Merge remote-tracking branch 'origin/main' into codex-api-server-user-guide
Signed-off-by: hsliu_ustc <hsliu_ustc@noreply.gitcode.com> # Conflicts: # docs/user_guide/examples/online_serving/text_to_audio.md # examples/online_serving/stable_audio/README.md
This commit is contained in:
hsliu_ustc committed
commit
7d41ea6183
1457 files changed
+164090
-40241
No files matched your search
@@ -1,64 +0,0 @@
|
||||
# AudioX online serving
|
||||
|
||||
Launches the `AudioXPipeline` behind vLLM-Omni's OpenAI-compatible chat endpoint and provides a
|
||||
minimal Python client that covers all six tasks (`t2a`, `t2m`, `v2a`, `v2m`, `tv2a`, `tv2m`).
|
||||
|
||||
## Start the server
|
||||
|
||||
```bash
|
||||
cd examples/online_serving/audiox
|
||||
bash run_server.sh # defaults: MODEL=zhangj1an/AudioX, PORT=8099
|
||||
```
|
||||
|
||||
Environment overrides: `MODEL`, `PORT`, `DIFFUSION_ATTENTION_BACKEND`.
|
||||
|
||||
## Call from Python
|
||||
|
||||
```bash
|
||||
# text-to-audio
|
||||
python openai_chat_client.py --task t2a \
|
||||
--prompt "Fireworks burst twice, followed by a period of silence before a clock begins ticking." \
|
||||
--output t2a.wav
|
||||
|
||||
# text-to-music
|
||||
python openai_chat_client.py --task t2m \
|
||||
--prompt "Uplifting ukulele tune for a travel vlog" \
|
||||
--output t2m.wav
|
||||
|
||||
# video-to-audio (no text)
|
||||
python openai_chat_client.py --task v2a --video path/to/clip.mp4 --output v2a.wav
|
||||
|
||||
# text+video-to-audio
|
||||
python openai_chat_client.py --task tv2a \
|
||||
--prompt "drum beating sound and human talking" \
|
||||
--video path/to/clip.mp4 \
|
||||
--output tv2a.wav
|
||||
```
|
||||
|
||||
The client sends:
|
||||
|
||||
- `num_inference_steps`, `guidance_scale`, `seed`, `audiox_task`, `seconds_start`,
|
||||
`seconds_total`, `sigma_min`, `sigma_max` as flattened keys under `extra_body`. The server
|
||||
routes the declared knobs (see `vllm_omni/model_extras/audiox.py`) into the pipeline's
|
||||
`sampling_params.extra_args`.
|
||||
- For `v2*` / `tv2*` tasks, the video as a `video_url` content item (data URI for local files)
|
||||
|
||||
## curl
|
||||
|
||||
```bash
|
||||
curl -sS -X POST http://localhost:8099/v1/chat/completions \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d '{
|
||||
"model": "zhangj1an/AudioX",
|
||||
"messages": [{"role": "user", "content": [{"type": "text", "text": "Uplifting ukulele"}]}],
|
||||
"extra_body": {
|
||||
"num_inference_steps": 250,
|
||||
"guidance_scale": 7.0,
|
||||
"seed": 42,
|
||||
"audiox_task": "t2m",
|
||||
"seconds_total": 10.0,
|
||||
"sigma_min": 0.3,
|
||||
"sigma_max": 500.0
|
||||
}
|
||||
}' > t2m.json
|
||||
```
|
||||
@@ -1,119 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""AudioX OpenAI-compatible chat client.
|
||||
|
||||
AudioX supports 6 tasks (t2a, t2m, v2a, v2m, tv2a, tv2m). Text-only tasks send the prompt as the
|
||||
chat message; video-conditioned tasks additionally attach the video via a ``video_url`` content
|
||||
item (data URI for local files). Task + generation knobs (steps, cfg, sigma range, seconds, seed)
|
||||
are sent as flattened keys under ``extra_body``; the server routes the declared knobs into
|
||||
``sampling_params.extra_args`` (see ``vllm_omni/model_extras/audiox.py``).
|
||||
|
||||
Usage:
|
||||
python openai_chat_client.py --task t2a --prompt "Fireworks burst twice..." --output t2a.wav
|
||||
python openai_chat_client.py --task tv2a --prompt "drum beating" --video clip.mp4 -o tv2a.wav
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import base64
|
||||
import io
|
||||
import mimetypes
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import requests
|
||||
import soundfile
|
||||
import torch
|
||||
|
||||
VIDEO_TASKS = frozenset({"v2a", "v2m", "tv2a", "tv2m"})
|
||||
TEXT_TASKS = frozenset({"t2a", "t2m", "tv2a", "tv2m"})
|
||||
|
||||
|
||||
def _to_data_url(path: str) -> str:
|
||||
mime, _ = mimetypes.guess_type(path)
|
||||
mime = mime or "video/mp4"
|
||||
with open(path, "rb") as f:
|
||||
data = base64.b64encode(f.read()).decode("ascii")
|
||||
return f"data:{mime};base64,{data}"
|
||||
|
||||
|
||||
def _save_wav(audio: torch.Tensor, path: Path, sample_rate: int) -> None:
|
||||
audio = audio.to(torch.float32)
|
||||
audio = audio / audio.abs().max().clamp(min=1e-8)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
# soundfile expects channels-last (T, C); project convention is (C, T).
|
||||
soundfile.write(str(path), audio.clamp(-1.0, 1.0).cpu().T.numpy(), sample_rate, subtype="PCM_16")
|
||||
|
||||
|
||||
def _decode_audio_from_response(body: dict) -> tuple[torch.Tensor, int]:
|
||||
for choice in body.get("choices", []):
|
||||
audio_obj = choice.get("message", {}).get("audio")
|
||||
if not (isinstance(audio_obj, dict) and audio_obj.get("data")):
|
||||
continue
|
||||
data, sr = soundfile.read(io.BytesIO(base64.b64decode(audio_obj["data"])), dtype="float32", always_2d=True)
|
||||
return torch.from_numpy(data).transpose(0, 1), sr
|
||||
brief = {k: v for k, v in body.items() if k != "choices"}
|
||||
raise RuntimeError(f"no audio in response message.audio: {brief}")
|
||||
|
||||
|
||||
def main() -> int:
|
||||
p = argparse.ArgumentParser(description="AudioX OpenAI chat client")
|
||||
p.add_argument("--task", required=True, choices=["t2a", "t2m", "v2a", "v2m", "tv2a", "tv2m"])
|
||||
p.add_argument("--prompt", "-p", default="", help="Text prompt (required for t2*/tv2*).")
|
||||
p.add_argument("--video", help="Video path or URL (required for v2*/tv2*).")
|
||||
p.add_argument("--output", "-o", default="audiox_out.wav")
|
||||
p.add_argument("--server", "-s", default="http://localhost:8099")
|
||||
p.add_argument("--model", default="zhangj1an/AudioX")
|
||||
p.add_argument("--steps", type=int, default=250)
|
||||
p.add_argument("--guidance-scale", type=float, default=7.0)
|
||||
p.add_argument("--seed", type=int, default=42)
|
||||
p.add_argument("--seconds-total", type=float, default=10.0)
|
||||
p.add_argument("--seconds-start", type=float, default=0.0)
|
||||
p.add_argument("--sigma-min", type=float, default=0.03)
|
||||
p.add_argument("--sigma-max", type=float, default=1000.0)
|
||||
args = p.parse_args()
|
||||
|
||||
if args.task in VIDEO_TASKS and not args.video:
|
||||
print(f"ERROR: task {args.task!r} requires --video", file=sys.stderr)
|
||||
return 2
|
||||
if args.task in TEXT_TASKS and not args.prompt.strip() and args.task not in {"v2a", "v2m"}:
|
||||
print(f"ERROR: task {args.task!r} requires --prompt", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
content: list[dict] = [{"type": "text", "text": args.prompt}]
|
||||
if args.task in VIDEO_TASKS:
|
||||
vurl = args.video if args.video.startswith(("http://", "https://")) else _to_data_url(args.video)
|
||||
content.append({"type": "video_url", "video_url": {"url": vurl}})
|
||||
|
||||
payload = {
|
||||
"model": args.model,
|
||||
"messages": [{"role": "user", "content": content}],
|
||||
"extra_body": {
|
||||
"num_inference_steps": args.steps,
|
||||
"guidance_scale": args.guidance_scale,
|
||||
"seed": args.seed,
|
||||
"audiox_task": args.task,
|
||||
"seconds_start": args.seconds_start,
|
||||
"seconds_total": args.seconds_total,
|
||||
"sigma_min": args.sigma_min,
|
||||
"sigma_max": args.sigma_max,
|
||||
},
|
||||
}
|
||||
|
||||
print(f"POST {args.server}/v1/chat/completions task={args.task} steps={args.steps}")
|
||||
r = requests.post(
|
||||
f"{args.server}/v1/chat/completions",
|
||||
headers={"Content-Type": "application/json"},
|
||||
json=payload,
|
||||
timeout=600,
|
||||
)
|
||||
r.raise_for_status()
|
||||
audio, sr = _decode_audio_from_response(r.json())
|
||||
_save_wav(audio, Path(args.output), sr)
|
||||
dur = audio.shape[-1] / sr
|
||||
print(f"saved {args.output} sr={sr}Hz duration={dur:.2f}s channels={audio.shape[0]}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -1,16 +0,0 @@
|
||||
#!/bin/bash
|
||||
# AudioX online serving startup script.
|
||||
|
||||
MODEL="${MODEL:-zhangj1an/AudioX}"
|
||||
PORT="${PORT:-8099}"
|
||||
DIFFUSION_ATTENTION_BACKEND="${DIFFUSION_ATTENTION_BACKEND:-FLASH_ATTN}"
|
||||
|
||||
echo "Starting AudioX server..."
|
||||
echo "Model: $MODEL"
|
||||
echo "Port: $PORT"
|
||||
echo "Diffusion attention backend: $DIFFUSION_ATTENTION_BACKEND"
|
||||
|
||||
DIFFUSION_ATTENTION_BACKEND="$DIFFUSION_ATTENTION_BACKEND" \
|
||||
vllm serve "$MODEL" --omni \
|
||||
--model-class-name AudioXPipeline \
|
||||
--port "$PORT"
|
||||
@@ -1,5 +1,19 @@
|
||||
# Image-To-Video
|
||||
|
||||
This example demonstrates how to deploy image-to-video models for online
|
||||
video generation using vLLM-Omni.
|
||||
|
||||
## Supported Models
|
||||
|
||||
| Model | Model ID |
|
||||
|-------|----------|
|
||||
| Wan2.2 I2V | `Wan-AI/Wan2.2-I2V-A14B-Diffusers` |
|
||||
| SANA-Video 2B (480p) | `Efficient-Large-Model/SANA-Video_2B_480p_diffusers` |
|
||||
| SANA-Video 2B (720p) | `Efficient-Large-Model/SANA-Video_2B_720p_diffusers` |
|
||||
|
||||
For SANA-Video 2B image-to-video online serving, see the
|
||||
[SANA-Video 2B recipe](../../../recipes/NVIDIA/SANA-Video-2B.md).
|
||||
|
||||
This example demonstrates online image-to-video generation with vLLM-Omni.
|
||||
The startup script defaults to Wan2.2, but `MODEL` can select any supported
|
||||
image-to-video model. The existing curl helper remains a concrete Wan2.2
|
||||
|
||||
@@ -0,0 +1,67 @@
|
||||
#!/bin/bash
|
||||
# Submit and download a standard SANA-Video-2B image-to-video request.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
INPUT_IMAGE="${INPUT_IMAGE:-}"
|
||||
BASE_URL="${BASE_URL:-http://localhost:8099}"
|
||||
OUTPUT_PATH="${OUTPUT_PATH:-sana_video_i2v.mp4}"
|
||||
WIDTH="${WIDTH:-832}"
|
||||
HEIGHT="${HEIGHT:-480}"
|
||||
POLL_INTERVAL="${POLL_INTERVAL:-2}"
|
||||
|
||||
if [ -z "$INPUT_IMAGE" ] || [ ! -f "$INPUT_IMAGE" ]; then
|
||||
echo "Set INPUT_IMAGE to an existing image file."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
create_response="$(
|
||||
curl -sS -X POST "${BASE_URL}/v1/videos" \
|
||||
-H "Accept: application/json" \
|
||||
-F "prompt=A cat turns toward the camera with smooth, natural motion." \
|
||||
-F "negative_prompt=blurry, low quality, temporal artifacts" \
|
||||
-F "input_reference=@${INPUT_IMAGE}" \
|
||||
-F "width=${WIDTH}" \
|
||||
-F "height=${HEIGHT}" \
|
||||
-F "num_frames=81" \
|
||||
-F "fps=16" \
|
||||
-F "num_inference_steps=50" \
|
||||
-F "guidance_scale=6.0" \
|
||||
-F "seed=42" \
|
||||
-F 'extra_params={"motion_score":30}'
|
||||
)"
|
||||
|
||||
video_id="$(echo "${create_response}" | jq -r '.id')"
|
||||
if [ -z "${video_id}" ] || [ "${video_id}" = "null" ]; then
|
||||
echo "Failed to create SANA-Video job:"
|
||||
echo "${create_response}" | jq .
|
||||
exit 1
|
||||
fi
|
||||
|
||||
while true; do
|
||||
status_response="$(curl -sS "${BASE_URL}/v1/videos/${video_id}")"
|
||||
status="$(echo "${status_response}" | jq -r '.status')"
|
||||
|
||||
case "${status}" in
|
||||
queued|in_progress)
|
||||
echo "Video job ${video_id} status: ${status}"
|
||||
sleep "${POLL_INTERVAL}"
|
||||
;;
|
||||
completed)
|
||||
break
|
||||
;;
|
||||
failed)
|
||||
echo "SANA-Video generation failed:"
|
||||
echo "${status_response}" | jq .
|
||||
exit 1
|
||||
;;
|
||||
*)
|
||||
echo "Unexpected status response:"
|
||||
echo "${status_response}" | jq .
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
curl -sS -L "${BASE_URL}/v1/videos/${video_id}/content" -o "${OUTPUT_PATH}"
|
||||
echo "Saved SANA-Video to ${OUTPUT_PATH}"
|
||||
@@ -0,0 +1,13 @@
|
||||
#!/bin/bash
|
||||
# SANA-Video-2B native image-to-video serving.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
MODEL="${MODEL:-Efficient-Large-Model/SANA-Video_2B_480p_diffusers}"
|
||||
PORT="${PORT:-8099}"
|
||||
|
||||
vllm serve "$MODEL" \
|
||||
--omni \
|
||||
--model-class-name SanaImageToVideoPipeline \
|
||||
--dtype bfloat16 \
|
||||
--port "$PORT"
|
||||
@@ -0,0 +1,18 @@
|
||||
#!/bin/bash
|
||||
# SANA-Video-2B image-to-video serving through the Diffusers adapter.
|
||||
# Both 480p and 720p checkpoints are validated. For 720p:
|
||||
# MODEL=Efficient-Large-Model/SANA-Video_2B_720p_diffusers bash run_server_sana_video_diffusers.sh
|
||||
# INPUT_IMAGE=input.jpg WIDTH=1280 HEIGHT=704 bash run_curl_sana_video.sh
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
MODEL="${MODEL:-Efficient-Large-Model/SANA-Video_2B_480p_diffusers}"
|
||||
PORT="${PORT:-8099}"
|
||||
|
||||
vllm serve "$MODEL" \
|
||||
--omni \
|
||||
--model-class-name SanaImageToVideoPipeline \
|
||||
--diffusion-load-format diffusers \
|
||||
--diffusion-attention-backend TORCH_SDPA \
|
||||
--dtype bfloat16 \
|
||||
--port "$PORT"
|
||||
+1
-1
@@ -328,7 +328,7 @@ def run_multimodal_generation(args) -> None:
|
||||
|
||||
extra_body = {
|
||||
"sampling_params_list": sampling_params_list
|
||||
# Optional, it has a default setting in stage_configs of the corresponding model.
|
||||
# Optional; defaults come from the resolved pipeline and deploy config.
|
||||
}
|
||||
|
||||
# Build messages list (filter None so concurrent tasks get valid structure)
|
||||
|
||||
@@ -32,12 +32,14 @@ GPU 0 (90%) and colocates the Talker (55%) and Code2Wav (35%) on GPU 1. That
|
||||
profile admits at most four concurrent sequences per stage.
|
||||
|
||||
| deploy config | GPUs | Notes |
|
||||
|---|---|---|
|
||||
| --- | --- | --- |
|
||||
| `minicpmo_4_5.yaml` (default) | 1 | Memory-constrained compatibility layout. |
|
||||
| `minicpmo_4_5_2gpu.yaml` | 2 | Recommended continuous-batching layout; Talker and Code2Wav share GPU 1. |
|
||||
| `minicpmo_4_5_3gpu.yaml` | 3 | One GPU per stage. |
|
||||
| `minicpmo_4_5_8x4090.yaml` | 8 | Full 8x4090 layout. |
|
||||
| `minicpmo_4_5_duplex.yaml` | 1 | Experimental native full-duplex overlay. |
|
||||
|
||||
Every profile sets `session_mode: duplex`, so native full-duplex is served
|
||||
from the same process as `/v1/chat/completions`.
|
||||
|
||||
The split pipeline preserves native-duplex epoch/turn identity, segment text,
|
||||
turn completion, reference voice, and terminal-audio metadata through
|
||||
@@ -55,8 +57,8 @@ vllm serve openbmb/MiniCPM-o-4_5 \
|
||||
```
|
||||
|
||||
For local ModelScope checkpoints, replace `openbmb/MiniCPM-o-4_5` with the
|
||||
checkpoint path. To start the experimental native duplex backend, use
|
||||
`vllm_omni/deploy/minicpmo_4_5_duplex.yaml`.
|
||||
checkpoint path. For native full-duplex, connect `/v1/realtime?duplex=1` on
|
||||
this same server.
|
||||
|
||||
### Per-stage overrides
|
||||
|
||||
@@ -145,7 +147,7 @@ assistant template, so it is not an apples-to-apples accuracy run.
|
||||
|
||||
## Run the Realtime duplex CLI demo
|
||||
|
||||
After the duplex backend is running, stream one WAV through the Realtime
|
||||
After the server is running, stream one WAV through the Realtime
|
||||
WebSocket endpoint:
|
||||
|
||||
```bash
|
||||
@@ -157,6 +159,33 @@ python examples/online_serving/minicpmo/realtime_duplex_demo.py \
|
||||
--output-dir /tmp/minicpmo_realtime_duplex_demo
|
||||
```
|
||||
|
||||
Video input uses the same session. PyAV demuxes JPEG frames at `--video-fps`
|
||||
(default 1.0) and vLLM `load_audio` extracts a 16 kHz mono WAV unless
|
||||
`--input-wav` overrides the soundtrack. Frames keep their capture size
|
||||
(`--frame-max-side 0`); the server `process_image` normalizes at 448.
|
||||
|
||||
`--stack-frames N` raises the visual refresh rate the way official duplex does:
|
||||
each 1 s unit also samples the `N-1` sub-frames captured inside it and tiles
|
||||
them into a single composite image sent next to that unit's base frame. The
|
||||
audio cadence never changes — a unit is always one second — and the wire always
|
||||
carries 2 images per unit however large `N` is, because the sub-frames share one
|
||||
composite. Official uses 5 for high refresh rate mode:
|
||||
|
||||
```bash
|
||||
python examples/online_serving/minicpmo/realtime_duplex_demo.py \
|
||||
--url ws://localhost:8099/v1/realtime?duplex=1 \
|
||||
--model openbmb/MiniCPM-o-4_5 \
|
||||
--input-video /path/to/clip.mp4 \
|
||||
--stack-frames 5 \
|
||||
--ref-audio /path/to/MiniCPM-o-Demo/assets/ref_audio/ref_minicpm_signature.wav \
|
||||
--output-dir /tmp/minicpmo_realtime_duplex_video_demo
|
||||
```
|
||||
|
||||
Detail inside a composite is capped by `scale_resolution=448` at
|
||||
`max_slice_nums=1`: official suggests HD slicing (`max_slice_nums=[2, 1]`) for
|
||||
stacked frames, which the duplex adapter does not implement yet. Reading small
|
||||
text or digits out of a wide scene is limited by that, not by frame timing.
|
||||
|
||||
## Open the experimental browser client
|
||||
|
||||
The browser UI serves the page and proxies the same-origin Realtime WebSocket to
|
||||
@@ -190,6 +219,64 @@ stronger `response-required` mode is diagnostic: it requires a purpose-built
|
||||
two-response WAV, its `--input-sha256`, and an
|
||||
`--expect-second-response-substring` value.
|
||||
|
||||
## Run Omni-DuplexEval
|
||||
|
||||
Omni-DuplexEval generation uses the vLLM-Omni native MiniCPM-o duplex
|
||||
endpoint. Evaluation uses a separate OpenAI-compatible multimodal judge
|
||||
served from the same vLLM-Omni environment. This validates the vLLM-Omni
|
||||
duplex client and local judge integration; it does not reproduce the paper's
|
||||
original MiniCPM-o inference implementation.
|
||||
|
||||
Start the duplex generation server:
|
||||
|
||||
```bash
|
||||
vllm serve openbmb/MiniCPM-o-4_5 --omni --trust-remote-code \
|
||||
--deploy-config vllm_omni/deploy/minicpmo_4_5_duplex.yaml \
|
||||
--served-model-name openbmb/MiniCPM-o-4_5 \
|
||||
--host 0.0.0.0 --port 8099
|
||||
```
|
||||
|
||||
Start a separate multimodal judge. The allowed path must contain any local
|
||||
videos passed with `--judge-video-mode video_url`:
|
||||
|
||||
```bash
|
||||
vllm serve Qwen/Qwen2.5-VL-7B-Instruct \
|
||||
--served-model-name Qwen/Qwen2.5-VL-7B-Instruct \
|
||||
--allowed-local-media-path /data/omni-duplex-eval \
|
||||
--host 0.0.0.0 --port 8000
|
||||
```
|
||||
|
||||
Generate, evaluate, and summarize:
|
||||
|
||||
```bash
|
||||
vllm bench omni-duplex-eval --omni generate \
|
||||
--url ws://127.0.0.1:8099/v1/realtime?duplex=1 \
|
||||
--model openbmb/MiniCPM-o-4_5 \
|
||||
--ref-audio /data/ref.wav \
|
||||
--dataset Hothan/Omni-DuplexEval \
|
||||
--concurrency 2 \
|
||||
--response-root /data/omni-duplex-eval/responses
|
||||
|
||||
vllm bench omni-duplex-eval --omni evaluate \
|
||||
--dataset Hothan/Omni-DuplexEval \
|
||||
--response-root /data/omni-duplex-eval/responses \
|
||||
--score-root /data/omni-duplex-eval/scores \
|
||||
--judge-base-url http://127.0.0.1:8000 \
|
||||
--judge-model Qwen/Qwen2.5-VL-7B-Instruct \
|
||||
--judge-video-mode video_url \
|
||||
--eval-workers 4
|
||||
|
||||
vllm bench omni-duplex-eval --omni summarize \
|
||||
--score-root /data/omni-duplex-eval/scores
|
||||
```
|
||||
|
||||
The dataset names such as `RTD_OCR` and `PR_correction` are Hugging Face
|
||||
splits, not dataset configurations. Pass one with `--split`, or omit it to
|
||||
run all nine splits. `--limit 1` is useful for a smoke test. Realtime
|
||||
generation records `clock=media`; artifacts generated with
|
||||
`--pace as-fast-as-possible` record `clock=invalid` and evaluation rejects
|
||||
them unless `--allow-invalid-clock` is explicit.
|
||||
|
||||
## Related examples
|
||||
|
||||
- [Offline MiniCPM-o inference](../../offline_inference/minicpmo/)
|
||||
|
||||
@@ -1,3 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
"""Minimal single-input MiniCPM-o 4.5 Realtime duplex demo.
|
||||
|
||||
Run this after starting the duplex server. Strict lifecycle, overlap, and
|
||||
@@ -9,7 +12,9 @@ from __future__ import annotations
|
||||
import argparse
|
||||
import asyncio
|
||||
import base64
|
||||
import io
|
||||
import json
|
||||
import math
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
@@ -28,6 +33,17 @@ from vllm_omni.experimental.fullduplex.client import ( # noqa: E402
|
||||
wait_for,
|
||||
write_pcm16_wav,
|
||||
)
|
||||
from vllm_omni.experimental.fullduplex.client import chunk_period_ms as _chunk_period_ms # noqa: E402
|
||||
from vllm_omni.experimental.fullduplex.client import ( # noqa: E402
|
||||
has_residual_model_unit as _has_residual_model_unit,
|
||||
)
|
||||
from vllm_omni.experimental.fullduplex.client import ( # noqa: E402
|
||||
reference_audio_data_url as _ref_audio_data_url,
|
||||
)
|
||||
from vllm_omni.experimental.fullduplex.video_stacking import ( # noqa: E402
|
||||
concat_frames_b64,
|
||||
unit_subframe_offsets,
|
||||
)
|
||||
|
||||
|
||||
class _StreamingOutputWriter:
|
||||
@@ -133,31 +149,29 @@ def _latest_model_decision(
|
||||
return decision
|
||||
|
||||
|
||||
def _chunk_period_ms(events: list[dict[str, object]]) -> int:
|
||||
for event in reversed(events):
|
||||
session = event.get("session")
|
||||
if not isinstance(session, dict):
|
||||
continue
|
||||
capabilities = session.get("capabilities")
|
||||
if not isinstance(capabilities, dict):
|
||||
continue
|
||||
chunk_period_ms = capabilities.get("chunk_period_ms")
|
||||
if isinstance(chunk_period_ms, int) and chunk_period_ms > 0:
|
||||
return chunk_period_ms
|
||||
return 1000
|
||||
|
||||
|
||||
def _has_residual_model_unit(pcm16: bytes, *, chunk_period_ms: int) -> bool:
|
||||
unit_bytes = PCM16_SAMPLE_RATE * PCM16_BYTES_PER_SAMPLE * chunk_period_ms // 1000
|
||||
return bool(unit_bytes > 0 and len(pcm16) % unit_bytes)
|
||||
|
||||
|
||||
def _response_in_progress(events: list[dict[str, object]]) -> bool:
|
||||
return sum(event.get("type") == "response.created" for event in events) > sum(
|
||||
event.get("type") == "response.done" for event in events
|
||||
)
|
||||
|
||||
|
||||
async def _commit_input_after_playback_checkpoint(client: RealtimeDuplexClient) -> float:
|
||||
"""Commit input only after preserving already-played assistant history.
|
||||
|
||||
Native duplex can finish several responses while the user is still
|
||||
streaming. Under the ``ack_only`` playback policy, acknowledging those
|
||||
responses after the final input commit is intentionally rejected: it could
|
||||
append an old assistant message after the newly committed user message.
|
||||
Checkpoint playback first so the server commits each assistant item at its
|
||||
original history position. A second acknowledgement after the commit can
|
||||
then safely advance response-local playback that drained in the meantime.
|
||||
"""
|
||||
await client.acknowledge_playback()
|
||||
commit_sent_at_s = time.monotonic()
|
||||
await client.commit()
|
||||
return commit_sent_at_s
|
||||
|
||||
|
||||
def _event_count_after(
|
||||
events: list[dict[str, object]],
|
||||
event_type: str,
|
||||
@@ -168,21 +182,258 @@ def _event_count_after(
|
||||
return sum(event.get("type") == event_type for event in events[index + 1 :])
|
||||
|
||||
|
||||
def _ref_audio_data_url(path: str | None) -> str | None:
|
||||
if path is None:
|
||||
return None
|
||||
ref_path = Path(path).expanduser()
|
||||
return "data:audio/wav;base64," + base64.b64encode(ref_path.read_bytes()).decode("ascii")
|
||||
def _probe_video_frames_and_fps(video_path: Path) -> tuple[int, float]:
|
||||
"""Frame count and average FPS via PyAV (same approach as Daily-Omni)."""
|
||||
import av
|
||||
|
||||
with av.open(str(video_path)) as container:
|
||||
stream = container.streams.video[0]
|
||||
avg_fps = float(stream.average_rate) if stream.average_rate else 0.0
|
||||
num_frames = stream.frames or 0
|
||||
if num_frames <= 0:
|
||||
duration: float | None = None
|
||||
if stream.duration is not None and stream.time_base is not None:
|
||||
duration = float(stream.duration * stream.time_base)
|
||||
elif container.duration is not None:
|
||||
duration = container.duration / av.time_base
|
||||
if duration and avg_fps:
|
||||
num_frames = int(math.floor(duration * avg_fps))
|
||||
if num_frames <= 0:
|
||||
stream.thread_type = "AUTO"
|
||||
num_frames = sum(1 for _ in container.decode(stream))
|
||||
if num_frames <= 0:
|
||||
raise ValueError(f"No decodable video frames in {video_path}")
|
||||
return num_frames, avg_fps or 1.0
|
||||
|
||||
|
||||
def _upright(image, rotation: int):
|
||||
"""Apply a container display-matrix rotation, in degrees, to ``image``.
|
||||
|
||||
PyAV hands back the stored raster and leaves the display matrix for the
|
||||
caller, unlike the ffmpeg CLI which autorotates. Phone clips routinely
|
||||
carry +/-90, so skipping this feeds the model a sideways frame: arrows and
|
||||
digits stop being readable and the model mirrors the motion it reports.
|
||||
"""
|
||||
from PIL import Image
|
||||
|
||||
quarter_turns = {
|
||||
90: Image.Transpose.ROTATE_90,
|
||||
180: Image.Transpose.ROTATE_180,
|
||||
270: Image.Transpose.ROTATE_270,
|
||||
}
|
||||
turn = quarter_turns.get(int(rotation) % 360)
|
||||
return image.transpose(turn) if turn is not None else image
|
||||
|
||||
|
||||
def _decode_video_frames_rgb(video_path: Path, frame_idx: list[int]) -> list:
|
||||
"""Decode presentation-order ``frame_idx`` into upright RGB ``PIL.Image``s."""
|
||||
import av
|
||||
|
||||
wanted = sorted({int(i) for i in frame_idx})
|
||||
if not wanted:
|
||||
return []
|
||||
|
||||
decoded: dict[int, object] = {}
|
||||
with av.open(str(video_path)) as container:
|
||||
stream = container.streams.video[0]
|
||||
stream.thread_type = "AUTO"
|
||||
cursor = 0
|
||||
for position, frame in enumerate(container.decode(stream)):
|
||||
if wanted[cursor] > position:
|
||||
continue
|
||||
image = _upright(frame.to_image(), getattr(frame, "rotation", 0) or 0)
|
||||
while cursor < len(wanted) and wanted[cursor] <= position:
|
||||
decoded[wanted[cursor]] = image
|
||||
cursor += 1
|
||||
if cursor >= len(wanted):
|
||||
break
|
||||
|
||||
if not decoded:
|
||||
raise ValueError(f"Decoded no frames from {video_path} for indices {wanted[:8]}")
|
||||
last_image = decoded[max(decoded)]
|
||||
return [decoded.get(int(i), last_image) for i in frame_idx]
|
||||
|
||||
|
||||
def _resize_max_side(image, max_side: int):
|
||||
width, height = image.size
|
||||
longest = max(width, height)
|
||||
if longest <= max_side or max_side <= 0:
|
||||
return image
|
||||
scale = max_side / float(longest)
|
||||
new_size = (max(1, int(round(width * scale))), max(1, int(round(height * scale))))
|
||||
return image.resize(new_size)
|
||||
|
||||
|
||||
def _jpeg_bytes(image, *, quality: int = 95) -> bytes:
|
||||
buf = io.BytesIO()
|
||||
image.convert("RGB").save(buf, format="JPEG", quality=quality)
|
||||
return buf.getvalue()
|
||||
|
||||
|
||||
def _to_pcm16_bytes(audio_np) -> bytes:
|
||||
import numpy as np
|
||||
|
||||
arr = np.asarray(audio_np).reshape(-1)
|
||||
if arr.dtype != np.int16:
|
||||
arr = (np.clip(arr, -1.0, 1.0) * 32767.0).astype(np.int16)
|
||||
return arr.tobytes()
|
||||
|
||||
|
||||
def _pcm16_wav_duration_s(path: Path) -> float:
|
||||
pcm16 = read_pcm16_wav(path)
|
||||
return len(pcm16) / float(PCM16_SAMPLE_RATE * PCM16_BYTES_PER_SAMPLE)
|
||||
|
||||
|
||||
def _extract_video_frames(
|
||||
video_path: Path,
|
||||
work_dir: Path,
|
||||
*,
|
||||
fps: float,
|
||||
max_side: int,
|
||||
duration_s: float,
|
||||
stack_frames: int = 1,
|
||||
) -> tuple[list[str], list[str | None]]:
|
||||
"""Demux JPEG frames only. Frame count follows ``duration_s``, not soundtrack.
|
||||
|
||||
Returns the base frames (one per model unit) and the parallel stacked track.
|
||||
With ``stack_frames=N`` each unit also samples its ``N-1`` interior
|
||||
sub-frames and tiles them into one composite, which is how official duplex
|
||||
raises the visual refresh rate while keeping one second of audio per unit.
|
||||
A unit whose interior falls past the end of the clip stacks nothing.
|
||||
"""
|
||||
num_frames, avg_fps = _probe_video_frames_and_fps(video_path)
|
||||
sample_fps = max(float(fps), 1e-6)
|
||||
unit_s = 1.0 / sample_fps
|
||||
timeline_s = max(float(duration_s), unit_s)
|
||||
num_samples = max(1, int(math.ceil(timeline_s * sample_fps - 1e-9)))
|
||||
offsets = unit_subframe_offsets(stack_frames, unit_s=unit_s)
|
||||
|
||||
def frame_at(seconds: float) -> int:
|
||||
# Official get_video_frame_audio_segments uses truncating int(), not
|
||||
# round(), so 29.97 fps lands on the same presentation index.
|
||||
return min(int(seconds * avg_fps), num_frames - 1)
|
||||
|
||||
base_idx = [frame_at(unit * unit_s) for unit in range(num_samples)]
|
||||
sub_times = [
|
||||
[unit * unit_s + offset for offset in offsets if unit * unit_s + offset < timeline_s]
|
||||
for unit in range(num_samples)
|
||||
]
|
||||
sub_idx = [frame_at(seconds) for times in sub_times for seconds in times]
|
||||
images = _decode_video_frames_rgb(video_path, base_idx + sub_idx)
|
||||
sub_images = iter(images[len(base_idx) :])
|
||||
|
||||
frame_dir = work_dir / "frames"
|
||||
frame_dir.mkdir(parents=True, exist_ok=True)
|
||||
for stale in frame_dir.glob("*.jpg"):
|
||||
stale.unlink()
|
||||
frames_b64: list[str] = []
|
||||
stacked_b64: list[str | None] = []
|
||||
for index, (image, times) in enumerate(zip(images[: len(base_idx)], sub_times), start=1):
|
||||
jpeg = _jpeg_bytes(_resize_max_side(image, max_side))
|
||||
(frame_dir / f"frame_{index:05d}.jpg").write_bytes(jpeg)
|
||||
frames_b64.append(base64.b64encode(jpeg).decode("ascii"))
|
||||
interior = [_resize_max_side(next(sub_images), max_side) for _ in times]
|
||||
if not interior:
|
||||
stacked_b64.append(None)
|
||||
continue
|
||||
composite = concat_frames_b64(interior)
|
||||
(frame_dir / f"stack_{index:05d}.jpg").write_bytes(base64.b64decode(composite))
|
||||
stacked_b64.append(composite)
|
||||
return frames_b64, stacked_b64
|
||||
|
||||
|
||||
def _extract_video_soundtrack(video_path: Path, work_dir: Path) -> Path:
|
||||
"""Write a 16 kHz mono WAV from the video soundtrack."""
|
||||
try:
|
||||
from vllm.multimodal.media.audio import load_audio
|
||||
except ImportError as exc: # pragma: no cover
|
||||
raise SystemExit("--input-video needs vLLM audio helpers (load_audio)") from exc
|
||||
|
||||
audio_np, sr = load_audio(str(video_path), sr=PCM16_SAMPLE_RATE, mono=True)
|
||||
work_dir.mkdir(parents=True, exist_ok=True)
|
||||
audio_path = work_dir / "input_16k.wav"
|
||||
write_pcm16_wav(audio_path, _to_pcm16_bytes(audio_np), sample_rate_hz=int(sr))
|
||||
return audio_path
|
||||
|
||||
|
||||
def _extract_video_input(
|
||||
video_path: Path,
|
||||
work_dir: Path,
|
||||
*,
|
||||
fps: float,
|
||||
max_side: int,
|
||||
stack_frames: int,
|
||||
) -> tuple[Path, list[str], list[str | None]]:
|
||||
"""Demux a video into duplex inputs: 16 kHz mono WAV plus JPEG frames.
|
||||
|
||||
Use this only when the video soundtrack is the audio source. When the
|
||||
caller already has ``--input-wav``, extract frames against that WAV
|
||||
duration instead so a silent clip cannot fail ``load_audio``.
|
||||
"""
|
||||
audio_path = _extract_video_soundtrack(video_path, work_dir)
|
||||
frames_b64, stacked_b64 = _extract_video_frames(
|
||||
video_path,
|
||||
work_dir,
|
||||
fps=fps,
|
||||
max_side=max_side,
|
||||
duration_s=_pcm16_wav_duration_s(audio_path),
|
||||
stack_frames=stack_frames,
|
||||
)
|
||||
return audio_path, frames_b64, stacked_b64
|
||||
|
||||
|
||||
def _resolve_duplex_av_inputs(
|
||||
*,
|
||||
input_wav: str | None,
|
||||
input_video: str | None,
|
||||
work_dir: Path,
|
||||
fps: float,
|
||||
max_side: int,
|
||||
stack_frames: int = 1,
|
||||
) -> tuple[str, list[str], list[str | None]]:
|
||||
"""Return ``(wav_path, frames_b64, stacked_b64)``. External WAV wins over video audio."""
|
||||
if input_video:
|
||||
video_path = Path(input_video)
|
||||
if input_wav:
|
||||
frames, stacked = _extract_video_frames(
|
||||
video_path,
|
||||
work_dir,
|
||||
fps=fps,
|
||||
max_side=max_side,
|
||||
duration_s=_pcm16_wav_duration_s(Path(input_wav)),
|
||||
stack_frames=stack_frames,
|
||||
)
|
||||
return input_wav, frames, stacked
|
||||
extracted_wav, frames, stacked = _extract_video_input(
|
||||
video_path,
|
||||
work_dir,
|
||||
fps=fps,
|
||||
max_side=max_side,
|
||||
stack_frames=stack_frames,
|
||||
)
|
||||
return str(extracted_wav), frames, stacked
|
||||
if not input_wav:
|
||||
raise SystemExit("provide --input-wav, --input-video, or both")
|
||||
return input_wav, [], []
|
||||
|
||||
|
||||
async def run_demo(args: argparse.Namespace) -> dict[str, object]:
|
||||
input_pcm16 = read_pcm16_wav(Path(args.input_wav))
|
||||
if not input_pcm16:
|
||||
raise ValueError("input WAV has no audio")
|
||||
|
||||
output_dir = Path(args.output_dir)
|
||||
stream_writer = _StreamingOutputWriter(output_dir)
|
||||
|
||||
input_wav, video_frames, stacked_video_frames = _resolve_duplex_av_inputs(
|
||||
input_wav=args.input_wav,
|
||||
input_video=args.input_video,
|
||||
work_dir=output_dir / "video_input",
|
||||
fps=args.video_fps,
|
||||
max_side=args.frame_max_side,
|
||||
stack_frames=args.stack_frames,
|
||||
)
|
||||
|
||||
input_pcm16 = read_pcm16_wav(Path(input_wav))
|
||||
if not input_pcm16:
|
||||
raise ValueError("input WAV has no audio")
|
||||
|
||||
url = build_realtime_url(
|
||||
args.url,
|
||||
args.model,
|
||||
@@ -200,10 +451,12 @@ async def run_demo(args: argparse.Namespace) -> dict[str, object]:
|
||||
timeout_s=args.timeout_s,
|
||||
)
|
||||
stream_event_cursor = len(client.events.events)
|
||||
await client.stream_pcm16(
|
||||
frames_sent = await client.stream_pcm16(
|
||||
input_pcm16,
|
||||
chunk_ms=args.chunk_ms,
|
||||
realtime=not args.no_realtime_pacing,
|
||||
video_frames=video_frames,
|
||||
stacked_video_frames=stacked_video_frames,
|
||||
)
|
||||
commit_event_cursor = len(client.events.events)
|
||||
stream_decision = _latest_model_decision(client.events.events, stream_event_cursor)
|
||||
@@ -212,8 +465,7 @@ async def run_demo(args: argparse.Namespace) -> dict[str, object]:
|
||||
chunk_period_ms=_chunk_period_ms(client.events.events),
|
||||
)
|
||||
wait_for_post_commit_decision = False
|
||||
commit_sent_at_s = time.monotonic()
|
||||
await client.commit()
|
||||
commit_sent_at_s = await _commit_input_after_playback_checkpoint(client)
|
||||
wait_error: str | None = None
|
||||
committed_index: int | None = None
|
||||
post_commit_decision: str | None = None
|
||||
@@ -326,6 +578,10 @@ async def run_demo(args: argparse.Namespace) -> dict[str, object]:
|
||||
committed_index,
|
||||
),
|
||||
},
|
||||
"input_video": args.input_video,
|
||||
"video_frames_extracted": len(video_frames),
|
||||
"video_frames_sent": frames_sent,
|
||||
"video_stacked_frames_extracted": sum(frame is not None for frame in stacked_video_frames),
|
||||
"audio_bytes": len(audio),
|
||||
"audio_chunk_count": len(stream_writer.audio_chunk_paths),
|
||||
"audio_chunk_files": [str(path) for path in stream_writer.audio_chunk_paths],
|
||||
@@ -372,7 +628,48 @@ def parse_args() -> argparse.Namespace:
|
||||
parser.add_argument("--url", default="ws://localhost:8099/v1/realtime?duplex=1")
|
||||
parser.add_argument("--model", default="openbmb/MiniCPM-o-4_5")
|
||||
parser.add_argument("--session-id")
|
||||
parser.add_argument("--input-wav", required=True)
|
||||
parser.add_argument(
|
||||
"--input-wav",
|
||||
help="Mono 16 kHz PCM16 clip to stream. Optional when --input-video carries the audio track.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--input-video",
|
||||
help=(
|
||||
"Video to stream as omni duplex input. PyAV demuxes frames and "
|
||||
"vLLM load_audio extracts a 16 kHz mono WAV (unless --input-wav overrides the audio)."
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--video-fps",
|
||||
type=float,
|
||||
default=1.0,
|
||||
help="Frame extraction rate. The wire contract carries ~1 frame per second of audio.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--frame-max-side",
|
||||
type=int,
|
||||
default=0,
|
||||
help=(
|
||||
"Optional client-side downscale of the longest side, in pixels. "
|
||||
"0 (default) keeps the capture resolution so the server "
|
||||
"process_image can normalize at scale_resolution=448 from a "
|
||||
"higher-quality source. The web demo never resizes on the client."
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--stack-frames",
|
||||
type=int,
|
||||
default=1,
|
||||
help=(
|
||||
"Frames sampled per 1 s audio unit. 1 (default) sends the unit's "
|
||||
"frame alone. N > 1 also samples the N-1 sub-frames inside the unit "
|
||||
"and tiles them into one composite sent next to the base frame, so "
|
||||
"motion (arrows, doors, digits changing) is visible at N fps while "
|
||||
"the audio stays 1 s per unit. Official uses 5 for high refresh "
|
||||
"rate mode; the wire carries 2 images per unit at any N because the "
|
||||
"sub-frames share one composite."
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--ref-audio",
|
||||
required=True,
|
||||
@@ -388,7 +685,7 @@ def parse_args() -> argparse.Namespace:
|
||||
"--temperature",
|
||||
type=float,
|
||||
default=None,
|
||||
help="Stage0 sampling temperature; response-required soft-interrupt uses 0.0.",
|
||||
help="Stage 0 sampling temperature; omit to preserve the model default.",
|
||||
)
|
||||
parser.add_argument("--no-realtime-pacing", action="store_true")
|
||||
parser.add_argument("--require-audio", action="store_true")
|
||||
|
||||
@@ -415,7 +415,7 @@ def run_multimodal_generation(args, client: OpenAI) -> None:
|
||||
audio_path=audio_path,
|
||||
)
|
||||
extra_body = {
|
||||
# Optional, it has default settings in stage configs. you can override them here.
|
||||
# Optional; defaults come from the resolved pipeline and deploy config.
|
||||
}
|
||||
if args.query_type == "use_audio_in_video":
|
||||
extra_body["mm_processor_kwargs"] = {"use_audio_in_video": True}
|
||||
|
||||
@@ -38,7 +38,7 @@ full-duplex control plane:
|
||||
CUDA_VISIBLE_DEVICES=0 python -m vllm_omni.entrypoints.cli.main serve \
|
||||
/path/to/personaplex-7b-v1 \
|
||||
--omni \
|
||||
--stage-configs-path vllm_omni/deploy/personaplex.yaml
|
||||
--deploy-config vllm_omni/deploy/personaplex.yaml
|
||||
```
|
||||
|
||||
Validate the actual `/v1/realtime?duplex=1` scheduler path with paced 24 kHz
|
||||
|
||||
@@ -0,0 +1,104 @@
|
||||
# π0 (Pi-Zero) VLA — OpenPI realtime serving
|
||||
|
||||
[π0](https://www.physicalintelligence.company/blog/pi0) is a Vision-Language-Action
|
||||
model from Physical Intelligence: multi-camera images + a language instruction +
|
||||
robot proprioceptive state → a continuous action chunk via flow-matching denoising
|
||||
(it does **not** emit text tokens). This example serves π0 over the OpenPI realtime
|
||||
websocket protocol at `/v1/realtime/robot/openpi`.
|
||||
|
||||
## Install extras
|
||||
|
||||
The core `pip install -e .` does not include the OpenPI client used here:
|
||||
|
||||
- `openpi-client` (from the [openpi](https://github.com/Physical-Intelligence/openpi)
|
||||
repo: `pip install -e packages/openpi-client`), `websockets`, `msgpack`, `msgpack-numpy`
|
||||
|
||||
## Weights
|
||||
|
||||
π0 uses the LeRobot `lerobot/pi0_base` checkpoint (HF, ~13 GB). Either let the server
|
||||
download it (`MODEL=lerobot/pi0_base`) or point at a local copy
|
||||
(`MODEL=/path/to/pi0_base`).
|
||||
|
||||
## Run the server
|
||||
|
||||
```bash
|
||||
vllm serve lerobot/pi0_base --omni --port 8000 \
|
||||
--served-model-name pi0 \
|
||||
--deploy-config vllm_omni/deploy/pi0.yaml \
|
||||
--enforce-eager --disable-log-stats
|
||||
```
|
||||
|
||||
The deploy config (`vllm_omni/deploy/pi0.yaml`) declares a single diffusion stage
|
||||
(`Pi0Pipeline`), float32, `max_num_seqs: 1`, and the `policy_server_config` handshake
|
||||
metadata (3 cameras, `joint_position`, action horizon 50, action dim 32).
|
||||
|
||||
### Commonly adjusted `model_config` keys
|
||||
|
||||
These live under `stages[0].model_config` in `vllm_omni/deploy/pi0.yaml`:
|
||||
|
||||
| Key | Default | Meaning |
|
||||
|---|---|---|
|
||||
| `chunk_size` | `50` | Action-chunk length (timesteps) the model predicts per inference. |
|
||||
| `num_inference_steps` | `10` | Flow-matching Euler denoising steps. |
|
||||
| `max_action_dim` | `32` | Action dimensionality (state/action are padded to this). |
|
||||
| `max_state_dim` | `32` | Proprioceptive-state dimensionality (zero-padded to this). |
|
||||
| `image_resolution` | `[224, 224]` | Per-camera input size (square; SigLIP). |
|
||||
| `tokenizer_max_length` | `48` | Max PaliGemma prompt tokens. |
|
||||
| `max_cameras` | `3` | Camera slots the model attends to (real + `-1`-padded). |
|
||||
| `image_feature_keys` | 3 `observation.images.*` keys | Camera order the model attends to. |
|
||||
| `image_key_map` | `{}` | Map raw obs camera keys → `image_feature_keys` (empty = verbatim). |
|
||||
|
||||
The `policy_server_config` block below them is the OpenPI handshake metadata
|
||||
advertised to the client; keep its `action_horizon` / `action_dim` /
|
||||
`image_resolution` in sync with the `model_config` values above. The e2e test
|
||||
`tests/e2e/online_serving/test_pi0_expansion.py::test_pi0_openpi_online` connects to a
|
||||
live server and asserts the advertised metadata matches these `pi0.yaml` values.
|
||||
|
||||
## Run the client
|
||||
|
||||
```bash
|
||||
python examples/online_serving/pi0/openpi_client.py --host 127.0.0.1 --port 8000 \
|
||||
--prompt "pick up the red block and place it in the bin"
|
||||
```
|
||||
|
||||
It connects, prints the server metadata, sends robot observations, and prints the
|
||||
returned `[action_horizon, action_dim] = [50, 32]` action chunks. Replace the blank
|
||||
cameras / zero state in `_make_dummy_obs` with real frames (HWC uint8) and
|
||||
proprioceptive state to drive a robot.
|
||||
|
||||
### Observation format
|
||||
|
||||
The client sends a flat dict per inference:
|
||||
|
||||
```python
|
||||
{
|
||||
"observation.images.base_0_rgb": np.uint8[H, W, 3],
|
||||
"observation.images.left_wrist_0_rgb": np.uint8[H, W, 3],
|
||||
"observation.images.right_wrist_0_rgb":np.uint8[H, W, 3],
|
||||
"state": np.float32[state_dim], # zero-padded to max_state_dim=32 server-side
|
||||
"prompt": "pick up the red block",
|
||||
"session_id": "<uuid>", # accepted but ignored (π0 is stateless)
|
||||
}
|
||||
```
|
||||
|
||||
Camera keys must match the server's `image_feature_keys` (the checkpoint's
|
||||
`input_features` order). If your robot uses different camera names, set
|
||||
`model_config.image_key_map` in `pi0.yaml` to map raw obs keys → feature keys.
|
||||
|
||||
## Correctness
|
||||
|
||||
π0's flow-matching kernel is bit-for-bit matched to the LeRobot `PI0Policy` reference
|
||||
(`max|Δ| = 7.15e-07`, CPU/float32, fixed noise; see
|
||||
`tests/diffusion/models/pi0/test_pi0_parity.py::test_pi0_vllm_omni_vs_lerobot`) and is
|
||||
version-stable across transformers releases (the version-stability checks in
|
||||
`tests/diffusion/models/pi0/test_pi0_units.py`). An OpenPI websocket online-serving e2e
|
||||
lives in `tests/e2e/online_serving/test_pi0_expansion.py::test_pi0_openpi_online`.
|
||||
|
||||
## Limitations
|
||||
|
||||
- **Normalization stats**: `lerobot/pi0_base` uses identity normalization, which is
|
||||
fully supported (state passes through, actions are returned in the model's space).
|
||||
Per-dataset `norm_stats` declared in a checkpoint's `config.json` are honored, but
|
||||
stats stored only in LeRobot's `policy_preprocessor.json` companion safetensors are
|
||||
not yet loaded — a fine-tuned checkpoint that relies on those would need that bridge
|
||||
added before its actions are in real-world units.
|
||||
@@ -0,0 +1,119 @@
|
||||
#!/usr/bin/env python3
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Minimal OpenPI client for the π0 (Pi-Zero) VLA policy server.
|
||||
|
||||
Connects to a running π0 server (see README.md), sends a single robot
|
||||
observation (3 cameras + state + language instruction), and prints the returned
|
||||
continuous action chunk ``[action_horizon, action_dim]``.
|
||||
|
||||
The server speaks the OpenPI websocket protocol:
|
||||
connect -> server sends msgpack(policy_server_config metadata)
|
||||
infer -> client sends msgpack(obs), server replies msgpack(actions ndarray)
|
||||
|
||||
Usage:
|
||||
python openpi_client.py --host 127.0.0.1 --port 8000 \
|
||||
--prompt "pick up the red block"
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
import sys
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
|
||||
try:
|
||||
# Avoid shadowing the installed ``openpi_client`` package with this dir.
|
||||
example_dir = str(Path(__file__).resolve().parent)
|
||||
removed = sys.path and sys.path[0] == example_dir
|
||||
if removed:
|
||||
sys.path.pop(0)
|
||||
try:
|
||||
from openpi_client.websocket_client_policy import WebsocketClientPolicy
|
||||
finally:
|
||||
if removed:
|
||||
sys.path.insert(0, example_dir)
|
||||
except ImportError as exc: # pragma: no cover - runtime dependency guard
|
||||
raise ImportError("π0 OpenPI example requires `openpi-client`.") from exc
|
||||
|
||||
logger = logging.getLogger("pi0_openpi_client")
|
||||
|
||||
DEFAULT_HOST = "127.0.0.1"
|
||||
DEFAULT_PORT = 8000
|
||||
DEFAULT_PATH = "/v1/realtime/robot/openpi"
|
||||
DEFAULT_PROMPT = "pick up the red block and place it in the bin"
|
||||
|
||||
# π0 / pi0_base camera identities (must match the server's image_feature_keys).
|
||||
CAMERA_KEYS = (
|
||||
"observation.images.base_0_rgb",
|
||||
"observation.images.left_wrist_0_rgb",
|
||||
"observation.images.right_wrist_0_rgb",
|
||||
)
|
||||
STATE_DIM = 32
|
||||
|
||||
|
||||
def _make_dummy_obs(prompt: str, session_id: str, image_size: int = 224) -> dict[str, Any]:
|
||||
"""Build a single π0 observation with blank cameras + zero state.
|
||||
|
||||
Replace the zeros with real camera frames (HWC uint8) and proprioceptive
|
||||
state to drive an actual robot.
|
||||
"""
|
||||
obs: dict[str, Any] = {cam: np.zeros((image_size, image_size, 3), dtype=np.uint8) for cam in CAMERA_KEYS}
|
||||
obs["state"] = np.zeros(STATE_DIM, dtype=np.float32)
|
||||
obs["prompt"] = prompt
|
||||
obs["session_id"] = session_id
|
||||
return obs
|
||||
|
||||
|
||||
def _ws_uri(host: str, port: int, path: str) -> str:
|
||||
"""Build the full websocket URI. WebsocketClientPolicy takes the whole URI as
|
||||
``host`` (it has no separate path arg)."""
|
||||
if host.startswith(("ws://", "wss://")):
|
||||
return host
|
||||
return f"ws://{host}:{port}{path}"
|
||||
|
||||
|
||||
def _as_action_array(response: Any) -> np.ndarray:
|
||||
if isinstance(response, dict) and response.get("type") == "error":
|
||||
raise RuntimeError(f"Inference failed: {response.get('message', response)}")
|
||||
return np.asarray(response, dtype=np.float32)
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="π0 OpenPI client")
|
||||
parser.add_argument("--host", default=DEFAULT_HOST)
|
||||
parser.add_argument("--port", type=int, default=DEFAULT_PORT)
|
||||
parser.add_argument("--path", default=DEFAULT_PATH)
|
||||
parser.add_argument("--prompt", default=DEFAULT_PROMPT)
|
||||
parser.add_argument("--num-steps", type=int, default=2, help="Inference observations to send.")
|
||||
args = parser.parse_args()
|
||||
logging.basicConfig(level=logging.INFO, format="%(message)s")
|
||||
|
||||
policy = WebsocketClientPolicy(host=_ws_uri(args.host, args.port, args.path))
|
||||
metadata = dict(policy.get_server_metadata())
|
||||
logger.info("Server metadata: %s", metadata)
|
||||
|
||||
session_id = str(uuid.uuid4())
|
||||
for step in range(args.num_steps):
|
||||
obs = _make_dummy_obs(args.prompt, session_id)
|
||||
actions = _as_action_array(policy.infer(obs))
|
||||
logger.info(
|
||||
"[step %d] actions shape=%s mean=%.4f std=%.4f",
|
||||
step,
|
||||
actions.shape,
|
||||
float(actions.mean()),
|
||||
float(actions.std()),
|
||||
)
|
||||
if not np.isfinite(actions).all():
|
||||
raise RuntimeError("Server returned non-finite actions.")
|
||||
logger.info("Done — π0 OpenPI round-trip OK.")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -51,7 +51,7 @@ code2wav_sampling_params='{
|
||||
"detokenize": true,
|
||||
"repetition_penalty": 1.1
|
||||
}'
|
||||
# Above is optional, it has a default setting in stage_configs of the corresponding model.
|
||||
# The block above is optional; defaults come from the resolved pipeline and deploy config.
|
||||
|
||||
# Define URLs for assets
|
||||
MARY_HAD_LAMB_AUDIO_URL="https://vllm-public-assets.s3.us-west-2.amazonaws.com/multimodal_asset/mary_had_lamb.ogg"
|
||||
|
||||
@@ -24,6 +24,18 @@ vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091 \
|
||||
--deploy-config /path/to/your_deploy_config.yaml
|
||||
```
|
||||
|
||||
To serve **thinker-only** (text output, no talker / code2wav loaded) with Instruct
|
||||
weights, pass the bundled thinker-only deploy YAML. The `pipeline:` key
|
||||
`qwen3_omni_moe_thinker_only` overrides the HF `enable_audio_output` resolver:
|
||||
|
||||
```bash
|
||||
vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091 \
|
||||
--deploy-config vllm_omni/deploy/qwen3_omni_moe_thinking.yaml
|
||||
```
|
||||
|
||||
Captioner / Thinking checkpoints (`enable_audio_output=false`) still auto-select
|
||||
the same single-stage pipeline without `--deploy-config`.
|
||||
|
||||
For a 3x-GPU multi-replica layout (talker/code2wav scale-out on cuda:1,2),
|
||||
use `--stage-overrides` on top of the default config:
|
||||
|
||||
@@ -513,7 +525,7 @@ The script supports the following arguments:
|
||||
vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091
|
||||
```
|
||||
|
||||
If you have custom stage configs file:
|
||||
If you have a custom deploy config file:
|
||||
```bash
|
||||
vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091 --deploy-config /path/to/deploy_config_file
|
||||
```
|
||||
|
||||
@@ -50,7 +50,7 @@ code2wav_sampling_params='{
|
||||
"detokenize": true,
|
||||
"repetition_penalty": 1.1
|
||||
}'
|
||||
# Above is optional, it has a default setting in stage_configs of the corresponding model.
|
||||
# The block above is optional; defaults come from the resolved pipeline and deploy config.
|
||||
|
||||
# Define URLs for assets
|
||||
MARY_HAD_LAMB_AUDIO_URL="https://vllm-public-assets.s3.us-west-2.amazonaws.com/multimodal_asset/mary_had_lamb.ogg"
|
||||
|
||||
@@ -15,7 +15,7 @@ so an output is a deterministic function of its own input and nothing else.
|
||||
Measuring replica scaling
|
||||
-------------------------
|
||||
This client is replica-agnostic. Vary the *server's* replica count
|
||||
(`runtime.num_replicas` in the stage config; see run_server.sh) and keep the
|
||||
(`stages[].num_replicas` in the deploy config; see run_server.sh) and keep the
|
||||
client workload fixed, then compare throughput:
|
||||
|
||||
N=1 -> baseline
|
||||
|
||||
@@ -1,234 +0,0 @@
|
||||
# Stable Audio Online Serving
|
||||
|
||||
Generate audio from text prompts using Stable Audio models via an OpenAI-compatible API endpoint.
|
||||
|
||||
## Features
|
||||
|
||||
- **OpenAI-compatible API**: Use `/v1/audio/generate` endpoint
|
||||
- **Flexible control**: Adjust audio length, guidance scale, inference steps
|
||||
- **Quality control**: Use negative prompts to avoid unwanted characteristics
|
||||
- **Reproducible**: Set random seed for deterministic generation
|
||||
|
||||
## Quick Start
|
||||
|
||||
### 1. Start the Server
|
||||
|
||||
```bash
|
||||
vllm serve stabilityai/stable-audio-open-1.0 \
|
||||
--host 0.0.0.0 \
|
||||
--port 8091 \
|
||||
--gpu-memory-utilization 0.9 \
|
||||
--trust-remote-code \
|
||||
--enforce-eager \
|
||||
--omni
|
||||
```
|
||||
|
||||
### 2. Generate Audio
|
||||
|
||||
#### Using curl
|
||||
|
||||
```bash
|
||||
curl -X POST http://localhost:8091/v1/audio/generate \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"input": "The sound of a cat purring",
|
||||
"audio_length": 10.0
|
||||
}' --output cat.wav
|
||||
```
|
||||
|
||||
#### Using Python Client
|
||||
|
||||
```bash
|
||||
python stable_audio_client.py \
|
||||
--text "The sound of a cat purring" \
|
||||
--audio_length 10.0 \
|
||||
--output cat.wav
|
||||
```
|
||||
|
||||
#### Using Bash Script
|
||||
|
||||
```bash
|
||||
bash curl_examples.sh
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
### Endpoint
|
||||
|
||||
```
|
||||
POST /v1/audio/generate
|
||||
```
|
||||
|
||||
### Request Body
|
||||
|
||||
```json
|
||||
{
|
||||
"input": "Text description of the audio",
|
||||
"audio_length": 10.0,
|
||||
"audio_start": 0.0,
|
||||
"negative_prompt": "Low quality",
|
||||
"guidance_scale": 7.0,
|
||||
"num_inference_steps": 100,
|
||||
"seed": 42,
|
||||
"response_format": "wav"
|
||||
}
|
||||
```
|
||||
|
||||
### Parameters
|
||||
|
||||
| Parameter | Type | Default | Description |
|
||||
|-----------|------|---------|-------------|
|
||||
| `input` | string | **required** | Text prompt describing the audio to generate |
|
||||
| `audio_length` | float | ~47s | Audio duration in seconds (max ~47s for stable-audio-open-1.0) |
|
||||
| `audio_start` | float | 0.0 | Audio start time in seconds |
|
||||
| `negative_prompt` | string | null | Text describing what to avoid in generation |
|
||||
| `guidance_scale` | float | 7.0 | Classifier-free guidance scale (higher = more adherence to prompt) |
|
||||
| `num_inference_steps` | int | 50 | Number of denoising steps (higher = better quality, slower) |
|
||||
| `seed` | int | null | Random seed for reproducibility |
|
||||
| `response_format` | string | "wav" | Output format: wav, mp3, flac, pcm |
|
||||
|
||||
### Response
|
||||
|
||||
Returns audio data in the requested format (default: WAV).
|
||||
|
||||
## Usage Examples
|
||||
|
||||
### Basic Generation
|
||||
|
||||
```bash
|
||||
curl -X POST http://localhost:8091/v1/audio/generate \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"input": "The sound of ocean waves"
|
||||
}' --output ocean.wav
|
||||
```
|
||||
|
||||
### Custom Duration
|
||||
|
||||
```bash
|
||||
curl -X POST http://localhost:8091/v1/audio/generate \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"input": "A dog barking",
|
||||
"audio_length": 5.0
|
||||
}' --output dog_5s.wav
|
||||
```
|
||||
|
||||
### High Quality with Negative Prompt
|
||||
|
||||
```bash
|
||||
curl -X POST http://localhost:8091/v1/audio/generate \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"input": "A piano playing a gentle melody",
|
||||
"audio_length": 10.0,
|
||||
"negative_prompt": "Low quality, distorted, noisy",
|
||||
"guidance_scale": 8.0,
|
||||
"num_inference_steps": 150
|
||||
}' --output piano_hq.wav
|
||||
```
|
||||
|
||||
### Reproducible Generation
|
||||
|
||||
```bash
|
||||
curl -X POST http://localhost:8091/v1/audio/generate \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"input": "Thunder and rain sounds",
|
||||
"audio_length": 15.0,
|
||||
"seed": 42
|
||||
}' --output thunder.wav
|
||||
```
|
||||
|
||||
### Quick Generation (Fewer Steps)
|
||||
|
||||
For faster generation with slightly lower quality:
|
||||
|
||||
```bash
|
||||
curl -X POST http://localhost:8091/v1/audio/generate \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"input": "Birds chirping in a forest",
|
||||
"audio_length": 8.0,
|
||||
"num_inference_steps": 50
|
||||
}' --output birds_quick.wav
|
||||
```
|
||||
|
||||
## Python Client Examples
|
||||
|
||||
### Simple Generation
|
||||
|
||||
```bash
|
||||
python stable_audio_client.py \
|
||||
--text "The sound of a cat purring"
|
||||
```
|
||||
|
||||
### Custom Parameters
|
||||
|
||||
```bash
|
||||
python stable_audio_client.py \
|
||||
--text "Thunder and rain" \
|
||||
--audio_length 15.0 \
|
||||
--negative_prompt "Low quality" \
|
||||
--guidance_scale 7.0 \
|
||||
--num_inference_steps 100 \
|
||||
--seed 42 \
|
||||
--output thunder.wav
|
||||
```
|
||||
|
||||
### Different Output Format
|
||||
|
||||
```bash
|
||||
python stable_audio_client.py \
|
||||
--text "Guitar playing" \
|
||||
--response_format mp3 \
|
||||
--output guitar.mp3
|
||||
```
|
||||
|
||||
## Tips
|
||||
|
||||
1. **Audio Length**: Keep under 47 seconds for `stable-audio-open-1.0`
|
||||
2. **Quality vs Speed**:
|
||||
- 50 steps: Fast, decent quality
|
||||
- 100 steps: Good balance (default)
|
||||
- 150+ steps: High quality, slower
|
||||
3. **Guidance Scale**:
|
||||
- Lower (3-5): More creative/varied
|
||||
- Default (7): Good balance
|
||||
- Higher (10+): More literal to prompt
|
||||
4. **Negative Prompts**: Use to avoid "Low quality", "distorted", "noisy", etc.
|
||||
5. **Seeds**: Use same seed for reproducible results
|
||||
|
||||
## Performance
|
||||
|
||||
| Inference Steps | Quality | Speed | Use Case |
|
||||
|----------------|---------|-------|----------|
|
||||
| 50 | Good | Fast | Quick previews |
|
||||
| 100 (default) | Very Good | Medium | Production |
|
||||
| 150+ | Excellent | Slow | Final/critical audio |
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Server not responding
|
||||
- Check if server is running: `curl http://localhost:8091/health`
|
||||
- Check server logs for errors
|
||||
|
||||
### Audio quality issues
|
||||
- Increase `num_inference_steps` (e.g., 150)
|
||||
- Add negative prompts: `"Low quality, distorted, noisy"`
|
||||
- Increase `guidance_scale` for more prompt adherence
|
||||
|
||||
### Generation timeout
|
||||
- Reduce `num_inference_steps`
|
||||
- Reduce `audio_length`
|
||||
- Check GPU memory with `nvidia-smi`
|
||||
|
||||
### Wrong audio length
|
||||
- Ensure `audio_length` is within model limits (~47s max)
|
||||
- Adjust `audio_start` if trimming is needed
|
||||
|
||||
## See Also
|
||||
|
||||
- [Offline Inference Example](../../offline_inference/text_to_audio/README.md)
|
||||
- [Stable Audio Model Card](https://huggingface.co/stabilityai/stable-audio-open-1.0)
|
||||
- [vLLM-Omni Documentation](https://github.com/vllm-project/vllm-omni)
|
||||
@@ -1,54 +0,0 @@
|
||||
#!/bin/bash
|
||||
# Examples for using Stable Audio with curl via /v1/audio/generate endpoint
|
||||
|
||||
# Example 1: Simple request with default parameters
|
||||
echo "Example 1: Simple request with default parameters"
|
||||
curl -X POST http://localhost:8091/v1/audio/generate \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"input": "The sound audience clapping and cheering in a stadium"
|
||||
}' --output stadium.wav
|
||||
|
||||
# Example 2: Request with custom audio_length
|
||||
echo "Example 2: Custom audio length (5 seconds)"
|
||||
curl -X POST http://localhost:8091/v1/audio/generate \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"input": "The sound of a dog barking",
|
||||
"audio_length": 5.0
|
||||
}' --output dog_5s.wav
|
||||
|
||||
# Example 3: Request with negative prompt for quality control
|
||||
echo "Example 3: With negative prompt"
|
||||
curl -X POST http://localhost:8091/v1/audio/generate \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"input": "A piano playing a gentle melody",
|
||||
"audio_length": 10.0,
|
||||
"negative_prompt": "Low quality, distorted, noisy"
|
||||
}' --output piano.wav
|
||||
|
||||
# Example 4: Full control with all parameters
|
||||
echo "Example 4: Full control (custom length, guidance, steps, seed)"
|
||||
curl -X POST http://localhost:8091/v1/audio/generate \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"input": "Thunder and rain sounds",
|
||||
"audio_length": 15.0,
|
||||
"negative_prompt": "Low quality",
|
||||
"guidance_scale": 7.0,
|
||||
"num_inference_steps": 100,
|
||||
"seed": 42
|
||||
}' --output thunder_rain.wav
|
||||
|
||||
# Example 5: Quick generation with fewer steps (faster but lower quality)
|
||||
echo "Example 5: Quick generation (fewer steps)"
|
||||
curl -X POST http://localhost:8091/v1/audio/generate \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"input": "Ocean waves crashing on a beach",
|
||||
"audio_length": 8.0,
|
||||
"num_inference_steps": 50
|
||||
}' --output ocean.wav
|
||||
|
||||
echo "All examples completed!"
|
||||
@@ -1,170 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
OpenAI-compatible client for Stable Audio via /v1/audio/generate endpoint.
|
||||
|
||||
This script demonstrates how to use the OpenAI-compatible speech API
|
||||
to generate audio from text using Stable Audio models.
|
||||
|
||||
Examples:
|
||||
# Simple generation
|
||||
python stable_audio_client.py --text "The sound of a cat purring"
|
||||
|
||||
# With custom duration
|
||||
python stable_audio_client.py --text "A dog barking" --audio_length 5.0
|
||||
|
||||
# With all parameters
|
||||
python stable_audio_client.py --text "Thunder and rain" \
|
||||
--audio_length 15.0 \
|
||||
--negative_prompt "Low quality" \
|
||||
--guidance_scale 7.0 \
|
||||
--num_inference_steps 100 \
|
||||
--seed 42 \
|
||||
--output thunder.wav
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import sys
|
||||
|
||||
import requests
|
||||
|
||||
|
||||
def parse_args():
|
||||
parser = argparse.ArgumentParser(description="Generate audio with Stable Audio via OpenAI-compatible API")
|
||||
parser.add_argument(
|
||||
"--api_url",
|
||||
default="http://localhost:8091/v1/audio/generate",
|
||||
help="API endpoint URL",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--text",
|
||||
default="The sound of a cat purring",
|
||||
help="Text prompt for audio generation",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--audio_length",
|
||||
type=float,
|
||||
default=10.0,
|
||||
help="Audio length in seconds (max ~47s for stable-audio-open-1.0)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--audio_start",
|
||||
type=float,
|
||||
default=0.0,
|
||||
help="Audio start time in seconds",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--negative_prompt",
|
||||
default="Low quality",
|
||||
help="Negative prompt for classifier-free guidance",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--guidance_scale",
|
||||
type=float,
|
||||
default=7.0,
|
||||
help="Guidance scale for diffusion (higher = more adherence to prompt)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--num_inference_steps",
|
||||
type=int,
|
||||
default=100,
|
||||
help="Number of inference steps (higher = better quality, slower)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--seed",
|
||||
type=int,
|
||||
default=None,
|
||||
help="Random seed for reproducibility",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output",
|
||||
default="stable_audio_output.wav",
|
||||
help="Output file path",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--response_format",
|
||||
default="wav",
|
||||
choices=["wav", "mp3", "flac", "pcm"],
|
||||
help="Audio output format",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def generate_audio(args):
|
||||
"""Generate audio using the API."""
|
||||
|
||||
# Build request payload
|
||||
payload = {
|
||||
"input": args.text,
|
||||
"audio_length": args.audio_length,
|
||||
"audio_start": args.audio_start,
|
||||
"response_format": args.response_format,
|
||||
}
|
||||
|
||||
# Add optional parameters
|
||||
if args.negative_prompt:
|
||||
payload["negative_prompt"] = args.negative_prompt
|
||||
if args.guidance_scale:
|
||||
payload["guidance_scale"] = args.guidance_scale
|
||||
if args.num_inference_steps:
|
||||
payload["num_inference_steps"] = args.num_inference_steps
|
||||
if args.seed is not None:
|
||||
payload["seed"] = args.seed
|
||||
|
||||
print(f"\n{'=' * 60}")
|
||||
print("Stable Audio - Text-to-Audio Generation")
|
||||
print(f"{'=' * 60}")
|
||||
print(f"API URL: {args.api_url}")
|
||||
print(f"Prompt: {args.text}")
|
||||
print(f"Audio length: {args.audio_length}s")
|
||||
print(f"Negative prompt: {args.negative_prompt}")
|
||||
print(f"Guidance scale: {args.guidance_scale}")
|
||||
print(f"Inference steps: {args.num_inference_steps}")
|
||||
if args.seed is not None:
|
||||
print(f"Seed: {args.seed}")
|
||||
print(f"Output: {args.output}")
|
||||
print(f"{'=' * 60}\n")
|
||||
|
||||
try:
|
||||
# Make the API request
|
||||
print("Generating audio...")
|
||||
response = requests.post(
|
||||
args.api_url,
|
||||
json=payload,
|
||||
headers={"Content-Type": "application/json"},
|
||||
timeout=300, # 5 minute timeout for long generations
|
||||
)
|
||||
|
||||
# Check for errors
|
||||
if response.status_code != 200:
|
||||
print(f"Error: API returned status code {response.status_code}")
|
||||
print(f"Response: {response.text}")
|
||||
return False
|
||||
|
||||
# Save the audio
|
||||
with open(args.output, "wb") as f:
|
||||
f.write(response.content)
|
||||
|
||||
print(f"✓ Audio saved to {args.output}")
|
||||
print(f" File size: {len(response.content) / 1024:.1f} KB")
|
||||
return True
|
||||
|
||||
except requests.exceptions.Timeout:
|
||||
print("Error: Request timed out. Try reducing inference steps or audio length.")
|
||||
return False
|
||||
except requests.exceptions.ConnectionError:
|
||||
print(f"Error: Could not connect to {args.api_url}")
|
||||
print("Make sure the server is running.")
|
||||
return False
|
||||
except Exception as e:
|
||||
print(f"Error: {e}")
|
||||
return False
|
||||
|
||||
|
||||
def main():
|
||||
args = parse_args()
|
||||
success = generate_audio(args)
|
||||
sys.exit(0 if success else 1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -189,5 +189,5 @@ Async chunk reduces TTFP by **67%** by streaming audio token chunks from Thinker
|
||||
- Check server logs for errors
|
||||
|
||||
### Out of memory
|
||||
- Reduce `gpu_memory_utilization` in stage configs
|
||||
- Reduce `gpu_memory_utilization` in the deploy config
|
||||
- Use a smaller batch size
|
||||
@@ -25,7 +25,6 @@ For the full list of supported architectures across all modalities, see
|
||||
| Qwen3-TTS | `Qwen/Qwen3-TTS-12Hz-1.7B-{CustomVoice,VoiceDesign,Base}` | ✓ (Base) | ✓ (PCM + WebSocket) | ✓ (presets + `/v1/audio/voices` upload) | ✓ (standard + FastRTC) |
|
||||
| VoxCPM2 | `openbmb/VoxCPM2` | ✓ | ✓ (AudioWorklet via gradio) | — | ✓ |
|
||||
| Voxtral TTS | `mistralai/Voxtral-4B-TTS-2603` | ✓ (gated upstream) | ✓ | ✓ (presets) | ✓ |
|
||||
| SoulX-Singer | `Soul-AILab/SoulX-Singer` | ✓ (prompt audio) | — (batch only) | — (prompt + target audio) | — (chat client) |
|
||||
|
||||
CosyVoice3 is intentionally absent: no online example exists for it yet. See its [offline section](../../offline_inference/text_to_speech/README.md#cosyvoice3) instead.
|
||||
|
||||
@@ -570,7 +569,7 @@ curl -X POST http://localhost:8091/v1/audio/speech \
|
||||
"response_format": "pcm"
|
||||
}' --no-buffer | play -t raw -r 24000 -e signed -b 16 -c 1 -
|
||||
```
|
||||
Raw PCM streaming requires `stream_format="audio"`, `response_format="pcm"`, and `async_chunk: true` on the stage config (default in `qwen3_tts.yaml`). `speed` is not supported when streaming.
|
||||
Raw PCM streaming requires `stream_format="audio"`, `response_format="pcm"`, and `async_chunk: true` in the deploy config (default in `qwen3_tts.yaml`). `speed` is not supported when streaming.
|
||||
|
||||
### Streaming WebSocket
|
||||
The `/v1/audio/speech/stream` endpoint accepts text incrementally and synthesizes the buffered text as one continuous request on `input.done`:
|
||||
@@ -715,65 +714,3 @@ The demo handles voice-preset selection and reference-audio upload. `voxtral_tts
|
||||
- Voice presets are listed on the HF model card (`mistralai/Voxtral-4B-TTS-2603`).
|
||||
- Voice cloning is gated upstream and may require a recent `mistral_common`.
|
||||
- A standalone CLI client is not yet shipped; the gradio demo is the canonical reference for now.
|
||||
|
||||
---
|
||||
|
||||
## SoulX-Singer
|
||||
|
||||
Singing voice synthesis (SVS) and conversion (SVC) at 24 kHz. Single-stage DiT with inline preprocess. Uses the `/v1/chat/completions` endpoint with multimodal input (`prompt_audio` + `target_audio`).
|
||||
|
||||
### Prerequisites
|
||||
|
||||
Download DiT and preprocess weights, then set up separate SVS / SVC view directories and install dependencies as described in the [offline README](../../offline_inference/text_to_speech/README.md#soulx-singer). `config.json` `architectures` field is the single source of truth for SVS vs SVC — point `MODEL` at the matching directory.
|
||||
|
||||
### Launch
|
||||
|
||||
```bash
|
||||
# SVS (default)
|
||||
export MODEL=/path/to/SoulX-Singer
|
||||
export PREPROCESS=/path/to/SoulX-Singer-Preprocess
|
||||
bash examples/online_serving/text_to_speech/soulxsinger/run_server.sh
|
||||
|
||||
# SVC
|
||||
export MODE=svc
|
||||
export MODEL=/path/to/SoulX-Singer-svc
|
||||
bash examples/online_serving/text_to_speech/soulxsinger/run_server.sh
|
||||
```
|
||||
|
||||
Or equivalently, set `SOULX_PREPROCESS_WEIGHTS_DIR` and launch directly:
|
||||
```bash
|
||||
export SOULX_PREPROCESS_WEIGHTS_DIR=$PREPROCESS
|
||||
vllm serve $MODEL --omni \
|
||||
--deploy-config vllm_omni/deploy/soulxsinger_${MODE}.yaml \
|
||||
--port 8192 --trust-remote-code --enforce-eager
|
||||
```
|
||||
|
||||
### Sending requests
|
||||
|
||||
Audio paths must be reachable from the server host (local filesystem or data URL). The client sends prompt vocal via `input_audio` and target accompaniment via `extra_args['target_audio']`.
|
||||
|
||||
```bash
|
||||
# Default demo audio: tests/assets/soulxsinger/zh_prompt.mp3 + music.mp3
|
||||
python examples/online_serving/text_to_speech/soulxsinger/openai_chat_client.py \
|
||||
--prompt-audio /path/on/server/zh_prompt.mp3 \
|
||||
--target-audio /path/on/server/music.mp3 \
|
||||
--preprocess-weights-dir /path/on/server/SoulX-Singer-Preprocess \
|
||||
-o output.wav
|
||||
```
|
||||
|
||||
Use precomputed metadata to skip online preprocess with following command:
|
||||
```bash
|
||||
python examples/online_serving/text_to_speech/soulxsinger/openai_chat_client.py \
|
||||
--prompt-metadata-path /path/on/server/zh_prompt.json \
|
||||
--target-metadata-path /path/on/server/music.json \
|
||||
--audio-path /path/on/server/zh_prompt.mp3 \
|
||||
-o output.wav
|
||||
```
|
||||
|
||||
`SOULX_PREPROCESS_WEIGHTS_DIR` makes `--preprocess-weights-dir` optional. See `openai_chat_client.py --help` for `--vocal-sep`, `--language`, `--num-inference-steps`, `--guidance-scale`, and `--seed`.
|
||||
|
||||
### Notes
|
||||
|
||||
- Output: 24 kHz mono WAV; batch only.
|
||||
- Defaults match upstream: `--guidance-scale 3.0`, `--seed 42`, `--auto-shift` on.
|
||||
- SVS `--control`: `score` or `melody`. MIDI / lyric QC: upstream `midi_editor` only.
|
||||
@@ -1,186 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""SoulX-Singer OpenAI-compatible chat client (SVS / SVC).
|
||||
|
||||
Sends prompt audio via ``input_audio`` and target accompaniment via
|
||||
``extra_args['target_audio']`` (server-local path). For integrated preprocess,
|
||||
also pass ``preprocess_weights_dir`` in ``extra_args``.
|
||||
|
||||
Usage:
|
||||
python openai_chat_client.py \\
|
||||
--prompt-audio /path/on/server/zh_prompt.mp3 \\
|
||||
--target-audio /path/on/server/music.mp3 \\
|
||||
--preprocess-weights-dir /path/on/server/SoulX-Singer-Preprocess \\
|
||||
-o output.wav
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import base64
|
||||
import io
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import requests
|
||||
import soundfile
|
||||
import torch
|
||||
|
||||
|
||||
def _audio_to_data_url(path: Path) -> str:
|
||||
with path.open("rb") as handle:
|
||||
data = base64.b64encode(handle.read()).decode("ascii")
|
||||
return f"data:audio/mpeg;base64,{data}"
|
||||
|
||||
|
||||
def _save_wav(audio: torch.Tensor, path: Path, sample_rate: int) -> None:
|
||||
audio = audio.to(torch.float32)
|
||||
peak = audio.abs().max().clamp(min=1e-8)
|
||||
audio = audio / peak
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
soundfile.write(str(path), audio.clamp(-1.0, 1.0).cpu().T.numpy(), sample_rate, subtype="PCM_16")
|
||||
|
||||
|
||||
def _decode_audio_from_response(body: dict) -> tuple[torch.Tensor, int]:
|
||||
for choice in body.get("choices", []):
|
||||
audio_obj = choice.get("message", {}).get("audio")
|
||||
if isinstance(audio_obj, dict) and audio_obj.get("data"):
|
||||
data, sr = soundfile.read(
|
||||
io.BytesIO(base64.b64decode(audio_obj["data"])),
|
||||
dtype="float32",
|
||||
always_2d=True,
|
||||
)
|
||||
return torch.from_numpy(data).transpose(0, 1), sr
|
||||
brief = {k: v for k, v in body.items() if k != "choices"}
|
||||
raise RuntimeError(f"no audio in response message.audio: {brief}")
|
||||
|
||||
|
||||
def main() -> int:
|
||||
repo_root = Path(__file__).resolve().parents[4]
|
||||
default_assets = repo_root / "tests" / "assets" / "soulxsinger"
|
||||
|
||||
parser = argparse.ArgumentParser(description="SoulX-Singer online chat client")
|
||||
parser.add_argument("--port", type=int, default=8192)
|
||||
parser.add_argument("--model", default="Soul-AILab/SoulX-Singer")
|
||||
parser.add_argument(
|
||||
"--prompt-audio",
|
||||
default=str(default_assets / "zh_prompt.mp3"),
|
||||
help="Prompt vocal audio (path on server if using extra_args, or local for input_audio)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--target-audio",
|
||||
default=str(default_assets / "music.mp3"),
|
||||
help="Target accompaniment path on the server (extra_args['target_audio'])",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--prompt-metadata-path",
|
||||
default=None,
|
||||
help="SVS precomputed prompt metadata.json",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--target-metadata-path",
|
||||
default=None,
|
||||
help="SVS precomputed target metadata.json",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--audio-path",
|
||||
default=None,
|
||||
help="SVS prompt vocal wav for precomputed metadata",
|
||||
)
|
||||
parser.add_argument("--preprocess-weights-dir", default=None)
|
||||
parser.add_argument("--output", "-o", default="soulxsinger_out.wav")
|
||||
parser.add_argument("--svc", action="store_true", help="Use SVC mode knobs")
|
||||
parser.add_argument("--language", default="Mandarin")
|
||||
parser.add_argument("--num-inference-steps", type=int, default=32)
|
||||
parser.add_argument("--guidance-scale", type=float, default=3.0)
|
||||
parser.add_argument(
|
||||
"--seed",
|
||||
type=int,
|
||||
default=42,
|
||||
help="Optional CFM seed. Omit for non-deterministic sampling.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--auto-shift",
|
||||
action=argparse.BooleanOptionalAction,
|
||||
default=True,
|
||||
help="Auto pitch shift (default: on, original upstream infer.sh)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--control",
|
||||
default="melody",
|
||||
choices=["melody", "score"],
|
||||
help="SVS control mode",
|
||||
)
|
||||
parser.add_argument("--vocal-sep", action="store_true")
|
||||
args = parser.parse_args()
|
||||
|
||||
meta_paths = (args.prompt_metadata_path, args.target_metadata_path, args.audio_path)
|
||||
if any(meta_paths) and not all(meta_paths):
|
||||
print(
|
||||
"ERROR: precomputed metadata requires --prompt-metadata-path, "
|
||||
"--target-metadata-path, and --audio-path together.",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 2
|
||||
|
||||
extra_args: dict = {
|
||||
"vocal_sep": args.vocal_sep,
|
||||
"auto_shift": args.auto_shift,
|
||||
"pitch_shift": 0,
|
||||
}
|
||||
if all(meta_paths):
|
||||
extra_args.update(
|
||||
{
|
||||
"prompt_metadata_path": str(Path(args.prompt_metadata_path).expanduser().resolve()),
|
||||
"target_metadata_path": str(Path(args.target_metadata_path).expanduser().resolve()),
|
||||
"audio_path": str(Path(args.audio_path).expanduser().resolve()),
|
||||
}
|
||||
)
|
||||
content = [{"type": "text", "text": "soulx-singer"}]
|
||||
else:
|
||||
prompt_path = Path(args.prompt_audio).expanduser().resolve()
|
||||
if not prompt_path.is_file():
|
||||
print(f"ERROR: prompt audio not found: {prompt_path}", file=sys.stderr)
|
||||
return 2
|
||||
extra_args["prompt_audio"] = str(prompt_path)
|
||||
extra_args["target_audio"] = str(Path(args.target_audio).expanduser().resolve())
|
||||
if args.preprocess_weights_dir:
|
||||
extra_args["preprocess_weights_dir"] = str(Path(args.preprocess_weights_dir).expanduser().resolve())
|
||||
content = [
|
||||
{"type": "text", "text": "soulx-singer"},
|
||||
{
|
||||
"type": "input_audio",
|
||||
"input_audio": {"data": _audio_to_data_url(prompt_path), "format": "mp3"},
|
||||
},
|
||||
]
|
||||
if not args.svc:
|
||||
extra_args["language"] = args.language
|
||||
extra_args["control"] = args.control
|
||||
|
||||
payload = {
|
||||
"model": args.model,
|
||||
"modalities": ["audio"],
|
||||
"messages": [{"role": "user", "content": content}],
|
||||
"num_inference_steps": args.num_inference_steps,
|
||||
"guidance_scale": args.guidance_scale,
|
||||
"extra_args": extra_args,
|
||||
}
|
||||
if args.seed is not None:
|
||||
payload["seed"] = args.seed
|
||||
|
||||
print(f"POST http://localhost:{args.port}/v1/chat/completions")
|
||||
response = requests.post(
|
||||
f"http://localhost:{args.port}/v1/chat/completions",
|
||||
headers={"Content-Type": "application/json"},
|
||||
json=payload,
|
||||
timeout=1800,
|
||||
)
|
||||
response.raise_for_status()
|
||||
audio, sample_rate = _decode_audio_from_response(response.json())
|
||||
_save_wav(audio, Path(args.output), sample_rate)
|
||||
duration = audio.shape[-1] / sample_rate
|
||||
print(f"saved {args.output} sr={sample_rate}Hz duration={duration:.2f}s")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -1,39 +0,0 @@
|
||||
#!/bin/bash
|
||||
# Launch vLLM-Omni server for SoulX-Singer (single-stage DiT, preprocess inline).
|
||||
#
|
||||
# Usage:
|
||||
# MODEL=/path/to/SoulX-Singer PREPROCESS=/path/to/Preprocess \
|
||||
#
|
||||
# Audio paths in client extra_args must be readable on the server host.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
REPO_ROOT="$(cd "$SCRIPT_DIR/../../../.." && pwd)"
|
||||
|
||||
MODEL="${MODEL:-Soul-AILab/SoulX-Singer}"
|
||||
MODE="${MODE:-svs}"
|
||||
PORT="${PORT:-8192}"
|
||||
GPUS="${GPUS:-0}"
|
||||
|
||||
if [[ "$MODE" == "svc" ]]; then
|
||||
DEPLOY_CONFIG="${DEPLOY_CONFIG:-$REPO_ROOT/vllm_omni/deploy/soulxsinger_svc.yaml}"
|
||||
else
|
||||
DEPLOY_CONFIG="${DEPLOY_CONFIG:-$REPO_ROOT/vllm_omni/deploy/soulxsinger_svs.yaml}"
|
||||
fi
|
||||
|
||||
echo "Starting SoulX-Singer server"
|
||||
echo " MODEL=$MODEL"
|
||||
echo " MODE=$MODE"
|
||||
echo " PORT=$PORT"
|
||||
echo " DEPLOY_CONFIG=$DEPLOY_CONFIG"
|
||||
echo " CUDA_VISIBLE_DEVICES=$GPUS"
|
||||
|
||||
CUDA_VISIBLE_DEVICES="$GPUS" \
|
||||
vllm serve "$MODEL" \
|
||||
--omni \
|
||||
--deploy-config "$DEPLOY_CONFIG" \
|
||||
--host 0.0.0.0 \
|
||||
--port "$PORT" \
|
||||
--trust-remote-code \
|
||||
--enforce-eager
|
||||
@@ -11,6 +11,11 @@ This example demonstrates how to deploy text-to-video models for online video ge
|
||||
| Wan2.2 T2V | `Wan-AI/Wan2.2-T2V-A14B-Diffusers` |
|
||||
| LTX-2 | `Lightricks/LTX-2` |
|
||||
| Helios (Base / Mid / Distilled) | `BestWishYsh/Helios-Base`, `Helios-Mid`, `Helios-Distilled` |
|
||||
| SANA-Video 2B (480p) | `Efficient-Large-Model/SANA-Video_2B_480p_diffusers` |
|
||||
| SANA-Video 2B (720p) | `Efficient-Large-Model/SANA-Video_2B_720p_diffusers` |
|
||||
|
||||
For SANA-Video 2B text-to-video online serving, see the
|
||||
[SANA-Video 2B recipe](../../../recipes/NVIDIA/SANA-Video-2B.md).
|
||||
|
||||
## Wan2.2 T2V
|
||||
|
||||
|
||||
@@ -0,0 +1,16 @@
|
||||
#!/bin/bash
|
||||
|
||||
BASE_URL="${BASE_URL:-http://localhost:8091}"
|
||||
OUTPUT_PATH="${OUTPUT_PATH:-sana_video.mp4}"
|
||||
|
||||
curl -sS -X POST "$BASE_URL/v1/videos/sync" \
|
||||
-F "prompt=A cinematic tracking shot of a sailboat crossing the ocean at sunset" \
|
||||
-F "height=480" \
|
||||
-F "width=832" \
|
||||
-F "num_frames=81" \
|
||||
-F "fps=16" \
|
||||
-F "num_inference_steps=50" \
|
||||
-F "guidance_scale=6.0" \
|
||||
-F "seed=42" \
|
||||
-F 'extra_params={"motion_score":30}' \
|
||||
--output "$OUTPUT_PATH"
|
||||
@@ -0,0 +1,11 @@
|
||||
#!/bin/bash
|
||||
# SANA-Video-2B text-to-video serving.
|
||||
|
||||
MODEL="${MODEL:-Efficient-Large-Model/SANA-Video_2B_480p_diffusers}"
|
||||
PORT="${PORT:-8091}"
|
||||
|
||||
vllm serve "$MODEL" \
|
||||
--omni \
|
||||
--model-class-name SanaVideoPipeline \
|
||||
--dtype bfloat16 \
|
||||
--port "$PORT"
|
||||
@@ -0,0 +1,12 @@
|
||||
#!/bin/bash
|
||||
# SANA-Video through the black-box Diffusers backend adapter.
|
||||
|
||||
MODEL="${MODEL:-Efficient-Large-Model/SANA-Video_2B_480p_diffusers}"
|
||||
PORT="${PORT:-8091}"
|
||||
|
||||
vllm serve "$MODEL" \
|
||||
--omni \
|
||||
--diffusion-load-format diffusers \
|
||||
--diffusion-attention-backend TORCH_SDPA \
|
||||
--dtype bfloat16 \
|
||||
--port "$PORT"
|
||||
Reference in new issue
Block a user