Merge branch 'main' into feat/moss-tts-nano
Signed-off-by: Yueqian Lin <70319226+linyueqian@users.noreply.github.com>
This commit is contained in:
611 files changed
+48775
-14783
No files matched your search
@@ -29,15 +29,12 @@ Features:
|
||||
## Launch the Server
|
||||
|
||||
```bash
|
||||
vllm-omni serve fishaudio/s2-pro \
|
||||
--stage-configs-path vllm_omni/model_executor/stage_configs/fish_speech_s2_pro.yaml \
|
||||
--omni \
|
||||
--port 8091 \
|
||||
--trust-remote-code \
|
||||
--enforce-eager \
|
||||
--gpu-memory-utilization 0.9
|
||||
vllm serve fishaudio/s2-pro --omni --port 8091
|
||||
```
|
||||
|
||||
The deploy config is auto-loaded from `vllm_omni/deploy/fish_qwen3_omni.yaml`
|
||||
(the HF `model_type` on the fishaudio checkpoints is `fish_qwen3_omni`).
|
||||
|
||||
Or use the convenience script:
|
||||
|
||||
```bash
|
||||
|
||||
@@ -11,18 +11,13 @@ MODEL="${MODEL:-fishaudio/s2-pro}"
|
||||
PORT="${PORT:-8091}"
|
||||
GRADIO_PORT="${GRADIO_PORT:-7860}"
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
REPO_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)"
|
||||
|
||||
echo "Starting Fish Speech S2 Pro server (port $PORT)..."
|
||||
FLASHINFER_DISABLE_VERSION_CHECK=1 \
|
||||
vllm-omni serve "$MODEL" \
|
||||
--stage-configs-path "$REPO_ROOT/vllm_omni/model_executor/stage_configs/fish_speech_s2_pro.yaml" \
|
||||
vllm serve "$MODEL" \
|
||||
--omni \
|
||||
--host 0.0.0.0 \
|
||||
--port "$PORT" \
|
||||
--gpu-memory-utilization 0.9 \
|
||||
--trust-remote-code \
|
||||
--enforce-eager \
|
||||
--omni &
|
||||
--port "$PORT" &
|
||||
SERVER_PID=$!
|
||||
|
||||
cleanup() {
|
||||
|
||||
@@ -13,11 +13,7 @@ PORT="${PORT:-8091}"
|
||||
echo "Starting Fish Speech S2 Pro server with model: $MODEL"
|
||||
|
||||
FLASHINFER_DISABLE_VERSION_CHECK=1 \
|
||||
vllm-omni serve "$MODEL" \
|
||||
--stage-configs-path vllm_omni/model_executor/stage_configs/fish_speech_s2_pro.yaml \
|
||||
vllm serve "$MODEL" \
|
||||
--omni \
|
||||
--host 0.0.0.0 \
|
||||
--port "$PORT" \
|
||||
--gpu-memory-utilization 0.9 \
|
||||
--trust-remote-code \
|
||||
--enforce-eager \
|
||||
--omni
|
||||
--port "$PORT"
|
||||
@@ -13,10 +13,10 @@ Please refer to [README.md](../../../README.md)
|
||||
```bash
|
||||
export MIMO_AUDIO_TOKENIZER_PATH="XiaomiMiMo/MiMo-Audio-Tokenizer"
|
||||
|
||||
vllm-omni serve XiaomiMiMo/MiMo-Audio-7B-Instruct --omni \
|
||||
--served-model-name "MiMo-Audio-7B-Instruct" \
|
||||
--port 18091 --stage-configs-path ./vllm_omni/model_executor/stage_configs/mimo_audio.yaml \
|
||||
--chat-template ./examples/online_serving/mimo_audio/chat_template.jinja
|
||||
vllm serve XiaomiMiMo/MiMo-Audio-7B-Instruct --omni \
|
||||
--served-model-name "MiMo-Audio-7B-Instruct" \
|
||||
--port 18091 \
|
||||
--chat-template ./examples/online_serving/mimo_audio/chat_template.jinja
|
||||
```
|
||||
> ⚠️ **Important**
|
||||
> **MiMo-Audio is not compatible with the default chat template.**
|
||||
|
||||
@@ -0,0 +1,204 @@
|
||||
# Ming-flash-omni 2.0
|
||||
|
||||
## Installation
|
||||
|
||||
Please refer to [README.md](../../../README.md)
|
||||
|
||||
## Run examples (Ming-flash-omni 2.0)
|
||||
|
||||
### Launch the Server
|
||||
|
||||
```bash
|
||||
vllm serve Jonathan1909/Ming-flash-omni-2.0 --omni --port 8091
|
||||
```
|
||||
|
||||
If you have custom stage configs file, launch the server with command below
|
||||
```bash
|
||||
vllm serve Jonathan1909/Ming-flash-omni-2.0 --omni --port 8091 --stage-configs-path /path/to/stage_configs_file
|
||||
```
|
||||
|
||||
### Send Multi-modal Request
|
||||
|
||||
#### Send request via python
|
||||
|
||||
```bash
|
||||
python examples/online_serving/openai_chat_completion_client_for_multimodal_generation.py --model Jonathan1909/Ming-flash-omni-2.0 --query-type use_mixed_modalities --port 8091 --host "localhost" --modalities text
|
||||
```
|
||||
|
||||
The Python client supports the following command-line arguments:
|
||||
|
||||
- `--query-type` (or `-q`): Query type. Options: `text`, `use_audio`, `use_image`, `use_video`, `use_mixed_modalities`
|
||||
- `--video-path` (or `-v`): Path to local video file or URL. If not provided and query-type uses video, uses default video URL. Supports local file paths (automatically encoded to base64) or HTTP/HTTPS URLs. Example: `--video-path /path/to/video.mp4` or `--video-path https://example.com/video.mp4`
|
||||
- `--image-path` (or `-i`): Path to local image file or URL. If not provided and query-type uses image, uses default image URL. Supports local file paths (automatically encoded to base64) or HTTP/HTTPS URLs and common image formats: JPEG, PNG, GIF, WebP. Example: `--image-path /path/to/image.jpg` or `--image-path https://example.com/image.png`
|
||||
- `--audio-path` (or `-a`): Path to local audio file or URL. If not provided and query-type uses audio, uses default audio URL. Supports local file paths (automatically encoded to base64) or HTTP/HTTPS URLs and common audio formats: MP3, WAV, OGG, FLAC, M4A. Example: `--audio-path /path/to/audio.wav` or `--audio-path https://example.com/audio.mp3`
|
||||
- `--prompt` (or `-p`): Custom text prompt/question. If not provided, uses default prompt for the selected query type. Example: `--prompt "What are the main activities shown in this video?"`
|
||||
- `--modalities`: Output modalities. For now, only `text` is supported. Example: `--modalities text`
|
||||
|
||||
|
||||
#### Send request via curl
|
||||
|
||||
```bash
|
||||
bash run_curl_multimodal_generation.sh text
|
||||
bash run_curl_multimodal_generation.sh use_image
|
||||
bash run_curl_multimodal_generation.sh use_audio
|
||||
bash run_curl_multimodal_generation.sh use_video
|
||||
bash run_curl_multimodal_generation.sh use_mixed_modalities
|
||||
```
|
||||
|
||||
## Modality control
|
||||
|
||||
Ming-flash-omni 2.0 currently supports text output only (thinker stage).
|
||||
|
||||
| Modalities | Output |
|
||||
|------------|--------|
|
||||
| `["text"]` | Text only |
|
||||
| Not specified | Text only (default) |
|
||||
|
||||
### Using curl
|
||||
|
||||
```bash
|
||||
curl http://localhost:8091/v1/chat/completions \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"model": "Jonathan1909/Ming-flash-omni-2.0",
|
||||
"messages": [
|
||||
{"role": "system", "content": [{"type": "text", "text": "你是一个友好的AI助手。\n\ndetailed thinking off"}]},
|
||||
{"role": "user", "content": "请详细介绍鹦鹉的生活习性。"}
|
||||
],
|
||||
"modalities": ["text"]
|
||||
}'
|
||||
```
|
||||
|
||||
### Using OpenAI Python SDK
|
||||
|
||||
```python
|
||||
from openai import OpenAI
|
||||
|
||||
client = OpenAI(base_url="http://localhost:8091/v1", api_key="EMPTY")
|
||||
|
||||
response = client.chat.completions.create(
|
||||
model="Jonathan1909/Ming-flash-omni-2.0",
|
||||
messages=[
|
||||
{"role": "system", "content": [{"type": "text", "text": "你是一个友好的AI助手。\n\ndetailed thinking off"}]},
|
||||
{"role": "user", "content": "请详细介绍鹦鹉的生活习性。"},
|
||||
],
|
||||
modalities=["text"],
|
||||
)
|
||||
print(response.choices[0].message.content)
|
||||
```
|
||||
|
||||
### Multi-modal input with OpenAI Python SDK
|
||||
|
||||
```python
|
||||
from openai import OpenAI
|
||||
|
||||
client = OpenAI(base_url="http://localhost:8091/v1", api_key="EMPTY")
|
||||
|
||||
response = client.chat.completions.create(
|
||||
model="Jonathan1909/Ming-flash-omni-2.0",
|
||||
messages=[
|
||||
{"role": "system", "content": [{"type": "text", "text": "你是一个友好的AI助手。\n\ndetailed thinking off"}]},
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "image_url", "image_url": {"url": "https://vllm-public-assets.s3.us-west-2.amazonaws.com/vision_model_images/cherry_blossom.jpg"}},
|
||||
{"type": "text", "text": "Describe this image in detail."},
|
||||
],
|
||||
},
|
||||
],
|
||||
modalities=["text"],
|
||||
)
|
||||
print(response.choices[0].message.content)
|
||||
```
|
||||
|
||||
## Streaming Output
|
||||
|
||||
To enable streaming output:
|
||||
|
||||
```bash
|
||||
python examples/online_serving/openai_chat_completion_client_for_multimodal_generation.py \
|
||||
--query-type use_image \
|
||||
--model Jonathan1909/Ming-flash-omni-2.0 \
|
||||
--modalities text \
|
||||
--stream
|
||||
```
|
||||
|
||||
Or with the OpenAI Python SDK:
|
||||
|
||||
```python
|
||||
from openai import OpenAI
|
||||
|
||||
client = OpenAI(base_url="http://localhost:8091/v1", api_key="EMPTY")
|
||||
|
||||
response = client.chat.completions.create(
|
||||
model="Jonathan1909/Ming-flash-omni-2.0",
|
||||
messages=[
|
||||
{"role": "system", "content": [{"type": "text", "text": "你是一个友好的AI助手。\n\ndetailed thinking off"}]},
|
||||
{"role": "user", "content": "请详细介绍鹦鹉的生活习性。"},
|
||||
],
|
||||
modalities=["text"],
|
||||
stream=True,
|
||||
)
|
||||
for chunk in response:
|
||||
for choice in chunk.choices:
|
||||
if hasattr(choice, "delta") and choice.delta.content:
|
||||
print(choice.delta.content, end="", flush=True)
|
||||
print()
|
||||
```
|
||||
|
||||
Or using curl:
|
||||
|
||||
```bash
|
||||
curl http://localhost:8091/v1/chat/completions \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"model": "Jonathan1909/Ming-flash-omni-2.0",
|
||||
"messages": [
|
||||
{"role": "system", "content": [{"type": "text", "text": "你是一个友好的AI助手。\n\ndetailed thinking off"}]},
|
||||
{"role": "user", "content": "请详细介绍鹦鹉的生活习性。"}
|
||||
],
|
||||
"modalities": ["text"],
|
||||
"stream": true,
|
||||
}'
|
||||
```
|
||||
|
||||
|
||||
## Reasoning (Thinking Mode)
|
||||
|
||||
To enable reasoning/thinking mode, change `detailed thinking off` to `detailed thinking on` in the system prompt:
|
||||
|
||||
### Using curl
|
||||
|
||||
```bash
|
||||
curl http://localhost:8091/v1/chat/completions \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"model": "Jonathan1909/Ming-flash-omni-2.0",
|
||||
"messages": [
|
||||
{"role": "system", "content": [{"type": "text", "text": "你是一个友好的AI助手。\n\ndetailed thinking on"}]},
|
||||
{"role": "user", "content": [
|
||||
{"type": "image_url", "image_url": {"url": "https://example.com/math_problem.png"}},
|
||||
{"type": "text", "text": "Solve this math problem step by step."}
|
||||
]}
|
||||
],
|
||||
"modalities": ["text"]
|
||||
}'
|
||||
```
|
||||
|
||||
### Using OpenAI Python SDK
|
||||
|
||||
```python
|
||||
from openai import OpenAI
|
||||
|
||||
client = OpenAI(base_url="http://localhost:8091/v1", api_key="EMPTY")
|
||||
|
||||
response = client.chat.completions.create(
|
||||
model="Jonathan1909/Ming-flash-omni-2.0",
|
||||
messages=[
|
||||
{"role": "system", "content": [{"type": "text", "text": "你是一个友好的AI助手。\n\ndetailed thinking on"}]},
|
||||
{"role": "user", "content": "If a train travels 120 km in 2 hours, what is its average speed?"},
|
||||
],
|
||||
modalities=["text"],
|
||||
)
|
||||
print(response.choices[0].message.content)
|
||||
```
|
||||
@@ -0,0 +1,145 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
# Server port
|
||||
PORT="${PORT:-8091}"
|
||||
# Default query type
|
||||
QUERY_TYPE="${1:-text}"
|
||||
|
||||
# Validate query type
|
||||
if [[ ! "$QUERY_TYPE" =~ ^(text|use_audio|use_image|use_video|use_mixed_modalities)$ ]]; then
|
||||
echo "Error: Invalid query type '$QUERY_TYPE'"
|
||||
echo "Usage: $0 [text|use_audio|use_image|use_video|use_mixed_modalities]"
|
||||
echo " text: Text-only query"
|
||||
echo " use_audio: Audio + Text query"
|
||||
echo " use_image: Image + Text query"
|
||||
echo " use_video: Video + Text query"
|
||||
echo " use_mixed_modalities: Audio + Image + Video + Text query"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
thinker_sampling_params='{
|
||||
"temperature": 0.4,
|
||||
"top_p": 0.9,
|
||||
"top_k": -1,
|
||||
"max_tokens": 16384,
|
||||
"seed": 42,
|
||||
"detokenize": true,
|
||||
"repetition_penalty": 1.05
|
||||
}'
|
||||
# Above is optional, it has a default setting in stage_configs of the corresponding model.
|
||||
|
||||
# Define URLs for assets
|
||||
MARY_HAD_LAMB_AUDIO_URL="https://vllm-public-assets.s3.us-west-2.amazonaws.com/multimodal_asset/mary_had_lamb.ogg"
|
||||
CHERRY_BLOSSOM_IMAGE_URL="https://vllm-public-assets.s3.us-west-2.amazonaws.com/vision_model_images/cherry_blossom.jpg"
|
||||
SAMPLE_VIDEO_URL="https://huggingface.co/datasets/raushan-testing-hf/videos-test/resolve/main/sample_demo_1.mp4"
|
||||
|
||||
# Build user content based on query type
|
||||
case "$QUERY_TYPE" in
|
||||
text)
|
||||
user_content='[
|
||||
{
|
||||
"type": "text",
|
||||
"text": "请详细介绍鹦鹉的生活习性。"
|
||||
}
|
||||
]'
|
||||
;;
|
||||
use_image)
|
||||
user_content='[
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {
|
||||
"url": "'"$CHERRY_BLOSSOM_IMAGE_URL"'"
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "text",
|
||||
"text": "Describe this image in detail."
|
||||
}
|
||||
]'
|
||||
;;
|
||||
use_audio)
|
||||
user_content='[
|
||||
{
|
||||
"type": "audio_url",
|
||||
"audio_url": {
|
||||
"url": "'"$MARY_HAD_LAMB_AUDIO_URL"'"
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "text",
|
||||
"text": "Please recognize the language of this speech and transcribe it. Format: oral."
|
||||
}
|
||||
]'
|
||||
;;
|
||||
use_video)
|
||||
user_content='[
|
||||
{
|
||||
"type": "video_url",
|
||||
"video_url": {
|
||||
"url": "'"$SAMPLE_VIDEO_URL"'"
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "text",
|
||||
"text": "Describe what is happening in this video."
|
||||
}
|
||||
]'
|
||||
;;
|
||||
use_mixed_modalities)
|
||||
user_content='[
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {
|
||||
"url": "'"$CHERRY_BLOSSOM_IMAGE_URL"'"
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "audio_url",
|
||||
"audio_url": {
|
||||
"url": "'"$MARY_HAD_LAMB_AUDIO_URL"'"
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "text",
|
||||
"text": "Describe the image, and recognize the language of this speech and transcribe it. Format: oral"
|
||||
}
|
||||
]'
|
||||
;;
|
||||
esac
|
||||
|
||||
echo "Running query type: $QUERY_TYPE"
|
||||
echo ""
|
||||
|
||||
request_body=$(cat <<EOF
|
||||
{
|
||||
"model": "Jonathan1909/Ming-flash-omni-2.0",
|
||||
"sampling_params_list": [
|
||||
$thinker_sampling_params
|
||||
],
|
||||
"modalities": ["text"],
|
||||
"messages": [
|
||||
{
|
||||
"role": "system",
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "你是一个友好的AI助手。\n\ndetailed thinking off"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": $user_content
|
||||
}
|
||||
]
|
||||
}
|
||||
EOF
|
||||
)
|
||||
|
||||
output=$(curl -sS --retry 3 --retry-delay 3 --retry-connrefused \
|
||||
-X POST http://localhost:${PORT}/v1/chat/completions \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "$request_body")
|
||||
|
||||
echo "Output of request: $(echo "$output" | jq '.choices[0].message.content')"
|
||||
@@ -12,17 +12,159 @@ Please refer to [README.md](../../../README.md)
|
||||
vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091
|
||||
```
|
||||
|
||||
If you want to open async chunking for qwen3-omni, launch the server with command below
|
||||
The default deploy config at `vllm_omni/deploy/qwen3_omni_moe.yaml` is loaded
|
||||
automatically by the model registry — no `--deploy-config` flag needed for the
|
||||
common case. Async-chunk streaming is **enabled by default** in the bundled config.
|
||||
NPU / ROCm / XPU per-platform deltas are merged in automatically from the
|
||||
`platforms:` section of the same YAML.
|
||||
|
||||
**Note:** The OpenAI-style **`/v1/realtime`** WebSocket (streaming PCM audio in, audio + transcription out) is **not supported** when `async_chunk` is enabled. Use the default omni layout or a stage config with `async_chunk: false` for realtime sessions.
|
||||
|
||||
If you have a custom deploy YAML, point at it explicitly:
|
||||
|
||||
```bash
|
||||
vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091 --stage-configs-path vllm_omni/model_executor/stage_configs/qwen3_omni_moe_async_chunk.yaml
|
||||
vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091 \
|
||||
--deploy-config /path/to/your_deploy_config.yaml
|
||||
```
|
||||
|
||||
If you have custom stage configs file, launch the server with command below
|
||||
### Tuning deployment parameters
|
||||
|
||||
Most engine knobs (`max_num_batched_tokens`, `max_model_len`, `enforce_eager`,
|
||||
`gpu_memory_utilization`, `tensor_parallel_size`, …) can be tuned without
|
||||
editing the YAML. There are three layers, in increasing specificity:
|
||||
|
||||
#### 1. Global CLI flags (apply to every stage)
|
||||
|
||||
```bash
|
||||
vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091 --stage-configs-path /path/to/stage_configs_file
|
||||
# Tighter memory budget on a smaller GPU
|
||||
vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091 \
|
||||
--gpu-memory-utilization 0.85
|
||||
|
||||
# Disable cudagraphs (e.g. for debugging)
|
||||
vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091 \
|
||||
--enforce-eager
|
||||
|
||||
# Reduce context length
|
||||
vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091 \
|
||||
--max-model-len 32768
|
||||
|
||||
# Toggle prefix caching on every stage (yaml default: off)
|
||||
vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091 \
|
||||
--enable-prefix-caching
|
||||
# ...or force it off if the yaml turned it on
|
||||
vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091 \
|
||||
--no-enable-prefix-caching
|
||||
|
||||
# Toggle pipeline-wide async chunked streaming between stages
|
||||
# (yaml default for qwen3_omni_moe: on)
|
||||
vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091 \
|
||||
--no-async-chunk
|
||||
```
|
||||
|
||||
For the TTS counterpart (synchronous codec variant), see
|
||||
[qwen3_tts README](../qwen3_tts/README.md#sync-vs-async-chunk-mode).
|
||||
|
||||
Explicit CLI flags **override** the deploy YAML (which itself overrides the
|
||||
parser defaults). If you don't pass a flag, the YAML value wins.
|
||||
|
||||
> **Note on `--no-async-chunk`**: Flips the deploy yaml's `async_chunk:`
|
||||
> bool. Pipelines that implement alternate processor functions for
|
||||
> chunked vs end-to-end modes (e.g. qwen3_tts code2wav) dispatch
|
||||
> automatically based on that bool — no extra flag or variant yaml is
|
||||
> needed.
|
||||
|
||||
> ⚠️ **For multi-stage models that share GPUs (qwen3_omni_moe by default
|
||||
> shares cuda:1 between stages 1 and 2), avoid using global memory flags.**
|
||||
> A global `--gpu-memory-utilization 0.85` would apply to every stage and
|
||||
> oversubscribe the shared device. Use per-stage overrides instead — see
|
||||
> below.
|
||||
|
||||
#### 2. Per-stage overrides via `--stage-overrides` (recommended for memory)
|
||||
|
||||
```bash
|
||||
# Lower stage 1's memory budget; leave others at the YAML default
|
||||
vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091 \
|
||||
--stage-overrides '{
|
||||
"1": {"gpu_memory_utilization": 0.5},
|
||||
"2": {"max_num_batched_tokens": 65536}
|
||||
}'
|
||||
```
|
||||
|
||||
Per-stage values are always treated as explicit and beat YAML defaults for
|
||||
the named stage. Other stages keep their YAML values.
|
||||
|
||||
#### 3. Custom deploy YAML
|
||||
|
||||
When per-stage overrides get long, write a small overlay YAML that inherits
|
||||
from the bundled default:
|
||||
|
||||
```yaml
|
||||
# my_qwen3_omni_overrides.yaml
|
||||
base_config: /path/to/vllm_omni/deploy/qwen3_omni_moe.yaml
|
||||
|
||||
stages:
|
||||
- stage_id: 0
|
||||
max_num_batched_tokens: 65536
|
||||
enforce_eager: true
|
||||
- stage_id: 1
|
||||
gpu_memory_utilization: 0.5
|
||||
- stage_id: 2
|
||||
max_model_len: 8192
|
||||
```
|
||||
|
||||
Then start the server with `--deploy-config my_qwen3_omni_overrides.yaml`.
|
||||
The `base_config:` line tells the loader to inherit everything else (stages,
|
||||
connectors, edges, platforms section) from the bundled production YAML, so
|
||||
you only need to spell out the deltas.
|
||||
|
||||
#### 4. Multi-node deployment (cross-host transfer connector)
|
||||
|
||||
The bundled `qwen3_omni_moe.yaml` uses `SharedMemoryConnector` between stages,
|
||||
which only works when all stages run on the same physical host. For
|
||||
**cross-node** deployments, write a small overlay YAML that swaps in a
|
||||
network-capable connector (e.g. `MooncakeStoreConnector`) and re-points each
|
||||
stage's connector wiring at it. The connector spec carries your own server
|
||||
addresses — there is no checked-in default because every cluster is
|
||||
different.
|
||||
|
||||
```yaml
|
||||
# my_qwen3_omni_multinode.yaml
|
||||
base_config: /path/to/vllm_omni/deploy/qwen3_omni_moe.yaml
|
||||
|
||||
connectors:
|
||||
mooncake_connector:
|
||||
name: MooncakeStoreConnector
|
||||
extra:
|
||||
host: "127.0.0.1"
|
||||
metadata_server: "http://YOUR_METADATA_HOST:8080/metadata"
|
||||
master: "YOUR_MASTER_HOST:50051"
|
||||
segment: 512000000 # 512 MB transfer segment
|
||||
localbuf: 64000000 # 64 MB local buffer
|
||||
proto: "tcp"
|
||||
|
||||
stages:
|
||||
- stage_id: 0
|
||||
output_connectors:
|
||||
to_stage_1: mooncake_connector
|
||||
- stage_id: 1
|
||||
input_connectors:
|
||||
from_stage_0: mooncake_connector
|
||||
output_connectors:
|
||||
to_stage_2: mooncake_connector
|
||||
- stage_id: 2
|
||||
input_connectors:
|
||||
from_stage_1: mooncake_connector
|
||||
```
|
||||
|
||||
Then launch with `--deploy-config my_qwen3_omni_multinode.yaml`. Same
|
||||
pattern works for Qwen2.5-Omni — replace `base_config:` with the path to
|
||||
`vllm_omni/deploy/qwen2_5_omni.yaml`.
|
||||
|
||||
> ⚠️ Replace `YOUR_METADATA_HOST` / `YOUR_MASTER_HOST` with the actual
|
||||
> mooncake server addresses for your cluster. The `base_config:` overlay
|
||||
> inherits all stage budgets, devices, and edges from the bundled prod
|
||||
> YAML — you only need to spell out the connector swap.
|
||||
|
||||
### Send Multi-modal Request
|
||||
|
||||
Get into the example folder
|
||||
@@ -38,36 +180,43 @@ python examples/online_serving/openai_chat_completion_client_for_multimodal_gene
|
||||
|
||||
#### Realtime WebSocket client (`openai_realtime_client.py`)
|
||||
|
||||
[`openai_realtime_client.py`](./openai_realtime_client.py) connects to **`ws://<host>:<port>/v1/realtime`**, uploads a local audio file as **PCM16 mono @ 16 kHz** chunks (OpenAI-style `input_audio_buffer.append` / `commit`), and prints **streaming transcription** (`transcription.delta` / `transcription.done`).
|
||||
[`openai_realtime_client.py`](./openai_realtime_client.py) connects to **`ws://<host>:<port>/v1/realtime`**, streams a local WAV as **PCM16 mono @ 16 kHz** in fixed-size chunks (OpenAI-style `input_audio_buffer.append` / `commit`), and receives **`response.audio.delta`** (incremental PCM for the reply) plus **`transcription.*`** events. By default it concatenates audio deltas and writes **`--output-wav`** (model output is typically **24 kHz**). Optional **`--delta-dump-dir`** saves each delta as `delta_000001.wav`, … for debugging.
|
||||
|
||||
Streaming input works well for translation-style use cases; if the Thinker runs while input is still incomplete, consider limiting **`max_tokens`** in your session / server defaults to avoid over-generation.
|
||||
|
||||
**Dependencies:**
|
||||
|
||||
```bash
|
||||
pip install websockets numpy
|
||||
pip install websockets
|
||||
```
|
||||
|
||||
**From this directory** (`examples/online_serving/qwen3_omni`):
|
||||
|
||||
```bash
|
||||
python openai_realtime_client.py \
|
||||
--host localhost \
|
||||
--port 8091 \
|
||||
--url ws://localhost:8091/v1/realtime \
|
||||
--model Qwen/Qwen3-Omni-30B-A3B-Instruct \
|
||||
--audio_path /path/to/your.wav
|
||||
--input-wav /path/to/input_16k_mono.wav \
|
||||
--output-wav realtime_output.wav \
|
||||
--delta-dump-dir ./rt_delta_wavs
|
||||
```
|
||||
|
||||
If `--audio_path` is omitted, the script uses a bundled default clip (`mary_had_lamb` via vLLM assets).
|
||||
|
||||
**Arguments:**
|
||||
|
||||
| Flag | Default | Description |
|
||||
|------|---------|-------------|
|
||||
| `--host` | `localhost` | API server host |
|
||||
| `--port` | `8000` | API server port (match your `vllm serve` port, e.g. `8091`) |
|
||||
| `--model` | `Qwen/Qwen3-Omni-30B-A3B-Instruct` | Must match the served model (also sent in `session.update`) |
|
||||
| `--audio_path` | *(optional)* | Path to input audio; resampled to 16 kHz mono inside the client |
|
||||
| `--url` | `ws://localhost:8091/v1/realtime` | Full WebSocket URL including path |
|
||||
| `--model` | `Qwen/Qwen3-Omni-30B-A3B-Instruct` | Must match the served model (sent in `session.update`) |
|
||||
| `--input-wav` | *(required)* | Input WAV: mono, 16-bit PCM, **16 kHz** |
|
||||
| `--output-wav` | `realtime_output.wav` | Output path for concatenated reply audio |
|
||||
| `--output-text` | *(optional)* | If set, write final transcription text to this path |
|
||||
| `--chunk-ms` | `200` | Size of each uploaded audio chunk (milliseconds of audio) |
|
||||
| `--send-delay-ms` | `0` | Delay between chunk sends (simulate realtime upload) |
|
||||
| `--delta-dump-dir` | *(optional)* | Directory to write per-`response.audio.delta` WAV files |
|
||||
| `--num-requests` | `1` | Number of sequential sessions (see `--concurrency`) |
|
||||
| `--concurrency` | `1` | Max concurrent WebSocket sessions when `--num-requests` > 1 |
|
||||
|
||||
Ensure the vLLM-Omni server is running with realtime support for this endpoint, for example:
|
||||
Ensure the server is running **without** `async_chunk` if you use `/v1/realtime`, for example:
|
||||
|
||||
```bash
|
||||
vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091
|
||||
@@ -276,7 +425,7 @@ The script supports the following arguments:
|
||||
- `--model`: Model name/path (default: Qwen/Qwen3-Omni-30B-A3B-Instruct)
|
||||
- `--server-port`: Port for vLLM server (default: 8091)
|
||||
- `--gradio-port`: Port for Gradio demo (default: 7861)
|
||||
- `--stage-configs-path`: Path to custom stage configs YAML file (optional)
|
||||
- `--deploy-config`: Path to custom deploy config YAML file (optional)
|
||||
- `--server-host`: Host for vLLM server (default: 0.0.0.0)
|
||||
- `--gradio-ip`: IP for Gradio demo (default: 127.0.0.1)
|
||||
- `--share`: Share Gradio demo publicly (creates a public link)
|
||||
@@ -291,7 +440,7 @@ vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091
|
||||
|
||||
If you have custom stage configs file:
|
||||
```bash
|
||||
vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091 --stage-configs-path /path/to/stage_configs_file
|
||||
vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091 --deploy-config /path/to/deploy_config_file
|
||||
```
|
||||
|
||||
**Step 2: Run the Gradio demo**
|
||||
|
||||
@@ -1,81 +1,118 @@
|
||||
"""Realtime client for vLLM-Omni /v1/realtime (audio + text events).
|
||||
|
||||
This client:
|
||||
1) Reads a local WAV file (must be mono, 16-bit PCM, 16kHz),
|
||||
2) Streams PCM16 chunks to /v1/realtime with OpenAI-style events,
|
||||
3) Receives response.audio.* and transcription.* events,
|
||||
4) Saves synthesized audio to an output WAV file and optional text file.
|
||||
|
||||
By default each ``response.audio.delta`` is treated as an **incremental PCM**
|
||||
chunk and all chunks are concatenated into the final ``--output-wav``.
|
||||
|
||||
Optional debugging: pass ``--delta-dump-dir DIR`` to write every
|
||||
``response.audio.delta`` payload as ``delta_000001.wav``, ``delta_000002.wav``, …
|
||||
|
||||
Usage:
|
||||
python openai_realtime_client.py \
|
||||
--url ws://localhost:8091/v1/realtime \
|
||||
--model Qwen/Qwen3-Omni-30B-A3B-Instruct \
|
||||
--input-wav input_16k_mono.wav \
|
||||
--output-wav realtime_output.wav \
|
||||
--delta-dump-dir ./rt_delta_wavs
|
||||
|
||||
Dependencies:
|
||||
pip install websockets
|
||||
"""
|
||||
This script demonstrates how to use the vLLM-Omni Realtime WebSocket API to perform
|
||||
audio transcription by uploading an audio file.
|
||||
|
||||
Before running this script, you must start the vLLM-Omni server with a realtime-capable
|
||||
model, for example:
|
||||
|
||||
vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni
|
||||
|
||||
Requirements:
|
||||
- vllm with audio support
|
||||
- websockets
|
||||
- soundfile
|
||||
- numpy
|
||||
|
||||
The script:
|
||||
1. Connects to the Realtime WebSocket endpoint
|
||||
2. Converts an audio file to PCM16 @ 16kHz
|
||||
3. Sends audio chunks to the server
|
||||
4. Receives and prints transcription as it streams
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import base64
|
||||
import json
|
||||
import wave
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import websockets
|
||||
from vllm.assets.audio import AudioAsset
|
||||
from vllm.multimodal.media.audio import load_audio
|
||||
try:
|
||||
import websockets
|
||||
except ImportError:
|
||||
print("Please install websockets: pip install websockets")
|
||||
raise SystemExit(1)
|
||||
|
||||
|
||||
def audio_to_pcm16_base64(audio_path: str) -> str:
|
||||
"""
|
||||
Load an audio file and convert it to base64-encoded PCM16 @ 16kHz.
|
||||
"""
|
||||
# Load audio and resample to 16kHz mono
|
||||
audio, _ = load_audio(audio_path, sr=16000, mono=True)
|
||||
# Convert to PCM16
|
||||
pcm16 = (audio * 32767).astype(np.int16)
|
||||
# Encode as base64
|
||||
return base64.b64encode(pcm16.tobytes()).decode("utf-8")
|
||||
def _read_wav_pcm16(path: Path) -> bytes:
|
||||
with wave.open(str(path), "rb") as wf:
|
||||
nchannels = wf.getnchannels()
|
||||
sampwidth = wf.getsampwidth()
|
||||
framerate = wf.getframerate()
|
||||
comptype = wf.getcomptype()
|
||||
nframes = wf.getnframes()
|
||||
|
||||
if nchannels != 1:
|
||||
raise ValueError(f"Input WAV must be mono (got {nchannels} channels).")
|
||||
if sampwidth != 2:
|
||||
raise ValueError(f"Input WAV must be 16-bit PCM (got sample width={sampwidth}).")
|
||||
if framerate != 16000:
|
||||
raise ValueError(f"Input WAV must be 16kHz (got {framerate} Hz).")
|
||||
if comptype != "NONE":
|
||||
raise ValueError(f"Input WAV must be uncompressed PCM (got comptype={comptype}).")
|
||||
if nframes <= 0:
|
||||
raise ValueError("Input WAV has no audio frames.")
|
||||
|
||||
return wf.readframes(nframes)
|
||||
|
||||
|
||||
async def realtime_transcribe(audio_path: str, host: str, port: int, model: str):
|
||||
"""
|
||||
Connect to the Realtime API and transcribe an audio file.
|
||||
"""
|
||||
uri = f"ws://{host}:{port}/v1/realtime"
|
||||
def _write_wav_pcm16(path: Path, pcm16_bytes: bytes, sample_rate_hz: int) -> None:
|
||||
with wave.open(str(path), "wb") as wf:
|
||||
wf.setnchannels(1)
|
||||
wf.setsampwidth(2)
|
||||
wf.setframerate(sample_rate_hz)
|
||||
wf.writeframes(pcm16_bytes)
|
||||
|
||||
async with websockets.connect(uri) as ws:
|
||||
# Wait for session.created
|
||||
response = json.loads(await ws.recv())
|
||||
if response["type"] == "session.created":
|
||||
print(f"Session created: {response['id']}")
|
||||
else:
|
||||
print(f"Unexpected response: {response}")
|
||||
return
|
||||
|
||||
# Validate model
|
||||
await ws.send(json.dumps({"type": "session.update", "model": model}))
|
||||
async def run_client(
|
||||
url: str,
|
||||
model: str,
|
||||
input_wav: Path,
|
||||
output_wav: Path,
|
||||
output_text: Path | None,
|
||||
chunk_ms: int,
|
||||
send_delay_ms: int,
|
||||
delta_dump_dir: Path | None,
|
||||
request_idx: int = 1,
|
||||
total_requests: int = 1,
|
||||
) -> None:
|
||||
log_prefix = f"[req {request_idx:02d}/{total_requests:02d}] " if total_requests > 1 else ""
|
||||
pcm16 = _read_wav_pcm16(input_wav)
|
||||
bytes_per_ms = 16000 * 2 // 1000 # mono PCM16 at 16kHz
|
||||
chunk_bytes = max(bytes_per_ms * chunk_ms, 2)
|
||||
|
||||
# Signal ready to start
|
||||
await ws.send(json.dumps({"type": "input_audio_buffer.commit"}))
|
||||
incremental_pcm_parts: list[bytes] = []
|
||||
output_sample_rate = 24000
|
||||
delta_index = 0
|
||||
text_chunks: list[str] = []
|
||||
final_text: str = ""
|
||||
|
||||
# Convert audio file to base64 PCM16
|
||||
print(f"Loading audio from: {audio_path}")
|
||||
audio_base64 = audio_to_pcm16_base64(audio_path)
|
||||
if delta_dump_dir is not None:
|
||||
delta_dump_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Send audio in chunks (4KB of raw audio = ~8KB base64)
|
||||
chunk_size = 4096
|
||||
audio_bytes = base64.b64decode(audio_base64)
|
||||
total_chunks = (len(audio_bytes) + chunk_size - 1) // chunk_size
|
||||
async with websockets.connect(url, max_size=64 * 1024 * 1024) as ws:
|
||||
# 1) Validate model.
|
||||
await ws.send(
|
||||
json.dumps(
|
||||
{
|
||||
"type": "session.update",
|
||||
"model": model,
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
print(f"Sending {total_chunks} audio chunks...")
|
||||
for i in range(0, len(audio_bytes), chunk_size):
|
||||
chunk = audio_bytes[i : i + chunk_size]
|
||||
# 2) Start generation once (non-final commit).
|
||||
await ws.send(json.dumps({"type": "input_audio_buffer.commit", "final": False}))
|
||||
|
||||
# 3) Stream audio chunks.
|
||||
for i in range(0, len(pcm16), chunk_bytes):
|
||||
chunk = pcm16[i : i + chunk_bytes]
|
||||
await ws.send(
|
||||
json.dumps(
|
||||
{
|
||||
@@ -84,63 +121,212 @@ async def realtime_transcribe(audio_path: str, host: str, port: int, model: str)
|
||||
}
|
||||
)
|
||||
)
|
||||
if send_delay_ms > 0:
|
||||
await asyncio.sleep(send_delay_ms / 1000.0)
|
||||
|
||||
# Signal all audio is sent
|
||||
# 4) Final commit closes input stream.
|
||||
await ws.send(json.dumps({"type": "input_audio_buffer.commit", "final": True}))
|
||||
print("Audio sent. Waiting for transcription...\n")
|
||||
|
||||
# Receive transcription
|
||||
print("Transcription: ", end="", flush=True)
|
||||
# 5) Receive server events until audio done.
|
||||
while True:
|
||||
response = json.loads(await ws.recv())
|
||||
if response["type"] == "transcription.delta":
|
||||
print(response["delta"], end="", flush=True)
|
||||
elif response["type"] == "transcription.done":
|
||||
print(f"\n\nFinal transcription: {response['text']}")
|
||||
if response.get("usage"):
|
||||
print(f"Usage: {response['usage']}")
|
||||
break
|
||||
elif response["type"] == "error":
|
||||
print(f"\nError: {response['error']}")
|
||||
message = await ws.recv()
|
||||
if isinstance(message, bytes):
|
||||
# We only expect JSON text frames.
|
||||
continue
|
||||
|
||||
event = json.loads(message)
|
||||
event_type = event.get("type")
|
||||
|
||||
if event_type == "session.created":
|
||||
continue
|
||||
|
||||
if event_type == "response.audio.delta":
|
||||
sr = event.get("sample_rate_hz")
|
||||
if isinstance(sr, int) and sr > 0:
|
||||
output_sample_rate = sr
|
||||
audio_b64 = event.get("audio", "")
|
||||
if audio_b64:
|
||||
pcm_delta = base64.b64decode(audio_b64)
|
||||
incremental_pcm_parts.append(pcm_delta)
|
||||
if delta_dump_dir is not None and pcm_delta:
|
||||
delta_index += 1
|
||||
dump_path = delta_dump_dir / f"delta_{delta_index:06d}.wav"
|
||||
_write_wav_pcm16(dump_path, pcm_delta, output_sample_rate)
|
||||
print(
|
||||
f"{log_prefix}delta dump #{delta_index}: {dump_path} "
|
||||
f"(pcm bytes={len(pcm_delta)}, sr={output_sample_rate})"
|
||||
)
|
||||
continue
|
||||
|
||||
if event_type == "transcription.delta":
|
||||
delta = event.get("delta", "")
|
||||
if delta:
|
||||
text_chunks.append(delta)
|
||||
print(delta, end="", flush=True)
|
||||
continue
|
||||
|
||||
if event_type == "transcription.done":
|
||||
final_text = event.get("text", "") or "".join(text_chunks)
|
||||
usage = event.get("usage")
|
||||
final_text_with_tag = f"Final transcription: {final_text}"
|
||||
if text_chunks:
|
||||
print()
|
||||
print(f"{log_prefix}{final_text_with_tag}")
|
||||
if usage:
|
||||
print(f"{log_prefix}text usage: {usage}")
|
||||
continue
|
||||
|
||||
if event_type == "response.audio.done":
|
||||
break
|
||||
|
||||
if event_type == "error":
|
||||
raise RuntimeError(f"Server error: {event}")
|
||||
|
||||
def main(args):
|
||||
if args.audio_path:
|
||||
audio_path = args.audio_path
|
||||
all_pcm16 = b"".join(incremental_pcm_parts)
|
||||
if not all_pcm16:
|
||||
raise RuntimeError("No audio received from server.")
|
||||
|
||||
output_wav.parent.mkdir(parents=True, exist_ok=True)
|
||||
_write_wav_pcm16(output_wav, all_pcm16, output_sample_rate)
|
||||
print(f"{log_prefix}Saved realtime audio to: {output_wav} (incremental chunks joined)")
|
||||
|
||||
if output_text is not None:
|
||||
text_to_save = final_text if final_text else "".join(text_chunks)
|
||||
output_text.parent.mkdir(parents=True, exist_ok=True)
|
||||
output_text.write_text(text_to_save, encoding="utf-8")
|
||||
print(f"{log_prefix}Saved realtime text to: {output_text}")
|
||||
|
||||
|
||||
def _indexed_output_path(path: Path | None, index: int, total: int) -> Path | None:
|
||||
if path is None or total <= 1:
|
||||
return path
|
||||
return path.with_name(f"{path.stem}_{index:02d}{path.suffix}")
|
||||
|
||||
|
||||
async def run_clients_concurrent(
|
||||
*,
|
||||
url: str,
|
||||
model: str,
|
||||
input_wav: Path,
|
||||
output_wav: Path,
|
||||
output_text: Path | None,
|
||||
chunk_ms: int,
|
||||
send_delay_ms: int,
|
||||
delta_dump_dir: Path | None,
|
||||
num_requests: int,
|
||||
concurrency: int,
|
||||
) -> None:
|
||||
sem = asyncio.Semaphore(concurrency)
|
||||
|
||||
async def _run_one(index: int) -> tuple[int, bool, str | None]:
|
||||
per_output_wav = _indexed_output_path(output_wav, index, num_requests)
|
||||
per_output_text = _indexed_output_path(output_text, index, num_requests)
|
||||
per_delta_dir = None
|
||||
if delta_dump_dir is not None:
|
||||
per_delta_dir = delta_dump_dir / f"req_{index:02d}"
|
||||
async with sem:
|
||||
try:
|
||||
await run_client(
|
||||
url=url,
|
||||
model=model,
|
||||
input_wav=input_wav,
|
||||
output_wav=per_output_wav,
|
||||
output_text=per_output_text,
|
||||
chunk_ms=chunk_ms,
|
||||
send_delay_ms=send_delay_ms,
|
||||
delta_dump_dir=per_delta_dir,
|
||||
request_idx=index,
|
||||
total_requests=num_requests,
|
||||
)
|
||||
return index, True, None
|
||||
except Exception as exc:
|
||||
return index, False, str(exc)
|
||||
|
||||
tasks = [asyncio.create_task(_run_one(i), name=f"rt-client-{i}") for i in range(1, num_requests + 1)]
|
||||
results = await asyncio.gather(*tasks)
|
||||
|
||||
failed = [(idx, err) for idx, ok, err in results if not ok]
|
||||
succeeded = num_requests - len(failed)
|
||||
print(f"[summary] succeeded={succeeded}, failed={len(failed)}, total={num_requests}")
|
||||
if failed:
|
||||
for idx, err in failed:
|
||||
print(f"[summary] req {idx:02d} failed: {err}")
|
||||
raise RuntimeError(f"{len(failed)} concurrent request(s) failed")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Realtime audio/text client for vLLM-Omni")
|
||||
parser.add_argument("--url", default="ws://localhost:8091/v1/realtime", help="WebSocket URL")
|
||||
parser.add_argument(
|
||||
"--model",
|
||||
default="Qwen/Qwen3-Omni-30B-A3B-Instruct",
|
||||
help="Model name for session.update",
|
||||
)
|
||||
parser.add_argument("--input-wav", required=True, type=Path, help="Input WAV (mono, PCM16, 16kHz)")
|
||||
parser.add_argument("--output-wav", default=Path("realtime_output.wav"), type=Path, help="Output WAV path")
|
||||
parser.add_argument(
|
||||
"--output-text",
|
||||
default=None,
|
||||
type=Path,
|
||||
help="Optional output text path for final transcription",
|
||||
)
|
||||
parser.add_argument("--chunk-ms", type=int, default=200, help="Input chunk size in milliseconds")
|
||||
parser.add_argument(
|
||||
"--send-delay-ms",
|
||||
type=int,
|
||||
default=0,
|
||||
help="Delay between chunk sends; set >0 to simulate realtime upload",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--delta-dump-dir",
|
||||
type=Path,
|
||||
default=None,
|
||||
help="If set, each response.audio.delta is saved as delta_NNNNNN.wav under this directory",
|
||||
)
|
||||
parser.add_argument("--num-requests", type=int, default=1, help="Total number of requests to send")
|
||||
parser.add_argument(
|
||||
"--concurrency",
|
||||
type=int,
|
||||
default=1,
|
||||
help="Maximum number of concurrent websocket requests",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.num_requests <= 0:
|
||||
raise ValueError("--num-requests must be >= 1")
|
||||
if args.concurrency <= 0:
|
||||
raise ValueError("--concurrency must be >= 1")
|
||||
concurrency = min(args.concurrency, args.num_requests)
|
||||
|
||||
if args.num_requests == 1:
|
||||
asyncio.run(
|
||||
run_client(
|
||||
url=args.url,
|
||||
model=args.model,
|
||||
input_wav=args.input_wav,
|
||||
output_wav=args.output_wav,
|
||||
output_text=args.output_text,
|
||||
chunk_ms=args.chunk_ms,
|
||||
send_delay_ms=args.send_delay_ms,
|
||||
delta_dump_dir=args.delta_dump_dir,
|
||||
)
|
||||
)
|
||||
else:
|
||||
# Use default audio asset
|
||||
audio_path = str(AudioAsset("mary_had_lamb").get_local_path())
|
||||
print(f"No audio path provided, using default: {audio_path}")
|
||||
|
||||
asyncio.run(realtime_transcribe(audio_path, args.host, args.port, args.model))
|
||||
asyncio.run(
|
||||
run_clients_concurrent(
|
||||
url=args.url,
|
||||
model=args.model,
|
||||
input_wav=args.input_wav,
|
||||
output_wav=args.output_wav,
|
||||
output_text=args.output_text,
|
||||
chunk_ms=args.chunk_ms,
|
||||
send_delay_ms=args.send_delay_ms,
|
||||
delta_dump_dir=args.delta_dump_dir,
|
||||
num_requests=args.num_requests,
|
||||
concurrency=concurrency,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(description="Realtime WebSocket Transcription Client")
|
||||
parser.add_argument(
|
||||
"--model",
|
||||
type=str,
|
||||
default="Qwen/Qwen3-Omni-30B-A3B-Instruct",
|
||||
help="Model that is served and should be pinged.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--audio_path",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Path to the audio file to transcribe.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--host",
|
||||
type=str,
|
||||
default="localhost",
|
||||
help="vLLM-Omni server host (default: localhost)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--port",
|
||||
type=int,
|
||||
default=8000,
|
||||
help="vLLM-Omni server port (default: 8000)",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
main(args)
|
||||
main()
|
||||
@@ -43,7 +43,7 @@ Then open http://localhost:7860 in your browser.
|
||||
|
||||
### Launch the Server
|
||||
|
||||
The default stage config is located at `vllm_omni/model_executor/stage_configs/qwen3_tts.yaml`. For other platforms (e.g., NPU), refer to `vllm_omni/platforms/npu/stage_configs/qwen3_tts.yaml`.
|
||||
The default deploy config is located at `vllm_omni/deploy/qwen3_tts.yaml` and is loaded automatically by the model registry — no `--deploy-config` flag needed for default use. Platform-specific deltas (NPU, ROCm, XPU) are merged in automatically from the `platforms:` block of the same YAML based on the detected runtime.
|
||||
|
||||
```bash
|
||||
# CustomVoice model (predefined speakers)
|
||||
@@ -70,6 +70,22 @@ vllm serve Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice \
|
||||
--port 8091
|
||||
```
|
||||
|
||||
#### Sync vs async-chunk mode
|
||||
|
||||
Qwen3-TTS supports both **chunked streaming** (default, lower latency) and
|
||||
**synchronous end-to-end** modes from the same deploy YAML. The bundled
|
||||
`qwen3_tts.yaml` ships with `async_chunk: true`; flip with `--no-async-chunk`
|
||||
and the pipeline automatically dispatches to the end-to-end codec processor:
|
||||
|
||||
```bash
|
||||
vllm serve Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice --omni --port 8091 \
|
||||
--no-async-chunk
|
||||
```
|
||||
|
||||
No variant YAML or extra flag is needed — the `StagePipelineConfig` on each
|
||||
stage declares both processor functions and the runtime picks based on the
|
||||
`async_chunk:` bool.
|
||||
|
||||
Alternatively, use the convenience script:
|
||||
```bash
|
||||
./run_server.sh # Default: CustomVoice model
|
||||
|
||||
@@ -5,11 +5,13 @@ A particularly useful scenario is voice cloning: set ref_audio once at the
|
||||
batch level and generate many utterances in the cloned voice without repeating
|
||||
the reference for each item.
|
||||
|
||||
Start the server (with batch-optimized config for best throughput):
|
||||
Start the server (with batch-optimized stage settings for best throughput):
|
||||
|
||||
vllm serve Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice \
|
||||
--stage-configs-path vllm_omni/model_executor/stage_configs/qwen3_tts_batch.yaml \
|
||||
--trust-remote-code
|
||||
--omni \
|
||||
--trust-remote-code \
|
||||
--stage-overrides '{"0":{"max_num_seqs":4,"gpu_memory_utilization":0.2},
|
||||
"1":{"max_num_seqs":4,"gpu_memory_utilization":0.2}}'
|
||||
|
||||
Examples:
|
||||
# Batch with a predefined voice
|
||||
|
||||
@@ -127,7 +127,7 @@ echo "Starting vLLM server..."
|
||||
LOG_FILE="/tmp/vllm_tts_server_${SERVER_PORT}.log"
|
||||
|
||||
vllm-omni serve "$MODEL" \
|
||||
--stage-configs-path vllm_omni/model_executor/stage_configs/qwen3_tts.yaml \
|
||||
--deploy-config vllm_omni/deploy/qwen3_tts.yaml \
|
||||
--host "$SERVER_HOST" \
|
||||
--port "$SERVER_PORT" \
|
||||
--gpu-memory-utilization 0.9 \
|
||||
|
||||
@@ -31,7 +31,7 @@ esac
|
||||
echo "Starting Qwen3-TTS server with model: $MODEL"
|
||||
|
||||
vllm-omni serve "$MODEL" \
|
||||
--stage-configs-path vllm_omni/model_executor/stage_configs/qwen3_tts.yaml \
|
||||
--deploy-config vllm_omni/deploy/qwen3_tts.yaml \
|
||||
--host 0.0.0.0 \
|
||||
--port 8091 \
|
||||
--gpu-memory-utilization 0.9 \
|
||||
|
||||
@@ -5,7 +5,7 @@ speaker encoder from a Qwen3-TTS checkpoint, then interpolates between them
|
||||
using SLERP and sends the result to the /v1/audio/speech API.
|
||||
|
||||
Requirements:
|
||||
pip install torch resampy soundfile numpy httpx
|
||||
pip install torch soundfile numpy httpx
|
||||
|
||||
Examples:
|
||||
# Extract and save an embedding
|
||||
@@ -143,11 +143,12 @@ def _load_speaker_encoder_weights(encoder: torch.nn.Module, model_path: str) ->
|
||||
|
||||
def compute_mel_spectrogram(audio: np.ndarray, sr: int = 24000) -> torch.Tensor:
|
||||
"""Compute 128-bin mel spectrogram matching Qwen3-TTS's extraction pipeline."""
|
||||
from vllm.multimodal.audio import resample_audio_resampy
|
||||
from vllm.multimodal.audio import AudioResampler
|
||||
|
||||
# Resample to 24kHz if needed
|
||||
if sr != 24000:
|
||||
audio = resample_audio_resampy(audio.astype(np.float32), orig_sr=sr, target_sr=24000)
|
||||
resampler = AudioResampler(target_sr=24000)
|
||||
audio = resampler.resample(audio.astype(np.float32), orig_sr=sr)
|
||||
|
||||
y = torch.from_numpy(audio).unsqueeze(0).float()
|
||||
|
||||
|
||||
@@ -0,0 +1,166 @@
|
||||
# VoxCPM
|
||||
|
||||
## Prerequisites
|
||||
|
||||
Install VoxCPM in one of these ways:
|
||||
|
||||
```bash
|
||||
pip install voxcpm
|
||||
```
|
||||
|
||||
or point vLLM-Omni to a local VoxCPM source tree:
|
||||
|
||||
```bash
|
||||
export VLLM_OMNI_VOXCPM_CODE_PATH=/path/to/VoxCPM/src
|
||||
```
|
||||
|
||||
If the native VoxCPM `config.json` lacks HF metadata such as `model_type`,
|
||||
prepare a persistent HF-compatible config directory and export:
|
||||
|
||||
```bash
|
||||
export VLLM_OMNI_VOXCPM_HF_CONFIG_PATH=/tmp/voxcpm_hf_config
|
||||
mkdir -p "$VLLM_OMNI_VOXCPM_HF_CONFIG_PATH"
|
||||
cp "$VOXCPM_MODEL/config.json" "$VLLM_OMNI_VOXCPM_HF_CONFIG_PATH/config.json"
|
||||
cp "$VOXCPM_MODEL/generation_config.json" "$VLLM_OMNI_VOXCPM_HF_CONFIG_PATH/generation_config.json" 2>/dev/null || true
|
||||
python3 -c 'import json, os; p=os.path.join(os.environ["VLLM_OMNI_VOXCPM_HF_CONFIG_PATH"], "config.json"); cfg=json.load(open(p, "r", encoding="utf-8")); cfg["model_type"]="voxcpm"; cfg.setdefault("architectures", ["VoxCPMForConditionalGeneration"]); json.dump(cfg, open(p, "w", encoding="utf-8"), indent=2, ensure_ascii=False)'
|
||||
```
|
||||
|
||||
The VoxCPM stage configs read `VLLM_OMNI_VOXCPM_HF_CONFIG_PATH` directly. The `python3 -c` form above avoids heredoc/indentation issues in interactive shells.
|
||||
|
||||
## Launch the Server
|
||||
|
||||
Use the async-chunk stage config by default:
|
||||
|
||||
```bash
|
||||
export VOXCPM_MODEL=/path/to/voxcpm-model
|
||||
cd examples/online_serving/voxcpm
|
||||
./run_server.sh
|
||||
```
|
||||
|
||||
Use the non-streaming stage config:
|
||||
|
||||
```bash
|
||||
./run_server.sh sync
|
||||
```
|
||||
|
||||
You can also launch the server directly:
|
||||
|
||||
```bash
|
||||
vllm serve "$VOXCPM_MODEL" \
|
||||
--stage-configs-path vllm_omni/model_executor/stage_configs/voxcpm_async_chunk.yaml \
|
||||
--trust-remote-code \
|
||||
--enforce-eager \
|
||||
--omni \
|
||||
--port 8091
|
||||
```
|
||||
|
||||
## Send Requests
|
||||
|
||||
### Basic text-to-speech
|
||||
|
||||
```bash
|
||||
python openai_speech_client.py \
|
||||
--model "$VOXCPM_MODEL" \
|
||||
--text "This is a VoxCPM online text-to-speech example."
|
||||
```
|
||||
|
||||
### Voice cloning
|
||||
|
||||
```bash
|
||||
python openai_speech_client.py \
|
||||
--model "$VOXCPM_MODEL" \
|
||||
--text "This sentence is synthesized with a cloned voice." \
|
||||
--ref-audio /path/to/reference.wav \
|
||||
--ref-text "The exact transcript spoken in reference.wav."
|
||||
```
|
||||
|
||||
`ref_text` must be the real transcript of the reference audio. Placeholder text or mismatched text will usually degrade quality badly.
|
||||
|
||||
### Streaming PCM output
|
||||
|
||||
```bash
|
||||
python openai_speech_client.py \
|
||||
--model "$VOXCPM_MODEL" \
|
||||
--text "This is a streaming VoxCPM request." \
|
||||
--stream \
|
||||
--output voxcpm_stream.pcm
|
||||
```
|
||||
|
||||
### Using curl
|
||||
|
||||
```bash
|
||||
curl -X POST http://localhost:8091/v1/audio/speech \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"model": "OpenBMB/VoxCPM1.5",
|
||||
"input": "Hello from VoxCPM online serving.",
|
||||
"response_format": "wav"
|
||||
}' --output output.wav
|
||||
```
|
||||
|
||||
Voice cloning:
|
||||
|
||||
```bash
|
||||
curl -X POST http://localhost:8091/v1/audio/speech \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"model": "OpenBMB/VoxCPM1.5",
|
||||
"input": "This sentence uses a cloned voice.",
|
||||
"ref_audio": "https://example.com/reference.wav",
|
||||
"ref_text": "The exact transcript spoken in the reference audio.",
|
||||
"response_format": "wav"
|
||||
}' --output cloned.wav
|
||||
```
|
||||
|
||||
Streaming PCM:
|
||||
|
||||
```bash
|
||||
curl -X POST http://localhost:8091/v1/audio/speech \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"model": "OpenBMB/VoxCPM1.5",
|
||||
"input": "This is a streaming VoxCPM request.",
|
||||
"stream": true,
|
||||
"response_format": "pcm"
|
||||
}' --output output.pcm
|
||||
```
|
||||
|
||||
## Supported Request Shape
|
||||
|
||||
VoxCPM online serving currently supports:
|
||||
|
||||
- plain text-to-speech
|
||||
- voice cloning with `ref_audio` + `ref_text`
|
||||
- `stream=true` with `response_format=pcm` or `wav`
|
||||
|
||||
VoxCPM online serving does not use these generic TTS fields:
|
||||
|
||||
- `voice`
|
||||
- `instructions`
|
||||
- `language`
|
||||
- `speaker_embedding`
|
||||
- `x_vector_only_mode`
|
||||
|
||||
## Streaming vs Non-Streaming
|
||||
|
||||
- `voxcpm_async_chunk.yaml` enables async-chunk streaming and is best for single-request streaming latency.
|
||||
- `voxcpm.yaml` performs one-shot latent generation then VAE decode.
|
||||
|
||||
Like native VoxCPM, the async streaming path should be treated as single-request. If you need stable throughput benchmarking, prefer `voxcpm.yaml`.
|
||||
|
||||
Do not use `voxcpm_async_chunk.yaml` for concurrent online streaming or `/v1/audio/speech/batch`. For multiple requests, prefer `voxcpm.yaml`.
|
||||
|
||||
## Benchmark
|
||||
|
||||
The serving benchmark reports TTFP and RTF:
|
||||
|
||||
```bash
|
||||
python benchmarks/voxcpm/vllm_omni/bench_tts_serve.py \
|
||||
--host 127.0.0.1 \
|
||||
--port 8091 \
|
||||
--num-prompts 10 \
|
||||
--max-concurrency 1 \
|
||||
--result-dir /tmp/voxcpm_bench
|
||||
```
|
||||
|
||||
For the async-chunk server, keep `--max-concurrency 1`.
|
||||
@@ -0,0 +1,155 @@
|
||||
"""OpenAI-compatible client for VoxCPM via /v1/audio/speech.
|
||||
|
||||
Examples:
|
||||
# Basic text-to-speech
|
||||
python openai_speech_client.py --text "Hello from VoxCPM"
|
||||
|
||||
# Voice cloning
|
||||
python openai_speech_client.py \
|
||||
--text "This sentence uses the cloned voice." \
|
||||
--ref-audio /path/to/reference.wav \
|
||||
--ref-text "The exact transcript spoken in the reference audio."
|
||||
|
||||
# Streaming PCM output
|
||||
python openai_speech_client.py \
|
||||
--text "This is a streaming VoxCPM request." \
|
||||
--stream \
|
||||
--output output.pcm
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import base64
|
||||
import os
|
||||
|
||||
import httpx
|
||||
|
||||
DEFAULT_API_BASE = "http://localhost:8091"
|
||||
DEFAULT_API_KEY = "EMPTY"
|
||||
DEFAULT_MODEL = "OpenBMB/VoxCPM1.5"
|
||||
|
||||
|
||||
def encode_audio_to_base64(audio_path: str) -> str:
|
||||
"""Encode a local audio file to base64 data URL."""
|
||||
if not os.path.exists(audio_path):
|
||||
raise FileNotFoundError(f"Audio file not found: {audio_path}")
|
||||
|
||||
ext = audio_path.lower().rsplit(".", 1)[-1]
|
||||
mime_map = {
|
||||
"wav": "audio/wav",
|
||||
"mp3": "audio/mpeg",
|
||||
"flac": "audio/flac",
|
||||
"ogg": "audio/ogg",
|
||||
}
|
||||
mime_type = mime_map.get(ext, "audio/wav")
|
||||
|
||||
with open(audio_path, "rb") as f:
|
||||
audio_b64 = base64.b64encode(f.read()).decode("utf-8")
|
||||
return f"data:{mime_type};base64,{audio_b64}"
|
||||
|
||||
|
||||
def build_payload(args) -> dict[str, object]:
|
||||
payload: dict[str, object] = {
|
||||
"model": args.model,
|
||||
"input": args.text,
|
||||
"response_format": "pcm" if args.stream else args.response_format,
|
||||
}
|
||||
|
||||
if args.ref_audio:
|
||||
if args.ref_audio.startswith(("http://", "https://", "data:")):
|
||||
payload["ref_audio"] = args.ref_audio
|
||||
else:
|
||||
payload["ref_audio"] = encode_audio_to_base64(args.ref_audio)
|
||||
if args.ref_text:
|
||||
payload["ref_text"] = args.ref_text
|
||||
if args.max_new_tokens is not None:
|
||||
payload["max_new_tokens"] = args.max_new_tokens
|
||||
if args.stream:
|
||||
payload["stream"] = True
|
||||
|
||||
return payload
|
||||
|
||||
|
||||
def run_tts(args) -> None:
|
||||
payload = build_payload(args)
|
||||
api_url = f"{args.api_base}/v1/audio/speech"
|
||||
headers = {
|
||||
"Content-Type": "application/json",
|
||||
"Authorization": f"Bearer {args.api_key}",
|
||||
}
|
||||
|
||||
print(f"Model: {args.model}")
|
||||
print(f"Text: {args.text}")
|
||||
if args.ref_audio:
|
||||
print("Mode: voice cloning")
|
||||
print(f"Reference audio: {args.ref_audio}")
|
||||
else:
|
||||
print("Mode: text-to-speech")
|
||||
|
||||
if args.stream:
|
||||
output_path = args.output or "voxcpm_output.pcm"
|
||||
with httpx.Client(timeout=300.0) as client:
|
||||
with client.stream("POST", api_url, json=payload, headers=headers) as response:
|
||||
if response.status_code != 200:
|
||||
print(f"Error: {response.status_code}")
|
||||
print(response.read().decode("utf-8", errors="ignore"))
|
||||
return
|
||||
|
||||
total_bytes = 0
|
||||
with open(output_path, "wb") as f:
|
||||
for chunk in response.iter_bytes():
|
||||
if not chunk:
|
||||
continue
|
||||
f.write(chunk)
|
||||
total_bytes += len(chunk)
|
||||
print(f"Streamed {total_bytes} bytes to: {output_path}")
|
||||
return
|
||||
|
||||
with httpx.Client(timeout=300.0) as client:
|
||||
response = client.post(api_url, json=payload, headers=headers)
|
||||
|
||||
if response.status_code != 200:
|
||||
print(f"Error: {response.status_code}")
|
||||
print(response.text)
|
||||
return
|
||||
|
||||
try:
|
||||
text = response.content.decode("utf-8")
|
||||
if text.startswith('{"error"'):
|
||||
print(f"Error: {text}")
|
||||
return
|
||||
except UnicodeDecodeError:
|
||||
pass
|
||||
|
||||
output_path = args.output or "voxcpm_output.wav"
|
||||
with open(output_path, "wb") as f:
|
||||
f.write(response.content)
|
||||
print(f"Audio saved to: {output_path}")
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="VoxCPM OpenAI-compatible speech client")
|
||||
parser.add_argument("--api-base", default=DEFAULT_API_BASE, help="API base URL")
|
||||
parser.add_argument("--api-key", default=DEFAULT_API_KEY, help="API key")
|
||||
parser.add_argument("--model", "-m", default=DEFAULT_MODEL, help="Model name or path")
|
||||
parser.add_argument("--text", required=True, help="Text to synthesize")
|
||||
parser.add_argument("--ref-audio", default=None, help="Reference audio path, URL, or data URL")
|
||||
parser.add_argument(
|
||||
"--ref-text",
|
||||
default=None,
|
||||
help="The exact transcript spoken in the reference audio",
|
||||
)
|
||||
parser.add_argument("--stream", action="store_true", help="Enable streaming PCM output")
|
||||
parser.add_argument(
|
||||
"--response-format",
|
||||
default="wav",
|
||||
choices=["wav", "pcm", "flac", "mp3", "aac", "opus"],
|
||||
help="Audio format for non-streaming mode (default: wav)",
|
||||
)
|
||||
parser.add_argument("--max-new-tokens", type=int, default=None, help="Maximum tokens to generate")
|
||||
parser.add_argument("--output", "-o", default=None, help="Output file path")
|
||||
args = parser.parse_args()
|
||||
run_tts(args)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+38
@@ -0,0 +1,38 @@
|
||||
#!/bin/bash
|
||||
# Launch vLLM-Omni server for VoxCPM online speech serving.
|
||||
#
|
||||
# Usage:
|
||||
# ./run_server.sh # default: async_chunk stage config
|
||||
# ./run_server.sh async # async_chunk stage config
|
||||
# ./run_server.sh sync # no-async-chunk stage config
|
||||
# VOXCPM_MODEL=/path/to/model ./run_server.sh
|
||||
|
||||
set -e
|
||||
|
||||
MODE="${1:-async}"
|
||||
MODEL="${VOXCPM_MODEL:-OpenBMB/VoxCPM1.5}"
|
||||
|
||||
case "$MODE" in
|
||||
async)
|
||||
STAGE_CONFIG="vllm_omni/model_executor/stage_configs/voxcpm_async_chunk.yaml"
|
||||
;;
|
||||
sync)
|
||||
STAGE_CONFIG="vllm_omni/model_executor/stage_configs/voxcpm.yaml"
|
||||
;;
|
||||
*)
|
||||
echo "Unknown mode: $MODE"
|
||||
echo "Supported: async, sync"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
echo "Starting VoxCPM server with model: $MODEL"
|
||||
echo "Stage config: $STAGE_CONFIG"
|
||||
|
||||
vllm serve "$MODEL" \
|
||||
--stage-configs-path "$STAGE_CONFIG" \
|
||||
--host 0.0.0.0 \
|
||||
--port 8091 \
|
||||
--trust-remote-code \
|
||||
--enforce-eager \
|
||||
--omni
|
||||
@@ -5,12 +5,13 @@ Serve VoxCPM2 TTS via the OpenAI-compatible `/v1/audio/speech` endpoint.
|
||||
## Start the Server
|
||||
|
||||
```bash
|
||||
python -m vllm_omni.entrypoints.openai.api_server \
|
||||
--model openbmb/VoxCPM2 \
|
||||
--stage-configs-path vllm_omni/model_executor/stage_configs/voxcpm2.yaml \
|
||||
--host 0.0.0.0 --port 8000
|
||||
vllm serve openbmb/VoxCPM2 --omni --host 0.0.0.0 --port 8000
|
||||
```
|
||||
|
||||
The deploy config is auto-loaded from `vllm_omni/deploy/voxcpm2.yaml`. Pass
|
||||
`--deploy-config <path>` to override, or `--stage-N-<field> <value>` (e.g.
|
||||
`--stage-0-max-num-seqs 8`) for per-stage runtime overrides.
|
||||
|
||||
## Zero-shot Synthesis
|
||||
|
||||
```bash
|
||||
|
||||
@@ -0,0 +1,599 @@
|
||||
"""Gradio demo for VoxCPM2 TTS with gapless streaming audio playback.
|
||||
|
||||
Uses a custom AudioWorklet-based player for gap-free streaming
|
||||
(adapted from the Qwen3-TTS demo). Audio is streamed from the vLLM
|
||||
server through a same-origin proxy and played via the Web Audio API's
|
||||
AudioWorklet, which maintains a FIFO buffer queue and plays samples at
|
||||
the audio clock rate.
|
||||
|
||||
Usage:
|
||||
# Start the vLLM server first:
|
||||
vllm serve openbmb/VoxCPM2 --omni --host 0.0.0.0 --port 8000
|
||||
|
||||
# Then launch the demo:
|
||||
python gradio_demo.py --api-base http://localhost:8000
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import base64
|
||||
import io
|
||||
import json
|
||||
import logging
|
||||
|
||||
import gradio as gr
|
||||
import httpx
|
||||
import numpy as np
|
||||
import soundfile as sf
|
||||
from fastapi import FastAPI, Request
|
||||
from fastapi.responses import Response, StreamingResponse
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
SAMPLE_RATE = 48000
|
||||
|
||||
# ── AudioWorklet processor (loaded in browser via Blob URL) ──────────
|
||||
WORKLET_JS = r"""
|
||||
class TTSPlaybackProcessor extends AudioWorkletProcessor {
|
||||
constructor() {
|
||||
super();
|
||||
this.queue = [];
|
||||
this.buf = null;
|
||||
this.pos = 0;
|
||||
this.playing = false;
|
||||
this.played = 0;
|
||||
this.port.onmessage = (e) => {
|
||||
if (e.data && e.data.type === 'clear') {
|
||||
this.queue = []; this.buf = null; this.pos = 0; this.played = 0;
|
||||
if (this.playing) { this.playing = false; this.port.postMessage({type:'stopped'}); }
|
||||
return;
|
||||
}
|
||||
this.queue.push(e.data);
|
||||
};
|
||||
}
|
||||
process(inputs, outputs) {
|
||||
const out = outputs[0][0];
|
||||
for (let i = 0; i < out.length; i++) {
|
||||
if (!this.buf || this.pos >= this.buf.length) {
|
||||
if (this.queue.length > 0) {
|
||||
this.buf = this.queue.shift(); this.pos = 0;
|
||||
} else {
|
||||
for (let j = i; j < out.length; j++) out[j] = 0;
|
||||
if (this.playing) { this.playing = false; this.port.postMessage({type:'stopped', played:this.played}); }
|
||||
return true;
|
||||
}
|
||||
}
|
||||
out[i] = this.buf[this.pos++] / 32768;
|
||||
this.played++;
|
||||
}
|
||||
if (!this.playing) { this.playing = true; this.port.postMessage({type:'started'}); }
|
||||
return true;
|
||||
}
|
||||
}
|
||||
registerProcessor('tts-playback-processor', TTSPlaybackProcessor);
|
||||
"""
|
||||
|
||||
PLAYER_HTML = """
|
||||
<div id="tts-player">
|
||||
<div style="display:flex; align-items:center; gap:10px;">
|
||||
<div id="tts-status-dot" style="width:10px;height:10px;border-radius:50%;background:#ccc;flex-shrink:0;"></div>
|
||||
<span id="tts-status" style="font-weight:600;font-size:1.05em;">Ready</span>
|
||||
<button id="tts-stop-btn" onclick="window.ttsStop()"
|
||||
style="display:none; margin-left:auto; padding:5px 16px; border-radius:6px; border:1px solid #EF5552;
|
||||
background:#fff; color:#EF5552; cursor:pointer; font-size:0.85em;">Stop</button>
|
||||
</div>
|
||||
<div id="tts-metrics" style="display:none; grid-template-columns:repeat(4,1fr); gap:10px; margin-top:12px;">
|
||||
<div style="background:#f8f9fa;border-radius:6px;padding:8px 10px;text-align:center;">
|
||||
<div style="font-size:0.7em;text-transform:uppercase;color:#888;letter-spacing:0.5px;margin-bottom:2px;">TTFP</div>
|
||||
<div id="tts-m-ttfp" style="font-size:1.2em;font-weight:700;color:#333;">—</div>
|
||||
</div>
|
||||
<div style="background:#f8f9fa;border-radius:6px;padding:8px 10px;text-align:center;">
|
||||
<div style="font-size:0.7em;text-transform:uppercase;color:#888;letter-spacing:0.5px;margin-bottom:2px;">RTF</div>
|
||||
<div id="tts-m-rtf" style="font-size:1.2em;font-weight:700;color:#333;">—</div>
|
||||
</div>
|
||||
<div style="background:#f8f9fa;border-radius:6px;padding:8px 10px;text-align:center;">
|
||||
<div style="font-size:0.7em;text-transform:uppercase;color:#888;letter-spacing:0.5px;margin-bottom:2px;">Audio</div>
|
||||
<div id="tts-m-dur" style="font-size:1.2em;font-weight:700;color:#333;">—</div>
|
||||
</div>
|
||||
<div style="background:#f8f9fa;border-radius:6px;padding:8px 10px;text-align:center;">
|
||||
<div style="font-size:0.7em;text-transform:uppercase;color:#888;letter-spacing:0.5px;margin-bottom:2px;">Speed</div>
|
||||
<div id="tts-m-speed" style="font-size:1.2em;font-weight:700;color:#333;">—</div>
|
||||
</div>
|
||||
</div>
|
||||
<div id="tts-rtf-bar-wrap" style="display:none; background:#e8ecf1; border-radius:4px; height:20px; overflow:hidden; position:relative; margin-top:10px;">
|
||||
<div id="tts-rtf-bar" style="height:100%; border-radius:4px; transition:width 0.3s ease, background 0.3s ease; width:0%;"></div>
|
||||
<span id="tts-rtf-label" style="position:absolute;top:50%;left:50%;transform:translate(-50%,-50%);font-size:0.75em;font-weight:600;color:#444;"></span>
|
||||
</div>
|
||||
<div id="tts-elapsed" style="display:none; margin-top:6px; font-size:0.8em; color:#999; text-align:right;"></div>
|
||||
</div>
|
||||
"""
|
||||
|
||||
|
||||
def _build_player_js() -> str:
|
||||
return f"""
|
||||
<script>
|
||||
const SR = {SAMPLE_RATE};
|
||||
const WC = {json.dumps(WORKLET_JS)};
|
||||
let ctx = null, node = null, abort = null, gen = false, st = {{}};
|
||||
|
||||
async function init() {{
|
||||
if (ctx) return;
|
||||
ctx = new AudioContext({{ sampleRate: SR }});
|
||||
const b = new Blob([WC], {{ type: 'application/javascript' }});
|
||||
const u = URL.createObjectURL(b);
|
||||
await ctx.audioWorklet.addModule(u);
|
||||
URL.revokeObjectURL(u);
|
||||
node = new AudioWorkletNode(ctx, 'tts-playback-processor');
|
||||
node.connect(ctx.destination);
|
||||
node.port.onmessage = (e) => {{
|
||||
if (e.data.type === 'started') setStatus('Playing...', '#64dd17');
|
||||
else if (e.data.type === 'stopped' && !gen) {{
|
||||
setStatus('Done', '#64dd17'); showStats(true);
|
||||
const btn = document.getElementById('tts-stop-btn');
|
||||
if (btn) btn.style.display = 'none';
|
||||
}}
|
||||
}};
|
||||
}}
|
||||
|
||||
function setStatus(text, color) {{
|
||||
const s = document.getElementById('tts-status');
|
||||
const d = document.getElementById('tts-status-dot');
|
||||
if (s) s.textContent = text;
|
||||
if (d) d.style.background = color || '#ccc';
|
||||
}}
|
||||
|
||||
function showStats(fin) {{
|
||||
if (!st.t0) return;
|
||||
const elapsed = (fin && st.streamEnd ? (st.streamEnd - st.t0) : (performance.now() - st.t0)) / 1000;
|
||||
const dur = st.samples / SR;
|
||||
const mTtfp = document.getElementById('tts-m-ttfp');
|
||||
const mRtf = document.getElementById('tts-m-rtf');
|
||||
const mDur = document.getElementById('tts-m-dur');
|
||||
const mSpeed = document.getElementById('tts-m-speed');
|
||||
const bar = document.getElementById('tts-rtf-bar');
|
||||
const barLabel = document.getElementById('tts-rtf-label');
|
||||
const elapsedEl = document.getElementById('tts-elapsed');
|
||||
if (mTtfp && st.ttfp != null) mTtfp.textContent = st.ttfp.toFixed(0) + 'ms';
|
||||
if (mDur) mDur.textContent = dur.toFixed(1) + 's';
|
||||
if (dur > 0 && elapsed > 0) {{
|
||||
const rtf = elapsed / dur;
|
||||
const speed = 1 / rtf;
|
||||
if (mRtf) {{
|
||||
mRtf.textContent = rtf.toFixed(2) + 'x';
|
||||
mRtf.style.color = rtf < 1 ? '#64dd17' : rtf < 1.5 ? '#e8a317' : '#EF5552';
|
||||
}}
|
||||
if (mSpeed) {{
|
||||
mSpeed.textContent = speed.toFixed(1) + 'x';
|
||||
mSpeed.style.color = speed > 1 ? '#64dd17' : speed > 0.7 ? '#e8a317' : '#EF5552';
|
||||
}}
|
||||
if (bar) {{
|
||||
const pct = Math.min(speed / 10 * 100, 100);
|
||||
bar.style.width = pct + '%';
|
||||
bar.style.background = speed > 1 ? 'linear-gradient(90deg,#4A90D9,#64dd17)' : 'linear-gradient(90deg,#EF5552,#f87171)';
|
||||
}}
|
||||
if (barLabel) barLabel.textContent = speed.toFixed(1) + 'x realtime';
|
||||
}}
|
||||
if (elapsedEl) {{
|
||||
elapsedEl.style.display = 'block';
|
||||
elapsedEl.textContent = fin ? 'Completed in ' + elapsed.toFixed(1) + 's (' + st.chunks + ' chunks)' : elapsed.toFixed(1) + 's elapsed (' + st.chunks + ' chunks)';
|
||||
}}
|
||||
}}
|
||||
|
||||
window.ttsStop = function() {{
|
||||
if (abort) abort.abort();
|
||||
if (node) node.port.postMessage({{ type: 'clear' }});
|
||||
gen = false;
|
||||
setStatus('Stopped', '#999');
|
||||
const btn = document.getElementById('tts-stop-btn');
|
||||
if (btn) btn.style.display = 'none';
|
||||
}};
|
||||
|
||||
window.ttsGenerate = async function(payload) {{
|
||||
try {{ await init(); if (ctx.state === 'suspended') await ctx.resume(); }}
|
||||
catch (e) {{ setStatus('Audio init error: ' + e.message, '#EF5552'); return; }}
|
||||
if (abort) abort.abort();
|
||||
node.port.postMessage({{ type: 'clear' }});
|
||||
await new Promise(r => setTimeout(r, 50));
|
||||
node.port.postMessage({{ type: 'clear' }});
|
||||
|
||||
gen = true;
|
||||
st = {{ t0: null, chunks: 0, samples: 0, ttfp: null }};
|
||||
setStatus('Connecting...', '#4A90D9');
|
||||
const bEl = document.getElementById('tts-stop-btn');
|
||||
if (bEl) bEl.style.display = 'inline-block';
|
||||
const mp = document.getElementById('tts-metrics');
|
||||
if (mp) {{ mp.style.display = 'grid'; ['tts-m-ttfp','tts-m-rtf','tts-m-dur','tts-m-speed'].forEach(id => {{ const e = document.getElementById(id); if(e) {{ e.textContent = '\\u2014'; e.style.color = '#333'; }} }}); }}
|
||||
const bw = document.getElementById('tts-rtf-bar-wrap');
|
||||
if (bw) bw.style.display = 'block';
|
||||
const bar = document.getElementById('tts-rtf-bar');
|
||||
if (bar) bar.style.width = '0%';
|
||||
const bl = document.getElementById('tts-rtf-label');
|
||||
if (bl) bl.textContent = '';
|
||||
const ee = document.getElementById('tts-elapsed');
|
||||
if (ee) {{ ee.style.display = 'none'; ee.textContent = ''; }}
|
||||
abort = new AbortController();
|
||||
|
||||
try {{
|
||||
st.t0 = performance.now();
|
||||
const r = await fetch('/proxy/v1/audio/speech', {{
|
||||
method: 'POST',
|
||||
headers: {{ 'Content-Type': 'application/json' }},
|
||||
body: JSON.stringify(payload),
|
||||
signal: abort.signal,
|
||||
}});
|
||||
if (!r.ok) {{ const t = await r.text(); throw new Error('Server ' + r.status + ': ' + t.slice(0, 200)); }}
|
||||
setStatus('Streaming...', '#4A90D9');
|
||||
const reader = r.body.getReader();
|
||||
let left = new Uint8Array(0);
|
||||
while (true) {{
|
||||
const {{ done, value }} = await reader.read();
|
||||
if (done) break;
|
||||
let raw;
|
||||
if (left.length > 0) {{
|
||||
raw = new Uint8Array(left.length + value.length);
|
||||
raw.set(left); raw.set(value, left.length);
|
||||
}} else {{ raw = value; }}
|
||||
const usable = raw.length - (raw.length % 2);
|
||||
left = usable < raw.length ? raw.slice(usable) : new Uint8Array(0);
|
||||
if (usable > 0) {{
|
||||
const ab = new ArrayBuffer(usable);
|
||||
new Uint8Array(ab).set(raw.subarray(0, usable));
|
||||
const pcm = new Int16Array(ab);
|
||||
node.port.postMessage(pcm);
|
||||
st.chunks++;
|
||||
st.samples += pcm.length;
|
||||
if (st.ttfp == null) st.ttfp = performance.now() - st.t0;
|
||||
showStats(false);
|
||||
}}
|
||||
}}
|
||||
}} catch (e) {{
|
||||
if (e.name !== 'AbortError') {{
|
||||
setStatus('Error: ' + e.message, '#EF5552');
|
||||
console.error('TTS error:', e);
|
||||
}}
|
||||
}} finally {{
|
||||
st.streamEnd = performance.now();
|
||||
showStats(true);
|
||||
gen = false;
|
||||
if (st.samples > 0) setStatus('Finishing playback...', '#64dd17');
|
||||
else {{
|
||||
setStatus('No audio received', '#999');
|
||||
if (bEl) bEl.style.display = 'none';
|
||||
}}
|
||||
}}
|
||||
}};
|
||||
</script>
|
||||
"""
|
||||
|
||||
|
||||
def _encode_audio(audio_data: tuple) -> str:
|
||||
sr, audio_np = audio_data
|
||||
if audio_np.dtype in (np.float32, np.float64):
|
||||
audio_np = np.clip(audio_np, -1.0, 1.0)
|
||||
audio_np = (audio_np * 32767).astype(np.int16)
|
||||
elif audio_np.dtype != np.int16:
|
||||
audio_np = audio_np.astype(np.int16)
|
||||
buf = io.BytesIO()
|
||||
sf.write(buf, audio_np, sr, format="WAV")
|
||||
return f"data:audio/wav;base64,{base64.b64encode(buf.getvalue()).decode()}"
|
||||
|
||||
|
||||
def create_app(api_base: str):
|
||||
app = FastAPI()
|
||||
_pending: dict[str, dict] = {}
|
||||
|
||||
@app.post("/proxy/v1/audio/speech")
|
||||
async def proxy_speech(request: Request):
|
||||
body = await request.json()
|
||||
req_id = body.get("_req_id")
|
||||
if req_id and req_id in _pending:
|
||||
body = _pending.pop(req_id)
|
||||
logger.info("Proxy: %s", {k: (f"<{len(str(v))} chars>" if k == "ref_audio" else v) for k, v in body.items()})
|
||||
try:
|
||||
client = httpx.AsyncClient(timeout=300)
|
||||
resp = await client.send(
|
||||
client.build_request(
|
||||
"POST",
|
||||
f"{api_base}/v1/audio/speech",
|
||||
json=body,
|
||||
headers={"Authorization": "Bearer EMPTY", "Content-Type": "application/json"},
|
||||
),
|
||||
stream=True,
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.exception("Proxy connection error")
|
||||
await client.aclose()
|
||||
return Response(content=str(exc), status_code=502)
|
||||
if resp.status_code != 200:
|
||||
content = await resp.aread()
|
||||
await resp.aclose()
|
||||
await client.aclose()
|
||||
return Response(content=content, status_code=resp.status_code)
|
||||
|
||||
async def relay():
|
||||
try:
|
||||
async for chunk in resp.aiter_bytes():
|
||||
yield chunk
|
||||
finally:
|
||||
await resp.aclose()
|
||||
await client.aclose()
|
||||
|
||||
return StreamingResponse(relay(), media_type="application/octet-stream")
|
||||
|
||||
css = """
|
||||
#generate-btn button { width: 100%; }
|
||||
#streaming-player { border: 1px solid var(--border-color-primary) !important; border-radius: var(--block-radius) !important; padding: var(--block-padding) !important; }
|
||||
"""
|
||||
theme = gr.themes.Default(
|
||||
primary_hue=gr.themes.Color(
|
||||
c50="#f0f5ff",
|
||||
c100="#dce6f9",
|
||||
c200="#b8cef3",
|
||||
c300="#8eb2eb",
|
||||
c400="#6496e0",
|
||||
c500="#4A90D9",
|
||||
c600="#3a7bc8",
|
||||
c700="#2d66b0",
|
||||
c800="#1f4f8f",
|
||||
c900="#163a6e",
|
||||
c950="#0e2650",
|
||||
),
|
||||
)
|
||||
|
||||
with gr.Blocks(title="VoxCPM2 TTS Demo") as demo:
|
||||
gr.HTML(f"""
|
||||
<div style="display:flex; align-items:center; gap:16px; margin-bottom:8px;">
|
||||
<img src="https://raw.githubusercontent.com/vllm-project/vllm-omni/main/docs/source/logos/vllm-omni-logo.png"
|
||||
alt="vLLM-Omni" style="height:42px;">
|
||||
<div>
|
||||
<h1 style="margin:0; font-size:1.5em;">VoxCPM2 Streaming Demo</h1>
|
||||
<span style="font-size:0.85em; color:#666;">
|
||||
Served by <a href="https://github.com/vllm-project/vllm-omni" target="_blank"
|
||||
style="color:#4A90D9; text-decoration:none; font-weight:600;">vLLM-Omni</a>
|
||||
· <code style="background:#eef2f7; padding:2px 6px; border-radius:4px; font-size:0.9em;">{api_base}</code>
|
||||
· 48 kHz
|
||||
</span>
|
||||
</div>
|
||||
</div>
|
||||
""")
|
||||
|
||||
gr.Markdown(
|
||||
"**Three modes:** "
|
||||
"**Voice Design** (control instruction only) · "
|
||||
"**Controllable Cloning** (ref audio + optional style control) · "
|
||||
"**Ultimate Cloning** (ref audio + transcript for audio continuation)"
|
||||
)
|
||||
|
||||
with gr.Row():
|
||||
with gr.Column(scale=3):
|
||||
text_input = gr.Textbox(
|
||||
label="Target Text",
|
||||
placeholder="Enter text to synthesize...",
|
||||
lines=4,
|
||||
)
|
||||
control_instruction = gr.Textbox(
|
||||
label="Control Instruction (optional)",
|
||||
placeholder="e.g. A warm young woman / Excited and fast-paced",
|
||||
lines=2,
|
||||
info="Describe voice style, emotion, pace. Works for both Voice Design and Controllable Cloning.",
|
||||
)
|
||||
|
||||
with gr.Accordion("Voice Cloning", open=False):
|
||||
ref_audio = gr.Audio(
|
||||
label="Reference Audio (upload for cloning)",
|
||||
type="numpy",
|
||||
sources=["upload", "microphone"],
|
||||
)
|
||||
ref_audio_url = gr.Textbox(
|
||||
label="or Reference Audio URL",
|
||||
placeholder="https://example.com/reference.wav",
|
||||
)
|
||||
ultimate_clone = gr.Checkbox(
|
||||
label="Ultimate Cloning Mode",
|
||||
value=False,
|
||||
info="Provide transcript of ref audio for audio continuation (disables control instruction)",
|
||||
)
|
||||
prompt_text = gr.Textbox(
|
||||
label="Reference Audio Transcript",
|
||||
placeholder="Transcript of your reference audio (for ultimate cloning)",
|
||||
lines=2,
|
||||
visible=False,
|
||||
)
|
||||
|
||||
with gr.Row():
|
||||
stream_checkbox = gr.Checkbox(
|
||||
label="Stream (gapless)",
|
||||
value=True,
|
||||
info="AudioWorklet streaming",
|
||||
)
|
||||
with gr.Row():
|
||||
generate_btn = gr.Button(
|
||||
"Generate Speech",
|
||||
variant="primary",
|
||||
size="lg",
|
||||
elem_id="generate-btn",
|
||||
scale=3,
|
||||
)
|
||||
reset_btn = gr.Button("Reset", variant="secondary", size="lg", scale=1)
|
||||
|
||||
with gr.Column(scale=2):
|
||||
player_html = gr.HTML(
|
||||
value=PLAYER_HTML,
|
||||
visible=True,
|
||||
label="streaming player",
|
||||
elem_id="streaming-player",
|
||||
)
|
||||
audio_output = gr.Audio(
|
||||
label="generated audio",
|
||||
interactive=False,
|
||||
autoplay=True,
|
||||
visible=False,
|
||||
)
|
||||
gr.Examples(
|
||||
examples=[
|
||||
["Hello, this is a VoxCPM2 demo running on vLLM-Omni.", ""],
|
||||
[
|
||||
"I have a dream that my four little children will one day live in a nation "
|
||||
"where they will not be judged by the color of their skin but by the content "
|
||||
"of their character.",
|
||||
"",
|
||||
],
|
||||
[
|
||||
"I never asked you to stay. It's not like I care or anything. "
|
||||
"But why does it still hurt so much now that you're gone?",
|
||||
"A young girl with a soft, sweet voice. Speaks slowly with a melancholic tone.",
|
||||
],
|
||||
],
|
||||
inputs=[text_input, control_instruction],
|
||||
label="examples",
|
||||
)
|
||||
gr.HTML("""
|
||||
<div style="text-align:center; padding:8px 0; margin-top:4px;">
|
||||
<a href="https://github.com/vllm-project/vllm-omni" target="_blank">
|
||||
<img src="https://raw.githubusercontent.com/vllm-project/vllm-omni/main/docs/source/logos/vllm-omni-logo.png"
|
||||
alt="vLLM-Omni" style="height:28px; opacity:0.7;">
|
||||
</a>
|
||||
</div>
|
||||
""")
|
||||
|
||||
hidden_payload = gr.Textbox(visible=False, elem_id="tts-payload")
|
||||
|
||||
def on_ultimate_toggle(checked):
|
||||
return (
|
||||
gr.update(visible=checked), # prompt_text
|
||||
gr.update(interactive=not checked), # control_instruction
|
||||
)
|
||||
|
||||
ultimate_clone.change(
|
||||
fn=on_ultimate_toggle,
|
||||
inputs=[ultimate_clone],
|
||||
outputs=[prompt_text, control_instruction],
|
||||
)
|
||||
|
||||
def on_stream_change(stream: bool):
|
||||
if stream:
|
||||
return gr.update(visible=True), gr.update(visible=False)
|
||||
return gr.update(visible=False), gr.update(visible=True)
|
||||
|
||||
stream_checkbox.change(
|
||||
fn=on_stream_change,
|
||||
inputs=[stream_checkbox],
|
||||
outputs=[player_html, audio_output],
|
||||
)
|
||||
|
||||
def on_reset():
|
||||
return "", "", None, "", False, "", PLAYER_HTML
|
||||
|
||||
reset_btn.click(
|
||||
fn=on_reset,
|
||||
outputs=[
|
||||
text_input,
|
||||
control_instruction,
|
||||
audio_output,
|
||||
hidden_payload,
|
||||
ultimate_clone,
|
||||
prompt_text,
|
||||
player_html,
|
||||
],
|
||||
js="() => { if (window.ttsStop) window.ttsStop(); }",
|
||||
)
|
||||
|
||||
def on_generate(stream_enabled, text, ctrl_instr, ref_a, ref_url, ult_clone, p_text):
|
||||
import time as _time
|
||||
|
||||
if not text or not text.strip():
|
||||
raise gr.Error("Please enter text to synthesize.")
|
||||
|
||||
# VoxCPM2 uses "(instruction)text" format for control
|
||||
ctrl = ctrl_instr.strip() if ctrl_instr and not ult_clone else ""
|
||||
final_text = f"({ctrl}){text.strip()}" if ctrl else text.strip()
|
||||
|
||||
payload: dict = {
|
||||
"input": final_text,
|
||||
"voice": "default",
|
||||
"response_format": "pcm" if stream_enabled else "wav",
|
||||
"stream": stream_enabled,
|
||||
}
|
||||
|
||||
# Reference audio for cloning
|
||||
ref_url_s = ref_url.strip() if ref_url else ""
|
||||
if ref_url_s:
|
||||
payload["ref_audio"] = ref_url_s
|
||||
elif ref_a is not None:
|
||||
payload["ref_audio"] = _encode_audio(ref_a)
|
||||
|
||||
# Ultimate cloning: prompt_audio + prompt_text for continuation
|
||||
if ult_clone and p_text and p_text.strip():
|
||||
if ref_url_s:
|
||||
payload["prompt_audio"] = ref_url_s
|
||||
elif ref_a is not None:
|
||||
payload["prompt_audio"] = payload.get("ref_audio", "")
|
||||
payload["prompt_text"] = p_text.strip()
|
||||
|
||||
if stream_enabled:
|
||||
if ref_a is not None and not ref_url_s:
|
||||
req_id = f"req-{int(_time.time() * 1000)}"
|
||||
_pending[req_id] = payload
|
||||
browser_payload = {"_req_id": req_id, "_nonce": int(_time.time() * 1000)}
|
||||
return json.dumps(browser_payload), gr.update()
|
||||
payload["_nonce"] = int(_time.time() * 1000)
|
||||
return json.dumps(payload), gr.update()
|
||||
else:
|
||||
try:
|
||||
with httpx.Client(timeout=300.0) as client:
|
||||
resp = client.post(
|
||||
f"{api_base}/v1/audio/speech",
|
||||
json=payload,
|
||||
headers={"Content-Type": "application/json", "Authorization": "Bearer EMPTY"},
|
||||
)
|
||||
except httpx.ConnectError:
|
||||
raise gr.Error(f"Cannot connect to server at {api_base}.")
|
||||
if resp.status_code != 200:
|
||||
raise gr.Error(f"Server error ({resp.status_code}): {resp.text[:200]}")
|
||||
audio_np, sr = sf.read(io.BytesIO(resp.content))
|
||||
if audio_np.ndim > 1:
|
||||
audio_np = audio_np[:, 0]
|
||||
return "", (sr, audio_np.astype(np.float32))
|
||||
|
||||
generate_btn.click(
|
||||
fn=on_generate,
|
||||
inputs=[
|
||||
stream_checkbox,
|
||||
text_input,
|
||||
control_instruction,
|
||||
ref_audio,
|
||||
ref_audio_url,
|
||||
ultimate_clone,
|
||||
prompt_text,
|
||||
],
|
||||
outputs=[hidden_payload, audio_output],
|
||||
).then(
|
||||
fn=lambda p: p,
|
||||
inputs=[hidden_payload],
|
||||
outputs=[hidden_payload],
|
||||
js="(p) => { if (p && p.trim()) { const d = JSON.parse(p); delete d._nonce; window.ttsGenerate(d); } return p; }",
|
||||
)
|
||||
|
||||
demo.queue()
|
||||
|
||||
return gr.mount_gradio_app(app, demo, path="/", css=css, theme=theme, head=_build_player_js())
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="VoxCPM2 streaming Gradio demo")
|
||||
parser.add_argument("--api-base", default="http://localhost:8000", help="vLLM API server URL")
|
||||
parser.add_argument("--host", default="0.0.0.0", help="Gradio server host")
|
||||
parser.add_argument("--port", type=int, default=7860, help="Gradio server port")
|
||||
args = parser.parse_args()
|
||||
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
print(f"Connecting to vLLM server at: {args.api_base}")
|
||||
|
||||
import uvicorn
|
||||
|
||||
uvicorn.run(create_app(args.api_base), host=args.host, port=args.port)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -13,10 +13,7 @@ Examples:
|
||||
--ref-audio "https://example.com/reference.wav"
|
||||
|
||||
Server setup:
|
||||
python -m vllm_omni.entrypoints.openai.api_server \
|
||||
--model openbmb/VoxCPM2 \
|
||||
--stage-configs-path vllm_omni/model_executor/stage_configs/voxcpm2.yaml \
|
||||
--host 0.0.0.0 --port 8000
|
||||
vllm serve openbmb/VoxCPM2 --omni --host 0.0.0.0 --port 8000
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
Reference in new issue
Block a user