Merge remote-tracking branch 'origin/main' into dev/vllm-align
This commit is contained in:
commit
279b6b5050
45 files changed
+2224
-349
No files matched your search
@@ -151,6 +151,24 @@ The default yaml configuration deploys Thinker and DiT on the same GPU. You can
|
||||
|
||||
------
|
||||
|
||||
#### Tensor Parallelism (TP)
|
||||
|
||||
For larger models or multi-GPU environments, you can enable Tensor Parallelism (TP) by modifying the stage configuration (e.g., [`bagel.yaml`](../../../vllm_omni/model_executor/stage_configs/bagel.yaml)).
|
||||
|
||||
1. **Set `tensor_parallel_size`**: Increase this value (e.g., to `2` or `4`).
|
||||
2. **Set `devices`**: Specify the comma-separated GPU IDs to be used for the stage (e.g., `"0,1"`).
|
||||
|
||||
Example configuration for TP=2 on GPUs 0 and 1:
|
||||
```yaml
|
||||
engine_args:
|
||||
tensor_parallel_size: 2
|
||||
...
|
||||
runtime:
|
||||
devices: "0,1"
|
||||
```
|
||||
|
||||
------
|
||||
|
||||
#### 🔗 Runtime Configuration
|
||||
|
||||
| Parameter | Value | Description |
|
||||
|
||||
@@ -118,12 +118,6 @@ def parse_args() -> argparse.Namespace:
|
||||
default=1,
|
||||
help="Number of ready layers (blocks) to keep on GPU during generation.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--tensor_parallel_size",
|
||||
type=int,
|
||||
default=1,
|
||||
help="Number of GPUs used for tensor parallelism (TP) inside the DiT.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--vae_use_slicing",
|
||||
action="store_true",
|
||||
@@ -134,6 +128,18 @@ def parse_args() -> argparse.Namespace:
|
||||
action="store_true",
|
||||
help="Enable VAE tiling for memory optimization.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--tensor_parallel_size",
|
||||
type=int,
|
||||
default=1,
|
||||
help="Number of GPUs used for tensor parallelism (TP) inside the DiT.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--vae_patch_parallel_size",
|
||||
type=int,
|
||||
default=1,
|
||||
help="Number of ranks used for VAE patch/tile parallelism (decode/encode).",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
@@ -176,6 +182,7 @@ def main():
|
||||
ring_degree=args.ring_degree,
|
||||
cfg_parallel_size=args.cfg_parallel_size,
|
||||
tensor_parallel_size=args.tensor_parallel_size,
|
||||
vae_patch_parallel_size=args.vae_patch_parallel_size,
|
||||
)
|
||||
|
||||
# Check if profiling is requested via environment variable
|
||||
@@ -207,8 +214,10 @@ def main():
|
||||
print(f" Cache backend: {args.cache_backend if args.cache_backend else 'None (no acceleration)'}")
|
||||
print(
|
||||
f" Parallel configuration: tensor_parallel_size={args.tensor_parallel_size}, "
|
||||
f"ulysses_degree={args.ulysses_degree}, ring_degree={args.ring_degree}, cfg_parallel_size={args.cfg_parallel_size}"
|
||||
f"ulysses_degree={args.ulysses_degree}, ring_degree={args.ring_degree}, cfg_parallel_size={args.cfg_parallel_size}, "
|
||||
f"vae_patch_parallel_size={args.vae_patch_parallel_size}"
|
||||
)
|
||||
print(f" CPU offload: {args.enable_cpu_offload}")
|
||||
print(f" Image size: {args.width}x{args.height}")
|
||||
print(f"{'=' * 60}\n")
|
||||
|
||||
|
||||
@@ -22,12 +22,29 @@ cd /workspace/vllm-omni/examples/online_serving/bagel
|
||||
bash run_server.sh
|
||||
```
|
||||
|
||||
If you have a custom stage configs file, launch the server with the command below:
|
||||
|
||||
```bash
|
||||
vllm serve ByteDance-Seed/BAGEL-7B-MoT --omni --port 8091 --stage-configs-path /path/to/stage_configs_file
|
||||
```
|
||||
|
||||
#### 🚀 Tensor Parallelism (TP)
|
||||
|
||||
For larger models or multi-GPU environments, you can enable Tensor Parallelism (TP) for the server.
|
||||
|
||||
1. **Modify Stage Config**: Create or modify a stage configuration yaml (e.g., [`bagel.yaml`](../../../vllm_omni/model_executor/stage_configs/bagel.yaml)). Set `tensor_parallel_size` to `2` (or more) and update `devices` to include multiple GPU IDs (e.g., `"0,1"`).
|
||||
|
||||
```yaml
|
||||
engine_args:
|
||||
tensor_parallel_size: 2
|
||||
...
|
||||
runtime:
|
||||
devices: "0,1"
|
||||
```
|
||||
|
||||
2. **Launch Server**:
|
||||
```bash
|
||||
vllm serve ByteDance-Seed/BAGEL-7B-MoT --omni --port 8091 --stage-configs-path /path/to/your/custom_bagel.yaml
|
||||
```
|
||||
|
||||
### Send Multi-modal Request
|
||||
|
||||
Get into the bagel folder:
|
||||
|
||||
@@ -76,6 +76,10 @@ def edit_image(
|
||||
|
||||
# Build extra_body with generation parameters
|
||||
extra_body = {}
|
||||
if height is not None:
|
||||
extra_body["height"] = height
|
||||
if width is not None:
|
||||
extra_body["width"] = width
|
||||
if steps is not None:
|
||||
extra_body["num_inference_steps"] = steps
|
||||
if guidance_scale is not None:
|
||||
|
||||
@@ -19,26 +19,25 @@ if [[ ! -f "$INPUT_IMG" ]]; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
IMG_B64=$(base64 -w0 "$INPUT_IMG")
|
||||
REQUEST_JSON_FILE=$(mktemp)
|
||||
trap 'rm -f "$REQUEST_JSON_FILE"' EXIT
|
||||
|
||||
REQUEST_JSON=$(
|
||||
jq -n --arg prompt "$PROMPT" --arg img "$IMG_B64" '{
|
||||
# Pipe base64 into jq via stdin to avoid ARG_MAX limit on large images
|
||||
base64 -w0 "$INPUT_IMG" \
|
||||
| jq -Rs --arg prompt "$PROMPT" '{
|
||||
messages: [{
|
||||
role: "user",
|
||||
content: [
|
||||
{"type": "text", "text": $prompt},
|
||||
{"type": "image_url", "image_url": {"url": ("data:image/png;base64," + $img)}}
|
||||
{"type": "image_url", "image_url": {"url": ("data:image/png;base64," + .)}}
|
||||
]
|
||||
}],
|
||||
extra_body: {
|
||||
height: 1024,
|
||||
width: 1024,
|
||||
num_inference_steps: 50,
|
||||
guidance_scale: 1,
|
||||
seed: 42
|
||||
}
|
||||
}'
|
||||
)
|
||||
}' > "$REQUEST_JSON_FILE"
|
||||
|
||||
echo "Generating edited image..."
|
||||
echo "Server: $SERVER"
|
||||
@@ -48,7 +47,7 @@ echo "Output: $OUTPUT"
|
||||
|
||||
curl -s "$SERVER/v1/chat/completions" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d "$REQUEST_JSON" \
|
||||
-d @"$REQUEST_JSON_FILE" \
|
||||
| jq -r '.choices[0].message.content[0].image_url.url' \
|
||||
| cut -d',' -f2 \
|
||||
| base64 -d > "$OUTPUT"
|
||||
|
||||
@@ -9,6 +9,8 @@ This directory contains examples for running Qwen3-TTS models with vLLM-Omni's o
|
||||
| `Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice` | CustomVoice | Predefined speaker voices with optional style control |
|
||||
| `Qwen/Qwen3-TTS-12Hz-1.7B-VoiceDesign` | VoiceDesign | Natural language voice style description |
|
||||
| `Qwen/Qwen3-TTS-12Hz-1.7B-Base` | Base | Voice cloning from reference audio |
|
||||
| `Qwen/Qwen3-TTS-12Hz-0.6B-CustomVoice` | CustomVoice | Smaller/faster variant |
|
||||
| `Qwen/Qwen3-TTS-12Hz-0.6B-Base` | Base | Smaller/faster variant for voice cloning |
|
||||
|
||||
## Quick Start
|
||||
|
||||
@@ -24,19 +26,27 @@ This directory contains examples for running Qwen3-TTS models with vLLM-Omni's o
|
||||
./run_server.sh Base
|
||||
```
|
||||
|
||||
Or launch directly with vllm serve:
|
||||
|
||||
```bash
|
||||
vllm serve Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice \
|
||||
--stage-configs-path vllm_omni/model_executor/stage_configs/qwen3_tts.yaml \
|
||||
--omni --port 8091 --trust-remote-code --enforce-eager
|
||||
```
|
||||
|
||||
### 2. Run the Client
|
||||
|
||||
```bash
|
||||
# CustomVoice: Use predefined speaker
|
||||
python openai_speech_client.py \
|
||||
--text "你好,我是通义千问" \
|
||||
--voice Vivian \
|
||||
--voice vivian \
|
||||
--language Chinese
|
||||
|
||||
# CustomVoice with style instruction
|
||||
python openai_speech_client.py \
|
||||
--text "今天天气真好" \
|
||||
--voice Ryan \
|
||||
--voice ryan \
|
||||
--instructions "用开心的语气说"
|
||||
|
||||
# VoiceDesign: Describe the voice style
|
||||
@@ -59,25 +69,25 @@ python openai_speech_client.py \
|
||||
|
||||
```bash
|
||||
# Simple TTS request
|
||||
curl -X POST http://localhost:8000/v1/audio/speech \
|
||||
curl -X POST http://localhost:8091/v1/audio/speech \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"input": "Hello, how are you?",
|
||||
"voice": "Vivian",
|
||||
"voice": "vivian",
|
||||
"language": "English"
|
||||
}' --output output.wav
|
||||
|
||||
# With style instruction
|
||||
curl -X POST http://localhost:8000/v1/audio/speech \
|
||||
curl -X POST http://localhost:8091/v1/audio/speech \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"input": "I am so excited!",
|
||||
"voice": "Vivian",
|
||||
"voice": "vivian",
|
||||
"instructions": "Speak with great enthusiasm"
|
||||
}' --output excited.wav
|
||||
|
||||
# List available voices in CustomVoice models
|
||||
curl http://localhost:8000/v1/audio/voices
|
||||
curl http://localhost:8091/v1/audio/voices
|
||||
```
|
||||
|
||||
## API Reference
|
||||
@@ -90,12 +100,26 @@ POST /v1/audio/speech
|
||||
|
||||
This endpoint follows the [OpenAI Audio Speech API](https://platform.openai.com/docs/api-reference/audio/createSpeech) format with additional Qwen3-TTS parameters.
|
||||
|
||||
### Voices Endpoint
|
||||
|
||||
```
|
||||
GET /v1/audio/voices
|
||||
```
|
||||
|
||||
Lists available voices for the loaded model:
|
||||
|
||||
```json
|
||||
{
|
||||
"voices": ["aiden", "dylan", "eric", "one_anna", "ryan", "serena", "sohee", "uncle_fu", "vivian"]
|
||||
}
|
||||
```
|
||||
|
||||
### Request Body
|
||||
|
||||
```json
|
||||
{
|
||||
"input": "Text to synthesize",
|
||||
"voice": "Vivian",
|
||||
"voice": "vivian",
|
||||
"response_format": "wav",
|
||||
"task_type": "CustomVoice",
|
||||
"language": "Auto",
|
||||
@@ -115,45 +139,64 @@ Returns audio data in the requested format (default: WAV).
|
||||
|
||||
## Parameters
|
||||
|
||||
### Standard OpenAI Parameters
|
||||
### OpenAI Standard Parameters
|
||||
|
||||
| Parameter | Type | Default | Description |
|
||||
|-----------|------|---------|-------------|
|
||||
| `input` | string | required | Text to synthesize |
|
||||
| `voice` | string | "Vivian" | Speaker/voice name |
|
||||
| `input` | string | **required** | Text to synthesize |
|
||||
| `model` | string | server's model | Model to use (optional, should match server if specified) |
|
||||
| `voice` | string | "vivian" | Speaker name (e.g., vivian, ryan, aiden) |
|
||||
| `response_format` | string | "wav" | Audio format: wav, mp3, flac, pcm, aac, opus |
|
||||
| `speed` | float | 1.0 | Playback speed (0.25-4.0) |
|
||||
| `model` | string | optional | Model name (optional when serving single model) |
|
||||
|
||||
### Qwen3-TTS Parameters
|
||||
### vLLM-Omni Extension Parameters
|
||||
|
||||
| Parameter | Type | Default | Description |
|
||||
|-----------|------|---------|-------------|
|
||||
| `task_type` | string | "CustomVoice" | Task: CustomVoice, VoiceDesign, or Base |
|
||||
| `language` | string | "Auto" | Language: Auto, Chinese, English, Japanese, Korean |
|
||||
| `language` | string | "Auto" | Language (see supported languages below) |
|
||||
| `instructions` | string | "" | Voice style/emotion instructions |
|
||||
| `max_new_tokens` | int | 2048 | Maximum tokens to generate |
|
||||
|
||||
**Supported languages:** Auto, Chinese, English, Japanese, Korean, German, French, Russian, Portuguese, Spanish, Italian
|
||||
|
||||
### Voice Clone Parameters (Base task)
|
||||
|
||||
| Parameter | Type | Required | Description |
|
||||
| Parameter | Type | Default | Description |
|
||||
|-----------|------|----------|-------------|
|
||||
| `ref_audio` | string | Yes* | Reference audio (file path, URL, or base64) |
|
||||
| `ref_text` | string | No | Transcript of reference audio (for ICL mode) |
|
||||
| `x_vector_only_mode` | bool | false | Use speaker embedding only (no ICL) |
|
||||
| `ref_audio` | string | null | Reference audio (URL or base64 data URL) |
|
||||
| `ref_text` | string | null | Transcript of reference audio |
|
||||
| `x_vector_only_mode` | bool | null | Use speaker embedding only (no ICL) |
|
||||
|
||||
## Python Usage
|
||||
|
||||
### Using OpenAI SDK
|
||||
|
||||
```python
|
||||
from openai import OpenAI
|
||||
|
||||
client = OpenAI(base_url="http://localhost:8091/v1", api_key="none")
|
||||
|
||||
response = client.audio.speech.create(
|
||||
model="Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice",
|
||||
voice="vivian",
|
||||
input="Hello, how are you?",
|
||||
)
|
||||
|
||||
response.stream_to_file("output.wav")
|
||||
```
|
||||
|
||||
### Using httpx
|
||||
|
||||
```python
|
||||
import httpx
|
||||
|
||||
# Simple request
|
||||
response = httpx.post(
|
||||
"http://localhost:8000/v1/audio/speech",
|
||||
"http://localhost:8091/v1/audio/speech",
|
||||
json={
|
||||
"model": "Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice",
|
||||
"input": "Hello world",
|
||||
"voice": "Vivian",
|
||||
"input": "Hello, how are you?",
|
||||
"voice": "vivian",
|
||||
"language": "English",
|
||||
},
|
||||
timeout=300.0,
|
||||
)
|
||||
@@ -169,7 +212,8 @@ with open("output.wav", "wb") as f:
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
1. **Connection refused**: Make sure the server is running on the correct port
|
||||
2. **Out of memory**: Reduce `--gpu-memory-utilization` in run_server.sh
|
||||
3. **Unsupported speaker**: Check supported speakers via model documentation
|
||||
4. **Voice clone fails**: Ensure you're using the Base model variant for voice cloning
|
||||
1. **"TTS model did not produce audio output"**: Ensure you're using the correct model variant for your task type (CustomVoice task → CustomVoice model, etc.)
|
||||
2. **Connection refused**: Make sure the server is running on the correct port
|
||||
3. **Out of memory**: Use smaller model variant (`Qwen/Qwen3-TTS-12Hz-0.6B-CustomVoice`) or reduce `--gpu-memory-utilization`
|
||||
4. **Unsupported speaker**: Use `/v1/audio/voices` to list available voices for the loaded model
|
||||
5. **Voice clone fails**: Ensure you're using the Base model variant for voice cloning
|
||||
@@ -5,10 +5,10 @@ to generate audio from text using Qwen3-TTS models.
|
||||
|
||||
Examples:
|
||||
# CustomVoice task (predefined speaker)
|
||||
python openai_speech_client.py --text "Hello, how are you?" --voice Vivian
|
||||
python openai_speech_client.py --text "Hello, how are you?" --voice vivian
|
||||
|
||||
# CustomVoice with emotion instruction
|
||||
python openai_speech_client.py --text "I'm so happy!" --voice Vivian \
|
||||
python openai_speech_client.py --text "I'm so happy!" --voice vivian \
|
||||
--instructions "Speak with excitement"
|
||||
|
||||
# VoiceDesign task (voice from description)
|
||||
@@ -30,7 +30,7 @@ import os
|
||||
import httpx
|
||||
|
||||
# Default server configuration
|
||||
DEFAULT_API_BASE = "http://localhost:8000"
|
||||
DEFAULT_API_BASE = "http://localhost:8091"
|
||||
DEFAULT_API_KEY = "EMPTY"
|
||||
|
||||
|
||||
@@ -178,8 +178,8 @@ def parse_args():
|
||||
parser.add_argument(
|
||||
"--voice",
|
||||
type=str,
|
||||
default="Vivian",
|
||||
help="Speaker/voice name (default: Vivian). Options: Vivian, Ryan, etc.",
|
||||
default="vivian",
|
||||
help="Speaker/voice name (default: vivian). Options: vivian, ryan, aiden, etc.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--language",
|
||||
|
||||
@@ -33,7 +33,7 @@ echo "Starting Qwen3-TTS server with model: $MODEL"
|
||||
vllm-omni serve "$MODEL" \
|
||||
--stage-configs-path vllm_omni/model_executor/stage_configs/qwen3_tts.yaml \
|
||||
--host 0.0.0.0 \
|
||||
--port 8000 \
|
||||
--port 8091 \
|
||||
--gpu-memory-utilization 0.9 \
|
||||
--trust-remote-code \
|
||||
--enforce-eager \
|
||||
|
||||
Reference in new issue
Block a user