diff --git a/docs/contributing/ci/CI_5levels.md b/docs/contributing/ci/CI_5levels.md
index dbf87d352..c8493af0a 100644
--- a/docs/contributing/ci/CI_5levels.md
+++ b/docs/contributing/ci/CI_5levels.md
@@ -67,8 +67,8 @@ Through five levels (L1-L5) and common (Common) specifications, the system clari
Online & Offline (multiple deployment scenarios): real model, normal inference function, normal accuracy |
<30min |
- /tests/e2e/online_serving/test_{model_name}_expansion.py
- /tests/e2e/offline_inference/test_{model_name}_expansion.py
+ /tests/e2e/online_serving/test_{model_name}.py
+ /tests/e2e/offline_inference/test_{model_name}.py
|
Chapter 2
diff --git a/docs/design/feature/async_chunk_design.md b/docs/design/feature/async_chunk_design.md
index 40b8f215a..5ae5f5967 100644
--- a/docs/design/feature/async_chunk_design.md
+++ b/docs/design/feature/async_chunk_design.md
@@ -19,7 +19,7 @@ The `async_chunk` feature enables asynchronous, chunked processing of data acros
For qwen3-omni:
- **Thinker → Talker**: Per decode step (typically chunk_size=1)
-- **Talker → Code2Wav**: Accumulated to `codec_chunk_frames` (default=25) before sending. Set `initial_codec_chunk_frames` to emit smaller chunks during the initial phase for reduced TTFA
+- **Talker → Code2Wav**: Accumulated to `codec_chunk_frames` (default=25) before sending. During the initial phase, a dynamic initial chunk size (IC) is automatically selected based on server load to reduce TTFA. Use the per-request `initial_codec_chunk_frames` API field to override.
- **Code2Wav**: Streaming decode with code2wav chunk_size
With `async_chunk`:
diff --git a/docs/getting_started/installation/gpu/cuda.inc.md b/docs/getting_started/installation/gpu/cuda.inc.md
index 1e1737e43..d8585ad68 100644
--- a/docs/getting_started/installation/gpu/cuda.inc.md
+++ b/docs/getting_started/installation/gpu/cuda.inc.md
@@ -34,13 +34,13 @@ uv pip install vllm-omni
# --8<-- [start:build-wheel-from-source]
#### Installation of vLLM
-If you do not need to modify source code of vLLM, you can directly install the stable 0.16.0 release version of the library
+If you do not need to modify source code of vLLM, you can directly install the stable 0.17.0 release version of the library
```bash
-uv pip install vllm==0.16.0 --torch-backend=auto
+uv pip install vllm==0.17.0 --torch-backend=auto
```
-The release 0.16.0 of vLLM is based on PyTorch 2.9.0 which requires CUDA 12.9 environment.
+The release 0.17.0 of vLLM is based on PyTorch 2.10.0 which requires CUDA 12.9 environment.
#### Installation of vLLM-Omni
Since vllm-omni is rapidly evolving, it's recommended to install it from source
@@ -56,11 +56,11 @@ If you want to check, modify or debug with source code of vLLM, install the libr
```bash
git clone https://github.com/vllm-project/vllm.git
cd vllm
-git checkout v0.16.0
+git checkout v0.17.0
```
Set up environment variables to get pre-built wheels. If there are internet problems, just download the whl file manually. And set `VLLM_PRECOMPILED_WHEEL_LOCATION` as your local absolute path of whl file.
```bash
-export VLLM_PRECOMPILED_WHEEL_LOCATION=https://github.com/vllm-project/vllm/releases/download/v0.16.0/vllm-0.16.0-cp38-abi3-manylinux_2_31_x86_64.whl
+export VLLM_PRECOMPILED_WHEEL_LOCATION=https://github.com/vllm-project/vllm/releases/download/v0.17.0/vllm-0.17.0-cp38-abi3-manylinux_2_31_x86_64.whl
```
Install vllm with command below (If you have no existing PyTorch).
```bash
diff --git a/docs/getting_started/installation/gpu/rocm.inc.md b/docs/getting_started/installation/gpu/rocm.inc.md
index dc280bb94..701741cd0 100644
--- a/docs/getting_started/installation/gpu/rocm.inc.md
+++ b/docs/getting_started/installation/gpu/rocm.inc.md
@@ -13,7 +13,7 @@ vLLM-Omni current recommends the steps in under setup through Docker Images.
vLLM-Omni is built based on vLLM. Please install it with command below.
```bash
-uv pip install vllm==0.16.0+rocm700 --extra-index-url https://wheels.vllm.ai/rocm/0.16.0/rocm700
+uv pip install vllm==0.17.0+rocm700 --extra-index-url https://wheels.vllm.ai/rocm/0.17.0/rocm700
```
#### Installation of vLLM-Omni
@@ -34,13 +34,13 @@ uv pip install onnxruntime-rocm sox
# --8<-- [start:build-wheel-from-source]
#### Installation of vLLM
-If you do not need to modify source code of vLLM, you can directly install the stable 0.16.0 release version of the library
+If you do not need to modify source code of vLLM, you can directly install the stable 0.17.0 release version of the library
```bash
-uv pip install vllm==0.16.0+rocm700 --extra-index-url https://wheels.vllm.ai/rocm/0.16.0/rocm700
+uv pip install vllm==0.17.0+rocm700 --extra-index-url https://wheels.vllm.ai/rocm/0.17.0/rocm700
```
-The release 0.16.0 of vLLM requires ROCm 7.0 environment.
+The release 0.17.0 of vLLM requires ROCm 7.0 environment.
#### Installation of vLLM-Omni
Since vllm-omni is rapidly evolving, it's recommended to install it from source
@@ -58,7 +58,7 @@ If you want to check, modify or debug with source code of vLLM, install the libr
```bash
git clone https://github.com/vllm-project/vllm.git
cd vllm
-git checkout v0.16.0
+git checkout v0.17.0
python3 -m pip install -r requirements/rocm.txt
python3 setup.py develop
```
diff --git a/docs/getting_started/quickstart.md b/docs/getting_started/quickstart.md
index a3eece127..6d8197193 100644
--- a/docs/getting_started/quickstart.md
+++ b/docs/getting_started/quickstart.md
@@ -19,10 +19,10 @@ uv venv --python 3.12 --seed
source .venv/bin/activate
# On CUDA
-uv pip install vllm==0.16.0 --torch-backend=auto
+uv pip install vllm==0.17.0 --torch-backend=auto
# On ROCm
-uv pip install vllm==0.16.0 --extra-index-url https://wheels.vllm.ai/rocm/0.16.0/rocm700
+uv pip install vllm==0.17.0 --extra-index-url https://wheels.vllm.ai/rocm/0.17.0/rocm700
git clone https://github.com/vllm-project/vllm-omni.git
cd vllm-omni
diff --git a/docs/serving/audio_generate_api.md b/docs/serving/audio_generate_api.md
deleted file mode 100644
index e7eaef186..000000000
--- a/docs/serving/audio_generate_api.md
+++ /dev/null
@@ -1,338 +0,0 @@
-# Audio Generate API
-
-vLLM-Omni provides an API for text-to-audio generation using diffusion-based models such as Stable Audio.
-
-Unlike the [Speech API](speech_api.md) which targets text-to-speech synthesis, the Audio Generate API is designed for general-purpose audio generation from text descriptions (sound effects, music, ambient soundscapes, etc.).
-
-Each server instance runs a single model (specified at startup via `vllm-omni serve --omni`).
-
-## Quick Start
-
-### Start the Server
-
-```bash
-vllm-omni serve stabilityai/stable-audio-open-1.0 \
- --host 0.0.0.0 \
- --port 8000 \
- --gpu-memory-utilization 0.9 \
- --trust-remote-code \
- --enforce-eager \
- --omni
-```
-
-### Generate Audio
-
-**Using curl:**
-
-```bash
-curl -X POST http://localhost:8000/v1/audio/generate \
- -H "Content-Type: application/json" \
- -d '{
- "input": "The sound of a cat purring",
- "audio_length": 10.0
- }' --output cat.wav
-```
-
-**Using Python:**
-
-```python
-import httpx
-
-response = httpx.post(
- "http://localhost:8000/v1/audio/generate",
- json={
- "input": "The sound of a cat purring",
- "audio_length": 10.0,
- },
- timeout=300.0,
-)
-
-with open("cat.wav", "wb") as f:
- f.write(response.content)
-```
-
-## API Reference
-
-### Endpoint
-
-```
-POST /v1/audio/generate
-Content-Type: application/json
-```
-
-### Request Parameters
-
-| Parameter | Type | Default | Description |
-|-----------|------|---------|-------------|
-| `input` | string | **required** | Text prompt describing the audio to generate |
-| `model` | string | server's model | Model to use (optional, should match server if specified) |
-| `response_format` | string | "wav" | Audio format: wav, mp3, flac, pcm, aac, opus |
-| `speed` | float | 1.0 | Playback speed (0.25 - 4.0) |
-
-#### Diffusion Parameters
-
-| Parameter | Type | Default | Description |
-|-----------|------|---------|-------------|
-| `audio_length` | float | null | Audio duration in seconds (default value is the max ~47s for `stable-audio-open-1.0`) |
-| `audio_start` | float | 0.0 | Audio start time in seconds |
-| `negative_prompt` | string | null | Text describing what to avoid in generation |
-| `guidance_scale` | float | model default | Classifier-free guidance scale (higher = more adherence to prompt) |
-| `num_inference_steps` | int | model default | Number of denoising steps (higher = better quality, slower) |
-| `seed` | int | null | Random seed for reproducible generation |
-
-### Response Format
-
-Returns binary audio data with the appropriate `Content-Type` header:
-
-| `response_format` | Content-Type |
-|--------------------|--------------|
-| `wav` | `audio/wav` |
-| `mp3` | `audio/mpeg` |
-| `flac` | `audio/flac` |
-| `pcm` | `audio/pcm` |
-| `aac` | `audio/aac` |
-| `opus` | `audio/opus` |
-
-## Examples
-
-### Basic Generation
-
-Generate audio with only a text prompt (model defaults for all other parameters):
-
-```bash
-curl -X POST http://localhost:8000/v1/audio/generate \
- -H "Content-Type: application/json" \
- -d '{
- "input": "The sound of ocean waves crashing on a beach"
- }' --output ocean.wav
-```
-
-### Custom Duration
-
-Specify an explicit audio length in seconds:
-
-```bash
-curl -X POST http://localhost:8000/v1/audio/generate \
- -H "Content-Type: application/json" \
- -d '{
- "input": "A dog barking",
- "audio_length": 5.0
- }' --output dog_5s.wav
-```
-
-### High Quality with Negative Prompt
-
-Use a negative prompt to steer generation away from undesired characteristics, and increase inference steps for higher quality:
-
-```bash
-curl -X POST http://localhost:8000/v1/audio/generate \
- -H "Content-Type: application/json" \
- -d '{
- "input": "A piano playing a gentle melody",
- "audio_length": 10.0,
- "negative_prompt": "Low quality, distorted, noisy",
- "guidance_scale": 8.0,
- "num_inference_steps": 150
- }' --output piano_hq.wav
-```
-
-### Reproducible Generation
-
-Set a `seed` to get deterministic results across runs:
-
-```bash
-curl -X POST http://localhost:8000/v1/audio/generate \
- -H "Content-Type: application/json" \
- -d '{
- "input": "Thunder and rain sounds",
- "audio_length": 15.0,
- "seed": 42
- }' --output thunder.wav
-```
-
-### Full Control
-
-Combine all parameters for precise control over generation:
-
-```bash
-curl -X POST http://localhost:8000/v1/audio/generate \
- -H "Content-Type: application/json" \
- -d '{
- "input": "Thunder and rain sounds",
- "audio_length": 15.0,
- "negative_prompt": "Low quality",
- "guidance_scale": 7.0,
- "num_inference_steps": 100,
- "seed": 42
- }' --output thunder_rain.wav
-```
-
-### Quick Generation (Fewer Steps)
-
-For faster generation with slightly lower quality:
-
-```bash
-curl -X POST http://localhost:8000/v1/audio/generate \
- -H "Content-Type: application/json" \
- -d '{
- "input": "Birds chirping in a forest",
- "audio_length": 8.0,
- "num_inference_steps": 50
- }' --output birds_quick.wav
-```
-
-### Python Client
-
-```python
-import httpx
-
-response = httpx.post(
- "http://localhost:8000/v1/audio/generate",
- json={
- "input": "Thunder and rain",
- "audio_length": 15.0,
- "negative_prompt": "Low quality",
- "guidance_scale": 7.0,
- "num_inference_steps": 100,
- "seed": 42,
- "response_format": "wav",
- },
- timeout=300.0,
-)
-
-with open("thunder.wav", "wb") as f:
- f.write(response.content)
-```
-
-## Parameter Tuning Guide
-
-### `guidance_scale`
-
-Controls how closely the generated audio follows the text prompt.
-
-| Range | Behaviour |
-|-------|-----------|
-| 3 - 5 | More creative / varied output |
-| 7 (default) | Balanced adherence |
-| 10+ | Strict adherence to the prompt |
-
-### `num_inference_steps`
-
-Controls the number of denoising steps in the diffusion process.
-
-| Steps | Quality | Speed | Use Case |
-|-------|---------|-------|----------|
-| 50 | Good | Fast | Quick previews |
-| 100 | Very Good | Medium | General purpose |
-| 150+ | Excellent | Slow | Final / critical audio |
-
-### `audio_length`
-
-Duration of the generated audio clip. For `stable-audio-open-1.0`, the maximum is approximately 47 seconds. If omitted, the model uses its own default length.
-
-### `negative_prompt`
-
-Describes characteristics to avoid. Common negative prompts include:
-
-- `"Low quality, distorted, noisy"`
-- `"Silence, static"`
-- `"Music"` (when generating sound effects only)
-
-## Supported Models
-
-| Model | Description |
-|-------|-------------|
-| `stabilityai/stable-audio-open-1.0` | Open-source audio generation model, up to ~47 seconds, 44.1 kHz stereo |
-
-## Error Responses
-
-### 400 Bad Request
-
-Invalid or missing parameters:
-
-```json
-{
- "error": {
- "message": "Audio generation model did not produce audio output.",
- "type": "BadRequestError",
- "param": null,
- "code": 400
- }
-}
-```
-
-### 404 Not Found
-
-Model mismatch:
-
-```json
-{
- "error": {
- "message": "The model `xxx` does not exist.",
- "type": "NotFoundError",
- "param": "model",
- "code": 404
- }
-}
-```
-
-### 422 Unprocessable Entity
-
-Pydantic validation failure (e.g. invalid `response_format`, `speed` out of range):
-
-```json
-{
- "detail": [
- {
- "type": "literal_error",
- "msg": "Input should be 'wav', 'pcm', 'flac', 'mp3', 'aac' or 'opus'",
- ...
- }
- ]
-}
-```
-
-## Troubleshooting
-
-### "Audio generation model did not produce audio output"
-
-The model finished but returned no audio data. Verify the server started successfully and the model loaded without errors.
-
-### Server Not Responding
-
-```bash
-# Check if the server is healthy
-curl http://localhost:8000/health
-```
-
-### Audio Quality Issues
-
-- Increase `num_inference_steps` (e.g. 150).
-- Add a negative prompt: `"Low quality, distorted, noisy"`.
-- Increase `guidance_scale` for stronger prompt adherence.
-
-### Generation Timeout
-
-- Reduce `num_inference_steps`.
-- Reduce `audio_length`.
-- Check GPU memory with `nvidia-smi`.
-
-### Out of Memory
-
-- Lower `--gpu-memory-utilization` (e.g. 0.8).
-- Reduce `audio_length`.
-
-## Development
-
-Enable debug logging:
-
-```bash
-vllm-omni serve stabilityai/stable-audio-open-1.0 \
- --host 0.0.0.0 \
- --port 8000 \
- --gpu-memory-utilization 0.9 \
- --trust-remote-code \
- --enforce-eager \
- --omni \
- --uvicorn-log-level debug
-```
diff --git a/docs/serving/speech_api.md b/docs/serving/speech_api.md
index ca5f7a2ac..2f0695298 100644
--- a/docs/serving/speech_api.md
+++ b/docs/serving/speech_api.md
@@ -96,7 +96,7 @@ Content-Type: application/json
| `language` | string | "Auto" | Language (see supported languages below) |
| `instructions` | string | "" | Voice style/emotion instructions |
| `max_new_tokens` | integer | 2048 | Maximum tokens to generate |
-| `initial_codec_chunk_frames` | integer | null | Initial chunk size for reduced TTFA (overrides stage config) |
+| `initial_codec_chunk_frames` | integer | null | Per-request initial chunk size override for TTFA tuning. When null, IC is computed dynamically based on server load. |
**Supported languages:** Auto, Chinese, English, Japanese, Korean, German, French, Russian, Portuguese, Spanish, Italian
diff --git a/docs/user_guide/examples/offline_inference/qwen3_tts.md b/docs/user_guide/examples/offline_inference/qwen3_tts.md
index afadd39f4..352c076ad 100644
--- a/docs/user_guide/examples/offline_inference/qwen3_tts.md
+++ b/docs/user_guide/examples/offline_inference/qwen3_tts.md
@@ -99,7 +99,8 @@ python end2end.py --query-type CustomVoice --streaming --output-dir /tmp/out_str
```
Each Code2Wav chunk is logged as it arrives (default 25 frames; configurable via `codec_chunk_frames`
-and `initial_codec_chunk_frames` in the stage config). The final WAV file is written once generation
+in the stage config). The initial chunk size is dynamically selected based on server load for reduced
+TTFA, and can be overridden per-request via the `initial_codec_chunk_frames` API field. The final WAV file is written once generation
completes. This demonstrates that audio data is available progressively rather than only at the end.
> **Note:** Streaming uses `AsyncOmni` internally. The non-streaming path (`Omni`) is unchanged.
diff --git a/docs/user_guide/examples/online_serving/qwen3_tts.md b/docs/user_guide/examples/online_serving/qwen3_tts.md
index e3de00515..993505b87 100644
--- a/docs/user_guide/examples/online_serving/qwen3_tts.md
+++ b/docs/user_guide/examples/online_serving/qwen3_tts.md
@@ -17,6 +17,31 @@ Please refer to [README.md](https://github.com/vllm-project/vllm-omni/tree/main/
| `Qwen/Qwen3-TTS-12Hz-0.6B-CustomVoice` | CustomVoice | Smaller/faster variant |
| `Qwen/Qwen3-TTS-12Hz-0.6B-Base` | Base | Smaller/faster variant for voice cloning |
+## Gradio Demo
+
+Two interactive Gradio demos are available, both supporting all 3 task types:
+
+| Demo | File | Transport | Streaming Quality |
+| ---- | ---- | --------- | ----------------- |
+| Standard | `gradio_demo.py` | HTTP chunked | May have small gaps between chunks |
+| FastRTC | `gradio_fastrtc_demo.py` | WebRTC | Gapless streaming (requires `pip install fastrtc`) |
+
+```bash
+# Option 1: Launch server + Standard Gradio together
+./run_gradio_demo.sh # CustomVoice (default)
+./run_gradio_demo.sh --task-type VoiceDesign # VoiceDesign
+./run_gradio_demo.sh --task-type Base # Voice cloning
+
+# Option 2: If server is already running
+python gradio_demo.py --api-base http://localhost:8000
+
+# Option 3: FastRTC demo (gapless streaming)
+pip install fastrtc
+python gradio_fastrtc_demo.py --api-base http://localhost:8000
+```
+
+Then open http://localhost:7860 in your browser.
+
## Run examples (Qwen3-TTS)
### Launch the Server
@@ -112,7 +137,7 @@ The Python client supports the following command-line arguments:
- `--voice`: Speaker/voice name (default: `vivian`). Options: `vivian`, `ryan`, `aiden`, etc.
- `--language`: Language. Options: `Auto`, `Chinese`, `English`, `Japanese`, `Korean`, `German`, `French`, `Russian`, `Portuguese`, `Spanish`, `Italian`
- `--instructions`: Voice style/emotion instructions
-- `--ref-audio`: Reference audio file path or URL for voice cloning (Base task). Local paths are automatically base64-encoded by the client before sending to the server.
+- `--ref-audio`: Reference audio file path or URL for voice cloning (Base task)
- `--ref-text`: Reference audio transcript for voice cloning (Base task)
- `--response-format`: Audio output format (default: `wav`). Options: `wav`, `mp3`, `flac`, `pcm`, `aac`, `opus`
- `--output` (or `-o`): Output audio file path (default: `tts_output.wav`)
@@ -187,6 +212,59 @@ sudo apt install ffmpeg
## API Reference
+### Voices Endpoint
+
+#### GET /v1/audio/voices
+
+List all available voices/speakers from the loaded model, including both built-in model voices and uploaded custom voices.
+
+**Response Example:**
+```json
+{
+ "voices": ["vivian", "ryan", "custom_voice_1"],
+ "uploaded_voices": [
+ {
+ "name": "custom_voice_1",
+ "consent": "user_consent_id",
+ "created_at": 1738660000,
+ "file_size": 1024000,
+ "mime_type": "audio/wav"
+ }
+ ]
+}
+```
+
+#### POST /v1/audio/voices
+
+Upload a new voice sample for voice cloning in Base task TTS requests.
+
+**Form Parameters:**
+- `audio_sample` (required): Audio file (max 10MB, supported formats: wav, mp3, flac, ogg, aac, webm, mp4)
+- `consent` (required): Consent recording ID
+- `name` (required): Name for the new voice
+
+**Response Example:**
+```json
+{
+ "success": true,
+ "voice": {
+ "name": "custom_voice_1",
+ "consent": "user_consent_id",
+ "created_at": 1738660000,
+ "mime_type": "audio/wav",
+ "file_size": 1024000
+ }
+}
+```
+
+**Usage Example:**
+```bash
+curl -X POST http://localhost:8000/v1/audio/voices \
+ -F "audio_sample=@/path/to/voice_sample.wav" \
+ -F "consent=user_consent_id" \
+ -F "name=custom_voice_1"
+```
+
### Endpoint
```
@@ -196,20 +274,6 @@ Content-Type: application/json
This endpoint follows the [OpenAI Audio Speech API](https://platform.openai.com/docs/api-reference/audio/createSpeech) format with additional Qwen3-TTS parameters.
-### Voices Endpoint
-
-```
-GET /v1/audio/voices
-```
-
-Lists available voices for the loaded model:
-
-```json
-{
- "voices": ["aiden", "dylan", "eric", "one_anna", "ryan", "serena", "sohee", "uncle_fu", "vivian"]
-}
-```
-
### Request Body
```json
@@ -220,7 +284,7 @@ Lists available voices for the loaded model:
"task_type": "CustomVoice",
"language": "Auto",
"instructions": "Optional style instructions",
- "ref_audio": "HTTP URL, base64 data URL, or file:// URI for voice cloning",
+ "ref_audio": "URL or base64 for voice cloning",
"ref_text": "Reference audio transcript",
"x_vector_only_mode": false,
"max_new_tokens": 2048
@@ -253,6 +317,7 @@ Returns binary audio data with appropriate `Content-Type` header (e.g., `audio/w
| `language` | string | "Auto" | Language (see supported languages below) |
| `instructions` | string | "" | Voice style/emotion instructions |
| `max_new_tokens` | int | 2048 | Maximum tokens to generate |
+| `initial_codec_chunk_frames` | int | null | Per-request initial chunk size override for TTFA tuning. When null, IC is computed dynamically based on server load. |
| `stream` | bool | false | Stream raw PCM chunks as they are decoded (requires `response_format="pcm"`) |
**Supported languages:** Auto, Chinese, English, Japanese, Korean, German, French, Russian, Portuguese, Spanish, Italian
@@ -261,7 +326,7 @@ Returns binary audio data with appropriate `Content-Type` header (e.g., `audio/w
| Parameter | Type | Required | Description |
| -------------------- | ------ | -------- | -------------------------------------------- |
-| `ref_audio` | string | **Yes** | Reference audio (HTTP URL, base64 data URL, or `file://` URI with `--allowed-local-media-path`) |
+| `ref_audio` | string | **Yes** | Reference audio (URL or base64 data URL) |
| `ref_text` | string | No | Transcript of reference audio (for ICL mode) |
| `x_vector_only_mode` | bool | No | Use speaker embedding only (no ICL) |
@@ -301,11 +366,27 @@ curl -X POST http://localhost:8091/v1/audio/speech \
## Example materials
+??? abstract "gradio_demo.py"
+ ``````py
+ --8<-- "examples/online_serving/qwen3_tts/gradio_demo.py"
+ ``````
+??? abstract "gradio_fastrtc_demo.py"
+ ``````py
+ --8<-- "examples/online_serving/qwen3_tts/gradio_fastrtc_demo.py"
+ ``````
??? abstract "openai_speech_client.py"
``````py
--8<-- "examples/online_serving/qwen3_tts/openai_speech_client.py"
``````
+??? abstract "run_gradio_demo.sh"
+ ``````sh
+ --8<-- "examples/online_serving/qwen3_tts/run_gradio_demo.sh"
+ ``````
??? abstract "run_server.sh"
``````sh
--8<-- "examples/online_serving/qwen3_tts/run_server.sh"
``````
+??? abstract "tts_common.py"
+ ``````py
+ --8<-- "examples/online_serving/qwen3_tts/tts_common.py"
+ ``````
diff --git a/docs/user_guide/examples/online_serving/text_to_audio.md b/docs/user_guide/examples/online_serving/text_to_audio.md
deleted file mode 100644
index b84bde6c4..000000000
--- a/docs/user_guide/examples/online_serving/text_to_audio.md
+++ /dev/null
@@ -1,193 +0,0 @@
-# Text-To-Audio
-
-Source .
-
-This example demonstrates how to deploy Stable Audio models for online text-to-audio generation using vLLM-Omni.
-
-## Supported Models
-
-| Model | Description |
-|-------|-------------|
-| `stabilityai/stable-audio-open-1.0` | Open-source audio generation, up to ~47 seconds, 44.1 kHz stereo |
-
-## Start Server
-
-### Basic Start
-
-```bash
-vllm-omni serve stabilityai/stable-audio-open-1.0 \
- --host 0.0.0.0 \
- --port 8000 \
- --gpu-memory-utilization 0.9 \
- --trust-remote-code \
- --enforce-eager \
- --omni
-```
-
-## API Calls
-
-### Method 1: Using curl
-
-```bash
-# Run all curl examples
-bash curl_examples.sh
-
-# Or execute directly
-curl -X POST http://localhost:8000/v1/audio/generate \
- -H "Content-Type: application/json" \
- -d '{
- "input": "The sound of a cat purring",
- "audio_length": 10.0
- }' --output cat.wav
-```
-
-### Method 2: Using Python Client
-
-```bash
-cd examples/online_serving/stable_audio
-
-# Simple generation
-python stable_audio_client.py \
- --text "The sound of a cat purring"
-
-# With custom duration
-python stable_audio_client.py \
- --text "A dog barking" \
- --audio_length 5.0
-
-# With all parameters
-python stable_audio_client.py \
- --text "Thunder and rain" \
- --audio_length 15.0 \
- --negative_prompt "Low quality" \
- --guidance_scale 7.0 \
- --num_inference_steps 100 \
- --seed 42 \
- --output thunder.wav
-```
-
-The Python client supports the following command-line arguments:
-
-- `--api_url`: API endpoint URL (default: `http://localhost:8000/v1/audio/generate`)
-- `--text`: Text prompt for audio generation (default: `"The sound of a cat purring"`)
-- `--audio_length`: Audio length in seconds (default: `10.0`, max ~47s for `stable-audio-open-1.0`)
-- `--audio_start`: Audio start time in seconds (default: `0.0`)
-- `--negative_prompt`: Negative prompt for classifier-free guidance (default: `"Low quality"`)
-- `--guidance_scale`: Guidance scale for diffusion (default: `7.0`)
-- `--num_inference_steps`: Number of inference steps (default: `100`)
-- `--seed`: Random seed for reproducibility (default: `None`)
-- `--response_format`: Audio output format (default: `wav`). Options: `wav`, `mp3`, `flac`, `pcm`
-- `--output`: Output file path (default: `stable_audio_output.wav`)
-
-### Method 3: Using Python httpx
-
-```python
-import httpx
-
-response = httpx.post(
- "http://localhost:8000/v1/audio/generate",
- json={
- "input": "The sound of ocean waves crashing on a beach",
- "audio_length": 10.0,
- "negative_prompt": "Low quality, distorted",
- "guidance_scale": 7.0,
- "num_inference_steps": 100,
- },
- timeout=300.0,
-)
-
-with open("ocean.wav", "wb") as f:
- f.write(response.content)
-```
-
-## Request Format
-
-### Simple Generation
-
-```json
-{
- "input": "The sound of ocean waves"
-}
-```
-
-### Generation with Parameters
-
-```json
-{
- "input": "A piano playing a gentle melody",
- "audio_length": 10.0,
- "negative_prompt": "Low quality, distorted, noisy",
- "guidance_scale": 8.0,
- "num_inference_steps": 150,
- "seed": 42,
- "response_format": "wav"
-}
-```
-
-## API Reference
-
-### Endpoint
-
-```
-POST /v1/audio/generate
-Content-Type: application/json
-```
-
-### Generation Parameters
-
-| Parameter | Type | Default | Description |
-|-----------|------|---------|-------------|
-| `input` | string | **required** | Text prompt describing the audio to generate |
-| `model` | string | server's model | Model to use (optional, should match server if specified) |
-| `response_format` | string | "wav" | Audio format: wav, mp3, flac, pcm, aac, opus |
-| `speed` | float | 1.0 | Playback speed (0.25 - 4.0) |
-| `audio_length` | float | null | Audio duration in seconds (max ~47s for `stable-audio-open-1.0`) |
-| `audio_start` | float | 0.0 | Audio start time in seconds |
-| `negative_prompt` | string | null | Text describing what to avoid in generation |
-| `guidance_scale` | float | model default | Classifier-free guidance scale (higher = more adherence to prompt) |
-| `num_inference_steps` | int | model default | Number of denoising steps (higher = better quality, slower) |
-| `seed` | int | null | Random seed for reproducible generation |
-
-### Response Format
-
-Returns binary audio data with appropriate `Content-Type` header (e.g., `audio/wav`).
-
-## Tuning Tips
-
-1. **Audio Length**: Keep under 47 seconds for `stable-audio-open-1.0`.
-2. **Quality vs Speed**:
- - 50 steps: Fast, decent quality (quick previews)
- - 100 steps: Good balance (general purpose)
- - 150+ steps: High quality, slower (final / critical audio)
-3. **Guidance Scale**:
- - Lower (3 - 5): More creative / varied output
- - Default (7): Good balance
- - Higher (10+): Strict adherence to the prompt
-4. **Negative Prompts**: Use to avoid unwanted characteristics such as `"Low quality"`, `"distorted"`, `"noisy"`.
-5. **Seeds**: Set a fixed seed to get deterministic, reproducible results.
-
-## File Description
-
-| File | Description |
-|------|-------------|
-| `curl_examples.sh` | Curl examples covering common use cases |
-| `stable_audio_client.py` | Python client with full CLI argument support |
-
-## Troubleshooting
-
-1. **Audio generation model did not produce audio output**: Verify the server started successfully and the model loaded without errors.
-2. **Connection refused**: Make sure the server is running on the correct port.
-3. **Generation timeout**: Reduce `num_inference_steps` or `audio_length`, and check GPU memory with `nvidia-smi`.
-4. **Out of memory**: Lower `--gpu-memory-utilization` or reduce `audio_length`.
-5. **Audio quality issues**: Increase `num_inference_steps`, add a negative prompt, or raise `guidance_scale`.
-
-## Example materials
-
-??? abstract "stable_audio_client.py"
- ``````py
- --8<-- "examples/online_serving/stable_audio/stable_audio_client.py"
- ``````
-??? abstract "curl_examples.sh"
- ``````sh
- --8<-- "examples/online_serving/stable_audio/curl_examples.sh"
- ``````
diff --git a/examples/offline_inference/qwen3_tts/README.md b/examples/offline_inference/qwen3_tts/README.md
index c198f2c6e..ce2dd588e 100644
--- a/examples/offline_inference/qwen3_tts/README.md
+++ b/examples/offline_inference/qwen3_tts/README.md
@@ -96,7 +96,8 @@ python end2end.py --query-type CustomVoice --streaming --output-dir /tmp/out_str
```
Each Code2Wav chunk is logged as it arrives (default 25 frames; configurable via `codec_chunk_frames`
-and `initial_codec_chunk_frames` in the stage config). The final WAV file is written once generation
+in the stage config). The initial chunk size is dynamically selected based on server load for reduced
+TTFA, and can be overridden per-request via the `initial_codec_chunk_frames` API field. The final WAV file is written once generation
completes. This demonstrates that audio data is available progressively rather than only at the end.
> **Note:** Streaming uses `AsyncOmni` internally. The non-streaming path (`Omni`) is unchanged.
diff --git a/examples/online_serving/qwen3_tts/README.md b/examples/online_serving/qwen3_tts/README.md
index 5088a4ca6..99442c29e 100644
--- a/examples/online_serving/qwen3_tts/README.md
+++ b/examples/online_serving/qwen3_tts/README.md
@@ -14,6 +14,31 @@ Please refer to [README.md](https://github.com/vllm-project/vllm-omni/tree/main/
| `Qwen/Qwen3-TTS-12Hz-0.6B-CustomVoice` | CustomVoice | Smaller/faster variant |
| `Qwen/Qwen3-TTS-12Hz-0.6B-Base` | Base | Smaller/faster variant for voice cloning |
+## Gradio Demo
+
+Two interactive Gradio demos are available, both supporting all 3 task types:
+
+| Demo | File | Transport | Streaming Quality |
+| ---- | ---- | --------- | ----------------- |
+| Standard | `gradio_demo.py` | HTTP chunked | May have small gaps between chunks |
+| FastRTC | `gradio_fastrtc_demo.py` | WebRTC | Gapless streaming (requires `pip install fastrtc`) |
+
+```bash
+# Option 1: Launch server + Standard Gradio together
+./run_gradio_demo.sh # CustomVoice (default)
+./run_gradio_demo.sh --task-type VoiceDesign # VoiceDesign
+./run_gradio_demo.sh --task-type Base # Voice cloning
+
+# Option 2: If server is already running
+python gradio_demo.py --api-base http://localhost:8000
+
+# Option 3: FastRTC demo (gapless streaming)
+pip install fastrtc
+python gradio_fastrtc_demo.py --api-base http://localhost:8000
+```
+
+Then open http://localhost:7860 in your browser.
+
## Run examples (Qwen3-TTS)
### Launch the Server
@@ -289,7 +314,7 @@ Returns binary audio data with appropriate `Content-Type` header (e.g., `audio/w
| `language` | string | "Auto" | Language (see supported languages below) |
| `instructions` | string | "" | Voice style/emotion instructions |
| `max_new_tokens` | int | 2048 | Maximum tokens to generate |
-| `initial_codec_chunk_frames` | int | null | Initial chunk size for reduced TTFA (overrides stage config) |
+| `initial_codec_chunk_frames` | int | null | Per-request initial chunk size override for TTFA tuning. When null, IC is computed dynamically based on server load. |
| `stream` | bool | false | Stream raw PCM chunks as they are decoded (requires `response_format="pcm"`) |
**Supported languages:** Auto, Chinese, English, Japanese, Korean, German, French, Russian, Portuguese, Spanish, Italian
diff --git a/examples/online_serving/qwen3_tts/gradio_demo.py b/examples/online_serving/qwen3_tts/gradio_demo.py
new file mode 100644
index 000000000..b8a54c6a6
--- /dev/null
+++ b/examples/online_serving/qwen3_tts/gradio_demo.py
@@ -0,0 +1,404 @@
+"""Gradio demo for Qwen3-TTS online serving via /v1/audio/speech API.
+
+Uses Gradio's built-in audio component for playback. Supports streaming
+(progressive PCM chunks) and non-streaming (full audio download) modes.
+
+For gapless streaming playback, see gradio_fastrtc_demo.py which uses WebRTC.
+
+Supports all 3 task types:
+ - CustomVoice: Predefined speaker with optional style instructions
+ - VoiceDesign: Natural language voice description
+ - Base: Voice cloning from reference audio (upload or URL)
+
+Usage:
+ # Start the server first (see run_server.sh), then:
+ python gradio_demo.py --api-base http://localhost:8000
+
+ # Or use run_gradio_demo.sh to start both server and demo together.
+"""
+
+import argparse
+import io
+
+import gradio as gr
+import httpx
+import numpy as np
+import soundfile as sf
+from tts_common import (
+ PCM_SAMPLE_RATE,
+ SUPPORTED_LANGUAGES,
+ TASK_TYPES,
+ add_common_args,
+ build_payload,
+ fetch_voices,
+ on_task_type_change,
+ stream_pcm_chunks,
+)
+
+
+def generate_speech(
+ api_base: str,
+ text: str,
+ task_type: str,
+ voice: str,
+ language: str,
+ instructions: str,
+ ref_audio: tuple | None,
+ ref_audio_url: str,
+ ref_text: str,
+ x_vector_only: bool,
+ response_format: str,
+ speed: float,
+):
+ """Non-streaming: call /v1/audio/speech and return audio."""
+ payload = build_payload(
+ text,
+ task_type,
+ voice,
+ language,
+ instructions,
+ ref_audio,
+ ref_audio_url,
+ ref_text,
+ x_vector_only,
+ response_format,
+ speed,
+ stream=False,
+ )
+
+ try:
+ with httpx.Client(timeout=300.0) as client:
+ resp = client.post(
+ f"{api_base}/v1/audio/speech",
+ json=payload,
+ headers={
+ "Content-Type": "application/json",
+ "Authorization": "Bearer EMPTY",
+ },
+ )
+ except httpx.TimeoutException:
+ raise gr.Error("Request timed out. The server may be busy.")
+ except httpx.ConnectError:
+ raise gr.Error(f"Cannot connect to server at {api_base}. Make sure the vLLM server is running.")
+
+ if resp.status_code != 200:
+ raise gr.Error(f"Server error ({resp.status_code}): {resp.text}")
+
+ content_type = resp.headers.get("content-type", "")
+ if "application/json" in content_type:
+ try:
+ error_data = resp.json()
+ raise gr.Error(f"Server error: {error_data}")
+ except ValueError:
+ pass
+
+ try:
+ if response_format == "pcm":
+ audio_np = np.frombuffer(resp.content, dtype=np.int16).astype(np.float32) / 32767.0
+ return (PCM_SAMPLE_RATE, audio_np)
+ audio_np, sample_rate = sf.read(io.BytesIO(resp.content))
+ if audio_np.ndim > 1:
+ audio_np = audio_np[:, 0]
+ return (sample_rate, audio_np.astype(np.float32))
+ except Exception as e:
+ raise gr.Error(f"Failed to decode audio response: {e}")
+
+
+def generate_speech_stream(
+ api_base: str,
+ text: str,
+ task_type: str,
+ voice: str,
+ language: str,
+ instructions: str,
+ ref_audio: tuple | None,
+ ref_audio_url: str,
+ ref_text: str,
+ x_vector_only: bool,
+ response_format: str,
+ speed: float,
+ chunk_seconds: float = 0.25,
+):
+ """Streaming: yield individual PCM chunks for progressive playback."""
+ payload = build_payload(
+ text,
+ task_type,
+ voice,
+ language,
+ instructions,
+ ref_audio,
+ ref_audio_url,
+ ref_text,
+ x_vector_only,
+ response_format,
+ speed,
+ stream=True,
+ )
+
+ min_chunk_samples = int(PCM_SAMPLE_RATE * chunk_seconds)
+ # Pre-buffer 2 chunks before first playback to avoid the gap
+ # between chunk 1 and chunk 2 during progressive streaming.
+ prebuffer_chunks = 2
+ pending: list[np.ndarray] = []
+ pending_len = 0
+ chunks_yielded = 0
+ prebuffer: list[np.ndarray] = []
+
+ try:
+ for samples in stream_pcm_chunks(api_base, payload):
+ float_samples = samples.astype(np.float32) / 32767.0
+ pending.append(float_samples)
+ pending_len += len(float_samples)
+ if pending_len >= min_chunk_samples:
+ audio_chunk = np.concatenate(pending)
+ pending.clear()
+ pending_len = 0
+
+ if chunks_yielded < prebuffer_chunks:
+ prebuffer.append(audio_chunk)
+ chunks_yielded += 1
+ if chunks_yielded == prebuffer_chunks:
+ yield (PCM_SAMPLE_RATE, np.concatenate(prebuffer))
+ prebuffer.clear()
+ else:
+ yield (PCM_SAMPLE_RATE, audio_chunk)
+ # Flush: prebuffer (if stream was shorter than 2 chunks) + remaining pending
+ remaining = prebuffer + pending
+ if remaining:
+ yield (PCM_SAMPLE_RATE, np.concatenate(remaining))
+ except httpx.TimeoutException:
+ raise gr.Error("Request timed out. The server may be busy.")
+ except httpx.ConnectError:
+ raise gr.Error(f"Cannot connect to server at {api_base}. Make sure the vLLM server is running.")
+
+
+def on_stream_change(stream: bool):
+ """When streaming is enabled, lock format to PCM and disable speed."""
+ if stream:
+ return (
+ gr.update(value="pcm", interactive=False),
+ gr.update(interactive=False),
+ )
+ return (
+ gr.update(value="wav", interactive=True),
+ gr.update(interactive=True),
+ )
+
+
+def build_interface(api_base: str, stream_chunk_seconds: float = 0.25):
+ """Build the Gradio interface."""
+ voices = fetch_voices(api_base)
+
+ css = """
+ #generate-btn button { width: 100%; }
+ .task-info { padding: 8px 12px; border-radius: 6px;
+ background: #f0f4ff; margin-bottom: 8px; }
+ """
+
+ with gr.Blocks(css=css, title="Qwen3-TTS Demo") as demo:
+ gr.Markdown("# Qwen3-TTS Online Serving Demo")
+ gr.Markdown(f"**Server:** `{api_base}`")
+
+ with gr.Row():
+ # Left column: inputs
+ with gr.Column(scale=3):
+ text_input = gr.Textbox(
+ label="Text to Synthesize",
+ placeholder="Enter text here, e.g., Hello, how are you?",
+ lines=4,
+ )
+
+ with gr.Row():
+ task_type = gr.Radio(
+ choices=TASK_TYPES,
+ value="CustomVoice",
+ label="Task Type",
+ scale=2,
+ )
+ language = gr.Dropdown(
+ choices=SUPPORTED_LANGUAGES,
+ value="Auto",
+ label="Language",
+ scale=1,
+ )
+
+ voice = gr.Dropdown(
+ choices=voices,
+ value=voices[0] if voices else None,
+ label="Speaker",
+ visible=True,
+ )
+
+ instructions = gr.Textbox(
+ label="Instructions",
+ placeholder=("e.g., Speak with excitement / A warm, friendly female voice"),
+ lines=2,
+ visible=True,
+ info="Optional style/emotion instructions",
+ )
+
+ ref_audio = gr.Audio(
+ label="Reference Audio (upload for voice cloning)",
+ type="numpy",
+ sources=["upload", "microphone"],
+ visible=False,
+ )
+ ref_audio_url = gr.Textbox(
+ label="Reference Audio URL",
+ placeholder=("https://example.com/reference.wav (alternative to uploading)"),
+ lines=1,
+ visible=False,
+ )
+ ref_text = gr.Textbox(
+ label="Reference Audio Transcript",
+ placeholder=("Transcript of the reference audio (optional, improves quality)"),
+ lines=2,
+ visible=False,
+ )
+ x_vector_only = gr.Checkbox(
+ label="Use x-vector only",
+ value=False,
+ visible=False,
+ info=("Skip reference transcript, use speaker embedding only (lower quality)"),
+ )
+
+ with gr.Row():
+ response_format = gr.Dropdown(
+ choices=["wav", "mp3", "flac", "pcm", "aac", "opus"],
+ value="wav",
+ label="Audio Format",
+ scale=1,
+ )
+ speed = gr.Slider(
+ minimum=0.25,
+ maximum=4.0,
+ value=1.0,
+ step=0.05,
+ label="Speed",
+ scale=1,
+ )
+ stream_checkbox = gr.Checkbox(
+ label="Stream output",
+ value=False,
+ info=(
+ "Enable streaming (PCM format, speed disabled). "
+ "For gapless streaming, use gradio_fastrtc_demo.py"
+ ),
+ scale=1,
+ )
+
+ generate_btn = gr.Button(
+ "Generate Speech",
+ variant="primary",
+ size="lg",
+ elem_id="generate-btn",
+ )
+
+ # Right column: output
+ with gr.Column(scale=2):
+ audio_output = gr.Audio(
+ label="Generated Audio",
+ interactive=False,
+ streaming=True,
+ autoplay=True,
+ )
+ gr.Markdown(
+ "### Task Types\n"
+ "- **CustomVoice**: Use a predefined speaker "
+ "(Vivian, Ryan, etc.) with optional style instructions\n"
+ "- **VoiceDesign**: Describe the desired voice in natural "
+ "language (instructions required)\n"
+ "- **Base**: Clone a voice from reference audio "
+ "(upload a file or provide a URL)"
+ )
+
+ # Dynamic UI updates
+ task_type.change(
+ fn=on_task_type_change,
+ inputs=[task_type],
+ outputs=[
+ voice,
+ instructions,
+ ref_audio,
+ ref_audio_url,
+ ref_text,
+ x_vector_only,
+ ],
+ )
+
+ stream_checkbox.change(
+ fn=on_stream_change,
+ inputs=[stream_checkbox],
+ outputs=[response_format, speed],
+ )
+
+ all_inputs = [
+ text_input,
+ task_type,
+ voice,
+ language,
+ instructions,
+ ref_audio,
+ ref_audio_url,
+ ref_text,
+ x_vector_only,
+ response_format,
+ speed,
+ ]
+
+ def dispatch(stream_enabled, *args):
+ if stream_enabled:
+ yield from generate_speech_stream(
+ api_base,
+ *args,
+ chunk_seconds=stream_chunk_seconds,
+ )
+ else:
+ yield generate_speech(api_base, *args)
+
+ generate_btn.click(
+ fn=dispatch,
+ inputs=[stream_checkbox] + all_inputs,
+ outputs=[audio_output],
+ )
+
+ demo.queue()
+ return demo
+
+
+def parse_args():
+ parser = argparse.ArgumentParser(description="Gradio demo for Qwen3-TTS online serving.")
+ add_common_args(parser)
+ parser.add_argument(
+ "--stream-chunk-seconds",
+ type=float,
+ default=0.25,
+ help=(
+ "Seconds of audio to buffer per streaming chunk (default: 0.25). "
+ "Lower = smoother but more overhead, higher = chunkier."
+ ),
+ )
+ return parser.parse_args()
+
+
+def main():
+ args = parse_args()
+
+ print(f"Connecting to vLLM server at: {args.api_base}")
+ demo = build_interface(
+ args.api_base,
+ stream_chunk_seconds=args.stream_chunk_seconds,
+ )
+
+ try:
+ demo.launch(
+ server_name=args.host,
+ server_port=args.port,
+ share=args.share,
+ )
+ except KeyboardInterrupt:
+ print("\nShutting down...")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/examples/online_serving/qwen3_tts/gradio_fastrtc_demo.py b/examples/online_serving/qwen3_tts/gradio_fastrtc_demo.py
new file mode 100644
index 000000000..a40ebfd89
--- /dev/null
+++ b/examples/online_serving/qwen3_tts/gradio_fastrtc_demo.py
@@ -0,0 +1,274 @@
+"""Gradio demo for Qwen3-TTS using FastRTC for gapless streaming audio.
+
+Uses WebRTC via FastRTC to deliver audio chunks as a continuous real-time
+stream, avoiding the inter-chunk gaps present in Gradio's built-in streaming.
+
+Requires: pip install fastrtc
+
+Supports all 3 task types:
+ - CustomVoice: Predefined speaker with optional style instructions
+ - VoiceDesign: Natural language voice description
+ - Base: Voice cloning from reference audio (URL only in this demo)
+
+Usage:
+ # Start the server first (see run_server.sh), then:
+ python gradio_fastrtc_demo.py --api-base http://localhost:8000
+
+ # With public sharing:
+ python gradio_fastrtc_demo.py --api-base http://localhost:8000 --share
+"""
+
+import argparse
+import logging
+from collections.abc import Generator
+
+import gradio as gr
+import numpy as np
+from fastrtc import WebRTC
+from tts_common import (
+ PCM_SAMPLE_RATE,
+ SUPPORTED_LANGUAGES,
+ TASK_TYPES,
+ add_common_args,
+ build_payload,
+ fetch_voices,
+ stream_pcm_chunks,
+)
+
+logger = logging.getLogger(__name__)
+
+
+def make_tts_generator(api_base: str):
+ """Create a generator function for FastRTC's mode='receive'.
+
+ FastRTC calls this with the additional_inputs values whenever the user
+ clicks "Start Stream". It must be a synchronous generator yielding
+ (sample_rate, int16_ndarray) tuples.
+ """
+
+ def tts_generate(
+ text: str,
+ task_type: str,
+ voice: str,
+ language: str,
+ instructions: str,
+ ref_audio_url: str,
+ ref_text: str,
+ x_vector_only: bool,
+ ) -> Generator[tuple[int, np.ndarray], None, None]:
+ if not text or not text.strip():
+ return
+
+ payload = build_payload(
+ text,
+ task_type,
+ voice,
+ language,
+ instructions,
+ ref_audio=None, # WebRTC UI doesn't support file upload
+ ref_audio_url=ref_audio_url,
+ ref_text=ref_text,
+ x_vector_only=x_vector_only,
+ stream=True,
+ )
+
+ try:
+ for samples in stream_pcm_chunks(api_base, payload):
+ yield (PCM_SAMPLE_RATE, samples)
+ except Exception:
+ logger.exception("TTS streaming error")
+
+ return tts_generate
+
+
+def build_interface(api_base: str):
+ """Build a custom Gradio Blocks UI matching gradio_demo.py layout."""
+ voices = fetch_voices(api_base)
+ handler = make_tts_generator(api_base)
+
+ css = """
+ #generate-btn button { width: 100%; }
+ .task-info { padding: 8px 12px; border-radius: 6px;
+ background: #f0f4ff; margin-bottom: 8px; }
+ """
+
+ with gr.Blocks(css=css, title="Qwen3-TTS (FastRTC)") as demo:
+ gr.Markdown("# Qwen3-TTS Online Serving Demo (FastRTC)")
+ gr.Markdown(f"**Server:** `{api_base}` | **Transport:** WebRTC (gapless streaming)")
+
+ with gr.Row():
+ # Left column: inputs
+ with gr.Column(scale=3):
+ text_input = gr.Textbox(
+ label="Text to Synthesize",
+ placeholder="Enter text here, e.g., Hello, how are you?",
+ lines=4,
+ )
+
+ with gr.Row():
+ task_type = gr.Radio(
+ choices=TASK_TYPES,
+ value="CustomVoice",
+ label="Task Type",
+ scale=2,
+ )
+ language = gr.Dropdown(
+ choices=SUPPORTED_LANGUAGES,
+ value="Auto",
+ label="Language",
+ scale=1,
+ )
+
+ voice = gr.Dropdown(
+ choices=voices,
+ value=voices[0] if voices else None,
+ label="Speaker",
+ visible=True,
+ )
+
+ instructions = gr.Textbox(
+ label="Instructions",
+ placeholder=("e.g., Speak with excitement / A warm, friendly female voice"),
+ lines=2,
+ visible=True,
+ info="Optional style/emotion instructions",
+ )
+
+ # Base (voice clone) controls
+ # Note: file upload is not supported with WebRTC,
+ # use a URL instead.
+ ref_audio_url = gr.Textbox(
+ label="Reference Audio URL",
+ placeholder=("https://example.com/reference.wav (alternative to uploading)"),
+ lines=1,
+ visible=False,
+ )
+ ref_text = gr.Textbox(
+ label="Reference Audio Transcript",
+ placeholder=("Transcript of the reference audio (optional, improves quality)"),
+ lines=2,
+ visible=False,
+ )
+ x_vector_only = gr.Checkbox(
+ label="Use x-vector only",
+ value=False,
+ visible=False,
+ info=("Skip reference transcript, use speaker embedding only (lower quality)"),
+ )
+
+ generate_btn = gr.Button(
+ "Generate Speech",
+ variant="primary",
+ size="lg",
+ elem_id="generate-btn",
+ )
+
+ # Right column: output
+ with gr.Column(scale=2):
+ webrtc_output = WebRTC(
+ label="Generated Audio (WebRTC)",
+ mode="receive",
+ modality="audio",
+ )
+ gr.Markdown(
+ "### Task Types\n"
+ "- **CustomVoice**: Use a predefined speaker "
+ "(Vivian, Ryan, etc.) with optional style instructions\n"
+ "- **VoiceDesign**: Describe the desired voice in natural "
+ "language (instructions required)\n"
+ "- **Base**: Clone a voice from reference audio "
+ "(provide a URL)"
+ )
+
+ # Dynamic UI updates for task type
+ # FastRTC doesn't support file upload, so we only toggle
+ # 5 components (no ref_audio)
+ def on_task_type_change_fastrtc(task_type: str):
+ if task_type == "CustomVoice":
+ return (
+ gr.update(visible=True), # voice
+ gr.update(visible=True, info="Optional style/emotion instructions"),
+ gr.update(visible=False), # ref_audio_url
+ gr.update(visible=False), # ref_text
+ gr.update(visible=False), # x_vector_only
+ )
+ elif task_type == "VoiceDesign":
+ return (
+ gr.update(visible=False),
+ gr.update(visible=True, info="Required: describe the voice style"),
+ gr.update(visible=False),
+ gr.update(visible=False),
+ gr.update(visible=False),
+ )
+ elif task_type == "Base":
+ return (
+ gr.update(visible=False),
+ gr.update(visible=False),
+ gr.update(visible=True),
+ gr.update(visible=True),
+ gr.update(visible=True),
+ )
+ return (
+ gr.update(visible=True),
+ gr.update(visible=True),
+ gr.update(visible=False),
+ gr.update(visible=False),
+ gr.update(visible=False),
+ )
+
+ task_type.change(
+ fn=on_task_type_change_fastrtc,
+ inputs=[task_type],
+ outputs=[
+ voice,
+ instructions,
+ ref_audio_url,
+ ref_text,
+ x_vector_only,
+ ],
+ )
+
+ # Wire up WebRTC streaming
+ all_inputs = [
+ text_input,
+ task_type,
+ voice,
+ language,
+ instructions,
+ ref_audio_url,
+ ref_text,
+ x_vector_only,
+ ]
+
+ webrtc_output.stream(
+ fn=handler,
+ inputs=all_inputs,
+ outputs=[webrtc_output],
+ trigger=generate_btn.click,
+ )
+
+ demo.queue()
+ return demo
+
+
+def parse_args():
+ parser = argparse.ArgumentParser(description="Gradio FastRTC demo for Qwen3-TTS (gapless streaming).")
+ add_common_args(parser)
+ return parser.parse_args()
+
+
+def main():
+ args = parse_args()
+ logging.basicConfig(level=logging.INFO)
+ print(f"Connecting to vLLM server at: {args.api_base}")
+
+ demo = build_interface(args.api_base)
+ demo.launch(
+ server_name=args.host,
+ server_port=args.port,
+ share=args.share,
+ )
+
+
+if __name__ == "__main__":
+ main()
diff --git a/examples/online_serving/qwen3_tts/run_gradio_demo.sh b/examples/online_serving/qwen3_tts/run_gradio_demo.sh
new file mode 100644
index 000000000..1c7fe9e37
--- /dev/null
+++ b/examples/online_serving/qwen3_tts/run_gradio_demo.sh
@@ -0,0 +1,203 @@
+#!/bin/bash
+# Launch both vLLM server and Gradio demo for Qwen3-TTS
+#
+# Usage:
+# ./run_gradio_demo.sh # Default: CustomVoice
+# ./run_gradio_demo.sh --task-type VoiceDesign # VoiceDesign model
+# ./run_gradio_demo.sh --task-type Base --gradio-port 7861
+#
+# Options:
+# --task-type TYPE Task type: CustomVoice, VoiceDesign, Base (default: CustomVoice)
+# --server-port PORT Port for vLLM server (default: 8000)
+# --gradio-port PORT Port for Gradio demo (default: 7860)
+# --server-host HOST Host for vLLM server (default: 0.0.0.0)
+# --gradio-ip IP IP for Gradio demo (default: 127.0.0.1)
+# --share Share Gradio demo publicly
+
+set -e
+
+# Default values
+TASK_TYPE="CustomVoice"
+SERVER_PORT=8000
+GRADIO_PORT=7860
+SERVER_HOST="0.0.0.0"
+GRADIO_IP="127.0.0.1"
+GRADIO_SHARE=false
+
+# Parse command line arguments
+while [[ $# -gt 0 ]]; do
+ case $1 in
+ --task-type)
+ TASK_TYPE="$2"
+ shift 2
+ ;;
+ --server-port)
+ SERVER_PORT="$2"
+ shift 2
+ ;;
+ --gradio-port)
+ GRADIO_PORT="$2"
+ shift 2
+ ;;
+ --server-host)
+ SERVER_HOST="$2"
+ shift 2
+ ;;
+ --gradio-ip)
+ GRADIO_IP="$2"
+ shift 2
+ ;;
+ --share)
+ GRADIO_SHARE=true
+ shift
+ ;;
+ --help)
+ echo "Usage: $0 [OPTIONS]"
+ echo ""
+ echo "Options:"
+ echo " --task-type TYPE Task type: CustomVoice, VoiceDesign, Base (default: CustomVoice)"
+ echo " --server-port PORT Port for vLLM server (default: 8000)"
+ echo " --gradio-port PORT Port for Gradio demo (default: 7860)"
+ echo " --server-host HOST Host for vLLM server (default: 0.0.0.0)"
+ echo " --gradio-ip IP IP for Gradio demo (default: 127.0.0.1)"
+ echo " --share Share Gradio demo publicly"
+ echo ""
+ exit 0
+ ;;
+ *)
+ echo "Unknown option: $1"
+ echo "Use --help for usage information"
+ exit 1
+ ;;
+ esac
+done
+
+# Map task type to model
+case "$TASK_TYPE" in
+ CustomVoice)
+ MODEL="Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice"
+ ;;
+ VoiceDesign)
+ MODEL="Qwen/Qwen3-TTS-12Hz-1.7B-VoiceDesign"
+ ;;
+ Base)
+ MODEL="Qwen/Qwen3-TTS-12Hz-1.7B-Base"
+ ;;
+ *)
+ echo "Unknown task type: $TASK_TYPE"
+ echo "Supported: CustomVoice, VoiceDesign, Base"
+ exit 1
+ ;;
+esac
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+API_BASE="http://localhost:${SERVER_PORT}"
+
+echo "=========================================="
+echo "Qwen3-TTS Gradio Demo"
+echo "=========================================="
+echo "Task Type : $TASK_TYPE"
+echo "Model : $MODEL"
+echo "Server : http://${SERVER_HOST}:${SERVER_PORT}"
+echo "Gradio : http://${GRADIO_IP}:${GRADIO_PORT}"
+echo "=========================================="
+
+# Cleanup on exit
+cleanup() {
+ echo ""
+ echo "Shutting down..."
+ if [ -n "$SERVER_PID" ]; then
+ echo "Stopping vLLM server (PID: $SERVER_PID)..."
+ kill "$SERVER_PID" 2>/dev/null || true
+ wait "$SERVER_PID" 2>/dev/null || true
+ fi
+ if [ -n "$GRADIO_PID" ]; then
+ echo "Stopping Gradio demo (PID: $GRADIO_PID)..."
+ kill "$GRADIO_PID" 2>/dev/null || true
+ wait "$GRADIO_PID" 2>/dev/null || true
+ fi
+ echo "Cleanup complete"
+ exit 0
+}
+trap cleanup SIGINT SIGTERM
+
+# Start vLLM server
+echo ""
+echo "Starting vLLM server..."
+LOG_FILE="/tmp/vllm_tts_server_${SERVER_PORT}.log"
+
+vllm-omni serve "$MODEL" \
+ --stage-configs-path vllm_omni/model_executor/stage_configs/qwen3_tts.yaml \
+ --host "$SERVER_HOST" \
+ --port "$SERVER_PORT" \
+ --gpu-memory-utilization 0.9 \
+ --trust-remote-code \
+ --enforce-eager \
+ --omni 2>&1 | tee "$LOG_FILE" &
+SERVER_PID=$!
+
+# Wait for server startup
+echo ""
+echo "Waiting for vLLM server to be ready..."
+STARTUP_FLAG="/tmp/vllm_tts_startup_flag_${SERVER_PORT}.tmp"
+rm -f "$STARTUP_FLAG"
+
+(
+ tail -f "$LOG_FILE" 2>/dev/null | grep -m 1 "Application startup complete" > /dev/null && touch "$STARTUP_FLAG"
+) &
+TAIL_PID=$!
+
+MAX_WAIT=300
+ELAPSED=0
+while [ $ELAPSED -lt $MAX_WAIT ]; do
+ if [ -f "$STARTUP_FLAG" ]; then
+ kill "$TAIL_PID" 2>/dev/null || true
+ wait "$TAIL_PID" 2>/dev/null || true
+ echo ""
+ echo "vLLM server is ready!"
+ break
+ fi
+ if ! kill -0 "$SERVER_PID" 2>/dev/null; then
+ kill "$TAIL_PID" 2>/dev/null || true
+ echo ""
+ echo "Error: vLLM server failed to start"
+ exit 1
+ fi
+ sleep 1
+ ELAPSED=$((ELAPSED + 1))
+done
+
+rm -f "$STARTUP_FLAG"
+
+if [ $ELAPSED -ge $MAX_WAIT ]; then
+ kill "$TAIL_PID" 2>/dev/null || true
+ echo "Error: Server startup timed out after ${MAX_WAIT}s"
+ kill "$SERVER_PID" 2>/dev/null || true
+ exit 1
+fi
+
+# Start Gradio demo
+echo ""
+echo "Starting Gradio demo..."
+cd "$SCRIPT_DIR"
+GRADIO_CMD=("python" "gradio_demo.py" "--api-base" "$API_BASE" "--ip" "$GRADIO_IP" "--port" "$GRADIO_PORT")
+if [ "$GRADIO_SHARE" = true ]; then
+ GRADIO_CMD+=("--share")
+fi
+
+"${GRADIO_CMD[@]}" &
+GRADIO_PID=$!
+
+echo ""
+echo "=========================================="
+echo "Both services are running!"
+echo "=========================================="
+echo "vLLM Server : http://${SERVER_HOST}:${SERVER_PORT}"
+echo "Gradio Demo : http://${GRADIO_IP}:${GRADIO_PORT}"
+echo ""
+echo "Press Ctrl+C to stop both services"
+echo "=========================================="
+echo ""
+
+wait $SERVER_PID $GRADIO_PID || true
+cleanup
diff --git a/examples/online_serving/qwen3_tts/tts_common.py b/examples/online_serving/qwen3_tts/tts_common.py
new file mode 100644
index 000000000..8e1c77c39
--- /dev/null
+++ b/examples/online_serving/qwen3_tts/tts_common.py
@@ -0,0 +1,218 @@
+"""Shared constants, helpers, and payload building for Qwen3-TTS Gradio demos."""
+
+import base64
+import io
+
+import gradio as gr
+import httpx
+import numpy as np
+import soundfile as sf
+
+SUPPORTED_LANGUAGES = [
+ "Auto",
+ "Chinese",
+ "English",
+ "Japanese",
+ "Korean",
+ "German",
+ "French",
+ "Russian",
+ "Portuguese",
+ "Spanish",
+ "Italian",
+]
+
+TASK_TYPES = ["CustomVoice", "VoiceDesign", "Base"]
+
+PCM_SAMPLE_RATE = 24000
+
+DEFAULT_API_BASE = "http://localhost:8000"
+
+
+def fetch_voices(api_base: str) -> list[str]:
+ """Fetch available voices from the server."""
+ try:
+ with httpx.Client(timeout=10.0) as client:
+ resp = client.get(
+ f"{api_base}/v1/audio/voices",
+ headers={"Authorization": "Bearer EMPTY"},
+ )
+ if resp.status_code == 200:
+ data = resp.json()
+ return data.get("voices", ["Vivian", "Ryan"])
+ except Exception:
+ pass
+ return ["Vivian", "Ryan"]
+
+
+def encode_audio_to_base64(audio_data: tuple) -> str:
+ """Encode Gradio audio input (sample_rate, numpy_array) to base64 data URL."""
+ sample_rate, audio_np = audio_data
+
+ if audio_np.dtype != np.int16:
+ if audio_np.dtype in (np.float32, np.float64):
+ audio_np = np.clip(audio_np, -1.0, 1.0)
+ audio_np = (audio_np * 32767).astype(np.int16)
+ else:
+ audio_np = audio_np.astype(np.int16)
+
+ buf = io.BytesIO()
+ sf.write(buf, audio_np, sample_rate, format="WAV")
+ wav_b64 = base64.b64encode(buf.getvalue()).decode("utf-8")
+ return f"data:audio/wav;base64,{wav_b64}"
+
+
+def build_payload(
+ text: str,
+ task_type: str,
+ voice: str,
+ language: str,
+ instructions: str,
+ ref_audio: tuple | None,
+ ref_audio_url: str,
+ ref_text: str,
+ x_vector_only: bool,
+ response_format: str = "pcm",
+ speed: float = 1.0,
+ stream: bool = True,
+) -> dict:
+ """Build the /v1/audio/speech request payload.
+
+ Raises gr.Error for invalid input so callers don't need to validate.
+ """
+ if not text or not text.strip():
+ raise gr.Error("Please enter text to synthesize.")
+
+ payload: dict = {
+ "input": text.strip(),
+ "response_format": "pcm" if stream else response_format,
+ "stream": stream,
+ }
+ if not stream:
+ payload["speed"] = speed
+
+ if task_type:
+ payload["task_type"] = task_type
+ if language:
+ payload["language"] = language
+
+ if task_type == "CustomVoice":
+ if voice:
+ payload["voice"] = voice
+ if instructions and instructions.strip():
+ payload["instructions"] = instructions.strip()
+
+ elif task_type == "VoiceDesign":
+ if not instructions or not instructions.strip():
+ raise gr.Error("VoiceDesign task requires voice style instructions.")
+ payload["instructions"] = instructions.strip()
+
+ elif task_type == "Base":
+ ref_audio_url_stripped = ref_audio_url.strip() if ref_audio_url else ""
+ if ref_audio_url_stripped:
+ payload["ref_audio"] = ref_audio_url_stripped
+ elif ref_audio is not None:
+ payload["ref_audio"] = encode_audio_to_base64(ref_audio)
+ else:
+ raise gr.Error("Base (voice clone) task requires reference audio. Upload a file or provide a URL.")
+ if ref_text and ref_text.strip():
+ payload["ref_text"] = ref_text.strip()
+ if x_vector_only:
+ payload["x_vector_only_mode"] = True
+
+ return payload
+
+
+def on_task_type_change(task_type: str):
+ """Update UI visibility based on selected task type."""
+ if task_type == "CustomVoice":
+ return (
+ gr.update(visible=True), # voice dropdown
+ gr.update(visible=True, info="Optional style/emotion instructions"),
+ gr.update(visible=False), # ref_audio
+ gr.update(visible=False), # ref_audio_url
+ gr.update(visible=False), # ref_text
+ gr.update(visible=False), # x_vector_only
+ )
+ elif task_type == "VoiceDesign":
+ return (
+ gr.update(visible=False),
+ gr.update(visible=True, info="Required: describe the voice style"),
+ gr.update(visible=False),
+ gr.update(visible=False),
+ gr.update(visible=False),
+ gr.update(visible=False),
+ )
+ elif task_type == "Base":
+ return (
+ gr.update(visible=False),
+ gr.update(visible=False),
+ gr.update(visible=True),
+ gr.update(visible=True),
+ gr.update(visible=True),
+ gr.update(visible=True),
+ )
+ return (
+ gr.update(visible=True),
+ gr.update(visible=True),
+ gr.update(visible=False),
+ gr.update(visible=False),
+ gr.update(visible=False),
+ gr.update(visible=False),
+ )
+
+
+def stream_pcm_chunks(api_base: str, payload: dict):
+ """Stream raw PCM bytes from the server, yielding int16 numpy arrays.
+
+ Handles odd-byte boundaries between network chunks.
+ """
+ leftover = b""
+ with httpx.Client(timeout=300.0) as client:
+ with client.stream(
+ "POST",
+ f"{api_base}/v1/audio/speech",
+ json=payload,
+ headers={
+ "Content-Type": "application/json",
+ "Authorization": "Bearer EMPTY",
+ },
+ ) as resp:
+ if resp.status_code != 200:
+ resp.read()
+ raise gr.Error(f"Server error ({resp.status_code}): {resp.text}")
+ for chunk in resp.iter_bytes():
+ if not chunk:
+ continue
+ raw = leftover + chunk
+ usable = len(raw) - (len(raw) % 2)
+ leftover = raw[usable:]
+ if usable == 0:
+ continue
+ yield np.frombuffer(raw[:usable], dtype=np.int16).copy()
+
+
+def add_common_args(parser):
+ """Add CLI arguments shared by both demos."""
+ parser.add_argument(
+ "--api-base",
+ default=DEFAULT_API_BASE,
+ help=f"Base URL for the vLLM API server (default: {DEFAULT_API_BASE}).",
+ )
+ parser.add_argument(
+ "--host",
+ default="0.0.0.0",
+ help="Host/IP for Gradio server (default: 0.0.0.0).",
+ )
+ parser.add_argument(
+ "--port",
+ type=int,
+ default=7860,
+ help="Port for Gradio server (default: 7860).",
+ )
+ parser.add_argument(
+ "--share",
+ action="store_true",
+ help="Share the Gradio demo publicly.",
+ )
+ return parser
diff --git a/examples/online_serving/stable_audio/README.md b/examples/online_serving/stable_audio/README.md
deleted file mode 100644
index c8ffaadca..000000000
--- a/examples/online_serving/stable_audio/README.md
+++ /dev/null
@@ -1,234 +0,0 @@
-# Stable Audio Online Serving
-
-Generate audio from text prompts using Stable Audio models via an OpenAI-compatible API endpoint.
-
-## Features
-
-- **OpenAI-compatible API**: Use `/v1/audio/generate` endpoint
-- **Flexible control**: Adjust audio length, guidance scale, inference steps
-- **Quality control**: Use negative prompts to avoid unwanted characteristics
-- **Reproducible**: Set random seed for deterministic generation
-
-## Quick Start
-
-### 1. Start the Server
-
-```bash
-vllm-omni serve stabilityai/stable-audio-open-1.0 \
- --host 0.0.0.0 \
- --port 8000 \
- --gpu-memory-utilization 0.9 \
- --trust-remote-code \
- --enforce-eager \
- --omni
-```
-
-### 2. Generate Audio
-
-#### Using curl
-
-```bash
-curl -X POST http://localhost:8000/v1/audio/generate \
- -H "Content-Type: application/json" \
- -d '{
- "input": "The sound of a cat purring",
- "audio_length": 10.0
- }' --output cat.wav
-```
-
-#### Using Python Client
-
-```bash
-python stable_audio_client.py \
- --text "The sound of a cat purring" \
- --audio_length 10.0 \
- --output cat.wav
-```
-
-#### Using Bash Script
-
-```bash
-bash curl_examples.sh
-```
-
-## API Reference
-
-### Endpoint
-
-```
-POST /v1/audio/generate
-```
-
-### Request Body
-
-```json
-{
- "input": "Text description of the audio",
- "audio_length": 10.0,
- "audio_start": 0.0,
- "negative_prompt": "Low quality",
- "guidance_scale": 7.0,
- "num_inference_steps": 100,
- "seed": 42,
- "response_format": "wav"
-}
-```
-
-### Parameters
-
-| Parameter | Type | Default | Description |
-|-----------|------|---------|-------------|
-| `input` | string | **required** | Text prompt describing the audio to generate |
-| `audio_length` | float | ~47s | Audio duration in seconds (max ~47s for stable-audio-open-1.0) |
-| `audio_start` | float | 0.0 | Audio start time in seconds |
-| `negative_prompt` | string | null | Text describing what to avoid in generation |
-| `guidance_scale` | float | 7.0 | Classifier-free guidance scale (higher = more adherence to prompt) |
-| `num_inference_steps` | int | 50 | Number of denoising steps (higher = better quality, slower) |
-| `seed` | int | null | Random seed for reproducibility |
-| `response_format` | string | "wav" | Output format: wav, mp3, flac, pcm |
-
-### Response
-
-Returns audio data in the requested format (default: WAV).
-
-## Usage Examples
-
-### Basic Generation
-
-```bash
-curl -X POST http://localhost:8000/v1/audio/generate \
- -H "Content-Type: application/json" \
- -d '{
- "input": "The sound of ocean waves"
- }' --output ocean.wav
-```
-
-### Custom Duration
-
-```bash
-curl -X POST http://localhost:8000/v1/audio/generate \
- -H "Content-Type: application/json" \
- -d '{
- "input": "A dog barking",
- "audio_length": 5.0
- }' --output dog_5s.wav
-```
-
-### High Quality with Negative Prompt
-
-```bash
-curl -X POST http://localhost:8000/v1/audio/generate \
- -H "Content-Type: application/json" \
- -d '{
- "input": "A piano playing a gentle melody",
- "audio_length": 10.0,
- "negative_prompt": "Low quality, distorted, noisy",
- "guidance_scale": 8.0,
- "num_inference_steps": 150
- }' --output piano_hq.wav
-```
-
-### Reproducible Generation
-
-```bash
-curl -X POST http://localhost:8000/v1/audio/generate \
- -H "Content-Type: application/json" \
- -d '{
- "input": "Thunder and rain sounds",
- "audio_length": 15.0,
- "seed": 42
- }' --output thunder.wav
-```
-
-### Quick Generation (Fewer Steps)
-
-For faster generation with slightly lower quality:
-
-```bash
-curl -X POST http://localhost:8000/v1/audio/generate \
- -H "Content-Type: application/json" \
- -d '{
- "input": "Birds chirping in a forest",
- "audio_length": 8.0,
- "num_inference_steps": 50
- }' --output birds_quick.wav
-```
-
-## Python Client Examples
-
-### Simple Generation
-
-```bash
-python stable_audio_client.py \
- --text "The sound of a cat purring"
-```
-
-### Custom Parameters
-
-```bash
-python stable_audio_client.py \
- --text "Thunder and rain" \
- --audio_length 15.0 \
- --negative_prompt "Low quality" \
- --guidance_scale 7.0 \
- --num_inference_steps 100 \
- --seed 42 \
- --output thunder.wav
-```
-
-### Different Output Format
-
-```bash
-python stable_audio_client.py \
- --text "Guitar playing" \
- --response_format mp3 \
- --output guitar.mp3
-```
-
-## Tips
-
-1. **Audio Length**: Keep under 47 seconds for `stable-audio-open-1.0`
-2. **Quality vs Speed**:
- - 50 steps: Fast, decent quality
- - 100 steps: Good balance (default)
- - 150+ steps: High quality, slower
-3. **Guidance Scale**:
- - Lower (3-5): More creative/varied
- - Default (7): Good balance
- - Higher (10+): More literal to prompt
-4. **Negative Prompts**: Use to avoid "Low quality", "distorted", "noisy", etc.
-5. **Seeds**: Use same seed for reproducible results
-
-## Performance
-
-| Inference Steps | Quality | Speed | Use Case |
-|----------------|---------|-------|----------|
-| 50 | Good | Fast | Quick previews |
-| 100 (default) | Very Good | Medium | Production |
-| 150+ | Excellent | Slow | Final/critical audio |
-
-## Troubleshooting
-
-### Server not responding
-- Check if server is running: `curl http://localhost:8000/health`
-- Check server logs for errors
-
-### Audio quality issues
-- Increase `num_inference_steps` (e.g., 150)
-- Add negative prompts: `"Low quality, distorted, noisy"`
-- Increase `guidance_scale` for more prompt adherence
-
-### Generation timeout
-- Reduce `num_inference_steps`
-- Reduce `audio_length`
-- Check GPU memory with `nvidia-smi`
-
-### Wrong audio length
-- Ensure `audio_length` is within model limits (~47s max)
-- Adjust `audio_start` if trimming is needed
-
-## See Also
-
-- [Offline Inference Example](../../offline_inference/text_to_audio/README.md)
-- [Stable Audio Model Card](https://huggingface.co/stabilityai/stable-audio-open-1.0)
-- [vLLM-Omni Documentation](https://github.com/vllm-project/vllm-omni)
diff --git a/examples/online_serving/stable_audio/curl_examples.sh b/examples/online_serving/stable_audio/curl_examples.sh
deleted file mode 100755
index 8c4b0c846..000000000
--- a/examples/online_serving/stable_audio/curl_examples.sh
+++ /dev/null
@@ -1,54 +0,0 @@
-#!/bin/bash
-# Examples for using Stable Audio with curl via /v1/audio/generate endpoint
-
-# Example 1: Simple request with default parameters
-echo "Example 1: Simple request with default parameters"
-curl -X POST http://localhost:8000/v1/audio/generate \
- -H "Content-Type: application/json" \
- -d '{
- "input": "The sound audience clapping and cheering in a stadium"
- }' --output stadium.wav
-
-# Example 2: Request with custom audio_length
-echo "Example 2: Custom audio length (5 seconds)"
-curl -X POST http://localhost:8000/v1/audio/generate \
- -H "Content-Type: application/json" \
- -d '{
- "input": "The sound of a dog barking",
- "audio_length": 5.0
- }' --output dog_5s.wav
-
-# Example 3: Request with negative prompt for quality control
-echo "Example 3: With negative prompt"
-curl -X POST http://localhost:8000/v1/audio/generate \
- -H "Content-Type: application/json" \
- -d '{
- "input": "A piano playing a gentle melody",
- "audio_length": 10.0,
- "negative_prompt": "Low quality, distorted, noisy"
- }' --output piano.wav
-
-# Example 4: Full control with all parameters
-echo "Example 4: Full control (custom length, guidance, steps, seed)"
-curl -X POST http://localhost:8000/v1/audio/generate \
- -H "Content-Type: application/json" \
- -d '{
- "input": "Thunder and rain sounds",
- "audio_length": 15.0,
- "negative_prompt": "Low quality",
- "guidance_scale": 7.0,
- "num_inference_steps": 100,
- "seed": 42
- }' --output thunder_rain.wav
-
-# Example 5: Quick generation with fewer steps (faster but lower quality)
-echo "Example 5: Quick generation (fewer steps)"
-curl -X POST http://localhost:8000/v1/audio/generate \
- -H "Content-Type: application/json" \
- -d '{
- "input": "Ocean waves crashing on a beach",
- "audio_length": 8.0,
- "num_inference_steps": 50
- }' --output ocean.wav
-
-echo "All examples completed!"
diff --git a/examples/online_serving/stable_audio/stable_audio_client.py b/examples/online_serving/stable_audio/stable_audio_client.py
deleted file mode 100755
index 8e6a59d36..000000000
--- a/examples/online_serving/stable_audio/stable_audio_client.py
+++ /dev/null
@@ -1,170 +0,0 @@
-#!/usr/bin/env python3
-"""
-OpenAI-compatible client for Stable Audio via /v1/audio/generate endpoint.
-
-This script demonstrates how to use the OpenAI-compatible speech API
-to generate audio from text using Stable Audio models.
-
-Examples:
- # Simple generation
- python stable_audio_client.py --text "The sound of a cat purring"
-
- # With custom duration
- python stable_audio_client.py --text "A dog barking" --audio_length 5.0
-
- # With all parameters
- python stable_audio_client.py --text "Thunder and rain" \
- --audio_length 15.0 \
- --negative_prompt "Low quality" \
- --guidance_scale 7.0 \
- --num_inference_steps 100 \
- --seed 42 \
- --output thunder.wav
-"""
-
-import argparse
-import sys
-
-import requests
-
-
-def parse_args():
- parser = argparse.ArgumentParser(description="Generate audio with Stable Audio via OpenAI-compatible API")
- parser.add_argument(
- "--api_url",
- default="http://localhost:8000/v1/audio/generate",
- help="API endpoint URL",
- )
- parser.add_argument(
- "--text",
- default="The sound of a cat purring",
- help="Text prompt for audio generation",
- )
- parser.add_argument(
- "--audio_length",
- type=float,
- default=10.0,
- help="Audio length in seconds (max ~47s for stable-audio-open-1.0)",
- )
- parser.add_argument(
- "--audio_start",
- type=float,
- default=0.0,
- help="Audio start time in seconds",
- )
- parser.add_argument(
- "--negative_prompt",
- default="Low quality",
- help="Negative prompt for classifier-free guidance",
- )
- parser.add_argument(
- "--guidance_scale",
- type=float,
- default=7.0,
- help="Guidance scale for diffusion (higher = more adherence to prompt)",
- )
- parser.add_argument(
- "--num_inference_steps",
- type=int,
- default=100,
- help="Number of inference steps (higher = better quality, slower)",
- )
- parser.add_argument(
- "--seed",
- type=int,
- default=None,
- help="Random seed for reproducibility",
- )
- parser.add_argument(
- "--output",
- default="stable_audio_output.wav",
- help="Output file path",
- )
- parser.add_argument(
- "--response_format",
- default="wav",
- choices=["wav", "mp3", "flac", "pcm"],
- help="Audio output format",
- )
- return parser.parse_args()
-
-
-def generate_audio(args):
- """Generate audio using the API."""
-
- # Build request payload
- payload = {
- "input": args.text,
- "audio_length": args.audio_length,
- "audio_start": args.audio_start,
- "response_format": args.response_format,
- }
-
- # Add optional parameters
- if args.negative_prompt:
- payload["negative_prompt"] = args.negative_prompt
- if args.guidance_scale:
- payload["guidance_scale"] = args.guidance_scale
- if args.num_inference_steps:
- payload["num_inference_steps"] = args.num_inference_steps
- if args.seed is not None:
- payload["seed"] = args.seed
-
- print(f"\n{'=' * 60}")
- print("Stable Audio - Text-to-Audio Generation")
- print(f"{'=' * 60}")
- print(f"API URL: {args.api_url}")
- print(f"Prompt: {args.text}")
- print(f"Audio length: {args.audio_length}s")
- print(f"Negative prompt: {args.negative_prompt}")
- print(f"Guidance scale: {args.guidance_scale}")
- print(f"Inference steps: {args.num_inference_steps}")
- if args.seed is not None:
- print(f"Seed: {args.seed}")
- print(f"Output: {args.output}")
- print(f"{'=' * 60}\n")
-
- try:
- # Make the API request
- print("Generating audio...")
- response = requests.post(
- args.api_url,
- json=payload,
- headers={"Content-Type": "application/json"},
- timeout=300, # 5 minute timeout for long generations
- )
-
- # Check for errors
- if response.status_code != 200:
- print(f"Error: API returned status code {response.status_code}")
- print(f"Response: {response.text}")
- return False
-
- # Save the audio
- with open(args.output, "wb") as f:
- f.write(response.content)
-
- print(f"✓ Audio saved to {args.output}")
- print(f" File size: {len(response.content) / 1024:.1f} KB")
- return True
-
- except requests.exceptions.Timeout:
- print("Error: Request timed out. Try reducing inference steps or audio length.")
- return False
- except requests.exceptions.ConnectionError:
- print(f"Error: Could not connect to {args.api_url}")
- print("Make sure the server is running.")
- return False
- except Exception as e:
- print(f"Error: {e}")
- return False
-
-
-def main():
- args = parse_args()
- success = generate_audio(args)
- sys.exit(0 if success else 1)
-
-
-if __name__ == "__main__":
- main()
diff --git a/tests/conftest.py b/tests/conftest.py
index c775ae349..619dae3b8 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -1057,15 +1057,16 @@ class OmniServer:
max_wait = 1200 # 20 minutes
start_time = time.time()
while time.time() - start_time < max_wait:
- try:
- with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
- sock.settimeout(1)
- result = sock.connect_ex((self.host, self.port))
- if result == 0:
- print(f"Server ready on {self.host}:{self.port}")
- return
- except Exception:
- pass
+ # Check for process status
+ ret = self.proc.poll()
+ if ret is not None:
+ raise RuntimeError(f"Server processes exited with code {ret} before becoming ready.")
+ with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
+ sock.settimeout(1)
+ result = sock.connect_ex((self.host, self.port))
+ if result == 0:
+ print(f"Server ready on {self.host}:{self.port}")
+ return
time.sleep(2)
raise RuntimeError(f"Server failed to start within {max_wait} seconds")
diff --git a/tests/diffusion/distributed/test_hsdp.py b/tests/diffusion/distributed/test_hsdp.py
index 6fcaf1e81..e6d782c0a 100644
--- a/tests/diffusion/distributed/test_hsdp.py
+++ b/tests/diffusion/distributed/test_hsdp.py
@@ -11,7 +11,7 @@ import torch.nn as nn
from vllm_omni.diffusion.data import DiffusionParallelConfig
from vllm_omni.diffusion.distributed.hsdp import HSDPInferenceConfig
-pytestmark = [pytest.mark.diffusion, pytest.mark.parallel, pytest.mark.cpu]
+pytestmark = [pytest.mark.diffusion, pytest.mark.parallel, pytest.mark.cpu, pytest.mark.core_model]
class TestHSDPInferenceConfig:
diff --git a/tests/diffusion/distributed/test_parallel_state_sp_groups.py b/tests/diffusion/distributed/test_parallel_state_sp_groups.py
index ac8b1700a..9dda5610b 100644
--- a/tests/diffusion/distributed/test_parallel_state_sp_groups.py
+++ b/tests/diffusion/distributed/test_parallel_state_sp_groups.py
@@ -11,10 +11,7 @@ import torch
from vllm_omni.diffusion.distributed.parallel_state import RankGenerator, set_seq_parallel_pg
-pytestmark = [
- pytest.mark.diffusion,
- pytest.mark.parallel,
-]
+pytestmark = [pytest.mark.diffusion, pytest.mark.parallel, pytest.mark.core_model, pytest.mark.cpu]
def _fake_new_group_factory(created_groups: list[SimpleNamespace]):
diff --git a/tests/diffusion/distributed/test_sp_plan_hooks.py b/tests/diffusion/distributed/test_sp_plan_hooks.py
index 6883ae09f..72ec9aaf2 100644
--- a/tests/diffusion/distributed/test_sp_plan_hooks.py
+++ b/tests/diffusion/distributed/test_sp_plan_hooks.py
@@ -47,6 +47,8 @@ requires_distributed = pytest.mark.skipif(
pytestmark = [
pytest.mark.diffusion,
pytest.mark.parallel,
+ pytest.mark.core_model,
+ pytest.mark.cpu,
]
# =============================================================================
diff --git a/tests/diffusion/distributed/test_vae_patch_parallel.py b/tests/diffusion/distributed/test_vae_patch_parallel.py
index 43bbc71d5..db74fff1b 100644
--- a/tests/diffusion/distributed/test_vae_patch_parallel.py
+++ b/tests/diffusion/distributed/test_vae_patch_parallel.py
@@ -8,6 +8,8 @@ import torch
from vllm_omni.diffusion.distributed import vae_patch_parallel as vae_patch_parallel
+pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
+
class _DummyConfig:
def __init__(self, **attrs):
diff --git a/tests/diffusion/models/nextstep_1_1/test_nextstep_cfg_parallel_layout.py b/tests/diffusion/models/nextstep_1_1/test_nextstep_cfg_parallel_layout.py
index 9caf667be..dd87d2249 100644
--- a/tests/diffusion/models/nextstep_1_1/test_nextstep_cfg_parallel_layout.py
+++ b/tests/diffusion/models/nextstep_1_1/test_nextstep_cfg_parallel_layout.py
@@ -8,6 +8,8 @@ import vllm_omni.diffusion.models.nextstep_1_1.pipeline_nextstep_1_1 as nextstep
from vllm_omni.diffusion.models.nextstep_1_1.modeling_nextstep_heads import FlowMatchingHead
from vllm_omni.diffusion.models.nextstep_1_1.pipeline_nextstep_1_1 import NextStep11Pipeline
+pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
+
class _DummyImageHead:
def __init__(self, token_dim: int):
diff --git a/tests/diffusion/models/z_image/test_zimage_tp_constraints.py b/tests/diffusion/models/z_image/test_zimage_tp_constraints.py
index a276274a0..b87d51c36 100644
--- a/tests/diffusion/models/z_image/test_zimage_tp_constraints.py
+++ b/tests/diffusion/models/z_image/test_zimage_tp_constraints.py
@@ -2,6 +2,8 @@ import pytest
from vllm_omni.diffusion.models.z_image.z_image_transformer import validate_zimage_tp_constraints
+pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
+
def test_validate_zimage_tp_constraints_tp2_ok():
ffn_hidden_dim, final_out_dims, supported_tp = validate_zimage_tp_constraints(
diff --git a/tests/diffusion/test_multiproc_executor_concurrency.py b/tests/diffusion/test_multiproc_executor_concurrency.py
index ee3b8f4b8..76caa9844 100644
--- a/tests/diffusion/test_multiproc_executor_concurrency.py
+++ b/tests/diffusion/test_multiproc_executor_concurrency.py
@@ -12,7 +12,7 @@ from vllm_omni.diffusion.data import DiffusionOutput
from vllm_omni.diffusion.executor.multiproc_executor import MultiprocDiffusionExecutor
from vllm_omni.diffusion.scheduler import Scheduler
-pytestmark = [pytest.mark.diffusion]
+pytestmark = [pytest.mark.diffusion, pytest.mark.core_model, pytest.mark.cpu]
# ───────────────────────────────────────────── helpers ─────────────────────
diff --git a/tests/distributed/omni_connectors/test_mooncake_transfer_engine_buffer.py b/tests/distributed/omni_connectors/test_mooncake_transfer_engine_buffer.py
index 968109976..55351efcb 100755
--- a/tests/distributed/omni_connectors/test_mooncake_transfer_engine_buffer.py
+++ b/tests/distributed/omni_connectors/test_mooncake_transfer_engine_buffer.py
@@ -17,7 +17,7 @@ from vllm_omni.distributed.omni_connectors.connectors.mooncake_transfer_engine_c
)
# All tests in this file are pure-CPU unit tests for the memory allocator.
-pytestmark = [pytest.mark.cpu, pytest.mark.parallel]
+pytestmark = [pytest.mark.cpu, pytest.mark.parallel, pytest.mark.core_model]
@pytest.mark.core_model
diff --git a/tests/distributed/omni_connectors/test_omni_connector_configs.py b/tests/distributed/omni_connectors/test_omni_connector_configs.py
index ca73e1ec4..3dc80b57b 100644
--- a/tests/distributed/omni_connectors/test_omni_connector_configs.py
+++ b/tests/distributed/omni_connectors/test_omni_connector_configs.py
@@ -8,6 +8,8 @@ import pytest
# Use the new import path for initialization utilities
from vllm_omni.distributed.omni_connectors.utils.initialization import load_omni_transfer_config
+pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
+
def get_config_files():
"""Helper to find config files."""
diff --git a/tests/entrypoints/openai_api/test_serving_audio_generate.py b/tests/entrypoints/openai_api/test_serving_audio_generate.py
deleted file mode 100644
index fcf22bcef..000000000
--- a/tests/entrypoints/openai_api/test_serving_audio_generate.py
+++ /dev/null
@@ -1,509 +0,0 @@
-# tests/entrypoints/openai_api/test_serving_audio_generate.py
-import logging
-from inspect import Signature, signature
-from unittest.mock import MagicMock, patch
-
-import pytest
-import torch
-from fastapi import FastAPI
-from fastapi.testclient import TestClient
-
-from vllm_omni.entrypoints.openai.protocol.audio import (
- CreateAudio,
- OpenAICreateAudioGenerateRequest,
-)
-from vllm_omni.entrypoints.openai.serving_audio_generate import (
- OmniOpenAIServingAudioGenerate,
-)
-from vllm_omni.inputs.data import OmniDiffusionSamplingParams
-from vllm_omni.outputs import OmniRequestOutput
-
-pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
-
-logger = logging.getLogger(__name__)
-
-
-# Helper: create a mock audio output for endpoint tests
-def create_mock_audio_output(
- request_id: str = "audiogen-mock-123",
- sample_rate: int = 44100,
- num_samples: int = 44100,
- audio_key: str = "audio",
-) -> OmniRequestOutput:
- """Return an OmniRequestOutput mimicking diffusion audio model output."""
-
- audio_tensor = torch.sin(torch.linspace(0, 440 * 2 * torch.pi, num_samples))
-
- return OmniRequestOutput.from_diffusion(
- request_id=request_id,
- images=[],
- prompt=None,
- metrics={},
- multimodal_output={audio_key: audio_tensor, "sr": sample_rate},
- )
-
-
-def _make_engine_client(*, audio_key: str = "audio", sample_rate: int = 44100):
- """Build a mock engine client producing audio output."""
- mock_engine_client = MagicMock()
- mock_engine_client.errored = False
- mock_engine_client.model_type = "StableAudioPipeline"
- mock_engine_client.default_sampling_params_list = [{}]
-
- async def mock_generate_fn(*args, **kwargs):
- yield create_mock_audio_output(
- request_id=kwargs.get("request_id", "audiogen-mock"),
- sample_rate=sample_rate,
- audio_key=audio_key,
- )
-
- mock_engine_client.generate = MagicMock(side_effect=mock_generate_fn)
- return mock_engine_client
-
-
-def _make_server(engine_client=None):
- """Build an OmniOpenAIServingAudioGenerate with mocks."""
- if engine_client is None:
- engine_client = _make_engine_client()
-
- mock_models = MagicMock()
- mock_models.is_base_model.return_value = True
-
- return OmniOpenAIServingAudioGenerate(
- engine_client=engine_client,
- models=mock_models,
- request_logger=MagicMock(),
- )
-
-
-@pytest.fixture
-def test_app():
- server = _make_server()
-
- original_fn = server.create_audio_generate
- sig = signature(original_fn)
- new_params = [p for name, p in sig.parameters.items() if name != "raw_request"]
- new_sig = Signature(parameters=new_params, return_annotation=sig.return_annotation)
-
- async def patched_create_audio_generate(*args, **kwargs):
- return await original_fn(*args, **kwargs)
-
- patched_create_audio_generate.__signature__ = new_sig
- server.create_audio_generate = patched_create_audio_generate
-
- app = FastAPI()
- app.add_api_route(
- "/v1/audio/generate",
- server.create_audio_generate,
- methods=["POST"],
- response_model=None,
- )
- return app
-
-
-@pytest.fixture
-def client(test_app):
- return TestClient(test_app)
-
-
-# Request Validation (Pydantic model)
-class TestRequestValidation:
- """Validate OpenAICreateAudioGenerateRequest pydantic constraints."""
-
- def test_valid_minimal_request(self):
- req = OpenAICreateAudioGenerateRequest(input="A calm piano melody")
- assert req.input == "A calm piano melody"
- assert req.response_format == "wav"
- assert req.speed == 1.0
-
- def test_fields_are_wired_correctly(self):
- req = OpenAICreateAudioGenerateRequest(
- input="rain sounds",
- model="stable-audio",
- response_format="flac",
- speed=1.5,
- audio_length=10.0,
- audio_start=2.0,
- negative_prompt="noise",
- guidance_scale=7.5,
- num_inference_steps=100,
- seed=42,
- )
- assert req.input == "rain sounds"
- assert req.model == "stable-audio"
- assert req.response_format == "flac"
- assert req.speed == 1.5
- assert req.audio_length == 10.0
- assert req.audio_start == 2.0
- assert req.negative_prompt == "noise"
- assert req.guidance_scale == 7.5
- assert req.num_inference_steps == 100
- assert req.seed == 42
-
- def test_invalid_response_format(self):
- with pytest.raises(Exception):
- OpenAICreateAudioGenerateRequest(input="test", response_format="invalid_format")
-
- def test_speed_lower_bound(self):
- with pytest.raises(Exception):
- OpenAICreateAudioGenerateRequest(input="test", speed=0.1)
-
- def test_speed_upper_bound(self):
- with pytest.raises(Exception):
- OpenAICreateAudioGenerateRequest(input="test", speed=5.0)
-
- def test_speed_at_boundaries(self):
- req_low = OpenAICreateAudioGenerateRequest(input="test", speed=0.25)
- assert req_low.speed == 0.25
- req_high = OpenAICreateAudioGenerateRequest(input="test", speed=4.0)
- assert req_high.speed == 4.0
-
- def test_stream_format_sse_rejected(self):
- with pytest.raises(Exception):
- OpenAICreateAudioGenerateRequest(input="test", stream_format="sse")
-
- def test_stream_format_audio_accepted(self):
- req = OpenAICreateAudioGenerateRequest(input="test", stream_format="audio")
- assert req.stream_format == "audio"
-
-
-# Constructor & Class Methods
-class TestConstructor:
- def test_default_init(self):
- server = _make_server()
- assert server.diffusion_mode is False
-
- def test_for_diffusion_factory(self):
- engine_client = _make_engine_client()
- mock_models = MagicMock()
- mock_models.is_base_model.return_value = True
-
- server = OmniOpenAIServingAudioGenerate.for_diffusion(
- engine_client=engine_client,
- models=mock_models,
- request_logger=MagicMock(),
- )
- assert server.diffusion_mode is True
-
- def test_is_stable_audio_model_true(self):
- server = _make_server()
- assert server._is_stable_audio_model() is True
-
- def test_is_stable_audio_model_false(self):
- engine = _make_engine_client()
- engine.model_type = "SomeOtherModel"
- server = _make_server(engine_client=engine)
- assert server._is_stable_audio_model() is False
-
-
-# Parameter Wiring — verify request params reach the engine
-class TestParameterWiring:
- """Ensure request parameters are correctly forwarded to the engine."""
-
- @pytest.fixture
- def server_and_engine(self):
- engine = _make_engine_client()
- server = _make_server(engine_client=engine)
- return server, engine
-
- @pytest.mark.asyncio
- async def test_prompt_wiring(self, server_and_engine):
- server, engine = server_and_engine
- req = OpenAICreateAudioGenerateRequest(input="birds chirping")
- await server.create_audio_generate(req)
-
- engine.generate.assert_called_once()
- call_kwargs = engine.generate.call_args[1]
- assert call_kwargs["prompt"]["prompt"] == "birds chirping"
- assert call_kwargs["output_modalities"] == ["audio"]
-
- @pytest.mark.asyncio
- async def test_negative_prompt_wiring(self, server_and_engine):
- server, engine = server_and_engine
- req = OpenAICreateAudioGenerateRequest(input="a calm ocean", negative_prompt="noise distortion")
- await server.create_audio_generate(req)
-
- call_kwargs = engine.generate.call_args[1]
- assert call_kwargs["prompt"]["negative_prompt"] == "noise distortion"
-
- @pytest.mark.asyncio
- async def test_negative_prompt_absent(self, server_and_engine):
- server, engine = server_and_engine
- req = OpenAICreateAudioGenerateRequest(input="a calm ocean")
- await server.create_audio_generate(req)
-
- call_kwargs = engine.generate.call_args[1]
- assert "negative_prompt" not in call_kwargs["prompt"]
-
- @pytest.mark.asyncio
- async def test_guidance_scale_wiring(self, server_and_engine):
- server, engine = server_and_engine
- req = OpenAICreateAudioGenerateRequest(input="test", guidance_scale=12.0)
- await server.create_audio_generate(req)
-
- call_kwargs = engine.generate.call_args[1]
- sp = call_kwargs["sampling_params_list"][0]
- assert isinstance(sp, OmniDiffusionSamplingParams)
- assert sp.guidance_scale == 12.0
-
- @pytest.mark.asyncio
- async def test_num_inference_steps_wiring(self, server_and_engine):
- server, engine = server_and_engine
- req = OpenAICreateAudioGenerateRequest(input="test", num_inference_steps=200)
- await server.create_audio_generate(req)
-
- sp = engine.generate.call_args[1]["sampling_params_list"][0]
- assert sp.num_inference_steps == 200
-
- @pytest.mark.asyncio
- async def test_seed_creates_generator(self, server_and_engine):
- server, engine = server_and_engine
- req = OpenAICreateAudioGenerateRequest(input="test", seed=42)
-
- with patch("vllm_omni.entrypoints.openai.serving_audio_generate.torch") as mock_torch:
- mock_gen = MagicMock()
- mock_gen.manual_seed.return_value = mock_gen
- mock_torch.Generator.return_value = mock_gen
-
- await server.create_audio_generate(req)
-
- mock_torch.Generator.assert_called_once()
- mock_gen.manual_seed.assert_called_once_with(42)
-
- @pytest.mark.asyncio
- async def test_seed_none_skips_generator(self, server_and_engine):
- server, engine = server_and_engine
- req = OpenAICreateAudioGenerateRequest(input="test")
-
- await server.create_audio_generate(req)
-
- sp = engine.generate.call_args[1]["sampling_params_list"][0]
- assert sp.generator is None
-
- @pytest.mark.asyncio
- async def test_audio_length_wiring(self, server_and_engine):
- server, engine = server_and_engine
- req = OpenAICreateAudioGenerateRequest(input="test", audio_length=10.0, audio_start=2.0)
- await server.create_audio_generate(req)
-
- sp = engine.generate.call_args[1]["sampling_params_list"][0]
- assert sp.extra_args["audio_start_in_s"] == 2.0
- assert sp.extra_args["audio_end_in_s"] == 12.0 # start + length
-
- @pytest.mark.asyncio
- async def test_audio_length_default_start(self, server_and_engine):
- server, engine = server_and_engine
- req = OpenAICreateAudioGenerateRequest(input="test", audio_length=5.0)
- await server.create_audio_generate(req)
-
- sp = engine.generate.call_args[1]["sampling_params_list"][0]
- assert sp.extra_args["audio_start_in_s"] == 0.0
- assert sp.extra_args["audio_end_in_s"] == 5.0
-
- @pytest.mark.asyncio
- async def test_no_audio_length_skips_extra_args(self, server_and_engine):
- server, engine = server_and_engine
- req = OpenAICreateAudioGenerateRequest(input="test")
- await server.create_audio_generate(req)
-
- sp = engine.generate.call_args[1]["sampling_params_list"][0]
- assert sp.extra_args == {}
-
- @pytest.mark.asyncio
- async def test_defaults_not_set_when_omitted(self, server_and_engine):
- """Guidance scale and num_inference_steps keep dataclass defaults when not in request."""
- server, engine = server_and_engine
- req = OpenAICreateAudioGenerateRequest(input="test")
- await server.create_audio_generate(req)
-
- sp = engine.generate.call_args[1]["sampling_params_list"][0]
- defaults = OmniDiffusionSamplingParams()
- assert sp.guidance_scale == defaults.guidance_scale
- assert sp.num_inference_steps == defaults.num_inference_steps
-
-
-# Audio Response Format
-class TestAudioResponseFormat:
- def test_wav_response(self, client):
- payload = {"input": "a gentle rain", "response_format": "wav"}
- response = client.post("/v1/audio/generate", json=payload)
- assert response.status_code == 200
- assert response.headers["content-type"] == "audio/wav"
- assert len(response.content) > 0
-
- def test_mp3_response(self, client):
- payload = {"input": "a gentle rain", "response_format": "mp3"}
- response = client.post("/v1/audio/generate", json=payload)
- assert response.status_code == 200
- assert response.headers["content-type"] == "audio/mpeg"
- assert len(response.content) > 0
-
- def test_flac_response(self, client):
- payload = {"input": "a gentle rain", "response_format": "flac"}
- response = client.post("/v1/audio/generate", json=payload)
- assert response.status_code == 200
- assert response.headers["content-type"] == "audio/flac"
- assert len(response.content) > 0
-
- def test_invalid_format_rejected(self, client):
- payload = {"input": "test", "response_format": "banana"}
- response = client.post("/v1/audio/generate", json=payload)
- assert response.status_code == 422
-
- @patch("vllm_omni.entrypoints.openai.serving_audio_generate.OmniOpenAIServingAudioGenerate.create_audio")
- def test_speed_parameter_forwarded(self, mock_create_audio, test_app):
- mock_audio_response = MagicMock()
- mock_audio_response.audio_data = b"dummy_audio"
- mock_audio_response.media_type = "audio/wav"
- mock_create_audio.return_value = mock_audio_response
-
- c = TestClient(test_app)
- payload = {"input": "test", "response_format": "wav", "speed": 2.5}
- c.post("/v1/audio/generate", json=payload)
-
- mock_create_audio.assert_called_once()
- audio_obj = mock_create_audio.call_args[0][0]
- assert isinstance(audio_obj, CreateAudio)
- assert audio_obj.speed == 2.5
-
- @patch("vllm_omni.entrypoints.openai.serving_audio_generate.OmniOpenAIServingAudioGenerate.create_audio")
- def test_sample_rate_from_output(self, mock_create_audio, test_app):
- mock_audio_response = MagicMock()
- mock_audio_response.audio_data = b"dummy"
- mock_audio_response.media_type = "audio/wav"
- mock_create_audio.return_value = mock_audio_response
-
- c = TestClient(test_app)
- payload = {"input": "test"}
- c.post("/v1/audio/generate", json=payload)
-
- audio_obj = mock_create_audio.call_args[0][0]
- assert audio_obj.sample_rate == 44100 # Stable Audio default
-
-
-# Error Handling
-class TestErrorHandling:
- @pytest.mark.asyncio
- async def test_no_output_returns_error(self):
- engine = _make_engine_client()
-
- async def empty_gen(*args, **kwargs):
- return
- yield # noqa: unreachable – makes this an async generator
-
- engine.generate = MagicMock(side_effect=empty_gen)
- server = _make_server(engine_client=engine)
- req = OpenAICreateAudioGenerateRequest(input="test")
- resp = await server.create_audio_generate(req)
-
- # create_error_response returns an ErrorResponse with .error.message
- assert "No output generated" in resp.error.message
-
- @pytest.mark.asyncio
- async def test_no_audio_in_output_returns_error(self):
- engine = _make_engine_client()
-
- async def gen_without_audio(*args, **kwargs):
- yield OmniRequestOutput.from_diffusion(
- request_id="test",
- images=[],
- prompt=None,
- metrics={},
- multimodal_output={}, # no audio key
- )
-
- engine.generate = MagicMock(side_effect=gen_without_audio)
- server = _make_server(engine_client=engine)
- req = OpenAICreateAudioGenerateRequest(input="test")
- resp = await server.create_audio_generate(req)
-
- assert "did not produce audio" in resp.error.message
-
- @pytest.mark.asyncio
- async def test_engine_errored_raises(self):
- engine = _make_engine_client()
- engine.errored = True
- engine.dead_error = RuntimeError("engine is dead")
- server = _make_server(engine_client=engine)
-
- req = OpenAICreateAudioGenerateRequest(input="test")
- with pytest.raises(RuntimeError, match="engine is dead"):
- await server.create_audio_generate(req)
-
- @pytest.mark.asyncio
- async def test_model_outputs_key_fallback(self):
- """Audio data under 'model_outputs' key should be accepted."""
- engine = _make_engine_client(audio_key="model_outputs")
- server = _make_server(engine_client=engine)
- req = OpenAICreateAudioGenerateRequest(input="test")
- resp = await server.create_audio_generate(req)
-
- # Should succeed and return a Response with audio bytes
- assert hasattr(resp, "body")
- assert len(resp.body) > 0
-
- @pytest.mark.asyncio
- async def test_value_error_returns_error_response(self):
- engine = _make_engine_client()
-
- async def gen_value_error(*args, **kwargs):
- raise ValueError("bad value")
- yield # noqa: unreachable
-
- engine.generate = MagicMock(side_effect=gen_value_error)
- server = _make_server(engine_client=engine)
- req = OpenAICreateAudioGenerateRequest(input="test")
- resp = await server.create_audio_generate(req)
-
- assert "bad value" in resp.error.message
-
- @pytest.mark.asyncio
- async def test_generic_exception_returns_error_response(self):
- engine = _make_engine_client()
-
- async def gen_runtime_error(*args, **kwargs):
- raise RuntimeError("something went wrong")
- yield # noqa: unreachable
-
- engine.generate = MagicMock(side_effect=gen_runtime_error)
- server = _make_server(engine_client=engine)
- req = OpenAICreateAudioGenerateRequest(input="test")
- resp = await server.create_audio_generate(req)
-
- assert "Audio generation failed" in resp.error.message
-
-
-# End-to-End via TestClient
-class TestAudioGenerateAPI:
- def test_basic_success(self, client):
- payload = {"input": "ambient forest sounds"}
- response = client.post("/v1/audio/generate", json=payload)
- assert response.status_code == 200
- assert len(response.content) > 0
-
- def test_with_all_params(self, client):
- payload = {
- "input": "gentle piano",
- "response_format": "wav",
- "speed": 1.0,
- "audio_length": 5.0,
- "audio_start": 0.0,
- "negative_prompt": "noise",
- "guidance_scale": 7.0,
- "num_inference_steps": 50,
- "seed": 123,
- }
- response = client.post("/v1/audio/generate", json=payload)
- assert response.status_code == 200
- assert response.headers["content-type"] == "audio/wav"
-
- def test_missing_input_rejected(self, client):
- payload = {}
- response = client.post("/v1/audio/generate", json=payload)
- assert response.status_code == 422
-
- def test_extra_unknown_fields_ignored(self, client):
- payload = {"input": "test", "unknown_field": "value"}
- response = client.post("/v1/audio/generate", json=payload)
- # Pydantic v2 ignores extra fields by default
- assert response.status_code == 200
diff --git a/tests/entrypoints/test_omni_diffusion.py b/tests/entrypoints/test_omni_diffusion.py
index 2fbe7a8b4..90b0fd053 100644
--- a/tests/entrypoints/test_omni_diffusion.py
+++ b/tests/entrypoints/test_omni_diffusion.py
@@ -612,7 +612,7 @@ def test_initialize_stage_configs_called_when_none(
"""Test that stage configs are auto-loaded when stage_configs_path is None."""
def _fake_loader(
- config_path: str,
+ model: str,
stage_configs_path: str | None = None,
base_engine_args: dict | None = None,
default_stage_cfg_factory=None,
@@ -687,7 +687,7 @@ def test_generate_raises_on_length_mismatch(monkeypatch: pytest.MonkeyPatch, moc
"""Test that generate raises ValueError when sampling_params_list length doesn't match."""
def _fake_loader(
- config_path: str,
+ model: str,
stage_configs_path: str | None = None,
base_engine_args: dict | None = None,
default_stage_cfg_factory=None,
@@ -742,7 +742,7 @@ def test_generate_pipeline_and_final_outputs(monkeypatch: pytest.MonkeyPatch, mo
stage_cfg1["processed_input"] = ["processed-for-stage-1"]
def _fake_loader(
- config_path: str,
+ model: str,
stage_configs_path: str | None = None,
base_engine_args: dict | None = None,
default_stage_cfg_factory=None,
@@ -848,7 +848,7 @@ def test_generate_pipeline_with_batch_input(monkeypatch: pytest.MonkeyPatch, moc
stage_cfg1["stage_id"] = 1
def _fake_loader(
- config_path: str,
+ model: str,
stage_configs_path: str | None = None,
base_engine_args: dict | None = None,
default_stage_cfg_factory=None,
@@ -969,7 +969,7 @@ def test_generate_no_final_output_returns_empty(
stage_cfg1["final_output"] = False
def _fake_loader(
- config_path: str,
+ model: str,
stage_configs_path: str | None = None,
base_engine_args: dict | None = None,
default_stage_cfg_factory=None,
@@ -1062,7 +1062,7 @@ def test_generate_sampling_params_none_use_default(
stage_cfg1["final_output"] = False
def _fake_loader(
- config_path: str,
+ model: str,
stage_configs_path: str | None = None,
base_engine_args: dict | None = None,
default_stage_cfg_factory=None,
@@ -1140,7 +1140,7 @@ def test_wait_for_stages_ready_timeout(monkeypatch: pytest.MonkeyPatch, mocker:
"""Test that _wait_for_stages_ready handles timeout correctly."""
def _fake_loader(
- config_path: str,
+ model: str,
stage_configs_path: str | None = None,
base_engine_args: dict | None = None,
default_stage_cfg_factory=None,
@@ -1202,7 +1202,7 @@ def test_generate_handles_error_messages(monkeypatch: pytest.MonkeyPatch, mocker
"""Test that generate handles error messages from stages correctly."""
def _fake_loader(
- config_path: str,
+ model: str,
stage_configs_path: str | None = None,
base_engine_args: dict | None = None,
default_stage_cfg_factory=None,
@@ -1283,7 +1283,7 @@ def test_close_sends_shutdown_signal(monkeypatch: pytest.MonkeyPatch, mocker: Mo
"""Test that close() sends shutdown signal to all input queues."""
def _fake_loader(
- config_path: str,
+ model: str,
stage_configs_path: str | None = None,
base_engine_args: dict | None = None,
default_stage_cfg_factory=None,
diff --git a/tests/entrypoints/test_omni_llm.py b/tests/entrypoints/test_omni_llm.py
index 0e1745c68..b11b14c89 100644
--- a/tests/entrypoints/test_omni_llm.py
+++ b/tests/entrypoints/test_omni_llm.py
@@ -577,7 +577,7 @@ def test_initialize_stage_configs_called_when_none(
"""Test that stage configs are auto-loaded when stage_configs_path is None."""
def _fake_loader(
- config_path: str,
+ model: str,
stage_configs_path: str | None = None,
base_engine_args: dict | None = None,
default_stage_cfg_factory=None,
@@ -654,7 +654,7 @@ def test_generate_raises_on_length_mismatch(monkeypatch: pytest.MonkeyPatch, moc
"""Test that generate raises ValueError when sampling_params_list length doesn't match."""
def _fake_loader(
- config_path: str,
+ model: str,
stage_configs_path: str | None = None,
base_engine_args: dict | None = None,
default_stage_cfg_factory=None,
@@ -711,7 +711,7 @@ def test_generate_pipeline_and_final_outputs(monkeypatch: pytest.MonkeyPatch, mo
stage_cfg1["processed_input"] = ["processed-for-stage-1"]
def _fake_loader(
- config_path: str,
+ model: str,
stage_configs_path: str | None = None,
base_engine_args: dict | None = None,
default_stage_cfg_factory=None,
@@ -819,7 +819,7 @@ def test_generate_no_final_output_returns_empty(
stage_cfg1["final_output"] = False
def _fake_loader(
- config_path: str,
+ model: str,
stage_configs_path: str | None = None,
base_engine_args: dict | None = None,
default_stage_cfg_factory=None,
@@ -911,7 +911,7 @@ def test_generate_sampling_params_none_use_default(
stage_cfg1["final_output"] = False
def _fake_loader(
- config_path: str,
+ model: str,
stage_configs_path: str | None = None,
base_engine_args: dict | None = None,
default_stage_cfg_factory=None,
@@ -988,7 +988,7 @@ def test_wait_for_stages_ready_timeout(monkeypatch: pytest.MonkeyPatch, mocker:
"""Test that _wait_for_stages_ready handles timeout correctly."""
def _fake_loader(
- config_path: str,
+ model: str,
stage_configs_path: str | None = None,
base_engine_args: dict | None = None,
default_stage_cfg_factory=None,
@@ -1052,7 +1052,7 @@ def test_generate_handles_error_messages(monkeypatch: pytest.MonkeyPatch, mocker
"""Test that generate handles error messages from stages correctly."""
def _fake_loader(
- config_path: str,
+ model: str,
stage_configs_path: str | None = None,
base_engine_args: dict | None = None,
default_stage_cfg_factory=None,
@@ -1135,7 +1135,7 @@ def test_close_sends_shutdown_signal(monkeypatch: pytest.MonkeyPatch, mocker: Mo
"""Test that close() sends shutdown signal to all input queues."""
def _fake_loader(
- config_path: str,
+ model: str,
stage_configs_path: str | None = None,
base_engine_args: dict | None = None,
default_stage_cfg_factory=None,
diff --git a/tests/model_executor/models/qwen2_5_omni/test_qwen2_5_omni_embed.py b/tests/model_executor/models/qwen2_5_omni/test_qwen2_5_omni_embed.py
index 9bf230d55..84112336e 100644
--- a/tests/model_executor/models/qwen2_5_omni/test_qwen2_5_omni_embed.py
+++ b/tests/model_executor/models/qwen2_5_omni/test_qwen2_5_omni_embed.py
@@ -19,6 +19,8 @@ from vllm.model_executor.models.qwen2_5_omni_thinker import (
merge_interleaved_embeddings,
)
+pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
+
# Fake token IDs
AUDIO_TOKEN_ID = 1001
IMAGE_TOKEN_ID = 1002
diff --git a/tests/model_executor/stage_input_processors/test_qwen3_tts_async_chunk.py b/tests/model_executor/stage_input_processors/test_qwen3_tts_async_chunk.py
index 1b6fe2e4b..8d6d21430 100644
--- a/tests/model_executor/stage_input_processors/test_qwen3_tts_async_chunk.py
+++ b/tests/model_executor/stage_input_processors/test_qwen3_tts_async_chunk.py
@@ -7,16 +7,20 @@ from types import SimpleNamespace
import pytest
import torch
+from vllm_omni.model_executor.stage_input_processors.chunk_size_utils import (
+ compute_dynamic_initial_chunk_size,
+ max_ic_for_chunk_size,
+)
from vllm_omni.model_executor.stage_input_processors.qwen3_tts import (
talker2code2wav,
talker2code2wav_async_chunk,
)
-_FRAME = [1, 2, 3, 4] # 4-codebook frame
-_Q = len(_FRAME) # num quantizers
+_FRAME = [1, 2, 3, 4]
+_Q = len(_FRAME)
-def _req(rid: str, *, finished: bool, initial_codec_chunk_frames: int | None = None):
+def _req(rid, *, finished, initial_codec_chunk_frames=None):
ai = None
if initial_codec_chunk_frames is not None:
entry = SimpleNamespace(list_data=[initial_codec_chunk_frames])
@@ -28,9 +32,10 @@ def _req(rid: str, *, finished: bool, initial_codec_chunk_frames: int | None = N
)
-def _tm(*, chunk_frames=25, left_context=25, initial_chunk=0):
+def _tm(*, chunk_frames=25, left_context=25, max_num_seqs=1):
return SimpleNamespace(
code_prompt_token_ids=defaultdict(list),
+ scheduler_max_num_seqs=max_num_seqs,
put_req_chunk=defaultdict(int),
request_payload={},
connector=SimpleNamespace(
@@ -38,16 +43,14 @@ def _tm(*, chunk_frames=25, left_context=25, initial_chunk=0):
"extra": {
"codec_chunk_frames": chunk_frames,
"codec_left_context_frames": left_context,
- "initial_codec_chunk_frames": initial_chunk,
}
}
),
)
-def _call(tm, rid, *, n_frames, put_req=0, finished=False, req_ic=None):
+def _call(tm, rid, *, n_frames, finished=False, req_ic=None):
tm.code_prompt_token_ids[rid] = [_FRAME[:] for _ in range(n_frames)]
- tm.put_req_chunk[rid] = put_req
return talker2code2wav_async_chunk(
transfer_manager=tm,
pooling_output={"audio_codes": torch.zeros((0,))},
@@ -56,84 +59,68 @@ def _call(tm, rid, *, n_frames, put_req=0, finished=False, req_ic=None):
)
-def test_does_not_emit_empty_chunk_when_not_finished():
+def test_empty_returns_none():
tm = _tm()
- request = _req("rid-empty", finished=False)
-
- payload = talker2code2wav_async_chunk(
+ p = talker2code2wav_async_chunk(
transfer_manager=tm,
pooling_output={"audio_codes": torch.zeros((0,))},
- request=request,
+ request=_req("r", finished=False),
)
-
- assert payload is None
+ assert p is None
-def test_flushes_tail_when_finished_without_pooler_output():
+def test_eof_marker_when_finished_empty():
tm = _tm()
- rid = "rid-tail"
- tm.code_prompt_token_ids[rid] = [_FRAME[:] for _ in range(24)]
- request = _req(rid, finished=True)
-
- payload = talker2code2wav_async_chunk(
+ p = talker2code2wav_async_chunk(
transfer_manager=tm,
pooling_output=None,
- request=request,
+ request=_req("r", finished=True),
+ is_finished=True,
)
-
- assert payload is not None
- assert payload["finished"].item() is True
- assert len(payload["code_predictor_codes"]) == _Q * 24
+ assert p == {"code_predictor_codes": [], "finished": torch.tensor(True, dtype=torch.bool)}
-def test_emits_eof_marker_when_finished_with_no_frames():
+def test_flush_on_finish():
tm = _tm()
- request = _req("rid-eof", finished=True)
-
- payload = talker2code2wav_async_chunk(
+ tm.code_prompt_token_ids["r"] = [_FRAME[:] for _ in range(24)]
+ p = talker2code2wav_async_chunk(
transfer_manager=tm,
pooling_output=None,
- request=request,
+ request=_req("r", finished=True),
+ is_finished=True,
)
-
- assert payload == {
- "code_predictor_codes": [],
- "finished": torch.tensor(True, dtype=torch.bool),
- }
+ assert p is not None
+ assert p["finished"].item() is True
+ assert len(p["code_predictor_codes"]) == _Q * 24
_CASES = [
- # Normal path (initial=0): emit at chunk_size boundaries
- ((25, 25, 0), (24, 0, False), None),
- ((25, 25, 0), (25, 0, False), (0, 25)),
- # Initial-chunk phase: hold, first emit, second emit
- ((25, 25, 10), (9, 0, False), None),
- ((25, 25, 10), (10, 0, False), (0, 10)),
- ((25, 25, 10), (20, 1, False), (10, 20)),
- # Non-divisible: holds at chunk boundary
- ((25, 25, 12), (25, 2, False), None),
- # Normal phase: offset by initial_coverage (chunk//initial * initial)
- ((25, 25, 10), (45, 2, False), (20, 45)),
- # Second normal emit (offset must stay stable)
- ((25, 25, 10), (70, 3, False), (25, 50)),
- # initial >= chunk clamps to chunk_size (behaves as normal)
- ((25, 25, 30), (25, 0, False), (0, 25)),
- # finished=True flushes IC tail
- ((25, 25, 10), (5, 0, True), (0, 5)),
- # finished=True flushes non-divisible IC residual
- ((25, 25, 12), (25, 2, True), (24, 25)),
- # finished=True flushes normal phase tail
- ((25, 25, 10), (30, 2, True), (20, 30)),
+ # Dynamic IC=16, cs=25, initial_coverage=16
+ ((25, 25, 0), 24, False, None), # IC phase: 24%16!=0 -> hold
+ ((25, 25, 0), 25, False, None), # transition: adjusted=9, hold (no replay)
+ ((25, 25, 0), 41, False, (16, 41)), # first normal emit, lc=16
+ # Per-request IC=10, cs=25, initial_coverage=20
+ ((25, 25, 10), 9, False, None), # IC: hold
+ ((25, 25, 10), 10, False, (0, 10)), # IC: emit at boundary
+ ((25, 25, 10), 25, False, None), # transition: hold (no replay)
+ ((25, 25, 10), 45, False, (20, 45)), # first normal emit, lc=20
+ ((25, 25, 10), 5, True, (0, 5)), # finished flushes IC tail
+ ((25, 25, 10), 33, True, (20, 33)), # finished flushes normal tail
+ # IC=8, cs=16: IC divides chunk_size evenly (edge case)
+ ((16, 25, 8), 8, False, (0, 8)), # IC: emit
+ ((16, 25, 8), 16, False, None), # transition: hold (no replay)
+ ((16, 25, 8), 24, False, (8, 24)), # first normal emit, lc=8
+ # Per-request override: IC=15 at n_frames=10 -> 10%15!=0 -> hold
+ ((25, 25, 15), 10, False, None),
]
-@pytest.mark.parametrize("config, state, expected", _CASES)
-def test_streaming_decoding_with_variable_initial(config, state, expected):
- chunk_frames, left_context, initial_chunk = config
- n_frames, put_req, finished = state
-
- tm = _tm(chunk_frames=chunk_frames, left_context=left_context, initial_chunk=initial_chunk)
- payload = _call(tm, "r", n_frames=n_frames, put_req=put_req, finished=finished)
+@pytest.mark.parametrize("config, n_frames, finished, expected", _CASES)
+def test_streaming_phases(config, n_frames, finished, expected):
+ chunk_frames, left_context, req_ic_val = config
+ tm = _tm(chunk_frames=chunk_frames, left_context=left_context)
+ req_ic = req_ic_val if req_ic_val > 0 else None
+ payload = _call(tm, "r", n_frames=n_frames, finished=finished, req_ic=req_ic)
if expected is None:
assert payload is None
@@ -144,22 +131,86 @@ def test_streaming_decoding_with_variable_initial(config, state, expected):
assert len(payload["code_predictor_codes"]) == _Q * exp_window
-def test_per_request_override_activates_initial_phase():
- tm = _tm(initial_chunk=0)
- payload = _call(tm, "r-override", n_frames=10, req_ic=10)
- assert payload is not None
- assert payload["left_context_size"] == 0
- assert len(payload["code_predictor_codes"]) == _Q * 10
+def test_dynamic_ic_adapts_to_load():
+ # chunk_size=25 -> max_ic=16, steps=[2,4,8,16]
+ tm = _tm(max_num_seqs=8)
+
+ # Low load (1/8) -> IC=2 -> emit at 2
+ p1 = _call(tm, "r", n_frames=2)
+ assert p1 is not None
+ assert len(p1["code_predictor_codes"]) == _Q * 2
+
+ # High load: add 4 others -> active=5/8 -> IC=8 -> emit at 8
+ for i in range(4):
+ tm.code_prompt_token_ids[f"other-{i}"] = [[0]]
+ p2 = _call(tm, "r", n_frames=8)
+ assert p2 is not None
+ assert len(p2["code_predictor_codes"]) == _Q * 8
+
+ # Requests past initial phase still count in load factor
+ tm2 = _tm(max_num_seqs=4)
+ for i in range(3):
+ tm2.code_prompt_token_ids[f"long-{i}"] = [[0]] * 50 # well past cs=25
+ # active=4/4=1.0 -> IC=16
+ p3 = _call(tm2, "new", n_frames=16)
+ assert p3 is not None
+ assert len(p3["code_predictor_codes"]) == _Q * 16
-def test_per_request_override_wins_over_stage_config():
- tm = _tm(initial_chunk=5)
- payload = _call(tm, "r-override2", n_frames=10, put_req=0, req_ic=15)
- assert payload is None
+def test_ic_load_change_mid_request():
+ """IC stateless: load spike mid-request shifts initial_coverage."""
+ tm = _tm(chunk_frames=25, left_context=25, max_num_seqs=8)
+
+ # Low load -> IC=2 -> emit at frame 2
+ p1 = _call(tm, "r", n_frames=2)
+ assert p1 is not None
+
+ # Spike load: 6 others -> IC=16 -> initial_coverage=16
+ for i in range(6):
+ tm.code_prompt_token_ids[f"other-{i}"] = [[0]] * 10
+
+ # adjusted=25-16=9, 9%25!=0 -> hold
+ assert _call(tm, "r", n_frames=25) is None
+
+ # First normal emit at 16+25=41
+ p3 = _call(tm, "r", n_frames=41)
+ assert p3 is not None
+ assert p3["left_context_size"] == 16
+
+
+@pytest.mark.parametrize(
+ "active,max_bs,max_ic,expected",
+ [
+ (0, 4, 32, 2), # zero load -> min step
+ (2, 4, 32, 8), # mid load
+ (4, 4, 32, 32), # full load
+ (10, 4, 16, 16), # over capacity, capped
+ (0, 4, 1, 1), # max_ic below min step
+ (0, 0, 16, 2), # zero capacity edge case
+ ],
+)
+def test_compute_dynamic_initial_chunk_size(active, max_bs, max_ic, expected):
+ assert compute_dynamic_initial_chunk_size(active, max_bs, max_ic) == expected
+
+
+@pytest.mark.parametrize(
+ "chunk_size,expected",
+ [
+ (25, 16),
+ (50, 32),
+ (70, 64),
+ (8, 4),
+ (4, 2),
+ (2, 1),
+ (1, 1),
+ ],
+)
+def test_max_ic_for_chunk_size(chunk_size, expected):
+ assert max_ic_for_chunk_size(chunk_size) == expected
def test_first_streaming_chunk_prepends_ref_code_context():
- tm = _tm(initial_chunk=10)
+ tm = _tm()
rid = "r-ref"
tm.code_prompt_token_ids[rid] = [_FRAME[:] for _ in range(10)]
ref_code = torch.tensor([[9, 9, 9, 9], [8, 8, 8, 8]], dtype=torch.long)
@@ -177,7 +228,7 @@ def test_first_streaming_chunk_prepends_ref_code_context():
def test_ref_code_context_only_applies_to_first_streaming_chunk():
- tm = _tm(initial_chunk=10)
+ tm = _tm()
rid = "r-ref2"
tm.code_prompt_token_ids[rid] = [_FRAME[:] for _ in range(20)]
tm.put_req_chunk[rid] = 1
@@ -196,7 +247,7 @@ def test_ref_code_context_only_applies_to_first_streaming_chunk():
def test_ref_code_context_can_be_buffered_before_first_emit():
- tm = _tm(initial_chunk=10)
+ tm = _tm()
rid = "r-ref-buffered"
ref_code = torch.tensor([[9, 9, 9, 9], [8, 8, 8, 8]], dtype=torch.long)
diff --git a/vllm_omni/diffusion/models/stable_audio/pipeline_stable_audio.py b/vllm_omni/diffusion/models/stable_audio/pipeline_stable_audio.py
index a1eb74eaf..22dfc06c5 100644
--- a/vllm_omni/diffusion/models/stable_audio/pipeline_stable_audio.py
+++ b/vllm_omni/diffusion/models/stable_audio/pipeline_stable_audio.py
@@ -356,6 +356,7 @@ class StableAudioPipeline(nn.Module, SupportAudioOutput):
negative_prompt: str | list[str] | None = None,
audio_end_in_s: float | None = None,
audio_start_in_s: float = 0.0,
+ num_inference_steps: int = 100,
guidance_scale: float = 7.0,
num_waveforms_per_prompt: int = 1,
generator: torch.Generator | list[torch.Generator] | None = None,
@@ -373,6 +374,7 @@ class StableAudioPipeline(nn.Module, SupportAudioOutput):
negative_prompt: Negative prompt for CFG
audio_end_in_s: Audio end time in seconds (max ~47s for stable-audio-open-1.0)
audio_start_in_s: Audio start time in seconds
+ num_inference_steps: Number of denoising steps
guidance_scale: CFG scale
num_waveforms_per_prompt: Number of audio outputs per prompt
generator: Random generator for reproducibility
@@ -393,7 +395,7 @@ class StableAudioPipeline(nn.Module, SupportAudioOutput):
elif req.prompts:
negative_prompt = ["" if isinstance(p, str) else (p.get("negative_prompt") or "") for p in req.prompts]
- num_inference_steps = req.sampling_params.num_inference_steps
+ num_inference_steps = req.sampling_params.num_inference_steps or num_inference_steps
if req.sampling_params.guidance_scale_provided:
guidance_scale = req.sampling_params.guidance_scale
diff --git a/vllm_omni/distributed/omni_connectors/transfer_adapter/chunk_transfer_adapter.py b/vllm_omni/distributed/omni_connectors/transfer_adapter/chunk_transfer_adapter.py
index db8d5dbaa..d96ef74db 100644
--- a/vllm_omni/distributed/omni_connectors/transfer_adapter/chunk_transfer_adapter.py
+++ b/vllm_omni/distributed/omni_connectors/transfer_adapter/chunk_transfer_adapter.py
@@ -236,6 +236,9 @@ class OmniChunkTransferAdapter(OmniTransferAdapterBase):
self.put_req_chunk[request_id] += 1
logger.debug(f"[Stage-{stage_id}] Sent {connector_put_key}")
+ if is_finished:
+ self.code_prompt_token_ids.pop(request_id, None)
+
########################################################################
# Cleanup
########################################################################
diff --git a/vllm_omni/entrypoints/omni_llm.py b/vllm_omni/entrypoints/omni_llm.py
index 91c1c6bed..27f470b4b 100644
--- a/vllm_omni/entrypoints/omni_llm.py
+++ b/vllm_omni/entrypoints/omni_llm.py
@@ -28,7 +28,6 @@ from vllm_omni.entrypoints.utils import (
load_stage_configs_from_model,
load_stage_configs_from_yaml,
resolve_model_config_path,
- resolve_model_type,
)
logger = init_logger(__name__)
@@ -85,13 +84,12 @@ class OmniLLM(LLM):
self.worker_backend = kwargs.get("worker_backend", "multi_process")
self.ray_address = kwargs.get("ray_address", None)
self.batch_timeout = batch_timeout
- self.model_type = resolve_model_type(model)
self.log_stats: bool = bool(log_stats)
# Load stage configurations
if stage_configs_path is None:
- self.config_path = resolve_model_config_path(self.model_type)
- self.stage_configs = load_stage_configs_from_model(config_path=self.config_path)
+ self.config_path = resolve_model_config_path(model)
+ self.stage_configs = load_stage_configs_from_model(model)
else:
self.config_path = stage_configs_path
self.stage_configs = load_stage_configs_from_yaml(stage_configs_path)
diff --git a/vllm_omni/entrypoints/openai/api_server.py b/vllm_omni/entrypoints/openai/api_server.py
index bb4649bf1..6db97dd2d 100644
--- a/vllm_omni/entrypoints/openai/api_server.py
+++ b/vllm_omni/entrypoints/openai/api_server.py
@@ -87,10 +87,7 @@ from vllm_omni.entrypoints.openai.image_api_utils import (
encode_image_base64,
parse_size,
)
-from vllm_omni.entrypoints.openai.protocol.audio import (
- OpenAICreateAudioGenerateRequest,
- OpenAICreateSpeechRequest,
-)
+from vllm_omni.entrypoints.openai.protocol.audio import OpenAICreateSpeechRequest
from vllm_omni.entrypoints.openai.protocol.images import (
ImageData,
ImageGenerationRequest,
@@ -100,7 +97,6 @@ from vllm_omni.entrypoints.openai.protocol.videos import (
VideoGenerationRequest,
VideoGenerationResponse,
)
-from vllm_omni.entrypoints.openai.serving_audio_generate import OmniOpenAIServingAudioGenerate
from vllm_omni.entrypoints.openai.serving_chat import OmniOpenAIServingChat
from vllm_omni.entrypoints.openai.serving_speech import OmniOpenAIServingSpeech
from vllm_omni.entrypoints.openai.serving_video import OmniOpenAIServingVideo
@@ -179,29 +175,8 @@ class _DiffusionServingModels:
provide a lightweight fallback.
"""
- class _NullModelConfig:
- def __getattr__(self, name):
- return None
-
- class _Unsupported:
- def __init__(self, name: str):
- self.name = name
-
- def __call__(self, *args, **kwargs):
- raise NotImplementedError(f"{self.name} is not supported in diffusion mode")
-
- def __getattr__(self, attr):
- raise NotImplementedError(f"{self.name}.{attr} is not supported in diffusion mode")
-
def __init__(self, base_model_paths: list[BaseModelPath]) -> None:
self._base_model_paths = base_model_paths
- self.model_config = self._NullModelConfig()
-
- def __getattr__(self, name):
- """Return a sentinel that raises NotImplementedError if called or
- accessed, so any use of unsupported OpenAIServingModels features in
- diffusion mode fails loudly with a descriptive message."""
- return self._Unsupported(name)
async def show_available_models(self) -> ModelList:
return ModelList(
@@ -459,10 +434,10 @@ async def omni_init_app_state(
# For omni models
state.stage_configs = engine_client.stage_configs if hasattr(engine_client, "stage_configs") else None
- model_name = served_model_names[0] if served_model_names else args.model
# Pure Diffusion mode: use simplified initialization logic
if is_pure_diffusion:
+ model_name = served_model_names[0] if served_model_names else args.model
state.vllm_config = None
state.diffusion_engine = engine_client
state.openai_serving_models = _DiffusionServingModels(base_model_paths)
@@ -475,16 +450,6 @@ async def omni_init_app_state(
model_name=model_name,
)
- # audio related
- state.openai_serving_speech = None
- state.openai_serving_audio_generate = OmniOpenAIServingAudioGenerate.for_diffusion(
- engine_client,
- state.openai_serving_models,
- request_logger=request_logger,
- model_name=model_name,
- )
-
- # video related
diffusion_stage_configs = engine_client.stage_configs if hasattr(engine_client, "stage_configs") else None
state.openai_serving_video = OmniOpenAIServingVideo.for_diffusion(
diffusion_engine=engine_client, # type: ignore
@@ -492,6 +457,7 @@ async def omni_init_app_state(
stage_configs=diffusion_stage_configs,
)
+ state.enable_server_load_tracking = getattr(args, "enable_server_load_tracking", False)
state.server_load_metrics = 0
logger.info("Pure diffusion API server initialized for model: %s", model_name)
return
@@ -766,11 +732,7 @@ async def omni_init_app_state(
)
state.openai_serving_speech = OmniOpenAIServingSpeech(
- engine_client, state.openai_serving_models, request_logger=request_logger, model_name=model_name
- )
-
- state.openai_serving_audio_generate = OmniOpenAIServingAudioGenerate(
- engine_client, state.openai_serving_models, request_logger=request_logger, model_name=model_name
+ engine_client, state.openai_serving_models, request_logger=request_logger
)
state.openai_serving_video = OmniOpenAIServingVideo(
@@ -795,10 +757,6 @@ def Omnispeech(request: Request) -> OmniOpenAIServingSpeech | None:
return request.app.state.openai_serving_speech
-def OmniAudioGenerate(request: Request) -> OmniOpenAIServingAudioGenerate | None:
- return getattr(request.app.state, "openai_serving_audio_generate", None)
-
-
@router.post(
"/v1/chat/completions",
dependencies=[Depends(validate_json_request)],
@@ -909,34 +867,6 @@ async def create_speech(request: OpenAICreateSpeechRequest, raw_request: Request
raise HTTPException(status_code=HTTPStatus.INTERNAL_SERVER_ERROR.value, detail=str(e)) from e
-@router.post(
- "/v1/audio/generate",
- dependencies=[Depends(validate_json_request)],
- responses={
- HTTPStatus.OK.value: {"content": {"audio/*": {}}},
- HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
- HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse},
- HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
- },
-)
-@with_cancellation
-@load_aware_call
-async def create_audio_generate(request: OpenAICreateAudioGenerateRequest, raw_request: Request):
- handler = OmniAudioGenerate(raw_request)
- if handler is None:
- base_server = getattr(raw_request.app.state, "openai_serving_tokenization", None)
- if base_server is None:
- raise HTTPException(
- status_code=HTTPStatus.NOT_FOUND.value,
- detail="The model does not support Audio Generate API",
- )
- return base_server.create_error_response(message="The model does not support Audio Generate API")
- try:
- return await handler.create_audio_generate(request, raw_request)
- except Exception as e:
- raise HTTPException(status_code=HTTPStatus.INTERNAL_SERVER_ERROR.value, detail=str(e)) from e
-
-
@router.get(
"/v1/audio/voices",
responses={
diff --git a/vllm_omni/entrypoints/openai/audio_utils_mixin.py b/vllm_omni/entrypoints/openai/audio_utils_mixin.py
index 3023b67ad..13df32ebe 100644
--- a/vllm_omni/entrypoints/openai/audio_utils_mixin.py
+++ b/vllm_omni/entrypoints/openai/audio_utils_mixin.py
@@ -45,10 +45,6 @@ class AudioMixin:
"Only mono (1D) and stereo (2D) are supported."
)
- if audio_tensor.ndim == 2 and audio_tensor.shape[0] == 2:
- # Convert from [channels, samples] to [samples, channels]
- audio_tensor = audio_tensor.T
-
audio_tensor, sample_rate = self._apply_speed_adjustment(audio_tensor, speed, sample_rate)
supported_formats = {
diff --git a/vllm_omni/entrypoints/openai/protocol/audio.py b/vllm_omni/entrypoints/openai/protocol/audio.py
index 91690cbf0..871787173 100644
--- a/vllm_omni/entrypoints/openai/protocol/audio.py
+++ b/vllm_omni/entrypoints/openai/protocol/audio.py
@@ -58,7 +58,7 @@ class OpenAICreateSpeechRequest(BaseModel):
initial_codec_chunk_frames: int | None = Field(
default=None,
ge=0,
- description="Initial chunk size for reduced TTFA. Overrides stage config for this request.",
+ description="Per-request initial chunk size override. If null, computed dynamically based on server load.",
)
@field_validator("stream_format")
@@ -85,53 +85,6 @@ class OpenAICreateSpeechRequest(BaseModel):
return self
-class OpenAICreateAudioGenerateRequest(BaseModel):
- """Request model for audio generation via diffusion models (e.g. Stable Audio)."""
-
- input: str = Field(
- description="Text prompt describing the audio to generate",
- )
- model: str | None = None
- response_format: Literal["wav", "pcm", "flac", "mp3", "aac", "opus"] = "wav"
- speed: float | None = Field(
- default=1.0,
- ge=0.25,
- le=4.0,
- )
- stream_format: Literal["sse", "audio"] | None = "audio"
- audio_length: float | None = Field(
- default=None,
- description="Audio length in seconds",
- )
- audio_start: float | None = Field(
- default=0.0,
- description="Audio start time in seconds",
- )
- negative_prompt: str | None = Field(
- default=None,
- description="Negative prompt for classifier-free guidance",
- )
- guidance_scale: float | None = Field(
- default=None,
- description="Guidance scale for diffusion models",
- )
- num_inference_steps: int | None = Field(
- default=None,
- description="Number of inference steps",
- )
- seed: int | None = Field(
- default=None,
- description="Random seed for reproducibility",
- )
-
- @field_validator("stream_format")
- @classmethod
- def validate_stream_format(cls, v: str) -> str:
- if v == "sse":
- raise ValueError("'sse' is not a supported stream_format yet. Please use 'audio'.")
- return v
-
-
class CreateAudio(BaseModel):
audio_tensor: np.ndarray
sample_rate: int = 24000
diff --git a/vllm_omni/entrypoints/openai/serving_audio_generate.py b/vllm_omni/entrypoints/openai/serving_audio_generate.py
deleted file mode 100644
index bd5c37157..000000000
--- a/vllm_omni/entrypoints/openai/serving_audio_generate.py
+++ /dev/null
@@ -1,168 +0,0 @@
-import asyncio
-
-import torch
-from fastapi import Request
-from fastapi.responses import Response
-from vllm.entrypoints.openai.engine.serving import OpenAIServing
-from vllm.logger import init_logger
-from vllm.utils import random_uuid
-
-from vllm_omni.entrypoints.openai.audio_utils_mixin import AudioMixin
-from vllm_omni.entrypoints.openai.protocol.audio import (
- AudioResponse,
- CreateAudio,
- OpenAICreateAudioGenerateRequest,
-)
-from vllm_omni.inputs.data import OmniDiffusionSamplingParams
-from vllm_omni.outputs import OmniRequestOutput
-
-logger = init_logger(__name__)
-
-
-class OmniOpenAIServingAudioGenerate(OpenAIServing, AudioMixin):
- """Serving class for audio generation via diffusion models (e.g. Stable Audio)."""
-
- def __init__(self, *args, **kwargs):
- self.model_name = kwargs.pop("model_name", None)
- super().__init__(*args, **kwargs)
- self.diffusion_mode = False
-
- @classmethod
- def for_diffusion(cls, *args, **kwargs) -> "OmniOpenAIServingAudioGenerate":
- """Create an instance configured to run in diffusion mode."""
- instance = cls(*args, **kwargs)
- instance.diffusion_mode = True
- return instance
-
- def _is_stable_audio_model(self) -> bool:
- return self.engine_client.model_type == "StableAudioPipeline"
-
- async def create_audio_generate(
- self,
- request: OpenAICreateAudioGenerateRequest,
- raw_request: Request | None = None,
- ):
- """
- Generate audio using diffusion-based models (e.g. Stable Audio).
-
- This endpoint is designed for audio generation models as
- opposed to TTS models that specifically generate speech.
- """
-
- error_check_ret = await self._check_model(request)
- if error_check_ret is not None:
- logger.error("Error with model %s", error_check_ret)
- return error_check_ret
-
- if self.engine_client.errored:
- raise self.engine_client.dead_error
-
- request_id = f"audiogen-{random_uuid()}"
-
- try:
- default_sr = 44100 # Default sample rate for Stable Audio
-
- # Build prompt for diffusion audio generation
- prompt = {
- "prompt": request.input,
- }
- if request.negative_prompt:
- prompt["negative_prompt"] = request.negative_prompt
-
- # Build sampling params for diffusion
- sampling_params_list = [OmniDiffusionSamplingParams(num_outputs_per_prompt=1)]
-
- # Create generator if seed provided
- if request.seed is not None:
- from vllm_omni.platforms import current_omni_platform
-
- generator = torch.Generator(device=current_omni_platform.device_type).manual_seed(request.seed)
- sampling_params_list[0].generator = generator
-
- if request.guidance_scale is not None:
- sampling_params_list[0].guidance_scale = request.guidance_scale
-
- if request.num_inference_steps is not None:
- sampling_params_list[0].num_inference_steps = request.num_inference_steps
-
- # Set up audio duration parameters
- if request.audio_length is not None:
- audio_length = request.audio_length
- audio_start = request.audio_start if request.audio_start is not None else 0.0
- audio_end_in_s = audio_start + audio_length
- sampling_params_list[0].extra_args = {
- "audio_start_in_s": audio_start,
- "audio_end_in_s": audio_end_in_s,
- }
-
- logger.info(
- "Audio generation request %s: prompt=%r",
- request_id,
- request.input[:50] + "..." if len(request.input) > 50 else request.input,
- )
-
- generator = self.engine_client.generate(
- prompt=prompt,
- request_id=request_id,
- sampling_params_list=sampling_params_list,
- output_modalities=["audio"],
- )
-
- final_output: OmniRequestOutput | None = None
- async for res in generator:
- final_output = res
-
- if final_output is None:
- return self.create_error_response("No output generated from the model.")
-
- # Extract audio from output
- audio_output = None
- if hasattr(final_output, "multimodal_output") and final_output.multimodal_output:
- audio_output = final_output.multimodal_output
- if not audio_output and hasattr(final_output, "request_output"):
- if final_output.request_output and hasattr(final_output.request_output, "multimodal_output"):
- audio_output = final_output.request_output.multimodal_output
-
- # Check for audio data using either "audio" or "model_outputs" key
- audio_key = None
- if audio_output:
- if "audio" in audio_output:
- audio_key = "audio"
- elif "model_outputs" in audio_output:
- audio_key = "model_outputs"
-
- if not audio_output or audio_key is None:
- return self.create_error_response("Audio generation model did not produce audio output.")
-
- audio_tensor = audio_output[audio_key]
- sample_rate = audio_output.get("sr", default_sr)
- if hasattr(sample_rate, "item"):
- sample_rate = sample_rate.item()
-
- # Convert tensor to numpy
- if hasattr(audio_tensor, "float"):
- audio_tensor = audio_tensor.float().detach().cpu().numpy()
-
- # Squeeze batch dimension if present, but preserve channel dimension for stereo
- if audio_tensor.ndim > 1:
- audio_tensor = audio_tensor.squeeze()
-
- audio_obj = CreateAudio(
- audio_tensor=audio_tensor,
- sample_rate=int(sample_rate),
- response_format=request.response_format or "wav",
- speed=request.speed or 1.0,
- stream_format=request.stream_format,
- base64_encode=False,
- )
-
- audio_response: AudioResponse = self.create_audio(audio_obj)
- return Response(content=audio_response.audio_data, media_type=audio_response.media_type)
-
- except asyncio.CancelledError:
- return self.create_error_response("Client disconnected")
- except ValueError as e:
- return self.create_error_response(e)
- except Exception as e:
- logger.exception("Audio generation failed: %s", e)
- return self.create_error_response(f"Audio generation failed: {e}")
diff --git a/vllm_omni/entrypoints/openai/serving_speech.py b/vllm_omni/entrypoints/openai/serving_speech.py
index 75cf956fa..7bcf75ace 100644
--- a/vllm_omni/entrypoints/openai/serving_speech.py
+++ b/vllm_omni/entrypoints/openai/serving_speech.py
@@ -84,7 +84,6 @@ def _validate_path_within_directory(file_path: Path, directory: Path) -> bool:
class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
def __init__(self, *args, **kwargs):
- self.model_name = kwargs.pop("model_name", None)
super().__init__(*args, **kwargs)
# Initialize uploaded speakers storage
speech_voice_samples_dir = os.environ.get("SPEECH_VOICE_SAMPLES", "/tmp/voice_samples")
@@ -757,8 +756,6 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
request_id = f"speech-{random_uuid()}"
try:
- sampling_params_list = self.engine_client.default_sampling_params_list
- default_sr = 24000 # Default sample rate for TTS models
if self._is_tts:
# Validate TTS parameters
validation_error = self._validate_tts_request(request)
@@ -784,6 +781,8 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
tts_params.get("task_type", ["unknown"])[0],
)
+ sampling_params_list = self.engine_client.default_sampling_params_list
+
generator = self.engine_client.generate(
prompt=prompt,
request_id=request_id,
@@ -810,7 +809,7 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
return self.create_error_response("TTS model did not produce audio output.")
audio_tensor = audio_output[audio_key]
- sr_raw = audio_output.get("sr", default_sr)
+ sr_raw = audio_output.get("sr", 24000)
sr_val = sr_raw[-1] if isinstance(sr_raw, list) and sr_raw else sr_raw
sample_rate = sr_val.item() if hasattr(sr_val, "item") else int(sr_val)
diff --git a/vllm_omni/entrypoints/utils.py b/vllm_omni/entrypoints/utils.py
index beec074d8..69ce73fc4 100644
--- a/vllm_omni/entrypoints/utils.py
+++ b/vllm_omni/entrypoints/utils.py
@@ -169,46 +169,22 @@ def _convert_dataclasses_to_dict(obj: Any) -> Any:
return obj
-def resolve_model_config_path(model_type: str) -> str | None:
- """Resolve the stage config file path from the model type.
+def resolve_model_config_path(model: str) -> str:
+ """Resolve the stage config file path from the model name.
Resolves stage configuration path based on the model type and device type.
First tries to find a device-specific YAML file from stage_configs/{device_type}/
directory. If not found, falls back to the default config file.
Args:
- model_type: Model type string
+ model: Model name or path (used to determine model_type)
Returns:
- String path to the stage configuration file if found, None otherwise
- """
- default_config_path = current_omni_platform.get_default_stage_config_path()
- config_file_name = f"{model_type}.yaml"
- complete_config_path = PROJECT_ROOT / default_config_path / config_file_name
- if os.path.exists(complete_config_path):
- return str(complete_config_path)
-
- # Fall back to default config
- stage_config_file = f"vllm_omni/model_executor/stage_configs/{model_type}.yaml"
- stage_config_path = PROJECT_ROOT / stage_config_file
- if not os.path.exists(stage_config_path):
- return None
- return str(stage_config_path)
-
-
-def resolve_model_type(model: str) -> str:
- """Resolve the model type from the model name.
-
- Args:
- model: Model name or path
-
- Returns:
- Model type string (e.g. ``"Qwen3TTSForConditionalGeneration"``,
- ``"StableAudioPipeline"``).
+ String path to the stage configuration file
Raises:
- ValueError: If the model type cannot be determined from any
- available configuration file.
+ ValueError: If model_type cannot be determined
+ FileNotFoundError: If no stage config file exists for the model type
"""
# Try to get config from standard transformers format first
try:
@@ -241,26 +217,42 @@ def resolve_model_type(model: str) -> str:
f"Please ensure the model has proper configuration files with 'model_type' field"
)
- return model_type
+ default_config_path = current_omni_platform.get_default_stage_config_path()
+ model_type_str = f"{model_type}.yaml"
+ complete_config_path = PROJECT_ROOT / default_config_path / model_type_str
+ if os.path.exists(complete_config_path):
+ return str(complete_config_path)
+
+ # Fall back to default config
+ stage_config_file = f"vllm_omni/model_executor/stage_configs/{model_type}.yaml"
+ stage_config_path = PROJECT_ROOT / stage_config_file
+ if not os.path.exists(stage_config_path):
+ return None
+ return str(stage_config_path)
-def load_stage_configs_from_model(config_path: str | None, base_engine_args: dict | None = None) -> list:
- """Load stage configurations from a resolved config file path.
+def load_stage_configs_from_model(model: str, base_engine_args: dict | None = None) -> list:
+ """Load stage configurations from model's default config file.
+
+ Loads stage configurations based on the model type and device type.
+ First tries to load a device-specific YAML file from stage_configs/{device_type}/
+ directory. If not found, falls back to the default config file.
Args:
- config_path: Path to the YAML configuration file, or None.
- When None, returns an empty list.
- base_engine_args: Optional engine arguments to merge with stage configs.
+ model: Model name or path (used to determine model_type)
Returns:
- List of stage configuration dictionaries, or empty list if
- config_path is None.
+ List of stage configuration dictionaries
+
+ Raises:
+ FileNotFoundError: If no stage config file exists for the model type
"""
if base_engine_args is None:
base_engine_args = {}
- if config_path is None:
+ stage_config_path = resolve_model_config_path(model)
+ if stage_config_path is None:
return []
- stage_configs = load_stage_configs_from_yaml(config_path=config_path, base_engine_args=base_engine_args)
+ stage_configs = load_stage_configs_from_yaml(config_path=stage_config_path, base_engine_args=base_engine_args)
return stage_configs
@@ -314,10 +306,9 @@ def load_and_resolve_stage_configs(
Returns:
Tuple of (config_path, stage_configs)
"""
- model_type = resolve_model_type(model)
if stage_configs_path is None:
- config_path = resolve_model_config_path(model_type)
- stage_configs = load_stage_configs_from_model(config_path, base_engine_args=kwargs)
+ config_path = resolve_model_config_path(model)
+ stage_configs = load_stage_configs_from_model(model, base_engine_args=kwargs)
if not stage_configs:
if default_stage_cfg_factory is not None:
default_stage_cfg = default_stage_cfg_factory()
diff --git a/vllm_omni/model_executor/models/mimo_audio/mimo_audio.py b/vllm_omni/model_executor/models/mimo_audio/mimo_audio.py
index 162c4eb00..9528f0bfb 100644
--- a/vllm_omni/model_executor/models/mimo_audio/mimo_audio.py
+++ b/vllm_omni/model_executor/models/mimo_audio/mimo_audio.py
@@ -255,7 +255,7 @@ class MiMoAudioLLMDummyInputsBuilder(BaseDummyInputsBuilder[MiMoAudioLLMProcessi
dummy_processor_inputs = ProcessorInputs(
prompt=dummy_text,
- mm_items=dummy_mm_items,
+ mm_data_items=dummy_mm_items,
)
return dummy_processor_inputs
@@ -309,10 +309,6 @@ class MiMoAudioDataParser(MultiModalDataParser):
if data is None:
return AudioProcessorItems(None)
- # also check single audio item with sampling rate
- if self._is_empty(data) or (isinstance(data, tuple) and self._is_empty(data[0])):
- return None
-
if (
is_list_of(data, float)
or isinstance(data, (np.ndarray, torch.Tensor))
diff --git a/vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py b/vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py
index ec928195a..6dcd278ac 100644
--- a/vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py
+++ b/vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py
@@ -159,6 +159,12 @@ class Qwen3OmniMoeForConditionalGeneration(
# suppress tokens by setting their probability to ~1e-9 (finite very small)
self.suppressed_tokens = self._get_talker_suppressed_tokens()
self.requires_raw_input_tokens = True
+ # Keys that should stay on GPU in model_intermediate_buffer to avoid CPU↔GPU round-trips
+ self.gpu_resident_buffer_keys: set[str] = {
+ "last_talker_hidden",
+ "trailing_text_hidden",
+ "tts_pad_embed_projected",
+ }
elif self.model_stage == "code2wav":
self.enable_update_additional_information = True
@@ -223,14 +229,16 @@ class Qwen3OmniMoeForConditionalGeneration(
# ==================== Forward Pass ====================
def _get_talker_suppressed_tokens(self):
- return [
- i
- for i in range(
- self.config.talker_config.text_config.vocab_size - 1024,
- self.config.talker_config.text_config.vocab_size,
- )
- if i != self.config.talker_config.codec_eos_token_id
- ]
+ """Return a boolean mask on GPU for suppressed token positions."""
+ vocab_size = self.config.talker_config.text_config.vocab_size
+ mask = torch.zeros(vocab_size, dtype=torch.bool)
+ start = vocab_size - 1024
+ eos_id = self.config.talker_config.codec_eos_token_id
+ for i in range(start, vocab_size):
+ if i != eos_id:
+ mask[i] = True
+ # Will be moved to the correct device on first use
+ return mask
def get_mrope_input_positions(
self,
@@ -578,7 +586,7 @@ class Qwen3OmniMoeForConditionalGeneration(
Postprocess the talker hidden states.
"""
update_dict = {}
- update_dict["last_talker_hidden"] = hidden_states[-1, :].detach().to("cpu").contiguous()
+ update_dict["last_talker_hidden"] = hidden_states[-1, :].detach()
return update_dict
def talker_preprocess(self, input_ids: torch.Tensor, input_embeds: torch.Tensor, **info_dict: dict):
@@ -632,9 +640,9 @@ class Qwen3OmniMoeForConditionalGeneration(
if inputs_embeds.shape[-1] == 2048:
inputs_embeds = self.text_projection(inputs_embeds)
code_predictor_codes, summed_embeddings = self.talker.code_predictor_forward(
- input_ids, inputs_embeds.clone(), last_talker_hidden=last_talker_hidden
+ input_ids, inputs_embeds, last_talker_hidden=last_talker_hidden
)
- inputs_embeds = summed_embeddings.clone()
+ inputs_embeds = summed_embeddings
inputs_embeds = (inputs_embeds + text_step).reshape(-1, self.talker_config.text_config.hidden_size)
return inputs_embeds, code_predictor_codes.squeeze(-1)
@@ -758,7 +766,7 @@ class Qwen3OmniMoeForConditionalGeneration(
# compatible with old shape [1,S,D]
rem_tail = trailing_text_hidden.squeeze(0)
if rem_tail.shape[0] > 0:
- update_dict["trailing_text_hidden"] = rem_tail.detach().to("cpu").contiguous()
+ update_dict["trailing_text_hidden"] = rem_tail.detach()
# Also persist projected tts_pad for decode fallback if needed
if isinstance(tts_pad_thinker, torch.Tensor):
pad_in = tts_pad_thinker
@@ -767,7 +775,7 @@ class Qwen3OmniMoeForConditionalGeneration(
if pad_in.ndim == 1:
pad_in = pad_in.view(1, 1, -1)
pad_proj = self.talker.text_projection(pad_in.to(self._module_device(self.talker)))
- update_dict["tts_pad_embed_projected"] = pad_proj.detach().to("cpu").contiguous()
+ update_dict["tts_pad_embed_projected"] = pad_proj.detach()
except Exception:
pass
self._talker_cache_thinker_decode_embeds(info_dict, update_dict)
@@ -926,7 +934,7 @@ class Qwen3OmniMoeForConditionalGeneration(
if isinstance(q_tail, torch.Tensor) and q_tail.numel() > 0:
use_vec = q_tail[0:1, :]
new_q_tail = (
- q_tail[1:, :].detach().to("cpu").contiguous()
+ q_tail[1:, :].detach()
if q_tail.shape[0] > 1
else self.tts_pad_embed.to(input_embeds.device, dtype=input_embeds.dtype)
)
@@ -1122,16 +1130,10 @@ class Qwen3OmniMoeForConditionalGeneration(
# implemented by assigning their logits to log(1e-9).
if getattr(self, "model_stage", None) == "talker" and isinstance(logits, torch.Tensor):
- try:
- logits_cpu = logits.cpu()
- logits_cpu[:, self.suppressed_tokens] = -1e9
- logits = logits_cpu.to(logits.device)
- except Exception as e:
- print(f"Error in logits suppression: {e}")
- print(f"logits.shape: {logits.shape}")
- print(f"self.suppressed_tokens: {self.suppressed_tokens}")
- raise e
- logits[:, self.suppressed_tokens] = -1e9
+ # Move mask to device once (lazy), then reuse every step
+ if self.suppressed_tokens.device != logits.device:
+ self.suppressed_tokens = self.suppressed_tokens.to(logits.device)
+ logits.masked_fill_(self.suppressed_tokens.unsqueeze(0), -1e9)
return logits
def sample(
diff --git a/vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_code_predictor_mtp.py b/vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_code_predictor_mtp.py
index fc7402890..ad372c8f0 100644
--- a/vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_code_predictor_mtp.py
+++ b/vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_code_predictor_mtp.py
@@ -1,27 +1,16 @@
-"""Qwen3-Omni Code Predictor with MTP (Multi-Token Prediction) support.
+"""Qwen3-Omni Code Predictor -- optimized re-prefill, no KV cache.
-This module implements the code predictor component for Qwen3-Omni talker models.
-
-The code predictor generates residual RVQ (Residual Vector Quantization) codes
-autoregressively, predicting layers 1 to N based on layer-0 codes from the talker.
+* SDPA attention (F.scaled_dot_product_attention) -- no HF backend fallback
+* Persistent pre-allocated buffers (_proj_buf, _pos_ids) -- zero per-call alloc
+* Inline top-k sampling -- no LogitsProcessorList / custom-op overhead
+* torch.compile on inner transformer by default
+* No @support_torch_compile / static_forward_context / namedtuple
"""
-from collections import namedtuple
-from typing import Any
-
import torch
import torch.nn as nn
import torch.nn.functional as F
-from transformers import Cache, PretrainedConfig
-from transformers.generation.logits_process import (
- LogitsProcessorList,
- TopKLogitsWarper,
- TopPLogitsWarper,
-)
-from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS
-from vllm.compilation.decorators import support_torch_compile
-from vllm.config import CacheConfig, ModelConfig, VllmConfig, get_current_vllm_config
-from vllm.forward_context import get_forward_context
+from vllm.config import VllmConfig
from vllm.logger import init_logger
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.linear import (
@@ -32,47 +21,51 @@ from vllm.model_executor.layers.linear import (
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.model_executor.layers.rotary_embedding import get_rope
from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding
-from vllm.utils.torch_utils import direct_register_custom_op
logger = init_logger(__name__)
-# ============================================================================
-# Code Predictor Attention Layer
-# ============================================================================
+
+# ===================================================================
+# Standalone Attention (SDPA, no KV cache, no HF backend fallback)
+# ===================================================================
class Qwen3OmniCodePredictorAttention(nn.Module):
- """Multi-head self-attention for code predictor with vLLM optimization."""
+ """Multi-head self-attention for code predictor.
+
+ Uses ``F.scaled_dot_product_attention`` directly. No KV cache -- the code
+ predictor always re-prefills the full (short) sequence each AR step.
+
+ Input : [B, seq_len, hidden_size]
+ Output: [B, seq_len, hidden_size]
+ """
def __init__(
self,
config,
- layer_idx: int,
- vllm_config: VllmConfig = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
):
super().__init__()
-
- self.num_heads = config.code_predictor_config.num_attention_heads
- self.num_key_value_heads = config.code_predictor_config.num_key_value_heads
+ cp_cfg = config.code_predictor_config
+ self.num_heads = cp_cfg.num_attention_heads
+ self.num_kv_heads = cp_cfg.num_key_value_heads
self.head_dim = getattr(
- config.code_predictor_config,
+ cp_cfg,
"head_dim",
- config.code_predictor_config.hidden_size // config.code_predictor_config.num_attention_heads,
+ cp_cfg.hidden_size // cp_cfg.num_attention_heads,
)
- self.hidden_size = config.code_predictor_config.hidden_size
-
- if self.num_heads % self.num_key_value_heads != 0:
- raise ValueError("num_attention_heads must be divisible by num_key_value_heads")
-
- self.num_key_value_groups = self.num_heads // self.num_key_value_heads
+ self.hidden_size = cp_cfg.hidden_size
+ self.q_size = self.num_heads * self.head_dim
+ self.kv_size = self.num_kv_heads * self.head_dim
+ self.scaling = self.head_dim**-0.5
+ self._use_gqa = self.num_kv_heads != self.num_heads
self.qkv_proj = QKVParallelLinear(
hidden_size=self.hidden_size,
head_size=self.head_dim,
total_num_heads=self.num_heads,
- total_num_kv_heads=self.num_key_value_heads,
+ total_num_kv_heads=self.num_kv_heads,
bias=False,
quant_config=quant_config,
prefix=f"{prefix}.qkv_proj",
@@ -88,120 +81,55 @@ class Qwen3OmniCodePredictorAttention(nn.Module):
)
self.rotary_emb = get_rope(
self.head_dim,
- max_position=config.code_predictor_config.max_position_embeddings,
+ max_position=cp_cfg.max_position_embeddings,
rope_parameters=None,
dual_chunk_attention_config=None,
)
-
- self.q_size = self.num_heads * self.head_dim
- self.kv_size = self.num_key_value_heads * self.head_dim
-
- # Query/Key normalization
- self.q_norm = RMSNorm(self.head_dim, eps=config.code_predictor_config.rms_norm_eps)
- self.k_norm = RMSNorm(self.head_dim, eps=config.code_predictor_config.rms_norm_eps)
- self.is_causal = True
- self.config = config
-
- self.attention_backends = ["flash_attention_2", "xformers", "eager", "sdpa"]
- cudagraph_mode = get_current_vllm_config().compilation_config.cudagraph_mode
- if "flash_attention_2" in ALL_ATTENTION_FUNCTIONS and cudagraph_mode.has_full_cudagraphs():
- logger.warning(
- f"CUDAGraphMode.{cudagraph_mode.name} is currently not supported "
- f"with flash attention for Qwen3-Omni talker MTP."
- f"removing flash attention from attention_backends"
- )
- self.attention_backends.remove("flash_attention_2")
+ self.q_norm = RMSNorm(self.head_dim, eps=cp_cfg.rms_norm_eps)
+ self.k_norm = RMSNorm(self.head_dim, eps=cp_cfg.rms_norm_eps)
def forward(
self,
hidden_states: torch.Tensor,
- past_key_values: Cache | None = None,
- cache_position: torch.LongTensor | None = None,
- use_cache: bool = False,
- position_ids: torch.LongTensor | None = None,
+ position_ids: torch.Tensor,
) -> torch.Tensor:
bsz, seq_len, _ = hidden_states.shape
- qkv, _ = self.qkv_proj(hidden_states)
+ # Flatten to 2-D so vLLM rotary_emb gets [num_tokens, size]
+ qkv, _ = self.qkv_proj(hidden_states.reshape(bsz * seq_len, -1))
q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
- # Reshape for attention
- q = q.reshape(bsz, seq_len, self.num_heads, self.head_dim)
- k = k.reshape(bsz, seq_len, self.num_key_value_heads, self.head_dim)
- v = v.reshape(bsz, seq_len, self.num_key_value_heads, self.head_dim)
-
- # Apply normalization
- q = self.q_norm(q).contiguous()
- k = self.k_norm(k).contiguous()
- q = q.reshape(-1, self.q_size)
- k = k.reshape(-1, self.kv_size)
-
- # Apply RoPE
+ # QK-norm -> RoPE (both 2-D)
+ q = self.q_norm(q.view(-1, self.num_heads, self.head_dim)).view(q.shape)
+ k = self.k_norm(k.view(-1, self.num_kv_heads, self.head_dim)).view(k.shape)
q, k = self.rotary_emb(position_ids, q, k)
- # Reshape for attention
- q = q.reshape(bsz, seq_len, self.num_heads, self.head_dim)
- k = k.reshape(bsz, seq_len, self.num_key_value_heads, self.head_dim)
+ # [B, heads, seq, head_dim] for SDPA
+ q = q.view(bsz, seq_len, self.num_heads, self.head_dim).transpose(1, 2)
+ k = k.view(bsz, seq_len, self.num_kv_heads, self.head_dim).transpose(1, 2)
+ v = v.view(bsz, seq_len, self.num_kv_heads, self.head_dim).transpose(1, 2)
- v_heads = v.transpose(1, 2).contiguous()
- q_heads = q.transpose(1, 2).contiguous()
- k_heads = k.transpose(1, 2).contiguous()
+ attn_out = F.scaled_dot_product_attention(
+ q,
+ k,
+ v,
+ scale=self.scaling,
+ is_causal=True,
+ enable_gqa=self._use_gqa,
+ )
- if past_key_values is not None:
- sin, cos = self.rotary_emb.get_cos_sin(seq_len)
- # sin and cos are specific to RoPE models; cache_position needed for the static cache
- cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position}
- k_heads, v_heads = past_key_values.update(k_heads, v_heads, self.layer_idx, cache_kwargs)
-
- # Try attention backends in order of preference, with runtime error handling
- # This handles cases where the backend is registered but not actually available
- attn_output = None
- last_error = None
-
- for backend_name in self.attention_backends:
- if backend_name not in ALL_ATTENTION_FUNCTIONS:
- continue
-
- try:
- attention_interface = ALL_ATTENTION_FUNCTIONS[backend_name]
- attn_output, _ = attention_interface(
- self,
- q_heads,
- k_heads,
- v_heads,
- None,
- dropout=0.0 if not self.training else getattr(self, "attention_dropout", 0.0),
- scaling=self.head_dim**-0.5,
- sliding_window=None,
- use_cache=use_cache,
- position_ids=position_ids[:seq_len].unsqueeze(0),
- output_hidden_states=True,
- output_attentions=False,
- )
- break
- except (ValueError, ImportError, RuntimeError, AttributeError) as e:
- # Store error and try next backend
- last_error = e
- continue
-
- if attn_output is None:
- raise RuntimeError(
- f"All attention backends failed. Last error: {last_error}. "
- "Please install flash-attn, or ensure PyTorch's scaled_dot_product_attention is available."
- )
- attn_output = attn_output.reshape(*(hidden_states.shape[:-1]), -1).contiguous()
-
- attn_output, _ = self.o_proj(attn_output)
- return attn_output
+ attn_out = attn_out.transpose(1, 2).reshape(bsz * seq_len, -1)
+ output, _ = self.o_proj(attn_out)
+ return output.view(bsz, seq_len, -1)
-# ============================================================================
-# Code Predictor MLP Layer
-# ============================================================================
+# ===================================================================
+# MLP
+# ===================================================================
class Qwen3OmniCodePredictorMLP(nn.Module):
- """Feed-forward network for code predictor with fused gate/up projection."""
+ """SiLU-gated MLP for code predictor."""
def __init__(
self,
@@ -221,7 +149,6 @@ class Qwen3OmniCodePredictorMLP(nn.Module):
prefix=f"{prefix}.gate_up_proj",
disable_tp=True,
)
-
self.down_proj = RowParallelLinear(
input_size=intermediate_size,
output_size=hidden_size,
@@ -238,35 +165,23 @@ class Qwen3OmniCodePredictorMLP(nn.Module):
return down
-# ============================================================================
-# MTP Layer (Multi-Token Prediction Layer)
-# ============================================================================
+# ===================================================================
+# Decoder Layer
+# ===================================================================
-class Qwen3OmniCodePredictorMTPLayer(nn.Module):
- """MTP layer for speculative decoding - predicts next residual code layer."""
+class Qwen3OmniCodePredictorDecoderLayer(nn.Module):
+ """Transformer decoder layer (SDPA, no KV cache)."""
def __init__(
self,
- config: PretrainedConfig,
- prefix: str,
- model_config: ModelConfig,
- layer_idx: int,
- cache_config: CacheConfig | None = None,
+ config,
quant_config: QuantizationConfig | None = None,
+ prefix: str = "",
) -> None:
super().__init__()
- self.layer_idx = layer_idx
- self.config = config
-
self.self_attn = Qwen3OmniCodePredictorAttention(
config,
- layer_idx,
- vllm_config=type(
- "VllmConfig",
- (),
- {"cache_config": cache_config, "quant_config": quant_config, "model_config": model_config},
- )(),
quant_config=quant_config,
prefix=f"{prefix}.self_attn",
)
@@ -275,260 +190,243 @@ class Qwen3OmniCodePredictorMTPLayer(nn.Module):
quant_config=quant_config,
prefix=f"{prefix}.mlp",
)
- self.input_layernorm = RMSNorm(
- config.code_predictor_config.hidden_size, eps=config.code_predictor_config.rms_norm_eps
- )
- self.post_attention_layernorm = RMSNorm(
- config.code_predictor_config.hidden_size, eps=config.code_predictor_config.rms_norm_eps
- )
+ cp_cfg = config.code_predictor_config
+ self.input_layernorm = RMSNorm(cp_cfg.hidden_size, eps=cp_cfg.rms_norm_eps)
+ self.post_attention_layernorm = RMSNorm(cp_cfg.hidden_size, eps=cp_cfg.rms_norm_eps)
- def mtp_block(
+ def forward(
self,
hidden_states: torch.Tensor,
- past_key_values: Cache | None = None,
- cache_position: torch.LongTensor | None = None,
- use_cache: bool = False,
- position_ids: torch.LongTensor | None = None,
+ position_ids: torch.Tensor,
) -> torch.Tensor:
- # Self-attention with residual
residual = hidden_states
hidden_states = self.input_layernorm(hidden_states)
- hidden_states = self.self_attn(hidden_states, past_key_values, cache_position, use_cache, position_ids)
+ hidden_states = self.self_attn(hidden_states, position_ids)
hidden_states = residual + hidden_states
- # MLP with residual
residual = hidden_states
hidden_states = self.post_attention_layernorm(hidden_states)
hidden_states = self.mlp(hidden_states)
hidden_states = residual + hidden_states
-
return hidden_states
-class Qwen3OmniCodePredictorBaseModel(nn.Module):
- """
- Base model for code predictor - matches HF Qwen3OmniMoeTalkerCodePredictorModel structure.
+# ===================================================================
+# Base Transformer Model (re-prefill, no KV cache)
+# ===================================================================
- This is a simple transformer that processes inputs_embeds and outputs hidden states.
+
+class Qwen3OmniCodePredictorBaseModel(nn.Module):
+ """Inner transformer for code predictor.
+
+ Signature: ``forward(inputs_embeds, position_ids) -> hidden_states``
+ -- plain Tensor in, plain Tensor out (no namedtuple).
"""
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
config = vllm_config.model_config.hf_config.code_predictor_config
-
self.config = config
- self.vocab_size = config.vocab_size
- self.num_code_groups = config.num_code_groups
- # Codec embeddings (for layers 1-num_code_groups-1)
self.codec_embedding = nn.ModuleList(
- [
- VocabParallelEmbedding(
- config.vocab_size,
- config.hidden_size,
- )
- for _ in range(config.num_code_groups - 1)
- ]
+ [VocabParallelEmbedding(config.vocab_size, config.hidden_size) for _ in range(config.num_code_groups - 1)]
)
- # Decoder layers
self.layers = nn.ModuleList(
[
- Qwen3OmniCodePredictorMTPLayer(
+ Qwen3OmniCodePredictorDecoderLayer(
vllm_config.model_config.hf_config,
- f"{prefix}.layers.{idx}",
- model_config=vllm_config.model_config,
- layer_idx=idx,
- cache_config=vllm_config.cache_config,
quant_config=vllm_config.quant_config,
+ prefix=f"{prefix}.layers.{idx}",
)
for idx in range(config.num_hidden_layers)
]
)
-
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
def forward(
self,
inputs_embeds: torch.Tensor,
- attention_mask: torch.Tensor | None = None,
- position_ids: torch.LongTensor | None = None,
- past_key_values: Any | None = None,
- use_cache: bool | None = False,
- cache_position: torch.LongTensor | None = None,
- **kwargs: Any,
- ) -> Any:
- """
- Forward pass matching HF structure.
-
- Args:
- inputs_embeds: [batch, seq_len, hidden_size]
- position_ids: Optional position IDs tensor
- past_key_values: Optional cached key-value pairs
- use_cache: Whether to use cache
- cache_position: Optional cache position tensor
- **kwargs: Additional keyword arguments
-
- Returns:
- Named tuple with .last_hidden_state and .past_key_values attributes
- """
- batch_size, seq_len, _ = inputs_embeds.shape
- # Forward through decoder layers
+ position_ids: torch.Tensor,
+ ) -> torch.Tensor:
hidden_states = inputs_embeds
-
for layer in self.layers:
- hidden_states = layer.mtp_block(hidden_states, past_key_values, cache_position, use_cache, position_ids)
-
- # Final norm
+ hidden_states = layer(hidden_states, position_ids)
hidden_states = self.norm(hidden_states)
-
- # Return in HF-compatible format
- Output = namedtuple("Output", ["last_hidden_state", "past_key_values"])
- return Output(last_hidden_state=hidden_states, past_key_values=None) # [batch, num_code_groups-1, hidden_size]
-
- def get_input_embeddings(self):
- """Return codec embeddings for HF compatibility."""
- return self.codec_embedding
+ return hidden_states
-def code_predictor_sample(
- logits: torch.Tensor,
- layer_name: str,
-) -> torch.Tensor:
- forward_context = get_forward_context()
- self = forward_context.no_compile_layers[layer_name]
- logits = self.logits_processors(None, logits[:, -1])
- probs = F.softmax(logits, dim=-1)
- code = torch.multinomial(probs.squeeze(1), num_samples=1) # [batch, 1]
- return code
+# ===================================================================
+# Code Predictor Wrapper (optimized re-prefill, persistent buffers)
+# ===================================================================
-def code_predictor_sample_fake(
- logits: torch.Tensor,
- layer_name: str,
-) -> torch.Tensor:
- return torch.empty((logits.shape[0], 1), dtype=torch.int64, device=logits.device)
-
-
-direct_register_custom_op(
- op_name="qwen3_omni_code_predictor_sample",
- op_func=code_predictor_sample,
- fake_impl=code_predictor_sample_fake,
-)
-
-
-@support_torch_compile
class Qwen3OmniMoeTalkerCodePredictor(nn.Module):
- """
- Code predictor wrapper matching HF structure.
+ """Optimized code predictor -- re-prefill approach, no KV cache.
- Structure:
- - self.model: Qwen3OmniCodePredictorBaseModel (transformer)
- - self.lm_head: ModuleList of output heads
+ Each AR step forwards the full growing sequence (len 2 -> num_code_groups+1)
+ through the transformer. The extra O(T^2) FLOPs are negligible for
+ short sequences, and this avoids all KV-cache management overhead.
+
+ Optimizations:
+ 1. Pre-allocated embedding buffer -- no torch.cat per step.
+ 2. Pre-allocated position_ids -- no torch.arange per step.
+ 3. Inline top-k sampling -- no LogitsProcessorList / custom op.
+ 4. Cached module references -- bypass ModuleList indexing.
+ 5. torch.compile on inner transformer.
"""
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
- talker_code_predictor_config = vllm_config.model_config.hf_config
+ config = vllm_config.model_config.hf_config
+ self.config = config
self.quant_config = vllm_config.quant_config
self.prefix = prefix
- self.config = talker_code_predictor_config
- self.vocab_size = self.config.code_predictor_config.vocab_size
- self.num_code_groups = self.config.code_predictor_config.num_code_groups
+ self.num_code_groups = config.code_predictor_config.num_code_groups
+ self._hidden_size = config.code_predictor_config.hidden_size
- # Base transformer model (matches HF structure)
- self.model = Qwen3OmniCodePredictorBaseModel(vllm_config=vllm_config, prefix=prefix)
+ self.model = Qwen3OmniCodePredictorBaseModel(
+ vllm_config=vllm_config,
+ prefix=prefix,
+ )
- # Output heads for each residual layer (1-num_layers-1)
+ # One lm_head per residual layer (layers 1 .. G-1)
self.lm_head = nn.ModuleList(
[
nn.Linear(
- self.config.code_predictor_config.hidden_size,
- self.config.code_predictor_config.vocab_size,
+ config.code_predictor_config.hidden_size,
+ config.code_predictor_config.vocab_size,
bias=False,
)
for _ in range(self.num_code_groups - 1)
]
)
- self.logits_processors = LogitsProcessorList(
- [
- TopKLogitsWarper(top_k=50),
- TopPLogitsWarper(top_p=0.8),
- ]
+
+ # Sampling hyperparams (inlined)
+ self._top_k = 50
+
+ # Persistent buffers (lazily initialised on first forward)
+ self._proj_buf: torch.Tensor | None = None
+ self._pos_ids: torch.Tensor | None = None
+
+ # Cached plain-list refs (set once)
+ self._lm_heads: list | None = None
+ self._codec_embeds: list | None = None
+
+ # Model forward (optionally compiled)
+ self._model_fwd: object | None = None
+
+ # ------------------------------------------------------------------
+ # Lazy-init helpers
+ # ------------------------------------------------------------------
+
+ def _ensure_buffers(self, bsz: int, device: torch.device, dtype: torch.dtype) -> None:
+ max_seq = self.num_code_groups + 1
+ if (
+ self._proj_buf is not None
+ and self._proj_buf.shape[0] >= bsz
+ and self._proj_buf.device == device
+ and self._proj_buf.dtype == dtype
+ ):
+ return
+ self._proj_buf = torch.zeros(bsz, max_seq, self._hidden_size, dtype=dtype, device=device)
+ self._pos_ids = torch.arange(max_seq, dtype=torch.long, device=device)
+
+ def _ensure_cached_refs(self) -> None:
+ if self._lm_heads is not None:
+ return
+ self._lm_heads = list(self.lm_head)
+ self._codec_embeds = list(self.model.codec_embedding)
+
+ def _ensure_model_fwd(self) -> None:
+ if self._model_fwd is not None:
+ return
+ self._model_fwd = torch.compile(
+ self.model.forward,
+ mode="default",
+ dynamic=True,
)
+ logger.info("code_predictor: torch.compile enabled")
- compilation_config = get_current_vllm_config().compilation_config
- if prefix in compilation_config.static_forward_context:
- raise ValueError(f"Duplicate layer name: {prefix}")
- compilation_config.static_forward_context[prefix] = self
- self.layer_name = prefix
+ # ------------------------------------------------------------------
+ # Forward -- re-prefill + persistent buffers + inline sampling
+ # ------------------------------------------------------------------
+ @torch.inference_mode()
def forward(
self,
layer0_code: torch.Tensor,
layer0_embed: torch.Tensor,
last_talker_hidden: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor]:
- """
- Forward pass for code predictor.
+ """Predict residual codebooks 1..G-1 autoregressively via re-prefill.
Args:
- layer0_code:
- Code index for code-group (layer) 0.
- Shape: [batch_size, 1], dtype typically int64.
-
- last_talker_hidden:
-
- Shape: [batch_size, hidden_size].
+ layer0_code: [bsz, 1] int64
+ layer0_embed: [bsz, 1, hidden_size]
+ last_talker_hidden: [bsz, 1, hidden_size]
Returns:
- pos_all_layers:
- Predicted codes for all code groups, including `layer0_code`.
- Shape: [batch_size, num_code_groups, 1].
-
- current_input:
- The final input embedding sequence after appending embeddings of all
- predicted codes (one token per predicted layer).
- Shape: [batch_size, num_code_groups + 2, hidden_size].
+ all_codes: [bsz, num_code_groups, 1]
+ proj_buf: [bsz, num_code_groups + 1, hidden_size]
+ pos 0 = last_talker_hidden (NOT a codec embed)
+ pos 1 = layer0_embed
+ pos 2.. = codec_embedding[i](predicted_code_i)
"""
- pos_codes = [layer0_code] # Start with layer 0: [batch, 1]
- try:
- current_input = torch.cat([last_talker_hidden, layer0_embed], dim=1) # [batch, 2, hidden_size]
- except Exception as e:
- print(f"Error in current_input: {e}")
- print(f"last_talker_hidden shape: {last_talker_hidden.shape}")
- print(f"prev_embed shape: {layer0_embed.shape}")
- raise e
- batch_size = current_input.shape[0]
+ bsz = int(layer0_code.shape[0])
+ device = layer0_code.device
+ dtype = last_talker_hidden.dtype
+ num_groups = self.num_code_groups
+ top_k = self._top_k
- # Predict all residual layers (layers 1 to num_code_groups-1) autoregressively
- for layer_idx in range(self.num_code_groups - 1):
- seq_len = layer_idx + 2
- # Compute position_ids dynamically to avoid torch.compile specializing batch_size
- position_ids = torch.arange(seq_len, device=current_input.device, dtype=torch.int64).repeat(batch_size)
- # Forward through code_predictor model
- outputs = self.model(
- inputs_embeds=current_input,
- attention_mask=None,
- position_ids=position_ids,
- past_key_values=None,
- use_cache=False,
- cache_position=None,
- )
- hidden_state = outputs.last_hidden_state # [batch, 2, hidden_size]
+ # Lazy init
+ self._ensure_buffers(bsz, device, dtype)
+ self._ensure_model_fwd()
+ self._ensure_cached_refs()
- # Use the corresponding lm_head for this layer
- logits = self.lm_head[layer_idx](hidden_state[:, -1:, :])
- code = torch.ops.vllm.qwen3_omni_code_predictor_sample(logits, self.layer_name)
- pos_codes.append(code)
- # Update prev_embed for next layer (if not last layer)
- # layer_idx=0 predicts layer 1, embed with codec_embedding[1]
- new_embed = self.model.codec_embedding[layer_idx](code) # [batch, 1, hidden_size]
- current_input = torch.cat([current_input, new_embed], dim=1) # [batch, 3~n, hidden_size]
- pos_all_layers = torch.stack(pos_codes, dim=1) # [batch, num_code_groups, 1]
- return pos_all_layers, current_input
+ proj_buf = self._proj_buf
+ pos_ids = self._pos_ids
+ model_fwd = self._model_fwd
+ lm_heads = self._lm_heads
+ codec_embeds = self._codec_embeds
+
+ # Output codes
+ all_codes = torch.empty(bsz, num_groups, 1, dtype=torch.int64, device=device)
+ all_codes[:, 0] = layer0_code
+
+ # Fill buffer positions 0 & 1
+ proj_buf[:bsz, 0:1, :] = last_talker_hidden
+ proj_buf[:bsz, 1:2, :] = layer0_embed
+
+ # Autoregressive loop: predict layers 1..G-1
+ for step in range(1, num_groups):
+ seq_len = step + 1
+ projected = proj_buf[:bsz, :seq_len, :]
+ step_pos_ids = pos_ids[:seq_len] if bsz == 1 else pos_ids[:seq_len].repeat(bsz)
+
+ hidden_out = model_fwd(projected, step_pos_ids)
+
+ # Inline top-k sampling
+ logits = lm_heads[step - 1](hidden_out[:, -1, :])
+ if top_k > 0:
+ topk_vals, _ = logits.topk(top_k, dim=-1)
+ logits = logits.masked_fill(logits < topk_vals[:, -1:], float("-inf"))
+ probs = F.softmax(logits, dim=-1)
+ code = torch.multinomial(probs, num_samples=1) # [bsz, 1]
+
+ all_codes[:, step] = code
+
+ # Embed predicted code -> next buffer position
+ new_embed = codec_embeds[step - 1](code)
+ proj_buf[:bsz, step + 1 : step + 2, :] = new_embed
+
+ return all_codes, proj_buf[:bsz]
+
+ # ------------------------------------------------------------------
+ # Weight loading
+ # ------------------------------------------------------------------
def load_weights(self, weights: list[tuple[str, torch.Tensor]]) -> set[str]:
"""Load weights with mapping for fused QKV and gate_up projections.
diff --git a/vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_talker.py b/vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_talker.py
index 3362ae17f..9f5c20cf7 100644
--- a/vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_talker.py
+++ b/vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_talker.py
@@ -129,111 +129,70 @@ class Qwen3OmniMoeTalkerForConditionalGeneration(
self.code_predictor = Qwen3OmniMoeTalkerCodePredictor(
vllm_config=vllm_config, prefix=maybe_prefix(prefix, "code_predictor")
)
- max_batch_size = max(
- vllm_config.scheduler_config.max_num_seqs, vllm_config.compilation_config.max_cudagraph_capture_size
- )
- self.layer0_embed_buffer = torch.zeros(
- (max_batch_size, 1, self.config.text_config.hidden_size),
- dtype=vllm_config.model_config.dtype,
- )
def code_predictor_forward(
self,
input_ids: torch.Tensor,
inputs_embeds: torch.Tensor | None = None,
*,
- temperature: float = 1.0,
- top_k: int = 50, # Match transformers default
- top_p: float = 0.8, # Match transformers default
- generation_steps: int | None = None,
last_talker_hidden: torch.Tensor | None = None,
**_: object,
) -> tuple[torch.Tensor, torch.Tensor]:
- """
- Generate full RVQ codec codes for the provided sequence.
+ """Generate full RVQ codes + summed embeddings (single-loop, no KV cache).
- The code predictor consumes the layer-0 codec codes produced by the talker
- alongside the talker's hidden states, and autoregressively predicts the remaining
- residual layers (to num_codec_groups).
+ The code predictor uses re-prefill: each AR step re-forwards the full
+ (short) sequence through the transformer. The returned ``proj_buf``
+ already contains all codec embeddings at positions 1..G,
+ so summed_embeddings = proj_buf[:, 1:, :].sum(dim=1) — no second
+ loop or re-embedding needed.
Returns:
- tuple containing:
- - residual_codes: A tensor of shape [batch, num_code_groups, seq_len] containing
- the complete set of codec codes
- - summed_embeddings: A tensor of shape [batch, seq_len, hidden_size]
- Sum of all layer embeddings at each position (like Transformers)
+ result_codes: [batch, num_code_groups, seq_len]
+ summed_embeddings: [batch, seq_len, hidden_size]
"""
if input_ids is None:
- raise ValueError("`input_ids` containing layer-0 codec codes must be provided.")
+ raise ValueError("`input_ids` (layer-0 codes) must be provided.")
if inputs_embeds is None:
- raise ValueError("`inputs_embeds` containing talker hidden states must be provided.")
+ raise ValueError("`inputs_embeds` (talker hidden states) must be provided.")
if inputs_embeds.ndim == 2:
inputs_embeds = inputs_embeds.unsqueeze(0)
if input_ids.ndim == 1:
input_ids = input_ids.unsqueeze(0)
- # Ensure the tensors are contiguous for the autoregressive sampling loop
- inputs_embeds = inputs_embeds.contiguous()
- input_ids = input_ids.contiguous()
-
- # Generate full codec codes using MTP
- # This will be the parallel prediction implementation
batch_size, seq_len = input_ids.shape
+ device = input_ids.device
+ embed_fn = self.language_model.model.codec_embedding
+ hidden_size = self.config.code_predictor_config.hidden_size
- # For now, use sequential generation (TODO: implement parallel)
- # Result will be [batch, num_code_groups, seq_len]
- # - all_codes_per_position will collect [batch, num_code_groups, 1] for each position
- all_codes_per_position = []
- middle_hidden_states = [] # Collect hidden states for each position
-
- # Generate residual layers for each position
- for pos in range(seq_len):
- layer0_code = input_ids[:, pos : pos + 1] # [batch, 1]
-
- # Initial input: [last_talker_hidden, layer0_embed]
- layer0_embed = self.embed_input_ids(layer0_code)
- self.layer0_embed_buffer[:batch_size].copy_(layer0_embed)
- pos_all_layers, current_input = self.code_predictor(
- layer0_code, self.layer0_embed_buffer[:batch_size], last_talker_hidden
- )
-
- # Stack all layers for this position: [batch, num_code_groups, 1]
- all_codes_per_position.append(pos_all_layers)
- middle_hidden_states.append(current_input[:, 2:-1, :])
-
- # Concatenate across positions: [batch, num_code_groups, seq_len]
- result_codes = torch.cat(all_codes_per_position, dim=2)
-
- # Build summed embeddings for each position (like Transformers)
- # This combines layer-0 embed, mid layers hidden states, and last layer embed
- all_summed_embeddings = []
+ result_codes = torch.empty(
+ batch_size,
+ self.num_code_groups,
+ seq_len,
+ dtype=torch.int64,
+ device=device,
+ )
+ summed_embeddings = torch.empty(
+ batch_size,
+ seq_len,
+ hidden_size,
+ dtype=inputs_embeds.dtype,
+ device=device,
+ )
for pos in range(seq_len):
- # Layer 0 embedding
- layer0_code = result_codes[:, 0, pos : pos + 1] # [batch, 1]
- layer0_embed = self.embed_input_ids(layer0_code) # [batch, 1, hidden_size]
+ layer0_code = input_ids[:, pos : pos + 1]
+ layer0_embed = embed_fn(layer0_code)
- # mid layers hidden states (from CodePredictor)
- mid_residual_hiddens = middle_hidden_states[pos] # [batch, num_code_groups-2, hidden_size]
- mid_list = list(mid_residual_hiddens.split(1, dim=1))
-
- # last layer embedding
- last_layer_code = result_codes[:, -1, pos : pos + 1] # [batch, 1]
- last_residual_hidden = self.code_predictor.model.codec_embedding[-1](last_layer_code)
-
- # Concatenate all layers: [batch, num_code_groups, hidden_size]
- pos_codec_hiddens = torch.cat(
- [layer0_embed] + mid_list + [last_residual_hidden],
- dim=1,
+ pos_all_layers, proj_buf = self.code_predictor(
+ layer0_code,
+ layer0_embed,
+ last_talker_hidden,
)
- # Sum across layers: [batch, 1, hidden_size] (like Transformers)
- pos_summed = pos_codec_hiddens.sum(dim=1, keepdim=True)
- all_summed_embeddings.append(pos_summed)
-
- # Concatenate across positions: [batch, seq_len, hidden_size]
- summed_embeddings = torch.cat(all_summed_embeddings, dim=1).squeeze(1)
+ result_codes[:, :, pos : pos + 1] = pos_all_layers
+ # proj_buf layout: [0]=talker_hidden, [1..G]=codec embeds
+ summed_embeddings[:, pos, :] = proj_buf[:, 1:, :].sum(dim=1)
return result_codes, summed_embeddings
diff --git a/vllm_omni/model_executor/models/qwen3_tts/cuda_graph_decoder_wrapper.py b/vllm_omni/model_executor/models/qwen3_tts/cuda_graph_decoder_wrapper.py
index de278edf7..5ff259193 100644
--- a/vllm_omni/model_executor/models/qwen3_tts/cuda_graph_decoder_wrapper.py
+++ b/vllm_omni/model_executor/models/qwen3_tts/cuda_graph_decoder_wrapper.py
@@ -29,7 +29,7 @@ class CUDAGraphDecoderWrapper:
output = wrapper.decode(codes) # Automatically uses CUDA graph if possible
"""
- DEFAULT_CAPTURE_SIZES = [25, 50, 100, 150, 200, 250, 300]
+ DEFAULT_CAPTURE_SIZES = [2, 4, 8, 16, 25, 32, 50, 100, 150, 200, 250, 300]
def __init__(
self,
diff --git a/vllm_omni/model_executor/stage_configs/qwen3_tts.yaml b/vllm_omni/model_executor/stage_configs/qwen3_tts.yaml
index 19502add8..095aafe08 100644
--- a/vllm_omni/model_executor/stage_configs/qwen3_tts.yaml
+++ b/vllm_omni/model_executor/stage_configs/qwen3_tts.yaml
@@ -96,10 +96,6 @@ runtime:
# Align with Omni: small chunks with sufficient context overlap.
codec_chunk_frames: 25
codec_left_context_frames: 25
- # First chunk size for reduced TTFA (0 = disabled).
- # When > 0, emits small chunks every N frames during the initial phase,
- # then switches to codec_chunk_frames cadence.
- initial_codec_chunk_frames: 0
edges:
- from: 0
diff --git a/vllm_omni/model_executor/stage_configs/qwen3_tts_batch.yaml b/vllm_omni/model_executor/stage_configs/qwen3_tts_batch.yaml
index 43df70fc2..d73739109 100644
--- a/vllm_omni/model_executor/stage_configs/qwen3_tts_batch.yaml
+++ b/vllm_omni/model_executor/stage_configs/qwen3_tts_batch.yaml
@@ -98,10 +98,6 @@ runtime:
# Align with Omni: small chunks with sufficient context overlap.
codec_chunk_frames: 25
codec_left_context_frames: 25
- # First chunk size for reduced TTFA (0 = disabled).
- # When > 0, emits small chunks every N frames during the initial phase,
- # then switches to codec_chunk_frames cadence.
- initial_codec_chunk_frames: 0
edges:
- from: 0
diff --git a/vllm_omni/model_executor/stage_input_processors/chunk_size_utils.py b/vllm_omni/model_executor/stage_input_processors/chunk_size_utils.py
new file mode 100644
index 000000000..7bdec0bb3
--- /dev/null
+++ b/vllm_omni/model_executor/stage_input_processors/chunk_size_utils.py
@@ -0,0 +1,33 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+
+def max_ic_for_chunk_size(chunk_size: int) -> int:
+ """Largest power of 2 strictly less than chunk_size."""
+ if chunk_size <= 2:
+ return 1
+ return 1 << ((chunk_size - 1).bit_length() - 1)
+
+
+def compute_dynamic_initial_chunk_size(
+ active_requests: int,
+ max_batch_size: int,
+ max_ic: int,
+) -> int:
+ """Select IC from power-of-2 steps [2, 4, ..., max_ic] based on load factor.
+
+ - Low load: small IC (faster TTFA).
+ - High load: large IC (amortise decode cost).
+ """
+ steps: list[int] = []
+ v = 2
+ while v <= max_ic:
+ steps.append(v)
+ v <<= 1
+ if not steps:
+ return max(1, max_ic)
+ if max_batch_size <= 0:
+ return steps[0]
+ load_factor = min(active_requests / max_batch_size, 1.0)
+ idx = int(round(load_factor * (len(steps) - 1)))
+ return steps[idx]
diff --git a/vllm_omni/model_executor/stage_input_processors/qwen3_tts.py b/vllm_omni/model_executor/stage_input_processors/qwen3_tts.py
index eb9753262..435e45f03 100644
--- a/vllm_omni/model_executor/stage_input_processors/qwen3_tts.py
+++ b/vllm_omni/model_executor/stage_input_processors/qwen3_tts.py
@@ -5,6 +5,11 @@ from typing import Any
import torch
from vllm.logger import init_logger
+from vllm_omni.model_executor.stage_input_processors.chunk_size_utils import (
+ compute_dynamic_initial_chunk_size,
+ max_ic_for_chunk_size,
+)
+
logger = init_logger(__name__)
@@ -14,7 +19,7 @@ def talker2code2wav(
prompt: Any = None,
requires_multimodal_data: bool = False,
) -> list[Any]:
- """Non-async processor: wait for talker to finish, then pass all codes to code2wav at once."""
+ """Non-async: collect all talker codes, then pass to code2wav at once."""
from vllm_omni.inputs.data import OmniTokensPrompt
from vllm_omni.model_executor.stage_input_processors.qwen3_omni import _validate_stage_inputs
@@ -94,9 +99,12 @@ def talker2code2wav_async_chunk(
cfg = raw_cfg.get("extra", raw_cfg) if isinstance(raw_cfg, dict) else {}
chunk_size = int(cfg.get("codec_chunk_frames", 25))
left_context_size_config = int(cfg.get("codec_left_context_frames", 25))
- initial_chunk_size = int(cfg.get("initial_codec_chunk_frames", 0))
- # Per-request override (takes priority over stage config)
+
+ # Per-request override takes priority over dynamic IC.
+ per_request_override = False
+ initial_chunk_size = 0
additional_information = getattr(request, "additional_information", None)
+
if (
additional_information is not None
and hasattr(additional_information, "entries")
@@ -105,12 +113,32 @@ def talker2code2wav_async_chunk(
entry = additional_information.entries["initial_codec_chunk_frames"]
if entry.list_data is not None and len(entry.list_data) == 1:
initial_chunk_size = int(entry.list_data[0])
+ per_request_override = True
+
+ # Dynamic IC: always derived from chunk_size, recomputed every call to adapt to load.
+ if not per_request_override:
+ max_ic = max_ic_for_chunk_size(chunk_size)
+ active = sum(1 for v in transfer_manager.code_prompt_token_ids.values() if len(v) > 0)
+ capacity = getattr(transfer_manager, "scheduler_max_num_seqs", 1)
+ initial_chunk_size = compute_dynamic_initial_chunk_size(active, capacity, max_ic)
+ logger.debug(
+ "Dynamic IC: active=%d, capacity=%d, max_ic=%d, ic=%d, cs=%d, req=%s, keys=%d",
+ active,
+ capacity,
+ max_ic,
+ initial_chunk_size,
+ chunk_size,
+ request_id,
+ len(transfer_manager.code_prompt_token_ids),
+ )
+
if chunk_size <= 0 or left_context_size_config < 0 or initial_chunk_size < 0:
raise ValueError(
f"Invalid codec chunk config: codec_chunk_frames={chunk_size}, "
f"codec_left_context_frames={left_context_size_config}, "
f"initial_codec_chunk_frames={initial_chunk_size}"
)
+
if initial_chunk_size > chunk_size:
logger.warning(
"initial_codec_chunk_frames=%d > codec_chunk_frames=%d, clamping to codec_chunk_frames.",
@@ -120,8 +148,6 @@ def talker2code2wav_async_chunk(
initial_chunk_size = chunk_size
length = len(transfer_manager.code_prompt_token_ids[request_id])
- # Avoid emitting empty chunks during normal streaming. If the request is
- # finished and nothing was produced, emit an EOF marker.
if length <= 0:
if finished:
return {
@@ -130,34 +156,32 @@ def talker2code2wav_async_chunk(
}
return None
- in_initial_phase = initial_chunk_size > 0 and length <= chunk_size
+ in_initial_phase = initial_chunk_size > 0 and initial_chunk_size < chunk_size and length < chunk_size
if in_initial_phase:
- # Initial-chunk phase: emit every initial_chunk_size frames with full accumulated context.
- already_sent = transfer_manager.put_req_chunk[request_id] * initial_chunk_size
- pending = length - already_sent
- if pending <= 0:
+ # IC phase: emit every initial_chunk_size frames with growing left context.
+ if not finished and length % initial_chunk_size != 0:
return None
- if pending < initial_chunk_size and not finished:
- return None
- context_length = min(pending, initial_chunk_size)
- end_index = length
- left_context_size = max(0, length - context_length)
- window_frames = transfer_manager.code_prompt_token_ids[request_id][:length]
+ context_length = (
+ length % initial_chunk_size if (finished and length % initial_chunk_size != 0) else initial_chunk_size
+ )
else:
- # Normal phase: standard chunk_size cadence with left_context sliding window.
- # Offset by initial_coverage so normal starts from where the initial-chunk phase left off.
- initial_coverage = (chunk_size // initial_chunk_size) * initial_chunk_size if initial_chunk_size > 0 else 0
+ # Normal phase: offset so the first normal emit picks up after IC phase.
+ # IC is stateless (may change with load); any mismatch is absorbed by left_context.
+ initial_coverage = (
+ ((chunk_size - 1) // initial_chunk_size) * initial_chunk_size if 0 < initial_chunk_size < chunk_size else 0
+ )
adjusted = length - initial_coverage
- chunk_length = adjusted % chunk_size
- if chunk_length != 0 and not finished:
+ if not finished and adjusted % chunk_size != 0:
return None
+ chunk_length = adjusted % chunk_size
context_length = chunk_length if chunk_length != 0 else chunk_size
- end_index = min(length, left_context_size_config + context_length)
- left_context_size = max(0, int(end_index - context_length))
- window_frames = transfer_manager.code_prompt_token_ids[request_id][-end_index:]
- # Match offline Base ICL decoding by prepending ref_code only once to the first decoder window.
+ end_index = min(length, left_context_size_config + context_length)
+ left_context_size = max(0, end_index - context_length)
+ window_frames = transfer_manager.code_prompt_token_ids[request_id][-end_index:]
+
+ # Prepend ref_code to the first decoder window only.
if transfer_manager.put_req_chunk[request_id] == 0:
ref_code = request_payload.pop(request_id, None)
if isinstance(ref_code, torch.Tensor) and ref_code.numel() > 0:
@@ -165,7 +189,6 @@ def talker2code2wav_async_chunk(
window_frames = ref_frames + window_frames
left_context_size += len(ref_frames)
- # Pack context + chunk into codebook-major flat codes for adapter.
code_predictor_codes = torch.tensor(window_frames).transpose(0, 1).reshape(-1).tolist()
return {
diff --git a/vllm_omni/platforms/npu/stage_configs/qwen3_tts.yaml b/vllm_omni/platforms/npu/stage_configs/qwen3_tts.yaml
index 5ad0cc90e..60659a976 100644
--- a/vllm_omni/platforms/npu/stage_configs/qwen3_tts.yaml
+++ b/vllm_omni/platforms/npu/stage_configs/qwen3_tts.yaml
@@ -96,10 +96,6 @@ runtime:
# Align with Omni: small chunks with sufficient context overlap.
codec_chunk_frames: 25
codec_left_context_frames: 25
- # First chunk size for reduced TTFA (0 = disabled).
- # When > 0, emits small chunks every N frames during the initial phase,
- # then switches to codec_chunk_frames cadence.
- initial_codec_chunk_frames: 0
edges:
- from: 0
diff --git a/vllm_omni/worker/gpu_model_runner.py b/vllm_omni/worker/gpu_model_runner.py
index dc353d8d4..e17d52bdd 100644
--- a/vllm_omni/worker/gpu_model_runner.py
+++ b/vllm_omni/worker/gpu_model_runner.py
@@ -1339,10 +1339,17 @@ class OmniGPUModelRunner(GPUModelRunner):
req_state = self.requests.get(req_id)
if req_state is None:
return
+ # Check if the model declares keys that should stay on GPU
+ gpu_keys: set[str] = set()
+ if hasattr(self, "model") and hasattr(self.model, "gpu_resident_buffer_keys"):
+ gpu_keys = self.model.gpu_resident_buffer_keys
existing = self.model_intermediate_buffer.setdefault(req_id, {})
for k, v in upd.items():
if isinstance(v, torch.Tensor):
- existing[k] = v.detach().to("cpu").contiguous()
+ if k in gpu_keys:
+ existing[k] = v.detach().clone()
+ else:
+ existing[k] = v.detach().to("cpu").contiguous()
elif isinstance(v, list):
existing[k] = [
(item.detach().to("cpu").contiguous() if isinstance(item, torch.Tensor) else item) for item in v
|