diff --git a/docs/contributing/ci/CI_5levels.md b/docs/contributing/ci/CI_5levels.md index dbf87d352..c8493af0a 100644 --- a/docs/contributing/ci/CI_5levels.md +++ b/docs/contributing/ci/CI_5levels.md @@ -67,8 +67,8 @@ Through five levels (L1-L5) and common (Common) specifications, the system clari Online & Offline (multiple deployment scenarios):
real model, normal inference function, normal accuracy <30min - /tests/e2e/online_serving/test_{model_name}_expansion.py
- /tests/e2e/offline_inference/test_{model_name}_expansion.py + /tests/e2e/online_serving/test_{model_name}.py
+ /tests/e2e/offline_inference/test_{model_name}.py Chapter 2
diff --git a/docs/design/feature/async_chunk_design.md b/docs/design/feature/async_chunk_design.md index 40b8f215a..5ae5f5967 100644 --- a/docs/design/feature/async_chunk_design.md +++ b/docs/design/feature/async_chunk_design.md @@ -19,7 +19,7 @@ The `async_chunk` feature enables asynchronous, chunked processing of data acros For qwen3-omni: - **Thinker → Talker**: Per decode step (typically chunk_size=1) -- **Talker → Code2Wav**: Accumulated to `codec_chunk_frames` (default=25) before sending. Set `initial_codec_chunk_frames` to emit smaller chunks during the initial phase for reduced TTFA +- **Talker → Code2Wav**: Accumulated to `codec_chunk_frames` (default=25) before sending. During the initial phase, a dynamic initial chunk size (IC) is automatically selected based on server load to reduce TTFA. Use the per-request `initial_codec_chunk_frames` API field to override. - **Code2Wav**: Streaming decode with code2wav chunk_size With `async_chunk`: diff --git a/docs/getting_started/installation/gpu/cuda.inc.md b/docs/getting_started/installation/gpu/cuda.inc.md index 1e1737e43..d8585ad68 100644 --- a/docs/getting_started/installation/gpu/cuda.inc.md +++ b/docs/getting_started/installation/gpu/cuda.inc.md @@ -34,13 +34,13 @@ uv pip install vllm-omni # --8<-- [start:build-wheel-from-source] #### Installation of vLLM -If you do not need to modify source code of vLLM, you can directly install the stable 0.16.0 release version of the library +If you do not need to modify source code of vLLM, you can directly install the stable 0.17.0 release version of the library ```bash -uv pip install vllm==0.16.0 --torch-backend=auto +uv pip install vllm==0.17.0 --torch-backend=auto ``` -The release 0.16.0 of vLLM is based on PyTorch 2.9.0 which requires CUDA 12.9 environment. +The release 0.17.0 of vLLM is based on PyTorch 2.10.0 which requires CUDA 12.9 environment. #### Installation of vLLM-Omni Since vllm-omni is rapidly evolving, it's recommended to install it from source @@ -56,11 +56,11 @@ If you want to check, modify or debug with source code of vLLM, install the libr ```bash git clone https://github.com/vllm-project/vllm.git cd vllm -git checkout v0.16.0 +git checkout v0.17.0 ``` Set up environment variables to get pre-built wheels. If there are internet problems, just download the whl file manually. And set `VLLM_PRECOMPILED_WHEEL_LOCATION` as your local absolute path of whl file. ```bash -export VLLM_PRECOMPILED_WHEEL_LOCATION=https://github.com/vllm-project/vllm/releases/download/v0.16.0/vllm-0.16.0-cp38-abi3-manylinux_2_31_x86_64.whl +export VLLM_PRECOMPILED_WHEEL_LOCATION=https://github.com/vllm-project/vllm/releases/download/v0.17.0/vllm-0.17.0-cp38-abi3-manylinux_2_31_x86_64.whl ``` Install vllm with command below (If you have no existing PyTorch). ```bash diff --git a/docs/getting_started/installation/gpu/rocm.inc.md b/docs/getting_started/installation/gpu/rocm.inc.md index dc280bb94..701741cd0 100644 --- a/docs/getting_started/installation/gpu/rocm.inc.md +++ b/docs/getting_started/installation/gpu/rocm.inc.md @@ -13,7 +13,7 @@ vLLM-Omni current recommends the steps in under setup through Docker Images. vLLM-Omni is built based on vLLM. Please install it with command below. ```bash -uv pip install vllm==0.16.0+rocm700 --extra-index-url https://wheels.vllm.ai/rocm/0.16.0/rocm700 +uv pip install vllm==0.17.0+rocm700 --extra-index-url https://wheels.vllm.ai/rocm/0.17.0/rocm700 ``` #### Installation of vLLM-Omni @@ -34,13 +34,13 @@ uv pip install onnxruntime-rocm sox # --8<-- [start:build-wheel-from-source] #### Installation of vLLM -If you do not need to modify source code of vLLM, you can directly install the stable 0.16.0 release version of the library +If you do not need to modify source code of vLLM, you can directly install the stable 0.17.0 release version of the library ```bash -uv pip install vllm==0.16.0+rocm700 --extra-index-url https://wheels.vllm.ai/rocm/0.16.0/rocm700 +uv pip install vllm==0.17.0+rocm700 --extra-index-url https://wheels.vllm.ai/rocm/0.17.0/rocm700 ``` -The release 0.16.0 of vLLM requires ROCm 7.0 environment. +The release 0.17.0 of vLLM requires ROCm 7.0 environment. #### Installation of vLLM-Omni Since vllm-omni is rapidly evolving, it's recommended to install it from source @@ -58,7 +58,7 @@ If you want to check, modify or debug with source code of vLLM, install the libr ```bash git clone https://github.com/vllm-project/vllm.git cd vllm -git checkout v0.16.0 +git checkout v0.17.0 python3 -m pip install -r requirements/rocm.txt python3 setup.py develop ``` diff --git a/docs/getting_started/quickstart.md b/docs/getting_started/quickstart.md index a3eece127..6d8197193 100644 --- a/docs/getting_started/quickstart.md +++ b/docs/getting_started/quickstart.md @@ -19,10 +19,10 @@ uv venv --python 3.12 --seed source .venv/bin/activate # On CUDA -uv pip install vllm==0.16.0 --torch-backend=auto +uv pip install vllm==0.17.0 --torch-backend=auto # On ROCm -uv pip install vllm==0.16.0 --extra-index-url https://wheels.vllm.ai/rocm/0.16.0/rocm700 +uv pip install vllm==0.17.0 --extra-index-url https://wheels.vllm.ai/rocm/0.17.0/rocm700 git clone https://github.com/vllm-project/vllm-omni.git cd vllm-omni diff --git a/docs/serving/audio_generate_api.md b/docs/serving/audio_generate_api.md deleted file mode 100644 index e7eaef186..000000000 --- a/docs/serving/audio_generate_api.md +++ /dev/null @@ -1,338 +0,0 @@ -# Audio Generate API - -vLLM-Omni provides an API for text-to-audio generation using diffusion-based models such as Stable Audio. - -Unlike the [Speech API](speech_api.md) which targets text-to-speech synthesis, the Audio Generate API is designed for general-purpose audio generation from text descriptions (sound effects, music, ambient soundscapes, etc.). - -Each server instance runs a single model (specified at startup via `vllm-omni serve --omni`). - -## Quick Start - -### Start the Server - -```bash -vllm-omni serve stabilityai/stable-audio-open-1.0 \ - --host 0.0.0.0 \ - --port 8000 \ - --gpu-memory-utilization 0.9 \ - --trust-remote-code \ - --enforce-eager \ - --omni -``` - -### Generate Audio - -**Using curl:** - -```bash -curl -X POST http://localhost:8000/v1/audio/generate \ - -H "Content-Type: application/json" \ - -d '{ - "input": "The sound of a cat purring", - "audio_length": 10.0 - }' --output cat.wav -``` - -**Using Python:** - -```python -import httpx - -response = httpx.post( - "http://localhost:8000/v1/audio/generate", - json={ - "input": "The sound of a cat purring", - "audio_length": 10.0, - }, - timeout=300.0, -) - -with open("cat.wav", "wb") as f: - f.write(response.content) -``` - -## API Reference - -### Endpoint - -``` -POST /v1/audio/generate -Content-Type: application/json -``` - -### Request Parameters - -| Parameter | Type | Default | Description | -|-----------|------|---------|-------------| -| `input` | string | **required** | Text prompt describing the audio to generate | -| `model` | string | server's model | Model to use (optional, should match server if specified) | -| `response_format` | string | "wav" | Audio format: wav, mp3, flac, pcm, aac, opus | -| `speed` | float | 1.0 | Playback speed (0.25 - 4.0) | - -#### Diffusion Parameters - -| Parameter | Type | Default | Description | -|-----------|------|---------|-------------| -| `audio_length` | float | null | Audio duration in seconds (default value is the max ~47s for `stable-audio-open-1.0`) | -| `audio_start` | float | 0.0 | Audio start time in seconds | -| `negative_prompt` | string | null | Text describing what to avoid in generation | -| `guidance_scale` | float | model default | Classifier-free guidance scale (higher = more adherence to prompt) | -| `num_inference_steps` | int | model default | Number of denoising steps (higher = better quality, slower) | -| `seed` | int | null | Random seed for reproducible generation | - -### Response Format - -Returns binary audio data with the appropriate `Content-Type` header: - -| `response_format` | Content-Type | -|--------------------|--------------| -| `wav` | `audio/wav` | -| `mp3` | `audio/mpeg` | -| `flac` | `audio/flac` | -| `pcm` | `audio/pcm` | -| `aac` | `audio/aac` | -| `opus` | `audio/opus` | - -## Examples - -### Basic Generation - -Generate audio with only a text prompt (model defaults for all other parameters): - -```bash -curl -X POST http://localhost:8000/v1/audio/generate \ - -H "Content-Type: application/json" \ - -d '{ - "input": "The sound of ocean waves crashing on a beach" - }' --output ocean.wav -``` - -### Custom Duration - -Specify an explicit audio length in seconds: - -```bash -curl -X POST http://localhost:8000/v1/audio/generate \ - -H "Content-Type: application/json" \ - -d '{ - "input": "A dog barking", - "audio_length": 5.0 - }' --output dog_5s.wav -``` - -### High Quality with Negative Prompt - -Use a negative prompt to steer generation away from undesired characteristics, and increase inference steps for higher quality: - -```bash -curl -X POST http://localhost:8000/v1/audio/generate \ - -H "Content-Type: application/json" \ - -d '{ - "input": "A piano playing a gentle melody", - "audio_length": 10.0, - "negative_prompt": "Low quality, distorted, noisy", - "guidance_scale": 8.0, - "num_inference_steps": 150 - }' --output piano_hq.wav -``` - -### Reproducible Generation - -Set a `seed` to get deterministic results across runs: - -```bash -curl -X POST http://localhost:8000/v1/audio/generate \ - -H "Content-Type: application/json" \ - -d '{ - "input": "Thunder and rain sounds", - "audio_length": 15.0, - "seed": 42 - }' --output thunder.wav -``` - -### Full Control - -Combine all parameters for precise control over generation: - -```bash -curl -X POST http://localhost:8000/v1/audio/generate \ - -H "Content-Type: application/json" \ - -d '{ - "input": "Thunder and rain sounds", - "audio_length": 15.0, - "negative_prompt": "Low quality", - "guidance_scale": 7.0, - "num_inference_steps": 100, - "seed": 42 - }' --output thunder_rain.wav -``` - -### Quick Generation (Fewer Steps) - -For faster generation with slightly lower quality: - -```bash -curl -X POST http://localhost:8000/v1/audio/generate \ - -H "Content-Type: application/json" \ - -d '{ - "input": "Birds chirping in a forest", - "audio_length": 8.0, - "num_inference_steps": 50 - }' --output birds_quick.wav -``` - -### Python Client - -```python -import httpx - -response = httpx.post( - "http://localhost:8000/v1/audio/generate", - json={ - "input": "Thunder and rain", - "audio_length": 15.0, - "negative_prompt": "Low quality", - "guidance_scale": 7.0, - "num_inference_steps": 100, - "seed": 42, - "response_format": "wav", - }, - timeout=300.0, -) - -with open("thunder.wav", "wb") as f: - f.write(response.content) -``` - -## Parameter Tuning Guide - -### `guidance_scale` - -Controls how closely the generated audio follows the text prompt. - -| Range | Behaviour | -|-------|-----------| -| 3 - 5 | More creative / varied output | -| 7 (default) | Balanced adherence | -| 10+ | Strict adherence to the prompt | - -### `num_inference_steps` - -Controls the number of denoising steps in the diffusion process. - -| Steps | Quality | Speed | Use Case | -|-------|---------|-------|----------| -| 50 | Good | Fast | Quick previews | -| 100 | Very Good | Medium | General purpose | -| 150+ | Excellent | Slow | Final / critical audio | - -### `audio_length` - -Duration of the generated audio clip. For `stable-audio-open-1.0`, the maximum is approximately 47 seconds. If omitted, the model uses its own default length. - -### `negative_prompt` - -Describes characteristics to avoid. Common negative prompts include: - -- `"Low quality, distorted, noisy"` -- `"Silence, static"` -- `"Music"` (when generating sound effects only) - -## Supported Models - -| Model | Description | -|-------|-------------| -| `stabilityai/stable-audio-open-1.0` | Open-source audio generation model, up to ~47 seconds, 44.1 kHz stereo | - -## Error Responses - -### 400 Bad Request - -Invalid or missing parameters: - -```json -{ - "error": { - "message": "Audio generation model did not produce audio output.", - "type": "BadRequestError", - "param": null, - "code": 400 - } -} -``` - -### 404 Not Found - -Model mismatch: - -```json -{ - "error": { - "message": "The model `xxx` does not exist.", - "type": "NotFoundError", - "param": "model", - "code": 404 - } -} -``` - -### 422 Unprocessable Entity - -Pydantic validation failure (e.g. invalid `response_format`, `speed` out of range): - -```json -{ - "detail": [ - { - "type": "literal_error", - "msg": "Input should be 'wav', 'pcm', 'flac', 'mp3', 'aac' or 'opus'", - ... - } - ] -} -``` - -## Troubleshooting - -### "Audio generation model did not produce audio output" - -The model finished but returned no audio data. Verify the server started successfully and the model loaded without errors. - -### Server Not Responding - -```bash -# Check if the server is healthy -curl http://localhost:8000/health -``` - -### Audio Quality Issues - -- Increase `num_inference_steps` (e.g. 150). -- Add a negative prompt: `"Low quality, distorted, noisy"`. -- Increase `guidance_scale` for stronger prompt adherence. - -### Generation Timeout - -- Reduce `num_inference_steps`. -- Reduce `audio_length`. -- Check GPU memory with `nvidia-smi`. - -### Out of Memory - -- Lower `--gpu-memory-utilization` (e.g. 0.8). -- Reduce `audio_length`. - -## Development - -Enable debug logging: - -```bash -vllm-omni serve stabilityai/stable-audio-open-1.0 \ - --host 0.0.0.0 \ - --port 8000 \ - --gpu-memory-utilization 0.9 \ - --trust-remote-code \ - --enforce-eager \ - --omni \ - --uvicorn-log-level debug -``` diff --git a/docs/serving/speech_api.md b/docs/serving/speech_api.md index ca5f7a2ac..2f0695298 100644 --- a/docs/serving/speech_api.md +++ b/docs/serving/speech_api.md @@ -96,7 +96,7 @@ Content-Type: application/json | `language` | string | "Auto" | Language (see supported languages below) | | `instructions` | string | "" | Voice style/emotion instructions | | `max_new_tokens` | integer | 2048 | Maximum tokens to generate | -| `initial_codec_chunk_frames` | integer | null | Initial chunk size for reduced TTFA (overrides stage config) | +| `initial_codec_chunk_frames` | integer | null | Per-request initial chunk size override for TTFA tuning. When null, IC is computed dynamically based on server load. | **Supported languages:** Auto, Chinese, English, Japanese, Korean, German, French, Russian, Portuguese, Spanish, Italian diff --git a/docs/user_guide/examples/offline_inference/qwen3_tts.md b/docs/user_guide/examples/offline_inference/qwen3_tts.md index afadd39f4..352c076ad 100644 --- a/docs/user_guide/examples/offline_inference/qwen3_tts.md +++ b/docs/user_guide/examples/offline_inference/qwen3_tts.md @@ -99,7 +99,8 @@ python end2end.py --query-type CustomVoice --streaming --output-dir /tmp/out_str ``` Each Code2Wav chunk is logged as it arrives (default 25 frames; configurable via `codec_chunk_frames` -and `initial_codec_chunk_frames` in the stage config). The final WAV file is written once generation +in the stage config). The initial chunk size is dynamically selected based on server load for reduced +TTFA, and can be overridden per-request via the `initial_codec_chunk_frames` API field. The final WAV file is written once generation completes. This demonstrates that audio data is available progressively rather than only at the end. > **Note:** Streaming uses `AsyncOmni` internally. The non-streaming path (`Omni`) is unchanged. diff --git a/docs/user_guide/examples/online_serving/qwen3_tts.md b/docs/user_guide/examples/online_serving/qwen3_tts.md index e3de00515..993505b87 100644 --- a/docs/user_guide/examples/online_serving/qwen3_tts.md +++ b/docs/user_guide/examples/online_serving/qwen3_tts.md @@ -17,6 +17,31 @@ Please refer to [README.md](https://github.com/vllm-project/vllm-omni/tree/main/ | `Qwen/Qwen3-TTS-12Hz-0.6B-CustomVoice` | CustomVoice | Smaller/faster variant | | `Qwen/Qwen3-TTS-12Hz-0.6B-Base` | Base | Smaller/faster variant for voice cloning | +## Gradio Demo + +Two interactive Gradio demos are available, both supporting all 3 task types: + +| Demo | File | Transport | Streaming Quality | +| ---- | ---- | --------- | ----------------- | +| Standard | `gradio_demo.py` | HTTP chunked | May have small gaps between chunks | +| FastRTC | `gradio_fastrtc_demo.py` | WebRTC | Gapless streaming (requires `pip install fastrtc`) | + +```bash +# Option 1: Launch server + Standard Gradio together +./run_gradio_demo.sh # CustomVoice (default) +./run_gradio_demo.sh --task-type VoiceDesign # VoiceDesign +./run_gradio_demo.sh --task-type Base # Voice cloning + +# Option 2: If server is already running +python gradio_demo.py --api-base http://localhost:8000 + +# Option 3: FastRTC demo (gapless streaming) +pip install fastrtc +python gradio_fastrtc_demo.py --api-base http://localhost:8000 +``` + +Then open http://localhost:7860 in your browser. + ## Run examples (Qwen3-TTS) ### Launch the Server @@ -112,7 +137,7 @@ The Python client supports the following command-line arguments: - `--voice`: Speaker/voice name (default: `vivian`). Options: `vivian`, `ryan`, `aiden`, etc. - `--language`: Language. Options: `Auto`, `Chinese`, `English`, `Japanese`, `Korean`, `German`, `French`, `Russian`, `Portuguese`, `Spanish`, `Italian` - `--instructions`: Voice style/emotion instructions -- `--ref-audio`: Reference audio file path or URL for voice cloning (Base task). Local paths are automatically base64-encoded by the client before sending to the server. +- `--ref-audio`: Reference audio file path or URL for voice cloning (Base task) - `--ref-text`: Reference audio transcript for voice cloning (Base task) - `--response-format`: Audio output format (default: `wav`). Options: `wav`, `mp3`, `flac`, `pcm`, `aac`, `opus` - `--output` (or `-o`): Output audio file path (default: `tts_output.wav`) @@ -187,6 +212,59 @@ sudo apt install ffmpeg ## API Reference +### Voices Endpoint + +#### GET /v1/audio/voices + +List all available voices/speakers from the loaded model, including both built-in model voices and uploaded custom voices. + +**Response Example:** +```json +{ + "voices": ["vivian", "ryan", "custom_voice_1"], + "uploaded_voices": [ + { + "name": "custom_voice_1", + "consent": "user_consent_id", + "created_at": 1738660000, + "file_size": 1024000, + "mime_type": "audio/wav" + } + ] +} +``` + +#### POST /v1/audio/voices + +Upload a new voice sample for voice cloning in Base task TTS requests. + +**Form Parameters:** +- `audio_sample` (required): Audio file (max 10MB, supported formats: wav, mp3, flac, ogg, aac, webm, mp4) +- `consent` (required): Consent recording ID +- `name` (required): Name for the new voice + +**Response Example:** +```json +{ + "success": true, + "voice": { + "name": "custom_voice_1", + "consent": "user_consent_id", + "created_at": 1738660000, + "mime_type": "audio/wav", + "file_size": 1024000 + } +} +``` + +**Usage Example:** +```bash +curl -X POST http://localhost:8000/v1/audio/voices \ + -F "audio_sample=@/path/to/voice_sample.wav" \ + -F "consent=user_consent_id" \ + -F "name=custom_voice_1" +``` + ### Endpoint ``` @@ -196,20 +274,6 @@ Content-Type: application/json This endpoint follows the [OpenAI Audio Speech API](https://platform.openai.com/docs/api-reference/audio/createSpeech) format with additional Qwen3-TTS parameters. -### Voices Endpoint - -``` -GET /v1/audio/voices -``` - -Lists available voices for the loaded model: - -```json -{ - "voices": ["aiden", "dylan", "eric", "one_anna", "ryan", "serena", "sohee", "uncle_fu", "vivian"] -} -``` - ### Request Body ```json @@ -220,7 +284,7 @@ Lists available voices for the loaded model: "task_type": "CustomVoice", "language": "Auto", "instructions": "Optional style instructions", - "ref_audio": "HTTP URL, base64 data URL, or file:// URI for voice cloning", + "ref_audio": "URL or base64 for voice cloning", "ref_text": "Reference audio transcript", "x_vector_only_mode": false, "max_new_tokens": 2048 @@ -253,6 +317,7 @@ Returns binary audio data with appropriate `Content-Type` header (e.g., `audio/w | `language` | string | "Auto" | Language (see supported languages below) | | `instructions` | string | "" | Voice style/emotion instructions | | `max_new_tokens` | int | 2048 | Maximum tokens to generate | +| `initial_codec_chunk_frames` | int | null | Per-request initial chunk size override for TTFA tuning. When null, IC is computed dynamically based on server load. | | `stream` | bool | false | Stream raw PCM chunks as they are decoded (requires `response_format="pcm"`) | **Supported languages:** Auto, Chinese, English, Japanese, Korean, German, French, Russian, Portuguese, Spanish, Italian @@ -261,7 +326,7 @@ Returns binary audio data with appropriate `Content-Type` header (e.g., `audio/w | Parameter | Type | Required | Description | | -------------------- | ------ | -------- | -------------------------------------------- | -| `ref_audio` | string | **Yes** | Reference audio (HTTP URL, base64 data URL, or `file://` URI with `--allowed-local-media-path`) | +| `ref_audio` | string | **Yes** | Reference audio (URL or base64 data URL) | | `ref_text` | string | No | Transcript of reference audio (for ICL mode) | | `x_vector_only_mode` | bool | No | Use speaker embedding only (no ICL) | @@ -301,11 +366,27 @@ curl -X POST http://localhost:8091/v1/audio/speech \ ## Example materials +??? abstract "gradio_demo.py" + ``````py + --8<-- "examples/online_serving/qwen3_tts/gradio_demo.py" + `````` +??? abstract "gradio_fastrtc_demo.py" + ``````py + --8<-- "examples/online_serving/qwen3_tts/gradio_fastrtc_demo.py" + `````` ??? abstract "openai_speech_client.py" ``````py --8<-- "examples/online_serving/qwen3_tts/openai_speech_client.py" `````` +??? abstract "run_gradio_demo.sh" + ``````sh + --8<-- "examples/online_serving/qwen3_tts/run_gradio_demo.sh" + `````` ??? abstract "run_server.sh" ``````sh --8<-- "examples/online_serving/qwen3_tts/run_server.sh" `````` +??? abstract "tts_common.py" + ``````py + --8<-- "examples/online_serving/qwen3_tts/tts_common.py" + `````` diff --git a/docs/user_guide/examples/online_serving/text_to_audio.md b/docs/user_guide/examples/online_serving/text_to_audio.md deleted file mode 100644 index b84bde6c4..000000000 --- a/docs/user_guide/examples/online_serving/text_to_audio.md +++ /dev/null @@ -1,193 +0,0 @@ -# Text-To-Audio - -Source . - -This example demonstrates how to deploy Stable Audio models for online text-to-audio generation using vLLM-Omni. - -## Supported Models - -| Model | Description | -|-------|-------------| -| `stabilityai/stable-audio-open-1.0` | Open-source audio generation, up to ~47 seconds, 44.1 kHz stereo | - -## Start Server - -### Basic Start - -```bash -vllm-omni serve stabilityai/stable-audio-open-1.0 \ - --host 0.0.0.0 \ - --port 8000 \ - --gpu-memory-utilization 0.9 \ - --trust-remote-code \ - --enforce-eager \ - --omni -``` - -## API Calls - -### Method 1: Using curl - -```bash -# Run all curl examples -bash curl_examples.sh - -# Or execute directly -curl -X POST http://localhost:8000/v1/audio/generate \ - -H "Content-Type: application/json" \ - -d '{ - "input": "The sound of a cat purring", - "audio_length": 10.0 - }' --output cat.wav -``` - -### Method 2: Using Python Client - -```bash -cd examples/online_serving/stable_audio - -# Simple generation -python stable_audio_client.py \ - --text "The sound of a cat purring" - -# With custom duration -python stable_audio_client.py \ - --text "A dog barking" \ - --audio_length 5.0 - -# With all parameters -python stable_audio_client.py \ - --text "Thunder and rain" \ - --audio_length 15.0 \ - --negative_prompt "Low quality" \ - --guidance_scale 7.0 \ - --num_inference_steps 100 \ - --seed 42 \ - --output thunder.wav -``` - -The Python client supports the following command-line arguments: - -- `--api_url`: API endpoint URL (default: `http://localhost:8000/v1/audio/generate`) -- `--text`: Text prompt for audio generation (default: `"The sound of a cat purring"`) -- `--audio_length`: Audio length in seconds (default: `10.0`, max ~47s for `stable-audio-open-1.0`) -- `--audio_start`: Audio start time in seconds (default: `0.0`) -- `--negative_prompt`: Negative prompt for classifier-free guidance (default: `"Low quality"`) -- `--guidance_scale`: Guidance scale for diffusion (default: `7.0`) -- `--num_inference_steps`: Number of inference steps (default: `100`) -- `--seed`: Random seed for reproducibility (default: `None`) -- `--response_format`: Audio output format (default: `wav`). Options: `wav`, `mp3`, `flac`, `pcm` -- `--output`: Output file path (default: `stable_audio_output.wav`) - -### Method 3: Using Python httpx - -```python -import httpx - -response = httpx.post( - "http://localhost:8000/v1/audio/generate", - json={ - "input": "The sound of ocean waves crashing on a beach", - "audio_length": 10.0, - "negative_prompt": "Low quality, distorted", - "guidance_scale": 7.0, - "num_inference_steps": 100, - }, - timeout=300.0, -) - -with open("ocean.wav", "wb") as f: - f.write(response.content) -``` - -## Request Format - -### Simple Generation - -```json -{ - "input": "The sound of ocean waves" -} -``` - -### Generation with Parameters - -```json -{ - "input": "A piano playing a gentle melody", - "audio_length": 10.0, - "negative_prompt": "Low quality, distorted, noisy", - "guidance_scale": 8.0, - "num_inference_steps": 150, - "seed": 42, - "response_format": "wav" -} -``` - -## API Reference - -### Endpoint - -``` -POST /v1/audio/generate -Content-Type: application/json -``` - -### Generation Parameters - -| Parameter | Type | Default | Description | -|-----------|------|---------|-------------| -| `input` | string | **required** | Text prompt describing the audio to generate | -| `model` | string | server's model | Model to use (optional, should match server if specified) | -| `response_format` | string | "wav" | Audio format: wav, mp3, flac, pcm, aac, opus | -| `speed` | float | 1.0 | Playback speed (0.25 - 4.0) | -| `audio_length` | float | null | Audio duration in seconds (max ~47s for `stable-audio-open-1.0`) | -| `audio_start` | float | 0.0 | Audio start time in seconds | -| `negative_prompt` | string | null | Text describing what to avoid in generation | -| `guidance_scale` | float | model default | Classifier-free guidance scale (higher = more adherence to prompt) | -| `num_inference_steps` | int | model default | Number of denoising steps (higher = better quality, slower) | -| `seed` | int | null | Random seed for reproducible generation | - -### Response Format - -Returns binary audio data with appropriate `Content-Type` header (e.g., `audio/wav`). - -## Tuning Tips - -1. **Audio Length**: Keep under 47 seconds for `stable-audio-open-1.0`. -2. **Quality vs Speed**: - - 50 steps: Fast, decent quality (quick previews) - - 100 steps: Good balance (general purpose) - - 150+ steps: High quality, slower (final / critical audio) -3. **Guidance Scale**: - - Lower (3 - 5): More creative / varied output - - Default (7): Good balance - - Higher (10+): Strict adherence to the prompt -4. **Negative Prompts**: Use to avoid unwanted characteristics such as `"Low quality"`, `"distorted"`, `"noisy"`. -5. **Seeds**: Set a fixed seed to get deterministic, reproducible results. - -## File Description - -| File | Description | -|------|-------------| -| `curl_examples.sh` | Curl examples covering common use cases | -| `stable_audio_client.py` | Python client with full CLI argument support | - -## Troubleshooting - -1. **Audio generation model did not produce audio output**: Verify the server started successfully and the model loaded without errors. -2. **Connection refused**: Make sure the server is running on the correct port. -3. **Generation timeout**: Reduce `num_inference_steps` or `audio_length`, and check GPU memory with `nvidia-smi`. -4. **Out of memory**: Lower `--gpu-memory-utilization` or reduce `audio_length`. -5. **Audio quality issues**: Increase `num_inference_steps`, add a negative prompt, or raise `guidance_scale`. - -## Example materials - -??? abstract "stable_audio_client.py" - ``````py - --8<-- "examples/online_serving/stable_audio/stable_audio_client.py" - `````` -??? abstract "curl_examples.sh" - ``````sh - --8<-- "examples/online_serving/stable_audio/curl_examples.sh" - `````` diff --git a/examples/offline_inference/qwen3_tts/README.md b/examples/offline_inference/qwen3_tts/README.md index c198f2c6e..ce2dd588e 100644 --- a/examples/offline_inference/qwen3_tts/README.md +++ b/examples/offline_inference/qwen3_tts/README.md @@ -96,7 +96,8 @@ python end2end.py --query-type CustomVoice --streaming --output-dir /tmp/out_str ``` Each Code2Wav chunk is logged as it arrives (default 25 frames; configurable via `codec_chunk_frames` -and `initial_codec_chunk_frames` in the stage config). The final WAV file is written once generation +in the stage config). The initial chunk size is dynamically selected based on server load for reduced +TTFA, and can be overridden per-request via the `initial_codec_chunk_frames` API field. The final WAV file is written once generation completes. This demonstrates that audio data is available progressively rather than only at the end. > **Note:** Streaming uses `AsyncOmni` internally. The non-streaming path (`Omni`) is unchanged. diff --git a/examples/online_serving/qwen3_tts/README.md b/examples/online_serving/qwen3_tts/README.md index 5088a4ca6..99442c29e 100644 --- a/examples/online_serving/qwen3_tts/README.md +++ b/examples/online_serving/qwen3_tts/README.md @@ -14,6 +14,31 @@ Please refer to [README.md](https://github.com/vllm-project/vllm-omni/tree/main/ | `Qwen/Qwen3-TTS-12Hz-0.6B-CustomVoice` | CustomVoice | Smaller/faster variant | | `Qwen/Qwen3-TTS-12Hz-0.6B-Base` | Base | Smaller/faster variant for voice cloning | +## Gradio Demo + +Two interactive Gradio demos are available, both supporting all 3 task types: + +| Demo | File | Transport | Streaming Quality | +| ---- | ---- | --------- | ----------------- | +| Standard | `gradio_demo.py` | HTTP chunked | May have small gaps between chunks | +| FastRTC | `gradio_fastrtc_demo.py` | WebRTC | Gapless streaming (requires `pip install fastrtc`) | + +```bash +# Option 1: Launch server + Standard Gradio together +./run_gradio_demo.sh # CustomVoice (default) +./run_gradio_demo.sh --task-type VoiceDesign # VoiceDesign +./run_gradio_demo.sh --task-type Base # Voice cloning + +# Option 2: If server is already running +python gradio_demo.py --api-base http://localhost:8000 + +# Option 3: FastRTC demo (gapless streaming) +pip install fastrtc +python gradio_fastrtc_demo.py --api-base http://localhost:8000 +``` + +Then open http://localhost:7860 in your browser. + ## Run examples (Qwen3-TTS) ### Launch the Server @@ -289,7 +314,7 @@ Returns binary audio data with appropriate `Content-Type` header (e.g., `audio/w | `language` | string | "Auto" | Language (see supported languages below) | | `instructions` | string | "" | Voice style/emotion instructions | | `max_new_tokens` | int | 2048 | Maximum tokens to generate | -| `initial_codec_chunk_frames` | int | null | Initial chunk size for reduced TTFA (overrides stage config) | +| `initial_codec_chunk_frames` | int | null | Per-request initial chunk size override for TTFA tuning. When null, IC is computed dynamically based on server load. | | `stream` | bool | false | Stream raw PCM chunks as they are decoded (requires `response_format="pcm"`) | **Supported languages:** Auto, Chinese, English, Japanese, Korean, German, French, Russian, Portuguese, Spanish, Italian diff --git a/examples/online_serving/qwen3_tts/gradio_demo.py b/examples/online_serving/qwen3_tts/gradio_demo.py new file mode 100644 index 000000000..b8a54c6a6 --- /dev/null +++ b/examples/online_serving/qwen3_tts/gradio_demo.py @@ -0,0 +1,404 @@ +"""Gradio demo for Qwen3-TTS online serving via /v1/audio/speech API. + +Uses Gradio's built-in audio component for playback. Supports streaming +(progressive PCM chunks) and non-streaming (full audio download) modes. + +For gapless streaming playback, see gradio_fastrtc_demo.py which uses WebRTC. + +Supports all 3 task types: + - CustomVoice: Predefined speaker with optional style instructions + - VoiceDesign: Natural language voice description + - Base: Voice cloning from reference audio (upload or URL) + +Usage: + # Start the server first (see run_server.sh), then: + python gradio_demo.py --api-base http://localhost:8000 + + # Or use run_gradio_demo.sh to start both server and demo together. +""" + +import argparse +import io + +import gradio as gr +import httpx +import numpy as np +import soundfile as sf +from tts_common import ( + PCM_SAMPLE_RATE, + SUPPORTED_LANGUAGES, + TASK_TYPES, + add_common_args, + build_payload, + fetch_voices, + on_task_type_change, + stream_pcm_chunks, +) + + +def generate_speech( + api_base: str, + text: str, + task_type: str, + voice: str, + language: str, + instructions: str, + ref_audio: tuple | None, + ref_audio_url: str, + ref_text: str, + x_vector_only: bool, + response_format: str, + speed: float, +): + """Non-streaming: call /v1/audio/speech and return audio.""" + payload = build_payload( + text, + task_type, + voice, + language, + instructions, + ref_audio, + ref_audio_url, + ref_text, + x_vector_only, + response_format, + speed, + stream=False, + ) + + try: + with httpx.Client(timeout=300.0) as client: + resp = client.post( + f"{api_base}/v1/audio/speech", + json=payload, + headers={ + "Content-Type": "application/json", + "Authorization": "Bearer EMPTY", + }, + ) + except httpx.TimeoutException: + raise gr.Error("Request timed out. The server may be busy.") + except httpx.ConnectError: + raise gr.Error(f"Cannot connect to server at {api_base}. Make sure the vLLM server is running.") + + if resp.status_code != 200: + raise gr.Error(f"Server error ({resp.status_code}): {resp.text}") + + content_type = resp.headers.get("content-type", "") + if "application/json" in content_type: + try: + error_data = resp.json() + raise gr.Error(f"Server error: {error_data}") + except ValueError: + pass + + try: + if response_format == "pcm": + audio_np = np.frombuffer(resp.content, dtype=np.int16).astype(np.float32) / 32767.0 + return (PCM_SAMPLE_RATE, audio_np) + audio_np, sample_rate = sf.read(io.BytesIO(resp.content)) + if audio_np.ndim > 1: + audio_np = audio_np[:, 0] + return (sample_rate, audio_np.astype(np.float32)) + except Exception as e: + raise gr.Error(f"Failed to decode audio response: {e}") + + +def generate_speech_stream( + api_base: str, + text: str, + task_type: str, + voice: str, + language: str, + instructions: str, + ref_audio: tuple | None, + ref_audio_url: str, + ref_text: str, + x_vector_only: bool, + response_format: str, + speed: float, + chunk_seconds: float = 0.25, +): + """Streaming: yield individual PCM chunks for progressive playback.""" + payload = build_payload( + text, + task_type, + voice, + language, + instructions, + ref_audio, + ref_audio_url, + ref_text, + x_vector_only, + response_format, + speed, + stream=True, + ) + + min_chunk_samples = int(PCM_SAMPLE_RATE * chunk_seconds) + # Pre-buffer 2 chunks before first playback to avoid the gap + # between chunk 1 and chunk 2 during progressive streaming. + prebuffer_chunks = 2 + pending: list[np.ndarray] = [] + pending_len = 0 + chunks_yielded = 0 + prebuffer: list[np.ndarray] = [] + + try: + for samples in stream_pcm_chunks(api_base, payload): + float_samples = samples.astype(np.float32) / 32767.0 + pending.append(float_samples) + pending_len += len(float_samples) + if pending_len >= min_chunk_samples: + audio_chunk = np.concatenate(pending) + pending.clear() + pending_len = 0 + + if chunks_yielded < prebuffer_chunks: + prebuffer.append(audio_chunk) + chunks_yielded += 1 + if chunks_yielded == prebuffer_chunks: + yield (PCM_SAMPLE_RATE, np.concatenate(prebuffer)) + prebuffer.clear() + else: + yield (PCM_SAMPLE_RATE, audio_chunk) + # Flush: prebuffer (if stream was shorter than 2 chunks) + remaining pending + remaining = prebuffer + pending + if remaining: + yield (PCM_SAMPLE_RATE, np.concatenate(remaining)) + except httpx.TimeoutException: + raise gr.Error("Request timed out. The server may be busy.") + except httpx.ConnectError: + raise gr.Error(f"Cannot connect to server at {api_base}. Make sure the vLLM server is running.") + + +def on_stream_change(stream: bool): + """When streaming is enabled, lock format to PCM and disable speed.""" + if stream: + return ( + gr.update(value="pcm", interactive=False), + gr.update(interactive=False), + ) + return ( + gr.update(value="wav", interactive=True), + gr.update(interactive=True), + ) + + +def build_interface(api_base: str, stream_chunk_seconds: float = 0.25): + """Build the Gradio interface.""" + voices = fetch_voices(api_base) + + css = """ + #generate-btn button { width: 100%; } + .task-info { padding: 8px 12px; border-radius: 6px; + background: #f0f4ff; margin-bottom: 8px; } + """ + + with gr.Blocks(css=css, title="Qwen3-TTS Demo") as demo: + gr.Markdown("# Qwen3-TTS Online Serving Demo") + gr.Markdown(f"**Server:** `{api_base}`") + + with gr.Row(): + # Left column: inputs + with gr.Column(scale=3): + text_input = gr.Textbox( + label="Text to Synthesize", + placeholder="Enter text here, e.g., Hello, how are you?", + lines=4, + ) + + with gr.Row(): + task_type = gr.Radio( + choices=TASK_TYPES, + value="CustomVoice", + label="Task Type", + scale=2, + ) + language = gr.Dropdown( + choices=SUPPORTED_LANGUAGES, + value="Auto", + label="Language", + scale=1, + ) + + voice = gr.Dropdown( + choices=voices, + value=voices[0] if voices else None, + label="Speaker", + visible=True, + ) + + instructions = gr.Textbox( + label="Instructions", + placeholder=("e.g., Speak with excitement / A warm, friendly female voice"), + lines=2, + visible=True, + info="Optional style/emotion instructions", + ) + + ref_audio = gr.Audio( + label="Reference Audio (upload for voice cloning)", + type="numpy", + sources=["upload", "microphone"], + visible=False, + ) + ref_audio_url = gr.Textbox( + label="Reference Audio URL", + placeholder=("https://example.com/reference.wav (alternative to uploading)"), + lines=1, + visible=False, + ) + ref_text = gr.Textbox( + label="Reference Audio Transcript", + placeholder=("Transcript of the reference audio (optional, improves quality)"), + lines=2, + visible=False, + ) + x_vector_only = gr.Checkbox( + label="Use x-vector only", + value=False, + visible=False, + info=("Skip reference transcript, use speaker embedding only (lower quality)"), + ) + + with gr.Row(): + response_format = gr.Dropdown( + choices=["wav", "mp3", "flac", "pcm", "aac", "opus"], + value="wav", + label="Audio Format", + scale=1, + ) + speed = gr.Slider( + minimum=0.25, + maximum=4.0, + value=1.0, + step=0.05, + label="Speed", + scale=1, + ) + stream_checkbox = gr.Checkbox( + label="Stream output", + value=False, + info=( + "Enable streaming (PCM format, speed disabled). " + "For gapless streaming, use gradio_fastrtc_demo.py" + ), + scale=1, + ) + + generate_btn = gr.Button( + "Generate Speech", + variant="primary", + size="lg", + elem_id="generate-btn", + ) + + # Right column: output + with gr.Column(scale=2): + audio_output = gr.Audio( + label="Generated Audio", + interactive=False, + streaming=True, + autoplay=True, + ) + gr.Markdown( + "### Task Types\n" + "- **CustomVoice**: Use a predefined speaker " + "(Vivian, Ryan, etc.) with optional style instructions\n" + "- **VoiceDesign**: Describe the desired voice in natural " + "language (instructions required)\n" + "- **Base**: Clone a voice from reference audio " + "(upload a file or provide a URL)" + ) + + # Dynamic UI updates + task_type.change( + fn=on_task_type_change, + inputs=[task_type], + outputs=[ + voice, + instructions, + ref_audio, + ref_audio_url, + ref_text, + x_vector_only, + ], + ) + + stream_checkbox.change( + fn=on_stream_change, + inputs=[stream_checkbox], + outputs=[response_format, speed], + ) + + all_inputs = [ + text_input, + task_type, + voice, + language, + instructions, + ref_audio, + ref_audio_url, + ref_text, + x_vector_only, + response_format, + speed, + ] + + def dispatch(stream_enabled, *args): + if stream_enabled: + yield from generate_speech_stream( + api_base, + *args, + chunk_seconds=stream_chunk_seconds, + ) + else: + yield generate_speech(api_base, *args) + + generate_btn.click( + fn=dispatch, + inputs=[stream_checkbox] + all_inputs, + outputs=[audio_output], + ) + + demo.queue() + return demo + + +def parse_args(): + parser = argparse.ArgumentParser(description="Gradio demo for Qwen3-TTS online serving.") + add_common_args(parser) + parser.add_argument( + "--stream-chunk-seconds", + type=float, + default=0.25, + help=( + "Seconds of audio to buffer per streaming chunk (default: 0.25). " + "Lower = smoother but more overhead, higher = chunkier." + ), + ) + return parser.parse_args() + + +def main(): + args = parse_args() + + print(f"Connecting to vLLM server at: {args.api_base}") + demo = build_interface( + args.api_base, + stream_chunk_seconds=args.stream_chunk_seconds, + ) + + try: + demo.launch( + server_name=args.host, + server_port=args.port, + share=args.share, + ) + except KeyboardInterrupt: + print("\nShutting down...") + + +if __name__ == "__main__": + main() diff --git a/examples/online_serving/qwen3_tts/gradio_fastrtc_demo.py b/examples/online_serving/qwen3_tts/gradio_fastrtc_demo.py new file mode 100644 index 000000000..a40ebfd89 --- /dev/null +++ b/examples/online_serving/qwen3_tts/gradio_fastrtc_demo.py @@ -0,0 +1,274 @@ +"""Gradio demo for Qwen3-TTS using FastRTC for gapless streaming audio. + +Uses WebRTC via FastRTC to deliver audio chunks as a continuous real-time +stream, avoiding the inter-chunk gaps present in Gradio's built-in streaming. + +Requires: pip install fastrtc + +Supports all 3 task types: + - CustomVoice: Predefined speaker with optional style instructions + - VoiceDesign: Natural language voice description + - Base: Voice cloning from reference audio (URL only in this demo) + +Usage: + # Start the server first (see run_server.sh), then: + python gradio_fastrtc_demo.py --api-base http://localhost:8000 + + # With public sharing: + python gradio_fastrtc_demo.py --api-base http://localhost:8000 --share +""" + +import argparse +import logging +from collections.abc import Generator + +import gradio as gr +import numpy as np +from fastrtc import WebRTC +from tts_common import ( + PCM_SAMPLE_RATE, + SUPPORTED_LANGUAGES, + TASK_TYPES, + add_common_args, + build_payload, + fetch_voices, + stream_pcm_chunks, +) + +logger = logging.getLogger(__name__) + + +def make_tts_generator(api_base: str): + """Create a generator function for FastRTC's mode='receive'. + + FastRTC calls this with the additional_inputs values whenever the user + clicks "Start Stream". It must be a synchronous generator yielding + (sample_rate, int16_ndarray) tuples. + """ + + def tts_generate( + text: str, + task_type: str, + voice: str, + language: str, + instructions: str, + ref_audio_url: str, + ref_text: str, + x_vector_only: bool, + ) -> Generator[tuple[int, np.ndarray], None, None]: + if not text or not text.strip(): + return + + payload = build_payload( + text, + task_type, + voice, + language, + instructions, + ref_audio=None, # WebRTC UI doesn't support file upload + ref_audio_url=ref_audio_url, + ref_text=ref_text, + x_vector_only=x_vector_only, + stream=True, + ) + + try: + for samples in stream_pcm_chunks(api_base, payload): + yield (PCM_SAMPLE_RATE, samples) + except Exception: + logger.exception("TTS streaming error") + + return tts_generate + + +def build_interface(api_base: str): + """Build a custom Gradio Blocks UI matching gradio_demo.py layout.""" + voices = fetch_voices(api_base) + handler = make_tts_generator(api_base) + + css = """ + #generate-btn button { width: 100%; } + .task-info { padding: 8px 12px; border-radius: 6px; + background: #f0f4ff; margin-bottom: 8px; } + """ + + with gr.Blocks(css=css, title="Qwen3-TTS (FastRTC)") as demo: + gr.Markdown("# Qwen3-TTS Online Serving Demo (FastRTC)") + gr.Markdown(f"**Server:** `{api_base}` | **Transport:** WebRTC (gapless streaming)") + + with gr.Row(): + # Left column: inputs + with gr.Column(scale=3): + text_input = gr.Textbox( + label="Text to Synthesize", + placeholder="Enter text here, e.g., Hello, how are you?", + lines=4, + ) + + with gr.Row(): + task_type = gr.Radio( + choices=TASK_TYPES, + value="CustomVoice", + label="Task Type", + scale=2, + ) + language = gr.Dropdown( + choices=SUPPORTED_LANGUAGES, + value="Auto", + label="Language", + scale=1, + ) + + voice = gr.Dropdown( + choices=voices, + value=voices[0] if voices else None, + label="Speaker", + visible=True, + ) + + instructions = gr.Textbox( + label="Instructions", + placeholder=("e.g., Speak with excitement / A warm, friendly female voice"), + lines=2, + visible=True, + info="Optional style/emotion instructions", + ) + + # Base (voice clone) controls + # Note: file upload is not supported with WebRTC, + # use a URL instead. + ref_audio_url = gr.Textbox( + label="Reference Audio URL", + placeholder=("https://example.com/reference.wav (alternative to uploading)"), + lines=1, + visible=False, + ) + ref_text = gr.Textbox( + label="Reference Audio Transcript", + placeholder=("Transcript of the reference audio (optional, improves quality)"), + lines=2, + visible=False, + ) + x_vector_only = gr.Checkbox( + label="Use x-vector only", + value=False, + visible=False, + info=("Skip reference transcript, use speaker embedding only (lower quality)"), + ) + + generate_btn = gr.Button( + "Generate Speech", + variant="primary", + size="lg", + elem_id="generate-btn", + ) + + # Right column: output + with gr.Column(scale=2): + webrtc_output = WebRTC( + label="Generated Audio (WebRTC)", + mode="receive", + modality="audio", + ) + gr.Markdown( + "### Task Types\n" + "- **CustomVoice**: Use a predefined speaker " + "(Vivian, Ryan, etc.) with optional style instructions\n" + "- **VoiceDesign**: Describe the desired voice in natural " + "language (instructions required)\n" + "- **Base**: Clone a voice from reference audio " + "(provide a URL)" + ) + + # Dynamic UI updates for task type + # FastRTC doesn't support file upload, so we only toggle + # 5 components (no ref_audio) + def on_task_type_change_fastrtc(task_type: str): + if task_type == "CustomVoice": + return ( + gr.update(visible=True), # voice + gr.update(visible=True, info="Optional style/emotion instructions"), + gr.update(visible=False), # ref_audio_url + gr.update(visible=False), # ref_text + gr.update(visible=False), # x_vector_only + ) + elif task_type == "VoiceDesign": + return ( + gr.update(visible=False), + gr.update(visible=True, info="Required: describe the voice style"), + gr.update(visible=False), + gr.update(visible=False), + gr.update(visible=False), + ) + elif task_type == "Base": + return ( + gr.update(visible=False), + gr.update(visible=False), + gr.update(visible=True), + gr.update(visible=True), + gr.update(visible=True), + ) + return ( + gr.update(visible=True), + gr.update(visible=True), + gr.update(visible=False), + gr.update(visible=False), + gr.update(visible=False), + ) + + task_type.change( + fn=on_task_type_change_fastrtc, + inputs=[task_type], + outputs=[ + voice, + instructions, + ref_audio_url, + ref_text, + x_vector_only, + ], + ) + + # Wire up WebRTC streaming + all_inputs = [ + text_input, + task_type, + voice, + language, + instructions, + ref_audio_url, + ref_text, + x_vector_only, + ] + + webrtc_output.stream( + fn=handler, + inputs=all_inputs, + outputs=[webrtc_output], + trigger=generate_btn.click, + ) + + demo.queue() + return demo + + +def parse_args(): + parser = argparse.ArgumentParser(description="Gradio FastRTC demo for Qwen3-TTS (gapless streaming).") + add_common_args(parser) + return parser.parse_args() + + +def main(): + args = parse_args() + logging.basicConfig(level=logging.INFO) + print(f"Connecting to vLLM server at: {args.api_base}") + + demo = build_interface(args.api_base) + demo.launch( + server_name=args.host, + server_port=args.port, + share=args.share, + ) + + +if __name__ == "__main__": + main() diff --git a/examples/online_serving/qwen3_tts/run_gradio_demo.sh b/examples/online_serving/qwen3_tts/run_gradio_demo.sh new file mode 100644 index 000000000..1c7fe9e37 --- /dev/null +++ b/examples/online_serving/qwen3_tts/run_gradio_demo.sh @@ -0,0 +1,203 @@ +#!/bin/bash +# Launch both vLLM server and Gradio demo for Qwen3-TTS +# +# Usage: +# ./run_gradio_demo.sh # Default: CustomVoice +# ./run_gradio_demo.sh --task-type VoiceDesign # VoiceDesign model +# ./run_gradio_demo.sh --task-type Base --gradio-port 7861 +# +# Options: +# --task-type TYPE Task type: CustomVoice, VoiceDesign, Base (default: CustomVoice) +# --server-port PORT Port for vLLM server (default: 8000) +# --gradio-port PORT Port for Gradio demo (default: 7860) +# --server-host HOST Host for vLLM server (default: 0.0.0.0) +# --gradio-ip IP IP for Gradio demo (default: 127.0.0.1) +# --share Share Gradio demo publicly + +set -e + +# Default values +TASK_TYPE="CustomVoice" +SERVER_PORT=8000 +GRADIO_PORT=7860 +SERVER_HOST="0.0.0.0" +GRADIO_IP="127.0.0.1" +GRADIO_SHARE=false + +# Parse command line arguments +while [[ $# -gt 0 ]]; do + case $1 in + --task-type) + TASK_TYPE="$2" + shift 2 + ;; + --server-port) + SERVER_PORT="$2" + shift 2 + ;; + --gradio-port) + GRADIO_PORT="$2" + shift 2 + ;; + --server-host) + SERVER_HOST="$2" + shift 2 + ;; + --gradio-ip) + GRADIO_IP="$2" + shift 2 + ;; + --share) + GRADIO_SHARE=true + shift + ;; + --help) + echo "Usage: $0 [OPTIONS]" + echo "" + echo "Options:" + echo " --task-type TYPE Task type: CustomVoice, VoiceDesign, Base (default: CustomVoice)" + echo " --server-port PORT Port for vLLM server (default: 8000)" + echo " --gradio-port PORT Port for Gradio demo (default: 7860)" + echo " --server-host HOST Host for vLLM server (default: 0.0.0.0)" + echo " --gradio-ip IP IP for Gradio demo (default: 127.0.0.1)" + echo " --share Share Gradio demo publicly" + echo "" + exit 0 + ;; + *) + echo "Unknown option: $1" + echo "Use --help for usage information" + exit 1 + ;; + esac +done + +# Map task type to model +case "$TASK_TYPE" in + CustomVoice) + MODEL="Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice" + ;; + VoiceDesign) + MODEL="Qwen/Qwen3-TTS-12Hz-1.7B-VoiceDesign" + ;; + Base) + MODEL="Qwen/Qwen3-TTS-12Hz-1.7B-Base" + ;; + *) + echo "Unknown task type: $TASK_TYPE" + echo "Supported: CustomVoice, VoiceDesign, Base" + exit 1 + ;; +esac + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +API_BASE="http://localhost:${SERVER_PORT}" + +echo "==========================================" +echo "Qwen3-TTS Gradio Demo" +echo "==========================================" +echo "Task Type : $TASK_TYPE" +echo "Model : $MODEL" +echo "Server : http://${SERVER_HOST}:${SERVER_PORT}" +echo "Gradio : http://${GRADIO_IP}:${GRADIO_PORT}" +echo "==========================================" + +# Cleanup on exit +cleanup() { + echo "" + echo "Shutting down..." + if [ -n "$SERVER_PID" ]; then + echo "Stopping vLLM server (PID: $SERVER_PID)..." + kill "$SERVER_PID" 2>/dev/null || true + wait "$SERVER_PID" 2>/dev/null || true + fi + if [ -n "$GRADIO_PID" ]; then + echo "Stopping Gradio demo (PID: $GRADIO_PID)..." + kill "$GRADIO_PID" 2>/dev/null || true + wait "$GRADIO_PID" 2>/dev/null || true + fi + echo "Cleanup complete" + exit 0 +} +trap cleanup SIGINT SIGTERM + +# Start vLLM server +echo "" +echo "Starting vLLM server..." +LOG_FILE="/tmp/vllm_tts_server_${SERVER_PORT}.log" + +vllm-omni serve "$MODEL" \ + --stage-configs-path vllm_omni/model_executor/stage_configs/qwen3_tts.yaml \ + --host "$SERVER_HOST" \ + --port "$SERVER_PORT" \ + --gpu-memory-utilization 0.9 \ + --trust-remote-code \ + --enforce-eager \ + --omni 2>&1 | tee "$LOG_FILE" & +SERVER_PID=$! + +# Wait for server startup +echo "" +echo "Waiting for vLLM server to be ready..." +STARTUP_FLAG="/tmp/vllm_tts_startup_flag_${SERVER_PORT}.tmp" +rm -f "$STARTUP_FLAG" + +( + tail -f "$LOG_FILE" 2>/dev/null | grep -m 1 "Application startup complete" > /dev/null && touch "$STARTUP_FLAG" +) & +TAIL_PID=$! + +MAX_WAIT=300 +ELAPSED=0 +while [ $ELAPSED -lt $MAX_WAIT ]; do + if [ -f "$STARTUP_FLAG" ]; then + kill "$TAIL_PID" 2>/dev/null || true + wait "$TAIL_PID" 2>/dev/null || true + echo "" + echo "vLLM server is ready!" + break + fi + if ! kill -0 "$SERVER_PID" 2>/dev/null; then + kill "$TAIL_PID" 2>/dev/null || true + echo "" + echo "Error: vLLM server failed to start" + exit 1 + fi + sleep 1 + ELAPSED=$((ELAPSED + 1)) +done + +rm -f "$STARTUP_FLAG" + +if [ $ELAPSED -ge $MAX_WAIT ]; then + kill "$TAIL_PID" 2>/dev/null || true + echo "Error: Server startup timed out after ${MAX_WAIT}s" + kill "$SERVER_PID" 2>/dev/null || true + exit 1 +fi + +# Start Gradio demo +echo "" +echo "Starting Gradio demo..." +cd "$SCRIPT_DIR" +GRADIO_CMD=("python" "gradio_demo.py" "--api-base" "$API_BASE" "--ip" "$GRADIO_IP" "--port" "$GRADIO_PORT") +if [ "$GRADIO_SHARE" = true ]; then + GRADIO_CMD+=("--share") +fi + +"${GRADIO_CMD[@]}" & +GRADIO_PID=$! + +echo "" +echo "==========================================" +echo "Both services are running!" +echo "==========================================" +echo "vLLM Server : http://${SERVER_HOST}:${SERVER_PORT}" +echo "Gradio Demo : http://${GRADIO_IP}:${GRADIO_PORT}" +echo "" +echo "Press Ctrl+C to stop both services" +echo "==========================================" +echo "" + +wait $SERVER_PID $GRADIO_PID || true +cleanup diff --git a/examples/online_serving/qwen3_tts/tts_common.py b/examples/online_serving/qwen3_tts/tts_common.py new file mode 100644 index 000000000..8e1c77c39 --- /dev/null +++ b/examples/online_serving/qwen3_tts/tts_common.py @@ -0,0 +1,218 @@ +"""Shared constants, helpers, and payload building for Qwen3-TTS Gradio demos.""" + +import base64 +import io + +import gradio as gr +import httpx +import numpy as np +import soundfile as sf + +SUPPORTED_LANGUAGES = [ + "Auto", + "Chinese", + "English", + "Japanese", + "Korean", + "German", + "French", + "Russian", + "Portuguese", + "Spanish", + "Italian", +] + +TASK_TYPES = ["CustomVoice", "VoiceDesign", "Base"] + +PCM_SAMPLE_RATE = 24000 + +DEFAULT_API_BASE = "http://localhost:8000" + + +def fetch_voices(api_base: str) -> list[str]: + """Fetch available voices from the server.""" + try: + with httpx.Client(timeout=10.0) as client: + resp = client.get( + f"{api_base}/v1/audio/voices", + headers={"Authorization": "Bearer EMPTY"}, + ) + if resp.status_code == 200: + data = resp.json() + return data.get("voices", ["Vivian", "Ryan"]) + except Exception: + pass + return ["Vivian", "Ryan"] + + +def encode_audio_to_base64(audio_data: tuple) -> str: + """Encode Gradio audio input (sample_rate, numpy_array) to base64 data URL.""" + sample_rate, audio_np = audio_data + + if audio_np.dtype != np.int16: + if audio_np.dtype in (np.float32, np.float64): + audio_np = np.clip(audio_np, -1.0, 1.0) + audio_np = (audio_np * 32767).astype(np.int16) + else: + audio_np = audio_np.astype(np.int16) + + buf = io.BytesIO() + sf.write(buf, audio_np, sample_rate, format="WAV") + wav_b64 = base64.b64encode(buf.getvalue()).decode("utf-8") + return f"data:audio/wav;base64,{wav_b64}" + + +def build_payload( + text: str, + task_type: str, + voice: str, + language: str, + instructions: str, + ref_audio: tuple | None, + ref_audio_url: str, + ref_text: str, + x_vector_only: bool, + response_format: str = "pcm", + speed: float = 1.0, + stream: bool = True, +) -> dict: + """Build the /v1/audio/speech request payload. + + Raises gr.Error for invalid input so callers don't need to validate. + """ + if not text or not text.strip(): + raise gr.Error("Please enter text to synthesize.") + + payload: dict = { + "input": text.strip(), + "response_format": "pcm" if stream else response_format, + "stream": stream, + } + if not stream: + payload["speed"] = speed + + if task_type: + payload["task_type"] = task_type + if language: + payload["language"] = language + + if task_type == "CustomVoice": + if voice: + payload["voice"] = voice + if instructions and instructions.strip(): + payload["instructions"] = instructions.strip() + + elif task_type == "VoiceDesign": + if not instructions or not instructions.strip(): + raise gr.Error("VoiceDesign task requires voice style instructions.") + payload["instructions"] = instructions.strip() + + elif task_type == "Base": + ref_audio_url_stripped = ref_audio_url.strip() if ref_audio_url else "" + if ref_audio_url_stripped: + payload["ref_audio"] = ref_audio_url_stripped + elif ref_audio is not None: + payload["ref_audio"] = encode_audio_to_base64(ref_audio) + else: + raise gr.Error("Base (voice clone) task requires reference audio. Upload a file or provide a URL.") + if ref_text and ref_text.strip(): + payload["ref_text"] = ref_text.strip() + if x_vector_only: + payload["x_vector_only_mode"] = True + + return payload + + +def on_task_type_change(task_type: str): + """Update UI visibility based on selected task type.""" + if task_type == "CustomVoice": + return ( + gr.update(visible=True), # voice dropdown + gr.update(visible=True, info="Optional style/emotion instructions"), + gr.update(visible=False), # ref_audio + gr.update(visible=False), # ref_audio_url + gr.update(visible=False), # ref_text + gr.update(visible=False), # x_vector_only + ) + elif task_type == "VoiceDesign": + return ( + gr.update(visible=False), + gr.update(visible=True, info="Required: describe the voice style"), + gr.update(visible=False), + gr.update(visible=False), + gr.update(visible=False), + gr.update(visible=False), + ) + elif task_type == "Base": + return ( + gr.update(visible=False), + gr.update(visible=False), + gr.update(visible=True), + gr.update(visible=True), + gr.update(visible=True), + gr.update(visible=True), + ) + return ( + gr.update(visible=True), + gr.update(visible=True), + gr.update(visible=False), + gr.update(visible=False), + gr.update(visible=False), + gr.update(visible=False), + ) + + +def stream_pcm_chunks(api_base: str, payload: dict): + """Stream raw PCM bytes from the server, yielding int16 numpy arrays. + + Handles odd-byte boundaries between network chunks. + """ + leftover = b"" + with httpx.Client(timeout=300.0) as client: + with client.stream( + "POST", + f"{api_base}/v1/audio/speech", + json=payload, + headers={ + "Content-Type": "application/json", + "Authorization": "Bearer EMPTY", + }, + ) as resp: + if resp.status_code != 200: + resp.read() + raise gr.Error(f"Server error ({resp.status_code}): {resp.text}") + for chunk in resp.iter_bytes(): + if not chunk: + continue + raw = leftover + chunk + usable = len(raw) - (len(raw) % 2) + leftover = raw[usable:] + if usable == 0: + continue + yield np.frombuffer(raw[:usable], dtype=np.int16).copy() + + +def add_common_args(parser): + """Add CLI arguments shared by both demos.""" + parser.add_argument( + "--api-base", + default=DEFAULT_API_BASE, + help=f"Base URL for the vLLM API server (default: {DEFAULT_API_BASE}).", + ) + parser.add_argument( + "--host", + default="0.0.0.0", + help="Host/IP for Gradio server (default: 0.0.0.0).", + ) + parser.add_argument( + "--port", + type=int, + default=7860, + help="Port for Gradio server (default: 7860).", + ) + parser.add_argument( + "--share", + action="store_true", + help="Share the Gradio demo publicly.", + ) + return parser diff --git a/examples/online_serving/stable_audio/README.md b/examples/online_serving/stable_audio/README.md deleted file mode 100644 index c8ffaadca..000000000 --- a/examples/online_serving/stable_audio/README.md +++ /dev/null @@ -1,234 +0,0 @@ -# Stable Audio Online Serving - -Generate audio from text prompts using Stable Audio models via an OpenAI-compatible API endpoint. - -## Features - -- **OpenAI-compatible API**: Use `/v1/audio/generate` endpoint -- **Flexible control**: Adjust audio length, guidance scale, inference steps -- **Quality control**: Use negative prompts to avoid unwanted characteristics -- **Reproducible**: Set random seed for deterministic generation - -## Quick Start - -### 1. Start the Server - -```bash -vllm-omni serve stabilityai/stable-audio-open-1.0 \ - --host 0.0.0.0 \ - --port 8000 \ - --gpu-memory-utilization 0.9 \ - --trust-remote-code \ - --enforce-eager \ - --omni -``` - -### 2. Generate Audio - -#### Using curl - -```bash -curl -X POST http://localhost:8000/v1/audio/generate \ - -H "Content-Type: application/json" \ - -d '{ - "input": "The sound of a cat purring", - "audio_length": 10.0 - }' --output cat.wav -``` - -#### Using Python Client - -```bash -python stable_audio_client.py \ - --text "The sound of a cat purring" \ - --audio_length 10.0 \ - --output cat.wav -``` - -#### Using Bash Script - -```bash -bash curl_examples.sh -``` - -## API Reference - -### Endpoint - -``` -POST /v1/audio/generate -``` - -### Request Body - -```json -{ - "input": "Text description of the audio", - "audio_length": 10.0, - "audio_start": 0.0, - "negative_prompt": "Low quality", - "guidance_scale": 7.0, - "num_inference_steps": 100, - "seed": 42, - "response_format": "wav" -} -``` - -### Parameters - -| Parameter | Type | Default | Description | -|-----------|------|---------|-------------| -| `input` | string | **required** | Text prompt describing the audio to generate | -| `audio_length` | float | ~47s | Audio duration in seconds (max ~47s for stable-audio-open-1.0) | -| `audio_start` | float | 0.0 | Audio start time in seconds | -| `negative_prompt` | string | null | Text describing what to avoid in generation | -| `guidance_scale` | float | 7.0 | Classifier-free guidance scale (higher = more adherence to prompt) | -| `num_inference_steps` | int | 50 | Number of denoising steps (higher = better quality, slower) | -| `seed` | int | null | Random seed for reproducibility | -| `response_format` | string | "wav" | Output format: wav, mp3, flac, pcm | - -### Response - -Returns audio data in the requested format (default: WAV). - -## Usage Examples - -### Basic Generation - -```bash -curl -X POST http://localhost:8000/v1/audio/generate \ - -H "Content-Type: application/json" \ - -d '{ - "input": "The sound of ocean waves" - }' --output ocean.wav -``` - -### Custom Duration - -```bash -curl -X POST http://localhost:8000/v1/audio/generate \ - -H "Content-Type: application/json" \ - -d '{ - "input": "A dog barking", - "audio_length": 5.0 - }' --output dog_5s.wav -``` - -### High Quality with Negative Prompt - -```bash -curl -X POST http://localhost:8000/v1/audio/generate \ - -H "Content-Type: application/json" \ - -d '{ - "input": "A piano playing a gentle melody", - "audio_length": 10.0, - "negative_prompt": "Low quality, distorted, noisy", - "guidance_scale": 8.0, - "num_inference_steps": 150 - }' --output piano_hq.wav -``` - -### Reproducible Generation - -```bash -curl -X POST http://localhost:8000/v1/audio/generate \ - -H "Content-Type: application/json" \ - -d '{ - "input": "Thunder and rain sounds", - "audio_length": 15.0, - "seed": 42 - }' --output thunder.wav -``` - -### Quick Generation (Fewer Steps) - -For faster generation with slightly lower quality: - -```bash -curl -X POST http://localhost:8000/v1/audio/generate \ - -H "Content-Type: application/json" \ - -d '{ - "input": "Birds chirping in a forest", - "audio_length": 8.0, - "num_inference_steps": 50 - }' --output birds_quick.wav -``` - -## Python Client Examples - -### Simple Generation - -```bash -python stable_audio_client.py \ - --text "The sound of a cat purring" -``` - -### Custom Parameters - -```bash -python stable_audio_client.py \ - --text "Thunder and rain" \ - --audio_length 15.0 \ - --negative_prompt "Low quality" \ - --guidance_scale 7.0 \ - --num_inference_steps 100 \ - --seed 42 \ - --output thunder.wav -``` - -### Different Output Format - -```bash -python stable_audio_client.py \ - --text "Guitar playing" \ - --response_format mp3 \ - --output guitar.mp3 -``` - -## Tips - -1. **Audio Length**: Keep under 47 seconds for `stable-audio-open-1.0` -2. **Quality vs Speed**: - - 50 steps: Fast, decent quality - - 100 steps: Good balance (default) - - 150+ steps: High quality, slower -3. **Guidance Scale**: - - Lower (3-5): More creative/varied - - Default (7): Good balance - - Higher (10+): More literal to prompt -4. **Negative Prompts**: Use to avoid "Low quality", "distorted", "noisy", etc. -5. **Seeds**: Use same seed for reproducible results - -## Performance - -| Inference Steps | Quality | Speed | Use Case | -|----------------|---------|-------|----------| -| 50 | Good | Fast | Quick previews | -| 100 (default) | Very Good | Medium | Production | -| 150+ | Excellent | Slow | Final/critical audio | - -## Troubleshooting - -### Server not responding -- Check if server is running: `curl http://localhost:8000/health` -- Check server logs for errors - -### Audio quality issues -- Increase `num_inference_steps` (e.g., 150) -- Add negative prompts: `"Low quality, distorted, noisy"` -- Increase `guidance_scale` for more prompt adherence - -### Generation timeout -- Reduce `num_inference_steps` -- Reduce `audio_length` -- Check GPU memory with `nvidia-smi` - -### Wrong audio length -- Ensure `audio_length` is within model limits (~47s max) -- Adjust `audio_start` if trimming is needed - -## See Also - -- [Offline Inference Example](../../offline_inference/text_to_audio/README.md) -- [Stable Audio Model Card](https://huggingface.co/stabilityai/stable-audio-open-1.0) -- [vLLM-Omni Documentation](https://github.com/vllm-project/vllm-omni) diff --git a/examples/online_serving/stable_audio/curl_examples.sh b/examples/online_serving/stable_audio/curl_examples.sh deleted file mode 100755 index 8c4b0c846..000000000 --- a/examples/online_serving/stable_audio/curl_examples.sh +++ /dev/null @@ -1,54 +0,0 @@ -#!/bin/bash -# Examples for using Stable Audio with curl via /v1/audio/generate endpoint - -# Example 1: Simple request with default parameters -echo "Example 1: Simple request with default parameters" -curl -X POST http://localhost:8000/v1/audio/generate \ - -H "Content-Type: application/json" \ - -d '{ - "input": "The sound audience clapping and cheering in a stadium" - }' --output stadium.wav - -# Example 2: Request with custom audio_length -echo "Example 2: Custom audio length (5 seconds)" -curl -X POST http://localhost:8000/v1/audio/generate \ - -H "Content-Type: application/json" \ - -d '{ - "input": "The sound of a dog barking", - "audio_length": 5.0 - }' --output dog_5s.wav - -# Example 3: Request with negative prompt for quality control -echo "Example 3: With negative prompt" -curl -X POST http://localhost:8000/v1/audio/generate \ - -H "Content-Type: application/json" \ - -d '{ - "input": "A piano playing a gentle melody", - "audio_length": 10.0, - "negative_prompt": "Low quality, distorted, noisy" - }' --output piano.wav - -# Example 4: Full control with all parameters -echo "Example 4: Full control (custom length, guidance, steps, seed)" -curl -X POST http://localhost:8000/v1/audio/generate \ - -H "Content-Type: application/json" \ - -d '{ - "input": "Thunder and rain sounds", - "audio_length": 15.0, - "negative_prompt": "Low quality", - "guidance_scale": 7.0, - "num_inference_steps": 100, - "seed": 42 - }' --output thunder_rain.wav - -# Example 5: Quick generation with fewer steps (faster but lower quality) -echo "Example 5: Quick generation (fewer steps)" -curl -X POST http://localhost:8000/v1/audio/generate \ - -H "Content-Type: application/json" \ - -d '{ - "input": "Ocean waves crashing on a beach", - "audio_length": 8.0, - "num_inference_steps": 50 - }' --output ocean.wav - -echo "All examples completed!" diff --git a/examples/online_serving/stable_audio/stable_audio_client.py b/examples/online_serving/stable_audio/stable_audio_client.py deleted file mode 100755 index 8e6a59d36..000000000 --- a/examples/online_serving/stable_audio/stable_audio_client.py +++ /dev/null @@ -1,170 +0,0 @@ -#!/usr/bin/env python3 -""" -OpenAI-compatible client for Stable Audio via /v1/audio/generate endpoint. - -This script demonstrates how to use the OpenAI-compatible speech API -to generate audio from text using Stable Audio models. - -Examples: - # Simple generation - python stable_audio_client.py --text "The sound of a cat purring" - - # With custom duration - python stable_audio_client.py --text "A dog barking" --audio_length 5.0 - - # With all parameters - python stable_audio_client.py --text "Thunder and rain" \ - --audio_length 15.0 \ - --negative_prompt "Low quality" \ - --guidance_scale 7.0 \ - --num_inference_steps 100 \ - --seed 42 \ - --output thunder.wav -""" - -import argparse -import sys - -import requests - - -def parse_args(): - parser = argparse.ArgumentParser(description="Generate audio with Stable Audio via OpenAI-compatible API") - parser.add_argument( - "--api_url", - default="http://localhost:8000/v1/audio/generate", - help="API endpoint URL", - ) - parser.add_argument( - "--text", - default="The sound of a cat purring", - help="Text prompt for audio generation", - ) - parser.add_argument( - "--audio_length", - type=float, - default=10.0, - help="Audio length in seconds (max ~47s for stable-audio-open-1.0)", - ) - parser.add_argument( - "--audio_start", - type=float, - default=0.0, - help="Audio start time in seconds", - ) - parser.add_argument( - "--negative_prompt", - default="Low quality", - help="Negative prompt for classifier-free guidance", - ) - parser.add_argument( - "--guidance_scale", - type=float, - default=7.0, - help="Guidance scale for diffusion (higher = more adherence to prompt)", - ) - parser.add_argument( - "--num_inference_steps", - type=int, - default=100, - help="Number of inference steps (higher = better quality, slower)", - ) - parser.add_argument( - "--seed", - type=int, - default=None, - help="Random seed for reproducibility", - ) - parser.add_argument( - "--output", - default="stable_audio_output.wav", - help="Output file path", - ) - parser.add_argument( - "--response_format", - default="wav", - choices=["wav", "mp3", "flac", "pcm"], - help="Audio output format", - ) - return parser.parse_args() - - -def generate_audio(args): - """Generate audio using the API.""" - - # Build request payload - payload = { - "input": args.text, - "audio_length": args.audio_length, - "audio_start": args.audio_start, - "response_format": args.response_format, - } - - # Add optional parameters - if args.negative_prompt: - payload["negative_prompt"] = args.negative_prompt - if args.guidance_scale: - payload["guidance_scale"] = args.guidance_scale - if args.num_inference_steps: - payload["num_inference_steps"] = args.num_inference_steps - if args.seed is not None: - payload["seed"] = args.seed - - print(f"\n{'=' * 60}") - print("Stable Audio - Text-to-Audio Generation") - print(f"{'=' * 60}") - print(f"API URL: {args.api_url}") - print(f"Prompt: {args.text}") - print(f"Audio length: {args.audio_length}s") - print(f"Negative prompt: {args.negative_prompt}") - print(f"Guidance scale: {args.guidance_scale}") - print(f"Inference steps: {args.num_inference_steps}") - if args.seed is not None: - print(f"Seed: {args.seed}") - print(f"Output: {args.output}") - print(f"{'=' * 60}\n") - - try: - # Make the API request - print("Generating audio...") - response = requests.post( - args.api_url, - json=payload, - headers={"Content-Type": "application/json"}, - timeout=300, # 5 minute timeout for long generations - ) - - # Check for errors - if response.status_code != 200: - print(f"Error: API returned status code {response.status_code}") - print(f"Response: {response.text}") - return False - - # Save the audio - with open(args.output, "wb") as f: - f.write(response.content) - - print(f"✓ Audio saved to {args.output}") - print(f" File size: {len(response.content) / 1024:.1f} KB") - return True - - except requests.exceptions.Timeout: - print("Error: Request timed out. Try reducing inference steps or audio length.") - return False - except requests.exceptions.ConnectionError: - print(f"Error: Could not connect to {args.api_url}") - print("Make sure the server is running.") - return False - except Exception as e: - print(f"Error: {e}") - return False - - -def main(): - args = parse_args() - success = generate_audio(args) - sys.exit(0 if success else 1) - - -if __name__ == "__main__": - main() diff --git a/tests/conftest.py b/tests/conftest.py index c775ae349..619dae3b8 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1057,15 +1057,16 @@ class OmniServer: max_wait = 1200 # 20 minutes start_time = time.time() while time.time() - start_time < max_wait: - try: - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: - sock.settimeout(1) - result = sock.connect_ex((self.host, self.port)) - if result == 0: - print(f"Server ready on {self.host}:{self.port}") - return - except Exception: - pass + # Check for process status + ret = self.proc.poll() + if ret is not None: + raise RuntimeError(f"Server processes exited with code {ret} before becoming ready.") + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: + sock.settimeout(1) + result = sock.connect_ex((self.host, self.port)) + if result == 0: + print(f"Server ready on {self.host}:{self.port}") + return time.sleep(2) raise RuntimeError(f"Server failed to start within {max_wait} seconds") diff --git a/tests/diffusion/distributed/test_hsdp.py b/tests/diffusion/distributed/test_hsdp.py index 6fcaf1e81..e6d782c0a 100644 --- a/tests/diffusion/distributed/test_hsdp.py +++ b/tests/diffusion/distributed/test_hsdp.py @@ -11,7 +11,7 @@ import torch.nn as nn from vllm_omni.diffusion.data import DiffusionParallelConfig from vllm_omni.diffusion.distributed.hsdp import HSDPInferenceConfig -pytestmark = [pytest.mark.diffusion, pytest.mark.parallel, pytest.mark.cpu] +pytestmark = [pytest.mark.diffusion, pytest.mark.parallel, pytest.mark.cpu, pytest.mark.core_model] class TestHSDPInferenceConfig: diff --git a/tests/diffusion/distributed/test_parallel_state_sp_groups.py b/tests/diffusion/distributed/test_parallel_state_sp_groups.py index ac8b1700a..9dda5610b 100644 --- a/tests/diffusion/distributed/test_parallel_state_sp_groups.py +++ b/tests/diffusion/distributed/test_parallel_state_sp_groups.py @@ -11,10 +11,7 @@ import torch from vllm_omni.diffusion.distributed.parallel_state import RankGenerator, set_seq_parallel_pg -pytestmark = [ - pytest.mark.diffusion, - pytest.mark.parallel, -] +pytestmark = [pytest.mark.diffusion, pytest.mark.parallel, pytest.mark.core_model, pytest.mark.cpu] def _fake_new_group_factory(created_groups: list[SimpleNamespace]): diff --git a/tests/diffusion/distributed/test_sp_plan_hooks.py b/tests/diffusion/distributed/test_sp_plan_hooks.py index 6883ae09f..72ec9aaf2 100644 --- a/tests/diffusion/distributed/test_sp_plan_hooks.py +++ b/tests/diffusion/distributed/test_sp_plan_hooks.py @@ -47,6 +47,8 @@ requires_distributed = pytest.mark.skipif( pytestmark = [ pytest.mark.diffusion, pytest.mark.parallel, + pytest.mark.core_model, + pytest.mark.cpu, ] # ============================================================================= diff --git a/tests/diffusion/distributed/test_vae_patch_parallel.py b/tests/diffusion/distributed/test_vae_patch_parallel.py index 43bbc71d5..db74fff1b 100644 --- a/tests/diffusion/distributed/test_vae_patch_parallel.py +++ b/tests/diffusion/distributed/test_vae_patch_parallel.py @@ -8,6 +8,8 @@ import torch from vllm_omni.diffusion.distributed import vae_patch_parallel as vae_patch_parallel +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + class _DummyConfig: def __init__(self, **attrs): diff --git a/tests/diffusion/models/nextstep_1_1/test_nextstep_cfg_parallel_layout.py b/tests/diffusion/models/nextstep_1_1/test_nextstep_cfg_parallel_layout.py index 9caf667be..dd87d2249 100644 --- a/tests/diffusion/models/nextstep_1_1/test_nextstep_cfg_parallel_layout.py +++ b/tests/diffusion/models/nextstep_1_1/test_nextstep_cfg_parallel_layout.py @@ -8,6 +8,8 @@ import vllm_omni.diffusion.models.nextstep_1_1.pipeline_nextstep_1_1 as nextstep from vllm_omni.diffusion.models.nextstep_1_1.modeling_nextstep_heads import FlowMatchingHead from vllm_omni.diffusion.models.nextstep_1_1.pipeline_nextstep_1_1 import NextStep11Pipeline +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + class _DummyImageHead: def __init__(self, token_dim: int): diff --git a/tests/diffusion/models/z_image/test_zimage_tp_constraints.py b/tests/diffusion/models/z_image/test_zimage_tp_constraints.py index a276274a0..b87d51c36 100644 --- a/tests/diffusion/models/z_image/test_zimage_tp_constraints.py +++ b/tests/diffusion/models/z_image/test_zimage_tp_constraints.py @@ -2,6 +2,8 @@ import pytest from vllm_omni.diffusion.models.z_image.z_image_transformer import validate_zimage_tp_constraints +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + def test_validate_zimage_tp_constraints_tp2_ok(): ffn_hidden_dim, final_out_dims, supported_tp = validate_zimage_tp_constraints( diff --git a/tests/diffusion/test_multiproc_executor_concurrency.py b/tests/diffusion/test_multiproc_executor_concurrency.py index ee3b8f4b8..76caa9844 100644 --- a/tests/diffusion/test_multiproc_executor_concurrency.py +++ b/tests/diffusion/test_multiproc_executor_concurrency.py @@ -12,7 +12,7 @@ from vllm_omni.diffusion.data import DiffusionOutput from vllm_omni.diffusion.executor.multiproc_executor import MultiprocDiffusionExecutor from vllm_omni.diffusion.scheduler import Scheduler -pytestmark = [pytest.mark.diffusion] +pytestmark = [pytest.mark.diffusion, pytest.mark.core_model, pytest.mark.cpu] # ───────────────────────────────────────────── helpers ───────────────────── diff --git a/tests/distributed/omni_connectors/test_mooncake_transfer_engine_buffer.py b/tests/distributed/omni_connectors/test_mooncake_transfer_engine_buffer.py index 968109976..55351efcb 100755 --- a/tests/distributed/omni_connectors/test_mooncake_transfer_engine_buffer.py +++ b/tests/distributed/omni_connectors/test_mooncake_transfer_engine_buffer.py @@ -17,7 +17,7 @@ from vllm_omni.distributed.omni_connectors.connectors.mooncake_transfer_engine_c ) # All tests in this file are pure-CPU unit tests for the memory allocator. -pytestmark = [pytest.mark.cpu, pytest.mark.parallel] +pytestmark = [pytest.mark.cpu, pytest.mark.parallel, pytest.mark.core_model] @pytest.mark.core_model diff --git a/tests/distributed/omni_connectors/test_omni_connector_configs.py b/tests/distributed/omni_connectors/test_omni_connector_configs.py index ca73e1ec4..3dc80b57b 100644 --- a/tests/distributed/omni_connectors/test_omni_connector_configs.py +++ b/tests/distributed/omni_connectors/test_omni_connector_configs.py @@ -8,6 +8,8 @@ import pytest # Use the new import path for initialization utilities from vllm_omni.distributed.omni_connectors.utils.initialization import load_omni_transfer_config +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + def get_config_files(): """Helper to find config files.""" diff --git a/tests/entrypoints/openai_api/test_serving_audio_generate.py b/tests/entrypoints/openai_api/test_serving_audio_generate.py deleted file mode 100644 index fcf22bcef..000000000 --- a/tests/entrypoints/openai_api/test_serving_audio_generate.py +++ /dev/null @@ -1,509 +0,0 @@ -# tests/entrypoints/openai_api/test_serving_audio_generate.py -import logging -from inspect import Signature, signature -from unittest.mock import MagicMock, patch - -import pytest -import torch -from fastapi import FastAPI -from fastapi.testclient import TestClient - -from vllm_omni.entrypoints.openai.protocol.audio import ( - CreateAudio, - OpenAICreateAudioGenerateRequest, -) -from vllm_omni.entrypoints.openai.serving_audio_generate import ( - OmniOpenAIServingAudioGenerate, -) -from vllm_omni.inputs.data import OmniDiffusionSamplingParams -from vllm_omni.outputs import OmniRequestOutput - -pytestmark = [pytest.mark.core_model, pytest.mark.cpu] - -logger = logging.getLogger(__name__) - - -# Helper: create a mock audio output for endpoint tests -def create_mock_audio_output( - request_id: str = "audiogen-mock-123", - sample_rate: int = 44100, - num_samples: int = 44100, - audio_key: str = "audio", -) -> OmniRequestOutput: - """Return an OmniRequestOutput mimicking diffusion audio model output.""" - - audio_tensor = torch.sin(torch.linspace(0, 440 * 2 * torch.pi, num_samples)) - - return OmniRequestOutput.from_diffusion( - request_id=request_id, - images=[], - prompt=None, - metrics={}, - multimodal_output={audio_key: audio_tensor, "sr": sample_rate}, - ) - - -def _make_engine_client(*, audio_key: str = "audio", sample_rate: int = 44100): - """Build a mock engine client producing audio output.""" - mock_engine_client = MagicMock() - mock_engine_client.errored = False - mock_engine_client.model_type = "StableAudioPipeline" - mock_engine_client.default_sampling_params_list = [{}] - - async def mock_generate_fn(*args, **kwargs): - yield create_mock_audio_output( - request_id=kwargs.get("request_id", "audiogen-mock"), - sample_rate=sample_rate, - audio_key=audio_key, - ) - - mock_engine_client.generate = MagicMock(side_effect=mock_generate_fn) - return mock_engine_client - - -def _make_server(engine_client=None): - """Build an OmniOpenAIServingAudioGenerate with mocks.""" - if engine_client is None: - engine_client = _make_engine_client() - - mock_models = MagicMock() - mock_models.is_base_model.return_value = True - - return OmniOpenAIServingAudioGenerate( - engine_client=engine_client, - models=mock_models, - request_logger=MagicMock(), - ) - - -@pytest.fixture -def test_app(): - server = _make_server() - - original_fn = server.create_audio_generate - sig = signature(original_fn) - new_params = [p for name, p in sig.parameters.items() if name != "raw_request"] - new_sig = Signature(parameters=new_params, return_annotation=sig.return_annotation) - - async def patched_create_audio_generate(*args, **kwargs): - return await original_fn(*args, **kwargs) - - patched_create_audio_generate.__signature__ = new_sig - server.create_audio_generate = patched_create_audio_generate - - app = FastAPI() - app.add_api_route( - "/v1/audio/generate", - server.create_audio_generate, - methods=["POST"], - response_model=None, - ) - return app - - -@pytest.fixture -def client(test_app): - return TestClient(test_app) - - -# Request Validation (Pydantic model) -class TestRequestValidation: - """Validate OpenAICreateAudioGenerateRequest pydantic constraints.""" - - def test_valid_minimal_request(self): - req = OpenAICreateAudioGenerateRequest(input="A calm piano melody") - assert req.input == "A calm piano melody" - assert req.response_format == "wav" - assert req.speed == 1.0 - - def test_fields_are_wired_correctly(self): - req = OpenAICreateAudioGenerateRequest( - input="rain sounds", - model="stable-audio", - response_format="flac", - speed=1.5, - audio_length=10.0, - audio_start=2.0, - negative_prompt="noise", - guidance_scale=7.5, - num_inference_steps=100, - seed=42, - ) - assert req.input == "rain sounds" - assert req.model == "stable-audio" - assert req.response_format == "flac" - assert req.speed == 1.5 - assert req.audio_length == 10.0 - assert req.audio_start == 2.0 - assert req.negative_prompt == "noise" - assert req.guidance_scale == 7.5 - assert req.num_inference_steps == 100 - assert req.seed == 42 - - def test_invalid_response_format(self): - with pytest.raises(Exception): - OpenAICreateAudioGenerateRequest(input="test", response_format="invalid_format") - - def test_speed_lower_bound(self): - with pytest.raises(Exception): - OpenAICreateAudioGenerateRequest(input="test", speed=0.1) - - def test_speed_upper_bound(self): - with pytest.raises(Exception): - OpenAICreateAudioGenerateRequest(input="test", speed=5.0) - - def test_speed_at_boundaries(self): - req_low = OpenAICreateAudioGenerateRequest(input="test", speed=0.25) - assert req_low.speed == 0.25 - req_high = OpenAICreateAudioGenerateRequest(input="test", speed=4.0) - assert req_high.speed == 4.0 - - def test_stream_format_sse_rejected(self): - with pytest.raises(Exception): - OpenAICreateAudioGenerateRequest(input="test", stream_format="sse") - - def test_stream_format_audio_accepted(self): - req = OpenAICreateAudioGenerateRequest(input="test", stream_format="audio") - assert req.stream_format == "audio" - - -# Constructor & Class Methods -class TestConstructor: - def test_default_init(self): - server = _make_server() - assert server.diffusion_mode is False - - def test_for_diffusion_factory(self): - engine_client = _make_engine_client() - mock_models = MagicMock() - mock_models.is_base_model.return_value = True - - server = OmniOpenAIServingAudioGenerate.for_diffusion( - engine_client=engine_client, - models=mock_models, - request_logger=MagicMock(), - ) - assert server.diffusion_mode is True - - def test_is_stable_audio_model_true(self): - server = _make_server() - assert server._is_stable_audio_model() is True - - def test_is_stable_audio_model_false(self): - engine = _make_engine_client() - engine.model_type = "SomeOtherModel" - server = _make_server(engine_client=engine) - assert server._is_stable_audio_model() is False - - -# Parameter Wiring — verify request params reach the engine -class TestParameterWiring: - """Ensure request parameters are correctly forwarded to the engine.""" - - @pytest.fixture - def server_and_engine(self): - engine = _make_engine_client() - server = _make_server(engine_client=engine) - return server, engine - - @pytest.mark.asyncio - async def test_prompt_wiring(self, server_and_engine): - server, engine = server_and_engine - req = OpenAICreateAudioGenerateRequest(input="birds chirping") - await server.create_audio_generate(req) - - engine.generate.assert_called_once() - call_kwargs = engine.generate.call_args[1] - assert call_kwargs["prompt"]["prompt"] == "birds chirping" - assert call_kwargs["output_modalities"] == ["audio"] - - @pytest.mark.asyncio - async def test_negative_prompt_wiring(self, server_and_engine): - server, engine = server_and_engine - req = OpenAICreateAudioGenerateRequest(input="a calm ocean", negative_prompt="noise distortion") - await server.create_audio_generate(req) - - call_kwargs = engine.generate.call_args[1] - assert call_kwargs["prompt"]["negative_prompt"] == "noise distortion" - - @pytest.mark.asyncio - async def test_negative_prompt_absent(self, server_and_engine): - server, engine = server_and_engine - req = OpenAICreateAudioGenerateRequest(input="a calm ocean") - await server.create_audio_generate(req) - - call_kwargs = engine.generate.call_args[1] - assert "negative_prompt" not in call_kwargs["prompt"] - - @pytest.mark.asyncio - async def test_guidance_scale_wiring(self, server_and_engine): - server, engine = server_and_engine - req = OpenAICreateAudioGenerateRequest(input="test", guidance_scale=12.0) - await server.create_audio_generate(req) - - call_kwargs = engine.generate.call_args[1] - sp = call_kwargs["sampling_params_list"][0] - assert isinstance(sp, OmniDiffusionSamplingParams) - assert sp.guidance_scale == 12.0 - - @pytest.mark.asyncio - async def test_num_inference_steps_wiring(self, server_and_engine): - server, engine = server_and_engine - req = OpenAICreateAudioGenerateRequest(input="test", num_inference_steps=200) - await server.create_audio_generate(req) - - sp = engine.generate.call_args[1]["sampling_params_list"][0] - assert sp.num_inference_steps == 200 - - @pytest.mark.asyncio - async def test_seed_creates_generator(self, server_and_engine): - server, engine = server_and_engine - req = OpenAICreateAudioGenerateRequest(input="test", seed=42) - - with patch("vllm_omni.entrypoints.openai.serving_audio_generate.torch") as mock_torch: - mock_gen = MagicMock() - mock_gen.manual_seed.return_value = mock_gen - mock_torch.Generator.return_value = mock_gen - - await server.create_audio_generate(req) - - mock_torch.Generator.assert_called_once() - mock_gen.manual_seed.assert_called_once_with(42) - - @pytest.mark.asyncio - async def test_seed_none_skips_generator(self, server_and_engine): - server, engine = server_and_engine - req = OpenAICreateAudioGenerateRequest(input="test") - - await server.create_audio_generate(req) - - sp = engine.generate.call_args[1]["sampling_params_list"][0] - assert sp.generator is None - - @pytest.mark.asyncio - async def test_audio_length_wiring(self, server_and_engine): - server, engine = server_and_engine - req = OpenAICreateAudioGenerateRequest(input="test", audio_length=10.0, audio_start=2.0) - await server.create_audio_generate(req) - - sp = engine.generate.call_args[1]["sampling_params_list"][0] - assert sp.extra_args["audio_start_in_s"] == 2.0 - assert sp.extra_args["audio_end_in_s"] == 12.0 # start + length - - @pytest.mark.asyncio - async def test_audio_length_default_start(self, server_and_engine): - server, engine = server_and_engine - req = OpenAICreateAudioGenerateRequest(input="test", audio_length=5.0) - await server.create_audio_generate(req) - - sp = engine.generate.call_args[1]["sampling_params_list"][0] - assert sp.extra_args["audio_start_in_s"] == 0.0 - assert sp.extra_args["audio_end_in_s"] == 5.0 - - @pytest.mark.asyncio - async def test_no_audio_length_skips_extra_args(self, server_and_engine): - server, engine = server_and_engine - req = OpenAICreateAudioGenerateRequest(input="test") - await server.create_audio_generate(req) - - sp = engine.generate.call_args[1]["sampling_params_list"][0] - assert sp.extra_args == {} - - @pytest.mark.asyncio - async def test_defaults_not_set_when_omitted(self, server_and_engine): - """Guidance scale and num_inference_steps keep dataclass defaults when not in request.""" - server, engine = server_and_engine - req = OpenAICreateAudioGenerateRequest(input="test") - await server.create_audio_generate(req) - - sp = engine.generate.call_args[1]["sampling_params_list"][0] - defaults = OmniDiffusionSamplingParams() - assert sp.guidance_scale == defaults.guidance_scale - assert sp.num_inference_steps == defaults.num_inference_steps - - -# Audio Response Format -class TestAudioResponseFormat: - def test_wav_response(self, client): - payload = {"input": "a gentle rain", "response_format": "wav"} - response = client.post("/v1/audio/generate", json=payload) - assert response.status_code == 200 - assert response.headers["content-type"] == "audio/wav" - assert len(response.content) > 0 - - def test_mp3_response(self, client): - payload = {"input": "a gentle rain", "response_format": "mp3"} - response = client.post("/v1/audio/generate", json=payload) - assert response.status_code == 200 - assert response.headers["content-type"] == "audio/mpeg" - assert len(response.content) > 0 - - def test_flac_response(self, client): - payload = {"input": "a gentle rain", "response_format": "flac"} - response = client.post("/v1/audio/generate", json=payload) - assert response.status_code == 200 - assert response.headers["content-type"] == "audio/flac" - assert len(response.content) > 0 - - def test_invalid_format_rejected(self, client): - payload = {"input": "test", "response_format": "banana"} - response = client.post("/v1/audio/generate", json=payload) - assert response.status_code == 422 - - @patch("vllm_omni.entrypoints.openai.serving_audio_generate.OmniOpenAIServingAudioGenerate.create_audio") - def test_speed_parameter_forwarded(self, mock_create_audio, test_app): - mock_audio_response = MagicMock() - mock_audio_response.audio_data = b"dummy_audio" - mock_audio_response.media_type = "audio/wav" - mock_create_audio.return_value = mock_audio_response - - c = TestClient(test_app) - payload = {"input": "test", "response_format": "wav", "speed": 2.5} - c.post("/v1/audio/generate", json=payload) - - mock_create_audio.assert_called_once() - audio_obj = mock_create_audio.call_args[0][0] - assert isinstance(audio_obj, CreateAudio) - assert audio_obj.speed == 2.5 - - @patch("vllm_omni.entrypoints.openai.serving_audio_generate.OmniOpenAIServingAudioGenerate.create_audio") - def test_sample_rate_from_output(self, mock_create_audio, test_app): - mock_audio_response = MagicMock() - mock_audio_response.audio_data = b"dummy" - mock_audio_response.media_type = "audio/wav" - mock_create_audio.return_value = mock_audio_response - - c = TestClient(test_app) - payload = {"input": "test"} - c.post("/v1/audio/generate", json=payload) - - audio_obj = mock_create_audio.call_args[0][0] - assert audio_obj.sample_rate == 44100 # Stable Audio default - - -# Error Handling -class TestErrorHandling: - @pytest.mark.asyncio - async def test_no_output_returns_error(self): - engine = _make_engine_client() - - async def empty_gen(*args, **kwargs): - return - yield # noqa: unreachable – makes this an async generator - - engine.generate = MagicMock(side_effect=empty_gen) - server = _make_server(engine_client=engine) - req = OpenAICreateAudioGenerateRequest(input="test") - resp = await server.create_audio_generate(req) - - # create_error_response returns an ErrorResponse with .error.message - assert "No output generated" in resp.error.message - - @pytest.mark.asyncio - async def test_no_audio_in_output_returns_error(self): - engine = _make_engine_client() - - async def gen_without_audio(*args, **kwargs): - yield OmniRequestOutput.from_diffusion( - request_id="test", - images=[], - prompt=None, - metrics={}, - multimodal_output={}, # no audio key - ) - - engine.generate = MagicMock(side_effect=gen_without_audio) - server = _make_server(engine_client=engine) - req = OpenAICreateAudioGenerateRequest(input="test") - resp = await server.create_audio_generate(req) - - assert "did not produce audio" in resp.error.message - - @pytest.mark.asyncio - async def test_engine_errored_raises(self): - engine = _make_engine_client() - engine.errored = True - engine.dead_error = RuntimeError("engine is dead") - server = _make_server(engine_client=engine) - - req = OpenAICreateAudioGenerateRequest(input="test") - with pytest.raises(RuntimeError, match="engine is dead"): - await server.create_audio_generate(req) - - @pytest.mark.asyncio - async def test_model_outputs_key_fallback(self): - """Audio data under 'model_outputs' key should be accepted.""" - engine = _make_engine_client(audio_key="model_outputs") - server = _make_server(engine_client=engine) - req = OpenAICreateAudioGenerateRequest(input="test") - resp = await server.create_audio_generate(req) - - # Should succeed and return a Response with audio bytes - assert hasattr(resp, "body") - assert len(resp.body) > 0 - - @pytest.mark.asyncio - async def test_value_error_returns_error_response(self): - engine = _make_engine_client() - - async def gen_value_error(*args, **kwargs): - raise ValueError("bad value") - yield # noqa: unreachable - - engine.generate = MagicMock(side_effect=gen_value_error) - server = _make_server(engine_client=engine) - req = OpenAICreateAudioGenerateRequest(input="test") - resp = await server.create_audio_generate(req) - - assert "bad value" in resp.error.message - - @pytest.mark.asyncio - async def test_generic_exception_returns_error_response(self): - engine = _make_engine_client() - - async def gen_runtime_error(*args, **kwargs): - raise RuntimeError("something went wrong") - yield # noqa: unreachable - - engine.generate = MagicMock(side_effect=gen_runtime_error) - server = _make_server(engine_client=engine) - req = OpenAICreateAudioGenerateRequest(input="test") - resp = await server.create_audio_generate(req) - - assert "Audio generation failed" in resp.error.message - - -# End-to-End via TestClient -class TestAudioGenerateAPI: - def test_basic_success(self, client): - payload = {"input": "ambient forest sounds"} - response = client.post("/v1/audio/generate", json=payload) - assert response.status_code == 200 - assert len(response.content) > 0 - - def test_with_all_params(self, client): - payload = { - "input": "gentle piano", - "response_format": "wav", - "speed": 1.0, - "audio_length": 5.0, - "audio_start": 0.0, - "negative_prompt": "noise", - "guidance_scale": 7.0, - "num_inference_steps": 50, - "seed": 123, - } - response = client.post("/v1/audio/generate", json=payload) - assert response.status_code == 200 - assert response.headers["content-type"] == "audio/wav" - - def test_missing_input_rejected(self, client): - payload = {} - response = client.post("/v1/audio/generate", json=payload) - assert response.status_code == 422 - - def test_extra_unknown_fields_ignored(self, client): - payload = {"input": "test", "unknown_field": "value"} - response = client.post("/v1/audio/generate", json=payload) - # Pydantic v2 ignores extra fields by default - assert response.status_code == 200 diff --git a/tests/entrypoints/test_omni_diffusion.py b/tests/entrypoints/test_omni_diffusion.py index 2fbe7a8b4..90b0fd053 100644 --- a/tests/entrypoints/test_omni_diffusion.py +++ b/tests/entrypoints/test_omni_diffusion.py @@ -612,7 +612,7 @@ def test_initialize_stage_configs_called_when_none( """Test that stage configs are auto-loaded when stage_configs_path is None.""" def _fake_loader( - config_path: str, + model: str, stage_configs_path: str | None = None, base_engine_args: dict | None = None, default_stage_cfg_factory=None, @@ -687,7 +687,7 @@ def test_generate_raises_on_length_mismatch(monkeypatch: pytest.MonkeyPatch, moc """Test that generate raises ValueError when sampling_params_list length doesn't match.""" def _fake_loader( - config_path: str, + model: str, stage_configs_path: str | None = None, base_engine_args: dict | None = None, default_stage_cfg_factory=None, @@ -742,7 +742,7 @@ def test_generate_pipeline_and_final_outputs(monkeypatch: pytest.MonkeyPatch, mo stage_cfg1["processed_input"] = ["processed-for-stage-1"] def _fake_loader( - config_path: str, + model: str, stage_configs_path: str | None = None, base_engine_args: dict | None = None, default_stage_cfg_factory=None, @@ -848,7 +848,7 @@ def test_generate_pipeline_with_batch_input(monkeypatch: pytest.MonkeyPatch, moc stage_cfg1["stage_id"] = 1 def _fake_loader( - config_path: str, + model: str, stage_configs_path: str | None = None, base_engine_args: dict | None = None, default_stage_cfg_factory=None, @@ -969,7 +969,7 @@ def test_generate_no_final_output_returns_empty( stage_cfg1["final_output"] = False def _fake_loader( - config_path: str, + model: str, stage_configs_path: str | None = None, base_engine_args: dict | None = None, default_stage_cfg_factory=None, @@ -1062,7 +1062,7 @@ def test_generate_sampling_params_none_use_default( stage_cfg1["final_output"] = False def _fake_loader( - config_path: str, + model: str, stage_configs_path: str | None = None, base_engine_args: dict | None = None, default_stage_cfg_factory=None, @@ -1140,7 +1140,7 @@ def test_wait_for_stages_ready_timeout(monkeypatch: pytest.MonkeyPatch, mocker: """Test that _wait_for_stages_ready handles timeout correctly.""" def _fake_loader( - config_path: str, + model: str, stage_configs_path: str | None = None, base_engine_args: dict | None = None, default_stage_cfg_factory=None, @@ -1202,7 +1202,7 @@ def test_generate_handles_error_messages(monkeypatch: pytest.MonkeyPatch, mocker """Test that generate handles error messages from stages correctly.""" def _fake_loader( - config_path: str, + model: str, stage_configs_path: str | None = None, base_engine_args: dict | None = None, default_stage_cfg_factory=None, @@ -1283,7 +1283,7 @@ def test_close_sends_shutdown_signal(monkeypatch: pytest.MonkeyPatch, mocker: Mo """Test that close() sends shutdown signal to all input queues.""" def _fake_loader( - config_path: str, + model: str, stage_configs_path: str | None = None, base_engine_args: dict | None = None, default_stage_cfg_factory=None, diff --git a/tests/entrypoints/test_omni_llm.py b/tests/entrypoints/test_omni_llm.py index 0e1745c68..b11b14c89 100644 --- a/tests/entrypoints/test_omni_llm.py +++ b/tests/entrypoints/test_omni_llm.py @@ -577,7 +577,7 @@ def test_initialize_stage_configs_called_when_none( """Test that stage configs are auto-loaded when stage_configs_path is None.""" def _fake_loader( - config_path: str, + model: str, stage_configs_path: str | None = None, base_engine_args: dict | None = None, default_stage_cfg_factory=None, @@ -654,7 +654,7 @@ def test_generate_raises_on_length_mismatch(monkeypatch: pytest.MonkeyPatch, moc """Test that generate raises ValueError when sampling_params_list length doesn't match.""" def _fake_loader( - config_path: str, + model: str, stage_configs_path: str | None = None, base_engine_args: dict | None = None, default_stage_cfg_factory=None, @@ -711,7 +711,7 @@ def test_generate_pipeline_and_final_outputs(monkeypatch: pytest.MonkeyPatch, mo stage_cfg1["processed_input"] = ["processed-for-stage-1"] def _fake_loader( - config_path: str, + model: str, stage_configs_path: str | None = None, base_engine_args: dict | None = None, default_stage_cfg_factory=None, @@ -819,7 +819,7 @@ def test_generate_no_final_output_returns_empty( stage_cfg1["final_output"] = False def _fake_loader( - config_path: str, + model: str, stage_configs_path: str | None = None, base_engine_args: dict | None = None, default_stage_cfg_factory=None, @@ -911,7 +911,7 @@ def test_generate_sampling_params_none_use_default( stage_cfg1["final_output"] = False def _fake_loader( - config_path: str, + model: str, stage_configs_path: str | None = None, base_engine_args: dict | None = None, default_stage_cfg_factory=None, @@ -988,7 +988,7 @@ def test_wait_for_stages_ready_timeout(monkeypatch: pytest.MonkeyPatch, mocker: """Test that _wait_for_stages_ready handles timeout correctly.""" def _fake_loader( - config_path: str, + model: str, stage_configs_path: str | None = None, base_engine_args: dict | None = None, default_stage_cfg_factory=None, @@ -1052,7 +1052,7 @@ def test_generate_handles_error_messages(monkeypatch: pytest.MonkeyPatch, mocker """Test that generate handles error messages from stages correctly.""" def _fake_loader( - config_path: str, + model: str, stage_configs_path: str | None = None, base_engine_args: dict | None = None, default_stage_cfg_factory=None, @@ -1135,7 +1135,7 @@ def test_close_sends_shutdown_signal(monkeypatch: pytest.MonkeyPatch, mocker: Mo """Test that close() sends shutdown signal to all input queues.""" def _fake_loader( - config_path: str, + model: str, stage_configs_path: str | None = None, base_engine_args: dict | None = None, default_stage_cfg_factory=None, diff --git a/tests/model_executor/models/qwen2_5_omni/test_qwen2_5_omni_embed.py b/tests/model_executor/models/qwen2_5_omni/test_qwen2_5_omni_embed.py index 9bf230d55..84112336e 100644 --- a/tests/model_executor/models/qwen2_5_omni/test_qwen2_5_omni_embed.py +++ b/tests/model_executor/models/qwen2_5_omni/test_qwen2_5_omni_embed.py @@ -19,6 +19,8 @@ from vllm.model_executor.models.qwen2_5_omni_thinker import ( merge_interleaved_embeddings, ) +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + # Fake token IDs AUDIO_TOKEN_ID = 1001 IMAGE_TOKEN_ID = 1002 diff --git a/tests/model_executor/stage_input_processors/test_qwen3_tts_async_chunk.py b/tests/model_executor/stage_input_processors/test_qwen3_tts_async_chunk.py index 1b6fe2e4b..8d6d21430 100644 --- a/tests/model_executor/stage_input_processors/test_qwen3_tts_async_chunk.py +++ b/tests/model_executor/stage_input_processors/test_qwen3_tts_async_chunk.py @@ -7,16 +7,20 @@ from types import SimpleNamespace import pytest import torch +from vllm_omni.model_executor.stage_input_processors.chunk_size_utils import ( + compute_dynamic_initial_chunk_size, + max_ic_for_chunk_size, +) from vllm_omni.model_executor.stage_input_processors.qwen3_tts import ( talker2code2wav, talker2code2wav_async_chunk, ) -_FRAME = [1, 2, 3, 4] # 4-codebook frame -_Q = len(_FRAME) # num quantizers +_FRAME = [1, 2, 3, 4] +_Q = len(_FRAME) -def _req(rid: str, *, finished: bool, initial_codec_chunk_frames: int | None = None): +def _req(rid, *, finished, initial_codec_chunk_frames=None): ai = None if initial_codec_chunk_frames is not None: entry = SimpleNamespace(list_data=[initial_codec_chunk_frames]) @@ -28,9 +32,10 @@ def _req(rid: str, *, finished: bool, initial_codec_chunk_frames: int | None = N ) -def _tm(*, chunk_frames=25, left_context=25, initial_chunk=0): +def _tm(*, chunk_frames=25, left_context=25, max_num_seqs=1): return SimpleNamespace( code_prompt_token_ids=defaultdict(list), + scheduler_max_num_seqs=max_num_seqs, put_req_chunk=defaultdict(int), request_payload={}, connector=SimpleNamespace( @@ -38,16 +43,14 @@ def _tm(*, chunk_frames=25, left_context=25, initial_chunk=0): "extra": { "codec_chunk_frames": chunk_frames, "codec_left_context_frames": left_context, - "initial_codec_chunk_frames": initial_chunk, } } ), ) -def _call(tm, rid, *, n_frames, put_req=0, finished=False, req_ic=None): +def _call(tm, rid, *, n_frames, finished=False, req_ic=None): tm.code_prompt_token_ids[rid] = [_FRAME[:] for _ in range(n_frames)] - tm.put_req_chunk[rid] = put_req return talker2code2wav_async_chunk( transfer_manager=tm, pooling_output={"audio_codes": torch.zeros((0,))}, @@ -56,84 +59,68 @@ def _call(tm, rid, *, n_frames, put_req=0, finished=False, req_ic=None): ) -def test_does_not_emit_empty_chunk_when_not_finished(): +def test_empty_returns_none(): tm = _tm() - request = _req("rid-empty", finished=False) - - payload = talker2code2wav_async_chunk( + p = talker2code2wav_async_chunk( transfer_manager=tm, pooling_output={"audio_codes": torch.zeros((0,))}, - request=request, + request=_req("r", finished=False), ) - - assert payload is None + assert p is None -def test_flushes_tail_when_finished_without_pooler_output(): +def test_eof_marker_when_finished_empty(): tm = _tm() - rid = "rid-tail" - tm.code_prompt_token_ids[rid] = [_FRAME[:] for _ in range(24)] - request = _req(rid, finished=True) - - payload = talker2code2wav_async_chunk( + p = talker2code2wav_async_chunk( transfer_manager=tm, pooling_output=None, - request=request, + request=_req("r", finished=True), + is_finished=True, ) - - assert payload is not None - assert payload["finished"].item() is True - assert len(payload["code_predictor_codes"]) == _Q * 24 + assert p == {"code_predictor_codes": [], "finished": torch.tensor(True, dtype=torch.bool)} -def test_emits_eof_marker_when_finished_with_no_frames(): +def test_flush_on_finish(): tm = _tm() - request = _req("rid-eof", finished=True) - - payload = talker2code2wav_async_chunk( + tm.code_prompt_token_ids["r"] = [_FRAME[:] for _ in range(24)] + p = talker2code2wav_async_chunk( transfer_manager=tm, pooling_output=None, - request=request, + request=_req("r", finished=True), + is_finished=True, ) - - assert payload == { - "code_predictor_codes": [], - "finished": torch.tensor(True, dtype=torch.bool), - } + assert p is not None + assert p["finished"].item() is True + assert len(p["code_predictor_codes"]) == _Q * 24 _CASES = [ - # Normal path (initial=0): emit at chunk_size boundaries - ((25, 25, 0), (24, 0, False), None), - ((25, 25, 0), (25, 0, False), (0, 25)), - # Initial-chunk phase: hold, first emit, second emit - ((25, 25, 10), (9, 0, False), None), - ((25, 25, 10), (10, 0, False), (0, 10)), - ((25, 25, 10), (20, 1, False), (10, 20)), - # Non-divisible: holds at chunk boundary - ((25, 25, 12), (25, 2, False), None), - # Normal phase: offset by initial_coverage (chunk//initial * initial) - ((25, 25, 10), (45, 2, False), (20, 45)), - # Second normal emit (offset must stay stable) - ((25, 25, 10), (70, 3, False), (25, 50)), - # initial >= chunk clamps to chunk_size (behaves as normal) - ((25, 25, 30), (25, 0, False), (0, 25)), - # finished=True flushes IC tail - ((25, 25, 10), (5, 0, True), (0, 5)), - # finished=True flushes non-divisible IC residual - ((25, 25, 12), (25, 2, True), (24, 25)), - # finished=True flushes normal phase tail - ((25, 25, 10), (30, 2, True), (20, 30)), + # Dynamic IC=16, cs=25, initial_coverage=16 + ((25, 25, 0), 24, False, None), # IC phase: 24%16!=0 -> hold + ((25, 25, 0), 25, False, None), # transition: adjusted=9, hold (no replay) + ((25, 25, 0), 41, False, (16, 41)), # first normal emit, lc=16 + # Per-request IC=10, cs=25, initial_coverage=20 + ((25, 25, 10), 9, False, None), # IC: hold + ((25, 25, 10), 10, False, (0, 10)), # IC: emit at boundary + ((25, 25, 10), 25, False, None), # transition: hold (no replay) + ((25, 25, 10), 45, False, (20, 45)), # first normal emit, lc=20 + ((25, 25, 10), 5, True, (0, 5)), # finished flushes IC tail + ((25, 25, 10), 33, True, (20, 33)), # finished flushes normal tail + # IC=8, cs=16: IC divides chunk_size evenly (edge case) + ((16, 25, 8), 8, False, (0, 8)), # IC: emit + ((16, 25, 8), 16, False, None), # transition: hold (no replay) + ((16, 25, 8), 24, False, (8, 24)), # first normal emit, lc=8 + # Per-request override: IC=15 at n_frames=10 -> 10%15!=0 -> hold + ((25, 25, 15), 10, False, None), ] -@pytest.mark.parametrize("config, state, expected", _CASES) -def test_streaming_decoding_with_variable_initial(config, state, expected): - chunk_frames, left_context, initial_chunk = config - n_frames, put_req, finished = state - - tm = _tm(chunk_frames=chunk_frames, left_context=left_context, initial_chunk=initial_chunk) - payload = _call(tm, "r", n_frames=n_frames, put_req=put_req, finished=finished) +@pytest.mark.parametrize("config, n_frames, finished, expected", _CASES) +def test_streaming_phases(config, n_frames, finished, expected): + chunk_frames, left_context, req_ic_val = config + tm = _tm(chunk_frames=chunk_frames, left_context=left_context) + req_ic = req_ic_val if req_ic_val > 0 else None + payload = _call(tm, "r", n_frames=n_frames, finished=finished, req_ic=req_ic) if expected is None: assert payload is None @@ -144,22 +131,86 @@ def test_streaming_decoding_with_variable_initial(config, state, expected): assert len(payload["code_predictor_codes"]) == _Q * exp_window -def test_per_request_override_activates_initial_phase(): - tm = _tm(initial_chunk=0) - payload = _call(tm, "r-override", n_frames=10, req_ic=10) - assert payload is not None - assert payload["left_context_size"] == 0 - assert len(payload["code_predictor_codes"]) == _Q * 10 +def test_dynamic_ic_adapts_to_load(): + # chunk_size=25 -> max_ic=16, steps=[2,4,8,16] + tm = _tm(max_num_seqs=8) + + # Low load (1/8) -> IC=2 -> emit at 2 + p1 = _call(tm, "r", n_frames=2) + assert p1 is not None + assert len(p1["code_predictor_codes"]) == _Q * 2 + + # High load: add 4 others -> active=5/8 -> IC=8 -> emit at 8 + for i in range(4): + tm.code_prompt_token_ids[f"other-{i}"] = [[0]] + p2 = _call(tm, "r", n_frames=8) + assert p2 is not None + assert len(p2["code_predictor_codes"]) == _Q * 8 + + # Requests past initial phase still count in load factor + tm2 = _tm(max_num_seqs=4) + for i in range(3): + tm2.code_prompt_token_ids[f"long-{i}"] = [[0]] * 50 # well past cs=25 + # active=4/4=1.0 -> IC=16 + p3 = _call(tm2, "new", n_frames=16) + assert p3 is not None + assert len(p3["code_predictor_codes"]) == _Q * 16 -def test_per_request_override_wins_over_stage_config(): - tm = _tm(initial_chunk=5) - payload = _call(tm, "r-override2", n_frames=10, put_req=0, req_ic=15) - assert payload is None +def test_ic_load_change_mid_request(): + """IC stateless: load spike mid-request shifts initial_coverage.""" + tm = _tm(chunk_frames=25, left_context=25, max_num_seqs=8) + + # Low load -> IC=2 -> emit at frame 2 + p1 = _call(tm, "r", n_frames=2) + assert p1 is not None + + # Spike load: 6 others -> IC=16 -> initial_coverage=16 + for i in range(6): + tm.code_prompt_token_ids[f"other-{i}"] = [[0]] * 10 + + # adjusted=25-16=9, 9%25!=0 -> hold + assert _call(tm, "r", n_frames=25) is None + + # First normal emit at 16+25=41 + p3 = _call(tm, "r", n_frames=41) + assert p3 is not None + assert p3["left_context_size"] == 16 + + +@pytest.mark.parametrize( + "active,max_bs,max_ic,expected", + [ + (0, 4, 32, 2), # zero load -> min step + (2, 4, 32, 8), # mid load + (4, 4, 32, 32), # full load + (10, 4, 16, 16), # over capacity, capped + (0, 4, 1, 1), # max_ic below min step + (0, 0, 16, 2), # zero capacity edge case + ], +) +def test_compute_dynamic_initial_chunk_size(active, max_bs, max_ic, expected): + assert compute_dynamic_initial_chunk_size(active, max_bs, max_ic) == expected + + +@pytest.mark.parametrize( + "chunk_size,expected", + [ + (25, 16), + (50, 32), + (70, 64), + (8, 4), + (4, 2), + (2, 1), + (1, 1), + ], +) +def test_max_ic_for_chunk_size(chunk_size, expected): + assert max_ic_for_chunk_size(chunk_size) == expected def test_first_streaming_chunk_prepends_ref_code_context(): - tm = _tm(initial_chunk=10) + tm = _tm() rid = "r-ref" tm.code_prompt_token_ids[rid] = [_FRAME[:] for _ in range(10)] ref_code = torch.tensor([[9, 9, 9, 9], [8, 8, 8, 8]], dtype=torch.long) @@ -177,7 +228,7 @@ def test_first_streaming_chunk_prepends_ref_code_context(): def test_ref_code_context_only_applies_to_first_streaming_chunk(): - tm = _tm(initial_chunk=10) + tm = _tm() rid = "r-ref2" tm.code_prompt_token_ids[rid] = [_FRAME[:] for _ in range(20)] tm.put_req_chunk[rid] = 1 @@ -196,7 +247,7 @@ def test_ref_code_context_only_applies_to_first_streaming_chunk(): def test_ref_code_context_can_be_buffered_before_first_emit(): - tm = _tm(initial_chunk=10) + tm = _tm() rid = "r-ref-buffered" ref_code = torch.tensor([[9, 9, 9, 9], [8, 8, 8, 8]], dtype=torch.long) diff --git a/vllm_omni/diffusion/models/stable_audio/pipeline_stable_audio.py b/vllm_omni/diffusion/models/stable_audio/pipeline_stable_audio.py index a1eb74eaf..22dfc06c5 100644 --- a/vllm_omni/diffusion/models/stable_audio/pipeline_stable_audio.py +++ b/vllm_omni/diffusion/models/stable_audio/pipeline_stable_audio.py @@ -356,6 +356,7 @@ class StableAudioPipeline(nn.Module, SupportAudioOutput): negative_prompt: str | list[str] | None = None, audio_end_in_s: float | None = None, audio_start_in_s: float = 0.0, + num_inference_steps: int = 100, guidance_scale: float = 7.0, num_waveforms_per_prompt: int = 1, generator: torch.Generator | list[torch.Generator] | None = None, @@ -373,6 +374,7 @@ class StableAudioPipeline(nn.Module, SupportAudioOutput): negative_prompt: Negative prompt for CFG audio_end_in_s: Audio end time in seconds (max ~47s for stable-audio-open-1.0) audio_start_in_s: Audio start time in seconds + num_inference_steps: Number of denoising steps guidance_scale: CFG scale num_waveforms_per_prompt: Number of audio outputs per prompt generator: Random generator for reproducibility @@ -393,7 +395,7 @@ class StableAudioPipeline(nn.Module, SupportAudioOutput): elif req.prompts: negative_prompt = ["" if isinstance(p, str) else (p.get("negative_prompt") or "") for p in req.prompts] - num_inference_steps = req.sampling_params.num_inference_steps + num_inference_steps = req.sampling_params.num_inference_steps or num_inference_steps if req.sampling_params.guidance_scale_provided: guidance_scale = req.sampling_params.guidance_scale diff --git a/vllm_omni/distributed/omni_connectors/transfer_adapter/chunk_transfer_adapter.py b/vllm_omni/distributed/omni_connectors/transfer_adapter/chunk_transfer_adapter.py index db8d5dbaa..d96ef74db 100644 --- a/vllm_omni/distributed/omni_connectors/transfer_adapter/chunk_transfer_adapter.py +++ b/vllm_omni/distributed/omni_connectors/transfer_adapter/chunk_transfer_adapter.py @@ -236,6 +236,9 @@ class OmniChunkTransferAdapter(OmniTransferAdapterBase): self.put_req_chunk[request_id] += 1 logger.debug(f"[Stage-{stage_id}] Sent {connector_put_key}") + if is_finished: + self.code_prompt_token_ids.pop(request_id, None) + ######################################################################## # Cleanup ######################################################################## diff --git a/vllm_omni/entrypoints/omni_llm.py b/vllm_omni/entrypoints/omni_llm.py index 91c1c6bed..27f470b4b 100644 --- a/vllm_omni/entrypoints/omni_llm.py +++ b/vllm_omni/entrypoints/omni_llm.py @@ -28,7 +28,6 @@ from vllm_omni.entrypoints.utils import ( load_stage_configs_from_model, load_stage_configs_from_yaml, resolve_model_config_path, - resolve_model_type, ) logger = init_logger(__name__) @@ -85,13 +84,12 @@ class OmniLLM(LLM): self.worker_backend = kwargs.get("worker_backend", "multi_process") self.ray_address = kwargs.get("ray_address", None) self.batch_timeout = batch_timeout - self.model_type = resolve_model_type(model) self.log_stats: bool = bool(log_stats) # Load stage configurations if stage_configs_path is None: - self.config_path = resolve_model_config_path(self.model_type) - self.stage_configs = load_stage_configs_from_model(config_path=self.config_path) + self.config_path = resolve_model_config_path(model) + self.stage_configs = load_stage_configs_from_model(model) else: self.config_path = stage_configs_path self.stage_configs = load_stage_configs_from_yaml(stage_configs_path) diff --git a/vllm_omni/entrypoints/openai/api_server.py b/vllm_omni/entrypoints/openai/api_server.py index bb4649bf1..6db97dd2d 100644 --- a/vllm_omni/entrypoints/openai/api_server.py +++ b/vllm_omni/entrypoints/openai/api_server.py @@ -87,10 +87,7 @@ from vllm_omni.entrypoints.openai.image_api_utils import ( encode_image_base64, parse_size, ) -from vllm_omni.entrypoints.openai.protocol.audio import ( - OpenAICreateAudioGenerateRequest, - OpenAICreateSpeechRequest, -) +from vllm_omni.entrypoints.openai.protocol.audio import OpenAICreateSpeechRequest from vllm_omni.entrypoints.openai.protocol.images import ( ImageData, ImageGenerationRequest, @@ -100,7 +97,6 @@ from vllm_omni.entrypoints.openai.protocol.videos import ( VideoGenerationRequest, VideoGenerationResponse, ) -from vllm_omni.entrypoints.openai.serving_audio_generate import OmniOpenAIServingAudioGenerate from vllm_omni.entrypoints.openai.serving_chat import OmniOpenAIServingChat from vllm_omni.entrypoints.openai.serving_speech import OmniOpenAIServingSpeech from vllm_omni.entrypoints.openai.serving_video import OmniOpenAIServingVideo @@ -179,29 +175,8 @@ class _DiffusionServingModels: provide a lightweight fallback. """ - class _NullModelConfig: - def __getattr__(self, name): - return None - - class _Unsupported: - def __init__(self, name: str): - self.name = name - - def __call__(self, *args, **kwargs): - raise NotImplementedError(f"{self.name} is not supported in diffusion mode") - - def __getattr__(self, attr): - raise NotImplementedError(f"{self.name}.{attr} is not supported in diffusion mode") - def __init__(self, base_model_paths: list[BaseModelPath]) -> None: self._base_model_paths = base_model_paths - self.model_config = self._NullModelConfig() - - def __getattr__(self, name): - """Return a sentinel that raises NotImplementedError if called or - accessed, so any use of unsupported OpenAIServingModels features in - diffusion mode fails loudly with a descriptive message.""" - return self._Unsupported(name) async def show_available_models(self) -> ModelList: return ModelList( @@ -459,10 +434,10 @@ async def omni_init_app_state( # For omni models state.stage_configs = engine_client.stage_configs if hasattr(engine_client, "stage_configs") else None - model_name = served_model_names[0] if served_model_names else args.model # Pure Diffusion mode: use simplified initialization logic if is_pure_diffusion: + model_name = served_model_names[0] if served_model_names else args.model state.vllm_config = None state.diffusion_engine = engine_client state.openai_serving_models = _DiffusionServingModels(base_model_paths) @@ -475,16 +450,6 @@ async def omni_init_app_state( model_name=model_name, ) - # audio related - state.openai_serving_speech = None - state.openai_serving_audio_generate = OmniOpenAIServingAudioGenerate.for_diffusion( - engine_client, - state.openai_serving_models, - request_logger=request_logger, - model_name=model_name, - ) - - # video related diffusion_stage_configs = engine_client.stage_configs if hasattr(engine_client, "stage_configs") else None state.openai_serving_video = OmniOpenAIServingVideo.for_diffusion( diffusion_engine=engine_client, # type: ignore @@ -492,6 +457,7 @@ async def omni_init_app_state( stage_configs=diffusion_stage_configs, ) + state.enable_server_load_tracking = getattr(args, "enable_server_load_tracking", False) state.server_load_metrics = 0 logger.info("Pure diffusion API server initialized for model: %s", model_name) return @@ -766,11 +732,7 @@ async def omni_init_app_state( ) state.openai_serving_speech = OmniOpenAIServingSpeech( - engine_client, state.openai_serving_models, request_logger=request_logger, model_name=model_name - ) - - state.openai_serving_audio_generate = OmniOpenAIServingAudioGenerate( - engine_client, state.openai_serving_models, request_logger=request_logger, model_name=model_name + engine_client, state.openai_serving_models, request_logger=request_logger ) state.openai_serving_video = OmniOpenAIServingVideo( @@ -795,10 +757,6 @@ def Omnispeech(request: Request) -> OmniOpenAIServingSpeech | None: return request.app.state.openai_serving_speech -def OmniAudioGenerate(request: Request) -> OmniOpenAIServingAudioGenerate | None: - return getattr(request.app.state, "openai_serving_audio_generate", None) - - @router.post( "/v1/chat/completions", dependencies=[Depends(validate_json_request)], @@ -909,34 +867,6 @@ async def create_speech(request: OpenAICreateSpeechRequest, raw_request: Request raise HTTPException(status_code=HTTPStatus.INTERNAL_SERVER_ERROR.value, detail=str(e)) from e -@router.post( - "/v1/audio/generate", - dependencies=[Depends(validate_json_request)], - responses={ - HTTPStatus.OK.value: {"content": {"audio/*": {}}}, - HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse}, - HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse}, - HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse}, - }, -) -@with_cancellation -@load_aware_call -async def create_audio_generate(request: OpenAICreateAudioGenerateRequest, raw_request: Request): - handler = OmniAudioGenerate(raw_request) - if handler is None: - base_server = getattr(raw_request.app.state, "openai_serving_tokenization", None) - if base_server is None: - raise HTTPException( - status_code=HTTPStatus.NOT_FOUND.value, - detail="The model does not support Audio Generate API", - ) - return base_server.create_error_response(message="The model does not support Audio Generate API") - try: - return await handler.create_audio_generate(request, raw_request) - except Exception as e: - raise HTTPException(status_code=HTTPStatus.INTERNAL_SERVER_ERROR.value, detail=str(e)) from e - - @router.get( "/v1/audio/voices", responses={ diff --git a/vllm_omni/entrypoints/openai/audio_utils_mixin.py b/vllm_omni/entrypoints/openai/audio_utils_mixin.py index 3023b67ad..13df32ebe 100644 --- a/vllm_omni/entrypoints/openai/audio_utils_mixin.py +++ b/vllm_omni/entrypoints/openai/audio_utils_mixin.py @@ -45,10 +45,6 @@ class AudioMixin: "Only mono (1D) and stereo (2D) are supported." ) - if audio_tensor.ndim == 2 and audio_tensor.shape[0] == 2: - # Convert from [channels, samples] to [samples, channels] - audio_tensor = audio_tensor.T - audio_tensor, sample_rate = self._apply_speed_adjustment(audio_tensor, speed, sample_rate) supported_formats = { diff --git a/vllm_omni/entrypoints/openai/protocol/audio.py b/vllm_omni/entrypoints/openai/protocol/audio.py index 91690cbf0..871787173 100644 --- a/vllm_omni/entrypoints/openai/protocol/audio.py +++ b/vllm_omni/entrypoints/openai/protocol/audio.py @@ -58,7 +58,7 @@ class OpenAICreateSpeechRequest(BaseModel): initial_codec_chunk_frames: int | None = Field( default=None, ge=0, - description="Initial chunk size for reduced TTFA. Overrides stage config for this request.", + description="Per-request initial chunk size override. If null, computed dynamically based on server load.", ) @field_validator("stream_format") @@ -85,53 +85,6 @@ class OpenAICreateSpeechRequest(BaseModel): return self -class OpenAICreateAudioGenerateRequest(BaseModel): - """Request model for audio generation via diffusion models (e.g. Stable Audio).""" - - input: str = Field( - description="Text prompt describing the audio to generate", - ) - model: str | None = None - response_format: Literal["wav", "pcm", "flac", "mp3", "aac", "opus"] = "wav" - speed: float | None = Field( - default=1.0, - ge=0.25, - le=4.0, - ) - stream_format: Literal["sse", "audio"] | None = "audio" - audio_length: float | None = Field( - default=None, - description="Audio length in seconds", - ) - audio_start: float | None = Field( - default=0.0, - description="Audio start time in seconds", - ) - negative_prompt: str | None = Field( - default=None, - description="Negative prompt for classifier-free guidance", - ) - guidance_scale: float | None = Field( - default=None, - description="Guidance scale for diffusion models", - ) - num_inference_steps: int | None = Field( - default=None, - description="Number of inference steps", - ) - seed: int | None = Field( - default=None, - description="Random seed for reproducibility", - ) - - @field_validator("stream_format") - @classmethod - def validate_stream_format(cls, v: str) -> str: - if v == "sse": - raise ValueError("'sse' is not a supported stream_format yet. Please use 'audio'.") - return v - - class CreateAudio(BaseModel): audio_tensor: np.ndarray sample_rate: int = 24000 diff --git a/vllm_omni/entrypoints/openai/serving_audio_generate.py b/vllm_omni/entrypoints/openai/serving_audio_generate.py deleted file mode 100644 index bd5c37157..000000000 --- a/vllm_omni/entrypoints/openai/serving_audio_generate.py +++ /dev/null @@ -1,168 +0,0 @@ -import asyncio - -import torch -from fastapi import Request -from fastapi.responses import Response -from vllm.entrypoints.openai.engine.serving import OpenAIServing -from vllm.logger import init_logger -from vllm.utils import random_uuid - -from vllm_omni.entrypoints.openai.audio_utils_mixin import AudioMixin -from vllm_omni.entrypoints.openai.protocol.audio import ( - AudioResponse, - CreateAudio, - OpenAICreateAudioGenerateRequest, -) -from vllm_omni.inputs.data import OmniDiffusionSamplingParams -from vllm_omni.outputs import OmniRequestOutput - -logger = init_logger(__name__) - - -class OmniOpenAIServingAudioGenerate(OpenAIServing, AudioMixin): - """Serving class for audio generation via diffusion models (e.g. Stable Audio).""" - - def __init__(self, *args, **kwargs): - self.model_name = kwargs.pop("model_name", None) - super().__init__(*args, **kwargs) - self.diffusion_mode = False - - @classmethod - def for_diffusion(cls, *args, **kwargs) -> "OmniOpenAIServingAudioGenerate": - """Create an instance configured to run in diffusion mode.""" - instance = cls(*args, **kwargs) - instance.diffusion_mode = True - return instance - - def _is_stable_audio_model(self) -> bool: - return self.engine_client.model_type == "StableAudioPipeline" - - async def create_audio_generate( - self, - request: OpenAICreateAudioGenerateRequest, - raw_request: Request | None = None, - ): - """ - Generate audio using diffusion-based models (e.g. Stable Audio). - - This endpoint is designed for audio generation models as - opposed to TTS models that specifically generate speech. - """ - - error_check_ret = await self._check_model(request) - if error_check_ret is not None: - logger.error("Error with model %s", error_check_ret) - return error_check_ret - - if self.engine_client.errored: - raise self.engine_client.dead_error - - request_id = f"audiogen-{random_uuid()}" - - try: - default_sr = 44100 # Default sample rate for Stable Audio - - # Build prompt for diffusion audio generation - prompt = { - "prompt": request.input, - } - if request.negative_prompt: - prompt["negative_prompt"] = request.negative_prompt - - # Build sampling params for diffusion - sampling_params_list = [OmniDiffusionSamplingParams(num_outputs_per_prompt=1)] - - # Create generator if seed provided - if request.seed is not None: - from vllm_omni.platforms import current_omni_platform - - generator = torch.Generator(device=current_omni_platform.device_type).manual_seed(request.seed) - sampling_params_list[0].generator = generator - - if request.guidance_scale is not None: - sampling_params_list[0].guidance_scale = request.guidance_scale - - if request.num_inference_steps is not None: - sampling_params_list[0].num_inference_steps = request.num_inference_steps - - # Set up audio duration parameters - if request.audio_length is not None: - audio_length = request.audio_length - audio_start = request.audio_start if request.audio_start is not None else 0.0 - audio_end_in_s = audio_start + audio_length - sampling_params_list[0].extra_args = { - "audio_start_in_s": audio_start, - "audio_end_in_s": audio_end_in_s, - } - - logger.info( - "Audio generation request %s: prompt=%r", - request_id, - request.input[:50] + "..." if len(request.input) > 50 else request.input, - ) - - generator = self.engine_client.generate( - prompt=prompt, - request_id=request_id, - sampling_params_list=sampling_params_list, - output_modalities=["audio"], - ) - - final_output: OmniRequestOutput | None = None - async for res in generator: - final_output = res - - if final_output is None: - return self.create_error_response("No output generated from the model.") - - # Extract audio from output - audio_output = None - if hasattr(final_output, "multimodal_output") and final_output.multimodal_output: - audio_output = final_output.multimodal_output - if not audio_output and hasattr(final_output, "request_output"): - if final_output.request_output and hasattr(final_output.request_output, "multimodal_output"): - audio_output = final_output.request_output.multimodal_output - - # Check for audio data using either "audio" or "model_outputs" key - audio_key = None - if audio_output: - if "audio" in audio_output: - audio_key = "audio" - elif "model_outputs" in audio_output: - audio_key = "model_outputs" - - if not audio_output or audio_key is None: - return self.create_error_response("Audio generation model did not produce audio output.") - - audio_tensor = audio_output[audio_key] - sample_rate = audio_output.get("sr", default_sr) - if hasattr(sample_rate, "item"): - sample_rate = sample_rate.item() - - # Convert tensor to numpy - if hasattr(audio_tensor, "float"): - audio_tensor = audio_tensor.float().detach().cpu().numpy() - - # Squeeze batch dimension if present, but preserve channel dimension for stereo - if audio_tensor.ndim > 1: - audio_tensor = audio_tensor.squeeze() - - audio_obj = CreateAudio( - audio_tensor=audio_tensor, - sample_rate=int(sample_rate), - response_format=request.response_format or "wav", - speed=request.speed or 1.0, - stream_format=request.stream_format, - base64_encode=False, - ) - - audio_response: AudioResponse = self.create_audio(audio_obj) - return Response(content=audio_response.audio_data, media_type=audio_response.media_type) - - except asyncio.CancelledError: - return self.create_error_response("Client disconnected") - except ValueError as e: - return self.create_error_response(e) - except Exception as e: - logger.exception("Audio generation failed: %s", e) - return self.create_error_response(f"Audio generation failed: {e}") diff --git a/vllm_omni/entrypoints/openai/serving_speech.py b/vllm_omni/entrypoints/openai/serving_speech.py index 75cf956fa..7bcf75ace 100644 --- a/vllm_omni/entrypoints/openai/serving_speech.py +++ b/vllm_omni/entrypoints/openai/serving_speech.py @@ -84,7 +84,6 @@ def _validate_path_within_directory(file_path: Path, directory: Path) -> bool: class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): def __init__(self, *args, **kwargs): - self.model_name = kwargs.pop("model_name", None) super().__init__(*args, **kwargs) # Initialize uploaded speakers storage speech_voice_samples_dir = os.environ.get("SPEECH_VOICE_SAMPLES", "/tmp/voice_samples") @@ -757,8 +756,6 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): request_id = f"speech-{random_uuid()}" try: - sampling_params_list = self.engine_client.default_sampling_params_list - default_sr = 24000 # Default sample rate for TTS models if self._is_tts: # Validate TTS parameters validation_error = self._validate_tts_request(request) @@ -784,6 +781,8 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): tts_params.get("task_type", ["unknown"])[0], ) + sampling_params_list = self.engine_client.default_sampling_params_list + generator = self.engine_client.generate( prompt=prompt, request_id=request_id, @@ -810,7 +809,7 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): return self.create_error_response("TTS model did not produce audio output.") audio_tensor = audio_output[audio_key] - sr_raw = audio_output.get("sr", default_sr) + sr_raw = audio_output.get("sr", 24000) sr_val = sr_raw[-1] if isinstance(sr_raw, list) and sr_raw else sr_raw sample_rate = sr_val.item() if hasattr(sr_val, "item") else int(sr_val) diff --git a/vllm_omni/entrypoints/utils.py b/vllm_omni/entrypoints/utils.py index beec074d8..69ce73fc4 100644 --- a/vllm_omni/entrypoints/utils.py +++ b/vllm_omni/entrypoints/utils.py @@ -169,46 +169,22 @@ def _convert_dataclasses_to_dict(obj: Any) -> Any: return obj -def resolve_model_config_path(model_type: str) -> str | None: - """Resolve the stage config file path from the model type. +def resolve_model_config_path(model: str) -> str: + """Resolve the stage config file path from the model name. Resolves stage configuration path based on the model type and device type. First tries to find a device-specific YAML file from stage_configs/{device_type}/ directory. If not found, falls back to the default config file. Args: - model_type: Model type string + model: Model name or path (used to determine model_type) Returns: - String path to the stage configuration file if found, None otherwise - """ - default_config_path = current_omni_platform.get_default_stage_config_path() - config_file_name = f"{model_type}.yaml" - complete_config_path = PROJECT_ROOT / default_config_path / config_file_name - if os.path.exists(complete_config_path): - return str(complete_config_path) - - # Fall back to default config - stage_config_file = f"vllm_omni/model_executor/stage_configs/{model_type}.yaml" - stage_config_path = PROJECT_ROOT / stage_config_file - if not os.path.exists(stage_config_path): - return None - return str(stage_config_path) - - -def resolve_model_type(model: str) -> str: - """Resolve the model type from the model name. - - Args: - model: Model name or path - - Returns: - Model type string (e.g. ``"Qwen3TTSForConditionalGeneration"``, - ``"StableAudioPipeline"``). + String path to the stage configuration file Raises: - ValueError: If the model type cannot be determined from any - available configuration file. + ValueError: If model_type cannot be determined + FileNotFoundError: If no stage config file exists for the model type """ # Try to get config from standard transformers format first try: @@ -241,26 +217,42 @@ def resolve_model_type(model: str) -> str: f"Please ensure the model has proper configuration files with 'model_type' field" ) - return model_type + default_config_path = current_omni_platform.get_default_stage_config_path() + model_type_str = f"{model_type}.yaml" + complete_config_path = PROJECT_ROOT / default_config_path / model_type_str + if os.path.exists(complete_config_path): + return str(complete_config_path) + + # Fall back to default config + stage_config_file = f"vllm_omni/model_executor/stage_configs/{model_type}.yaml" + stage_config_path = PROJECT_ROOT / stage_config_file + if not os.path.exists(stage_config_path): + return None + return str(stage_config_path) -def load_stage_configs_from_model(config_path: str | None, base_engine_args: dict | None = None) -> list: - """Load stage configurations from a resolved config file path. +def load_stage_configs_from_model(model: str, base_engine_args: dict | None = None) -> list: + """Load stage configurations from model's default config file. + + Loads stage configurations based on the model type and device type. + First tries to load a device-specific YAML file from stage_configs/{device_type}/ + directory. If not found, falls back to the default config file. Args: - config_path: Path to the YAML configuration file, or None. - When None, returns an empty list. - base_engine_args: Optional engine arguments to merge with stage configs. + model: Model name or path (used to determine model_type) Returns: - List of stage configuration dictionaries, or empty list if - config_path is None. + List of stage configuration dictionaries + + Raises: + FileNotFoundError: If no stage config file exists for the model type """ if base_engine_args is None: base_engine_args = {} - if config_path is None: + stage_config_path = resolve_model_config_path(model) + if stage_config_path is None: return [] - stage_configs = load_stage_configs_from_yaml(config_path=config_path, base_engine_args=base_engine_args) + stage_configs = load_stage_configs_from_yaml(config_path=stage_config_path, base_engine_args=base_engine_args) return stage_configs @@ -314,10 +306,9 @@ def load_and_resolve_stage_configs( Returns: Tuple of (config_path, stage_configs) """ - model_type = resolve_model_type(model) if stage_configs_path is None: - config_path = resolve_model_config_path(model_type) - stage_configs = load_stage_configs_from_model(config_path, base_engine_args=kwargs) + config_path = resolve_model_config_path(model) + stage_configs = load_stage_configs_from_model(model, base_engine_args=kwargs) if not stage_configs: if default_stage_cfg_factory is not None: default_stage_cfg = default_stage_cfg_factory() diff --git a/vllm_omni/model_executor/models/mimo_audio/mimo_audio.py b/vllm_omni/model_executor/models/mimo_audio/mimo_audio.py index 162c4eb00..9528f0bfb 100644 --- a/vllm_omni/model_executor/models/mimo_audio/mimo_audio.py +++ b/vllm_omni/model_executor/models/mimo_audio/mimo_audio.py @@ -255,7 +255,7 @@ class MiMoAudioLLMDummyInputsBuilder(BaseDummyInputsBuilder[MiMoAudioLLMProcessi dummy_processor_inputs = ProcessorInputs( prompt=dummy_text, - mm_items=dummy_mm_items, + mm_data_items=dummy_mm_items, ) return dummy_processor_inputs @@ -309,10 +309,6 @@ class MiMoAudioDataParser(MultiModalDataParser): if data is None: return AudioProcessorItems(None) - # also check single audio item with sampling rate - if self._is_empty(data) or (isinstance(data, tuple) and self._is_empty(data[0])): - return None - if ( is_list_of(data, float) or isinstance(data, (np.ndarray, torch.Tensor)) diff --git a/vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py b/vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py index ec928195a..6dcd278ac 100644 --- a/vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py +++ b/vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py @@ -159,6 +159,12 @@ class Qwen3OmniMoeForConditionalGeneration( # suppress tokens by setting their probability to ~1e-9 (finite very small) self.suppressed_tokens = self._get_talker_suppressed_tokens() self.requires_raw_input_tokens = True + # Keys that should stay on GPU in model_intermediate_buffer to avoid CPU↔GPU round-trips + self.gpu_resident_buffer_keys: set[str] = { + "last_talker_hidden", + "trailing_text_hidden", + "tts_pad_embed_projected", + } elif self.model_stage == "code2wav": self.enable_update_additional_information = True @@ -223,14 +229,16 @@ class Qwen3OmniMoeForConditionalGeneration( # ==================== Forward Pass ==================== def _get_talker_suppressed_tokens(self): - return [ - i - for i in range( - self.config.talker_config.text_config.vocab_size - 1024, - self.config.talker_config.text_config.vocab_size, - ) - if i != self.config.talker_config.codec_eos_token_id - ] + """Return a boolean mask on GPU for suppressed token positions.""" + vocab_size = self.config.talker_config.text_config.vocab_size + mask = torch.zeros(vocab_size, dtype=torch.bool) + start = vocab_size - 1024 + eos_id = self.config.talker_config.codec_eos_token_id + for i in range(start, vocab_size): + if i != eos_id: + mask[i] = True + # Will be moved to the correct device on first use + return mask def get_mrope_input_positions( self, @@ -578,7 +586,7 @@ class Qwen3OmniMoeForConditionalGeneration( Postprocess the talker hidden states. """ update_dict = {} - update_dict["last_talker_hidden"] = hidden_states[-1, :].detach().to("cpu").contiguous() + update_dict["last_talker_hidden"] = hidden_states[-1, :].detach() return update_dict def talker_preprocess(self, input_ids: torch.Tensor, input_embeds: torch.Tensor, **info_dict: dict): @@ -632,9 +640,9 @@ class Qwen3OmniMoeForConditionalGeneration( if inputs_embeds.shape[-1] == 2048: inputs_embeds = self.text_projection(inputs_embeds) code_predictor_codes, summed_embeddings = self.talker.code_predictor_forward( - input_ids, inputs_embeds.clone(), last_talker_hidden=last_talker_hidden + input_ids, inputs_embeds, last_talker_hidden=last_talker_hidden ) - inputs_embeds = summed_embeddings.clone() + inputs_embeds = summed_embeddings inputs_embeds = (inputs_embeds + text_step).reshape(-1, self.talker_config.text_config.hidden_size) return inputs_embeds, code_predictor_codes.squeeze(-1) @@ -758,7 +766,7 @@ class Qwen3OmniMoeForConditionalGeneration( # compatible with old shape [1,S,D] rem_tail = trailing_text_hidden.squeeze(0) if rem_tail.shape[0] > 0: - update_dict["trailing_text_hidden"] = rem_tail.detach().to("cpu").contiguous() + update_dict["trailing_text_hidden"] = rem_tail.detach() # Also persist projected tts_pad for decode fallback if needed if isinstance(tts_pad_thinker, torch.Tensor): pad_in = tts_pad_thinker @@ -767,7 +775,7 @@ class Qwen3OmniMoeForConditionalGeneration( if pad_in.ndim == 1: pad_in = pad_in.view(1, 1, -1) pad_proj = self.talker.text_projection(pad_in.to(self._module_device(self.talker))) - update_dict["tts_pad_embed_projected"] = pad_proj.detach().to("cpu").contiguous() + update_dict["tts_pad_embed_projected"] = pad_proj.detach() except Exception: pass self._talker_cache_thinker_decode_embeds(info_dict, update_dict) @@ -926,7 +934,7 @@ class Qwen3OmniMoeForConditionalGeneration( if isinstance(q_tail, torch.Tensor) and q_tail.numel() > 0: use_vec = q_tail[0:1, :] new_q_tail = ( - q_tail[1:, :].detach().to("cpu").contiguous() + q_tail[1:, :].detach() if q_tail.shape[0] > 1 else self.tts_pad_embed.to(input_embeds.device, dtype=input_embeds.dtype) ) @@ -1122,16 +1130,10 @@ class Qwen3OmniMoeForConditionalGeneration( # implemented by assigning their logits to log(1e-9). if getattr(self, "model_stage", None) == "talker" and isinstance(logits, torch.Tensor): - try: - logits_cpu = logits.cpu() - logits_cpu[:, self.suppressed_tokens] = -1e9 - logits = logits_cpu.to(logits.device) - except Exception as e: - print(f"Error in logits suppression: {e}") - print(f"logits.shape: {logits.shape}") - print(f"self.suppressed_tokens: {self.suppressed_tokens}") - raise e - logits[:, self.suppressed_tokens] = -1e9 + # Move mask to device once (lazy), then reuse every step + if self.suppressed_tokens.device != logits.device: + self.suppressed_tokens = self.suppressed_tokens.to(logits.device) + logits.masked_fill_(self.suppressed_tokens.unsqueeze(0), -1e9) return logits def sample( diff --git a/vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_code_predictor_mtp.py b/vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_code_predictor_mtp.py index fc7402890..ad372c8f0 100644 --- a/vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_code_predictor_mtp.py +++ b/vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_code_predictor_mtp.py @@ -1,27 +1,16 @@ -"""Qwen3-Omni Code Predictor with MTP (Multi-Token Prediction) support. +"""Qwen3-Omni Code Predictor -- optimized re-prefill, no KV cache. -This module implements the code predictor component for Qwen3-Omni talker models. - -The code predictor generates residual RVQ (Residual Vector Quantization) codes -autoregressively, predicting layers 1 to N based on layer-0 codes from the talker. +* SDPA attention (F.scaled_dot_product_attention) -- no HF backend fallback +* Persistent pre-allocated buffers (_proj_buf, _pos_ids) -- zero per-call alloc +* Inline top-k sampling -- no LogitsProcessorList / custom-op overhead +* torch.compile on inner transformer by default +* No @support_torch_compile / static_forward_context / namedtuple """ -from collections import namedtuple -from typing import Any - import torch import torch.nn as nn import torch.nn.functional as F -from transformers import Cache, PretrainedConfig -from transformers.generation.logits_process import ( - LogitsProcessorList, - TopKLogitsWarper, - TopPLogitsWarper, -) -from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS -from vllm.compilation.decorators import support_torch_compile -from vllm.config import CacheConfig, ModelConfig, VllmConfig, get_current_vllm_config -from vllm.forward_context import get_forward_context +from vllm.config import VllmConfig from vllm.logger import init_logger from vllm.model_executor.layers.layernorm import RMSNorm from vllm.model_executor.layers.linear import ( @@ -32,47 +21,51 @@ from vllm.model_executor.layers.linear import ( from vllm.model_executor.layers.quantization import QuantizationConfig from vllm.model_executor.layers.rotary_embedding import get_rope from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding -from vllm.utils.torch_utils import direct_register_custom_op logger = init_logger(__name__) -# ============================================================================ -# Code Predictor Attention Layer -# ============================================================================ + +# =================================================================== +# Standalone Attention (SDPA, no KV cache, no HF backend fallback) +# =================================================================== class Qwen3OmniCodePredictorAttention(nn.Module): - """Multi-head self-attention for code predictor with vLLM optimization.""" + """Multi-head self-attention for code predictor. + + Uses ``F.scaled_dot_product_attention`` directly. No KV cache -- the code + predictor always re-prefills the full (short) sequence each AR step. + + Input : [B, seq_len, hidden_size] + Output: [B, seq_len, hidden_size] + """ def __init__( self, config, - layer_idx: int, - vllm_config: VllmConfig = None, quant_config: QuantizationConfig | None = None, prefix: str = "", ): super().__init__() - - self.num_heads = config.code_predictor_config.num_attention_heads - self.num_key_value_heads = config.code_predictor_config.num_key_value_heads + cp_cfg = config.code_predictor_config + self.num_heads = cp_cfg.num_attention_heads + self.num_kv_heads = cp_cfg.num_key_value_heads self.head_dim = getattr( - config.code_predictor_config, + cp_cfg, "head_dim", - config.code_predictor_config.hidden_size // config.code_predictor_config.num_attention_heads, + cp_cfg.hidden_size // cp_cfg.num_attention_heads, ) - self.hidden_size = config.code_predictor_config.hidden_size - - if self.num_heads % self.num_key_value_heads != 0: - raise ValueError("num_attention_heads must be divisible by num_key_value_heads") - - self.num_key_value_groups = self.num_heads // self.num_key_value_heads + self.hidden_size = cp_cfg.hidden_size + self.q_size = self.num_heads * self.head_dim + self.kv_size = self.num_kv_heads * self.head_dim + self.scaling = self.head_dim**-0.5 + self._use_gqa = self.num_kv_heads != self.num_heads self.qkv_proj = QKVParallelLinear( hidden_size=self.hidden_size, head_size=self.head_dim, total_num_heads=self.num_heads, - total_num_kv_heads=self.num_key_value_heads, + total_num_kv_heads=self.num_kv_heads, bias=False, quant_config=quant_config, prefix=f"{prefix}.qkv_proj", @@ -88,120 +81,55 @@ class Qwen3OmniCodePredictorAttention(nn.Module): ) self.rotary_emb = get_rope( self.head_dim, - max_position=config.code_predictor_config.max_position_embeddings, + max_position=cp_cfg.max_position_embeddings, rope_parameters=None, dual_chunk_attention_config=None, ) - - self.q_size = self.num_heads * self.head_dim - self.kv_size = self.num_key_value_heads * self.head_dim - - # Query/Key normalization - self.q_norm = RMSNorm(self.head_dim, eps=config.code_predictor_config.rms_norm_eps) - self.k_norm = RMSNorm(self.head_dim, eps=config.code_predictor_config.rms_norm_eps) - self.is_causal = True - self.config = config - - self.attention_backends = ["flash_attention_2", "xformers", "eager", "sdpa"] - cudagraph_mode = get_current_vllm_config().compilation_config.cudagraph_mode - if "flash_attention_2" in ALL_ATTENTION_FUNCTIONS and cudagraph_mode.has_full_cudagraphs(): - logger.warning( - f"CUDAGraphMode.{cudagraph_mode.name} is currently not supported " - f"with flash attention for Qwen3-Omni talker MTP." - f"removing flash attention from attention_backends" - ) - self.attention_backends.remove("flash_attention_2") + self.q_norm = RMSNorm(self.head_dim, eps=cp_cfg.rms_norm_eps) + self.k_norm = RMSNorm(self.head_dim, eps=cp_cfg.rms_norm_eps) def forward( self, hidden_states: torch.Tensor, - past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, - use_cache: bool = False, - position_ids: torch.LongTensor | None = None, + position_ids: torch.Tensor, ) -> torch.Tensor: bsz, seq_len, _ = hidden_states.shape - qkv, _ = self.qkv_proj(hidden_states) + # Flatten to 2-D so vLLM rotary_emb gets [num_tokens, size] + qkv, _ = self.qkv_proj(hidden_states.reshape(bsz * seq_len, -1)) q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1) - # Reshape for attention - q = q.reshape(bsz, seq_len, self.num_heads, self.head_dim) - k = k.reshape(bsz, seq_len, self.num_key_value_heads, self.head_dim) - v = v.reshape(bsz, seq_len, self.num_key_value_heads, self.head_dim) - - # Apply normalization - q = self.q_norm(q).contiguous() - k = self.k_norm(k).contiguous() - q = q.reshape(-1, self.q_size) - k = k.reshape(-1, self.kv_size) - - # Apply RoPE + # QK-norm -> RoPE (both 2-D) + q = self.q_norm(q.view(-1, self.num_heads, self.head_dim)).view(q.shape) + k = self.k_norm(k.view(-1, self.num_kv_heads, self.head_dim)).view(k.shape) q, k = self.rotary_emb(position_ids, q, k) - # Reshape for attention - q = q.reshape(bsz, seq_len, self.num_heads, self.head_dim) - k = k.reshape(bsz, seq_len, self.num_key_value_heads, self.head_dim) + # [B, heads, seq, head_dim] for SDPA + q = q.view(bsz, seq_len, self.num_heads, self.head_dim).transpose(1, 2) + k = k.view(bsz, seq_len, self.num_kv_heads, self.head_dim).transpose(1, 2) + v = v.view(bsz, seq_len, self.num_kv_heads, self.head_dim).transpose(1, 2) - v_heads = v.transpose(1, 2).contiguous() - q_heads = q.transpose(1, 2).contiguous() - k_heads = k.transpose(1, 2).contiguous() + attn_out = F.scaled_dot_product_attention( + q, + k, + v, + scale=self.scaling, + is_causal=True, + enable_gqa=self._use_gqa, + ) - if past_key_values is not None: - sin, cos = self.rotary_emb.get_cos_sin(seq_len) - # sin and cos are specific to RoPE models; cache_position needed for the static cache - cache_kwargs = {"sin": sin, "cos": cos, "cache_position": cache_position} - k_heads, v_heads = past_key_values.update(k_heads, v_heads, self.layer_idx, cache_kwargs) - - # Try attention backends in order of preference, with runtime error handling - # This handles cases where the backend is registered but not actually available - attn_output = None - last_error = None - - for backend_name in self.attention_backends: - if backend_name not in ALL_ATTENTION_FUNCTIONS: - continue - - try: - attention_interface = ALL_ATTENTION_FUNCTIONS[backend_name] - attn_output, _ = attention_interface( - self, - q_heads, - k_heads, - v_heads, - None, - dropout=0.0 if not self.training else getattr(self, "attention_dropout", 0.0), - scaling=self.head_dim**-0.5, - sliding_window=None, - use_cache=use_cache, - position_ids=position_ids[:seq_len].unsqueeze(0), - output_hidden_states=True, - output_attentions=False, - ) - break - except (ValueError, ImportError, RuntimeError, AttributeError) as e: - # Store error and try next backend - last_error = e - continue - - if attn_output is None: - raise RuntimeError( - f"All attention backends failed. Last error: {last_error}. " - "Please install flash-attn, or ensure PyTorch's scaled_dot_product_attention is available." - ) - attn_output = attn_output.reshape(*(hidden_states.shape[:-1]), -1).contiguous() - - attn_output, _ = self.o_proj(attn_output) - return attn_output + attn_out = attn_out.transpose(1, 2).reshape(bsz * seq_len, -1) + output, _ = self.o_proj(attn_out) + return output.view(bsz, seq_len, -1) -# ============================================================================ -# Code Predictor MLP Layer -# ============================================================================ +# =================================================================== +# MLP +# =================================================================== class Qwen3OmniCodePredictorMLP(nn.Module): - """Feed-forward network for code predictor with fused gate/up projection.""" + """SiLU-gated MLP for code predictor.""" def __init__( self, @@ -221,7 +149,6 @@ class Qwen3OmniCodePredictorMLP(nn.Module): prefix=f"{prefix}.gate_up_proj", disable_tp=True, ) - self.down_proj = RowParallelLinear( input_size=intermediate_size, output_size=hidden_size, @@ -238,35 +165,23 @@ class Qwen3OmniCodePredictorMLP(nn.Module): return down -# ============================================================================ -# MTP Layer (Multi-Token Prediction Layer) -# ============================================================================ +# =================================================================== +# Decoder Layer +# =================================================================== -class Qwen3OmniCodePredictorMTPLayer(nn.Module): - """MTP layer for speculative decoding - predicts next residual code layer.""" +class Qwen3OmniCodePredictorDecoderLayer(nn.Module): + """Transformer decoder layer (SDPA, no KV cache).""" def __init__( self, - config: PretrainedConfig, - prefix: str, - model_config: ModelConfig, - layer_idx: int, - cache_config: CacheConfig | None = None, + config, quant_config: QuantizationConfig | None = None, + prefix: str = "", ) -> None: super().__init__() - self.layer_idx = layer_idx - self.config = config - self.self_attn = Qwen3OmniCodePredictorAttention( config, - layer_idx, - vllm_config=type( - "VllmConfig", - (), - {"cache_config": cache_config, "quant_config": quant_config, "model_config": model_config}, - )(), quant_config=quant_config, prefix=f"{prefix}.self_attn", ) @@ -275,260 +190,243 @@ class Qwen3OmniCodePredictorMTPLayer(nn.Module): quant_config=quant_config, prefix=f"{prefix}.mlp", ) - self.input_layernorm = RMSNorm( - config.code_predictor_config.hidden_size, eps=config.code_predictor_config.rms_norm_eps - ) - self.post_attention_layernorm = RMSNorm( - config.code_predictor_config.hidden_size, eps=config.code_predictor_config.rms_norm_eps - ) + cp_cfg = config.code_predictor_config + self.input_layernorm = RMSNorm(cp_cfg.hidden_size, eps=cp_cfg.rms_norm_eps) + self.post_attention_layernorm = RMSNorm(cp_cfg.hidden_size, eps=cp_cfg.rms_norm_eps) - def mtp_block( + def forward( self, hidden_states: torch.Tensor, - past_key_values: Cache | None = None, - cache_position: torch.LongTensor | None = None, - use_cache: bool = False, - position_ids: torch.LongTensor | None = None, + position_ids: torch.Tensor, ) -> torch.Tensor: - # Self-attention with residual residual = hidden_states hidden_states = self.input_layernorm(hidden_states) - hidden_states = self.self_attn(hidden_states, past_key_values, cache_position, use_cache, position_ids) + hidden_states = self.self_attn(hidden_states, position_ids) hidden_states = residual + hidden_states - # MLP with residual residual = hidden_states hidden_states = self.post_attention_layernorm(hidden_states) hidden_states = self.mlp(hidden_states) hidden_states = residual + hidden_states - return hidden_states -class Qwen3OmniCodePredictorBaseModel(nn.Module): - """ - Base model for code predictor - matches HF Qwen3OmniMoeTalkerCodePredictorModel structure. +# =================================================================== +# Base Transformer Model (re-prefill, no KV cache) +# =================================================================== - This is a simple transformer that processes inputs_embeds and outputs hidden states. + +class Qwen3OmniCodePredictorBaseModel(nn.Module): + """Inner transformer for code predictor. + + Signature: ``forward(inputs_embeds, position_ids) -> hidden_states`` + -- plain Tensor in, plain Tensor out (no namedtuple). """ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): super().__init__() config = vllm_config.model_config.hf_config.code_predictor_config - self.config = config - self.vocab_size = config.vocab_size - self.num_code_groups = config.num_code_groups - # Codec embeddings (for layers 1-num_code_groups-1) self.codec_embedding = nn.ModuleList( - [ - VocabParallelEmbedding( - config.vocab_size, - config.hidden_size, - ) - for _ in range(config.num_code_groups - 1) - ] + [VocabParallelEmbedding(config.vocab_size, config.hidden_size) for _ in range(config.num_code_groups - 1)] ) - # Decoder layers self.layers = nn.ModuleList( [ - Qwen3OmniCodePredictorMTPLayer( + Qwen3OmniCodePredictorDecoderLayer( vllm_config.model_config.hf_config, - f"{prefix}.layers.{idx}", - model_config=vllm_config.model_config, - layer_idx=idx, - cache_config=vllm_config.cache_config, quant_config=vllm_config.quant_config, + prefix=f"{prefix}.layers.{idx}", ) for idx in range(config.num_hidden_layers) ] ) - self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps) def forward( self, inputs_embeds: torch.Tensor, - attention_mask: torch.Tensor | None = None, - position_ids: torch.LongTensor | None = None, - past_key_values: Any | None = None, - use_cache: bool | None = False, - cache_position: torch.LongTensor | None = None, - **kwargs: Any, - ) -> Any: - """ - Forward pass matching HF structure. - - Args: - inputs_embeds: [batch, seq_len, hidden_size] - position_ids: Optional position IDs tensor - past_key_values: Optional cached key-value pairs - use_cache: Whether to use cache - cache_position: Optional cache position tensor - **kwargs: Additional keyword arguments - - Returns: - Named tuple with .last_hidden_state and .past_key_values attributes - """ - batch_size, seq_len, _ = inputs_embeds.shape - # Forward through decoder layers + position_ids: torch.Tensor, + ) -> torch.Tensor: hidden_states = inputs_embeds - for layer in self.layers: - hidden_states = layer.mtp_block(hidden_states, past_key_values, cache_position, use_cache, position_ids) - - # Final norm + hidden_states = layer(hidden_states, position_ids) hidden_states = self.norm(hidden_states) - - # Return in HF-compatible format - Output = namedtuple("Output", ["last_hidden_state", "past_key_values"]) - return Output(last_hidden_state=hidden_states, past_key_values=None) # [batch, num_code_groups-1, hidden_size] - - def get_input_embeddings(self): - """Return codec embeddings for HF compatibility.""" - return self.codec_embedding + return hidden_states -def code_predictor_sample( - logits: torch.Tensor, - layer_name: str, -) -> torch.Tensor: - forward_context = get_forward_context() - self = forward_context.no_compile_layers[layer_name] - logits = self.logits_processors(None, logits[:, -1]) - probs = F.softmax(logits, dim=-1) - code = torch.multinomial(probs.squeeze(1), num_samples=1) # [batch, 1] - return code +# =================================================================== +# Code Predictor Wrapper (optimized re-prefill, persistent buffers) +# =================================================================== -def code_predictor_sample_fake( - logits: torch.Tensor, - layer_name: str, -) -> torch.Tensor: - return torch.empty((logits.shape[0], 1), dtype=torch.int64, device=logits.device) - - -direct_register_custom_op( - op_name="qwen3_omni_code_predictor_sample", - op_func=code_predictor_sample, - fake_impl=code_predictor_sample_fake, -) - - -@support_torch_compile class Qwen3OmniMoeTalkerCodePredictor(nn.Module): - """ - Code predictor wrapper matching HF structure. + """Optimized code predictor -- re-prefill approach, no KV cache. - Structure: - - self.model: Qwen3OmniCodePredictorBaseModel (transformer) - - self.lm_head: ModuleList of output heads + Each AR step forwards the full growing sequence (len 2 -> num_code_groups+1) + through the transformer. The extra O(T^2) FLOPs are negligible for + short sequences, and this avoids all KV-cache management overhead. + + Optimizations: + 1. Pre-allocated embedding buffer -- no torch.cat per step. + 2. Pre-allocated position_ids -- no torch.arange per step. + 3. Inline top-k sampling -- no LogitsProcessorList / custom op. + 4. Cached module references -- bypass ModuleList indexing. + 5. torch.compile on inner transformer. """ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): super().__init__() - talker_code_predictor_config = vllm_config.model_config.hf_config + config = vllm_config.model_config.hf_config + self.config = config self.quant_config = vllm_config.quant_config self.prefix = prefix - self.config = talker_code_predictor_config - self.vocab_size = self.config.code_predictor_config.vocab_size - self.num_code_groups = self.config.code_predictor_config.num_code_groups + self.num_code_groups = config.code_predictor_config.num_code_groups + self._hidden_size = config.code_predictor_config.hidden_size - # Base transformer model (matches HF structure) - self.model = Qwen3OmniCodePredictorBaseModel(vllm_config=vllm_config, prefix=prefix) + self.model = Qwen3OmniCodePredictorBaseModel( + vllm_config=vllm_config, + prefix=prefix, + ) - # Output heads for each residual layer (1-num_layers-1) + # One lm_head per residual layer (layers 1 .. G-1) self.lm_head = nn.ModuleList( [ nn.Linear( - self.config.code_predictor_config.hidden_size, - self.config.code_predictor_config.vocab_size, + config.code_predictor_config.hidden_size, + config.code_predictor_config.vocab_size, bias=False, ) for _ in range(self.num_code_groups - 1) ] ) - self.logits_processors = LogitsProcessorList( - [ - TopKLogitsWarper(top_k=50), - TopPLogitsWarper(top_p=0.8), - ] + + # Sampling hyperparams (inlined) + self._top_k = 50 + + # Persistent buffers (lazily initialised on first forward) + self._proj_buf: torch.Tensor | None = None + self._pos_ids: torch.Tensor | None = None + + # Cached plain-list refs (set once) + self._lm_heads: list | None = None + self._codec_embeds: list | None = None + + # Model forward (optionally compiled) + self._model_fwd: object | None = None + + # ------------------------------------------------------------------ + # Lazy-init helpers + # ------------------------------------------------------------------ + + def _ensure_buffers(self, bsz: int, device: torch.device, dtype: torch.dtype) -> None: + max_seq = self.num_code_groups + 1 + if ( + self._proj_buf is not None + and self._proj_buf.shape[0] >= bsz + and self._proj_buf.device == device + and self._proj_buf.dtype == dtype + ): + return + self._proj_buf = torch.zeros(bsz, max_seq, self._hidden_size, dtype=dtype, device=device) + self._pos_ids = torch.arange(max_seq, dtype=torch.long, device=device) + + def _ensure_cached_refs(self) -> None: + if self._lm_heads is not None: + return + self._lm_heads = list(self.lm_head) + self._codec_embeds = list(self.model.codec_embedding) + + def _ensure_model_fwd(self) -> None: + if self._model_fwd is not None: + return + self._model_fwd = torch.compile( + self.model.forward, + mode="default", + dynamic=True, ) + logger.info("code_predictor: torch.compile enabled") - compilation_config = get_current_vllm_config().compilation_config - if prefix in compilation_config.static_forward_context: - raise ValueError(f"Duplicate layer name: {prefix}") - compilation_config.static_forward_context[prefix] = self - self.layer_name = prefix + # ------------------------------------------------------------------ + # Forward -- re-prefill + persistent buffers + inline sampling + # ------------------------------------------------------------------ + @torch.inference_mode() def forward( self, layer0_code: torch.Tensor, layer0_embed: torch.Tensor, last_talker_hidden: torch.Tensor, ) -> tuple[torch.Tensor, torch.Tensor]: - """ - Forward pass for code predictor. + """Predict residual codebooks 1..G-1 autoregressively via re-prefill. Args: - layer0_code: - Code index for code-group (layer) 0. - Shape: [batch_size, 1], dtype typically int64. - - last_talker_hidden: - - Shape: [batch_size, hidden_size]. + layer0_code: [bsz, 1] int64 + layer0_embed: [bsz, 1, hidden_size] + last_talker_hidden: [bsz, 1, hidden_size] Returns: - pos_all_layers: - Predicted codes for all code groups, including `layer0_code`. - Shape: [batch_size, num_code_groups, 1]. - - current_input: - The final input embedding sequence after appending embeddings of all - predicted codes (one token per predicted layer). - Shape: [batch_size, num_code_groups + 2, hidden_size]. + all_codes: [bsz, num_code_groups, 1] + proj_buf: [bsz, num_code_groups + 1, hidden_size] + pos 0 = last_talker_hidden (NOT a codec embed) + pos 1 = layer0_embed + pos 2.. = codec_embedding[i](predicted_code_i) """ - pos_codes = [layer0_code] # Start with layer 0: [batch, 1] - try: - current_input = torch.cat([last_talker_hidden, layer0_embed], dim=1) # [batch, 2, hidden_size] - except Exception as e: - print(f"Error in current_input: {e}") - print(f"last_talker_hidden shape: {last_talker_hidden.shape}") - print(f"prev_embed shape: {layer0_embed.shape}") - raise e - batch_size = current_input.shape[0] + bsz = int(layer0_code.shape[0]) + device = layer0_code.device + dtype = last_talker_hidden.dtype + num_groups = self.num_code_groups + top_k = self._top_k - # Predict all residual layers (layers 1 to num_code_groups-1) autoregressively - for layer_idx in range(self.num_code_groups - 1): - seq_len = layer_idx + 2 - # Compute position_ids dynamically to avoid torch.compile specializing batch_size - position_ids = torch.arange(seq_len, device=current_input.device, dtype=torch.int64).repeat(batch_size) - # Forward through code_predictor model - outputs = self.model( - inputs_embeds=current_input, - attention_mask=None, - position_ids=position_ids, - past_key_values=None, - use_cache=False, - cache_position=None, - ) - hidden_state = outputs.last_hidden_state # [batch, 2, hidden_size] + # Lazy init + self._ensure_buffers(bsz, device, dtype) + self._ensure_model_fwd() + self._ensure_cached_refs() - # Use the corresponding lm_head for this layer - logits = self.lm_head[layer_idx](hidden_state[:, -1:, :]) - code = torch.ops.vllm.qwen3_omni_code_predictor_sample(logits, self.layer_name) - pos_codes.append(code) - # Update prev_embed for next layer (if not last layer) - # layer_idx=0 predicts layer 1, embed with codec_embedding[1] - new_embed = self.model.codec_embedding[layer_idx](code) # [batch, 1, hidden_size] - current_input = torch.cat([current_input, new_embed], dim=1) # [batch, 3~n, hidden_size] - pos_all_layers = torch.stack(pos_codes, dim=1) # [batch, num_code_groups, 1] - return pos_all_layers, current_input + proj_buf = self._proj_buf + pos_ids = self._pos_ids + model_fwd = self._model_fwd + lm_heads = self._lm_heads + codec_embeds = self._codec_embeds + + # Output codes + all_codes = torch.empty(bsz, num_groups, 1, dtype=torch.int64, device=device) + all_codes[:, 0] = layer0_code + + # Fill buffer positions 0 & 1 + proj_buf[:bsz, 0:1, :] = last_talker_hidden + proj_buf[:bsz, 1:2, :] = layer0_embed + + # Autoregressive loop: predict layers 1..G-1 + for step in range(1, num_groups): + seq_len = step + 1 + projected = proj_buf[:bsz, :seq_len, :] + step_pos_ids = pos_ids[:seq_len] if bsz == 1 else pos_ids[:seq_len].repeat(bsz) + + hidden_out = model_fwd(projected, step_pos_ids) + + # Inline top-k sampling + logits = lm_heads[step - 1](hidden_out[:, -1, :]) + if top_k > 0: + topk_vals, _ = logits.topk(top_k, dim=-1) + logits = logits.masked_fill(logits < topk_vals[:, -1:], float("-inf")) + probs = F.softmax(logits, dim=-1) + code = torch.multinomial(probs, num_samples=1) # [bsz, 1] + + all_codes[:, step] = code + + # Embed predicted code -> next buffer position + new_embed = codec_embeds[step - 1](code) + proj_buf[:bsz, step + 1 : step + 2, :] = new_embed + + return all_codes, proj_buf[:bsz] + + # ------------------------------------------------------------------ + # Weight loading + # ------------------------------------------------------------------ def load_weights(self, weights: list[tuple[str, torch.Tensor]]) -> set[str]: """Load weights with mapping for fused QKV and gate_up projections. diff --git a/vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_talker.py b/vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_talker.py index 3362ae17f..9f5c20cf7 100644 --- a/vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_talker.py +++ b/vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_talker.py @@ -129,111 +129,70 @@ class Qwen3OmniMoeTalkerForConditionalGeneration( self.code_predictor = Qwen3OmniMoeTalkerCodePredictor( vllm_config=vllm_config, prefix=maybe_prefix(prefix, "code_predictor") ) - max_batch_size = max( - vllm_config.scheduler_config.max_num_seqs, vllm_config.compilation_config.max_cudagraph_capture_size - ) - self.layer0_embed_buffer = torch.zeros( - (max_batch_size, 1, self.config.text_config.hidden_size), - dtype=vllm_config.model_config.dtype, - ) def code_predictor_forward( self, input_ids: torch.Tensor, inputs_embeds: torch.Tensor | None = None, *, - temperature: float = 1.0, - top_k: int = 50, # Match transformers default - top_p: float = 0.8, # Match transformers default - generation_steps: int | None = None, last_talker_hidden: torch.Tensor | None = None, **_: object, ) -> tuple[torch.Tensor, torch.Tensor]: - """ - Generate full RVQ codec codes for the provided sequence. + """Generate full RVQ codes + summed embeddings (single-loop, no KV cache). - The code predictor consumes the layer-0 codec codes produced by the talker - alongside the talker's hidden states, and autoregressively predicts the remaining - residual layers (to num_codec_groups). + The code predictor uses re-prefill: each AR step re-forwards the full + (short) sequence through the transformer. The returned ``proj_buf`` + already contains all codec embeddings at positions 1..G, + so summed_embeddings = proj_buf[:, 1:, :].sum(dim=1) — no second + loop or re-embedding needed. Returns: - tuple containing: - - residual_codes: A tensor of shape [batch, num_code_groups, seq_len] containing - the complete set of codec codes - - summed_embeddings: A tensor of shape [batch, seq_len, hidden_size] - Sum of all layer embeddings at each position (like Transformers) + result_codes: [batch, num_code_groups, seq_len] + summed_embeddings: [batch, seq_len, hidden_size] """ if input_ids is None: - raise ValueError("`input_ids` containing layer-0 codec codes must be provided.") + raise ValueError("`input_ids` (layer-0 codes) must be provided.") if inputs_embeds is None: - raise ValueError("`inputs_embeds` containing talker hidden states must be provided.") + raise ValueError("`inputs_embeds` (talker hidden states) must be provided.") if inputs_embeds.ndim == 2: inputs_embeds = inputs_embeds.unsqueeze(0) if input_ids.ndim == 1: input_ids = input_ids.unsqueeze(0) - # Ensure the tensors are contiguous for the autoregressive sampling loop - inputs_embeds = inputs_embeds.contiguous() - input_ids = input_ids.contiguous() - - # Generate full codec codes using MTP - # This will be the parallel prediction implementation batch_size, seq_len = input_ids.shape + device = input_ids.device + embed_fn = self.language_model.model.codec_embedding + hidden_size = self.config.code_predictor_config.hidden_size - # For now, use sequential generation (TODO: implement parallel) - # Result will be [batch, num_code_groups, seq_len] - # - all_codes_per_position will collect [batch, num_code_groups, 1] for each position - all_codes_per_position = [] - middle_hidden_states = [] # Collect hidden states for each position - - # Generate residual layers for each position - for pos in range(seq_len): - layer0_code = input_ids[:, pos : pos + 1] # [batch, 1] - - # Initial input: [last_talker_hidden, layer0_embed] - layer0_embed = self.embed_input_ids(layer0_code) - self.layer0_embed_buffer[:batch_size].copy_(layer0_embed) - pos_all_layers, current_input = self.code_predictor( - layer0_code, self.layer0_embed_buffer[:batch_size], last_talker_hidden - ) - - # Stack all layers for this position: [batch, num_code_groups, 1] - all_codes_per_position.append(pos_all_layers) - middle_hidden_states.append(current_input[:, 2:-1, :]) - - # Concatenate across positions: [batch, num_code_groups, seq_len] - result_codes = torch.cat(all_codes_per_position, dim=2) - - # Build summed embeddings for each position (like Transformers) - # This combines layer-0 embed, mid layers hidden states, and last layer embed - all_summed_embeddings = [] + result_codes = torch.empty( + batch_size, + self.num_code_groups, + seq_len, + dtype=torch.int64, + device=device, + ) + summed_embeddings = torch.empty( + batch_size, + seq_len, + hidden_size, + dtype=inputs_embeds.dtype, + device=device, + ) for pos in range(seq_len): - # Layer 0 embedding - layer0_code = result_codes[:, 0, pos : pos + 1] # [batch, 1] - layer0_embed = self.embed_input_ids(layer0_code) # [batch, 1, hidden_size] + layer0_code = input_ids[:, pos : pos + 1] + layer0_embed = embed_fn(layer0_code) - # mid layers hidden states (from CodePredictor) - mid_residual_hiddens = middle_hidden_states[pos] # [batch, num_code_groups-2, hidden_size] - mid_list = list(mid_residual_hiddens.split(1, dim=1)) - - # last layer embedding - last_layer_code = result_codes[:, -1, pos : pos + 1] # [batch, 1] - last_residual_hidden = self.code_predictor.model.codec_embedding[-1](last_layer_code) - - # Concatenate all layers: [batch, num_code_groups, hidden_size] - pos_codec_hiddens = torch.cat( - [layer0_embed] + mid_list + [last_residual_hidden], - dim=1, + pos_all_layers, proj_buf = self.code_predictor( + layer0_code, + layer0_embed, + last_talker_hidden, ) - # Sum across layers: [batch, 1, hidden_size] (like Transformers) - pos_summed = pos_codec_hiddens.sum(dim=1, keepdim=True) - all_summed_embeddings.append(pos_summed) - - # Concatenate across positions: [batch, seq_len, hidden_size] - summed_embeddings = torch.cat(all_summed_embeddings, dim=1).squeeze(1) + result_codes[:, :, pos : pos + 1] = pos_all_layers + # proj_buf layout: [0]=talker_hidden, [1..G]=codec embeds + summed_embeddings[:, pos, :] = proj_buf[:, 1:, :].sum(dim=1) return result_codes, summed_embeddings diff --git a/vllm_omni/model_executor/models/qwen3_tts/cuda_graph_decoder_wrapper.py b/vllm_omni/model_executor/models/qwen3_tts/cuda_graph_decoder_wrapper.py index de278edf7..5ff259193 100644 --- a/vllm_omni/model_executor/models/qwen3_tts/cuda_graph_decoder_wrapper.py +++ b/vllm_omni/model_executor/models/qwen3_tts/cuda_graph_decoder_wrapper.py @@ -29,7 +29,7 @@ class CUDAGraphDecoderWrapper: output = wrapper.decode(codes) # Automatically uses CUDA graph if possible """ - DEFAULT_CAPTURE_SIZES = [25, 50, 100, 150, 200, 250, 300] + DEFAULT_CAPTURE_SIZES = [2, 4, 8, 16, 25, 32, 50, 100, 150, 200, 250, 300] def __init__( self, diff --git a/vllm_omni/model_executor/stage_configs/qwen3_tts.yaml b/vllm_omni/model_executor/stage_configs/qwen3_tts.yaml index 19502add8..095aafe08 100644 --- a/vllm_omni/model_executor/stage_configs/qwen3_tts.yaml +++ b/vllm_omni/model_executor/stage_configs/qwen3_tts.yaml @@ -96,10 +96,6 @@ runtime: # Align with Omni: small chunks with sufficient context overlap. codec_chunk_frames: 25 codec_left_context_frames: 25 - # First chunk size for reduced TTFA (0 = disabled). - # When > 0, emits small chunks every N frames during the initial phase, - # then switches to codec_chunk_frames cadence. - initial_codec_chunk_frames: 0 edges: - from: 0 diff --git a/vllm_omni/model_executor/stage_configs/qwen3_tts_batch.yaml b/vllm_omni/model_executor/stage_configs/qwen3_tts_batch.yaml index 43df70fc2..d73739109 100644 --- a/vllm_omni/model_executor/stage_configs/qwen3_tts_batch.yaml +++ b/vllm_omni/model_executor/stage_configs/qwen3_tts_batch.yaml @@ -98,10 +98,6 @@ runtime: # Align with Omni: small chunks with sufficient context overlap. codec_chunk_frames: 25 codec_left_context_frames: 25 - # First chunk size for reduced TTFA (0 = disabled). - # When > 0, emits small chunks every N frames during the initial phase, - # then switches to codec_chunk_frames cadence. - initial_codec_chunk_frames: 0 edges: - from: 0 diff --git a/vllm_omni/model_executor/stage_input_processors/chunk_size_utils.py b/vllm_omni/model_executor/stage_input_processors/chunk_size_utils.py new file mode 100644 index 000000000..7bdec0bb3 --- /dev/null +++ b/vllm_omni/model_executor/stage_input_processors/chunk_size_utils.py @@ -0,0 +1,33 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + + +def max_ic_for_chunk_size(chunk_size: int) -> int: + """Largest power of 2 strictly less than chunk_size.""" + if chunk_size <= 2: + return 1 + return 1 << ((chunk_size - 1).bit_length() - 1) + + +def compute_dynamic_initial_chunk_size( + active_requests: int, + max_batch_size: int, + max_ic: int, +) -> int: + """Select IC from power-of-2 steps [2, 4, ..., max_ic] based on load factor. + + - Low load: small IC (faster TTFA). + - High load: large IC (amortise decode cost). + """ + steps: list[int] = [] + v = 2 + while v <= max_ic: + steps.append(v) + v <<= 1 + if not steps: + return max(1, max_ic) + if max_batch_size <= 0: + return steps[0] + load_factor = min(active_requests / max_batch_size, 1.0) + idx = int(round(load_factor * (len(steps) - 1))) + return steps[idx] diff --git a/vllm_omni/model_executor/stage_input_processors/qwen3_tts.py b/vllm_omni/model_executor/stage_input_processors/qwen3_tts.py index eb9753262..435e45f03 100644 --- a/vllm_omni/model_executor/stage_input_processors/qwen3_tts.py +++ b/vllm_omni/model_executor/stage_input_processors/qwen3_tts.py @@ -5,6 +5,11 @@ from typing import Any import torch from vllm.logger import init_logger +from vllm_omni.model_executor.stage_input_processors.chunk_size_utils import ( + compute_dynamic_initial_chunk_size, + max_ic_for_chunk_size, +) + logger = init_logger(__name__) @@ -14,7 +19,7 @@ def talker2code2wav( prompt: Any = None, requires_multimodal_data: bool = False, ) -> list[Any]: - """Non-async processor: wait for talker to finish, then pass all codes to code2wav at once.""" + """Non-async: collect all talker codes, then pass to code2wav at once.""" from vllm_omni.inputs.data import OmniTokensPrompt from vllm_omni.model_executor.stage_input_processors.qwen3_omni import _validate_stage_inputs @@ -94,9 +99,12 @@ def talker2code2wav_async_chunk( cfg = raw_cfg.get("extra", raw_cfg) if isinstance(raw_cfg, dict) else {} chunk_size = int(cfg.get("codec_chunk_frames", 25)) left_context_size_config = int(cfg.get("codec_left_context_frames", 25)) - initial_chunk_size = int(cfg.get("initial_codec_chunk_frames", 0)) - # Per-request override (takes priority over stage config) + + # Per-request override takes priority over dynamic IC. + per_request_override = False + initial_chunk_size = 0 additional_information = getattr(request, "additional_information", None) + if ( additional_information is not None and hasattr(additional_information, "entries") @@ -105,12 +113,32 @@ def talker2code2wav_async_chunk( entry = additional_information.entries["initial_codec_chunk_frames"] if entry.list_data is not None and len(entry.list_data) == 1: initial_chunk_size = int(entry.list_data[0]) + per_request_override = True + + # Dynamic IC: always derived from chunk_size, recomputed every call to adapt to load. + if not per_request_override: + max_ic = max_ic_for_chunk_size(chunk_size) + active = sum(1 for v in transfer_manager.code_prompt_token_ids.values() if len(v) > 0) + capacity = getattr(transfer_manager, "scheduler_max_num_seqs", 1) + initial_chunk_size = compute_dynamic_initial_chunk_size(active, capacity, max_ic) + logger.debug( + "Dynamic IC: active=%d, capacity=%d, max_ic=%d, ic=%d, cs=%d, req=%s, keys=%d", + active, + capacity, + max_ic, + initial_chunk_size, + chunk_size, + request_id, + len(transfer_manager.code_prompt_token_ids), + ) + if chunk_size <= 0 or left_context_size_config < 0 or initial_chunk_size < 0: raise ValueError( f"Invalid codec chunk config: codec_chunk_frames={chunk_size}, " f"codec_left_context_frames={left_context_size_config}, " f"initial_codec_chunk_frames={initial_chunk_size}" ) + if initial_chunk_size > chunk_size: logger.warning( "initial_codec_chunk_frames=%d > codec_chunk_frames=%d, clamping to codec_chunk_frames.", @@ -120,8 +148,6 @@ def talker2code2wav_async_chunk( initial_chunk_size = chunk_size length = len(transfer_manager.code_prompt_token_ids[request_id]) - # Avoid emitting empty chunks during normal streaming. If the request is - # finished and nothing was produced, emit an EOF marker. if length <= 0: if finished: return { @@ -130,34 +156,32 @@ def talker2code2wav_async_chunk( } return None - in_initial_phase = initial_chunk_size > 0 and length <= chunk_size + in_initial_phase = initial_chunk_size > 0 and initial_chunk_size < chunk_size and length < chunk_size if in_initial_phase: - # Initial-chunk phase: emit every initial_chunk_size frames with full accumulated context. - already_sent = transfer_manager.put_req_chunk[request_id] * initial_chunk_size - pending = length - already_sent - if pending <= 0: + # IC phase: emit every initial_chunk_size frames with growing left context. + if not finished and length % initial_chunk_size != 0: return None - if pending < initial_chunk_size and not finished: - return None - context_length = min(pending, initial_chunk_size) - end_index = length - left_context_size = max(0, length - context_length) - window_frames = transfer_manager.code_prompt_token_ids[request_id][:length] + context_length = ( + length % initial_chunk_size if (finished and length % initial_chunk_size != 0) else initial_chunk_size + ) else: - # Normal phase: standard chunk_size cadence with left_context sliding window. - # Offset by initial_coverage so normal starts from where the initial-chunk phase left off. - initial_coverage = (chunk_size // initial_chunk_size) * initial_chunk_size if initial_chunk_size > 0 else 0 + # Normal phase: offset so the first normal emit picks up after IC phase. + # IC is stateless (may change with load); any mismatch is absorbed by left_context. + initial_coverage = ( + ((chunk_size - 1) // initial_chunk_size) * initial_chunk_size if 0 < initial_chunk_size < chunk_size else 0 + ) adjusted = length - initial_coverage - chunk_length = adjusted % chunk_size - if chunk_length != 0 and not finished: + if not finished and adjusted % chunk_size != 0: return None + chunk_length = adjusted % chunk_size context_length = chunk_length if chunk_length != 0 else chunk_size - end_index = min(length, left_context_size_config + context_length) - left_context_size = max(0, int(end_index - context_length)) - window_frames = transfer_manager.code_prompt_token_ids[request_id][-end_index:] - # Match offline Base ICL decoding by prepending ref_code only once to the first decoder window. + end_index = min(length, left_context_size_config + context_length) + left_context_size = max(0, end_index - context_length) + window_frames = transfer_manager.code_prompt_token_ids[request_id][-end_index:] + + # Prepend ref_code to the first decoder window only. if transfer_manager.put_req_chunk[request_id] == 0: ref_code = request_payload.pop(request_id, None) if isinstance(ref_code, torch.Tensor) and ref_code.numel() > 0: @@ -165,7 +189,6 @@ def talker2code2wav_async_chunk( window_frames = ref_frames + window_frames left_context_size += len(ref_frames) - # Pack context + chunk into codebook-major flat codes for adapter. code_predictor_codes = torch.tensor(window_frames).transpose(0, 1).reshape(-1).tolist() return { diff --git a/vllm_omni/platforms/npu/stage_configs/qwen3_tts.yaml b/vllm_omni/platforms/npu/stage_configs/qwen3_tts.yaml index 5ad0cc90e..60659a976 100644 --- a/vllm_omni/platforms/npu/stage_configs/qwen3_tts.yaml +++ b/vllm_omni/platforms/npu/stage_configs/qwen3_tts.yaml @@ -96,10 +96,6 @@ runtime: # Align with Omni: small chunks with sufficient context overlap. codec_chunk_frames: 25 codec_left_context_frames: 25 - # First chunk size for reduced TTFA (0 = disabled). - # When > 0, emits small chunks every N frames during the initial phase, - # then switches to codec_chunk_frames cadence. - initial_codec_chunk_frames: 0 edges: - from: 0 diff --git a/vllm_omni/worker/gpu_model_runner.py b/vllm_omni/worker/gpu_model_runner.py index dc353d8d4..e17d52bdd 100644 --- a/vllm_omni/worker/gpu_model_runner.py +++ b/vllm_omni/worker/gpu_model_runner.py @@ -1339,10 +1339,17 @@ class OmniGPUModelRunner(GPUModelRunner): req_state = self.requests.get(req_id) if req_state is None: return + # Check if the model declares keys that should stay on GPU + gpu_keys: set[str] = set() + if hasattr(self, "model") and hasattr(self.model, "gpu_resident_buffer_keys"): + gpu_keys = self.model.gpu_resident_buffer_keys existing = self.model_intermediate_buffer.setdefault(req_id, {}) for k, v in upd.items(): if isinstance(v, torch.Tensor): - existing[k] = v.detach().to("cpu").contiguous() + if k in gpu_keys: + existing[k] = v.detach().clone() + else: + existing[k] = v.detach().to("cpu").contiguous() elif isinstance(v, list): existing[k] = [ (item.detach().to("cpu").contiguous() if isinstance(item, torch.Tensor) else item) for item in v