From ce1ae26084ec2d866c7d2454115d63ea1dca51a1 Mon Sep 17 00:00:00 2001 From: Alicia <115451386+congw729@users.noreply.github.com> Date: Tue, 3 Feb 2026 18:11:50 +0800 Subject: [PATCH] [Doc]Fix - Align with repo. (#1176) Signed-off-by: Alicia <115451386+congw729@users.noreply.github.com> --- docs/.nav.yml | 7 + docs/api/README.md | 31 ++- .../examples/offline_inference/bagel.md | 10 +- .../offline_inference/image_to_image.md | 7 +- .../offline_inference/image_to_video.md | 7 +- .../offline_inference/lora_inference.md | 5 +- .../examples/offline_inference/qwen3_tts.md | 9 + .../offline_inference/text_to_audio.md | 47 +++++ .../offline_inference/text_to_image.md | 9 +- .../offline_inference/text_to_video.md | 18 +- .../examples/online_serving/bagel.md | 16 +- .../examples/online_serving/lora_inference.md | 9 +- .../examples/online_serving/qwen3_tts.md | 189 ++++++++++++++++++ 13 files changed, 335 insertions(+), 29 deletions(-) create mode 100644 docs/user_guide/examples/offline_inference/text_to_audio.md create mode 100644 docs/user_guide/examples/online_serving/qwen3_tts.md diff --git a/docs/.nav.yml b/docs/.nav.yml index 428fd16b5..41754c9c7 100644 --- a/docs/.nav.yml +++ b/docs/.nav.yml @@ -11,17 +11,23 @@ nav: - Examples: - examples/README.md - Offline Inference: + - BAGEL-7B-MoT: user_guide/examples/offline_inference/bagel.md - Image-To-Image: user_guide/examples/offline_inference/image_to_image.md - Image-To-Video: user_guide/examples/offline_inference/image_to_video.md + - LoRA Inference Examples: user_guide/examples/offline_inference/lora_inference.md - Qwen2.5-Omni: user_guide/examples/offline_inference/qwen2_5_omni.md - Qwen3-Omni: user_guide/examples/offline_inference/qwen3_omni.md - Qwen3-TTS Offline Inference: user_guide/examples/offline_inference/qwen3_tts.md + - Text-To-Audio: user_guide/examples/offline_inference/text_to_audio.md - Text-To-Image: user_guide/examples/offline_inference/text_to_image.md - Text-To-Video: user_guide/examples/offline_inference/text_to_video.md - Online Serving: + - BAGEL-7B-MoT: user_guide/examples/online_serving/bagel.md - Image-To-Image: user_guide/examples/online_serving/image_to_image.md + - Online LoRA Inference (Diffusion): user_guide/examples/online_serving/lora_inference.md - Qwen2.5-Omni: user_guide/examples/online_serving/qwen2_5_omni.md - Qwen3-Omni: user_guide/examples/online_serving/qwen3_omni.md + - Qwen3-TTS Online Serving: user_guide/examples/online_serving/qwen3_tts.md - Text-To-Image: user_guide/examples/online_serving/text_to_image.md - General: - usage/* @@ -54,6 +60,7 @@ nav: - Feature Design: - design/feature/disaggregated_inference.md - design/feature/ray_based_execution.md + - design/feature/omni_connectors/ - Module Design: - design/module/ar_module.md - design/module/dit_module.md diff --git a/docs/api/README.md b/docs/api/README.md index 3a110f491..6219aee40 100644 --- a/docs/api/README.md +++ b/docs/api/README.md @@ -10,6 +10,9 @@ Main entry points for vLLM-Omni inference and serving. - [vllm_omni.entrypoints.chat_utils.OmniAsyncMultiModalContentParser][] - [vllm_omni.entrypoints.chat_utils.OmniAsyncMultiModalItemTracker][] - [vllm_omni.entrypoints.chat_utils.parse_chat_messages_futures][] +- [vllm_omni.entrypoints.cli.benchmark.base.OmniBenchmarkSubcommandBase][] +- [vllm_omni.entrypoints.cli.benchmark.main.OmniBenchmarkSubcommand][] +- [vllm_omni.entrypoints.cli.benchmark.serve.OmniBenchmarkServingSubcommand][] - [vllm_omni.entrypoints.cli.serve.OmniServeCommand][] - [vllm_omni.entrypoints.client_request_state.ClientRequestState][] - [vllm_omni.entrypoints.log_utils.OrchestratorMetrics][] @@ -26,7 +29,9 @@ Main entry points for vLLM-Omni inference and serving. Input data structures for multi-modal inputs. +- [vllm_omni.inputs.data.OmniDiffusionSamplingParams][] - [vllm_omni.inputs.data.OmniEmbedsPrompt][] +- [vllm_omni.inputs.data.OmniTextPrompt][] - [vllm_omni.inputs.data.OmniTokenInputs][] - [vllm_omni.inputs.data.OmniTokensPrompt][] - [vllm_omni.inputs.parse.parse_singleton_prompt_omni][] @@ -58,6 +63,7 @@ Core scheduling and caching components. - [vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler][] - [vllm_omni.core.sched.output.OmniCachedRequestData][] - [vllm_omni.core.sched.output.OmniNewRequestData][] +- [vllm_omni.core.sched.output.OmniSchedulerOutput][] - [vllm_omni.model_executor.models.qwen3_tts.tokenizer_25hz.vq.core_vq.DistributedGroupResidualVectorQuantization][] - [vllm_omni.model_executor.models.qwen3_tts.tokenizer_25hz.vq.core_vq.DistributedResidualVectorQuantization][] - [vllm_omni.model_executor.models.qwen3_tts.tokenizer_25hz.vq.core_vq.EuclideanCodebook][] @@ -88,20 +94,23 @@ Configuration classes. Worker classes and model runners for distributed inference. -- [vllm_omni.diffusion.worker.gpu_diffusion_model_runner.GPUDiffusionModelRunner][] -- [vllm_omni.diffusion.worker.gpu_diffusion_worker.GPUDiffusionWorker][] -- [vllm_omni.diffusion.worker.gpu_diffusion_worker.WorkerProc][] -- [vllm_omni.diffusion.worker.npu.npu_worker.NPUWorker][] -- [vllm_omni.diffusion.worker.npu.npu_worker.NPUWorkerProc][] +- [vllm_omni.diffusion.worker.diffusion_model_runner.DiffusionModelRunner][] +- [vllm_omni.diffusion.worker.diffusion_worker.DiffusionWorker][] +- [vllm_omni.diffusion.worker.diffusion_worker.WorkerProc][] +- [vllm_omni.platforms.npu.worker.npu_ar_model_runner.ExecuteModelState][] +- [vllm_omni.platforms.npu.worker.npu_ar_model_runner.NPUARModelRunner][] +- [vllm_omni.platforms.npu.worker.npu_ar_worker.NPUARWorker][] +- [vllm_omni.platforms.npu.worker.npu_generation_model_runner.NPUGenerationModelRunner][] +- [vllm_omni.platforms.npu.worker.npu_generation_worker.NPUGenerationWorker][] +- [vllm_omni.platforms.npu.worker.npu_model_runner.OmniNPUModelRunner][] +- [vllm_omni.platforms.xpu.worker.xpu_ar_model_runner.XPUARModelRunner][] +- [vllm_omni.platforms.xpu.worker.xpu_ar_worker.XPUARWorker][] +- [vllm_omni.platforms.xpu.worker.xpu_generation_model_runner.XPUGenerationModelRunner][] +- [vllm_omni.platforms.xpu.worker.xpu_generation_worker.XPUGenerationWorker][] - [vllm_omni.worker.gpu_ar_model_runner.ExecuteModelState][] - [vllm_omni.worker.gpu_ar_model_runner.GPUARModelRunner][] - [vllm_omni.worker.gpu_ar_worker.GPUARWorker][] - [vllm_omni.worker.gpu_generation_model_runner.GPUGenerationModelRunner][] - [vllm_omni.worker.gpu_generation_worker.GPUGenerationWorker][] - [vllm_omni.worker.gpu_model_runner.OmniGPUModelRunner][] -- [vllm_omni.worker.npu.npu_ar_model_runner.ExecuteModelState][] -- [vllm_omni.worker.npu.npu_ar_model_runner.NPUARModelRunner][] -- [vllm_omni.worker.npu.npu_ar_worker.NPUARWorker][] -- [vllm_omni.worker.npu.npu_generation_model_runner.NPUGenerationModelRunner][] -- [vllm_omni.worker.npu.npu_generation_worker.NPUGenerationWorker][] -- [vllm_omni.worker.npu.npu_model_runner.OmniNPUModelRunner][] +- [vllm_omni.worker.mixins.OmniWorkerMixin][] diff --git a/docs/user_guide/examples/offline_inference/bagel.md b/docs/user_guide/examples/offline_inference/bagel.md index b2ee5d241..8bab92977 100644 --- a/docs/user_guide/examples/offline_inference/bagel.md +++ b/docs/user_guide/examples/offline_inference/bagel.md @@ -2,6 +2,7 @@ Source . + ## Set up Please refer to the [stage configuration documentation](https://docs.vllm.ai/projects/vllm-omni/en/latest/configuration/stage_configs/) to configure memory allocation appropriately for your hardware setup. @@ -99,7 +100,7 @@ python end2end.py --model ByteDance-Seed/BAGEL-7B-MoT \ BAGEL-7B-MoT supports **multiple modality modes** for different use cases. -The default yaml configuration deploys Thinker and DiT on the same GPU. You can use the default configuration file: [`bagel.yaml`](../../../vllm_omni/model_executor/stage_configs/bagel.yaml) +The default yaml configuration deploys Thinker and DiT on the same GPU. You can use the default configuration file: [`bagel.yaml`](https://github.com/vllm-project/vllm-omni/tree/main/vllm_omni/model_executor/stage_configs/bagel.yaml) #### 📌 Command Line Arguments (end2end.py) @@ -177,3 +178,10 @@ sudo apt install ffmpeg | Stage-0 (Thinker) | **15.04 GiB** **+ KV Cache** | | Stage-1 (DiT) | **26.50 GiB** | | Total | **~42 GiB + KV Cache** | + +## Example materials + +??? abstract "end2end.py" + ``````py + --8<-- "examples/offline_inference/bagel/end2end.py" + `````` diff --git a/docs/user_guide/examples/offline_inference/image_to_image.md b/docs/user_guide/examples/offline_inference/image_to_image.md index c970106d2..701a4f768 100644 --- a/docs/user_guide/examples/offline_inference/image_to_image.md +++ b/docs/user_guide/examples/offline_inference/image_to_image.md @@ -47,10 +47,15 @@ Key arguments: - `--image`: path(s) to the source image(s) (PNG/JPG, converted to RGB). Can specify multiple images. - `--prompt` / `--negative_prompt`: text description (string). - `--cfg_scale`: true classifier-free guidance scale (default: 4.0). Classifier-free guidance is enabled by setting cfg_scale > 1 and providing a negative_prompt. Higher guidance scale encourages images closely linked to the text prompt, usually at the expense of lower image quality. -- `--cfg_parallel_size`: the number of devices to run CFG Parallel. CFG Parallel is valid only if classifier-free guidance is enabled and `cfg_parallel_size` is set to 2. - `--guidance_scale`: guidance scale for guidance-distilled models (default: 1.0, disabled). Unlike classifier-free guidance (--cfg_scale), guidance-distilled models take the guidance scale directly as an input parameter. Enabled when guidance_scale > 1. Ignored when not using guidance-distilled models. - `--num_inference_steps`: diffusion sampling steps (more steps = higher quality, slower). - `--output`: path to save the generated PNG. +- `--vae_use_slicing`: enable VAE slicing for memory optimization. +- `--vae_use_tiling`: enable VAE tiling for memory optimization. +- `--cfg_parallel_size`: set it to 2 to enable CFG Parallel. See more examples in [`user_guide`](https://github.com/vllm-project/vllm-omni/tree/main/docs/user_guide/diffusion/parallelism_acceleration.md#cfg-parallel). +- `--enable-cpu-offload`: enable CPU offloading for diffusion models. + +> ℹ️ If you encounter OOM errors, try using `--vae_use_slicing` and `--vae_use_tiling` to reduce memory usage. ## Example materials diff --git a/docs/user_guide/examples/offline_inference/image_to_video.md b/docs/user_guide/examples/offline_inference/image_to_video.md index d65839dd7..60f00a95b 100644 --- a/docs/user_guide/examples/offline_inference/image_to_video.md +++ b/docs/user_guide/examples/offline_inference/image_to_video.md @@ -52,12 +52,17 @@ Key arguments: - `--num_frames`: Number of frames (default 81). - `--guidance_scale` and `--guidance_scale_high`: CFG scale (applied to low/high-noise stages for MoE). - `--negative_prompt`: Optional list of artifacts to suppress. -- `--cfg_parallel_size`: the number of devices to run CFG Parallel. CFG Parallel is valid only if classifier-free guidance is enabled and `cfg_parallel_size` is set to 2. - `--boundary_ratio`: Boundary split ratio for two-stage MoE models. - `--flow_shift`: Scheduler flow shift (5.0 for 720p, 12.0 for 480p). - `--num_inference_steps`: Number of denoising steps (default 50). - `--fps`: Frames per second for the saved MP4 (requires `diffusers` export_to_video). - `--output`: Path to save the generated video. +- `--vae_use_slicing`: Enable VAE slicing for memory optimization. +- `--vae_use_tiling`: Enable VAE tiling for memory optimization. +- `--cfg_parallel_size`: set it to 2 to enable CFG Parallel. See more examples in [`user_guide`](https://github.com/vllm-project/vllm-omni/tree/main/docs/user_guide/diffusion/parallelism_acceleration.md#cfg-parallel). +- `--enable-cpu-offload`: enable CPU offloading for diffusion models. + +> ℹ️ If you encounter OOM errors, try using `--vae_use_slicing` and `--vae_use_tiling` to reduce memory usage. ## Example materials diff --git a/docs/user_guide/examples/offline_inference/lora_inference.md b/docs/user_guide/examples/offline_inference/lora_inference.md index dde42655e..791948374 100644 --- a/docs/user_guide/examples/offline_inference/lora_inference.md +++ b/docs/user_guide/examples/offline_inference/lora_inference.md @@ -1,8 +1,9 @@ -# LoRA-Inference +# LoRA Inference Examples Source . -This contains examples for using LoRA (Low-Rank Adaptation) adapters with vLLM-omni diffusion models for offline inference. + +This directory contains examples for using LoRA (Low-Rank Adaptation) adapters with vLLM-omni diffusion models for offline inference. The example uses the `stabilityai/stable-diffusion-3.5-medium` as the default model, but you can replace it with other models in vLLM-omni. ## Overview diff --git a/docs/user_guide/examples/offline_inference/qwen3_tts.md b/docs/user_guide/examples/offline_inference/qwen3_tts.md index 81bcb1c11..9007f9350 100644 --- a/docs/user_guide/examples/offline_inference/qwen3_tts.md +++ b/docs/user_guide/examples/offline_inference/qwen3_tts.md @@ -16,6 +16,15 @@ Qwen3 TTS provides multiple task variants for speech generation: ## Setup Please refer to the [stage configuration documentation](https://docs.vllm.ai/projects/vllm-omni/en/latest/configuration/stage_configs/) to configure memory allocation appropriately for your hardware setup. +### ROCm Dependencies + +You will need to install these two dependencies `onnxruntime-rocm` and `sox`. + +``` +pip uninstall onnxruntime # should be removed before we can install onnxruntime-rocm +pip install onnxruntime-rocm sox +``` + ## Quick Start Run a single sample for a task: diff --git a/docs/user_guide/examples/offline_inference/text_to_audio.md b/docs/user_guide/examples/offline_inference/text_to_audio.md new file mode 100644 index 000000000..617bb6958 --- /dev/null +++ b/docs/user_guide/examples/offline_inference/text_to_audio.md @@ -0,0 +1,47 @@ +# Text-To-Audio + +Source . + + +The `stabilityai/stable-audio-open-1.0` pipeline generates audio from text prompts. + +## Prerequisites + +If you use a gated model (e.g., `stabilityai/stable-audio-open-1.0`), ensure you have access: + +1. **Accept Model License**: Visit the model page on Hugging Face (e.g., [stabilityai/stable-audio-open-1.0]) and accept the user agreement. +2. **Authenticate**: Log in to Hugging Face locally to access the gated model. + ```bash + huggingface-cli login + ``` + +## Local CLI Usage + +```bash +python text_to_audio.py \ + --model stabilityai/stable-audio-open-1.0 \ + --prompt "The sound of a hammer hitting a wooden surface" \ + --negative_prompt "Low quality" \ + --seed 42 \ + --guidance_scale 7.0 \ + --audio_length 10.0 \ + --num_inference_steps 100 \ + --output stable_audio_output.wav +``` + +Key arguments: + +- `--prompt`: text description (string). +- `--negative_prompt`: negative prompt for classifier-free guidance. +- `--seed`: integer seed for deterministic generation. +- `--guidance_scale`: classifier-free guidance scale. +- `--audio_length`: audio duration in seconds. +- `--num_inference_steps`: diffusion sampling steps.(more steps = higher quality, slower). +- `--output`: path to save the generated WAV file. + +## Example materials + +??? abstract "text_to_audio.py" + ``````py + --8<-- "examples/offline_inference/text_to_audio/text_to_audio.py" + `````` diff --git a/docs/user_guide/examples/offline_inference/text_to_image.md b/docs/user_guide/examples/offline_inference/text_to_image.md index 486b9f63b..c0dbfd141 100644 --- a/docs/user_guide/examples/offline_inference/text_to_image.md +++ b/docs/user_guide/examples/offline_inference/text_to_image.md @@ -51,7 +51,7 @@ if __name__ == "__main__": For diffusion pipelines, the stage config field `stage_args.[].runtime.max_batch_size` is 1 by default, and the input list is sliced into single-item requests before feeding into the diffusion pipeline. For models that do internally support - batched inputs, you can [modify this configuration](../../../configuration/stage_configs.md) to let the model accept a longer batch of prompts. + batched inputs, you can [modify this configuration](https://github.com/vllm-project/vllm-omni/tree/main/configuration/stage_configs.md) to let the model accept a longer batch of prompts. Apart from string prompt, vLLM-Omni also supports dictionary prompts in the same style as vLLM. This is useful for models that support negative prompts. @@ -95,11 +95,16 @@ Key arguments: - `--prompt`: text description (string). - `--seed`: integer seed for deterministic sampling. - `--cfg_scale`: true CFG scale (model-specific guidance strength). -- `--cfg_parallel_size`: the number of devices to run CFG Parallel. CFG Parallel is valid only if classifier-free guidance is enabled and `cfg_parallel_size` is set to 2. - `--num_images_per_prompt`: number of images to generate per prompt (saves as `output`, `output_1`, ...). - `--num_inference_steps`: diffusion sampling steps (more steps = higher quality, slower). - `--height/--width`: output resolution (defaults 1024x1024). - `--output`: path to save the generated PNG. +- `--vae_use_slicing`: enable VAE slicing for memory optimization. +- `--vae_use_tiling`: enable VAE tiling for memory optimization. +- `--cfg_parallel_size`: set it to 2 to enable CFG Parallel. See more examples in [`user_guide`](https://github.com/vllm-project/vllm-omni/tree/main/docs/user_guide/diffusion/parallelism_acceleration.md#cfg-parallel). +- `--enable-cpu-offload`: enable CPU offloading for diffusion models. + +> ℹ️ If you encounter OOM errors, try using `--vae_use_slicing` and `--vae_use_tiling` to reduce memory usage. > ℹ️ Qwen-Image currently publishes best-effort presets at `1328x1328`, `1664x928`, `928x1664`, `1472x1140`, `1140x1472`, `1584x1056`, and `1056x1584`. Adjust `--height/--width` accordingly for the most reliable outcomes. diff --git a/docs/user_guide/examples/offline_inference/text_to_video.md b/docs/user_guide/examples/offline_inference/text_to_video.md index db0860b38..ec1da2756 100644 --- a/docs/user_guide/examples/offline_inference/text_to_video.md +++ b/docs/user_guide/examples/offline_inference/text_to_video.md @@ -12,10 +12,11 @@ python text_to_video.py \ --prompt "Two anthropomorphic cats in comfy boxing gear and bright gloves fight intensely on a spotlighted stage." \ --negative_prompt "" \ --height 480 \ - --width 640 \ - --num_frames 32 \ + --width 832 \ + --num_frames 33 \ --guidance_scale 4.0 \ --guidance_scale_high 3.0 \ + --flow_shift 12.0 \ --num_inference_steps 40 \ --fps 16 \ --output t2v_out.mp4 @@ -24,14 +25,19 @@ python text_to_video.py \ Key arguments: - `--prompt`: text description (string). -- `--height/--width`: output resolution (defaults 720x1280). Dimensions should align with Wan VAE downsampling (multiples of 8). +- `--height/--width`: output resolution (defaults 480x832, i.e. 480P). Dimensions should align with Wan VAE downsampling (multiples of 8). - `--num_frames`: Number of frames (Wan default is 81). -- `--guidance_scale` and `--guidance_scale_high`: CFG scale (applied to low/high).. +- `--guidance_scale` and `--guidance_scale_high`: CFG scale (applied to low/high). - `--negative_prompt`: optional list of artifacts to suppress (the PR demo used a long Chinese string). -- `--cfg_parallel_size`: the number of devices to run CFG Parallel. CFG Parallel is valid only if classifier-free guidance is enabled and `cfg_parallel_size` is set to 2. -- `--boundary_ratio`: Boundary split ratio for low/high DiT. +- `--boundary_ratio`: Boundary split ratio for low/high DiT. Default `0.875` uses both transformers for best quality. Set to `1.0` to load only the low-noise transformer (saves noticeable memory with good quality, recommended if memory is limited). Set to `0.0` loads only the high-noise transformer (not recommended, lower quality). - `--fps`: frames per second for the saved MP4 (requires `diffusers` export_to_video). - `--output`: path to save the generated video. +- `--vae_use_slicing`: enable VAE slicing for memory optimization. +- `--vae_use_tiling`: enable VAE tiling for memory optimization. +- `--cfg_parallel_size`: set it to 2 to enable CFG Parallel. See more examples in [`user_guide`](https://github.com/vllm-project/vllm-omni/tree/main/docs/user_guide/diffusion/parallelism_acceleration.md#cfg-parallel). +- `--enable-cpu-offload`: enable CPU offloading for diffusion models. + +> ℹ️ If you encounter OOM errors, try using `--vae_use_slicing` and `--vae_use_tiling` to reduce memory usage. ## Example materials diff --git a/docs/user_guide/examples/online_serving/bagel.md b/docs/user_guide/examples/online_serving/bagel.md index 107e42449..f29c095f4 100644 --- a/docs/user_guide/examples/online_serving/bagel.md +++ b/docs/user_guide/examples/online_serving/bagel.md @@ -2,9 +2,10 @@ Source . + ## 🛠️ Installation -Please refer to [README.md](../../../README.md) +Please refer to [README.md](https://github.com/vllm-project/vllm-omni/tree/main/README.md) ## Run examples (BAGEL-7B-MoT) @@ -74,7 +75,7 @@ python openai_chat_client.py \ BAGEL-7B-MoT supports **multiple modality modes** for different use cases. -The default yaml configuration deploys Thinker and DiT on the same GPU. You can use the default configuration file: [`bagel.yaml`](../../../vllm_omni/model_executor/stage_configs/bagel.yaml) +The default yaml configuration deploys Thinker and DiT on the same GPU. You can use the default configuration file: [`bagel.yaml`](https://github.com/vllm-project/vllm-omni/tree/main/vllm_omni/model_executor/stage_configs/bagel.yaml) | Modality | Input | Output | Description | | ----------- | ------------ | ------ | -------------------------------------- | @@ -230,3 +231,14 @@ sudo apt install ffmpeg | Stage-0 (Thinker) | **15.04 GiB** **+ KV Cache** | | Stage-1 (DiT) | **26.50 GiB** | | Total | **~42 GiB + KV Cache** | + +## Example materials + +??? abstract "openai_chat_client.py" + ``````py + --8<-- "examples/online_serving/bagel/openai_chat_client.py" + `````` +??? abstract "run_server.sh" + ``````sh + --8<-- "examples/online_serving/bagel/run_server.sh" + `````` diff --git a/docs/user_guide/examples/online_serving/lora_inference.md b/docs/user_guide/examples/online_serving/lora_inference.md index 4c8b215d2..253c04ecc 100644 --- a/docs/user_guide/examples/online_serving/lora_inference.md +++ b/docs/user_guide/examples/online_serving/lora_inference.md @@ -1,7 +1,8 @@ -# LoRA-Inference +# Online LoRA Inference (Diffusion) Source . + This example shows how to use **per-request LoRA** with vLLM-Omni diffusion models via the OpenAI-compatible Chat Completions API. > Note: The LoRA adapter path must be readable on the **server** machine (usually a local path or a mounted directory). @@ -55,15 +56,17 @@ lora_adapter/ └── adapter_model.safetensors ``` +## Example materials + ??? abstract "openai_chat_client.py" ``````py --8<-- "examples/online_serving/lora_inference/openai_chat_client.py" `````` ??? abstract "run_curl_lora_inference.sh" - ``````py + ``````sh --8<-- "examples/online_serving/lora_inference/run_curl_lora_inference.sh" `````` ??? abstract "run_server.sh" - ``````py + ``````sh --8<-- "examples/online_serving/lora_inference/run_server.sh" `````` diff --git a/docs/user_guide/examples/online_serving/qwen3_tts.md b/docs/user_guide/examples/online_serving/qwen3_tts.md new file mode 100644 index 000000000..48bd58b00 --- /dev/null +++ b/docs/user_guide/examples/online_serving/qwen3_tts.md @@ -0,0 +1,189 @@ +# Qwen3-TTS Online Serving + +Source . + + +This directory contains examples for running Qwen3-TTS models with vLLM-Omni's online serving API. + +## Supported Models + +| Model | Task Type | Description | +|-------|-----------|-------------| +| `Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice` | CustomVoice | Predefined speaker voices with optional style control | +| `Qwen/Qwen3-TTS-12Hz-1.7B-VoiceDesign` | VoiceDesign | Natural language voice style description | +| `Qwen/Qwen3-TTS-12Hz-1.7B-Base` | Base | Voice cloning from reference audio | + +## Quick Start + +### 1. Start the Server + +```bash +# CustomVoice model (default) +./run_server.sh + +# Or specify task type +./run_server.sh CustomVoice +./run_server.sh VoiceDesign +./run_server.sh Base +``` + +### 2. Run the Client + +```bash +# CustomVoice: Use predefined speaker +python openai_speech_client.py \ + --text "你好，我是通义千问" \ + --voice Vivian \ + --language Chinese + +# CustomVoice with style instruction +python openai_speech_client.py \ + --text "今天天气真好" \ + --voice Ryan \ + --instructions "用开心的语气说" + +# VoiceDesign: Describe the voice style +python openai_speech_client.py \ + --model Qwen/Qwen3-TTS-12Hz-1.7B-VoiceDesign \ + --task-type VoiceDesign \ + --text "哥哥，你回来啦" \ + --instructions "体现撒娇稚嫩的萝莉女声，音调偏高" + +# Base: Voice cloning +python openai_speech_client.py \ + --model Qwen/Qwen3-TTS-12Hz-1.7B-Base \ + --task-type Base \ + --text "Hello, this is a cloned voice" \ + --ref-audio /path/to/reference.wav \ + --ref-text "Original transcript of the reference audio" +``` + +### 3. Using curl + +```bash +# Simple TTS request +curl -X POST http://localhost:8000/v1/audio/speech \ + -H "Content-Type: application/json" \ + -d '{ + "input": "Hello, how are you?", + "voice": "Vivian", + "language": "English" + }' --output output.wav + +# With style instruction +curl -X POST http://localhost:8000/v1/audio/speech \ + -H "Content-Type: application/json" \ + -d '{ + "input": "I am so excited!", + "voice": "Vivian", + "instructions": "Speak with great enthusiasm" + }' --output excited.wav + +# List available voices in CustomVoice models +curl http://localhost:8000/v1/audio/voices +``` + +## API Reference + +### Endpoint + +``` +POST /v1/audio/speech +``` + +This endpoint follows the [OpenAI Audio Speech API](https://platform.openai.com/docs/api-reference/audio/createSpeech) format with additional Qwen3-TTS parameters. + +### Request Body + +```json +{ + "input": "Text to synthesize", + "voice": "Vivian", + "response_format": "wav", + "task_type": "CustomVoice", + "language": "Auto", + "instructions": "Optional style instructions", + "ref_audio": "URL or base64 for voice cloning", + "ref_text": "Reference audio transcript", + "x_vector_only_mode": false, + "max_new_tokens": 2048 +} +``` + +> **Note:** The `model` field is optional when serving a single model, as the server already knows which model is loaded. + +### Response + +Returns audio data in the requested format (default: WAV). + +## Parameters + +### Standard OpenAI Parameters + +| Parameter | Type | Default | Description | +|-----------|------|---------|-------------| +| `input` | string | required | Text to synthesize | +| `voice` | string | "Vivian" | Speaker/voice name | +| `response_format` | string | "wav" | Audio format: wav, mp3, flac, pcm, aac, opus | +| `speed` | float | 1.0 | Playback speed (0.25-4.0) | +| `model` | string | optional | Model name (optional when serving single model) | + +### Qwen3-TTS Parameters + +| Parameter | Type | Default | Description | +|-----------|------|---------|-------------| +| `task_type` | string | "CustomVoice" | Task: CustomVoice, VoiceDesign, or Base | +| `language` | string | "Auto" | Language: Auto, Chinese, English, Japanese, Korean | +| `instructions` | string | "" | Voice style/emotion instructions | +| `max_new_tokens` | int | 2048 | Maximum tokens to generate | + +### Voice Clone Parameters (Base task) + +| Parameter | Type | Required | Description | +|-----------|------|----------|-------------| +| `ref_audio` | string | Yes* | Reference audio (file path, URL, or base64) | +| `ref_text` | string | No | Transcript of reference audio (for ICL mode) | +| `x_vector_only_mode` | bool | false | Use speaker embedding only (no ICL) | + +## Python Usage + +```python +import httpx + +# Simple request +response = httpx.post( + "http://localhost:8000/v1/audio/speech", + json={ + "model": "Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice", + "input": "Hello world", + "voice": "Vivian", + }, + timeout=300.0, +) + +with open("output.wav", "wb") as f: + f.write(response.content) +``` + +## Limitations + +- **No streaming**: Audio is generated completely before being returned. Streaming will be supported after the pipeline is disaggregated (see RFC #938). +- **Single request**: Batch processing is not yet optimized for online serving. + +## Troubleshooting + +1. **Connection refused**: Make sure the server is running on the correct port +2. **Out of memory**: Reduce `--gpu-memory-utilization` in run_server.sh +3. **Unsupported speaker**: Check supported speakers via model documentation +4. **Voice clone fails**: Ensure you're using the Base model variant for voice cloning + +## Example materials + +??? abstract "openai_speech_client.py" + ``````py + --8<-- "examples/online_serving/qwen3_tts/openai_speech_client.py" + `````` +??? abstract "run_server.sh" + ``````sh + --8<-- "examples/online_serving/qwen3_tts/run_server.sh" + ``````