From b0efb6ede07adea33465cdefe4b236eb6559e5a0 Mon Sep 17 00:00:00 2001 From: Alicia <115451386+congw729@users.noreply.github.com> Date: Tue, 24 Mar 2026 11:20:50 +0800 Subject: [PATCH] [Doc] Sync and fix. (#2110) Signed-off-by: Alicia <115451386+congw729@users.noreply.github.com> --- docs/.nav.yml | 1 + docs/api/README.md | 41 +++ .../examples/offline_inference/fish_speech.md | 12 - .../examples/offline_inference/glm_image.md | 8 +- .../examples/offline_inference/helios.md | 203 ++++++++++++++ .../offline_inference/hunyuan_image3.md | 35 +++ .../offline_inference/image_to_video.md | 1 + .../mammothmodal2_preview.md | 46 ++++ .../examples/offline_inference/mimo_audio.md | 254 +++++++++++++++++ .../examples/offline_inference/qwen3_omni.md | 57 ++++ .../examples/offline_inference/qwen3_tts.md | 2 +- .../offline_inference/text_to_image.md | 258 ++++++++++++------ .../offline_inference/text_to_video.md | 126 ++++++++- .../examples/offline_inference/voxtral_tts.md | 74 +++++ .../offline_inference/x_to_video_audio.md | 14 + .../examples/online_serving/chart-helm.md | 226 +++++++++++++++ .../examples/online_serving/image_to_video.md | 10 +- .../examples/online_serving/mimo_audio.md | 72 +++++ .../examples/online_serving/qwen3_tts.md | 40 +++ .../examples/online_serving/text_to_image.md | 23 -- .../examples/online_serving/text_to_video.md | 11 +- mkdocs.yml | 7 +- .../qwen3_omni_moe_code_predictor_mtp.py | 2 +- 23 files changed, 1382 insertions(+), 141 deletions(-) create mode 100644 docs/user_guide/examples/offline_inference/helios.md create mode 100644 docs/user_guide/examples/offline_inference/hunyuan_image3.md create mode 100644 docs/user_guide/examples/offline_inference/mammothmodal2_preview.md create mode 100644 docs/user_guide/examples/offline_inference/mimo_audio.md create mode 100644 docs/user_guide/examples/offline_inference/voxtral_tts.md create mode 100644 docs/user_guide/examples/online_serving/chart-helm.md create mode 100644 docs/user_guide/examples/online_serving/mimo_audio.md diff --git a/docs/.nav.yml b/docs/.nav.yml index fd4bec2ef..1d7346669 100644 --- a/docs/.nav.yml +++ b/docs/.nav.yml @@ -28,6 +28,7 @@ nav: - Text-To-Audio: user_guide/examples/offline_inference/text_to_audio.md - Text-To-Image: user_guide/examples/offline_inference/text_to_image.md - Text-To-Video: user_guide/examples/offline_inference/text_to_video.md + - Voxtral TTS Offline Inference: user_guide/examples/offline_inference/voxtral_tts.md - X-To-Video-Audio: user_guide/examples/offline_inference/x_to_video_audio.md - Online Serving: - BAGEL-7B-MoT: user_guide/examples/online_serving/bagel.md diff --git a/docs/api/README.md b/docs/api/README.md index 549d8bb1d..2266a5241 100644 --- a/docs/api/README.md +++ b/docs/api/README.md @@ -6,6 +6,7 @@ Main entry points for vLLM-Omni inference and serving. - [vllm_omni.entrypoints.async_omni.AsyncOmni][] - [vllm_omni.entrypoints.async_omni_diffusion.AsyncOmniDiffusion][] +- [vllm_omni.entrypoints.cfg_companion_tracker.CfgCompanionTracker][] - [vllm_omni.entrypoints.cli.benchmark.base.OmniBenchmarkSubcommandBase][] - [vllm_omni.entrypoints.cli.benchmark.main.OmniBenchmarkSubcommand][] - [vllm_omni.entrypoints.cli.benchmark.serve.OmniBenchmarkServingSubcommand][] @@ -13,11 +14,13 @@ Main entry points for vLLM-Omni inference and serving. - [vllm_omni.entrypoints.client_request_state.ClientRequestState][] - [vllm_omni.entrypoints.omni.Omni][] - [vllm_omni.entrypoints.omni_base.OmniBase][] +- [vllm_omni.entrypoints.pd_utils.PDDisaggregationMixin][] ## Inputs Input data structures for multi-modal inputs. +- [vllm_omni.inputs.data.OmniCustomPrompt][] - [vllm_omni.inputs.data.OmniDiffusionSamplingParams][] - [vllm_omni.inputs.data.OmniEmbedsPrompt][] - [vllm_omni.inputs.data.OmniTextPrompt][] @@ -43,8 +46,18 @@ Engine classes for offline and online inference. - [vllm_omni.engine.OmniEngineCoreRequest][] - [vllm_omni.engine.PromptEmbedsPayload][] - [vllm_omni.engine.arg_utils.OmniEngineArgs][] +- [vllm_omni.engine.async_omni_engine.AsyncOmniEngine][] +- [vllm_omni.engine.mm_outputs.MultimodalCompletionOutput][] +- [vllm_omni.engine.mm_outputs.MultimodalPayload][] +- [vllm_omni.engine.orchestrator.Orchestrator][] +- [vllm_omni.engine.orchestrator.OrchestratorRequestState][] +- [vllm_omni.engine.output_modality.OutputModality][] +- [vllm_omni.engine.output_modality.TensorAccumulationStrategy][] - [vllm_omni.engine.output_processor.MultimodalOutputProcessor][] - [vllm_omni.engine.output_processor.OmniRequestState][] +- [vllm_omni.engine.stage_engine_core_client.StageEngineCoreClient][] +- [vllm_omni.engine.stage_init_utils.StageMetadata][] +- [vllm_omni.engine.stage_init_utils.StartedLlmStage][] ## Core @@ -56,6 +69,19 @@ Core scheduling and caching components. - [vllm_omni.core.sched.output.OmniCachedRequestData][] - [vllm_omni.core.sched.output.OmniNewRequestData][] - [vllm_omni.core.sched.output.OmniSchedulerOutput][] +- [vllm_omni.model_executor.models.cosyvoice3.code2wav_core.cfm.BASECFM][] +- [vllm_omni.model_executor.models.cosyvoice3.code2wav_core.cfm.CausalConditionalCFM][] +- [vllm_omni.model_executor.models.cosyvoice3.code2wav_core.cfm.CausalMaskedDiffWithDiT][] +- [vllm_omni.model_executor.models.cosyvoice3.code2wav_core.cfm.ConditionalCFM][] +- [vllm_omni.model_executor.models.cosyvoice3.code2wav_core.hifigan.CausalConv1d][] +- [vllm_omni.model_executor.models.cosyvoice3.code2wav_core.hifigan.CausalConv1dUpsample][] +- [vllm_omni.model_executor.models.cosyvoice3.code2wav_core.hifigan.CausalConvRNNF0Predictor][] +- [vllm_omni.model_executor.models.cosyvoice3.code2wav_core.hifigan.CausalHiFTGenerator][] +- [vllm_omni.model_executor.models.cosyvoice3.code2wav_core.hifigan.HiFTGenerator][] +- [vllm_omni.model_executor.models.cosyvoice3.code2wav_core.hifigan.SineGen][] +- [vllm_omni.model_executor.models.cosyvoice3.code2wav_core.hifigan.SineGen2][] +- [vllm_omni.model_executor.models.cosyvoice3.code2wav_core.hifigan.Snake][] +- [vllm_omni.model_executor.models.cosyvoice3.code2wav_core.hifigan.SourceModuleHnNSF][] - [vllm_omni.model_executor.models.qwen3_tts.tokenizer_25hz.vq.core_vq.DistributedGroupResidualVectorQuantization][] - [vllm_omni.model_executor.models.qwen3_tts.tokenizer_25hz.vq.core_vq.DistributedResidualVectorQuantization][] - [vllm_omni.model_executor.models.qwen3_tts.tokenizer_25hz.vq.core_vq.EuclideanCodebook][] @@ -67,9 +93,17 @@ Core scheduling and caching components. Configuration classes. - [vllm_omni.config.model.OmniModelConfig][] +- [vllm_omni.config.stage_config.ModelPipeline][] +- [vllm_omni.config.stage_config.StageConfig][] +- [vllm_omni.config.stage_config.StageConfigFactory][] +- [vllm_omni.config.stage_config.StageType][] - [vllm_omni.diffusion.cache.teacache.config.TeaCacheConfig][] - [vllm_omni.distributed.omni_connectors.utils.config.ConnectorSpec][] - [vllm_omni.distributed.omni_connectors.utils.config.OmniTransferConfig][] +- [vllm_omni.model_executor.models.cosyvoice3.config.CosyVoice3Config][] +- [vllm_omni.model_executor.models.fish_speech.configuration_fish_speech.FishSpeechConfig][] +- [vllm_omni.model_executor.models.fish_speech.configuration_fish_speech.FishSpeechFastARConfig][] +- [vllm_omni.model_executor.models.fish_speech.configuration_fish_speech.FishSpeechSlowARConfig][] - [vllm_omni.model_executor.models.mimo_audio.config_mimo_audio.MiMoAudioConfig][] - [vllm_omni.model_executor.models.mimo_audio.config_mimo_audio.MiMoAudioTokenizerConfig][] - [vllm_omni.model_executor.models.qwen3_tts.configuration_qwen3_tts.Qwen3TTSConfig][] @@ -83,6 +117,10 @@ Configuration classes. - [vllm_omni.model_executor.models.qwen3_tts.tokenizer_25hz.configuration_qwen3_tts_tokenizer_v1.Qwen3TTSTokenizerV1DecoderConfig][] - [vllm_omni.model_executor.models.qwen3_tts.tokenizer_25hz.configuration_qwen3_tts_tokenizer_v1.Qwen3TTSTokenizerV1DecoderDiTConfig][] - [vllm_omni.model_executor.models.qwen3_tts.tokenizer_25hz.configuration_qwen3_tts_tokenizer_v1.Qwen3TTSTokenizerV1EncoderConfig][] +- [vllm_omni.transformers_utils.configs.mammoth_moda2.Mammothmoda2Config][] +- [vllm_omni.transformers_utils.configs.mammoth_moda2.Mammothmoda2Qwen2_5_VLConfig][] +- [vllm_omni.transformers_utils.configs.mammoth_moda2.Mammothmoda2Qwen2_5_VLTextConfig][] +- [vllm_omni.transformers_utils.configs.mammoth_moda2.Mammothmoda2Qwen2_5_VLVisionConfig][] ## Workers @@ -93,6 +131,8 @@ Worker classes and model runners for distributed inference. - [vllm_omni.diffusion.worker.diffusion_worker.DiffusionWorker][] - [vllm_omni.diffusion.worker.diffusion_worker.WorkerProc][] - [vllm_omni.diffusion.worker.diffusion_worker.WorkerWrapperBase][] +- [vllm_omni.diffusion.worker.utils.DiffusionRequestState][] +- [vllm_omni.diffusion.worker.utils.RunnerOutput][] - [vllm_omni.platforms.npu.worker.npu_ar_model_runner.ExecuteModelState][] - [vllm_omni.platforms.npu.worker.npu_ar_model_runner.NPUARModelRunner][] - [vllm_omni.platforms.npu.worker.npu_ar_worker.NPUARWorker][] @@ -110,5 +150,6 @@ Worker classes and model runners for distributed inference. - [vllm_omni.worker.gpu_generation_model_runner.GPUGenerationModelRunner][] - [vllm_omni.worker.gpu_generation_worker.GPUGenerationWorker][] - [vllm_omni.worker.gpu_memory_utils.parse_cuda_visible_devices][] +- [vllm_omni.worker.gpu_model_runner.CUDAGraphWrapper][] - [vllm_omni.worker.gpu_model_runner.OmniGPUModelRunner][] - [vllm_omni.worker.mixins.OmniWorkerMixin][] diff --git a/docs/user_guide/examples/offline_inference/fish_speech.md b/docs/user_guide/examples/offline_inference/fish_speech.md index dd9066afd..0d5c538be 100644 --- a/docs/user_guide/examples/offline_inference/fish_speech.md +++ b/docs/user_guide/examples/offline_inference/fish_speech.md @@ -79,19 +79,7 @@ Each DAC decoder chunk is logged as it arrives. The final WAV file is written on ## Example materials -??? abstract "diag_codebook.py" - ``````py - --8<-- "examples/offline_inference/fish_speech/diag_codebook.py" - `````` -??? abstract "diag_reference_compare.py" - ``````py - --8<-- "examples/offline_inference/fish_speech/diag_reference_compare.py" - `````` ??? abstract "end2end.py" ``````py --8<-- "examples/offline_inference/fish_speech/end2end.py" `````` -??? abstract "standalone_test.py" - ``````py - --8<-- "examples/offline_inference/fish_speech/standalone_test.py" - `````` diff --git a/docs/user_guide/examples/offline_inference/glm_image.md b/docs/user_guide/examples/offline_inference/glm_image.md index 7b5ab25a0..4519e26fd 100644 --- a/docs/user_guide/examples/offline_inference/glm_image.md +++ b/docs/user_guide/examples/offline_inference/glm_image.md @@ -43,8 +43,7 @@ GLM-Image uses a 2-stage pipeline: ```bash python end2end.py \ - --model-path /path/to/glm-image \ - --config-path ../../vllm_omni/model_executor/stage_configs/glm_image.yaml \ + --config-path ../../../vllm_omni/model_executor/stage_configs/glm_image.yaml \ --prompt "A beautiful sunset over the ocean with sailing boats" \ --height 1024 \ --width 1024 \ @@ -55,8 +54,7 @@ python end2end.py \ ```bash python end2end.py \ - --model-path /path/to/glm-image \ - --config-path ../../vllm_omni/model_executor/stage_configs/glm_image.yaml \ + --config-path ../../../vllm_omni/model_executor/stage_configs/glm_image.yaml \ --prompt "Transform this scene into a winter wonderland" \ --image input.png \ --output output_i2i.png @@ -67,7 +65,7 @@ python end2end.py \ ```bash python end2end.py \ --model-path /path/to/glm-image \ - --config-path ../../vllm_omni/model_executor/stage_configs/glm_image.yaml \ + --config-path ../../../vllm_omni/model_executor/stage_configs/glm_image.yaml \ --prompt "A photorealistic cat sitting on a window sill" \ --height 1024 \ --width 1024 \ diff --git a/docs/user_guide/examples/offline_inference/helios.md b/docs/user_guide/examples/offline_inference/helios.md new file mode 100644 index 000000000..f83ebc2de --- /dev/null +++ b/docs/user_guide/examples/offline_inference/helios.md @@ -0,0 +1,203 @@ +# Helios Video Generation + +Source . + + +Helios is a text-to-video (T2V), image-to-video (I2V), and video-to-video (V2V) diffusion model. This example demonstrates end-to-end video generation using vLLM-Omni with three model variants: + +| Variant | Description | Key Features | +|---------|-------------|--------------| +| **Helios-Base** | Base model, Stage 1 only | Single-stage denoising, `guidance_scale=5.0` | +| **Helios-Mid** | Mid model, Stage 2 pyramid | Multi-stage pyramid denoising, CFG-Zero* support | +| **Helios-Distilled** | Distilled model, Stage 2+3 | Few-step inference with DMD, `guidance_scale=1.0` | + +## Setup + +Please refer to the [stage configuration documentation](https://docs.vllm.ai/projects/vllm-omni/en/latest/configuration/stage_configs/) to configure memory allocation appropriately for your hardware setup. + +## Run Examples + +Get into the example folder: +```bash +cd examples/offline_inference/helios +``` + +### Text-to-Video (T2V) + +**Helios-Base** (Stage 1 only): +```bash +python end2end.py \ + --model BestWishYsh/Helios-Base \ + --sample-type t2v \ + --prompt "A dynamic time-lapse video showing the rapidly moving scenery from the window of a speeding train." \ + --guidance-scale 5.0 \ + --output helios_t2v_base.mp4 +``` + +**Helios-Mid** (Stage 2 + CFG-Zero*): +```bash +python end2end.py \ + --model BestWishYsh/Helios-Mid \ + --sample-type t2v \ + --prompt "A dynamic time-lapse video showing the rapidly moving scenery from the window of a speeding train." \ + --guidance-scale 5.0 \ + --is-enable-stage2 \ + --pyramid-num-inference-steps-list 20 20 20 \ + --use-cfg-zero-star --use-zero-init --zero-steps 1 \ + --output helios_t2v_mid.mp4 +``` + +**Helios-Distilled** (Stage 2 pyramid + DMD): +```bash +python end2end.py \ + --model BestWishYsh/Helios-Distilled \ + --sample-type t2v \ + --prompt "A dynamic time-lapse video showing the rapidly moving scenery from the window of a speeding train." \ + --num-frames 240 \ + --guidance-scale 1.0 \ + --is-enable-stage2 \ + --pyramid-num-inference-steps-list 2 2 2 \ + --is-amplify-first-chunk \ + --output helios_t2v_distilled.mp4 +``` + +### Image-to-Video (I2V) + +**Helios-Base**: +```bash +python end2end.py \ + --model BestWishYsh/Helios-Base \ + --sample-type i2v \ + --image-path /path/to/image.jpg \ + --prompt "A towering emerald wave surges forward, its crest curling with raw power and energy." \ + --guidance-scale 5.0 \ + --output helios_i2v_base.mp4 +``` + +**Helios-Mid** (Stage 2 + CFG-Zero*): +```bash +python end2end.py \ + --model BestWishYsh/Helios-Mid \ + --sample-type i2v \ + --image-path /path/to/image.jpg \ + --prompt "A towering emerald wave surges forward, its crest curling with raw power and energy." \ + --guidance-scale 5.0 \ + --is-enable-stage2 \ + --pyramid-num-inference-steps-list 20 20 20 \ + --use-cfg-zero-star --use-zero-init --zero-steps 1 \ + --output helios_i2v_mid.mp4 +``` + +**Helios-Distilled**: +```bash +python end2end.py \ + --model BestWishYsh/Helios-Distilled \ + --sample-type i2v \ + --image-path /path/to/image.jpg \ + --prompt "A towering emerald wave surges forward, its crest curling with raw power and energy." \ + --num-frames 240 \ + --guidance-scale 1.0 \ + --is-enable-stage2 \ + --pyramid-num-inference-steps-list 2 2 2 \ + --is-amplify-first-chunk \ + --output helios_i2v_distilled.mp4 +``` + +### Video-to-Video (V2V) + +**Helios-Base**: +```bash +python end2end.py \ + --model BestWishYsh/Helios-Base \ + --sample-type v2v \ + --video-path /path/to/video.mp4 \ + --prompt "A bright yellow Lamborghini speeds along a curving mountain road." \ + --guidance-scale 5.0 \ + --output helios_v2v_base.mp4 +``` + +**Helios-Mid** (Stage 2 + CFG-Zero*): +```bash +python end2end.py \ + --model BestWishYsh/Helios-Mid \ + --sample-type v2v \ + --video-path /path/to/video.mp4 \ + --prompt "A bright yellow Lamborghini speeds along a curving mountain road." \ + --guidance-scale 5.0 \ + --is-enable-stage2 \ + --pyramid-num-inference-steps-list 20 20 20 \ + --use-cfg-zero-star --use-zero-init --zero-steps 1 \ + --output helios_v2v_mid.mp4 +``` + +**Helios-Distilled**: +```bash +python end2end.py \ + --model BestWishYsh/Helios-Distilled \ + --sample-type v2v \ + --video-path /path/to/video.mp4 \ + --prompt "A bright yellow Lamborghini speeds along a curving mountain road." \ + --num-frames 240 \ + --guidance-scale 1.0 \ + --is-enable-stage2 \ + --pyramid-num-inference-steps-list 2 2 2 \ + --is-amplify-first-chunk \ + --output helios_v2v_distilled.mp4 +``` + +## Common Parameters + +| Parameter | Default | Description | +|-----------|---------|-------------| +| `--model` | `BestWishYsh/Helios-Base` | Model ID or local path | +| `--sample-type` | `t2v` | Generation mode: `t2v`, `i2v`, or `v2v` | +| `--prompt` | — | Text prompt describing the video | +| `--negative-prompt` | *(see source)* | Negative prompt for CFG (includes anti-static terms) | +| `--image-path` | — | Input image (required for `i2v`) | +| `--video-path` | — | Input video (required for `v2v`) | +| `--height` | `384` | Video height in pixels | +| `--width` | `640` | Video width in pixels | +| `--num-frames` | `99` | Number of output frames | +| `--num-inference-steps` | `50` | Denoising steps (Stage 1 only) | +| `--guidance-scale` | `5.0` | CFG scale (`1.0` for Distilled) | +| `--seed` | `42` | Random seed | +| `--fps` | `16` | Output video frame rate | +| `--output` | `helios_output.mp4` | Output file path | + +### Stage 2 / Pyramid Parameters + +| Parameter | Default | Description | +|-----------|---------|-------------| +| `--is-enable-stage2` | off | Enable pyramid multi-stage denoising | +| `--pyramid-num-stages` | `3` | Number of pyramid stages | +| `--pyramid-num-inference-steps-list` | `10 10 10` | Steps per pyramid stage | +| `--is-amplify-first-chunk` | off | DMD amplification (Distilled) | + +### CFG-Zero* Parameters + +| Parameter | Default | Description | +|-----------|---------|-------------| +| `--use-cfg-zero-star` | off | Enable CFG-Zero* guidance (Mid) | +| `--use-zero-init` | off | Zero init for first steps | +| `--zero-steps` | `1` | Number of zero-init steps | + +### Memory & Parallelism + +| Parameter | Default | Description | +|-----------|---------|-------------| +| `--vae-use-slicing` | off | Enable VAE slicing | +| `--vae-use-tiling` | off | Enable VAE tiling | +| `--enforce-eager` | off | Disable torch.compile | +| `--enable-cpu-offload` | off | CPU offloading | +| `--enable-layerwise-offload` | off | Layerwise offloading | +| `--ulysses-degree` | `1` | Ulysses SP degree | +| `--ring-degree` | `1` | Ring SP degree | +| `--cfg-parallel-size` | `1` | CFG parallel size (1 or 2) | +| `--tensor-parallel-size` | `1` | Tensor parallelism size | + +## Example materials + +??? abstract "end2end.py" + ``````py + --8<-- "examples/offline_inference/helios/end2end.py" + `````` diff --git a/docs/user_guide/examples/offline_inference/hunyuan_image3.md b/docs/user_guide/examples/offline_inference/hunyuan_image3.md new file mode 100644 index 000000000..636aa0c11 --- /dev/null +++ b/docs/user_guide/examples/offline_inference/hunyuan_image3.md @@ -0,0 +1,35 @@ +# HunyuanImage-3.0 Image-to-Text Inference + +Source . + + +This example demonstrates how to run HunyuanImage-3.0 Image-to-Text with the vLLM-Omni. + +## Local CLI Usage + +Download the example image: + +```bash +wget https://vllm-public-assets.s3.us-west-2.amazonaws.com/vision_model_images/cherry_blossom.jpg +``` + +Run example: + +```bash +python image_to_text.py \ + --image cherry_blossom.jpg \ + --prompt "<|startoftext|>You are an assistant that understands images and outputs text.Describe the content of the picture." +``` + +Key arguments: + +- `--model`: Model used. Default is: tencent/HunyuanImage-3.0-Instruct (Optional). +- `--image`: Path to input image (required). +- `--prompt`: Text description used to guide image understanding (required). + +## Example materials + +??? abstract "image_to_text.py" + ``````py + --8<-- "examples/offline_inference/hunyuan_image3/image_to_text.py" + `````` diff --git a/docs/user_guide/examples/offline_inference/image_to_video.md b/docs/user_guide/examples/offline_inference/image_to_video.md index 8052c07af..7a750aeff 100644 --- a/docs/user_guide/examples/offline_inference/image_to_video.md +++ b/docs/user_guide/examples/offline_inference/image_to_video.md @@ -68,6 +68,7 @@ Key arguments: - `--vae-use-slicing`: Enable VAE slicing for memory optimization. - `--vae-use-tiling`: Enable VAE tiling for memory optimization. - `--cfg-parallel-size`: set it to 2 to enable CFG Parallel. See more examples in [`user_guide`](https://github.com/vllm-project/vllm-omni/tree/main/docs/user_guide/diffusion/parallelism_acceleration.md#cfg-parallel). +- `--tensor-parallel-size`: tensor parallel size (effective for models that support TP, e.g. LTX2). - `--enable-cpu-offload`: enable CPU offloading for diffusion models. - `--use-hsdp`: Enable Hybrid Sharded Data Parallel to shard model weights across GPUs. - `--hsdp-shard-size`: Number of GPUs to shard model weights across within each replica group. -1 (default) auto-calculates as world_size / replicate_size. diff --git a/docs/user_guide/examples/offline_inference/mammothmodal2_preview.md b/docs/user_guide/examples/offline_inference/mammothmodal2_preview.md new file mode 100644 index 000000000..85227b748 --- /dev/null +++ b/docs/user_guide/examples/offline_inference/mammothmodal2_preview.md @@ -0,0 +1,46 @@ +# MammothModa2-Preview + +Source . + + +## Run examples (MammothModa2-Preview) + +Download model +```bash +hf bytedance-research/MammothModa2-Preview --local-dir ./MammothModa2-Preview +``` + +### Text-to-Image (T2I) + +```bash +python examples/offline_inference/mammothmodal2_preview/run_mammothmoda2_t2i.py \ + --model ./MammothModa2-Preview \ + --stage-config ./vllm_omni/model_executor/stage_configs/mammoth_moda2.yaml \ + --prompt "A stylish woman riding a motorcycle in NYC, movie poster style" \ + --height 1024 \ + --width 1024 \ + --num-inference-steps 50 \ + --text-guidance-scale 4.0 \ + --out output.png +``` + +### Image Summary + +```bash +python examples/offline_inference/mammothmodal2_preview/run_mammothmoda2_image_summarize.py \ + --model ./MammothModa2-Preview \ + --stage-config ./vllm_omni/model_executor/stage_configs/mammoth_moda2_ar.yaml \ + --question "Summarize this image." \ + --image ./image.png +``` + +## Example materials + +??? abstract "run_mammothmoda2_image_summarize.py" + ``````py + --8<-- "examples/offline_inference/mammothmodal2_preview/run_mammothmoda2_image_summarize.py" + `````` +??? abstract "run_mammothmoda2_t2i.py" + ``````py + --8<-- "examples/offline_inference/mammothmodal2_preview/run_mammothmoda2_t2i.py" + `````` diff --git a/docs/user_guide/examples/offline_inference/mimo_audio.md b/docs/user_guide/examples/offline_inference/mimo_audio.md new file mode 100644 index 000000000..6af7bc72e --- /dev/null +++ b/docs/user_guide/examples/offline_inference/mimo_audio.md @@ -0,0 +1,254 @@ +# MiMo-Audio Offline Inference + +Source . + + +This directory contains an offline demo for running MiMo-Audio models with vLLM Omni. It builds task-specific inputs and generates WAV files or text outputs locally. + +## Model Overview + +MiMo-Audio provides multiple task variants for audio understanding and generation: + +- **tts_sft**: Basic text-to-speech generation from text input. +- **tts_sft_with_instruct**: TTS generation with explicit voice style instructions. +- **tts_sft_with_audio**: TTS generation with audio reference for voice cloning. +- **tts_sft_with_natural_instruction**: TTS generation from natural language descriptions embedded in text. +- **audio_trancribing_sft**: Transcribe audio to text (speech-to-text). +- **audio_understanding_sft**: Understand and analyze audio content with text queries. +- **audio_understanding_sft_with_thinking**: Audio understanding with reasoning chain. +- **spoken_dialogue_sft_multiturn**: Multi-turn spoken dialogue with audio input/output. +- **speech2text_dialogue_sft_multiturn**: Multi-turn dialogue converting speech to text. +- **text_dialogue_sft_multiturn**: Multi-turn text-only dialogue. + +## Setup + +Please refer to the [stage configuration documentation](https://docs.vllm.ai/projects/vllm-omni/en/latest/configuration/stage_configs/) to configure memory allocation appropriately for your hardware setup. + +### Environment Variables + +The `MIMO_AUDIO_TOKENIZER_PATH` environment variable is mandatory due to the specialized architecture: + +```bash +export MIMO_AUDIO_TOKENIZER_PATH="XiaomiMiMo/MiMo-Audio-Tokenizer" +``` + +## Quick Start + +Run a single sample for basic TTS: + +```bash +python3 -u end2end.py \ + --stage-configs-path vllm_omni/model_executor/stage_configs/mimo_audio.yaml \ + --model-name XiaomiMiMo/MiMo-Audio-7B-Instruct \ + --query-type tts_sft +``` + +Run batch samples for basic TTS: + +```bash +python3 -u end2end.py \ + --stage-configs-path vllm_omni/model_executor/stage_configs/mimo_audio.yaml \ + --model-name XiaomiMiMo/MiMo-Audio-7B-Instruct \ + --query-type tts_sft \ + --num-prompts {batch_size} +``` + +When enabling multi-batch processing, if the total number of tokens passed to the next stage exceeds the `max_model_len` value in the `mimo_audio.yaml` configuration file, you must also synchronously update the `max_position_embeddings` value in `MiMo-Audio-7B-Instruct/config.json` to match the modified value. + +Generated audio files are saved to `output_audio/` by default. `--num-prompts` also can be used to all tasks below. + +## Task Usage + +### tts_sft (Basic Text-to-Speech) + +Generate speech from text input: + +```bash +python3 -u end2end.py \ + --stage-configs-path vllm_omni/model_executor/stage_configs/mimo_audio.yaml \ + --model-name XiaomiMiMo/MiMo-Audio-7B-Instruct \ + --query-type tts_sft \ + --text "The weather is so nice today." +``` + +### tts_sft_with_instruct (TTS with Voice Instructions) + +Generate speech with explicit voice style instructions: + +```bash +python3 -u end2end.py \ + --stage-configs-path vllm_omni/model_executor/stage_configs/mimo_audio.yaml \ + --model-name XiaomiMiMo/MiMo-Audio-7B-Instruct \ + --query-type tts_sft_with_instruct \ + --text "The weather is so nice today." \ + --instruct "Speak happily in a child's voice" +``` + +### tts_sft_with_audio (TTS with Audio Reference) + +Generate speech using an audio reference for voice cloning: + +```bash +python3 -u end2end.py \ + --stage-configs-path vllm_omni/model_executor/stage_configs/mimo_audio.yaml \ + --model-name XiaomiMiMo/MiMo-Audio-7B-Instruct \ + --query-type tts_sft_with_audio \ + --text "The weather is so nice today." \ + --audio-path "./spoken_dialogue_assistant_turn_1.wav" +``` + +### tts_sft_with_natural_instruction (Natural Language TTS) + +Generate speech from text containing natural voice descriptions: + +```bash +python3 -u end2end.py \ + --stage-configs-path vllm_omni/model_executor/stage_configs/mimo_audio.yaml \ + --model-name XiaomiMiMo/MiMo-Audio-7B-Instruct \ + --query-type tts_sft_with_natural_instruction \ + --text "In a panting young male voice, he said: I can't run anymore, wait for me!" +``` + +### audio_trancribing_sft (Speech-to-Text) + +Transcribe audio to text: + +```bash +python3 -u end2end.py \ + --stage-configs-path vllm_omni/model_executor/stage_configs/mimo_audio.yaml \ + --model-name XiaomiMiMo/MiMo-Audio-7B-Instruct \ + --query-type audio_trancribing_sft \ + --audio-path "./spoken_dialogue_assistant_turn_1.wav" +``` + +### audio_understanding_sft (Audio Understanding) + +Understand and analyze audio content with text queries: + +```bash +python3 -u end2end.py \ + --stage-configs-path vllm_omni/model_executor/stage_configs/mimo_audio.yaml \ + --model-name XiaomiMiMo/MiMo-Audio-7B-Instruct \ + --query-type audio_understanding_sft \ + --text "Summarize the audio." \ + --audio-path "./spoken_dialogue_assistant_turn_1.wav" +``` + +### audio_understanding_sft_with_thinking (Audio Understanding with Reasoning) + +Audio understanding with reasoning chain: + +```bash +python3 -u end2end.py \ + --stage-configs-path vllm_omni/model_executor/stage_configs/mimo_audio.yaml \ + --model-name XiaomiMiMo/MiMo-Audio-7B-Instruct \ + --query-type audio_understanding_sft_with_thinking \ + --text "Summarize the audio." \ + --audio-path "./spoken_dialogue_assistant_turn_1.wav" +``` + +### spoken_dialogue_sft_multiturn (Multi-turn Spoken Dialogue) + +Multi-turn dialogue with audio input and output: + +```bash +python3 -u end2end.py \ + --stage-configs-path vllm_omni/model_executor/stage_configs/mimo_audio.yaml \ + --model-name XiaomiMiMo/MiMo-Audio-7B-Instruct \ + --query-type spoken_dialogue_sft_multiturn \ + --audio-path "./prompt_speech_zh_m.wav" +``` + +Note: This task uses hardcoded audio files in the script. The audio files used in examples are available at: https://github.com/XiaomiMiMo/MiMo-Audio/tree/main/examples + +### speech2text_dialogue_sft_multiturn (Speech-to-Text Dialogue) + +Multi-turn dialogue converting speech to text: + +```bash +python3 -u end2end.py \ + --stage-configs-path vllm_omni/model_executor/stage_configs/mimo_audio.yaml \ + --model-name XiaomiMiMo/MiMo-Audio-7B-Instruct \ + --query-type speech2text_dialogue_sft_multiturn +``` + +Note: This task uses hardcoded audio files and message lists in the script. + +### text_dialogue_sft_multiturn (Text Dialogue) + +Multi-turn text-only dialogue: + +```bash +python3 -u end2end.py \ + --stage-configs-path vllm_omni/model_executor/stage_configs/mimo_audio.yaml \ + --model-name XiaomiMiMo/MiMo-Audio-7B-Instruct \ + --query-type text_dialogue_sft_multiturn +``` + +Note: This task uses hardcoded message lists in the script. + +## Troubleshooting + +### Audio dependencies (soundfile, librosa) + +This example depends on **soundfile** (read/write WAV) and **librosa** (load audio including MP3). Install the project requirements first: + +```bash +pip install -r requirements/common.txt +# or at least: pip install soundfile>=0.13.1 librosa>=0.11.0 +``` + +- **`soundfile` / libsndfile not found** + `soundfile` uses the C library **libsndfile**. On Linux, install the system package before pip: + - Debian/Ubuntu: `sudo apt-get install libsndfile1` + - For development builds: `sudo apt-get install libsndfile1-dev` + - Then: `pip install soundfile` + +- **`librosa` fails to load MP3 or reports "No backend available"** + Loading MP3 (e.g. in `spoken_dialogue_sft_multiturn` with `.mp3` files) uses **ffmpeg** as the backend. Install ffmpeg: + - Debian/Ubuntu: `sudo apt-get install ffmpeg` + - macOS: `brew install ffmpeg` + +- **`ImportError: No module named 'soundfile'` or `ModuleNotFoundError: ... librosa`** + Ensure you are in the same Python environment where vLLM Omni and the example dependencies are installed, and that `requirements/common.txt` (or the packages above) are installed. + +### Tokenizer path + +- **`MIMO_AUDIO_TOKENIZER_PATH` not set or model fails to find tokenizer** + Export the tokenizer path before running: + ```bash + export MIMO_AUDIO_TOKENIZER_PATH="XiaomiMiMo/MiMo-Audio-Tokenizer" + ``` + See [Environment Variables](#environment-variables) in Setup. + +### Other + +- If the model or stage config fails to load, check [stage configuration documentation](https://docs.vllm.ai/projects/vllm-omni/en/latest/configuration/stage_configs/) for memory and GPU settings. +- For errors when reading/writing WAV (e.g. unsupported format), ensure input files are standard WAV/MP3 and that `soundfile` is linked to a working libsndfile (see above). + +## Notes + +- The script uses default model paths and audio files embedded in `end2end.py`. Update them if your local cache path differs. +- Use `--output-dir` to change the output folder (default: `./output_audio`). +- Use `--num-prompts` to generate multiple prompts in one run (default: 1). +- Audio files used in multi-turn dialogue examples are available at: https://github.com/XiaomiMiMo/MiMo-Audio/tree/main/examples +- The script supports various configuration options for initialization timeouts, batch timeouts, and shared memory thresholds. See `--help` for details. + +## Example materials + +??? abstract "end2end.py" + ``````py + --8<-- "examples/offline_inference/mimo_audio/end2end.py" + `````` +??? abstract "message_base64_wav.json" + ``````json + --8<-- "examples/offline_inference/mimo_audio/message_base64_wav.json" + `````` +??? abstract "message_convert.py" + ``````py + --8<-- "examples/offline_inference/mimo_audio/message_convert.py" + `````` +??? abstract "process_speechdata.py" + ``````py + --8<-- "examples/offline_inference/mimo_audio/process_speechdata.py" + `````` diff --git a/docs/user_guide/examples/offline_inference/qwen3_omni.md b/docs/user_guide/examples/offline_inference/qwen3_omni.md index 2e28f7dea..6577092bb 100644 --- a/docs/user_guide/examples/offline_inference/qwen3_omni.md +++ b/docs/user_guide/examples/offline_inference/qwen3_omni.md @@ -67,6 +67,51 @@ If media file paths are not provided, the script will use default assets. Suppor - `multi_audios`: Multiple audio inputs - `mixed_modalities`: Combination of video, image, and audio inputs +### Async-chunk (offline) + +For true stage-level concurrency -- where downstream stages (Talker, Code2Wav) +start **before** the upstream stage (Thinker) finishes -- use the async_chunk +example. This requires: + +1. A stage config YAML with ``async_chunk: true`` (e.g. + ``qwen3_omni_moe_async_chunk.yaml``). +2. Hardware that matches the config (e.g. 2x H100 for the default 3-stage + config). + +The async_chunk example uses ``AsyncOmni`` instead of the synchronous ``Omni`` +class, which enables the async orchestrator to receive stage-0 intermediate +outputs and trigger downstream stages early. Chunk data flows directly between +stage workers via the in-worker ``OmniChunkTransferAdapter`` / connector, +**not** through the orchestrator. + +#### Single prompt +```bash +cd examples/offline_inference/qwen3_omni +bash run_single_prompt_async_chunk.sh +``` + +#### Multiple prompts with concurrency control +```bash +bash run_multiple_prompts_async_chunk.sh --max-in-flight 4 +``` + +#### Text-only output (skip audio generation) +```bash +python end2end_async_chunk.py --query-type text --modalities text +``` + +#### Custom stage config +```bash +python end2end_async_chunk.py \ + --query-type use_audio \ + --stage-configs-path /path/to/your_async_chunk.yaml +``` + +> **Note**: The synchronous ``end2end.py`` (using ``Omni``) is still the +> recommended entry point for non-async-chunk workflows. Only use the +> async_chunk example when you need the stage-level concurrency semantics +> described in PR #962 / #1151. + ### FAQ If you encounter error about backend of librosa, try to install ffmpeg with command below. @@ -81,14 +126,26 @@ sudo apt install ffmpeg ``````py --8<-- "examples/offline_inference/qwen3_omni/end2end.py" `````` +??? abstract "end2end_async_chunk.py" + ``````py + --8<-- "examples/offline_inference/qwen3_omni/end2end_async_chunk.py" + `````` ??? abstract "run_multiple_prompts.sh" ``````sh --8<-- "examples/offline_inference/qwen3_omni/run_multiple_prompts.sh" `````` +??? abstract "run_multiple_prompts_async_chunk.sh" + ``````sh + --8<-- "examples/offline_inference/qwen3_omni/run_multiple_prompts_async_chunk.sh" + `````` ??? abstract "run_single_prompt.sh" ``````sh --8<-- "examples/offline_inference/qwen3_omni/run_single_prompt.sh" `````` +??? abstract "run_single_prompt_async_chunk.sh" + ``````sh + --8<-- "examples/offline_inference/qwen3_omni/run_single_prompt_async_chunk.sh" + `````` ??? abstract "run_single_prompt_tp.sh" ``````sh --8<-- "examples/offline_inference/qwen3_omni/run_single_prompt_tp.sh" diff --git a/docs/user_guide/examples/offline_inference/qwen3_tts.md b/docs/user_guide/examples/offline_inference/qwen3_tts.md index d04a42014..217822852 100644 --- a/docs/user_guide/examples/offline_inference/qwen3_tts.md +++ b/docs/user_guide/examples/offline_inference/qwen3_tts.md @@ -11,7 +11,7 @@ Qwen3 TTS provides multiple task variants for speech generation: - **CustomVoice**: Generate speech with a known speaker identity (speaker ID) and optional instruction. - **VoiceDesign**: Generate speech from text plus a descriptive instruction that designs a new voice. -- **Base**: Voice cloning using a reference audio + reference transcript, with optional mode selection. The `ref_audio` field accepts a local file path, HTTP URL, or base64 data URL. +- **Base**: Voice cloning using a reference audio + reference transcript, with optional mode selection. ## Setup Please refer to the [stage configuration documentation](https://docs.vllm.ai/projects/vllm-omni/en/latest/configuration/stage_configs/) to configure memory allocation appropriately for your hardware setup. diff --git a/docs/user_guide/examples/offline_inference/text_to_image.md b/docs/user_guide/examples/offline_inference/text_to_image.md index ce000cc8b..cea136b35 100644 --- a/docs/user_guide/examples/offline_inference/text_to_image.md +++ b/docs/user_guide/examples/offline_inference/text_to_image.md @@ -3,14 +3,50 @@ Source . -This folder provides several entrypoints for experimenting with `Qwen/Qwen-Image` `Qwen/Qwen-Image-2512` `Tongyi-MAI/Z-Image-Turbo` `stepfun-ai/NextStep-1.1` using vLLM-Omni, note that NextStep-1.1 has different architecture so we treat it differently regarding running arguments and pipeline. +Generate images from text prompts using vLLM-Omni's diffusion pipeline entrypoints. - `text_to_image.py`: command-line script for single image generation with advanced options. -- `web_demo.py`: lightweight Gradio UI for interactive prompt/seed/CFG exploration. +- `gradio_demo.py`: lightweight Gradio UI for interactive prompt/seed/CFG exploration. -Note that when you pass in multiple independent prompts, they will be processed sequentially. Batching requests is currently not supported. +## Table of Contents -## Basic Usage +- [Overview](#overview) +- [Quick Start](#quick-start) +- [Key Arguments](#key-arguments) +- [More CLI Examples](#more-cli-examples) +- [Web UI Demo](#web-ui-demo) + +## Overview + +This folder provides several entrypoints for experimenting with text-to-image diffusion models using vLLM-Omni. Note that `NextStep-1.1` has a different architecture, so it is treated differently regarding running arguments and pipeline. + +### Supported Models + +| Model | Image Shape | Peak VRAM (GiB) * | Model Weights (GiB) | +| ----- | ----------- | ----------- | ----------------- | +| `Qwen/Qwen-Image` | 1024 x 1024 | 60.0 | 53.7 | +| `Qwen/Qwen-Image-2512` |1024 x 1024 | 60.0 | 53.7 | +| `Tongyi-MAI/Z-Image-Turbo` | 1024 x 1024 | 24.8 | 19.2 | +| `stepfun-ai/NextStep-1.1` | 512 x 512 | 71.8 | 28.1 | +| `meituan-longcat/LongCat-Image` | 1024 x 1024 | 71.2 | 27.3 | +| `AIDC-AI/Ovis-Image-7B` | 1024 x 1024 | 71.8 | 17.1 | +| `OmniGen2/OmniGen2` | 1024 x 1024 | 20.1 | 14.7 | +| `stabilityai/stable-diffusion-3.5-medium` | 1024 x 1024 | 20.1 | 15.6 | +| `black-forest-labs/FLUX.1-dev` | 1024 x 1024 | 77.6 | 31.4 | +| `black-forest-labs/FLUX.2-klein-4B` | 1024 x 1024 | 72.7 | 14.9 | +| `black-forest-labs/FLUX.2-klein-9B` | 1024 x 1024 | 37.1 | 32.3 | +| `black-forest-labs/FLUX.2-dev` | 1024 x 1024 | 65.7 | >80 (CPU offload required) | + +!!! info +*Peak VRAM: based on basic single-card usage, batch size =1, without any acceleration/optimization features. FLUX.2-dev requires `--enable-cpu-offload` on a single 80 GiB GPU. + +Default model: `Qwen/Qwen-Image` + +## Quick Start + +### Python API + +Single-prompt generation: ```python from vllm_omni.entrypoints.omni import Omni @@ -23,78 +59,74 @@ if __name__ == "__main__": images[0].save("coffee.png") ``` -Or put more than one prompt in a request. +### Local CLI Usage -```python -from vllm_omni.entrypoints.omni import Omni - -if __name__ == "__main__": - omni = Omni(model="Qwen/Qwen-Image") - prompts = [ - "a cup of coffee on a table", - "a toy dinosaur on a sandy beach", - "a fox waking up in bed and yawning", - ] - outputs = omni.generate(prompts) - for i, output in enumerate(outputs): - image = output.request_output.images[0].save(f"{i}.jpg") +```bash +python text_to_image.py \ + --model Qwen/Qwen-Image \ + --prompt "a cup of coffee on the table" \ + --output coffee.png ``` -!!! info +## Key Arguments - However, it is not currently recommended to do so - because not all models support batch inference, - and batch requesting mostly does not provide significant performance improvement (despite the impression that it does). - This feature is primarily for the sake of interface compatibility with vLLM and to allow for future improvements. +**Common arguments:** -!!! info +| Argument | Type | Default | Description | +| -------- | ---- | ------- | ----------- | +| `--prompt` | str | `"a cup of coffee on the table"` | Text description for image generation | +| `--seed` | int | `142` | Integer seed for deterministic sampling | +| `--negative-prompt` | str | `None` | Negative prompt for classifier-free conditional guidance | +| `--cfg-scale` | float | `4.0` | True CFG scale (model-specific guidance strength) | +| `--guidance-scale` | float | `1.0` | Classifier-free guidance scale | +| `--num-images-per-prompt` | int | `1` | Number of images per prompt (saved as `output`, `output_1`, ...) | +| `--num-inference-steps` | int | `50` | Diffusion sampling steps (more steps = higher quality, slower) | +| `--height` | int | `1024` | Output image height in pixels | +| `--width` | int | `1024` | Output image width in pixels | +| `--output` | str | `"qwen_image_output.png"` | Path to save the generated image | +| `--vae-use-slicing` | flag | off | Enable VAE slicing for memory optimization | +| `--vae-use-tiling` | flag | off | Enable VAE tiling for memory optimization | +| `--cfg-parallel-size` | int | `1` | Set to `2` to enable CFG Parallel | +| `--enable-cpu-offload` | flag | off | Enable CPU offloading for diffusion models | +| `--lora-path` | str | — | Path to PEFT LoRA adapter folder | +| `--lora-scale` | float | `1.0` | Scale factor for LoRA weights | - For diffusion pipelines, the stage config field `stage_args.[].engine_args.max_num_seqs` is 1 by default, and the input - list is sliced into single-item requests before feeding into the diffusion pipeline. For models that do internally support - batched inputs, you can [modify this configuration](https://github.com/vllm-project/vllm-omni/tree/main/configuration/stage_configs.md) to let the model accept a longer batch of prompts. +**NextStep-1.1 specific arguments:** -Apart from string prompt, vLLM-Omni also supports dictionary prompts in the same style as vLLM. -This is useful for models that support negative prompts. +| Argument | Type | Default | Description | +| -------- | ---- | ------- | ----------- | +| `--guidance-scale-2` | float | `1.0` | Secondary guidance scale (e.g. image-level CFG) | +| `--timesteps-shift` | float | `1.0` | Timesteps shift parameter for sampling | +| `--cfg-schedule` | str | `"constant"` | CFG schedule type: `"constant"` or `"linear"` | +| `--use-norm` | flag | off | Apply layer normalization to sampled tokens | -```python -from vllm_omni.entrypoints.omni import Omni +> If you encounter OOM errors, try using `--vae-use-slicing` and `--vae-use-tiling` to reduce memory usage. -if __name__ == "__main__": - omni = Omni(model="Qwen/Qwen-Image") - outputs = omni.generate([ - { - "prompt": "a cup of coffee on a table"， - "negative_prompt": "low resolution" - }, - { - "prompt": "a toy dinosaur on a sandy beach"， - "negative_prompt": "cinematic, realistic" - } - ]) - for i, output in enumerate(outputs): - image = output.request_output.images[0].save(f"{i}.jpg") -``` +> Qwen-Image currently publishes best-effort presets at `1328x1328`, `1664x928`, `928x1664`, `1472x1140`, `1140x1472`, `1584x1056`, and `1056x1584`. Adjust `--height/--width` accordingly for the most reliable outcomes. -## Local CLI Usage +## More CLI Examples -### Qwen/Tongyi Models +### Tongyi Models ```bash python text_to_image.py \ --model Tongyi-MAI/Z-Image-Turbo \ --prompt "a cup of coffee on the table" \ --seed 42 \ - --cfg-scale 4.0 \ + --guidance-scale 0.0 \ --num-images-per-prompt 1 \ - --num-inference-steps 50 \ + --num-inference-steps 9 \ --height 1024 \ --width 1024 \ --output outputs/coffee.png ``` +`Tongyi-MAI/Z-Image-Turbo` is a distilled version of Z-Image. Distilled diffusion models usually require less number of inference steps (4~9), and Classifier-Free Guidance (CFG) is usually NOT applied. Similar distilled models are `black-forest-labs/FLUX.2-klein-4B` and `black-forest-labs/FLUX.2-klein-9B`. + ### NextStep Models -NextStep-1.1 can have extra arguments +NextStep-1.1 supports extra arguments for dual-level CFG control: + ```bash python text_to_image.py \ --model stepfun-ai/NextStep-1.1 \ @@ -109,38 +141,101 @@ python text_to_image.py \ --seed 42 ``` -### Key Arguments +### FLUX.2-dev Models -**Common arguments:** +To run FLUX.2-dev on a single GPU, `--enable-cpu-offload` is required because the model weights exceed 80 GiB: -- `--prompt`: text description (string). -- `--seed`: integer seed for deterministic sampling. -- `--cfg-scale`: true CFG scale (model-specific guidance strength). -- `--num-images-per-prompt`: number of images to generate per prompt (saves as `output`, `output_1`, ...). -- `--num-inference-steps`: diffusion sampling steps (more steps = higher quality, slower). -- `--height/--width`: output resolution (defaults 1024x1024). -- `--output`: path to save the generated PNG. -- `--vae-use-slicing`: enable VAE slicing for memory optimization. -- `--vae-use-tiling`: enable VAE tiling for memory optimization. -- `--cfg-parallel-size`: set it to 2 to enable CFG Parallel. See more examples in [`user_guide`](https://github.com/vllm-project/vllm-omni/tree/main/docs/user_guide/diffusion_acceleration.md#using-cfg-parallel). -- `--enable-cpu-offload`: enable CPU offloading for diffusion models. -- `--guidance-scale`: classifier-free guidance scale. +```bash +python examples/offline_inference/text_to_image/text_to_image.py \ + --model black-forest-labs/FLUX.2-dev \ + --prompt "a lovely bunny holding a sign that says 'vllm-omni'" \ + --seed 42 \ + --tensor-parallel-size 1 \ + --num-images-per-prompt 1 \ + --num-inference-steps 50 \ + --guidance-scale 4.0 \ + --height 1024 \ + --width 1024 \ + --enable-cpu-offload \ + --output flux2-dev.png +``` -**NextStep-1.1 specific:** -- `--guidance-scale-2`: secondary guidance scale, e.g. image-level CFG (default: 1.0). -- `--timesteps-shift`: timesteps shift parameter for sampling (default: 1.0). -- `--cfg-schedule`: CFG schedule type, "constant" or "linear" (default: "constant"). -- `--use-norm`: apply layer normalization to sampled tokens. +### Batch Requests (Multiple Prompts) -> ℹ️ If you encounter OOM errors, try using `--vae-use-slicing` and `--vae-use-tiling` to reduce memory usage. +You can pass multiple prompts in a single `generate` call. -> ℹ️ Qwen-Image currently publishes best-effort presets at `1328x1328`, `1664x928`, `928x1664`, `1472x1140`, `1140x1472`, `1584x1056`, and `1056x1584`. Adjust `--height/--width` accordingly for the most reliable outcomes. +```python +from vllm_omni.entrypoints.omni import Omni -## LoRA +if __name__ == "__main__": + omni = Omni(model="Qwen/Qwen-Image") + prompts = [ + "a cup of coffee on a table", + "a toy dinosaur on a sandy beach", + "a fox waking up in bed and yawning", + ] + outputs = omni.generate(prompts) + for i, output in enumerate(outputs): + output.request_output.images[0].save(f"{i}.jpg") +``` -This example supports Peft-compatible LoRA (Low-Rank Adaptation) adapters for diffusion models. Pass `--lora-path` to use a LoRA adapter and optionally `--lora-scale` (default 1.0); omit it to use the base model only. +!!! info -### Basic usage with LoRA + Not all models support batch inference, and batch requesting mostly does not provide significant + performance improvement. This feature is primarily for interface compatibility with vLLM and to + allow for future improvements. + +!!! info + + For diffusion pipelines, the stage config field `stage_args.[].runtime.max_batch_size` is 1 by + default, and the input list is sliced into single-item requests before feeding into the diffusion + pipeline. For models that do internally support batched inputs, you can + [modify this configuration](https://github.com/vllm-project/vllm-omni/tree/main/configuration/stage_configs.md) to let the model accept a + longer batch of prompts. + +### Negative Prompts + +vLLM-Omni supports dictionary prompts for models that accept negative prompts: + +```python +from vllm_omni.entrypoints.omni import Omni + +if __name__ == "__main__": + omni = Omni(model="Qwen/Qwen-Image") + outputs = omni.generate([ + { + "prompt": "a cup of coffee on a table", + "negative_prompt": "low resolution" + }, + { + "prompt": "a toy dinosaur on a sandy beach", + "negative_prompt": "cinematic, realistic" + } + ]) + for i, output in enumerate(outputs): + output.request_output.images[0].save(f"{i}.jpg") +``` + +You can also pass a negative prompt via the CLI argument `--negative-prompt`: + +```bash +python examples/offline_inference/text_to_image/text_to_image.py \ + --model Qwen/Qwen-Image \ + --prompt "a cup of coffee on a table" \ + --negative-prompt "low resolution, blurry" \ + --output coffee.png +``` + +### Advanced Features + +#### CFG Parallel + +Set `--cfg-parallel-size 2` to enable CFG Parallel for faster inference on multi-GPU setups. +See more examples in the [diffusion acceleration user guide](https://github.com/vllm-project/vllm-omni/tree/main/docs/user_guide/diffusion_acceleration.md#using-cfg-parallel). + +#### LoRA + +This example supports PEFT-compatible LoRA (Low-Rank Adaptation) adapters for diffusion models. Pass `--lora-path` to use a LoRA adapter and optionally `--lora-scale` (default `1.0`); omit it to use the base model only. ```bash python text_to_image.py \ @@ -151,14 +246,7 @@ python text_to_image.py \ --output output.png ``` -### LoRA parameters - -- `--lora-path`: Path to LoRA adapter folder (PEFT format). Loaded at initialization and used for generation. -- `--lora-scale`: Scale factor for LoRA weights (default: 1.0). Higher values increase the influence of the LoRA adapter. - -### LoRA adapter format - -LoRA adapters must be in PEFT (Parameter-Efficient Fine-Tuning) format. A typical LoRA adapter directory structure: +LoRA adapters must be in PEFT format. A typical adapter directory structure: ``` lora_adapter/ @@ -168,13 +256,13 @@ lora_adapter/ ## Web UI Demo -Launch the gradio demo: +Launch the Gradio demo: ```bash python gradio_demo.py --port 7862 ``` -Then open `http://localhost:7862/` on your local browser to interact with the web UI. +Then open `http://localhost:7862/` in your local browser to interact with the web UI. ## Example materials diff --git a/docs/user_guide/examples/offline_inference/text_to_video.md b/docs/user_guide/examples/offline_inference/text_to_video.md index 70bdfb3cc..4288c089c 100644 --- a/docs/user_guide/examples/offline_inference/text_to_video.md +++ b/docs/user_guide/examples/offline_inference/text_to_video.md @@ -3,13 +3,20 @@ Source . -The Wan T2V pipeline generates short videos from text prompts. Supported models include: +A unified script for text-to-video generation. Supports multiple models with model-aware defaults. -- `Wan-AI/Wan2.2-T2V-A14B-Diffusers` (14B, requires ~48GB VRAM) -- `Wan-AI/Wan2.1-T2V-1.3B-Diffusers` (1.3B, requires ~16GB VRAM, use `--flow-shift 3.0 --boundary-ratio 0.0`) +## Supported Models + +| Model | Default Resolution | Default Frames | Default Steps | Guidance | VRAM (BF16) | +|---|---|---|---|---|---| +| `Wan-AI/Wan2.2-T2V-A14B-Diffusers` | 720x1280 | 81 | 40 | 4.0 | ~60 GiB | +| `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v` | 480x832 | 121 | 50 | 6.0 | 1×A100 80GB | +| `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-720p_t2v` | 720x1280 | 121 | 50 | 6.0 | FP8 + VAE tiling required | ## Local CLI Usage +### Wan2.2 (default) + ```bash python text_to_video.py \ --prompt "Two anthropomorphic cats in comfy boxing gear and bright gloves fight intensely on a spotlighted stage." \ @@ -25,22 +32,119 @@ python text_to_video.py \ --output t2v_out.mp4 ``` -Key arguments: +LTX2 example: +```bash +python text_to_video.py \ + --model "Lightricks/LTX-2" \ + --prompt "A cinematic close-up of ocean waves at golden hour." \ + --negative-prompt "worst quality, inconsistent motion, blurry, jittery, distorted" \ + --height 512 \ + --width 768 \ + --num-frames 121 \ + --num-inference-steps 40 \ + --guidance-scale 4.0 \ + --frame-rate 24 \ + --output ltx2_out.mp4 +``` + +### HunyuanVideo-1.5 (480p) + +```bash +python text_to_video.py \ + --model hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v \ + --prompt "A cat walks through a sunlit garden, flowers swaying gently in the breeze." \ + --height 480 \ + --width 832 \ + --num-frames 121 \ + --guidance-scale 6.0 \ + --flow-shift 5.0 \ + --num-inference-steps 50 \ + --fps 24 \ + --output hunyuan_video_15_output.mp4 +``` + +### HunyuanVideo-1.5 (720p) + +```bash +python text_to_video.py \ + --model hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-720p_t2v \ + --prompt "A serene lakeside sunrise with mist over the water." \ + --height 720 \ + --width 1280 \ + --num-frames 121 \ + --guidance-scale 6.0 \ + --flow-shift 9.0 \ + --num-inference-steps 50 \ + --fps 24 \ + --output hunyuan_720p.mp4 +``` + +### HunyuanVideo-1.5 with FP8 Quantization + +```bash +python text_to_video.py \ + --model hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v \ + --prompt "A dog running across a field of golden wheat." \ + --quantization fp8 \ + --height 480 --width 832 --num-frames 121 \ + --guidance-scale 6.0 --flow-shift 5.0 \ + --output hunyuan_fp8.mp4 +``` + +Quick test (smaller resolution, fewer frames): + +```bash +python text_to_video.py \ + --model hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v \ + --prompt "A serene lakeside sunrise with mist over the water." \ + --height 320 --width 576 --num-frames 17 --num-inference-steps 30 \ + --flow-shift 5.0 \ + --output quick_test.mp4 +``` + +## Key Arguments + +### Common + +- `--model`: Diffusers model ID or local path. - `--prompt`: text description (string). -- `--height/--width`: output resolution (defaults 480x832, i.e. 480P). Dimensions should align with Wan VAE downsampling (multiples of 8). -- `--num-frames`: Number of frames (Wan default is 81). -- `--guidance-scale` and `--guidance-scale-high`: CFG scale (applied to low/high). -- `--negative-prompt`: optional list of artifacts to suppress (the PR demo used a long Chinese string). -- `--boundary-ratio`: Boundary split ratio for low/high DiT. Default `0.875` uses both transformers for best quality. Set to `1.0` to load only the low-noise transformer (saves noticeable memory with good quality, recommended if memory is limited). Set to `0.0` loads only the high-noise transformer (not recommended, lower quality). -- `--fps`: frames per second for the saved MP4 (requires `diffusers` export_to_video). +- `--height/--width`: output resolution. Default depends on model. +- `--num-frames`: number of frames. Default depends on model. +- `--guidance-scale`: CFG scale. Default depends on model. +- `--num-inference-steps`: sampling steps. Default depends on model. +- `--fps`: frames per second for the saved MP4. - `--output`: path to save the generated video. - `--vae-use-slicing`: enable VAE slicing for memory optimization. - `--vae-use-tiling`: enable VAE tiling for memory optimization. - `--cfg-parallel-size`: set it to 2 to enable CFG Parallel. See more examples in [`user_guide`](https://github.com/vllm-project/vllm-omni/tree/main/docs/user_guide/diffusion/parallelism_acceleration.md#cfg-parallel). +- `--tensor-parallel-size`: tensor parallel size (effective for models that support TP, e.g. LTX2). - `--enable-cpu-offload`: enable CPU offloading for diffusion models. +- `--enable-layerwise-offload`: enable layerwise offloading on DiT modules. +- `--frame-rate`: generation FPS for pipelines that require it (e.g., LTX2). +- `--audio-sample-rate`: audio sample rate for embedded audio (when the pipeline returns audio). +- `--quantization`: quantization method (`fp8` for FP8, `gguf` for GGUF). +- `--flow-shift`: scheduler flow_shift parameter. -> ℹ️ If you encounter OOM errors, try using `--vae-use-slicing` and `--vae-use-tiling` to reduce memory usage. +### Wan2.2-specific + +- `--negative-prompt`: artifacts to suppress. +- `--guidance-scale-high`: separate CFG scale for high-noise stage. +- `--boundary-ratio`: boundary split for low/high DiT (default 0.875). +- `--flow-shift`: scheduler flow_shift (5.0 for 720p, 12.0 for 480p). +- `--cache-backend`: `cache_dit` for acceleration. + +### HunyuanVideo-1.5 Optimal Configs + +| Variant | flow_shift | guidance_scale | steps | +|---------|-----------|----------------|-------| +| 480p T2V | 5.0 | 6.0 | 50 | +| 720p T2V | 9.0 | 6.0 | 50 | +| 480p I2V | 5.0 | 6.0 | 50 | +| 720p I2V | 7.0 | 6.0 | 50 | +| CFG-distilled | (same) | 1.0 | 50 | + +> If you encounter OOM errors, try `--vae-use-slicing`, `--vae-use-tiling`, `--enable-cpu-offload`, or `--quantization fp8`. ## Example materials diff --git a/docs/user_guide/examples/offline_inference/voxtral_tts.md b/docs/user_guide/examples/offline_inference/voxtral_tts.md new file mode 100644 index 000000000..db4c2e26b --- /dev/null +++ b/docs/user_guide/examples/offline_inference/voxtral_tts.md @@ -0,0 +1,74 @@ +# Voxtral TTS Offline Inference + +Source . + + +`end2end.py` runs Voxtral TTS end-to-end offline inference using vLLM. It supports both blocking (`Omni`) and streaming (`AsyncOmni`) generation, batched prompts with configurable concurrency, and voice selection via preset name or reference audio file. + +When `mistral_common` has `SpeechRequest` support, prompt token IDs are built via `encode_speech_request`. Otherwise, the script falls back to manual token construction. + +## Usage Examples + + +```bash +# Basic single-prompt with cheerful_female voice preset +python3 examples/offline_inference/voxtral_tts/end2end.py \ + --stage-configs-path vllm_omni/model_executor/stage_configs/voxtral_tts.yaml \ + --write-audio --voice cheerful_female \ + --model mistralai/Voxtral-4B-TTS-2603 \ + --text "That eerie silence after the first storm was just the calm before another round of chaos, wasn't it?" + +# 32 replicate prompts with cheerful_female voice preset +python3 examples/offline_inference/voxtral_tts/end2end.py \ + --stage-configs-path vllm_omni/model_executor/stage_configs/voxtral_tts.yaml \ + --num-prompts 32 --write-audio --voice cheerful_female \ + --model mistralai/Voxtral-4B-TTS-2603 \ + --text "That eerie silence after the first storm was just the calm before another round of chaos, wasn't it?" + +# Streaming with neutral_female voice preset +python3 examples/offline_inference/voxtral_tts/end2end.py \ + --stage-configs-path vllm_omni/model_executor/stage_configs/voxtral_tts.yaml \ + --streaming --write-audio --voice neutral_female \ + --model mistralai/Voxtral-4B-TTS-2603 \ + --text "That eerie silence after the first storm was just the calm before another round of chaos, wasn't it?" + +# 32 prompts, 8 concurrent requests per wave, streaming with neutral_female voice +python3 examples/offline_inference/voxtral_tts/end2end.py \ + --stage-configs-path vllm_omni/model_executor/stage_configs/voxtral_tts.yaml \ + --num-prompts 32 --concurrency 8 --streaming --write-audio --voice neutral_female \ + --model mistralai/Voxtral-4B-TTS-2603 \ + --text "That eerie silence after the first storm was just the calm before another round of chaos, wasn't it?" + +# Short debug prompt with reference audio +# Note: Reference audio capability is not yet released. +python3 examples/offline_inference/voxtral_tts/end2end.py \ + --stage-configs-path vllm_omni/model_executor/stage_configs/voxtral_tts.yaml \ + --write-audio \ + --model mistralai/Voxtral-4B-TTS-2603 \ + --text "This is a test message." \ + --audio-path path/to/reference_audio.wav +``` + +## Arguments + +| Argument | Description | +|---|---| +| `--model PATH` | HuggingFace repo ID or local directory path (default: `mistralai/Voxtral-4B-TTS-2603`) | +| `--text TEXT` | Text to synthesize (default: `"This is a test message."`) | +| `--audio-path PATH` | Path to reference audio file for voice cloning | +| `--output-dir DIR` | Directory to write output WAV files (default: `output_audio`) | +| `--stage-configs-path PATH` | Path to stage configs YAML (currently it must be set for VoxtralTTS) | +| `--num-prompts N` | Number of replicate prompts to run for measuring performance (default: 1) | +| `--streaming` | Use streaming generation via `AsyncOmni` (default: blocking `Omni`) | +| `--concurrency N` | Max concurrent requests per wave (must be used with `--streaming`, must evenly divide `--num-prompts`) | +| `--voice NAME` | Voice preset to use instead of reference audio file (e.g., casual_female, casual_male, cheerful_female, neutral_female, neutral_male) | +| `--write-audio` | Write generated audio to WAV files | +| `--profiling-mode` | Enable profiling mode (reduces max tokens to 50) | +| `--log-stats` | Enable detailed statistics logging | + +## Example materials + +??? abstract "end2end.py" + ``````py + --8<-- "examples/offline_inference/voxtral_tts/end2end.py" + `````` diff --git a/docs/user_guide/examples/offline_inference/x_to_video_audio.md b/docs/user_guide/examples/offline_inference/x_to_video_audio.md index 59b993a72..8ea39d811 100644 --- a/docs/user_guide/examples/offline_inference/x_to_video_audio.md +++ b/docs/user_guide/examples/offline_inference/x_to_video_audio.md @@ -1,5 +1,8 @@ # X-To-Video-Audio +Source . + + The `DreamID-Omni` pipeline generates short videos from text, image and video. ## Local CLI Usage @@ -56,3 +59,14 @@ Key arguments: - `--video-negative-prompt`: negative prompt for video generation. - `--audio-negative-prompt`: negative prompt for audio generation. - `--enable-cpu-offload`: enable CPU offload (defaults False). + +## Example materials + +??? abstract "download_dreamid_omni.py" + ``````py + --8<-- "examples/offline_inference/x_to_video_audio/download_dreamid_omni.py" + `````` +??? abstract "x_to_video_audio.py" + ``````py + --8<-- "examples/offline_inference/x_to_video_audio/x_to_video_audio.py" + `````` diff --git a/docs/user_guide/examples/online_serving/chart-helm.md b/docs/user_guide/examples/online_serving/chart-helm.md new file mode 100644 index 000000000..9b077a747 --- /dev/null +++ b/docs/user_guide/examples/online_serving/chart-helm.md @@ -0,0 +1,226 @@ +# vLLM-Omni Helm Chart + +Source . + + +Helm chart for deploying [vLLM-Omni](https://github.com/vllm-project/vllm-omni) on Kubernetes. vLLM-Omni extends vLLM with omni-modality model serving, supporting text-to-image, multimodal chat, text-to-speech, and more. + +## Prerequisites + +- Kubernetes 1.24+ +- Helm 3.x +- NVIDIA GPU nodes with [NVIDIA Device Plugin](https://github.com/NVIDIA/k8s-device-plugin) + +## Quick Start + +```bash +helm install my-release ./chart-helm \ + --set model=Tongyi-MAI/Z-Image-Turbo +``` + +## Configuration + +### Model Selection + +Set the `model` value to any supported HuggingFace model ID: + +| Model | Type | GPUs | Notes | +|-------|------|------|-------| +| `Tongyi-MAI/Z-Image-Turbo` | text-to-image | 1 | Small, fast (default) | +| `stabilityai/stable-diffusion-3.5-medium` | text-to-image | 1 | ~6GB VRAM | +| `Qwen/Qwen-Image` | text-to-image | 1 | Large, ~40GB+ VRAM | +| `Qwen/Qwen2.5-Omni-7B` | multimodal | 2 | Text + audio + image + video | +| `Qwen/Qwen3-Omni-7B-Chat` | multimodal | 2 | Latest omni model | +| `Qwen/Qwen3-TTS` | text-to-speech | 1 | TTS | + +### HuggingFace Token + +For gated models that require authentication: + +```bash +helm install my-release ./chart-helm \ + --set model=Qwen/Qwen2.5-Omni-7B \ + --set hfToken=hf_xxxxx \ + --set resources.requests."nvidia\.com/gpu"=2 \ + --set resources.limits."nvidia\.com/gpu"=2 +``` + +### Omni-Specific Flags + +Enable VAE memory optimizations for diffusion models: + +```bash +helm install my-release ./chart-helm \ + --set model=Qwen/Qwen-Image \ + --set omniArgs.vaeUseSlicing=true \ + --set omniArgs.vaeUseTiling=true +``` + +Enable CPU offloading: + +```bash +helm install my-release ./chart-helm \ + --set model=Qwen/Qwen-Image \ + --set omniArgs.enableCpuOffload=true +``` + +Pass additional raw CLI flags: + +```bash +helm install my-release ./chart-helm \ + --set model=Qwen/Qwen-Image \ + --set omniArgs.extraArgs[0]="--enable-layerwise-offload" +``` + +### Model Cache + +By default, a PersistentVolumeClaim is created for the HuggingFace model cache to avoid re-downloading models on pod restarts: + +```yaml +modelCache: + enabled: true + storageSize: "50Gi" + storageClassName: "" +``` + +To use an ephemeral volume instead: + +```bash +helm install my-release ./chart-helm \ + --set modelCache.enabled=false +``` + +### Custom Command Override + +To fully override the container command: + +```bash +helm install my-release ./chart-helm \ + --set image.command[0]=vllm \ + --set image.command[1]=serve \ + --set image.command[2]=my-model \ + --set image.command[3]=--omni \ + --set image.command[4]=--host \ + --set image.command[5]=0.0.0.0 +``` + +## API Endpoints + +Once deployed, vLLM-Omni exposes the following OpenAI-compatible endpoints: + +| Endpoint | Method | Description | +|----------|--------|-------------| +| `/health` | GET | Health check | +| `/v1/models` | GET | List available models | +| `/v1/chat/completions` | POST | Chat completions (text/multimodal) | +| `/v1/images/generations` | POST | Image generation | +| `/v1/images/edits` | POST | Image editing | +| `/v1/audio/speech` | POST | Text-to-speech | + +## Files + +| File | Description | +|------|-------------| +| `Chart.yaml` | Chart metadata (name, version, maintainers) | +| `values.yaml` | Default configuration values | +| `values.schema.json` | JSON schema for validating values | +| `templates/_helpers.tpl` | Helper templates for common configurations | +| `templates/deployment.yaml` | Kubernetes Deployment | +| `templates/service.yaml` | Kubernetes Service (ClusterIP) | +| `templates/secrets.yaml` | Secrets (generic + HuggingFace token) | +| `templates/pvc.yaml` | PersistentVolumeClaim for model cache | +| `templates/configmap.yaml` | Optional ConfigMap | +| `templates/hpa.yaml` | HorizontalPodAutoscaler | +| `templates/poddisruptionbudget.yaml` | PodDisruptionBudget | +| `templates/custom-objects.yaml` | Custom Kubernetes objects | + +## Running Tests + +This chart includes unit tests using [helm-unittest](https://github.com/helm-unittest/helm-unittest). Install the plugin and run tests: + +```bash +# Install plugin +helm plugin install https://github.com/helm-unittest/helm-unittest + +# Run tests +helm unittest . +``` + +## Example materials + +??? abstract ".helmignore" + `````` + --8<-- "examples/online_serving/chart-helm/.helmignore" + `````` +??? abstract "Chart.yaml" + ``````yaml + --8<-- "examples/online_serving/chart-helm/Chart.yaml" + `````` +??? abstract "ct.yaml" + ``````yaml + --8<-- "examples/online_serving/chart-helm/ct.yaml" + `````` +??? abstract "lintconf.yaml" + ``````yaml + --8<-- "examples/online_serving/chart-helm/lintconf.yaml" + `````` +??? abstract "templates/_helpers.tpl" + ``````tpl + --8<-- "examples/online_serving/chart-helm/templates/_helpers.tpl" + `````` +??? abstract "templates/configmap.yaml" + ``````yaml + --8<-- "examples/online_serving/chart-helm/templates/configmap.yaml" + `````` +??? abstract "templates/custom-objects.yaml" + ``````yaml + --8<-- "examples/online_serving/chart-helm/templates/custom-objects.yaml" + `````` +??? abstract "templates/deployment.yaml" + ``````yaml + --8<-- "examples/online_serving/chart-helm/templates/deployment.yaml" + `````` +??? abstract "templates/hpa.yaml" + ``````yaml + --8<-- "examples/online_serving/chart-helm/templates/hpa.yaml" + `````` +??? abstract "templates/ingress.yaml" + ``````yaml + --8<-- "examples/online_serving/chart-helm/templates/ingress.yaml" + `````` +??? abstract "templates/poddisruptionbudget.yaml" + ``````yaml + --8<-- "examples/online_serving/chart-helm/templates/poddisruptionbudget.yaml" + `````` +??? abstract "templates/pvc.yaml" + ``````yaml + --8<-- "examples/online_serving/chart-helm/templates/pvc.yaml" + `````` +??? abstract "templates/secrets.yaml" + ``````yaml + --8<-- "examples/online_serving/chart-helm/templates/secrets.yaml" + `````` +??? abstract "templates/service.yaml" + ``````yaml + --8<-- "examples/online_serving/chart-helm/templates/service.yaml" + `````` +??? abstract "tests/deployment_test.yaml" + ``````yaml + --8<-- "examples/online_serving/chart-helm/tests/deployment_test.yaml" + `````` +??? abstract "tests/ingress_test.yaml" + ``````yaml + --8<-- "examples/online_serving/chart-helm/tests/ingress_test.yaml" + `````` +??? abstract "tests/pvc_test.yaml" + ``````yaml + --8<-- "examples/online_serving/chart-helm/tests/pvc_test.yaml" + `````` +??? abstract "tests/secrets_test.yaml" + ``````yaml + --8<-- "examples/online_serving/chart-helm/tests/secrets_test.yaml" + `````` +??? abstract "values.yaml" + ``````yaml + --8<-- "examples/online_serving/chart-helm/values.yaml" + `````` diff --git a/docs/user_guide/examples/online_serving/image_to_video.md b/docs/user_guide/examples/online_serving/image_to_video.md index a2b30320b..912450acf 100644 --- a/docs/user_guide/examples/online_serving/image_to_video.md +++ b/docs/user_guide/examples/online_serving/image_to_video.md @@ -144,7 +144,7 @@ curl -X POST http://localhost:8091/v1/videos \ ## Create Response Format -`POST /v1/videos` returns a job record. +`POST /v1/videos` returns a job record, not inline base64 video data. ```json { @@ -201,6 +201,10 @@ done ## Example materials +??? abstract "run_curl_hunyuan_video_15.sh" + ``````sh + --8<-- "examples/online_serving/image_to_video/run_curl_hunyuan_video_15.sh" + `````` ??? abstract "run_curl_image_to_video.sh" ``````sh --8<-- "examples/online_serving/image_to_video/run_curl_image_to_video.sh" @@ -209,3 +213,7 @@ done ``````sh --8<-- "examples/online_serving/image_to_video/run_server.sh" `````` +??? abstract "run_server_hunyuan_video_15.sh" + ``````sh + --8<-- "examples/online_serving/image_to_video/run_server_hunyuan_video_15.sh" + `````` diff --git a/docs/user_guide/examples/online_serving/mimo_audio.md b/docs/user_guide/examples/online_serving/mimo_audio.md new file mode 100644 index 000000000..4737eca36 --- /dev/null +++ b/docs/user_guide/examples/online_serving/mimo_audio.md @@ -0,0 +1,72 @@ +# Online serving Example of vLLM-Omni for MiMo-Audio + +Source . + + +## 🛠️ Installation + +Please refer to [README.md](https://github.com/vllm-project/vllm-omni/tree/main/README.md) + +## Run examples (MiMo-Audio) + +### Launch the Server +```bash +export MIMO_AUDIO_TOKENIZER_PATH="XiaomiMiMo/MiMo-Audio-Tokenizer" + +vllm-omni serve XiaomiMiMo/MiMo-Audio-7B-Instruct --omni \ +--served-model-name "MiMo-Audio-7B-Instruct" \ +--port 18091 --stage-configs-path ./vllm_omni/model_executor/stage_configs/mimo_audio.yaml \ +--chat-template ./examples/online_serving/mimo_audio/chat_template.jinja +``` +> ⚠️ **Important** +> **MiMo-Audio is not compatible with the default chat template.** +> The provided `chat_template.jinja` implements MiMo-specific role, audio token, and instruction formatting and **must be used for all inference**. + + +### Send Multi-modal Request + +Get into the example folder +```bash +cd examples/online_serving/mimo_audio +``` + +#### Send request via python + +```bash +# Audio dialogue task +python openai_chat_completion_client_for_multimodal_generation.py \ +--query-type multi_audios \ +--message-json ../../offline_inference/mimo_audio/message_base64_wav.json +``` + +The Python client supports the following command-line arguments: + +- `--query-type` (or `-q`): Query type (default: `multi_audios`) + - Options: `multi_audios`, `text` +- `--message-json` (or `-m`): Path to `base64` multi rounds audio messages json file + - Do not pass any value for "text" query type + - Supports local file paths (automatically encoded to base64) or HTTP/HTTPS URLs, only for "Are these two audio clips the same?" task + - Example: `---message-json ./examples/offline_inference/mimo_audio/message_base64_wav.json` +- `--prompt` (or `-p`): Custom text prompt/question, only for query type is "text"(TTS task) + - Attention! Do not pass any value for "multi_audios" query type + - Example: `--prompt "What are the main activities shown in this video?"` + + +For example, to use multi rounds audios with local files: + +```bash +python openai_chat_completion_client_for_multimodal_generation.py \ +--query-type multi_audios \ +--message-json ../../offline_inference/mimo_audio/message_base64_wav.json +``` + +## Example materials + +??? abstract "chat_template.jinja" + ``````jinja + --8<-- "examples/online_serving/mimo_audio/chat_template.jinja" + `````` +??? abstract "openai_chat_completion_client_for_multimodal_generation.py" + ``````py + --8<-- "examples/online_serving/mimo_audio/openai_chat_completion_client_for_multimodal_generation.py" + `````` diff --git a/docs/user_guide/examples/online_serving/qwen3_tts.md b/docs/user_guide/examples/online_serving/qwen3_tts.md index 993505b87..faf2fe781 100644 --- a/docs/user_guide/examples/online_serving/qwen3_tts.md +++ b/docs/user_guide/examples/online_serving/qwen3_tts.md @@ -352,6 +352,42 @@ curl -X POST http://localhost:8091/v1/audio/speech \ - `speed` adjustment is not supported when streaming. - Requires the server stage config to have `async_chunk: true` (default in `qwen3_tts.yaml`). +## Streaming Text Input (WebSocket) + +The `/v1/audio/speech/stream` WebSocket endpoint accepts text incrementally, buffers and splits it at sentence boundaries, and generates audio per sentence. + +When `stream_audio=true`, each sentence is emitted as `audio.start`, one or more binary PCM frames, and `audio.done`. + +### Quick Start + +```bash +python streaming_speech_client.py \ + --text "Hello world. How are you? I am fine." + +python streaming_speech_client.py \ + --text "Hello world. How are you? I am fine." \ + --simulate-stt --stt-delay 0.1 +``` + +### WebSocket Protocol + +Client -> Server: + +```jsonc +{"type": "session.config", "voice": "Vivian", "task_type": "CustomVoice", "language": "Auto", "split_granularity": "sentence", "stream_audio": true, "response_format": "pcm"} +{"type": "input.text", "text": "Hello, how are you? "} +{"type": "input.done"} +``` + +Server -> Client: + +```jsonc +{"type": "audio.start", "sentence_index": 0, "sentence_text": "Hello, how are you?", "format": "pcm", "sample_rate": 24000} +// binary PCM frame(s) +{"type": "audio.done", "sentence_index": 0, "total_bytes": 96000, "error": false} +{"type": "session.done", "total_sentences": 1} +``` + ## Limitations - **Single request**: Batch processing is not yet optimized for online serving. @@ -386,6 +422,10 @@ curl -X POST http://localhost:8091/v1/audio/speech \ ``````sh --8<-- "examples/online_serving/qwen3_tts/run_server.sh" `````` +??? abstract "streaming_speech_client.py" + ``````py + --8<-- "examples/online_serving/qwen3_tts/streaming_speech_client.py" + `````` ??? abstract "tts_common.py" ``````py --8<-- "examples/online_serving/qwen3_tts/tts_common.py" diff --git a/docs/user_guide/examples/online_serving/text_to_image.md b/docs/user_guide/examples/online_serving/text_to_image.md index 793129488..be8c7d894 100644 --- a/docs/user_guide/examples/online_serving/text_to_image.md +++ b/docs/user_guide/examples/online_serving/text_to_image.md @@ -23,29 +23,6 @@ Or use the startup script: bash run_server.sh ``` -### Start with Parallelism Acceleration - -Enable Tensor Parallelism and VAE Patch Parallelism for faster inference: - -```bash -# With Tensor Parallelism (requires >= 2 GPUs) -vllm serve Qwen/Qwen-Image --omni --port 8091 --tensor-parallel-size 2 - -# With Tensor Parallelism and VAE Patch Parallelism (requires >= 2 GPUs) -vllm serve Qwen/Qwen-Image --omni --port 8091 --tensor-parallel-size 2 --vae-patch-parallel-size 2 --vae-use-tiling - -# With Sequence Parallelism (Ulysses-SP, requires >= 2 GPUs) -vllm serve Qwen/Qwen-Image --omni --port 8091 --usp 2 - -# With Ring-Attention (requires >= 2 GPUs) -vllm serve Qwen/Qwen-Image --omni --port 8091 --ring 2 - -# Combined: Ulysses + Ring (requires >= 4 GPUs) -vllm serve Qwen/Qwen-Image --omni --port 8091 --usp 2 --ring 2 -``` - -For more details on parallelism acceleration, see the [Parallelism Acceleration Guide](../../diffusion/parallelism_acceleration.md). - ## API Calls ### Method 1: Using curl diff --git a/docs/user_guide/examples/online_serving/text_to_video.md b/docs/user_guide/examples/online_serving/text_to_video.md index 309edb840..ea7a94ed9 100644 --- a/docs/user_guide/examples/online_serving/text_to_video.md +++ b/docs/user_guide/examples/online_serving/text_to_video.md @@ -46,6 +46,7 @@ The main endpoints are: ## Storage Generated video files are stored on local disk by the async video API. +Local file storage behavior can be controlled via the following environment variables: - `VLLM_OMNI_STORAGE_PATH`: directory used for generated files (default: `/tmp/storage`) - `VLLM_OMNI_STORAGE_MAX_CONCURRENCY`: max concurrent save/delete operations (default: `4`) @@ -147,7 +148,7 @@ curl -X POST http://localhost:8091/v1/videos \ ## Create Response Format -`POST /v1/videos` returns a job record. +`POST /v1/videos` returns a job record, not inline base64 video data. ```json { @@ -204,6 +205,10 @@ done ## Example materials +??? abstract "run_curl_hunyuan_video_15.sh" + ``````sh + --8<-- "examples/online_serving/text_to_video/run_curl_hunyuan_video_15.sh" + `````` ??? abstract "run_curl_text_to_video.sh" ``````sh --8<-- "examples/online_serving/text_to_video/run_curl_text_to_video.sh" @@ -212,3 +217,7 @@ done ``````sh --8<-- "examples/online_serving/text_to_video/run_server.sh" `````` +??? abstract "run_server_hunyuan_video_15.sh" + ``````sh + --8<-- "examples/online_serving/text_to_video/run_server_hunyuan_video_15.sh" + `````` diff --git a/mkdocs.yml b/mkdocs.yml index edecec07a..c81d384d9 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -76,6 +76,8 @@ plugins: # exclude files exclude: - api/* + # Hook-generated content under docs/generated/ is gitignored; no git history for per-file dates. + - generated/** - user_guide/examples/** - contributing/design_documents/api_design_template.md - DOCS_GUIDE.md @@ -122,7 +124,10 @@ plugins: - https://numpy.org/doc/stable/objects.inv # Temporarily disabled due to decompression errors # - https://pytorch.org/docs/stable/objects.inv - - https://psutil.readthedocs.io/en/stable/objects.inv + # Temporary disabled due to HTTP 404 error + # - https://psutil.readthedocs.io/en/stable/objects.inv + - https://psutil.readthedocs.io/objects.inv + markdown_extensions: - attr_list diff --git a/vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_code_predictor_mtp.py b/vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_code_predictor_mtp.py index b1f0b2237..379f5f24d 100644 --- a/vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_code_predictor_mtp.py +++ b/vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_code_predictor_mtp.py @@ -381,7 +381,7 @@ class Qwen3OmniMoeTalkerCodePredictor(nn.Module): proj_buf: [bsz, num_code_groups + 1, hidden_size] pos 0 = last_talker_hidden (NOT a codec embed) pos 1 = layer0_embed - pos 2.. = codec_embedding[i](predicted_code_i) + pos 2.. = `codec_embedding[i](predicted_code_i)` """ bsz = int(layer0_code.shape[0]) device = layer0_code.device