Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Roger Wang
|
2ec26c0fed |
No files matched your search
@@ -1,87 +1,6 @@
|
||||
steps:
|
||||
- label: ":docker: Build image"
|
||||
key: image-build
|
||||
commands:
|
||||
- "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7"
|
||||
- "docker build --file docker/Dockerfile.ci -t vllm-omni-ci ."
|
||||
- "docker tag vllm-omni-ci public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT"
|
||||
- "docker push public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT"
|
||||
agents:
|
||||
queue: "cpu_queue_premerge_us_east_1"
|
||||
|
||||
- label: "Simple Unit Test"
|
||||
commands:
|
||||
- ".buildkite/scripts/simple_test.sh"
|
||||
agents:
|
||||
queue: "cpu_queue_premerge"
|
||||
|
||||
- label: "Diffusion Model Test"
|
||||
timeout_in_minutes: 15
|
||||
depends_on: image-build
|
||||
commands:
|
||||
- pytest -s -v tests/e2e/offline_inference/test_t2i_model.py
|
||||
agents:
|
||||
queue: "gpu_1_queue" # g6.4xlarge instance on AWS, has 1 L4 GPU
|
||||
plugins:
|
||||
- docker#v5.2.0:
|
||||
image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
|
||||
always-pull: true
|
||||
propagate-environment: true
|
||||
environment:
|
||||
- "HF_HOME=/fsx/hf_cache"
|
||||
volumes:
|
||||
- "/fsx/hf_cache:/fsx/hf_cache"
|
||||
|
||||
- label: "Omni Model Test"
|
||||
timeout_in_minutes: 15
|
||||
depends_on: image-build
|
||||
commands:
|
||||
- export VLLM_LOGGING_LEVEL=DEBUG
|
||||
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
- pytest -s -v tests/e2e/offline_inference/test_qwen2_5_omni.py
|
||||
agents:
|
||||
queue: "gpu_4_queue" # g6.12xlarge instance on AWS, has 4 L4 GPU
|
||||
plugins:
|
||||
- docker#v5.2.0:
|
||||
image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
|
||||
always-pull: true
|
||||
propagate-environment: true
|
||||
environment:
|
||||
- "HF_HOME=/fsx/hf_cache"
|
||||
volumes:
|
||||
- "/fsx/hf_cache:/fsx/hf_cache"
|
||||
|
||||
- label: "Omni Model Test with H100"
|
||||
timeout_in_minutes: 20
|
||||
depends_on: image-build
|
||||
commands:
|
||||
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
- pytest -s -v tests/e2e/offline_inference/test_qwen3_omni.py tests/e2e/online_serving/test_qwen3_omni.py
|
||||
agents:
|
||||
queue: "mithril-h100-pool"
|
||||
plugins:
|
||||
- kubernetes:
|
||||
podSpec:
|
||||
containers:
|
||||
- image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
|
||||
resources:
|
||||
limits:
|
||||
nvidia.com/gpu: 2
|
||||
volumeMounts:
|
||||
- name: devshm
|
||||
mountPath: /dev/shm
|
||||
- name: hf-cache
|
||||
mountPath: /root/.cache/huggingface
|
||||
env:
|
||||
- name: HF_HOME
|
||||
value: /root/.cache/huggingface
|
||||
nodeSelector:
|
||||
node.kubernetes.io/instance-type: gpu-h100-sxm
|
||||
volumes:
|
||||
- name: devshm
|
||||
emptyDir:
|
||||
medium: Memory
|
||||
- name: hf-cache
|
||||
hostPath:
|
||||
path: /mnt/hf-cache
|
||||
type: DirectoryOrCreate
|
||||
@@ -1,5 +1,5 @@
|
||||
blank_issues_enabled: false
|
||||
contact_links:
|
||||
- name: Questions
|
||||
url: https://slack.vllm.ai
|
||||
about: Ask questions and discuss with other vLLM-Omni community members in our #sig-omni slack channel.
|
||||
url: https://github.com/vllm-project/vllm-omni/discussions
|
||||
about: Ask questions and discuss with other vLLM-Omni community members
|
||||
+3
-3
@@ -156,6 +156,9 @@ cython_debug/
|
||||
# cursor
|
||||
.cursor/
|
||||
|
||||
# docker
|
||||
docker/
|
||||
|
||||
# scripts
|
||||
/scripts/
|
||||
|
||||
@@ -179,7 +182,6 @@ checkpoints/
|
||||
|
||||
# Cache directories
|
||||
cache/
|
||||
!vllm_omni/diffusion/cache/
|
||||
.cache/
|
||||
diffusion_cache/
|
||||
kv_cache/
|
||||
@@ -230,5 +232,3 @@ configs/development.yaml
|
||||
# Docker
|
||||
.dockerignore
|
||||
Dockerfile.dev
|
||||
discussion
|
||||
tmp_test
|
||||
@@ -1,14 +0,0 @@
|
||||
ARG VLLM_BASE_IMAGE=vllm/vllm-openai
|
||||
ARG VLLM_BASE_TAG=v0.11.0
|
||||
FROM ${VLLM_BASE_IMAGE}:${VLLM_BASE_TAG}
|
||||
ARG APP_DIR=/workspace/vllm-omni
|
||||
WORKDIR ${APP_DIR}
|
||||
|
||||
COPY . .
|
||||
|
||||
# Install vllm-omni into the same uv-managed Python environment used by the base image.
|
||||
RUN uv pip install --python "$(python3 -c 'import sys; print(sys.executable)')" --no-cache-dir ".[dev]"
|
||||
|
||||
RUN ln -sf /usr/bin/python3 /usr/bin/python
|
||||
|
||||
ENTRYPOINT []
|
||||
+7
-17
@@ -7,21 +7,17 @@ nav:
|
||||
- Examples:
|
||||
- examples/README.md
|
||||
- Offline Inference:
|
||||
- Qwen2.5-Omni: user_guide/examples/offline_inference/qwen2_5_omni.md
|
||||
- Qwen3-Omni: user_guide/examples/offline_inference/qwen3_omni.md
|
||||
- Text-To-Image: user_guide/examples/offline_inference/text_to_image.md
|
||||
- Image-To-Image: user_guide/examples/offline_inference/image_to_image.md
|
||||
- Text-To-Video: user_guide/examples/offline_inference/text_to_video.md
|
||||
- Offline Example of vLLM-Omni for Qwen2.5-Omni: user_guide/examples/offline_inference/qwen2_5_omni.md
|
||||
- Offline Example of vLLM-Omni for Qwen3-Omni: user_guide/examples/offline_inference/qwen3_omni.md
|
||||
- Qwen-Image Offline Inference: user_guide/examples/offline_inference/qwen_image.md
|
||||
- Online Serving:
|
||||
- Qwen2.5-Omni: user_guide/examples/online_serving/qwen2_5_omni.md
|
||||
- Qwen3-Omni: user_guide/examples/online_serving/qwen3_omni.md
|
||||
- Online serving Example of vLLM-Omni for Qwen2.5-Omni: user_guide/examples/online_serving/qwen2_5_omni.md
|
||||
- Online serving Example of vLLM-Omni for Qwen3-omni: user_guide/examples/online_serving/qwen3_omni.md
|
||||
- General:
|
||||
- usage/*
|
||||
- Configuration:
|
||||
- configuration/README.md
|
||||
- configuration/*
|
||||
- Diffusion Acceleration:
|
||||
- TeaCache: user_guide/teacache.md
|
||||
- Models:
|
||||
- models/supported_models.md
|
||||
- Developer Guide:
|
||||
@@ -35,14 +31,8 @@ nav:
|
||||
- Design Documents:
|
||||
- design/index.md
|
||||
- design/architecture_overview.md
|
||||
- Feature Design:
|
||||
- design/feature/multi_request_streaming_design.md
|
||||
- design/feature/omni_connector_design.md
|
||||
- design/feature/ray_based_execution.md
|
||||
- Module Design:
|
||||
- design/module/ar_module_design.md
|
||||
- design/module/dit_module_design.md
|
||||
- design/module/entrypoint_module_design.md
|
||||
- design/vllm_omni_design.md
|
||||
- design/mrs_design.md
|
||||
- Docs Guide: contributing/DOCS_GUIDE.md
|
||||
- API Reference:
|
||||
- api/README.md
|
||||
|
||||
@@ -59,6 +59,4 @@ vLLM-Omni seamlessly supports most popular open-source models on HuggingFace, in
|
||||
- Multi-modality generation models (e.g. Qwen-Image)
|
||||
|
||||
For more information, checkout the following:
|
||||
|
||||
- [vllm-omni architecture design and recent roadmaps](https://docs.google.com/presentation/d/1Y7t2Zm3BIISPN-_X_sDpxZMNCWir2rAx/edit?usp=drive_link&ouid=110473603432222024453&rtpof=true&sd=true)
|
||||
- [vllm-omni announcement blogpost](https://blog.vllm.ai/2025/11/30/vllm-omni.html)
|
||||
+7
-5
@@ -49,9 +49,10 @@ Engine classes for offline and online inference.
|
||||
Core scheduling and caching components.
|
||||
|
||||
- [vllm_omni.core.dit_cache_manager.DiTCacheManager][]
|
||||
- [vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler][]
|
||||
- [vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler][]
|
||||
- [vllm_omni.core.sched.diffusion_scheduler.DiffusionScheduler][]
|
||||
- [vllm_omni.core.sched.generation_scheduler.GenerationScheduler][]
|
||||
- [vllm_omni.core.sched.output.OmniNewRequestData][]
|
||||
- [vllm_omni.core.sched.scheduler.OmniScheduler][]
|
||||
|
||||
## Model Executor
|
||||
|
||||
@@ -96,7 +97,6 @@ Model execution components.
|
||||
Configuration classes.
|
||||
|
||||
- [vllm_omni.config.model.OmniModelConfig][]
|
||||
- [vllm_omni.diffusion.cache.teacache.config.TeaCacheConfig][]
|
||||
|
||||
## Workers
|
||||
|
||||
@@ -108,11 +108,13 @@ Worker classes and model runners for distributed inference.
|
||||
- [vllm_omni.diffusion.worker.npu.npu_worker.NPUWorkerProc][]
|
||||
- [vllm_omni.worker.gpu_ar_model_runner.GPUARModelRunner][]
|
||||
- [vllm_omni.worker.gpu_ar_worker.GPUARWorker][]
|
||||
- [vllm_omni.worker.gpu_diffusion_model_runner.GPUDiffusionModelRunner][]
|
||||
- [vllm_omni.worker.gpu_diffusion_worker.GPUDiffusionWorker][]
|
||||
- [vllm_omni.worker.gpu_generation_model_runner.GPUGenerationModelRunner][]
|
||||
- [vllm_omni.worker.gpu_generation_worker.GPUGenerationWorker][]
|
||||
- [vllm_omni.worker.gpu_model_runner.OmniGPUModelRunner][]
|
||||
- [vllm_omni.worker.npu.npu_ar_model_runner.NPUARModelRunner][]
|
||||
- [vllm_omni.worker.npu.npu_ar_worker.NPUARWorker][]
|
||||
- [vllm_omni.worker.npu.npu_generation_model_runner.NPUGenerationModelRunner][]
|
||||
- [vllm_omni.worker.npu.npu_generation_worker.NPUGenerationWorker][]
|
||||
- [vllm_omni.worker.npu.npu_diffusion_model_runner.NPUDiffusionModelRunner][]
|
||||
- [vllm_omni.worker.npu.npu_diffusion_worker.NPUDiffusionWorker][]
|
||||
- [vllm_omni.worker.npu.npu_model_runner.OmniNPUModelRunner][]
|
||||
@@ -2,4 +2,4 @@
|
||||
|
||||
- For technical questions and feature requests, please use GitHub [Issues](https://github.com/vllm-project/vllm-omni/issues)
|
||||
- For coordinating contributions and development and discussing with other users and developers, please join `sig-omni` channel in our [Slack](https://slack.vllm.ai/) or use the [vLLM Forum](https://discuss.vllm.ai/)
|
||||
- For security disclosures, please use GitHub's [Security Advisories](https://github.com/vllm-project/vllm-omni/security/advisories) feature
|
||||
- For security disclosures, please use GitHub's [Security Advisories](https://github.com/vllm-project/vllm/security/advisories) feature
|
||||
@@ -1,12 +0,0 @@
|
||||
# Volunteers for Bugfix and CI
|
||||
|
||||
We encourage you to check current docs and [issues](https://github.com/vllm-project/vllm-omni/issues) to find possible solutions for your questions. If non of these can solve it, please propose an issue to describe your questions about bug or CI problems for developing.
|
||||
|
||||
If you have urgent need for locating and solving bugfix or CI problems, please find community volunteers below.
|
||||
|
||||
| Dec 4-Dec 12 | Dec 15-Dec 19 | Dec 22-Dec 26 | Dec 29- Jan 2, 2026| Jan 5-Jan 9 | Jan 12-Jan 16 |
|
||||
|----------|----------|----------|----------|----------|----------|
|
||||
| <a href="https://github.com/congw729">Conw729</a> | <a href="https://github.com/yinpeiqi">yinpeiqi</a> | <a href="https://github.com/tzhouam">tzhouam</a> | <a href="https://github.com/SamitHuang">SamitHuang</a> | <a href="https://github.com/gcanlin">gcanlin</a> | <a href="https://github.com/natureofnature">natureofnature</a> |
|
||||
| <a href="https://github.com/david6666666">david6666666</a> | <a href="https://github.com/R2-Y">R2-Y</a> | <a href="https://github.com/hsliuustc0106">hsliuustc0106</a> | <a href="https://github.com/Gaohan123">Gaohan123</a> | <a href="https://github.com/ZJY0516">ZJY0516</a> | <a href="https://github.com/qibaoyuan">qibaoyuan</a> |
|
||||
|
||||
We kindly welcome more contributors to fix bugs and contribute new features!
|
||||
@@ -9,7 +9,3 @@ Currently, the main options are maintained by stage configs for each model.
|
||||
For specific example, please refer to [Qwen2.5-omni stage config](stage_configs/qwen2_5_omni.yaml)
|
||||
|
||||
For introduction, please check [Introduction for stage config](./stage_configs.md)
|
||||
|
||||
## Optimization Features
|
||||
|
||||
- **[TeaCache Configuration](../user_guide/teacache.md)** - Enable TeaCache adaptive caching for DiT models to achieve 1.5x-2.0x speedup with minimal quality loss
|
||||
@@ -9,7 +9,7 @@ stage_args:
|
||||
model_stage: thinker
|
||||
model_arch: Qwen2_5OmniForConditionalGeneration
|
||||
worker_cls: vllm_omni.worker.gpu_ar_worker.GPUARWorker
|
||||
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
|
||||
scheduler_cls: vllm_omni.core.sched.scheduler.OmniScheduler
|
||||
gpu_memory_utilization: 0.8
|
||||
enforce_eager: true # Now we only support eager mode
|
||||
trust_remote_code: true
|
||||
@@ -35,7 +35,7 @@ stage_args:
|
||||
model_stage: talker
|
||||
model_arch: Qwen2_5OmniForConditionalGeneration
|
||||
worker_cls: vllm_omni.worker.gpu_ar_worker.GPUARWorker
|
||||
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
|
||||
scheduler_cls: vllm_omni.core.sched.scheduler.OmniScheduler
|
||||
gpu_memory_utilization: 0.8
|
||||
enforce_eager: true
|
||||
trust_remote_code: true
|
||||
|
||||
@@ -23,7 +23,7 @@ Install vLLM-Omni from source with development dependencies:
|
||||
|
||||
```bash
|
||||
git clone https://github.com/vllm-project/vllm-omni.git
|
||||
cd vllm-omni
|
||||
cd vllm_omni
|
||||
uv pip install -e ".[dev]"
|
||||
```
|
||||
|
||||
|
||||
@@ -1,195 +1,113 @@
|
||||
# Architecture Overview
|
||||
|
||||
This document outlines the architectural design for vLLM-Omni.
|
||||
# Introduction
|
||||
|
||||
<p align="center">
|
||||
<picture>
|
||||
<source media="(prefers-color-scheme: dark)" src="https://raw.githubusercontent.com/vllm-project/vllm-omni/refs/heads/main/docs/source/architecture/omni-modality-model-architecture.png">
|
||||
<img alt="Omni-Modality Model Architecture" src="https://raw.githubusercontent.com/vllm-project/vllm-omni/refs/heads/main/docs/source/architecture/omni-modality-model-architecture.png" width=55%>
|
||||
</picture>
|
||||
</p>
|
||||
This document outlines the architectural design for vLLM-Omni, a new repository developed under the vLLM project. vLLM-Omni supports multi-modality models with non-autoregressive structures and non-textual outputs, extending vLLM beyond traditional text-based, autoregressive generation.
|
||||
|
||||
# Goals
|
||||
|
||||
The primary goal of the vLLM-Omni project is to build the fastest and easiest-to-use open-source Omni-Modality model inference & serving engine. vLLM-Omni extends the original vLLM, which was created to support large language models for text-based autoregressive (AR) generation tasks. vLLM-Omni is designed to support:
|
||||
The primary goals for vLLM-Omni are:
|
||||
|
||||
* **Non-textual Output:** Enables the integration, efficient processing and output of various data types, including but not limited to, images, audio, and video, alongside text.
|
||||
* **Non-Autoregressive Structure:** Support model structure beyond autoregressive, especially Diffusion Transformer (DiT), which is widely used in visual and audio generation.
|
||||
* **Integration with vLLM Core:** Maintain compatibility and leverage existing vLLM key modules and optimizations where applicable.
|
||||
* **Support for Non-textual Output:** Enables the integration, efficient processing and output of various data types, including but not limited to, images, audio, and video, alongside text.
|
||||
* **Non-Autoregressive Structure:** Support model structure beyond autoregressive, especially Diffusion Transformer (DiT), which is widely used in visual and audio generation.
|
||||
* **Integration with vLLM Core:** Maintain compatibility and leverage existing vLLM key modules and optimizations where applicable.
|
||||
* **Extensibility:** Design a modular and flexible architecture that can easily accommodate new modalities, model architectures, and output formats.
|
||||
|
||||
# Expected Gains
|
||||
|
||||
# Representative omni-modality models
|
||||
* It breaks through the limitation of vLLM that only supports autoregressive text generation models, and provides architectural support for popular model structures, especially multimodal generation.
|
||||
* By supporting new output formats and new model architectures, developers can easily connect to mature ecosystems such as Higress Gateway and Ray, improving the ease of deployment and efficiency of corresponding models.
|
||||
* Integrating efficient cache management, multiple parallel strategies, disaggregation from vLLM, to improve inference efficiency of corresponding models.
|
||||
|
||||
# Non-Autoregressive Structure
|
||||
|
||||
* **Diffusion Transformer:** initialized with a random latent noise, the transformer model iteratively predicts latent output for multiple steps, under control of sampling timestamps. Finally, it is decoded by the decoder of Variational AutoEncoder (VAE). It is not supported by vLLM now.
|
||||
* **Autoregressive:** dominant generation paradigm for text, now supported by vLLM. It generates tokens conditioned on previous tokens one by one. The efficient KV Cache management provided by vLLM can accelerate inference effectively.
|
||||
![][image1] ![][image2]
|
||||
|
||||
|
||||
| | Autoregressive | DiT |
|
||||
| :---- | ----- | ----- |
|
||||
| Scenarios | Text generation | Multimodal generation |
|
||||
| Generation | Token-by-token, left to right | Continuous vector, iteratively refine |
|
||||
| Key advantages | Smooth, good local consistency | Global consistency, diversity and controlability |
|
||||
| Efficiency | More efficient with KV cache | Need generation repeatedly, slow |
|
||||
| Error accumulation | Easy to accumulate | Can fix possibly |
|
||||
| Key challenges | Limited global planning, long context generation | Large global computational cost |
|
||||
| Sequence length | Varied length | Fixed length |
|
||||
| Attention mask | Lower triangular | Full bi-directional matrix |
|
||||
| Parallel strategy | TP, DP, PP, .etc | DP, CP, .etc |
|
||||
| vLLM support | Supported | Unsupported |
|
||||
|
||||
# Targeted models to support
|
||||
|
||||
According to analysis for current popular open-source models, most of them have the combination of AR+DiT. Specifically, they can be further categorized into 3 types below:
|
||||
|
||||
**DiT as a main structure, with AR as text encoder (e.g.: Qwen-Image)**
|
||||
A powerful image generation foundation model capable of complex text rendering and precise image editing.
|
||||
* **DiT as a main structure, with AR as text encoder (e.g.: Qwen-Image)**
|
||||
A popular model for image generation and editing. Main visual generation models are similar to the structure, such as Flux.
|
||||
![][image3]
|
||||
* **AR as a main structure, with DiT as multi-modal generator (e.g. BAGEL)**
|
||||
A unified multimodal comprehension and generation model. Visual generation can utilize CoT text generation output. Main unified multimodal models are similar to the structure.
|
||||
![][image4]
|
||||
* **Extension, multiple AR+DiT (e.g. Qwen-omni)**
|
||||
A new model for multimodal input and output. It designs the Thinker-Talker-Codec structure, which is a double-AR+DiT format.
|
||||
![][image5]
|
||||
|
||||
<p align="center">
|
||||
<picture>
|
||||
<source media="(prefers-color-scheme: dark)" src="https://raw.githubusercontent.com/vllm-project/vllm-omni/refs/heads/main/docs/source/architecture/ar-main-architecture.png">
|
||||
<img alt="Qwen-Image" src="https://raw.githubusercontent.com/vllm-project/vllm-omni/refs/heads/main/docs/source/architecture/ar-main-architecture.png" width=30%>
|
||||
</picture>
|
||||
</p>
|
||||
|
||||
**AR as a main structure, with DiT as multi-modal generator (e.g. BAGEL)**
|
||||
A unified multimodal comprehension and generation model, with cot text output and visual generation.
|
||||
|
||||
<p align="center">
|
||||
<picture>
|
||||
<source media="(prefers-color-scheme: dark)" src="https://raw.githubusercontent.com/vllm-project/vllm-omni/refs/heads/main/docs/source/architecture/dit-main-architecture.png">
|
||||
<img alt="Bagel" src="https://raw.githubusercontent.com/vllm-project/vllm-omni/refs/heads/main/docs/source/architecture/dit-main-architecture.png" width=30%>
|
||||
</picture>
|
||||
</p>
|
||||
# Architectural Overview
|
||||
|
||||
**AR+DiT (e.g. Qwen-Omni)**
|
||||
A natively end-to-end omni-modal LLM for multimodal inputs (text/image/audio/video...) and outputs (text/audio...).
|
||||
vLLM-Omni will import core modules from existing vLLM framework and introduce several new components and modifications to achieve its goals. The core architecture will be based on a modular design, allowing for independent development and integration of different modality handlers and output generators.
|
||||
![][image6]
|
||||
|
||||
<p align="center">
|
||||
<picture>
|
||||
<source media="(prefers-color-scheme: dark)" src="https://raw.githubusercontent.com/vllm-project/vllm-omni/refs/heads/main/docs/source/architecture/ar-dit-main-architecture.png">
|
||||
<img alt="Qwen-Omni" src="https://raw.githubusercontent.com/vllm-project/vllm-omni/refs/heads/main/docs/source/architecture/ar-dit-main-architecture.png" width=30%>
|
||||
</picture>
|
||||
</p>
|
||||
## General ideas
|
||||
|
||||
# vLLM-Omni main architecture
|
||||
|
||||
<p align="center">
|
||||
<picture>
|
||||
<source media="(prefers-color-scheme: dark)" src="https://raw.githubusercontent.com/vllm-project/vllm-omni/refs/heads/main/docs/source/architecture/vllm-omni-main-architecture.png">
|
||||
<img alt="vLLM-Omni Main Architecture" src="https://raw.githubusercontent.com/vllm-project/vllm-omni/refs/heads/main/docs/source/architecture/vllm-omni-main-architecture.png" width=55%>
|
||||
</picture>
|
||||
</p>
|
||||
* It divides models in vLLM-Omni into multiple stages, which are processed by different Engine Cores. Each stage can choose AR Engine Core for AR model part, and Diffusion Engine Core for diffusion model part.
|
||||
* Designed a specialized diffusion engine core for DiT or dLLM.
|
||||
* Plug-in with external DiT acceleration modules such as xDiT, Cache-DiT.
|
||||
* Output linked with Higress API, Gradio for online demo, ComfyUI for less-code process.
|
||||
|
||||
## Key Components
|
||||
|
||||
| Component | Description |
|
||||
|-----------|-------------|
|
||||
| **OmniRouter** | provide an intelligent router for Omni-modality requests dispatch |
|
||||
| **EntryPoints** | define the APIs for offline/online serving (APIServer, Omni/AsyncOmni) and provide the OmniStage abstraction for different AR/DiT stages |
|
||||
| **AR** | adapted for omni-modality models while inheriting efficient features from vLLM, such as cache management |
|
||||
| **Diffusion** | natively implemented and optimized using acceleration components |
|
||||
| **OmniConnector** | supports fully disaggregation based on E/P/D/G (Encoding/Processing/Decoding/Generation) disaggregation across stages |
|
||||
* **Processor:** Input processing, tokenization or encoding
|
||||
* **Scheduler:** for order and queue of incoming requests
|
||||
* **Executor:** Prepare model input and run
|
||||
* **Multimodal output processor:** support multimodal outputs
|
||||
* **Hidden State Output processor:** support intermediate hidden vectors as request output
|
||||
* **Engine Core:** accelerate AR modules, imported from vLLM main repo
|
||||
* **Diffusion Engine Core:** Wrap and accelerate DiT modules.
|
||||
* **Diffusion model:** Wrap up diffusion models with vLLM format
|
||||
|
||||
Disaggregated stages are managed through configuration, such as in the Qwen3-Omni example, where stages like Thinker, Talker, and Code2wav are defined as separate OmniStage instances with specific resources and input/output type.
|
||||
# Examples data flow of specific models
|
||||
|
||||
## Main features
|
||||
1. Qwen-omni
|
||||
![][image7]![][image8]
|
||||
2. Qwen-Image
|
||||
![][image9] ![][image10]
|
||||
|
||||
vLLM-Omni aims to be fast, flexible, and easy to use with the following features:
|
||||
# Overview of function call dependency
|
||||
|
||||
### Performance and Acceleration
|
||||
![][image11]
|
||||
|
||||
The framework achieves high performance through several optimization techniques:
|
||||
|
||||
* **Efficient AR Support:** Leverages efficient KV cache management inherited from vLLM.
|
||||
* **Pipelined Execution:** Uses pipelined stage execution overlapping to ensure high throughput.
|
||||
* **Full Disaggregation:** Relies on the OmniConnector and dynamic resource allocation across stages.
|
||||
* **Diffusion Acceleration:** Includes integrated support for diffusion acceleration. This is managed by the acceleration layer, which handles:
|
||||
* **Cache:** Includes DBCache, TeaCache and third-party integration(e.g., [cache-dit](https://github.com/vipshop/cache-dit)).
|
||||
* **Parallelism:** Supports TP, CP, USP, and CFG.
|
||||
* **Attention:** Provides an interface for third-party integration (e.g., FA3, SAGE, MindIE-SD).
|
||||
* **Quantization:** Supports various quantization implementations including FP8 and AWQ.
|
||||
* **FusedOps:** Allows for custom and third-party integration.
|
||||
[image1]: <data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAALYAAAFwCAYAAAAVNoEZAABxw0lEQVR4XuydBVRcWb6ve90709OStnTScel4x93dIUaEuBJ3dzciRPDgEtzdCRpCBCIEEkIgQHCCBHe+t6uSvNvNnZ55r2/fng5zvrX+69TZp4qkan3nz2/vKup88sm/OWqXL3HxwnnOnj7JubOnuaB6jqtXLqJ28TxXRWmrX8Ha0gIzE0OuX1bFQFsTcxMDLMyMMDXS44bWVRxsLLG5aYKuxlVsLMxwc3bAWF8bNzdnHGwt8fb2wNXZHlcnW3HMDjsbC4Ju3cLO2hwvT3ecnexwsLcmIMAHHy83+X18PVxxtLMUt+1xcbTBw8URRwdr+f1tLM1wd3PB08NZlAs24ue4uTrKf0bD5yfxb8qWDWvZtnE9W0Vt37yBvTu3cWDPTlE7OHviCBfPnsTEwECIfg7Vk0e5fkkVoxtamBneQF1NlSsXTmMrZJaN6elcF7LexFaIZ6SngbOjvfy2l5DUSwhoa2mK9U1jfL09ibwTKaS3IPx2BA4ONnh7uuHr4yUX1d/Pi0B/X1xd7PHz8cTT3UUc85SfIMHBQUJ4a3EfH/mJ4OsjThoXB/njPNydJbEl3rFisTKrli5i9fLFrF21jC0bVdi2aR07tqzn+KF9nD91HF1NDU4eO8CZIwe5fO4UBjoacpEvnDnG5fMnsDQ1Rk/rOje0rwmxLeQd3chAC0d7W6zMjfBwc8JHiGlmpIu5oS6hISHcibgtOrE99+4+wE50dR8hraeHm1zOW4G+BAb4yjtyYICf/FiAED0wMICw0DDxODvR8QNEBeLv7yO6txPeotN7e7lLYku8Y8akccycOhGlGVNRVprJsoXzWLlEmbUrl7Jv5xbOHD8sBD/ApnWrOLZvN2ePH0FDRJVronMf3rdDiH1aiK4p9s+JE0AWS6xE59bA2EAHO6ub3DTRw1UI7Ovjg/oVEWV01Llz567ouH6i+wbz8FEMtiKaBPr7yzuvn6+nED9I3rH9/bwJDgqUSy3r1OHh4YSEBOPu7io6/W1CQ0PFSeAvxHeTd3B/X19JbIl3TB4zjOnjRzJz8jiUFCazYPZ0li1QYrXo4nt3bObE4QPs3LIRlRWLObh7h9jfj5rqGdGtj7N/11Yh9hk0r14W+yfQ0bgiZLZC8/pljPR1ROwwwdxYF2d7GyGfN1cunkJfiB0h4oe3lweRkfeIjn4iF9vf1wcnkbX9fL0ICb4ll1kutrgtkzdEyB4ZGUlQ0C35Y+/cuSPv3oHimOxkkHX2W6KjN3x+Ev+mjBzQmzGD+zJxxCAmjxrC9AmjUBZyL1dWEkJvEF17G6tEB1++aB47Nq0Xsm+Vy71PbHdv2yTP4LL9Q3t3onVdTYhsyCXV02LyqCuyuS762texlk0oXV05e/IQetrqBInuK5v8RUc/JiLiDvYiisgmf7Y2N0UssZJv1a9fFQJ7yiUPkXXrsBAePHiAn+jMsghy//590dlDRRTxk8cW2VhoWLAktsQ7Bv7UleH9egi5+8kFl3Xw+TOnsmTBLDGpXMsOMaFcunCuiChKbF27ml1CbtnEcvt6FbnY504eE517uxB+CxpXLwlxNbhwXjbhvCGXWFOMmZsZibztwMmj+7khsniA6Kwurs4iSoi87OKCuuj4urqa3DQ3xdHBVt6pXZwd0NJQJ1AmbGiw6NC3xYkQLbK6lxA6WC55yP8VWxZrbhEWHiKJLfGOPl06MKhnVwb06Cy2XZg0aqiIJJNYNHcG61cvl2dtZXF78byZrF22iHUrl7B53WqRwxcI6dfLO/WW9Wvk3f3i2VOonj7BeRFT9HW0ULtwRoh/FIMb2hgZ6otMLuvql9klos3COYri3xrOiEH96dujK907d+Snrp3o37c3Gzdu4MKFCxjqG7Bn9y7uRt6Vd+gHUQ9xFsKHiMmnbEy29ReRRbaaEhYWJo8qDZ+fxL8pfbt2pF+3TqJ+ZHCvLiKODGX2tAly8VRWLJFPJBcoKcpFX6E8j9WL300slyrPkXd0WdfeIE4A2UqKbHlQtpJy7tQxtNWvckZILYspOprX0dFSZ8+2DezcqMKsCaOZOGwQvTp1oHPbVnRo2YL2rZrz/TdN+O6rJrRp8YMQvA+q5y9w9NARbgUFEf3wEffuR2FrZyUXOiIiQnTyEPmkUSa2LNLcu/dAElviHb06taF357bIBB/cqzNjh/YXOXs0cxWnsWjebPlKyVwRTebNmILyLAWWzJ3NMuW5LJijIO/o61YvY/WShWxUWcmB3dvZLgQ/emAvZ04cZc/2zULmTaiKuLJv22ZmjxvJqH496dK6pagW9Gnfhp7tWtH5h+9p//23tPruG1qI+v6br/n2qy/47K9/odn337FGRQUHJ2fCbt/GyFhPxI4Q+eRRJrivr7d8AvngQZTo6tGS2BLv6P6jELtLO7nY/bv/yPCBPZkiBJylMIV5QuS5M6czR3EKStMnMU8m+5yZLFGazdwZU1mzfAmrly2UTyzXr1omjyMb1qxgn8jch/fvYdsGFXZsXMMusV0i7j+6V1eGdm3PGBF5pvXtztxBPZjetyujxL8/VPw/erRuzo9C8pZNheBNv+b7r7/iy88/o2PHjixcspTbtyO4IbK4bMnvzp13KySyzB0cIptYRktiS/wXnzdpzpdNmtGtQyt+6tiK/j06MXbEYCaPG8WMKROYOWUc0yaOY6ro4orittKMacxWmMysaeNZtmAei+fPEZ19FquWLmSdkHv5ovnyzL1dxJQ1Qvr1ohaOG8rxJTO4ukYJ/a1LODprDEsGdmdkux8Y2KopHb/+knZffk6LJp/xvRD5e3G7+Zdf0PKbr2gtJG8jZP/uqy9Zu24tqmdOEn77jrxjy96wkb1BExERyd2794m4c1cSW+Idn33ZlL99IaJA6x/o1KaZ6N7tGTmkP+NGDpXLPG38KLnkk8eNYPqkMXKpZ0ydIKQfK2LJHHkHV1aaIYReIM/jMtFlgss691KRzeeKn6G7YwVup3fgr3YAw83KLO/VkbX9OrNhYBfmd23JrC6tmNT+B7p89zVdRafu1uwbOjf9ivbffUUHsS/r4h3F/69Fs2YsWbhAZOn78owdGBgol1vWyWWTyYg7EZLYEu9o/cVXrO71E6aL53Nx1mT6iLw9sE9Phg/qx/jhQ+Q1doTYjhryXu6xTJNtx4scrjCVOSKiKImoIsvispJ19MULlEQGn8GsYf05vWI2+aGOZHqb4LJvNY+v7iP2+hEeXdpL2NF1uG9V5ti4nuwb1pUlPVoz88cWTGzblHGihrVuypA23zOsfXOGCPE7t/qBZt9+g7qmFmFh4XKxZXlb9kbPbdG1ZZ87afj8JP5NGd2mFfuHD8BCeQ43F8/hzMxxzB3al5H9fmJo/z4MH9CXEYMHMEpMKscNFxFl9AjGjxjERNHRFSbIJB/N9MljmDV9MjOnTUJh8nhmT5/CxNHDMNu/luI77pRGB5Dhb0t1zB2qnoRREuxEnpsx989sI3T/apzXKXBz6WRMl01Bf9F4VKcP5uj4Pmwd1p21g7ui3Lsj83p1oEerZrQV3fv7pt9y4uRxbgUHERIWRlh4BMFC9FvBoZLYEu/YObgfp0YO4ZLowtemjhViKXB1wVT2KIxBccRAxorOPVgIPnRQX8YMHSjv4GPeSz5p7EgmjhkhjywyoadPkmXxMfJMPmHoAOJstKhLjKLiaTiVCQ+pSY6jJj6K0jA3CjxNSdI/S4L2MR5c2sW9CzsIOqKC156lGC2bivr8MRye0I+dI3uyelBXlvXvxMA2zWnb7Ds+//QvjBkziiAxaQwJDRWTx3D5kqC/9Ja6xAcU27Rkcgvx67/59yzo2JrDo/thvVYZj51rCT20leCDm7DfvALN5XNRHj2UcQP7MKRPV9HJe4suLuvkQvZRw+SCTxg9XH57pDgZji6bSXVcJHWZidS+SaMuN43alGdysasfBlER6UNpsCNv/azJctQh1eoacdrHuX9hF67bFmK+SpFz04dwZHw/to3sxfqh3ZgssniXZl/LlwT/+p//ia+vn5g8+uPt7SX/YJSrq4sktsQ7un39Fd2/+ZoB337NpJZNWdezI/pKY3FSmU+o2mVCjc3xd/LCw9kHbSNrDl/QQFFpAaNHDGNov3dZfIzo3rLJ5tiR7/L4zJGDCBQ5ui4tntr8TOpF1WWnUJsUQ+2z+1RHBVF1L4DSUFeKAuzJdjEkzUaT+BtniLq8F+ftypitVOCsEPvoxH7sHt2XzUN7MLtHGwa0aEpLIfYXf/sMS0sLPDw8cHR0xN7ORl4Nn5/Evymd2rXgxzY/0EF0wj5dO3NE2xTj5CLMC+rQL6zDtKQe+/I6rEvrMS6owfBNFQY5leimlnLI0pOl2/aLDN6X0UPede9TG1dxS/sMiO5cHO5DcYgHed4OZNkZURbqSfWDYKrvBlN+25/iIDcK/Rx542JEpp0uTzWPEXl+Jw6b52O8fDrnFYZxcupg9o7pK7p2Dxb3aYdil5a0+eYrvvnicw4e3I/lTQsM9G+gr6eL3g1dSWyJd3T44Rt+bN2C+bsOc9w7ApMiuFkMN/JrsSyuwbm8FuuiWkzF/s3Caizf1mCZX4VpbgV6yW+5/iCRpVv3MkFMJBdMn0BehDf50SGQLaLH6xfUy+JHVCiVYZ6UeltRGexO9T2xHxFI0S2RtX0cyHMxI9P2Bo9Flw8/vQ3bjfMwWDqVc0LsE1MGsXd0H7aP6M6SPu2Z3bUVbb9pwjdffsnKlcvQ1rjO1SuXuHZVVpclsf8gNomSvdgfKuaXh//1qD3O4fLrKjRya9HJrUE3uxqr4mrcy2vwLKvDMKsKSyG0s9i3Lxad+20txqJj3xRlJeQ2yyhBNy6Dq0GPWLN8Kbn3g6jNEdEjL5uHphZcO3QRNyNHEnyCqY2NpfbFC2rjnlBxL0x0clkMuUmSmQZPdS9wR3U/t4Line truncated
|
||||
|
||||
### Flexibility and Usability
|
||||
[image2]: <data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAARUAAAEOCAMAAACQIXOIAAADAFBMVEUAAAAAAAD/AAAAAAAAAADfACAAAAAAAADvADAAAAAAAAAAAADqACsAAADgACkAAAAAAAAAAAAAAAAAAAAAAAAtAAbmAC0AAAAAAADqADAAAAAAAAAAAAAAAAAAAADnADAAAAAAAAAAAADqAC4AAAAAAAAAAAAAAADpADAAAAAAAADoAC4AAADnADAAAAAAAAAAAACxACQAAAAAAADoADAQAALkAC8AAAAAAAAAAADoAC8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADqAC4AAAAAAAB/ABoAAADpAC8AAAAAAAAAAADqAC8AAAAAAAAAAAAAAADpAC8AAAAAAADpAC8AAAAAAAAAAAAAAAAAAAAAAAAAAADpAC8AAAA0AAvhAC0AAABXABLqAC8AAAAAAADpAC8AAAAAAADqAC+rACLpAC/IACgAAADGACjqADDpAC8AAADVACsAAAAdHRoAAAAdHRojHBowGhw2GR08GB1JAA9XFSBdFCFmABVpEiKQDSWWDCenACKq4+q8Biu9ACbCBSvTACvcAi7jAS/pAC/4Y4L9gpv+h6D/kKcAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAn++oGAAAAdHRSTlMAAQEECAgKCxATFRYYGRkaHSAhJCgoKCsvMDY3Ojw+QEJFR0hJTE1QUFRYWF1gYmRoaWttcHJyc3R3eHyAgYSGiYyNj5CSk5SXmZ2en6Klpq+vsrO3uLu/wMXMzc/S1tbX19fg4eft7u7v7/H09Pb3+f3+/iGfBl4AABcuSURBVHhe7Z1dcBNXlsePaeE4MW23jLyOhASOsPiwwYAXg5kwZAOJcajNkDDxUMkkVcmmaqt2HlJTNTOv+7Cvm62amtpK1W7NJFWbbDIDMyRVmYUAkxAIWcAQMAx2+Su2wY6FQTYysh0jS2bvR7fUfdV2d0stS2Lv70G6ffuclvqv27dvdx+dC8DhcDgcDofD4TwcFLEVOWNlcR9bZRHJlSjGB1T11nGwFblC2Op/m62zyMrmRDH4kKhiA/c6AMp8oVFUDLPrrPEwqTI4CFDvCx9j661TMKqIq93SdNdAFB1rK4pvxn114pEoSOt8UVKnh7DBI00PXSZF7NIiXhmgLsfm8UhQKKo81YhfA6N/moai5mVHdvgAWg97D+C64GHdndzwpIheA9Hr6LXoWRG7eIN/XPHCEoBlf7zPWmspFFWWD6MfunpvVfMnaMHxk+73wN3y3OOdF6D6aXdNJ2uNqG+Gtg4o2tIyR9ZiF3eLu+XxLuyyWs9DxRK2Ik85/fvuKAwehdJivBQ8PjZ2HbWco2Nj31yGNawxoqIJjp4ZGwudhC3EA7tcP4FcjiEXWGMwICkUVcbI61TESRr3FXLMxNrwawhKE1ZJvGVBOvy5V0lHMdjl7lzs4gO8QLWdn0I5ggBcFStcS8UZXIyRE29kKXm7qzZKUAM3aG8T8ksjILtMTRYTlzmVoS6Foor/Wdx34v0zy7hSqFDXmqNQVDkQ+7pnDCpal7IrskKBqCLApXNs3YKElSbiTDYa8xRIb1tE982LDyNThGAz7YSdMzeYVSYokLYSg331E0vW90bMHkHXhl/6p9hAdMn6ecZ4C1MgbQUugrdu/fmv2Or5GT805AjUrY+lI0oe3V8xQJDkMYt5kMuc/nmbw+FwOBxOIVEw45V/YyvU/IKtyJA8GPHbub8LbutXhvdVFAplxL+4cFX04KrowVXRg6uiB1dFD66KHlwVPbgqenBV9OCq6MFV0YOrogdXRQ+uih5cFT24KnpwVfTgqujBVdGDq6IHV0UProoeXBU9uCp6FKAqcdOPANMm/54zrxueZKu07IHP2aoENj1RzT+W/WwZW8WwZw9bYzf5dwRNdm9nqxgurDXSLVPyTxW002wNg7FumaLuV36lKueSx6bZGoYlUtjWLuJt8qchFXnYVmDG6EvNzZSwVfZi9AVywZzhPs+UZPd7Z3fraWK4z9luLEafnxOM9zn6aFa/OBsX96/Mck74m5d/ZzCSgxfvzT+Us8IvdNXVrcw1tw+9aTQi+bgsm0O5vFQFgjcNRyTnN7A1NpKfqqB9NvrTWPAmW2MjeapK8OYmtorlvJFuGZCnqsD5LUY7HTTULX3yVZVgyHCnDXVLn3xVxURjMdYtbfJWFePGYqxb2uStKsY7baxb2uSvKsY7bahb2uSvKvCxbytbpSX459ezJEseq2LcFoybU5rksyrGO22oW5rksypop9kaBmPd0iOvVQmG2BqWLDWWvFbF+Fon+H1WGkt+q2J8rXMpK40lv1UxvtbpykpjyXNVjPc5K40lz1Ux3ueu7/VSC2ZInqvS9dVBtorlgy3r2KqMyYEq9b/8qUHCQxVdQPd5AadLBhcGacA++UiT3Q0fKnn/6kme5tjoZBtOSpwxl7Z2sVUMXfarYk9bcfhhG1OzYu1r2po0URrLAlxiKzLGnrZSK4E3GUkQ/HwWwLdd3HlWbZMuJhrLOiMLq9jSVoQauFRSnVweHxsba/9deNV8PYElTDQW2w8hW1Qpr4z1KNl1E8TaaC7ajDHRmxrqZhFbvrhXRNdxlS42z6pDYsJzxNXrH1zpVpaEjQEHDH4j5/6rDrge3OgiSe+EljJ4cENZIZY43/j+iuIEwgHFabO7v7u60XF6xMRBZhE7VHFsgxvRsORnVHHR5LxJSI5ar7IHFa041eaK7b/GC9JeH3r1Crgnoiu82/+C84Bjp+KqsPcUdaIrV2w/0YnW+COPoPOdNILOQjb3LHaoskvqOwtHWpvGVTm6xcpnxXc0TUXY7w++h35jHxmXoaW+/yDvL3yCF2bel8/j2IyuaNl0OEqd1m394JfKSuwl7N+3BmcVb+qTp3v44FWwVRYbVKkIQA/AxB1xjayK+y3y1qk9fjz+2Cnc8Ef6/XSJZL2O9+16bNrjhy+UwQ02Iysu+t01ndQJtQXipHjF+/zklDdzQnay+xiyobf1ijOD+Juik7OqNvbZUfwm1BCKcfrdO+QmUpxk+Q7ISyTttRPk1N/qFUGcuFd2urSVrldWku4HYDihu4kTlRUybyvotDzhQe9LImIDHaAEUcvf2dgi4aXyPbiTiB0aARdMaZKqyi0KYtK0dpVcfjAuuRSn5PBV8SI9uepWXZetjSVzVcoroeoFWlzVpuxc/LTQsBWrMkFDkkI4raoatPQdLT3QrEo1I9BdxisTXlqubrGzw81clc2i/E3LRPXJ+YxfKo4qxwsirj0hhaEP95fKQim2xcRVZaSl4nT1aXmlykvD7BU7G0vGqqBLoPP0wHG8rj45x44f3MaM+OlFATriAO+w6hIhBE5HUgm5XFRBcmVTs9mAvFLlpcXWxpJxb+uWYv20hN43q/rbocvqJYBeoBcF6IgD3F+qLhGGIyUNKeVaCXXiihMUk5GzxkvL7BUbQ8IyVUVoVM4Zyf2WaS8h9xQURvqhGfXK0gGSJx0vkXh9AdWN90LTTlSW1tIyrvfvgPZpxQmkZY8oXsTph7hSy1Wnm61Km0yPIPTLy1NH4HOppAxZMONQU6tajH9ZKb7S4fQEr+A+Ai89/1w3OD0foqUzlb6mrbfKxfPocuC05P/5QMwrQudZ2Sk06vR89qIge/0cO8W+TW5YZvZK08dsXbqw/27IbrytWDquPjsvd96NJJaXl95SyqIzel8ektAl5LX09T8HqZ3aScNWn3VZSLxtyr8bMm0r1ohENItj6qzpqgWtGV1KNAWNk4arRg9gTZNpv7J4XHUZ9RuzySvrDCkcVVBjYatYrhrpZpbCUcVMYzHUzSQFpIqJxmKom0kKSBUTjcVYN3MUkirGjcVYN3MUkirGO22smzkKShXjnTbUzRwFpQpcGn2RrdIy+9FP7JClsFSBaysNdnryulFzMkOBqWK80xd8BrqZocBUgQsr2RqGyQ4j3UxQaKpMkkdnC3Ghqgy9rlvF1luh0FSBC+Tfqwvs9GRvI1r/BpYmbQpOlcntaJ9fXWinLwQ2vvoGW2mNxb2/Ygdrb2/ayNZp8MZfZausUniqzC68z+u2LqyZKdg7lPlN8WPKrzj1vWaFgpCYeXkeA11S7lAWVr9imP4rHpnncZE1CkwVw51+MB0xUs4EhaUK3mm2iuX+vRhbZZkCUwXttOE+xyYsdCn65F8WPSOW7sFP2T74K1uvpu4Zz8IGBhRcW4HZz/6LDUtMoeMjtsYa+DlloXFnYLnr2m22VsNUycTCBg8jS//ecKhmmBpqIbT9yr9olhj+OVk0a7eQYdLMlBGTd/OB5osnZ0W0Jz1n4fUrlOyeJQpVlezCVdGDq6IHV0UProoeXBU9uCp6cFX04KrowVXRg6uiB1dFD66KHlwVPbgqenBV9OCq6MFV0YOrogdXRQ+uih5cFT24KnpwVTgcjt2YiNQQfJWTcbby4UYbb1u/610ajlfffJT+mb+++TdzzcsOGcbR2APJhwXwG/3/ti8eWlWGS6qpGC6oIO9CDUQXMVLine truncated
|
||||
|
||||
vLLM-Omni is designed to be flexible and straightforward for users:
|
||||
[image3]: <data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAPwAAAEqCAYAAAAxuz3hAACAAElEQVR4Xuy9B3QUV57/246AMTkpC6Gcc84555xzQgkFQBIIRAaTsQkGg43JGWMwwQRhG2MbbGzjgMceezyzO7sz7+y+8z9n98y+s+/N9/3ubZUolVpCIAkEut9zvqdbVdW3qqvv5/5+v1vVLZVKSOgR9e3p43hBpcLoF1UYO2YMtKcbQl/bgmyNmbqOmKXvAjMjd1gYe8LazBv2VoFwsA6Cs00YXG0j4WEfA0+HOHg7JcLHOQn+rqkIcEtHkEcWQrxyEOqdg3DfAkT4FSLSvwhR5OigEsQGlyIupALxoZVICJtNrkJSRA2SI2uRElWHtJh67oy4RmTGNyE7cR53TtJ85CTOR25SKz22dDk7obnb86z4FmTGNSMjdj7SY+ZRm01IJSdHNCApvB4JoXWIC6tDbGgtYkJqEBVUhcjA2YgIqES4fxnC/EoR6luCEJ9iBHsXcQd55Xe6AAEeefB3z0GAeza930z4OafBl+zlnAovpxR4OibTeWFOIifCwy4BbnbxcLGNpXPHHANn6yg4WUXC0TIcjhbhsLMIga15MDkQNmYBsLUIoPWBoI/peeXnJiT0SLp7+gQHfsxLz2PSuHHQ1zGBkZ4dge4IE0NXDruliResTH2oA/rDwSoYjtYhcLONgrtdNLwcGewJ8HVJhp9rCgLdM7pgD/PJ47BH+jLYCxEVUIzoQAZ7GYEuwV6NxPBqDjsDPTV6TjfQsxLmdoGem9yMvOQW5KUwL6Dn980GAOlRPRi0EvitHHwGfSpBnxo9FymRaugTw+Zw6JnjybEEfWxwNQ1IlTQ4VSDcr5wDL0GvBr+QO8irEIGezLkEfi78CHx/l3T4kRn0PhL0nfZg8Nsnwd0+Ea4c+jhuJ+sYOFpFwZmgd7KMIOhDYU/Q21kEqW0ZQANDsABeaPD0bSfwr7z8PKZMmAQDHXOC3QnGBiyye8B8lidFdy+C3pt3QDvLIIo6YRx2Ft0Z7CyyM9j9OyN7sGd2t8gexeyvhj0miEX2cm457OrITrBH1yM9tkEz7AS6BHR+ysIuS9DL/2bbdAM+Vh3lGfDMLMpL0MfJgI8OmI1IGfDB3sUEOIvwDPpCsjraM/AZ7Mws0vu5UpR3zaKBrxN6p1RuL0cp2ifBw6En8E7WauCZHcxDYG8Z2gU8y6ZcBPBCgykG/EsvqPDqmJegNVUbM/VsYDrTjWBnoJNn+WBOZSOWtS2jNNMfh/cewNIFy6mDRlDUiqVOHE8dOh6+BHyAezqHXR3d8zvT+EJ6ZLCTA0opnS8lsMrJ6jSeOTmyjiCsQ3Pda1i9ZDt/nhU/9z7wiQz2ZhoE5uGTa7dx8J0zBDWDu60T7s4on0iwdz7PSVxAbbDUXp3S56e24PrlGzRwtCIxlFL6kDk06NQS6LWU2tfQ8VUjMqCKBiYC3p8BX4ni3Ga8vm4XNq3diezkeg5+CIHPonuQpxp4P48c+Lplw8clCz6u6TQApqrtSGaRniJ7kGcGDYapcGMR3lad1jM7WsbAwTKaR3cHy0isWrQKrvYRPLW3swgm+INo+1ABvNDgiQH/MtXv48e+BN3pehTd7dQ1+ywvbjMjL1x67zi+/PQyjzofX76A9pbF1NkT4e4QQ/VuFpKjizpr98zO6J7LozszAz01ppLS+RLkpVL9HFmNyoIFKMtt5ZE9L6WJIno9CtPnY9XiHWitX4uKvEWYXbgUORTVm2avRmV+OwFL0Zpq819//hUH9pxAWc5iAn4Bf6zMX8oje2X+Esyrfa1zAGhFddGyTvib0TZ3A3758Seq61soujdSPd+A0qxFdOz1vI7PjKesIoGl+jUEfgXBXYqOi9fw8ZWr+OLGDRpkjiAquAQRgaWICS2her8YkUH0vj1pcPPLo9cUwI/ef7hfDg1w2QR4GgfemyJ70+wWZCWWEeTxiAjIoOyIojvV8X5u8Qj1SyXwKboT9Ls2vU7Ah/Mob2/JgA8WwAsNru6+dwJjRlE6P340jHRnwdjQkafwViY+vHZf0tqOvPQCHNv3LjxdonD1/PuoKqunx7Norm/D9s3bMa9mIZa0rKYUNgUhBDyr3SP8iuixiOrmKpw59gHam9fh4N5jOLr/JBY1v4bjh89g+6a9+PLmF6gtXYKvb93htfvONw7gP/7+77RuH4F2C3t3HcVP399DTsICAnQx/vzrr3h313H88d7PuHn9C3R8eINeexenjpyn7X7C6qU78ObrR/DNnbu4crED1698ijtffoszxy/g+6/vUjSfQ4NOI76j52to23vf/YAFTevx6fVPsXThFpyi4wom2LMS5+L8+xeQGl+H/XuO4EZHBz66cg2b1mzDv//lD9izfQ+dgwvY99Z+Op49OH34KN7cvBOH9+ynQacNW9e+TmAnID6sBPt3vY20+CJsXLEBZXk12L/zLRRlVmFl2wpU5NdgYWMbwR2JhXMXwdlWAl4NvatNiABeaPDEIvyEV16A1qTRMDGwhKmRG6xMvQl2H2xdtwnnTx7HwuaF+ODkMaQl5GHjqg1U26Zg26atePvN3aitaMKurW+hJK8RLtbx2PXG20iPm40IqtmjKIUPdMvFLz99j2+/+grleQvw5edfoWF2O25+9BnBvZ/A/QP27jxMsH5P9fQc3Lj+Oa5e/Ag7Nu3HX//8Ozas2IUlzVv5JFzT7HW4ce0mDS6v4zOC/U+//Er7O4A///E3rF+5m2DcjZ9/+IkGkW/wze2vsap9F5Uja3Dl/DV8/833+OC9y0iMmIu0uPkUuT9B/ewVdBxf4K3th7Bo3ibUlC3H+uW76P2VoyCjFRcJ+KKsJnz31R1sXL0Tp4+cxm8/36VBaB9uf3qDBrwVlPl8hN1bd1PmUImFTStRlt+E8oJ5mFe9GB52yVRC1OG9w4eRGleG948dR0JkPna/8SYW1i9Cflo5qovrUVFQy2v5mJBMOFoR8BZhZAG80BCIAT9x7AvQnjwGZjNtYW7swWfkrU39CPIT1Klb4GIXhk2rN/NavrFqPuLCcwmQZWisXYgPz52nqH0I8+cspYiWhHWrXuepO0vhowLK4O+Si7//2+84/94FxIfOxu5tB/Htl9/g2qWPKOpvxq8//YrbN29ThP0ciSH1uPXpHRzZ+z7amjbhG4rcn16/hdfXvstr9tqS1fj9l9/x092f+Wt/+u5nitQ/4ucff8bZkx/S9t/jzq1v8da247h390dcufAp5lSsxK9/+JWWf43bn32LBErnU6Ln4S5lAN/c+Y4ymK3YuukgastWYnHzG1RSrKPjrkJMQCUNUnfw1c1blD28h8ykRhzaexz/8tsfsXTBOvzxx+8oU6jEiYMncLPjGhY0rMSyhWuQnjAbzXOWUQSvh4dDKjLiqnD60FEaQOZgdftayiAOU7SnMieqiAaC/Ti5fz+/0uHuEI/lC5ZT2h/ZDXgXWwG80CCKpfRTxr0EvWmvwtLYlUd3W/MAfi3Y3IiBHwAHijo25sE8+rjYxsCNOqe7QwI8HJPoMQmeTqm8fmWpfFxoGcK8C3h0T4upxe7tB7Fy8RYkhFUiIYRACpyNaIIpNqiKauk6xIfU8MieFM5m5ucjLbqJoqX6OjubcGP1N6vVi9IXkxdx8JnZ37lJbeSFXY/ZlPZnJ7QgN3ERvwafGcdm9Bchi+r2lOj5VK/PRWpsM79Ex1J75oQwquFD5lAdTw6uo+OrQWxgLT1W8dQ+MrAc65bt5DV+sHcZAjyKEOxTggA39fV4f8pg/F1z6P1nwd89m8/Y+7qyCbxMPpnn55zOr8fz2XoaEF1t2fV49hhP5zUGDhaRcLaOJUfDjkB3tIrgwLOJOwa8uCwnNKhiwE+b8DL0p42DpakHrM18OfBslpjdBGJnHkIdkzqiNcFuF8UjkYdjPLyck+HtkkKdms1AZyLII5sAyEGoF6vf2TX3Uh7RC7PmUx1bxaN9QiiDuxqJYbUEeC1SIhso0jVyyNlMegbByC6lsfSdAcwm4gpSF6MwjWDvcjvB3s4fC7lpfWo7ClIWIy+JnEwDQWIb1fyLqJ02Dj9rKyN+AdJiFxDsDP5mJEc1UYRuohS/iUf9OAZ8SB3iOqGPJrNIH+ZTSduyQaGKX58P8y1DODnMp5TeazGC+Yx9AQI98hHomYcA987LdDQAcPNLdQQ/m8CjgdHTIQXu7Jq8XSKfxHNml+dsorkZ7ErgnQTwQoOp786cwPSJo2A4YyJFcfXNNeoZYmniKJR3QlZjutrF8sju6UQRyzmFolgaRbcMPjsf5JGDEC92o00RvwzHgI8JKueODZpNwFQTVDU0CNTwyJ4SOYfMZsubCPi5ncDP74zM7G45Nruujt55yYtQQC4kqIsIbmYGeX4Km6knJ9Pz5CVqyMnZ8cyd0JOz4hcS7K3cXcBHzkdKxDwkRzTy7IIBz03QxwQR7IHVHPho6VKdXyV3ONX33N5q4ENYxPcg4Bn0nuwOPAn4bG4fgp3Z25ldskuDN4v0BLwHAe9G0d7FJq7zzrtYfo4d2fV4Swa9+nq8o3WQAF5o8MSAnzFpNAE/mWD34zfX2Fuqo7qDpdqsEzpTBHKzj+NpPAPe2yWVX3dmwLObTlhHD/ZUAx/pX4IIcnRQGcFTQVGzilJmAp4iewKP7vX8bjcJdgn49Bh2K6wG4JPuA1/Aonmn1bCzyN7OrQZ9scxq6DM7gWewq4FvuQ98uBr4eErtlcBLsHcHvoI7zJuVLqUI8SwiqyN8EIEf4M5uub0P/X3gM+DllN4NeHeW3tvEy4BnUT5KDbwli/JB/DZmlQBeaLD0/ZmTBPwYGGlPJ9j9qbNRJ7MKVQNPNbsEO7tRxN0hsVt093FRR3d/Sl0DGfBe+Qgl4EMpwkewSbvASl6zx1N0Z+awR8wh2Ot5Kn8fePY4D2lRc5HBoW+hqMzulOse5ZmlqM6e53aCnpvIovt90LPiWNLine truncated
|
||||
|
||||
* **Heterogeneous Pipeline Abstraction:** Manages complex model workflows effectively.
|
||||
* **Hugging Face Integration:** Offers seamless integration with popular Hugging Face models.
|
||||
* **Distributed Inference:** Supports tensor, pipeline, data, and expert parallelism.
|
||||
* **Streaming Outputs:** Supports streaming outputs.
|
||||
* **Unified API:** Provides a consistent and unified API interface compatible with vLLM.
|
||||
* **OpenAI-compatible API Server:** Includes a FastAPI-based server for online serving that is compatible with the OpenAI API.
|
||||
[image4]: <data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAO0AAAFdCAYAAADv8OpOAABCCElEQVR4Xu2dB3gUxRvGlwRCh9BTCb2KVAWkKiDSiwKiCPxV7IACKlbABohBESk2pAkC0kR6SWjSW+i9Kr0TapL3P7N3c+x911I2l1z4fs/zPrmd/XZmd2fendkaTWMYhmEYhmEYhmEYhmEYhmEYhmEYhmEYhmEYhmEYJoUA8OtVqnqTj8rUfappcMmmOfyzNhXJKVJw1hxNWwSXbdo9vFK9IdUb59UYhjGHryrUfapFSIlnS2XPs6NOgdA9Imm3WSqcNfvuanmLrK0WGNxLYxgmZcyr1SL020qNhvlrmdCocDHMr9MR0fWfx+qGL7jVGqEV9Z/TtbJBF/1vVAPny8nYn6s1R818IagaWKhnEa1ITroeDMMkko/K1u1QPTDo4FslquNEszeBZz5AwtP9cbplT6FeLvVfi5641eZd3GjVD9fb9EVs63643fY9nG3Z2yFW6lLrdzC71tMQRUZny6zVoevBMEwi6BRSrmWpHHn3BmfLiXhh1GvCfBVzF5TGSpSWDWoDzOuL7Hn8sPLL9ujfqZpDDFWmTPrf2IjQnG9Pi6yVX2MYJvGUyJ63XW7/zIdr5w8Vpv1ADI076Mbq1KIEpg5vhCmRT1g03FGThz2Oazu7Aadfw8JfmuH6ru44uvJZTPu2kUOsUVPF/O8/fgz+fpkOly0V2puuE8Mw7mkrdLBx4QjsaPwSauYNwdLfWgBnegGHXsPtnS/jg1eroGeXiuhlot7u/hAqlgpEZj8NT7Ro+3vjlq3/R1eMYRgn+BtM+8/jXVExRyFsmN4WOPUmTq3qghfalHYY3pqtXLnzxGXLnn1f7wHfBmoMw7jHaNp1j3fTTbt+mjTtW3i1U3ndVN3efAf9Ph+GPoOGmqp+n32Nd78Yhi6vvomQsHCEFS/1xoABA9i4DOMOl6Y92wuBuTOjRr0ncPQOsP9aQop18AZ00fTj94C1B/9DoSJBl1s/+z++j8sw7vBk2gZPtdGNtufyvRTp0E1g75U47Lsaj/3X7fM7GAusP3YOgQUKXmrS8mk2LcO4w5Np6zdtnSLTHr4FLNq6Dy06dLadw/YdNATbTl+3M+26o2flvIuBgfl72q8hwzB2pMS0+8TQ9thd6L0onae08fgFNGvX0XJu3L40nn6yGDL5Z8ankaP0XpeaVohNyzDuSI5pd1+6hw3HzqPPwK9QskRxNG7eCgfEkPfA9QRi2jj0GfClbtj+r1bWbyHh8OsILxyArNlz68vsEUNmfXh8/LyMOyX0hv0aMgxjR3JMK3vXds93swx3C4Xpf4f9PFE3HzXt+18M0+e/9Exp/TYSTr6JIvkzI3uuQJtpZU+9at/xe8FhRd8SseXtVpBhGHuSYtoj4vx0+op1qFKjJrRSlaENnQNtTTy0OSd1YwYFh+gGlMNmW698ORa9P/5MzPfHIw8VQMVSeVAkJAwT/l6mX5jS870NLIs5dGPBgQNZ6foxDENIrGnlFd+f/pyPEqWsD1uMXgVtRSy0hRegLb8BrVUPPX3k73/aneMeEr3vplOXMOi70fDLkhu5Agvjt3lL7a4gy4tVy2IOxu5JSChAVo9hGEpiTLv3yj30/yrSYtZmYlg87QC0RRcthlWSxpXpIqZ4iZK6WdWFJil10Urek5U97K4Lt7FX3v6R92+tt3zEsleFmtutIMMw9ngy7RMt2uON9z5BtoAAaJ37iqHwKWHQ6/aG1XXRYtxGz+rGHTd3MQ6Tq8qyd5Xq/kZvVKxUCQOHj8bW/67i0C3b1ePrQi3s15BhGDvcmTZ/YFb4+flBKxgCrf/P0JZccWJWouViyPz7Lt245R96WDeu7EnlkPmN9z5G3pw5oFVpCG3A79AKhOhxE/5ernraK4ULF25tv4YMw1DaCR1q5My0eQMsQ+LBs6AtvuxoUKe6aDnXfayFvuy0Zf+Ic+K7mLlyoyWvnHmhTT8EbZnorUcshVa5HoIKFcJb73+MHNmyXQwLC3vTfvUYhrEjOCB72xz+/ocq5SmEnU1eRiWjacXwWCtUTBjsmjDtJScGdaMJ26HVbIpHHquLdz79AqVKl4UWUR7ax+OhLb1qOSdeIYbTU/dCy5YLosdHYGDgjR49ekyh68gwjAEMGOD3aGBwvWyZMs8pn7sA8vtntzftI80sJqOmTIyib0Gr2tDSwxYIhrY2wbn5ZZzozbOKofjSpUuRkJBQiq4nwzCECrkK1dUyaZeFEjbMaGeOaaV+3gDt4ToW4/60ztJr0xg5VJ59HFlFzOzZs6Vp36brxzCMEyZ81ahA+WKBH3dvVwZXdr6MgnmypNy0UtG3oY1ZDS0gK7TIBY7zlwojTz+InH7arSlTpqwUpq1K141hGBe0bhxeI1s2/7stGxRFzqx+0Go1T7lppeTFqS9nQPvziOM8q2kLZvW/2LZ1C/5mFMMklbLFC1fKlyvLy+LnXa1mM8vwVV44Si3J+77CtFrhkMs5m/D7tAyTXLJoWXNt1IKLWe69Lr5iue2TGlp5B9rwhdCCwo/6PdbieboiDMMkkor1Hn9MCys1TcscIM5Hs0PLki115JcZWbLm+Dc8vEAIXQeGYZJIoT7fNG3+ZBM8Xq8uGtatkypq0qBefL0XekTRshmGSSL37t1rGR8fvxxeQpT1XUJCQkm6HgyT4RENv49V7em8xCI8lEWYKIr4KlUR5V0UGkHXhWEyPAYTXBbG/VT8HCj/CtUSv/2E/KXEdFZnEvNyiV62lcrHm8TFxV0U61Dauh4B1r/+dBsZJkNBjeAKYeqrwiSXqcSsKzTWm4j1ui3XQ/y9JHTNmnZEGDiMbivDZAioCdI1wpPnb9DE+1w7fwZnrt6Vpo0Rpo2g28owGQLa8G2cmQGtwPP4S76Jng7YNPYl1KzUBDfpDAMbv2yCoGLNdNNOB4rSbWWYDAFt+IrYqCEo+MxP2HMpgc5KA+LQrHxpNB8UTWfYcWHtp6gZVAL32LRMRoY2fAvnMLNfQ7w5Zy9ui6mXH6uCRzoNRO0yRRFcpAiCitbFxpVj0aNhBRQuVBghD3cRtrIw5IV6KF00DEFBwQgNL4n6Lw615bpqTC80figcwSFhKN+gC16p/yhavT8HuHMKS0e+hvDQMISFh4rlyiLmYrx1qVvY/ctz6D7hMOThI+HKbvz+YRuEFyuGYkVD0azvROy8ZIm9c+Ef9K1REPPuxscMOMymZTIoNkcZubseg9sWx09bzuuTFXNp0HKVwLglG7Bu7rdoV8wPDz/eCj0+m4yZn7REsMjmrHTUjc3Im7cMnhk4GYdiVmDM241RwL8A/pOZJFxFxby5UbH9R4hZ/Qc+eroycmYLQqtv9yDqm2cQmisPJq3ciyNHtmHqhy3Q+puVOCu8GHdlHfrXCULUVcuqbR7fA6VzhWLnieM4tnspcmQqgI5jtuBigozdh8/r58PPV+JjfrzJpmUyKMqnRs4tGISGBTTs0y/6XBZGLI93l57T513a+Qc6BflhtfVU95/IjiiWORyx+lQCDh/Yi107tiFq8SyM7N8RZTLnwRVcx9/9G6L3zL24Y1kMuL0XBct1xZwT/0LzD0erj//Eju3bsV1oR8xi+Id2wsy9N7C036MoUKiyWgo7ZryLGnkz4/XIGdhw8LRIice9e3FIEKa9d3k7PqmdD7/cYNMyGRibGwysG/kCimsB0PvZu6uRv+LrWHHpnpi4g4OL3kc5/8LQ760IfnulBnLkeRLSsJd2z0eVMsUQHhaGsuXKoGRwLmgBFUUXuBlftQzB9Jj7l37jrq5GyZaRiPl3BzIF5EZIibIoWbKkrtJlyyLsiQ8QffwWvm4ahoIlnrUth4QziBrdE/kDNGQNfgT7r6iBuRhIn5mPl0rnxyIeHjMZmftuuM+g5kWgBXXUf1/88w00HroV1+Vp47WD+L1rBAJK3X+WokNpP4R3+Q1357+JMvlz3O9JD89Gt5IactV8TRh/A4a0K4rx2y39Me4dwYTuFdB16jncPbcdOav3wJwLasEE7Jo5BH/tkbeAgedLi3Pmpt/ov+PP7kSn93/FdlXIv7OghXbF3H2WsfOeMW1RJF8JxMXH71jA92mZjIq1+dvRpWwmZHt8EOTQc9WnT+DtJaeh97Mn1+DdatkR3KCvLbZGzmxoMiwatxb2Qrm8WXFKBgq++V9lZNH8UO7pwZBD7DWRHVD9peFYsikGP/ZriQjRUw7blSCvHiGrfwhaD1qAy6f3Y9XkfqhTJDPmH7EYUZo2tPlw/XfChS3Qcj+C92bs16dvn5qFLBV7Yckxy42gb5sXRb6yHeUtn50H2LRMRkVv7YSxg4di+vpj4tcprJwyFidiLU68emIVfo0chT9WHbFGXkXk8D+w+aTlBPfSrvlo07QJmrXvivUxe7Doh4/x1bgl1lhgxe8j8eWgz7A26js0L14IJ+9a0mNPrMPU7z5B15d7YsD307Dh5P27sUObRyC46lu26cOrfsfgPi+iYaMWeO71j3BJjY7jT6CgMOzEffJ6t/5U1E66rQyTIbC5wSSuXbqEaxbf2Di1diI+6NrdNn10THuE5iqCe4m4Bbykb20EFWtBk3H+0nXbbSbJqXn98NyoTbaHL+RjjXRbGSZDYGj3qcaFpV+gjhgOf/LzdPw1bwYeC82OQo/1oWFOOT2vJyqHVqTJ9pxZgj71y2IXecSRbivDZAjsm3kqEheLzevWYNWq1Th+4Rad65qEWzi6YbHLxxcTYk9g3cIFWHvQcuHKCN1WhskQ0IaeXnE5kpY3aF1At5VhmBQQpWmBUjSdYZh0iDRrtKZtk2LjMkw6x2pYGMXGZZh0iqGHtTOtTPtb0/IZYxmGSWOc9bBUbFyGSSe46WGpuMdlmLQmMT2sE23TF2YYxvsI07YVGigVrWlHox0NqnRUxUnZMmAYJu2IFh6OdjSrUtT9SIZh0gXRbFqG8S2i2bQM41Line truncated
|
||||
|
||||
# Interface design
|
||||
[image5]: <data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAR8AAAEwCAYAAACOrXWuAAA5IElEQVR4Xu2dB5xU1fXHH21FsSCisrTV2LBFMAhG/EfFilEES2LDQiIREZKoUdSIGI1ExW6CGAEBFYMoTXrZoQlIbwssbQHpZWlL3z3/d97M2b1z3ntT9s3dmTdzfp/P77Mzt5z35s6737n3vrKGIRKlkHZt2wt+t6lK/HOJRKIUF+/IfrTARyTyoXhHdvLu7ftgz84DtvRUscBHJPKheEd2c+GO/ba0VLHARyTyoXhHdnJg0jRoddvtsGbleji496gtP9kW+IhEPhTvyE4e8r+hYBaFvMUr4ciBYlt+si3wEYl8KN6RyTjNKtwenGp9/+1wyMrKguVL8i34qHmpYIGPSORD8Y6M3rGlEIYOGQGjR461Xg/7bmQQPktXwYE9R2DQV4PNvHGwd9dBW91kWOAjEvlQvCOjt2zcAS2uuRZa33GX9fqHYaMt+KxYthp2bN0Dl15yqZV39GCJra6b9+4qgn279cBK4CMS+VC8I6M3b9gOJ5xQHc4551zr9ajhYyz4rFy2BrZt2mmt/2BeyVGw1eXGKdpOE1jjx0yC774dbsuPx3jKH83TBT4ikQ9FHRjXcMCEyeEDxRZwatQ4Gc4773xX+GBe8RGwAFRyJBxCCBuETvFhjHcctm/eBfUbNLDq4TQO141wW04gcTOeaZs8caq1D/y0v8BHJPKhDu8/bk2Htm7aBT+YkJk1Y27s8DEBgms/Y0dNCAMJwmHrzzthwtjJsH7NJgs4F154kVUPy82aMcfaFk7pOGRU7y88BIfM/cMp25gfJkDbu+6GwYO+M9MPQ9HeI1Y+xhP4iEQ+1EMPtoOB/QfBqhXrLDjc1PJmCxyxwMfq9ZUqWe/xCmha10GgLVu8ErKqZUH/fl9agLjookZWOQTHTTfebL1esmC5VQ/r0GjGOpNmGtM//7QvPPLwozB39kLo93l/qFa1GrzdoycUHy+BF194GT75sBeszd8g8BGJ/CgERFezI6/NX28BAS8mLA98cCSCp+Q/793PGung4vSpp5wKXw0cZINPq9t+G7puKB9mTp9j1ulrgigPdmwutEYz5Kef6gInVj/RHF2Nh/59B8JJJ54EPd96D0pKSqDZVc3hkXaPwdKFKwQ+IpFPBX97rmspfG5v9du44YPGNZxfXdm0FCqrlq+D0049zYTPNzb43N7qDut1ft5aePnFV6z6A8wREq4VNWncBDr88UlYtmgldPxTJ6hSuSqMGjHWBp8mja8EHLUJfEQi/yph8LmqaTPrdTzw+ftL3aw6A7/42oTPXrjil1fAH9s/AUsXrRD4iERprsTA5xjAxY0usfKWLFweM3z+9uwLVv3PPwtO1x5/tD289eY71tktgY9IlMaqV68+vP6PN83OXmAB4Zabb40bPvger3Zev3YTrF25AYr2HLHuA8P1mgH9v7LBB7eBr5eakNqycbsFPtwmTrtwoZkWobs+/xLkNDwHJo7LDYdPcQncdmsr6Nzpz7B8ySqBj0jkR21av80Eyi7rlPjdbe6GHv982zoFHit87v/9g/DgAw+VXgCIxsXidas3Wms30wIzYee2PaXwQbC81aMn3NP2XutMFZ7ZcrveB/cLt4+v+5gjIwQd7h+eTUNYYT4CS+AjEvlQCAoEwvbNu2HKpOmweH5ezNf5lBwHmDF1Nvw4bXYYNBAmeGHhnJnzrfrbt+yG88+/wKqH+TjimTp5hgUPDhzVwRHQIQtQP/04D9547U0ImPu4zxxl4Wn9PTuL5Dofkciv4tBAxwyf0FXKbiMXTEdI7DDB1vHJp+GO394Zth1ePpKxfOGOA471BD4ikQ/FOzIagZOoe7vIa/LXQ37eGlt6IizwEYl8KN6R0TgdwsXcl158xXo94vsfgne1L10Ne8wp0BN/+JOVd2j/EVvdZFjgIxL5ULwjk+k2B3xtf5jYAdvNncm0wEck8qF4Ryar6zJ2+LifoUqGBT4ikQ/FO7KTvx88DKpUqWJduSzPcBaJRAkR78hOxpHOgT2HU2q0o1rgIxKJRCKRSCQSiUQikUhUsco1jMfQPF0kEom0CaETMAxATzKMx3m+SCQSaVHAMEoCIfigcwVAIpFIp0IjnjDwkAVAIpFIi9SplpsFQCKRKKGKNOLhFgCJRCItynWBEBiGXI0sEon0SeAjEomSIoGPSCRKigQ+IpEoKRL4iESipEjgIxKJkiKBj0gkSooEPiKRKCkS+IhEoqRI4CMSiUQiUSqrQYO6IBanm81DuzI/1kUpJv6licXpYEPgk/riX5pYnA42BD6pL/ULg4AhFvvWAh+fSeAjThcLfHwmgY84XSzw8ZkEPuJ0scDHZxL4iNPFAh+fSeAjThcLfHwmgY84XSzw8ZkEPuJ0scDHZxL4iNPFAh+fSeAjThcLfHwmgY84XSzw8ZkEPuJ0scDHZ8oE+Gz+zoBVXxmw7ht7nhf3ed6AKy+0pyfb119hwKz/2NPT3QIfnyld4XNskoEHoKMHdbOXL491wWftIANqnWrAtqHB97gd/hnQWdUMODA2vO6h8cG8NtcG3/d1qUvG/X/29wZ0e9SAklz7vvjJAh+fKV3hc33jYOe6/BcGFIU6KALp1qtSHz4Ig0qGCYPQe9rOgs+DI7gRbxrQ+e4yeBwYE15fHeHhZ8Y66OUDgnXoPY0GEVgIMr+PlgQ+PlM6wodGCjw90dYFHwTP16+UvaftcMhYZSsFP2ssoxYaFRHUVOfUCQKIp/vJAh+fKd3gM+3jYAeLdXSDUxwsr9qpE459xz5d+aiLHT48HsZSQYK+oYl9m+ooB6dMKkwiwYemaDRtoukmvudlI8EH2w1B5rQNv1jg4zOlG3xo1LNtmD3PybVOCU7Nhr4RnIbQdO1rBV7UMdVyWAatwodAgGnqNEeFGcGpR4eyMjiFIiBY6y+PhQMiEnwQNq1blAGrvPDBPBz9+HnqJfDxmdINPndeE+xgtM4TyQgVp1EOjZ4IQAgeXo46vQofp3IIDkzHxWOCGB8Jkd0AEAk+aAQW5ZcXPhQH980tP9Ut8PGZMhk+z/zOPm1CUyd99bHge1yDcSrH13ywDi9HwEH4EIjcIMLPcvHtONVTIZgI+Pj5rJfAx2dKN/jQtGt2L3ueaqvTXlN2SprnYQzMU1/zcip81FP711xm96xeQUhgvlvnLg98qA6NpgQ+Ah/fKN3gg50PRz98+uNk7NQ1qtvTqZPStAtf83KO0y6Hcnx7NArieejyTLv42a7ywoc+j0y7RBWmdIMPmtZsln5hz1NNi788naBEkKAOzuvS4jKlOZVzqoMLyjwPrQJATXeDz8ddgttT96G88HEDn58s8PGZ0hE+aJp+kXHqQ6/V0/A49cI0HCk93qqsjNpBCRqY3qihAb+6MNjh+al2tVyHO4Om2FSGAIBp111hwOmnhG8PwclBwz8LGWNwkJQXPnglNMLTr1MutMDHZ0pX+KjG09nje0ZehMb7vxb2safzMlM+jByHPPqt4Kl0nk7GPLdYCAGnNSadRmhGmjL6wQIfnykT4OM3V/TVxjRa4tM9v1ng4zMJfFLTdLEhT9dhhJ3bIrifLPDxmQQ+qetEPwLEzZGmh36ywMdnEviI08UCH59J4CNOFwt8fCaBjzhdLPDxmQQ+4nSxwMdnEviI08UCH59J4CNOFwt8fCb1CxOL08WGwCf1xb80sTgdbAh8Ul/8SxOL08GGwEckEolEIpFI5D/xMwterDO2ZcOohHFLAkaJLc+LIRhX5CyzvYttbebFIFMnkSnbgeHBOmNb9jl8SnL17rct36NL91vgI9Ih24HhwTpjWxb4hFvgI/KzbAeGB+uMbVngE26Bj8jPsh0YHqwztmWBT7gFPiI/y3ZgeLDO2JYFPuEW+Ij8LNuB4cE6Y1sW+IRb4CPys2wHhgfrjG1Z4BNugY/Iz7IdGB6sM7ZlgU+4BT4iP8t2YHiwztiWBT7hFviI/CzbgeHBOmNbFviEW+Aj8rNsB4YH64xtWeATboGPyM+yHRgerDO2ZYFPuAU+Ij/LdmB4sM7YlgU+4Rb4iPws24HhwTpjWxb4hFvgI/KzbAeGB+uMbVngE26Bj8jPsh0YHqwztmWBT7gFPiI/y3ZgeLDO2JYFPuEW+IhEolgk8AlZ4CMSiUQikUgUr/j/1RGLxWJdNtT/28UzxWKxWJcFPmKxOCl2hU9pokgkEiVIAh+RSJQUCXxEIlFSJPARiURJkcDHX2pvep7pk3hGEoXHRyrtj6rhRnD/SFVD7yvktgtRZAl84ld1I3gAR2oXyufuoxYqhzIBPueaPmD6LMPeftw1QnXcVJHw6Wn6NZ4ocpfAJz7hwTvC9Fum15u+Pzy7VNhmCAlVWN5rR013+FxruMcrMMJBEosqEj6oEkNf7LSTwCc+4a/ybtPNTL9r2AFDcoIPggPTz2Tp8Sjd4YNt6na8FRjueW6qaPgUmG7OE0XOEvjEJxq9oGj65SQn+FDHihU+txrB8mSM1zn0V+3sCES13BHDPiLLDeWpVoVA5fntwkoExcvhvuBfDp9doXR1n0gEAIIxmvYXXw8NveYqMOz7jSMNvt/qVCwafPA9llH3D9uT7/8DSj7FVPefhCM3bJNo00GRYWOMwCeKeOfAzsg7OgrLEXyyjTKQLC4tEVk4tcPyaqemKYkKHyqnSk2jaSKuoVxaWqJMlM+BRp2UPoNbOT5NonJqG/E0is0hg+kIARxVOqnAsH9WDj0EvAr9aPDB1075fP/VOhST7z+KyqqwErlI4BO7aIShwoZ3NBId1KpxsZl3FjdhefXXGOUEAKdy6vSO4OAESBTlO3V42m+UWznqbLQ/buVwnxCAKKqDC8qqcCQZL3y41O2gIsGH2ukRJR/3H/eBT51whEX7SzH5/pOwrMAnBgl8Yhd2fBy5tFL8pBE8ELuXFbOEafQL/GbofffS3KCcpkvU8fLine truncated
|
||||
|
||||
If you use vLLM, then you know how to use vLLM-Omni from Day 0:
|
||||
[image6]: <data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAdAAAAHUCAYAAACKzHkWAACAAElEQVR4XuydB1wUW5q3Z77dmdndibuzOzM36DXca85iwAiIZDEAKqJiwpxzzllERUUlGlCM16yIIkGSmHPOCkoGQQWhn6+qoBXLcFFE6dvn7+//6+pTp9Ir3U+/p8459ZvfCL0hhIRKWNnZ2bx8+VJZzs3N5cWLF8p7uTwvL08pf/78ORqNRlmfnp5OZmam8l5ISOhtyZ8X+fMjS/4Mye+1r4XryGXy5ygjI0Ox9vP2qVLzQ++lDpCQkJCQUOmWDEctLGVAZmVlKa8yVOUfqNrXzy01P/Re6gAJCRVX8fHxDBgw4JX79++Pi4uLstyvXz/69OmjlMmW3/ft2/eN+sLCuugRI0aQmpqq/jiUiLTZpjYTlQEqt97Ikl+1y59ban7ovdQB0kXJf0zyH5Hc7Kdrfvr06aummF+LLl++zG9/+1v5w8Zv/+23b/n/vaNMWFhXLf89y3/rf/jDH7h//77641BikjNO+btP+/2nvU1SklLzQ++lDpAuKjQ0lI4dO9K5c2ecnJx0yl27dlXO+9ekK1euKAB13biAiznnOJ15Qlj4V+urXGLEvGH8+c9/5sGDB+qPQ4lJBqicgebk5CjNtV+iz4CaH3ovdYB0UcHBwdjY2CjNJ+oMr7T71q1bdOjQQX1JOi0tQGd5Ticq4RhhD48KC/9qHZsazcAp/b84QGXJ2eeXyDy1UvND76UOkC5KC9CSavcvSd25cwc7Ozt1sU5LAFRYn/w1AfqlpeaH3ksdIF2UFqBf8pfY59Lt27cFQIWFddgCoHosdYB0UQKgpUsCoML6ZAFQPZY6QLooAdDSpXcC9NFRjidHEpMkXBxHJ0YQ+iD4rS9x4a9nAVA9ljpAuigB0NKldwH0yK1DjJg9nDELRgt/oscuHCPFdIYCUfWXuPDXswCoHksdIF2UAGjp0rsAeujGQarWrEblimWoWa2C8Ee6Xq2f+Nc//k8BaYyUyau/xIW/ngVA9VjqAOmiPidAk0KXMNRtL68mwdLksX1qJ2ytLTA3N8fcypmDd9LIS7nF9c1jWRKRUXjzj5Y+AbRilSrMneLA7TPzuXl6nnARffvCQk4cnUK92j8xZr4AaGmzAKgeSx0gXdTnA2gq8xpWpYbLWM4nFky6nJfNINNvWHjgLklJSSRHe1Cr6zye3TvBqWnVaLs+4c1dfKT0CaAVqlTGdZajFGYvSPYULqqz/Lh9bj61qlcUAC2FFgDVY6kDpIv6bAC9tgCjgQcInt+Pqfsv5pdpshlsVRH/ywV1EnZT3nAgcfdPcGJ2Xez8E19t/inSN4AumumIRoJCbuIa4SKaDB8lExUALZ0WANVjqQOki/pcAA0d1pZF16SFuI00GhVQUKphYvPfU7ZKHerXr0/tCoZM33mWvIcniJlcUwD0HRIA/bwWAC3dFgDVY6kDpIv6PAB9zhKL+rTuP46xg2z4t6qjySxYM87oTxhaOdGjRw96uszlRhq8fBBLxIQaAqDvkADo57UAaOm2AKgeSx0gXdTnAOiz85uoZupIWFg44cei8OpanaGhGuKfahhsXYG1Z9+cqDn73nEiJgqAvkslDVBSvCR7okl6XaYpKMt7R33ZecleSn3tq1KW9P76Rbenck/3ffspfLxPtQBo6bYAqB5LHSBdVPEBmse+6daYj96NFpPJO50oY7qMVZEPGWD8f7gfS39ji+w7UYSNKoel35M3yj9WAqAfYQl4pMxniFFFatp0ZWP4YglOEjhfrGShWQVqm9lwN2WD9GvIR4KafAypfoavtLyGwJkmTN68jCPjqjLxgC+axJVsXuTIweilkLUWnkr1nkuvadL+Mvykfa5VgJybIME51Sd/XZZUJ2l1wfnIx10PcZMwr96UMKkeyd7528p1n8n78GL/gDqM3bYcTdqaTwapAGjptgCoHksdIF1U8QH6krP7fybibsrrouzb+PsFsCM2gaj967kYn/N6naTcjMfERQdw8Fr+U+E/Vb9GgN64caMEATqXYa1McRnUmJ6+EyQw+cGpfpj3bUU/q3Y8SliE18whHApfhCZhOfu8B3D+6gqOLLRiyvTuOLUsi2FvZ+Zvm8/pXSM4d8WNkOWd8VjsTL/e1uw+PJsL3m1x7NSW4FMSXJ96kxY9mvE9jRg+xZm4RAnQqTIoXVk6xJypY81pVNOUiBQJsremcCOgA10djRkyrBtxEmAPD63PxB3LJTALgP5aLQCqx1IHSBdVfIB+PX1tgEZFRSnPU/1cDg8PZ926dfIHjakrJ5cAQOcw2NgKD3czWjo5kpjlzfHZ7Vm03YWhZm15Ej8R2zo2zPGfDgluTO7Zit0xywieZ8Y0vzlsGlSVIf5LeXB9Ca4DWuMfPouxBj8wZMUM7qyz4O/1m2M9YxSn17THamBvsm4toK9pTdbtm8XOsY3oMGUEEVJmu2tQQ2ymjOXBvu40LGvAqcduuHczwqKNA9ceeRLQryoDNi4ickwDAdBfuQVA9VjqAOmiBEA/XXXq1GH48OGMGjXqs3n06NFYmFkwfM4wohM/P0AHtTDj5+hheHRrRMAJN6YOsef4yeEMMLWVADoJu4a2LNg0AxLdmNHPnL3HJYDObc30ADcOjq3BhL1evHjkjttgczZJAJ1pXZNNZzagOdeXzrY92H9zBZlH+uIwSso4Tw6jdydbHjxZT3rsCOyc7Ri3fhoTLVsRdN8D0ucysLmZlIFKWentedzbNwjf1QOZ37MOvX3nETVWAPTXbgFQPZY6QLooAdBPVyuTVpTEk+zX+a1jwOR+JQTQ1kTfXEBqYDsshndj9BAH7t4YT/9XAG2H65aZEOfKGMfm7It1/yBAF7Srzf5rW8g97YKdTTd+vrycp4ddsB/pTLwE5j5agB4fTofuEkA3TGOCRSsO3csH6KDm5sQ8dGV+t6aYmVmxedtoVgyo+8UBGh4X8laZcMlbAFSPpQ6QLuqjAZp8Etf5c5k7dy4+O6O58mrevqIr4WwQ2zcGka5JZe+2fdzPLOKxVfraADVtbcqDU0GsmJsfD9lu7h4cPH2fW+H++PwcTHrBpEyy0q8E4rk2kPTn+dDNOn+QhfPyt1vitQ/tneI1q9YoXyolBdATcW5w2YUKteszakI/Eh9NUACaEDcB0xotGL5aAmjCfBwb1eHAqeWvALpvdHVmBq+XYLaCxYPyATq/bW32Xt38ToCmnJ9Cd6umXH2yibjtDhh1cWRp5BJ8OzfHLXIVxE+mfa1mRF+fSddWjWk0aJJ09Ts5NKgevdbNJ/oLAVRuKs9/Yot4WsuXtgCoHksdIF3URwH0xgbqGdRk0ro97NmziwltTajWcgZx6nq/oFOzzKnwp9Yc1zzj8rnLpGYXosxH6EMAffbsmbros6uVqSlPbpzh8B45HnvYe+AQQ1r+D108b7LH5U/8ppw1gTde/8LY6vT/+M333bmWlMuj42uoXMmGTT/nb+vZrxF1Bi4jVarnWWIAnY1LQ2MiUj3JfuGK44/fM2WzG5rHY+jZxIK7GX5sG1gF40aVsbNqhpmUKe4/sYygmcaM3+DKmTm1+d7WgZFe01k5xJT1oTOZZVGdXVekDPRUb2xad2HbJQmgh6TlIV3Jlo57YEZrjBpXp0XzOvhHrObp07VknexLk5o1adOzGVY/mnA8dTmbhjbHosyPNG5hyqLRTbGfMYHDo+oxZmsJ9cJ9dJSI+DBOpMbgHehJ7ca1mb92LtEJ4mktX9ICoHosdYB0UUUHaDaetpXovOX1/LWZ57exc0hbdibB4cGGlKlUixZ9l3D16laa/LMyVSpXpUyF+niEPCYvL5WVg834oVJVrJsZ8be/dORa3i16tOnJsYQXnPWfSJ3vK/FjxXK4n0gmowhMVQM0JSVF6cXq6OjI//7v/5ZI82phmZiY8DRTO2WElJxv6kkV65XIfYsPjCxPjTZjcPeP5lyuVJAdSpfydjQd3p/Lj16wpHsdZuwr9IXxIo5B5ob4nkzEa3UJALTAeYW2l/eVpx3XWVAul2lhpayX18ljPuVxoIXGfmqXC5dpVMvafSjHS5L3+3qbwsfQHl8pS8hfVs6r4Ljqa/gYqwEqf2HLEN17cRdjFo6ikXFDpdPWv77/lxJr0ZT7ZS0AqsdSB0gXVXSAnse2ohVHlHbGNDZPGMGAYWMZM3sZUfEaDvc3oM7c/ElvT65zJzI7f6u4vRPpNHobT66tpm7t4TyVyoLGNaPh/1pyJe+2BNBehN26Ri+Diux7ApqTrlRq5MzAQ2naA79XMkC7dOmiLK9atYoGDRrw7//+78oX4o4dO1S1P79MpQz0aUY+QNNPbKBpo04cLxjyumd4JZwmzMdqiBsjj+WRsc0O28WH6e/izPn7DxloWpvttwrtjBy2jnJgxIYzeHt6lhhA9c2FATrRbQKhD47Qb6ILZcp/r/ydaL1400IuZJ/lVMZxybHCX8iX8y4wfM5Q/vKXv5Ce/uZ48V+b3qSHkJ4B9CoOlRsRoMx9kMOjK5eICZiLm93/MjRSQ8iAegzcW5CNnVtJjaqVqVypEuW++z86uYVyP6A/9R39kI9ybdcM2lfqLgFUzkD7cVjKWG2qWXH2mQTQ1Bt0tbfEbPWd14d+j+7evav0hJXn2ZX/O7SWP4z29vZ0796dbt26vdfy+o+1s7Oz4l69elG9enWyNXmk7B7EQGcpEz//OhvdPeRHxu85RR/HmfRfuJtFFsasO3mZrt2duPAgjfF2NdlyvaBynpwpZ+PV14IRARcEQD+jtQCtXeNHuvRz5H/+8T9v/K3I/v1//B4zu9ZYOJhJNhf+grZ2tKJK7cr87ne/w8HBQZnyU/sZ+1RrP6elLaMthA4hWeoA6aKKDlCIHGtIpYFb0d5dTAycxhSzPzMkIh+gQw/kSKUJ9P/ub7jdz69zcmVXnKbsI+GCH63Npynbxiy0o9m37QoA2oLine truncated
|
||||
|
||||
<p align="center">
|
||||
<picture>
|
||||
<source media="(prefers-color-scheme: dark)" src="https://raw.githubusercontent.com/vllm-project/vllm-omni/refs/heads/main/docs/source/architecture/vllm-omni-user-interface.png">
|
||||
<img alt="vLLM-Omni interface design" src="https://raw.githubusercontent.com/vllm-project/vllm-omni/refs/heads/main/docs/source/architecture/vllm-omni-user-interface.png" width=55%>
|
||||
</picture>
|
||||
</p>
|
||||
[image7]: <data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAQwAAAEbCAYAAAAvX7qwAAA00ElEQVR4Xu2dCXgUVbbHC5AgoIgITkIIwYERFB0RFxj1jeuMy3MBlTcgIouKKIIL7jMjqOMyioyCDqCCAgqIIIiyCzQgAsq+LwJhFRDClrAn59Wp7pPcPlXVqXR1JVWd8/u+/5fue2+dqq6u8+97by3RNEHwmH27D0IQpVOOfxZBEDyGJ2JQJIYhCKUAT0Su7D2H4MDeHNj/22FTXWlKDEMQSgGeiKrQLDZv2ApzZ82DxT8tgwO/5ZjalJbEMAShFOCJqOrA3lwITZ8DXbs8Dm+98Q4cOXjC1Ka0JIYhCKUAT0RVB/cdgTGjxsFlTS+Hdm3bw/GcPFOb0pIYhiCUAjwRScdyTkGu3qP4+qtv4IrLr4T27TrCidx8yDlwDI4ePgl7dx0wLVOSEsMQhFKAJyIawq/bfoOrr7oGxo7+Fr4bN9EwjA4PdILfdJN49ZXX4c7b74ITR/JNSWwlnAc5lH3EVO5WYhiCUArwRMw9eBx2bt0DVaueAYM+HgITvplkGEbH9p1g94690PnhLlC/fgPIPwGmJLYS9kT27Mw2lbuVGIYglAI8ERNpGAf35cKUid/DZZddYbw/lH3U1KYo4fAH4/ByMQxBKAWOHT5lDBl27dgH3+nmsH/PYUeGkacbxsRvp8DkCdOMYQclMl6vsXTRSuPsypaNO+CrkV+DvprI9Ry5xjpwyMMNgITtjurbhGdk8PqPlnfdDaNGjIHD+48ZZnZ4/1GjjRiGIJQCbe9rB8OGjIANazcbib12xQbYtX1vkYZhZGy5csYymNjYC8C/aEBdH30cMjPrwZBPPzcmTbENJvz2LbuM1yuWrDHaolHRBWH4F7Vx3RZ44P728GyPF+DUsTyoeFpFePvN3pB3Kh9efP7v8MH7/WHT+q1iGIJQGqRUTIEX9ETctH6LkczrVv5SbMPAX340hmmTZ8Bvv+6HJ7s/Deef3xC+GDYiyjB2bNltvF69fD3M++Fn+GTgYN08Vhtmgz0H1Mqla6Hy6ZXhkj9eApAHUKVyFej97z6Qn58PV17RDB5o18FoI4YhCKUDPPvMC3EbBgr04Qleq3Ff67aGGTz1RA9o2LCRbhgjLQ1j/epN8PcX/2ksO1TvhYwbMx4ubXIpdH6oi7EdFcqfBhdecKHJMC5t0hSwRySGIQilR0IMA9u0bXO/Y8P4x0svG8sO+2w4jB093uhRPNTpYWO4IYYhCP4FntCHEG4M40TuKXj6qWeMxMaJzqIMY62+jqGffQF33dkCpk2aYdyvMnXSdFjw4yJYtWyd0SazbiZAfrRhNNSHOffc3cqYAxHDEIRSID29Drz26huwcV1WOJnjmPTES8i3bNoB2zb/CrkHjkO3rt3h97+vD0OHfGFpGCuXroFft+0xTArXhZOdOAmKWrNivW4W9eCG626A/JP5hYaRlw+33HyrHvsJvc0GMQxBKA0wiXfv2Gf0DO5ucbeR9Hja04lhtP7bfXBfm7bGaU4STlwOGTwMnu3xPMwJzTPuRdFXY5gBToje0/JeY9iBJqGejiXhhV54WvdX3UjwqlM0pTdff9s4+4LmgtuKbcQwBKEUUK9/OHLohCEn12HghVt4dgTFkx7PeuToPYocvbcxauQY0PSkxyTHthjfyijsdCwnzzj9yu9dEcMQhFKAJyhd6Vmp0unwyUefmQzjoQc7Q7165zm60hOFvYL1qzcar4tjFEVJDEMQSgGeiGQY5cqVh4H9B0UZxq7tv0EH/W/t2ukAJ50ZBg5Fjuc6u1GtOBLDEIRSgCciCec09up/6fb2Du06GvMIO7fuhq2bdia0txCPxDAEoRTgiUgybkvfdzTqeRj4AB2a3OTtS1piGIJQCvBEVIWnS7+WJ24JgkDwRFSF8w/zf/gJev7zFRjw34+N05y8TWlJDEMQBEEQBEEQBEEQhEQxU9M68DJBEAQTIU3L1wUoViUIglBISDELUlQDQRAEBIchIWYWKL28Y1RDQRDKNiGLngVXQWNBEMouIQdmQYosIgiCYJiHySR0l5ArMwVBMBMSwxAEwSkhMQxBEJwSEsMQBMEpITEMQRCcEhLDEATBKSExDEEQnBISwxAEwSkhMQxBEARBsCEjozaIRH6XfqiW58euUArwL0Yk8qM0MQx/wL8YkciP0sQw/IH6pUBIE4l8o8y6Yhi+QwxD5FeJYfgQMQyRXyWG4UPEMER+lRiGDxHDEPlVYhg+RAxD5FeJYfgQMQyRXyWG4UPEMER+lRiGDxHDEPlVYhg+RAxD5FeJYfgQMQyRXyWG4UOSzTDWDDX+gVCUFvQ3tyuuBj2nQdPzzeVuNKefBuXKaZAzKRxf3earLtKgzY3mZVA1qoW3BZcbzJbjmtpbg7uu0SB/pjmO3yWG4UOSyTC63W1OGNSIl81tiysvDAMTv+rp4dfcMEgpFXVjmBy9XGaqBi0iJlCUYaCpoCm93N68fr9LDMOHJIth3HFVOEGGvmSuS4QSbRjYu6BeAr6n+PSe1K97YeLzGFxHp4bb5rNyNBU0jaD1MsQwfEgyGAYlCi+30/Q+4S4/6q3O5nrUzjHh+msvCb+3MoyT0zW4+YrYcbDXQ3E2jywsx97F7rGF7+0MA9Xjb+Gexvz/ht8PeFqDsf8yt7MzDIoRtF6GGIYPSQbDwGSjrn1RqnGmBhf/XoMNX4R1XROz2eCvMZZhUmIbfI1SDWPTiHDZkkHhNndeFU5qXv9m53A9zq307FCYzOW06MSOZRhoTBgLhyH4/s6rw8nPewyxDAPr1O0LgsQwfEgyGMbT/2f+9bcSJZ46J8DL6L0ajyYnqQzbYNJSAqvLURJjvZ2JYVuevE4Mg9Yfj2EUbJ9FnV8lhuFDksEwcP7CiWHYGQtOIuKvP74uSFw20agOSXr8X8RkWHKjqdAwg85+8HWhsPfBzcRrw0AFbR5DDMOHJINhYLLxX2wuo1dwVXSvgETlmGjqa74ONWH1XVcwD6Jqfn9zb4OruIZBw5vh/yxcvxiGUCokg2FQghZ1vYXdXAf2MIZHTr1iLwTbqBOSNAQhw8A4OAehtuFSextcVvMJsQyDTouqw53iGoYMSYSEkAyGgdI/ipHoKz8z15Hol5qXq8mP8xXYhgyElqOLpagNJrHahgvbv9zBXI6i5FXL7AzD6rRqPIZhZVJ+lxiGD0kWw0DpH6dAODyg1+qFWzjkwDK8irLjreHXnW6LjoPJjuWN6mpw2fnhRO7bPXr+g8wHk7DzHeF2akJS8mIZnlI9+8ywgRScJWFzHGgY6vaTcHluAPEYBl6LgVd88nI/SwzDhySTYZDwGgq8JDqXTVyqmvW+BksHmct5m1gxUHjKdOK/7dthvVUcNAi6WpMvk2hRD8luiORXiWH4kGQ0jKAIexlWk7CJlnpKOEgSw/AhYhilJ7z6kl/AlWjhMIVP4gZFYhg+RAxD5FeJYfgQMQyRXyWG4UPEMER+lRiGDxHDEPlVYhg+RAxD5FeJYfgQMQyRXyWG4UNUwxCJ/CpNDMMf8C9GJPKjNDEMf8C/GJHIj9LEMARBEIRSJT+k5fPJs3jFYwuC4ACeSG7kdVwxDG/g+yZugQwtkh7Tl+5CXscNkmHw9cUt0Mp5ErdnYXKb6uKVGEbyY/rSXcjruGIYWuLiimEI8WD60l3I67hiGFri4ophCPFg+tJdyOu4Yhha4uKKYQjxYPrSXcjruGIYWuLiimEI8WD60l3I67hiGFri4ophCPFg+tJdyOu4Yhha4uKKYQjxYPrSXcjruGIYWuLiimEI8WD60l3I67hiGFri4ophCPFg+tJdyOu4Yhha4uKKYQjxYPrSXcjruGIYWuLiimEI8WD60l3I67hiGFri4ophCPFg+tJdyOu4Yhha4uKKYQjxYPrSXcjruGIYWuLiimEI8WD60l3I67hiGFri4ophCPFg+tJdyOu4Yhha4uKKYQiCIAiCIAiCIJjgjxQXiUQiVZr6bwZ4pUgkEqmyNYyCQkEQyjS6H+SJYQiC4AgxDEEQHCOGIQiCY8QwiuZdXcd5oQLuHxK2GxpdXWw6aeFYVXhFKZDI7bhG1yIten9xYX1VWsCC07RwO/4+6qpKl+TrGssLhTBiGLE5XdcWLfog5fCDHjUoqkXxSFbD2KfrXM28r1T5wTB6a2HTSGTMpEEMIzbjtfABiabRmtURPKnQZHhZcUhGw6DeBSdLizaAoigJw0DwO5BehgViGLGhXz084O2GJVZJhWVXsjI70nTNjaiFZm8YXbRwm+ct6pAbtMI42EYFE2twpC6kWS+P20HrwO1AYm0HSq3DdQyMvKZtQc7TCnsXnCzNbBgYZ7IWvR1qXVGG8Y6uD7TCnooaj+8X/Cy0jkm6mil10suwQAzDHjSJHF21Iu+tkgfh5ZggTvcZrgPb4kHbQNcbkfdqTIyXrYWNC9tQr0c1JKxHQ8N6VF+ljraH1oHJjO/VHhNtx/JIG6vtQNTtaBSpp+2g5MU2IS2cjAjOAdntjywtug7NEj/HJVp4HbQNRFGGgcMJ1djpszfRwvG+YfX4frgW3uZWum5X6rK0aAMRNDGMWOCBrnZL8aCyGpaoSYW/0tgOE88JmPy8q84TlQyCwCTBMto2SprGBS0KobZ8HVSmrkN9j5CJUBlPVsRqO3hXHtvY9c6ytOiYf9ait4HMRu0t8H1BhvF65HUbpR4NQd0ebjBYb7XNCNapsQRNDCMW+Lmpd4Hgr5/a4yCwnap20dXGQXorE4Htea+FD0nwNU946v0g2J7XE5RwfB00z9Ir8t6qDaKWYyy+HnU7eDITOByhNpwszXoZBHswX2nhehrO8HXQezK3XkodgmV8EhWHGhSPDMVq6IG9lVd4YVlHDMMa6spaiSeNmlT0Xh0u2BkGP/gJ1TCoDYrmDUihSL36K8/hvRNC7Q3YbQeibgfFstoOxC5OcQyDfvGxR/Line truncated
|
||||
|
||||
Taking **Qwen3-Omni** as an example:
|
||||
[image8]: <data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAVwAAAGJCAYAAADG5YuwAACAAElEQVR4XuydBXgV19aG23tv7//f+u3f21svFHd3d5fgkgRCcHeXIIFAkKDB3d1dgwa3lhYo7m4tUOT797fCnJ6chJaE5JCTs77nWU8ye/bImVnzztpr75l5CyqVSqVyit5yLFCpVCpV7EiBq1KpVE6SAlelUqmcJAWuSqVSOUkKXJVKpXKSFLgqlUrlJClwVSqVyklS4KpUKpWTpMBVqVQqJ0mBq1KpVE6SAlelUqmcJAWuSqVSOUkKXJVKpXKSFLgqlUrlJClwVSqVyklS4KpUKpWTpMBVqVQqJ0mBq1KpVE6SAlelUqmcJAWuSqVSOUkKXJVKpXKSFLgqlUrlJClwVSqVyklS4KpUKpWTpMB1Ez1//hyzZ89G+vTp8d133yFlypSoWrUqLl686Fg1zmjJkiVInDix7G/r1q3lN/yZqlWrJnXtrXTp0o7VIojHJUOGDFI/VapUmD9/Pq5du+ZYLdaUMWNGpE2bFvfv33ecpYpnUuC6iaZOnYoECRLgrbfestk//vEP5MqVCzdu3HCs/sb1448/IlGiRLZ9/fDDDzFy5EjHauH07bffhvt9tKRJkzpWC6d58+ZFOC4JEyZEvnz58PDhQ8fqsaJ//vOf+Nvf/oY7d+44zlLFMylw3UATJkywwaRGjRro0qULatasKREdy/7973/j3r17jou9MS1YsABJkiSRffP29kabNm3k5kAonTlzxrG66ObNmwIuRsT8fZb9GaQZQVvHpVKlSlK/Vq1atrKvvvoKjx8/dlwsxjVmzBgEBwc7ZVuqNysFbjzXgwcP8MknnwhA5s6dG27ekydP8P3338u8Dh06YOXKlfJ/y5Ytbc3b/v37S1n+/PltyzECZBmb+RSb5O+//76U5cmTBz///LOt+U94spwwIzj5/7vvvovy5cvj9u3btnXaq0yZMlIvderUtjJfX18p27Ztm13NP7R161aZ37x5c+zdu1fsz5rohNvXX38ty/CGZK/Vq1fLtjmvR48eUpY7d26ZXrp0qS2S/u9//4vz589j//79tmMcGBhoW8/169flRsEoe/HixfK7WYfbPXjwoK3exx9/LMfv7t27tjJV/JQCN56LMOBFzmZzZLKASKDwgv/ggw+QLl06LFq0SOYToBZc2MS+cuWKTNMOHz4s0Zk1/fe//13+/u///q9EihTXwzJGp/Z1aFYdRzHFce7cOdmWJQKf6yDMI1NQUFC47dAYuR86dMixqujEiRNS54svvnCcJdq4caPMZ36VKlCggG29b7/9tu13MNVhRd/WfOumwN9BkFrz7Otwu7///rvU+89//iP7qsCN/1LgxnMxIuMFXrhwYcdZoj179sj8ZMmSyXSOHDkEDJ6ennj06JHAwILF8ePHbemJzz77TKJERsisY0W0zIlyPoHEnKQF3G+++caWoyTQWcZtvYp69uwp9QsWLPjSjrPixYtLHUbdY8eORf369WWa0Xhk2rx5s8zPnj274yzRsWPHbMs/ffrUBlwrqqesqHbYsGEy3aRJE5m2ImYLuCzj9ij+Ft68mP5gBEwpcN1HCtx4ru3bt8sFz86xyLRz506Zb6UMrIiVAN63b5/8X7duXfnLPCNHAjCiq1y5sqQk3nvvPZkXmbHZbAGXALfk4eEhQH7ZPtmrVatWsjwjTSsifFVZnWGMlh1l/TaO2ohMR44csR0HQt4CLlMVltKkSYN33nkHFy5ckOk+ffpInUmTJsm0BVymDCwtXLhQ0hX/8z//Y+usVOC6jxS48VzMMRIKbAZbOVNGYhMnTpSOJquTaOjQoTKPEGBdRrT16tWTeadOnRJwMO3AiI+Q2rBhgwCQwGWUy0432m+//SbljH6fPXtmA26jRo1s+8T87asAl5DmsnXq1JF1vUycR4hzP7h9S1YkffLkSbvaYeLvZE6Vv5XHyF6MxFu0aCHLcr2UBdyjR4/a6hGcPLYW0CMDLo+PfdqCKRwFrvtKgRvPxeisUKFCAgJe6JcuXZJOLk5b5ti0z5Ytm20eYUBVr17dVtaxY0dbj3qKFCkEztOmTZNpNueLFi2KEiVKSEoiOsC9deuWdLJxOXbavYoyZ84s9fv27SvTjNwZiX/66acvhXXFihVlGUbCVq6XoyA4Rpkg5jxGwpQCVxUTUuC6gWbOnGmDJWHHiNG+84pDqUJDQ231vby8bPOsHKefn5+tzH60A1MMLCN4CVVrBANBx5RDdIDL/bXG4FaoUAGNGzeW5WkHDhyQOuzU4vSmTZtkmjcBC54sz5s3r0wTqi/TihUrbL+JNyUuxwclrDKORrDSGApcVUxIgesmsi50CyY09rCzM4v/sxPHGkbFIUzWOFhrmNOOHTtsy9k32wlVC240RrvdunXD1atXZT7hzHL7EQnswGMZn65yFDuoPv/883D7aW+zZs2SetZwtQEDBtiWK1myZLi6HGHxV+OLV61aJSkS++V48yhVqlS4Bx+sCNp+1IM1PMwaG9y1a1eZZh6cYqcYp5m6sMT9t5azOs2slI8++BD/pcB1IzG9wIuaTXb+tfKtgwcPRtOmTW312AT/9ddfpZ4V4XFZ5oBfFoWxnPUdx75yeZZzfZY4NpjrigyG9vsYmVn7w3QFp/nXXtZ+RAVejsclshQEfxfnE+yWuC3+Dqs+Ac06VrrFOmb2+8L9t5azRlxwvv20Kv5KgatSqVROkgJXpVKpnCQFrkqlUjlJClyVSqVykhS4KpVK5SQpcFUqlcpJUuCqVCqVk6TAValUKidJgatSqVROkgJXpVKpnCQFrkqlUjlJClyVSqVykhS4KpVK5SQpcFUqlcpJUuCqVCqVk6TAValUKidJgatSqVROkgLXiJ+DuXbtWpw061M1KpXK9aXANfrXv/6Fd/7xT/zj7/+IO/a3f+Cdv78jHyFUqVTxQwpco0/+/QmWHVmMvfdC44wd+HUPNp9bj/f+V4EbX8QPbNauXdtl7JNPPnH8CarXlAIXYcCds3MmtlzYGGds6+XNchN4/1/vO+6uykXFT7+7kpImTepYpHpNKXChwFU5RwpclQIXClyVc6TAVSlwocBVOUcKXJUCF+GBu+vmdoTe3IHQWzuw68a2CCB0lilw458UuCoFLv4AbsilTchdKBcy58iEDFkzoGTlkthxfauUb720OQyE5q/1v9jFsLKQi5sinW8/zTr2da35XL/Mt9+OAjfeSYGrUuAiPHDzl8iH/Q9244enh9BnbC8MnDEA7fu3hVeTmth2dQtqNfeGb2sfiYQJxfl75qK8ZzlMWjcB265sQZ1WPmjQoR52394p8PRtUwc1GlbH9qshmL19BsrXLIvBMwOx49pWAatn4xroMqSTLNt5cCczXRMbzqzFtstbIgXumDFjsHTp0nBlKtfQXwH38OHDSJEiBVq1aoXOnTsjffr0ePDggWO119a9e/fQtm1bx+IIUuDGvBS4CA9cRrf9Jvqj1xg/lKlWGnNDZ5toN73As7V/SwTNGSwgbtW7JTadW48iZQtj373daBvQBu0D2mLapskYNCMQ7cz0kNmDMGrJSCw9vAhjVwQjb9Hc+OHZYfQZ10tAXaVOZcwImQafFrUQbOplz5sN60+tCRfhvvcCuIsXL5Zxke+++67D3qtcRX8F3KNHj6Jdu3a26caNG+PHH3/EtGnTkDlzZlSsWFHKL168iGTJksHT0xN169bFpEmTsGnTJuzYsUNA/fjxY9SsWRNZs2bFyJEjcfPmTWTKlAnp0qXDDz/8gJ49e+LLL7/E9evXbduKTArcmJcCF+GBm6tgTizYO1ds5Y/LsOn8BhQuUwg7r2+DdxNPgeea4yslch27YjQqeHlItLrv/m54mig4VfrUSJc5nUS6y39YigIl8iN3odySRhi1eASSp0oGv5HdMXrZKOQpkhvrTq7G8AVD0SWoE0pXLhkupbDcAPfL/34poH3rrbfEeNE2atRILsYmTZqgefPmaNmyJdq0McBv3x4dO3ZEly5d0L17d/To0QO9evWCv78/AgICEBgYiEGDBiEoKAjDhw/HiBEjEBwcLFHz+PHj5cKdOnUqpk+fjpkzZ2L27NmYN28eFi5cKMBftmwZVq5ciTVr1mDt2rXYsGGDXOghISHYvn07du7cid27d2Pfvn3Yv38/Dh06JBAhNH7++WecPHkSp0+fxrlz53D+/HlcunQJV65cwY0bNwQK/Buf7fvvv3d0vXA6duwYPvroIyRMmBBfffWVnFsqQ4YMEpXyYQQe4759++LgwYO4fPkyGjZsKOeL54DHvnfv3vK3WbNmuH//PgoUKCDnjOf49u3bePLkiRzrTp06OWw9ohS4MS8FLsIDN2/RPNhsIGvlWDeeDYtit18LMVFtc4xYOBQBE/uiVa8WWHNiFfIWyYODv+1Dsx5N0K5/W8zdNdtExbPQZ0xPDJ8/FGtNnWmbJqHjwPZo2bM5TuMkvJt5Yf7uOahsItzZO2bCt3UdjFgwDCUrFRfQ2ke4H77/IUJDQ5E/f34BboIECeQdC7zYaIx2Lly4IBA7e/asAO2XX34RuJ04cUJAxwuZ0CP8jhw5IiA8cOCAGOG4d+9euUi5HV7QhOe2bduwdetWbNmyBZs3b8bGjRsFsOvWrRPg0latWiUX8/LlyyXNsWTJEixatEgAvWDBAsyfPx9z584VcBPgM2bMkGhtypQpmDx5sgB+4sSJmDBhgoB+3Lhx8dq+++47R9cLJ54f3jQpRqKEKcXolseIx5jnkhEqzysBypsvgctztn79egEuz2OdOnXkJsrzRvF8MXrm+eWNztrOn0mBG/NS4MIOuCYKbd2nZTjgbjq3QXK4jDy3XwlBtXpVJe/KEQzMu07eMAnFKxTDhDXjJO/q2agGKvlUlBzuZrNs1bpVJP/LacKadRnpMirm/MqmbruAtrKujoHtbR1okXWaEYDM8RGYKtLine truncated
|
||||
|
||||
## Offline Inference
|
||||
The **Omni** class provides a Python interface for offline batched inference. Users initialize the Omni class with a Hugging Face model name and use the generate method, passing inputs that include both text prompts and multi-modal data:
|
||||
[image9]: <data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAOsAAAEWCAYAAACHcWCmAAB5cklEQVR4XuydB3gVVfrG4/5XRXonjYQQ0nvvvffee++kQwKkQOgdpIogiPQmIBBAJXRUFERQV1111+7aV1d33fX9f2duJlwmNw0IIXDe53lz751y5szM+c33nZlzb5SUuLgUSH3cSAz4sxIe//MjUBmrifEq+mQjTFCzwCRNW+hq2cFgkiNM9F1hZugBC2Nv2JgEwM40CA7moXCyjICzVSTcbGLgYZcAT/tEeDsmw9c5FX4u6Qhwy0QgOcg9CyFeOeRchPkUINy3kFyESP8SRAVMRnRgKWKDyxEXUoH40EokhlcjKWKK4OSIqUiJnCY4OaJWcFJ4zS3vE0JrEB8yVXB0UBViAqsQHVCJSL9yRPiWIdS3FCE+kxHsXYJAzyIEeBTC370Afm658HHJFuzllNnqDHg6ppHT4W6fCje7ZLjbJcHdNgGuVrFwIjtaxcDRIpqOQRQcLMjmEbA3DYetaRisTULIwbAyJhsFwtIwABYGfjDX94GpvjdM9DxgrOtOr+4wM/AAnYY/Sc8LF1c7qY8dgSce/RMGPvZ/GK86CVrqppg4noFqA72J9gKoxnoMVHeYG3pRA/RrBTWEQA2Hi3UUXG2iCdR4eDkktYHq75KBAGYGqkc2gj1zCNJ8weG+xYjwKxZAZZDGBJUJoDJIE8KqBFCTIwnQqBrBqdG15OkyR8nMwBVfZRBPR2JYLUE7BXHBU6jMagK2GlH+FYLDfUoFh5FDCNhgjyK6iBTA3zWfQL0JrLdzlmAZsBnwcGBOIWhT4ErQulnHwZXswqC1jIGDZbRgewasWSTszCJgQ8AyW5uEwtIoGBaGgbAiYC0N/GEhAOvZZgsjTw4rV/ekNno4Bg34Pwx54lFoqhlCW8MaOhPsCFQH6Gs7wUjXRYgAFkbe8HKJkIFqFkxRJUSIqK4UUd1s44SI6uOUQqCmUESVQSoDNYtAzaOIWohQ7wKCpVAAVYimAaUEZhUyE6ZRRKVoGiYXSaOYa5ERV4fJ2XOQHlNHnoG0aNF15OnC+9QoGaxJ4bVURg1iKbrmpTYgO7GOQK0iV9IFokwANZRFV69iqlchRft8BLgWEKQ58CZQ/d1zCVCKrI4M2NYIy0ywutkxJ9P+JsDZmlkGrDMB60wR1p4irL15pGBbs3CCNRTWxgzWIHIAweoPawLW1sRXiK43YfXisHJ1T2pjhmMwgTp00OOYoG5MoLLUl4HqQJHUA2UFlQjwIiAdwrBzyzZK+UIJ1hDYUwRxsAiDC0VVd7s4IarKYE2jaJVBziRnERDZCKKoysyADfMpEtLfKAKVwVpTuhDzZ66VgRtW3Zb+soiaQPBWFC7A9s2HCMgZAqyyaNoaXSNkn5mTCNLEsKkCrNEBVTj94nls23QQET7lBGs5XSgmU0SdTOlwCYFaDH+3QoqshXRByRciaVZKDW1nF5KiyimKpsHbKUuIqp5kIaraJ8PFNokgTRRAZVHV0TyGUmKClSKrp0M8XbRiCFSKrCZhAqgsHbYwCIa5QZAQVaOD0jGvfh6lwF4EqhelwF4cVq7uazz1WYcPeQyjhw2iqGoupL762o706ohdW5/F06tWYW7DbJTkV+Hci8dw4vAhikxJKMgox9kXTyI3pZIibDSlvwzUdAFUb+rvHdx7BFs27MSWp3Zidt1yHNx9BG++fhUb12zH4b0nkJM0HYf3HUdN2SLMrV+FG9fewUH6/MqZ13Hj6g2CeLEQMZfO3YRrr7+FF4+cQUHaLJw/fRmXL13Fivlb8NqFN7D72WMoypiNK6+8hS3r96Ewo0lY/rOPP0bDlBUUxauQHj8DV169imc37kU8XQzOtVygZa5i6Zz18HXJw47N+3D9yhWKsJm4ePo0VixZj2kV8/DysZNYsXA9ntu4Azkp1bQ/W/HchuewZM4KHNq1H80HDlCkjCAww3Fs3z4saVqCyIB0PL9zB3Y9s4VS4mBse3oj9j37HF0oEhAVlIrFTQsoU/EmUJkZrN4cVq7uSVNlFMYOHwDVUUMwaYKV0Ec1mOSMEL9YnD5+COtXPonnqMFtWruBGvUWrF22DhtWr0dL8zFkJpehurgRjmZRlOpm3RJRd23dj5NHTqD5hRO4cPoCDuw6is0E7xuvvY6rr72BGZVL8PabbyGM+o8H9x7Hm5evEZgbCaLrOPPSRQJzJooz5+Kl5nOUBs/HX999D9uf2SdAeuPKX+gisgtvvnYNjTWrcebkBYLsNSxbsBmvE8hZKTNx+uQ55KU1ISakChfOvIpkSqk/ePd9PLlkM9at2Ipzpy7g2Q27KeXNweXzr2L7lj0UdYvwwr4XUFU8my4kB+iicJb2eTvt/3O42PIiTp9oxqJZy3D80CHUVc/FC3v2CrDamERj61MbEeydiv3btlMfNoyWW4CmaU2IDqYLmEcc5tbNo0gdicykYoLUp9XeHFau7muC6mgojxgAtdHDoKNlC0MdJwHW1YuXY+HseZhRM4Mi4S4c2L4Dy+Ytxb5tu/HM+o0ESDNB+zSyUythbRSGDaueodQxldLLLAS65xAQW/DBX25gYdNavP7KG3h+z2FcPHMJL+xvFsBjEfcvN24g3KeMouUrePn4Gaxb/hz2bDtEEW+D0B+tKlxMcL+FmbVP4tLpVykq76LtbMPf//oRlhDYyxdsxN7tR/D6xTexZumzlLIvwI0336Z6P4WjB19EZmIDYkOn4pVzr6O8cA5t/1U8vXYH6qcsp4vDm1gyewNF0zxcOHUexw83o2nGciyb/xQBXoKP3n9LiKSXL5xHTfkcvH7hDDau3oiE0AKcOt6M3LQqvEhZhr1pFMEaiYM7dyImNBeHCeDwgDRs37QVM8rrkRabh+KscuSnT6Y+bwQBnQAzfT8yh5Wrh9JWH0ugDoSm8lgY6DhSf8pVeKwQ6hsPw0keBK8nwgNTEOafQpEhGjHB2fBwikNSdCE1zjyE++XAzyWVQKmjVwYq9VE9cnHsUDNFzjdpWib1PStRUTCPUuY6RPoXUzRegHL6XJI1W7hzW1W0kFLZJmTE1mNew3qKSKvpfR2yExtRkb8QCxqfRmZcPXKTZmH29KcoKq9Cbeky6v/RxSJhJvJSZmN+w9PIim9Aac4CSq2XE2ArkBJTT/3EqRRVZ6CBIjB7pJMeT9OCqwigWYgNrkQI9V+jg8pQmDkTb1+9RhlCPnyc81CUU4/IwBIUZs2Al1M64kInU9l1lMoW0r5PpmkpVG4J9VkTqBsQT/tYQOuxm2tJlOJTH9knnfr14UiPK6F+eA4s9APh5xaHMN8U6sP688jK1XPpaSpj/Jgh0FbXbLvzy25+mOixGyA+MDf0haVxIGyp/2VnHkYpXgScrKPhbBMLN1v2XDUJno7J8HFMRUBrVGV3fLOSa6lhFlGaW0TRs5hcgki/ycLNnpjASsQSOAzU+JAaJIWzu7nsEcwMpEezu74NyIyVc1wjvcqcQZ8zYhppuQakRjbQOnVIiahDcjh7rad+bh3BMR2xIdPpwlJLINaQp1BfsgoR/lV0cakkkMqEG06hXqUI9ihBIAEb6F6EIM8i6lsWURqfS33ZXAKPXp1zaN+y4CXcbEqHh32acMPJXbg7nARX20S42JCtE+BiFSN7nGPO7g5Hws6U+rMmYbBij3CM2V3hQFgY+guwyu4IewmPw5Q4rFzdkcEEZWiMpRRYQ1t4nmqq7yE0JNkNEB9qXH6wooZmaxbSCmsUwRoDV5s4YZCAl0My9ftYRGEDILIFWFlk9XFmAyHyCAgGa4ng6IAyMoO1Sg5WdgeXDWyQPX5Ji64XnE7OJCiZs2JnEqgzZZCS0wjUtCgCNaKBIK1HUtgMMgHLoI2YgfiwaRJYp9JFopqiOhskUUEXkzKCspSAJVg9b8Lq5yJ77hroWgA/1zz4UYT1c6J9ccyEt30mvOwJVgasQ6oArPAoxzapDVbn1mevThbRcGSDJcwiKUUOlwFrHCw8xrEwDGiFVTZAgt1kUuKwcnVHhloq0Bw3EgZaxjA1YCNqvIVoam7A7Cc0LjYax848HPYWkXC0YlE1jmCNF545skbr5cBGK2VSfzVbcJBnLkI8CwRQQ3wpqvpORqRvqTA4gY0qig2qFsxgjQtmI4/YCCQG7HQBNhYt0yLrkRHdGkUlkKZGNgpOCmOgNrSaRVf2uY5SVhmoMlhrERU4FdH+UxDl1wqrL0tTWXS9CWuQOxskUSg8d2X2c8kX7Euw+jrlwNuBgG2NrB52FF3t2MimlDZY2bNXJ0qHHS3j2mC1p8hqR7DaGjNYQwSLwJoZyJ63cli5ui2jiarQUhkLI21rgtOTQJWlvuYUUVnaJouqYQKosqgaSw2TQKWo6kYpoAeD1TFNeOzhx2ClPmughwzUMO9i4W5vpH8ZgVompL+yqMpgZa9TEBtYjXgBWDaggUXXGW3ApkaxKNsgmL1PITNIUyJmEpgySBND69ucQPDGE6yxITMQFzSdLEbWmtY0uJLSYDaaiSIrpcChXpPbYA0gWNmzVz8hqlKEJVCZb8KaRWlwJu0vG4KYRqDeCqsIqqNFnDCayc4sWoistqZsRBOLrqFtwAqw6t8cHKHEYeXqjlhknaiqAmMdWwK0NaoSqGZG/kI/i0VVNiKHgcrsSLD6O0YhwDmK0uAkYXSPj1MafF2zCNYsgjXnFlhZ+hvlz6IqS4Erb0ZWgjQyoAbTs9IJ4GkCsAxW+egqAis6uS31pahKsCYyWMkM0gSCNT6URdX6DmEVQG1NgxmkIZ4M1snCIAmWBjNYRWBvwppHaXB2K6xZt8DqaptCoLKBEjejqoN5rDBgxI5AFSKrMPzwJqyy4Yc3YeWRlavbMqbIqjNeEya6DrAwZg/p/ZFs6IEDZq7wNKL+FcHKRuLYsyF1FDGivSOwOF4PJX7mcKSIwm66+DqmI8E/Fb4uORSh8gjWPASz0UreDNLine truncated
|
||||
|
||||
```
|
||||
# Create an omni_lm with HF model name.
|
||||
from vllm_omni.entrypoints.omni import Omni
|
||||
[image10]: <data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAXYAAAFiCAYAAADiGfPoAAB7FElEQVR4XuydB1gV19aGc2tuv3/KTUwxsSb2rthFELtSFMUKduzYC1aw94rSRFBBUUBFEVEEsffeYktssbfEWKJ+//4WzvF4wIIVcH/Ps55zZs2ePYXDO2vW7PIBtLS0tLQylT6wdGhpaWlpZWxpsGtpaWllMmmwa2lpaWUyabBraWlpZTJpsGtpaWllMmmwa2lpaWUyabBraWlpZTJpsGtpaWllMmmwa2lpaWUyabBraWlpZTJpsGtpaWllMmmwa2lpaWUyabBraWlpZTJpsGtpaWllMmmwa2lpaWUyabBraWlpZTJpsGtpaWllMmmwa2lpaWUyabBraWlpZTJpsGtpaWllMmmwa2lpaWUyabBraWlpZTJpsGtpaWllMmmwa2lpaWUyabBrvbJOnDiBDh06IEuWLPjkk09QpkwZxMTE4N69e5ZF37muXbsmx2dtbY2HDx9artbSyhTSYNd6JY0YMQIffPBBqlaqVCmBfnrS1atXkTdvXhQpUkSDXSvTSoNd66UVGhpqgnixYsWwefNmHDt2DD169DD5K1WqhN9++02i999///2J7R88eJDCd//+/RQ+LtNY3lzG9gQ0zSj3LKUGdvPtUquDx0RLTUZ54zhSk3mdxvlZnotR5mn70dJKizTYtV5Kd+7cQenSpQXe/LQE1ahRo/DHP/5R1q9evRr//ve/5fuyZctkPSFYoEAB8a1bt058BNv//vc/8d29excnT56Eo6Oj6Sbx1Vdf4fDhw6Z9uLm5ib9hw4ZyYzHKtW3bVlIuqckS7KdOncIf/vAH2W7q1KmmOpo2bYpffvkF48aNk+U//elPst4c3tHR0abyxnXYvn27af2NGzfQrl0703pnZ2dUrFhRvk+ePNlULjIy0lTmP//5j1w7nr+W1stKg13rpXTw4EEBEaG4atUqy9USpWfLlk3K9OzZE35+fvLd1tZW1u/cudMEMwKaioqKkuXixYtj3759JuDyCYCAtbKykuXExEQp37FjR1Mdffv2Ffvb3/4my23atDEdi7kswX7mzBnTDejDDz/E2LFj8fXXX5vq5bFw33/+85/leBYvXiz1+Pv7y/r4+HhZDgkJkeWPP/5Y9nHz5k3897//FR/3xzrc3d1N9c6cOVO243FyuXXr1lKGuX8uFy1aVK6hltbLSINd66WUkJAgAGIku2nTJsvVknr57rvvpAyBxuibAP373/8uwDJy8/R98803EvG3atVKfN7e3ujSpYt8b968uUTfBOWWLVvER9BTBti5H0NNmjQRn4uLi8lnrmeBPSAgQMrwBsHl//u//8OFCxfElzt3bvEx/UTdvn0bt27dkjp4bEuWLJH1fHnMbfjymDeCf/7zn3LuhniDYDne6BiVcx9cvnLlitRz9OhR09PNgQMHTNtpaaVFGuxaL6VDhw4JfAivuLg4y9UCPiNinzZtmgDQSEMwHWNnZ4d//etfqFmzpvi2bdsmqZm//vWvcqNwcHAQf2pWvnx52YcBdnt7e9N+27dvL75GjRqZfOZ6Fth37NghZYz0C1/+GqkX4yYVFhZmqisiIkKOkzcr49gMsDPVwmWmiMzTN4ULFxY/I/5Lly7JU4Ll+RnGVI+W1stIg13rpcQI88svvxQAeXh4iI8Aq1evnsCSKQpC+h//+Ad2794t6728vEwgzpo1K/LkyWOK3F1dXSV6zZcvn+S2jdx6586dpT7mws+ePYuff/5Z9k0ZYGeO3ZCR7ngZsDP9QzEdw+WSJUs+FeyEP29qjMC5r9GjR8t6A+w+Pj6yzHM0fyHKfdNPsF++fFnAzqce4xz5yXOk8T2GltbLSINd66VE4BnRMSPvBQsWSPqFIKOPLzr56enpadpm48aNphw4jfBes2aN3AAMX4sWLaRsnz59ZJkpkIsXL4qPTScJxL1798qyAfYGDRqY9vEqYDfqfR7YCeoKFSrIsgHtDRs2yDLBzuPluRrvCJYuXSplmHoxfEzF8Hp9+umnsswUDMWbF1NCQUFBcoPT0noZabBrvbTOnz9vitoJR0bbRqsWw9gZyGgpQrB9//33pnVM0TBqNeBGW758uZQ9fvy4KU3BfRQsWFCgyWWjvncFdvr4Etg4Fp4X00Nc5svT06dPC/D54pU+5tl5/N9++63pPI2Xp0OGDJFlXjeW+fzzz2WZzUR1xK71stJg13plnTt3Dl27dpXWJART9erVpZWIAUOmUwzNmjVLyhByRkqFL0Ppq1y5sqkcRThOmjQJ2bNnl/VsTmjexpwvOeln80ZDvXv3Fh8Bn5quX78uECaYCWge+xdffCHbsKUPxTQKl6tVq2YCe9myZcXHljsUj437582GrXUYsbM5I8vwqcIQmzLmzJlT9sGnGsKb14TXwRCPgWW4LdfzKUZL61Wkwa71RsWUQnrrffo2xDbsQ4cOlTbp5m3bjRYvvBFoab0pabBrab0B8cnAaC3DF6wcR+ezzz6TZUbmT+tApaX1OqTBrqX1hsS27IUKFTLl1Wls/mh0sNLSelPSYNfSeoNip6MffvhB2v3TdEsXrbchDXYtLS2tTCYNdi0tLa1MJg12LS0trUwmDXYtLS2tTCYNdi0tLa1MJg12LS0trUwmDXYtLS2tTCYNdi0tLa1MJg12LS0trUwmDXYtLS2tTCYNdi0tLa1MJg12LS0trUwmDXYtLS2tTCYNdi0tLa1MJg12LS0trUymNw72n3/+GQcOHMDhw4fTrZ06dcrysLW0tLQyrN442Ldt24YCBQogd+7c6c9y5ZaJkhMSEiwPW0tLSyvD6o2DfdeuXciZKydGBAzDpAUTMDFsfLqxyeGT4NzaGevWrbM8bK10pNu3b8t0cosXL053xtmRtLTSm94K2HPkzAGfxdOw4XwS1p5Zk25s06X1cGjuoMGeznXmzBm0b99eAP/rr7+mG7ty5QpGjx5tebhaWu9cGuwa7OleBHubNm0s3e9cvNFosGulR2mwa7Cne6VXsP/2228a7FrpUhrsGuzpXhrsWlppkwa7Bnu6lwa7llba9NbBvu5cIjZcSML682tlOelsQgrgvi3TYM8Y0mDX0kqb3irYN15ch4FTPNGqR0ssO7AU/cb1wZDpg5FwKh7L1fKKQ8uw+sRKLNkThVXHYwX6SecSsOLwMsQeXi43BQKZ62NUWa5fdSxWLUdi5dEYKRt/Mg5L90YhUdXJ5dXHWV+k1JtwarXaLhrR+xYj8XR8qmDnC7HAwEDEx8ebnYXWu9TLgD0kJAQ5c+bEZ599hhw5cqBjx46WRZ4QW7i4uLhYup8pDXat9Kp3AvZeo3ti373dGDV7BDoP6giPoV1QuqIVBk31RHXHarBvXBe2tSoL0BdsCkXdRnVQsWoFLNq2AF29uqKWc02UrmAlkHZs6oD6bvVQt3FtKd+oXSNUc6wq++HNoUptW9m+qr0d5q4NwbfZvoVr52aI/zEOmy9vELDzGO/evQtfX198//33+NOf/oStW7danorWO9LLgN1QlSpVTN8J+759++LYsWMIDw/HzZs3xT9//ny5mfMGcP/+fVP550mDXSu96q2DfXiAt3RYKlKyCKxrWCN6/2J0GdwJg6cNkvVNOjTG1qsb0bZ3Gwzz90KP4d0wOXwiVv4QgzmJs1FZbcPIfXrkVFXnVNRr7iTRNyG/4nA0qjtUk2h9xZFl0gmp04AO2HFzK5p3aga/5TNRzaGqKdfPiL1eCye0bNkSefLkwQcffCBGGBw8eBA7d+7E7t27sWfPHuzfv198HILg6NGjOH78OE6ePImffvoJp0+fxtmzZ3H+/HlcuHABly9flgjw+vXruHHjBm7duiUQeJrxKeFpdufOnWcab0hPs3v37qVqv//++1ONYHuaPXjw4Kn28OHDZ9qr6Ny5cy8N9kqVKsm1iIqKQpcuXZCUlIQKFSrI37B3795YsmQJpk+fLh2N6E/LsWqwa6VXvXWwS8Q+qid2/7YD265vxrqfE9F9mAcmhU9A/Ik4tOjmho2X1sFzUn94zRiCVt1bwCdqGrZe24SI7QtRupIV7JvYo55rPSzbvwQzlk5HdadqaNaxCVafjMP8DfMkOh8ROAwj1E2kx4ju2Hp9k9pnD8xc6gOXNg0lv2+AvX6LeqhXrx4++ugjE9hz5cqFfv36oUePHujWrRs8PDzQuXNndOrUSTrKtG3bFq1bt0arVq3QokULuLq6olmzZmjSpAkaNWqEhg0bwtnZWep1dHRE3bp1YW9vL+bg4CBGP83JyUmMZevXry/WoEEDMdZDY4qAxrpp3I9hTZs2FeP+mzdvLsbjcXNzE+Px0XisNB43jaCk8Vxo7dq1g7u7uxjPsUOHDmJMYfC8abwGNAKya9euYrw2NF6n7t27yzWj9ezZU6xXr15ihGifPn0kYjasf//+Yp6enmIDBgwQGzRoEAYOHCifgwcPlv3w+ryMCHbezLhvXq/x48fLcBKUv78/6tSpI995U65Ro4b5ps+VBrtWetU7AXtnFaEb+XLmwbt5d5Xomi9VrWtUknw44cwIu8+Y3ug/sR96j+mFoT6DUc2xmmw7ft5YTI+YqsDdDbtubYd7v7YCbkb6i7YuUJ+tVYQfLPWs+XEVKtpVQHB8EBq0cn4C7EzFbNq0SaJuAurvf/87smbNikuXLlmeyjuXEf0yQmYEbUTZBBejUkbwRqRP6PBJgcZekr/88oukHmh8iuDTxLVr18SuXr0qTxh80uB5X7x4UYxPHwQeB3KjMXLmkwlTIzReMw6gRuOTy48//ihPMSdOnJAnGhrTHoyOGRHTjhw5Ik89hw4dEuNTEAeJo/GpaN++fdi7d688JdH4xBQbGys3q5eRAXYvLy9MnDgRGzZskCcxasyYMXKz5fXi+Wqwa2UWvV2wX1iHKQsnYeSs4abWMPwcEzwSs+ICJHof5jcUFezKq4i9n9wIWLine truncated
|
||||
|
||||
omni_lm = Omni(model="Qwen/Qwen3-Omni-30B-A3B-Instruct")
|
||||
|
||||
# Example prompts.
|
||||
om_inputs = {"prompt": prompt,
|
||||
"multi_modal_data": {
|
||||
"video": video_frames,
|
||||
"audio": audio_signal,
|
||||
}}
|
||||
|
||||
# Generate texts and audio from the multi-modality inputs.
|
||||
outputs = omni_lm.generate(om_inputs, sampling_params_list)
|
||||
```
|
||||
|
||||
## Online Serving
|
||||
Similar to vLLM, vLLM-Omni also provides a FastAPI-based server for online serving. Users can launch the server using the vllm serve command with the `--omni` flag:
|
||||
|
||||
```
|
||||
vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091
|
||||
```
|
||||
|
||||
Users can send requests to the server using curl:
|
||||
|
||||
```
|
||||
# prepare user content
|
||||
user_content='[
|
||||
{
|
||||
"type": "video_url",
|
||||
"video_url": {
|
||||
"url": "'"$SAMPLE_VIDEO_URL"'"
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "text",
|
||||
"text": "Why is this video funny?"
|
||||
}
|
||||
]'
|
||||
sampling_params_list='[
|
||||
'"$thinker_sampling_params"',
|
||||
'"$talker_sampling_params"',
|
||||
'"$code2wav_sampling_params"'
|
||||
]'
|
||||
mm_processor_kwargs="{}"
|
||||
|
||||
# send the request
|
||||
curl -sS -X POST http://localhost:8091/v1/chat/completions \
|
||||
-H "Content-Type: application/json" \
|
||||
-d @- <<EOF
|
||||
{
|
||||
"model": "Qwen/Qwen3-Omni-30B-A3B-Instruct",
|
||||
"sampling_params_list": $sampling_params_list,
|
||||
"mm_processor_kwargs": $mm_processor_kwargs,
|
||||
"messages": [
|
||||
{
|
||||
"role": "system",
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": "You are Qwen, a virtual human developed by the Qwen Team, Alibaba Group, capable of perceiving auditory and visual inputs, as well as generating text and speech."
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": $user_content
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
For more usages, please refer to [examples](../user_guide/examples/).
|
||||
[image11]: <data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAABM8AAARlCAYAAACukOBGAACAAElEQVR4Xuy9B7hV13WuHSd5ch3Zia17rcQl13+K88dyXOLYlmPLRb13CVUkUEMIEB3Rkeiig+iiI3rvvffeO4gmeu+dM+56xz4TbRZ7IxDt7M33Ps98zll97rXGnGvOb4055l+YEEIIIYQQQgghhBAiJX8RXyGEEEIIIYQQQgghhEgg8UwIIYQQQgghhBBCiDRIPBNCCCGEEEIIIYQQIg0Sz4QQQgghhBBCCCGESIPEMyGEEEIIIYQQQggh0iDxTAghhBBCCCGEEEKINEg8E0IIIYQQQgghhBAiDRLPhBBCCCGEEEIIIYRIg8QzIYQQQgghhBBCCCHSIPFMCCGEEEIIIYQQQog0SDwTQgghhBBCCCGEECINEs+EuAhnz5jlKCnF09k0Kb49/B9fH98/1bazSfaXYhvr025PPm98e3xd8nJ826VcK36eVNeN/e/nih+Xajl+vaT1550jvu+lbIv/TfV/fDn5nPHz5yYhhBBCCCFE9iHxTIh05JgdP5Bjx3YrKcXSntTpeG5KXk61/lzafZFtsXPE07n1KfJz7pwp8nvB9ZL2uWBb8j7ptuWm+DWT973geqmOS943zfW+7Den+j++HD9Hym25/8eXk895wfmj8506mhOvRYQQQgghhBBZgMQzIdKBeLY/x45sz7GjO5SUvkhH0qT49vB/fH18/1TbLnbclx2bvC2+Pb4ueTm+Lb5Pqm2p9onvG///Upfj10ten25b/P/4cvzcqbalukb8nBecf7vEMyGEEEIIIbIViWdCXIQT+y3ROd6ppKSklD5RT0g8E0IIIYQQIjuReCZEOnIQz3ISHigpOstKSkpKIUk8E0IIIYQQInuReCZEOsKwTYlnSkpKX5IkngkhhBBCCJG9SDwTIh3yPFNSUrrEJPFMCCGEEEKI7EXimRDpkOeZkpLSJSaJZ0IIIYQQQmQvEs+ESIfEMyUlpUtMEs+EEEIIIYTIXiSeCZEOiWdKSkqXmCSeCSGEEEIIkb1IPBPiIpzYbxLPbmA6vjt6CEej57Dni+dgh83OHkgsu2CxL9rnpPlflk9wzIlon4Nf7HNyb2IfO56bonMe22UXXC+euD7nuZR9SX79KK/k8cuOYd/TkX3lHJKNZUOSeCaEEEIIIUT2IvFMiHRowoAbmhC8tq3bby0afWKfLd3mQhPrenToZ2MGT3ax7PRBs3lTllmJImVsxbz1dvpQ4pjypSrb8H7jfDv7LZu91t59s5gVzk1Vy9ew3RtQuOycoIZQhuCFsBYEtu1r91vfbkNs/+fsYC7K2ZGEKOYiGeuOJY4NQt6GKK9d2va0PZuOfnGu4wkB0MXAIOBFl58zabGNHDDBBTpSOD/58GXyd+ILYZB74PvkCm5nDnyxHL9/Stc3STwTQgghhBAie5F4JkQ6NGzzhiaEorWLt9hD9z9mcycvdVGJde++Ucwa1m7m4hMi1IAew+373/uBNW/Y1h/bkN6jfblRneYuPuHZNbDHCLvvrgds5viFtmDaUlsya5Xt2XjE5kxaZD079bcB3YfZwa2nXOBCdOvZaUC03zL7fNVuGzFgvK1bssUmjphhndv0sCmjZrvnG/v36NjPBvYc4fk9sSfHBa/5U5ZZwfxv2cblO23RjOXWu8tA69NlkO3//IQd3n7Gl7u172N7N+2zSmWq2aMPPm57Nh2wxTNXW6fW3WzCsOkuni2ZtSbK93Dr3qFvlI89djr6vRuWbY/y0N1mTljowtyaRZs9T3OnLPU8xe+h0vVLEs+EEEIIIYTIXiSeCXERNGzzxiWEMjzO8j39ki2YtsLFMtaVKVbeWjRs66IYAtKwvmPtyUeftVJFy9mhHcetecM2lu+pF619y09dcEPQGtpnjL30XH4XoXatP2CHo/3WR+f+nzvudE+0P995t/XpOtjGD5saXe9Fe+O1QvbgfY/YzAkLrGzxCtamWQe741f/YyXeLWP3/Pk+Wz5njbVq3N5KFStnBV9509o07eBeYORp0fSV5zzh7r/nQXvztXeiY+73/HRq3d3ee6ekVS73gQtolcpWs8ceftKFOfLfpF5Le+bxfDZ93Hx75YUC9nz025945Gmr8n4N2711X/S3ur1fspIVi84xacxUK/1eeav9QT17KV9+mzNxkZ2WgHbDksQzIYQQQgghsheJZ0JcBIlnNy5dqng2uNcoK1mkjL1fqrIN6D7CalerbzUq17HWTTu4eMZ+YwZNstv/4z8t/wsFrMDLr1vLxm1s7aLP7bmnXrAd6w5Yp1bdrH7NJlauRAUfErp1zT4r9EYRmzZmrpUrWTHa/xN7r3ApP1fZaJ9P2/exJx992lo1aW8fVKhlTz/+nB3YctLzE8SzZXPWuaj12dKt1rfrYKsV5avtx52sepS3kQNG277NB21g9+HWqM7HdmLP2SjvQ21Qr5H2wrMvW99Ph1iRQiVsanT9GePmW8WyVaPtw/28eNMd2XPUOrfuYX+68y4b2meYPfLA4+5pR/7i91Hp+iSJZ0IIIYQQQmQvEs+ESIeGbd7QhFC2bulWe+HZV1zock4kxLMOLbuee0yDeo6wmlU/cq8uvLVqVatnrZu0TwzjzPU8Q2B7u0DhRKyxXH1j3eIt7nW2c/2h6HyfutcXItzEETNt+7oDVrRQiYR4VqKifdK8k4tfDKdkqGW39n3tuadfsJpVPrKPG7T0oZOHtp2xnMNfiGcMu+TvphU7rVfHAVbnw4Z2cNux6Nge7s3W9ZNePuyzWb1WNnviYnv5+VetWf1W9uC9D/tQ0MrlPrSF01fYhOHTo99X14eSFn+39Lk4bd2jPNz9p/tcCKxfs7FNGjnTTu298D4qXZ8k8UwIIYQQQojsReKZEOmQeHZDE+LZ+mXb7Q+/v8uFq95detnMCXOtyFvF7eV8r1qvzr1sxMARLnzh/TV2yBS77Tu32bC+Y6xFo7ZWv1ZT9wTDGwvx7Df//VsX2Lq1722Deg6xuZOXWIGX37Bta/db8wZtrHHdFja490h7q0Bhq/NBA3vmiXw2bew8K/p2cR8KWrFMNc8XotqwfmPdY6xhnWY+bJLjT+xNzARKzLO3Cha2hdNX+vDPz5ZstS5te9iHlWpbuxado3O19mM/ad7Z+n06xAq88oa1b9HFr9e/21B75IHHXIwrWbSsTygwatBEK1uivK2cv87PhyhYqlhZ69Wlr4tzvToPsBLvlrbJo2b5PYvfR6XrkySeCSGEEEIIkb1IPBMiHRLPbmg6vjvH9m85YZ+262N1PmxgtarWs8G9R3nA/no1Gvsyccfw2po1YaHt3njExadd6w/ZwmkrfBbOk/uYtTPHA+03rdfSaler555pzeq3ttULNrlXF9fAW2zBtOW2d9Mxa1i7oRV45U2PZcZxU0fPskUzVvrwSfJFfLItq/b49Zi4oEGtprZpxS47tS8xacC2Nfv8vMzUOX7YNNuz8agtn7POxbrN0X51qzf0vByIrrt55S6Pp8YQT2boRFQbPWiSX2/K6Nm2dfVej82GMMaEALMmLvJhqUP6jLacI4nZOvk9vToNiOz0rN+z+H1Uuj5J4pkQQgghhBDZi8QzIdIh8eyGJ4ZJMlTTTuemI5aYZTMsn0gMy8yJ0ond5sMZmTGT4P2ITTw70sm9SceQTpqd2pc4F9dgf87F7JYli5axd94o4gIY52afMBkAeWKZ83EdO5VILIdr+frDudujv8d3J2K1cS7PB8ecTKz35ZOJ7f47o//5jZ6fQ4lznGKfQ4nze5y33PtwlGXyx/KxxO+I3z+l65ckngkhhBBCCJG9SDwTIh0Sz266hACFCIewpiGQSpeTJJ4JIYQQQgiRvUg8EyIdEs9uyoSAJi8upctNEs+EEEIIIYTIXiSeCZEOiWdKSkqXmCSeCSGEEEIIkb1IPBMiHRLPlJSULjFJPBNCCCGEECJ7kXgmRDoknikpKV1ikngmhBBCCCFE9iLxTIh0SDxTUlK6xCTxTAghhBBCiOxF4pkQ6ZB4pqSkdIlJ4pkQQgghhBDZi8QzIdIh8UxJKU8kZj89nsdnQJV4JoQQQgghRPYi8UyIdCCe7Ys6xdsTHWMlJaXrn45G6cRus5N77YJteSpF9cSpIxLPhBBCCCGEyEYkngmRhpyoH4wnCR1iJaXz0lG7cDms+7JtydtTLce3pTrvxY5LtRzfFj82+RrxY+P7xrelOzbVvun+j58nadvZE2Zd2/ewMsUr2IlDZ+x0/Nj4NeLnSZe/VMelOybVtvj/UTpzQuKZEEIIIYQQ2YjEMyGEEHmacePHWdu2bS0n52x8kxBCCCGEEEJccySeCSGEEEIIIYQQQgiRBolnQgghhBBCCCGEEEKkQeKZEEIIIYQQQgghhBBpkHgmhBAiTzNixAj76KOP7OxZxTwTQgghhBBCXH8kngkhhMjTFC9e3G699VY7ffp0fJMQQgghhBBCXHMkngkhhMjTVKxY0f7t3/7NTp06Fd8khBBCCCGEENcciWdCCCHyNJUqVbJ//dd/lXgmhBBCCCGEuCFIPBNCCJGnqVy5sv3Lv/yLxDMhhBBCCCHEDUHimRBCiDwNwzZ/+MMfSjwTQgghhBBC3BAkngkhhMjT4Hkm8UwIIYQQQghxo5B4JoQQIk+za9cuW7NmjeXk5MQ3CSGEEEIIIcQ1R+KZEEIIIYQQQgghhBBpkHgmhBBCCCGEEEIIIUQaJJ4JIYQQQgghhBBCCJEGiWdCCCHyNK1bt7Z77rlHEwYIIYQQQgghbggSz4QQQuRpqlSpYn//939vJ0+ejG8SQgghhBBCiGuOxDMhhBB5mpo1a9p3v/tdiWdCCCGEEEKIG4LEMyGEEHmaWrVq2T/+4z9KPBNCCCGEEELcECSeCSGEyNMgnv3DP/yDxDMhhBBCCCHEDUHimRBCiDxNtWrV7JZbbpF4JoQQQgghhLghSLine truncated
|
||||
@@ -1,200 +0,0 @@
|
||||
# OmniConnector: a unified connector interface of disaggregated inference for omni-modality models
|
||||
|
||||
This guide explains how to configure and use distributed connectors (vllm_omni/distributed/connectors) in vllm-omni for multi-stage pipelines.
|
||||
|
||||
## 1. Overview
|
||||
|
||||
Connectors enable data transfer between pipeline stages (e.g., Thinker -> Talker).
|
||||
Currently supported connectors operate in **D2H2D (Device-to-Host-to-Device)** mode:
|
||||
1. **SharedMemoryConnector**: Uses system shared memory.
|
||||
2. **MooncakeConnector**: Uses [Mooncake](https://github.com/kvcache-ai/Mooncake).
|
||||
|
||||
* **SharedMemoryConnector (Default)**: Zero-copy (on host), lowest latency. Best for **single-node** deployments. Auto-configured if no connectors are specified.
|
||||
* **MooncakeConnector**: TCP/RDMA based. Best for **multi-node** distributed deployments. Requires a Mooncake Master service.
|
||||
|
||||
## 2. API Design
|
||||
|
||||
The connector system is built around the `OmniConnectorBase` abstraction, which decouples data transport from stage logic.
|
||||
|
||||
### Core Interface
|
||||
|
||||
```python
|
||||
class OmniConnectorBase(ABC):
|
||||
@abstractmethod
|
||||
def put(self, from_stage: str, to_stage: str, request_id: str, data: Any) -> tuple[bool, int, Optional[dict]]:
|
||||
"""
|
||||
Store data.
|
||||
Returns: (success, serialized_size, metadata)
|
||||
"""
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def get(self, from_stage: str, to_stage: str, request_id: str, metadata: Optional[dict] = None) -> Optional[tuple[Any, int]]:
|
||||
"""
|
||||
Retrieve data.
|
||||
Args: metadata - Transport-specific handles returned by put() (e.g., SHM name).
|
||||
Returns: (object, serialized_size)
|
||||
"""
|
||||
pass
|
||||
```
|
||||
|
||||
### Key Concept: Metadata Passing
|
||||
Unlike a pure key-value store, some connectors (like `SharedMemoryConnector`) generate transient resources (e.g., a shared memory block name) during `put()`. This `metadata` **must be passed via the control plane** (e.g., HTTP headers, queue messages) from the producer stage to the consumer stage so `get()` can locate the data.
|
||||
|
||||
## 3. Backends & Use Cases
|
||||
|
||||
### 3.1 SharedMemoryConnector
|
||||
**Best for:** Single-node, high-performance IPC.
|
||||
|
||||
* **Mechanism:**
|
||||
* **Small Payloads (< Threshold)**: Data is serialized and passed directly "inline" within the `metadata` dictionary. This avoids the overhead of creating SHM blocks for tiny messages.
|
||||
* **Large Payloads (>= Threshold)**: Data is written to a named System Shared Memory block. The block name is returned in `metadata`.
|
||||
* **Configuration:**
|
||||
* `shm_threshold_bytes`: Size in bytes to switch from inline to SHM (default: 64KB).
|
||||
|
||||
### 3.2 MooncakeConnector
|
||||
**Best for:** Multi-node distributed inference.
|
||||
|
||||
* **Mechanism:** Uses Mooncake's distributed KVCache store.
|
||||
* **Data Plane**: TCP or RDMA for high-bandwidth transfer.
|
||||
* **Control Plane**: Uses a centralized Mooncake Master and Metadata Server.
|
||||
* **Keying**: Deterministic keys based on `request_id/from_stage_to_stage`.
|
||||
* **Requirements**: Requires a running Mooncake Master service.
|
||||
|
||||
## 4. Relationship with vLLM
|
||||
|
||||
vLLM provides specialized distributed mechanisms for specific artifacts:
|
||||
* **KV Transfer** (`vllm.distributed.kv_transfer`): Optimized for transferring KV caches between prefill and decode instances (using NCCL, Mooncake, etc.).
|
||||
* **EC Transfer** (`vllm.distributed.ec_transfer`): Optimized for sharing encoder embeddings.
|
||||
* **Device Communicators** (`vllm.distributed.device_communicators`): Low-level primitives (NCCL, SHM) for Tensor/Pipeline Parallelism.
|
||||
|
||||
`vllm-omni` complements this by introducing a **Generalized Connector Abstraction** (`OmniConnector`) for multimodal pipelines. While vLLM's connectors are artifact-specific, `vllm-omni`:
|
||||
|
||||
1. **Unifies Transport**: Provides a single API (`put`/`get`) to transport *any* stage artifact (Input Embeddings, Hidden States, Audio/Image Tensors, KV Cache, Final Output) between arbitrary pipeline stages (e.g., AudioEncoder -> LLM -> AudioGenerator).
|
||||
2. **Extends Connectivity**: Enables flexible construction of complex DAGs (Directed Acyclic Graphs) where stages can run in the same process, same node, or across nodes, using the most appropriate backend (SHM, Mooncake, etc.) for each edge.
|
||||
3. **Wraps & Adapts**: Can internally utilize vLLM's specialized `kv_transfer` for KV paths while using generic transports (SHM/Mooncake) for other data types, presenting a consistent interface to the application layer.
|
||||
|
||||
## 5. Installation (Mooncake)
|
||||
|
||||
If using `MooncakeConnector`, install the library first:
|
||||
|
||||
```bash
|
||||
# For CUDA-enabled systems (Recommended)
|
||||
pip install mooncake-transfer-engine
|
||||
|
||||
# For non-CUDA systems
|
||||
pip install mooncake-transfer-engine-non-cuda
|
||||
```
|
||||
|
||||
## 6. Using MooncakeConnector
|
||||
|
||||
### 6.1 Start Mooncake Master
|
||||
|
||||
Start the master service on your primary node:
|
||||
|
||||
```bash
|
||||
# if you use mooncake SSD storage
|
||||
mkdir -p ./mc_storage
|
||||
|
||||
mooncake_master \
|
||||
--rpc_port=50051 \
|
||||
--enable_http_metadata_server=true \
|
||||
--http_metadata_server_host=0.0.0.0 \
|
||||
--http_metadata_server_port=8080 \
|
||||
--metrics_port=9003 \
|
||||
--root_fs_dir=./mc_storage/ \
|
||||
--cluster_id=mc-local-1 &
|
||||
```
|
||||
|
||||
### 6.2 Configuration (YAML)
|
||||
|
||||
Edit your stage config (e.g., `qwen2_5_omni.yaml`).
|
||||
|
||||
**Step 1: Define Connector in Global Runtime**
|
||||
|
||||
```yaml
|
||||
runtime:
|
||||
connectors:
|
||||
connector_of_mooncake:
|
||||
name: MooncakeConnector
|
||||
extra:
|
||||
host: "127.0.0.1" # Local Worker IP
|
||||
metadata_server: "http://<MASTER_IP>:8080/metadata"
|
||||
master: "<MASTER_IP>:50051"
|
||||
segment: 512000000 # 512MB segment
|
||||
localbuf: 64000000 # 64MB buffer
|
||||
proto: "tcp" # "tcp" or "rdma"
|
||||
```
|
||||
|
||||
**Mooncake Configuration Parameters:**
|
||||
|
||||
* **host**: The hostname or IP address of the local machine (worker). Mooncake uses this to register itself in the metadata server so other nodes can find it.
|
||||
* **metadata_server**: The URL of the metadata server. This is used for service discovery and connection establishment (e.g., exchanging QP information for RDMA).
|
||||
* **master**: The address of the Mooncake Master Server (e.g., `<MASTER_IP>:50051`). This is used for global state management and control plane operations.
|
||||
* **segment**: The size of the global memory segment in bytes (default: ~512MB). This defines the shared memory region accessible by Mooncake for data transfer.
|
||||
* **localbuf**: The size of the local buffer in bytes (default: ~64MB). Used for local data buffering during transfer operations.
|
||||
* **proto**: The transport protocol to use. Options:
|
||||
* `tcp`: Standard TCP/IP (easier setup, universal compatibility).
|
||||
* `rdma`: Remote Direct Memory Access (higher performance, requires RDMA-capable hardware).
|
||||
|
||||
For more details, refer to the [Mooncake Repository](https://github.com/kvcache-ai/Mooncake).
|
||||
|
||||
**Step 2: Reference in Stages**
|
||||
|
||||
Explicitly link stages using `input_connectors` and `output_connectors`:
|
||||
|
||||
```yaml
|
||||
stage_args:
|
||||
- stage_id: 0
|
||||
# ...
|
||||
output_connectors:
|
||||
to_stage_1: connector_of_mooncake
|
||||
|
||||
- stage_id: 1
|
||||
# ...
|
||||
input_connectors:
|
||||
from_stage_0: connector_of_mooncake
|
||||
```
|
||||
|
||||
## 7. Using SharedMemoryConnector (Auto-Mode)
|
||||
|
||||
**Best for single-node.**
|
||||
|
||||
The system will automatically create `SharedMemoryConnector`s for any pipeline edge that does not have an explicit connector defined. This is inferred from:
|
||||
1. `runtime.edges` list in the config.
|
||||
2. `engine_input_source` dependencies defined in `stage_args`.
|
||||
|
||||
### Threshold Configuration
|
||||
By default, payloads larger than **64KB** (default threshold) are transferred via shared memory, while smaller ones use the control queue (inline).
|
||||
|
||||
To adjust this threshold (e.g., to 1GB), add the following to your `runtime.connectors`:
|
||||
|
||||
```yaml
|
||||
runtime:
|
||||
connectors:
|
||||
connector_of_shared_memory:
|
||||
name: SharedMemoryConnector
|
||||
extra:
|
||||
shm_threshold_bytes: 1024 # 1KB threshold
|
||||
```
|
||||
|
||||
## 8. Summary
|
||||
|
||||
| Use Case | Recommended Connector | Configuration |
|
||||
| :--- | :--- | :--- |
|
||||
| **Single Node** | `SharedMemoryConnector` | **None** (Automatic) or Custom Threshold |
|
||||
| **Multi Node** | `MooncakeConnector` | Explicit YAML + Mooncake Master |
|
||||
|
||||
## 9. Operational Notes (important)
|
||||
|
||||
- **Fail-fast config validation**: the loader raises if any expected edge is missing a connector. Define `input_connectors`/`output_connectors` or rely on auto-SHM filling; otherwise startup aborts.
|
||||
- **Missing payloads halt the stage**: workers expect connector payloads; if metadata or connector config is missing, the stage raises and stops. Verify connector wiring and metadata propagation before production.
|
||||
|
||||
## 10. Future Roadmap: Device-to-Device (D2D) Transport
|
||||
|
||||
The current implementations (`SharedMemoryConnector`, `MooncakeConnector`) utilize a **D2H2D (Device-to-Host-to-Device)** data path. Tensors are moved to CPU memory (Host) for transport, which incurs PCIe overhead.
|
||||
|
||||
As outlined in the design RFC, future versions will introduce **D2D (Device-to-Device)** connectors:
|
||||
|
||||
* **Goal**: Direct GPU-to-GPU transfer (via NCCL, UCX, or IPC) to minimize latency for large tensor payloads.
|
||||
* **Mechanism**: The `OmniConnector` API allows `put()` to initiate a transfer and return a lightweight handle (metadata) via the control plane, while the heavy payload flows directly between devices.
|
||||
@@ -1,61 +0,0 @@
|
||||
# VLLM-Omni Distributed Utilities
|
||||
|
||||
This directory (vllm_omni/distributed/ray_utils) contains utilities for distributed execution in vllm-omni, supporting both **Ray** and **Multiprocessing** backends.
|
||||
|
||||
## 1. Ray Utils
|
||||
|
||||
The `ray_utils` module provides helper functions for managing Ray clusters and actors, which is essential for:
|
||||
* **Multi-node deployment**: Running pipeline stages across different physical machines.
|
||||
* **Resource management**: Efficient GPU/CPU allocation.
|
||||
|
||||
### 1.1 Basic Usage
|
||||
|
||||
To use the Ray backend, specify `worker_backend="ray"` when initializing the engine.
|
||||
|
||||
**Command Line Example:**
|
||||
```bash
|
||||
vllm serve Qwen/Qwen2.5-Omni-7B \
|
||||
--omni \
|
||||
--port 8091 \
|
||||
--worker-backend ray \
|
||||
--ray-address auto
|
||||
```
|
||||
|
||||
### 1.2 Cluster Setup
|
||||
|
||||
**Step 1: Start Head Node**
|
||||
Run this on your primary machine:
|
||||
```bash
|
||||
ray start --head --port=6399
|
||||
```
|
||||
|
||||
**Step 2: Connect Worker Nodes**
|
||||
Run this on each worker machine:
|
||||
```bash
|
||||
ray start --address=<HEAD_NODE_IP>:6399
|
||||
```
|
||||
|
||||
> **Tip**: For a complete cluster setup script, refer to the vLLM example:
|
||||
> [run_cluster.sh](https://github.com/vllm-project/vllm/blob/main/examples/online_serving/run_cluster.sh)
|
||||
|
||||
### 1.3 Distributed Connector Support
|
||||
|
||||
When running on Ray, the system automatically adapts its communication strategy:
|
||||
|
||||
* **Cross-Node**: Recommended to use `MooncakeConnector` (requires separate configuration).
|
||||
* **Same-Node**: Can still use `SharedMemoryConnector` for efficiency, or Ray's native object store (plasma).
|
||||
* **SHM threshold default differs**: when `worker_backend="ray"`, the SharedMemoryConnector default threshold is set to `sys.maxsize`, which forces payloads to go inline (no SHM). Override `shm_threshold_bytes` in the connector config if you want SHM for Ray runs.
|
||||
|
||||
### 1.4 Internal Helpers
|
||||
|
||||
* **`initialize_ray_cluster`**: Connects to an existing Ray cluster or starts a local one.
|
||||
|
||||
## 2. Troubleshooting
|
||||
|
||||
* **Connection Issues**: Ensure the Ray head node is accessible and ports (default 6399 in this example) are open.
|
||||
* **Version Mismatch**: Ensure all nodes run the same version of Ray and Python.
|
||||
|
||||
### Installation
|
||||
```bash
|
||||
pip install "ray[default]"
|
||||
```
|
||||
+2
-12
@@ -5,15 +5,5 @@ This section contains design documents and architecture specifications for vLLM-
|
||||
## Architecture Documents
|
||||
|
||||
- [Architecture Overview](architecture_overview.md)
|
||||
|
||||
## Feature Design Documents
|
||||
|
||||
- [Multi-Request Streaming](feature/multi_request_streaming_design.md)
|
||||
- [Omni Connector Design](feature/omni_connector_design.md)
|
||||
- [Ray-based Execution](feature/ray_based_execution.md)
|
||||
|
||||
## Module Design Documents
|
||||
|
||||
- [AR Module Design](module/ar_module_design.md)
|
||||
- [DIT Module Design](module/dit_module_design.md)
|
||||
- [Entrypoint Module Design](module/entrypoint_module_design.md)
|
||||
- [vLLM-Omni Software Design Document](vllm_omni_design.md)
|
||||
- [Multi-Request Streaming](mrs_design.md)
|
||||
@@ -1 +0,0 @@
|
||||
Architecture design of the AR module (update soon)
|
||||
@@ -1 +0,0 @@
|
||||
Architecture design of the DiT module (update soon)
|
||||
@@ -1 +0,0 @@
|
||||
Architecture design of the entrypoint (update soon)
|
||||
File renamed without changes.
@@ -0,0 +1,509 @@
|
||||
# Architecture Design Principles
|
||||
|
||||
## Overview
|
||||
|
||||
vLLM-Omni is a multi-modality extension for vLLM that supports non-autoregressive structures and non-textual outputs. This document outlines the key software abstractions, APIs, and dependencies for the system, designed to maximize reuse of vLLM's proven architecture.
|
||||
|
||||
## Architecture Principles
|
||||
|
||||
1. **vLLM V1 Compatibility**: Built on vLLM's Engine V1 architecture with AsyncLLM and EngineCore patterns
|
||||
2. **Stage-based Processing**: Models are divided into multiple stages, each processed by different Engine Cores
|
||||
3. **Multiple Engine Core Support**: Each stage can use either AR Engine Core (reusing vLLM) or Diffusion Engine Core (new DiT support), or other new Cores
|
||||
4. **Worker Process Pattern**: Follows vLLM's multiprocess worker architecture for scalability
|
||||
5. **Extensibility**: Easy integration of new modalities, model architectures, and output formats
|
||||
|
||||
## Key Data Flow
|
||||
|
||||
API Server --> OmniLLM/AsyncOmni (New, including multi engines) --> LLMEngine/AsyncLLM --> Engine Core
|
||||
--> Scheduler (New one for DiT) --> Executor (New one for diffusers) --> Worker (New one for DiT)
|
||||
--> ModelRunner (New one for AR hiddenstate, New one for DiT) --> RequestState --> OutputProcessoer (New one for final multimodal output)
|
||||
|
||||
## Core Components (Ordered by Data Flow)
|
||||
|
||||
### 1. Installation
|
||||
```bash
|
||||
pip install vllm
|
||||
pip install vllm-omni
|
||||
```
|
||||
|
||||
### 2. Online inference launch (Entry Point)
|
||||
|
||||
Keep consistency with the vllm main branch
|
||||
```bash
|
||||
vllm serve Qwen/Qwen2.5-Omni-7B --omni --port 8000
|
||||
```
|
||||
|
||||
#### Design Architecture
|
||||
```mermaid
|
||||
graph TD
|
||||
A[vllm serve model --omni] --> B[vLLM-Omni CLI Wrapper]
|
||||
B --> C{Detect --omni flag}
|
||||
C -->|Yes| D[Parse OmniConfig]
|
||||
C -->|No| E[Forward to vLLM CLI]
|
||||
D --> F[Initialize AsyncOmni]
|
||||
F --> G[Start omni Server]
|
||||
G --> H[Multi-stage Processing]
|
||||
E --> I[Standard vLLM Pipeline]
|
||||
|
||||
subgraph "vLLM-Omni Components"
|
||||
F
|
||||
G
|
||||
H
|
||||
end
|
||||
|
||||
subgraph "vLLM Components"
|
||||
I
|
||||
end
|
||||
```
|
||||
|
||||
#### omni Serve Command Implementation
|
||||
File: vllm_omni/entrypoints/cli/main.py
|
||||
```python
|
||||
import sys
|
||||
import argparse
|
||||
from typing import List, Optional
|
||||
from vllm_omni.entrypoints.omni import OmniServeCommand
|
||||
|
||||
def main():
|
||||
"""Main CLI entry point that intercepts vLLM commands"""
|
||||
# Check if --omni flag is present
|
||||
if "--omni" in sys.argv:
|
||||
# Remove --omni flag and process with vLLM-Omni
|
||||
omni_args = [arg for arg in sys.argv[1:] if arg != "--omni"]
|
||||
omni_serve = OmniServeCommand()
|
||||
omni_serve.run(omni_args)
|
||||
else:
|
||||
# Forward to original vLLM CLI
|
||||
from vllm.entrypoints.cli.main import main as vllm_main
|
||||
vllm_main()
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
```
|
||||
|
||||
#### vLLM Plugin Integration
|
||||
File: vllm_omni/pyproject.toml
|
||||
```python
|
||||
[project.scripts]
|
||||
# Override vLLM's CLI entry point
|
||||
vllm = "vllm_omni.cli:main"
|
||||
|
||||
# Add entry point for vLLM integration
|
||||
[project.entry-points."vllm.plugins"]
|
||||
omni = "vllm_omni.plugin:OmniPlugin"
|
||||
```
|
||||
|
||||
### 3. Offline inference launch (Entry Point)
|
||||
Design an upper level class to incorporate multi Engines, each engine has a engine call.
|
||||
```python
|
||||
from vllm.entrypoints.llm import LLM
|
||||
from vllm.v1.engine.llm_engine import LLMEngine
|
||||
|
||||
class OmniLLM(LLM):
|
||||
"""Extended LLM supporting multiple engines and stage-based processing"""
|
||||
|
||||
def __init__(self, stage_configs: List[StageConfig]):
|
||||
super().__init__()
|
||||
self.stage_configs = stage_configs
|
||||
self.engine_list = [] # List of AsyncLLM instances for each stage
|
||||
self.output_processor = MultimodalOutputProcessor()
|
||||
self._initialize_stage_engines()
|
||||
|
||||
def _initialize_stage_engines(self) -> None:
|
||||
"""Initialize LLMEngine instances for each stage"""
|
||||
for stage_config in self.stage_configs:
|
||||
stage_llm = LLMEngine.from_vllm_config(
|
||||
vllm_config=stage_config.vllm_config,
|
||||
executor_class=self.executor_class,
|
||||
log_stats=self.log_stats
|
||||
)
|
||||
self.engine_list.append(stage_llm)
|
||||
|
||||
def generate(
|
||||
self,
|
||||
stage_args_list: List[stage_args],
|
||||
use_tqdm: Union[bool, Callable[..., tqdm]] = True,
|
||||
lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
|
||||
priority: Optional[list[int]] = None,
|
||||
) -> list[RequestOutput]:
|
||||
"""Main generation interface - orchestrates multi-stage processing"""
|
||||
|
||||
# Process through each stage sequentially
|
||||
for i, stage_config in enumerate(self.stage_configs):
|
||||
stage_engine = self.engine_list[i]
|
||||
|
||||
# Prepare input for this stage
|
||||
stage_args = stage_args_list[i]
|
||||
prompt_str, engine_request, tokenization_kwargs = self._process_stage_inputs(stage_config, **stage_args)
|
||||
|
||||
# Add inputs to Engine
|
||||
stage_engine.add_request(requesy_id, prompt_str, tokenization_kwargs)
|
||||
# Run Engine
|
||||
stage_output = stage_engine.step()
|
||||
|
||||
# Update stage input and output of each stage for later usage
|
||||
self._update_stage_io(stage_output, stage_config)
|
||||
|
||||
|
||||
# Process final output
|
||||
final_output = self.output_processor.process_output(
|
||||
self.stage_configs[-1].stage_output
|
||||
)
|
||||
return final_output
|
||||
|
||||
def _process_stage_inputs(stage_config, **stage_args) -> Any:
|
||||
"""Prepare input for specific stage"""
|
||||
if stage_config.engine_type == "AR":
|
||||
return self._process_ar_inputs(**stage_args)
|
||||
elif stage_config.engine_type == "DiT":
|
||||
return self._process_dit_inputs(**stage_args)
|
||||
else
|
||||
raise NotImplementedError
|
||||
|
||||
def _process_dit_inputs(**stage_args)-> Any:
|
||||
image_latent = self.vae.encode(**stage_args)
|
||||
|
||||
return image_latent
|
||||
```
|
||||
|
||||
### 4. Stage config for setting of different model stages
|
||||
```python
|
||||
@dataclass
|
||||
class StageConfig:
|
||||
"""Configuration for a processing stage"""
|
||||
stage_id: int
|
||||
engine_type: str # "AR" or "DiT"
|
||||
model_path: str
|
||||
input_modalities: List[str]
|
||||
output_modalities: List[str]
|
||||
vllm_config: Optional[VllmConfig] = None # For engine config of corresponding stage
|
||||
executor_class: type[Executor] # For execute class config of corresponding stage
|
||||
dit_config: Optional[DiTConfig] = None # For diffusion stages
|
||||
cache_config: Optional[DiTCacheConfig] = None
|
||||
```
|
||||
For AR stage, the setting is:
|
||||
```python
|
||||
from vllm.v1.core.sched.scheduler import Scheduler
|
||||
from vllm.v1.executor.multiproc_executor import MultiprocExecutor
|
||||
ar_stage_config = StageConfig()
|
||||
ar_stage_config.vllm_config.scheduler_config.scheduler_cls = Scheduler # original vllm scheduler for AR
|
||||
ar_stage_config.executor_class = MultiprocExecutor # original vllm executor for AR
|
||||
```
|
||||
For DiT stage, the setting is:
|
||||
```python
|
||||
from vllm.v1.executor.multiproc_executor import MultiprocExecutor
|
||||
dit_stage_config = StageConfig()
|
||||
dit_stage_config.vllm_config.scheduler_config.scheduler_cls = DiffusionScheduler # New scheduler for DiT
|
||||
|
||||
# For diffusion models without using diffusers
|
||||
dit_stage_config.executor_class = MultiprocExecutor
|
||||
|
||||
# For diffusion models using diffusers
|
||||
dit_stage_config.executor_class = DiffusersPipelineExecutor
|
||||
```
|
||||
|
||||
### 5. Online inference main class
|
||||
|
||||
Similar to OmniLLM in offline inference, add some asynchronous processing, referring to AsyncLLM
|
||||
```python
|
||||
from vllm.v1.engine.async_llm import AsyncLLM
|
||||
|
||||
|
||||
class AsyncOmni(AsyncLLM):
|
||||
"""Extended AsyncLLM supporting multiple engines and stage-based processing"""
|
||||
|
||||
def __init__(self, stage_configs: List[StageConfig]):
|
||||
super().__init__()
|
||||
```
|
||||
|
||||
### 6. Engine Core
|
||||
No need to change. The specific executor, scheduler and output type can be transferred into it with new configs.
|
||||
```python
|
||||
# class: EngineCore
|
||||
# EngineCore.step (simplified)
|
||||
class EngineCore:
|
||||
def step(self):
|
||||
scheduler_output = scheduler.schedule()
|
||||
model_output = executor.execute_model(scheduler_output)
|
||||
engine_outputs = scheduler.update_from_output(
|
||||
scheduler_output, model_output
|
||||
)
|
||||
return engine_outputs
|
||||
```
|
||||
|
||||
### 7. Scheduler
|
||||
Create a new Diffusion Scheduler for DiT, which is inherited from original vllm AR scheduler. At first, no complicated strategy.
|
||||
Just process the request first in first out. Then create a class of DiT Cache Manager for optimization in the future.
|
||||
```python
|
||||
from vllm.v1.core.sched.scheduler import Scheduler
|
||||
class DiffusionScheduler(Scheduler):
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
vllm_config: VllmConfig,
|
||||
kv_cache_config: KVCacheConfig,
|
||||
dit_cache_config: DiTCacheConfig,
|
||||
structured_output_manager: StructuredOutputManager,
|
||||
mm_registry: MultiModalRegistry = MULTIMODAL_REGISTRY,
|
||||
include_finished_set: bool = False,
|
||||
log_stats: bool = False,
|
||||
) -> None:
|
||||
|
||||
self.dit_cache_manager = DiTCacheManager(dit_cache_config)
|
||||
```
|
||||
For DiT Cache Manager (Can refer to xDiT):
|
||||
```python
|
||||
class DiTCacheManager:
|
||||
"""Manages DiT-specific caching"""
|
||||
|
||||
def __init__(self, config: DiTCacheConfig):
|
||||
self.cache_tensors: Dict[str, torch.Tensor] = {}
|
||||
self.cache_groups: List[DiTCacheTensor] = config.dit_cache_tensors
|
||||
|
||||
def allocate_cache(self, request_id: str, size: int) -> torch.Tensor
|
||||
def get_cache(self, request_id: str) -> Optional[torch.Tensor]
|
||||
def release_cache(self, request_id: str) -> None
|
||||
def clear_expired_cache(self) -> None
|
||||
```
|
||||
|
||||
### 8. Executor for DiT without diffusers
|
||||
|
||||
No need to change. Just import original AR executor.
|
||||
|
||||
### 9. Diffusers Pipeline Executor (no worker)
|
||||
|
||||
A single-process executor that directly runs the Diffusers pipeline without spawning workers or using RPC. Interfaces remain identical to `Executor` so the EngineCore loop is unchanged.
|
||||
|
||||
#### Function map（Pipeline Executor）
|
||||
|
||||
##### 1) Inherited and overridden
|
||||
```python
|
||||
from concurrent.futures import Future
|
||||
from typing import Optional, Union, Callable, Any
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from vllm.v1.executor.abstract import Executor
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig, KVCacheSpec
|
||||
from vllm.v1.outputs import ModelRunnerOutput
|
||||
from vllm.tasks import SupportedTask
|
||||
from diffusers import DiffusionPipeline
|
||||
|
||||
class DiffusersPipelineExecutor(Executor):
|
||||
supports_pp: bool = False # Single-process, no TP/PP/DP
|
||||
|
||||
def _init_executor(self) -> None:
|
||||
# Called by ExecutorBase.__init__
|
||||
self._failure_callback: Optional[Callable[[], None]] = None
|
||||
self._device = self._resolve_device()
|
||||
self._dtype = self._resolve_dtype()
|
||||
self._pipeline = self._build_pipeline(device=self._device, dtype=self._dtype)
|
||||
self._profiler = None
|
||||
self._is_failed = False
|
||||
self.is_sleeping = False
|
||||
self.sleeping_tags: set[str] = set()
|
||||
|
||||
# major functions to build/run diffusers pipeline
|
||||
def _build_pipeline(self, device:torch.device, dtype:torch.dtype)->DiffusionPipeline:
|
||||
model_name = "Qwen/Qwen-Image"
|
||||
|
||||
self.pipe = DiffusionPipeline.from_pretrained(model_name, torch_dtype=dtype)
|
||||
self.pipe = pipe.to(device)
|
||||
|
||||
def _run_pipeline(self, scheduler_output) -> ModelRunnerOutput: ...
|
||||
positive_magic = {
|
||||
"en": ", Ultra HD, 4K, cinematic composition.", # for english prompt
|
||||
"zh": ", 超清，4K，电影级构图." # for chinese prompt
|
||||
}
|
||||
|
||||
# Generate image
|
||||
prompt_embeds = self._get_and_process_prompt_embeds(scheduler_output, positive_magic)
|
||||
negative_prompt_embeds = self.pipe.embed_prompt(" ")
|
||||
|
||||
width, height = aspect_ratios["16:9"]
|
||||
|
||||
image = pipe(
|
||||
prompt_embeds=prompt_embeds,
|
||||
negative_prompt_embeds=negative_prompt_embeds,
|
||||
width=width,
|
||||
height=height,
|
||||
num_inference_steps=50,
|
||||
true_cfg_scale=4.0,
|
||||
generator=torch.Generator(device="cuda").manual_seed(42)
|
||||
).images[0]
|
||||
|
||||
output = self.wrap_image_as_ModelRunnerOutput(image)
|
||||
return output
|
||||
|
||||
# ---- Internal helpers (implementation-specific, not public API) ----
|
||||
def _resolve_device(self): ...
|
||||
def _resolve_dtype(self): ...
|
||||
def _get_model(self) -> nn.Module: ...
|
||||
def _get_and_process_prompt_embeds(self, scheduler_output, positive_magic):
|
||||
...
|
||||
#append the positive_magic to prompt and embed them to prompt embed tensors
|
||||
|
||||
# Functions related to workers should either raise NotImplementedError
|
||||
# or return empty defaults in this no-worker executor.
|
||||
|
||||
def collective_rpc(self, method, timeout=None, args=(), kwargs=None) -> list[Any]:
|
||||
# No workers in pipeline executor
|
||||
raise NotImplementedError("No workers in DiffusersPipelineExecutor")
|
||||
|
||||
def initialize_from_config(self, kv_cache_configs: list[KVCacheConfig]) -> None:
|
||||
return # no-op (pipeline already built in _init_executor)
|
||||
|
||||
def register_failure_callback(self, callback):
|
||||
self._failure_callback = callback
|
||||
|
||||
def determine_available_memory(self) -> list[int]: # bytes
|
||||
# Single device; return [available_bytes]. If CPU-only, return [0].
|
||||
return [self._determine_available_bytes(self._device)]
|
||||
|
||||
```
|
||||
|
||||
### 10. Worker
|
||||
|
||||
#### Inheritance strategy
|
||||
Prefer reusing the mature GPU Worker end-to-end. Worker class is selected by configuration (vllm_config). Do not add a new executor-specific worker binding. If customization is needed, override only `init_device` to construct the `DiffusionModelRunner`; all other behaviors (device init details, profiling, sleep/wake, PP/TP comms, execute path) remain from `vllm/v1/worker/gpu_worker.py::Worker`.
|
||||
|
||||
|
||||
Inherited and overridden
|
||||
```python
|
||||
# Optional: only if you need to plug a custom DiffusionModelRunner.
|
||||
from vllm.v1.worker.gpu_worker import Worker as GPUWorker
|
||||
from vllm.v1.worker.diffusion_model_runner import DiffusionModelRunner
|
||||
|
||||
class Worker(GPUWorker):
|
||||
def init_device(self) -> None:
|
||||
#those related to device check and init
|
||||
...
|
||||
|
||||
self.model_runner = DiffusionModelRunner(self.vllm_config, ...)
|
||||
```
|
||||
|
||||
### 11. Model Runner
|
||||
|
||||
### Function map (Model Runner)
|
||||
#### 1) Inherited and overridden
|
||||
Those parts relied to the KV Cache will be omitted if we do not register the model to the vllm config. The engine core will view it as do not require KV Cache, and handle it properly
|
||||
|
||||
Reuse `vllm/v1/outputs.py::ModelRunnerOutput`：
|
||||
- DiffusionModelRunner: Use the `pooler_output=[Tensor,...]` to return multi modal tensors
|
||||
- ARModelRunner: Use the `pooler_output=[Tensor,...]` to return hidden states.
|
||||
```python
|
||||
from typing import Optional, Union
|
||||
import torch
|
||||
from vllm.v1.worker.gpu_model_runner import GPUModelRunner
|
||||
from vllm.v1.outputs import ModelRunnerOutput
|
||||
|
||||
|
||||
class DiffusionModelRunner(GPUModelRunner):
|
||||
@torch.inference_mode()
|
||||
def execute_model(
|
||||
self,
|
||||
scheduler_output: "SchedulerOutput",
|
||||
intermediate_tensors: Optional[IntermediateTensors] = None,
|
||||
) -> Union[ModelRunnerOutput, IntermediateTensors]:
|
||||
...
|
||||
return ModelRunnerOutput(
|
||||
req_ids=[...],
|
||||
req_id_to_index={...},
|
||||
sampled_token_ids=[],
|
||||
spec_token_ids=None,
|
||||
logprobs=None,
|
||||
prompt_logprobs_dict={},
|
||||
pooler_output=[Tensor,...], # Return Hidden states
|
||||
kv_connector_output=None,
|
||||
num_nans_in_logits=None,
|
||||
)# return multi modal tensors via pooler_output=[Tensor,...]
|
||||
|
||||
|
||||
class ARModelRunner(GPUModelRunner):
|
||||
@torch.inference_mode()
|
||||
def execute_model(
|
||||
self,
|
||||
scheduler_output: "SchedulerOutput",
|
||||
intermediate_tensors: Optional[IntermediateTensors] = None,
|
||||
) -> Union[ModelRunnerOutput, IntermediateTensors]:
|
||||
...
|
||||
return ModelRunnerOutput(
|
||||
req_ids=[...],
|
||||
req_id_to_index={...},
|
||||
sampled_token_ids=[],
|
||||
spec_token_ids=None,
|
||||
logprobs=None,
|
||||
prompt_logprobs_dict={},
|
||||
pooler_output=[Tensor,...], # Return Hidden states
|
||||
kv_connector_output=None,
|
||||
num_nans_in_logits=None,
|
||||
)# return hidden states via pooler_output=[Tensor,...]
|
||||
```
|
||||
|
||||
### 12. RequestState Role and Usage
|
||||
```bash
|
||||
# RequestState serves as the per-request state tracker in OutputProcessor:
|
||||
# - Maintains request-specific state (tokens, logprobs, detokenizer, etc.)
|
||||
# - Converts EngineCoreOutput → RequestOutput/PoolingRequestOutput
|
||||
# - Manages request lifecycle from registration to completion
|
||||
AsyncLLM.add_request()
|
||||
↓
|
||||
OutputProcessor.add_request()
|
||||
↓
|
||||
RequestState.from_new_request() → create request state
|
||||
|
||||
AsyncLLM.__init__() / AsyncLLM.generate() / AsyncLLM.encode() → create a background loop that continuously pulls output from EngineCore
|
||||
↓
|
||||
OutputProcessor.process_outputs() → update state and process outputs
|
||||
↓
|
||||
RequestState.make_request_output() → convert to the final output, formatted as RequestOutput or PoolingRequestOutput
|
||||
↓
|
||||
RequestOutputCollector.put() → push to the queue (AsyncLLM)
|
||||
```
|
||||
|
||||
Need to add implementation for an existing method
|
||||
```python
|
||||
class RequestState:
|
||||
def __init__(self, request_id: str, parent_req: Optional[ParentRequest],
|
||||
request_index: int, lora_name: Optional[str],
|
||||
output_kind: RequestOutputKind, prompt: Optional[str],
|
||||
prompt_token_ids: list[int], logprobs_processor: Optional[LogprobsProcessor],
|
||||
detokenizer: Optional[IncrementalDetokenizer],
|
||||
max_tokens_param: Optional[int], arrival_time: float,
|
||||
queue: Optional[RequestOutputCollector], log_stats: bool):
|
||||
|
||||
def _new_pooling_output(self, pooling_output: torch.Tensor) -> PoolingOutput:
|
||||
"""Create PoolingOutput for multimodal/pooling requests"""
|
||||
return PoolingOutput(data=pooling_output)
|
||||
```
|
||||
|
||||
|
||||
### 13. OutputProcessor
|
||||
|
||||
For hidden state output, original OutputProcessor already support.
|
||||
|
||||
We only need to add new one for final multimodal output.
|
||||
```python
|
||||
class MultimodalOutputProcessor(OutputProcessor):
|
||||
"""Handles multimodal output processing"""
|
||||
|
||||
def __init__(self):
|
||||
self.output_handlers: Dict[str, OutputProcessor] = {}
|
||||
|
||||
def process_outputs(self, engine_core_outputs: list[EngineCoreOutput], ...):
|
||||
for engine_core_output in engine_core_outputs:
|
||||
# Option 1: Use output_type field (if EngineCoreOutput is extended)
|
||||
if engine_core_output.output_type == "image":
|
||||
self._process_image_output(engine_core_output)
|
||||
elif engine_core_output.output_type == "text+image":
|
||||
self._process_text_image_output(engine_core_output)
|
||||
elif engine_core_output.output_type == "latents":
|
||||
self._process_latents_output(engine_core_output)
|
||||
elif engine_core_output.output_type == "text":
|
||||
self._process_text_output(engine_core_output)
|
||||
else:
|
||||
# Fallback: use existing pooling_output logic
|
||||
if engine_core_output.pooling_output is not None:
|
||||
self._process_pooling_output(engine_core_output)
|
||||
else:
|
||||
self._process_text_output(engine_core_output)
|
||||
```
|
||||
@@ -1,8 +1,7 @@
|
||||
# Installation
|
||||
|
||||
vLLM-Omni supports the following hardware platforms:
|
||||
vLLM supports the following hardware platforms:
|
||||
|
||||
- [GPU](gpu.md)
|
||||
- [NVIDIA CUDA](gpu.md)
|
||||
- [AMD ROCm](gpu.md)
|
||||
- [NPU](npu.md)
|
||||
@@ -14,10 +14,6 @@ vLLM-Omni is a Python library that supports the following GPU variants. The libr
|
||||
|
||||
--8<-- "docs/getting_started/installation/gpu/cuda.inc.md:requirements"
|
||||
|
||||
=== "AMD ROCm"
|
||||
|
||||
--8<-- "docs/getting_started/installation/gpu/rocm.inc.md:requirements"
|
||||
|
||||
## Set up using Python
|
||||
|
||||
### Create a new Python environment
|
||||
@@ -30,11 +26,6 @@ vLLM-Omni is a Python library that supports the following GPU variants. The libr
|
||||
|
||||
--8<-- "docs/getting_started/installation/gpu/cuda.inc.md:pre-built-wheels"
|
||||
|
||||
|
||||
=== "AMD ROCm"
|
||||
|
||||
--8<-- "docs/getting_started/installation/gpu/rocm.inc.md:pre-built-wheels"
|
||||
|
||||
[](){ #build-from-source }
|
||||
|
||||
### Build wheel from source
|
||||
@@ -43,24 +34,7 @@ vLLM-Omni is a Python library that supports the following GPU variants. The libr
|
||||
|
||||
--8<-- "docs/getting_started/installation/gpu/cuda.inc.md:build-wheel-from-source"
|
||||
|
||||
=== "AMD ROCm"
|
||||
|
||||
--8<-- "docs/getting_started/installation/gpu/rocm.inc.md:build-wheel-from-source"
|
||||
|
||||
## Set up using Docker
|
||||
|
||||
### Build wheel from source
|
||||
|
||||
=== "NVIDIA CUDA"
|
||||
|
||||
--8<-- "docs/getting_started/installation/gpu/cuda.inc.md:build-wheel-from-source-in-docker"
|
||||
|
||||
=== "AMD ROCm"
|
||||
|
||||
--8<-- "docs/getting_started/installation/gpu/rocm.inc.md:build-wheel-from-source-in-docker"
|
||||
|
||||
### Pre-built images
|
||||
|
||||
=== "NVIDIA CUDA"
|
||||
|
||||
--8<-- "docs/getting_started/installation/gpu/cuda.inc.md:pre-built-images"
|
||||
To be released... Stay tuned!
|
||||
@@ -44,7 +44,7 @@ uv pip install vllm==0.11.0 --torch-backend=auto
|
||||
Install additional requirements for vLLM-Omni
|
||||
```bash
|
||||
git clone https://github.com/vllm-project/vllm-omni.git
|
||||
cd vllm-omni
|
||||
cd vllm_omni
|
||||
uv pip install -e .
|
||||
```
|
||||
|
||||
@@ -73,27 +73,3 @@ uv pip install --no-build-isolation --editable .
|
||||
</details>
|
||||
|
||||
# --8<-- [end:build-wheel-from-source]
|
||||
|
||||
# --8<-- [start:build-wheel-from-source-in-docker]
|
||||
|
||||
# --8<-- [end:build-wheel-from-source-in-docker]
|
||||
|
||||
# --8<-- [start:pre-built-images]
|
||||
|
||||
vLLM-Omni offers an official docker image for deployment. These images are built on top of vLLM docker images and available on Docker Hub as [vllm/vllm-omni](https://hub.docker.com/r/vllm/vllm-omni/tags). The version of vLLM-Omni indicates which release of vLLM it is based on.
|
||||
|
||||
Here's an example deployment command that has been verified on 2 x H100's:
|
||||
```bash
|
||||
docker run --runtime nvidia --gpus 2 \
|
||||
-v ~/.cache/huggingface:/root/.cache/huggingface \
|
||||
--env "HF_TOKEN=$HF_TOKEN" \
|
||||
-p 8091:8091 \
|
||||
--ipc=host \
|
||||
vllm/vllm-omni:v0.11.0rc1 \
|
||||
--model Qwen/Qwen3-Omni-30B-A3B-Instruct --port 8091
|
||||
```
|
||||
|
||||
!!! tip
|
||||
You can use this docker image to serve models the same way you would with in vLLM! To do so, make sure you overwrite the default entrypoint (`vllm serve --omni`) which works only for models supported in the vLLM-Omni project.
|
||||
|
||||
# --8<-- [end:pre-built-images]
|
||||
@@ -1,115 +0,0 @@
|
||||
# --8<-- [start:requirements]
|
||||
|
||||
- GPU: Validated on gfx942 (It should be supported on the AMD GPUs that are supported by vLLM.)
|
||||
|
||||
# --8<-- [end:requirements]
|
||||
# --8<-- [start:set-up-using-python]
|
||||
|
||||
vLLM-Omni current recommends the steps in under setup through Docker Images.
|
||||
|
||||
# --8<-- [start:pre-built-wheels]
|
||||
|
||||
# --8<-- [end:pre-built-wheels]
|
||||
|
||||
# --8<-- [start:build-wheel-from-source]
|
||||
|
||||
# --8<-- [end:build-wheel-from-source]
|
||||
|
||||
# --8<-- [start:build-wheel-from-source-in-docker]
|
||||
|
||||
#### Get the correct docker environment
|
||||
|
||||
The v0.11.0 releases is on Oct 3, 2025, so we will pull a AMD nightly docker image close to the release period. `rocm/vllm-dev:nightly_main_20251005` is picked as it has a correct `transformers` version that supports qwen3_omni.
|
||||
|
||||
`docker pull rocm/vllm-dev:nightly_main_20251005`
|
||||
|
||||
#### Launch the docker environment
|
||||
|
||||
In this docker environment, we have all the dependencies for vLLM installed, like `flash-attn` and `aiter`
|
||||
|
||||
```bash
|
||||
#!/bin/bash
|
||||
docker run -it \
|
||||
--privileged \
|
||||
--network=host \
|
||||
--group-add=video \
|
||||
--ipc=host \
|
||||
--cap-add=SYS_PTRACE \
|
||||
--security-opt seccomp=unconfined \
|
||||
--device /dev/kfd \
|
||||
--device /dev/dri \
|
||||
--name vllmomni \
|
||||
rocm/vllm-dev:nightly_main_20251005 \
|
||||
bash
|
||||
```
|
||||
|
||||
#### Update docker dependencies
|
||||
|
||||
```bash
|
||||
sudo apt update
|
||||
sudo apt install ffmpeg -y
|
||||
```
|
||||
|
||||
#### Installation of vLLM
|
||||
Install the v0.11.0 release of vLLM as vLLM-Omni is currently depend on v0.11.0.
|
||||
|
||||
```bash
|
||||
# uninstall existing vllm which is based on the main branch, not on the releases branch
|
||||
python3 -m pip uninstall -y vllm
|
||||
git clone https://github.com/vllm-project/vllm.git vllm0110
|
||||
cd vllm0110/
|
||||
|
||||
# checkout the version tag v0.11.0
|
||||
git checkout -b v0.11.0 v0.11.0
|
||||
|
||||
# you should see 0.11.0
|
||||
python -c "import setuptools_scm; print(setuptools_scm.get_version())"
|
||||
|
||||
# PYTORCH_ROCM_ARCH=<your-gpu-arch> python3 setup.py develop
|
||||
PYTORCH_ROCM_ARCH=gfx942 python3 setup.py develop
|
||||
```
|
||||
|
||||
!!! note
|
||||
vLLM release wheels based on the branch with prefix `releases/`, not from the tag as vLLM may cherry pick bugfixes after cutting a branch.
|
||||
|
||||
|
||||
#### Installation of vLLM-Omni
|
||||
|
||||
There are two approaches to installing `vllm-omni`
|
||||
|
||||
|
||||
1. Install from PyPI
|
||||
|
||||
```bash
|
||||
pip install vllm-omni
|
||||
|
||||
# you might want to also download the source code of the same version to run the examples
|
||||
```
|
||||
|
||||
2. Install from source
|
||||
|
||||
```bash
|
||||
# clone and checkout to the release version.
|
||||
git clone https://github.com/vllm-project/vllm-omni.git
|
||||
cd vllm-omni
|
||||
git checkout v0.11.0rc1
|
||||
pip install -e .
|
||||
```
|
||||
|
||||
#### Setup Environment Variables
|
||||
|
||||
After installing `vllm` and `vllm-omni` please setup the environment variable as well before running any of the examples and command.
|
||||
|
||||
```bash
|
||||
export MIOPEN_FIND_MODE=FAST
|
||||
export VLLM_ROCM_USE_AITER=1
|
||||
export VLLM_ROCM_USE_AITER_MHA=1
|
||||
export VLLM_ROCM_USE_AITER_LINEAR=0
|
||||
export VLLM_ROCM_USE_AITER_RMSNORM=0
|
||||
```
|
||||
|
||||
# --8<-- [end:build-wheel-from-source-in-docker]
|
||||
|
||||
# --8<-- [start:pre-built-images]
|
||||
|
||||
# --8<-- [end:pre-built-images]
|
||||
@@ -16,19 +16,7 @@ Please refer to [installation](installation/README.md)
|
||||
|
||||
## Offline Inference
|
||||
|
||||
Text-to-image generation quickstart with vLLM-Omni:
|
||||
|
||||
```python
|
||||
from vllm_omni.entrypoints.omni import Omni
|
||||
|
||||
if __name__ == "__main__":
|
||||
omni = Omni(model="Tongyi-MAI/Z-Image-Turbo")
|
||||
prompt = "a cup of coffee on the table"
|
||||
images = omni.generate(prompt)
|
||||
images[0].save("coffee.png")
|
||||
```
|
||||
|
||||
For more usages, please refer to [offline inference](../user_guide/examples/offline_inference/qwen2_5_omni.md)
|
||||
Please refer to [offline inference](../user_guide/examples/offline_inference/qwen2_5_omni.md)
|
||||
|
||||
## Online Serving with OpenAI-Completions API
|
||||
|
||||
|
||||
@@ -173,7 +173,7 @@ class Example:
|
||||
gh_file = gh_file.relative_to(ROOT_DIR)
|
||||
|
||||
# Make GitHub URL
|
||||
url = "https://github.com/vllm-project/vllm-omni/"
|
||||
url = "https://github.com/vllm-project/vllm/"
|
||||
url += "tree/main" if self.path.is_dir() else "blob/main"
|
||||
gh_url = f"{url}/{gh_file}"
|
||||
|
||||
@@ -183,7 +183,7 @@ class Example:
|
||||
|
||||
def generate(self) -> str:
|
||||
content = f"# {self.title}\n\n"
|
||||
url = "https://github.com/vllm-project/vllm-omni/"
|
||||
url = "https://github.com/vllm-project/vllm/"
|
||||
url += "tree/main" if self.path.is_dir() else "blob/main"
|
||||
content += f"Source <{url}/{self.path.relative_to(ROOT_DIR)}>.\n\n"
|
||||
|
||||
|
||||
@@ -25,11 +25,8 @@ a:not(:has(svg)):not(.md-icon):not(.autorefs-external) {
|
||||
|
||||
a[href*="localhost"]::after,
|
||||
a[href*="127.0.0.1"]::after,
|
||||
|
||||
/* Hide external link icons for all links */
|
||||
a[href^="//"]::after,
|
||||
a[href^="http://"]::after,
|
||||
a[href^="https://"]::after {
|
||||
a[href*="org.readthedocs.build"]::after,
|
||||
a[href*="docs.vllm.ai"]::after {
|
||||
display: none !important;
|
||||
}
|
||||
|
||||
|
||||
@@ -4,9 +4,9 @@ vLLM-Omni supports unified multimodal comprehension and generation models across
|
||||
|
||||
## Model Implementation
|
||||
|
||||
If vLLM-Omni natively supports a model, its implementation can be found in <gh-file:vllm_omni/model_executor/models> and <gh-file:vllm_omni/diffusion/models>.
|
||||
If vLLM-Omni natively supports a model, its implementation can be found in <gh-file:vllm-omni/model_executor/models> and <gh-file:vllm_omni/diffusion/models>.
|
||||
|
||||
## List of Supported Models for Nvidia GPU / AMD GPU
|
||||
## List of Supported Models for Nvidia GPU
|
||||
|
||||
<style>
|
||||
th {
|
||||
@@ -20,12 +20,9 @@ th {
|
||||
| `Qwen3OmniMoeForConditionalGeneration` | Qwen3-Omni | `Qwen/Qwen3-Omni-30B-A3B-Instruct` |
|
||||
| `Qwen2_5OmniForConditionalGeneration` | Qwen2.5-Omni | `Qwen/Qwen2.5-Omni-7B`, `Qwen/Qwen2.5-Omni-3B` |
|
||||
| `QwenImagePipeline` | Qwen-Image | `Qwen/Qwen-Image` |
|
||||
| `QwenImageEditPipeline` | Qwen-Image-Edit | `Qwen/Qwen-Image-Edit` |
|
||||
|`ZImagePipeline` | Z-Image | `Tongyi-MAI/Z-Image-Turbo` |
|
||||
| `WanPipeline` | Wan2.2 | `Wan-AI/Wan2.2-T2V-A14B-Diffusers` |
|
||||
|
||||
|
||||
## List of Supported Models for NPU
|
||||
## List of Supported Models for Ascend NPU
|
||||
|
||||
<style>
|
||||
th {
|
||||
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 107 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 97 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 126 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 367 KiB |
Binary file not shown.
|
Before Width: | Height: | Size: 480 KiB |
+1
-13
@@ -2,7 +2,7 @@
|
||||
|
||||
> Q: How many chips do I need to infer a model in vLLM-Omni?
|
||||
|
||||
A: Now, we support natively disaggregated deployment for different model stages within a model. There is a restriction that one chip can only have one AutoRegressive model stage. This is because the unified KV cache management of vLLM. Stages of other types can coexist within a chip. The restriction will be resolved in later version.
|
||||
A: Now, we support natively disaggregated deployment for different model stages within a model. There is a restriction that one chip can only has one AutoRegressive model stage. This is because the unified KV cache management of vLLM. Stages of other types can coexist with in a chip. The restriction will be resolved in later version.
|
||||
|
||||
> Q: When trying to run examples, I encounter error about backend of librosa or soundfile. How to solve it?
|
||||
|
||||
@@ -11,15 +11,3 @@ A: If you encounter error about backend of librosa, try to install ffmpeg with c
|
||||
sudo apt update
|
||||
sudo apt install ffmpeg
|
||||
```
|
||||
|
||||
> Q: I encounter some bugs or CI problems, which is urgent. How can I solve it?
|
||||
|
||||
A: At first, you can check current [issues](https://github.com/vllm-project/vllm-omni/issues) to find possible solutions. If none of these satisfy your demand and it is urgent, please find these [volunteers](https://docs.vllm.ai/projects/vllm-omni/en/latest/community/volunteers/) for help.
|
||||
|
||||
> Q: Does vLLM-Omni support AWQ or any other quantization?
|
||||
|
||||
A: vLLM-Omni partitions model into several stages. For AR stages, it will reuse main logic of LLMEngine in vLLM. So current quantization supported in vLLM should be also supported in vLLM-Omni for them. But systematic verification is ongoing. For quantization for DiffusionEngine, we are working on it. Please stay tuned and welcome contribution!
|
||||
|
||||
> Q: Does vLLM-Omni support multimodal streaming input and output?
|
||||
|
||||
A: Not yet. We already put it on the [Roadmap](https://github.com/vllm-project/vllm-omni/issues/165). Please stay tuned!
|
||||
@@ -1,30 +0,0 @@
|
||||
# Qwen-Image-Edit Offline Inference
|
||||
|
||||
Source <https://github.com/vllm-project/vllm-omni/tree/main/examples/offline_inference/image_to_image>. Added in PR [#196](https://github.com/vllm-project/vllm-omni/pull/196).
|
||||
|
||||
This example edits an input image with `Qwen/Qwen-Image-Edit` using the `image_edit.py` CLI.
|
||||
|
||||
## Local CLI Usage
|
||||
|
||||
```bash
|
||||
python image_edit.py \
|
||||
--image qwen_bear.png \
|
||||
--prompt "Let this mascot dance under the moon, surrounded by floating stars and poetic bubbles such as 'Be Kind'" \
|
||||
--output output_image_edit.png \
|
||||
--num_inference_steps 50 \
|
||||
--cfg_scale 4.0
|
||||
```
|
||||
|
||||
Key arguments:
|
||||
- `--image`: path to the source image (PNG/JPG, converted to RGB).
|
||||
- `--prompt` / `--negative_prompt`: text description (string).
|
||||
- `--cfg_scale`: true CFG scale for Qwen-Image-Edit (quality vs. fidelity).
|
||||
- `--num_inference_steps`: diffusion sampling steps (more steps = higher quality, slower).
|
||||
- `--output`: path to save the generated PNG.
|
||||
|
||||
## Example materials
|
||||
|
||||
??? abstract "image_edit.py"
|
||||
``````py
|
||||
--8<-- "examples/offline_inference/image_to_image/image_edit.py"
|
||||
``````
|
||||
@@ -1,14 +1,13 @@
|
||||
# Offline Example of vLLM-Omni for Qwen2.5-Omni
|
||||
|
||||
Source <https://github.com/vllm-project/vllm-omni/tree/main/examples/offline_inference/qwen2_5_omni>.
|
||||
Source <https://github.com/vllm-project/vllm/tree/main/examples/offline_inference/qwen2_5_omni>.
|
||||
|
||||
|
||||
## 🛠️ Installation
|
||||
|
||||
Please refer to [README.md](https://github.com/vllm-project/vllm-omni/tree/main/README.md)
|
||||
Please refer to [README.md](https://github.com/vllm-project/vllm/tree/main/README.md)
|
||||
|
||||
## Run examples (Qwen2.5-Omni)
|
||||
|
||||
### Multiple Prompts
|
||||
Download dataset from [seed_tts](https://drive.google.com/file/d/1GlSjVfSHkW3-leKKBlfrjuuTGqQ_xaLP/edit). To get the prompt, you can:
|
||||
```bash
|
||||
@@ -27,7 +26,6 @@ Then run the command below.
|
||||
```bash
|
||||
bash run_multiple_prompts.sh
|
||||
```
|
||||
|
||||
### Single Prompt
|
||||
Get into the example folder
|
||||
```bash
|
||||
@@ -38,34 +36,6 @@ Then run the command below.
|
||||
bash run_single_prompt.sh
|
||||
```
|
||||
|
||||
#### Using Local Media Files
|
||||
The `end2end.py` script supports local media files (audio, video, image) via CLI arguments:
|
||||
|
||||
```bash
|
||||
# Use single local media files
|
||||
python end2end.py --query-type use_image --image-path /path/to/image.jpg
|
||||
python end2end.py --query-type use_video --video-path /path/to/video.mp4
|
||||
python end2end.py --query-type use_audio --audio-path /path/to/audio.wav
|
||||
|
||||
# Combine multiple local media files
|
||||
python end2end.py --query-type mixed_modalities \
|
||||
--video-path /path/to/video.mp4 \
|
||||
--image-path /path/to/image.jpg \
|
||||
--audio-path /path/to/audio.wav
|
||||
|
||||
# Use audio from video file
|
||||
python end2end.py --query-type use_audio_in_video --video-path /path/to/video.mp4
|
||||
|
||||
```
|
||||
|
||||
If media file paths are not provided, the script will use default assets. Supported query types:
|
||||
- `use_image`: Image input only
|
||||
- `use_video`: Video input only
|
||||
- `use_audio`: Audio input only
|
||||
- `mixed_modalities`: Audio + image + video
|
||||
- `use_audio_in_video`: Extract audio from video
|
||||
- `text`: Text-only query
|
||||
|
||||
### FAQ
|
||||
|
||||
If you encounter error about backend of librosa, try to install ffmpeg with command below.
|
||||
|
||||
@@ -1,16 +1,15 @@
|
||||
# Offline Example of vLLM-Omni for Qwen3-Omni
|
||||
|
||||
Source <https://github.com/vllm-project/vllm-omni/tree/main/examples/offline_inference/qwen3_omni>.
|
||||
Source <https://github.com/vllm-project/vllm/tree/main/examples/offline_inference/qwen3_omni>.
|
||||
|
||||
|
||||
## 🛠️ Installation
|
||||
|
||||
Please refer to [README.md](https://github.com/vllm-project/vllm-omni/tree/main/README.md)
|
||||
Please refer to [README.md](https://github.com/vllm-project/vllm/tree/main/README.md)
|
||||
|
||||
## Run examples (Qwen3-Omni)
|
||||
|
||||
### Multiple Prompts
|
||||
Download dataset from [seed_tts](https://drive.google.com/file/d/1GlSjVfSHkW3-leKKBlfrjuuTGqQ_xaLP/edit). For processing dataset please refer to [Qwen2.5-Omni README.md](https://github.com/vllm-project/vllm-omni/tree/main/examples/offline_inference/qwen2_5_omni/README.md)
|
||||
Download dataset from [seed_tts](https://drive.google.com/file/d/1GlSjVfSHkW3-leKKBlfrjuuTGqQ_xaLP/edit). For processing dataset please refer to [Qwen2.5-Omni README.md](https://github.com/vllm-project/vllm/tree/main/examples/offline_inference/qwen2_5_omni/README.md)
|
||||
Get into the example folder
|
||||
```bash
|
||||
cd examples/offline_inference/qwen3_omni
|
||||
@@ -33,26 +32,6 @@ If you have not enough memory, you can set thinker with tensor parallel. Just ru
|
||||
bash run_single_prompt_tp.sh
|
||||
```
|
||||
|
||||
#### Using Local Media Files
|
||||
The `end2end.py` script supports local media files (audio, video, image) via command-line arguments:
|
||||
|
||||
```bash
|
||||
# Use local video file
|
||||
python end2end.py --query-type use_video --video-path /path/to/video.mp4
|
||||
|
||||
# Use local image file
|
||||
python end2end.py --query-type use_image --image-path /path/to/image.jpg
|
||||
|
||||
# Use local audio file
|
||||
python end2end.py --query-type use_audio --audio-path /path/to/audio.wav
|
||||
```
|
||||
|
||||
If media file paths are not provided, the script will use default assets. Supported query types:
|
||||
- `use_video`: Video input
|
||||
- `use_image`: Image input
|
||||
- `use_audio`: Audio input
|
||||
- `text`: Text-only query
|
||||
|
||||
### FAQ
|
||||
|
||||
If you encounter error about backend of librosa, try to install ffmpeg with command below.
|
||||
|
||||
+6
-5
@@ -1,9 +1,9 @@
|
||||
# Qwen-Image Offline Inference
|
||||
|
||||
Source <https://github.com/vllm-project/vllm-omni/tree/main/examples/offline_inference/text_to_image>.
|
||||
Source <https://github.com/vllm-project/vllm/tree/main/examples/offline_inference/qwen_image>.
|
||||
|
||||
|
||||
This folder provides two simple entrypoints for experimenting with `Qwen/Qwen-Image` `Tongyi-MAI/Z-Image-Turbo` using vLLM-Omni:
|
||||
This folder provides two simple entrypoints for experimenting with `Qwen/Qwen-Image` using vLLM-Omni:
|
||||
|
||||
- `text_to_image.py`: command-line script for single image generation.
|
||||
- `web_demo.py`: lightweight Gradio UI for interactive prompt/seed/CFG exploration.
|
||||
@@ -13,7 +13,6 @@ This folder provides two simple entrypoints for experimenting with `Qwen/Qwen-Im
|
||||
|
||||
```bash
|
||||
python text_to_image.py \
|
||||
--model Tongyi-MAI/Z-Image-Turbo \
|
||||
--prompt "a cup of coffee on the table" \
|
||||
--seed 42 \
|
||||
--cfg_scale 4.0 \
|
||||
@@ -34,6 +33,8 @@ Key arguments:
|
||||
- `--height/--width`: output resolution (defaults 1024x1024).
|
||||
- `--output`: path to save the generated PNG.
|
||||
|
||||
> ℹ️ Qwen-Image currently publishes best-effort presets at `1328x1328`, `1664x928`, `928x1664`, `1472x1140`, `1140x1472`, `1584x1056`, and `1056x1584`. Adjust `--height/--width` accordingly for the most reliable outcomes.
|
||||
|
||||
## Web UI Demo
|
||||
|
||||
Launch the gradio demo:
|
||||
@@ -48,9 +49,9 @@ Then open `http://localhost:7862/` on your local browser to interact with the we
|
||||
|
||||
??? abstract "gradio_demo.py"
|
||||
``````py
|
||||
--8<-- "examples/offline_inference/text_to_image/gradio_demo.py"
|
||||
--8<-- "examples/offline_inference/qwen_image/gradio_demo.py"
|
||||
``````
|
||||
??? abstract "text_to_image.py"
|
||||
``````py
|
||||
--8<-- "examples/offline_inference/text_to_image/text_to_image.py"
|
||||
--8<-- "examples/offline_inference/qwen_image/text_to_image.py"
|
||||
``````
|
||||
@@ -1,38 +0,0 @@
|
||||
# Wan2.2 Text-to-Video Offline Inference
|
||||
|
||||
Source <https://github.com/vllm-project/vllm-omni/tree/main/examples/offline_inference/text_to_video>. Added in PR [#202](https://github.com/vllm-project/vllm-omni/pull/202).
|
||||
|
||||
The `Wan-AI/Wan2.2-T2V-A14B-Diffusers` pipeline generates short videos from text prompts.
|
||||
|
||||
## Local CLI Usage
|
||||
|
||||
```bash
|
||||
python text_to_video.py \
|
||||
--prompt "Two anthropomorphic cats in comfy boxing gear and bright gloves fight intensely on a spotlighted stage." \
|
||||
--negative_prompt "<optional quality filter>" \
|
||||
--height 480 \
|
||||
--width 640 \
|
||||
--num_frames 32 \
|
||||
--guidance_scale 4.0 \
|
||||
--guidance_scale_high 3.0 \
|
||||
--num_inference_steps 40 \
|
||||
--fps 16 \
|
||||
--output t2v_out.mp4
|
||||
```
|
||||
|
||||
Key arguments:
|
||||
- `--prompt`: text description (string).
|
||||
- `--height/--width`: output resolution (defaults 720x1280). Dimensions should align with Wan VAE downsampling (multiples of 8).
|
||||
- `--num_frames`: Number of frames (Wan default is 81).
|
||||
- `--guidance_scale` and `--guidance_scale_high`: CFG scale (applied to low/high)..
|
||||
- `--negative_prompt`: optional list of artifacts to suppress (the PR demo used a long Chinese string).
|
||||
- `--boundary_ratio`: Boundary split ratio for low/high DiT.
|
||||
- `--fps`: frames per second for the saved MP4 (requires `diffusers` export_to_video).
|
||||
- `--output`: path to save the generated video.
|
||||
|
||||
## Example materials
|
||||
|
||||
??? abstract "text_to_video.py"
|
||||
``````py
|
||||
--8<-- "examples/offline_inference/wan22/text_to_video.py"
|
||||
``````
|
||||
@@ -1,16 +1,15 @@
|
||||
# Online serving Example of vLLM-Omni for Qwen2.5-Omni
|
||||
|
||||
Source <https://github.com/vllm-project/vllm-omni/tree/main/examples/online_serving/qwen2_5_omni>.
|
||||
Source <https://github.com/vllm-project/vllm/tree/main/examples/online_serving/qwen2_5_omni>.
|
||||
|
||||
|
||||
## 🛠️ Installation
|
||||
|
||||
Please refer to [README.md](https://github.com/vllm-project/vllm-omni/tree/main/README.md)
|
||||
Please refer to [README.md](https://github.com/vllm-project/vllm/tree/main/README.md)
|
||||
|
||||
## Run examples (Qwen2.5-Omni)
|
||||
|
||||
### Launch the Server
|
||||
|
||||
Launch the server
|
||||
```bash
|
||||
vllm serve Qwen/Qwen2.5-Omni-7B --omni --port 8091
|
||||
```
|
||||
@@ -20,59 +19,29 @@ If you have custom stage configs file, launch the server with command below
|
||||
vllm serve Qwen/Qwen2.5-Omni-7B --omni --port 8091 --stage-configs-path /path/to/stage_configs_file
|
||||
```
|
||||
|
||||
### Send Multi-modal Request
|
||||
|
||||
Get into the example folder
|
||||
```bash
|
||||
cd examples/online_serving/qwen2_5_omni
|
||||
```
|
||||
|
||||
#### Send request via python
|
||||
|
||||
Send request via python
|
||||
```bash
|
||||
python openai_chat_completion_client_for_multimodal_generation.py --query-type mixed_modalities
|
||||
```
|
||||
|
||||
The Python client supports the following command-line arguments:
|
||||
|
||||
- `--query-type` (or `-q`): Query type (default: `mixed_modalities`)
|
||||
- Options: `mixed_modalities`, `use_audio_in_video`, `multi_audios`, `text`
|
||||
- `--video-path` (or `-v`): Path to local video file or URL
|
||||
- If not provided and query-type uses video, uses default video URL
|
||||
- Supports local file paths (automatically encoded to base64) or HTTP/HTTPS URLs
|
||||
- Example: `--video-path /path/to/video.mp4` or `--video-path https://example.com/video.mp4`
|
||||
- `--image-path` (or `-i`): Path to local image file or URL
|
||||
- If not provided and query-type uses image, uses default image URL
|
||||
- Supports local file paths (automatically encoded to base64) or HTTP/HTTPS URLs
|
||||
- Supports common image formats: JPEG, PNG, GIF, WebP
|
||||
- Example: `--image-path /path/to/image.jpg` or `--image-path https://example.com/image.png`
|
||||
- `--audio-path` (or `-a`): Path to local audio file or URL
|
||||
- If not provided and query-type uses audio, uses default audio URL
|
||||
- Supports local file paths (automatically encoded to base64) or HTTP/HTTPS URLs
|
||||
- Supports common audio formats: MP3, WAV, OGG, FLAC, M4A
|
||||
- Example: `--audio-path /path/to/audio.wav` or `--audio-path https://example.com/audio.mp3`
|
||||
- `--prompt` (or `-p`): Custom text prompt/question
|
||||
- If not provided, uses default prompt for the selected query type
|
||||
- Example: `--prompt "What are the main activities shown in this video?"`
|
||||
|
||||
|
||||
For example, to use mixed modalities with all local files:
|
||||
|
||||
```bash
|
||||
python openai_chat_completion_client_for_multimodal_generation.py \
|
||||
--query-type mixed_modalities \
|
||||
--video-path /path/to/your/video.mp4 \
|
||||
--image-path /path/to/your/image.jpg \
|
||||
--audio-path /path/to/your/audio.wav \
|
||||
--prompt "Analyze all the media content and provide a comprehensive summary."
|
||||
```
|
||||
|
||||
#### Send request via curl
|
||||
|
||||
Send request via curl
|
||||
```bash
|
||||
bash run_curl_multimodal_generation.sh mixed_modalities
|
||||
```
|
||||
|
||||
### FAQ
|
||||
|
||||
If you encounter error about backend of librosa, try to install ffmpeg with command below.
|
||||
```
|
||||
sudo apt update
|
||||
sudo apt install ffmpeg
|
||||
```
|
||||
|
||||
## Run Local Web UI Demo
|
||||
|
||||
This Web UI demo allows users to interact with the model through a web browser.
|
||||
@@ -87,22 +56,25 @@ python gradio_demo.py --model Qwen/Qwen2.5-Omni-7B --port 7861
|
||||
|
||||
Then open `http://localhost:7861/` on your local browser to interact with the web UI.
|
||||
|
||||
The gradio script supports the following arguments:
|
||||
|
||||
- `--model`: Model name
|
||||
- `--ip`: Host/IP for Gradio server (default: 127.0.0.1)
|
||||
- `--port`: Port for Gradio server (default: 7861)
|
||||
- `--stage-configs-path`: Path to custom stage configs YAML file (optional)
|
||||
- `--share`: Share the Gradio demo publicly (creates a public link)
|
||||
### Options
|
||||
|
||||
### FAQ
|
||||
You can customize its basic launch parameters:
|
||||
|
||||
If you encounter error about backend of librosa, try to install ffmpeg with command below.
|
||||
```
|
||||
sudo apt update
|
||||
sudo apt install ffmpeg
|
||||
```bash
|
||||
python gradio_demo.py \
|
||||
--model Qwen/Qwen2.5-Omni-7B \
|
||||
--ip 127.0.0.1 \
|
||||
--port 7861 \
|
||||
--stage-configs-path /path/to/stage_configs.yaml
|
||||
```
|
||||
|
||||
- `--model`: Local model checkpoint to load (default `Qwen/Qwen2.5-Omni-7B`).
|
||||
- `--ip`: Host/IP for the Gradio server (default `127.0.0.1`).
|
||||
- `--port`: Port for the Gradio server (default `7861`).
|
||||
- `--stage-configs-path`: Optional path to custom stage configs YAML.
|
||||
- `--share`: Set to expose a temporary public link via Gradio.
|
||||
|
||||
## Example materials
|
||||
|
||||
??? abstract "gradio_demo.py"
|
||||
|
||||
@@ -1,16 +1,15 @@
|
||||
# Online serving Example of vLLM-Omni for Qwen3-omni
|
||||
|
||||
Source <https://github.com/vllm-project/vllm-omni/tree/main/examples/online_serving/qwen3_omni>.
|
||||
Source <https://github.com/vllm-project/vllm/tree/main/examples/online_serving/qwen3_omni>.
|
||||
|
||||
|
||||
## 🛠️ Installation
|
||||
|
||||
Please refer to [README.md](https://github.com/vllm-project/vllm-omni/tree/main/README.md)
|
||||
Please refer to [README.md](https://github.com/vllm-project/vllm/tree/main/README.md)
|
||||
|
||||
## Run examples (Qwen3-Omni)
|
||||
|
||||
### Launch the Server
|
||||
|
||||
Launch the server
|
||||
```bash
|
||||
vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091
|
||||
```
|
||||
@@ -20,59 +19,21 @@ If you have custom stage configs file, launch the server with command below
|
||||
vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091 --stage-configs-path /path/to/stage_configs_file
|
||||
```
|
||||
|
||||
### Send Multi-modal Request
|
||||
|
||||
Get into the example folder
|
||||
```bash
|
||||
cd examples/online_serving/qwen3_omni
|
||||
```
|
||||
|
||||
#### Send request via python
|
||||
|
||||
Send request via python
|
||||
```bash
|
||||
python openai_chat_completion_client_for_multimodal_generation.py --query-type use_image
|
||||
```
|
||||
|
||||
The Python client supports the following command-line arguments:
|
||||
|
||||
- `--query-type` (or `-q`): Query type (default: `use_video`)
|
||||
- Options: `text`, `use_audio`, `use_image`, `use_video`
|
||||
- `--model` (or `-m`): Model name/path (default: `Qwen/Qwen3-Omni-30B-A3B-Instruct`)
|
||||
- `--video-path` (or `-v`): Path to local video file or URL
|
||||
- If not provided and query-type is `use_video`, uses default video URL
|
||||
- Supports local file paths (automatically encoded to base64) or HTTP/HTTPS URLs
|
||||
- Example: `--video-path /path/to/video.mp4` or `--video-path https://example.com/video.mp4`
|
||||
- `--image-path` (or `-i`): Path to local image file or URL
|
||||
- If not provided and query-type is `use_image`, uses default image URL
|
||||
- Supports local file paths (automatically encoded to base64) or HTTP/HTTPS URLs
|
||||
- Supports common image formats: JPEG, PNG, GIF, WebP
|
||||
- Example: `--image-path /path/to/image.jpg` or `--image-path https://example.com/image.png`
|
||||
- `--audio-path` (or `-a`): Path to local audio file or URL
|
||||
- If not provided and query-type is `use_audio`, uses default audio URL
|
||||
- Supports local file paths (automatically encoded to base64) or HTTP/HTTPS URLs
|
||||
- Supports common audio formats: MP3, WAV, OGG, FLAC, M4A
|
||||
- Example: `--audio-path /path/to/audio.wav` or `--audio-path https://example.com/audio.mp3`
|
||||
- `--prompt` (or `-p`): Custom text prompt/question
|
||||
- If not provided, uses default prompt for the selected query type
|
||||
- Example: `--prompt "What are the main activities shown in this video?"`
|
||||
|
||||
|
||||
For example, to use a local video file with custom prompt:
|
||||
|
||||
```bash
|
||||
python openai_chat_completion_client_for_multimodal_generation.py \
|
||||
--query-type use_video \
|
||||
--video-path /path/to/your/video.mp4 \
|
||||
--prompt "What are the main activities shown in this video?"
|
||||
```
|
||||
|
||||
#### Send request via curl
|
||||
|
||||
Send request via curl
|
||||
```bash
|
||||
bash run_curl_multimodal_generation.sh use_image
|
||||
```
|
||||
|
||||
|
||||
### FAQ
|
||||
|
||||
If you encounter error about backend of librosa, try to install ffmpeg with command below.
|
||||
@@ -96,9 +57,23 @@ python gradio_demo.py --model Qwen/Qwen3-Omni-30B-A3B-Instruct --port 7861
|
||||
Then open `http://localhost:7861/` on your local browser to interact with the web UI.
|
||||
|
||||
|
||||
The gradio script supports the following arguments:
|
||||
### Options
|
||||
|
||||
The gradio demo also supports running with an existing API server and can be customized with the following arguments.
|
||||
|
||||
|
||||
```bash
|
||||
python gradio_demo.py \
|
||||
--model Qwen/Qwen3-Omni-30B-A3B-Instruct \
|
||||
--use-api-server \
|
||||
--api-base http://localhost:8091/v1 \
|
||||
--ip 127.0.0.1 \
|
||||
--port 7861
|
||||
```
|
||||
|
||||
- `--model`: Model name
|
||||
- `--use-api-server`: If set, connect to an existing vLLM HTTP API server instead of running AsyncOmni locally.
|
||||
- `--api-base`: Base URL for vllm serve (only used when `use-api-server` is set, default: http://localhost:8091/v1)
|
||||
- `--ip`: Host/IP for Gradio server (default: 127.0.0.1)
|
||||
- `--port`: Port for Gradio server (default: 7861)
|
||||
- `--stage-configs-path`: Path to custom stage configs YAML file (optional)
|
||||
@@ -114,10 +89,6 @@ The gradio script supports the following arguments:
|
||||
``````py
|
||||
--8<-- "examples/online_serving/qwen3_omni/openai_chat_completion_client_for_multimodal_generation.py"
|
||||
``````
|
||||
??? abstract "qwen3_omni_moe_thinking.yaml"
|
||||
``````yaml
|
||||
--8<-- "examples/online_serving/qwen3_omni/qwen3_omni_moe_thinking.yaml"
|
||||
``````
|
||||
??? abstract "run_curl_multimodal_generation.sh"
|
||||
``````sh
|
||||
--8<-- "examples/online_serving/qwen3_omni/run_curl_multimodal_generation.sh"
|
||||
|
||||
@@ -1,99 +0,0 @@
|
||||
# TeaCache Configuration Guide
|
||||
|
||||
TeaCache speeds up diffusion model inference by caching transformer computations when consecutive timesteps are similar. This typically provides **1.5x-2.0x speedup** with minimal quality loss.
|
||||
|
||||
## Quick Start
|
||||
|
||||
Enable TeaCache by setting `cache_adapter` to `"tea_cache"`:
|
||||
|
||||
```python
|
||||
from vllm_omni import Omni
|
||||
|
||||
# Simple configuration - model_class_name and model_type are auto-detected
|
||||
omni = Omni(
|
||||
model="Qwen/Qwen-Image",
|
||||
cache_adapter="tea_cache",
|
||||
cache_config={
|
||||
"rel_l1_thresh": 0.2 # Optional, defaults to 0.2
|
||||
}
|
||||
)
|
||||
outputs = omni.generate(prompt="A cat sitting on a windowsill", num_inference_steps=50)
|
||||
```
|
||||
|
||||
### Using Environment Variable
|
||||
|
||||
You can also enable TeaCache via environment variable:
|
||||
|
||||
```bash
|
||||
export DIFFUSION_CACHE_ADAPTER=tea_cache
|
||||
```
|
||||
|
||||
Then initialize without explicitly setting `cache_adapter`:
|
||||
|
||||
```python
|
||||
from vllm_omni import Omni
|
||||
|
||||
omni = Omni(
|
||||
model="Qwen/Qwen-Image",
|
||||
cache_config={"rel_l1_thresh": 0.2} # Optional
|
||||
)
|
||||
```
|
||||
|
||||
## Configuration Parameters
|
||||
|
||||
### `rel_l1_thresh` (float, default: `0.2`)
|
||||
|
||||
Controls the balance between speed and quality. Lower values prioritize quality, higher values prioritize speed.
|
||||
|
||||
**Recommended values:**
|
||||
|
||||
- `0.2` - **~1.5x speedup** with minimal quality loss (recommended)
|
||||
- `0.4` - **~1.8x speedup** with slight quality loss
|
||||
- `0.6` - **~2.0x speedup** with noticeable quality loss
|
||||
- `0.8` - **~2.25x speedup** with significant quality loss
|
||||
|
||||
## Examples
|
||||
|
||||
### Python API
|
||||
|
||||
```python
|
||||
from vllm_omni import Omni
|
||||
|
||||
omni = Omni(
|
||||
model="Qwen/Qwen-Image",
|
||||
cache_adapter="tea_cache",
|
||||
cache_config={"rel_l1_thresh": 0.2}
|
||||
)
|
||||
outputs = omni.generate(prompt="A cat sitting on a windowsill", num_inference_steps=50)
|
||||
```
|
||||
|
||||
## Performance Tuning
|
||||
|
||||
Start with the default `rel_l1_thresh=0.2` and adjust based on your needs:
|
||||
|
||||
- **Maximum quality**: Use `0.1-0.2`
|
||||
- **Balanced**: Use `0.2-0.4` (recommended)
|
||||
- **Maximum speed**: Use `0.6-0.8` (may reduce quality)
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Quality Degradation
|
||||
|
||||
If you notice quality issues, lower the threshold:
|
||||
|
||||
```python
|
||||
cache_config={"rel_l1_thresh": 0.1} # More conservative caching
|
||||
```
|
||||
|
||||
## Supported Models
|
||||
|
||||
Currently supported models:
|
||||
|
||||
- **QwenImagePipeline**
|
||||
- Model identifier: `"Qwen/Qwen-Image"`
|
||||
- Example: `model="Qwen/Qwen-Image"`
|
||||
|
||||
Future support (extractors in development):
|
||||
|
||||
- **FluxPipeline** - Coming soon
|
||||
- **CogVideoXPipeline** - Coming soon
|
||||
@@ -1,143 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""
|
||||
Example script for image editing with Qwen-Image-Edit.
|
||||
|
||||
Usage:
|
||||
python image_edit.py \
|
||||
--image input.png \
|
||||
--prompt "Let this mascot dance under the moon, surrounded by floating stars and poetic bubbles such as 'Be Kind'" \
|
||||
--output output_image_edit.png \
|
||||
--num_inference_steps 50 \
|
||||
--cfg_scale 4.0
|
||||
|
||||
For more options, run:
|
||||
python image_edit.py --help
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
import torch
|
||||
from PIL import Image
|
||||
|
||||
from vllm_omni.entrypoints.omni import Omni
|
||||
from vllm_omni.utils.platform_utils import detect_device_type, is_npu
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description="Edit an image with Qwen-Image-Edit.")
|
||||
parser.add_argument(
|
||||
"--model",
|
||||
default="Qwen/Qwen-Image-Edit",
|
||||
help="Diffusion model name or local path.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--image",
|
||||
type=str,
|
||||
required=True,
|
||||
help="Path to input image file (PNG, JPG, etc.).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--prompt",
|
||||
type=str,
|
||||
required=True,
|
||||
help="Text prompt describing the edit to make to the image.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--negative_prompt",
|
||||
type=str,
|
||||
default=" ",
|
||||
required=False,
|
||||
)
|
||||
parser.add_argument(
|
||||
"--seed",
|
||||
type=int,
|
||||
default=0,
|
||||
help="Random seed for deterministic results.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--cfg_scale",
|
||||
type=float,
|
||||
default=4.0,
|
||||
help="True classifier-free guidance scale specific to Qwen-Image-Edit.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output",
|
||||
type=str,
|
||||
default="output_image_edit.png",
|
||||
help="Path to save the edited image (PNG).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--num_outputs_per_prompt",
|
||||
type=int,
|
||||
default=1,
|
||||
help="Number of images to generate for the given prompt.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--num_inference_steps",
|
||||
type=int,
|
||||
default=50,
|
||||
help="Number of denoising steps for the diffusion sampler.",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def main():
|
||||
args = parse_args()
|
||||
|
||||
# Validate input image exists
|
||||
if not os.path.exists(args.image):
|
||||
raise FileNotFoundError(f"Input image not found: {args.image}")
|
||||
|
||||
# Load input image
|
||||
input_image = Image.open(args.image).convert("RGB")
|
||||
print(f"Loaded input image from {args.image} (size: {input_image.size})")
|
||||
|
||||
device = detect_device_type()
|
||||
generator = torch.Generator(device=device).manual_seed(args.seed)
|
||||
|
||||
# Enable VAE memory optimizations on NPU
|
||||
vae_use_slicing = is_npu()
|
||||
vae_use_tiling = is_npu()
|
||||
|
||||
# Initialize Omni with QwenImageEditPipeline
|
||||
omni = Omni(
|
||||
model=args.model,
|
||||
model_class_name="QwenImageEditPipeline",
|
||||
vae_use_slicing=vae_use_slicing,
|
||||
vae_use_tiling=vae_use_tiling,
|
||||
)
|
||||
print("Pipeline loaded")
|
||||
|
||||
# Generate edited image
|
||||
images = omni.generate(
|
||||
prompt=args.prompt,
|
||||
pil_image=input_image,
|
||||
negative_prompt=args.negative_prompt,
|
||||
generator=generator,
|
||||
true_cfg_scale=args.cfg_scale,
|
||||
num_inference_steps=args.num_inference_steps,
|
||||
num_outputs_per_prompt=args.num_outputs_per_prompt,
|
||||
)
|
||||
|
||||
# Save output image(s)
|
||||
output_path = Path(args.output)
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
suffix = output_path.suffix or ".png"
|
||||
stem = output_path.stem or "output_image_edit"
|
||||
|
||||
if args.num_outputs_per_prompt <= 1:
|
||||
images[0].save(output_path)
|
||||
print(f"Saved edited image to {os.path.abspath(output_path)}")
|
||||
else:
|
||||
for idx, img in enumerate(images):
|
||||
save_path = output_path.parent / f"{stem}_{idx}{suffix}"
|
||||
img.save(save_path)
|
||||
print(f"Saved edited image to {os.path.abspath(save_path)}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,30 +0,0 @@
|
||||
# Qwen-Image-Edit Offline Inference
|
||||
|
||||
Source <https://github.com/vllm-project/vllm-omni/tree/main/examples/offline_inference/image_to_image>. Added in PR [#196](https://github.com/vllm-project/vllm-omni/pull/196).
|
||||
|
||||
This example edits an input image with `Qwen/Qwen-Image-Edit` using the `image_edit.py` CLI.
|
||||
|
||||
## Local CLI Usage
|
||||
|
||||
```bash
|
||||
python image_edit.py \
|
||||
--image qwen_bear.png \
|
||||
--prompt "Let this mascot dance under the moon, surrounded by floating stars and poetic bubbles such as 'Be Kind'" \
|
||||
--output output_image_edit.png \
|
||||
--num_inference_steps 50 \
|
||||
--cfg_scale 4.0
|
||||
```
|
||||
|
||||
Key arguments:
|
||||
- `--image`: path to the source image (PNG/JPG, converted to RGB).
|
||||
- `--prompt` / `--negative_prompt`: text description (string).
|
||||
- `--cfg_scale`: true CFG scale for Qwen-Image-Edit (quality vs. fidelity).
|
||||
- `--num_inference_steps`: diffusion sampling steps (more steps = higher quality, slower).
|
||||
- `--output`: path to save the generated PNG.
|
||||
|
||||
## Example materials
|
||||
|
||||
??? abstract "image_edit.py"
|
||||
``````py
|
||||
--8<-- "examples/offline_inference/image_to_image/image_edit.py"
|
||||
``````
|
||||
@@ -5,7 +5,6 @@
|
||||
Please refer to [README.md](../../../README.md)
|
||||
|
||||
## Run examples (Qwen2.5-Omni)
|
||||
|
||||
### Multiple Prompts
|
||||
Download dataset from [seed_tts](https://drive.google.com/file/d/1GlSjVfSHkW3-leKKBlfrjuuTGqQ_xaLP/edit). To get the prompt, you can:
|
||||
```bash
|
||||
@@ -24,7 +23,6 @@ Then run the command below.
|
||||
```bash
|
||||
bash run_multiple_prompts.sh
|
||||
```
|
||||
|
||||
### Single Prompt
|
||||
Get into the example folder
|
||||
```bash
|
||||
@@ -35,34 +33,6 @@ Then run the command below.
|
||||
bash run_single_prompt.sh
|
||||
```
|
||||
|
||||
#### Using Local Media Files
|
||||
The `end2end.py` script supports local media files (audio, video, image) via CLI arguments:
|
||||
|
||||
```bash
|
||||
# Use single local media files
|
||||
python end2end.py --query-type use_image --image-path /path/to/image.jpg
|
||||
python end2end.py --query-type use_video --video-path /path/to/video.mp4
|
||||
python end2end.py --query-type use_audio --audio-path /path/to/audio.wav
|
||||
|
||||
# Combine multiple local media files
|
||||
python end2end.py --query-type mixed_modalities \
|
||||
--video-path /path/to/video.mp4 \
|
||||
--image-path /path/to/image.jpg \
|
||||
--audio-path /path/to/audio.wav
|
||||
|
||||
# Use audio from video file
|
||||
python end2end.py --query-type use_audio_in_video --video-path /path/to/video.mp4
|
||||
|
||||
```
|
||||
|
||||
If media file paths are not provided, the script will use default assets. Supported query types:
|
||||
- `use_image`: Image input only
|
||||
- `use_video`: Video input only
|
||||
- `use_audio`: Audio input only
|
||||
- `mixed_modalities`: Audio + image + video
|
||||
- `use_audio_in_video`: Extract audio from video
|
||||
- `text`: Text-only query
|
||||
|
||||
### FAQ
|
||||
|
||||
If you encounter error about backend of librosa, try to install ffmpeg with command below.
|
||||
|
||||
@@ -6,15 +6,12 @@ with the correct prompt format on Qwen2.5-Omni
|
||||
"""
|
||||
|
||||
import os
|
||||
from typing import NamedTuple, Optional
|
||||
from typing import NamedTuple
|
||||
|
||||
import librosa
|
||||
import numpy as np
|
||||
import soundfile as sf
|
||||
from PIL import Image
|
||||
from vllm.assets.audio import AudioAsset
|
||||
from vllm.assets.image import ImageAsset
|
||||
from vllm.assets.video import VideoAsset, video_to_ndarrays
|
||||
from vllm.assets.video import VideoAsset
|
||||
from vllm.multimodal.image import convert_image_mode
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.utils import FlexibleArgumentParser
|
||||
@@ -57,13 +54,7 @@ def get_text_query(question: str = None) -> QueryResult:
|
||||
)
|
||||
|
||||
|
||||
def get_mixed_modalities_query(
|
||||
video_path: Optional[str] = None,
|
||||
image_path: Optional[str] = None,
|
||||
audio_path: Optional[str] = None,
|
||||
num_frames: int = 16,
|
||||
sampling_rate: int = 16000,
|
||||
) -> QueryResult:
|
||||
def get_mixed_modalities_query() -> QueryResult:
|
||||
question = "What is recited in the audio? What is the content of this image? Why is this video funny?"
|
||||
prompt = (
|
||||
f"<|im_start|>system\n{default_system}<|im_end|>\n"
|
||||
@@ -73,49 +64,20 @@ def get_mixed_modalities_query(
|
||||
f"{question}<|im_end|>\n"
|
||||
f"<|im_start|>assistant\n"
|
||||
)
|
||||
|
||||
# Load video
|
||||
if video_path:
|
||||
if not os.path.exists(video_path):
|
||||
raise FileNotFoundError(f"Video file not found: {video_path}")
|
||||
video_frames = video_to_ndarrays(video_path, num_frames=num_frames)
|
||||
else:
|
||||
video_frames = VideoAsset(name="baby_reading", num_frames=num_frames).np_ndarrays
|
||||
|
||||
# Load image
|
||||
if image_path:
|
||||
if not os.path.exists(image_path):
|
||||
raise FileNotFoundError(f"Image file not found: {image_path}")
|
||||
pil_image = Image.open(image_path)
|
||||
image_data = convert_image_mode(pil_image, "RGB")
|
||||
else:
|
||||
image_data = convert_image_mode(ImageAsset("cherry_blossom").pil_image, "RGB")
|
||||
|
||||
# Load audio
|
||||
if audio_path:
|
||||
if not os.path.exists(audio_path):
|
||||
raise FileNotFoundError(f"Audio file not found: {audio_path}")
|
||||
audio_signal, sr = librosa.load(audio_path, sr=sampling_rate)
|
||||
audio_data = (audio_signal.astype(np.float32), sr)
|
||||
else:
|
||||
audio_data = AudioAsset("mary_had_lamb").audio_and_sample_rate
|
||||
|
||||
return QueryResult(
|
||||
inputs={
|
||||
"prompt": prompt,
|
||||
"multi_modal_data": {
|
||||
"audio": audio_data,
|
||||
"image": image_data,
|
||||
"video": video_frames,
|
||||
"audio": AudioAsset("mary_had_lamb").audio_and_sample_rate,
|
||||
"image": convert_image_mode(ImageAsset("cherry_blossom").pil_image, "RGB"),
|
||||
"video": VideoAsset(name="baby_reading", num_frames=16).np_ndarrays,
|
||||
},
|
||||
},
|
||||
limit_mm_per_prompt={"audio": 1, "image": 1, "video": 1},
|
||||
)
|
||||
|
||||
|
||||
def get_use_audio_in_video_query(
|
||||
video_path: Optional[str] = None, num_frames: int = 16, sampling_rate: int = 16000
|
||||
) -> QueryResult:
|
||||
def get_use_audio_in_video_query() -> QueryResult:
|
||||
question = "Describe the content of the video, then convert what the baby say into text."
|
||||
prompt = (
|
||||
f"<|im_start|>system\n{default_system}<|im_end|>\n"
|
||||
@@ -123,24 +85,14 @@ def get_use_audio_in_video_query(
|
||||
f"{question}<|im_end|>\n"
|
||||
f"<|im_start|>assistant\n"
|
||||
)
|
||||
|
||||
if video_path:
|
||||
if not os.path.exists(video_path):
|
||||
raise FileNotFoundError(f"Video file not found: {video_path}")
|
||||
video_frames = video_to_ndarrays(video_path, num_frames=num_frames)
|
||||
# Extract audio from video file
|
||||
audio_signal, sr = librosa.load(video_path, sr=sampling_rate)
|
||||
audio = (audio_signal.astype(np.float32), sr)
|
||||
else:
|
||||
asset = VideoAsset(name="baby_reading", num_frames=num_frames)
|
||||
video_frames = asset.np_ndarrays
|
||||
audio = asset.get_audio(sampling_rate=sampling_rate)
|
||||
asset = VideoAsset(name="baby_reading", num_frames=16)
|
||||
audio = asset.get_audio(sampling_rate=16000)
|
||||
|
||||
return QueryResult(
|
||||
inputs={
|
||||
"prompt": prompt,
|
||||
"multi_modal_data": {
|
||||
"video": video_frames,
|
||||
"video": asset.np_ndarrays,
|
||||
"audio": audio,
|
||||
},
|
||||
"mm_processor_kwargs": {
|
||||
@@ -151,7 +103,7 @@ def get_use_audio_in_video_query(
|
||||
)
|
||||
|
||||
|
||||
def get_multi_audios_query(audio_path: Optional[str] = None, sampling_rate: int = 16000) -> QueryResult:
|
||||
def get_multi_audios_query() -> QueryResult:
|
||||
question = "Are these two audio clips the same?"
|
||||
prompt = (
|
||||
f"<|im_start|>system\n{default_system}<|im_end|>\n"
|
||||
@@ -160,28 +112,14 @@ def get_multi_audios_query(audio_path: Optional[str] = None, sampling_rate: int
|
||||
f"{question}<|im_end|>\n"
|
||||
f"<|im_start|>assistant\n"
|
||||
)
|
||||
|
||||
if audio_path:
|
||||
if not os.path.exists(audio_path):
|
||||
raise FileNotFoundError(f"Audio file not found: {audio_path}")
|
||||
audio_signal, sr = librosa.load(audio_path, sr=sampling_rate)
|
||||
audio_data = (audio_signal.astype(np.float32), sr)
|
||||
# Use the provided audio as the first audio, default as second
|
||||
audio_list = [
|
||||
audio_data,
|
||||
AudioAsset("mary_had_lamb").audio_and_sample_rate,
|
||||
]
|
||||
else:
|
||||
audio_list = [
|
||||
AudioAsset("winning_call").audio_and_sample_rate,
|
||||
AudioAsset("mary_had_lamb").audio_and_sample_rate,
|
||||
]
|
||||
|
||||
return QueryResult(
|
||||
inputs={
|
||||
"prompt": prompt,
|
||||
"multi_modal_data": {
|
||||
"audio": audio_list,
|
||||
"audio": [
|
||||
AudioAsset("winning_call").audio_and_sample_rate,
|
||||
AudioAsset("mary_had_lamb").audio_and_sample_rate,
|
||||
],
|
||||
},
|
||||
},
|
||||
limit_mm_per_prompt={
|
||||
@@ -190,135 +128,17 @@ def get_multi_audios_query(audio_path: Optional[str] = None, sampling_rate: int
|
||||
)
|
||||
|
||||
|
||||
def get_image_query(question: str = None, image_path: Optional[str] = None) -> QueryResult:
|
||||
if question is None:
|
||||
question = "What is the content of this image?"
|
||||
prompt = (
|
||||
f"<|im_start|>system\n{default_system}<|im_end|>\n"
|
||||
"<|im_start|>user\n<|vision_bos|><|IMAGE|><|vision_eos|>"
|
||||
f"{question}<|im_end|>\n"
|
||||
f"<|im_start|>assistant\n"
|
||||
)
|
||||
|
||||
if image_path:
|
||||
if not os.path.exists(image_path):
|
||||
raise FileNotFoundError(f"Image file not found: {image_path}")
|
||||
pil_image = Image.open(image_path)
|
||||
image_data = convert_image_mode(pil_image, "RGB")
|
||||
else:
|
||||
image_data = convert_image_mode(ImageAsset("cherry_blossom").pil_image, "RGB")
|
||||
|
||||
return QueryResult(
|
||||
inputs={
|
||||
"prompt": prompt,
|
||||
"multi_modal_data": {
|
||||
"image": image_data,
|
||||
},
|
||||
},
|
||||
limit_mm_per_prompt={"image": 1},
|
||||
)
|
||||
|
||||
|
||||
def get_video_query(question: str = None, video_path: Optional[str] = None, num_frames: int = 16) -> QueryResult:
|
||||
if question is None:
|
||||
question = "Why is this video funny?"
|
||||
prompt = (
|
||||
f"<|im_start|>system\n{default_system}<|im_end|>\n"
|
||||
"<|im_start|>user\n<|vision_bos|><|VIDEO|><|vision_eos|>"
|
||||
f"{question}<|im_end|>\n"
|
||||
f"<|im_start|>assistant\n"
|
||||
)
|
||||
|
||||
if video_path:
|
||||
if not os.path.exists(video_path):
|
||||
raise FileNotFoundError(f"Video file not found: {video_path}")
|
||||
video_frames = video_to_ndarrays(video_path, num_frames=num_frames)
|
||||
else:
|
||||
video_frames = VideoAsset(name="baby_reading", num_frames=num_frames).np_ndarrays
|
||||
|
||||
return QueryResult(
|
||||
inputs={
|
||||
"prompt": prompt,
|
||||
"multi_modal_data": {
|
||||
"video": video_frames,
|
||||
},
|
||||
},
|
||||
limit_mm_per_prompt={"video": 1},
|
||||
)
|
||||
|
||||
|
||||
def get_audio_query(question: str = None, audio_path: Optional[str] = None, sampling_rate: int = 16000) -> QueryResult:
|
||||
if question is None:
|
||||
question = "What is the content of this audio?"
|
||||
prompt = (
|
||||
f"<|im_start|>system\n{default_system}<|im_end|>\n"
|
||||
"<|im_start|>user\n<|audio_bos|><|AUDIO|><|audio_eos|>"
|
||||
f"{question}<|im_end|>\n"
|
||||
f"<|im_start|>assistant\n"
|
||||
)
|
||||
|
||||
if audio_path:
|
||||
if not os.path.exists(audio_path):
|
||||
raise FileNotFoundError(f"Audio file not found: {audio_path}")
|
||||
audio_signal, sr = librosa.load(audio_path, sr=sampling_rate)
|
||||
audio_data = (audio_signal.astype(np.float32), sr)
|
||||
else:
|
||||
audio_data = AudioAsset("mary_had_lamb").audio_and_sample_rate
|
||||
|
||||
return QueryResult(
|
||||
inputs={
|
||||
"prompt": prompt,
|
||||
"multi_modal_data": {
|
||||
"audio": audio_data,
|
||||
},
|
||||
},
|
||||
limit_mm_per_prompt={"audio": 1},
|
||||
)
|
||||
|
||||
|
||||
query_map = {
|
||||
"mixed_modalities": get_mixed_modalities_query,
|
||||
"use_audio_in_video": get_use_audio_in_video_query,
|
||||
"multi_audios": get_multi_audios_query,
|
||||
"use_image": get_image_query,
|
||||
"use_video": get_video_query,
|
||||
"use_audio": get_audio_query,
|
||||
"text": get_text_query,
|
||||
}
|
||||
|
||||
|
||||
def main(args):
|
||||
model_name = "Qwen/Qwen2.5-Omni-7B"
|
||||
|
||||
# Get paths from args
|
||||
video_path = getattr(args, "video_path", None)
|
||||
image_path = getattr(args, "image_path", None)
|
||||
audio_path = getattr(args, "audio_path", None)
|
||||
num_frames = getattr(args, "num_frames", 16)
|
||||
sampling_rate = getattr(args, "sampling_rate", 16000)
|
||||
|
||||
# Get the query function and call it with appropriate parameters
|
||||
query_func = query_map[args.query_type]
|
||||
if args.query_type == "mixed_modalities":
|
||||
query_result = query_func(
|
||||
video_path=video_path,
|
||||
image_path=image_path,
|
||||
audio_path=audio_path,
|
||||
num_frames=num_frames,
|
||||
sampling_rate=sampling_rate,
|
||||
)
|
||||
elif args.query_type == "use_audio_in_video":
|
||||
query_result = query_func(video_path=video_path, num_frames=num_frames, sampling_rate=sampling_rate)
|
||||
elif args.query_type == "multi_audios":
|
||||
query_result = query_func(audio_path=audio_path, sampling_rate=sampling_rate)
|
||||
elif args.query_type == "use_image":
|
||||
query_result = query_func(image_path=image_path)
|
||||
elif args.query_type == "use_video":
|
||||
query_result = query_func(video_path=video_path, num_frames=num_frames)
|
||||
elif args.query_type == "use_audio":
|
||||
query_result = query_func(audio_path=audio_path, sampling_rate=sampling_rate)
|
||||
else:
|
||||
query_result = query_func()
|
||||
query_result = query_map[args.query_type]()
|
||||
|
||||
omni_llm = Omni(
|
||||
model=model_name,
|
||||
@@ -463,48 +283,7 @@ def parse_args():
|
||||
default=None,
|
||||
help="Path to a .txt file with one prompt per line (preferred).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--video-path",
|
||||
"-v",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Path to local video file. If not provided, uses default video asset.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--image-path",
|
||||
"-i",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Path to local image file. If not provided, uses default image asset.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--audio-path",
|
||||
"-a",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Path to local audio file. If not provided, uses default audio asset.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--num-frames",
|
||||
type=int,
|
||||
default=16,
|
||||
help="Number of frames to extract from video (default: 16).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--sampling-rate",
|
||||
type=int,
|
||||
default=16000,
|
||||
help="Sampling rate for audio loading (default: 16000).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--worker-backend", type=str, default="multi_process", choices=["multi_process", "ray"], help="backend"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--ray-address",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Address of the Ray cluster.",
|
||||
)
|
||||
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
|
||||
@@ -5,7 +5,6 @@
|
||||
Please refer to [README.md](../../../README.md)
|
||||
|
||||
## Run examples (Qwen3-Omni)
|
||||
|
||||
### Multiple Prompts
|
||||
Download dataset from [seed_tts](https://drive.google.com/file/d/1GlSjVfSHkW3-leKKBlfrjuuTGqQ_xaLP/edit). For processing dataset please refer to [Qwen2.5-Omni README.md](../qwen2_5_omni/README.md)
|
||||
Get into the example folder
|
||||
@@ -30,26 +29,6 @@ If you have not enough memory, you can set thinker with tensor parallel. Just ru
|
||||
bash run_single_prompt_tp.sh
|
||||
```
|
||||
|
||||
#### Using Local Media Files
|
||||
The `end2end.py` script supports local media files (audio, video, image) via command-line arguments:
|
||||
|
||||
```bash
|
||||
# Use local video file
|
||||
python end2end.py --query-type use_video --video-path /path/to/video.mp4
|
||||
|
||||
# Use local image file
|
||||
python end2end.py --query-type use_image --image-path /path/to/image.jpg
|
||||
|
||||
# Use local audio file
|
||||
python end2end.py --query-type use_audio --audio-path /path/to/audio.wav
|
||||
```
|
||||
|
||||
If media file paths are not provided, the script will use default assets. Supported query types:
|
||||
- `use_video`: Video input
|
||||
- `use_image`: Image input
|
||||
- `use_audio`: Audio input
|
||||
- `text`: Text-only query
|
||||
|
||||
### FAQ
|
||||
|
||||
If you encounter error about backend of librosa, try to install ffmpeg with command below.
|
||||
|
||||
@@ -6,16 +6,13 @@ with the correct prompt format on Qwen3-Omni (thinker only).
|
||||
"""
|
||||
|
||||
import os
|
||||
from typing import NamedTuple, Optional
|
||||
from typing import NamedTuple
|
||||
|
||||
import librosa
|
||||
import numpy as np
|
||||
import soundfile as sf
|
||||
from PIL import Image
|
||||
from vllm import SamplingParams
|
||||
from vllm.assets.audio import AudioAsset
|
||||
from vllm.assets.image import ImageAsset
|
||||
from vllm.assets.video import VideoAsset, video_to_ndarrays
|
||||
from vllm.assets.video import VideoAsset
|
||||
from vllm.multimodal.image import convert_image_mode
|
||||
from vllm.utils import FlexibleArgumentParser
|
||||
|
||||
@@ -57,7 +54,7 @@ def get_text_query(question: str = None) -> QueryResult:
|
||||
)
|
||||
|
||||
|
||||
def get_video_query(question: str = None, video_path: Optional[str] = None, num_frames: int = 16) -> QueryResult:
|
||||
def get_video_query(question: str = None) -> QueryResult:
|
||||
if question is None:
|
||||
question = "Why is this video funny?"
|
||||
prompt = (
|
||||
@@ -67,25 +64,18 @@ def get_video_query(question: str = None, video_path: Optional[str] = None, num_
|
||||
f"<|im_start|>assistant\n"
|
||||
)
|
||||
|
||||
if video_path:
|
||||
if not os.path.exists(video_path):
|
||||
raise FileNotFoundError(f"Video file not found: {video_path}")
|
||||
video_frames = video_to_ndarrays(video_path, num_frames=num_frames)
|
||||
else:
|
||||
video_frames = VideoAsset(name="baby_reading", num_frames=num_frames).np_ndarrays
|
||||
|
||||
return QueryResult(
|
||||
inputs={
|
||||
"prompt": prompt,
|
||||
"multi_modal_data": {
|
||||
"video": video_frames,
|
||||
"video": VideoAsset(name="baby_reading", num_frames=16).np_ndarrays,
|
||||
},
|
||||
},
|
||||
limit_mm_per_prompt={"video": 1},
|
||||
)
|
||||
|
||||
|
||||
def get_image_query(question: str = None, image_path: Optional[str] = None) -> QueryResult:
|
||||
def get_image_query(question: str = None) -> QueryResult:
|
||||
if question is None:
|
||||
question = "What is the content of this image?"
|
||||
prompt = (
|
||||
@@ -94,27 +84,18 @@ def get_image_query(question: str = None, image_path: Optional[str] = None) -> Q
|
||||
f"{question}<|im_end|>\n"
|
||||
f"<|im_start|>assistant\n"
|
||||
)
|
||||
|
||||
if image_path:
|
||||
if not os.path.exists(image_path):
|
||||
raise FileNotFoundError(f"Image file not found: {image_path}")
|
||||
pil_image = Image.open(image_path)
|
||||
image_data = convert_image_mode(pil_image, "RGB")
|
||||
else:
|
||||
image_data = convert_image_mode(ImageAsset("cherry_blossom").pil_image, "RGB")
|
||||
|
||||
return QueryResult(
|
||||
inputs={
|
||||
"prompt": prompt,
|
||||
"multi_modal_data": {
|
||||
"image": image_data,
|
||||
"image": convert_image_mode(ImageAsset("cherry_blossom").pil_image, "RGB"),
|
||||
},
|
||||
},
|
||||
limit_mm_per_prompt={"image": 1},
|
||||
)
|
||||
|
||||
|
||||
def get_audio_query(question: str = None, audio_path: Optional[str] = None, sampling_rate: int = 16000) -> QueryResult:
|
||||
def get_audio_query(question: str = None) -> QueryResult:
|
||||
if question is None:
|
||||
question = "What is the content of this audio?"
|
||||
prompt = (
|
||||
@@ -123,20 +104,11 @@ def get_audio_query(question: str = None, audio_path: Optional[str] = None, samp
|
||||
f"{question}<|im_end|>\n"
|
||||
f"<|im_start|>assistant\n"
|
||||
)
|
||||
|
||||
if audio_path:
|
||||
if not os.path.exists(audio_path):
|
||||
raise FileNotFoundError(f"Audio file not found: {audio_path}")
|
||||
audio_signal, sr = librosa.load(audio_path, sr=sampling_rate)
|
||||
audio_data = (audio_signal.astype(np.float32), sr)
|
||||
else:
|
||||
audio_data = AudioAsset("mary_had_lamb").audio_and_sample_rate
|
||||
|
||||
return QueryResult(
|
||||
inputs={
|
||||
"prompt": prompt,
|
||||
"multi_modal_data": {
|
||||
"audio": audio_data,
|
||||
"audio": AudioAsset("mary_had_lamb").audio_and_sample_rate,
|
||||
},
|
||||
},
|
||||
limit_mm_per_prompt={"audio": 1},
|
||||
@@ -153,22 +125,7 @@ query_map = {
|
||||
|
||||
def main(args):
|
||||
model_name = "Qwen/Qwen3-Omni-30B-A3B-Instruct"
|
||||
|
||||
# Get paths from args
|
||||
video_path = getattr(args, "video_path", None)
|
||||
image_path = getattr(args, "image_path", None)
|
||||
audio_path = getattr(args, "audio_path", None)
|
||||
|
||||
# Get the query function and call it with appropriate parameters
|
||||
query_func = query_map[args.query_type]
|
||||
if args.query_type == "use_video":
|
||||
query_result = query_func(video_path=video_path, num_frames=getattr(args, "num_frames", 16))
|
||||
elif args.query_type == "use_image":
|
||||
query_result = query_func(image_path=image_path)
|
||||
elif args.query_type == "use_audio":
|
||||
query_result = query_func(audio_path=audio_path, sampling_rate=getattr(args, "sampling_rate", 16000))
|
||||
else:
|
||||
query_result = query_func()
|
||||
query_result = query_map[args.query_type]()
|
||||
|
||||
omni_llm = Omni(
|
||||
model=model_name,
|
||||
@@ -326,39 +283,6 @@ def parse_args():
|
||||
default=None,
|
||||
help="Path to a stage configs file.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--video-path",
|
||||
"-v",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Path to local video file. If not provided, uses default video asset.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--image-path",
|
||||
"-i",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Path to local image file. If not provided, uses default image asset.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--audio-path",
|
||||
"-a",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Path to local audio file. If not provided, uses default audio asset.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--num-frames",
|
||||
type=int,
|
||||
default=16,
|
||||
help="Number of frames to extract from video (default: 16).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--sampling-rate",
|
||||
type=int,
|
||||
default=16000,
|
||||
help="Sampling rate for audio loading (default: 16000).",
|
||||
)
|
||||
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
+2
-13
@@ -1,21 +1,10 @@
|
||||
# Qwen-Image Offline Inference
|
||||
|
||||
This folder provides several entrypoints for experimenting with `Qwen/Qwen-Image` using vLLM-Omni:
|
||||
This folder provides two simple entrypoints for experimenting with `Qwen/Qwen-Image` using vLLM-Omni:
|
||||
|
||||
- `text_to_image.py`: command-line script for single image generation with advanced options.
|
||||
- `text_to_image.py`: command-line script for single image generation.
|
||||
- `web_demo.py`: lightweight Gradio UI for interactive prompt/seed/CFG exploration.
|
||||
|
||||
## Basic Usage
|
||||
|
||||
```python
|
||||
from vllm_omni.entrypoints.omni import Omni
|
||||
|
||||
if __name__ == "__main__":
|
||||
omni = Omni(model="Qwen/Qwen-Image")
|
||||
prompt = "a cup of coffee on the table"
|
||||
images = omni.generate(prompt)
|
||||
images[0].save("coffee.png")
|
||||
```
|
||||
|
||||
## Local CLI Usage
|
||||
|
||||
+1
@@ -105,6 +105,7 @@ def build_demo(args: argparse.Namespace) -> gr.Blocks:
|
||||
generator=generator,
|
||||
true_cfg_scale=float(cfg_scale_value),
|
||||
num_inference_steps=num_steps,
|
||||
num_images_per_prompt=num_images,
|
||||
num_outputs_per_prompt=num_images,
|
||||
)
|
||||
return [img for img in images[:num_images]]
|
||||
+1
@@ -65,6 +65,7 @@ def main():
|
||||
generator=generator,
|
||||
true_cfg_scale=args.cfg_scale,
|
||||
num_inference_steps=args.num_inference_steps,
|
||||
num_images_per_prompt=args.num_images_per_prompt,
|
||||
num_outputs_per_prompt=args.num_images_per_prompt,
|
||||
)
|
||||
|
||||
@@ -1,38 +0,0 @@
|
||||
# Wan2.2 Text-to-Video Offline Inference
|
||||
|
||||
Source <https://github.com/vllm-project/vllm-omni/tree/main/examples/offline_inference/text_to_video>. Added in PR [#202](https://github.com/vllm-project/vllm-omni/pull/202).
|
||||
|
||||
The `Wan-AI/Wan2.2-T2V-A14B-Diffusers` pipeline generates short videos from text prompts.
|
||||
|
||||
## Local CLI Usage
|
||||
|
||||
```bash
|
||||
python text_to_video.py \
|
||||
--prompt "Two anthropomorphic cats in comfy boxing gear and bright gloves fight intensely on a spotlighted stage." \
|
||||
--negative_prompt "<optional quality filter>" \
|
||||
--height 480 \
|
||||
--width 640 \
|
||||
--num_frames 32 \
|
||||
--guidance_scale 4.0 \
|
||||
--guidance_scale_high 3.0 \
|
||||
--num_inference_steps 40 \
|
||||
--fps 16 \
|
||||
--output t2v_out.mp4
|
||||
```
|
||||
|
||||
Key arguments:
|
||||
- `--prompt`: text description (string).
|
||||
- `--height/--width`: output resolution (defaults 720x1280). Dimensions should align with Wan VAE downsampling (multiples of 8).
|
||||
- `--num_frames`: Number of frames (Wan default is 81).
|
||||
- `--guidance_scale` and `--guidance_scale_high`: CFG scale (applied to low/high)..
|
||||
- `--negative_prompt`: optional list of artifacts to suppress (the PR demo used a long Chinese string).
|
||||
- `--boundary_ratio`: Boundary split ratio for low/high DiT.
|
||||
- `--fps`: frames per second for the saved MP4 (requires `diffusers` export_to_video).
|
||||
- `--output`: path to save the generated video.
|
||||
|
||||
## Example materials
|
||||
|
||||
??? abstract "text_to_video.py"
|
||||
``````py
|
||||
--8<-- "examples/offline_inference/wan22/text_to_video.py"
|
||||
``````
|
||||
@@ -1,105 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
from vllm_omni.entrypoints.omni import Omni
|
||||
from vllm_omni.utils.platform_utils import detect_device_type, is_npu
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description="Generate a video with Wan2.2 T2V.")
|
||||
parser.add_argument(
|
||||
"--model",
|
||||
default="Wan-AI/Wan2.2-T2V-A14B-Diffusers",
|
||||
help="Diffusers Wan2.2 model ID or local path.",
|
||||
)
|
||||
parser.add_argument("--prompt", default="A serene lakeside sunrise with mist over the water.", help="Text prompt.")
|
||||
parser.add_argument("--negative_prompt", default="", help="Negative prompt.")
|
||||
parser.add_argument("--seed", type=int, default=42, help="Random seed.")
|
||||
parser.add_argument("--guidance_scale", type=float, default=4.0, help="CFG scale (applied to low/high).")
|
||||
parser.add_argument("--guidance_scale_high", type=float, default=None, help="Optional separate CFG for high-noise.")
|
||||
parser.add_argument("--height", type=int, default=720, help="Video height.")
|
||||
parser.add_argument("--width", type=int, default=1280, help="Video width.")
|
||||
parser.add_argument("--num_frames", type=int, default=81, help="Number of frames (Wan default is 81).")
|
||||
parser.add_argument("--num_inference_steps", type=int, default=40, help="Sampling steps.")
|
||||
parser.add_argument("--boundary_ratio", type=float, default=0.875, help="Boundary split ratio for low/high DiT.")
|
||||
parser.add_argument(
|
||||
"--flow_shift", type=float, default=5.0, help="Scheduler flow_shift (5.0 for 720p, 12.0 for 480p)."
|
||||
)
|
||||
parser.add_argument("--output", type=str, default="wan22_output.mp4", help="Path to save the video (mp4).")
|
||||
parser.add_argument("--fps", type=int, default=24, help="Frames per second for the output video.")
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def main():
|
||||
args = parse_args()
|
||||
device = detect_device_type()
|
||||
generator = torch.Generator(device=device).manual_seed(args.seed)
|
||||
|
||||
# Enable VAE memory optimizations on NPU
|
||||
vae_use_slicing = is_npu()
|
||||
vae_use_tiling = is_npu()
|
||||
|
||||
omni = Omni(
|
||||
model=args.model,
|
||||
vae_use_slicing=vae_use_slicing,
|
||||
vae_use_tiling=vae_use_tiling,
|
||||
boundary_ratio=args.boundary_ratio,
|
||||
flow_shift=args.flow_shift,
|
||||
)
|
||||
|
||||
frames = omni.generate(
|
||||
args.prompt,
|
||||
negative_prompt=args.negative_prompt,
|
||||
height=args.height,
|
||||
width=args.width,
|
||||
generator=generator,
|
||||
guidance_scale=args.guidance_scale,
|
||||
guidance_scale_2=args.guidance_scale_high,
|
||||
num_inference_steps=args.num_inference_steps,
|
||||
num_frames=args.num_frames,
|
||||
)
|
||||
|
||||
output_path = Path(args.output)
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
try:
|
||||
from diffusers.utils import export_to_video
|
||||
except ImportError:
|
||||
raise ImportError("diffusers is required for export_to_video.")
|
||||
|
||||
# frames may be np.ndarray (preferred) or torch.Tensor
|
||||
# export_to_video expects a list of frames with values in [0, 1]
|
||||
if isinstance(frames, torch.Tensor):
|
||||
video_tensor = frames.detach().cpu()
|
||||
if video_tensor.dim() == 5:
|
||||
# [B, C, F, H, W] or [B, F, H, W, C]
|
||||
if video_tensor.shape[1] in (3, 4):
|
||||
video_tensor = video_tensor[0].permute(1, 2, 3, 0)
|
||||
else:
|
||||
video_tensor = video_tensor[0]
|
||||
elif video_tensor.dim() == 4 and video_tensor.shape[0] in (3, 4):
|
||||
video_tensor = video_tensor.permute(1, 2, 3, 0)
|
||||
# If float, assume [-1,1] and normalize to [0,1]
|
||||
if video_tensor.is_floating_point():
|
||||
video_tensor = video_tensor.clamp(-1, 1) * 0.5 + 0.5
|
||||
video_array = video_tensor.float().numpy()
|
||||
else:
|
||||
video_array = frames
|
||||
if hasattr(video_array, "shape") and video_array.ndim == 5:
|
||||
video_array = video_array[0]
|
||||
|
||||
# Convert 4D array (frames, H, W, C) to list of frames for export_to_video
|
||||
if isinstance(video_array, np.ndarray) and video_array.ndim == 4:
|
||||
video_array = list(video_array)
|
||||
|
||||
export_to_video(video_array, str(output_path), fps=args.fps)
|
||||
print(f"Saved generated video to {output_path}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -91,58 +91,9 @@ def parse_args():
|
||||
default=None,
|
||||
help="Path to custom stage configs YAML file (optional).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--log-stats",
|
||||
action="store_true",
|
||||
help="Enable statistics logging for AsyncOmni.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--log-file",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Path prefix for AsyncOmni log files.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--init-sleep-seconds",
|
||||
type=int,
|
||||
default=30,
|
||||
help="Seconds to sleep between starting stage processes.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--shm-threshold-bytes",
|
||||
type=int,
|
||||
default=65536,
|
||||
help="Threshold in bytes for using shared memory IPC.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--batch-timeout",
|
||||
type=int,
|
||||
default=10,
|
||||
help="Batching timeout (seconds) inside each stage.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--init-timeout",
|
||||
type=int,
|
||||
default=ASYNC_INIT_TIMEOUT,
|
||||
help="Timeout (seconds) for initializing all stages.",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def build_async_omni_cli_args(base_args: argparse.Namespace) -> argparse.Namespace:
|
||||
"""Construct the minimal CLI args Namespace expected by AsyncOmni."""
|
||||
return argparse.Namespace(
|
||||
model=base_args.model,
|
||||
stage_configs_path=getattr(base_args, "stage_configs_path", None),
|
||||
log_stats=bool(getattr(base_args, "log_stats", False)),
|
||||
log_file=getattr(base_args, "log_file", None),
|
||||
init_sleep_seconds=int(getattr(base_args, "init_sleep_seconds", 30)),
|
||||
shm_threshold_bytes=int(getattr(base_args, "shm_threshold_bytes", 65536)),
|
||||
batch_timeout=int(getattr(base_args, "batch_timeout", 10)),
|
||||
init_timeout=int(getattr(base_args, "init_timeout", ASYNC_INIT_TIMEOUT)),
|
||||
)
|
||||
|
||||
|
||||
def build_sampling_params(seed: int, model_key: str) -> list[SamplingParams]:
|
||||
"""Build SamplingParams objects by reusing the dict definitions."""
|
||||
return [SamplingParams(**params_dict) for params_dict in build_sampling_params_dict(seed, model_key)]
|
||||
@@ -549,8 +500,11 @@ def main():
|
||||
print(f"Using custom stage configs: {args.stage_configs_path}")
|
||||
|
||||
sampling_params = build_sampling_params(SEED, model_name)
|
||||
cli_args = build_async_omni_cli_args(args)
|
||||
omni = AsyncOmni(model=args.model, cli_args=cli_args)
|
||||
omni = AsyncOmni(
|
||||
model=args.model,
|
||||
stage_configs_path=args.stage_configs_path,
|
||||
init_timeout=ASYNC_INIT_TIMEOUT,
|
||||
)
|
||||
print("✓ AsyncOmni initialized successfully")
|
||||
prompt_args_template = create_prompt_args(args)
|
||||
|
||||
|
||||
@@ -94,58 +94,9 @@ def parse_args():
|
||||
default=None,
|
||||
help="Path to custom stage configs YAML file (optional).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--log-stats",
|
||||
action="store_true",
|
||||
help="Enable statistics logging for AsyncOmni.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--log-file",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Path prefix for AsyncOmni log files.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--init-sleep-seconds",
|
||||
type=int,
|
||||
default=30,
|
||||
help="Seconds to sleep between starting stage processes.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--shm-threshold-bytes",
|
||||
type=int,
|
||||
default=65536,
|
||||
help="Threshold in bytes for using shared memory IPC.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--batch-timeout",
|
||||
type=int,
|
||||
default=10,
|
||||
help="Batching timeout (seconds) inside each stage.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--init-timeout",
|
||||
type=int,
|
||||
default=ASYNC_INIT_TIMEOUT,
|
||||
help="Timeout (seconds) for initializing all stages.",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def build_async_omni_cli_args(base_args: argparse.Namespace) -> argparse.Namespace:
|
||||
"""Construct the minimal CLI args Namespace expected by AsyncOmni."""
|
||||
return argparse.Namespace(
|
||||
model=base_args.model,
|
||||
stage_configs_path=getattr(base_args, "stage_configs_path", None),
|
||||
log_stats=bool(getattr(base_args, "log_stats", False)),
|
||||
log_file=getattr(base_args, "log_file", None),
|
||||
init_sleep_seconds=int(getattr(base_args, "init_sleep_seconds", 30)),
|
||||
shm_threshold_bytes=int(getattr(base_args, "shm_threshold_bytes", 65536)),
|
||||
batch_timeout=int(getattr(base_args, "batch_timeout", 10)),
|
||||
init_timeout=int(getattr(base_args, "init_timeout", ASYNC_INIT_TIMEOUT)),
|
||||
)
|
||||
|
||||
|
||||
def build_sampling_params(seed: int, model_key: str) -> list[SamplingParams]:
|
||||
"""Build SamplingParams objects by reusing the dict definitions."""
|
||||
return [SamplingParams(**params_dict) for params_dict in build_sampling_params_dict(seed, model_key)]
|
||||
@@ -555,8 +506,11 @@ def main():
|
||||
print(f"Using custom stage configs: {args.stage_configs_path}")
|
||||
|
||||
sampling_params = build_sampling_params(SEED, model_name)
|
||||
cli_args = build_async_omni_cli_args(args)
|
||||
omni = AsyncOmni(model=args.model, cli_args=cli_args)
|
||||
omni = AsyncOmni(
|
||||
model=args.model,
|
||||
stage_configs_path=args.stage_configs_path,
|
||||
init_timeout=ASYNC_INIT_TIMEOUT,
|
||||
)
|
||||
print("✓ AsyncOmni initialized successfully")
|
||||
prompt_args_template = create_prompt_args(args)
|
||||
|
||||
|
||||
@@ -1,36 +0,0 @@
|
||||
# Stage config for running Qwen3-Omni-MoE-Thinking (text-only output)
|
||||
# This config is for models like Qwen3-Omni-30B-A3B-Thinking that only have the
|
||||
# thinker component and do not support audio output.
|
||||
#
|
||||
# Single stage: Thinker (multimodal understanding + text generation)
|
||||
|
||||
# The following config has been verified on 2x H100-80G GPUs.
|
||||
stage_args:
|
||||
- stage_id: 0
|
||||
runtime:
|
||||
devices: "0,1"
|
||||
max_batch_size: 1
|
||||
engine_args:
|
||||
model_stage: thinker
|
||||
model_arch: Qwen3OmniMoeForConditionalGeneration
|
||||
worker_cls: vllm_omni.worker.gpu_ar_worker.GPUARWorker
|
||||
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
|
||||
gpu_memory_utilization: 0.9
|
||||
enforce_eager: true
|
||||
trust_remote_code: true
|
||||
engine_output_type: text
|
||||
distributed_executor_backend: "mp"
|
||||
enable_prefix_caching: false
|
||||
hf_config_name: thinker_config
|
||||
tensor_parallel_size: 2
|
||||
final_output: true
|
||||
final_output_type: text
|
||||
is_comprehension: true
|
||||
default_sampling_params:
|
||||
temperature: 0.4
|
||||
top_p: 0.9
|
||||
top_k: 1
|
||||
max_tokens: 2048
|
||||
seed: 42
|
||||
detokenize: True
|
||||
repetition_penalty: 1.05
|
||||
@@ -156,7 +156,6 @@ markers = [
|
||||
"integration: Integration tests",
|
||||
"benchmark: Benchmark tests",
|
||||
"slow: Slow tests",
|
||||
"core_model: enable this model test in each PR instead of only nightly",
|
||||
]
|
||||
|
||||
[tool.typos.default]
|
||||
|
||||
@@ -1,36 +0,0 @@
|
||||
import os
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from vllm.logger import init_logger
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def clean_gpu_memory_between_tests():
|
||||
if os.getenv("VLLM_TEST_CLEAN_GPU_MEMORY", "0") != "1":
|
||||
yield
|
||||
return
|
||||
|
||||
# Wait for GPU memory to be cleared before starting the test
|
||||
import gc
|
||||
|
||||
from tests.utils import wait_for_gpu_memory_to_clear
|
||||
|
||||
num_gpus = torch.cuda.device_count()
|
||||
if num_gpus > 0:
|
||||
try:
|
||||
wait_for_gpu_memory_to_clear(
|
||||
devices=list(range(num_gpus)),
|
||||
threshold_ratio=0.1,
|
||||
)
|
||||
except ValueError as e:
|
||||
logger.info("Failed to clean GPU memory: %s", e)
|
||||
|
||||
yield
|
||||
|
||||
# Clean up GPU memory after the test
|
||||
if torch.cuda.is_available():
|
||||
torch.cuda.empty_cache()
|
||||
gc.collect()
|
||||
@@ -1,236 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm_omni.distributed.omni_connectors.adapter import try_recv_via_connector, try_send_via_connector
|
||||
from vllm_omni.distributed.omni_connectors.connectors.shm_connector import SharedMemoryConnector
|
||||
from vllm_omni.distributed.omni_connectors.utils.config import ConnectorSpec, OmniTransferConfig
|
||||
from vllm_omni.distributed.omni_connectors.utils.initialization import get_connectors_config_for_stage
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_objects():
|
||||
return {"connector": MagicMock(), "metrics": MagicMock(), "queue_fn": MagicMock()}
|
||||
|
||||
|
||||
def test_send_success(mock_objects):
|
||||
"""Test try_send_via_connector success path."""
|
||||
# Setup
|
||||
mock_connector = mock_objects["connector"]
|
||||
mock_metrics = mock_objects["metrics"]
|
||||
mock_queue_fn = mock_objects["queue_fn"]
|
||||
|
||||
stage_id = 0
|
||||
next_stage_id = 1
|
||||
req_id = "req_123"
|
||||
inputs = {"input_ids": [1, 2, 3]}
|
||||
sampling_params = {"temperature": 0.7}
|
||||
prompt = "test prompt"
|
||||
|
||||
# Mock connector.put return
|
||||
# Returns: (success, size, metadata)
|
||||
mock_metadata = {"handle": "xyz"}
|
||||
mock_connector.put.return_value = (True, 100, mock_metadata)
|
||||
|
||||
# Execute
|
||||
result = try_send_via_connector(
|
||||
connector=mock_connector,
|
||||
stage_id=stage_id,
|
||||
next_stage_id=next_stage_id,
|
||||
req_id=req_id,
|
||||
next_inputs=inputs,
|
||||
sampling_params=sampling_params,
|
||||
original_prompt=prompt,
|
||||
next_stage_queue_submit_fn=mock_queue_fn,
|
||||
metrics=mock_metrics,
|
||||
)
|
||||
|
||||
# Verify
|
||||
assert result is True
|
||||
|
||||
# 1. Verify connector.put called correctly
|
||||
mock_connector.put.assert_called_once()
|
||||
args, _ = mock_connector.put.call_args
|
||||
assert args[0] == "0" # from_stage
|
||||
assert args[1] == "1" # to_stage
|
||||
assert args[2] == req_id
|
||||
# Verify payload structure in put
|
||||
payload = args[3]
|
||||
assert payload["engine_inputs"] == inputs
|
||||
assert payload["sampling_params"] == sampling_params
|
||||
|
||||
# 2. Verify queue notification submitted
|
||||
mock_queue_fn.assert_called_once()
|
||||
notify_payload = mock_queue_fn.call_args[0][0]
|
||||
assert notify_payload["request_id"] == req_id
|
||||
assert notify_payload["from_connector"] is True
|
||||
assert notify_payload["connector_metadata"] == mock_metadata
|
||||
|
||||
# 3. Verify metrics recorded
|
||||
mock_metrics.on_forward.assert_called_once()
|
||||
|
||||
|
||||
def test_send_fail(mock_objects):
|
||||
"""Test try_send_via_connector when connector fails."""
|
||||
mock_connector = mock_objects["connector"]
|
||||
mock_metrics = mock_objects["metrics"]
|
||||
mock_queue_fn = mock_objects["queue_fn"]
|
||||
|
||||
mock_connector.put.return_value = (False, 0, None)
|
||||
|
||||
result = try_send_via_connector(
|
||||
connector=mock_connector,
|
||||
stage_id=0,
|
||||
next_stage_id=1,
|
||||
req_id="req_fail",
|
||||
next_inputs={},
|
||||
sampling_params={},
|
||||
original_prompt="",
|
||||
next_stage_queue_submit_fn=mock_queue_fn,
|
||||
metrics=mock_metrics,
|
||||
)
|
||||
|
||||
assert result is False
|
||||
mock_queue_fn.assert_not_called()
|
||||
|
||||
|
||||
def test_recv_success(mock_objects):
|
||||
"""Test try_recv_via_connector success path."""
|
||||
mock_connector = mock_objects["connector"]
|
||||
|
||||
# Setup task received from queue
|
||||
task = {
|
||||
"request_id": "req_recv",
|
||||
"from_connector": True,
|
||||
"from_stage": "0",
|
||||
"connector_metadata": {"handle": "xyz"},
|
||||
}
|
||||
|
||||
# Setup connectors dict
|
||||
connectors = {("0", "1"): mock_connector}
|
||||
|
||||
# Mock connector.get return
|
||||
expected_data = {"engine_inputs": {"ids": [1]}}
|
||||
# get returns: (data_obj, size)
|
||||
mock_connector.get.return_value = (expected_data, 50)
|
||||
# serialize_obj needed for metrics calculation if size not returned directly
|
||||
mock_connector.serialize_obj.return_value = b"bytes"
|
||||
|
||||
# Execute
|
||||
# We are stage 1 receiving from stage 0
|
||||
inputs, rx_metrics = try_recv_via_connector(task, connectors, stage_id=1)
|
||||
|
||||
# Verify
|
||||
assert inputs == expected_data["engine_inputs"]
|
||||
assert rx_metrics is not None
|
||||
mock_connector.get.assert_called_once_with("0", "1", "req_recv", metadata={"handle": "xyz"})
|
||||
|
||||
|
||||
def test_recv_no_connector():
|
||||
"""Test recv fails when no connector exists for edge."""
|
||||
task = {"request_id": "req_missing", "from_connector": True, "from_stage": "0"}
|
||||
connectors = {} # Empty connectors
|
||||
|
||||
inputs, _ = try_recv_via_connector(task, connectors, stage_id=1)
|
||||
assert inputs is None
|
||||
|
||||
|
||||
def test_shm_connector_flow():
|
||||
"""
|
||||
Verify the full flow: Send -> Adapter -> Connector -> Adapter -> Recv.
|
||||
Using real SharedMemoryConnector (inline mode for simplicity).
|
||||
"""
|
||||
# 1. Setup Connector
|
||||
config = {"shm_threshold_bytes": 1024} # Large threshold to use inline
|
||||
connector = SharedMemoryConnector(config)
|
||||
connectors_map = {("0", "1"): connector}
|
||||
|
||||
# 2. Setup Data
|
||||
stage_id = 0
|
||||
next_stage_id = 1
|
||||
req_id = "flow_req"
|
||||
inputs = {"tokens": [10, 20, 30]}
|
||||
sampling_params = {"n": 1}
|
||||
|
||||
# Queue capture mechanism
|
||||
queue_capture = []
|
||||
|
||||
def mock_submit(payload):
|
||||
queue_capture.append(payload)
|
||||
|
||||
mock_metrics = MagicMock()
|
||||
|
||||
# 3. Send
|
||||
success = try_send_via_connector(
|
||||
connector=connector,
|
||||
stage_id=stage_id,
|
||||
next_stage_id=next_stage_id,
|
||||
req_id=req_id,
|
||||
next_inputs=inputs,
|
||||
sampling_params=sampling_params,
|
||||
original_prompt="prompt",
|
||||
next_stage_queue_submit_fn=mock_submit,
|
||||
metrics=mock_metrics,
|
||||
)
|
||||
assert success is True
|
||||
assert len(queue_capture) == 1
|
||||
|
||||
# 4. Recv
|
||||
# The 'task' is what would be popped from the queue
|
||||
received_task = queue_capture[0]
|
||||
|
||||
# Verify queue payload contains what we expect
|
||||
assert received_task["from_connector"] is True
|
||||
assert received_task["from_stage"] == "0"
|
||||
|
||||
# Decode
|
||||
decoded_inputs, _ = try_recv_via_connector(received_task, connectors_map, stage_id=1)
|
||||
|
||||
# 5. Verify Data Integrity
|
||||
assert decoded_inputs == inputs
|
||||
|
||||
|
||||
def test_get_connectors_for_stage():
|
||||
"""Test filtering logic for stage config."""
|
||||
# Config has edges: 0->1, 1->2
|
||||
config = OmniTransferConfig(connectors={("0", "1"): ConnectorSpec(name="C1"), ("1", "2"): ConnectorSpec(name="C2")})
|
||||
|
||||
# Get config for Stage 1
|
||||
# Stage 1 receives from 0 (input) and sends to 2 (output)
|
||||
# get_connectors_config_for_stage ONLY returns INPUT connectors for the worker to initialize
|
||||
|
||||
stage_config = get_connectors_config_for_stage(config, stage_id=1)
|
||||
|
||||
# Should contain "from_stage_0"
|
||||
assert "from_stage_0" in stage_config
|
||||
assert stage_config["from_stage_0"]["spec"]["name"] == "C1"
|
||||
|
||||
# Should NOT contain "from_stage_1" or related to output
|
||||
assert "from_stage_1" not in stage_config
|
||||
|
||||
# Verify Stage 2
|
||||
stage_2_config = get_connectors_config_for_stage(config, stage_id=2)
|
||||
assert "from_stage_1" in stage_2_config
|
||||
assert stage_2_config["from_stage_1"]["spec"]["name"] == "C2"
|
||||
|
||||
|
||||
def test_recv_with_missing_metadata():
|
||||
"""Test recv when queue payload is malformed (missing metadata)."""
|
||||
# Connector expects metadata but task doesn't have it
|
||||
task = {
|
||||
"request_id": "req_bad",
|
||||
"from_connector": True,
|
||||
"from_stage": "0",
|
||||
# Missing "connector_metadata"
|
||||
}
|
||||
mock_conn = MagicMock()
|
||||
# If get is called with None metadata, connector usually handles it or adapter handles exception
|
||||
mock_conn.get.side_effect = Exception("Get failed")
|
||||
|
||||
connectors = {("0", "1"): mock_conn}
|
||||
|
||||
inputs, _ = try_recv_via_connector(task, connectors, stage_id=1)
|
||||
assert inputs is None
|
||||
@@ -1,99 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm_omni.distributed.omni_connectors.connectors.shm_connector import SharedMemoryConnector
|
||||
from vllm_omni.distributed.omni_connectors.factory import OmniConnectorFactory
|
||||
from vllm_omni.distributed.omni_connectors.utils.config import ConnectorSpec
|
||||
from vllm_omni.distributed.omni_connectors.utils.serialization import OmniSerializer
|
||||
|
||||
|
||||
def test_pickle_serialization():
|
||||
"""Test basic pickle serialization."""
|
||||
data = {"key": "value", "list": [1, 2, 3]}
|
||||
serialized = OmniSerializer.serialize(data, method="cloudpickle")
|
||||
assert isinstance(serialized, bytes)
|
||||
|
||||
deserialized = OmniSerializer.deserialize(serialized, method="cloudpickle")
|
||||
assert data == deserialized
|
||||
|
||||
|
||||
def test_create_shm_connector():
|
||||
"""Test creating SharedMemoryConnector via Factory."""
|
||||
spec = ConnectorSpec(name="SharedMemoryConnector", extra={"shm_threshold_bytes": 1024})
|
||||
connector = OmniConnectorFactory.create_connector(spec)
|
||||
assert isinstance(connector, SharedMemoryConnector)
|
||||
assert connector.threshold == 1024
|
||||
|
||||
|
||||
def test_create_unknown_connector():
|
||||
"""Test error when creating unknown connector."""
|
||||
spec = ConnectorSpec(name="UnknownConnector")
|
||||
with pytest.raises(ValueError):
|
||||
OmniConnectorFactory.create_connector(spec)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def shm_connector():
|
||||
config = {"shm_threshold_bytes": 100} # Small threshold for testing
|
||||
return SharedMemoryConnector(config)
|
||||
|
||||
|
||||
def test_put_get_inline(shm_connector):
|
||||
"""Test inline transfer for small data."""
|
||||
data = {"small": "data"}
|
||||
# Ensure data is smaller than threshold (100 bytes)
|
||||
|
||||
success, size, metadata = shm_connector.put("stage_0", "stage_1", "req_1", data)
|
||||
assert success is True
|
||||
assert "inline_bytes" in metadata
|
||||
assert "shm" not in metadata
|
||||
|
||||
# Retrieve
|
||||
retrieved_data, ret_size = shm_connector.get("stage_0", "stage_1", "req_1", metadata)
|
||||
assert data == retrieved_data
|
||||
assert size == ret_size
|
||||
|
||||
|
||||
def test_put_get_shm(shm_connector, monkeypatch):
|
||||
"""Test SHM transfer logic for large data (Mocked)."""
|
||||
# Create data larger than 100 bytes
|
||||
data = {"large": "x" * 200}
|
||||
|
||||
# Mock SHM return values
|
||||
mock_handle = {"name": "test_shm", "size": 200}
|
||||
mock_write = MagicMock(return_value=mock_handle)
|
||||
monkeypatch.setattr("vllm_omni.distributed.omni_connectors.connectors.shm_connector.shm_write_bytes", mock_write)
|
||||
|
||||
# When reading, return the serialized bytes of the data
|
||||
serialized_data = shm_connector.serialize_obj(data)
|
||||
mock_read = MagicMock(return_value=serialized_data)
|
||||
monkeypatch.setattr("vllm_omni.distributed.omni_connectors.connectors.shm_connector.shm_read_bytes", mock_read)
|
||||
|
||||
# Put
|
||||
success, size, metadata = shm_connector.put("stage_0", "stage_1", "req_2", data)
|
||||
|
||||
assert success is True
|
||||
# Should use SHM because data > threshold
|
||||
assert "shm" in metadata
|
||||
assert metadata["shm"] == mock_handle
|
||||
assert "inline_bytes" not in metadata
|
||||
|
||||
mock_write.assert_called_once()
|
||||
|
||||
# Get
|
||||
retrieved_data, ret_size = shm_connector.get("stage_0", "stage_1", "req_2", metadata)
|
||||
|
||||
assert data == retrieved_data
|
||||
mock_read.assert_called_once_with(mock_handle)
|
||||
|
||||
|
||||
def test_get_invalid_metadata(shm_connector):
|
||||
"""Test get with invalid metadata."""
|
||||
result = shm_connector.get("stage_0", "stage_1", "req_3", {})
|
||||
assert result is None
|
||||
|
||||
result = shm_connector.get("stage_0", "stage_1", "req_3", {"unknown": "format"})
|
||||
assert result is None
|
||||
@@ -1,64 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
# Use the new import path for initialization utilities
|
||||
from vllm_omni.distributed.omni_connectors.utils.initialization import load_omni_transfer_config
|
||||
|
||||
|
||||
def get_config_files():
|
||||
"""Helper to find config files."""
|
||||
# Go up two levels from 'tests/distributed/omni_connectors' (approx) to 'vllm-omni' root
|
||||
# Adjust based on file location: vllm-omni/tests/distributed/omni_connectors/test_omni_connector_configs.py
|
||||
# This file is 4 levels deep from root if we count from tests?
|
||||
# vllm-omni/tests/distributed/omni_connectors -> parent -> distributed -> parent -> tests -> parent -> vllm-omni
|
||||
# Let's use resolve to be safe.
|
||||
|
||||
# Path(__file__) = .../vllm-omni/tests/distributed/omni_connectors/test_omni_connector_configs.py
|
||||
# .parent = omni_connectors
|
||||
# .parent = distributed
|
||||
# .parent = tests
|
||||
# .parent = vllm-omni
|
||||
|
||||
base_dir = Path(__file__).resolve().parent.parent.parent.parent
|
||||
config_dir = base_dir / "vllm_omni" / "model_executor" / "stage_configs"
|
||||
|
||||
if not config_dir.exists():
|
||||
return []
|
||||
|
||||
return list(config_dir.glob("qwen*.yaml"))
|
||||
|
||||
|
||||
# Collect files at module level for parametrization
|
||||
config_files = get_config_files()
|
||||
|
||||
|
||||
@pytest.mark.skipif(len(config_files) == 0, reason="No config files found or directory missing")
|
||||
@pytest.mark.parametrize("yaml_file", config_files, ids=lambda p: p.name)
|
||||
def test_load_qwen_yaml_configs(yaml_file):
|
||||
"""
|
||||
Scan and test loading of all qwen*.yaml config files.
|
||||
This ensures that existing stage configs are compatible with the OmniConnector system.
|
||||
"""
|
||||
print(f"Testing config load: {yaml_file.name}")
|
||||
try:
|
||||
# Attempt to load the config
|
||||
# default_shm_threshold doesn't matter much for loading correctness, using default
|
||||
config = load_omni_transfer_config(yaml_file)
|
||||
|
||||
assert config is not None, "Config should not be None"
|
||||
|
||||
# Basic validation
|
||||
# Note: Some configs might not have 'runtime' or 'connectors' section if they rely on auto-shm
|
||||
# but the load function should succeed regardless.
|
||||
|
||||
# If the config defines stages, we expect connectors to be populated (either explicit or auto SHM)
|
||||
# We can't strictly assert len(config.connectors) > 0 because a single stage pipeline might have 0 edges.
|
||||
|
||||
print(f" -> Successfully loaded. Connectors: {len(config.connectors)}")
|
||||
|
||||
except Exception as e:
|
||||
pytest.fail(f"Failed to load config {yaml_file.name}: {e}")
|
||||
Whitespace-only changes.
@@ -1,341 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Pytest configuration and fixtures for vllm-omni tests.
|
||||
"""
|
||||
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
from vllm.distributed.parallel_state import cleanup_dist_env_and_memory
|
||||
from vllm.sampling_params import SamplingParams
|
||||
|
||||
from vllm_omni.entrypoints.omni import Omni
|
||||
|
||||
PromptAudioInput = list[tuple[Any, int]] | tuple[Any, int] | None
|
||||
PromptImageInput = list[Any] | Any | None
|
||||
PromptVideoInput = list[Any] | Any | None
|
||||
|
||||
|
||||
class OmniRunner:
|
||||
"""
|
||||
Test runner for Omni models.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
model_name: str,
|
||||
seed: int = 42,
|
||||
init_sleep_seconds: int = 20,
|
||||
batch_timeout: int = 10,
|
||||
init_timeout: int = 300,
|
||||
shm_threshold_bytes: int = 65536,
|
||||
log_stats: bool = False,
|
||||
stage_configs_path: str | None = None,
|
||||
**kwargs,
|
||||
) -> None:
|
||||
"""
|
||||
Initialize an OmniRunner for testing.
|
||||
|
||||
Args:
|
||||
model_name: The model name or path
|
||||
seed: Random seed for reproducibility
|
||||
init_sleep_seconds: Sleep time after starting each stage
|
||||
batch_timeout: Timeout for batching in seconds
|
||||
init_timeout: Timeout for initializing stages in seconds
|
||||
shm_threshold_bytes: Threshold for using shared memory
|
||||
log_stats: Enable detailed statistics logging
|
||||
stage_configs_path: Optional path to YAML stage config file
|
||||
**kwargs: Additional arguments passed to Omni
|
||||
"""
|
||||
self.model_name = model_name
|
||||
self.seed = seed
|
||||
|
||||
self.omni = Omni(
|
||||
model=model_name,
|
||||
log_stats=log_stats,
|
||||
init_sleep_seconds=init_sleep_seconds,
|
||||
batch_timeout=batch_timeout,
|
||||
init_timeout=init_timeout,
|
||||
shm_threshold_bytes=shm_threshold_bytes,
|
||||
stage_configs_path=stage_configs_path,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
def get_default_sampling_params_list(self) -> list[SamplingParams]:
|
||||
"""
|
||||
Get a list of default sampling parameters for all stages.
|
||||
|
||||
Returns:
|
||||
List of SamplingParams with default decoding for each stage
|
||||
"""
|
||||
return [st.default_sampling_params for st in self.omni.instance.stage_list]
|
||||
|
||||
def get_omni_inputs(
|
||||
self,
|
||||
prompts: list[str] | str,
|
||||
system_prompt: str | None = None,
|
||||
audios: PromptAudioInput = None,
|
||||
images: PromptImageInput = None,
|
||||
videos: PromptVideoInput = None,
|
||||
mm_processor_kwargs: dict[str, Any] | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""
|
||||
Construct Omni input format from prompts and multimodal data.
|
||||
|
||||
Args:
|
||||
prompts: Text prompt(s) - either a single string or list of strings
|
||||
system_prompt: Optional system prompt (defaults to Qwen system prompt)
|
||||
audios: Audio input(s) - tuple of (audio_array, sample_rate) or list of tuples
|
||||
images: Image input(s) - PIL Image or list of PIL Images
|
||||
videos: Video input(s) - numpy array or list of numpy arrays
|
||||
mm_processor_kwargs: Optional processor kwargs (e.g., use_audio_in_video)
|
||||
|
||||
Returns:
|
||||
List of prompt dictionaries suitable for Omni.generate()
|
||||
"""
|
||||
if system_prompt is None:
|
||||
system_prompt = (
|
||||
"You are Qwen, a virtual human developed by the Qwen Team, Alibaba "
|
||||
"Group, capable of perceiving auditory and visual inputs, as well as "
|
||||
"generating text and speech."
|
||||
)
|
||||
|
||||
video_padding_token = "<|VIDEO|>"
|
||||
image_padding_token = "<|IMAGE|>"
|
||||
audio_padding_token = "<|AUDIO|>"
|
||||
|
||||
if self.model_name == "Qwen/Qwen3-Omni-30B-A3B-Instruct":
|
||||
video_padding_token = "<|video_pad|>"
|
||||
image_padding_token = "<|image_pad|>"
|
||||
audio_padding_token = "<|audio_pad|>"
|
||||
|
||||
if isinstance(prompts, str):
|
||||
prompts = [prompts]
|
||||
|
||||
def _normalize_mm_input(mm_input, num_prompts):
|
||||
if mm_input is None:
|
||||
return [None] * num_prompts
|
||||
if isinstance(mm_input, list):
|
||||
if len(mm_input) != num_prompts:
|
||||
raise ValueError(
|
||||
f"Multimodal input list length ({len(mm_input)}) must match prompts length ({num_prompts})"
|
||||
)
|
||||
return mm_input
|
||||
return [mm_input] * num_prompts
|
||||
|
||||
num_prompts = len(prompts)
|
||||
audios_list = _normalize_mm_input(audios, num_prompts)
|
||||
images_list = _normalize_mm_input(images, num_prompts)
|
||||
videos_list = _normalize_mm_input(videos, num_prompts)
|
||||
|
||||
omni_inputs = []
|
||||
for i, prompt_text in enumerate(prompts):
|
||||
user_content = ""
|
||||
multi_modal_data = {}
|
||||
|
||||
audio = audios_list[i]
|
||||
if audio is not None:
|
||||
if isinstance(audio, list):
|
||||
for _ in audio:
|
||||
user_content += f"<|audio_bos|>{audio_padding_token}<|audio_eos|>"
|
||||
multi_modal_data["audio"] = audio
|
||||
else:
|
||||
user_content += f"<|audio_bos|>{audio_padding_token}<|audio_eos|>"
|
||||
multi_modal_data["audio"] = audio
|
||||
|
||||
image = images_list[i]
|
||||
if image is not None:
|
||||
if isinstance(image, list):
|
||||
for _ in image:
|
||||
user_content += f"<|vision_bos|>{image_padding_token}<|vision_eos|>"
|
||||
multi_modal_data["image"] = image
|
||||
else:
|
||||
user_content += f"<|vision_bos|>{image_padding_token}<|vision_eos|>"
|
||||
multi_modal_data["image"] = image
|
||||
|
||||
video = videos_list[i]
|
||||
if video is not None:
|
||||
if isinstance(video, list):
|
||||
for _ in video:
|
||||
user_content += f"<|vision_bos|>{video_padding_token}<|vision_eos|>"
|
||||
multi_modal_data["video"] = video
|
||||
else:
|
||||
user_content += f"<|vision_bos|>{video_padding_token}<|vision_eos|>"
|
||||
multi_modal_data["video"] = video
|
||||
|
||||
user_content += prompt_text
|
||||
|
||||
full_prompt = (
|
||||
f"<|im_start|>system\n{system_prompt}<|im_end|>\n"
|
||||
f"<|im_start|>user\n{user_content}<|im_end|>\n"
|
||||
f"<|im_start|>assistant\n"
|
||||
)
|
||||
|
||||
input_dict: dict[str, Any] = {"prompt": full_prompt}
|
||||
if multi_modal_data:
|
||||
input_dict["multi_modal_data"] = multi_modal_data
|
||||
if mm_processor_kwargs:
|
||||
input_dict["mm_processor_kwargs"] = mm_processor_kwargs
|
||||
|
||||
omni_inputs.append(input_dict)
|
||||
|
||||
return omni_inputs
|
||||
|
||||
def generate(
|
||||
self,
|
||||
prompts: list[dict[str, Any]],
|
||||
sampling_params_list: list[SamplingParams] | None = None,
|
||||
) -> list[Any]:
|
||||
"""
|
||||
Generate outputs for the given prompts.
|
||||
|
||||
Args:
|
||||
prompts: List of prompt dictionaries with 'prompt' and optionally
|
||||
'multi_modal_data' keys
|
||||
sampling_params_list: List of sampling parameters for each stage.
|
||||
If None, uses default parameters.
|
||||
|
||||
Returns:
|
||||
List of OmniRequestOutput objects from stages with final_output=True
|
||||
"""
|
||||
if sampling_params_list is None:
|
||||
sampling_params_list = self.get_default_sampling_params_list()
|
||||
|
||||
return self.omni.generate(prompts, sampling_params_list)
|
||||
|
||||
def generate_multimodal(
|
||||
self,
|
||||
prompts: list[str] | str,
|
||||
sampling_params_list: list[SamplingParams] | None = None,
|
||||
system_prompt: str | None = None,
|
||||
audios: PromptAudioInput = None,
|
||||
images: PromptImageInput = None,
|
||||
videos: PromptVideoInput = None,
|
||||
mm_processor_kwargs: dict[str, Any] | None = None,
|
||||
) -> list[Any]:
|
||||
"""
|
||||
Convenience method to generate with multimodal inputs.
|
||||
|
||||
Args:
|
||||
prompts: Text prompt(s)
|
||||
sampling_params_list: List of sampling parameters for each stage
|
||||
system_prompt: Optional system prompt
|
||||
audios: Audio input(s)
|
||||
images: Image input(s)
|
||||
videos: Video input(s)
|
||||
mm_processor_kwargs: Optional processor kwargs
|
||||
|
||||
Returns:
|
||||
List of OmniRequestOutput objects from stages with final_output=True
|
||||
"""
|
||||
omni_inputs = self.get_omni_inputs(
|
||||
prompts=prompts,
|
||||
system_prompt=system_prompt,
|
||||
audios=audios,
|
||||
images=images,
|
||||
videos=videos,
|
||||
mm_processor_kwargs=mm_processor_kwargs,
|
||||
)
|
||||
return self.generate(omni_inputs, sampling_params_list)
|
||||
|
||||
def generate_audio(
|
||||
self,
|
||||
prompts: list[str] | str,
|
||||
sampling_params_list: list[SamplingParams] | None = None,
|
||||
system_prompt: str | None = None,
|
||||
audios: PromptAudioInput = None,
|
||||
mm_processor_kwargs: dict[str, Any] | None = None,
|
||||
) -> list[Any]:
|
||||
"""
|
||||
Convenience method to generate with multimodal inputs.
|
||||
Args:
|
||||
prompts: Text prompt(s)
|
||||
sampling_params_list: List of sampling parameters for each stage
|
||||
system_prompt: Optional system prompt
|
||||
audios: Audio input(s)
|
||||
mm_processor_kwargs: Optional processor kwargs
|
||||
Returns:
|
||||
List of OmniRequestOutput objects from stages with final_output=True
|
||||
"""
|
||||
omni_inputs = self.get_omni_inputs(
|
||||
prompts=prompts,
|
||||
system_prompt=system_prompt,
|
||||
audios=audios,
|
||||
mm_processor_kwargs=mm_processor_kwargs,
|
||||
)
|
||||
return self.generate(omni_inputs, sampling_params_list)
|
||||
|
||||
def generate_video(
|
||||
self,
|
||||
prompts: list[str] | str,
|
||||
sampling_params_list: list[SamplingParams] | None = None,
|
||||
system_prompt: str | None = None,
|
||||
videos: PromptVideoInput = None,
|
||||
mm_processor_kwargs: dict[str, Any] | None = None,
|
||||
) -> list[Any]:
|
||||
"""
|
||||
Convenience method to generate with multimodal inputs.
|
||||
Args:
|
||||
prompts: Text prompt(s)
|
||||
sampling_params_list: List of sampling parameters for each stage
|
||||
system_prompt: Optional system prompt
|
||||
videos: Video input(s)
|
||||
mm_processor_kwargs: Optional processor kwargs
|
||||
Returns:
|
||||
List of OmniRequestOutput objects from stages with final_output=True
|
||||
"""
|
||||
omni_inputs = self.get_omni_inputs(
|
||||
prompts=prompts,
|
||||
system_prompt=system_prompt,
|
||||
videos=videos,
|
||||
mm_processor_kwargs=mm_processor_kwargs,
|
||||
)
|
||||
return self.generate(omni_inputs, sampling_params_list)
|
||||
|
||||
def generate_image(
|
||||
self,
|
||||
prompts: list[str] | str,
|
||||
sampling_params_list: list[SamplingParams] | None = None,
|
||||
system_prompt: str | None = None,
|
||||
images: PromptImageInput = None,
|
||||
mm_processor_kwargs: dict[str, Any] | None = None,
|
||||
) -> list[Any]:
|
||||
"""
|
||||
Convenience method to generate with multimodal inputs.
|
||||
Args:
|
||||
prompts: Text prompt(s)
|
||||
sampling_params_list: List of sampling parameters for each stage
|
||||
system_prompt: Optional system prompt
|
||||
images: Image input(s)
|
||||
mm_processor_kwargs: Optional processor kwargs
|
||||
Returns:
|
||||
List of OmniRequestOutput objects from stages with final_output=True
|
||||
"""
|
||||
omni_inputs = self.get_omni_inputs(
|
||||
prompts=prompts,
|
||||
system_prompt=system_prompt,
|
||||
images=images,
|
||||
mm_processor_kwargs=mm_processor_kwargs,
|
||||
)
|
||||
return self.generate(omni_inputs, sampling_params_list)
|
||||
|
||||
def __enter__(self):
|
||||
"""Context manager entry."""
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc_val, exc_tb):
|
||||
"""Context manager exit - cleanup resources."""
|
||||
self.close()
|
||||
del self.omni
|
||||
cleanup_dist_env_and_memory()
|
||||
|
||||
def close(self):
|
||||
"""Close and cleanup the Omni instance."""
|
||||
if hasattr(self.omni.instance, "close"):
|
||||
self.omni.instance.close()
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def omni_runner():
|
||||
return OmniRunner
|
||||
@@ -1,105 +0,0 @@
|
||||
# stage config for running qwen2.5-omni with architecture of OmniLLM.
|
||||
|
||||
# The following config has been verified on 2x 24GB GPU (L4/RTX3090/RTX4090).
|
||||
# This config is optimized for CI e2e tests.
|
||||
stage_args:
|
||||
- stage_id: 0
|
||||
runtime:
|
||||
process: true # Run this stage in a separate process
|
||||
devices: "0" # Visible devices for this stage (CUDA_VISIBLE_DEVICES/torch.cuda.set_device)
|
||||
max_batch_size: 1
|
||||
engine_args:
|
||||
model_stage: thinker
|
||||
model_arch: Qwen2_5OmniForConditionalGeneration
|
||||
worker_cls: vllm_omni.worker.gpu_ar_worker.GPUARWorker
|
||||
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
|
||||
max_model_len: 896
|
||||
max_num_batched_tokens: 896
|
||||
max_num_seqs: 1
|
||||
gpu_memory_utilization: 0.8
|
||||
skip_mm_profiling: true
|
||||
enforce_eager: true # Now we only support eager mode
|
||||
trust_remote_code: true
|
||||
engine_output_type: latent
|
||||
enable_prefix_caching: false
|
||||
is_comprehension: true
|
||||
final_output: true
|
||||
final_output_type: text
|
||||
default_sampling_params:
|
||||
temperature: 0.0
|
||||
top_p: 1.0
|
||||
top_k: -1
|
||||
max_tokens: 128
|
||||
seed: 42
|
||||
detokenize: True
|
||||
repetition_penalty: 1.1
|
||||
- stage_id: 1
|
||||
runtime:
|
||||
process: true
|
||||
devices: "1"
|
||||
max_batch_size: 1
|
||||
engine_args:
|
||||
model_stage: talker
|
||||
model_arch: Qwen2_5OmniForConditionalGeneration
|
||||
worker_cls: vllm_omni.worker.gpu_ar_worker.GPUARWorker
|
||||
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
|
||||
max_model_len: 896
|
||||
max_num_batched_tokens: 896
|
||||
max_num_seqs: 1
|
||||
gpu_memory_utilization: 0.8
|
||||
skip_mm_profiling: true
|
||||
enforce_eager: true
|
||||
trust_remote_code: true
|
||||
enable_prefix_caching: false
|
||||
engine_output_type: latent
|
||||
engine_input_source: [0]
|
||||
custom_process_input_func: vllm_omni.model_executor.stage_input_processors.qwen2_5_omni.thinker2talker
|
||||
default_sampling_params:
|
||||
temperature: 0.9
|
||||
top_p: 0.8
|
||||
top_k: 40
|
||||
max_tokens: 128
|
||||
seed: 42
|
||||
detokenize: True
|
||||
repetition_penalty: 1.05
|
||||
stop_token_ids: [8294]
|
||||
- stage_id: 2
|
||||
runtime:
|
||||
process: true
|
||||
devices: "0" # Example: use a different GPU than the previous stage; use "0" if single GPU
|
||||
max_batch_size: 1
|
||||
engine_args:
|
||||
model_stage: code2wav
|
||||
model_arch: Qwen2_5OmniForConditionalGeneration
|
||||
worker_cls: vllm_omni.worker.gpu_generation_worker.GPUGenerationWorker
|
||||
scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler
|
||||
gpu_memory_utilization: 0.15
|
||||
enforce_eager: true
|
||||
trust_remote_code: true
|
||||
enable_prefix_caching: false
|
||||
engine_output_type: audio
|
||||
engine_input_source: [1]
|
||||
final_output: true
|
||||
final_output_type: audio
|
||||
default_sampling_params:
|
||||
temperature: 0.0
|
||||
top_p: 1.0
|
||||
top_k: -1
|
||||
max_tokens: 128
|
||||
seed: 42
|
||||
detokenize: True
|
||||
repetition_penalty: 1.1
|
||||
|
||||
# Top-level runtime config (concise): default windows and stage edges
|
||||
runtime:
|
||||
enabled: true
|
||||
defaults:
|
||||
window_size: -1 # Simplified: trigger downstream only after full upstream completion
|
||||
max_inflight: 1 # Simplified: process serially within each stage
|
||||
edges:
|
||||
- from: 0 # thinker → talker: trigger only after receiving full input (-1)
|
||||
to: 1
|
||||
window_size: -1
|
||||
- from: 1 # talker → code2wav: trigger only after receiving full input (-1)
|
||||
to: 2
|
||||
window_size: -1
|
||||
@@ -1,98 +0,0 @@
|
||||
# Stage config for running Qwen3-Omni-MoE with 3-stage architecture
|
||||
# Stage 0: Thinker (multimodal understanding + text generation)
|
||||
# Stage 1: Talker (text embeddings → 16-layer RVQ codec codes)
|
||||
# Stage 2: Code2Wav (8-layer RVQ codes → audio waveform)
|
||||
|
||||
# The following config has been verified on 2x H100-80G GPUs.
|
||||
stage_args:
|
||||
- stage_id: 0
|
||||
runtime:
|
||||
devices: "0,1"
|
||||
max_batch_size: 1
|
||||
engine_args:
|
||||
model_stage: thinker
|
||||
model_arch: Qwen3OmniMoeForConditionalGeneration
|
||||
worker_cls: vllm_omni.worker.gpu_ar_worker.GPUARWorker
|
||||
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
|
||||
gpu_memory_utilization: 0.6
|
||||
enforce_eager: true
|
||||
trust_remote_code: true
|
||||
engine_output_type: latent # Output hidden states for talker
|
||||
distributed_executor_backend: "mp"
|
||||
enable_prefix_caching: false
|
||||
hf_config_name: thinker_config
|
||||
tensor_parallel_size: 2
|
||||
load_format: dummy
|
||||
final_output: true
|
||||
final_output_type: text
|
||||
is_comprehension: true
|
||||
default_sampling_params:
|
||||
temperature: 0.4
|
||||
top_p: 0.9
|
||||
top_k: 1
|
||||
max_tokens: 100
|
||||
seed: 42
|
||||
detokenize: True
|
||||
repetition_penalty: 1.05
|
||||
|
||||
- stage_id: 1
|
||||
runtime:
|
||||
devices: "1"
|
||||
max_batch_size: 1
|
||||
engine_args:
|
||||
model_stage: talker
|
||||
model_arch: Qwen3OmniMoeForConditionalGeneration
|
||||
worker_cls: vllm_omni.worker.gpu_ar_worker.GPUARWorker
|
||||
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
|
||||
gpu_memory_utilization: 0.5
|
||||
enforce_eager: true
|
||||
trust_remote_code: true
|
||||
engine_output_type: latent # Output codec codes for code2wav
|
||||
# tensor_parallel_size: 2
|
||||
enable_prefix_caching: false
|
||||
distributed_executor_backend: "mp"
|
||||
hf_config_name: talker_config
|
||||
load_format: dummy
|
||||
engine_input_source: [0]
|
||||
custom_process_input_func: vllm_omni.model_executor.stage_input_processors.qwen3_omni.thinker2talker
|
||||
# final_output: true
|
||||
# final_output_type: text
|
||||
default_sampling_params:
|
||||
temperature: 0.9
|
||||
top_k: 50
|
||||
max_tokens: 100
|
||||
seed: 42
|
||||
detokenize: False
|
||||
repetition_penalty: 1.05
|
||||
stop_token_ids: [2150]
|
||||
|
||||
- stage_id: 2
|
||||
runtime:
|
||||
devices: "0"
|
||||
max_batch_size: 1
|
||||
engine_args:
|
||||
model_stage: code2wav
|
||||
model_arch: Qwen3OmniMoeForConditionalGeneration
|
||||
worker_cls: vllm_omni.worker.gpu_generation_worker.GPUGenerationWorker
|
||||
scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler
|
||||
enforce_eager: true
|
||||
trust_remote_code: true
|
||||
enable_prefix_caching: false
|
||||
engine_output_type: audio # Final output: audio waveform
|
||||
gpu_memory_utilization: 0.1
|
||||
distributed_executor_backend: "mp"
|
||||
max_num_batched_tokens: 1000000
|
||||
hf_config_name: thinker_config
|
||||
load_format: dummy
|
||||
engine_input_source: [1]
|
||||
custom_process_input_func: vllm_omni.model_executor.stage_input_processors.qwen3_omni.talker2code2wav
|
||||
final_output: true
|
||||
final_output_type: audio
|
||||
default_sampling_params:
|
||||
temperature: 0.0
|
||||
top_p: 1.0
|
||||
top_k: -1
|
||||
max_tokens: 200
|
||||
seed: 42
|
||||
detokenize: True
|
||||
repetition_penalty: 1.1
|
||||
@@ -1,77 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
E2E tests for Qwen2.5-Omni model with mixed modality inputs and audio output.
|
||||
"""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from vllm.assets.audio import AudioAsset
|
||||
from vllm.assets.image import ImageAsset
|
||||
from vllm.assets.video import VideoAsset
|
||||
from vllm.multimodal.image import convert_image_mode
|
||||
|
||||
from .conftest import OmniRunner
|
||||
from .utils import create_new_process_for_each_test
|
||||
|
||||
models = ["Qwen/Qwen2.5-Omni-3B"]
|
||||
|
||||
# CI stage config optimized for 24GB GPU (L4/RTX3090)
|
||||
stage_configs = [str(Path(__file__).parent / "stage_configs" / "qwen2_5_omni_ci.yaml")]
|
||||
|
||||
# Create parameter combinations for model and stage config
|
||||
test_params = [(model, stage_config) for model in models for stage_config in stage_configs]
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.parametrize("test_config", test_params)
|
||||
@create_new_process_for_each_test()
|
||||
def test_mixed_modalities_to_audio(omni_runner: type[OmniRunner], test_config: tuple[str, str]) -> None:
|
||||
"""Test processing audio, image, and video together, generating audio output."""
|
||||
model, stage_config_path = test_config
|
||||
with omni_runner(model, seed=42, stage_configs_path=stage_config_path) as runner:
|
||||
# Prepare multimodal inputs
|
||||
question = "What is recited in the audio? What is in this image? Describe the video briefly."
|
||||
audio = AudioAsset("mary_had_lamb").audio_and_sample_rate
|
||||
audio = (audio[0][: 16000 * 5], audio[1]) # Trim to first 5 seconds
|
||||
image = convert_image_mode(ImageAsset("cherry_blossom").pil_image.resize((128, 128)), "RGB")
|
||||
video = VideoAsset(name="baby_reading", num_frames=4).np_ndarrays
|
||||
|
||||
outputs = runner.generate_multimodal(
|
||||
prompts=question,
|
||||
audios=audio,
|
||||
images=image,
|
||||
videos=video,
|
||||
)
|
||||
|
||||
# Verify we got outputs from multiple stages
|
||||
assert len(outputs) > 0
|
||||
|
||||
# Find and verify text output (thinker stage)
|
||||
text_output = None
|
||||
for stage_output in outputs:
|
||||
if stage_output.final_output_type == "text":
|
||||
text_output = stage_output
|
||||
break
|
||||
|
||||
assert text_output is not None
|
||||
assert len(text_output.request_output) > 0
|
||||
text_content = text_output.request_output[0].outputs[0].text
|
||||
assert text_content is not None
|
||||
assert len(text_content.strip()) > 0
|
||||
|
||||
# Find and verify audio output (code2wav stage)
|
||||
audio_output = None
|
||||
for stage_output in outputs:
|
||||
if stage_output.final_output_type == "audio":
|
||||
audio_output = stage_output
|
||||
break
|
||||
|
||||
assert audio_output is not None
|
||||
assert len(audio_output.request_output) > 0
|
||||
|
||||
# Verify audio tensor exists and has content
|
||||
audio_tensor = audio_output.request_output[0].multimodal_output["audio"]
|
||||
assert audio_tensor is not None
|
||||
assert audio_tensor.numel() > 0
|
||||
@@ -1,69 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
E2E offline tests for Omni model with video input and audio output.
|
||||
"""
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from vllm.assets.video import VideoAsset
|
||||
|
||||
from .conftest import OmniRunner
|
||||
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
|
||||
models = ["Qwen/Qwen3-Omni-30B-A3B-Instruct"]
|
||||
|
||||
# CI stage config for 2xH100-80G GPUs
|
||||
stage_configs = [str(Path(__file__).parent / "stage_configs" / "qwen3_omni_ci.yaml")]
|
||||
|
||||
# Create parameter combinations for model and stage config
|
||||
test_params = [(model, stage_config) for model in models for stage_config in stage_configs]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("test_config", test_params)
|
||||
def test_video_to_audio(omni_runner: type[OmniRunner], test_config) -> None:
|
||||
"""Test processing video, generating audio output."""
|
||||
model, stage_config_path = test_config
|
||||
with omni_runner(model, seed=42, stage_configs_path=stage_config_path) as runner:
|
||||
# Prepare inputs
|
||||
question = "Describe the video briefly."
|
||||
video = VideoAsset(name="baby_reading", num_frames=4).np_ndarrays
|
||||
|
||||
outputs = runner.generate_multimodal(
|
||||
prompts=question,
|
||||
videos=video,
|
||||
)
|
||||
|
||||
# Verify we got outputs from multiple stages
|
||||
assert len(outputs) > 0
|
||||
|
||||
# Find and verify text output (thinker stage)
|
||||
text_output = None
|
||||
for stage_output in outputs:
|
||||
if stage_output.final_output_type == "text":
|
||||
text_output = stage_output
|
||||
break
|
||||
|
||||
assert text_output is not None
|
||||
assert len(text_output.request_output) > 0
|
||||
text_content = text_output.request_output[0].outputs[0].text
|
||||
assert text_content is not None
|
||||
assert len(text_content.strip()) > 0
|
||||
|
||||
# Find and verify audio output (code2wav stage)
|
||||
audio_output = None
|
||||
for stage_output in outputs:
|
||||
if stage_output.final_output_type == "audio":
|
||||
audio_output = stage_output
|
||||
break
|
||||
|
||||
assert audio_output is not None
|
||||
assert len(audio_output.request_output) > 0
|
||||
|
||||
# Verify audio tensor exists and has content
|
||||
audio_tensor = audio_output.request_output[0].multimodal_output["audio"]
|
||||
assert audio_tensor is not None
|
||||
assert audio_tensor.numel() > 0
|
||||
@@ -1,39 +0,0 @@
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
# ruff: noqa: E402
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
if str(REPO_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(REPO_ROOT))
|
||||
|
||||
from vllm_omni import Omni
|
||||
|
||||
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "1"
|
||||
|
||||
models = ["Tongyi-MAI/Z-Image-Turbo", "riverclouds/qwen_image_random"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_name", models)
|
||||
def test_diffusion_model(model_name: str):
|
||||
m = Omni(model=model_name)
|
||||
# high resolution may cause OOM on L4
|
||||
height = 256
|
||||
width = 256
|
||||
images = m.generate(
|
||||
"a photo of a cat sitting on a laptop keyboard",
|
||||
height=height,
|
||||
width=width,
|
||||
num_inference_steps=2,
|
||||
guidance_scale=0.0,
|
||||
generator=torch.Generator("cuda").manual_seed(42),
|
||||
num_outputs_per_prompt=2,
|
||||
)
|
||||
assert len(images) == 2
|
||||
# check image size
|
||||
assert images[0].width == width
|
||||
assert images[0].height == height
|
||||
images[0].save("image_output.png")
|
||||
@@ -1,48 +0,0 @@
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
# ruff: noqa: E402
|
||||
REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
if str(REPO_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(REPO_ROOT))
|
||||
|
||||
from vllm_omni import Omni
|
||||
|
||||
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "1"
|
||||
|
||||
models = ["Wan-AI/Wan2.2-T2V-A14B-Diffusers"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_name", models)
|
||||
def test_video_diffusion_model(model_name: str):
|
||||
m = Omni(
|
||||
model=model_name,
|
||||
boundary_ratio=0.875,
|
||||
flow_shift=5.0,
|
||||
)
|
||||
# Use minimal settings for testing
|
||||
# num_frames must satisfy: num_frames % vae_scale_factor_temporal == 1
|
||||
# For Wan2.2, vae_scale_factor_temporal=4, so valid values are 5, 9, 13, 17, ...
|
||||
height = 480
|
||||
width = 640
|
||||
num_frames = 5
|
||||
frames = m.generate(
|
||||
"A cat sitting on a table",
|
||||
height=height,
|
||||
width=width,
|
||||
num_frames=num_frames,
|
||||
num_inference_steps=2,
|
||||
guidance_scale=1.0,
|
||||
generator=torch.Generator("cuda").manual_seed(42),
|
||||
)
|
||||
|
||||
assert frames is not None
|
||||
assert hasattr(frames, "shape")
|
||||
# frames shape: (batch, num_frames, height, width, channels)
|
||||
assert frames.shape[1] == num_frames
|
||||
assert frames.shape[2] == height
|
||||
assert frames.shape[3] == width
|
||||
@@ -1,206 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import contextlib
|
||||
import functools
|
||||
import os
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from collections.abc import Callable
|
||||
from contextlib import ExitStack, suppress
|
||||
from pathlib import Path
|
||||
from typing import Any, Literal
|
||||
|
||||
import cloudpickle
|
||||
from typing_extensions import ParamSpec
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
VLLM_PATH = Path(__file__).parent.parent.parent
|
||||
"""Path to root of the vLLM repository."""
|
||||
|
||||
|
||||
_P = ParamSpec("_P")
|
||||
|
||||
|
||||
def fork_new_process_for_each_test(func: Callable[_P, None]) -> Callable[_P, None]:
|
||||
"""Decorator to fork a new process for each test function.
|
||||
See https://github.com/vllm-project/vllm/issues/7053 for more details.
|
||||
"""
|
||||
|
||||
@functools.wraps(func)
|
||||
def wrapper(*args: _P.args, **kwargs: _P.kwargs) -> None:
|
||||
# Make the process the leader of its own process group
|
||||
# to avoid sending SIGTERM to the parent process
|
||||
os.setpgrp()
|
||||
from _pytest.outcomes import Skipped
|
||||
|
||||
# Create a unique temporary file to store exception info from child
|
||||
# process. Use test function name and process ID to avoid collisions.
|
||||
with (
|
||||
tempfile.NamedTemporaryFile(
|
||||
delete=False,
|
||||
mode="w+b",
|
||||
prefix=f"vllm_test_{func.__name__}_{os.getpid()}_",
|
||||
suffix=".exc",
|
||||
) as exc_file,
|
||||
ExitStack() as delete_after,
|
||||
):
|
||||
exc_file_path = exc_file.name
|
||||
delete_after.callback(os.remove, exc_file_path)
|
||||
|
||||
pid = os.fork()
|
||||
print(f"Fork a new process to run a test {pid}")
|
||||
if pid == 0:
|
||||
# Parent process responsible for deleting, don't delete
|
||||
# in child.
|
||||
delete_after.pop_all()
|
||||
try:
|
||||
func(*args, **kwargs)
|
||||
except Skipped as e:
|
||||
# convert Skipped to exit code 0
|
||||
print(str(e))
|
||||
os._exit(0)
|
||||
except Exception as e:
|
||||
import traceback
|
||||
|
||||
tb_string = traceback.format_exc()
|
||||
|
||||
# Try to serialize the exception object first
|
||||
exc_to_serialize: dict[str, Any]
|
||||
try:
|
||||
# First, try to pickle the actual exception with
|
||||
# its traceback.
|
||||
exc_to_serialize = {"pickled_exception": e}
|
||||
# Test if it can be pickled
|
||||
cloudpickle.dumps(exc_to_serialize)
|
||||
except (Exception, KeyboardInterrupt):
|
||||
# Fall back to string-based approach.
|
||||
exc_to_serialize = {
|
||||
"exception_type": type(e).__name__,
|
||||
"exception_msg": str(e),
|
||||
"traceback": tb_string,
|
||||
}
|
||||
try:
|
||||
with open(exc_file_path, "wb") as f:
|
||||
cloudpickle.dump(exc_to_serialize, f)
|
||||
except Exception:
|
||||
# Fallback: just print the traceback.
|
||||
print(tb_string)
|
||||
os._exit(1)
|
||||
else:
|
||||
os._exit(0)
|
||||
else:
|
||||
pgid = os.getpgid(pid)
|
||||
_pid, _exitcode = os.waitpid(pid, 0)
|
||||
# ignore SIGTERM signal itself
|
||||
old_signal_handler = signal.signal(signal.SIGTERM, signal.SIG_IGN)
|
||||
# kill all child processes
|
||||
os.killpg(pgid, signal.SIGTERM)
|
||||
# restore the signal handler
|
||||
signal.signal(signal.SIGTERM, old_signal_handler)
|
||||
if _exitcode != 0:
|
||||
# Try to read the exception from the child process
|
||||
exc_info = {}
|
||||
if os.path.exists(exc_file_path):
|
||||
with (
|
||||
contextlib.suppress(Exception),
|
||||
open(exc_file_path, "rb") as f,
|
||||
):
|
||||
exc_info = cloudpickle.load(f)
|
||||
|
||||
if (original_exception := exc_info.get("pickled_exception")) is not None:
|
||||
# Re-raise the actual exception object if it was
|
||||
# successfully pickled.
|
||||
assert isinstance(original_exception, Exception)
|
||||
raise original_exception
|
||||
|
||||
if (original_tb := exc_info.get("traceback")) is not None:
|
||||
# Use string-based traceback for fallback case
|
||||
raise AssertionError(
|
||||
f"Test {func.__name__} failed when called with"
|
||||
f" args {args} and kwargs {kwargs}"
|
||||
f" (exit code: {_exitcode}):\n{original_tb}"
|
||||
) from None
|
||||
|
||||
# Fallback to the original generic error
|
||||
raise AssertionError(
|
||||
f"function {func.__name__} failed when called with"
|
||||
f" args {args} and kwargs {kwargs}"
|
||||
f" (exit code: {_exitcode})"
|
||||
) from None
|
||||
|
||||
return wrapper
|
||||
|
||||
|
||||
def spawn_new_process_for_each_test(f: Callable[_P, None]) -> Callable[_P, None]:
|
||||
"""Decorator to spawn a new process for each test function."""
|
||||
|
||||
@functools.wraps(f)
|
||||
def wrapper(*args: _P.args, **kwargs: _P.kwargs) -> None:
|
||||
# Check if we're already in a subprocess
|
||||
if os.environ.get("RUNNING_IN_SUBPROCESS") == "1":
|
||||
# If we are, just run the function directly
|
||||
return f(*args, **kwargs)
|
||||
|
||||
import torch.multiprocessing as mp
|
||||
|
||||
with suppress(RuntimeError):
|
||||
mp.set_start_method("spawn")
|
||||
|
||||
# Get the module
|
||||
module_name = f.__module__
|
||||
|
||||
# Create a process with environment variable set
|
||||
env = os.environ.copy()
|
||||
env["RUNNING_IN_SUBPROCESS"] = "1"
|
||||
|
||||
with tempfile.TemporaryDirectory() as tempdir:
|
||||
output_filepath = os.path.join(tempdir, "new_process.tmp")
|
||||
|
||||
# `cloudpickle` allows pickling complex functions directly
|
||||
input_bytes = cloudpickle.dumps((f, output_filepath))
|
||||
|
||||
repo_root = str(VLLM_PATH.resolve())
|
||||
|
||||
env = dict(env or os.environ)
|
||||
env["PYTHONPATH"] = repo_root + os.pathsep + env.get("PYTHONPATH", "")
|
||||
|
||||
cmd = [sys.executable, "-m", f"{module_name}"]
|
||||
|
||||
returned = subprocess.run(cmd, input=input_bytes, capture_output=True, env=env)
|
||||
|
||||
# check if the subprocess is successful
|
||||
try:
|
||||
returned.check_returncode()
|
||||
except Exception as e:
|
||||
# wrap raised exception to provide more information
|
||||
raise RuntimeError(f"Error raised in subprocess:\n{returned.stderr.decode()}") from e
|
||||
|
||||
return wrapper
|
||||
|
||||
|
||||
def create_new_process_for_each_test(
|
||||
method: Literal["spawn", "fork"] | None = None,
|
||||
) -> Callable[[Callable[_P, None]], Callable[_P, None]]:
|
||||
"""Creates a decorator that runs each test function in a new process.
|
||||
|
||||
Args:
|
||||
method: The process creation method. Can be either "spawn" or "fork".
|
||||
If not specified, it defaults to "spawn" on ROCm and XPU
|
||||
platforms and "fork" otherwise.
|
||||
|
||||
Returns:
|
||||
A decorator to run test functions in separate processes.
|
||||
"""
|
||||
if method is None:
|
||||
use_spawn = current_platform.is_rocm() or current_platform.is_xpu()
|
||||
method = "spawn" if use_spawn else "fork"
|
||||
|
||||
assert method in ["spawn", "fork"], "Method must be either 'spawn' or 'fork'"
|
||||
|
||||
if method == "fork":
|
||||
return fork_new_process_for_each_test
|
||||
|
||||
return spawn_new_process_for_each_test
|
||||
Whitespace-only changes.
@@ -1,98 +0,0 @@
|
||||
# Stage config for running Qwen3-Omni-MoE with 3-stage architecture
|
||||
# Stage 0: Thinker (multimodal understanding + text generation)
|
||||
# Stage 1: Talker (text embeddings → 16-layer RVQ codec codes)
|
||||
# Stage 2: Code2Wav (8-layer RVQ codes → audio waveform)
|
||||
|
||||
# The following config has been verified on 2x H100-80G GPUs.
|
||||
stage_args:
|
||||
- stage_id: 0
|
||||
runtime:
|
||||
devices: "0,1"
|
||||
max_batch_size: 1
|
||||
engine_args:
|
||||
model_stage: thinker
|
||||
model_arch: Qwen3OmniMoeForConditionalGeneration
|
||||
worker_cls: vllm_omni.worker.gpu_ar_worker.GPUARWorker
|
||||
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
|
||||
gpu_memory_utilization: 0.6
|
||||
enforce_eager: true
|
||||
trust_remote_code: true
|
||||
engine_output_type: latent # Output hidden states for talker
|
||||
distributed_executor_backend: "mp"
|
||||
enable_prefix_caching: false
|
||||
hf_config_name: thinker_config
|
||||
tensor_parallel_size: 2
|
||||
load_format: dummy
|
||||
final_output: true
|
||||
final_output_type: text
|
||||
is_comprehension: true
|
||||
default_sampling_params:
|
||||
temperature: 0.4
|
||||
top_p: 0.9
|
||||
top_k: 1
|
||||
max_tokens: 100
|
||||
seed: 42
|
||||
detokenize: True
|
||||
repetition_penalty: 1.05
|
||||
|
||||
- stage_id: 1
|
||||
runtime:
|
||||
devices: "1"
|
||||
max_batch_size: 1
|
||||
engine_args:
|
||||
model_stage: talker
|
||||
model_arch: Qwen3OmniMoeForConditionalGeneration
|
||||
worker_cls: vllm_omni.worker.gpu_ar_worker.GPUARWorker
|
||||
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
|
||||
gpu_memory_utilization: 0.3
|
||||
enforce_eager: true
|
||||
trust_remote_code: true
|
||||
engine_output_type: latent # Output codec codes for code2wav
|
||||
# tensor_parallel_size: 2
|
||||
enable_prefix_caching: false
|
||||
distributed_executor_backend: "mp"
|
||||
hf_config_name: talker_config
|
||||
load_format: dummy
|
||||
engine_input_source: [0]
|
||||
custom_process_input_func: vllm_omni.model_executor.stage_input_processors.qwen3_omni.thinker2talker
|
||||
# final_output: true
|
||||
# final_output_type: text
|
||||
default_sampling_params:
|
||||
temperature: 0.9
|
||||
top_k: 50
|
||||
max_tokens: 100
|
||||
seed: 42
|
||||
detokenize: False
|
||||
repetition_penalty: 1.05
|
||||
stop_token_ids: [2150]
|
||||
|
||||
- stage_id: 2
|
||||
runtime:
|
||||
devices: "0"
|
||||
max_batch_size: 1
|
||||
engine_args:
|
||||
model_stage: code2wav
|
||||
model_arch: Qwen3OmniMoeForConditionalGeneration
|
||||
worker_cls: vllm_omni.worker.gpu_generation_worker.GPUGenerationWorker
|
||||
scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler
|
||||
enforce_eager: true
|
||||
trust_remote_code: true
|
||||
enable_prefix_caching: false
|
||||
engine_output_type: audio # Final output: audio waveform
|
||||
gpu_memory_utilization: 0.1
|
||||
distributed_executor_backend: "mp"
|
||||
max_num_batched_tokens: 1000000
|
||||
hf_config_name: thinker_config
|
||||
load_format: dummy
|
||||
engine_input_source: [1]
|
||||
custom_process_input_func: vllm_omni.model_executor.stage_input_processors.qwen3_omni.talker2code2wav
|
||||
final_output: true
|
||||
final_output_type: audio
|
||||
default_sampling_params:
|
||||
temperature: 0.0
|
||||
top_p: 1.0
|
||||
top_k: -1
|
||||
max_tokens: 200
|
||||
seed: 42
|
||||
detokenize: True
|
||||
repetition_penalty: 1.1
|
||||
@@ -1,206 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
E2E Online tests for Qwen3-Omni model with video input and audio output.
|
||||
"""
|
||||
|
||||
import os
|
||||
import socket
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import openai
|
||||
import pytest
|
||||
from vllm.assets.video import VideoAsset
|
||||
from vllm.utils import get_open_port
|
||||
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
|
||||
models = ["Qwen/Qwen3-Omni-30B-A3B-Instruct"]
|
||||
|
||||
# CI stage config for 2*H100-80G GPUs
|
||||
stage_configs = [str(Path(__file__).parent / "stage_configs" / "qwen3_omni_ci.yaml")]
|
||||
|
||||
# Create parameter combinations for model and stage config
|
||||
test_params = [(model, stage_config) for model in models for stage_config in stage_configs]
|
||||
|
||||
|
||||
class OmniServer:
|
||||
"""Omniserver for vLLM-Omni tests."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
model: str,
|
||||
serve_args: list[str],
|
||||
*,
|
||||
env_dict: dict[str, str] | None = None,
|
||||
) -> None:
|
||||
self.model = model
|
||||
self.serve_args = serve_args
|
||||
self.env_dict = env_dict
|
||||
self.proc: subprocess.Popen | None = None
|
||||
self.host = "127.0.0.1"
|
||||
self.port = get_open_port()
|
||||
|
||||
def _start_server(self) -> None:
|
||||
"""Start the vLLM-Omni server subprocess."""
|
||||
env = os.environ.copy()
|
||||
env["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
if self.env_dict is not None:
|
||||
env.update(self.env_dict)
|
||||
|
||||
cmd = [
|
||||
sys.executable,
|
||||
"-m",
|
||||
"vllm_omni.entrypoints.cli.main",
|
||||
"serve",
|
||||
self.model,
|
||||
"--omni",
|
||||
"--host",
|
||||
self.host,
|
||||
"--port",
|
||||
str(self.port),
|
||||
] + self.serve_args
|
||||
|
||||
print(f"Launching OmniServer with: {' '.join(cmd)}")
|
||||
self.proc = subprocess.Popen(
|
||||
cmd,
|
||||
env=env,
|
||||
cwd=os.path.dirname(os.path.dirname(os.path.abspath(__file__))), # Set working directory to vllm-omni root
|
||||
)
|
||||
|
||||
# Wait for server to be ready
|
||||
max_wait = 600 # 10 minutes
|
||||
start_time = time.time()
|
||||
while time.time() - start_time < max_wait:
|
||||
try:
|
||||
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
|
||||
sock.settimeout(1)
|
||||
result = sock.connect_ex((self.host, self.port))
|
||||
if result == 0:
|
||||
print(f"Server ready on {self.host}:{self.port}")
|
||||
return
|
||||
except Exception:
|
||||
pass
|
||||
time.sleep(2)
|
||||
|
||||
raise RuntimeError(f"Server failed to start within {max_wait} seconds")
|
||||
|
||||
def __enter__(self):
|
||||
self._start_server()
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc_val, exc_tb):
|
||||
if self.proc:
|
||||
self.proc.terminate()
|
||||
try:
|
||||
self.proc.wait(timeout=30)
|
||||
except subprocess.TimeoutExpired:
|
||||
self.proc.kill()
|
||||
self.proc.wait()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def omni_server(request):
|
||||
"""Start vLLM-Omni server as a subprocess with actual model weights.
|
||||
Uses module scope so the server starts only once for all tests.
|
||||
Multi-stage initialization can take 10-20+ minutes.
|
||||
"""
|
||||
model, stage_config_path = request.param
|
||||
with OmniServer(model, ["--stage-configs-path", stage_config_path]) as server:
|
||||
yield server
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def client(omni_server):
|
||||
"""OpenAI client for the running vLLM-Omni server."""
|
||||
return openai.OpenAI(
|
||||
base_url=f"http://{omni_server.host}:{omni_server.port}/v1",
|
||||
api_key="EMPTY",
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(scope="session")
|
||||
def base64_encoded_video() -> str:
|
||||
"""Base64 encoded video for testing."""
|
||||
import base64
|
||||
|
||||
video = VideoAsset(name="baby_reading", num_frames=4)
|
||||
with open(video.video_path, "rb") as f:
|
||||
content = f.read()
|
||||
return base64.b64encode(content).decode("utf-8")
|
||||
|
||||
|
||||
def get_system_prompt():
|
||||
return {
|
||||
"role": "system",
|
||||
"content": [
|
||||
{
|
||||
"type": "text",
|
||||
"text": (
|
||||
"You are Qwen, a virtual human developed by the Qwen Team, "
|
||||
"Alibaba Group, capable of perceiving auditory and visual inputs, "
|
||||
"as well as generating text and speech."
|
||||
),
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def dummy_messages_from_video_data(
|
||||
video_data_url: str,
|
||||
content_text: str = "Describe the video briefly.",
|
||||
):
|
||||
"""Create messages with video data URL for OpenAI API."""
|
||||
return [
|
||||
get_system_prompt(),
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "video_url", "video_url": {"url": video_data_url}},
|
||||
{"type": "text", "text": content_text},
|
||||
],
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("omni_server", test_params, indirect=True)
|
||||
def test_video_to_audio(
|
||||
client: openai.OpenAI,
|
||||
omni_server,
|
||||
base64_encoded_video: str,
|
||||
) -> None:
|
||||
"""Test processing video, generating audio output via OpenAI API."""
|
||||
# Create data URL for the base64 encoded video
|
||||
video_data_url = f"data:video/mp4;base64,{base64_encoded_video}"
|
||||
|
||||
messages = dummy_messages_from_video_data(video_data_url)
|
||||
|
||||
# Test single completion
|
||||
chat_completion = client.chat.completions.create(
|
||||
model=omni_server.model,
|
||||
messages=messages,
|
||||
)
|
||||
|
||||
assert len(chat_completion.choices) == 2 # 1 for text output, 1 for audio output
|
||||
|
||||
# Verify text output
|
||||
text_choice = chat_completion.choices[0]
|
||||
assert text_choice.finish_reason == "length"
|
||||
|
||||
# Verify we got a response
|
||||
text_message = text_choice.message
|
||||
assert text_message.content is not None and len(text_message.content) >= 10
|
||||
assert text_message.role == "assistant"
|
||||
|
||||
# Verify audio output
|
||||
audio_choice = chat_completion.choices[1]
|
||||
assert audio_choice.finish_reason == "stop"
|
||||
audio_message = audio_choice.message
|
||||
|
||||
# Check if audio was generated
|
||||
if hasattr(audio_message, "audio") and audio_message.audio:
|
||||
assert audio_message.audio.data is not None
|
||||
assert len(audio_message.audio.data) > 0
|
||||
@@ -1,58 +0,0 @@
|
||||
import os
|
||||
import sys
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm_omni.entrypoints.stage_utils import set_stage_devices
|
||||
|
||||
|
||||
def _make_dummy_torch(call_log):
|
||||
class _Props:
|
||||
def __init__(self, total):
|
||||
self.total_memory = total
|
||||
|
||||
class _Cuda:
|
||||
@staticmethod
|
||||
def is_available():
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def set_device(idx):
|
||||
call_log.append(idx)
|
||||
|
||||
@staticmethod
|
||||
def device_count():
|
||||
return 2
|
||||
|
||||
@staticmethod
|
||||
def get_device_properties(idx):
|
||||
return _Props(total=16000)
|
||||
|
||||
@staticmethod
|
||||
def mem_get_info(idx):
|
||||
return (8000, 16000)
|
||||
|
||||
@staticmethod
|
||||
def get_device_name(idx):
|
||||
return f"gpu-{idx}"
|
||||
|
||||
class _Torch:
|
||||
cuda = _Cuda
|
||||
|
||||
return _Torch
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("clean_gpu_memory_between_tests")
|
||||
def test_set_stage_devices_respects_logical_ids(monkeypatch):
|
||||
# Preserve an existing logical mapping and ensure devices "0,1" map through it.
|
||||
monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "6,7")
|
||||
call_log: list[int] = []
|
||||
dummy_torch = _make_dummy_torch(call_log)
|
||||
monkeypatch.setitem(sys.modules, "torch", dummy_torch)
|
||||
monkeypatch.setattr("vllm_omni.utils.detect_device_type", lambda: "cuda")
|
||||
monkeypatch.setattr("vllm_omni.utils.get_device_control_env_var", lambda: "CUDA_VISIBLE_DEVICES")
|
||||
|
||||
set_stage_devices(stage_id=0, devices="0,1")
|
||||
|
||||
assert os.environ["CUDA_VISIBLE_DEVICES"] == "6,7"
|
||||
assert call_log and call_log[0] == 0 # current device set after remap
|
||||
@@ -104,7 +104,6 @@ class _FakeStage:
|
||||
shm_threshold_bytes: int = 65536,
|
||||
ctx=None,
|
||||
batch_timeout: int = 10,
|
||||
**kwargs,
|
||||
):
|
||||
"""Mock init_stage_worker: don't start real process, just send stage_ready message."""
|
||||
# Create a mock process object
|
||||
|
||||
-107
@@ -1,107 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import os
|
||||
import time
|
||||
from contextlib import contextmanager
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if current_platform.is_rocm():
|
||||
from amdsmi import (
|
||||
amdsmi_get_gpu_vram_usage,
|
||||
amdsmi_get_processor_handles,
|
||||
amdsmi_init,
|
||||
amdsmi_shut_down,
|
||||
)
|
||||
|
||||
@contextmanager
|
||||
def _nvml():
|
||||
try:
|
||||
amdsmi_init()
|
||||
yield
|
||||
finally:
|
||||
amdsmi_shut_down()
|
||||
elif current_platform.is_cuda():
|
||||
from vllm.third_party.pynvml import (
|
||||
nvmlDeviceGetHandleByIndex,
|
||||
nvmlDeviceGetMemoryInfo,
|
||||
nvmlInit,
|
||||
nvmlShutdown,
|
||||
)
|
||||
|
||||
@contextmanager
|
||||
def _nvml():
|
||||
try:
|
||||
nvmlInit()
|
||||
yield
|
||||
finally:
|
||||
nvmlShutdown()
|
||||
else:
|
||||
|
||||
@contextmanager
|
||||
def _nvml():
|
||||
yield
|
||||
|
||||
|
||||
def get_physical_device_indices(devices):
|
||||
visible_devices = os.environ.get("CUDA_VISIBLE_DEVICES")
|
||||
if visible_devices is None:
|
||||
return devices
|
||||
|
||||
visible_indices = [int(x) for x in visible_devices.split(",")]
|
||||
index_mapping = {i: physical for i, physical in enumerate(visible_indices)}
|
||||
return [index_mapping[i] for i in devices if i in index_mapping]
|
||||
|
||||
|
||||
@_nvml()
|
||||
def wait_for_gpu_memory_to_clear(
|
||||
*,
|
||||
devices: list[int],
|
||||
threshold_bytes: int | None = None,
|
||||
threshold_ratio: float | None = None,
|
||||
timeout_s: float = 120,
|
||||
) -> None:
|
||||
assert threshold_bytes is not None or threshold_ratio is not None
|
||||
# Use nvml instead of pytorch to reduce measurement error from torch cuda
|
||||
# context.
|
||||
devices = get_physical_device_indices(devices)
|
||||
start_time = time.time()
|
||||
while True:
|
||||
output: dict[int, str] = {}
|
||||
output_raw: dict[int, tuple[float, float]] = {}
|
||||
for device in devices:
|
||||
if current_platform.is_rocm():
|
||||
dev_handle = amdsmi_get_processor_handles()[device]
|
||||
mem_info = amdsmi_get_gpu_vram_usage(dev_handle)
|
||||
gb_used = mem_info["vram_used"] / 2**10
|
||||
gb_total = mem_info["vram_total"] / 2**10
|
||||
else:
|
||||
dev_handle = nvmlDeviceGetHandleByIndex(device)
|
||||
mem_info = nvmlDeviceGetMemoryInfo(dev_handle)
|
||||
gb_used = mem_info.used / 2**30
|
||||
gb_total = mem_info.total / 2**30
|
||||
output_raw[device] = (gb_used, gb_total)
|
||||
output[device] = f"{gb_used:.02f}/{gb_total:.02f}"
|
||||
|
||||
print("gpu memory used/total (GiB): ", end="")
|
||||
for k, v in output.items():
|
||||
print(f"{k}={v}; ", end="")
|
||||
print("")
|
||||
|
||||
if threshold_bytes is not None:
|
||||
is_free = lambda used, total: used <= threshold_bytes / 2**30 # noqa E731
|
||||
threshold = f"{threshold_bytes / 2**30} GiB"
|
||||
else:
|
||||
is_free = lambda used, total: used / total <= threshold_ratio # noqa E731
|
||||
threshold = f"{threshold_ratio:.2f}"
|
||||
|
||||
dur_s = time.time() - start_time
|
||||
if all(is_free(used, total) for used, total in output_raw.values()):
|
||||
print(f"Done waiting for free GPU memory on devices {devices=} ({threshold=}) {dur_s=:.02f}")
|
||||
break
|
||||
|
||||
if dur_s >= timeout_s:
|
||||
raise ValueError(f"Memory of devices {devices=} not free after {dur_s=:.02f} ({threshold=})")
|
||||
|
||||
time.sleep(5)
|
||||
@@ -2,12 +2,12 @@
|
||||
Scheduling components for vLLM-Omni.
|
||||
"""
|
||||
|
||||
from .omni_ar_scheduler import OmniARScheduler
|
||||
from .omni_generation_scheduler import OmniGenerationScheduler
|
||||
from .diffusion_scheduler import DiffusionScheduler
|
||||
from .output import OmniNewRequestData
|
||||
from .scheduler import OmniScheduler
|
||||
|
||||
__all__ = [
|
||||
"OmniARScheduler",
|
||||
"OmniGenerationScheduler",
|
||||
"OmniScheduler",
|
||||
"DiffusionScheduler",
|
||||
"OmniNewRequestData",
|
||||
]
|
||||
@@ -0,0 +1,315 @@
|
||||
import time
|
||||
from collections import defaultdict
|
||||
from typing import Optional
|
||||
|
||||
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
|
||||
from vllm.v1.core.sched.request_queue import create_request_queue
|
||||
from vllm.v1.core.sched.scheduler import Request, RequestStatus, SchedulerOutput, SpecDecodingStats
|
||||
from vllm.v1.core.sched.utils import remove_all
|
||||
from vllm.v1.engine import EngineCoreEventType, EngineCoreOutput, EngineCoreOutputs
|
||||
|
||||
from vllm_omni.core.sched.output import OmniNewRequestData
|
||||
from vllm_omni.core.sched.scheduler import OmniScheduler
|
||||
from vllm_omni.outputs import OmniModelRunnerOutput
|
||||
|
||||
|
||||
class DiffusionScheduler(OmniScheduler):
|
||||
"""Scheduler for diffusion-based non-autoregressive stages.
|
||||
|
||||
Implements a fast-path scheduling strategy for diffusion models where
|
||||
all input tokens are fed at once. Falls back to standard vLLM scheduling
|
||||
if the token budget cannot accommodate all tokens in a single iteration.
|
||||
"""
|
||||
|
||||
def schedule(self) -> SchedulerOutput:
|
||||
"""Schedule requests for diffusion model execution.
|
||||
|
||||
Uses a fast-path approach that feeds all input tokens at once
|
||||
(or 1 placeholder token if input is empty). Falls back to
|
||||
standard scheduling if token budget is insufficient.
|
||||
|
||||
Returns:
|
||||
SchedulerOutput containing scheduled requests and token allocations
|
||||
"""
|
||||
|
||||
# Select requests with zero prompt and using pooling
|
||||
# (diffusion results returned via pooler_output)
|
||||
token_budget = self.max_num_scheduled_tokens
|
||||
capacity = self.max_num_running_reqs - len(self.running)
|
||||
scheduled_timestamp = time.monotonic()
|
||||
|
||||
scheduled_new_reqs: list[Request] = []
|
||||
scheduled_resumed_reqs: list[Request] = []
|
||||
scheduled_running_reqs: list[Request] = []
|
||||
|
||||
req_to_new_blocks: dict[str, KVCacheBlocks] = {}
|
||||
num_scheduled_tokens: dict[str, int] = {}
|
||||
scheduled_spec_decode_tokens: dict[str, list[int]] = {}
|
||||
scheduled_encoder_inputs: dict[str, list[int]] = {}
|
||||
|
||||
# Temporary queue: preserve waiting order, do not disturb non-diffusion requests
|
||||
skipped_waiting_requests = create_request_queue(self.policy)
|
||||
|
||||
# Fast path selection and scheduling (treat all as diffusion requests,
|
||||
# independent of pooling_params)
|
||||
while self.waiting and token_budget > 0 and capacity > 0:
|
||||
request = self.waiting.peek_request()
|
||||
# Uniformly treat as diffusion. A feature flag can be added later
|
||||
# via config or request tag.
|
||||
|
||||
# Allocate all input tokens for the request in one shot
|
||||
# (allocate 1 placeholder if zero)
|
||||
required_tokens = max(getattr(request, "num_prompt_tokens", 0), 1)
|
||||
if required_tokens > token_budget:
|
||||
# Insufficient budget to process all inputs at once;
|
||||
# stop fast path attempt
|
||||
break
|
||||
num_new_tokens = required_tokens
|
||||
new_blocks = self.kv_cache_manager.allocate_slots(
|
||||
request,
|
||||
num_new_tokens,
|
||||
num_lookahead_tokens=self.num_lookahead_tokens,
|
||||
)
|
||||
if new_blocks is None:
|
||||
# Allocation failed (e.g., VRAM pressure); stop fast path and
|
||||
# fall back to default scheduling
|
||||
# Put the current request back to the head of the waiting queue
|
||||
# Note: the original queue order is preserved
|
||||
break
|
||||
|
||||
# Officially schedule this request
|
||||
request = self.waiting.pop_request()
|
||||
self.running.append(request)
|
||||
request.status = RequestStatus.RUNNING
|
||||
if self.log_stats:
|
||||
request.record_event(EngineCoreEventType.SCHEDULED, scheduled_timestamp)
|
||||
|
||||
req_to_new_blocks[request.request_id] = new_blocks
|
||||
num_scheduled_tokens[request.request_id] = num_new_tokens
|
||||
token_budget -= num_new_tokens
|
||||
capacity -= 1
|
||||
scheduled_new_reqs.append(request)
|
||||
|
||||
# Return skipped waiting requests
|
||||
if skipped_waiting_requests:
|
||||
self.waiting.prepend_requests(skipped_waiting_requests)
|
||||
|
||||
# If fast path scheduled none, fall back to the original scheduling
|
||||
if not num_scheduled_tokens:
|
||||
return super().schedule()
|
||||
|
||||
# Compute common prefix blocks (aligned with v1)
|
||||
num_common_prefix_blocks = [0] * len(self.kv_cache_config.kv_cache_groups)
|
||||
if self.running:
|
||||
any_request = self.running[0]
|
||||
num_common_prefix_blocks = self.kv_cache_manager.get_num_common_prefix_blocks(
|
||||
any_request, len(self.running)
|
||||
)
|
||||
|
||||
# Assemble SchedulerOutput
|
||||
new_reqs_data = [
|
||||
OmniNewRequestData.from_request(req, req_to_new_blocks[req.request_id].get_block_ids())
|
||||
for req in scheduled_new_reqs
|
||||
]
|
||||
cached_reqs_data = self._make_cached_request_data(
|
||||
scheduled_running_reqs,
|
||||
scheduled_resumed_reqs,
|
||||
num_scheduled_tokens,
|
||||
scheduled_spec_decode_tokens,
|
||||
req_to_new_blocks,
|
||||
)
|
||||
scheduled_requests = scheduled_new_reqs + scheduled_running_reqs + scheduled_resumed_reqs
|
||||
structured_output_request_ids, grammar_bitmask = self.get_grammar_bitmask(
|
||||
scheduled_requests, scheduled_spec_decode_tokens
|
||||
)
|
||||
|
||||
total_num_scheduled_tokens = sum(num_scheduled_tokens.values())
|
||||
scheduler_output = SchedulerOutput(
|
||||
scheduled_new_reqs=new_reqs_data,
|
||||
scheduled_cached_reqs=cached_reqs_data,
|
||||
num_scheduled_tokens=num_scheduled_tokens,
|
||||
total_num_scheduled_tokens=total_num_scheduled_tokens,
|
||||
scheduled_spec_decode_tokens=scheduled_spec_decode_tokens,
|
||||
scheduled_encoder_inputs=scheduled_encoder_inputs,
|
||||
num_common_prefix_blocks=num_common_prefix_blocks,
|
||||
finished_req_ids=self.finished_req_ids,
|
||||
free_encoder_mm_hashes=self.encoder_cache_manager.get_freed_mm_hashes(),
|
||||
structured_output_request_ids=structured_output_request_ids,
|
||||
grammar_bitmask=grammar_bitmask,
|
||||
)
|
||||
|
||||
# KVTransfer: package metadata
|
||||
if self.connector is not None:
|
||||
meta = self.connector.build_connector_meta(scheduler_output)
|
||||
scheduler_output.kv_connector_metadata = meta
|
||||
|
||||
# Update internal state (advance num_computed_tokens, free encoder inputs,
|
||||
# etc.)
|
||||
self._update_after_schedule(scheduler_output)
|
||||
return scheduler_output
|
||||
|
||||
"""
|
||||
Scheduler for the diffusion model.
|
||||
This scheduler is modified to stop the request immediately for the diffusion model.
|
||||
This is because the diffusion model can generate the final image/audio in one step.
|
||||
Note: This is just a minimal modification to the original scheduler,
|
||||
and there should be some further efforts to optimize the scheduler.
|
||||
The original scheduler is still used for the AR model.
|
||||
"""
|
||||
|
||||
def update_from_output(
|
||||
self,
|
||||
scheduler_output: SchedulerOutput,
|
||||
model_runner_output: OmniModelRunnerOutput,
|
||||
) -> dict[int, EngineCoreOutputs]:
|
||||
"""Update the scheduler state based on the model runner output.
|
||||
|
||||
This method is modified to stop the request immediately for the diffusion model.
|
||||
"""
|
||||
sampled_token_ids = model_runner_output.sampled_token_ids
|
||||
logprobs = model_runner_output.logprobs
|
||||
prompt_logprobs_dict = model_runner_output.prompt_logprobs_dict
|
||||
num_scheduled_tokens = scheduler_output.num_scheduled_tokens
|
||||
pooler_outputs = model_runner_output.pooler_output
|
||||
num_nans_in_logits = model_runner_output.num_nans_in_logits
|
||||
kv_connector_output = model_runner_output.kv_connector_output
|
||||
|
||||
outputs: dict[int, list[EngineCoreOutput]] = defaultdict(list)
|
||||
spec_decoding_stats: Optional[SpecDecodingStats] = None
|
||||
kv_connector_stats = kv_connector_output.kv_connector_stats if kv_connector_output else None
|
||||
|
||||
# NOTE(woosuk): As len(num_scheduled_tokens) can be up to 1K or more,
|
||||
# the below loop can be a performance bottleneck. We should do our best
|
||||
# to avoid expensive operations inside the loop.
|
||||
stopped_running_reqs: set[Request] = set()
|
||||
stopped_preempted_reqs: set[Request] = set()
|
||||
for req_id, num_tokens_scheduled in num_scheduled_tokens.items():
|
||||
assert num_tokens_scheduled > 0
|
||||
request = self.requests.get(req_id)
|
||||
if request is None:
|
||||
# The request is already finished. This can happen if the
|
||||
# request is aborted while the model is executing it (e.g.,
|
||||
# in pipeline parallelism).
|
||||
continue
|
||||
|
||||
req_index = model_runner_output.req_id_to_index[req_id]
|
||||
generated_token_ids = sampled_token_ids[req_index] if sampled_token_ids else []
|
||||
|
||||
scheduled_spec_token_ids = scheduler_output.scheduled_spec_decode_tokens.get(req_id)
|
||||
if scheduled_spec_token_ids:
|
||||
num_draft_tokens = len(scheduled_spec_token_ids)
|
||||
num_accepted = len(generated_token_ids) - 1
|
||||
num_rejected = num_draft_tokens - num_accepted
|
||||
# num_computed_tokens represents the number of tokens
|
||||
# processed in the current step, considering scheduled
|
||||
# tokens and rejections. If some tokens are rejected,
|
||||
# num_computed_tokens is decreased by the number of rejected
|
||||
# tokens.
|
||||
request.num_computed_tokens -= num_rejected
|
||||
spec_decoding_stats = self.make_spec_decoding_stats(
|
||||
spec_decoding_stats,
|
||||
num_draft_tokens=num_draft_tokens,
|
||||
num_accepted_tokens=num_accepted,
|
||||
)
|
||||
|
||||
new_logprobs = None
|
||||
new_token_ids = generated_token_ids
|
||||
kv_transfer_params = None
|
||||
status_before_stop = request.status
|
||||
pooler_output = None
|
||||
if pooler_outputs:
|
||||
pooler_output = pooler_outputs[req_index]
|
||||
|
||||
# Diffusion request: completes in one step; mark finished and free resources
|
||||
request.status = RequestStatus.FINISHED_STOPPED
|
||||
# Optional: set a stop_reason for front-end clarity
|
||||
# (does not affect protocol)
|
||||
request.stop_reason = request.stop_reason or "diffusion_done"
|
||||
kv_transfer_params = self._free_request(request)
|
||||
if status_before_stop == RequestStatus.RUNNING:
|
||||
stopped_running_reqs.add(request)
|
||||
else:
|
||||
stopped_preempted_reqs.add(request)
|
||||
|
||||
# Extract sample logprobs if needed.
|
||||
if request.sampling_params is not None and request.sampling_params.logprobs is not None and logprobs:
|
||||
# NOTE: once we support N tokens per step (spec decode),
|
||||
# the outer lists can be of length > 1.
|
||||
new_logprobs = logprobs.slice(req_index, req_index + 1)
|
||||
|
||||
if new_token_ids and self.structured_output_manager.should_advance(request):
|
||||
# NOTE: structured_output_request should not be None if
|
||||
# use_structured_output, we have check above, so safe to ignore
|
||||
# type warning
|
||||
request.structured_output_request.grammar.accept_tokens(req_id, new_token_ids) # type: ignore[union-attr] # noqa: E501
|
||||
|
||||
# spec_token_ids comes from the model runner output
|
||||
if num_nans_in_logits is not None and req_id in num_nans_in_logits:
|
||||
request.num_nans_in_logits = num_nans_in_logits[req_id]
|
||||
|
||||
# Get prompt logprobs for this request.
|
||||
prompt_logprobs_tensors = prompt_logprobs_dict.get(req_id)
|
||||
# Convert pooler_output tensor to dict format expected by OmniEngineCoreOutput
|
||||
pooling_output_dict = None
|
||||
if pooler_output is not None:
|
||||
# Wrap tensor in dict to match OmniEngineCoreOutput.pooling_output type
|
||||
# which expects Optional[dict[str, torch.Tensor]]
|
||||
pooling_output_dict = {"model_outputs": pooler_output}
|
||||
if new_token_ids or pooler_output is not None or kv_transfer_params:
|
||||
# Add EngineCoreOutput for this Request.
|
||||
outputs[request.client_index].append(
|
||||
EngineCoreOutput(
|
||||
request_id=req_id,
|
||||
new_token_ids=new_token_ids,
|
||||
finish_reason=request.get_finished_reason(),
|
||||
new_logprobs=new_logprobs,
|
||||
new_prompt_logprobs_tensors=prompt_logprobs_tensors,
|
||||
pooling_output=pooling_output_dict,
|
||||
stop_reason=request.stop_reason,
|
||||
events=request.take_events(),
|
||||
kv_transfer_params=kv_transfer_params,
|
||||
trace_headers=request.trace_headers,
|
||||
num_cached_tokens=request.num_cached_tokens,
|
||||
)
|
||||
)
|
||||
else:
|
||||
# Invariant: EngineCore returns no partial prefill outputs.
|
||||
assert not prompt_logprobs_tensors
|
||||
|
||||
# Remove the stopped requests from the running and waiting queues.
|
||||
if stopped_running_reqs:
|
||||
if stopped_running_reqs:
|
||||
self.running = remove_all(self.running, stopped_running_reqs)
|
||||
if stopped_preempted_reqs:
|
||||
# This is a rare case and unlikely to impact performance.
|
||||
self.waiting.remove_requests(stopped_preempted_reqs)
|
||||
|
||||
# KV Connector: update state for finished KV Transfers.
|
||||
if model_runner_output.kv_connector_output:
|
||||
self._update_from_kv_xfer_finished(model_runner_output.kv_connector_output)
|
||||
|
||||
# Create EngineCoreOutputs for all clients that have requests with
|
||||
# outputs in this step.
|
||||
engine_core_outputs = {client_index: EngineCoreOutputs(outputs=outs) for client_index, outs in outputs.items()}
|
||||
|
||||
finished_req_ids = self.finished_req_ids_dict
|
||||
if finished_req_ids:
|
||||
# Include ids of requests that finished since last outputs
|
||||
# were sent.
|
||||
for client_index, finished_set in finished_req_ids.items():
|
||||
# Set finished request set in EngineCoreOutputs for this client.
|
||||
if (eco := engine_core_outputs.get(client_index)) is not None:
|
||||
eco.finished_requests = finished_set
|
||||
else:
|
||||
engine_core_outputs[client_index] = EngineCoreOutputs(finished_requests=finished_set)
|
||||
finished_req_ids.clear()
|
||||
|
||||
if (stats := self.make_stats(spec_decoding_stats, kv_connector_stats)) is not None:
|
||||
# Return stats to only one of the front-ends.
|
||||
if (eco := next(iter(engine_core_outputs.values()), None)) is None:
|
||||
# We must return the stats even if there are no request
|
||||
# outputs this step.
|
||||
engine_core_outputs[0] = eco = EngineCoreOutputs()
|
||||
eco.scheduler_stats = stats
|
||||
|
||||
return engine_core_outputs
|
||||
+2
-2
@@ -10,15 +10,15 @@ from vllm.v1.core.sched.scheduler import (
|
||||
SchedulerOutput,
|
||||
SpecDecodingStats,
|
||||
)
|
||||
from vllm.v1.core.sched.scheduler import Scheduler as VLLMScheduler
|
||||
from vllm.v1.core.sched.utils import remove_all
|
||||
from vllm.v1.engine import EngineCoreEventType, EngineCoreOutput, EngineCoreOutputs
|
||||
|
||||
from vllm_omni.core.sched.output import OmniNewRequestData
|
||||
from vllm_omni.core.sched.scheduler import OmniScheduler
|
||||
from vllm_omni.outputs import OmniModelRunnerOutput
|
||||
|
||||
|
||||
class OmniGenerationScheduler(VLLMScheduler):
|
||||
class GenerationScheduler(OmniScheduler):
|
||||
def schedule(self) -> SchedulerOutput:
|
||||
"""Diffusion fast path:
|
||||
- Feed all input tokens of the request at once
|
||||
@@ -12,7 +12,7 @@ from vllm.v1.request import Request, RequestStatus
|
||||
from vllm.v1.spec_decode.metrics import SpecDecodingStats
|
||||
|
||||
|
||||
class OmniARScheduler(VLLMScheduler):
|
||||
class OmniScheduler(VLLMScheduler):
|
||||
"""
|
||||
OmniScheduler: Scheduler for vLLM-Omni multimodal processing.
|
||||
|
||||
@@ -1,2 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
@@ -1,2 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
@@ -1,78 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from dataclasses import dataclass
|
||||
from typing import Generic, TypeVar
|
||||
|
||||
import torch
|
||||
|
||||
|
||||
class AttentionBackend(ABC):
|
||||
"""Abstract class for diffusion attention backends."""
|
||||
|
||||
accept_output_buffer: bool = False
|
||||
|
||||
@staticmethod
|
||||
@abstractmethod
|
||||
def get_name() -> str:
|
||||
raise NotImplementedError
|
||||
|
||||
@staticmethod
|
||||
@abstractmethod
|
||||
def get_impl_cls() -> type["AttentionImpl"]:
|
||||
raise NotImplementedError
|
||||
|
||||
@staticmethod
|
||||
@abstractmethod
|
||||
def get_metadata_cls() -> type["AttentionMetadata"]:
|
||||
raise NotImplementedError
|
||||
|
||||
@staticmethod
|
||||
@abstractmethod
|
||||
def get_builder_cls(): # -> Type["AttentionMetadataBuilder"]:
|
||||
raise NotImplementedError
|
||||
|
||||
@staticmethod
|
||||
@abstractmethod
|
||||
def get_supported_head_sizes() -> list[int]:
|
||||
"""Get the list of supported head sizes for this backend."""
|
||||
raise NotImplementedError
|
||||
|
||||
@classmethod
|
||||
def supports_head_size(cls, head_size: int) -> bool:
|
||||
supported_head_sizes = cls.get_supported_head_sizes()
|
||||
return (not supported_head_sizes) or head_size in supported_head_sizes
|
||||
|
||||
|
||||
@dataclass
|
||||
class AttentionMetadata:
|
||||
attn_mask: torch.Tensor | None = None
|
||||
|
||||
|
||||
T = TypeVar("T", bound=AttentionMetadata)
|
||||
|
||||
|
||||
class AttentionImpl(ABC, Generic[T]):
|
||||
@abstractmethod
|
||||
def __init__(
|
||||
self,
|
||||
num_heads: int,
|
||||
head_size: int,
|
||||
softmax_scale: float,
|
||||
causal: bool = False,
|
||||
num_kv_heads: int | None = None,
|
||||
prefix: str = "",
|
||||
**extra_impl_args,
|
||||
) -> None:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def forward(
|
||||
self,
|
||||
query: torch.Tensor,
|
||||
key: torch.Tensor,
|
||||
value: torch.Tensor,
|
||||
attn_metadata: T,
|
||||
) -> torch.Tensor:
|
||||
raise NotImplementedError
|
||||
@@ -1,74 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import torch
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from vllm_omni.diffusion.attention.backends.abstract import (
|
||||
AttentionBackend,
|
||||
AttentionImpl,
|
||||
AttentionMetadata,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
try:
|
||||
# only tested with flash_attn v3
|
||||
# from flash_attn_interface import flash_attn_func as flash_attn_3_func # not available in flash-attn 2.8.1
|
||||
from flash_attn import flash_attn_func # can be FA2 or FA3
|
||||
except ImportError:
|
||||
logger.warning(
|
||||
"FlashAttentionBackend is not available. You may install flash-attn "
|
||||
"by running `uv pip install flash-attn==2.8.1 --no-build-isolation`"
|
||||
" or install pre-built flash-attn from https://github.com/Dao-AILab/flash-attention/releases"
|
||||
)
|
||||
flash_attn_3_func = None
|
||||
|
||||
|
||||
class FlashAttentionBackend(AttentionBackend):
|
||||
accept_output_buffer: bool = True
|
||||
|
||||
@staticmethod
|
||||
def get_supported_head_sizes() -> list[int]:
|
||||
return [64, 96, 128, 192, 256]
|
||||
|
||||
@staticmethod
|
||||
def get_name() -> str:
|
||||
return "FLASH_ATTN"
|
||||
|
||||
@staticmethod
|
||||
def get_impl_cls() -> type["FlashAttentionImpl"]:
|
||||
return FlashAttentionImpl
|
||||
|
||||
|
||||
class FlashAttentionImpl(AttentionImpl):
|
||||
def __init__(
|
||||
self,
|
||||
num_heads: int,
|
||||
head_size: int,
|
||||
softmax_scale: float,
|
||||
causal: bool = False,
|
||||
num_kv_heads: int | None = None,
|
||||
prefix: str = "",
|
||||
**extra_impl_args,
|
||||
) -> None:
|
||||
self.num_heads = num_heads
|
||||
self.causal = causal
|
||||
self.softmax_scale = softmax_scale
|
||||
|
||||
def forward(
|
||||
self,
|
||||
query: torch.Tensor,
|
||||
key: torch.Tensor,
|
||||
value: torch.Tensor,
|
||||
attn_metadata: AttentionMetadata = None,
|
||||
) -> torch.Tensor:
|
||||
# TODO: flash_attn_func does not support attn_mask.
|
||||
out: torch.Tensor = flash_attn_func(
|
||||
query,
|
||||
key,
|
||||
value,
|
||||
causal=self.causal,
|
||||
softmax_scale=self.softmax_scale,
|
||||
)
|
||||
return out
|
||||
@@ -1,66 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import torch
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from vllm_omni.diffusion.attention.backends.abstract import (
|
||||
AttentionBackend,
|
||||
AttentionImpl,
|
||||
AttentionMetadata,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class SDPABackend(AttentionBackend):
|
||||
accept_output_buffer: bool = True
|
||||
|
||||
@staticmethod
|
||||
def get_supported_head_sizes() -> list[int]:
|
||||
return [x for x in range(1024)] # todo
|
||||
|
||||
@staticmethod
|
||||
def get_name() -> str:
|
||||
return "SDPA"
|
||||
|
||||
@staticmethod
|
||||
def get_impl_cls() -> type["SDPAImpl"]:
|
||||
return SDPAImpl
|
||||
|
||||
|
||||
class SDPAImpl(AttentionImpl):
|
||||
def __init__(
|
||||
self,
|
||||
num_heads: int,
|
||||
head_size: int,
|
||||
softmax_scale: float,
|
||||
causal: bool = False,
|
||||
num_kv_heads: int | None = None,
|
||||
prefix: str = "",
|
||||
**extra_impl_args,
|
||||
) -> None:
|
||||
self.causal = causal
|
||||
self.softmax_scale = softmax_scale
|
||||
|
||||
def forward(
|
||||
self,
|
||||
query: torch.Tensor,
|
||||
key: torch.Tensor,
|
||||
value: torch.Tensor,
|
||||
attn_metadata: AttentionMetadata = None,
|
||||
) -> torch.Tensor:
|
||||
query, key, value = (x.permute(0, 2, 1, 3) for x in (query, key, value))
|
||||
attention_mask = attn_metadata.attn_mask if attn_metadata else None
|
||||
|
||||
output = torch.nn.functional.scaled_dot_product_attention(
|
||||
query,
|
||||
key,
|
||||
value,
|
||||
attn_mask=attention_mask,
|
||||
dropout_p=0.0,
|
||||
is_causal=self.causal,
|
||||
scale=self.softmax_scale,
|
||||
)
|
||||
out = output.permute(0, 2, 1, 3)
|
||||
return out
|
||||
@@ -1,43 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm_omni.diffusion.attention.backends.abstract import (
|
||||
AttentionMetadata,
|
||||
)
|
||||
from vllm_omni.diffusion.attention.selector import get_attn_backend
|
||||
|
||||
|
||||
class Attention(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
num_heads: int,
|
||||
head_size: int,
|
||||
causal: bool,
|
||||
softmax_scale: float,
|
||||
num_kv_heads: int | None = None,
|
||||
prefix: str = "",
|
||||
):
|
||||
super().__init__()
|
||||
self.attn_backend = get_attn_backend(-1)
|
||||
self.attn_impl_cls = self.attn_backend.get_impl_cls()
|
||||
self.attention = self.attn_impl_cls(
|
||||
num_heads=num_heads,
|
||||
head_size=head_size,
|
||||
softmax_scale=softmax_scale,
|
||||
causal=causal,
|
||||
num_kv_heads=num_kv_heads,
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
query: torch.Tensor,
|
||||
key: torch.Tensor,
|
||||
value: torch.Tensor,
|
||||
attn_metadata: AttentionMetadata = None,
|
||||
) -> torch.Tensor:
|
||||
# shape: (batch_size, seq_len, num_heads, head_size)
|
||||
attn_output = self.attention.forward(query, key, value, attn_metadata)
|
||||
return attn_output
|
||||
@@ -1,53 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import os
|
||||
from functools import cache
|
||||
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from vllm_omni.diffusion.attention.backends.abstract import (
|
||||
AttentionBackend,
|
||||
)
|
||||
from vllm_omni.diffusion.attention.backends.flash_attn import FlashAttentionBackend
|
||||
from vllm_omni.diffusion.attention.backends.sdpa import SDPABackend
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# environment variable value -> backend class
|
||||
SUPPORTED_BACKENDS = {
|
||||
"FLASH_ATTN": FlashAttentionBackend,
|
||||
"TORCH_SDPA": SDPABackend,
|
||||
}
|
||||
|
||||
|
||||
@cache
|
||||
def get_attn_backend(head_size: int) -> type[AttentionBackend]:
|
||||
"""
|
||||
Get attention backend for diffusion models.
|
||||
|
||||
The backend is selected based on the following priority:
|
||||
1. DIFFUSION_ATTENTION_BACKEND environment variable (if set, e.g. export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN)
|
||||
2. Default backend (SDPA)
|
||||
|
||||
Args:
|
||||
head_size: Head size (currently not used for selection, but kept for API compatibility)
|
||||
|
||||
Returns:
|
||||
The selected attention backend class
|
||||
"""
|
||||
# Check environment variable
|
||||
backend_name: str | None = os.environ.get("DIFFUSION_ATTENTION_BACKEND")
|
||||
|
||||
if backend_name is not None:
|
||||
backend_name_upper = backend_name.upper()
|
||||
if backend_name_upper not in SUPPORTED_BACKENDS:
|
||||
valid_backends = list(SUPPORTED_BACKENDS.keys())
|
||||
raise ValueError(
|
||||
f"Invalid attention backend for diffusion: '{backend_name}'. Valid backends are: {valid_backends}"
|
||||
)
|
||||
logger.info(f"Using attention backend '{backend_name_upper}' for diffusion")
|
||||
return SUPPORTED_BACKENDS[backend_name_upper]
|
||||
|
||||
# Default to SDPA
|
||||
return SDPABackend
|
||||
-41
@@ -1,41 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""
|
||||
Cache module for diffusion model inference acceleration.
|
||||
|
||||
This module provides a unified cache adapter system for different caching strategies:
|
||||
- TeaCache: Timestep Embedding Aware Cache for adaptive transformer caching
|
||||
- (Future: DeepCache, DistriFusion, etc.)
|
||||
|
||||
The cache adapter system uses a registry pattern where cache types can be
|
||||
dynamically registered and instantiated. Cache is configured via OmniDiffusionConfig
|
||||
or DIFFUSION_CACHE_ADAPTER environment variable.
|
||||
"""
|
||||
|
||||
from vllm_omni.diffusion.cache.apply import setup_cache
|
||||
from vllm_omni.diffusion.cache.base import CacheAdapter
|
||||
from vllm_omni.diffusion.cache.registry import (
|
||||
CacheType,
|
||||
get_cache_adapter,
|
||||
register_cache_adapter,
|
||||
)
|
||||
from vllm_omni.diffusion.cache.teacache import (
|
||||
CacheContext,
|
||||
TeaCacheConfig,
|
||||
apply_teacache_hook,
|
||||
)
|
||||
|
||||
# Import teacache adapter to trigger registration
|
||||
from vllm_omni.diffusion.cache.teacache.adapter import TeaCacheAdapter # noqa: F401
|
||||
|
||||
__all__ = [
|
||||
"CacheAdapter",
|
||||
"CacheType",
|
||||
"TeaCacheConfig",
|
||||
"CacheContext",
|
||||
"setup_cache",
|
||||
"get_cache_adapter",
|
||||
"register_cache_adapter",
|
||||
"apply_teacache_hook",
|
||||
]
|
||||
Vendored
-82
@@ -1,82 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""
|
||||
Unified cache setup function for diffusion models.
|
||||
|
||||
This module provides the main entry point for setting up cache adapters
|
||||
on transformer models. It handles cache type selection, adapter instantiation,
|
||||
and application to the model.
|
||||
"""
|
||||
|
||||
from typing import Any, Optional
|
||||
|
||||
import torch
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from vllm_omni.diffusion.cache.base import CacheAdapter
|
||||
from vllm_omni.diffusion.cache.registry import get_cache_adapter
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def setup_cache(
|
||||
transformer: torch.nn.Module,
|
||||
cache_type: str = "none",
|
||||
cache_config: Optional[dict[str, Any]] = None,
|
||||
) -> Optional[CacheAdapter]:
|
||||
"""
|
||||
Setup cache adapter for transformer (one-time initialization).
|
||||
|
||||
This is the main entry point for applying caching to diffusion transformers.
|
||||
It's called once during pipeline initialization and applies the appropriate
|
||||
cache adapter based on configuration.
|
||||
|
||||
Args:
|
||||
transformer: Transformer module to apply cache to
|
||||
cache_type: Type of cache to apply. Options:
|
||||
- "none": No caching (default)
|
||||
- "tea_cache": TeaCache adaptive caching
|
||||
- Additional types can be registered via register_cache_adapter()
|
||||
cache_config: Cache-specific configuration dictionary. Contents depend
|
||||
on cache type:
|
||||
- For "tea_cache": {"rel_l1_thresh": 0.2, "model_type": "QwenImagePipeline"}
|
||||
- For other types: see their respective adapter documentation
|
||||
|
||||
Returns:
|
||||
CacheAdapter instance for state management, or None if cache_type="none"
|
||||
|
||||
Example:
|
||||
>>> # Setup TeaCache during pipeline initialization
|
||||
>>> adapter = setup_cache(
|
||||
... transformer,
|
||||
... cache_type="tea_cache",
|
||||
... cache_config={"rel_l1_thresh": 0.2}
|
||||
... )
|
||||
>>>
|
||||
>>> # Reset cache state before each generation
|
||||
>>> if adapter is not None:
|
||||
... adapter.reset(transformer)
|
||||
"""
|
||||
# No caching requested
|
||||
if cache_type == "none":
|
||||
logger.debug("No cache adapter requested (cache_type='none')")
|
||||
return None
|
||||
|
||||
# Get adapter instance from registry
|
||||
try:
|
||||
adapter = get_cache_adapter(cache_type, cache_config or {})
|
||||
except ValueError as e:
|
||||
logger.error(f"Failed to get cache adapter: {e}")
|
||||
raise
|
||||
|
||||
# Apply cache to transformer
|
||||
logger.info(f"Applying {cache_type} cache to transformer")
|
||||
try:
|
||||
adapter.apply(transformer)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to apply cache adapter: {e}")
|
||||
raise
|
||||
|
||||
logger.info(f"Cache adapter '{cache_type}' successfully applied")
|
||||
return adapter
|
||||
Vendored
-67
@@ -1,67 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""
|
||||
Base cache adapter interface for diffusion models.
|
||||
|
||||
This module defines the abstract base class that all cache adapters must implement.
|
||||
Cache adapters provide a unified interface for applying different caching strategies
|
||||
(TeaCache, DeepCache, etc.) to transformer models using hooks.
|
||||
"""
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
|
||||
class CacheAdapter(ABC):
|
||||
"""
|
||||
Abstract base class for cache adapters.
|
||||
|
||||
Cache adapters apply caching strategies to transformer models to accelerate
|
||||
inference. Each adapter type (TeaCache, DeepCache, etc.) implements the
|
||||
apply() and reset() methods to manage cache lifecycle.
|
||||
|
||||
Attributes:
|
||||
config: Dictionary containing cache-specific configuration parameters
|
||||
"""
|
||||
|
||||
def __init__(self, config: dict[str, Any]):
|
||||
"""
|
||||
Initialize cache adapter with configuration.
|
||||
|
||||
Args:
|
||||
config: Cache-specific configuration dictionary
|
||||
"""
|
||||
self.config = config
|
||||
|
||||
@abstractmethod
|
||||
def apply(self, transformer: torch.nn.Module) -> None:
|
||||
"""
|
||||
Apply cache to transformer using hooks.
|
||||
|
||||
This method should register the appropriate hooks on the transformer
|
||||
to enable caching during inference. Called once during pipeline
|
||||
initialization.
|
||||
|
||||
Args:
|
||||
transformer: Transformer module to apply cache to
|
||||
"""
|
||||
raise NotImplementedError("Subclasses must implement apply()")
|
||||
|
||||
@abstractmethod
|
||||
def reset(self, transformer: torch.nn.Module) -> None:
|
||||
"""
|
||||
Reset cache state for new generation.
|
||||
|
||||
This method should clear any cached values and reset counters/accumulators.
|
||||
Called at the start of each generation to ensure clean state.
|
||||
|
||||
Args:
|
||||
transformer: Transformer module to reset cache on
|
||||
"""
|
||||
raise NotImplementedError("Subclasses must implement reset()")
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return f"{self.__class__.__name__}(config={self.config})"
|
||||
-100
@@ -1,100 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""
|
||||
Cache adapter registry for diffusion models.
|
||||
|
||||
This module provides a registry pattern for cache adapters, allowing dynamic
|
||||
registration and instantiation of different cache types (TeaCache, DeepCache, etc.).
|
||||
"""
|
||||
|
||||
from enum import Enum
|
||||
from typing import Any
|
||||
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from vllm_omni.diffusion.cache.base import CacheAdapter
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class CacheType(Enum):
|
||||
"""Supported cache adapter types."""
|
||||
|
||||
NONE = "none"
|
||||
TEA_CACHE = "tea_cache"
|
||||
# Future cache types can be added here:
|
||||
# DEEP_CACHE = "deep_cache"
|
||||
# DISTRI_FUSION = "distri_fusion"
|
||||
|
||||
|
||||
# Global registry mapping cache types to adapter classes
|
||||
CACHE_ADAPTER_REGISTRY: dict[CacheType, type[CacheAdapter]] = {}
|
||||
|
||||
|
||||
def register_cache_adapter(cache_type: CacheType, adapter_class: type[CacheAdapter]) -> None:
|
||||
"""
|
||||
Register a cache adapter class for a given cache type.
|
||||
|
||||
This allows extending the system with new cache types without modifying
|
||||
the core cache infrastructure.
|
||||
|
||||
Args:
|
||||
cache_type: CacheType enum value
|
||||
adapter_class: CacheAdapter subclass to register
|
||||
|
||||
Example:
|
||||
>>> register_cache_adapter(CacheType.TEA_CACHE, TeaCacheAdapter)
|
||||
"""
|
||||
if not issubclass(adapter_class, CacheAdapter):
|
||||
raise TypeError(f"{adapter_class} must be a subclass of CacheAdapter")
|
||||
|
||||
CACHE_ADAPTER_REGISTRY[cache_type] = adapter_class
|
||||
logger.debug(f"Registered cache adapter: {cache_type.value} -> {adapter_class.__name__}")
|
||||
|
||||
|
||||
def get_cache_adapter(cache_type: str, config: dict[str, Any]) -> CacheAdapter:
|
||||
"""
|
||||
Factory function to get cache adapter instance.
|
||||
|
||||
Converts string cache type to enum, looks up in registry, and instantiates
|
||||
the appropriate adapter class with the provided configuration.
|
||||
|
||||
Args:
|
||||
cache_type: String name of cache type ("tea_cache", "deep_cache", etc.)
|
||||
config: Configuration dictionary to pass to adapter constructor
|
||||
|
||||
Returns:
|
||||
Instantiated CacheAdapter subclass
|
||||
|
||||
Raises:
|
||||
ValueError: If cache_type is unknown or not registered
|
||||
|
||||
Example:
|
||||
>>> adapter = get_cache_adapter("tea_cache", {"rel_l1_thresh": 0.2})
|
||||
>>> adapter.apply(transformer)
|
||||
"""
|
||||
# Normalize cache type string
|
||||
cache_type_str = cache_type.lower().strip()
|
||||
|
||||
# Convert string to enum
|
||||
try:
|
||||
cache_enum = CacheType(cache_type_str)
|
||||
except ValueError:
|
||||
available = [ct.value for ct in CacheType if ct != CacheType.NONE]
|
||||
raise ValueError(f"Unknown cache type: '{cache_type}'. Available types: {available}")
|
||||
|
||||
# Check if it's the special "none" case
|
||||
if cache_enum == CacheType.NONE:
|
||||
raise ValueError("Cannot instantiate adapter for cache_type='none'. Use setup_cache() which handles this case.")
|
||||
|
||||
# Lookup in registry
|
||||
if cache_enum not in CACHE_ADAPTER_REGISTRY:
|
||||
raise ValueError(
|
||||
f"Cache type '{cache_type}' is not registered. Registered types: {list(CACHE_ADAPTER_REGISTRY.keys())}"
|
||||
)
|
||||
|
||||
adapter_class = CACHE_ADAPTER_REGISTRY[cache_enum]
|
||||
|
||||
# Instantiate and return
|
||||
return adapter_class(config)
|
||||
-45
@@ -1,45 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""
|
||||
TeaCache: Timestep Embedding Aware Cache for diffusion model acceleration.
|
||||
|
||||
TeaCache speeds up diffusion inference by reusing transformer block computations
|
||||
when consecutive timestep embeddings are similar.
|
||||
|
||||
This implementation uses a hooks-based approach that requires zero changes to
|
||||
model code. Model developers only need to add an extractor function to support
|
||||
new models.
|
||||
|
||||
Usage:
|
||||
from vllm_omni import Omni
|
||||
|
||||
omni = Omni(
|
||||
model="Qwen/Qwen-Image",
|
||||
cache_adapter="tea_cache",
|
||||
cache_config={"rel_l1_thresh": 0.2}
|
||||
)
|
||||
images = omni.generate("a cat")
|
||||
|
||||
# Alternative: Using environment variable
|
||||
# export DIFFUSION_CACHE_ADAPTER=tea_cache
|
||||
"""
|
||||
|
||||
from vllm_omni.diffusion.cache.teacache.adapter import TeaCacheAdapter
|
||||
from vllm_omni.diffusion.cache.teacache.config import TeaCacheConfig
|
||||
from vllm_omni.diffusion.cache.teacache.extractors import (
|
||||
CacheContext,
|
||||
register_extractor,
|
||||
)
|
||||
from vllm_omni.diffusion.cache.teacache.hook import TeaCacheHook, apply_teacache_hook
|
||||
from vllm_omni.diffusion.cache.teacache.state import TeaCacheState
|
||||
|
||||
__all__ = [
|
||||
"TeaCacheAdapter",
|
||||
"TeaCacheConfig",
|
||||
"TeaCacheState",
|
||||
"TeaCacheHook",
|
||||
"apply_teacache_hook",
|
||||
"register_extractor",
|
||||
"CacheContext",
|
||||
]
|
||||
-96
@@ -1,96 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""
|
||||
TeaCache adapter implementation.
|
||||
|
||||
This module provides the TeaCache adapter that implements the CacheAdapter
|
||||
interface using the hooks-based TeaCache system.
|
||||
"""
|
||||
|
||||
import torch
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from vllm_omni.diffusion.cache.base import CacheAdapter
|
||||
from vllm_omni.diffusion.cache.registry import CacheType, register_cache_adapter
|
||||
from vllm_omni.diffusion.cache.teacache.config import TeaCacheConfig
|
||||
from vllm_omni.diffusion.cache.teacache.hook import TeaCacheHook, apply_teacache_hook
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class TeaCacheAdapter(CacheAdapter):
|
||||
"""
|
||||
TeaCache implementation using hooks.
|
||||
|
||||
TeaCache (Timestep Embedding Aware Cache) is an adaptive caching technique
|
||||
that speeds up diffusion inference by reusing transformer block computations
|
||||
when consecutive timestep embeddings are similar.
|
||||
|
||||
The adapter applies TeaCache hooks to the transformer which intercept the
|
||||
forward pass and implement the caching logic transparently.
|
||||
|
||||
Example:
|
||||
>>> adapter = TeaCacheAdapter({"rel_l1_thresh": 0.2})
|
||||
>>> adapter.apply(transformer)
|
||||
>>> # Generate with cache enabled
|
||||
>>> adapter.reset(transformer) # Reset before each generation
|
||||
"""
|
||||
|
||||
def apply(self, transformer: torch.nn.Module) -> None:
|
||||
"""
|
||||
Apply TeaCache to transformer using hooks.
|
||||
|
||||
This creates a TeaCacheConfig from the adapter's config dict and
|
||||
applies the TeaCache hook to the transformer.
|
||||
|
||||
Args:
|
||||
transformer: Transformer module to apply TeaCache to
|
||||
"""
|
||||
# Validate that model_type is provided
|
||||
if "model_type" not in self.config:
|
||||
raise ValueError(
|
||||
"model_type must be provided in cache_config. "
|
||||
"Should be set to OmniDiffusionConfig.model_class_name (e.g., 'QwenImagePipeline')"
|
||||
)
|
||||
|
||||
# Create TeaCacheConfig from dict
|
||||
try:
|
||||
teacache_config = TeaCacheConfig(**self.config)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to create TeaCacheConfig: {e}")
|
||||
raise ValueError(
|
||||
f"Invalid TeaCache configuration: {e}. "
|
||||
f"Expected keys: rel_l1_thresh, model_type (pipeline class name), coefficients (optional)"
|
||||
)
|
||||
|
||||
# Apply hook to transformer
|
||||
apply_teacache_hook(transformer, teacache_config)
|
||||
logger.info(
|
||||
f"TeaCache applied with rel_l1_thresh={teacache_config.rel_l1_thresh}, "
|
||||
f"model_type={teacache_config.model_type}"
|
||||
)
|
||||
|
||||
def reset(self, transformer: torch.nn.Module) -> None:
|
||||
"""
|
||||
Reset TeaCache state for new generation.
|
||||
|
||||
Clears all cached residuals and resets counters/accumulators.
|
||||
Should be called before each generation to ensure clean state.
|
||||
|
||||
Args:
|
||||
transformer: Transformer module to reset cache on
|
||||
"""
|
||||
if hasattr(transformer, "_hook_registry"):
|
||||
hook = transformer._hook_registry.get_hook(TeaCacheHook._HOOK_NAME)
|
||||
if hook is not None:
|
||||
transformer._hook_registry.reset_hook(TeaCacheHook._HOOK_NAME)
|
||||
logger.debug("TeaCache state reset")
|
||||
else:
|
||||
logger.warning("TeaCache hook not found, nothing to reset")
|
||||
else:
|
||||
logger.warning("Transformer has no hook registry, TeaCache may not be applied")
|
||||
|
||||
|
||||
# Register TeaCache adapter in the global registry
|
||||
register_cache_adapter(CacheType.TEA_CACHE, TeaCacheAdapter)
|
||||
-67
@@ -1,67 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Optional
|
||||
|
||||
# Model-specific polynomial coefficients for rescaling L1 distances
|
||||
# These coefficients account for model-specific characteristics in how embeddings change
|
||||
# Source: TeaCache paper and ComfyUI-TeaCache empirical tuning
|
||||
_MODEL_COEFFICIENTS = {
|
||||
# FLUX model coefficients from TeaCache paper
|
||||
"FluxPipeline": [
|
||||
4.98651651e02,
|
||||
-2.83781631e02,
|
||||
5.58554382e01,
|
||||
-3.82021401e00,
|
||||
2.64230861e-01,
|
||||
],
|
||||
# Qwen-Image model coefficients from ComfyUI-TeaCache
|
||||
# Tuned specifically for Qwen's dual-stream transformer architecture
|
||||
"QwenImagePipeline": [
|
||||
-4.50000000e02,
|
||||
2.80000000e02,
|
||||
-4.50000000e01,
|
||||
3.20000000e00,
|
||||
-2.00000000e-02,
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class TeaCacheConfig:
|
||||
"""
|
||||
Configuration for TeaCache applied to transformer models.
|
||||
|
||||
TeaCache (Timestep Embedding Aware Cache) is an adaptive caching technique that speeds up
|
||||
diffusion model inference by reusing transformer block computations when consecutive
|
||||
timestep embeddings are similar.
|
||||
|
||||
Args:
|
||||
rel_l1_thresh: Threshold for accumulated relative L1 distance. When below threshold,
|
||||
cached residual is reused. Values in [0.1, 0.3] work best:
|
||||
- 0.2: ~1.5x speedup with minimal quality loss
|
||||
- 0.4: ~1.8x speedup with slight quality loss
|
||||
- 0.6: ~2.0x speedup with noticeable quality loss
|
||||
coefficients: Polynomial coefficients for rescaling L1 distance. If None, uses
|
||||
model-specific defaults based on model_type.
|
||||
model_type: Pipeline class name (e.g., "QwenImagePipeline", "FluxPipeline").
|
||||
Must match OmniDiffusionConfig.model_class_name for proper extractor lookup.
|
||||
Defaults to "QwenImagePipeline".
|
||||
"""
|
||||
|
||||
rel_l1_thresh: float = 0.2
|
||||
coefficients: Optional[list[float]] = None
|
||||
model_type: str = "QwenImagePipeline"
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
"""Validate and set default coefficients."""
|
||||
if self.rel_l1_thresh <= 0:
|
||||
raise ValueError(f"rel_l1_thresh must be positive, got {self.rel_l1_thresh}")
|
||||
|
||||
if self.coefficients is None:
|
||||
# Use model-specific coefficients, fallback to FluxPipeline if not found
|
||||
self.coefficients = _MODEL_COEFFICIENTS.get(self.model_type, _MODEL_COEFFICIENTS["FluxPipeline"])
|
||||
|
||||
if len(self.coefficients) != 5:
|
||||
raise ValueError(f"coefficients must contain exactly 5 elements, got {len(self.coefficients)}")
|
||||
-334
@@ -1,334 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""
|
||||
Model-specific extractors for TeaCache.
|
||||
|
||||
This module provides a registry of extractor functions that know how to extract
|
||||
modulated inputs from different transformer architectures. Adding support for
|
||||
a new model requires only adding a new extractor function to the registry.
|
||||
|
||||
With Option B enhancement, extractors now return a CacheContext object containing
|
||||
all model-specific information needed for generic caching, including preprocessing,
|
||||
transformer execution, and postprocessing logic.
|
||||
"""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Callable, Union
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
|
||||
@dataclass
|
||||
class CacheContext:
|
||||
"""
|
||||
Context object containing all model-specific information for caching.
|
||||
|
||||
This allows the TeaCacheHook to remain completely generic - all model-specific
|
||||
logic is encapsulated in the extractor that returns this context.
|
||||
|
||||
Attributes:
|
||||
modulated_input: Tensor used for cache decision (similarity comparison).
|
||||
Must be a torch.Tensor extracted from the first transformer block,
|
||||
typically after applying normalization and modulation.
|
||||
|
||||
hidden_states: Current hidden states (will be modified by caching).
|
||||
Must be a torch.Tensor representing the main image/latent states
|
||||
after preprocessing but before transformer blocks.
|
||||
|
||||
encoder_hidden_states: Optional encoder states (for dual-stream models).
|
||||
Set to None for single-stream models (e.g., Flux).
|
||||
For dual-stream models (e.g., Qwen), contains text encoder outputs.
|
||||
|
||||
temb: Timestep embedding tensor.
|
||||
Must be a torch.Tensor containing the timestep conditioning.
|
||||
|
||||
run_transformer_blocks: Callable that executes model-specific transformer blocks.
|
||||
Signature: () -> tuple[torch.Tensor, ...]
|
||||
|
||||
Returns:
|
||||
tuple containing:
|
||||
- [0]: processed hidden_states (required)
|
||||
- [1]: processed encoder_hidden_states (optional, only for dual-stream)
|
||||
|
||||
Example for single-stream:
|
||||
def run_blocks():
|
||||
h = hidden_states
|
||||
for block in module.transformer_blocks:
|
||||
h = block(h, temb=temb)
|
||||
return (h,)
|
||||
|
||||
Example for dual-stream:
|
||||
def run_blocks():
|
||||
h, e = hidden_states, encoder_hidden_states
|
||||
for block in module.transformer_blocks:
|
||||
e, h = block(h, e, temb=temb)
|
||||
return (h, e)
|
||||
|
||||
postprocess: Callable that does model-specific output postprocessing.
|
||||
Signature: (torch.Tensor) -> Union[torch.Tensor, Transformer2DModelOutput, tuple]
|
||||
|
||||
Takes the processed hidden_states and applies final transformations
|
||||
(normalization, projection) to produce the model output.
|
||||
|
||||
Example:
|
||||
def postprocess(h):
|
||||
h = module.norm_out(h, temb)
|
||||
output = module.proj_out(h)
|
||||
return Transformer2DModelOutput(sample=output)
|
||||
|
||||
extra_states: Optional dict for additional model-specific state.
|
||||
Use this for models that need to pass additional context beyond
|
||||
the standard fields.
|
||||
"""
|
||||
|
||||
modulated_input: torch.Tensor
|
||||
hidden_states: torch.Tensor
|
||||
encoder_hidden_states: torch.Tensor | None
|
||||
temb: torch.Tensor
|
||||
run_transformer_blocks: Callable[[], tuple[torch.Tensor, ...]]
|
||||
postprocess: Callable[[torch.Tensor], Any]
|
||||
extra_states: dict[str, Any] | None = None
|
||||
|
||||
def validate(self) -> None:
|
||||
"""
|
||||
Validate that the CacheContext contains valid data.
|
||||
|
||||
Raises:
|
||||
TypeError: If fields have wrong types
|
||||
ValueError: If tensors have invalid properties
|
||||
RuntimeError: If callables fail basic invocation tests
|
||||
|
||||
This method should be called after creating a CacheContext to catch
|
||||
common developer errors early with clear error messages.
|
||||
"""
|
||||
# Validate tensor fields
|
||||
if not isinstance(self.modulated_input, torch.Tensor):
|
||||
raise TypeError(f"modulated_input must be torch.Tensor, got {type(self.modulated_input)}")
|
||||
|
||||
if not isinstance(self.hidden_states, torch.Tensor):
|
||||
raise TypeError(f"hidden_states must be torch.Tensor, got {type(self.hidden_states)}")
|
||||
|
||||
if self.encoder_hidden_states is not None and not isinstance(self.encoder_hidden_states, torch.Tensor):
|
||||
raise TypeError(
|
||||
f"encoder_hidden_states must be torch.Tensor or None, got {type(self.encoder_hidden_states)}"
|
||||
)
|
||||
|
||||
if not isinstance(self.temb, torch.Tensor):
|
||||
raise TypeError(f"temb must be torch.Tensor, got {type(self.temb)}")
|
||||
|
||||
# Validate callables
|
||||
if not callable(self.run_transformer_blocks):
|
||||
raise TypeError(f"run_transformer_blocks must be callable, got {type(self.run_transformer_blocks)}")
|
||||
|
||||
if not callable(self.postprocess):
|
||||
raise TypeError(f"postprocess must be callable, got {type(self.postprocess)}")
|
||||
|
||||
# Validate tensor shapes are compatible
|
||||
if self.modulated_input.shape[0] != self.hidden_states.shape[0]:
|
||||
raise ValueError(
|
||||
f"Batch size mismatch: modulated_input has batch size "
|
||||
f"{self.modulated_input.shape[0]}, but hidden_states has "
|
||||
f"{self.hidden_states.shape[0]}"
|
||||
)
|
||||
|
||||
# Validate devices match
|
||||
if self.modulated_input.device != self.hidden_states.device:
|
||||
raise ValueError(
|
||||
f"Device mismatch: modulated_input on {self.modulated_input.device}, "
|
||||
f"hidden_states on {self.hidden_states.device}"
|
||||
)
|
||||
|
||||
|
||||
def extract_qwen_context(
|
||||
module: nn.Module,
|
||||
hidden_states: torch.Tensor,
|
||||
encoder_hidden_states: torch.Tensor,
|
||||
encoder_hidden_states_mask: torch.Tensor,
|
||||
timestep: Union[torch.Tensor, float, int],
|
||||
img_shapes: torch.Tensor,
|
||||
txt_seq_lens: torch.Tensor,
|
||||
guidance: torch.Tensor | None = None,
|
||||
attention_kwargs: dict[str, Any] | None = None,
|
||||
**kwargs: Any,
|
||||
) -> CacheContext:
|
||||
"""
|
||||
Extract cache context for QwenImageTransformer2DModel.
|
||||
|
||||
This is the ONLY Qwen-specific code needed for TeaCache support.
|
||||
It encapsulates preprocessing, modulated input extraction, transformer execution,
|
||||
and postprocessing logic.
|
||||
|
||||
Args:
|
||||
module: QwenImageTransformer2DModel instance
|
||||
hidden_states: Input hidden states tensor
|
||||
encoder_hidden_states: Text encoder outputs
|
||||
encoder_hidden_states_mask: Mask for text encoder
|
||||
timestep: Current diffusion timestep
|
||||
img_shapes: Image shapes for position embedding
|
||||
txt_seq_lens: Text sequence lengths
|
||||
guidance: Optional guidance scale for CFG
|
||||
attention_kwargs: Additional attention arguments
|
||||
**kwargs: Additional keyword arguments ignored by this extractor
|
||||
|
||||
Returns:
|
||||
CacheContext with all information needed for generic caching
|
||||
"""
|
||||
from diffusers.models.modeling_outputs import Transformer2DModelOutput
|
||||
|
||||
if not hasattr(module, "transformer_blocks") or len(module.transformer_blocks) == 0:
|
||||
raise ValueError("Module must have transformer_blocks")
|
||||
|
||||
# ============================================================================
|
||||
# PREPROCESSING (Qwen-specific)
|
||||
# ============================================================================
|
||||
hidden_states = module.img_in(hidden_states)
|
||||
timestep = timestep.to(device=hidden_states.device, dtype=hidden_states.dtype)
|
||||
encoder_hidden_states = module.txt_norm(encoder_hidden_states)
|
||||
encoder_hidden_states = module.txt_in(encoder_hidden_states)
|
||||
|
||||
if guidance is not None:
|
||||
guidance = guidance.to(hidden_states.dtype) * 1000
|
||||
|
||||
temb = (
|
||||
module.time_text_embed(timestep, hidden_states)
|
||||
if guidance is None
|
||||
else module.time_text_embed(timestep, guidance, hidden_states)
|
||||
)
|
||||
|
||||
image_rotary_emb = module.pos_embed(img_shapes, txt_seq_lens, device=hidden_states.device)
|
||||
|
||||
# ============================================================================
|
||||
# EXTRACT MODULATED INPUT (for cache decision)
|
||||
# ============================================================================
|
||||
block = module.transformer_blocks[0]
|
||||
img_mod_params = block.img_mod(temb)
|
||||
img_mod1, _ = img_mod_params.chunk(2, dim=-1)
|
||||
img_normed = block.img_norm1(hidden_states)
|
||||
img_modulated, _ = block._modulate(img_normed, img_mod1)
|
||||
|
||||
# ============================================================================
|
||||
# DEFINE TRANSFORMER EXECUTION (Qwen-specific)
|
||||
# ============================================================================
|
||||
def run_transformer_blocks():
|
||||
"""Execute all Qwen transformer blocks."""
|
||||
h = hidden_states
|
||||
e = encoder_hidden_states
|
||||
for block in module.transformer_blocks:
|
||||
e, h = block(
|
||||
hidden_states=h,
|
||||
encoder_hidden_states=e,
|
||||
encoder_hidden_states_mask=encoder_hidden_states_mask,
|
||||
temb=temb,
|
||||
image_rotary_emb=image_rotary_emb,
|
||||
joint_attention_kwargs=attention_kwargs,
|
||||
)
|
||||
return (h, e)
|
||||
|
||||
# ============================================================================
|
||||
# DEFINE POSTPROCESSING (Qwen-specific)
|
||||
# ============================================================================
|
||||
return_dict = kwargs.get("return_dict", True)
|
||||
|
||||
def postprocess(h):
|
||||
"""Apply Qwen-specific output postprocessing."""
|
||||
h = module.norm_out(h, temb)
|
||||
output = module.proj_out(h)
|
||||
if not return_dict:
|
||||
return (output,)
|
||||
return Transformer2DModelOutput(sample=output)
|
||||
|
||||
# ============================================================================
|
||||
# RETURN CONTEXT
|
||||
# ============================================================================
|
||||
return CacheContext(
|
||||
modulated_input=img_modulated,
|
||||
hidden_states=hidden_states,
|
||||
encoder_hidden_states=encoder_hidden_states,
|
||||
temb=temb,
|
||||
run_transformer_blocks=run_transformer_blocks,
|
||||
postprocess=postprocess,
|
||||
)
|
||||
|
||||
|
||||
# Registry for model-specific extractors
|
||||
# Key: Pipeline class name (from OmniDiffusionConfig.model_class_name)
|
||||
# Value: extractor function with signature (module, *args, **kwargs) -> CacheContext
|
||||
#
|
||||
# Note: Use the exact pipeline class name as specified in OmniDiffusionConfig.model_class_name
|
||||
# to ensure consistent lookup without substring matching.
|
||||
EXTRACTOR_REGISTRY: dict[str, Callable] = {
|
||||
"QwenImagePipeline": extract_qwen_context,
|
||||
# Future models:
|
||||
# "FluxPipeline": extract_flux_context,
|
||||
# "CogVideoXPipeline": extract_cogvideox_context,
|
||||
}
|
||||
|
||||
|
||||
def register_extractor(model_identifier: str, extractor_fn: Callable) -> None:
|
||||
"""
|
||||
Register a new extractor function for a model type.
|
||||
|
||||
This allows extending TeaCache support to new models without modifying
|
||||
the core TeaCache code.
|
||||
|
||||
Args:
|
||||
model_identifier: Model type identifier (class name or type string)
|
||||
extractor_fn: Function with signature (module, *args, **kwargs) -> CacheContext
|
||||
|
||||
Example:
|
||||
>>> def extract_flux_context(module, hidden_states, timestep, guidance=None, **kwargs):
|
||||
... # Preprocessing
|
||||
... temb = module.time_text_embed(timestep, guidance)
|
||||
... # Extract modulated input
|
||||
... modulated = module.transformer_blocks[0].norm1(hidden_states, emb=temb)
|
||||
... # Define execution
|
||||
... def run_blocks():
|
||||
... h = hidden_states
|
||||
... for block in module.transformer_blocks:
|
||||
... h = block(h, temb=temb)
|
||||
... return (h,)
|
||||
... # Define postprocessing
|
||||
... def postprocess(h):
|
||||
... return module.proj_out(module.norm_out(h, temb))
|
||||
... # Return context
|
||||
... return CacheContext(modulated, hidden_states, None, temb, run_blocks, postprocess)
|
||||
>>> register_extractor("FluxTransformer2DModel", extract_flux_context)
|
||||
"""
|
||||
EXTRACTOR_REGISTRY[model_identifier] = extractor_fn
|
||||
|
||||
|
||||
def get_extractor(model_type: str) -> Callable:
|
||||
"""
|
||||
Get extractor function for given model type.
|
||||
|
||||
This function looks up the extractor based on the exact model_type string,
|
||||
which should match the pipeline class name from OmniDiffusionConfig.model_class_name.
|
||||
|
||||
Args:
|
||||
model_type: Model type string (e.g., "QwenImagePipeline", "FluxPipeline")
|
||||
Must exactly match a key in EXTRACTOR_REGISTRY.
|
||||
|
||||
Returns:
|
||||
Extractor function with signature (module, *args, **kwargs) -> CacheContext
|
||||
|
||||
Raises:
|
||||
ValueError: If model type not found in registry
|
||||
|
||||
Example:
|
||||
>>> # Get extractor for Qwen model
|
||||
>>> extractor = get_extractor("QwenImagePipeline")
|
||||
>>> ctx = extractor(transformer, hidden_states, encoder_hidden_states, timestep, ...)
|
||||
"""
|
||||
# Direct lookup - no substring matching
|
||||
if model_type in EXTRACTOR_REGISTRY:
|
||||
return EXTRACTOR_REGISTRY[model_type]
|
||||
|
||||
# No match found
|
||||
available_types = list(EXTRACTOR_REGISTRY.keys())
|
||||
raise ValueError(
|
||||
f"Unknown model type: '{model_type}'. "
|
||||
f"Available types: {available_types}\n"
|
||||
f"To add support for a new model, use register_extractor() or add to EXTRACTOR_REGISTRY."
|
||||
)
|
||||
-246
@@ -1,246 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""
|
||||
Hook-based TeaCache implementation for vLLM-Omni.
|
||||
|
||||
This module implements a diffusers-style hook system that completely intercepts
|
||||
the transformer forward pass, eliminating the need for any TeaCache-specific
|
||||
code in model definitions. Model developers only need to add an extractor function
|
||||
to support new models.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
from vllm_omni.diffusion.cache.teacache.config import TeaCacheConfig
|
||||
from vllm_omni.diffusion.cache.teacache.extractors import get_extractor
|
||||
from vllm_omni.diffusion.cache.teacache.state import TeaCacheState
|
||||
from vllm_omni.diffusion.hooks import HookRegistry, ModelHook, StateManager
|
||||
|
||||
|
||||
class TeaCacheHook(ModelHook):
|
||||
"""
|
||||
ModelHook implementing TeaCache for transformer models.
|
||||
|
||||
This hook completely intercepts the transformer's forward pass and implements
|
||||
adaptive caching based on timestep embedding similarity. It's model-agnostic
|
||||
and supports multiple model types through extractor functions.
|
||||
|
||||
Key features:
|
||||
- Zero changes to model code
|
||||
- CFG-aware with separate states for positive/negative branches
|
||||
- Model-specific polynomial rescaling
|
||||
- Auto-detection of model types
|
||||
|
||||
Attributes:
|
||||
config: TeaCache configuration with thresholds and callbacks
|
||||
rescale_func: Polynomial function for rescaling L1 distances
|
||||
state_manager: Manages TeaCacheState across forward passes
|
||||
extractor_fn: Model-specific function to extract modulated input
|
||||
"""
|
||||
|
||||
_HOOK_NAME = "teacache"
|
||||
|
||||
def __init__(self, config: TeaCacheConfig):
|
||||
"""
|
||||
Initialize TeaCacheHook.
|
||||
|
||||
Args:
|
||||
config: TeaCache configuration object.
|
||||
"""
|
||||
super().__init__()
|
||||
self.config = config
|
||||
self.rescale_func = np.poly1d(config.coefficients)
|
||||
self.state_manager = StateManager(TeaCacheState)
|
||||
self.extractor_fn = None
|
||||
|
||||
def initialize_hook(self, module: torch.nn.Module) -> torch.nn.Module:
|
||||
"""
|
||||
Initialize hook with extractor from config model type.
|
||||
|
||||
Args:
|
||||
module: The module to initialize the hook for.
|
||||
|
||||
Returns:
|
||||
The initialized module.
|
||||
"""
|
||||
# Get extractor function based on model_type from config
|
||||
# model_type should be the pipeline class name (e.g., "QwenImagePipeline")
|
||||
self.extractor_fn = get_extractor(self.config.model_type)
|
||||
|
||||
# Set default context
|
||||
self.state_manager.set_context("teacache")
|
||||
|
||||
return module
|
||||
|
||||
def new_forward(self, module: torch.nn.Module, *args: Any, **kwargs: Any) -> Any:
|
||||
"""
|
||||
Generic forward handler that works for ANY model.
|
||||
|
||||
This method is completely model-agnostic. All model-specific logic
|
||||
is encapsulated in the extractor function that returns a CacheContext.
|
||||
|
||||
The extractor does:
|
||||
- Model-specific preprocessing
|
||||
- Extraction of modulated input for cache decision
|
||||
- Providing transformer execution callable
|
||||
- Providing postprocessing callable
|
||||
|
||||
This hook does:
|
||||
- CFG-aware state management
|
||||
- Cache decision logic (generic)
|
||||
- Residual caching and reuse
|
||||
|
||||
Args:
|
||||
module: Transformer module (any architecture)
|
||||
*args: Positional arguments for model forward
|
||||
**kwargs: Keyword arguments for model forward
|
||||
|
||||
Returns:
|
||||
Model output (format depends on model)
|
||||
"""
|
||||
# Get model-specific context from extractor
|
||||
# The extractor encapsulates ALL model-specific logic
|
||||
ctx = self.extractor_fn(module, *args, **kwargs)
|
||||
|
||||
# ============================================================================
|
||||
# GENERIC CACHING LOGIC (works for all models)
|
||||
# ============================================================================
|
||||
# Set context based on CFG branch for separate state tracking
|
||||
cache_branch = kwargs.get("cache_branch", "default")
|
||||
context_name = f"teacache_{cache_branch}"
|
||||
self.state_manager.set_context(context_name)
|
||||
state = self.state_manager.get_state()
|
||||
|
||||
# Decide whether to compute or cache based on modulated input similarity
|
||||
should_compute = self._should_compute_full_transformer(state, ctx.modulated_input)
|
||||
|
||||
if not should_compute and state.previous_residual is not None:
|
||||
# ============================================================================
|
||||
# FAST PATH: Reuse cached residuals
|
||||
# ============================================================================
|
||||
ctx.hidden_states = ctx.hidden_states + state.previous_residual
|
||||
if state.previous_residual_encoder is not None and ctx.encoder_hidden_states is not None:
|
||||
ctx.encoder_hidden_states = ctx.encoder_hidden_states + state.previous_residual_encoder
|
||||
output = ctx.hidden_states
|
||||
else:
|
||||
# ============================================================================
|
||||
# SLOW PATH: Full transformer computation
|
||||
# ============================================================================
|
||||
ori_hidden_states = ctx.hidden_states.clone()
|
||||
ori_encoder_hidden_states = (
|
||||
ctx.encoder_hidden_states.clone() if ctx.encoder_hidden_states is not None else None
|
||||
)
|
||||
|
||||
# Run transformer blocks using model-specific callable
|
||||
outputs = ctx.run_transformer_blocks()
|
||||
|
||||
# Update context with outputs
|
||||
ctx.hidden_states = outputs[0]
|
||||
if len(outputs) > 1 and ctx.encoder_hidden_states is not None:
|
||||
ctx.encoder_hidden_states = outputs[1]
|
||||
|
||||
# Cache residuals for next timestep
|
||||
state.previous_residual = (ctx.hidden_states - ori_hidden_states).detach()
|
||||
if ori_encoder_hidden_states is not None:
|
||||
state.previous_residual_encoder = (ctx.encoder_hidden_states - ori_encoder_hidden_states).detach()
|
||||
|
||||
output = ctx.hidden_states
|
||||
|
||||
# Update state
|
||||
state.previous_modulated_input = ctx.modulated_input.detach()
|
||||
state.cnt += 1
|
||||
|
||||
# ============================================================================
|
||||
# POSTPROCESSING (model-specific, via callable)
|
||||
# ============================================================================
|
||||
return ctx.postprocess(output)
|
||||
|
||||
def _should_compute_full_transformer(self, state: TeaCacheState, modulated_inp: torch.Tensor) -> bool:
|
||||
"""
|
||||
Determine whether to compute full transformer or reuse cached residual.
|
||||
|
||||
This implements the core TeaCache algorithm:
|
||||
1. Always compute first timestep
|
||||
2. For intermediate steps:
|
||||
- Compute relative L1 distance between current and previous modulated inputs
|
||||
- Apply polynomial rescaling with model-specific coefficients
|
||||
- Accumulate rescaled distances
|
||||
- Compare to threshold: below = cache, above = compute
|
||||
|
||||
Args:
|
||||
state: Current TeaCacheState containing counters and cached values
|
||||
modulated_inp: Modulated input extracted from first transformer block
|
||||
|
||||
Returns:
|
||||
True to compute full transformer, False to reuse cached residual
|
||||
"""
|
||||
# First timestep: always compute
|
||||
if state.cnt == 0:
|
||||
state.accumulated_rel_l1_distance = 0.0
|
||||
return True
|
||||
|
||||
# Need previous input for comparison
|
||||
if state.previous_modulated_input is None:
|
||||
return True
|
||||
|
||||
# Compute relative L1 distance between consecutive modulated inputs
|
||||
rel_distance = (
|
||||
(
|
||||
(modulated_inp - state.previous_modulated_input).abs().mean()
|
||||
/ (state.previous_modulated_input.abs().mean() + 1e-8)
|
||||
)
|
||||
.cpu()
|
||||
.item()
|
||||
)
|
||||
|
||||
# Apply model-specific polynomial rescaling
|
||||
rescaled_distance = float(self.rescale_func(rel_distance))
|
||||
state.accumulated_rel_l1_distance += abs(rescaled_distance)
|
||||
|
||||
# Decision: below threshold = cache, above = compute
|
||||
if state.accumulated_rel_l1_distance < self.config.rel_l1_thresh:
|
||||
return False # Use cache
|
||||
else:
|
||||
state.accumulated_rel_l1_distance = 0.0 # Reset accumulator
|
||||
return True # Compute
|
||||
|
||||
def reset_state(self, module: torch.nn.Module) -> torch.nn.Module:
|
||||
"""
|
||||
Reset all cached states for a new inference run.
|
||||
|
||||
Args:
|
||||
module: The module to reset state for.
|
||||
|
||||
Returns:
|
||||
The module with reset state.
|
||||
"""
|
||||
self.state_manager.reset()
|
||||
return module
|
||||
|
||||
|
||||
def apply_teacache_hook(module: torch.nn.Module, config: TeaCacheConfig) -> None:
|
||||
"""
|
||||
Apply TeaCache optimization to a transformer module.
|
||||
|
||||
This function registers a TeaCacheHook that completely intercepts the
|
||||
module's forward pass, implementing adaptive caching without any changes
|
||||
to the model code.
|
||||
|
||||
Args:
|
||||
module: Transformer model to optimize (e.g., QwenImageTransformer2DModel)
|
||||
config: TeaCacheConfig specifying caching parameters
|
||||
|
||||
Example:
|
||||
>>> config = TeaCacheConfig(rel_l1_thresh=0.2, model_type="Qwen")
|
||||
>>> apply_teacache_hook(transformer, config)
|
||||
>>> # Model now uses TeaCache automatically, no code changes needed!
|
||||
"""
|
||||
registry = HookRegistry.get_or_create(module)
|
||||
hook = TeaCacheHook(config)
|
||||
registry.register_hook(TeaCacheHook._HOOK_NAME, hook)
|
||||
-40
@@ -1,40 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""
|
||||
TeaCache state management.
|
||||
|
||||
This module manages the state for TeaCache hooks across diffusion timesteps.
|
||||
"""
|
||||
|
||||
from typing import Optional
|
||||
|
||||
import torch
|
||||
|
||||
|
||||
class TeaCacheState:
|
||||
"""
|
||||
State management for TeaCache hook.
|
||||
|
||||
Tracks caching state across diffusion timesteps, managing counters,
|
||||
accumulated distances, and cached residuals for the TeaCache algorithm.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
"""Initialize empty TeaCache state."""
|
||||
# Timestep tracking
|
||||
self.cnt = 0
|
||||
|
||||
# Caching state
|
||||
self.accumulated_rel_l1_distance = 0.0
|
||||
self.previous_modulated_input: Optional[torch.Tensor] = None
|
||||
self.previous_residual: Optional[torch.Tensor] = None
|
||||
self.previous_residual_encoder: Optional[torch.Tensor] = None
|
||||
|
||||
def reset(self) -> None:
|
||||
"""Reset all state variables for a new inference run."""
|
||||
self.cnt = 0
|
||||
self.accumulated_rel_l1_distance = 0.0
|
||||
self.previous_modulated_input = None
|
||||
self.previous_residual = None
|
||||
self.previous_residual_encoder = None
|
||||
@@ -2,7 +2,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import enum
|
||||
import os
|
||||
import random
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Callable
|
||||
@@ -15,32 +14,6 @@ from vllm_omni.diffusion.utils.network_utils import is_port_available
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class TransformerConfig:
|
||||
"""Container for raw transformer configuration dictionaries."""
|
||||
|
||||
params: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: dict[str, Any]) -> "TransformerConfig":
|
||||
if not isinstance(data, dict):
|
||||
raise TypeError(f"Expected transformer config dict, got {type(data)!r}")
|
||||
return cls(params=dict(data))
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
return dict(self.params)
|
||||
|
||||
def get(self, key: str, default: Any | None = None) -> Any:
|
||||
return self.params.get(key, default)
|
||||
|
||||
def __getattr__(self, item: str) -> Any:
|
||||
params = object.__getattribute__(self, "params")
|
||||
try:
|
||||
return params[item]
|
||||
except KeyError as exc:
|
||||
raise AttributeError(item) from exc
|
||||
|
||||
|
||||
@dataclass
|
||||
class OmniDiffusionConfig:
|
||||
# Model and path configuration (for convenience)
|
||||
@@ -48,10 +21,6 @@ class OmniDiffusionConfig:
|
||||
|
||||
model_class_name: str | None = None
|
||||
|
||||
dtype: torch.dtype = torch.bfloat16
|
||||
|
||||
tf_model_config: TransformerConfig = field(default_factory=TransformerConfig)
|
||||
|
||||
# Attention
|
||||
# attention_backend: str = None
|
||||
|
||||
@@ -61,13 +30,9 @@ class OmniDiffusionConfig:
|
||||
# Workload type
|
||||
# workload_type: WorkloadType = WorkloadType.T2V
|
||||
|
||||
# Cache strategy (legacy)
|
||||
# Cache strategy
|
||||
cache_strategy: str = "none"
|
||||
|
||||
# Cache adapter configuration (NEW)
|
||||
cache_adapter: str = "none" # "tea_cache", "deep_cache", etc.
|
||||
cache_config: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
# Distributed executor backend
|
||||
distributed_executor_backend: str = "mp"
|
||||
nccl_port: int | None = None
|
||||
@@ -167,8 +132,6 @@ class OmniDiffusionConfig:
|
||||
|
||||
# MoE parameters used by Wan2.2
|
||||
boundary_ratio: float | None = None
|
||||
# Scheduler flow_shift for Wan2.2 (12.0 for 480p, 5.0 for 720p)
|
||||
flow_shift: float | None = None
|
||||
|
||||
# Logging
|
||||
log_level: str = "info"
|
||||
@@ -213,17 +176,8 @@ class OmniDiffusionConfig:
|
||||
initial_master_port = (self.master_port or 30005) + random.randint(0, 100)
|
||||
self.master_port = self.settle_port(initial_master_port, 37)
|
||||
|
||||
# Automatically inject model_class_name into cache_config if not present
|
||||
if self.cache_adapter != "none" and self.model_class_name:
|
||||
if "model_type" not in self.cache_config:
|
||||
self.cache_config["model_type"] = self.model_class_name
|
||||
logger.debug(f"Auto-injected model_type='{self.model_class_name}' into cache_config")
|
||||
|
||||
@classmethod
|
||||
def from_kwargs(cls, **kwargs: Any) -> "OmniDiffusionConfig":
|
||||
# Check environment variable as fallback for cache_adapter
|
||||
if "cache_adapter" not in kwargs:
|
||||
kwargs["cache_adapter"] = os.environ.get("DIFFUSION_CACHE_ADAPTER", "none").lower()
|
||||
return cls(**kwargs)
|
||||
|
||||
|
||||
@@ -258,7 +212,3 @@ class AttentionBackendEnum(enum.Enum):
|
||||
|
||||
def __str__(self):
|
||||
return self.name.lower()
|
||||
|
||||
|
||||
# Special message broadcast via scheduler queues to signal worker shutdown.
|
||||
SHUTDOWN_MESSAGE = {"type": "shutdown"}
|
||||
@@ -2,12 +2,11 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import multiprocessing as mp
|
||||
import time
|
||||
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from vllm_omni.diffusion.data import SHUTDOWN_MESSAGE, OmniDiffusionConfig
|
||||
from vllm_omni.diffusion.registry import get_diffusion_post_process_func, get_diffusion_pre_process_func
|
||||
from vllm_omni.diffusion.data import OmniDiffusionConfig
|
||||
from vllm_omni.diffusion.registry import get_diffusion_post_process_func
|
||||
from vllm_omni.diffusion.request import OmniDiffusionRequest
|
||||
from vllm_omni.diffusion.scheduler import scheduler
|
||||
from vllm_omni.utils.platform_utils import get_diffusion_worker_class
|
||||
@@ -27,32 +26,16 @@ class DiffusionEngine:
|
||||
self.od_config = od_config
|
||||
|
||||
self.post_process_func = get_diffusion_post_process_func(od_config)
|
||||
self.pre_process_func = get_diffusion_pre_process_func(od_config)
|
||||
|
||||
self._processes: list[mp.Process] = []
|
||||
self._closed = False
|
||||
self._make_client()
|
||||
|
||||
def step(self, requests: list[OmniDiffusionRequest]):
|
||||
try:
|
||||
# Apply pre-processing if available
|
||||
if self.pre_process_func is not None:
|
||||
preprocess_start_time = time.time()
|
||||
requests = self.pre_process_func(requests)
|
||||
preprocess_time = time.time() - preprocess_start_time
|
||||
logger.info(f"Pre-processing completed in {preprocess_time:.4f} seconds")
|
||||
|
||||
output = self.add_req_and_wait_for_response(requests)
|
||||
if output.error:
|
||||
raise Exception(f"{output.error}")
|
||||
logger.info("Generation completed successfully.")
|
||||
|
||||
postprocess_start_time = time.time()
|
||||
result = self.post_process_func(output.output)
|
||||
postprocess_time = time.time() - postprocess_start_time
|
||||
logger.info(f"Post-processing completed in {postprocess_time:.4f} seconds")
|
||||
|
||||
return result
|
||||
return self.post_process_func(output.output)
|
||||
except Exception as e:
|
||||
logger.error(f"Generation failed: {e}")
|
||||
return None
|
||||
@@ -85,8 +68,6 @@ class DiffusionEngine:
|
||||
else:
|
||||
logger.error("Failed to get result queue handle from workers")
|
||||
|
||||
self._processes = processes
|
||||
|
||||
def _launch_workers(self, broadcast_handle):
|
||||
od_config = self.od_config
|
||||
logger.info("Starting server...")
|
||||
@@ -150,32 +131,3 @@ class DiffusionEngine:
|
||||
|
||||
def add_req_and_wait_for_response(self, requests: list[OmniDiffusionRequest]):
|
||||
return scheduler.add_req(requests)
|
||||
|
||||
def close(self, *, timeout_s: float = 30.0) -> None:
|
||||
if self._closed:
|
||||
return
|
||||
self._closed = True
|
||||
|
||||
# Send shutdown signal to worker processes via broadcast queue
|
||||
try:
|
||||
if getattr(scheduler, "mq", None) is not None:
|
||||
for _ in range(self.od_config.num_gpus or 1):
|
||||
scheduler.mq.enqueue(SHUTDOWN_MESSAGE)
|
||||
except Exception as exc: # pragma: no cover - best effort cleanup
|
||||
logger.warning("Failed to send shutdown signal: %s", exc)
|
||||
|
||||
# Join all worker processes, terminate if they refuse to exit
|
||||
for proc in self._processes:
|
||||
if not proc.is_alive():
|
||||
continue
|
||||
proc.join(timeout_s)
|
||||
if proc.is_alive():
|
||||
logger.warning("Terminating diffusion worker %s after timeout", proc.name)
|
||||
proc.terminate()
|
||||
proc.join(timeout_s)
|
||||
|
||||
scheduler.close()
|
||||
self._processes = []
|
||||
|
||||
def __del__(self): # pragma: no cover - best effort cleanup
|
||||
self.close()
|
||||
Loaded 100 of 159 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user