comment

Signed-off-by: Roger Wang <hey@rogerw.io>

.buildkite/pipeline.yml

@@ -1,6 +1,87 @@
 steps:
+  - label: ":docker: Build image"
+    key: image-build
+    commands:
+      - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7"
+      - "docker build --file docker/Dockerfile.ci -t vllm-omni-ci ."
+      - "docker tag vllm-omni-ci public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT"
+      - "docker push public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT"
+    agents:
+      queue: "cpu_queue_premerge_us_east_1"
+
   - label: "Simple Unit Test"
     commands:
       - ".buildkite/scripts/simple_test.sh"
     agents:
       queue: "cpu_queue_premerge"
+
+  - label: "Diffusion Model Test"
+    timeout_in_minutes: 15
+    depends_on: image-build
+    commands:
+      - pytest -s -v tests/e2e/offline_inference/test_t2i_model.py
+    agents:
+      queue: "gpu_1_queue" # g6.4xlarge instance on AWS, has 1 L4 GPU
+    plugins:
+      - docker#v5.2.0:
+          image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
+          always-pull: true
+          propagate-environment: true
+          environment:
+            - "HF_HOME=/fsx/hf_cache"
+          volumes:
+            - "/fsx/hf_cache:/fsx/hf_cache"
+
+  - label: "Omni Model Test"
+    timeout_in_minutes: 15
+    depends_on: image-build
+    commands:
+      - export VLLM_LOGGING_LEVEL=DEBUG
+      - export VLLM_WORKER_MULTIPROC_METHOD=spawn
+      - pytest -s -v tests/e2e/offline_inference/test_qwen2_5_omni.py
+    agents:
+      queue: "gpu_4_queue" # g6.12xlarge instance on AWS, has 4 L4 GPU
+    plugins:
+      - docker#v5.2.0:
+          image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
+          always-pull: true
+          propagate-environment: true
+          environment:
+            - "HF_HOME=/fsx/hf_cache"
+          volumes:
+            - "/fsx/hf_cache:/fsx/hf_cache"
+
+  - label: "Omni Model Test with H100"
+    timeout_in_minutes: 20
+    depends_on: image-build
+    commands:
+      - export VLLM_WORKER_MULTIPROC_METHOD=spawn
+      - pytest -s -v tests/e2e/offline_inference/test_qwen3_omni.py tests/e2e/online_serving/test_qwen3_omni.py
+    agents:
+      queue: "mithril-h100-pool"
+    plugins:
+      - kubernetes:
+          podSpec:
+            containers:
+              - image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
+                resources:
+                  limits:
+                    nvidia.com/gpu: 2
+                volumeMounts:
+                  - name: devshm
+                    mountPath: /dev/shm
+                  - name: hf-cache
+                    mountPath: /root/.cache/huggingface
+                env:
+                  - name: HF_HOME
+                    value: /root/.cache/huggingface
+            nodeSelector:
+              node.kubernetes.io/instance-type: gpu-h100-sxm
+            volumes:
+              - name: devshm
+                emptyDir:
+                  medium: Memory
+              - name: hf-cache
+                hostPath:
+                  path: /mnt/hf-cache
+                  type: DirectoryOrCreate

.github/ISSUE_TEMPLATE/config.yml

@@ -1,5 +1,5 @@
 blank_issues_enabled: false
 contact_links:
   - name: Questions
-    url: https://github.com/vllm-project/vllm-omni/discussions
-    about: Ask questions and discuss with other vLLM-Omni community members
+    url: https://slack.vllm.ai
+    about: Ask questions and discuss with other vLLM-Omni community members in our #sig-omni slack channel.

.gitignore

@@ -156,15 +156,9 @@ cython_debug/
 # cursor
 .cursor/
 
-# docker
-docker/
-
 # scripts
 /scripts/
 
-# tests
-tests/
-
 # PyCharm
 #  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
 #  be added to the global gitignore or merged into this project gitignore.  For a PyCharm
@@ -185,6 +179,7 @@ checkpoints/
 
 # Cache directories
 cache/
+!vllm_omni/diffusion/cache/
 .cache/
 diffusion_cache/
 kv_cache/
@@ -235,3 +230,5 @@ configs/development.yaml
 # Docker
 .dockerignore
 Dockerfile.dev
+discussion
+tmp_test

CONTRIBUTING.md

@@ -1,3 +1,3 @@
 # Contributing to vLLM-Omni
 
-You may find information about contributing to vLLM-Omni on [Contributing](docs/contributing/)
+You may find information about contributing to vLLM-Omni on [Contributing](https://vllm-omni.readthedocs.io/en/latest/contributing/)

README.md

@@ -1,7 +1,7 @@
 <p align="center">
   <picture>
-    <source media="(prefers-color-scheme: dark)" src="./docs/source/logos/vllm-omni-logo-text-dark.png">
-    <img alt="vllm-omni" src="./docs/source/logos/vllm-omni-logo-text-dark.png" width=55%>
+    <source media="(prefers-color-scheme: dark)" src="https://raw.githubusercontent.com/vllm-project/vllm-omni/refs/heads/main/docs/source/logos/vllm-omni-logo.png">
+    <img alt="vllm-omni" src="https://raw.githubusercontent.com/vllm-project/vllm-omni/refs/heads/main/docs/source/logos/vllm-omni-logo.png" width=55%>
   </picture>
 </p>
 <h3 align="center">
@@ -9,7 +9,7 @@ Easy, fast, and cheap omni-modality model serving for everyone
 </h3>
 
 <p align="center">
-| <a href="https://vllm-omni.readthedocs.io/en/latest/"><b>Documentation</b></a> | <a href="https://tinyurl.com/vllm-omni-meeting"><b>Weekly Meeting</b></a> |
+| <a href="https://vllm-omni.readthedocs.io/en/latest/"><b>Documentation</b></a> | <a href="https://discuss.vllm.ai"><b>User Forum</b></a> | <a href="https://slack.vllm.ai"><b>Developer Slack</b></a> |
 </p>
 
 ---
@@ -30,7 +30,7 @@ Easy, fast, and cheap omni-modality model serving for everyone
 
 <p align="center">
   <picture>
-    <img alt="vllm-omni" src="./docs/source/architecture/omni-modality model architecture.png" width=55%>
+    <img alt="vllm-omni" src="https://raw.githubusercontent.com/vllm-project/vllm-omni/refs/heads/main/docs/source/architecture/omni-modality-model-architecture.png" width=55%>
   </picture>
 </p>
 
@@ -57,8 +57,6 @@ vLLM-Omni seamlessly supports most popular open-source models on HuggingFace, in
 
 Visit our [documentation](https://vllm-omni.readthedocs.io/en/latest/) to learn more.
 
-Visit our documentation to learn more.
-
 - [Installation](https://vllm-omni.readthedocs.io/en/latest/getting_started/installation/)
 - [Quickstart](https://vllm-omni.readthedocs.io/en/latest/getting_started/quickstart/)
 - [List of Supported Models](https://vllm-omni.readthedocs.io/en/latest/models/supported_models/)
@@ -68,10 +66,8 @@ Visit our documentation to learn more.
 We welcome and value any contributions and collaborations.
 Please check out [Contributing to vLLM-Omni](https://vllm-omni.readthedocs.io/en/latest/contributing/) for how to get involved.
 
-## Weekly Meeting
-
-- vLLM Omni Weekly Meeting: https://tinyurl.com/vllm-omni-meeting
-- Wednesday, 11:30 - 12:30 (UTC+8, [Convert to your timezone](https://dateful.com/convert/gmt8?t=15))
+## Join the Community
+Feel free to ask questions, provide feedbacks and discuss with fellow users of vLLM-Omni in `#sig-omni` slack channel at [slack.vllm.ai](https://slack.vllm.ai) or vLLM user forum at [discuss.vllm.ai](https://discuss.vllm.ai).
 
 ## License
 

docker/Dockerfile.ci

@@ -0,0 +1,14 @@
+ARG VLLM_BASE_IMAGE=vllm/vllm-openai
+ARG VLLM_BASE_TAG=v0.11.0
+FROM ${VLLM_BASE_IMAGE}:${VLLM_BASE_TAG}
+ARG APP_DIR=/workspace/vllm-omni
+WORKDIR ${APP_DIR}
+
+COPY . .
+
+# Install vllm-omni into the same uv-managed Python environment used by the base image.
+RUN uv pip install --python "$(python3 -c 'import sys; print(sys.executable)')" --no-cache-dir ".[dev]"
+
+RUN ln -sf /usr/bin/python3 /usr/bin/python
+
+ENTRYPOINT []

docs/.nav.yml

@@ -1,44 +1,55 @@
 nav:
-  - Home: README.md
-  - User Guide:
-    - Getting Started:
-      - getting_started/quickstart.md
-      - getting_started/installation
-    - Examples:
-      - examples/README.md
-      - Offline Inference:
-        - Qwen2.5-Omni: user_guide/examples/offline_inference/qwen2_5_omni.md
-      - Online Serving:
-        - Qwen2.5-Omni: user_guide/examples/online_serving/qwen2_5_omni.md
-        - Qwen2.5-Image: user_guide/examples/offline_inference/qwen_image.md
-    - General:
-      - usage/*
-    - Configuration:
-      - configuration/README.md
-      - configuration/*
-    - Models:
-      - models/supported_models.md
-  - Developer Guide:
-    - General:
-      - contributing/README.md
-      - glob: contributing/*
-        flatten_single_child_sections: true
-    - Model Implementation:
-      - contributing/model/README.md
-    - CI: contributing/ci
-    - Design Documents:
-      - design/index.md
-      - design/architecture_overview.md
-      - design/vllm_omni_design.md
-      - design/mrs_design.md
-      - design/api_design_doc.md
-    - Docs Guide: contributing/DOCS_GUIDE.md
-  - API Reference:
-    - api/README.md
-    - api/vllm_omni
-  - CLI Reference: cli
-  - Community:
-    - community/*
-    - Slack: https://slack.vllm.ai
-    - Blog: https://blog.vllm.ai
-    - Forum: https://discuss.vllm.ai
+- Home: README.md
+- User Guide:
+  - Getting Started:
+    - getting_started/quickstart.md
+    - getting_started/installation
+  - Examples:
+    - examples/README.md
+    - Offline Inference:
+      - Qwen2.5-Omni: user_guide/examples/offline_inference/qwen2_5_omni.md
+      - Qwen3-Omni: user_guide/examples/offline_inference/qwen3_omni.md
+      - Text-To-Image: user_guide/examples/offline_inference/text_to_image.md
+      - Image-To-Image: user_guide/examples/offline_inference/image_to_image.md
+      - Text-To-Video: user_guide/examples/offline_inference/text_to_video.md
+    - Online Serving:
+      - Qwen2.5-Omni: user_guide/examples/online_serving/qwen2_5_omni.md
+      - Qwen3-Omni: user_guide/examples/online_serving/qwen3_omni.md
+  - General:
+    - usage/*
+  - Configuration:
+    - configuration/README.md
+    - configuration/*
+  - Diffusion Acceleration:
+     - TeaCache: user_guide/teacache.md
+  - Models:
+    - models/supported_models.md
+- Developer Guide:
+  - General:
+    - contributing/README.md
+    - glob: contributing/*
+      flatten_single_child_sections: true
+  - Model Implementation:
+    - contributing/model/README.md
+  - CI: contributing/ci
+  - Design Documents:
+    - design/index.md
+    - design/architecture_overview.md
+    - Feature Design:
+      - design/feature/multi_request_streaming_design.md
+      - design/feature/omni_connector_design.md
+      - design/feature/ray_based_execution.md
+    - Module Design:
+      - design/module/ar_module_design.md
+      - design/module/dit_module_design.md
+      - design/module/entrypoint_module_design.md
+  - Docs Guide: contributing/DOCS_GUIDE.md
+- API Reference:
+  - api/README.md
+  - api/vllm_omni
+- CLI Reference: cli
+- Community:
+  - community/*
+  - Slack: https://slack.vllm.ai
+  - Blog: https://blog.vllm.ai
+  - Forum: https://discuss.vllm.ai

docs/README.md

@@ -8,8 +8,8 @@ hide:
 
 <p align="center">
   <picture>
-    <source media="(prefers-color-scheme: dark)" src="./source/logos/vllm-omni-logo-text-dark.png">
-    <img alt="vllm-omni" src="./source/logos/vllm-omni-logo-text-dark.png" width=55%>
+    <source media="(prefers-color-scheme: dark)" src="./source/logos/vllm-omni-logo.png">
+    <img alt="vllm-omni" src="./source/logos/vllm-omni-logo.png" width=55%>
   </picture>
 </p>
 <h3 align="center">
@@ -34,7 +34,8 @@ Easy, fast, and cheap omni-modality model serving for everyone
 
 <p align="center">
   <picture>
-    <img alt="vllm-omni" src="./source/architecture/omni-modality model architecture.png" width=55%>
+    <source media="(prefers-color-scheme: dark)" src="./source/architecture/omni-modality-model-architecture.png">
+    <img alt="vllm-omni-arch" src="./source/architecture/omni-modality-model-architecture.png" width=55%>
   </picture>
 </p>
 
@@ -58,4 +59,6 @@ vLLM-Omni seamlessly supports most popular open-source models on HuggingFace, in
 - Multi-modality generation models (e.g. Qwen-Image)
 
 For more information, checkout the following:
-- [Placeholder for vllm-omni blogpost]()
+
+- [vllm-omni architecture design and recent roadmaps](https://docs.google.com/presentation/d/1Y7t2Zm3BIISPN-_X_sDpxZMNCWir2rAx/edit?usp=drive_link&ouid=110473603432222024453&rtpof=true&sd=true)
+- [vllm-omni announcement blogpost](https://blog.vllm.ai/2025/11/30/vllm-omni.html)

docs/api/README.md

@@ -4,8 +4,8 @@
 
 Main entry points for vLLM-Omni inference and serving.
 
-- [vllm_omni.entrypoints.async_omni_llm.AsyncOmniLLM][]
-- [vllm_omni.entrypoints.async_omni_llm.AsyncOmniStageLLM][]
+- [vllm_omni.entrypoints.async_omni.AsyncOmni][]
+- [vllm_omni.entrypoints.async_omni.AsyncOmniStageLLM][]
 - [vllm_omni.entrypoints.chat_utils.OmniAsyncMultiModalContentParser][]
 - [vllm_omni.entrypoints.chat_utils.OmniAsyncMultiModalItemTracker][]
 - [vllm_omni.entrypoints.chat_utils.parse_chat_messages_futures][]
@@ -34,6 +34,8 @@ Engine classes for offline and online inference.
 - [vllm_omni.diffusion.diffusion_engine.DiffusionEngine][]
 - [vllm_omni.engine.AdditionalInformationEntry][]
 - [vllm_omni.engine.AdditionalInformationPayload][]
+- [vllm_omni.engine.OmniEngineCoreOutput][]
+- [vllm_omni.engine.OmniEngineCoreOutputs][]
 - [vllm_omni.engine.OmniEngineCoreRequest][]
 - [vllm_omni.engine.PromptEmbedsPayload][]
 - [vllm_omni.engine.arg_utils.AsyncOmniEngineArgs][]
@@ -47,15 +49,15 @@ Engine classes for offline and online inference.
 Core scheduling and caching components.
 
 - [vllm_omni.core.dit_cache_manager.DiTCacheManager][]
-- [vllm_omni.core.sched.diffusion_scheduler.DiffusionScheduler][]
+- [vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler][]
+- [vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler][]
 - [vllm_omni.core.sched.output.OmniNewRequestData][]
-- [vllm_omni.core.sched.scheduler.OmniScheduler][]
 
 ## Model Executor
 
 Model execution components.
 
-- [vllm_omni.model_executor.models.qwen2_5_omni.qwen2_5_omni.OmniOutput][]
+- [vllm_omni.model_executor.models.output_templates.OmniOutput][]
 - [vllm_omni.model_executor.models.qwen2_5_omni.qwen2_5_omni.Qwen2_5OmniForConditionalGeneration][]
 - [vllm_omni.model_executor.models.qwen2_5_omni.qwen2_5_omni_talker.Qwen2_5OmniTalkerForConditionalGeneration][]
 - [vllm_omni.model_executor.models.qwen2_5_omni.qwen2_5_omni_thinker.Qwen2_5OmniAudioFeatureInputs][]
@@ -72,12 +74,29 @@ Model execution components.
 - [vllm_omni.model_executor.models.qwen2_5_omni.qwen2_old.Qwen2EmbeddingModel][]
 - [vllm_omni.model_executor.models.qwen2_5_omni.qwen2_old.Qwen2ForCausalLM][]
 - [vllm_omni.model_executor.models.qwen2_5_omni.qwen2_old.Qwen2Model][]
+- [vllm_omni.model_executor.models.qwen3_omni.qwen3_moe.Qwen3MoeForCausalLM][]
+- [vllm_omni.model_executor.models.qwen3_omni.qwen3_omni.Qwen3OmniMoeForConditionalGeneration][]
+- [vllm_omni.model_executor.models.qwen3_omni.qwen3_omni_code2wav.Qwen3OmniMoeCode2Wav][]
+- [vllm_omni.model_executor.models.qwen3_omni.qwen3_omni_moe_code_predictor_mtp.Qwen3OmniCodePredictorBaseModel][]
+- [vllm_omni.model_executor.models.qwen3_omni.qwen3_omni_moe_code_predictor_mtp.Qwen3OmniMoeTalkerCodePredictor][]
+- [vllm_omni.model_executor.models.qwen3_omni.qwen3_omni_moe_talker.Qwen3OmniMoeModel][]
+- [vllm_omni.model_executor.models.qwen3_omni.qwen3_omni_moe_talker.Qwen3OmniMoeTalkerForConditionalGeneration][]
+- [vllm_omni.model_executor.models.qwen3_omni.qwen3_omni_moe_thinker.Qwen3MoeLLMForCausalLM][]
+- [vllm_omni.model_executor.models.qwen3_omni.qwen3_omni_moe_thinker.Qwen3MoeLLMModel][]
+- [vllm_omni.model_executor.models.qwen3_omni.qwen3_omni_moe_thinker.Qwen3OmniMoeConditionalGenerationMixin][]
+- [vllm_omni.model_executor.models.qwen3_omni.qwen3_omni_moe_thinker.Qwen3OmniMoeThinkerForConditionalGeneration][]
+- [vllm_omni.model_executor.models.qwen3_omni.qwen3_omni_moe_thinker.Qwen3OmniMoeThinkerMultiModalProcessor][]
+- [vllm_omni.model_executor.models.qwen3_omni.qwen3_omni_moe_thinker.Qwen3OmniMoeThinkerProcessingInfo][]
+- [vllm_omni.model_executor.models.qwen3_omni.qwen3_omni_moe_thinker.Qwen3Omni_VisionTransformer][]
+- [vllm_omni.model_executor.models.qwen3_omni.qwen3_omni_moe_thinker.Qwen3_VisionPatchEmbed][]
+- [vllm_omni.model_executor.models.qwen3_omni.qwen3_omni_moe_thinker.Qwen3_VisionPatchMerger][]
 
 ## Configuration
 
 Configuration classes.
 
 - [vllm_omni.config.model.OmniModelConfig][]
+- [vllm_omni.diffusion.cache.teacache.config.TeaCacheConfig][]
 
 ## Workers
 
@@ -85,8 +104,15 @@ Worker classes and model runners for distributed inference.
 
 - [vllm_omni.diffusion.worker.gpu_worker.GPUWorker][]
 - [vllm_omni.diffusion.worker.gpu_worker.WorkerProc][]
+- [vllm_omni.diffusion.worker.npu.npu_worker.NPUWorker][]
+- [vllm_omni.diffusion.worker.npu.npu_worker.NPUWorkerProc][]
 - [vllm_omni.worker.gpu_ar_model_runner.GPUARModelRunner][]
 - [vllm_omni.worker.gpu_ar_worker.GPUARWorker][]
-- [vllm_omni.worker.gpu_diffusion_model_runner.GPUDiffusionModelRunner][]
-- [vllm_omni.worker.gpu_diffusion_worker.GPUDiffusionWorker][]
+- [vllm_omni.worker.gpu_generation_model_runner.GPUGenerationModelRunner][]
+- [vllm_omni.worker.gpu_generation_worker.GPUGenerationWorker][]
 - [vllm_omni.worker.gpu_model_runner.OmniGPUModelRunner][]
+- [vllm_omni.worker.npu.npu_ar_model_runner.NPUARModelRunner][]
+- [vllm_omni.worker.npu.npu_ar_worker.NPUARWorker][]
+- [vllm_omni.worker.npu.npu_generation_model_runner.NPUGenerationModelRunner][]
+- [vllm_omni.worker.npu.npu_generation_worker.NPUGenerationWorker][]
+- [vllm_omni.worker.npu.npu_model_runner.OmniNPUModelRunner][]

docs/community/contact_us.md

@@ -2,4 +2,4 @@
 
 - For technical questions and feature requests, please use GitHub [Issues](https://github.com/vllm-project/vllm-omni/issues)
 - For coordinating contributions and development and discussing with other users and developers, please join `sig-omni` channel in our [Slack](https://slack.vllm.ai/) or use the [vLLM Forum](https://discuss.vllm.ai/)
-- For security disclosures, please use GitHub's [Security Advisories](https://github.com/vllm-project/vllm/security/advisories) feature
+- For security disclosures, please use GitHub's [Security Advisories](https://github.com/vllm-project/vllm-omni/security/advisories) feature

docs/community/volunteers.md

@@ -0,0 +1,12 @@
+# Volunteers for Bugfix and CI
+
+We encourage you to check current docs and [issues](https://github.com/vllm-project/vllm-omni/issues) to find possible solutions for your questions. If non of these can solve it, please propose an issue to describe your questions about bug or CI problems for developing.
+
+If you have urgent need for locating and solving bugfix or CI problems, please find community volunteers below.
+
+| Dec 4-Dec 12 | Dec 15-Dec 19 | Dec 22-Dec 26 | Dec 29- Jan 2, 2026| Jan 5-Jan 9 | Jan 12-Jan 16 |
+|----------|----------|----------|----------|----------|----------|
+| <a href="https://github.com/congw729">Conw729</a> | <a href="https://github.com/yinpeiqi">yinpeiqi</a> | <a href="https://github.com/tzhouam">tzhouam</a> | <a href="https://github.com/SamitHuang">SamitHuang</a> | <a href="https://github.com/gcanlin">gcanlin</a> | <a href="https://github.com/natureofnature">natureofnature</a> |
+| <a href="https://github.com/david6666666">david6666666</a> | <a href="https://github.com/R2-Y">R2-Y</a> | <a href="https://github.com/hsliuustc0106">hsliuustc0106</a> | <a href="https://github.com/Gaohan123">Gaohan123</a> | <a href="https://github.com/ZJY0516">ZJY0516</a> | <a href="https://github.com/qibaoyuan">qibaoyuan</a> |
+
+We kindly welcome more contributors to fix bugs and contribute new features!

docs/configuration/README.md

@@ -9,3 +9,7 @@ Currently, the main options are maintained by stage configs for each model.
 For specific example, please refer to [Qwen2.5-omni stage config](stage_configs/qwen2_5_omni.yaml)
 
 For introduction, please check [Introduction for stage config](./stage_configs.md)
+
+## Optimization Features
+
+- **[TeaCache Configuration](../user_guide/teacache.md)** - Enable TeaCache adaptive caching for DiT models to achieve 1.5x-2.0x speedup with minimal quality loss

docs/configuration/stage_configs.md

@@ -2,6 +2,9 @@
 
 In vLLM-Omni, the target model is separated into multiple stages, which are processed by different LLMEngines, DiffusionEngines or other types of engines. Depending on different types of stages, such as Autoregressive (AR) stage or Diffusion transformer (DiT) stage, each can choose corresponding schedulers, model workers to load with the Engines in a plug-in fashion.
 
+!!! note
+    Default stage config YAMLs (for example, `vllm_omni/model_executor/stage_configs/qwen2_5_omni.yaml` and `vllm_omni/model_executor/stage_configs/qwen3_omni_moe.yaml`) are bundled and loaded automatically when `stage_configs_path` is not provided. They have been verified to work on 1xH100 for Qwen2.5-Omni and 2xH100 for Qwen3-Omni.
+
 Therefore, as a core part of vLLM-Omni, the stage configs for a model have several main functions:
 
 - Claim partition of stages and their corresponding class implementation in `model_executor/models`.
@@ -44,7 +47,7 @@ stage_args:
       trust_remote_code: true # Needed by huggingface config parsing
       engine_output_type: latent  # It claims that the stage will input latent hiddenstates besides token ids
       enable_prefix_caching: false # For request with hiddenstates output, the prefix caching is not supported now
-    is_comprehension: true # If the stage is a text or multimodal comprehension module. If it is, the AsyncOmniLLM will use its tokenizer as default
+    is_comprehension: true # If the stage is a text or multimodal comprehension module. If it is, the AsyncOmni will use its tokenizer as default
     final_output: true # If the stage has output as part of final outputs. If it is false, which means that the stage only works as a intermediate role.
     final_output_type: text # What is the final output type. It can be text and audio now.
     default_sampling_params: # sampling parameters for the stage. Their meaning aligns with vLLM.
@@ -206,7 +209,7 @@ Default: `false`
 
 ### `is_comprehension`
 
-Whether this stage is a text or multimodal comprehension module. When set to `true`, the stage acts as a comprehension module that processes input text or multimodal content. If this is the first comprehension stage, `AsyncOmniLLM` will use its tokenizer as the default tokenizer for the entire pipeline.
+Whether this stage is a text or multimodal comprehension module. When set to `true`, the stage acts as a comprehension module that processes input text or multimodal content. If this is the first comprehension stage, `AsyncOmni` will use its tokenizer as the default tokenizer for the entire pipeline.
 
 Default: `true`
 

docs/configuration/stage_configs/qwen2_5_omni.yaml

@@ -9,7 +9,7 @@ stage_args:
       model_stage: thinker
       model_arch: Qwen2_5OmniForConditionalGeneration
       worker_cls: vllm_omni.worker.gpu_ar_worker.GPUARWorker
-      scheduler_cls: vllm_omni.core.sched.scheduler.OmniScheduler
+      scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
       gpu_memory_utilization: 0.8
       enforce_eager: true  # Now we only support eager mode
       trust_remote_code: true
@@ -35,7 +35,7 @@ stage_args:
       model_stage: talker
       model_arch: Qwen2_5OmniForConditionalGeneration
       worker_cls: vllm_omni.worker.gpu_ar_worker.GPUARWorker
-      scheduler_cls: vllm_omni.core.sched.scheduler.OmniScheduler
+      scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
       gpu_memory_utilization: 0.8
       enforce_eager: true
       trust_remote_code: true

docs/contributing/DOCS_GUIDE.md

@@ -9,7 +9,7 @@ This directory contains the source files for the vLLM-Omni documentation.
 Install documentation dependencies:
 
 ```bash
-pip install -e ".[docs]"
+uv pip install -e ".[docs]"
 ```
 
 ### Build and Serve Documentation
@@ -84,7 +84,7 @@ The documentation is automatically deployed to GitHub Pages using GitHub Actions
    ```
 
 3. **Documentation will be available at**:
-   - `https://vllm-project.github.io/vllm-omni/`
+   - `https://vllm-omni.readthedocs.io`
 
 The GitHub Actions workflow (`.github/workflows/docs.yml`) will automatically:
 - Build the documentation when you push to `main` branch

docs/contributing/README.md

@@ -2,6 +2,9 @@
 
 Thank you for your interest in contributing to vLLM-Omni! This document provides guidelines and instructions for contributing.
 
+!!! note
+    We host weekly developer-facing online meetings to discuss milestones and updates **every Tuesday at 19:30 PDT**. Meeting link as well as the past meeting notes can be found [here](https://tinyurl.com/vllm-omni-meeting).
+
 ## Getting Started
 
 vLLM-Omni uses `uv` as the environment manager, to create and manage Python environments. Please follow the documentation to install `uv`. After installing `uv`, you can create a new Python environment using the following commands:
@@ -20,7 +23,7 @@ Install vLLM-Omni from source with development dependencies:
 
 ```bash
 git clone https://github.com/vllm-project/vllm-omni.git
-cd vllm_omni
+cd vllm-omni
 uv pip install -e ".[dev]"
 ```
 

docs/design/api_design_doc.md

@@ -1,143 +0,0 @@
-# API Design Documentation
-
-This document contains comprehensive API design documentation for all core modules in vLLM-Omni. These templates provide a standardized structure for designing and implementing the core, engine, executor, and worker modules.
-
-## 📋 Module API
-
-### Core Module API
-**Core module** provides fundamental scheduling, caching, and resource management functionality.
-
-**Key Components:**
-- Request scheduling and prioritization
-- DiT cache management for diffusion models
-- Resource allocation and coordination
-- Inter-module communication
-
-### Engine Module API
-**Engine module** handles model loading, inference execution, and output processing.
-
-**Key Components:**
-- Model loading and initialization
-- Inference execution for AR and diffusion models
-- Input preprocessing for various modalities
-- Output postprocessing and formatting
-
-### Executor Module API
-**Executor module** coordinates and manages request execution across different engines and workers.
-
-**Key Components:**
-- Request routing and load balancing
-- Execution pipeline coordination
-- Worker management and task distribution
-- Error handling and recovery
-
-### Worker Module API
-**Worker module** provides the actual execution environment for model inference.
-
-**Key Components:**
-- Model execution and inference
-- GPU resource management
-- Request batching and processing
-- Performance optimization
-
-
-## 🔧 Implementation Guidelines
-
-### Code Standards
-- Use type hints for all methods
-- Include comprehensive docstrings
-- Follow PEP 8 style guidelines
-- Use async/await for I/O operations
-
-### Error Handling
-- Define specific exception types
-- Provide meaningful error messages
-- Include error recovery strategies
-- Log errors appropriately
-
-### Configuration
-- Use dataclasses for configuration
-- Provide sensible defaults
-- Include validation methods
-- Support environment variables
-
-### Testing
-- Write unit tests for all public methods
-- Include integration tests
-- Test error conditions
-- Validate configuration options
-
-## 🚀 Getting Started
-
-### For Developers
-
-1. **Choose a Module**
-   - Pick the module you want to work on
-   - Read the corresponding API documentation
-   - Understand the responsibilities and interfaces
-
-2. **Set Up Development Environment**
-   ```bash
-   # Clone the repository
-   git clone https://github.com/vllm-project/vllm-omni.git
-   cd vllm-omni
-
-   # Install dependencies
-   pip install -e ".[dev]"
-
-   # Set up pre-commit hooks
-   pre-commit install
-   ```
-
-3. **Start Implementation**
-   - Create the module directory
-   - Implement base classes first
-   - Add concrete implementations
-   - Write tests as you go
-
-### For Contributors
-
-1. **Review API Documentation**
-   - Read through all module APIs
-   - Understand the overall architecture
-   - Identify areas for improvement
-
-2. **Propose Changes**
-   - Create issues for API changes
-   - Discuss in pull requests
-   - Update documentation accordingly
-
-3. **Submit Contributions**
-   - Follow the coding standards
-   - Include tests for new features
-   - Update documentation
-   - Submit pull requests
-
-## 🤝 Contributing
-
-We welcome contributions to improve the API documentation and implementation. Please:
-
-1. **Follow the Template Structure**
-   - Use the [api template](https://github.com/vllm-project/vllm-omni/docs/contributing/design_documents/api_design_template.md) as a guide
-   - Maintain consistency across modules
-   - Include all required sections
-
-2. **Provide Working Examples**
-   - Test all code examples
-   - Include both basic and advanced usage
-   - Show integration patterns
-
-3. **Keep Documentation Updated**
-   - Update docs when APIs change
-   - Version control documentation changes
-   - Maintain backward compatibility notes
-
-## 📝 Notes
-
-- All API documentation should be kept up-to-date with implementation
-- Code examples should be tested and working
-- Configuration options should be validated
-- Error handling should be comprehensive
-- Performance considerations should be documented
-
-For questions or suggestions about the API documentation, please open an issue or start a discussion in the repository.

docs/design/api_design_template.md

@@ -1,336 +0,0 @@
-# API Design Template for vLLM-Omni Modules
-
-This template provides a standardized structure for designing APIs for core, engine, executor, and worker modules in vLLM-Omni. Use this template to ensure consistency and completeness across all modules.
-
-## Template Structure
-
-### 1. Module Overview
-- **Purpose**: What this module does
-- **Responsibilities**: Key responsibilities of the module
-- **Dependencies**: What other modules this depends on
-- **Integration Points**: How it integrates with other modules
-
-### 2. Core Classes/Interfaces
-- **Base Classes**: Abstract base classes and interfaces
-- **Implementation Classes**: Concrete implementations
-- **Data Structures**: Key data structures and models
-
-### 3. Public API Methods
-- **Initialization**: Constructor and setup methods
-- **Core Operations**: Main functionality methods
-- **Configuration**: Configuration and parameter methods
-- **Lifecycle Management**: Start, stop, cleanup methods
-- **Monitoring**: Status, metrics, and debugging methods
-
-### 4. Configuration
-- **Configuration Classes**: Dataclasses or config objects
-- **Required Parameters**: Must-have configuration
-- **Optional Parameters**: Optional configuration with defaults
-- **Validation**: Parameter validation rules
-
-### 5. Error Handling
-- **Custom Exceptions**: Module-specific exceptions
-- **Error Codes**: Standardized error codes
-- **Recovery Strategies**: How to handle and recover from errors
-
-### 6. Examples
-- **Basic Usage**: Simple usage examples
-- **Advanced Usage**: Complex scenarios
-- **Integration Examples**: How to use with other modules
-
----
-
-## Example: Core Module Template
-
-### 1. Module Overview
-
-**Purpose**: The core module provides fundamental scheduling and caching functionality for vLLM-Omni.
-
-**Responsibilities**:
-- Request scheduling and prioritization
-- DiT cache management
-- Resource allocation and coordination
-- Inter-module communication
-
-**Dependencies**:
-- `vllm_omni.request` - Request handling
-- `vllm_omni.config` - Configuration management
-- `vllm_omni.utils` - Utility functions
-
-**Integration Points**:
-- Receives requests from entrypoints
-- Coordinates with engine modules
-- Manages worker allocation
-- Provides status to monitoring systems
-
-### 2. Core Classes/Interfaces
-
-```python
-from abc import ABC, abstractmethod
-from typing import List, Optional, Dict, Any
-from dataclasses import dataclass
-from enum import Enum
-
-class DiTCacheManager:
-    """Manages DiT cache for diffusion models."""
-
-    def __init__(self, config: DiTCacheConfig):
-        self.config = config
-        self.cache = {}
-        self.cache_stats = {}
-
-    async def get_cache(self, cache_key: str) -> Optional[Any]:
-        """Retrieve cached data."""
-        pass
-
-    async def set_cache(self, cache_key: str, data: Any) -> None:
-        """Store data in cache."""
-        pass
-
-    async def invalidate_cache(self, cache_key: str) -> None:
-        """Invalidate cached data."""
-        pass
-```
-
-### 3. Public API Methods
-
-#### Initialization
-```python
-class OmniScheduler():
-    def __init__(self, config: SchedulerConfig):
-        """Initialize the core scheduler."""
-        self.config = config
-        self.scheduler = self._create_scheduler()
-        self.cache_manager = DiTCacheManager(config.dit_cache_config)
-        self._running = False
-
-    def _create_scheduler(self) -> BaseScheduler:
-        """Factory method to create appropriate scheduler."""
-        if self.config.scheduler_type == SchedulerType.FIFO:
-            return FIFOScheduler(self.config)
-        elif self.config.scheduler_type == SchedulerType.PRIORITY:
-            return PriorityScheduler(self.config)
-        # ... other scheduler types
-```
-
-#### Core Operations
-```python
-    async def schedule(self, request: OmniRequest) -> bool:
-        """
-        Schedule a request for processing.
-
-        Args:
-            request: The request to schedule
-
-        Returns:
-            bool: True if successfully scheduled, False otherwise
-
-        Raises:
-            SchedulerError: If scheduling fails
-            QueueFullError: If queue is at capacity
-        """
-        pass
-```
-
-#### Configuration
-```python
-    def update_config(self, new_config: SchedulerConfig) -> None:
-        """Update scheduler configuration."""
-        pass
-
-    def get_config(self) -> SchedulerConfig:
-        """Get current configuration."""
-        pass
-```
-
-#### Lifecycle Management
-```python
-    async def start(self) -> None:
-        """Start the scheduler."""
-        self._running = True
-        # Start background tasks
-
-    async def stop(self) -> None:
-        """Stop the scheduler gracefully."""
-        self._running = False
-        # Cleanup resources
-
-    async def shutdown(self) -> None:
-        """Force shutdown the scheduler."""
-        # Immediate cleanup
-```
-
-#### Monitoring
-```python
-    def get_status(self) -> Dict[str, Any]:
-        """Get current scheduler status."""
-        return {
-            "running": self._running,
-            "queue_size": self.scheduler.queue_size(),
-            "processed_requests": self.scheduler.processed_count(),
-            "cache_hit_rate": self.cache_manager.get_hit_rate()
-        }
-
-    def get_metrics(self) -> Dict[str, float]:
-        """Get performance metrics."""
-        pass
-```
-
-### 4. Configuration
-
-```python
-@dataclass
-class OmniConfig:
-    """Configuration for the core module."""
-
-    # Scheduler configuration
-    scheduler: SchedulerConfig = field(default_factory=SchedulerConfig)
-
-    # Cache configuration
-    dit_cache: DiTCacheConfig = field(default_factory=DiTCacheConfig)
-
-    # Resource limits
-    max_memory_gb: float = 16.0
-    max_gpu_utilization: float = 0.8
-
-    # Timeouts
-    request_timeout: int = 300
-    worker_timeout: int = 60
-
-    def validate(self) -> None:
-        """Validate configuration parameters."""
-        if self.max_memory_gb <= 0:
-            raise ValueError("max_memory_gb must be positive")
-        if not 0 < self.max_gpu_utilization <= 1:
-            raise ValueError("max_gpu_utilization must be between 0 and 1")
-```
-
-### 5. Error Handling
-
-```python
-class CoreModuleError(Exception):
-    """Base exception for core module errors."""
-    pass
-
-class SchedulerError(CoreModuleError):
-    """Scheduler-related errors."""
-    pass
-
-class QueueFullError(SchedulerError):
-    """Queue is at capacity."""
-    pass
-
-class CacheError(CoreModuleError):
-    """Cache-related errors."""
-    pass
-
-class ResourceError(CoreModuleError):
-    """Resource allocation errors."""
-    pass
-```
-
-### 6. Examples
-
-#### Basic Usage
-```python
-from vllm_omni.core import CoreScheduler, CoreModuleConfig
-from vllm_omni.request import create_text_request
-
-# Create configuration
-config = CoreModuleConfig(
-    scheduler=SchedulerConfig(scheduler_type=SchedulerType.FIFO),
-    max_memory_gb=8.0
-)
-
-# Initialize scheduler
-scheduler = CoreScheduler(config)
-await scheduler.start()
-
-# Create and schedule a request
-request = create_text_request(
-    request_id="req_001",
-    prompt="Hello, world!",
-    sampling_params=sampling_params
-)
-
-success = await scheduler.schedule_request(request)
-if success:
-    result = await scheduler.process_request(request)
-    print(f"Result: {result}")
-```
-
-#### Advanced Usage
-```python
-# Custom scheduler with priority handling
-config = CoreModuleConfig(
-    scheduler=SchedulerConfig(
-        scheduler_type=SchedulerType.PRIORITY,
-        priority_weights={"high": 2.0, "normal": 1.0, "low": 0.5}
-    )
-)
-
-scheduler = CoreScheduler(config)
-await scheduler.start()
-
-# Monitor scheduler status
-status = scheduler.get_status()
-print(f"Queue size: {status['queue_size']}")
-print(f"Cache hit rate: {status['cache_hit_rate']:.2%}")
-```
-
----
-
-## Module-Specific Guidelines
-
-### Core Module
-- Focus on scheduling, caching, and resource management
-- Provide clear interfaces for other modules
-- Handle concurrency and thread safety
-- Implement comprehensive monitoring
-
-### Engine Module
-- Handle model loading and inference
-- Support both AR and diffusion models
-- Provide unified interface for different model types
-- Implement efficient memory management
-
-### Executor Module
-- Coordinate between different engines
-- Handle request routing and load balancing
-- Manage execution pipelines
-- Provide error recovery mechanisms
-
-### Worker Module
-- Handle actual model execution
-- Manage GPU resources
-- Implement batching strategies
-- Provide performance optimization
-
----
-
-## Checklist for API Design
-
-- [ ] **Clear Purpose**: Module purpose is well-defined
-- [ ] **Complete Interface**: All public methods are documented
-- [ ] **Error Handling**: Comprehensive error handling strategy
-- [ ] **Configuration**: Flexible configuration system
-- [ ] **Examples**: Basic and advanced usage examples
-- [ ] **Type Hints**: All methods have proper type hints
-- [ ] **Documentation**: Comprehensive docstrings
-- [ ] **Testing**: Unit tests for all public methods
-- [ ] **Integration**: Clear integration points with other modules
-- [ ] **Performance**: Performance considerations documented
-
----
-
-## Submission Guidelines
-
-1. Create a new file: `docs/api/[module_name]_api.md`
-2. Follow the template structure exactly
-3. Include all required sections
-4. Provide working code examples
-5. Ensure all methods have proper docstrings
-6. Include error handling strategies
-7. Submit for review before implementation
-
-This template ensures consistency and completeness across all vLLM-Omni modules.

docs/design/architecture_overview.md

@@ -1,113 +1,195 @@
 # Architecture Overview
 
-# Introduction
+This document outlines the architectural design for vLLM-Omni.
 
-This document outlines the architectural design for vLLM-Omni, a new repository developed under the vLLM project. vLLM-Omni supports multi-modality models with non-autoregressive structures and non-textual outputs, extending vLLM beyond traditional text-based, autoregressive generation.
+<p align="center">
+  <picture>
+    <source media="(prefers-color-scheme: dark)" src="https://raw.githubusercontent.com/vllm-project/vllm-omni/refs/heads/main/docs/source/architecture/omni-modality-model-architecture.png">
+    <img alt="Omni-Modality Model Architecture" src="https://raw.githubusercontent.com/vllm-project/vllm-omni/refs/heads/main/docs/source/architecture/omni-modality-model-architecture.png" width=55%>
+  </picture>
+</p>
 
 # Goals
 
-The primary goals for vLLM-Omni are:
+The primary goal of the vLLM-Omni project is to build the fastest and easiest-to-use open-source Omni-Modality model inference & serving engine. vLLM-Omni extends the original vLLM, which was created to support large language models for text-based autoregressive (AR) generation tasks. vLLM-Omni is designed to support:
 
-* **Support for Non-textual Output:** Enables the integration, efficient processing and output of various data types, including but not limited to, images, audio, and video, alongside text.  
-* **Non-Autoregressive Structure:** Support model structure beyond autoregressive, especially Diffusion Transformer (DiT), which is widely used in visual and audio generation.  
-* **Integration with vLLM Core:** Maintain compatibility and leverage existing vLLM key modules and optimizations where applicable.  
+* **Non-textual Output:** Enables the integration, efficient processing and output of various data types, including but not limited to, images, audio, and video, alongside text.
+* **Non-Autoregressive Structure:** Support model structure beyond autoregressive, especially Diffusion Transformer (DiT), which is widely used in visual and audio generation.
+* **Integration with vLLM Core:** Maintain compatibility and leverage existing vLLM key modules and optimizations where applicable.
 * **Extensibility:** Design a modular and flexible architecture that can easily accommodate new modalities, model architectures, and output formats.
 
-# Expected Gains
 
-* It breaks through the limitation of vLLM that only supports autoregressive text generation models, and provides architectural support for popular model structures, especially multimodal generation.  
-* By supporting new output formats and new model architectures, developers can easily connect to mature ecosystems such as Higress Gateway and Ray, improving the ease of deployment and efficiency of corresponding models.  
-* Integrating efficient cache management, multiple parallel strategies, disaggregation from vLLM, to improve inference efficiency of corresponding models.
-
-# Non-Autoregressive Structure
-
-* **Diffusion Transformer:** initialized with a random latent noise, the transformer model iteratively predicts latent output for multiple steps, under control of sampling timestamps. Finally, it is decoded by the decoder of Variational AutoEncoder (VAE). It is not supported by vLLM now.  
-* **Autoregressive:** dominant generation paradigm for text, now supported by vLLM. It generates tokens conditioned on previous tokens one by one. The efficient KV Cache management provided by vLLM can accelerate inference effectively.  
-  ![][image1]                       ![][image2]
-
-
-|  | Autoregressive | DiT |
-| :---- | ----- | ----- |
-| Scenarios | Text generation | Multimodal generation |
-| Generation | Token-by-token, left to right | Continuous vector, iteratively refine |
-| Key advantages | Smooth, good local consistency | Global consistency, diversity and controlability |
-| Efficiency | More efficient with KV cache | Need generation repeatedly, slow |
-| Error accumulation | Easy to accumulate | Can fix possibly |
-| Key challenges | Limited global planning, long context generation | Large global computational cost |
-| Sequence length | Varied length | Fixed length |
-| Attention mask | Lower triangular | Full bi-directional matrix |
-| Parallel strategy | TP, DP, PP, .etc | DP, CP, .etc |
-| vLLM support | Supported | Unsupported |
-
-# Targeted models to support
+# Representative omni-modality models
 
 According to analysis for current popular open-source models, most of them have the combination of AR+DiT. Specifically, they can be further categorized into 3 types below:
 
-* **DiT as a main structure, with AR as text encoder (e.g.: Qwen-Image)**  
-  A popular model for image generation and editing. Main visual generation models are similar to the structure, such as Flux.  
-  ![][image3]  
-* **AR as a main structure, with DiT as multi-modal generator (e.g. BAGEL)**  
-  A unified multimodal comprehension and generation model. Visual generation can utilize CoT text generation output. Main unified multimodal models are similar to the structure.  
-      ![][image4]  
-* **Extension, multiple AR+DiT (e.g. Qwen-omni)**  
-  A new model for multimodal input and output. It designs the Thinker-Talker-Codec structure, which is a double-AR+DiT format.  
-  ![][image5]  
+**DiT as a main structure, with AR as text encoder (e.g.: Qwen-Image)**
+  A powerful image generation foundation model capable of complex text rendering and precise image editing.
 
+<p align="center">
+  <picture>
+    <source media="(prefers-color-scheme: dark)" src="https://raw.githubusercontent.com/vllm-project/vllm-omni/refs/heads/main/docs/source/architecture/ar-main-architecture.png">
+    <img alt="Qwen-Image" src="https://raw.githubusercontent.com/vllm-project/vllm-omni/refs/heads/main/docs/source/architecture/ar-main-architecture.png" width=30%>
+  </picture>
+</p>
 
+**AR as a main structure, with DiT as multi-modal generator (e.g. BAGEL)**
+  A unified multimodal comprehension and generation model, with cot text output and visual generation.
 
-# Architectural Overview
+<p align="center">
+  <picture>
+    <source media="(prefers-color-scheme: dark)" src="https://raw.githubusercontent.com/vllm-project/vllm-omni/refs/heads/main/docs/source/architecture/dit-main-architecture.png">
+    <img alt="Bagel" src="https://raw.githubusercontent.com/vllm-project/vllm-omni/refs/heads/main/docs/source/architecture/dit-main-architecture.png" width=30%>
+  </picture>
+</p>
 
-vLLM-Omni will import core modules from existing vLLM framework and introduce several new components and modifications to achieve its goals. The core architecture will be based on a modular design, allowing for independent development and integration of different modality handlers and output generators.  
-![][image6]
+**AR+DiT (e.g. Qwen-Omni)**
+  A natively end-to-end omni-modal LLM for multimodal inputs (text/image/audio/video...) and outputs (text/audio...).
 
-## General ideas
+<p align="center">
+  <picture>
+    <source media="(prefers-color-scheme: dark)" src="https://raw.githubusercontent.com/vllm-project/vllm-omni/refs/heads/main/docs/source/architecture/ar-dit-main-architecture.png">
+    <img alt="Qwen-Omni" src="https://raw.githubusercontent.com/vllm-project/vllm-omni/refs/heads/main/docs/source/architecture/ar-dit-main-architecture.png" width=30%>
+  </picture>
+</p>
 
-* It divides models in vLLM-Omni into multiple stages, which are processed by different Engine Cores. Each stage can choose AR Engine Core for AR model part, and Diffusion Engine Core for diffusion model part.  
-* Designed a specialized diffusion engine core for DiT or dLLM.  
-* Plug-in with external DiT acceleration modules such as xDiT, Cache-DiT.  
-* Output linked with Higress API, Gradio for online demo, ComfyUI for less-code process.
+# vLLM-Omni main architecture
+
+<p align="center">
+  <picture>
+    <source media="(prefers-color-scheme: dark)" src="https://raw.githubusercontent.com/vllm-project/vllm-omni/refs/heads/main/docs/source/architecture/vllm-omni-main-architecture.png">
+    <img alt="vLLM-Omni Main Architecture" src="https://raw.githubusercontent.com/vllm-project/vllm-omni/refs/heads/main/docs/source/architecture/vllm-omni-main-architecture.png" width=55%>
+  </picture>
+</p>
 
 ## Key Components
 
-* **Processor:** Input processing, tokenization or encoding  
-* **Scheduler:** for order and queue of incoming requests  
-* **Executor:**  Prepare model input and run  
-* **Multimodal output processor:** support multimodal outputs  
-* **Hidden State Output processor:** support intermediate hidden vectors as request output  
-* **Engine Core:** accelerate AR modules, imported from vLLM main repo  
-* **Diffusion Engine Core:** Wrap and accelerate DiT modules.  
-* **Diffusion model:** Wrap up diffusion models with vLLM format
+| Component | Description |
+|-----------|-------------|
+| **OmniRouter** | provide an intelligent router for Omni-modality requests dispatch |
+| **EntryPoints** | define the APIs for offline/online serving (APIServer, Omni/AsyncOmni) and provide the OmniStage abstraction for different AR/DiT stages |
+| **AR** | adapted for omni-modality models while inheriting efficient features from vLLM, such as cache management |
+| **Diffusion** | natively implemented and optimized using acceleration components |
+| **OmniConnector** | supports fully disaggregation based on E/P/D/G (Encoding/Processing/Decoding/Generation) disaggregation across stages |
 
-# Examples data flow of specific models
+Disaggregated stages are managed through configuration, such as in the Qwen3-Omni example, where stages like Thinker, Talker, and Code2wav are defined as separate OmniStage instances with specific resources and input/output type.
 
-1. Qwen-omni  
-   ![][image7]![][image8]  
-2. Qwen-Image  
-   ![][image9]      ![][image10]
+## Main features
 
-# Overview of function call dependency
+vLLM-Omni aims to be fast, flexible, and easy to use with the following features:
 
-![][image11]
+### Performance and Acceleration
 
+The framework achieves high performance through several optimization techniques:
 
-[image1]: <data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAALYAAAFwCAYAAAAVNoEZAABxw0lEQVR4XuydBVRcWb6ve90709OStnTScel4x93dIUaEuBJ3dzciRPDgEtzdCRpCBCIEEkIgQHCCBHe+t6uSvNvNnZ55r2/fng5zvrX+69TZp4qkan3nz2/vKup88sm/OWqXL3HxwnnOnj7JubOnuaB6jqtXLqJ28TxXRWmrX8Ha0gIzE0OuX1bFQFsTcxMDLMyMMDXS44bWVRxsLLG5aYKuxlVsLMxwc3bAWF8bNzdnHGwt8fb2wNXZHlcnW3HMDjsbC4Ju3cLO2hwvT3ecnexwsLcmIMAHHy83+X18PVxxtLMUt+1xcbTBw8URRwdr+f1tLM1wd3PB08NZlAs24ue4uTrKf0bD5yfxb8qWDWvZtnE9W0Vt37yBvTu3cWDPTlE7OHviCBfPnsTEwECIfg7Vk0e5fkkVoxtamBneQF1NlSsXTmMrZJaN6elcF7LexFaIZ6SngbOjvfy2l5DUSwhoa2mK9U1jfL09ibwTKaS3IPx2BA4ONnh7uuHr4yUX1d/Pi0B/X1xd7PHz8cTT3UUc85SfIMHBQUJ4a3EfH/mJ4OsjThoXB/njPNydJbEl3rFisTKrli5i9fLFrF21jC0bVdi2aR07tqzn+KF9nD91HF1NDU4eO8CZIwe5fO4UBjoacpEvnDnG5fMnsDQ1Rk/rOje0rwmxLeQd3chAC0d7W6zMjfBwc8JHiGlmpIu5oS6hISHcibgtOrE99+4+wE50dR8hraeHm1zOW4G+BAb4yjtyYICf/FiAED0wMICw0DDxODvR8QNEBeLv7yO6txPeotN7e7lLYku8Y8akccycOhGlGVNRVprJsoXzWLlEmbUrl7Jv5xbOHD8sBD/ApnWrOLZvN2ePH0FDRJVronMf3rdDiH1aiK4p9s+JE0AWS6xE59bA2EAHO6ub3DTRw1UI7Ovjg/oVEWV01Llz567ouH6i+wbz8FEMtiKaBPr7yzuvn6+nED9I3rH9/bwJDgqUSy3r1OHh4YSEBOPu7io6/W1CQ0PFSeAvxHeTd3B/X19JbIl3TB4zjOnjRzJz8jiUFCazYPZ0li1QYrXo4nt3bObE4QPs3LIRlRWLObh7h9jfj5rqGdGtj7N/11Yh9hk0r14W+yfQ0bgiZLZC8/pljPR1ROwwwdxYF2d7GyGfN1cunkJfiB0h4oe3lweRkfeIjn4iF9vf1wcnkbX9fL0ICb4ll1kutrgtkzdEyB4ZGUlQ0C35Y+/cuSPv3oHimOxkkHX2W6KjN3x+Ev+mjBzQmzGD+zJxxCAmjxrC9AmjUBZyL1dWEkJvEF17G6tEB1++aB47Nq0Xsm+Vy71PbHdv2yTP4LL9Q3t3onVdTYhsyCXV02LyqCuyuS762texlk0oXV05e/IQetrqBInuK5v8RUc/JiLiDvYiisgmf7Y2N0UssZJv1a9fFQJ7yiUPkXXrsBAePHiAn+jMsghy//590dlDRRTxk8cW2VhoWLAktsQ7Bv7UleH9egi5+8kFl3Xw+TOnsmTBLDGpXMsOMaFcunCuiChKbF27ml1CbtnEcvt6FbnY504eE517uxB+CxpXLwlxNbhwXjbhvCGXWFOMmZsZibztwMmj+7khsniA6Kwurs4iSoi87OKCuuj4urqa3DQ3xdHBVt6pXZwd0NJQJ1AmbGiw6NC3xYkQLbK6lxA6WC55yP8VWxZrbhEWHiKJLfGOPl06MKhnVwb06Cy2XZg0aqiIJJNYNHcG61cvl2dtZXF78byZrF22iHUrl7B53WqRwxcI6dfLO/WW9Wvk3f3i2VOonj7BeRFT9HW0ULtwRoh/FIMb2hgZ6otMLuvql9klos3COYri3xrOiEH96dujK907d+Snrp3o37c3Gzdu4MKFCxjqG7Bn9y7uRt6Vd+gHUQ9xFsKHiMmnbEy29ReRRbaaEhYWJo8qDZ+fxL8pfbt2pF+3TqJ+ZHCvLiKODGX2tAly8VRWLJFPJBcoKcpFX6E8j9WL300slyrPkXd0WdfeIE4A2UqKbHlQtpJy7tQxtNWvckZILYspOprX0dFSZ8+2DezcqMKsCaOZOGwQvTp1oHPbVnRo2YL2rZrz/TdN+O6rJrRp8YMQvA+q5y9w9NARbgUFEf3wEffuR2FrZyUXOiIiQnTyEPmkUSa2LNLcu/dAElviHb06taF357bIBB/cqzNjh/YXOXs0cxWnsWjebPlKyVwRTebNmILyLAWWzJ3NMuW5LJijIO/o61YvY/WShWxUWcmB3dvZLgQ/emAvZ04cZc/2zULmTaiKuLJv22ZmjxvJqH496dK6pagW9Gnfhp7tWtH5h+9p//23tPruG1qI+v6br/n2qy/47K9/odn337FGRQUHJ2fCbt/GyFhPxI4Q+eRRJrivr7d8AvngQZTo6tGS2BLv6P6jELtLO7nY/bv/yPCBPZkiBJylMIV5QuS5M6czR3EKStMnMU8m+5yZLFGazdwZU1mzfAmrly2UTyzXr1omjyMb1qxgn8jch/fvYdsGFXZsXMMusV0i7j+6V1eGdm3PGBF5pvXtztxBPZjetyujxL8/VPw/erRuzo9C8pZNheBNv+b7r7/iy88/o2PHjixcspTbtyO4IbK4bMnvzp13KySyzB0cIptYRktiS/wXnzdpzpdNmtGtQyt+6tiK/j06MXbEYCaPG8WMKROYOWUc0yaOY6ro4orittKMacxWmMysaeNZtmAei+fPEZ19FquWLmSdkHv5ovnyzL1dxJQ1Qvr1ohaOG8rxJTO4ukYJ/a1LODprDEsGdmdkux8Y2KopHb/+knZffk6LJp/xvRD5e3G7+Zdf0PKbr2gtJG8jZP/uqy9Zu24tqmdOEn77jrxjy96wkb1BExERyd2794m4c1cSW+Idn33ZlL99IaJA6x/o1KaZ6N7tGTmkP+NGDpXLPG38KLnkk8eNYPqkMXKpZ0ydIKQfK2LJHHkHV1aaIYReIM/jMtFlgss691KRzeeKn6G7YwVup3fgr3YAw83KLO/VkbX9OrNhYBfmd23JrC6tmNT+B7p89zVdRafu1uwbOjf9ivbffUUHsS/r4h3F/69Fs2YsWbhAZOn78owdGBgol1vWyWWTyYg7EZLYEu9o/cVXrO71E6aL53Nx1mT6iLw9sE9Phg/qx/jhQ+Q1doTYjhryXu6xTJNtx4scrjCVOSKiKImoIsvispJ19MULlEQGn8GsYf05vWI2+aGOZHqb4LJvNY+v7iP2+hEeXdpL2NF1uG9V5ti4nuwb1pUlPVoz88cWTGzblHGihrVuypA23zOsfXOGCPE7t/qBZt9+g7qmFmFh4XKxZXlb9kbPbdG1ZZ87afj8JP5NGd2mFfuHD8BCeQ43F8/hzMxxzB3al5H9fmJo/z4MH9CXEYMHMEpMKscNFxFl9AjGjxjERNHRFSbIJB/N9MljmDV9MjOnTUJh8nhmT5/CxNHDMNu/luI77pRGB5Dhb0t1zB2qnoRREuxEnpsx989sI3T/apzXKXBz6WRMl01Bf9F4VKcP5uj4Pmwd1p21g7ui3Lsj83p1oEerZrQV3fv7pt9y4uRxbgUHERIWRlh4BMFC9FvBoZLYEu/YObgfp0YO4ZLowtemjhViKXB1wVT2KIxBccRAxorOPVgIPnRQX8YMHSjv4GPeSz5p7EgmjhkhjywyoadPkmXxMfJMPmHoAOJstKhLjKLiaTiVCQ+pSY6jJj6K0jA3CjxNSdI/S4L2MR5c2sW9CzsIOqKC156lGC2bivr8MRye0I+dI3uyelBXlvXvxMA2zWnb7Ds+//QvjBkziiAxaQwJDRWTx3D5kqC/9Ja6xAcU27Rkcgvx67/59yzo2JrDo/thvVYZj51rCT20leCDm7DfvALN5XNRHj2UcQP7MKRPV9HJe4suLuvkQvZRw+SCTxg9XH57pDgZji6bSXVcJHWZidS+SaMuN43alGdysasfBlER6UNpsCNv/azJctQh1eoacdrHuX9hF67bFmK+SpFz04dwZHw/to3sxfqh3ZgssniXZl/LlwT/+p//ia+vn5g8+uPt7SX/YJSrq4sktsQ7un39Fd2/+ZoB337NpJZNWdezI/pKY3FSmU+o2mVCjc3xd/LCw9kHbSNrDl/QQFFpAaNHDGNov3dZfIzo3rLJ5tiR7/L4zJGDCBQ5ui4tntr8TOpF1WWnUJsUQ+2z+1RHBVF1L4DSUFeKAuzJdjEkzUaT+BtniLq8F+ftypitVOCsEPvoxH7sHt2XzUN7MLtHGwa0aEpLIfYXf/sMS0sLPDw8cHR0xN7ORl4Nn5/Evymd2rXgxzY/0EF0wj5dO3NE2xTj5CLMC+rQL6zDtKQe+/I6rEvrMS6owfBNFQY5leimlnLI0pOl2/aLDN6X0UPede9TG1dxS/sMiO5cHO5DcYgHed4OZNkZURbqSfWDYKrvBlN+25/iIDcK/Rx542JEpp0uTzWPEXl+Jw6b52O8fDrnFYZxcupg9o7pK7p2Dxb3aYdil5a0+eYrvvnicw4e3I/lTQsM9G+gr6eL3g1dSWyJd3T44Rt+bN2C+bsOc9w7ApMiuFkMN/JrsSyuwbm8FuuiWkzF/s3Caizf1mCZX4VpbgV6yW+5/iCRpVv3MkFMJBdMn0BehDf50SGQLaLH6xfUy+JHVCiVYZ6UeltRGexO9T2xHxFI0S2RtX0cyHMxI9P2Bo9Flw8/vQ3bjfMwWDqVc0LsE1MGsXd0H7aP6M6SPu2Z3bUVbb9pwjdffsnKlcvQ1rjO1SuXuHZVVpclsf8gNomSvdgfKuaXh//1qD3O4fLrKjRya9HJrUE3uxqr4mrcy2vwLKvDMKsKSyG0s9i3Lxad+20txqJj3xRlJeQ2yyhBNy6Dq0GPWLN8Kbn3g6jNEdEjL5uHphZcO3QRNyNHEnyCqY2NpfbFC2rjnlBxL0x0clkMuUmSmQZPdS9wR3U/t4Line truncated
+* **Efficient AR Support:** Leverages efficient KV cache management inherited from vLLM.
+* **Pipelined Execution:** Uses pipelined stage execution overlapping to ensure high throughput.
+* **Full Disaggregation:** Relies on the OmniConnector and dynamic resource allocation across stages.
+* **Diffusion Acceleration:** Includes integrated support for diffusion acceleration. This is managed by the acceleration layer, which handles:
+    * **Cache:** Includes DBCache, TeaCache and third-party integration(e.g., [cache-dit](https://github.com/vipshop/cache-dit)).
+    * **Parallelism:** Supports TP, CP, USP, and CFG.
+    * **Attention:** Provides an interface for third-party integration (e.g., FA3, SAGE, MindIE-SD).
+    * **Quantization:** Supports various quantization implementations including FP8 and AWQ.
+    * **FusedOps:** Allows for custom and third-party integration.
 
-[image2]: <data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAARUAAAEOCAMAAACQIXOIAAADAFBMVEUAAAAAAAD/AAAAAAAAAADfACAAAAAAAADvADAAAAAAAAAAAADqACsAAADgACkAAAAAAAAAAAAAAAAAAAAAAAAtAAbmAC0AAAAAAADqADAAAAAAAAAAAAAAAAAAAADnADAAAAAAAAAAAADqAC4AAAAAAAAAAAAAAADpADAAAAAAAADoAC4AAADnADAAAAAAAAAAAACxACQAAAAAAADoADAQAALkAC8AAAAAAAAAAADoAC8AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADqAC4AAAAAAAB/ABoAAADpAC8AAAAAAAAAAADqAC8AAAAAAAAAAAAAAADpAC8AAAAAAADpAC8AAAAAAAAAAAAAAAAAAAAAAAAAAADpAC8AAAA0AAvhAC0AAABXABLqAC8AAAAAAADpAC8AAAAAAADqAC+rACLpAC/IACgAAADGACjqADDpAC8AAADVACsAAAAdHRoAAAAdHRojHBowGhw2GR08GB1JAA9XFSBdFCFmABVpEiKQDSWWDCenACKq4+q8Biu9ACbCBSvTACvcAi7jAS/pAC/4Y4L9gpv+h6D/kKcAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAn++oGAAAAdHRSTlMAAQEECAgKCxATFRYYGRkaHSAhJCgoKCsvMDY3Ojw+QEJFR0hJTE1QUFRYWF1gYmRoaWttcHJyc3R3eHyAgYSGiYyNj5CSk5SXmZ2en6Klpq+vsrO3uLu/wMXMzc/S1tbX19fg4eft7u7v7/H09Pb3+f3+/iGfBl4AABcuSURBVHhe7Z1dcBNXlsePaeE4MW23jLyOhASOsPiwwYAXg5kwZAOJcajNkDDxUMkkVcmmaqt2HlJTNTOv+7Cvm62amtpK1W7NJFWbbDIDMyRVmYUAkxAIWcAQMAx2+Su2wY6FQTYysh0jS2bvR7fUfdV2d0stS2Lv70G6ffuclvqv27dvdx+dC8DhcDgcDofD4TwcFLEVOWNlcR9bZRHJlSjGB1T11nGwFblC2Op/m62zyMrmRDH4kKhiA/c6AMp8oVFUDLPrrPEwqTI4CFDvCx9j661TMKqIq93SdNdAFB1rK4pvxn114pEoSOt8UVKnh7DBI00PXSZF7NIiXhmgLsfm8UhQKKo81YhfA6N/moai5mVHdvgAWg97D+C64GHdndzwpIheA9Hr6LXoWRG7eIN/XPHCEoBlf7zPWmspFFWWD6MfunpvVfMnaMHxk+73wN3y3OOdF6D6aXdNJ2uNqG+Gtg4o2tIyR9ZiF3eLu+XxLuyyWs9DxRK2Ik85/fvuKAwehdJivBQ8PjZ2HbWco2Nj31yGNawxoqIJjp4ZGwudhC3EA7tcP4FcjiEXWGMwICkUVcbI61TESRr3FXLMxNrwawhKE1ZJvGVBOvy5V0lHMdjl7lzs4gO8QLWdn0I5ggBcFStcS8UZXIyRE29kKXm7qzZKUAM3aG8T8ksjILtMTRYTlzmVoS6Foor/Wdx34v0zy7hSqFDXmqNQVDkQ+7pnDCpal7IrskKBqCLApXNs3YKElSbiTDYa8xRIb1tE982LDyNThGAz7YSdMzeYVSYokLYSg331E0vW90bMHkHXhl/6p9hAdMn6ecZ4C1MgbQUugrdu/fmv2Or5GT805AjUrY+lI0oe3V8xQJDkMYt5kMuc/nmbw+FwOBxOIVEw45V/YyvU/IKtyJA8GPHbub8LbutXhvdVFAplxL+4cFX04KrowVXRg6uiB1dFD66KHlwVPbgqenBV9OCq6MFV0YOrogdXRQ+uih5cFT24KnpwVfTgqujBVdGDq6IHV0UProoeXBU9uCp6FKAqcdOPANMm/54zrxueZKu07IHP2aoENj1RzT+W/WwZW8WwZw9bYzf5dwRNdm9nqxgurDXSLVPyTxW002wNg7FumaLuV36lKueSx6bZGoYlUtjWLuJt8qchFXnYVmDG6EvNzZSwVfZi9AVywZzhPs+UZPd7Z3fraWK4z9luLEafnxOM9zn6aFa/OBsX96/Mck74m5d/ZzCSgxfvzT+Us8IvdNXVrcw1tw+9aTQi+bgsm0O5vFQFgjcNRyTnN7A1NpKfqqB9NvrTWPAmW2MjeapK8OYmtorlvJFuGZCnqsD5LUY7HTTULX3yVZVgyHCnDXVLn3xVxURjMdYtbfJWFePGYqxb2uStKsY7baxb2uSvKsY7bahb2uSvKvCxbytbpSX459ezJEseq2LcFoybU5rksyrGO22oW5rksypop9kaBmPd0iOvVQmG2BqWLDWWvFbF+Fon+H1WGkt+q2J8rXMpK40lv1UxvtbpykpjyXNVjPc5K40lz1Ux3ueu7/VSC2ZInqvS9dVBtorlgy3r2KqMyYEq9b/8qUHCQxVdQPd5AadLBhcGacA++UiT3Q0fKnn/6kme5tjoZBtOSpwxl7Z2sVUMXfarYk9bcfhhG1OzYu1r2po0URrLAlxiKzLGnrZSK4E3GUkQ/HwWwLdd3HlWbZMuJhrLOiMLq9jSVoQauFRSnVweHxsba/9deNV8PYElTDQW2w8hW1Qpr4z1KNl1E8TaaC7ajDHRmxrqZhFbvrhXRNdxlS42z6pDYsJzxNXrH1zpVpaEjQEHDH4j5/6rDrge3OgiSe+EljJ4cENZIZY43/j+iuIEwgHFabO7v7u60XF6xMRBZhE7VHFsgxvRsORnVHHR5LxJSI5ar7IHFa041eaK7b/GC9JeH3r1Crgnoiu82/+C84Bjp+KqsPcUdaIrV2w/0YnW+COPoPOdNILOQjb3LHaoskvqOwtHWpvGVTm6xcpnxXc0TUXY7w++h35jHxmXoaW+/yDvL3yCF2bel8/j2IyuaNl0OEqd1m394JfKSuwl7N+3BmcVb+qTp3v44FWwVRYbVKkIQA/AxB1xjayK+y3y1qk9fjz+2Cnc8Ef6/XSJZL2O9+16bNrjhy+UwQ02Iysu+t01ndQJtQXipHjF+/zklDdzQnay+xiyobf1ijOD+Juik7OqNvbZUfwm1BCKcfrdO+QmUpxk+Q7ISyTttRPk1N/qFUGcuFd2urSVrldWku4HYDihu4kTlRUybyvotDzhQe9LImIDHaAEUcvf2dgi4aXyPbiTiB0aARdMaZKqyi0KYtK0dpVcfjAuuRSn5PBV8SI9uepWXZetjSVzVcoroeoFWlzVpuxc/LTQsBWrMkFDkkI4raoatPQdLT3QrEo1I9BdxisTXlqubrGzw81clc2i/E3LRPXJ+YxfKo4qxwsirj0hhaEP95fKQim2xcRVZaSl4nT1aXmlykvD7BU7G0vGqqBLoPP0wHG8rj45x44f3MaM+OlFATriAO+w6hIhBE5HUgm5XFRBcmVTs9mAvFLlpcXWxpJxb+uWYv20hN43q/rbocvqJYBeoBcF6IgD3F+qLhGGIyUNKeVaCXXiihMUk5GzxkvL7BUbQ8IyVUVoVM4Zyf2WaS8h9xQURvqhGfXK0gGSJx0vkXh9AdWN90LTTlSW1tIyrvfvgPZpxQmkZY8oXsTph7hSy1Wnm61Km0yPIPTLy1NH4HOppAxZMONQU6tajH9ZKb7S4fQEr+A+Ai89/1w3OD0foqUzlb6mrbfKxfPocuC05P/5QMwrQudZ2Sk06vR89qIge/0cO8W+TW5YZvZK08dsXbqw/27IbrytWDquPjsvd96NJJaXl95SyqIzel8ektAl5LX09T8HqZ3aScNWn3VZSLxtyr8bMm0r1ohENItj6qzpqgWtGV1KNAWNk4arRg9gTZNpv7J4XHUZ9RuzySvrDCkcVVBjYatYrhrpZpbCUcVMYzHUzSQFpIqJxmKom0kKSBUTjcVYN3MUkirGjcVYN3MUkirGO22smzkKShXjnTbUzRwFpQpcGn2RrdIy+9FP7JClsFSBaysNdnryulFzMkOBqWK80xd8BrqZocBUgQsr2RqGyQ4j3UxQaKpMkkdnC3Ghqgy9rlvF1luh0FSBC+Tfqwvs9GRvI1r/BpYmbQpOlcntaJ9fXWinLwQ2vvoGW2mNxb2/Ygdrb2/ayNZp8MZfZausUniqzC68z+u2LqyZKdg7lPlN8WPKrzj1vWaFgpCYeXkeA11S7lAWVr9imP4rHpnncZE1CkwVw51+MB0xUs4EhaUK3mm2iuX+vRhbZZkCUwXttOE+xyYsdCn65F8WPSOW7sFP2T74K1uvpu4Zz8IGBhRcW4HZz/6LDUtMoeMjtsYa+DlloXFnYLnr2m22VsNUycTCBg8jS//ecKhmmBpqIbT9yr9olhj+OVk0a7eQYdLMlBGTd/OB5osnZ0W0Jz1n4fUrlOyeJQpVlezCVdGDq6IHV0UProoeXBU9uCp6cFX04KrowVXRg6uiB1dFD66KHlwVPbgqenBV9OCq6MFV0YOrogdXRQ+uih5cFT24KnpwVTgcjt2YiNQQfJWTcbby4UYbb1u/610ajlfffJT+mb+++TdzzcsOGcbR2APJhwXwG/3/ti8eWlWGS6qpGC6oIO9CDUQXMVLine truncated
+### Flexibility and Usability
 
-[image3]: <data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAPwAAAEqCAYAAAAxuz3hAACAAElEQVR4Xuy9B3QUV57/246AMTkpC6Gcc84555xzQgkFQBIIRAaTsQkGg43JGWMwwQRhG2MbbGzjgMceezyzO7sz7+y+8z9n98y+s+/N9/3ubZUolVpCIAkEut9zvqdbVdW3qqvv5/5+v1vVLZVKSOgR9e3p43hBpcLoF1UYO2YMtKcbQl/bgmyNmbqOmKXvAjMjd1gYe8LazBv2VoFwsA6Cs00YXG0j4WEfA0+HOHg7JcLHOQn+rqkIcEtHkEcWQrxyEOqdg3DfAkT4FSLSvwhR5OigEsQGlyIupALxoZVICJtNrkJSRA2SI2uRElWHtJh67oy4RmTGNyE7cR53TtJ85CTOR25SKz22dDk7obnb86z4FmTGNSMjdj7SY+ZRm01IJSdHNCApvB4JoXWIC6tDbGgtYkJqEBVUhcjA2YgIqES4fxnC/EoR6luCEJ9iBHsXcQd55Xe6AAEeefB3z0GAeza930z4OafBl+zlnAovpxR4OibTeWFOIifCwy4BbnbxcLGNpXPHHANn6yg4WUXC0TIcjhbhsLMIga15MDkQNmYBsLUIoPWBoI/peeXnJiT0SLp7+gQHfsxLz2PSuHHQ1zGBkZ4dge4IE0NXDruliResTH2oA/rDwSoYjtYhcLONgrtdNLwcGewJ8HVJhp9rCgLdM7pgD/PJ47BH+jLYCxEVUIzoQAZ7GYEuwV6NxPBqDjsDPTV6TjfQsxLmdoGem9yMvOQW5KUwL6Dn980GAOlRPRi0EvitHHwGfSpBnxo9FymRaugTw+Zw6JnjybEEfWxwNQ1IlTQ4VSDcr5wDL0GvBr+QO8irEIGezLkEfi78CHx/l3T4kRn0PhL0nfZg8Nsnwd0+Ea4c+jhuJ+sYOFpFwZmgd7KMIOhDYU/Q21kEqW0ZQANDsABeaPD0bSfwr7z8PKZMmAQDHXOC3QnGBiyye8B8lidFdy+C3pt3QDvLIIo6YRx2Ft0Z7CyyM9j9OyN7sGd2t8gexeyvhj0miEX2cm457OrITrBH1yM9tkEz7AS6BHR+ysIuS9DL/2bbdAM+Vh3lGfDMLMpL0MfJgI8OmI1IGfDB3sUEOIvwDPpCsjraM/AZ7Mws0vu5UpR3zaKBrxN6p1RuL0cp2ifBw6En8E7WauCZHcxDYG8Z2gU8y6ZcBPBCgykG/EsvqPDqmJegNVUbM/VsYDrTjWBnoJNn+WBOZSOWtS2jNNMfh/cewNIFy6mDRlDUiqVOHE8dOh6+BHyAezqHXR3d8zvT+EJ6ZLCTA0opnS8lsMrJ6jSeOTmyjiCsQ3Pda1i9ZDt/nhU/9z7wiQz2ZhoE5uGTa7dx8J0zBDWDu60T7s4on0iwdz7PSVxAbbDUXp3S56e24PrlGzRwtCIxlFL6kDk06NQS6LWU2tfQ8VUjMqCKBiYC3p8BX4ni3Ga8vm4XNq3diezkeg5+CIHPonuQpxp4P48c+Lplw8clCz6u6TQApqrtSGaRniJ7kGcGDYapcGMR3lad1jM7WsbAwTKaR3cHy0isWrQKrvYRPLW3swgm+INo+1ABvNDgiQH/MtXv48e+BN3pehTd7dQ1+ywvbjMjL1x67zi+/PQyjzofX76A9pbF1NkT4e4QQ/VuFpKjizpr98zO6J7LozszAz01ppLS+RLkpVL9HFmNyoIFKMtt5ZE9L6WJIno9CtPnY9XiHWitX4uKvEWYXbgUORTVm2avRmV+OwFL0Zpq819//hUH9pxAWc5iAn4Bf6zMX8oje2X+Esyrfa1zAGhFddGyTvib0TZ3A3758Seq61soujdSPd+A0qxFdOz1vI7PjKesIoGl+jUEfgXBXYqOi9fw8ZWr+OLGDRpkjiAquAQRgaWICS2her8YkUH0vj1pcPPLo9cUwI/ef7hfDg1w2QR4GgfemyJ70+wWZCWWEeTxiAjIoOyIojvV8X5u8Qj1SyXwKboT9Ls2vU7Ah/Mob2/JgA8WwAsNru6+dwJjRlE6P340jHRnwdjQkafwViY+vHZf0tqOvPQCHNv3LjxdonD1/PuoKqunx7Norm/D9s3bMa9mIZa0rKYUNgUhBDyr3SP8iuixiOrmKpw59gHam9fh4N5jOLr/JBY1v4bjh89g+6a9+PLmF6gtXYKvb93htfvONw7gP/7+77RuH4F2C3t3HcVP399DTsICAnQx/vzrr3h313H88d7PuHn9C3R8eINeexenjpyn7X7C6qU78ObrR/DNnbu4crED1698ijtffoszxy/g+6/vUjSfQ4NOI76j52to23vf/YAFTevx6fVPsXThFpyi4wom2LMS5+L8+xeQGl+H/XuO4EZHBz66cg2b1mzDv//lD9izfQ+dgwvY99Z+Op49OH34KN7cvBOH9+ynQacNW9e+TmAnID6sBPt3vY20+CJsXLEBZXk12L/zLRRlVmFl2wpU5NdgYWMbwR2JhXMXwdlWAl4NvatNiABeaPDEIvyEV16A1qTRMDGwhKmRG6xMvQl2H2xdtwnnTx7HwuaF+ODkMaQl5GHjqg1U26Zg26atePvN3aitaMKurW+hJK8RLtbx2PXG20iPm40IqtmjKIUPdMvFLz99j2+/+grleQvw5edfoWF2O25+9BnBvZ/A/QP27jxMsH5P9fQc3Lj+Oa5e/Ag7Nu3HX//8Ozas2IUlzVv5JFzT7HW4ce0mDS6v4zOC/U+//Er7O4A///E3rF+5m2DcjZ9/+IkGkW/wze2vsap9F5Uja3Dl/DV8/833+OC9y0iMmIu0uPkUuT9B/ewVdBxf4K3th7Bo3ibUlC3H+uW76P2VoyCjFRcJ+KKsJnz31R1sXL0Tp4+cxm8/36VBaB9uf3qDBrwVlPl8hN1bd1PmUImFTStRlt+E8oJ5mFe9GB52yVRC1OG9w4eRGleG948dR0JkPna/8SYW1i9Cflo5qovrUVFQy2v5mJBMOFoR8BZhZAG80BCIAT9x7AvQnjwGZjNtYW7swWfkrU39CPIT1Klb4GIXhk2rN/NavrFqPuLCcwmQZWisXYgPz52nqH0I8+cspYiWhHWrXuepO0vhowLK4O+Si7//2+84/94FxIfOxu5tB/Htl9/g2qWPKOpvxq8//YrbN29ThP0ciSH1uPXpHRzZ+z7amjbhG4rcn16/hdfXvstr9tqS1fj9l9/x092f+Wt/+u5nitQ/4ucff8bZkx/S9t/jzq1v8da247h390dcufAp5lSsxK9/+JWWf43bn32LBErnU6Ln4S5lAN/c+Y4ymK3YuukgastWYnHzG1RSrKPjrkJMQCUNUnfw1c1blD28h8ykRhzaexz/8tsfsXTBOvzxx+8oU6jEiYMncLPjGhY0rMSyhWuQnjAbzXOWUQSvh4dDKjLiqnD60FEaQOZgdftayiAOU7SnMieqiAaC/Ti5fz+/0uHuEI/lC5ZT2h/ZDXgXWwG80CCKpfRTxr0EvWmvwtLYlUd3W/MAfi3Y3IiBHwAHijo25sE8+rjYxsCNOqe7QwI8HJPoMQmeTqm8fmWpfFxoGcK8C3h0T4upxe7tB7Fy8RYkhFUiIYRACpyNaIIpNqiKauk6xIfU8MieFM5m5ucjLbqJoqX6OjubcGP1N6vVi9IXkxdx8JnZ37lJbeSFXY/ZlPZnJ7QgN3ERvwafGcdm9Bchi+r2lOj5VK/PRWpsM79Ex1J75oQwquFD5lAdTw6uo+OrQWxgLT1W8dQ+MrAc65bt5DV+sHcZAjyKEOxTggA39fV4f8pg/F1z6P1nwd89m8/Y+7qyCbxMPpnn55zOr8fz2XoaEF1t2fV49hhP5zUGDhaRcLaOJUfDjkB3tIrgwLOJOwa8uCwnNKhiwE+b8DL0p42DpakHrM18OfBslpjdBGJnHkIdkzqiNcFuF8UjkYdjPLyck+HtkkKdms1AZyLII5sAyEGoF6vf2TX3Uh7RC7PmUx1bxaN9QiiDuxqJYbUEeC1SIhso0jVyyNlMegbByC6lsfSdAcwm4gpSF6MwjWDvcjvB3s4fC7lpfWo7ClIWIy+JnEwDQWIb1fyLqJ02Dj9rKyN+AdJiFxDsDP5mJEc1UYRuohS/iUf9OAZ8SB3iOqGPJrNIH+ZTSduyQaGKX58P8y1DODnMp5TeazGC+Yx9AQI98hHomYcA987LdDQAcPNLdQQ/m8CjgdHTIQXu7Jq8XSKfxHNml+dsorkZ7ErgnQTwQoOp786cwPSJo2A4YyJFcfXNNeoZYmniKJR3QlZjutrF8sju6UQRyzmFolgaRbcMPjsf5JGDEC92o00RvwzHgI8JKueODZpNwFQTVDU0CNTwyJ4SOYfMZsubCPi5ncDP74zM7G45Nruujt55yYtQQC4kqIsIbmYGeX4Km6knJ9Pz5CVqyMnZ8cyd0JOz4hcS7K3cXcBHzkdKxDwkRzTy7IIBz03QxwQR7IHVHPho6VKdXyV3ONX33N5q4ENYxPcg4Bn0nuwOPAn4bG4fgp3Z25ldskuDN4v0BLwHAe9G0d7FJq7zzrtYfo4d2fV4Swa9+nq8o3WQAF5o8MSAnzFpNAE/mWD34zfX2Fuqo7qDpdqsEzpTBHKzj+NpPAPe2yWVX3dmwLObTlhHD/ZUAx/pX4IIcnRQGcFTQVGzilJmAp4iewKP7vX8bjcJdgn49Bh2K6wG4JPuA1/Aonmn1bCzyN7OrQZ9scxq6DM7gWewq4FvuQ98uBr4eErtlcBLsHcHvoI7zJuVLqUI8SwiqyN8EIEf4M5uub0P/X3gM+DllN4NeHeW3tvEy4BnUT5KDbwli/JB/DZmlQBeaLD0/ZmTBPwYGGlPJ9j9qbNRJ7MKVQNPNbsEO7tRxN0hsVt093FRR3d/Sl0DGfBe+Qgl4EMpwkewSbvASl6zx1N0Z+awR8wh2Ot5Kn8fePY4D2lRc5HBoW+hqMzulOse5ZmlqM6e53aCnpvIovt90LPiWNLine truncated
+vLLM-Omni is designed to be flexible and straightforward for users:
 
-[image4]: <data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAO0AAAFdCAYAAADv8OpOAABCCElEQVR4Xu2dB3gUxRvGlwRCh9BTCb2KVAWkKiDSiwKiCPxV7IACKlbABohBESk2pAkC0kR6SWjSW+i9Kr0TapL3P7N3c+x911I2l1z4fs/zPrmd/XZmd2fendkaTWMYhmEYhmEYhmEYhmEYhmEYhmEYhmEYhmEYhmEYJoUA8OtVqnqTj8rUfappcMmmOfyzNhXJKVJw1hxNWwSXbdo9vFK9IdUb59UYhjGHryrUfapFSIlnS2XPs6NOgdA9Imm3WSqcNfvuanmLrK0WGNxLYxgmZcyr1SL020qNhvlrmdCocDHMr9MR0fWfx+qGL7jVGqEV9Z/TtbJBF/1vVAPny8nYn6s1R818IagaWKhnEa1ITroeDMMkko/K1u1QPTDo4FslquNEszeBZz5AwtP9cbplT6FeLvVfi5641eZd3GjVD9fb9EVs63643fY9nG3Z2yFW6lLrdzC71tMQRUZny6zVoevBMEwi6BRSrmWpHHn3BmfLiXhh1GvCfBVzF5TGSpSWDWoDzOuL7Hn8sPLL9ujfqZpDDFWmTPrf2IjQnG9Pi6yVX2MYJvGUyJ63XW7/zIdr5w8Vpv1ADI076Mbq1KIEpg5vhCmRT1g03FGThz2Oazu7Aadfw8JfmuH6ru44uvJZTPu2kUOsUVPF/O8/fgz+fpkOly0V2puuE8Mw7mkrdLBx4QjsaPwSauYNwdLfWgBnegGHXsPtnS/jg1eroGeXiuhlot7u/hAqlgpEZj8NT7Ro+3vjlq3/R1eMYRgn+BtM+8/jXVExRyFsmN4WOPUmTq3qghfalHYY3pqtXLnzxGXLnn1f7wHfBmoMw7jHaNp1j3fTTbt+mjTtW3i1U3ndVN3efAf9Ph+GPoOGmqp+n32Nd78Yhi6vvomQsHCEFS/1xoABA9i4DOMOl6Y92wuBuTOjRr0ncPQOsP9aQop18AZ00fTj94C1B/9DoSJBl1s/+z++j8sw7vBk2gZPtdGNtufyvRTp0E1g75U47Lsaj/3X7fM7GAusP3YOgQUKXmrS8mk2LcO4w5Np6zdtnSLTHr4FLNq6Dy06dLadw/YdNATbTl+3M+26o2flvIuBgfl72q8hwzB2pMS0+8TQ9thd6L0onae08fgFNGvX0XJu3L40nn6yGDL5Z8ankaP0XpeaVohNyzDuSI5pd1+6hw3HzqPPwK9QskRxNG7eCgfEkPfA9QRi2jj0GfClbtj+r1bWbyHh8OsILxyArNlz68vsEUNmfXh8/LyMOyX0hv0aMgxjR3JMK3vXds93swx3C4Xpf4f9PFE3HzXt+18M0+e/9Exp/TYSTr6JIvkzI3uuQJtpZU+9at/xe8FhRd8SseXtVpBhGHuSYtoj4vx0+op1qFKjJrRSlaENnQNtTTy0OSd1YwYFh+gGlMNmW698ORa9P/5MzPfHIw8VQMVSeVAkJAwT/l6mX5jS870NLIs5dGPBgQNZ6foxDENIrGnlFd+f/pyPEqWsD1uMXgVtRSy0hRegLb8BrVUPPX3k73/aneMeEr3vplOXMOi70fDLkhu5Agvjt3lL7a4gy4tVy2IOxu5JSChAVo9hGEpiTLv3yj30/yrSYtZmYlg87QC0RRcthlWSxpXpIqZ4iZK6WdWFJil10Urek5U97K4Lt7FX3v6R92+tt3zEsleFmtutIMMw9ngy7RMt2uON9z5BtoAAaJ37iqHwKWHQ6/aG1XXRYtxGz+rGHTd3MQ6Tq8qyd5Xq/kZvVKxUCQOHj8bW/67i0C3b1ePrQi3s15BhGDvcmTZ/YFb4+flBKxgCrf/P0JZccWJWouViyPz7Lt245R96WDeu7EnlkPmN9z5G3pw5oFVpCG3A79AKhOhxE/5ernraK4ULF25tv4YMw1DaCR1q5My0eQMsQ+LBs6AtvuxoUKe6aDnXfayFvuy0Zf+Ic+K7mLlyoyWvnHmhTT8EbZnorUcshVa5HoIKFcJb73+MHNmyXQwLC3vTfvUYhrEjOCB72xz+/ocq5SmEnU1eRiWjacXwWCtUTBjsmjDtJScGdaMJ26HVbIpHHquLdz79AqVKl4UWUR7ax+OhLb1qOSdeIYbTU/dCy5YLosdHYGDgjR49ekyh68gwjAEMGOD3aGBwvWyZMs8pn7sA8vtntzftI80sJqOmTIyib0Gr2tDSwxYIhrY2wbn5ZZzozbOKofjSpUuRkJBQiq4nwzCECrkK1dUyaZeFEjbMaGeOaaV+3gDt4ToW4/60ztJr0xg5VJ59HFlFzOzZs6Vp36brxzCMEyZ81ahA+WKBH3dvVwZXdr6MgnmypNy0UtG3oY1ZDS0gK7TIBY7zlwojTz+InH7arSlTpqwUpq1K141hGBe0bhxeI1s2/7stGxRFzqx+0Go1T7lppeTFqS9nQPvziOM8q2kLZvW/2LZ1C/5mFMMklbLFC1fKlyvLy+LnXa1mM8vwVV44Si3J+77CtFrhkMs5m/D7tAyTXLJoWXNt1IKLWe69Lr5iue2TGlp5B9rwhdCCwo/6PdbieboiDMMkkor1Hn9MCys1TcscIM5Hs0PLki115JcZWbLm+Dc8vEAIXQeGYZJIoT7fNG3+ZBM8Xq8uGtatkypq0qBefL0XekTRshmGSSL37t1rGR8fvxxeQpT1XUJCQkm6HgyT4RENv49V7em8xCI8lEWYKIr4KlUR5V0UGkHXhWEyPAYTXBbG/VT8HCj/CtUSv/2E/KXEdFZnEvNyiV62lcrHm8TFxV0U61Dauh4B1r/+dBsZJkNBjeAKYeqrwiSXqcSsKzTWm4j1ui3XQ/y9JHTNmnZEGDiMbivDZAioCdI1wpPnb9DE+1w7fwZnrt6Vpo0Rpo2g28owGQLa8G2cmQGtwPP4S76Jng7YNPYl1KzUBDfpDAMbv2yCoGLNdNNOB4rSbWWYDAFt+IrYqCEo+MxP2HMpgc5KA+LQrHxpNB8UTWfYcWHtp6gZVAL32LRMRoY2fAvnMLNfQ7w5Zy9ui6mXH6uCRzoNRO0yRRFcpAiCitbFxpVj0aNhBRQuVBghD3cRtrIw5IV6KF00DEFBwQgNL4n6Lw615bpqTC80figcwSFhKN+gC16p/yhavT8HuHMKS0e+hvDQMISFh4rlyiLmYrx1qVvY/ctz6D7hMOThI+HKbvz+YRuEFyuGYkVD0azvROy8ZIm9c+Ef9K1REPPuxscMOMymZTIoNkcZubseg9sWx09bzuuTFXNp0HKVwLglG7Bu7rdoV8wPDz/eCj0+m4yZn7REsMjmrHTUjc3Im7cMnhk4GYdiVmDM241RwL8A/pOZJFxFxby5UbH9R4hZ/Qc+eroycmYLQqtv9yDqm2cQmisPJq3ciyNHtmHqhy3Q+puVOCu8GHdlHfrXCULUVcuqbR7fA6VzhWLnieM4tnspcmQqgI5jtuBigozdh8/r58PPV+JjfrzJpmUyKMqnRs4tGISGBTTs0y/6XBZGLI93l57T513a+Qc6BflhtfVU95/IjiiWORyx+lQCDh/Yi107tiFq8SyM7N8RZTLnwRVcx9/9G6L3zL24Y1kMuL0XBct1xZwT/0LzD0erj//Eju3bsV1oR8xi+Id2wsy9N7C036MoUKiyWgo7ZryLGnkz4/XIGdhw8LRIice9e3FIEKa9d3k7PqmdD7/cYNMyGRibGwysG/kCimsB0PvZu6uRv+LrWHHpnpi4g4OL3kc5/8LQ760IfnulBnLkeRLSsJd2z0eVMsUQHhaGsuXKoGRwLmgBFUUXuBlftQzB9Jj7l37jrq5GyZaRiPl3BzIF5EZIibIoWbKkrtJlyyLsiQ8QffwWvm4ahoIlnrUth4QziBrdE/kDNGQNfgT7r6iBuRhIn5mPl0rnxyIeHjMZmftuuM+g5kWgBXXUf1/88w00HroV1+Vp47WD+L1rBAJK3X+WokNpP4R3+Q1357+JMvlz3O9JD89Gt5IactV8TRh/A4a0K4rx2y39Me4dwYTuFdB16jncPbcdOav3wJwLasEE7Jo5BH/tkbeAgedLi3Pmpt/ov+PP7kSn93/FdlXIv7OghXbF3H2WsfOeMW1RJF8JxMXH71jA92mZjIq1+dvRpWwmZHt8EOTQc9WnT+DtJaeh97Mn1+DdatkR3KCvLbZGzmxoMiwatxb2Qrm8WXFKBgq++V9lZNH8UO7pwZBD7DWRHVD9peFYsikGP/ZriQjRUw7blSCvHiGrfwhaD1qAy6f3Y9XkfqhTJDPmH7EYUZo2tPlw/XfChS3Qcj+C92bs16dvn5qFLBV7Yckxy42gb5sXRb6yHeUtn50H2LRMRkVv7YSxg4di+vpj4tcprJwyFidiLU68emIVfo0chT9WHbFGXkXk8D+w+aTlBPfSrvlo07QJmrXvivUxe7Doh4/x1bgl1lhgxe8j8eWgz7A26js0L14IJ+9a0mNPrMPU7z5B15d7YsD307Dh5P27sUObRyC46lu26cOrfsfgPi+iYaMWeO71j3BJjY7jT6CgMOzEffJ6t/5U1E66rQyTIbC5wSSuXbqEaxbf2Di1diI+6NrdNn10THuE5iqCe4m4Bbykb20EFWtBk3H+0nXbbSbJqXn98NyoTbaHL+RjjXRbGSZDYGj3qcaFpV+gjhgOf/LzdPw1bwYeC82OQo/1oWFOOT2vJyqHVqTJ9pxZgj71y2IXecSRbivDZAjsm3kqEheLzevWYNWq1Th+4Rad65qEWzi6YbHLxxcTYk9g3cIFWHvQcuHKCN1WhskQ0IaeXnE5kpY3aF1At5VhmBQQpWmBUjSdYZh0iDRrtKZtk2LjMkw6x2pYGMXGZZh0iqGHtTOtTPtb0/IZYxmGSWOc9bBUbFyGSSe46WGpuMdlmLQmMT2sE23TF2YYxvsI07YVGigVrWlHox0NqnRUxUnZMmAYJu2IFh6OdjSrUtT9SIZh0gXRbFqG8S2i2bQM41Line truncated
+* **Heterogeneous Pipeline Abstraction:** Manages complex model workflows effectively.
+* **Hugging Face Integration:** Offers seamless integration with popular Hugging Face models.
+* **Distributed Inference:** Supports tensor, pipeline, data, and expert parallelism.
+* **Streaming Outputs:** Supports streaming outputs.
+* **Unified API:** Provides a consistent and unified API interface compatible with vLLM.
+* **OpenAI-compatible API Server:** Includes a FastAPI-based server for online serving that is compatible with the OpenAI API.
 
-[image5]: <data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAR8AAAEwCAYAAACOrXWuAAA5IElEQVR4Xu2dB5xU1fXHH21FsSCisrTV2LBFMAhG/EfFilEES2LDQiIREZKoUdSIGI1ExW6CGAEBFYMoTXrZoQlIbwssbQHpZWlL3z3/d97M2b1z3ntT9s3dmTdzfp/P77Mzt5z35s6737n3vrKGIRKlkHZt2wt+t6lK/HOJRKIUF+/IfrTARyTyoXhHdvLu7ftgz84DtvRUscBHJPKheEd2c+GO/ba0VLHARyTyoXhHdnJg0jRoddvtsGbleji496gtP9kW+IhEPhTvyE4e8r+hYBaFvMUr4ciBYlt+si3wEYl8KN6RyTjNKtwenGp9/+1wyMrKguVL8i34qHmpYIGPSORD8Y6M3rGlEIYOGQGjR461Xg/7bmQQPktXwYE9R2DQV4PNvHGwd9dBW91kWOAjEvlQvCOjt2zcAS2uuRZa33GX9fqHYaMt+KxYthp2bN0Dl15yqZV39GCJra6b9+4qgn279cBK4CMS+VC8I6M3b9gOJ5xQHc4551zr9ajhYyz4rFy2BrZt2mmt/2BeyVGw1eXGKdpOE1jjx0yC774dbsuPx3jKH83TBT4ikQ9FHRjXcMCEyeEDxRZwatQ4Gc4773xX+GBe8RGwAFRyJBxCCBuETvFhjHcctm/eBfUbNLDq4TQO141wW04gcTOeaZs8caq1D/y0v8BHJPKhDu8/bk2Htm7aBT+YkJk1Y27s8DEBgms/Y0dNCAMJwmHrzzthwtjJsH7NJgs4F154kVUPy82aMcfaFk7pOGRU7y88BIfM/cMp25gfJkDbu+6GwYO+M9MPQ9HeI1Y+xhP4iEQ+1EMPtoOB/QfBqhXrLDjc1PJmCxyxwMfq9ZUqWe/xCmha10GgLVu8ErKqZUH/fl9agLjookZWOQTHTTfebL1esmC5VQ/r0GjGOpNmGtM//7QvPPLwozB39kLo93l/qFa1GrzdoycUHy+BF194GT75sBeszd8g8BGJ/CgERFezI6/NX28BAS8mLA98cCSCp+Q/793PGung4vSpp5wKXw0cZINPq9t+G7puKB9mTp9j1ulrgigPdmwutEYz5Kef6gInVj/RHF2Nh/59B8JJJ54EPd96D0pKSqDZVc3hkXaPwdKFKwQ+IpFPBX97rmspfG5v9du44YPGNZxfXdm0FCqrlq+D0049zYTPNzb43N7qDut1ft5aePnFV6z6A8wREq4VNWncBDr88UlYtmgldPxTJ6hSuSqMGjHWBp8mja8EHLUJfEQi/yph8LmqaTPrdTzw+ftL3aw6A7/42oTPXrjil1fAH9s/AUsXrRD4iERprsTA5xjAxY0usfKWLFweM3z+9uwLVv3PPwtO1x5/tD289eY71tktgY9IlMaqV68+vP6PN83OXmAB4Zabb40bPvger3Zev3YTrF25AYr2HLHuA8P1mgH9v7LBB7eBr5eakNqycbsFPtwmTrtwoZkWobs+/xLkNDwHJo7LDYdPcQncdmsr6Nzpz7B8ySqBj0jkR21av80Eyi7rlPjdbe6GHv982zoFHit87v/9g/DgAw+VXgCIxsXidas3Wms30wIzYee2PaXwQbC81aMn3NP2XutMFZ7ZcrveB/cLt4+v+5gjIwQd7h+eTUNYYT4CS+AjEvlQCAoEwvbNu2HKpOmweH5ezNf5lBwHmDF1Nvw4bXYYNBAmeGHhnJnzrfrbt+yG88+/wKqH+TjimTp5hgUPDhzVwRHQIQtQP/04D9547U0ImPu4zxxl4Wn9PTuL5Dofkciv4tBAxwyf0FXKbiMXTEdI7DDB1vHJp+GO394Zth1ePpKxfOGOA471BD4ikQ/FOzIagZOoe7vIa/LXQ37eGlt6IizwEYl8KN6R0TgdwsXcl158xXo94vsfgne1L10Ne8wp0BN/+JOVd2j/EVvdZFjgIxL5ULwjk+k2B3xtf5jYAdvNncm0wEck8qF4Ryar6zJ2+LifoUqGBT4ikQ/FO7KTvx88DKpUqWJduSzPcBaJRAkR78hOxpHOgT2HU2q0o1rgIxKJRCKRSCQSiUQikUhUsco1jMfQPF0kEom0CaETMAxATzKMx3m+SCQSaVHAMEoCIfigcwVAIpFIp0IjnjDwkAVAIpFIi9SplpsFQCKRKKGKNOLhFgCJRCItynWBEBiGXI0sEon0SeAjEomSIoGPSCRKigQ+IpEoKRL4iESipEjgIxKJkiKBj0gkSooEPiKRKCkS+IhEoqRI4CMSiUQiUSqrQYO6IBanm81DuzI/1kUpJv6licXpYEPgk/riX5pYnA42BD6pL/ULg4AhFvvWAh+fSeAjThcLfHwmgY84XSzw8ZkEPuJ0scDHZxL4iNPFAh+fSeAjThcLfHwmgY84XSzw8ZkEPuJ0scDHZxL4iNPFAh+fSeAjThcLfHwmgY84XSzw8ZkEPuJ0scDHZ8oE+Gz+zoBVXxmw7ht7nhf3ed6AKy+0pyfb119hwKz/2NPT3QIfnyld4XNskoEHoKMHdbOXL491wWftIANqnWrAtqHB97gd/hnQWdUMODA2vO6h8cG8NtcG3/d1qUvG/X/29wZ0e9SAklz7vvjJAh+fKV3hc33jYOe6/BcGFIU6KALp1qtSHz4Ig0qGCYPQe9rOgs+DI7gRbxrQ+e4yeBwYE15fHeHhZ8Y66OUDgnXoPY0GEVgIMr+PlgQ+PlM6wodGCjw90dYFHwTP16+UvaftcMhYZSsFP2ssoxYaFRHUVOfUCQKIp/vJAh+fKd3gM+3jYAeLdXSDUxwsr9qpE459xz5d+aiLHT48HsZSQYK+oYl9m+ooB6dMKkwiwYemaDRtoukmvudlI8EH2w1B5rQNv1jg4zOlG3xo1LNtmD3PybVOCU7Nhr4RnIbQdO1rBV7UMdVyWAatwodAgGnqNEeFGcGpR4eyMjiFIiBY6y+PhQMiEnwQNq1blAGrvPDBPBz9+HnqJfDxmdINPndeE+xgtM4TyQgVp1EOjZ4IQAgeXo46vQofp3IIDkzHxWOCGB8Jkd0AEAk+aAQW5ZcXPhQH980tP9Ut8PGZMhk+z/zOPm1CUyd99bHge1yDcSrH13ywDi9HwEH4EIjcIMLPcvHtONVTIZgI+Pj5rJfAx2dKN/jQtGt2L3ueaqvTXlN2SprnYQzMU1/zcip81FP711xm96xeQUhgvlvnLg98qA6NpgQ+Ah/fKN3gg50PRz98+uNk7NQ1qtvTqZPStAtf83KO0y6Hcnx7NArieejyTLv42a7ywoc+j0y7RBWmdIMPmtZsln5hz1NNi788naBEkKAOzuvS4jKlOZVzqoMLyjwPrQJATXeDz8ddgttT96G88HEDn58s8PGZ0hE+aJp+kXHqQ6/V0/A49cI0HCk93qqsjNpBCRqY3qihAb+6MNjh+al2tVyHO4Om2FSGAIBp111hwOmnhG8PwclBwz8LGWNwkJQXPnglNMLTr1MutMDHZ0pX+KjG09nje0ZehMb7vxb2safzMlM+jByHPPqt4Kl0nk7GPLdYCAGnNSadRmhGmjL6wQIfnykT4OM3V/TVxjRa4tM9v1ng4zMJfFLTdLEhT9dhhJ3bIrifLPDxmQQ+qetEPwLEzZGmh36ywMdnEviI08UCH59J4CNOFwt8fCaBjzhdLPDxmQQ+4nSxwMdnEviI08UCH59J4CNOFwt8fCb1CxOL08WGwCf1xb80sTgdbAh8Ul/8SxOL08GGwEckEolEIpFI5D/xMwterDO2ZcOohHFLAkaJLc+LIRhX5CyzvYttbebFIFMnkSnbgeHBOmNb9jl8SnL17rct36NL91vgI9Ih24HhwTpjWxb4hFvgI/KzbAeGB+uMbVngE26Bj8jPsh0YHqwztmWBT7gFPiI/y3ZgeLDO2JYFPuEW+Ij8LNuB4cE6Y1sW+IRb4CPys2wHhgfrjG1Z4BNugY/Iz7IdGB6sM7ZlgU+4BT4iP8t2YHiwztiWBT7hFviI/CzbgeHBOmNbFviEW+Aj8rNsB4YH64xtWeATboGPyM+yHRgerDO2ZYFPuAU+Ij/LdmB4sM7YlgU+4Rb4iPws24HhwTpjWxb4hFvgI/KzbAeGB+uMbVngE26Bj8jPsh0YHqwztmWBT7gFPiI/y3ZgeLDO2JYFPuEW+IhEolgk8AlZ4CMSiUQikUgUr/j/1RGLxWJdNtT/28UzxWKxWJcFPmKxOCl2hU9pokgkEiVIAh+RSJQUCXxEIlFSJPARiURJkcDHX2pvep7pk3hGEoXHRyrtj6rhRnD/SFVD7yvktgtRZAl84ld1I3gAR2oXyufuoxYqhzIBPueaPmD6LMPeftw1QnXcVJHw6Wn6NZ4ocpfAJz7hwTvC9Fum15u+Pzy7VNhmCAlVWN5rR013+FxruMcrMMJBEosqEj6oEkNf7LSTwCc+4a/ybtPNTL9r2AFDcoIPggPTz2Tp8Sjd4YNt6na8FRjueW6qaPgUmG7OE0XOEvjEJxq9oGj65SQn+FDHihU+txrB8mSM1zn0V+3sCES13BHDPiLLDeWpVoVA5fntwkoExcvhvuBfDp9doXR1n0gEAIIxmvYXXw8NveYqMOz7jSMNvt/qVCwafPA9llH3D9uT7/8DSj7FVPefhCM3bJNo00GRYWOMwCeKeOfAzsg7OgrLEXyyjTKQLC4tEVk4tcPyaqemKYkKHyqnSk2jaSKuoVxaWqJMlM+BRp2UPoNbOT5NonJqG/E0is0hg+kIARxVOqnAsH9WDj0EvAr9aPDB1075fP/VOhST7z+KyqqwErlI4BO7aIShwoZ3NBId1KpxsZl3FjdhefXXGOUEAKdy6vSO4OAESBTlO3V42m+UWznqbLQ/buVwnxCAKKqDC8qqcCQZL3y41O2gIsGH2ukRJR/3H/eBT51whEX7SzH5/pOwrMAnBgl8Yhd2fBy5tFL8pBE8ELuXFbOEafQL/GbofffS3KCcpkvU8fLine truncated
+# Interface design
 
-[image6]: <data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAdAAAAHUCAYAAACKzHkWAACAAElEQVR4XuydB1wUW5q3Z77dmdndibuzOzM36DXca85iwAiIZDEAKqJiwpxzzllERUUlGlCM16yIIkGSmHPOCkoGQQWhn6+qoBXLcFFE6dvn7+//6+pTp9Ir3U+/p8459ZvfCL0hhIRKWNnZ2bx8+VJZzs3N5cWLF8p7uTwvL08pf/78ORqNRlmfnp5OZmam8l5ISOhtyZ8X+fMjS/4Mye+1r4XryGXy5ygjI0Ox9vP2qVLzQ++lDpCQkJCQUOmWDEctLGVAZmVlKa8yVOUfqNrXzy01P/Re6gAJCRVX8fHxDBgw4JX79++Pi4uLstyvXz/69OmjlMmW3/ft2/eN+sLCuugRI0aQmpqq/jiUiLTZpjYTlQEqt97Ikl+1y59ban7ovdQB0kXJf0zyH5Hc7Kdrfvr06aummF+LLl++zG9/+1v5w8Zv/+23b/n/vaNMWFhXLf89y3/rf/jDH7h//77641BikjNO+btP+/2nvU1SklLzQ++lDpAuKjQ0lI4dO9K5c2ecnJx0yl27dlXO+9ekK1euKAB13biAiznnOJ15Qlj4V+urXGLEvGH8+c9/5sGDB+qPQ4lJBqicgebk5CjNtV+iz4CaH3ovdYB0UcHBwdjY2CjNJ+oMr7T71q1bdOjQQX1JOi0tQGd5Ticq4RhhD48KC/9qHZsazcAp/b84QGXJ2eeXyDy1UvND76UOkC5KC9CSavcvSd25cwc7Ozt1sU5LAFRYn/w1AfqlpeaH3ksdIF2UFqBf8pfY59Lt27cFQIWFddgCoHosdYB0UQKgpUsCoML6ZAFQPZY6QLooAdDSpXcC9NFRjidHEpMkXBxHJ0YQ+iD4rS9x4a9nAVA9ljpAuigB0NKldwH0yK1DjJg9nDELRgt/oscuHCPFdIYCUfWXuPDXswCoHksdIF2UAGjp0rsAeujGQarWrEblimWoWa2C8Ee6Xq2f+Nc//k8BaYyUyau/xIW/ngVA9VjqAOmiPidAk0KXMNRtL68mwdLksX1qJ2ytLTA3N8fcypmDd9LIS7nF9c1jWRKRUXjzj5Y+AbRilSrMneLA7TPzuXl6nnARffvCQk4cnUK92j8xZr4AaGmzAKgeSx0gXdTnA2gq8xpWpYbLWM4nFky6nJfNINNvWHjgLklJSSRHe1Cr6zye3TvBqWnVaLs+4c1dfKT0CaAVqlTGdZajFGYvSPYULqqz/Lh9bj61qlcUAC2FFgDVY6kDpIv6bAC9tgCjgQcInt+Pqfsv5pdpshlsVRH/ywV1EnZT3nAgcfdPcGJ2Xez8E19t/inSN4AumumIRoJCbuIa4SKaDB8lExUALZ0WANVjqQOki/pcAA0d1pZF16SFuI00GhVQUKphYvPfU7ZKHerXr0/tCoZM33mWvIcniJlcUwD0HRIA/bwWAC3dFgDVY6kDpIv6PAB9zhKL+rTuP46xg2z4t6qjySxYM87oTxhaOdGjRw96uszlRhq8fBBLxIQaAqDvkADo57UAaOm2AKgeSx0gXdTnAOiz85uoZupIWFg44cei8OpanaGhGuKfahhsXYG1Z9+cqDn73nEiJgqAvkslDVBSvCR7okl6XaYpKMt7R33ZecleSn3tq1KW9P76Rbenck/3ffspfLxPtQBo6bYAqB5LHSBdVPEBmse+6daYj96NFpPJO50oY7qMVZEPGWD8f7gfS39ji+w7UYSNKoel35M3yj9WAqAfYQl4pMxniFFFatp0ZWP4YglOEjhfrGShWQVqm9lwN2WD9GvIR4KafAypfoavtLyGwJkmTN68jCPjqjLxgC+axJVsXuTIweilkLUWnkr1nkuvadL+Mvykfa5VgJybIME51Sd/XZZUJ2l1wfnIx10PcZMwr96UMKkeyd7528p1n8n78GL/gDqM3bYcTdqaTwapAGjptgCoHksdIF1U8QH6krP7fybibsrrouzb+PsFsCM2gaj967kYn/N6naTcjMfERQdw8Fr+U+E/Vb9GgN64caMEATqXYa1McRnUmJ6+EyQw+cGpfpj3bUU/q3Y8SliE18whHApfhCZhOfu8B3D+6gqOLLRiyvTuOLUsi2FvZ+Zvm8/pXSM4d8WNkOWd8VjsTL/e1uw+PJsL3m1x7NSW4FMSXJ96kxY9mvE9jRg+xZm4RAnQqTIoXVk6xJypY81pVNOUiBQJsremcCOgA10djRkyrBtxEmAPD63PxB3LJTALgP5aLQCqx1IHSBdVfIB+PX1tgEZFRSnPU/1cDg8PZ926dfIHjakrJ5cAQOcw2NgKD3czWjo5kpjlzfHZ7Vm03YWhZm15Ej8R2zo2zPGfDgluTO7Zit0xywieZ8Y0vzlsGlSVIf5LeXB9Ca4DWuMfPouxBj8wZMUM7qyz4O/1m2M9YxSn17THamBvsm4toK9pTdbtm8XOsY3oMGUEEVJmu2tQQ2ymjOXBvu40LGvAqcduuHczwqKNA9ceeRLQryoDNi4ickwDAdBfuQVA9VjqAOmiBEA/XXXq1GH48OGMGjXqs3n06NFYmFkwfM4wohM/P0AHtTDj5+hheHRrRMAJN6YOsef4yeEMMLWVADoJu4a2LNg0AxLdmNHPnL3HJYDObc30ADcOjq3BhL1evHjkjttgczZJAJ1pXZNNZzagOdeXzrY92H9zBZlH+uIwSso4Tw6jdydbHjxZT3rsCOyc7Ri3fhoTLVsRdN8D0ucysLmZlIFKWentedzbNwjf1QOZ37MOvX3nETVWAPTXbgFQPZY6QLooAdBPVyuTVpTEk+zX+a1jwOR+JQTQ1kTfXEBqYDsshndj9BAH7t4YT/9XAG2H65aZEOfKGMfm7It1/yBAF7Srzf5rW8g97YKdTTd+vrycp4ddsB/pTLwE5j5agB4fTofuEkA3TGOCRSsO3csH6KDm5sQ8dGV+t6aYmVmxedtoVgyo+8UBGh4X8laZcMlbAFSPpQ6QLuqjAZp8Etf5c5k7dy4+O6O58mrevqIr4WwQ2zcGka5JZe+2fdzPLOKxVfraADVtbcqDU0GsmJsfD9lu7h4cPH2fW+H++PwcTHrBpEyy0q8E4rk2kPTn+dDNOn+QhfPyt1vitQ/tneI1q9YoXyolBdATcW5w2YUKteszakI/Eh9NUACaEDcB0xotGL5aAmjCfBwb1eHAqeWvALpvdHVmBq+XYLaCxYPyATq/bW32Xt38ToCmnJ9Cd6umXH2yibjtDhh1cWRp5BJ8OzfHLXIVxE+mfa1mRF+fSddWjWk0aJJ09Ts5NKgevdbNJ/oLAVRuKs9/Yot4WsuXtgCoHksdIF3URwH0xgbqGdRk0ro97NmziwltTajWcgZx6nq/oFOzzKnwp9Yc1zzj8rnLpGYXosxH6EMAffbsmbros6uVqSlPbpzh8B45HnvYe+AQQ1r+D108b7LH5U/8ppw1gTde/8LY6vT/+M333bmWlMuj42uoXMmGTT/nb+vZrxF1Bi4jVarnWWIAnY1LQ2MiUj3JfuGK44/fM2WzG5rHY+jZxIK7GX5sG1gF40aVsbNqhpmUKe4/sYygmcaM3+DKmTm1+d7WgZFe01k5xJT1oTOZZVGdXVekDPRUb2xad2HbJQmgh6TlIV3Jlo57YEZrjBpXp0XzOvhHrObp07VknexLk5o1adOzGVY/mnA8dTmbhjbHosyPNG5hyqLRTbGfMYHDo+oxZmsJ9cJ9dJSI+DBOpMbgHehJ7ca1mb92LtEJ4mktX9ICoHosdYB0UUUHaDaetpXovOX1/LWZ57exc0hbdibB4cGGlKlUixZ9l3D16laa/LMyVSpXpUyF+niEPCYvL5WVg834oVJVrJsZ8be/dORa3i16tOnJsYQXnPWfSJ3vK/FjxXK4n0gmowhMVQM0JSVF6cXq6OjI//7v/5ZI82phmZiY8DRTO2WElJxv6kkV65XIfYsPjCxPjTZjcPeP5lyuVJAdSpfydjQd3p/Lj16wpHsdZuwr9IXxIo5B5ob4nkzEa3UJALTAeYW2l/eVpx3XWVAul2lhpayX18ljPuVxoIXGfmqXC5dpVMvafSjHS5L3+3qbwsfQHl8pS8hfVs6r4Ljqa/gYqwEqf2HLEN17cRdjFo6ikXFDpdPWv77/lxJr0ZT7ZS0AqsdSB0gXVXSAnse2ohVHlHbGNDZPGMGAYWMZM3sZUfEaDvc3oM7c/ElvT65zJzI7f6u4vRPpNHobT66tpm7t4TyVyoLGNaPh/1pyJe+2BNBehN26Ri+Diux7ApqTrlRq5MzAQ2naA79XMkC7dOmiLK9atYoGDRrw7//+78oX4o4dO1S1P79MpQz0aUY+QNNPbKBpo04cLxjyumd4JZwmzMdqiBsjj+WRsc0O28WH6e/izPn7DxloWpvttwrtjBy2jnJgxIYzeHt6lhhA9c2FATrRbQKhD47Qb6ILZcp/r/ydaL1400IuZJ/lVMZxybHCX8iX8y4wfM5Q/vKXv5Ce/uZ48V+b3qSHkJ4B9CoOlRsRoMx9kMOjK5eICZiLm93/MjRSQ8iAegzcW5CNnVtJjaqVqVypEuW++z86uYVyP6A/9R39kI9ybdcM2lfqLgFUzkD7cVjKWG2qWXH2mQTQ1Bt0tbfEbPWd14d+j+7evav0hJXn2ZX/O7SWP4z29vZ0796dbt26vdfy+o+1s7Oz4l69elG9enWyNXmk7B7EQGcpEz//OhvdPeRHxu85RR/HmfRfuJtFFsasO3mZrt2duPAgjfF2NdlyvaBynpwpZ+PV14IRARcEQD+jtQCtXeNHuvRz5H/+8T9v/K3I/v1//B4zu9ZYOJhJNhf+grZ2tKJK7cr87ne/w8HBQZnyU/sZ+1RrP6elLaMthA4hWeoA6aKKDlCIHGtIpYFb0d5dTAycxhSzPzMkIh+gQw/kSKUJ9P/ub7jdz69zcmVXnKbsI+GCH63Npynbxiy0o9m37QoA2oLine truncated
+If you use vLLM, then you know how to use vLLM-Omni from Day 0:
 
-[image7]: <data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAQwAAAEbCAYAAAAvX7qwAAA00ElEQVR4Xu2dCXgUVbbHC5AgoIgITkIIwYERFB0RFxj1jeuMy3MBlTcgIouKKIIL7jMjqOMyioyCDqCCAgqIIIiyCzQgAsq+LwJhFRDClrAn59Wp7pPcPlXVqXR1JVWd8/u+/5fue2+dqq6u8+97by3RNEHwmH27D0IQpVOOfxZBEDyGJ2JQJIYhCKUAT0Su7D2H4MDeHNj/22FTXWlKDEMQSgGeiKrQLDZv2ApzZ82DxT8tgwO/5ZjalJbEMAShFOCJqOrA3lwITZ8DXbs8Dm+98Q4cOXjC1Ka0JIYhCKUAT0RVB/cdgTGjxsFlTS+Hdm3bw/GcPFOb0pIYhiCUAjwRScdyTkGu3qP4+qtv4IrLr4T27TrCidx8yDlwDI4ePgl7dx0wLVOSEsMQhFKAJyIawq/bfoOrr7oGxo7+Fr4bN9EwjA4PdILfdJN49ZXX4c7b74ITR/JNSWwlnAc5lH3EVO5WYhiCUArwRMw9eBx2bt0DVaueAYM+HgITvplkGEbH9p1g94690PnhLlC/fgPIPwGmJLYS9kT27Mw2lbuVGIYglAI8ERNpGAf35cKUid/DZZddYbw/lH3U1KYo4fAH4/ByMQxBKAWOHT5lDBl27dgH3+nmsH/PYUeGkacbxsRvp8DkCdOMYQclMl6vsXTRSuPsypaNO+CrkV+DvprI9Ry5xjpwyMMNgITtjurbhGdk8PqPlnfdDaNGjIHD+48ZZnZ4/1GjjRiGIJQCbe9rB8OGjIANazcbib12xQbYtX1vkYZhZGy5csYymNjYC8C/aEBdH30cMjPrwZBPPzcmTbENJvz2LbuM1yuWrDHaolHRBWH4F7Vx3RZ44P728GyPF+DUsTyoeFpFePvN3pB3Kh9efP7v8MH7/WHT+q1iGIJQGqRUTIEX9ETctH6LkczrVv5SbMPAX340hmmTZ8Bvv+6HJ7s/Deef3xC+GDYiyjB2bNltvF69fD3M++Fn+GTgYN08Vhtmgz0H1Mqla6Hy6ZXhkj9eApAHUKVyFej97z6Qn58PV17RDB5o18FoI4YhCKUDPPvMC3EbBgr04Qleq3Ff67aGGTz1RA9o2LCRbhgjLQ1j/epN8PcX/2ksO1TvhYwbMx4ubXIpdH6oi7EdFcqfBhdecKHJMC5t0hSwRySGIQilR0IMA9u0bXO/Y8P4x0svG8sO+2w4jB093uhRPNTpYWO4IYYhCP4FntCHEG4M40TuKXj6qWeMxMaJzqIMY62+jqGffQF33dkCpk2aYdyvMnXSdFjw4yJYtWyd0SazbiZAfrRhNNSHOffc3cqYAxHDEIRSID29Drz26huwcV1WOJnjmPTES8i3bNoB2zb/CrkHjkO3rt3h97+vD0OHfGFpGCuXroFft+0xTArXhZOdOAmKWrNivW4W9eCG626A/JP5hYaRlw+33HyrHvsJvc0GMQxBKA0wiXfv2Gf0DO5ucbeR9Hja04lhtP7bfXBfm7bGaU4STlwOGTwMnu3xPMwJzTPuRdFXY5gBToje0/JeY9iBJqGejiXhhV54WvdX3UjwqlM0pTdff9s4+4LmgtuKbcQwBKEUUK9/OHLohCEn12HghVt4dgTFkx7PeuToPYocvbcxauQY0PSkxyTHthjfyijsdCwnzzj9yu9dEcMQhFKAJyhd6Vmp0unwyUefmQzjoQc7Q7165zm60hOFvYL1qzcar4tjFEVJDEMQSgGeiGQY5cqVh4H9B0UZxq7tv0EH/W/t2ukAJ50ZBg5Fjuc6u1GtOBLDEIRSgCciCec09up/6fb2Du06GvMIO7fuhq2bdia0txCPxDAEoRTgiUgybkvfdzTqeRj4AB2a3OTtS1piGIJQCvBEVIWnS7+WJ24JgkDwRFSF8w/zf/gJev7zFRjw34+N05y8TWlJDEMQBEEQBEEQBEEQhEQxU9M68DJBEAQTIU3L1wUoViUIglBISDELUlQDQRAEBIchIWYWKL28Y1RDQRDKNiGLngVXQWNBEMouIQdmQYosIgiCYJiHySR0l5ArMwVBMBMSwxAEwSkhMQxBEJwSEsMQBMEpITEMQRCcEhLDEATBKSExDEEQnBISwxAEwSkhMQxBEARBsCEjozaIRH6XfqiW58euUArwL0Yk8qM0MQx/wL8YkciP0sQw/IH6pUBIE4l8o8y6Yhi+QwxD5FeJYfgQMQyRXyWG4UPEMER+lRiGDxHDEPlVYhg+RAxD5FeJYfgQMQyRXyWG4UPEMER+lRiGDxHDEPlVYhg+RAxD5FeJYfgQMQyRXyWG4UOSzTDWDDX+gVCUFvQ3tyuuBj2nQdPzzeVuNKefBuXKaZAzKRxf3earLtKgzY3mZVA1qoW3BZcbzJbjmtpbg7uu0SB/pjmO3yWG4UOSyTC63W1OGNSIl81tiysvDAMTv+rp4dfcMEgpFXVjmBy9XGaqBi0iJlCUYaCpoCm93N68fr9LDMOHJIth3HFVOEGGvmSuS4QSbRjYu6BeAr6n+PSe1K97YeLzGFxHp4bb5rNyNBU0jaD1MsQwfEgyGAYlCi+30/Q+4S4/6q3O5nrUzjHh+msvCb+3MoyT0zW4+YrYcbDXQ3E2jywsx97F7rGF7+0MA9Xjb+Gexvz/ht8PeFqDsf8yt7MzDIoRtF6GGIYPSQbDwGSjrn1RqnGmBhf/XoMNX4R1XROz2eCvMZZhUmIbfI1SDWPTiHDZkkHhNndeFU5qXv9m53A9zq307FCYzOW06MSOZRhoTBgLhyH4/s6rw8nPewyxDAPr1O0LgsQwfEgyGMbT/2f+9bcSJZ46J8DL6L0ajyYnqQzbYNJSAqvLURJjvZ2JYVuevE4Mg9Yfj2EUbJ9FnV8lhuFDksEwcP7CiWHYGQtOIuKvP74uSFw20agOSXr8X8RkWHKjqdAwg85+8HWhsPfBzcRrw0AFbR5DDMOHJINhYLLxX2wuo1dwVXSvgETlmGjqa74ONWH1XVcwD6Jqfn9zb4OruIZBw5vh/yxcvxiGUCokg2FQghZ1vYXdXAf2MIZHTr1iLwTbqBOSNAQhw8A4OAehtuFSextcVvMJsQyDTouqw53iGoYMSYSEkAyGgdI/ipHoKz8z15Hol5qXq8mP8xXYhgyElqOLpagNJrHahgvbv9zBXI6i5FXL7AzD6rRqPIZhZVJ+lxiGD0kWw0DpH6dAODyg1+qFWzjkwDK8irLjreHXnW6LjoPJjuWN6mpw2fnhRO7bPXr+g8wHk7DzHeF2akJS8mIZnlI9+8ywgRScJWFzHGgY6vaTcHluAPEYBl6LgVd88nI/SwzDhySTYZDwGgq8JDqXTVyqmvW+BksHmct5m1gxUHjKdOK/7dthvVUcNAi6WpMvk2hRD8luiORXiWH4kGQ0jKAIexlWk7CJlnpKOEgSw/AhYhilJ7z6kl/AlWjhMIVP4gZFYhg+RAxD5FeJYfgQMQyRXyWG4UPEMER+lRiGDxHDEPlVYhg+RAxD5FeJYfgQMQyRXyWG4UNUwxCJ/CpNDMMf8C9GJPKjNDEMf8C/GJHIj9LEMARBEIRSJT+k5fPJs3jFYwuC4ACeSG7kdVwxDG/g+yZugQwtkh7Tl+5CXscNkmHw9cUt0Mp5ErdnYXKb6uKVGEbyY/rSXcjruGIYWuLiimEI8WD60l3I67hiGFri4ophCPFg+tJdyOu4Yhha4uKKYQjxYPrSXcjruGIYWuLiimEI8WD60l3I67hiGFri4ophCPFg+tJdyOu4Yhha4uKKYQjxYPrSXcjruGIYWuLiimEI8WD60l3I67hiGFri4ophCPFg+tJdyOu4Yhha4uKKYQjxYPrSXcjruGIYWuLiimEI8WD60l3I67hiGFri4ophCPFg+tJdyOu4Yhha4uKKYQjxYPrSXcjruGIYWuLiimEI8WD60l3I67hiGFri4ophCPFg+tJdyOu4Yhha4uKKYQiCIAiCIAiCIJjgjxQXiUQiVZr6bwZ4pUgkEqmyNYyCQkEQyjS6H+SJYQiC4AgxDEEQHCOGIQiCY8QwiuZdXcd5oQLuHxK2GxpdXWw6aeFYVXhFKZDI7bhG1yIten9xYX1VWsCC07RwO/4+6qpKl+TrGssLhTBiGLE5XdcWLfog5fCDHjUoqkXxSFbD2KfrXM28r1T5wTB6a2HTSGTMpEEMIzbjtfABiabRmtURPKnQZHhZcUhGw6DeBSdLizaAoigJw0DwO5BehgViGLGhXz084O2GJVZJhWVXsjI70nTNjaiFZm8YXbRwm+ct6pAbtMI42EYFE2twpC6kWS+P20HrwO1AYm0HSq3DdQyMvKZtQc7TCnsXnCzNbBgYZ7IWvR1qXVGG8Y6uD7TCnooaj+8X/Cy0jkm6mil10suwQAzDHjSJHF21Iu+tkgfh5ZggTvcZrgPb4kHbQNcbkfdqTIyXrYWNC9tQr0c1JKxHQ8N6VF+ljraH1oHJjO/VHhNtx/JIG6vtQNTtaBSpp+2g5MU2IS2cjAjOAdntjywtug7NEj/HJVp4HbQNRFGGgcMJ1djpszfRwvG+YfX4frgW3uZWum5X6rK0aAMRNDGMWOCBrnZL8aCyGpaoSYW/0tgOE88JmPy8q84TlQyCwCTBMto2SprGBS0KobZ8HVSmrkN9j5CJUBlPVsRqO3hXHtvY9c6ytOiYf9ait4HMRu0t8H1BhvF65HUbpR4NQd0ebjBYb7XNCNapsQRNDCMW+Lmpd4Hgr5/a4yCwnap20dXGQXorE4Htea+FD0nwNU946v0g2J7XE5RwfB00z9Ir8t6qDaKWYyy+HnU7eDITOByhNpwszXoZBHswX2nhehrO8HXQezK3XkodgmV8EhWHGhSPDMVq6IG9lVd4YVlHDMMa6spaiSeNmlT0Xh0u2BkGP/gJ1TCoDYrmDUihSL36K8/hvRNC7Q3YbQeibgfFstoOxC5OcQyDfvGxR/Line truncated
+<p align="center">
+  <picture>
+    <source media="(prefers-color-scheme: dark)" src="https://raw.githubusercontent.com/vllm-project/vllm-omni/refs/heads/main/docs/source/architecture/vllm-omni-user-interface.png">
+    <img alt="vLLM-Omni interface design" src="https://raw.githubusercontent.com/vllm-project/vllm-omni/refs/heads/main/docs/source/architecture/vllm-omni-user-interface.png" width=55%>
+  </picture>
+</p>
 
-[image8]: <data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAVwAAAGJCAYAAADG5YuwAACAAElEQVR4XuydBXgV19aG23tv7//f+u3f21svFHd3d5fgkgRCcHeXIIFAkKDB3d1dgwa3lhYo7m4tUOT797fCnJ6chJaE5JCTs77nWU8ye/bImVnzztpr75l5CyqVSqVyit5yLFCpVCpV7EiBq1KpVE6SAlelUqmcJAWuSqVSOUkKXJVKpXKSFLgqlUrlJClwVSqVyklS4KpUKpWTpMBVqVQqJ0mBq1KpVE6SAlelUqmcJAWuSqVSOUkKXJVKpXKSFLgqlUrlJClwVSqVyklS4KpUKpWTpMBVqVQqJ0mBq1KpVE6SAlelUqmcJAWuSqVSOUkKXJVKpXKSFLgqlUrlJClwVSqVyklS4KpUKpWTpMB1Ez1//hyzZ89G+vTp8d133yFlypSoWrUqLl686Fg1zmjJkiVInDix7G/r1q3lN/yZqlWrJnXtrXTp0o7VIojHJUOGDFI/VapUmD9/Pq5du+ZYLdaUMWNGpE2bFvfv33ecpYpnUuC6iaZOnYoECRLgrbfestk//vEP5MqVCzdu3HCs/sb1448/IlGiRLZ9/fDDDzFy5EjHauH07bffhvt9tKRJkzpWC6d58+ZFOC4JEyZEvnz58PDhQ8fqsaJ//vOf+Nvf/oY7d+44zlLFMylw3UATJkywwaRGjRro0qULatasKREdy/7973/j3r17jou9MS1YsABJkiSRffP29kabNm3k5kAonTlzxrG66ObNmwIuRsT8fZb9GaQZQVvHpVKlSlK/Vq1atrKvvvoKjx8/dlwsxjVmzBgEBwc7ZVuqNysFbjzXgwcP8MknnwhA5s6dG27ekydP8P3338u8Dh06YOXKlfJ/y5Ytbc3b/v37S1n+/PltyzECZBmb+RSb5O+//76U5cmTBz///LOt+U94spwwIzj5/7vvvovy5cvj9u3btnXaq0yZMlIvderUtjJfX18p27Ztm13NP7R161aZ37x5c+zdu1fsz5rohNvXX38ty/CGZK/Vq1fLtjmvR48eUpY7d26ZXrp0qS2S/u9//4vz589j//79tmMcGBhoW8/169flRsEoe/HixfK7WYfbPXjwoK3exx9/LMfv7t27tjJV/JQCN56LMOBFzmZzZLKASKDwgv/ggw+QLl06LFq0SOYToBZc2MS+cuWKTNMOHz4s0Zk1/fe//13+/u///q9EihTXwzJGp/Z1aFYdRzHFce7cOdmWJQKf6yDMI1NQUFC47dAYuR86dMixqujEiRNS54svvnCcJdq4caPMZ36VKlCggG29b7/9tu13MNVhRd/WfOumwN9BkFrz7Otwu7///rvU+89//iP7qsCN/1LgxnMxIuMFXrhwYcdZoj179sj8ZMmSyXSOHDkEDJ6ennj06JHAwILF8ePHbemJzz77TKJERsisY0W0zIlyPoHEnKQF3G+++caWoyTQWcZtvYp69uwp9QsWLPjSjrPixYtLHUbdY8eORf369WWa0Xhk2rx5s8zPnj274yzRsWPHbMs/ffrUBlwrqqesqHbYsGEy3aRJE5m2ImYLuCzj9ij+Ft68mP5gBEwpcN1HCtx4ru3bt8sFz86xyLRz506Zb6UMrIiVAN63b5/8X7duXfnLPCNHAjCiq1y5sqQk3nvvPZkXmbHZbAGXALfk4eEhQH7ZPtmrVatWsjwjTSsifFVZnWGMlh1l/TaO2ohMR44csR0HQt4CLlMVltKkSYN33nkHFy5ckOk+ffpInUmTJsm0BVymDCwtXLhQ0hX/8z//Y+usVOC6jxS48VzMMRIKbAZbOVNGYhMnTpSOJquTaOjQoTKPEGBdRrT16tWTeadOnRJwMO3AiI+Q2rBhgwCQwGWUy0432m+//SbljH6fPXtmA26jRo1s+8T87asAl5DmsnXq1JF1vUycR4hzP7h9S1YkffLkSbvaYeLvZE6Vv5XHyF6MxFu0aCHLcr2UBdyjR4/a6hGcPLYW0CMDLo+PfdqCKRwFrvtKgRvPxeisUKFCAgJe6JcuXZJOLk5b5ti0z5Ytm20eYUBVr17dVtaxY0dbj3qKFCkEztOmTZNpNueLFi2KEiVKSEoiOsC9deuWdLJxOXbavYoyZ84s9fv27SvTjNwZiX/66acvhXXFihVlGUbCVq6XoyA4Rpkg5jxGwpQCVxUTUuC6gWbOnGmDJWHHiNG+84pDqUJDQ231vby8bPOsHKefn5+tzH60A1MMLCN4CVVrBANBx5RDdIDL/bXG4FaoUAGNGzeW5WkHDhyQOuzU4vSmTZtkmjcBC54sz5s3r0wTqi/TihUrbL+JNyUuxwclrDKORrDSGApcVUxIgesmsi50CyY09rCzM4v/sxPHGkbFIUzWOFhrmNOOHTtsy9k32wlVC240RrvdunXD1atXZT7hzHL7EQnswGMZn65yFDuoPv/883D7aW+zZs2SetZwtQEDBtiWK1myZLi6HGHxV+OLV61aJSkS++V48yhVqlS4Bx+sCNp+1IM1PMwaG9y1a1eZZh6cYqcYp5m6sMT9t5azOs2slI8++BD/pcB1IzG9wIuaTXb+tfKtgwcPRtOmTW312AT/9ddfpZ4V4XFZ5oBfFoWxnPUdx75yeZZzfZY4NpjrigyG9vsYmVn7w3QFp/nXXtZ+RAVejsclshQEfxfnE+yWuC3+Dqs+Ac06VrrFOmb2+8L9t5azRlxwvv20Kv5KgatSqVROkgJXpVKpnCQFrkqlUjlJClyVSqVykhS4KpVK5SQpcFUqlcpJUuCqVCqVk6TAValUKidJgatSqVROkgJXpVKpnCQFrkqlUjlJClyVSqVykhS4KpVK5SQpcFUqlcpJUuCqVCqVk6TAValUKidJgatSqVROkgLXiJ+DuXbtWpw061M1KpXK9aXANfrXv/6Fd/7xT/zj7/+IO/a3f+Cdv78jHyFUqVTxQwpco0/+/QmWHVmMvfdC44wd+HUPNp9bj/f+V4EbX8QPbNauXdtl7JNPPnH8CarXlAIXYcCds3MmtlzYGGds6+XNchN4/1/vO+6uykXFT7+7kpImTepYpHpNKXChwFU5RwpclQIXClyVc6TAVSlwocBVOUcKXJUCF+GBu+vmdoTe3IHQWzuw68a2CCB0lilw458UuCoFLv4AbsilTchdKBcy58iEDFkzoGTlkthxfauUb720OQyE5q/1v9jFsLKQi5sinW8/zTr2da35XL/Mt9+OAjfeSYGrUuAiPHDzl8iH/Q9244enh9BnbC8MnDEA7fu3hVeTmth2dQtqNfeGb2sfiYQJxfl75qK8ZzlMWjcB265sQZ1WPmjQoR52394p8PRtUwc1GlbH9qshmL19BsrXLIvBMwOx49pWAatn4xroMqSTLNt5cCczXRMbzqzFtstbIgXumDFjsHTp0nBlKtfQXwH38OHDSJEiBVq1aoXOnTsjffr0ePDggWO119a9e/fQtm1bx+IIUuDGvBS4CA9cRrf9Jvqj1xg/lKlWGnNDZ5toN73As7V/SwTNGSwgbtW7JTadW48iZQtj373daBvQBu0D2mLapskYNCMQ7cz0kNmDMGrJSCw9vAhjVwQjb9Hc+OHZYfQZ10tAXaVOZcwImQafFrUQbOplz5sN60+tCRfhvvcCuIsXL5Zxke+++67D3qtcRX8F3KNHj6Jdu3a26caNG+PHH3/EtGnTkDlzZlSsWFHKL168iGTJksHT0xN169bFpEmTsGnTJuzYsUNA/fjxY9SsWRNZs2bFyJEjcfPmTWTKlAnp0qXDDz/8gJ49e+LLL7/E9evXbduKTArcmJcCF+GBm6tgTizYO1ds5Y/LsOn8BhQuUwg7r2+DdxNPgeea4yslch27YjQqeHlItLrv/m54mig4VfrUSJc5nUS6y39YigIl8iN3odySRhi1eASSp0oGv5HdMXrZKOQpkhvrTq7G8AVD0SWoE0pXLhkupbDcAPfL/34poH3rrbfEeNE2atRILsYmTZqgefPmaNmyJdq0McBv3x4dO3ZEly5d0L17d/To0QO9evWCv78/AgICEBgYiEGDBiEoKAjDhw/HiBEjEBwcLFHz+PHj5cKdOnUqpk+fjpkzZ2L27NmYN28eFi5cKMBftmwZVq5ciTVr1mDt2rXYsGGDXOghISHYvn07du7cid27d2Pfvn3Yv38/Dh06JBAhNH7++WecPHkSp0+fxrlz53D+/HlcunQJV65cwY0bNwQK/Buf7fvvv3d0vXA6duwYPvroIyRMmBBfffWVnFsqQ4YMEpXyYQQe4759++LgwYO4fPkyGjZsKOeL54DHvnfv3vK3WbNmuH//PgoUKCDnjOf49u3bePLkiRzrTp06OWw9ohS4MS8FLsIDN2/RPNhsIGvlWDeeDYtit18LMVFtc4xYOBQBE/uiVa8WWHNiFfIWyYODv+1Dsx5N0K5/W8zdNdtExbPQZ0xPDJ8/FGtNnWmbJqHjwPZo2bM5TuMkvJt5Yf7uOahsItzZO2bCt3UdjFgwDCUrFRfQ2ke4H77/IUJDQ5E/f34BboIECeQdC7zYaIx2Lly4IBA7e/asAO2XX34RuJ04cUJAxwuZ0CP8jhw5IiA8cOCAGOG4d+9euUi5HV7QhOe2bduwdetWbNmyBZs3b8bGjRsFsOvWrRPg0latWiUX8/LlyyXNsWTJEixatEgAvWDBAsyfPx9z584VcBPgM2bMkGhtypQpmDx5sgB+4sSJmDBhgoB+3Lhx8dq+++47R9cLJ54f3jQpRqKEKcXolseIx5jnkhEqzysBypsvgctztn79egEuz2OdOnXkJsrzRvF8MXrm+eWNztrOn0mBG/NS4MIOuCYKbd2nZTjgbjq3QXK4jDy3XwlBtXpVJe/KEQzMu07eMAnFKxTDhDXjJO/q2agGKvlUlBzuZrNs1bpVJP/LacKadRnpMirm/MqmbruAtrKujoHtbR1okXWaEYDM8RGYKtLine truncated
+Taking **Qwen3-Omni** as an example:
 
-[image9]: <data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAOsAAAEWCAYAAACHcWCmAAB5cklEQVR4XuydB3gVVfrG4/5XRXonjYQQ0nvvvffee++kQwKkQOgdpIogiPQmIBBAJXRUFERQV1111+7aV1d33fX9f2duJlwmNw0IIXDe53lz751y5szM+c33nZlzb5SUuLgUSH3cSAz4sxIe//MjUBmrifEq+mQjTFCzwCRNW+hq2cFgkiNM9F1hZugBC2Nv2JgEwM40CA7moXCyjICzVSTcbGLgYZcAT/tEeDsmw9c5FX4u6Qhwy0QgOcg9CyFeOeRchPkUINy3kFyESP8SRAVMRnRgKWKDyxEXUoH40EokhlcjKWKK4OSIqUiJnCY4OaJWcFJ4zS3vE0JrEB8yVXB0UBViAqsQHVCJSL9yRPiWIdS3FCE+kxHsXYJAzyIEeBTC370Afm658HHJFuzllNnqDHg6ppHT4W6fCje7ZLjbJcHdNgGuVrFwIjtaxcDRIpqOQRQcLMjmEbA3DYetaRisTULIwbAyJhsFwtIwABYGfjDX94GpvjdM9DxgrOtOr+4wM/AAnYY/Sc8LF1c7qY8dgSce/RMGPvZ/GK86CVrqppg4noFqA72J9gKoxnoMVHeYG3pRA/RrBTWEQA2Hi3UUXG2iCdR4eDkktYHq75KBAGYGqkc2gj1zCNJ8weG+xYjwKxZAZZDGBJUJoDJIE8KqBFCTIwnQqBrBqdG15OkyR8nMwBVfZRBPR2JYLUE7BXHBU6jMagK2GlH+FYLDfUoFh5FDCNhgjyK6iBTA3zWfQL0JrLdzlmAZsBnwcGBOIWhT4ErQulnHwZXswqC1jIGDZbRgewasWSTszCJgQ8AyW5uEwtIoGBaGgbAiYC0N/GEhAOvZZgsjTw4rV/ekNno4Bg34Pwx54lFoqhlCW8MaOhPsCFQH6Gs7wUjXRYgAFkbe8HKJkIFqFkxRJUSIqK4UUd1s44SI6uOUQqCmUESVQSoDNYtAzaOIWohQ7wKCpVAAVYimAaUEZhUyE6ZRRKVoGiYXSaOYa5ERV4fJ2XOQHlNHnoG0aNF15OnC+9QoGaxJ4bVURg1iKbrmpTYgO7GOQK0iV9IFokwANZRFV69iqlchRft8BLgWEKQ58CZQ/d1zCVCKrI4M2NYIy0ywutkxJ9P+JsDZmlkGrDMB60wR1p4irL15pGBbs3CCNRTWxgzWIHIAweoPawLW1sRXiK43YfXisHJ1T2pjhmMwgTp00OOYoG5MoLLUl4HqQJHUA2UFlQjwIiAdwrBzyzZK+UIJ1hDYUwRxsAiDC0VVd7s4IarKYE2jaJVBziRnERDZCKKoysyADfMpEtLfKAKVwVpTuhDzZ66VgRtW3Zb+soiaQPBWFC7A9s2HCMgZAqyyaNoaXSNkn5mTCNLEsKkCrNEBVTj94nls23QQET7lBGs5XSgmU0SdTOlwCYFaDH+3QoqshXRByRciaVZKDW1nF5KiyimKpsHbKUuIqp5kIaraJ8PFNokgTRRAZVHV0TyGUmKClSKrp0M8XbRiCFSKrCZhAqgsHbYwCIa5QZAQVaOD0jGvfh6lwF4EqhelwF4cVq7uazz1WYcPeQyjhw2iqGoupL762o706ohdW5/F06tWYW7DbJTkV+Hci8dw4vAhikxJKMgox9kXTyI3pZIibDSlvwzUdAFUb+rvHdx7BFs27MSWp3Zidt1yHNx9BG++fhUb12zH4b0nkJM0HYf3HUdN2SLMrV+FG9fewUH6/MqZ13Hj6g2CeLEQMZfO3YRrr7+FF4+cQUHaLJw/fRmXL13Fivlb8NqFN7D72WMoypiNK6+8hS3r96Ewo0lY/rOPP0bDlBUUxauQHj8DV169imc37kU8XQzOtVygZa5i6Zz18HXJw47N+3D9yhWKsJm4ePo0VixZj2kV8/DysZNYsXA9ntu4Azkp1bQ/W/HchuewZM4KHNq1H80HDlCkjCAww3Fs3z4saVqCyIB0PL9zB3Y9s4VS4mBse3oj9j37HF0oEhAVlIrFTQsoU/EmUJkZrN4cVq7uSVNlFMYOHwDVUUMwaYKV0Ec1mOSMEL9YnD5+COtXPonnqMFtWruBGvUWrF22DhtWr0dL8zFkJpehurgRjmZRlOpm3RJRd23dj5NHTqD5hRO4cPoCDuw6is0E7xuvvY6rr72BGZVL8PabbyGM+o8H9x7Hm5evEZgbCaLrOPPSRQJzJooz5+Kl5nOUBs/HX999D9uf2SdAeuPKX+gisgtvvnYNjTWrcebkBYLsNSxbsBmvE8hZKTNx+uQ55KU1ISakChfOvIpkSqk/ePd9PLlkM9at2Ipzpy7g2Q27KeXNweXzr2L7lj0UdYvwwr4XUFU8my4kB+iicJb2eTvt/3O42PIiTp9oxqJZy3D80CHUVc/FC3v2CrDamERj61MbEeydiv3btlMfNoyWW4CmaU2IDqYLmEcc5tbNo0gdicykYoLUp9XeHFau7muC6mgojxgAtdHDoKNlC0MdJwHW1YuXY+HseZhRM4Mi4S4c2L4Dy+Ytxb5tu/HM+o0ESDNB+zSyUythbRSGDaueodQxldLLLAS65xAQW/DBX25gYdNavP7KG3h+z2FcPHMJL+xvFsBjEfcvN24g3KeMouUrePn4Gaxb/hz2bDtEEW+D0B+tKlxMcL+FmbVP4tLpVykq76LtbMPf//oRlhDYyxdsxN7tR/D6xTexZumzlLIvwI0336Z6P4WjB19EZmIDYkOn4pVzr6O8cA5t/1U8vXYH6qcsp4vDm1gyewNF0zxcOHUexw83o2nGciyb/xQBXoKP3n9LiKSXL5xHTfkcvH7hDDau3oiE0AKcOt6M3LQqvEhZhr1pFMEaiYM7dyImNBeHCeDwgDRs37QVM8rrkRabh+KscuSnT6Y+bwQBnQAzfT8yh5Wrh9JWH0ugDoSm8lgY6DhSf8pVeKwQ6hsPw0keBK8nwgNTEOafQpEhGjHB2fBwikNSdCE1zjyE++XAzyWVQKmjVwYq9VE9cnHsUDNFzjdpWib1PStRUTCPUuY6RPoXUzRegHL6XJI1W7hzW1W0kFLZJmTE1mNew3qKSKvpfR2yExtRkb8QCxqfRmZcPXKTZmH29KcoKq9Cbeky6v/RxSJhJvJSZmN+w9PIim9Aac4CSq2XE2ArkBJTT/3EqRRVZ6CBIjB7pJMeT9OCqwigWYgNrkQI9V+jg8pQmDkTb1+9RhlCPnyc81CUU4/IwBIUZs2Al1M64kInU9l1lMoW0r5PpmkpVG4J9VkTqBsQT/tYQOuxm2tJlOJTH9knnfr14UiPK6F+eA4s9APh5xaHMN8U6sP688jK1XPpaSpj/Jgh0FbXbLvzy25+mOixGyA+MDf0haVxIGyp/2VnHkYpXgScrKPhbBMLN1v2XDUJno7J8HFMRUBrVGV3fLOSa6lhFlGaW0TRs5hcgki/ycLNnpjASsQSOAzU+JAaJIWzu7nsEcwMpEezu74NyIyVc1wjvcqcQZ8zYhppuQakRjbQOnVIiahDcjh7rad+bh3BMR2xIdPpwlJLINaQp1BfsgoR/lV0cakkkMqEG06hXqUI9ihBIAEb6F6EIM8i6lsWURqfS33ZXAKPXp1zaN+y4CXcbEqHh32acMPJXbg7nARX20S42JCtE+BiFSN7nGPO7g5Hws6U+rMmYbBij3CM2V3hQFgY+guwyu4IewmPw5Q4rFzdkcEEZWiMpRRYQ1t4nmqq7yE0JNkNEB9qXH6wooZmaxbSCmsUwRoDV5s4YZCAl0My9ftYRGEDILIFWFlk9XFmAyHyCAgGa4ng6IAyMoO1Sg5WdgeXDWyQPX5Ji64XnE7OJCiZs2JnEqgzZZCS0wjUtCgCNaKBIK1HUtgMMgHLoI2YgfiwaRJYp9JFopqiOhskUUEXkzKCspSAJVg9b8Lq5yJ77hroWgA/1zz4UYT1c6J9ccyEt30mvOwJVgasQ6oArPAoxzapDVbn1mevThbRcGSDJcwiKUUOlwFrHCw8xrEwDGiFVTZAgt1kUuKwcnVHhloq0Bw3EgZaxjA1YCNqvIVoam7A7Cc0LjYax848HPYWkXC0YlE1jmCNF545skbr5cBGK2VSfzVbcJBnLkI8CwRQQ3wpqvpORqRvqTA4gY0qig2qFsxgjQtmI4/YCCQG7HQBNhYt0yLrkRHdGkUlkKZGNgpOCmOgNrSaRVf2uY5SVhmoMlhrERU4FdH+UxDl1wqrL0tTWXS9CWuQOxskUSg8d2X2c8kX7Euw+jrlwNuBgG2NrB52FF3t2MimlDZY2bNXJ0qHHS3j2mC1p8hqR7DaGjNYQwSLwJoZyJ63cli5ui2jiarQUhkLI21rgtOTQJWlvuYUUVnaJouqYQKosqgaSw2TQKWo6kYpoAeD1TFNeOzhx2ClPmughwzUMO9i4W5vpH8ZgVompL+yqMpgZa9TEBtYjXgBWDaggUXXGW3ApkaxKNsgmL1PITNIUyJmEpgySBND69ucQPDGE6yxITMQFzSdLEbWmtY0uJLSYDaaiSIrpcChXpPbYA0gWNmzVz8hqlKEJVCZb8KaRWlwJu0vG4KYRqDeCqsIqqNFnDCayc4sWoistqZsRBOLrqFtwAqw6t8cHKHEYeXqjlhknaiqAmMdWwK0NaoSqGZG/kI/i0VVNiKHgcrsSLD6O0YhwDmK0uAkYXSPj1MafF2zCNYsgjXnFlhZ+hvlz6IqS4Erb0ZWgjQyoAbTs9IJ4GkCsAxW+egqAis6uS31pahKsCYyWMkM0gSCNT6URdX6DmEVQG1NgxmkIZ4M1snCIAmWBjNYRWBvwppHaXB2K6xZt8DqaptCoLKBEjejqoN5rDBgxI5AFSKrMPzwJqyy4Yc3YeWRlavbMqbIqjNeEya6DrAwZg/p/ZFs6IEDZq7wNKL+FcHKRuLYsyF1FDGivSOwOF4PJX7mcKSIwm66+DqmI8E/Fb4uORSh8gjWPASz0UreDNLine truncated
+## Offline Inference
+The **Omni** class provides a Python interface for offline batched inference. Users initialize the Omni class with a Hugging Face model name and use the generate method, passing inputs that include both text prompts and multi-modal data:
 
-[image10]: <data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAXYAAAFiCAYAAADiGfPoAAB7FElEQVR4XuydB1gV19aGc2tuv3/KTUwxsSb2rthFELtSFMUKduzYC1aw94rSRFBBUUBFEVEEsffeYktssbfEWKJ+//4WzvF4wIIVcH/Ps55zZs2ePYXDO2vW7PIBtLS0tLQylT6wdGhpaWlpZWxpsGtpaWllMmmwa2lpaWUyabBraWlpZTJpsGtpaWllMmmwa2lpaWUyabBraWlpZTJpsGtpaWllMmmwa2lpaWUyabBraWlpZTJpsGtpaWllMmmwa2lpaWUyabBraWlpZTJpsGtpaWllMmmwa2lpaWUyabBraWlpZTJpsGtpaWllMmmwa2lpaWUyabBraWlpZTJpsGtpaWllMmmwa2lpaWUyabBraWlpZTJpsGtpaWllMmmwa2lpaWUyabBrvbJOnDiBDh06IEuWLPjkk09QpkwZxMTE4N69e5ZF37muXbsmx2dtbY2HDx9artbSyhTSYNd6JY0YMQIffPBBqlaqVCmBfnrS1atXkTdvXhQpUkSDXSvTSoNd66UVGhpqgnixYsWwefNmHDt2DD169DD5K1WqhN9++02i999///2J7R88eJDCd//+/RQ+LtNY3lzG9gQ0zSj3LKUGdvPtUquDx0RLTUZ54zhSk3mdxvlZnotR5mn70dJKizTYtV5Kd+7cQenSpQXe/LQE1ahRo/DHP/5R1q9evRr//ve/5fuyZctkPSFYoEAB8a1bt058BNv//vc/8d29excnT56Eo6Oj6Sbx1Vdf4fDhw6Z9uLm5ib9hw4ZyYzHKtW3bVlIuqckS7KdOncIf/vAH2W7q1KmmOpo2bYpffvkF48aNk+U//elPst4c3tHR0abyxnXYvn27af2NGzfQrl0703pnZ2dUrFhRvk+ePNlULjIy0lTmP//5j1w7nr+W1stKg13rpXTw4EEBEaG4atUqy9USpWfLlk3K9OzZE35+fvLd1tZW1u/cudMEMwKaioqKkuXixYtj3759JuDyCYCAtbKykuXExEQp37FjR1Mdffv2Ffvb3/4my23atDEdi7kswX7mzBnTDejDDz/E2LFj8fXXX5vq5bFw33/+85/leBYvXiz1+Pv7y/r4+HhZDgkJkeWPP/5Y9nHz5k3897//FR/3xzrc3d1N9c6cOVO243FyuXXr1lKGuX8uFy1aVK6hltbLSINd66WUkJAgAGIku2nTJsvVknr57rvvpAyBxuibAP373/8uwDJy8/R98803EvG3atVKfN7e3ujSpYt8b968uUTfBOWWLVvER9BTBti5H0NNmjQRn4uLi8lnrmeBPSAgQMrwBsHl//u//8OFCxfElzt3bvEx/UTdvn0bt27dkjp4bEuWLJH1fHnMbfjymDeCf/7zn3LuhniDYDne6BiVcx9cvnLlitRz9OhR09PNgQMHTNtpaaVFGuxaL6VDhw4JfAivuLg4y9UCPiNinzZtmgDQSEMwHWNnZ4d//etfqFmzpvi2bdsmqZm//vWvcqNwcHAQf2pWvnx52YcBdnt7e9N+27dvL75GjRqZfOZ6Fth37NghZYz0C1/+GqkX4yYVFhZmqisiIkKOkzcr49gMsDPVwmWmiMzTN4ULFxY/I/5Lly7JU4Ll+RnGVI+W1stIg13rpcQI88svvxQAeXh4iI8Aq1evnsCSKQpC+h//+Ad2794t6728vEwgzpo1K/LkyWOK3F1dXSV6zZcvn+S2jdx6586dpT7mws+ePYuff/5Z9k0ZYGeO3ZCR7ngZsDP9QzEdw+WSJUs+FeyEP29qjMC5r9GjR8t6A+w+Pj6yzHM0fyHKfdNPsF++fFnAzqce4xz5yXOk8T2GltbLSINd66VE4BnRMSPvBQsWSPqFIKOPLzr56enpadpm48aNphw4jfBes2aN3AAMX4sWLaRsnz59ZJkpkIsXL4qPTScJxL1798qyAfYGDRqY9vEqYDfqfR7YCeoKFSrIsgHtDRs2yDLBzuPluRrvCJYuXSplmHoxfEzF8Hp9+umnsswUDMWbF1NCQUFBcoPT0noZabBrvbTOnz9vitoJR0bbRqsWw9gZyGgpQrB9//33pnVM0TBqNeBGW758uZQ9fvy4KU3BfRQsWFCgyWWjvncFdvr4Etg4Fp4X00Nc5svT06dPC/D54pU+5tl5/N9++63pPI2Xp0OGDJFlXjeW+fzzz2WZzUR1xK71stJg13plnTt3Dl27dpXWJART9erVpZWIAUOmUwzNmjVLyhByRkqFL0Ppq1y5sqkcRThOmjQJ2bNnl/VsTmjexpwvOeln80ZDvXv3Fh8Bn5quX78uECaYCWge+xdffCHbsKUPxTQKl6tVq2YCe9myZcXHljsUj437582GrXUYsbM5I8vwqcIQmzLmzJlT9sGnGsKb14TXwRCPgWW4LdfzKUZL61Wkwa71RsWUQnrrffo2xDbsQ4cOlTbp5m3bjRYvvBFoab0pabBrab0B8cnAaC3DF6wcR+ezzz6TZUbmT+tApaX1OqTBrqX1hsS27IUKFTLl1Wls/mh0sNLSelPSYNfSeoNip6MffvhB2v3TdEsXrbchDXYtLS2tTCYNdi0tLa1MJg12LS0trUwmDXYtLS2tTCYNdi0tLa1MJg12LS0trUwmDXYtLS2tTCYNdi0tLa1MJg12LS0trUwmDXYtLS2tTCYNdi0tLa1MJg12LS0trUwmDXYtLS2tTCYNdi0tLa1MJg12LS0trUymNw72n3/+GQcOHMDhw4fTrZ06dcrysLW0tLQyrN442Ldt24YCBQogd+7c6c9y5ZaJkhMSEiwPW0tLSyvD6o2DfdeuXciZKydGBAzDpAUTMDFsfLqxyeGT4NzaGevWrbM8bK10pNu3b8t0cosXL053xtmRtLTSm94K2HPkzAGfxdOw4XwS1p5Zk25s06X1cGjuoMGeznXmzBm0b99eAP/rr7+mG7ty5QpGjx5tebhaWu9cGuwa7OleBHubNm0s3e9cvNFosGulR2mwa7Cne6VXsP/2228a7FrpUhrsGuzpXhrsWlppkwa7Bnu6lwa7llba9NbBvu5cIjZcSML682tlOelsQgrgvi3TYM8Y0mDX0kqb3irYN15ch4FTPNGqR0ssO7AU/cb1wZDpg5FwKh7L1fKKQ8uw+sRKLNkThVXHYwX6SecSsOLwMsQeXi43BQKZ62NUWa5fdSxWLUdi5dEYKRt/Mg5L90YhUdXJ5dXHWV+k1JtwarXaLhrR+xYj8XR8qmDnC7HAwEDEx8ebnYXWu9TLgD0kJAQ5c+bEZ599hhw5cqBjx46WRZ4QW7i4uLhYup8pDXat9Kp3AvZeo3ti373dGDV7BDoP6giPoV1QuqIVBk31RHXHarBvXBe2tSoL0BdsCkXdRnVQsWoFLNq2AF29uqKWc02UrmAlkHZs6oD6bvVQt3FtKd+oXSNUc6wq++HNoUptW9m+qr0d5q4NwbfZvoVr52aI/zEOmy9vELDzGO/evQtfX198//33+NOf/oStW7danorWO9LLgN1QlSpVTN8J+759++LYsWMIDw/HzZs3xT9//ny5mfMGcP/+fVP550mDXSu96q2DfXiAt3RYKlKyCKxrWCN6/2J0GdwJg6cNkvVNOjTG1qsb0bZ3Gwzz90KP4d0wOXwiVv4QgzmJs1FZbcPIfXrkVFXnVNRr7iTRNyG/4nA0qjtUk2h9xZFl0gmp04AO2HFzK5p3aga/5TNRzaGqKdfPiL1eCye0bNkSefLkwQcffCBGGBw8eBA7d+7E7t27sWfPHuzfv198HILg6NGjOH78OE6ePImffvoJp0+fxtmzZ3H+/HlcuHABly9flgjw+vXruHHjBm7duiUQeJrxKeFpdufOnWcab0hPs3v37qVqv//++1ONYHuaPXjw4Kn28OHDZ9qr6Ny5cy8N9kqVKsm1iIqKQpcuXZCUlIQKFSrI37B3795YsmQJpk+fLh2N6E/LsWqwa6VXvXWwS8Q+qid2/7YD265vxrqfE9F9mAcmhU9A/Ik4tOjmho2X1sFzUn94zRiCVt1bwCdqGrZe24SI7QtRupIV7JvYo55rPSzbvwQzlk5HdadqaNaxCVafjMP8DfMkOh8ROAwj1E2kx4ju2Hp9k9pnD8xc6gOXNg0lv2+AvX6LeqhXrx4++ugjE9hz5cqFfv36oUePHujWrRs8PDzQuXNndOrUSTrKtG3bFq1bt0arVq3QokULuLq6olmzZmjSpAkaNWqEhg0bwtnZWep1dHRE3bp1YW9vL+bg4CBGP83JyUmMZevXry/WoEEDMdZDY4qAxrpp3I9hTZs2FeP+mzdvLsbjcXNzE+Px0XisNB43jaCk8Vxo7dq1g7u7uxjPsUOHDmJMYfC8abwGNAKya9euYrw2NF6n7t27yzWj9ezZU6xXr15ihGifPn0kYjasf//+Yp6enmIDBgwQGzRoEAYOHCifgwcPlv3w+ryMCHbezLhvXq/x48fLcBKUv78/6tSpI995U65Ro4b5ps+VBrtWetU7AXtnFaEb+XLmwbt5d5Xomi9VrWtUknw44cwIu8+Y3ug/sR96j+mFoT6DUc2xmmw7ft5YTI+YqsDdDbtubYd7v7YCbkb6i7YuUJ+tVYQfLPWs+XEVKtpVQHB8EBq0cn4C7EzFbNq0SaJuAurvf/87smbNikuXLlmeyjuXEf0yQmYEbUTZBBejUkbwRqRP6PBJgcZekr/88oukHmh8iuDTxLVr18SuXr0qTxh80uB5X7x4UYxPHwQeB3KjMXLmkwlTIzReMw6gRuOTy48//ihPMSdOnJAnGhrTHoyOGRHTjhw5Ik89hw4dEuNTEAeJo/GpaN++fdi7d688JdH4xBQbGys3q5eRAXYvLy9MnDgRGzZskCcxasyYMXKz5fXi+Wqwa2UWvV2wX1iHKQsnYeSs4abWMPwcEzwSs+ICJHof5jcUFezKq4i9n9wIWLine truncated
+```
+# Create an omni_lm with HF model name.
+from vllm_omni.entrypoints.omni import Omni
 
-[image11]: <data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAABM8AAARlCAYAAACukOBGAACAAElEQVR4Xuy9B7hV13WuHSd5ch3Zia17rcQl13+K88dyXOLYlmPLRb13CVUkUEMIEB3Rkeiig+iiI3rvvffeO4gmeu+dM+56xz4TbRZ7IxDt7M33Ps98zll97rXGnGvOb4055l+YEEIIIYQQQgghhBAiJX8RXyGEEEIIIYQQQgghhEgg8UwIIYQQQgghhBBCiDRIPBNCCCGEEEIIIYQQIg0Sz4QQQgghhBBCCCGESIPEMyGEEEIIIYQQQggh0iDxTAghhBBCCCGEEEKINEg8E0IIIYQQQgghhBAiDRLPhBBCCCGEEEIIIYRIg8QzIYQQQgghhBBCCCHSIPFMCCGEEEIIIYQQQog0SDwTQgghhBBCCCGEECINEs+EuAhnz5jlKCnF09k0Kb49/B9fH98/1bazSfaXYhvr025PPm98e3xd8nJ826VcK36eVNeN/e/nih+Xajl+vaT1550jvu+lbIv/TfV/fDn5nPHz5yYhhBBCCCFE9iHxTIh05JgdP5Bjx3YrKcXSntTpeG5KXk61/lzafZFtsXPE07n1KfJz7pwp8nvB9ZL2uWBb8j7ptuWm+DWT973geqmOS943zfW+7Den+j++HD9Hym25/8eXk895wfmj8506mhOvRYQQQgghhBBZgMQzIdKBeLY/x45sz7GjO5SUvkhH0qT49vB/fH18/1TbLnbclx2bvC2+Pb4ueTm+Lb5Pqm2p9onvG///Upfj10ten25b/P/4cvzcqbalukb8nBecf7vEMyGEEEIIIbIViWdCXIQT+y3ROd6ppKSklD5RT0g8E0IIIYQQIjuReCZEOnIQz3ISHigpOstKSkpKIUk8E0IIIYQQInuReCZEOsKwTYlnSkpKX5IkngkhhBBCCJG9SDwTIh3yPFNSUrrEJPFMCCGEEEKI7EXimRDpkOeZkpLSJSaJZ0IIIYQQQmQvEs+ESIfEMyUlpUtMEs+EEEIIIYTIXiSeCZEOiWdKSkqXmCSeCSGEEEIIkb1IPBPiIpzYbxLPbmA6vjt6CEej57Dni+dgh83OHkgsu2CxL9rnpPlflk9wzIlon4Nf7HNyb2IfO56bonMe22UXXC+euD7nuZR9SX79KK/k8cuOYd/TkX3lHJKNZUOSeCaEEEIIIUT2IvFMiHRowoAbmhC8tq3bby0afWKfLd3mQhPrenToZ2MGT3ax7PRBs3lTllmJImVsxbz1dvpQ4pjypSrb8H7jfDv7LZu91t59s5gVzk1Vy9ew3RtQuOycoIZQhuCFsBYEtu1r91vfbkNs/+fsYC7K2ZGEKOYiGeuOJY4NQt6GKK9d2va0PZuOfnGu4wkB0MXAIOBFl58zabGNHDDBBTpSOD/58GXyd+ILYZB74PvkCm5nDnyxHL9/Stc3STwTQgghhBAie5F4JkQ6NGzzhiaEorWLt9hD9z9mcycvdVGJde++Ucwa1m7m4hMi1IAew+373/uBNW/Y1h/bkN6jfblRneYuPuHZNbDHCLvvrgds5viFtmDaUlsya5Xt2XjE5kxaZD079bcB3YfZwa2nXOBCdOvZaUC03zL7fNVuGzFgvK1bssUmjphhndv0sCmjZrvnG/v36NjPBvYc4fk9sSfHBa/5U5ZZwfxv2cblO23RjOXWu8tA69NlkO3//IQd3n7Gl7u172N7N+2zSmWq2aMPPm57Nh2wxTNXW6fW3WzCsOkuni2ZtSbK93Dr3qFvlI89djr6vRuWbY/y0N1mTljowtyaRZs9T3OnLPU8xe+h0vVLEs+EEEIIIYTIXiSeCXERNGzzxiWEMjzO8j39ki2YtsLFMtaVKVbeWjRs66IYAtKwvmPtyUeftVJFy9mhHcetecM2lu+pF619y09dcEPQGtpnjL30XH4XoXatP2CHo/3WR+f+nzvudE+0P995t/XpOtjGD5saXe9Fe+O1QvbgfY/YzAkLrGzxCtamWQe741f/YyXeLWP3/Pk+Wz5njbVq3N5KFStnBV9509o07eBeYORp0fSV5zzh7r/nQXvztXeiY+73/HRq3d3ee6ekVS73gQtolcpWs8ceftKFOfLfpF5Le+bxfDZ93Hx75YUC9nz025945Gmr8n4N2711X/S3ur1fspIVi84xacxUK/1eeav9QT17KV9+mzNxkZ2WgHbDksQzIYQQQgghsheJZ0JcBIlnNy5dqng2uNcoK1mkjL1fqrIN6D7CalerbzUq17HWTTu4eMZ+YwZNstv/4z8t/wsFrMDLr1vLxm1s7aLP7bmnXrAd6w5Yp1bdrH7NJlauRAUfErp1zT4r9EYRmzZmrpUrWTHa/xN7r3ApP1fZaJ9P2/exJx992lo1aW8fVKhlTz/+nB3YctLzE8SzZXPWuaj12dKt1rfrYKsV5avtx52sepS3kQNG277NB21g9+HWqM7HdmLP2SjvQ21Qr5H2wrMvW99Ph1iRQiVsanT9GePmW8WyVaPtw/28eNMd2XPUOrfuYX+68y4b2meYPfLA4+5pR/7i91Hp+iSJZ0IIIYQQQmQvEs+ESIeGbd7QhFC2bulWe+HZV1zock4kxLMOLbuee0yDeo6wmlU/cq8uvLVqVatnrZu0TwzjzPU8Q2B7u0DhRKyxXH1j3eIt7nW2c/2h6HyfutcXItzEETNt+7oDVrRQiYR4VqKifdK8k4tfDKdkqGW39n3tuadfsJpVPrKPG7T0oZOHtp2xnMNfiGcMu+TvphU7rVfHAVbnw4Z2cNux6Nge7s3W9ZNePuyzWb1WNnviYnv5+VetWf1W9uC9D/tQ0MrlPrSF01fYhOHTo99X14eSFn+39Lk4bd2jPNz9p/tcCKxfs7FNGjnTTu298D4qXZ8k8UwIIYQQQojsReKZEOmQeHZDE+LZ+mXb7Q+/v8uFq95detnMCXOtyFvF7eV8r1qvzr1sxMARLnzh/TV2yBS77Tu32bC+Y6xFo7ZWv1ZT9wTDGwvx7Df//VsX2Lq1722Deg6xuZOXWIGX37Bta/db8wZtrHHdFja490h7q0Bhq/NBA3vmiXw2bew8K/p2cR8KWrFMNc8XotqwfmPdY6xhnWY+bJLjT+xNzARKzLO3Cha2hdNX+vDPz5ZstS5te9iHlWpbuxado3O19mM/ad7Z+n06xAq88oa1b9HFr9e/21B75IHHXIwrWbSsTygwatBEK1uivK2cv87PhyhYqlhZ69Wlr4tzvToPsBLvlrbJo2b5PYvfR6XrkySeCSGEEEIIkb1IPBMiHRLPbmg6vjvH9m85YZ+262N1PmxgtarWs8G9R3nA/no1Gvsyccfw2po1YaHt3njExadd6w/ZwmkrfBbOk/uYtTPHA+03rdfSaler555pzeq3ttULNrlXF9fAW2zBtOW2d9Mxa1i7oRV45U2PZcZxU0fPskUzVvrwSfJFfLItq/b49Zi4oEGtprZpxS47tS8xacC2Nfv8vMzUOX7YNNuz8agtn7POxbrN0X51qzf0vByIrrt55S6Pp8YQT2boRFQbPWiSX2/K6Nm2dfVej82GMMaEALMmLvJhqUP6jLacI4nZOvk9vToNiOz0rN+z+H1Uuj5J4pkQQgghhBDZi8QzIdIh8eyGJ4ZJMlTTTuemI5aYZTMsn0gMy8yJ0ond5sMZmTGT4P2ITTw70sm9SceQTpqd2pc4F9dgf87F7JYli5axd94o4gIY52afMBkAeWKZ83EdO5VILIdr+frDudujv8d3J2K1cS7PB8ecTKz35ZOJ7f47o//5jZ6fQ4lznGKfQ4nze5y33PtwlGXyx/KxxO+I3z+l65ckngkhhBBCCJG9SDwTIh0Sz266hACFCIewpiGQSpeTJJ4JIYQQQgiRvUg8EyIdEs9uyoSAJi8upctNEs+EEEIIIYTIXiSeCZEOiWdKSkqXmCSeCSGEEEIIkb1IPBMiHRLPlJSULjFJPBNCCCGEECJ7kXgmRDoknikpKV1ikngmhBBCCCFE9iLxTIh0SDxTUlK6xCTxTAghhBBCiOxF4pkQ6ZB4pqSkdIlJ4pkQQgghhBDZi8QzIdIh8UxJKU8kZj89nsdnQJV4JoQQQgghRPYi8UyIdCCe7Ys6xdsTHWMlJaXrn45G6cRus5N77YJteSpF9cSpIxLPhBBCCCGEyEYkngmRhpyoH4wnCR1iJaXz0lG7cDms+7JtydtTLce3pTrvxY5LtRzfFj82+RrxY+P7xrelOzbVvun+j58nadvZE2Zd2/ewMsUr2IlDZ+x0/Nj4NeLnSZe/VMelOybVtvj/UTpzQuKZEEIIIYQQ2YjEMyGEEHmacePHWdu2bS0n52x8kxBCCCGEEEJccySeCSGEEEIIIYQQQgiRBolnQgghhBBCCCGEEEKkQeKZEEIIIYQQQgghhBBpkHgmhBAiTzNixAj76KOP7OxZxTwTQgghhBBCXH8kngkhhMjTFC9e3G699VY7ffp0fJMQQgghhBBCXHMkngkhhMjTVKxY0f7t3/7NTp06Fd8khBBCCCGEENcciWdCCCHyNJUqVbJ//dd/lXgmhBBCCCGEuCFIPBNCCJGnqVy5sv3Lv/yLxDMhhBBCCCHEDUHimRBCiDwNwzZ/+MMfSjwTQgghhBBC3BAkngkhhMjT4Hkm8UwIIYQQQghxo5B4JoQQIk+za9cuW7NmjeXk5MQ3CSGEEEIIIcQ1R+KZEEIIIYQQQgghhBBpkHgmhBBCCCGEEEIIIUQaJJ4JIYQQQgghhBBCCJEGiWdCCCHyNK1bt7Z77rlHEwYIIYQQQgghbggSz4QQQuRpqlSpYn//939vJ0+ejG8SQgghhBBCiGuOxDMhhBB5mpo1a9p3v/tdiWdCCCGEEEKIG4LEMyGEEHmaWrVq2T/+4z9KPBNCCCGEEELcECSeCSGEyNMgnv3DP/yDxDMhhBBCCCHEDUHimRBCiDxNtWrV7JZbbpF4JoQQQgghhLghSLine truncated
+omni_lm = Omni(model="Qwen/Qwen3-Omni-30B-A3B-Instruct")
+
+# Example prompts.
+om_inputs = {"prompt": prompt,
+             "multi_modal_data": {
+                 "video": video_frames,
+                 "audio": audio_signal,
+             }}
+
+# Generate texts and audio from the multi-modality inputs.
+outputs = omni_lm.generate(om_inputs, sampling_params_list)
+```
+
+## Online Serving
+Similar to vLLM, vLLM-Omni also provides a FastAPI-based server for online serving. Users can launch the server using the vllm serve command with the `--omni` flag:
+
+```
+vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091
+```
+
+Users can send requests to the server using curl:
+
+```
+# prepare user content
+user_content='[
+        {
+          "type": "video_url",
+          "video_url": {
+            "url": "'"$SAMPLE_VIDEO_URL"'"
+          }
+        },
+        {
+          "type": "text",
+          "text": "Why is this video funny?"
+        }
+      ]'
+    sampling_params_list='[
+      '"$thinker_sampling_params"',
+      '"$talker_sampling_params"',
+      '"$code2wav_sampling_params"'
+    ]'
+    mm_processor_kwargs="{}"
+
+# send the request
+curl -sS -X POST http://localhost:8091/v1/chat/completions \
+    -H "Content-Type: application/json" \
+    -d @- <<EOF
+{
+  "model": "Qwen/Qwen3-Omni-30B-A3B-Instruct",
+  "sampling_params_list": $sampling_params_list,
+  "mm_processor_kwargs": $mm_processor_kwargs,
+  "messages": [
+    {
+      "role": "system",
+      "content": [
+        {
+          "type": "text",
+          "text": "You are Qwen, a virtual human developed by the Qwen Team, Alibaba Group, capable of perceiving auditory and visual inputs, as well as generating text and speech."
+        }
+      ]
+    },
+    {
+      "role": "user",
+      "content": $user_content
+    }
+  ]
+}
+```
+
+For more usages, please refer to [examples](../user_guide/examples/).

docs/design/feature/omni_connector_design.md

@@ -0,0 +1,200 @@
+# OmniConnector: a unified connector interface of disaggregated inference for omni-modality models
+
+This guide explains how to configure and use distributed connectors (vllm_omni/distributed/connectors) in vllm-omni for multi-stage pipelines.
+
+## 1. Overview
+
+Connectors enable data transfer between pipeline stages (e.g., Thinker -> Talker).
+Currently supported connectors operate in **D2H2D (Device-to-Host-to-Device)** mode:
+1. **SharedMemoryConnector**: Uses system shared memory.
+2. **MooncakeConnector**: Uses [Mooncake](https://github.com/kvcache-ai/Mooncake).
+
+*   **SharedMemoryConnector (Default)**: Zero-copy (on host), lowest latency. Best for **single-node** deployments. Auto-configured if no connectors are specified.
+*   **MooncakeConnector**: TCP/RDMA based. Best for **multi-node** distributed deployments. Requires a Mooncake Master service.
+
+## 2. API Design
+
+The connector system is built around the `OmniConnectorBase` abstraction, which decouples data transport from stage logic.
+
+### Core Interface
+
+```python
+class OmniConnectorBase(ABC):
+    @abstractmethod
+    def put(self, from_stage: str, to_stage: str, request_id: str, data: Any) -> tuple[bool, int, Optional[dict]]:
+        """
+        Store data.
+        Returns: (success, serialized_size, metadata)
+        """
+        pass
+
+    @abstractmethod
+    def get(self, from_stage: str, to_stage: str, request_id: str, metadata: Optional[dict] = None) -> Optional[tuple[Any, int]]:
+        """
+        Retrieve data.
+        Args: metadata - Transport-specific handles returned by put() (e.g., SHM name).
+        Returns: (object, serialized_size)
+        """
+        pass
+```
+
+### Key Concept: Metadata Passing
+Unlike a pure key-value store, some connectors (like `SharedMemoryConnector`) generate transient resources (e.g., a shared memory block name) during `put()`. This `metadata` **must be passed via the control plane** (e.g., HTTP headers, queue messages) from the producer stage to the consumer stage so `get()` can locate the data.
+
+## 3. Backends & Use Cases
+
+### 3.1 SharedMemoryConnector
+**Best for:** Single-node, high-performance IPC.
+
+*   **Mechanism:**
+    *   **Small Payloads (< Threshold)**: Data is serialized and passed directly "inline" within the `metadata` dictionary. This avoids the overhead of creating SHM blocks for tiny messages.
+    *   **Large Payloads (>= Threshold)**: Data is written to a named System Shared Memory block. The block name is returned in `metadata`.
+*   **Configuration:**
+    *   `shm_threshold_bytes`: Size in bytes to switch from inline to SHM (default: 64KB).
+
+### 3.2 MooncakeConnector
+**Best for:** Multi-node distributed inference.
+
+*   **Mechanism:** Uses Mooncake's distributed KVCache store.
+    *   **Data Plane**: TCP or RDMA for high-bandwidth transfer.
+    *   **Control Plane**: Uses a centralized Mooncake Master and Metadata Server.
+    *   **Keying**: Deterministic keys based on `request_id/from_stage_to_stage`.
+*   **Requirements**: Requires a running Mooncake Master service.
+
+## 4. Relationship with vLLM
+
+vLLM provides specialized distributed mechanisms for specific artifacts:
+*   **KV Transfer** (`vllm.distributed.kv_transfer`): Optimized for transferring KV caches between prefill and decode instances (using NCCL, Mooncake, etc.).
+*   **EC Transfer** (`vllm.distributed.ec_transfer`): Optimized for sharing encoder embeddings.
+*   **Device Communicators** (`vllm.distributed.device_communicators`): Low-level primitives (NCCL, SHM) for Tensor/Pipeline Parallelism.
+
+`vllm-omni` complements this by introducing a **Generalized Connector Abstraction** (`OmniConnector`) for multimodal pipelines. While vLLM's connectors are artifact-specific, `vllm-omni`:
+
+1.  **Unifies Transport**: Provides a single API (`put`/`get`) to transport *any* stage artifact (Input Embeddings, Hidden States, Audio/Image Tensors, KV Cache, Final Output) between arbitrary pipeline stages (e.g., AudioEncoder -> LLM -> AudioGenerator).
+2.  **Extends Connectivity**: Enables flexible construction of complex DAGs (Directed Acyclic Graphs) where stages can run in the same process, same node, or across nodes, using the most appropriate backend (SHM, Mooncake, etc.) for each edge.
+3.  **Wraps & Adapts**: Can internally utilize vLLM's specialized `kv_transfer` for KV paths while using generic transports (SHM/Mooncake) for other data types, presenting a consistent interface to the application layer.
+
+## 5. Installation (Mooncake)
+
+If using `MooncakeConnector`, install the library first:
+
+```bash
+# For CUDA-enabled systems (Recommended)
+pip install mooncake-transfer-engine
+
+# For non-CUDA systems
+pip install mooncake-transfer-engine-non-cuda
+```
+
+## 6. Using MooncakeConnector
+
+### 6.1 Start Mooncake Master
+
+Start the master service on your primary node:
+
+```bash
+# if you use mooncake SSD storage
+mkdir -p ./mc_storage
+
+mooncake_master \
+  --rpc_port=50051 \
+  --enable_http_metadata_server=true \
+  --http_metadata_server_host=0.0.0.0 \
+  --http_metadata_server_port=8080 \
+  --metrics_port=9003 \
+  --root_fs_dir=./mc_storage/ \
+  --cluster_id=mc-local-1 &
+```
+
+### 6.2 Configuration (YAML)
+
+Edit your stage config (e.g., `qwen2_5_omni.yaml`).
+
+**Step 1: Define Connector in Global Runtime**
+
+```yaml
+runtime:
+  connectors:
+    connector_of_mooncake:
+      name: MooncakeConnector
+      extra:
+        host: "127.0.0.1"           # Local Worker IP
+        metadata_server: "http://<MASTER_IP>:8080/metadata"
+        master: "<MASTER_IP>:50051"
+        segment: 512000000          # 512MB segment
+        localbuf: 64000000          # 64MB buffer
+        proto: "tcp"                # "tcp" or "rdma"
+    ```
+
+**Mooncake Configuration Parameters:**
+
+*   **host**: The hostname or IP address of the local machine (worker). Mooncake uses this to register itself in the metadata server so other nodes can find it.
+*   **metadata_server**: The URL of the metadata server. This is used for service discovery and connection establishment (e.g., exchanging QP information for RDMA).
+*   **master**: The address of the Mooncake Master Server (e.g., `<MASTER_IP>:50051`). This is used for global state management and control plane operations.
+*   **segment**: The size of the global memory segment in bytes (default: ~512MB). This defines the shared memory region accessible by Mooncake for data transfer.
+*   **localbuf**: The size of the local buffer in bytes (default: ~64MB). Used for local data buffering during transfer operations.
+*   **proto**: The transport protocol to use. Options:
+    *   `tcp`: Standard TCP/IP (easier setup, universal compatibility).
+    *   `rdma`: Remote Direct Memory Access (higher performance, requires RDMA-capable hardware).
+
+For more details, refer to the [Mooncake Repository](https://github.com/kvcache-ai/Mooncake).
+
+    **Step 2: Reference in Stages**
+
+Explicitly link stages using `input_connectors` and `output_connectors`:
+
+```yaml
+stage_args:
+  - stage_id: 0
+    # ...
+    output_connectors:
+      to_stage_1: connector_of_mooncake
+
+  - stage_id: 1
+    # ...
+    input_connectors:
+      from_stage_0: connector_of_mooncake
+```
+
+## 7. Using SharedMemoryConnector (Auto-Mode)
+
+**Best for single-node.**
+
+The system will automatically create `SharedMemoryConnector`s for any pipeline edge that does not have an explicit connector defined. This is inferred from:
+1.  `runtime.edges` list in the config.
+2.  `engine_input_source` dependencies defined in `stage_args`.
+
+### Threshold Configuration
+By default, payloads larger than **64KB** (default threshold) are transferred via shared memory, while smaller ones use the control queue (inline).
+
+To adjust this threshold (e.g., to 1GB), add the following to your `runtime.connectors`:
+
+```yaml
+runtime:
+  connectors:
+    connector_of_shared_memory:
+      name: SharedMemoryConnector
+      extra:
+        shm_threshold_bytes: 1024 # 1KB threshold
+```
+
+## 8. Summary
+
+| Use Case | Recommended Connector | Configuration |
+| :--- | :--- | :--- |
+| **Single Node** | `SharedMemoryConnector` | **None** (Automatic) or Custom Threshold |
+| **Multi Node** | `MooncakeConnector` | Explicit YAML + Mooncake Master |
+
+## 9. Operational Notes (important)
+
+- **Fail-fast config validation**: the loader raises if any expected edge is missing a connector. Define `input_connectors`/`output_connectors` or rely on auto-SHM filling; otherwise startup aborts.
+- **Missing payloads halt the stage**: workers expect connector payloads; if metadata or connector config is missing, the stage raises and stops. Verify connector wiring and metadata propagation before production.
+
+## 10. Future Roadmap: Device-to-Device (D2D) Transport
+
+The current implementations (`SharedMemoryConnector`, `MooncakeConnector`) utilize a **D2H2D (Device-to-Host-to-Device)** data path. Tensors are moved to CPU memory (Host) for transport, which incurs PCIe overhead.
+
+As outlined in the design RFC, future versions will introduce **D2D (Device-to-Device)** connectors:
+
+*   **Goal**: Direct GPU-to-GPU transfer (via NCCL, UCX, or IPC) to minimize latency for large tensor payloads.
+*   **Mechanism**: The `OmniConnector` API allows `put()` to initiate a transfer and return a lightweight handle (metadata) via the control plane, while the heavy payload flows directly between devices.

docs/design/feature/ray_based_execution.md

@@ -0,0 +1,61 @@
+# VLLM-Omni Distributed Utilities
+
+This directory (vllm_omni/distributed/ray_utils) contains utilities for distributed execution in vllm-omni, supporting both **Ray** and **Multiprocessing** backends.
+
+## 1. Ray Utils
+
+The `ray_utils` module provides helper functions for managing Ray clusters and actors, which is essential for:
+*   **Multi-node deployment**: Running pipeline stages across different physical machines.
+*   **Resource management**: Efficient GPU/CPU allocation.
+
+### 1.1 Basic Usage
+
+To use the Ray backend, specify `worker_backend="ray"` when initializing the engine.
+
+**Command Line Example:**
+```bash
+vllm serve Qwen/Qwen2.5-Omni-7B \
+  --omni \
+  --port 8091 \
+  --worker-backend ray \
+  --ray-address auto
+```
+
+### 1.2 Cluster Setup
+
+**Step 1: Start Head Node**
+Run this on your primary machine:
+```bash
+ray start --head --port=6399
+```
+
+**Step 2: Connect Worker Nodes**
+Run this on each worker machine:
+```bash
+ray start --address=<HEAD_NODE_IP>:6399
+```
+
+> **Tip**: For a complete cluster setup script, refer to the vLLM example:
+> [run_cluster.sh](https://github.com/vllm-project/vllm/blob/main/examples/online_serving/run_cluster.sh)
+
+### 1.3 Distributed Connector Support
+
+When running on Ray, the system automatically adapts its communication strategy:
+
+*   **Cross-Node**: Recommended to use `MooncakeConnector` (requires separate configuration).
+*   **Same-Node**: Can still use `SharedMemoryConnector` for efficiency, or Ray's native object store (plasma).
+*   **SHM threshold default differs**: when `worker_backend="ray"`, the SharedMemoryConnector default threshold is set to `sys.maxsize`, which forces payloads to go inline (no SHM). Override `shm_threshold_bytes` in the connector config if you want SHM for Ray runs.
+
+### 1.4 Internal Helpers
+
+*   **`initialize_ray_cluster`**: Connects to an existing Ray cluster or starts a local one.
+
+## 2. Troubleshooting
+
+*   **Connection Issues**: Ensure the Ray head node is accessible and ports (default 6399 in this example) are open.
+*   **Version Mismatch**: Ensure all nodes run the same version of Ray and Python.
+
+### Installation
+```bash
+pip install "ray[default]"
+```

docs/design/index.md

@@ -5,9 +5,15 @@ This section contains design documents and architecture specifications for vLLM-
 ## Architecture Documents
 
 - [Architecture Overview](architecture_overview.md)
-- [vLLM-Omni Software Design Document](vllm_omni_design.md)
-- [Multi-Request Streaming](mrs_design.md)
 
-## API Design Documentation
+## Feature Design Documents
 
-- [vLLM-Omni API Design Documentation](api_design_doc.md)
+- [Multi-Request Streaming](feature/multi_request_streaming_design.md)
+- [Omni Connector Design](feature/omni_connector_design.md)
+- [Ray-based Execution](feature/ray_based_execution.md)
+
+## Module Design Documents
+
+- [AR Module Design](module/ar_module_design.md)
+- [DIT Module Design](module/dit_module_design.md)
+- [Entrypoint Module Design](module/entrypoint_module_design.md)

docs/design/module/ar_module_design.md

@@ -0,0 +1 @@
+Architecture design of the AR module (update soon)

docs/design/module/dit_module_design.md

@@ -0,0 +1 @@
+Architecture design of the DiT module (update soon)

docs/design/module/entrypoint_module_design.md

@@ -0,0 +1 @@
+Architecture design of the entrypoint (update soon)

docs/design/vllm_omni_design.md

@@ -1,509 +0,0 @@
-# Architecture Design Principles
-
-## Overview
-
-vLLM-Omni is a multi-modality extension for vLLM that supports non-autoregressive structures and non-textual outputs. This document outlines the key software abstractions, APIs, and dependencies for the system, designed to maximize reuse of vLLM's proven architecture.
-
-## Architecture Principles
-
-1. **vLLM V1 Compatibility**: Built on vLLM's Engine V1 architecture with AsyncLLM and EngineCore patterns
-2. **Stage-based Processing**: Models are divided into multiple stages, each processed by different Engine Cores
-3. **Multiple Engine Core Support**: Each stage can use either AR Engine Core (reusing vLLM) or Diffusion Engine Core (new DiT support), or other new Cores
-4. **Worker Process Pattern**: Follows vLLM's multiprocess worker architecture for scalability
-5. **Extensibility**: Easy integration of new modalities, model architectures, and output formats
-
-## Key Data Flow
-
-API Server --> OmniLLM/AsyncOmniLLM (New, including multi engines) --> LLMEngine/AsyncLLM --> Engine Core
- --> Scheduler (New one for DiT) --> Executor (New one for diffusers) --> Worker (New one for DiT)
- --> ModelRunner (New one for AR hiddenstate, New one for DiT) --> RequestState --> OutputProcessoer (New one for final multimodal output)
-
-## Core Components (Ordered by Data Flow)
-
-### 1. Installation
-```bash
-pip install vllm
-pip install vllm-omni
-```
-
-### 2. Online inference launch (Entry Point)
-
-Keep consistency with the vllm main branch
-```bash
-vllm serve Qwen/Qwen2.5-Omni-7B --omni --port 8000
-```
-
-#### Design Architecture
-```mermaid
-graph TD
-    A[vllm serve model --omni] --> B[vLLM-Omni CLI Wrapper]
-    B --> C{Detect --omni flag}
-    C -->|Yes| D[Parse OmniConfig]
-    C -->|No| E[Forward to vLLM CLI]
-    D --> F[Initialize AsyncOmniLLM]
-    F --> G[Start omni Server]
-    G --> H[Multi-stage Processing]
-    E --> I[Standard vLLM Pipeline]
-
-    subgraph "vLLM-Omni Components"
-        F
-        G
-        H
-    end
-
-    subgraph "vLLM Components"
-        I
-    end
-```
-
-#### omni Serve Command Implementation
-File: vllm_omni/entrypoints/cli/main.py
-```python
-import sys
-import argparse
-from typing import List, Optional
-from vllm_omni.entrypoints.omni import OmniServeCommand
-
-def main():
-    """Main CLI entry point that intercepts vLLM commands"""
-    # Check if --omni flag is present
-    if "--omni" in sys.argv:
-        # Remove --omni flag and process with vLLM-Omni
-        omni_args = [arg for arg in sys.argv[1:] if arg != "--omni"]
-        omni_serve = OmniServeCommand()
-        omni_serve.run(omni_args)
-    else:
-        # Forward to original vLLM CLI
-        from vllm.entrypoints.cli.main import main as vllm_main
-        vllm_main()
-
-if __name__ == "__main__":
-    main()
-```
-
-#### vLLM Plugin Integration
-File: vllm_omni/pyproject.toml
-```python
-[project.scripts]
-# Override vLLM's CLI entry point
-vllm = "vllm_omni.cli:main"
-
-# Add entry point for vLLM integration
-[project.entry-points."vllm.plugins"]
-omni = "vllm_omni.plugin:OmniPlugin"
-```
-
-### 3. Offline inference launch (Entry Point)
-Design an upper level class to incorporate multi Engines, each engine has a engine call.
-```python
-from vllm.entrypoints.llm import LLM
-from vllm.v1.engine.llm_engine import LLMEngine
-
-class OmniLLM(LLM):
-    """Extended LLM supporting multiple engines and stage-based processing"""
-
-    def __init__(self, stage_configs: List[StageConfig]):
-        super().__init__()
-        self.stage_configs = stage_configs
-        self.engine_list = []  # List of AsyncLLM instances for each stage
-        self.output_processor = MultimodalOutputProcessor()
-        self._initialize_stage_engines()
-
-    def _initialize_stage_engines(self) -> None:
-        """Initialize LLMEngine instances for each stage"""
-        for stage_config in self.stage_configs:
-            stage_llm = LLMEngine.from_vllm_config(
-                    vllm_config=stage_config.vllm_config,
-                    executor_class=self.executor_class,
-                    log_stats=self.log_stats
-                )
-            self.engine_list.append(stage_llm)
-
-    def generate(
-        self,
-        stage_args_list: List[stage_args],
-        use_tqdm: Union[bool, Callable[..., tqdm]] = True,
-        lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
-        priority: Optional[list[int]] = None,
-    ) -> list[RequestOutput]:
-        """Main generation interface - orchestrates multi-stage processing"""
-
-        # Process through each stage sequentially
-        for i, stage_config in enumerate(self.stage_configs):
-            stage_engine = self.engine_list[i]
-
-            # Prepare input for this stage
-            stage_args = stage_args_list[i]
-            prompt_str, engine_request, tokenization_kwargs = self._process_stage_inputs(stage_config, **stage_args)
-
-            # Add inputs to Engine
-            stage_engine.add_request(requesy_id, prompt_str, tokenization_kwargs)
-            # Run Engine
-            stage_output = stage_engine.step()
-
-            # Update stage input and output of each stage for later usage
-            self._update_stage_io(stage_output, stage_config)
-
-
-        # Process final output
-        final_output = self.output_processor.process_output(
-            self.stage_configs[-1].stage_output
-        )
-        return final_output
-
-    def _process_stage_inputs(stage_config, **stage_args) -> Any:
-        """Prepare input for specific stage"""
-        if stage_config.engine_type == "AR":
-            return self._process_ar_inputs(**stage_args)
-        elif stage_config.engine_type == "DiT":
-            return self._process_dit_inputs(**stage_args)
-        else
-            raise NotImplementedError
-
-    def _process_dit_inputs(**stage_args)-> Any:
-        image_latent = self.vae.encode(**stage_args)
-
-        return image_latent
-```
-
-### 4. Stage config for setting of different model stages
-```python
-@dataclass
-class StageConfig:
-    """Configuration for a processing stage"""
-    stage_id: int
-    engine_type: str  # "AR" or "DiT"
-    model_path: str
-    input_modalities: List[str]
-    output_modalities: List[str]
-    vllm_config: Optional[VllmConfig] = None  # For engine config of corresponding stage
-    executor_class: type[Executor]  # For execute class config of corresponding stage
-    dit_config: Optional[DiTConfig] = None  # For diffusion stages
-    cache_config: Optional[DiTCacheConfig] = None
-```
-For AR stage, the setting is:
-```python
-from vllm.v1.core.sched.scheduler import Scheduler
-from vllm.v1.executor.multiproc_executor import MultiprocExecutor
-ar_stage_config = StageConfig()
-ar_stage_config.vllm_config.scheduler_config.scheduler_cls = Scheduler # original vllm scheduler for AR
-ar_stage_config.executor_class = MultiprocExecutor # original vllm executor for AR
-```
-For DiT stage, the setting is:
-```python
-from vllm.v1.executor.multiproc_executor import MultiprocExecutor
-dit_stage_config = StageConfig()
-dit_stage_config.vllm_config.scheduler_config.scheduler_cls = DiffusionScheduler # New scheduler for DiT
-
-# For diffusion models without using diffusers
-dit_stage_config.executor_class = MultiprocExecutor
-
-# For diffusion models using diffusers
-dit_stage_config.executor_class = DiffusersPipelineExecutor
-```
-
-### 5. Online inference main class
-
-Similar to OmniLLM in offline inference, add some asynchronous processing, referring to AsyncLLM
-```python
-from vllm.v1.engine.async_llm import AsyncLLM
-
-
-class AsyncOmniLLM(AsyncLLM):
-    """Extended AsyncLLM supporting multiple engines and stage-based processing"""
-
-    def __init__(self, stage_configs: List[StageConfig]):
-        super().__init__()
-```
-
-### 6. Engine Core
-No need to change. The specific executor, scheduler and output type can be transferred into it with new configs.
-```python
-# class: EngineCore
-# EngineCore.step (simplified)
-class EngineCore:
-    def step(self):
-        scheduler_output = scheduler.schedule()
-        model_output = executor.execute_model(scheduler_output)
-        engine_outputs = scheduler.update_from_output(
-            scheduler_output, model_output
-        )
-        return engine_outputs
-```
-
-### 7. Scheduler
-Create a new Diffusion Scheduler for DiT, which is inherited from original vllm AR scheduler. At first, no complicated strategy.
-Just process the request first in first out. Then create a class of DiT Cache Manager for optimization in the future.
-```python
-from vllm.v1.core.sched.scheduler import Scheduler
-class DiffusionScheduler(Scheduler):
-
-    def __init__(
-        self,
-        vllm_config: VllmConfig,
-        kv_cache_config: KVCacheConfig,
-        dit_cache_config: DiTCacheConfig,
-        structured_output_manager: StructuredOutputManager,
-        mm_registry: MultiModalRegistry = MULTIMODAL_REGISTRY,
-        include_finished_set: bool = False,
-        log_stats: bool = False,
-    ) -> None:
-
-    self.dit_cache_manager = DiTCacheManager(dit_cache_config)
-```
-For DiT Cache Manager (Can refer to xDiT):
-```python
-class DiTCacheManager:
-    """Manages DiT-specific caching"""
-
-    def __init__(self, config: DiTCacheConfig):
-        self.cache_tensors: Dict[str, torch.Tensor] = {}
-        self.cache_groups: List[DiTCacheTensor] = config.dit_cache_tensors
-
-    def allocate_cache(self, request_id: str, size: int) -> torch.Tensor
-    def get_cache(self, request_id: str) -> Optional[torch.Tensor]
-    def release_cache(self, request_id: str) -> None
-    def clear_expired_cache(self) -> None
-```
-
-### 8. Executor for DiT without diffusers
-
-No need to change. Just import original AR executor.
-
-### 9. Diffusers Pipeline Executor (no worker)
-
-A single-process executor that directly runs the Diffusers pipeline without spawning workers or using RPC. Interfaces remain identical to `Executor` so the EngineCore loop is unchanged.
-
-#### Function map（Pipeline Executor）
-
-##### 1) Inherited and overridden
-```python
-from concurrent.futures import Future
-from typing import Optional, Union, Callable, Any
-import torch
-import torch.nn as nn
-from vllm.v1.executor.abstract import Executor
-from vllm.v1.kv_cache_interface import KVCacheConfig, KVCacheSpec
-from vllm.v1.outputs import ModelRunnerOutput
-from vllm.tasks import SupportedTask
-from diffusers import DiffusionPipeline
-
-class DiffusersPipelineExecutor(Executor):
-    supports_pp: bool = False  # Single-process, no TP/PP/DP
-
-    def _init_executor(self) -> None:
-        # Called by ExecutorBase.__init__
-        self._failure_callback: Optional[Callable[[], None]] = None
-        self._device = self._resolve_device()
-        self._dtype = self._resolve_dtype()
-        self._pipeline = self._build_pipeline(device=self._device, dtype=self._dtype)
-        self._profiler = None
-        self._is_failed = False
-        self.is_sleeping = False
-        self.sleeping_tags: set[str] = set()
-
-    # major functions to build/run diffusers pipeline
-    def _build_pipeline(self, device:torch.device, dtype:torch.dtype)->DiffusionPipeline:
-        model_name = "Qwen/Qwen-Image"
-
-        self.pipe = DiffusionPipeline.from_pretrained(model_name, torch_dtype=dtype)
-        self.pipe = pipe.to(device)
-
-    def _run_pipeline(self, scheduler_output) -> ModelRunnerOutput: ...
-        positive_magic = {
-            "en": ", Ultra HD, 4K, cinematic composition.", # for english prompt
-            "zh": ", 超清，4K，电影级构图." # for chinese prompt
-        }
-
-        # Generate image
-        prompt_embeds = self._get_and_process_prompt_embeds(scheduler_output, positive_magic)
-        negative_prompt_embeds = self.pipe.embed_prompt(" ")
-
-        width, height = aspect_ratios["16:9"]
-
-        image = pipe(
-            prompt_embeds=prompt_embeds,
-            negative_prompt_embeds=negative_prompt_embeds,
-            width=width,
-            height=height,
-            num_inference_steps=50,
-            true_cfg_scale=4.0,
-            generator=torch.Generator(device="cuda").manual_seed(42)
-        ).images[0]
-
-        output = self.wrap_image_as_ModelRunnerOutput(image)
-        return output
-
-    # ---- Internal helpers (implementation-specific, not public API) ----
-    def _resolve_device(self): ...
-    def _resolve_dtype(self): ...
-    def _get_model(self) -> nn.Module: ...
-    def _get_and_process_prompt_embeds(self, scheduler_output, positive_magic):
-        ...
-        #append the positive_magic to prompt and embed them to prompt embed tensors
-
-    # Functions related to workers should either raise NotImplementedError
-    # or return empty defaults in this no-worker executor.
-
-    def collective_rpc(self, method, timeout=None, args=(), kwargs=None) -> list[Any]:
-        # No workers in pipeline executor
-        raise NotImplementedError("No workers in DiffusersPipelineExecutor")
-
-    def initialize_from_config(self, kv_cache_configs: list[KVCacheConfig]) -> None:
-        return  # no-op (pipeline already built in _init_executor)
-
-    def register_failure_callback(self, callback):
-        self._failure_callback = callback
-
-    def determine_available_memory(self) -> list[int]:  # bytes
-        # Single device; return [available_bytes]. If CPU-only, return [0].
-        return [self._determine_available_bytes(self._device)]
-
-```
-
-### 10. Worker
-
-#### Inheritance strategy
-Prefer reusing the mature GPU Worker end-to-end. Worker class is selected by configuration (vllm_config). Do not add a new executor-specific worker binding. If customization is needed, override only `init_device` to construct the `DiffusionModelRunner`; all other behaviors (device init details, profiling, sleep/wake, PP/TP comms, execute path) remain from `vllm/v1/worker/gpu_worker.py::Worker`.
-
-
-Inherited and overridden
-```python
-# Optional: only if you need to plug a custom DiffusionModelRunner.
-from vllm.v1.worker.gpu_worker import Worker as GPUWorker
-from vllm.v1.worker.diffusion_model_runner import DiffusionModelRunner
-
-class Worker(GPUWorker):
-    def init_device(self) -> None:
-        #those related to device check and init
-        ...
-
-        self.model_runner = DiffusionModelRunner(self.vllm_config, ...)
-```
-
-### 11. Model Runner
-
-### Function map (Model Runner)
-#### 1) Inherited and overridden
-Those parts relied to the KV Cache will be omitted if we do not register the model to the vllm config. The engine core will view it as do not require KV Cache, and handle it properly
-
-Reuse `vllm/v1/outputs.py::ModelRunnerOutput`：
-- DiffusionModelRunner: Use the `pooler_output=[Tensor,...]` to return multi modal tensors
-- ARModelRunner: Use the `pooler_output=[Tensor,...]` to return hidden states.
-```python
-from typing import Optional, Union
-import torch
-from vllm.v1.worker.gpu_model_runner import GPUModelRunner
-from vllm.v1.outputs import ModelRunnerOutput
-
-
-class DiffusionModelRunner(GPUModelRunner):
-    @torch.inference_mode()
-    def execute_model(
-        self,
-        scheduler_output: "SchedulerOutput",
-        intermediate_tensors: Optional[IntermediateTensors] = None,
-    ) -> Union[ModelRunnerOutput, IntermediateTensors]:
-        ...
-        return ModelRunnerOutput(
-            req_ids=[...],
-            req_id_to_index={...},
-            sampled_token_ids=[],
-            spec_token_ids=None,
-            logprobs=None,
-            prompt_logprobs_dict={},
-            pooler_output=[Tensor,...],         # Return Hidden states
-            kv_connector_output=None,
-            num_nans_in_logits=None,
-        )# return multi modal tensors via pooler_output=[Tensor,...]
-
-
-class ARModelRunner(GPUModelRunner):
-    @torch.inference_mode()
-    def execute_model(
-        self,
-        scheduler_output: "SchedulerOutput",
-        intermediate_tensors: Optional[IntermediateTensors] = None,
-    ) -> Union[ModelRunnerOutput, IntermediateTensors]:
-        ...
-        return ModelRunnerOutput(
-            req_ids=[...],
-            req_id_to_index={...},
-            sampled_token_ids=[],
-            spec_token_ids=None,
-            logprobs=None,
-            prompt_logprobs_dict={},
-            pooler_output=[Tensor,...],         # Return Hidden states
-            kv_connector_output=None,
-            num_nans_in_logits=None,
-        )# return hidden states via pooler_output=[Tensor,...]
-```
-
-### 12. RequestState Role and Usage
-```bash
-# RequestState serves as the per-request state tracker in OutputProcessor:
-# - Maintains request-specific state (tokens, logprobs, detokenizer, etc.)
-# - Converts EngineCoreOutput → RequestOutput/PoolingRequestOutput
-# - Manages request lifecycle from registration to completion
-AsyncLLM.add_request()
-↓
-OutputProcessor.add_request()
-↓
-RequestState.from_new_request() → 创建请求状态
-
-AsyncLLM.__init__() / AsyncLLM.generate() / AsyncLLM.encode() →  创建一个Background loop 持续从EngineCore获取输出
-↓
-OutputProcessor.process_outputs() → 更新状态并处理输出
-↓
-RequestState.make_request_output() → 转换为最终输出,格式为RequestOutput或者PoolingRequestOutput
-↓
-RequestOutputCollector.put() → 推送到队列（AsyncLLM）
-```
-
-Need to add implementation for an existing method
-```python
-class RequestState:
-    def __init__(self, request_id: str, parent_req: Optional[ParentRequest],
-                 request_index: int, lora_name: Optional[str],
-                 output_kind: RequestOutputKind, prompt: Optional[str],
-                 prompt_token_ids: list[int], logprobs_processor: Optional[LogprobsProcessor],
-                 detokenizer: Optional[IncrementalDetokenizer],
-                 max_tokens_param: Optional[int], arrival_time: float,
-                 queue: Optional[RequestOutputCollector], log_stats: bool):
-
-    def _new_pooling_output(self, pooling_output: torch.Tensor) -> PoolingOutput:
-        """Create PoolingOutput for multimodal/pooling requests"""
-        return PoolingOutput(data=pooling_output)
-```
-
-
-### 13. OutputProcessor
-
-For hidden state output, original OutputProcessor already support.
-
-We only need to add new one for final multimodal output.
-```python
-class MultimodalOutputProcessor(OutputProcessor):
-    """Handles multimodal output processing"""
-
-    def __init__(self):
-        self.output_handlers: Dict[str, OutputProcessor] = {}
-
-    def process_outputs(self, engine_core_outputs: list[EngineCoreOutput], ...):
-        for engine_core_output in engine_core_outputs:
-            # Option 1: Use output_type field (if EngineCoreOutput is extended)
-            if engine_core_output.output_type == "image":
-                self._process_image_output(engine_core_output)
-            elif engine_core_output.output_type == "text+image":
-                self._process_text_image_output(engine_core_output)
-            elif engine_core_output.output_type == "latents":
-                self._process_latents_output(engine_core_output)
-            elif engine_core_output.output_type == "text":
-                self._process_text_output(engine_core_output)
-            else:
-                # Fallback: use existing pooling_output logic
-                if engine_core_output.pooling_output is not None:
-                    self._process_pooling_output(engine_core_output)
-                else:
-                    self._process_text_output(engine_core_output)
-```

docs/getting_started/installation/README.md

@@ -1,7 +1,8 @@
 # Installation
 
-vLLM supports the following hardware platforms:
+vLLM-Omni supports the following hardware platforms:
 
 - [GPU](gpu.md)
     - [NVIDIA CUDA](gpu.md)
-- [Ascend NPU](npu.md)
+    - [AMD ROCm](gpu.md)
+- [NPU](npu.md)

docs/getting_started/installation/gpu.md

@@ -14,6 +14,10 @@ vLLM-Omni is a Python library that supports the following GPU variants. The libr
 
     --8<-- "docs/getting_started/installation/gpu/cuda.inc.md:requirements"
 
+=== "AMD ROCm"
+
+    --8<-- "docs/getting_started/installation/gpu/rocm.inc.md:requirements"
+
 ## Set up using Python
 
 ### Create a new Python environment
@@ -26,6 +30,11 @@ vLLM-Omni is a Python library that supports the following GPU variants. The libr
 
     --8<-- "docs/getting_started/installation/gpu/cuda.inc.md:pre-built-wheels"
 
+
+=== "AMD ROCm"
+
+    --8<-- "docs/getting_started/installation/gpu/rocm.inc.md:pre-built-wheels"
+
 [](){ #build-from-source }
 
 ### Build wheel from source
@@ -34,7 +43,24 @@ vLLM-Omni is a Python library that supports the following GPU variants. The libr
 
     --8<-- "docs/getting_started/installation/gpu/cuda.inc.md:build-wheel-from-source"
 
+=== "AMD ROCm"
+
+    --8<-- "docs/getting_started/installation/gpu/rocm.inc.md:build-wheel-from-source"
+
 ## Set up using Docker
 
+### Build wheel from source
+
+=== "NVIDIA CUDA"
+
+    --8<-- "docs/getting_started/installation/gpu/cuda.inc.md:build-wheel-from-source-in-docker"
+
+=== "AMD ROCm"
+
+    --8<-- "docs/getting_started/installation/gpu/rocm.inc.md:build-wheel-from-source-in-docker"
+
 ### Pre-built images
-To be released... Stay tuned!
+
+=== "NVIDIA CUDA"
+
+    --8<-- "docs/getting_started/installation/gpu/cuda.inc.md:pre-built-images"

docs/getting_started/installation/gpu/cuda.inc.md

@@ -44,7 +44,7 @@ uv pip install vllm==0.11.0 --torch-backend=auto
 Install additional requirements for vLLM-Omni
 ```bash
 git clone https://github.com/vllm-project/vllm-omni.git
-cd vllm_omni
+cd vllm-omni
 uv pip install -e .
 ```
 
@@ -73,3 +73,27 @@ uv pip install --no-build-isolation --editable .
 </details>
 
 # --8<-- [end:build-wheel-from-source]
+
+# --8<-- [start:build-wheel-from-source-in-docker]
+
+# --8<-- [end:build-wheel-from-source-in-docker]
+
+# --8<-- [start:pre-built-images]
+
+vLLM-Omni offers an official docker image for deployment. These images are built on top of vLLM docker images and available on Docker Hub as [vllm/vllm-omni](https://hub.docker.com/r/vllm/vllm-omni/tags). The version of vLLM-Omni indicates which release of vLLM it is based on.
+
+Here's an example deployment command that has been verified on 2 x H100's:
+```bash
+docker run --runtime nvidia --gpus 2 \
+    -v ~/.cache/huggingface:/root/.cache/huggingface \
+    --env "HF_TOKEN=$HF_TOKEN" \
+    -p 8091:8091 \
+    --ipc=host \
+    vllm/vllm-omni:v0.11.0rc1 \
+    --model Qwen/Qwen3-Omni-30B-A3B-Instruct --port 8091
+```
+
+!!! tip
+    You can use this docker image to serve models the same way you would with in vLLM! To do so, make sure you overwrite the default entrypoint (`vllm serve --omni`) which works only for models supported in the vLLM-Omni project.
+
+# --8<-- [end:pre-built-images]

docs/getting_started/installation/gpu/rocm.inc.md

@@ -0,0 +1,115 @@
+# --8<-- [start:requirements]
+
+- GPU: Validated on gfx942 (It should be supported on the AMD GPUs that are supported by vLLM.)
+
+# --8<-- [end:requirements]
+# --8<-- [start:set-up-using-python]
+
+vLLM-Omni current recommends the steps in under setup through Docker Images.
+
+# --8<-- [start:pre-built-wheels]
+
+# --8<-- [end:pre-built-wheels]
+
+# --8<-- [start:build-wheel-from-source]
+
+# --8<-- [end:build-wheel-from-source]
+
+# --8<-- [start:build-wheel-from-source-in-docker]
+
+#### Get the correct docker environment
+
+The v0.11.0 releases is on Oct 3, 2025, so we will pull a AMD nightly docker image close to the release period. `rocm/vllm-dev:nightly_main_20251005` is picked as it has a correct `transformers` version that supports qwen3_omni.
+
+`docker pull rocm/vllm-dev:nightly_main_20251005`
+
+#### Launch the docker environment
+
+In this docker environment, we have all the dependencies for vLLM installed, like `flash-attn` and `aiter`
+
+```bash
+#!/bin/bash
+docker run -it \
+   --privileged \
+   --network=host \
+   --group-add=video \
+   --ipc=host \
+   --cap-add=SYS_PTRACE \
+   --security-opt seccomp=unconfined \
+   --device /dev/kfd \
+   --device /dev/dri \
+   --name vllmomni \
+   rocm/vllm-dev:nightly_main_20251005 \
+   bash
+```
+
+#### Update docker dependencies
+
+```bash
+sudo apt update
+sudo apt install ffmpeg -y
+```
+
+#### Installation of vLLM
+Install the v0.11.0 release of vLLM as vLLM-Omni is currently depend on v0.11.0.
+
+```bash
+# uninstall existing vllm which is based on the main branch, not on the releases branch
+python3 -m pip uninstall -y vllm
+git clone https://github.com/vllm-project/vllm.git vllm0110
+cd vllm0110/
+
+# checkout the version tag v0.11.0
+git checkout -b v0.11.0 v0.11.0
+
+# you should see 0.11.0
+python -c "import setuptools_scm; print(setuptools_scm.get_version())"
+
+# PYTORCH_ROCM_ARCH=<your-gpu-arch> python3 setup.py develop
+PYTORCH_ROCM_ARCH=gfx942 python3 setup.py develop
+```
+
+!!! note
+    vLLM release wheels based on the branch with prefix `releases/`, not from the tag as vLLM may cherry pick bugfixes after cutting a branch.
+
+
+#### Installation of vLLM-Omni
+
+There are two approaches to installing `vllm-omni`
+
+
+1. Install from PyPI
+
+```bash
+pip install vllm-omni
+
+# you might want to also download the source code of the same version to run the examples
+```
+
+2. Install from source
+
+```bash
+# clone and checkout to the release version.
+git clone https://github.com/vllm-project/vllm-omni.git
+cd vllm-omni
+git checkout v0.11.0rc1
+pip install -e .
+```
+
+#### Setup Environment Variables
+
+After installing `vllm` and `vllm-omni` please setup the environment variable as well before running any of the examples and command.
+
+```bash
+export MIOPEN_FIND_MODE=FAST
+export VLLM_ROCM_USE_AITER=1
+export VLLM_ROCM_USE_AITER_MHA=1
+export VLLM_ROCM_USE_AITER_LINEAR=0
+export VLLM_ROCM_USE_AITER_RMSNORM=0
+```
+
+# --8<-- [end:build-wheel-from-source-in-docker]
+
+# --8<-- [start:pre-built-images]
+
+# --8<-- [end:pre-built-images]

docs/getting_started/installation/npu.md

@@ -1,3 +1,23 @@
-# Ascend-NPU
+# NPU
 
-vLLM-Omni is a Python library that supports the following NPU variants. Select your NPU type to see vendor specific instructions:
+vLLM-Omni supports NPU through the vLLM Ascend Plugin (vllm-ascend). This is a community maintained hardware plugin for running vLLM on NPU.
+
+## Requirements
+
+- OS: Linux
+- Python: 3.12
+
+!!! note
+    vLLM-Omni is currently not natively supported on Windows.
+
+=== "NPU"
+
+    --8<-- "docs/getting_started/installation/npu/npu.inc.md:requirements"
+
+## Installation
+
+### Recommended
+
+=== "NPU"
+
+    --8<-- "docs/getting_started/installation/npu/npu.inc.md:installation"

docs/getting_started/installation/npu/npu.inc.md

@@ -1,5 +1,49 @@
+# --8<-- [start:requirements]
+
+For detailed hardware and software requirements, please refer to the [vllm-ascend installation documentation](https://docs.vllm.ai/projects/ascend/en/latest/installation.html).
+
+# --8<-- [end:requirements]
 # --8<-- [start:installation]
 
-vLLM-Omni mainly contains python implementations for framework and models.
+The recommended way to use vLLM-Omni on NPU is through the vllm-ascend pre-built Docker images:
+
+```bash
+# Update DEVICE according to your NPUs (/dev/davinci[0-7])
+export DEVICE0=/dev/davinci0
+export DEVICE1=/dev/davinci1
+# Update the vllm-ascend image
+# Atlas A2:
+# export IMAGE=quay.io/ascend/vllm-ascend:v0.11.0rc2
+# Atlas A3:
+# export IMAGE=quay.io/ascend/vllm-ascend:v0.11.0rc2-a3
+export IMAGE=quay.io/ascend/vllm-ascend:v0.11.0rc2
+docker run --rm \
+    --name vllm-omni-npu \
+    --shm-size=1g \
+    --device $DEVICE0 \
+    --device $DEVICE1 \
+    --device /dev/davinci_manager \
+    --device /dev/devmm_svm \
+    --device /dev/hisi_hdc \
+    -v /usr/local/dcmi:/usr/local/dcmi \
+    -v /usr/local/bin/npu-smi:/usr/local/bin/npu-smi \
+    -v /usr/local/Ascend/driver/lib64/:/usr/local/Ascend/driver/lib64/ \
+    -v /usr/local/Ascend/driver/version.info:/usr/local/Ascend/driver/version.info \
+    -v /etc/ascend_install.info:/etc/ascend_install.info \
+    -v /root/.cache:/root/.cache \
+    -p 8000:8000 \
+    -it $IMAGE bash
+
+# Inside the container, install vLLM-Omni from source
+cd /vllm-workspace
+git clone https://github.com/vllm-project/vllm-omni.git
+cd vllm-omni
+pip install -v -e .
+export VLLM_WORKER_MULTIPROC_METHOD=spawn
+```
+
+The default workdir is `/workspace`, with vLLM, vLLM-Ascend and vLLM-Omni code placed in `/vllm-workspace` installed in development mode.
+
+For other installation methods (pip installation, building from source, custom Docker builds), please refer to the [vllm-ascend installation guide](https://docs.vllm.ai/projects/ascend/en/latest/installation.html).
 
 # --8<-- [end:installation]

docs/getting_started/quickstart.md

@@ -16,7 +16,19 @@ Please refer to [installation](installation/README.md)
 
 ## Offline Inference
 
-Please refer to [offline inference](../user_guide/examples/offline_inference/qwen2_5_omni.md)
+Text-to-image generation quickstart with vLLM-Omni:
+
+```python
+from vllm_omni.entrypoints.omni import Omni
+
+if __name__ == "__main__":
+    omni = Omni(model="Tongyi-MAI/Z-Image-Turbo")
+    prompt = "a cup of coffee on the table"
+    images = omni.generate(prompt)
+    images[0].save("coffee.png")
+```
+
+For more usages, please refer to [offline inference](../user_guide/examples/offline_inference/qwen2_5_omni.md)
 
 ## Online Serving with OpenAI-Completions API
 

docs/mkdocs/hooks/generate_examples.py

@@ -0,0 +1,336 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+import itertools
+import logging
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Literal
+
+import regex as re
+import yaml
+
+logger = logging.getLogger("mkdocs")
+
+ROOT_DIR = Path(__file__).parent.parent.parent.parent
+ROOT_DIR_RELATIVE = "../../../../.."
+EXAMPLE_DIR = ROOT_DIR / "examples"
+EXAMPLE_DOC_DIR = ROOT_DIR / "docs/user_guide/examples"
+NAV_FILE = ROOT_DIR / "docs/.nav.yml"
+
+
+def fix_case(text: str) -> str:
+    subs = {
+        "api": "API",
+        "cli": "CLI",
+        "cpu": "CPU",
+        "llm": "LLM",
+        "mae": "MAE",
+        "tpu": "TPU",
+        "gguf": "GGUF",
+        "lora": "LoRA",
+        "rlhf": "RLHF",
+        "vllm": "vLLM",
+        "openai": "OpenAI",
+        "lmcache": "LMCache",
+        "multilora": "MultiLoRA",
+        "mlpspeculator": "MLPSpeculator",
+        r"fp\d+": lambda x: x.group(0).upper(),  # e.g. fp16, fp32
+        r"int\d+": lambda x: x.group(0).upper(),  # e.g. int8, int16
+    }
+    for pattern, repl in subs.items():
+        text = re.sub(rf"\b{pattern}\b", repl, text, flags=re.IGNORECASE)
+    return text
+
+
+@dataclass
+class Example:
+    """
+    Example class for generating documentation content from a given path.
+
+    Attributes:
+        path (Path): The path to the main directory or file.
+        category (str): The category of the document.
+        main_file (Path): The main file in the directory.
+        other_files (list[Path]): list of other files in the directory.
+        title (str): The title of the document.
+
+    Methods:
+        __post_init__(): Initializes the main_file, other_files, and title attributes.
+        determine_main_file() -> Path: Determines the main file in the given path.
+        determine_other_files() -> list[Path]: Determines other files in the directory excluding the main file.
+        determine_title() -> str: Determines the title of the document.
+        generate() -> str: Generates the documentation content.
+    """  # noqa: E501
+
+    path: Path
+    category: str = None
+    main_file: Path = field(init=False)
+    other_files: list[Path] = field(init=False)
+    title: str = field(init=False)
+
+    def __post_init__(self):
+        self.main_file = self.determine_main_file()
+        self.other_files = self.determine_other_files()
+        self.title = self.determine_title()
+
+    @property
+    def is_code(self) -> bool:
+        return self.main_file.suffix != ".md"
+
+    def determine_main_file(self) -> Path:
+        """
+        Determines the main file in the given path.
+        If the path is a file, it returns the path itself. Otherwise, it searches
+        for Markdown files (*.md) in the directory and returns the first one found.
+        Returns:
+            Path: The main file path, either the original path if it's a file or the first
+            Markdown file found in the directory.
+        Raises:
+            IndexError: If no Markdown files are found in the directory.
+        """  # noqa: E501
+        return self.path if self.path.is_file() else list(self.path.glob("*.md")).pop()
+
+    def determine_other_files(self) -> list[Path]:
+        """
+        Determine other files in the directory excluding the main file.
+
+        This method checks if the given path is a file. If it is, it returns an empty list.
+        Otherwise, it recursively searches through the directory and returns a list of all
+        files that are not the main file.
+
+        Returns:
+            list[Path]: A list of Path objects representing the other files in the directory.
+        """  # noqa: E501
+        if self.path.is_file():
+            return []
+        # Binary file extensions to exclude
+        binary_extensions = {
+            ".wav",
+            ".mp3",
+            ".mp4",
+            ".avi",
+            ".mov",
+            ".mkv",  # Audio/Video
+            ".png",
+            ".jpg",
+            ".jpeg",
+            ".gif",
+            ".bmp",
+            ".ico",
+            ".svg",  # Images
+            ".pdf",
+            ".zip",
+            ".tar",
+            ".gz",
+            ".bz2",
+            ".xz",  # Archives/Documents
+            ".exe",
+            ".so",
+            ".dll",
+            ".dylib",  # Binaries
+            ".bin",
+            ".dat",
+            ".db",
+            ".sqlite",  # Data files
+        }
+
+        def is_other_file(file: Path) -> bool:
+            return file.is_file() and file != self.main_file and file.suffix.lower() not in binary_extensions
+
+        return [file for file in self.path.rglob("*") if is_other_file(file)]
+
+    def determine_title(self) -> str:
+        if not self.is_code:
+            # Specify encoding for building on Windows
+            with open(self.main_file, encoding="utf-8") as f:
+                first_line = f.readline().strip()
+            match = re.match(r"^#\s+(?P<title>.+)$", first_line)
+            if match:
+                return match.group("title")
+        return fix_case(self.path.stem.replace("_", " ").title())
+
+    def fix_relative_links(self, content: str) -> str:
+        """
+        Fix relative links in markdown content by converting them to gh-file
+        format.
+
+        Args:
+            content (str): The markdown content to process
+
+        Returns:
+            str: Content with relative links converted to gh-file format
+        """
+        # Regex to match markdown links [text](relative_path)
+        # This matches links that don't start with http, https, ftp, or #
+        link_pattern = r"\[([^\]]*)\]\((?!(?:https?|ftp)://|#)([^)]+)\)"
+
+        def replace_link(match):
+            link_text = match.group(1)
+            relative_path = match.group(2)
+
+            # Make relative to repo root
+            gh_file = (self.main_file.parent / relative_path).resolve()
+            gh_file = gh_file.relative_to(ROOT_DIR)
+
+            # Make GitHub URL
+            url = "https://github.com/vllm-project/vllm-omni/"
+            url += "tree/main" if self.path.is_dir() else "blob/main"
+            gh_url = f"{url}/{gh_file}"
+
+            return f"[{link_text}]({gh_url})"
+
+        return re.sub(link_pattern, replace_link, content)
+
+    def generate(self) -> str:
+        content = f"# {self.title}\n\n"
+        url = "https://github.com/vllm-project/vllm-omni/"
+        url += "tree/main" if self.path.is_dir() else "blob/main"
+        content += f"Source <{url}/{self.path.relative_to(ROOT_DIR)}>.\n\n"
+
+        # Use long code fence to avoid issues with
+        # included files containing code fences too
+        code_fence = "``````"
+
+        if self.is_code:
+            main_file_rel = self.main_file.relative_to(ROOT_DIR)
+            content += f'{code_fence}{self.main_file.suffix[1:]}\n--8<-- "{main_file_rel}"\n{code_fence}\n'
+        else:
+            with open(self.main_file) as f:
+                # Skip the title from md snippets as it's been included above
+                main_content = f.readlines()[1:]
+            content += self.fix_relative_links("".join(main_content))
+        content += "\n"
+
+        if not self.other_files:
+            return content
+
+        content += "## Example materials\n\n"
+        for file in sorted(self.other_files):
+            content += f'??? abstract "{file.relative_to(self.path)}"\n'
+            if file.suffix != ".md":
+                content += f"    {code_fence}{file.suffix[1:]}\n"
+            content += f'    --8<-- "{file.relative_to(ROOT_DIR)}"\n'
+            if file.suffix != ".md":
+                content += f"    {code_fence}\n"
+
+        return content
+
+
+def update_nav_file(examples: list[Example]):
+    """
+    Update the .nav.yml file to include all generated examples.
+    This function completely regenerates the examples section based on the actual
+    folder structure, ensuring consistency between the examples folder and nav file.
+
+    Args:
+        examples: List of Example objects that have been generated
+    """
+    if not NAV_FILE.exists():
+        logger.warning("Navigation file not found: %s", NAV_FILE)
+        return
+
+    # Read the current nav file
+    with open(NAV_FILE, encoding="utf-8") as f:
+        nav_data = yaml.safe_load(f) or {}
+
+    nav_list = nav_data.get("nav", [])
+
+    # Find the "User Guide" section
+    user_guide_idx = None
+    examples_idx = None
+    for i, item in enumerate(nav_list):
+        if isinstance(item, dict) and "User Guide" in item:
+            user_guide_idx = i
+            user_guide_content = item["User Guide"]
+            # Find the "Examples" subsection
+            for j, subitem in enumerate(user_guide_content):
+                if isinstance(subitem, dict) and "Examples" in subitem:
+                    examples_idx = j
+                    break
+            break
+
+    if user_guide_idx is None or examples_idx is None:
+        logger.warning("Could not find 'User Guide' -> 'Examples' section in nav file")
+        return
+
+    # Get existing Examples section to preserve non-example items (like README.md)
+    existing_examples_content = nav_list[user_guide_idx]["User Guide"][examples_idx]["Examples"]
+
+    # Preserve string items (like "examples/README.md") that are not example categories
+    preserved_items = [
+        item
+        for item in existing_examples_content
+        if isinstance(item, str) and not item.startswith("user_guide/examples/")
+    ]
+
+    # Group examples by category
+    examples_by_category = {}
+    for example in examples:
+        category = example.category
+        if category not in examples_by_category:
+            examples_by_category[category] = []
+        examples_by_category[category].append(example)
+
+    # Build the new Examples section - start with preserved items
+    examples_section = preserved_items.copy()
+
+    # Add examples grouped by category, sorted by category name
+    for category in sorted(examples_by_category.keys()):
+        category_examples = sorted(examples_by_category[category], key=lambda e: e.path.stem)
+        category_items = []
+        for example in category_examples:
+            doc_path = EXAMPLE_DOC_DIR / example.category / f"{example.path.stem}.md"
+            rel_path = doc_path.relative_to(ROOT_DIR / "docs")
+            category_items.append({example.title: str(rel_path)})
+
+        if category_items:
+            # Format category name (e.g., "offline_inference" -> "Offline Inference")
+            category_title = fix_case(category.replace("_", " ").title())
+            examples_section.append({category_title: category_items})
+
+    # Update the nav structure
+    nav_list[user_guide_idx]["User Guide"][examples_idx]["Examples"] = examples_section
+
+    # Write back to file
+    nav_data["nav"] = nav_list
+    with open(NAV_FILE, "w", encoding="utf-8") as f:
+        yaml.dump(nav_data, f, default_flow_style=False, sort_keys=False, allow_unicode=True)
+    logger.info("Updated navigation file: %s", NAV_FILE.relative_to(ROOT_DIR))
+
+
+def on_startup(command: Literal["build", "gh-deploy", "serve"], dirty: bool):
+    logger.info("Generating example documentation")
+    logger.debug("Root directory: %s", ROOT_DIR.resolve())
+    logger.debug("Example directory: %s", EXAMPLE_DIR.resolve())
+    logger.debug("Example document directory: %s", EXAMPLE_DOC_DIR.resolve())
+
+    # Create the EXAMPLE_DOC_DIR if it doesn't exist
+    if not EXAMPLE_DOC_DIR.exists():
+        EXAMPLE_DOC_DIR.mkdir(parents=True)
+
+    categories = sorted(p for p in EXAMPLE_DIR.iterdir() if p.is_dir())
+
+    examples = []
+    glob_patterns = ["*.py", "*.md", "*.sh"]
+    # Find categorised examples
+    for category in categories:
+        globs = [category.glob(pattern) for pattern in glob_patterns]
+        for path in itertools.chain(*globs):
+            examples.append(Example(path, category.stem))
+        # Find examples in subdirectories
+        for path in category.glob("*/*.md"):
+            examples.append(Example(path.parent, category.stem))
+
+    # Generate the example documentation
+    for example in sorted(examples, key=lambda e: e.path.stem):
+        example_name = f"{example.path.stem}.md"
+        doc_path = EXAMPLE_DOC_DIR / example.category / example_name
+        if not doc_path.parent.exists():
+            doc_path.parent.mkdir(parents=True)
+        # Specify encoding for building on Windows
+        with open(doc_path, "w+", encoding="utf-8") as f:
+            f.write(example.generate())
+        logger.debug("Example generated: %s", doc_path.relative_to(ROOT_DIR))
+
+    # Update the navigation file
+    update_nav_file(examples)

docs/mkdocs/stylesheets/extra.css

@@ -25,8 +25,11 @@ a:not(:has(svg)):not(.md-icon):not(.autorefs-external) {
 
 a[href*="localhost"]::after,
 a[href*="127.0.0.1"]::after,
-a[href*="org.readthedocs.build"]::after,
-a[href*="docs.vllm.ai"]::after {
+
+/* Hide external link icons for all links */
+a[href^="//"]::after,
+a[href^="http://"]::after,
+a[href^="https://"]::after {
     display: none !important;
 }
 

docs/models/supported_models.md

@@ -4,9 +4,9 @@ vLLM-Omni supports unified multimodal comprehension and generation models across
 
 ## Model Implementation
 
-If vLLM-Omni natively supports a model, its implementation can be found in <gh-file:vllm-omni/model_executor/models> and <gh-file:vllm_omni/diffusion/models>.
+If vLLM-Omni natively supports a model, its implementation can be found in <gh-file:vllm_omni/model_executor/models> and <gh-file:vllm_omni/diffusion/models>.
 
-## List of Supported Models for Nvidia GPU
+## List of Supported Models for Nvidia GPU / AMD GPU
 
 <style>
 th {
@@ -17,11 +17,15 @@ th {
 
 | Architecture | Models | Example HF Models |
 |--------------|--------|-------------------|
+| `Qwen3OmniMoeForConditionalGeneration` | Qwen3-Omni | `Qwen/Qwen3-Omni-30B-A3B-Instruct` |
 | `Qwen2_5OmniForConditionalGeneration` | Qwen2.5-Omni | `Qwen/Qwen2.5-Omni-7B`, `Qwen/Qwen2.5-Omni-3B` |
 | `QwenImagePipeline` | Qwen-Image | `Qwen/Qwen-Image` |
+| `QwenImageEditPipeline` | Qwen-Image-Edit | `Qwen/Qwen-Image-Edit` |
+|`ZImagePipeline` | Z-Image | `Tongyi-MAI/Z-Image-Turbo` |
+| `WanPipeline` | Wan2.2 | `Wan-AI/Wan2.2-T2V-A14B-Diffusers` |
 
 
-## List of Supported Models for Ascend NPU
+## List of Supported Models for NPU
 
 <style>
 th {
@@ -33,3 +37,4 @@ th {
 | Architecture | Models | Example HF Models |
 |--------------|--------|-------------------|
 | `Qwen2_5OmniForConditionalGeneration` | Qwen2.5-Omni | `Qwen/Qwen2.5-Omni-7B`, `Qwen/Qwen2.5-Omni-3B`|
+| `QwenImagePipeline` | Qwen-Image | `Qwen/Qwen-Image` |

docs/usage/faq.md

@@ -2,7 +2,7 @@
 
 > Q: How many chips do I need to infer a model in vLLM-Omni?
 
-A: Now, we support natively disaggregated deployment for different model stages within a model. There is a restriction that one chip can only has one AutoRegressive model stage. This is because the unified KV cache management of vLLM. Stages of other types can coexist with in a chip. The restriction will be resolved in later version.
+A: Now, we support natively disaggregated deployment for different model stages within a model. There is a restriction that one chip can only have one AutoRegressive model stage. This is because the unified KV cache management of vLLM. Stages of other types can coexist within a chip. The restriction will be resolved in later version.
 
 > Q: When trying to run examples, I encounter error about backend of librosa or soundfile. How to solve it?
 
@@ -11,3 +11,15 @@ A: If you encounter error about backend of librosa, try to install ffmpeg with c
 sudo apt update
 sudo apt install ffmpeg
 ```
+
+> Q: I encounter some bugs or CI problems, which is urgent. How can I solve it?
+
+A: At first, you can check current [issues](https://github.com/vllm-project/vllm-omni/issues) to find possible solutions. If none of these satisfy your demand and it is urgent, please find these [volunteers](https://docs.vllm.ai/projects/vllm-omni/en/latest/community/volunteers/) for help.
+
+> Q: Does vLLM-Omni support AWQ or any other quantization?
+
+A: vLLM-Omni partitions model into several stages. For AR stages, it will reuse main logic of LLMEngine in vLLM. So current quantization supported in vLLM should be also supported in vLLM-Omni for them. But systematic verification is ongoing. For quantization for DiffusionEngine, we are working on it. Please stay tuned and welcome contribution!
+
+> Q: Does vLLM-Omni support multimodal streaming input and output?
+
+A: Not yet. We already put it on the [Roadmap](https://github.com/vllm-project/vllm-omni/issues/165). Please stay tuned!

docs/user_guide/examples/offline_inference/image_to_image.md

@@ -0,0 +1,30 @@
+# Qwen-Image-Edit Offline Inference
+
+Source <https://github.com/vllm-project/vllm-omni/tree/main/examples/offline_inference/image_to_image>. Added in PR [#196](https://github.com/vllm-project/vllm-omni/pull/196).
+
+This example edits an input image with `Qwen/Qwen-Image-Edit` using the `image_edit.py` CLI.
+
+## Local CLI Usage
+
+```bash
+python image_edit.py \
+  --image qwen_bear.png \
+  --prompt "Let this mascot dance under the moon, surrounded by floating stars and poetic bubbles such as 'Be Kind'" \
+  --output output_image_edit.png \
+  --num_inference_steps 50 \
+  --cfg_scale 4.0
+```
+
+Key arguments:
+- `--image`: path to the source image (PNG/JPG, converted to RGB).
+- `--prompt` / `--negative_prompt`: text description (string).
+- `--cfg_scale`: true CFG scale for Qwen-Image-Edit (quality vs. fidelity).
+- `--num_inference_steps`: diffusion sampling steps (more steps = higher quality, slower).
+- `--output`: path to save the generated PNG.
+
+## Example materials
+
+??? abstract "image_edit.py"
+    ``````py
+    --8<-- "examples/offline_inference/image_to_image/image_edit.py"
+    ``````

docs/user_guide/examples/offline_inference/qwen2_5_omni.md

@@ -1,55 +1,70 @@
-# Offline Inference Example of vLLM-Omni for Qwen2.5-Omni
+# Offline Example of vLLM-Omni for Qwen2.5-Omni
 
 Source <https://github.com/vllm-project/vllm-omni/tree/main/examples/offline_inference/qwen2_5_omni>.
 
 
 ## 🛠️ Installation
 
-Please refer to [installation](../../../getting_started/installation/README.md).
+Please refer to [README.md](https://github.com/vllm-project/vllm-omni/tree/main/README.md)
 
-## Run Offline inference with Qwen2.5-Omni.
-First, navigate to the example folder
-```bash
-cd examples/offline_inference/qwen2_5_omni
-```
-Inside the directory, `end2end.py` is a comprehensive demo suite for initializing a model instance of `Qwen/Qwen2.5-Omni-7B` and use it for various offline inference tasks.
-??? abstract "end2end.py"
-    ``````py
-    --8<-- "examples/offline_inference/qwen2_5_omni/end2end.py"
-    ``````
-
-Below we also provide simple bash scripts to execute this file.
-### Single Prompt
-```bash
-bash run_single_prompt.sh
-```
-??? abstract "run_single_prompt.sh"
-    ``````sh
-    --8<-- "examples/offline_inference/qwen2_5_omni/run_single_prompt.sh"
-    ``````
+## Run examples (Qwen2.5-Omni)
 
 ### Multiple Prompts
-Download dataset from [seed_tts](https://drive.google.com/file/d/1GlSjVfSHkW3-leKKBlfrjuuTGqQ_xaLP/edit) and extract the prompts with `extract_prompts.py`
-??? abstract "extract_prompts.py"
-    ``````py
-    --8<-- "examples/offline_inference/qwen2_5_omni/extract_prompts.py"
-    ``````
+Download dataset from [seed_tts](https://drive.google.com/file/d/1GlSjVfSHkW3-leKKBlfrjuuTGqQ_xaLP/edit). To get the prompt, you can:
 ```bash
 tar -xf <Your Download Path>/seedtts_testset.tar
 cp seedtts_testset/en/meta.lst examples/offline_inference/qwen2_5_omni/meta.lst
 python3 examples/offline_inference/qwen2_5_omni/extract_prompts.py \
   --input examples/offline_inference/qwen2_5_omni/meta.lst \
   --output examples/offline_inference/qwen2_5_omni/top100.txt \
-  --topk 10
+  --topk 100
+```
+Get into the example folder
+```bash
+cd examples/offline_inference/qwen2_5_omni
 ```
 Then run the command below.
 ```bash
 bash run_multiple_prompts.sh
 ```
-??? abstract "run_multiple_prompts.sh"
-    ``````sh
-    --8<-- "examples/offline_inference/qwen2_5_omni/run_multiple_prompts.sh"
-    ``````
+
+### Single Prompt
+Get into the example folder
+```bash
+cd examples/offline_inference/qwen2_5_omni
+```
+Then run the command below.
+```bash
+bash run_single_prompt.sh
+```
+
+#### Using Local Media Files
+The `end2end.py` script supports local media files (audio, video, image) via CLI arguments:
+
+```bash
+# Use single local media files
+python end2end.py --query-type use_image --image-path /path/to/image.jpg
+python end2end.py --query-type use_video --video-path /path/to/video.mp4
+python end2end.py --query-type use_audio --audio-path /path/to/audio.wav
+
+# Combine multiple local media files
+python end2end.py --query-type mixed_modalities \
+    --video-path /path/to/video.mp4 \
+    --image-path /path/to/image.jpg \
+    --audio-path /path/to/audio.wav
+
+# Use audio from video file
+python end2end.py --query-type use_audio_in_video --video-path /path/to/video.mp4
+
+```
+
+If media file paths are not provided, the script will use default assets. Supported query types:
+- `use_image`: Image input only
+- `use_video`: Video input only
+- `use_audio`: Audio input only
+- `mixed_modalities`: Audio + image + video
+- `use_audio_in_video`: Extract audio from video
+- `text`: Text-only query
 
 ### FAQ
 
@@ -58,3 +73,22 @@ If you encounter error about backend of librosa, try to install ffmpeg with comm
 sudo apt update
 sudo apt install ffmpeg
 ```
+
+## Example materials
+
+??? abstract "end2end.py"
+    ``````py
+    --8<-- "examples/offline_inference/qwen2_5_omni/end2end.py"
+    ``````
+??? abstract "extract_prompts.py"
+    ``````py
+    --8<-- "examples/offline_inference/qwen2_5_omni/extract_prompts.py"
+    ``````
+??? abstract "run_multiple_prompts.sh"
+    ``````sh
+    --8<-- "examples/offline_inference/qwen2_5_omni/run_multiple_prompts.sh"
+    ``````
+??? abstract "run_single_prompt.sh"
+    ``````sh
+    --8<-- "examples/offline_inference/qwen2_5_omni/run_single_prompt.sh"
+    ``````

docs/user_guide/examples/offline_inference/qwen3_omni.md

@@ -0,0 +1,81 @@
+# Offline Example of vLLM-Omni for Qwen3-Omni
+
+Source <https://github.com/vllm-project/vllm-omni/tree/main/examples/offline_inference/qwen3_omni>.
+
+
+## 🛠️ Installation
+
+Please refer to [README.md](https://github.com/vllm-project/vllm-omni/tree/main/README.md)
+
+## Run examples (Qwen3-Omni)
+
+### Multiple Prompts
+Download dataset from [seed_tts](https://drive.google.com/file/d/1GlSjVfSHkW3-leKKBlfrjuuTGqQ_xaLP/edit). For processing dataset please refer to [Qwen2.5-Omni README.md](https://github.com/vllm-project/vllm-omni/tree/main/examples/offline_inference/qwen2_5_omni/README.md)
+Get into the example folder
+```bash
+cd examples/offline_inference/qwen3_omni
+```
+Then run the command below.
+```bash
+bash run_multiple_prompts.sh
+```
+### Single Prompt
+Get into the example folder
+```bash
+cd examples/offline_inference/qwen3_omni
+```
+Then run the command below.
+```bash
+bash run_single_prompt.sh
+```
+If you have not enough memory, you can set thinker with tensor parallel. Just run the command below.
+```bash
+bash run_single_prompt_tp.sh
+```
+
+#### Using Local Media Files
+The `end2end.py` script supports local media files (audio, video, image) via command-line arguments:
+
+```bash
+# Use local video file
+python end2end.py --query-type use_video --video-path /path/to/video.mp4
+
+# Use local image file
+python end2end.py --query-type use_image --image-path /path/to/image.jpg
+
+# Use local audio file
+python end2end.py --query-type use_audio --audio-path /path/to/audio.wav
+```
+
+If media file paths are not provided, the script will use default assets. Supported query types:
+- `use_video`: Video input
+- `use_image`: Image input
+- `use_audio`: Audio input
+- `text`: Text-only query
+
+### FAQ
+
+If you encounter error about backend of librosa, try to install ffmpeg with command below.
+```
+sudo apt update
+sudo apt install ffmpeg
+```
+
+## Example materials
+
+??? abstract "end2end.py"
+    ``````py
+    --8<-- "examples/offline_inference/qwen3_omni/end2end.py"
+    ``````
+??? abstract "run_multiple_prompts.sh"
+    ``````sh
+    --8<-- "examples/offline_inference/qwen3_omni/run_multiple_prompts.sh"
+    ``````
+??? abstract "run_single_prompt.sh"
+    ``````sh
+    --8<-- "examples/offline_inference/qwen3_omni/run_single_prompt.sh"
+    ``````
+??? abstract "run_single_prompt_tp.sh"
+    ``````sh
+    --8<-- "examples/offline_inference/qwen3_omni/run_single_prompt_tp.sh"
+    ``````

docs/user_guide/examples/offline_inference/text_to_image.md

@@ -0,0 +1,56 @@
+# Qwen-Image Offline Inference
+
+Source <https://github.com/vllm-project/vllm-omni/tree/main/examples/offline_inference/text_to_image>.
+
+
+This folder provides two simple entrypoints for experimenting with `Qwen/Qwen-Image` `Tongyi-MAI/Z-Image-Turbo` using vLLM-Omni:
+
+- `text_to_image.py`: command-line script for single image generation.
+- `web_demo.py`: lightweight Gradio UI for interactive prompt/seed/CFG exploration.
+
+
+## Local CLI Usage
+
+```bash
+python text_to_image.py \
+  --model Tongyi-MAI/Z-Image-Turbo \
+  --prompt "a cup of coffee on the table" \
+  --seed 42 \
+  --cfg_scale 4.0 \
+  --num_images_per_prompt 1 \
+  --num_inference_steps 50 \
+  --height 1024 \
+  --width 1024 \
+  --output outputs/coffee.png
+```
+
+Key arguments:
+
+- `--prompt`: text description (string).
+- `--seed`: integer seed for deterministic sampling.
+- `--cfg_scale`: true CFG scale (model-specific guidance strength).
+- `--num_images_per_prompt`: number of images to generate per prompt (saves as `output`, `output_1`, ...).
+- `--num_inference_steps`: diffusion sampling steps (more steps = higher quality, slower).
+- `--height/--width`: output resolution (defaults 1024x1024).
+- `--output`: path to save the generated PNG.
+
+## Web UI Demo
+
+Launch the gradio demo:
+
+```bash
+python gradio_demo.py --port 7862
+```
+
+Then open `http://localhost:7862/` on your local browser to interact with the web UI.
+
+## Example materials
+
+??? abstract "gradio_demo.py"
+    ``````py
+    --8<-- "examples/offline_inference/text_to_image/gradio_demo.py"
+    ``````
+??? abstract "text_to_image.py"
+    ``````py
+    --8<-- "examples/offline_inference/text_to_image/text_to_image.py"
+    ``````

docs/user_guide/examples/offline_inference/text_to_video.md

@@ -0,0 +1,38 @@
+# Wan2.2 Text-to-Video Offline Inference
+
+Source <https://github.com/vllm-project/vllm-omni/tree/main/examples/offline_inference/text_to_video>. Added in PR [#202](https://github.com/vllm-project/vllm-omni/pull/202).
+
+The `Wan-AI/Wan2.2-T2V-A14B-Diffusers` pipeline generates short videos from text prompts.
+
+## Local CLI Usage
+
+```bash
+python text_to_video.py \
+  --prompt "Two anthropomorphic cats in comfy boxing gear and bright gloves fight intensely on a spotlighted stage." \
+  --negative_prompt "<optional quality filter>" \
+  --height 480 \
+  --width 640 \
+  --num_frames 32 \
+  --guidance_scale 4.0 \
+  --guidance_scale_high 3.0 \
+  --num_inference_steps 40 \
+  --fps 16 \
+  --output t2v_out.mp4
+```
+
+Key arguments:
+- `--prompt`: text description (string).
+- `--height/--width`: output resolution (defaults 720x1280). Dimensions should align with Wan VAE downsampling (multiples of 8).
+- `--num_frames`: Number of frames (Wan default is 81).
+- `--guidance_scale` and `--guidance_scale_high`: CFG scale (applied to low/high)..
+- `--negative_prompt`: optional list of artifacts to suppress (the PR demo used a long Chinese string).
+- `--boundary_ratio`: Boundary split ratio for low/high DiT.
+- `--fps`: frames per second for the saved MP4 (requires `diffusers` export_to_video).
+- `--output`: path to save the generated video.
+
+## Example materials
+
+??? abstract "text_to_video.py"
+    ``````py
+    --8<-- "examples/offline_inference/wan22/text_to_video.py"
+    ``````

docs/user_guide/examples/online_serving/qwen2_5_omni.md

@@ -1,43 +1,99 @@
-# Online Serving Example of vLLM-Omni for Qwen2.5-Omni
+# Online serving Example of vLLM-Omni for Qwen2.5-Omni
 
-Source <https://github.com/vllm-project/vllm-omni/blob/main/examples/online_serving/README.md>.
+Source <https://github.com/vllm-project/vllm-omni/tree/main/examples/online_serving/qwen2_5_omni>.
 
 
 ## 🛠️ Installation
 
-Please refer to [installation](../../../getting_started/installation/README.md).
+Please refer to [README.md](https://github.com/vllm-project/vllm-omni/tree/main/README.md)
 
-## Deploy Qwen/Qwen2.5-Omni-7B
+## Run examples (Qwen2.5-Omni)
+
+### Launch the Server
 
-First launch the OpenAI-compatible inference server
 ```bash
 vllm serve Qwen/Qwen2.5-Omni-7B --omni --port 8091
 ```
+
 If you have custom stage configs file, launch the server with command below
 ```bash
 vllm serve Qwen/Qwen2.5-Omni-7B --omni --port 8091 --stage-configs-path /path/to/stage_configs_file
 ```
-## Query the model
-Navigate to the example folder
+
+### Send Multi-modal Request
+
+Get into the example folder
 ```bash
-cd examples/online_serving
+cd examples/online_serving/qwen2_5_omni
 ```
-Query the model server with OpenAI Python API client:
+
+####  Send request via python
+
 ```bash
 python openai_chat_completion_client_for_multimodal_generation.py --query-type mixed_modalities
 ```
-??? abstract "openai_chat_completion_client_for_multimodal_generation.py"
-    ``````py
-    --8<-- "examples/online_serving/openai_chat_completion_client_for_multimodal_generation.py"
-    ``````
-You can also query the model with `curl` command:
+
+The Python client supports the following command-line arguments:
+
+- `--query-type` (or `-q`): Query type (default: `mixed_modalities`)
+  - Options: `mixed_modalities`, `use_audio_in_video`, `multi_audios`, `text`
+- `--video-path` (or `-v`): Path to local video file or URL
+  - If not provided and query-type uses video, uses default video URL
+  - Supports local file paths (automatically encoded to base64) or HTTP/HTTPS URLs
+  - Example: `--video-path /path/to/video.mp4` or `--video-path https://example.com/video.mp4`
+- `--image-path` (or `-i`): Path to local image file or URL
+  - If not provided and query-type uses image, uses default image URL
+  - Supports local file paths (automatically encoded to base64) or HTTP/HTTPS URLs
+  - Supports common image formats: JPEG, PNG, GIF, WebP
+  - Example: `--image-path /path/to/image.jpg` or `--image-path https://example.com/image.png`
+- `--audio-path` (or `-a`): Path to local audio file or URL
+  - If not provided and query-type uses audio, uses default audio URL
+  - Supports local file paths (automatically encoded to base64) or HTTP/HTTPS URLs
+  - Supports common audio formats: MP3, WAV, OGG, FLAC, M4A
+  - Example: `--audio-path /path/to/audio.wav` or `--audio-path https://example.com/audio.mp3`
+- `--prompt` (or `-p`): Custom text prompt/question
+  - If not provided, uses default prompt for the selected query type
+  - Example: `--prompt "What are the main activities shown in this video?"`
+
+
+For example, to use mixed modalities with all local files:
+
+```bash
+python openai_chat_completion_client_for_multimodal_generation.py \
+    --query-type mixed_modalities \
+    --video-path /path/to/your/video.mp4 \
+    --image-path /path/to/your/image.jpg \
+    --audio-path /path/to/your/audio.wav \
+    --prompt "Analyze all the media content and provide a comprehensive summary."
+```
+
+####  Send request via curl
+
 ```bash
 bash run_curl_multimodal_generation.sh mixed_modalities
 ```
-??? abstract "run_curl_multimodal_generation.sh"
-    ``````py
-    --8<-- "examples/online_serving/run_curl_multimodal_generation.sh"
-    ``````
+
+## Run Local Web UI Demo
+
+This Web UI demo allows users to interact with the model through a web browser.
+
+### Running Gradio Demo
+
+Once vllm and vllm-omni are installed, you can launch the web service built on AsyncOmni by
+
+```bash
+python gradio_demo.py  --model Qwen/Qwen2.5-Omni-7B --port 7861
+```
+
+Then open `http://localhost:7861/` on your local browser to interact with the web UI.
+
+The gradio script supports the following arguments:
+
+- `--model`: Model name
+- `--ip`: Host/IP for Gradio server (default: 127.0.0.1)
+- `--port`: Port for Gradio server (default: 7861)
+- `--stage-configs-path`: Path to custom stage configs YAML file (optional)
+- `--share`: Share the Gradio demo publicly (creates a public link)
 
 ### FAQ
 
@@ -47,34 +103,17 @@ sudo apt update
 sudo apt install ffmpeg
 ```
 
-## Run Local Web UI Demo
+## Example materials
 
-You can also deploy a Gradio Web UI that allows users to interact with the model through a web browser. Below is an example on how to do so with `Qwen/Qwen2.5-Omni-7B`.
-
-### Running Gradio Demo
-
-Install gradio with `uv pip install "gradio>=5.49.1,<6.0.0"`, then you can launch the web service built on AsyncOmni by
-
-```bash
-python gradio_demo.py  --model Qwen/Qwen2.5-Omni-7B --port 7861
-```
-Now you can interact with model via the web UI at `http://localhost:7861/` on your local browser.
-
-
-### Options
-
-You can customize its basic launch parameters:
-
-```bash
-python gradio_demo.py \
-    --model Qwen/Qwen2.5-Omni-7B \
-    --ip 127.0.0.1 \
-    --port 7861 \
-    --stage-configs-path /path/to/stage_configs.yaml
-```
-
-- `--model`: Local model checkpoint to load (default `Qwen/Qwen2.5-Omni-7B`).
-- `--ip`: Host/IP for the Gradio server (default `127.0.0.1`).
-- `--port`: Port for the Gradio server (default `7861`).
-- `--stage-configs-path`: Optional path to custom stage configs YAML.
-- `--share`: Set to expose a temporary public link via Gradio.
+??? abstract "gradio_demo.py"
+    ``````py
+    --8<-- "examples/online_serving/qwen2_5_omni/gradio_demo.py"
+    ``````
+??? abstract "openai_chat_completion_client_for_multimodal_generation.py"
+    ``````py
+    --8<-- "examples/online_serving/qwen2_5_omni/openai_chat_completion_client_for_multimodal_generation.py"
+    ``````
+??? abstract "run_curl_multimodal_generation.sh"
+    ``````sh
+    --8<-- "examples/online_serving/qwen2_5_omni/run_curl_multimodal_generation.sh"
+    ``````

docs/user_guide/examples/online_serving/qwen3_omni.md

@@ -0,0 +1,124 @@
+# Online serving Example of vLLM-Omni for Qwen3-omni
+
+Source <https://github.com/vllm-project/vllm-omni/tree/main/examples/online_serving/qwen3_omni>.
+
+
+## 🛠️ Installation
+
+Please refer to [README.md](https://github.com/vllm-project/vllm-omni/tree/main/README.md)
+
+## Run examples (Qwen3-Omni)
+
+### Launch the Server
+
+```bash
+vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091
+```
+
+If you have custom stage configs file, launch the server with command below
+```bash
+vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091 --stage-configs-path /path/to/stage_configs_file
+```
+
+### Send Multi-modal Request
+
+Get into the example folder
+```bash
+cd examples/online_serving/qwen3_omni
+```
+
+####  Send request via python
+
+```bash
+python openai_chat_completion_client_for_multimodal_generation.py --query-type use_image
+```
+
+The Python client supports the following command-line arguments:
+
+- `--query-type` (or `-q`): Query type (default: `use_video`)
+  - Options: `text`, `use_audio`, `use_image`, `use_video`
+- `--model` (or `-m`): Model name/path (default: `Qwen/Qwen3-Omni-30B-A3B-Instruct`)
+- `--video-path` (or `-v`): Path to local video file or URL
+  - If not provided and query-type is `use_video`, uses default video URL
+  - Supports local file paths (automatically encoded to base64) or HTTP/HTTPS URLs
+  - Example: `--video-path /path/to/video.mp4` or `--video-path https://example.com/video.mp4`
+- `--image-path` (or `-i`): Path to local image file or URL
+  - If not provided and query-type is `use_image`, uses default image URL
+  - Supports local file paths (automatically encoded to base64) or HTTP/HTTPS URLs
+  - Supports common image formats: JPEG, PNG, GIF, WebP
+  - Example: `--image-path /path/to/image.jpg` or `--image-path https://example.com/image.png`
+- `--audio-path` (or `-a`): Path to local audio file or URL
+  - If not provided and query-type is `use_audio`, uses default audio URL
+  - Supports local file paths (automatically encoded to base64) or HTTP/HTTPS URLs
+  - Supports common audio formats: MP3, WAV, OGG, FLAC, M4A
+  - Example: `--audio-path /path/to/audio.wav` or `--audio-path https://example.com/audio.mp3`
+- `--prompt` (or `-p`): Custom text prompt/question
+  - If not provided, uses default prompt for the selected query type
+  - Example: `--prompt "What are the main activities shown in this video?"`
+
+
+For example, to use a local video file with custom prompt:
+
+```bash
+python openai_chat_completion_client_for_multimodal_generation.py \
+    --query-type use_video \
+    --video-path /path/to/your/video.mp4 \
+    --prompt "What are the main activities shown in this video?"
+```
+
+####  Send request via curl
+
+```bash
+bash run_curl_multimodal_generation.sh use_image
+```
+
+
+### FAQ
+
+If you encounter error about backend of librosa, try to install ffmpeg with command below.
+```
+sudo apt update
+sudo apt install ffmpeg
+```
+
+## Run Local Web UI Demo
+
+This Web UI demo allows users to interact with the model through a web browser.
+
+### Running Gradio Demo
+
+Once vllm and vllm-omni are installed, you can launch the web service built on AsyncOmni by
+
+```bash
+python gradio_demo.py  --model Qwen/Qwen3-Omni-30B-A3B-Instruct --port 7861
+```
+
+Then open `http://localhost:7861/` on your local browser to interact with the web UI.
+
+
+The gradio script supports the following arguments:
+
+- `--model`: Model name
+- `--ip`: Host/IP for Gradio server (default: 127.0.0.1)
+- `--port`: Port for Gradio server (default: 7861)
+- `--stage-configs-path`: Path to custom stage configs YAML file (optional)
+- `--share`: Share the Gradio demo publicly (creates a public link)
+
+## Example materials
+
+??? abstract "gradio_demo.py"
+    ``````py
+    --8<-- "examples/online_serving/qwen3_omni/gradio_demo.py"
+    ``````
+??? abstract "openai_chat_completion_client_for_multimodal_generation.py"
+    ``````py
+    --8<-- "examples/online_serving/qwen3_omni/openai_chat_completion_client_for_multimodal_generation.py"
+    ``````
+??? abstract "qwen3_omni_moe_thinking.yaml"
+    ``````yaml
+    --8<-- "examples/online_serving/qwen3_omni/qwen3_omni_moe_thinking.yaml"
+    ``````
+??? abstract "run_curl_multimodal_generation.sh"
+    ``````sh
+    --8<-- "examples/online_serving/qwen3_omni/run_curl_multimodal_generation.sh"
+    ``````

docs/user_guide/teacache.md

@@ -0,0 +1,99 @@
+# TeaCache Configuration Guide
+
+TeaCache speeds up diffusion model inference by caching transformer computations when consecutive timesteps are similar. This typically provides **1.5x-2.0x speedup** with minimal quality loss.
+
+## Quick Start
+
+Enable TeaCache by setting `cache_adapter` to `"tea_cache"`:
+
+```python
+from vllm_omni import Omni
+
+# Simple configuration - model_class_name and model_type are auto-detected
+omni = Omni(
+    model="Qwen/Qwen-Image",
+    cache_adapter="tea_cache",
+    cache_config={
+        "rel_l1_thresh": 0.2  # Optional, defaults to 0.2
+    }
+)
+outputs = omni.generate(prompt="A cat sitting on a windowsill", num_inference_steps=50)
+```
+
+### Using Environment Variable
+
+You can also enable TeaCache via environment variable:
+
+```bash
+export DIFFUSION_CACHE_ADAPTER=tea_cache
+```
+
+Then initialize without explicitly setting `cache_adapter`:
+
+```python
+from vllm_omni import Omni
+
+omni = Omni(
+    model="Qwen/Qwen-Image",
+    cache_config={"rel_l1_thresh": 0.2}  # Optional
+)
+```
+
+## Configuration Parameters
+
+### `rel_l1_thresh` (float, default: `0.2`)
+
+Controls the balance between speed and quality. Lower values prioritize quality, higher values prioritize speed.
+
+**Recommended values:**
+
+- `0.2` - **~1.5x speedup** with minimal quality loss (recommended)
+- `0.4` - **~1.8x speedup** with slight quality loss
+- `0.6` - **~2.0x speedup** with noticeable quality loss
+- `0.8` - **~2.25x speedup** with significant quality loss
+
+## Examples
+
+### Python API
+
+```python
+from vllm_omni import Omni
+
+omni = Omni(
+    model="Qwen/Qwen-Image",
+    cache_adapter="tea_cache",
+    cache_config={"rel_l1_thresh": 0.2}
+)
+outputs = omni.generate(prompt="A cat sitting on a windowsill", num_inference_steps=50)
+```
+
+## Performance Tuning
+
+Start with the default `rel_l1_thresh=0.2` and adjust based on your needs:
+
+- **Maximum quality**: Use `0.1-0.2`
+- **Balanced**: Use `0.2-0.4` (recommended)
+- **Maximum speed**: Use `0.6-0.8` (may reduce quality)
+
+## Troubleshooting
+
+### Quality Degradation
+
+If you notice quality issues, lower the threshold:
+
+```python
+cache_config={"rel_l1_thresh": 0.1}  # More conservative caching
+```
+
+## Supported Models
+
+Currently supported models:
+
+- **QwenImagePipeline**
+  - Model identifier: `"Qwen/Qwen-Image"`
+  - Example: `model="Qwen/Qwen-Image"`
+
+Future support (extractors in development):
+
+- **FluxPipeline** - Coming soon
+- **CogVideoXPipeline** - Coming soon

examples/offline_inference/image_to_image/image_edit.py

@@ -0,0 +1,143 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+"""
+Example script for image editing with Qwen-Image-Edit.
+
+Usage:
+    python image_edit.py \
+        --image input.png \
+        --prompt "Let this mascot dance under the moon, surrounded by floating stars and poetic bubbles such as 'Be Kind'" \
+        --output output_image_edit.png \
+        --num_inference_steps 50 \
+        --cfg_scale 4.0
+
+For more options, run:
+    python image_edit.py --help
+"""
+
+import argparse
+import os
+from pathlib import Path
+
+import torch
+from PIL import Image
+
+from vllm_omni.entrypoints.omni import Omni
+from vllm_omni.utils.platform_utils import detect_device_type, is_npu
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description="Edit an image with Qwen-Image-Edit.")
+    parser.add_argument(
+        "--model",
+        default="Qwen/Qwen-Image-Edit",
+        help="Diffusion model name or local path.",
+    )
+    parser.add_argument(
+        "--image",
+        type=str,
+        required=True,
+        help="Path to input image file (PNG, JPG, etc.).",
+    )
+    parser.add_argument(
+        "--prompt",
+        type=str,
+        required=True,
+        help="Text prompt describing the edit to make to the image.",
+    )
+    parser.add_argument(
+        "--negative_prompt",
+        type=str,
+        default=" ",
+        required=False,
+    )
+    parser.add_argument(
+        "--seed",
+        type=int,
+        default=0,
+        help="Random seed for deterministic results.",
+    )
+    parser.add_argument(
+        "--cfg_scale",
+        type=float,
+        default=4.0,
+        help="True classifier-free guidance scale specific to Qwen-Image-Edit.",
+    )
+    parser.add_argument(
+        "--output",
+        type=str,
+        default="output_image_edit.png",
+        help="Path to save the edited image (PNG).",
+    )
+    parser.add_argument(
+        "--num_outputs_per_prompt",
+        type=int,
+        default=1,
+        help="Number of images to generate for the given prompt.",
+    )
+    parser.add_argument(
+        "--num_inference_steps",
+        type=int,
+        default=50,
+        help="Number of denoising steps for the diffusion sampler.",
+    )
+    return parser.parse_args()
+
+
+def main():
+    args = parse_args()
+
+    # Validate input image exists
+    if not os.path.exists(args.image):
+        raise FileNotFoundError(f"Input image not found: {args.image}")
+
+    # Load input image
+    input_image = Image.open(args.image).convert("RGB")
+    print(f"Loaded input image from {args.image} (size: {input_image.size})")
+
+    device = detect_device_type()
+    generator = torch.Generator(device=device).manual_seed(args.seed)
+
+    # Enable VAE memory optimizations on NPU
+    vae_use_slicing = is_npu()
+    vae_use_tiling = is_npu()
+
+    # Initialize Omni with QwenImageEditPipeline
+    omni = Omni(
+        model=args.model,
+        model_class_name="QwenImageEditPipeline",
+        vae_use_slicing=vae_use_slicing,
+        vae_use_tiling=vae_use_tiling,
+    )
+    print("Pipeline loaded")
+
+    # Generate edited image
+    images = omni.generate(
+        prompt=args.prompt,
+        pil_image=input_image,
+        negative_prompt=args.negative_prompt,
+        generator=generator,
+        true_cfg_scale=args.cfg_scale,
+        num_inference_steps=args.num_inference_steps,
+        num_outputs_per_prompt=args.num_outputs_per_prompt,
+    )
+
+    # Save output image(s)
+    output_path = Path(args.output)
+    output_path.parent.mkdir(parents=True, exist_ok=True)
+    suffix = output_path.suffix or ".png"
+    stem = output_path.stem or "output_image_edit"
+
+    if args.num_outputs_per_prompt <= 1:
+        images[0].save(output_path)
+        print(f"Saved edited image to {os.path.abspath(output_path)}")
+    else:
+        for idx, img in enumerate(images):
+            save_path = output_path.parent / f"{stem}_{idx}{suffix}"
+            img.save(save_path)
+            print(f"Saved edited image to {os.path.abspath(save_path)}")
+
+
+if __name__ == "__main__":
+    main()

examples/offline_inference/image_to_image/image_to_image.md

@@ -0,0 +1,30 @@
+# Qwen-Image-Edit Offline Inference
+
+Source <https://github.com/vllm-project/vllm-omni/tree/main/examples/offline_inference/image_to_image>. Added in PR [#196](https://github.com/vllm-project/vllm-omni/pull/196).
+
+This example edits an input image with `Qwen/Qwen-Image-Edit` using the `image_edit.py` CLI.
+
+## Local CLI Usage
+
+```bash
+python image_edit.py \
+  --image qwen_bear.png \
+  --prompt "Let this mascot dance under the moon, surrounded by floating stars and poetic bubbles such as 'Be Kind'" \
+  --output output_image_edit.png \
+  --num_inference_steps 50 \
+  --cfg_scale 4.0
+```
+
+Key arguments:
+- `--image`: path to the source image (PNG/JPG, converted to RGB).
+- `--prompt` / `--negative_prompt`: text description (string).
+- `--cfg_scale`: true CFG scale for Qwen-Image-Edit (quality vs. fidelity).
+- `--num_inference_steps`: diffusion sampling steps (more steps = higher quality, slower).
+- `--output`: path to save the generated PNG.
+
+## Example materials
+
+??? abstract "image_edit.py"
+    ``````py
+    --8<-- "examples/offline_inference/image_to_image/image_edit.py"
+    ``````

examples/offline_inference/qwen2_5_omni/README.md

@@ -1,10 +1,11 @@
-# Offline Example of vLLM-Omni for Qwen2.5-omni
+# Offline Example of vLLM-Omni for Qwen2.5-Omni
 
 ## 🛠️ Installation
 
 Please refer to [README.md](../../../README.md)
 
-## Run examples (Qwen2.5-omni)
+## Run examples (Qwen2.5-Omni)
+
 ### Multiple Prompts
 Download dataset from [seed_tts](https://drive.google.com/file/d/1GlSjVfSHkW3-leKKBlfrjuuTGqQ_xaLP/edit). To get the prompt, you can:
 ```bash
@@ -23,7 +24,8 @@ Then run the command below.
 ```bash
 bash run_multiple_prompts.sh
 ```
-### Single Prompts
+
+### Single Prompt
 Get into the example folder
 ```bash
 cd examples/offline_inference/qwen2_5_omni
@@ -33,6 +35,34 @@ Then run the command below.
 bash run_single_prompt.sh
 ```
 
+#### Using Local Media Files
+The `end2end.py` script supports local media files (audio, video, image) via CLI arguments:
+
+```bash
+# Use single local media files
+python end2end.py --query-type use_image --image-path /path/to/image.jpg
+python end2end.py --query-type use_video --video-path /path/to/video.mp4
+python end2end.py --query-type use_audio --audio-path /path/to/audio.wav
+
+# Combine multiple local media files
+python end2end.py --query-type mixed_modalities \
+    --video-path /path/to/video.mp4 \
+    --image-path /path/to/image.jpg \
+    --audio-path /path/to/audio.wav
+
+# Use audio from video file
+python end2end.py --query-type use_audio_in_video --video-path /path/to/video.mp4
+
+```
+
+If media file paths are not provided, the script will use default assets. Supported query types:
+- `use_image`: Image input only
+- `use_video`: Video input only
+- `use_audio`: Audio input only
+- `mixed_modalities`: Audio + image + video
+- `use_audio_in_video`: Extract audio from video
+- `text`: Text-only query
+
 ### FAQ
 
 If you encounter error about backend of librosa, try to install ffmpeg with command below.

examples/offline_inference/qwen2_5_omni/end2end.py

@@ -6,17 +6,20 @@ with the correct prompt format on Qwen2.5-Omni
 """
 
 import os
-from typing import NamedTuple
+from typing import NamedTuple, Optional
 
+import librosa
+import numpy as np
 import soundfile as sf
+from PIL import Image
 from vllm.assets.audio import AudioAsset
 from vllm.assets.image import ImageAsset
-from vllm.assets.video import VideoAsset
+from vllm.assets.video import VideoAsset, video_to_ndarrays
 from vllm.multimodal.image import convert_image_mode
 from vllm.sampling_params import SamplingParams
 from vllm.utils import FlexibleArgumentParser
 
-from vllm_omni.entrypoints.omni_llm import OmniLLM
+from vllm_omni.entrypoints.omni import Omni
 
 SEED = 42
 
@@ -54,7 +57,13 @@ def get_text_query(question: str = None) -> QueryResult:
     )
 
 
-def get_mixed_modalities_query() -> QueryResult:
+def get_mixed_modalities_query(
+    video_path: Optional[str] = None,
+    image_path: Optional[str] = None,
+    audio_path: Optional[str] = None,
+    num_frames: int = 16,
+    sampling_rate: int = 16000,
+) -> QueryResult:
     question = "What is recited in the audio? What is the content of this image? Why is this video funny?"
     prompt = (
         f"<|im_start|>system\n{default_system}<|im_end|>\n"
@@ -64,20 +73,49 @@ def get_mixed_modalities_query() -> QueryResult:
         f"{question}<|im_end|>\n"
         f"<|im_start|>assistant\n"
     )
+
+    # Load video
+    if video_path:
+        if not os.path.exists(video_path):
+            raise FileNotFoundError(f"Video file not found: {video_path}")
+        video_frames = video_to_ndarrays(video_path, num_frames=num_frames)
+    else:
+        video_frames = VideoAsset(name="baby_reading", num_frames=num_frames).np_ndarrays
+
+    # Load image
+    if image_path:
+        if not os.path.exists(image_path):
+            raise FileNotFoundError(f"Image file not found: {image_path}")
+        pil_image = Image.open(image_path)
+        image_data = convert_image_mode(pil_image, "RGB")
+    else:
+        image_data = convert_image_mode(ImageAsset("cherry_blossom").pil_image, "RGB")
+
+    # Load audio
+    if audio_path:
+        if not os.path.exists(audio_path):
+            raise FileNotFoundError(f"Audio file not found: {audio_path}")
+        audio_signal, sr = librosa.load(audio_path, sr=sampling_rate)
+        audio_data = (audio_signal.astype(np.float32), sr)
+    else:
+        audio_data = AudioAsset("mary_had_lamb").audio_and_sample_rate
+
     return QueryResult(
         inputs={
             "prompt": prompt,
             "multi_modal_data": {
-                "audio": AudioAsset("mary_had_lamb").audio_and_sample_rate,
-                "image": convert_image_mode(ImageAsset("cherry_blossom").pil_image, "RGB"),
-                "video": VideoAsset(name="baby_reading", num_frames=16).np_ndarrays,
+                "audio": audio_data,
+                "image": image_data,
+                "video": video_frames,
             },
         },
         limit_mm_per_prompt={"audio": 1, "image": 1, "video": 1},
     )
 
 
-def get_use_audio_in_video_query() -> QueryResult:
+def get_use_audio_in_video_query(
+    video_path: Optional[str] = None, num_frames: int = 16, sampling_rate: int = 16000
+) -> QueryResult:
     question = "Describe the content of the video, then convert what the baby say into text."
     prompt = (
         f"<|im_start|>system\n{default_system}<|im_end|>\n"
@@ -85,14 +123,24 @@ def get_use_audio_in_video_query() -> QueryResult:
         f"{question}<|im_end|>\n"
         f"<|im_start|>assistant\n"
     )
-    asset = VideoAsset(name="baby_reading", num_frames=16)
-    audio = asset.get_audio(sampling_rate=16000)
+
+    if video_path:
+        if not os.path.exists(video_path):
+            raise FileNotFoundError(f"Video file not found: {video_path}")
+        video_frames = video_to_ndarrays(video_path, num_frames=num_frames)
+        # Extract audio from video file
+        audio_signal, sr = librosa.load(video_path, sr=sampling_rate)
+        audio = (audio_signal.astype(np.float32), sr)
+    else:
+        asset = VideoAsset(name="baby_reading", num_frames=num_frames)
+        video_frames = asset.np_ndarrays
+        audio = asset.get_audio(sampling_rate=sampling_rate)
 
     return QueryResult(
         inputs={
             "prompt": prompt,
             "multi_modal_data": {
-                "video": asset.np_ndarrays,
+                "video": video_frames,
                 "audio": audio,
             },
             "mm_processor_kwargs": {
@@ -103,7 +151,7 @@ def get_use_audio_in_video_query() -> QueryResult:
     )
 
 
-def get_multi_audios_query() -> QueryResult:
+def get_multi_audios_query(audio_path: Optional[str] = None, sampling_rate: int = 16000) -> QueryResult:
     question = "Are these two audio clips the same?"
     prompt = (
         f"<|im_start|>system\n{default_system}<|im_end|>\n"
@@ -112,14 +160,28 @@ def get_multi_audios_query() -> QueryResult:
         f"{question}<|im_end|>\n"
         f"<|im_start|>assistant\n"
     )
+
+    if audio_path:
+        if not os.path.exists(audio_path):
+            raise FileNotFoundError(f"Audio file not found: {audio_path}")
+        audio_signal, sr = librosa.load(audio_path, sr=sampling_rate)
+        audio_data = (audio_signal.astype(np.float32), sr)
+        # Use the provided audio as the first audio, default as second
+        audio_list = [
+            audio_data,
+            AudioAsset("mary_had_lamb").audio_and_sample_rate,
+        ]
+    else:
+        audio_list = [
+            AudioAsset("winning_call").audio_and_sample_rate,
+            AudioAsset("mary_had_lamb").audio_and_sample_rate,
+        ]
+
     return QueryResult(
         inputs={
             "prompt": prompt,
             "multi_modal_data": {
-                "audio": [
-                    AudioAsset("winning_call").audio_and_sample_rate,
-                    AudioAsset("mary_had_lamb").audio_and_sample_rate,
-                ],
+                "audio": audio_list,
             },
         },
         limit_mm_per_prompt={
@@ -128,19 +190,137 @@ def get_multi_audios_query() -> QueryResult:
     )
 
 
+def get_image_query(question: str = None, image_path: Optional[str] = None) -> QueryResult:
+    if question is None:
+        question = "What is the content of this image?"
+    prompt = (
+        f"<|im_start|>system\n{default_system}<|im_end|>\n"
+        "<|im_start|>user\n<|vision_bos|><|IMAGE|><|vision_eos|>"
+        f"{question}<|im_end|>\n"
+        f"<|im_start|>assistant\n"
+    )
+
+    if image_path:
+        if not os.path.exists(image_path):
+            raise FileNotFoundError(f"Image file not found: {image_path}")
+        pil_image = Image.open(image_path)
+        image_data = convert_image_mode(pil_image, "RGB")
+    else:
+        image_data = convert_image_mode(ImageAsset("cherry_blossom").pil_image, "RGB")
+
+    return QueryResult(
+        inputs={
+            "prompt": prompt,
+            "multi_modal_data": {
+                "image": image_data,
+            },
+        },
+        limit_mm_per_prompt={"image": 1},
+    )
+
+
+def get_video_query(question: str = None, video_path: Optional[str] = None, num_frames: int = 16) -> QueryResult:
+    if question is None:
+        question = "Why is this video funny?"
+    prompt = (
+        f"<|im_start|>system\n{default_system}<|im_end|>\n"
+        "<|im_start|>user\n<|vision_bos|><|VIDEO|><|vision_eos|>"
+        f"{question}<|im_end|>\n"
+        f"<|im_start|>assistant\n"
+    )
+
+    if video_path:
+        if not os.path.exists(video_path):
+            raise FileNotFoundError(f"Video file not found: {video_path}")
+        video_frames = video_to_ndarrays(video_path, num_frames=num_frames)
+    else:
+        video_frames = VideoAsset(name="baby_reading", num_frames=num_frames).np_ndarrays
+
+    return QueryResult(
+        inputs={
+            "prompt": prompt,
+            "multi_modal_data": {
+                "video": video_frames,
+            },
+        },
+        limit_mm_per_prompt={"video": 1},
+    )
+
+
+def get_audio_query(question: str = None, audio_path: Optional[str] = None, sampling_rate: int = 16000) -> QueryResult:
+    if question is None:
+        question = "What is the content of this audio?"
+    prompt = (
+        f"<|im_start|>system\n{default_system}<|im_end|>\n"
+        "<|im_start|>user\n<|audio_bos|><|AUDIO|><|audio_eos|>"
+        f"{question}<|im_end|>\n"
+        f"<|im_start|>assistant\n"
+    )
+
+    if audio_path:
+        if not os.path.exists(audio_path):
+            raise FileNotFoundError(f"Audio file not found: {audio_path}")
+        audio_signal, sr = librosa.load(audio_path, sr=sampling_rate)
+        audio_data = (audio_signal.astype(np.float32), sr)
+    else:
+        audio_data = AudioAsset("mary_had_lamb").audio_and_sample_rate
+
+    return QueryResult(
+        inputs={
+            "prompt": prompt,
+            "multi_modal_data": {
+                "audio": audio_data,
+            },
+        },
+        limit_mm_per_prompt={"audio": 1},
+    )
+
+
 query_map = {
     "mixed_modalities": get_mixed_modalities_query,
     "use_audio_in_video": get_use_audio_in_video_query,
     "multi_audios": get_multi_audios_query,
+    "use_image": get_image_query,
+    "use_video": get_video_query,
+    "use_audio": get_audio_query,
     "text": get_text_query,
 }
 
 
 def main(args):
     model_name = "Qwen/Qwen2.5-Omni-7B"
-    query_result = query_map[args.query_type]()
 
-    omni_llm = OmniLLM(
+    # Get paths from args
+    video_path = getattr(args, "video_path", None)
+    image_path = getattr(args, "image_path", None)
+    audio_path = getattr(args, "audio_path", None)
+    num_frames = getattr(args, "num_frames", 16)
+    sampling_rate = getattr(args, "sampling_rate", 16000)
+
+    # Get the query function and call it with appropriate parameters
+    query_func = query_map[args.query_type]
+    if args.query_type == "mixed_modalities":
+        query_result = query_func(
+            video_path=video_path,
+            image_path=image_path,
+            audio_path=audio_path,
+            num_frames=num_frames,
+            sampling_rate=sampling_rate,
+        )
+    elif args.query_type == "use_audio_in_video":
+        query_result = query_func(video_path=video_path, num_frames=num_frames, sampling_rate=sampling_rate)
+    elif args.query_type == "multi_audios":
+        query_result = query_func(audio_path=audio_path, sampling_rate=sampling_rate)
+    elif args.query_type == "use_image":
+        query_result = query_func(image_path=image_path)
+    elif args.query_type == "use_video":
+        query_result = query_func(video_path=video_path, num_frames=num_frames)
+    elif args.query_type == "use_audio":
+        query_result = query_func(audio_path=audio_path, sampling_rate=sampling_rate)
+    else:
+        query_result = query_func()
+
+    omni_llm = Omni(
         model=model_name,
         log_stats=args.enable_stats,
         log_file=("omni_llm_pipeline.log" if args.enable_stats else None),
@@ -283,7 +463,48 @@ def parse_args():
         default=None,
         help="Path to a .txt file with one prompt per line (preferred).",
     )
-
+    parser.add_argument(
+        "--video-path",
+        "-v",
+        type=str,
+        default=None,
+        help="Path to local video file. If not provided, uses default video asset.",
+    )
+    parser.add_argument(
+        "--image-path",
+        "-i",
+        type=str,
+        default=None,
+        help="Path to local image file. If not provided, uses default image asset.",
+    )
+    parser.add_argument(
+        "--audio-path",
+        "-a",
+        type=str,
+        default=None,
+        help="Path to local audio file. If not provided, uses default audio asset.",
+    )
+    parser.add_argument(
+        "--num-frames",
+        type=int,
+        default=16,
+        help="Number of frames to extract from video (default: 16).",
+    )
+    parser.add_argument(
+        "--sampling-rate",
+        type=int,
+        default=16000,
+        help="Sampling rate for audio loading (default: 16000).",
+    )
+    parser.add_argument(
+        "--worker-backend", type=str, default="multi_process", choices=["multi_process", "ray"], help="backend"
+    )
+    parser.add_argument(
+        "--ray-address",
+        type=str,
+        default=None,
+        help="Address of the Ray cluster.",
+    )
     return parser.parse_args()
 
 

examples/offline_inference/qwen2_5_omni/extract_prompts.py

@@ -4,7 +4,7 @@ from typing import Optional
 
 
 def extract_prompt(line: str) -> Optional[str]:
-    # 提取第一个 '|' 与第二个 '|' 之间的内容
+    # Extract the content between the first '|' and the second '|'
     i = line.find("|")
     if i == -1:
         return None

examples/offline_inference/qwen2_5_omni/run_single_prompt.sh

@@ -1,2 +1,2 @@
 python end2end.py --output-wav output_audio \
-                  --query-type use_audio_in_video
+                  --query-type mixed_modalities

examples/offline_inference/qwen3_omni/README.md

@@ -1,12 +1,13 @@
-# Offline Example of vLLM-Omni for Qwen2.5-omni
+# Offline Example of vLLM-Omni for Qwen3-Omni
 
 ## 🛠️ Installation
 
 Please refer to [README.md](../../../README.md)
 
-## Run examples (Qwen3-omni)
+## Run examples (Qwen3-Omni)
+
 ### Multiple Prompts
-Download dataset from [seed_tts](https://drive.google.com/file/d/1GlSjVfSHkW3-leKKBlfrjuuTGqQ_xaLP/edit). For processing dataset please refer to [Qwen2.5-omni README.md](../qwen2_5_omni/README.md)
+Download dataset from [seed_tts](https://drive.google.com/file/d/1GlSjVfSHkW3-leKKBlfrjuuTGqQ_xaLP/edit). For processing dataset please refer to [Qwen2.5-Omni README.md](../qwen2_5_omni/README.md)
 Get into the example folder
 ```bash
 cd examples/offline_inference/qwen3_omni
@@ -15,7 +16,7 @@ Then run the command below.
 ```bash
 bash run_multiple_prompts.sh
 ```
-### Single Prompts
+### Single Prompt
 Get into the example folder
 ```bash
 cd examples/offline_inference/qwen3_omni
@@ -24,6 +25,30 @@ Then run the command below.
 ```bash
 bash run_single_prompt.sh
 ```
+If you have not enough memory, you can set thinker with tensor parallel. Just run the command below.
+```bash
+bash run_single_prompt_tp.sh
+```
+
+#### Using Local Media Files
+The `end2end.py` script supports local media files (audio, video, image) via command-line arguments:
+
+```bash
+# Use local video file
+python end2end.py --query-type use_video --video-path /path/to/video.mp4
+
+# Use local image file
+python end2end.py --query-type use_image --image-path /path/to/image.jpg
+
+# Use local audio file
+python end2end.py --query-type use_audio --audio-path /path/to/audio.wav
+```
+
+If media file paths are not provided, the script will use default assets. Supported query types:
+- `use_video`: Video input
+- `use_image`: Image input
+- `use_audio`: Audio input
+- `text`: Text-only query
 
 ### FAQ
 

examples/offline_inference/qwen3_omni/end2end.py

@@ -6,17 +6,20 @@ with the correct prompt format on Qwen3-Omni (thinker only).
 """
 
 import os
-from typing import NamedTuple
+from typing import NamedTuple, Optional
 
+import librosa
+import numpy as np
 import soundfile as sf
+from PIL import Image
 from vllm import SamplingParams
 from vllm.assets.audio import AudioAsset
 from vllm.assets.image import ImageAsset
-from vllm.assets.video import VideoAsset
+from vllm.assets.video import VideoAsset, video_to_ndarrays
 from vllm.multimodal.image import convert_image_mode
 from vllm.utils import FlexibleArgumentParser
 
-from vllm_omni.entrypoints.omni_llm import OmniLLM
+from vllm_omni.entrypoints.omni import Omni
 
 SEED = 42
 
@@ -54,7 +57,7 @@ def get_text_query(question: str = None) -> QueryResult:
     )
 
 
-def get_video_query(question: str = None) -> QueryResult:
+def get_video_query(question: str = None, video_path: Optional[str] = None, num_frames: int = 16) -> QueryResult:
     if question is None:
         question = "Why is this video funny?"
     prompt = (
@@ -64,18 +67,25 @@ def get_video_query(question: str = None) -> QueryResult:
         f"<|im_start|>assistant\n"
     )
 
+    if video_path:
+        if not os.path.exists(video_path):
+            raise FileNotFoundError(f"Video file not found: {video_path}")
+        video_frames = video_to_ndarrays(video_path, num_frames=num_frames)
+    else:
+        video_frames = VideoAsset(name="baby_reading", num_frames=num_frames).np_ndarrays
+
     return QueryResult(
         inputs={
             "prompt": prompt,
             "multi_modal_data": {
-                "video": VideoAsset(name="baby_reading", num_frames=16).np_ndarrays,
+                "video": video_frames,
             },
         },
         limit_mm_per_prompt={"video": 1},
     )
 
 
-def get_image_query(question: str = None) -> QueryResult:
+def get_image_query(question: str = None, image_path: Optional[str] = None) -> QueryResult:
     if question is None:
         question = "What is the content of this image?"
     prompt = (
@@ -84,18 +94,27 @@ def get_image_query(question: str = None) -> QueryResult:
         f"{question}<|im_end|>\n"
         f"<|im_start|>assistant\n"
     )
+
+    if image_path:
+        if not os.path.exists(image_path):
+            raise FileNotFoundError(f"Image file not found: {image_path}")
+        pil_image = Image.open(image_path)
+        image_data = convert_image_mode(pil_image, "RGB")
+    else:
+        image_data = convert_image_mode(ImageAsset("cherry_blossom").pil_image, "RGB")
+
     return QueryResult(
         inputs={
             "prompt": prompt,
             "multi_modal_data": {
-                "image": convert_image_mode(ImageAsset("cherry_blossom").pil_image, "RGB"),
+                "image": image_data,
             },
         },
         limit_mm_per_prompt={"image": 1},
     )
 
 
-def get_audio_query(question: str = None) -> QueryResult:
+def get_audio_query(question: str = None, audio_path: Optional[str] = None, sampling_rate: int = 16000) -> QueryResult:
     if question is None:
         question = "What is the content of this audio?"
     prompt = (
@@ -104,11 +123,20 @@ def get_audio_query(question: str = None) -> QueryResult:
         f"{question}<|im_end|>\n"
         f"<|im_start|>assistant\n"
     )
+
+    if audio_path:
+        if not os.path.exists(audio_path):
+            raise FileNotFoundError(f"Audio file not found: {audio_path}")
+        audio_signal, sr = librosa.load(audio_path, sr=sampling_rate)
+        audio_data = (audio_signal.astype(np.float32), sr)
+    else:
+        audio_data = AudioAsset("mary_had_lamb").audio_and_sample_rate
+
     return QueryResult(
         inputs={
             "prompt": prompt,
             "multi_modal_data": {
-                "audio": AudioAsset("mary_had_lamb").audio_and_sample_rate,
+                "audio": audio_data,
             },
         },
         limit_mm_per_prompt={"audio": 1},
@@ -125,10 +153,26 @@ query_map = {
 
 def main(args):
     model_name = "Qwen/Qwen3-Omni-30B-A3B-Instruct"
-    query_result = query_map[args.query_type]()
 
-    omni_llm = OmniLLM(
+    # Get paths from args
+    video_path = getattr(args, "video_path", None)
+    image_path = getattr(args, "image_path", None)
+    audio_path = getattr(args, "audio_path", None)
+
+    # Get the query function and call it with appropriate parameters
+    query_func = query_map[args.query_type]
+    if args.query_type == "use_video":
+        query_result = query_func(video_path=video_path, num_frames=getattr(args, "num_frames", 16))
+    elif args.query_type == "use_image":
+        query_result = query_func(image_path=image_path)
+    elif args.query_type == "use_audio":
+        query_result = query_func(audio_path=audio_path, sampling_rate=getattr(args, "sampling_rate", 16000))
+    else:
+        query_result = query_func()
+
+    omni_llm = Omni(
         model=model_name,
+        stage_configs_path=args.stage_configs_path,
     )
 
     thinker_sampling_params = SamplingParams(
@@ -276,6 +320,45 @@ def parse_args():
         default=None,
         help="Path to a .txt file with one prompt per line (preferred).",
     )
+    parser.add_argument(
+        "--stage-configs-path",
+        type=str,
+        default=None,
+        help="Path to a stage configs file.",
+    )
+    parser.add_argument(
+        "--video-path",
+        "-v",
+        type=str,
+        default=None,
+        help="Path to local video file. If not provided, uses default video asset.",
+    )
+    parser.add_argument(
+        "--image-path",
+        "-i",
+        type=str,
+        default=None,
+        help="Path to local image file. If not provided, uses default image asset.",
+    )
+    parser.add_argument(
+        "--audio-path",
+        "-a",
+        type=str,
+        default=None,
+        help="Path to local audio file. If not provided, uses default audio asset.",
+    )
+    parser.add_argument(
+        "--num-frames",
+        type=int,
+        default=16,
+        help="Number of frames to extract from video (default: 16).",
+    )
+    parser.add_argument(
+        "--sampling-rate",
+        type=int,
+        default=16000,
+        help="Sampling rate for audio loading (default: 16000).",
+    )
 
     return parser.parse_args()
 
@@ -283,7 +366,3 @@ def parse_args():
 if __name__ == "__main__":
     args = parse_args()
     main(args)
-
-    # use examples:
-    # python end2end.py --model Qwen/Qwen3-Omni-30B-A3B-Instruct --query-type text
-    # python end2end.py --model /custom_path/Qwen3-Omni-30B-A3B-Instruct --query-type use_video_only

examples/offline_inference/qwen3_omni/run_single_prompt_tp.sh

@@ -0,0 +1,5 @@
+python end2end.py --output-wav output_audio \
+                  --query-type use_audio \
+                  --init-sleep-seconds 90
+
+# init-sleep-seconds works to avoid two vLLM stages initialized at the same time within a card.

examples/offline_inference/text_to_image/README.md

@@ -1,10 +1,21 @@
 # Qwen-Image Offline Inference
 
-This folder provides two simple entrypoints for experimenting with `Qwen/Qwen-Image` using vLLM-Omni:
+This folder provides several entrypoints for experimenting with `Qwen/Qwen-Image` using vLLM-Omni:
 
-- `text_to_image.py`: command-line script for single image generation.
+- `text_to_image.py`: command-line script for single image generation with advanced options.
 - `web_demo.py`: lightweight Gradio UI for interactive prompt/seed/CFG exploration.
 
+## Basic Usage
+
+```python
+from vllm_omni.entrypoints.omni import Omni
+
+if __name__ == "__main__":
+    omni = Omni(model="Qwen/Qwen-Image")
+    prompt = "a cup of coffee on the table"
+    images = omni.generate(prompt)
+    images[0].save("coffee.png")
+```
 
 ## Local CLI Usage
 

examples/offline_inference/text_to_image/gradio_demo.py

@@ -5,6 +5,7 @@ import gradio as gr
 import torch
 
 from vllm_omni.entrypoints.omni import Omni
+from vllm_omni.utils.platform_utils import detect_device_type, is_npu
 
 ASPECT_RATIOS: dict[str, tuple[int, int]] = {
     "1:1": (1328, 1328),
@@ -58,11 +59,18 @@ def parse_args() -> argparse.Namespace:
 
 @lru_cache(maxsize=1)
 def get_omni(model_name: str) -> Omni:
-    return Omni(model=model_name)
+    # Enable VAE memory optimizations on NPU
+    vae_use_slicing = is_npu()
+    vae_use_tiling = is_npu()
+    return Omni(
+        model=model_name,
+        vae_use_slicing=vae_use_slicing,
+        vae_use_tiling=vae_use_tiling,
+    )
 
 
 def build_demo(args: argparse.Namespace) -> gr.Blocks:
-    device = "cuda"
+    device = detect_device_type()
     omni = get_omni(args.model)
 
     def run_inference(
@@ -97,7 +105,6 @@ def build_demo(args: argparse.Namespace) -> gr.Blocks:
             generator=generator,
             true_cfg_scale=float(cfg_scale_value),
             num_inference_steps=num_steps,
-            num_images_per_prompt=num_images,
             num_outputs_per_prompt=num_images,
         )
         return [img for img in images[:num_images]]

examples/offline_inference/text_to_image/text_to_image.py

@@ -7,6 +7,7 @@ from pathlib import Path
 import torch
 
 from vllm_omni.entrypoints.omni import Omni
+from vllm_omni.utils.platform_utils import detect_device_type, is_npu
 
 
 def parse_args() -> argparse.Namespace:
@@ -45,10 +46,18 @@ def parse_args() -> argparse.Namespace:
 
 def main():
     args = parse_args()
-    device = "cuda" if torch.cuda.is_available() else "cpu"
+    device = detect_device_type()
     generator = torch.Generator(device=device).manual_seed(args.seed)
 
-    omni = Omni(model=args.model)
+    # Enable VAE memory optimizations on NPU
+    vae_use_slicing = is_npu()
+    vae_use_tiling = is_npu()
+
+    omni = Omni(
+        model=args.model,
+        vae_use_slicing=vae_use_slicing,
+        vae_use_tiling=vae_use_tiling,
+    )
     images = omni.generate(
         args.prompt,
         height=args.height,
@@ -56,7 +65,6 @@ def main():
         generator=generator,
         true_cfg_scale=args.cfg_scale,
         num_inference_steps=args.num_inference_steps,
-        num_images_per_prompt=args.num_images_per_prompt,
         num_outputs_per_prompt=args.num_images_per_prompt,
     )
 

examples/offline_inference/text_to_video/text_to_video.md

@@ -0,0 +1,38 @@
+# Wan2.2 Text-to-Video Offline Inference
+
+Source <https://github.com/vllm-project/vllm-omni/tree/main/examples/offline_inference/text_to_video>. Added in PR [#202](https://github.com/vllm-project/vllm-omni/pull/202).
+
+The `Wan-AI/Wan2.2-T2V-A14B-Diffusers` pipeline generates short videos from text prompts.
+
+## Local CLI Usage
+
+```bash
+python text_to_video.py \
+  --prompt "Two anthropomorphic cats in comfy boxing gear and bright gloves fight intensely on a spotlighted stage." \
+  --negative_prompt "<optional quality filter>" \
+  --height 480 \
+  --width 640 \
+  --num_frames 32 \
+  --guidance_scale 4.0 \
+  --guidance_scale_high 3.0 \
+  --num_inference_steps 40 \
+  --fps 16 \
+  --output t2v_out.mp4
+```
+
+Key arguments:
+- `--prompt`: text description (string).
+- `--height/--width`: output resolution (defaults 720x1280). Dimensions should align with Wan VAE downsampling (multiples of 8).
+- `--num_frames`: Number of frames (Wan default is 81).
+- `--guidance_scale` and `--guidance_scale_high`: CFG scale (applied to low/high)..
+- `--negative_prompt`: optional list of artifacts to suppress (the PR demo used a long Chinese string).
+- `--boundary_ratio`: Boundary split ratio for low/high DiT.
+- `--fps`: frames per second for the saved MP4 (requires `diffusers` export_to_video).
+- `--output`: path to save the generated video.
+
+## Example materials
+
+??? abstract "text_to_video.py"
+    ``````py
+    --8<-- "examples/offline_inference/wan22/text_to_video.py"
+    ``````

examples/offline_inference/text_to_video/text_to_video.py

@@ -0,0 +1,105 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+import argparse
+from pathlib import Path
+
+import numpy as np
+import torch
+
+from vllm_omni.entrypoints.omni import Omni
+from vllm_omni.utils.platform_utils import detect_device_type, is_npu
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description="Generate a video with Wan2.2 T2V.")
+    parser.add_argument(
+        "--model",
+        default="Wan-AI/Wan2.2-T2V-A14B-Diffusers",
+        help="Diffusers Wan2.2 model ID or local path.",
+    )
+    parser.add_argument("--prompt", default="A serene lakeside sunrise with mist over the water.", help="Text prompt.")
+    parser.add_argument("--negative_prompt", default="", help="Negative prompt.")
+    parser.add_argument("--seed", type=int, default=42, help="Random seed.")
+    parser.add_argument("--guidance_scale", type=float, default=4.0, help="CFG scale (applied to low/high).")
+    parser.add_argument("--guidance_scale_high", type=float, default=None, help="Optional separate CFG for high-noise.")
+    parser.add_argument("--height", type=int, default=720, help="Video height.")
+    parser.add_argument("--width", type=int, default=1280, help="Video width.")
+    parser.add_argument("--num_frames", type=int, default=81, help="Number of frames (Wan default is 81).")
+    parser.add_argument("--num_inference_steps", type=int, default=40, help="Sampling steps.")
+    parser.add_argument("--boundary_ratio", type=float, default=0.875, help="Boundary split ratio for low/high DiT.")
+    parser.add_argument(
+        "--flow_shift", type=float, default=5.0, help="Scheduler flow_shift (5.0 for 720p, 12.0 for 480p)."
+    )
+    parser.add_argument("--output", type=str, default="wan22_output.mp4", help="Path to save the video (mp4).")
+    parser.add_argument("--fps", type=int, default=24, help="Frames per second for the output video.")
+    return parser.parse_args()
+
+
+def main():
+    args = parse_args()
+    device = detect_device_type()
+    generator = torch.Generator(device=device).manual_seed(args.seed)
+
+    # Enable VAE memory optimizations on NPU
+    vae_use_slicing = is_npu()
+    vae_use_tiling = is_npu()
+
+    omni = Omni(
+        model=args.model,
+        vae_use_slicing=vae_use_slicing,
+        vae_use_tiling=vae_use_tiling,
+        boundary_ratio=args.boundary_ratio,
+        flow_shift=args.flow_shift,
+    )
+
+    frames = omni.generate(
+        args.prompt,
+        negative_prompt=args.negative_prompt,
+        height=args.height,
+        width=args.width,
+        generator=generator,
+        guidance_scale=args.guidance_scale,
+        guidance_scale_2=args.guidance_scale_high,
+        num_inference_steps=args.num_inference_steps,
+        num_frames=args.num_frames,
+    )
+
+    output_path = Path(args.output)
+    output_path.parent.mkdir(parents=True, exist_ok=True)
+    try:
+        from diffusers.utils import export_to_video
+    except ImportError:
+        raise ImportError("diffusers is required for export_to_video.")
+
+    # frames may be np.ndarray (preferred) or torch.Tensor
+    # export_to_video expects a list of frames with values in [0, 1]
+    if isinstance(frames, torch.Tensor):
+        video_tensor = frames.detach().cpu()
+        if video_tensor.dim() == 5:
+            # [B, C, F, H, W] or [B, F, H, W, C]
+            if video_tensor.shape[1] in (3, 4):
+                video_tensor = video_tensor[0].permute(1, 2, 3, 0)
+            else:
+                video_tensor = video_tensor[0]
+        elif video_tensor.dim() == 4 and video_tensor.shape[0] in (3, 4):
+            video_tensor = video_tensor.permute(1, 2, 3, 0)
+        # If float, assume [-1,1] and normalize to [0,1]
+        if video_tensor.is_floating_point():
+            video_tensor = video_tensor.clamp(-1, 1) * 0.5 + 0.5
+        video_array = video_tensor.float().numpy()
+    else:
+        video_array = frames
+        if hasattr(video_array, "shape") and video_array.ndim == 5:
+            video_array = video_array[0]
+
+    # Convert 4D array (frames, H, W, C) to list of frames for export_to_video
+    if isinstance(video_array, np.ndarray) and video_array.ndim == 4:
+        video_array = list(video_array)
+
+    export_to_video(video_array, str(output_path), fps=args.fps)
+    print(f"Saved generated video to {output_path}")
+
+
+if __name__ == "__main__":
+    main()

examples/online_serving/qwen2_5_omni/README.md

@@ -1,12 +1,13 @@
-# Online serving Example of vLLM-Omni for Qwen2.5-omni
+# Online serving Example of vLLM-Omni for Qwen2.5-Omni
 
 ## 🛠️ Installation
 
-Please refer to [README.md](../../README.md)
+Please refer to [README.md](../../../README.md)
 
-## Run examples (Qwen2.5-omni)
+## Run examples (Qwen2.5-Omni)
+
+### Launch the Server
 
-Launch the server
 ```bash
 vllm serve Qwen/Qwen2.5-Omni-7B --omni --port 8091
 ```
@@ -16,36 +17,66 @@ If you have custom stage configs file, launch the server with command below
 vllm serve Qwen/Qwen2.5-Omni-7B --omni --port 8091 --stage-configs-path /path/to/stage_configs_file
 ```
 
+### Send Multi-modal Request
+
 Get into the example folder
 ```bash
-cd examples/online_serving
+cd examples/online_serving/qwen2_5_omni
 ```
 
-Send request via python
+####  Send request via python
+
 ```bash
 python openai_chat_completion_client_for_multimodal_generation.py --query-type mixed_modalities
 ```
 
-Send request via curl
+The Python client supports the following command-line arguments:
+
+- `--query-type` (or `-q`): Query type (default: `mixed_modalities`)
+  - Options: `mixed_modalities`, `use_audio_in_video`, `multi_audios`, `text`
+- `--video-path` (or `-v`): Path to local video file or URL
+  - If not provided and query-type uses video, uses default video URL
+  - Supports local file paths (automatically encoded to base64) or HTTP/HTTPS URLs
+  - Example: `--video-path /path/to/video.mp4` or `--video-path https://example.com/video.mp4`
+- `--image-path` (or `-i`): Path to local image file or URL
+  - If not provided and query-type uses image, uses default image URL
+  - Supports local file paths (automatically encoded to base64) or HTTP/HTTPS URLs
+  - Supports common image formats: JPEG, PNG, GIF, WebP
+  - Example: `--image-path /path/to/image.jpg` or `--image-path https://example.com/image.png`
+- `--audio-path` (or `-a`): Path to local audio file or URL
+  - If not provided and query-type uses audio, uses default audio URL
+  - Supports local file paths (automatically encoded to base64) or HTTP/HTTPS URLs
+  - Supports common audio formats: MP3, WAV, OGG, FLAC, M4A
+  - Example: `--audio-path /path/to/audio.wav` or `--audio-path https://example.com/audio.mp3`
+- `--prompt` (or `-p`): Custom text prompt/question
+  - If not provided, uses default prompt for the selected query type
+  - Example: `--prompt "What are the main activities shown in this video?"`
+
+
+For example, to use mixed modalities with all local files:
+
+```bash
+python openai_chat_completion_client_for_multimodal_generation.py \
+    --query-type mixed_modalities \
+    --video-path /path/to/your/video.mp4 \
+    --image-path /path/to/your/image.jpg \
+    --audio-path /path/to/your/audio.wav \
+    --prompt "Analyze all the media content and provide a comprehensive summary."
+```
+
+####  Send request via curl
+
 ```bash
 bash run_curl_multimodal_generation.sh mixed_modalities
 ```
 
-### FAQ
-
-If you encounter error about backend of librosa, try to install ffmpeg with command below.
-```
-sudo apt update
-sudo apt install ffmpeg
-```
-
 ## Run Local Web UI Demo
 
 This Web UI demo allows users to interact with the model through a web browser.
 
 ### Running Gradio Demo
 
-Once vllm and vllm-omni are installed, you can launch the web service built on AsyncOmniLLM by
+Once vllm and vllm-omni are installed, you can launch the web service built on AsyncOmni by
 
 ```bash
 python gradio_demo.py  --model Qwen/Qwen2.5-Omni-7B --port 7861
@@ -53,21 +84,18 @@ python gradio_demo.py  --model Qwen/Qwen2.5-Omni-7B --port 7861
 
 Then open `http://localhost:7861/` on your local browser to interact with the web UI.
 
+The gradio script supports the following arguments:
 
-### Options
+- `--model`: Model name
+- `--ip`: Host/IP for Gradio server (default: 127.0.0.1)
+- `--port`: Port for Gradio server (default: 7861)
+- `--stage-configs-path`: Path to custom stage configs YAML file (optional)
+- `--share`: Share the Gradio demo publicly (creates a public link)
 
-You can customize its basic launch parameters:
+### FAQ
 
-```bash
-python gradio_demo.py \
-    --model Qwen/Qwen2.5-Omni-7B \
-    --ip 127.0.0.1 \
-    --port 7861 \
-    --stage-configs-path /path/to/stage_configs.yaml
+If you encounter error about backend of librosa, try to install ffmpeg with command below.
+```
+sudo apt update
+sudo apt install ffmpeg
 ```
-
-- `--model`: Local model checkpoint to load (default `Qwen/Qwen2.5-Omni-7B`).
-- `--ip`: Host/IP for the Gradio server (default `127.0.0.1`).
-- `--port`: Port for the Gradio server (default `7861`).
-- `--stage-configs-path`: Optional path to custom stage configs YAML.
-- `--share`: Set to expose a temporary public link via Gradio.

examples/online_serving/qwen2_5_omni/gradio_demo.py

@@ -15,7 +15,7 @@ from PIL import Image
 from vllm.assets.video import video_get_metadata, video_to_ndarrays
 from vllm.sampling_params import SamplingParams
 
-from vllm_omni.entrypoints.async_omni_llm import AsyncOmniLLM
+from vllm_omni.entrypoints.async_omni import AsyncOmni
 
 # Import utils from offline inference example
 sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../offline_inference/qwen2_5_omni"))
@@ -91,9 +91,58 @@ def parse_args():
         default=None,
         help="Path to custom stage configs YAML file (optional).",
     )
+    parser.add_argument(
+        "--log-stats",
+        action="store_true",
+        help="Enable statistics logging for AsyncOmni.",
+    )
+    parser.add_argument(
+        "--log-file",
+        type=str,
+        default=None,
+        help="Path prefix for AsyncOmni log files.",
+    )
+    parser.add_argument(
+        "--init-sleep-seconds",
+        type=int,
+        default=30,
+        help="Seconds to sleep between starting stage processes.",
+    )
+    parser.add_argument(
+        "--shm-threshold-bytes",
+        type=int,
+        default=65536,
+        help="Threshold in bytes for using shared memory IPC.",
+    )
+    parser.add_argument(
+        "--batch-timeout",
+        type=int,
+        default=10,
+        help="Batching timeout (seconds) inside each stage.",
+    )
+    parser.add_argument(
+        "--init-timeout",
+        type=int,
+        default=ASYNC_INIT_TIMEOUT,
+        help="Timeout (seconds) for initializing all stages.",
+    )
     return parser.parse_args()
 
 
+def build_async_omni_cli_args(base_args: argparse.Namespace) -> argparse.Namespace:
+    """Construct the minimal CLI args Namespace expected by AsyncOmni."""
+    return argparse.Namespace(
+        model=base_args.model,
+        stage_configs_path=getattr(base_args, "stage_configs_path", None),
+        log_stats=bool(getattr(base_args, "log_stats", False)),
+        log_file=getattr(base_args, "log_file", None),
+        init_sleep_seconds=int(getattr(base_args, "init_sleep_seconds", 30)),
+        shm_threshold_bytes=int(getattr(base_args, "shm_threshold_bytes", 65536)),
+        batch_timeout=int(getattr(base_args, "batch_timeout", 10)),
+        init_timeout=int(getattr(base_args, "init_timeout", ASYNC_INIT_TIMEOUT)),
+    )
+
+
 def build_sampling_params(seed: int, model_key: str) -> list[SamplingParams]:
     """Build SamplingParams objects by reusing the dict definitions."""
     return [SamplingParams(**params_dict) for params_dict in build_sampling_params_dict(seed, model_key)]
@@ -236,8 +285,8 @@ def process_video_file(
     return frames, metadata, audio_tuple
 
 
-async def run_inference_async_omni_llm(
-    omni_llm: AsyncOmniLLM,
+async def run_inference_async_omni(
+    omni: AsyncOmni,
     sampling_params: list[SamplingParams],
     prompt_args_template: SimpleNamespace,
     user_prompt: str,
@@ -246,7 +295,7 @@ async def run_inference_async_omni_llm(
     video_file: Optional[str] = None,
     use_audio_in_video: bool = False,
 ):
-    """Run inference using AsyncOmniLLM directly with multimodal support."""
+    """Run inference using AsyncOmni directly with multimodal support."""
     if not user_prompt.strip() and not audio_file and not image_file and not video_file:
         return "Please provide at least a text prompt or multimodal input.", None
 
@@ -328,7 +377,7 @@ async def run_inference_async_omni_llm(
         text_outputs: list[str] = []
         audio_output = None
 
-        async for stage_outputs in omni_llm.generate(
+        async for stage_outputs in omni.generate(
             prompt=omni_prompt,
             request_id=request_id,
             sampling_params_list=sampling_params,
@@ -362,12 +411,12 @@ async def run_inference_async_omni_llm(
 
 
 def build_interface(
-    omni_llm: AsyncOmniLLM,
+    omni: AsyncOmni,
     sampling_params: list[SamplingParams],
     prompt_args_template: SimpleNamespace,
     model: str,
 ):
-    """Build Gradio interface for AsyncOmniLLM mode."""
+    """Build Gradio interface for AsyncOmni mode."""
 
     async def run_inference(
         user_prompt: str,
@@ -376,8 +425,8 @@ def build_interface(
         video_file: Optional[str],
         use_audio_in_video: bool,
     ):
-        return await run_inference_async_omni_llm(
-            omni_llm,
+        return await run_inference_async_omni(
+            omni,
             sampling_params,
             prompt_args_template,
             user_prompt,
@@ -475,7 +524,7 @@ def build_interface(
 
 def main():
     args = parse_args()
-    omni_llm = None
+    omni = None
 
     model_name = "/".join(args.model.split("/")[-2:])
     assert model_name in SUPPORTED_MODELS, (
@@ -485,9 +534,9 @@ def main():
     # Register signal handlers for graceful shutdown
     def signal_handler(sig, frame):
         print("\nReceived interrupt signal, shutting down...")
-        if omni_llm is not None:
+        if omni is not None:
             try:
-                omni_llm.shutdown()
+                omni.shutdown()
             except Exception as e:
                 print(f"Error during shutdown: {e}")
         sys.exit(0)
@@ -495,21 +544,18 @@ def main():
     signal.signal(signal.SIGINT, signal_handler)
     signal.signal(signal.SIGTERM, signal_handler)
 
-    print(f"Initializing AsyncOmniLLM with model: {args.model}")
+    print(f"Initializing AsyncOmni with model: {args.model}")
     if args.stage_configs_path:
         print(f"Using custom stage configs: {args.stage_configs_path}")
 
     sampling_params = build_sampling_params(SEED, model_name)
-    omni_llm = AsyncOmniLLM(
-        model=args.model,
-        stage_configs_path=args.stage_configs_path,
-        init_timeout=ASYNC_INIT_TIMEOUT,
-    )
-    print("✓ AsyncOmniLLM initialized successfully")
+    cli_args = build_async_omni_cli_args(args)
+    omni = AsyncOmni(model=args.model, cli_args=cli_args)
+    print("✓ AsyncOmni initialized successfully")
     prompt_args_template = create_prompt_args(args)
 
     demo = build_interface(
-        omni_llm,
+        omni,
         sampling_params,
         prompt_args_template,
         args.model,
@@ -524,9 +570,9 @@ def main():
         print("\nShutting down...")
     finally:
         # Cleanup
-        if omni_llm is not None:
+        if omni is not None:
             try:
-                omni_llm.shutdown()
+                omni.shutdown()
             except Exception as e:
                 print(f"Error during cleanup: {e}")
 

examples/online_serving/qwen2_5_omni/openai_chat_completion_client_for_multimodal_generation.py

@@ -1,4 +1,6 @@
 import base64
+import os
+from typing import Optional
 
 import requests
 from openai import OpenAI
@@ -28,6 +30,129 @@ def encode_base64_content_from_url(content_url: str) -> str:
     return result
 
 
+def encode_base64_content_from_file(file_path: str) -> str:
+    """Encode a local file to base64 format."""
+    with open(file_path, "rb") as f:
+        content = f.read()
+        result = base64.b64encode(content).decode("utf-8")
+    return result
+
+
+def get_video_url_from_path(video_path: Optional[str]) -> str:
+    """Convert a video path (local file or URL) to a video URL format for the API.
+
+    If video_path is None or empty, returns the default URL.
+    If video_path is a local file path, encodes it to base64 data URL.
+    If video_path is a URL, returns it as-is.
+    """
+    if not video_path:
+        # Default video URL
+        return "https://huggingface.co/datasets/raushan-testing-hf/videos-test/resolve/main/sample_demo_1.mp4"
+
+    # Check if it's a URL (starts with http:// or https://)
+    if video_path.startswith(("http://", "https://")):
+        return video_path
+
+    # Otherwise, treat it as a local file path
+    if not os.path.exists(video_path):
+        raise FileNotFoundError(f"Video file not found: {video_path}")
+
+    # Detect video MIME type from file extension
+    video_path_lower = video_path.lower()
+    if video_path_lower.endswith(".mp4"):
+        mime_type = "video/mp4"
+    elif video_path_lower.endswith(".webm"):
+        mime_type = "video/webm"
+    elif video_path_lower.endswith(".mov"):
+        mime_type = "video/quicktime"
+    elif video_path_lower.endswith(".avi"):
+        mime_type = "video/x-msvideo"
+    elif video_path_lower.endswith(".mkv"):
+        mime_type = "video/x-matroska"
+    else:
+        # Default to mp4 if extension is unknown
+        mime_type = "video/mp4"
+
+    video_base64 = encode_base64_content_from_file(video_path)
+    return f"data:{mime_type};base64,{video_base64}"
+
+
+def get_image_url_from_path(image_path: Optional[str]) -> str:
+    """Convert an image path (local file or URL) to an image URL format for the API.
+
+    If image_path is None or empty, returns the default URL.
+    If image_path is a local file path, encodes it to base64 data URL.
+    If image_path is a URL, returns it as-is.
+    """
+    if not image_path:
+        # Default image URL
+        return "https://vllm-public-assets.s3.us-west-2.amazonaws.com/vision_model_images/cherry_blossom.jpg"
+
+    # Check if it's a URL (starts with http:// or https://)
+    if image_path.startswith(("http://", "https://")):
+        return image_path
+
+    # Otherwise, treat it as a local file path
+    if not os.path.exists(image_path):
+        raise FileNotFoundError(f"Image file not found: {image_path}")
+
+    # Detect image MIME type from file extension
+    image_path_lower = image_path.lower()
+    if image_path_lower.endswith((".jpg", ".jpeg")):
+        mime_type = "image/jpeg"
+    elif image_path_lower.endswith(".png"):
+        mime_type = "image/png"
+    elif image_path_lower.endswith(".gif"):
+        mime_type = "image/gif"
+    elif image_path_lower.endswith(".webp"):
+        mime_type = "image/webp"
+    else:
+        # Default to jpeg if extension is unknown
+        mime_type = "image/jpeg"
+
+    image_base64 = encode_base64_content_from_file(image_path)
+    return f"data:{mime_type};base64,{image_base64}"
+
+
+def get_audio_url_from_path(audio_path: Optional[str]) -> str:
+    """Convert an audio path (local file or URL) to an audio URL format for the API.
+
+    If audio_path is None or empty, returns the default URL.
+    If audio_path is a local file path, encodes it to base64 data URL.
+    If audio_path is a URL, returns it as-is.
+    """
+    if not audio_path:
+        # Default audio URL
+        return AudioAsset("mary_had_lamb").url
+
+    # Check if it's a URL (starts with http:// or https://)
+    if audio_path.startswith(("http://", "https://")):
+        return audio_path
+
+    # Otherwise, treat it as a local file path
+    if not os.path.exists(audio_path):
+        raise FileNotFoundError(f"Audio file not found: {audio_path}")
+
+    # Detect audio MIME type from file extension
+    audio_path_lower = audio_path.lower()
+    if audio_path_lower.endswith((".mp3", ".mpeg")):
+        mime_type = "audio/mpeg"
+    elif audio_path_lower.endswith(".wav"):
+        mime_type = "audio/wav"
+    elif audio_path_lower.endswith(".ogg"):
+        mime_type = "audio/ogg"
+    elif audio_path_lower.endswith(".flac"):
+        mime_type = "audio/flac"
+    elif audio_path_lower.endswith(".m4a"):
+        mime_type = "audio/mp4"
+    else:
+        # Default to wav if extension is unknown
+        mime_type = "audio/wav"
+
+    audio_base64 = encode_base64_content_from_file(audio_path)
+    return f"data:{mime_type};base64,{audio_base64}"
+
+
 def get_system_prompt():
     return {
         "role": "system",
@@ -44,8 +169,10 @@ def get_system_prompt():
     }
 
 
-def get_text_query():
-    question = "Explain the system architecture for a scalable audio generation pipeline. Answer in 15 words."
+def get_text_query(custom_prompt: Optional[str] = None):
+    question = (
+        custom_prompt or "Explain the system architecture for a scalable audio generation pipeline. Answer in 15 words."
+    )
     prompt = {
         "role": "user",
         "content": [
@@ -58,26 +185,32 @@ def get_text_query():
     return prompt
 
 
-def get_mixed_modalities_query():
-    question = "What is recited in the audio? What is the content of this image? Why is this video funny?"
+def get_mixed_modalities_query(
+    video_path: Optional[str] = None,
+    image_path: Optional[str] = None,
+    audio_path: Optional[str] = None,
+    custom_prompt: Optional[str] = None,
+):
+    question = (
+        custom_prompt or "What is recited in the audio? What is the content of this image? Why is this video funny?"
+    )
+    video_url = get_video_url_from_path(video_path)
+    image_url = get_image_url_from_path(image_path)
+    audio_url = get_audio_url_from_path(audio_path)
     prompt = {
         "role": "user",
         "content": [
             {
                 "type": "audio_url",
-                "audio_url": {"url": AudioAsset("mary_had_lamb").url},
+                "audio_url": {"url": audio_url},
             },
             {
                 "type": "image_url",
-                "image_url": {
-                    "url": "https://vllm-public-assets.s3.us-west-2.amazonaws.com/vision_model_images/cherry_blossom.jpg"
-                },
+                "image_url": {"url": image_url},
             },
             {
                 "type": "video_url",
-                "video_url": {
-                    "url": "https://huggingface.co/datasets/raushan-testing-hf/videos-test/resolve/main/sample_demo_1.mp4"
-                },
+                "video_url": {"url": video_url},
             },
             {
                 "type": "text",
@@ -89,8 +222,9 @@ def get_mixed_modalities_query():
     return prompt
 
 
-def get_use_audio_in_video_query():
-    question = "Describe the content of the video, then convert what the baby say into text."
+def get_use_audio_in_video_query(video_path: Optional[str] = None, custom_prompt: Optional[str] = None):
+    question = custom_prompt or "Describe the content of the video, then convert what the baby say into text."
+    video_url = get_video_url_from_path(video_path)
 
     prompt = {
         "role": "user",
@@ -98,7 +232,7 @@ def get_use_audio_in_video_query():
             {
                 "type": "video_url",
                 "video_url": {
-                    "url": "https://huggingface.co/datasets/raushan-testing-hf/videos-test/resolve/main/sample_demo_1.mp4",
+                    "url": video_url,
                     "num_frames": 16,
                 },
             },
@@ -112,14 +246,15 @@ def get_use_audio_in_video_query():
     return prompt
 
 
-def get_multi_audios_query():
-    question = "Are these two audio clips the same?"
+def get_multi_audios_query(audio_path: Optional[str] = None, custom_prompt: Optional[str] = None):
+    question = custom_prompt or "Are these two audio clips the same?"
+    audio_url = get_audio_url_from_path(audio_path)
     prompt = {
         "role": "user",
         "content": [
             {
                 "type": "audio_url",
-                "audio_url": {"url": AudioAsset("mary_had_lamb").url},
+                "audio_url": {"url": audio_url},
             },
             {
                 "type": "audio_url",
@@ -179,7 +314,27 @@ def run_multimodal_generation(args) -> None:
         code2wav_sampling_params,
     ]
 
-    prompt = query_map[args.query_type]()
+    # Get paths and custom prompt from args
+    video_path = getattr(args, "video_path", None)
+    image_path = getattr(args, "image_path", None)
+    audio_path = getattr(args, "audio_path", None)
+    custom_prompt = getattr(args, "prompt", None)
+
+    # Get the query function and call it with appropriate parameters
+    query_func = query_map[args.query_type]
+    if args.query_type == "mixed_modalities":
+        prompt = query_func(
+            video_path=video_path, image_path=image_path, audio_path=audio_path, custom_prompt=custom_prompt
+        )
+    elif args.query_type == "use_audio_in_video":
+        prompt = query_func(video_path=video_path, custom_prompt=custom_prompt)
+    elif args.query_type == "multi_audios":
+        prompt = query_func(audio_path=audio_path, custom_prompt=custom_prompt)
+    elif args.query_type == "text":
+        prompt = query_func(custom_prompt=custom_prompt)
+    else:
+        prompt = query_func()
+
     extra_body = {
         "sampling_params_list": sampling_params_list  # Optional, it has a default setting in stage_configs of the corresponding model.
     }
@@ -219,6 +374,34 @@ def parse_args():
         choices=query_map.keys(),
         help="Query type.",
     )
+    parser.add_argument(
+        "--video-path",
+        "-v",
+        type=str,
+        default=None,
+        help="Path to local video file or URL. If not provided and query-type uses video, uses default video URL.",
+    )
+    parser.add_argument(
+        "--image-path",
+        "-i",
+        type=str,
+        default=None,
+        help="Path to local image file or URL. If not provided and query-type uses image, uses default image URL.",
+    )
+    parser.add_argument(
+        "--audio-path",
+        "-a",
+        type=str,
+        default=None,
+        help="Path to local audio file or URL. If not provided and query-type uses audio, uses default audio URL.",
+    )
+    parser.add_argument(
+        "--prompt",
+        "-p",
+        type=str,
+        default=None,
+        help="Custom text prompt/question to use instead of the default prompt for the selected query type.",
+    )
 
     return parser.parse_args()
 

examples/online_serving/qwen3_omni/README.md

@@ -1,12 +1,13 @@
-# Online serving Example of vLLM-omni for Qwen2.5-omni
+# Online serving Example of vLLM-Omni for Qwen3-omni
 
 ## 🛠️ Installation
 
-Please refer to [README.md](../../README.md)
+Please refer to [README.md](../../../README.md)
 
 ## Run examples (Qwen3-Omni)
 
-Launch the server
+### Launch the Server
+
 ```bash
 vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091
 ```
@@ -16,21 +17,59 @@ If you have custom stage configs file, launch the server with command below
 vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091 --stage-configs-path /path/to/stage_configs_file
 ```
 
+### Send Multi-modal Request
+
 Get into the example folder
 ```bash
-cd examples/online_serving
+cd examples/online_serving/qwen3_omni
 ```
 
-Send request via python
+####  Send request via python
+
 ```bash
-python openai_chat_completion_client_for_multimodal_generation.py --query-type mixed_modalities
+python openai_chat_completion_client_for_multimodal_generation.py --query-type use_image
 ```
 
-Send request via curl
+The Python client supports the following command-line arguments:
+
+- `--query-type` (or `-q`): Query type (default: `use_video`)
+  - Options: `text`, `use_audio`, `use_image`, `use_video`
+- `--model` (or `-m`): Model name/path (default: `Qwen/Qwen3-Omni-30B-A3B-Instruct`)
+- `--video-path` (or `-v`): Path to local video file or URL
+  - If not provided and query-type is `use_video`, uses default video URL
+  - Supports local file paths (automatically encoded to base64) or HTTP/HTTPS URLs
+  - Example: `--video-path /path/to/video.mp4` or `--video-path https://example.com/video.mp4`
+- `--image-path` (or `-i`): Path to local image file or URL
+  - If not provided and query-type is `use_image`, uses default image URL
+  - Supports local file paths (automatically encoded to base64) or HTTP/HTTPS URLs
+  - Supports common image formats: JPEG, PNG, GIF, WebP
+  - Example: `--image-path /path/to/image.jpg` or `--image-path https://example.com/image.png`
+- `--audio-path` (or `-a`): Path to local audio file or URL
+  - If not provided and query-type is `use_audio`, uses default audio URL
+  - Supports local file paths (automatically encoded to base64) or HTTP/HTTPS URLs
+  - Supports common audio formats: MP3, WAV, OGG, FLAC, M4A
+  - Example: `--audio-path /path/to/audio.wav` or `--audio-path https://example.com/audio.mp3`
+- `--prompt` (or `-p`): Custom text prompt/question
+  - If not provided, uses default prompt for the selected query type
+  - Example: `--prompt "What are the main activities shown in this video?"`
+
+
+For example, to use a local video file with custom prompt:
+
 ```bash
-bash run_curl_multimodal_generation.sh mixed_modalities
+python openai_chat_completion_client_for_multimodal_generation.py \
+    --query-type use_video \
+    --video-path /path/to/your/video.mp4 \
+    --prompt "What are the main activities shown in this video?"
 ```
 
+####  Send request via curl
+
+```bash
+bash run_curl_multimodal_generation.sh use_image
+```
+
+
 ### FAQ
 
 If you encounter error about backend of librosa, try to install ffmpeg with command below.
@@ -45,7 +84,7 @@ This Web UI demo allows users to interact with the model through a web browser.
 
 ### Running Gradio Demo
 
-Once vllm and vllm-omni are installed, you can launch the web service built on AsyncOmniLLM by
+Once vllm and vllm-omni are installed, you can launch the web service built on AsyncOmni by
 
 ```bash
 python gradio_demo.py  --model Qwen/Qwen3-Omni-30B-A3B-Instruct --port 7861
@@ -54,23 +93,9 @@ python gradio_demo.py  --model Qwen/Qwen3-Omni-30B-A3B-Instruct --port 7861
 Then open `http://localhost:7861/` on your local browser to interact with the web UI.
 
 
-### Options
-
-The gradio demo also supports running with an existing API server and can be customized with the following arguments.
-
-
-```bash
-python gradio_demo.py \
-    --model Qwen/Qwen3-Omni-30B-A3B-Instruct \
-    --use-api-server \
-    --api-base http://localhost:8091/v1 \
-    --ip 127.0.0.1 \
-    --port 7861
-```
+The gradio script supports the following arguments:
 
 - `--model`: Model name
-- `--use-api-server`: If set, connect to an existing vLLM HTTP API server instead of running AsyncOmniLLM locally.
-- `--api-base`: Base URL for vllm serve (only used when `use-api-server` is set, default: http://localhost:8091/v1)
 - `--ip`: Host/IP for Gradio server (default: 127.0.0.1)
 - `--port`: Port for Gradio server (default: 7861)
 - `--stage-configs-path`: Path to custom stage configs YAML file (optional)

examples/online_serving/qwen3_omni/gradio_demo.py

@@ -15,7 +15,7 @@ from PIL import Image
 from vllm.assets.video import video_get_metadata, video_to_ndarrays
 from vllm.sampling_params import SamplingParams
 
-from vllm_omni.entrypoints.async_omni_llm import AsyncOmniLLM
+from vllm_omni.entrypoints.async_omni import AsyncOmni
 
 # Import utils from offline inference example
 sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../offline_inference/qwen3_omni"))
@@ -42,19 +42,20 @@ SUPPORTED_MODELS: dict[str, dict[str, Any]] = {
                 "seed": SEED,
             },
             "talker": {
-                "temperature": 0.0,
-                "top_p": 1.0,
-                "top_k": -1,
+                "temperature": 0.9,
+                "top_k": 50,
                 "max_tokens": 4096,
-                "detokenize": True,
-                "repetition_penalty": 1.1,
+                "seed": SEED,
+                "detokenize": False,
+                "repetition_penalty": 1.05,
                 "stop_token_ids": [2150],
             },
             "code2wav": {
                 "temperature": 0.0,
                 "top_p": 1.0,
                 "top_k": -1,
-                "max_tokens": 2048,
+                "max_tokens": 4096 * 16,
+                "seed": SEED,
                 "detokenize": True,
                 "repetition_penalty": 1.1,
             },
@@ -93,9 +94,58 @@ def parse_args():
         default=None,
         help="Path to custom stage configs YAML file (optional).",
     )
+    parser.add_argument(
+        "--log-stats",
+        action="store_true",
+        help="Enable statistics logging for AsyncOmni.",
+    )
+    parser.add_argument(
+        "--log-file",
+        type=str,
+        default=None,
+        help="Path prefix for AsyncOmni log files.",
+    )
+    parser.add_argument(
+        "--init-sleep-seconds",
+        type=int,
+        default=30,
+        help="Seconds to sleep between starting stage processes.",
+    )
+    parser.add_argument(
+        "--shm-threshold-bytes",
+        type=int,
+        default=65536,
+        help="Threshold in bytes for using shared memory IPC.",
+    )
+    parser.add_argument(
+        "--batch-timeout",
+        type=int,
+        default=10,
+        help="Batching timeout (seconds) inside each stage.",
+    )
+    parser.add_argument(
+        "--init-timeout",
+        type=int,
+        default=ASYNC_INIT_TIMEOUT,
+        help="Timeout (seconds) for initializing all stages.",
+    )
     return parser.parse_args()
 
 
+def build_async_omni_cli_args(base_args: argparse.Namespace) -> argparse.Namespace:
+    """Construct the minimal CLI args Namespace expected by AsyncOmni."""
+    return argparse.Namespace(
+        model=base_args.model,
+        stage_configs_path=getattr(base_args, "stage_configs_path", None),
+        log_stats=bool(getattr(base_args, "log_stats", False)),
+        log_file=getattr(base_args, "log_file", None),
+        init_sleep_seconds=int(getattr(base_args, "init_sleep_seconds", 30)),
+        shm_threshold_bytes=int(getattr(base_args, "shm_threshold_bytes", 65536)),
+        batch_timeout=int(getattr(base_args, "batch_timeout", 10)),
+        init_timeout=int(getattr(base_args, "init_timeout", ASYNC_INIT_TIMEOUT)),
+    )
+
+
 def build_sampling_params(seed: int, model_key: str) -> list[SamplingParams]:
     """Build SamplingParams objects by reusing the dict definitions."""
     return [SamplingParams(**params_dict) for params_dict in build_sampling_params_dict(seed, model_key)]
@@ -238,8 +288,8 @@ def process_video_file(
     return frames, metadata, audio_tuple
 
 
-async def run_inference_async_omni_llm(
-    omni_llm: AsyncOmniLLM,
+async def run_inference_async_omni(
+    omni: AsyncOmni,
     sampling_params: list[SamplingParams],
     prompt_args_template: SimpleNamespace,
     user_prompt: str,
@@ -248,7 +298,7 @@ async def run_inference_async_omni_llm(
     video_file: Optional[str] = None,
     use_audio_in_video: bool = False,
 ):
-    """Run inference using AsyncOmniLLM directly with multimodal support."""
+    """Run inference using AsyncOmni directly with multimodal support."""
     if not user_prompt.strip() and not audio_file and not image_file and not video_file:
         return "Please provide at least a text prompt or multimodal input.", None
 
@@ -263,7 +313,7 @@ async def run_inference_async_omni_llm(
         # Process audio
         audio_data = process_audio_file(audio_file)
         if audio_data is not None:
-            multi_modal_data["audio"] = audio_data
+            multi_modal_data["audio"] = [audio_data] if not isinstance(audio_data, list) else audio_data
 
         # Process image
         image_data = process_image_file(image_file)
@@ -281,7 +331,9 @@ async def run_inference_async_omni_llm(
                 video_entry = video_frames
             multi_modal_data["video"] = video_entry
             if use_audio_in_video and extracted_audio is not None and "audio" not in multi_modal_data:
-                multi_modal_data["audio"] = extracted_audio
+                multi_modal_data["audio"] = (
+                    [extracted_audio] if not isinstance(extracted_audio, list) else extracted_audio
+                )
                 mm_processor_kwargs["use_audio_in_video"] = True
 
         # Build the prompt input
@@ -310,11 +362,11 @@ async def run_inference_async_omni_llm(
             )
             prompt = f"<|im_start|>system\n{default_system}<|im_end|>\n<|im_start|>user\n"
             if audio_data:
-                prompt += "<|audio_bos|><|AUDIO|><|audio_eos|>"
+                prompt += "<|audio_start|><|audio_pad|><|audio_end|>"
             if image_data:
-                prompt += "<|vision_bos|><|IMAGE|><|vision_eos|>"
+                prompt += "<|vision_start|><|image_pad|><|vision_end|>"
             if video_payload is not None:
-                prompt += "<|vision_bos|><|VIDEO|><|vision_eos|>"
+                prompt += "<|vision_start|><|video_pad|><|vision_end|>"
             if user_prompt.strip():
                 prompt += f"{user_prompt}"
             prompt += "<|im_end|>\n<|im_start|>assistant\n"
@@ -330,7 +382,7 @@ async def run_inference_async_omni_llm(
         text_outputs: list[str] = []
         audio_output = None
 
-        async for stage_outputs in omni_llm.generate(
+        async for stage_outputs in omni.generate(
             prompt=omni_prompt,
             request_id=request_id,
             sampling_params_list=sampling_params,
@@ -344,14 +396,15 @@ async def run_inference_async_omni_llm(
                             text_outputs.append(output.text)
             elif stage_outputs.final_output_type == "audio":
                 # multimodal_output is on the RequestOutput object
-                # See vllm_omni/entrypoints/openai/serving_chat.py:680 for reference
+                # See vllm_omni/entrypoints/openai/serving_chat.py:733 for reference
                 if hasattr(request_output, "multimodal_output") and request_output.multimodal_output:
                     if "audio" in request_output.multimodal_output:
                         audio_tensor = request_output.multimodal_output["audio"]
                         # Ensure audio is 1D (flatten if needed)
                         if hasattr(audio_tensor, "ndim") and audio_tensor.ndim > 1:
                             audio_tensor = audio_tensor.flatten()
-                        audio_np = audio_tensor.detach().cpu().numpy()
+                        # Convert BFloat16 to Float32 before numpy conversion (numpy doesn't support BFloat16)
+                        audio_np = audio_tensor.float().detach().cpu().numpy()
                         audio_output = (
                             24000,  # sampling rate in Hz
                             audio_np,
@@ -364,12 +417,12 @@ async def run_inference_async_omni_llm(
 
 
 def build_interface(
-    omni_llm: AsyncOmniLLM,
+    omni: AsyncOmni,
     sampling_params: list[SamplingParams],
     prompt_args_template: SimpleNamespace,
     model: str,
 ):
-    """Build Gradio interface for AsyncOmniLLM mode."""
+    """Build Gradio interface for AsyncOmni mode."""
 
     async def run_inference(
         user_prompt: str,
@@ -378,8 +431,8 @@ def build_interface(
         video_file: Optional[str],
         use_audio_in_video: bool,
     ):
-        return await run_inference_async_omni_llm(
-            omni_llm,
+        return await run_inference_async_omni(
+            omni,
             sampling_params,
             prompt_args_template,
             user_prompt,
@@ -477,7 +530,7 @@ def build_interface(
 
 def main():
     args = parse_args()
-    omni_llm = None
+    omni = None
 
     model_name = "/".join(args.model.split("/")[-2:])
     assert model_name in SUPPORTED_MODELS, (
@@ -487,9 +540,9 @@ def main():
     # Register signal handlers for graceful shutdown
     def signal_handler(sig, frame):
         print("\nReceived interrupt signal, shutting down...")
-        if omni_llm is not None:
+        if omni is not None:
             try:
-                omni_llm.shutdown()
+                omni.shutdown()
             except Exception as e:
                 print(f"Error during shutdown: {e}")
         sys.exit(0)
@@ -497,21 +550,18 @@ def main():
     signal.signal(signal.SIGINT, signal_handler)
     signal.signal(signal.SIGTERM, signal_handler)
 
-    print(f"Initializing AsyncOmniLLM with model: {args.model}")
+    print(f"Initializing AsyncOmni with model: {args.model}")
     if args.stage_configs_path:
         print(f"Using custom stage configs: {args.stage_configs_path}")
 
     sampling_params = build_sampling_params(SEED, model_name)
-    omni_llm = AsyncOmniLLM(
-        model=args.model,
-        stage_configs_path=args.stage_configs_path,
-        init_timeout=ASYNC_INIT_TIMEOUT,
-    )
-    print("✓ AsyncOmniLLM initialized successfully")
+    cli_args = build_async_omni_cli_args(args)
+    omni = AsyncOmni(model=args.model, cli_args=cli_args)
+    print("✓ AsyncOmni initialized successfully")
     prompt_args_template = create_prompt_args(args)
 
     demo = build_interface(
-        omni_llm,
+        omni,
         sampling_params,
         prompt_args_template,
         args.model,
@@ -526,9 +576,9 @@ def main():
         print("\nShutting down...")
     finally:
         # Cleanup
-        if omni_llm is not None:
+        if omni is not None:
             try:
-                omni_llm.shutdown()
+                omni.shutdown()
             except Exception as e:
                 print(f"Error during cleanup: {e}")
 

examples/online_serving/qwen3_omni/openai_chat_completion_client_for_multimodal_generation.py

@@ -1,5 +1,6 @@
 import base64
-from typing import NamedTuple
+import os
+from typing import NamedTuple, Optional
 
 import requests
 from openai import OpenAI
@@ -34,6 +35,129 @@ def encode_base64_content_from_url(content_url: str) -> str:
     return result
 
 
+def encode_base64_content_from_file(file_path: str) -> str:
+    """Encode a local file to base64 format."""
+    with open(file_path, "rb") as f:
+        content = f.read()
+        result = base64.b64encode(content).decode("utf-8")
+    return result
+
+
+def get_video_url_from_path(video_path: Optional[str]) -> str:
+    """Convert a video path (local file or URL) to a video URL format for the API.
+
+    If video_path is None or empty, returns the default URL.
+    If video_path is a local file path, encodes it to base64 data URL.
+    If video_path is a URL, returns it as-is.
+    """
+    if not video_path:
+        # Default video URL
+        return "https://huggingface.co/datasets/raushan-testing-hf/videos-test/resolve/main/sample_demo_1.mp4"
+
+    # Check if it's a URL (starts with http:// or https://)
+    if video_path.startswith(("http://", "https://")):
+        return video_path
+
+    # Otherwise, treat it as a local file path
+    if not os.path.exists(video_path):
+        raise FileNotFoundError(f"Video file not found: {video_path}")
+
+    # Detect video MIME type from file extension
+    video_path_lower = video_path.lower()
+    if video_path_lower.endswith(".mp4"):
+        mime_type = "video/mp4"
+    elif video_path_lower.endswith(".webm"):
+        mime_type = "video/webm"
+    elif video_path_lower.endswith(".mov"):
+        mime_type = "video/quicktime"
+    elif video_path_lower.endswith(".avi"):
+        mime_type = "video/x-msvideo"
+    elif video_path_lower.endswith(".mkv"):
+        mime_type = "video/x-matroska"
+    else:
+        # Default to mp4 if extension is unknown
+        mime_type = "video/mp4"
+
+    video_base64 = encode_base64_content_from_file(video_path)
+    return f"data:{mime_type};base64,{video_base64}"
+
+
+def get_image_url_from_path(image_path: Optional[str]) -> str:
+    """Convert an image path (local file or URL) to an image URL format for the API.
+
+    If image_path is None or empty, returns the default URL.
+    If image_path is a local file path, encodes it to base64 data URL.
+    If image_path is a URL, returns it as-is.
+    """
+    if not image_path:
+        # Default image URL
+        return "https://vllm-public-assets.s3.us-west-2.amazonaws.com/vision_model_images/cherry_blossom.jpg"
+
+    # Check if it's a URL (starts with http:// or https://)
+    if image_path.startswith(("http://", "https://")):
+        return image_path
+
+    # Otherwise, treat it as a local file path
+    if not os.path.exists(image_path):
+        raise FileNotFoundError(f"Image file not found: {image_path}")
+
+    # Detect image MIME type from file extension
+    image_path_lower = image_path.lower()
+    if image_path_lower.endswith((".jpg", ".jpeg")):
+        mime_type = "image/jpeg"
+    elif image_path_lower.endswith(".png"):
+        mime_type = "image/png"
+    elif image_path_lower.endswith(".gif"):
+        mime_type = "image/gif"
+    elif image_path_lower.endswith(".webp"):
+        mime_type = "image/webp"
+    else:
+        # Default to jpeg if extension is unknown
+        mime_type = "image/jpeg"
+
+    image_base64 = encode_base64_content_from_file(image_path)
+    return f"data:{mime_type};base64,{image_base64}"
+
+
+def get_audio_url_from_path(audio_path: Optional[str]) -> str:
+    """Convert an audio path (local file or URL) to an audio URL format for the API.
+
+    If audio_path is None or empty, returns the default URL.
+    If audio_path is a local file path, encodes it to base64 data URL.
+    If audio_path is a URL, returns it as-is.
+    """
+    if not audio_path:
+        # Default audio URL
+        return AudioAsset("mary_had_lamb").url
+
+    # Check if it's a URL (starts with http:// or https://)
+    if audio_path.startswith(("http://", "https://")):
+        return audio_path
+
+    # Otherwise, treat it as a local file path
+    if not os.path.exists(audio_path):
+        raise FileNotFoundError(f"Audio file not found: {audio_path}")
+
+    # Detect audio MIME type from file extension
+    audio_path_lower = audio_path.lower()
+    if audio_path_lower.endswith((".mp3", ".mpeg")):
+        mime_type = "audio/mpeg"
+    elif audio_path_lower.endswith(".wav"):
+        mime_type = "audio/wav"
+    elif audio_path_lower.endswith(".ogg"):
+        mime_type = "audio/ogg"
+    elif audio_path_lower.endswith(".flac"):
+        mime_type = "audio/flac"
+    elif audio_path_lower.endswith(".m4a"):
+        mime_type = "audio/mp4"
+    else:
+        # Default to wav if extension is unknown
+        mime_type = "audio/wav"
+
+    audio_base64 = encode_base64_content_from_file(audio_path)
+    return f"data:{mime_type};base64,{audio_base64}"
+
+
 def get_system_prompt():
     return {
         "role": "system",
@@ -50,8 +174,10 @@ def get_system_prompt():
     }
 
 
-def get_text_query():
-    question = "Explain the system architecture for a scalable audio generation pipeline. Answer in 15 words."
+def get_text_query(custom_prompt: Optional[str] = None):
+    question = (
+        custom_prompt or "Explain the system architecture for a scalable audio generation pipeline. Answer in 15 words."
+    )
     prompt = {
         "role": "user",
         "content": [
@@ -71,16 +197,15 @@ default_system = (
 )
 
 
-def get_video_query() -> QueryResult:
-    question = "Why is this video funny?"
+def get_video_query(video_path: Optional[str] = None, custom_prompt: Optional[str] = None):
+    question = custom_prompt or "Why is this video funny?"
+    video_url = get_video_url_from_path(video_path)
     prompt = {
         "role": "user",
         "content": [
             {
                 "type": "video_url",
-                "video_url": {
-                    "url": "https://huggingface.co/datasets/raushan-testing-hf/videos-test/resolve/main/sample_demo_1.mp4"
-                },
+                "video_url": {"url": video_url},
             },
             {
                 "type": "text",
@@ -91,16 +216,15 @@ def get_video_query() -> QueryResult:
     return prompt
 
 
-def get_image_query() -> QueryResult:
-    question = "What is the content of this image?"
+def get_image_query(image_path: Optional[str] = None, custom_prompt: Optional[str] = None):
+    question = custom_prompt or "What is the content of this image?"
+    image_url = get_image_url_from_path(image_path)
     prompt = {
         "role": "user",
         "content": [
             {
                 "type": "image_url",
-                "image_url": {
-                    "url": "https://vllm-public-assets.s3.us-west-2.amazonaws.com/vision_model_images/cherry_blossom.jpg"
-                },
+                "image_url": {"url": image_url},
             },
             {
                 "type": "text",
@@ -111,14 +235,15 @@ def get_image_query() -> QueryResult:
     return prompt
 
 
-def get_audio_query() -> QueryResult:
-    question = "What is the content of this audio?"
+def get_audio_query(audio_path: Optional[str] = None, custom_prompt: Optional[str] = None):
+    question = custom_prompt or "What is the content of this audio?"
+    audio_url = get_audio_url_from_path(audio_path)
     prompt = {
         "role": "user",
         "content": [
             {
                 "type": "audio_url",
-                "audio_url": {"url": AudioAsset("mary_had_lamb").url},
+                "audio_url": {"url": audio_url},
             },
             {
                 "type": "text",
@@ -178,7 +303,25 @@ def run_multimodal_generation(args) -> None:
         code2wav_sampling_params,
     ]
 
-    prompt = query_map[args.query_type]()
+    # Get paths and custom prompt from args
+    video_path = getattr(args, "video_path", None)
+    image_path = getattr(args, "image_path", None)
+    audio_path = getattr(args, "audio_path", None)
+    custom_prompt = getattr(args, "prompt", None)
+
+    # Get the query function and call it with appropriate parameters
+    query_func = query_map[args.query_type]
+    if args.query_type == "use_video":
+        prompt = query_func(video_path=video_path, custom_prompt=custom_prompt)
+    elif args.query_type == "use_image":
+        prompt = query_func(image_path=image_path, custom_prompt=custom_prompt)
+    elif args.query_type == "use_audio":
+        prompt = query_func(audio_path=audio_path, custom_prompt=custom_prompt)
+    elif args.query_type == "text":
+        prompt = query_func(custom_prompt=custom_prompt)
+    else:
+        prompt = query_func()
+
     extra_body = {
         "sampling_params_list": sampling_params_list  # Optional, it has a default setting in stage_configs of the corresponding model.
     }
@@ -197,7 +340,6 @@ def run_multimodal_generation(args) -> None:
 
     count = 0
     for choice in chat_completion.choices:
-        print(f"choice: {choice}")
         if choice.message.audio:
             audio_data = base64.b64decode(choice.message.audio.data)
             audio_file_path = f"audio_{count}.wav"
@@ -226,6 +368,34 @@ def parse_args():
         default="Qwen/Qwen3-Omni-30B-A3B-Instruct",
         help="Model Name / Path",
     )
+    parser.add_argument(
+        "--video-path",
+        "-v",
+        type=str,
+        default=None,
+        help="Path to local video file or URL. If not provided and query-type is 'use_video', uses default video URL.",
+    )
+    parser.add_argument(
+        "--image-path",
+        "-i",
+        type=str,
+        default=None,
+        help="Path to local image file or URL. If not provided and query-type is 'use_image', uses default image URL.",
+    )
+    parser.add_argument(
+        "--audio-path",
+        "-a",
+        type=str,
+        default=None,
+        help="Path to local audio file or URL. If not provided and query-type is 'use_audio', uses default audio URL.",
+    )
+    parser.add_argument(
+        "--prompt",
+        "-p",
+        type=str,
+        default=None,
+        help="Custom text prompt/question to use instead of the default prompt for the selected query type.",
+    )
 
     return parser.parse_args()
 

examples/online_serving/qwen3_omni/qwen3_omni_moe_thinking.yaml

@@ -0,0 +1,36 @@
+# Stage config for running Qwen3-Omni-MoE-Thinking (text-only output)
+# This config is for models like Qwen3-Omni-30B-A3B-Thinking that only have the
+# thinker component and do not support audio output.
+#
+# Single stage: Thinker (multimodal understanding + text generation)
+
+# The following config has been verified on 2x H100-80G GPUs.
+stage_args:
+  - stage_id: 0
+    runtime:
+      devices: "0,1"
+      max_batch_size: 1
+    engine_args:
+      model_stage: thinker
+      model_arch: Qwen3OmniMoeForConditionalGeneration
+      worker_cls: vllm_omni.worker.gpu_ar_worker.GPUARWorker
+      scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
+      gpu_memory_utilization: 0.9
+      enforce_eager: true
+      trust_remote_code: true
+      engine_output_type: text
+      distributed_executor_backend: "mp"
+      enable_prefix_caching: false
+      hf_config_name: thinker_config
+      tensor_parallel_size: 2
+    final_output: true
+    final_output_type: text
+    is_comprehension: true
+    default_sampling_params:
+      temperature: 0.4
+      top_p: 0.9
+      top_k: 1
+      max_tokens: 2048
+      seed: 42
+      detokenize: True
+      repetition_penalty: 1.05

examples/online_serving/qwen3_omni/run_curl_multimodal_generation.sh

@@ -2,16 +2,16 @@
 set -euo pipefail
 
 # Default query type
-QUERY_TYPE="${1:-mixed_modalities}"
+QUERY_TYPE="${1:-use_video}"
 
 # Validate query type
-if [[ ! "$QUERY_TYPE" =~ ^(mixed_modalities|use_audio_in_video|multi_audios|text)$ ]]; then
+if [[ ! "$QUERY_TYPE" =~ ^(text|use_audio|use_image|use_video)$ ]]; then
     echo "Error: Invalid query type '$QUERY_TYPE'"
-    echo "Usage: $0 [mixed_modalities|use_audio_in_video|multi_audios|text]"
-    echo "  mixed_modalities: Audio + Image + Video + Text query"
-    echo "  use_audio_in_video: Video + Text query (with audio extraction from video)"
-    echo "  multi_audios: Two audio clips + Text query"
+    echo "Usage: $0 [text|use_audio|use_image|use_video]"
     echo "  text: Text query"
+    echo "  use_audio: Audio + Text query"
+    echo "  use_image: Image + Text query"
+    echo "  use_video: Video + Text query"
     exit 1
 fi
 
@@ -23,7 +23,6 @@ thinker_sampling_params='{
   "top_k": 1,
   "max_tokens": 16384,
   "seed": 42,
-  "detokenize": true,
   "repetition_penalty": 1.05,
   "stop_token_ids": [151645]
 }'
@@ -33,7 +32,7 @@ talker_sampling_params='{
   "top_k": 50,
   "max_tokens": 4096,
   "seed": 42,
-  "detokenize": true,
+  "detokenize": false,
   "repetition_penalty": 1.05,
   "stop_token_ids": [2150]
 }'
@@ -42,7 +41,7 @@ code2wav_sampling_params='{
   "temperature": 0.0,
   "top_p": 1.0,
   "top_k": -1,
-  "max_tokens": 4096 * 16,
+  "max_tokens": 65536,
   "seed": 42,
   "detokenize": true,
   "repetition_penalty": 1.1
@@ -51,7 +50,6 @@ code2wav_sampling_params='{
 
 # Define URLs for assets
 MARY_HAD_LAMB_AUDIO_URL="https://vllm-public-assets.s3.us-west-2.amazonaws.com/multimodal_asset/mary_had_lamb.ogg"
-WINNING_CALL_AUDIO_URL="https://vllm-public-assets.s3.us-west-2.amazonaws.com/multimodal_asset/winning_call.ogg"
 CHERRY_BLOSSOM_IMAGE_URL="https://vllm-public-assets.s3.us-west-2.amazonaws.com/vision_model_images/cherry_blossom.jpg"
 SAMPLE_VIDEO_URL="https://huggingface.co/datasets/raushan-testing-hf/videos-test/resolve/main/sample_demo_1.mp4"
 
@@ -71,7 +69,7 @@ case "$QUERY_TYPE" in
     ]'
     mm_processor_kwargs="{}"
     ;;
-  mixed_modalities)
+  use_audio)
     user_content='[
         {
           "type": "audio_url",
@@ -79,21 +77,9 @@ case "$QUERY_TYPE" in
             "url": "'"$MARY_HAD_LAMB_AUDIO_URL"'"
           }
         },
-        {
-          "type": "image_url",
-          "image_url": {
-            "url": "'"$CHERRY_BLOSSOM_IMAGE_URL"'"
-          }
-        },
-        {
-          "type": "video_url",
-          "video_url": {
-            "url": "'"$SAMPLE_VIDEO_URL"'"
-          }
-        },
         {
           "type": "text",
-          "text": "What is recited in the audio? What is the content of this image? Why is this video funny?"
+          "text": "What is the content of this audio?"
         }
       ]'
     sampling_params_list='[
@@ -103,7 +89,27 @@ case "$QUERY_TYPE" in
     ]'
     mm_processor_kwargs="{}"
     ;;
-  use_audio_in_video)
+  use_image)
+    user_content='[
+        {
+          "type": "image_url",
+          "image_url": {
+            "url": "'"$CHERRY_BLOSSOM_IMAGE_URL"'"
+          }
+        },
+        {
+          "type": "text",
+          "text": "What is the content of this image?"
+        }
+      ]'
+    sampling_params_list='[
+      '"$thinker_sampling_params"',
+      '"$talker_sampling_params"',
+      '"$code2wav_sampling_params"'
+    ]'
+    mm_processor_kwargs="{}"
+    ;;
+  use_video)
     user_content='[
         {
           "type": "video_url",
@@ -113,35 +119,7 @@ case "$QUERY_TYPE" in
         },
         {
           "type": "text",
-          "text": "Describe the content of the video, then convert what the baby say into text."
-        }
-      ]'
-    sampling_params_list='[
-      '"$thinker_sampling_params"',
-      '"$talker_sampling_params"',
-      '"$code2wav_sampling_params"'
-    ]'
-    mm_processor_kwargs='{
-      "use_audio_in_video": true
-    }'
-    ;;
-  multi_audios)
-    user_content='[
-        {
-          "type": "audio_url",
-          "audio_url": {
-            "url": "'"$MARY_HAD_LAMB_AUDIO_URL"'"
-          }
-        },
-        {
-          "type": "audio_url",
-          "audio_url": {
-            "url": "'"$WINNING_CALL_AUDIO_URL"'"
-          }
-        },
-        {
-          "type": "text",
-          "text": "Are these two audio clips the same?"
+          "text": "Why is this video funny?"
         }
       ]'
     sampling_params_list='[
@@ -161,7 +139,7 @@ output=$(curl -sS -X POST http://localhost:8091/v1/chat/completions \
     -H "Content-Type: application/json" \
     -d @- <<EOF
 {
-  "model": "Qwen/Qwen2.5-Omni-7B",
+  "model": "Qwen/Qwen3-Omni-30B-A3B-Instruct",
   "sampling_params_list": $sampling_params_list,
   "mm_processor_kwargs": $mm_processor_kwargs,
   "messages": [

mkdocs.yml

@@ -1,5 +1,5 @@
 site_name: vLLM-Omni
-site_description: Multi-modality models inference and serving with non-autoregressive structures
+site_description: Efficient omni-modality model serving for everyone
 site_author: vLLM-Omni Team
 site_url: https://vllm-project.github.io/vllm-omni/
 
@@ -58,6 +58,7 @@ theme:
 hooks:
   - docs/mkdocs/hooks/generate_api_readme.py
   - docs/mkdocs/hooks/url_schemes.py
+  - docs/mkdocs/hooks/generate_examples.py
 
 # Plugins
 plugins:
@@ -87,6 +88,7 @@ plugins:
       show_full_namespace: false  # Show only module name, not full path
       exclude:
         - "re:vllm_omni\\._.*"  # Internal modules
+        - "vllm_omni.diffusion.models.qwen_image"  # avoid importing vllm in mkdocs building
   - mkdocstrings:
       handlers:
         python:

pyproject.toml

@@ -5,12 +5,12 @@ build-backend = "setuptools.build_meta"
 [project]
 name = "vllm-omni"
 version = "0.11.0rc1"
-description = "vLLM-Omni: Multi-modality models inference and serving with non-autoregressive structures"
+description = "A framework for efficient model inference with omni-modality models"
 readme = "README.md"
 requires-python = ">=3.9,<3.14"  # Align with vLLM v0.11 supported Python versions
 license = {text = "Apache-2.0"}
 authors = [
-    {name = "vLLM-Omni Team", email = "vllm-project@gmail.com"}
+    {name = "vLLM-Omni Team"}
 ]
 keywords = ["vllm", "multimodal", "diffusion", "transformer", "inference", "serving"]
 classifiers = [
@@ -27,6 +27,7 @@ classifiers = [
     "Topic :: Software Development :: Libraries :: Python Modules",
 ]
 
+
 dependencies = [
     # Core runtime dependencies (required for actual usage)
     "omegaconf>=2.3.0",
@@ -155,6 +156,7 @@ markers = [
     "integration: Integration tests",
     "benchmark: Benchmark tests",
     "slow: Slow tests",
+    "core_model: enable this model test in each PR instead of only nightly",
 ]
 
 [tool.typos.default]

tests/__init__.py

@@ -1,6 +1,6 @@
 """
-Test suite for vLLM-omni.
+Test suite for vLLM-Omni.
 
 This package contains unit tests, integration tests, and benchmarks
-for vLLM-omni.
+for vLLM-Omni.
 """

tests/conftest.py

@@ -0,0 +1,36 @@
+import os
+
+import pytest
+import torch
+from vllm.logger import init_logger
+
+logger = init_logger(__name__)
+
+
+@pytest.fixture(autouse=True)
+def clean_gpu_memory_between_tests():
+    if os.getenv("VLLM_TEST_CLEAN_GPU_MEMORY", "0") != "1":
+        yield
+        return
+
+    # Wait for GPU memory to be cleared before starting the test
+    import gc
+
+    from tests.utils import wait_for_gpu_memory_to_clear
+
+    num_gpus = torch.cuda.device_count()
+    if num_gpus > 0:
+        try:
+            wait_for_gpu_memory_to_clear(
+                devices=list(range(num_gpus)),
+                threshold_ratio=0.1,
+            )
+        except ValueError as e:
+            logger.info("Failed to clean GPU memory: %s", e)
+
+    yield
+
+    # Clean up GPU memory after the test
+    if torch.cuda.is_available():
+        torch.cuda.empty_cache()
+        gc.collect()

tests/distributed/omni_connectors/test_adapter_and_flow.py

@@ -0,0 +1,236 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+from unittest.mock import MagicMock
+
+import pytest
+
+from vllm_omni.distributed.omni_connectors.adapter import try_recv_via_connector, try_send_via_connector
+from vllm_omni.distributed.omni_connectors.connectors.shm_connector import SharedMemoryConnector
+from vllm_omni.distributed.omni_connectors.utils.config import ConnectorSpec, OmniTransferConfig
+from vllm_omni.distributed.omni_connectors.utils.initialization import get_connectors_config_for_stage
+
+
+@pytest.fixture
+def mock_objects():
+    return {"connector": MagicMock(), "metrics": MagicMock(), "queue_fn": MagicMock()}
+
+
+def test_send_success(mock_objects):
+    """Test try_send_via_connector success path."""
+    # Setup
+    mock_connector = mock_objects["connector"]
+    mock_metrics = mock_objects["metrics"]
+    mock_queue_fn = mock_objects["queue_fn"]
+
+    stage_id = 0
+    next_stage_id = 1
+    req_id = "req_123"
+    inputs = {"input_ids": [1, 2, 3]}
+    sampling_params = {"temperature": 0.7}
+    prompt = "test prompt"
+
+    # Mock connector.put return
+    # Returns: (success, size, metadata)
+    mock_metadata = {"handle": "xyz"}
+    mock_connector.put.return_value = (True, 100, mock_metadata)
+
+    # Execute
+    result = try_send_via_connector(
+        connector=mock_connector,
+        stage_id=stage_id,
+        next_stage_id=next_stage_id,
+        req_id=req_id,
+        next_inputs=inputs,
+        sampling_params=sampling_params,
+        original_prompt=prompt,
+        next_stage_queue_submit_fn=mock_queue_fn,
+        metrics=mock_metrics,
+    )
+
+    # Verify
+    assert result is True
+
+    # 1. Verify connector.put called correctly
+    mock_connector.put.assert_called_once()
+    args, _ = mock_connector.put.call_args
+    assert args[0] == "0"  # from_stage
+    assert args[1] == "1"  # to_stage
+    assert args[2] == req_id
+    # Verify payload structure in put
+    payload = args[3]
+    assert payload["engine_inputs"] == inputs
+    assert payload["sampling_params"] == sampling_params
+
+    # 2. Verify queue notification submitted
+    mock_queue_fn.assert_called_once()
+    notify_payload = mock_queue_fn.call_args[0][0]
+    assert notify_payload["request_id"] == req_id
+    assert notify_payload["from_connector"] is True
+    assert notify_payload["connector_metadata"] == mock_metadata
+
+    # 3. Verify metrics recorded
+    mock_metrics.on_forward.assert_called_once()
+
+
+def test_send_fail(mock_objects):
+    """Test try_send_via_connector when connector fails."""
+    mock_connector = mock_objects["connector"]
+    mock_metrics = mock_objects["metrics"]
+    mock_queue_fn = mock_objects["queue_fn"]
+
+    mock_connector.put.return_value = (False, 0, None)
+
+    result = try_send_via_connector(
+        connector=mock_connector,
+        stage_id=0,
+        next_stage_id=1,
+        req_id="req_fail",
+        next_inputs={},
+        sampling_params={},
+        original_prompt="",
+        next_stage_queue_submit_fn=mock_queue_fn,
+        metrics=mock_metrics,
+    )
+
+    assert result is False
+    mock_queue_fn.assert_not_called()
+
+
+def test_recv_success(mock_objects):
+    """Test try_recv_via_connector success path."""
+    mock_connector = mock_objects["connector"]
+
+    # Setup task received from queue
+    task = {
+        "request_id": "req_recv",
+        "from_connector": True,
+        "from_stage": "0",
+        "connector_metadata": {"handle": "xyz"},
+    }
+
+    # Setup connectors dict
+    connectors = {("0", "1"): mock_connector}
+
+    # Mock connector.get return
+    expected_data = {"engine_inputs": {"ids": [1]}}
+    # get returns: (data_obj, size)
+    mock_connector.get.return_value = (expected_data, 50)
+    # serialize_obj needed for metrics calculation if size not returned directly
+    mock_connector.serialize_obj.return_value = b"bytes"
+
+    # Execute
+    # We are stage 1 receiving from stage 0
+    inputs, rx_metrics = try_recv_via_connector(task, connectors, stage_id=1)
+
+    # Verify
+    assert inputs == expected_data["engine_inputs"]
+    assert rx_metrics is not None
+    mock_connector.get.assert_called_once_with("0", "1", "req_recv", metadata={"handle": "xyz"})
+
+
+def test_recv_no_connector():
+    """Test recv fails when no connector exists for edge."""
+    task = {"request_id": "req_missing", "from_connector": True, "from_stage": "0"}
+    connectors = {}  # Empty connectors
+
+    inputs, _ = try_recv_via_connector(task, connectors, stage_id=1)
+    assert inputs is None
+
+
+def test_shm_connector_flow():
+    """
+    Verify the full flow: Send -> Adapter -> Connector -> Adapter -> Recv.
+    Using real SharedMemoryConnector (inline mode for simplicity).
+    """
+    # 1. Setup Connector
+    config = {"shm_threshold_bytes": 1024}  # Large threshold to use inline
+    connector = SharedMemoryConnector(config)
+    connectors_map = {("0", "1"): connector}
+
+    # 2. Setup Data
+    stage_id = 0
+    next_stage_id = 1
+    req_id = "flow_req"
+    inputs = {"tokens": [10, 20, 30]}
+    sampling_params = {"n": 1}
+
+    # Queue capture mechanism
+    queue_capture = []
+
+    def mock_submit(payload):
+        queue_capture.append(payload)
+
+    mock_metrics = MagicMock()
+
+    # 3. Send
+    success = try_send_via_connector(
+        connector=connector,
+        stage_id=stage_id,
+        next_stage_id=next_stage_id,
+        req_id=req_id,
+        next_inputs=inputs,
+        sampling_params=sampling_params,
+        original_prompt="prompt",
+        next_stage_queue_submit_fn=mock_submit,
+        metrics=mock_metrics,
+    )
+    assert success is True
+    assert len(queue_capture) == 1
+
+    # 4. Recv
+    # The 'task' is what would be popped from the queue
+    received_task = queue_capture[0]
+
+    # Verify queue payload contains what we expect
+    assert received_task["from_connector"] is True
+    assert received_task["from_stage"] == "0"
+
+    # Decode
+    decoded_inputs, _ = try_recv_via_connector(received_task, connectors_map, stage_id=1)
+
+    # 5. Verify Data Integrity
+    assert decoded_inputs == inputs
+
+
+def test_get_connectors_for_stage():
+    """Test filtering logic for stage config."""
+    # Config has edges: 0->1, 1->2
+    config = OmniTransferConfig(connectors={("0", "1"): ConnectorSpec(name="C1"), ("1", "2"): ConnectorSpec(name="C2")})
+
+    # Get config for Stage 1
+    # Stage 1 receives from 0 (input) and sends to 2 (output)
+    # get_connectors_config_for_stage ONLY returns INPUT connectors for the worker to initialize
+
+    stage_config = get_connectors_config_for_stage(config, stage_id=1)
+
+    # Should contain "from_stage_0"
+    assert "from_stage_0" in stage_config
+    assert stage_config["from_stage_0"]["spec"]["name"] == "C1"
+
+    # Should NOT contain "from_stage_1" or related to output
+    assert "from_stage_1" not in stage_config
+
+    # Verify Stage 2
+    stage_2_config = get_connectors_config_for_stage(config, stage_id=2)
+    assert "from_stage_1" in stage_2_config
+    assert stage_2_config["from_stage_1"]["spec"]["name"] == "C2"
+
+
+def test_recv_with_missing_metadata():
+    """Test recv when queue payload is malformed (missing metadata)."""
+    # Connector expects metadata but task doesn't have it
+    task = {
+        "request_id": "req_bad",
+        "from_connector": True,
+        "from_stage": "0",
+        # Missing "connector_metadata"
+    }
+    mock_conn = MagicMock()
+    # If get is called with None metadata, connector usually handles it or adapter handles exception
+    mock_conn.get.side_effect = Exception("Get failed")
+
+    connectors = {("0", "1"): mock_conn}
+
+    inputs, _ = try_recv_via_connector(task, connectors, stage_id=1)
+    assert inputs is None

tests/distributed/omni_connectors/test_basic_connectors.py

@@ -0,0 +1,99 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+from unittest.mock import MagicMock
+
+import pytest
+
+from vllm_omni.distributed.omni_connectors.connectors.shm_connector import SharedMemoryConnector
+from vllm_omni.distributed.omni_connectors.factory import OmniConnectorFactory
+from vllm_omni.distributed.omni_connectors.utils.config import ConnectorSpec
+from vllm_omni.distributed.omni_connectors.utils.serialization import OmniSerializer
+
+
+def test_pickle_serialization():
+    """Test basic pickle serialization."""
+    data = {"key": "value", "list": [1, 2, 3]}
+    serialized = OmniSerializer.serialize(data, method="cloudpickle")
+    assert isinstance(serialized, bytes)
+
+    deserialized = OmniSerializer.deserialize(serialized, method="cloudpickle")
+    assert data == deserialized
+
+
+def test_create_shm_connector():
+    """Test creating SharedMemoryConnector via Factory."""
+    spec = ConnectorSpec(name="SharedMemoryConnector", extra={"shm_threshold_bytes": 1024})
+    connector = OmniConnectorFactory.create_connector(spec)
+    assert isinstance(connector, SharedMemoryConnector)
+    assert connector.threshold == 1024
+
+
+def test_create_unknown_connector():
+    """Test error when creating unknown connector."""
+    spec = ConnectorSpec(name="UnknownConnector")
+    with pytest.raises(ValueError):
+        OmniConnectorFactory.create_connector(spec)
+
+
+@pytest.fixture
+def shm_connector():
+    config = {"shm_threshold_bytes": 100}  # Small threshold for testing
+    return SharedMemoryConnector(config)
+
+
+def test_put_get_inline(shm_connector):
+    """Test inline transfer for small data."""
+    data = {"small": "data"}
+    # Ensure data is smaller than threshold (100 bytes)
+
+    success, size, metadata = shm_connector.put("stage_0", "stage_1", "req_1", data)
+    assert success is True
+    assert "inline_bytes" in metadata
+    assert "shm" not in metadata
+
+    # Retrieve
+    retrieved_data, ret_size = shm_connector.get("stage_0", "stage_1", "req_1", metadata)
+    assert data == retrieved_data
+    assert size == ret_size
+
+
+def test_put_get_shm(shm_connector, monkeypatch):
+    """Test SHM transfer logic for large data (Mocked)."""
+    # Create data larger than 100 bytes
+    data = {"large": "x" * 200}
+
+    # Mock SHM return values
+    mock_handle = {"name": "test_shm", "size": 200}
+    mock_write = MagicMock(return_value=mock_handle)
+    monkeypatch.setattr("vllm_omni.distributed.omni_connectors.connectors.shm_connector.shm_write_bytes", mock_write)
+
+    # When reading, return the serialized bytes of the data
+    serialized_data = shm_connector.serialize_obj(data)
+    mock_read = MagicMock(return_value=serialized_data)
+    monkeypatch.setattr("vllm_omni.distributed.omni_connectors.connectors.shm_connector.shm_read_bytes", mock_read)
+
+    # Put
+    success, size, metadata = shm_connector.put("stage_0", "stage_1", "req_2", data)
+
+    assert success is True
+    # Should use SHM because data > threshold
+    assert "shm" in metadata
+    assert metadata["shm"] == mock_handle
+    assert "inline_bytes" not in metadata
+
+    mock_write.assert_called_once()
+
+    # Get
+    retrieved_data, ret_size = shm_connector.get("stage_0", "stage_1", "req_2", metadata)
+
+    assert data == retrieved_data
+    mock_read.assert_called_once_with(mock_handle)
+
+
+def test_get_invalid_metadata(shm_connector):
+    """Test get with invalid metadata."""
+    result = shm_connector.get("stage_0", "stage_1", "req_3", {})
+    assert result is None
+
+    result = shm_connector.get("stage_0", "stage_1", "req_3", {"unknown": "format"})
+    assert result is None

tests/distributed/omni_connectors/test_omni_connector_configs.py

@@ -0,0 +1,64 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+from pathlib import Path
+
+import pytest
+
+# Use the new import path for initialization utilities
+from vllm_omni.distributed.omni_connectors.utils.initialization import load_omni_transfer_config
+
+
+def get_config_files():
+    """Helper to find config files."""
+    # Go up two levels from 'tests/distributed/omni_connectors' (approx) to 'vllm-omni' root
+    # Adjust based on file location: vllm-omni/tests/distributed/omni_connectors/test_omni_connector_configs.py
+    # This file is 4 levels deep from root if we count from tests?
+    # vllm-omni/tests/distributed/omni_connectors -> parent -> distributed -> parent -> tests -> parent -> vllm-omni
+    # Let's use resolve to be safe.
+
+    # Path(__file__) = .../vllm-omni/tests/distributed/omni_connectors/test_omni_connector_configs.py
+    # .parent = omni_connectors
+    # .parent = distributed
+    # .parent = tests
+    # .parent = vllm-omni
+
+    base_dir = Path(__file__).resolve().parent.parent.parent.parent
+    config_dir = base_dir / "vllm_omni" / "model_executor" / "stage_configs"
+
+    if not config_dir.exists():
+        return []
+
+    return list(config_dir.glob("qwen*.yaml"))
+
+
+# Collect files at module level for parametrization
+config_files = get_config_files()
+
+
+@pytest.mark.skipif(len(config_files) == 0, reason="No config files found or directory missing")
+@pytest.mark.parametrize("yaml_file", config_files, ids=lambda p: p.name)
+def test_load_qwen_yaml_configs(yaml_file):
+    """
+    Scan and test loading of all qwen*.yaml config files.
+    This ensures that existing stage configs are compatible with the OmniConnector system.
+    """
+    print(f"Testing config load: {yaml_file.name}")
+    try:
+        # Attempt to load the config
+        # default_shm_threshold doesn't matter much for loading correctness, using default
+        config = load_omni_transfer_config(yaml_file)
+
+        assert config is not None, "Config should not be None"
+
+        # Basic validation
+        # Note: Some configs might not have 'runtime' or 'connectors' section if they rely on auto-shm
+        # but the load function should succeed regardless.
+
+        # If the config defines stages, we expect connectors to be populated (either explicit or auto SHM)
+        # We can't strictly assert len(config.connectors) > 0 because a single stage pipeline might have 0 edges.
+
+        print(f"  -> Successfully loaded. Connectors: {len(config.connectors)}")
+
+    except Exception as e:
+        pytest.fail(f"Failed to load config {yaml_file.name}: {e}")

tests/e2e/offline_inference/conftest.py

@@ -0,0 +1,341 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""
+Pytest configuration and fixtures for vllm-omni tests.
+"""
+
+from typing import Any
+
+import pytest
+from vllm.distributed.parallel_state import cleanup_dist_env_and_memory
+from vllm.sampling_params import SamplingParams
+
+from vllm_omni.entrypoints.omni import Omni
+
+PromptAudioInput = list[tuple[Any, int]] | tuple[Any, int] | None
+PromptImageInput = list[Any] | Any | None
+PromptVideoInput = list[Any] | Any | None
+
+
+class OmniRunner:
+    """
+    Test runner for Omni models.
+    """
+
+    def __init__(
+        self,
+        model_name: str,
+        seed: int = 42,
+        init_sleep_seconds: int = 20,
+        batch_timeout: int = 10,
+        init_timeout: int = 300,
+        shm_threshold_bytes: int = 65536,
+        log_stats: bool = False,
+        stage_configs_path: str | None = None,
+        **kwargs,
+    ) -> None:
+        """
+        Initialize an OmniRunner for testing.
+
+        Args:
+            model_name: The model name or path
+            seed: Random seed for reproducibility
+            init_sleep_seconds: Sleep time after starting each stage
+            batch_timeout: Timeout for batching in seconds
+            init_timeout: Timeout for initializing stages in seconds
+            shm_threshold_bytes: Threshold for using shared memory
+            log_stats: Enable detailed statistics logging
+            stage_configs_path: Optional path to YAML stage config file
+            **kwargs: Additional arguments passed to Omni
+        """
+        self.model_name = model_name
+        self.seed = seed
+
+        self.omni = Omni(
+            model=model_name,
+            log_stats=log_stats,
+            init_sleep_seconds=init_sleep_seconds,
+            batch_timeout=batch_timeout,
+            init_timeout=init_timeout,
+            shm_threshold_bytes=shm_threshold_bytes,
+            stage_configs_path=stage_configs_path,
+            **kwargs,
+        )
+
+    def get_default_sampling_params_list(self) -> list[SamplingParams]:
+        """
+        Get a list of default sampling parameters for all stages.
+
+        Returns:
+            List of SamplingParams with default decoding for each stage
+        """
+        return [st.default_sampling_params for st in self.omni.instance.stage_list]
+
+    def get_omni_inputs(
+        self,
+        prompts: list[str] | str,
+        system_prompt: str | None = None,
+        audios: PromptAudioInput = None,
+        images: PromptImageInput = None,
+        videos: PromptVideoInput = None,
+        mm_processor_kwargs: dict[str, Any] | None = None,
+    ) -> list[dict[str, Any]]:
+        """
+        Construct Omni input format from prompts and multimodal data.
+
+        Args:
+            prompts: Text prompt(s) - either a single string or list of strings
+            system_prompt: Optional system prompt (defaults to Qwen system prompt)
+            audios: Audio input(s) - tuple of (audio_array, sample_rate) or list of tuples
+            images: Image input(s) - PIL Image or list of PIL Images
+            videos: Video input(s) - numpy array or list of numpy arrays
+            mm_processor_kwargs: Optional processor kwargs (e.g., use_audio_in_video)
+
+        Returns:
+            List of prompt dictionaries suitable for Omni.generate()
+        """
+        if system_prompt is None:
+            system_prompt = (
+                "You are Qwen, a virtual human developed by the Qwen Team, Alibaba "
+                "Group, capable of perceiving auditory and visual inputs, as well as "
+                "generating text and speech."
+            )
+
+        video_padding_token = "<|VIDEO|>"
+        image_padding_token = "<|IMAGE|>"
+        audio_padding_token = "<|AUDIO|>"
+
+        if self.model_name == "Qwen/Qwen3-Omni-30B-A3B-Instruct":
+            video_padding_token = "<|video_pad|>"
+            image_padding_token = "<|image_pad|>"
+            audio_padding_token = "<|audio_pad|>"
+
+        if isinstance(prompts, str):
+            prompts = [prompts]
+
+        def _normalize_mm_input(mm_input, num_prompts):
+            if mm_input is None:
+                return [None] * num_prompts
+            if isinstance(mm_input, list):
+                if len(mm_input) != num_prompts:
+                    raise ValueError(
+                        f"Multimodal input list length ({len(mm_input)}) must match prompts length ({num_prompts})"
+                    )
+                return mm_input
+            return [mm_input] * num_prompts
+
+        num_prompts = len(prompts)
+        audios_list = _normalize_mm_input(audios, num_prompts)
+        images_list = _normalize_mm_input(images, num_prompts)
+        videos_list = _normalize_mm_input(videos, num_prompts)
+
+        omni_inputs = []
+        for i, prompt_text in enumerate(prompts):
+            user_content = ""
+            multi_modal_data = {}
+
+            audio = audios_list[i]
+            if audio is not None:
+                if isinstance(audio, list):
+                    for _ in audio:
+                        user_content += f"<|audio_bos|>{audio_padding_token}<|audio_eos|>"
+                    multi_modal_data["audio"] = audio
+                else:
+                    user_content += f"<|audio_bos|>{audio_padding_token}<|audio_eos|>"
+                    multi_modal_data["audio"] = audio
+
+            image = images_list[i]
+            if image is not None:
+                if isinstance(image, list):
+                    for _ in image:
+                        user_content += f"<|vision_bos|>{image_padding_token}<|vision_eos|>"
+                    multi_modal_data["image"] = image
+                else:
+                    user_content += f"<|vision_bos|>{image_padding_token}<|vision_eos|>"
+                    multi_modal_data["image"] = image
+
+            video = videos_list[i]
+            if video is not None:
+                if isinstance(video, list):
+                    for _ in video:
+                        user_content += f"<|vision_bos|>{video_padding_token}<|vision_eos|>"
+                    multi_modal_data["video"] = video
+                else:
+                    user_content += f"<|vision_bos|>{video_padding_token}<|vision_eos|>"
+                    multi_modal_data["video"] = video
+
+            user_content += prompt_text
+
+            full_prompt = (
+                f"<|im_start|>system\n{system_prompt}<|im_end|>\n"
+                f"<|im_start|>user\n{user_content}<|im_end|>\n"
+                f"<|im_start|>assistant\n"
+            )
+
+            input_dict: dict[str, Any] = {"prompt": full_prompt}
+            if multi_modal_data:
+                input_dict["multi_modal_data"] = multi_modal_data
+            if mm_processor_kwargs:
+                input_dict["mm_processor_kwargs"] = mm_processor_kwargs
+
+            omni_inputs.append(input_dict)
+
+        return omni_inputs
+
+    def generate(
+        self,
+        prompts: list[dict[str, Any]],
+        sampling_params_list: list[SamplingParams] | None = None,
+    ) -> list[Any]:
+        """
+        Generate outputs for the given prompts.
+
+        Args:
+            prompts: List of prompt dictionaries with 'prompt' and optionally
+                    'multi_modal_data' keys
+            sampling_params_list: List of sampling parameters for each stage.
+                                 If None, uses default parameters.
+
+        Returns:
+            List of OmniRequestOutput objects from stages with final_output=True
+        """
+        if sampling_params_list is None:
+            sampling_params_list = self.get_default_sampling_params_list()
+
+        return self.omni.generate(prompts, sampling_params_list)
+
+    def generate_multimodal(
+        self,
+        prompts: list[str] | str,
+        sampling_params_list: list[SamplingParams] | None = None,
+        system_prompt: str | None = None,
+        audios: PromptAudioInput = None,
+        images: PromptImageInput = None,
+        videos: PromptVideoInput = None,
+        mm_processor_kwargs: dict[str, Any] | None = None,
+    ) -> list[Any]:
+        """
+        Convenience method to generate with multimodal inputs.
+
+        Args:
+            prompts: Text prompt(s)
+            sampling_params_list: List of sampling parameters for each stage
+            system_prompt: Optional system prompt
+            audios: Audio input(s)
+            images: Image input(s)
+            videos: Video input(s)
+            mm_processor_kwargs: Optional processor kwargs
+
+        Returns:
+            List of OmniRequestOutput objects from stages with final_output=True
+        """
+        omni_inputs = self.get_omni_inputs(
+            prompts=prompts,
+            system_prompt=system_prompt,
+            audios=audios,
+            images=images,
+            videos=videos,
+            mm_processor_kwargs=mm_processor_kwargs,
+        )
+        return self.generate(omni_inputs, sampling_params_list)
+
+    def generate_audio(
+        self,
+        prompts: list[str] | str,
+        sampling_params_list: list[SamplingParams] | None = None,
+        system_prompt: str | None = None,
+        audios: PromptAudioInput = None,
+        mm_processor_kwargs: dict[str, Any] | None = None,
+    ) -> list[Any]:
+        """
+        Convenience method to generate with multimodal inputs.
+        Args:
+            prompts: Text prompt(s)
+            sampling_params_list: List of sampling parameters for each stage
+            system_prompt: Optional system prompt
+            audios: Audio input(s)
+            mm_processor_kwargs: Optional processor kwargs
+        Returns:
+            List of OmniRequestOutput objects from stages with final_output=True
+        """
+        omni_inputs = self.get_omni_inputs(
+            prompts=prompts,
+            system_prompt=system_prompt,
+            audios=audios,
+            mm_processor_kwargs=mm_processor_kwargs,
+        )
+        return self.generate(omni_inputs, sampling_params_list)
+
+    def generate_video(
+        self,
+        prompts: list[str] | str,
+        sampling_params_list: list[SamplingParams] | None = None,
+        system_prompt: str | None = None,
+        videos: PromptVideoInput = None,
+        mm_processor_kwargs: dict[str, Any] | None = None,
+    ) -> list[Any]:
+        """
+        Convenience method to generate with multimodal inputs.
+        Args:
+            prompts: Text prompt(s)
+            sampling_params_list: List of sampling parameters for each stage
+            system_prompt: Optional system prompt
+            videos: Video input(s)
+            mm_processor_kwargs: Optional processor kwargs
+        Returns:
+            List of OmniRequestOutput objects from stages with final_output=True
+        """
+        omni_inputs = self.get_omni_inputs(
+            prompts=prompts,
+            system_prompt=system_prompt,
+            videos=videos,
+            mm_processor_kwargs=mm_processor_kwargs,
+        )
+        return self.generate(omni_inputs, sampling_params_list)
+
+    def generate_image(
+        self,
+        prompts: list[str] | str,
+        sampling_params_list: list[SamplingParams] | None = None,
+        system_prompt: str | None = None,
+        images: PromptImageInput = None,
+        mm_processor_kwargs: dict[str, Any] | None = None,
+    ) -> list[Any]:
+        """
+        Convenience method to generate with multimodal inputs.
+        Args:
+            prompts: Text prompt(s)
+            sampling_params_list: List of sampling parameters for each stage
+            system_prompt: Optional system prompt
+            images: Image input(s)
+            mm_processor_kwargs: Optional processor kwargs
+        Returns:
+            List of OmniRequestOutput objects from stages with final_output=True
+        """
+        omni_inputs = self.get_omni_inputs(
+            prompts=prompts,
+            system_prompt=system_prompt,
+            images=images,
+            mm_processor_kwargs=mm_processor_kwargs,
+        )
+        return self.generate(omni_inputs, sampling_params_list)
+
+    def __enter__(self):
+        """Context manager entry."""
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        """Context manager exit - cleanup resources."""
+        self.close()
+        del self.omni
+        cleanup_dist_env_and_memory()
+
+    def close(self):
+        """Close and cleanup the Omni instance."""
+        if hasattr(self.omni.instance, "close"):
+            self.omni.instance.close()
+
+
+@pytest.fixture(scope="session")
+def omni_runner():
+    return OmniRunner

tests/e2e/offline_inference/stage_configs/qwen2_5_omni_ci.yaml

@@ -0,0 +1,105 @@
+# stage config for running qwen2.5-omni with architecture of OmniLLM.
+
+# The following config has been verified on 2x 24GB GPU (L4/RTX3090/RTX4090).
+# This config is optimized for CI e2e tests.
+stage_args:
+  - stage_id: 0
+    runtime:
+      process: true            # Run this stage in a separate process
+      devices: "0"            # Visible devices for this stage (CUDA_VISIBLE_DEVICES/torch.cuda.set_device)
+      max_batch_size: 1
+    engine_args:
+      model_stage: thinker
+      model_arch: Qwen2_5OmniForConditionalGeneration
+      worker_cls: vllm_omni.worker.gpu_ar_worker.GPUARWorker
+      scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
+      max_model_len: 896
+      max_num_batched_tokens: 896
+      max_num_seqs: 1
+      gpu_memory_utilization: 0.8
+      skip_mm_profiling: true
+      enforce_eager: true  # Now we only support eager mode
+      trust_remote_code: true
+      engine_output_type: latent
+      enable_prefix_caching: false
+    is_comprehension: true
+    final_output: true
+    final_output_type: text
+    default_sampling_params:
+      temperature: 0.0
+      top_p: 1.0
+      top_k: -1
+      max_tokens: 128
+      seed: 42
+      detokenize: True
+      repetition_penalty: 1.1
+  - stage_id: 1
+    runtime:
+      process: true
+      devices: "1"
+      max_batch_size: 1
+    engine_args:
+      model_stage: talker
+      model_arch: Qwen2_5OmniForConditionalGeneration
+      worker_cls: vllm_omni.worker.gpu_ar_worker.GPUARWorker
+      scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
+      max_model_len: 896
+      max_num_batched_tokens: 896
+      max_num_seqs: 1
+      gpu_memory_utilization: 0.8
+      skip_mm_profiling: true
+      enforce_eager: true
+      trust_remote_code: true
+      enable_prefix_caching: false
+      engine_output_type: latent
+    engine_input_source: [0]
+    custom_process_input_func: vllm_omni.model_executor.stage_input_processors.qwen2_5_omni.thinker2talker
+    default_sampling_params:
+      temperature: 0.9
+      top_p: 0.8
+      top_k: 40
+      max_tokens: 128
+      seed: 42
+      detokenize: True
+      repetition_penalty: 1.05
+      stop_token_ids: [8294]
+  - stage_id: 2
+    runtime:
+      process: true
+      devices: "0"            # Example: use a different GPU than the previous stage; use "0" if single GPU
+      max_batch_size: 1
+    engine_args:
+      model_stage: code2wav
+      model_arch: Qwen2_5OmniForConditionalGeneration
+      worker_cls: vllm_omni.worker.gpu_generation_worker.GPUGenerationWorker
+      scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler
+      gpu_memory_utilization: 0.15
+      enforce_eager: true
+      trust_remote_code: true
+      enable_prefix_caching: false
+      engine_output_type: audio
+    engine_input_source: [1]
+    final_output: true
+    final_output_type: audio
+    default_sampling_params:
+      temperature: 0.0
+      top_p: 1.0
+      top_k: -1
+      max_tokens: 128
+      seed: 42
+      detokenize: True
+      repetition_penalty: 1.1
+
+# Top-level runtime config (concise): default windows and stage edges
+runtime:
+  enabled: true
+  defaults:
+    window_size: -1             # Simplified: trigger downstream only after full upstream completion
+    max_inflight: 1             # Simplified: process serially within each stage
+  edges:
+    - from: 0                   # thinker → talker: trigger only after receiving full input (-1)
+      to: 1
+      window_size: -1
+    - from: 1                   # talker → code2wav: trigger only after receiving full input (-1)
+      to: 2
+      window_size: -1

tests/e2e/offline_inference/stage_configs/qwen3_omni_ci.yaml

@@ -0,0 +1,98 @@
+# Stage config for running Qwen3-Omni-MoE with 3-stage architecture
+# Stage 0: Thinker (multimodal understanding + text generation)
+# Stage 1: Talker (text embeddings → 16-layer RVQ codec codes)
+# Stage 2: Code2Wav (8-layer RVQ codes → audio waveform)
+
+# The following config has been verified on 2x H100-80G GPUs.
+stage_args:
+  - stage_id: 0
+    runtime:
+      devices: "0,1"
+      max_batch_size: 1
+    engine_args:
+      model_stage: thinker
+      model_arch: Qwen3OmniMoeForConditionalGeneration
+      worker_cls: vllm_omni.worker.gpu_ar_worker.GPUARWorker
+      scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
+      gpu_memory_utilization: 0.6
+      enforce_eager: true
+      trust_remote_code: true
+      engine_output_type: latent  # Output hidden states for talker
+      distributed_executor_backend: "mp"
+      enable_prefix_caching: false
+      hf_config_name: thinker_config
+      tensor_parallel_size: 2
+      load_format: dummy
+    final_output: true
+    final_output_type: text
+    is_comprehension: true
+    default_sampling_params:
+      temperature: 0.4
+      top_p: 0.9
+      top_k: 1
+      max_tokens: 100
+      seed: 42
+      detokenize: True
+      repetition_penalty: 1.05
+
+  - stage_id: 1
+    runtime:
+       devices: "1"
+       max_batch_size: 1
+    engine_args:
+       model_stage: talker
+       model_arch: Qwen3OmniMoeForConditionalGeneration
+       worker_cls: vllm_omni.worker.gpu_ar_worker.GPUARWorker
+       scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
+       gpu_memory_utilization: 0.5
+       enforce_eager: true
+       trust_remote_code: true
+       engine_output_type: latent  # Output codec codes for code2wav
+      #  tensor_parallel_size: 2
+       enable_prefix_caching: false
+       distributed_executor_backend: "mp"
+       hf_config_name: talker_config
+       load_format: dummy
+    engine_input_source: [0]
+    custom_process_input_func: vllm_omni.model_executor.stage_input_processors.qwen3_omni.thinker2talker
+    # final_output: true
+    # final_output_type: text
+    default_sampling_params:
+      temperature: 0.9
+      top_k: 50
+      max_tokens: 100
+      seed: 42
+      detokenize: False
+      repetition_penalty: 1.05
+      stop_token_ids: [2150]
+
+  - stage_id: 2
+    runtime:
+      devices: "0"
+      max_batch_size: 1
+    engine_args:
+      model_stage: code2wav
+      model_arch: Qwen3OmniMoeForConditionalGeneration
+      worker_cls: vllm_omni.worker.gpu_generation_worker.GPUGenerationWorker
+      scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler
+      enforce_eager: true
+      trust_remote_code: true
+      enable_prefix_caching: false
+      engine_output_type: audio  # Final output: audio waveform
+      gpu_memory_utilization: 0.1
+      distributed_executor_backend: "mp"
+      max_num_batched_tokens: 1000000
+      hf_config_name: thinker_config
+      load_format: dummy
+    engine_input_source: [1]
+    custom_process_input_func: vllm_omni.model_executor.stage_input_processors.qwen3_omni.talker2code2wav
+    final_output: true
+    final_output_type: audio
+    default_sampling_params:
+      temperature: 0.0
+      top_p: 1.0
+      top_k: -1
+      max_tokens: 200
+      seed: 42
+      detokenize: True
+      repetition_penalty: 1.1

tests/e2e/offline_inference/test_qwen2_5_omni.py

@@ -0,0 +1,77 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""
+E2E tests for Qwen2.5-Omni model with mixed modality inputs and audio output.
+"""
+
+from pathlib import Path
+
+import pytest
+from vllm.assets.audio import AudioAsset
+from vllm.assets.image import ImageAsset
+from vllm.assets.video import VideoAsset
+from vllm.multimodal.image import convert_image_mode
+
+from .conftest import OmniRunner
+from .utils import create_new_process_for_each_test
+
+models = ["Qwen/Qwen2.5-Omni-3B"]
+
+# CI stage config optimized for 24GB GPU (L4/RTX3090)
+stage_configs = [str(Path(__file__).parent / "stage_configs" / "qwen2_5_omni_ci.yaml")]
+
+# Create parameter combinations for model and stage config
+test_params = [(model, stage_config) for model in models for stage_config in stage_configs]
+
+
+@pytest.mark.core_model
+@pytest.mark.parametrize("test_config", test_params)
+@create_new_process_for_each_test()
+def test_mixed_modalities_to_audio(omni_runner: type[OmniRunner], test_config: tuple[str, str]) -> None:
+    """Test processing audio, image, and video together, generating audio output."""
+    model, stage_config_path = test_config
+    with omni_runner(model, seed=42, stage_configs_path=stage_config_path) as runner:
+        # Prepare multimodal inputs
+        question = "What is recited in the audio? What is in this image? Describe the video briefly."
+        audio = AudioAsset("mary_had_lamb").audio_and_sample_rate
+        audio = (audio[0][: 16000 * 5], audio[1])  # Trim to first 5 seconds
+        image = convert_image_mode(ImageAsset("cherry_blossom").pil_image.resize((128, 128)), "RGB")
+        video = VideoAsset(name="baby_reading", num_frames=4).np_ndarrays
+
+        outputs = runner.generate_multimodal(
+            prompts=question,
+            audios=audio,
+            images=image,
+            videos=video,
+        )
+
+        # Verify we got outputs from multiple stages
+        assert len(outputs) > 0
+
+        # Find and verify text output (thinker stage)
+        text_output = None
+        for stage_output in outputs:
+            if stage_output.final_output_type == "text":
+                text_output = stage_output
+                break
+
+        assert text_output is not None
+        assert len(text_output.request_output) > 0
+        text_content = text_output.request_output[0].outputs[0].text
+        assert text_content is not None
+        assert len(text_content.strip()) > 0
+
+        # Find and verify audio output (code2wav stage)
+        audio_output = None
+        for stage_output in outputs:
+            if stage_output.final_output_type == "audio":
+                audio_output = stage_output
+                break
+
+        assert audio_output is not None
+        assert len(audio_output.request_output) > 0
+
+        # Verify audio tensor exists and has content
+        audio_tensor = audio_output.request_output[0].multimodal_output["audio"]
+        assert audio_tensor is not None
+        assert audio_tensor.numel() > 0

tests/e2e/offline_inference/test_qwen3_omni.py

@@ -0,0 +1,69 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""
+E2E offline tests for Omni model with video input and audio output.
+"""
+
+import os
+from pathlib import Path
+
+import pytest
+from vllm.assets.video import VideoAsset
+
+from .conftest import OmniRunner
+
+os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
+
+models = ["Qwen/Qwen3-Omni-30B-A3B-Instruct"]
+
+# CI stage config for 2xH100-80G GPUs
+stage_configs = [str(Path(__file__).parent / "stage_configs" / "qwen3_omni_ci.yaml")]
+
+# Create parameter combinations for model and stage config
+test_params = [(model, stage_config) for model in models for stage_config in stage_configs]
+
+
+@pytest.mark.parametrize("test_config", test_params)
+def test_video_to_audio(omni_runner: type[OmniRunner], test_config) -> None:
+    """Test processing video, generating audio output."""
+    model, stage_config_path = test_config
+    with omni_runner(model, seed=42, stage_configs_path=stage_config_path) as runner:
+        # Prepare inputs
+        question = "Describe the video briefly."
+        video = VideoAsset(name="baby_reading", num_frames=4).np_ndarrays
+
+        outputs = runner.generate_multimodal(
+            prompts=question,
+            videos=video,
+        )
+
+        # Verify we got outputs from multiple stages
+        assert len(outputs) > 0
+
+        # Find and verify text output (thinker stage)
+        text_output = None
+        for stage_output in outputs:
+            if stage_output.final_output_type == "text":
+                text_output = stage_output
+                break
+
+        assert text_output is not None
+        assert len(text_output.request_output) > 0
+        text_content = text_output.request_output[0].outputs[0].text
+        assert text_content is not None
+        assert len(text_content.strip()) > 0
+
+        # Find and verify audio output (code2wav stage)
+        audio_output = None
+        for stage_output in outputs:
+            if stage_output.final_output_type == "audio":
+                audio_output = stage_output
+                break
+
+        assert audio_output is not None
+        assert len(audio_output.request_output) > 0
+
+        # Verify audio tensor exists and has content
+        audio_tensor = audio_output.request_output[0].multimodal_output["audio"]
+        assert audio_tensor is not None
+        assert audio_tensor.numel() > 0

tests/e2e/offline_inference/test_t2i_model.py

@@ -0,0 +1,39 @@
+import os
+import sys
+from pathlib import Path
+
+import pytest
+import torch
+
+# ruff: noqa: E402
+REPO_ROOT = Path(__file__).resolve().parents[2]
+if str(REPO_ROOT) not in sys.path:
+    sys.path.insert(0, str(REPO_ROOT))
+
+from vllm_omni import Omni
+
+os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "1"
+
+models = ["Tongyi-MAI/Z-Image-Turbo", "riverclouds/qwen_image_random"]
+
+
+@pytest.mark.parametrize("model_name", models)
+def test_diffusion_model(model_name: str):
+    m = Omni(model=model_name)
+    # high resolution may cause OOM on L4
+    height = 256
+    width = 256
+    images = m.generate(
+        "a photo of a cat sitting on a laptop keyboard",
+        height=height,
+        width=width,
+        num_inference_steps=2,
+        guidance_scale=0.0,
+        generator=torch.Generator("cuda").manual_seed(42),
+        num_outputs_per_prompt=2,
+    )
+    assert len(images) == 2
+    # check image size
+    assert images[0].width == width
+    assert images[0].height == height
+    images[0].save("image_output.png")

tests/e2e/offline_inference/test_t2v_model.py

@@ -0,0 +1,48 @@
+import os
+import sys
+from pathlib import Path
+
+import pytest
+import torch
+
+# ruff: noqa: E402
+REPO_ROOT = Path(__file__).resolve().parents[2]
+if str(REPO_ROOT) not in sys.path:
+    sys.path.insert(0, str(REPO_ROOT))
+
+from vllm_omni import Omni
+
+os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "1"
+
+models = ["Wan-AI/Wan2.2-T2V-A14B-Diffusers"]
+
+
+@pytest.mark.parametrize("model_name", models)
+def test_video_diffusion_model(model_name: str):
+    m = Omni(
+        model=model_name,
+        boundary_ratio=0.875,
+        flow_shift=5.0,
+    )
+    # Use minimal settings for testing
+    # num_frames must satisfy: num_frames % vae_scale_factor_temporal == 1
+    # For Wan2.2, vae_scale_factor_temporal=4, so valid values are 5, 9, 13, 17, ...
+    height = 480
+    width = 640
+    num_frames = 5
+    frames = m.generate(
+        "A cat sitting on a table",
+        height=height,
+        width=width,
+        num_frames=num_frames,
+        num_inference_steps=2,
+        guidance_scale=1.0,
+        generator=torch.Generator("cuda").manual_seed(42),
+    )
+
+    assert frames is not None
+    assert hasattr(frames, "shape")
+    # frames shape: (batch, num_frames, height, width, channels)
+    assert frames.shape[1] == num_frames
+    assert frames.shape[2] == height
+    assert frames.shape[3] == width

tests/e2e/offline_inference/utils.py

@@ -0,0 +1,206 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+import contextlib
+import functools
+import os
+import signal
+import subprocess
+import sys
+import tempfile
+from collections.abc import Callable
+from contextlib import ExitStack, suppress
+from pathlib import Path
+from typing import Any, Literal
+
+import cloudpickle
+from typing_extensions import ParamSpec
+from vllm.platforms import current_platform
+
+VLLM_PATH = Path(__file__).parent.parent.parent
+"""Path to root of the vLLM repository."""
+
+
+_P = ParamSpec("_P")
+
+
+def fork_new_process_for_each_test(func: Callable[_P, None]) -> Callable[_P, None]:
+    """Decorator to fork a new process for each test function.
+    See https://github.com/vllm-project/vllm/issues/7053 for more details.
+    """
+
+    @functools.wraps(func)
+    def wrapper(*args: _P.args, **kwargs: _P.kwargs) -> None:
+        # Make the process the leader of its own process group
+        # to avoid sending SIGTERM to the parent process
+        os.setpgrp()
+        from _pytest.outcomes import Skipped
+
+        # Create a unique temporary file to store exception info from child
+        # process. Use test function name and process ID to avoid collisions.
+        with (
+            tempfile.NamedTemporaryFile(
+                delete=False,
+                mode="w+b",
+                prefix=f"vllm_test_{func.__name__}_{os.getpid()}_",
+                suffix=".exc",
+            ) as exc_file,
+            ExitStack() as delete_after,
+        ):
+            exc_file_path = exc_file.name
+            delete_after.callback(os.remove, exc_file_path)
+
+            pid = os.fork()
+            print(f"Fork a new process to run a test {pid}")
+            if pid == 0:
+                # Parent process responsible for deleting, don't delete
+                # in child.
+                delete_after.pop_all()
+                try:
+                    func(*args, **kwargs)
+                except Skipped as e:
+                    # convert Skipped to exit code 0
+                    print(str(e))
+                    os._exit(0)
+                except Exception as e:
+                    import traceback
+
+                    tb_string = traceback.format_exc()
+
+                    # Try to serialize the exception object first
+                    exc_to_serialize: dict[str, Any]
+                    try:
+                        # First, try to pickle the actual exception with
+                        # its traceback.
+                        exc_to_serialize = {"pickled_exception": e}
+                        # Test if it can be pickled
+                        cloudpickle.dumps(exc_to_serialize)
+                    except (Exception, KeyboardInterrupt):
+                        # Fall back to string-based approach.
+                        exc_to_serialize = {
+                            "exception_type": type(e).__name__,
+                            "exception_msg": str(e),
+                            "traceback": tb_string,
+                        }
+                    try:
+                        with open(exc_file_path, "wb") as f:
+                            cloudpickle.dump(exc_to_serialize, f)
+                    except Exception:
+                        # Fallback: just print the traceback.
+                        print(tb_string)
+                    os._exit(1)
+                else:
+                    os._exit(0)
+            else:
+                pgid = os.getpgid(pid)
+                _pid, _exitcode = os.waitpid(pid, 0)
+                # ignore SIGTERM signal itself
+                old_signal_handler = signal.signal(signal.SIGTERM, signal.SIG_IGN)
+                # kill all child processes
+                os.killpg(pgid, signal.SIGTERM)
+                # restore the signal handler
+                signal.signal(signal.SIGTERM, old_signal_handler)
+                if _exitcode != 0:
+                    # Try to read the exception from the child process
+                    exc_info = {}
+                    if os.path.exists(exc_file_path):
+                        with (
+                            contextlib.suppress(Exception),
+                            open(exc_file_path, "rb") as f,
+                        ):
+                            exc_info = cloudpickle.load(f)
+
+                    if (original_exception := exc_info.get("pickled_exception")) is not None:
+                        # Re-raise the actual exception object if it was
+                        # successfully pickled.
+                        assert isinstance(original_exception, Exception)
+                        raise original_exception
+
+                    if (original_tb := exc_info.get("traceback")) is not None:
+                        # Use string-based traceback for fallback case
+                        raise AssertionError(
+                            f"Test {func.__name__} failed when called with"
+                            f" args {args} and kwargs {kwargs}"
+                            f" (exit code: {_exitcode}):\n{original_tb}"
+                        ) from None
+
+                    # Fallback to the original generic error
+                    raise AssertionError(
+                        f"function {func.__name__} failed when called with"
+                        f" args {args} and kwargs {kwargs}"
+                        f" (exit code: {_exitcode})"
+                    ) from None
+
+    return wrapper
+
+
+def spawn_new_process_for_each_test(f: Callable[_P, None]) -> Callable[_P, None]:
+    """Decorator to spawn a new process for each test function."""
+
+    @functools.wraps(f)
+    def wrapper(*args: _P.args, **kwargs: _P.kwargs) -> None:
+        # Check if we're already in a subprocess
+        if os.environ.get("RUNNING_IN_SUBPROCESS") == "1":
+            # If we are, just run the function directly
+            return f(*args, **kwargs)
+
+        import torch.multiprocessing as mp
+
+        with suppress(RuntimeError):
+            mp.set_start_method("spawn")
+
+        # Get the module
+        module_name = f.__module__
+
+        # Create a process with environment variable set
+        env = os.environ.copy()
+        env["RUNNING_IN_SUBPROCESS"] = "1"
+
+        with tempfile.TemporaryDirectory() as tempdir:
+            output_filepath = os.path.join(tempdir, "new_process.tmp")
+
+            # `cloudpickle` allows pickling complex functions directly
+            input_bytes = cloudpickle.dumps((f, output_filepath))
+
+            repo_root = str(VLLM_PATH.resolve())
+
+            env = dict(env or os.environ)
+            env["PYTHONPATH"] = repo_root + os.pathsep + env.get("PYTHONPATH", "")
+
+            cmd = [sys.executable, "-m", f"{module_name}"]
+
+            returned = subprocess.run(cmd, input=input_bytes, capture_output=True, env=env)
+
+            # check if the subprocess is successful
+            try:
+                returned.check_returncode()
+            except Exception as e:
+                # wrap raised exception to provide more information
+                raise RuntimeError(f"Error raised in subprocess:\n{returned.stderr.decode()}") from e
+
+    return wrapper
+
+
+def create_new_process_for_each_test(
+    method: Literal["spawn", "fork"] | None = None,
+) -> Callable[[Callable[_P, None]], Callable[_P, None]]:
+    """Creates a decorator that runs each test function in a new process.
+
+    Args:
+        method: The process creation method. Can be either "spawn" or "fork".
+               If not specified, it defaults to "spawn" on ROCm and XPU
+               platforms and "fork" otherwise.
+
+    Returns:
+        A decorator to run test functions in separate processes.
+    """
+    if method is None:
+        use_spawn = current_platform.is_rocm() or current_platform.is_xpu()
+        method = "spawn" if use_spawn else "fork"
+
+    assert method in ["spawn", "fork"], "Method must be either 'spawn' or 'fork'"
+
+    if method == "fork":
+        return fork_new_process_for_each_test
+
+    return spawn_new_process_for_each_test

tests/e2e/online_serving/stage_configs/qwen3_omni_ci.yaml

@@ -0,0 +1,98 @@
+# Stage config for running Qwen3-Omni-MoE with 3-stage architecture
+# Stage 0: Thinker (multimodal understanding + text generation)
+# Stage 1: Talker (text embeddings → 16-layer RVQ codec codes)
+# Stage 2: Code2Wav (8-layer RVQ codes → audio waveform)
+
+# The following config has been verified on 2x H100-80G GPUs.
+stage_args:
+  - stage_id: 0
+    runtime:
+      devices: "0,1"
+      max_batch_size: 1
+    engine_args:
+      model_stage: thinker
+      model_arch: Qwen3OmniMoeForConditionalGeneration
+      worker_cls: vllm_omni.worker.gpu_ar_worker.GPUARWorker
+      scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
+      gpu_memory_utilization: 0.6
+      enforce_eager: true
+      trust_remote_code: true
+      engine_output_type: latent  # Output hidden states for talker
+      distributed_executor_backend: "mp"
+      enable_prefix_caching: false
+      hf_config_name: thinker_config
+      tensor_parallel_size: 2
+      load_format: dummy
+    final_output: true
+    final_output_type: text
+    is_comprehension: true
+    default_sampling_params:
+      temperature: 0.4
+      top_p: 0.9
+      top_k: 1
+      max_tokens: 100
+      seed: 42
+      detokenize: True
+      repetition_penalty: 1.05
+
+  - stage_id: 1
+    runtime:
+       devices: "1"
+       max_batch_size: 1
+    engine_args:
+       model_stage: talker
+       model_arch: Qwen3OmniMoeForConditionalGeneration
+       worker_cls: vllm_omni.worker.gpu_ar_worker.GPUARWorker
+       scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
+       gpu_memory_utilization: 0.3
+       enforce_eager: true
+       trust_remote_code: true
+       engine_output_type: latent  # Output codec codes for code2wav
+      #  tensor_parallel_size: 2
+       enable_prefix_caching: false
+       distributed_executor_backend: "mp"
+       hf_config_name: talker_config
+       load_format: dummy
+    engine_input_source: [0]
+    custom_process_input_func: vllm_omni.model_executor.stage_input_processors.qwen3_omni.thinker2talker
+    # final_output: true
+    # final_output_type: text
+    default_sampling_params:
+      temperature: 0.9
+      top_k: 50
+      max_tokens: 100
+      seed: 42
+      detokenize: False
+      repetition_penalty: 1.05
+      stop_token_ids: [2150]
+
+  - stage_id: 2
+    runtime:
+      devices: "0"
+      max_batch_size: 1
+    engine_args:
+      model_stage: code2wav
+      model_arch: Qwen3OmniMoeForConditionalGeneration
+      worker_cls: vllm_omni.worker.gpu_generation_worker.GPUGenerationWorker
+      scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler
+      enforce_eager: true
+      trust_remote_code: true
+      enable_prefix_caching: false
+      engine_output_type: audio  # Final output: audio waveform
+      gpu_memory_utilization: 0.1
+      distributed_executor_backend: "mp"
+      max_num_batched_tokens: 1000000
+      hf_config_name: thinker_config
+      load_format: dummy
+    engine_input_source: [1]
+    custom_process_input_func: vllm_omni.model_executor.stage_input_processors.qwen3_omni.talker2code2wav
+    final_output: true
+    final_output_type: audio
+    default_sampling_params:
+      temperature: 0.0
+      top_p: 1.0
+      top_k: -1
+      max_tokens: 200
+      seed: 42
+      detokenize: True
+      repetition_penalty: 1.1

tests/e2e/online_serving/test_qwen3_omni.py

@@ -0,0 +1,206 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""
+E2E Online tests for Qwen3-Omni model with video input and audio output.
+"""
+
+import os
+import socket
+import subprocess
+import sys
+import time
+from pathlib import Path
+
+import openai
+import pytest
+from vllm.assets.video import VideoAsset
+from vllm.utils import get_open_port
+
+os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
+
+models = ["Qwen/Qwen3-Omni-30B-A3B-Instruct"]
+
+# CI stage config for 2*H100-80G GPUs
+stage_configs = [str(Path(__file__).parent / "stage_configs" / "qwen3_omni_ci.yaml")]
+
+# Create parameter combinations for model and stage config
+test_params = [(model, stage_config) for model in models for stage_config in stage_configs]
+
+
+class OmniServer:
+    """Omniserver for vLLM-Omni tests."""
+
+    def __init__(
+        self,
+        model: str,
+        serve_args: list[str],
+        *,
+        env_dict: dict[str, str] | None = None,
+    ) -> None:
+        self.model = model
+        self.serve_args = serve_args
+        self.env_dict = env_dict
+        self.proc: subprocess.Popen | None = None
+        self.host = "127.0.0.1"
+        self.port = get_open_port()
+
+    def _start_server(self) -> None:
+        """Start the vLLM-Omni server subprocess."""
+        env = os.environ.copy()
+        env["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
+        if self.env_dict is not None:
+            env.update(self.env_dict)
+
+        cmd = [
+            sys.executable,
+            "-m",
+            "vllm_omni.entrypoints.cli.main",
+            "serve",
+            self.model,
+            "--omni",
+            "--host",
+            self.host,
+            "--port",
+            str(self.port),
+        ] + self.serve_args
+
+        print(f"Launching OmniServer with: {' '.join(cmd)}")
+        self.proc = subprocess.Popen(
+            cmd,
+            env=env,
+            cwd=os.path.dirname(os.path.dirname(os.path.abspath(__file__))),  # Set working directory to vllm-omni root
+        )
+
+        # Wait for server to be ready
+        max_wait = 600  # 10 minutes
+        start_time = time.time()
+        while time.time() - start_time < max_wait:
+            try:
+                with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock:
+                    sock.settimeout(1)
+                    result = sock.connect_ex((self.host, self.port))
+                    if result == 0:
+                        print(f"Server ready on {self.host}:{self.port}")
+                        return
+            except Exception:
+                pass
+            time.sleep(2)
+
+        raise RuntimeError(f"Server failed to start within {max_wait} seconds")
+
+    def __enter__(self):
+        self._start_server()
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        if self.proc:
+            self.proc.terminate()
+            try:
+                self.proc.wait(timeout=30)
+            except subprocess.TimeoutExpired:
+                self.proc.kill()
+                self.proc.wait()
+
+
+@pytest.fixture
+def omni_server(request):
+    """Start vLLM-Omni server as a subprocess with actual model weights.
+    Uses module scope so the server starts only once for all tests.
+    Multi-stage initialization can take 10-20+ minutes.
+    """
+    model, stage_config_path = request.param
+    with OmniServer(model, ["--stage-configs-path", stage_config_path]) as server:
+        yield server
+
+
+@pytest.fixture
+def client(omni_server):
+    """OpenAI client for the running vLLM-Omni server."""
+    return openai.OpenAI(
+        base_url=f"http://{omni_server.host}:{omni_server.port}/v1",
+        api_key="EMPTY",
+    )
+
+
+@pytest.fixture(scope="session")
+def base64_encoded_video() -> str:
+    """Base64 encoded video for testing."""
+    import base64
+
+    video = VideoAsset(name="baby_reading", num_frames=4)
+    with open(video.video_path, "rb") as f:
+        content = f.read()
+        return base64.b64encode(content).decode("utf-8")
+
+
+def get_system_prompt():
+    return {
+        "role": "system",
+        "content": [
+            {
+                "type": "text",
+                "text": (
+                    "You are Qwen, a virtual human developed by the Qwen Team, "
+                    "Alibaba Group, capable of perceiving auditory and visual inputs, "
+                    "as well as generating text and speech."
+                ),
+            }
+        ],
+    }
+
+
+def dummy_messages_from_video_data(
+    video_data_url: str,
+    content_text: str = "Describe the video briefly.",
+):
+    """Create messages with video data URL for OpenAI API."""
+    return [
+        get_system_prompt(),
+        {
+            "role": "user",
+            "content": [
+                {"type": "video_url", "video_url": {"url": video_data_url}},
+                {"type": "text", "text": content_text},
+            ],
+        },
+    ]
+
+
+@pytest.mark.parametrize("omni_server", test_params, indirect=True)
+def test_video_to_audio(
+    client: openai.OpenAI,
+    omni_server,
+    base64_encoded_video: str,
+) -> None:
+    """Test processing video, generating audio output via OpenAI API."""
+    # Create data URL for the base64 encoded video
+    video_data_url = f"data:video/mp4;base64,{base64_encoded_video}"
+
+    messages = dummy_messages_from_video_data(video_data_url)
+
+    # Test single completion
+    chat_completion = client.chat.completions.create(
+        model=omni_server.model,
+        messages=messages,
+    )
+
+    assert len(chat_completion.choices) == 2  # 1 for text output, 1 for audio output
+
+    # Verify text output
+    text_choice = chat_completion.choices[0]
+    assert text_choice.finish_reason == "length"
+
+    # Verify we got a response
+    text_message = text_choice.message
+    assert text_message.content is not None and len(text_message.content) >= 10
+    assert text_message.role == "assistant"
+
+    # Verify audio output
+    audio_choice = chat_completion.choices[1]
+    assert audio_choice.finish_reason == "stop"
+    audio_message = audio_choice.message
+
+    # Check if audio was generated
+    if hasattr(audio_message, "audio") and audio_message.audio:
+        assert audio_message.audio.data is not None
+        assert len(audio_message.audio.data) > 0

tests/entrypoints/test_stage_utils.py

@@ -0,0 +1,58 @@
+import os
+import sys
+
+import pytest
+
+from vllm_omni.entrypoints.stage_utils import set_stage_devices
+
+
+def _make_dummy_torch(call_log):
+    class _Props:
+        def __init__(self, total):
+            self.total_memory = total
+
+    class _Cuda:
+        @staticmethod
+        def is_available():
+            return True
+
+        @staticmethod
+        def set_device(idx):
+            call_log.append(idx)
+
+        @staticmethod
+        def device_count():
+            return 2
+
+        @staticmethod
+        def get_device_properties(idx):
+            return _Props(total=16000)
+
+        @staticmethod
+        def mem_get_info(idx):
+            return (8000, 16000)
+
+        @staticmethod
+        def get_device_name(idx):
+            return f"gpu-{idx}"
+
+    class _Torch:
+        cuda = _Cuda
+
+    return _Torch
+
+
+@pytest.mark.usefixtures("clean_gpu_memory_between_tests")
+def test_set_stage_devices_respects_logical_ids(monkeypatch):
+    # Preserve an existing logical mapping and ensure devices "0,1" map through it.
+    monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "6,7")
+    call_log: list[int] = []
+    dummy_torch = _make_dummy_torch(call_log)
+    monkeypatch.setitem(sys.modules, "torch", dummy_torch)
+    monkeypatch.setattr("vllm_omni.utils.detect_device_type", lambda: "cuda")
+    monkeypatch.setattr("vllm_omni.utils.get_device_control_env_var", lambda: "CUDA_VISIBLE_DEVICES")
+
+    set_stage_devices(stage_id=0, devices="0,1")
+
+    assert os.environ["CUDA_VISIBLE_DEVICES"] == "6,7"
+    assert call_log and call_log[0] == 0  # current device set after remap

tests/test_omni_llm.py

@@ -104,6 +104,7 @@ class _FakeStage:
         shm_threshold_bytes: int = 65536,
         ctx=None,
         batch_timeout: int = 10,
+        **kwargs,
     ):
         """Mock init_stage_worker: don't start real process, just send stage_ready message."""
         # Create a mock process object
@@ -342,14 +343,14 @@ def _setup_log_mocks(monkeypatch):
 def mock_get_config(monkeypatch):
     """Auto-mock get_config and related model loading functions to avoid model path validation."""
     # CRITICAL: Mock tokenizer-related imports FIRST, before any module imports
-    # This prevents ImportError when async_omni_llm is imported (which happens via omni_stage)
+    # This prevents ImportError when async_omni is imported (which happens via omni_stage)
     import sys
 
     fake_tokenizer = MagicMock()
     fake_tokenizer.encode = MagicMock(return_value=[1, 2, 3])
     fake_tokenizer.decode = MagicMock(return_value="test")
 
-    # Mock init_tokenizer_from_configs (used in async_omni_llm)
+    # Mock init_tokenizer_from_configs (used in async_omni)
     def _mock_init_tokenizer_from_configs(model_config=None, **kwargs):
         return fake_tokenizer
 
@@ -367,8 +368,9 @@ def mock_get_config(monkeypatch):
         tokenizer_module = sys.modules[tokenizer_module_path]
         setattr(tokenizer_module, "init_tokenizer_from_configs", _mock_init_tokenizer_from_configs)
 
-    # CRITICAL: Mock length_from_prompt_token_ids_or_embeds BEFORE trying to mock async_omni_llm
-    # This is because async_omni_llm imports processor.py, which imports this function at module level
+    # CRITICAL: Mock length_from_prompt_token_ids_or_embeds BEFORE trying to mock async_omni
+
+    # This is because async_omni imports processor.py, which imports this function at module level
     # Mock length_from_prompt_token_ids_or_embeds (used in processor.py)
     def _mock_length_from_prompt_token_ids_or_embeds(prompt_token_ids=None, prompt_embeds=None):
         # Return a reasonable default length
@@ -402,19 +404,19 @@ def mock_get_config(monkeypatch):
             processor_module, "length_from_prompt_token_ids_or_embeds", _mock_length_from_prompt_token_ids_or_embeds
         )
 
-    # Strategy 3: Now mock async_omni_llm AFTER length_from_prompt_token_ids_or_embeds is mocked
-    # This prevents ImportError when async_omni_llm imports processor.py
+    # Strategy 3: Now mock async_omni AFTER length_from_prompt_token_ids_or_embeds is mocked
+    # This prevents ImportError when async_omni imports processor.py
     monkeypatch.setattr(
-        "vllm_omni.entrypoints.async_omni_llm.init_tokenizer_from_configs",
+        "vllm_omni.entrypoints.async_omni.init_tokenizer_from_configs",
         _mock_init_tokenizer_from_configs,
         raising=False,
     )
 
-    # Strategy 4: If async_omni_llm is already imported, patch it directly
-    async_omni_llm_path = "vllm_omni.entrypoints.async_omni_llm"
-    if async_omni_llm_path in sys.modules:
-        async_omni_llm_module = sys.modules[async_omni_llm_path]
-        setattr(async_omni_llm_module, "init_tokenizer_from_configs", _mock_init_tokenizer_from_configs)
+    # Strategy 4: If async_omni is already imported, patch it directly
+    async_omni_path = "vllm_omni.entrypoints.async_omni"
+    if async_omni_path in sys.modules:
+        async_omni_module = sys.modules[async_omni_path]
+        setattr(async_omni_module, "init_tokenizer_from_configs", _mock_init_tokenizer_from_configs)
 
     # Now mock get_config and other functions
     fake_hf_config = MagicMock()

tests/utils.py

@@ -0,0 +1,107 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+import os
+import time
+from contextlib import contextmanager
+
+from vllm.platforms import current_platform
+
+if current_platform.is_rocm():
+    from amdsmi import (
+        amdsmi_get_gpu_vram_usage,
+        amdsmi_get_processor_handles,
+        amdsmi_init,
+        amdsmi_shut_down,
+    )
+
+    @contextmanager
+    def _nvml():
+        try:
+            amdsmi_init()
+            yield
+        finally:
+            amdsmi_shut_down()
+elif current_platform.is_cuda():
+    from vllm.third_party.pynvml import (
+        nvmlDeviceGetHandleByIndex,
+        nvmlDeviceGetMemoryInfo,
+        nvmlInit,
+        nvmlShutdown,
+    )
+
+    @contextmanager
+    def _nvml():
+        try:
+            nvmlInit()
+            yield
+        finally:
+            nvmlShutdown()
+else:
+
+    @contextmanager
+    def _nvml():
+        yield
+
+
+def get_physical_device_indices(devices):
+    visible_devices = os.environ.get("CUDA_VISIBLE_DEVICES")
+    if visible_devices is None:
+        return devices
+
+    visible_indices = [int(x) for x in visible_devices.split(",")]
+    index_mapping = {i: physical for i, physical in enumerate(visible_indices)}
+    return [index_mapping[i] for i in devices if i in index_mapping]
+
+
+@_nvml()
+def wait_for_gpu_memory_to_clear(
+    *,
+    devices: list[int],
+    threshold_bytes: int | None = None,
+    threshold_ratio: float | None = None,
+    timeout_s: float = 120,
+) -> None:
+    assert threshold_bytes is not None or threshold_ratio is not None
+    # Use nvml instead of pytorch to reduce measurement error from torch cuda
+    # context.
+    devices = get_physical_device_indices(devices)
+    start_time = time.time()
+    while True:
+        output: dict[int, str] = {}
+        output_raw: dict[int, tuple[float, float]] = {}
+        for device in devices:
+            if current_platform.is_rocm():
+                dev_handle = amdsmi_get_processor_handles()[device]
+                mem_info = amdsmi_get_gpu_vram_usage(dev_handle)
+                gb_used = mem_info["vram_used"] / 2**10
+                gb_total = mem_info["vram_total"] / 2**10
+            else:
+                dev_handle = nvmlDeviceGetHandleByIndex(device)
+                mem_info = nvmlDeviceGetMemoryInfo(dev_handle)
+                gb_used = mem_info.used / 2**30
+                gb_total = mem_info.total / 2**30
+            output_raw[device] = (gb_used, gb_total)
+            output[device] = f"{gb_used:.02f}/{gb_total:.02f}"
+
+        print("gpu memory used/total (GiB): ", end="")
+        for k, v in output.items():
+            print(f"{k}={v}; ", end="")
+        print("")
+
+        if threshold_bytes is not None:
+            is_free = lambda used, total: used <= threshold_bytes / 2**30  # noqa E731
+            threshold = f"{threshold_bytes / 2**30} GiB"
+        else:
+            is_free = lambda used, total: used / total <= threshold_ratio  # noqa E731
+            threshold = f"{threshold_ratio:.2f}"
+
+        dur_s = time.time() - start_time
+        if all(is_free(used, total) for used, total in output_raw.values()):
+            print(f"Done waiting for free GPU memory on devices {devices=} ({threshold=}) {dur_s=:.02f}")
+            break
+
+        if dur_s >= timeout_s:
+            raise ValueError(f"Memory of devices {devices=} not free after {dur_s=:.02f} ({threshold=})")
+
+        time.sleep(5)