update

Signed-off-by: Roger Wang <hey@rogerw.io>

.buildkite/pipeline.yml

@@ -1,23 +1,6 @@
 steps:
-  - label: ":docker: Build image"
-    key: image-build
+  - label: ":toolbox: Qwen2.5 omni offline example"
     commands:
-      - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7"
-      - "docker build --file docker/Dockerfile.ci -t vllm-omni-ci ."
-      - "docker tag vllm-omni-ci public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:omni-latest"
-      - "docker push public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:omni-latest"
+      - ".buildkite/scripts/run_qwen_offline_example.sh"
     agents:
-      queue: "cpu_queue_premerge_us_east_1"
-
-  - label: "Simple Unit Test"
-    depends_on: image-build
-    commands:
-      - ".buildkite/scripts/simple_test.sh"
-    agents:
-      queue: "cpu_queue_premerge"
-    plugins:
-      - docker#v5.2.0:
-          image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:omni-latest
-          always-pull: true
-          propagate-environment: true
-
+      queue: "gpu_1_queue" # g6.4xlarge instance on AWS, has 1 L4 GPU

.buildkite/scripts/run_qwen_offline_example.sh

@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+# Move to repo root (script lives in .buildkite/scripts/)
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+cd "${ROOT_DIR}"
+
+UV_INSTALL_DIR="${ROOT_DIR}/.local/bin"
+case ":${PATH}:" in
+  *":${UV_INSTALL_DIR}:"*) ;;
+  *) export PATH="${UV_INSTALL_DIR}:${PATH}" ;;
+esac
+
+if ! command -v uv >/dev/null 2>&1; then
+  mkdir -p "${UV_INSTALL_DIR}"
+  curl -LsSf https://astral.sh/uv/install.sh | env UV_INSTALL_DIR="${UV_INSTALL_DIR}" sh
+fi
+
+UV_BIN=(uv)
+
+"${UV_BIN[@]}" python install 3.12
+"${UV_BIN[@]}" venv --python 3.12 .venv
+source .venv/bin/activate
+
+"${UV_BIN[@]}" pip install -e .
+"${UV_BIN[@]}" pip install vllm==0.11.0 --torch-backend=auto
+
+EXAMPLE_DIR="examples/offline_inference/qwen2_5_omni"
+cd "${EXAMPLE_DIR}"
+
+python end2end.py --query-type text \
+                  --no-save-results

.buildkite/scripts/simple_test.sh

@@ -1,52 +0,0 @@
-#!/usr/bin/env bash
-
-set -euo pipefail
-
-SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
-REPO_ROOT="$(cd -- "${SCRIPT_DIR}/../.." && pwd)"
-UV_BIN="uv"
-PYTHON_BOOTSTRAP="${PYTHON:-python3}"
-VENV_DIR="${VENV_DIR:-${REPO_ROOT}/.venv-simple-test}"
-TARGET_PY_VERSION="3.12"
-
-if ! command -v "${PYTHON_BOOTSTRAP}" >/dev/null 2>&1; then
-  PYTHON_BOOTSTRAP="python"
-fi
-
-cd "${REPO_ROOT}"
-
-# Ensure pip-installed scripts (including uv) are on PATH
-PYTHON_SCRIPTS="$(${PYTHON_BOOTSTRAP} -c 'import sysconfig; print(sysconfig.get_path("scripts"))')"
-USER_BASE="$(${PYTHON_BOOTSTRAP} -m site --user-base 2>/dev/null || true)"
-PATH="${PYTHON_SCRIPTS}:${PATH}"
-if [[ -n "${USER_BASE}" ]]; then
-  PATH="${USER_BASE}/bin:${PATH}"
-fi
-
-if ! command -v "${UV_BIN}" >/dev/null 2>&1; then
-  "${PYTHON_BOOTSTRAP}" -m pip install --upgrade pip
-  "${PYTHON_BOOTSTRAP}" -m pip install uv
-  hash -r
-fi
-
-TARGET_PY_PATH="$(${UV_BIN} python find "${TARGET_PY_VERSION}" 2>/dev/null | head -n1 || true)"
-if [[ -z "${TARGET_PY_PATH}" ]]; then
-  "${UV_BIN}" python install "${TARGET_PY_VERSION}"
-  TARGET_PY_PATH="$(${UV_BIN} python find "${TARGET_PY_VERSION}" 2>/dev/null | head -n1 || true)"
-fi
-
-if [[ -z "${TARGET_PY_PATH}" ]]; then
-  echo "Failed to locate python ${TARGET_PY_VERSION}" >&2
-  exit 1
-fi
-
-if [[ ! -d "${VENV_DIR}" ]]; then
-  "${UV_BIN}" venv --python "${TARGET_PY_PATH}" "${VENV_DIR}"
-fi
-
-VENV_PYTHON="${VENV_DIR}/bin/python"
-[[ -x "${VENV_PYTHON}" ]] || { echo "Python not found in ${VENV_DIR}"; exit 1; }
-
-"${UV_BIN}" pip install --python "${VENV_PYTHON}" vllm==0.11.0
-"${UV_BIN}" pip install --python "${VENV_PYTHON}" -e ".[dev]"
-"${VENV_PYTHON}" -m pytest -v -s tests/test_omni_llm.py

.github/ISSUE_TEMPLATE/config.yml

@@ -1,5 +1,5 @@
 blank_issues_enabled: false
 contact_links:
   - name: Questions
-    url: https://slack.vllm.ai
-    about: Ask questions and discuss with other vLLM-Omni community members in our #sig-omni slack channel.
+    url: https://github.com/vllm-project/vllm-omni/discussions
+    about: Ask questions and discuss with other vLLM-omni community members

.gitignore

@@ -156,9 +156,15 @@ cython_debug/
 # cursor
 .cursor/
 
+# docker
+docker/
+
 # scripts
 /scripts/
 
+# tests
+tests/
+
 # PyCharm
 #  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
 #  be added to the global gitignore or merged into this project gitignore.  For a PyCharm
@@ -169,7 +175,7 @@ cython_debug/
 #  *.iws
 #  .idea/
 
-# vLLM-Omni specific
+# vLLM-omni specific
 # Model files and checkpoints
 checkpoints/
 *.bin

CONTRIBUTING.md

@@ -1,3 +1,19 @@
-# Contributing to vLLM-Omni
+# Contributing to vLLM
 
-You may find information about contributing to vLLM-Omni on [Contributing](https://vllm-omni.readthedocs.io/en/latest/contributing/)
+## Getting Started
+
+Setup environment with uv
+```bash
+uv venv --python 3.12 --seed
+source .venv/bin/activate
+```
+
+Pre-commit has been linked to git commit. For run checks locally, please install pre-commit:
+```bash
+uv pip install pre-commit
+```
+
+Then run:
+```bash
+pre-commit run --show-diff-on-failure --color=always --all-files
+```

README.md

@@ -1,7 +1,7 @@
 <p align="center">
   <picture>
-    <source media="(prefers-color-scheme: dark)" src="https://raw.githubusercontent.com/vllm-project/vllm-omni/refs/heads/main/docs/source/logos/vllm-omni-logo.png">
-    <img alt="vllm-omni" src="https://raw.githubusercontent.com/vllm-project/vllm-omni/refs/heads/main/docs/source/logos/vllm-omni-logo.png" width=55%>
+    <source media="(prefers-color-scheme: dark)" src="./docs/source/logos/vllm-omni-logo-text-dark.png">
+    <img alt="vllm-omni" src="./docs/source/logos/vllm-omni-logo-text-dark.png" width=55%>
   </picture>
 </p>
 <h3 align="center">
@@ -9,7 +9,7 @@ Easy, fast, and cheap omni-modality model serving for everyone
 </h3>
 
 <p align="center">
-| <a href="https://vllm-omni.readthedocs.io/en/latest/"><b>Documentation</b></a> | <a href="https://discuss.vllm.ai"><b>User Forum</b></a> | <a href="https://slack.vllm.ai"><b>Developer Slack</b></a> |
+| <a href="https://vllm-omni.readthedocs.io/en/latest/"><b>Documentation</b></a> | <a href="https://tinyurl.com/vllm-omni-meeting"><b>Weekly Meeting</b></a> |
 </p>
 
 ---
@@ -22,7 +22,7 @@ Easy, fast, and cheap omni-modality model serving for everyone
 
 ## About
 
-[vLLM](https://github.com/vllm-project/vllm) was originally designed to support large language models for text-based autoregressive generation tasks. vLLM-Omni is a framework that extends its support for omni-modality model inference and serving:
+[vLLM](https://github.com/vllm-project/vllm) was originally designed to support large language models for text-based autoregressive generation tasks. vLLM-Omni extends its support for omni-modality model inference and serving:
 
 - **Omni-modality**: Text, image, video, and audio data processing
 - **Non-autoregressive Architectures**: extend the AR support of vLLM to Diffusion Transformers (DiT) and other parallel generation models
@@ -30,44 +30,107 @@ Easy, fast, and cheap omni-modality model serving for everyone
 
 <p align="center">
   <picture>
-    <img alt="vllm-omni" src="https://raw.githubusercontent.com/vllm-project/vllm-omni/refs/heads/main/docs/source/architecture/omni-modality-model-architecture.png" width=55%>
+    <img alt="vllm-omni" src="./docs/source/architecture/omni-modality model architecture.png" width=55%>
   </picture>
 </p>
 
-vLLM-Omni is fast with:
+vLLM-omni is fast with:
 
-- State-of-the-art AR support by leveraging efficient KV cache management from vLLM
-- Pipelined stage execution overlapping for high throughput performance
-- Fully disaggregation based on OmniConnector and dynamic resource allocation across stages
+- Seamless AR support for efficient KV Cache management with vLLM
+- Pipelined stage execution overlapping
+- Fully disaggregation based on omniConnector and dynamic resource allocation across stages
 
-vLLM-Omni is flexible and easy to use with:
+vLLM-omni is flexible and easy to use with:
 
-- Heterogeneous pipeline abstraction to manage complex model workflows
 - Seamless integration with popular Hugging Face models
+- High-throughput serving with various decoding algorithms, including *parallel sampling*, *beam search*, and more
 - Tensor, pipeline, data and expert parallelism support for distributed inference
 - Streaming outputs
 - OpenAI-compatible API server
 
-vLLM-Omni seamlessly supports most popular open-source models on HuggingFace, including:
+vLLM seamlessly supports most popular open-source models on HuggingFace, including:
 
-- Omni-modality models (e.g. Qwen-Omni)
-- Multi-modality generation models (e.g. Qwen-Image)
+- Multi-modality generation models (e.g. Qwen-image)
+- Omni-modality models (e.g. Qwen-omni)
 
 ## Getting Started
 
-Visit our [documentation](https://vllm-omni.readthedocs.io/en/latest/) to learn more.
+Install vLLM with `pip` or [from source](https://docs.vllm.ai/en/latest/getting_started/installation/gpu/index.html#build-wheel-from-source):
 
-- [Installation](https://vllm-omni.readthedocs.io/en/latest/getting_started/installation/)
-- [Quickstart](https://vllm-omni.readthedocs.io/en/latest/getting_started/quickstart/)
-- [List of Supported Models](https://vllm-omni.readthedocs.io/en/latest/models/supported_models/)
+```bash
+pip install vllm
+```
+
+## 🛠️ Installation
+
+### Environment setup
+
+Use Docker to keep consistent basic environment (Optional, Recommended)
+```bash
+docker run --gpus all --ipc=host --network=host -v $source_dir:$container_dir --name $container_name -it nvcr.io/nvidia/pytorch:25.01-py3 bash
+```
+
+Set up basic uv environment
+```bash
+pip install uv
+uv venv --python 3.12 --seed
+source .venv/bin/activate
+```
+
+### Installation of vLLM (for users)
+
+Now we build it based on vLLM v0.11.0. Please install it with command below.
+```bash
+uv pip install vllm==0.11.0 --torch-backend=auto
+```
+
+### Installation of vLLM (for developers)
+
+If you want to check or debug with source code of vLLM, install stable release version of vllm with 0.11.0 from source with pre-built wheel file.
+
+```bash
+git clone https://github.com/vllm-project/vllm.git
+cd vllm
+git checkout v0.11.0
+```
+Set up environment variables to get pre-built wheels. If there are internet problems, just download the whl file manually. And set VLLM_PRECOMPILED_WHEEL_LOCATION as your local absolute path of whl file.
+```bash
+export VLLM_PRECOMPILED_WHEEL_LOCATION=https://github.com/vllm-project/vllm/releases/download/v0.11.0/vllm-0.11.0-cp38-abi3-manylinux1_x86_64.whl
+```
+Install vllm with command below (If you have no existing PyTorch).
+```bash
+uv pip install --editable .
+```
+
+Install vllm with command below (If you already have PyTorch).
+```bash
+python use_existing_torch.py
+uv pip install -r requirements/build.txt
+uv pip install --no-build-isolation --editable .
+```
+
+### Installation of vLLM-omni
+Install additional requirements for vllm-omni
+```bash
+git clone https://github.com/vllm-project/vllm-omni.git
+cd vllm_omni
+uv pip install -e .
+```
+
+
+## Run examples (Qwen2.5-omni)
+
+Please check the folder of [examples](examples)
 
 ## Contributing
 
 We welcome and value any contributions and collaborations.
-Please check out [Contributing to vLLM-Omni](https://vllm-omni.readthedocs.io/en/latest/contributing/) for how to get involved.
+Please check out [Contributing to vLLM-omni](https://docs.vllm.ai/en/latest/contributing/index.html) for how to get involved.
 
-## Join the Community
-Feel free to ask questions, provide feedbacks and discuss with fellow users of vLLM-Omni in `#sig-omni` slack channel at [slack.vllm.ai](https://slack.vllm.ai) or vLLM user forum at [discuss.vllm.ai](https://discuss.vllm.ai).
+## Weekly Meeting
+
+- vLLM Omni Weekly Meeting: https://tinyurl.com/vllm-omni-meeting
+- Wednesday, 11:30 - 12:30 (UTC+8, [Convert to your timezone](https://dateful.com/convert/gmt8?t=15))
 
 ## License
 

docker/Dockerfile.ci

@@ -1,14 +0,0 @@
-ARG VLLM_BASE_IMAGE=vllm/vllm-openai
-ARG VLLM_BASE_TAG=v0.11.0
-FROM ${VLLM_BASE_IMAGE}:${VLLM_BASE_TAG}
-ARG APP_DIR=/workspace/vllm-omni
-WORKDIR ${APP_DIR}
-
-COPY . .
-
-# Install vllm-omni into the same uv-managed Python environment used by the base image.
-RUN uv pip install --python "$(python3 -c 'import sys; print(sys.executable)')" --no-cache-dir ".[dev]"
-
-RUN ln -sf /usr/bin/python3 /usr/bin/python
-
-ENTRYPOINT []

docs/.nav.yml

@@ -1,45 +0,0 @@
-nav:
-- Home: README.md
-- User Guide:
-  - Getting Started:
-    - getting_started/quickstart.md
-    - getting_started/installation
-  - Examples:
-    - examples/README.md
-    - Offline Inference:
-      - Offline Example of vLLM-Omni for Qwen2.5-Omni: user_guide/examples/offline_inference/qwen2_5_omni.md
-      - Offline Example of vLLM-Omni for Qwen3-Omni: user_guide/examples/offline_inference/qwen3_omni.md
-      - Qwen-Image Offline Inference: user_guide/examples/offline_inference/qwen_image.md
-    - Online Serving:
-      - Online serving Example of vLLM-Omni for Qwen2.5-Omni: user_guide/examples/online_serving/qwen2_5_omni.md
-      - Online serving Example of vLLM-Omni for Qwen3-omni: user_guide/examples/online_serving/qwen3_omni.md
-  - General:
-    - usage/*
-  - Configuration:
-    - configuration/README.md
-    - configuration/*
-  - Models:
-    - models/supported_models.md
-- Developer Guide:
-  - General:
-    - contributing/README.md
-    - glob: contributing/*
-      flatten_single_child_sections: true
-  - Model Implementation:
-    - contributing/model/README.md
-  - CI: contributing/ci
-  - Design Documents:
-    - design/index.md
-    - design/architecture_overview.md
-    - design/vllm_omni_design.md
-    - design/mrs_design.md
-  - Docs Guide: contributing/DOCS_GUIDE.md
-- API Reference:
-  - api/README.md
-  - api/vllm_omni
-- CLI Reference: cli
-- Community:
-  - community/*
-  - Slack: https://slack.vllm.ai
-  - Blog: https://blog.vllm.ai
-  - Forum: https://discuss.vllm.ai

docs/DOCS_GUIDE.md

@@ -1,6 +1,6 @@
 # Documentation Build Guide
 
-This directory contains the source files for the vLLM-Omni documentation.
+This directory contains the source files for the vLLM-omni documentation.
 
 ## Building Documentation Locally
 
@@ -9,7 +9,7 @@ This directory contains the source files for the vLLM-Omni documentation.
 Install documentation dependencies:
 
 ```bash
-uv pip install -e ".[docs]"
+pip install -e ".[docs]"
 ```
 
 ### Build and Serve Documentation
@@ -38,8 +38,8 @@ The documentation automatically extracts docstrings from the code using mkdocstr
 Example docstring:
 
 ```python
-class Omni:
-    """Main entry point for vLLM-Omni inference.
+class OmniLLM:
+    """Main entry point for vLLM-omni inference.
 
     This class provides a high-level interface for running multi-modal
     inference with non-autoregressive models.
@@ -50,7 +50,7 @@ class Omni:
         **kwargs: Additional arguments passed to the engine
 
     Example:
-        >>> llm = Omni(model="Qwen/Qwen2.5-Omni")
+        >>> llm = OmniLLM(model="Qwen/Qwen2.5-Omni")
         >>> outputs = llm.generate(prompts="Hello")
     """
 ```
@@ -84,7 +84,7 @@ The documentation is automatically deployed to GitHub Pages using GitHub Actions
    ```
 
 3. **Documentation will be available at**:
-   - `https://vllm-omni.readthedocs.io`
+   - `https://vllm-project.github.io/vllm-omni/`
 
 The GitHub Actions workflow (`.github/workflows/docs.yml`) will automatically:
 - Build the documentation when you push to `main` branch

docs/README.md

@@ -1,62 +0,0 @@
----
-hide:
-  - navigation
-  - toc
----
-
-# Welcome to vLLM-Omni
-
-<p align="center">
-  <picture>
-    <source media="(prefers-color-scheme: dark)" src="./source/logos/vllm-omni-logo.png">
-    <img alt="vllm-omni" src="./source/logos/vllm-omni-logo.png" width=55%>
-  </picture>
-</p>
-<h3 align="center">
-Easy, fast, and cheap omni-modality model serving for everyone
-</h3>
-
-<p style="text-align:center">
-<script async defer src="https://buttons.github.io/buttons.js"></script>
-<a class="github-button" href="https://github.com/vllm-project/vllm-omni" data-show-count="true" data-size="large" aria-label="Star">Star</a>
-<a class="github-button" href="https://github.com/vllm-project/vllm-omni/subscription" data-show-count="true" data-icon="octicon-eye" data-size="large" aria-label="Watch">Watch</a>
-<a class="github-button" href="https://github.com/vllm-project/vllm-omni/fork" data-show-count="true" data-icon="octicon-repo-forked" data-size="large" aria-label="Fork">Fork</a>
-</p>
-
-
-## About
-
-[vLLM](https://github.com/vllm-project/vllm) was originally designed to support large language models for text-based autoregressive generation tasks. vLLM-Omni is a framework that extends its support for omni-modality model inference and serving:
-
-- **Omni-modality**: Text, image, video, and audio data processing
-- **Non-autoregressive Architectures**: extend the AR support of vLLM to Diffusion Transformers (DiT) and other parallel generation models
-- **Heterogeneous outputs**: from traditional text generation to multimodal outputs
-
-<p align="center">
-  <picture>
-    <source media="(prefers-color-scheme: dark)" src="./source/architecture/omni-modality-model-architecture.png">
-    <img alt="vllm-omni-arch" src="./source/architecture/omni-modality-model-architecture.png" width=55%>
-  </picture>
-</p>
-
-vLLM-Omni is fast with:
-
-- State-of-the-art AR support by leveraging efficient KV cache management from vLLM
-- Pipelined stage execution overlapping for high throughput performance
-- Fully disaggregation based on OmniConnector and dynamic resource allocation across stages
-
-vLLM-Omni is flexible and easy to use with:
-
-- Heterogeneous pipeline abstraction to manage complex model workflows
-- Seamless integration with popular Hugging Face models
-- Tensor, pipeline, data and expert parallelism support for distributed inference
-- Streaming outputs
-- OpenAI-compatible API server
-
-vLLM-Omni seamlessly supports most popular open-source models on HuggingFace, including:
-
-- Omni-modality models (e.g. Qwen2.5-Omni, Qwen3-Omni)
-- Multi-modality generation models (e.g. Qwen-Image)
-
-For more information, checkout the following:
-- [vllm-omni announcement blogpost](https://blog.vllm.ai/2025/11/30/vllm-omni.html)

docs/api/README.md

@@ -1,120 +0,0 @@
-# Summary
-
-## Entry Points
-
-Main entry points for vLLM-Omni inference and serving.
-
-- [vllm_omni.entrypoints.async_omni.AsyncOmni][]
-- [vllm_omni.entrypoints.async_omni.AsyncOmniStageLLM][]
-- [vllm_omni.entrypoints.chat_utils.OmniAsyncMultiModalContentParser][]
-- [vllm_omni.entrypoints.chat_utils.OmniAsyncMultiModalItemTracker][]
-- [vllm_omni.entrypoints.chat_utils.parse_chat_messages_futures][]
-- [vllm_omni.entrypoints.cli.serve.OmniServeCommand][]
-- [vllm_omni.entrypoints.log_utils.OrchestratorMetrics][]
-- [vllm_omni.entrypoints.omni.Omni][]
-- [vllm_omni.entrypoints.omni_llm.OmniLLM][]
-- [vllm_omni.entrypoints.omni_llm.OmniStageLLM][]
-- [vllm_omni.entrypoints.omni_stage.OmniStage][]
-- [vllm_omni.entrypoints.openai.serving_chat.OmniOpenAIServingChat][]
-
-## Inputs
-
-Input data structures for multi-modal inputs.
-
-- [vllm_omni.inputs.data.OmniEmbedsPrompt][]
-- [vllm_omni.inputs.data.OmniTokenInputs][]
-- [vllm_omni.inputs.data.OmniTokensPrompt][]
-- [vllm_omni.inputs.parse.parse_singleton_prompt_omni][]
-- [vllm_omni.inputs.preprocess.OmniInputPreprocessor][]
-
-## Engine
-
-Engine classes for offline and online inference.
-
-- [vllm_omni.diffusion.diffusion_engine.DiffusionEngine][]
-- [vllm_omni.engine.AdditionalInformationEntry][]
-- [vllm_omni.engine.AdditionalInformationPayload][]
-- [vllm_omni.engine.OmniEngineCoreOutput][]
-- [vllm_omni.engine.OmniEngineCoreOutputs][]
-- [vllm_omni.engine.OmniEngineCoreRequest][]
-- [vllm_omni.engine.PromptEmbedsPayload][]
-- [vllm_omni.engine.arg_utils.AsyncOmniEngineArgs][]
-- [vllm_omni.engine.arg_utils.OmniEngineArgs][]
-- [vllm_omni.engine.output_processor.MultimodalOutputProcessor][]
-- [vllm_omni.engine.output_processor.OmniRequestState][]
-- [vllm_omni.engine.processor.OmniProcessor][]
-
-## Core
-
-Core scheduling and caching components.
-
-- [vllm_omni.core.dit_cache_manager.DiTCacheManager][]
-- [vllm_omni.core.sched.diffusion_scheduler.DiffusionScheduler][]
-- [vllm_omni.core.sched.generation_scheduler.GenerationScheduler][]
-- [vllm_omni.core.sched.output.OmniNewRequestData][]
-- [vllm_omni.core.sched.scheduler.OmniScheduler][]
-
-## Model Executor
-
-Model execution components.
-
-- [vllm_omni.model_executor.models.output_templates.OmniOutput][]
-- [vllm_omni.model_executor.models.qwen2_5_omni.qwen2_5_omni.Qwen2_5OmniForConditionalGeneration][]
-- [vllm_omni.model_executor.models.qwen2_5_omni.qwen2_5_omni_talker.Qwen2_5OmniTalkerForConditionalGeneration][]
-- [vllm_omni.model_executor.models.qwen2_5_omni.qwen2_5_omni_thinker.Qwen2_5OmniAudioFeatureInputs][]
-- [vllm_omni.model_executor.models.qwen2_5_omni.qwen2_5_omni_thinker.Qwen2_5OmniConditionalGenerationMixin][]
-- [vllm_omni.model_executor.models.qwen2_5_omni.qwen2_5_omni_thinker.Qwen2_5OmniThinkerDummyInputsBuilder][]
-- [vllm_omni.model_executor.models.qwen2_5_omni.qwen2_5_omni_thinker.Qwen2_5OmniThinkerForConditionalGeneration][]
-- [vllm_omni.model_executor.models.qwen2_5_omni.qwen2_5_omni_thinker.Qwen2_5OmniThinkerMultiModalDataParser][]
-- [vllm_omni.model_executor.models.qwen2_5_omni.qwen2_5_omni_thinker.Qwen2_5OmniThinkerMultiModalProcessor][]
-- [vllm_omni.model_executor.models.qwen2_5_omni.qwen2_5_omni_thinker.Qwen2_5OmniThinkerProcessingInfo][]
-- [vllm_omni.model_executor.models.qwen2_5_omni.qwen2_5_omni_token2wav.Qwen2_5OmniToken2WavBigVGANModel][]
-- [vllm_omni.model_executor.models.qwen2_5_omni.qwen2_5_omni_token2wav.Qwen2_5OmniToken2WavDiTModel][]
-- [vllm_omni.model_executor.models.qwen2_5_omni.qwen2_5_omni_token2wav.Qwen2_5OmniToken2WavForConditionalGenerationVLLM][]
-- [vllm_omni.model_executor.models.qwen2_5_omni.qwen2_5_omni_token2wav.Qwen2_5OmniToken2WavModel][]
-- [vllm_omni.model_executor.models.qwen2_5_omni.qwen2_old.Qwen2EmbeddingModel][]
-- [vllm_omni.model_executor.models.qwen2_5_omni.qwen2_old.Qwen2ForCausalLM][]
-- [vllm_omni.model_executor.models.qwen2_5_omni.qwen2_old.Qwen2Model][]
-- [vllm_omni.model_executor.models.qwen3_omni.qwen3_moe.Qwen3MoeForCausalLM][]
-- [vllm_omni.model_executor.models.qwen3_omni.qwen3_omni.Qwen3OmniMoeForConditionalGeneration][]
-- [vllm_omni.model_executor.models.qwen3_omni.qwen3_omni_code2wav.Qwen3OmniMoeCode2Wav][]
-- [vllm_omni.model_executor.models.qwen3_omni.qwen3_omni_moe_code_predictor_mtp.Qwen3OmniCodePredictorBaseModel][]
-- [vllm_omni.model_executor.models.qwen3_omni.qwen3_omni_moe_code_predictor_mtp.Qwen3OmniMoeTalkerCodePredictor][]
-- [vllm_omni.model_executor.models.qwen3_omni.qwen3_omni_moe_talker.Qwen3OmniMoeModel][]
-- [vllm_omni.model_executor.models.qwen3_omni.qwen3_omni_moe_talker.Qwen3OmniMoeTalkerForConditionalGeneration][]
-- [vllm_omni.model_executor.models.qwen3_omni.qwen3_omni_moe_thinker.Qwen3MoeLLMForCausalLM][]
-- [vllm_omni.model_executor.models.qwen3_omni.qwen3_omni_moe_thinker.Qwen3MoeLLMModel][]
-- [vllm_omni.model_executor.models.qwen3_omni.qwen3_omni_moe_thinker.Qwen3OmniMoeConditionalGenerationMixin][]
-- [vllm_omni.model_executor.models.qwen3_omni.qwen3_omni_moe_thinker.Qwen3OmniMoeThinkerForConditionalGeneration][]
-- [vllm_omni.model_executor.models.qwen3_omni.qwen3_omni_moe_thinker.Qwen3OmniMoeThinkerMultiModalProcessor][]
-- [vllm_omni.model_executor.models.qwen3_omni.qwen3_omni_moe_thinker.Qwen3OmniMoeThinkerProcessingInfo][]
-- [vllm_omni.model_executor.models.qwen3_omni.qwen3_omni_moe_thinker.Qwen3Omni_VisionTransformer][]
-- [vllm_omni.model_executor.models.qwen3_omni.qwen3_omni_moe_thinker.Qwen3_VisionPatchEmbed][]
-- [vllm_omni.model_executor.models.qwen3_omni.qwen3_omni_moe_thinker.Qwen3_VisionPatchMerger][]
-
-## Configuration
-
-Configuration classes.
-
-- [vllm_omni.config.model.OmniModelConfig][]
-
-## Workers
-
-Worker classes and model runners for distributed inference.
-
-- [vllm_omni.diffusion.worker.gpu_worker.GPUWorker][]
-- [vllm_omni.diffusion.worker.gpu_worker.WorkerProc][]
-- [vllm_omni.diffusion.worker.npu.npu_worker.NPUWorker][]
-- [vllm_omni.diffusion.worker.npu.npu_worker.NPUWorkerProc][]
-- [vllm_omni.worker.gpu_ar_model_runner.GPUARModelRunner][]
-- [vllm_omni.worker.gpu_ar_worker.GPUARWorker][]
-- [vllm_omni.worker.gpu_diffusion_model_runner.GPUDiffusionModelRunner][]
-- [vllm_omni.worker.gpu_diffusion_worker.GPUDiffusionWorker][]
-- [vllm_omni.worker.gpu_generation_model_runner.GPUGenerationModelRunner][]
-- [vllm_omni.worker.gpu_generation_worker.GPUGenerationWorker][]
-- [vllm_omni.worker.gpu_model_runner.OmniGPUModelRunner][]
-- [vllm_omni.worker.npu.npu_ar_model_runner.NPUARModelRunner][]
-- [vllm_omni.worker.npu.npu_ar_worker.NPUARWorker][]
-- [vllm_omni.worker.npu.npu_diffusion_model_runner.NPUDiffusionModelRunner][]
-- [vllm_omni.worker.npu.npu_diffusion_worker.NPUDiffusionWorker][]
-- [vllm_omni.worker.npu.npu_model_runner.OmniNPUModelRunner][]

docs/api/overview.md

@@ -0,0 +1,68 @@
+# Summary
+
+## Entry Points
+
+Main entry points for vLLM-omni inference and serving.
+
+- [vllm_omni.entrypoints.omni_llm.OmniLLM][]
+- [vllm_omni.entrypoints.omni_llm.OmniStageLLM][]
+- [vllm_omni.entrypoints.omni_stage.OmniStage][]
+- [vllm_omni.entrypoints.cli.serve.OmniServeCommand][]
+
+## Inputs
+
+Input data structures for multi-modal inputs.
+
+- [vllm_omni.inputs.data.OmniTokenInputs][]
+- [vllm_omni.inputs.data.OmniTokensPrompt][]
+- [vllm_omni.inputs.data.OmniEmbedsPrompt][]
+- [vllm_omni.inputs.parse.parse_singleton_prompt_omni][]
+- [vllm_omni.inputs.preprocess.OmniInputPreprocessor][]
+
+## Engine
+
+Engine classes for offline and online inference.
+
+- [vllm_omni.engine.arg_utils.OmniEngineArgs][]
+- [vllm_omni.engine.processor.OmniProcessor][]
+- [vllm_omni.engine.output_processor.OmniRequestState][]
+- [vllm_omni.engine.output_processor.MultimodalOutputProcessor][]
+- [vllm_omni.engine.PromptEmbedsPayload][]
+- [vllm_omni.engine.AdditionalInformationPayload][]
+- [vllm_omni.engine.OmniEngineCoreRequest][]
+
+## Core
+
+Core scheduling and caching components.
+
+- [vllm_omni.core.sched.scheduler.OmniScheduler][]
+- [vllm_omni.core.sched.diffusion_scheduler.DiffusionScheduler][]
+- [vllm_omni.core.sched.output.OmniNewRequestData][]
+- [vllm_omni.core.dit_cache_manager.DiTCacheManager][]
+
+## Model Executor
+
+Model execution components.
+
+- [vllm_omni.model_executor.models.qwen2_5_omni.Qwen2_5OmniForConditionalGeneration][]
+- [vllm_omni.model_executor.models.qwen2_5_omni_talker.Qwen2_5OmniTalkerForConditionalGeneration][]
+- [vllm_omni.model_executor.models.qwen2_5_omni_thinker.Qwen2_5OmniThinkerForConditionalGeneration][]
+- [vllm_omni.model_executor.models.qwen2_5_omni_token2wav.Qwen2_5OmniToken2WavForConditionalGenerationVLLM][]
+- [vllm_omni.model_executor.layers.mrope.MRotaryEmbedding][]
+- [vllm_omni.model_executor.models.registry.OmniModelRegistry][]
+
+## Configuration
+
+Configuration classes.
+
+- [vllm_omni.config.OmniModelConfig][]
+
+## Workers
+
+Worker classes and model runners for distributed inference.
+
+- [vllm_omni.worker.gpu_ar_worker.GPUARWorker][]
+- [vllm_omni.worker.gpu_diffusion_worker.GPUDiffusionWorker][]
+- [vllm_omni.worker.gpu_model_runner.OmniGPUModelRunner][]
+- [vllm_omni.worker.gpu_ar_model_runner.GPUARModelRunner][]
+- [vllm_omni.worker.gpu_diffusion_model_runner.GPUDiffusionModelRunner][]

docs/cli/README.md

@@ -1,24 +0,0 @@
-# vLLM-Omni CLI Guide
-
-The CLI for vLLM-Omni inherits from vllm with some additional arguments.
-
-## serve
-
-Starts the vLLM-Omni OpenAI Compatible API server.
-
-Start with a model:
-
-```bash
-vllm serve Qwen/Qwen2.5-Omni-7B --omni
-```
-
-Specify the port:
-
-```bash
-vllm serve Qwen/Qwen2.5-Omni-7B --omni --port 8091
-```
-
-If you have custom stage configs file, launch the server with command below
-```bash
-vllm serve Qwen/Qwen2.5-Omni-7B --omni --stage-configs-path /path/to/stage_configs_file
-```

docs/community/contact_us.md

@@ -1,5 +0,0 @@
-# Contact Us
-
-- For technical questions and feature requests, please use GitHub [Issues](https://github.com/vllm-project/vllm-omni/issues)
-- For coordinating contributions and development and discussing with other users and developers, please join `sig-omni` channel in our [Slack](https://slack.vllm.ai/) or use the [vLLM Forum](https://discuss.vllm.ai/)
-- For security disclosures, please use GitHub's [Security Advisories](https://github.com/vllm-project/vllm-omni/security/advisories) feature

docs/community/meetups.md

@@ -1 +0,0 @@
-# Meetups

docs/community/volunteers.md

@@ -1,12 +0,0 @@
-# Volunteers for Bugfix and CI
-
-We encourage you to check current docs and [issues](https://github.com/vllm-project/vllm-omni/issues) to find possible solutions for your questions. If non of these can solve it, please propose an issue to describe your questions about bug or CI problems for developing.
-
-If you have urgent need for locating and solving bugfix or CI problems, please find community volunteers below.
-
-| Dec 4-Dec 12 | Dec 15-Dec 19 | Dec 22-Dec 26 | Dec 29- Jan 2, 2026| Jan 5-Jan 9 | Jan 12-Jan 16 |
-|----------|----------|----------|----------|----------|----------|
-| <a href="https://github.com/congw729">Conw729</a> | <a href="https://github.com/yinpeiqi">yinpeiqi</a> | <a href="https://github.com/tzhouam">tzhouam</a> | <a href="https://github.com/SamitHuang">SamitHuang</a> | <a href="https://github.com/gcanlin">gcanlin</a> | <a href="https://github.com/natureofnature">natureofnature</a> |
-| <a href="https://github.com/david6666666">david6666666</a> | <a href="https://github.com/R2-Y">R2-Y</a> | <a href="https://github.com/hsliuustc0106">hsliuustc0106</a> | <a href="https://github.com/Gaohan123">Gaohan123</a> | <a href="https://github.com/ZJY0516">ZJY0516</a> | <a href="https://github.com/qibaoyuan">qibaoyuan</a> |
-
-We kindly welcome more contributors to fix bugs and contribute new features!

docs/configuration/README.md

@@ -1,11 +0,0 @@
-# Configuration Options
-
-This section lists the most common options for running vLLM-Omni.
-
-For options within a vLLM Engine. Please refer to [vLLM Configuration](https://docs.vllm.ai/en/v0.11.0/configuration/index.html)
-
-Currently, the main options are maintained by stage configs for each model.
-
-For specific example, please refer to [Qwen2.5-omni stage config](stage_configs/qwen2_5_omni.yaml)
-
-For introduction, please check [Introduction for stage config](./stage_configs.md)

docs/configuration/stage_configs.md

@@ -1,272 +0,0 @@
-# Stage configs for vLLM-Omni
-
-In vLLM-Omni, the target model is separated into multiple stages, which are processed by different LLMEngines, DiffusionEngines or other types of engines. Depending on different types of stages, such as Autoregressive (AR) stage or Diffusion transformer (DiT) stage, each can choose corresponding schedulers, model workers to load with the Engines in a plug-in fashion.
-
-!!! note
-    Default stage config YAMLs (for example, `vllm_omni/model_executor/stage_configs/qwen2_5_omni.yaml` and `vllm_omni/model_executor/stage_configs/qwen3_omni_moe.yaml`) are bundled and loaded automatically when `stage_configs_path` is not provided. They have been verified to work on 1xH100 for Qwen2.5-Omni and 2xH100 for Qwen3-Omni.
-
-Therefore, as a core part of vLLM-Omni, the stage configs for a model have several main functions:
-
-- Claim partition of stages and their corresponding class implementation in `model_executor/models`.
-- The disaggregated configuration for each stage and the communication topology among them.
-- Engine arguments for each engine within the stage.
-- Input and output dependencies for each stage.
-- Default input parameters.
-
-If users want to modify some part of it. The custom stage_configs file can be input as input argument in both online and offline. Just like examples below:
-
-For offline (Assume necessary dependencies have ben imported):
-```python
-model_name = "Qwen/Qwen2.5-Omni-7B"
-omni_llm = OmniLLM(model=model_name, stage_configs_path="/path/to/custom_stage_configs.yaml")
-```
-
-For online serving:
-```bash
-vllm serve Qwen/Qwen2.5-Omni-7B --omni --port 8091 --stage-configs-path /path/to/stage_configs_file
-```
-!!! important
-    We are actively iterating on the definition of stage configs, and we welcome all feedbacks from both community users and developers to help us shape the development!
-
-Below is a specific example of stage_configs.yaml in Qwen2.5-omni.
-```python
-# stage config for running qwen2.5-omni with architecture of OmniLLM.
-stage_args:
-  - stage_id: 0 # mark the unique id for each stage
-    runtime: # The disaggregated configuration
-      process: true  # Run this stage in a separate process
-      devices: "0" # Visible devices for this stage (CUDA_VISIBLE_DEVICES/torch.cuda.set_device)
-      max_batch_size: 1 # the batch_size for offline inference
-    engine_args: # Engine arguments for a certain engine
-      model_stage: thinker
-      model_arch: Qwen2_5OmniForConditionalGeneration # The model implementation registered in model_executor/models/registry.py
-      worker_cls: vllm_omni.worker.gpu_ar_worker.GPUARWorker # The specific worker used
-      scheduler_cls: vllm_omni.core.sched.scheduler.OmniScheduler # The specific scehduler used
-      gpu_memory_utilization: 0.8 # The gpu memory allocation for the stage within a single chip
-      enforce_eager: true  # Now we only support eager mode
-      trust_remote_code: true # Needed by huggingface config parsing
-      engine_output_type: latent  # It claims that the stage will input latent hiddenstates besides token ids
-      enable_prefix_caching: false # For request with hiddenstates output, the prefix caching is not supported now
-    is_comprehension: true # If the stage is a text or multimodal comprehension module. If it is, the AsyncOmni will use its tokenizer as default
-    final_output: true # If the stage has output as part of final outputs. If it is false, which means that the stage only works as a intermediate role.
-    final_output_type: text # What is the final output type. It can be text and audio now.
-    default_sampling_params: # sampling parameters for the stage. Their meaning aligns with vLLM.
-      temperature: 0.0
-      top_p: 1.0
-      top_k: -1
-      max_tokens: 2048
-      seed: 42
-      detokenize: True
-      repetition_penalty: 1.1
-  - stage_id: 1
-    runtime:
-      process: true
-      devices: "1"
-      max_batch_size: 3
-    engine_args:
-      model_stage: talker
-      model_arch: Qwen2_5OmniForConditionalGeneration
-      worker_cls: vllm_omni.worker.gpu_ar_worker.GPUARWorker
-      scheduler_cls: vllm_omni.core.sched.scheduler.OmniScheduler
-      gpu_memory_utilization: 0.8
-      enforce_eager: true
-      trust_remote_code: true
-      enable_prefix_caching: false
-      engine_output_type: latent
-    engine_input_source: [0]
-    custom_process_input_func: vllm_omni.model_executor.stage_input_processors.qwen2_5_omni.thinker2talker
-    default_sampling_params:
-      temperature: 0.9
-      top_p: 0.8
-      top_k: 40
-      max_tokens: 2048
-      seed: 42
-      detokenize: True
-      repetition_penalty: 1.05
-      stop_token_ids: [8294]
-  - stage_id: 2
-    runtime:
-      process: true
-      devices: "0"            # Example: use a different GPU than the previous stage; use "0" if single GPU
-      max_batch_size: 1
-    engine_args:
-      model_stage: code2wav
-      model_arch: Qwen2_5OmniForConditionalGeneration
-      worker_cls: vllm_omni.worker.gpu_diffusion_worker.GPUDiffusionWorker
-      scheduler_cls: vllm_omni.core.sched.diffusion_scheduler.DiffusionScheduler
-      gpu_memory_utilization: 0.15
-      enforce_eager: true
-      trust_remote_code: true
-      enable_prefix_caching: false
-      engine_output_type: audio
-    engine_input_source: [1]
-    final_output: true
-    final_output_type: audio
-    default_sampling_params:
-      temperature: 0.0
-      top_p: 1.0
-      top_k: -1
-      max_tokens: 2048
-      seed: 42
-      detokenize: True
-      repetition_penalty: 1.1
-
-# Top-level runtime config (concise): default windows and stage edges
-runtime:
-  enabled: true
-  defaults:
-    window_size: -1             # Simplified: trigger downstream only after full upstream completion
-    max_inflight: 1             # Simplified: process serially within each stage
-  edges:
-    - from: 0                   # thinker → talker: trigger only after receiving full input (-1)
-      to: 1
-      window_size: -1
-    - from: 1                   # talker → code2wav: trigger only after receiving full input (-1)
-      to: 2
-      window_size: -1
-
-```
-
-## Stage Configuration Arguments
-
-Each stage in the `stage_args` list contains the following configuration options:
-
-### `stage_id`
-
-A unique identifier for each stage in the multi-stage pipeline. Stages are numbered sequentially starting from 0, and this ID is used to reference stages in inter-stage dependencies (e.g., `engine_input_source`).
-
-### `runtime`
-
-Configuration for disaggregated execution of the stage, controlling how the stage is deployed and executed.
-
-#### `runtime.process`
-
-Whether to run this stage in a separate process. When set to `true`, the stage will be executed in an isolated process, enabling better resource isolation and parallel execution across different stages. This is essential for multi-GPU deployments where different stages run on different devices.
-
-Default: `true`
-
-#### `runtime.devices`
-
-Visible devices for this stage, specified as a string. This controls which GPU devices are available to the stage process, similar to setting `CUDA_VISIBLE_DEVICES` or using `torch.cuda.set_device()`. For example, `"0"` uses GPU 0, `"1"` uses GPU 1, and `"0,1"` makes both GPUs 0 and 1 visible.
-
-Default: `"0"`
-
-#### `runtime.max_batch_size`
-
-The maximum batch size for offline inference in this stage. This limits how many sequences can be processed together in a single batch during offline inference operations.
-
-Default: `1`
-
-### `engine_args`
-
-Engine arguments for configuring the LLM engine, diffusion engine, or other engine types used by this stage.
-
-#### `engine_args.model_stage`
-
-The name identifier for this model stage within the multi-stage architecture. This is used internally to distinguish different stages of the same model (e.g., "thinker", "talker", "code2wav" in Qwen2.5-Omni).
-
-#### `engine_args.model_arch`
-
-The model architecture class name that is registered in `model_executor/models/registry.py`. This specifies which model implementation to use for this stage. The class must be registered in the model registry for vLLM-Omni to locate and instantiate it.
-
-#### `engine_args.worker_cls`
-
-The specific worker class to use for this stage. This determines how the model computations are executed. Examples include `vllm_omni.worker.gpu_ar_worker.GPUARWorker` for autoregressive stages and `vllm_omni.worker.gpu_diffusion_worker.GPUDiffusionWorker` for diffusion-based stages.
-
-#### `engine_args.scheduler_cls`
-
-The scheduler class to use for this stage. The scheduler manages request queuing, batching, and execution order. Examples include `vllm_omni.core.sched.scheduler.OmniScheduler` for standard stages and `vllm_omni.core.sched.diffusion_scheduler.DiffusionScheduler` for diffusion stages.
-
-#### `engine_args.gpu_memory_utilization`
-
-The fraction of GPU memory to allocate for this stage within a single GPU chip. This is a value between 0.0 and 1.0, where 0.8 means 80% of the GPU memory will be used by this stage. This allows fine-grained control over memory allocation when multiple stages share the same GPU or when reserving memory for other operations.
-
-Default: `0.8`
-
-#### `engine_args.enforce_eager`
-
-Whether to enforce eager execution mode. When set to `true`, the engine will run in eager mode without using CUDA graphs or other compilation optimizations. Currently, vLLM-Omni only supports eager mode.
-
-Default: `true`
-
-#### `engine_args.trust_remote_code`
-
-Whether to trust remote code when loading models from Hugging Face. This is required for models that use custom code in their configuration files. Set to `true` when loading models that require custom model implementations.
-
-Default: `true`
-
-#### `engine_args.engine_output_type`
-
-Specifies the type of output produced by this stage's engine. This determines what kind of data flows to downstream stages. Possible values include `latent` (hidden states), `text` (tokenized text), and `audio` (audio waveforms). When set to `latent`, the stage outputs latent hidden states in addition to token IDs, which are consumed by downstream stages.
-
-Default: `latent`
-
-#### `engine_args.enable_prefix_caching`
-
-Whether to enable prefix caching for this stage. Prefix caching can improve performance by caching KV cache for common prompt prefixes. However, for requests that output hidden states (when `engine_output_type` is `latent`), prefix caching is not currently supported and should be set to `false`.
-
-Default: `false`
-
-### `is_comprehension`
-
-Whether this stage is a text or multimodal comprehension module. When set to `true`, the stage acts as a comprehension module that processes input text or multimodal content. If this is the first comprehension stage, `AsyncOmni` will use its tokenizer as the default tokenizer for the entire pipeline.
-
-Default: `true`
-
-### `final_output`
-
-Whether this stage produces output that is part of the final outputs returned to the user. When set to `false`, the stage only works as an intermediate stage, processing data that flows to downstream stages but not contributing directly to the final response.
-
-Default: `true`
-
-### `final_output_type`
-
-The type of final output produced by this stage. This specifies what format the output will be in when returned to the user. Currently supported values are `text` (for text generation) and `audio` (for audio generation).
-
-Default: `text`
-
-### `default_sampling_params`
-
-Default sampling parameters for this stage. These parameters control the generation behavior and align with vLLM's sampling parameter semantics. These defaults are used when no explicit sampling parameters are provided in the request.
-
-#### `default_sampling_params.temperature`
-
-Sampling temperature for controlling randomness. Lower values (e.g., 0.0) make the output more deterministic and focused, while higher values increase randomness.
-
-Default: `0.0`
-
-#### `default_sampling_params.top_p`
-
-Nucleus sampling parameter. Only tokens with cumulative probability mass up to `top_p` are considered. This helps filter out low-probability tokens while maintaining diversity.
-
-Default: `1.0`
-
-#### `default_sampling_params.top_k`
-
-Top-k sampling parameter. Only the top `k` most likely tokens are considered. Set to `-1` to disable top-k filtering and consider all tokens.
-
-Default: `-1`
-
-#### `default_sampling_params.max_tokens`
-
-Maximum number of tokens to generate in this stage. This limits the length of the output sequence.
-
-Default: `2048`
-
-#### `default_sampling_params.seed`
-
-Random seed for reproducible generation. When set, the random number generator will be initialized with this seed to ensure consistent outputs across runs.
-
-Default: `42`
-
-#### `default_sampling_params.detokenize`
-
-Whether to detokenize the output tokens into text. When set to `true`, token IDs are converted back to readable text strings.
-
-Default: `True`
-
-#### `default_sampling_params.repetition_penalty`
-
-Penalty applied to tokens that have already appeared in the generated sequence. Values greater than 1.0 discourage repetition, while values less than 1.0 encourage it. A value of 1.0 applies no penalty.
-
-Default: `1.1`

docs/configuration/stage_configs/qwen2_5_omni.yaml

@@ -1,94 +0,0 @@
-# stage config for running qwen2.5-omni with architecture of OmniLLM.
-stage_args:
-  - stage_id: 0
-    runtime:
-      process: true            # Run this stage in a separate process
-      devices: "0"            # Visible devices for this stage (CUDA_VISIBLE_DEVICES/torch.cuda.set_device)
-      max_batch_size: 1
-    engine_args:
-      model_stage: thinker
-      model_arch: Qwen2_5OmniForConditionalGeneration
-      worker_cls: vllm_omni.worker.gpu_ar_worker.GPUARWorker
-      scheduler_cls: vllm_omni.core.sched.scheduler.OmniScheduler
-      gpu_memory_utilization: 0.8
-      enforce_eager: true  # Now we only support eager mode
-      trust_remote_code: true
-      engine_output_type: latent
-      enable_prefix_caching: false
-    is_comprehension: true
-    final_output: true
-    final_output_type: text
-    default_sampling_params:
-      temperature: 0.0
-      top_p: 1.0
-      top_k: -1
-      max_tokens: 2048
-      seed: 42
-      detokenize: True
-      repetition_penalty: 1.1
-  - stage_id: 1
-    runtime:
-      process: true
-      devices: "1"
-      max_batch_size: 1
-    engine_args:
-      model_stage: talker
-      model_arch: Qwen2_5OmniForConditionalGeneration
-      worker_cls: vllm_omni.worker.gpu_ar_worker.GPUARWorker
-      scheduler_cls: vllm_omni.core.sched.scheduler.OmniScheduler
-      gpu_memory_utilization: 0.8
-      enforce_eager: true
-      trust_remote_code: true
-      enable_prefix_caching: false
-      engine_output_type: latent
-    engine_input_source: [0]
-    custom_process_input_func: vllm_omni.model_executor.stage_input_processors.qwen2_5_omni.thinker2talker
-    default_sampling_params:
-      temperature: 0.9
-      top_p: 0.8
-      top_k: 40
-      max_tokens: 2048
-      seed: 42
-      detokenize: True
-      repetition_penalty: 1.05
-      stop_token_ids: [8294]
-  - stage_id: 2
-    runtime:
-      process: true
-      devices: "0"            # Example: use a different GPU than the previous stage; use "0" if single GPU
-      max_batch_size: 1
-    engine_args:
-      model_stage: code2wav
-      model_arch: Qwen2_5OmniForConditionalGeneration
-      worker_cls: vllm_omni.worker.gpu_diffusion_worker.GPUDiffusionWorker
-      scheduler_cls: vllm_omni.core.sched.diffusion_scheduler.DiffusionScheduler
-      gpu_memory_utilization: 0.15
-      enforce_eager: true
-      trust_remote_code: true
-      enable_prefix_caching: false
-      engine_output_type: audio
-    engine_input_source: [1]
-    final_output: true
-    final_output_type: audio
-    default_sampling_params:
-      temperature: 0.0
-      top_p: 1.0
-      top_k: -1
-      max_tokens: 2048
-      seed: 42
-      detokenize: True
-      repetition_penalty: 1.1
-
-# Top-level runtime config (concise): default windows and stage edges
-runtime:
-  enabled: true
-  defaults:
-    window_size: -1             # Simplified: trigger downstream only after full upstream completion
-    max_inflight: 1             # Simplified: process serially within each stage
-  edges:
-    - from: 0                   # thinker → talker: trigger only after receiving full input (-1)
-      to: 1
-      window_size: -1
-    - from: 1                   # talker → code2wav: trigger only after receiving full input (-1)
-      to: 2
-      window_size: -1

docs/contributing/README.md

@@ -1,52 +1,55 @@
-# Contributing to vLLM-Omni
+# Contributing to vLLM-omni
 
-Thank you for your interest in contributing to vLLM-Omni! This document provides guidelines and instructions for contributing.
-
-!!! note
-    We host weekly developer-facing online meetings to discuss milestones and updates **every Tuesday at 19:30 PDT**. Meeting link as well as the past meeting notes can be found [here](https://tinyurl.com/vllm-omni-meeting).
+Thank you for your interest in contributing to vLLM-omni! This document provides guidelines and instructions for contributing.
 
 ## Getting Started
 
-vLLM-Omni uses `uv` as the environment manager, to create and manage Python environments. Please follow the documentation to install `uv`. After installing `uv`, you can create a new Python environment using the following commands:
+### Development Environment
+
+vLLM-omni uses `uv` as the environment manager, to create and manage Python environments. Please follow the documentation to install `uv`. After installing `uv`, you can create a new Python environment using the following commands:
 
 ```bash
 uv venv --python 3.12 --seed
 source .venv/bin/activate
 ```
-### Development Environment for vLLM
 
-vLLM-Omni is built on top of the stable release 0.11.0 of vLLM. You may install `vllm==0.11.0` directly or install the library from source if you need to check, modify or debug with source code of vLLM.
+Install vLLM from source:
 
-### Development Environment for vLLM-Omni
+```bash
+git clone https://github.com/vllm-project/vllm.git
+cd vllm
+git checkout v0.11.0
+VLLM_USE_PRECOMPILED=1 uv pip install -e .
+```
 
-Install vLLM-Omni from source with development dependencies:
+Install vLLM-omni from source:
 
 ```bash
 git clone https://github.com/vllm-project/vllm-omni.git
-cd vllm-omni
-uv pip install -e ".[dev]"
+cd vllm_omni
+uv pip install -e .
 ```
 
 !!! tip
-    vLLM-Omni is compatible with Python versions 3.10 to 3.12. However, we recommend developing with Python 3.12 to minimize the chance of your local environment clashing with our CI environment.
+    vLLM-omni is compatible with Python versions 3.10 to 3.12. However, we recommend developing with Python 3.12 to minimize the chance of your local environment clashing with our CI environment.
 
 ### Linting
 
-vLLM-Omni uses `pre-commit` to lint and format the codebase. See [pre-commit documentation](https://pre-commit.com/#usage) if `pre-commit` is new to you. Setting up `pre-commit` is as easy as:
+vLLM-omni uses `pre-commit` to lint and format the codebase. See [pre-commit documentation](https://pre-commit.com/#usage) if `pre-commit` is new to you. Setting up `pre-commit` is as easy as:
 
 ```bash
 uv pip install pre-commit
 pre-commit install
 ```
 
-vLLM-Omni's `pre-commit` hooks will now run automatically every time you commit.
+vLLM-omni's `pre-commit` hooks will now run automatically every time you commit.
 
 !!! tip
     You can manually run the `pre-commit` hooks using:
 
     ```bash
     pre-commit run     # runs on staged files
-    pre-commit run --show-diff-on-failure --color=always --all-files  # runs on all files (short for --all-files)
+    pre-commit run -a  # runs on all files (short for --all-files)
     ```
 
 ### Documentation
@@ -75,14 +78,20 @@ For additional features and advanced configurations, refer to the:
 
 ### Testing
 
-vLLM-Omni uses `pytest` to test the codebase.
+vLLM-omni uses `pytest` to test the codebase.
 
 ```bash
+# Install the test dependencies used in CI (CUDA only)
+uv pip install -e ".[dev]" --torch-backend=auto
+
+# Install some common test dependencies (hardware agnostic)
+uv pip install pytest pytest-asyncio
+
 # Run all tests
 pytest tests/
 
 # Run tests for a single test file with detailed output
-pytest -s -v tests/test_omni_llm.py
+pytest -s -v tests/test_logger.py
 ```
 
 !!! warning
@@ -97,7 +106,7 @@ If you encounter a bug or have a feature request, please search existing issues
 
 ## Pull Requests & Code Reviews
 
-Thank you for your contribution to vLLM-Omni! Before submitting the pull request, please ensure the PR meets the following criteria. This helps vLLM-Omni maintain the code quality and improve the efficiency of the review process.
+Thank you for your contribution to vLLM-omni! Before submitting the pull request, please ensure the PR meets the following criteria. This helps vLLM-omni maintain the code quality and improve the efficiency of the review process.
 
 ### DCO and Signed-off-by
 
@@ -119,9 +128,9 @@ Only specific types of PRs will be reviewed. The PR title is prefixed appropriat
 - `[CI/Build]` for build or continuous integration improvements.
 - `[Doc]` for documentation fixes and improvements.
 - `[Model]` for adding a new model or improving an existing model. Model name should appear in the title.
-- `[Frontend]` For changes on the vLLM-Omni frontend (e.g., OpenAI API server, `OmniLLM` class, etc.)
+- `[Frontend]` For changes on the vLLM-omni frontend (e.g., OpenAI API server, `OmniLLM` class, etc.)
 - `[Kernel]` for changes affecting CUDA kernels or other compute kernels.
-- `[Core]` for changes in the core vLLM-Omni logic (e.g., `OmniProcessor`, `OmniScheduler`, etc.)
+- `[Core]` for changes in the core vLLM-omni logic (e.g., `OmniProcessor`, `OmniScheduler`, etc.)
 - `[Hardware][Vendor]` for hardware-specific changes. Vendor name should appear in the prefix, such as [Ascend] for Ascend NPUs.
 - `[Misc]` for PRs that do not fit the above categories. Please use this sparingly.
 
@@ -136,7 +145,7 @@ The PR needs to meet the following code quality standards:
 - Pass all linter checks.
 - The code needs to be well-documented to ensure future contributors can easily understand the code.
 - Include sufficient tests to ensure the project stays correct and robust. This includes both unit tests and integration tests.
-- Please add documentation to `docs/` if the PR modifies the user-facing behaviors of vLLM-Omni. It helps vLLM-Omni users understand and utilize the new features or changes.
+- Please add documentation to `docs/` if the PR modifies the user-facing behaviors of vLLM-omni. It helps vLLM-omni users understand and utilize the new features or changes.
 
 ### Notes for Large Changes
 
@@ -144,17 +153,18 @@ Please keep the changes as concise as possible. For major architectural changes
 
 ### What to Expect for the Reviews
 
-The goal of the vLLM-Omni team is to be a _transparent reviewing machine_. We would like to make the review process transparent and efficient and make sure no contributor feels confused or frustrated. However, the vLLM-Omni team is small, so we need to prioritize some PRs over others. Here is what you can expect from the review process:
+The goal of the vLLM-omni team is to be a _transparent reviewing machine_. We would like to make the review process transparent and efficient and make sure no contributor feels confused or frustrated. However, the vLLM-omni team is small, so we need to prioritize some PRs over others. Here is what you can expect from the review process:
 
 - After the PR is submitted, the PR will be assigned to a reviewer. Every reviewer will pick up the PRs based on their expertise and availability.
-- After the PR is assigned, the reviewer will provide status updates every 2-3 days. If the PR is not reviewed within 7 days, please feel free to ping the reviewer or the vLLM-Omni team.
+- After the PR is assigned, the reviewer will provide status updates every 2-3 days. If the PR is not reviewed within 7 days, please feel free to ping the reviewer or the vLLM-omni team.
 - After the review, the reviewer will put an `action-required` label on the PR if there are changes required. The contributor should address the comments and ping the reviewer to re-review the PR.
 - Please respond to all comments within a reasonable time frame. If a comment isn't clear or you disagree with a suggestion, feel free to ask for clarification or discuss the suggestion.
+- Note that not all CI checks will be executed due to limited computational resources. The reviewer will add `ready` label to the PR when the PR is ready to merge or a full CI run is needed.
 
 ## Additional Resources
 
-- [Design Documents](../design/index.md) - Architecture and design documentation
+- [Design Documents](design_documents/index.md) - Architecture and design documentation
 
 ## Thank You
 
-Finally, thank you for taking the time to read these guidelines and for your interest in contributing to vLLM-Omni. All of your contributions help make vLLM-Omni a great tool and community for everyone!
+Finally, thank you for taking the time to read these guidelines and for your interest in contributing to vLLM-omni. All of your contributions help make vLLM-omni a great tool and community for everyone!

docs/contributing/ci/failures.md

@@ -1,4 +0,0 @@
-# CI Failures
-
-What should I do when a CI job fails on my PR, but I don't think my PR caused
-the failure?

docs/contributing/design_documents/api_design_doc.md

@@ -0,0 +1,143 @@
+# API Design Documentation
+
+This document contains comprehensive API design documentation for all core modules in vLLM-omni. These templates provide a standardized structure for designing and implementing the core, engine, executor, and worker modules.
+
+## 📋 Module API
+
+### Core Module API
+**Core module** provides fundamental scheduling, caching, and resource management functionality.
+
+**Key Components:**
+- Request scheduling and prioritization
+- DiT cache management for diffusion models
+- Resource allocation and coordination
+- Inter-module communication
+
+### Engine Module API
+**Engine module** handles model loading, inference execution, and output processing.
+
+**Key Components:**
+- Model loading and initialization
+- Inference execution for AR and diffusion models
+- Input preprocessing for various modalities
+- Output postprocessing and formatting
+
+### Executor Module API
+**Executor module** coordinates and manages request execution across different engines and workers.
+
+**Key Components:**
+- Request routing and load balancing
+- Execution pipeline coordination
+- Worker management and task distribution
+- Error handling and recovery
+
+### Worker Module API
+**Worker module** provides the actual execution environment for model inference.
+
+**Key Components:**
+- Model execution and inference
+- GPU resource management
+- Request batching and processing
+- Performance optimization
+
+
+## 🔧 Implementation Guidelines
+
+### Code Standards
+- Use type hints for all methods
+- Include comprehensive docstrings
+- Follow PEP 8 style guidelines
+- Use async/await for I/O operations
+
+### Error Handling
+- Define specific exception types
+- Provide meaningful error messages
+- Include error recovery strategies
+- Log errors appropriately
+
+### Configuration
+- Use dataclasses for configuration
+- Provide sensible defaults
+- Include validation methods
+- Support environment variables
+
+### Testing
+- Write unit tests for all public methods
+- Include integration tests
+- Test error conditions
+- Validate configuration options
+
+## 🚀 Getting Started
+
+### For Developers
+
+1. **Choose a Module**
+   - Pick the module you want to work on
+   - Read the corresponding API documentation
+   - Understand the responsibilities and interfaces
+
+2. **Set Up Development Environment**
+   ```bash
+   # Clone the repository
+   git clone https://github.com/vllm-project/vllm-omni.git
+   cd vllm-omni
+
+   # Install dependencies
+   pip install -e ".[dev]"
+
+   # Set up pre-commit hooks
+   pre-commit install
+   ```
+
+3. **Start Implementation**
+   - Create the module directory
+   - Implement base classes first
+   - Add concrete implementations
+   - Write tests as you go
+
+### For Contributors
+
+1. **Review API Documentation**
+   - Read through all module APIs
+   - Understand the overall architecture
+   - Identify areas for improvement
+
+2. **Propose Changes**
+   - Create issues for API changes
+   - Discuss in pull requests
+   - Update documentation accordingly
+
+3. **Submit Contributions**
+   - Follow the coding standards
+   - Include tests for new features
+   - Update documentation
+   - Submit pull requests
+
+## 🤝 Contributing
+
+We welcome contributions to improve the API documentation and implementation. Please:
+
+1. **Follow the Template Structure**
+   - Use the [api template](https://github.com/vllm-project/vllm-omni/docs/contributing/design_documents/api_design_template.md) as a guide
+   - Maintain consistency across modules
+   - Include all required sections
+
+2. **Provide Working Examples**
+   - Test all code examples
+   - Include both basic and advanced usage
+   - Show integration patterns
+
+3. **Keep Documentation Updated**
+   - Update docs when APIs change
+   - Version control documentation changes
+   - Maintain backward compatibility notes
+
+## 📝 Notes
+
+- All API documentation should be kept up-to-date with implementation
+- Code examples should be tested and working
+- Configuration options should be validated
+- Error handling should be comprehensive
+- Performance considerations should be documented
+
+For questions or suggestions about the API documentation, please open an issue or start a discussion in the repository.

docs/contributing/design_documents/api_design_template.md

@@ -0,0 +1,336 @@
+# API Design Template for vLLM-omni Modules
+
+This template provides a standardized structure for designing APIs for core, engine, executor, and worker modules in vLLM-omni. Use this template to ensure consistency and completeness across all modules.
+
+## Template Structure
+
+### 1. Module Overview
+- **Purpose**: What this module does
+- **Responsibilities**: Key responsibilities of the module
+- **Dependencies**: What other modules this depends on
+- **Integration Points**: How it integrates with other modules
+
+### 2. Core Classes/Interfaces
+- **Base Classes**: Abstract base classes and interfaces
+- **Implementation Classes**: Concrete implementations
+- **Data Structures**: Key data structures and models
+
+### 3. Public API Methods
+- **Initialization**: Constructor and setup methods
+- **Core Operations**: Main functionality methods
+- **Configuration**: Configuration and parameter methods
+- **Lifecycle Management**: Start, stop, cleanup methods
+- **Monitoring**: Status, metrics, and debugging methods
+
+### 4. Configuration
+- **Configuration Classes**: Dataclasses or config objects
+- **Required Parameters**: Must-have configuration
+- **Optional Parameters**: Optional configuration with defaults
+- **Validation**: Parameter validation rules
+
+### 5. Error Handling
+- **Custom Exceptions**: Module-specific exceptions
+- **Error Codes**: Standardized error codes
+- **Recovery Strategies**: How to handle and recover from errors
+
+### 6. Examples
+- **Basic Usage**: Simple usage examples
+- **Advanced Usage**: Complex scenarios
+- **Integration Examples**: How to use with other modules
+
+---
+
+## Example: Core Module Template
+
+### 1. Module Overview
+
+**Purpose**: The core module provides fundamental scheduling and caching functionality for vLLM-omni.
+
+**Responsibilities**:
+- Request scheduling and prioritization
+- DiT cache management
+- Resource allocation and coordination
+- Inter-module communication
+
+**Dependencies**:
+- `vllm_omni.request` - Request handling
+- `vllm_omni.config` - Configuration management
+- `vllm_omni.utils` - Utility functions
+
+**Integration Points**:
+- Receives requests from entrypoints
+- Coordinates with engine modules
+- Manages worker allocation
+- Provides status to monitoring systems
+
+### 2. Core Classes/Interfaces
+
+```python
+from abc import ABC, abstractmethod
+from typing import List, Optional, Dict, Any
+from dataclasses import dataclass
+from enum import Enum
+
+class DiTCacheManager:
+    """Manages DiT cache for diffusion models."""
+
+    def __init__(self, config: DiTCacheConfig):
+        self.config = config
+        self.cache = {}
+        self.cache_stats = {}
+
+    async def get_cache(self, cache_key: str) -> Optional[Any]:
+        """Retrieve cached data."""
+        pass
+
+    async def set_cache(self, cache_key: str, data: Any) -> None:
+        """Store data in cache."""
+        pass
+
+    async def invalidate_cache(self, cache_key: str) -> None:
+        """Invalidate cached data."""
+        pass
+```
+
+### 3. Public API Methods
+
+#### Initialization
+```python
+class OmniScheduler():
+    def __init__(self, config: SchedulerConfig):
+        """Initialize the core scheduler."""
+        self.config = config
+        self.scheduler = self._create_scheduler()
+        self.cache_manager = DiTCacheManager(config.dit_cache_config)
+        self._running = False
+
+    def _create_scheduler(self) -> BaseScheduler:
+        """Factory method to create appropriate scheduler."""
+        if self.config.scheduler_type == SchedulerType.FIFO:
+            return FIFOScheduler(self.config)
+        elif self.config.scheduler_type == SchedulerType.PRIORITY:
+            return PriorityScheduler(self.config)
+        # ... other scheduler types
+```
+
+#### Core Operations
+```python
+    async def schedule(self, request: OmniRequest) -> bool:
+        """
+        Schedule a request for processing.
+
+        Args:
+            request: The request to schedule
+
+        Returns:
+            bool: True if successfully scheduled, False otherwise
+
+        Raises:
+            SchedulerError: If scheduling fails
+            QueueFullError: If queue is at capacity
+        """
+        pass
+```
+
+#### Configuration
+```python
+    def update_config(self, new_config: SchedulerConfig) -> None:
+        """Update scheduler configuration."""
+        pass
+
+    def get_config(self) -> SchedulerConfig:
+        """Get current configuration."""
+        pass
+```
+
+#### Lifecycle Management
+```python
+    async def start(self) -> None:
+        """Start the scheduler."""
+        self._running = True
+        # Start background tasks
+
+    async def stop(self) -> None:
+        """Stop the scheduler gracefully."""
+        self._running = False
+        # Cleanup resources
+
+    async def shutdown(self) -> None:
+        """Force shutdown the scheduler."""
+        # Immediate cleanup
+```
+
+#### Monitoring
+```python
+    def get_status(self) -> Dict[str, Any]:
+        """Get current scheduler status."""
+        return {
+            "running": self._running,
+            "queue_size": self.scheduler.queue_size(),
+            "processed_requests": self.scheduler.processed_count(),
+            "cache_hit_rate": self.cache_manager.get_hit_rate()
+        }
+
+    def get_metrics(self) -> Dict[str, float]:
+        """Get performance metrics."""
+        pass
+```
+
+### 4. Configuration
+
+```python
+@dataclass
+class OmniConfig:
+    """Configuration for the core module."""
+
+    # Scheduler configuration
+    scheduler: SchedulerConfig = field(default_factory=SchedulerConfig)
+
+    # Cache configuration
+    dit_cache: DiTCacheConfig = field(default_factory=DiTCacheConfig)
+
+    # Resource limits
+    max_memory_gb: float = 16.0
+    max_gpu_utilization: float = 0.8
+
+    # Timeouts
+    request_timeout: int = 300
+    worker_timeout: int = 60
+
+    def validate(self) -> None:
+        """Validate configuration parameters."""
+        if self.max_memory_gb <= 0:
+            raise ValueError("max_memory_gb must be positive")
+        if not 0 < self.max_gpu_utilization <= 1:
+            raise ValueError("max_gpu_utilization must be between 0 and 1")
+```
+
+### 5. Error Handling
+
+```python
+class CoreModuleError(Exception):
+    """Base exception for core module errors."""
+    pass
+
+class SchedulerError(CoreModuleError):
+    """Scheduler-related errors."""
+    pass
+
+class QueueFullError(SchedulerError):
+    """Queue is at capacity."""
+    pass
+
+class CacheError(CoreModuleError):
+    """Cache-related errors."""
+    pass
+
+class ResourceError(CoreModuleError):
+    """Resource allocation errors."""
+    pass
+```
+
+### 6. Examples
+
+#### Basic Usage
+```python
+from vllm_omni.core import CoreScheduler, CoreModuleConfig
+from vllm_omni.request import create_text_request
+
+# Create configuration
+config = CoreModuleConfig(
+    scheduler=SchedulerConfig(scheduler_type=SchedulerType.FIFO),
+    max_memory_gb=8.0
+)
+
+# Initialize scheduler
+scheduler = CoreScheduler(config)
+await scheduler.start()
+
+# Create and schedule a request
+request = create_text_request(
+    request_id="req_001",
+    prompt="Hello, world!",
+    sampling_params=sampling_params
+)
+
+success = await scheduler.schedule_request(request)
+if success:
+    result = await scheduler.process_request(request)
+    print(f"Result: {result}")
+```
+
+#### Advanced Usage
+```python
+# Custom scheduler with priority handling
+config = CoreModuleConfig(
+    scheduler=SchedulerConfig(
+        scheduler_type=SchedulerType.PRIORITY,
+        priority_weights={"high": 2.0, "normal": 1.0, "low": 0.5}
+    )
+)
+
+scheduler = CoreScheduler(config)
+await scheduler.start()
+
+# Monitor scheduler status
+status = scheduler.get_status()
+print(f"Queue size: {status['queue_size']}")
+print(f"Cache hit rate: {status['cache_hit_rate']:.2%}")
+```
+
+---
+
+## Module-Specific Guidelines
+
+### Core Module
+- Focus on scheduling, caching, and resource management
+- Provide clear interfaces for other modules
+- Handle concurrency and thread safety
+- Implement comprehensive monitoring
+
+### Engine Module
+- Handle model loading and inference
+- Support both AR and diffusion models
+- Provide unified interface for different model types
+- Implement efficient memory management
+
+### Executor Module
+- Coordinate between different engines
+- Handle request routing and load balancing
+- Manage execution pipelines
+- Provide error recovery mechanisms
+
+### Worker Module
+- Handle actual model execution
+- Manage GPU resources
+- Implement batching strategies
+- Provide performance optimization
+
+---
+
+## Checklist for API Design
+
+- [ ] **Clear Purpose**: Module purpose is well-defined
+- [ ] **Complete Interface**: All public methods are documented
+- [ ] **Error Handling**: Comprehensive error handling strategy
+- [ ] **Configuration**: Flexible configuration system
+- [ ] **Examples**: Basic and advanced usage examples
+- [ ] **Type Hints**: All methods have proper type hints
+- [ ] **Documentation**: Comprehensive docstrings
+- [ ] **Testing**: Unit tests for all public methods
+- [ ] **Integration**: Clear integration points with other modules
+- [ ] **Performance**: Performance considerations documented
+
+---
+
+## Submission Guidelines
+
+1. Create a new file: `docs/api/[module_name]_api.md`
+2. Follow the template structure exactly
+3. Include all required sections
+4. Provide working code examples
+5. Ensure all methods have proper docstrings
+6. Include error handling strategies
+7. Submit for review before implementation
+
+This template ensures consistency and completeness across all vLLM-omni modules.

docs/contributing/design_documents/architecture_overview.md

@@ -2,11 +2,11 @@
 
 # Introduction
 
-This document outlines the architectural design for vLLM-Omni, a new repository developed under the vLLM project. vLLM-Omni supports multi-modality models with non-autoregressive structures and non-textual outputs, extending vLLM beyond traditional text-based, autoregressive generation.
+This document outlines the architectural design for vLLM-omni, a new repository developed under the vLLM project. vLLM-omni supports multi-modality models with non-autoregressive structures and non-textual outputs, extending vLLM beyond traditional text-based, autoregressive generation.
 
 # Goals
 
-The primary goals for vLLM-Omni are:
+The primary goals for vLLM-omni are:
 
 * **Support for Non-textual Output:** Enables the integration, efficient processing and output of various data types, including but not limited to, images, audio, and video, alongside text.  
 * **Non-Autoregressive Structure:** Support model structure beyond autoregressive, especially Diffusion Transformer (DiT), which is widely used in visual and audio generation.  
@@ -57,12 +57,12 @@ According to analysis for current popular open-source models, most of them have
 
 # Architectural Overview
 
-vLLM-Omni will import core modules from existing vLLM framework and introduce several new components and modifications to achieve its goals. The core architecture will be based on a modular design, allowing for independent development and integration of different modality handlers and output generators.  
+vLLM-omni will import core modules from existing vLLM framework and introduce several new components and modifications to achieve its goals. The core architecture will be based on a modular design, allowing for independent development and integration of different modality handlers and output generators.  
 ![][image6]
 
 ## General ideas
 
-* It divides models in vLLM-Omni into multiple stages, which are processed by different Engine Cores. Each stage can choose AR Engine Core for AR model part, and Diffusion Engine Core for diffusion model part.  
+* It divides models in vLLM-omni into multiple stages, which are processed by different Engine Cores. Each stage can choose AR Engine Core for AR model part, and Diffusion Engine Core for diffusion model part.  
 * Designed a specialized diffusion engine core for DiT or dLLM.  
 * Plug-in with external DiT acceleration modules such as xDiT, Cache-DiT.  
 * Output linked with Higress API, Gradio for online demo, ComfyUI for less-code process.
@@ -89,6 +89,32 @@ vLLM-Omni will import core modules from existing vLLM framework and introduce se
 
 ![][image11]
 
+# Roadmap
+
+* AsyncLLM:  
+- [ ] Initializer: resource allocation for different stages (2025 H2)  
+- [ ] Hidden state Output Processor (2025 H2)  
+- [ ] Multimodal Output Processor (2025 H2)  
+- [ ] Interface for API, Gradio, ComfyUI  
+* OmniEngine:  
+- [ ] Basic Diffusion Engine core: Diffusion step manager, Cache manager (2025 H2)  
+- [ ] Pipeline of different Engine Cores (2025 H2)  
+- [ ] Communication between Engine Cores via AsyncLLM (2025 H2)  
+* Model support (Included but not limited to):  
+- [ ] DiT: Qwen-Image (2025 H2), Flux-Kontext  
+- [ ] Unified-MLLM: BAGEL (2025 H2), OmniGen2, Janus-Pro  
+- [ ] Extensions of AR+DiT: Qwen-omni (2025 H2)  
+* Optimization  
+- [ ] Integration of other DiT acceleration modules, such as xDiT, Cache-DiT  
+- [ ] Disaggregation for AR and DiT  
+- [ ] Integration with vLLM's existing KV cache for potential future hybrid models.  
+- [ ] Support for streaming multi-modal inputs and outputs
+
+# Contact
+
+For any questions or feedback regarding this design document, please contact  
+Han Gao: [hgaoaf@connect.ust.hk](mailto:hgaoaf@connect.ust.hk),  
+Hongsheng Liu: [vllm-project@gmail.com](mailto:vllm-project@gmail.com)
 
 [image1]: <data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAALYAAAFwCAYAAAAVNoEZAABxw0lEQVR4XuydBVRcWb6ve90709OStnTScel4x93dIUaEuBJ3dzciRPDgEtzdCRpCBCIEEkIgQHCCBHe+t6uSvNvNnZ55r2/fng5zvrX+69TZp4qkan3nz2/vKup88sm/OWqXL3HxwnnOnj7JubOnuaB6jqtXLqJ28TxXRWmrX8Ha0gIzE0OuX1bFQFsTcxMDLMyMMDXS44bWVRxsLLG5aYKuxlVsLMxwc3bAWF8bNzdnHGwt8fb2wNXZHlcnW3HMDjsbC4Ju3cLO2hwvT3ecnexwsLcmIMAHHy83+X18PVxxtLMUt+1xcbTBw8URRwdr+f1tLM1wd3PB08NZlAs24ue4uTrKf0bD5yfxb8qWDWvZtnE9W0Vt37yBvTu3cWDPTlE7OHviCBfPnsTEwECIfg7Vk0e5fkkVoxtamBneQF1NlSsXTmMrZJaN6elcF7LexFaIZ6SngbOjvfy2l5DUSwhoa2mK9U1jfL09ibwTKaS3IPx2BA4ONnh7uuHr4yUX1d/Pi0B/X1xd7PHz8cTT3UUc85SfIMHBQUJ4a3EfH/mJ4OsjThoXB/njPNydJbEl3rFisTKrli5i9fLFrF21jC0bVdi2aR07tqzn+KF9nD91HF1NDU4eO8CZIwe5fO4UBjoacpEvnDnG5fMnsDQ1Rk/rOje0rwmxLeQd3chAC0d7W6zMjfBwc8JHiGlmpIu5oS6hISHcibgtOrE99+4+wE50dR8hraeHm1zOW4G+BAb4yjtyYICf/FiAED0wMICw0DDxODvR8QNEBeLv7yO6txPeotN7e7lLYku8Y8akccycOhGlGVNRVprJsoXzWLlEmbUrl7Jv5xbOHD8sBD/ApnWrOLZvN2ePH0FDRJVronMf3rdDiH1aiK4p9s+JE0AWS6xE59bA2EAHO6ub3DTRw1UI7Ovjg/oVEWV01Llz567ouH6i+wbz8FEMtiKaBPr7yzuvn6+nED9I3rH9/bwJDgqUSy3r1OHh4YSEBOPu7io6/W1CQ0PFSeAvxHeTd3B/X19JbIl3TB4zjOnjRzJz8jiUFCazYPZ0li1QYrXo4nt3bObE4QPs3LIRlRWLObh7h9jfj5rqGdGtj7N/11Yh9hk0r14W+yfQ0bgiZLZC8/pljPR1ROwwwdxYF2d7GyGfN1cunkJfiB0h4oe3lweRkfeIjn4iF9vf1wcnkbX9fL0ICb4ll1kutrgtkzdEyB4ZGUlQ0C35Y+/cuSPv3oHimOxkkHX2W6KjN3x+Ev+mjBzQmzGD+zJxxCAmjxrC9AmjUBZyL1dWEkJvEF17G6tEB1++aB47Nq0Xsm+Vy71PbHdv2yTP4LL9Q3t3onVdTYhsyCXV02LyqCuyuS762texlk0oXV05e/IQetrqBInuK5v8RUc/JiLiDvYiisgmf7Y2N0UssZJv1a9fFQJ7yiUPkXXrsBAePHiAn+jMsghy//590dlDRRTxk8cW2VhoWLAktsQ7Bv7UleH9egi5+8kFl3Xw+TOnsmTBLDGpXMsOMaFcunCuiChKbF27ml1CbtnEcvt6FbnY504eE517uxB+CxpXLwlxNbhwXjbhvCGXWFOMmZsZibztwMmj+7khsniA6Kwurs4iSoi87OKCuuj4urqa3DQ3xdHBVt6pXZwd0NJQJ1AmbGiw6NC3xYkQLbK6lxA6WC55yP8VWxZrbhEWHiKJLfGOPl06MKhnVwb06Cy2XZg0aqiIJJNYNHcG61cvl2dtZXF78byZrF22iHUrl7B53WqRwxcI6dfLO/WW9Wvk3f3i2VOonj7BeRFT9HW0ULtwRoh/FIMb2hgZ6otMLuvql9klos3COYri3xrOiEH96dujK907d+Snrp3o37c3Gzdu4MKFCxjqG7Bn9y7uRt6Vd+gHUQ9xFsKHiMmnbEy29ReRRbaaEhYWJo8qDZ+fxL8pfbt2pF+3TqJ+ZHCvLiKODGX2tAly8VRWLJFPJBcoKcpFX6E8j9WL300slyrPkXd0WdfeIE4A2UqKbHlQtpJy7tQxtNWvckZILYspOprX0dFSZ8+2DezcqMKsCaOZOGwQvTp1oHPbVnRo2YL2rZrz/TdN+O6rJrRp8YMQvA+q5y9w9NARbgUFEf3wEffuR2FrZyUXOiIiQnTyEPmkUSa2LNLcu/dAElviHb06taF357bIBB/cqzNjh/YXOXs0cxWnsWjebPlKyVwRTebNmILyLAWWzJ3NMuW5LJijIO/o61YvY/WShWxUWcmB3dvZLgQ/emAvZ04cZc/2zULmTaiKuLJv22ZmjxvJqH496dK6pagW9Gnfhp7tWtH5h+9p//23tPruG1qI+v6br/n2qy/47K9/odn337FGRQUHJ2fCbt/GyFhPxI4Q+eRRJrivr7d8AvngQZTo6tGS2BLv6P6jELtLO7nY/bv/yPCBPZkiBJylMIV5QuS5M6czR3EKStMnMU8m+5yZLFGazdwZU1mzfAmrly2UTyzXr1omjyMb1qxgn8jch/fvYdsGFXZsXMMusV0i7j+6V1eGdm3PGBF5pvXtztxBPZjetyujxL8/VPw/erRuzo9C8pZNheBNv+b7r7/iy88/o2PHjixcspTbtyO4IbK4bMnvzp13KySyzB0cIptYRktiS/wXnzdpzpdNmtGtQyt+6tiK/j06MXbEYCaPG8WMKROYOWUc0yaOY6ro4orittKMacxWmMysaeNZtmAei+fPEZ19FquWLmSdkHv5ovnyzL1dxJQ1Qvr1ohaOG8rxJTO4ukYJ/a1LODprDEsGdmdkux8Y2KopHb/+knZffk6LJp/xvRD5e3G7+Zdf0PKbr2gtJG8jZP/uqy9Zu24tqmdOEn77jrxjy96wkb1BExERyd2794m4c1cSW+Idn33ZlL99IaJA6x/o1KaZ6N7tGTmkP+NGDpXLPG38KLnkk8eNYPqkMXKpZ0ydIKQfK2LJHHkHV1aaIYReIM/jMtFlgss691KRzeeKn6G7YwVup3fgr3YAw83KLO/VkbX9OrNhYBfmd23JrC6tmNT+B7p89zVdRafu1uwbOjf9ivbffUUHsS/r4h3F/69Fs2YsWbhAZOn78owdGBgol1vWyWWTyYg7EZLYEu9o/cVXrO71E6aL53Nx1mT6iLw9sE9Phg/qx/jhQ+Q1doTYjhryXu6xTJNtx4scrjCVOSKiKImoIsvispJ19MULlEQGn8GsYf05vWI2+aGOZHqb4LJvNY+v7iP2+hEeXdpL2NF1uG9V5ti4nuwb1pUlPVoz88cWTGzblHGihrVuypA23zOsfXOGCPE7t/qBZt9+g7qmFmFh4XKxZXlb9kbPbdG1ZZ87afj8JP5NGd2mFfuHD8BCeQ43F8/hzMxxzB3al5H9fmJo/z4MH9CXEYMHMEpMKscNFxFl9AjGjxjERNHRFSbIJB/N9MljmDV9MjOnTUJh8nhmT5/CxNHDMNu/luI77pRGB5Dhb0t1zB2qnoRREuxEnpsx989sI3T/apzXKXBz6WRMl01Bf9F4VKcP5uj4Pmwd1p21g7ui3Lsj83p1oEerZrQV3fv7pt9y4uRxbgUHERIWRlh4BMFC9FvBoZLYEu/YObgfp0YO4ZLowtemjhViKXB1wVT2KIxBccRAxorOPVgIPnRQX8YMHSjv4GPeSz5p7EgmjhkhjywyoadPkmXxMfJMPmHoAOJstKhLjKLiaTiVCQ+pSY6jJj6K0jA3CjxNSdI/S4L2MR5c2sW9CzsIOqKC156lGC2bivr8MRye0I+dI3uyelBXlvXvxMA2zWnb7Ds+//QvjBkziiAxaQwJDRWTx3D5kqC/9Ja6xAcU27Rkcgvx67/59yzo2JrDo/thvVYZj51rCT20leCDm7DfvALN5XNRHj2UcQP7MKRPV9HJe4suLuvkQvZRw+SCTxg9XH57pDgZji6bSXVcJHWZidS+SaMuN43alGdysasfBlER6UNpsCNv/azJctQh1eoacdrHuX9hF67bFmK+SpFz04dwZHw/to3sxfqh3ZgssniXZl/LlwT/+p//ia+vn5g8+uPt7SX/YJSrq4sktsQ7un39Fd2/+ZoB337NpJZNWdezI/pKY3FSmU+o2mVCjc3xd/LCw9kHbSNrDl/QQFFpAaNHDGNov3dZfIzo3rLJ5tiR7/L4zJGDCBQ5ui4tntr8TOpF1WWnUJsUQ+2z+1RHBVF1L4DSUFeKAuzJdjEkzUaT+BtniLq8F+ftypitVOCsEPvoxH7sHt2XzUN7MLtHGwa0aEpLIfYXf/sMS0sLPDw8cHR0xN7ORl4Nn5/Evymd2rXgxzY/0EF0wj5dO3NE2xTj5CLMC+rQL6zDtKQe+/I6rEvrMS6owfBNFQY5leimlnLI0pOl2/aLDN6X0UPede9TG1dxS/sMiO5cHO5DcYgHed4OZNkZURbqSfWDYKrvBlN+25/iIDcK/Rx542JEpp0uTzWPEXl+Jw6b52O8fDrnFYZxcupg9o7pK7p2Dxb3aYdil5a0+eYrvvnicw4e3I/lTQsM9G+gr6eL3g1dSWyJd3T44Rt+bN2C+bsOc9w7ApMiuFkMN/JrsSyuwbm8FuuiWkzF/s3Caizf1mCZX4VpbgV6yW+5/iCRpVv3MkFMJBdMn0BehDf50SGQLaLH6xfUy+JHVCiVYZ6UeltRGexO9T2xHxFI0S2RtX0cyHMxI9P2Bo9Flw8/vQ3bjfMwWDqVc0LsE1MGsXd0H7aP6M6SPu2Z3bUVbb9pwjdffsnKlcvQ1rjO1SuXuHZVVpclsf8gNomSvdgfKuaXh//1qD3O4fLrKjRya9HJrUE3uxqr4mrcy2vwLKvDMKsKSyG0s9i3Lxad+20txqJj3xRlJeQ2yyhBNy6Dq0GPWLN8Kbn3g6jNEdEjL5uHphZcO3QRNyNHEnyCqY2NpfbFC2rjnlBxL0x0clkMuUmSmQZPdS9wR3U/t4Line truncated
 

docs/contributing/design_documents/index.md

@@ -1,9 +1,12 @@
 # Design Documents
 
-This section contains design documents and architecture specifications for vLLM-Omni.
+This section contains design documents and architecture specifications for vLLM-omni.
 
 ## Architecture Documents
 
+- [vLLM-omni Software Design Document](vllm_omni_design.md)
 - [Architecture Overview](architecture_overview.md)
-- [vLLM-Omni Software Design Document](vllm_omni_design.md)
-- [Multi-Request Streaming](mrs_design.md)
+
+## API Design Documentation
+
+- [vLLM-omni API Documentation](api_design_doc.md)

docs/contributing/design_documents/vllm_omni_design.md

@@ -2,7 +2,7 @@
 
 ## Overview
 
-vLLM-Omni is a multi-modality extension for vLLM that supports non-autoregressive structures and non-textual outputs. This document outlines the key software abstractions, APIs, and dependencies for the system, designed to maximize reuse of vLLM's proven architecture.
+vLLM-omni is a multi-modality extension for vLLM that supports non-autoregressive structures and non-textual outputs. This document outlines the key software abstractions, APIs, and dependencies for the system, designed to maximize reuse of vLLM's proven architecture.
 
 ## Architecture Principles
 
@@ -14,7 +14,7 @@ vLLM-Omni is a multi-modality extension for vLLM that supports non-autoregressiv
 
 ## Key Data Flow
 
-API Server --> OmniLLM/AsyncOmni (New, including multi engines) --> LLMEngine/AsyncLLM --> Engine Core
+API Server --> OmniLLM/AsyncOmniLLM (New, including multi engines) --> LLMEngine/AsyncLLM --> Engine Core
  --> Scheduler (New one for DiT) --> Executor (New one for diffusers) --> Worker (New one for DiT)
  --> ModelRunner (New one for AR hiddenstate, New one for DiT) --> RequestState --> OutputProcessoer (New one for final multimodal output)
 
@@ -36,16 +36,16 @@ vllm serve Qwen/Qwen2.5-Omni-7B --omni --port 8000
 #### Design Architecture
 ```mermaid
 graph TD
-    A[vllm serve model --omni] --> B[vLLM-Omni CLI Wrapper]
+    A[vllm serve model --omni] --> B[vLLM-omni CLI Wrapper]
     B --> C{Detect --omni flag}
     C -->|Yes| D[Parse OmniConfig]
     C -->|No| E[Forward to vLLM CLI]
-    D --> F[Initialize AsyncOmni]
+    D --> F[Initialize AsyncOmniLLM]
     F --> G[Start omni Server]
     G --> H[Multi-stage Processing]
     E --> I[Standard vLLM Pipeline]
 
-    subgraph "vLLM-Omni Components"
+    subgraph "vLLM-omni Components"
         F
         G
         H
@@ -68,7 +68,7 @@ def main():
     """Main CLI entry point that intercepts vLLM commands"""
     # Check if --omni flag is present
     if "--omni" in sys.argv:
-        # Remove --omni flag and process with vLLM-Omni
+        # Remove --omni flag and process with vLLM-omni
         omni_args = [arg for arg in sys.argv[1:] if arg != "--omni"]
         omni_serve = OmniServeCommand()
         omni_serve.run(omni_args)
@@ -209,7 +209,7 @@ Similar to OmniLLM in offline inference, add some asynchronous processing, refer
 from vllm.v1.engine.async_llm import AsyncLLM
 
 
-class AsyncOmni(AsyncLLM):
+class AsyncOmniLLM(AsyncLLM):
     """Extended AsyncLLM supporting multiple engines and stage-based processing"""
 
     def __init__(self, stage_configs: List[StageConfig]):
@@ -449,15 +449,15 @@ AsyncLLM.add_request()
 ↓
 OutputProcessor.add_request()
 ↓
-RequestState.from_new_request() → create request state
+RequestState.from_new_request() → 创建请求状态
 
-AsyncLLM.__init__() / AsyncLLM.generate() / AsyncLLM.encode() →  create a background loop that continuously pulls output from EngineCore
+AsyncLLM.__init__() / AsyncLLM.generate() / AsyncLLM.encode() →  创建一个Background loop 持续从EngineCore获取输出
 ↓
-OutputProcessor.process_outputs() → update state and process outputs
+OutputProcessor.process_outputs() → 更新状态并处理输出
 ↓
-RequestState.make_request_output() → convert to the final output, formatted as RequestOutput or PoolingRequestOutput
+RequestState.make_request_output() → 转换为最终输出,格式为RequestOutput或者PoolingRequestOutput
 ↓
-RequestOutputCollector.put() → push to the queue (AsyncLLM)
+RequestOutputCollector.put() → 推送到队列（AsyncLLM）
 ```
 
 Need to add implementation for an existing method

docs/contributing/model/README.md

@@ -1,2 +0,0 @@
-# Adding a New Model
-Stay tuned for guidance on how to add a new model on vLLM-Omni!

docs/examples/README.md

@@ -1,6 +0,0 @@
-# Examples
-
-vLLM-Omni's examples are split into two categories:
-
-- If you are using vLLM-Omni from within Python code, see the *Offline Inference* section.
-- If you are using vLLM-Omni from an HTTP application or client, see the *Online Serving* section.

docs/getting_started/installation/.nav.yml

@@ -1,4 +0,0 @@
-nav:
-  - README.md
-  - gpu.md
-  - npu.md

docs/getting_started/installation/README.md

@@ -1,7 +0,0 @@
-# Installation
-
-vLLM supports the following hardware platforms:
-
-- [GPU](gpu.md)
-    - [NVIDIA CUDA](gpu.md)
-- [NPU](npu.md)

docs/getting_started/installation/gpu.md

@@ -1,43 +0,0 @@
-# GPU
-
-vLLM-Omni is a Python library that supports the following GPU variants. The library itself mainly contains python implementations for framework and models.
-
-## Requirements
-
-- OS: Linux
-- Python: 3.12
-
-!!! note
-    vLLM-Omni is currently not natively supported on Windows.
-
-=== "NVIDIA CUDA"
-
-    --8<-- "docs/getting_started/installation/gpu/cuda.inc.md:requirements"
-
-## Set up using Python
-
-### Create a new Python environment
-
---8<-- "docs/getting_started/installation/python_env_setup.inc.md"
-
-### Pre-built wheels
-
-=== "NVIDIA CUDA"
-
-    --8<-- "docs/getting_started/installation/gpu/cuda.inc.md:pre-built-wheels"
-
-[](){ #build-from-source }
-
-### Build wheel from source
-
-=== "NVIDIA CUDA"
-
-    --8<-- "docs/getting_started/installation/gpu/cuda.inc.md:build-wheel-from-source"
-
-## Set up using Docker
-
-### Pre-built images
-
-=== "NVIDIA CUDA"
-
-    --8<-- "docs/getting_started/installation/gpu/cuda.inc.md:pre-built-images"

docs/getting_started/installation/gpu/cuda.inc.md

@@ -1,96 +0,0 @@
-# --8<-- [start:requirements]
-
-- GPU: compute capability 7.0 or higher (e.g., V100, T4, RTX20xx, A100, L4, H100, etc.)
-
-# --8<-- [end:requirements]
-# --8<-- [start:set-up-using-python]
-
-vLLM-Omni depends vLLM. So please follow instructions below mainly for vLLM.
-
-!!! note
-    PyTorch installed via `conda` will statically link `NCCL` library, which can cause issues when vLLM tries to use `NCCL`. See <gh-issue:8420> for more details.
-
-In order to be performant, vLLM has to compile many cuda kernels. The compilation unfortunately introduces binary incompatibility with other CUDA versions and PyTorch versions, even for the same PyTorch version with different building configurations.
-
-Therefore, it is recommended to install vLLM and vLLM-Omni with a **fresh new** environment. If either you have a different CUDA version or you want to use an existing PyTorch installation, you need to build vLLM from source. See [build-from-source-vllm](https://docs.vllm.ai/en/stable/getting_started/installation/gpu/#build-wheel-from-source) for more details.
-
-# --8<-- [start:pre-built-wheels]
-
-#### Installation of vLLM
-
-vLLM-Omni is built based on vLLM v0.11.0. Please install it with command below.
-```bash
-uv pip install vllm==0.11.0 --torch-backend=auto
-```
-
-#### Installation of vLLM-Omni
-
-```bash
-uv pip install vllm-omni
-```
-
-# --8<-- [end:pre-built-wheels]
-
-# --8<-- [start:build-wheel-from-source]
-
-#### Installation of vLLM
-If you do not need to modify source code of vLLM, you can directly install the stable 0.11.0 release version of the library
-
-```bash
-uv pip install vllm==0.11.0 --torch-backend=auto
-```
-
-#### Installation of vLLM-Omni
-Install additional requirements for vLLM-Omni
-```bash
-git clone https://github.com/vllm-project/vllm-omni.git
-cd vllm-omni
-uv pip install -e .
-```
-
-<details><summary>(Optional) Installation of vLLM from source</summary>
-If you want to check, modify or debug with source code of vLLM, install the library from source with the following instructions:
-
-```bash
-git clone https://github.com/vllm-project/vllm.git
-cd vllm
-git checkout v0.11.0
-```
-Set up environment variables to get pre-built wheels. If there are internet problems, just download the whl file manually. And set `VLLM_PRECOMPILED_WHEEL_LOCATION` as your local absolute path of whl file.
-```bash
-export VLLM_PRECOMPILED_WHEEL_LOCATION=https://github.com/vllm-project/vllm/releases/download/v0.11.0/vllm-0.11.0-cp38-abi3-manylinux1_x86_64.whl
-```
-Install vllm with command below (If you have no existing PyTorch).
-```bash
-uv pip install --editable .
-```
-Install vllm with command below (If you already have PyTorch).
-```bash
-python use_existing_torch.py
-uv pip install -r requirements/build.txt
-uv pip install --no-build-isolation --editable .
-```
-</details>
-
-# --8<-- [end:build-wheel-from-source]
-
-
-# --8<-- [start:pre-built-images]
-
-vLLM-Omni offers an official docker image for deployment. These images are built on top of vLLM docker images and available on Docker Hub as [vllm/vllm-omni](https://hub.docker.com/r/vllm/vllm-omni/tags). The version of vLLM-Omni indicates which release of vLLM it is based on.
-
-Here's an example deployment command that has been verified on 2 x H100's:
-```bash
-docker run --runtime nvidia --gpus 2 \
-    -v ~/.cache/huggingface:/root/.cache/huggingface \
-    --env "HF_TOKEN=$HF_TOKEN" \
-    -p 8091:8091 \
-    --ipc=host \
-    vllm/vllm-omni:v0.11.0rc1 \
-    --model Qwen/Qwen3-Omni-30B-A3B-Instruct --port 8091
-```
-
-!!! tip
-    You can use this docker image to serve models the same way you would with in vLLM! To do so, make sure you overwrite the default entrypoint (`vllm serve --omni`) which works only for models supported in the vLLM-Omni project.
-
-# --8<-- [end:pre-built-images]

docs/getting_started/installation/npu.md

@@ -1,23 +0,0 @@
-# NPU
-
-vLLM-Omni supports NPU through the vLLM Ascend Plugin (vllm-ascend). This is a community maintained hardware plugin for running vLLM on NPU.
-
-## Requirements
-
-- OS: Linux
-- Python: 3.12
-
-!!! note
-    vLLM-Omni is currently not natively supported on Windows.
-
-=== "NPU"
-
-    --8<-- "docs/getting_started/installation/npu/npu.inc.md:requirements"
-
-## Installation
-
-### Recommended
-
-=== "NPU"
-
-    --8<-- "docs/getting_started/installation/npu/npu.inc.md:installation"

docs/getting_started/installation/npu/npu.inc.md

@@ -1,49 +0,0 @@
-# --8<-- [start:requirements]
-
-For detailed hardware and software requirements, please refer to the [vllm-ascend installation documentation](https://docs.vllm.ai/projects/ascend/en/latest/installation.html).
-
-# --8<-- [end:requirements]
-# --8<-- [start:installation]
-
-The recommended way to use vLLM-Omni on NPU is through the vllm-ascend pre-built Docker images:
-
-```bash
-# Update DEVICE according to your NPUs (/dev/davinci[0-7])
-export DEVICE0=/dev/davinci0
-export DEVICE1=/dev/davinci1
-# Update the vllm-ascend image
-# Atlas A2:
-# export IMAGE=quay.io/ascend/vllm-ascend:v0.11.0rc2
-# Atlas A3:
-# export IMAGE=quay.io/ascend/vllm-ascend:v0.11.0rc2-a3
-export IMAGE=quay.io/ascend/vllm-ascend:v0.11.0rc2
-docker run --rm \
-    --name vllm-omni-npu \
-    --shm-size=1g \
-    --device $DEVICE0 \
-    --device $DEVICE1 \
-    --device /dev/davinci_manager \
-    --device /dev/devmm_svm \
-    --device /dev/hisi_hdc \
-    -v /usr/local/dcmi:/usr/local/dcmi \
-    -v /usr/local/bin/npu-smi:/usr/local/bin/npu-smi \
-    -v /usr/local/Ascend/driver/lib64/:/usr/local/Ascend/driver/lib64/ \
-    -v /usr/local/Ascend/driver/version.info:/usr/local/Ascend/driver/version.info \
-    -v /etc/ascend_install.info:/etc/ascend_install.info \
-    -v /root/.cache:/root/.cache \
-    -p 8000:8000 \
-    -it $IMAGE bash
-
-# Inside the container, install vLLM-Omni from source
-cd /vllm-workspace
-git clone https://github.com/vllm-project/vllm-omni.git
-cd vllm-omni
-pip install -v -e .
-export VLLM_WORKER_MULTIPROC_METHOD=spawn
-```
-
-The default workdir is `/workspace`, with vLLM, vLLM-Ascend and vLLM-Omni code placed in `/vllm-workspace` installed in development mode.
-
-For other installation methods (pip installation, building from source, custom Docker builds), please refer to the [vllm-ascend installation guide](https://docs.vllm.ai/projects/ascend/en/latest/installation.html).
-
-# --8<-- [end:installation]

docs/getting_started/installation/python_env_setup.inc.md

@@ -1,6 +0,0 @@
-It's recommended to use [uv](https://docs.astral.sh/uv/), a very fast Python environment manager, to create and manage Python environments. Please follow the [documentation](https://docs.astral.sh/uv/#getting-started) to install `uv`. After installing `uv`, you can create a new Python environment using the following commands:
-
-```bash
-uv venv --python 3.12 --seed
-source .venv/bin/activate
-```

docs/getting_started/quickstart.md

@@ -1,23 +0,0 @@
-# Quickstart
-
-This guide will help you quickly get started with vLLM-Omni to perform:
-
-- Offline batched inference
-- Online serving using OpenAI-compatible server
-
-## Prerequisites
-
-- OS: Linux
-- Python: 3.12
-
-## Installation
-
-Please refer to [installation](installation/README.md)
-
-## Offline Inference
-
-Please refer to [offline inference](../user_guide/examples/offline_inference/qwen2_5_omni.md)
-
-## Online Serving with OpenAI-Completions API
-
-Please refer to [online serving](../user_guide/examples/online_serving/qwen2_5_omni.md)

docs/index.md

@@ -0,0 +1,32 @@
+# Welcome to vLLM-omni
+
+**vLLM-omni** extends vLLM capabilities to support multi-modality model inference and serving, particularly focusing on non-autoregressive structures and non-textual outputs. It aims to support:
+
+- **Multi-modal Models**: Text, image, video, audio, and sensor data processing
+- **Non-autoregressive Architectures**: Diffusion Transformers (DiT) and other parallel generation models
+- **Heterogeneous Outputs**: Beyond traditional text generation to multimodal outputs
+
+## <span class="twemoji">🎯</span> Key Features
+
+### Multi-Engine Support
+
+- **Autoregressive Engine**: Traditional text generation with enhanced KV-caching
+- **Diffusion Engine**: Support for DiT models and iterative generation
+- **Hybrid Engine**: Combined AR+DiT processing pipelines
+
+### Modality Processing
+
+- **Text**: Advanced tokenization and embedding generation
+- **Image**: Vision encoder integration (CLIP, etc.)
+- **Audio**: Speech processing and audio embedding
+
+## <span class="twemoji">🤖</span> Supported Models
+
+- Qwen-omni (Thinker-Talker-Codec structure)
+- HunyunaImage 3.0 (Ongoing)
+- Qwen-Image (Ongoing)
+
+## <span class="twemoji">📚</span> Documentation Navigation
+
+- To run open-source models on vLLM-Omni, we recommend starting with the [:material-code-tags: User Quide](user_guide/getting_started/quickstart.md)
+- To develop and contribute to vLLM-Omni, we recommend starting with the [:material-tools: Developer Guide](contributing/README.md)

docs/mkdocs/hooks/generate_api_readme.py

@@ -1,268 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
-"""
-Hook to automatically generate docs/api/README.md from the codebase.
-
-This script scans the vllm_omni module for public classes and functions,
-categorizes them, and generates a summary README file.
-"""
-
-import ast
-import logging
-from pathlib import Path
-
-logger = logging.getLogger("mkdocs")
-
-ROOT_DIR = Path(__file__).parent.parent.parent.parent
-API_README_PATH = ROOT_DIR / "docs" / "api" / "README.md"
-
-# Category mappings: module prefix -> category name and description
-CATEGORIES = {
-    "entrypoints": {
-        "name": "Entry Points",
-        "description": "Main entry points for vLLM-Omni inference and serving.",
-    },
-    "inputs": {
-        "name": "Inputs",
-        "description": "Input data structures for multi-modal inputs.",
-    },
-    "engine": {
-        "name": "Engine",
-        "description": "Engine classes for offline and online inference.",
-    },
-    "core": {
-        "name": "Core",
-        "description": "Core scheduling and caching components.",
-    },
-    "model_executor": {
-        "name": "Model Executor",
-        "description": "Model execution components.",
-    },
-    "config": {
-        "name": "Configuration",
-        "description": "Configuration classes.",
-    },
-    "worker": {
-        "name": "Workers",
-        "description": "Worker classes and model runners for distributed inference.",
-    },
-}
-
-
-class APIVisitor(ast.NodeVisitor):
-    """AST visitor to extract public classes and module-level functions."""
-
-    def __init__(self, module_path: str):
-        self.module_path = module_path
-        self.classes: list[str] = []
-        self.functions: list[str] = []
-        self._class_stack: list[str] = []  # Track nested class definitions
-
-    def visit_ClassDef(self, node: ast.ClassDef) -> None:
-        """Visit class definitions."""
-        if not node.name.startswith("_"):
-            self.classes.append(f"{self.module_path}.{node.name}")
-        # Track that we're entering a class
-        self._class_stack.append(node.name)
-        self.generic_visit(node)
-        # Remove from stack when done visiting
-        self._class_stack.pop()
-
-    def visit_FunctionDef(self, node: ast.FunctionDef) -> None:
-        """Visit function definitions - only collect module-level functions."""
-        # Only collect if we're not inside a class (stack is empty)
-        if not self._class_stack and not node.name.startswith("_"):
-            self.functions.append(f"{self.module_path}.{node.name}")
-        self.generic_visit(node)
-
-    def visit_AsyncFunctionDef(self, node: ast.AsyncFunctionDef) -> None:
-        """Visit async function definitions - only collect module-level functions."""
-        # Only collect if we're not inside a class (stack is empty)
-        if not self._class_stack and not node.name.startswith("_"):
-            self.functions.append(f"{self.module_path}.{node.name}")
-        self.generic_visit(node)
-
-
-def parse_file_for_symbols(file_path: Path, module_path: str) -> tuple[list[str], list[str]]:
-    """
-    Parse a Python file and extract public classes and functions.
-
-    Returns:
-        Tuple of (classes, functions)
-    """
-    try:
-        # If this is __init__.py, use parent module path
-        if file_path.name == "__init__.py":
-            # Remove .__init__ from module path
-            if module_path.endswith(".__init__"):
-                module_path = module_path[:-9]
-
-        with open(file_path, encoding="utf-8") as f:
-            content = f.read()
-
-        tree = ast.parse(content, filename=str(file_path))
-        visitor = APIVisitor(module_path)
-        visitor.visit(tree)
-
-        return visitor.classes, visitor.functions
-    except Exception as e:
-        logger.debug(f"Could not parse {file_path}: {e}")
-        return [], []
-
-
-def scan_package(package_name: str = "vllm_omni") -> dict[str, list[str]]:
-    """
-    Scan the vllm_omni package and categorize public symbols.
-
-    Returns:
-        Dict mapping category names to lists of symbol full names
-    """
-    categorized: dict[str, list[str]] = {cat["name"]: [] for cat in CATEGORIES.values()}
-
-    try:
-        # Find the package directory
-        package_path = ROOT_DIR / package_name
-        if not package_path.exists():
-            logger.warning(f"Package path not found: {package_path}")
-            return categorized
-
-        # Walk through all Python files
-        for py_file in package_path.rglob("*.py"):
-            # Skip __init__.py and private modules
-            if py_file.name.startswith("_") and py_file.name != "__init__.py":
-                continue
-
-            # Get module path
-            relative_path = py_file.relative_to(ROOT_DIR)
-            module_path = str(relative_path.with_suffix("")).replace("/", ".").replace("\\", ".")
-
-            # Handle __init__.py - use parent module path
-            if py_file.name == "__init__.py":
-                # Remove .__init__ from module path
-                if module_path.endswith(".__init__"):
-                    module_path = module_path[:-9]
-
-            # Determine category from module path
-            category = None
-            for prefix, cat_info in CATEGORIES.items():
-                if prefix in module_path:
-                    category = cat_info["name"]
-                    break
-
-            if not category:
-                continue
-
-            # Parse file for symbols
-            classes, functions = parse_file_for_symbols(py_file, module_path)
-
-            # Filter out internal implementation classes
-            # Skip classes that look like internal components (DiT layers, etc.)
-            internal_patterns = [
-                "Block",
-                "Layer",
-                "Net",
-                "Embedding",
-                "Norm",
-                "Activation",
-                "Solver",
-                "Pooling",
-                "Attention",
-                "MLP",
-                "DecoderLayer",
-                "InputEmbedding",
-                "TimestepEmbedding",
-                "CodecEmbedding",
-                "DownSample",
-                "UpSample",
-                "Res2Net",
-                "SqueezeExcitation",
-                "TimeDelay",
-                "TorchActivation",
-                "SnakeBeta",
-                "SinusPosition",
-                "RungeKutta",
-                "AMPBlock",
-                "AdaLayerNorm",
-            ]
-
-            # Add classes (filter out internal ones)
-            for class_name in classes:
-                class_short_name = class_name.split(".")[-1]
-                # Skip if it matches internal patterns (unless it's a main model class)
-                if any(pattern in class_short_name for pattern in internal_patterns):
-                    # But include main model classes
-                    if not any(
-                        main_class in class_short_name
-                        for main_class in [
-                            "ForConditionalGeneration",
-                            "Model",
-                            "Registry",
-                            "Worker",
-                            "Runner",
-                            "Scheduler",
-                            "Manager",
-                            "Processor",
-                            "Config",
-                        ]
-                    ):
-                        continue
-                categorized[category].append(class_name)
-
-            # Add important functions (parse, preprocess, etc.)
-            for func_name in functions:
-                # Include functions that match certain patterns
-                if any(keyword in func_name.lower() for keyword in ["parse", "preprocess"]):
-                    categorized[category].append(func_name)
-
-        # Sort symbols within each category
-        for category in categorized:
-            categorized[category].sort()
-
-    except Exception as e:
-        logger.error(f"Error scanning package: {e}", exc_info=True)
-
-    return categorized
-
-
-def generate_readme(categorized: dict[str, list[str]]) -> str:
-    """Generate the API README markdown content."""
-    lines = ["# Summary", ""]
-
-    # Generate sections for each category
-    for prefix, cat_info in CATEGORIES.items():
-        category_name = cat_info["name"]
-        description = cat_info["description"]
-        symbols = categorized.get(category_name, [])
-
-        if not symbols:
-            continue
-
-        lines.append(f"## {category_name}")
-        lines.append("")
-        lines.append(description)
-        lines.append("")
-
-        for symbol in symbols:
-            lines.append(f"- [{symbol}][]")
-
-        lines.append("")
-
-    return "\n".join(lines)
-
-
-def on_startup(command, dirty: bool):
-    """MkDocs hook entry point."""
-    logger.info("Generating API README documentation")
-
-    # Scan the package
-    categorized = scan_package()
-
-    # Generate README content
-    content = generate_readme(categorized)
-
-    # Write to file
-    API_README_PATH.parent.mkdir(parents=True, exist_ok=True)
-    with open(API_README_PATH, "w", encoding="utf-8") as f:
-        f.write(content)
-
-    logger.info(f"API README generated: {API_README_PATH.relative_to(ROOT_DIR)}")

docs/mkdocs/hooks/generate_examples.py

@@ -1,5 +1,5 @@
 # SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-omni project
 import itertools
 import logging
 from dataclasses import dataclass, field
@@ -7,7 +7,6 @@ from pathlib import Path
 from typing import Literal
 
 import regex as re
-import yaml
 
 logger = logging.getLogger("mkdocs")
 
@@ -15,7 +14,6 @@ ROOT_DIR = Path(__file__).parent.parent.parent.parent
 ROOT_DIR_RELATIVE = "../../../../.."
 EXAMPLE_DIR = ROOT_DIR / "examples"
 EXAMPLE_DOC_DIR = ROOT_DIR / "docs/user_guide/examples"
-NAV_FILE = ROOT_DIR / "docs/.nav.yml"
 
 
 def fix_case(text: str) -> str:
@@ -30,6 +28,7 @@ def fix_case(text: str) -> str:
         "lora": "LoRA",
         "rlhf": "RLHF",
         "vllm": "vLLM",
+        "omni": "Omni",
         "openai": "OpenAI",
         "lmcache": "LMCache",
         "multilora": "MultiLoRA",
@@ -103,40 +102,20 @@ class Example:
         """  # noqa: E501
         if self.path.is_file():
             return []
-        # Binary file extensions to exclude
-        binary_extensions = {
-            ".wav",
-            ".mp3",
-            ".mp4",
-            ".avi",
-            ".mov",
-            ".mkv",  # Audio/Video
-            ".png",
-            ".jpg",
-            ".jpeg",
-            ".gif",
-            ".bmp",
-            ".ico",
-            ".svg",  # Images
-            ".pdf",
-            ".zip",
-            ".tar",
-            ".gz",
-            ".bz2",
-            ".xz",  # Archives/Documents
-            ".exe",
-            ".so",
-            ".dll",
-            ".dylib",  # Binaries
-            ".bin",
-            ".dat",
-            ".db",
-            ".sqlite",  # Data files
+        # Text file extensions that can be safely included
+        text_extensions = {
+            ".py",
+            ".md",
+            ".sh",
+            ".yaml",
+            ".yml",
+            ".json",
+            ".txt",
+            ".toml",
+            ".cfg",
+            ".ini",
         }
-
-        def is_other_file(file: Path) -> bool:
-            return file.is_file() and file != self.main_file and file.suffix.lower() not in binary_extensions
-
+        is_other_file = lambda file: (file.is_file() and file != self.main_file and file.suffix in text_extensions)  # noqa: E731
         return [file for file in self.path.rglob("*") if is_other_file(file)]
 
     def determine_title(self) -> str:
@@ -192,10 +171,11 @@ class Example:
         code_fence = "``````"
 
         if self.is_code:
-            main_file_rel = self.main_file.relative_to(ROOT_DIR)
-            content += f'{code_fence}{self.main_file.suffix[1:]}\n--8<-- "{main_file_rel}"\n{code_fence}\n'
+            # Use relative path from ROOT_DIR for snippet inclusion
+            main_file_rel_path = self.main_file.relative_to(ROOT_DIR)
+            content += f'{code_fence}{self.main_file.suffix[1:]}\n--8<-- "{main_file_rel_path}"\n{code_fence}\n'
         else:
-            with open(self.main_file) as f:
+            with open(self.main_file, encoding="utf-8") as f:
                 # Skip the title from md snippets as it's been included above
                 main_content = f.readlines()[1:]
             content += self.fix_relative_links("".join(main_content))
@@ -206,98 +186,18 @@ class Example:
 
         content += "## Example materials\n\n"
         for file in sorted(self.other_files):
+            # Use relative path from ROOT_DIR for snippet inclusion
+            file_rel_path = file.relative_to(ROOT_DIR)
             content += f'??? abstract "{file.relative_to(self.path)}"\n'
             if file.suffix != ".md":
                 content += f"    {code_fence}{file.suffix[1:]}\n"
-            content += f'    --8<-- "{file.relative_to(ROOT_DIR)}"\n'
+            content += f'    --8<-- "{file_rel_path}"\n'
             if file.suffix != ".md":
                 content += f"    {code_fence}\n"
 
         return content
 
 
-def update_nav_file(examples: list[Example]):
-    """
-    Update the .nav.yml file to include all generated examples.
-    This function completely regenerates the examples section based on the actual
-    folder structure, ensuring consistency between the examples folder and nav file.
-
-    Args:
-        examples: List of Example objects that have been generated
-    """
-    if not NAV_FILE.exists():
-        logger.warning("Navigation file not found: %s", NAV_FILE)
-        return
-
-    # Read the current nav file
-    with open(NAV_FILE, encoding="utf-8") as f:
-        nav_data = yaml.safe_load(f) or {}
-
-    nav_list = nav_data.get("nav", [])
-
-    # Find the "User Guide" section
-    user_guide_idx = None
-    examples_idx = None
-    for i, item in enumerate(nav_list):
-        if isinstance(item, dict) and "User Guide" in item:
-            user_guide_idx = i
-            user_guide_content = item["User Guide"]
-            # Find the "Examples" subsection
-            for j, subitem in enumerate(user_guide_content):
-                if isinstance(subitem, dict) and "Examples" in subitem:
-                    examples_idx = j
-                    break
-            break
-
-    if user_guide_idx is None or examples_idx is None:
-        logger.warning("Could not find 'User Guide' -> 'Examples' section in nav file")
-        return
-
-    # Get existing Examples section to preserve non-example items (like README.md)
-    existing_examples_content = nav_list[user_guide_idx]["User Guide"][examples_idx]["Examples"]
-
-    # Preserve string items (like "examples/README.md") that are not example categories
-    preserved_items = [
-        item
-        for item in existing_examples_content
-        if isinstance(item, str) and not item.startswith("user_guide/examples/")
-    ]
-
-    # Group examples by category
-    examples_by_category = {}
-    for example in examples:
-        category = example.category
-        if category not in examples_by_category:
-            examples_by_category[category] = []
-        examples_by_category[category].append(example)
-
-    # Build the new Examples section - start with preserved items
-    examples_section = preserved_items.copy()
-
-    # Add examples grouped by category, sorted by category name
-    for category in sorted(examples_by_category.keys()):
-        category_examples = sorted(examples_by_category[category], key=lambda e: e.path.stem)
-        category_items = []
-        for example in category_examples:
-            doc_path = EXAMPLE_DOC_DIR / example.category / f"{example.path.stem}.md"
-            rel_path = doc_path.relative_to(ROOT_DIR / "docs")
-            category_items.append({example.title: str(rel_path)})
-
-        if category_items:
-            # Format category name (e.g., "offline_inference" -> "Offline Inference")
-            category_title = fix_case(category.replace("_", " ").title())
-            examples_section.append({category_title: category_items})
-
-    # Update the nav structure
-    nav_list[user_guide_idx]["User Guide"][examples_idx]["Examples"] = examples_section
-
-    # Write back to file
-    nav_data["nav"] = nav_list
-    with open(NAV_FILE, "w", encoding="utf-8") as f:
-        yaml.dump(nav_data, f, default_flow_style=False, sort_keys=False, allow_unicode=True)
-    logger.info("Updated navigation file: %s", NAV_FILE.relative_to(ROOT_DIR))
-
-
 def on_startup(command: Literal["build", "gh-deploy", "serve"], dirty: bool):
     logger.info("Generating example documentation")
     logger.debug("Root directory: %s", ROOT_DIR.resolve())
@@ -311,7 +211,7 @@ def on_startup(command: Literal["build", "gh-deploy", "serve"], dirty: bool):
     categories = sorted(p for p in EXAMPLE_DIR.iterdir() if p.is_dir())
 
     examples = []
-    glob_patterns = ["*.py", "*.md", "*.sh"]
+    glob_patterns = ["*.py", "*.md"]
     # Find categorised examples
     for category in categories:
         globs = [category.glob(pattern) for pattern in glob_patterns]
@@ -331,6 +231,3 @@ def on_startup(command: Literal["build", "gh-deploy", "serve"], dirty: bool):
         with open(doc_path, "w+", encoding="utf-8") as f:
             f.write(example.generate())
         logger.debug("Example generated: %s", doc_path.relative_to(ROOT_DIR))
-
-    # Update the navigation file
-    update_nav_file(examples)

docs/mkdocs/hooks/url_schemes.py

@@ -1,121 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-"""
-This is basically a port of MyST parser’s external URL resolution mechanism
-(https://myst-parser.readthedocs.io/en/latest/syntax/cross-referencing.html#customising-external-url-resolution)
-to work with MkDocs.
-
-It allows Markdown authors to use GitHub shorthand links like:
-
-  - [Text](gh-issue:123)
-  - <gh-pr:456>
-  - [File](gh-file:path/to/file.py#L10)
-
-These are automatically rewritten into fully qualified GitHub URLs pointing to
-issues, pull requests, files, directories, or projects in the
-`vllm-project/vllm-omni` repository.
-
-The goal is to simplify cross-referencing common GitHub resources
-in project docs.
-"""
-
-import regex as re
-from mkdocs.config.defaults import MkDocsConfig
-from mkdocs.structure.files import Files
-from mkdocs.structure.pages import Page
-
-
-def on_page_markdown(markdown: str, *, page: Page, config: MkDocsConfig, files: Files) -> str:
-    """
-    Custom MkDocs plugin hook to rewrite special GitHub reference links
-    in Markdown.
-
-    This function scans the given Markdown content for specially formatted
-    GitHub shorthand links, such as:
-      - `[Link text](gh-issue:123)`
-      - `<gh-pr:456>`
-
-    And rewrites them into fully-qualified GitHub URLs with GitHub icons:
-      - `[:octicons-mark-github-16: Link text](https://github.com/vllm-project/vllm/issues/123)`
-      - `[:octicons-mark-github-16: Pull Request #456](https://github.com/vllm-project/vllm/pull/456)`
-
-    Supported shorthand types:
-      - `gh-issue`
-      - `gh-pr`
-      - `gh-project`
-      - `gh-dir`
-      - `gh-file`
-
-    Args:
-        markdown (str): The raw Markdown content of the page.
-        page (Page): The MkDocs page object being processed.
-        config (MkDocsConfig): The MkDocs site configuration.
-        files (Files): The collection of files in the MkDocs build.
-
-    Returns:
-        str: The updated Markdown content with GitHub shorthand links replaced.
-    """
-    gh_icon = ":octicons-mark-github-16:"
-    gh_url = "https://github.com"
-    repo_url = f"{gh_url}/vllm-project/vllm-omni"
-    org_url = f"{gh_url}/orgs/vllm-project"
-
-    # Mapping of shorthand types to their corresponding GitHub base URLs
-    urls = {
-        "issue": f"{repo_url}/issues",
-        "pr": f"{repo_url}/pull",
-        "project": f"{org_url}/projects",
-        "dir": f"{repo_url}/tree/main",
-        "file": f"{repo_url}/blob/main",
-    }
-
-    # Default title prefixes for auto links
-    titles = {
-        "issue": "Issue #",
-        "pr": "Pull Request #",
-        "project": "Project #",
-        "dir": "",
-        "file": "",
-    }
-
-    # Regular expression to match GitHub shorthand links
-    scheme = r"gh-(?P<type>.+?):(?P<path>.+?)(#(?P<fragment>.+?))?"
-    inline_link = re.compile(r"\[(?P<title>[^\[]+?)\]\(" + scheme + r"\)")
-    auto_link = re.compile(f"<{scheme}>")
-
-    def replace_inline_link(match: re.Match) -> str:
-        """
-        Replaces a matched inline-style GitHub shorthand link
-        with a full Markdown link.
-
-        Example:
-            [My issue](gh-issue:123) → [:octicons-mark-github-16: My issue](https://github.com/vllm-project/vllm/issues/123)
-        """
-        url = f"{urls[match.group('type')]}/{match.group('path')}"
-        if fragment := match.group("fragment"):
-            url += f"#{fragment}"
-
-        return f"[{gh_icon} {match.group('title')}]({url})"
-
-    def replace_auto_link(match: re.Match) -> str:
-        """
-        Replaces a matched autolink-style GitHub shorthand
-        with a full Markdown link.
-
-        Example:
-            <gh-pr:456> → [:octicons-mark-github-16: Pull Request #456](https://github.com/vllm-project/vllm/pull/456)
-        """
-        type = match.group("type")
-        path = match.group("path")
-        title = f"{titles[type]}{path}"
-        url = f"{urls[type]}/{path}"
-        if fragment := match.group("fragment"):
-            url += f"#{fragment}"
-
-        return f"[{gh_icon} {title}]({url})"
-
-    # Replace both inline and autolinks
-    markdown = inline_link.sub(replace_inline_link, markdown)
-    markdown = auto_link.sub(replace_auto_link, markdown)
-
-    return markdown

docs/mkdocs/javascript/edit_and_feedback.js

@@ -1,47 +0,0 @@
-/**
- * edit_and_feedback.js
- *
- * Enhances MkDocs Material docs pages by:
- *
- * 1. Adding a "Question? Give us feedback" link
- *    below the "Edit" button.
- *
- *    - The link opens a GitHub issue with a template,
- *      auto-filled with the current page URL and path.
- *
- * 2. Ensuring the edit button opens in a new tab
- *    with target="_blank" and rel="noopener".
- */
-document.addEventListener("DOMContentLoaded", function () {
-    const url = window.location.href;
-    const page = document.body.dataset.mdUrl || location.pathname;
-
-    const feedbackLink = document.createElement("a");
-    feedbackLink.href = `https://github.com/vllm-project/vllm-omni/issues/new?template=100-documentation.yml&title=${encodeURIComponent(
-      `[Docs] Feedback for \`${page}\``
-    )}&body=${encodeURIComponent(`📄 **Reference:**\n${url}\n\n📝 **Feedback:**\n_Your response_`)}`;
-    feedbackLink.target = "_blank";
-    feedbackLink.rel = "noopener";
-    feedbackLink.title = "Provide feedback";
-    feedbackLink.className = "md-content__button";
-    feedbackLink.innerHTML = `
-    <svg
-      xmlns="http://www.w3.org/2000/svg"
-      height="24px"
-      viewBox="0 -960 960 960"
-      width="24px"
-      fill="currentColor"
-    >
-      <path d="M280-280h280v-80H280v80Zm0-160h400v-80H280v80Zm0-160h400v-80H280v80Zm-80 480q-33 0-56.5-23.5T120-200v-560q0-33 23.5-56.5T200-840h560q33 0 56.5 23.5T840-760v560q0 33-23.5 56.5T760-120H200Zm0-80h560v-560H200v560Zm0-560v560-560Z"/>
-    </svg>
-  `;
-
-    const editButton = document.querySelector('.md-content__button[href*="edit"]');
-
-    if (editButton && editButton.parentNode) {
-      editButton.insertAdjacentElement("beforebegin", feedbackLink);
-
-      editButton.setAttribute("target", "_blank");
-      editButton.setAttribute("rel", "noopener");
-    }
-  });

docs/mkdocs/javascript/mathjax.js

@@ -1,20 +0,0 @@
-// Enables MathJax rendering
-window.MathJax = {
-    tex: {
-      inlineMath: [["\\(", "\\)"]],
-      displayMath: [["\\[", "\\]"]],
-      processEscapes: true,
-      processEnvironments: true
-    },
-    options: {
-      ignoreHtmlClass: ".*|",
-      processHtmlClass: "arithmatex"
-    }
-  };
-
-  document$.subscribe(() => {
-    MathJax.startup.output.clearCache()
-    MathJax.typesetClear()
-    MathJax.texReset()
-    MathJax.typesetPromise()
-  })

docs/mkdocs/javascript/slack_and_forum.js

@@ -1,56 +0,0 @@
-/**
- * slack_and_forum.js
- *
- * Adds a custom Slack and Forum button to the MkDocs Material header.
- *
- */
-
-window.addEventListener('DOMContentLoaded', () => {
-    const headerInner = document.querySelector('.md-header__inner');
-
-    if (headerInner) {
-      const slackButton = document.createElement('button');
-      slackButton.className = 'slack-button';
-      slackButton.title = 'Join us on Slack';
-      slackButton.style.border = 'none';
-      slackButton.style.background = 'transparent';
-      slackButton.style.cursor = 'pointer';
-
-      slackButton.innerHTML = `
-        <img src="https://a.slack-edge.com/80588/marketing/img/icons/icon_slack_hash_colored.png"
-             style="height: 1.1rem;"
-             alt="Slack">
-      `;
-
-      slackButton.addEventListener('click', () => {
-        window.open('https://slack.vllm.ai', '_blank', 'noopener');
-      });
-
-      const forumButton = document.createElement('button');
-      forumButton.className = 'forum-button';
-      forumButton.title = 'Join the Forum';
-      forumButton.style.border = 'none';
-      forumButton.style.background = 'transparent';
-      forumButton.style.cursor = 'pointer';
-
-      forumButton.innerHTML = `
-        <svg
-          xmlns="http://www.w3.org/2000/svg"
-          viewBox="0 -960 960 960"
-          fill="currentColor"
-        >
-          <path d="M817.85-198.15 698.46-317.54H320q-24.48 0-41.47-16.99T261.54-376v-11.69h424.61q25.39 0 43.47-18.08 18.07-18.08 18.07-43.46v-268.92h11.69q24.48 0 41.47 16.99 17 16.99 17 41.47v461.54ZM179.08-434.69l66.84-66.85h363.31q10.77 0 17.69-6.92 6.93-6.92 6.93-17.69v-246.77q0-10.77-6.93-17.7-6.92-6.92-17.69-6.92H203.69q-10.77 0-17.69 6.92-6.92 6.93-6.92 17.7v338.23Zm-36.93 89.46v-427.69q0-25.39 18.08-43.46 18.08-18.08 43.46-18.08h405.54q25.39 0 43.46 18.08 18.08 18.07 18.08 43.46v246.77q0 25.38-18.08 43.46-18.07 18.07-43.46 18.07H261.54L142.15-345.23Zm36.93-180.92V-797.54v271.39Z"/>
-        </svg>
-      `;
-
-      forumButton.addEventListener('click', () => {
-        window.open('https://discuss.vllm.ai/', '_blank', 'noopener');
-      });
-
-      const githubSource = document.querySelector('.md-header__source');
-      if (githubSource) {
-        githubSource.parentNode.insertBefore(slackButton, githubSource.nextSibling);
-        githubSource.parentNode.insertBefore(forumButton, slackButton.nextSibling);
-      }
-    }
-  });

docs/mkdocs/overrides/main.html

@@ -1 +0,0 @@
-{% extends "base.html" %}

docs/mkdocs/overrides/partials/toc-item.html

@@ -1,21 +0,0 @@
-<!-- Enables the use of toc_depth in document frontmatter https://github.com/squidfunk/mkdocs-material/issues/4827#issuecomment-1869812019 -->
-<li class="md-nav__item">
-    <a href="{{ toc_item.url }}" class="md-nav__link">
-      <span class="md-ellipsis">
-        {{ toc_item.title }}
-      </span>
-    </a>
-
-    <!-- Table of contents list -->
-    {% if toc_item.children %}
-      <nav class="md-nav" aria-label="{{ toc_item.title | striptags }}">
-        <ul class="md-nav__list">
-          {% for toc_item in toc_item.children %}
-          {% if not page.meta.toc_depth or toc_item.level <= page.meta.toc_depth %}
-            {% include "partials/toc-item.html" %}
-          {% endif %}
-          {% endfor %}
-        </ul>
-      </nav>
-    {% endif %}
-  </li>

docs/mkdocs/stylesheets/extra.css

@@ -1,221 +1,55 @@
-/* Warning for latest docs */
-.md-banner {
-    background-color: var(--md-warning-bg-color);
-    color: var(--md-warning-fg-color);
+/* Custom CSS for vLLM-omni documentation - based on vLLM's extra.css */
+
+/* Improve code block readability */
+.highlight {
+    border-radius: 4px;
 }
 
-/* https://christianoliff.com/blog/styling-external-links-with-an-icon-in-css/ */
-a:not(:has(svg)):not(.md-icon):not(.autorefs-external) {
-    align-items: center;
-
-    &[href^="//"]::after,
-    &[href^="http://"]::after,
-    &[href^="https://"]::after {
-        content: "";
-        width: 12px;
-        height: 12px;
-        margin-left: 4px;
-        background-image: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' width='16' height='16' stroke='gray' viewBox='0 0 16 16'%3E%3Cpath fill-rule='evenodd' d='M8.636 3.5a.5.5 0 0 0-.5-.5H1.5A1.5 1.5 0 0 0 0 4.5v10A1.5 1.5 0 0 0 1.5 16h10a1.5 1.5 0 0 0 1.5-1.5V7.864a.5.5 0 0 0-1 0V14.5a.5.5 0 0 1-.5.5h-10a.5.5 0 0 1-.5-.5v-10a.5.5 0 0 1 .5-.5h6.636a.5.5 0 0 0 .5-.5z'/%3E%3Cpath fill-rule='evenodd' d='M16 .5a.5.5 0 0 0-.5-.5h-5a.5.5 0 0 0 0 1h3.793L6.146 9.146a.5.5 0 1 0 .708.708L15 1.707V5.5a.5.5 0 0 0 1 0v-5z'/%3E%3C/svg%3E");
-        background-position: center;
-        background-repeat: no-repeat;
-        background-size: contain;
-        display: inline-block;
-    }
+/* Style API documentation */
+.md-typeset .doc-contents {
+    margin-bottom: 1.5em;
 }
 
-a[href*="localhost"]::after,
-a[href*="127.0.0.1"]::after,
-
-/* Hide external link icons for all links */
-a[href^="//"]::after,
-a[href^="http://"]::after,
-a[href^="https://"]::after {
-    display: none !important;
+/* Improve table readability */
+.md-typeset table {
+    display: table;
+    width: 100%;
 }
 
-/* Light mode: darker section titles */
+/* Light mode: darker section titles (from vLLM) */
 body[data-md-color-scheme="default"] .md-nav__item--section > label.md-nav__link .md-ellipsis {
   color: rgba(0, 0, 0, 0.7) !important;
   font-weight: 700;
 }
 
-/* Dark mode: lighter gray section titles */
+/* Dark mode: lighter gray section titles (from vLLM) */
 body[data-md-color-scheme="slate"] .md-nav__item--section > label.md-nav__link .md-ellipsis {
   color: rgba(255, 255, 255, 0.75) !important;
   font-weight: 700;
 }
 
-/* Custom admonitions */
-:root {
-  --md-admonition-icon--announcement: url('data:image/svg+xml;charset=utf-8,<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 16 16" width="16" height="16"><path d="M3.25 9a.75.75 0 0 1 .75.75c0 2.142.456 3.828.733 4.653a.122.122 0 0 0 .05.064.212.212 0 0 0 .117.033h1.31c.085 0 .18-.042.258-.152a.45.45 0 0 0 .075-.366A16.743 16.743 0 0 1 6 9.75a.75.75 0 0 1 1.5 0c0 1.588.25 2.926.494 3.85.293 1.113-.504 2.4-1.783 2.4H4.9c-.686 0-1.35-.41-1.589-1.12A16.4 16.4 0 0 1 2.5 9.75.75.75 0 0 1 3.25 9Z"></path><path d="M0 6a4 4 0 0 1 4-4h2.75a.75.75 0 0 1 .75.75v6.5a.75.75 0 0 1-.75.75H4a4 4 0 0 1-4-4Zm4-2.5a2.5 2.5 0 1 0 0 5h2v-5Z"></path><path d="M15.59.082A.75.75 0 0 1 16 .75v10.5a.75.75 0 0 1-1.189.608l-.002-.001h.001l-.014-.01a5.775 5.775 0 0 0-.422-.25 10.63 10.63 0 0 0-1.469-.64C11.576 10.484 9.536 10 6.75 10a.75.75 0 0 1 0-1.5c2.964 0 5.174.516 6.658 1.043.423.151.787.302 1.092.443V2.014c-.305.14-.669.292-1.092.443C11.924 2.984 9.713 3.5 6.75 3.5a.75.75 0 0 1 0-1.5c2.786 0 4.826-.484 6.155-.957.665-.236 1.154-.47 1.47-.64.144-.077.284-.161.421-.25l.014-.01a.75.75 0 0 1 .78-.061Z"></path></svg>');
-  --md-admonition-icon--important: url('data:image/svg+xml;charset=utf-8,<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 16 16" width="16" height="16"><path d="M4.47.22A.749.749 0 0 1 5 0h6c.199 0 .389.079.53.22l4.25 4.25c.141.14.22.331.22.53v6a.749.749 0 0 1-.22.53l-4.25 4.25A.749.749 0 0 1 11 16H5a.749.749 0 0 1-.53-.22L.22 11.53A.749.749 0 0 1 0 11V5c0-.199.079-.389.22-.53Zm.84 1.28L1.5 5.31v5.38l3.81 3.81h5.38l3.81-3.81V5.31L10.69 1.5ZM8 4a.75.75 0 0 1 .75.75v3.5a.75.75 0 0 1-1.5 0v-3.5A.75.75 0 0 1 8 4Zm0 8a1 1 0 1 1 0-2 1 1 0 0 1 0 2Z"></path></svg>');
-  --md-admonition-icon--code: url('data:image/svg+xml;charset=utf-8,<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 16 16"><path d="m11.28 3.22 4.25 4.25a.75.75 0 0 1 0 1.06l-4.25 4.25a.749.749 0 0 1-1.275-.326.75.75 0 0 1 .215-.734L13.94 8l-3.72-3.72a.749.749 0 0 1 .326-1.275.75.75 0 0 1 .734.215m-6.56 0a.75.75 0 0 1 1.042.018.75.75 0 0 1 .018 1.042L2.06 8l3.72 3.72a.749.749 0 0 1-.326 1.275.75.75 0 0 1-.734-.215L.47 8.53a.75.75 0 0 1 0-1.06Z"/></svg>');
-  --md-admonition-icon--console: url('data:image/svg+xml;charset=utf-8,<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 16 16"><path d="M0 2.75C0 1.784.784 1 1.75 1h12.5c.966 0 1.75.784 1.75 1.75v10.5A1.75 1.75 0 0 1 14.25 15H1.75A1.75 1.75 0 0 1 0 13.25Zm1.75-.25a.25.25 0 0 0-.25.25v10.5c0 .138.112.25.25.25h12.5a.25.25 0 0 0 .25-.25V2.75a.25.25 0 0 0-.25-.25ZM7.25 8a.75.75 0 0 1-.22.53l-2.25 2.25a.749.749 0 0 1-1.275-.326.75.75 0 0 1 .215-.734L5.44 8 3.72 6.28a.749.749 0 0 1 .326-1.275.75.75 0 0 1 .734.215l2.25 2.25c.141.14.22.331.22.53m1.5 1.5h3a.75.75 0 0 1 0 1.5h-3a.75.75 0 0 1 0-1.5"/></svg>');
+/* Navigation: Expand first level of API Reference by default, collapse deeper levels */
+/* Show direct children of API Reference section (vllm_omni's children: config, core, entrypoints, etc.) */
+.md-nav__item--nested[data-md-state] > .md-nav {
+    display: none;
 }
 
-.md-typeset .admonition.announcement,
-.md-typeset details.announcement {
-  border-color: rgb(255, 110, 66);
-}
-.md-typeset .admonition.important,
-.md-typeset details.important {
-  border-color: rgb(239, 85, 82);
-}
-.md-typeset .admonition.code,
-.md-typeset details.code {
-  border-color: #64dd17
-}
-.md-typeset .admonition.console,
-.md-typeset details.console {
-  border-color: #64dd17
+/* Expand active items */
+.md-nav__item--nested.md-nav__item--active > .md-nav {
+    display: block;
 }
 
-.md-typeset .announcement > .admonition-title,
-.md-typeset .announcement > summary {
-  background-color: rgb(255, 110, 66, 0.1);
-}
-.md-typeset .important > .admonition-title,
-.md-typeset .important > summary {
-  background-color: rgb(239, 85, 82, 0.1);
-}
-.md-typeset .code > .admonition-title,
-.md-typeset .code > summary {
-  background-color: #64dd171a;
-}
-.md-typeset .console > .admonition-title,
-.md-typeset .console > summary {
-  background-color: #64dd171a;
+/* For API Reference: show first level children (config, core, entrypoints, etc.) by default */
+.md-nav__item--section:has([href*="api/vllm_omni/index"]) > .md-nav > .md-nav__list > .md-nav__item--nested > .md-nav {
+    display: block;
 }
 
-.md-typeset .announcement > .admonition-title::before,
-.md-typeset .announcement > summary::before {
-  background-color: rgb(239, 85, 82);
-  -webkit-mask-image: var(--md-admonition-icon--announcement);
-          mask-image: var(--md-admonition-icon--announcement);
-}
-.md-typeset .important > .admonition-title::before,
-.md-typeset .important > summary::before {
-  background-color: rgb(239, 85, 82);
-  -webkit-mask-image: var(--md-admonition-icon--important);
-          mask-image: var(--md-admonition-icon--important);
-}
-.md-typeset .code > .admonition-title::before,
-.md-typeset .code > summary::before {
-  background-color: #64dd17;
-  -webkit-mask-image: var(--md-admonition-icon--code);
-          mask-image: var(--md-admonition-icon--code);
-}
-.md-typeset .console > .admonition-title::before,
-.md-typeset .console > summary::before {
-  background-color: #64dd17;
-  -webkit-mask-image: var(--md-admonition-icon--console);
-          mask-image: var(--md-admonition-icon--console);
+/* But keep deeper levels collapsed */
+.md-nav__item--section:has([href*="api/vllm_omni/index"]) > .md-nav > .md-nav__list > .md-nav__item--nested > .md-nav > .md-nav__list > .md-nav__item--nested > .md-nav {
+    display: none;
 }
 
-/* Make label fully visible on hover */
-.md-content__button[href*="edit"]:hover::after {
-  opacity: 1;
-}
-
-/* Hide edit button on generated docs/examples pages */
-@media (min-width: 960px) {
-  .md-content__button[href*="docs/examples/"] {
-    display: none !important;
-  }
-}
-
-.md-content__button-wrapper {
-  position: absolute;
-  top: 0.6rem;
-  right: 0.8rem;
-  display: flex;
-  flex-direction: row;
-  align-items: center;
-  gap: 0.4rem;
-  z-index: 1;
-}
-
-.md-content__button-wrapper a {
-  display: inline-flex;
-  align-items: center;
-  justify-content: center;
-  height: 24px;
-  width: 24px;
-  color: var(--md-default-fg-color);
-  text-decoration: none;
-}
-
-.md-content__button-wrapper a:hover {
-  color: var(--md-accent-fg-color);
-}
-
-/* Slack and Forum css */
-.slack-button,
-.forum-button {
-  display: inline-flex;
-  align-items: center;
-  justify-content: center;
-  margin-left: 0.4rem;
-  height: 24px;
-}
-
-.slack-button img {
-  height: 18px;
-  filter: none !important;
-}
-
-.slack-button:hover,
-.forum-button:hover {
-  opacity: 0.7;
-}
-
-.forum-button svg {
-  height: 28px;
-  opacity: 0.9;
-  transform: translateY(2px);
-}
-
-/* For logo css */
-[data-md-color-scheme="default"] .logo-dark {
-  display: none;
-}
-
-[data-md-color-scheme="slate"] .logo-light {
-  display: none;
-}
-
-/* Outline for content tabs */
-.md-typeset .tabbed-set {
-  border: 0.075rem solid var(--md-default-fg-color);
-  border-radius: 0.2rem;
-}
-
-.md-typeset .tabbed-content {
-  padding: 0 0.6em;
-}
-
-/* Hide link icon in header logo */
-.md-header__button.md-logo :is(img, svg) {
-  pointer-events: none;
-}
-
-.md-header__button.md-logo::after {
-  display: none !important;
-}
-
-/* Hide link icons in content tabs (tabbed content) */
-.md-typeset .tabbed-set > label::after,
-.md-typeset .tabbed-set > input:checked + label::after {
-  display: none !important;
-}
-
-/* Hide link icons in navigation tabs */
-.md-nav__link[href]::after,
-.md-nav__item--nested > .md-nav__link::after {
-  display: none !important;
-}
-
-/* Hide link icons in top navigation tabs */
-.md-tabs__link::after {
-  display: none !important;
+/* Expand deeper levels only when active */
+.md-nav__item--section:has([href*="api/vllm_omni/index"]) > .md-nav > .md-nav__list > .md-nav__item--nested > .md-nav > .md-nav__list > .md-nav__item--nested.md-nav__item--active > .md-nav {
+    display: block;
 }

docs/models/supported_models.md

@@ -1,38 +0,0 @@
-# Supported Models
-
-vLLM-Omni supports unified multimodal comprehension and generation models across various tasks.
-
-## Model Implementation
-
-If vLLM-Omni natively supports a model, its implementation can be found in <gh-file:vllm-omni/model_executor/models> and <gh-file:vllm_omni/diffusion/models>.
-
-## List of Supported Models for Nvidia GPU
-
-<style>
-th {
-  white-space: nowrap;
-  min-width: 0 !important;
-}
-</style>
-
-| Architecture | Models | Example HF Models |
-|--------------|--------|-------------------|
-| `Qwen3OmniMoeForConditionalGeneration` | Qwen3-Omni | `Qwen/Qwen3-Omni-30B-A3B-Instruct` |
-| `Qwen2_5OmniForConditionalGeneration` | Qwen2.5-Omni | `Qwen/Qwen2.5-Omni-7B`, `Qwen/Qwen2.5-Omni-3B` |
-| `QwenImagePipeline` | Qwen-Image | `Qwen/Qwen-Image` |
-|`ZImagePipeline` | Z-Image | `Tongyi-MAI/Z-Image-Turbo` |
-
-
-## List of Supported Models for NPU
-
-<style>
-th {
-  white-space: nowrap;
-  min-width: 0 !important;
-}
-</style>
-
-| Architecture | Models | Example HF Models |
-|--------------|--------|-------------------|
-| `Qwen2_5OmniForConditionalGeneration` | Qwen2.5-Omni | `Qwen/Qwen2.5-Omni-7B`, `Qwen/Qwen2.5-Omni-3B`|
-| `QwenImagePipeline` | Qwen-Image | `Qwen/Qwen-Image` |

docs/usage/faq.md

@@ -1,25 +0,0 @@
-# Frequently Asked Questions
-
-> Q: How many chips do I need to infer a model in vLLM-Omni?
-
-A: Now, we support natively disaggregated deployment for different model stages within a model. There is a restriction that one chip can only has one AutoRegressive model stage. This is because the unified KV cache management of vLLM. Stages of other types can coexist with in a chip. The restriction will be resolved in later version.
-
-> Q: When trying to run examples, I encounter error about backend of librosa or soundfile. How to solve it?
-
-A: If you encounter error about backend of librosa, try to install ffmpeg with command below.
-```
-sudo apt update
-sudo apt install ffmpeg
-```
-
-> Q: I encounter some bugs or CI problems, which is urgent. How can I solve it?
-
-A: At first, you can check current [issues](https://github.com/vllm-project/vllm-omni/issues) to find possible solutions. If non of these satisfy your demand and it is urgent, please find these [volunteers](https://docs.vllm.ai/projects/vllm-omni/en/latest/community/volunteers/) for help.
-
-> Q: Does vLLM-Omni support AWQ or any other quantization?
-
-A: vLLM-Omni partitions model into several stages. For AR stages, it will reuse main logic of LLMEngine in vLLM. So current quantization supported in vLLM should be also supported in vLLM-Omni for them. But systematic verification is ongoing. For quantization for DiffusionEngine, we are working on it. Please stay tuned and welcome contribution!
-
-> Q: Does vLLM-Omni support multimodal streaming input and output?
-
-A: Not yet. We already put it on the [Roadmap](https://github.com/vllm-project/vllm-omni/issues/165). Please stay tuned!

docs/user_guide/examples/index.md

@@ -0,0 +1,8 @@
+# Examples
+
+vLLM-omni's examples are split into two categories:
+
+- If you are using vLLM-omni from within Python code, see the *Offline Inference* section.
+- If you are using vLLM-omni from an HTTP application or client, see the *Online Serving* section.
+
+For detailed example documentation, check the [examples directory](https://github.com/vllm-project/vllm-omni/tree/main/examples) in the repository.

docs/user_guide/examples/offline_inference/qwen2_5_omni.md

@@ -1,14 +1,13 @@
-# Offline Example of vLLM-Omni for Qwen2.5-Omni
+# Offline Example of vLLM-omni for Qwen2.5-omni
 
-Source <https://github.com/vllm-project/vllm-omni/tree/main/examples/offline_inference/qwen2_5_omni>.
+Source <https://github.com/vllm-project/vllm-omni/tree/main/examples\offline_inference\qwen2_5_omni>.
 
 
 ## 🛠️ Installation
 
 Please refer to [README.md](https://github.com/vllm-project/vllm-omni/tree/main/README.md)
 
-## Run examples (Qwen2.5-Omni)
-
+## Run examples (Qwen2.5-omni)
 ### Multiple Prompts
 Download dataset from [seed_tts](https://drive.google.com/file/d/1GlSjVfSHkW3-leKKBlfrjuuTGqQ_xaLP/edit). To get the prompt, you can:
 ```bash
@@ -27,8 +26,7 @@ Then run the command below.
 ```bash
 bash run_multiple_prompts.sh
 ```
-
-### Single Prompt
+### Single Prompts
 Get into the example folder
 ```bash
 cd examples/offline_inference/qwen2_5_omni
@@ -38,57 +36,29 @@ Then run the command below.
 bash run_single_prompt.sh
 ```
 
-#### Using Local Media Files
-The `end2end.py` script supports local media files (audio, video, image) via CLI arguments:
-
-```bash
-# Use single local media files
-python end2end.py --query-type use_image --image-path /path/to/image.jpg
-python end2end.py --query-type use_video --video-path /path/to/video.mp4
-python end2end.py --query-type use_audio --audio-path /path/to/audio.wav
-
-# Combine multiple local media files
-python end2end.py --query-type mixed_modalities \
-    --video-path /path/to/video.mp4 \
-    --image-path /path/to/image.jpg \
-    --audio-path /path/to/audio.wav
-
-# Use audio from video file
-python end2end.py --query-type use_audio_in_video --video-path /path/to/video.mp4
-
-```
-
-If media file paths are not provided, the script will use default assets. Supported query types:
-- `use_image`: Image input only
-- `use_video`: Video input only
-- `use_audio`: Audio input only
-- `mixed_modalities`: Audio + image + video
-- `use_audio_in_video`: Extract audio from video
-- `text`: Text-only query
-
-### FAQ
-
-If you encounter error about backend of librosa, try to install ffmpeg with command below.
-```
-sudo apt update
-sudo apt install ffmpeg
-```
-
 ## Example materials
 
 ??? abstract "end2end.py"
     ``````py
-    --8<-- "examples/offline_inference/qwen2_5_omni/end2end.py"
+    --8<-- "examples\offline_inference\qwen2_5_omni\end2end.py"
     ``````
 ??? abstract "extract_prompts.py"
     ``````py
-    --8<-- "examples/offline_inference/qwen2_5_omni/extract_prompts.py"
+    --8<-- "examples\offline_inference\qwen2_5_omni\extract_prompts.py"
+    ``````
+??? abstract "processing_omni.py"
+    ``````py
+    --8<-- "examples\offline_inference\qwen2_5_omni\processing_omni.py"
     ``````
 ??? abstract "run_multiple_prompts.sh"
     ``````sh
-    --8<-- "examples/offline_inference/qwen2_5_omni/run_multiple_prompts.sh"
+    --8<-- "examples\offline_inference\qwen2_5_omni\run_multiple_prompts.sh"
     ``````
 ??? abstract "run_single_prompt.sh"
     ``````sh
-    --8<-- "examples/offline_inference/qwen2_5_omni/run_single_prompt.sh"
+    --8<-- "examples\offline_inference\qwen2_5_omni\run_single_prompt.sh"
+    ``````
+??? abstract "utils.py"
+    ``````py
+    --8<-- "examples\offline_inference\qwen2_5_omni\utils.py"
     ``````

docs/user_guide/examples/offline_inference/qwen3_omni.md

@@ -1,81 +0,0 @@
-# Offline Example of vLLM-Omni for Qwen3-Omni
-
-Source <https://github.com/vllm-project/vllm-omni/tree/main/examples/offline_inference/qwen3_omni>.
-
-
-## 🛠️ Installation
-
-Please refer to [README.md](https://github.com/vllm-project/vllm-omni/tree/main/README.md)
-
-## Run examples (Qwen3-Omni)
-
-### Multiple Prompts
-Download dataset from [seed_tts](https://drive.google.com/file/d/1GlSjVfSHkW3-leKKBlfrjuuTGqQ_xaLP/edit). For processing dataset please refer to [Qwen2.5-Omni README.md](https://github.com/vllm-project/vllm-omni/tree/main/examples/offline_inference/qwen2_5_omni/README.md)
-Get into the example folder
-```bash
-cd examples/offline_inference/qwen3_omni
-```
-Then run the command below.
-```bash
-bash run_multiple_prompts.sh
-```
-### Single Prompt
-Get into the example folder
-```bash
-cd examples/offline_inference/qwen3_omni
-```
-Then run the command below.
-```bash
-bash run_single_prompt.sh
-```
-If you have not enough memory, you can set thinker with tensor parallel. Just run the command below.
-```bash
-bash run_single_prompt_tp.sh
-```
-
-#### Using Local Media Files
-The `end2end.py` script supports local media files (audio, video, image) via command-line arguments:
-
-```bash
-# Use local video file
-python end2end.py --query-type use_video --video-path /path/to/video.mp4
-
-# Use local image file
-python end2end.py --query-type use_image --image-path /path/to/image.jpg
-
-# Use local audio file
-python end2end.py --query-type use_audio --audio-path /path/to/audio.wav
-```
-
-If media file paths are not provided, the script will use default assets. Supported query types:
-- `use_video`: Video input
-- `use_image`: Image input
-- `use_audio`: Audio input
-- `text`: Text-only query
-
-### FAQ
-
-If you encounter error about backend of librosa, try to install ffmpeg with command below.
-```
-sudo apt update
-sudo apt install ffmpeg
-```
-
-## Example materials
-
-??? abstract "end2end.py"
-    ``````py
-    --8<-- "examples/offline_inference/qwen3_omni/end2end.py"
-    ``````
-??? abstract "run_multiple_prompts.sh"
-    ``````sh
-    --8<-- "examples/offline_inference/qwen3_omni/run_multiple_prompts.sh"
-    ``````
-??? abstract "run_single_prompt.sh"
-    ``````sh
-    --8<-- "examples/offline_inference/qwen3_omni/run_single_prompt.sh"
-    ``````
-??? abstract "run_single_prompt_tp.sh"
-    ``````sh
-    --8<-- "examples/offline_inference/qwen3_omni/run_single_prompt_tp.sh"
-    ``````

docs/user_guide/examples/offline_inference/qwen_2_5_omni.md

@@ -0,0 +1,97 @@
+# Offline Example of vLLM-omni for Qwen2.5-omni
+
+Source <https://github.com/vllm-project/vllm-omni/tree/main/examples/offline_inference/qwen_2_5_omni>.
+
+
+## 🛠️ Installation
+
+### Environment setup
+
+Use Docker to keep consistent basic environment (Optional, Recommended)
+```bash
+docker run --gpus all --ipc=host --network=host -v $source_dir:$container_dir --name $container_name -it nvcr.io/nvidia/pytorch:25.01-py3 bash
+```
+
+Set up basic uv environment
+```bash
+pip install uv
+uv venv --python 3.12 --seed
+source .venv/bin/activate
+```
+
+### Installation of vLLM (for users)
+
+Now we build it based on vLLM v0.11.0. Please install it with command below.
+```bash
+uv pip install vllm==0.11.0 --torch-backend=auto
+```
+
+### Installation of vLLM (for developers)
+
+If you want to check or debug with source code of vLLM, install stable release version of vllm with 0.11.0 from source with pre-built wheel file.
+
+```bash
+git clone https://github.com/vllm-project/vllm.git
+cd vllm
+git checkout v0.11.0
+```
+Set up environment variables to get pre-built wheels. If there are internet problems, just download the whl file manually. And set VLLM_PRECOMPILED_WHEEL_LOCATION as your local absolute path of whl file.
+```bash
+export VLLM_PRECOMPILED_WHEEL_LOCATION=https://github.com/vllm-project/vllm/releases/download/v0.11.0/vllm-0.11.0-cp38-abi3-manylinux1_x86_64.whl
+```
+Install vllm with command below (If you have no existing PyTorch).
+```bash
+uv pip install --editable .
+```
+
+Install vllm with command below (If you already have PyTorch).
+```bash
+python use_existing_torch.py
+uv pip install -r requirements/build.txt
+uv pip install --no-build-isolation --editable .
+```
+
+### Verification for successful installation of vLLM
+Just run the command below. If no error, it demonstrates that the installation is successful.
+```bash
+python -c "import vllm._C"
+```
+
+### Installation of vLLM-omni
+Install additional requirements for vllm-omni
+```bash
+git clone https://github.com/vllm-project/vllm-omni.git
+cd vllm_omni
+uv pip install -e .
+```
+
+## Run examples (Qwen2.5-omni)
+
+Get into the example folder
+```bash
+cd examples/offline_inference/qwen_2_5_omni
+```
+Then run
+```bash
+bash run.sh
+```
+The output audio is saved in ./output_audio
+
+## Example materials
+
+??? abstract "end2end.py"
+    ``````py
+    --8<-- "examples/offline_inference/qwen_2_5_omni/end2end.py"
+    ``````
+??? abstract "processing_omni.py"
+    ``````py
+    --8<-- "examples/offline_inference/qwen_2_5_omni/processing_omni.py"
+    ``````
+??? abstract "run.sh"
+    ``````sh
+    --8<-- "examples/offline_inference/qwen_2_5_omni/run.sh"
+    ``````
+??? abstract "utils.py"
+    ``````py
+    --8<-- "examples/offline_inference/qwen_2_5_omni/utils.py"
+    ``````

docs/user_guide/examples/offline_inference/qwen_image.md

@@ -1,57 +0,0 @@
-# Qwen-Image Offline Inference
-
-Source <https://github.com/vllm-project/vllm-omni/tree/main/examples/offline_inference/qwen_image>.
-
-
-This folder provides two simple entrypoints for experimenting with `Qwen/Qwen-Image` using vLLM-Omni:
-
-- `text_to_image.py`: command-line script for single image generation.
-- `web_demo.py`: lightweight Gradio UI for interactive prompt/seed/CFG exploration.
-
-
-## Local CLI Usage
-
-```bash
-python text_to_image.py \
-  --prompt "a cup of coffee on the table" \
-  --seed 42 \
-  --cfg_scale 4.0 \
-  --num_images_per_prompt 1 \
-  --num_inference_steps 50 \
-  --height 1024 \
-  --width 1024 \
-  --output outputs/coffee.png
-```
-
-Key arguments:
-
-- `--prompt`: text description (string).
-- `--seed`: integer seed for deterministic sampling.
-- `--cfg_scale`: true CFG scale (model-specific guidance strength).
-- `--num_images_per_prompt`: number of images to generate per prompt (saves as `output`, `output_1`, ...).
-- `--num_inference_steps`: diffusion sampling steps (more steps = higher quality, slower).
-- `--height/--width`: output resolution (defaults 1024x1024).
-- `--output`: path to save the generated PNG.
-
-> ℹ️ Qwen-Image currently publishes best-effort presets at `1328x1328`, `1664x928`, `928x1664`, `1472x1140`, `1140x1472`, `1584x1056`, and `1056x1584`. Adjust `--height/--width` accordingly for the most reliable outcomes.
-
-## Web UI Demo
-
-Launch the gradio demo:
-
-```bash
-python gradio_demo.py --port 7862
-```
-
-Then open `http://localhost:7862/` on your local browser to interact with the web UI.
-
-## Example materials
-
-??? abstract "gradio_demo.py"
-    ``````py
-    --8<-- "examples/offline_inference/qwen_image/gradio_demo.py"
-    ``````
-??? abstract "text_to_image.py"
-    ``````py
-    --8<-- "examples/offline_inference/qwen_image/text_to_image.py"
-    ``````

docs/user_guide/examples/online_serving/README.md

@@ -0,0 +1,35 @@
+# Online serving Example of vLLM-omni for Qwen2.5-omni
+
+Source <https://github.com/vllm-project/vllm-omni/blob/main/examples\online_serving\README.md>.
+
+
+## 🛠️ Installation
+
+Please refer to [README.md](https://github.com/vllm-project/vllm-omni/blob/main/README.md)
+
+## Run examples (Qwen2.5-omni)
+
+Launch the server
+```bash
+vllm serve Qwen/Qwen2.5-Omni-7B --omni --port 8091
+```
+
+If you have custom stage configs file, launch the server with command below
+```bash
+vllm serve Qwen/Qwen2.5-Omni-7B --omni --port 8091 --stage-configs-path /path/to/stage_configs_file
+```
+
+Get into the example folder
+```bash
+cd examples/online_serving
+```
+
+Send request via python
+```bash
+python openai_chat_completion_client_for_multimodal_generation.py
+```
+
+Send request via curl
+```bash
+bash run_curl_multimodal_generation.sh
+```

docs/user_guide/examples/online_serving/gradio_demo.md

@@ -1,7 +0,0 @@
-# Gradio Demo
-
-Source <https://github.com/vllm-project/vllm-omni/blob/main/examples/online_serving/gradio_demo.py>.
-
-``````py
---8<-- "examples/online_serving/gradio_demo.py"
-``````

docs/user_guide/examples/online_serving/index.md

@@ -0,0 +1,5 @@
+# Online Serving
+
+## Available Examples
+
+Online serving is under active development. Coming soon.

docs/user_guide/examples/online_serving/openai_chat_completion_client_for_multimodal_generation.md

@@ -1,7 +1,7 @@
 # OpenAI Chat Completion Client For Multimodal Generation
 
-Source <https://github.com/vllm-project/vllm-omni/blob/main/examples/online_serving/openai_chat_completion_client_for_multimodal_generation.py>.
+Source <https://github.com/vllm-project/vllm-omni/blob/main/examples\online_serving\openai_chat_completion_client_for_multimodal_generation.py>.
 
 ``````py
---8<-- "examples/online_serving/openai_chat_completion_client_for_multimodal_generation.py"
+--8<-- "examples\online_serving\openai_chat_completion_client_for_multimodal_generation.py"
 ``````

docs/user_guide/examples/online_serving/qwen2_5_omni.md

@@ -1,119 +0,0 @@
-# Online serving Example of vLLM-Omni for Qwen2.5-Omni
-
-Source <https://github.com/vllm-project/vllm-omni/tree/main/examples/online_serving/qwen2_5_omni>.
-
-
-## 🛠️ Installation
-
-Please refer to [README.md](https://github.com/vllm-project/vllm-omni/tree/main/README.md)
-
-## Run examples (Qwen2.5-Omni)
-
-### Launch the Server
-
-```bash
-vllm serve Qwen/Qwen2.5-Omni-7B --omni --port 8091
-```
-
-If you have custom stage configs file, launch the server with command below
-```bash
-vllm serve Qwen/Qwen2.5-Omni-7B --omni --port 8091 --stage-configs-path /path/to/stage_configs_file
-```
-
-### Send Multi-modal Request
-
-Get into the example folder
-```bash
-cd examples/online_serving/qwen2_5_omni
-```
-
-####  Send request via python
-
-```bash
-python openai_chat_completion_client_for_multimodal_generation.py --query-type mixed_modalities
-```
-
-The Python client supports the following command-line arguments:
-
-- `--query-type` (or `-q`): Query type (default: `mixed_modalities`)
-  - Options: `mixed_modalities`, `use_audio_in_video`, `multi_audios`, `text`
-- `--video-path` (or `-v`): Path to local video file or URL
-  - If not provided and query-type uses video, uses default video URL
-  - Supports local file paths (automatically encoded to base64) or HTTP/HTTPS URLs
-  - Example: `--video-path /path/to/video.mp4` or `--video-path https://example.com/video.mp4`
-- `--image-path` (or `-i`): Path to local image file or URL
-  - If not provided and query-type uses image, uses default image URL
-  - Supports local file paths (automatically encoded to base64) or HTTP/HTTPS URLs
-  - Supports common image formats: JPEG, PNG, GIF, WebP
-  - Example: `--image-path /path/to/image.jpg` or `--image-path https://example.com/image.png`
-- `--audio-path` (or `-a`): Path to local audio file or URL
-  - If not provided and query-type uses audio, uses default audio URL
-  - Supports local file paths (automatically encoded to base64) or HTTP/HTTPS URLs
-  - Supports common audio formats: MP3, WAV, OGG, FLAC, M4A
-  - Example: `--audio-path /path/to/audio.wav` or `--audio-path https://example.com/audio.mp3`
-- `--prompt` (or `-p`): Custom text prompt/question
-  - If not provided, uses default prompt for the selected query type
-  - Example: `--prompt "What are the main activities shown in this video?"`
-
-
-For example, to use mixed modalities with all local files:
-
-```bash
-python openai_chat_completion_client_for_multimodal_generation.py \
-    --query-type mixed_modalities \
-    --video-path /path/to/your/video.mp4 \
-    --image-path /path/to/your/image.jpg \
-    --audio-path /path/to/your/audio.wav \
-    --prompt "Analyze all the media content and provide a comprehensive summary."
-```
-
-####  Send request via curl
-
-```bash
-bash run_curl_multimodal_generation.sh mixed_modalities
-```
-
-## Run Local Web UI Demo
-
-This Web UI demo allows users to interact with the model through a web browser.
-
-### Running Gradio Demo
-
-Once vllm and vllm-omni are installed, you can launch the web service built on AsyncOmni by
-
-```bash
-python gradio_demo.py  --model Qwen/Qwen2.5-Omni-7B --port 7861
-```
-
-Then open `http://localhost:7861/` on your local browser to interact with the web UI.
-
-The gradio script supports the following arguments:
-
-- `--model`: Model name
-- `--ip`: Host/IP for Gradio server (default: 127.0.0.1)
-- `--port`: Port for Gradio server (default: 7861)
-- `--stage-configs-path`: Path to custom stage configs YAML file (optional)
-- `--share`: Share the Gradio demo publicly (creates a public link)
-
-### FAQ
-
-If you encounter error about backend of librosa, try to install ffmpeg with command below.
-```
-sudo apt update
-sudo apt install ffmpeg
-```
-
-## Example materials
-
-??? abstract "gradio_demo.py"
-    ``````py
-    --8<-- "examples/online_serving/qwen2_5_omni/gradio_demo.py"
-    ``````
-??? abstract "openai_chat_completion_client_for_multimodal_generation.py"
-    ``````py
-    --8<-- "examples/online_serving/qwen2_5_omni/openai_chat_completion_client_for_multimodal_generation.py"
-    ``````
-??? abstract "run_curl_multimodal_generation.sh"
-    ``````sh
-    --8<-- "examples/online_serving/qwen2_5_omni/run_curl_multimodal_generation.sh"
-    ``````

docs/user_guide/examples/online_serving/qwen3_omni.md

@@ -1,120 +0,0 @@
-# Online serving Example of vLLM-Omni for Qwen3-omni
-
-Source <https://github.com/vllm-project/vllm-omni/tree/main/examples/online_serving/qwen3_omni>.
-
-
-## 🛠️ Installation
-
-Please refer to [README.md](https://github.com/vllm-project/vllm-omni/tree/main/README.md)
-
-## Run examples (Qwen3-Omni)
-
-### Launch the Server
-
-```bash
-vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091
-```
-
-If you have custom stage configs file, launch the server with command below
-```bash
-vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091 --stage-configs-path /path/to/stage_configs_file
-```
-
-### Send Multi-modal Request
-
-Get into the example folder
-```bash
-cd examples/online_serving/qwen3_omni
-```
-
-####  Send request via python
-
-```bash
-python openai_chat_completion_client_for_multimodal_generation.py --query-type use_image
-```
-
-The Python client supports the following command-line arguments:
-
-- `--query-type` (or `-q`): Query type (default: `use_video`)
-  - Options: `text`, `use_audio`, `use_image`, `use_video`
-- `--model` (or `-m`): Model name/path (default: `Qwen/Qwen3-Omni-30B-A3B-Instruct`)
-- `--video-path` (or `-v`): Path to local video file or URL
-  - If not provided and query-type is `use_video`, uses default video URL
-  - Supports local file paths (automatically encoded to base64) or HTTP/HTTPS URLs
-  - Example: `--video-path /path/to/video.mp4` or `--video-path https://example.com/video.mp4`
-- `--image-path` (or `-i`): Path to local image file or URL
-  - If not provided and query-type is `use_image`, uses default image URL
-  - Supports local file paths (automatically encoded to base64) or HTTP/HTTPS URLs
-  - Supports common image formats: JPEG, PNG, GIF, WebP
-  - Example: `--image-path /path/to/image.jpg` or `--image-path https://example.com/image.png`
-- `--audio-path` (or `-a`): Path to local audio file or URL
-  - If not provided and query-type is `use_audio`, uses default audio URL
-  - Supports local file paths (automatically encoded to base64) or HTTP/HTTPS URLs
-  - Supports common audio formats: MP3, WAV, OGG, FLAC, M4A
-  - Example: `--audio-path /path/to/audio.wav` or `--audio-path https://example.com/audio.mp3`
-- `--prompt` (or `-p`): Custom text prompt/question
-  - If not provided, uses default prompt for the selected query type
-  - Example: `--prompt "What are the main activities shown in this video?"`
-
-
-For example, to use a local video file with custom prompt:
-
-```bash
-python openai_chat_completion_client_for_multimodal_generation.py \
-    --query-type use_video \
-    --video-path /path/to/your/video.mp4 \
-    --prompt "What are the main activities shown in this video?"
-```
-
-####  Send request via curl
-
-```bash
-bash run_curl_multimodal_generation.sh use_image
-```
-
-
-### FAQ
-
-If you encounter error about backend of librosa, try to install ffmpeg with command below.
-```
-sudo apt update
-sudo apt install ffmpeg
-```
-
-## Run Local Web UI Demo
-
-This Web UI demo allows users to interact with the model through a web browser.
-
-### Running Gradio Demo
-
-Once vllm and vllm-omni are installed, you can launch the web service built on AsyncOmni by
-
-```bash
-python gradio_demo.py  --model Qwen/Qwen3-Omni-30B-A3B-Instruct --port 7861
-```
-
-Then open `http://localhost:7861/` on your local browser to interact with the web UI.
-
-
-The gradio script supports the following arguments:
-
-- `--model`: Model name
-- `--ip`: Host/IP for Gradio server (default: 127.0.0.1)
-- `--port`: Port for Gradio server (default: 7861)
-- `--stage-configs-path`: Path to custom stage configs YAML file (optional)
-- `--share`: Share the Gradio demo publicly (creates a public link)
-
-## Example materials
-
-??? abstract "gradio_demo.py"
-    ``````py
-    --8<-- "examples/online_serving/qwen3_omni/gradio_demo.py"
-    ``````
-??? abstract "openai_chat_completion_client_for_multimodal_generation.py"
-    ``````py
-    --8<-- "examples/online_serving/qwen3_omni/openai_chat_completion_client_for_multimodal_generation.py"
-    ``````
-??? abstract "run_curl_multimodal_generation.sh"
-    ``````sh
-    --8<-- "examples/online_serving/qwen3_omni/run_curl_multimodal_generation.sh"
-    ``````

docs/user_guide/getting_started/quickstart.md

@@ -0,0 +1,55 @@
+# Quickstart
+
+This guide will help you get started with vLLM-omni.
+
+## Installation
+
+Use Docker to keep consistent basic environment (Optional, Recommended)
+
+```bash
+docker run --gpus all --ipc=host --network=host -v $source_dir:$container_dir --rm --name $container_name -it nvcr.io/nvidia/pytorch:25.01-py3 bash
+```
+
+Set up basic uv environment
+
+```bash
+pip install uv
+uv venv --python 3.12 --seed
+source .venv/bin/activate
+```
+
+Install vLLM
+
+```bash
+uv pip install vllm==0.11.0 --torch-backend=auto
+```
+
+Install vLLM-omni
+
+```bash
+git clone https://github.com/vllm-project/vllm-omni.git
+cd vllm_omni
+uv pip install -e .
+```
+
+## Run examples (Qwen2.5-omni)
+
+Get into the example folder
+
+```bash
+cd examples/offline_inference/qwen_2_5_omni
+```
+
+Then run.
+
+```bash
+bash run.sh
+```
+
+The output audio is saved in ./output_audio
+
+## Next Steps
+
+- Read the [architecture documentation](../../contributing/design_documents/vllm_omni_design.md)
+- Check out the [API reference](../../api/overview.md)
+- Explore the [examples](../examples/index.md)

examples/offline_inference/qwen2_5_omni/README.md

@@ -1,11 +1,10 @@
-# Offline Example of vLLM-Omni for Qwen2.5-Omni
+# Offline Example of vLLM-omni for Qwen2.5-omni
 
 ## 🛠️ Installation
 
 Please refer to [README.md](../../../README.md)
 
-## Run examples (Qwen2.5-Omni)
-
+## Run examples (Qwen2.5-omni)
 ### Multiple Prompts
 Download dataset from [seed_tts](https://drive.google.com/file/d/1GlSjVfSHkW3-leKKBlfrjuuTGqQ_xaLP/edit). To get the prompt, you can:
 ```bash
@@ -24,8 +23,7 @@ Then run the command below.
 ```bash
 bash run_multiple_prompts.sh
 ```
-
-### Single Prompt
+### Single Prompts
 Get into the example folder
 ```bash
 cd examples/offline_inference/qwen2_5_omni
@@ -35,34 +33,6 @@ Then run the command below.
 bash run_single_prompt.sh
 ```
 
-#### Using Local Media Files
-The `end2end.py` script supports local media files (audio, video, image) via CLI arguments:
-
-```bash
-# Use single local media files
-python end2end.py --query-type use_image --image-path /path/to/image.jpg
-python end2end.py --query-type use_video --video-path /path/to/video.mp4
-python end2end.py --query-type use_audio --audio-path /path/to/audio.wav
-
-# Combine multiple local media files
-python end2end.py --query-type mixed_modalities \
-    --video-path /path/to/video.mp4 \
-    --image-path /path/to/image.jpg \
-    --audio-path /path/to/audio.wav
-
-# Use audio from video file
-python end2end.py --query-type use_audio_in_video --video-path /path/to/video.mp4
-
-```
-
-If media file paths are not provided, the script will use default assets. Supported query types:
-- `use_image`: Image input only
-- `use_video`: Video input only
-- `use_audio`: Audio input only
-- `mixed_modalities`: Audio + image + video
-- `use_audio_in_video`: Extract audio from video
-- `text`: Text-only query
-
 ### FAQ
 
 If you encounter error about backend of librosa, try to install ffmpeg with command below.

examples/offline_inference/qwen2_5_omni/end2end.py

@@ -1,25 +1,23 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
 """
-This example shows how to use vLLM-Omni for running offline inference
+This example shows how to use vLLM-omni for running offline inference
 with the correct prompt format on Qwen2.5-Omni
 """
 
+import argparse
 import os
-from typing import NamedTuple, Optional
+from typing import NamedTuple
 
-import librosa
-import numpy as np
 import soundfile as sf
-from PIL import Image
 from vllm.assets.audio import AudioAsset
 from vllm.assets.image import ImageAsset
-from vllm.assets.video import VideoAsset, video_to_ndarrays
+from vllm.assets.video import VideoAsset
 from vllm.multimodal.image import convert_image_mode
 from vllm.sampling_params import SamplingParams
 from vllm.utils import FlexibleArgumentParser
 
-from vllm_omni.entrypoints.omni import Omni
+from vllm_omni import OmniLLM
 
 SEED = 42
 
@@ -57,13 +55,7 @@ def get_text_query(question: str = None) -> QueryResult:
     )
 
 
-def get_mixed_modalities_query(
-    video_path: Optional[str] = None,
-    image_path: Optional[str] = None,
-    audio_path: Optional[str] = None,
-    num_frames: int = 16,
-    sampling_rate: int = 16000,
-) -> QueryResult:
+def get_mixed_modalities_query() -> QueryResult:
     question = "What is recited in the audio? What is the content of this image? Why is this video funny?"
     prompt = (
         f"<|im_start|>system\n{default_system}<|im_end|>\n"
@@ -73,49 +65,20 @@ def get_mixed_modalities_query(
         f"{question}<|im_end|>\n"
         f"<|im_start|>assistant\n"
     )
-
-    # Load video
-    if video_path:
-        if not os.path.exists(video_path):
-            raise FileNotFoundError(f"Video file not found: {video_path}")
-        video_frames = video_to_ndarrays(video_path, num_frames=num_frames)
-    else:
-        video_frames = VideoAsset(name="baby_reading", num_frames=num_frames).np_ndarrays
-
-    # Load image
-    if image_path:
-        if not os.path.exists(image_path):
-            raise FileNotFoundError(f"Image file not found: {image_path}")
-        pil_image = Image.open(image_path)
-        image_data = convert_image_mode(pil_image, "RGB")
-    else:
-        image_data = convert_image_mode(ImageAsset("cherry_blossom").pil_image, "RGB")
-
-    # Load audio
-    if audio_path:
-        if not os.path.exists(audio_path):
-            raise FileNotFoundError(f"Audio file not found: {audio_path}")
-        audio_signal, sr = librosa.load(audio_path, sr=sampling_rate)
-        audio_data = (audio_signal.astype(np.float32), sr)
-    else:
-        audio_data = AudioAsset("mary_had_lamb").audio_and_sample_rate
-
     return QueryResult(
         inputs={
             "prompt": prompt,
             "multi_modal_data": {
-                "audio": audio_data,
-                "image": image_data,
-                "video": video_frames,
+                "audio": AudioAsset("mary_had_lamb").audio_and_sample_rate,
+                "image": convert_image_mode(ImageAsset("cherry_blossom").pil_image, "RGB"),
+                "video": VideoAsset(name="baby_reading", num_frames=16).np_ndarrays,
             },
         },
         limit_mm_per_prompt={"audio": 1, "image": 1, "video": 1},
     )
 
 
-def get_use_audio_in_video_query(
-    video_path: Optional[str] = None, num_frames: int = 16, sampling_rate: int = 16000
-) -> QueryResult:
+def get_use_audio_in_video_query() -> QueryResult:
     question = "Describe the content of the video, then convert what the baby say into text."
     prompt = (
         f"<|im_start|>system\n{default_system}<|im_end|>\n"
@@ -123,24 +86,14 @@ def get_use_audio_in_video_query(
         f"{question}<|im_end|>\n"
         f"<|im_start|>assistant\n"
     )
-
-    if video_path:
-        if not os.path.exists(video_path):
-            raise FileNotFoundError(f"Video file not found: {video_path}")
-        video_frames = video_to_ndarrays(video_path, num_frames=num_frames)
-        # Extract audio from video file
-        audio_signal, sr = librosa.load(video_path, sr=sampling_rate)
-        audio = (audio_signal.astype(np.float32), sr)
-    else:
-        asset = VideoAsset(name="baby_reading", num_frames=num_frames)
-        video_frames = asset.np_ndarrays
-        audio = asset.get_audio(sampling_rate=sampling_rate)
+    asset = VideoAsset(name="baby_reading", num_frames=16)
+    audio = asset.get_audio(sampling_rate=16000)
 
     return QueryResult(
         inputs={
             "prompt": prompt,
             "multi_modal_data": {
-                "video": video_frames,
+                "video": asset.np_ndarrays,
                 "audio": audio,
             },
             "mm_processor_kwargs": {
@@ -151,7 +104,7 @@ def get_use_audio_in_video_query(
     )
 
 
-def get_multi_audios_query(audio_path: Optional[str] = None, sampling_rate: int = 16000) -> QueryResult:
+def get_multi_audios_query() -> QueryResult:
     question = "Are these two audio clips the same?"
     prompt = (
         f"<|im_start|>system\n{default_system}<|im_end|>\n"
@@ -160,28 +113,14 @@ def get_multi_audios_query(audio_path: Optional[str] = None, sampling_rate: int
         f"{question}<|im_end|>\n"
         f"<|im_start|>assistant\n"
     )
-
-    if audio_path:
-        if not os.path.exists(audio_path):
-            raise FileNotFoundError(f"Audio file not found: {audio_path}")
-        audio_signal, sr = librosa.load(audio_path, sr=sampling_rate)
-        audio_data = (audio_signal.astype(np.float32), sr)
-        # Use the provided audio as the first audio, default as second
-        audio_list = [
-            audio_data,
-            AudioAsset("mary_had_lamb").audio_and_sample_rate,
-        ]
-    else:
-        audio_list = [
-            AudioAsset("winning_call").audio_and_sample_rate,
-            AudioAsset("mary_had_lamb").audio_and_sample_rate,
-        ]
-
     return QueryResult(
         inputs={
             "prompt": prompt,
             "multi_modal_data": {
-                "audio": audio_list,
+                "audio": [
+                    AudioAsset("winning_call").audio_and_sample_rate,
+                    AudioAsset("mary_had_lamb").audio_and_sample_rate,
+                ],
             },
         },
         limit_mm_per_prompt={
@@ -190,137 +129,19 @@ def get_multi_audios_query(audio_path: Optional[str] = None, sampling_rate: int
     )
 
 
-def get_image_query(question: str = None, image_path: Optional[str] = None) -> QueryResult:
-    if question is None:
-        question = "What is the content of this image?"
-    prompt = (
-        f"<|im_start|>system\n{default_system}<|im_end|>\n"
-        "<|im_start|>user\n<|vision_bos|><|IMAGE|><|vision_eos|>"
-        f"{question}<|im_end|>\n"
-        f"<|im_start|>assistant\n"
-    )
-
-    if image_path:
-        if not os.path.exists(image_path):
-            raise FileNotFoundError(f"Image file not found: {image_path}")
-        pil_image = Image.open(image_path)
-        image_data = convert_image_mode(pil_image, "RGB")
-    else:
-        image_data = convert_image_mode(ImageAsset("cherry_blossom").pil_image, "RGB")
-
-    return QueryResult(
-        inputs={
-            "prompt": prompt,
-            "multi_modal_data": {
-                "image": image_data,
-            },
-        },
-        limit_mm_per_prompt={"image": 1},
-    )
-
-
-def get_video_query(question: str = None, video_path: Optional[str] = None, num_frames: int = 16) -> QueryResult:
-    if question is None:
-        question = "Why is this video funny?"
-    prompt = (
-        f"<|im_start|>system\n{default_system}<|im_end|>\n"
-        "<|im_start|>user\n<|vision_bos|><|VIDEO|><|vision_eos|>"
-        f"{question}<|im_end|>\n"
-        f"<|im_start|>assistant\n"
-    )
-
-    if video_path:
-        if not os.path.exists(video_path):
-            raise FileNotFoundError(f"Video file not found: {video_path}")
-        video_frames = video_to_ndarrays(video_path, num_frames=num_frames)
-    else:
-        video_frames = VideoAsset(name="baby_reading", num_frames=num_frames).np_ndarrays
-
-    return QueryResult(
-        inputs={
-            "prompt": prompt,
-            "multi_modal_data": {
-                "video": video_frames,
-            },
-        },
-        limit_mm_per_prompt={"video": 1},
-    )
-
-
-def get_audio_query(question: str = None, audio_path: Optional[str] = None, sampling_rate: int = 16000) -> QueryResult:
-    if question is None:
-        question = "What is the content of this audio?"
-    prompt = (
-        f"<|im_start|>system\n{default_system}<|im_end|>\n"
-        "<|im_start|>user\n<|audio_bos|><|AUDIO|><|audio_eos|>"
-        f"{question}<|im_end|>\n"
-        f"<|im_start|>assistant\n"
-    )
-
-    if audio_path:
-        if not os.path.exists(audio_path):
-            raise FileNotFoundError(f"Audio file not found: {audio_path}")
-        audio_signal, sr = librosa.load(audio_path, sr=sampling_rate)
-        audio_data = (audio_signal.astype(np.float32), sr)
-    else:
-        audio_data = AudioAsset("mary_had_lamb").audio_and_sample_rate
-
-    return QueryResult(
-        inputs={
-            "prompt": prompt,
-            "multi_modal_data": {
-                "audio": audio_data,
-            },
-        },
-        limit_mm_per_prompt={"audio": 1},
-    )
-
-
 query_map = {
     "mixed_modalities": get_mixed_modalities_query,
     "use_audio_in_video": get_use_audio_in_video_query,
     "multi_audios": get_multi_audios_query,
-    "use_image": get_image_query,
-    "use_video": get_video_query,
-    "use_audio": get_audio_query,
     "text": get_text_query,
 }
 
 
 def main(args):
     model_name = "Qwen/Qwen2.5-Omni-7B"
+    query_result = query_map[args.query_type]()
 
-    # Get paths from args
-    video_path = getattr(args, "video_path", None)
-    image_path = getattr(args, "image_path", None)
-    audio_path = getattr(args, "audio_path", None)
-    num_frames = getattr(args, "num_frames", 16)
-    sampling_rate = getattr(args, "sampling_rate", 16000)
-
-    # Get the query function and call it with appropriate parameters
-    query_func = query_map[args.query_type]
-    if args.query_type == "mixed_modalities":
-        query_result = query_func(
-            video_path=video_path,
-            image_path=image_path,
-            audio_path=audio_path,
-            num_frames=num_frames,
-            sampling_rate=sampling_rate,
-        )
-    elif args.query_type == "use_audio_in_video":
-        query_result = query_func(video_path=video_path, num_frames=num_frames, sampling_rate=sampling_rate)
-    elif args.query_type == "multi_audios":
-        query_result = query_func(audio_path=audio_path, sampling_rate=sampling_rate)
-    elif args.query_type == "use_image":
-        query_result = query_func(image_path=image_path)
-    elif args.query_type == "use_video":
-        query_result = query_func(video_path=video_path, num_frames=num_frames)
-    elif args.query_type == "use_audio":
-        query_result = query_func(audio_path=audio_path, sampling_rate=sampling_rate)
-    else:
-        query_result = query_func()
-
-    omni_llm = Omni(
+    omni_llm = OmniLLM(
         model=model_name,
         log_stats=args.enable_stats,
         log_file=("omni_llm_pipeline.log" if args.enable_stats else None),
@@ -375,35 +196,43 @@ def main(args):
 
     omni_outputs = omni_llm.generate(prompts, sampling_params_list)
 
-    # Determine output directory: prefer --output-dir; fallback to --output-wav
-    output_dir = args.output_dir if getattr(args, "output_dir", None) else args.output_wav
-    os.makedirs(output_dir, exist_ok=True)
+    output_dir = None
+    if args.save_results:
+        # Determine output directory: prefer --output-dir; fallback to --output-wav
+        output_dir = args.output_dir if getattr(args, "output_dir", None) else args.output_wav
+        os.makedirs(output_dir, exist_ok=True)
     for stage_outputs in omni_outputs:
         if stage_outputs.final_output_type == "text":
             for output in stage_outputs.request_output:
                 request_id = int(output.request_id)
                 text_output = output.outputs[0].text
-                # Save aligned text file per request
-                prompt_text = prompts[request_id]["prompt"]
-                out_txt = os.path.join(output_dir, f"{request_id:05d}.txt")
-                lines = []
-                lines.append("Prompt:\n")
-                lines.append(str(prompt_text) + "\n")
-                lines.append("vllm_text_output:\n")
-                lines.append(str(text_output).strip() + "\n")
-                try:
-                    with open(out_txt, "w", encoding="utf-8") as f:
-                        f.writelines(lines)
-                except Exception as e:
-                    print(f"[Warn] Failed writing text file {out_txt}: {e}")
-                print(f"Request ID: {request_id}, Text saved to {out_txt}")
+                if args.save_results:
+                    # Save aligned text file per request
+                    prompt_text = prompts[request_id]["prompt"]
+                    out_txt = os.path.join(output_dir, f"{request_id:05d}.txt")
+                    lines = []
+                    lines.append("Prompt:\n")
+                    lines.append(str(prompt_text) + "\n")
+                    lines.append("vllm_text_output:\n")
+                    lines.append(str(text_output).strip() + "\n")
+                    try:
+                        with open(out_txt, "w", encoding="utf-8") as f:
+                            f.writelines(lines)
+                    except Exception as e:
+                        print(f"[Warn] Failed writing text file {out_txt}: {e}")
+                    print(f"Request ID: {request_id}, Text saved to {out_txt}")
+                else:
+                    print(f"Request ID: {request_id}, Text output:\n{text_output}")
         elif stage_outputs.final_output_type == "audio":
             for output in stage_outputs.request_output:
                 request_id = int(output.request_id)
                 audio_tensor = output.multimodal_output["audio"]
-                output_wav = os.path.join(output_dir, f"output_{output.request_id}.wav")
-                sf.write(output_wav, audio_tensor.detach().cpu().numpy(), samplerate=24000)
-                print(f"Request ID: {request_id}, Saved audio to {output_wav}")
+                if args.save_results:
+                    output_wav = os.path.join(output_dir, f"output_{output.request_id}.wav")
+                    sf.write(output_wav, audio_tensor.detach().cpu().numpy(), samplerate=24000)
+                    print(f"Request ID: {request_id}, Saved audio to {output_wav}")
+                else:
+                    print(f"Request ID: {request_id}, Received audio output (not saved).")
 
 
 def parse_args():
@@ -446,6 +275,12 @@ def parse_args():
         default=65536,
         help="Threshold for using shared memory in bytes (default: 65536)",
     )
+    parser.add_argument(
+        "--save-results",
+        action=argparse.BooleanOptionalAction,
+        default=True,
+        help="Save text/audio outputs to files (default: enabled). Use --no-save-results to disable.",
+    )
     parser.add_argument(
         "--output-wav",
         default="output_audio",
@@ -463,39 +298,6 @@ def parse_args():
         default=None,
         help="Path to a .txt file with one prompt per line (preferred).",
     )
-    parser.add_argument(
-        "--video-path",
-        "-v",
-        type=str,
-        default=None,
-        help="Path to local video file. If not provided, uses default video asset.",
-    )
-    parser.add_argument(
-        "--image-path",
-        "-i",
-        type=str,
-        default=None,
-        help="Path to local image file. If not provided, uses default image asset.",
-    )
-    parser.add_argument(
-        "--audio-path",
-        "-a",
-        type=str,
-        default=None,
-        help="Path to local audio file. If not provided, uses default audio asset.",
-    )
-    parser.add_argument(
-        "--num-frames",
-        type=int,
-        default=16,
-        help="Number of frames to extract from video (default: 16).",
-    )
-    parser.add_argument(
-        "--sampling-rate",
-        type=int,
-        default=16000,
-        help="Sampling rate for audio loading (default: 16000).",
-    )
 
     return parser.parse_args()
 

examples/offline_inference/qwen2_5_omni/extract_prompts.py

@@ -4,7 +4,7 @@ from typing import Optional
 
 
 def extract_prompt(line: str) -> Optional[str]:
-    # Extract the content between the first '|' and the second '|'
+    # 提取第一个 '|' 与第二个 '|' 之间的内容
     i = line.find("|")
     if i == -1:
         return None

examples/offline_inference/qwen2_5_omni/run_single_prompt.sh

@@ -1,2 +1,2 @@
 python end2end.py --output-wav output_audio \
-                  --query-type mixed_modalities
+                  --query-type use_audio_in_video

examples/offline_inference/qwen3_omni/README.md

@@ -1,59 +0,0 @@
-# Offline Example of vLLM-Omni for Qwen3-Omni
-
-## 🛠️ Installation
-
-Please refer to [README.md](../../../README.md)
-
-## Run examples (Qwen3-Omni)
-
-### Multiple Prompts
-Download dataset from [seed_tts](https://drive.google.com/file/d/1GlSjVfSHkW3-leKKBlfrjuuTGqQ_xaLP/edit). For processing dataset please refer to [Qwen2.5-Omni README.md](../qwen2_5_omni/README.md)
-Get into the example folder
-```bash
-cd examples/offline_inference/qwen3_omni
-```
-Then run the command below.
-```bash
-bash run_multiple_prompts.sh
-```
-### Single Prompt
-Get into the example folder
-```bash
-cd examples/offline_inference/qwen3_omni
-```
-Then run the command below.
-```bash
-bash run_single_prompt.sh
-```
-If you have not enough memory, you can set thinker with tensor parallel. Just run the command below.
-```bash
-bash run_single_prompt_tp.sh
-```
-
-#### Using Local Media Files
-The `end2end.py` script supports local media files (audio, video, image) via command-line arguments:
-
-```bash
-# Use local video file
-python end2end.py --query-type use_video --video-path /path/to/video.mp4
-
-# Use local image file
-python end2end.py --query-type use_image --image-path /path/to/image.jpg
-
-# Use local audio file
-python end2end.py --query-type use_audio --audio-path /path/to/audio.wav
-```
-
-If media file paths are not provided, the script will use default assets. Supported query types:
-- `use_video`: Video input
-- `use_image`: Image input
-- `use_audio`: Audio input
-- `text`: Text-only query
-
-### FAQ
-
-If you encounter error about backend of librosa, try to install ffmpeg with command below.
-```
-sudo apt update
-sudo apt install ffmpeg
-```

examples/offline_inference/qwen3_omni/end2end.py

@@ -1,368 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-"""
-This example shows how to use vLLM for running offline inference
-with the correct prompt format on Qwen3-Omni (thinker only).
-"""
-
-import os
-from typing import NamedTuple, Optional
-
-import librosa
-import numpy as np
-import soundfile as sf
-from PIL import Image
-from vllm import SamplingParams
-from vllm.assets.audio import AudioAsset
-from vllm.assets.image import ImageAsset
-from vllm.assets.video import VideoAsset, video_to_ndarrays
-from vllm.multimodal.image import convert_image_mode
-from vllm.utils import FlexibleArgumentParser
-
-from vllm_omni.entrypoints.omni import Omni
-
-SEED = 42
-
-
-class QueryResult(NamedTuple):
-    inputs: dict
-    limit_mm_per_prompt: dict[str, int]
-
-
-# NOTE: The default `max_num_seqs` and `max_model_len` may result in OOM on
-# lower-end GPUs.
-# Unless specified, these settings have been tested to work on a single L4.
-
-default_system = (
-    "You are Qwen, a virtual human developed by the Qwen Team, Alibaba "
-    "Group, capable of perceiving auditory and visual inputs, as well as "
-    "generating text and speech."
-)
-
-
-def get_text_query(question: str = None) -> QueryResult:
-    if question is None:
-        question = "Explain the system architecture for a scalable audio generation pipeline. Answer in 15 words."
-    prompt = (
-        f"<|im_start|>system\n{default_system}<|im_end|>\n"
-        "<|im_start|>user\n"
-        f"{question}<|im_end|>\n"
-        f"<|im_start|>assistant\n"
-    )
-    return QueryResult(
-        inputs={
-            "prompt": prompt,
-        },
-        limit_mm_per_prompt={},
-    )
-
-
-def get_video_query(question: str = None, video_path: Optional[str] = None, num_frames: int = 16) -> QueryResult:
-    if question is None:
-        question = "Why is this video funny?"
-    prompt = (
-        f"<|im_start|>system\n{default_system}<|im_end|>\n"
-        "<|im_start|>user\n<|vision_start|><|video_pad|><|vision_end|>"
-        f"{question}<|im_end|>\n"
-        f"<|im_start|>assistant\n"
-    )
-
-    if video_path:
-        if not os.path.exists(video_path):
-            raise FileNotFoundError(f"Video file not found: {video_path}")
-        video_frames = video_to_ndarrays(video_path, num_frames=num_frames)
-    else:
-        video_frames = VideoAsset(name="baby_reading", num_frames=num_frames).np_ndarrays
-
-    return QueryResult(
-        inputs={
-            "prompt": prompt,
-            "multi_modal_data": {
-                "video": video_frames,
-            },
-        },
-        limit_mm_per_prompt={"video": 1},
-    )
-
-
-def get_image_query(question: str = None, image_path: Optional[str] = None) -> QueryResult:
-    if question is None:
-        question = "What is the content of this image?"
-    prompt = (
-        f"<|im_start|>system\n{default_system}<|im_end|>\n"
-        "<|im_start|>user\n<|vision_start|><|image_pad|><|vision_end|>"
-        f"{question}<|im_end|>\n"
-        f"<|im_start|>assistant\n"
-    )
-
-    if image_path:
-        if not os.path.exists(image_path):
-            raise FileNotFoundError(f"Image file not found: {image_path}")
-        pil_image = Image.open(image_path)
-        image_data = convert_image_mode(pil_image, "RGB")
-    else:
-        image_data = convert_image_mode(ImageAsset("cherry_blossom").pil_image, "RGB")
-
-    return QueryResult(
-        inputs={
-            "prompt": prompt,
-            "multi_modal_data": {
-                "image": image_data,
-            },
-        },
-        limit_mm_per_prompt={"image": 1},
-    )
-
-
-def get_audio_query(question: str = None, audio_path: Optional[str] = None, sampling_rate: int = 16000) -> QueryResult:
-    if question is None:
-        question = "What is the content of this audio?"
-    prompt = (
-        f"<|im_start|>system\n{default_system}<|im_end|>\n"
-        "<|im_start|>user\n<|audio_start|><|audio_pad|><|audio_end|>"
-        f"{question}<|im_end|>\n"
-        f"<|im_start|>assistant\n"
-    )
-
-    if audio_path:
-        if not os.path.exists(audio_path):
-            raise FileNotFoundError(f"Audio file not found: {audio_path}")
-        audio_signal, sr = librosa.load(audio_path, sr=sampling_rate)
-        audio_data = (audio_signal.astype(np.float32), sr)
-    else:
-        audio_data = AudioAsset("mary_had_lamb").audio_and_sample_rate
-
-    return QueryResult(
-        inputs={
-            "prompt": prompt,
-            "multi_modal_data": {
-                "audio": audio_data,
-            },
-        },
-        limit_mm_per_prompt={"audio": 1},
-    )
-
-
-query_map = {
-    "text": get_text_query,
-    "use_audio": get_audio_query,
-    "use_image": get_image_query,
-    "use_video": get_video_query,
-}
-
-
-def main(args):
-    model_name = "Qwen/Qwen3-Omni-30B-A3B-Instruct"
-
-    # Get paths from args
-    video_path = getattr(args, "video_path", None)
-    image_path = getattr(args, "image_path", None)
-    audio_path = getattr(args, "audio_path", None)
-
-    # Get the query function and call it with appropriate parameters
-    query_func = query_map[args.query_type]
-    if args.query_type == "use_video":
-        query_result = query_func(video_path=video_path, num_frames=getattr(args, "num_frames", 16))
-    elif args.query_type == "use_image":
-        query_result = query_func(image_path=image_path)
-    elif args.query_type == "use_audio":
-        query_result = query_func(audio_path=audio_path, sampling_rate=getattr(args, "sampling_rate", 16000))
-    else:
-        query_result = query_func()
-
-    omni_llm = Omni(
-        model=model_name,
-        stage_configs_path=args.stage_configs_path,
-    )
-
-    thinker_sampling_params = SamplingParams(
-        temperature=0.4,
-        top_p=0.9,
-        top_k=-1,
-        max_tokens=1200,
-        repetition_penalty=1.05,
-        logit_bias={},
-        seed=SEED,
-    )
-
-    talker_sampling_params = SamplingParams(
-        temperature=0.9,
-        top_k=50,
-        max_tokens=4096,
-        seed=SEED,
-        detokenize=False,
-        repetition_penalty=1.05,
-        stop_token_ids=[2150],  # TALKER_CODEC_EOS_TOKEN_ID
-    )
-
-    # Sampling parameters for Code2Wav stage (audio generation)
-    code2wav_sampling_params = SamplingParams(
-        temperature=0.0,
-        top_p=1.0,
-        top_k=-1,
-        max_tokens=4096 * 16,
-        seed=SEED,
-        detokenize=True,
-        repetition_penalty=1.1,
-    )
-
-    sampling_params_list = [
-        thinker_sampling_params,
-        talker_sampling_params,  # code predictor is integrated into talker for Qwen3 Omni
-        code2wav_sampling_params,
-    ]
-
-    if args.txt_prompts is None:
-        prompts = [query_result.inputs for _ in range(args.num_prompts)]
-    else:
-        assert args.query_type == "text", "txt-prompts is only supported for text query type"
-        with open(args.txt_prompts, encoding="utf-8") as f:
-            lines = [ln.strip() for ln in f.readlines()]
-            prompts = [get_text_query(ln).inputs for ln in lines if ln != ""]
-            print(f"[Info] Loaded {len(prompts)} prompts from {args.txt_prompts}")
-
-    omni_outputs = omni_llm.generate(prompts, sampling_params_list)
-    # Determine output directory: prefer --output-dir; fallback to --output-wav
-    output_dir = args.output_dir if getattr(args, "output_dir", None) else args.output_wav
-    os.makedirs(output_dir, exist_ok=True)
-
-    for stage_outputs in omni_outputs:
-        if stage_outputs.final_output_type == "text":
-            for output in stage_outputs.request_output:
-                request_id = int(output.request_id)
-                text_output = output.outputs[0].text
-                # Save aligned text file per request
-                prompt_text = prompts[request_id]["prompt"]
-                out_txt = os.path.join(output_dir, f"{request_id:05d}.txt")
-                lines = []
-                lines.append("Prompt:\n")
-                lines.append(str(prompt_text) + "\n")
-                lines.append("vllm_text_output:\n")
-                lines.append(str(text_output).strip() + "\n")
-                try:
-                    with open(out_txt, "w", encoding="utf-8") as f:
-                        f.writelines(lines)
-                except Exception as e:
-                    print(f"[Warn] Failed writing text file {out_txt}: {e}")
-                print(f"Request ID: {request_id}, Text saved to {out_txt}")
-        elif stage_outputs.final_output_type == "audio":
-            for output in stage_outputs.request_output:
-                request_id = int(output.request_id)
-                audio_tensor = output.multimodal_output["audio"]
-                output_wav = os.path.join(output_dir, f"output_{output.request_id}.wav")
-
-                # Convert to numpy array and ensure correct format
-                audio_numpy = audio_tensor.float().detach().cpu().numpy()
-
-                # Ensure audio is 1D (flatten if needed)
-                if audio_numpy.ndim > 1:
-                    audio_numpy = audio_numpy.flatten()
-
-                # Save audio file with explicit WAV format
-                sf.write(output_wav, audio_numpy, samplerate=24000, format="WAV")
-                print(f"Request ID: {request_id}, Saved audio to {output_wav}")
-
-
-def parse_args():
-    parser = FlexibleArgumentParser(description="Demo on using vLLM for offline inference with audio language models")
-    parser.add_argument(
-        "--query-type",
-        "-q",
-        type=str,
-        default="mixed_modalities",
-        choices=query_map.keys(),
-        help="Query type.",
-    )
-    parser.add_argument(
-        "--enable-stats",
-        action="store_true",
-        default=False,
-        help="Enable writing detailed statistics (default: disabled)",
-    )
-    parser.add_argument(
-        "--init-sleep-seconds",
-        type=int,
-        default=20,
-        help="Sleep seconds after starting each stage process to allow initialization (default: 20)",
-    )
-    parser.add_argument(
-        "--batch-timeout",
-        type=int,
-        default=5,
-        help="Timeout for batching in seconds (default: 5)",
-    )
-    parser.add_argument(
-        "--init-timeout",
-        type=int,
-        default=300,
-        help="Timeout for initializing stages in seconds (default: 300)",
-    )
-    parser.add_argument(
-        "--shm-threshold-bytes",
-        type=int,
-        default=65536,
-        help="Threshold for using shared memory in bytes (default: 65536)",
-    )
-    parser.add_argument(
-        "--output-wav",
-        default="output_audio",
-        help="[Deprecated] Output wav directory (use --output-dir).",
-    )
-    parser.add_argument(
-        "--num-prompts",
-        type=int,
-        default=1,
-        help="Number of prompts to generate.",
-    )
-    parser.add_argument(
-        "--txt-prompts",
-        type=str,
-        default=None,
-        help="Path to a .txt file with one prompt per line (preferred).",
-    )
-    parser.add_argument(
-        "--stage-configs-path",
-        type=str,
-        default=None,
-        help="Path to a stage configs file.",
-    )
-    parser.add_argument(
-        "--video-path",
-        "-v",
-        type=str,
-        default=None,
-        help="Path to local video file. If not provided, uses default video asset.",
-    )
-    parser.add_argument(
-        "--image-path",
-        "-i",
-        type=str,
-        default=None,
-        help="Path to local image file. If not provided, uses default image asset.",
-    )
-    parser.add_argument(
-        "--audio-path",
-        "-a",
-        type=str,
-        default=None,
-        help="Path to local audio file. If not provided, uses default audio asset.",
-    )
-    parser.add_argument(
-        "--num-frames",
-        type=int,
-        default=16,
-        help="Number of frames to extract from video (default: 16).",
-    )
-    parser.add_argument(
-        "--sampling-rate",
-        type=int,
-        default=16000,
-        help="Sampling rate for audio loading (default: 16000).",
-    )
-
-    return parser.parse_args()
-
-
-if __name__ == "__main__":
-    args = parse_args()
-    main(args)

examples/offline_inference/qwen3_omni/run_multiple_prompts.sh

@@ -1,3 +0,0 @@
-python end2end.py --output-wav output_audio \
-                  --query-type text \
-                  --txt-prompts top10.txt

examples/offline_inference/qwen3_omni/run_single_prompt.sh

@@ -1,2 +0,0 @@
-python end2end.py --output-wav output_audio \
-                  --query-type use_audio

examples/offline_inference/qwen3_omni/run_single_prompt_tp.sh

@@ -1,5 +0,0 @@
-python end2end.py --output-wav output_audio \
-                  --query-type use_audio \
-                  --init-sleep-seconds 90
-
-# init-sleep-seconds works to avoid two vLLM stages initialized at the same time within a card.

examples/offline_inference/qwen_image/README.md

@@ -1,43 +0,0 @@
-# Qwen-Image Offline Inference
-
-This folder provides two simple entrypoints for experimenting with `Qwen/Qwen-Image` using vLLM-Omni:
-
-- `text_to_image.py`: command-line script for single image generation.
-- `web_demo.py`: lightweight Gradio UI for interactive prompt/seed/CFG exploration.
-
-
-## Local CLI Usage
-
-```bash
-python text_to_image.py \
-  --prompt "a cup of coffee on the table" \
-  --seed 42 \
-  --cfg_scale 4.0 \
-  --num_images_per_prompt 1 \
-  --num_inference_steps 50 \
-  --height 1024 \
-  --width 1024 \
-  --output outputs/coffee.png
-```
-
-Key arguments:
-
-- `--prompt`: text description (string).
-- `--seed`: integer seed for deterministic sampling.
-- `--cfg_scale`: true CFG scale (model-specific guidance strength).
-- `--num_images_per_prompt`: number of images to generate per prompt (saves as `output`, `output_1`, ...).
-- `--num_inference_steps`: diffusion sampling steps (more steps = higher quality, slower).
-- `--height/--width`: output resolution (defaults 1024x1024).
-- `--output`: path to save the generated PNG.
-
-> ℹ️ Qwen-Image currently publishes best-effort presets at `1328x1328`, `1664x928`, `928x1664`, `1472x1140`, `1140x1472`, `1584x1056`, and `1056x1584`. Adjust `--height/--width` accordingly for the most reliable outcomes.
-
-## Web UI Demo
-
-Launch the gradio demo:
-
-```bash
-python gradio_demo.py --port 7862
-```
-
-Then open `http://localhost:7862/` on your local browser to interact with the web UI.

examples/offline_inference/qwen_image/gradio_demo.py

@@ -1,222 +0,0 @@
-import argparse
-from functools import lru_cache
-
-import gradio as gr
-import torch
-
-from vllm_omni.entrypoints.omni import Omni
-from vllm_omni.utils.platform_utils import detect_device_type, is_npu
-
-ASPECT_RATIOS: dict[str, tuple[int, int]] = {
-    "1:1": (1328, 1328),
-    "16:9": (1664, 928),
-    "9:16": (928, 1664),
-    "4:3": (1472, 1140),
-    "3:4": (1140, 1472),
-    "3:2": (1584, 1056),
-    "2:3": (1056, 1584),
-}
-ASPECT_RATIO_CHOICES = [f"{ratio} ({w}x{h})" for ratio, (w, h) in ASPECT_RATIOS.items()]
-
-
-def parse_args() -> argparse.Namespace:
-    parser = argparse.ArgumentParser(description="Gradio demo for Qwen-Image offline inference.")
-    parser.add_argument("--model", default="Qwen/Qwen-Image", help="Diffusion model name or local path.")
-    parser.add_argument(
-        "--height",
-        type=int,
-        default=1328,
-        help="Default image height (must match one of the supported presets).",
-    )
-    parser.add_argument(
-        "--width",
-        type=int,
-        default=1328,
-        help="Default image width (must match one of the supported presets).",
-    )
-    parser.add_argument("--default-prompt", default="a cup of coffee on the table", help="Initial prompt shown in UI.")
-    parser.add_argument("--default-seed", type=int, default=42, help="Initial seed shown in UI.")
-    parser.add_argument("--default-cfg-scale", type=float, default=4.0, help="Initial CFG scale shown in UI.")
-    parser.add_argument(
-        "--num_inference_steps",
-        type=int,
-        default=50,
-        help="Default number of denoising steps shown in the UI.",
-    )
-    parser.add_argument("--ip", default="127.0.0.1", help="Host/IP for Gradio `launch`.")
-    parser.add_argument("--port", type=int, default=7862, help="Port for Gradio `launch`.")
-    parser.add_argument("--share", action="store_true", help="Share the Gradio demo publicly.")
-    args = parser.parse_args()
-    args.aspect_ratio_label = next(
-        (ratio for ratio, dims in ASPECT_RATIOS.items() if dims == (args.width, args.height)),
-        None,
-    )
-    if args.aspect_ratio_label is None:
-        supported = ", ".join(f"{ratio} ({w}x{h})" for ratio, (w, h) in ASPECT_RATIOS.items())
-        parser.error(f"Unsupported resolution {args.width}x{args.height}. Please pick one of: {supported}.")
-    return args
-
-
-@lru_cache(maxsize=1)
-def get_omni(model_name: str) -> Omni:
-    # Enable VAE memory optimizations on NPU
-    vae_use_slicing = is_npu()
-    vae_use_tiling = is_npu()
-    return Omni(
-        model=model_name,
-        vae_use_slicing=vae_use_slicing,
-        vae_use_tiling=vae_use_tiling,
-    )
-
-
-def build_demo(args: argparse.Namespace) -> gr.Blocks:
-    device = detect_device_type()
-    omni = get_omni(args.model)
-
-    def run_inference(
-        prompt: str,
-        seed_value: float,
-        cfg_scale_value: float,
-        resolution_choice: str,
-        num_steps_value: float,
-        num_images_choice: float,
-    ):
-        if not prompt or not prompt.strip():
-            raise gr.Error("Please enter a non-empty prompt.")
-        ratio_label = resolution_choice.split(" ", 1)[0]
-        if ratio_label not in ASPECT_RATIOS:
-            raise gr.Error(f"Unsupported aspect ratio: {ratio_label}")
-        width, height = ASPECT_RATIOS[ratio_label]
-        try:
-            seed = int(seed_value)
-            num_steps = int(num_steps_value)
-            num_images = int(num_images_choice)
-        except (TypeError, ValueError) as exc:
-            raise gr.Error("Seed, inference steps, and number of images must be valid integers.") from exc
-        if num_steps <= 0:
-            raise gr.Error("Inference steps must be a positive integer.")
-        if num_images not in {1, 2, 3, 4}:
-            raise gr.Error("Number of images must be 1, 2, 3, or 4.")
-        generator = torch.Generator(device=device).manual_seed(seed)
-        images = omni.generate(
-            prompt.strip(),
-            height=height,
-            width=width,
-            generator=generator,
-            true_cfg_scale=float(cfg_scale_value),
-            num_inference_steps=num_steps,
-            num_images_per_prompt=num_images,
-            num_outputs_per_prompt=num_images,
-        )
-        return [img for img in images[:num_images]]
-
-    with gr.Blocks(
-        title="vLLM-Omni Web Serving Demo",
-        css="""
-        /* Left column button width */
-        .left-column button {
-            width: 100%;
-        }
-        /* Right preview area: fixed height, hide unnecessary buttons */
-        .fixed-image {
-            height: 660px;
-            display: flex;
-            flex-direction: column;
-            justify-content: center;
-            align-items: center;
-        }
-        .fixed-image .duplicate-button,
-        .fixed-image .svelte-drgfj2 {
-            display: none !important;
-        }
-        /* Gallery container: fill available space and center content */
-        #image-gallery {
-            width: 100%;
-            height: 100%;
-            display: flex;
-            align-items: center;
-            justify-content: center;
-        }
-        /* Gallery grid: center horizontally and vertically, set gap */
-        #image-gallery .grid {
-            display: flex;
-            flex-wrap: wrap;
-            justify-content: center;
-            align-items: center;
-            align-content: center;
-            gap: 16px;
-            width: 100%;
-            height: 100%;
-        }
-        /* Gallery grid items: center content */
-        #image-gallery .grid > div {
-            display: flex;
-            align-items: center;
-            justify-content: center;
-        }
-        /* Gallery images: limit max height, maintain aspect ratio */
-        .fixed-image img {
-            max-height: 660px !important;
-            width: auto !important;
-            object-fit: contain;
-        }
-        """,
-    ) as demo:
-        gr.Markdown("# vLLM-Omni Web Serving Demo")
-        gr.Markdown(f"**Model:** {args.model}")
-
-        with gr.Row():
-            with gr.Column(scale=1, elem_classes="left-column"):
-                prompt_input = gr.Textbox(
-                    label="Prompt",
-                    value=args.default_prompt,
-                    placeholder="Describe the image you want to generate...",
-                    lines=5,
-                )
-                seed_input = gr.Number(label="Seed", value=args.default_seed, precision=0)
-                cfg_input = gr.Number(label="CFG Scale", value=args.default_cfg_scale)
-                steps_input = gr.Number(
-                    label="Inference Steps",
-                    value=args.num_inference_steps,
-                    precision=0,
-                    minimum=1,
-                )
-                aspect_dropdown = gr.Dropdown(
-                    label="Aspect Ratio (W:H)",
-                    choices=ASPECT_RATIO_CHOICES,
-                    value=f"{args.aspect_ratio_label} ({ASPECT_RATIOS[args.aspect_ratio_label][0]}x{ASPECT_RATIOS[args.aspect_ratio_label][1]})",
-                )
-                num_images = gr.Dropdown(
-                    label="Number of images",
-                    choices=["1", "2", "3", "4"],
-                    value="1",
-                )
-                generate_btn = gr.Button("Generate", variant="primary")
-            with gr.Column(scale=2, elem_classes="fixed-image"):
-                gallery = gr.Gallery(
-                    label="Preview",
-                    columns=2,
-                    rows=2,
-                    height=660,
-                    allow_preview=True,
-                    show_label=True,
-                    elem_id="image-gallery",
-                )
-
-        generate_btn.click(
-            fn=run_inference,
-            inputs=[prompt_input, seed_input, cfg_input, aspect_dropdown, steps_input, num_images],
-            outputs=gallery,
-        )
-
-    return demo
-
-
-def main():
-    args = parse_args()
-    demo = build_demo(args)
-    demo.launch(server_name=args.ip, server_port=args.port, share=args.share)
-
-
-if __name__ == "__main__":
-    main()

examples/offline_inference/qwen_image/text_to_image.py

@@ -1,87 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-
-import argparse
-from pathlib import Path
-
-import torch
-
-from vllm_omni.entrypoints.omni import Omni
-from vllm_omni.utils.platform_utils import detect_device_type, is_npu
-
-
-def parse_args() -> argparse.Namespace:
-    parser = argparse.ArgumentParser(description="Generate an image with Qwen-Image.")
-    parser.add_argument("--model", default="Qwen/Qwen-Image", help="Diffusion model name or local path.")
-    parser.add_argument("--prompt", default="a cup of coffee on the table", help="Text prompt for image generation.")
-    parser.add_argument("--seed", type=int, default=42, help="Random seed for deterministic results.")
-    parser.add_argument(
-        "--cfg_scale",
-        type=float,
-        default=4.0,
-        help="True classifier-free guidance scale specific to Qwen-Image.",
-    )
-    parser.add_argument("--height", type=int, default=1024, help="Height of generated image.")
-    parser.add_argument("--width", type=int, default=1024, help="Width of generated image.")
-    parser.add_argument(
-        "--output",
-        type=str,
-        default="qwen_image_output.png",
-        help="Path to save the generated image (PNG).",
-    )
-    parser.add_argument(
-        "--num_images_per_prompt",
-        type=int,
-        default=1,
-        help="Number of images to generate for the given prompt.",
-    )
-    parser.add_argument(
-        "--num_inference_steps",
-        type=int,
-        default=50,
-        help="Number of denoising steps for the diffusion sampler.",
-    )
-    return parser.parse_args()
-
-
-def main():
-    args = parse_args()
-    device = detect_device_type()
-    generator = torch.Generator(device=device).manual_seed(args.seed)
-
-    # Enable VAE memory optimizations on NPU
-    vae_use_slicing = is_npu()
-    vae_use_tiling = is_npu()
-
-    omni = Omni(
-        model=args.model,
-        vae_use_slicing=vae_use_slicing,
-        vae_use_tiling=vae_use_tiling,
-    )
-    images = omni.generate(
-        args.prompt,
-        height=args.height,
-        width=args.width,
-        generator=generator,
-        true_cfg_scale=args.cfg_scale,
-        num_inference_steps=args.num_inference_steps,
-        num_images_per_prompt=args.num_images_per_prompt,
-        num_outputs_per_prompt=args.num_images_per_prompt,
-    )
-
-    output_path = Path(args.output)
-    output_path.parent.mkdir(parents=True, exist_ok=True)
-    suffix = output_path.suffix or ".png"
-    stem = output_path.stem or "qwen_image_output"
-    if args.num_images_per_prompt <= 1:
-        images[0].save(output_path)
-        print(f"Saved generated image to {output_path}")
-    else:
-        for idx, img in enumerate(images):
-            save_path = output_path.parent / f"{stem}_{idx}{suffix}"
-            img.save(save_path)
-            print(f"Saved generated image to {save_path}")
-
-
-if __name__ == "__main__":
-    main()

examples/online_serving/README.md

@@ -0,0 +1,77 @@
+# Online serving Example of vLLM-omni for Qwen2.5-omni
+
+## 🛠️ Installation
+
+Please refer to [README.md](../../README.md)
+
+## Run examples (Qwen2.5-omni)
+
+Launch the server
+```bash
+vllm serve Qwen/Qwen2.5-Omni-7B --omni --port 8091
+```
+
+If you have custom stage configs file, launch the server with command below
+```bash
+vllm serve Qwen/Qwen2.5-Omni-7B --omni --port 8091 --stage-configs-path /path/to/stage_configs_file
+```
+
+Get into the example folder
+```bash
+cd examples/online_serving
+```
+
+Send request via python
+```bash
+python openai_chat_completion_client_for_multimodal_generation.py --query-type mixed_modalities
+```
+
+Send request via curl
+```bash
+bash run_curl_multimodal_generation.sh mixed_modalities
+```
+
+### FAQ
+
+If you encounter error about backend of librosa, try to install ffmpeg with command below.
+```
+sudo apt update
+sudo apt install ffmpeg
+```
+
+## Run Local Web UI Demo
+
+This Web UI demo allows users to interact with the model through a web browser.
+
+### Running Gradio Demo
+
+Once vllm and vllm-omni are installed, you can launch the web service built on AsyncOmniLLM by
+
+```bash
+python gradio_demo.py  --model Qwen/Qwen2.5-Omni-7B --port 7861
+```
+
+Then open `http://localhost:7861/` on your local browser to interact with the web UI.
+
+
+### Options
+
+The gradio demo also supports running with an existing API server and can be customized with the following arguments.
+
+
+```bash
+python gradio_demo.py \
+    --model Qwen/Qwen2.5-Omni-7B \
+    --use-api-server \
+    --api-base http://localhost:8091/v1 \
+    --ip 127.0.0.1 \
+    --port 7861
+```
+
+- `--model`: Model name
+- `--use-api-server`: If set, connect to an existing vLLM HTTP API server instead of running AsyncOmniLLM locally.
+- `--api-base`: Base URL for vllm serve (only used when `use-api-server` is set, default: http://localhost:8091/v1)
+- `--ip`: Host/IP for Gradio server (default: 127.0.0.1)
+- `--port`: Port for Gradio server (default: 7861)
+- `--stage-configs-path`: Path to custom stage configs YAML file (optional)
+- `--share`: Share the Gradio demo publicly (creates a public link)

examples/online_serving/gradio_demo.py

@@ -15,7 +15,7 @@ from PIL import Image
 from vllm.assets.video import video_get_metadata, video_to_ndarrays
 from vllm.sampling_params import SamplingParams
 
-from vllm_omni.entrypoints.async_omni import AsyncOmni
+from vllm_omni.entrypoints.async_omni_llm import AsyncOmniLLM
 
 # Import utils from offline inference example
 sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../offline_inference/qwen2_5_omni"))
@@ -236,8 +236,8 @@ def process_video_file(
     return frames, metadata, audio_tuple
 
 
-async def run_inference_async_omni(
-    omni: AsyncOmni,
+async def run_inference_async_omni_llm(
+    omni_llm: AsyncOmniLLM,
     sampling_params: list[SamplingParams],
     prompt_args_template: SimpleNamespace,
     user_prompt: str,
@@ -246,7 +246,7 @@ async def run_inference_async_omni(
     video_file: Optional[str] = None,
     use_audio_in_video: bool = False,
 ):
-    """Run inference using AsyncOmni directly with multimodal support."""
+    """Run inference using AsyncOmniLLM directly with multimodal support."""
     if not user_prompt.strip() and not audio_file and not image_file and not video_file:
         return "Please provide at least a text prompt or multimodal input.", None
 
@@ -328,7 +328,7 @@ async def run_inference_async_omni(
         text_outputs: list[str] = []
         audio_output = None
 
-        async for stage_outputs in omni.generate(
+        async for stage_outputs in omni_llm.generate(
             prompt=omni_prompt,
             request_id=request_id,
             sampling_params_list=sampling_params,
@@ -362,12 +362,12 @@ async def run_inference_async_omni(
 
 
 def build_interface(
-    omni: AsyncOmni,
+    omni_llm: AsyncOmniLLM,
     sampling_params: list[SamplingParams],
     prompt_args_template: SimpleNamespace,
     model: str,
 ):
-    """Build Gradio interface for AsyncOmni mode."""
+    """Build Gradio interface for AsyncOmniLLM mode."""
 
     async def run_inference(
         user_prompt: str,
@@ -376,8 +376,8 @@ def build_interface(
         video_file: Optional[str],
         use_audio_in_video: bool,
     ):
-        return await run_inference_async_omni(
-            omni,
+        return await run_inference_async_omni_llm(
+            omni_llm,
             sampling_params,
             prompt_args_template,
             user_prompt,
@@ -475,7 +475,7 @@ def build_interface(
 
 def main():
     args = parse_args()
-    omni = None
+    omni_llm = None
 
     model_name = "/".join(args.model.split("/")[-2:])
     assert model_name in SUPPORTED_MODELS, (
@@ -485,9 +485,9 @@ def main():
     # Register signal handlers for graceful shutdown
     def signal_handler(sig, frame):
         print("\nReceived interrupt signal, shutting down...")
-        if omni is not None:
+        if omni_llm is not None:
             try:
-                omni.shutdown()
+                omni_llm.shutdown()
             except Exception as e:
                 print(f"Error during shutdown: {e}")
         sys.exit(0)
@@ -495,21 +495,21 @@ def main():
     signal.signal(signal.SIGINT, signal_handler)
     signal.signal(signal.SIGTERM, signal_handler)
 
-    print(f"Initializing AsyncOmni with model: {args.model}")
+    print(f"Initializing AsyncOmniLLM with model: {args.model}")
     if args.stage_configs_path:
         print(f"Using custom stage configs: {args.stage_configs_path}")
 
     sampling_params = build_sampling_params(SEED, model_name)
-    omni = AsyncOmni(
+    omni_llm = AsyncOmniLLM(
         model=args.model,
         stage_configs_path=args.stage_configs_path,
         init_timeout=ASYNC_INIT_TIMEOUT,
     )
-    print("✓ AsyncOmni initialized successfully")
+    print("✓ AsyncOmniLLM initialized successfully")
     prompt_args_template = create_prompt_args(args)
 
     demo = build_interface(
-        omni,
+        omni_llm,
         sampling_params,
         prompt_args_template,
         args.model,
@@ -524,9 +524,9 @@ def main():
         print("\nShutting down...")
     finally:
         # Cleanup
-        if omni is not None:
+        if omni_llm is not None:
             try:
-                omni.shutdown()
+                omni_llm.shutdown()
             except Exception as e:
                 print(f"Error during cleanup: {e}")
 

examples/online_serving/openai_chat_completion_client_for_multimodal_generation.py

@@ -0,0 +1,228 @@
+import base64
+
+import requests
+from openai import OpenAI
+from vllm.assets.audio import AudioAsset
+from vllm.utils import FlexibleArgumentParser
+
+# Modify OpenAI's API key and API base to use vLLM's API server.
+openai_api_key = "EMPTY"
+openai_api_base = "http://localhost:8091/v1"
+
+client = OpenAI(
+    # defaults to os.environ.get("OPENAI_API_KEY")
+    api_key=openai_api_key,
+    base_url=openai_api_base,
+)
+
+SEED = 42
+
+
+def encode_base64_content_from_url(content_url: str) -> str:
+    """Encode a content retrieved from a remote url to base64 format."""
+
+    with requests.get(content_url) as response:
+        response.raise_for_status()
+        result = base64.b64encode(response.content).decode("utf-8")
+
+    return result
+
+
+def get_system_prompt():
+    return {
+        "role": "system",
+        "content": [
+            {
+                "type": "text",
+                "text": (
+                    "You are Qwen, a virtual human developed by the Qwen Team, "
+                    "Alibaba Group, capable of perceiving auditory and visual inputs, "
+                    "as well as generating text and speech."
+                ),
+            }
+        ],
+    }
+
+
+def get_text_query():
+    question = "Explain the system architecture for a scalable audio generation pipeline. Answer in 15 words."
+    prompt = {
+        "role": "user",
+        "content": [
+            {
+                "type": "text",
+                "text": f"{question}",
+            }
+        ],
+    }
+    return prompt
+
+
+def get_mixed_modalities_query():
+    question = "What is recited in the audio? What is the content of this image? Why is this video funny?"
+    prompt = {
+        "role": "user",
+        "content": [
+            {
+                "type": "audio_url",
+                "audio_url": {"url": AudioAsset("mary_had_lamb").url},
+            },
+            {
+                "type": "image_url",
+                "image_url": {
+                    "url": "https://vllm-public-assets.s3.us-west-2.amazonaws.com/vision_model_images/cherry_blossom.jpg"
+                },
+            },
+            {
+                "type": "video_url",
+                "video_url": {
+                    "url": "https://huggingface.co/datasets/raushan-testing-hf/videos-test/resolve/main/sample_demo_1.mp4"
+                },
+            },
+            {
+                "type": "text",
+                "text": f"{question}",
+            },
+        ],
+    }
+
+    return prompt
+
+
+def get_use_audio_in_video_query():
+    question = "Describe the content of the video, then convert what the baby say into text."
+
+    prompt = {
+        "role": "user",
+        "content": [
+            {
+                "type": "video_url",
+                "video_url": {
+                    "url": "https://huggingface.co/datasets/raushan-testing-hf/videos-test/resolve/main/sample_demo_1.mp4",
+                    "num_frames": 16,
+                },
+            },
+            {
+                "type": "text",
+                "text": f"{question}",
+            },
+        ],
+    }
+
+    return prompt
+
+
+def get_multi_audios_query():
+    question = "Are these two audio clips the same?"
+    prompt = {
+        "role": "user",
+        "content": [
+            {
+                "type": "audio_url",
+                "audio_url": {"url": AudioAsset("mary_had_lamb").url},
+            },
+            {
+                "type": "audio_url",
+                "audio_url": {"url": AudioAsset("winning_call").url},
+            },
+            {
+                "type": "text",
+                "text": f"{question}",
+            },
+        ],
+    }
+    return prompt
+
+
+query_map = {
+    "mixed_modalities": get_mixed_modalities_query,
+    "use_audio_in_video": get_use_audio_in_video_query,
+    "multi_audios": get_multi_audios_query,
+    "text": get_text_query,
+}
+
+
+def run_multimodal_generation(args) -> None:
+    model_name = "Qwen/Qwen2.5-Omni-7B"
+    thinker_sampling_params = {
+        "temperature": 0.0,  # Deterministic - no randomness
+        "top_p": 1.0,  # Disable nucleus sampling
+        "top_k": -1,  # Disable top-k sampling
+        "max_tokens": 2048,
+        "seed": SEED,  # Fixed seed for sampling
+        "detokenize": True,
+        "repetition_penalty": 1.1,
+    }
+    talker_sampling_params = {
+        "temperature": 0.9,
+        "top_p": 0.8,
+        "top_k": 40,
+        "max_tokens": 2048,
+        "seed": SEED,  # Fixed seed for sampling
+        "detokenize": True,
+        "repetition_penalty": 1.05,
+        "stop_token_ids": [8294],
+    }
+    code2wav_sampling_params = {
+        "temperature": 0.0,  # Deterministic - no randomness
+        "top_p": 1.0,  # Disable nucleus sampling
+        "top_k": -1,  # Disable top-k sampling
+        "max_tokens": 2048,
+        "seed": SEED,  # Fixed seed for sampling
+        "detokenize": True,
+        "repetition_penalty": 1.1,
+    }
+
+    sampling_params_list = [
+        thinker_sampling_params,
+        talker_sampling_params,
+        code2wav_sampling_params,
+    ]
+
+    prompt = query_map[args.query_type]()
+    extra_body = {
+        "sampling_params_list": sampling_params_list  # Optional, it has a default setting in stage_configs of the corresponding model.
+    }
+
+    if args.query_type == "use_audio_in_video":
+        extra_body["mm_processor_kwargs"] = {"use_audio_in_video": True}
+
+    chat_completion = client.chat.completions.create(
+        messages=[
+            get_system_prompt(),
+            prompt,
+        ],
+        model=model_name,
+        extra_body=extra_body,
+    )
+
+    count = 0
+    for choice in chat_completion.choices:
+        if choice.message.audio:
+            audio_data = base64.b64decode(choice.message.audio.data)
+            audio_file_path = f"audio_{count}.wav"
+            with open(audio_file_path, "wb") as f:
+                f.write(audio_data)
+            print(f"Audio saved to {audio_file_path}")
+            count += 1
+        elif choice.message.content:
+            print("Chat completion output from text:", choice.message.content)
+
+
+def parse_args():
+    parser = FlexibleArgumentParser(description="Demo on using vLLM for offline inference with audio language models")
+    parser.add_argument(
+        "--query-type",
+        "-q",
+        type=str,
+        default="mixed_modalities",
+        choices=query_map.keys(),
+        help="Query type.",
+    )
+
+    return parser.parse_args()
+
+
+if __name__ == "__main__":
+    args = parse_args()
+    run_multimodal_generation(args)

examples/online_serving/qwen2_5_omni/README.md

@@ -1,101 +0,0 @@
-# Online serving Example of vLLM-Omni for Qwen2.5-Omni
-
-## 🛠️ Installation
-
-Please refer to [README.md](../../../README.md)
-
-## Run examples (Qwen2.5-Omni)
-
-### Launch the Server
-
-```bash
-vllm serve Qwen/Qwen2.5-Omni-7B --omni --port 8091
-```
-
-If you have custom stage configs file, launch the server with command below
-```bash
-vllm serve Qwen/Qwen2.5-Omni-7B --omni --port 8091 --stage-configs-path /path/to/stage_configs_file
-```
-
-### Send Multi-modal Request
-
-Get into the example folder
-```bash
-cd examples/online_serving/qwen2_5_omni
-```
-
-####  Send request via python
-
-```bash
-python openai_chat_completion_client_for_multimodal_generation.py --query-type mixed_modalities
-```
-
-The Python client supports the following command-line arguments:
-
-- `--query-type` (or `-q`): Query type (default: `mixed_modalities`)
-  - Options: `mixed_modalities`, `use_audio_in_video`, `multi_audios`, `text`
-- `--video-path` (or `-v`): Path to local video file or URL
-  - If not provided and query-type uses video, uses default video URL
-  - Supports local file paths (automatically encoded to base64) or HTTP/HTTPS URLs
-  - Example: `--video-path /path/to/video.mp4` or `--video-path https://example.com/video.mp4`
-- `--image-path` (or `-i`): Path to local image file or URL
-  - If not provided and query-type uses image, uses default image URL
-  - Supports local file paths (automatically encoded to base64) or HTTP/HTTPS URLs
-  - Supports common image formats: JPEG, PNG, GIF, WebP
-  - Example: `--image-path /path/to/image.jpg` or `--image-path https://example.com/image.png`
-- `--audio-path` (or `-a`): Path to local audio file or URL
-  - If not provided and query-type uses audio, uses default audio URL
-  - Supports local file paths (automatically encoded to base64) or HTTP/HTTPS URLs
-  - Supports common audio formats: MP3, WAV, OGG, FLAC, M4A
-  - Example: `--audio-path /path/to/audio.wav` or `--audio-path https://example.com/audio.mp3`
-- `--prompt` (or `-p`): Custom text prompt/question
-  - If not provided, uses default prompt for the selected query type
-  - Example: `--prompt "What are the main activities shown in this video?"`
-
-
-For example, to use mixed modalities with all local files:
-
-```bash
-python openai_chat_completion_client_for_multimodal_generation.py \
-    --query-type mixed_modalities \
-    --video-path /path/to/your/video.mp4 \
-    --image-path /path/to/your/image.jpg \
-    --audio-path /path/to/your/audio.wav \
-    --prompt "Analyze all the media content and provide a comprehensive summary."
-```
-
-####  Send request via curl
-
-```bash
-bash run_curl_multimodal_generation.sh mixed_modalities
-```
-
-## Run Local Web UI Demo
-
-This Web UI demo allows users to interact with the model through a web browser.
-
-### Running Gradio Demo
-
-Once vllm and vllm-omni are installed, you can launch the web service built on AsyncOmni by
-
-```bash
-python gradio_demo.py  --model Qwen/Qwen2.5-Omni-7B --port 7861
-```
-
-Then open `http://localhost:7861/` on your local browser to interact with the web UI.
-
-The gradio script supports the following arguments:
-
-- `--model`: Model name
-- `--ip`: Host/IP for Gradio server (default: 127.0.0.1)
-- `--port`: Port for Gradio server (default: 7861)
-- `--stage-configs-path`: Path to custom stage configs YAML file (optional)
-- `--share`: Share the Gradio demo publicly (creates a public link)
-
-### FAQ
-
-If you encounter error about backend of librosa, try to install ffmpeg with command below.
-```
-sudo apt update
-sudo apt install ffmpeg
-```

examples/online_serving/qwen2_5_omni/openai_chat_completion_client_for_multimodal_generation.py

@@ -1,411 +0,0 @@
-import base64
-import os
-from typing import Optional
-
-import requests
-from openai import OpenAI
-from vllm.assets.audio import AudioAsset
-from vllm.utils import FlexibleArgumentParser
-
-# Modify OpenAI's API key and API base to use vLLM's API server.
-openai_api_key = "EMPTY"
-openai_api_base = "http://localhost:8091/v1"
-
-client = OpenAI(
-    # defaults to os.environ.get("OPENAI_API_KEY")
-    api_key=openai_api_key,
-    base_url=openai_api_base,
-)
-
-SEED = 42
-
-
-def encode_base64_content_from_url(content_url: str) -> str:
-    """Encode a content retrieved from a remote url to base64 format."""
-
-    with requests.get(content_url) as response:
-        response.raise_for_status()
-        result = base64.b64encode(response.content).decode("utf-8")
-
-    return result
-
-
-def encode_base64_content_from_file(file_path: str) -> str:
-    """Encode a local file to base64 format."""
-    with open(file_path, "rb") as f:
-        content = f.read()
-        result = base64.b64encode(content).decode("utf-8")
-    return result
-
-
-def get_video_url_from_path(video_path: Optional[str]) -> str:
-    """Convert a video path (local file or URL) to a video URL format for the API.
-
-    If video_path is None or empty, returns the default URL.
-    If video_path is a local file path, encodes it to base64 data URL.
-    If video_path is a URL, returns it as-is.
-    """
-    if not video_path:
-        # Default video URL
-        return "https://huggingface.co/datasets/raushan-testing-hf/videos-test/resolve/main/sample_demo_1.mp4"
-
-    # Check if it's a URL (starts with http:// or https://)
-    if video_path.startswith(("http://", "https://")):
-        return video_path
-
-    # Otherwise, treat it as a local file path
-    if not os.path.exists(video_path):
-        raise FileNotFoundError(f"Video file not found: {video_path}")
-
-    # Detect video MIME type from file extension
-    video_path_lower = video_path.lower()
-    if video_path_lower.endswith(".mp4"):
-        mime_type = "video/mp4"
-    elif video_path_lower.endswith(".webm"):
-        mime_type = "video/webm"
-    elif video_path_lower.endswith(".mov"):
-        mime_type = "video/quicktime"
-    elif video_path_lower.endswith(".avi"):
-        mime_type = "video/x-msvideo"
-    elif video_path_lower.endswith(".mkv"):
-        mime_type = "video/x-matroska"
-    else:
-        # Default to mp4 if extension is unknown
-        mime_type = "video/mp4"
-
-    video_base64 = encode_base64_content_from_file(video_path)
-    return f"data:{mime_type};base64,{video_base64}"
-
-
-def get_image_url_from_path(image_path: Optional[str]) -> str:
-    """Convert an image path (local file or URL) to an image URL format for the API.
-
-    If image_path is None or empty, returns the default URL.
-    If image_path is a local file path, encodes it to base64 data URL.
-    If image_path is a URL, returns it as-is.
-    """
-    if not image_path:
-        # Default image URL
-        return "https://vllm-public-assets.s3.us-west-2.amazonaws.com/vision_model_images/cherry_blossom.jpg"
-
-    # Check if it's a URL (starts with http:// or https://)
-    if image_path.startswith(("http://", "https://")):
-        return image_path
-
-    # Otherwise, treat it as a local file path
-    if not os.path.exists(image_path):
-        raise FileNotFoundError(f"Image file not found: {image_path}")
-
-    # Detect image MIME type from file extension
-    image_path_lower = image_path.lower()
-    if image_path_lower.endswith((".jpg", ".jpeg")):
-        mime_type = "image/jpeg"
-    elif image_path_lower.endswith(".png"):
-        mime_type = "image/png"
-    elif image_path_lower.endswith(".gif"):
-        mime_type = "image/gif"
-    elif image_path_lower.endswith(".webp"):
-        mime_type = "image/webp"
-    else:
-        # Default to jpeg if extension is unknown
-        mime_type = "image/jpeg"
-
-    image_base64 = encode_base64_content_from_file(image_path)
-    return f"data:{mime_type};base64,{image_base64}"
-
-
-def get_audio_url_from_path(audio_path: Optional[str]) -> str:
-    """Convert an audio path (local file or URL) to an audio URL format for the API.
-
-    If audio_path is None or empty, returns the default URL.
-    If audio_path is a local file path, encodes it to base64 data URL.
-    If audio_path is a URL, returns it as-is.
-    """
-    if not audio_path:
-        # Default audio URL
-        return AudioAsset("mary_had_lamb").url
-
-    # Check if it's a URL (starts with http:// or https://)
-    if audio_path.startswith(("http://", "https://")):
-        return audio_path
-
-    # Otherwise, treat it as a local file path
-    if not os.path.exists(audio_path):
-        raise FileNotFoundError(f"Audio file not found: {audio_path}")
-
-    # Detect audio MIME type from file extension
-    audio_path_lower = audio_path.lower()
-    if audio_path_lower.endswith((".mp3", ".mpeg")):
-        mime_type = "audio/mpeg"
-    elif audio_path_lower.endswith(".wav"):
-        mime_type = "audio/wav"
-    elif audio_path_lower.endswith(".ogg"):
-        mime_type = "audio/ogg"
-    elif audio_path_lower.endswith(".flac"):
-        mime_type = "audio/flac"
-    elif audio_path_lower.endswith(".m4a"):
-        mime_type = "audio/mp4"
-    else:
-        # Default to wav if extension is unknown
-        mime_type = "audio/wav"
-
-    audio_base64 = encode_base64_content_from_file(audio_path)
-    return f"data:{mime_type};base64,{audio_base64}"
-
-
-def get_system_prompt():
-    return {
-        "role": "system",
-        "content": [
-            {
-                "type": "text",
-                "text": (
-                    "You are Qwen, a virtual human developed by the Qwen Team, "
-                    "Alibaba Group, capable of perceiving auditory and visual inputs, "
-                    "as well as generating text and speech."
-                ),
-            }
-        ],
-    }
-
-
-def get_text_query(custom_prompt: Optional[str] = None):
-    question = (
-        custom_prompt or "Explain the system architecture for a scalable audio generation pipeline. Answer in 15 words."
-    )
-    prompt = {
-        "role": "user",
-        "content": [
-            {
-                "type": "text",
-                "text": f"{question}",
-            }
-        ],
-    }
-    return prompt
-
-
-def get_mixed_modalities_query(
-    video_path: Optional[str] = None,
-    image_path: Optional[str] = None,
-    audio_path: Optional[str] = None,
-    custom_prompt: Optional[str] = None,
-):
-    question = (
-        custom_prompt or "What is recited in the audio? What is the content of this image? Why is this video funny?"
-    )
-    video_url = get_video_url_from_path(video_path)
-    image_url = get_image_url_from_path(image_path)
-    audio_url = get_audio_url_from_path(audio_path)
-    prompt = {
-        "role": "user",
-        "content": [
-            {
-                "type": "audio_url",
-                "audio_url": {"url": audio_url},
-            },
-            {
-                "type": "image_url",
-                "image_url": {"url": image_url},
-            },
-            {
-                "type": "video_url",
-                "video_url": {"url": video_url},
-            },
-            {
-                "type": "text",
-                "text": f"{question}",
-            },
-        ],
-    }
-
-    return prompt
-
-
-def get_use_audio_in_video_query(video_path: Optional[str] = None, custom_prompt: Optional[str] = None):
-    question = custom_prompt or "Describe the content of the video, then convert what the baby say into text."
-    video_url = get_video_url_from_path(video_path)
-
-    prompt = {
-        "role": "user",
-        "content": [
-            {
-                "type": "video_url",
-                "video_url": {
-                    "url": video_url,
-                    "num_frames": 16,
-                },
-            },
-            {
-                "type": "text",
-                "text": f"{question}",
-            },
-        ],
-    }
-
-    return prompt
-
-
-def get_multi_audios_query(audio_path: Optional[str] = None, custom_prompt: Optional[str] = None):
-    question = custom_prompt or "Are these two audio clips the same?"
-    audio_url = get_audio_url_from_path(audio_path)
-    prompt = {
-        "role": "user",
-        "content": [
-            {
-                "type": "audio_url",
-                "audio_url": {"url": audio_url},
-            },
-            {
-                "type": "audio_url",
-                "audio_url": {"url": AudioAsset("winning_call").url},
-            },
-            {
-                "type": "text",
-                "text": f"{question}",
-            },
-        ],
-    }
-    return prompt
-
-
-query_map = {
-    "mixed_modalities": get_mixed_modalities_query,
-    "use_audio_in_video": get_use_audio_in_video_query,
-    "multi_audios": get_multi_audios_query,
-    "text": get_text_query,
-}
-
-
-def run_multimodal_generation(args) -> None:
-    model_name = "Qwen/Qwen2.5-Omni-7B"
-    thinker_sampling_params = {
-        "temperature": 0.0,  # Deterministic - no randomness
-        "top_p": 1.0,  # Disable nucleus sampling
-        "top_k": -1,  # Disable top-k sampling
-        "max_tokens": 2048,
-        "seed": SEED,  # Fixed seed for sampling
-        "detokenize": True,
-        "repetition_penalty": 1.1,
-    }
-    talker_sampling_params = {
-        "temperature": 0.9,
-        "top_p": 0.8,
-        "top_k": 40,
-        "max_tokens": 2048,
-        "seed": SEED,  # Fixed seed for sampling
-        "detokenize": True,
-        "repetition_penalty": 1.05,
-        "stop_token_ids": [8294],
-    }
-    code2wav_sampling_params = {
-        "temperature": 0.0,  # Deterministic - no randomness
-        "top_p": 1.0,  # Disable nucleus sampling
-        "top_k": -1,  # Disable top-k sampling
-        "max_tokens": 2048,
-        "seed": SEED,  # Fixed seed for sampling
-        "detokenize": True,
-        "repetition_penalty": 1.1,
-    }
-
-    sampling_params_list = [
-        thinker_sampling_params,
-        talker_sampling_params,
-        code2wav_sampling_params,
-    ]
-
-    # Get paths and custom prompt from args
-    video_path = getattr(args, "video_path", None)
-    image_path = getattr(args, "image_path", None)
-    audio_path = getattr(args, "audio_path", None)
-    custom_prompt = getattr(args, "prompt", None)
-
-    # Get the query function and call it with appropriate parameters
-    query_func = query_map[args.query_type]
-    if args.query_type == "mixed_modalities":
-        prompt = query_func(
-            video_path=video_path, image_path=image_path, audio_path=audio_path, custom_prompt=custom_prompt
-        )
-    elif args.query_type == "use_audio_in_video":
-        prompt = query_func(video_path=video_path, custom_prompt=custom_prompt)
-    elif args.query_type == "multi_audios":
-        prompt = query_func(audio_path=audio_path, custom_prompt=custom_prompt)
-    elif args.query_type == "text":
-        prompt = query_func(custom_prompt=custom_prompt)
-    else:
-        prompt = query_func()
-
-    extra_body = {
-        "sampling_params_list": sampling_params_list  # Optional, it has a default setting in stage_configs of the corresponding model.
-    }
-
-    if args.query_type == "use_audio_in_video":
-        extra_body["mm_processor_kwargs"] = {"use_audio_in_video": True}
-
-    chat_completion = client.chat.completions.create(
-        messages=[
-            get_system_prompt(),
-            prompt,
-        ],
-        model=model_name,
-        extra_body=extra_body,
-    )
-
-    count = 0
-    for choice in chat_completion.choices:
-        if choice.message.audio:
-            audio_data = base64.b64decode(choice.message.audio.data)
-            audio_file_path = f"audio_{count}.wav"
-            with open(audio_file_path, "wb") as f:
-                f.write(audio_data)
-            print(f"Audio saved to {audio_file_path}")
-            count += 1
-        elif choice.message.content:
-            print("Chat completion output from text:", choice.message.content)
-
-
-def parse_args():
-    parser = FlexibleArgumentParser(description="Demo on using vLLM for offline inference with audio language models")
-    parser.add_argument(
-        "--query-type",
-        "-q",
-        type=str,
-        default="mixed_modalities",
-        choices=query_map.keys(),
-        help="Query type.",
-    )
-    parser.add_argument(
-        "--video-path",
-        "-v",
-        type=str,
-        default=None,
-        help="Path to local video file or URL. If not provided and query-type uses video, uses default video URL.",
-    )
-    parser.add_argument(
-        "--image-path",
-        "-i",
-        type=str,
-        default=None,
-        help="Path to local image file or URL. If not provided and query-type uses image, uses default image URL.",
-    )
-    parser.add_argument(
-        "--audio-path",
-        "-a",
-        type=str,
-        default=None,
-        help="Path to local audio file or URL. If not provided and query-type uses audio, uses default audio URL.",
-    )
-    parser.add_argument(
-        "--prompt",
-        "-p",
-        type=str,
-        default=None,
-        help="Custom text prompt/question to use instead of the default prompt for the selected query type.",
-    )
-
-    return parser.parse_args()
-
-
-if __name__ == "__main__":
-    args = parse_args()
-    run_multimodal_generation(args)

examples/online_serving/qwen3_omni/README.md

@@ -1,102 +0,0 @@
-# Online serving Example of vLLM-Omni for Qwen3-omni
-
-## 🛠️ Installation
-
-Please refer to [README.md](../../../README.md)
-
-## Run examples (Qwen3-Omni)
-
-### Launch the Server
-
-```bash
-vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091
-```
-
-If you have custom stage configs file, launch the server with command below
-```bash
-vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091 --stage-configs-path /path/to/stage_configs_file
-```
-
-### Send Multi-modal Request
-
-Get into the example folder
-```bash
-cd examples/online_serving/qwen3_omni
-```
-
-####  Send request via python
-
-```bash
-python openai_chat_completion_client_for_multimodal_generation.py --query-type use_image
-```
-
-The Python client supports the following command-line arguments:
-
-- `--query-type` (or `-q`): Query type (default: `use_video`)
-  - Options: `text`, `use_audio`, `use_image`, `use_video`
-- `--model` (or `-m`): Model name/path (default: `Qwen/Qwen3-Omni-30B-A3B-Instruct`)
-- `--video-path` (or `-v`): Path to local video file or URL
-  - If not provided and query-type is `use_video`, uses default video URL
-  - Supports local file paths (automatically encoded to base64) or HTTP/HTTPS URLs
-  - Example: `--video-path /path/to/video.mp4` or `--video-path https://example.com/video.mp4`
-- `--image-path` (or `-i`): Path to local image file or URL
-  - If not provided and query-type is `use_image`, uses default image URL
-  - Supports local file paths (automatically encoded to base64) or HTTP/HTTPS URLs
-  - Supports common image formats: JPEG, PNG, GIF, WebP
-  - Example: `--image-path /path/to/image.jpg` or `--image-path https://example.com/image.png`
-- `--audio-path` (or `-a`): Path to local audio file or URL
-  - If not provided and query-type is `use_audio`, uses default audio URL
-  - Supports local file paths (automatically encoded to base64) or HTTP/HTTPS URLs
-  - Supports common audio formats: MP3, WAV, OGG, FLAC, M4A
-  - Example: `--audio-path /path/to/audio.wav` or `--audio-path https://example.com/audio.mp3`
-- `--prompt` (or `-p`): Custom text prompt/question
-  - If not provided, uses default prompt for the selected query type
-  - Example: `--prompt "What are the main activities shown in this video?"`
-
-
-For example, to use a local video file with custom prompt:
-
-```bash
-python openai_chat_completion_client_for_multimodal_generation.py \
-    --query-type use_video \
-    --video-path /path/to/your/video.mp4 \
-    --prompt "What are the main activities shown in this video?"
-```
-
-####  Send request via curl
-
-```bash
-bash run_curl_multimodal_generation.sh use_image
-```
-
-
-### FAQ
-
-If you encounter error about backend of librosa, try to install ffmpeg with command below.
-```
-sudo apt update
-sudo apt install ffmpeg
-```
-
-## Run Local Web UI Demo
-
-This Web UI demo allows users to interact with the model through a web browser.
-
-### Running Gradio Demo
-
-Once vllm and vllm-omni are installed, you can launch the web service built on AsyncOmni by
-
-```bash
-python gradio_demo.py  --model Qwen/Qwen3-Omni-30B-A3B-Instruct --port 7861
-```
-
-Then open `http://localhost:7861/` on your local browser to interact with the web UI.
-
-
-The gradio script supports the following arguments:
-
-- `--model`: Model name
-- `--ip`: Host/IP for Gradio server (default: 127.0.0.1)
-- `--port`: Port for Gradio server (default: 7861)
-- `--stage-configs-path`: Path to custom stage configs YAML file (optional)
-- `--share`: Share the Gradio demo publicly (creates a public link)

examples/online_serving/qwen3_omni/gradio_demo.py

@@ -1,541 +0,0 @@
-import argparse
-import os
-import random
-import signal
-import sys
-from pathlib import Path
-from types import SimpleNamespace
-from typing import Any, Optional
-
-import gradio as gr
-import numpy as np
-import soundfile as sf
-import torch
-from PIL import Image
-from vllm.assets.video import video_get_metadata, video_to_ndarrays
-from vllm.sampling_params import SamplingParams
-
-from vllm_omni.entrypoints.async_omni import AsyncOmni
-
-# Import utils from offline inference example
-sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../offline_inference/qwen3_omni"))
-try:
-    from utils import make_omni_prompt
-except ImportError:
-    # Fallback if utils doesn't exist - we'll build prompts directly
-    make_omni_prompt = None
-
-SEED = 42
-ASYNC_INIT_TIMEOUT = 600
-
-SUPPORTED_MODELS: dict[str, dict[str, Any]] = {
-    "Qwen/Qwen3-Omni-30B-A3B-Instruct": {
-        "sampling_params": {
-            "thinker": {
-                "temperature": 0.4,
-                "top_p": 0.9,
-                "top_k": 1,
-                "max_tokens": 16384,
-                "detokenize": True,
-                "repetition_penalty": 1.05,
-                "stop_token_ids": [151645],
-                "seed": SEED,
-            },
-            "talker": {
-                "temperature": 0.9,
-                "top_k": 50,
-                "max_tokens": 4096,
-                "seed": SEED,
-                "detokenize": False,
-                "repetition_penalty": 1.05,
-                "stop_token_ids": [2150],
-            },
-            "code2wav": {
-                "temperature": 0.0,
-                "top_p": 1.0,
-                "top_k": -1,
-                "max_tokens": 4096 * 16,
-                "seed": SEED,
-                "detokenize": True,
-                "repetition_penalty": 1.1,
-            },
-        },
-    },
-}
-# Ensure deterministic behavior across runs.
-random.seed(SEED)
-np.random.seed(SEED)
-torch.manual_seed(SEED)
-torch.cuda.manual_seed(SEED)
-torch.cuda.manual_seed_all(SEED)
-torch.backends.cudnn.deterministic = True
-torch.backends.cudnn.benchmark = False
-os.environ["PYTHONHASHSEED"] = str(SEED)
-os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8"
-
-
-def parse_args():
-    parser = argparse.ArgumentParser(description="Gradio demo for Qwen3-Omni online inference.")
-    parser.add_argument(
-        "--model",
-        default="Qwen/Qwen3-Omni-30B-A3B-Instruct",
-        help="Path to local model checkpoint.",
-    )
-    parser.add_argument(
-        "--ip",
-        default="127.0.0.1",
-        help="Host/IP for gradio `launch`.",
-    )
-    parser.add_argument("--port", type=int, default=7861, help="Port for gradio `launch`.")
-    parser.add_argument("--share", action="store_true", help="Share the Gradio demo publicly.")
-    parser.add_argument(
-        "--stage-configs-path",
-        type=str,
-        default=None,
-        help="Path to custom stage configs YAML file (optional).",
-    )
-    return parser.parse_args()
-
-
-def build_sampling_params(seed: int, model_key: str) -> list[SamplingParams]:
-    """Build SamplingParams objects by reusing the dict definitions."""
-    return [SamplingParams(**params_dict) for params_dict in build_sampling_params_dict(seed, model_key)]
-
-
-def build_sampling_params_dict(seed: int, model_key: str) -> list[dict]:
-    """Build sampling params as dict for HTTP API mode."""
-    model_conf = SUPPORTED_MODELS.get(model_key)
-    if model_conf is None:
-        raise ValueError(f"Unsupported model '{model_key}'")
-
-    sampling_templates: dict[str, dict[str, Any]] = model_conf["sampling_params"]
-    sampling_params: list[dict] = []
-    for stage_name, template in sampling_templates.items():
-        params = dict(template)
-        params["seed"] = seed
-        sampling_params.append(params)
-    return sampling_params
-
-
-def create_prompt_args(base_args: argparse.Namespace) -> SimpleNamespace:
-    # The prompt builder expects a minimal namespace with these attributes.
-    return SimpleNamespace(
-        model=base_args.model,
-        prompt_type="text",
-        tokenize=True,
-        use_torchvision=True,
-        legacy_omni_video=False,
-    )
-
-
-def process_audio_file(
-    audio_file: Optional[Any],
-) -> Optional[tuple[np.ndarray, int]]:
-    """Normalize Gradio audio input to (np.ndarray, sample_rate)."""
-    if audio_file is None:
-        return None
-
-    sample_rate: Optional[int] = None
-    audio_np: Optional[np.ndarray] = None
-
-    def _load_from_path(path_str: str) -> Optional[tuple[np.ndarray, int]]:
-        if not path_str:
-            return None
-        path = Path(path_str)
-        if not path.exists():
-            return None
-        data, sr = sf.read(path)
-        if data.ndim > 1:
-            data = data[:, 0]
-        return data.astype(np.float32), int(sr)
-
-    if isinstance(audio_file, tuple):
-        if len(audio_file) == 2:
-            first, second = audio_file
-            # Case 1: (sample_rate, np.ndarray)
-            if isinstance(first, (int, float)) and isinstance(second, np.ndarray):
-                sample_rate = int(first)
-                audio_np = second
-            # Case 2: (filepath, (sample_rate, np.ndarray or list))
-            elif isinstance(first, str):
-                if isinstance(second, tuple) and len(second) == 2:
-                    sr_candidate, data_candidate = second
-                    if isinstance(sr_candidate, (int, float)) and isinstance(data_candidate, np.ndarray):
-                        sample_rate = int(sr_candidate)
-                        audio_np = data_candidate
-                if audio_np is None:
-                    loaded = _load_from_path(first)
-                    if loaded is not None:
-                        audio_np, sample_rate = loaded
-            # Case 3: (None, (sample_rate, np.ndarray))
-            elif first is None and isinstance(second, tuple) and len(second) == 2:
-                sr_candidate, data_candidate = second
-                if isinstance(sr_candidate, (int, float)) and isinstance(data_candidate, np.ndarray):
-                    sample_rate = int(sr_candidate)
-                    audio_np = data_candidate
-        elif len(audio_file) == 1 and isinstance(audio_file[0], str):
-            loaded = _load_from_path(audio_file[0])
-            if loaded is not None:
-                audio_np, sample_rate = loaded
-    elif isinstance(audio_file, str):
-        loaded = _load_from_path(audio_file)
-        if loaded is not None:
-            audio_np, sample_rate = loaded
-
-    if audio_np is None or sample_rate is None:
-        return None
-
-    if audio_np.ndim > 1:
-        audio_np = audio_np[:, 0]
-
-    return audio_np.astype(np.float32), sample_rate
-
-
-def process_image_file(image_file: Optional[Image.Image]) -> Optional[Image.Image]:
-    """Process image file from Gradio input.
-
-    Returns:
-        PIL Image in RGB mode or None if no image provided.
-    """
-    if image_file is None:
-        return None
-    # Convert to RGB if needed
-    if image_file.mode != "RGB":
-        image_file = image_file.convert("RGB")
-    return image_file
-
-
-def process_video_file(
-    video_file: Optional[str],
-    enable_audio_in_video: bool = False,
-    max_frames: int = 32,
-) -> Optional[tuple[np.ndarray, dict[str, Any], Optional[tuple[np.ndarray, int]]]]:
-    """Process video file and optionally extract audio track."""
-    if video_file is None:
-        return None
-
-    video_path = Path(video_file)
-    if not video_path.exists():
-        print(f"Video file not found: {video_path}")
-        return None
-
-    try:
-        frames = video_to_ndarrays(str(video_path), num_frames=max_frames)
-        metadata = video_get_metadata(str(video_path), num_frames=frames.shape[0])
-    except Exception as exc:  # pylint: disable=broad-except
-        print(f"Failed to decode video {video_path}: {exc}")
-        return None
-
-    audio_tuple: Optional[tuple[np.ndarray, int]] = None
-    if enable_audio_in_video:
-        try:
-            import librosa  # type: ignore import
-
-            audio_signal, sampling_rate = librosa.load(str(video_path), sr=16000)
-            audio_tuple = (audio_signal.astype(np.float32), sampling_rate)
-        except Exception as exc:  # pylint: disable=broad-except
-            print(f"Failed to extract audio from video {video_path}: {exc}")
-
-    return frames, metadata, audio_tuple
-
-
-async def run_inference_async_omni(
-    omni: AsyncOmni,
-    sampling_params: list[SamplingParams],
-    prompt_args_template: SimpleNamespace,
-    user_prompt: str,
-    audio_file: Optional[tuple[str, tuple[int, np.ndarray]]] = None,
-    image_file: Optional[Image.Image] = None,
-    video_file: Optional[str] = None,
-    use_audio_in_video: bool = False,
-):
-    """Run inference using AsyncOmni directly with multimodal support."""
-    if not user_prompt.strip() and not audio_file and not image_file and not video_file:
-        return "Please provide at least a text prompt or multimodal input.", None
-
-    try:
-        # Build prompt with multimodal data
-        prompt_args = SimpleNamespace(**prompt_args_template.__dict__)
-
-        # Process multimodal inputs
-        multi_modal_data = {}
-        mm_processor_kwargs = {}
-
-        # Process audio
-        audio_data = process_audio_file(audio_file)
-        if audio_data is not None:
-            multi_modal_data["audio"] = [audio_data] if not isinstance(audio_data, list) else audio_data
-
-        # Process image
-        image_data = process_image_file(image_file)
-        if image_data is not None:
-            multi_modal_data["image"] = image_data
-
-        # Process video
-        video_payload = process_video_file(video_file, enable_audio_in_video=use_audio_in_video)
-        if video_payload is not None:
-            video_frames, video_metadata, extracted_audio = video_payload
-            video_entry: Any
-            if video_metadata:
-                video_entry = (video_frames, video_metadata)
-            else:
-                video_entry = video_frames
-            multi_modal_data["video"] = video_entry
-            if use_audio_in_video and extracted_audio is not None and "audio" not in multi_modal_data:
-                multi_modal_data["audio"] = (
-                    [extracted_audio] if not isinstance(extracted_audio, list) else extracted_audio
-                )
-                mm_processor_kwargs["use_audio_in_video"] = True
-
-        # Build the prompt input
-        if make_omni_prompt is not None:
-            omni_prompt = make_omni_prompt(prompt_args, user_prompt)
-            # Add multimodal data if present
-            if multi_modal_data:
-                if isinstance(omni_prompt, dict):
-                    omni_prompt["multi_modal_data"] = multi_modal_data
-                    if mm_processor_kwargs:
-                        omni_prompt["mm_processor_kwargs"] = mm_processor_kwargs
-                else:
-                    # If make_omni_prompt returns a string, we need to wrap it
-                    omni_prompt = {
-                        "prompt": omni_prompt,
-                        "multi_modal_data": multi_modal_data,
-                    }
-                    if mm_processor_kwargs:
-                        omni_prompt["mm_processor_kwargs"] = mm_processor_kwargs
-        else:
-            # Fallback: build prompt directly
-            default_system = (
-                "You are Qwen, a virtual human developed by the Qwen Team, "
-                "Alibaba Group, capable of perceiving auditory and visual inputs, "
-                "as well as generating text and speech."
-            )
-            prompt = f"<|im_start|>system\n{default_system}<|im_end|>\n<|im_start|>user\n"
-            if audio_data:
-                prompt += "<|audio_start|><|audio_pad|><|audio_end|>"
-            if image_data:
-                prompt += "<|vision_start|><|image_pad|><|vision_end|>"
-            if video_payload is not None:
-                prompt += "<|vision_start|><|video_pad|><|vision_end|>"
-            if user_prompt.strip():
-                prompt += f"{user_prompt}"
-            prompt += "<|im_end|>\n<|im_start|>assistant\n"
-
-            omni_prompt = {
-                "prompt": prompt,
-                "multi_modal_data": multi_modal_data,
-            }
-            if mm_processor_kwargs:
-                omni_prompt["mm_processor_kwargs"] = mm_processor_kwargs
-
-        request_id = "0"
-        text_outputs: list[str] = []
-        audio_output = None
-
-        async for stage_outputs in omni.generate(
-            prompt=omni_prompt,
-            request_id=request_id,
-            sampling_params_list=sampling_params,
-        ):
-            # stage_outputs.request_output is a RequestOutput object, not a list
-            request_output = stage_outputs.request_output
-            if stage_outputs.final_output_type == "text":
-                if request_output.outputs:
-                    for output in request_output.outputs:
-                        if output.text:
-                            text_outputs.append(output.text)
-            elif stage_outputs.final_output_type == "audio":
-                # multimodal_output is on the RequestOutput object
-                # See vllm_omni/entrypoints/openai/serving_chat.py:733 for reference
-                if hasattr(request_output, "multimodal_output") and request_output.multimodal_output:
-                    if "audio" in request_output.multimodal_output:
-                        audio_tensor = request_output.multimodal_output["audio"]
-                        # Ensure audio is 1D (flatten if needed)
-                        if hasattr(audio_tensor, "ndim") and audio_tensor.ndim > 1:
-                            audio_tensor = audio_tensor.flatten()
-                        # Convert BFloat16 to Float32 before numpy conversion (numpy doesn't support BFloat16)
-                        audio_np = audio_tensor.float().detach().cpu().numpy()
-                        audio_output = (
-                            24000,  # sampling rate in Hz
-                            audio_np,
-                        )
-
-        text_response = "\n\n".join(text_outputs) if text_outputs else "No text output."
-        return text_response, audio_output
-    except Exception as exc:  # pylint: disable=broad-except
-        return f"Inference failed: {exc}", None
-
-
-def build_interface(
-    omni: AsyncOmni,
-    sampling_params: list[SamplingParams],
-    prompt_args_template: SimpleNamespace,
-    model: str,
-):
-    """Build Gradio interface for AsyncOmni mode."""
-
-    async def run_inference(
-        user_prompt: str,
-        audio_file: Optional[tuple[str, tuple[int, np.ndarray]]],
-        image_file: Optional[Image.Image],
-        video_file: Optional[str],
-        use_audio_in_video: bool,
-    ):
-        return await run_inference_async_omni(
-            omni,
-            sampling_params,
-            prompt_args_template,
-            user_prompt,
-            audio_file,
-            image_file,
-            video_file,
-            use_audio_in_video,
-        )
-
-    css = """
-    .media-input-container {
-        display: flex;
-        gap: 10px;
-    }
-    .media-input-container > div {
-        flex: 1;
-    }
-    .media-input-container .image-input,
-    .media-input-container .audio-input {
-        height: 300px;
-    }
-    .media-input-container .video-column {
-        height: 300px;
-        display: flex;
-        flex-direction: column;
-    }
-    .media-input-container .video-input {
-        flex: 1;
-        min-height: 0;
-    }
-    #generate-btn button {
-        width: 100%;
-    }
-    """
-
-    with gr.Blocks(css=css) as demo:
-        gr.Markdown("# vLLM-Omni Online Serving Demo")
-        gr.Markdown(f"**Model:** {model} \n\n")
-
-        with gr.Column():
-            with gr.Row():
-                input_box = gr.Textbox(
-                    label="Text Prompt",
-                    placeholder="For example: Describe what happens in the media inputs.",
-                    lines=4,
-                    scale=1,
-                )
-            with gr.Row(elem_classes="media-input-container"):
-                image_input = gr.Image(
-                    label="Image Input (optional)",
-                    type="pil",
-                    sources=["upload"],
-                    scale=1,
-                    elem_classes="image-input",
-                )
-                with gr.Column(scale=1, elem_classes="video-column"):
-                    video_input = gr.Video(
-                        label="Video Input (optional)",
-                        sources=["upload"],
-                        elem_classes="video-input",
-                    )
-                    use_audio_in_video_checkbox = gr.Checkbox(
-                        label="Use audio from video",
-                        value=False,
-                        info="Extract the video's audio track when provided.",
-                    )
-                audio_input = gr.Audio(
-                    label="Audio Input (optional)",
-                    type="numpy",
-                    sources=["upload", "microphone"],
-                    scale=1,
-                    elem_classes="audio-input",
-                )
-
-        with gr.Row():
-            generate_btn = gr.Button(
-                "Generate",
-                variant="primary",
-                size="lg",
-                elem_id="generate-btn",
-            )
-
-        with gr.Row():
-            text_output = gr.Textbox(label="Text Output", lines=10, scale=2)
-            audio_output = gr.Audio(label="Audio Output", interactive=False, scale=1)
-
-        generate_btn.click(
-            fn=run_inference,
-            inputs=[input_box, audio_input, image_input, video_input, use_audio_in_video_checkbox],
-            outputs=[text_output, audio_output],
-        )
-        demo.queue()
-    return demo
-
-
-def main():
-    args = parse_args()
-    omni = None
-
-    model_name = "/".join(args.model.split("/")[-2:])
-    assert model_name in SUPPORTED_MODELS, (
-        f"Unsupported model '{model_name}'. Supported models: {SUPPORTED_MODELS.keys()}"
-    )
-
-    # Register signal handlers for graceful shutdown
-    def signal_handler(sig, frame):
-        print("\nReceived interrupt signal, shutting down...")
-        if omni is not None:
-            try:
-                omni.shutdown()
-            except Exception as e:
-                print(f"Error during shutdown: {e}")
-        sys.exit(0)
-
-    signal.signal(signal.SIGINT, signal_handler)
-    signal.signal(signal.SIGTERM, signal_handler)
-
-    print(f"Initializing AsyncOmni with model: {args.model}")
-    if args.stage_configs_path:
-        print(f"Using custom stage configs: {args.stage_configs_path}")
-
-    sampling_params = build_sampling_params(SEED, model_name)
-    omni = AsyncOmni(
-        model=args.model,
-        stage_configs_path=args.stage_configs_path,
-        init_timeout=ASYNC_INIT_TIMEOUT,
-    )
-    print("✓ AsyncOmni initialized successfully")
-    prompt_args_template = create_prompt_args(args)
-
-    demo = build_interface(
-        omni,
-        sampling_params,
-        prompt_args_template,
-        args.model,
-    )
-    try:
-        demo.launch(
-            server_name=args.ip,
-            server_port=args.port,
-            share=args.share,
-        )
-    except KeyboardInterrupt:
-        print("\nShutting down...")
-    finally:
-        # Cleanup
-        if omni is not None:
-            try:
-                omni.shutdown()
-            except Exception as e:
-                print(f"Error during cleanup: {e}")
-
-
-if __name__ == "__main__":
-    main()

examples/online_serving/qwen3_omni/openai_chat_completion_client_for_multimodal_generation.py

@@ -1,405 +0,0 @@
-import base64
-import os
-from typing import NamedTuple, Optional
-
-import requests
-from openai import OpenAI
-from vllm.assets.audio import AudioAsset
-from vllm.utils import FlexibleArgumentParser
-
-# Modify OpenAI's API key and API base to use vLLM's API server.
-openai_api_key = "EMPTY"
-openai_api_base = "http://localhost:8091/v1"
-
-client = OpenAI(
-    # defaults to os.environ.get("OPENAI_API_KEY")
-    api_key=openai_api_key,
-    base_url=openai_api_base,
-)
-
-SEED = 42
-
-
-class QueryResult(NamedTuple):
-    inputs: dict
-    limit_mm_per_prompt: dict[str, int]
-
-
-def encode_base64_content_from_url(content_url: str) -> str:
-    """Encode a content retrieved from a remote url to base64 format."""
-
-    with requests.get(content_url) as response:
-        response.raise_for_status()
-        result = base64.b64encode(response.content).decode("utf-8")
-
-    return result
-
-
-def encode_base64_content_from_file(file_path: str) -> str:
-    """Encode a local file to base64 format."""
-    with open(file_path, "rb") as f:
-        content = f.read()
-        result = base64.b64encode(content).decode("utf-8")
-    return result
-
-
-def get_video_url_from_path(video_path: Optional[str]) -> str:
-    """Convert a video path (local file or URL) to a video URL format for the API.
-
-    If video_path is None or empty, returns the default URL.
-    If video_path is a local file path, encodes it to base64 data URL.
-    If video_path is a URL, returns it as-is.
-    """
-    if not video_path:
-        # Default video URL
-        return "https://huggingface.co/datasets/raushan-testing-hf/videos-test/resolve/main/sample_demo_1.mp4"
-
-    # Check if it's a URL (starts with http:// or https://)
-    if video_path.startswith(("http://", "https://")):
-        return video_path
-
-    # Otherwise, treat it as a local file path
-    if not os.path.exists(video_path):
-        raise FileNotFoundError(f"Video file not found: {video_path}")
-
-    # Detect video MIME type from file extension
-    video_path_lower = video_path.lower()
-    if video_path_lower.endswith(".mp4"):
-        mime_type = "video/mp4"
-    elif video_path_lower.endswith(".webm"):
-        mime_type = "video/webm"
-    elif video_path_lower.endswith(".mov"):
-        mime_type = "video/quicktime"
-    elif video_path_lower.endswith(".avi"):
-        mime_type = "video/x-msvideo"
-    elif video_path_lower.endswith(".mkv"):
-        mime_type = "video/x-matroska"
-    else:
-        # Default to mp4 if extension is unknown
-        mime_type = "video/mp4"
-
-    video_base64 = encode_base64_content_from_file(video_path)
-    return f"data:{mime_type};base64,{video_base64}"
-
-
-def get_image_url_from_path(image_path: Optional[str]) -> str:
-    """Convert an image path (local file or URL) to an image URL format for the API.
-
-    If image_path is None or empty, returns the default URL.
-    If image_path is a local file path, encodes it to base64 data URL.
-    If image_path is a URL, returns it as-is.
-    """
-    if not image_path:
-        # Default image URL
-        return "https://vllm-public-assets.s3.us-west-2.amazonaws.com/vision_model_images/cherry_blossom.jpg"
-
-    # Check if it's a URL (starts with http:// or https://)
-    if image_path.startswith(("http://", "https://")):
-        return image_path
-
-    # Otherwise, treat it as a local file path
-    if not os.path.exists(image_path):
-        raise FileNotFoundError(f"Image file not found: {image_path}")
-
-    # Detect image MIME type from file extension
-    image_path_lower = image_path.lower()
-    if image_path_lower.endswith((".jpg", ".jpeg")):
-        mime_type = "image/jpeg"
-    elif image_path_lower.endswith(".png"):
-        mime_type = "image/png"
-    elif image_path_lower.endswith(".gif"):
-        mime_type = "image/gif"
-    elif image_path_lower.endswith(".webp"):
-        mime_type = "image/webp"
-    else:
-        # Default to jpeg if extension is unknown
-        mime_type = "image/jpeg"
-
-    image_base64 = encode_base64_content_from_file(image_path)
-    return f"data:{mime_type};base64,{image_base64}"
-
-
-def get_audio_url_from_path(audio_path: Optional[str]) -> str:
-    """Convert an audio path (local file or URL) to an audio URL format for the API.
-
-    If audio_path is None or empty, returns the default URL.
-    If audio_path is a local file path, encodes it to base64 data URL.
-    If audio_path is a URL, returns it as-is.
-    """
-    if not audio_path:
-        # Default audio URL
-        return AudioAsset("mary_had_lamb").url
-
-    # Check if it's a URL (starts with http:// or https://)
-    if audio_path.startswith(("http://", "https://")):
-        return audio_path
-
-    # Otherwise, treat it as a local file path
-    if not os.path.exists(audio_path):
-        raise FileNotFoundError(f"Audio file not found: {audio_path}")
-
-    # Detect audio MIME type from file extension
-    audio_path_lower = audio_path.lower()
-    if audio_path_lower.endswith((".mp3", ".mpeg")):
-        mime_type = "audio/mpeg"
-    elif audio_path_lower.endswith(".wav"):
-        mime_type = "audio/wav"
-    elif audio_path_lower.endswith(".ogg"):
-        mime_type = "audio/ogg"
-    elif audio_path_lower.endswith(".flac"):
-        mime_type = "audio/flac"
-    elif audio_path_lower.endswith(".m4a"):
-        mime_type = "audio/mp4"
-    else:
-        # Default to wav if extension is unknown
-        mime_type = "audio/wav"
-
-    audio_base64 = encode_base64_content_from_file(audio_path)
-    return f"data:{mime_type};base64,{audio_base64}"
-
-
-def get_system_prompt():
-    return {
-        "role": "system",
-        "content": [
-            {
-                "type": "text",
-                "text": (
-                    "You are Qwen, a virtual human developed by the Qwen Team, "
-                    "Alibaba Group, capable of perceiving auditory and visual inputs, "
-                    "as well as generating text and speech."
-                ),
-            }
-        ],
-    }
-
-
-def get_text_query(custom_prompt: Optional[str] = None):
-    question = (
-        custom_prompt or "Explain the system architecture for a scalable audio generation pipeline. Answer in 15 words."
-    )
-    prompt = {
-        "role": "user",
-        "content": [
-            {
-                "type": "text",
-                "text": f"{question}",
-            }
-        ],
-    }
-    return prompt
-
-
-default_system = (
-    "You are Qwen, a virtual human developed by the Qwen Team, Alibaba "
-    "Group, capable of perceiving auditory and visual inputs, as well as "
-    "generating text and speech."
-)
-
-
-def get_video_query(video_path: Optional[str] = None, custom_prompt: Optional[str] = None):
-    question = custom_prompt or "Why is this video funny?"
-    video_url = get_video_url_from_path(video_path)
-    prompt = {
-        "role": "user",
-        "content": [
-            {
-                "type": "video_url",
-                "video_url": {"url": video_url},
-            },
-            {
-                "type": "text",
-                "text": f"{question}",
-            },
-        ],
-    }
-    return prompt
-
-
-def get_image_query(image_path: Optional[str] = None, custom_prompt: Optional[str] = None):
-    question = custom_prompt or "What is the content of this image?"
-    image_url = get_image_url_from_path(image_path)
-    prompt = {
-        "role": "user",
-        "content": [
-            {
-                "type": "image_url",
-                "image_url": {"url": image_url},
-            },
-            {
-                "type": "text",
-                "text": f"{question}",
-            },
-        ],
-    }
-    return prompt
-
-
-def get_audio_query(audio_path: Optional[str] = None, custom_prompt: Optional[str] = None):
-    question = custom_prompt or "What is the content of this audio?"
-    audio_url = get_audio_url_from_path(audio_path)
-    prompt = {
-        "role": "user",
-        "content": [
-            {
-                "type": "audio_url",
-                "audio_url": {"url": audio_url},
-            },
-            {
-                "type": "text",
-                "text": f"{question}",
-            },
-        ],
-    }
-    return prompt
-
-
-query_map = {
-    "text": get_text_query,
-    "use_audio": get_audio_query,
-    "use_image": get_image_query,
-    "use_video": get_video_query,
-}
-
-
-def run_multimodal_generation(args) -> None:
-    model_name = args.model
-    thinker_sampling_params = {
-        "temperature": 0.4,  # Deterministic
-        "top_p": 0.9,
-        "top_k": 1,
-        "max_tokens": 16384,
-        "repetition_penalty": 1.05,
-        "stop_token_ids": [151645],  # Qwen EOS token <|im_end|>
-        "seed": SEED,
-    }
-
-    # Sampling parameters for Talker stage (codec generation)
-    # Stop at codec EOS token
-    talker_sampling_params = {
-        "temperature": 0.9,
-        "top_k": 50,
-        "max_tokens": 4096,
-        "seed": SEED,
-        "detokenize": False,
-        "repetition_penalty": 1.05,
-        "stop_token_ids": [2150],  # TALKER_CODEC_EOS_TOKEN_ID
-    }
-
-    # # Sampling parameters for Code2Wav stage (audio generation)
-    code2wav_sampling_params = {
-        "temperature": 0.0,
-        "top_p": 1.0,
-        "top_k": -1,
-        "max_tokens": 4096 * 16,
-        "seed": SEED,
-        "detokenize": True,
-        "repetition_penalty": 1.1,
-    }
-
-    sampling_params_list = [
-        thinker_sampling_params,
-        talker_sampling_params,
-        code2wav_sampling_params,
-    ]
-
-    # Get paths and custom prompt from args
-    video_path = getattr(args, "video_path", None)
-    image_path = getattr(args, "image_path", None)
-    audio_path = getattr(args, "audio_path", None)
-    custom_prompt = getattr(args, "prompt", None)
-
-    # Get the query function and call it with appropriate parameters
-    query_func = query_map[args.query_type]
-    if args.query_type == "use_video":
-        prompt = query_func(video_path=video_path, custom_prompt=custom_prompt)
-    elif args.query_type == "use_image":
-        prompt = query_func(image_path=image_path, custom_prompt=custom_prompt)
-    elif args.query_type == "use_audio":
-        prompt = query_func(audio_path=audio_path, custom_prompt=custom_prompt)
-    elif args.query_type == "text":
-        prompt = query_func(custom_prompt=custom_prompt)
-    else:
-        prompt = query_func()
-
-    extra_body = {
-        "sampling_params_list": sampling_params_list  # Optional, it has a default setting in stage_configs of the corresponding model.
-    }
-
-    if args.query_type == "use_audio_in_video":
-        extra_body["mm_processor_kwargs"] = {"use_audio_in_video": True}
-
-    chat_completion = client.chat.completions.create(
-        messages=[
-            get_system_prompt(),
-            prompt,
-        ],
-        model=model_name,
-        extra_body=extra_body,
-    )
-
-    count = 0
-    for choice in chat_completion.choices:
-        if choice.message.audio:
-            audio_data = base64.b64decode(choice.message.audio.data)
-            audio_file_path = f"audio_{count}.wav"
-            with open(audio_file_path, "wb") as f:
-                f.write(audio_data)
-            print(f"Audio saved to {audio_file_path}")
-            count += 1
-        elif choice.message.content:
-            print("Chat completion output from text:", choice.message.content)
-
-
-def parse_args():
-    parser = FlexibleArgumentParser(description="Demo on using vLLM for offline inference with audio language models")
-    parser.add_argument(
-        "--query-type",
-        "-q",
-        type=str,
-        default="use_video",
-        choices=query_map.keys(),
-        help="Query type.",
-    )
-    parser.add_argument(
-        "--model",
-        "-m",
-        type=str,
-        default="Qwen/Qwen3-Omni-30B-A3B-Instruct",
-        help="Model Name / Path",
-    )
-    parser.add_argument(
-        "--video-path",
-        "-v",
-        type=str,
-        default=None,
-        help="Path to local video file or URL. If not provided and query-type is 'use_video', uses default video URL.",
-    )
-    parser.add_argument(
-        "--image-path",
-        "-i",
-        type=str,
-        default=None,
-        help="Path to local image file or URL. If not provided and query-type is 'use_image', uses default image URL.",
-    )
-    parser.add_argument(
-        "--audio-path",
-        "-a",
-        type=str,
-        default=None,
-        help="Path to local audio file or URL. If not provided and query-type is 'use_audio', uses default audio URL.",
-    )
-    parser.add_argument(
-        "--prompt",
-        "-p",
-        type=str,
-        default=None,
-        help="Custom text prompt/question to use instead of the default prompt for the selected query type.",
-    )
-
-    return parser.parse_args()
-
-
-if __name__ == "__main__":
-    args = parse_args()
-    run_multimodal_generation(args)

examples/online_serving/qwen3_omni/run_curl_multimodal_generation.sh

@@ -1,165 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-
-# Default query type
-QUERY_TYPE="${1:-use_video}"
-
-# Validate query type
-if [[ ! "$QUERY_TYPE" =~ ^(text|use_audio|use_image|use_video)$ ]]; then
-    echo "Error: Invalid query type '$QUERY_TYPE'"
-    echo "Usage: $0 [text|use_audio|use_image|use_video]"
-    echo "  text: Text query"
-    echo "  use_audio: Audio + Text query"
-    echo "  use_image: Image + Text query"
-    echo "  use_video: Video + Text query"
-    exit 1
-fi
-
-SEED=42
-
-thinker_sampling_params='{
-  "temperature": 0.4,
-  "top_p": 0.9,
-  "top_k": 1,
-  "max_tokens": 16384,
-  "seed": 42,
-  "repetition_penalty": 1.05,
-  "stop_token_ids": [151645]
-}'
-
-talker_sampling_params='{
-  "temperature": 0.9,
-  "top_k": 50,
-  "max_tokens": 4096,
-  "seed": 42,
-  "detokenize": false,
-  "repetition_penalty": 1.05,
-  "stop_token_ids": [2150]
-}'
-
-code2wav_sampling_params='{
-  "temperature": 0.0,
-  "top_p": 1.0,
-  "top_k": -1,
-  "max_tokens": 65536,
-  "seed": 42,
-  "detokenize": true,
-  "repetition_penalty": 1.1
-}'
-# Above is optional, it has a default setting in stage_configs of the corresponding model.
-
-# Define URLs for assets
-MARY_HAD_LAMB_AUDIO_URL="https://vllm-public-assets.s3.us-west-2.amazonaws.com/multimodal_asset/mary_had_lamb.ogg"
-CHERRY_BLOSSOM_IMAGE_URL="https://vllm-public-assets.s3.us-west-2.amazonaws.com/vision_model_images/cherry_blossom.jpg"
-SAMPLE_VIDEO_URL="https://huggingface.co/datasets/raushan-testing-hf/videos-test/resolve/main/sample_demo_1.mp4"
-
-# Build user content and extra fields based on query type
-case "$QUERY_TYPE" in
-  text)
-    user_content='[
-      {
-        "type": "text",
-        "text": "Explain the system architecture for a scalable audio generation pipeline. Answer in 15 words."
-      }
-    ]'
-    sampling_params_list='[
-      '"$thinker_sampling_params"',
-      '"$talker_sampling_params"',
-      '"$code2wav_sampling_params"'
-    ]'
-    mm_processor_kwargs="{}"
-    ;;
-  use_audio)
-    user_content='[
-        {
-          "type": "audio_url",
-          "audio_url": {
-            "url": "'"$MARY_HAD_LAMB_AUDIO_URL"'"
-          }
-        },
-        {
-          "type": "text",
-          "text": "What is the content of this audio?"
-        }
-      ]'
-    sampling_params_list='[
-      '"$thinker_sampling_params"',
-      '"$talker_sampling_params"',
-      '"$code2wav_sampling_params"'
-    ]'
-    mm_processor_kwargs="{}"
-    ;;
-  use_image)
-    user_content='[
-        {
-          "type": "image_url",
-          "image_url": {
-            "url": "'"$CHERRY_BLOSSOM_IMAGE_URL"'"
-          }
-        },
-        {
-          "type": "text",
-          "text": "What is the content of this image?"
-        }
-      ]'
-    sampling_params_list='[
-      '"$thinker_sampling_params"',
-      '"$talker_sampling_params"',
-      '"$code2wav_sampling_params"'
-    ]'
-    mm_processor_kwargs="{}"
-    ;;
-  use_video)
-    user_content='[
-        {
-          "type": "video_url",
-          "video_url": {
-            "url": "'"$SAMPLE_VIDEO_URL"'"
-          }
-        },
-        {
-          "type": "text",
-          "text": "Why is this video funny?"
-        }
-      ]'
-    sampling_params_list='[
-      '"$thinker_sampling_params"',
-      '"$talker_sampling_params"',
-      '"$code2wav_sampling_params"'
-    ]'
-    mm_processor_kwargs="{}"
-    ;;
-esac
-
-echo "Running query type: $QUERY_TYPE"
-echo ""
-
-
-output=$(curl -sS -X POST http://localhost:8091/v1/chat/completions \
-    -H "Content-Type: application/json" \
-    -d @- <<EOF
-{
-  "model": "Qwen/Qwen3-Omni-30B-A3B-Instruct",
-  "sampling_params_list": $sampling_params_list,
-  "mm_processor_kwargs": $mm_processor_kwargs,
-  "messages": [
-    {
-      "role": "system",
-      "content": [
-        {
-          "type": "text",
-          "text": "You are Qwen, a virtual human developed by the Qwen Team, Alibaba Group, capable of perceiving auditory and visual inputs, as well as generating text and speech."
-        }
-      ]
-    },
-    {
-      "role": "user",
-      "content": $user_content
-    }
-  ]
-}
-EOF
-  )
-
-# Here it only shows the text content of the first choice. Audio content has many binaries, so it's not displayed here.
-echo "Output of request: $(echo "$output" | jq '.choices[0].message.content')"

mkdocs.yml

@@ -1,6 +1,6 @@
-site_name: vLLM-Omni
-site_description: Efficient omni-modality model serving for everyone
-site_author: vLLM-Omni Team
+site_name: vLLM-omni
+site_description: Multi-modality models inference and serving with non-autoregressive structures
+site_author: vLLM-omni Team
 site_url: https://vllm-project.github.io/vllm-omni/
 
 repo_name: vllm-project/vllm-omni
@@ -8,36 +8,30 @@ repo_url: https://github.com/vllm-project/vllm-omni
 edit_uri: edit/main/docs/
 
 # Copyright
-copyright: Copyright © 2025 vLLM-Omni Team
+copyright: Copyright © 2025 vLLM-omni Team
 
 # Theme
 theme:
   name: material
-  logo: source/logos/vllm-logo-only-light.ico
-  favicon: source/logos/vllm-logo-only-light.ico
   palette:
-    # Palette toggle for automatic mode
-    - media: "(prefers-color-scheme)"
-      toggle:
-        icon: material/brightness-auto
-        name: Switch to light mode
-    # Palette toggle for light mode
-    - media: "(prefers-color-scheme: light)"
-      scheme: default
-      primary: white
+    # Light mode
+    - scheme: default
+      primary: blue
+      accent: blue
       toggle:
         icon: material/brightness-7
         name: Switch to dark mode
-    # Palette toggle for dark mode
-    - media: "(prefers-color-scheme: dark)"
-      scheme: slate
-      primary: black
+    # Dark mode
+    - scheme: slate
+      primary: blue
+      accent: blue
       toggle:
-        icon: material/brightness-2
-        name: Switch to system preference
+        icon: material/brightness-4
+        name: Switch to light mode
   features:
     - content.action.edit
     - content.code.copy
+    - content.tabs.link
     - navigation.instant
     - navigation.instant.progress
     - navigation.tracking
@@ -53,19 +47,17 @@ theme:
     - content.tabs
     - content.tooltips
     - toc.follow
-  custom_dir: docs/mkdocs/overrides
+  icon:
+    repo: fontawesome/brands/github
 
 hooks:
-  - docs/mkdocs/hooks/generate_api_readme.py
-  - docs/mkdocs/hooks/url_schemes.py
   - docs/mkdocs/hooks/generate_examples.py
 
 # Plugins
 plugins:
   - meta
   - search
-  - autorefs
-  - awesome-nav
+  # - awesome-nav  # Uncomment if installed
   - glightbox
   - git-revision-date-localized:
       # exclude files
@@ -79,8 +71,6 @@ plugins:
       minify_js: true
       minify_css: true
       cache_safe: true
-      js_files: [docs/mkdocs/javascript/*.js]
-      css_files: [docs/mkdocs/stylesheets/*.css]
   - api-autonav:
       modules: ["vllm_omni"]
       api_root_uri: "api"
@@ -88,31 +78,36 @@ plugins:
       show_full_namespace: false  # Show only module name, not full path
       exclude:
         - "re:vllm_omni\\._.*"  # Internal modules
-        - "vllm_omni.diffusion.models.qwen_image"  # avoid importing vllm in mkdocs building
   - mkdocstrings:
       handlers:
         python:
+          paths: [vllm_omni]
           options:
             show_symbol_type_heading: true
             show_symbol_type_toc: true
-            filters:
-              - "!^_"  # Exclude private members (methods/classes starting with underscore)
-            summary:
-              modules: true
+            show_source: true
+            show_root_heading: true
+            show_root_toc_entry: true
+            show_root_full_path: false
+            show_object_full_path: false
             show_if_no_docstring: true
             show_signature_annotations: true
             separate_signature: true
             show_overloads: true
             signature_crossrefs: true
+            filters: []
+            summary:
+              modules: true
           inventories:
-          - https://docs.python.org/3/objects.inv
-          - https://typing-extensions.readthedocs.io/en/latest/objects.inv
-          - https://docs.aiohttp.org/en/stable/objects.inv
-          - https://pillow.readthedocs.io/en/stable/objects.inv
-          - https://numpy.org/doc/stable/objects.inv
-          - https://pytorch.org/docs/stable/objects.inv
-          - https://psutil.readthedocs.io/en/stable/objects.inv
+            - https://docs.python.org/3/objects.inv
+            - https://typing-extensions.readthedocs.io/en/latest/objects.inv
+            - https://docs.aiohttp.org/en/stable/objects.inv
+            - https://pillow.readthedocs.io/en/stable/objects.inv
+            - https://numpy.org/doc/stable/objects.inv
+            - https://pytorch.org/docs/stable/objects.inv
+            - https://psutil.readthedocs.io/en/stable/objects.inv
 
+# Markdown extensions
 markdown_extensions:
   - attr_list
   - md_in_html
@@ -142,12 +137,44 @@ markdown_extensions:
   # For math rendering
   - pymdownx.arithmatex:
       generic: true
+  # For task lists
+  - pymdownx.tasklist:
+      custom_checkbox: true
+  # For tables
+  - tables
 
+# Navigation
+nav:
+  - Home: index.md
+  - User Guide:
+    - Getting Started:
+      - user_guide/getting_started/quickstart.md
+    - Examples:
+      - user_guide/examples/index.md
+      - Offline Inference:
+        - user_guide/examples/offline_inference/qwen_2_5_omni.md
+      - Online Serving:
+        - user_guide/examples/online_serving/index.md
+  - API Reference:
+      - Summary: api/overview.md
+  - Developer Guide:
+    - contributing/README.md
+    - Design Documents:
+      - contributing/design_documents/index.md
+      - contributing/design_documents/vllm_omni_design.md
+      - contributing/design_documents/architecture_overview.md
+      - contributing/design_documents/api_design_doc.md
+
+# Extra
+extra:
+  version:
+    provider: mike
+
+# Extra CSS
 extra_css:
   - mkdocs/stylesheets/extra.css
 
-extra_javascript:
-  - mkdocs/javascript/mathjax.js
-  - https://unpkg.com/mathjax@3.2.2/es5/tex-mml-chtml.js
-  - mkdocs/javascript/edit_and_feedback.js
-  - mkdocs/javascript/slack_and_forum.js
+# Makes the url format end in .html rather than act as a dir
+# So index.md generates as index.html and is available under URL /index.html
+# https://www.mkdocs.org/user-guide/configuration/#use_directory_urls
+use_directory_urls: false

pyproject.toml

@@ -5,12 +5,12 @@ build-backend = "setuptools.build_meta"
 [project]
 name = "vllm-omni"
 version = "0.11.0rc1"
-description = "A framework for efficient model inference with omni-modality models"
+description = "vLLM-omni: Multi-modality models inference and serving with non-autoregressive structures"
 readme = "README.md"
 requires-python = ">=3.9,<3.14"  # Align with vLLM v0.11 supported Python versions
 license = {text = "Apache-2.0"}
 authors = [
-    {name = "vLLM-Omni Team"}
+    {name = "vLLM-omni Team", email = "vllm-project@gmail.com"}
 ]
 keywords = ["vllm", "multimodal", "diffusion", "transformer", "inference", "serving"]
 classifiers = [
@@ -27,17 +27,12 @@ classifiers = [
     "Topic :: Software Development :: Libraries :: Python Modules",
 ]
 
-
 dependencies = [
     # Core runtime dependencies (required for actual usage)
     "omegaconf>=2.3.0",
     # "decord>=0.6.0", # install manually
     "librosa>=0.11.0",
     "resampy>=0.4.3",
-    "diffusers==0.35.2",
-    "accelerate==1.12.0",
-    "gradio==5.50",
-    "soundfile>=0.13.1"
     # "vllm==0.11.0",  # TODO: fix the entrypoints overwrite problem
 ]
 
@@ -51,6 +46,8 @@ dev = [
     "isort>=5.12.0",
     "mypy>=1.0.0",
     "pre-commit>=3.0.0",
+    "gradio>=5.49.1",
+    "soundfile>=0.13.1"
 ]
 
 docs = [
@@ -59,13 +56,12 @@ docs = [
     "mkdocs-material",
     "mkdocstrings-python",
     "mkdocs-gen-files",
-    "mkdocs-awesome-nav",
+    # "mkdocs-awesome-nav",  # Not available on PyPI
     "mkdocs-glightbox",
     "mkdocs-git-revision-date-localized-plugin",
     "mkdocs-minify-plugin",
     "regex",
     "pydantic",
-    "black>=23.0.0",
 ]
 
 
@@ -163,9 +159,5 @@ extend-ignore-identifiers-re = [
     ".*_thw",
     ".*thw",
     "ein",
-    ".*arange",
-    ".*MoBA",
-    ".*temperal_downsample",
-    ".*nothink.*",
-    ".*NOTHINK.*"
+    ".*arange"
 ]

tests/__init__.py

@@ -1,6 +1,6 @@
 """
-Test suite for vLLM-Omni.
+Test suite for vLLM-omni.
 
 This package contains unit tests, integration tests, and benchmarks
-for vLLM-Omni.
+for vLLM-omni.
 """

vllm_omni/__init__.py

@@ -1,5 +1,5 @@
 """
-vLLM-Omni: Multi-modality models inference and serving with
+vLLM-omni: Multi-modality models inference and serving with
 non-autoregressive structures.
 
 This package extends vLLM beyond traditional text-based, autoregressive
@@ -19,22 +19,21 @@ except ModuleNotFoundError as exc:  # pragma: no cover - optional dependency
         raise
     # Allow importing vllm_omni without vllm (e.g., documentation builds)
     patch = None  # type: ignore
-
 from .config import OmniModelConfig
-from .entrypoints.async_omni import AsyncOmni
+from .entrypoints.async_omni_llm import AsyncOmniLLM
 
 # Main entry points
-from .entrypoints.omni import Omni
+from .entrypoints.omni_llm import OmniLLM
 
-from .version import __version__, __version_tuple__  # isort:skip
+# from .version import __version__, __version_tuple__  # isort:skip
 
 
 __all__ = [
-    "__version__",
-    "__version_tuple__",
+    # "__version__",  # TODO: add __version__ back when it is implemented
+    # "__version_tuple__",  # TODO: add __version_tuple__ back when it is implemented
     # Main components
-    "Omni",
-    "AsyncOmni",
+    "OmniLLM",
+    "AsyncOmniLLM",
     # Configuration
     "OmniModelConfig",
     # All other components are available through their respective modules

vllm_omni/config/__init__.py

@@ -1,5 +1,5 @@
 """
-Configuration module for vLLM-Omni.
+Configuration module for vLLM-omni.
 """
 
 from vllm_omni.config.model import OmniModelConfig

vllm_omni/config/model.py

@@ -1,72 +1,21 @@
-import json
-import warnings
-from importlib.util import find_spec
-from typing import Any, Literal, Optional
+from typing import Optional
 
-import torch
-import vllm.envs as envs
-from omegaconf import DictConfig, OmegaConf
 from pydantic import ConfigDict
 from pydantic.dataclasses import dataclass
 from vllm.config import ModelConfig, config
-from vllm.config.model import (
-    _RUNNER_CONVERTS,
-    _RUNNER_TASKS,
-    ConvertType,
-    RunnerOption,
-    TaskOption,
-    _get_and_verify_dtype,
-    get_served_model_name,
-)
-from vllm.config.multimodal import MMCacheType, MMEncoderTPMode, MultiModalConfig
-from vllm.config.pooler import PoolerConfig
-from vllm.logger import init_logger
-from vllm.platforms import current_platform
-from vllm.transformers_utils.config import (
-    get_config,
-    get_hf_image_processor_config,
-    get_hf_text_config,
-    get_pooling_config,
-    is_interleaved,
-)
-from vllm.transformers_utils.utils import maybe_model_redirect
 
 import vllm_omni.model_executor.models as me_models
 
-logger = init_logger(__name__)
-
 
 @config
 @dataclass(config=ConfigDict(arbitrary_types_allowed=True))
 class OmniModelConfig(ModelConfig):
-    """Configuration for Omni models, extending the base ModelConfig.
-
-    This configuration class extends the base vLLM ModelConfig with
-    omni-specific fields for multi-stage pipeline processing.
-
-    Attributes:
-        stage_id: Identifier for the stage in a multi-stage pipeline (default: 0)
-        model_stage: Stage type identifier, e.g., "thinker" or "talker"
-            (default: "thinker")
-        model_arch: Model architecture name
-            (default: "Qwen2_5OmniForConditionalGeneration")
-        engine_output_type: Optional output type specification for the engine.
-            Used to route outputs to appropriate processors (e.g., "image",
-            "audio", "latents"). If None, output type is inferred.
-
-    Example:
-        >>> config = OmniModelConfig(
-        ...     stage_id=0,
-        ...     model_stage="thinker",
-        ...     model_arch="Qwen2_5OmniForConditionalGeneration"
-        ... )
-    """
+    """Configuration for Omni models, extending the base ModelConfig."""
 
     stage_id: int = 0
     model_stage: str = "thinker"
     model_arch: str = "Qwen2_5OmniForConditionalGeneration"
     engine_output_type: Optional[str] = None
-    hf_config_name: Optional[str] = None
 
     @property
     def registry(self):
@@ -75,313 +24,3 @@ class OmniModelConfig(ModelConfig):
     @property
     def architectures(self) -> list[str]:
         return [self.model_arch]
-
-    def draw_hf_text_config(self):
-        # transformers' get_text_config method is used to get the text config from thinker_config.
-        # to handle the case that each model stage has their own text config,
-        # we need to draw the text config from the corresponding model stage.
-        if self.hf_config_name is None:
-            return get_hf_text_config(self.hf_config)
-        return getattr(self.hf_config, self.hf_config_name).get_text_config()
-
-    def __post_init__(
-        self,
-        # Multimodal config init vars
-        limit_mm_per_prompt: Optional[dict[str, int]],
-        media_io_kwargs: Optional[dict[str, dict[str, Any]]],
-        mm_processor_kwargs: Optional[dict[str, Any]],
-        mm_processor_cache_gb: Optional[float],
-        mm_processor_cache_type: Optional[MMCacheType],
-        mm_shm_cache_max_object_size_mb: Optional[int],
-        mm_encoder_tp_mode: Optional[MMEncoderTPMode],
-        interleave_mm_strings: Optional[bool],
-        skip_mm_profiling: Optional[bool],
-        video_pruning_rate: Optional[float],
-    ) -> None:
-        # Set the default seed to 0 in V1.
-        if envs.VLLM_USE_V1 and self.seed is None:
-            self.seed = 0
-            if not envs.VLLM_ENABLE_V1_MULTIPROCESSING:
-                logger.warning(
-                    "The global random seed is set to %d. Since "
-                    "VLLM_ENABLE_V1_MULTIPROCESSING is set to False, this may "
-                    "affect the random state of the Python process that "
-                    "launched vLLM.",
-                    self.seed,
-                )
-
-        # Keep set served_model_name before maybe_model_redirect(self.model)
-        self.served_model_name = get_served_model_name(self.model, self.served_model_name)
-        self.model = maybe_model_redirect(self.model)
-        # The tokenizer is consistent with the model by default.
-        if self.tokenizer is None:
-            self.tokenizer = self.model
-        if self.tokenizer_revision is None:
-            self.tokenizer_revision = self.revision
-        self.tokenizer = maybe_model_redirect(self.tokenizer)
-
-        if isinstance(self.hf_config_path, str):
-            self.hf_config_path = maybe_model_redirect(self.hf_config_path)
-
-        if callable(self.hf_overrides):
-            hf_overrides_kw: dict[str, Any] = {}
-            hf_overrides_fn = self.hf_overrides
-            dict_overrides: dict[str, Any] = {}
-        else:
-            # Separate dict overrides from flat ones; convert OmegaConf to dict
-            hf_overrides_kw = {}
-            dict_overrides = {}
-            for key, value in self.hf_overrides.items():
-                if isinstance(value, DictConfig):
-                    value = OmegaConf.to_container(value, resolve=True, structured_config_mode=None)
-                if isinstance(value, dict):
-                    dict_overrides[key] = value
-                else:
-                    hf_overrides_kw[key] = value
-            hf_overrides_fn = None
-
-        if self.rope_scaling:
-            hf_override: dict[str, Any] = {"rope_scaling": self.rope_scaling}
-            hf_overrides_kw.update(hf_override)
-            hf_overrides_str = json.dumps(hf_overrides_kw)
-            msg = (
-                "`--rope-scaling` will be removed in a future release. "
-                f"'Please instead use `--hf-overrides '{hf_overrides_str}'`"
-            )
-            warnings.warn(DeprecationWarning(msg), stacklevel=2)
-        if self.rope_theta is not None:
-            hf_override = {"rope_theta": self.rope_theta}
-            hf_overrides_kw.update(hf_override)
-            hf_overrides_str = json.dumps(hf_overrides_kw)
-            msg = (
-                "`--rope-theta` will be removed in a future release. "
-                f"'Please instead use `--hf-overrides '{hf_overrides_str}'`"
-            )
-            warnings.warn(DeprecationWarning(msg), stacklevel=2)
-
-        self.maybe_pull_model_tokenizer_for_runai(self.model, self.tokenizer)
-
-        if (backend := envs.VLLM_ATTENTION_BACKEND) and backend == "FLASHINFER" and find_spec("flashinfer") is None:
-            raise ValueError(
-                "VLLM_ATTENTION_BACKEND is set to FLASHINFER, but flashinfer "
-                "module was not found. See "
-                "https://github.com/vllm-project/vllm/blob/main/docker/Dockerfile "
-                "for instructions on how to install it."
-            )
-
-        if self.override_attention_dtype is not None and not current_platform.is_rocm():
-            warnings.warn("override-attention-dtype is set but not using ROCm platform", stacklevel=2)
-
-        if self.enable_sleep_mode and not current_platform.is_sleep_mode_available():
-            raise ValueError("Sleep mode is not supported on current platform.")
-
-        hf_config = get_config(
-            self.hf_config_path or self.model,
-            self.trust_remote_code,
-            self.revision,
-            self.code_revision,
-            self.config_format,
-            hf_overrides_kw=hf_overrides_kw,
-            hf_overrides_fn=hf_overrides_fn,
-        )
-
-        if dict_overrides:
-            self._apply_dict_overrides(hf_config, dict_overrides)
-
-        self.hf_config = hf_config
-        self.hf_text_config = self.draw_hf_text_config()
-        self.attention_chunk_size = getattr(self.hf_text_config, "attention_chunk_size", None)
-        self.encoder_config = self._get_encoder_config()
-        self.hf_image_processor_config = get_hf_image_processor_config(
-            self.model, hf_token=self.hf_token, revision=self.revision
-        )
-
-        architectures = self.architectures
-        registry = self.registry
-        is_generative_model = registry.is_text_generation_model(architectures, self)
-        is_pooling_model = registry.is_pooling_model(architectures, self)
-
-        def _task_to_convert(task: TaskOption) -> ConvertType:
-            if task == "embedding" or task == "embed":
-                return "embed"
-            if task == "classify":
-                return "classify"
-            if task == "reward":
-                return "reward"
-            if task == "score":
-                new_task = self._get_default_pooling_task(architectures)
-                return "classify" if new_task == "classify" else "embed"
-
-            return "none"
-
-        if self.task is not None:
-            runner: RunnerOption = "auto"
-            convert: ConvertType | Literal["auto"] = "auto"  # type: ignore[name-defined]
-            msg_prefix = (
-                "The 'task' option has been deprecated and will be removed in v0.13.0 or v1.0, whichever comes first."
-            )
-            msg_hint = "Please remove this option."
-
-            is_generative_task = self.task in _RUNNER_TASKS["generate"]
-            is_pooling_task = self.task in _RUNNER_TASKS["pooling"]
-
-            if is_generative_model and is_pooling_model:
-                if is_generative_task:
-                    runner = "generate"
-                    convert = "auto"  # type: ignore[assignment]
-                    msg_hint = (
-                        "Please replace this option with `--runner "
-                        "generate` to continue using this model "
-                        "as a generative model."
-                    )
-                elif is_pooling_task:
-                    runner = "pooling"
-                    convert = "auto"  # type: ignore[assignment]
-                    msg_hint = (
-                        "Please replace this option with `--runner "
-                        "pooling` to continue using this model "
-                        "as a pooling model."
-                    )
-                else:  # task == "auto"
-                    pass
-            elif is_generative_model or is_pooling_model:
-                if is_generative_task:
-                    runner = "generate"
-                    convert = "auto"  # type: ignore[assignment]
-                    msg_hint = "Please remove this option"
-                elif is_pooling_task:
-                    runner = "pooling"
-                    convert = _task_to_convert(self.task)
-                    msg_hint = (
-                        "Please replace this option with `--convert "
-                        f"{convert}` to continue using this model "
-                        "as a pooling model."
-                    )
-                else:  # task == "auto"
-                    pass
-            else:
-                raise AssertionError(
-                    f"The model should be a generative or pooling model when task is set to {self.task!r}."
-                )
-
-            self.runner = runner
-            self.convert = convert  # type: ignore[assignment]
-
-            msg = f"{msg_prefix} {msg_hint}"
-            warnings.warn(msg, DeprecationWarning, stacklevel=2)
-
-        self.runner_type = self._get_runner_type(architectures, self.runner)
-        self.convert_type = self._get_convert_type(architectures, self.runner_type, self.convert)
-
-        if self.runner_type == "generate" and not is_generative_model:
-            generate_converts = _RUNNER_CONVERTS["generate"]
-            if self.convert_type not in generate_converts:
-                raise ValueError("This model does not support `--runner generate`.")
-        if self.runner_type == "pooling" and not is_pooling_model:
-            pooling_converts = _RUNNER_CONVERTS["pooling"]
-            if self.convert_type not in pooling_converts:
-                convert_option = "<" + "|".join(pooling_converts) + ">"
-                raise ValueError(
-                    "This model does not support `--runner pooling`. "
-                    f"You can pass `--convert {convert_option} to adapt "
-                    "it into a pooling model."
-                )
-
-        # Note: Initialize these attributes early because transformers fallback
-        # may fail to load dynamic modules in child processes
-        model_info, arch = registry.inspect_model_cls(architectures, self)
-        self._model_info = model_info
-        self._architecture = arch
-        logger.info("Resolved architecture: %s", arch)
-
-        # Init pooler config if needed
-        if self.runner_type == "pooling":
-            if self.override_pooler_config is not None:
-                logger.warning_once(
-                    "`override_pooler_config` is deprecated and will be "
-                    "removed in v0.12.0 or v1.0.0, whichever is sooner. "
-                    "Please use `pooler_config` instead."
-                )
-
-                if isinstance(self.override_pooler_config, dict):
-                    self.pooler_config = PoolerConfig(**self.override_pooler_config)
-                else:
-                    self.pooler_config = self.override_pooler_config
-
-            if self.pooler_config is None:
-                self.pooler_config = PoolerConfig()
-
-            base_config = get_pooling_config(self.model, self.revision)
-            if base_config is not None:
-                for k, v in base_config.items():
-                    if getattr(self.pooler_config, k) is None:
-                        setattr(self.pooler_config, k, v)
-
-            default_pooling_type = self._model_info.default_pooling_type
-            if self.pooler_config.pooling_type is None:
-                self.pooler_config.pooling_type = default_pooling_type
-
-        self.dtype: torch.dtype = _get_and_verify_dtype(
-            self.model,
-            self.hf_config,
-            self.dtype,
-            is_pooling_model=self.runner_type == "pooling",
-            revision=self.revision,
-        )
-
-        # Interleaved attention is not supported by some backends in V0
-        if (
-            not self.disable_sliding_window
-            and is_interleaved(self.hf_text_config)
-            and not envs.VLLM_USE_V1
-            and (backend := envs.VLLM_ATTENTION_BACKEND) in ("XFORMERS", "FLASHINFER")
-        ):
-            logger.warning_once(
-                "%s has interleaved attention, which is currently not "
-                "supported by the %s backend. Disabling sliding window and "
-                "capping the max length to the sliding window size (%d).",
-                self.hf_text_config.model_type,
-                backend,
-                self.hf_text_config.sliding_window,
-            )
-            self.disable_sliding_window = True
-
-        self.original_max_model_len = self.max_model_len
-        self.max_model_len = self.get_and_verify_max_len(self.max_model_len)
-        # Init multimodal config if needed
-        if self._model_info.supports_multimodal:
-            if mm_encoder_tp_mode == "data" and not self._model_info.supports_multimodal_encoder_tp_data:
-                logger.warning_once(
-                    "This model does not support `--mm-encoder-tp-mode data`. "
-                    "Falling back to `--mm-encoder-tp-mode weights`."
-                )
-                mm_encoder_tp_mode = "weights"
-
-            mm_config_kwargs = dict(
-                limit_per_prompt=limit_mm_per_prompt,
-                media_io_kwargs=media_io_kwargs,
-                mm_processor_kwargs=mm_processor_kwargs,
-                mm_processor_cache_gb=mm_processor_cache_gb,
-                mm_processor_cache_type=mm_processor_cache_type,
-                mm_shm_cache_max_object_size_mb=mm_shm_cache_max_object_size_mb,
-                mm_encoder_tp_mode=mm_encoder_tp_mode,
-                interleave_mm_strings=interleave_mm_strings,
-                skip_mm_profiling=skip_mm_profiling,
-                video_pruning_rate=video_pruning_rate,
-            )
-
-            mm_config_kwargs = {k: v for k, v in mm_config_kwargs.items() if v is not None}
-
-            self.multimodal_config = MultiModalConfig(**mm_config_kwargs)
-
-        if self.disable_sliding_window:
-            self.hf_text_config.sliding_window = None
-
-        if not self.skip_tokenizer_init:
-            self._verify_tokenizer_mode()
-
-        self.config_updated = False
-
-        self._verify_quantization()
-        self._verify_cuda_graph()
-        self._verify_bnb_config()

vllm_omni/core/dit_cache_manager.py

@@ -1,5 +1,5 @@
 """
-DiT Cache Manager for vLLM-Omni.
+DiT Cache Manager for vLLM-omni.
 
 This module provides caching functionality for DiT (Diffusion Transformer) models
 to optimize inference performance and memory usage.

vllm_omni/core/sched/__init__.py

@@ -1,5 +1,5 @@
 """
-Scheduling components for vLLM-Omni.
+Scheduling components for vLLM-omni.
 """
 
 from .diffusion_scheduler import DiffusionScheduler

vllm_omni/core/sched/diffusion_scheduler.py

@@ -4,9 +4,9 @@ from typing import Optional
 
 from vllm.v1.core.kv_cache_manager import KVCacheBlocks
 from vllm.v1.core.sched.request_queue import create_request_queue
-from vllm.v1.core.sched.scheduler import Request, RequestStatus, SchedulerOutput, SpecDecodingStats
+from vllm.v1.core.sched.scheduler import EngineCoreOutputs, Request, RequestStatus, SchedulerOutput, SpecDecodingStats
 from vllm.v1.core.sched.utils import remove_all
-from vllm.v1.engine import EngineCoreEventType, EngineCoreOutput, EngineCoreOutputs
+from vllm.v1.engine import EngineCoreEventType, EngineCoreOutput
 
 from vllm_omni.core.sched.output import OmniNewRequestData
 from vllm_omni.core.sched.scheduler import OmniScheduler
@@ -14,22 +14,12 @@ from vllm_omni.outputs import OmniModelRunnerOutput
 
 
 class DiffusionScheduler(OmniScheduler):
-    """Scheduler for diffusion-based non-autoregressive stages.
-
-    Implements a fast-path scheduling strategy for diffusion models where
-    all input tokens are fed at once. Falls back to standard vLLM scheduling
-    if the token budget cannot accommodate all tokens in a single iteration.
-    """
-
     def schedule(self) -> SchedulerOutput:
-        """Schedule requests for diffusion model execution.
-
-        Uses a fast-path approach that feeds all input tokens at once
-        (or 1 placeholder token if input is empty). Falls back to
-        standard scheduling if token budget is insufficient.
-
-        Returns:
-            SchedulerOutput containing scheduled requests and token allocations
+        """Diffusion fast path:
+        - Feed all input tokens of the request at once
+          (if 0, allocate 1 placeholder token).
+        - If the token budget cannot be satisfied at once, fall back to the
+          default vLLM scheduling.
         """
 
         # Select requests with zero prompt and using pooling
@@ -249,12 +239,6 @@ class DiffusionScheduler(OmniScheduler):
 
             # Get prompt logprobs for this request.
             prompt_logprobs_tensors = prompt_logprobs_dict.get(req_id)
-            # Convert pooler_output tensor to dict format expected by OmniEngineCoreOutput
-            pooling_output_dict = None
-            if pooler_output is not None:
-                # Wrap tensor in dict to match OmniEngineCoreOutput.pooling_output type
-                # which expects Optional[dict[str, torch.Tensor]]
-                pooling_output_dict = {"model_outputs": pooler_output}
             if new_token_ids or pooler_output is not None or kv_transfer_params:
                 # Add EngineCoreOutput for this Request.
                 outputs[request.client_index].append(
@@ -264,7 +248,7 @@ class DiffusionScheduler(OmniScheduler):
                         finish_reason=request.get_finished_reason(),
                         new_logprobs=new_logprobs,
                         new_prompt_logprobs_tensors=prompt_logprobs_tensors,
-                        pooling_output=pooling_output_dict,
+                        pooling_output=pooler_output,
                         stop_reason=request.stop_reason,
                         events=request.take_events(),
                         kv_transfer_params=kv_transfer_params,