Initial plan

2025-10-24 04:05:48 +00:00
127 changed files with 7065 additions and 12942 deletions
@@ -1,6 +0,0 @@
-steps:
-  - label: ":toolbox: Qwen2.5 omni offline example"
-    commands:
-      - ".buildkite/scripts/run_qwen_offline_example.sh"
-    agents:
-      queue: "gpu_1_queue" # g6.4xlarge instance on AWS, has 1 L4 GPU
@@ -1,33 +0,0 @@
-#!/usr/bin/env bash
-
-set -euo pipefail
-
-# Move to repo root (script lives in .buildkite/scripts/)
-ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
-cd "${ROOT_DIR}"
-
-UV_INSTALL_DIR="${ROOT_DIR}/.local/bin"
-case ":${PATH}:" in
-  *":${UV_INSTALL_DIR}:"*) ;;
-  *) export PATH="${UV_INSTALL_DIR}:${PATH}" ;;
-esac
-
-if ! command -v uv >/dev/null 2>&1; then
-  mkdir -p "${UV_INSTALL_DIR}"
-  curl -LsSf https://astral.sh/uv/install.sh | env UV_INSTALL_DIR="${UV_INSTALL_DIR}" sh
-fi
-
-UV_BIN=(uv)
-
-"${UV_BIN[@]}" python install 3.12
-"${UV_BIN[@]}" venv --python 3.12 .venv
-source .venv/bin/activate
-
-"${UV_BIN[@]}" pip install -e .
-"${UV_BIN[@]}" pip install vllm==0.11.0 --torch-backend=auto
-
-EXAMPLE_DIR="examples/offline_inference/qwen2_5_omni"
-cd "${EXAMPLE_DIR}"
-
-python end2end.py --query-type text \
-                  --no-save-results
@@ -1,44 +0,0 @@
-# Set default behavior to automatically normalize line endings
-* text=auto
-
-# Explicitly declare text files you want to always be normalized and converted
-# to native line endings on checkout
-*.py text eol=lf
-*.yaml text eol=lf
-*.yml text eol=lf
-*.md text eol=lf
-*.txt text eol=lf
-*.json text eol=lf
-*.toml text eol=lf
-*.sh text eol=lf
-*.bat text eol=crlf
-*.cmd text eol=crlf
-
-# Declare files that will always have CRLF line endings on checkout
-*.bat text eol=crlf
-*.cmd text eol=crlf
-
-# Denote all files that are truly binary and should not be modified
-*.png binary
-*.jpg binary
-*.jpeg binary
-*.gif binary
-*.ico binary
-*.mov binary
-*.mp4 binary
-*.mp3 binary
-*.flv binary
-*.fla binary
-*.swf binary
-*.gz binary
-*.zip binary
-*.7z binary
-*.ttf binary
-*.eot binary
-*.woff binary
-*.woff2 binary
-*.pyc binary
-*.pdf binary
-*.ai binary
-*.psd binary
-*.xcf binary
@@ -1,17 +0,0 @@
-# CODEOWNERS is used to automatically assign PR reviewers
-# Format: path pattern @username or @team
-# More info: https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/customizing-your-repository/about-code-owners
-
-# Default owners (fallback reviewers for all files)
-# later, we can add more reviewers here
-* @hsliuustc0106
-
-# Assign reviewers by directory
-/.github/ @owner1
-/tests/ @owner1
-/docs/ @owner1
-
-# Example: reviewers for specific directories
-# /examples/offline_inference/ @offline-inference-team
-# /vllm_omni/core/ @core-team
-# /vllm_omni/model_executor/ @model-team
@@ -7,14 +7,14 @@ body:
 - type: markdown
  attributes:
    value: >
-      #### Before submitting an issue, please make sure the issue hasn't been already addressed by searching through [the existing and past issues](https://github.com/vllm-project/vllm-omni/issues?q=is%3Aissue+sort%3Acreated-desc+).
+      #### Before submitting an issue, please make sure the issue hasn't been already addressed by searching through [the existing and past issues](https://github.com/hsliuustc0106/vllm-omni/issues?q=is%3Aissue+sort%3Acreated-desc+).
 - type: textarea
  attributes:
    label: Your current environment
    description: |
      Please run the following and paste the output below.
      ```sh
-      wget https://raw.githubusercontent.com/vllm-project/vllm-omni/main/vllm_omni/collect_env.py
+      wget https://raw.githubusercontent.com/hsliuustc0106/vllm-omni/main/vllm_omni/collect_env.py
      # For security purposes, please feel free to check the contents of collect_env.py before running it.
      python collect_env.py
      ```
@@ -7,7 +7,7 @@ body:
 - type: markdown
  attributes:
    value: >
-      #### Before submitting an issue, please make sure the issue hasn't been already addressed by searching through [the existing and past issues](https://github.com/vllm-project/vllm-omni/issues?q=is%3Aissue+sort%3Acreated-desc+).
+      #### Before submitting an issue, please make sure the issue hasn't been already addressed by searching through [the existing and past issues](https://github.com/hsliuustc0106/vllm-omni/issues?q=is%3Aissue+sort%3Acreated-desc+).
 - type: markdown
  attributes:
    value: |
@@ -16,7 +16,7 @@ body:
      - Passwords or authentication credentials
      - Private URLs or endpoints
      - Personal or confidential data
-
+      
      Consider redacting or replacing sensitive values with placeholders like `<YOUR_TOKEN_HERE>` when sharing configuration or code examples.
 - type: textarea
  attributes:
@@ -24,7 +24,7 @@ body:
    description: |
      Please run the following and paste the output below.
      ```sh
-      wget https://raw.githubusercontent.com/vllm-project/vllm-omni/main/vllm_omni/collect_env.py
+      wget https://raw.githubusercontent.com/hsliuustc0106/vllm-omni/main/vllm_omni/collect_env.py
      # For security purposes, please feel free to check the contents of collect_env.py before running it.
      python collect_env.py
      ```
@@ -58,7 +58,7 @@ body:
          input_modalities=["text"],
          output_modalities=["text"]
      )
-
+      
      dit_config = create_dit_stage_config(
          stage_id=1,
          model_path="stabilityai/stable-diffusion-2-1",
@@ -7,7 +7,7 @@ body:
 - type: markdown
  attributes:
    value: >
-      #### Before submitting an issue, please make sure the issue hasn't been already addressed by searching through [the existing and past issues](https://github.com/vllm-project/vllm-omni/issues?q=is%3Aissue+sort%3Acreated-desc+).
+      #### Before submitting an issue, please make sure the issue hasn't been already addressed by searching through [the existing and past issues](https://github.com/hsliuustc0106/vllm-omni/issues?q=is%3Aissue+sort%3Acreated-desc+).
 - type: textarea
  attributes:
    label: 🚀 The feature, motivation and pitch
@@ -7,7 +7,7 @@ body:
 - type: markdown
  attributes:
    value: >
-      #### Before submitting an issue, please make sure the issue hasn't been already addressed by searching through [the existing and past issues](https://github.com/vllm-project/vllm-omni/issues?q=is%3Aissue+sort%3Acreated-desc+).
+      #### Before submitting an issue, please make sure the issue hasn't been already addressed by searching through [the existing and past issues](https://github.com/hsliuustc0106/vllm-omni/issues?q=is%3Aissue+sort%3Acreated-desc+).

      #### We also highly recommend you read the contributing guidelines first to understand how to add a new model.
 - type: textarea
@@ -7,7 +7,7 @@ body:
 - type: markdown
  attributes:
    value: >
-      #### Before submitting an issue, please make sure the issue hasn't been already addressed by searching through [the existing and past issues](https://github.com/vllm-project/vllm-omni/issues?q=is%3Aissue+sort%3Acreated-desc+).
+      #### Before submitting an issue, please make sure the issue hasn't been already addressed by searching through [the existing and past issues](https://github.com/hsliuustc0106/vllm-omni/issues?q=is%3Aissue+sort%3Acreated-desc+).
 - type: textarea
  attributes:
    label: Proposal to improve performance
@@ -19,7 +19,7 @@ body:
  attributes:
    label: Report of performance regression
    description: >
-      Please provide detailed description of performance comparison to confirm the regression. You may want to run the benchmark script at https://github.com/vllm-project/vllm-omni/tree/main/tests/benchmarks .
+      Please provide detailed description of performance comparison to confirm the regression. You may want to run the benchmark script at https://github.com/hsliuustc0106/vllm-omni/tree/main/tests/benchmarks .
  validations:
    required: false
 - type: textarea
@@ -35,7 +35,7 @@ body:
    description: |
      Please run the following and paste the output below.
      ```sh
-      wget https://raw.githubusercontent.com/vllm-project/vllm-omni/main/vllm_omni/collect_env.py
+      wget https://raw.githubusercontent.com/hsliuustc0106/vllm-omni/main/vllm_omni/collect_env.py
      # For security purposes, please feel free to check the contents of collect_env.py before running it.
      python collect_env.py
      ```
@@ -7,7 +7,7 @@ body:
 - type: markdown
  attributes:
    value: >
-      #### Please take a look at previous [RFCs](https://github.com/vllm-project/vllm-omni/issues?q=label%3ARFC+sort%3Aupdated-desc) for reference.
+      #### Please take a look at previous [RFCs](https://github.com/hsliuustc0106/vllm-omni/issues?q=label%3ARFC+sort%3Aupdated-desc) for reference.
 - type: textarea
  attributes:
    label: Motivation.
@@ -1,5 +1,5 @@
 blank_issues_enabled: false
 contact_links:
  - name: Questions
-    url: https://github.com/vllm-project/vllm-omni/discussions
+    url: https://github.com/hsliuustc0106/vllm-omni/discussions
    about: Ask questions and discuss with other vLLM-omni community members
@@ -18,4 +18,4 @@ PLEASE FILL IN THE PR DESCRIPTION HERE ENSURING ALL CHECKLIST ITEMS (AT THE BOTT
 - [ ] (Optional) Release notes update. If your change is user facing, please update the release notes draft.
 </details>

-**BEFORE SUBMITTING, PLEASE READ <https://github.com/vllm-project/vllm-omni/blob/main/CONTRIBUTING.md>** (anything written below this line will be removed by GitHub Actions)
+**BEFORE SUBMITTING, PLEASE READ <https://github.com/hsliuustc0106/vllm-omni/blob/main/CONTRIBUTING.md>** (anything written below this line will be removed by GitHub Actions)
@@ -1,89 +0,0 @@
-#!/bin/bash
-# Script to detect changed files and find corresponding test files
-# Usage: ./detect_changed_tests.sh [base_sha] [head_sha] [event_type]
-#
-# Outputs to GITHUB_OUTPUT:
-#   test_files: Space-separated list of test files to run
-#   has_tests: 'true' if test files found, 'false' otherwise
-
-set -e
-
-BASE_SHA="${1:-}"
-HEAD_SHA="${2:-}"
-EVENT_TYPE="${3:-push}"
-
-# Determine base and head SHA based on event type
-if [ "$EVENT_TYPE" = "pull_request" ]; then
-    BASE_SHA="${BASE_SHA:-$GITHUB_BASE_REF}"
-    HEAD_SHA="${HEAD_SHA:-$GITHUB_SHA}"
-elif [ -z "$BASE_SHA" ] || [ -z "$HEAD_SHA" ]; then
-    # For push events, use git to find the previous commit
-    BASE_SHA="${BASE_SHA:-$(git rev-parse HEAD~1 2>/dev/null || echo HEAD)}"
-    HEAD_SHA="${HEAD_SHA:-HEAD}"
-fi
-
-echo "Event type: $EVENT_TYPE"
-echo "Base SHA: $BASE_SHA"
-echo "Head SHA: $HEAD_SHA"
-
-# Get changed Python files
-CHANGED_FILES=$(git diff --name-only --diff-filter=ACMRT "$BASE_SHA" "$HEAD_SHA" | grep -E '\.(py)$' || true)
-
-if [ -z "$CHANGED_FILES" ]; then
-    echo "No Python files changed"
-    echo "test_files=" >> "$GITHUB_OUTPUT"
-    echo "has_tests=false" >> "$GITHUB_OUTPUT"
-    exit 0
-fi
-
-echo "Changed Python files:"
-echo "$CHANGED_FILES"
-
-# Extract related test files
-TEST_FILES=""
-for file in $CHANGED_FILES; do
-    # If changed file is a test file, add it directly
-    if [[ "$file" == tests/* ]]; then
-        TEST_FILES="$TEST_FILES $file"
-    # If changed file is source code, find corresponding test file
-    elif [[ "$file" == vllm_omni/* ]]; then
-        # Convert vllm_omni/path/to/module.py to possible test file paths
-        REL_PATH=$(echo "$file" | sed 's|^vllm_omni/||' | sed 's|\.py$||')
-        MODULE_NAME=$(basename "$REL_PATH")
-        DIR_PATH=$(dirname "$REL_PATH")
-
-        # Try multiple possible test file naming patterns
-        POSSIBLE_TESTS=(
-            "tests/test_${REL_PATH//\//_}.py"
-            "tests/${REL_PATH//\//_}_test.py"
-        )
-
-        # If multi-level directory, also try directory structure
-        if [ "$DIR_PATH" != "." ]; then
-            POSSIBLE_TESTS+=(
-                "tests/${DIR_PATH}/test_${MODULE_NAME}.py"
-                "tests/${DIR_PATH}/${MODULE_NAME}_test.py"
-            )
-        fi
-
-        for test_file in "${POSSIBLE_TESTS[@]}"; do
-            if [ -f "$test_file" ]; then
-                TEST_FILES="$TEST_FILES $test_file"
-                break
-            fi
-        done
-    fi
-done
-
-# Deduplicate and format
-TEST_FILES=$(echo "$TEST_FILES" | tr ' ' '\n' | grep -v '^$' | sort -u | tr '\n' ' ' | sed 's/^ *//;s/ *$//')
-
-if [ -z "$TEST_FILES" ]; then
-    echo "No related test files found, will run all tests"
-    echo "test_files=" >> "$GITHUB_OUTPUT"
-    echo "has_tests=false" >> "$GITHUB_OUTPUT"
-else
-    echo "Found test files: $TEST_FILES"
-    echo "test_files=$TEST_FILES" >> "$GITHUB_OUTPUT"
-    echo "has_tests=true" >> "$GITHUB_OUTPUT"
-fi
@@ -1,26 +0,0 @@
-name: pre-commit
-
-on:
-  pull_request:
-    branches: [main]
-  push:
-    branches: [main]
-  workflow_dispatch: # for manual trigger
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
-
-permissions:
-  contents: read
-
-jobs:
-  pre-commit:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-      - uses: actions/setup-python@v5
-        with:
-          python-version: "3.12"
-          cache: 'pip'
-      - uses: pre-commit/action@v3.0.1
@@ -20,7 +20,6 @@ parts/
 sdist/
 var/
 wheels/
-vllm/
 share/python-wheels/
 *.egg-info/
 .installed.cfg
@@ -160,7 +159,7 @@ cython_debug/
 docker/

 # scripts
-/scripts/
+scripts/

 # tests
 tests/
@@ -177,6 +176,7 @@ tests/

 # vLLM-omni specific
 # Model files and checkpoints
+models/
 checkpoints/
 *.bin
 *.safetensors
@@ -1,60 +0,0 @@
-default_install_hook_types:
-  - pre-commit
-  - commit-msg
-repos:
-  - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.6.0
-    hooks:
-      # list of supported hooks: https://pre-commit.com/hooks.html
-      - id: check-yaml
-        args: ["--unsafe"]
-      - id: debug-statements
-      - id: end-of-file-fixer
-      - id: mixed-line-ending
-        args: ["--fix=lf"]
-      - id: trailing-whitespace
-        args: ["--markdown-linebreak-ext=md"]
-
-  - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.14.0
-    hooks:
-    - id: ruff-check
-      args: [--output-format, github, --fix]
-    - id: ruff-format
-
-  - repo: https://github.com/crate-ci/typos
-    rev: v1.35.5
-    hooks:
-      - id: typos
-        # only for staged files
-
-  - repo: https://github.com/rhysd/actionlint
-    rev: v1.7.7
-    hooks:
-      - id: actionlint
-        files: ^\.github/workflows/.*\.ya?ml$
-
-
-  - repo: local
-    hooks:
-    - id: signoff-commit
-      name: Sign-off Commit
-      entry: bash
-      args:
-        - -c
-        - |
-          if ! grep -q "^Signed-off-by: $(git config user.name) <$(git config user.email)>" "$(git rev-parse --git-path COMMIT_EDITMSG)"; then
-            printf "\nSigned-off-by: $(git config user.name) <$(git config user.email)>\n" >> "$(git rev-parse --git-path COMMIT_EDITMSG)"
-          fi
-      language: system
-      verbose: true
-      stages: [commit-msg]
-
-    # Keep `suggestion` last
-    - id: suggestion
-      name: Suggestion
-      entry: bash -c 'echo "To bypass all the pre-commit hooks, add --no-verify to git commit. To skip a specific hook, prefix the commit command with SKIP=<hook-id>."'
-      language: system
-      verbose: true
-      pass_filenames: false
-    # Insert new entries above the `suggestion` entry
@@ -1,24 +0,0 @@
-# Read the Docs configuration file
-# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
-
-version: 2
-
-build:
-  os: ubuntu-22.04
-  tools:
-    python: "3.12"
-  jobs:
-    post_checkout:
-      - git fetch --unshallow || true
-
-mkdocs:
-  configuration: mkdocs.yml
-  fail_on_warning: true
-
-# Optionally declare the Python requirements required to build your docs
-python:
-  install:
-    - method: pip
-      path: .
-      extra_requirements:
-        - docs
@@ -1,19 +0,0 @@
-# Contributing to vLLM
-
-## Getting Started
-
-Setup environment with uv
-```bash
-uv venv --python 3.12 --seed
-source .venv/bin/activate
-```
-
-Pre-commit has been linked to git commit. For run checks locally, please install pre-commit:
-```bash
-uv pip install pre-commit
-```
-
-Then run:
-```bash
-pre-commit run --show-diff-on-failure --color=always --all-files
-```
@@ -1,137 +1,75 @@
-<p align="center">
-  <picture>
-    <source media="(prefers-color-scheme: dark)" src="./docs/source/logos/vllm-omni-logo-text-dark.png">
-    <img alt="vllm-omni" src="./docs/source/logos/vllm-omni-logo-text-dark.png" width=55%>
-  </picture>
-</p>
-<h3 align="center">
-Easy, fast, and cheap omni-modality model serving for everyone
-</h3>
+# vLLM-omni: Multi-modal Extension for vLLM

-<p align="center">
-| <a href="https://vllm-omni.readthedocs.io/en/latest/"><b>Documentation</b></a> | <a href="https://tinyurl.com/vllm-omni-meeting"><b>Weekly Meeting</b></a> |
-</p>
+vLLM-omni is designed to extend vLLM capabilities to support multi-modality model inference and serving, particularly focusing on non-autoregressive structures and non-textual outputs.

---
+## 🎯 Overview

-*Latest News* 🔥
+Traditional vLLM systems are limited to text-based, autoregressive generation. vLLM-omni addresses this limitation by enabling support for:

- [2025/11] vLLM community officially released [vllm-project/vllm-omni](https://github.com/vllm-project/vllm-omni) in order to support omni-modality models serving.
+- **Multi-modal Models**: Text, image, video, audio, and sensor data processing
+- **Non-autoregressive Architectures**: Diffusion Transformers (DiT) and other parallel generation models
+- **Heterogeneous Outputs**: Beyond traditional text generation to multimodal outputs

---
+## 🏗️ Architecture

-## About
+vLLM-omni is built on a modular architecture that extends vLLM's core functionality:

-[vLLM](https://github.com/vllm-project/vllm) was originally designed to support large language models for text-based autoregressive generation tasks. vLLM-Omni extends its support for omni-modality model inference and serving:

- **Omni-modality**: Text, image, video, and audio data processing
- **Non-autoregressive Architectures**: extend the AR support of vLLM to Diffusion Transformers (DiT) and other parallel generation models
- **Heterogeneous outputs**: from traditional text generation to multimodal outputs
+## 🚀 Key Features

-<p align="center">
-  <picture>
-    <img alt="vllm-omni" src="./docs/source/architecture/omni-modality model architecture.png" width=55%>
-  </picture>
-</p>
+### Multi-Engine Support

-vLLM-omni is fast with:
+- **Autoregressive Engine**: Traditional text generation with enhanced KV-caching
+- **Diffusion Engine**: Support for DiT models and iterative generation
+- **Hybrid Engine**: Combined AR+DiT processing pipelines

- Seamless AR support for efficient KV Cache management with vLLM
- Pipelined stage execution overlapping
- Fully disaggregation based on omniConnector and dynamic resource allocation across stages
+### Modality Processing

-vLLM-omni is flexible and easy to use with:
+- **Text**: Advanced tokenization and embedding generation
+- **Image**: Vision encoder integration (CLIP, etc.)
+- **Audio**: Speech processing and audio embedding

- Seamless integration with popular Hugging Face models
- High-throughput serving with various decoding algorithms, including *parallel sampling*, *beam search*, and more
- Tensor, pipeline, data and expert parallelism support for distributed inference
- Streaming outputs
- OpenAI-compatible API server
+## 📋 Supported Models

-vLLM seamlessly supports most popular open-source models on HuggingFace, including:
-
- Multi-modality generation models (e.g. Qwen-image)
- Omni-modality models (e.g. Qwen-omni)
-
-## Getting Started
-
-Install vLLM with `pip` or [from source](https://docs.vllm.ai/en/latest/getting_started/installation/gpu/index.html#build-wheel-from-source):
-
-```bash
-pip install vllm
-```
+### AR + Diffusion Transformer (DiT) Models
+- Qwen-omni (Thinker-Talker-Codec structure)
+- HunyunaImage 3.0 (Ongoing)
+- Qwen-Image (Ongoing)

 ## 🛠️ Installation

-### Environment setup
-
-Use Docker to keep consistent basic environment (Optional, Recommended)
+Set up basic environments
 ```bash
-docker run --gpus all --ipc=host --network=host -v $source_dir:$container_dir --name $container_name -it nvcr.io/nvidia/pytorch:25.01-py3 bash
-```
-
-Set up basic uv environment
-```bash
-pip install uv
 uv venv --python 3.12 --seed
 source .venv/bin/activate
 ```
-
-### Installation of vLLM (for users)
-
-Now we build it based on vLLM v0.11.0. Please install it with command below.
-```bash
-uv pip install vllm==0.11.0 --torch-backend=auto
-```
-
-### Installation of vLLM (for developers)
-
-If you want to check or debug with source code of vLLM, install stable release version of vllm with 0.11.0 from source with pre-built wheel file.
+Install certain version of vllm with commitid: 808a7b69df479b6b3a16181711cac7ca28a9b941

 ```bash
 git clone https://github.com/vllm-project/vllm.git
 cd vllm
-git checkout v0.11.0
+git checkout 808a7b69df479b6b3a16181711cac7ca28a9b941
+VLLM_USE_PRECOMPILED=1 uv pip install --editable .
 ```
-Set up environment variables to get pre-built wheels. If there are internet problems, just download the whl file manually. And set VLLM_PRECOMPILED_WHEEL_LOCATION as your local absolute path of whl file.
-```bash
-export VLLM_PRECOMPILED_WHEEL_LOCATION=https://github.com/vllm-project/vllm/releases/download/v0.11.0/vllm-0.11.0-cp38-abi3-manylinux1_x86_64.whl
-```
-Install vllm with command below (If you have no existing PyTorch).
-```bash
-uv pip install --editable .
-```
-
-Install vllm with command below (If you already have PyTorch).
-```bash
-python use_existing_torch.py
-uv pip install -r requirements/build.txt
-uv pip install --no-build-isolation --editable .
-```
-
-### Installation of vLLM-omni
-Install additional requirements for vllm-omni
-```bash
-git clone https://github.com/vllm-project/vllm-omni.git
-cd vllm_omni
-uv pip install -e .
-```
-

 ## Run examples (Qwen2.5-omni)

-Please check the folder of [examples](examples)
+Get into the example folder
+```bash
+cd vllm_omni
+cd examples/offline_inference/qwen2_5_omni
+```
+Modify PYTHONPATH in run.sh as your path of vllm_omni. Then run.
+```bash
+bash run.sh
+```
+The output audio is saved in ./output_audio

-## Contributing
+## To-do list
+- [x] Offline inference example for Qwen2.5-omni with single request
+- [ ] Adaptation from current vllm branch to stable vllm v0.11.0
+- [ ] Offline inference example for Qwen2.5-omni with streaming multiple requests
+- [ ] Online inference support
+- [ ] Support for other models

-We welcome and value any contributions and collaborations.
-Please check out [Contributing to vLLM-omni](https://docs.vllm.ai/en/latest/contributing/index.html) for how to get involved.
-
-## Weekly Meeting
-
- vLLM Omni Weekly Meeting: https://tinyurl.com/vllm-omni-meeting
- Wednesday, 11:30 - 12:30 (UTC+8, [Convert to your timezone](https://dateful.com/convert/gmt8?t=15))
-
-## License
-
-Apache License 2.0, as found in the [LICENSE](./LICENSE) file.
+For detailed model management, see [vllm_omni_design.md](docs/architecture/vllm_omni_design.md) and [high_level_arch_design.md](docs/architecture/high_level_arch_design.md).
@@ -1,139 +0,0 @@
-# Documentation Build Guide
-
-This directory contains the source files for the vLLM-omni documentation.
-
-## Building Documentation Locally
-
-### Prerequisites
-
-Install documentation dependencies:
-
-```bash
-pip install -e ".[docs]"
-```
-
-### Build and Serve Documentation
-
-From the project root:
-
-```bash
-# Serve documentation locally (auto-reload on changes)
-# This starts a local web server at http://127.0.0.1:8000
-mkdocs serve
-
-# Build static site (generates HTML files in site/ directory)
-mkdocs build
-```
-
-When using `mkdocs serve`, the documentation will be automatically available at `http://127.0.0.1:8000`. The server will automatically reload when you make changes to the documentation files.
-
-## Auto-generating API Documentation
-
-The documentation automatically extracts docstrings from the code using mkdocstrings. To ensure your code is documented:
-
-1. Add docstrings to all public classes, functions, and methods
-2. Use Google or NumPy style docstrings (both are supported)
-3. Rebuild the documentation to see changes
-
-Example docstring:
-
-```python
-class OmniLLM:
-    """Main entry point for vLLM-omni inference.
-
-    This class provides a high-level interface for running multi-modal
-    inference with non-autoregressive models.
-
-    Args:
-        model: Model name or path
-        stage_configs: Optional stage configurations
-        **kwargs: Additional arguments passed to the engine
-
-    Example:
-        >>> llm = OmniLLM(model="Qwen/Qwen2.5-Omni")
-        >>> outputs = llm.generate(prompts="Hello")
-    """
-```
-
-## Documentation Structure
-
-```
-docs/
-├── index.md              # Main documentation page
-├── getting_started/      # Getting started guides
-├── architecture/        # Architecture documentation
-├── api/                 # API reference (auto-generated from code)
-├── examples/            # Code examples
-└── stylesheets/         # Custom CSS
-```
-
-## Publishing Documentation
-
-### GitHub Pages (Recommended)
-
-The documentation is automatically deployed to GitHub Pages using GitHub Actions.
-
-1. **Enable GitHub Pages**:
-   - Go to repository `Settings` → `Pages`
-   - Set `Source` to `GitHub Actions`
-   - Save settings
-
-2. **Push changes**:
-   ```bash
-   git push origin main
-   ```
-
-3. **Documentation will be available at**:
-   - `https://vllm-project.github.io/vllm-omni/`
-
-The GitHub Actions workflow (`.github/workflows/docs.yml`) will automatically:
- Build the documentation when you push to `main` branch
- Deploy it to GitHub Pages
- Update the documentation whenever you make changes
-
-
-### Read the Docs (Alternative)
-
-You can also use Read the Docs for hosting:
-
-1. Sign up at https://readthedocs.org/
-2. Import the `vllm-project/vllm-omni` repository
-3. Read the Docs will automatically build using `.readthedocs.yml`
-4. Documentation will be available at: `https://vllm-omni.readthedocs.io/`
-
-## Configuration
-
-The documentation configuration is in `mkdocs.yml` at the project root.
-
-## Tips
-
- **API Documentation**: API docs are automatically generated using `mkdocs-api-autonav` and `mkdocstrings`
-  - No need to manually create API pages - they're generated automatically
-  - Use `[module.name.ClassName][]` syntax for cross-references in Summary pages
- **Code Snippets**: Use `--8<-- "path/to/file.py"` for including code snippets
- **Markdown**: Use Markdown for all documentation (no need for RST)
- **Material Theme**: Use Material theme features like:
-  - Admonitions: `!!! note`, `!!! warning`, etc.
-  - Code blocks with syntax highlighting
-  - Tabs for organizing content
-  - Math formulas using `pymdownx.arithmatex`
-
-## Troubleshooting
-
-### Documentation not updating
-
- Make sure you've saved all files
- If using `mkdocs serve`, it should auto-reload
- Check for syntax errors in `mkdocs.yml`
-
-### API links not working
-
- Ensure class names match exactly (case-sensitive)
- Check that the module is imported correctly
- Run `mkdocs build --strict` to check for errors
-
-### Build errors
-
- Check Python version (requires 3.9+)
- Ensure all dependencies are installed: `pip install -e ".[docs]"`
- Check `mkdocs.yml` syntax with `mkdocs build --strict`
@@ -73,20 +73,20 @@ from enum import Enum

 class DiTCacheManager:
    """Manages DiT cache for diffusion models."""
-
+    
    def __init__(self, config: DiTCacheConfig):
        self.config = config
        self.cache = {}
        self.cache_stats = {}
-
+    
    async def get_cache(self, cache_key: str) -> Optional[Any]:
        """Retrieve cached data."""
        pass
-
+    
    async def set_cache(self, cache_key: str, data: Any) -> None:
        """Store data in cache."""
        pass
-
+    
    async def invalidate_cache(self, cache_key: str) -> None:
        """Invalidate cached data."""
        pass
@@ -103,7 +103,7 @@ class OmniScheduler():
        self.scheduler = self._create_scheduler()
        self.cache_manager = DiTCacheManager(config.dit_cache_config)
        self._running = False
-
+    
    def _create_scheduler(self) -> BaseScheduler:
        """Factory method to create appropriate scheduler."""
        if self.config.scheduler_type == SchedulerType.FIFO:
@@ -118,13 +118,13 @@ class OmniScheduler():
    async def schedule(self, request: OmniRequest) -> bool:
        """
        Schedule a request for processing.
-
+        
        Args:
            request: The request to schedule
-
+            
        Returns:
            bool: True if successfully scheduled, False otherwise
-
+            
        Raises:
            SchedulerError: If scheduling fails
            QueueFullError: If queue is at capacity
@@ -137,7 +137,7 @@ class OmniScheduler():
    def update_config(self, new_config: SchedulerConfig) -> None:
        """Update scheduler configuration."""
        pass
-
+    
    def get_config(self) -> SchedulerConfig:
        """Get current configuration."""
        pass
@@ -149,12 +149,12 @@ class OmniScheduler():
        """Start the scheduler."""
        self._running = True
        # Start background tasks
-
+    
    async def stop(self) -> None:
        """Stop the scheduler gracefully."""
        self._running = False
        # Cleanup resources
-
+    
    async def shutdown(self) -> None:
        """Force shutdown the scheduler."""
        # Immediate cleanup
@@ -170,7 +170,7 @@ class OmniScheduler():
            "processed_requests": self.scheduler.processed_count(),
            "cache_hit_rate": self.cache_manager.get_hit_rate()
        }
-
+    
    def get_metrics(self) -> Dict[str, float]:
        """Get performance metrics."""
        pass
@@ -182,21 +182,21 @@ class OmniScheduler():
@dataclass
 class OmniConfig:
    """Configuration for the core module."""
-
+    
    # Scheduler configuration
    scheduler: SchedulerConfig = field(default_factory=SchedulerConfig)
-
+    
    # Cache configuration
    dit_cache: DiTCacheConfig = field(default_factory=DiTCacheConfig)
-
+    
    # Resource limits
    max_memory_gb: float = 16.0
    max_gpu_utilization: float = 0.8
-
+    
    # Timeouts
    request_timeout: int = 300
    worker_timeout: int = 60
-
+    
    def validate(self) -> None:
        """Validate configuration parameters."""
        if self.max_memory_gb <= 0:
@@ -1,10 +1,13 @@
-# API Design Documentation
+# vLLM-omni API Documentation

-This document contains comprehensive API design documentation for all core modules in vLLM-omni. These templates provide a standardized structure for designing and implementing the core, engine, executor, and worker modules.
+This directory contains comprehensive API design documentation for all core modules in vLLM-omni. These templates provide a standardized structure for designing and implementing the core, engine, executor, and worker modules.

-## 📋 Module API
+## 📋 API Design Templates

-### Core Module API
+### 1. [API Design Template](API_DESIGN_TEMPLATE.md)
+**Master template** for creating API documentation for any module in vLLM-omni. Use this as a starting point for new modules.
+
+### 2. [Core Module API](core_module_api.md)
 **Core module** provides fundamental scheduling, caching, and resource management functionality.

 **Key Components:**
@@ -13,7 +16,7 @@ This document contains comprehensive API design documentation for all core modul
 - Resource allocation and coordination
 - Inter-module communication

-### Engine Module API
+### 3. [Engine Module API](engine_module_api.md)
 **Engine module** handles model loading, inference execution, and output processing.

 **Key Components:**
@@ -22,7 +25,7 @@ This document contains comprehensive API design documentation for all core modul
 - Input preprocessing for various modalities
 - Output postprocessing and formatting

-### Executor Module API
+### 4. [Executor Module API](executor_module_api.md)
 **Executor module** coordinates and manages request execution across different engines and workers.

 **Key Components:**
@@ -31,7 +34,7 @@ This document contains comprehensive API design documentation for all core modul
 - Worker management and task distribution
 - Error handling and recovery

-### Worker Module API
+### 5. [Worker Module API](worker_module_api.md)
 **Worker module** provides the actual execution environment for model inference.

 **Key Components:**
@@ -40,6 +43,79 @@ This document contains comprehensive API design documentation for all core modul
 - Request batching and processing
 - Performance optimization

+## 🎯 How to Use These Templates
+
+### For New Module Development
+
+1. **Start with the Master Template**
+   - Copy `API_DESIGN_TEMPLATE.md`
+   - Rename to `[module_name]_api.md`
+   - Follow the structure exactly
+
+2. **Fill in Module-Specific Details**
+   - Update the module overview
+   - Define core classes and interfaces
+   - Specify public API methods
+   - Add configuration options
+   - Define error handling strategy
+   - Provide usage examples
+
+3. **Review and Validate**
+   - Ensure all sections are complete
+   - Verify code examples work
+   - Check for consistency with other modules
+   - Validate configuration options
+
+### For Existing Module Updates
+
+1. **Update API Documentation**
+   - Modify existing API files
+   - Add new methods and classes
+   - Update examples and configuration
+   - Maintain backward compatibility notes
+
+2. **Version Control**
+   - Track changes in git
+   - Use clear commit messages
+   - Tag major API changes
+
+## 📚 Template Structure
+
+Each API template follows this standardized structure:
+
+### 1. Module Overview
+- **Purpose**: What the module does
+- **Responsibilities**: Key responsibilities
+- **Dependencies**: Required modules
+- **Integration Points**: How it connects with other modules
+
+### 2. Core Classes/Interfaces
+- **Base Classes**: Abstract base classes
+- **Implementation Classes**: Concrete implementations
+- **Data Structures**: Key data models
+
+### 3. Public API Methods
+- **Initialization**: Constructor and setup
+- **Core Operations**: Main functionality
+- **Configuration**: Settings management
+- **Lifecycle Management**: Start/stop/cleanup
+- **Monitoring**: Status and metrics
+
+### 4. Configuration
+- **Configuration Classes**: Dataclasses for settings
+- **Required Parameters**: Must-have settings
+- **Optional Parameters**: Optional settings with defaults
+- **Validation**: Parameter validation rules
+
+### 5. Error Handling
+- **Custom Exceptions**: Module-specific errors
+- **Error Codes**: Standardized error codes
+- **Recovery Strategies**: Error recovery approaches
+
+### 6. Examples
+- **Basic Usage**: Simple usage examples
+- **Advanced Usage**: Complex scenarios
+- **Integration Examples**: Multi-module usage

 ## 🔧 Implementation Guidelines

@@ -79,12 +155,12 @@ This document contains comprehensive API design documentation for all core modul
 2. **Set Up Development Environment**
   ```bash
   # Clone the repository
-   git clone https://github.com/vllm-project/vllm-omni.git
+   git clone https://github.com/hsliuustc0106/vllm-omni.git
   cd vllm-omni
-
+   
   # Install dependencies
-   pip install -e ".[dev]"
-
+   pip install -r requirements-dev.txt
+   
   # Set up pre-commit hooks
   pre-commit install
   ```
@@ -113,12 +189,28 @@ This document contains comprehensive API design documentation for all core modul
   - Update documentation
   - Submit pull requests

+## 📖 Additional Resources
+
+### Architecture Overview
+- [vLLM-omni Architecture Design](../architecture/vLLM-omni%20arch%20design%20doc.md)
+- [Component Categorization](../../ARCHITECTURE_CATEGORIZATION.md)
+
+### Development Guides
+- [Development Setup](../../README.md#development)
+- [Testing Guidelines](../../tests/README.md)
+- [Contributing Guidelines](../../CONTRIBUTING.md)
+
+### Examples
+- [Basic Examples](../../examples/basic/)
+- [Advanced Examples](../../examples/advanced/)
+- [Multimodal Examples](../../examples/multimodal/)
+
 ## 🤝 Contributing

 We welcome contributions to improve the API documentation and implementation. Please:

 1. **Follow the Template Structure**
-   - Use the [api template](https://github.com/vllm-project/vllm-omni/docs/contributing/design_documents/api_design_template.md) as a guide
+   - Use the master template as a guide
   - Maintain consistency across modules
   - Include all required sections

@@ -1,68 +0,0 @@
-# Summary
-
-## Entry Points
-
-Main entry points for vLLM-omni inference and serving.
-
- [vllm_omni.entrypoints.omni_llm.OmniLLM][]
- [vllm_omni.entrypoints.omni_llm.OmniStageLLM][]
- [vllm_omni.entrypoints.omni_stage.OmniStage][]
- [vllm_omni.entrypoints.cli.serve.OmniServeCommand][]
-
-## Inputs
-
-Input data structures for multi-modal inputs.
-
- [vllm_omni.inputs.data.OmniTokenInputs][]
- [vllm_omni.inputs.data.OmniTokensPrompt][]
- [vllm_omni.inputs.data.OmniEmbedsPrompt][]
- [vllm_omni.inputs.parse.parse_singleton_prompt_omni][]
- [vllm_omni.inputs.preprocess.OmniInputPreprocessor][]
-
-## Engine
-
-Engine classes for offline and online inference.
-
- [vllm_omni.engine.arg_utils.OmniEngineArgs][]
- [vllm_omni.engine.processor.OmniProcessor][]
- [vllm_omni.engine.output_processor.OmniRequestState][]
- [vllm_omni.engine.output_processor.MultimodalOutputProcessor][]
- [vllm_omni.engine.PromptEmbedsPayload][]
- [vllm_omni.engine.AdditionalInformationPayload][]
- [vllm_omni.engine.OmniEngineCoreRequest][]
-
-## Core
-
-Core scheduling and caching components.
-
- [vllm_omni.core.sched.scheduler.OmniScheduler][]
- [vllm_omni.core.sched.diffusion_scheduler.DiffusionScheduler][]
- [vllm_omni.core.sched.output.OmniNewRequestData][]
- [vllm_omni.core.dit_cache_manager.DiTCacheManager][]
-
-## Model Executor
-
-Model execution components.
-
- [vllm_omni.model_executor.models.qwen2_5_omni.Qwen2_5OmniForConditionalGeneration][]
- [vllm_omni.model_executor.models.qwen2_5_omni_talker.Qwen2_5OmniTalkerForConditionalGeneration][]
- [vllm_omni.model_executor.models.qwen2_5_omni_thinker.Qwen2_5OmniThinkerForConditionalGeneration][]
- [vllm_omni.model_executor.models.qwen2_5_omni_token2wav.Qwen2_5OmniToken2WavForConditionalGenerationVLLM][]
- [vllm_omni.model_executor.layers.mrope.MRotaryEmbedding][]
- [vllm_omni.model_executor.models.registry.OmniModelRegistry][]
-
-## Configuration
-
-Configuration classes.
-
- [vllm_omni.config.OmniModelConfig][]
-
-## Workers
-
-Worker classes and model runners for distributed inference.
-
- [vllm_omni.worker.gpu_ar_worker.GPUARWorker][]
- [vllm_omni.worker.gpu_diffusion_worker.GPUDiffusionWorker][]
- [vllm_omni.worker.gpu_model_runner.OmniGPUModelRunner][]
- [vllm_omni.worker.gpu_ar_model_runner.GPUARModelRunner][]
- [vllm_omni.worker.gpu_diffusion_model_runner.GPUDiffusionModelRunner][]
@@ -1,4 +1,4 @@
-# Architecture Overview
+# vLLM-omni Architecture Design Document

 # Introduction

@@ -52,7 +52,7 @@ According to analysis for current popular open-source models, most of them have
 * **Extension, multiple AR+DiT (e.g. Qwen-omni)**  
  A new model for multimodal input and output. It designs the Thinker-Talker-Codec structure, which is a double-AR+DiT format.  
  ![][image5]  
-
+  


 # Architectural Overview
@@ -112,9 +112,9 @@ vLLM-omni will import core modules from existing vLLM framework and introduce se

 # Contact

-For any questions or feedback regarding this design document, please contact  
-Han Gao: [hgaoaf@connect.ust.hk](mailto:hgaoaf@connect.ust.hk),  
-Hongsheng Liu: [vllm-project@gmail.com](mailto:vllm-project@gmail.com)
+For any questions or feedback regarding this design document, please contact   
+Han Gao: [hgaoaf@connect.ust.hk](mailto:hgaoaf@connect.ust.hk),   
+Hongsheng Liu: [hsliuustc@gmail.com](mailto:hsliuustc@gmail.com)

 [image1]: <data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAALYAAAFwCAYAAAAVNoEZAABxw0lEQVR4XuydBVRcWb6ve90709OStnTScel4x93dIUaEuBJ3dzciRPDgEtzdCRpCBCIEEkIgQHCCBHe+t6uSvNvNnZ55r2/fng5zvrX+69TZp4qkan3nz2/vKup88sm/OWqXL3HxwnnOnj7JubOnuaB6jqtXLqJ28TxXRWmrX8Ha0gIzE0OuX1bFQFsTcxMDLMyMMDXS44bWVRxsLLG5aYKuxlVsLMxwc3bAWF8bNzdnHGwt8fb2wNXZHlcnW3HMDjsbC4Ju3cLO2hwvT3ecnexwsLcmIMAHHy83+X18PVxxtLMUt+1xcbTBw8URRwdr+f1tLM1wd3PB08NZlAs24ue4uTrKf0bD5yfxb8qWDWvZtnE9W0Vt37yBvTu3cWDPTlE7OHviCBfPnsTEwECIfg7Vk0e5fkkVoxtamBneQF1NlSsXTmMrZJaN6elcF7LexFaIZ6SngbOjvfy2l5DUSwhoa2mK9U1jfL09ibwTKaS3IPx2BA4ONnh7uuHr4yUX1d/Pi0B/X1xd7PHz8cTT3UUc85SfIMHBQUJ4a3EfH/mJ4OsjThoXB/njPNydJbEl3rFisTKrli5i9fLFrF21jC0bVdi2aR07tqzn+KF9nD91HF1NDU4eO8CZIwe5fO4UBjoacpEvnDnG5fMnsDQ1Rk/rOje0rwmxLeQd3chAC0d7W6zMjfBwc8JHiGlmpIu5oS6hISHcibgtOrE99+4+wE50dR8hraeHm1zOW4G+BAb4yjtyYICf/FiAED0wMICw0DDxODvR8QNEBeLv7yO6txPeotN7e7lLYku8Y8akccycOhGlGVNRVprJsoXzWLlEmbUrl7Jv5xbOHD8sBD/ApnWrOLZvN2ePH0FDRJVronMf3rdDiH1aiK4p9s+JE0AWS6xE59bA2EAHO6ub3DTRw1UI7Ovjg/oVEWV01Llz567ouH6i+wbz8FEMtiKaBPr7yzuvn6+nED9I3rH9/bwJDgqUSy3r1OHh4YSEBOPu7io6/W1CQ0PFSeAvxHeTd3B/X19JbIl3TB4zjOnjRzJz8jiUFCazYPZ0li1QYrXo4nt3bObE4QPs3LIRlRWLObh7h9jfj5rqGdGtj7N/11Yh9hk0r14W+yfQ0bgiZLZC8/pljPR1ROwwwdxYF2d7GyGfN1cunkJfiB0h4oe3lweRkfeIjn4iF9vf1wcnkbX9fL0ICb4ll1kutrgtkzdEyB4ZGUlQ0C35Y+/cuSPv3oHimOxkkHX2W6KjN3x+Ev+mjBzQmzGD+zJxxCAmjxrC9AmjUBZyL1dWEkJvEF17G6tEB1++aB47Nq0Xsm+Vy71PbHdv2yTP4LL9Q3t3onVdTYhsyCXV02LyqCuyuS762texlk0oXV05e/IQetrqBInuK5v8RUc/JiLiDvYiisgmf7Y2N0UssZJv1a9fFQJ7yiUPkXXrsBAePHiAn+jMsghy//590dlDRRTxk8cW2VhoWLAktsQ7Bv7UleH9egi5+8kFl3Xw+TOnsmTBLDGpXMsOMaFcunCuiChKbF27ml1CbtnEcvt6FbnY504eE517uxB+CxpXLwlxNbhwXjbhvCGXWFOMmZsZibztwMmj+7khsniA6Kwurs4iSoi87OKCuuj4urqa3DQ3xdHBVt6pXZwd0NJQJ1AmbGiw6NC3xYkQLbK6lxA6WC55yP8VWxZrbhEWHiKJLfGOPl06MKhnVwb06Cy2XZg0aqiIJJNYNHcG61cvl2dtZXF78byZrF22iHUrl7B53WqRwxcI6dfLO/WW9Wvk3f3i2VOonj7BeRFT9HW0ULtwRoh/FIMb2hgZ6otMLuvql9klos3COYri3xrOiEH96dujK907d+Snrp3o37c3Gzdu4MKFCxjqG7Bn9y7uRt6Vd+gHUQ9xFsKHiMmnbEy29ReRRbaaEhYWJo8qDZ+fxL8pfbt2pF+3TqJ+ZHCvLiKODGX2tAly8VRWLJFPJBcoKcpFX6E8j9WL300slyrPkXd0WdfeIE4A2UqKbHlQtpJy7tQxtNWvckZILYspOprX0dFSZ8+2DezcqMKsCaOZOGwQvTp1oHPbVnRo2YL2rZrz/TdN+O6rJrRp8YMQvA+q5y9w9NARbgUFEf3wEffuR2FrZyUXOiIiQnTyEPmkUSa2LNLcu/dAElviHb06taF357bIBB/cqzNjh/YXOXs0cxWnsWjebPlKyVwRTebNmILyLAWWzJ3NMuW5LJijIO/o61YvY/WShWxUWcmB3dvZLgQ/emAvZ04cZc/2zULmTaiKuLJv22ZmjxvJqH496dK6pagW9Gnfhp7tWtH5h+9p//23tPruG1qI+v6br/n2qy/47K9/odn337FGRQUHJ2fCbt/GyFhPxI4Q+eRRJrivr7d8AvngQZTo6tGS2BLv6P6jELtLO7nY/bv/yPCBPZkiBJylMIV5QuS5M6czR3EKStMnMU8m+5yZLFGazdwZU1mzfAmrly2UTyzXr1omjyMb1qxgn8jch/fvYdsGFXZsXMMusV0i7j+6V1eGdm3PGBF5pvXtztxBPZjetyujxL8/VPw/erRuzo9C8pZNheBNv+b7r7/iy88/o2PHjixcspTbtyO4IbK4bMnvzp13KySyzB0cIptYRktiS/wXnzdpzpdNmtGtQyt+6tiK/j06MXbEYCaPG8WMKROYOWUc0yaOY6ro4orittKMacxWmMysaeNZtmAei+fPEZ19FquWLmSdkHv5ovnyzL1dxJQ1Qvr1ohaOG8rxJTO4ukYJ/a1LODprDEsGdmdkux8Y2KopHb/+knZffk6LJp/xvRD5e3G7+Zdf0PKbr2gtJG8jZP/uqy9Zu24tqmdOEn77jrxjy96wkb1BExERyd2794m4c1cSW+Idn33ZlL99IaJA6x/o1KaZ6N7tGTmkP+NGDpXLPG38KLnkk8eNYPqkMXKpZ0ydIKQfK2LJHHkHV1aaIYReIM/jMtFlgss691KRzeeKn6G7YwVup3fgr3YAw83KLO/VkbX9OrNhYBfmd23JrC6tmNT+B7p89zVdRafu1uwbOjf9ivbffUUHsS/r4h3F/69Fs2YsWbhAZOn78owdGBgol1vWyWWTyYg7EZLYEu9o/cVXrO71E6aL53Nx1mT6iLw9sE9Phg/qx/jhQ+Q1doTYjhryXu6xTJNtx4scrjCVOSKiKImoIsvispJ19MULlEQGn8GsYf05vWI2+aGOZHqb4LJvNY+v7iP2+hEeXdpL2NF1uG9V5ti4nuwb1pUlPVoz88cWTGzblHGihrVuypA23zOsfXOGCPE7t/qBZt9+g7qmFmFh4XKxZXlb9kbPbdG1ZZ87afj8JP5NGd2mFfuHD8BCeQ43F8/hzMxxzB3al5H9fmJo/z4MH9CXEYMHMEpMKscNFxFl9AjGjxjERNHRFSbIJB/N9MljmDV9MjOnTUJh8nhmT5/CxNHDMNu/luI77pRGB5Dhb0t1zB2qnoRREuxEnpsx989sI3T/apzXKXBz6WRMl01Bf9F4VKcP5uj4Pmwd1p21g7ui3Lsj83p1oEerZrQV3fv7pt9y4uRxbgUHERIWRlh4BMFC9FvBoZLYEu/YObgfp0YO4ZLowtemjhViKXB1wVT2KIxBccRAxorOPVgIPnRQX8YMHSjv4GPeSz5p7EgmjhkhjywyoadPkmXxMfJMPmHoAOJstKhLjKLiaTiVCQ+pSY6jJj6K0jA3CjxNSdI/S4L2MR5c2sW9CzsIOqKC156lGC2bivr8MRye0I+dI3uyelBXlvXvxMA2zWnb7Ds+//QvjBkziiAxaQwJDRWTx3D5kqC/9Ja6xAcU27Rkcgvx67/59yzo2JrDo/thvVYZj51rCT20leCDm7DfvALN5XNRHj2UcQP7MKRPV9HJe4suLuvkQvZRw+SCTxg9XH57pDgZji6bSXVcJHWZidS+SaMuN43alGdysasfBlER6UNpsCNv/azJctQh1eoacdrHuX9hF67bFmK+SpFz04dwZHw/to3sxfqh3ZgssniXZl/LlwT/+p//ia+vn5g8+uPt7SX/YJSrq4sktsQ7un39Fd2/+ZoB337NpJZNWdezI/pKY3FSmU+o2mVCjc3xd/LCw9kHbSNrDl/QQFFpAaNHDGNov3dZfIzo3rLJ5tiR7/L4zJGDCBQ5ui4tntr8TOpF1WWnUJsUQ+2z+1RHBVF1L4DSUFeKAuzJdjEkzUaT+BtniLq8F+ftypitVOCsEPvoxH7sHt2XzUN7MLtHGwa0aEpLIfYXf/sMS0sLPDw8cHR0xN7ORl4Nn5/Evymd2rXgxzY/0EF0wj5dO3NE2xTj5CLMC+rQL6zDtKQe+/I6rEvrMS6owfBNFQY5leimlnLI0pOl2/aLDN6X0UPede9TG1dxS/sMiO5cHO5DcYgHed4OZNkZURbqSfWDYKrvBlN+25/iIDcK/Rx542JEpp0uTzWPEXl+Jw6b52O8fDrnFYZxcupg9o7pK7p2Dxb3aYdil5a0+eYrvvnicw4e3I/lTQsM9G+gr6eL3g1dSWyJd3T44Rt+bN2C+bsOc9w7ApMiuFkMN/JrsSyuwbm8FuuiWkzF/s3Caizf1mCZX4VpbgV6yW+5/iCRpVv3MkFMJBdMn0BehDf50SGQLaLH6xfUy+JHVCiVYZ6UeltRGexO9T2xHxFI0S2RtX0cyHMxI9P2Bo9Flw8/vQ3bjfMwWDqVc0LsE1MGsXd0H7aP6M6SPu2Z3bUVbb9pwjdffsnKlcvQ1rjO1SuXuHZVVpclsf8gNomSvdgfKuaXh//1qD3O4fLrKjRya9HJrUE3uxqr4mrcy2vwLKvDMKsKSyG0s9i3Lxad+20txqJj3xRlJeQ2yyhBNy6Dq0GPWLN8Kbn3g6jNEdEjL5uHphZcO3QRNyNHEnyCqY2NpfbFC2rjnlBxL0x0clkMuUmSmQZPdS9wR3U/t4Line truncated

@@ -0,0 +1,353 @@
+# vLLM-omni Implementation Architecture
+
+## 1. Package Structure
+
+```
+vllm_omni/
+├── __init__.py                 # Main package exports
+├── config/                     # Configuration management
+│   ├── __init__.py
+│   ├── stage_config.py        # OmniStageConfig implementation
+│   ├── dit_config.py          # DiT-specific configurations
+│   └── cache_config.py        # Caching configurations
+├── core/                       # Core processing components
+│   ├── __init__.py
+│   ├── stage_manager.py       # Multi-stage orchestration
+│   ├── dit_cache_manager.py   # DiT caching system
+│   └── sched/                 # Schedulers
+│       ├── __init__.py
+│       ├── scheduler.py       # Base scheduler interface
+
+├── engine/                     # Engine components
+│   ├── __init__.py
+│   ├── processor.py           # Output processing
+│   └── output_processor.py    # Multimodal output handling
+├── executor/                   # Executor implementations
+│   ├── __init__.py
+│   ├── base_executor.py       # Base executor interface
+│   └── diffusers_executor.py  # Diffusers pipeline executor
+├── model_executor/            # Model execution components
+│   ├── __init__.py
+├── worker/                     # Worker implementations
+│   ├── __init__.py
+│   ├── gpu_diffusion_model_runner.py  # Loads and runs diffusers pipelines
+│   └── gpu_diffusion_worker.py        # Worker facade around the diffusion runner
+├── entrypoints/               # Entry points and CLI
+│   ├── __init__.py
+│   ├── omni.py               # OmniServeCommand
+│   ├── omni_llm.py            # OmniLLM and AsyncOmniLLM
+│   └── cli/                  # CLI integration
+│       ├── __init__.py
+│       └── main.py           # CLI main entry point
+├── request.py                 # Request handling
+├── dit_cache_interface.py     # DiT cache interface
+└── utils/                     # Utility functions
+    ├── __init__.py
+    ├── multimodal.py         # Multimodal utilities
+    └── vae.py                # VAE utilities for image processing
+```
+
+## 2. Core Module Dependencies
+
+### 2.1 vLLM Integration Points
+
+```python
+# Key vLLM imports and extensions
+from vllm.v1.engine.llm_engine import LLMEngine
+from vllm.v1.engine.async_llm import AsyncLLM
+from vllm.v1.core.sched.scheduler import SchedulerInterface
+from vllm.v1.executor.multiproc_executor import MultiprocExecutor
+from vllm.v1.worker.gpu_worker import Worker as GPUWorker
+from vllm.v1.worker.gpu_model_runner import GPUModelRunner
+from vllm.v1.outputs import ModelRunnerOutput
+```
+
+### 2.2 Internal Dependencies
+
+```python
+# Internal module dependencies
+vllm_omni.config → vllm_omni.core
+vllm_omni.core → vllm_omni.engine, vllm_omni.executor, vllm_omni.worker
+vllm_omni.engine → vllm_omni.model_executor
+vllm_omni.entrypoints → vllm_omni.core
+```
+
+## 3. Configuration System
+
+### 3.1 Stage Configuration
+
+```python
+# vllm_omni/config/stage_config.py
+@dataclass
+class OmniStageConfig:
+    stage_id: int
+    engine_type: Literal["AR", "DiT"]
+    model_path: str
+    input_modalities: List[str]
+    output_modalities: List[str]
+    vllm_config: Optional[VllmConfig] = None
+    dit_config: Optional[DiTConfig] = None
+    executor_class: Optional[type[Executor]] = None
+    stage_output: Optional[Any] = None
+```
+
+### 3.2 DiT Configuration
+
+```python
+# vllm_omni/config/stage_config.py
+@dataclass
+class DiTConfig:
+    model_config: Optional[ModelConfig] = None
+    scheduler_config: Optional[Any] = None
+    device_config: Optional[DeviceConfig] = None
+    load_config: Optional[LoadConfig] = None
+    compilation_config: Optional[CompilationConfig] = None
+    dit_cache_config: Optional[DiTCacheConfig] = None
+    num_inference_steps: int
+    guidance_scale: float = 7.5
+    use_diffusers: bool = False
+    diffusers_pipeline: Optional[str] = None
+    height: int = 512
+    width: int = 512
+```
+
+### 3.3 Cache Configuration
+
+```python
+# vllm_omni/config/cache_config.py
+@dataclass
+class DiTCacheConfig:
+    cache_tensors: List[DiTCacheTensor]
+    max_cache_size: int
+    cache_strategy: str = "fifo"
+    enable_optimization: bool = True
+```
+
+## 4. Core Implementation Details
+
+### 4.1 OmniLLM Implementation
+
+```python
+# vllm_omni/core/omni_llm.py
+class OmniLLM(LLM):
+    def __init__(self, stage_configs: List[OmniStageConfig]):
+        super().__init__()
+        self.stage_configs = stage_configs
+        self.engine_list: List[LLMEngine] = []
+        self.output_processor = MultimodalOutputProcessor()
+        self._initialize_stage_engines()
+    
+    def _initialize_stage_engines(self) -> None:
+        """Initialize LLMEngine instances for each stage"""
+        for stage_config in self.stage_configs:
+            if stage_config.engine_type == "AR":
+                engine = self._create_ar_engine(stage_config)
+            elif stage_config.engine_type == "DiT":
+                engine = self._create_dit_engine(stage_config)
+            self.engine_list.append(engine)
+    
+    def generate(self, stage_args_list: List[Dict], **kwargs) -> List[RequestOutput]:
+        """Main generation interface - orchestrates multi-stage processing"""
+        current_output = None
+        
+        for i, (stage_config, stage_args) in enumerate(zip(self.stage_configs, stage_args_list)):
+            stage_engine = self.engine_list[i]
+            
+            # Prepare input for this stage
+            processed_input = self._process_stage_inputs(stage_config, stage_args, current_output)
+            
+            # Execute stage
+            stage_output = self._execute_stage(stage_engine, processed_input)
+            
+            # Update for next stage
+            current_output = stage_output
+            stage_config.stage_output = stage_output
+        
+        # Process final output
+        return self.output_processor.process_output(current_output)
+```
+
+### 4.2 AsyncOmniLLM Implementation
+
+```python
+# vllm_omni/core/omni_llm.py
+class AsyncOmniLLM(AsyncLLM):
+    def __init__(self, stage_configs: List[OmniStageConfig]):
+        super().__init__()
+        self.stage_configs = stage_configs
+        self.async_engine_list: List[AsyncLLM] = []
+        self.output_processor = MultimodalOutputProcessor()
+        self._initialize_async_stage_engines()
+    
+    async def generate_async(self, stage_args_list: List[Dict], **kwargs) -> List[RequestOutput]:
+        """Async generation interface"""
+        # Similar to OmniLLM but with async/await patterns
+        pass
+``
+
+### 4.3 Model Runners
+
+```python
+# vllm_omni/model_executor/dit_model_runner.py
+class OmniDiffusionModelRunner(GPUModelRunner):
+    def execute_model(
+        self,
+        scheduler_output: "SchedulerOutput",
+        intermediate_tensors: Optional[IntermediateTensors] = None,
+    ) -> Union[ModelRunnerOutput, IntermediateTensors]:
+        # DiT model execution logic
+        return ModelRunnerOutput(
+            req_ids=[...],
+            req_id_to_index={...},
+            sampled_token_ids=[],
+            spec_token_ids=None,
+            logprobs=None,
+            prompt_logprobs_dict={},
+            pooler_output=[tensor, ...],  # DiT output tensors
+            kv_connector_output=None,
+            num_nans_in_logits=None,
+        )
+
+# vllm_omni/model_executor/ar_model_runner.py
+class OmniARModelRunner(GPUModelRunner):
+    def execute_model(
+        self,
+        scheduler_output: "SchedulerOutput",
+        intermediate_tensors: Optional[IntermediateTensors] = None,
+    ) -> Union[ModelRunnerOutput, IntermediateTensors]:
+        # AR model execution with hidden state output
+        return ModelRunnerOutput(
+            req_ids=[...],
+            req_id_to_index={...},
+            sampled_token_ids=[],
+            spec_token_ids=None,
+            logprobs=None,
+            prompt_logprobs_dict={},
+            pooler_output=[hidden_states, ...],  # Hidden states
+            kv_connector_output=None,
+            num_nans_in_logits=None,
+        )
+```
+
+### 4.5 Output Processing
+
+```python
+# vllm_omni/engine/output_processor.py
+class MultimodalOutputProcessor(OutputProcessor):
+    def __init__(self):
+        self.output_handlers: Dict[str, Callable] = {
+            "image": self._process_image_output,
+            "text+image": self._process_text_image_output,
+            "latents": self._process_latents_output,
+            "text": self._process_text_output,
+        }
+    
+    def process_outputs(self, engine_core_outputs: List[EngineCoreOutput], ...):
+        """Process multimodal outputs based on type"""
+        for engine_core_output in engine_core_outputs:
+            output_type = self._detect_output_type(engine_core_output)
+            handler = self.output_handlers.get(output_type, self._process_pooling_output)
+            handler(engine_core_output)
+```
+
+## 5. CLI Integration
+
+### 5.1 Entry Point Override
+
+```python
+# vllm_omni/entrypoints/cli/main.py
+def main():
+    """Main CLI entry point that intercepts vLLM commands"""
+    if "--omni" in sys.argv:
+        omni_args = [arg for arg in sys.argv[1:] if arg != "--omni"]
+        omni_serve = OmniServeCommand()
+        omni_serve.run(omni_args)
+    else:
+        from vllm.entrypoints.cli.main import main as vllm_main
+        vllm_main()
+```
+
+### 5.2 Package Configuration
+
+```toml
+# pyproject.toml updates
+[project.scripts]
+vllm = "vllm_omni.entrypoints.cli.main:main"
+vllm-omni = "vllm_omni.entrypoints.cli.main:main"
+
+[project.entry-points."vllm.plugins"]
+omni = "vllm_omni.plugin:OmniPlugin"
+```
+
+## 6. Testing Strategy
+
+### 6.1 Unit Tests
+- Individual component testing
+- Mock vLLM dependencies
+- Configuration validation
+
+### 6.2 Integration Tests
+- End-to-end pipeline testing
+- vLLM compatibility testing
+- Multi-stage processing validation
+
+### 6.3 Performance Tests
+- Benchmarking against native vLLM
+- Memory usage profiling
+- Latency measurements
+
+## 7. Installation and Setup
+
+### 7.1 Package Installation
+```bash
+pip install vllm>=0.10.2
+pip install vllm-omni
+```
+
+### 7.2 Development Setup
+```bash
+git clone https://github.com/hsliuustc0106/vllm-omni
+cd vllm-omni
+pip install -e ".[dev]"
+```
+
+### 7.3 Usage
+```bash
+vllm serve Qwen/Qwen2.5-Omni-7B --omni --port 8000
+```
+
+## 8. Memory Management
+
+### 8.1 DiT Cache Management
+- Tensor caching for intermediate results
+- Memory pooling for efficient allocation
+- Cache eviction strategies
+
+### 8.2 Multi-Stage Memory
+- Inter-stage data passing optimization
+- Memory sharing between stages
+- Garbage collection optimization
+
+## 9. Error Handling
+
+### 9.1 Stage Failure Handling
+- Graceful degradation on stage failures
+- Error propagation and reporting
+- Recovery mechanisms
+
+### 9.2 vLLM Compatibility
+- Version compatibility checks
+- API change detection
+- Fallback mechanisms
+
+## 10. Monitoring and Logging
+
+### 10.1 Performance Metrics
+- Stage execution times
+- Memory usage per stage
+- Throughput measurements
+
+### 10.2 Debug Information
+- Request tracing across stages
+- Cache hit/miss ratios
+- Error logging and reporting
@@ -1,170 +0,0 @@
-# Contributing to vLLM-omni
-
-Thank you for your interest in contributing to vLLM-omni! This document provides guidelines and instructions for contributing.
-
-## Getting Started
-
-### Development Environment
-
-vLLM-omni uses `uv` as the environment manager, to create and manage Python environments. Please follow the documentation to install `uv`. After installing `uv`, you can create a new Python environment using the following commands:
-
-```bash
-uv venv --python 3.12 --seed
-source .venv/bin/activate
-```
-
-Install vLLM from source:
-
-```bash
-git clone https://github.com/vllm-project/vllm.git
-cd vllm
-git checkout v0.11.0
-VLLM_USE_PRECOMPILED=1 uv pip install -e .
-```
-
-Install vLLM-omni from source:
-
-```bash
-git clone https://github.com/vllm-project/vllm-omni.git
-cd vllm_omni
-uv pip install -e .
-```
-
-!!! tip
-    vLLM-omni is compatible with Python versions 3.10 to 3.12. However, we recommend developing with Python 3.12 to minimize the chance of your local environment clashing with our CI environment.
-
-### Linting
-
-vLLM-omni uses `pre-commit` to lint and format the codebase. See [pre-commit documentation](https://pre-commit.com/#usage) if `pre-commit` is new to you. Setting up `pre-commit` is as easy as:
-
-```bash
-uv pip install pre-commit
-pre-commit install
-```
-
-vLLM-omni's `pre-commit` hooks will now run automatically every time you commit.
-
-!!! tip
-    You can manually run the `pre-commit` hooks using:
-
-    ```bash
-    pre-commit run     # runs on staged files
-    pre-commit run -a  # runs on all files (short for --all-files)
-    ```
-
-### Documentation
-
-MkDocs is a fast, simple and downright gorgeous static site generator that's geared towards building project documentation. Documentation source files are written in Markdown, and configured with a single YAML configuration file, `mkdocs.yml`.
-
-Get started with:
-
-```bash
-uv pip install -e ".[docs]"
-```
-
-MkDocs comes with a built-in dev-server that lets you preview your documentation as you work on it. From the root of the repository, run:
-
-```bash
-mkdocs serve                           # with API ref (~10 minutes)
-API_AUTONAV_EXCLUDE=vllm_omni mkdocs serve  # API ref off (~15 seconds)
-```
-
-Once you see `Serving on http://127.0.0.1:8000/` in the logs, the live preview is ready! Open <http://127.0.0.1:8000/> in your browser to see it.
-
-For additional features and advanced configurations, refer to the:
-
- [MkDocs documentation](https://www.mkdocs.org/)
- [Material for MkDocs documentation](https://squidfunk.github.io/mkdocs-material/) (the MkDocs theme we use)
-
-### Testing
-
-vLLM-omni uses `pytest` to test the codebase.
-
-```bash
-# Install the test dependencies used in CI (CUDA only)
-uv pip install -e ".[dev]" --torch-backend=auto
-
-# Install some common test dependencies (hardware agnostic)
-uv pip install pytest pytest-asyncio
-
-# Run all tests
-pytest tests/
-
-# Run tests for a single test file with detailed output
-pytest -s -v tests/test_logger.py
-```
-
-!!! warning
-    Currently, not all unit tests pass when run on CPU platforms. If you don't have access to a GPU platform to run unit tests locally, rely on the continuous integration system to run the tests for now.
-
-## Issues
-
-If you encounter a bug or have a feature request, please search existing issues first to see if it has already been reported. If not, please file a new issue, providing as much relevant information as possible.
-
-!!! important
-    If you discover a security vulnerability, please report it by creating a GitHub issue with the `security` label.
-
-## Pull Requests & Code Reviews
-
-Thank you for your contribution to vLLM-omni! Before submitting the pull request, please ensure the PR meets the following criteria. This helps vLLM-omni maintain the code quality and improve the efficiency of the review process.
-
-### DCO and Signed-off-by
-
-When contributing changes to this project, you must agree to the [DCO](https://developercertificate.org/). Commits must include a `Signed-off-by:` header which certifies agreement with the terms of the DCO.
-
-Using `-s` with `git commit` will automatically add this header.
-
-!!! tip
-    You can enable automatic sign-off via your IDE:
-
-    - **PyCharm**: Click on the `Show Commit Options` icon to the right of the `Commit and Push...` button in the `Commit` window. It will bring up a `git` window where you can modify the `Author` and enable `Sign-off commit`.
-    - **VSCode**: Open the Settings editor and enable the `Git: Always Sign Off` (`git.alwaysSignOff`) field.
-
-### PR Title and Classification
-
-Only specific types of PRs will be reviewed. The PR title is prefixed appropriately to indicate the type of change. Please use one of the following:
-
- `[Bugfix]` for bug fixes.
- `[CI/Build]` for build or continuous integration improvements.
- `[Doc]` for documentation fixes and improvements.
- `[Model]` for adding a new model or improving an existing model. Model name should appear in the title.
- `[Frontend]` For changes on the vLLM-omni frontend (e.g., OpenAI API server, `OmniLLM` class, etc.)
- `[Kernel]` for changes affecting CUDA kernels or other compute kernels.
- `[Core]` for changes in the core vLLM-omni logic (e.g., `OmniProcessor`, `OmniScheduler`, etc.)
- `[Hardware][Vendor]` for hardware-specific changes. Vendor name should appear in the prefix, such as [Ascend] for Ascend NPUs.
- `[Misc]` for PRs that do not fit the above categories. Please use this sparingly.
-
-!!! note
-    If the PR spans more than one category, please include all relevant prefixes.
-
-### Code Quality
-
-The PR needs to meet the following code quality standards:
-
- We adhere to Google Python style guide and Google C++ style guide.
- Pass all linter checks.
- The code needs to be well-documented to ensure future contributors can easily understand the code.
- Include sufficient tests to ensure the project stays correct and robust. This includes both unit tests and integration tests.
- Please add documentation to `docs/` if the PR modifies the user-facing behaviors of vLLM-omni. It helps vLLM-omni users understand and utilize the new features or changes.
-
-### Notes for Large Changes
-
-Please keep the changes as concise as possible. For major architectural changes (>500 LOC excluding kernel/data/config/test), we would expect a GitHub issue (RFC) discussing the technical design and justification. Otherwise, we will tag it with `rfc-required` and might not go through the PR.
-
-### What to Expect for the Reviews
-
-The goal of the vLLM-omni team is to be a _transparent reviewing machine_. We would like to make the review process transparent and efficient and make sure no contributor feels confused or frustrated. However, the vLLM-omni team is small, so we need to prioritize some PRs over others. Here is what you can expect from the review process:
-
- After the PR is submitted, the PR will be assigned to a reviewer. Every reviewer will pick up the PRs based on their expertise and availability.
- After the PR is assigned, the reviewer will provide status updates every 2-3 days. If the PR is not reviewed within 7 days, please feel free to ping the reviewer or the vLLM-omni team.
- After the review, the reviewer will put an `action-required` label on the PR if there are changes required. The contributor should address the comments and ping the reviewer to re-review the PR.
- Please respond to all comments within a reasonable time frame. If a comment isn't clear or you disagree with a suggestion, feel free to ask for clarification or discuss the suggestion.
- Note that not all CI checks will be executed due to limited computational resources. The reviewer will add `ready` label to the PR when the PR is ready to merge or a full CI run is needed.
-
-## Additional Resources
-
- [Design Documents](design_documents/index.md) - Architecture and design documentation
-
-## Thank You
-
-Finally, thank you for taking the time to read these guidelines and for your interest in contributing to vLLM-omni. All of your contributions help make vLLM-omni a great tool and community for everyone!
@@ -1,12 +0,0 @@
-# Design Documents
-
-This section contains design documents and architecture specifications for vLLM-omni.
-
-## Architecture Documents
-
- [vLLM-omni Software Design Document](vllm_omni_design.md)
- [Architecture Overview](architecture_overview.md)
-
-## API Design Documentation
-
- [vLLM-omni API Documentation](api_design_doc.md)
@@ -1,509 +0,0 @@
-# Architecture Design Principles
-
-## Overview
-
-vLLM-omni is a multi-modality extension for vLLM that supports non-autoregressive structures and non-textual outputs. This document outlines the key software abstractions, APIs, and dependencies for the system, designed to maximize reuse of vLLM's proven architecture.
-
-## Architecture Principles
-
-1. **vLLM V1 Compatibility**: Built on vLLM's Engine V1 architecture with AsyncLLM and EngineCore patterns
-2. **Stage-based Processing**: Models are divided into multiple stages, each processed by different Engine Cores
-3. **Multiple Engine Core Support**: Each stage can use either AR Engine Core (reusing vLLM) or Diffusion Engine Core (new DiT support), or other new Cores
-4. **Worker Process Pattern**: Follows vLLM's multiprocess worker architecture for scalability
-5. **Extensibility**: Easy integration of new modalities, model architectures, and output formats
-
-## Key Data Flow
-
-API Server --> OmniLLM/AsyncOmniLLM (New, including multi engines) --> LLMEngine/AsyncLLM --> Engine Core
- --> Scheduler (New one for DiT) --> Executor (New one for diffusers) --> Worker (New one for DiT)
- --> ModelRunner (New one for AR hiddenstate, New one for DiT) --> RequestState --> OutputProcessoer (New one for final multimodal output)
-
-## Core Components (Ordered by Data Flow)
-
-### 1. Installation
-```bash
-pip install vllm
-pip install vllm-omni
-```
-
-### 2. Online inference launch (Entry Point)
-
-Keep consistency with the vllm main branch
-```bash
-vllm serve Qwen/Qwen2.5-Omni-7B --omni --port 8000
-```
-
-#### Design Architecture
-```mermaid
-graph TD
-    A[vllm serve model --omni] --> B[vLLM-omni CLI Wrapper]
-    B --> C{Detect --omni flag}
-    C -->|Yes| D[Parse OmniConfig]
-    C -->|No| E[Forward to vLLM CLI]
-    D --> F[Initialize AsyncOmniLLM]
-    F --> G[Start omni Server]
-    G --> H[Multi-stage Processing]
-    E --> I[Standard vLLM Pipeline]
-
-    subgraph "vLLM-omni Components"
-        F
-        G
-        H
-    end
-
-    subgraph "vLLM Components"
-        I
-    end
-```
-
-#### omni Serve Command Implementation
-File: vllm_omni/entrypoints/cli/main.py
-```python
-import sys
-import argparse
-from typing import List, Optional
-from vllm_omni.entrypoints.omni import OmniServeCommand
-
-def main():
-    """Main CLI entry point that intercepts vLLM commands"""
-    # Check if --omni flag is present
-    if "--omni" in sys.argv:
-        # Remove --omni flag and process with vLLM-omni
-        omni_args = [arg for arg in sys.argv[1:] if arg != "--omni"]
-        omni_serve = OmniServeCommand()
-        omni_serve.run(omni_args)
-    else:
-        # Forward to original vLLM CLI
-        from vllm.entrypoints.cli.main import main as vllm_main
-        vllm_main()
-
-if __name__ == "__main__":
-    main()
-```
-
-#### vLLM Plugin Integration
-File: vllm_omni/pyproject.toml
-```python
-[project.scripts]
-# Override vLLM's CLI entry point
-vllm = "vllm_omni.cli:main"
-
-# Add entry point for vLLM integration
-[project.entry-points."vllm.plugins"]
-omni = "vllm_omni.plugin:OmniPlugin"
-```
-
-### 3. Offline inference launch (Entry Point)
-Design an upper level class to incorporate multi Engines, each engine has a engine call.
-```python
-from vllm.entrypoints.llm import LLM
-from vllm.v1.engine.llm_engine import LLMEngine
-
-class OmniLLM(LLM):
-    """Extended LLM supporting multiple engines and stage-based processing"""
-
-    def __init__(self, stage_configs: List[StageConfig]):
-        super().__init__()
-        self.stage_configs = stage_configs
-        self.engine_list = []  # List of AsyncLLM instances for each stage
-        self.output_processor = MultimodalOutputProcessor()
-        self._initialize_stage_engines()
-
-    def _initialize_stage_engines(self) -> None:
-        """Initialize LLMEngine instances for each stage"""
-        for stage_config in self.stage_configs:
-            stage_llm = LLMEngine.from_vllm_config(
-                    vllm_config=stage_config.vllm_config,
-                    executor_class=self.executor_class,
-                    log_stats=self.log_stats
-                )
-            self.engine_list.append(stage_llm)
-
-    def generate(
-        self,
-        stage_args_list: List[stage_args],
-        use_tqdm: Union[bool, Callable[..., tqdm]] = True,
-        lora_request: Optional[Union[list[LoRARequest], LoRARequest]] = None,
-        priority: Optional[list[int]] = None,
-    ) -> list[RequestOutput]:
-        """Main generation interface - orchestrates multi-stage processing"""
-
-        # Process through each stage sequentially
-        for i, stage_config in enumerate(self.stage_configs):
-            stage_engine = self.engine_list[i]
-
-            # Prepare input for this stage
-            stage_args = stage_args_list[i]
-            prompt_str, engine_request, tokenization_kwargs = self._process_stage_inputs(stage_config, **stage_args)
-
-            # Add inputs to Engine
-            stage_engine.add_request(requesy_id, prompt_str, tokenization_kwargs)
-            # Run Engine
-            stage_output = stage_engine.step()
-
-            # Update stage input and output of each stage for later usage
-            self._update_stage_io(stage_output, stage_config)
-
-
-        # Process final output
-        final_output = self.output_processor.process_output(
-            self.stage_configs[-1].stage_output
-        )
-        return final_output
-
-    def _process_stage_inputs(stage_config, **stage_args) -> Any:
-        """Prepare input for specific stage"""
-        if stage_config.engine_type == "AR":
-            return self._process_ar_inputs(**stage_args)
-        elif stage_config.engine_type == "DiT":
-            return self._process_dit_inputs(**stage_args)
-        else
-            raise NotImplementedError
-
-    def _process_dit_inputs(**stage_args)-> Any:
-        image_latent = self.vae.encode(**stage_args)
-
-        return image_latent
-```
-
-### 4. Stage config for setting of different model stages
-```python
-@dataclass
-class StageConfig:
-    """Configuration for a processing stage"""
-    stage_id: int
-    engine_type: str  # "AR" or "DiT"
-    model_path: str
-    input_modalities: List[str]
-    output_modalities: List[str]
-    vllm_config: Optional[VllmConfig] = None  # For engine config of corresponding stage
-    executor_class: type[Executor]  # For execute class config of corresponding stage
-    dit_config: Optional[DiTConfig] = None  # For diffusion stages
-    cache_config: Optional[DiTCacheConfig] = None
-```
-For AR stage, the setting is:
-```python
-from vllm.v1.core.sched.scheduler import Scheduler
-from vllm.v1.executor.multiproc_executor import MultiprocExecutor
-ar_stage_config = StageConfig()
-ar_stage_config.vllm_config.scheduler_config.scheduler_cls = Scheduler # original vllm scheduler for AR
-ar_stage_config.executor_class = MultiprocExecutor # original vllm executor for AR
-```
-For DiT stage, the setting is:
-```python
-from vllm.v1.executor.multiproc_executor import MultiprocExecutor
-dit_stage_config = StageConfig()
-dit_stage_config.vllm_config.scheduler_config.scheduler_cls = DiffusionScheduler # New scheduler for DiT
-
-# For diffusion models without using diffusers
-dit_stage_config.executor_class = MultiprocExecutor
-
-# For diffusion models using diffusers
-dit_stage_config.executor_class = DiffusersPipelineExecutor
-```
-
-### 5. Online inference main class
-
-Similar to OmniLLM in offline inference, add some asynchronous processing, referring to AsyncLLM
-```python
-from vllm.v1.engine.async_llm import AsyncLLM
-
-
-class AsyncOmniLLM(AsyncLLM):
-    """Extended AsyncLLM supporting multiple engines and stage-based processing"""
-
-    def __init__(self, stage_configs: List[StageConfig]):
-        super().__init__()
-```
-
-### 6. Engine Core
-No need to change. The specific executor, scheduler and output type can be transferred into it with new configs.
-```python
-# class: EngineCore
-# EngineCore.step (simplified)
-class EngineCore:
-    def step(self):
-        scheduler_output = scheduler.schedule()
-        model_output = executor.execute_model(scheduler_output)
-        engine_outputs = scheduler.update_from_output(
-            scheduler_output, model_output
-        )
-        return engine_outputs
-```
-
-### 7. Scheduler
-Create a new Diffusion Scheduler for DiT, which is inherited from original vllm AR scheduler. At first, no complicated strategy.
-Just process the request first in first out. Then create a class of DiT Cache Manager for optimization in the future.
-```python
-from vllm.v1.core.sched.scheduler import Scheduler
-class DiffusionScheduler(Scheduler):
-
-    def __init__(
-        self,
-        vllm_config: VllmConfig,
-        kv_cache_config: KVCacheConfig,
-        dit_cache_config: DiTCacheConfig,
-        structured_output_manager: StructuredOutputManager,
-        mm_registry: MultiModalRegistry = MULTIMODAL_REGISTRY,
-        include_finished_set: bool = False,
-        log_stats: bool = False,
-    ) -> None:
-
-    self.dit_cache_manager = DiTCacheManager(dit_cache_config)
-```
-For DiT Cache Manager (Can refer to xDiT):
-```python
-class DiTCacheManager:
-    """Manages DiT-specific caching"""
-
-    def __init__(self, config: DiTCacheConfig):
-        self.cache_tensors: Dict[str, torch.Tensor] = {}
-        self.cache_groups: List[DiTCacheTensor] = config.dit_cache_tensors
-
-    def allocate_cache(self, request_id: str, size: int) -> torch.Tensor
-    def get_cache(self, request_id: str) -> Optional[torch.Tensor]
-    def release_cache(self, request_id: str) -> None
-    def clear_expired_cache(self) -> None
-```
-
-### 8. Executor for DiT without diffusers
-
-No need to change. Just import original AR executor.
-
-### 9. Diffusers Pipeline Executor (no worker)
-
-A single-process executor that directly runs the Diffusers pipeline without spawning workers or using RPC. Interfaces remain identical to `Executor` so the EngineCore loop is unchanged.
-
-#### Function map（Pipeline Executor）
-
-##### 1) Inherited and overridden
-```python
-from concurrent.futures import Future
-from typing import Optional, Union, Callable, Any
-import torch
-import torch.nn as nn
-from vllm.v1.executor.abstract import Executor
-from vllm.v1.kv_cache_interface import KVCacheConfig, KVCacheSpec
-from vllm.v1.outputs import ModelRunnerOutput
-from vllm.tasks import SupportedTask
-from diffusers import DiffusionPipeline
-
-class DiffusersPipelineExecutor(Executor):
-    supports_pp: bool = False  # Single-process, no TP/PP/DP
-
-    def _init_executor(self) -> None:
-        # Called by ExecutorBase.__init__
-        self._failure_callback: Optional[Callable[[], None]] = None
-        self._device = self._resolve_device()
-        self._dtype = self._resolve_dtype()
-        self._pipeline = self._build_pipeline(device=self._device, dtype=self._dtype)
-        self._profiler = None
-        self._is_failed = False
-        self.is_sleeping = False
-        self.sleeping_tags: set[str] = set()
-
-    # major functions to build/run diffusers pipeline
-    def _build_pipeline(self, device:torch.device, dtype:torch.dtype)->DiffusionPipeline:
-        model_name = "Qwen/Qwen-Image"
-
-        self.pipe = DiffusionPipeline.from_pretrained(model_name, torch_dtype=dtype)
-        self.pipe = pipe.to(device)
-
-    def _run_pipeline(self, scheduler_output) -> ModelRunnerOutput: ...
-        positive_magic = {
-            "en": ", Ultra HD, 4K, cinematic composition.", # for english prompt
-            "zh": ", 超清，4K，电影级构图." # for chinese prompt
-        }
-
-        # Generate image
-        prompt_embeds = self._get_and_process_prompt_embeds(scheduler_output, positive_magic)
-        negative_prompt_embeds = self.pipe.embed_prompt(" ")
-
-        width, height = aspect_ratios["16:9"]
-
-        image = pipe(
-            prompt_embeds=prompt_embeds,
-            negative_prompt_embeds=negative_prompt_embeds,
-            width=width,
-            height=height,
-            num_inference_steps=50,
-            true_cfg_scale=4.0,
-            generator=torch.Generator(device="cuda").manual_seed(42)
-        ).images[0]
-
-        output = self.wrap_image_as_ModelRunnerOutput(image)
-        return output
-
-    # ---- Internal helpers (implementation-specific, not public API) ----
-    def _resolve_device(self): ...
-    def _resolve_dtype(self): ...
-    def _get_model(self) -> nn.Module: ...
-    def _get_and_process_prompt_embeds(self, scheduler_output, positive_magic):
-        ...
-        #append the positive_magic to prompt and embed them to prompt embed tensors
-
-    # Functions related to workers should either raise NotImplementedError
-    # or return empty defaults in this no-worker executor.
-
-    def collective_rpc(self, method, timeout=None, args=(), kwargs=None) -> list[Any]:
-        # No workers in pipeline executor
-        raise NotImplementedError("No workers in DiffusersPipelineExecutor")
-
-    def initialize_from_config(self, kv_cache_configs: list[KVCacheConfig]) -> None:
-        return  # no-op (pipeline already built in _init_executor)
-
-    def register_failure_callback(self, callback):
-        self._failure_callback = callback
-
-    def determine_available_memory(self) -> list[int]:  # bytes
-        # Single device; return [available_bytes]. If CPU-only, return [0].
-        return [self._determine_available_bytes(self._device)]
-
-```
-
-### 10. Worker
-
-#### Inheritance strategy
-Prefer reusing the mature GPU Worker end-to-end. Worker class is selected by configuration (vllm_config). Do not add a new executor-specific worker binding. If customization is needed, override only `init_device` to construct the `DiffusionModelRunner`; all other behaviors (device init details, profiling, sleep/wake, PP/TP comms, execute path) remain from `vllm/v1/worker/gpu_worker.py::Worker`.
-
-
-Inherited and overridden
-```python
-# Optional: only if you need to plug a custom DiffusionModelRunner.
-from vllm.v1.worker.gpu_worker import Worker as GPUWorker
-from vllm.v1.worker.diffusion_model_runner import DiffusionModelRunner
-
-class Worker(GPUWorker):
-    def init_device(self) -> None:
-        #those related to device check and init
-        ...
-
-        self.model_runner = DiffusionModelRunner(self.vllm_config, ...)
-```
-
-### 11. Model Runner
-
-### Function map (Model Runner)
-#### 1) Inherited and overridden
-Those parts relied to the KV Cache will be omitted if we do not register the model to the vllm config. The engine core will view it as do not require KV Cache, and handle it properly
-
-Reuse `vllm/v1/outputs.py::ModelRunnerOutput`：
- DiffusionModelRunner: Use the `pooler_output=[Tensor,...]` to return multi modal tensors
- ARModelRunner: Use the `pooler_output=[Tensor,...]` to return hidden states.
-```python
-from typing import Optional, Union
-import torch
-from vllm.v1.worker.gpu_model_runner import GPUModelRunner
-from vllm.v1.outputs import ModelRunnerOutput
-
-
-class DiffusionModelRunner(GPUModelRunner):
-    @torch.inference_mode()
-    def execute_model(
-        self,
-        scheduler_output: "SchedulerOutput",
-        intermediate_tensors: Optional[IntermediateTensors] = None,
-    ) -> Union[ModelRunnerOutput, IntermediateTensors]:
-        ...
-        return ModelRunnerOutput(
-            req_ids=[...],
-            req_id_to_index={...},
-            sampled_token_ids=[],
-            spec_token_ids=None,
-            logprobs=None,
-            prompt_logprobs_dict={},
-            pooler_output=[Tensor,...],         # Return Hidden states
-            kv_connector_output=None,
-            num_nans_in_logits=None,
-        )# return multi modal tensors via pooler_output=[Tensor,...]
-
-
-class ARModelRunner(GPUModelRunner):
-    @torch.inference_mode()
-    def execute_model(
-        self,
-        scheduler_output: "SchedulerOutput",
-        intermediate_tensors: Optional[IntermediateTensors] = None,
-    ) -> Union[ModelRunnerOutput, IntermediateTensors]:
-        ...
-        return ModelRunnerOutput(
-            req_ids=[...],
-            req_id_to_index={...},
-            sampled_token_ids=[],
-            spec_token_ids=None,
-            logprobs=None,
-            prompt_logprobs_dict={},
-            pooler_output=[Tensor,...],         # Return Hidden states
-            kv_connector_output=None,
-            num_nans_in_logits=None,
-        )# return hidden states via pooler_output=[Tensor,...]
-```
-
-### 12. RequestState Role and Usage
-```bash
-# RequestState serves as the per-request state tracker in OutputProcessor:
-# - Maintains request-specific state (tokens, logprobs, detokenizer, etc.)
-# - Converts EngineCoreOutput → RequestOutput/PoolingRequestOutput
-# - Manages request lifecycle from registration to completion
-AsyncLLM.add_request()
-↓
-OutputProcessor.add_request()
-↓
-RequestState.from_new_request() → 创建请求状态
-
-AsyncLLM.__init__() / AsyncLLM.generate() / AsyncLLM.encode() →  创建一个Background loop 持续从EngineCore获取输出
-↓
-OutputProcessor.process_outputs() → 更新状态并处理输出
-↓
-RequestState.make_request_output() → 转换为最终输出,格式为RequestOutput或者PoolingRequestOutput
-↓
-RequestOutputCollector.put() → 推送到队列（AsyncLLM）
-```
-
-Need to add implementation for an existing method
-```python
-class RequestState:
-    def __init__(self, request_id: str, parent_req: Optional[ParentRequest],
-                 request_index: int, lora_name: Optional[str],
-                 output_kind: RequestOutputKind, prompt: Optional[str],
-                 prompt_token_ids: list[int], logprobs_processor: Optional[LogprobsProcessor],
-                 detokenizer: Optional[IncrementalDetokenizer],
-                 max_tokens_param: Optional[int], arrival_time: float,
-                 queue: Optional[RequestOutputCollector], log_stats: bool):
-
-    def _new_pooling_output(self, pooling_output: torch.Tensor) -> PoolingOutput:
-        """Create PoolingOutput for multimodal/pooling requests"""
-        return PoolingOutput(data=pooling_output)
-```
-
-
-### 13. OutputProcessor
-
-For hidden state output, original OutputProcessor already support.
-
-We only need to add new one for final multimodal output.
-```python
-class MultimodalOutputProcessor(OutputProcessor):
-    """Handles multimodal output processing"""
-
-    def __init__(self):
-        self.output_handlers: Dict[str, OutputProcessor] = {}
-
-    def process_outputs(self, engine_core_outputs: list[EngineCoreOutput], ...):
-        for engine_core_output in engine_core_outputs:
-            # Option 1: Use output_type field (if EngineCoreOutput is extended)
-            if engine_core_output.output_type == "image":
-                self._process_image_output(engine_core_output)
-            elif engine_core_output.output_type == "text+image":
-                self._process_text_image_output(engine_core_output)
-            elif engine_core_output.output_type == "latents":
-                self._process_latents_output(engine_core_output)
-            elif engine_core_output.output_type == "text":
-                self._process_text_output(engine_core_output)
-            else:
-                # Fallback: use existing pooling_output logic
-                if engine_core_output.pooling_output is not None:
-                    self._process_pooling_output(engine_core_output)
-                else:
-                    self._process_text_output(engine_core_output)
-```
@@ -1,32 +0,0 @@
-# Welcome to vLLM-omni
-
-**vLLM-omni** extends vLLM capabilities to support multi-modality model inference and serving, particularly focusing on non-autoregressive structures and non-textual outputs. It aims to support:
-
- **Multi-modal Models**: Text, image, video, audio, and sensor data processing
- **Non-autoregressive Architectures**: Diffusion Transformers (DiT) and other parallel generation models
- **Heterogeneous Outputs**: Beyond traditional text generation to multimodal outputs
-
-## <span class="twemoji">🎯</span> Key Features
-
-### Multi-Engine Support
-
- **Autoregressive Engine**: Traditional text generation with enhanced KV-caching
- **Diffusion Engine**: Support for DiT models and iterative generation
- **Hybrid Engine**: Combined AR+DiT processing pipelines
-
-### Modality Processing
-
- **Text**: Advanced tokenization and embedding generation
- **Image**: Vision encoder integration (CLIP, etc.)
- **Audio**: Speech processing and audio embedding
-
-## <span class="twemoji">🤖</span> Supported Models
-
- Qwen-omni (Thinker-Talker-Codec structure)
- HunyunaImage 3.0 (Ongoing)
- Qwen-Image (Ongoing)
-
-## <span class="twemoji">📚</span> Documentation Navigation
-
- To run open-source models on vLLM-Omni, we recommend starting with the [:material-code-tags: User Quide](user_guide/getting_started/quickstart.md)
- To develop and contribute to vLLM-Omni, we recommend starting with the [:material-tools: Developer Guide](contributing/README.md)
@@ -1,233 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM-omni project
-import itertools
-import logging
-from dataclasses import dataclass, field
-from pathlib import Path
-from typing import Literal
-
-import regex as re
-
-logger = logging.getLogger("mkdocs")
-
-ROOT_DIR = Path(__file__).parent.parent.parent.parent
-ROOT_DIR_RELATIVE = "../../../../.."
-EXAMPLE_DIR = ROOT_DIR / "examples"
-EXAMPLE_DOC_DIR = ROOT_DIR / "docs/user_guide/examples"
-
-
-def fix_case(text: str) -> str:
-    subs = {
-        "api": "API",
-        "cli": "CLI",
-        "cpu": "CPU",
-        "llm": "LLM",
-        "mae": "MAE",
-        "tpu": "TPU",
-        "gguf": "GGUF",
-        "lora": "LoRA",
-        "rlhf": "RLHF",
-        "vllm": "vLLM",
-        "omni": "Omni",
-        "openai": "OpenAI",
-        "lmcache": "LMCache",
-        "multilora": "MultiLoRA",
-        "mlpspeculator": "MLPSpeculator",
-        r"fp\d+": lambda x: x.group(0).upper(),  # e.g. fp16, fp32
-        r"int\d+": lambda x: x.group(0).upper(),  # e.g. int8, int16
-    }
-    for pattern, repl in subs.items():
-        text = re.sub(rf"\b{pattern}\b", repl, text, flags=re.IGNORECASE)
-    return text
-
-
-@dataclass
-class Example:
-    """
-    Example class for generating documentation content from a given path.
-
-    Attributes:
-        path (Path): The path to the main directory or file.
-        category (str): The category of the document.
-        main_file (Path): The main file in the directory.
-        other_files (list[Path]): list of other files in the directory.
-        title (str): The title of the document.
-
-    Methods:
-        __post_init__(): Initializes the main_file, other_files, and title attributes.
-        determine_main_file() -> Path: Determines the main file in the given path.
-        determine_other_files() -> list[Path]: Determines other files in the directory excluding the main file.
-        determine_title() -> str: Determines the title of the document.
-        generate() -> str: Generates the documentation content.
-    """  # noqa: E501
-
-    path: Path
-    category: str = None
-    main_file: Path = field(init=False)
-    other_files: list[Path] = field(init=False)
-    title: str = field(init=False)
-
-    def __post_init__(self):
-        self.main_file = self.determine_main_file()
-        self.other_files = self.determine_other_files()
-        self.title = self.determine_title()
-
-    @property
-    def is_code(self) -> bool:
-        return self.main_file.suffix != ".md"
-
-    def determine_main_file(self) -> Path:
-        """
-        Determines the main file in the given path.
-        If the path is a file, it returns the path itself. Otherwise, it searches
-        for Markdown files (*.md) in the directory and returns the first one found.
-        Returns:
-            Path: The main file path, either the original path if it's a file or the first
-            Markdown file found in the directory.
-        Raises:
-            IndexError: If no Markdown files are found in the directory.
-        """  # noqa: E501
-        return self.path if self.path.is_file() else list(self.path.glob("*.md")).pop()
-
-    def determine_other_files(self) -> list[Path]:
-        """
-        Determine other files in the directory excluding the main file.
-
-        This method checks if the given path is a file. If it is, it returns an empty list.
-        Otherwise, it recursively searches through the directory and returns a list of all
-        files that are not the main file.
-
-        Returns:
-            list[Path]: A list of Path objects representing the other files in the directory.
-        """  # noqa: E501
-        if self.path.is_file():
-            return []
-        # Text file extensions that can be safely included
-        text_extensions = {
-            ".py",
-            ".md",
-            ".sh",
-            ".yaml",
-            ".yml",
-            ".json",
-            ".txt",
-            ".toml",
-            ".cfg",
-            ".ini",
-        }
-        is_other_file = lambda file: (file.is_file() and file != self.main_file and file.suffix in text_extensions)  # noqa: E731
-        return [file for file in self.path.rglob("*") if is_other_file(file)]
-
-    def determine_title(self) -> str:
-        if not self.is_code:
-            # Specify encoding for building on Windows
-            with open(self.main_file, encoding="utf-8") as f:
-                first_line = f.readline().strip()
-            match = re.match(r"^#\s+(?P<title>.+)$", first_line)
-            if match:
-                return match.group("title")
-        return fix_case(self.path.stem.replace("_", " ").title())
-
-    def fix_relative_links(self, content: str) -> str:
-        """
-        Fix relative links in markdown content by converting them to gh-file
-        format.
-
-        Args:
-            content (str): The markdown content to process
-
-        Returns:
-            str: Content with relative links converted to gh-file format
-        """
-        # Regex to match markdown links [text](relative_path)
-        # This matches links that don't start with http, https, ftp, or #
-        link_pattern = r"\[([^\]]*)\]\((?!(?:https?|ftp)://|#)([^)]+)\)"
-
-        def replace_link(match):
-            link_text = match.group(1)
-            relative_path = match.group(2)
-
-            # Make relative to repo root
-            gh_file = (self.main_file.parent / relative_path).resolve()
-            gh_file = gh_file.relative_to(ROOT_DIR)
-
-            # Make GitHub URL
-            url = "https://github.com/vllm-project/vllm-omni/"
-            url += "tree/main" if self.path.is_dir() else "blob/main"
-            gh_url = f"{url}/{gh_file}"
-
-            return f"[{link_text}]({gh_url})"
-
-        return re.sub(link_pattern, replace_link, content)
-
-    def generate(self) -> str:
-        content = f"# {self.title}\n\n"
-        url = "https://github.com/vllm-project/vllm-omni/"
-        url += "tree/main" if self.path.is_dir() else "blob/main"
-        content += f"Source <{url}/{self.path.relative_to(ROOT_DIR)}>.\n\n"
-
-        # Use long code fence to avoid issues with
-        # included files containing code fences too
-        code_fence = "``````"
-
-        if self.is_code:
-            # Use relative path from ROOT_DIR for snippet inclusion
-            main_file_rel_path = self.main_file.relative_to(ROOT_DIR)
-            content += f'{code_fence}{self.main_file.suffix[1:]}\n--8<-- "{main_file_rel_path}"\n{code_fence}\n'
-        else:
-            with open(self.main_file, encoding="utf-8") as f:
-                # Skip the title from md snippets as it's been included above
-                main_content = f.readlines()[1:]
-            content += self.fix_relative_links("".join(main_content))
-        content += "\n"
-
-        if not self.other_files:
-            return content
-
-        content += "## Example materials\n\n"
-        for file in sorted(self.other_files):
-            # Use relative path from ROOT_DIR for snippet inclusion
-            file_rel_path = file.relative_to(ROOT_DIR)
-            content += f'??? abstract "{file.relative_to(self.path)}"\n'
-            if file.suffix != ".md":
-                content += f"    {code_fence}{file.suffix[1:]}\n"
-            content += f'    --8<-- "{file_rel_path}"\n'
-            if file.suffix != ".md":
-                content += f"    {code_fence}\n"
-
-        return content
-
-
-def on_startup(command: Literal["build", "gh-deploy", "serve"], dirty: bool):
-    logger.info("Generating example documentation")
-    logger.debug("Root directory: %s", ROOT_DIR.resolve())
-    logger.debug("Example directory: %s", EXAMPLE_DIR.resolve())
-    logger.debug("Example document directory: %s", EXAMPLE_DOC_DIR.resolve())
-
-    # Create the EXAMPLE_DOC_DIR if it doesn't exist
-    if not EXAMPLE_DOC_DIR.exists():
-        EXAMPLE_DOC_DIR.mkdir(parents=True)
-
-    categories = sorted(p for p in EXAMPLE_DIR.iterdir() if p.is_dir())
-
-    examples = []
-    glob_patterns = ["*.py", "*.md"]
-    # Find categorised examples
-    for category in categories:
-        globs = [category.glob(pattern) for pattern in glob_patterns]
-        for path in itertools.chain(*globs):
-            examples.append(Example(path, category.stem))
-        # Find examples in subdirectories
-        for path in category.glob("*/*.md"):
-            examples.append(Example(path.parent, category.stem))
-
-    # Generate the example documentation
-    for example in sorted(examples, key=lambda e: e.path.stem):
-        example_name = f"{example.path.stem}.md"
-        doc_path = EXAMPLE_DOC_DIR / example.category / example_name
-        if not doc_path.parent.exists():
-            doc_path.parent.mkdir(parents=True)
-        # Specify encoding for building on Windows
-        with open(doc_path, "w+", encoding="utf-8") as f:
-            f.write(example.generate())
-        logger.debug("Example generated: %s", doc_path.relative_to(ROOT_DIR))
@@ -1,55 +0,0 @@
-/* Custom CSS for vLLM-omni documentation - based on vLLM's extra.css */
-
-/* Improve code block readability */
-.highlight {
-    border-radius: 4px;
-}
-
-/* Style API documentation */
-.md-typeset .doc-contents {
-    margin-bottom: 1.5em;
-}
-
-/* Improve table readability */
-.md-typeset table {
-    display: table;
-    width: 100%;
-}
-
-/* Light mode: darker section titles (from vLLM) */
-body[data-md-color-scheme="default"] .md-nav__item--section > label.md-nav__link .md-ellipsis {
-  color: rgba(0, 0, 0, 0.7) !important;
-  font-weight: 700;
-}
-
-/* Dark mode: lighter gray section titles (from vLLM) */
-body[data-md-color-scheme="slate"] .md-nav__item--section > label.md-nav__link .md-ellipsis {
-  color: rgba(255, 255, 255, 0.75) !important;
-  font-weight: 700;
-}
-
-/* Navigation: Expand first level of API Reference by default, collapse deeper levels */
-/* Show direct children of API Reference section (vllm_omni's children: config, core, entrypoints, etc.) */
-.md-nav__item--nested[data-md-state] > .md-nav {
-    display: none;
-}
-
-/* Expand active items */
-.md-nav__item--nested.md-nav__item--active > .md-nav {
-    display: block;
-}
-
-/* For API Reference: show first level children (config, core, entrypoints, etc.) by default */
-.md-nav__item--section:has([href*="api/vllm_omni/index"]) > .md-nav > .md-nav__list > .md-nav__item--nested > .md-nav {
-    display: block;
-}
-
-/* But keep deeper levels collapsed */
-.md-nav__item--section:has([href*="api/vllm_omni/index"]) > .md-nav > .md-nav__list > .md-nav__item--nested > .md-nav > .md-nav__list > .md-nav__item--nested > .md-nav {
-    display: none;
-}
-
-/* Expand deeper levels only when active */
-.md-nav__item--section:has([href*="api/vllm_omni/index"]) > .md-nav > .md-nav__list > .md-nav__item--nested > .md-nav > .md-nav__list > .md-nav__item--nested.md-nav__item--active > .md-nav {
-    display: block;
-}
@@ -1,188 +0,0 @@
-## Multi-Request Streaming (MRS) on a Single Machine
-
-### 1. Background & Scope
- All processing runs on a single physical machine with multi-process, per-stage workers. No proxy or network transport involved.
- Current alignment with vllm-omni: `OmniLLM` supports multiple stages (`OmniStage`). GPU runners already expose streamable steps (prefill/decoding/diffusion), but the entry layer still collects lists and lacks intra-stage streaming and window scheduling.
- Goal: implement multi-stage, multi-request streaming (MRS) locally. Each stage outputs segments; downstream stages stitch and trigger compute based on configured windows. Shared memory and zero-copy strategies reduce data movement overhead.
-
-### 2. Key Constraints
- Multi-process per stage: each stage is an independent process with a while loop; device visibility can be configured (`CUDA_VISIBLE_DEVICES`/`torch.cuda.set_device`).
- Simple IPC (copy-based): use `multiprocessing.Queue`/Pipe for inter-process communication with CPU copies/serialization; do not rely on CUDA IPC/SHM zero-copy in this version.
- Cross-stage pipeline: different stages can process different requests concurrently (e.g., stage A handles request 1 while stage B handles request 0).
-
-### 3. Architecture Overview
- Processes & IPC queues
-  - Each "sub-stage" is an OS process (worker). The loop: take from input_queue → compute → put to output_queue.
-  - Inter-stage connection via IPC: copy-based `multiprocessing.Queue` passing dict payloads; use shared memory for large objects.
-  - Each link is SPSC (single-producer/single-consumer): the upstream is the orchestrator and the downstream is a single stage process; queues are unbounded (maxsize=0) on the orchestrator side.
- Device visibility
-  - Each stage sets `CUDA_VISIBLE_DEVICES` or calls `torch.cuda.set_device` to bind to GPU sets.
-  - A stage may use multiple GPUs internally (TP/PP/DP) but presents as a single stage unit.
- Simplified IPC: copy-based queues/pipes for data transfer; zero-copy is future work.
- Pipeline progression: when a stage finishes a request, it enqueues outputs to the downstream stage; if downstream is idle, it starts immediately.
- Scheduling
-  - A downstream stage triggers only after the upstream completes the request.
-  - Windowed segmentation/stitched triggering is not implemented; intra-stage streaming is not provided.
-
-### 7. IPC Implementation (simplified: copy-based)
- Use `multiprocessing.Queue`/Pipe for inter-process communication (control + data).
- Data is serialized/copied via CPU; no CUDA IPC/SHM zero-copy in this version.
- Backpressure: queues are unbounded; pressure manifests as compute-rate differences. Optional SHM reduces large-object transfer cost; RX/decoding overhead is recorded for observability.
-
-### 8. Scheduling & Cancellation (simplified)
- Pipeline: when a stage finishes a request, it enqueues to the next stage; that stage immediately pulls the next request from its input queue, enabling cross-stage concurrency.
- Cancellation/timeout: explicit cancellation/timeouts are not provided; graceful shutdown uses a `None` sentinel sent to each stage input queue.
-
-#### Short sequence example (req0/req1, stage A→B)
-1) t0: stage A handles req0
-2) t1: req0 completes on A → enters B; A immediately starts req1
-3) t2: B handles req0 while A handles req1 (parallel across stages)
-
-### 9. Integration Points (by file)
- `vllm_omni/entrypoints/omni_llm.py` (Orchestrator)
-  - Class `OmniLLM` orchestrates multi-process stages; constructs `OmniStage` instances in parallel and spawns per-stage workers.
-  - Spawns stage processes per config (set `CUDA_VISIBLE_DEVICES`/`torch.cuda.set_device`), creates control/data channels, builds simple full-trigger flow.
-  - Stats/logging are disabled by default; per-stage and orchestrator stats are only written when explicitly enabled.
-  - Manages process lifecycle: start/wait for readiness, graceful shutdown; forwards results between stages using copy-based IPC and optional SHM.
-  - Stage readiness: each stage emits `{"type": "stage_ready"}` after initialization; the orchestrator waits for all stages or times out and logs diagnostic suggestions.
-
-### 9.1 Process Device Visibility
- Device binding: set `CUDA_VISIBLE_DEVICES` before process start, or call `torch.cuda.set_device` early in init; configs can be single/multi-GPU.
- Cross-device transfer (simplified): via CPU copies; zero-copy paths are out of scope for now.
-
-### 10. Worker Template (pseudo-code)
-```python
-def stage_worker(input_q, output_q, runtime, shm_threshold):
-    # Device binding
-    set_stage_gpu_devices(runtime.devices)
-    engine = OmniStageLLM(model, **engine_args)
-    batch_seq = 0
-    while True:
-        first = input_q.get()
-        if first is None:
-            break
-        # Batch intake (up to runtime.max_batch_size)
-        batch = [first]
-        while len(batch) < int(runtime.max_batch_size or 1) and not input_q.empty():
-            nxt = input_q.get()
-            if nxt is None:
-                input_q.put(None)
-                break
-            batch.append(nxt)
-        # Decode IPC payload and sampling params
-        request_ids = []
-        engine_inputs = []
-        rx_bytes = {}
-        rx_decode_ms = {}
-        for t in batch:
-            ein, rx = maybe_load_from_ipc_with_metrics(t, "engine_inputs", "engine_inputs_shm")
-            request_ids.append(t["request_id"])
-            engine_inputs.extend(ein if isinstance(ein, list) else [ein])
-            rx_bytes[t["request_id"]] = rx["rx_transfer_bytes"]
-            rx_decode_ms[t["request_id"]] = rx["rx_decode_time_ms"]
-        # Generate and dispatch (grouped by request_id)
-        batch_seq += 1
-        outputs = list(engine.generate(engine_inputs, batch[0]["sampling_params"], use_tqdm=False))
-        grouped = group_by_request_id(outputs, request_ids)
-        for rid in request_ids:
-            r_out = grouped.get(rid, [])
-            use_shm, payload = maybe_dump_to_shm(r_out, shm_threshold)
-            msg = {
-                "request_id": rid,
-                "metrics": {
-                    "batch_id": batch_seq,
-                    "rx_transfer_bytes": int(rx_bytes.get(rid, 0)),
-                    "rx_decode_time_ms": float(rx_decode_ms.get(rid, 0.0)),
-                    # omitted: generation latency and token stats
-                },
-            }
-            if use_shm:
-                msg["engine_outputs_shm"] = payload
-            else:
-                msg["engine_outputs"] = payload
-            output_q.put(msg)
-```
-
-### 12. Observability
- Metrics (as implemented):
-  - per-request (emitted by stages): `num_tokens_out`, `stage_gen_time_ms`, `rx_transfer_bytes`, `rx_decode_time_ms`, `rx_in_flight_time_ms`, `batch_id`
-  - orchestrator aggregates: E2E latency, tokens/s (written only when stats are enabled)
-  - optional per-stage JSONL: `{log_file}.stage{stage_id}.stats.jsonl`
- Logs/tracing:
-  - optional per-stage log files: `{log_file}.stage{stage_id}.log`
-  - the orchestrator can log readiness, forward size/time, and summary information
-
-### 15. Risks & Mitigations
-— End-to-end latency: full-trigger sacrifices interactivity; start simple/correct, add windowed streaming later.
-— CPU copy overhead: copy-based IPC for maintainability; add SHM/zero-copy later as an optimization path.
-
-### 16. Refactor: Sink LLM init and process into Stage (diagram)
-
-Objective: without changing external call patterns, encapsulate per-stage LLM init, process creation, and worker logic inside `Stage`, so `PipelinedOmniLLM` focuses on orchestration (seeding, polling, forwarding, collecting) while keeping multi-process, shared memory, and device mapping capabilities.
-
-Key changes (vs. current):
- Before: Orchestrator created processes directly, passing `_stage_worker`; device setup, LLM init, batching, and SHM were implemented there.
- After: enhanced `OmniStage` owns its subprocess (`init_stage_worker`), with `_stage_worker` as the entry; device setup, LLM init, batching, and SHM live in `OmniStage`.
-
-Classes & responsibilities
- OmniLLM (orchestrator)
-  - Build `OmniStage` list in parallel (preserve `process_engine_inputs` wiring)
-  - Connect adjacent `Stage` input/output queues; seed requests to stage 0
-  - Poll stage outputs, decode results, call `process_engine_inputs`, then encode and forward to next stage
-  - Termination & cleanup: distribute shutdown signals; join/terminate subprocesses
- Stage (stage unit)
-  - Members: `stage_config` (with `engine_args` and `runtime`), `in_q/out_q`, subprocess handle, stats
-  - API:
-    - `init_stage_worker(...)`: spawn subprocess
-    - `stop_stage_worker()`: graceful exit
-    - `submit(ipc_payload)`: submit to input queue (may use shared memory)
-    - `try_collect() -> Optional[payload]`: non-blocking get from output queue
-    - `process_engine_inputs(stages, prompts)`: reuse/delegate existing logic
-  - Subprocess entry `worker_main()`:
-    - `set_stage_gpu_devices` (`CUDA_VISIBLE_DEVICES` & logical index mapping)
-    - build `OmniStageLLM(model, **engine_args)`
-    - while-loop:
-      - take first task + up to `runtime.max_batch_size-1` non-blocking; support `max_batch_size>=1`
-      - `maybe_load_from_ipc_with_metrics` per `engine_inputs` in batch
-      - `stage_engine.generate(batched_inputs, sampling_params)` (window `-1`)
-      - write per-request aggregated outputs to output queue after `maybe_dump_to_shm`
-      - log start/finish; report `error` per request on exceptions
-
-Data flow (sketch)
-```
-OmniLLM
-  ├─ Stage[0].start_process(in0, out0)
-  ├─ Stage[1].start_process(in1, out1)
-  ├─ ...
-  └─ Orchestrator loop:
-       - stage0.submit(encode(engine_inputs))
-       - for each stage:
-           res = stage.try_collect()
-           outputs = decode(res)
-           stage.set_engine_outputs(outputs)
-           if !final: next_inputs = next_stage.process_engine_inputs(stages, original_prompt)
-                    next_stage.submit(encode(next_inputs))
-```
-
-Config (preserve & extend)
- `runtime.devices`:
-  - Comma-separated visibility list (e.g., "2,5,7"): set visible set; default logical index 0 (first GPU)
-  - Number/string: treat as logical index (0/1-based per implementation) mapping to physical GPUs
- `runtime.max_batch_size`: max requests taken per intake (default 1)
- `shm_threshold_bytes`: objects larger than this go via shared memory (default 64KB; injected into stages)
- Stage link and inputs:
-  - `engine_input_source`: list of upstream stage_id(s) from which the downstream stage takes inputs (typically a single source)
-  - `custom_process_input_func`: function to construct downstream engine_inputs from upstream outputs
- Output flags:
-  - `final_output`, `final_output_type`: marks that a stage produces the final external output and its type
-
-Kept & strengthened
- Multi-process: each stage in its own process; parallel across stages; serial/batching inside a stage controlled by `max_batch_size`
- Shared memory: large objects via SHM with strict `memoryview -> close -> unlink`
- Device mapping: unified `set_stage_gpu_devices`, support comma lists and logical indices
-
-Migration (execution order)
-1) Enhanced `OmniStage` with `init_stage_worker/stop_stage_worker/submit/try_collect/_stage_worker`.
-2) `OmniLLM` holds `OmniStage` instances, replacing the older pipelined class; uses Stage APIs.
-3) Reuse `entrypoints/stage_utils`: device/SHM/encode-decode utilities live here.
-4) Regression: single/multi-request, various `max_batch_size`, SHM threshold, device mapping, multi-stage parallelism, error paths, and shutdown flows.
@@ -1,8 +0,0 @@
-# Examples
-
-vLLM-omni's examples are split into two categories:
-
- If you are using vLLM-omni from within Python code, see the *Offline Inference* section.
- If you are using vLLM-omni from an HTTP application or client, see the *Online Serving* section.
-
-For detailed example documentation, check the [examples directory](https://github.com/vllm-project/vllm-omni/tree/main/examples) in the repository.
@@ -1,64 +0,0 @@
-# Offline Example of vLLM-omni for Qwen2.5-omni
-
-Source <https://github.com/vllm-project/vllm-omni/tree/main/examples\offline_inference\qwen2_5_omni>.
-
-
-## 🛠️ Installation
-
-Please refer to [README.md](https://github.com/vllm-project/vllm-omni/tree/main/README.md)
-
-## Run examples (Qwen2.5-omni)
-### Multiple Prompts
-Download dataset from [seed_tts](https://drive.google.com/file/d/1GlSjVfSHkW3-leKKBlfrjuuTGqQ_xaLP/edit). To get the prompt, you can:
-```bash
-tar -xf <Your Download Path>/seedtts_testset.tar
-cp seedtts_testset/en/meta.lst examples/offline_inference/qwen2_5_omni/meta.lst
-python3 examples/offline_inference/qwen2_5_omni/extract_prompts.py \
-  --input examples/offline_inference/qwen2_5_omni/meta.lst \
-  --output examples/offline_inference/qwen2_5_omni/top100.txt \
-  --topk 100
-```
-Get into the example folder
-```bash
-cd examples/offline_inference/qwen2_5_omni
-```
-Then run the command below.
-```bash
-bash run_multiple_prompts.sh
-```
-### Single Prompts
-Get into the example folder
-```bash
-cd examples/offline_inference/qwen2_5_omni
-```
-Then run the command below.
-```bash
-bash run_single_prompt.sh
-```
-
-## Example materials
-
-??? abstract "end2end.py"
-    ``````py
-    --8<-- "examples\offline_inference\qwen2_5_omni\end2end.py"
-    ``````
-??? abstract "extract_prompts.py"
-    ``````py
-    --8<-- "examples\offline_inference\qwen2_5_omni\extract_prompts.py"
-    ``````
-??? abstract "processing_omni.py"
-    ``````py
-    --8<-- "examples\offline_inference\qwen2_5_omni\processing_omni.py"
-    ``````
-??? abstract "run_multiple_prompts.sh"
-    ``````sh
-    --8<-- "examples\offline_inference\qwen2_5_omni\run_multiple_prompts.sh"
-    ``````
-??? abstract "run_single_prompt.sh"
-    ``````sh
-    --8<-- "examples\offline_inference\qwen2_5_omni\run_single_prompt.sh"
-    ``````
-??? abstract "utils.py"
-    ``````py
-    --8<-- "examples\offline_inference\qwen2_5_omni\utils.py"
-    ``````
@@ -1,97 +0,0 @@
-# Offline Example of vLLM-omni for Qwen2.5-omni
-
-Source <https://github.com/vllm-project/vllm-omni/tree/main/examples/offline_inference/qwen_2_5_omni>.
-
-
-## 🛠️ Installation
-
-### Environment setup
-
-Use Docker to keep consistent basic environment (Optional, Recommended)
-```bash
-docker run --gpus all --ipc=host --network=host -v $source_dir:$container_dir --name $container_name -it nvcr.io/nvidia/pytorch:25.01-py3 bash
-```
-
-Set up basic uv environment
-```bash
-pip install uv
-uv venv --python 3.12 --seed
-source .venv/bin/activate
-```
-
-### Installation of vLLM (for users)
-
-Now we build it based on vLLM v0.11.0. Please install it with command below.
-```bash
-uv pip install vllm==0.11.0 --torch-backend=auto
-```
-
-### Installation of vLLM (for developers)
-
-If you want to check or debug with source code of vLLM, install stable release version of vllm with 0.11.0 from source with pre-built wheel file.
-
-```bash
-git clone https://github.com/vllm-project/vllm.git
-cd vllm
-git checkout v0.11.0
-```
-Set up environment variables to get pre-built wheels. If there are internet problems, just download the whl file manually. And set VLLM_PRECOMPILED_WHEEL_LOCATION as your local absolute path of whl file.
-```bash
-export VLLM_PRECOMPILED_WHEEL_LOCATION=https://github.com/vllm-project/vllm/releases/download/v0.11.0/vllm-0.11.0-cp38-abi3-manylinux1_x86_64.whl
-```
-Install vllm with command below (If you have no existing PyTorch).
-```bash
-uv pip install --editable .
-```
-
-Install vllm with command below (If you already have PyTorch).
-```bash
-python use_existing_torch.py
-uv pip install -r requirements/build.txt
-uv pip install --no-build-isolation --editable .
-```
-
-### Verification for successful installation of vLLM
-Just run the command below. If no error, it demonstrates that the installation is successful.
-```bash
-python -c "import vllm._C"
-```
-
-### Installation of vLLM-omni
-Install additional requirements for vllm-omni
-```bash
-git clone https://github.com/vllm-project/vllm-omni.git
-cd vllm_omni
-uv pip install -e .
-```
-
-## Run examples (Qwen2.5-omni)
-
-Get into the example folder
-```bash
-cd examples/offline_inference/qwen_2_5_omni
-```
-Then run
-```bash
-bash run.sh
-```
-The output audio is saved in ./output_audio
-
-## Example materials
-
-??? abstract "end2end.py"
-    ``````py
-    --8<-- "examples/offline_inference/qwen_2_5_omni/end2end.py"
-    ``````
-??? abstract "processing_omni.py"
-    ``````py
-    --8<-- "examples/offline_inference/qwen_2_5_omni/processing_omni.py"
-    ``````
-??? abstract "run.sh"
-    ``````sh
-    --8<-- "examples/offline_inference/qwen_2_5_omni/run.sh"
-    ``````
-??? abstract "utils.py"
-    ``````py
-    --8<-- "examples/offline_inference/qwen_2_5_omni/utils.py"
-    ``````
@@ -1,35 +0,0 @@
-# Online serving Example of vLLM-omni for Qwen2.5-omni
-
-Source <https://github.com/vllm-project/vllm-omni/blob/main/examples\online_serving\README.md>.
-
-
-## 🛠️ Installation
-
-Please refer to [README.md](https://github.com/vllm-project/vllm-omni/blob/main/README.md)
-
-## Run examples (Qwen2.5-omni)
-
-Launch the server
-```bash
-vllm serve Qwen/Qwen2.5-Omni-7B --omni --port 8091
-```
-
-If you have custom stage configs file, launch the server with command below
-```bash
-vllm serve Qwen/Qwen2.5-Omni-7B --omni --port 8091 --stage-configs-path /path/to/stage_configs_file
-```
-
-Get into the example folder
-```bash
-cd examples/online_serving
-```
-
-Send request via python
-```bash
-python openai_chat_completion_client_for_multimodal_generation.py
-```
-
-Send request via curl
-```bash
-bash run_curl_multimodal_generation.sh
-```
@@ -1,5 +0,0 @@
-# Online Serving
-
-## Available Examples
-
-Online serving is under active development. Coming soon.
@@ -1,7 +0,0 @@
-# OpenAI Chat Completion Client For Multimodal Generation
-
-Source <https://github.com/vllm-project/vllm-omni/blob/main/examples\online_serving\openai_chat_completion_client_for_multimodal_generation.py>.
-
-``````py
--8<-- "examples\online_serving\openai_chat_completion_client_for_multimodal_generation.py"
-``````
@@ -1,55 +0,0 @@
-# Quickstart
-
-This guide will help you get started with vLLM-omni.
-
-## Installation
-
-Use Docker to keep consistent basic environment (Optional, Recommended)
-
-```bash
-docker run --gpus all --ipc=host --network=host -v $source_dir:$container_dir --rm --name $container_name -it nvcr.io/nvidia/pytorch:25.01-py3 bash
-```
-
-Set up basic uv environment
-
-```bash
-pip install uv
-uv venv --python 3.12 --seed
-source .venv/bin/activate
-```
-
-Install vLLM
-
-```bash
-uv pip install vllm==0.11.0 --torch-backend=auto
-```
-
-Install vLLM-omni
-
-```bash
-git clone https://github.com/vllm-project/vllm-omni.git
-cd vllm_omni
-uv pip install -e .
-```
-
-## Run examples (Qwen2.5-omni)
-
-Get into the example folder
-
-```bash
-cd examples/offline_inference/qwen_2_5_omni
-```
-
-Then run.
-
-```bash
-bash run.sh
-```
-
-The output audio is saved in ./output_audio
-
-## Next Steps
-
- Read the [architecture documentation](../../contributing/design_documents/vllm_omni_design.md)
- Check out the [API reference](../../api/overview.md)
- Explore the [examples](../examples/index.md)
@@ -0,0 +1,128 @@
+#!/usr/bin/env python3
+"""
+Simple API client example for vLLM-omni server.
+
+This example shows how to interact with the vLLM-omni API server
+using HTTP requests.
+"""
+
+import requests
+import json
+import time
+
+
+def test_health(host="localhost", port=8000):
+    """Test if the server is healthy."""
+    url = f"http://{host}:{port}/health"
+    try:
+        response = requests.get(url, timeout=5)
+        if response.status_code == 200:
+            print("✅ Server is healthy")
+            return True
+        else:
+            print(f"❌ Server health check failed: {response.status_code}")
+            return False
+    except requests.exceptions.RequestException as e:
+        print(f"❌ Cannot connect to server: {e}")
+        return False
+
+
+def generate_text(prompt, host="localhost", port=8000, **kwargs):
+    """Generate text using the API server."""
+    url = f"http://{host}:{port}/generate"
+    
+    # Default parameters
+    params = {
+        "prompts": [prompt],
+        "max_tokens": kwargs.get("max_tokens", 100),
+        "temperature": kwargs.get("temperature", 0.7),
+        "top_p": kwargs.get("top_p", 1.0),
+        "frequency_penalty": kwargs.get("frequency_penalty", 0.0),
+        "presence_penalty": kwargs.get("presence_penalty", 0.0),
+    }
+    
+    # Add optional parameters
+    if "stop" in kwargs:
+        params["stop"] = kwargs["stop"]
+    
+    try:
+        response = requests.post(url, json=params, timeout=30)
+        response.raise_for_status()
+        return response.json()
+    except requests.exceptions.RequestException as e:
+        print(f"❌ API request failed: {e}")
+        return None
+
+
+def main():
+    """Main example function."""
+    print("vLLM-omni API Client Example")
+    print("=" * 40)
+    
+    # Test server health
+    if not test_health():
+        print("Please start the server first:")
+        print("  vllm serve Qwen/Qwen3-0.6B --omni --port 8000")
+        return
+    
+    # Example 1: Simple generation
+    print("\n1. Simple text generation:")
+    result = generate_text("Hello, how are you?", max_tokens=50)
+    if result:
+        text = result["outputs"][0]["text"]
+        print(f"   Input: Hello, how are you?")
+        print(f"   Output: {text}")
+    
+    # Example 2: Creative writing
+    print("\n2. Creative writing:")
+    result = generate_text(
+        "Write a short story about a robot learning to paint",
+        max_tokens=150,
+        temperature=0.9
+    )
+    if result:
+        text = result["outputs"][0]["text"]
+        print(f"   Input: Write a short story about a robot learning to paint")
+        print(f"   Output: {text}")
+    
+    # Example 3: Question answering
+    print("\n3. Question answering:")
+    result = generate_text(
+        "What is the capital of France?",
+        max_tokens=30,
+        temperature=0.3
+    )
+    if result:
+        text = result["outputs"][0]["text"]
+        print(f"   Input: What is the capital of France?")
+        print(f"   Output: {text}")
+    
+    # Example 4: Multiple prompts
+    print("\n4. Multiple prompts:")
+    prompts = [
+        "Tell me a joke",
+        "What's 2+2?",
+        "Describe the color blue"
+    ]
+    
+    for prompt in prompts:
+        result = generate_text(prompt, max_tokens=30)
+        if result:
+            text = result["outputs"][0]["text"]
+            print(f"   Q: {prompt}")
+            print(f"   A: {text}")
+    
+    # Example 5: Show full response structure
+    print("\n5. Full response structure:")
+    result = generate_text("Hello world", max_tokens=20)
+    if result:
+        print("   Full API response:")
+        print(json.dumps(result, indent=2))
+    
+    print("\n" + "=" * 40)
+    print("API client examples completed!")
+
+
+if __name__ == "__main__":
+    main()
+
@@ -0,0 +1,137 @@
+#!/usr/bin/env python3
+"""
+Simple vLLM-omni usage example.
+
+This example demonstrates how to use vLLM-omni for basic text generation
+with a single AR (Autoregressive) stage.
+"""
+
+import asyncio
+from vllm_omni.entrypoints.omni_llm import OmniLLM, AsyncOmniLLM
+from vllm_omni.config import create_ar_stage_config
+
+
+def sync_example():
+    """Synchronous usage example."""
+    print("=== Synchronous vLLM-omni Example ===")
+    
+    # Create a simple AR stage configuration
+    stage_config = create_ar_stage_config(
+        stage_id=0,
+        model_path="Qwen/Qwen3-0.6B",
+        input_modalities=["text"],
+        output_modalities=["text"]
+    )
+    
+    # Initialize OmniLLM
+    omni_llm = OmniLLM([stage_config])
+    
+    # Prepare stage arguments
+    stage_args = [{
+        "prompt": "Hello, how are you today?",
+        "max_tokens": 50,
+        "temperature": 0.7
+    }]
+    
+    # Generate text
+    print(f"Input: {stage_args[0]['prompt']}")
+    outputs = omni_llm.generate(stage_args)
+    
+    # Display results
+    for i, output in enumerate(outputs):
+        print(f"Output {i}:")
+        if hasattr(output, 'outputs') and output.outputs:
+            for completion in output.outputs:
+                print(f"  Text: {completion.text}")
+                print(f"  Finished: {completion.finish_reason != 'length'}")
+                print(f"  Tokens: {len(completion.token_ids)} tokens")
+
+
+async def async_example():
+    """Asynchronous usage example."""
+    print("\n=== Asynchronous vLLM-omni Example ===")
+    
+    # Create a simple AR stage configuration
+    stage_config = create_ar_stage_config(
+        stage_id=0,
+        model_path="Qwen/Qwen3-0.6B",
+        input_modalities=["text"],
+        output_modalities=["text"]
+    )
+    
+    # Initialize AsyncOmniLLM
+    omni_llm = AsyncOmniLLM([stage_config])
+    
+    # Prepare stage arguments
+    stage_args = [{
+        "prompt": "What is artificial intelligence?",
+        "max_tokens": 100,
+        "temperature": 0.8
+    }]
+    
+    # Generate text asynchronously
+    print(f"Input: {stage_args[0]['prompt']}")
+    outputs = await omni_llm.generate_async(stage_args)
+    
+    # Display results
+    for i, output in enumerate(outputs):
+        print(f"Output {i}:")
+        if hasattr(output, 'outputs') and output.outputs:
+            for completion in output.outputs:
+                print(f"  Text: {completion.text}")
+                print(f"  Finished: {completion.finish_reason != 'length'}")
+                print(f"  Tokens: {len(completion.token_ids)} tokens")
+
+
+def multi_prompt_example():
+    """Example with multiple prompts."""
+    print("\n=== Multi-Prompt Example ===")
+    
+    # Create stage configuration
+    stage_config = create_ar_stage_config(
+        stage_id=0,
+        model_path="Qwen/Qwen3-0.6B",
+        input_modalities=["text"],
+        output_modalities=["text"]
+    )
+    
+    # Initialize OmniLLM
+    omni_llm = OmniLLM([stage_config])
+    
+    # Multiple prompts
+    prompts = [
+        "Tell me a joke",
+        "What's the weather like?",
+        "Explain quantum computing"
+    ]
+    
+    for prompt in prompts:
+        print(f"\nInput: {prompt}")
+        stage_args = [{
+            "prompt": prompt,
+            "max_tokens": 30,
+            "temperature": 0.9
+        }]
+        
+        outputs = omni_llm.generate(stage_args)
+        
+        if outputs and hasattr(outputs[0], 'outputs') and outputs[0].outputs:
+            completion = outputs[0].outputs[0]
+            print(f"Output: {completion.text}")
+
+
+if __name__ == "__main__":
+    print("vLLM-omni Simple Usage Examples")
+    print("=" * 40)
+    
+    # Run synchronous example
+    sync_example()
+    
+    # Run asynchronous example
+    asyncio.run(async_example())
+    
+    # Run multi-prompt example
+    multi_prompt_example()
+    
+    print("\n" + "=" * 40)
+    print("Examples completed!")
@@ -1,42 +0,0 @@
-# Offline Example of vLLM-omni for Qwen2.5-omni
-
-## 🛠️ Installation
-
-Please refer to [README.md](../../../README.md)
-
-## Run examples (Qwen2.5-omni)
-### Multiple Prompts
-Download dataset from [seed_tts](https://drive.google.com/file/d/1GlSjVfSHkW3-leKKBlfrjuuTGqQ_xaLP/edit). To get the prompt, you can:
-```bash
-tar -xf <Your Download Path>/seedtts_testset.tar
-cp seedtts_testset/en/meta.lst examples/offline_inference/qwen2_5_omni/meta.lst
-python3 examples/offline_inference/qwen2_5_omni/extract_prompts.py \
-  --input examples/offline_inference/qwen2_5_omni/meta.lst \
-  --output examples/offline_inference/qwen2_5_omni/top100.txt \
-  --topk 100
-```
-Get into the example folder
-```bash
-cd examples/offline_inference/qwen2_5_omni
-```
-Then run the command below.
-```bash
-bash run_multiple_prompts.sh
-```
-### Single Prompts
-Get into the example folder
-```bash
-cd examples/offline_inference/qwen2_5_omni
-```
-Then run the command below.
-```bash
-bash run_single_prompt.sh
-```
-
-### FAQ
-
-If you encounter error about backend of librosa, try to install ffmpeg with command below.
-```
-sudo apt update
-sudo apt install ffmpeg
-```
@@ -1,307 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-"""
-This example shows how to use vLLM-omni for running offline inference
-with the correct prompt format on Qwen2.5-Omni
-"""
-
-import argparse
-import os
-from typing import NamedTuple
-
-import soundfile as sf
-from vllm.assets.audio import AudioAsset
-from vllm.assets.image import ImageAsset
-from vllm.assets.video import VideoAsset
-from vllm.multimodal.image import convert_image_mode
-from vllm.sampling_params import SamplingParams
-from vllm.utils import FlexibleArgumentParser
-
-from vllm_omni import OmniLLM
-
-SEED = 42
-
-
-class QueryResult(NamedTuple):
-    inputs: dict
-    limit_mm_per_prompt: dict[str, int]
-
-
-# NOTE: The default `max_num_seqs` and `max_model_len` may result in OOM on
-# lower-end GPUs.
-# Unless specified, these settings have been tested to work on a single L4.
-
-default_system = (
-    "You are Qwen, a virtual human developed by the Qwen Team, Alibaba "
-    "Group, capable of perceiving auditory and visual inputs, as well as "
-    "generating text and speech."
-)
-
-
-def get_text_query(question: str = None) -> QueryResult:
-    if question is None:
-        question = "Explain the system architecture for a scalable audio generation pipeline. Answer in 15 words."
-    prompt = (
-        f"<|im_start|>system\n{default_system}<|im_end|>\n"
-        "<|im_start|>user\n"
-        f"{question}<|im_end|>\n"
-        f"<|im_start|>assistant\n"
-    )
-    return QueryResult(
-        inputs={
-            "prompt": prompt,
-        },
-        limit_mm_per_prompt={},
-    )
-
-
-def get_mixed_modalities_query() -> QueryResult:
-    question = "What is recited in the audio? What is the content of this image? Why is this video funny?"
-    prompt = (
-        f"<|im_start|>system\n{default_system}<|im_end|>\n"
-        "<|im_start|>user\n<|audio_bos|><|AUDIO|><|audio_eos|>"
-        "<|vision_bos|><|IMAGE|><|vision_eos|>"
-        "<|vision_bos|><|VIDEO|><|vision_eos|>"
-        f"{question}<|im_end|>\n"
-        f"<|im_start|>assistant\n"
-    )
-    return QueryResult(
-        inputs={
-            "prompt": prompt,
-            "multi_modal_data": {
-                "audio": AudioAsset("mary_had_lamb").audio_and_sample_rate,
-                "image": convert_image_mode(ImageAsset("cherry_blossom").pil_image, "RGB"),
-                "video": VideoAsset(name="baby_reading", num_frames=16).np_ndarrays,
-            },
-        },
-        limit_mm_per_prompt={"audio": 1, "image": 1, "video": 1},
-    )
-
-
-def get_use_audio_in_video_query() -> QueryResult:
-    question = "Describe the content of the video, then convert what the baby say into text."
-    prompt = (
-        f"<|im_start|>system\n{default_system}<|im_end|>\n"
-        "<|im_start|>user\n<|vision_bos|><|VIDEO|><|vision_eos|>"
-        f"{question}<|im_end|>\n"
-        f"<|im_start|>assistant\n"
-    )
-    asset = VideoAsset(name="baby_reading", num_frames=16)
-    audio = asset.get_audio(sampling_rate=16000)
-
-    return QueryResult(
-        inputs={
-            "prompt": prompt,
-            "multi_modal_data": {
-                "video": asset.np_ndarrays,
-                "audio": audio,
-            },
-            "mm_processor_kwargs": {
-                "use_audio_in_video": True,
-            },
-        },
-        limit_mm_per_prompt={"audio": 1, "video": 1},
-    )
-
-
-def get_multi_audios_query() -> QueryResult:
-    question = "Are these two audio clips the same?"
-    prompt = (
-        f"<|im_start|>system\n{default_system}<|im_end|>\n"
-        "<|im_start|>user\n<|audio_bos|><|AUDIO|><|audio_eos|>"
-        "<|audio_bos|><|AUDIO|><|audio_eos|>"
-        f"{question}<|im_end|>\n"
-        f"<|im_start|>assistant\n"
-    )
-    return QueryResult(
-        inputs={
-            "prompt": prompt,
-            "multi_modal_data": {
-                "audio": [
-                    AudioAsset("winning_call").audio_and_sample_rate,
-                    AudioAsset("mary_had_lamb").audio_and_sample_rate,
-                ],
-            },
-        },
-        limit_mm_per_prompt={
-            "audio": 2,
-        },
-    )
-
-
-query_map = {
-    "mixed_modalities": get_mixed_modalities_query,
-    "use_audio_in_video": get_use_audio_in_video_query,
-    "multi_audios": get_multi_audios_query,
-    "text": get_text_query,
-}
-
-
-def main(args):
-    model_name = "Qwen/Qwen2.5-Omni-7B"
-    query_result = query_map[args.query_type]()
-
-    omni_llm = OmniLLM(
-        model=model_name,
-        log_stats=args.enable_stats,
-        log_file=("omni_llm_pipeline.log" if args.enable_stats else None),
-        init_sleep_seconds=args.init_sleep_seconds,
-        batch_timeout=args.batch_timeout,
-        init_timeout=args.init_timeout,
-        shm_threshold_bytes=args.shm_threshold_bytes,
-    )
-    thinker_sampling_params = SamplingParams(
-        temperature=0.0,  # Deterministic - no randomness
-        top_p=1.0,  # Disable nucleus sampling
-        top_k=-1,  # Disable top-k sampling
-        max_tokens=2048,
-        seed=SEED,  # Fixed seed for sampling
-        detokenize=True,
-        repetition_penalty=1.1,
-    )
-    talker_sampling_params = SamplingParams(
-        temperature=0.9,
-        top_p=0.8,
-        top_k=40,
-        max_tokens=2048,
-        seed=SEED,  # Fixed seed for sampling
-        detokenize=True,
-        repetition_penalty=1.05,
-        stop_token_ids=[8294],
-    )
-    code2wav_sampling_params = SamplingParams(
-        temperature=0.0,  # Deterministic - no randomness
-        top_p=1.0,  # Disable nucleus sampling
-        top_k=-1,  # Disable top-k sampling
-        max_tokens=2048,
-        seed=SEED,  # Fixed seed for sampling
-        detokenize=True,
-        repetition_penalty=1.1,
-    )
-
-    sampling_params_list = [
-        thinker_sampling_params,
-        talker_sampling_params,
-        code2wav_sampling_params,
-    ]
-
-    if args.txt_prompts is None:
-        prompts = [query_result.inputs for _ in range(args.num_prompts)]
-    else:
-        assert args.query_type == "text", "txt-prompts is only supported for text query type"
-        with open(args.txt_prompts, encoding="utf-8") as f:
-            lines = [ln.strip() for ln in f.readlines()]
-            prompts = [get_text_query(ln).inputs for ln in lines if ln != ""]
-            print(f"[Info] Loaded {len(prompts)} prompts from {args.txt_prompts}")
-
-    omni_outputs = omni_llm.generate(prompts, sampling_params_list)
-
-    output_dir = None
-    if args.save_results:
-        # Determine output directory: prefer --output-dir; fallback to --output-wav
-        output_dir = args.output_dir if getattr(args, "output_dir", None) else args.output_wav
-        os.makedirs(output_dir, exist_ok=True)
-    for stage_outputs in omni_outputs:
-        if stage_outputs.final_output_type == "text":
-            for output in stage_outputs.request_output:
-                request_id = int(output.request_id)
-                text_output = output.outputs[0].text
-                if args.save_results:
-                    # Save aligned text file per request
-                    prompt_text = prompts[request_id]["prompt"]
-                    out_txt = os.path.join(output_dir, f"{request_id:05d}.txt")
-                    lines = []
-                    lines.append("Prompt:\n")
-                    lines.append(str(prompt_text) + "\n")
-                    lines.append("vllm_text_output:\n")
-                    lines.append(str(text_output).strip() + "\n")
-                    try:
-                        with open(out_txt, "w", encoding="utf-8") as f:
-                            f.writelines(lines)
-                    except Exception as e:
-                        print(f"[Warn] Failed writing text file {out_txt}: {e}")
-                    print(f"Request ID: {request_id}, Text saved to {out_txt}")
-                else:
-                    print(f"Request ID: {request_id}, Text output:\n{text_output}")
-        elif stage_outputs.final_output_type == "audio":
-            for output in stage_outputs.request_output:
-                request_id = int(output.request_id)
-                audio_tensor = output.multimodal_output["audio"]
-                if args.save_results:
-                    output_wav = os.path.join(output_dir, f"output_{output.request_id}.wav")
-                    sf.write(output_wav, audio_tensor.detach().cpu().numpy(), samplerate=24000)
-                    print(f"Request ID: {request_id}, Saved audio to {output_wav}")
-                else:
-                    print(f"Request ID: {request_id}, Received audio output (not saved).")
-
-
-def parse_args():
-    parser = FlexibleArgumentParser(description="Demo on using vLLM for offline inference with audio language models")
-    parser.add_argument(
-        "--query-type",
-        "-q",
-        type=str,
-        default="mixed_modalities",
-        choices=query_map.keys(),
-        help="Query type.",
-    )
-    parser.add_argument(
-        "--enable-stats",
-        action="store_true",
-        default=False,
-        help="Enable writing detailed statistics (default: disabled)",
-    )
-    parser.add_argument(
-        "--init-sleep-seconds",
-        type=int,
-        default=20,
-        help="Sleep seconds after starting each stage process to allow initialization (default: 20)",
-    )
-    parser.add_argument(
-        "--batch-timeout",
-        type=int,
-        default=5,
-        help="Timeout for batching in seconds (default: 5)",
-    )
-    parser.add_argument(
-        "--init-timeout",
-        type=int,
-        default=300,
-        help="Timeout for initializing stages in seconds (default: 300)",
-    )
-    parser.add_argument(
-        "--shm-threshold-bytes",
-        type=int,
-        default=65536,
-        help="Threshold for using shared memory in bytes (default: 65536)",
-    )
-    parser.add_argument(
-        "--save-results",
-        action=argparse.BooleanOptionalAction,
-        default=True,
-        help="Save text/audio outputs to files (default: enabled). Use --no-save-results to disable.",
-    )
-    parser.add_argument(
-        "--output-wav",
-        default="output_audio",
-        help="[Deprecated] Output wav directory (use --output-dir).",
-    )
-    parser.add_argument(
-        "--num-prompts",
-        type=int,
-        default=1,
-        help="Number of prompts to generate.",
-    )
-    parser.add_argument(
-        "--txt-prompts",
-        type=str,
-        default=None,
-        help="Path to a .txt file with one prompt per line (preferred).",
-    )
-
-    return parser.parse_args()
-
-
-if __name__ == "__main__":
-    args = parse_args()
-    main(args)
@@ -1,45 +0,0 @@
-#!/usr/bin/env python3
-import argparse
-from typing import Optional
-
-
-def extract_prompt(line: str) -> Optional[str]:
-    # 提取第一个 '|' 与第二个 '|' 之间的内容
-    i = line.find("|")
-    if i == -1:
-        return None
-    j = line.find("|", i + 1)
-    if j == -1:
-        return None
-    return line[i + 1 : j].strip()
-
-
-def main() -> None:
-    parser = argparse.ArgumentParser()
-    parser.add_argument("--input", "-i", required=True, help="Input .lst file path")
-    parser.add_argument("--output", "-o", required=True, help="Output file path")
-    parser.add_argument(
-        "--topk",
-        "-k",
-        type=int,
-        default=100,
-        help="Extract the top K prompts (default: 100)",
-    )
-    args = parser.parse_args()
-
-    prompts = []
-    with open(args.input, encoding="utf-8", errors="ignore") as f:
-        for line in f:
-            if len(prompts) >= args.topk:
-                break
-            p = extract_prompt(line.rstrip("\n"))
-            if p:
-                prompts.append(p)
-
-    with open(args.output, "w", encoding="utf-8") as f:
-        for p in prompts:
-            f.write(p + "\n")
-
-
-if __name__ == "__main__":
-    main()
@@ -1,3 +0,0 @@
-python end2end.py --output-wav output_audio \
-                  --query-type text \
-                  --txt-prompts top10.txt
@@ -1,2 +0,0 @@
-python end2end.py --output-wav output_audio \
-                  --query-type use_audio_in_video
@@ -0,0 +1,37 @@
+# Offline Example of vLLM-omni for Qwen2.5-omni
+
+## Installation
+
+Set up basic environments
+```bash
+uv venv --python 3.12 --seed
+source .venv/bin/activate
+```
+Install certain version of vllm with commitid: 808a7b69df479b6b3a16181711cac7ca28a9b941
+
+```bash
+git clone https://github.com/vllm-project/vllm.git
+cd vllm
+git checkout 808a7b69df479b6b3a16181711cac7ca28a9b941
+VLLM_USE_PRECOMPILED=1 uv pip install --editable .
+```
+
+## Run examples
+
+Get into the example folder
+```bash
+cd vllm_omni
+cd examples/offline_inference/qwen2_5_omni
+```
+Modify PYTHONPATH in run.sh as your path of vllm_omni. Then run.
+```bash
+bash run.sh
+```
+The output audio is saved in ./output_audio
+
+## To-do list
+- [x] Offline inference example for Qwen2.5-omni with single request
+- [ ] Adaptation from current vllm branch to stable vllm v0.11.0
+- [ ] Offline inference example for Qwen2.5-omni with streaming multiple requests
+- [ ] Online inference support
+- [ ] Support for other models
@@ -0,0 +1,130 @@
+import argparse
+import os
+import soundfile as sf
+import random
+import numpy as np
+import torch
+
+from vllm.sampling_params import SamplingParams
+
+import os as _os_env_toggle
+_os_env_toggle.environ["VLLM_USE_V1"] = "1"
+
+from vllm_omni.entrypoints.omni_llm import OmniLLM
+from utils import make_omni_prompt
+
+
+SEED = 42
+# Set all random seeds
+random.seed(SEED)
+np.random.seed(SEED)
+torch.manual_seed(SEED)
+torch.cuda.manual_seed(SEED)
+torch.cuda.manual_seed_all(SEED)
+
+# Make PyTorch deterministic
+torch.backends.cudnn.deterministic = True
+torch.backends.cudnn.benchmark = False
+
+# Set environment variables for deterministic behavior
+os.environ["PYTHONHASHSEED"] = str(SEED)
+os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8"
+
+
+def parse_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--model', required=True, help='Path to merged model directory (will be created if downloading).')
+    parser.add_argument('--thinker-model', type=str, default=None)
+    parser.add_argument('--talker-model', type=str, default=None)
+    parser.add_argument('--code2wav-model', type=str, default=None)
+    parser.add_argument('--hf-hub-id', default='Qwen/Qwen2.5-Omni-7B', help='Hugging Face repo id to download if needed.')
+    parser.add_argument('--hf-revision', default=None, help='Optional HF revision (branch/tag/commit).')
+    parser.add_argument('--prompts', required=True, nargs='+', help='Input text prompts.')
+    parser.add_argument('--voice-type', default='default', help='Voice type, e.g., m02, f030, default.')
+    parser.add_argument('--code2wav-dir', default=None, help='Path to code2wav folder (contains spk_dict.pt).')
+    parser.add_argument('--dit-ckpt', default=None, help='Path to DiT checkpoint file (e.g., dit.pt).')
+    parser.add_argument('--bigvgan-ckpt', default=None, help='Path to BigVGAN checkpoint file.')
+    parser.add_argument('--dtype', default='bfloat16', choices=['float16', 'bfloat16', 'float32'])
+    parser.add_argument('--max-model-len', type=int, default=32768)
+
+    parser.add_argument("--thinker-only", action="store_true")
+    parser.add_argument("--text-only", action="store_true")
+    parser.add_argument("--do-wave", action="store_true")
+    parser.add_argument('--prompt_type',
+                        choices=[
+                            'text', 'audio', 'audio-long', 'audio-long-chunks',
+                            'audio-long-expand-chunks', 'image', 'video',
+                            'video-frames', 'audio-in-video', 'audio-in-video-v2',
+                            "audio-multi-round", "badcase-vl", "badcase-text",
+                            "badcase-image-early-stop", "badcase-two-audios",
+                            "badcase-two-videos", "badcase-multi-round",
+                            "badcase-voice-type", "badcase-voice-type-v2",
+                            "badcase-audio-tower-1", "badcase-audio-only"
+                        ],
+                        default='text')
+    parser.add_argument('--use-torchvision', action='store_true')
+    parser.add_argument('--tokenize', action='store_true')
+    parser.add_argument('--output-wav', default="output.wav", help='Output wav file path.')
+    parser.add_argument('--thinker-hidden-states-dir', default="thinker_hidden_states", help='Path to thinker hidden states directory.')
+    args = parser.parse_args()
+    return args
+
+
+def main():
+    args = parse_args()
+    model_name = args.model
+    omni_llm = OmniLLM(model=model_name)
+    thinker_sampling_params = SamplingParams(
+                                            temperature=0.0,    # Deterministic - no randomness
+                                            top_p=1.0,          # Disable nucleus sampling
+                                            top_k=-1,           # Disable top-k sampling
+                                            max_tokens=2048,
+                                            seed=SEED,          # Fixed seed for sampling
+                                            detokenize=True,
+                                            repetition_penalty=1.1,
+                                            )
+    talker_sampling_params = SamplingParams(
+                                            temperature=0.0,    # Deterministic - no randomness
+                                            top_p=1.0,          # Disable nucleus sampling
+                                            top_k=-1,           # Disable top-k sampling
+                                            max_tokens=2048,
+                                            seed=SEED,          # Fixed seed for sampling
+                                            detokenize=True,
+                                            repetition_penalty=1.1,
+                                            stop_token_ids=[8294]
+                                            )
+    code2wav_sampling_params = SamplingParams(
+                                            temperature=0.0,    # Deterministic - no randomness
+                                            top_p=1.0,          # Disable nucleus sampling
+                                            top_k=-1,           # Disable top-k sampling
+                                            max_tokens=2048,
+                                            seed=SEED,          # Fixed seed for sampling
+                                            detokenize=True,
+                                            repetition_penalty=1.1,
+                                            )
+
+    sampling_params_list = [thinker_sampling_params,
+                            talker_sampling_params,
+                            code2wav_sampling_params]
+    
+    prompt = [make_omni_prompt(args, prompt) for prompt in args.prompts]
+    omni_outputs = omni_llm.generate(prompt, sampling_params_list)
+
+    os.makedirs(args.output_wav, exist_ok=True)
+    for stage_outputs in omni_outputs:
+        if stage_outputs.final_output_type == "text":
+            for output in stage_outputs.request_output:
+                request_id = output.request_id
+                text_output = output.outputs[0].text
+                print(f"Request ID: {request_id}, Text Output: {text_output}")
+        elif stage_outputs.final_output_type == "audio":
+            for output in stage_outputs.request_output:
+                request_id = output.request_id
+                audio_tensor = output.multimodal_output["audio"]
+                output_wav = os.path.join(args.output_wav, f"output_{output.request_id}.wav")
+                sf.write(output_wav, audio_tensor.detach().cpu().numpy(), samplerate=24000)
+                print(f"Request ID: {request_id}, Saved audio to {output_wav}")
+
+
+if __name__ == "__main__":
+    main()
@@ -0,0 +1,373 @@
+from __future__ import annotations
+
+import base64
+import logging
+import math
+import os
+import time
+import warnings
+from functools import lru_cache
+from io import BytesIO
+import requests
+import torch
+import torchvision
+from packaging import version
+from PIL import Image
+from torchvision import io, transforms
+from torchvision.transforms import InterpolationMode
+
+
+logger = logging.getLogger(__name__)
+
+IMAGE_FACTOR = 28
+MIN_PIXELS = 4 * 28 * 28
+MAX_PIXELS = 16384 * 28 * 28
+MAX_RATIO = 200
+
+VIDEO_MIN_PIXELS = 128 * 28 * 28
+VIDEO_MAX_PIXELS = 768 * 28 * 28
+VIDEO_TOTAL_PIXELS = 24576 * 28 * 28
+FRAME_FACTOR = 2
+FPS = 2.0
+FPS_MIN_FRAMES = 4
+FPS_MAX_FRAMES = 768
+
+temporal_patch_size = 2
+spatial_patch_size = 14
+spatial_merge_size = 2
+
+
+def round_by_factor(number: int, factor: int) -> int:
+    """Returns the closest integer to 'number' that is divisible by 'factor'."""
+    return round(number / factor) * factor
+
+
+def ceil_by_factor(number: int, factor: int) -> int:
+    """Returns the smallest integer greater than or equal to 'number' that is divisible by 'factor'."""
+    return math.ceil(number / factor) * factor
+
+
+def floor_by_factor(number: int, factor: int) -> int:
+    """Returns the largest integer less than or equal to 'number' that is divisible by 'factor'."""
+    return math.floor(number / factor) * factor
+
+
+def smart_resize(height: int,
+                 width: int,
+                 factor: int = IMAGE_FACTOR,
+                 min_pixels: int = MIN_PIXELS,
+                 max_pixels: int = MAX_PIXELS) -> tuple[int, int]:
+    """
+    Rescales the image so that the following conditions are met:
+
+    1. Both dimensions (height and width) are divisible by 'factor'.
+
+    2. The total number of pixels is within the range ['min_pixels', 'max_pixels'].
+
+    3. The aspect ratio of the image is maintained as closely as possible.
+    """
+    if max(height, width) / min(height, width) > MAX_RATIO:
+        raise ValueError(
+            f"absolute aspect ratio must be smaller than {MAX_RATIO}, got {max(height, width) / min(height, width)}"
+        )
+    h_bar = max(factor, round_by_factor(height, factor))
+    w_bar = max(factor, round_by_factor(width, factor))
+    if h_bar * w_bar > max_pixels:
+        beta = math.sqrt((height * width) / max_pixels)
+        h_bar = floor_by_factor(height / beta, factor)
+        w_bar = floor_by_factor(width / beta, factor)
+    elif h_bar * w_bar < min_pixels:
+        beta = math.sqrt(min_pixels / (height * width))
+        h_bar = ceil_by_factor(height * beta, factor)
+        w_bar = ceil_by_factor(width * beta, factor)
+    return h_bar, w_bar
+
+
+def fetch_image(ele: dict[str, str | Image.Image],
+                size_factor: int = IMAGE_FACTOR) -> Image.Image:
+    if "image" in ele:
+        image = ele["image"]
+    else:
+        image = ele["image_url"]
+    image_obj = None
+    if isinstance(image, Image.Image):
+        image_obj = image
+    elif image.startswith("http://") or image.startswith("https://"):
+        image_obj = Image.open(requests.get(image, stream=True).raw)
+    elif image.startswith("file://"):
+        image_obj = Image.open(image[7:])
+    elif image.startswith("data:image"):
+        if "base64," in image:
+            _, base64_data = image.split("base64,", 1)
+            data = base64.b64decode(base64_data)
+            image_obj = Image.open(BytesIO(data))
+    else:
+        image_obj = Image.open(image)
+    if image_obj is None:
+        raise ValueError(
+            f"Unrecognized image input, support local path, http url, base64 and PIL.Image, got {image}"
+        )
+    image = image_obj.convert("RGB")
+    ## resize
+    if "resized_height" in ele and "resized_width" in ele:
+        resized_height, resized_width = smart_resize(
+            ele["resized_height"],
+            ele["resized_width"],
+            factor=size_factor,
+        )
+    else:
+        width, height = image.size
+        min_pixels = ele.get("min_pixels", MIN_PIXELS)
+        max_pixels = ele.get("max_pixels", MAX_PIXELS)
+        resized_height, resized_width = smart_resize(
+            height,
+            width,
+            factor=size_factor,
+            min_pixels=min_pixels,
+            max_pixels=max_pixels,
+        )
+    image = image.resize((resized_width, resized_height))
+
+    return image
+
+
+def smart_nframes(
+    ele: dict,
+    total_frames: int,
+    video_fps: int | float,
+) -> int:
+    """calculate the number of frames for video used for model inputs.
+
+    Args:
+        ele (dict): a dict contains the configuration of video.
+            support either `fps` or `nframes`:
+                - nframes: the number of frames to extract for model inputs.
+                - fps: the fps to extract frames for model inputs.
+                    - min_frames: the minimum number of frames of the video, only used when fps is provided.
+                    - max_frames: the maximum number of frames of the video, only used when fps is provided.
+        total_frames (int): the original total number of frames of the video.
+        video_fps (int | float): the original fps of the video.
+
+    Raises:
+        ValueError: nframes should in interval [FRAME_FACTOR, total_frames].
+
+    Returns:
+        int: the number of frames for video used for model inputs.
+    """
+    assert not ("fps" in ele
+                and "nframes" in ele), "Only accept either `fps` or `nframes`"
+    if "nframes" in ele:
+        nframes = round_by_factor(ele["nframes"], FRAME_FACTOR)
+    else:
+        fps = ele.get("fps", FPS)
+        min_frames = ceil_by_factor(ele.get("min_frames", FPS_MIN_FRAMES),
+                                    FRAME_FACTOR)
+        max_frames = floor_by_factor(
+            ele.get("max_frames", min(FPS_MAX_FRAMES, total_frames)),
+            FRAME_FACTOR)
+        nframes = total_frames / video_fps * fps
+        nframes = min(max(nframes, min_frames), max_frames)
+        nframes = round_by_factor(nframes, FRAME_FACTOR)
+    if not (FRAME_FACTOR <= nframes and nframes <= total_frames):
+        raise ValueError(
+            f"nframes should in interval [{FRAME_FACTOR}, {total_frames}], but got {nframes}."
+        )
+    return nframes
+
+
+def _read_video_torchvision(ele: dict, ) -> torch.Tensor:
+    """read video using torchvision.io.read_video
+
+    Args:
+        ele (dict): a dict contains the configuration of video.
+        support keys:
+            - video: the path of video. support "file://", "http://", "https://" and local path.
+            - video_start: the start time of video.
+            - video_end: the end time of video.
+    Returns:
+        torch.Tensor: the video tensor with shape (T, C, H, W).
+    """
+    video_path = ele["video"]
+    if version.parse(torchvision.__version__) < version.parse("0.19.0"):
+        if "http://" in video_path or "https://" in video_path:
+            warnings.warn(
+                "torchvision < 0.19.0 does not support http/https video path, please upgrade to 0.19.0."
+            )
+        if "file://" in video_path:
+            video_path = video_path[7:]
+    st = time.time()
+    video, audio, info = io.read_video(
+        video_path,
+        start_pts=ele.get("video_start", 0.0),
+        end_pts=ele.get("video_end", None),
+        pts_unit="sec",
+        output_format="TCHW",
+    )
+    total_frames, video_fps = video.size(0), info["video_fps"]
+    total_duration = round(total_frames / video_fps, 3)
+    logger.info(
+        f"torchvision:  {video_path=}, {total_frames=}, {video_fps=}, duration={total_duration}s, time={time.time() - st:.3f}s"
+    )
+    nframes = smart_nframes(ele,
+                            total_frames=total_frames,
+                            video_fps=video_fps)
+    idx = torch.linspace(0, total_frames - 1, nframes).round().long()
+    video = video[idx]
+    return video, total_duration, nframes
+
+
+def is_decord_available() -> bool:
+    import importlib.util
+
+    return importlib.util.find_spec("decord") is not None
+
+
+def _read_video_decord(ele: dict, ) -> torch.Tensor:
+    """read video using decord.VideoReader
+
+    Args:
+        ele (dict): a dict contains the configuration of video.
+        support keys:
+            - video: the path of video. support "file://", "http://", "https://" and local path.
+            - video_start: the start time of video.
+            - video_end: the end time of video.
+    Returns:
+        torch.Tensor: the video tensor with shape (T, C, H, W).
+    """
+    import decord
+    video_path = ele["video"]
+    st = time.time()
+    vr = decord.VideoReader(video_path)
+    # TODO: support start_pts and end_pts
+    if 'video_start' in ele or 'video_end' in ele:
+        raise NotImplementedError(
+            "not support start_pts and end_pts in decord for now.")
+    total_frames, video_fps = len(vr), vr.get_avg_fps()
+    total_duration = round(total_frames / video_fps, 3)
+    logger.info(
+        f"decord:  {video_path=}, {total_frames=}, {video_fps=}, time={time.time() - st:.3f}s"
+    )
+    nframes = smart_nframes(ele,
+                            total_frames=total_frames,
+                            video_fps=video_fps)
+    idx = torch.linspace(0, total_frames - 1, nframes).round().long().tolist()
+    video = vr.get_batch(idx).asnumpy()
+    video = torch.tensor(video).permute(0, 3, 1, 2)  # Convert to TCHW format
+    return video, total_duration, nframes
+
+
+VIDEO_READER_BACKENDS = {
+    "decord": _read_video_decord,
+    "torchvision": _read_video_torchvision,
+}
+
+FORCE_QWENVL_VIDEO_READER = os.getenv("FORCE_QWENVL_VIDEO_READER", None)
+
+
+@lru_cache(maxsize=1)
+def get_video_reader_backend() -> str:
+    if FORCE_QWENVL_VIDEO_READER is not None:
+        video_reader_backend = FORCE_QWENVL_VIDEO_READER
+    elif is_decord_available():
+        video_reader_backend = "decord"
+    else:
+        video_reader_backend = "torchvision"
+    # print(f"qwen-vl-utils using {video_reader_backend} to read video.", file=sys.stderr)
+    return video_reader_backend
+
+
+def fetch_video(
+        ele: dict,
+        image_factor: int = IMAGE_FACTOR) -> torch.Tensor | list[Image.Image]:
+    if isinstance(ele["video"], str):
+        video_reader_backend = get_video_reader_backend()
+        video, total_dur, nframes = VIDEO_READER_BACKENDS[
+            video_reader_backend](ele)
+        frame_timestamps = total_dur * torch.arange(1, nframes + 1) / nframes
+        grid_timestamps = frame_timestamps[::FRAME_FACTOR]
+        second_per_grid = grid_timestamps[1] - grid_timestamps[0]
+        nframes, _, height, width = video.shape
+        factor = spatial_patch_size * spatial_merge_size
+        min_pixels = ele.get("min_pixels", VIDEO_MIN_PIXELS)
+        total_pixels = ele.get("total_pixels", VIDEO_TOTAL_PIXELS)
+        max_pixels = max(
+            min(VIDEO_MAX_PIXELS, total_pixels / nframes * FRAME_FACTOR),
+            int(min_pixels * 1.05))
+        max_pixels = ele.get("max_pixels", max_pixels)
+        # min_pixels = (factor ** 2) * 52
+        # max_pixels = (factor ** 2) * min(768, (16384 / nframes * temporal_patch_size))
+        if "resized_height" in ele and "resized_width" in ele:
+            resized_height, resized_width = smart_resize(
+                ele["resized_height"],
+                ele["resized_width"],
+                factor=image_factor,
+            )
+        else:
+            resized_height, resized_width = smart_resize(
+                height,
+                width,
+                factor=image_factor,
+                min_pixels=min_pixels,
+                max_pixels=max_pixels,
+            )
+        video = transforms.functional.resize(
+            video,
+            [resized_height, resized_width],
+            interpolation=InterpolationMode.BICUBIC,
+            antialias=True,
+        ).float()
+        return video, total_dur, nframes, second_per_grid
+    else:
+        assert isinstance(ele["video"], (list, tuple))
+        process_info = ele.copy()
+        process_info.pop("type", None)
+        process_info.pop("video", None)
+        images = [
+            fetch_image({
+                "image": video_element,
+                **process_info
+            },
+                        size_factor=image_factor)
+            for video_element in ele["video"]
+        ]
+        nframes = ceil_by_factor(len(images), FRAME_FACTOR)
+        if len(images) < nframes:
+            images.extend([images[-1]] * (nframes - len(images)))
+        return images, None, None, None
+
+
+def extract_vision_info(
+        conversations: list[dict] | list[list[dict]]) -> list[dict]:
+    vision_infos = []
+    if isinstance(conversations[0], dict):
+        conversations = [conversations]
+    for conversation in conversations:
+        for message in conversation:
+            if isinstance(message["content"], list):
+                for ele in message["content"]:
+                    if ("image" in ele or "image_url" in ele or "video" in ele
+                            or ele["type"] in ("image", "image_url", "video")):
+                        vision_infos.append(ele)
+    return vision_infos
+
+
+def process_vision_info(
+    conversations: list[dict] | list[list[dict]],
+) -> tuple[list[Image.Image] | None, list[torch.Tensor | list[Image.Image]]
+           | None]:
+    vision_infos = extract_vision_info(conversations)
+    ## Read images or videos
+    image_inputs = []
+    video_inputs = []
+    for vision_info in vision_infos:
+        if "image" in vision_info or "image_url" in vision_info:
+            image_inputs.append(fetch_image(vision_info))
+        elif "video" in vision_info:
+            video_inputs.append(fetch_video(vision_info))
+        else:
+            raise ValueError("image, image_url or video should in content.")
+    if len(image_inputs) == 0:
+        image_inputs = None
+    if len(video_inputs) == 0:
+        video_inputs = None
+    return image_inputs, video_inputs
@@ -0,0 +1,9 @@
+export PYTHONPATH=/path/to/vllm-omni:$PYTHONPATH
+export HF_ENDPOINT=https://hf-mirror.com
+python end2end.py --model Qwen/Qwen2.5-Omni-7B \
+                                 --prompts "Explain the system architecture for a scalable audio generation pipeline. Answer in 15 words." \
+                                 --voice-type "m02" \
+                                 --dit-ckpt none \
+                                 --bigvgan-ckpt none \
+                                 --output-wav output_audio \
+                                 --prompt_type text
@@ -0,0 +1,305 @@
+import tempfile
+from urllib.request import urlopen
+import librosa
+import soundfile as sf
+import resampy
+from typing import Dict, Optional
+import torch
+import requests
+import torchvision.io
+
+from typing import Union, List
+from vllm.inputs import TextPrompt
+from vllm_omni.inputs.data import OmniTokensPrompt
+from processing_omni import fetch_image, fetch_video
+
+
+def get_system_prompt():
+    
+    return {
+        'role':
+        'system',
+        'content': [{
+            'type':
+            'text',
+            'text':
+            'You are Qwen, a virtual human developed by the Qwen Team, Alibaba Group, capable of perceiving auditory and visual inputs, as well as generating text and speech.'
+        }]
+    }
+
+
+def resample_wav_to_16khz(input_filepath):
+    data, original_sample_rate = sf.read(input_filepath)
+    # Only use the first channel
+    if len(data.shape) > 1:
+        data = data[:, 0]
+    # resample to 16kHz
+    data_resampled = resampy.resample(data,
+                                      sr_orig=original_sample_rate,
+                                      sr_new=16000)
+    return data_resampled
+
+
+def fetch_and_read_video(args, video_url: str, fps=2):
+
+    def read_video_with_torchvision(video_file_name: str):
+        video, audio, info = torchvision.io.read_video(
+            video_file_name,
+            start_pts=0.0,
+            end_pts=None,
+            pts_unit="sec",
+            output_format="TCHW",
+        )
+
+        total_frames, video_fps = video.size(0), info["video_fps"]
+        total_duration = round(total_frames / video_fps, 3)
+        nframes = int(total_frames / video_fps * fps)
+
+        frame_timestamps = total_duration * torch.arange(1,
+                                                         nframes + 1) / nframes
+        grid_timestamps = frame_timestamps[::2]
+        second_per_grid = grid_timestamps[1] - grid_timestamps[0]
+
+        idx = torch.linspace(0, video.size(0) - 1, nframes).round().long()
+        video = video[idx]
+
+        if args.legacy_omni_video:
+            return [video, total_duration, nframes, second_per_grid.item()]
+        else:
+            return video
+
+    def read_video_with_transformers(video_file_name: Union[str, List[str]]):
+        video, total_duration, nframes, second_per_grid = fetch_video(
+            {'video': video_file_name})
+        if total_duration is None and nframes is None:
+            nframes = len(video)
+            total_duration = 0.5 * nframes
+            second_per_grid = 1.0
+        if args.legacy_omni_video:
+            return [video, total_duration, nframes, second_per_grid]
+        else:
+            return video
+
+    def read_video(video_file_name: str):
+        if args.use_torchvision:
+            return read_video_with_torchvision(video_file_name)
+        else:
+            return read_video_with_transformers(video_file_name)
+
+    if isinstance(video_url, str) and video_url.startswith("http"):
+        with tempfile.NamedTemporaryFile(delete=True) as temp_video_file:
+            resp = requests.get(video_url)
+            assert resp.status_code == requests.codes.ok, f"Failed to fetch video from {video_url}, status_code:{resp.status_code}, resp:{resp}"
+
+            temp_video_file.write(urlopen(video_url).read())
+            temp_video_file_path = temp_video_file.name
+            video_file_name = temp_video_file_path
+            return read_video(video_file_name)
+    else:
+        video_file_name = video_url
+        return read_video(video_file_name)
+
+
+def make_inputs_qwen2_omni(
+    args,
+    messages: List[Dict[str, Union[str, List[Dict[str, str]]]]],
+    use_audio_in_video: Optional[bool] = False,
+    tokenize: bool = False,
+) -> Union[OmniTokensPrompt, TextPrompt]:
+    
+    from transformers import AutoConfig, AutoProcessor, AutoTokenizer
+    processor = AutoProcessor.from_pretrained(args.model)
+    tokenizer = AutoTokenizer.from_pretrained(args.model)
+
+    try:
+        config = AutoConfig.from_pretrained(args.model)
+        if 'Qwen2_5OmniModel' in config.architectures:
+            args.legacy_omni_video = False
+        else:
+            args.legacy_omni_video = True
+    except:
+        args.legacy_omni_video = True
+
+    audios, images, videos = [], [], []
+    for message in messages:
+        if not isinstance(message['content'], list):
+            message['content'] = [{
+                'type': 'text',
+                'text': message['content'],
+            }]
+        index, num_contents = 0, len(message['content'])
+        while index < num_contents:
+            ele = message['content'][index]
+            if 'type' not in ele:
+                if 'text' in ele:
+                    ele['type'] = 'text'
+                elif 'audio' in ele:
+                    ele['type'] = 'audio'
+                elif 'audio_url' in ele:
+                    ele['type'] = 'audio_url'
+                elif 'image' in ele:
+                    ele['type'] = 'image'
+                elif 'image_url' in ele:
+                    ele['type'] = 'image_url'
+                elif 'video' in ele:
+                    ele['type'] = 'video'
+                elif 'video_url' in ele:
+                    ele['type'] = 'video_url'
+                else:
+                    raise ValueError(f'Unknown ele: {ele}')
+
+            if ele['type'] == 'audio' or ele['type'] == 'audio_url':
+                if 'audio_url' in ele:
+                    audio_key = 'audio_url'
+                    with tempfile.NamedTemporaryFile(
+                            delete=True) as temp_audio_file:
+                        temp_audio_file.write(urlopen(ele[audio_key]).read())
+                        temp_audio_file_path = temp_audio_file.name
+                        audios.append(
+                            resample_wav_to_16khz(temp_audio_file_path))
+                        ele['audio'] = temp_audio_file_path
+                elif 'audio' in ele:
+                    audio_key = 'audio'
+                    audios.append(resample_wav_to_16khz(ele[audio_key]))
+                else:
+                    raise ValueError(f'Unknown ele {ele}')
+            elif use_audio_in_video and (ele['type'] == 'video'
+                                         or ele['type'] == 'video_url'):
+                # use video as audio as well
+                if 'video_url' in ele:
+                    audio_key = 'video_url'
+                    with tempfile.NamedTemporaryFile(
+                            delete=True) as temp_video_file:
+                        temp_video_file.write(urlopen(ele[audio_key]).read())
+                        temp_video_file_path = temp_video_file.name
+                        ele[audio_key] = temp_video_file_path
+                        audios.append(
+                            librosa.load(temp_video_file_path, sr=16000)[0])
+                        videos.append(
+                            fetch_and_read_video(args, temp_video_file_path))
+                        ele['video'] = temp_video_file_path
+                elif 'video' in ele:
+                    audio_key = 'video'
+                    audios.append(librosa.load(ele[audio_key], sr=16000)[0])
+                    videos.append(fetch_and_read_video(args, audio_key))
+                else:
+                    raise ValueError("Unknown ele {}".format(ele))
+                # insert a audio after the video
+                message['content'].insert(index + 1, {
+                    "type": "audio",
+                    "audio": ele[audio_key],
+                })
+                # no need to load the added audio again
+                index += 1
+            elif ele['type'] == 'video' or ele['type'] == 'video_url':
+                if 'video_url' in ele:
+                    video_key = 'video_url'
+                    with tempfile.NamedTemporaryFile(
+                            delete=True) as temp_video_file:
+                        temp_video_file.write(urlopen(ele['video_url']).read())
+                        temp_video_file_path = temp_video_file.name
+                        videos.append(fetch_and_read_video(args, temp_video_file))
+                        ele['video'] = temp_video_file_path
+                else:
+                    video_key = 'video'
+                    videos.append(fetch_and_read_video(args, ele[video_key]))
+            elif ele['type'] == 'image' or ele['type'] == 'image_url':
+                images.append(fetch_image(ele))
+
+            # move to the next content
+            index += 1
+
+    prompt = processor.apply_chat_template(
+        messages,
+        tokenize=tokenize,
+        add_generation_prompt=True,
+        add_vision_id=True,
+    )
+
+    audios = audios if len(audios) > 0 else None
+    images = images if len(images) > 0 else None
+    videos = videos if len(videos) > 0 else None
+
+    multi_modal_data = {}
+    if audios:
+        multi_modal_data["audio"] = audios
+    if images:
+        multi_modal_data["image"] = images
+    if videos:
+        multi_modal_data["video"] = videos
+
+    if isinstance(prompt, list) and isinstance(prompt[0], (list, str)):
+        prompt = prompt[0]
+
+    if tokenize:
+        return OmniTokensPrompt(
+            prompt_token_ids=prompt,
+            multi_modal_data=multi_modal_data,
+        )
+    else:
+        return TextPrompt(
+            prompt=prompt,
+            multi_modal_data=multi_modal_data,
+        )
+
+
+def make_text_prompt(args, prompt):
+    messages = [
+        get_system_prompt(),
+        {
+            "role": "user",
+            "content": [
+                {
+                    "type": "text",
+                    "text": prompt
+                },
+            ]
+        },
+    ]
+
+    prompt = make_inputs_qwen2_omni(args, messages, tokenize=args.tokenize)
+    return prompt
+
+
+def make_audio_in_video_v2_prompt(args):
+    messages = [
+        {
+            'role':
+            'system',
+            'content': [{
+                'type':
+                'text',
+                'text':
+                'You are Qwen, a virtual human developed by the Qwen Team, Alibaba Group, capable of perceiving auditory and visual inputs, as well as generating text and speech.'
+            }]
+        },
+        {
+            "role":
+            "user",
+            "content": [
+                {
+                    "type":
+                    "video_url",
+                    "video_url":
+                    "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen2.5-Omni/draw_small.mp4"
+                },
+            ]
+        },
+    ]
+    prompt = make_inputs_qwen2_omni(
+        args,
+        messages,
+        use_audio_in_video=True,
+        tokenize=args.tokenize,
+    )
+    return prompt
+
+
+def make_omni_prompt(args, prompt = None) -> Union[OmniTokensPrompt, List[OmniTokensPrompt]]:
+    if args.prompt_type == 'text':
+        prompt = make_text_prompt(args, prompt)
+    elif args.prompt_type == 'audio-in-video-v2':
+        prompt = make_audio_in_video_v2_prompt(args)
+    else:
+        raise ValueError(f'Unsupported prompt type: {args.prompt_type}')
+    return prompt
@@ -0,0 +1,31 @@
+# Omni Examples
+
+Examples showcasing multi-stage AR + DiT pipelines powered by vLLM-omni.
+
+## AR ➜ DiT with Diffusers Backend
+
+```bash
+# Optional: customise models
+export AR_MODEL=Qwen/Qwen3-0.6B
+export DIT_MODEL=./models/stable-diffusion-2-1/
+
+# Run with defaults (20 steps @ 512x512)
+python examples/omni/ar_dit_diffusers.py
+
+# Faster run
+python examples/omni/ar_dit_diffusers.py --steps 14 --height 384 --width 384 --guidance 4.5
+```
+
+Arguments:
+- `--ar-model`: Override AR stage model (default `Qwen/Qwen3-0.6B`).
+- `--dit-model`: Override DiT stage model (default env `DIT_MODEL`, else diffusers repo).
+- `--steps`: Number of diffusion steps (default 20).
+- `--guidance`: Guidance scale (default 5.0).
+- `--height` / `--width`: Output resolution (default 512).
+- `--seed`: Optional RNG seed.
+- `--prompt`, `--temperature`, `--max-tokens`: Control AR stage generation.
+- `--output`: Destination path for the generated image file.
+
+Generated images are saved next to the script as `omni_dit_output.png` by default.
+
+See the script for additional tweaks (scheduler, prompt chaining, etc.).
@@ -0,0 +1,175 @@
+"""Run the AR → DiT (diffusers) pipeline using YAML configuration defaults."""
+
+from __future__ import annotations
+
+import argparse
+import os
+from pathlib import Path
+from typing import Dict, List
+
+import yaml
+
+from vllm_omni import (
+    OmniLLM,
+    DiTConfig,
+    create_ar_stage_config,
+    create_dit_stage_config,
+)
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(
+        description="Run a two-stage AR → DiT pipeline with config-derived defaults."
+    )
+    parser.add_argument(
+        "--config",
+        type=Path,
+        default=Path(__file__).resolve().parent / "configs" / "ar_dit_local.yaml",
+        help="Path to a YAML configuration describing the pipeline stages.",
+    )
+    parser.add_argument(
+        "--prompt",
+        default="A scenic watercolor painting of a lighthouse at sunset",
+        help="Prompt passed to the AR stage.",
+    )
+    parser.add_argument(
+        "--negative-prompt",
+        default=None,
+        help="Optional negative prompt forwarded to the diffusion stage.",
+    )
+    parser.add_argument(
+        "--seed",
+        type=int,
+        default=None,
+        help="Optional seed for deterministic diffusion sampling.",
+    )
+    parser.add_argument(
+        "--output",
+        type=Path,
+        default=Path("./omni_dit_output.png"),
+        help="Destination path for the generated image.",
+    )
+    return parser.parse_args()
+
+
+def _normalize_modalities(values: List[str] | None, default: List[str]) -> List[str]:
+    if not values:
+        return default
+    return list(values)
+
+
+def _load_stage_configs_from_yaml(config_path: Path):
+    with config_path.open("r", encoding="utf-8") as f:
+        config = yaml.safe_load(f) or {}
+
+    stages = config.get("stages", [])
+    if not stages:
+        raise ValueError(f"No stages defined in config: {config_path}")
+
+    stage_configs = []
+    for stage in stages:
+        stage_id = stage["stage_id"]
+        engine_type = stage["engine_type"]
+        default_stage_args = stage.get("default_stage_args")
+
+        if engine_type == "AR":
+            stage_configs.append(
+                create_ar_stage_config(
+                    stage_id=stage_id,
+                    model_path=stage["model_path"],
+                    input_modalities=_normalize_modalities(
+                        stage.get("input_modalities"), ["text"]
+                    ),
+                    output_modalities=_normalize_modalities(
+                        stage.get("output_modalities"), ["text"]
+                    ),
+                    default_stage_args=default_stage_args,
+                )
+            )
+        elif engine_type == "DiT":
+            dit_cfg_dict: Dict = dict(stage.get("dit_config", {}))
+            if "num_inference_steps" not in dit_cfg_dict:
+                raise ValueError(
+                    "DiT stage requires 'num_inference_steps' in dit_config"
+                )
+            dit_cfg = DiTConfig(**dit_cfg_dict)
+
+            stage_configs.append(
+                create_dit_stage_config(
+                    stage_id=stage_id,
+                    model_path=stage["model_path"],
+                    input_modalities=_normalize_modalities(
+                        stage.get("input_modalities"), ["text"]
+                    ),
+                    output_modalities=_normalize_modalities(
+                        stage.get("output_modalities"), ["image"]
+                    ),
+                    dit_config=dit_cfg,
+                    default_stage_args=default_stage_args,
+                )
+            )
+        else:
+            raise ValueError(f"Unsupported engine_type '{engine_type}' in config")
+
+    return stage_configs
+
+
+def _apply_env_overrides(stage_configs):
+    ar_override = os.environ.get("AR_MODEL")
+    dit_override = os.environ.get("DIT_MODEL")
+
+    for stage_config in stage_configs:
+        if ar_override and stage_config.engine_type == "AR":
+            stage_config.model_path = ar_override
+        if dit_override and stage_config.engine_type == "DiT":
+            stage_config.model_path = dit_override
+
+
+def main():
+    args = parse_args()
+
+    stage_configs = _load_stage_configs_from_yaml(args.config)
+    _apply_env_overrides(stage_configs)
+
+    omni = OmniLLM(stage_configs)
+
+    stage_overrides: Dict[int, Dict[str, object]] = {}
+    if args.seed is not None or args.negative_prompt:
+        for stage_config in stage_configs:
+            if stage_config.engine_type != "DiT":
+                continue
+            override = stage_overrides.setdefault(stage_config.stage_id, {})
+            if args.seed is not None:
+                override["seed"] = args.seed
+            if args.negative_prompt:
+                override["negative_prompt"] = args.negative_prompt
+
+    outputs = omni.generate(
+        prompt=args.prompt,
+        stage_overrides=stage_overrides if stage_overrides else None,
+    )
+
+    image = None
+    for request_output in outputs:
+        for completion in getattr(request_output, "outputs", []) or []:
+            candidate = getattr(completion, "image", None)
+            if candidate is not None:
+                image = candidate
+                break
+        if image is not None:
+            break
+
+    if image is None:
+        print("No image found in outputs.")
+        return
+
+    out_path = args.output.resolve()
+    try:
+        image.save(out_path)
+        print(f"Saved image to {out_path}")
+    except Exception:
+        print("Generated output is not a PIL.Image; skipping save.")
+
+
+if __name__ == "__main__":
+    main()
@@ -0,0 +1,28 @@
+# Example config for running OmniLLM with local AR + DiT models.
+stages:
+  - stage_id: 0
+    engine_type: AR
+    model_path: Qwen/Qwen3-0.6B
+    input_modalities: [text]
+    output_modalities: [text]
+    default_stage_args:
+      max_tokens: 16
+      temperature: 0.7
+  - stage_id: 1
+    engine_type: DiT
+    model_path: stabilityai/stable-diffusion-2-1
+    input_modalities: [text]
+    output_modalities: [image]
+    default_stage_args:
+      height: 512
+      width: 512
+    dit_config:
+      num_inference_steps: 20
+      guidance_scale: 5.0
+      use_diffusers: true
+      diffusers_pipeline: auto
+      height: 512
+      width: 512
+      device_config:
+        device: mps
+        dtype: float16
@@ -1,77 +0,0 @@
-# Online serving Example of vLLM-omni for Qwen2.5-omni
-
-## 🛠️ Installation
-
-Please refer to [README.md](../../README.md)
-
-## Run examples (Qwen2.5-omni)
-
-Launch the server
-```bash
-vllm serve Qwen/Qwen2.5-Omni-7B --omni --port 8091
-```
-
-If you have custom stage configs file, launch the server with command below
-```bash
-vllm serve Qwen/Qwen2.5-Omni-7B --omni --port 8091 --stage-configs-path /path/to/stage_configs_file
-```
-
-Get into the example folder
-```bash
-cd examples/online_serving
-```
-
-Send request via python
-```bash
-python openai_chat_completion_client_for_multimodal_generation.py --query-type mixed_modalities
-```
-
-Send request via curl
-```bash
-bash run_curl_multimodal_generation.sh mixed_modalities
-```
-
-### FAQ
-
-If you encounter error about backend of librosa, try to install ffmpeg with command below.
-```
-sudo apt update
-sudo apt install ffmpeg
-```
-
-## Run Local Web UI Demo
-
-This Web UI demo allows users to interact with the model through a web browser.
-
-### Running Gradio Demo
-
-Once vllm and vllm-omni are installed, you can launch the web service built on AsyncOmniLLM by
-
-```bash
-python gradio_demo.py  --model Qwen/Qwen2.5-Omni-7B --port 7861
-```
-
-Then open `http://localhost:7861/` on your local browser to interact with the web UI.
-
-
-### Options
-
-The gradio demo also supports running with an existing API server and can be customized with the following arguments.
-
-
-```bash
-python gradio_demo.py \
-    --model Qwen/Qwen2.5-Omni-7B \
-    --use-api-server \
-    --api-base http://localhost:8091/v1 \
-    --ip 127.0.0.1 \
-    --port 7861
-```
-
- `--model`: Model name
- `--use-api-server`: If set, connect to an existing vLLM HTTP API server instead of running AsyncOmniLLM locally.
- `--api-base`: Base URL for vllm serve (only used when `use-api-server` is set, default: http://localhost:8091/v1)
- `--ip`: Host/IP for Gradio server (default: 127.0.0.1)
- `--port`: Port for Gradio server (default: 7861)
- `--stage-configs-path`: Path to custom stage configs YAML file (optional)
- `--share`: Share the Gradio demo publicly (creates a public link)
@@ -1,535 +0,0 @@
-import argparse
-import os
-import random
-import signal
-import sys
-from pathlib import Path
-from types import SimpleNamespace
-from typing import Any, Optional
-
-import gradio as gr
-import numpy as np
-import soundfile as sf
-import torch
-from PIL import Image
-from vllm.assets.video import video_get_metadata, video_to_ndarrays
-from vllm.sampling_params import SamplingParams
-
-from vllm_omni.entrypoints.async_omni_llm import AsyncOmniLLM
-
-# Import utils from offline inference example
-sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../offline_inference/qwen2_5_omni"))
-try:
-    from utils import make_omni_prompt
-except ImportError:
-    # Fallback if utils doesn't exist - we'll build prompts directly
-    make_omni_prompt = None
-
-SEED = 42
-ASYNC_INIT_TIMEOUT = 600
-
-SUPPORTED_MODELS: dict[str, dict[str, Any]] = {
-    "Qwen/Qwen2.5-Omni-7B": {
-        "sampling_params": {
-            "thinker": {
-                "temperature": 0.0,
-                "top_p": 1.0,
-                "top_k": -1,
-                "max_tokens": 2048,
-                "detokenize": True,
-                "repetition_penalty": 1.1,
-            },
-            "talker": {
-                "temperature": 0.0,
-                "top_p": 1.0,
-                "top_k": -1,
-                "max_tokens": 2048,
-                "detokenize": True,
-                "repetition_penalty": 1.1,
-                "stop_token_ids": [8294],
-            },
-            "code2wav": {
-                "temperature": 0.0,
-                "top_p": 1.0,
-                "top_k": -1,
-                "max_tokens": 2048,
-                "detokenize": True,
-                "repetition_penalty": 1.1,
-            },
-        },
-    },
-}
-# Ensure deterministic behavior across runs.
-random.seed(SEED)
-np.random.seed(SEED)
-torch.manual_seed(SEED)
-torch.cuda.manual_seed(SEED)
-torch.cuda.manual_seed_all(SEED)
-torch.backends.cudnn.deterministic = True
-torch.backends.cudnn.benchmark = False
-os.environ["PYTHONHASHSEED"] = str(SEED)
-os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8"
-
-
-def parse_args():
-    parser = argparse.ArgumentParser(description="Gradio demo for Qwen2.5-Omni online inference.")
-    parser.add_argument(
-        "--model",
-        default="Qwen/Qwen2.5-Omni-7B",
-        help="Path to local model checkpoint.",
-    )
-    parser.add_argument(
-        "--ip",
-        default="127.0.0.1",
-        help="Host/IP for gradio `launch`.",
-    )
-    parser.add_argument("--port", type=int, default=7861, help="Port for gradio `launch`.")
-    parser.add_argument("--share", action="store_true", help="Share the Gradio demo publicly.")
-    parser.add_argument(
-        "--stage-configs-path",
-        type=str,
-        default=None,
-        help="Path to custom stage configs YAML file (optional).",
-    )
-    return parser.parse_args()
-
-
-def build_sampling_params(seed: int, model_key: str) -> list[SamplingParams]:
-    """Build SamplingParams objects by reusing the dict definitions."""
-    return [SamplingParams(**params_dict) for params_dict in build_sampling_params_dict(seed, model_key)]
-
-
-def build_sampling_params_dict(seed: int, model_key: str) -> list[dict]:
-    """Build sampling params as dict for HTTP API mode."""
-    model_conf = SUPPORTED_MODELS.get(model_key)
-    if model_conf is None:
-        raise ValueError(f"Unsupported model '{model_key}'")
-
-    sampling_templates: dict[str, dict[str, Any]] = model_conf["sampling_params"]
-    sampling_params: list[dict] = []
-    for stage_name, template in sampling_templates.items():
-        params = dict(template)
-        params["seed"] = seed
-        sampling_params.append(params)
-    return sampling_params
-
-
-def create_prompt_args(base_args: argparse.Namespace) -> SimpleNamespace:
-    # The prompt builder expects a minimal namespace with these attributes.
-    return SimpleNamespace(
-        model=base_args.model,
-        prompt_type="text",
-        tokenize=True,
-        use_torchvision=True,
-        legacy_omni_video=False,
-    )
-
-
-def process_audio_file(
-    audio_file: Optional[Any],
-) -> Optional[tuple[np.ndarray, int]]:
-    """Normalize Gradio audio input to (np.ndarray, sample_rate)."""
-    if audio_file is None:
-        return None
-
-    sample_rate: Optional[int] = None
-    audio_np: Optional[np.ndarray] = None
-
-    def _load_from_path(path_str: str) -> Optional[tuple[np.ndarray, int]]:
-        if not path_str:
-            return None
-        path = Path(path_str)
-        if not path.exists():
-            return None
-        data, sr = sf.read(path)
-        if data.ndim > 1:
-            data = data[:, 0]
-        return data.astype(np.float32), int(sr)
-
-    if isinstance(audio_file, tuple):
-        if len(audio_file) == 2:
-            first, second = audio_file
-            # Case 1: (sample_rate, np.ndarray)
-            if isinstance(first, (int, float)) and isinstance(second, np.ndarray):
-                sample_rate = int(first)
-                audio_np = second
-            # Case 2: (filepath, (sample_rate, np.ndarray or list))
-            elif isinstance(first, str):
-                if isinstance(second, tuple) and len(second) == 2:
-                    sr_candidate, data_candidate = second
-                    if isinstance(sr_candidate, (int, float)) and isinstance(data_candidate, np.ndarray):
-                        sample_rate = int(sr_candidate)
-                        audio_np = data_candidate
-                if audio_np is None:
-                    loaded = _load_from_path(first)
-                    if loaded is not None:
-                        audio_np, sample_rate = loaded
-            # Case 3: (None, (sample_rate, np.ndarray))
-            elif first is None and isinstance(second, tuple) and len(second) == 2:
-                sr_candidate, data_candidate = second
-                if isinstance(sr_candidate, (int, float)) and isinstance(data_candidate, np.ndarray):
-                    sample_rate = int(sr_candidate)
-                    audio_np = data_candidate
-        elif len(audio_file) == 1 and isinstance(audio_file[0], str):
-            loaded = _load_from_path(audio_file[0])
-            if loaded is not None:
-                audio_np, sample_rate = loaded
-    elif isinstance(audio_file, str):
-        loaded = _load_from_path(audio_file)
-        if loaded is not None:
-            audio_np, sample_rate = loaded
-
-    if audio_np is None or sample_rate is None:
-        return None
-
-    if audio_np.ndim > 1:
-        audio_np = audio_np[:, 0]
-
-    return audio_np.astype(np.float32), sample_rate
-
-
-def process_image_file(image_file: Optional[Image.Image]) -> Optional[Image.Image]:
-    """Process image file from Gradio input.
-
-    Returns:
-        PIL Image in RGB mode or None if no image provided.
-    """
-    if image_file is None:
-        return None
-    # Convert to RGB if needed
-    if image_file.mode != "RGB":
-        image_file = image_file.convert("RGB")
-    return image_file
-
-
-def process_video_file(
-    video_file: Optional[str],
-    enable_audio_in_video: bool = False,
-    max_frames: int = 32,
-) -> Optional[tuple[np.ndarray, dict[str, Any], Optional[tuple[np.ndarray, int]]]]:
-    """Process video file and optionally extract audio track."""
-    if video_file is None:
-        return None
-
-    video_path = Path(video_file)
-    if not video_path.exists():
-        print(f"Video file not found: {video_path}")
-        return None
-
-    try:
-        frames = video_to_ndarrays(str(video_path), num_frames=max_frames)
-        metadata = video_get_metadata(str(video_path), num_frames=frames.shape[0])
-    except Exception as exc:  # pylint: disable=broad-except
-        print(f"Failed to decode video {video_path}: {exc}")
-        return None
-
-    audio_tuple: Optional[tuple[np.ndarray, int]] = None
-    if enable_audio_in_video:
-        try:
-            import librosa  # type: ignore import
-
-            audio_signal, sampling_rate = librosa.load(str(video_path), sr=16000)
-            audio_tuple = (audio_signal.astype(np.float32), sampling_rate)
-        except Exception as exc:  # pylint: disable=broad-except
-            print(f"Failed to extract audio from video {video_path}: {exc}")
-
-    return frames, metadata, audio_tuple
-
-
-async def run_inference_async_omni_llm(
-    omni_llm: AsyncOmniLLM,
-    sampling_params: list[SamplingParams],
-    prompt_args_template: SimpleNamespace,
-    user_prompt: str,
-    audio_file: Optional[tuple[str, tuple[int, np.ndarray]]] = None,
-    image_file: Optional[Image.Image] = None,
-    video_file: Optional[str] = None,
-    use_audio_in_video: bool = False,
-):
-    """Run inference using AsyncOmniLLM directly with multimodal support."""
-    if not user_prompt.strip() and not audio_file and not image_file and not video_file:
-        return "Please provide at least a text prompt or multimodal input.", None
-
-    try:
-        # Build prompt with multimodal data
-        prompt_args = SimpleNamespace(**prompt_args_template.__dict__)
-
-        # Process multimodal inputs
-        multi_modal_data = {}
-        mm_processor_kwargs = {}
-
-        # Process audio
-        audio_data = process_audio_file(audio_file)
-        if audio_data is not None:
-            multi_modal_data["audio"] = audio_data
-
-        # Process image
-        image_data = process_image_file(image_file)
-        if image_data is not None:
-            multi_modal_data["image"] = image_data
-
-        # Process video
-        video_payload = process_video_file(video_file, enable_audio_in_video=use_audio_in_video)
-        if video_payload is not None:
-            video_frames, video_metadata, extracted_audio = video_payload
-            video_entry: Any
-            if video_metadata:
-                video_entry = (video_frames, video_metadata)
-            else:
-                video_entry = video_frames
-            multi_modal_data["video"] = video_entry
-            if use_audio_in_video and extracted_audio is not None and "audio" not in multi_modal_data:
-                multi_modal_data["audio"] = extracted_audio
-                mm_processor_kwargs["use_audio_in_video"] = True
-
-        # Build the prompt input
-        if make_omni_prompt is not None:
-            omni_prompt = make_omni_prompt(prompt_args, user_prompt)
-            # Add multimodal data if present
-            if multi_modal_data:
-                if isinstance(omni_prompt, dict):
-                    omni_prompt["multi_modal_data"] = multi_modal_data
-                    if mm_processor_kwargs:
-                        omni_prompt["mm_processor_kwargs"] = mm_processor_kwargs
-                else:
-                    # If make_omni_prompt returns a string, we need to wrap it
-                    omni_prompt = {
-                        "prompt": omni_prompt,
-                        "multi_modal_data": multi_modal_data,
-                    }
-                    if mm_processor_kwargs:
-                        omni_prompt["mm_processor_kwargs"] = mm_processor_kwargs
-        else:
-            # Fallback: build prompt directly
-            default_system = (
-                "You are Qwen, a virtual human developed by the Qwen Team, "
-                "Alibaba Group, capable of perceiving auditory and visual inputs, "
-                "as well as generating text and speech."
-            )
-            prompt = f"<|im_start|>system\n{default_system}<|im_end|>\n<|im_start|>user\n"
-            if audio_data:
-                prompt += "<|audio_bos|><|AUDIO|><|audio_eos|>"
-            if image_data:
-                prompt += "<|vision_bos|><|IMAGE|><|vision_eos|>"
-            if video_payload is not None:
-                prompt += "<|vision_bos|><|VIDEO|><|vision_eos|>"
-            if user_prompt.strip():
-                prompt += f"{user_prompt}"
-            prompt += "<|im_end|>\n<|im_start|>assistant\n"
-
-            omni_prompt = {
-                "prompt": prompt,
-                "multi_modal_data": multi_modal_data,
-            }
-            if mm_processor_kwargs:
-                omni_prompt["mm_processor_kwargs"] = mm_processor_kwargs
-
-        request_id = "0"
-        text_outputs: list[str] = []
-        audio_output = None
-
-        async for stage_outputs in omni_llm.generate(
-            prompt=omni_prompt,
-            request_id=request_id,
-            sampling_params_list=sampling_params,
-        ):
-            # stage_outputs.request_output is a RequestOutput object, not a list
-            request_output = stage_outputs.request_output
-            if stage_outputs.final_output_type == "text":
-                if request_output.outputs:
-                    for output in request_output.outputs:
-                        if output.text:
-                            text_outputs.append(output.text)
-            elif stage_outputs.final_output_type == "audio":
-                # multimodal_output is on the RequestOutput object
-                # See vllm_omni/entrypoints/openai/serving_chat.py:680 for reference
-                if hasattr(request_output, "multimodal_output") and request_output.multimodal_output:
-                    if "audio" in request_output.multimodal_output:
-                        audio_tensor = request_output.multimodal_output["audio"]
-                        # Ensure audio is 1D (flatten if needed)
-                        if hasattr(audio_tensor, "ndim") and audio_tensor.ndim > 1:
-                            audio_tensor = audio_tensor.flatten()
-                        audio_np = audio_tensor.detach().cpu().numpy()
-                        audio_output = (
-                            24000,  # sampling rate in Hz
-                            audio_np,
-                        )
-
-        text_response = "\n\n".join(text_outputs) if text_outputs else "No text output."
-        return text_response, audio_output
-    except Exception as exc:  # pylint: disable=broad-except
-        return f"Inference failed: {exc}", None
-
-
-def build_interface(
-    omni_llm: AsyncOmniLLM,
-    sampling_params: list[SamplingParams],
-    prompt_args_template: SimpleNamespace,
-    model: str,
-):
-    """Build Gradio interface for AsyncOmniLLM mode."""
-
-    async def run_inference(
-        user_prompt: str,
-        audio_file: Optional[tuple[str, tuple[int, np.ndarray]]],
-        image_file: Optional[Image.Image],
-        video_file: Optional[str],
-        use_audio_in_video: bool,
-    ):
-        return await run_inference_async_omni_llm(
-            omni_llm,
-            sampling_params,
-            prompt_args_template,
-            user_prompt,
-            audio_file,
-            image_file,
-            video_file,
-            use_audio_in_video,
-        )
-
-    css = """
-    .media-input-container {
-        display: flex;
-        gap: 10px;
-    }
-    .media-input-container > div {
-        flex: 1;
-    }
-    .media-input-container .image-input,
-    .media-input-container .audio-input {
-        height: 300px;
-    }
-    .media-input-container .video-column {
-        height: 300px;
-        display: flex;
-        flex-direction: column;
-    }
-    .media-input-container .video-input {
-        flex: 1;
-        min-height: 0;
-    }
-    #generate-btn button {
-        width: 100%;
-    }
-    """
-
-    with gr.Blocks(css=css) as demo:
-        gr.Markdown("# vLLM-Omni Online Serving Demo")
-        gr.Markdown(f"**Model:** {model} \n\n")
-
-        with gr.Column():
-            with gr.Row():
-                input_box = gr.Textbox(
-                    label="Text Prompt",
-                    placeholder="For example: Describe what happens in the media inputs.",
-                    lines=4,
-                    scale=1,
-                )
-            with gr.Row(elem_classes="media-input-container"):
-                image_input = gr.Image(
-                    label="Image Input (optional)",
-                    type="pil",
-                    sources=["upload"],
-                    scale=1,
-                    elem_classes="image-input",
-                )
-                with gr.Column(scale=1, elem_classes="video-column"):
-                    video_input = gr.Video(
-                        label="Video Input (optional)",
-                        sources=["upload"],
-                        elem_classes="video-input",
-                    )
-                    use_audio_in_video_checkbox = gr.Checkbox(
-                        label="Use audio from video",
-                        value=False,
-                        info="Extract the video's audio track when provided.",
-                    )
-                audio_input = gr.Audio(
-                    label="Audio Input (optional)",
-                    type="numpy",
-                    sources=["upload", "microphone"],
-                    scale=1,
-                    elem_classes="audio-input",
-                )
-
-        with gr.Row():
-            generate_btn = gr.Button(
-                "Generate",
-                variant="primary",
-                size="lg",
-                elem_id="generate-btn",
-            )
-
-        with gr.Row():
-            text_output = gr.Textbox(label="Text Output", lines=10, scale=2)
-            audio_output = gr.Audio(label="Audio Output", interactive=False, scale=1)
-
-        generate_btn.click(
-            fn=run_inference,
-            inputs=[input_box, audio_input, image_input, video_input, use_audio_in_video_checkbox],
-            outputs=[text_output, audio_output],
-        )
-        demo.queue()
-    return demo
-
-
-def main():
-    args = parse_args()
-    omni_llm = None
-
-    model_name = "/".join(args.model.split("/")[-2:])
-    assert model_name in SUPPORTED_MODELS, (
-        f"Unsupported model '{model_name}'. Supported models: {SUPPORTED_MODELS.keys()}"
-    )
-
-    # Register signal handlers for graceful shutdown
-    def signal_handler(sig, frame):
-        print("\nReceived interrupt signal, shutting down...")
-        if omni_llm is not None:
-            try:
-                omni_llm.shutdown()
-            except Exception as e:
-                print(f"Error during shutdown: {e}")
-        sys.exit(0)
-
-    signal.signal(signal.SIGINT, signal_handler)
-    signal.signal(signal.SIGTERM, signal_handler)
-
-    print(f"Initializing AsyncOmniLLM with model: {args.model}")
-    if args.stage_configs_path:
-        print(f"Using custom stage configs: {args.stage_configs_path}")
-
-    sampling_params = build_sampling_params(SEED, model_name)
-    omni_llm = AsyncOmniLLM(
-        model=args.model,
-        stage_configs_path=args.stage_configs_path,
-        init_timeout=ASYNC_INIT_TIMEOUT,
-    )
-    print("✓ AsyncOmniLLM initialized successfully")
-    prompt_args_template = create_prompt_args(args)
-
-    demo = build_interface(
-        omni_llm,
-        sampling_params,
-        prompt_args_template,
-        args.model,
-    )
-    try:
-        demo.launch(
-            server_name=args.ip,
-            server_port=args.port,
-            share=args.share,
-        )
-    except KeyboardInterrupt:
-        print("\nShutting down...")
-    finally:
-        # Cleanup
-        if omni_llm is not None:
-            try:
-                omni_llm.shutdown()
-            except Exception as e:
-                print(f"Error during cleanup: {e}")
-
-
-if __name__ == "__main__":
-    main()
@@ -1,228 +0,0 @@
-import base64
-
-import requests
-from openai import OpenAI
-from vllm.assets.audio import AudioAsset
-from vllm.utils import FlexibleArgumentParser
-
-# Modify OpenAI's API key and API base to use vLLM's API server.
-openai_api_key = "EMPTY"
-openai_api_base = "http://localhost:8091/v1"
-
-client = OpenAI(
-    # defaults to os.environ.get("OPENAI_API_KEY")
-    api_key=openai_api_key,
-    base_url=openai_api_base,
-)
-
-SEED = 42
-
-
-def encode_base64_content_from_url(content_url: str) -> str:
-    """Encode a content retrieved from a remote url to base64 format."""
-
-    with requests.get(content_url) as response:
-        response.raise_for_status()
-        result = base64.b64encode(response.content).decode("utf-8")
-
-    return result
-
-
-def get_system_prompt():
-    return {
-        "role": "system",
-        "content": [
-            {
-                "type": "text",
-                "text": (
-                    "You are Qwen, a virtual human developed by the Qwen Team, "
-                    "Alibaba Group, capable of perceiving auditory and visual inputs, "
-                    "as well as generating text and speech."
-                ),
-            }
-        ],
-    }
-
-
-def get_text_query():
-    question = "Explain the system architecture for a scalable audio generation pipeline. Answer in 15 words."
-    prompt = {
-        "role": "user",
-        "content": [
-            {
-                "type": "text",
-                "text": f"{question}",
-            }
-        ],
-    }
-    return prompt
-
-
-def get_mixed_modalities_query():
-    question = "What is recited in the audio? What is the content of this image? Why is this video funny?"
-    prompt = {
-        "role": "user",
-        "content": [
-            {
-                "type": "audio_url",
-                "audio_url": {"url": AudioAsset("mary_had_lamb").url},
-            },
-            {
-                "type": "image_url",
-                "image_url": {
-                    "url": "https://vllm-public-assets.s3.us-west-2.amazonaws.com/vision_model_images/cherry_blossom.jpg"
-                },
-            },
-            {
-                "type": "video_url",
-                "video_url": {
-                    "url": "https://huggingface.co/datasets/raushan-testing-hf/videos-test/resolve/main/sample_demo_1.mp4"
-                },
-            },
-            {
-                "type": "text",
-                "text": f"{question}",
-            },
-        ],
-    }
-
-    return prompt
-
-
-def get_use_audio_in_video_query():
-    question = "Describe the content of the video, then convert what the baby say into text."
-
-    prompt = {
-        "role": "user",
-        "content": [
-            {
-                "type": "video_url",
-                "video_url": {
-                    "url": "https://huggingface.co/datasets/raushan-testing-hf/videos-test/resolve/main/sample_demo_1.mp4",
-                    "num_frames": 16,
-                },
-            },
-            {
-                "type": "text",
-                "text": f"{question}",
-            },
-        ],
-    }
-
-    return prompt
-
-
-def get_multi_audios_query():
-    question = "Are these two audio clips the same?"
-    prompt = {
-        "role": "user",
-        "content": [
-            {
-                "type": "audio_url",
-                "audio_url": {"url": AudioAsset("mary_had_lamb").url},
-            },
-            {
-                "type": "audio_url",
-                "audio_url": {"url": AudioAsset("winning_call").url},
-            },
-            {
-                "type": "text",
-                "text": f"{question}",
-            },
-        ],
-    }
-    return prompt
-
-
-query_map = {
-    "mixed_modalities": get_mixed_modalities_query,
-    "use_audio_in_video": get_use_audio_in_video_query,
-    "multi_audios": get_multi_audios_query,
-    "text": get_text_query,
-}
-
-
-def run_multimodal_generation(args) -> None:
-    model_name = "Qwen/Qwen2.5-Omni-7B"
-    thinker_sampling_params = {
-        "temperature": 0.0,  # Deterministic - no randomness
-        "top_p": 1.0,  # Disable nucleus sampling
-        "top_k": -1,  # Disable top-k sampling
-        "max_tokens": 2048,
-        "seed": SEED,  # Fixed seed for sampling
-        "detokenize": True,
-        "repetition_penalty": 1.1,
-    }
-    talker_sampling_params = {
-        "temperature": 0.9,
-        "top_p": 0.8,
-        "top_k": 40,
-        "max_tokens": 2048,
-        "seed": SEED,  # Fixed seed for sampling
-        "detokenize": True,
-        "repetition_penalty": 1.05,
-        "stop_token_ids": [8294],
-    }
-    code2wav_sampling_params = {
-        "temperature": 0.0,  # Deterministic - no randomness
-        "top_p": 1.0,  # Disable nucleus sampling
-        "top_k": -1,  # Disable top-k sampling
-        "max_tokens": 2048,
-        "seed": SEED,  # Fixed seed for sampling
-        "detokenize": True,
-        "repetition_penalty": 1.1,
-    }
-
-    sampling_params_list = [
-        thinker_sampling_params,
-        talker_sampling_params,
-        code2wav_sampling_params,
-    ]
-
-    prompt = query_map[args.query_type]()
-    extra_body = {
-        "sampling_params_list": sampling_params_list  # Optional, it has a default setting in stage_configs of the corresponding model.
-    }
-
-    if args.query_type == "use_audio_in_video":
-        extra_body["mm_processor_kwargs"] = {"use_audio_in_video": True}
-
-    chat_completion = client.chat.completions.create(
-        messages=[
-            get_system_prompt(),
-            prompt,
-        ],
-        model=model_name,
-        extra_body=extra_body,
-    )
-
-    count = 0
-    for choice in chat_completion.choices:
-        if choice.message.audio:
-            audio_data = base64.b64decode(choice.message.audio.data)
-            audio_file_path = f"audio_{count}.wav"
-            with open(audio_file_path, "wb") as f:
-                f.write(audio_data)
-            print(f"Audio saved to {audio_file_path}")
-            count += 1
-        elif choice.message.content:
-            print("Chat completion output from text:", choice.message.content)
-
-
-def parse_args():
-    parser = FlexibleArgumentParser(description="Demo on using vLLM for offline inference with audio language models")
-    parser.add_argument(
-        "--query-type",
-        "-q",
-        type=str,
-        default="mixed_modalities",
-        choices=query_map.keys(),
-        help="Query type.",
-    )
-
-    return parser.parse_args()
-
-
-if __name__ == "__main__":
-    args = parse_args()
-    run_multimodal_generation(args)
@@ -1,187 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-
-# Default query type
-QUERY_TYPE="${1:-mixed_modalities}"
-
-# Validate query type
-if [[ ! "$QUERY_TYPE" =~ ^(mixed_modalities|use_audio_in_video|multi_audios|text)$ ]]; then
-    echo "Error: Invalid query type '$QUERY_TYPE'"
-    echo "Usage: $0 [mixed_modalities|use_audio_in_video|multi_audios|text]"
-    echo "  mixed_modalities: Audio + Image + Video + Text query"
-    echo "  use_audio_in_video: Video + Text query (with audio extraction from video)"
-    echo "  multi_audios: Two audio clips + Text query"
-    echo "  text: Text query"
-    exit 1
-fi
-
-SEED=42
-
-thinker_sampling_params='{
-  "temperature": 0.0,
-  "top_p": 1.0,
-  "top_k": -1,
-  "max_tokens": 2048,
-  "seed": 42,
-  "detokenize": true,
-  "repetition_penalty": 1.1
-}'
-
-talker_sampling_params='{
-  "temperature": 0.9,
-  "top_p": 0.8,
-  "top_k": 40,
-  "max_tokens": 2048,
-  "seed": 42,
-  "detokenize": true,
-  "repetition_penalty": 1.05,
-  "stop_token_ids": [8294]
-}'
-
-code2wav_sampling_params='{
-  "temperature": 0.0,
-  "top_p": 1.0,
-  "top_k": -1,
-  "max_tokens": 2048,
-  "seed": 42,
-  "detokenize": true,
-  "repetition_penalty": 1.1
-}'
-# Above is optional, it has a default setting in stage_configs of the corresponding model.
-
-# Define URLs for assets
-MARY_HAD_LAMB_AUDIO_URL="https://vllm-public-assets.s3.us-west-2.amazonaws.com/multimodal_asset/mary_had_lamb.ogg"
-WINNING_CALL_AUDIO_URL="https://vllm-public-assets.s3.us-west-2.amazonaws.com/multimodal_asset/winning_call.ogg"
-CHERRY_BLOSSOM_IMAGE_URL="https://vllm-public-assets.s3.us-west-2.amazonaws.com/vision_model_images/cherry_blossom.jpg"
-SAMPLE_VIDEO_URL="https://huggingface.co/datasets/raushan-testing-hf/videos-test/resolve/main/sample_demo_1.mp4"
-
-# Build user content and extra fields based on query type
-case "$QUERY_TYPE" in
-  text)
-    user_content='[
-      {
-        "type": "text",
-        "text": "Explain the system architecture for a scalable audio generation pipeline. Answer in 15 words."
-      }
-    ]'
-    sampling_params_list='[
-      '"$thinker_sampling_params"',
-      '"$talker_sampling_params"',
-      '"$code2wav_sampling_params"'
-    ]'
-    mm_processor_kwargs="{}"
-    ;;
-  mixed_modalities)
-    user_content='[
-        {
-          "type": "audio_url",
-          "audio_url": {
-            "url": "'"$MARY_HAD_LAMB_AUDIO_URL"'"
-          }
-        },
-        {
-          "type": "image_url",
-          "image_url": {
-            "url": "'"$CHERRY_BLOSSOM_IMAGE_URL"'"
-          }
-        },
-        {
-          "type": "video_url",
-          "video_url": {
-            "url": "'"$SAMPLE_VIDEO_URL"'"
-          }
-        },
-        {
-          "type": "text",
-          "text": "What is recited in the audio? What is the content of this image? Why is this video funny?"
-        }
-      ]'
-    sampling_params_list='[
-      '"$thinker_sampling_params"',
-      '"$talker_sampling_params"',
-      '"$code2wav_sampling_params"'
-    ]'
-    mm_processor_kwargs="{}"
-    ;;
-  use_audio_in_video)
-    user_content='[
-        {
-          "type": "video_url",
-          "video_url": {
-            "url": "'"$SAMPLE_VIDEO_URL"'"
-          }
-        },
-        {
-          "type": "text",
-          "text": "Describe the content of the video, then convert what the baby say into text."
-        }
-      ]'
-    sampling_params_list='[
-      '"$thinker_sampling_params"',
-      '"$talker_sampling_params"',
-      '"$code2wav_sampling_params"'
-    ]'
-    mm_processor_kwargs='{
-      "use_audio_in_video": true
-    }'
-    ;;
-  multi_audios)
-    user_content='[
-        {
-          "type": "audio_url",
-          "audio_url": {
-            "url": "'"$MARY_HAD_LAMB_AUDIO_URL"'"
-          }
-        },
-        {
-          "type": "audio_url",
-          "audio_url": {
-            "url": "'"$WINNING_CALL_AUDIO_URL"'"
-          }
-        },
-        {
-          "type": "text",
-          "text": "Are these two audio clips the same?"
-        }
-      ]'
-    sampling_params_list='[
-      '"$thinker_sampling_params"',
-      '"$talker_sampling_params"',
-      '"$code2wav_sampling_params"'
-    ]'
-    mm_processor_kwargs="{}"
-    ;;
-esac
-
-echo "Running query type: $QUERY_TYPE"
-echo ""
-
-
-output=$(curl -sS -X POST http://localhost:8091/v1/chat/completions \
-    -H "Content-Type: application/json" \
-    -d @- <<EOF
-{
-  "model": "Qwen/Qwen2.5-Omni-7B",
-  "sampling_params_list": $sampling_params_list,
-  "mm_processor_kwargs": $mm_processor_kwargs,
-  "messages": [
-    {
-      "role": "system",
-      "content": [
-        {
-          "type": "text",
-          "text": "You are Qwen, a virtual human developed by the Qwen Team, Alibaba Group, capable of perceiving auditory and visual inputs, as well as generating text and speech."
-        }
-      ]
-    },
-    {
-      "role": "user",
-      "content": $user_content
-    }
-  ]
-}
-EOF
-  )
-
-# Here it only shows the text content of the first choice. Audio content has many binaries, so it's not displayed here.
-echo "Output of request: $(echo "$output" | jq '.choices[0].message.content')"
@@ -1,180 +0,0 @@
-site_name: vLLM-omni
-site_description: Multi-modality models inference and serving with non-autoregressive structures
-site_author: vLLM-omni Team
-site_url: https://vllm-project.github.io/vllm-omni/
-
-repo_name: vllm-project/vllm-omni
-repo_url: https://github.com/vllm-project/vllm-omni
-edit_uri: edit/main/docs/
-
-# Copyright
-copyright: Copyright &copy; 2025 vLLM-omni Team
-
-# Theme
-theme:
-  name: material
-  palette:
-    # Light mode
-    - scheme: default
-      primary: blue
-      accent: blue
-      toggle:
-        icon: material/brightness-7
-        name: Switch to dark mode
-    # Dark mode
-    - scheme: slate
-      primary: blue
-      accent: blue
-      toggle:
-        icon: material/brightness-4
-        name: Switch to light mode
-  features:
-    - content.action.edit
-    - content.code.copy
-    - content.tabs.link
-    - navigation.instant
-    - navigation.instant.progress
-    - navigation.tracking
-    - navigation.tabs
-    - navigation.tabs.sticky
-    - navigation.sections
-    - navigation.indexes
-    - navigation.top
-    - search.suggest
-    - search.highlight
-    - search.share
-    - content.code.annotate
-    - content.tabs
-    - content.tooltips
-    - toc.follow
-  icon:
-    repo: fontawesome/brands/github
-
-hooks:
-  - docs/mkdocs/hooks/generate_examples.py
-
-# Plugins
-plugins:
-  - meta
-  - search
-  # - awesome-nav  # Uncomment if installed
-  - glightbox
-  - git-revision-date-localized:
-      # exclude files
-      exclude:
-        - api/*
-        - user_guide/examples/**
-        - contributing/design_documents/api_design_template.md
-        - DOCS_GUIDE.md
-  - minify:
-      minify_html: true
-      minify_js: true
-      minify_css: true
-      cache_safe: true
-  - api-autonav:
-      modules: ["vllm_omni"]
-      api_root_uri: "api"
-      nav_item_prefix: ""  # No prefix in navigation tree (clean names)
-      show_full_namespace: false  # Show only module name, not full path
-      exclude:
-        - "re:vllm_omni\\._.*"  # Internal modules
-  - mkdocstrings:
-      handlers:
-        python:
-          paths: [vllm_omni]
-          options:
-            show_symbol_type_heading: true
-            show_symbol_type_toc: true
-            show_source: true
-            show_root_heading: true
-            show_root_toc_entry: true
-            show_root_full_path: false
-            show_object_full_path: false
-            show_if_no_docstring: true
-            show_signature_annotations: true
-            separate_signature: true
-            show_overloads: true
-            signature_crossrefs: true
-            filters: []
-            summary:
-              modules: true
-          inventories:
-            - https://docs.python.org/3/objects.inv
-            - https://typing-extensions.readthedocs.io/en/latest/objects.inv
-            - https://docs.aiohttp.org/en/stable/objects.inv
-            - https://pillow.readthedocs.io/en/stable/objects.inv
-            - https://numpy.org/doc/stable/objects.inv
-            - https://pytorch.org/docs/stable/objects.inv
-            - https://psutil.readthedocs.io/en/stable/objects.inv
-
-# Markdown extensions
-markdown_extensions:
-  - attr_list
-  - md_in_html
-  - admonition
-  - pymdownx.details
-  # For content tabs
-  - pymdownx.superfences
-  - pymdownx.tabbed:
-      slugify: !!python/object/apply:pymdownx.slugs.slugify
-        kwds:
-          case: lower
-      alternate_style: true
-  # For code highlighting
-  - pymdownx.highlight:
-      anchor_linenums: true
-      line_spans: __span
-      pygments_lang_class: true
-  - pymdownx.inlinehilite
-  - pymdownx.snippets
-  # For emoji and icons
-  - pymdownx.emoji:
-      emoji_index: !!python/name:material.extensions.emoji.twemoji
-      emoji_generator: !!python/name:material.extensions.emoji.to_svg
-  # For in page [TOC] (not sidebar)
-  - toc:
-      permalink: true
-  # For math rendering
-  - pymdownx.arithmatex:
-      generic: true
-  # For task lists
-  - pymdownx.tasklist:
-      custom_checkbox: true
-  # For tables
-  - tables
-
-# Navigation
-nav:
-  - Home: index.md
-  - User Guide:
-    - Getting Started:
-      - user_guide/getting_started/quickstart.md
-    - Examples:
-      - user_guide/examples/index.md
-      - Offline Inference:
-        - user_guide/examples/offline_inference/qwen_2_5_omni.md
-      - Online Serving:
-        - user_guide/examples/online_serving/index.md
-  - API Reference:
-      - Summary: api/overview.md
-  - Developer Guide:
-    - contributing/README.md
-    - Design Documents:
-      - contributing/design_documents/index.md
-      - contributing/design_documents/vllm_omni_design.md
-      - contributing/design_documents/architecture_overview.md
-      - contributing/design_documents/api_design_doc.md
-
-# Extra
-extra:
-  version:
-    provider: mike
-
-# Extra CSS
-extra_css:
-  - mkdocs/stylesheets/extra.css
-
-# Makes the url format end in .html rather than act as a dir
-# So index.md generates as index.html and is available under URL /index.html
-# https://www.mkdocs.org/user-guide/configuration/#use_directory_urls
-use_directory_urls: false
@@ -4,13 +4,13 @@ build-backend = "setuptools.build_meta"

 [project]
 name = "vllm-omni"
-version = "0.11.0rc1"
+version = "0.1.0"
 description = "vLLM-omni: Multi-modality models inference and serving with non-autoregressive structures"
 readme = "README.md"
-requires-python = ">=3.9,<3.14"  # Align with vLLM v0.11 supported Python versions
+requires-python = ">=3.8"
 license = {text = "Apache-2.0"}
 authors = [
-    {name = "vLLM-omni Team", email = "vllm-project@gmail.com"}
+    {name = "vLLM-omni Team", email = "hsliuustc@gmail.com"}
 ]
 keywords = ["vllm", "multimodal", "diffusion", "transformer", "inference", "serving"]
 classifiers = [
@@ -19,108 +19,92 @@ classifiers = [
    "Intended Audience :: Science/Research",
    "License :: OSI Approved :: Apache Software License",
    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.8",
    "Programming Language :: Python :: 3.9",
    "Programming Language :: Python :: 3.10",
    "Programming Language :: Python :: 3.11",
-    "Programming Language :: Python :: 3.12",
    "Topic :: Scientific/Engineering :: Artificial Intelligence",
    "Topic :: Software Development :: Libraries :: Python Modules",
 ]

 dependencies = [
-    # Core runtime dependencies (required for actual usage)
-    "omegaconf>=2.3.0",
-    # "decord>=0.6.0", # install manually
-    "librosa>=0.11.0",
-    "resampy>=0.4.3",
-    # "vllm==0.11.0",  # TODO: fix the entrypoints overwrite problem
+    "vllm>=0.2.0",
+    "torch>=2.0.0",
+    "transformers>=4.30.0",
+    "numpy>=1.21.0",
+    "pillow>=8.0.0",
+    "opencv-python>=4.5.0",
+    "PyYAML>=6.0",
+    "gradio>=3.0.0",
+    "fastapi>=0.100.0",
+    "uvicorn>=0.20.0",
+    "pydantic>=2.0.0",
+    "asyncio-mqtt>=0.11.0",
+    "ray>=2.0.0",
 ]

 [project.optional-dependencies]
-
 dev = [
    "pytest>=7.0.0",
    "pytest-asyncio>=0.21.0",
    "pytest-cov>=4.0.0",
    "black>=23.0.0",
    "isort>=5.12.0",
+    "flake8>=6.0.0",
    "mypy>=1.0.0",
    "pre-commit>=3.0.0",
-    "gradio>=5.49.1",
-    "soundfile>=0.13.1"
 ]
-
 docs = [
-    "mkdocs>=1.5.0",
-    "mkdocs-api-autonav",
-    "mkdocs-material",
-    "mkdocstrings-python",
-    "mkdocs-gen-files",
-    # "mkdocs-awesome-nav",  # Not available on PyPI
-    "mkdocs-glightbox",
-    "mkdocs-git-revision-date-localized-plugin",
-    "mkdocs-minify-plugin",
-    "regex",
-    "pydantic",
+    "sphinx>=6.0.0",
+    "sphinx-rtd-theme>=1.2.0",
+    "myst-parser>=1.0.0",
+]
+examples = [
+    "jupyter>=1.0.0",
+    "matplotlib>=3.5.0",
+    "seaborn>=0.11.0",
 ]
-

 [project.urls]
-Homepage = "https://github.com/vllm-project/vllm-omni"
-Repository = "https://github.com/vllm-project/vllm-omni"
+Homepage = "https://github.com/hsliuustc0106/vllm-omni"
+Repository = "https://github.com/hsliuustc0106/vllm-omni"
 Documentation = "https://vllm-omni.readthedocs.io"
-"Bug Tracker" = "https://github.com/vllm-project/vllm-omni/issues"
+"Bug Tracker" = "https://github.com/hsliuustc0106/vllm-omni/issues"

 [project.scripts]
 vllm = "vllm_omni.entrypoints.cli.main:main"
 vllm-omni = "vllm_omni.entrypoints.cli.main:main"

-
 [tool.setuptools.packages.find]
 where = ["."]
 include = ["vllm_omni*"]

-[tool.setuptools.package-data]
-"vllm_omni.model_executor.stage_configs" = ["*.yaml"]
+[tool.black]
+line-length = 88
+target-version = ['py38', 'py39', 'py310', 'py311']
+include = '\.pyi?$'
+extend-exclude = '''
+/(
+  # directories
+  \.eggs
+  | \.git
+  | \.hg
+  | \.mypy_cache
+  | \.tox
+  | \.venv
+  | build
+  | dist
+)/
+'''

-[tool.ruff]
-line-length = 120
-exclude = [
-    ".eggs",
-    ".git",
-    ".hg",
-    ".mypy_cache",
-    ".tox",
-    ".venv",
-    "build",
-    "dist",
-    "vllm_omni.egg-info",
-]
-
-[tool.ruff.lint]
-select = [
-    "E",  # pycodestyle errors
-    "W",  # pycodestyle warnings
-    "F",  # pyflakes
-    "I",  # isort (handled separately, but included for compatibility)
-    "N",  # pep8-naming
-    "UP", # pyupgrade
-]
-ignore = [
-    "E203",  # whitespace before ':' (conflicts with black)
-    # W503 is not needed in ruff as it's compatible with black by default
-    "N801",  # class names should use CapWords convention
-    "N802",  # function name should follow snake_case
-    "N806",  # variable in function should follow snake_case
-    "N812",  # lowercase imported as non-lowercase: functional as F
-]
-
-[tool.ruff.lint.per-file-ignores]
-"examples/**" = ["E501"]  # Allow long lines in examples
-"tests/**" = ["E501"]  # Allow long lines in tests
+[tool.isort]
+profile = "black"
+multi_line_output = 3
+line_length = 88
+known_first_party = ["vllm_omni"]

 [tool.mypy]
-python_version = "3.12, 3.13"
+python_version = "3.8"
 warn_return_any = true
 warn_unused_configs = true
 disallow_untyped_defs = true
@@ -153,11 +137,3 @@ markers = [
    "benchmark: Benchmark tests",
    "slow: Slow tests",
 ]
-
-[tool.typos.default]
-extend-ignore-identifiers-re = [
-    ".*_thw",
-    ".*thw",
-    "ein",
-    ".*arange"
-]
@@ -0,0 +1,27 @@
+# Core dependencies
+vllm>=0.10.2
+torch>=2.7
+transformers>=4.30.0
+numpy>=1.21.0
+pillow>=8.0.0
+opencv-python>=4.5.0
+PyYAML>=6.0
+
+# API and serving
+fastapi>=0.100.0
+uvicorn>=0.20.0
+pydantic>=2.0.0
+gradio>=3.0.0
+
+# Async and distributed computing
+asyncio-mqtt>=0.11.0
+ray>=2.0.0
+
+# Development dependencies
+pytest>=7.0.0
+pytest-asyncio>=0.21.0
+pytest-cov>=4.0.0
+black>=23.0.0
+isort>=5.12.0
+flake8>=6.0.0
+mypy>=1.0.0
@@ -0,0 +1,109 @@
+# vLLM-omni Testing Scripts
+
+This directory contains automated testing scripts for vLLM-omni serving functionality.
+
+## Scripts Overview
+
+### 1. `test_serving.sh` - Comprehensive Test Suite
+Full-featured testing script that performs complete validation of the vLLM-omni serving functionality.
+
+**Features:**
+- Model existence validation
+- Environment setup verification
+- Import functionality testing
+- Server startup and health checks
+- Text generation testing
+- Performance benchmarking
+- API client integration testing
+- Comprehensive logging and error handling
+
+**Usage:**
+```bash
+# Use default model and port
+./scripts/test_serving.sh
+
+# Use specific model and port
+./scripts/test_serving.sh ./models/Qwen3-0.6B 8000
+
+# Use HuggingFace model
+./scripts/test_serving.sh Qwen/Qwen3-0.6B 8001
+
+# Show help
+./scripts/test_serving.sh --help
+```
+
+**Test Coverage:**
+- ✅ Model loading and server startup
+- ✅ Health and info endpoints
+- ✅ Text generation functionality
+- ✅ Performance metrics
+- ✅ API client integration
+- ✅ All imports working correctly
+
+### 2. `quick_test.sh` - Fast Validation
+Lightweight script for quick validation after making changes.
+
+**Features:**
+- Fast import testing
+- Basic server startup
+- Health endpoint validation
+- Simple text generation test
+- Retry mechanism for reliability
+
+**Usage:**
+```bash
+# Use default port (8000)
+./scripts/quick_test.sh
+
+# Use specific port
+./scripts/quick_test.sh 8001
+```
+
+**Test Coverage:**
+- ✅ Import functionality
+- ✅ Server startup
+- ✅ Health endpoint
+- ✅ Basic text generation
+
+## Prerequisites
+
+1. **Conda Environment**: Ensure `vllm_omni` environment is activated
+2. **Model Available**: Qwen3-0.6B model should be available in `./models/Qwen3-0.6B/`
+3. **Dependencies**: All vLLM-omni dependencies should be installed
+
+## Environment Setup
+
+```bash
+# Activate conda environment
+conda activate vllm_omni
+
+# Verify installation
+python -c "import vllm_omni; print('vLLM-omni ready')"
+```
+
+## Model Setup
+
+The scripts expect the Qwen3-0.6B model to be available. You can:
+
+1. **Use local model**: Place model in `./models/Qwen3-0.6B/`
+2. **Use HuggingFace model**: Pass `Qwen/Qwen3-0.6B` as model path
+3. **Download model**: Use the `download_models.py` script
+
+```bash
+# Download model using the provided script
+python scripts/download_models.py
+```
+
+## Output and Logging
+
+### Test Results
+- **Success**: Green `[SUCCESS]` messages
+- **Info**: Blue `[INFO]` messages  
+- **Warnings**: Yellow `[WARNING]` messages
+- **Errors**: Red `[ERROR]` messages
+
+### Log Files
+- `server.log`: Server startup and runtime logs
+- `api_client_test.log`: API client test results
+- `simple_usage_test.log`: Simple usage test results
+
@@ -1,139 +0,0 @@
-#!/usr/bin/env bash
-
-set -euo pipefail
-
-SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
-REPO_ROOT="$(cd -- "${SCRIPT_DIR}/.." && pwd)"
-PROJECT_NAME="vllm-omni"
-RUN_QUALITY=false
-SKIP_CLEAN=false
-CREATE_VENV=false
-VENV_DIR=".venv-build"
-PYTHON_BIN="python"
-UV_BIN="uv"
-
-log() {
-  local level="$1"
-  shift
-  printf '[%s] %s\n' "${level}" "$*"
-}
-
-abort() {
-  log "ERROR" "$*"
-  exit 1
-}
-
-usage() {
-  cat <<EOF
-Usage: $(basename "$0") [options]
-
-Options:
-  --run-quality    Run pre-commit, install dev deps, and pytest before building
-  --skip-clean     Skip removing previous build artifacts
-  --create-venv    Build inside a fresh virtual environment (default path: .venv-build)
-  --venv-dir PATH  Custom directory for the virtual environment (implies --create-venv)
-  --python PATH    Python executable to use (default: python)
-  -h, --help       Show this help message
-EOF
-}
-
-while [[ $# -gt 0 ]]; do
-  case "$1" in
-    --run-quality)
-      RUN_QUALITY=true
-      ;;
-    --skip-clean)
-      SKIP_CLEAN=true
-      ;;
-    --create-venv)
-      CREATE_VENV=true
-      ;;
-    --venv-dir)
-      CREATE_VENV=true
-      shift
-      [[ $# -gt 0 ]] || abort "--venv-dir requires a path"
-      VENV_DIR="$1"
-      ;;
-    --python)
-      shift
-      [[ $# -gt 0 ]] || abort "--python requires a path"
-      PYTHON_BIN="$1"
-      ;;
-    -h|--help)
-      usage
-      exit 0
-      ;;
-    *)
-      usage
-      abort "Unknown option: $1"
-      ;;
-  esac
-  shift
-done
-
-HOST_PYTHON="${PYTHON_BIN}"
-
-log "INFO" "Switching to repository root: ${REPO_ROOT}"
-cd "${REPO_ROOT}" || abort "Cannot enter repository root"
-
-[[ -f pyproject.toml ]] || abort "pyproject.toml not found, please ensure correct script location"
-
-ensure_uv() {
-  if ! command -v "${UV_BIN}" >/dev/null 2>&1; then
-    log "INFO" "uv not found, installing via ${HOST_PYTHON}"
-    "${HOST_PYTHON}" -m pip install --upgrade pip
-    "${HOST_PYTHON}" -m pip install uv
-  fi
-}
-
-ensure_uv
-
-if [[ "${CREATE_VENV}" == "true" ]]; then
-  log "INFO" "Creating fresh virtual environment at ${VENV_DIR} via uv"
-  "${UV_BIN}" venv --python "${HOST_PYTHON}" --seed "${VENV_DIR}"
-  PYTHON_BIN="${VENV_DIR}/bin/python"
-  [[ -x "${PYTHON_BIN}" ]] || abort "Failed to locate python inside ${VENV_DIR}"
-  log "INFO" "Installing build module inside virtual environment"
-  "${UV_BIN}" pip install --python "${PYTHON_BIN}" build
-else
-  log "INFO" "Ensuring build module is available via uv pip"
-  "${UV_BIN}" pip install --python "${PYTHON_BIN}" build
-fi
-
-log "INFO" "Checking build module"
-if ! "${PYTHON_BIN}" -m build --version >/dev/null 2>&1; then
-  abort "${PYTHON_BIN} -m build is not available, install build first"
-fi
-
-run_quality_steps() {
-  log "INFO" "Running quality checks"
-  "${UV_BIN}" pip install --python "${PYTHON_BIN}" -e ".[dev]"
-  "${PYTHON_BIN}" -m pre_commit run --all-files
-  "${PYTHON_BIN}" -m pytest tests/ -v -m "not slow"
-}
-
-if [[ "${RUN_QUALITY}" == "true" ]]; then
-  run_quality_steps
-else
-  log "INFO" "Quality steps available via --run-quality"
-  log "INFO" "  - pre-commit run --all-files"
-  log "INFO" "  - pip install -e '.[dev]'"
-  log "INFO" "  - pytest tests/ -v -m \"not slow\""
-fi
-
-cleanup_artifacts() {
-  log "INFO" "Cleaning previous build artifacts"
-  rm -rf build dist "${PROJECT_NAME}.egg-info" "${PROJECT_NAME//-/_}.egg-info"
-}
-
-if [[ "${SKIP_CLEAN}" == "true" ]]; then
-  log "INFO" "Skipping cleanup as requested"
-else
-  cleanup_artifacts
-fi
-
-log "INFO" "Building source and wheel distributions"
-"${PYTHON_BIN}" -m build
-
-log "INFO" "Build finished, artifacts:"
-ls -lh dist
@@ -0,0 +1,363 @@
+#!/bin/bash
+
+# vLLM-omni Serving Functionality Test Script
+# This script tests the complete serving functionality of vLLM-omni
+# Usage: ./scripts/test_serving.sh [model_path] [port]
+
+set -e  # Exit on any error
+
+# Configuration
+MODEL_PATH=${1:-"./models/Qwen3-0.6B"}
+PORT=${2:-8000}
+HOST="localhost"
+TIMEOUT=30
+SERVER_STARTUP_WAIT=15
+
+# Colors for output
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+BLUE='\033[0;34m'
+NC='\033[0m' # No Color
+
+# Logging functions
+log_info() {
+    echo -e "${BLUE}[INFO]${NC} $1"
+}
+
+log_success() {
+    echo -e "${GREEN}[SUCCESS]${NC} $1"
+}
+
+log_warning() {
+    echo -e "${YELLOW}[WARNING]${NC} $1"
+}
+
+log_error() {
+    echo -e "${RED}[ERROR]${NC} $1"
+}
+
+# Cleanup function
+cleanup() {
+    log_info "Cleaning up..."
+    if [ ! -z "$SERVER_PID" ]; then
+        log_info "Stopping server (PID: $SERVER_PID)..."
+        kill $SERVER_PID 2>/dev/null || true
+        wait $SERVER_PID 2>/dev/null || true
+    fi
+    # Kill any remaining vllm processes
+    pkill -f "vllm serve" 2>/dev/null || true
+    log_info "Cleanup completed"
+}
+
+# Set up trap for cleanup on exit
+trap cleanup EXIT
+
+# Check if model exists
+check_model() {
+    log_info "Checking if model exists at: $MODEL_PATH"
+    if [ ! -d "$MODEL_PATH" ]; then
+        log_error "Model directory not found: $MODEL_PATH"
+        log_info "Available models:"
+        ls -la models/ 2>/dev/null || log_warning "No models directory found"
+        exit 1
+    fi
+    log_success "Model found: $MODEL_PATH"
+}
+
+# Check if conda environment is activated
+check_environment() {
+    log_info "Checking conda environment..."
+    if [[ "$CONDA_DEFAULT_ENV" != "vllm_omni" ]]; then
+        log_warning "vllm_omni conda environment not activated"
+        log_info "Attempting to activate vllm_omni environment..."
+        source $(conda info --base)/etc/profile.d/conda.sh
+        conda activate vllm_omni
+        if [[ "$CONDA_DEFAULT_ENV" != "vllm_omni" ]]; then
+            log_error "Failed to activate vllm_omni environment"
+            exit 1
+        fi
+    fi
+    log_success "Environment check passed: $CONDA_DEFAULT_ENV"
+}
+
+# Test import functionality
+test_imports() {
+    log_info "Testing imports..."
+    python -c "
+import vllm_omni
+from vllm_omni.entrypoints.omni_llm import OmniLLM, AsyncOmniLLM
+print('✅ All imports successful')
+" || {
+        log_error "Import test failed"
+        exit 1
+    }
+    log_success "Import test passed"
+}
+
+# Start the server
+start_server() {
+    log_info "Starting vLLM-omni server on port $PORT..."
+    log_info "Command: vllm serve $MODEL_PATH --omni --port $PORT"
+    
+    # Start server in background
+    vllm serve "$MODEL_PATH" --omni --port "$PORT" > server.log 2>&1 &
+    SERVER_PID=$!
+    
+    log_info "Server started with PID: $SERVER_PID"
+    log_info "Waiting $SERVER_STARTUP_WAIT seconds for server to initialize..."
+    sleep $SERVER_STARTUP_WAIT
+    
+    # Check if server is still running
+    if ! kill -0 $SERVER_PID 2>/dev/null; then
+        log_error "Server failed to start"
+        log_info "Server logs:"
+        cat server.log
+        exit 1
+    fi
+    
+    log_success "Server appears to be running"
+}
+
+# Test health endpoint
+test_health() {
+    log_info "Testing health endpoint..."
+    for attempt in {1..10}; do
+        local response=$(curl -s -w "%{http_code}" http://$HOST:$PORT/health 2>/dev/null || echo "000")
+        local http_code="${response: -3}"
+        local body="${response%???}"
+
+        if [ "$http_code" = "200" ]; then
+            log_success "Health check passed: $body"
+            return 0
+        fi
+
+        log_warning "Health check attempt $attempt failed (HTTP $http_code). Retrying in 3s..."
+        sleep 3
+    done
+
+    log_error "Health check failed after multiple attempts"
+    return 1
+}
+
+# Test info endpoint
+test_info() {
+    log_info "Testing info endpoint..."
+    local response=$(curl -s -w "%{http_code}" http://$HOST:$PORT/info 2>/dev/null || echo "000")
+    local http_code="${response: -3}"
+    local body="${response%???}"
+    
+    if [ "$http_code" = "200" ]; then
+        log_success "Info endpoint working"
+        echo "$body" | python -m json.tool 2>/dev/null || log_warning "Info response not valid JSON"
+    else
+        log_error "Info endpoint failed (HTTP $http_code): $body"
+        return 1
+    fi
+}
+
+# Test text generation
+test_generation() {
+    log_info "Testing text generation..."
+    
+    # Test 1: Simple generation
+    local response=$(curl -s -X POST http://$HOST:$PORT/generate \
+        -H "Content-Type: application/json" \
+        -d '{"prompts": ["Test the server functionality"], "max_tokens": 20, "temperature": 0.7}' \
+        2>/dev/null || echo "{}")
+    
+    if echo "$response" | python -c "import sys, json; data=json.load(sys.stdin); exit(0 if 'outputs' in data and len(data['outputs']) > 0 else 1)" 2>/dev/null; then
+        log_success "Text generation test passed"
+        echo "$response" | python -c "import sys, json; data=json.load(sys.stdin); print('Generated text:', data['outputs'][0]['text'][:100] + '...' if len(data['outputs'][0]['text']) > 100 else data['outputs'][0]['text'])" 2>/dev/null
+    else
+        log_error "Text generation test failed"
+        echo "Response: $response"
+        return 1
+    fi
+}
+
+# Test API client example
+test_api_client() {
+    log_info "Testing API client example..."
+    if [ -f "examples/basic/api_client.py" ]; then
+        python examples/basic/api_client.py > api_client_test.log 2>&1 || {
+            log_error "API client test failed"
+            log_info "API client logs:"
+            cat api_client_test.log
+            return 1
+        }
+        log_success "API client test passed"
+    else
+        log_warning "API client example not found, skipping"
+    fi
+}
+
+# Test AR → DiT pipeline example
+test_ar_dit_pipeline() {
+    log_info "Testing AR → DiT diffusers pipeline example..."
+
+    if ! python - 2>/dev/null <<'PY'
+import importlib
+for pkg in ("torch", "diffusers"):
+    if importlib.util.find_spec(pkg) is None:
+        raise ImportError(pkg)
+PY
+    then
+        log_warning "Required packages for diffusers pipeline not found, skipping AR → DiT test"
+        return 0
+    fi
+
+    if [ ! -f "examples/omni/ar_dit_diffusers.py" ]; then
+        log_warning "AR → DiT example not found, skipping"
+        return 0
+    fi
+
+    local output_path="logs/ar_dit_pipeline.png"
+    local log_path="logs/ar_dit_pipeline.log"
+    rm -f "$output_path"
+
+    local cmd=(python examples/omni/ar_dit_diffusers.py
+        --prompt "Smoke test prompt of a lighthouse"
+        --seed 0
+        --output "$output_path")
+
+    local env_prefix=()
+    if [ -n "$AR_MODEL" ]; then
+        env_prefix+=("AR_MODEL=$AR_MODEL")
+    fi
+    if [ -n "$DIT_MODEL" ]; then
+        env_prefix+=("DIT_MODEL=$DIT_MODEL")
+    fi
+
+    if ! env "${env_prefix[@]}" "${cmd[@]}" >"$log_path" 2>&1; then
+        log_error "AR → DiT pipeline test failed"
+        log_info "AR → DiT logs:"
+        cat "$log_path"
+        return 1
+    fi
+
+    if [ ! -f "$output_path" ]; then
+        log_error "AR → DiT pipeline did not produce an image"
+        log_info "AR → DiT logs:"
+        cat "$log_path"
+        return 1
+    fi
+
+    log_success "AR → DiT pipeline test passed (output: $output_path)"
+}
+
+# Test simple usage example
+test_simple_usage() {
+    log_info "Testing simple usage example..."
+    if [ -f "examples/basic/simple_usage.py" ]; then
+        # Run with timeout to avoid hanging
+        timeout 60 python examples/basic/simple_usage.py > simple_usage_test.log 2>&1 || {
+            local exit_code=$?
+            if [ $exit_code -eq 124 ]; then
+                log_warning "Simple usage test timed out (60s), but this might be normal for model loading"
+            else
+                log_error "Simple usage test failed (exit code: $exit_code)"
+                log_info "Simple usage logs:"
+                cat simple_usage_test.log
+                return 1
+            fi
+        }
+        log_success "Simple usage test passed"
+    else
+        log_warning "Simple usage example not found, skipping"
+    fi
+}
+
+# Performance test
+test_performance() {
+    log_info "Running performance test..."
+    
+    local start_time=$(date +%s)
+    local response=$(curl -s -X POST http://$HOST:$PORT/generate \
+        -H "Content-Type: application/json" \
+        -d '{"prompts": ["Performance test prompt"], "max_tokens": 50, "temperature": 0.7}' \
+        2>/dev/null)
+    local end_time=$(date +%s)
+    local duration=$((end_time - start_time))
+    
+    if echo "$response" | python -c "import sys, json; data=json.load(sys.stdin); exit(0 if 'outputs' in data else 1)" 2>/dev/null; then
+        log_success "Performance test passed (${duration}s response time)"
+    else
+        log_error "Performance test failed"
+        return 1
+    fi
+}
+
+# Main test function
+run_tests() {
+    log_info "Starting vLLM-omni serving functionality tests..."
+    log_info "Model: $MODEL_PATH"
+    log_info "Port: $PORT"
+    log_info "Host: $HOST"
+    echo "=========================================="
+    
+    # Run all tests
+    check_model
+    check_environment
+    test_imports
+    start_server
+    
+    # Wait a bit more for server to be fully ready
+    sleep 5
+    
+    test_health
+    test_info
+    test_generation
+    test_performance
+    test_api_client
+    test_ar_dit_pipeline
+
+    # Note: Simple usage test is commented out as it starts its own server
+    # test_simple_usage
+
+    log_success "All tests completed successfully!"
+    echo "=========================================="
+    log_info "Test Summary:"
+    log_info "✅ Model loading and server startup"
+    log_info "✅ Health and info endpoints"
+    log_info "✅ Text generation functionality"
+    log_info "✅ Performance metrics"
+    log_info "✅ API client integration"
+    log_info "✅ AR → DiT diffusers pipeline"
+    log_info "✅ All imports working correctly"
+}
+
+# Show usage
+show_usage() {
+    echo "Usage: $0 [model_path] [port]"
+    echo ""
+    echo "Arguments:"
+    echo "  model_path    Path to the model directory (default: ./models/Qwen3-0.6B)"
+    echo "  port          Port to run the server on (default: 8000)"
+    echo ""
+    echo "Examples:"
+    echo "  $0                                    # Use default model and port"
+    echo "  $0 ./models/Qwen3-0.6B 8001          # Use specific model and port"
+    echo "  $0 Qwen/Qwen3-0.6B 8000              # Use HuggingFace model"
+}
+
+# Main execution
+main() {
+    # Check for help flag
+    if [[ "$1" == "-h" || "$1" == "--help" ]]; then
+        show_usage
+        exit 0
+    fi
+    
+    # Change to script directory
+    cd "$(dirname "$0")/.."
+    
+    # Create logs directory if it doesn't exist
+    mkdir -p logs
+    
+    # Run tests
+    run_tests
+}
+
+# Run main function with all arguments
+main "$@"
@@ -1,6 +1,5 @@
 """
 Test suite for vLLM-omni.

-This package contains unit tests, integration tests, and benchmarks
-for vLLM-omni.
+This package contains unit tests, integration tests, and benchmarks for vLLM-omni.
 """
@@ -0,0 +1,82 @@
+"""
+Shared fixtures and configuration for vLLM-omni tests.
+"""
+
+import pytest
+import torch
+from unittest.mock import Mock
+from vllm_omni.config import OmniStageConfig, DiTConfig, DiTCacheConfig, create_ar_stage_config, create_dit_stage_config
+
+
+@pytest.fixture(scope="session")
+def device():
+    """Get available device for testing."""
+    return "cuda" if torch.cuda.is_available() else "cpu"
+
+
+@pytest.fixture
+def sample_ar_stage_config():
+    """Sample AR stage configuration for testing."""
+    return create_ar_stage_config(
+        stage_id=0,
+        model_path="test-ar-model",
+        input_modalities=["text"],
+        output_modalities=["text"]
+    )
+
+
+@pytest.fixture
+def sample_dit_stage_config():
+    """Sample DiT stage configuration for testing."""
+    dit_config = DiTConfig(
+        model_type="dit",
+        scheduler_type="ddpm",
+        num_inference_steps=10,
+        guidance_scale=7.5
+    )
+    
+    return create_dit_stage_config(
+        stage_id=1,
+        model_path="test-dit-model",
+        input_modalities=["text"],
+        output_modalities=["image"],
+        dit_config=dit_config
+    )
+
+
+@pytest.fixture
+def sample_stage_configs(sample_ar_stage_config, sample_dit_stage_config):
+    """Sample stage configurations for testing."""
+    return [sample_ar_stage_config, sample_dit_stage_config]
+
+
+@pytest.fixture
+def mock_vllm_config():
+    """Mock vLLM configuration."""
+    config = Mock()
+    config.model = "test-model"
+    config.tensor_parallel_size = 1
+    config.pipeline_parallel_size = 1
+    return config
+
+
+@pytest.fixture
+def mock_dit_cache_config():
+    """Mock DiT cache configuration."""
+    from vllm_omni.config import DiTCacheTensor
+    
+    cache_tensors = [
+        DiTCacheTensor(
+            name="test_tensor",
+            shape=[1, 512, 512],
+            dtype="float32",
+            persistent=True
+        )
+    ]
+    
+    return DiTCacheConfig(
+        cache_tensors=cache_tensors,
+        max_cache_size=1024 * 1024 * 1024,  # 1GB
+        cache_strategy="fifo",
+        enable_optimization=True
+    )
@@ -0,0 +1,179 @@
+"""
+Unit tests for configuration modules.
+"""
+
+import pytest
+from vllm_omni.config import (
+    OmniStageConfig,
+    DiTConfig,
+    DiTCacheConfig,
+    DiTCacheTensor,
+    create_ar_stage_config,
+    create_dit_stage_config,
+)
+
+
+class TestOmniStageConfig:
+    def test_ar_stage_config_creation(self):
+        """Test AR stage configuration creation."""
+        config = OmniStageConfig(
+            stage_id=0,
+            engine_type="AR",
+            model_path="test-model",
+            input_modalities=["text"],
+            output_modalities=["text"]
+        )
+        assert config.engine_type == "AR"
+        assert config.stage_id == 0
+        assert config.model_path == "test-model"
+        assert config.input_modalities == ["text"]
+        assert config.output_modalities == ["text"]
+    
+    def test_dit_stage_config_creation(self):
+        """Test DiT stage configuration creation."""
+        dit_config = DiTConfig(
+            model_type="dit",
+            scheduler_type="ddpm",
+            num_inference_steps=50
+        )
+        config = OmniStageConfig(
+            stage_id=1,
+            engine_type="DiT",
+            model_path="test-dit-model",
+            input_modalities=["text"],
+            output_modalities=["image"],
+            dit_config=dit_config
+        )
+        assert config.engine_type == "DiT"
+        assert config.dit_config is not None
+        assert config.dit_config.model_type == "dit"
+    
+    def test_invalid_engine_type(self):
+        """Test validation of engine type."""
+        with pytest.raises(ValueError):
+            OmniStageConfig(
+                stage_id=0,
+                engine_type="INVALID",
+                model_path="test-model",
+                input_modalities=["text"],
+                output_modalities=["text"]
+            )
+    
+    def test_empty_modalities(self):
+        """Test validation of empty modalities."""
+        with pytest.raises(ValueError):
+            OmniStageConfig(
+                stage_id=0,
+                engine_type="AR",
+                model_path="test-model",
+                input_modalities=[],
+                output_modalities=["text"]
+            )
+        
+        with pytest.raises(ValueError):
+            OmniStageConfig(
+                stage_id=0,
+                engine_type="AR",
+                model_path="test-model",
+                input_modalities=["text"],
+                output_modalities=[]
+            )
+    
+    def test_dit_requires_config(self):
+        """Test that DiT engine requires dit_config."""
+        with pytest.raises(ValueError):
+            OmniStageConfig(
+                stage_id=1,
+                engine_type="DiT",
+                model_path="test-dit-model",
+                input_modalities=["text"],
+                output_modalities=["image"]
+            )
+
+
+class TestDiTConfig:
+    def test_dit_config_defaults(self):
+        """Test DiT configuration with defaults."""
+        config = DiTConfig(
+            model_type="dit",
+            scheduler_type="ddpm",
+            num_inference_steps=50
+        )
+        assert config.use_diffusers is False
+        assert config.diffusers_pipeline is None
+        assert config.guidance_scale == 7.5
+        assert config.height == 512
+        assert config.width == 512
+    
+    def test_diffusers_config(self):
+        """Test DiT configuration with diffusers."""
+        config = DiTConfig(
+            model_type="dit",
+            scheduler_type="ddpm",
+            num_inference_steps=50,
+            use_diffusers=True,
+            diffusers_pipeline="stable-diffusion"
+        )
+        assert config.use_diffusers is True
+        assert config.diffusers_pipeline == "stable-diffusion"
+
+
+class TestDiTCacheConfig:
+    def test_cache_config_creation(self):
+        """Test cache configuration creation."""
+        cache_tensors = [
+            DiTCacheTensor(
+                name="test_tensor",
+                shape=[1, 512, 512],
+                dtype="float32",
+                persistent=True
+            )
+        ]
+        
+        config = DiTCacheConfig(
+            cache_tensors=cache_tensors,
+            max_cache_size=1024 * 1024 * 1024,
+            cache_strategy="fifo"
+        )
+        
+        assert len(config.cache_tensors) == 1
+        assert config.max_cache_size == 1024 * 1024 * 1024
+        assert config.cache_strategy == "fifo"
+        assert config.enable_optimization is True
+
+
+class TestHelperFunctions:
+    def test_create_ar_stage_config(self):
+        """Test create_ar_stage_config helper function."""
+        config = create_ar_stage_config(
+            stage_id=0,
+            model_path="test-model"
+        )
+        
+        assert config.stage_id == 0
+        assert config.engine_type == "AR"
+        assert config.model_path == "test-model"
+        assert config.input_modalities == ["text"]
+        assert config.output_modalities == ["text"]
+    
+    def test_create_dit_stage_config(self):
+        """Test create_dit_stage_config helper function."""
+        dit_config = DiTConfig(
+            model_type="dit",
+            scheduler_type="ddpm",
+            num_inference_steps=50
+        )
+        
+        config = create_dit_stage_config(
+            stage_id=1,
+            model_path="test-dit-model",
+            dit_config=dit_config
+        )
+        
+        assert config.stage_id == 1
+        assert config.engine_type == "DiT"
+        assert config.model_path == "test-dit-model"
+        assert config.input_modalities == ["text"]
+        assert config.output_modalities == ["image"]
+        assert config.dit_config is not None
+
@@ -1,41 +1,45 @@
 """
-vLLM-omni: Multi-modality models inference and serving with
-non-autoregressive structures.
+vLLM-omni: Multi-modality models inference and serving with non-autoregressive structures.

-This package extends vLLM beyond traditional text-based, autoregressive
-generation to support multi-modality models with non-autoregressive
-structures and non-textual outputs.
+This package extends vLLM beyond traditional text-based, autoregressive generation
+to support multi-modality models with non-autoregressive structures and non-textual outputs.

 Architecture:
- 🟡 Modified: vLLM components modified for multimodal support
- 🔴 Added: New components for multimodal and non-autoregressive
-  processing
+- 🟡 Modified: vLLM components modified for multimodal support  
+- 🔴 Added: New components for multimodal and non-autoregressive processing
 """

-try:
-    from . import patch  # noqa: F401
-except ModuleNotFoundError as exc:  # pragma: no cover - optional dependency
-    if exc.name != "vllm":
-        raise
-    # Allow importing vllm_omni without vllm (e.g., documentation builds)
-    patch = None  # type: ignore
-from .config import OmniModelConfig
-from .entrypoints.async_omni_llm import AsyncOmniLLM
+__version__ = "0.1.0"
+__author__ = "vLLM-omni Team"
+__email__ = "hsliuustc@gmail.com"

 # Main entry points
-from .entrypoints.omni_llm import OmniLLM
-
-# from .version import __version__, __version_tuple__  # isort:skip
-
+from .entrypoints.omni_llm import OmniLLM, AsyncOmniLLM
+from .config import (
+    OmniStageConfig,
+    DiTConfig,
+    DiTCacheConfig,
+    create_ar_stage_config,
+    create_dit_stage_config,
+)

 __all__ = [
-    # "__version__",  # TODO: add __version__ back when it is implemented
-    # "__version_tuple__",  # TODO: add __version_tuple__ back when it is implemented
+    # Version info
+    "__version__",
+    "__author__", 
+    "__email__",
+    
    # Main components
    "OmniLLM",
    "AsyncOmniLLM",
+    
    # Configuration
-    "OmniModelConfig",
+    "OmniStageConfig",
+    "DiTConfig",
+    "DiTCacheConfig",
+    "create_ar_stage_config",
+    "create_dit_stage_config",
+    
    # All other components are available through their respective modules
    # processors.*, schedulers.*, executors.*, etc.
-]
+]
@@ -1,9 +1,50 @@
 """
 Configuration module for vLLM-omni.
 """
+from typing import Optional
+from pydantic.dataclasses import dataclass
+from pydantic import ConfigDict
+
+from vllm.config import ModelConfig
+from vllm.config import config
+
+import vllm_omni.model_executor.models as me_models
+
+from .stage_config import (
+    OmniStageConfig,
+    DiTConfig,
+    DiTCacheConfig,
+    DiTCacheTensor,
+    create_ar_stage_config,
+    create_dit_stage_config,
+)
+
+
+@config
+@dataclass(config=ConfigDict(arbitrary_types_allowed=True))
+class OmniModelConfig(ModelConfig):
+    """Configuration for Omni models, extending the base ModelConfig."""
+    
+    stage_id: int = 0
+    model_stage: str = "thinker"
+    model_arch: str = "Qwen2_5OmniForConditionalGeneration"
+    engine_output_type: Optional[str] = None
+    
+    @property
+    def registry(self):
+        return me_models.OmniModelRegistry
+    
+    @property
+    def architectures(self) -> list[str]:
+        return [self.model_arch]

-from vllm_omni.config.model import OmniModelConfig

 __all__ = [
    "OmniModelConfig",
-]
+    "OmniStageConfig",
+    "DiTConfig", 
+    "DiTCacheConfig",
+    "DiTCacheTensor",
+    "create_ar_stage_config",
+    "create_dit_stage_config",
+]
@@ -1,26 +0,0 @@
-from typing import Optional
-
-from pydantic import ConfigDict
-from pydantic.dataclasses import dataclass
-from vllm.config import ModelConfig, config
-
-import vllm_omni.model_executor.models as me_models
-
-
-@config
-@dataclass(config=ConfigDict(arbitrary_types_allowed=True))
-class OmniModelConfig(ModelConfig):
-    """Configuration for Omni models, extending the base ModelConfig."""
-
-    stage_id: int = 0
-    model_stage: str = "thinker"
-    model_arch: str = "Qwen2_5OmniForConditionalGeneration"
-    engine_output_type: Optional[str] = None
-
-    @property
-    def registry(self):
-        return me_models.OmniModelRegistry
-
-    @property
-    def architectures(self) -> list[str]:
-        return [self.model_arch]
@@ -0,0 +1,166 @@
+"""
+Stage configuration for vLLM-omni multi-stage processing.
+"""
+
+from dataclasses import dataclass
+from typing import List, Optional, Type, Literal, Any, Dict
+from vllm.config import (
+    VllmConfig,
+    DeviceConfig,
+    LoadConfig,
+    ModelConfig,
+    CompilationConfig,
+)
+from vllm.executor.executor_base import ExecutorBase as Executor
+
+@dataclass
+class DiTCacheTensor:
+    """Configuration for DiT cache tensors."""
+    name: str
+    shape: List[int]
+    dtype: str = "float32"
+    persistent: bool = True
+
+
+@dataclass
+class DiTCacheConfig:
+    """Configuration for DiT caching system."""
+    cache_tensors: List[DiTCacheTensor]
+    max_cache_size: int = 1024 * 1024 * 1024  # 1GB
+    cache_strategy: str = "fifo"  # fifo, lru, lfu
+    enable_optimization: bool = True
+    cache_compression: bool = False
+
+
+@dataclass
+class DiTConfig:
+    """Configuration for DiT (Diffusion Transformer) stages."""
+
+    num_inference_steps: int
+    """Number of diffusion inference steps."""
+
+    model_config: Optional[ModelConfig] = None
+    """Model configuration."""
+    scheduler_config: Optional[Any] = None
+    """Scheduler configuration."""
+    device_config: Optional[DeviceConfig] = None
+    """Device configuration."""
+    load_config: Optional[LoadConfig] = None
+    """Model loading configuration."""
+    compilation_config: Optional[CompilationConfig] = None
+    """Compilation configuration."""
+    dit_cache_config: Optional[DiTCacheConfig] = None
+    """DiT cache configuration."""
+    guidance_scale: float = 7.5
+    use_diffusers: bool = False
+    diffusers_pipeline: Optional[str] = None
+    height: int = 512
+    width: int = 512
+    batch_size: int = 1
+
+
+@dataclass
+class OmniStageConfig:
+    """Configuration for a processing stage in vLLM-omni."""
+
+    stage_id: int
+    engine_type: Literal["AR", "DiT"]
+    model_path: str
+    input_modalities: List[str]
+    output_modalities: List[str]
+    vllm_config: Optional[VllmConfig] = None
+    dit_config: Optional[DiTConfig] = None
+    executor_class: Type[Executor] = None  # Will be set based on engine_type
+    default_stage_args: Optional[Dict[str, Any]] = None
+    stage_output: Optional[Any] = None
+    
+    def __post_init__(self):
+        """Validate configuration after initialization."""
+        if self.engine_type not in ["AR", "DiT"]:
+            raise ValueError(f"Invalid engine_type: {self.engine_type}. Must be 'AR' or 'DiT'")
+        
+        if self.engine_type == "DiT" and self.dit_config is None:
+            raise ValueError("DiT engine requires dit_config")
+        
+        if not self.input_modalities:
+            raise ValueError("input_modalities cannot be empty")
+        
+        if not self.output_modalities:
+            raise ValueError("output_modalities cannot be empty")
+    
+    def get_executor_class(self) -> Type[Executor]:
+        """Get the appropriate executor class for this stage."""
+        if self.executor_class is not None:
+            return self.executor_class
+        
+        if self.engine_type == "AR":
+            from vllm.executor.uniproc_executor import UniProcExecutor
+            return UniProcExecutor
+        elif self.engine_type == "DiT":
+            if self.dit_config and self.dit_config.use_diffusers:
+                from vllm_omni.executor.diffusers_executor import DiffusersPipelineExecutor
+                return DiffusersPipelineExecutor
+            else:
+                from vllm.executor.uniproc_executor import UniProcExecutor
+                return UniProcExecutor
+        
+        raise ValueError(f"No executor class available for engine_type: {self.engine_type}")
+
+
+def create_ar_stage_config(
+    stage_id: int,
+    model_path: str,
+    input_modalities: List[str] = None,
+    output_modalities: List[str] = None,
+    vllm_config: Optional[VllmConfig] = None,
+    executor_class: Optional[Type[Executor]] = None,
+    default_stage_args: Optional[Dict[str, Any]] = None,
+) -> OmniStageConfig:
+    """Create a configuration for an AR (Autoregressive) stage."""
+    if input_modalities is None:
+        input_modalities = ["text"]
+    if output_modalities is None:
+        output_modalities = ["text"]
+    
+    # For now, we'll create minimal configs
+    # vllm_config and executor_class will be handled by the LLM class
+    return OmniStageConfig(
+        stage_id=stage_id,
+        engine_type="AR",
+        model_path=model_path,
+        input_modalities=input_modalities,
+        output_modalities=output_modalities,
+        vllm_config=vllm_config,
+        executor_class=executor_class,
+        default_stage_args=default_stage_args,
+    )
+
+
+def create_dit_stage_config(
+    stage_id: int,
+    model_path: str,
+    input_modalities: Optional[List[str]] = None,
+    output_modalities: Optional[List[str]] = None,
+    dit_config: Optional[DiTConfig] = None,
+    executor_class: Optional[Type[Executor]] = None,
+    default_stage_args: Optional[Dict[str, Any]] = None,
+) -> OmniStageConfig:
+    """Create a configuration for a DiT (Diffusion Transformer) stage."""
+    if input_modalities is None:
+        input_modalities = ["text"]
+    if output_modalities is None:
+        output_modalities = ["image"]
+    
+    if dit_config is None:
+        dit_config = DiTConfig(num_inference_steps=50)
+
+    return OmniStageConfig(
+        stage_id=stage_id,
+        engine_type="DiT",
+        model_path=model_path,
+        input_modalities=input_modalities,
+        output_modalities=output_modalities,
+        dit_config=dit_config,
+        executor_class=executor_class,
+        default_stage_args=default_stage_args,
+    )
@@ -5,9 +5,174 @@ This module provides caching functionality for DiT (Diffusion Transformer) model
 to optimize inference performance and memory usage.
 """

+import time
+import torch
+from typing import Dict, List, Optional, Any
+from dataclasses import dataclass
+from ..config import DiTCacheConfig, DiTCacheTensor
+

 class DiTCacheManager:
    """Manages DiT-specific caching for optimized inference."""
-
-    def __init__(self, config):
-        pass
+    
+    def __init__(self, config: DiTCacheConfig):
+        self.config = config
+        self.cache_tensors: Dict[str, DiTCacheTensor] = {}
+        self.cache_groups: List[DiTCacheTensor] = config.cache_tensors
+        self.max_cache_size = config.max_cache_size
+        self.current_cache_size = 0
+        self.cache_hits = 0
+        self.cache_misses = 0
+        self.cache_timeout = 3600  # 1 hour default timeout
+    
+    def allocate_cache(self, request_id: str, size: int) -> torch.Tensor:
+        """Allocate cache for a specific request."""
+        # Check if we have enough space
+        if self.current_cache_size + size > self.max_cache_size:
+            self._evict_cache()
+        
+        # Create tensor
+        tensor = torch.zeros(size, dtype=torch.float32)
+        
+        # Store in cache
+        cache_tensor = DiTCacheTensor(
+            name=request_id,
+            tensor=tensor,
+            timestamp=time.time(),
+            access_count=0,
+            size_bytes=size * 4  # Assuming float32
+        )
+        
+        self.cache_tensors[request_id] = cache_tensor
+        self.current_cache_size += cache_tensor.size_bytes
+        
+        return tensor
+    
+    def get_cache(self, request_id: str) -> Optional[torch.Tensor]:
+        """Get cached tensor for a request."""
+        if request_id in self.cache_tensors:
+            cache_tensor = self.cache_tensors[request_id]
+            cache_tensor.access_count += 1
+            self.cache_hits += 1
+            return cache_tensor.tensor
+        else:
+            self.cache_misses += 1
+            return None
+    
+    def store_cache(self, request_id: str, tensor: torch.Tensor) -> None:
+        """Store a tensor in the cache."""
+        if request_id in self.cache_tensors:
+            # Update existing cache
+            old_tensor = self.cache_tensors[request_id]
+            self.current_cache_size -= old_tensor.size_bytes
+            
+            new_size_bytes = tensor.numel() * 4  # Assuming float32
+            self.current_cache_size += new_size_bytes
+            
+            self.cache_tensors[request_id] = DiTCacheTensor(
+                name=request_id,
+                tensor=tensor,
+                timestamp=time.time(),
+                access_count=old_tensor.access_count,
+                size_bytes=new_size_bytes
+            )
+        else:
+            # Create new cache entry
+            new_size_bytes = tensor.numel() * 4
+            if self.current_cache_size + new_size_bytes > self.max_cache_size:
+                self._evict_cache()
+            
+            self.cache_tensors[request_id] = DiTCacheTensor(
+                name=request_id,
+                tensor=tensor,
+                timestamp=time.time(),
+                access_count=0,
+                size_bytes=new_size_bytes
+            )
+            self.current_cache_size += new_size_bytes
+    
+    def release_cache(self, request_id: str) -> None:
+        """Release cache for a request."""
+        if request_id in self.cache_tensors:
+            cache_tensor = self.cache_tensors[request_id]
+            self.current_cache_size -= cache_tensor.size_bytes
+            del self.cache_tensors[request_id]
+    
+    def clear_expired_cache(self) -> None:
+        """Clear expired cache entries."""
+        current_time = time.time()
+        expired_keys = []
+        
+        for request_id, cache_tensor in self.cache_tensors.items():
+            if current_time - cache_tensor.timestamp > self.cache_timeout:
+                expired_keys.append(request_id)
+        
+        for request_id in expired_keys:
+            self.release_cache(request_id)
+    
+    def _evict_cache(self) -> None:
+        """Evict cache entries based on strategy."""
+        if self.config.cache_strategy == "fifo":
+            self._evict_fifo()
+        elif self.config.cache_strategy == "lru":
+            self._evict_lru()
+        elif self.config.cache_strategy == "lfu":
+            self._evict_lfu()
+        else:
+            self._evict_fifo()  # Default to FIFO
+    
+    def _evict_fifo(self) -> None:
+        """Evict cache entries using FIFO strategy."""
+        if not self.cache_tensors:
+            return
+        
+        # Find oldest entry
+        oldest_key = min(self.cache_tensors.keys(), 
+                        key=lambda k: self.cache_tensors[k].timestamp)
+        self.release_cache(oldest_key)
+    
+    def _evict_lru(self) -> None:
+        """Evict cache entries using LRU strategy."""
+        if not self.cache_tensors:
+            return
+        
+        # Find least recently used entry
+        lru_key = min(self.cache_tensors.keys(),
+                     key=lambda k: self.cache_tensors[k].access_count)
+        self.release_cache(lru_key)
+    
+    def _evict_lfu(self) -> None:
+        """Evict cache entries using LFU strategy."""
+        if not self.cache_tensors:
+            return
+        
+        # Find least frequently used entry
+        lfu_key = min(self.cache_tensors.keys(),
+                     key=lambda k: self.cache_tensors[k].access_count)
+        self.release_cache(lfu_key)
+    
+    def get_statistics(self) -> Dict[str, Any]:
+        """Get cache statistics."""
+        total_requests = self.cache_hits + self.cache_misses
+        hit_rate = self.cache_hits / total_requests if total_requests > 0 else 0
+        
+        return {
+            "cache_size": self.current_cache_size,
+            "max_cache_size": self.max_cache_size,
+            "cache_utilization": self.current_cache_size / self.max_cache_size,
+            "cache_hits": self.cache_hits,
+            "cache_misses": self.cache_misses,
+            "hit_rate": hit_rate,
+            "num_cached_tensors": len(self.cache_tensors)
+        }
+    
+    def clear_all_cache(self) -> None:
+        """Clear all cache entries."""
+        self.cache_tensors.clear()
+        self.current_cache_size = 0
+        self.cache_hits = 0
+        self.cache_misses = 0
+    
+    def set_cache_timeout(self, timeout: float) -> None:
+        """Set cache timeout in seconds."""
+        self.cache_timeout = timeout
@@ -1,13 +0,0 @@
-"""
-Scheduling components for vLLM-omni.
-"""
-
-from .diffusion_scheduler import DiffusionScheduler
-from .output import OmniNewRequestData
-from .scheduler import OmniScheduler
-
-__all__ = [
-    "OmniScheduler",
-    "DiffusionScheduler",
-    "OmniNewRequestData",
-]
@@ -1,299 +0,0 @@
-import time
-from collections import defaultdict
-from typing import Optional
-
-from vllm.v1.core.kv_cache_manager import KVCacheBlocks
-from vllm.v1.core.sched.request_queue import create_request_queue
-from vllm.v1.core.sched.scheduler import EngineCoreOutputs, Request, RequestStatus, SchedulerOutput, SpecDecodingStats
-from vllm.v1.core.sched.utils import remove_all
-from vllm.v1.engine import EngineCoreEventType, EngineCoreOutput
-
-from vllm_omni.core.sched.output import OmniNewRequestData
-from vllm_omni.core.sched.scheduler import OmniScheduler
-from vllm_omni.outputs import OmniModelRunnerOutput
-
-
-class DiffusionScheduler(OmniScheduler):
-    def schedule(self) -> SchedulerOutput:
-        """Diffusion fast path:
-        - Feed all input tokens of the request at once
-          (if 0, allocate 1 placeholder token).
-        - If the token budget cannot be satisfied at once, fall back to the
-          default vLLM scheduling.
-        """
-
-        # Select requests with zero prompt and using pooling
-        # (diffusion results returned via pooler_output)
-        token_budget = self.max_num_scheduled_tokens
-        capacity = self.max_num_running_reqs - len(self.running)
-        scheduled_timestamp = time.monotonic()
-
-        scheduled_new_reqs: list[Request] = []
-        scheduled_resumed_reqs: list[Request] = []
-        scheduled_running_reqs: list[Request] = []
-
-        req_to_new_blocks: dict[str, KVCacheBlocks] = {}
-        num_scheduled_tokens: dict[str, int] = {}
-        scheduled_spec_decode_tokens: dict[str, list[int]] = {}
-        scheduled_encoder_inputs: dict[str, list[int]] = {}
-
-        # Temporary queue: preserve waiting order, do not disturb non-diffusion requests
-        skipped_waiting_requests = create_request_queue(self.policy)
-
-        # Fast path selection and scheduling (treat all as diffusion requests,
-        # independent of pooling_params)
-        while self.waiting and token_budget > 0 and capacity > 0:
-            request = self.waiting.peek_request()
-            # Uniformly treat as diffusion. A feature flag can be added later
-            # via config or request tag.
-
-            # Allocate all input tokens for the request in one shot
-            # (allocate 1 placeholder if zero)
-            required_tokens = max(getattr(request, "num_prompt_tokens", 0), 1)
-            if required_tokens > token_budget:
-                # Insufficient budget to process all inputs at once;
-                # stop fast path attempt
-                break
-            num_new_tokens = required_tokens
-            new_blocks = self.kv_cache_manager.allocate_slots(
-                request,
-                num_new_tokens,
-                num_lookahead_tokens=self.num_lookahead_tokens,
-            )
-            if new_blocks is None:
-                # Allocation failed (e.g., VRAM pressure); stop fast path and
-                # fall back to default scheduling
-                # Put the current request back to the head of the waiting queue
-                # Note: the original queue order is preserved
-                break
-
-            # Officially schedule this request
-            request = self.waiting.pop_request()
-            self.running.append(request)
-            request.status = RequestStatus.RUNNING
-            if self.log_stats:
-                request.record_event(EngineCoreEventType.SCHEDULED, scheduled_timestamp)
-
-            req_to_new_blocks[request.request_id] = new_blocks
-            num_scheduled_tokens[request.request_id] = num_new_tokens
-            token_budget -= num_new_tokens
-            capacity -= 1
-            scheduled_new_reqs.append(request)
-
-        # Return skipped waiting requests
-        if skipped_waiting_requests:
-            self.waiting.prepend_requests(skipped_waiting_requests)
-
-        # If fast path scheduled none, fall back to the original scheduling
-        if not num_scheduled_tokens:
-            return super().schedule()
-
-        # Compute common prefix blocks (aligned with v1)
-        num_common_prefix_blocks = [0] * len(self.kv_cache_config.kv_cache_groups)
-        if self.running:
-            any_request = self.running[0]
-            num_common_prefix_blocks = self.kv_cache_manager.get_num_common_prefix_blocks(
-                any_request, len(self.running)
-            )
-
-        # Assemble SchedulerOutput
-        new_reqs_data = [
-            OmniNewRequestData.from_request(req, req_to_new_blocks[req.request_id].get_block_ids())
-            for req in scheduled_new_reqs
-        ]
-        cached_reqs_data = self._make_cached_request_data(
-            scheduled_running_reqs,
-            scheduled_resumed_reqs,
-            num_scheduled_tokens,
-            scheduled_spec_decode_tokens,
-            req_to_new_blocks,
-        )
-        scheduled_requests = scheduled_new_reqs + scheduled_running_reqs + scheduled_resumed_reqs
-        structured_output_request_ids, grammar_bitmask = self.get_grammar_bitmask(
-            scheduled_requests, scheduled_spec_decode_tokens
-        )
-
-        total_num_scheduled_tokens = sum(num_scheduled_tokens.values())
-        scheduler_output = SchedulerOutput(
-            scheduled_new_reqs=new_reqs_data,
-            scheduled_cached_reqs=cached_reqs_data,
-            num_scheduled_tokens=num_scheduled_tokens,
-            total_num_scheduled_tokens=total_num_scheduled_tokens,
-            scheduled_spec_decode_tokens=scheduled_spec_decode_tokens,
-            scheduled_encoder_inputs=scheduled_encoder_inputs,
-            num_common_prefix_blocks=num_common_prefix_blocks,
-            finished_req_ids=self.finished_req_ids,
-            free_encoder_mm_hashes=self.encoder_cache_manager.get_freed_mm_hashes(),
-            structured_output_request_ids=structured_output_request_ids,
-            grammar_bitmask=grammar_bitmask,
-        )
-
-        # KVTransfer: package metadata
-        if self.connector is not None:
-            meta = self.connector.build_connector_meta(scheduler_output)
-            scheduler_output.kv_connector_metadata = meta
-
-        # Update internal state (advance num_computed_tokens, free encoder inputs,
-        # etc.)
-        self._update_after_schedule(scheduler_output)
-        return scheduler_output
-
-    """
-    Scheduler for the diffusion model.
-    This scheduler is modified to stop the request immediately for the diffusion model.
-    This is because the diffusion model can generate the final image/audio in one step.
-    Note: This is just a minimal modification to the original scheduler,
-    and there should be some further efforts to optimize the scheduler.
-    The original scheduler is still used for the AR model.
-    """
-
-    def update_from_output(
-        self,
-        scheduler_output: SchedulerOutput,
-        model_runner_output: OmniModelRunnerOutput,
-    ) -> dict[int, EngineCoreOutputs]:
-        """Update the scheduler state based on the model runner output.
-
-        This method is modified to stop the request immediately for the diffusion model.
-        """
-        sampled_token_ids = model_runner_output.sampled_token_ids
-        logprobs = model_runner_output.logprobs
-        prompt_logprobs_dict = model_runner_output.prompt_logprobs_dict
-        num_scheduled_tokens = scheduler_output.num_scheduled_tokens
-        pooler_outputs = model_runner_output.pooler_output
-        num_nans_in_logits = model_runner_output.num_nans_in_logits
-        kv_connector_output = model_runner_output.kv_connector_output
-
-        outputs: dict[int, list[EngineCoreOutput]] = defaultdict(list)
-        spec_decoding_stats: Optional[SpecDecodingStats] = None
-        kv_connector_stats = kv_connector_output.kv_connector_stats if kv_connector_output else None
-
-        # NOTE(woosuk): As len(num_scheduled_tokens) can be up to 1K or more,
-        # the below loop can be a performance bottleneck. We should do our best
-        # to avoid expensive operations inside the loop.
-        stopped_running_reqs: set[Request] = set()
-        stopped_preempted_reqs: set[Request] = set()
-        for req_id, num_tokens_scheduled in num_scheduled_tokens.items():
-            assert num_tokens_scheduled > 0
-            request = self.requests.get(req_id)
-            if request is None:
-                # The request is already finished. This can happen if the
-                # request is aborted while the model is executing it (e.g.,
-                # in pipeline parallelism).
-                continue
-
-            req_index = model_runner_output.req_id_to_index[req_id]
-            generated_token_ids = sampled_token_ids[req_index] if sampled_token_ids else []
-
-            scheduled_spec_token_ids = scheduler_output.scheduled_spec_decode_tokens.get(req_id)
-            if scheduled_spec_token_ids:
-                num_draft_tokens = len(scheduled_spec_token_ids)
-                num_accepted = len(generated_token_ids) - 1
-                num_rejected = num_draft_tokens - num_accepted
-                # num_computed_tokens represents the number of tokens
-                # processed in the current step, considering scheduled
-                # tokens and rejections. If some tokens are rejected,
-                # num_computed_tokens is decreased by the number of rejected
-                # tokens.
-                request.num_computed_tokens -= num_rejected
-                spec_decoding_stats = self.make_spec_decoding_stats(
-                    spec_decoding_stats,
-                    num_draft_tokens=num_draft_tokens,
-                    num_accepted_tokens=num_accepted,
-                )
-
-            new_logprobs = None
-            new_token_ids = generated_token_ids
-            kv_transfer_params = None
-            status_before_stop = request.status
-            pooler_output = None
-            if pooler_outputs:
-                pooler_output = pooler_outputs[req_index]
-
-            # Diffusion request: completes in one step; mark finished and free resources
-            request.status = RequestStatus.FINISHED_STOPPED
-            # Optional: set a stop_reason for front-end clarity
-            # (does not affect protocol)
-            request.stop_reason = request.stop_reason or "diffusion_done"
-            kv_transfer_params = self._free_request(request)
-            if status_before_stop == RequestStatus.RUNNING:
-                stopped_running_reqs.add(request)
-            else:
-                stopped_preempted_reqs.add(request)
-
-            # Extract sample logprobs if needed.
-            if request.sampling_params is not None and request.sampling_params.logprobs is not None and logprobs:
-                # NOTE: once we support N tokens per step (spec decode),
-                # the outer lists can be of length > 1.
-                new_logprobs = logprobs.slice(req_index, req_index + 1)
-
-            if new_token_ids and self.structured_output_manager.should_advance(request):
-                # NOTE: structured_output_request should not be None if
-                # use_structured_output, we have check above, so safe to ignore
-                # type warning
-                request.structured_output_request.grammar.accept_tokens(req_id, new_token_ids)  # type: ignore[union-attr]  # noqa: E501
-
-            # spec_token_ids comes from the model runner output
-            if num_nans_in_logits is not None and req_id in num_nans_in_logits:
-                request.num_nans_in_logits = num_nans_in_logits[req_id]
-
-            # Get prompt logprobs for this request.
-            prompt_logprobs_tensors = prompt_logprobs_dict.get(req_id)
-            if new_token_ids or pooler_output is not None or kv_transfer_params:
-                # Add EngineCoreOutput for this Request.
-                outputs[request.client_index].append(
-                    EngineCoreOutput(
-                        request_id=req_id,
-                        new_token_ids=new_token_ids,
-                        finish_reason=request.get_finished_reason(),
-                        new_logprobs=new_logprobs,
-                        new_prompt_logprobs_tensors=prompt_logprobs_tensors,
-                        pooling_output=pooler_output,
-                        stop_reason=request.stop_reason,
-                        events=request.take_events(),
-                        kv_transfer_params=kv_transfer_params,
-                        trace_headers=request.trace_headers,
-                        num_cached_tokens=request.num_cached_tokens,
-                    )
-                )
-            else:
-                # Invariant: EngineCore returns no partial prefill outputs.
-                assert not prompt_logprobs_tensors
-
-        # Remove the stopped requests from the running and waiting queues.
-        if stopped_running_reqs:
-            if stopped_running_reqs:
-                self.running = remove_all(self.running, stopped_running_reqs)
-        if stopped_preempted_reqs:
-            # This is a rare case and unlikely to impact performance.
-            self.waiting.remove_requests(stopped_preempted_reqs)
-
-        # KV Connector: update state for finished KV Transfers.
-        if model_runner_output.kv_connector_output:
-            self._update_from_kv_xfer_finished(model_runner_output.kv_connector_output)
-
-        # Create EngineCoreOutputs for all clients that have requests with
-        # outputs in this step.
-        engine_core_outputs = {client_index: EngineCoreOutputs(outputs=outs) for client_index, outs in outputs.items()}
-
-        finished_req_ids = self.finished_req_ids_dict
-        if finished_req_ids:
-            # Include ids of requests that finished since last outputs
-            # were sent.
-            for client_index, finished_set in finished_req_ids.items():
-                # Set finished request set in EngineCoreOutputs for this client.
-                if (eco := engine_core_outputs.get(client_index)) is not None:
-                    eco.finished_requests = finished_set
-                else:
-                    engine_core_outputs[client_index] = EngineCoreOutputs(finished_requests=finished_set)
-            finished_req_ids.clear()
-
-        if (stats := self.make_stats(spec_decoding_stats, kv_connector_stats)) is not None:
-            # Return stats to only one of the front-ends.
-            if (eco := next(iter(engine_core_outputs.values()), None)) is None:
-                # We must return the stats even if there are no request
-                # outputs this step.
-                engine_core_outputs[0] = eco = EngineCoreOutputs()
-            eco.scheduler_stats = stats
-
-        return engine_core_outputs
@@ -1,34 +0,0 @@
-from dataclasses import dataclass
-from typing import Optional
-
-from vllm.v1.core.sched.output import NewRequestData
-from vllm.v1.request import Request
-
-from vllm_omni.engine import AdditionalInformationPayload, PromptEmbedsPayload
-
-
-@dataclass
-class OmniNewRequestData(NewRequestData):
-    # Optional serialized prompt embeddings
-    prompt_embeds: Optional[PromptEmbedsPayload] = None
-    # Optional serialized additional information
-    additional_information: Optional[AdditionalInformationPayload] = None
-
-    @classmethod
-    def from_request(
-        cls,
-        request: Request,
-        block_ids: tuple[list[int], ...],
-    ) -> "OmniNewRequestData":
-        return cls(
-            req_id=request.request_id,
-            prompt_token_ids=request.prompt_token_ids,
-            mm_features=request.mm_features,
-            sampling_params=request.sampling_params,
-            pooling_params=request.pooling_params,
-            block_ids=block_ids,
-            num_computed_tokens=request.num_computed_tokens,
-            lora_request=request.lora_request,
-            prompt_embeds=request.prompt_embeds,
-            additional_information=request.additional_information,
-        )
@@ -1,52 +1,40 @@
-from __future__ import annotations

-from vllm.logger import init_logger
-from vllm.v1.core.sched.output import SchedulerOutput
-from vllm.v1.core.sched.scheduler import Scheduler as VLLMScheduler
+from vllm_omni.request import OmniRequest
+from typing import List
+from threading import Lock, Condition
+from vllm_omni.config import OmniConfig
+
+from vllm.v1.kv_cache_interface import KVCacheConfig
+from vllm.v1.core.sched import SchedulerInterface
+from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalRegistry
+from vllm.v1.structured_output import StructuredOutputManager


-class OmniScheduler(VLLMScheduler):
+class OmniScheduler(SchedulerInterface):
    """
    OmniScheduler: Scheduler for vLLM-omni multimodal processing.

-    This scheduler extends vLLM's scheduler to support multimodal and
-    non-autoregressive processing with additional fields and methods
-    specific to vLLM-omni.
+    This scheduler extends vLLM's scheduler to support multimodal and non-autoregressive
+    processing with additional fields and methods specific to vLLM-omni.
    """
-
-    # Ensure scheduled_new_reqs carry omni-specific payloads
-    # (e.g., additional_information)
-    def schedule(self) -> SchedulerOutput:  # type: ignore[override]
-        scheduler_output = super().schedule()
-        try:
-            # Late import to avoid circulars in some launch modes
-            from .output import OmniNewRequestData
-
-            # Rewrap base NewRequestData entries with OmniNewRequestData,
-            # enriching with request-level payloads
-            new_list = []
-            for nr in scheduler_output.scheduled_new_reqs:
-                req_id = getattr(nr, "req_id", None)
-                request = self.requests.get(req_id) if req_id else None
-                # Build omni entry preserving all base fields
-                omni_nr = OmniNewRequestData(
-                    req_id=nr.req_id,
-                    prompt_token_ids=nr.prompt_token_ids,
-                    mm_features=nr.mm_features,
-                    sampling_params=nr.sampling_params,
-                    pooling_params=nr.pooling_params,
-                    block_ids=nr.block_ids,
-                    num_computed_tokens=nr.num_computed_tokens,
-                    lora_request=nr.lora_request,
-                    # Enrich with omni payloads from the live request object
-                    prompt_embeds=(getattr(request, "prompt_embeds", None) if request else None),
-                    additional_information=(getattr(request, "additional_information", None) if request else None),
-                )
-                new_list.append(omni_nr)
-
-            scheduler_output.scheduled_new_reqs = new_list  # type: ignore[assignment]
-        except Exception:
-            # If anything goes wrong, leave the original output unchanged
-            init_logger(__name__).exception("Failed to wrap scheduled_new_reqs with OmniNewRequestData")
-
-        return scheduler_output
+    def __init__(self,
+        omni_config: OmniConfig,
+        kv_cache_config: KVCacheConfig,
+        structured_output_manager: StructuredOutputManager,
+        mm_registry: MultiModalRegistry = MULTIMODAL_REGISTRY,
+        include_finished_set: bool = False,
+        log_stats: bool = False,
+    ):
+        super().__init__(
+            vllm_config=omni_config.vllm_config,
+            kv_cache_config=kv_cache_config,
+            multimodal_registry=mm_registry,
+            structured_output_manager=structured_output_manager,
+            include_finished_set=include_finished_set,
+            log_stats=log_stats,
+        )
+        self.omni_config = omni_config
+            
+    def schedule(self, requests: List[OmniRequest]) -> List[OmniRequest]:
+        # TODO: Implement scheduling logic
+        pass
@@ -2,54 +2,8 @@
 Engine components for vLLM-omni.
 """

-from typing import Any, Optional
+from .output_processor import MultimodalOutputProcessor

-import msgspec
-from vllm.v1.engine import EngineCoreRequest
-
-
-class PromptEmbedsPayload(msgspec.Struct):
-    """Serialized prompt embeddings payload for direct transfer.
-
-    data: raw bytes of the tensor in row-major order
-    shape: [seq_len, hidden_size]
-    dtype: torch dtype name (e.g., "float16", "float32")
-    """
-
-    data: bytes
-    shape: list[int]
-    dtype: str
-
-
-class AdditionalInformationEntry(msgspec.Struct):
-    """One entry of additional_information.
-
-    Two supported forms are encoded:
-      - tensor: data/shape/dtype
-      - list: a Python list (msgspec-serializable)
-    Exactly one of (tensor_data, list_data) should be non-None.
-    """
-
-    # Tensor form
-    tensor_data: Optional[bytes] = None
-    tensor_shape: Optional[list[int]] = None
-    tensor_dtype: Optional[str] = None
-
-    # List form
-    list_data: Optional[list[Any]] = None
-
-
-class AdditionalInformationPayload(msgspec.Struct):
-    """Serialized dictionary payload for additional_information.
-
-    Keys are strings; values are encoded as AdditionalInformationEntry.
-    """
-
-    entries: dict[str, AdditionalInformationEntry]
-
-
-class OmniEngineCoreRequest(EngineCoreRequest):
-    # Optional prompt embeddings (direct-transfer version)
-    prompt_embeds: Optional[PromptEmbedsPayload] = None
-    # Optional additional information dictionary (serialized)
-    additional_information: Optional[AdditionalInformationPayload] = None
+__all__ = [
+    "MultimodalOutputProcessor",
+]
@@ -1,8 +1,7 @@
-from dataclasses import dataclass
 from typing import Optional
-
+from dataclasses import dataclass
 from vllm.engine.arg_utils import EngineArgs
-from vllm.v1.engine.async_llm import AsyncEngineArgs
+from vllm.utils import FlexibleArgumentParser

 from vllm_omni.config import OmniModelConfig

@@ -14,50 +13,39 @@ class OmniEngineArgs(EngineArgs):
    model_arch: str = "Qwen2_5OmniForConditionalGeneration"
    engine_output_type: Optional[str] = None

-    def create_model_config(self) -> OmniModelConfig:
-        # First, get the base ModelConfig from the parent class
-        base_config = super().create_model_config()
-
-        # Create OmniModelConfig by copying all base config attributes
-        # and adding the new omni-specific fields
-        config_dict = base_config.__dict__.copy()
-
-        # Add the new omni-specific fields
-        config_dict["stage_id"] = self.stage_id
-        config_dict["model_stage"] = self.model_stage
-        config_dict["model_arch"] = self.model_arch
-        config_dict["engine_output_type"] = self.engine_output_type
-
-        # Create and return the OmniModelConfig instance
-        omni_config = OmniModelConfig(**config_dict)
-        omni_config.hf_config.architectures = omni_config.architectures
-
-        return omni_config
-
-
-@dataclass
-class AsyncOmniEngineArgs(AsyncEngineArgs):
-    stage_id: int = 0
-    model_stage: str = "thinker"
-    model_arch: str = "Qwen2_5OmniForConditionalGeneration"
-    engine_output_type: Optional[str] = None
+    @staticmethod
+    def add_cli_args(parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
+        """Shared CLI arguments for vLLM engine."""
+        parser.add_argument(
+            "--engine-output-type",
+            type=str,
+            default=EngineArgs.engine_output_type,
+            help=(
+                "Declare EngineCoreOutput.output_type (e.g., 'text', 'image', "
+                "'text+image', 'latent'). This will be written into "
+                "model_config.engine_output_type for schedulers to use."
+            ),
+        )
+        parser.add_argument("--model-stage", type=str, default=OmniEngineArgs.model_stage, 
+        help="Declare model stage (e.g., 'thinker', 'talker', 'token2wav'). This will be written into model_config.model_stage for schedulers to use.")
+        return parser

    def create_model_config(self) -> OmniModelConfig:
        # First, get the base ModelConfig from the parent class
        base_config = super().create_model_config()
-
+        
        # Create OmniModelConfig by copying all base config attributes
        # and adding the new omni-specific fields
        config_dict = base_config.__dict__.copy()
-
+        
        # Add the new omni-specific fields
-        config_dict["stage_id"] = self.stage_id
-        config_dict["model_stage"] = self.model_stage
-        config_dict["model_arch"] = self.model_arch
-        config_dict["engine_output_type"] = self.engine_output_type
-
+        config_dict['stage_id'] = self.stage_id
+        config_dict['model_stage'] = self.model_stage
+        config_dict['model_arch'] = self.model_arch
+        config_dict['engine_output_type'] = self.engine_output_type
+        
        # Create and return the OmniModelConfig instance
        omni_config = OmniModelConfig(**config_dict)
        omni_config.hf_config.architectures = omni_config.architectures
-
-        return omni_config
+        
+        return omni_config
@@ -0,0 +1,133 @@
+"""Diffusion engine adapters for diffusers-backed DiT stages."""
+
+from __future__ import annotations
+
+from typing import Any, Optional
+
+from ..config import DiTConfig
+from ..worker.gpu_diffusion_worker import (
+    DiffusionGPUWorker,
+    DiffusionRunnerOutput,
+)
+
+
+class DiffusionEngine:
+    """Base diffusion engine storing shared DiT configuration."""
+
+    def __init__(
+        self,
+        dit_config: DiTConfig,
+        model_path: Optional[str] = None,
+        log_stats: bool = False,
+        multiprocess_mode: bool = False,
+    ) -> None:
+        self.dit_config = dit_config
+
+        model_cfg = getattr(dit_config, "model_config", None)
+        config_model_path = None
+        if model_cfg is not None:
+            if isinstance(model_cfg, dict):
+                config_model_path = (
+                    model_cfg.get("model")
+                    or model_cfg.get("model_path")
+                )
+            else:
+                config_model_path = (
+                    getattr(model_cfg, "model", None)
+                    or getattr(model_cfg, "model_path", None)
+                )
+
+        self.model_path = model_path or config_model_path
+        self.model_config = dit_config.model_config
+        self.cache_config = dit_config.dit_cache_config
+
+        self.height = dit_config.height
+        self.width = dit_config.width
+        self.num_inference_steps = dit_config.num_inference_steps
+        self.guidance_scale = dit_config.guidance_scale
+
+        self.log_stats = log_stats
+        self.multiprocess_mode = multiprocess_mode
+
+    def generate(self, *args, **kwargs):  # pragma: no cover - placeholder
+        raise NotImplementedError("This method should be implemented in subclasses.")
+
+
+class DiffusersPipelineEngine(DiffusionEngine):
+    """Adapter that invokes a diffusers pipeline via DiffusionGPUWorker."""
+
+    def __init__(
+        self,
+        dit_config: DiTConfig,
+        model_path: Optional[str] = None,
+        log_stats: bool = False,
+        multiprocess_mode: bool = False,
+    ) -> None:
+        super().__init__(dit_config, model_path, log_stats, multiprocess_mode)
+
+        resolved_model_path = self.model_path
+        if resolved_model_path is None:
+            raise ValueError("Model path must be provided for DiffusersPipelineEngine.")
+
+        device_cfg = getattr(dit_config, "device_config", None)
+        model_cfg = getattr(dit_config, "model_config", None)
+
+        device = None
+        dtype = None
+
+        if device_cfg:
+            if isinstance(device_cfg, dict):
+                device = device_cfg.get("device")
+                dtype = device_cfg.get("dtype")
+            else:
+                device = getattr(device_cfg, "device", None)
+                dtype = getattr(device_cfg, "dtype", None)
+
+        if dtype is None and model_cfg:
+            if isinstance(model_cfg, dict):
+                dtype = model_cfg.get("dtype")
+            else:
+                dtype = getattr(model_cfg, "dtype", None)
+
+        self.worker = DiffusionGPUWorker(
+            resolved_model_path,
+            pipeline_name=dit_config.diffusers_pipeline,
+            device=device,
+            dtype=dtype,
+        )
+
+    def generate(
+        self,
+        prompt: str,
+        *,
+        height: Optional[int] = None,
+        width: Optional[int] = None,
+        num_inference_steps: Optional[int] = None,
+        guidance_scale: Optional[float] = None,
+        negative_prompt: Optional[str] = None,
+        seed: Optional[int] = None,
+        image: Optional[Any] = None,
+    ) -> DiffusionRunnerOutput:
+        height = int(height) if height is not None else self.height
+        width = int(width) if width is not None else self.width
+        num_steps = (
+            int(num_inference_steps)
+            if num_inference_steps is not None
+            else self.num_inference_steps
+        )
+        guidance = (
+            float(guidance_scale)
+            if guidance_scale is not None
+            else self.guidance_scale
+        )
+
+        return self.worker.generate(
+            prompt=prompt,
+            height=height,
+            width=width,
+            num_inference_steps=num_steps,
+            guidance_scale=guidance,
+            negative_prompt=negative_prompt,
+            seed=seed,
+            image=image,
+        )
@@ -1,24 +1,27 @@
-from typing import Any, Callable, Optional, Union
+from typing import Dict, Callable, Optional, Any, Union

 import torch
-from vllm.logger import init_logger
-from vllm.outputs import PoolingRequestOutput
+
+from vllm.v1.engine.output_processor import OutputProcessor as VLLMOutputProcessor
+from vllm.v1.engine.output_processor import OutputProcessorOutput, RequestState, RequestOutputCollector
+from vllm.transformers_utils.tokenizer_group import TokenizerGroup
+from vllm.v1.engine import FinishReason
+from vllm.v1.metrics.stats import IterationStats
 from vllm.sampling_params import RequestOutputKind
 from vllm.transformers_utils.tokenizer import AnyTokenizer
-from vllm.v1.engine import EngineCoreOutput, EngineCoreRequest, FinishReason
+from vllm.v1.engine import EngineCoreRequest
 from vllm.v1.engine.detokenizer import IncrementalDetokenizer
 from vllm.v1.engine.logprobs import LogprobsProcessor
-from vllm.v1.engine.output_processor import OutputProcessor as VLLMOutputProcessor
-from vllm.v1.engine.output_processor import OutputProcessorOutput, RequestOutputCollector, RequestState
 from vllm.v1.engine.parallel_sampling import ParentRequest
-from vllm.v1.metrics.stats import IterationStats
+from vllm.logger import init_logger
+from vllm.v1.engine import EngineCoreOutput

-from vllm_omni.outputs import OmniRequestOutput

 logger = init_logger(__name__)


 class OmniRequestState(RequestState):
+
    def __init__(
        self,
        *args,
@@ -39,6 +42,7 @@ class OmniRequestState(RequestState):
        queue: Optional[Any],
        log_stats: bool,
    ) -> "OmniRequestState":
+
        if sampling_params := request.sampling_params:
            if not sampling_params.detokenize:
                tokenizer = None
@@ -52,16 +56,10 @@ class OmniRequestState(RequestState):
                request=request,
            )
            max_tokens_param = sampling_params.max_tokens
-            top_p = sampling_params.top_p
-            n = sampling_params.n
-            temperature = sampling_params.temperature
        else:
            logprobs_processor = None
            detokenizer = None
            max_tokens_param = None
-            top_p = None
-            n = None
-            temperature = None
            assert request.pooling_params is not None
            output_kind = request.pooling_params.output_kind

@@ -69,23 +67,21 @@ class OmniRequestState(RequestState):
            request_id=request.request_id,
            parent_req=parent_req,
            request_index=request_index,
-            lora_name=(request.lora_request.name if request.lora_request is not None else None),
+            lora_name=(request.lora_request.name
+                       if request.lora_request is not None else None),
            output_kind=output_kind,
            prompt=prompt,
            prompt_token_ids=request.prompt_token_ids,
-            prompt_embeds=request.prompt_embeds,
            logprobs_processor=logprobs_processor,
            detokenizer=detokenizer,
            max_tokens_param=max_tokens_param,
-            top_p=top_p,
-            n=n,
-            temperature=temperature,
            arrival_time=request.arrival_time,
            queue=queue,
            log_stats=log_stats,
        )

-    def add_multimodal_tensor(self, tensor: Optional[torch.Tensor], mm_type: Optional[str]) -> None:
+    def add_multimodal_tensor(self, tensor: Optional[torch.Tensor],
+                               mm_type: Optional[str]) -> None:
        if tensor is None:
            return
        try:
@@ -114,43 +110,52 @@ class OmniRequestState(RequestState):
        finish_reason: Optional[FinishReason],
        stop_reason: Optional[Union[int, str]],
        kv_transfer_params: Optional[dict[str, Any]] = None,
-    ) -> Optional[Union[OmniRequestOutput, PoolingRequestOutput]]:
+        num_cached_tokens: Optional[int] = None,
+    ) -> Optional[Any]:
        finished = finish_reason is not None
        final_only = self.output_kind == RequestOutputKind.FINAL_ONLY

        if not finished and final_only:
            return None

+        if num_cached_tokens is not None:
+            # Keep num_cached_tokens in RequestOutput for compatibility
+            try:
+                self.num_cached_tokens = num_cached_tokens  # type: ignore[attr-defined]
+            except Exception:
+                pass
+
        request_id = self.request_id
-        output = self._new_completion_output(new_token_ids, finish_reason, stop_reason)
+        output = self._new_completion_output(new_token_ids, finish_reason,
+                                             stop_reason)

        if self.parent_req is None:
            outputs = [output]
        else:
-            request_id, outputs, finished = self.parent_req.get_outputs(request_id, output)
+            request_id, outputs, finished = self.parent_req.get_outputs(
+                request_id, output)
            if not outputs:
                return None

-        return self._new_request_output(request_id, outputs, finished, kv_transfer_params)
+        return self._new_request_output(request_id, outputs, finished,
+                                        kv_transfer_params)

    def _new_completion_output(
        self,
        token_ids: list[int],
        finish_reason: Optional[FinishReason],
-        stop_reason: Optional[Union[int, str]],
+        stop_reason: Optional[Union[int, str]]
    ) -> Any:
        # Reuse base text/logprobs logic, then annotate with pooling_result.
-        base_output = super()._new_completion_output(token_ids, finish_reason, stop_reason)
+        base_output = super()._new_completion_output(token_ids, finish_reason,
+                                                     stop_reason)
        try:
            if self.mm_accumulated is not None:
                tensor = self.mm_accumulated
                try:
                    tensor = tensor.detach().to("cpu")
                except Exception:
-                    logger.debug(
-                        "Failed to move accumulated multimodal tensor to CPU",
-                        exc_info=True,
-                    )
+                    logger.debug("Failed to move accumulated multimodal tensor to CPU", exc_info=True)
                # Attach on the completion output for downstream consumers.
                if not hasattr(base_output, "multimodal_output"):
                    setattr(base_output, "multimodal_output", {})
@@ -172,22 +177,17 @@ class MultimodalOutputProcessor(VLLMOutputProcessor):
      produce the correct RequestOutput/PoolingRequestOutput.
    - Allow custom per-modality handlers via register_handler().
    """
-
-    def __init__(
-        self,
-        tokenizer: AnyTokenizer,
-        log_stats: bool,
-        engine_core_output_type: Optional[str] = None,
-    ):
+    def __init__(self, tokenizer: TokenizerGroup, log_stats: bool, engine_core_output_type: Optional[str] = None):
        super().__init__(tokenizer=tokenizer, log_stats=log_stats)
-        self.output_handlers: dict[str, Callable[[EngineCoreOutput], None]] = {}
-        self._reqid_to_mm_type: dict[str, str] = {}
+        self.output_handlers: Dict[str, Callable[[EngineCoreOutput], None]] = {}
+        self._reqid_to_mm_type: Dict[str, str] = {}
        self.request_states: dict[str, OmniRequestState] = {}
        self.engine_core_output_type = engine_core_output_type
-
-    def register_handler(self, modality: str, handler: Callable[[EngineCoreOutput], None]) -> None:
+        
+    def register_handler(self, modality: str,
+                         handler: Callable[[EngineCoreOutput], None]) -> None:
        self.output_handlers[modality.lower()] = handler
-
+    
    def add_request(
        self,
        request: EngineCoreRequest,
@@ -200,20 +200,21 @@ class MultimodalOutputProcessor(VLLMOutputProcessor):
        if request_id in self.request_states:
            raise ValueError(f"Request id {request_id} already running.")

-        req_state = OmniRequestState.from_new_request(
-            tokenizer=self.tokenizer,
-            request=request,
-            prompt=prompt,
-            parent_req=parent_req,
-            request_index=request_index,
-            queue=queue,
-            log_stats=self.log_stats,
-        )
+        tokenizer = None if not self.tokenizer else \
+            self.tokenizer.get_lora_tokenizer(request.lora_request)
+
+        req_state = OmniRequestState.from_new_request(tokenizer=tokenizer,
+                                                  request=request,
+                                                  prompt=prompt,
+                                                  parent_req=parent_req,
+                                                  request_index=request_index,
+                                                  queue=queue,
+                                                  log_stats=self.log_stats)
        self.request_states[request_id] = req_state
        self.lora_states.add_request(req_state)
        if parent_req:
            self.parent_requests[parent_req.request_id] = parent_req
-
+    
    def process_outputs(
        self,
        engine_core_outputs: list[EngineCoreOutput],
@@ -222,7 +223,7 @@ class MultimodalOutputProcessor(VLLMOutputProcessor):
    ) -> OutputProcessorOutput:
        self._reqid_to_mm_type.clear()
        for eco in engine_core_outputs:
-            mm_type = (self.engine_core_output_type or "").lower()
+            mm_type = (getattr(eco, "output_type", None) or "").lower()
            if mm_type:
                self._reqid_to_mm_type[eco.request_id] = mm_type
            self._route_and_normalize(eco)
@@ -237,20 +238,23 @@ class MultimodalOutputProcessor(VLLMOutputProcessor):
                continue

            # 1) Stats
-            self._update_stats_from_output(req_state, eco, engine_core_timestamp, iteration_stats)
+            self._update_stats_from_output(req_state, eco,
+                                           engine_core_timestamp,
+                                           iteration_stats)

            new_token_ids = eco.new_token_ids
            pooling_output = eco.pooling_output
            finish_reason = eco.finish_reason
            stop_reason = eco.stop_reason
            kv_transfer_params = eco.kv_transfer_params
-            req_state.num_cached_tokens = eco.num_cached_tokens
+            num_cached_tokens = eco.num_cached_tokens
            req_state.is_prefilling = False

            # 2) Detokenize and logprobs when text path
            assert req_state.detokenizer is not None
            assert req_state.logprobs_processor is not None
-            stop_string = req_state.detokenizer.update(new_token_ids, finish_reason == FinishReason.STOP)
+            stop_string = req_state.detokenizer.update(
+                new_token_ids, finish_reason == FinishReason.STOP)
            if stop_string:
                finish_reason = FinishReason.STOP
                stop_reason = stop_string
@@ -262,11 +266,7 @@ class MultimodalOutputProcessor(VLLMOutputProcessor):
                if pooling_output is not None and isinstance(req_state, OmniRequestState):
                    req_state.add_multimodal_tensor(pooling_output, mm_type)
            except Exception:
-                logger.debug(
-                    "Failed to accumulate multimodal tensor for request %s",
-                    req_id,
-                    exc_info=True,
-                )
+                logger.debug("Failed to accumulate multimodal tensor for request %s", req_id, exc_info=True)

            # 3) Create RequestOutput objects, forcing combined mode to keep ids
            pooling_for_make = pooling_output
@@ -274,13 +274,10 @@ class MultimodalOutputProcessor(VLLMOutputProcessor):
                # Do not consume pooling path now; keep ids and attach mm later
                pooling_for_make = None

-            ro = req_state.make_request_output(
-                new_token_ids,
-                pooling_for_make,
-                finish_reason,
-                stop_reason,
-                kv_transfer_params,
-            )
+            ro = req_state.make_request_output(new_token_ids, pooling_for_make,
+                                               finish_reason, stop_reason,
+                                               kv_transfer_params,
+                                               num_cached_tokens)
            if ro:
                # Attach accumulated multimodal payload if any
                try:
@@ -304,9 +301,8 @@ class MultimodalOutputProcessor(VLLMOutputProcessor):
                    self.parent_requests.pop(parent_req.request_id, None)
                if not eco.finished:
                    reqs_to_abort.append(req_id)
-                self._update_stats_from_finished(req_state, finish_reason, iteration_stats)
-                if self.tracer:
-                    self.do_tracing(eco, req_state, iteration_stats)
+                self._update_stats_from_finished(req_state, finish_reason,
+                                                 iteration_stats)
                # Cleanup per-request mm state
                if isinstance(req_state, OmniRequestState):
                    req_state.mm_accumulated = None
@@ -316,7 +312,7 @@ class MultimodalOutputProcessor(VLLMOutputProcessor):
            request_outputs=request_outputs,
            reqs_to_abort=reqs_to_abort,
        )
-
+    
    # ---- routing helpers ----
    def _route_and_normalize(self, eco: EngineCoreOutput) -> None:
        output_type = (getattr(eco, "output_type", self.engine_core_output_type) or "").lower()
@@ -350,7 +346,8 @@ class MultimodalOutputProcessor(VLLMOutputProcessor):
    def _process_image_output(self, eco: EngineCoreOutput) -> None:
        """Ensure image tensors are surfaced via pooling_output for vLLM."""
        if eco.pooling_output is None:
-            tensor = self._extract_from_multimodal_outputs(eco, keys=("image", "images", "pixel_values", "pixels"))
+            tensor = self._extract_from_multimodal_outputs(
+                eco, keys=("image", "images", "pixel_values", "pixels"))
            if tensor is not None:
                eco.pooling_output = tensor

@@ -360,24 +357,16 @@ class MultimodalOutputProcessor(VLLMOutputProcessor):
        # Preserve text tokens as-is; ensure pooling_output carries image/latents
        if eco.pooling_output is None:
            tensor = self._extract_from_multimodal_outputs(
-                eco,
-                keys=(
-                    "image",
-                    "images",
-                    "pixel_values",
-                    "pixels",
-                    "latent",
-                    "latents",
-                    "z",
-                ),
-            )
+                eco, keys=("image", "images", "pixel_values", "pixels",
+                           "latent", "latents", "z"))
            if tensor is not None:
                eco.pooling_output = tensor

    def _process_latents_output(self, eco: EngineCoreOutput) -> None:
        """Ensure latent tensors are surfaced via pooling_output."""
        if eco.pooling_output is None:
-            tensor = self._extract_from_multimodal_outputs(eco, keys=("latent", "latents", "z", "posterior"))
+            tensor = self._extract_from_multimodal_outputs(
+                eco, keys=("latent", "latents", "z", "posterior"))
            if tensor is not None:
                eco.pooling_output = tensor

@@ -385,8 +374,8 @@ class MultimodalOutputProcessor(VLLMOutputProcessor):
        """Ensure audio tensors are surfaced via pooling_output."""
        if eco.pooling_output is None:
            tensor = self._extract_from_multimodal_outputs(
-                eco, keys=("audio", "audios", "wav", "waveform", "audio_pcm", "pcm")
-            )
+                eco, keys=("audio", "audios", "wav", "waveform",
+                           "audio_pcm", "pcm"))
            if tensor is not None:
                eco.pooling_output = tensor

@@ -405,7 +394,9 @@ class MultimodalOutputProcessor(VLLMOutputProcessor):
            except Exception:
                pass

-    def _extract_from_multimodal_outputs(self, eco: EngineCoreOutput, keys: tuple[str, ...]) -> Optional[torch.Tensor]:
+    def _extract_from_multimodal_outputs(
+        self, eco: EngineCoreOutput, keys: tuple[str, ...]
+    ) -> Optional[torch.Tensor]:
        mm = getattr(eco, "multimodal_outputs", None)
        if not isinstance(mm, dict):
            return None
@@ -417,4 +408,4 @@ class MultimodalOutputProcessor(VLLMOutputProcessor):
        for v in mm.values():
            if isinstance(v, torch.Tensor):
                return v
-        return None
+        return None
@@ -1,235 +0,0 @@
-import time
-from collections.abc import Mapping
-from typing import Any, Optional, Union
-
-import torch
-from vllm.config import VllmConfig
-from vllm.inputs import ProcessorInputs, PromptType
-from vllm.inputs.parse import split_enc_dec_inputs
-from vllm.logger import init_logger
-from vllm.lora.request import LoRARequest
-from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalRegistry
-from vllm.multimodal.inputs import MultiModalFeatureSpec
-from vllm.multimodal.utils import argsort_mm_positions
-from vllm.platforms import current_platform
-from vllm.pooling_params import PoolingParams
-from vllm.sampling_params import SamplingParams
-from vllm.transformers_utils.tokenizer import AnyTokenizer
-from vllm.utils import length_from_prompt_token_ids_or_embeds
-from vllm.v1.engine.processor import Processor
-
-from vllm_omni.engine import (
-    AdditionalInformationEntry,
-    AdditionalInformationPayload,
-    OmniEngineCoreRequest,
-    PromptEmbedsPayload,
-)
-from vllm_omni.inputs.preprocess import OmniInputPreprocessor
-
-logger = init_logger(__name__)
-
-
-class OmniProcessor(Processor):
-    @staticmethod
-    def _dtype_to_name(dtype: torch.dtype) -> str:
-        mapping = {
-            torch.float32: "float32",
-            torch.float: "float32",
-            torch.float16: "float16",
-            torch.half: "float16",
-            torch.bfloat16: "bfloat16",
-            torch.float64: "float64",
-            torch.double: "float64",
-            torch.int64: "int64",
-            torch.long: "int64",
-            torch.int32: "int32",
-            torch.int: "int32",
-            torch.int16: "int16",
-            torch.short: "int16",
-            torch.int8: "int8",
-            torch.uint8: "uint8",
-            torch.bool: "bool",
-        }
-        return mapping.get(dtype, str(dtype).replace("torch.", ""))
-
-    def __init__(
-        self,
-        vllm_config: VllmConfig,
-        tokenizer: AnyTokenizer,
-        mm_registry: MultiModalRegistry = MULTIMODAL_REGISTRY,
-    ):
-        super().__init__(vllm_config, tokenizer, mm_registry)
-        self.input_preprocessor = OmniInputPreprocessor(
-            self.model_config,
-            self.tokenizer,
-            mm_registry,
-            mm_processor_cache=self.mm_processor_cache,
-        )
-
-    def process_inputs(
-        self,
-        request_id: str,
-        prompt: PromptType,
-        params: Union[SamplingParams, PoolingParams],
-        arrival_time: Optional[float] = None,
-        lora_request: Optional[LoRARequest] = None,
-        tokenization_kwargs: Optional[dict[str, Any]] = None,
-        trace_headers: Optional[Mapping[str, str]] = None,
-        priority: int = 0,
-        data_parallel_rank: Optional[int] = None,
-    ) -> tuple[Optional[str], OmniEngineCoreRequest]:
-        # TODO(woosuk): Support pooling models.
-        # TODO(woosuk): Support encoder-decoder models.
-        self._validate_lora(lora_request)
-        self._validate_params(params)
-
-        data_parallel_size = self.vllm_config.parallel_config.data_parallel_size
-        if data_parallel_rank is not None and not (0 <= data_parallel_rank < data_parallel_size):
-            raise ValueError(f"data_parallel_rank {data_parallel_rank} is out of range [0, {data_parallel_size}).")
-
-        if arrival_time is None:
-            arrival_time = time.time()
-
-        # Optionally generate multimodal hash overrides to avoid hashing
-        # multimodal data items by their content as their identifiers.
-
-        # NOTE: when users explicitly turn off BOTH prefix caching and input
-        # processing caching, no multimodal features or embeddings will be
-        # reused across requests, therefore identifying multimodal data items
-        # by their content is no longer necessary, and we create uuids with
-        # request id-modality-index as multimodal hash overrides.
-        if (
-            self.model_config.multimodal_config
-            and self.model_config.multimodal_config.mm_processor_cache_gb == 0
-            and not self.cache_config.enable_prefix_caching
-        ):
-            mm_uuids = self._maybe_build_mm_uuids(request_id, prompt)
-        else:
-            # Otherwise, use user-provided uuids as multimodal hash overrides
-            # if provided.
-            self._validate_multi_modal_uuids(prompt)
-            if isinstance(prompt, dict):
-                mm_uuids = prompt.get("multi_modal_uuids")
-            else:
-                mm_uuids = None
-
-        # Process inputs, which includes:
-        # 1. Tokenize text prompt, with LoRA request if one exists.
-        # 2. For multimodal models with a merged preprocessor, preprocess
-        #   multimodal data and expand prompt token ids accordingly.
-        processed_inputs: ProcessorInputs = self.input_preprocessor.preprocess(
-            prompt,
-            tokenization_kwargs=tokenization_kwargs,
-            mm_uuids=mm_uuids,
-        )
-        current_platform.validate_request(
-            prompt=prompt,
-            params=params,
-            processed_inputs=processed_inputs,
-        )
-        eos_token_id = self.input_preprocessor.get_eos_token_id()
-
-        encoder_inputs, decoder_inputs = split_enc_dec_inputs(processed_inputs)
-        self._validate_model_inputs(encoder_inputs, decoder_inputs)
-
-        # Mypy does not always properly infer the types of some elements of
-        # discriminated unions of TypedDicts, because of how it handles
-        # inheritance of TypedDict. If we explicitly extract the items we want
-        # we can avoid type errors from using `dict.get` later in the method.
-        prompt_str: Optional[str] = None if decoder_inputs["type"] == "embeds" else decoder_inputs.get("prompt")
-        prompt_token_ids = decoder_inputs["prompt_token_ids"] if decoder_inputs["type"] != "embeds" else None
-        prompt_embeds = decoder_inputs["prompt_embeds"] if decoder_inputs["type"] == "embeds" else None
-
-        sampling_params = None
-        pooling_params = None
-        if isinstance(params, SamplingParams):
-            # TODO: can we avoid cloning here in multiproc case?
-            sampling_params = params.clone()
-            # If unset max tokens, then generate up to the max_model_len.
-            if sampling_params.max_tokens is None:
-                seq_len = length_from_prompt_token_ids_or_embeds(prompt_token_ids, prompt_embeds)
-                sampling_params.max_tokens = self.model_config.max_model_len - seq_len
-            sampling_params.update_from_generation_config(self.generation_config_fields, eos_token_id)
-            if self.tokenizer is not None:
-                sampling_params.update_from_tokenizer(self.tokenizer)
-        else:
-            pooling_params = params.clone()
-
-        # Multimodal related.
-        mm_features: Optional[list[MultiModalFeatureSpec]] = None
-
-        if decoder_inputs["type"] == "multimodal":
-            decoder_mm_inputs = decoder_inputs["mm_kwargs"]
-            decoder_mm_positions = decoder_inputs["mm_placeholders"]
-            decoder_mm_hashes = decoder_inputs["mm_hashes"]
-
-            # Merge and flatten multimodal placeholders, hashes and inputs
-            # from dictionaries to lists, and sort them by each item's position
-            # in the input sequence.
-            sorted_mm_idxs = argsort_mm_positions(decoder_mm_positions)
-
-            mm_features = []
-            for modality, idx in sorted_mm_idxs:
-                mm_features.append(
-                    MultiModalFeatureSpec(
-                        data=decoder_mm_inputs[modality][idx],
-                        modality=modality,
-                        identifier=decoder_mm_hashes[modality][idx],
-                        mm_position=decoder_mm_positions[modality][idx],
-                    )
-                )
-
-        # Serialize prompt_embeds and additional_information if provided
-        # (direct-transfer path)
-        prompt_embeds_payload: Optional[PromptEmbedsPayload] = None
-        additional_information_payload: Optional[AdditionalInformationPayload] = None
-        if "prompt_embeds" in decoder_inputs:  # type: ignore[operator]
-            pe: torch.Tensor = decoder_inputs["prompt_embeds"]  # type: ignore[index]
-            if pe.ndim != 2:
-                raise ValueError("prompt_embeds must be of shape (seq_len, hidden_size)")
-            # Move to CPU and ensure contiguous memory for stable serialization
-            pe_cpu = pe.detach().to("cpu").contiguous()
-            seq_len, hidden_size = pe_cpu.shape
-            dtype_str = self._dtype_to_name(pe_cpu.dtype)
-            data_bytes = pe_cpu.numpy().tobytes()
-            prompt_embeds_payload = PromptEmbedsPayload(
-                data=data_bytes,
-                shape=[int(seq_len), int(hidden_size)],
-                dtype=dtype_str,
-            )
-        if "additional_information" in decoder_inputs:  # type: ignore[operator]
-            raw_info: dict[str, Any] = decoder_inputs["additional_information"]  # type: ignore[index]  # noqa: E501
-            entries: dict[str, AdditionalInformationEntry] = {}
-            for key, value in raw_info.items():
-                if isinstance(value, torch.Tensor):
-                    v_cpu = value.detach().to("cpu").contiguous()
-                    dtype_str = self._dtype_to_name(v_cpu.dtype)
-                    data_bytes = v_cpu.numpy().tobytes()
-                    entry = AdditionalInformationEntry(
-                        tensor_data=data_bytes,
-                        tensor_shape=[int(x) for x in list(v_cpu.shape)],
-                        tensor_dtype=dtype_str,
-                    )
-                elif isinstance(value, list):
-                    entry = AdditionalInformationEntry(list_data=value)
-                else:
-                    raise ValueError("additional_information values must be Tensor or list")
-                entries[key] = entry
-            additional_information_payload = AdditionalInformationPayload(entries=entries)
-
-        return prompt_str, OmniEngineCoreRequest(
-            request_id=request_id,
-            prompt_token_ids=prompt_token_ids,
-            mm_features=mm_features,
-            sampling_params=sampling_params,
-            pooling_params=pooling_params,
-            eos_token_id=eos_token_id,
-            arrival_time=arrival_time,
-            lora_request=lora_request,
-            cache_salt=decoder_inputs.get("cache_salt"),
-            priority=priority,
-            data_parallel_rank=data_parallel_rank,
-            trace_headers=trace_headers,
-            prompt_embeds=prompt_embeds_payload,
-            additional_information=additional_information_payload,
-        )
@@ -0,0 +1,146 @@
+"""
+API server for vLLM-omni.
+"""
+
+import asyncio
+from typing import Dict, Any, List
+from fastapi import FastAPI, HTTPException
+from fastapi.responses import JSONResponse
+from pydantic import BaseModel
+import uvicorn
+
+from .omni_llm import AsyncOmniLLM
+
+
+class GenerateRequest(BaseModel):
+    """Request model for generation."""
+    prompts: List[str]
+    max_tokens: int = 100
+    temperature: float = 0.7
+    stage_args: List[Dict[str, Any]] = None
+
+
+class GenerateResponse(BaseModel):
+    """Response model for generation."""
+    outputs: List[Dict[str, Any]]
+    stage_outputs: List[Dict[str, Any]] = None
+
+
+app = FastAPI(
+    title="vLLM-omni API",
+    description="Multi-modality models inference and serving",
+    version="0.1.0"
+)
+
+# Global omni_llm instance
+omni_llm: AsyncOmniLLM = None
+
+
+@app.on_event("startup")
+async def startup_event():
+    """Initialize the omni_llm instance on startup."""
+    global omni_llm
+    # This will be set by the run_server function
+    pass
+
+
+@app.post("/generate", response_model=GenerateResponse)
+async def generate(request: GenerateRequest):
+    """Generate text or multimodal content."""
+    try:
+        if omni_llm is None:
+            raise HTTPException(status_code=500, detail="OmniLLM not initialized")
+        
+        # Prepare stage arguments
+        if request.stage_args is None:
+            # Create default stage arguments - one per stage config
+            # For now, we'll process all prompts in the first stage
+            stage_args = [{
+                "prompt": " ".join(request.prompts) if request.prompts else "",
+                "max_tokens": request.max_tokens,
+                "temperature": request.temperature
+            }]
+        else:
+            stage_args = request.stage_args
+        
+        # Generate using omni_llm
+        try:
+            outputs = await omni_llm.generate_async(stage_args)
+        except Exception as e:
+            # Fallback to synchronous generation
+            outputs = omni_llm.generate(stage_args)
+        
+        # Convert outputs to response format
+        response_outputs = []
+        for output in outputs:
+            if hasattr(output, 'outputs') and output.outputs:
+                for out in output.outputs:
+                    response_outputs.append({
+                        "text": getattr(out, 'text', ''),
+                        "finished": getattr(out, 'finish_reason', 'length') != 'length',
+                        "tokens": getattr(out, 'token_ids', [])
+                    })
+            else:
+                response_outputs.append({
+                    "text": "",
+                    "finished": True,
+                    "tokens": []
+                })
+        
+        return GenerateResponse(
+            outputs=response_outputs,
+            stage_outputs=[{"stage": i, "output": "processed"} for i in range(len(stage_args))]
+        )
+    
+    except Exception as e:
+        raise HTTPException(status_code=500, detail=str(e))
+
+
+@app.get("/health")
+async def health_check():
+    """Health check endpoint."""
+    return {"status": "healthy", "service": "vllm-omni"}
+
+
+@app.get("/info")
+async def get_info():
+    """Get information about the service."""
+    if omni_llm is None:
+        return {"error": "OmniLLM not initialized"}
+    
+    return {
+        "service": "vllm-omni",
+        "version": "0.1.0",
+        "num_stages": omni_llm.get_num_stages() if hasattr(omni_llm, 'get_num_stages') else 0,
+        "stage_configs": [
+            {
+                "stage_id": config.stage_id,
+                "engine_type": config.engine_type,
+                "model_path": config.model_path,
+                "input_modalities": config.input_modalities,
+                "output_modalities": config.output_modalities
+            }
+            for config in omni_llm.stage_configs
+        ]
+    }
+
+
+async def run_server(omni_llm_instance: AsyncOmniLLM, host: str = "0.0.0.0", port: int = 8000):
+    """Run the API server."""
+    global omni_llm
+    omni_llm = omni_llm_instance
+    
+    config = uvicorn.Config(
+        app=app,
+        host=host,
+        port=port,
+        log_level="info"
+    )
+    
+    server = uvicorn.Server(config)
+    await server.serve()
+
+
+if __name__ == "__main__":
+    # This is for testing purposes
+    asyncio.run(run_server(None))
@@ -1,742 +0,0 @@
-import asyncio
-import multiprocessing as mp
-import os
-import socket
-import sys
-import time
-from collections.abc import AsyncGenerator, Iterable, Mapping, Sequence
-from concurrent.futures import ThreadPoolExecutor, as_completed
-from typing import Any, Optional, Union
-
-import torch
-
-# External library imports (vLLM)
-import vllm.envs as envs
-from vllm.config import VllmConfig
-from vllm.engine.protocol import EngineClient
-from vllm.inputs import PromptType
-from vllm.inputs.preprocess import InputPreprocessor
-from vllm.logger import init_logger
-from vllm.lora.request import LoRARequest
-from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalRegistry
-from vllm.sampling_params import SamplingParams
-from vllm.tracing import init_tracer
-from vllm.transformers_utils.config import maybe_register_config_serialize_by_value
-from vllm.transformers_utils.tokenizer import AnyTokenizer, init_tokenizer_from_configs
-from vllm.usage.usage_lib import UsageContext
-from vllm.utils import Device, deprecate_kwargs
-from vllm.v1.engine.async_llm import AsyncLLM
-from vllm.v1.engine.core_client import EngineCoreClient
-from vllm.v1.engine.exceptions import EngineDeadError
-from vllm.v1.executor.abstract import Executor
-from vllm.v1.metrics.loggers import StatLoggerFactory, StatLoggerManager
-
-# Internal imports (our code)
-from vllm_omni.config import OmniModelConfig
-from vllm_omni.engine.arg_utils import AsyncOmniEngineArgs
-from vllm_omni.engine.output_processor import MultimodalOutputProcessor
-from vllm_omni.engine.processor import OmniProcessor
-from vllm_omni.entrypoints.log_utils import (
-    OrchestratorMetrics,
-    configure_orchestrator_logger,
-    init_stats_paths,
-    remove_old_logs,
-)
-from vllm_omni.entrypoints.omni_stage import OmniStage
-from vllm_omni.entrypoints.stage_utils import encode_for_ipc as _encode
-from vllm_omni.entrypoints.stage_utils import maybe_load_from_ipc as _load
-from vllm_omni.entrypoints.stage_utils import serialize_obj as _set
-from vllm_omni.entrypoints.utils import load_stage_configs_from_model, load_stage_configs_from_yaml
-from vllm_omni.outputs import OmniRequestOutput
-
-logger = init_logger(__name__)
-
-
-class AsyncOmniLLM(EngineClient):
-    def __init__(
-        self,
-        model: str,
-        stage_configs_path: Optional[str] = None,
-        log_stats: bool = False,
-        log_file: Optional[str] = None,
-        init_sleep_seconds: int = 20,
-        shm_threshold_bytes: int = 65536,
-        batch_timeout: int = 10,
-        init_timeout: int = 300,
-        **kwargs,
-    ):
-        self.batch_timeout = batch_timeout
-        self._enable_stats: bool = bool(log_stats)
-
-        if stage_configs_path is None:
-            self.stage_configs = load_stage_configs_from_model(model)
-        else:
-            self.stage_configs = load_stage_configs_from_yaml(stage_configs_path)
-
-        self.stage_list: list[OmniStage] = []
-        self.default_sampling_params_list: list[SamplingParams] = []
-        # Optional file handler for orchestrator
-        self._log_file = log_file
-        if self._log_file:
-            remove_old_logs(self._log_file, len(self.stage_list))
-            configure_orchestrator_logger(logger, self._log_file)
-
-        self._stats_file, self._overall_stats_file = init_stats_paths(self._enable_stats, self._log_file)
-        self._initialize_stages(model, init_sleep_seconds, shm_threshold_bytes, init_timeout)
-
-    def _initialize_stages(
-        self,
-        model: str,
-        init_sleep_seconds: int,
-        shm_threshold_bytes: int,
-        init_timeout: int,
-    ) -> None:
-        self.stage_list: list[OmniStage] = []
-
-        # Build OmniStage instances in parallel, preserve original order
-        def _build_stage(idx_cfg: tuple[int, Any]) -> tuple[int, OmniStage]:
-            idx, cfg = idx_cfg
-            return idx, OmniStage(cfg)
-
-        with ThreadPoolExecutor(max_workers=min(len(self.stage_configs), max(1, os.cpu_count() or 1))) as executor:
-            futures = [executor.submit(_build_stage, (idx, cfg)) for idx, cfg in enumerate(self.stage_configs)]
-            results: list[tuple[int, OmniStage]] = []
-            for fut in as_completed(futures):
-                results.append(fut.result())
-        results.sort(key=lambda x: x[0])
-        self.stage_list = [st for _, st in results]
-        self.default_sampling_params_list = [st.default_sampling_params for st in self.stage_list]
-        logger.debug("[Orchestrator] Loaded %d stages", len(self.stage_list))
-
-        self._ctx = mp.get_context("spawn")
-        self._stage_in_queues: list[mp.Queue] = []
-        self._stage_out_queues: list[mp.Queue] = []
-        self._init_sleep_seconds = max(0, int(init_sleep_seconds))
-        self._shm_threshold_bytes = max(0, int(shm_threshold_bytes))
-        self._start_stage_processes(model)
-        # Wait for all stages to report readiness before seeding
-        self._stages_ready: set[int] = set()
-        self._wait_for_stages_ready(timeout=init_timeout)
-
-    def _start_stage_processes(self, model: str) -> None:
-        for stage_id, stage in enumerate(self.stage_list):
-            # Use unbounded queues to avoid deadlock when seeding many requests
-            in_q: mp.Queue = self._ctx.Queue(maxsize=0)
-            out_q: mp.Queue = self._ctx.Queue(maxsize=0)
-            self._stage_in_queues.append(in_q)
-            self._stage_out_queues.append(out_q)
-
-            # Attach queues and start Stage-owned worker process
-            stage.attach_queues(in_q, out_q)
-            stage.init_stage_worker(
-                model,
-                is_async=True,
-                log_file=self._log_file,
-                shm_threshold_bytes=self._shm_threshold_bytes,
-                ctx=self._ctx,
-                batch_timeout=self.batch_timeout,
-            )
-            logger.debug("[Orchestrator] Stage-%s process started", stage_id)
-            time.sleep(self._init_sleep_seconds)
-
-    def close(self) -> None:
-        for q in self._stage_in_queues:
-            try:
-                q.put_nowait(None)
-            except Exception as e:
-                logger.warning(
-                    "[Orchestrator] Failed to send shutdown signal to \
-                        stage input queue: %s",
-                    e,
-                )
-        for stage in self.stage_list:
-            try:
-                stage.stop_stage_worker()
-            except Exception as e:
-                logger.warning("[Orchestrator] Failed to stop stage worker: %s", e)
-
-    def __del__(self) -> None:  # best-effort
-        try:
-            self.close()
-        except Exception as e:
-            logger.debug("[Orchestrator] __del__ close() raised: %s", e, exc_info=True)
-
-    def shutdown(self):
-        """Shutdown, cleaning up the background proc and IPC."""
-        try:
-            self.close()
-        except Exception as e:
-            logger.debug("[Orchestrator] __del__ close() raised: %s", e, exc_info=True)
-
-    async def generate(
-        self,
-        prompt: PromptType,
-        request_id: str,
-        sampling_params_list: Optional[Union[SamplingParams, Sequence[SamplingParams]]] = None,
-        lora_request: Optional[LoRARequest] = None,
-        trace_headers: Optional[Mapping[str, str]] = None,
-        priority: int = 0,
-        data_parallel_rank: Optional[int] = None,
-    ) -> AsyncGenerator[OmniRequestOutput, None]:
-        logger.debug("[Orchestrator] generate() called")
-        if sampling_params_list is None:
-            sampling_params_list = self.default_sampling_params_list
-        if len(sampling_params_list) != len(self.stage_list):
-            raise ValueError(
-                f"Expected {len(self.stage_list)} sampling params, \
-                got {len(sampling_params_list)}"
-            )
-
-        # Orchestrator keeps stage objects for input derivation
-        num_stages = len(self.stage_list)
-        # Track per-request start time for end-to-end timing
-        _req_start_ts: dict[int, float] = {}
-        _wall_start_ts: float = time.time()
-        # _last_finish_ts: float = _wall_start_ts
-
-        # Determine the final stage for E2E stats (highest stage_id with
-        # final_output=True; fallback to last stage)
-        final_stage_id_for_e2e = -1
-        try:
-            for _sid, _st in enumerate(self.stage_list):
-                if getattr(_st, "final_output", False):
-                    final_stage_id_for_e2e = max(final_stage_id_for_e2e, _sid)
-            if final_stage_id_for_e2e < 0:
-                final_stage_id_for_e2e = len(self.stage_list) - 1
-        except Exception as e:
-            logger.debug(
-                "[Orchestrator] Failed to determine final stage for E2E; \
-                    falling back to last: %s",
-                e,
-                exc_info=True,
-            )
-            final_stage_id_for_e2e = len(self.stage_list) - 1
-        # Metrics/aggregation helper
-        metrics = OrchestratorMetrics(
-            num_stages,
-            self._enable_stats,
-            self._stats_file,
-            self._overall_stats_file,
-            _wall_start_ts,
-        )
-        # Seed stage-0 queue with all requests
-        logger.debug("[Orchestrator] Seeding request into stage-0")
-        # Mark first input time for stage-0
-        metrics.stage_first_ts[0] = metrics.stage_first_ts[0] or time.time()
-
-        sp0: SamplingParams = sampling_params_list[0]  # type: ignore[index]
-        task = {
-            "request_id": request_id,
-            "engine_inputs": prompt,
-            "sampling_params": sp0,
-        }
-        self.stage_list[0].submit(task)
-        _req_start_ts[request_id] = time.time()
-        logger.debug("[Orchestrator] Enqueued request %s to stage-0", request_id)
-
-        logger.debug("[Orchestrator] Entering scheduling loop: stages=%d", num_stages)
-        finished = False
-        while not finished:
-            made_progress = False
-            for stage_id, stage in enumerate(self.stage_list):
-                result = stage.try_collect()
-                if result is None:
-                    continue
-
-                made_progress = True
-                req_id = result.get("request_id")
-                if "error" in result:
-                    logger.error(
-                        "Stage %s error on request %s: %s",
-                        stage_id,
-                        req_id,
-                        result["error"],
-                    )
-                    continue
-
-                if result.get("type") == "stage_ready":
-                    # Only happens when stage is initialized slower than expected,
-                    # so we wait for a short time and try again
-                    time.sleep(0.05)
-                    continue
-
-                engine_outputs = _load(result, obj_key="engine_outputs", shm_key="engine_outputs_shm")
-                # Mark last output time for this stage whenever we receive outputs
-                metrics.stage_last_ts[stage_id] = max(metrics.stage_last_ts[stage_id] or 0.0, time.time())
-                try:
-                    _m = result.get("metrics")
-                    if _m is not None:
-                        metrics.on_stage_metrics(stage_id, req_id, _m)
-                except Exception as e:
-                    logger.exception(
-                        "[Orchestrator] Failed to process metrics for stage %s, \
-                            req %s: %s",
-                        stage_id,
-                        req_id,
-                        e,
-                    )
-                logger.debug(
-                    "[Orchestrator] Stage-%s completed request %s; \
-                        forwarding or finalizing",
-                    stage_id,
-                    req_id,
-                )
-                stage.set_engine_outputs(engine_outputs)
-
-                if getattr(stage, "final_output", False):
-                    logger.debug(
-                        "[Orchestrator] Request %s finalized at stage-%s",
-                        req_id,
-                        stage_id,
-                    )
-
-                    # End-to-end timing and time-per-token for final output
-                    # (only once per request at the designated final stage)
-                    try:
-                        rid_int = int(req_id) if isinstance(req_id, (int, str)) and str(req_id).isdigit() else req_id
-                        if stage_id == final_stage_id_for_e2e and rid_int not in metrics.e2e_done:
-                            metrics.on_finalize_request(
-                                stage_id,
-                                req_id,
-                                engine_outputs,
-                                _req_start_ts.get(req_id, _wall_start_ts),
-                            )
-                    except Exception as e:
-                        logger.exception(
-                            "[Orchestrator] Finalize request handling error for \
-                                req %s at stage %s: %s",
-                            req_id,
-                            stage_id,
-                            e,
-                        )
-
-                    if isinstance(engine_outputs, list):
-                        engine_outputs = engine_outputs[0]
-                    yield OmniRequestOutput(
-                        stage_id=stage_id,
-                        final_output_type=stage.final_output_type,
-                        request_output=engine_outputs,
-                    )
-
-                next_stage_id = stage_id + 1
-                if next_stage_id < num_stages:
-                    next_stage: OmniStage = self.stage_list[next_stage_id]
-                    next_inputs = next_stage.process_engine_inputs(self.stage_list, prompt)
-                    sp_next: SamplingParams = sampling_params_list[next_stage_id]
-                    try:
-                        # Measure transfer size and time (encode + enqueue)
-                        size_bytes = 0
-                        try:
-                            size_bytes = len(_set(next_inputs))
-                        except Exception:
-                            size_bytes = 0
-                        t0 = time.time()
-                        ipc_payload = _encode(
-                            next_inputs,
-                            getattr(self, "_shm_threshold_bytes", 65536),
-                            obj_key="engine_inputs",
-                            shm_key="engine_inputs_shm",
-                        )
-                        ipc_payload.update(
-                            {
-                                "request_id": req_id,
-                                "sampling_params": sp_next,
-                                "sent_ts": time.time(),
-                            }
-                        )
-                        self.stage_list[next_stage_id].submit(ipc_payload)
-                        t1 = time.time()
-                        tx_ms = (t1 - t0) * 1000.0
-                        metrics.on_forward(
-                            stage_id,
-                            next_stage_id,
-                            req_id,
-                            int(size_bytes),
-                            float(tx_ms),
-                            bool("engine_inputs_shm" in ipc_payload),
-                        )
-                    except Exception as e:
-                        logger.warning(
-                            "[Orchestrator] IPC encode failed for req %s: %s; \
-                                falling back to inline payload",
-                            req_id,
-                            e,
-                        )
-                        self.stage_list[next_stage_id].submit(
-                            {
-                                "request_id": req_id,
-                                "engine_inputs": next_inputs,
-                                "sampling_params": sp_next,
-                            }
-                        )
-                    logger.debug(
-                        "[Orchestrator] Forwarded request %s to stage-%s",
-                        req_id,
-                        next_stage_id,
-                    )
-                else:
-                    finished = True
-                    logger.debug("[Orchestrator] Request %s fully completed", req_id)
-
-            if not made_progress:
-                time.sleep(0.005)
-
-        logger.debug("[Orchestrator] All requests completed")
-
-        # Summarize and print stats
-        try:
-            summary = metrics.build_and_log_summary(final_stage_id_for_e2e)
-            logger.info("[Summary] %s", summary)
-        except Exception as e:
-            logger.exception("[Orchestrator] Failed to build/log summary: %s", e)
-
-    def _wait_for_stages_ready(self, timeout: int = 120) -> None:
-        deadline = time.time() + max(0, int(timeout))
-        num_stages = len(self.stage_list)
-        while len(self._stages_ready) < num_stages and time.time() < deadline:
-            progressed = False
-            for stage_id, stage in enumerate(self.stage_list):
-                if stage_id in self._stages_ready:
-                    continue
-                result = stage.try_collect()
-                if result is None:
-                    continue
-                progressed = True
-                if result.get("type") == "stage_ready":
-                    self._stages_ready.add(stage_id)
-                    # Store vllm_config received from worker process
-                    vllm_config = result.get("vllm_config")
-                    if vllm_config is not None:
-                        stage.set_vllm_config(vllm_config)
-                    tokenizer = result.get("tokenizer")
-                    if tokenizer is not None:
-                        stage.set_tokenizer(tokenizer)
-                    # input_preprocessor = result.get("input_preprocessor")
-                    # if input_preprocessor is not None:
-                    #     stage.set_input_preprocessor(input_preprocessor)
-                    is_tracing_enabled = result.get("is_tracing_enabled")
-                    if is_tracing_enabled is not None:
-                        stage.set_is_tracing_enabled(is_tracing_enabled)
-                    logger.debug("[Orchestrator] Stage-%s reported ready", stage_id)
-                else:
-                    # No user data should arrive before seeding; ignore other messages
-                    pass
-            if not progressed:
-                time.sleep(0.01)
-        if len(self._stages_ready) < num_stages:
-            not_ready = sorted(set(range(num_stages)) - set(self._stages_ready))
-            logger.warning(
-                "[Orchestrator] Initialization timeout: only %s/%s stages are \
-                    ready; not ready: %s",
-                len(self._stages_ready),
-                num_stages,
-                not_ready,
-            )
-            # Provide actionable suggestions before shutdown
-            try:
-                suggestions = [
-                    "Verify GPU/device assignment in config (runtime.devices) is \
-                        correct.",
-                    "Check GPU/host memory availability; reduce model or batch size if needed.",  # noqa: E501
-                    "Check model weights path and network reachability (if loading remotely).",  # noqa: E501
-                    "Increase initialization wait time (init_sleep_seconds or \
-                        call-site timeout).",
-                ]
-                if getattr(self, "_log_file", None):
-                    suggestions.append(
-                        f"Inspect per-stage log files for details: \
-                            {self._log_file}.stage<id>.log"
-                    )
-                logger.error(
-                    "[Orchestrator] Stage initialization failed, shutting down. \
-                        Suggestions:\n- %s",
-                    "\n- ".join(suggestions),
-                )
-            except Exception:
-                # Best-effort logging of suggestions
-                logger.error(
-                    "[Orchestrator] Stage initialization failed and an error \
-                        occurred while logging suggestions",
-                )
-
-            # Attempt graceful shutdown of all stages before exiting
-            try:
-                self.close()
-            except Exception:
-                pass
-
-            # Terminate the current process with non-zero exit code
-            try:
-                sys.exit(1)
-            except SystemExit:
-                raise
-            except Exception:
-                os._exit(1)
-
-    @property
-    def is_running(self) -> bool:
-        # Is None before the loop is started.
-        return len(self._stage_in_queues) > 0
-
-    @property
-    def is_stopped(self) -> bool:
-        return self.errored
-
-    @property
-    def errored(self) -> bool:
-        return not self.is_running
-
-    @property
-    def dead_error(self) -> BaseException:
-        return EngineDeadError()
-
-    async def abort(self, request_id: Union[str, Iterable[str]]) -> None:
-        pass
-
-    async def get_vllm_config(self) -> VllmConfig:
-        for stage in self.stage_list:
-            if stage.is_comprehension:
-                # Use the vllm_config received from worker process
-                if stage.vllm_config is not None:
-                    return stage.vllm_config
-        return None
-
-    async def get_model_config(self) -> OmniModelConfig:
-        for stage in self.stage_list:
-            if stage.is_comprehension:
-                # Use the vllm_config received from worker process
-                if stage.vllm_config is not None:
-                    return stage.vllm_config.model_config
-        return None
-
-    async def get_input_preprocessor(self) -> InputPreprocessor:
-        # for stage in self.stage_list:
-        #     if stage.is_comprehension:
-        #         return stage.input_preprocessor
-        return None
-
-    async def get_tokenizer(self) -> AnyTokenizer:
-        for stage in self.stage_list:
-            if stage.is_comprehension:
-                return stage.tokenizer
-        return None
-
-    async def is_tracing_enabled(self) -> bool:
-        for stage in self.stage_list:
-            if stage.is_comprehension:
-                return stage.is_tracing_enabled
-        return False
-
-    async def do_log_stats(self) -> None:
-        pass
-
-    async def check_health(self) -> None:
-        pass
-
-    async def reset_mm_cache(self) -> None:
-        pass
-
-    async def reset_prefix_cache(self, device: Optional[Device] = None) -> None:
-        pass
-
-    async def sleep(self, level: int = 1) -> None:
-        pass
-
-    async def wake_up(self, tags: Optional[list[str]] = None) -> None:
-        pass
-
-    async def is_sleeping(self) -> bool:
-        """Check whether the engine is sleeping"""
-        return False
-
-    async def add_lora(self, lora_request: LoRARequest) -> bool:
-        """Load a new LoRA adapter into the engine for future requests."""
-        return False
-
-    async def encode(
-        self,
-        *args,
-        **kwargs,
-    ):
-        """Generate outputs for a request from a pooling model."""
-        raise NotImplementedError("encode() is not implemented for AsyncOmniLLM")
-
-    async def start_profile(self) -> None:
-        raise NotImplementedError("start_profile() is not implemented for AsyncOmniLLM")
-
-    async def stop_profile(self) -> None:
-        raise NotImplementedError("stop_profile() is not implemented for AsyncOmniLLM")
-
-
-class AsyncOmniStageLLM(AsyncLLM):
-    def __init__(
-        self,
-        engine_args: AsyncOmniEngineArgs,
-        vllm_config: VllmConfig,
-        executor_class: type[Executor],
-        log_stats: bool,
-        usage_context: UsageContext = UsageContext.ENGINE_CONTEXT,
-        mm_registry: MultiModalRegistry = MULTIMODAL_REGISTRY,
-        use_cached_outputs: bool = False,
-        log_requests: bool = True,
-        start_engine_loop: bool = True,
-        stat_loggers: Optional[list[StatLoggerFactory]] = None,
-        client_addresses: Optional[dict[str, str]] = None,
-        client_count: int = 1,
-        client_index: int = 0,
-    ) -> None:
-        """
-        Create an AsyncLLM.
-
-        Args:
-            vllm_config: global configuration.
-            executor_class: an Executor impl, e.g. MultiprocExecutor.
-            log_stats: Whether to log stats.
-            usage_context: Usage context of the LLM.
-            mm_registry: Multi-modal registry.
-            use_cached_outputs: Whether to use cached outputs.
-            log_requests: Whether to log requests.
-            start_engine_loop: Whether to start the engine loop.
-            stat_loggers: customized stat loggers for the engine.
-                If not provided, default stat loggers will be used.
-                PLEASE BE AWARE THAT STAT LOGGER IS NOT STABLE
-                IN V1, AND ITS BASE CLASS INTERFACE MIGHT CHANGE.
-
-        Returns:
-            None
-        """
-        if not envs.VLLM_USE_V1:
-            raise ValueError(
-                "Using V1 AsyncLLMEngine, but envs.VLLM_USE_V1=False. "
-                "This should not happen. As a workaround, try using "
-                "AsyncLLMEngine.from_vllm_config(...) or explicitly set "
-                "VLLM_USE_V1=0 or 1 and report this issue on Github."
-            )
-
-        # Ensure we can serialize custom transformer configs
-        maybe_register_config_serialize_by_value()
-
-        self.model_config = vllm_config.model_config
-        self.vllm_config = vllm_config
-        self.observability_config = vllm_config.observability_config
-        self.log_requests = log_requests
-
-        self.log_stats = log_stats or (stat_loggers is not None)
-        if not log_stats and stat_loggers is not None:
-            logger.info(
-                "AsyncLLM created with log_stats=False and non-empty custom logger list; "
-                "enabling logging without default stat loggers"
-            )
-
-        if self.model_config.skip_tokenizer_init:
-            self.tokenizer = None
-        else:
-            # Tokenizer (+ ensure liveness if running in another process).
-            self.tokenizer = init_tokenizer_from_configs(model_config=vllm_config.model_config)
-
-        # Processor (converts Inputs --> EngineCoreRequests).
-        self.processor = OmniProcessor(
-            vllm_config=vllm_config,
-            tokenizer=self.tokenizer,
-            mm_registry=mm_registry,
-        )
-
-        # OutputProcessor (converts EngineCoreOutputs --> RequestOutput).
-        self.output_processor = MultimodalOutputProcessor(
-            tokenizer=self.tokenizer,
-            log_stats=self.log_stats,
-            engine_core_output_type=engine_args.engine_output_type,
-        )
-        if self.observability_config.otlp_traces_endpoint is not None:
-            tracer = init_tracer("vllm.llm_engine", self.observability_config.otlp_traces_endpoint)
-            self.output_processor.tracer = tracer
-
-        # EngineCore (starts the engine in background process).
-        self.engine_core = EngineCoreClient.make_async_mp_client(
-            vllm_config=vllm_config,
-            executor_class=executor_class,
-            log_stats=self.log_stats,
-            client_addresses=client_addresses,
-            client_count=client_count,
-            client_index=client_index,
-        )
-
-        # Loggers.
-        self.logger_manager: Optional[StatLoggerManager] = None
-        if self.log_stats:
-            self.logger_manager = StatLoggerManager(
-                vllm_config=vllm_config,
-                engine_idxs=self.engine_core.engine_ranks_managed,
-                custom_stat_loggers=stat_loggers,
-                enable_default_loggers=log_stats,
-                client_count=client_count,
-            )
-            self.logger_manager.log_engine_initialized()
-
-        self.output_handler: Optional[asyncio.Task] = None
-        try:
-            # Start output handler eagerly if we are in the asyncio eventloop.
-            asyncio.get_running_loop()
-            self._run_output_handler()
-        except RuntimeError:
-            pass
-
-        if envs.VLLM_TORCH_PROFILER_DIR:
-            logger.info(
-                "Torch profiler enabled. AsyncLLM CPU traces will be collected under %s",  # noqa: E501
-                envs.VLLM_TORCH_PROFILER_DIR,
-            )
-            worker_name = f"{socket.gethostname()}_{os.getpid()}.async_llm"
-            self.profiler = torch.profiler.profile(
-                activities=[
-                    torch.profiler.ProfilerActivity.CPU,
-                ],
-                with_stack=envs.VLLM_TORCH_PROFILER_WITH_STACK,
-                on_trace_ready=torch.profiler.tensorboard_trace_handler(
-                    envs.VLLM_TORCH_PROFILER_DIR, worker_name=worker_name, use_gzip=True
-                ),
-            )
-        else:
-            self.profiler = None
-
-    @classmethod
-    @deprecate_kwargs(
-        "disable_log_requests",
-        additional_message=("This argument will have no effect. Use `enable_log_requests` instead."),
-    )
-    def from_vllm_config(
-        cls,
-        vllm_config: VllmConfig,
-        engine_args: AsyncOmniEngineArgs,
-        start_engine_loop: bool = True,
-        usage_context: UsageContext = UsageContext.ENGINE_CONTEXT,
-        stat_loggers: Optional[list[StatLoggerFactory]] = None,
-        enable_log_requests: bool = False,
-        disable_log_stats: bool = False,
-        client_addresses: Optional[dict[str, str]] = None,
-        client_count: int = 1,
-        client_index: int = 0,
-        disable_log_requests: bool = True,  # Deprecated, will be removed
-    ) -> "AsyncLLM":
-        if not envs.VLLM_USE_V1:
-            raise ValueError(
-                "Using V1 AsyncLLMEngine, but envs.VLLM_USE_V1=False. "
-                "This should not happen. As a workaround, try using "
-                "AsyncLLMEngine.from_vllm_config(...) or explicitly set "
-                "VLLM_USE_V1=0 or 1 and report this issue on Github."
-            )
-
-        # Create the LLMEngine.
-        return cls(
-            vllm_config=vllm_config,
-            executor_class=Executor.get_class(vllm_config),
-            start_engine_loop=start_engine_loop,
-            stat_loggers=stat_loggers,
-            log_requests=enable_log_requests,
-            log_stats=not disable_log_stats,
-            usage_context=usage_context,
-            client_addresses=client_addresses,
-            client_count=client_count,
-            client_index=client_index,
-            engine_args=engine_args,
-        )
@@ -1,242 +0,0 @@
-from collections.abc import Awaitable, Iterable
-from typing import Any, Optional, Union, cast
-
-import numpy as np
-from openai.types.chat import ChatCompletionContentPartTextParam
-from vllm.config import ModelConfig
-from vllm.entrypoints.chat_utils import (
-    AsyncMultiModalContentParser,
-    AsyncMultiModalItemTracker,
-    BaseMultiModalContentParser,
-    BaseMultiModalItemTracker,
-    ChatCompletionContentPartParam,
-    ChatCompletionMessageParam,
-    ConversationMessage,
-    MultiModalDataDict,
-    MultiModalUUIDDict,
-    _AssistantParser,
-    _ChatTemplateContentFormat,
-    _ContentPart,
-    _get_full_multimodal_text_prompt,
-    _parse_chat_message_content_part,
-    _postprocess_messages,
-    _ToolParser,
-)
-from vllm.transformers_utils.tokenizer import AnyTokenizer
-
-
-class OmniAsyncMultiModalItemTracker(AsyncMultiModalItemTracker):
-    def create_parser(self) -> "BaseMultiModalContentParser":
-        return OmniAsyncMultiModalContentParser(self)
-
-
-class OmniAsyncMultiModalContentParser(AsyncMultiModalContentParser):
-    def __init__(self, tracker: AsyncMultiModalItemTracker) -> None:
-        super().__init__(tracker=tracker)
-        self._mm_processor_kwargs: Optional[dict[str, Any]] = None
-
-    def set_mm_processor_kwargs(self, mm_processor_kwargs: Optional[dict[str, Any]]) -> None:
-        """Set mm_processor_kwargs for use in parsing."""
-        self._mm_processor_kwargs = mm_processor_kwargs
-
-    def parse_video(self, video_url: Optional[str], uuid: Optional[str] = None) -> None:
-        video = self._connector.fetch_video_async(video_url=video_url) if video_url else None
-
-        placeholder = self._tracker.add("video", video, uuid)
-        self._add_placeholder("video", placeholder)
-
-        # Extract audio from video if use_audio_in_video is True
-        if video_url and self._mm_processor_kwargs and self._mm_processor_kwargs.get("use_audio_in_video", False):
-            audio_coro = self._extract_audio_from_video_async(video_url)
-            audio_placeholder = self._tracker.add("audio", audio_coro, uuid)
-            self._add_placeholder("audio", audio_placeholder)
-
-    async def _extract_audio_from_video_async(self, video_url: str) -> tuple[np.ndarray, Union[int, float]]:
-        """
-        Extract audio from video URL using librosa.
-        Returns tuple of (audio_array, sample_rate) compatible with audio format.
-
-        All blocking I/O operations are run in a thread pool to avoid blocking the event loop.
-        """
-        import asyncio
-        import os
-        import tempfile
-        from urllib.parse import urlparse
-
-        # Parse URL to determine type
-        parsed_url = urlparse(video_url)
-        temp_video_file_path = None
-
-        def _download_video_sync(url: str) -> bytes:
-            """Synchronous video download - runs in thread pool."""
-            from urllib.request import urlopen
-
-            return urlopen(url).read()
-
-        def _write_temp_file_sync(data: bytes, suffix: str) -> str:
-            """Synchronous temp file write - runs in thread pool."""
-            with tempfile.NamedTemporaryFile(delete=False, suffix=suffix) as temp_file:
-                temp_file.write(data)
-                return temp_file.name
-
-        def _load_audio_sync(file_path: str) -> tuple[np.ndarray, Union[int, float]]:
-            """Synchronous audio loading with librosa - runs in thread pool."""
-            import librosa
-
-            return librosa.load(file_path, sr=16000)
-
-        def _cleanup_file_sync(file_path: str) -> None:
-            """Synchronous file deletion - runs in thread pool."""
-            try:
-                if os.path.exists(file_path):
-                    os.unlink(file_path)
-            except OSError:
-                pass
-
-        try:
-            if parsed_url.scheme in ("http", "https"):
-                # Download video from HTTP/HTTPS URL asynchronously
-                video_data = await asyncio.to_thread(_download_video_sync, video_url)
-                # Write temp file asynchronously
-                temp_video_file_path = await asyncio.to_thread(_write_temp_file_sync, video_data, ".mp4")
-            elif parsed_url.scheme == "file":
-                # Use file path directly (handle Windows paths)
-                from urllib.request import url2pathname
-
-                temp_video_file_path = url2pathname(parsed_url.path)
-            elif parsed_url.scheme == "data":
-                # Handle data URL (base64 encoded video)
-                import base64
-
-                header, data = video_url.split(",", 1)
-                video_data = base64.b64decode(data)
-                # Write temp file asynchronously
-                temp_video_file_path = await asyncio.to_thread(_write_temp_file_sync, video_data, ".mp4")
-            else:
-                # Assume it's a local file path
-                temp_video_file_path = video_url
-
-            # Extract audio using librosa asynchronously (CPU-intensive, runs in thread pool)
-            audio_array, sample_rate = await asyncio.to_thread(_load_audio_sync, temp_video_file_path)
-
-            return audio_array, sample_rate
-        finally:
-            # Clean up temporary file if we created one (asynchronously)
-            if temp_video_file_path and parsed_url.scheme in ("http", "https", "data"):
-                await asyncio.to_thread(_cleanup_file_sync, temp_video_file_path)
-
-
-def parse_chat_messages_futures(
-    messages: list[ChatCompletionMessageParam],
-    model_config: ModelConfig,
-    tokenizer: AnyTokenizer,
-    content_format: _ChatTemplateContentFormat,
-    mm_processor_kwargs: Optional[dict[str, Any]] = None,
-) -> tuple[
-    list[ConversationMessage],
-    Awaitable[Optional[MultiModalDataDict]],
-    Optional[MultiModalUUIDDict],
-]:
-    conversation: list[ConversationMessage] = []
-    mm_tracker = OmniAsyncMultiModalItemTracker(model_config, tokenizer)
-
-    for msg in messages:
-        sub_messages = _parse_chat_message_content(
-            msg,
-            mm_tracker,
-            content_format,
-            interleave_strings=(
-                content_format == "string"
-                and model_config.multimodal_config is not None
-                and model_config.multimodal_config.interleave_mm_strings
-            ),
-            mm_processor_kwargs=mm_processor_kwargs,
-        )
-
-        conversation.extend(sub_messages)
-
-    _postprocess_messages(conversation)
-
-    return conversation, mm_tracker.all_mm_data(), mm_tracker.all_mm_uuids()
-
-
-def _parse_chat_message_content(
-    message: ChatCompletionMessageParam,
-    mm_tracker: BaseMultiModalItemTracker,
-    content_format: _ChatTemplateContentFormat,
-    interleave_strings: bool,
-    mm_processor_kwargs: Optional[dict[str, Any]] = None,
-) -> list[ConversationMessage]:
-    role = message["role"]
-    content = message.get("content")
-
-    if content is None:
-        content = []
-    elif isinstance(content, str):
-        content = [ChatCompletionContentPartTextParam(type="text", text=content)]
-    result = _parse_chat_message_content_parts(
-        role,
-        content,  # type: ignore
-        mm_tracker,
-        wrap_dicts=(content_format == "openai"),
-        interleave_strings=interleave_strings,
-        mm_processor_kwargs=mm_processor_kwargs,
-    )
-
-    for result_msg in result:
-        if role == "assistant":
-            parsed_msg = _AssistantParser(message)
-
-            # The 'tool_calls' is not None check ensures compatibility.
-            # It's needed only if downstream code doesn't strictly
-            # follow the OpenAI spec.
-            if "tool_calls" in parsed_msg and parsed_msg["tool_calls"] is not None:
-                result_msg["tool_calls"] = list(parsed_msg["tool_calls"])
-        elif role == "tool":
-            parsed_msg = _ToolParser(message)
-            if "tool_call_id" in parsed_msg:
-                result_msg["tool_call_id"] = parsed_msg["tool_call_id"]
-
-        if "name" in message and isinstance(message["name"], str):
-            result_msg["name"] = message["name"]
-
-    return result
-
-
-def _parse_chat_message_content_parts(
-    role: str,
-    parts: Iterable[ChatCompletionContentPartParam],
-    mm_tracker: BaseMultiModalItemTracker,
-    *,
-    wrap_dicts: bool,
-    interleave_strings: bool,
-    mm_processor_kwargs: Optional[dict[str, Any]] = None,
-) -> list[ConversationMessage]:
-    content = list[_ContentPart]()
-
-    mm_parser = mm_tracker.create_parser()
-    # Set mm_processor_kwargs if parser supports it
-    if hasattr(mm_parser, "set_mm_processor_kwargs"):
-        mm_parser.set_mm_processor_kwargs(mm_processor_kwargs)
-
-    for part in parts:
-        parse_res = _parse_chat_message_content_part(
-            part,
-            mm_parser,
-            wrap_dicts=wrap_dicts,
-            interleave_strings=interleave_strings,
-        )
-        if parse_res:
-            content.append(parse_res)
-
-    if wrap_dicts:
-        # Parsing wraps images and texts as interleaved dictionaries
-        return [ConversationMessage(role=role, content=content)]  # type: ignore
-    texts = cast(list[str], content)
-    mm_placeholder_storage = mm_parser.mm_placeholder_storage()
-    if mm_placeholder_storage:
-        text_prompt = _get_full_multimodal_text_prompt(mm_placeholder_storage, texts, interleave_strings)
-    else:
-        text_prompt = "\n".join(texts)
-
-    return [ConversationMessage(role=role, content=text_prompt)]
@@ -2,55 +2,24 @@
 CLI entry point for vLLM-omni that intercepts vLLM commands.
 """

-import importlib.metadata
 import sys
+import argparse
+from typing import List, Optional
+from vllm_omni.entrypoints.cli.serve import OmniServeCommand


 def main():
    """Main CLI entry point that intercepts vLLM commands."""
    # Check if --omni flag is present
-    if "--omni" not in sys.argv:
-        from vllm.entrypoints.cli.main import main as vllm_main
-
-        vllm_main()
-        return
+    if "--omni" in sys.argv:
+        # Remove --omni flag and "serve" command, process with vLLM-omni
+        omni_args = [arg for arg in sys.argv[1:] if arg not in ["--omni", "serve"]]
+        omni_serve = OmniServeCommand()
+        omni_serve.run(omni_args)
    else:
-        from vllm.entrypoints.utils import VLLM_SUBCMD_PARSER_EPILOG, cli_env_setup
-        from vllm.utils import FlexibleArgumentParser
-
-        import vllm_omni.entrypoints.cli.serve
-
-        CMD_MODULES = [
-            vllm_omni.entrypoints.cli.serve,
-        ]
-
-        cli_env_setup()
-
-        parser = FlexibleArgumentParser(
-            description="vLLM OMNI CLI",
-            epilog=VLLM_SUBCMD_PARSER_EPILOG.format(subcmd="[subcommand]"),
-        )
-        parser.add_argument(
-            "-v",
-            "--version",
-            action="version",
-            version=importlib.metadata.version("vllm_omni"),
-        )
-        subparsers = parser.add_subparsers(required=False, dest="subparser")
-        cmds = {}
-        for cmd_module in CMD_MODULES:
-            new_cmds = cmd_module.cmd_init()
-            for cmd in new_cmds:
-                cmd.subparser_init(subparsers).set_defaults(dispatch_function=cmd.cmd)
-                cmds[cmd.name] = cmd
-        args = parser.parse_args()
-        if args.subparser in cmds:
-            cmds[args.subparser].validate(args)
-
-        if hasattr(args, "dispatch_function"):
-            args.dispatch_function(args)
-        else:
-            parser.print_help()
+        # Forward to original vLLM CLI
+        from vllm.entrypoints.cli.main import main as vllm_main
+        vllm_main()


 if __name__ == "__main__":
@@ -3,65 +3,206 @@ Omni serve command for vLLM-omni.
 """

 import argparse
+import asyncio
+from typing import List, Optional

-import uvloop
-from vllm.entrypoints.cli.types import CLISubcommand
-from vllm.entrypoints.openai.cli_args import make_arg_parser, validate_parsed_serve_args
-from vllm.entrypoints.utils import VLLM_SUBCMD_PARSER_EPILOG
-from vllm.logger import init_logger
-from vllm.utils import FlexibleArgumentParser
-
-from vllm_omni.entrypoints.openai.api_server import omni_run_server
-
-logger = init_logger(__name__)
-
-DESCRIPTION = """Launch a local OpenAI-compatible API server to serve Omni LLM
-completions via HTTP. Defaults to Qwen/Qwen2.5-Omni-7B if no model is specified.
-
-Search by using: `--help=<ConfigGroup>` to explore options by section (e.g.,
--help=ModelConfig, --help=Frontend)
-  Use `--help=all` to show all available flags at once.
-"""
+from ..omni_llm import AsyncOmniLLM
+from vllm_omni.config import (
+    DiTConfig,
+    create_ar_stage_config,
+    create_dit_stage_config,
+)


-class OmniServeCommand(CLISubcommand):
-    """The `serve` subcommand for the vLLM CLI."""
-
-    name = "serve"
-
-    @staticmethod
-    def cmd(args: argparse.Namespace) -> None:
-        # If model is specified in CLI (as positional arg), it takes precedence
-        if hasattr(args, "model_tag") and args.model_tag is not None:
-            args.model = args.model_tag
-
-        uvloop.run(omni_run_server(args))
-
-    def validate(self, args: argparse.Namespace) -> None:
-        validate_parsed_serve_args(args)
-
-    def subparser_init(self, subparsers: argparse._SubParsersAction) -> FlexibleArgumentParser:
-        serve_parser = subparsers.add_parser(
-            self.name,
-            description=DESCRIPTION,
-            usage="vllm serve [model_tag] --omni [options]",
+class OmniServeCommand:
+    """Command handler for vLLM-omni serve command."""
+    
+    def __init__(self):
+        self.parser = self._create_parser()
+    
+    def _create_parser(self) -> argparse.ArgumentParser:
+        """Create argument parser for omni serve command."""
+        parser = argparse.ArgumentParser(
+            description="vLLM-omni: Multi-modality models inference and serving"
        )
-
-        serve_parser = make_arg_parser(serve_parser)
-        serve_parser.epilog = VLLM_SUBCMD_PARSER_EPILOG.format(subcmd=self.name)
-        serve_parser.add_argument(
-            "--omni",
-            action="store_true",
-            help="Enable vLLM-omni mode",
+        
+        # Model arguments - make it optional with default
+        parser.add_argument(
+            "model",
+            nargs="?",
+            default="Qwen/Qwen3-0.6B",
+            help="Path to the model or model name (default: Qwen/Qwen3-0.6B)"
        )
-        serve_parser.add_argument(
-            "--stage-configs-path",
+        
+        # Server arguments
+        parser.add_argument(
+            "--port",
+            type=int,
+            default=8000,
+            help="Port to run the server on"
+        )
+        
+        parser.add_argument(
+            "--host",
            type=str,
-            default=None,
-            help="Path to the stage configs file. If not specified, the stage configs will be loaded from the model.",
+            default="0.0.0.0",
+            help="Host to run the server on"
        )
-        return serve_parser
-
-
-def cmd_init() -> list[CLISubcommand]:
-    return [OmniServeCommand()]
+        
+        # Stage configuration arguments
+        parser.add_argument(
+            "--ar-stage",
+            type=str,
+            help="AR stage model path"
+        )
+        
+        parser.add_argument(
+            "--dit-stage",
+            type=str,
+            help="DiT stage model path"
+        )
+        
+        parser.add_argument(
+            "--dit-steps",
+            type=int,
+            default=50,
+            help="Number of DiT inference steps"
+        )
+        
+        parser.add_argument(
+            "--dit-guidance-scale",
+            type=float,
+            default=7.5,
+            help="DiT guidance scale"
+        )
+        
+        parser.add_argument(
+            "--use-diffusers",
+            action="store_true",
+            help="Use diffusers pipeline for DiT stage"
+        )
+        
+        # Other arguments
+        parser.add_argument(
+            "--log-stats",
+            action="store_true",
+            help="Enable logging statistics"
+        )
+        
+        return parser
+    
+    def run(self, args: List[str]) -> None:
+        """Run the omni serve command."""
+        parsed_args = self.parser.parse_args(args)
+        
+        # Create stage configurations
+        stage_configs = self._create_stage_configs(parsed_args)
+        
+        # Create AsyncOmniLLM instance
+        omni_llm = AsyncOmniLLM(
+            stage_configs=stage_configs,
+            log_stats=parsed_args.log_stats
+        )
+        
+        # Start the server
+        asyncio.run(self._start_server(omni_llm, parsed_args))
+    
+    def _create_stage_configs(self, args) -> List:
+        """Create stage configurations based on arguments."""
+        stage_configs = []
+        stage_id = 0
+        
+        # Add AR stage - use main model if ar_stage not specified
+        ar_model = args.ar_stage if args.ar_stage else args.model
+        ar_config = create_ar_stage_config(
+            stage_id=stage_id,
+            model_path=ar_model,
+            input_modalities=["text"],
+            output_modalities=["text"]
+        )
+        stage_configs.append(ar_config)
+        stage_id += 1
+        
+        # Add DiT stage if specified
+        if args.dit_stage:
+            dit_model = args.dit_stage
+        elif args.use_diffusers:
+            # Use a default DiT model if diffusers is enabled
+            dit_model = "stabilityai/stable-diffusion-2-1"
+        else:
+            dit_model = None
+            
+        if dit_model:
+            dit_config = DiTConfig(
+                num_inference_steps=args.dit_steps,
+                guidance_scale=args.dit_guidance_scale,
+                use_diffusers=args.use_diffusers,
+                diffusers_pipeline="auto" if args.use_diffusers else None,
+            )
+            
+            dit_stage_config = create_dit_stage_config(
+                stage_id=stage_id,
+                model_path=dit_model,
+                input_modalities=["text"],
+                output_modalities=["image"],
+                dit_config=dit_config
+            )
+            stage_configs.append(dit_stage_config)
+            stage_id += 1
+        
+        # If no specific stages are specified, use the main model
+        if not stage_configs:
+            # Try to detect if it's a multimodal model
+            if "omni" in args.model.lower() or "multimodal" in args.model.lower():
+                # Assume it's a multimodal model that can handle both AR and DiT
+                ar_config = create_ar_stage_config(
+                    stage_id=0,
+                    model_path=args.model,
+                    input_modalities=["text"],
+                    output_modalities=["text"]
+                )
+                stage_configs.append(ar_config)
+                
+                dit_config = DiTConfig(
+                    num_inference_steps=args.dit_steps,
+                    guidance_scale=args.dit_guidance_scale,
+                    use_diffusers=args.use_diffusers,
+                    diffusers_pipeline="auto" if args.use_diffusers else None,
+                )
+                
+                dit_stage_config = create_dit_stage_config(
+                    stage_id=1,
+                    model_path=args.model,
+                    input_modalities=["text"],
+                    output_modalities=["image"],
+                    dit_config=dit_config
+                )
+                stage_configs.append(dit_stage_config)
+            else:
+                # Default to AR stage
+                ar_config = create_ar_stage_config(
+                    stage_id=0,
+                    model_path=args.model,
+                    input_modalities=["text"],
+                    output_modalities=["text"]
+                )
+                stage_configs.append(ar_config)
+        
+        return stage_configs
+    
+    async def _start_server(self, omni_llm: AsyncOmniLLM, args) -> None:
+        """Start the API server."""
+        try:
+            # Import here to avoid circular imports
+            from vllm_omni.entrypoints.api_server import run_server
+            
+            await run_server(
+                omni_llm_instance=omni_llm,
+                host=args.host,
+                port=args.port
+            )
+        except KeyboardInterrupt:
+            print("\nShutting down server...")
+        except Exception as e:
+            print(f"Error starting server: {e}")
+            raise
@@ -1,693 +0,0 @@
-from __future__ import annotations
-
-import logging
-import os
-import time
-from typing import Any
-
-from vllm_omni.entrypoints.stage_utils import append_jsonl as _append_jsonl
-
-
-def remove_old_logs(log_file: str | None, num_stages: int) -> None:
-    try:
-        if not log_file:
-            return
-        try:
-            if os.path.exists(log_file):
-                os.remove(log_file)
-        except Exception:
-            pass
-        # Per-stage logs and stats
-        for sid in range(num_stages):
-            try:
-                p = f"{log_file}.stage{sid}.log"
-                if os.path.exists(p):
-                    os.remove(p)
-            except Exception:
-                pass
-            try:
-                p = f"{log_file}.stage{sid}.stats.jsonl"
-                if os.path.exists(p):
-                    os.remove(p)
-            except Exception:
-                pass
-        # Orchestrator stats files
-        try:
-            p = f"{log_file}.orchestrator.stats.jsonl"
-            if os.path.exists(p):
-                os.remove(p)
-        except Exception:
-            pass
-        try:
-            p = f"{log_file}.overall.stats.jsonl"
-            if os.path.exists(p):
-                os.remove(p)
-        except Exception:
-            pass
-    except Exception:
-        pass
-
-
-def configure_orchestrator_logger(logger: logging.Logger, log_file: str | None) -> None:
-    try:
-        if not log_file:
-            return
-        has_file_handler = any(isinstance(h, logging.FileHandler) for h in logger.handlers)
-        if not has_file_handler:
-            fh = logging.FileHandler(log_file)
-            fh.setLevel(logging.DEBUG)
-            fh.setFormatter(logging.Formatter("%(asctime)s [PID:%(process)d] %(levelname)s: %(message)s"))
-            logger.addHandler(fh)
-            logger.setLevel(logging.DEBUG)
-    except Exception:
-        pass
-
-
-def init_stats_paths(enable_stats: bool, log_file: str | None) -> tuple[str | None, str | None]:
-    stats_file: str | None = None
-    overall_file: str | None = None
-    try:
-        if enable_stats and log_file:
-            stats_file = f"{log_file}.orchestrator.stats.jsonl"
-            overall_file = f"{log_file}.overall.stats.jsonl"
-    except Exception:
-        stats_file = None
-        overall_file = None
-    return stats_file, overall_file
-
-
-def _safe_append_jsonl(path: str | None, record: dict[str, Any]) -> None:
-    if not path:
-        return
-    try:
-        _append_jsonl(path, record)  # type: ignore[arg-type]
-    except Exception:
-        pass
-
-
-def log_transfer_tx(
-    stats_file: str | None,
-    from_stage: int,
-    to_stage: int,
-    request_id: Any,
-    size_bytes: int,
-    tx_time_ms: float,
-    used_shm: bool,
-) -> None:
-    _safe_append_jsonl(
-        stats_file,
-        {
-            "type": "transfer_stats",
-            "from_stage": from_stage,
-            "to_stage": to_stage,
-            "request_id": request_id,
-            "size_bytes": int(size_bytes),
-            "tx_time_ms": float(tx_time_ms),
-            "tx_mbps": (float(size_bytes) * 8.0) / (max(tx_time_ms, 1e-6) * 1000.0),
-            "used_shm": bool(used_shm),
-        },
-    )
-
-
-def log_transfer_rx(
-    stats_file: str | None,
-    from_stage: int,
-    to_stage: int,
-    request_id: Any,
-    rx_bytes: int,
-    rx_decode_time_ms: float,
-    in_flight_time_ms: float,
-) -> None:
-    _safe_append_jsonl(
-        stats_file,
-        {
-            "type": "transfer_rx_stats",
-            "from_stage": from_stage,
-            "to_stage": to_stage,
-            "request_id": request_id,
-            "rx_bytes": int(rx_bytes),
-            "rx_decode_time_ms": float(rx_decode_time_ms),
-            "in_flight_time_ms": float(in_flight_time_ms),
-            "rx_time_per_kb_ms": (
-                (float(rx_decode_time_ms) / max(float(rx_bytes) / 1024.0, 1e-6)) if rx_bytes > 0 else 0.0
-            ),
-        },
-    )
-
-
-def log_transfer_total(
-    stats_file: str | None,
-    from_stage: int,
-    to_stage: int,
-    request_id: Any,
-    size_bytes: int,
-    tx_time_ms: float,
-    in_flight_time_ms: float,
-    rx_decode_time_ms: float,
-    total_time_ms: float,
-) -> None:
-    _safe_append_jsonl(
-        stats_file,
-        {
-            "type": "transfer_total_stats",
-            "from_stage": from_stage,
-            "to_stage": to_stage,
-            "request_id": request_id,
-            "size_bytes": int(size_bytes),
-            "tx_time_ms": float(tx_time_ms),
-            "in_flight_time_ms": float(in_flight_time_ms),
-            "rx_decode_time_ms": float(rx_decode_time_ms),
-            "total_time_ms": float(total_time_ms),
-            "total_time_per_kb_ms": (
-                float(total_time_ms) / max(float(size_bytes) / 1024.0, 1e-6) if size_bytes > 0 else 0.0
-            ),
-        },
-    )
-
-
-def log_orchestrator_e2e(
-    stats_file: str | None,
-    request_id: Any,
-    final_stage_id: int,
-    e2e_time_ms: float,
-    num_tokens_out: int,
-    e2e_time_per_token_ms: float,
-) -> None:
-    _safe_append_jsonl(
-        stats_file,
-        {
-            "type": "orchestrator_request_e2e",
-            "request_id": request_id,
-            "final_stage_id": final_stage_id,
-            "e2e_time_ms": float(e2e_time_ms),
-            "num_tokens_out": int(num_tokens_out),
-            "e2e_time_per_token_ms": float(e2e_time_per_token_ms),
-        },
-    )
-
-
-def log_orchestrator_summary(stats_file: str | None, summary: dict[str, Any]) -> None:
-    _safe_append_jsonl(stats_file, {"type": "orchestrator_summary", **summary})
-
-
-def log_overall_summary(overall_stats_file: str | None, summary: dict[str, Any]) -> None:
-    _safe_append_jsonl(overall_stats_file, {"type": "overall_summary", **summary})
-
-
-def log_overall_record(overall_stats_file: str | None, record: dict[str, Any]) -> None:
-    _safe_append_jsonl(overall_stats_file, record)
-
-
-def log_stage_request_stats(
-    stats_file: str | None,
-    stage_id: int,
-    request_id: Any,
-    batch_size: int,
-    num_tokens_out: int,
-    stage_gen_time_ms: float,
-    tokens_per_s: float,
-    rx_transfer_bytes: int,
-    rx_decode_time_ms: float,
-    rx_mbps: float,
-) -> None:
-    _safe_append_jsonl(
-        stats_file,
-        {
-            "type": "stage_request_stats",
-            "stage_id": stage_id,
-            "request_id": request_id,
-            "batch_size": int(batch_size),
-            "num_tokens_out": int(num_tokens_out),
-            "stage_gen_time_ms": float(stage_gen_time_ms),
-            "tokens_per_s": float(tokens_per_s),
-            "rx_transfer_bytes": int(rx_transfer_bytes),
-            "rx_decode_time_ms": float(rx_decode_time_ms),
-            "rx_mbps": float(rx_mbps),
-        },
-    )
-
-
-def log_stage_running_avg(
-    stats_file: str | None,
-    stage_id: int,
-    total_tokens: int,
-    total_gen_time_ms: float,
-    avg_tokens_per_s: float,
-) -> None:
-    _safe_append_jsonl(
-        stats_file,
-        {
-            "type": "stage_running_avg",
-            "stage_id": stage_id,
-            "total_tokens": int(total_tokens),
-            "total_gen_time_ms": float(total_gen_time_ms),
-            "avg_tokens_per_s": float(avg_tokens_per_s),
-        },
-    )
-
-
-def log_stage_batch_stats(
-    stats_file: str | None,
-    stage_id: int,
-    batch_size: int,
-    batch_gen_time_ms: float,
-    request_ids: list[Any],
-) -> None:
-    _safe_append_jsonl(
-        stats_file,
-        {
-            "type": "stage_batch_stats",
-            "stage_id": stage_id,
-            "batch_size": int(batch_size),
-            "batch_gen_time_ms": float(batch_gen_time_ms),
-            "request_ids": list(request_ids),
-        },
-    )
-
-
-def compute_and_log_stage_request_stats(
-    stats_file: str | None,
-    stage_id: int,
-    request_id: Any,
-    batch_size: int,
-    engine_outputs: list[Any],
-    stage_gen_time_ms: float,
-    rx_transfer_bytes: int,
-    rx_decode_time_ms: float,
-) -> None:
-    """Compute per-request metrics and log them in one call."""
-    num_tokens = count_tokens_from_outputs(engine_outputs)
-    tokens_per_s = (num_tokens * 1000.0 / stage_gen_time_ms) if stage_gen_time_ms > 0 else 0.0
-    rx_mbps = (
-        (float(rx_transfer_bytes) * 8.0) / (max(float(rx_decode_time_ms), 1e-6) * 1000.0)
-        if rx_transfer_bytes > 0
-        else 0.0
-    )
-    log_stage_request_stats(
-        stats_file,
-        stage_id,
-        request_id,
-        int(batch_size),
-        int(num_tokens),
-        float(stage_gen_time_ms),
-        float(tokens_per_s),
-        int(rx_transfer_bytes),
-        float(rx_decode_time_ms),
-        float(rx_mbps),
-    )
-
-
-# ----------------- Aggregation helpers for orchestrator -----------------
-
-
-def record_stage_metrics(
-    per_request: dict[int, dict[str, Any]],
-    stage_req_counts: list[int],
-    stage_total_time_ms: list[float],
-    stage_total_tokens: list[int],
-    stage_id: int,
-    req_id: Any,
-    metrics: dict[str, Any],
-) -> None:
-    try:
-        stage_req_counts[stage_id] += 1
-        stage_total_tokens[stage_id] += int(metrics.get("num_tokens_out", 0))
-        rid_int = int(req_id)
-        pr = per_request.setdefault(rid_int, {"stages": {}, "transfers_ms": 0.0, "transfers_bytes": 0})
-        pr_stages = pr["stages"]  # type: ignore[index]
-        pr_stages[stage_id] = {
-            "stage_gen_time_ms": float(metrics.get("stage_gen_time_ms", 0.0)),
-            "num_tokens_out": int(metrics.get("num_tokens_out", 0)),
-        }
-    except Exception:
-        pass
-
-
-def aggregate_rx_and_maybe_total(
-    transfer_edge_req: dict[tuple[int, int, int], dict[str, float]],
-    transfer_agg: dict[tuple[int, int], dict[str, float]],
-    per_request: dict[int, dict[str, Any]],
-    stage_id: int,
-    req_id: Any,
-    rx_bytes: float,
-    rx_ms: float,
-    in_flight_ms: float,
-) -> tuple[int, float, float] | None:
-    try:
-        # Update RX aggregates for (stage_id-1 -> stage_id)
-        if stage_id > 0:
-            key = (stage_id - 1, stage_id)
-            agg = transfer_agg.get(key)
-            if agg is None:
-                agg = {
-                    "sum_bytes": 0.0,
-                    "sum_ms": 0.0,
-                    "count": 0.0,
-                    "sum_rx_bytes": 0.0,
-                    "sum_rx_ms": 0.0,
-                    "rx_count": 0.0,
-                    "sum_total_ms": 0.0,
-                    "total_count": 0.0,
-                }
-                transfer_agg[key] = agg
-            agg["sum_rx_bytes"] += float(rx_bytes)
-            agg["sum_rx_ms"] += float(rx_ms)
-            agg["rx_count"] += 1.0
-
-            # Try combine with sender-side timing if present
-            try:
-                rid_int = int(req_id)
-            except Exception:
-                rid_int = req_id  # best effort
-            s = transfer_edge_req.get((stage_id - 1, stage_id, rid_int))
-            if s is None:
-                return None
-            tx_ms = float(s.get("tx_ms", 0.0))
-            size_b = float(s.get("size_bytes", rx_bytes))
-            total_ms = tx_ms + float(in_flight_ms) + float(rx_ms)
-            agg["sum_total_ms"] += total_ms
-            agg["total_count"] += 1.0
-            # accumulate per-request transfer totals
-            try:
-                pr = per_request.setdefault(rid_int, {"stages": {}, "transfers_ms": 0.0, "transfers_bytes": 0})
-                pr["transfers_ms"] = float(pr.get("transfers_ms", 0.0)) + total_ms  # type: ignore[index]
-                pr["transfers_bytes"] = int(pr.get("transfers_bytes", 0)) + int(rx_bytes)  # type: ignore[index]
-            except Exception:
-                pass
-            return int(size_b), float(tx_ms), float(total_ms)
-        return None
-    except Exception:
-        return None
-
-
-def record_sender_transfer_agg(
-    transfer_agg: dict[tuple[int, int], dict[str, float]],
-    transfer_edge_req: dict[tuple[int, int, int], dict[str, float]],
-    from_stage: int,
-    to_stage: int,
-    req_id: Any,
-    size_bytes: int,
-    tx_ms: float,
-) -> None:
-    try:
-        key = (from_stage, to_stage)
-        agg = transfer_agg.get(key)
-        if agg is None:
-            agg = {
-                "sum_bytes": 0.0,
-                "sum_ms": 0.0,
-                "count": 0.0,
-                "sum_rx_bytes": 0.0,
-                "sum_rx_ms": 0.0,
-                "rx_count": 0.0,
-                "sum_total_ms": 0.0,
-                "total_count": 0.0,
-            }
-            transfer_agg[key] = agg
-        agg["sum_bytes"] += float(size_bytes)
-        agg["sum_ms"] += float(tx_ms)
-        agg["count"] += 1.0
-        # Store sender-side timing for per-request combination
-        try:
-            rid_int = int(req_id)
-        except Exception:
-            rid_int = req_id
-        transfer_edge_req[(from_stage, to_stage, rid_int)] = {
-            "tx_ms": float(tx_ms),
-            "size_bytes": float(size_bytes),
-        }
-    except Exception:
-        pass
-
-
-def count_tokens_from_outputs(engine_outputs: list[Any]) -> int:
-    total = 0
-    for _ro in engine_outputs:
-        try:
-            outs = getattr(_ro, "outputs", None)
-            if outs and len(outs) > 0:
-                tokens = getattr(outs[0], "token_ids", None)
-                if tokens is not None:
-                    total += len(tokens)
-        except Exception:
-            pass
-    return total
-
-
-def build_stage_summary(
-    stage_req_counts: list[int],
-    stage_total_tokens: list[int],
-    stage_total_time_ms: list[float],
-) -> list[dict[str, Any]]:
-    summary: list[dict[str, Any]] = []
-    for sid in range(len(stage_req_counts)):
-        reqs = stage_req_counts[sid]
-        tokens = stage_total_tokens[sid]
-        total_ms = float(stage_total_time_ms[sid])
-        avg_req = (total_ms / reqs) if reqs > 0 else 0.0
-        avg_tok = (tokens * 1000.0 / total_ms) if total_ms > 0 else 0.0
-        summary.append(
-            {
-                "stage_id": sid,
-                "requests": int(reqs),
-                "tokens": int(tokens),
-                "total_time_ms": total_ms,
-                "avg_time_per_request_ms": avg_req,
-                "avg_tokens_per_s": avg_tok,
-            }
-        )
-    return summary
-
-
-def build_transfer_summary(
-    transfer_agg: dict[tuple[int, int], dict[str, float]],
-) -> list[dict[str, Any]]:
-    summary: list[dict[str, Any]] = []
-    for (src, dst), agg in transfer_agg.items():
-        sum_bytes = float(agg.get("sum_bytes", 0.0))
-        sum_ms = float(agg.get("sum_ms", 0.0))
-        samples = int(agg.get("count", 0.0))
-        tx_mbps = (sum_bytes * 8.0) / (max(sum_ms, 1e-6) * 1000.0) if sum_bytes > 0 else 0.0
-        sum_rx_bytes = float(agg.get("sum_rx_bytes", 0.0))
-        sum_rx_ms = float(agg.get("sum_rx_ms", 0.0))
-        samples_rx = int(agg.get("rx_count", 0.0))
-        rx_mbps = (sum_rx_bytes * 8.0) / (max(sum_rx_ms, 1e-6) * 1000.0) if sum_rx_bytes > 0 else 0.0
-        sum_total_ms = float(agg.get("sum_total_ms", 0.0))
-        samples_total = int(agg.get("total_count", 0.0))
-        total_mbps = (sum_bytes * 8.0) / (max(sum_total_ms, 1e-6) * 1000.0) if sum_bytes > 0 else 0.0
-        summary.append(
-            {
-                "from_stage": src,
-                "to_stage": dst,
-                "samples": samples,
-                "total_bytes": int(sum_bytes),
-                "total_time_ms": sum_ms,
-                "tx_mbps": tx_mbps,
-                "rx_samples": samples_rx,
-                "rx_total_bytes": int(sum_rx_bytes),
-                "rx_total_time_ms": sum_rx_ms,
-                "rx_mbps": rx_mbps,
-                "total_samples": samples_total,
-                "total_transfer_time_ms": sum_total_ms,
-                "total_mbps": total_mbps,
-            }
-        )
-    return summary
-
-
-class OrchestratorMetrics:
-    def __init__(
-        self,
-        num_stages: int,
-        enable_stats: bool,
-        stats_file: str | None,
-        overall_stats_file: str | None,
-        wall_start_ts: float,
-    ) -> None:
-        self.num_stages = int(num_stages)
-        self.enable_stats = bool(enable_stats)
-        self.stats_file = stats_file
-        self.overall_stats_file = overall_stats_file
-        self.stage_total_time_ms: list[float] = [0.0 for _ in range(self.num_stages)]
-        self.stage_total_tokens: list[int] = [0 for _ in range(self.num_stages)]
-        self.stage_req_counts: list[int] = [0 for _ in range(self.num_stages)]
-        self.transfer_agg: dict[tuple[int, int], dict[str, float]] = {}
-        self.transfer_edge_req: dict[tuple[int, int, int], dict[str, float]] = {}
-        self.e2e_total_ms: float = 0.0
-        self.e2e_total_tokens: int = 0
-        self.e2e_count: int = 0
-        self.e2e_done: set[int] = set()
-        self.per_request: dict[int, dict[str, Any]] = {}
-        self.sum_per_request_transfer_ms: float = 0.0
-        self.wall_start_ts: float = float(wall_start_ts)
-        self.last_finish_ts: float = float(wall_start_ts)
-        self.stage_seen_batches: dict[int, set] = {sid: set() for sid in range(self.num_stages)}
-        self.stage_first_ts: list[float | None] = [None for _ in range(self.num_stages)]
-        self.stage_last_ts: list[float | None] = [None for _ in range(self.num_stages)]
-
-    def on_stage_metrics(self, stage_id: int, req_id: Any, metrics: dict[str, Any]) -> None:
-        record_stage_metrics(
-            self.per_request,
-            self.stage_req_counts,
-            self.stage_total_time_ms,
-            self.stage_total_tokens,
-            stage_id,
-            req_id,
-            metrics,
-        )
-        try:
-            batch_id_raw = metrics.get("batch_id", None)
-            if batch_id_raw is not None:
-                batch_id = int(batch_id_raw)
-                if batch_id not in self.stage_seen_batches[stage_id]:
-                    self.stage_total_time_ms[stage_id] += float(metrics.get("stage_gen_time_ms", 0.0))
-                    self.stage_seen_batches[stage_id].add(batch_id)
-        except Exception:
-            pass
-        rx_b = float(metrics.get("rx_transfer_bytes", 0.0))
-        rx_ms = float(metrics.get("rx_decode_time_ms", 0.0))
-        in_flight_ms = float(metrics.get("rx_in_flight_time_ms", 0.0))
-        combined = aggregate_rx_and_maybe_total(
-            self.transfer_edge_req,
-            self.transfer_agg,
-            self.per_request,
-            stage_id,
-            req_id,
-            rx_b,
-            rx_ms,
-            in_flight_ms,
-        )
-        if self.enable_stats and self.stats_file and stage_id > 0:
-            log_transfer_rx(
-                self.stats_file,
-                stage_id - 1,
-                stage_id,
-                req_id,
-                int(rx_b),
-                float(rx_ms),
-                float(in_flight_ms),
-            )
-            if combined is not None:
-                size_b_c, tx_ms_c, total_ms_c = combined
-                log_transfer_total(
-                    self.stats_file,
-                    stage_id - 1,
-                    stage_id,
-                    req_id,
-                    int(size_b_c),
-                    float(tx_ms_c),
-                    float(in_flight_ms),
-                    float(rx_ms),
-                    float(total_ms_c),
-                )
-
-    def on_forward(
-        self,
-        from_stage: int,
-        to_stage: int,
-        req_id: Any,
-        size_bytes: int,
-        tx_ms: float,
-        used_shm: bool,
-    ) -> None:
-        # Mark first input time for the destination stage if not set
-        if self.stage_first_ts[to_stage] is None:
-            self.stage_first_ts[to_stage] = time.time()
-        if self.enable_stats and self.stats_file:
-            log_transfer_tx(
-                self.stats_file,
-                from_stage,
-                to_stage,
-                req_id,
-                int(size_bytes),
-                float(tx_ms),
-                bool(used_shm),
-            )
-        record_sender_transfer_agg(
-            self.transfer_agg,
-            self.transfer_edge_req,
-            from_stage,
-            to_stage,
-            req_id,
-            int(size_bytes),
-            float(tx_ms),
-        )
-
-    def on_finalize_request(self, stage_id: int, req_id: Any, engine_outputs: list[Any], req_start_ts: float) -> None:
-        try:
-            rid_int = int(req_id)
-        except Exception:
-            rid_int = req_id
-        _t0 = float(req_start_ts)
-        _t1 = time.time()
-        # Update last output time for this stage
-        prev_last = self.stage_last_ts[stage_id]
-        self.stage_last_ts[stage_id] = _t1 if prev_last is None else max(prev_last, _t1)
-        self.last_finish_ts = max(self.last_finish_ts, _t1)
-        e2e_ms = (_t1 - _t0) * 1000.0
-        num_tokens = count_tokens_from_outputs(engine_outputs)
-        self.e2e_total_ms += e2e_ms
-        self.e2e_total_tokens += int(num_tokens)
-        self.e2e_count += 1
-        self.e2e_done.add(rid_int)
-        pr = self.per_request.setdefault(rid_int, {"stages": {}, "transfers_ms": 0.0, "transfers_bytes": 0})
-        per_req_record = {
-            "type": "overall_request",
-            "request_id": rid_int,
-            "e2e_time_ms": e2e_ms,
-            "num_tokens_out": int(num_tokens),
-            "transfers_total_time_ms": float(pr.get("transfers_ms", 0.0)),
-            "transfers_total_bytes": int(pr.get("transfers_bytes", 0)),
-            "stages": pr.get("stages", {}),
-        }
-        self.sum_per_request_transfer_ms += float(pr.get("transfers_ms", 0.0))
-        if self.enable_stats and self.overall_stats_file:
-            log_overall_record(self.overall_stats_file, per_req_record)
-        if self.enable_stats and self.stats_file:
-            e2e_tpt = (e2e_ms / num_tokens) if num_tokens > 0 else 0.0
-            log_orchestrator_e2e(self.stats_file, req_id, stage_id, e2e_ms, int(num_tokens), e2e_tpt)
-
-    def build_and_log_summary(self, final_stage_id_for_e2e: int) -> dict[str, Any]:
-        # Compute stage summary using wall time between first input and last output per stage
-        stage_summary: list[dict[str, Any]] = []
-        for sid in range(self.num_stages):
-            first_ts = self.stage_first_ts[sid]
-            last_ts = self.stage_last_ts[sid]
-            total_ms = (
-                (max(0.0, (last_ts - first_ts)) * 1000.0) if (first_ts is not None and last_ts is not None) else 0.0
-            )
-            reqs = self.stage_req_counts[sid]
-            tokens = self.stage_total_tokens[sid]
-            avg_req = (total_ms / reqs) if reqs > 0 else 0.0
-            avg_tok = (tokens * 1000.0 / total_ms) if total_ms > 0 else 0.0
-            stage_summary.append(
-                {
-                    "stage_id": sid,
-                    "requests": int(reqs),
-                    "tokens": int(tokens),
-                    "total_time_ms": float(total_ms),
-                    "avg_time_per_request_ms": float(avg_req),
-                    "avg_tokens_per_s": float(avg_tok),
-                }
-            )
-        transfer_summary = build_transfer_summary(self.transfer_agg)
-        e2e_avg_req = (self.e2e_total_ms / self.e2e_count) if self.e2e_count > 0 else 0.0
-        e2e_avg_tok = (self.e2e_total_tokens * 1000.0 / self.e2e_total_ms) if self.e2e_total_ms > 0 else 0.0
-        wall_time_ms = max(0.0, (self.last_finish_ts - self.wall_start_ts) * 1000.0)
-        summary: dict[str, Any] = {
-            "e2e_requests": int(self.e2e_count),
-            "e2e_total_time_ms": float(wall_time_ms),
-            "e2e_sum_time_ms": float(self.e2e_total_ms),
-            "e2e_total_tokens": int(self.e2e_total_tokens),
-            "e2e_avg_time_per_request_ms": e2e_avg_req,
-            "e2e_avg_tokens_per_s": e2e_avg_tok,
-            "wall_time_ms": wall_time_ms,
-            "final_stage_id": final_stage_id_for_e2e,
-            "stages": stage_summary,
-            "transfers": transfer_summary,
-        }
-        if self.enable_stats and self.stats_file:
-            log_orchestrator_summary(self.stats_file, summary)
-        if self.enable_stats and self.overall_stats_file:
-            log_overall_summary(self.overall_stats_file, summary)
-        return summary
@@ -1,478 +1,88 @@
-import multiprocessing as mp
-import os
-import sys
-import time
-from collections.abc import Sequence
-from concurrent.futures import ThreadPoolExecutor, as_completed
-from typing import Any, Optional, Union
-
+from typing import Union, Sequence, Optional, Any
 import cloudpickle
 from pydantic import ValidationError

-# External library imports (vLLM)
-from vllm.config import CompilationConfig, StructuredOutputsConfig, is_init_field
-from vllm.engine.arg_utils import HfOverrides
-from vllm.entrypoints.llm import LLM
 from vllm.inputs import PromptType
-from vllm.logger import init_logger
-from vllm.plugins.io_processors import get_io_processor
 from vllm.sampling_params import SamplingParams
-from vllm.usage.usage_lib import UsageContext
-from vllm.utils import Counter
-from vllm.v1.engine.llm_engine import LLMEngine
+from vllm.entrypoints.llm import LLM

-# Internal imports (our code)
+from vllm.v1.engine.llm_engine import LLMEngine
+from vllm.engine.arg_utils import HfOverrides
+from vllm.usage.usage_lib import UsageContext
+from vllm.config import CompilationConfig, is_init_field
+from vllm.utils import Counter
+from vllm.logger import init_logger
+import vllm.envs as envs
+
+from vllm_omni.entrypoints.utils import load_stage_configs_from_model
+from vllm_omni.entrypoints.omni_stage import OmniStage
 from vllm_omni.engine.arg_utils import OmniEngineArgs
 from vllm_omni.engine.output_processor import MultimodalOutputProcessor
 from vllm_omni.engine.processor import OmniProcessor
-from vllm_omni.entrypoints.log_utils import (
-    OrchestratorMetrics,
-    configure_orchestrator_logger,
-    init_stats_paths,
-    remove_old_logs,
-)
-from vllm_omni.entrypoints.omni_stage import OmniStage
-from vllm_omni.entrypoints.stage_utils import encode_for_ipc as _encode
-from vllm_omni.entrypoints.stage_utils import maybe_load_from_ipc as _load
-from vllm_omni.entrypoints.stage_utils import serialize_obj as _set
-from vllm_omni.entrypoints.utils import load_stage_configs_from_model, load_stage_configs_from_yaml
 from vllm_omni.outputs import OmniRequestOutput

+
 logger = init_logger(__name__)


 class OmniLLM:
-    """Multi-process pipelined OmniLLM.
-
-    - Per-stage process with copy-based IPC (Queues)
-    - window=-1 across stages (downstream starts when upstream finishes)
-    - max_inflight=1 per stage (serial within a stage), but pipeline across stages
-    """
-
-    def __init__(
-        self,
-        model: str,
-        stage_configs_path: Optional[str] = None,
-        log_stats: bool = False,
-        log_file: Optional[str] = None,
-        init_sleep_seconds: int = 20,
-        shm_threshold_bytes: int = 65536,
-        batch_timeout: int = 10,
-        init_timeout: int = 300,
-        **kwargs,
-    ):
-        self.batch_timeout = batch_timeout
-        self._enable_stats: bool = bool(log_stats)
-        # Do NOT call super().__init__ to avoid creating OmniStageLLM instances in parent.
-        if stage_configs_path is None:
-            self.stage_configs = load_stage_configs_from_model(model)
+    def __init__(self, model: str, stage_configs = None, log_stats: bool = False, **kwargs):
+        if stage_configs is None:
+            self.initialize_stage_configs(model)
        else:
-            self.stage_configs = load_stage_configs_from_yaml(stage_configs_path)
-
-        # Optional file handler for orchestrator
-        self._log_file = log_file
-        if self._log_file:
-            remove_old_logs(self._log_file, len(self.stage_list))
-            configure_orchestrator_logger(logger, self._log_file)
-
-        self._stats_file, self._overall_stats_file = init_stats_paths(self._enable_stats, self._log_file)
-        self._initialize_stages(model, init_sleep_seconds, shm_threshold_bytes, init_timeout)
-
-    def _initialize_stages(
-        self,
-        model: str,
-        init_sleep_seconds: int,
-        shm_threshold_bytes: int,
-        init_timeout: int,
-    ) -> None:
-        self.stage_list: list[OmniStage] = []
-
-        # Build OmniStage instances in parallel, preserve original order
-        def _build_stage(idx_cfg: tuple[int, Any]) -> tuple[int, OmniStage]:
-            idx, cfg = idx_cfg
-            return idx, OmniStage(cfg)
-
-        with ThreadPoolExecutor(max_workers=min(len(self.stage_configs), max(1, os.cpu_count() or 1))) as executor:
-            futures = [executor.submit(_build_stage, (idx, cfg)) for idx, cfg in enumerate(self.stage_configs)]
-            results: list[tuple[int, OmniStage]] = []
-            for fut in as_completed(futures):
-                results.append(fut.result())
-        results.sort(key=lambda x: x[0])
-        self.stage_list = [st for _, st in results]
-        logger.debug("[Orchestrator] Loaded %d stages", len(self.stage_list))
-
-        self._ctx = mp.get_context("spawn")
-        self._stage_in_queues: list[mp.Queue] = []
-        self._stage_out_queues: list[mp.Queue] = []
-        self._init_sleep_seconds = max(0, int(init_sleep_seconds))
-        self._shm_threshold_bytes = max(0, int(shm_threshold_bytes))
-        self._start_stage_processes(model)
-        # Wait for all stages to report readiness before seeding
-        self._stages_ready: set[int] = set()
-        self._wait_for_stages_ready(timeout=init_timeout)
-
-    def _start_stage_processes(self, model: str) -> None:
-        for stage_id, stage in enumerate(self.stage_list):
-            # Use unbounded queues to avoid deadlock when seeding many requests
-            in_q: mp.Queue = self._ctx.Queue(maxsize=0)
-            out_q: mp.Queue = self._ctx.Queue(maxsize=0)
-            self._stage_in_queues.append(in_q)
-            self._stage_out_queues.append(out_q)
-
-            # Attach queues and start Stage-owned worker process
-            stage.attach_queues(in_q, out_q)
-            stage.init_stage_worker(
-                model,
-                log_file=self._log_file,
-                shm_threshold_bytes=self._shm_threshold_bytes,
-                ctx=self._ctx,
-                batch_timeout=self.batch_timeout,
-            )
-            logger.debug("[Orchestrator] Stage-%s process started", stage_id)
-            time.sleep(self._init_sleep_seconds)
-
-    def close(self) -> None:
-        for q in self._stage_in_queues:
-            try:
-                q.put_nowait(None)
-            except Exception as e:
-                logger.warning(
-                    "[Orchestrator] Failed to send shutdown signal to stage input queue: %s",
-                    e,
-                )
-        for stage in self.stage_list:
-            try:
-                stage.stop_stage_worker()
-            except Exception as e:
-                logger.warning("[Orchestrator] Failed to stop stage worker: %s", e)
-
-    def __del__(self) -> None:  # best-effort
-        try:
-            self.close()
-        except Exception as e:
-            logger.debug("[Orchestrator] __del__ close() raised: %s", e, exc_info=True)
-
+            self.stage_configs = stage_configs
+        
+        self.stage_list = []
+        self.initialize_stages(model)
+        
+    def initialize_stage_configs(self, model: str):
+        self.stage_configs = load_stage_configs_from_model(model)
+    
+    def initialize_stages(self, model: str):
+        for stage_config in self.stage_configs:
+            stage = OmniStage(stage_config)
+            stage_llm = OmniStageLLM(model=model, **stage_config.engine_args)
+            stage.set_engine(stage_llm)
+            self.stage_list.append(stage)
+    
    def generate(
        self,
        prompts: Union[PromptType, Sequence[PromptType]],
-        sampling_params_list: Optional[Union[SamplingParams, Sequence[SamplingParams]]] = None,
+        sampling_params_list: Optional[Union[SamplingParams,
+                                        Sequence[SamplingParams]]] = None,
    ) -> list[OmniRequestOutput]:
-        try:
-            return self._run_generation(prompts, sampling_params_list)
-        except Exception as e:
-            logger.exception("[Orchestrator] Failed to run generation: %s", e)
-            raise e
-        finally:
-            self.close()
-
-    def _run_generation(
-        self,
-        prompts: Union[PromptType, Sequence[PromptType]],
-        sampling_params_list: Optional[Union[SamplingParams, Sequence[SamplingParams]]] = None,
-    ) -> list[OmniRequestOutput]:
-        logger.debug("[Orchestrator] generate() called")
-        if sampling_params_list is None:
-            raise ValueError("sampling_params_list is required for pipelined generation")
+        """Generate text outputs for the given prompts."""
+        final_outputs: list[OmniRequestOutput] = []
        if len(sampling_params_list) != len(self.stage_list):
            raise ValueError(f"Expected {len(self.stage_list)} sampling params, got {len(sampling_params_list)}")
-
-        # Normalize prompts to a list for per-request iteration
-        if not isinstance(prompts, (list, tuple)):
-            request_prompts: list[PromptType] = [prompts]
-        else:
-            request_prompts = list(prompts)
-
-        final_outputs: list[OmniRequestOutput] = []
-
-        # Orchestrator keeps stage objects for input derivation
-        num_stages = len(self.stage_list)
-
-        # Map from request_id to original prompt
-        request_id_to_prompt: dict[int, PromptType] = {i: p for i, p in enumerate(request_prompts)}
-
-        # Track per-request start time for end-to-end timing
-        _req_start_ts: dict[int, float] = {}
-        _wall_start_ts: float = time.time()
-
-        # Determine the final stage for E2E stats (highest stage_id with final_output=True; fallback to last stage)
-        final_stage_id_for_e2e = -1
-        try:
-            for _sid, _st in enumerate(self.stage_list):
-                if getattr(_st, "final_output", False):
-                    final_stage_id_for_e2e = max(final_stage_id_for_e2e, _sid)
-            if final_stage_id_for_e2e < 0:
-                final_stage_id_for_e2e = len(self.stage_list) - 1
-        except Exception as e:
-            logger.debug(
-                "[Orchestrator] Failed to determine final stage for E2E; falling back to last: %s",
-                e,
-                exc_info=True,
-            )
-            final_stage_id_for_e2e = len(self.stage_list) - 1
-        # Metrics/aggregation helper
-        metrics = OrchestratorMetrics(
-            num_stages,
-            self._enable_stats,
-            self._stats_file,
-            self._overall_stats_file,
-            _wall_start_ts,
-        )
-
-        # Seed stage-0 queue with all requests
-        logger.debug("[Orchestrator] Seeding %d requests into stage-0", len(request_prompts))
-        # Mark first input time for stage-0
-        metrics.stage_first_ts[0] = metrics.stage_first_ts[0] or time.time()
-
-        for req_id, prompt in request_id_to_prompt.items():
-            sp0: SamplingParams = sampling_params_list[0]  # type: ignore[index]
-            task = {
-                "request_id": req_id,
-                "engine_inputs": prompt,
-                "sampling_params": sp0,
-            }
-            self.stage_list[0].submit(task)
-            _req_start_ts[req_id] = time.time()
-            logger.debug("[Orchestrator] Enqueued request %s to stage-0", req_id)
-
-        # For each stage, forward results to next stage; collect finals at the end
-        # We pipeline by continually polling output queues in stage order
-        remaining_by_stage: list[int] = [len(request_prompts)] + [0] * (num_stages - 1)
-        completed_requests = 0
-        total_requests = len(request_prompts)
-
-        logger.debug(
-            "[Orchestrator] Entering scheduling loop: total_requests=%d, stages=%d",
-            total_requests,
-            num_stages,
-        )
-        while completed_requests < total_requests:
-            made_progress = False
-            for stage_id, stage in enumerate(self.stage_list):
-                result = stage.try_collect()
-                if result is None:
-                    continue
-
-                made_progress = True
-                req_id = result.get("request_id")
-                if "error" in result:
-                    logger.error(
-                        "Stage %s error on request %s: %s",
-                        stage_id,
-                        req_id,
-                        result["error"],
-                    )
-                    continue
-
-                if result.get("type") == "stage_ready":
-                    # Only happens when stage is initialized slower than expected,
-                    # so we wait for a short time and try again
-                    time.sleep(0.05)
-                    continue
-
-                engine_outputs = _load(result, obj_key="engine_outputs", shm_key="engine_outputs_shm")
-                # Mark last output time for this stage whenever we receive outputs
-                metrics.stage_last_ts[stage_id] = max(metrics.stage_last_ts[stage_id] or 0.0, time.time())
-                try:
-                    _m = result.get("metrics")
-                    if _m is not None:
-                        metrics.on_stage_metrics(stage_id, req_id, _m)
-                except Exception as e:
-                    logger.exception(
-                        "[Orchestrator] Failed to process metrics for stage %s, req %s: %s",
-                        stage_id,
-                        req_id,
-                        e,
-                    )
-                logger.debug(
-                    "[Orchestrator] Stage-%s completed request %s; forwarding or finalizing",
-                    stage_id,
-                    req_id,
-                )
-                stage.set_engine_outputs(engine_outputs)
-
-                if getattr(stage, "final_output", False):
-                    final_outputs.append(
-                        OmniRequestOutput(
-                            stage_id=stage_id,
-                            final_output_type=stage.final_output_type,  # type: ignore[attr-defined]
-                            request_output=engine_outputs,
-                        )
-                    )
-                    logger.debug(
-                        "[Orchestrator] Request %s finalized at stage-%s",
-                        req_id,
-                        stage_id,
-                    )
-
-                    # End-to-end timing and time-per-token for final output
-                    # (only once per request at the designated final stage)
-                    try:
-                        rid_int = int(req_id) if isinstance(req_id, (int, str)) and str(req_id).isdigit() else req_id
-                        if stage_id == final_stage_id_for_e2e and rid_int not in metrics.e2e_done:
-                            metrics.on_finalize_request(
-                                stage_id,
-                                req_id,
-                                engine_outputs,
-                                _req_start_ts.get(req_id, _wall_start_ts),
-                            )
-                    except Exception as e:
-                        logger.exception(
-                            "[Orchestrator] Finalize request handling error for req %s at stage %s: %s",
-                            req_id,
-                            stage_id,
-                            e,
-                        )
-
-                next_stage_id = stage_id + 1
-                if next_stage_id < num_stages:
-                    next_stage: OmniStage = self.stage_list[next_stage_id]
-                    next_inputs = next_stage.process_engine_inputs(self.stage_list, [request_id_to_prompt[req_id]])
-                    sp_next: SamplingParams = sampling_params_list[next_stage_id]  # type: ignore[index]
-                    try:
-                        # Measure transfer size and time (encode + enqueue)
-                        size_bytes = 0
-                        try:
-                            size_bytes = len(_set(next_inputs))
-                        except Exception:
-                            size_bytes = 0
-                        t0 = time.time()
-                        ipc_payload = _encode(
-                            next_inputs,
-                            getattr(self, "_shm_threshold_bytes", 65536),
-                            obj_key="engine_inputs",
-                            shm_key="engine_inputs_shm",
-                        )
-                        ipc_payload.update(
-                            {
-                                "request_id": req_id,
-                                "sampling_params": sp_next,
-                                "sent_ts": time.time(),
-                            }
-                        )
-                        self.stage_list[next_stage_id].submit(ipc_payload)
-                        t1 = time.time()
-                        tx_ms = (t1 - t0) * 1000.0
-                        metrics.on_forward(
-                            stage_id,
-                            next_stage_id,
-                            req_id,
-                            int(size_bytes),
-                            float(tx_ms),
-                            bool("engine_inputs_shm" in ipc_payload),
-                        )
-                    except Exception as e:
-                        logger.warning(
-                            "[Orchestrator] IPC encode failed for req %s: %s; falling back to inline payload",
-                            req_id,
-                            e,
-                        )
-                        self.stage_list[next_stage_id].submit(
-                            {
-                                "request_id": req_id,
-                                "engine_inputs": next_inputs,
-                                "sampling_params": sp_next,
-                            }
-                        )
-                    logger.debug(
-                        "[Orchestrator] Forwarded request %s to stage-%s",
-                        req_id,
-                        next_stage_id,
-                    )
-                    remaining_by_stage[next_stage_id] += 1
-                else:
-                    completed_requests += 1
-                    logger.debug(
-                        "[Orchestrator] Request %s fully completed (%d/%d)",
-                        req_id,
-                        completed_requests,
-                        total_requests,
-                    )
-
-            if not made_progress:
-                time.sleep(0.005)
-        logger.debug("[Orchestrator] All requests completed")
-
-        # Summarize and print stats
-        try:
-            summary = metrics.build_and_log_summary(final_stage_id_for_e2e)
-            logger.info("[Summary] %s", summary)
-        except Exception as e:
-            logger.exception("[Orchestrator] Failed to build/log summary: %s", e)
-
+        for stage_id, stage in enumerate(self.stage_list):
+            if stage_id > 0:
+                engine_inputs = stage.process_engine_inputs(self.stage_list, prompts)
+            else:
+                engine_inputs = prompts
+            engine_outputs = self._run_generation(stage, sampling_params_list[stage_id], engine_inputs)
+            stage.set_engine_outputs(engine_outputs)
+            if hasattr(stage, 'final_output') and stage.final_output:
+                final_outputs.append(OmniRequestOutput(
+                    stage_id=stage_id, 
+                    final_output_type=stage.final_output_type, 
+                    request_output=engine_outputs))
        return final_outputs
-
-    def _wait_for_stages_ready(self, timeout: int = 120) -> None:
-        deadline = time.time() + max(0, int(timeout))
-        num_stages = len(self.stage_list)
-        while len(self._stages_ready) < num_stages and time.time() < deadline:
-            progressed = False
-            for stage_id, stage in enumerate(self.stage_list):
-                if stage_id in self._stages_ready:
-                    continue
-                result = stage.try_collect()
-                if result is None:
-                    continue
-                progressed = True
-                if result.get("type") == "stage_ready":
-                    self._stages_ready.add(stage_id)
-                    logger.debug("[Orchestrator] Stage-%s reported ready", stage_id)
-                else:
-                    # No user data should arrive before seeding; ignore other messages
-                    pass
-            if not progressed:
-                time.sleep(0.01)
-        if len(self._stages_ready) < num_stages:
-            not_ready = sorted(set(range(num_stages)) - set(self._stages_ready))
-            logger.warning(
-                "[Orchestrator] Initialization timeout: only %s/%s stages are ready; not ready: %s",
-                len(self._stages_ready),
-                num_stages,
-                not_ready,
-            )
-            # Provide actionable suggestions before shutdown
-            try:
-                suggestions = [
-                    "Verify GPU/device assignment in config (runtime.devices) is correct.",
-                    "Check GPU/host memory availability; reduce model or batch size if needed.",
-                    "Check model weights path and network reachability (if loading remotely).",
-                    "Increase initialization wait time (init_sleep_seconds or call-site timeout).",
-                ]
-                if getattr(self, "_log_file", None):
-                    suggestions.append(f"Inspect per-stage log files for details: {self._log_file}.stage<id>.log")
-                logger.error(
-                    "[Orchestrator] Stage initialization failed, shutting down. Suggestions:\n- %s",
-                    "\n- ".join(suggestions),
-                )
-            except Exception:
-                # Best-effort logging of suggestions
-                logger.error(
-                    "[Orchestrator] Stage initialization failed and an error occurred while logging suggestions",
-                )
-
-            # Attempt graceful shutdown of all stages before exiting
-            try:
-                self.close()
-            except Exception:
-                pass
-
-            # Terminate the current process with non-zero exit code
-            try:
-                sys.exit(1)
-            except SystemExit:
-                raise
-            except Exception:
-                os._exit(1)
+    
+    def _run_generation(self, stage: OmniStage, sampling_params: SamplingParams, prompts: Union[PromptType, Sequence[PromptType]]):
+        engine_outputs = []
+        for ro in stage.engine.generate(prompts, sampling_params):
+            engine_outputs.append(ro)
+        return engine_outputs


-class OmniStageLLM(LLM):
-    def __init__(
-        self,
-        model: str,
-        compilation_config: Optional[Union[int, dict[str, Any], CompilationConfig]] = None,
-        hf_overrides: Optional[HfOverrides] = None,
-        structured_outputs_config: Optional[Union[dict[str, Any], StructuredOutputsConfig]] = None,
-        **kwargs,
-    ):
+class OmniStageLLM(LLM): 
+    def __init__(self, 
+                model: str, 
+                compilation_config: Optional[Union[int, dict[str, Any],
+                                           CompilationConfig]] = None,
+                hf_overrides: Optional[HfOverrides] = None,
+                **kwargs):
        """LLM constructor."""
        if "disable_log_stats" not in kwargs:
            kwargs["disable_log_stats"] = True
@@ -484,76 +94,64 @@ class OmniStageLLM(LLM):
            if isinstance(worker_cls, type):
                kwargs["worker_cls"] = cloudpickle.dumps(worker_cls)

-        if "kv_transfer_config" in kwargs and isinstance(kwargs["kv_transfer_config"], dict):
-            from vllm.config.kv_transfer import KVTransferConfig
-
+        if "kv_transfer_config" in kwargs and isinstance(
+                kwargs["kv_transfer_config"], dict):
+            from vllm.config import KVTransferConfig
            raw_config_dict = kwargs["kv_transfer_config"]
            try:
-                kwargs["kv_transfer_config"] = KVTransferConfig(**raw_config_dict)
+                kwargs["kv_transfer_config"] = KVTransferConfig(
+                    **raw_config_dict)
            except ValidationError as e:
                logger.error(
-                    "Failed to convert 'kv_transfer_config' dict to KVTransferConfig object. Dict: %s. Error: %s",
-                    raw_config_dict,
-                    e,
-                )
+                    "Failed to convert 'kv_transfer_config' dict to "
+                    "KVTransferConfig object. Dict: %s. Error: %s",
+                    raw_config_dict, e)
                # Consider re-raising a more specific vLLM error or ValueError
                # to provide better context to the user.
-                raise ValueError(f"Invalid 'kv_transfer_config' provided: {e}") from e
+                raise ValueError(
+                    f"Invalid 'kv_transfer_config' provided: {e}") from e

        if hf_overrides is None:
            hf_overrides = {}

        if compilation_config is not None:
            if isinstance(compilation_config, int):
-                compilation_config_instance = CompilationConfig(level=compilation_config)
-            elif isinstance(compilation_config, dict):
                compilation_config_instance = CompilationConfig(
-                    **{k: v for k, v in compilation_config.items() if is_init_field(CompilationConfig, k)}
-                )
+                    level=compilation_config)
+            elif isinstance(compilation_config, dict):
+                predicate = lambda x: is_init_field(CompilationConfig, x[0])
+                compilation_config_instance = CompilationConfig(
+                    **dict(filter(predicate, compilation_config.items())))
            else:
                compilation_config_instance = compilation_config
        else:
            compilation_config_instance = CompilationConfig()

-        if structured_outputs_config is not None:
-            if isinstance(structured_outputs_config, dict):
-                structured_outputs_instance = StructuredOutputsConfig(
-                    **{k: v for k, v in structured_outputs_config.items() if is_init_field(StructuredOutputsConfig, k)}
-                )
-            else:
-                structured_outputs_instance = structured_outputs_config
-        else:
-            structured_outputs_instance = StructuredOutputsConfig()
-
        engine_args = OmniEngineArgs(
            model=model,
            hf_overrides=hf_overrides,
            compilation_config=compilation_config_instance,
-            structured_outputs_config=structured_outputs_instance,
            **kwargs,
        )

        # Create the Engine (autoselects V0 vs V1)
-        self.llm_engine = LLMEngine.from_engine_args(engine_args=engine_args, usage_context=UsageContext.LLM_CLASS)
-        self.llm_engine.output_processor = MultimodalOutputProcessor(
-            tokenizer=self.llm_engine.tokenizer,
-            log_stats=self.llm_engine.log_stats,
-            engine_core_output_type=engine_args.engine_output_type,
-        )
-        self.llm_engine.processor = OmniProcessor(
-            vllm_config=self.llm_engine.vllm_config, tokenizer=self.llm_engine.tokenizer
-        )
+        self.llm_engine = LLMEngine.from_engine_args(
+            engine_args=engine_args, usage_context=UsageContext.LLM_CLASS)
+        self.llm_engine.output_processor = MultimodalOutputProcessor(tokenizer=self.llm_engine.tokenizer, 
+                                                                    log_stats=self.llm_engine.log_stats)
+        self.llm_engine.processor = OmniProcessor(vllm_config=self.llm_engine.vllm_config,
+                                                  tokenizer=self.llm_engine.tokenizer)
        self.engine_class = type(self.llm_engine)

        self.request_counter = Counter()
        self.default_sampling_params: Union[dict[str, Any], None] = None

-        supported_tasks = self.llm_engine.get_supported_tasks()  # type: ignore
+        if envs.VLLM_USE_V1:
+            supported_tasks = self.llm_engine \
+                .get_supported_tasks()  # type: ignore
+        else:
+            supported_tasks = self.llm_engine.model_config.supported_tasks

        logger.info("Supported_tasks: %s", supported_tasks)

-        self.supported_tasks = supported_tasks
-
-        # Load the Input/Output processor plugin if any
-        io_processor_plugin = self.llm_engine.model_config.io_processor_plugin
-        self.io_processor = get_io_processor(self.llm_engine.vllm_config, io_processor_plugin)
+        self.supported_tasks = supported_tasks
@@ -1,79 +1,48 @@
 """
 Stage manager for orchestrating multiple engines in vLLM-omni.
-
-Enhanced to encapsulate per-stage process lifecycle and worker logic
-(device setup, LLM init, batching, shared-memory IPC), while preserving
-the original input processing utilities for cross-stage data wiring.
-
-Enhanced to encapsulate per-stage process lifecycle and worker logic
-(device setup, LLM init, batching, shared-memory IPC), while preserving
-the original input processing utilities for cross-stage data wiring.
 """

-import asyncio
 import importlib
-import logging
-import multiprocessing as mp
-from typing import Any, Optional, Union
-
-from vllm.inputs import TextPrompt
-from vllm.inputs.preprocess import InputPreprocessor
-from vllm.logger import init_logger
-from vllm.sampling_params import SamplingParams
-from vllm.transformers_utils.tokenizer import AnyTokenizer
-from vllm.usage.usage_lib import UsageContext
-from vllm.v1.engine import EngineCoreOutput
-from vllm.v1.engine.async_llm import AsyncLLM
+from typing import List, Union
 from vllm.v1.engine.llm_engine import LLMEngine
+from vllm.v1.engine.async_llm import AsyncLLM
+from vllm.inputs import TextPrompt
+from vllm.v1.engine import EngineCoreOutput

-from vllm_omni.engine.arg_utils import AsyncOmniEngineArgs
-from vllm_omni.entrypoints.stage_utils import (
-    _to_dict,
-    maybe_dump_to_shm,
-    maybe_load_from_ipc_with_metrics,
-    set_stage_gpu_devices,
-)
 from vllm_omni.inputs.data import OmniTokensPrompt

-logger = init_logger(__name__)
-

 class OmniStage:
    def __init__(self, stage_config):
        self.stage_config = stage_config
        self.engine = None
        self.async_engine = None
-        self.vllm_config = None
-        self.tokenizer = None
-        self.input_preprocessor = None
-        self.is_tracing_enabled = False
        self.stage_id = stage_config.stage_id
        self.engine_args = stage_config.engine_args
        self.model_stage = stage_config.engine_args.model_stage
-        self.requires_multimodal_data = getattr(stage_config.runtime, "requires_multimodal_data", False)
-        self.engine_input_source = getattr(stage_config, "engine_input_source", [])
+        if hasattr(stage_config, 'engine_input_source'):
+            self.engine_input_source = stage_config.engine_input_source
+        else:
+            self.engine_input_source = []
        self.engine_output_type = stage_config.engine_args.engine_output_type
        self.engine_outputs = None
-        self.is_comprehension = getattr(stage_config, "is_comprehension", False)
-        if hasattr(stage_config, "custom_process_input_func"):
+        if hasattr(stage_config, 'custom_process_input_func'):
            # Import the module specified in the config (already a full module path)
-            module_path, func_name = stage_config.custom_process_input_func.rsplit(".", 1)
+            module_path, func_name = stage_config.custom_process_input_func.rsplit('.', 1)
            module = importlib.import_module(module_path)
            self.custom_process_input_func = getattr(module, func_name)
        else:
            self.custom_process_input_func = None

-        self.final_output = getattr(stage_config, "final_output", False)
-        self.final_output_type = getattr(stage_config, "final_output_type", None)
-        default_sampling_params = getattr(stage_config, "default_sampling_params", {})
-        self.default_sampling_params = SamplingParams(**_to_dict(default_sampling_params))
-        # Runtime orchestration state (added)
-        self._in_q: Optional[mp.Queue] = None
-        self._out_q: Optional[mp.Queue] = None
-        self._proc: Optional[mp.Process] = None
-        self._log_file: Optional[str] = None
-        self._shm_threshold_bytes: int = 65536
-        self._logger = logging.getLogger(__name__)
+        if hasattr(stage_config, 'final_output'):
+            self.final_output = stage_config.final_output
+        else:
+            self.final_output = False
+
+        if hasattr(stage_config, 'final_output_type'):
+            self.final_output_type = stage_config.final_output_type
+        else:
+            self.final_output_type = None

    def set_engine(self, engine: LLMEngine) -> None:
        """Initialize the engine for the stage."""
@@ -82,113 +51,12 @@ class OmniStage:
    def set_async_engine(self, async_engine: AsyncLLM) -> None:
        """Initialize the async engine for the stage."""
        self.async_engine = async_engine
-
-    def set_vllm_config(self, vllm_config) -> None:
-        """Set the vllm_config for the stage (received from worker process)."""
-        self.vllm_config = vllm_config
-
-    def set_tokenizer(self, tokenizer: AnyTokenizer) -> None:
-        """Set the tokenizer for the stage (received from worker process)."""
-        self.tokenizer = tokenizer
-
-    def set_input_preprocessor(self, input_preprocessor: InputPreprocessor) -> None:
-        """Set the input preprocessor for the stage (received from worker process)."""
-        self.input_preprocessor = input_preprocessor
-
-    def set_is_tracing_enabled(self, is_tracing_enabled: bool) -> None:
-        """Set the is_tracing_enabled for the stage (received from worker process)."""
-        self.is_tracing_enabled = is_tracing_enabled
-
+    
    def set_engine_outputs(self, engine_outputs: EngineCoreOutput) -> None:
        """Set the engine output for the stage."""
        self.engine_outputs = engine_outputs
-
-    # ----------------- New Orchestration APIs -----------------
-    def attach_queues(self, in_q: mp.Queue, out_q: mp.Queue) -> None:
-        self._in_q = in_q
-        self._out_q = out_q
-
-    def init_stage_worker(
-        self,
-        model: str,
-        *,
-        is_async: bool = False,
-        log_file: Optional[str] = None,
-        shm_threshold_bytes: int = 65536,
-        ctx: Optional[mp.context.BaseContext] = None,
-        batch_timeout: int = 10,
-    ) -> None:
-        assert self._in_q is not None and self._out_q is not None, "Queues must be attached before start_process"
-        self._log_file = log_file
-        self._shm_threshold_bytes = shm_threshold_bytes
-        ctx = ctx or mp.get_context("spawn")
-        # Prepare lightweight dict config for worker
-        engine_args = _to_dict(self.engine_args)
-        runtime_cfg = _to_dict(getattr(self.stage_config, "runtime", {}))
-        stage_payload: dict[str, Any] = {
-            "stage_id": self.stage_id,
-            "engine_args": engine_args,
-            "runtime": runtime_cfg,
-            "shm_threshold_bytes": self._shm_threshold_bytes,
-        }
-        if is_async:
-            self._proc = ctx.Process(
-                target=_stage_worker_async_entry,
-                args=(
-                    self,
-                    model,
-                    stage_payload,
-                    self._in_q,
-                    self._out_q,
-                    self._log_file,
-                    batch_timeout,
-                ),
-            )
-        else:
-            self._proc = ctx.Process(
-                target=_stage_worker,
-                args=(
-                    model,
-                    stage_payload,
-                    self._in_q,
-                    self._out_q,
-                    self._log_file,
-                    batch_timeout,
-                ),
-            )
-        self._proc.start()
-
-    def stop_stage_worker(self) -> None:
-        if self._in_q is not None:
-            try:
-                self._in_q.put_nowait(None)
-            except Exception as e:
-                self._logger.warning("[Stage-%s] Failed to send shutdown to in_q: %s", self.stage_id, e)
-        if self._proc is not None:
-            try:
-                self._proc.join(timeout=5)
-            except Exception as e:
-                self._logger.debug("[Stage-%s] join() failed: %s", self.stage_id, e, exc_info=True)
-            if self._proc.is_alive():
-                try:
-                    self._proc.terminate()
-                except Exception as e:
-                    self._logger.warning("[Stage-%s] terminate() failed: %s", self.stage_id, e)
-
-    def submit(self, payload: dict[str, Any]) -> None:
-        assert self._in_q is not None
-        self._in_q.put(payload)
-
-    def try_collect(self) -> Optional[dict[str, Any]]:
-        assert self._out_q is not None
-        try:
-            return self._out_q.get_nowait()
-        except Exception:
-            return None
-
-    def process_engine_inputs(
-        self, stage_list, prompt: Union[OmniTokensPrompt, TextPrompt] = None
-    ) -> list[Union[OmniTokensPrompt, TextPrompt]]:
+    
+    def process_engine_inputs(self, stage_list, prompt: Union[OmniTokensPrompt, TextPrompt] = None) -> List[Union[OmniTokensPrompt, TextPrompt]]:
        """Process the engine input for the stage."""
        if self.custom_process_input_func is None:
            engine_inputs = []
@@ -196,535 +64,17 @@ class OmniStage:
                raise ValueError("engine_input_source is empty")
            source_stage_id = self.engine_input_source[0]
            source_outputs = stage_list[source_stage_id].engine_outputs
-            if not isinstance(prompt, list):
-                prompt = [prompt]
-            multi_modal_data = {
-                source_output.request_id: p.get("multi_modal_data", None)
-                for source_output, p in zip(source_outputs, prompt)
-            }
+            multi_modal_data = {source_output.request_id: 
+            prompt.get('multi_modal_data', None) for source_output, prompt in zip(source_outputs, prompt)}

            for source_output in source_outputs:
                engine_input = OmniTokensPrompt(
-                    prompt_token_ids=source_output.outputs[0].token_ids,
-                    multi_modal_data=(
-                        multi_modal_data[source_output.request_id]
-                        if self.requires_multimodal_data and multi_modal_data
-                        else None
-                    ),
+                    prompt_token_ids = source_output.outputs[0].token_ids,
+                    multi_modal_data=multi_modal_data[source_output.request_id] if multi_modal_data else None,
                )
                engine_inputs.append(engine_input)
            return engine_inputs

        else:
            engine_input_source = self.engine_input_source
-            return self.custom_process_input_func(
-                stage_list, engine_input_source, prompt, self.requires_multimodal_data
-            )
-
-
-def _stage_worker(
-    model: str,
-    stage_payload: dict[str, Any],
-    in_q: mp.Queue,
-    out_q: mp.Queue,
-    log_file: Optional[str] = None,
-    batch_timeout: int = 10,
-) -> None:
-    """Stage worker entry: device setup, LLM init, batching, SHM IPC."""
-    import logging as _logging
-    import time as _time
-
-    from vllm_omni.entrypoints.log_utils import (
-        compute_and_log_stage_request_stats,
-        count_tokens_from_outputs,
-        log_stage_batch_stats,
-        log_stage_running_avg,
-    )
-    from vllm_omni.entrypoints.omni_llm import OmniStageLLM
-
-    # no inline JSONL/serialization imports; logging handled by utilities
-
-    stage_id = stage_payload["stage_id"]
-    engine_args = stage_payload.get("engine_args", {})
-    runtime_cfg = stage_payload.get("runtime", {})
-    shm_threshold_bytes = int(stage_payload.get("shm_threshold_bytes", 65536))
-
-    # Per-stage file logger (optional)
-    try:
-        if log_file:
-            stage_log = _logging.getLogger(__name__)
-            stage_log.setLevel(_logging.DEBUG)
-            fh = _logging.FileHandler(f"{log_file}.stage{stage_id}.log")
-            fh.setLevel(_logging.DEBUG)
-            fh.setFormatter(
-                _logging.Formatter("%(asctime)s [PID:%(process)d] [Stage-%(stage)s] %(levelname)s: %(message)s")
-            )
-
-            class _StageFilter(_logging.Filter):
-                def filter(self, record: _logging.LogRecord) -> bool:
-                    setattr(record, "stage", stage_id)
-                    return True
-
-            fh.addFilter(_StageFilter())
-            stage_log.addHandler(fh)
-    except Exception:
-        pass
-
-    # Stage stats JSONL file
-    _stats_file = f"{log_file}.stage{stage_id}.stats.jsonl" if log_file else None
-
-    # Aggregates for running average
-    _agg_total_tokens = 0
-    _agg_total_gen_time_ms = 0.0
-    # Monotonic batch id per stage process for orchestrator dedup on time aggregation
-    _batch_seq = 0
-
-    # Device mapping
-    try:
-        set_stage_gpu_devices(stage_id, runtime_cfg.get("devices"))
-    except Exception as e:
-        _logging.getLogger(__name__).warning("[Stage-%s] Device setup failed: %s", stage_id, e)
-
-    # Init LLM
-    _logging.getLogger(__name__).debug(
-        "[Stage-%s] Initializing engine with args keys=%s",
-        stage_id,
-        list(engine_args.keys()),
-    )
-    stage_engine = OmniStageLLM(model=model, **engine_args)
-    _logging.getLogger(__name__).debug("[Stage-%s] Engine initialized", stage_id)
-    # Signal readiness to orchestrator
-    try:
-        out_q.put({"type": "stage_ready", "stage_id": stage_id})
-    except Exception:
-        pass
-
-    # Batch processing loop
-    while True:
-        task = in_q.get()
-        _recv_dequeue_ts = _time.time()
-        if task is None:
-            _logging.getLogger(__name__).debug("[Stage-%s] Received shutdown signal", stage_id)
-            break
-
-        max_batch_size = int(runtime_cfg.get("max_batch_size", 1) or 1)
-        batch_tasks: list[dict[str, Any]] = [task]
-        if max_batch_size > 1:
-            while len(batch_tasks) < max_batch_size:
-                if not in_q.empty():
-                    extra = in_q.get(timeout=batch_timeout)
-                    if extra is None:
-                        in_q.put(None)
-                        break
-                    batch_tasks.append(extra)
-                else:
-                    break
-
-        batch_request_ids: list[Any] = []
-        batch_engine_inputs: list[Any] = []
-        _rx_bytes_by_rid: dict[Any, int] = {}
-        _rx_decode_ms_by_rid: dict[Any, float] = {}
-        _in_flight_ms_by_rid: dict[Any, float] = {}
-        for t in batch_tasks:
-            rid = t["request_id"]
-            try:
-                sent_ts = float(t.get("sent_ts", None)) if isinstance(t, dict) else None
-                if sent_ts is not None:
-                    _in_flight_ms_by_rid[rid] = (_recv_dequeue_ts - sent_ts) * 1000.0
-                else:
-                    _in_flight_ms_by_rid[rid] = 0.0
-            except Exception:
-                _in_flight_ms_by_rid[rid] = 0.0
-            ein, _rx_metrics = maybe_load_from_ipc_with_metrics(t, obj_key="engine_inputs", shm_key="engine_inputs_shm")
-            _rx_decode_ms_by_rid[rid] = float(_rx_metrics.get("rx_decode_time_ms", 0.0))
-            _rx_bytes_by_rid[rid] = int(_rx_metrics.get("rx_transfer_bytes", 0))
-            batch_request_ids.append(rid)
-            if isinstance(ein, list):
-                batch_engine_inputs.extend(ein)
-            elif isinstance(ein, dict):
-                batch_engine_inputs.append(ein)
-            else:
-                _logging.getLogger(__name__).exception("[Stage-%s] Invalid engine input type: %s", stage_id, type(ein))
-        sampling_params = batch_tasks[0]["sampling_params"]
-        _logging.getLogger(__name__).debug(
-            "[Stage-%s] Received batch size=%d, request_ids=%s",
-            stage_id,
-            len(batch_tasks),
-            batch_request_ids,
-        )
-        print("--------------------------------", flush=True)
-        print(
-            f"[Stage-{stage_id}] Received batch size={len(batch_tasks)}, request_ids={batch_request_ids}",
-            flush=True,
-        )
-        print("--------------------------------", flush=True)
-        try:
-            _batch_seq += 1
-            gen_outputs: list[Any] = []
-            _gen_t0 = _time.time()
-            for ro in stage_engine.generate(batch_engine_inputs, sampling_params, use_tqdm=False):
-                gen_outputs.append(ro)
-            _gen_t1 = _time.time()
-            _gen_ms = (_gen_t1 - _gen_t0) * 1000.0
-
-            # Group outputs per request id with fallback
-            req_to_outputs: dict[Any, list[Any]] = {rid: [] for rid in batch_request_ids}
-            unmapped: list[Any] = []
-            for ro in gen_outputs:
-                rid = getattr(ro, "request_id", None)
-                if rid in req_to_outputs:
-                    req_to_outputs[rid].append(ro)
-                else:
-                    unmapped.append(ro)
-            if unmapped:
-                idx = 0
-                for ro in unmapped:
-                    target_rid = batch_request_ids[idx % len(batch_request_ids)]
-                    req_to_outputs[target_rid].append(ro)
-                    idx += 1
-
-            # Per-request stats logging and aggregates
-            for rid in batch_request_ids:
-                _r_outputs = req_to_outputs.get(rid, [])
-                _num_tokens = count_tokens_from_outputs(_r_outputs)
-                _agg_total_tokens += _num_tokens
-                _agg_total_gen_time_ms += _gen_ms
-
-            if _stats_file:
-                _avg_tokens_per_s = (
-                    (_agg_total_tokens * 1000.0 / _agg_total_gen_time_ms) if _agg_total_gen_time_ms > 0 else 0.0
-                )
-                log_stage_running_avg(
-                    _stats_file,
-                    stage_id,
-                    int(_agg_total_tokens),
-                    float(_agg_total_gen_time_ms),
-                    float(_avg_tokens_per_s),
-                )
-                log_stage_batch_stats(
-                    _stats_file,
-                    stage_id,
-                    len(batch_tasks),
-                    float(_gen_ms),
-                    list(batch_request_ids),
-                )
-
-            # Emit per-request results
-            for rid in batch_request_ids:
-                r_outputs = req_to_outputs.get(rid, [])
-                try:
-                    use_shm, payload = maybe_dump_to_shm(r_outputs, shm_threshold_bytes)
-                    _metrics = {
-                        "num_tokens_out": int(count_tokens_from_outputs(r_outputs)),
-                        "stage_gen_time_ms": _gen_ms,
-                        "batch_id": int(_batch_seq),
-                        "rx_decode_time_ms": float(_rx_decode_ms_by_rid.get(rid, 0.0)),
-                        "rx_transfer_bytes": int(_rx_bytes_by_rid.get(rid, 0)),
-                        "rx_in_flight_time_ms": float(_in_flight_ms_by_rid.get(rid, 0.0)),
-                    }
-                    if _stats_file:
-                        compute_and_log_stage_request_stats(
-                            _stats_file,
-                            stage_id,
-                            rid,
-                            len(batch_tasks),
-                            r_outputs,
-                            float(_gen_ms),
-                            int(_metrics["rx_transfer_bytes"]),  # type: ignore[index]
-                            float(_metrics["rx_decode_time_ms"]),  # type: ignore[index]
-                        )
-                    if use_shm:
-                        out_q.put(
-                            {
-                                "request_id": rid,
-                                "stage_id": stage_id,
-                                "engine_outputs_shm": payload,
-                                "metrics": _metrics,
-                            }
-                        )
-                    else:
-                        out_q.put(
-                            {
-                                "request_id": rid,
-                                "stage_id": stage_id,
-                                "engine_outputs": payload,
-                                "metrics": _metrics,
-                            }
-                        )
-                except Exception:
-                    out_q.put(
-                        {
-                            "request_id": rid,
-                            "stage_id": stage_id,
-                            "engine_outputs": r_outputs,
-                            "metrics": {
-                                "num_tokens_out": int(count_tokens_from_outputs(r_outputs)),
-                                "stage_gen_time_ms": _gen_ms,
-                                "rx_decode_time_ms": float(_rx_decode_ms_by_rid.get(rid, 0.0)),
-                                "rx_transfer_bytes": int(_rx_bytes_by_rid.get(rid, 0)),
-                                "rx_in_flight_time_ms": float(_in_flight_ms_by_rid.get(rid, 0.0)),
-                            },
-                        }
-                    )
-                _logging.getLogger(__name__).debug(
-                    "[Stage-%s] Enqueued result for request %s to downstream",
-                    stage_id,
-                    rid,
-                )
-        except Exception as e:
-            _logging.getLogger(__name__).exception("[Stage-%s] Failed on batch %s: %s", stage_id, batch_request_ids, e)
-            for rid in batch_request_ids:
-                out_q.put(
-                    {
-                        "request_id": rid,
-                        "stage_id": stage_id,
-                        "error": str(e),
-                    }
-                )
-
-
-def _stage_worker_async_entry(
-    omni_stage: OmniStage,
-    model: str,
-    stage_payload: dict[str, Any],
-    in_q: mp.Queue,
-    out_q: mp.Queue,
-    log_file: Optional[str] = None,
-    batch_timeout: int = 10,
-) -> None:
-    asyncio.run(_stage_worker_async(omni_stage, model, stage_payload, in_q, out_q, log_file, batch_timeout))
-
-
-async def _stage_worker_async(
-    omni_stage: OmniStage,
-    model: str,
-    stage_payload: dict[str, Any],
-    in_q: mp.Queue,
-    out_q: mp.Queue,
-    log_file: Optional[str] = None,
-    batch_timeout: int = 10,
-) -> None:
-    """Stage worker entry: device setup, LLM init, batching, SHM IPC."""
-    import logging as _logging
-    import time as _time
-
-    from vllm_omni.entrypoints.async_omni_llm import AsyncOmniStageLLM
-    from vllm_omni.entrypoints.log_utils import (
-        compute_and_log_stage_request_stats,
-        count_tokens_from_outputs,
-        log_stage_batch_stats,
-        log_stage_running_avg,
-    )
-
-    # no inline JSONL/serialization imports; logging handled by utilities
-
-    stage_id = stage_payload["stage_id"]
-    engine_args = stage_payload.get("engine_args", {})
-    runtime_cfg = stage_payload.get("runtime", {})
-    shm_threshold_bytes = int(stage_payload.get("shm_threshold_bytes", 65536))
-
-    # Per-stage file logger (optional)
-    try:
-        if log_file:
-            stage_log = _logging.getLogger(__name__)
-            stage_log.setLevel(_logging.DEBUG)
-            fh = _logging.FileHandler(f"{log_file}.stage{stage_id}.log")
-            fh.setLevel(_logging.DEBUG)
-            fh.setFormatter(
-                _logging.Formatter("%(asctime)s [PID:%(process)d] [Stage-%(stage)s] %(levelname)s: %(message)s")
-            )  # noqa: E501
-
-            class _StageFilter(_logging.Filter):
-                def filter(self, record: _logging.LogRecord) -> bool:
-                    setattr(record, "stage", stage_id)
-                    return True
-
-            fh.addFilter(_StageFilter())
-            stage_log.addHandler(fh)
-    except Exception:
-        pass
-
-    # Stage stats JSONL file
-    _stats_file = f"{log_file}.stage{stage_id}.stats.jsonl" if log_file else None
-
-    # Aggregates for running average
-    _agg_total_tokens = 0
-    _agg_total_gen_time_ms = 0.0
-    # Monotonic batch id per stage process for orchestrator dedup on time
-    # aggregation
-    _batch_seq = 0
-
-    # Device mapping
-    try:
-        set_stage_gpu_devices(stage_id, runtime_cfg.get("devices"))
-    except Exception as e:
-        _logging.getLogger(__name__).warning("[Stage-%s] Device setup failed: %s", stage_id, e)
-
-    # Init LLM
-    _logging.getLogger(__name__).debug(
-        "[Stage-%s] Initializing engine with args keys=%s",
-        stage_id,
-        list(engine_args.keys()),
-    )
-    omni_engine_args = AsyncOmniEngineArgs(model=model, **engine_args)
-    usage_context = UsageContext.OPENAI_API_SERVER
-    vllm_config = omni_engine_args.create_engine_config(usage_context=usage_context)
-    stage_engine = AsyncOmniStageLLM.from_vllm_config(
-        vllm_config=vllm_config,
-        usage_context=usage_context,
-        engine_args=omni_engine_args,
-    )
-    omni_stage.set_async_engine(stage_engine)
-    # Don't keep the dummy data in memory
-    await stage_engine.reset_mm_cache()
-    _logging.getLogger(__name__).debug("[Stage-%s] Engine initialized", stage_id)
-    # Signal readiness to orchestrator and send vllm_config back to main process
-    try:
-        # Send vllm_config back to main process so it can be accessed via
-        # get_vllm_config(). This is needed because async_engine is only available
-        # in the worker process
-
-        # input_preprocessor = await stage_engine.get_input_preprocessor()
-        out_q.put(
-            {
-                "type": "stage_ready",
-                "stage_id": stage_id,
-                "vllm_config": vllm_config,
-                "tokenizer": getattr(stage_engine, "tokenizer", None),
-                "is_tracing_enabled": await stage_engine.is_tracing_enabled(),
-                # "input_preprocessor": input_preprocessor,
-            }
-        )
-    except Exception as e:
-        _logging.getLogger(__name__).warning("[Stage-%s] Failed to send stage ready signal: %s", stage_id, e)
-
-    # Batch processing loop
-    while True:
-        task = in_q.get()
-        _recv_dequeue_ts = _time.time()
-        if task is None:
-            _logging.getLogger(__name__).debug("[Stage-%s] Received shutdown signal", stage_id)
-            break
-
-        _rx_bytes_by_rid: dict[Any, int] = {}
-        _rx_decode_ms_by_rid: dict[Any, float] = {}
-        _in_flight_ms_by_rid: dict[Any, float] = {}
-
-        rid = task["request_id"]
-        try:
-            sent_ts = float(task.get("sent_ts", None)) if isinstance(task, dict) else None
-            if sent_ts is not None:
-                _in_flight_ms_by_rid[rid] = (_recv_dequeue_ts - sent_ts) * 1000.0
-            else:
-                _in_flight_ms_by_rid[rid] = 0.0
-        except Exception:
-            _in_flight_ms_by_rid[rid] = 0.0
-        ein, _rx_metrics = maybe_load_from_ipc_with_metrics(task, obj_key="engine_inputs", shm_key="engine_inputs_shm")
-        _rx_decode_ms_by_rid[rid] = float(_rx_metrics.get("rx_decode_time_ms", 0.0))
-        _rx_bytes_by_rid[rid] = int(_rx_metrics.get("rx_transfer_bytes", 0))
-
-        sampling_params = task["sampling_params"]
-        _logging.getLogger(__name__).debug("[Stage-%s] Received batch size=1, request_ids=%d", stage_id, rid)
-        print("--------------------------------", flush=True)
-        print(f"[Stage-{stage_id}] Received batch size=1, request_ids={rid}", flush=True)
-        print("--------------------------------", flush=True)
-        try:
-            _batch_seq += 1
-            _gen_t0 = _time.time()
-            if isinstance(ein, list):
-                ein = ein[0]
-
-            async for res in stage_engine.generate(ein, sampling_params, rid):
-                gen_output = res
-            _gen_t1 = _time.time()
-            _gen_ms = (_gen_t1 - _gen_t0) * 1000.0
-
-            r_outputs = [gen_output]
-            _num_tokens = count_tokens_from_outputs(r_outputs)
-            _agg_total_tokens += _num_tokens
-            _agg_total_gen_time_ms += _gen_ms
-
-            if _stats_file:
-                _avg_tokens_per_s = (
-                    (_agg_total_tokens * 1000.0 / _agg_total_gen_time_ms) if _agg_total_gen_time_ms > 0 else 0.0
-                )
-                log_stage_running_avg(
-                    _stats_file,
-                    stage_id,
-                    int(_agg_total_tokens),
-                    float(_agg_total_gen_time_ms),
-                    float(_avg_tokens_per_s),
-                )
-                log_stage_batch_stats(_stats_file, stage_id, 1, float(_gen_ms), [rid])
-
-            try:
-                use_shm, payload = maybe_dump_to_shm(r_outputs, shm_threshold_bytes)
-                _metrics = {
-                    "num_tokens_out": int(count_tokens_from_outputs(r_outputs)),
-                    "stage_gen_time_ms": _gen_ms,
-                    "batch_id": int(_batch_seq),
-                    "rx_decode_time_ms": float(_rx_decode_ms_by_rid.get(rid, 0.0)),
-                    "rx_transfer_bytes": int(_rx_bytes_by_rid.get(rid, 0)),
-                    "rx_in_flight_time_ms": float(_in_flight_ms_by_rid.get(rid, 0.0)),
-                }
-                if _stats_file:
-                    compute_and_log_stage_request_stats(
-                        _stats_file,
-                        stage_id,
-                        rid,
-                        1,
-                        r_outputs,
-                        float(_gen_ms),
-                        int(_metrics["rx_transfer_bytes"]),  # type: ignore[index]
-                        float(_metrics["rx_decode_time_ms"]),  # type: ignore[index]
-                    )
-                if use_shm:
-                    out_q.put(
-                        {
-                            "request_id": rid,
-                            "stage_id": stage_id,
-                            "engine_outputs_shm": payload,
-                            "metrics": _metrics,
-                        }
-                    )
-                else:
-                    out_q.put(
-                        {
-                            "request_id": rid,
-                            "stage_id": stage_id,
-                            "engine_outputs": payload,
-                            "metrics": _metrics,
-                        }
-                    )
-            except Exception as e:
-                _logging.getLogger(__name__).exception(
-                    "[Stage-%s] Failed to enqueue result for request %s: %s",
-                    stage_id,
-                    rid,
-                    e,
-                )
-                out_q.put(
-                    {
-                        "request_id": rid,
-                        "stage_id": stage_id,
-                        "engine_outputs": r_outputs,
-                        "metrics": {
-                            "num_tokens_out": int(count_tokens_from_outputs(r_outputs)),
-                            "stage_gen_time_ms": _gen_ms,
-                            "rx_decode_time_ms": float(_rx_decode_ms_by_rid.get(rid, 0.0)),
-                            "rx_transfer_bytes": int(_rx_bytes_by_rid.get(rid, 0)),
-                            "rx_in_flight_time_ms": float(_in_flight_ms_by_rid.get(rid, 0.0)),
-                        },
-                    }
-                )
-            _logging.getLogger(__name__).debug("[Stage-%s] Enqueued result for request %s to downstream", stage_id, rid)
-
-        except Exception as e:
-            _logging.getLogger(__name__).exception("[Stage-%s] Failed on request %s: %s", stage_id, rid, e)
-            out_q.put(
-                {
-                    "request_id": rid,
-                    "stage_id": stage_id,
-                    "error": str(e),
-                }
-            )
+            return self.custom_process_input_func(stage_list, engine_input_source, prompt)
@@ -1,313 +0,0 @@
-import multiprocessing
-import multiprocessing.forkserver as forkserver
-import os
-from argparse import Namespace
-from collections.abc import AsyncIterator
-from contextlib import asynccontextmanager
-from http import HTTPStatus
-from typing import Any, Optional
-
-import vllm.envs as envs
-from fastapi import Depends, HTTPException, Request
-from fastapi.responses import JSONResponse, StreamingResponse
-from starlette.datastructures import State
-from vllm.config import VllmConfig
-from vllm.engine.protocol import EngineClient
-from vllm.entrypoints.chat_utils import load_chat_template, resolve_hf_chat_template, resolve_mistral_chat_template
-from vllm.entrypoints.launcher import serve_http
-from vllm.entrypoints.logger import RequestLogger
-from vllm.entrypoints.openai.api_server import (
-    base,
-    build_app,
-    load_log_config,
-    maybe_register_tokenizer_info_endpoint,
-    router,
-    setup_server,
-    validate_json_request,
-)
-from vllm.entrypoints.openai.protocol import ChatCompletionRequest, ChatCompletionResponse, ErrorResponse
-from vllm.entrypoints.openai.serving_models import BaseModelPath, LoRAModulePath, OpenAIServingModels
-from vllm.entrypoints.openai.tool_parsers import ToolParserManager
-
-# yapf conflicts with isort for this block
-# yapf: disable
-# yapf: enable
-from vllm.entrypoints.tool_server import DemoToolServer, MCPToolServer, ToolServer
-from vllm.entrypoints.utils import load_aware_call, with_cancellation
-from vllm.logger import init_logger
-from vllm.transformers_utils.tokenizer import MistralTokenizer
-from vllm.utils import decorate_logs
-
-from vllm_omni.entrypoints.async_omni_llm import AsyncOmniLLM
-from vllm_omni.entrypoints.openai.serving_chat import OmniOpenAIServingChat
-
-logger = init_logger(__name__)
-
-
-async def omni_run_server(args, **uvicorn_kwargs) -> None:
-    """Run a single-worker API server."""
-
-    # Add process-specific prefix to stdout and stderr.
-    decorate_logs("APIServer")
-
-    listen_address, sock = setup_server(args)
-    await omni_run_server_worker(listen_address, sock, args, **uvicorn_kwargs)
-
-
-async def omni_run_server_worker(listen_address, sock, args, client_config=None, **uvicorn_kwargs) -> None:
-    """Run a single API server worker."""
-
-    if args.tool_parser_plugin and len(args.tool_parser_plugin) > 3:
-        ToolParserManager.import_tool_parser(args.tool_parser_plugin)
-
-    # Load logging config for uvicorn if specified
-    log_config = load_log_config(args.log_config_file)
-    if log_config is not None:
-        uvicorn_kwargs["log_config"] = log_config
-
-    async with build_async_omni_llm(
-        args,
-        client_config=client_config,
-    ) as engine_client:
-        maybe_register_tokenizer_info_endpoint(args)
-        app = build_app(args)
-
-        vllm_config = await engine_client.get_vllm_config()
-        await omni_init_app_state(engine_client, vllm_config, app.state, args)
-
-        logger.info(
-            "Starting vLLM API server %d on %s",
-            vllm_config.parallel_config._api_process_rank,
-            listen_address,
-        )
-        shutdown_task = await serve_http(
-            app,
-            sock=sock,
-            enable_ssl_refresh=args.enable_ssl_refresh,
-            host=args.host,
-            port=args.port,
-            log_level=args.uvicorn_log_level,
-            # NOTE: When the 'disable_uvicorn_access_log' value is True,
-            # no access log will be output.
-            access_log=not args.disable_uvicorn_access_log,
-            timeout_keep_alive=envs.VLLM_HTTP_TIMEOUT_KEEP_ALIVE,
-            ssl_keyfile=args.ssl_keyfile,
-            ssl_certfile=args.ssl_certfile,
-            ssl_ca_certs=args.ssl_ca_certs,
-            ssl_cert_reqs=args.ssl_cert_reqs,
-            h11_max_incomplete_event_size=args.h11_max_incomplete_event_size,
-            h11_max_header_count=args.h11_max_header_count,
-            **uvicorn_kwargs,
-        )
-
-    # NB: Await server shutdown only after the backend context is exited
-    try:
-        await shutdown_task
-    finally:
-        sock.close()
-
-
-@asynccontextmanager
-async def build_async_omni_llm(
-    args: Namespace,
-    *,
-    disable_frontend_multiprocessing: Optional[bool] = None,
-    client_config: Optional[dict[str, Any]] = None,
-) -> AsyncIterator[EngineClient]:
-    if os.getenv("VLLM_WORKER_MULTIPROC_METHOD") == "forkserver":
-        # The executor is expected to be mp.
-        # Pre-import heavy modules in the forkserver process
-        logger.debug("Setup forkserver with pre-imports")
-        multiprocessing.set_start_method("forkserver")
-        multiprocessing.set_forkserver_preload(["vllm.v1.engine.async_llm"])
-        forkserver.ensure_running()
-        logger.debug("Forkserver setup complete!")
-
-    # Context manager to handle async_omni_llm lifecycle
-    # Ensures everything is shutdown and cleaned up on error/exit
-    async with build_async_omni_llm_from_stage_config(
-        args,
-        disable_frontend_multiprocessing=disable_frontend_multiprocessing,
-    ) as async_omni_llm:
-        yield async_omni_llm
-
-
-@asynccontextmanager
-async def build_async_omni_llm_from_stage_config(
-    args: Namespace,
-    *,
-    disable_frontend_multiprocessing: bool = False,
-    client_config: Optional[dict[str, Any]] = None,
-) -> AsyncIterator[EngineClient]:
-    """
-    Create AsyncOmniLLM, either:
-        - in-process using the AsyncOmniLLM Directly
-        - multiprocess using AsyncOmniLLM RPC
-
-    Returns the AsyncOmniLLM or None if the creation failed.
-    """
-
-    # V1 AsyncLLM.
-    assert envs.VLLM_USE_V1
-
-    if disable_frontend_multiprocessing:
-        logger.warning(
-            "V1 is enabled, but got --disable-frontend-multiprocessing. "
-            "To disable frontend multiprocessing, set VLLM_USE_V1=0."
-        )
-
-    async_omni_llm: Optional[EngineClient] = None
-
-    try:
-        if getattr(args, "stage_configs_path", None):
-            async_omni_llm = AsyncOmniLLM(model=args.model, stage_configs_path=args.stage_configs_path)
-        else:
-            async_omni_llm = AsyncOmniLLM(model=args.model)
-
-        # # Don't keep the dummy data in memory
-        # await async_llm.reset_mm_cache()
-
-        yield async_omni_llm
-    finally:
-        if async_omni_llm:
-            async_omni_llm.shutdown()
-
-
-async def omni_init_app_state(
-    engine_client: EngineClient,
-    vllm_config: VllmConfig,
-    state: State,
-    args: Namespace,
-) -> None:
-    if args.served_model_name is not None:
-        served_model_names = args.served_model_name
-    else:
-        served_model_names = [args.model]
-
-    if args.enable_log_requests:
-        request_logger = RequestLogger(max_log_len=args.max_log_len)
-    else:
-        request_logger = None
-
-    base_model_paths = [BaseModelPath(name=name, model_path=args.model) for name in served_model_names]
-    state.engine_client = engine_client
-    state.log_stats = not args.disable_log_stats
-    state.vllm_config = vllm_config
-    model_config = vllm_config.model_config
-    state.log_stats = not args.disable_log_stats
-
-    # For omni models
-    state.stage_configs = engine_client.stage_configs
-
-    resolved_chat_template = load_chat_template(args.chat_template)
-    if resolved_chat_template is not None:
-        # Get the tokenizer to check official template
-        tokenizer = await engine_client.get_tokenizer()
-
-        if isinstance(tokenizer, MistralTokenizer):
-            # The warning is logged in resolve_mistral_chat_template.
-            resolved_chat_template = resolve_mistral_chat_template(chat_template=resolved_chat_template)
-        else:
-            hf_chat_template = resolve_hf_chat_template(
-                tokenizer=tokenizer,
-                chat_template=None,
-                tools=None,
-                model_config=vllm_config.model_config,
-            )
-
-            if hf_chat_template != resolved_chat_template:
-                logger.warning(
-                    "Using supplied chat template: %s\nIt is different from official chat template '%s'. This discrepancy may lead to performance degradation.",  # noqa: E501
-                    resolved_chat_template,
-                    args.model,
-                )
-
-    if args.tool_server == "demo":
-        tool_server: Optional[ToolServer] = DemoToolServer()
-        assert isinstance(tool_server, DemoToolServer)
-        await tool_server.init_and_validate()
-    elif args.tool_server:
-        tool_server = MCPToolServer()
-        await tool_server.add_tool_server(args.tool_server)
-    else:
-        tool_server = None
-
-    # Merge default_mm_loras into the static lora_modules
-    default_mm_loras = vllm_config.lora_config.default_mm_loras if vllm_config.lora_config is not None else {}
-
-    lora_modules = args.lora_modules
-    if default_mm_loras:
-        default_mm_lora_paths = [
-            LoRAModulePath(
-                name=modality,
-                path=lora_path,
-            )
-            for modality, lora_path in default_mm_loras.items()
-        ]
-        if args.lora_modules is None:
-            lora_modules = default_mm_lora_paths
-        else:
-            lora_modules += default_mm_lora_paths
-
-    state.openai_serving_models = OpenAIServingModels(
-        engine_client=engine_client,
-        model_config=model_config,
-        base_model_paths=base_model_paths,
-        lora_modules=lora_modules,
-    )
-    await state.openai_serving_models.init_static_loras()
-    state.openai_serving_chat = OmniOpenAIServingChat(
-        engine_client,
-        model_config,
-        state.openai_serving_models,
-        args.response_role,
-        request_logger=request_logger,
-        chat_template=resolved_chat_template,
-        chat_template_content_format=args.chat_template_content_format,
-        trust_request_chat_template=args.trust_request_chat_template,
-        return_tokens_as_token_ids=args.return_tokens_as_token_ids,
-        enable_auto_tools=args.enable_auto_tool_choice,
-        exclude_tools_when_tool_choice_none=args.exclude_tools_when_tool_choice_none,
-        tool_parser=args.tool_call_parser,
-        reasoning_parser=args.structured_outputs_config.reasoning_parser,
-        enable_prompt_tokens_details=args.enable_prompt_tokens_details,
-        enable_force_include_usage=args.enable_force_include_usage,
-        enable_log_outputs=args.enable_log_outputs,
-        log_error_stack=args.log_error_stack,
-    )
-
-    state.enable_server_load_tracking = args.enable_server_load_tracking
-    state.server_load_metrics = 0
-
-
-def Omnichat(request: Request) -> Optional[OmniOpenAIServingChat]:
-    return request.app.state.openai_serving_chat
-
-
-@router.post(
-    "/v1/chat/completions",
-    dependencies=[Depends(validate_json_request)],
-    responses={
-        HTTPStatus.OK.value: {"content": {"text/event-stream": {}}},
-        HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
-        HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse},
-        HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
-    },
-)
-@with_cancellation
-@load_aware_call
-async def create_chat_completion(request: ChatCompletionRequest, raw_request: Request):
-    handler = Omnichat(raw_request)
-    if handler is None:
-        return base(raw_request).create_error_response(message="The model does not support Chat Completions API")
-    try:
-        generator = await handler.create_chat_completion(request, raw_request)
-    except Exception as e:
-        raise HTTPException(status_code=HTTPStatus.INTERNAL_SERVER_ERROR.value, detail=str(e)) from e
-    if isinstance(generator, ErrorResponse):
-        return JSONResponse(content=generator.model_dump(), status_code=generator.error.code)
-
-    elif isinstance(generator, ChatCompletionResponse):
-        return JSONResponse(content=generator.model_dump())
-
-    return StreamingResponse(content=generator, media_type="text/event-stream")
@@ -1,779 +0,0 @@
-import asyncio
-import base64
-import json
-import time
-import uuid
-from collections.abc import AsyncGenerator, AsyncIterator, Sequence
-from datetime import datetime, timedelta, timezone
-from io import BytesIO
-from typing import Any, Callable, Optional, Union
-
-import jinja2
-from fastapi import Request
-from pydantic import TypeAdapter
-
-try:
-    import soundfile
-except ImportError:
-    soundfile = None
-
-from openai.types.chat.chat_completion_audio import ChatCompletionAudio as OpenAIChatCompletionAudio
-from vllm.entrypoints.chat_utils import (
-    ChatCompletionMessageParam,
-    ChatTemplateContentFormatOption,
-    ConversationMessage,
-    apply_hf_chat_template,
-    apply_mistral_chat_template,
-    get_history_tool_calls_cnt,
-    make_tool_call_id,
-    resolve_chat_template_content_format,
-)
-from vllm.entrypoints.harmony_utils import parse_chat_output
-from vllm.entrypoints.openai.protocol import (
-    ChatCompletionNamedToolChoiceParam,
-    ChatCompletionRequest,
-    ChatCompletionResponse,
-    ChatCompletionResponseChoice,
-    ChatMessage,
-    ErrorResponse,
-    FunctionCall,
-    FunctionDefinition,
-    PromptTokenUsageInfo,
-    RequestResponseMetadata,
-    ToolCall,
-    UsageInfo,
-)
-from vllm.entrypoints.openai.serving_chat import OpenAIServingChat
-from vllm.entrypoints.openai.serving_engine import (
-    ChatLikeRequest,
-    EngineTokensPrompt,
-    RequestPrompt,
-    ResponsesRequest,
-    TextTokensPrompt,
-    clamp_prompt_logprobs,
-    is_list_of,
-)
-from vllm.entrypoints.openai.tool_parsers import ToolParser
-from vllm.entrypoints.openai.tool_parsers.mistral_tool_parser import MistralToolCall
-from vllm.inputs.data import PromptType
-from vllm.logger import init_logger
-from vllm.lora.request import LoRARequest
-from vllm.outputs import RequestOutput
-from vllm.sampling_params import SamplingParams
-from vllm.transformers_utils.tokenizer import AnyTokenizer, MistralTokenizer
-from vllm.transformers_utils.tokenizers import (
-    maybe_serialize_tool_calls,
-    truncate_tool_call_ids,
-    validate_request_params,
-)
-from vllm.utils import as_list
-
-from vllm_omni.entrypoints.chat_utils import parse_chat_messages_futures
-from vllm_omni.outputs import OmniRequestOutput
-
-logger = init_logger(__name__)
-
-
-class OmniOpenAIServingChat(OpenAIServingChat):
-    async def create_chat_completion(
-        self,
-        request: ChatCompletionRequest,
-        raw_request: Optional[Request] = None,
-    ) -> Union[AsyncGenerator[str, None], ChatCompletionResponse, ErrorResponse]:
-        """
-        Chat Completion API similar to OpenAI's API.
-
-        See https://platform.openai.com/docs/api-reference/chat/create
-        for the API specification. This API mimics the OpenAI
-        Chat Completion API.
-        """
-        error_check_ret = await self._check_model(request)
-        if error_check_ret is not None:
-            logger.error("Error with model %s", error_check_ret)
-            return error_check_ret
-
-        # If the engine is dead, raise the engine's DEAD_ERROR.
-        # This is required for the streaming case, where we return a
-        # success status before we actually start generating text :).
-        if self.engine_client.errored:
-            raise self.engine_client.dead_error
-
-        try:
-            lora_request = self._maybe_get_adapters(request, supports_default_mm_loras=True)
-
-            model_name = self.models.model_name(lora_request)
-
-            tokenizer = await self.engine_client.get_tokenizer()
-
-            tool_parser = self.tool_parser
-
-            if isinstance(tokenizer, MistralTokenizer):
-                # because of issues with pydantic we need to potentially
-                # re-serialize the tool_calls field of the request
-                # for more info: see comment in `maybe_serialize_tool_calls`
-                maybe_serialize_tool_calls(request)
-                truncate_tool_call_ids(request)
-                validate_request_params(request)
-
-            if (
-                request.tool_choice == "auto"
-                and not (self.enable_auto_tools and tool_parser is not None)
-                and not isinstance(tokenizer, MistralTokenizer)
-                and not self.use_harmony
-            ):
-                # for hf tokenizers, "auto" tools requires
-                # --enable-auto-tool-choice and --tool-call-parser
-                return self.create_error_response(
-                    '"auto" tool choice requires --enable-auto-tool-choice and --tool-call-parser to be set'
-                )
-
-            if request.tools is None or (request.tool_choice == "none" and self.exclude_tools_when_tool_choice_none):
-                tool_dicts = None
-            else:
-                tool_dicts = [tool.model_dump() for tool in request.tools]
-
-            # Common case.
-            request_chat_template = request.chat_template
-            chat_template_kwargs = request.chat_template_kwargs
-            if not self.trust_request_chat_template and (
-                request_chat_template is not None
-                or (chat_template_kwargs and chat_template_kwargs.get("chat_template") is not None)
-            ):
-                return self.create_error_response(
-                    "Chat template is passed with request, but --trust-request-chat-template is not set. "
-                    "Refused request with untrusted chat template."
-                )
-            (
-                conversation,
-                request_prompts,
-                engine_prompts,
-            ) = await self._preprocess_chat(
-                request,
-                tokenizer,
-                request.messages,
-                chat_template=request_chat_template or self.chat_template,
-                chat_template_content_format=self.chat_template_content_format,
-                add_generation_prompt=request.add_generation_prompt,
-                continue_final_message=request.continue_final_message,
-                tool_dicts=tool_dicts,
-                documents=request.documents,
-                chat_template_kwargs=request.chat_template_kwargs,
-                tool_parser=tool_parser,
-                add_special_tokens=request.add_special_tokens,
-            )
-
-        except (ValueError, TypeError, RuntimeError, jinja2.TemplateError) as e:
-            logger.exception("Error in preprocessing prompt inputs")
-            return self.create_error_response(f"{e} {e.__cause__}")
-
-        request_id = f"chatcmpl-{self._base_request_id(raw_request, request.request_id)}"
-
-        request_metadata = RequestResponseMetadata(request_id=request_id)
-        if raw_request:
-            raw_request.state.request_metadata = request_metadata
-
-        # Schedule the request and get the result generator.
-        generators: list[AsyncGenerator[RequestOutput, None]] = []
-        try:
-            for i, engine_prompt in enumerate(engine_prompts):
-                if hasattr(request, "sampling_params_list"):
-                    sampling_params_list = self._to_sampling_params_list(request.sampling_params_list)
-                else:
-                    sampling_params_list = None
-
-                self._log_inputs(
-                    request_id,
-                    request_prompts[i],
-                    params_list=sampling_params_list,
-                    lora_request=lora_request,
-                )
-
-                trace_headers = None if raw_request is None else await self._get_trace_headers(raw_request.headers)
-
-                generator = self.engine_client.generate(
-                    prompt=engine_prompt,
-                    request_id=request_id,
-                    sampling_params_list=sampling_params_list,
-                    lora_request=lora_request,
-                    trace_headers=trace_headers,
-                    priority=request.priority,
-                )
-
-                generators.append(generator)
-        except ValueError as e:
-            # TODO: Use a vllm-specific Validation Error
-            return self.create_error_response(str(e))
-
-        assert len(generators) == 1
-        (result_generator,) = generators
-
-        # Streaming response
-        if request.stream:
-            return self.chat_completion_stream_generator(
-                request,
-                result_generator,
-                request_id,
-                model_name,
-                conversation,
-                tokenizer,
-                request_metadata,
-                enable_force_include_usage=self.enable_force_include_usage,
-            )
-
-        try:
-            return await self.chat_completion_full_generator(
-                request,
-                result_generator,
-                request_id,
-                model_name,
-                conversation,
-                tokenizer,
-                request_metadata,
-            )
-        except ValueError as e:
-            # TODO: Use a vllm-specific Validation Error
-            return self.create_error_response(str(e))
-
-    async def _preprocess_chat(
-        self,
-        request: Union[ChatLikeRequest, ResponsesRequest],
-        tokenizer: AnyTokenizer,
-        messages: list[ChatCompletionMessageParam],
-        chat_template: Optional[str],
-        chat_template_content_format: ChatTemplateContentFormatOption,
-        add_generation_prompt: bool = True,
-        continue_final_message: bool = False,
-        tool_dicts: Optional[list[dict[str, Any]]] = None,
-        documents: Optional[list[dict[str, str]]] = None,
-        chat_template_kwargs: Optional[dict[str, Any]] = None,
-        tool_parser: Optional[Callable[[AnyTokenizer], ToolParser]] = None,
-        add_special_tokens: bool = False,
-    ) -> tuple[
-        list[ConversationMessage],
-        Sequence[RequestPrompt],
-        list[EngineTokensPrompt],
-    ]:
-        model_config = self.model_config
-
-        resolved_content_format = resolve_chat_template_content_format(
-            chat_template,
-            tool_dicts,
-            chat_template_content_format,
-            tokenizer,
-            model_config=model_config,
-        )
-        conversation, mm_data_future, mm_uuids = parse_chat_messages_futures(
-            messages,
-            model_config,
-            tokenizer,
-            content_format=resolved_content_format,
-            mm_processor_kwargs=getattr(request, "mm_processor_kwargs", None),
-        )
-
-        _chat_template_kwargs: dict[str, Any] = dict(
-            chat_template=chat_template,
-            add_generation_prompt=add_generation_prompt,
-            continue_final_message=continue_final_message,
-            tools=tool_dicts,
-            documents=documents,
-        )
-        _chat_template_kwargs.update(chat_template_kwargs or {})
-
-        request_prompt: Union[str, list[int]]
-
-        if tokenizer is None:
-            request_prompt = "placeholder"
-        elif isinstance(tokenizer, MistralTokenizer):
-            request_prompt = apply_mistral_chat_template(
-                tokenizer,
-                messages=messages,
-                **_chat_template_kwargs,
-            )
-        else:
-            request_prompt = apply_hf_chat_template(
-                tokenizer=tokenizer,
-                conversation=conversation,
-                model_config=model_config,
-                **_chat_template_kwargs,
-            )
-
-        mm_data = await mm_data_future
-
-        # tool parsing is done only if a tool_parser has been set and if
-        # tool_choice is not "none" (if tool_choice is "none" but a tool_parser
-        # is set, we want to prevent parsing a tool_call hallucinated by the LLM
-        should_parse_tools = tool_parser is not None and (
-            hasattr(request, "tool_choice") and request.tool_choice != "none"
-        )
-
-        if should_parse_tools:
-            if not isinstance(request, ChatCompletionRequest):
-                msg = "Tool usage is only supported for Chat Completions API"
-                raise NotImplementedError(msg)
-
-            request = tool_parser(tokenizer).adjust_request(  # type: ignore
-                request=request
-            )
-
-        if tokenizer is None:
-            assert isinstance(request_prompt, str), (
-                "Prompt has to be a string",
-                "when the tokenizer is not initialised",
-            )
-            prompt_inputs = TextTokensPrompt(prompt=request_prompt, prompt_token_ids=[1])
-        elif isinstance(request_prompt, str):
-            prompt_inputs = await self._tokenize_prompt_input_async(
-                request,
-                tokenizer,
-                request_prompt,
-                add_special_tokens=add_special_tokens,
-            )
-        else:
-            # For MistralTokenizer
-            assert is_list_of(request_prompt, int), "Prompt has to be either a string or a list of token ids"
-            prompt_inputs = TextTokensPrompt(
-                prompt=tokenizer.decode(request_prompt),
-                prompt_token_ids=request_prompt,
-            )
-
-        engine_prompt = EngineTokensPrompt(prompt_token_ids=prompt_inputs["prompt_token_ids"])
-        if mm_data is not None:
-            engine_prompt["multi_modal_data"] = mm_data
-
-        if mm_uuids is not None:
-            engine_prompt["multi_modal_uuids"] = mm_uuids
-
-        if request.mm_processor_kwargs is not None:
-            engine_prompt["mm_processor_kwargs"] = request.mm_processor_kwargs
-
-        if hasattr(request, "cache_salt") and request.cache_salt is not None:
-            engine_prompt["cache_salt"] = request.cache_salt
-
-        return conversation, [request_prompt], [engine_prompt]
-
-    def _to_sampling_params_list(self, sampling_params_list: list[dict]) -> list[SamplingParams]:
-        final_sampling_params_list = []
-        for sampling_params in sampling_params_list:
-            if isinstance(sampling_params, dict):
-                final_sampling_params_list.append(SamplingParams(**sampling_params))
-            elif isinstance(sampling_params, SamplingParams):
-                final_sampling_params_list.append(sampling_params)
-            else:
-                raise ValueError(f"Invalid sampling params: {sampling_params}")
-        return final_sampling_params_list
-
-    def _log_inputs(
-        self,
-        request_id: str,
-        inputs: Union[RequestPrompt, PromptType],
-        params_list: Optional[list[SamplingParams]],
-        lora_request: Optional[LoRARequest],
-    ) -> None:
-        if self.request_logger is None:
-            return
-        prompt, prompt_token_ids, prompt_embeds = None, None, None
-        if isinstance(inputs, str):
-            prompt = inputs
-        elif isinstance(inputs, list):
-            prompt_token_ids = inputs
-        else:
-            prompt = getattr(inputs, "prompt", None)
-            prompt_token_ids = getattr(inputs, "prompt_token_ids", None)
-
-        logger.info(
-            "Received request %s: prompt: %r, params_list: %s, prompt_token_ids: %s, prompt_embeds shape: %s, lora_request: %s.",  # noqa: E501
-            request_id,
-            prompt,
-            params_list,
-            prompt_token_ids,
-            prompt_embeds.shape if prompt_embeds is not None else None,
-            lora_request,
-        )
-
-    async def chat_completion_full_generator(
-        self,
-        request: ChatCompletionRequest,
-        result_generator: AsyncIterator[RequestOutput],
-        request_id: str,
-        model_name: str,
-        conversation: list[ConversationMessage],
-        tokenizer: AnyTokenizer,
-        request_metadata: RequestResponseMetadata,
-    ) -> Union[ErrorResponse, ChatCompletionResponse]:
-        created_time = int(time.time())
-        final_res: Optional[RequestOutput] = None
-
-        final_outputs: list[OmniRequestOutput] = []
-        try:
-            async for res in result_generator:
-                final_outputs.append(res)
-        except asyncio.CancelledError:
-            return self.create_error_response("Client disconnected")
-        except ValueError as e:
-            # TODO: Use a vllm-specific Validation Error
-            return self.create_error_response(str(e))
-
-        assert final_outputs is not None
-
-        choices: list[ChatCompletionResponseChoice] = []
-
-        usage = UsageInfo(prompt_tokens=0, completion_tokens=0, total_tokens=0)
-        role = self.get_chat_request_role(request)
-        prompt_logprobs = None
-        prompt_token_ids = None
-        kv_transfer_params = None
-
-        for omni_outputs in final_outputs:
-            choices_data = []
-            if omni_outputs.final_output_type == "text":
-                (
-                    choices_data,
-                    usage,
-                    prompt_logprobs,
-                    prompt_token_ids,
-                    kv_transfer_params,
-                ) = self._create_text_choice(request, omni_outputs, tokenizer, conversation, role)
-            elif omni_outputs.final_output_type == "audio":
-                choices_data = self._create_audio_choice(omni_outputs, role)
-            else:
-                logger.warning(f"Unsupported final output type: {omni_outputs.final_output_type}")
-                continue
-            choices.extend(choices_data)
-
-        response = ChatCompletionResponse(
-            id=request_id,
-            created=created_time,
-            model=model_name,
-            choices=choices,
-            usage=usage,
-            prompt_logprobs=prompt_logprobs,
-            prompt_token_ids=prompt_token_ids,
-            kv_transfer_params=kv_transfer_params,
-        )
-
-        # Log complete response if output logging is enabled
-        if self.enable_log_outputs and self.request_logger:
-            for choice in choices:
-                output_text = ""
-                if choice.message.content:
-                    output_text = choice.message.content
-                elif choice.message.tool_calls:
-                    # For tool calls, log the function name and arguments
-                    tool_call_descriptions = []
-                    for tc in choice.message.tool_calls:
-                        if hasattr(tc.function, "name") and hasattr(tc.function, "arguments"):
-                            tool_call_descriptions.append(f"{tc.function.name}({tc.function.arguments})")
-                    tool_calls_str = ", ".join(tool_call_descriptions)
-                    output_text = f"[tool_calls: {tool_calls_str}]"
-
-                if output_text:
-                    # Get the corresponding output token IDs
-                    output_token_ids = None
-                    if choice.index < len(final_res.outputs):
-                        output_token_ids = final_res.outputs[choice.index].token_ids
-
-                    self.request_logger.log_outputs(
-                        request_id=request_id,
-                        outputs=output_text,
-                        output_token_ids=output_token_ids,
-                        finish_reason=choice.finish_reason,
-                        is_streaming=False,
-                        delta=False,
-                    )
-
-        return response
-
-    def _create_text_choice(
-        self,
-        request: ChatCompletionRequest,
-        omni_outputs: OmniRequestOutput,
-        tokenizer: AnyTokenizer,
-        conversation: list[ConversationMessage],
-        role: str,
-    ):
-        final_res = omni_outputs.request_output
-        if self.tool_call_id_type == "kimi_k2":
-            history_tool_call_cnt = get_history_tool_calls_cnt(conversation)
-        else:
-            history_tool_call_cnt = 0
-
-        choices: list[ChatCompletionResponseChoice] = []
-
-        for output in final_res.outputs:
-            token_ids = output.token_ids
-            out_logprobs = output.logprobs
-            tool_call_info = None
-
-            if request.logprobs and request.top_logprobs is not None:
-                assert out_logprobs is not None, "Did not output logprobs"
-                logprobs = self._create_chat_logprobs(
-                    token_ids=token_ids,
-                    top_logprobs=out_logprobs,
-                    num_output_top_logprobs=request.top_logprobs,
-                    tokenizer=tokenizer,
-                    return_as_token_id=request.return_tokens_as_token_ids,
-                )
-            else:
-                logprobs = None
-
-            if self.use_harmony:
-                reasoning_content, content, _ = parse_chat_output(token_ids)
-                if not request.include_reasoning:
-                    reasoning_content = None
-
-                if self.tool_parser is not None:
-                    tool_parser = self.tool_parser(tokenizer)
-                    # NOTE: We use token_ids for openai tool parser
-                    tool_call_info = tool_parser.extract_tool_calls(
-                        "",
-                        request=request,
-                        token_ids=token_ids,  # type: ignore
-                    )
-                    content = tool_call_info.content
-                    message = ChatMessage(
-                        role=role,
-                        reasoning_content=reasoning_content,
-                        content=content,
-                        tool_calls=tool_call_info.tool_calls,
-                    )
-                else:
-                    message = ChatMessage(
-                        role=role,
-                        reasoning_content=reasoning_content,
-                        content=content,
-                    )
-
-                choice_data = ChatCompletionResponseChoice(
-                    index=output.index,
-                    message=message,
-                    logprobs=logprobs,
-                    finish_reason=(
-                        "tool_calls"
-                        if (tool_call_info is not None and tool_call_info.tools_called)
-                        else (output.finish_reason if output.finish_reason else "stop")
-                    ),
-                    stop_reason=output.stop_reason,
-                )
-                choices.append(choice_data)
-                continue
-
-            if self.reasoning_parser:
-                try:
-                    reasoning_parser = self.reasoning_parser(tokenizer)
-                except RuntimeError as e:
-                    logger.exception("Error in reasoning parser creation.")
-                    return self.create_error_response(str(e))
-                # If the reasoning parser is enabled,
-                # tool calls are extracted exclusively from the content.
-                reasoning_content, content = reasoning_parser.extract_reasoning_content(output.text, request=request)
-                if not request.include_reasoning:
-                    reasoning_content = None
-            else:
-                reasoning_content = None
-                content = output.text
-
-            auto_tools_called = False
-            # if auto tools are not enabled, and a named tool choice using
-            #   outlines is not being used
-            if (not self.enable_auto_tools or not self.tool_parser) and (
-                not isinstance(request.tool_choice, ChatCompletionNamedToolChoiceParam)
-                and request.tool_choice != "required"
-            ):
-                message = ChatMessage(role=role, reasoning_content=reasoning_content, content=content)
-
-            # if the request uses tools and specified a tool choice
-            elif request.tool_choice and type(request.tool_choice) is ChatCompletionNamedToolChoiceParam:
-                tool_call_class = MistralToolCall if isinstance(tokenizer, MistralTokenizer) else ToolCall
-                message = ChatMessage(
-                    role=role,
-                    reasoning_content=reasoning_content,
-                    content="",
-                    tool_calls=[
-                        tool_call_class(
-                            function=FunctionCall(
-                                name=request.tool_choice.function.name,
-                                arguments=content,
-                            )
-                        )
-                    ],
-                )
-
-            elif request.tool_choice and request.tool_choice == "required":
-                tool_call_class = MistralToolCall if isinstance(tokenizer, MistralTokenizer) else ToolCall
-
-                # the fields of FunctionDefinition are a superset of the
-                # tool call outputs and can be used for parsing
-                assert content is not None
-                tool_calls = TypeAdapter(list[FunctionDefinition]).validate_json(content)
-                tool_call_ids = []
-                for tool_call in tool_calls:
-                    tool_call_ids.append(
-                        make_tool_call_id(
-                            id_type=self.tool_call_id_type,
-                            func_name=tool_call.name,
-                            idx=history_tool_call_cnt,
-                        )
-                    )
-                    history_tool_call_cnt += 1
-                message = ChatMessage(
-                    role=role,
-                    content="",
-                    tool_calls=[
-                        tool_call_class(
-                            id=tool_call_ids[i],
-                            function=FunctionCall(
-                                name=tool_call.name,
-                                arguments=json.dumps(tool_call.parameters, ensure_ascii=False),
-                            ),
-                        )
-                        for i, tool_call in enumerate(tool_calls)
-                    ],
-                    reasoning_content=reasoning_content,
-                )
-
-            # if the request doesn't use tool choice
-            # OR specifies to not use a tool
-            elif not request.tool_choice or request.tool_choice == "none":
-                message = ChatMessage(role=role, reasoning_content=reasoning_content, content=content)
-
-            # handle when there are tools and tool choice is auto
-            elif (
-                request.tools
-                and (request.tool_choice == "auto" or request.tool_choice is None)
-                and self.enable_auto_tools
-                and self.tool_parser
-            ):
-                try:
-                    tool_parser = self.tool_parser(tokenizer)
-                except RuntimeError as e:
-                    logger.exception("Error in tool parser creation.")
-                    return self.create_error_response(str(e))
-
-                tool_call_info = tool_parser.extract_tool_calls(content if content is not None else "", request=request)
-                # In the OpenAI API the finish_reason is "tools_called"
-                # if the tool choice is auto and the model produced a tool
-                # call. The same is not true for named function calls
-                auto_tools_called = tool_call_info.tools_called
-                if tool_call_info.tools_called:
-                    message = ChatMessage(
-                        role=role,
-                        reasoning_content=reasoning_content,
-                        content=tool_call_info.content,
-                        tool_calls=tool_call_info.tool_calls,
-                    )
-
-                else:
-                    # FOR NOW make it a chat message; we will have to detect
-                    # the type to make it later.
-                    ret_content = content
-
-                    # try to use content return from tool parser first,
-                    # tool parser may do some modify for the content.
-                    if tool_call_info.content and len(tool_call_info.content) > 0:
-                        ret_content = tool_call_info.content
-                    message = ChatMessage(
-                        role=role,
-                        reasoning_content=reasoning_content,
-                        content=ret_content,
-                    )
-
-            # undetermined case that is still important to handle
-            else:
-                logger.error(
-                    "Error in chat_completion_full_generator - cannot determine if tools should be extracted. "
-                    "Returning a standard chat completion."
-                )
-                message = ChatMessage(role=role, reasoning_content=reasoning_content, content=content)
-
-            choice_data = ChatCompletionResponseChoice(
-                index=output.index,
-                message=message,
-                logprobs=logprobs,
-                finish_reason=(
-                    "tool_calls" if auto_tools_called else output.finish_reason if output.finish_reason else "stop"
-                ),
-                stop_reason=output.stop_reason,
-                token_ids=(as_list(output.token_ids) if request.return_token_ids else None),
-            )
-            choices.append(choice_data)
-
-        if request.echo:
-            last_msg_content: Union[str, list[dict[str, str]]] = ""
-            if conversation and "content" in conversation[-1] and conversation[-1].get("role") == role:
-                last_msg_content = conversation[-1]["content"] or ""
-            if isinstance(last_msg_content, list):
-                last_msg_content = "\n".join(msg["text"] for msg in last_msg_content)
-
-            for choice in choices:
-                full_message = last_msg_content + (choice.message.content or "")
-                choice.message.content = full_message
-
-        assert final_res.prompt_token_ids is not None
-        num_prompt_tokens = len(final_res.prompt_token_ids)
-        if final_res.encoder_prompt_token_ids is not None:
-            num_prompt_tokens += len(final_res.encoder_prompt_token_ids)
-        num_generated_tokens = sum(len(output.token_ids) for output in final_res.outputs)
-        usage = UsageInfo(
-            prompt_tokens=num_prompt_tokens,
-            completion_tokens=num_generated_tokens,
-            total_tokens=num_prompt_tokens + num_generated_tokens,
-        )
-        if self.enable_prompt_tokens_details and final_res.num_cached_tokens:
-            usage.prompt_tokens_details = PromptTokenUsageInfo(cached_tokens=final_res.num_cached_tokens)
-
-        prompt_logprobs = clamp_prompt_logprobs(final_res.prompt_logprobs)
-        prompt_token_ids = final_res.prompt_token_ids if request.return_token_ids else None
-        kv_transfer_params = final_res.kv_transfer_params
-
-        return choices, usage, prompt_logprobs, prompt_token_ids, kv_transfer_params
-
-    def _create_audio_choice(self, omni_outputs: OmniRequestOutput, role: str):
-        choices: list[ChatCompletionResponseChoice] = []
-        final_res = omni_outputs.request_output
-        audio_tensor = final_res.multimodal_output["audio"].detach().cpu().numpy()
-
-        # Convert numpy array to WAV bytes and encode as base64
-        if soundfile is None:
-            raise ImportError(
-                "soundfile is required for audio generation. Please install it with: pip install soundfile"
-            )
-
-        # Default sample rate for TTS models (typically 24000 Hz)
-        # You may need to adjust this based on your model's configuration
-        sample_rate = 24000
-
-        # Ensure audio is 1D (flatten if needed)
-        if audio_tensor.ndim > 1:
-            audio_tensor = audio_tensor.flatten()
-
-        # Convert to WAV format and encode as base64
-        with BytesIO() as buffer:
-            soundfile.write(buffer, audio_tensor, sample_rate, format="WAV")
-            wav_bytes = buffer.getvalue()
-
-        audio_base64 = base64.b64encode(wav_bytes).decode("utf-8")
-
-        # Generate unique ID for the audio
-        audio_id = f"audio-{uuid.uuid4().hex[:16]}"
-
-        # Set expiration time (e.g., 24 hours from now) as Unix timestamp
-        expires_at = int((datetime.now(timezone.utc) + timedelta(hours=24)).timestamp())
-
-        # Create OpenAIChatCompletionAudio object with all required fields
-        audio_obj = OpenAIChatCompletionAudio(
-            id=audio_id,
-            data=audio_base64,
-            expires_at=expires_at,
-            transcript="",  # Empty transcript if not available
-        )
-
-        for output in final_res.outputs:
-            choice_data = ChatCompletionResponseChoice(
-                index=output.index,
-                message=ChatMessage(role=role, audio=audio_obj),
-                logprobs=None,
-                finish_reason="stop",
-                stop_reason=None,
-            )
-            choices.append(choice_data)
-        return choices
@@ -1,271 +0,0 @@
-from __future__ import annotations
-
-import json
-import logging
-import os
-import pickle
-from multiprocessing import shared_memory as _shm
-from typing import Any
-
-import cloudpickle
-from omegaconf import OmegaConf
-
-logger = logging.getLogger(__name__)
-
-
-def set_stage_gpu_devices(stage_id: int, devices: str | int | None) -> None:
-    """Configure per-stage CUDA visibility and current device.
-
-    Behavior
-    - Comma-separated string (e.g. "2,5,7"): set CUDA_VISIBLE_DEVICES exactly
-      to this list; logical index 0 is used as current device.
-    - Integer or digit-string: treat as logical index (0-based) into the current
-      CUDA_VISIBLE_DEVICES mapping; map to the physical device, and then set
-      CUDA_VISIBLE_DEVICES to this single device.
-    - None/"cpu": keep default visibility.
-    - Otherwise: set CUDA_VISIBLE_DEVICES to the provided single device string.
-    """
-    try:
-        selected_physical: int | None = None
-        logical_idx: int | None = None
-
-        if isinstance(devices, str) and "," in devices:
-            os.environ["CUDA_VISIBLE_DEVICES"] = devices
-            toks = [t.strip() for t in devices.split(",") if t.strip() != ""]
-            if toks:
-                try:
-                    selected_physical = int(toks[0])
-                    logger.debug(
-                        "[Stage-%s] Set CUDA_VISIBLE_DEVICES to %s; logical 0 -> physical %s",
-                        stage_id,
-                        devices,
-                        selected_physical,
-                    )
-                except Exception as e:
-                    logger.debug("[Stage-%s] Failed to parse first CUDA device: %s", stage_id, e)
-                    selected_physical = None
-        elif isinstance(devices, (int, str)) and (isinstance(devices, int) or str(devices).isdigit()):
-            logical_idx = max(0, int(devices))
-            vis = os.environ.get("CUDA_VISIBLE_DEVICES")
-            if vis:
-                try:
-                    mapping = [int(x) for x in vis.split(",") if x.strip() != ""]
-                    if 0 <= logical_idx < len(mapping):
-                        selected_physical = mapping[logical_idx]
-                except Exception as e:
-                    logger.debug(
-                        "[Stage-%s] Failed to map logical index via CUDA_VISIBLE_DEVICES: %s",
-                        stage_id,
-                        e,
-                    )
-                    selected_physical = None
-            if selected_physical is None:
-                selected_physical = int(logical_idx)
-            os.environ["CUDA_VISIBLE_DEVICES"] = str(selected_physical)
-            logger.debug(
-                "[Stage-%s] Logical index %d -> physical %s; set CUDA_VISIBLE_DEVICES to single device",
-                stage_id,
-                logical_idx + 1,
-                selected_physical,
-            )
-        elif devices in (None, "cpu"):
-            logger.debug(
-                "[Stage-%s] Using default device visibility (devices=%s)",
-                stage_id,
-                devices,
-            )
-        else:
-            selected_physical = int(str(devices))
-            os.environ["CUDA_VISIBLE_DEVICES"] = str(selected_physical)
-            logger.debug(
-                "[Stage-%s] Set CUDA_VISIBLE_DEVICES to single device %s (fallback)",
-                stage_id,
-                selected_physical,
-            )
-
-        try:
-            import torch  # noqa: WPS433
-
-            if torch.cuda.is_available():
-                try:
-                    torch.cuda.set_device(0)
-                except Exception as e:
-                    logger.debug(
-                        "[Stage-%s] torch.cuda.set_device(0) failed: %s",
-                        stage_id,
-                        e,
-                        exc_info=True,
-                    )
-                num = torch.cuda.device_count()
-                info = []
-                for i in range(num):
-                    total = torch.cuda.get_device_properties(i).total_memory
-                    free, _ = torch.cuda.mem_get_info(i)
-                    info.append(
-                        {
-                            "idx": i,
-                            "name": torch.cuda.get_device_name(i),
-                            "total": int(total),
-                            "free": int(free),
-                        }
-                    )
-                logger.debug("[Stage-%s] CUDA devices visible=%s info=%s", stage_id, num, info)
-        except Exception as e:
-            logger.debug(
-                "[Stage-%s] Failed to query CUDA devices: %s",
-                stage_id,
-                e,
-                exc_info=True,
-            )
-    except Exception as e:
-        logger.warning("Failed to interpret devices for stage %s: %s", stage_id, e)
-
-
-def serialize_obj(obj: Any) -> bytes:
-    """Serialize a Python object to bytes using cloudpickle."""
-    return cloudpickle.dumps(obj)
-
-
-def shm_write_bytes(payload: bytes) -> dict[str, Any]:
-    """Write bytes into SharedMemory and return meta dict {name,size}.
-
-    Caller should close the segment; the receiver should unlink.
-    """
-    shm = _shm.SharedMemory(create=True, size=len(payload))
-    mv = memoryview(shm.buf)
-    mv[: len(payload)] = payload
-    del mv
-    meta = {"name": shm.name, "size": len(payload)}
-    try:
-        shm.close()
-    except Exception as e:
-        logger.debug("Failed to close shared memory: %s", e)
-    return meta
-
-
-def shm_read_bytes(meta: dict[str, Any]) -> bytes:
-    """Read bytes from SharedMemory by meta {name,size} and cleanup."""
-    shm = _shm.SharedMemory(name=meta["name"])  # type: ignore[index]
-    mv = memoryview(shm.buf)
-    data = bytes(mv[: meta["size"]])
-    del mv
-    try:
-        shm.close()
-    except Exception:
-        pass
-    try:
-        shm.unlink()
-    except Exception:
-        pass
-    return data
-
-
-def _ensure_parent_dir(path: str) -> None:
-    """Ensure the parent directory for a file path exists (best-effort)."""
-    try:
-        parent = os.path.dirname(path)
-        if parent:
-            os.makedirs(parent, exist_ok=True)
-    except Exception:
-        pass
-
-
-def append_jsonl(path: str, record: dict[str, Any]) -> None:
-    """Append a JSON record as one line to a JSONL file (best-effort).
-
-    This is safe to call from multiple processes when each process writes
-    to a distinct file. For concurrent writes to the same file, OS append
-    semantics typically suffice, but no additional locking is provided.
-    """
-    try:
-        _ensure_parent_dir(path)
-        line = json.dumps(record, ensure_ascii=False)
-        fd = os.open(path, os.O_APPEND | os.O_CREAT | os.O_WRONLY, 0o644)
-        with os.fdopen(fd, "a", encoding="utf-8") as f:
-            f.write(line + "\n")
-    except Exception:
-        logger.exception("Failed to append JSONL to %s", path)
-
-
-def maybe_dump_to_shm(obj: Any, threshold: int) -> tuple[bool, Any]:
-    """Dump object to SHM if serialized size exceeds threshold.
-
-    Returns (True, meta) when dumped; otherwise (False, original_obj).
-    """
-    payload = serialize_obj(obj)
-    if len(payload) > threshold:
-        return True, shm_write_bytes(payload)
-    return False, obj
-
-
-def maybe_load_from_ipc(container: dict[str, Any], obj_key: str, shm_key: str) -> Any:
-    """Load object from container that may carry SHM or inline object.
-
-    Deprecated: prefer `maybe_load_from_ipc_with_metrics` to also obtain
-    decode-time and size metrics.
-    """
-    if shm_key in container:
-        return pickle.loads(shm_read_bytes(container[shm_key]))
-    return container[obj_key]
-
-
-def maybe_load_from_ipc_with_metrics(
-    container: dict[str, Any], obj_key: str, shm_key: str
-) -> tuple[Any, dict[str, float]]:
-    """Load object and return (object, metrics) with RX bytes and decode time.
-
-    Metrics keys:
-      - rx_transfer_bytes: int
-      - rx_decode_time_ms: float
-    """
-    import time as _time  # local import to avoid overhead at module import
-
-    t0 = _time.time()
-    if shm_key in container:
-        meta = container[shm_key]  # type: ignore[index]
-        payload = shm_read_bytes(meta)
-        obj = pickle.loads(payload)
-        try:
-            rx_bytes = int(meta.get("size", len(payload)))  # type: ignore[call-arg]
-        except Exception:
-            rx_bytes = len(payload)
-    else:
-        obj = container[obj_key]
-        try:
-            rx_bytes = len(serialize_obj(obj))
-        except Exception:
-            rx_bytes = 0
-    t1 = _time.time()
-    rx_decode_ms = (t1 - t0) * 1000.0
-    return obj, {
-        "rx_transfer_bytes": int(rx_bytes),
-        "rx_decode_time_ms": float(rx_decode_ms),
-    }
-
-
-def encode_for_ipc(obj: Any, threshold: int, obj_key: str, shm_key: str) -> dict[str, Any]:
-    """Return a dict payload for IPC: inline (obj_key) or SHM (shm_key).
-
-    When serialized size exceeds threshold, returns {shm_key: {name,size}};
-    otherwise returns {obj_key: obj}.
-    """
-    payload: dict[str, Any] = {}
-    use_shm, data = maybe_dump_to_shm(obj, threshold)
-    if use_shm:
-        payload[shm_key] = data
-    else:
-        payload[obj_key] = data
-    return payload
-
-
-# Convert OmegaConf/objects to plain dicts
-def _to_dict(x: Any) -> dict[str, Any]:
-    try:
-        if isinstance(x, dict):
-            return dict(x)
-        return OmegaConf.to_container(x, resolve=True)  # type: ignore[arg-type]
-    except Exception:
-        try:
-            return dict(x)
-        except Exception:
-            return {}
@@ -1,14 +1,8 @@
-from __future__ import annotations
-
-import logging
 import os
 from pathlib import Path
-
 from omegaconf import OmegaConf
 from vllm.transformers_utils.config import get_config

-logger = logging.getLogger(__name__)
-
 # Get the project root directory (2 levels up from this file)
 PROJECT_ROOT = Path(__file__).parent.parent.parent

@@ -28,4 +22,4 @@ def load_stage_configs_from_model(model: str):
 def load_stage_configs_from_yaml(config_path: str):
    """Load stage configs from yaml file."""
    config_data = OmegaConf.load(config_path)
-    return config_data.stage_args
+    return config_data.stage_args
@@ -1,13 +1,6 @@
-from typing import Any, Optional
-
-try:
-    from typing import NotRequired
-except ImportError:
-    # Python < 3.11: use typing_extensions
-    from typing_extensions import NotRequired
-
+from vllm.inputs.data import TokensPrompt
+from typing import Any, NotRequired
 import torch
-from vllm.inputs.data import EmbedsPrompt, TokenInputs, TokensPrompt


 class OmniTokensPrompt(TokensPrompt):
@@ -16,44 +9,4 @@ class OmniTokensPrompt(TokensPrompt):

    # New: optional additional information dictionary
    # Values may be torch.Tensor or list
-    additional_information: NotRequired[dict[str, Any]]
-
-
-class OmniTokenInputs(TokenInputs):
-    # New: optional prompt embeddings aligned with token ids
-    prompt_embeds: NotRequired[torch.Tensor]
-
-    # New: optional additional information dictionary
-    # Values may be torch.Tensor or list
-    additional_information: NotRequired[dict[str, Any]]
-
-
-class OmniEmbedsPrompt(EmbedsPrompt):
-    # New: optional prompt embeddings aligned with token ids
-    prompt_embeds: NotRequired[torch.Tensor]
-
-    # New: optional additional information dictionary
-    # Values may be torch.Tensor or list
-    additional_information: NotRequired[dict[str, Any]]
-
-
-def token_inputs_omni(
-    prompt_token_ids: list[int],
-    prompt: Optional[str] = None,
-    cache_salt: Optional[str] = None,
-    prompt_embeds: Optional[torch.Tensor] = None,
-    additional_information: Optional[dict[str, Any]] = None,
-) -> OmniTokenInputs:
-    """Construct token inputs with optional embeddings and metadata."""
-    inputs = OmniTokenInputs(type="token", prompt_token_ids=prompt_token_ids)
-
-    if prompt is not None:
-        inputs["prompt"] = prompt
-    if cache_salt is not None:
-        inputs["cache_salt"] = cache_salt
-    if prompt_embeds is not None:
-        inputs["prompt_embeds"] = prompt_embeds
-    if additional_information is not None:
-        inputs["additional_information"] = additional_information
-
-    return inputs
+    additional_information: NotRequired[dict[str, Any]]
@@ -1,25 +0,0 @@
-from vllm.inputs.data import SingletonPrompt
-from vllm.inputs.parse import (
-    ParsedEmbedsPrompt,
-    ParsedSingletonPrompt,
-    ParsedStrPrompt,
-    ParsedTextPrompt,
-    ParsedTokensPrompt,
-)
-
-
-def parse_singleton_prompt_omni(prompt: SingletonPrompt) -> ParsedSingletonPrompt:
-    if isinstance(prompt, str):
-        return ParsedStrPrompt(type="str", content=prompt)
-    elif isinstance(prompt, dict):
-        # Type ignores are because mypy does not correctly infer the TypedDicts
-        # Pyright does succeed.
-        # Priority tokens: When both tokens and embeds exist, keep both and
-        # follow the tokens path
-        if "prompt_token_ids" in prompt:
-            return ParsedTokensPrompt(type="tokens", content=prompt)  # type: ignore[typeddict-item]
-        elif "prompt_embeds" in prompt:
-            return ParsedEmbedsPrompt(type="embeds", content=prompt)  # type: ignore[typeddict-item]
-        elif "prompt" in prompt:
-            return ParsedTextPrompt(type="text", content=prompt)
-    raise TypeError("inputs must be a string, TextPrompt, TokensPrompt, or EmbedsPrompt")