update news

Signed-off-by: Roger Wang <hey@rogerw.io>

.buildkite/pipeline.yml

@@ -18,83 +18,18 @@ steps:
   #     queue: "ascend"
 
   - label: "Simple Unit Test"
-    depends_on: image-build
+    depends_on: ~
     commands:
-      - pytest -v -s tests/entrypoints/
-      - pytest -v -s tests/diffusion/cache/
-      - pytest -v -s tests/diffusion/lora/
-      - pytest -v -s tests/model_executor/models/qwen2_5_omni/test_audio_length.py
-      - pytest -v -s tests/worker/
-      - pytest -v -s tests/distributed/omni_connectors/test_kv_flow.py
+      - ".buildkite/scripts/simple_test.sh"
     agents:
-      queue: "gpu_1_queue"
-    plugins:
-      - docker#v5.2.0:
-          image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-          always-pull: true
-          propagate-environment: true
-          environment:
-            - "HF_HOME=/fsx/hf_cache"
-          volumes:
-            - "/fsx/hf_cache:/fsx/hf_cache"
+      queue: "cpu_queue_premerge"
 
   - label: "Diffusion Model Test"
     timeout_in_minutes: 20
     depends_on: image-build
     commands:
       - pytest -s -v tests/e2e/offline_inference/test_t2i_model.py
-    agents:
-      queue: "gpu_1_queue" # g6.4xlarge instance on AWS, has 1 L4 GPU
-    plugins:
-      - docker#v5.2.0:
-          image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-          always-pull: true
-          propagate-environment: true
-          environment:
-            - "HF_HOME=/fsx/hf_cache"
-          volumes:
-            - "/fsx/hf_cache:/fsx/hf_cache"
-
-  - label: "Diffusion Images API LoRA E2E"
-    timeout_in_minutes: 20
-    depends_on: image-build
-    commands:
-      - pytest -s -v tests/e2e/online_serving/test_images_generations_lora.py
-    agents:
-      queue: "gpu_1_queue" # g6.4xlarge instance on AWS, has 1 L4 GPU
-    plugins:
-      - docker#v5.2.0:
-          image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-          always-pull: true
-          propagate-environment: true
-          environment:
-            - "HF_HOME=/fsx/hf_cache"
-          volumes:
-            - "/fsx/hf_cache:/fsx/hf_cache"
-
-  - label: "Diffusion Model CPU offloading Test"
-    timeout_in_minutes: 20
-    depends_on: image-build
-    commands:
-      - pytest -s -v tests/e2e/offline_inference/test_diffusion_cpu_offload.py
-      - pytest -s -v tests/e2e/offline_inference/test_diffusion_layerwise_offload.py
-    agents:
-      queue: "gpu_1_queue" # g6.4xlarge instance on AWS, has 1 L4 GPU
-    plugins:
-      - docker#v5.2.0:
-          image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-          always-pull: true
-          propagate-environment: true
-          environment:
-            - "HF_HOME=/fsx/hf_cache"
-          volumes:
-            - "/fsx/hf_cache:/fsx/hf_cache"
-
-  - label: "Audio Generation Model Test"
-    timeout_in_minutes: 20
-    depends_on: image-build
-    commands:
-      - pytest -s -v tests/e2e/offline_inference/test_stable_audio_model.py
+      - pytest -s -v tests/e2e/offline_inference/test_ovis_image.py
     agents:
       queue: "gpu_1_queue" # g6.4xlarge instance on AWS, has 1 L4 GPU
     plugins:
@@ -124,8 +59,8 @@ steps:
           volumes:
             - "/fsx/hf_cache:/fsx/hf_cache"
 
-  - label: "Diffusion Sequence Parallelism Test"
-    timeout_in_minutes: 20
+  - label: "Diffusion Parallelism Test"
+    timeout_in_minutes: 15
     depends_on: image-build
     commands:
       - pytest -s -v tests/e2e/offline_inference/test_sequence_parallel.py
@@ -142,29 +77,11 @@ steps:
           volumes:
             - "/fsx/hf_cache:/fsx/hf_cache"
 
-  - label: "Diffusion Tensor Parallelism Test"
-    timeout_in_minutes: 20
-    depends_on: image-build
-    commands:
-      - pytest -s -v tests/e2e/offline_inference/test_zimage_tensor_parallel.py
-    agents:
-      queue: "gpu_4_queue" # g6.12xlarge instance on AWS, has 4 L4 GPU
-    plugins:
-      - docker#v5.2.0:
-          image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-          always-pull: true
-          propagate-environment: true
-          shm-size: "8gb"
-          environment:
-            - "HF_HOME=/fsx/hf_cache"
-          volumes:
-            - "/fsx/hf_cache:/fsx/hf_cache"
-
   - label: "Diffusion GPU Worker Test"
     timeout_in_minutes: 20
     depends_on: image-build
     commands:
-      - pytest -s -v tests/diffusion/test_diffusion_worker.py
+      - pytest -s -v tests/diffusion/test_gpu_worker.py
     agents:
       queue: "gpu_4_queue" # g6.12xlarge instance on AWS, has 4 L4 GPU
     plugins:
@@ -178,13 +95,31 @@ steps:
           volumes:
             - "/fsx/hf_cache:/fsx/hf_cache"
 
-
-  - label: "Benchmark Test"
+  - label: "Omni Model Test"
     timeout_in_minutes: 15
     depends_on: image-build
     commands:
+      - export VLLM_LOGGING_LEVEL=DEBUG
       - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-      - pytest -s -v tests/benchmarks/test_serve_cli.py
+      - pytest -s -v tests/e2e/offline_inference/test_qwen2_5_omni.py
+    agents:
+      queue: "gpu_4_queue" # g6.12xlarge instance on AWS, has 4 L4 GPU
+    plugins:
+      - docker#v5.2.0:
+          image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
+          always-pull: true
+          propagate-environment: true
+          environment:
+            - "HF_HOME=/fsx/hf_cache"
+          volumes:
+            - "/fsx/hf_cache:/fsx/hf_cache"
+
+  - label: "Omni Model Test with H100"
+    timeout_in_minutes: 30
+    depends_on: image-build
+    commands:
+      - export VLLM_WORKER_MULTIPROC_METHOD=spawn
+      - pytest -s -v tests/e2e/offline_inference/test_qwen3_omni.py tests/e2e/online_serving/test_qwen3_omni.py tests/e2e/online_serving/test_async_omni.py
     agents:
       queue: "mithril-h100-pool"
     plugins:
@@ -214,69 +149,12 @@ steps:
                   path: /mnt/hf-cache
                   type: DirectoryOrCreate
 
-  - label: "Omni Model Test"
-    timeout_in_minutes: 15
-    depends_on: image-build
-    commands:
-    - export VLLM_LOGGING_LEVEL=DEBUG
-    - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-    - pytest -s -v tests/e2e/offline_inference/test_qwen2_5_omni.py
-    agents:
-      queue: "gpu_4_queue" # g6.12xlarge instance on AWS, has 4 L4 GPU
-    plugins:
-    - docker#v5.2.0:
-        image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-        always-pull: true
-        propagate-environment: true
-        environment:
-        - "HF_HOME=/fsx/hf_cache"
-        volumes:
-        - "/fsx/hf_cache:/fsx/hf_cache"
-
-  # - label: "Omni Model Test with H100"
-  #   timeout_in_minutes: 30
-  #   depends_on: image-build
-  #   commands:
-  #     - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-  #     - export VLLM_TEST_CLEAN_GPU_MEMORY="1"
-  #     - pytest -s -v tests/e2e/offline_inference/test_qwen3_omni.py
-  #     - pytest -s -v tests/e2e/online_serving/test_qwen3_omni.py
-  #     - pytest -s -v tests/e2e/online_serving/test_async_omni.py
-  #   agents:
-  #     queue: "mithril-h100-pool"
-  #   plugins:
-  #     - kubernetes:
-  #         podSpec:
-  #           containers:
-  #             - image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-  #               resources:
-  #                 limits:
-  #                   nvidia.com/gpu: 2
-  #               volumeMounts:
-  #                 - name: devshm
-  #                   mountPath: /dev/shm
-  #                 - name: hf-cache
-  #                   mountPath: /root/.cache/huggingface
-  #               env:
-  #                 - name: HF_HOME
-  #                   value: /root/.cache/huggingface
-  #           nodeSelector:
-  #             node.kubernetes.io/instance-type: gpu-h100-sxm
-  #           volumes:
-  #             - name: devshm
-  #               emptyDir:
-  #                 medium: Memory
-  #             - name: hf-cache
-  #               hostPath:
-  #                 path: /mnt/hf-cache
-  #                 type: DirectoryOrCreate
-
   - label: "Diffusion Image Edit Test with H100 (1 GPU)"
     timeout_in_minutes: 20
     depends_on: image-build
     commands:
       - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-      - pytest -s -v tests/e2e/online_serving/test_image_gen_edit.py
+      - pytest -s -v tests/e2e/online_serving/test_i2i_multi_image_input.py
     agents:
       queue: "mithril-h100-pool"
     plugins:
@@ -305,38 +183,3 @@ steps:
                 hostPath:
                   path: /mnt/hf-cache
                   type: DirectoryOrCreate
-
-  # - label: "Bagel Text2Img Model Test with H100"
-  #   timeout_in_minutes: 30
-  #   depends_on: image-build
-  #   commands:
-  #     - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-  #     - pytest -s -v tests/e2e/offline_inference/test_bagel_text2img.py
-  #   agents:
-  #     queue: "mithril-h100-pool"
-  #   plugins:
-  #     - kubernetes:
-  #         podSpec:
-  #           containers:
-  #             - image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-  #               resources:
-  #                 limits:
-  #                   nvidia.com/gpu: 1
-  #               volumeMounts:
-  #                 - name: devshm
-  #                   mountPath: /dev/shm
-  #                 - name: hf-cache
-  #                   mountPath: /root/.cache/huggingface
-  #               env:
-  #                 - name: HF_HOME
-  #                   value: /root/.cache/huggingface
-  #           nodeSelector:
-  #             node.kubernetes.io/instance-type: gpu-h100-sxm
-  #           volumes:
-  #             - name: devshm
-  #               emptyDir:
-  #                 medium: Memory
-  #             - name: hf-cache
-  #               hostPath:
-  #                 path: /mnt/hf-cache
-  #                 type: DirectoryOrCreate

.buildkite/scripts/docker_login_ecr_public.sh

@@ -1,73 +0,0 @@
-#!/bin/bash
-# Helper function to safely login to ECR Public with per-job config isolation
-# Uses DOCKER_CONFIG environment variable to prevent race conditions
-#
-# This script prevents the "device or resource busy" error by giving each
-# Buildkite job its own isolated Docker config directory.
-#
-# Usage:
-#   source docker_login_ecr_public.sh && safe_docker_login_ecr_public
-
-set -euo pipefail
-
-# Configuration
-ECR_REGISTRY="public.ecr.aws"
-
-setup_isolated_docker_config() {
-    # Use BUILDKITE_JOB_ID for job-specific isolation
-    # Fallback to PID if running outside Buildkite
-    local job_id="${BUILDKITE_JOB_ID:-$$}"
-
-    # Set Docker config to job-specific directory
-    export DOCKER_CONFIG="/tmp/docker-config-${job_id}"
-
-    # Create directory if it doesn't exist
-    mkdir -p "$DOCKER_CONFIG"
-
-    echo "[docker-config] Using isolated Docker config: $DOCKER_CONFIG"
-}
-
-check_docker_auth() {
-    # Check if already authenticated to the given registry
-    # Returns 0 if authenticated, 1 if not
-    local registry="$1"
-
-    # Check if credentials exist in the isolated config
-    if [[ -f "$DOCKER_CONFIG/config.json" ]]; then
-        # Check if registry is present in config
-        if grep -q "$registry" "$DOCKER_CONFIG/config.json" 2>/dev/null; then
-            return 0
-        fi
-    fi
-
-    return 1
-}
-
-safe_docker_login_ecr_public() {
-    # Setup isolated config first
-    setup_isolated_docker_config
-
-    local registry="$ECR_REGISTRY"
-
-    # Check if already authenticated (within this job)
-    if check_docker_auth "$registry"; then
-        echo "[docker-login] Already authenticated to $registry in this job"
-        return 0
-    fi
-
-    # Perform login to isolated config directory
-    echo "[docker-login] Logging in to $ECR_REGISTRY (isolated config)..."
-    if aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin "$ECR_REGISTRY"; then
-        echo "[docker-login] Login successful (config: $DOCKER_CONFIG)"
-        return 0
-    else
-        local exit_code=$?
-        echo "[docker-login] ERROR: Login failed with exit code $exit_code" >&2
-        return $exit_code
-    fi
-}
-
-# Execute if run as script (not sourced)
-if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then
-    safe_docker_login_ecr_public
-fi

.buildkite/scripts/hardware_ci/run-amd-test.sh

@@ -66,14 +66,6 @@ done
 echo "--- Pulling container"
 image_name="public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:${BUILDKITE_COMMIT}-rocm-omni"
 container_name="rocm_${BUILDKITE_COMMIT}_$(tr -dc A-Za-z0-9 < /dev/urandom | head -c 10; echo)"
-
-# Install AWS CLI to authenticate to ECR Public Gallery to get higher rate limit for pulling images
-sudo apt-get update && sudo apt-get install -y awscli
-# Use safe docker login helper to prevent race conditions
-source "$(dirname "${BASH_SOURCE[0]}")/../docker_login_ecr_public.sh"
-safe_docker_login_ecr_public
-# Pull the container from ECR Public Gallery
-
 docker pull "${image_name}"
 
 remove_docker_container() {
@@ -116,9 +108,6 @@ if [[ $commands == *"--shard-id="* ]]; then
         --shm-size=16gb \
         --group-add "$render_gid" \
         --rm \
-        -e MIOPEN_DEBUG_CONV_DIRECT=0 \
-        -e MIOPEN_DEBUG_CONV_GEMM=0 \
-        -e VLLM_ROCM_USE_AITER=1 \
         -e HIP_VISIBLE_DEVICES="${GPU}" \
         -e HF_TOKEN \
         -e AWS_ACCESS_KEY_ID \
@@ -151,9 +140,6 @@ else
           --shm-size=16gb \
           --group-add "$render_gid" \
           --rm \
-          -e MIOPEN_DEBUG_CONV_DIRECT=0 \
-          -e MIOPEN_DEBUG_CONV_GEMM=0 \
-          -e VLLM_ROCM_USE_AITER=1 \
           -e HF_TOKEN \
           -e AWS_ACCESS_KEY_ID \
           -e AWS_SECRET_ACCESS_KEY \

.buildkite/scripts/simple_test.sh

@@ -0,0 +1,54 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="$(cd -- "${SCRIPT_DIR}/../.." && pwd)"
+UV_BIN="uv"
+PYTHON_BOOTSTRAP="${PYTHON:-python3}"
+VENV_DIR="${VENV_DIR:-${REPO_ROOT}/.venv-simple-test}"
+TARGET_PY_VERSION="3.12"
+
+if ! command -v "${PYTHON_BOOTSTRAP}" >/dev/null 2>&1; then
+  PYTHON_BOOTSTRAP="python"
+fi
+
+cd "${REPO_ROOT}"
+
+# Ensure pip-installed scripts (including uv) are on PATH
+PYTHON_SCRIPTS="$(${PYTHON_BOOTSTRAP} -c 'import sysconfig; print(sysconfig.get_path("scripts"))')"
+USER_BASE="$(${PYTHON_BOOTSTRAP} -m site --user-base 2>/dev/null || true)"
+PATH="${PYTHON_SCRIPTS}:${PATH}"
+if [[ -n "${USER_BASE}" ]]; then
+  PATH="${USER_BASE}/bin:${PATH}"
+fi
+
+if ! command -v "${UV_BIN}" >/dev/null 2>&1; then
+  "${PYTHON_BOOTSTRAP}" -m pip install --upgrade pip
+  "${PYTHON_BOOTSTRAP}" -m pip install uv
+  hash -r
+fi
+
+TARGET_PY_PATH="$(${UV_BIN} python find "${TARGET_PY_VERSION}" 2>/dev/null | head -n1 || true)"
+if [[ -z "${TARGET_PY_PATH}" ]]; then
+  "${UV_BIN}" python install "${TARGET_PY_VERSION}"
+  TARGET_PY_PATH="$(${UV_BIN} python find "${TARGET_PY_VERSION}" 2>/dev/null | head -n1 || true)"
+fi
+
+if [[ -z "${TARGET_PY_PATH}" ]]; then
+  echo "Failed to locate python ${TARGET_PY_VERSION}" >&2
+  exit 1
+fi
+
+if [[ ! -d "${VENV_DIR}" ]]; then
+  "${UV_BIN}" venv --python "${TARGET_PY_PATH}" "${VENV_DIR}"
+fi
+
+VENV_PYTHON="${VENV_DIR}/bin/python"
+[[ -x "${VENV_PYTHON}" ]] || { echo "Python not found in ${VENV_DIR}"; exit 1; }
+
+"${UV_BIN}" pip install --python "${VENV_PYTHON}" vllm==0.12.0
+"${UV_BIN}" pip install --python "${VENV_PYTHON}" -e ".[dev]"
+"${VENV_PYTHON}" -m pytest -v -s tests/entrypoints/
+"${VENV_PYTHON}" -m pytest -v -s tests/diffusion/cache/
+"${VENV_PYTHON}" -m pytest -v -s tests/model_executor/models/qwen2_5_omni/test_audio_length.py

.buildkite/test-amd.yaml

@@ -1,116 +1,53 @@
 steps:
 
 - label: "Diffusion Model Test"
-  timeout_in_minutes: 20
+  timeout_in_minutes: 15
   agent_pool: mi325_2
   depends_on: amd-build
-  mirror_hardwares: [amdproduction]
+  mirror_hardwares: [amdexperimental, amdproduction, amdtentative]
   grade: Blocking
   commands:
     - export GPU_ARCHS=gfx942
+    - export MIOPEN_DEBUG_CONV_DIRECT=0
+    - export MIOPEN_DEBUG_CONV_GEMM=0
+    - export VLLM_ROCM_USE_AITER=1
+    - export VLLM_ROCM_USE_AITER_MHA=1
+    - export VLLM_ROCM_USE_AITER_LINEAR=0
+    - export VLLM_ROCM_USE_AITER_RMSNORM=0
     - pytest -s -v tests/e2e/offline_inference/test_t2i_model.py
 
-- label: "Diffusion Images API LoRA E2E"
-  timeout_in_minutes: 20
-  agent_pool: mi325_1
-  depends_on: amd-build
-  mirror_hardwares: [amdproduction]
-  grade: Blocking
-  commands:
-    - export GPU_ARCHS=gfx942
-    - export VLLM_LOGGING_LEVEL=DEBUG
-    - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-    - pytest -s -v tests/e2e/online_serving/test_images_generations_lora.py
-
-- label: "Diffusion Model CPU offloading Test"
-  timeout_in_minutes: 20
-  agent_pool: mi325_1
-  depends_on: amd-build
-  mirror_hardwares: [amdproduction]
-  grade: Blocking
-  commands:
-    - export GPU_ARCHS=gfx942
-    - export VLLM_LOGGING_LEVEL=DEBUG
-    - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-    - pytest -s -v tests/e2e/offline_inference/test_diffusion_cpu_offload.py
-
 - label: "Diffusion Cache Backend Test"
   timeout_in_minutes: 15
   agent_pool: mi325_1
   depends_on: amd-build
-  mirror_hardwares: [amdproduction]
+  mirror_hardwares: [amdexperimental, amdproduction, amdtentative]
   grade: Blocking
   commands:
     - export GPU_ARCHS=gfx942
     - export VLLM_LOGGING_LEVEL=DEBUG
     - export VLLM_WORKER_MULTIPROC_METHOD=spawn
+    - export MIOPEN_DEBUG_CONV_DIRECT=0
+    - export MIOPEN_DEBUG_CONV_GEMM=0
+    - export VLLM_ROCM_USE_AITER=1
+    - export VLLM_ROCM_USE_AITER_MHA=1
+    - export VLLM_ROCM_USE_AITER_LINEAR=0
+    - export VLLM_ROCM_USE_AITER_RMSNORM=0
     - pytest -s -v tests/e2e/offline_inference/test_cache_dit.py tests/e2e/offline_inference/test_teacache.py
 
-- label: "Diffusion Sequence Parallelism Test"
-  timeout_in_minutes: 20
-  agent_pool: mi325_2
-  depends_on: amd-build
-  mirror_hardwares: [amdproduction]
-  grade: Blocking
-  commands:
-    - export GPU_ARCHS=gfx942
-    - export VLLM_LOGGING_LEVEL=DEBUG
-    - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-    - pytest -s -v tests/e2e/offline_inference/test_sequence_parallel.py
-
-- label: "Diffusion Tensor Parallelism Test"
-  timeout_in_minutes: 20
-  agent_pool: mi325_2
-  depends_on: amd-build
-  commands:
-    - export GPU_ARCHS=gfx942
-    - export VLLM_LOGGING_LEVEL=DEBUG
-    - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-    - pytest -s -v tests/e2e/offline_inference/test_zimage_tensor_parallel.py
-
-- label: "Diffusion GPU Worker Test"
-  timeout_in_minutes: 20
-  agent_pool: mi325_2
-  depends_on: amd-build
-  mirror_hardwares: [amdproduction]
-  grade: Blocking
-  commands:
-    - pytest -s -v tests/diffusion/test_diffusion_worker.py
-
 - label: "Omni Model Test Qwen2-5-Omni"
   timeout_in_minutes: 15
   agent_pool: mi325_2
   depends_on: amd-build
-  mirror_hardwares: [amdproduction]
+  mirror_hardwares: [amdexperimental, amdproduction, amdtentative]
   grade: Blocking
   commands:
     - export GPU_ARCHS=gfx942
     - export VLLM_LOGGING_LEVEL=DEBUG
     - export VLLM_WORKER_MULTIPROC_METHOD=spawn
+    - export MIOPEN_DEBUG_CONV_DIRECT=0
+    - export MIOPEN_DEBUG_CONV_GEMM=0
+    - export VLLM_ROCM_USE_AITER=1
+    - export VLLM_ROCM_USE_AITER_MHA=1
+    - export VLLM_ROCM_USE_AITER_LINEAR=0
+    - export VLLM_ROCM_USE_AITER_RMSNORM=0
     - pytest -s -v tests/e2e/offline_inference/test_qwen2_5_omni.py
-
-- label: "Omni Model Test Qwen3-Omni"
-  timeout_in_minutes: 15
-  agent_pool: mi325_2
-  depends_on: amd-build
-  mirror_hardwares: [amdproduction]
-  grade: Blocking
-  commands:
-    - export VLLM_LOGGING_LEVEL=DEBUG
-    - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-    - export VLLM_TEST_CLEAN_GPU_MEMORY="1"
-    - pytest -s -v tests/e2e/offline_inference/test_qwen3_omni.py
-    - pytest -s -v tests/e2e/online_serving/test_qwen3_omni.py
-    - pytest -s -v tests/e2e/online_serving/test_async_omni.py
-
-- label: "Diffusion Image Edit Test"
-  timeout_in_minutes: 15
-  agent_pool: mi325_1
-  depends_on: amd-build
-  mirror_hardwares: [amdproduction]
-  grade: Blocking
-  commands:
-    - export GPU_ARCHS=gfx942
-    - export VLLM_LOGGING_LEVEL=DEBUG
-    - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-    - pytest -s -v tests/e2e/online_serving/test_image_gen_edit.py

.buildkite/test-template-amd-omni.j2

@@ -13,8 +13,8 @@
         depends_on: ~
         soft_fail: false
         commands:
-          - "source .buildkite/scripts/docker_login_ecr_public.sh && safe_docker_login_ecr_public"
-          - "docker build -f docker/Dockerfile.rocm -t {{ docker_image_amd }} --target final --progress plain ."
+          - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7"
+          - "docker build -f docker/Dockerfile.rocm -t {{ docker_image_amd }} --progress plain ."
           - "docker push {{ docker_image_amd }}"
         key: "amd-build"
         env:

.github/ISSUE_TEMPLATE/400-bug-report.yml

@@ -40,31 +40,6 @@ body:
       </details>
   validations:
     required: true
-
-- type: textarea
-  attributes:
-    label: Your code version
-    description: |
-      Please provide a description of which commit id or version the code is.
-
-    value: |
-      <details>
-      <summary>The commit id or version of vllm</summary>
-
-      ```text
-
-      ```
-      </details>
-      <details>
-      <summary>The commit id or version of vllm-omni</summary>
-
-      ```text
-
-      ```
-      </details>
-  validations:
-    required: true
-
 - type: textarea
   attributes:
     label: 🐛 Describe the bug

.github/ISSUE_TEMPLATE/750-RFC.yml

@@ -20,8 +20,6 @@ body:
     label: Proposed Change.
     description: >
       The proposed change of the RFC.
-    value: |
-      Please provide the detailed design document of the RFC using the [template](https://docs.google.com/document/d/12YxSsVeD1jvL-InClkeAEnZyWFDndz_65JmXvsamuV4/edit?tab=t.0#heading=h.4ef4szrsgspp).
   validations:
     required: true
 - type: textarea

.pre-commit-config.yaml

@@ -29,10 +29,7 @@ repos:
         # only for staged files
 
   - repo: https://github.com/rhysd/actionlint
-    # v1.7.8+ sets `go 1.24.0` in go.mod, which older Go toolchains (and most
-    # current CI images) cannot parse. Pin to v1.7.7 until actionlint fixes the
-    # go.mod directive.
-    rev: v1.7.7
+    rev: v1.7.9
     hooks:
       - id: actionlint
         files: ^\.github/workflows/.*\.ya?ml$

README.md

@@ -9,15 +9,14 @@ Easy, fast, and cheap omni-modality model serving for everyone
 </h3>
 
 <p align="center">
-| <a href="https://vllm-omni.readthedocs.io/en/latest/"><b>Documentation</b></a> | <a href="https://discuss.vllm.ai"><b>User Forum</b></a> | <a href="https://slack.vllm.ai"><b>Developer Slack</b></a> | <a href="docs/assets/WeChat.jpg"><b>WeChat</b></a> |
+| <a href="https://vllm-omni.readthedocs.io/en/latest/"><b>Documentation</b></a> | <a href="https://discuss.vllm.ai"><b>User Forum</b></a> | <a href="https://slack.vllm.ai"><b>Developer Slack</b></a> |
 </p>
 
 ---
 
 *Latest News* 🔥
 
-- [2026/02] We released [0.14.0](https://github.com/vllm-project/vllm-omni/releases/tag/v0.14.0) - This is the first **stable release** of vLLM-Omni that expands Omni’s diffusion / image-video generation and audio / TTS stack, improves distributed execution and memory efficiency, and broadens platform/backend coverage (GPU/ROCm/NPU/XPU). It also brings meaningful upgrades to serving APIs, profiling & benchmarking, and overall stability. Please check our latest [paper](https://arxiv.org/abs/2602.02204) for architecture design and performance results.
-- [2026/01] We released [0.12.0rc1](https://github.com/vllm-project/vllm-omni/releases/tag/v0.12.0rc1) - a major RC milestone focused on maturing the diffusion stack, strengthening OpenAI-compatible serving, expanding omni-model coverage, and improving stability across platforms (GPU/NPU/ROCm), please check our latest [design](https://docs.google.com/presentation/d/1qv4qMW1rKAqDREMXiUDLIgqqHQe7TDPj/edit?usp=sharing&ouid=110473603432222024453&rtpof=true&sd=true).
+- [2026/01] We released [0.12.0rc1](https://github.com/vllm-project/vllm-omni/releases/tag/v0.12.0rc1) - a major RC milestone focused on maturing the diffusion stack, strengthening OpenAI-compatible serving, expanding omni-model coverage, and improving stability across platforms (GPU/NPU/ROCm).
 - [2025/11] vLLM community officially released [vllm-project/vllm-omni](https://github.com/vllm-project/vllm-omni) in order to support omni-modality models serving.
 
 ---
@@ -68,26 +67,9 @@ Visit our [documentation](https://vllm-omni.readthedocs.io/en/latest/) to learn
 We welcome and value any contributions and collaborations.
 Please check out [Contributing to vLLM-Omni](https://vllm-omni.readthedocs.io/en/latest/contributing/) for how to get involved.
 
-## Citation
-
-If you use vLLM-Omni for your research, please cite our [paper](https://arxiv.org/abs/2602.02204):
-
-```bibtex
-@article{yin2026vllmomni,
-  title={vLLM-Omni: Fully Disaggregated Serving for Any-to-Any Multimodal Models},
-  author={Peiqi Yin, Jiangyun Zhu, Han Gao, Chenguang Zheng, Yongxiang Huang, Taichang Zhou, Ruirui Yang, Weizhi Liu, Weiqing Chen, Canlin Guo, Didan Deng, Zifeng Mo, Cong Wang, James Cheng, Roger Wang, Hongsheng Liu},
-  journal={arXiv preprint arXiv:2602.02204},
-  year={2026}
-}
-```
-
 ## Join the Community
 Feel free to ask questions, provide feedbacks and discuss with fellow users of vLLM-Omni in `#sig-omni` slack channel at [slack.vllm.ai](https://slack.vllm.ai) or vLLM user forum at [discuss.vllm.ai](https://discuss.vllm.ai).
 
-## Star History
-
-[![Star History Chart](https://api.star-history.com/svg?repos=vllm-project/vllm-omni&type=date&legend=top-left)](https://www.star-history.com/#vllm-project/vllm-omni&type=date&legend=top-left)
-
 ## License
 
 Apache License 2.0, as found in the [LICENSE](./LICENSE) file.

benchmarks/diffusion/README.md

@@ -1,117 +0,0 @@
-
-# Diffusion Serving Benchmark (Image/Video)
-
-This folder contains an online-serving benchmark script for diffusion models.
-It sends requests to a vLLM OpenAI-compatible endpoint and reports throughput,
-latency percentiles, and optional SLO attainment.
-
-The main entrypoint is:
-
-- `benchmarks/diffusion/diffusion_benchmark_serving.py`
-
-## 1. Quick Start
-
-1. Start the server:
-
-```bash
-vllm serve Qwen/Qwen-Image --omni --port 8099
-```
-
-2. Run a minimal benchmark:
-
-```bash
-python3 benchmarks/diffusion/diffusion_benchmark_serving.py \
-	--base-url http://localhost:8099 \
-	--model Qwen/Qwen-Image \
-	--task t2i \
-	--dataset vbench \
-	--num-prompts 5
-```
-
-**Notes**
-
-- The benchmark talks to `http://<host>:<port>/v1/chat/completions`.
-- If you run the server on another host or port, pass `--base-url` accordingly.
-
-## 2. Supported Datasets
-
-The benchmark supports three dataset modes via `--dataset`:
-
-- `vbench`: Built-in prompt/data loader.
-- `trace`: Heterogeneous request traces (each request can have different resolution/frames/steps).
-- `random`: Synthetic prompts for quick smoke tests.
-
-### VBench dataset
-
-If you use i2v/i2i bench datasets and need auto-download support, you may need:
-
-```bash
-uv pip install gdown
-```
-
-### Trace dataset
-
-Use `--dataset trace` to replay a trace file. The trace can specify per-request fields such as:
-
-- `width`, `height`
-- `num_frames` (video)
-- `num_inference_steps`
-- `seed`, `fps`
-- optional `slo_ms` (per-request SLO target)
-
-By default (when `--dataset-path` is not provided), the script downloads a default trace from
-the HuggingFace dataset repo `asukaqaqzz/Dit_Trace`. The default filename can depend on `--task`
-(e.g., `t2v` uses a video trace).
-
-Current defaults:
-
-- `--task t2i` -> `sd3_trace.txt`
-- `--task t2v` -> `cogvideox_trace.txt`
-
-You can point to your own trace using `--dataset-path`.
-
-## 3. Benchmark Parameters
-
-### Basic flags
-
-- `--base-url`: Server address (the script calls `.../v1/chat/completions`).
-- `--model`: The OpenAI-compatible `model` field.
-- `--task`: Task type (e.g., `t2i`, `t2v`, `i2i`, `i2v`).
-- `--dataset`: Dataset mode (`vbench` / `trace` / `random`).
-- `--num-prompts`: Number of requests to send.
-
-Common optional flags:
-
-- `--output-file`: Write metrics to a JSON file.
-- `--disable-tqdm`: Disable the progress bar.
-
-### Resolution / frames / steps: CLI defaults vs dataset fields
-
-Related flags: `--width`, `--height`, `--num-frames`, `--fps`, `--num-inference-steps`.
-
-- For `vbench` / `random`: these CLI flags act as global defaults for all generated requests.
-- For `trace`: each request can carry its own fields (e.g., `width/height/num_frames/num_inference_steps`).
-
-Precedence rules for `trace` (i.e., what actually gets sent):
-
-- `width/height`: if either `--width` or `--height` is explicitly set, it overrides per-request values from the trace; otherwise per-request values are used when present.
-- `num_frames`: per-request `num_frames` takes precedence; otherwise fall back to `--num-frames`.
-- `num_inference_steps`: per-request `num_inference_steps` takes precedence; otherwise fall back to `--num-inference-steps`.
-
-### SLO, warmup, and max concurrency
-
-Enable SLO evaluation with `--slo`.
-
-- If a request in the trace already has `slo_ms`, that value is used.
-- Otherwise, the script runs warmup requests to infer a base unit time, estimates `expected_ms` by linearly scaling with area/frames/steps, and then sets `slo_ms = expected_ms * --slo-scale`.
-
-Warmup flags:
-
-- `--warmup-requests`: Number of warmup requests.
-- `--warmup-num-inference-steps`: Steps used during warmup.
-- For `--task t2v`: warmup requests are forced to use `num_frames=1` to make warmup faster and less noisy.
-
-Traffic / concurrency flags:
-
-- `--request-rate`: Target request rate (requests/second). If set to `inf`, the script sends all requests immediately.
-- `--max-concurrency`: Max number of in-flight requests (default: `1`). This can hard-cap the achieved QPS: if it is too small, requests will queue behind the semaphore, and both achieved throughput and observed SLO attainment can be skewed.

benchmarks/diffusion_benchmark_serving.py

@@ -9,29 +9,28 @@ If you want to use i2v, i2i dataset, you should `uv pip install gdown` first
 Usage:
     # Video
     t2v:
-    python3 benchmarks/diffusion/diffusion_benchmark_serving.py \
+    python3 benchmarks/diffusion_benchmark_serving.py \
         --dataset vbench --task t2v --num-prompts 10 \
         --height 480 --width 640 --fps 16 --num-frames 80
 
     i2v:
-    python3 benchmarks/diffusion/diffusion_benchmark_serving.py \
+    python3 benchmarks/diffusion_benchmark_serving.py \
         --dataset vbench --task i2v --num-prompts 10
 
 
     # Image
     t2i:
-    python3 benchmarks/diffusion/diffusion_benchmark_serving.py \
+    python3 benchmarks/diffusion_benchmark_serving.py \
         --dataset vbench --task t2i --num-prompts 10 \
         --height 1024 --width 1024
 
     i2i:
-    python3 benchmarks/diffusion/diffusion_benchmark_serving.py \
+    python3 benchmarks/diffusion_benchmark_serving.py \
         --dataset vbench --task i2i --num-prompts 10
 
 """
 
 import argparse
-import ast
 import asyncio
 import base64
 import glob
@@ -41,7 +40,6 @@ import os
 import time
 import uuid
 from abc import ABC, abstractmethod
-from collections.abc import AsyncGenerator
 from dataclasses import dataclass, field, replace
 from typing import Any
 
@@ -62,8 +60,6 @@ class RequestFuncInput:
     num_inference_steps: int | None = None
     seed: int | None = None
     fps: int | None = None
-    timestamp: float | None = None
-    slo_ms: float | None = None
     extra_body: dict[str, Any] = field(default_factory=dict)
     image_paths: list[str] | None = None
     request_id: str = field(default_factory=lambda: str(uuid.uuid4()))
@@ -77,7 +73,6 @@ class RequestFuncOutput:
     start_time: float = 0.0
     response_body: dict[str, Any] = field(default_factory=dict)
     peak_memory_mb: float = 0.0
-    slo_achieved: bool | None = None
 
 
 class BaseDataset(ABC):
@@ -315,226 +310,6 @@ class VBenchDataset(BaseDataset):
         return [self[i] for i in range(len(self))]
 
 
-class TraceDataset(BaseDataset):
-    """Trace-based dataset loader for heterogeneous diffusion requests."""
-
-    DEFAULT_REPO_ID = "asukaqaqzz/Dit_Trace"
-    DEFAULT_FILENAME = "sd3_trace.txt"
-    DEFAULT_FILENAME_BY_TASK: dict[str, str] = {
-        # Text-to-image traces (e.g., SD3)
-        "t2i": "sd3_trace.txt",
-        # Text-to-video traces (e.g., CogVideoX)
-        "t2v": "cogvideox_trace.txt",
-    }
-
-    def __init__(self, args, api_url: str, model: str):
-        super().__init__(args, api_url, model)
-        self.cache_dir = os.path.join(os.path.expanduser("~"), ".cache", "vllm-omni", "trace")
-        self.default_filename = self.DEFAULT_FILENAME_BY_TASK.get(getattr(args, "task", ""), self.DEFAULT_FILENAME)
-        dataset_root = args.dataset_path
-        if not dataset_root:
-            dataset_root = self._download_default_trace()
-        self.items = self._load_items(dataset_root)
-
-    @staticmethod
-    def _coerce_int(x: Any) -> int | None:
-        if x is None:
-            return None
-        if isinstance(x, bool):
-            return None
-        if isinstance(x, int):
-            return x
-        try:
-            s = str(x).strip()
-            if not s:
-                return None
-            return int(float(s))
-        except Exception:
-            return None
-
-    @staticmethod
-    def _coerce_float(x: Any) -> float | None:
-        if x is None:
-            return None
-        if isinstance(x, float):
-            return x
-        if isinstance(x, int):
-            return float(x)
-        try:
-            s = str(x).strip()
-            if not s:
-                return None
-            return float(s)
-        except Exception:
-            return None
-
-    def _download_default_trace(self) -> str:
-        """Download default trace file from HuggingFace Hub if not provided."""
-
-        try:
-            from huggingface_hub import hf_hub_download
-        except ImportError as exc:
-            raise ImportError(
-                "huggingface_hub is required to download the default trace dataset. "
-                "Install via `pip install huggingface_hub`."
-            ) from exc
-
-        os.makedirs(self.cache_dir, exist_ok=True)
-        return hf_hub_download(
-            repo_id=self.DEFAULT_REPO_ID,
-            filename=self.default_filename,
-            repo_type="dataset",
-            local_dir=self.cache_dir,
-            local_dir_use_symlinks=False,
-        )
-
-    def _expand_paths(self, dataset_path: str | None) -> list[str]:
-        if not dataset_path:
-            return []
-
-        parts = [p.strip() for p in str(dataset_path).split(",") if p.strip()]
-        paths: list[str] = []
-        for p in parts:
-            if any(ch in p for ch in ["*", "?", "["]):
-                paths.extend(sorted(glob.glob(p)))
-            elif os.path.isdir(p):
-                paths.extend(sorted(glob.glob(os.path.join(p, "**", "*.txt"), recursive=True)))
-            else:
-                paths.append(p)
-
-        seen = set()
-        unique_paths = []
-        for p in paths:
-            if p not in seen:
-                seen.add(p)
-                unique_paths.append(p)
-        return unique_paths
-
-    def _parse_trace_file(self, path: str) -> list[dict[str, Any]]:
-        rows: list[dict[str, Any]] = []
-
-        def parse_request_repr_line(line: str) -> dict[str, Any] | None:
-            text = line.strip()
-            if not text:
-                return None
-            if not (text.startswith("Request(") and text.endswith(")")):
-                return None
-            inner = text[len("Request(") : -1]
-            try:
-                expr = ast.parse(f"f({inner})", mode="eval")
-                if not isinstance(expr.body, ast.Call):
-                    return None
-                call = expr.body
-                out: dict[str, Any] = {}
-                for kw in call.keywords:
-                    if kw.arg is None:
-                        continue
-                    out[kw.arg] = ast.literal_eval(kw.value)
-                return out
-            except Exception:
-                return None
-
-        # detect first non-empty line to pick parser
-        first_non_empty = None
-        with open(path, encoding="utf-8") as f:
-            for _ in range(50):
-                pos = f.tell()
-                line = f.readline()
-                if not line:
-                    break
-                if line.strip():
-                    first_non_empty = line.strip()
-                    f.seek(pos)
-                    break
-
-        if first_non_empty is None:
-            return rows
-
-        if first_non_empty.startswith("Request("):
-            with open(path, encoding="utf-8") as f:
-                for line in f:
-                    parsed = parse_request_repr_line(line)
-                    if isinstance(parsed, dict):
-                        rows.append(parsed)
-            return rows
-
-        # txt fallback: parse Request(...) lines only
-        with open(path, encoding="utf-8") as f:
-            for line in f:
-                parsed = parse_request_repr_line(line)
-                if isinstance(parsed, dict):
-                    rows.append(parsed)
-        return rows
-
-    def _load_items(self, dataset_root: str) -> list[dict[str, Any]]:
-        paths = self._expand_paths(dataset_root)
-        if not paths:
-            raise ValueError("No trace files found. Provide --dataset-path or rely on default HuggingFace download.")
-
-        items: list[dict[str, Any]] = []
-        for p in paths:
-            if not os.path.exists(p):
-                continue
-            for row in self._parse_trace_file(p):
-                if isinstance(row, dict):
-                    row = dict(row)
-                    row.setdefault("_source", p)
-                    items.append(row)
-
-        if not items:
-            raise ValueError("Trace dataset is empty after parsing provided paths.")
-
-        if self.args.num_prompts is not None:
-            items = items[: self.args.num_prompts]
-
-        return items
-
-    def __len__(self) -> int:
-        return len(self.items)
-
-    def __getitem__(self, idx: int) -> RequestFuncInput:
-        row = self.items[idx]
-        prompt = row.get("prompt") or row.get("text") or ""
-
-        row_height = self._coerce_int(row.get("height"))
-        row_width = self._coerce_int(row.get("width"))
-        num_frames = self._coerce_int(row.get("num_frames"))
-        num_steps = self._coerce_int(row.get("num_inference_steps"))
-        seed = self._coerce_int(row.get("seed"))
-        fps = self._coerce_int(row.get("fps"))
-        timestamp = self._coerce_float(row.get("timestamp"))
-        slo_ms = self._coerce_float(row.get("slo_ms"))
-        image_paths = row.get("image_paths")
-
-        override_w = self.args.width
-        override_h = self.args.height
-        if override_w is not None or override_h is not None:
-            width = override_w
-            height = override_h
-        else:
-            width = row_width
-            height = row_height
-
-        return RequestFuncInput(
-            prompt=str(prompt),
-            api_url=self.api_url,
-            model=self.model,
-            width=width,
-            height=height,
-            num_frames=num_frames if num_frames is not None else self.args.num_frames,
-            num_inference_steps=num_steps if num_steps is not None else self.args.num_inference_steps,
-            seed=seed if seed is not None else self.args.seed,
-            fps=fps if fps is not None else self.args.fps,
-            timestamp=timestamp,
-            slo_ms=slo_ms,
-            image_paths=image_paths,
-            request_id=str(row.get("request_id")) if row.get("request_id") is not None else str(uuid.uuid4()),
-        )
-
-    def get_requests(self) -> list[RequestFuncInput]:
-        return [self[i] for i in range(len(self))]
-
-
 class RandomDataset(BaseDataset):
     def __init__(self, args, api_url: str, model: str):
         self.args = args
@@ -562,128 +337,6 @@ class RandomDataset(BaseDataset):
         return [self[i] for i in range(len(self))]
 
 
-def _compute_expected_latency_ms_from_base(req: RequestFuncInput, args, base_time_ms: float | None) -> float | None:
-    """Compute expected execution time (ms) based on a base per-step-per-frame unit time.
-
-    Assumes linear scaling with pixel area, frame count, and num_inference_steps.
-    The base unit represents latency for a 16x16 resolution, single frame, single step.
-    """
-
-    if base_time_ms is None:
-        return None
-
-    width = req.width if req.width is not None else args.width
-    height = req.height if req.height is not None else args.height
-    if width is None or height is None:
-        return None
-
-    frames = req.num_frames if req.num_frames is not None else args.num_frames
-    steps = req.num_inference_steps if req.num_inference_steps is not None else args.num_inference_steps
-
-    frame_scale = frames if isinstance(frames, int) and frames > 0 else 1
-    step_scale = steps if isinstance(steps, int) and steps > 0 else 1
-
-    area_units = max((float(width) * float(height)) / float(16 * 16), 1.0)
-    return float(base_time_ms) * area_units * frame_scale * step_scale
-
-
-def _infer_slo_base_time_ms_from_warmups(
-    warmup_pairs: list[tuple[RequestFuncInput, RequestFuncOutput]],
-    args,
-) -> float | None:
-    """Infer base SLO unit time from warmup requests.
-
-    Returns the median base latency (ms) for a 16x16 resolution, single-frame,
-    single-step request. Only uses warmups that succeeded and have resolvable
-    width/height.
-    """
-
-    candidates_ms: list[float] = []
-    for req, out in warmup_pairs:
-        if not out.success or out.latency <= 0:
-            continue
-
-        width = req.width if req.width is not None else args.width
-        height = req.height if req.height is not None else args.height
-        if width is None or height is None:
-            continue
-
-        frames = req.num_frames if req.num_frames is not None else args.num_frames
-        steps = req.num_inference_steps if req.num_inference_steps is not None else args.num_inference_steps
-
-        frame_scale = int(frames) if isinstance(frames, int) and frames > 0 else 1
-        step_scale = int(steps) if isinstance(steps, int) and steps > 0 else 1
-
-        area_units = max((float(width) * float(height)) / float(16 * 16), 1.0)
-        denom = area_units * float(frame_scale) * float(step_scale)
-        if denom <= 0:
-            continue
-
-        candidates_ms.append((out.latency * 1000.0) / denom)
-
-    if not candidates_ms:
-        return None
-    return float(np.median(candidates_ms))
-
-
-def _populate_slo_ms_from_warmups(
-    requests_list: list[RequestFuncInput],
-    warmup_pairs: list[tuple[RequestFuncInput, RequestFuncOutput]],
-    args,
-) -> list[RequestFuncInput]:
-    """Populate missing RequestFuncInput.slo_ms using warmup outputs.
-
-    - If a request already has slo_ms (e.g., trace-provided), it is kept as-is.
-    - If any request has slo_ms is None and we can infer base time from warmups,
-      we estimate each missing request's expected execution time and set:
-        req.slo_ms = expected_latency_ms * args.slo_scale
-
-    Returns updated requests_list.
-    """
-
-    if not any(req.slo_ms is None for req in requests_list):
-        return requests_list
-
-    base_time_ms = _infer_slo_base_time_ms_from_warmups(warmup_pairs, args)
-    if base_time_ms is None:
-        return requests_list
-
-    slo_scale = float(getattr(args, "slo_scale", 3.0))
-    if slo_scale <= 0:
-        raise ValueError(f"slo_scale must be positive, got {slo_scale}.")
-
-    updated: list[RequestFuncInput] = []
-    for req in requests_list:
-        if req.slo_ms is not None:
-            updated.append(req)
-            continue
-        expected_ms = _compute_expected_latency_ms_from_base(req, args, base_time_ms)
-        updated.append(replace(req, slo_ms=(expected_ms * slo_scale) if expected_ms is not None else None))
-
-    return updated
-
-
-async def iter_requests(
-    requests_list: list[RequestFuncInput],
-    request_rate: float,
-) -> AsyncGenerator[RequestFuncInput, None]:
-    """Yield requests using a fixed interval if request_rate is set.
-
-    - If request_rate is inf, all requests are yielded immediately (no sleep).
-    - Otherwise, requests are emitted at a fixed cadence of 1 / request_rate seconds.
-    """
-
-    if request_rate != float("inf"):
-        if request_rate <= 0:
-            raise ValueError(f"request_rate must be positive or inf, got {request_rate}.")
-        interval_s = 1.0 / float(request_rate)
-
-    for i, req in enumerate(requests_list):
-        if request_rate != float("inf") and i > 0:
-            await asyncio.sleep(interval_s)
-        yield req
-
-
 def _guess_mime_type(path: str) -> str:
     mime, _ = mimetypes.guess_type(path)
     return mime or "application/octet-stream"
@@ -762,21 +415,12 @@ async def async_request_chat_completions(
 
     output.latency = time.perf_counter() - output.start_time
 
-    if output.success and input.slo_ms is not None:
-        output.slo_achieved = (output.latency * 1000.0) <= float(input.slo_ms)
-
     if pbar:
         pbar.update(1)
     return output
 
 
-def calculate_metrics(
-    outputs: list[RequestFuncOutput],
-    total_duration: float,
-    requests_list: list[RequestFuncInput],
-    args,
-    slo_enabled: bool,
-):
+def calculate_metrics(outputs: list[RequestFuncOutput], total_duration: float):
     success_outputs = [o for o in outputs if o.success]
     error_outputs = [o for o in outputs if not o.success]
 
@@ -798,29 +442,6 @@ def calculate_metrics(
         "peak_memory_mb_median": np.median(peak_memories) if peak_memories else 0,
     }
 
-    if slo_enabled:
-        slo_defined_total = 0
-        slo_met_success = 0
-
-        for req, out in zip(requests_list, outputs):
-            if req.slo_ms is None:
-                continue
-            slo_defined_total += 1
-            if out.slo_achieved is None:
-                continue
-            if out.slo_achieved:
-                slo_met_success += 1
-
-        slo_attain_all = (slo_met_success / slo_defined_total) if slo_defined_total > 0 else 0.0
-
-        metrics.update(
-            {
-                "slo_attainment_rate": slo_attain_all,
-                "slo_met_success": slo_met_success,
-                "slo_scale": getattr(args, "slo_scale", 3.0),
-            }
-        )
-
     return metrics
 
 
@@ -854,8 +475,6 @@ async def benchmark(args):
 
     if args.dataset == "vbench":
         dataset = VBenchDataset(args, api_url, args.model)
-    elif args.dataset == "trace":
-        dataset = TraceDataset(args, api_url, args.model)
     elif args.dataset == "random":
         dataset = RandomDataset(args, api_url, args.model)
     else:
@@ -882,7 +501,6 @@ async def benchmark(args):
     pbar = tqdm(total=len(requests_list), disable=args.disable_tqdm)
 
     async with aiohttp.ClientSession() as session:
-        warmup_pairs: list[tuple[RequestFuncInput, RequestFuncOutput]] = []
         if args.warmup_requests and requests_list:
             print(
                 f"Running {args.warmup_requests} warmup request(s) \
@@ -895,20 +513,16 @@ async def benchmark(args):
                         warm_req,
                         num_inference_steps=args.warmup_num_inference_steps,
                     )
-                warm_out = await limited_request_func(warm_req, session, None)
-                warmup_pairs.append((warm_req, warm_out))
-
-        if args.slo:
-            # Prefer trace-provided per-request slo_ms. Only populate when missing.
-            requests_list = _populate_slo_ms_from_warmups(
-                requests_list=requests_list,
-                warmup_pairs=warmup_pairs,
-                args=args,
-            )
+                await limited_request_func(warm_req, session, None)
 
         start_time = time.perf_counter()
         tasks = []
-        async for req in iter_requests(requests_list=requests_list, request_rate=args.request_rate):
+        for req in requests_list:
+            if args.request_rate != float("inf"):
+                # Poisson process: inter-arrival times follow exponential distribution
+                interval = np.random.exponential(1.0 / args.request_rate)
+                await asyncio.sleep(interval)
+
             task = asyncio.create_task(limited_request_func(req, session, pbar))
             tasks.append(task)
 
@@ -918,7 +532,7 @@ async def benchmark(args):
     pbar.close()
 
     # Calculate metrics
-    metrics = calculate_metrics(outputs, total_duration, requests_list, args, args.slo)
+    metrics = calculate_metrics(outputs, total_duration)
 
     print("\n{s:{c}^{n}}".format(s=" Serving Benchmark Result ", n=60, c="="))
 
@@ -947,12 +561,6 @@ async def benchmark(args):
     print("{:<40} {:<15.4f}".format("Latency Median (s):", metrics["latency_median"]))
     print("{:<40} {:<15.4f}".format("Latency P99 (s):", metrics["latency_p99"]))
 
-    if args.slo:
-        print(f"{'-' * 50}")
-        print("{:<40} {:<15.2%}".format("SLO Attainment Rate (all):", metrics.get("slo_attainment_rate", 0.0)))
-        print("{:<40} {:<15}".format("SLO Met (success count):", str(metrics.get("slo_met_success", 0))))
-        print("{:<40} {:<15}".format("SLO Scale:", str(metrics.get("slo_scale", 3.0))))
-
     if metrics["peak_memory_mb_max"] > 0:
         print(f"{'-' * 50}")
         print("{:<40} {:<15.2f}".format("Peak Memory Max (MB):", metrics["peak_memory_mb_max"]))
@@ -973,16 +581,16 @@ if __name__ == "__main__":
         "--base-url",
         type=str,
         default=None,
-        help="Base URL of the server (e.g., http://localhost:8091). Overrides host/port.",
+        help="Base URL of the server (e.g., http://localhost:30000). Overrides host/port.",
     )
     parser.add_argument("--host", type=str, default="localhost", help="Server host.")
-    parser.add_argument("--port", type=int, default=8091, help="Server port.")
+    parser.add_argument("--port", type=int, default=30000, help="Server port.")
     parser.add_argument("--model", type=str, default="default", help="Model name.")
     parser.add_argument(
         "--dataset",
         type=str,
         default="vbench",
-        choices=["vbench", "trace", "random"],
+        choices=["vbench", "random"],
         help="Dataset to use.",
     )
     parser.add_argument(
@@ -1037,7 +645,7 @@ if __name__ == "__main__":
     parser.add_argument(
         "--num-inference-steps",
         type=int,
-        default=50,
+        default=None,
         help="Number of inference steps (for diffusion models).",
     )
     parser.add_argument(
@@ -1048,21 +656,6 @@ if __name__ == "__main__":
     )
     parser.add_argument("--fps", type=int, default=None, help="FPS (for video).")
     parser.add_argument("--output-file", type=str, default=None, help="Output JSON file for metrics.")
-    parser.add_argument(
-        "--slo",
-        action="store_true",
-        help=(
-            "Enable SLO calculation and reporting. If trace provides per-request slo_ms, it is used. "
-            "Otherwise, warmup request(s) are used to infer expected execution time assuming linear "
-            "scaling by resolution, frames, and steps, then slo_ms = expected_time * --slo-scale."
-        ),
-    )
-    parser.add_argument(
-        "--slo-scale",
-        type=float,
-        default=3.0,
-        help="SLO target multiplier: slo_ms = estimated_exec_time_ms * slo_scale (default: 3).",
-    )
     parser.add_argument("--disable-tqdm", action="store_true", help="Disable progress bar.")
 
     args = parser.parse_args()

collect_env.py

@@ -57,7 +57,6 @@ SystemEnv = namedtuple(
         "cpu_info",
         "rocm_version",  # vllm specific field
         "vllm_version",  # vllm specific field
-        "vllm_omni_version",  # vllm-omni specific field
         "vllm_build_flags",  # vllm specific field
         "gpu_topo",  # vllm specific field
         "env_vars",
@@ -290,31 +289,6 @@ def get_vllm_version():
     return __version__
 
 
-def get_vllm_omni_version(run_lambda):
-    try:
-        import vllm_omni
-        from vllm_omni import __version__, __version_tuple__
-
-        version_str = __version_tuple__[-1]
-        if isinstance(version_str, str) and version_str.startswith("g"):
-            if "." in version_str:
-                git_sha = version_str.split(".")[0][1:]
-                date = version_str.split(".")[-1][1:]
-                return f"{__version__} (git sha: {git_sha}, date: {date})"
-            else:
-                git_sha = version_str[1:]
-                return f"{__version__} (git sha: {git_sha})"
-
-        package_dir = os.path.dirname(os.path.abspath(vllm_omni.__file__))
-        git_sha = run_and_read_all(run_lambda, f"git -C {package_dir} rev-parse --short HEAD")
-        if git_sha:
-            return f"{__version__} (git sha: {git_sha})"
-
-        return __version__
-    except ImportError:
-        return "N/A (vllm_omni not installed)"
-
-
 def summarize_vllm_build_flags():
     # This could be a static method if the flags are constant, or dynamic if you need to check environment variables, etc.
     return "CUDA Archs: {}; ROCm: {}".format(
@@ -550,7 +524,6 @@ def get_env_info():
 
     rocm_version = get_rocm_version(run_lambda)
     vllm_version = get_vllm_version()
-    vllm_omni_version = get_vllm_omni_version(run_lambda)
     vllm_build_flags = summarize_vllm_build_flags()
     gpu_topo = get_gpu_topo(run_lambda)
 
@@ -582,7 +555,6 @@ def get_env_info():
         cpu_info=get_cpu_info(run_lambda),
         rocm_version=rocm_version,
         vllm_version=vllm_version,
-        vllm_omni_version=vllm_omni_version,
         vllm_build_flags=vllm_build_flags,
         gpu_topo=gpu_topo,
         env_vars=get_env_vars(),
@@ -649,7 +621,6 @@ env_info_fmt += """
 ==============================
 ROCM Version                 : {rocm_version}
 vLLM Version                 : {vllm_version}
-vLLM-Omni Version            : {vllm_omni_version}
 vLLM Build Flags:
   {vllm_build_flags}
 GPU Topology:

docker/Dockerfile.ci

@@ -1,17 +1,11 @@
 ARG VLLM_BASE_IMAGE=vllm/vllm-openai
-ARG VLLM_BASE_TAG=v0.15.0
+ARG VLLM_BASE_TAG=v0.12.0
 FROM ${VLLM_BASE_IMAGE}:${VLLM_BASE_TAG}
 ARG APP_DIR=/workspace/vllm-omni
 WORKDIR ${APP_DIR}
 
 COPY . .
 
-# Install system dependencies
-RUN apt-get update && \
-    apt-get install -y ffmpeg && \
-    apt-get clean && \
-    rm -rf /var/lib/apt/lists/*
-
 # Install vllm-omni into the same uv-managed Python environment used by the base image.
 RUN uv pip install --python "$(python3 -c 'import sys; print(sys.executable)')" --no-cache-dir ".[dev]"
 

docker/Dockerfile.rocm

@@ -1,7 +1,9 @@
-ARG BASE_IMAGE=vllm/vllm-openai-rocm:v0.15.0
-FROM ${BASE_IMAGE} AS final
+ARG BASE_IMAGE=rocm/vllm-dev:nightly_main_20251205
+FROM ${BASE_IMAGE}
 
 ARG COMMON_WORKDIR=/app
+ARG VLLM_VERSION=v0.12.0
+ARG PYTORCH_ROCM_ARCH="gfx942;gfx950"
 
 WORKDIR ${COMMON_WORKDIR}
 
@@ -11,22 +13,30 @@ RUN apt-get update && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/*
 
+# Step 2: Reinstall vllm from source
+RUN python3 -m pip uninstall -y vllm && rm -rf vllm &&\
+    git clone https://github.com/vllm-project/vllm.git && \
+    cd vllm && \
+    git checkout ${VLLM_VERSION} && \
+    python3 -m pip install -r requirements/rocm.txt && \
+    python3 setup.py clean --all && \
+    PYTORCH_ROCM_ARCH=${PYTORCH_ROCM_ARCH} python3 setup.py develop && \
+    cd ../ && \
+    rm -rf vllm/.git
+
 RUN mkdir -p ${COMMON_WORKDIR}/vllm-omni
 
-# Step 2: Copy vllm-omni code and install without uv
+# Step 3: Copy vllm-omni code and install without uv
 COPY . ${COMMON_WORKDIR}/vllm-omni
-RUN cd ${COMMON_WORKDIR}/vllm-omni && uv pip install --python "$(python3 -c 'import sys; print(sys.executable)')" --no-cache-dir ".[dev]"
-
-# When we are installing onnxruntime-rocm, we need to uninstall the system-installed onnxruntime first.
-# These are the dependencies of Qwen3-TTS.
-RUN uv pip uninstall onnxruntime --system && uv pip install --no-cache-dir onnxruntime-rocm sox --system
+RUN cd ${COMMON_WORKDIR}/vllm-omni && python3 -m pip install --no-cache-dir ".[dev]"
 
+# Create python symlink
+# `GPU_ARCHS` is an environment variable that is used to set the GPU archs for the AITER.
+# This is needed to prevent the AITER automatic GPU arch detection from failing on MI325X.
+# The AITER version used in this dockerfile has issues with handling
+# the GPU archs of MI325X (CI machine) correctly. So we manually set the GPU archs here.
+# We reuse AITER_ROCM_ARCH from the base image to avoid duplication.
+ENV GPU_ARCHS=${AITER_ROCM_ARCH}
 RUN ln -sf /usr/bin/python3 /usr/bin/python
 
-CMD ["/bin/bash"]
-
 ENTRYPOINT []
-
-#Set entrypoint for vllm-openai official images
-FROM final AS vllm-openai
-ENTRYPOINT ["vllm", "serve", "--omni"]

docs/.nav.yml

@@ -3,19 +3,16 @@ nav:
 - User Guide:
   - Getting Started:
     - getting_started/quickstart.md
-    - getting_started/installation/*
+    - getting_started/installation
   - Serving:
     - OpenAI-Compatible API:
       - Image Generation: serving/image_generation_api.md
-      - Image Edit: serving/image_edit_api.md
   - Examples:
     - examples/README.md
     - Offline Inference:
       - Image-To-Image: user_guide/examples/offline_inference/image_to_image.md
-      - Image-To-Video: user_guide/examples/offline_inference/image_to_video.md
       - Qwen2.5-Omni: user_guide/examples/offline_inference/qwen2_5_omni.md
       - Qwen3-Omni: user_guide/examples/offline_inference/qwen3_omni.md
-      - Qwen3-TTS Offline Inference: user_guide/examples/offline_inference/qwen3_tts.md
       - Text-To-Image: user_guide/examples/offline_inference/text_to_image.md
       - Text-To-Video: user_guide/examples/offline_inference/text_to_video.md
     - Online Serving:
@@ -28,16 +25,14 @@ nav:
   - Configuration:
     - configuration/README.md
     - configuration/*
+  - Diffusion Acceleration:
+    - Overview: user_guide/diffusion_acceleration.md
+    - Acceleration Methods:
+      - TeaCache: user_guide/acceleration/teacache.md
+      - Cache-DiT: user_guide/acceleration/cache_dit_acceleration.md
+      - Parallelism Acceleration: user_guide/acceleration/parallelism_acceleration.md
   - Models:
     - models/supported_models.md
-  - Features:
-    - Sleep Mode: features/sleep_mode.md
-    - Diffusion Features:
-      - Overview: user_guide/diffusion_acceleration.md
-      - TeaCache: user_guide/diffusion/teacache.md
-      - Cache-DiT: user_guide/diffusion/cache_dit_acceleration.md
-      - Parallelism Acceleration: user_guide/diffusion/parallelism_acceleration.md
-      - CPU Offloading: user_guide/diffusion/cpu_offload_diffusion.md
 - Developer Guide:
   - General:
     - contributing/README.md
@@ -48,6 +43,7 @@ nav:
     - contributing/model/adding_omni_model.md
     - contributing/model/adding_diffusion_model.md
   - CI: contributing/ci
+  - Tests: contributing/tests
   - Design Documents:
     - design/index.md
     - design/architecture_overview.md

docs/README.md

@@ -60,5 +60,5 @@ vLLM-Omni seamlessly supports most popular open-source models on HuggingFace, in
 
 For more information, checkout the following:
 
-- [vllm-omni architecture design and recent roadmaps](https://docs.google.com/presentation/d/1qv4qMW1rKAqDREMXiUDLIgqqHQe7TDPj/edit?usp=sharing&ouid=110473603432222024453&rtpof=true&sd=true)
+- [vllm-omni architecture design and recent roadmaps](https://docs.google.com/presentation/d/1Y7t2Zm3BIISPN-_X_sDpxZMNCWir2rAx/edit?usp=drive_link&ouid=110473603432222024453&rtpof=true&sd=true)
 - [vllm-omni announcement blogpost](https://blog.vllm.ai/2025/11/30/vllm-omni.html)

docs/api/README.md

@@ -36,6 +36,7 @@ Input data structures for multi-modal inputs.
 
 Engine classes for offline and online inference.
 
+- [vllm_omni.diffusion.diffusion_engine.BackgroundResources][]
 - [vllm_omni.diffusion.diffusion_engine.DiffusionEngine][]
 - [vllm_omni.engine.AdditionalInformationEntry][]
 - [vllm_omni.engine.AdditionalInformationPayload][]
@@ -53,16 +54,40 @@ Engine classes for offline and online inference.
 
 Core scheduling and caching components.
 
-- [vllm_omni.core.sched.omni_ar_scheduler.KVCacheTransferData][]
 - [vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler][]
 - [vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler][]
-- [vllm_omni.core.sched.output.OmniCachedRequestData][]
 - [vllm_omni.core.sched.output.OmniNewRequestData][]
-- [vllm_omni.model_executor.models.qwen3_tts.tokenizer_25hz.vq.core_vq.DistributedGroupResidualVectorQuantization][]
-- [vllm_omni.model_executor.models.qwen3_tts.tokenizer_25hz.vq.core_vq.DistributedResidualVectorQuantization][]
-- [vllm_omni.model_executor.models.qwen3_tts.tokenizer_25hz.vq.core_vq.EuclideanCodebook][]
-- [vllm_omni.model_executor.models.qwen3_tts.tokenizer_25hz.vq.core_vq.VectorQuantization][]
-- [vllm_omni.model_executor.models.qwen3_tts.tokenizer_25hz.vq.core_vq.preprocess][]
+
+## Model Executor
+
+Model execution components.
+
+- [vllm_omni.model_executor.custom_process_mixin.CustomProcessMixin][]
+- [vllm_omni.model_executor.models.output_templates.OmniOutput][]
+- [vllm_omni.model_executor.models.qwen2_5_omni.qwen2_5_omni.Qwen2_5OmniForConditionalGeneration][]
+- [vllm_omni.model_executor.models.qwen2_5_omni.qwen2_5_omni_talker.Qwen2_5OmniTalkerForConditionalGeneration][]
+- [vllm_omni.model_executor.models.qwen2_5_omni.qwen2_5_omni_thinker.Qwen2_5OmniConditionalGenerationMixin][]
+- [vllm_omni.model_executor.models.qwen2_5_omni.qwen2_5_omni_thinker.Qwen2_5OmniThinkerForConditionalGeneration][]
+- [vllm_omni.model_executor.models.qwen2_5_omni.qwen2_5_omni_token2wav.Qwen2_5OmniToken2WavBigVGANModel][]
+- [vllm_omni.model_executor.models.qwen2_5_omni.qwen2_5_omni_token2wav.Qwen2_5OmniToken2WavDiTModel][]
+- [vllm_omni.model_executor.models.qwen2_5_omni.qwen2_5_omni_token2wav.Qwen2_5OmniToken2WavForConditionalGenerationVLLM][]
+- [vllm_omni.model_executor.models.qwen2_5_omni.qwen2_5_omni_token2wav.Qwen2_5OmniToken2WavModel][]
+- [vllm_omni.model_executor.models.qwen2_5_omni.qwen2_old.Qwen2EmbeddingModel][]
+- [vllm_omni.model_executor.models.qwen2_5_omni.qwen2_old.Qwen2ForCausalLM][]
+- [vllm_omni.model_executor.models.qwen2_5_omni.qwen2_old.Qwen2Model][]
+- [vllm_omni.model_executor.models.qwen3_omni.qwen3_moe.Qwen3MoeForCausalLM][]
+- [vllm_omni.model_executor.models.qwen3_omni.qwen3_omni.Qwen3OmniMoeForConditionalGeneration][]
+- [vllm_omni.model_executor.models.qwen3_omni.qwen3_omni_code2wav.Qwen3OmniMoeCode2Wav][]
+- [vllm_omni.model_executor.models.qwen3_omni.qwen3_omni_moe_code_predictor_mtp.Qwen3OmniCodePredictorBaseModel][]
+- [vllm_omni.model_executor.models.qwen3_omni.qwen3_omni_moe_code_predictor_mtp.Qwen3OmniMoeTalkerCodePredictor][]
+- [vllm_omni.model_executor.models.qwen3_omni.qwen3_omni_moe_talker.Qwen3OmniMoeModel][]
+- [vllm_omni.model_executor.models.qwen3_omni.qwen3_omni_moe_talker.Qwen3OmniMoeTalkerForConditionalGeneration][]
+- [vllm_omni.model_executor.models.qwen3_omni.qwen3_omni_moe_thinker.Qwen3MoeLLMForCausalLM][]
+- [vllm_omni.model_executor.models.qwen3_omni.qwen3_omni_moe_thinker.Qwen3MoeLLMModel][]
+- [vllm_omni.model_executor.models.qwen3_omni.qwen3_omni_moe_thinker.Qwen3OmniMoeConditionalGenerationMixin][]
+- [vllm_omni.model_executor.models.qwen3_omni.qwen3_omni_moe_thinker.Qwen3OmniMoeThinkerForConditionalGeneration][]
+- [vllm_omni.model_executor.models.qwen3_omni.qwen3_omni_moe_thinker.Qwen3OmniMoeThinkerMultiModalProcessor][]
+- [vllm_omni.model_executor.models.qwen3_omni.qwen3_omni_moe_thinker.Qwen3OmniMoeThinkerProcessingInfo][]
 
 ## Configuration
 
@@ -72,25 +97,13 @@ Configuration classes.
 - [vllm_omni.diffusion.cache.teacache.config.TeaCacheConfig][]
 - [vllm_omni.distributed.omni_connectors.utils.config.ConnectorSpec][]
 - [vllm_omni.distributed.omni_connectors.utils.config.OmniTransferConfig][]
-- [vllm_omni.model_executor.models.qwen3_tts.configuration_qwen3_tts.Qwen3TTSConfig][]
-- [vllm_omni.model_executor.models.qwen3_tts.configuration_qwen3_tts.Qwen3TTSSpeakerEncoderConfig][]
-- [vllm_omni.model_executor.models.qwen3_tts.configuration_qwen3_tts.Qwen3TTSTalkerCodePredictorConfig][]
-- [vllm_omni.model_executor.models.qwen3_tts.configuration_qwen3_tts.Qwen3TTSTalkerConfig][]
-- [vllm_omni.model_executor.models.qwen3_tts.tokenizer_12hz.configuration_qwen3_tts_tokenizer_v2.Qwen3TTSTokenizerV2Config][]
-- [vllm_omni.model_executor.models.qwen3_tts.tokenizer_12hz.configuration_qwen3_tts_tokenizer_v2.Qwen3TTSTokenizerV2DecoderConfig][]
-- [vllm_omni.model_executor.models.qwen3_tts.tokenizer_25hz.configuration_qwen3_tts_tokenizer_v1.Qwen3TTSTokenizerV1Config][]
-- [vllm_omni.model_executor.models.qwen3_tts.tokenizer_25hz.configuration_qwen3_tts_tokenizer_v1.Qwen3TTSTokenizerV1DecoderBigVGANConfig][]
-- [vllm_omni.model_executor.models.qwen3_tts.tokenizer_25hz.configuration_qwen3_tts_tokenizer_v1.Qwen3TTSTokenizerV1DecoderConfig][]
-- [vllm_omni.model_executor.models.qwen3_tts.tokenizer_25hz.configuration_qwen3_tts_tokenizer_v1.Qwen3TTSTokenizerV1DecoderDiTConfig][]
-- [vllm_omni.model_executor.models.qwen3_tts.tokenizer_25hz.configuration_qwen3_tts_tokenizer_v1.Qwen3TTSTokenizerV1EncoderConfig][]
 
 ## Workers
 
 Worker classes and model runners for distributed inference.
 
-- [vllm_omni.diffusion.worker.gpu_diffusion_model_runner.GPUDiffusionModelRunner][]
-- [vllm_omni.diffusion.worker.gpu_diffusion_worker.GPUDiffusionWorker][]
-- [vllm_omni.diffusion.worker.gpu_diffusion_worker.WorkerProc][]
+- [vllm_omni.diffusion.worker.gpu_worker.GPUWorker][]
+- [vllm_omni.diffusion.worker.gpu_worker.WorkerProc][]
 - [vllm_omni.diffusion.worker.npu.npu_worker.NPUWorker][]
 - [vllm_omni.diffusion.worker.npu.npu_worker.NPUWorkerProc][]
 - [vllm_omni.worker.gpu_ar_model_runner.ExecuteModelState][]

docs/cli/README.md

@@ -22,21 +22,3 @@ If you have custom stage configs file, launch the server with command below
 ```bash
 vllm serve Qwen/Qwen2.5-Omni-7B --omni --stage-configs-path /path/to/stage_configs_file
 ```
-
-
-## bench
-
-Run benchmark tests for online serving throughput.
-Available Commands:
-
-```bash
-vllm bench serve --omni \
-    --model Qwen/Qwen2.5-Omni-7B \
-    --host server-host \
-    --port server-port \
-    --random-input-len 32 \
-    --random-output-len 4  \
-    --num-prompts  5
-```
-
-See [vllm bench serve](./bench/serve.md) for the full reference of all available arguments.

docs/cli/bench/serve.md

@@ -1,359 +0,0 @@
-# vLLM-Omni Benchmark CLI Guide
-The vllm bench command launches the vLLM-Omni benchmark to evaluate the performance of multimodal models.
-
-## Notes
-We currently only support using the "openai-chat-omni" backend.
-
-## Basic Parameter Description
-You can use `vllm bench serve --omni --help=all` to get descriptions of all parameters. The commonly used parameters are described below:
-- `--omni`  
-  Enable Omni (multimodal) mode, supporting multimodal inputs and outputs such as images, videos, and audio.
-
-- `--backend`  
-  Specify the backend adapter as openai-chat-omni, using OpenAI Chat compatible API behavior as the protocol. Currently only openai-chat-omni is supported.
-
-- `--model`  
-  The model identifier to load, filled according to the models supported by vLLM-Omni.
-
-- `--endpoint`  
-  The API endpoint exposed externally, to which clients send their requests.
-
-- `--dataset-name`  
-  The name of the dataset used; random-mm indicates generating random multimodal inputs (images, videos, audio).
-
-- `--num-prompts`  
-  The total number of requests to send, an integer.
-
-- `--max-concurrency`  
-  "Maximum number of concurrent requests. This can be used "
-        "to help simulate an environment where a higher level component "
-        "is enforcing a maximum number of concurrent requests. While the "
-        "--request-rate argument controls the rate at which requests are "
-        "initiated, this argument will control how many are actually allowed "
-        "to execute at a time. This means that when used in combination, the "
-        "actual request rate may be lower than specified with --request-rate, "
-        "if the server is not processing requests fast enough to keep up."
-
-- `--request-rate`  
-  "Number of requests per second. If this is inf, "
-        "then all the requests are sent at time 0. "
-        "Otherwise, we use Poisson process or gamma distribution "
-        "to synthesize the request arrival times."
-
-- `--ignore-eos`  
-  "Set ignore_eos flag when sending the benchmark request."
-
-- `--metric-percentiles`  
-  Comma-separated list of percentiles for selected metrics. "
-        "To report 25-th, 50-th, and 75-th percentiles, use \"25,50,75\". "
-        "Default value is \"99\"."
-        "Use \"--percentile-metrics\" to select metrics.
-
-- `--percentile-metrics`  
-        "Comma-separated list of selected metrics to report percentiles."
-                    "This argument specifies the metrics to report percentiles."
-                    'Allowed metric names are "ttft", "tpot", "itl", "e2el", "audio_ttfp", "audio_rtf". '
-
-- `--save-result`  
-Specify to save benchmark results to a json file
-
-- `--save-detailed`  
-"When saving the results, whether to include per request "
-        "information such as response, error, ttfs, tpots, etc."
-
-- `--result-dir`  
- "Specify directory to save benchmark json results."
-        "If not specified, results are saved in the current directory."
-
-- `--result-filename`  
-"Specify the filename to save benchmark json results."
-        "If not specified, results will be saved in "
-        "{label}-{args.request_rate}qps-{base_model_id}-{current_dt}.json"
-
-- `--random-prefix-len`  
-  Number of fixed prefix tokens before the random context in a request.
-  The total input length is the sum of random-prefix-len and a random
-  context length sampled from [input_len * (1 - range_ratio),
-  input_len * (1 + range_ratio)].Only the random and random-mm modes
-  support this parameter.
-
-- `--random-input-len`  
-  Number of input tokens per request.Only the random and random-mm modes support this parameter.
-
-- `--random-output-len`  
-  Number of output tokens per request.Only the random and random-mm modes support this parameter.
-
-- `--random-range-ratio`  
-  Range ratio for sampling input/output length,
-  used only for random sampling. Must be in the range [0, 1) to define
-  a symmetric sampling range
-  [length * (1 - range_ratio), length * (1 + range_ratio)].
-  Only the random and random-mm modes support this parameter.
-
-- `--random-mm-base-items-per-request`  
-  Base number of multimodal items per request for random-mm.
-  Actual per-request count is sampled around this base using
-  --random-mm-num-mm-items-range-ratio.
-  Only the random-mm mode supports this parameter.
-
-- `--random-mm-limit-mm-per-prompt`  
-  Per-modality hard caps for items attached per request, e.g.
-  '{"image": 3, "video": 1, "audio": 1}'. The sampled per-request item
-  count is clamped to the sum of these limits. When a modality
-  reaches its cap, its buckets are excluded and probabilities are
-  renormalized.
-  Only the random-mm mode supports this parameter.
-
-- `--random-mm-num-mm-items-range-ratio`  
-  Range ratio r in [0, 1] for sampling items per request.
-  We sample uniformly from the closed integer range
-  [floor(n*(1-r)), ceil(n*(1+r))]
-  where n is the base items per request.
-  r=0 keeps it fixed; r=1 allows 0 items. The maximum is clamped
-  to the sum of per-modality limits from
-  --random-mm-limit-mm-per-prompt.
-  An error is raised if the computed min exceeds the max.
-  Only the random-mm mode supports this parameter.
-
-- `--random-mm-bucket-config`  
-  The bucket config is a dictionary mapping a multimodal item
-  sampling configuration to a probability.
-  Currently allows for 3 modalities: audio, images and videos.
-  A bucket key is a tuple of (height, width, num_frames)
-  The value is the probability of sampling that specific item.
-  Example:
-  --random-mm-bucket-config
-  "{(256, 256, 1): 0.5, (720, 1280, 16): 0.4, (0, 1, 5): 0.10}"
-  First item: images with resolution 256x256 w.p. 0.5
-  Second item: videos with resolution 720x1280 and 16 frames
-  Third item: audios with 1s duration and 5 channels w.p. 0.1
-  OBS.: If the probabilities do not sum to 1, they are normalized.
-  Only the random-mm mode supports this parameter
-
-## Usage Examples
-
-### Online Benchmark
-<details class="admonition abstract" markdown="1">
-<summary>Show more</summary>
-
-First start serving your model:
-
-```bash
-vllm serve Qwen/Qwen2.5-Omni-7B --omni
-```
-
-Then run the benchmarking for sharegpt:
-
-```bash
-# download dataset
-# wget https://huggingface.co/datasets/anon8231489123/ShareGPT_Vicuna_unfiltered/resolve/main/ShareGPT_V3_unfiltered_cleaned_split.json
-vllm bench serve \
-  --omni \
-  --port 43845 \
-  --model /home/models/Qwen/Qwen3-Omni-30B-A3B-Instruct \
-  --endpoint /v1/chat/completions \
-  --backend openai-chat-omni \
-  --num-prompts 2 \
-  --dataset-name sharegpt \
-  --dataset-path ShareGPT_V3_unfiltered_cleaned_split.json \
-  --percentile-metrics ttft,tpot,itl,e2el
-```
-If successful, you will see the following output:
-```text
-============ Serving Benchmark Result ============
-Successful requests:                     2
-Failed requests:                         0
-Benchmark duration (s):                  81.63
-Request throughput (req/s):              0.02
-Peak concurrent requests:                2.00
-----------------End-to-end Latency----------------
-Mean E2EL (ms):                          56966.13
-Median E2EL (ms):                        56966.13
-P99 E2EL (ms):                           81016.80
-================== Text Result ===================
-Total input tokens:                      36
-Total generated tokens:                  5926
-Output token throughput (tok/s):         72.60
-Peak output token throughput (tok/s):    103.00
-Peak concurrent requests:                2.00
-Total Token throughput (tok/s):          73.04
----------------Time to First Token----------------
-Mean TTFT (ms):                          124.76
-Median TTFT (ms):                        124.76
-P99 TTFT (ms):                           156.10
------Time per Output Token (excl. 1st token)------
-Mean TPOT (ms):                          481.30
-Median TPOT (ms):                        481.30
-P99 TPOT (ms):                           947.55
----------------Inter-token Latency----------------
-Mean ITL (ms):                           25.11
-Median ITL (ms):                         0.33
-P99 ITL (ms):                            25.17
-================== Audio Result ==================
-Total audio duration generated(s):       3.95
-Total audio frames generated:            94890
-Audio throughput(audio duration/s):      0.05
-==================================================
-```
-
-Or run the benchmarking for random:
-
-```bash
-vllm bench serve \
-  --omni \
-  --port 43845 \
-  --endpoint /v1/chat/completions \
-  --backend openai-chat-omni \
-  --model /home/models/Qwen/Qwen3-Omni-30B-A3B-Instruct \
-  --dataset-name random \
-  --num-prompts 2 \
-  --random-prefix-len 5 \
-  --random-input-len 10 \
-  --random-output-len 100 \
-  --percentile-metrics ttft,tpot,itl,e2el,audio_ttfp,audio_rtf \
-  --ignore-eos
-```
-
-If successful, you will see the following output:
-
-```text
-============ Serving Benchmark Result ============
-Successful requests:                     2
-Failed requests:                         0
-Benchmark duration (s):                  24.35
-Request throughput (req/s):              0.08
-Peak concurrent requests:                2.00
-----------------End-to-end Latency----------------
-Mean E2EL (ms):                          22576.23
-Median E2EL (ms):                        22576.23
-P99 E2EL (ms):                           24205.72
-================== Text Result ===================
-Total input tokens:                      30
-Total generated tokens:                  8973
-Output token throughput (tok/s):         368.52
-Peak output token throughput (tok/s):    81.00
-Peak concurrent requests:                2.00
-Total Token throughput (tok/s):          369.76
----------------Time to First Token----------------
-Mean TTFT (ms):                          125.16
-Median TTFT (ms):                        125.16
-P99 TTFT (ms):                           155.88
------Time per Output Token (excl. 1st token)------
-Mean TPOT (ms):                          5.01
-Median TPOT (ms):                        5.01
-P99 TPOT (ms):                           5.42
----------------Inter-token Latency----------------
-Mean ITL (ms):                           34.15
-Median ITL (ms):                         0.01
-P99 ITL (ms):                            376.19
-================== Audio Result ==================
-Total audio duration generated(s):       3.95
-Total audio frames generated:            94890
-Audio throughput(audio duration/s):      0.16
----------------Time to First Packet---------------
-Mean AUDIO_TTFP (ms):                    11756.89
-Median AUDIO_TTFP (ms):                  11756.89
-P99 AUDIO_TTFP (ms):                     20854.25
------------------Real Time Factor-----------------
-Mean AUDIO_RTF:                          3.75
-Median AUDIO_RTF:                        3.75
-P99 AUDIO_RTF:                           7.39
-==================================================
-```
-Notes:
-We use (audio generation time - first packet latency) / audio duration to calculate RTF.
-
-</details>
-
-### Multi-Modal Benchmark
-
-<details class="admonition abstract" markdown="1">
-<summary>Show more</summary>
-
-Benchmark the performance of multi-modal requests in vLLM-Omni.
-
-Generate synthetic image、video、audio inputs alongside random text prompts to stress-test vision models without external datasets.
-
-Notes:
-
-- Works only with online benchmark via the OpenAI backend (`--backend openai-chat-omni`) and endpoint `/v1/chat/completions`.
-
-Start the server (example):
-
-```bash
-vllm serve Qwen/Qwen2.5-Omni-7B --omni
-```
-
-It is recommended to use the flag `--ignore-eos` to simulate real responses. You can set the size of the output via the arg `random-output-len`.
-
-Then run the benchmarking script:
-```bash
-vllm bench serve \
-    --omni \
-  --dataset-name random-mm \
-  --port 40849 \
-  --model /home/models/Qwen/Qwen3-Omni-30B-A3B-Instruct \
-  --endpoint /v1/chat/completions \
-  --backend openai-chat-omni \
-  --request-rate 1 \
-  --num-prompts 1 \
-  --random-input-len 10 \
-  --random-range-ratio 0.0 \
-  --random-mm-base-items-per-request 2 \
-  --random-mm-num-mm-items-range-ratio 0 \
-  --random-mm-limit-mm-per-prompt '{"image":1,"video":1, "audio": 1}' \
-  --random-mm-bucket-config '{"(32, 32, 1)": 0.5, "(0, 1, 1)": 0.1, "(32, 32, 2)":0.4}' \
-  --ignore-eos \
-  --percentile-metrics ttft,tpot,itl \
-  --random-output-len 2 \
-  --extra_body '{"modalities": ["text"]}'
-```
-
-If successful, you will see the following output:
-
-```text
-============ Serving Benchmark Result ============
-Successful requests:                     1
-Failed requests:                         0
-Request rate configured (RPS):           1.00
-Benchmark duration (s):                  1.21
-Request throughput (req/s):              0.83
-Peak concurrent requests:                1.00
-================== Text Result ===================
-Total input tokens:                      10
-Total generated tokens:                  3
-Output token throughput (tok/s):         2.49
-Peak output token throughput (tok/s):    3.00
-Peak concurrent requests:                1.00
-Total Token throughput (tok/s):          10.77
----------------Time to First Token----------------
-Mean TTFT (ms):                          179.74
-Median TTFT (ms):                        179.74
-P99 TTFT (ms):                           179.74
------Time per Output Token (excl. 1st token)------
-Mean TPOT (ms):                          12.76
-Median TPOT (ms):                        12.76
-P99 TPOT (ms):                           12.76
----------------Inter-token Latency----------------
-Mean ITL (ms):                           12.76
-Median ITL (ms):                         12.76
-P99 ITL (ms):                            25.24
-================== Audio Result ==================
-Total audio duration generated(s):       0.00
-Total audio frames generated:            0
-Audio throughput(audio duration/s):      0.00
-==================================================
-```
-
-Behavioral notes:
-
-- If the requested base item count cannot be satisfied under the provided per-prompt limits, the tool raises an error rather than silently clamping.
-
-How sampling works:
-
-- Determine per-request item count k by sampling uniformly from the integer range defined by `--random-mm-base-items-per-request` and `--random-mm-num-mm-items-range-ratio`, then clamp k to at most the sum of per-modality limits.
-- For each of the k items, sample a bucket (H, W, T) according to the normalized probabilities in `--random-mm-bucket-config`, while tracking how many items of each modality have been added.
-- If a modality (e.g., image) reaches its limit from `--random-mm-limit-mm-per-prompt`, all buckets of that modality are excluded and the remaining bucket probabilities are renormalized before continuing.
-This should be seen as an edge case, and if this behavior can be avoided by setting `--random-mm-limit-mm-per-prompt` to a large number. Note that this might result in errors due to engine config `--limit-mm-per-prompt`.
-- The resulting request contains synthetic image data in `multi_modal_data` (OpenAI Chat format). When `random-mm` is used with the OpenAI Chat backend, prompts remain text and MM content is attached via `multi_modal_data`.
-</details>

docs/configuration/README.md

@@ -2,7 +2,7 @@
 
 This section lists the most common options for running vLLM-Omni.
 
-For options within a vLLM Engine. Please refer to [vLLM Configuration](https://docs.vllm.ai/en/v0.14.0/configuration/index.html)
+For options within a vLLM Engine. Please refer to [vLLM Configuration](https://docs.vllm.ai/en/v0.12.0/configuration/index.html)
 
 Currently, the main options are maintained by stage configs for each model.
 
@@ -16,6 +16,6 @@ For introduction, please check [Introduction for stage config](./stage_configs.m
 
 ## Optimization Features
 
-- **[TeaCache Configuration](../user_guide/diffusion/teacache.md)** - Enable TeaCache adaptive caching for DiT models to achieve 1.5x-2.0x speedup with minimal quality loss
-- **[Cache-DiT Configuration](../user_guide/diffusion/cache_dit_acceleration.md)** - Enable Cache-DiT as cache acceleration backends for DiT models
-- **[Parallelism Configuration](../user_guide/diffusion/parallelism_acceleration.md)** - Enable parallelism (e.g., sequence parallelism) for for DiT models
+- **[TeaCache Configuration](../user_guide/acceleration/teacache.md)** - Enable TeaCache adaptive caching for DiT models to achieve 1.5x-2.0x speedup with minimal quality loss
+- **[Cache-DiT Configuration](../user_guide/acceleration/cache_dit_acceleration.md)** - Enable Cache-DiT as cache acceleration backends for DiT models
+- **[Parallelism Configuration](../user_guide/acceleration/parallelism_acceleration.md)** - Enable parallelism (e.g., sequence parallelism) for for DiT models

docs/configuration/stage_configs.md

@@ -40,7 +40,7 @@ stage_args:
     engine_args: # Engine arguments for a certain engine
       model_stage: thinker
       model_arch: Qwen2_5OmniForConditionalGeneration # The model implementation registered in model_executor/models/registry.py
-      worker_type: ar # The specific worker used
+      worker_cls: vllm_omni.worker.gpu_ar_worker.GPUARWorker # The specific worker used
       scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler # The specific scehduler used
       gpu_memory_utilization: 0.8 # The gpu memory allocation for the stage within a single chip
       enforce_eager: true  # Now we only support eager mode
@@ -66,7 +66,7 @@ stage_args:
     engine_args:
       model_stage: talker
       model_arch: Qwen2_5OmniForConditionalGeneration
-      worker_type: ar
+      worker_cls: vllm_omni.worker.gpu_ar_worker.GPUARWorker
       scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
       gpu_memory_utilization: 0.8
       enforce_eager: true
@@ -92,7 +92,7 @@ stage_args:
     engine_args:
       model_stage: code2wav
       model_arch: Qwen2_5OmniForConditionalGeneration
-      worker_type: generation
+      worker_cls: vllm_omni.worker.gpu_generation_worker.GPUGenerationWorker
       scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler
       gpu_memory_utilization: 0.15
       enforce_eager: true

docs/configuration/stage_configs/qwen2_5_omni.yaml

@@ -8,7 +8,7 @@ stage_args:
     engine_args:
       model_stage: thinker
       model_arch: Qwen2_5OmniForConditionalGeneration
-      worker_type: ar
+      worker_cls: vllm_omni.worker.gpu_ar_worker.GPUARWorker
       scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
       gpu_memory_utilization: 0.8
       enforce_eager: true  # Now we only support eager mode
@@ -34,7 +34,7 @@ stage_args:
     engine_args:
       model_stage: talker
       model_arch: Qwen2_5OmniForConditionalGeneration
-      worker_type: ar
+      worker_cls: vllm_omni.worker.gpu_ar_worker.GPUARWorker
       scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
       gpu_memory_utilization: 0.8
       enforce_eager: true
@@ -60,7 +60,7 @@ stage_args:
     engine_args:
       model_stage: code2wav
       model_arch: Qwen2_5OmniForConditionalGeneration
-      worker_type: generation
+      worker_cls: vllm_omni.worker.gpu_generation_worker.GPUGenerationWorker
       scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler
       gpu_memory_utilization: 0.15
       enforce_eager: true

docs/contributing/ci/tests_markers.md

@@ -1,160 +0,0 @@
-# Markers for Tests
-
-By adding markers before test functions, tests can later be executed uniformly by simply declaring the corresponding marker type.
-
-## Current Markers
-Defined in `pyproject.toml`:
-
-| Marker             | Description                                             |
-| ------------------ | ------------------------------------------------------- |
-| `core_model`       | Core model tests (run in each PR)                       |
-| `diffusion`        | Diffusion model tests                                   |
-| `omni`             | Omni model tests                                        |
-| `cache`            | Cache backend tests                                     |
-| `parallel`         | Parallelism/distributed tests                           |
-| `cpu`              | Tests that run on CPU                                   |
-| `gpu`              | Tests that run on GPU (auto-added)                      |
-| `cuda`             | Tests that run on CUDA (auto-added)                     |
-| `rocm`             | Tests that run on AMD/ROCm (auto-added)                 |
-| `npu`              | Tests that run on NPU/Ascend (auto-added)               |
-| `H100`             | Tests that require H100 GPU                             |
-| `L4`               | Tests that require L4 GPU                               |
-| `MI325`            | Tests that require MI325 GPU (AMD/ROCm)                 |
-| `A2`               | Tests that require A2 NPU                               |
-| `A3`               | Tests that require A3 NPU                               |
-| `distributed_cuda` | Tests that require multi cards on CUDA platform         |
-| `distributed_rocm` | Tests that require multi cards on ROCm platform         |
-| `distributed_npu`  | Tests that require multi cards on NPU platform          |
-| `skipif_cuda`      | Skip if the num of CUDA cards is less than the required |
-| `skipif_rocm`      | Skip if the num of ROCm cards is less than the required |
-| `skipif_npu`       | Skip if the num of NPU cards is less than the required  |
-| `slow`             | Slow tests (may skip in quick CI)                       |
-| `benchmark`        | Benchmark tests                                         |
-
-For those markers shown as auto-added, they will be added by the `@hardware_test` decorator.
-
-### Example usage for markers
-
-```python
-from tests.utils import hardware_test
-
-@pytest.mark.core_model
-@pytest.mark.omni
-@hardware_test(
-   res={"cuda": "L4", "rocm": "MI325", "npu": "A2"},
-   num_cards=2,
-)
-@pytest.mark.parametrize("omni_server", test_params, indirect=True)
-def test_video_to_audio()
-    ...
-```
-### Decorator: `@hardware_test`
-
-This decorator is intended to make hardware-aware, cross-platform test authoring easier and more robust for CI/CD environments. The `hardware_test` decorator in `vllm-omni/tests/utils.py` performs the following actions:
-
-1. **Applies platform and resource markers**  
-   Adds the appropriate pytest markers for each specified hardware platform (e.g., `cuda`, `rocm`, `npu`) and resource type (e.g., `L4`, `H100`, `MI325`, `A2`, `A3`).
-   ```
-   @pytest.mark.cuda
-   @pytest.mark.L4
-   ```
-2. **Handles multi-card (distributed) scenarios**  
-   For tests requiring multiple cards, it automatically adds distributed markers such as `distributed_cuda`, `distributed_rocm`, or `distributed_npu`.
-   ```
-   @pytest.mark.distributed_cuda(num_cards=num_cards)
-   ```
-3. **Supports flexible card requirements**  
-   Accepts `num_cards` as either a single integer for all platforms or as a dictionary with per-platform values. If not specified, defaults to 1 card per platform.
-
-4. **Integrates resource validation**  
-   On CUDA, adds a skip marker (`skipif_cuda`) if the system does not have the required number of devices.
-   Support for `skipif_rocm` and `skipif_npu` will be implemented later.
-
-
-5. **Runs each test in a new process**  
-   Automatically wraps the distributed test with a decorator (`@create_new_process_for_each_test`) to ensure isolation and compatibility with multi-process hardware backends.
-
-6. **Works with pytest filtering**  
-   Allows tests to be filtered and selected at runtime using standard pytest marker expressions (e.g., `-m "distributed_cuda and L4"`).
-
-#### Example usage for decorator
-- Single call for multiple platforms:
-    ```python
-    @hardware_test(
-        res={"cuda": "L4", "rocm": "MI325", "npu": "A2"},
-        num_cards={"cuda": 2, "rocm": 2, "npu": 2},
-    )
-    ```
-    or
-    ```python
-    @hardware_test(
-        res={"cuda": "L4", "rocm": "MI325", "npu": "A2"},
-        num_cards=2,
-    )
-    ```
-- `res` must be a dict; supported resources: CUDA (L4/H100), ROCm (MI325), NPU (A2/A3)
-- `num_cards` can be int (all platforms) or dict (per platform); defaults to 1 when missing
-- `hardware_test` automatically applies `@create_new_process_for_each_test` for distributed tests.
-- Distributed markers (`distributed_cuda`, `distributed_rocm`, `distributed_npu`) are auto-added for multi-card cases
-- Filtering examples:
-    - CUDA only: `pytest -m "distributed_cuda and L4"`
-    - ROCm only: `pytest -m "distributed_rocm and MI325"`
-    - NPU only: `pytest -m "distributed_npu"`
-
-## Add Support for a New Platform
-
-If you want to add support for a new platform (e.g., "tpu" for a new accelerator), follow these steps:
-
-1. **Extend the marker list in your pytest config** so that platform/resource markers are defined:
-   ```toml
-   # In pyproject.toml or pytest.ini
-   [tool.pytest.ini_options]
-   markers = [
-       # ... existing markers ...
-       "tpu: Tests that require TPU device",
-       "TPU_V3: Tests that require TPU v3 hardware",
-       "distributed_tpu: Tests that require multiple TPU devices",
-   ]
-   ```
-2. **Implement a marker construction function for your platform** in `vllm-omni/tests/utils.py`:
-   ```python
-   # In vllm-omni/tests/utils.py
-
-   def tpu_marks(*, res: str, num_cards: int):
-       test_platform = pytest.mark.tpu
-       if res == "TPU_V3":
-           test_resource = pytest.mark.TPU_V3
-       else:
-           raise ValueError(
-               f"Invalid TPU resource type: {res}. Supported: TPU_V3")
-
-       if num_cards == 1:
-           return [test_platform, test_resource]
-       else:
-           test_distributed = pytest.mark.distributed_tpu(num_cards=num_cards)
-           # Optionally: add skipif_tpu when implemented
-           return [test_platform, test_resource, test_distributed]
-   ```
-3. **Update `hardware_test` to recognize your new platform**:
-    In the relevant place (see the `hardware_test` implementation), add:
-    ```python
-    if platform == "tpu":
-        marks = tpu_marks(res=resource, num_cards=cards)
-    ```
-4. **(Recommended) Add a test using your new markers**:
-   ```python
-   @hardware_test(
-       res={"tpu": "TPU_V3"},
-       num_cards=2,
-   )
-   def test_my_tpu_feature():
-       ...
-   ```
-
-**Summary**:  
-- Add pytest markers for your new platform/resources  
-- Implement a marker function (`xxx_marks`)  
-- Plug into `hardware_test`  
-- You're done: tests decorated with `@hardware_test` using your platform now automatically get the correct markers, distribution, and isolation!
-
-See code in `vllm-omni/tests/utils.py` for existing examples (`cuda_marks`, `rocm_marks`, `npu_marks`).

docs/contributing/model/adding_diffusion_model.md

@@ -44,7 +44,7 @@ vllm_omni/
 
 # Step-by-step-implementation
 ## Step 1: Model Implementation
-The diffusion pipeline’s implementation follows **HuggingFace Diffusers**.
+The diffusion pipeline’s implementation follows **HuggingFace Diffusers**, and components that do not need modification can be imported directly.
 ### 1.1 Define the Pipeline Class
 Define the pipeline class, e.g., `QwenImageEditPipeline`, and initialize all required submodules, either from HuggingFace `diffusers` or custom implementations. In `QwenImageEditPipeline`, only `QwenImageTransformer2DModel` is re-implemented to support optimizations such as Ulysses-SP. When adding new models in the future, you can either reuse this re-implemented `QwenImageTransformer2DModel` or extend it as needed.
 
@@ -73,75 +73,10 @@ def get_qwen_image_edit_post_process_func(
 The forward function of `QwenImageEditPipeline` follows the HuggingFace `diffusers` design for the most part. The key differences are:
 + As described in the overview, arguments are passed through `OnniDiffusionRequest`, so we need to get user parameters from it accordingly.
 ```python
-prompt = req.prompt
+prompt = req.prompt if req.prompt is not None else prompt
 ```
 + pre/post-processing are handled by the framework elsewhere, so skip them.
 
-### 1.4 Replace some ops or layers in DiT component
-
-vLLM-Omni provides a set of optimized operators with better performance and built-in support for parallelism, including attention, rotary embeddings (RoPE), and linear layers.
-
-Below is an example showing how to replace standard Transformer attention and FFN layers with vLLM-Omni implementations:
-
-```python
-from vllm_omni.diffusion.attention.layer import Attention
-from vllm_omni.diffusion.layers.rope import RotaryEmbedding
-
-class MyAttention(nn.Module):
-    def __init__(self):
-        super().__init__()
-        self.attn = Attention()
-        self.to_qkv = QKVParallelLinear()
-        self.to_out = RowParallelLinear()
-        self.rope = RotaryEmbedding(is_neox_style=False)
-
-    def forward(self, hidden_states):
-        qkv, _ = self.to_qkv(hidden_states)
-        q, k, v = qkv.split(...)
-        q, k = self.rope(...)
-        attn_output = self.attn(q, k, v)
-        output = self.to_out(attn_output)
-
-class MyFFN(nn.Module):
-    def __init__(self):
-        super().__init__()
-        self.fc1 = ColumnParallelLinear()
-        self.fc2 = RowParallelLinear()
-        self.act = F.gelu
-
-    def forward(self, hidden_states):
-        hidden, _ = self.fc1(hidden_states)
-        hidden = self.act(hidden)
-        output = self.fc2(hidden)
-        return output
-```
-
-In this example:
-
-+ Attention uses vLLM-Omni’s optimized attention kernel together with parallel QKV projection and RoPE.
-
-+ Linear layers are replaced with column- and row-parallel variants to enable tensor parallelism.
-
-+ The FFN follows a standard two-layer structure and can be further optimized (e.g., using fused or merged projections) if needed.
-
-
-### 1.5 Provide a `_repeated_blocks` in DiT model
-`_repeated_blocks` is the small and frequently-repeated block(s) of a model -- typically a transformer layer.
-
-It's used for torch compile optimizations.
-```python
-_repeated_blocks = ["QwenImageTransformerBlock"]
-```
-
-
-### 1.6 (Optional) implement sequence parallelism
-vLLM-Omni has a non-intrusive `_sp_plan` that enable sequence parallel without modifying `forward()` logic.
-You can refer to [How to parallelize a new model](../../user_guide/diffusion/parallelism_acceleration.md)
-
-
-### 1.7 (Optional) integrate with Cache-Dit
-vLLM-Omni supports acceleration via [Cache-Dit](../../user_guide/diffusion/cache_dit_acceleration.md). Most models compatible with Diffusers can use Cache-Dit seamlessly. For new models, you can extend support by modifying`cache_dit_backend.py`
-
 ## Step 2: Extend OmniDiffusionRequest Fields
 User-provided inputs are ultimately passed to the model’s forward method through OmniDiffusionRequest, so we add the required fields here to support the new model.
 ```python
@@ -204,167 +139,8 @@ Key point for writing the example:
 
 + Save or display the generated results so users can validate the integration.
 
-## Step 5: TeaCache Coefficient Estimation (Optional)
-
-If your model supports TeaCache acceleration, you need to estimate the polynomial coefficients for optimal caching performance.
-
-### 5.1 Add Extractor Function
-
-First, implement an extractor function in `vllm_omni/diffusion/cache/teacache/extractors.py`. The extractor extracts the modulated input and defines how to run transformer blocks:
-
-```python
-def extract_your_model_context(
-    module: nn.Module,
-    hidden_states: torch.Tensor,
-    timestep: torch.Tensor,
-    **kwargs: Any,
-) -> CacheContext:
-    # 1. Preprocessing
-    temb = module.time_embed(timestep)
-
-    # 2. Extract modulated input (for cache decision)
-    modulated_input = module.transformer_blocks[0].norm1(hidden_states, temb)
-
-    # 3. Define transformer execution
-    def run_transformer_blocks():
-        h = hidden_states
-        for block in module.transformer_blocks:
-            h = block(h, temb=temb)
-        return (h,)
-
-    # 4. Define postprocessing
-    def postprocess(h):
-        return module.proj_out(module.norm_out(h, temb))
-
-    return CacheContext(
-        modulated_input=modulated_input,
-        hidden_states=hidden_states,
-        encoder_hidden_states=None,
-        temb=temb,
-        run_transformer_blocks=run_transformer_blocks,
-        postprocess=postprocess,
-    )
-```
-
-Register it in `EXTRACTOR_REGISTRY`:
-```python
-EXTRACTOR_REGISTRY = {
-    ...
-    "YourTransformer2DModel": extract_your_model_context,
-}
-```
-
-### 5.2 Add Adapter for Coefficient Estimation
-
-Add an adapter in `vllm_omni/diffusion/cache/teacache/coefficient_estimator.py`:
-
-```python
-class YourModelAdapter:
-    @staticmethod
-    def load_pipeline(model_path: str, device: str, dtype: torch.dtype) -> Any:
-        # Load your pipeline
-        ...
-
-    @staticmethod
-    def get_transformer(pipeline: Any) -> tuple[Any, str]:
-        return pipeline.transformer, "YourTransformer2DModel"
-
-    @staticmethod
-    def install_hook(transformer: Any, hook: DataCollectionHook) -> None:
-        registry = HookRegistry.get_or_create(transformer)
-        registry.register_hook(hook._HOOK_NAME, hook)
-
-_MODEL_ADAPTERS["YourModel"] = YourModelAdapter
-```
-
-### 5.3 Run Coefficient Estimation
-
-Use the provided script to estimate coefficients:
-
-```python
-from vllm_omni.diffusion.cache.teacache.coefficient_estimator import (
-    TeaCacheCoefficientEstimator,
-)
-from datasets import load_dataset
-from tqdm import tqdm
-
-# Load model
-estimator = TeaCacheCoefficientEstimator(
-    model_path="/path/to/model",
-    model_type="Bagel",  # Your model type
-    device="cuda",
-)
-
-# Load prompts (paper suggests ~70 prompts)
-dataset = load_dataset("nateraw/parti-prompts", split="train")
-prompts = dataset["Prompt"][:70]
-
-# Collect data
-for prompt in tqdm(prompts):
-    estimator.collect_from_prompt(prompt, num_inference_steps=50)
-
-# Estimate coefficients
-coeffs = estimator.estimate(poly_order=4)
-print(f"Coefficients: {coeffs}")
-```
-
-### 5.4 Interpreting Coefficient Estimation Results
-
-The estimator outputs statistics and polynomial coefficients. Here's how to interpret them:
-
-**Example Output:**
-```
-Data statistics:
-Count: 48
-Input Diffs (x): min=1.1089e-02, max=5.2555e-02, mean=2.8435e-02
-Output Diffs (y): min=2.8242e-02, max=2.9792e-01, mean=7.0312e-02
-Coefficients: [1333131.29, -168644.23, 7950.51, -163.75, 1.26]
-```
-
-**What to Check:**
-- **Count**: Number of timestep pairs analyzed. Should be at least 30-50 for reliable estimation. Low count suggests insufficient prompts or inference steps.
-- **Input/Output Ranges**: Verify output differences correlate with input differences. If ranges seem unusual, check your prompt diversity.
-- **Coefficient Magnitude**: Extremely large values (>1e8) may indicate numerical instability - try collecting more diverse data.
-
-**Troubleshooting:**
-- If results seem unreliable, try:
-  - Increasing number of prompts (100+ recommended)
-  - Using more diverse prompts from multiple datasets
-  - Adjusting `num_inference_steps` (try 20, 50, 100)
-
-### 5.5 Add Coefficients to Config
-
-Add the estimated coefficients to `vllm_omni/diffusion/cache/teacache/config.py`:
-
-```python
-_MODEL_COEFFICIENTS = {
-    ...
-    "YourTransformer2DModel": [
-        1.04730573e+06,  # a4
-        -1.34150749e+05, # a3
-        6.51517806e+03,  # a2
-        -1.41209108e+02, # a1
-        1.17241808e+00,  # a0
-    ],
-}
-```
-## Step 6: Open a Pull Request
-
-When submitting a pull request to add support for a new model, please include the following information in the PR description:
-
-+ Output verification: provide generation outputs to verify correctness and model behavior.
-
-+ Inference speed: provide a comparison with the corresponding implementation in Diffusers.
-
-+ Parallelism support: specify the supported parallel sizes and any relevant limitations.
-
-+ Cache acceleration: check whether the model can be accelerated using Cache-Dit or not.
-
-
-Providing these details helps reviewers evaluate correctness, performance improvements, and parallel scalability of the new model integration.
-
 # Testing
-For comprehensive testing guidelines, please refer to the [Test File Structure and Style Guide](../ci/tests_style.md).
+For comprehensive testing guidelines, please refer to the [Test File Structure and Style Guide](../tests/tests_style.md).
 
 
 ## Adding a Model Recipe

docs/contributing/model/adding_omni_model.md

@@ -572,7 +572,7 @@ def talker2code2wav(
 
 ## Testing
 
-For comprehensive testing guidelines, please refer to the [Test File Structure and Style Guide](../ci/tests_style.md).
+For comprehensive testing guidelines, please refer to the [Test File Structure and Style Guide](../tests/tests_style.md).
 
 ## Adding a Model Recipe
 

docs/contributing/profiling.md

@@ -1,30 +1,48 @@
 # Profiling vLLM-Omni
 
-> **Warning:** Profiling incurs significant overhead. Use only for development and debugging, never in production.
+Performance Profiling Guidelines Profiling capabilities in vLLM-Omni are reserved for development and maintenance tasks aimed at temporal analysis of the codebase. Production use is **strongly discouraged**; enabling the profiler incurs a substantial overhead that negatively impacts inference latency.
 
-vLLM-Omni uses the PyTorch Profiler to analyze performance across both **multi-stage omni-modality models** and **diffusion models**.
+**Mechanism**: vLLM-Omni implements cross-stage profiling via the PyTorch Profiler. To accommodate the architecture—where stages operate as distinct engine instances in separate processes—the profiling interface supports both holistic capturing (all stages) and targeted capturing (specific stages).
 
-### 1. Set the Output Directory
-Before running any script, set this environment variable. The system detects this and automatically saves traces here.
+**1. Enabling the Profiler**
 
-```bash
-export VLLM_TORCH_PROFILER_DIR=./profiles
+Before running your script, you must set the ```VLLM_TORCH_PROFILER_DIR``` environment
+variable.
+
+```Bash
+export VLLM_TORCH_PROFILER_DIR=/path/to/save/traces
 ```
 
-### 2. Profiling Omni-Modality Models
+**Highly Recommended: Limit Profiling to a Single Iteration**  
+For most use cases (especially when profiling audio stages), you should limit the profiler to just **one iteration** to keep trace files small and readable.
 
-It is best to limit profiling to one iteration to keep trace files manageable.
 
 ```bash
 export VLLM_PROFILER_MAX_ITERS=1
 ```
 
+**2. Offline Inference**
+
+For offlinie processing using ```OmniLLM```, you can wrap your ```generate``` calls with ```start_profile``` and ```stop_profile()```.
+
+Basic Usage(All Stages)
+```Python
+from vllm_omni import OmniLLM
+
+omni_llm = OmniLLM.from_engine_args(engine_args)
+
+# Start profiling all active stages
+omni_llm.start_profile()
+
+outputs = omni_llm.generate(prompts, sampling_params)
+
+# Stop profiling and save traces
+omni_llm.stop_profile()
+```
+
 **Selective Stage Profiling**
 The profiler is default to function across all stages. But It is highly recommended to profile specific stages by passing the stages list, preventing from producing too large trace files:
 ```python
-# Profile all stages
-omni_llm.start_profile()
-
 # Only profile Stage 1
 omni_llm.start_profile(stages=[1])
 ```
@@ -34,116 +52,48 @@ omni_llm.start_profile(stages=[1])
 omni_llm.start_profile(stages=[0, 2])
 ```
 
-**Python Usage**: Wrap your generation logic with `start_profile()` and `stop_profile()`.
-
-```python
-from vllm_omni import omni_llm
-
-profiler_enabled = bool(os.getenv("VLLM_TORCH_PROFILER_DIR"))
-
-# 1. Start profiling if enabled
-if profiler_enabled:
-    omni_llm.start_profile(stages=[0])
-
-# Initialize generator
-omni_generator = omni_llm.generate(prompts, sampling_params_list, py_generator=args.py_generator)
-
-total_requests = len(prompts)
-processed_count = 0
-
-# Main Processing Loop
-for stage_outputs in omni_generator:
-
-    # ... [Output processing logic for text/audio would go here] ...
-
-    # Update count to track when to stop profiling
-    processed_count += len(stage_outputs.request_output)
-
-    # 2. Check if all requests are done to stop the profiler safely
-    if profiler_enabled and processed_count >= total_requests:
-        print(f"[Info] Processed {processed_count}/{total_requests}. Stopping profiler inside active loop...")
-
-        # Stop the profiler while workers are still active
-        omni_llm.stop_profile()
-
-        # Wait for traces to flush to disk
-        print("[Info] Waiting 30s for workers to write trace files to disk...")
-        time.sleep(30)
-        print("[Info] Trace export wait time finished.")
-
-omni_llm.close()
-```
-
-
 **Examples**:
 
-1. **Qwen2.5-Omni**:  [https://github.com/vllm-project/vllm-omni/blob/main/examples/offline_inference/qwen2_5_omni/end2end.py](https://github.com/vllm-project/vllm-omni/blob/main/examples/offline_inference/qwen2_5_omni/end2end.py)
+1. **Qwen-omni 2.5**:  [https://github.com/vllm-project/vllm-omni/blob/main/examples/offline_inference/qwen2_5_omni/end2end.py](https://github.com/vllm-project/vllm-omni/blob/main/examples/offline_inference/qwen2_5_omni/end2end.py)
 
-2. **Qwen3-Omni**:   [https://github.com/vllm-project/vllm-omni/blob/main/examples/offline_inference/qwen3_omni/end2end.py](https://github.com/vllm-project/vllm-omni/blob/main/examples/offline_inference/qwen3_omni/end2end.py)
+2. **Qwen-omni 3.0**:   [https://github.com/vllm-project/vllm-omni/blob/main/examples/offline_inference/qwen3_omni/end2end.py](https://github.com/vllm-project/vllm-omni/blob/main/examples/offline_inference/qwen3_omni/end2end.py)
 
+**3. Online Inference(Async)**
 
-### 3. Profiling diffusion models
+For online serving using AsyncOmni, the methods are asynchronous. This allows you to toggle profiling dynamically without restarting the server.
 
-Diffusion profiling is End-to-End, capturing encoding, denoising loops, and decoding.
-
-**CLI Usage:**
 ```python
+from vllm_omni import AsyncOmni
 
-python image_to_video.py \
-    --model Wan-AI/Wan2.2-I2V-A14B-Diffusers \
-    --image qwen-bear.png \
-    --prompt "A cat playing with yarn, smooth motion" \
-    \
-    # Minimize Spatial Dimensions (Optional but helpful):
-    #    Drastically reduces memory usage so the profiler doesn't
-    #    crash due to overhead, though for accurate performance
-    #    tuning you often want target resolutions.
-    --height 48 \
-    --width 64 \
-    \
-    # Minimize Temporal Dimension (Frames):
-    #    Video models process 3D tensors (Time, Height, Width).
-    #    Reducing frames to the absolute minimum (2) keeps the
-    #    tensor size small, ensuring the trace file doesn't become
-    #    multi-gigabytes in size.
-    --num_frames 2 \
-    \
-    # Minimize Iteration Loop (Steps):
-    #    This is the most critical setting for profiling.
-    #    Diffusion models run the same loop X times.
-    #    Profiling 2 steps gives you the exact same performance
-    #    data as 50 steps, but saves minutes of runtime and
-    #    prevents the trace viewer from freezing.
-    --num_inference_steps 2 \
-    \
-    --guidance_scale 5.0 \
-    --guidance_scale_high 6.0 \
-    --boundary_ratio 0.875 \
-    --flow_shift 12.0 \
-    --fps 16 \
-    --output i2v_output.mp4
+# Inside an async function:
+async_omni = AsyncOmni.from_engine_args(engine_args)
 
+await async_omni.start_profile()
+
+async for output in async_omni.generate(prompt, sampling_params, request_id):
+    # Process outputs...
+    pass
+
+await async_omni.stop_profile()
 ```
 
-**Examples**:
+**4. Analyzing Omni Traces**
 
-1. **Qwen image edit**:  [https://github.com/vllm-project/vllm-omni/blob/main/examples/offline_inference/image_to_image/image_edit.py](https://github.com/vllm-project/vllm-omni/blob/main/examples/offline_inference/image_to_image/image_edit.py)
+After ``stop_profile()`` completes (and the file write wait time has passed), the directory specified in ```VLLM_TORCH_PROFILER_DIR``` will contain the trace files.
 
-2. **Wan-AI/Wan2.2-I2V-A14B-Diffusers**:   [https://github.com/vllm-project/vllm-omni/tree/main/examples/offline_inference/image_to_video](https://github.com/vllm-project/vllm-omni/tree/main/examples/offline_inference/image_to_video)
-
-> **Note:**
-As of now, asynchronous (online) profiling is not fully supported in vLLM-Omni. While start_profile() and stop_profile() methods exist, they are only reliable in offline inference scripts (e.g., the provided end2end.py examples). Do not use them in server-mode or streaming scenarios—traces may be incomplete or fail to flush.
-
-### 4. Analyzing Omni Traces
-
-Output files are saved to your configured ```VLLM_TORCH_PROFILER_DIR```.
-
-**Output**
-**Chrome Trace** (```.json.gz```): Visual timeline of kernels and stages. Open in Perfetto UI.
+```
+Output/
+│── ...rank-0.pt.trace.json.gz   # GPU 0 trace (TP=2 Example)
+│── ...rank-1.pt.trace.json.gz   # GPU 1 trace (TP=2 Example)
+│       # Load these into Perfetto to visualize synchronization
+│
+│── profiler_out_.txt            # Summary tables (CPU/CUDA time %)
+```
 
 **Viewing Tools:**
+     - [Perfetto](https://ui.perfetto.dev/): (Recommended): Best for handling large audio trace files.
+     - ```chrome://tracing```: Good for smaller text-only traces.
 
-- [Perfetto](https://ui.perfetto.dev/)(recommended)
-- ```chrome://tracing```(Chrome only)
 
-**Note**: vLLM-Omni reuses the PyTorch Profiler infrastructure from vLLM. See the official vLLM profiler documentation:  [vLLM Profiling Guide](https://docs.vllm.ai/en/stable/contributing/profiling/)
+**Note**: vLLM-Omni reuses the PyTorch Profiler infrastructure from vLLM.  
+For more advanced configuration options (memory profiling, custom activities, etc.), see the official vLLM profiler documentation:  [vLLM Profiling Guide](https://docs.vllm.ai/en/latest/dev/profiling.html)

docs/contributing/tests/tests_style.md

@@ -113,7 +113,7 @@ vllm_omni/                          tests/
 
 ### Naming Conventions
 
-- **Unit Tests**: Use `test_<module_name>.py` format. Example: `omni_llm.py` → `test_omni_llm.py`
+- **Unit/System Tests**: Use `test_<module_name>.py` format. Example: `omni_llm.py` → `test_omni_llm.py`
 
 - **E2E Tests**: Place in `tests/e2e/offline_inference/` or `tests/e2e/online_serving/` with descriptive names. Example: `tests/e2e/offline_inference/test_qwen3_omni.py`, `tests/e2e/offline_inference/test_diffusion_model.py`
 
@@ -139,7 +139,7 @@ vllm_omni/                          tests/
 4. **Documentation**: Add docstrings to all test functions
 5. **Environment variables**: Set uniformly in `conftest.py` or at the top of files
 6. **Type annotations**: Add type annotations to all test function parameters
-7. **Pytest Markers**: Add necessary markers like `@pytest.mark.core_model` and use `@hardware_test` to declare hardware requirements (check detailed in [Markers for Tests](../ci/tests_markers.md)).
+7. **Resources**, Using pytest tag to specify the computation resources the test required.
 
 ### Template
 #### E2E - Online serving
@@ -155,7 +155,6 @@ from pathlib import Path
 import pytest
 import openai
 
-from tests.utils import hardware_test
 
 # Optional: set process start method for workers
 os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
@@ -185,12 +184,6 @@ def base64_encoded_video() -> str:
 def dummy_messages_from_video_data(video_data_url: str, content_text: str) -> str:
     xxx
 
-@pytest.mark.core_model
-@pytest.mark.omni
-@hardware_test(
-    res={"cuda": "L4", "rocm": "MI325", "npu": "A2"},
-    num_cards={"cuda": 2, "rocm": 2, "npu": 4},
-)
 @pytest.mark.parametrize("omni_server", test_params, indirect=True)
 def test_video_to_audio(
     client: openai.OpenAI,
@@ -233,7 +226,6 @@ from pathlib import Path
 import pytest
 from vllm.assets.video import VideoAsset
 
-from tests.utils import hardware_test
 from ..multi_stages.conftest import OmniRunner
 
 # Optional: set process start method for workers
@@ -247,12 +239,7 @@ test_params = [(model, stage_config) for model in models for stage_config in sta
 
 # function name: test_{input_modality}_to_{output_modality}
 # modality candidate: text, image, audio, video, mixed_modalities
-@pytest.mark.core_model
-@pytest.mark.omni
-@hardware_test(
-    res={"cuda": "L4", "rocm": "MI325", "npu": "A2"},
-    num_cards=2,
-)
+@pytest.mark.gpu_mem_high  # requires high-memory GPU node
 @pytest.mark.parametrize("test_config", test_params)
 def test_video_to_audio(omni_runner: type[OmniRunner], model: str) -> None:
     """Offline inference: video input, audio output."""
@@ -276,5 +263,4 @@ def test_video_to_audio(omni_runner: type[OmniRunner], model: str) -> None:
 
 1. The file is saved in an appropriate place and the file name is clear.
 2. The coding style follows the requirements outlined above.
-3. **All test functions have appropriate pytest markers**
-4. For tests that need run in CI, please ensure the test is configured under the `./buildkite/` folder.
+3. For e2e model test, please ensure the test is configured under the `./buildkite/` folder.

docs/design/architecture_overview.md

@@ -64,13 +64,13 @@ According to analysis for current popular open-source models, most of them have
 
 ## Key Components
 
-| Component         | Description                                                                                                                              |
-| ----------------- | ---------------------------------------------------------------------------------------------------------------------------------------- |
-| **OmniRouter**    | provide an intelligent router for Omni-modality requests dispatch                                                                        |
-| **EntryPoints**   | define the APIs for offline/online serving (APIServer, Omni/AsyncOmni) and provide the OmniStage abstraction for different AR/DiT stages |
-| **AR**            | adapted for omni-modality models while inheriting efficient features from vLLM, such as cache management                                 |
-| **Diffusion**     | natively implemented and optimized using acceleration components                                                                         |
-| **OmniConnector** | supports fully disaggregation based on E/P/D/G (Encoding/Processing/Decoding/Generation) disaggregation across stages                    |
+| Component | Description |
+|-----------|-------------|
+| **OmniRouter** | provide an intelligent router for Omni-modality requests dispatch |
+| **EntryPoints** | define the APIs for offline/online serving (APIServer, Omni/AsyncOmni) and provide the OmniStage abstraction for different AR/DiT stages |
+| **AR** | adapted for omni-modality models while inheriting efficient features from vLLM, such as cache management |
+| **Diffusion** | natively implemented and optimized using acceleration components |
+| **OmniConnector** | supports fully disaggregation based on E/P/D/G (Encoding/Processing/Decoding/Generation) disaggregation across stages |
 
 Disaggregated stages are managed through configuration, such as in the Qwen3-Omni example, where stages like Thinker, Talker, and Code2wav are defined as separate OmniStage instances with specific resources and input/output type.
 
@@ -192,4 +192,4 @@ curl -sS -X POST http://localhost:8091/v1/chat/completions \
 }
 ```
 
-For more usages, please refer to [examples](https://github.com/vllm-project/vllm-omni/tree/main/examples).
+For more usages, please refer to [examples](../user_guide/examples/).

docs/design/feature/disaggregated_inference.md

@@ -1,35 +1,27 @@
 # Disaggregated Inference for Omni-Modality Models
 
-This guide explains how to configure and use distributed connectors
-(`vllm_omni/distributed/omni_connectors`) in vllm-omni for multi-stage pipelines.
+This guide explains how to configure and use distributed connectors (vllm_omni/distributed/connectors) in vllm-omni for multi-stage pipelines.
 
-Backend-specific setup lives in separate docs:
-
-- [SharedMemoryConnector](omni_connectors/shared_memory_connector.md)
-- [MooncakeConnector](omni_connectors/mooncake_connector.md)
-- [YuanrongConnector](omni_connectors/yuanrong_connector.md)
-
-## Overview
+## 1. Overview
 
 Connectors enable data transfer between pipeline stages (e.g., Thinker -> Talker).
-Current connectors operate in D2H2D (device to host to device) mode.
+Currently supported connectors operate in **D2H2D (Device-to-Host-to-Device)** mode:
+1. **SharedMemoryConnector**: Uses system shared memory.
+2. **MooncakeConnector**: Uses [Mooncake](https://github.com/kvcache-ai/Mooncake).
 
-## Connector Choices
+*   **SharedMemoryConnector (Default)**: Zero-copy (on host), lowest latency. Best for **single-node** deployments. Auto-configured if no connectors are specified.
+*   **MooncakeConnector**: TCP/RDMA based. Best for **multi-node** distributed deployments. Requires a Mooncake Master service.
 
-| Use Case | Recommended Connector | Notes |
-| :--- | :--- | :--- |
-| Single node | SharedMemoryConnector | Auto-configured if no connector is specified. |
-| Multi node (Mooncake) | MooncakeConnector | Requires Mooncake Master + metadata server. |
-| Multi node (Yuanrong) | YuanrongConnector | Requires Yuanrong Datasystem + etcd. |
+## 2. API Design
 
-## Core API
+The connector system is built around the `OmniConnectorBase` abstraction, which decouples data transport from stage logic.
 
-The connector system is built around `OmniConnectorBase`.
+### Core Interface
 
 ```python
 class OmniConnectorBase(ABC):
     @abstractmethod
-    def put(self, from_stage: str, to_stage: str, put_key: str, data: Any) -> tuple[bool, int, Optional[dict]]:
+    def put(self, from_stage: str, to_stage: str, request_id: str, data: Any) -> tuple[bool, int, Optional[dict]]:
         """
         Store data.
         Returns: (success, serialized_size, metadata)
@@ -37,23 +29,145 @@ class OmniConnectorBase(ABC):
         pass
 
     @abstractmethod
-    def get(self, from_stage: str, to_stage: str, get_key: str, metadata: Optional[dict] = None) -> Optional[tuple[Any, int]]:
+    def get(self, from_stage: str, to_stage: str, request_id: str, metadata: Optional[dict] = None) -> Optional[tuple[Any, int]]:
         """
         Retrieve data.
-        Args: metadata - transport-specific handles returned by put() (e.g., SHM name).
+        Args: metadata - Transport-specific handles returned by put() (e.g., SHM name).
         Returns: (object, serialized_size)
         """
         pass
 ```
 
-### Metadata Passing
+### Key Concept: Metadata Passing
+Unlike a pure key-value store, some connectors (like `SharedMemoryConnector`) generate transient resources (e.g., a shared memory block name) during `put()`. This `metadata` **must be passed via the control plane** (e.g., HTTP headers, queue messages) from the producer stage to the consumer stage so `get()` can locate the data.
 
-Some connectors (e.g., SharedMemoryConnector) generate transient resources during `put()`.
-This `metadata` must be passed through the control plane so `get()` can locate the data.
+## 3. Backends & Use Cases
 
-## Configuration Model
+### 3.1 SharedMemoryConnector
+**Best for:** Single-node, high-performance IPC.
 
-Define connectors in runtime:
+*   **Mechanism:**
+    *   **Small Payloads (< Threshold)**: Data is serialized and passed directly "inline" within the `metadata` dictionary. This avoids the overhead of creating SHM blocks for tiny messages.
+    *   **Large Payloads (>= Threshold)**: Data is written to a named System Shared Memory block. The block name is returned in `metadata`.
+*   **Configuration:**
+    *   `shm_threshold_bytes`: Size in bytes to switch from inline to SHM (default: 64KB).
+
+### 3.2 MooncakeConnector
+**Best for:** Multi-node distributed inference.
+
+*   **Mechanism:** Uses Mooncake's distributed KVCache store.
+    *   **Data Plane**: TCP or RDMA for high-bandwidth transfer.
+    *   **Control Plane**: Uses a centralized Mooncake Master and Metadata Server.
+    *   **Keying**: Deterministic keys based on `request_id/from_stage_to_stage`.
+*   **Requirements**: Requires a running Mooncake Master service.
+
+## 4. Relationship with vLLM
+
+vLLM provides specialized distributed mechanisms for specific artifacts:
+*   **KV Transfer** (`vllm.distributed.kv_transfer`): Optimized for transferring KV caches between prefill and decode instances (using NCCL, Mooncake, etc.).
+*   **EC Transfer** (`vllm.distributed.ec_transfer`): Optimized for sharing encoder embeddings.
+*   **Device Communicators** (`vllm.distributed.device_communicators`): Low-level primitives (NCCL, SHM) for Tensor/Pipeline Parallelism.
+
+`vllm-omni` complements this by introducing a **Generalized Connector Abstraction** (`OmniConnector`) for multimodal pipelines. While vLLM's connectors are artifact-specific, `vllm-omni`:
+
+1.  **Unifies Transport**: Provides a single API (`put`/`get`) to transport *any* stage artifact (Input Embeddings, Hidden States, Audio/Image Tensors, KV Cache, Final Output) between arbitrary pipeline stages (e.g., AudioEncoder -> LLM -> AudioGenerator).
+2.  **Extends Connectivity**: Enables flexible construction of complex DAGs (Directed Acyclic Graphs) where stages can run in the same process, same node, or across nodes, using the most appropriate backend (SHM, Mooncake, etc.) for each edge.
+3.  **Wraps & Adapts**: Can internally utilize vLLM's specialized `kv_transfer` for KV paths while using generic transports (SHM/Mooncake) for other data types, presenting a consistent interface to the application layer.
+
+## 5. Installation (Mooncake)
+
+If using `MooncakeConnector`, install the library first:
+
+```bash
+# For CUDA-enabled systems (Recommended)
+pip install mooncake-transfer-engine
+
+# For non-CUDA systems
+pip install mooncake-transfer-engine-non-cuda
+```
+
+## 6. Using MooncakeConnector
+
+### 6.1 Start Mooncake Master
+
+Start the master service on your primary node:
+
+```bash
+# if you use mooncake SSD storage
+mkdir -p ./mc_storage
+
+mooncake_master \
+  --rpc_port=50051 \
+  --enable_http_metadata_server=true \
+  --http_metadata_server_host=0.0.0.0 \
+  --http_metadata_server_port=8080 \
+  --metrics_port=9003 \
+  --root_fs_dir=./mc_storage/ \
+  --cluster_id=mc-local-1 &
+```
+
+### 6.2 Configuration (YAML)
+
+Edit your stage config (e.g., `qwen2_5_omni.yaml`).
+
+**Step 1: Define Connector in Global Runtime**
+
+```yaml
+runtime:
+  connectors:
+    connector_of_mooncake:
+      name: MooncakeConnector
+      extra:
+        host: "127.0.0.1"           # Local Worker IP
+        metadata_server: "http://<MASTER_IP>:8080/metadata"
+        master: "<MASTER_IP>:50051"
+        segment: 512000000          # 512MB segment
+        localbuf: 64000000          # 64MB buffer
+        proto: "tcp"                # "tcp" or "rdma"
+    ```
+
+**Mooncake Configuration Parameters:**
+
+*   **host**: The hostname or IP address of the local machine (worker). Mooncake uses this to register itself in the metadata server so other nodes can find it.
+*   **metadata_server**: The URL of the metadata server. This is used for service discovery and connection establishment (e.g., exchanging QP information for RDMA).
+*   **master**: The address of the Mooncake Master Server (e.g., `<MASTER_IP>:50051`). This is used for global state management and control plane operations.
+*   **segment**: The size of the global memory segment in bytes (default: ~512MB). This defines the shared memory region accessible by Mooncake for data transfer.
+*   **localbuf**: The size of the local buffer in bytes (default: ~64MB). Used for local data buffering during transfer operations.
+*   **proto**: The transport protocol to use. Options:
+    *   `tcp`: Standard TCP/IP (easier setup, universal compatibility).
+    *   `rdma`: Remote Direct Memory Access (higher performance, requires RDMA-capable hardware).
+
+For more details, refer to the [Mooncake Repository](https://github.com/kvcache-ai/Mooncake).
+
+    **Step 2: Reference in Stages**
+
+Explicitly link stages using `input_connectors` and `output_connectors`:
+
+```yaml
+stage_args:
+  - stage_id: 0
+    # ...
+    output_connectors:
+      to_stage_1: connector_of_mooncake
+
+  - stage_id: 1
+    # ...
+    input_connectors:
+      from_stage_0: connector_of_mooncake
+```
+
+## 7. Using SharedMemoryConnector (Auto-Mode)
+
+**Best for single-node.**
+
+The system will automatically create `SharedMemoryConnector`s for any pipeline edge that does not have an explicit connector defined. This is inferred from:
+1.  `runtime.edges` list in the config.
+2.  `engine_input_source` dependencies defined in `stage_args`.
+
+### Threshold Configuration
+By default, payloads larger than **64KB** (default threshold) are transferred via shared memory, while smaller ones use the control queue (inline).
+
+To adjust this threshold (e.g., to 1GB), add the following to your `runtime.connectors`:
 
 ```yaml
 runtime:
@@ -61,46 +175,26 @@ runtime:
     connector_of_shared_memory:
       name: SharedMemoryConnector
       extra:
-        shm_threshold_bytes: 65536
+        shm_threshold_bytes: 1024 # 1KB threshold
 ```
 
-Wire stages to connectors:
+## 8. Summary
 
-```yaml
-stage_args:
-  - stage_id: 0
-    output_connectors:
-      to_stage_1: connector_of_shared_memory
+| Use Case | Recommended Connector | Configuration |
+| :--- | :--- | :--- |
+| **Single Node** | `SharedMemoryConnector` | **None** (Automatic) or Custom Threshold |
+| **Multi Node** | `MooncakeConnector` | Explicit YAML + Mooncake Master |
 
-  - stage_id: 1
-    input_connectors:
-      from_stage_0: connector_of_shared_memory
-```
+## 9. Operational Notes (important)
 
-If a pipeline edge has no explicit connector, the system auto-creates a
-SharedMemoryConnector for that edge.
+- **Fail-fast config validation**: the loader raises if any expected edge is missing a connector. Define `input_connectors`/`output_connectors` or rely on auto-SHM filling; otherwise startup aborts.
+- **Missing payloads halt the stage**: workers expect connector payloads; if metadata or connector config is missing, the stage raises and stops. Verify connector wiring and metadata propagation before production.
 
-## Relationship with vLLM
+## 10. Future Roadmap: Device-to-Device (D2D) Transport
 
-vLLM provides specialized distributed mechanisms for specific artifacts:
+The current implementations (`SharedMemoryConnector`, `MooncakeConnector`) utilize a **D2H2D (Device-to-Host-to-Device)** data path. Tensors are moved to CPU memory (Host) for transport, which incurs PCIe overhead.
 
-- KV Transfer (`vllm.distributed.kv_transfer`): optimized for KV caches.
-- EC Transfer (`vllm.distributed.ec_transfer`): optimized for encoder embeddings.
-- Device Communicators (`vllm.distributed.device_communicators`): low-level primitives (NCCL, SHM).
+As outlined in the design RFC, future versions will introduce **D2D (Device-to-Device)** connectors:
 
-vllm-omni complements this with a generalized connector abstraction:
-
-1. Unifies transport via a single `put`/`get` API for any stage artifact.
-2. Enables DAG-style pipelines across processes or nodes with per-edge transports.
-3. Can wrap vLLM-specific transfers for KV paths while keeping a consistent interface.
-
-## Operational Notes
-
-- Fail-fast config validation: missing expected edges cause startup failures.
-- Missing payloads halt stages: verify connector wiring and metadata propagation.
-
-## Future Roadmap: D2D Transport
-
-Current connectors use D2H2D paths. Future versions will introduce direct
-device-to-device connectors (NCCL, UCX, IPC) to reduce latency for large
-tensor payloads.
+*   **Goal**: Direct GPU-to-GPU transfer (via NCCL, UCX, or IPC) to minimize latency for large tensor payloads.
+*   **Mechanism**: The `OmniConnector` API allows `put()` to initiate a transfer and return a lightweight handle (metadata) via the control plane, while the heavy payload flows directly between devices.

docs/design/feature/omni_connectors/mooncake_connector.md

@@ -1,74 +0,0 @@
-# MooncakeConnector
-
-## When to Use
-
-Best for multi-node distributed inference using Mooncake.
-
-## Installation
-
-```bash
-# For CUDA-enabled systems (recommended)
-pip install mooncake-transfer-engine
-
-# For non-CUDA systems
-pip install mooncake-transfer-engine-non-cuda
-```
-
-## Start Mooncake Master
-
-```bash
-# If you use Mooncake SSD storage
-mkdir -p ./mc_storage
-
-mooncake_master \
-  --rpc_port=50051 \
-  --enable_http_metadata_server=true \
-  --http_metadata_server_host=0.0.0.0 \
-  --http_metadata_server_port=8080 \
-  --metrics_port=9003 \
-  --root_fs_dir=./mc_storage/ \
-  --cluster_id=mc-local-1 &
-```
-
-## Configuration
-
-Define the connector in runtime:
-
-```yaml
-runtime:
-  connectors:
-    connector_of_mooncake:
-      name: MooncakeConnector
-      extra:
-        host: "127.0.0.1"
-        metadata_server: "http://<MASTER_IP>:8080/metadata"
-        master: "<MASTER_IP>:50051"
-        segment: 512000000
-        localbuf: 64000000
-        proto: "tcp"
-```
-
-Wire stages to the connector:
-
-```yaml
-stage_args:
-  - stage_id: 0
-    output_connectors:
-      to_stage_1: connector_of_mooncake
-
-  - stage_id: 1
-    input_connectors:
-      from_stage_0: connector_of_mooncake
-```
-
-Parameters:
-
-- host: local worker IP registered in the metadata server.
-- metadata_server: metadata server URL for discovery and setup.
-- master: Mooncake Master address.
-- segment: global memory segment size in bytes.
-- localbuf: local buffer size in bytes.
-- proto: transport protocol ("tcp" or "rdma").
-
-For more details, refer to the
-[Mooncake repository](https://github.com/kvcache-ai/Mooncake).

docs/design/feature/omni_connectors/shared_memory_connector.md

@@ -1,27 +0,0 @@
-# SharedMemoryConnector
-
-## When to Use
-
-Best for single-node deployments where stages run on the same host. It is
-auto-configured when no explicit connector is specified for an edge.
-
-## How It Works
-
-- Small payloads (< threshold): serialized and passed inline in metadata.
-- Large payloads (>= threshold): stored in shared memory; the SHM name is
-  returned in metadata.
-
-## Configuration
-
-```yaml
-runtime:
-  connectors:
-    connector_of_shared_memory:
-      name: SharedMemoryConnector
-      extra:
-        shm_threshold_bytes: 65536
-```
-
-## Notes
-
-- Auto-mode uses SharedMemoryConnector if no connector is declared for an edge.

docs/design/feature/omni_connectors/yuanrong_connector.md

@@ -1,101 +0,0 @@
-# YuanrongConnector
-
-## When to Use
-
-Best for multi-node distributed inference using Yuanrong Datasystem.
-
-## Mechanism
-
-Uses Yuanrong Datasystem's distributed KV store (`datasystem.kv_client`).
-
-- Data Plane: TCP or RDMA for high-bandwidth transfer.
-- Control Plane: Yuanrong Datasystem workers and etcd.
-- Keying: deterministic keys based on `put_key` (often composed as `request_id:fromStage_toStage`).
-
-## Installation
-
-```bash
-pip install openyuanrong-datasystem
-```
-
-## Start etcd
-
-```bash
-# Download and install etcd (v3.5.12 or higher)
-ETCD_VERSION="v3.5.12"
-ETCD_ARCH="linux-arm64"
-wget https://github.com/etcd-io/etcd/releases/download/${ETCD_VERSION}/etcd-${ETCD_VERSION}-${ETCD_ARCH}.tar.gz
-tar -xvf etcd-${ETCD_VERSION}-${ETCD_ARCH}.tar.gz
-cd etcd-${ETCD_VERSION}-${ETCD_ARCH}
-sudo cp etcd etcdctl /usr/local/bin/
-
-# Start etcd
-etcd \
-  --name etcd-single \
-  --data-dir /tmp/etcd-data \
-  --listen-client-urls http://0.0.0.0:2379 \
-  --advertise-client-urls http://0.0.0.0:2379 \
-  --listen-peer-urls http://0.0.0.0:2380 \
-  --initial-advertise-peer-urls http://0.0.0.0:2380 \
-  --initial-cluster etcd-single=http://0.0.0.0:2380 &
-
-# Verify etcd is running
-etcdctl --endpoints "127.0.0.1:2379" put key "value"
-etcdctl --endpoints "127.0.0.1:2379" get key
-```
-
-For production environments, refer to the
-[official etcd clustering documentation](https://etcd.io/docs/current/op-guide/clustering/).
-
-## Start Datasystem Worker
-
-```bash
-# Replace ${ETCD_IP} with etcd node IP, ${WORKER_IP} with local node IP
-dscli start -w \
-  --worker_address "${WORKER_IP}:31501" \
-  --etcd_address "${ETCD_IP}:2379" \
-  --shared_memory_size_mb 20480
-```
-
-To stop the worker:
-
-```bash
-dscli stop --worker_address "${WORKER_IP}:31501"
-```
-
-## Configuration
-
-Define the connector in runtime:
-
-```yaml
-runtime:
-  connectors:
-    connector_of_yuanrong:
-      name: YuanrongConnector
-      extra:
-        host: "127.0.0.1"
-        port: 31501
-        get_sub_timeout_ms: 1000
-```
-
-Wire stages to the connector:
-
-```yaml
-stage_args:
-  - stage_id: 0
-    output_connectors:
-      to_stage_1: connector_of_yuanrong
-
-  - stage_id: 1
-    input_connectors:
-      from_stage_0: connector_of_yuanrong
-```
-
-Parameters:
-
-- host: datasystem worker host.
-- port: datasystem worker port.
-- get_sub_timeout_ms: get timeout in milliseconds (0 for no timeout).
-
-For more details, refer to the
-[Yuanrong Datasystem repository](https://atomgit.com/openeuler/yuanrong-datasystem).

docs/features/sleep_mode.md

@@ -1,39 +0,0 @@
-# Sleep Mode
-
-vLLM-Omni’s **Sleep Mode** allows you to temporarily release most GPU memory used by a model—such as model weights and key-value (KV) caches (for autoregressive models)—**without stopping the server or unloading the Docker container**.
-
-This feature is inherited from [vLLM’s Sleep Mode](https://blog.vllm.ai/2025/10/26/sleep-mode.html), which provides zero-reload model switching for multi-model serving.  
-
-It is especially useful in **RLHF**, **training**, or **cost-saving scenarios**, where GPU resources must be freed between inference workloads.
-
----
-
-## Omni Model
-
-Omni model inherit the feature from vLLM' Sleep Mode
-
-This means:
-
-- Support both Level 1 and Level 2 sleep, allow to release and reset both model weights and KV Cache
-
-## Diffusion Model Extension
-
-We added Sleep Mode support for **diffusion models**, which previously lacked this functionality.  
-In diffusion pipelines, this currently only offloads **model weight memory**, as these models typically do not use KV caches.
-
-This means:
-
-- Diffusion models can now enter Level 1 sleep.
-- Pipeline states (e.g., noise schedulers, buffers) remain intact after waking.
-- Useful for releasing VRAM between image generation or training cycles.
-
----
-
-## Enable sleep mode
-To enable sleep mode, set the `enable_sleep_mode` in `engine_args` to `True`
-
-
-Example:
-```python
-omni = Omni(model=...,enable_sleep_mode=True)
-```

docs/getting_started/installation/gpu.md

@@ -49,18 +49,15 @@ vLLM-Omni is a Python library that supports the following GPU variants. The libr
 
 ## Set up using Docker
 
-### Pre-built images
-
-=== "NVIDIA CUDA"
-
-    --8<-- "docs/getting_started/installation/gpu/cuda.inc.md:pre-built-images"
-
-=== "AMD ROCm"
-
-    --8<-- "docs/getting_started/installation/gpu/rocm.inc.md:pre-built-images"
 
 ### Build your own docker image
 
 === "AMD ROCm"
 
     --8<-- "docs/getting_started/installation/gpu/rocm.inc.md:build-docker"
+
+### Pre-built images
+
+=== "NVIDIA CUDA"
+
+    --8<-- "docs/getting_started/installation/gpu/cuda.inc.md:pre-built-images"

docs/getting_started/installation/gpu/cuda.inc.md

@@ -17,12 +17,12 @@ Therefore, it is recommended to install vLLM and vLLM-Omni with a **fresh new**
 # --8<-- [start:pre-built-wheels]
 
 #### Installation of vLLM
-Note: Pre-built wheels are currently only available for vLLM-Omni 0.11.0rc1, 0.12.0rc1, 0.14.0rc1, 0.14.0. For the latest version, please [build from source](https://docs.vllm.ai/projects/vllm-omni/en/latest/getting_started/installation/gpu/#build-wheel-from-source).
+Note: Pre-built wheels are currently only available for vLLM-Omni 0.11.0rc1 and 0.12.0rc1. For the latest version, please [build from source](https://docs.vllm.ai/projects/vllm-omni/en/latest/getting_started/installation/gpu/#build-wheel-from-source).
 
 
 vLLM-Omni is built based on vLLM. Please install it with command below.
 ```bash
-uv pip install vllm==0.14.0 --torch-backend=auto
+uv pip install vllm==0.12.0 --torch-backend=auto
 ```
 
 #### Installation of vLLM-Omni
@@ -36,13 +36,13 @@ uv pip install vllm-omni
 # --8<-- [start:build-wheel-from-source]
 
 #### Installation of vLLM
-If you do not need to modify source code of vLLM, you can directly install the stable 0.14.0 release version of the library
+If you do not need to modify source code of vLLM, you can directly install the stable 0.12.0 release version of the library
 
 ```bash
-uv pip install vllm==0.14.0 --torch-backend=auto
+uv pip install vllm==0.12.0 --torch-backend=auto
 ```
 
-The release 0.14.0 of vLLM is based on PyTorch 2.9.0 which requires CUDA 12.9 environment.
+The release 0.12.0 of vLLM is based on PyTorch 2.9.0 which requires CUDA 12.9 environment.
 
 #### Installation of vLLM-Omni
 Since vllm-omni is rapidly evolving, it's recommended to install it from source
@@ -58,11 +58,11 @@ If you want to check, modify or debug with source code of vLLM, install the libr
 ```bash
 git clone https://github.com/vllm-project/vllm.git
 cd vllm
-git checkout v0.14.0
+git checkout v0.12.0
 ```
 Set up environment variables to get pre-built wheels. If there are internet problems, just download the whl file manually. And set `VLLM_PRECOMPILED_WHEEL_LOCATION` as your local absolute path of whl file.
 ```bash
-export VLLM_PRECOMPILED_WHEEL_LOCATION=https://github.com/vllm-project/vllm/releases/download/v0.14.0/vllm-0.14.0-cp38-abi3-manylinux_2_31_x86_64.whl
+export VLLM_PRECOMPILED_WHEEL_LOCATION=https://github.com/vllm-project/vllm/releases/download/v0.12.0/vllm-0.12.0-cp38-abi3-manylinux_2_31_x86_64.whl
 ```
 Install vllm with command below (If you have no existing PyTorch).
 ```bash
@@ -93,7 +93,7 @@ docker run --runtime nvidia --gpus 2 \
     --env "HF_TOKEN=$HF_TOKEN" \
     -p 8091:8091 \
     --ipc=host \
-    vllm/vllm-omni:v0.14.0 \
+    vllm/vllm-omni:v0.12.0rc1 \
     --model Qwen/Qwen3-Omni-30B-A3B-Instruct --port 8091
 ```
 

docs/getting_started/installation/gpu/rocm.inc.md

@@ -23,30 +23,19 @@ vLLM-Omni current recommends the steps in under setup through Docker Images.
 DOCKER_BUILDKIT=1 docker build -f docker/Dockerfile.rocm -t vllm-omni-rocm .
 ```
 
+If you want to specify which GPU Arch to build for to cutdown build time:
+
+```bash
+DOCKER_BUILDKIT=1 docker build \
+  -f docker/Dockerfile.rocm \
+  --build-arg PYTORCH_ROCM_ARCH="gfx942;gfx950" \
+  -t vllm-omni-rocm .
+```
+
 #### Launch the docker image
 
-##### Launch with OpenAI API Server
-
 ```
-docker run --rm \
---group-add=video \
---ipc=host \
---cap-add=SYS_PTRACE \
---security-opt seccomp=unconfined \
---device /dev/kfd \
---device /dev/dri \
--v ~/.cache/huggingface:/root/.cache/huggingface \
---env "HF_TOKEN=$HF_TOKEN" \
--p 8091:8091 \
---ipc=host \
-vllm-omni-rocm \
---model Qwen/Qwen3-Omni-30B-A3B-Instruct --port 8091
-```
-
-##### Launch with interactive session for development
-
-```
-docker run --rm -it \
+docker run -it \
 --network=host \
 --group-add=video \
 --ipc=host \
@@ -55,51 +44,12 @@ docker run --rm -it \
 --device /dev/kfd \
 --device /dev/dri \
 -v <path/to/model>:/app/model \
--v ~/.cache/huggingface:/root/.cache/huggingface \
---entrypoint bash \
-vllm-omni-rocm
+vllm-omni-rocm \
+bash
 ```
 
 # --8<-- [end:build-docker]
 
 # --8<-- [start:pre-built-images]
 
-vLLM-Omni offers an official docker image for deployment. These images are built on top of vLLM docker images and available on Docker Hub as [vllm/vllm-omni-rocm](https://hub.docker.com/r/vllm/vllm-omni-rocm/tags). The version of vLLM-Omni indicates which release of vLLM it is based on.
-
-#### Launch vLLM-Omni Server
-Here's an example deployment command that has been verified on 2 x MI300's:
-```bash
-docker run --rm \
-  --group-add=video \
-  --ipc=host \
-  --cap-add=SYS_PTRACE \
-  --security-opt seccomp=unconfined \
-  --device /dev/kfd \
-  --device /dev/dri \
-  -v <path/to/model>:/app/model \
-  -v ~/.cache/huggingface:/root/.cache/huggingface \
-  --env "HF_TOKEN=$HF_TOKEN" \
-  -p 8091:8091 \
-  vllm/vllm-omni-rocm:v0.14.0 \
-  --model Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091
-```
-
-#### Launch an interactive terminal with prebuilt docker image.
-If you want to run in dev environment you can launch the docker image as follows:
-```bash
-docker run --rm -it \
-  --network=host \
-  --group-add=video \
-  --ipc=host \
-  --cap-add=SYS_PTRACE \
-  --security-opt seccomp=unconfined \
-  --device /dev/kfd \
-  --device /dev/dri \
-  -v <path/to/model>:/app/model \
-  -v ~/.cache/huggingface:/root/.cache/huggingface \
-  --env "HF_TOKEN=$HF_TOKEN" \
-  --entrypoint bash \
-  vllm/vllm-omni-rocm:v0.14.0
-```
-
 # --8<-- [end:pre-built-images]

docs/getting_started/installation/npu/npu.inc.md

@@ -13,10 +13,10 @@ export DEVICE0=/dev/davinci0
 export DEVICE1=/dev/davinci1
 # Update the vllm-ascend image
 # Atlas A2:
-# export IMAGE=quay.io/ascend/vllm-ascend:v0.14.0
+# export IMAGE=quay.io/ascend/vllm-ascend:v0.12.0rc1
 # Atlas A3:
-# export IMAGE=quay.io/ascend/vllm-ascend:v0.14.0-a3
-export IMAGE=quay.io/ascend/vllm-ascend:v0.14.0
+# export IMAGE=quay.io/ascend/vllm-ascend:v0.12.0rc1-a3
+export IMAGE=quay.io/ascend/vllm-ascend:v0.12.0rc1
 docker run --rm \
     --name vllm-omni-npu \
     --shm-size=1g \
@@ -42,7 +42,7 @@ source ~/.bashrc
 
 # Inside the container, install vLLM-Omni from source
 cd /vllm-workspace
-git clone -b v0.14.0 https://github.com/vllm-project/vllm-omni.git
+git clone -b v0.12.0rc1 https://github.com/vllm-project/vllm-omni.git
 cd vllm-omni
 pip install -v -e .
 export VLLM_WORKER_MULTIPROC_METHOD=spawn

docs/getting_started/quickstart.md

@@ -17,13 +17,7 @@ For installation on GPU from source:
 ```bash
 uv venv --python 3.12 --seed
 source .venv/bin/activate
-
-# On CUDA
-uv pip install vllm==0.14.0 --torch-backend=auto
-
-# On ROCm
-uv pip install vllm==0.14.0 --extra-index-url https://wheels.vllm.ai/rocm/0.14.0/rocm700
-
+uv pip install vllm==0.12.0 --torch-backend=auto
 git clone https://github.com/vllm-project/vllm-omni.git
 cd vllm-omni
 uv pip install -e .
@@ -46,46 +40,6 @@ if __name__ == "__main__":
     images[0].save("coffee.png")
 ```
 
-You can pass a list of prompts and wait for them to process altogether, shown below.
-
-!!! info
-
-    However, it is not currently recommended to do so
-    because not all models support batch inference,
-    and batch requesting mostly does not provide significant performance improvement (despite the impression that it does).
-    This feature is primarily for the sake of interface compatibility with vLLM and to allow for future improvements.
-
-```python
-from vllm_omni.entrypoints.omni import Omni
-
-if __name__ == "__main__":
-    omni = Omni(
-        model="Tongyi-MAI/Z-Image-Turbo",
-        # stage_configs_path="./stage-config.yaml",  # See below
-    )
-    prompts = [
-        "a cup of coffee on a table",
-        "a toy dinosaur on a sandy beach",
-        "a fox waking up in bed and yawning",
-    ]
-    omni_outputs = omni.generate(prompts)
-    for i_prompt, prompt_output in enumerate(omni_outputs):
-        this_request_output = prompt_output.request_output[0]
-        this_images = this_request_output.images
-        for i_image, image in enumerate(this_images):
-            image.save(f"p{i_prompt}-img{i_image}.jpg")
-            print("saved to", f"p{i_prompt}-img{i_image}.jpg")
-            # saved to p0-img0.jpg
-            # saved to p1-img0.jpg
-            # saved to p2-img0.jpg
-```
-
-!!! info
-
-    For diffusion pipelines, the stage config field `stage_args.[].runtime.max_batch_size` is 1 by default, and the input
-    list is sliced into single-item requests before feeding into the diffusion pipeline. For models that do internally support
-    batched inputs, you can [modify this configuration](../configuration/stage_configs.md) to let the model accept a longer batch of prompts.
-
 For more usages, please refer to [offline inference](../user_guide/examples/offline_inference/qwen2_5_omni.md)
 
 ## Online Serving with OpenAI-Completions API

docs/mkdocs/hooks/generate_api_readme.py

@@ -34,10 +34,10 @@ CATEGORIES = {
         "name": "Core",
         "description": "Core scheduling and caching components.",
     },
-    # "model_executor": {
-    #     "name": "Model Executor",
-    #     "description": "Model execution components.",
-    # },
+    "model_executor": {
+        "name": "Model Executor",
+        "description": "Model execution components.",
+    },
     "config": {
         "name": "Configuration",
         "description": "Configuration classes.",

docs/models/supported_models.md

@@ -21,7 +21,6 @@ th {
 | `Qwen2_5OmniForConditionalGeneration` | Qwen2.5-Omni | `Qwen/Qwen2.5-Omni-7B`, `Qwen/Qwen2.5-Omni-3B` |
 | `BagelForConditionalGeneration` | BAGEL (DiT-only) | `ByteDance-Seed/BAGEL-7B-MoT` |
 | `QwenImagePipeline` | Qwen-Image | `Qwen/Qwen-Image` |
-| `QwenImagePipeline` | Qwen-Image-2512 | `Qwen/Qwen-Image-2512` |
 | `QwenImageEditPipeline` | Qwen-Image-Edit | `Qwen/Qwen-Image-Edit` |
 | `QwenImageEditPlusPipeline` | Qwen-Image-Edit-2509 | `Qwen/Qwen-Image-Edit-2509` |
 | `QwenImageLayeredPipeline` | Qwen-Image-Layered | `Qwen/Qwen-Image-Layered` |
@@ -32,12 +31,6 @@ th {
 |`LongcatImagePipeline` | LongCat-Image | `meituan-longcat/LongCat-Image` |
 |`LongCatImageEditPipeline` | LongCat-Image-Edit | `meituan-longcat/LongCat-Image-Edit` |
 |`StableDiffusion3Pipeline` | Stable-Diffusion-3 | `stabilityai/stable-diffusion-3.5-medium` |
-|`Flux2KleinPipeline` | FLUX.2-klein | `black-forest-labs/FLUX.2-klein-4B`, `black-forest-labs/FLUX.2-klein-9B` |
-|`FluxPipeline` | FLUX.1-dev | `black-forest-labs/FLUX.1-dev` |
-|`StableAudioPipeline` | Stable-Audio-Open | `stabilityai/stable-audio-open-1.0` |
-|`Qwen3TTSForConditionalGeneration` | Qwen3-TTS-12Hz-1.7B-CustomVoice | `Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice` |
-|`Qwen3TTSForConditionalGeneration` | Qwen3-TTS-12Hz-1.7B-VoiceDesign | `Qwen/Qwen3-TTS-12Hz-1.7B-VoiceDesign` |
-|`Qwen3TTSForConditionalGeneration` | Qwen3-TTS-12Hz-1.7B-Base | `Qwen/Qwen3-TTS-12Hz-0.6B-Base` |
 
 
 ## List of Supported Models for NPU

docs/serving/image_edit_api.md

@@ -1,205 +0,0 @@
-# Image Edit API
-
-vLLM-Omni provides an OpenAI DALL-E compatible API for image edit using diffusion models.
-
-Each server instance runs a single model (specified at startup via `vllm serve <model> --omni`).
-
-## Quick Start
-
-### Start the Server
-
-For example...
-
-```bash
-# Qwen-Image
-vllm serve Qwen/Qwen-Image-Edit-2511 --omni --port 8000
-
-
-### Generate Images
-
-**Using curl:**
-
-```bash
-curl -s -D >(grep -i x-request-id >&2) \
-  -o >(jq -r '.data[0].b64_json' | base64 --decode > gift-basket.png) \
-  -X POST "http://localhost:8000/v1/images/edits" \
-  -F "model=xxx" \
-  -F "image=@./xx.png" \
-  -F "prompt='this bear is wearing sportwear. holding a basketball, and bending one leg.'" \
-  -F "size=1024x1024" \
-  -F "output_format=png"
-```
-
-
-**Using OpenAI SDK:**
-
-```python
-import base64
-from openai import OpenAI
-from pathlib import Path
-client = OpenAI(
-    api_key="None",
-    base_url="http://localhost:8000/v1"
-)
-
-input_image_url = "https://vllm-public-assets.s3.us-west-2.amazonaws.com/omni-assets/qwen-bear.png"
-
-result = client.images.edit(
-    image=[],
-    model="Qwen-Image-Edit-2511",
-    prompt="Change the bears in the two input images into walking together.",
-    size='512x512',
-    stream=False,
-    output_format='jpeg',
-    # url格式
-    extra_body={
-        "url": [input_image_url1,input_image_url],
-        "num_inference_steps": 50,
-        "guidance_scale": 1,
-        "seed": 777,
-    }
-)
-
-image_base64 = result.data[0].b64_json
-image_bytes = base64.b64decode(image_base64)
-
-# Save the image to a file
-with open("edit_out_http.jpeg", "wb") as f:
-    f.write(image_bytes)
-```
-
-## API Reference
-
-### Endpoint
-
-```
-POST /v1/images/edits
-Content-Type: multipart/form-data
-```
-
-### Request Parameters
-
-#### OpenAI Standard Parameters
-
-| Parameter | Type | Default | Description |
-|-----------|------|---------|-------------|
-| `prompt` | string | **required** | A text description of the desired image |
-| `model` | string | server's model | Model to use (optional, should match server if specified) |
-| `image` | string or array | **required** | The image(s) to edit. |
-| `n` | integer | 1 | Number of images to generate (1-10) |
-| `size` | string | "auto" | Image dimensions in WxH format (e.g., "1024x1024", "512x512"), when set to auto, it decide size from first input image. |
-| `response_format` | string | "b64_json" | Response format (only "b64_json" supported) |
-| `user` | string | null | User identifier for tracking |
-| `output_format` | string | "png" | The format in which the generated images are returned. Must be one of "png", "jpg", "jpeg", "webp". |
-| `output_compression` | integer | 100 | The compression level (0-100%) for the generated images. |
-| `background` | string or null | "auto" | Allows to set transparency for the background of the generated image(s).
-
-#### vllm-omni Extension Parameters
-
-| Parameter | Type | Default | Description |
-|-----------|------|---------|-------------|
-| `url` | string or array | None | The image(s) to edit. |
-| `negative_prompt` | string | null | Text describing what to avoid in the image |
-| `num_inference_steps` | integer | model defaults | Number of diffusion steps |
-| `guidance_scale` | float | model defaults | Classifier-free guidance scale (typically 0.0-20.0) |
-| `true_cfg_scale` | float | model defaults | True CFG scale (model-specific parameter, may be ignored if not supported) |
-| `seed` | integer | null | Random seed for reproducibility |
-
-### Response Format
-
-```json
-{
-  "created": 1701234567,
-  "data": [
-    {
-      "b64_json": "<base64-encoded PNG>",
-      "url": null,
-      "revised_prompt": null
-    }
-  ],
-  "output_format": null,
-  "size": null,
-}
-```
-
-## Examples
-
-### Multiple Images input
-
-```bash
-curl -s -D >(grep -i x-request-id >&2) \
-  -o >(jq -r '.data[0].b64_json' | base64 --decode > gift-basket.png) \
-  -X POST "http://localhost:8000/v1/images/edits" \
-  -F "model=xxx" \
-  -F "image=@xx.png" \
-  -F "image=@xx.png"
-  -F "prompt='this bear is wearing sportwear. holding a basketball, and bending one leg.'" \
-  -F "size=1024x1024" \
-  -F "output_format=png"
-```
-
-
-## Parameter Handling
-
-The API passes parameters directly to the diffusion pipeline without model-specific transformation:
-
-- **Default values**: When parameters are not specified, the underlying model uses its own defaults
-- **Pass-through design**: User-provided values are forwarded directly to the diffusion engine
-- **Minimal validation**: Only basic type checking and range validation at the API level
-
-### Parameter Compatibility
-
-The API passes parameters directly to the diffusion pipeline without model-specific validation.
-
-- Unsupported parameters may be silently ignored by the model
-- Incompatible values will result in errors from the underlying pipeline
-- Recommended values vary by model - consult model documentation
-
-**Best Practice:** Start with the model's recommended parameters, then adjust based on your needs.
-
-## Error Responses
-
-### 400 Bad Request
-
-Invalid parameters (e.g., model mismatch):
-
-```json
-{
-  "detail": "Invalid size format: '1024x'. Expected format: 'WIDTHxHEIGHT' (e.g., '1024x1024')."
-}
-```
-
-### 422 Unprocessable Entity
-
-Validation errors (missing required fields):
-
-```json
-{
-  "detail": "Field 'image' or 'url' is required"
-}
-```
-
-## Troubleshooting
-
-### Server Not Running
-
-```bash
-# Check if server is responding
-curl -X http://localhost:8000/v1/images/edit \
-  -F "prompt='test'"
-```
-
-### Out of Memory
-
-If you encounter OOM errors:
-1. Reduce image size: `"size": "512x512"`
-2. Reduce inference steps: `"num_inference_steps": 25`
-
-## Development
-
-Enable debug logging to see prompts and generation details:
-
-```bash
-vllm serve Qwen/Qwen-Image-Edit-2511 --omni \
-  --uvicorn-log-level debug
-```

docs/user_guide/acceleration/cache_dit_acceleration.md

@@ -18,7 +18,6 @@ Enable cache-dit acceleration by simply setting `cache_backend="cache_dit"`. Cac
 
 ```python
 from vllm_omni.entrypoints.omni import Omni
-from vllm_omni.inputs.data import OmniDiffusionSamplingParams
 
 # Simplest way: just enable cache-dit with default parameters
 omni = Omni(
@@ -28,7 +27,7 @@ omni = Omni(
 
 images = omni.generate(
     "a beautiful landscape",
-    OmniDiffusionSamplingParams(num_inference_steps=50),
+    num_inference_steps=50,
 )
 ```
 
@@ -70,7 +69,7 @@ vllm serve Qwen/Qwen-Image --omni --port 8091 \
 
 ## Acceleration Methods
 
-For comprehensive illustration, please view cache-dit [User_Guide](https://cache-dit.readthedocs.io/en/latest/user_guide/OVERVIEWS/)
+For comprehensive illustration, please view cache-dit [User_Guide](https://github.com/vipshop/cache-dit/blob/main/docs/User_Guide.md)
 
 ### 1. DBCache (Dual Block Cache)
 
@@ -223,6 +222,6 @@ To test another model, you can modify `--model` with the target model identifier
 
 ## Additional Resources
 
-- [Cache-DiT User Guide](https://cache-dit.readthedocs.io/en/latest/user_guide/OVERVIEWS/)
-- [Cache-DiT Benchmark](https://cache-dit.readthedocs.io/en/latest/benchmark/HYBRID_CACHE/)
-- [DBCache Technical Details](https://cache-dit.readthedocs.io/en/latest/user_guide/CACHE_API/)
+- [Cache-DiT User Guide](https://github.com/vipshop/cache-dit/blob/main/docs/User_Guide.md)
+- [Cache-DiT Benchmark](https://github.com/vipshop/cache-dit/tree/main/bench)
+- [DBCache Technical Details](https://github.com/vipshop/cache-dit/blob/main/docs/DBCache.md)

docs/user_guide/acceleration/parallelism_acceleration.md

@@ -0,0 +1,274 @@
+# Parallelism Acceleration Guide
+
+This guide includes how to use parallelism methods in vLLM-Omni to speed up diffusion model inference as well as reduce the memory requirement on each device.
+
+## Overview
+
+The following parallelism methods are currently supported in vLLM-Omni:
+
+1. DeepSpeed Ulysses Sequence Parallel (DeepSpeed Ulysses-SP) ([arxiv paper](https://arxiv.org/pdf/2309.14509)): Ulysses-SP splits the input along the sequence dimension and uses all-to-all communication to allow each device to compute only a subset of attention heads.
+
+2. [Ring-Attention](#ring-attention) - splits the input along the sequence dimension and uses ring-based P2P communication to accumulate attention results, keeping the sequence dimension sharded
+
+
+The following table shows which models are currently supported by parallelism method:
+
+### ImageGen
+
+| Model | Model Identifier | Ulysses-SP | Ring-SP |
+|-------|------------------|-----------|---------|
+| **LongCat-Image** | `meituan-longcat/LongCat-Image` | ❌ | ❌ |
+| **LongCat-Image-Edit** | `meituan-longcat/LongCat-Image-Edit` | ❌ | ❌ |
+| **Ovis-Image** | `OvisAI/Ovis-Image` | ❌ | ❌ |
+| **Qwen-Image** | `Qwen/Qwen-Image` | ✅ | ✅ |
+| **Qwen-Image-Edit** | `Qwen/Qwen-Image-Edit` | ✅ | ✅ |
+| **Qwen-Image-Edit-2509** | `Qwen/Qwen-Image-Edit-2509` | ✅ | ✅ |
+| **Qwen-Image-Layered** | `Qwen/Qwen-Image-Layered` | ✅ | ✅ |
+| **Z-Image** | `Tongyi-MAI/Z-Image-Turbo` | ❌ | ❌ |
+
+### VideoGen
+
+| Model | Model Identifier | Ulysses-SP | Ring-SP |
+|-------|------------------|-----------|---------|
+| **Wan2.2** | `Wan-AI/Wan2.2-T2V-A14B-Diffusers` | ❌ | ❌ |
+
+### Sequence Parallelism
+
+#### Ulysses-SP
+
+##### Offline Inference
+
+An example of offline inference script using [Ulysses-SP](https://arxiv.org/pdf/2309.14509) is shown below:
+```python
+from vllm_omni import Omni
+from vllm_omni.diffusion.data import DiffusionParallelConfig
+ulysses_degree = 2
+
+omni = Omni(
+    model="Qwen/Qwen-Image",
+    parallel_config=DiffusionParallelConfig(ulysses_degree=2)
+)
+
+outputs = omni.generate(prompt="A cat sitting on a windowsill", num_inference_steps=50, width=2048, height=2048)
+```
+
+See `examples/offline_inference/text_to_image/text_to_image.py` for a complete working example.
+
+##### Online Serving
+
+You can enable Ulysses-SP in online serving for diffusion models via `--usp`:
+
+```bash
+# Text-to-image (requires >= 2 GPUs)
+vllm serve Qwen/Qwen-Image --omni --port 8091 --usp 2
+```
+
+##### Benchmarks
+!!! note "Benchmark Disclaimer"
+    These benchmarks are provided for **general reference only**. The configurations shown use default or common parameter settings and have not been exhaustively optimized for maximum performance. Actual performance may vary based on:
+
+    - Specific model and use case
+    - Hardware configuration
+    - Careful parameter tuning
+    - Different inference settings (e.g., number of steps, image resolution)
+
+
+To measure the parallelism methods, we run benchmarks with **Qwen/Qwen-Image** model generating images (**2048x2048** as long sequence input) with 50 inference steps. The hardware devices are NVIDIA H800 GPUs. `sdpa` is the attention backends.
+
+| Configuration | Ulysses degree |Generation Time | Speedup |
+|---------------|----------------|---------|---------|
+| **Baseline (diffusers)** | - | 112.5s | 1.0x |
+| Ulysses-SP  |  2  |  65.2s | 1.73x |
+| Ulysses-SP  |  4  | 39.6s | 2.84x |
+| Ulysses-SP  |  8  | 30.8s | 3.65x |
+
+#### Ring-Attention
+
+Ring-Attention ([arxiv paper](https://arxiv.org/abs/2310.01889)) splits the input along the sequence dimension and uses ring-based P2P communication to accumulate attention results. Unlike Ulysses-SP which uses all-to-all communication, Ring-Attention keeps the sequence dimension sharded throughout the computation and circulates Key/Value blocks through a ring topology.
+
+##### Offline Inference
+
+An example of offline inference script using Ring-Attention is shown below:
+```python
+from vllm_omni import Omni
+from vllm_omni.diffusion.data import DiffusionParallelConfig
+ring_degree = 2
+
+omni = Omni(
+    model="Qwen/Qwen-Image",
+    parallel_config=DiffusionParallelConfig(ring_degree=2)
+)
+
+outputs = omni.generate(prompt="A cat sitting on a windowsill", num_inference_steps=50, width=2048, height=2048)
+```
+
+See `examples/offline_inference/text_to_image/text_to_image.py` for a complete working example.
+
+
+##### Online Serving
+
+You can enable Ring-Attention in online serving for diffusion models via `--ring`:
+
+```bash
+# Text-to-image (requires >= 2 GPUs)
+vllm serve Qwen/Qwen-Image --omni --port 8091 --ring 2
+```
+
+##### Benchmarks
+!!! note "Benchmark Disclaimer"
+    These benchmarks are provided for **general reference only**. The configurations shown use default or common parameter settings and have not been exhaustively optimized for maximum performance. Actual performance may vary based on:
+
+    - Specific model and use case
+    - Hardware configuration
+    - Careful parameter tuning
+    - Different inference settings (e.g., number of steps, image resolution)
+
+
+To measure the parallelism methods, we run benchmarks with **Qwen/Qwen-Image** model generating images (**1024x1024** as long sequence input) with 50 inference steps. The hardware devices are NVIDIA A100 GPUs. `flash_attn` is the attention backends.
+
+| Configuration | Ring degree |Generation Time | Speedup |
+|---------------|----------------|---------|---------|
+| **Baseline (diffusers)** | - | 45.2s | 1.0x |
+| Ring-Attention  |  2  |  29.9s | 1.51x |
+| Ring-Attention  |  4  | 23.3s | 1.94x |
+
+
+#### Hybrid Ulysses + Ring
+
+You can combine both Ulysses-SP and Ring-Attention for larger scale parallelism. The total sequence parallel size equals `ulysses_degree × ring_degree`.
+
+##### Offline Inference
+
+```python
+from vllm_omni import Omni
+from vllm_omni.diffusion.data import DiffusionParallelConfig
+
+# Hybrid: 2 Ulysses × 2 Ring = 4 GPUs total
+omni = Omni(
+    model="Qwen/Qwen-Image",
+    parallel_config=DiffusionParallelConfig(ulysses_degree=2, ring_degree=2)
+)
+
+outputs = omni.generate(prompt="A cat sitting on a windowsill", num_inference_steps=50, width=2048, height=2048)
+```
+
+##### Online Serving
+
+```bash
+# Text-to-image (requires >= 4 GPUs)
+vllm serve Qwen/Qwen-Image --omni --port 8091 --usp 2 --ring 2
+```
+
+##### Benchmarks
+!!! note "Benchmark Disclaimer"
+    These benchmarks are provided for **general reference only**. The configurations shown use default or common parameter settings and have not been exhaustively optimized for maximum performance. Actual performance may vary based on:
+
+    - Specific model and use case
+    - Hardware configuration
+    - Careful parameter tuning
+    - Different inference settings (e.g., number of steps, image resolution)
+
+
+To measure the parallelism methods, we run benchmarks with **Qwen/Qwen-Image** model generating images (**1024x1024** as long sequence input) with 50 inference steps. The hardware devices are NVIDIA A100 GPUs. `flash_attn` is the attention backends.
+
+| Configuration | Ulysses degree | Ring degree | Generation Time | Speedup |
+|---------------|----------------|-------------|-----------------|---------|
+| **Baseline (diffusers)** | - | - | 45.2s | 1.0x |
+| Hybrid Ulysses + Ring  |  2  |  2  |  24.3s | 1.87x |
+
+
+##### How to parallelize a new model
+
+If a diffusion model has been deployed in vLLM-Omni and supports single-card inference, you can refer to the following instructions to parallelize it with [Ulysses-SP](https://arxiv.org/pdf/2309.14509).
+
+This section uses **Qwen-Image** (`QwenImageTransformer2DModel`) as the reference implementation. Qwen-Image is a **dual-stream** transformer (text + image) that performs **joint attention** across the concatenated sequences. Because of that, when enabling sequence parallel you typically:
+
+- Chunk **image tokens** (`hidden_states`) across SP ranks along the **sequence dimension**.
+- Keep **text embeddings** (`encoder_hidden_states`) **replicated** on all SP ranks for correctness (unless you implement full joint-SP semantics and explicitly split text too).
+- Chunk **image RoPE freqs** to match the chunked image tokens.
+- `all_gather` the final output back along the sequence dimension.
+
+First, add the sequence-parallel helpers and (for Qwen-Image) the forward-context flag. Then, in the transformer's `forward()`:
+
+- Chunk `hidden_states` by SP world size.
+- Set `get_forward_context().split_text_embed_in_sp = False` because Qwen-Image uses joint attention and we would use a full text embedding in each rank.
+- After `pos_embed`, chunk `img_freqs` on `dim=0` (token axis) to match the chunked image tokens; only chunk `txt_freqs` if you explicitly split text embeddings in SP.
+
+Taking `vllm_omni/diffusion/models/qwen_image/qwen_image_transformer.py` as an example:
+
+```diff
+from vllm_omni.diffusion.distributed.parallel_state import (
+    get_sequence_parallel_rank,
+    get_sequence_parallel_world_size,
+    get_sp_group,
+)
+from vllm_omni.diffusion.forward_context import get_forward_context
+
+class QwenImageTransformer2DModel(...):
+    def forward(
+        self,
+        hidden_states: torch.Tensor,
+        encoder_hidden_states: torch.Tensor = None,
+        encoder_hidden_states_mask: torch.Tensor = None,
+        timestep: torch.LongTensor = None,
+        img_shapes: list[tuple[int, int, int]] | None = None,
+        txt_seq_lens: list[int] | None = None,
+        ...
+    ):
++       if self.parallel_config.sequence_parallel_size > 1:
++           # Chunk image tokens along sequence dimension.
++           hidden_states = torch.chunk(
++               hidden_states,
++               get_sequence_parallel_world_size(),
++               dim=-2,
++           )[get_sequence_parallel_rank()]
++
++           # Qwen-Image uses *dual-stream* (text + image) and runs *joint attention*.
++           # Text embeddings should be replicated across SP ranks for correctness.
++           get_forward_context().split_text_embed_in_sp = False
+
+        hidden_states = self.img_in(hidden_states)
+
+        ...
+        image_rotary_emb = self.pos_embed(img_shapes, txt_seq_lens, device=hidden_states.device)
+
++       def get_rotary_emb_chunk(freqs):
++           return torch.chunk(freqs, get_sequence_parallel_world_size(), dim=0)[
++               get_sequence_parallel_rank()
++           ]
++
++       if self.parallel_config.sequence_parallel_size > 1:
++           img_freqs, txt_freqs = image_rotary_emb
++           img_freqs = get_rotary_emb_chunk(img_freqs)
++           if get_forward_context().split_text_embed_in_sp:
++               txt_freqs = get_rotary_emb_chunk(txt_freqs)
++           image_rotary_emb = (img_freqs, txt_freqs)
+```
+
+Next, at the end of the `forward()` function, call `get_sp_group().all_gather` to gather the chunked outputs across devices and concatenate them along the sequence dimension (matching the earlier `dim=-2` chunking):
+
+```diff
+class QwenImageTransformer2DModel(...):
+    def forward(...):
+        ...
+        hidden_states = self.norm_out(hidden_states, temb)
+        output = self.proj_out(hidden_states)
+
++       if self.parallel_config.sequence_parallel_size > 1:
++           output = get_sp_group().all_gather(output, dim=-2)
+        return Transformer2DModelOutput(sample=output)
+```
+
+Finally, you can set the parallel configuration and pass it to `Omni` and start parallel inference with:
+```diff
+from vllm_omni import Omni
++from vllm_omni.diffusion.data import DiffusionParallelConfig
+ulysses_degree = 2
+
+omni = Omni(
+    model="Qwen/Qwen-Image",
++    parallel_config=DiffusionParallelConfig(ulysses_degree=2)
+)
+
+outputs = omni.generate(prompt="A cat sitting on a windowsill", num_inference_steps=50)
+```

docs/user_guide/acceleration/teacache.md

@@ -8,7 +8,6 @@ Enable TeaCache by setting `cache_backend` to `"tea_cache"`:
 
 ```python
 from vllm_omni import Omni
-from vllm_omni.inputs.data import OmniDiffusionSamplingParams
 
 # Simple configuration - model_type is automatically extracted from pipeline.__class__.__name__
 omni = Omni(
@@ -18,12 +17,7 @@ omni = Omni(
         "rel_l1_thresh": 0.2  # Optional, defaults to 0.2
     }
 )
-outputs = omni.generate(
-    "A cat sitting on a windowsill",
-    OmniDiffusionSamplingParams(
-        num_inference_steps=50,
-    ),
-)
+outputs = omni.generate(prompt="A cat sitting on a windowsill", num_inference_steps=50)
 ```
 
 ### Using Environment Variable
@@ -74,19 +68,13 @@ Controls the balance between speed and quality. Lower values prioritize quality,
 
 ```python
 from vllm_omni import Omni
-from vllm_omni.inputs.data import OmniDiffusionSamplingParams
 
 omni = Omni(
     model="Qwen/Qwen-Image",
     cache_backend="tea_cache",
     cache_config={"rel_l1_thresh": 0.2}
 )
-outputs = omni.generate(
-    "A cat sitting on a windowsill",
-    OmniDiffusionSamplingParams(
-        num_inference_steps=50,
-    ),
-)
+outputs = omni.generate(prompt="A cat sitting on a windowsill", num_inference_steps=50)
 ```
 
 ## Performance Tuning
@@ -124,7 +112,6 @@ th {
 | `QwenImageEditPipeline` | Qwen-Image-Edit | `Qwen/Qwen-Image-Edit` |
 | `QwenImageEditPlusPipeline` | Qwen-Image-Edit-2509 | `Qwen/Qwen-Image-Edit-2509` |
 | `QwenImageLayeredPipeline` | Qwen-Image-Layered | `Qwen/Qwen-Image-Layered` |
-| `BagelForConditionalGeneration` | BAGEL (DiT-only) | `ByteDance-Seed/BAGEL-7B-MoT` |
 
 ### VideoGen
 

docs/user_guide/diffusion/cpu_offload_diffusion.md

@@ -1,101 +0,0 @@
-# CPU Offloading for Diffusion Model
-
-## Overview
-
-vLLM-Omni provides two offloading strategies to reduce GPU memory usage for diffusion models, allowing you to run larger models on GPUs with limited VRAM:
-
-1. **Model-level (Component) Offloading**: Swaps entire model components (DiT transformer, VAE, encoders) between GPU and CPU.
-2. **Layerwise (Blockwise) Offloading**: Keeps only a single or a few transformer blocks on GPU at a time, with compute - memory copy overlap.
-
-Both approaches use pinned memory for faster CPU-GPU transfers. For now, the two offloading strategies could not be used at the same time.
-
-
-## Model-level CPU Offloading
-
-### Implementation
-
-CPU offload lets the diffusion worker move large model components between GPU and CPU memory on demand. It keeps the DiT transformer resident on GPU only while it is actively running, and swaps it out when encoders modules need the device. This reduces peak VRAM usage so bigger checkpoints run on smaller GPUs, or multiple requests can share the same GPU.
-
-**Execution Flow**:
-1. Text encoders run on GPU while the DiT transformer is offloaded to CPU.
-2. Before denoising, weights are prefetched back to GPU, honoring pinned-memory copies for speed.
-3. After the diffusion step, the transformer returns to CPU and the process repeats as needed.
-
-Transfers use pinned host buffers, and the worker coordinates swaps via mutex-style hooks so components never compete for memory.
-
-### Configuration
-You can enable CPU offload in two ways:
-
-1. **Python API**: set `enable_cpu_offload=True`.
-
-```python
-from vllm_omni import Omni
-
-if __name__ == "__main__":
-
-    m = Omni(model="Qwen/Qwen-Image",enable_cpu_offload=True)
-```
-
-2. **CLI**: pass `--enable-cpu-offload` to the diffusion service entrypoint.
-
-### Limitations
-- Cold start latency increases for over one minute for some models(e.g., Qwen-Image)
-
-
-## Layerwise (Blockwise) Offloading
-
-### Implementation
-Layerwise offload operates at transformer block granularity, keeping a single transformer block, or a specified number of blocks, on GPU while others stay in CPU memory.
-
-Unlike full model-wise CPU offload which swaps entire components like DiT and encoders, layerwise offloading applies a sliding window way of loading and offloading weights between gpu and cpu: while block `i` computes, block `i+1` get prefetched asynchronously via pinned memory. In this way, only partial blocks(s) reside on GPU at any moment during inference, so that greatly decrease the memory occupancy.
-
-**Execution Flow**:
-
-1. During model initialization, all components are loaded to CPU first. Then components other than DiT model(s) in the pipeline, such as VAE and encoders, are moved to GPU. The weights of target transformer blocks are collected as contiguous tensors per layer on CPU with pinned memory; and non-block modules (embeddings, norms, etc) in the DiT model are moved to and stay on GPU.
-2. The first block(s) are transferred to GPU during initialization of `LayerwiseOffloader`, before the first denoising step of the very first request.
-3. As each block executes, the next block prefetches on a separate CUDA stream for compute - memory copy overlap. After execution, the current block is immediately freed from GPU memory.
-4. When the last block completes, the first block prefetches for the next denoising step.
-
-
-Example of hook executions of a DiT model with n layers, by default keep a single layer on GPU:
-| Layer (block) idx | forward pre-hook               | forward          | forward post-hook         |
-|-------------------|--------------------------------|------------------|---------------------------|
-| layer-0           | prefetch layer 1 (copy stream) | compute layer 0  | free layer-0 gpu weights  |
-| layer-1           | prefetch layer 2 (copy stream) | compute layer 1  | free layer-1 gpu weights  |
-| layer-2           | prefetch layer 3 (copy stream) | compute layer 2  | free layer-2 gpu weights  |
-| ...               | ...                            | ...              | ...                       |
-| layer-(n-1)       | **prefetch layer 0 (copy stream)** | compute layer (n-1) | free layer (n-1) gpu weights  |
-
-
-### Configuration
-
-1. **Python API**: set `enable_layerwise_offload=True` and optionally `layerwise_num_gpu_layers`.
-
-```python
-from vllm_omni import Omni
-
-if __name__ == "__main__":
-    m = Omni(
-        model="Wan-AI/Wan2.2-T2V-A14B-Diffusers",
-        enable_layerwise_offload=True,
-        ...
-    )
-```
-
-2. **CLI**: pass `--enable-layerwise-offload` and `--layerwise-num-gpu-layers` to the diffusion service entrypoint.
-
-### Supported Models
-
-| Architecture | Models | Example HF Models | DiT Model Cls | Blocks Attr Name |
-|--------------|--------|-------------------|----------|----------|
-| `QwenImagePipeline` | Qwen-Image-Edit | `Qwen/Qwen-Image` | `QwenImageTransformer2DModel` | "transformer_blocks" |
-| `Wan22Pipeline` | Wan2.2 | `Wan-AI/Wan2.2-T2V-A14B-Diffusers` | `WanTransformer3DModel` | "blocks" |
-
-NOTE: Models must define `_layerwise_offload_blocks_attr` class attribute so that the layerwise offloader finds the target transformer blocks.
-
-### Limitations
-- Cold start latency increases because of
-    1) components are loaded to CPU first at the very first during initialization,  
-    2) weight consolidation and pinning
-- Performance depends on CPU <-> GPU interconnection (e.g., PCIe bandwidth).
-- Support single GPU only for now

docs/user_guide/diffusion/parallelism_acceleration.md

@@ -1,567 +0,0 @@
-# Parallelism Acceleration Guide
-
-This guide includes how to use parallelism methods in vLLM-Omni to speed up diffusion model inference as well as reduce the memory requirement on each device.
-
-## Overview
-
-The following parallelism methods are currently supported in vLLM-Omni:
-
-1. DeepSpeed Ulysses Sequence Parallel (DeepSpeed Ulysses-SP) ([arxiv paper](https://arxiv.org/pdf/2309.14509)): Ulysses-SP splits the input along the sequence dimension and uses all-to-all communication to allow each device to compute only a subset of attention heads.
-
-2. [Ring-Attention](#ring-attention) - splits the input along the sequence dimension and uses ring-based P2P communication to accumulate attention results, keeping the sequence dimension sharded
-
-3. Classifier-Free-Guidance Parallel (CFG-Parallel): CFG-Parallel runs the positive/negative prompts of classifier-free guidance (CFG) on different devices, then merges on a single device to perform the scheduler step.
-
-4. [Tensor Parallelism](#tensor-parallelism): Tensor parallelism shards model weights across devices. This can reduce per-GPU memory usage. Note that for diffusion models we currently shard the majority of layers within the DiT.
-
-The following table shows which models are currently supported by parallelism method:
-
-### ImageGen
-
-| Model                    | Model Identifier                     | Ulysses-SP | Ring-SP | CFG-Parallel | Tensor-Parallel |
-|--------------------------|--------------------------------------|:----------:|:-------:|:------------:|:---------------:|
-| **LongCat-Image**        | `meituan-longcat/LongCat-Image`      |     ✅      |    ✅    |      ❌       |        ✅        |
-| **LongCat-Image-Edit**   | `meituan-longcat/LongCat-Image-Edit` |     ✅      |    ✅    |      ❌       |        ✅        |
-| **Ovis-Image**           | `OvisAI/Ovis-Image`                  |     ❌      |    ❌    |      ❌       |        ❌        |
-| **Qwen-Image**           | `Qwen/Qwen-Image`                    |     ✅      |    ✅    |      ✅       |        ✅        |
-| **Qwen-Image-Edit**      | `Qwen/Qwen-Image-Edit`               |     ✅      |    ✅    |      ✅       |        ✅        |
-| **Qwen-Image-Edit-2509** | `Qwen/Qwen-Image-Edit-2509`          |     ✅      |    ✅    |      ✅       |        ✅        |
-| **Qwen-Image-Layered**   | `Qwen/Qwen-Image-Layered`            |     ✅      |    ✅    |      ✅       |        ✅        |
-| **Z-Image**              | `Tongyi-MAI/Z-Image-Turbo`           |     ✅      |    ✅    |      ❌       |  ✅ (TP=2 only)  |
-| **Stable-Diffusion3.5**  | `stabilityai/stable-diffusion-3.5`   |     ❌      |    ❌    |      ❌       |        ❌        |
-| **FLUX.2-klein**         | `black-forest-labs/FLUX.2-klein-4B`  |     ❌      |    ❌    |      ❌       |        ✅        |
-| **FLUX.1-dev**           | `black-forest-labs/FLUX.1-dev`       |     ❌      |    ❌    |      ❌       |        ✅        |
-
-!!! note "TP Limitations for Diffusion Models"
-    We currently implement Tensor Parallelism (TP) only for the DiT (Diffusion Transformer) blocks. This is because the `text_encoder` component in vLLM-Omni uses the original Transformers implementation, which does not yet support TP.
-
-    - Good news: The text_encoder typically has minimal impact on overall inference performance.
-    - Bad news: When TP is enabled, every TP process retains a full copy of the text_encoder weights, leading to significant GPU memory waste.
-
-    We are actively refactoring this design to address this. For details and progress, please refer to [Issue #771](https://github.com/vllm-project/vllm-omni/issues/771).
-
-
-!!! note "Why Z-Image is TP=2 only"
-    Z-Image Turbo is currently limited to `tensor_parallel_size` of **1 or 2** due to model shape divisibility constraints.
-    For example, the model has `n_heads=30` and a final projection out dimension of `64`, so valid TP sizes must divide both 30 and 64; the only common divisors are **1 and 2**.
-
-### VideoGen
-
-| Model | Model Identifier | Ulysses-SP | Ring-SP | Tensor-Parallel |
-|-------|------------------|------------|---------|--------------------------|
-| **Wan2.2** | `Wan-AI/Wan2.2-T2V-A14B-Diffusers` | ✅ | ✅ | ❌ |
-
-### Tensor Parallelism
-
-Tensor parallelism splits model parameters across GPUs. In vLLM-Omni, tensor parallelism is configured via `DiffusionParallelConfig.tensor_parallel_size`.
-
-#### Offline Inference
-
-```python
-from vllm_omni import Omni
-from vllm_omni.diffusion.data import DiffusionParallelConfig
-
-omni = Omni(
-    model="Tongyi-MAI/Z-Image-Turbo",
-    parallel_config=DiffusionParallelConfig(tensor_parallel_size=2),
-)
-
-outputs = omni.generate(
-    "a cat reading a book",
-    OmniDiffusionSamplingParams(
-        num_inference_steps=9,
-        width=512,
-        height=512,
-    ),
-)
-```
-
-### Sequence Parallelism
-
-#### Ulysses-SP
-
-##### Offline Inference
-
-An example of offline inference script using [Ulysses-SP](https://arxiv.org/pdf/2309.14509) is shown below:
-```python
-from vllm_omni import Omni
-from vllm_omni.inputs.data import OmniDiffusionSamplingParams
-from vllm_omni.diffusion.data import DiffusionParallelConfig
-ulysses_degree = 2
-
-omni = Omni(
-    model="Qwen/Qwen-Image",
-    parallel_config=DiffusionParallelConfig(ulysses_degree=2)
-)
-
-outputs = omni.generate(
-    "A cat sitting on a windowsill",
-    OmniDiffusionSamplingParams(num_inference_steps=50, width=2048, height=2048),
-)
-```
-
-See `examples/offline_inference/text_to_image/text_to_image.py` for a complete working example.
-
-##### Online Serving
-
-You can enable Ulysses-SP in online serving for diffusion models via `--usp`:
-
-```bash
-# Text-to-image (requires >= 2 GPUs)
-vllm serve Qwen/Qwen-Image --omni --port 8091 --usp 2
-```
-
-##### Benchmarks
-!!! note "Benchmark Disclaimer"
-    These benchmarks are provided for **general reference only**. The configurations shown use default or common parameter settings and have not been exhaustively optimized for maximum performance. Actual performance may vary based on:
-
-    - Specific model and use case
-    - Hardware configuration
-    - Careful parameter tuning
-    - Different inference settings (e.g., number of steps, image resolution)
-
-
-To measure the parallelism methods, we run benchmarks with **Qwen/Qwen-Image** model generating images (**2048x2048** as long sequence input) with 50 inference steps. The hardware devices are NVIDIA H800 GPUs. `sdpa` is the attention backends.
-
-| Configuration | Ulysses degree |Generation Time | Speedup |
-|---------------|----------------|---------|---------|
-| **Baseline (diffusers)** | - | 112.5s | 1.0x |
-| Ulysses-SP  |  2  |  65.2s | 1.73x |
-| Ulysses-SP  |  4  | 39.6s | 2.84x |
-| Ulysses-SP  |  8  | 30.8s | 3.65x |
-
-#### Ring-Attention
-
-Ring-Attention ([arxiv paper](https://arxiv.org/abs/2310.01889)) splits the input along the sequence dimension and uses ring-based P2P communication to accumulate attention results. Unlike Ulysses-SP which uses all-to-all communication, Ring-Attention keeps the sequence dimension sharded throughout the computation and circulates Key/Value blocks through a ring topology.
-
-##### Offline Inference
-
-An example of offline inference script using Ring-Attention is shown below:
-```python
-from vllm_omni import Omni
-from vllm_omni.inputs.data import OmniDiffusionSamplingParams
-from vllm_omni.diffusion.data import DiffusionParallelConfig
-ring_degree = 2
-
-omni = Omni(
-    model="Qwen/Qwen-Image",
-    parallel_config=DiffusionParallelConfig(ring_degree=2)
-)
-
-outputs = omni.generate(
-    "A cat sitting on a windowsill",
-    OmniDiffusionSamplingParams(num_inference_steps=50, width=2048, height=2048),
-)
-```
-
-See `examples/offline_inference/text_to_image/text_to_image.py` for a complete working example.
-
-
-##### Online Serving
-
-You can enable Ring-Attention in online serving for diffusion models via `--ring`:
-
-```bash
-# Text-to-image (requires >= 2 GPUs)
-vllm serve Qwen/Qwen-Image --omni --port 8091 --ring 2
-```
-
-##### Benchmarks
-!!! note "Benchmark Disclaimer"
-    These benchmarks are provided for **general reference only**. The configurations shown use default or common parameter settings and have not been exhaustively optimized for maximum performance. Actual performance may vary based on:
-
-    - Specific model and use case
-    - Hardware configuration
-    - Careful parameter tuning
-    - Different inference settings (e.g., number of steps, image resolution)
-
-
-To measure the parallelism methods, we run benchmarks with **Qwen/Qwen-Image** model generating images (**1024x1024** as long sequence input) with 50 inference steps. The hardware devices are NVIDIA A100 GPUs. `flash_attn` is the attention backends.
-
-| Configuration | Ring degree |Generation Time | Speedup |
-|---------------|----------------|---------|---------|
-| **Baseline (diffusers)** | - | 45.2s | 1.0x |
-| Ring-Attention  |  2  |  29.9s | 1.51x |
-| Ring-Attention  |  4  | 23.3s | 1.94x |
-
-
-#### Hybrid Ulysses + Ring
-
-You can combine both Ulysses-SP and Ring-Attention for larger scale parallelism. The total sequence parallel size equals `ulysses_degree × ring_degree`.
-
-##### Offline Inference
-
-```python
-from vllm_omni import Omni
-from vllm_omni.inputs.data import OmniDiffusionSamplingParams
-from vllm_omni.diffusion.data import DiffusionParallelConfig
-
-# Hybrid: 2 Ulysses × 2 Ring = 4 GPUs total
-omni = Omni(
-    model="Qwen/Qwen-Image",
-    parallel_config=DiffusionParallelConfig(ulysses_degree=2, ring_degree=2)
-)
-
-outputs = omni.generate(
-    "A cat sitting on a windowsill",
-    OmniDiffusionSamplingParams(num_inference_steps=50, width=2048, height=2048),
-)
-```
-
-##### Online Serving
-
-```bash
-# Text-to-image (requires >= 4 GPUs)
-vllm serve Qwen/Qwen-Image --omni --port 8091 --usp 2 --ring 2
-```
-
-##### Benchmarks
-!!! note "Benchmark Disclaimer"
-    These benchmarks are provided for **general reference only**. The configurations shown use default or common parameter settings and have not been exhaustively optimized for maximum performance. Actual performance may vary based on:
-
-    - Specific model and use case
-    - Hardware configuration
-    - Careful parameter tuning
-    - Different inference settings (e.g., number of steps, image resolution)
-
-
-To measure the parallelism methods, we run benchmarks with **Qwen/Qwen-Image** model generating images (**1024x1024** as long sequence input) with 50 inference steps. The hardware devices are NVIDIA A100 GPUs. `flash_attn` is the attention backends.
-
-| Configuration | Ulysses degree | Ring degree | Generation Time | Speedup |
-|---------------|----------------|-------------|-----------------|---------|
-| **Baseline (diffusers)** | - | - | 45.2s | 1.0x |
-| Hybrid Ulysses + Ring  |  2  |  2  |  24.3s | 1.87x |
-
-
-##### How to parallelize a new model
-
-NOTE: "Terminology: SP vs CP"
-    Our "Sequence Parallelism" (SP) corresponds to "Context Parallelism" (CP) in the [diffusers library](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/_modeling_parallel.py).
-    We use "Sequence Parallelism" to align with vLLM-Omni's terminology.
-
----
-
-###### Non-intrusive `_sp_plan` (Recommended)
-
-The `_sp_plan` mechanism allows SP without modifying `forward()` logic. The framework automatically registers hooks to shard inputs and gather outputs at module boundaries.
-
-**Requirements for `forward()` function:**
-
-- Tensor operations that need sharding/gathering must happen at **`nn.Module` boundaries** (not inline Python operations)
-- If your `forward()` contains inline tensor operations (e.g., `torch.cat`, `pad_sequence`) that need sharding, **extract them into a submodule**
-
-**When to create a submodule:**
-
-```python
-# ❌ BAD: Inline operations - hooks cannot intercept
-def forward(self, x, cap_feats):
-    unified = torch.cat([x, cap_feats], dim=1)  # Cannot be sharded via _sp_plan
-    ...
-
-# ✅ GOOD: Extract into a submodule
-class UnifiedPrepare(nn.Module):
-    def forward(self, x, cap_feats):
-        return torch.cat([x, cap_feats], dim=1)  # Now can be sharded via _sp_plan
-
-class MyModel(nn.Module):
-    def __init__(self):
-        self.unified_prepare = UnifiedPrepare()  # Submodule
-
-    def forward(self, x, cap_feats):
-        unified = self.unified_prepare(x, cap_feats)  # Hook can intercept here
-```
-
----
-
-###### Defining `_sp_plan`
-
-**Type definitions** (see [diffusers `_modeling_parallel.py`](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/_modeling_parallel.py) for reference):
-
-```python
-from vllm_omni.diffusion.distributed.sp_plan import (
-    SequenceParallelInput,   # Corresponds to diffusers' ContextParallelInput
-    SequenceParallelOutput,  # Corresponds to diffusers' ContextParallelOutput
-)
-```
-
-| Parameter | Description |
-|-----------|-------------|
-| `split_dim` | Dimension to split/gather (usually `1` for sequence) |
-| `expected_dims` | Expected tensor rank for validation (optional) |
-| `split_output` | `False`: shard **input** parameters; `True`: shard **output** tensors |
-| `auto_pad` | Auto-pad if sequence not divisible by world_size (Ulysses only) |
-
-**Key naming convention:**
-
-| Key | Meaning | Python equivalent |
-|-----|---------|-------------------|
-| `""` | Root model | `model` |
-| `"blocks.0"` | First element of ModuleList | `model.blocks[0]` |
-| `"blocks.*"` | All elements of ModuleList | `for b in model.blocks` |
-| `"outputs.main"` | ModuleDict entry | `model.outputs["main"]` |
-
-**Dictionary key types:**
-
-| Key type | `split_output` | Description |
-|----------|----------------|-------------|
-| `"param_name"` (str) | `False` | Shard **input parameter** by name |
-| `0`, `1` (int) | `True` | Shard **output tuple** by index |
-
-**Example** (similar to [diffusers `transformer_wan.py`](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/transformers/transformer_wan.py)):
-
-```python
-class MyTransformer(nn.Module):
-    _sp_plan = {
-        # Shard rope module OUTPUTS (returns tuple)
-        "rope": {
-            0: SequenceParallelInput(split_dim=1, expected_dims=4, split_output=True),  # cos
-            1: SequenceParallelInput(split_dim=1, expected_dims=4, split_output=True),  # sin
-        },
-        # Shard transformer block INPUT parameter
-        "blocks.0": {
-            "hidden_states": SequenceParallelInput(split_dim=1, expected_dims=3),
-        },
-        # Gather at final projection
-        "proj_out": SequenceParallelOutput(gather_dim=1, expected_dims=3),
-    }
-```
-
----
-
-###### Hook flow
-
-```
-Input → [SequenceParallelSplitHook: pre_forward] → Module.forward() → [post_forward] → ...
-                                                                              ↓
-... → [SequenceParallelGatherHook: post_forward] → Output
-```
-
-1. **SplitHook** shards tensors before/after the target module
-2. **Attention layers** handle Ulysses/Ring communication internally
-3. **GatherHook** collects sharded outputs
-
-The framework automatically applies these hooks when `sequence_parallel_size > 1`.
-
----
-
-###### Method 2: Intrusive modification (For complex cases)
-
-For models with dynamic sharding logic that cannot be expressed via `_sp_plan`:
-
-```python
-from vllm_omni.diffusion.distributed.sp_sharding import sp_shard, sp_gather
-
-def forward(self, hidden_states, ...):
-    if self.parallel_config.sequence_parallel_size > 1:
-        hidden_states = sp_shard(hidden_states, dim=1)
-        # ... computation ...
-        output = sp_gather(output, dim=1)
-    return output
-```
-
----
-
-###### Choosing the right approach
-
-| Scenario | Approach |
-|----------|----------|
-| Standard transformer | `_sp_plan` |
-| Inline tensor ops need sharding | Extract to submodule + `_sp_plan` |
-| Dynamic/conditional sharding | Intrusive modification |
-
-
-### CFG-Parallel
-
-#### Offline Inference
-
-CFG-Parallel is enabled through `DiffusionParallelConfig(cfg_parallel_size=2)`, which runs one rank for the positive branch and one rank for the negative branch.
-
-An example of offline inference using CFG-Parallel (image-to-image) is shown below:
-
-```python
-from vllm_omni import Omni
-from vllm_omni.diffusion.data import DiffusionParallelConfig
-
-image_path = "path_to_image.png"
-omni = Omni(
-    model="Qwen/Qwen-Image-Edit",
-    parallel_config=DiffusionParallelConfig(cfg_parallel_size=2),
-)
-input_image = Image.open(image_path).convert("RGB")
-
-outputs = omni.generate(
-    {
-        "prompt": "turn this cat to a dog",
-        "negative_prompt": "low quality, blurry",
-        "multi_modal_data": {"image": input_image},
-    },
-    OmniDiffusionSamplingParams(
-        true_cfg_scale=4.0,
-        num_inference_steps=50,
-    ),
-)
-```
-
-Notes:
-
-- CFG-Parallel is only effective when a `negative_prompt` is provided AND a guidance scale (or `cfg_scale`) is greater than 1.
-
-See `examples/offline_inference/image_to_image/image_edit.py` for a complete working example.
-```bash
-cd examples/offline_inference/image_to_image/
-python image_edit.py \
-  --model "Qwen/Qwen-Image-Edit" \
-  --image "qwen_image_output.png" \
-  --prompt "turn this cat to a dog" \
-  --negative_prompt "low quality, blurry" \
-  --cfg_scale 4.0 \
-  --output "edited_image.png" \
-  --cfg_parallel_size 2
-```
-
-#### Online Serving
-
-You can enable CFG-Parallel in online serving for diffusion models via `--cfg-parallel-size`:
-
-```bash
-vllm serve Qwen/Qwen-Image-Edit --omni --port 8091 --cfg-parallel-size 2
-```
-
-#### How to parallelize a pipeline
-
-This section describes how to add CFG-Parallel to a diffusion **pipeline**. We use the Qwen-Image pipeline (`vllm_omni/diffusion/models/qwen_image/pipeline_qwen_image.py`) as the reference implementation.
-
-In `QwenImagePipeline`, each diffusion step runs two denoiser forward passes sequentially:
-
-- positive (prompt-conditioned)
-- negative (negative-prompt-conditioned)
-
-CFG-Parallel assigns these two branches to different ranks in the **CFG group** and synchronizes the results.
-
-vLLM-omni provides `CFGParallelMixin` base class that encapsulates the CFG parallel logic. By inheriting from this mixin and calling its methods, pipelines can easily implement CFG parallel without writing repetitive code.
-
-**Key Methods in CFGParallelMixin:**
-- `predict_noise_maybe_with_cfg()`: Automatically handles CFG parallel noise prediction
-- `scheduler_step_maybe_with_cfg()`: Scheduler step with automatic CFG rank synchronization
-
-**Example Implementation:**
-
-```python
-class QwenImageCFGParallelMixin(CFGParallelMixin):
-    """
-    Base Mixin class for Qwen Image pipelines providing shared CFG methods.
-    """
-
-    def diffuse(
-        self,
-        prompt_embeds: torch.Tensor,
-        prompt_embeds_mask: torch.Tensor,
-        negative_prompt_embeds: torch.Tensor,
-        negative_prompt_embeds_mask: torch.Tensor,
-        latents: torch.Tensor,
-        img_shapes: torch.Tensor,
-        txt_seq_lens: torch.Tensor,
-        negative_txt_seq_lens: torch.Tensor,
-        timesteps: torch.Tensor,
-        do_true_cfg: bool,
-        guidance: torch.Tensor,
-        true_cfg_scale: float,
-        image_latents: torch.Tensor | None = None,
-        cfg_normalize: bool = True,
-        additional_transformer_kwargs: dict[str, Any] | None = None,
-    ) -> torch.Tensor:
-        self.transformer.do_true_cfg = do_true_cfg
-
-        for i, t in enumerate(timesteps):
-            timestep = t.expand(latents.shape[0]).to(device=latents.device, dtype=latents.dtype)
-
-            # Prepare kwargs for positive (conditional) prediction
-            positive_kwargs = {
-                "hidden_states": latents,
-                "timestep": timestep / 1000,
-                "guidance": guidance,
-                "encoder_hidden_states_mask": prompt_embeds_mask,
-                "encoder_hidden_states": prompt_embeds,
-                "img_shapes": img_shapes,
-                "txt_seq_lens": txt_seq_lens,
-            }
-
-            # Prepare kwargs for negative (unconditional) prediction
-            if do_true_cfg:
-                negative_kwargs = {
-                    "hidden_states": latents,
-                    "timestep": timestep / 1000,
-                    "guidance": guidance,
-                    "encoder_hidden_states_mask": negative_prompt_embeds_mask,
-                    "encoder_hidden_states": negative_prompt_embeds,
-                    "img_shapes": img_shapes,
-                    "txt_seq_lens": negative_txt_seq_lens,
-                }
-            else:
-                negative_kwargs = None
-
-            # Predict noise with automatic CFG parallel handling
-            # - In CFG parallel mode: rank0 computes positive, rank1 computes negative
-            # - Automatically gathers results and combines them on rank0
-            noise_pred = self.predict_noise_maybe_with_cfg(
-                do_true_cfg=do_true_cfg,
-                true_cfg_scale=true_cfg_scale,
-                positive_kwargs=positive_kwargs,
-                negative_kwargs=negative_kwargs,
-                cfg_normalize=cfg_normalize,
-            )
-
-            # Step scheduler with automatic CFG synchronization
-            # - Only rank0 computes the scheduler step
-            # - Automatically broadcasts updated latents to all ranks
-            latents = self.scheduler_step_maybe_with_cfg(
-                noise_pred, t, latents, do_true_cfg
-            )
-
-        return latents
-```
-
-**How it works:**
-1. Prepare separate `positive_kwargs` and `negative_kwargs` for conditional and unconditional predictions
-2. Call `predict_noise_maybe_with_cfg()` which:
-   - Detects if CFG parallel is enabled (`get_classifier_free_guidance_world_size() > 1`)
-   - Distributes computation: rank0 processes positive, rank1 processes negative
-   - Gathers predictions and combines them using `combine_cfg_noise()` on rank0
-   - Returns combined noise prediction (only valid on rank0)
-3. Call `scheduler_step_maybe_with_cfg()` which:
-   - Only rank0 computes the scheduler step
-   - Broadcasts the updated latents to all ranks for synchronization
-
-**How to customize**
-
-Some pipelines may need to customize the following functions in `CFGParallelMixin`:
-1. You may need to edit `predict_noise` function for custom behaviors.
-```python
-def predict_noise(self, *args, **kwargs):
-    """
-    Forward pass through transformer to predict noise.
-
-    Subclasses should override this if they need custom behavior,
-    but the default implementation calls self.transformer.
-    """
-    return self.transformer(*args, **kwargs)[0]
-
-```
-2. The default normalization function after combining the noise predictions from both branches is as follows. You may need to customize it.
-```python
-def cfg_normalize_function(self, noise_pred, comb_pred):
-    """
-    Normalize the combined noise prediction.
-
-    Args:
-        noise_pred: positive noise prediction
-        comb_pred: combined noise prediction after CFG
-
-    Returns:
-        Normalized noise prediction tensor
-    """
-    cond_norm = torch.norm(noise_pred, dim=-1, keepdim=True)
-    noise_norm = torch.norm(comb_pred, dim=-1, keepdim=True)
-    noise_pred = comb_pred * (cond_norm / noise_norm)
-    return noise_pred
-```

docs/user_guide/diffusion_acceleration.md

@@ -6,21 +6,18 @@ vLLM-Omni supports various cache acceleration methods to speed up diffusion mode
 
 vLLM-Omni currently supports two main cache acceleration backends:
 
-1. **[TeaCache](diffusion/teacache.md)** - Hook-based adaptive caching that caches transformer computations when consecutive timesteps are similar
-2. **[Cache-DiT](diffusion/cache_dit_acceleration.md)** - Library-based acceleration using multiple techniques:
+1. **[TeaCache](acceleration/teacache.md)** - Hook-based adaptive caching that caches transformer computations when consecutive timesteps are similar
+2. **[Cache-DiT](acceleration/cache_dit_acceleration.md)** - Library-based acceleration using multiple techniques:
     - **DBCache** (Dual Block Cache): Caches intermediate transformer block outputs based on residual differences
     - **TaylorSeer**: Uses Taylor expansion-based forecasting for faster inference
     - **SCM** (Step Computation Masking): Selectively computes steps based on adaptive masking
 
 Both methods can provide significant speedups (typically **1.5x-2.0x**) while maintaining high output quality.
 
-vLLM-Omni also supports parallelism methods for diffusion models, including:
+vLLM-Omni also supports the sequence parallelism (SP) for the diffusion model, that includes:
 
-1. [Ulysses-SP](diffusion/parallelism_acceleration.md#ulysses-sp) - splits the input along the sequence dimension and uses all-to-all communication to allow each device to compute only a subset of attention heads.
-
-2. [Ring-Attention](diffusion/parallelism_acceleration.md#ring-attention) - splits the input along the sequence dimension and uses ring-based P2P communication to accumulate attention results, keeping the sequence dimension sharded.
-
-3. [CFG-Parallel](diffusion/parallelism_acceleration.md#cfg-parallel) - runs the positive/negative prompts of classifier-free guidance (CFG) on different devices, then merges on a single device to perform the scheduler step.
+1. [Ulysses-SP](acceleration/parallelism_acceleration.md#ulysses-sp) - splits the input along the sequence dimension and uses all-to-all communication to allow each device to compute only a subset of attention heads.
+2. [Ring-Attention](acceleration/parallelism_acceleration.md#ring-attention) - splits the input along the sequence dimension and uses ring-based P2P communication to accumulate attention results, keeping the sequence dimension sharded.
 
 ## Quick Comparison
 
@@ -37,25 +34,26 @@ The following table shows which models are currently supported by each accelerat
 
 ### ImageGen
 
-| Model | Model Identifier | TeaCache | Cache-DiT | Ulysses-SP | Ring-Attention | CFG-Parallel |
-|-------|------------------|:----------:|:-----------:|:-----------:|:----------------:|:----------------:|
-| **LongCat-Image** | `meituan-longcat/LongCat-Image` | ❌ | ✅ | ❌ | ❌ | ✅ |
-| **LongCat-Image-Edit** | `meituan-longcat/LongCat-Image-Edit` | ❌ | ✅ | ❌ | ❌ | ✅ |
-| **Ovis-Image** | `OvisAI/Ovis-Image` | ❌ | ✅ | ❌ | ❌ | ✅ |
-| **Qwen-Image** | `Qwen/Qwen-Image` | ✅ | ✅ | ✅ | ✅ | ✅ |
-| **Qwen-Image-2512** | `Qwen/Qwen-Image-2512` | ✅ | ✅ | ✅ | ✅ | ✅ |
-| **Qwen-Image-Edit** | `Qwen/Qwen-Image-Edit` | ✅ | ✅ | ✅ | ✅ | ✅ |
-| **Qwen-Image-Edit-2509** | `Qwen/Qwen-Image-Edit-2509` | ✅ | ✅ | ✅ | ✅ | ✅ |
-| **Qwen-Image-Layered** | `Qwen/Qwen-Image-Layered` | ❌ | ✅ | ✅ | ✅ | ✅ |
-| **Z-Image** | `Tongyi-MAI/Z-Image-Turbo` | ✅ | ✅ | ❌ | ❌ | ❌ |
-| **Stable-Diffusion3.5** | `stabilityai/stable-diffusion-3.5` | ❌ | ✅ | ❌ | ❌ | ✅ |
-| **Bagel** | `ByteDance-Seed/BAGEL-7B-MoT` | ✅ | ✅ | ❌ | ❌ | ❌ |
+| Model | Model Identifier | TeaCache | Cache-DiT | Ulysses-SP | Ring-Attention |
+|-------|------------------|----------|-----------|-----------|----------------|
+| **LongCat-Image** | `meituan-longcat/LongCat-Image` | ❌ | ✅ | ❌ | ❌ |
+| **LongCat-Image-Edit** | `meituan-longcat/LongCat-Image-Edit` | ❌ | ✅ | ❌ | ❌ |
+| **Ovis-Image** | `OvisAI/Ovis-Image` | ❌ | ✅ | ❌ | ❌ |
+| **Qwen-Image** | `Qwen/Qwen-Image` | ✅ | ✅ | ✅ | ✅ |
+| **Qwen-Image-Edit** | `Qwen/Qwen-Image-Edit` | ✅ | ✅ | ✅ | ✅ |
+| **Qwen-Image-Edit-2509** | `Qwen/Qwen-Image-Edit-2509` | ❌ | ✅ | ✅ | ✅ |
+| **Qwen-Image-Layered** | `Qwen/Qwen-Image-Layered` | ❌ | ✅ | ✅ | ✅ |
+| **Z-Image** | `Tongyi-MAI/Z-Image-Turbo` | ❌ | ✅ |❌ | ❌ |
+| **LongCat-Image** | `meituan-longcat/LongCat-Image` | ❌ | ✅ | ❌ | ❌ |
+| **LongCat-Image-Edit** | `meituan-longcat/LongCat-Image-Edit` | ❌ | ✅ | ❌ | ❌ |
+| **Ovis-Image** | `OvisAI/Ovis-Image` | ❌ | ✅ | ❌ | ❌ |
+| **Stable-Diffusion3.5** | `stabilityai/stable-diffusion-3.5` | ❌ | ✅ | ❌ | ❌ |
 
 ### VideoGen
 
-| Model | Model Identifier | TeaCache | Cache-DiT | Ulysses-SP | Ring-Attention |CFG-Parallel |
-|-------|------------------|:--------:|:---------:|:----------:|:--------------:|:----------------:|
-| **Wan2.2** | `Wan-AI/Wan2.2-T2V-A14B-Diffusers` | ❌ | ✅ | ✅ | ✅ | ✅ |
+| Model | Model Identifier | TeaCache | Cache-DiT | Ulysses-SP |
+|-------|------------------|----------|-----------|-----------|
+| **Wan2.2** | `Wan-AI/Wan2.2-T2V-A14B-Diffusers` | ❌ | ✅ | ❌ |
 
 
 ## Performance Benchmarks
@@ -96,7 +94,6 @@ To measure the parallelism methods, we run benchmarks with **Qwen/Qwen-Image** m
 
 ```python
 from vllm_omni import Omni
-from vllm_omni.inputs.data import OmniDiffusionSamplingParams
 
 omni = Omni(
     model="Qwen/Qwen-Image",
@@ -104,19 +101,13 @@ omni = Omni(
     cache_config={"rel_l1_thresh": 0.2}  # Optional, defaults to 0.2
 )
 
-outputs = omni.generate(
-    "A cat sitting on a windowsill",
-    OmniDiffusionSamplingParams(
-        num_inference_steps=50,
-    ),
-)
+outputs = omni.generate(prompt="A cat sitting on a windowsill", num_inference_steps=50)
 ```
 
 ### Using Cache-DiT
 
 ```python
 from vllm_omni import Omni
-from vllm_omni.inputs.data import OmniDiffusionSamplingParams
 
 omni = Omni(
     model="Qwen/Qwen-Image",
@@ -130,12 +121,7 @@ omni = Omni(
     }
 )
 
-outputs = omni.generate(
-    "A cat sitting on a windowsill",
-    OmniDiffusionSamplingParams(
-        num_inference_steps=50,
-    ),
-)
+outputs = omni.generate(prompt="A cat sitting on a windowsill", num_inference_steps=50)
 ```
 
 ### Using Ulysses-SP
@@ -143,41 +129,31 @@ outputs = omni.generate(
 Run text-to-image:
 ```python
 from vllm_omni import Omni
-from vllm_omni.inputs.data import OmniDiffusionSamplingParams
 from vllm_omni.diffusion.data import DiffusionParallelConfig
 ulysses_degree = 2
 
 omni = Omni(
     model="Qwen/Qwen-Image",
-    parallel_config=DiffusionParallelConfig(ulysses_degree=ulysses_degree)
+    parallel_config=DiffusionParallelConfig(ulysses_degree=2)
 )
 
-outputs = omni.generate(
-    "A cat sitting on a windowsill",
-    OmniDiffusionSamplingParams(num_inference_steps=50, width=2048, height=2048),
-)
+outputs = omni.generate(prompt="A cat sitting on a windowsill", num_inference_steps=50, width=2048, height=2048)
 ```
 
 
 Run image-to-image:
 ```python
 from vllm_omni import Omni
-from vllm_omni.inputs.data import OmniDiffusionSamplingParams
 from vllm_omni.diffusion.data import DiffusionParallelConfig
 ulysses_degree = 2
 
 omni = Omni(
     model="Qwen/Qwen-Image-Edit",
-    parallel_config=DiffusionParallelConfig(ulysses_degree=ulysses_degree)
+    parallel_config=DiffusionParallelConfig(ulysses_degree=2)
 )
 
-outputs = omni.generate(
-    {
-        "prompt": "turn this cat to a dog",
-        "multi_modal_data": {"image": input_image}
-    },
-    OmniDiffusionSamplingParams(num_inference_steps=50),
-)
+outputs = omni.generate(prompt="turn this cat to a dog",
+        pil_image=input_image, num_inference_steps=50)
 ```
 
 ### Using Ring-Attention
@@ -185,7 +161,6 @@ outputs = omni.generate(
 Run text-to-image:
 ```python
 from vllm_omni import Omni
-from vllm_omni.inputs.data import OmniDiffusionSamplingParams
 from vllm_omni.diffusion.data import DiffusionParallelConfig
 ring_degree = 2
 
@@ -194,43 +169,13 @@ omni = Omni(
     parallel_config=DiffusionParallelConfig(ring_degree=2)
 )
 
-outputs = omni.generate(
-    "A cat sitting on a windowsill",
-    OmniDiffusionSamplingParams(num_inference_steps=50, width=2048, height=2048),
-)
-```
-
-### Using CFG-Parallel
-
-Run image-to-image:
-
-CFG-Parallel splits the CFG positive/negative branches across GPUs. Use it when you set a non-trivial `true_cfg_scale`.
-
-```python
-from vllm_omni import Omni
-from vllm_omni.inputs.data import OmniDiffusionSamplingParams
-from vllm_omni.diffusion.data import DiffusionParallelConfig
-cfg_parallel_size = 2
-
-omni = Omni(
-    model="Qwen/Qwen-Image-Edit",
-    parallel_config=DiffusionParallelConfig(cfg_parallel_size=cfg_parallel_size)
-)
-
-outputs = omni.generate(
-    {
-        "prompt": "turn this cat to a dog",
-        "multi_modal_data": {"image": input_image}
-    },
-    OmniDiffusionSamplingParams(num_inference_steps=50, true_cfg_scale=4.0),
-)
+outputs = omni.generate(prompt="A cat sitting on a windowsill", num_inference_steps=50, width=2048, height=2048)
 ```
 
 ## Documentation
 
 For detailed information on each acceleration method:
 
-- **[TeaCache Guide](diffusion/teacache.md)** - Complete TeaCache documentation, configuration options, and best practices
-- **[Cache-DiT Acceleration Guide](diffusion/cache_dit_acceleration.md)** - Comprehensive Cache-DiT guide covering DBCache, TaylorSeer, SCM, and configuration parameters
-- **[Sequence Parallelism](diffusion/parallelism_acceleration.md#sequence-parallelism)** - Guidance on how to set sequence parallelism with configuration.
-- **[CFG-Parallel](diffusion/parallelism_acceleration.md#cfg-parallel)** - Guidance on how to set CFG-Parallel to run positive/negative branches across ranks.
+- **[TeaCache Guide](acceleration/teacache.md)** - Complete TeaCache documentation, configuration options, and best practices
+- **[Cache-DiT Acceleration Guide](acceleration/cache_dit_acceleration.md)** - Comprehensive Cache-DiT guide covering DBCache, TaylorSeer, SCM, and configuration parameters
+- **[Sequence Parallelism](acceleration/parallelism_acceleration.md#sequence-parallelism)** - Guidance on how to set sequence parallelism with configuration.

docs/user_guide/examples/offline_inference/bagel.md

@@ -1,179 +0,0 @@
-# BAGEL-7B-MoT
-
-Source <https://github.com/vllm-project/vllm-omni/tree/main/examples/offline_inference/bagel>.
-
-## Set up
-
-Please refer to the [stage configuration documentation](https://docs.vllm.ai/projects/vllm-omni/en/latest/configuration/stage_configs/) to configure memory allocation appropriately for your hardware setup.
-
-## Run examples
-
-**Note**: These examples work with the default configuration on an **NVIDIA A100 (80GB)**. We also tested on dual **NVIDIA RTX 5000 Ada (32GB each)**. For dual-GPU setups, please modify the stage configuration to distribute the model across devices.
-
-Get into the bagel folder
-
-```bash
-cd examples/offline_inference/bagel
-```
-
-### Modality Control
-
-BAGEL-7B-MoT supports multiple modality modes. You can control the mode using the `--modality` argument:
-
-#### Text to Image (text2img)
-
-- **Pipeline**: Text → Thinker  → DiT → VAE Decode → Image
-- **Stages Used**: Stage 0 (Thinker) + Stage 1 (DiT)
-- **KV Transfer**: Thinker sends KV cache to DiT for conditioned generation
-
-Generate images from text prompts:
-
-```bash
-python end2end.py --model ByteDance-Seed/BAGEL-7B-MoT \
-                  --modality text2img \
-                  --prompts "A cute cat"
-```
-
-#### Image to Image (img2img)
-
-- **Pipeline**: Image → VAE Encode → DiT → VAE Decode → New Image
-- **Stages Used**: Stage 1 (DiT) only
-- **Special**: Bypasses the Thinker stage, direct image-to-image transformation
-
-Transform images based on text prompts:
-
-```bash
-python end2end.py --model ByteDance-Seed/BAGEL-7B-MoT \
-                  --modality img2img \
-                  --image-path /path/to/image.jpg \
-                  --prompts "Let the woman wear a blue dress"
-```
-
-#### Image to Text (img2text)
-
-- **Pipeline**: Image → ViT + VAE Encode → Thinker → Text Output
-- **Stages Used**: Stage 0 (Thinker) only
-- **Special**: Uses both VAE latent encoding AND ViT semantic encoding for comprehensive image understanding
-
-Generate text descriptions from images:
-
-```bash
-python end2end.py --model ByteDance-Seed/BAGEL-7B-MoT \
-                  --modality img2text \
-                  --image-path /path/to/image.jpg \
-                  --prompts "Describe this image in detail"
-```
-
-#### Text to Text (text2text)
-
-- **Pipeline**: Text → Thinker → Text Output
-- **Stages Used**: Stage 0 (Thinker) only
-- **Special**: No visual components involved, operates as pure language model
-
-Pure text generation:
-
-```bash
-python end2end.py --model ByteDance-Seed/BAGEL-7B-MoT \
-                  --modality text2text \
-                  --prompts "What is the capital of France?"
-
-# You can load prompts from a text file (one prompt per line):  
-python end2end.py --model ByteDance-Seed/BAGEL-7B-MoT \
-                  --modality text2text \
-                  --txt-prompts /path/to/prompts.txt
-```
-
-### Inference Steps
-
-Control the number of inference steps for image generation:
-
-```bash
-# You can adjust steps to 100 to improve image quality
-python end2end.py --model ByteDance-Seed/BAGEL-7B-MoT \
-                  --modality text2img \
-                  --steps 50 \
-                  --prompts "A cute cat"
-```
-
-### Key arguments
-
-BAGEL-7B-MoT supports **multiple modality modes** for different use cases.
-
-The default yaml configuration deploys Thinker and DiT on the same GPU. You can use the default configuration file: [`bagel.yaml`](../../../vllm_omni/model_executor/stage_configs/bagel.yaml)
-
-#### 📌 Command Line Arguments (end2end.py)
-
-| Argument               | Type   | Default                       | Description                                                  |
-| :--------------------- | :----- | :---------------------------- | :----------------------------------------------------------- |
-| `--model`              | string | `ByteDance-Seed/BAGEL-7B-MoT` | Model path or name                                           |
-| `--modality`           | choice | `text2img`                    | Modality mode: `text2img`, `img2img`, `img2text`, `text2text` |
-| `--prompts`            | list   | `None`                        | Input text prompts directly                                  |
-| `--txt-prompts`        | string | `None`                        | Path to txt file with one prompt per line                    |
-| `--image-path`         | string | `None`                        | Input image path (for `img2img`/`img2text`)                  |
-| `--steps`              | int    | `50`                          | Number of inference steps                                    |
-| `--stage-configs-path` | string | `None`                        | Custom stage config file path                                |
-| `--worker-backend`     | choice | `process`                     | Worker backend: `process` or `ray`                           |
-| `--ray-address`        | string | `None`                        | Ray cluster address                                          |
-| `--enable-stats`       | flag   | `False`                       | Enable statistics logging                                    |
-| `--init-sleep-seconds` | int    | `20`                          | Initialization sleep time                                    |
-| `--batch-timeout`      | int    | `5`                           | Batch timeout                                                |
-| `--init-timeout`       | int    | `300`                         | Initialization timeout                                       |
-
-------
-
-#### ⚙️ Stage Configuration Parameters (bagel.yaml)
-
- **Stage 0 - Thinker (LLM Stage)**
-
-| Parameter                        | Value                           | Description              |
-| :------------------------------- | :------------------------------ | :----------------------- |
-| `stage_type`                     | `llm`                           | Stage type               |
-| `devices`                        | `"0"`                           | GPU device ID            |
-| `max_batch_size`                 | `1`                             | Maximum batch size       |
-| `model_stage`                    | `thinker`                       | Model stage identifier   |
-| `model_arch`                     | `BagelForConditionalGeneration` | Model architecture       |
-| `gpu_memory_utilization`         | `0.4`                           | GPU memory utilization   |
-| `tensor_parallel_size`           | `1`                             | Tensor parallel size     |
-| `max_num_batched_tokens`         | `32768`                         | Maximum batched tokens   |
-| `omni_kv_config.need_send_cache` | `true`                          | Whether to send KV cache |
-
-------
-
-**Stage 1 - DiT (Diffusion Stage)**
-
-| Parameter                        | Value       | Description                 |
-| :------------------------------- | :---------- | :-------------------------- |
-| `stage_type`                     | `diffusion` | Stage type                  |
-| `devices`                        | `"0"`       | GPU device ID               |
-| `max_batch_size`                 | `1`         | Maximum batch size          |
-| `model_stage`                    | `dit`       | Model stage identifier      |
-| `gpu_memory_utilization`         | `0.4`       | GPU memory utilization      |
-| `omni_kv_config.need_recv_cache` | `true`      | Whether to receive KV cache |
-| `engine_input_source`            | `[0]`       | Input source from Stage 0   |
-
-------
-
-#### 🔗 Runtime Configuration
-
-| Parameter             | Value   | Description                      |
-| :-------------------- | :------ | :------------------------------- |
-| `window_size`         | `-1`    | Window size (-1 means unlimited) |
-| `max_inflight`        | `1`     | Maximum inflight requests        |
-| `shm_threshold_bytes` | `65536` | Shared memory threshold (64KB)   |
-
-## FAQ
-
-- If you encounter an error about the backend of librosa, try to install ffmpeg with the command below.
-
-```bash
-sudo apt update
-sudo apt install ffmpeg
-```
-
-- If you don’t know how much VRAM is needed for the model or encounter the OOM error, you can try to decrease the max_model_len.
-
-| Stage               | VRAM                         |
-| :------------------ | :--------------------------- |
-| Stage-0 (Thinker)   | **15.04 GiB** **+ KV Cache** |
-| Stage-1 (DiT)       | **26.50 GiB**                |
-| Total               | **~42 GiB + KV Cache**       |

docs/user_guide/examples/offline_inference/image_to_image.md

@@ -47,7 +47,6 @@ Key arguments:
 - `--image`: path(s) to the source image(s) (PNG/JPG, converted to RGB). Can specify multiple images.
 - `--prompt` / `--negative_prompt`: text description (string).
 - `--cfg_scale`: true classifier-free guidance scale (default: 4.0). Classifier-free guidance is enabled by setting cfg_scale > 1 and providing a negative_prompt. Higher guidance scale encourages images closely linked to the text prompt, usually at the expense of lower image quality.
-- `--cfg_parallel_size`: the number of devices to run CFG Parallel. CFG Parallel is valid only if classifier-free guidance is enabled and `cfg_parallel_size` is set to 2.
 - `--guidance_scale`: guidance scale for guidance-distilled models (default: 1.0, disabled). Unlike classifier-free guidance (--cfg_scale), guidance-distilled models take the guidance scale directly as an input parameter. Enabled when guidance_scale > 1. Ignored when not using guidance-distilled models.
 - `--num_inference_steps`: diffusion sampling steps (more steps = higher quality, slower).
 - `--output`: path to save the generated PNG.

docs/user_guide/examples/offline_inference/image_to_video.md

@@ -1,67 +0,0 @@
-# Image-To-Video
-
-Source <https://github.com/vllm-project/vllm-omni/tree/main/examples/offline_inference/image_to_video>.
-
-
-This example demonstrates how to generate videos from images using Wan2.2 Image-to-Video models with vLLM-Omni's offline inference API.
-
-## Local CLI Usage
-
-### Wan2.2-I2V-A14B-Diffusers (MoE)
-```bash
-python image_to_video.py \
-  --model Wan-AI/Wan2.2-I2V-A14B-Diffusers \
-  --image input.png \
-  --prompt "A cat playing with yarn, smooth motion" \
-  --negative_prompt "<optional quality filter>" \
-  --height 480 \
-  --width 832 \
-  --num_frames 48 \
-  --guidance_scale 5.0 \
-  --guidance_scale_high 6.0 \
-  --num_inference_steps 40 \
-  --boundary_ratio 0.875 \
-  --flow_shift 12.0 \
-  --fps 16 \
-  --output i2v_output.mp4
-```
-
-### Wan2.2-TI2V-5B-Diffusers (Unified)
-```bash
-python image_to_video.py \
-  --model Wan-AI/Wan2.2-TI2V-5B-Diffusers \
-  --image input.png \
-  --prompt "A cat playing with yarn, smooth motion" \
-  --negative_prompt "<optional quality filter>" \
-  --height 480 \
-  --width 832 \
-  --num_frames 48 \
-  --guidance_scale 4.0 \
-  --num_inference_steps 40 \
-  --flow_shift 12.0 \
-  --fps 16 \
-  --output i2v_output.mp4
-```
-
-Key arguments:
-
-- `--model`: Model ID (I2V-A14B for MoE, TI2V-5B for unified T2V+I2V).
-- `--image`: Path to input image (required).
-- `--prompt`: Text description of desired motion/animation.
-- `--height/--width`: Output resolution (auto-calculated from image if not set). Dimensions should be multiples of 16.
-- `--num_frames`: Number of frames (default 81).
-- `--guidance_scale` and `--guidance_scale_high`: CFG scale (applied to low/high-noise stages for MoE).
-- `--negative_prompt`: Optional list of artifacts to suppress.
-- `--cfg_parallel_size`: the number of devices to run CFG Parallel. CFG Parallel is valid only if classifier-free guidance is enabled and `cfg_parallel_size` is set to 2.
-- `--boundary_ratio`: Boundary split ratio for two-stage MoE models.
-- `--flow_shift`: Scheduler flow shift (5.0 for 720p, 12.0 for 480p).
-- `--num_inference_steps`: Number of denoising steps (default 50).
-- `--fps`: Frames per second for the saved MP4 (requires `diffusers` export_to_video).
-- `--output`: Path to save the generated video.
-
-## Example materials
-
-??? abstract "image_to_video.py"
-    ``````py
-    --8<-- "examples/offline_inference/image_to_video/image_to_video.py"
-    ``````

docs/user_guide/examples/offline_inference/lora_inference.md

@@ -1,107 +0,0 @@
-# LoRA-Inference
-
-Source <https://github.com/vllm-project/vllm-omni/tree/main/examples/offline_inference/lora_inference>.
-
-This contains examples for using LoRA (Low-Rank Adaptation) adapters with vLLM-omni diffusion models for offline inference.
-The example uses the  `stabilityai/stable-diffusion-3.5-medium` as the default model, but you can replace it with other models in vLLM-omni.
-
-## Overview
-
-Similar to vLLM, vLLM-omni uses a unified LoRA handling mechanism:
-
-- **Pre-loaded LoRA**: Loaded at initialization via `--lora-path` (pre-loaded into cache)
-- **Per-request LoRA**: Loaded on-demand. In the example, the LoRA is loaded via `--lora-request-path` in each request
-
-Both approaches use the same underlying mechanism - all LoRA adapters are handled uniformly through `set_active_adapter()`. If no LoRA request is provided in a request, all adapters are deactivated.
-
-## Usage
-
-### Pre-loaded LoRA (via --lora-path)
-
-Load a LoRA adapter at initialization. This adapter is pre-loaded into the cache and can be activated by requests:
-
-```bash
-python -m examples.offline_inference.lora_inference.lora_inference \
-    --prompt "A piece of cheesecake" \
-    --lora-path /path/to/lora/ \
-    --lora-scale 1.0 \
-    --num_inference_steps 50 \
-    --height 1024 \
-    --width 1024 \
-    --output output_preloaded.png
-```
-
-**Note**: When using `--lora-path`, the adapter is loaded at init time with a stable ID derived from the adapter path. This example activates it automatically for the request.
-
-### Per-request LoRA (via --lora-request-path)
-
-Load a LoRA adapter on-demand for each request:
-
-```bash
-python -m examples.offline_inference.lora_inference.lora_inference \
-    --prompt "A piece of cheesecake" \
-    --lora-request-path /path/to/lora/ \
-    --lora-scale 1.0 \
-    --num_inference_steps 50 \
-    --height 1024 \
-    --width 1024 \
-    --output output_per_request.png
-```
-
-### No LoRA
-
-If no LoRA request is provided, we will use the base model without any LoRA adapters:
-
-```bash
-python -m examples.offline_inference.lora_inference.lora_inference \
-    --prompt "A piece of cheesecake" \
-    --num_inference_steps 50 \
-    --height 1024 \
-    --width 1024 \
-    --output output_no_lora.png
-```
-
-## Parameters
-
-### LoRA Parameters
-
-- `--lora-path`: Path to LoRA adapter folder to pre-load at initialization (loads into cache with a stable ID derived from the path)
-- `--lora-request-path`: Path to LoRA adapter folder for per-request loading
-- `--lora-request-id`: Integer ID for the LoRA adapter (optional). If not provided and `--lora-request-path` is set, will derive a stable ID from the path.
-- `--lora-scale`: Scale factor for LoRA weights (default: 1.0). Higher values increase the influence of the LoRA adapter.
-
-### Standard Parameters
-
-- `--prompt`: Text prompt for image generation (required)
-- `--seed`: Random seed for reproducibility (default: 42)
-- `--height`: Image height in pixels (default: 1024)
-- `--width`: Image width in pixels (default: 1024)
-- `--num_inference_steps`: Number of denoising steps (default: 50)
-- `--output`: Output file path (default: `lora_output.png`)
-
-## How LoRA Works
-
-All LoRA adapters are handled uniformly:
-
-1. **Initialization**: If `--lora-path` is provided, the adapter is loaded into cache with a stable ID derived from the adapter path
-2. **Per-request**: If `--lora-request-path` is provided, the adapter is loaded/activated for that request
-3. **No LoRA**: If no LoRA request is provided (`req.lora_request` is None), all adapters are deactivated
-
-The system uses LRU cache management - adapters are cached and evicted when the cache is full (unless pinned).
-
-## LoRA Adapter Format
-
-LoRA adapters must be in PEFT (Parameter-Efficient Fine-Tuning) format. A typical LoRA adapter directory structure:
-
-```
-lora_adapter/
-├── adapter_config.json
-└── adapter_model.safetensors
-```
-
-## Example materials
-
-??? abstract "lora_inference.py"
-    ``````py
-    --8<-- "examples/offline_inference/lora_inference/lora_inference.py"
-    ``````

docs/user_guide/examples/offline_inference/qwen2_5_omni.md

@@ -2,7 +2,6 @@
 
 Source <https://github.com/vllm-project/vllm-omni/tree/main/examples/offline_inference/qwen2_5_omni>.
 
-
 ## Setup
 Please refer to the [stage configuration documentation](https://docs.vllm.ai/projects/vllm-omni/en/latest/configuration/stage_configs/) to configure memory allocation appropriately for your hardware setup.
 

docs/user_guide/examples/offline_inference/qwen3_omni.md

@@ -2,7 +2,6 @@
 
 Source <https://github.com/vllm-project/vllm-omni/tree/main/examples/offline_inference/qwen3_omni>.
 
-
 ## Setup
 Please refer to the [stage configuration documentation](https://docs.vllm.ai/projects/vllm-omni/en/latest/configuration/stage_configs/) to configure memory allocation appropriately for your hardware setup.
 

docs/user_guide/examples/offline_inference/qwen3_tts.md

@@ -1,94 +0,0 @@
-# Qwen3-TTS Offline Inference
-
-Source <https://github.com/vllm-project/vllm-omni/tree/main/examples/offline_inference/qwen3_tts>.
-
-
-This directory contains an offline demo for running Qwen3 TTS models with vLLM Omni. It builds task-specific inputs and generates WAV files locally.
-
-## Model Overview
-
-Qwen3 TTS provides multiple task variants for speech generation:
-
-- **CustomVoice**: Generate speech with a known speaker identity (speaker ID) and optional instruction.
-- **VoiceDesign**: Generate speech from text plus a descriptive instruction that designs a new voice.
-- **Base**: Voice cloning using a reference audio + reference transcript, with optional mode selection.
-
-## Setup
-Please refer to the [stage configuration documentation](https://docs.vllm.ai/projects/vllm-omni/en/latest/configuration/stage_configs/) to configure memory allocation appropriately for your hardware setup.
-
-## Quick Start
-
-Run a single sample for a task:
-
-```
-python end2end.py --query-type CustomVoice
-```
-
-Generated audio files are saved to `output_audio/` by default.
-
-## Task Usage
-
-### CustomVoice
-
-Single sample:
-
-```
-python end2end.py --query-type CustomVoice
-```
-
-Batch sample (multiple prompts in one run):
-
-```
-python end2end.py --query-type CustomVoice --use-batch-sample
-```
-
-### VoiceDesign
-
-Single sample:
-
-```
-python end2end.py --query-type VoiceDesign
-```
-
-Batch sample:
-
-```
-python end2end.py --query-type VoiceDesign --use-batch-sample
-```
-
-### Base (Voice Clone)
-
-Single sample:
-
-```
-python end2end.py --query-type Base
-```
-
-Batch sample:
-
-```
-python end2end.py --query-type Base --use-batch-sample
-```
-
-Mode selection for Base:
-
-- `--mode-tag icl` (default): standard mode
-- `--mode-tag xvec_only`: enable `x_vector_only_mode` in the request
-
-Examples:
-
-```
-python end2end.py --query-type Base --mode-tag icl
-```
-
-## Notes
-
-- The script uses the model paths embedded in `end2end.py`. Update them if your local cache path differs.
-- Use `--output-dir` (preferred) or `--output-wav` to change the output folder.
-
-## Example materials
-
-??? abstract "end2end.py"
-    ``````py
-    --8<-- "examples/offline_inference/qwen3_tts/end2end.py"
-    ``````

docs/user_guide/examples/offline_inference/text_to_image.md

@@ -23,7 +23,7 @@ if __name__ == "__main__":
     images[0].save("coffee.png")
 ```
 
-Or put more than one prompt in a request.
+Or put more than one prompt in a request, processing them sequentially.
 
 ```python
 from vllm_omni.entrypoints.omni import Omni
@@ -40,41 +40,6 @@ if __name__ == "__main__":
       image = output.request_output[0].images[0].save(f"{i}.jpg")
 ```
 
-!!! info
-
-    However, it is not currently recommended to do so
-    because not all models support batch inference,
-    and batch requesting mostly does not provide significant performance improvement (despite the impression that it does).
-    This feature is primarily for the sake of interface compatibility with vLLM and to allow for future improvements.
-
-!!! info
-
-    For diffusion pipelines, the stage config field `stage_args.[].runtime.max_batch_size` is 1 by default, and the input
-    list is sliced into single-item requests before feeding into the diffusion pipeline. For models that do internally support
-    batched inputs, you can [modify this configuration](../../../configuration/stage_configs.md) to let the model accept a longer batch of prompts.
-
-Apart from string prompt, vLLM-Omni also supports dictionary prompts in the same style as vLLM.
-This is useful for models that support negative prompts.
-
-```python
-from vllm_omni.entrypoints.omni import Omni
-
-if __name__ == "__main__":
-    omni = Omni(model="Qwen/Qwen-Image")
-    outputs = omni.generate([
-      {
-        "prompt": "a cup of coffee on a table"，
-        "negative_prompt": "low resolution"
-      },
-      {
-        "prompt": "a toy dinosaur on a sandy beach"，
-        "negative_prompt": "cinematic, realistic"
-      }
-    ])
-    for i, output in enumerate(outputs):
-      image = output.request_output[0].images[0].save(f"{i}.jpg")
-```
-
 ## Local CLI Usage
 
 ```bash
@@ -95,7 +60,6 @@ Key arguments:
 - `--prompt`: text description (string).
 - `--seed`: integer seed for deterministic sampling.
 - `--cfg_scale`: true CFG scale (model-specific guidance strength).
-- `--cfg_parallel_size`: the number of devices to run CFG Parallel. CFG Parallel is valid only if classifier-free guidance is enabled and `cfg_parallel_size` is set to 2.
 - `--num_images_per_prompt`: number of images to generate per prompt (saves as `output`, `output_1`, ...).
 - `--num_inference_steps`: diffusion sampling steps (more steps = higher quality, slower).
 - `--height/--width`: output resolution (defaults 1024x1024).

docs/user_guide/examples/offline_inference/text_to_video.md

@@ -28,7 +28,6 @@ Key arguments:
 - `--num_frames`: Number of frames (Wan default is 81).
 - `--guidance_scale` and `--guidance_scale_high`: CFG scale (applied to low/high)..
 - `--negative_prompt`: optional list of artifacts to suppress (the PR demo used a long Chinese string).
-- `--cfg_parallel_size`: the number of devices to run CFG Parallel. CFG Parallel is valid only if classifier-free guidance is enabled and `cfg_parallel_size` is set to 2.
 - `--boundary_ratio`: Boundary split ratio for low/high DiT.
 - `--fps`: frames per second for the saved MP4 (requires `diffusers` export_to_video).
 - `--output`: path to save the generated video.

docs/user_guide/examples/online_serving/bagel.md

@@ -1,232 +0,0 @@
-# BAGEL-7B-MoT
-
-Source <https://github.com/vllm-project/vllm-omni/tree/main/examples/online_serving/bagel>.
-
-## 🛠️ Installation
-
-Please refer to [README.md](../../../README.md)
-
-## Run examples (BAGEL-7B-MoT)
-
-**Note**: These examples work with the default configuration on an **NVIDIA A100 (80GB)**. We also tested on dual **NVIDIA RTX 5000 Ada (32GB each)**. For dual-GPU setups, please modify the stage configuration to distribute the model across devices.
-
-### Launch the Server
-
-```bash
-# Use default configuration
-vllm serve ByteDance-Seed/BAGEL-7B-MoT --omni --port 8091
-```
-
-Or use the convenience script:
-
-```bash
-cd /workspace/vllm-omni/examples/online_serving/bagel
-bash run_server.sh
-```
-
-If you have a custom stage configs file, launch the server with the command below:
-
-```bash
-vllm serve ByteDance-Seed/BAGEL-7B-MoT --omni --port 8091 --stage-configs-path /path/to/stage_configs_file
-```
-
-### Send Multi-modal Request
-
-Get into the bagel folder:
-
-```bash
-cd examples/online_serving/bagel
-```
-
-Send request via Python
-
-```bash
-python openai_chat_client.py --prompt "A cute cat" --modality text2img
-```
-
-The Python client supports the following command-line arguments:
-
-- `--prompt` (or `-p`): Text prompt for generation (default: `A cute cat`)
-- `--output` (or `-o`): Output file path for image results (default: `bagel_output.png`)
-- `--server` (or `-s`): Server URL (default: `http://localhost:8091`)
-- `--image-url` (or `-i`): Input image URL or local file path (for img2img/img2text modes)
-- `--modality` (or `-m`): Task modality (default: `text2img`). Options: `text2img`, `img2img`, `img2text`, `text2text`
-- `--height`: Image height in pixels (default: 512)
-- `--width`: Image width in pixels (default: 512)
-- `--steps`: Number of inference steps (default: 25)
-- `--seed`: Random seed (default: 42)
-- `--negative`: Negative prompt for image generation
-
-Example with custom parameters:
-
-```bash
-python openai_chat_client.py \
-    --prompt "A futuristic city" \
-    --modality text2img \
-    --height 768 \
-    --width 768 \
-    --steps 50 \
-    --seed 42 \
-    --negative "blurry, low quality"
-```
-
-## Modality Control
-
-BAGEL-7B-MoT supports **multiple modality modes** for different use cases.
-
-The default yaml configuration deploys Thinker and DiT on the same GPU. You can use the default configuration file: [`bagel.yaml`](../../../vllm_omni/model_executor/stage_configs/bagel.yaml)
-
-| Modality    | Input        | Output | Description                            |
-| ----------- | ------------ | ------ | -------------------------------------- |
-| `text2img`  | Text         | Image  | Generate images from text prompts      |
-| `img2img`   | Image + Text | Image  | Transform images using text guidance   |
-| `img2text`  | Image + Text | Text   | Generate text descriptions from images |
-| `text2text` | Text         | Text   | Pure text generation                   |
-
-### Text to Image (text2img)
-
-Generate images from text prompts:
-
-**Using Python client**
-
-```bash
-python openai_chat_client.py \
-    --prompt "A beautiful sunset over mountains" \
-    --modality text2img \
-    --output sunset.png \
-    --steps 50
-```
-
-**Using curl**
-
-```bash
-curl http://localhost:8091/v1/chat/completions \
-  -H "Content-Type: application/json" \
-  -d '{
-    "messages": [{"role": "user", "content": [{"type": "text", "text": "<|im_start|>A beautiful sunset over mountains<|im_end|>"}]}],
-    "modalities": ["image"],
-    "height": 512,
-    "width": 512,
-    "num_inference_steps": 50,
-    "seed": 42
-  }'
-```
-
-
-### Image to Image (img2img)
-
-Transform images based on text prompts:
-
-**Using Python client**
-
-```bash
-python openai_chat_client.py \
-    --prompt "Make the cat stand up" \
-    --modality img2img \
-    --image-url /path/to/input.jpg \
-    --output transformed.png
-```
-
-**Using curl**
-
-```bash
-IMAGE_BASE64=$(base64 -w 0 cat.jpg)
-
-cat <<EOF > payload.json
-{
-    "messages": [{
-      "role": "user",
-      "content": [
-        {"type": "text", "text": "<|im_start|>Make the cat stand up<|im_end|>"},
-        {"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,${IMAGE_BASE64}"}}
-      ]
-    }],
-    "modalities": ["image"],
-    "height": 512,
-    "width": 512,
-    "num_inference_steps": 50,
-    "seed": 42
-}
-EOF
-
-curl http://localhost:8091/v1/chat/completions \
-  -H "Content-Type: application/json" \
-  -d @payload.json
-
-```
-
-### Image to Text (img2text)
-
-Generate text descriptions from images:
-
-**Using Python client**
-
-```bash
-python openai_chat_client.py \
-    --prompt "Describe this image in detail" \
-    --modality img2text \
-    --image-url /path/to/image.jpg
-```
-
-**Using curl**
-
-```bash
-IMAGE_BASE64=$(base64 -w 0 cat.jpg)
-
-cat <<EOF > payload.json
-{
-  "messages": [{
-    "role": "user",
-    "content": [
-      {"type": "text", "text": "<|im_start|>user\n<|image_pad|>\nDescribe this image in detail<|im_end|>\n<|im_start|>assistant\n"},
-      {"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,${IMAGE_BASE64}"}}
-    ]
-  }],
-  "modalities": ["text"]
-}
-EOF
-
-curl http://localhost:8091/v1/chat/completions \
-  -H "Content-Type: application/json" \
-  -d @payload.json
-```
-
-### Text to Text (text2text)
-
-Pure text generation:
-
-**Using Python client**
-
-```bash
-python openai_chat_client.py \
-    --prompt "What is the capital of France?" \
-    --modality text2text
-```
-
-**Using curl**
-
-```bash
-curl http://localhost:8091/v1/chat/completions \
-  -H "Content-Type: application/json" \
-  -d '{
-    "messages": [{"role": "user", "content": [{"type": "text", "text": "<|im_start|>user\nWhat is the capital of France?<|im_end|>\n<|im_start|>assistant\n"}]}]
-    "modalities": ["text"]
-  }'
-```
-
-## FAQ
-
-- If you encounter an error about the backend of librosa, try to install ffmpeg with the command below.
-
-```bash
-sudo apt update
-sudo apt install ffmpeg
-```
-
-- If you don’t know how much VRAM is needed for the model or encounter the OOM error, you can try to decrease the max_model_len.
-
-| Stage               | VRAM                         |
-| :------------------ | :--------------------------- |
-| Stage-0 (Thinker)   | **15.04 GiB** **+ KV Cache** |
-| Stage-1 (DiT)       | **26.50 GiB**                |
-| Total               | **~42 GiB + KV Cache**       |

docs/user_guide/examples/online_serving/gradio_demo.md

@@ -0,0 +1,7 @@
+# Gradio Demo
+
+Source <https://github.com/vllm-project/vllm-omni/blob/main/examples/online_serving/gradio_demo.py>.
+
+``````py
+--8<-- "examples/online_serving/gradio_demo.py"
+``````

docs/user_guide/examples/online_serving/image_to_image.md

@@ -230,10 +230,6 @@ Provide multiple images in `content` (order matters):
     ``````py
     --8<-- "examples/online_serving/image_to_image/openai_chat_client.py"
     ``````
-??? abstract "run_curl_image_edit.sh"
-    ``````sh
-    --8<-- "examples/online_serving/image_to_image/run_curl_image_edit.sh"
-    ``````
 ??? abstract "run_server.sh"
     ``````sh
     --8<-- "examples/online_serving/image_to_image/run_server.sh"

docs/user_guide/examples/online_serving/lora_inference.md

@@ -1,69 +0,0 @@
-# LoRA-Inference
-
-Source <https://github.com/vllm-project/vllm-omni/tree/main/examples/online_serving/lora_inference>.
-
-This example shows how to use **per-request LoRA** with vLLM-Omni diffusion models via the OpenAI-compatible Chat Completions API.
-
-> Note: The LoRA adapter path must be readable on the **server** machine (usually a local path or a mounted directory).
-> Note: This example uses `/v1/chat/completions`. LoRA payloads for other OpenAI endpoints are not implemented here.
-
-## Start Server
-
-```bash
-# Pick a diffusion model (examples)
-# export MODEL=stabilityai/stable-diffusion-3.5-medium
-# export MODEL=Qwen/Qwen-Image
-
-bash run_server.sh
-```
-
-## Call API (curl)
-
-```bash
-# Required: local LoRA folder on the server
-export LORA_PATH=/path/to/lora_adapter
-
-# Optional
-export SERVER=http://localhost:8091
-export PROMPT="A piece of cheesecake"
-export LORA_NAME=my_lora
-export LORA_SCALE=1.0
-# Optional: if omitted, the server derives a stable id from LORA_PATH.
-# export LORA_INT_ID=123
-
-bash run_curl_lora_inference.sh
-```
-
-## Call API (Python)
-
-```bash
-python openai_chat_client.py \
-  --prompt "A piece of cheesecake" \
-  --lora-path /path/to/lora_adapter \
-  --lora-name my_lora \
-  --lora-scale 1.0 \
-  --output output.png
-```
-
-## LoRA Format
-
-LoRA adapters should be in PEFT format, for example:
-
-```
-lora_adapter/
-├── adapter_config.json
-└── adapter_model.safetensors
-```
-
-??? abstract "openai_chat_client.py"
-    ``````py
-    --8<-- "examples/online_serving/lora_inference/openai_chat_client.py"
-    ``````
-??? abstract "run_curl_lora_inference.sh"
-    ``````py
-    --8<-- "examples/online_serving/lora_inference/run_curl_lora_inference.sh"
-    ``````
-??? abstract "run_server.sh"
-    ``````py
-    --8<-- "examples/online_serving/lora_inference/run_server.sh"
-    ``````

docs/user_guide/examples/online_serving/openai_chat_completion_client_for_multimodal_generation.md

@@ -0,0 +1,7 @@
+# OpenAI Chat Completion Client For Multimodal Generation
+
+Source <https://github.com/vllm-project/vllm-omni/blob/main/examples/online_serving/openai_chat_completion_client_for_multimodal_generation.py>.
+
+``````py
+--8<-- "examples/online_serving/openai_chat_completion_client_for_multimodal_generation.py"
+``````

docs/user_guide/examples/online_serving/qwen2_5_omni.md

@@ -137,7 +137,6 @@ response = client.chat.completions.create(
 # Response contains two choices: one with text, one with audio
 print(response.choices[0].message.content)  # Text response
 print(response.choices[1].message.audio)    # Audio response
-```
 
 ## Streaming Output
 If you want to enable streaming output, please set the argument as below. The final output will be obtained just after generated by corresponding stage. Now we only support text streaming output. Other modalities can output normally.

docs/user_guide/examples/online_serving/qwen3_omni.md

@@ -15,12 +15,6 @@ Please refer to [README.md](https://github.com/vllm-project/vllm-omni/tree/main/
 vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091
 ```
 
-If you want to open async chunking for qwen3-omni, launch the server with command below
-
-```bash
-vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091 --stage-configs-path /vllm_omni/model_executor/stage_configs/qwen3_omni_moe_async_chunk.yaml
-```
-
 If you have custom stage configs file, launch the server with command below
 ```bash
 vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091 --stage-configs-path /path/to/stage_configs_file
@@ -151,7 +145,6 @@ response = client.chat.completions.create(
 # Response contains two choices: one with text, one with audio
 print(response.choices[0].message.content)  # Text response
 print(response.choices[1].message.audio)    # Audio response
-```
 
 ## Streaming Output
 If you want to enable streaming output, please set the argument as below. The final output will be obtained just after generated by corresponding stage. Now we only support text streaming output. Other modalities can output normally.

docs/user_guide/examples/online_serving/text_to_image.md

@@ -12,8 +12,6 @@ This example demonstrates how to deploy Qwen-Image model for online image genera
 ```bash
 vllm serve Qwen/Qwen-Image --omni --port 8091
 ```
-!!! note
-    If you encounter Out-of-Memory (OOM) issues or have limited GPU memory, you can enable VAE slicing and tiling to reduce memory usage, --vae-use-slicing --vae-use-tiling
 
 ### Start with Parameters
 
@@ -45,7 +43,7 @@ curl -s http://localhost:8091/v1/chat/completions \
       "true_cfg_scale": 4.0,
       "seed": 42
     }
-  }' | jq -r '.choices[0].message.content[0].image_url.url' | cut -d',' -f2- | base64 -d > output.png
+  }' | jq -r '.choices[0].message.content[0].image_url.url' | cut -d',' -f2 | base64 -d > output.png
 ```
 
 ### Method 2: Using Python Client
@@ -119,7 +117,6 @@ Use `extra_body` to pass generation parameters:
 | `seed`                   | int   | None    | Random seed (reproducible)     |
 | `negative_prompt`        | str   | None    | Negative prompt                |
 | `num_outputs_per_prompt` | int   | 1       | Number of images to generate   |
-| `--cfg-parallel-size`.   | int   | 1       | Number of GPUs for CFG parallelism |
 
 ## Response Format
 
@@ -149,7 +146,7 @@ Use `extra_body` to pass generation parameters:
 
 ```bash
 # Extract base64 from response and decode to image
-cat response.json | jq -r '.choices[0].message.content[0].image_url.url' | cut -d',' -f2- | base64 -d > output.png
+cat response.json | jq -r '.choices[0].message.content[0].image_url.url' | cut -d',' -f2 | base64 -d > output.png
 ```
 
 ## File Description

examples/offline_inference/bagel/README.md

@@ -1,177 +0,0 @@
-# BAGEL-7B-MoT
-
-## Set up
-
-Please refer to the [stage configuration documentation](https://docs.vllm.ai/projects/vllm-omni/en/latest/configuration/stage_configs/) to configure memory allocation appropriately for your hardware setup.
-
-## Run examples
-
-**Note**: These examples work with the default configuration on an **NVIDIA A100 (80GB)**. We also tested on dual **NVIDIA RTX 5000 Ada (32GB each)**. For dual-GPU setups, please modify the stage configuration to distribute the model across devices.
-
-Get into the bagel folder
-
-```bash
-cd examples/offline_inference/bagel
-```
-
-### Modality Control
-
-BAGEL-7B-MoT supports multiple modality modes. You can control the mode using the `--modality` argument:
-
-#### Text to Image (text2img)
-
-- **Pipeline**: Text → Thinker  → DiT → VAE Decode → Image
-- **Stages Used**: Stage 0 (Thinker) + Stage 1 (DiT)
-- **KV Transfer**: Thinker sends KV cache to DiT for conditioned generation
-
-Generate images from text prompts:
-
-```bash
-python end2end.py --model ByteDance-Seed/BAGEL-7B-MoT \
-                  --modality text2img \
-                  --prompts "A cute cat"
-```
-
-#### Image to Image (img2img)
-
-- **Pipeline**: Image → VAE Encode → DiT → VAE Decode → New Image
-- **Stages Used**: Stage 1 (DiT) only
-- **Special**: Bypasses the Thinker stage, direct image-to-image transformation
-
-Transform images based on text prompts:
-
-```bash
-python end2end.py --model ByteDance-Seed/BAGEL-7B-MoT \
-                  --modality img2img \
-                  --image-path /path/to/image.jpg \
-                  --prompts "Let the woman wear a blue dress"
-```
-
-#### Image to Text (img2text)
-
-- **Pipeline**: Image → ViT + VAE Encode → Thinker → Text Output
-- **Stages Used**: Stage 0 (Thinker) only
-- **Special**: Uses both VAE latent encoding AND ViT semantic encoding for comprehensive image understanding
-
-Generate text descriptions from images:
-
-```bash
-python end2end.py --model ByteDance-Seed/BAGEL-7B-MoT \
-                  --modality img2text \
-                  --image-path /path/to/image.jpg \
-                  --prompts "Describe this image in detail"
-```
-
-#### Text to Text (text2text)
-
-- **Pipeline**: Text → Thinker → Text Output
-- **Stages Used**: Stage 0 (Thinker) only
-- **Special**: No visual components involved, operates as pure language model
-
-Pure text generation:
-
-```bash
-python end2end.py --model ByteDance-Seed/BAGEL-7B-MoT \
-                  --modality text2text \
-                  --prompts "What is the capital of France?"
-
-# You can load prompts from a text file (one prompt per line):  
-python end2end.py --model ByteDance-Seed/BAGEL-7B-MoT \
-                  --modality text2text \
-                  --txt-prompts /path/to/prompts.txt
-```
-
-### Inference Steps
-
-Control the number of inference steps for image generation:
-
-```bash
-# You can adjust steps to 100 to improve image quality
-python end2end.py --model ByteDance-Seed/BAGEL-7B-MoT \
-                  --modality text2img \
-                  --steps 50 \
-                  --prompts "A cute cat"
-```
-
-### Key arguments
-
-BAGEL-7B-MoT supports **multiple modality modes** for different use cases.
-
-The default yaml configuration deploys Thinker and DiT on the same GPU. You can use the default configuration file: [`bagel.yaml`](../../../vllm_omni/model_executor/stage_configs/bagel.yaml)
-
-#### 📌 Command Line Arguments (end2end.py)
-
-| Argument               | Type   | Default                       | Description                                                  |
-| :--------------------- | :----- | :---------------------------- | :----------------------------------------------------------- |
-| `--model`              | string | `ByteDance-Seed/BAGEL-7B-MoT` | Model path or name                                           |
-| `--modality`           | choice | `text2img`                    | Modality mode: `text2img`, `img2img`, `img2text`, `text2text` |
-| `--prompts`            | list   | `None`                        | Input text prompts directly                                  |
-| `--txt-prompts`        | string | `None`                        | Path to txt file with one prompt per line                    |
-| `--image-path`         | string | `None`                        | Input image path (for `img2img`/`img2text`)                  |
-| `--steps`              | int    | `50`                          | Number of inference steps                                    |
-| `--stage-configs-path` | string | `None`                        | Custom stage config file path                                |
-| `--worker-backend`     | choice | `process`                     | Worker backend: `process` or `ray`                           |
-| `--ray-address`        | string | `None`                        | Ray cluster address                                          |
-| `--enable-stats`       | flag   | `False`                       | Enable statistics logging                                    |
-| `--init-sleep-seconds` | int    | `20`                          | Initialization sleep time                                    |
-| `--batch-timeout`      | int    | `5`                           | Batch timeout                                                |
-| `--init-timeout`       | int    | `300`                         | Initialization timeout                                       |
-
-------
-
-#### ⚙️ Stage Configuration Parameters (bagel.yaml)
-
- **Stage 0 - Thinker (LLM Stage)**
-
-| Parameter                        | Value                           | Description              |
-| :------------------------------- | :------------------------------ | :----------------------- |
-| `stage_type`                     | `llm`                           | Stage type               |
-| `devices`                        | `"0"`                           | GPU device ID            |
-| `max_batch_size`                 | `1`                             | Maximum batch size       |
-| `model_stage`                    | `thinker`                       | Model stage identifier   |
-| `model_arch`                     | `BagelForConditionalGeneration` | Model architecture       |
-| `gpu_memory_utilization`         | `0.4`                           | GPU memory utilization   |
-| `tensor_parallel_size`           | `1`                             | Tensor parallel size     |
-| `max_num_batched_tokens`         | `32768`                         | Maximum batched tokens   |
-| `omni_kv_config.need_send_cache` | `true`                          | Whether to send KV cache |
-
-------
-
-**Stage 1 - DiT (Diffusion Stage)**
-
-| Parameter                        | Value       | Description                 |
-| :------------------------------- | :---------- | :-------------------------- |
-| `stage_type`                     | `diffusion` | Stage type                  |
-| `devices`                        | `"0"`       | GPU device ID               |
-| `max_batch_size`                 | `1`         | Maximum batch size          |
-| `model_stage`                    | `dit`       | Model stage identifier      |
-| `gpu_memory_utilization`         | `0.4`       | GPU memory utilization      |
-| `omni_kv_config.need_recv_cache` | `true`      | Whether to receive KV cache |
-| `engine_input_source`            | `[0]`       | Input source from Stage 0   |
-
-------
-
-#### 🔗 Runtime Configuration
-
-| Parameter             | Value   | Description                      |
-| :-------------------- | :------ | :------------------------------- |
-| `window_size`         | `-1`    | Window size (-1 means unlimited) |
-| `max_inflight`        | `1`     | Maximum inflight requests        |
-| `shm_threshold_bytes` | `65536` | Shared memory threshold (64KB)   |
-
-## FAQ
-
-- If you encounter an error about the backend of librosa, try to install ffmpeg with the command below.
-
-```bash
-sudo apt update
-sudo apt install ffmpeg
-```
-
-- If you don’t know how much VRAM is needed for the model or encounter the OOM error, you can try to decrease the max_model_len.
-
-| Stage               | VRAM                         |
-| :------------------ | :--------------------------- |
-| Stage-0 (Thinker)   | **15.04 GiB** **+ KV Cache** |
-| Stage-1 (DiT)       | **26.50 GiB**                |
-| Total               | **~42 GiB + KV Cache**       |

examples/offline_inference/bagel/end2end.py

@@ -1,189 +0,0 @@
-import argparse
-import os
-from typing import cast
-
-from vllm_omni.inputs.data import OmniDiffusionSamplingParams, OmniPromptType
-
-
-def parse_args():
-    parser = argparse.ArgumentParser()
-    parser.add_argument(
-        "--model",
-        default="ByteDance-Seed/BAGEL-7B-MoT",
-        help="Path to merged model directory.",
-    )
-    parser.add_argument("--prompts", nargs="+", default=None, help="Input text prompts.")
-    parser.add_argument(
-        "--txt-prompts",
-        type=str,
-        default=None,
-        help="Path to a .txt file with one prompt per line (preferred).",
-    )
-    parser.add_argument("--prompt_type", default="text", choices=["text"])
-
-    parser.add_argument(
-        "--modality",
-        default="text2img",
-        choices=["text2img", "img2img", "img2text", "text2text"],
-        help="Modality mode to control stage execution.",
-    )
-
-    parser.add_argument(
-        "--image-path",
-        type=str,
-        default=None,
-        help="Path to input image for img2img.",
-    )
-
-    # OmniLLM init args
-    parser.add_argument("--enable-stats", action="store_true", default=False)
-    parser.add_argument("--init-sleep-seconds", type=int, default=20)
-    parser.add_argument("--batch-timeout", type=int, default=5)
-    parser.add_argument("--init-timeout", type=int, default=300)
-    parser.add_argument("--shm-threshold-bytes", type=int, default=65536)
-    parser.add_argument("--worker-backend", type=str, default="process", choices=["process", "ray"])
-    parser.add_argument("--ray-address", type=str, default=None)
-    parser.add_argument("--stage-configs-path", type=str, default=None)
-    parser.add_argument("--steps", type=int, default=50, help="Number of inference steps.")
-
-    args = parser.parse_args()
-    return args
-
-
-def main():
-    args = parse_args()
-    model_name = args.model
-    prompts: list[OmniPromptType] = []
-    try:
-        # Preferred: load from txt file (one prompt per line)
-        if getattr(args, "txt_prompts", None) and args.prompt_type == "text":
-            with open(args.txt_prompts, encoding="utf-8") as f:
-                lines = [ln.strip() for ln in f.readlines()]
-            prompts = [ln for ln in lines if ln != ""]
-            print(f"[Info] Loaded {len(prompts)} prompts from {args.txt_prompts}")
-        else:
-            prompts = args.prompts
-    except Exception as e:
-        print(f"[Error] Failed to load prompts: {e}")
-        raise
-
-    if not prompts:
-        # Default prompt for text2img test if none provided
-        prompts = ["<|im_start|>A cute cat<|im_end|>"]
-        print(f"[Info] No prompts provided, using default: {prompts}")
-    omni_outputs = []
-
-    from PIL import Image
-
-    if args.modality == "img2img":
-        from PIL import Image
-
-        from vllm_omni.entrypoints.omni_diffusion import OmniDiffusion
-
-        print("[Info] Running in img2img mode (Stage 1 only)")
-        client = OmniDiffusion(model=model_name)
-
-        if args.image_path:
-            if os.path.exists(args.image_path):
-                loaded_image = Image.open(args.image_path).convert("RGB")
-                prompts = [
-                    {
-                        "prompt": cast(str, p),
-                        "multi_modal_data": {"image": loaded_image},
-                    }
-                    for p in prompts
-                ]
-            else:
-                print(f"[Warning] Image path {args.image_path} does not exist.")
-
-        result = client.generate(
-            prompts,
-            OmniDiffusionSamplingParams(
-                seed=52,
-                need_kv_receive=False,
-                num_inference_steps=args.steps,
-            ),
-        )
-
-        # Ensure result is a list for iteration
-        if not isinstance(result, list):
-            omni_outputs = [result]
-        else:
-            omni_outputs = result
-
-    else:
-        from vllm_omni.entrypoints.omni import Omni
-
-        omni_kwargs = {}
-        if args.stage_configs_path:
-            omni_kwargs["stage_configs_path"] = args.stage_configs_path
-
-        omni_kwargs.update(
-            {
-                "log_stats": args.enable_stats,
-                "init_sleep_seconds": args.init_sleep_seconds,
-                "batch_timeout": args.batch_timeout,
-                "init_timeout": args.init_timeout,
-                "shm_threshold_bytes": args.shm_threshold_bytes,
-                "worker_backend": args.worker_backend,
-                "ray_address": args.ray_address,
-            }
-        )
-
-        omni = Omni(model=model_name, **omni_kwargs)
-
-        formatted_prompts = []
-        for p in args.prompts:
-            if args.modality == "img2text":
-                if args.image_path:
-                    loaded_image = Image.open(args.image_path).convert("RGB")
-                    final_prompt_text = f"<|im_start|>user\n<|image_pad|>\n{p}<|im_end|>\n<|im_start|>assistant\n"
-                    prompt_dict = {
-                        "prompt": final_prompt_text,
-                        "multi_modal_data": {"image": loaded_image},
-                        "modalities": ["text"],
-                    }
-                    formatted_prompts.append(prompt_dict)
-            elif args.modality == "text2text":
-                final_prompt_text = f"<|im_start|>user\n{p}<|im_end|>\n<|im_start|>assistant\n"
-                prompt_dict = {"prompt": final_prompt_text, "modalities": ["text"]}
-                formatted_prompts.append(prompt_dict)
-            else:
-                # text2img
-                final_prompt_text = f"<|im_start|>{p}<|im_end|>"
-                prompt_dict = {"prompt": final_prompt_text, "modalities": ["image"]}
-                formatted_prompts.append(prompt_dict)
-
-        params_list = omni.default_sampling_params_list
-        if args.modality == "text2img":
-            params_list[0].max_tokens = 1  # type: ignore # The first stage is a SamplingParam (vllm)
-            if len(params_list) > 1:
-                params_list[1].num_inference_steps = args.steps  # type: ignore # The second stage is an OmniDiffusionSamplingParam
-
-        omni_outputs = list(omni.generate(prompts=formatted_prompts, sampling_params_list=params_list))
-
-    for i, req_output in enumerate(omni_outputs):
-        images = getattr(req_output, "images", None)
-        if not images and hasattr(req_output, "output"):
-            if isinstance(req_output.output, list):
-                images = req_output.output
-            else:
-                images = [req_output.output]
-
-        if images:
-            for j, img in enumerate(images):
-                img.save(f"output_{i}_{j}.png")
-
-        if hasattr(req_output, "request_output") and req_output.request_output:
-            for stage_out in req_output.request_output:
-                if hasattr(stage_out, "images") and stage_out.images:
-                    for k, img in enumerate(stage_out.images):
-                        save_path = f"output_{i}_stage_{getattr(stage_out, 'stage_id', '?')}_{k}.png"
-                        img.save(save_path)
-                        print(f"[Info] Saved stage output image to {save_path}")
-
-    print(omni_outputs)
-
-
-if __name__ == "__main__":
-    main()

examples/offline_inference/image_to_image/image_edit.py

@@ -49,22 +49,6 @@ Usage (layered):
         --layers 4 \
         --color-format "RGBA"
 
-Usage (with CFG Parallel):
-    python image_edit.py \
-        --image input.png \
-        --prompt "Edit description" \
-        --cfg_parallel_size 2 \
-        --num_inference_steps 50 \
-        --cfg_scale 4.0
-
-Usage (disable torch.compile):
-    python image_edit.py \
-        --image input.png \
-        --prompt "Edit description" \
-        --enforce_eager \
-        --num_inference_steps 50 \
-        --cfg_scale 4.0
-
 For more options, run:
     python image_edit.py --help
 """
@@ -77,11 +61,10 @@ from pathlib import Path
 import torch
 from PIL import Image
 
-from vllm_omni.diffusion.data import DiffusionParallelConfig
+from vllm_omni.diffusion.data import DiffusionParallelConfig, logger
 from vllm_omni.entrypoints.omni import Omni
-from vllm_omni.inputs.data import OmniDiffusionSamplingParams
 from vllm_omni.outputs import OmniRequestOutput
-from vllm_omni.platforms import current_omni_platform
+from vllm_omni.utils.platform_utils import detect_device_type, is_npu
 
 
 def parse_args() -> argparse.Namespace:
@@ -111,7 +94,7 @@ def parse_args() -> argparse.Namespace:
     parser.add_argument(
         "--negative_prompt",
         type=str,
-        default=None,
+        default=" ",
         required=False,
     )
     parser.add_argument(
@@ -182,12 +165,6 @@ def parse_args() -> argparse.Namespace:
         default=1,
         help="Number of GPUs used for ring sequence parallelism.",
     )
-    parser.add_argument(
-        "--tensor_parallel_size",
-        type=int,
-        default=1,
-        help="Number of GPUs used for tensor parallelism (TP) inside the DiT.",
-    )
     parser.add_argument("--layers", type=int, default=4, help="Number of layers to decompose the input image into.")
     parser.add_argument(
         "--resolution",
@@ -268,44 +245,7 @@ def parse_args() -> argparse.Namespace:
         default=0.2,
         help="[tea_cache] Threshold for accumulated relative L1 distance.",
     )
-    parser.add_argument(
-        "--cfg_parallel_size",
-        type=int,
-        default=1,
-        choices=[1, 2],
-        help="Number of GPUs used for classifier free guidance parallel size.",
-    )
-    parser.add_argument(
-        "--enforce_eager",
-        action="store_true",
-        help="Disable torch.compile and force eager execution.",
-    )
-    parser.add_argument(
-        "--vae_use_slicing",
-        action="store_true",
-        help="Enable VAE slicing for memory optimization.",
-    )
-    parser.add_argument(
-        "--vae_use_tiling",
-        action="store_true",
-        help="Enable VAE tiling for memory optimization.",
-    )
-    parser.add_argument(
-        "--enable-cpu-offload",
-        action="store_true",
-        help="Enable CPU offloading for diffusion models.",
-    )
-    parser.add_argument(
-        "--enable-layerwise-offload",
-        action="store_true",
-        help="Enable layerwise (blockwise) offloading on DiT modules.",
-    )
-    parser.add_argument(
-        "--layerwise-num-gpu-layers",
-        type=int,
-        default=1,
-        help="Number of ready layers (blocks) to keep on GPU during generation.",
-    )
+
     return parser.parse_args()
 
 
@@ -327,15 +267,13 @@ def main():
     else:
         input_image = input_images
 
-    generator = torch.Generator(device=current_omni_platform.device_type).manual_seed(args.seed)
-
-    parallel_config = DiffusionParallelConfig(
-        ulysses_degree=args.ulysses_degree,
-        ring_degree=args.ring_degree,
-        cfg_parallel_size=args.cfg_parallel_size,
-        tensor_parallel_size=args.tensor_parallel_size,
-    )
+    device = detect_device_type()
+    generator = torch.Generator(device=device).manual_seed(args.seed)
 
+    # Enable VAE memory optimizations on NPU
+    vae_use_slicing = is_npu()
+    vae_use_tiling = is_npu()
+    parallel_config = DiffusionParallelConfig(ulysses_degree=args.ulysses_degree, ring_degree=args.ring_degree)
     # Configure cache based on backend type
     cache_config = None
     if args.cache_backend == "cache_dit":
@@ -361,21 +299,14 @@ def main():
     # Initialize Omni with appropriate pipeline
     omni = Omni(
         model=args.model,
-        enable_layerwise_offload=args.enable_layerwise_offload,
-        layerwise_num_gpu_layers=args.layerwise_num_gpu_layers,
-        vae_use_slicing=args.vae_use_slicing,
-        vae_use_tiling=args.vae_use_tiling,
+        vae_use_slicing=vae_use_slicing,
+        vae_use_tiling=vae_use_tiling,
         cache_backend=args.cache_backend,
         cache_config=cache_config,
         parallel_config=parallel_config,
-        enforce_eager=args.enforce_eager,
-        enable_cpu_offload=args.enable_cpu_offload,
     )
     print("Pipeline loaded")
 
-    # Check if profiling is requested via environment variable
-    profiler_enabled = bool(os.getenv("VLLM_TORCH_PROFILER_DIR"))
-
     # Time profiling for generation
     print(f"\n{'=' * 60}")
     print("Generation Configuration:")
@@ -388,59 +319,34 @@ def main():
             print(f"    Image {idx + 1} size: {img.size}")
     else:
         print(f"  Input image size: {input_image.size}")
-    print(
-        f"  Parallel configuration: ulysses_degree={args.ulysses_degree}, ring_degree={args.ring_degree}, cfg_parallel_size={args.cfg_parallel_size}, tensor_parallel_size={args.tensor_parallel_size}"
-    )
+    print(f"  Parallel configuration: ulysses_degree={args.ulysses_degree}, ring_degree={args.ring_degree}")
     print(f"{'=' * 60}\n")
 
     generation_start = time.perf_counter()
-
-    if profiler_enabled:
-        print("[Profiler] Starting profiling...")
-        omni.start_profile()
-
     # Generate edited image
-    outputs = omni.generate(
-        {
-            "prompt": args.prompt,
-            "negative_prompt": args.negative_prompt,
-            "multi_modal_data": {"image": input_image},
-        },
-        OmniDiffusionSamplingParams(
-            generator=generator,
-            true_cfg_scale=args.cfg_scale,
-            guidance_scale=args.guidance_scale,
-            num_inference_steps=args.num_inference_steps,
-            num_outputs_per_prompt=args.num_outputs_per_prompt,
-            layers=args.layers,
-            resolution=args.resolution,
-        ),
-    )
+    generate_kwargs = {
+        "prompt": args.prompt,
+        "pil_image": input_image,
+        "negative_prompt": args.negative_prompt,
+        "generator": generator,
+        "true_cfg_scale": args.cfg_scale,
+        "guidance_scale": args.guidance_scale,
+        "num_inference_steps": args.num_inference_steps,
+        "num_outputs_per_prompt": args.num_outputs_per_prompt,
+        "layers": args.layers,
+        "resolution": args.resolution,
+    }
+
+    outputs = omni.generate(**generate_kwargs)
     generation_end = time.perf_counter()
     generation_time = generation_end - generation_start
 
     # Print profiling results
     print(f"Total generation time: {generation_time:.4f} seconds ({generation_time * 1000:.2f} ms)")
 
-    if profiler_enabled:
-        print("\n[Profiler] Stopping profiler and collecting results...")
-        profile_results = omni.stop_profile()
-        if profile_results and isinstance(profile_results, dict):
-            traces = profile_results.get("traces", [])
-            print("\n" + "=" * 60)
-            print("PROFILING RESULTS:")
-            for rank, trace in enumerate(traces):
-                print(f"\nRank {rank}:")
-                if trace:
-                    print(f"  • Trace: {trace}")
-            if not traces:
-                print("  No traces collected.")
-            print("=" * 60)
-        else:
-            print("[Profiler] No valid profiling data returned.")
-
     if not outputs:
         raise ValueError("No output generated from omni.generate()")
+    logger.info("Outputs: %s", outputs)
 
     # Extract images from OmniRequestOutput
     # omni.generate() returns list[OmniRequestOutput], extract images from request_output[0].images

examples/offline_inference/image_to_image/image_to_image.md

@@ -47,9 +47,3 @@ Key arguments:
 - `--guidance_scale`: guidance scale for guidance-distilled models (default: 1.0, disabled). Unlike classifier-free guidance (--cfg_scale), guidance-distilled models take the guidance scale directly as an input parameter. Enabled when guidance_scale > 1. Ignored when not using guidance-distilled models.
 - `--num_inference_steps`: diffusion sampling steps (more steps = higher quality, slower).
 - `--output`: path to save the generated PNG.
-- `--vae_use_slicing`: enable VAE slicing for memory optimization.
-- `--vae_use_tiling`: enable VAE tiling for memory optimization.
-- `--cfg_parallel_size`: set it to 2 to enable CFG Parallel. See more examples in [`user_guide`](../../../docs/user_guide/diffusion/parallelism_acceleration.md#cfg-parallel).
-- `--enable-cpu-offload`: enable CPU offloading for diffusion models.
-
-> ℹ️ If you encounter OOM errors, try using `--vae_use_slicing` and `--vae_use_tiling` to reduce memory usage.

examples/offline_inference/image_to_video/README.md

@@ -54,9 +54,3 @@ Key arguments:
 - `--num_inference_steps`: Number of denoising steps (default 50).
 - `--fps`: Frames per second for the saved MP4 (requires `diffusers` export_to_video).
 - `--output`: Path to save the generated video.
-- `--vae_use_slicing`: Enable VAE slicing for memory optimization.
-- `--vae_use_tiling`: Enable VAE tiling for memory optimization.
-- `--cfg_parallel_size`: set it to 2 to enable CFG Parallel. See more examples in [`user_guide`](../../../docs/user_guide/diffusion/parallelism_acceleration.md#cfg-parallel).
-- `--enable-cpu-offload`: enable CPU offloading for diffusion models.
-
-> ℹ️ If you encounter OOM errors, try using `--vae_use_slicing` and `--vae_use_tiling` to reduce memory usage.

examples/offline_inference/image_to_video/image_to_video.py

@@ -19,18 +19,15 @@ Usage:
 """
 
 import argparse
-import os
 from pathlib import Path
 
 import numpy as np
 import PIL.Image
 import torch
 
-from vllm_omni.diffusion.data import DiffusionParallelConfig
 from vllm_omni.entrypoints.omni import Omni
-from vllm_omni.inputs.data import OmniDiffusionSamplingParams
 from vllm_omni.outputs import OmniRequestOutput
-from vllm_omni.platforms import current_omni_platform
+from vllm_omni.utils.platform_utils import detect_device_type, is_npu
 
 
 def parse_args() -> argparse.Namespace:
@@ -60,44 +57,6 @@ def parse_args() -> argparse.Namespace:
     )
     parser.add_argument("--output", type=str, default="i2v_output.mp4", help="Path to save the video (mp4).")
     parser.add_argument("--fps", type=int, default=16, help="Frames per second for the output video.")
-    parser.add_argument(
-        "--vae_use_slicing",
-        action="store_true",
-        help="Enable VAE slicing for memory optimization.",
-    )
-    parser.add_argument(
-        "--vae_use_tiling",
-        action="store_true",
-        help="Enable VAE tiling for memory optimization.",
-    )
-    parser.add_argument(
-        "--enable-cpu-offload",
-        action="store_true",
-        help="Enable CPU offloading for diffusion models.",
-    )
-    parser.add_argument(
-        "--enable-layerwise-offload",
-        action="store_true",
-        help="Enable layerwise (blockwise) offloading on DiT modules.",
-    )
-    parser.add_argument(
-        "--layerwise-num-gpu-layers",
-        type=int,
-        default=1,
-        help="Number of ready layers (blocks) to keep on GPU during generation.",
-    )
-    parser.add_argument(
-        "--cfg_parallel_size",
-        type=int,
-        default=1,
-        choices=[1, 2],
-        help="Number of GPUs used for classifier free guidance parallel size.",
-    )
-    parser.add_argument(
-        "--enforce_eager",
-        action="store_true",
-        help="Disable torch.compile and force eager execution.",
-    )
     return parser.parse_args()
 
 
@@ -114,7 +73,8 @@ def calculate_dimensions(image: PIL.Image.Image, max_area: int = 480 * 832) -> t
 
 def main():
     args = parse_args()
-    generator = torch.Generator(device=current_omni_platform.device_type).manual_seed(args.seed)
+    device = detect_device_type()
+    generator = torch.Generator(device=device).manual_seed(args.seed)
 
     # Load input image
     image = PIL.Image.open(args.image).convert("RGB")
@@ -131,54 +91,30 @@ def main():
     # Resize image to target dimensions
     image = image.resize((width, height), PIL.Image.Resampling.LANCZOS)
 
-    # Check if profiling is requested via environment variable
-    profiler_enabled = bool(os.getenv("VLLM_TORCH_PROFILER_DIR"))
-    parallel_config = DiffusionParallelConfig(
-        cfg_parallel_size=args.cfg_parallel_size,
-    )
+    # Enable VAE memory optimizations on NPU
+    vae_use_slicing = is_npu()
+    vae_use_tiling = is_npu()
+
     omni = Omni(
         model=args.model,
-        enable_layerwise_offload=args.enable_layerwise_offload,
-        layerwise_num_gpu_layers=args.layerwise_num_gpu_layers,
-        vae_use_slicing=args.vae_use_slicing,
-        vae_use_tiling=args.vae_use_tiling,
+        vae_use_slicing=vae_use_slicing,
+        vae_use_tiling=vae_use_tiling,
         boundary_ratio=args.boundary_ratio,
         flow_shift=args.flow_shift,
-        enable_cpu_offload=args.enable_cpu_offload,
-        parallel_config=parallel_config,
-        enforce_eager=args.enforce_eager,
     )
 
-    if profiler_enabled:
-        print("[Profiler] Starting profiling...")
-        omni.start_profile()
-
-    # Print generation configuration
-    print(f"\n{'=' * 60}")
-    print("Generation Configuration:")
-    print(f"  Model: {args.model}")
-    print(f"  Inference steps: {args.num_inference_steps}")
-    print(f"  Frames: {args.num_frames}")
-    print(f"  Parallel configuration: cfg_parallel_size={args.cfg_parallel_size}")
-    print(f"  Video size: {args.width}x{args.height}")
-    print(f"{'=' * 60}\n")
-
     # omni.generate() returns Generator[OmniRequestOutput, None, None]
     frames = omni.generate(
-        {
-            "prompt": args.prompt,
-            "negative_prompt": args.negative_prompt,
-            "multi_modal_data": {"image": image},
-        },
-        OmniDiffusionSamplingParams(
-            height=height,
-            width=width,
-            generator=generator,
-            guidance_scale=args.guidance_scale,
-            guidance_scale_2=args.guidance_scale_high,
-            num_inference_steps=args.num_inference_steps,
-            num_frames=args.num_frames,
-        ),
+        args.prompt,
+        negative_prompt=args.negative_prompt,
+        pil_image=image,
+        height=height,
+        width=width,
+        generator=generator,
+        guidance_scale=args.guidance_scale,
+        guidance_scale_2=args.guidance_scale_high,
+        num_inference_steps=args.num_inference_steps,
+        num_frames=args.num_frames,
     )
 
     # Extract video frames from OmniRequestOutput
@@ -242,23 +178,6 @@ def main():
     export_to_video(video_array, str(output_path), fps=args.fps)
     print(f"Saved generated video to {output_path}")
 
-    if profiler_enabled:
-        print("\n[Profiler] Stopping profiler and collecting results...")
-        profile_results = omni.stop_profile()
-        if profile_results and isinstance(profile_results, dict):
-            traces = profile_results.get("traces", [])
-            print("\n" + "=" * 60)
-            print("PROFILING RESULTS:")
-            for rank, trace in enumerate(traces):
-                print(f"\nRank {rank}:")
-                if trace:
-                    print(f"  • Trace: {trace}")
-            if not traces:
-                print("  No traces collected.")
-            print("=" * 60)
-        else:
-            print("[Profiler] No valid profiling data returned.")
-
 
 if __name__ == "__main__":
     main()

examples/offline_inference/lora_inference/README.md

@@ -1,98 +0,0 @@
-# LoRA Inference Examples
-
-This directory contains examples for using LoRA (Low-Rank Adaptation) adapters with vLLM-omni diffusion models for offline inference.
-The example uses the  `stabilityai/stable-diffusion-3.5-medium` as the default model, but you can replace it with other models in vLLM-omni.
-
-## Overview
-
-Similar to vLLM, vLLM-omni uses a unified LoRA handling mechanism:
-
-- **Pre-loaded LoRA**: Loaded at initialization via `--lora-path` (pre-loaded into cache)
-- **Per-request LoRA**: Loaded on-demand. In the example, the LoRA is loaded via `--lora-request-path` in each request
-
-Both approaches use the same underlying mechanism - all LoRA adapters are handled uniformly through `set_active_adapter()`. If no LoRA request is provided in a request, all adapters are deactivated.
-
-## Usage
-
-### Pre-loaded LoRA (via --lora-path)
-
-Load a LoRA adapter at initialization. This adapter is pre-loaded into the cache and can be activated by requests:
-
-```bash
-python -m examples.offline_inference.lora_inference.lora_inference \
-    --prompt "A piece of cheesecake" \
-    --lora-path /path/to/lora/ \
-    --lora-scale 1.0 \
-    --num_inference_steps 50 \
-    --height 1024 \
-    --width 1024 \
-    --output output_preloaded.png
-```
-
-**Note**: When using `--lora-path`, the adapter is loaded at init time with a stable ID derived from the adapter path. This example activates it automatically for the request.
-
-### Per-request LoRA (via --lora-request-path)
-
-Load a LoRA adapter on-demand for each request:
-
-```bash
-python -m examples.offline_inference.lora_inference.lora_inference \
-    --prompt "A piece of cheesecake" \
-    --lora-request-path /path/to/lora/ \
-    --lora-scale 1.0 \
-    --num_inference_steps 50 \
-    --height 1024 \
-    --width 1024 \
-    --output output_per_request.png
-```
-
-### No LoRA
-
-If no LoRA request is provided, we will use the base model without any LoRA adapters:
-
-```bash
-python -m examples.offline_inference.lora_inference.lora_inference \
-    --prompt "A piece of cheesecake" \
-    --num_inference_steps 50 \
-    --height 1024 \
-    --width 1024 \
-    --output output_no_lora.png
-```
-
-## Parameters
-
-### LoRA Parameters
-
-- `--lora-path`: Path to LoRA adapter folder to pre-load at initialization (loads into cache with a stable ID derived from the path)
-- `--lora-request-path`: Path to LoRA adapter folder for per-request loading
-- `--lora-request-id`: Integer ID for the LoRA adapter (optional). If not provided and `--lora-request-path` is set, will derive a stable ID from the path.
-- `--lora-scale`: Scale factor for LoRA weights (default: 1.0). Higher values increase the influence of the LoRA adapter.
-
-### Standard Parameters
-
-- `--prompt`: Text prompt for image generation (required)
-- `--seed`: Random seed for reproducibility (default: 42)
-- `--height`: Image height in pixels (default: 1024)
-- `--width`: Image width in pixels (default: 1024)
-- `--num_inference_steps`: Number of denoising steps (default: 50)
-- `--output`: Output file path (default: `lora_output.png`)
-
-## How LoRA Works
-
-All LoRA adapters are handled uniformly:
-
-1. **Initialization**: If `--lora-path` is provided, the adapter is loaded into cache with a stable ID derived from the adapter path
-2. **Per-request**: If `--lora-request-path` is provided, the adapter is loaded/activated for that request
-3. **No LoRA**: If no LoRA request is provided (`req.lora_request` is None), all adapters are deactivated
-
-The system uses LRU cache management - adapters are cached and evicted when the cache is full (unless pinned).
-
-## LoRA Adapter Format
-
-LoRA adapters must be in PEFT (Parameter-Efficient Fine-Tuning) format. A typical LoRA adapter directory structure:
-
-```
-lora_adapter/
-├── adapter_config.json
-└── adapter_model.safetensors
-```

examples/offline_inference/lora_inference/lora_inference.py

@@ -1,147 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-
-import argparse
-from pathlib import Path
-
-from vllm_omni.entrypoints.omni import Omni
-from vllm_omni.inputs.data import OmniDiffusionSamplingParams
-from vllm_omni.lora.request import LoRARequest
-from vllm_omni.lora.utils import stable_lora_int_id
-
-
-def parse_args() -> argparse.Namespace:
-    parser = argparse.ArgumentParser(description="Generate images with LoRA adapters.")
-    parser.add_argument("--model", default="stabilityai/stable-diffusion-3.5-medium", help="Model name or path.")
-    parser.add_argument("--prompt", required=True, help="Text prompt for image generation.")
-    parser.add_argument("--seed", type=int, default=42, help="Random seed for deterministic results.")
-    parser.add_argument("--height", type=int, default=1024, help="Height of generated image.")
-    parser.add_argument("--width", type=int, default=1024, help="Width of generated image.")
-    parser.add_argument(
-        "--num_inference_steps",
-        type=int,
-        default=50,
-        help="Number of denoising steps for the diffusion sampler.",
-    )
-    parser.add_argument(
-        "--output",
-        type=str,
-        default="lora_output.png",
-        help="Path to save the generated image (PNG).",
-    )
-    parser.add_argument(
-        "--lora-path",
-        type=str,
-        default=None,
-        help="Path to LoRA adapter folder to pre-load at initialization (PEFT format). "
-        "Note: pre-loading populates the cache; you still need to pass a lora_request to activate it.",
-    )
-    parser.add_argument(
-        "--lora-request-path",
-        type=str,
-        default=None,
-        help="Path to LoRA adapter folder for per-request activation (dynamic LoRA). "
-        "If --lora-request-id is not provided, a stable ID will be derived from this path.",
-    )
-    parser.add_argument(
-        "--lora-request-id",
-        type=int,
-        default=None,
-        help="Integer ID for the LoRA adapter (for dynamic LoRA). "
-        "If not provided and --lora-request-path is set, will derive a stable ID from the path.",
-    )
-    parser.add_argument(
-        "--lora-scale",
-        type=float,
-        default=1.0,
-        help="Scale factor for LoRA weights (default: 1.0).",
-    )
-    return parser.parse_args()
-
-
-def main():
-    args = parse_args()
-
-    model = args.model
-
-    omni_kwargs = {}
-
-    if args.lora_path:
-        omni_kwargs["lora_path"] = args.lora_path
-        print(f"Using static LoRA from: {args.lora_path}")
-
-    omni = Omni(model=model, **omni_kwargs)
-
-    lora_request = None
-    if args.lora_request_path:
-        if args.lora_request_id is None:
-            lora_request_id = stable_lora_int_id(args.lora_request_path)
-        else:
-            lora_request_id = args.lora_request_id
-
-        lora_name = Path(args.lora_request_path).stem
-        lora_request = LoRARequest(
-            lora_name=lora_name,
-            lora_int_id=lora_request_id,
-            lora_path=args.lora_request_path,
-        )
-        print(f"Using per-request LoRA: name={lora_name}, id={lora_request_id}, scale={args.lora_scale}")
-    elif args.lora_path:
-        # pre-loaded LoRA
-        lora_request_id = stable_lora_int_id(args.lora_path)
-        lora_request = LoRARequest(
-            lora_name="preloaded",
-            lora_int_id=lora_request_id,
-            lora_path=args.lora_path,
-        )
-        print(f"Activating pre-loaded LoRA: id={lora_request_id}, scale={args.lora_scale}")
-
-    sampling_params = OmniDiffusionSamplingParams(
-        height=args.height,
-        width=args.width,
-        num_inference_steps=args.num_inference_steps,
-    )
-
-    if lora_request:
-        sampling_params.lora_request = lora_request
-        sampling_params.lora_scale = args.lora_scale
-
-    outputs = omni.generate(args.prompt, sampling_params)
-
-    if not outputs or len(outputs) == 0:
-        raise ValueError("No output generated from omni.generate()")
-
-    if isinstance(outputs, list):
-        first_output = outputs[0]
-    else:
-        first_output = outputs
-
-    images = None
-    if hasattr(first_output, "images") and first_output.images:
-        images = first_output.images
-    elif hasattr(first_output, "request_output") and first_output.request_output:
-        req_out = first_output.request_output
-        if isinstance(req_out, list) and len(req_out) > 0:
-            req_out = req_out[0]
-        if hasattr(req_out, "images") and req_out.images:
-            images = req_out.images
-
-    if not images:
-        raise ValueError("No images found in request_output")
-
-    output_path = Path(args.output)
-    output_path.parent.mkdir(parents=True, exist_ok=True)
-    suffix = output_path.suffix or ".png"
-    stem = output_path.stem or "lora_output"
-    if len(images) <= 1:
-        images[0].save(output_path)
-        print(f"Saved generated image to {output_path}")
-    else:
-        for idx, img in enumerate(images):
-            save_path = output_path.parent / f"{stem}_{idx}{suffix}"
-            img.save(save_path)
-            print(f"Saved generated image to {save_path}")
-
-
-if __name__ == "__main__":
-    main()

examples/offline_inference/qwen2_5_omni/end2end.py

@@ -278,9 +278,9 @@ def get_audio_query(question: str = None, audio_path: str | None = None, samplin
 
 
 query_map = {
-    "use_mixed_modalities": get_mixed_modalities_query,
+    "mixed_modalities": get_mixed_modalities_query,
     "use_audio_in_video": get_use_audio_in_video_query,
-    "use_multi_audios": get_multi_audios_query,
+    "multi_audios": get_multi_audios_query,
     "use_image": get_image_query,
     "use_video": get_video_query,
     "use_audio": get_audio_query,
@@ -410,7 +410,7 @@ def main(args):
         elif stage_outputs.final_output_type == "audio":
             for output in stage_outputs.request_output:
                 request_id = output.request_id
-                audio_tensor = output.outputs[0].multimodal_output["audio"]
+                audio_tensor = output.multimodal_output["audio"]
                 output_wav = os.path.join(output_dir, f"output_{request_id}.wav")
                 sf.write(output_wav, audio_tensor.detach().cpu().numpy(), samplerate=24000)
                 print(f"Request ID: {request_id}, Saved audio to {output_wav}")
@@ -434,7 +434,7 @@ def parse_args():
         "--query-type",
         "-q",
         type=str,
-        default="use_mixed_modalities",
+        default="mixed_modalities",
         choices=query_map.keys(),
         help="Query type.",
     )

examples/offline_inference/qwen2_5_omni/run_single_prompt.sh

@@ -1,2 +1,2 @@
 python end2end.py --output-wav output_audio \
-                  --query-type use_mixed_modalities
+                  --query-type mixed_modalities

examples/offline_inference/qwen3_omni/end2end.py

@@ -12,7 +12,6 @@ from typing import NamedTuple
 import librosa
 import numpy as np
 import soundfile as sf
-import vllm
 from PIL import Image
 from vllm import SamplingParams
 from vllm.assets.audio import AudioAsset
@@ -226,76 +225,18 @@ def get_multi_audios_query() -> QueryResult:
     )
 
 
-# def get_use_audio_in_video_query(video_path: str | None = None) -> QueryResult:
-# question = (
-#     "Describe the content of the video in details, then convert what the "
-#     "baby say into text."
-# )
-# prompt = (
-#     f"<|im_start|>system\n{default_system}<|im_end|>\n"
-#     "<|im_start|>user\n<|vision_start|><|video_pad|><|vision_end|>"
-#     f"{question}<|im_end|>\n"
-#     f"<|im_start|>assistant\n"
-# )
-# if video_path:
-#     if not os.path.exists(video_path):
-#         raise FileNotFoundError(f"Video file not found: {video_path}")
-#     video_frames = video_to_ndarrays(video_path, num_frames=16)
-# else:
-#     video_frames = VideoAsset(name="baby_reading", num_frames=16).np_ndarrays
-# audio = extract_video_audio(video_path, sampling_rate=16000)
-# return QueryResult(
-#     inputs={
-#         "prompt": prompt,
-#         "multi_modal_data": {
-#             "video": video_frames,
-#             "audio": audio,
-#         },
-#         "mm_processor_kwargs": {
-#             "use_audio_in_video": True,
-#         },
-#     },
-#     limit_mm_per_prompt={"audio": 1, "video": 1},
-# )
-def get_use_audio_in_video_query() -> QueryResult:
-    question = "Describe the content of the video in details, then convert what the baby say into text."
-    prompt = (
-        f"<|im_start|>system\n{default_system}<|im_end|>\n"
-        "<|im_start|>user\n<|vision_start|><|video_pad|><|vision_end|>"
-        f"{question}<|im_end|>\n"
-        f"<|im_start|>assistant\n"
-    )
-    asset = VideoAsset(name="baby_reading", num_frames=16)
-    audio = asset.get_audio(sampling_rate=16000)
-    return QueryResult(
-        inputs={
-            "prompt": prompt,
-            "multi_modal_data": {
-                "video": asset.np_ndarrays,
-                "audio": audio,
-            },
-            "mm_processor_kwargs": {
-                "use_audio_in_video": True,
-            },
-        },
-        limit_mm_per_prompt={"audio": 1, "video": 1},
-    )
-
-
 query_map = {
     "text": get_text_query,
     "use_audio": get_audio_query,
     "use_image": get_image_query,
     "use_video": get_video_query,
-    "use_multi_audios": get_multi_audios_query,
-    "use_mixed_modalities": get_mixed_modalities_query,
-    "use_audio_in_video": get_use_audio_in_video_query,
+    "multi_audios": get_multi_audios_query,
+    "mixed_modalities": get_mixed_modalities_query,
 }
 
 
 def main(args):
     model_name = "Qwen/Qwen3-Omni-30B-A3B-Instruct"
-    print("=" * 20, "\n", f"vllm version: {vllm.__version__}", "\n", "=" * 20)
 
     # Get paths from args
     video_path = getattr(args, "video_path", None)
@@ -318,10 +259,6 @@ def main(args):
             num_frames=getattr(args, "num_frames", 16),
             sampling_rate=getattr(args, "sampling_rate", 16000),
         )
-    elif args.query_type == "multi_audios":
-        query_result = query_func()
-    elif args.query_type == "use_audio_in_video":
-        query_result = query_func()
     else:
         query_result = query_func()
 
@@ -333,7 +270,7 @@ def main(args):
     )
 
     thinker_sampling_params = SamplingParams(
-        temperature=0.9,
+        temperature=0.4,
         top_p=0.9,
         top_k=-1,
         max_tokens=1200,
@@ -394,8 +331,6 @@ def main(args):
     total_requests = len(prompts)
     processed_count = 0
 
-    print(f"query type: {args.query_type}")
-
     for stage_outputs in omni_generator:
         if stage_outputs.final_output_type == "text":
             for output in stage_outputs.request_output:
@@ -418,7 +353,7 @@ def main(args):
         elif stage_outputs.final_output_type == "audio":
             for output in stage_outputs.request_output:
                 request_id = output.request_id
-                audio_tensor = output.outputs[0].multimodal_output["audio"]
+                audio_tensor = output.multimodal_output["audio"]
                 output_wav = os.path.join(output_dir, f"output_{request_id}.wav")
 
                 # Convert to numpy array and ensure correct format
@@ -450,7 +385,7 @@ def parse_args():
         "--query-type",
         "-q",
         type=str,
-        default="use_mixed_modalities",
+        default="mixed_modalities",
         choices=query_map.keys(),
         help="Query type.",
     )

examples/offline_inference/qwen3_tts/README.md

@@ -1,93 +0,0 @@
-# Qwen3-TTS Offline Inference
-
-This directory contains an offline demo for running Qwen3 TTS models with vLLM Omni. It builds task-specific inputs and generates WAV files locally.
-
-## Model Overview
-
-Qwen3 TTS provides multiple task variants for speech generation:
-
-- **CustomVoice**: Generate speech with a known speaker identity (speaker ID) and optional instruction.
-- **VoiceDesign**: Generate speech from text plus a descriptive instruction that designs a new voice.
-- **Base**: Voice cloning using a reference audio + reference transcript, with optional mode selection.
-
-## Setup
-Please refer to the [stage configuration documentation](https://docs.vllm.ai/projects/vllm-omni/en/latest/configuration/stage_configs/) to configure memory allocation appropriately for your hardware setup.
-
-### ROCm Dependencies
-
-You will need to install these two dependencies `onnxruntime-rocm` and `sox`.
-
-```
-pip uninstall onnxruntime # should be removed before we can install onnxruntime-rocm
-pip install onnxruntime-rocm sox
-```
-
-## Quick Start
-
-Run a single sample for a task:
-
-```
-python end2end.py --query-type CustomVoice
-```
-
-Generated audio files are saved to `output_audio/` by default.
-
-## Task Usage
-
-### CustomVoice
-
-Single sample:
-
-```
-python end2end.py --query-type CustomVoice
-```
-
-Batch sample (multiple prompts in one run):
-
-```
-python end2end.py --query-type CustomVoice --use-batch-sample
-```
-
-### VoiceDesign
-
-Single sample:
-
-```
-python end2end.py --query-type VoiceDesign
-```
-
-Batch sample:
-
-```
-python end2end.py --query-type VoiceDesign --use-batch-sample
-```
-
-### Base (Voice Clone)
-
-Single sample:
-
-```
-python end2end.py --query-type Base
-```
-
-Batch sample:
-
-```
-python end2end.py --query-type Base --use-batch-sample
-```
-
-Mode selection for Base:
-
-- `--mode-tag icl` (default): standard mode
-- `--mode-tag xvec_only`: enable `x_vector_only_mode` in the request
-
-Examples:
-
-```
-python end2end.py --query-type Base --mode-tag icl
-```
-
-## Notes
-
-- The script uses the model paths embedded in `end2end.py`. Update them if your local cache path differs.
-- Use `--output-dir` (preferred) or `--output-wav` to change the output folder.

examples/offline_inference/qwen3_tts/end2end.py

@@ -1,381 +0,0 @@
-"""Offline inference demo for Qwen3 TTS via vLLM Omni.
-
-Provides single and batch sample inputs for CustomVoice, VoiceDesign, and Base
-tasks, then runs Omni generation and saves output wav files.
-"""
-
-import os
-from typing import NamedTuple
-
-import soundfile as sf
-
-os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
-
-from vllm import SamplingParams
-from vllm.utils.argparse_utils import FlexibleArgumentParser
-
-from vllm_omni import Omni
-
-
-class QueryResult(NamedTuple):
-    """Container for a prepared Omni request."""
-
-    inputs: dict
-    model_name: str
-
-
-def get_custom_voice_query(use_batch_sample: bool = False) -> QueryResult:
-    """Build CustomVoice sample inputs.
-
-    Args:
-        use_batch_sample: When True, return a batch of prompts; otherwise a single prompt.
-
-    Returns:
-        QueryResult with Omni inputs and the CustomVoice model path.
-    """
-    task_type = "CustomVoice"
-    if use_batch_sample:
-        texts = ["其实我真的有发现，我是一个特别善于观察别人情绪的人。", "She said she would be here by noon."]
-        instructs = ["", "Very happy."]
-        languages = ["Chinese", "English"]
-        speakers = ["Vivian", "Ryan"]
-        inputs = []
-        for text, instruct, language, speaker in zip(texts, instructs, languages, speakers):
-            prompt = f"<|im_start|>assistant\n{text}<|im_end|>\n<|im_start|>assistant\n"
-            inputs.append(
-                {
-                    "prompt": prompt,
-                    "additional_information": {
-                        "task_type": [task_type],
-                        "text": [text],
-                        "instruct": [instruct],
-                        "language": [language],
-                        "speaker": [speaker],
-                        "max_new_tokens": [2048],
-                    },
-                }
-            )
-    else:
-        text = "其实我真的有发现，我是一个特别善于观察别人情绪的人。"
-        language = "Chinese"
-        speaker = "Vivian"
-        instruct = "用特别愤怒的语气说"
-        prompts = f"<|im_start|>assistant\n{text}<|im_end|>\n<|im_start|>assistant\n"
-        inputs = {
-            "prompt": prompts,
-            "additional_information": {
-                "task_type": [task_type],
-                "text": [text],
-                "language": [language],
-                "speaker": [speaker],
-                "instruct": [instruct],
-                "max_new_tokens": [2048],
-            },
-        }
-    return QueryResult(
-        inputs=inputs,
-        model_name="Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice",
-    )
-
-
-def get_voice_design_query(use_batch_sample: bool = False) -> QueryResult:
-    """Build VoiceDesign sample inputs.
-
-    Args:
-        use_batch_sample: When True, return a batch of prompts; otherwise a single prompt.
-
-    Returns:
-        QueryResult with Omni inputs and the VoiceDesign model path.
-    """
-    task_type = "VoiceDesign"
-    if use_batch_sample:
-        texts = [
-            "哥哥，你回来啦，人家等了你好久好久了，要抱抱！",
-            "It's in the top drawer... wait, it's empty? No way, that's impossible! I'm sure I put it there!",
-        ]
-        instructs = [
-            "体现撒娇稚嫩的萝莉女声，音调偏高且起伏明显，营造出黏人、做作又刻意卖萌的听觉效果。",
-            "Speak in an incredulous tone, but with a hint of panic beginning to creep into your voice.",
-        ]
-        languages = ["Chinese", "English"]
-        inputs = []
-        for text, instruct, language in zip(texts, instructs, languages):
-            prompt = f"<|im_start|>assistant\n{text}<|im_end|>\n<|im_start|>assistant\n"
-            inputs.append(
-                {
-                    "prompt": prompt,
-                    "additional_information": {
-                        "task_type": [task_type],
-                        "text": [text],
-                        "language": [language],
-                        "instruct": [instruct],
-                        "max_new_tokens": [2048],
-                        "non_streaming_mode": [True],
-                    },
-                }
-            )
-    else:
-        text = "哥哥，你回来啦，人家等了你好久好久了，要抱抱！"
-        instruct = "体现撒娇稚嫩的萝莉女声，音调偏高且起伏明显，营造出黏人、做作又刻意卖萌的听觉效果。"
-        language = "Chinese"
-        prompt = f"<|im_start|>assistant\n{text}<|im_end|>\n<|im_start|>assistant\n"
-        inputs = {
-            "prompt": prompt,
-            "additional_information": {
-                "task_type": [task_type],
-                "text": [text],
-                "language": [language],
-                "instruct": [instruct],
-                "max_new_tokens": [2048],
-                "non_streaming_mode": [True],
-            },
-        }
-    return QueryResult(
-        inputs=inputs,
-        model_name="Qwen/Qwen3-TTS-12Hz-1.7B-VoiceDesign",
-    )
-
-
-def get_base_query(use_batch_sample: bool = False, mode_tag: str = "icl") -> QueryResult:
-    """Build Base (voice clone) sample inputs.
-
-    Args:
-        use_batch_sample: When True, return a batch of prompts (Case 2).
-        mode_tag: "icl" or "xvec_only" to control x_vector_only_mode behavior.
-
-    Returns:
-        QueryResult with Omni inputs and the Base model path.
-    """
-    task_type = "Base"
-    ref_audio_path_1 = "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen3-TTS-Repo/clone_2.wav"
-    ref_audio_single = ref_audio_path_1
-    ref_text_single = (
-        "Okay. Yeah. I resent you. I love you. I respect you. But you know what? You blew it! And thanks to you."
-    )
-    syn_text_single = "Good one. Okay, fine, I'm just gonna leave this sock monkey here. Goodbye."
-    syn_lang_single = "Auto"
-    x_vector_only_mode = mode_tag == "xvec_only"
-    if use_batch_sample:
-        syn_text_batch = [
-            "Good one. Okay, fine, I'm just gonna leave this sock monkey here. Goodbye.",
-            "其实我真的有发现，我是一个特别善于观察别人情绪的人。",
-        ]
-        syn_lang_batch = ["Chinese", "English"]
-        inputs = []
-        for text, language in zip(syn_text_batch, syn_lang_batch):
-            prompt = f"<|im_start|>assistant\n{text}<|im_end|>\n<|im_start|>assistant\n"
-            inputs.append(
-                {
-                    "prompt": prompt,
-                    "additional_information": {
-                        "task_type": [task_type],
-                        "ref_audio": [ref_audio_single],
-                        "ref_text": [ref_text_single],
-                        "text": [text],
-                        "language": [language],
-                        "x_vector_only_mode": [x_vector_only_mode],
-                        "max_new_tokens": [2048],
-                    },
-                }
-            )
-    else:
-        prompt = f"<|im_start|>assistant\n{syn_text_single}<|im_end|>\n<|im_start|>assistant\n"
-        inputs = {
-            "prompt": prompt,
-            "additional_information": {
-                "task_type": [task_type],
-                "ref_audio": [ref_audio_single],
-                "ref_text": [ref_text_single],
-                "text": [syn_text_single],
-                "language": [syn_lang_single],
-                "x_vector_only_mode": [x_vector_only_mode],
-                "max_new_tokens": [2048],
-            },
-        }
-    return QueryResult(
-        inputs=inputs,
-        model_name="Qwen/Qwen3-TTS-12Hz-1.7B-Base",
-    )
-
-
-def main(args):
-    """Run offline inference with Omni using prepared sample inputs.
-
-    Args:
-        args: Parsed CLI args from parse_args().
-    """
-    query_func = query_map[args.query_type]
-    if args.query_type in {"CustomVoice", "VoiceDesign"}:
-        query_result = query_func(use_batch_sample=args.use_batch_sample)
-    elif args.query_type == "Base":
-        query_result = query_func(
-            use_batch_sample=args.use_batch_sample,
-            mode_tag=args.mode_tag,
-        )
-    else:
-        query_result = query_func()
-
-    model_name = query_result.model_name
-    omni = Omni(
-        model=model_name,
-        stage_configs_path=args.stage_configs_path,
-        log_stats=args.enable_stats,
-        stage_init_timeout=args.stage_init_timeout,
-    )
-
-    sampling_params = SamplingParams(
-        temperature=0.9,
-        top_p=1.0,
-        top_k=50,
-        max_tokens=2048,
-        seed=42,
-        detokenize=False,
-        repetition_penalty=1.05,
-    )
-
-    sampling_params_list = [
-        sampling_params,
-    ]
-
-    output_dir = args.output_dir if getattr(args, "output_dir", None) else args.output_wav
-    os.makedirs(output_dir, exist_ok=True)
-
-    omni_generator = omni.generate(query_result.inputs, sampling_params_list)
-    for stage_outputs in omni_generator:
-        for output in stage_outputs.request_output:
-            request_id = output.request_id
-            audio_tensor = output.outputs[0].multimodal_output["audio"]
-            output_wav = os.path.join(output_dir, f"output_{request_id}.wav")
-            audio_samplerate = output.outputs[0].multimodal_output["sr"].item()
-            # Convert to numpy array and ensure correct format
-            audio_numpy = audio_tensor.float().detach().cpu().numpy()
-
-            # Ensure audio is 1D (flatten if needed)
-            if audio_numpy.ndim > 1:
-                audio_numpy = audio_numpy.flatten()
-
-            # Save audio file with explicit WAV format
-            sf.write(output_wav, audio_numpy, samplerate=audio_samplerate, format="WAV")
-            print(f"Request ID: {request_id}, Saved audio to {output_wav}")
-
-
-def parse_args():
-    """Parse CLI arguments for offline TTS inference.
-
-    Returns:
-        argparse.Namespace with CLI options.
-    """
-    parser = FlexibleArgumentParser(description="Demo on using vLLM for offline inference with audio language models")
-    parser.add_argument(
-        "--query-type",
-        "-q",
-        type=str,
-        default="CustomVoice",
-        choices=query_map.keys(),
-        help="Query type.",
-    )
-    parser.add_argument(
-        "--enable-stats",
-        action="store_true",
-        default=False,
-        help="Enable writing detailed statistics (default: disabled)",
-    )
-    parser.add_argument(
-        "--stage-init-timeout",
-        type=int,
-        default=300,
-        help="Timeout for initializing a single stage in seconds (default: 300)",
-    )
-    parser.add_argument(
-        "--batch-timeout",
-        type=int,
-        default=5,
-        help="Timeout for batching in seconds (default: 5)",
-    )
-    parser.add_argument(
-        "--init-timeout",
-        type=int,
-        default=300,
-        help="Timeout for initializing stages in seconds (default: 300)",
-    )
-    parser.add_argument(
-        "--shm-threshold-bytes",
-        type=int,
-        default=65536,
-        help="Threshold for using shared memory in bytes (default: 65536)",
-    )
-    parser.add_argument(
-        "--output-wav",
-        default="output_audio",
-        help="[Deprecated] Output wav directory (use --output-dir).",
-    )
-    parser.add_argument(
-        "--num-prompts",
-        type=int,
-        default=1,
-        help="Number of prompts to generate.",
-    )
-    parser.add_argument(
-        "--txt-prompts",
-        type=str,
-        default=None,
-        help="Path to a .txt file with one prompt per line (preferred).",
-    )
-    parser.add_argument(
-        "--stage-configs-path",
-        type=str,
-        default=None,
-        help="Path to a stage configs file.",
-    )
-    parser.add_argument(
-        "--audio-path",
-        "-a",
-        type=str,
-        default=None,
-        help="Path to local audio file. If not provided, uses default audio asset.",
-    )
-    parser.add_argument(
-        "--sampling-rate",
-        type=int,
-        default=16000,
-        help="Sampling rate for audio loading (default: 16000).",
-    )
-    parser.add_argument(
-        "--log-dir",
-        type=str,
-        default="logs",
-        help="Log directory (default: logs).",
-    )
-    parser.add_argument(
-        "--py-generator",
-        action="store_true",
-        default=False,
-        help="Use py_generator mode. The returned type of Omni.generate() is a Python Generator object.",
-    )
-    parser.add_argument(
-        "--use-batch-sample",
-        action="store_true",
-        default=False,
-        help="Use batch input sample for CustomVoice/VoiceDesign/Base query.",
-    )
-    parser.add_argument(
-        "--mode-tag",
-        type=str,
-        default="icl",
-        choices=["icl", "xvec_only"],
-        help="Mode tag for Base query x_vector_only_mode (default: icl).",
-    )
-
-    return parser.parse_args()
-
-
-query_map = {
-    "CustomVoice": get_custom_voice_query,
-    "VoiceDesign": get_voice_design_query,
-    "Base": get_base_query,
-}
-
-
-if __name__ == "__main__":
-    args = parse_args()
-    main(args)

examples/offline_inference/text_to_audio/README.md

@@ -1,37 +0,0 @@
-# Text-To-Audio
-
-The `stabilityai/stable-audio-open-1.0` pipeline generates audio from text prompts.
-
-## Prerequisites
-
-If you use a gated model (e.g., `stabilityai/stable-audio-open-1.0`), ensure you have access:
-
-1. **Accept Model License**: Visit the model page on Hugging Face (e.g., [stabilityai/stable-audio-open-1.0]) and accept the user agreement.
-2. **Authenticate**: Log in to Hugging Face locally to access the gated model.
-   ```bash
-   huggingface-cli login
-   ```
-
-## Local CLI Usage
-
-```bash
-python text_to_audio.py \
-  --model stabilityai/stable-audio-open-1.0 \
-  --prompt "The sound of a hammer hitting a wooden surface" \
-  --negative_prompt "Low quality" \
-  --seed 42 \
-  --guidance_scale 7.0 \
-  --audio_length 10.0 \
-  --num_inference_steps 100 \
-  --output stable_audio_output.wav
-```
-
-Key arguments:
-
-- `--prompt`: text description (string).
-- `--negative_prompt`: negative prompt for classifier-free guidance.
-- `--seed`: integer seed for deterministic generation.
-- `--guidance_scale`: classifier-free guidance scale.
-- `--audio_length`: audio duration in seconds.
-- `--num_inference_steps`: diffusion sampling steps.(more steps = higher quality, slower).
-- `--output`: path to save the generated WAV file.

examples/offline_inference/text_to_audio/text_to_audio.py

@@ -1,219 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-
-"""
-Example script for text-to-audio generation using Stable Audio Open.
-
-This script demonstrates how to generate audio from text prompts using
-the Stable Audio Open model with vLLM-Omni.
-
-Usage:
-    python text_to_audio.py --prompt "The sound of a dog barking"
-    python text_to_audio.py --prompt "A piano playing a gentle melody" --audio_length 10.0
-    python text_to_audio.py --prompt "Thunder and rain sounds" --negative_prompt "Low quality"
-"""
-
-import argparse
-import time
-from pathlib import Path
-
-import numpy as np
-import torch
-
-from vllm_omni.entrypoints.omni import Omni
-from vllm_omni.inputs.data import OmniDiffusionSamplingParams
-from vllm_omni.platforms import current_omni_platform
-
-
-def parse_args() -> argparse.Namespace:
-    parser = argparse.ArgumentParser(description="Generate audio with Stable Audio Open.")
-    parser.add_argument(
-        "--model",
-        default="stabilityai/stable-audio-open-1.0",
-        help="Stable Audio model name or local path.",
-    )
-    parser.add_argument(
-        "--prompt",
-        default="The sound of a hammer hitting a wooden surface.",
-        help="Text prompt for audio generation.",
-    )
-    parser.add_argument(
-        "--negative_prompt",
-        default="Low quality.",
-        help="Negative prompt for classifier-free guidance.",
-    )
-    parser.add_argument(
-        "--seed",
-        type=int,
-        default=42,
-        help="Random seed for deterministic results.",
-    )
-    parser.add_argument(
-        "--guidance_scale",
-        type=float,
-        default=7.0,
-        help="Classifier-free guidance scale.",
-    )
-    parser.add_argument(
-        "--audio_start",
-        type=float,
-        default=0.0,
-        help="Audio start time in seconds.",
-    )
-    parser.add_argument(
-        "--audio_length",
-        type=float,
-        default=10.0,
-        help="Audio length in seconds (max ~47s for stable-audio-open-1.0).",
-    )
-    parser.add_argument(
-        "--num_inference_steps",
-        type=int,
-        default=100,
-        help="Number of denoising steps for the diffusion sampler.",
-    )
-    parser.add_argument(
-        "--num_waveforms",
-        type=int,
-        default=1,
-        help="Number of audio waveforms to generate for the given prompt.",
-    )
-    parser.add_argument(
-        "--output",
-        type=str,
-        default="stable_audio_output.wav",
-        help="Path to save the generated audio (WAV format).",
-    )
-    parser.add_argument(
-        "--sample_rate",
-        type=int,
-        default=44100,
-        help="Sample rate for output audio (Stable Audio uses 44100 Hz).",
-    )
-    return parser.parse_args()
-
-
-def save_audio(audio_data: np.ndarray, output_path: str, sample_rate: int = 44100):
-    """Save audio data to a WAV file."""
-    try:
-        import soundfile as sf
-
-        sf.write(output_path, audio_data, sample_rate)
-    except ImportError:
-        try:
-            import scipy.io.wavfile as wav
-
-            # Ensure audio is in the correct format for scipy
-            if audio_data.dtype == np.float32 or audio_data.dtype == np.float64:
-                # Normalize to int16 range
-                audio_data = np.clip(audio_data, -1.0, 1.0)
-                audio_data = (audio_data * 32767).astype(np.int16)
-            wav.write(output_path, sample_rate, audio_data)
-        except ImportError:
-            raise ImportError(
-                "Either 'soundfile' or 'scipy' is required to save audio files. "
-                "Install with: pip install soundfile or pip install scipy"
-            )
-
-
-def main():
-    args = parse_args()
-    generator = torch.Generator(device=current_omni_platform.device_type).manual_seed(args.seed)
-
-    print(f"\n{'=' * 60}")
-    print("Stable Audio Open - Text-to-Audio Generation")
-    print(f"{'=' * 60}")
-    print(f"  Model: {args.model}")
-    print(f"  Prompt: {args.prompt}")
-    print(f"  Negative prompt: {args.negative_prompt}")
-    print(f"  Audio length: {args.audio_length}s")
-    print(f"  Inference steps: {args.num_inference_steps}")
-    print(f"  Guidance scale: {args.guidance_scale}")
-    print(f"  Seed: {args.seed}")
-    print(f"{'=' * 60}\n")
-
-    # Initialize Omni with Stable Audio model
-    omni = Omni(model=args.model)
-
-    # Calculate audio end time
-    audio_end_in_s = args.audio_start + args.audio_length
-
-    # Time profiling for generation
-    generation_start = time.perf_counter()
-
-    # Generate audio
-    outputs = omni.generate(
-        {
-            "prompt": args.prompt,
-            "negative_prompt": args.negative_prompt,
-        },
-        OmniDiffusionSamplingParams(
-            generator=generator,
-            guidance_scale=args.guidance_scale,
-            num_inference_steps=args.num_inference_steps,
-            num_outputs_per_prompt=args.num_waveforms,
-            extra_args={
-                "audio_start_in_s": args.audio_start,
-                "audio_end_in_s": audio_end_in_s,
-            },
-        ),
-    )
-
-    generation_end = time.perf_counter()
-    generation_time = generation_end - generation_start
-
-    print(f"Total generation time: {generation_time:.2f} seconds")
-
-    # Process and save audio
-    output_path = Path(args.output)
-    output_path.parent.mkdir(parents=True, exist_ok=True)
-    suffix = output_path.suffix or ".wav"
-    stem = output_path.stem or "stable_audio_output"
-
-    # Extract audio from omni.generate() outputs
-    if not outputs:
-        raise ValueError("No output generated from omni.generate()")
-
-    output = outputs[0]
-    if not hasattr(output, "request_output") or not output.request_output:
-        raise ValueError("No request_output found in OmniRequestOutput")
-    request_output = output.request_output[0]
-    if not hasattr(request_output, "multimodal_output"):
-        raise ValueError("No multimodal_output found in request_output")
-
-    audio = request_output.multimodal_output.get("audio")
-    if audio is None:
-        raise ValueError("No audio output found in request_output")
-
-    # Handle different output formats
-    if isinstance(audio, torch.Tensor):
-        audio = audio.cpu().float().numpy()
-
-    # Audio shape is typically [batch, channels, samples] or [channels, samples]
-    if audio.ndim == 3:
-        # [batch, channels, samples]
-        if args.num_waveforms <= 1:
-            audio_data = audio[0].T  # [samples, channels]
-            save_audio(audio_data, str(output_path), args.sample_rate)
-            print(f"Saved generated audio to {output_path}")
-        else:
-            for idx in range(audio.shape[0]):
-                audio_data = audio[idx].T  # [samples, channels]
-                save_path = output_path.parent / f"{stem}_{idx}{suffix}"
-                save_audio(audio_data, str(save_path), args.sample_rate)
-                print(f"Saved generated audio to {save_path}")
-    elif audio.ndim == 2:
-        # [channels, samples]
-        audio_data = audio.T  # [samples, channels]
-        save_audio(audio_data, str(output_path), args.sample_rate)
-        print(f"Saved generated audio to {output_path}")
-    else:
-        # [samples] - mono audio
-        save_audio(audio, str(output_path), args.sample_rate)
-        print(f"Saved generated audio to {output_path}")
-
-    print(f"\nGenerated {args.audio_length}s of audio at {args.sample_rate} Hz")
-
-
-if __name__ == "__main__":
-    main()

examples/offline_inference/text_to_image/README.md

@@ -20,7 +20,7 @@ if __name__ == "__main__":
     images[0].save("coffee.png")
 ```
 
-Or put more than one prompt in a request.
+Or put more than one prompt in a request, processing them sequentially.
 
 ```python
 from vllm_omni.entrypoints.omni import Omni
@@ -37,41 +37,6 @@ if __name__ == "__main__":
       image = output.request_output[0].images[0].save(f"{i}.jpg")
 ```
 
-!!! info
-
-    However, it is not currently recommended to do so
-    because not all models support batch inference,
-    and batch requesting mostly does not provide significant performance improvement (despite the impression that it does).
-    This feature is primarily for the sake of interface compatibility with vLLM and to allow for future improvements.
-
-!!! info
-
-    For diffusion pipelines, the stage config field `stage_args.[].runtime.max_batch_size` is 1 by default, and the input
-    list is sliced into single-item requests before feeding into the diffusion pipeline. For models that do internally support
-    batched inputs, you can [modify this configuration](../../../configuration/stage_configs.md) to let the model accept a longer batch of prompts.
-
-Apart from string prompt, vLLM-Omni also supports dictionary prompts in the same style as vLLM.
-This is useful for models that support negative prompts.
-
-```python
-from vllm_omni.entrypoints.omni import Omni
-
-if __name__ == "__main__":
-    omni = Omni(model="Qwen/Qwen-Image")
-    outputs = omni.generate([
-      {
-        "prompt": "a cup of coffee on a table"，
-        "negative_prompt": "low resolution"
-      },
-      {
-        "prompt": "a toy dinosaur on a sandy beach"，
-        "negative_prompt": "cinematic, realistic"
-      }
-    ])
-    for i, output in enumerate(outputs):
-      image = output.request_output[0].images[0].save(f"{i}.jpg")
-```
-
 ## Local CLI Usage
 
 ```bash
@@ -96,12 +61,6 @@ Key arguments:
 - `--num_inference_steps`: diffusion sampling steps (more steps = higher quality, slower).
 - `--height/--width`: output resolution (defaults 1024x1024).
 - `--output`: path to save the generated PNG.
-- `--vae_use_slicing`: enable VAE slicing for memory optimization.
-- `--vae_use_tiling`: enable VAE tiling for memory optimization.
-- `--cfg_parallel_size`: set it to 2 to enable CFG Parallel. See more examples in [`user_guide`](../../../docs/user_guide/diffusion/parallelism_acceleration.md#cfg-parallel).
-- `--enable-cpu-offload`: enable CPU offloading for diffusion models.
-
-> ℹ️ If you encounter OOM errors, try using `--vae_use_slicing` and `--vae_use_tiling` to reduce memory usage.
 
 > ℹ️ Qwen-Image currently publishes best-effort presets at `1328x1328`, `1664x928`, `928x1664`, `1472x1140`, `1140x1472`, `1584x1056`, and `1056x1584`. Adjust `--height/--width` accordingly for the most reliable outcomes.
 

examples/offline_inference/text_to_image/gradio_demo.py

@@ -5,9 +5,8 @@ import gradio as gr
 import torch
 
 from vllm_omni.entrypoints.omni import Omni
-from vllm_omni.inputs.data import OmniDiffusionSamplingParams
 from vllm_omni.outputs import OmniRequestOutput
-from vllm_omni.platforms import current_omni_platform
+from vllm_omni.utils.platform_utils import detect_device_type, is_npu
 
 ASPECT_RATIOS: dict[str, tuple[int, int]] = {
     "1:1": (1328, 1328),
@@ -62,8 +61,8 @@ def parse_args() -> argparse.Namespace:
 @lru_cache(maxsize=1)
 def get_omni(model_name: str) -> Omni:
     # Enable VAE memory optimizations on NPU
-    vae_use_slicing = current_omni_platform.is_npu()
-    vae_use_tiling = current_omni_platform.is_npu()
+    vae_use_slicing = is_npu()
+    vae_use_tiling = is_npu()
     return Omni(
         model=model_name,
         vae_use_slicing=vae_use_slicing,
@@ -72,6 +71,7 @@ def get_omni(model_name: str) -> Omni:
 
 
 def build_demo(args: argparse.Namespace) -> gr.Blocks:
+    device = detect_device_type()
     omni = get_omni(args.model)
 
     def run_inference(
@@ -98,17 +98,15 @@ def build_demo(args: argparse.Namespace) -> gr.Blocks:
             raise gr.Error("Inference steps must be a positive integer.")
         if num_images not in {1, 2, 3, 4}:
             raise gr.Error("Number of images must be 1, 2, 3, or 4.")
-        generator = torch.Generator(device=current_omni_platform.device_type).manual_seed(seed)
+        generator = torch.Generator(device=device).manual_seed(seed)
         outputs = omni.generate(
             prompt.strip(),
-            OmniDiffusionSamplingParams(
-                height=height,
-                width=width,
-                generator=generator,
-                true_cfg_scale=float(cfg_scale_value),
-                num_inference_steps=num_steps,
-                num_outputs_per_prompt=num_images,
-            ),
+            height=height,
+            width=width,
+            generator=generator,
+            true_cfg_scale=float(cfg_scale_value),
+            num_inference_steps=num_steps,
+            num_outputs_per_prompt=num_images,
         )
         images_outputs = []
         for output in outputs:

examples/offline_inference/text_to_image/text_to_image.py

@@ -2,7 +2,6 @@
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
 
 import argparse
-import os
 import time
 from pathlib import Path
 
@@ -10,9 +9,8 @@ import torch
 
 from vllm_omni.diffusion.data import DiffusionParallelConfig, logger
 from vllm_omni.entrypoints.omni import Omni
-from vllm_omni.inputs.data import OmniDiffusionSamplingParams
 from vllm_omni.outputs import OmniRequestOutput
-from vllm_omni.platforms import current_omni_platform
+from vllm_omni.utils.platform_utils import detect_device_type, is_npu
 
 
 def parse_args() -> argparse.Namespace:
@@ -25,9 +23,7 @@ def parse_args() -> argparse.Namespace:
     )
     parser.add_argument("--prompt", default="a cup of coffee on the table", help="Text prompt for image generation.")
     parser.add_argument(
-        "--negative_prompt",
-        default=None,
-        help="negative prompt for classifier-free conditional guidance.",
+        "--negative_prompt", default="", help="negative prompt for classifier-free conditional guidance."
     )
     parser.add_argument("--seed", type=int, default=142, help="Random seed for deterministic results.")
     parser.add_argument(
@@ -73,11 +69,6 @@ def parse_args() -> argparse.Namespace:
             "Default: None (no cache acceleration)."
         ),
     )
-    parser.add_argument(
-        "--enable-cache-dit-summary",
-        action="store_true",
-        help="Enable cache-dit summary logging after diffusion forward passes.",
-    )
     parser.add_argument(
         "--ulysses_degree",
         type=int,
@@ -90,56 +81,17 @@ def parse_args() -> argparse.Namespace:
         default=1,
         help="Number of GPUs used for ring sequence parallelism.",
     )
-    parser.add_argument(
-        "--cfg_parallel_size",
-        type=int,
-        default=1,
-        choices=[1, 2],
-        help="Number of GPUs used for classifier free guidance parallel size.",
-    )
-    parser.add_argument(
-        "--enforce_eager",
-        action="store_true",
-        help="Disable torch.compile and force eager execution.",
-    )
-    parser.add_argument(
-        "--enable-cpu-offload",
-        action="store_true",
-        help="Enable CPU offloading for diffusion models.",
-    )
-    parser.add_argument(
-        "--enable-layerwise-offload",
-        action="store_true",
-        help="Enable layerwise (blockwise) offloading on DiT modules.",
-    )
-    parser.add_argument(
-        "--layerwise-num-gpu-layers",
-        type=int,
-        default=1,
-        help="Number of ready layers (blocks) to keep on GPU during generation.",
-    )
-    parser.add_argument(
-        "--tensor_parallel_size",
-        type=int,
-        default=1,
-        help="Number of GPUs used for tensor parallelism (TP) inside the DiT.",
-    )
-    parser.add_argument(
-        "--vae_use_slicing",
-        action="store_true",
-        help="Enable VAE slicing for memory optimization.",
-    )
-    parser.add_argument(
-        "--vae_use_tiling",
-        action="store_true",
-        help="Enable VAE tiling for memory optimization.",
-    )
     return parser.parse_args()
 
 
 def main():
     args = parse_args()
-    generator = torch.Generator(device=current_omni_platform.device_type).manual_seed(args.seed)
+    device = detect_device_type()
+    generator = torch.Generator(device=device).manual_seed(args.seed)
+
+    # Enable VAE memory optimizations on NPU
+    vae_use_slicing = is_npu()
+    vae_use_tiling = is_npu()
 
     # Configure cache based on backend type
     cache_config = None
@@ -171,62 +123,37 @@ def main():
         }
 
     # assert args.ring_degree == 1, "Ring attention is not supported yet"
-    parallel_config = DiffusionParallelConfig(
-        ulysses_degree=args.ulysses_degree,
-        ring_degree=args.ring_degree,
-        cfg_parallel_size=args.cfg_parallel_size,
-        tensor_parallel_size=args.tensor_parallel_size,
-    )
-
-    # Check if profiling is requested via environment variable
-    profiler_enabled = bool(os.getenv("VLLM_TORCH_PROFILER_DIR"))
-
+    parallel_config = DiffusionParallelConfig(ulysses_degree=args.ulysses_degree, ring_degree=args.ring_degree)
     omni = Omni(
         model=args.model,
-        enable_layerwise_offload=args.enable_layerwise_offload,
-        layerwise_num_gpu_layers=args.layerwise_num_gpu_layers,
-        vae_use_slicing=args.vae_use_slicing,
-        vae_use_tiling=args.vae_use_tiling,
+        vae_use_slicing=vae_use_slicing,
+        vae_use_tiling=vae_use_tiling,
         cache_backend=args.cache_backend,
         cache_config=cache_config,
-        enable_cache_dit_summary=args.enable_cache_dit_summary,
         parallel_config=parallel_config,
-        enforce_eager=args.enforce_eager,
-        enable_cpu_offload=args.enable_cpu_offload,
     )
 
-    if profiler_enabled:
-        print("[Profiler] Starting profiling...")
-        omni.start_profile()
-
     # Time profiling for generation
     print(f"\n{'=' * 60}")
     print("Generation Configuration:")
     print(f"  Model: {args.model}")
     print(f"  Inference steps: {args.num_inference_steps}")
     print(f"  Cache backend: {args.cache_backend if args.cache_backend else 'None (no acceleration)'}")
-    print(
-        f"  Parallel configuration: tensor_parallel_size={args.tensor_parallel_size}, "
-        f"ulysses_degree={args.ulysses_degree}, ring_degree={args.ring_degree}, cfg_parallel_size={args.cfg_parallel_size}"
-    )
+    print(f"  Parallel configuration: ulysses_degree={args.ulysses_degree}, ring_degree={args.ring_degree}")
     print(f"  Image size: {args.width}x{args.height}")
     print(f"{'=' * 60}\n")
 
     generation_start = time.perf_counter()
     outputs = omni.generate(
-        {
-            "prompt": args.prompt,
-            "negative_prompt": args.negative_prompt,
-        },
-        OmniDiffusionSamplingParams(
-            height=args.height,
-            width=args.width,
-            generator=generator,
-            true_cfg_scale=args.cfg_scale,
-            guidance_scale=args.guidance_scale,
-            num_inference_steps=args.num_inference_steps,
-            num_outputs_per_prompt=args.num_images_per_prompt,
-        ),
+        args.prompt,
+        negative_prompt=args.negative_prompt,
+        height=args.height,
+        width=args.width,
+        generator=generator,
+        true_cfg_scale=args.cfg_scale,
+        guidance_scale=args.guidance_scale,
+        num_inference_steps=args.num_inference_steps,
+        num_outputs_per_prompt=args.num_images_per_prompt,
     )
     generation_end = time.perf_counter()
     generation_time = generation_end - generation_start
@@ -234,23 +161,6 @@ def main():
     # Print profiling results
     print(f"Total generation time: {generation_time:.4f} seconds ({generation_time * 1000:.2f} ms)")
 
-    if profiler_enabled:
-        print("\n[Profiler] Stopping profiler and collecting results...")
-        profile_results = omni.stop_profile()
-        if profile_results and isinstance(profile_results, dict):
-            traces = profile_results.get("traces", [])
-            print("\n" + "=" * 60)
-            print("PROFILING RESULTS:")
-            for rank, trace in enumerate(traces):
-                print(f"\nRank {rank}:")
-                if trace:
-                    print(f"  • Trace: {trace}")
-            if not traces:
-                print("  No traces collected.")
-            print("=" * 60)
-        else:
-            print("[Profiler] No valid profiling data returned.")
-
     # Extract images from OmniRequestOutput
     # omni.generate() returns list[OmniRequestOutput], extract images from the first output
     if not outputs or len(outputs) == 0:

examples/offline_inference/text_to_video/text_to_video.md

@@ -9,11 +9,10 @@ python text_to_video.py \
   --prompt "Two anthropomorphic cats in comfy boxing gear and bright gloves fight intensely on a spotlighted stage." \
   --negative_prompt "<optional quality filter>" \
   --height 480 \
-  --width 832 \
-  --num_frames 33 \
+  --width 640 \
+  --num_frames 32 \
   --guidance_scale 4.0 \
   --guidance_scale_high 3.0 \
-  --flow_shift 12.0 \
   --num_inference_steps 40 \
   --fps 16 \
   --output t2v_out.mp4
@@ -22,16 +21,10 @@ python text_to_video.py \
 Key arguments:
 
 - `--prompt`: text description (string).
-- `--height/--width`: output resolution (defaults 480x832, i.e. 480P). Dimensions should align with Wan VAE downsampling (multiples of 8).
+- `--height/--width`: output resolution (defaults 720x1280). Dimensions should align with Wan VAE downsampling (multiples of 8).
 - `--num_frames`: Number of frames (Wan default is 81).
-- `--guidance_scale` and `--guidance_scale_high`: CFG scale (applied to low/high).
+- `--guidance_scale` and `--guidance_scale_high`: CFG scale (applied to low/high)..
 - `--negative_prompt`: optional list of artifacts to suppress (the PR demo used a long Chinese string).
-- `--boundary_ratio`: Boundary split ratio for low/high DiT. Default `0.875` uses both transformers for best quality. Set to `1.0` to load only the low-noise transformer (saves noticeable memory with good quality, recommended if memory is limited). Set to `0.0` loads only the high-noise transformer (not recommended, lower quality).
+- `--boundary_ratio`: Boundary split ratio for low/high DiT.
 - `--fps`: frames per second for the saved MP4 (requires `diffusers` export_to_video).
 - `--output`: path to save the generated video.
-- `--vae_use_slicing`: enable VAE slicing for memory optimization.
-- `--vae_use_tiling`: enable VAE tiling for memory optimization.
-- `--cfg_parallel_size`: set it to 2 to enable CFG Parallel. See more examples in [`user_guide`](../../../docs/user_guide/diffusion/parallelism_acceleration.md#cfg-parallel).
-- `--enable-cpu-offload`: enable CPU offloading for diffusion models.
-
-> ℹ️ If you encounter OOM errors, try using `--vae_use_slicing` and `--vae_use_tiling` to reduce memory usage.

examples/offline_inference/text_to_video/text_to_video.py

@@ -2,18 +2,14 @@
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
 
 import argparse
-import os
-import time
 from pathlib import Path
 
 import numpy as np
 import torch
 
-from vllm_omni.diffusion.data import DiffusionParallelConfig
 from vllm_omni.entrypoints.omni import Omni
-from vllm_omni.inputs.data import OmniDiffusionSamplingParams
 from vllm_omni.outputs import OmniRequestOutput
-from vllm_omni.platforms import current_omni_platform
+from vllm_omni.utils.platform_utils import detect_device_type, is_npu
 
 
 def parse_args() -> argparse.Namespace:
@@ -32,173 +28,43 @@ def parse_args() -> argparse.Namespace:
     parser.add_argument("--width", type=int, default=1280, help="Video width.")
     parser.add_argument("--num_frames", type=int, default=81, help="Number of frames (Wan default is 81).")
     parser.add_argument("--num_inference_steps", type=int, default=40, help="Sampling steps.")
-    parser.add_argument(
-        "--boundary_ratio",
-        type=float,
-        default=0.875,
-        help="Boundary split ratio for low/high DiT. Default 0.875 uses both transformers for best quality. Set to 1.0 to load only the low-noise transformer (saves noticeable memory with good quality, recommended if memory is limited).",
-    )
+    parser.add_argument("--boundary_ratio", type=float, default=0.875, help="Boundary split ratio for low/high DiT.")
     parser.add_argument(
         "--flow_shift", type=float, default=5.0, help="Scheduler flow_shift (5.0 for 720p, 12.0 for 480p)."
     )
-    parser.add_argument(
-        "--cache_backend",
-        type=str,
-        default=None,
-        choices=["cache_dit"],
-        help=(
-            "Cache backend to use for acceleration. "
-            "Options: 'cache_dit' (DBCache + SCM + TaylorSeer). "
-            "Default: None (no cache acceleration)."
-        ),
-    )
-    parser.add_argument(
-        "--enable-cache-dit-summary",
-        action="store_true",
-        help="Enable cache-dit summary logging after diffusion forward passes.",
-    )
     parser.add_argument("--output", type=str, default="wan22_output.mp4", help="Path to save the video (mp4).")
     parser.add_argument("--fps", type=int, default=24, help="Frames per second for the output video.")
-    parser.add_argument(
-        "--vae_use_slicing",
-        action="store_true",
-        help="Enable VAE slicing for memory optimization.",
-    )
-    parser.add_argument(
-        "--vae_use_tiling",
-        action="store_true",
-        help="Enable VAE tiling for memory optimization.",
-    )
-    parser.add_argument(
-        "--enforce_eager",
-        action="store_true",
-        help="Disable torch.compile and force eager execution.",
-    )
-    parser.add_argument(
-        "--enable-cpu-offload",
-        action="store_true",
-        help="Enable CPU offloading for diffusion models.",
-    )
-    parser.add_argument(
-        "--enable-layerwise-offload",
-        action="store_true",
-        help="Enable layerwise (blockwise) offloading on DiT modules.",
-    )
-    parser.add_argument(
-        "--layerwise-num-gpu-layers",
-        type=int,
-        default=1,
-        help="Number of ready layers (blocks) to keep on GPU during generation.",
-    )
-    parser.add_argument(
-        "--ulysses_degree",
-        type=int,
-        default=1,
-        help="Number of GPUs used for ulysses sequence parallelism.",
-    )
-    parser.add_argument(
-        "--ring_degree",
-        type=int,
-        default=1,
-        help="Number of GPUs used for ring sequence parallelism.",
-    )
-    parser.add_argument(
-        "--cfg_parallel_size",
-        type=int,
-        default=1,
-        choices=[1, 2],
-        help="Number of GPUs used for classifier free guidance parallel size.",
-    )
-
     return parser.parse_args()
 
 
 def main():
     args = parse_args()
-    generator = torch.Generator(device=current_omni_platform.device_type).manual_seed(args.seed)
+    device = detect_device_type()
+    generator = torch.Generator(device=device).manual_seed(args.seed)
 
-    # Wan2.2 cache-dit tuning (from cache-dit examples and cache_alignment).
-    cache_config = None
-    if args.cache_backend == "cache_dit":
-        cache_config = {
-            # DBCache parameters [cache-dit only]
-            "Fn_compute_blocks": 1,  # Optimized for single-transformer models
-            "Bn_compute_blocks": 0,  # Number of backward compute blocks
-            "max_warmup_steps": 4,  # Maximum warmup steps (works for few-step models)
-            "max_cached_steps": 20,
-            "residual_diff_threshold": 0.24,  # Higher threshold for more aggressive caching
-            "max_continuous_cached_steps": 3,  # Limit to prevent precision degradation
-            # TaylorSeer parameters [cache-dit only]
-            "enable_taylorseer": False,  # Disabled by default (not suitable for few-step models)
-            "taylorseer_order": 1,  # TaylorSeer polynomial order
-            # SCM (Step Computation Masking) parameters [cache-dit only]
-            "scm_steps_mask_policy": None,  # SCM mask policy: None (disabled), "slow", "medium", "fast", "ultra"
-            "scm_steps_policy": "dynamic",  # SCM steps policy: "dynamic" or "static"
-        }
-    # Configure parallel settings (only SP is supported for Wan)
-    # Note: cfg_parallel and tensor_parallel are not implemented for Wan models
-    parallel_config = DiffusionParallelConfig(
-        ulysses_degree=args.ulysses_degree,
-        ring_degree=args.ring_degree,
-        cfg_parallel_size=args.cfg_parallel_size,
-    )
-
-    # Check if profiling is requested via environment variable
-    profiler_enabled = bool(os.getenv("VLLM_TORCH_PROFILER_DIR"))
+    # Enable VAE memory optimizations on NPU
+    vae_use_slicing = is_npu()
+    vae_use_tiling = is_npu()
 
     omni = Omni(
         model=args.model,
-        enable_layerwise_offload=args.enable_layerwise_offload,
-        layerwise_num_gpu_layers=args.layerwise_num_gpu_layers,
-        vae_use_slicing=args.vae_use_slicing,
-        vae_use_tiling=args.vae_use_tiling,
+        vae_use_slicing=vae_use_slicing,
+        vae_use_tiling=vae_use_tiling,
         boundary_ratio=args.boundary_ratio,
         flow_shift=args.flow_shift,
-        cache_backend=args.cache_backend,
-        cache_config=cache_config,
-        enable_cache_dit_summary=args.enable_cache_dit_summary,
-        enable_cpu_offload=args.enable_cpu_offload,
-        parallel_config=parallel_config,
-        enforce_eager=args.enforce_eager,
     )
 
-    if profiler_enabled:
-        print("[Profiler] Starting profiling...")
-        omni.start_profile()
-
-    # Print generation configuration
-    print(f"\n{'=' * 60}")
-    print("Generation Configuration:")
-    print(f"  Model: {args.model}")
-    print(f"  Inference steps: {args.num_inference_steps}")
-    print(f"  Frames: {args.num_frames}")
-    print(
-        f"  Parallel configuration: ulysses_degree={args.ulysses_degree}, ring_degree={args.ring_degree}, cfg_parallel_size={args.cfg_parallel_size}"
-    )
-    print(f"  Video size: {args.width}x{args.height}")
-    print(f"{'=' * 60}\n")
-
-    generation_start = time.perf_counter()
     frames = omni.generate(
-        {
-            "prompt": args.prompt,
-            "negative_prompt": args.negative_prompt,
-        },
-        OmniDiffusionSamplingParams(
-            height=args.height,
-            width=args.width,
-            generator=generator,
-            guidance_scale=args.guidance_scale,
-            guidance_scale_2=args.guidance_scale_high,
-            num_inference_steps=args.num_inference_steps,
-            num_frames=args.num_frames,
-        ),
+        args.prompt,
+        negative_prompt=args.negative_prompt,
+        height=args.height,
+        width=args.width,
+        generator=generator,
+        guidance_scale=args.guidance_scale,
+        guidance_scale_2=args.guidance_scale_high,
+        num_inference_steps=args.num_inference_steps,
+        num_frames=args.num_frames,
     )
-    generation_end = time.perf_counter()
-    generation_time = generation_end - generation_start
-
-    # Print profiling results
-    print(f"Total generation time: {generation_time:.4f} seconds ({generation_time * 1000:.2f} ms)")
 
     # Extract video frames from OmniRequestOutput
     if isinstance(frames, list) and len(frames) > 0:
@@ -260,23 +126,6 @@ def main():
     export_to_video(video_array, str(output_path), fps=args.fps)
     print(f"Saved generated video to {output_path}")
 
-    if profiler_enabled:
-        print("\n[Profiler] Stopping profiler and collecting results...")
-        profile_results = omni.stop_profile()
-        if profile_results and isinstance(profile_results, dict):
-            traces = profile_results.get("traces", [])
-            print("\n" + "=" * 60)
-            print("PROFILING RESULTS:")
-            for rank, trace in enumerate(traces):
-                print(f"\nRank {rank}:")
-                if trace:
-                    print(f"  • Trace: {trace}")
-            if not traces:
-                print("  No traces collected.")
-            print("=" * 60)
-        else:
-            print("[Profiler] No valid profiling data returned.")
-
 
 if __name__ == "__main__":
     main()

examples/online_serving/bagel/README.md

@@ -1,230 +0,0 @@
-# BAGEL-7B-MoT
-
-## 🛠️ Installation
-
-Please refer to [README.md](../../../README.md)
-
-## Run examples (BAGEL-7B-MoT)
-
-**Note**: These examples work with the default configuration on an **NVIDIA A100 (80GB)**. We also tested on dual **NVIDIA RTX 5000 Ada (32GB each)**. For dual-GPU setups, please modify the stage configuration to distribute the model across devices.
-
-### Launch the Server
-
-```bash
-# Use default configuration
-vllm serve ByteDance-Seed/BAGEL-7B-MoT --omni --port 8091
-```
-
-Or use the convenience script:
-
-```bash
-cd /workspace/vllm-omni/examples/online_serving/bagel
-bash run_server.sh
-```
-
-If you have a custom stage configs file, launch the server with the command below:
-
-```bash
-vllm serve ByteDance-Seed/BAGEL-7B-MoT --omni --port 8091 --stage-configs-path /path/to/stage_configs_file
-```
-
-### Send Multi-modal Request
-
-Get into the bagel folder:
-
-```bash
-cd examples/online_serving/bagel
-```
-
-Send request via Python
-
-```bash
-python openai_chat_client.py --prompt "A cute cat" --modality text2img
-```
-
-The Python client supports the following command-line arguments:
-
-- `--prompt` (or `-p`): Text prompt for generation (default: `A cute cat`)
-- `--output` (or `-o`): Output file path for image results (default: `bagel_output.png`)
-- `--server` (or `-s`): Server URL (default: `http://localhost:8091`)
-- `--image-url` (or `-i`): Input image URL or local file path (for img2img/img2text modes)
-- `--modality` (or `-m`): Task modality (default: `text2img`). Options: `text2img`, `img2img`, `img2text`, `text2text`
-- `--height`: Image height in pixels (default: 512)
-- `--width`: Image width in pixels (default: 512)
-- `--steps`: Number of inference steps (default: 25)
-- `--seed`: Random seed (default: 42)
-- `--negative`: Negative prompt for image generation
-
-Example with custom parameters:
-
-```bash
-python openai_chat_client.py \
-    --prompt "A futuristic city" \
-    --modality text2img \
-    --height 768 \
-    --width 768 \
-    --steps 50 \
-    --seed 42 \
-    --negative "blurry, low quality"
-```
-
-## Modality Control
-
-BAGEL-7B-MoT supports **multiple modality modes** for different use cases.
-
-The default yaml configuration deploys Thinker and DiT on the same GPU. You can use the default configuration file: [`bagel.yaml`](../../../vllm_omni/model_executor/stage_configs/bagel.yaml)
-
-| Modality    | Input        | Output | Description                            |
-| ----------- | ------------ | ------ | -------------------------------------- |
-| `text2img`  | Text         | Image  | Generate images from text prompts      |
-| `img2img`   | Image + Text | Image  | Transform images using text guidance   |
-| `img2text`  | Image + Text | Text   | Generate text descriptions from images |
-| `text2text` | Text         | Text   | Pure text generation                   |
-
-### Text to Image (text2img)
-
-Generate images from text prompts:
-
-**Using Python client**
-
-```bash
-python openai_chat_client.py \
-    --prompt "A beautiful sunset over mountains" \
-    --modality text2img \
-    --output sunset.png \
-    --steps 50
-```
-
-**Using curl**
-
-```bash
-curl http://localhost:8091/v1/chat/completions \
-  -H "Content-Type: application/json" \
-  -d '{
-    "messages": [{"role": "user", "content": [{"type": "text", "text": "<|im_start|>A beautiful sunset over mountains<|im_end|>"}]}],
-    "modalities": ["image"],
-    "height": 512,
-    "width": 512,
-    "num_inference_steps": 50,
-    "seed": 42
-  }'
-```
-
-
-### Image to Image (img2img)
-
-Transform images based on text prompts:
-
-**Using Python client**
-
-```bash
-python openai_chat_client.py \
-    --prompt "Make the cat stand up" \
-    --modality img2img \
-    --image-url /path/to/input.jpg \
-    --output transformed.png
-```
-
-**Using curl**
-
-```bash
-IMAGE_BASE64=$(base64 -w 0 cat.jpg)
-
-cat <<EOF > payload.json
-{
-    "messages": [{
-      "role": "user",
-      "content": [
-        {"type": "text", "text": "<|im_start|>Make the cat stand up<|im_end|>"},
-        {"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,${IMAGE_BASE64}"}}
-      ]
-    }],
-    "modalities": ["image"],
-    "height": 512,
-    "width": 512,
-    "num_inference_steps": 50,
-    "seed": 42
-}
-EOF
-
-curl http://localhost:8091/v1/chat/completions \
-  -H "Content-Type: application/json" \
-  -d @payload.json
-
-```
-
-### Image to Text (img2text)
-
-Generate text descriptions from images:
-
-**Using Python client**
-
-```bash
-python openai_chat_client.py \
-    --prompt "Describe this image in detail" \
-    --modality img2text \
-    --image-url /path/to/image.jpg
-```
-
-**Using curl**
-
-```bash
-IMAGE_BASE64=$(base64 -w 0 cat.jpg)
-
-cat <<EOF > payload.json
-{
-  "messages": [{
-    "role": "user",
-    "content": [
-      {"type": "text", "text": "<|im_start|>user\n<|image_pad|>\nDescribe this image in detail<|im_end|>\n<|im_start|>assistant\n"},
-      {"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,${IMAGE_BASE64}"}}
-    ]
-  }],
-  "modalities": ["text"]
-}
-EOF
-
-curl http://localhost:8091/v1/chat/completions \
-  -H "Content-Type: application/json" \
-  -d @payload.json
-```
-
-### Text to Text (text2text)
-
-Pure text generation:
-
-**Using Python client**
-
-```bash
-python openai_chat_client.py \
-    --prompt "What is the capital of France?" \
-    --modality text2text
-```
-
-**Using curl**
-
-```bash
-curl http://localhost:8091/v1/chat/completions \
-  -H "Content-Type: application/json" \
-  -d '{
-    "messages": [{"role": "user", "content": [{"type": "text", "text": "<|im_start|>user\nWhat is the capital of France?<|im_end|>\n<|im_start|>assistant\n"}]}]
-    "modalities": ["text"]
-  }'
-```
-
-## FAQ
-
-- If you encounter an error about the backend of librosa, try to install ffmpeg with the command below.
-
-```bash
-sudo apt update
-sudo apt install ffmpeg
-```
-
-- If you don’t know how much VRAM is needed for the model or encounter the OOM error, you can try to decrease the max_model_len.
-
-| Stage               | VRAM                         |
-| :------------------ | :--------------------------- |
-| Stage-0 (Thinker)   | **15.04 GiB** **+ KV Cache** |
-| Stage-1 (DiT)       | **26.50 GiB**                |
-| Total               | **~42 GiB + KV Cache**       |

examples/online_serving/bagel/openai_chat_client.py

@@ -1,184 +0,0 @@
-#!/usr/bin/env python3
-"""
-Bagel OpenAI-compatible chat client for image generation and multimodal tasks.
-
-Usage:
-    python openai_chat_client.py --prompt "A cute cat" --output output.png
-    python openai_chat_client.py --prompt "Describe this image" --image-url https://example.com/image.png
-"""
-
-import argparse
-import base64
-from pathlib import Path
-
-import requests
-
-
-def generate_image(
-    prompt: str,
-    server_url: str = "http://localhost:8091",
-    image_url: str | None = None,
-    height: int | None = None,
-    width: int | None = None,
-    steps: int | None = None,
-    seed: int | None = None,
-    negative_prompt: str | None = None,
-    modality: str = "text2img",  # "text2img" (default), "img2img", "img2text", "text2text"
-) -> bytes | str | None:
-    """Generate an image or text using the chat completions API.
-
-    Args:
-        prompt: Text description or prompt
-        server_url: Server URL
-        image_url: URL or path to input image (for img2img/img2text)
-        height: Image height in pixels
-        width: Image width in pixels
-        steps: Number of inference steps
-        seed: Random seed
-        negative_prompt: Negative prompt
-        modality: Task modality hint
-
-    Returns:
-        Image bytes (for image outputs) or Text string (for text outputs) or None if failed
-    """
-
-    # Construct Message Content
-    content = [{"type": "text", "text": f"<|im_start|>{prompt}<|im_end|>"}]
-
-    if image_url:
-        # Check if local file
-        if Path(image_url).exists():
-            with open(image_url, "rb") as f:
-                b64_data = base64.b64encode(f.read()).decode("utf-8")
-                final_image_url = f"data:image/jpeg;base64,{b64_data}"
-        else:
-            final_image_url = image_url
-
-        content.append({"type": "image_url", "image_url": {"url": final_image_url}})
-
-    messages = [{"role": "user", "content": content}]
-
-    # Build request payload with all parameters at top level
-    # Note: vLLM ignores "extra_body", so we put parameters directly in the payload
-    payload = {"messages": messages}
-
-    # Set output modalities at top level
-    if modality == "text2img" or modality == "img2img":
-        payload["modalities"] = ["image"]
-    elif modality == "img2text" or modality == "text2text":
-        payload["modalities"] = ["text"]
-
-    # Add generation parameters directly to payload
-    if height is not None:
-        payload["height"] = height
-    if width is not None:
-        payload["width"] = width
-    if steps is not None:
-        payload["num_inference_steps"] = steps
-    if seed is not None:
-        payload["seed"] = seed
-    if negative_prompt:
-        payload["negative_prompt"] = negative_prompt
-
-    # Send request
-    try:
-        print(f"Sending request to {server_url} with modality {modality}...")
-        response = requests.post(
-            f"{server_url}/v1/chat/completions",
-            headers={"Content-Type": "application/json"},
-            json=payload,
-            timeout=300,
-        )
-        response.raise_for_status()
-        data = response.json()
-
-        # Extract content - check ALL choices since server may return multiple
-        # (e.g., text in choices[0], image in choices[1])
-        choices = data.get("choices", [])
-
-        # First pass: look for image output in any choice
-        for choice in choices:
-            choice_content = choice.get("message", {}).get("content")
-
-            # Handle Image Output
-            if isinstance(choice_content, list) and len(choice_content) > 0:
-                first_item = choice_content[0]
-                if isinstance(first_item, dict) and "image_url" in first_item:
-                    img_url_str = first_item["image_url"].get("url", "")
-                    if img_url_str.startswith("data:image"):
-                        _, b64_data = img_url_str.split(",", 1)
-                        return base64.b64decode(b64_data)
-
-        # Second pass: look for text output if no image found
-        for choice in choices:
-            choice_content = choice.get("message", {}).get("content")
-            if isinstance(choice_content, str) and choice_content:
-                return choice_content
-
-        print(f"Unexpected response format: {choices}")
-        return None
-
-    except Exception as e:
-        print(f"Error: {e}")
-        return None
-
-
-def main():
-    parser = argparse.ArgumentParser(description="Bagel multimodal chat client")
-    parser.add_argument("--prompt", "-p", default="<|im_start|>A cute cat<|im_end|>", help="Text prompt")
-    parser.add_argument("--output", "-o", default="bagel_output.png", help="Output file (for image results)")
-    parser.add_argument("--server", "-s", default="http://localhost:8091", help="Server URL")
-
-    # Modality Control
-    parser.add_argument("--image-url", "-i", type=str, help="Input image URL or local path")
-    parser.add_argument(
-        "--modality",
-        "-m",
-        default="text2img",
-        choices=["text2img", "img2img", "img2text", "text2text"],
-        help="Task modality",
-    )
-
-    # Generation Params
-    parser.add_argument("--height", type=int, default=512, help="Image height")
-    parser.add_argument("--width", type=int, default=512, help="Image width")
-    parser.add_argument("--steps", type=int, default=25, help="Inference steps")
-    parser.add_argument("--seed", type=int, default=42, help="Random seed")
-    parser.add_argument("--negative", help="Negative prompt")
-
-    args = parser.parse_args()
-
-    print(f"Mode: {args.modality}")
-    if args.image_url:
-        print(f"Input Image: {args.image_url}")
-
-    result = generate_image(
-        prompt=args.prompt,
-        server_url=args.server,
-        image_url=args.image_url,
-        height=args.height,
-        width=args.width,
-        steps=args.steps,
-        seed=args.seed,
-        negative_prompt=args.negative,
-        modality=args.modality,
-    )
-
-    if result:
-        if isinstance(result, bytes):
-            # It's an image
-            output_path = Path(args.output)
-            output_path.write_bytes(result)
-            print(f"Image saved to: {output_path}")
-            print(f"Size: {len(result) / 1024:.1f} KB")
-        elif isinstance(result, str):
-            # It's text
-            print("Response:")
-            print(result)
-    else:
-        print("Failed to generate response")
-        exit(1)
-
-
-if __name__ == "__main__":
-    main()

examples/online_serving/bagel/run_server.sh

@@ -1,12 +0,0 @@
-#!/bin/bash
-# Bagel online serving startup script
-
-MODEL="${MODEL:-ByteDance-Seed/BAGEL-7B-MoT}"
-PORT="${PORT:-8091}"
-
-echo "Starting Bagel server..."
-echo "Model: $MODEL"
-echo "Port: $PORT"
-
-vllm serve "$MODEL" --omni \
-    --port "$PORT"

examples/online_serving/image_to_image/openai_chat_client.py

@@ -127,7 +127,7 @@ def main():
     parser.add_argument("--width", type=int, default=1024, help="Output image width")
     parser.add_argument("--steps", type=int, default=50, help="Inference steps")
     parser.add_argument("--guidance", type=float, default=7.5, help="Guidance scale")
-    parser.add_argument("--seed", type=int, default=0, help="Random seed")
+    parser.add_argument("--seed", type=int, help="Random seed")
     parser.add_argument("--negative", help="Negative prompt")
 
     args = parser.parse_args()

examples/online_serving/lora_inference/README.md

@@ -1,54 +0,0 @@
-# Online LoRA Inference (Diffusion)
-
-This example shows how to use **per-request LoRA** with vLLM-Omni diffusion models via the OpenAI-compatible Chat Completions API.
-
-> Note: The LoRA adapter path must be readable on the **server** machine (usually a local path or a mounted directory).
-> Note: This example uses `/v1/chat/completions`. LoRA payloads for other OpenAI endpoints are not implemented here.
-
-## Start Server
-
-```bash
-# Pick a diffusion model (examples)
-# export MODEL=stabilityai/stable-diffusion-3.5-medium
-# export MODEL=Qwen/Qwen-Image
-
-bash run_server.sh
-```
-
-## Call API (curl)
-
-```bash
-# Required: local LoRA folder on the server
-export LORA_PATH=/path/to/lora_adapter
-
-# Optional
-export SERVER=http://localhost:8091
-export PROMPT="A piece of cheesecake"
-export LORA_NAME=my_lora
-export LORA_SCALE=1.0
-# Optional: if omitted, the server derives a stable id from LORA_PATH.
-# export LORA_INT_ID=123
-
-bash run_curl_lora_inference.sh
-```
-
-## Call API (Python)
-
-```bash
-python openai_chat_client.py \
-  --prompt "A piece of cheesecake" \
-  --lora-path /path/to/lora_adapter \
-  --lora-name my_lora \
-  --lora-scale 1.0 \
-  --output output.png
-```
-
-## LoRA Format
-
-LoRA adapters should be in PEFT format, for example:
-
-```
-lora_adapter/
-├── adapter_config.json
-└── adapter_model.safetensors
-```

examples/online_serving/lora_inference/openai_chat_client.py

@@ -1,124 +0,0 @@
-#!/usr/bin/env python3
-"""
-OpenAI-compatible chat client for diffusion LoRA inference.
-
-Example:
-  python openai_chat_client.py \
-    --server http://localhost:8091 \
-    --prompt "A piece of cheesecake" \
-    --lora-path /path/to/lora_adapter \
-    --lora-name my_lora \
-    --lora-scale 1.0 \
-    --output output.png
-"""
-
-import argparse
-import base64
-from pathlib import Path
-
-import requests
-
-
-def generate_image(
-    *,
-    prompt: str,
-    server_url: str,
-    height: int | None,
-    width: int | None,
-    num_inference_steps: int | None,
-    seed: int | None,
-    lora_name: str | None,
-    lora_path: str | None,
-    lora_scale: float | None,
-    lora_int_id: int | None,
-) -> bytes | None:
-    messages = [{"role": "user", "content": prompt}]
-
-    extra_body: dict = {}
-    if height is not None:
-        extra_body["height"] = height
-    if width is not None:
-        extra_body["width"] = width
-    if num_inference_steps is not None:
-        extra_body["num_inference_steps"] = num_inference_steps
-    if seed is not None:
-        extra_body["seed"] = seed
-
-    if lora_path:
-        lora_body: dict = {
-            "local_path": lora_path,
-            "name": lora_name or Path(lora_path).stem,
-        }
-        if lora_scale is not None:
-            lora_body["scale"] = float(lora_scale)
-        if lora_int_id is not None:
-            lora_body["int_id"] = int(lora_int_id)
-        extra_body["lora"] = lora_body
-
-    payload = {"messages": messages}
-    if extra_body:
-        payload["extra_body"] = extra_body
-
-    response = requests.post(
-        f"{server_url}/v1/chat/completions",
-        headers={"Content-Type": "application/json"},
-        json=payload,
-        timeout=300,
-    )
-    response.raise_for_status()
-    data = response.json()
-
-    content = data["choices"][0]["message"]["content"]
-    if isinstance(content, list) and content:
-        image_url = content[0].get("image_url", {}).get("url", "")
-        if image_url.startswith("data:image"):
-            _, b64_data = image_url.split(",", 1)
-            return base64.b64decode(b64_data)
-
-    raise RuntimeError(f"Unexpected response format: {content!r}")
-
-
-def main() -> None:
-    parser = argparse.ArgumentParser(description="Diffusion LoRA OpenAI chat client")
-    parser.add_argument("--server", default="http://localhost:8091", help="Server URL")
-    parser.add_argument("--prompt", default="A piece of cheesecake", help="Text prompt")
-    parser.add_argument("--output", default="lora_online_output.png", help="Output image path")
-
-    parser.add_argument("--height", type=int, default=1024, help="Image height")
-    parser.add_argument("--width", type=int, default=1024, help="Image width")
-    parser.add_argument("--steps", type=int, default=50, help="num_inference_steps")
-    parser.add_argument("--seed", type=int, default=42, help="Random seed")
-
-    parser.add_argument("--lora-path", default=None, help="Server-local LoRA adapter folder (PEFT format)")
-    parser.add_argument("--lora-name", default=None, help="LoRA name (optional)")
-    parser.add_argument("--lora-scale", type=float, default=1.0, help="LoRA scale")
-    parser.add_argument(
-        "--lora-int-id",
-        type=int,
-        default=None,
-        help="LoRA integer id (cache key). If omitted, the server derives a stable id from lora_path.",
-    )
-
-    args = parser.parse_args()
-
-    image_bytes = generate_image(
-        prompt=args.prompt,
-        server_url=args.server,
-        height=args.height,
-        width=args.width,
-        num_inference_steps=args.steps,
-        seed=args.seed,
-        lora_name=args.lora_name,
-        lora_path=args.lora_path,
-        lora_scale=args.lora_scale if args.lora_path else None,
-        lora_int_id=args.lora_int_id if args.lora_path else None,
-    )
-
-    out_path = Path(args.output)
-    out_path.parent.mkdir(parents=True, exist_ok=True)
-    out_path.write_bytes(image_bytes)
-    print(f"Saved: {out_path} ({len(image_bytes) / 1024:.1f} KiB)")
-
-
-if __name__ == "__main__":
-    main()

examples/online_serving/lora_inference/run_curl_lora_inference.sh

@@ -1,61 +0,0 @@
-#!/bin/bash
-# Online diffusion LoRA inference via OpenAI-compatible chat API.
-
-SERVER="${SERVER:-http://localhost:8091}"
-PROMPT="${PROMPT:-A piece of cheesecake}"
-
-LORA_PATH="${LORA_PATH:-}"
-LORA_NAME="${LORA_NAME:-lora}"
-LORA_SCALE="${LORA_SCALE:-1.0}"
-LORA_INT_ID="${LORA_INT_ID:-}"
-
-HEIGHT="${HEIGHT:-1024}"
-WIDTH="${WIDTH:-1024}"
-NUM_INFERENCE_STEPS="${NUM_INFERENCE_STEPS:-50}"
-SEED="${SEED:-42}"
-
-CURRENT_TIME=$(date +%Y%m%d%H%M%S)
-OUTPUT="${OUTPUT:-lora_online_output_${CURRENT_TIME}.png}"
-
-if [ -z "$LORA_PATH" ]; then
-  echo "ERROR: LORA_PATH is required (must be a server-local path)."
-  exit 1
-fi
-
-echo "Generating image with LoRA..."
-echo "Server: $SERVER"
-echo "Prompt: $PROMPT"
-echo "LoRA: name=$LORA_NAME id=${LORA_INT_ID:-auto} scale=$LORA_SCALE path=$LORA_PATH"
-echo "Output: $OUTPUT"
-
-LORA_INT_ID_FIELD=""
-if [ -n "$LORA_INT_ID" ]; then
-  LORA_INT_ID_FIELD=", \"int_id\": $LORA_INT_ID"
-fi
-
-curl -s "$SERVER/v1/chat/completions" \
-  -H "Content-Type: application/json" \
-  -d "{
-    \"messages\": [
-      {\"role\": \"user\", \"content\": \"$PROMPT\"}
-    ],
-    \"extra_body\": {
-      \"height\": $HEIGHT,
-      \"width\": $WIDTH,
-      \"num_inference_steps\": $NUM_INFERENCE_STEPS,
-      \"seed\": $SEED,
-      \"lora\": {
-        \"name\": \"$LORA_NAME\",
-        \"local_path\": \"$LORA_PATH\",
-        \"scale\": $LORA_SCALE$LORA_INT_ID_FIELD
-      }
-    }
-  }" | jq -r '.choices[0].message.content[0].image_url.url' | sed 's/^data:image[^,]*,\s*//' | base64 -d > "$OUTPUT"
-
-if [ -f "$OUTPUT" ]; then
-  echo "Image saved to: $OUTPUT"
-  echo "Size: $(du -h "$OUTPUT" | cut -f1)"
-else
-  echo "Failed to generate image"
-  exit 1
-fi

examples/online_serving/lora_inference/run_server.sh

@@ -1,20 +0,0 @@
-#!/bin/bash
-# Online diffusion serving with vLLM-Omni (OpenAI-compatible API).
-
-MODEL="${MODEL:-stabilityai/stable-diffusion-3.5-medium}"
-PORT="${PORT:-8091}"
-
-echo "Starting vLLM-Omni diffusion server..."
-echo "Model: $MODEL"
-echo "Port: $PORT"
-
-if [ -z "${VLLM_BIN:-}" ]; then
-  if command -v vllm-omni >/dev/null 2>&1; then
-    VLLM_BIN="vllm-omni"
-  else
-    VLLM_BIN="vllm"
-  fi
-fi
-
-"$VLLM_BIN" serve "$MODEL" --omni \
-  --port "$PORT"

examples/online_serving/qwen2_5_omni/README.md

@@ -134,7 +134,6 @@ response = client.chat.completions.create(
 # Response contains two choices: one with text, one with audio
 print(response.choices[0].message.content)  # Text response
 print(response.choices[1].message.audio)    # Audio response
-```
 
 ## Streaming Output
 If you want to enable streaming output, please set the argument as below. The final output will be obtained just after generated by corresponding stage. Now we only support text streaming output. Other modalities can output normally.

examples/online_serving/qwen3_omni/README.md

@@ -12,12 +12,6 @@ Please refer to [README.md](../../../README.md)
 vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091
 ```
 
-If you want to open async chunking for qwen3-omni, launch the server with command below
-
-```bash
-vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091 --stage-configs-path /vllm_omni/model_executor/stage_configs/qwen3_omni_moe_async_chunk.yaml
-```
-
 If you have custom stage configs file, launch the server with command below
 ```bash
 vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091 --stage-configs-path /path/to/stage_configs_file
@@ -148,7 +142,6 @@ response = client.chat.completions.create(
 # Response contains two choices: one with text, one with audio
 print(response.choices[0].message.content)  # Text response
 print(response.choices[1].message.audio)    # Audio response
-```
 
 ## Streaming Output
 If you want to enable streaming output, please set the argument as below. The final output will be obtained just after generated by corresponding stage. Now we only support text streaming output. Other modalities can output normally.

examples/online_serving/qwen3_omni/openai_chat_completion_client_for_multimodal_generation.py

@@ -1,5 +1,4 @@
 import base64
-import concurrent.futures
 import os
 from typing import NamedTuple
 
@@ -306,26 +305,6 @@ def get_multi_audios_query(custom_prompt: str | None = None):
     }
 
 
-def get_use_audio_in_video_query(
-    video_path: str | None = None,
-    audio_path: str | None = None,
-    custom_prompt: str | None = None,
-):
-    question = custom_prompt or (
-        "Describe the content of the video in details, then convert what the baby say into text."
-    )
-    video_url = get_video_url_from_path(video_path)
-    audio_url = get_audio_url_from_path(audio_path)
-    return {
-        "role": "user",
-        "content": [
-            {"type": "video_url", "video_url": {"url": video_url}},
-            {"type": "audio_url", "audio_url": {"url": audio_url}},
-            {"type": "text", "text": question},
-        ],
-    }
-
-
 query_map = {
     "text": get_text_query,
     "use_audio": get_audio_query,
@@ -333,7 +312,6 @@ query_map = {
     "use_video": get_video_query,
     "use_mixed_modalities": get_mixed_modalities_query,
     "use_multi_audios": get_multi_audios_query,
-    "use_audio_in_video": get_use_audio_in_video_query,
 }
 
 
@@ -394,12 +372,6 @@ def run_multimodal_generation(args) -> None:
         prompt = query_func(audio_path=audio_path, custom_prompt=custom_prompt)
     elif args.query_type == "text":
         prompt = query_func(custom_prompt=custom_prompt)
-    elif args.query_type == "use_audio_in_video":
-        prompt = query_func(
-            video_path=video_path,
-            audio_path=audio_path,
-            custom_prompt=custom_prompt,
-        )
     else:
         prompt = query_func()
 
@@ -415,67 +387,51 @@ def run_multimodal_generation(args) -> None:
     else:
         output_modalities = None
 
-    # Test multiple concurrent completions
-    num_concurrent_requests = args.num_concurrent_requests
+    chat_completion = client.chat.completions.create(
+        messages=[
+            get_system_prompt(),
+            prompt,
+        ],
+        model=model_name,
+        modalities=output_modalities,
+        extra_body=extra_body,
+        stream=args.stream,
+    )
 
-    with concurrent.futures.ThreadPoolExecutor(max_workers=num_concurrent_requests) as executor:
-        # Submit multiple completion requests concurrently
-        futures = [
-            executor.submit(
-                client.chat.completions.create,
-                messages=[
-                    get_system_prompt(),
-                    prompt,
-                ],
-                model=model_name,
-                modalities=output_modalities,
-                extra_body=extra_body,
-                stream=args.stream,
-            )
-            for _ in range(num_concurrent_requests)
-        ]
-
-        # Wait for all requests to complete and collect results
-        chat_completions = [future.result() for future in concurrent.futures.as_completed(futures)]
-
-    assert len(chat_completions) == num_concurrent_requests
     count = 0
     if not args.stream:
-        # Verify all completions succeeded
-        for chat_completion in chat_completions:
-            for choice in chat_completion.choices:
-                if choice.message.audio:
-                    audio_data = base64.b64decode(choice.message.audio.data)
+        for choice in chat_completion.choices:
+            if choice.message.audio:
+                audio_data = base64.b64decode(choice.message.audio.data)
+                audio_file_path = f"audio_{count}.wav"
+                with open(audio_file_path, "wb") as f:
+                    f.write(audio_data)
+                print(f"Audio saved to {audio_file_path}")
+                count += 1
+            elif choice.message.content:
+                print("Chat completion output from text:", choice.message.content)
+    else:
+        printed_content = False
+        for chunk in chat_completion:
+            for choice in chunk.choices:
+                if hasattr(choice, "delta"):
+                    content = getattr(choice.delta, "content", None)
+                else:
+                    content = None
+
+                if getattr(chunk, "modality", None) == "audio" and content:
+                    audio_data = base64.b64decode(content)
                     audio_file_path = f"audio_{count}.wav"
                     with open(audio_file_path, "wb") as f:
                         f.write(audio_data)
-                    print(f"Audio saved to {audio_file_path}")
+                    print(f"\nAudio saved to {audio_file_path}")
                     count += 1
-                elif choice.message.content:
-                    print("Chat completion output from text:", choice.message.content)
-    else:
-        printed_content = False
-        for chat_completion in chat_completions:
-            for chunk in chat_completion:
-                for choice in chunk.choices:
-                    if hasattr(choice, "delta"):
-                        content = getattr(choice.delta, "content", None)
-                    else:
-                        content = None
 
-                    if getattr(chunk, "modality", None) == "audio" and content:
-                        audio_data = base64.b64decode(content)
-                        audio_file_path = f"audio_{count}.wav"
-                        with open(audio_file_path, "wb") as f:
-                            f.write(audio_data)
-                        print(f"\nAudio saved to {audio_file_path}")
-                        count += 1
-
-                    elif getattr(chunk, "modality", None) == "text":
-                        if not printed_content:
-                            printed_content = True
-                            print("\ncontent:", end="", flush=True)
-                        print(content, end="", flush=True)
+                elif getattr(chunk, "modality", None) == "text":
+                    if not printed_content:
+                        printed_content = True
+                        print("\ncontent:", end="", flush=True)
+                    print(content, end="", flush=True)
 
 
 def parse_args():
@@ -534,12 +490,6 @@ def parse_args():
         action="store_true",
         help="Stream the response.",
     )
-    parser.add_argument(
-        "--num-concurrent-requests",
-        type=int,
-        default=1,
-        help="Number of concurrent requests to send. Default is 1.",
-    )
 
     return parser.parse_args()
 

examples/online_serving/qwen3_omni/qwen3_omni_moe_thinking.yaml

@@ -13,7 +13,7 @@ stage_args:
     engine_args:
       model_stage: thinker
       model_arch: Qwen3OmniMoeForConditionalGeneration
-      worker_type: ar
+      worker_cls: vllm_omni.worker.gpu_ar_worker.GPUARWorker
       scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
       gpu_memory_utilization: 0.9
       enforce_eager: true

examples/online_serving/qwen3_tts/README.md

@@ -1,175 +0,0 @@
-# Qwen3-TTS Online Serving
-
-This directory contains examples for running Qwen3-TTS models with vLLM-Omni's online serving API.
-
-## Supported Models
-
-| Model | Task Type | Description |
-|-------|-----------|-------------|
-| `Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice` | CustomVoice | Predefined speaker voices with optional style control |
-| `Qwen/Qwen3-TTS-12Hz-1.7B-VoiceDesign` | VoiceDesign | Natural language voice style description |
-| `Qwen/Qwen3-TTS-12Hz-1.7B-Base` | Base | Voice cloning from reference audio |
-
-## Quick Start
-
-### 1. Start the Server
-
-```bash
-# CustomVoice model (default)
-./run_server.sh
-
-# Or specify task type
-./run_server.sh CustomVoice
-./run_server.sh VoiceDesign
-./run_server.sh Base
-```
-
-### 2. Run the Client
-
-```bash
-# CustomVoice: Use predefined speaker
-python openai_speech_client.py \
-    --text "你好，我是通义千问" \
-    --voice Vivian \
-    --language Chinese
-
-# CustomVoice with style instruction
-python openai_speech_client.py \
-    --text "今天天气真好" \
-    --voice Ryan \
-    --instructions "用开心的语气说"
-
-# VoiceDesign: Describe the voice style
-python openai_speech_client.py \
-    --model Qwen/Qwen3-TTS-12Hz-1.7B-VoiceDesign \
-    --task-type VoiceDesign \
-    --text "哥哥，你回来啦" \
-    --instructions "体现撒娇稚嫩的萝莉女声，音调偏高"
-
-# Base: Voice cloning
-python openai_speech_client.py \
-    --model Qwen/Qwen3-TTS-12Hz-1.7B-Base \
-    --task-type Base \
-    --text "Hello, this is a cloned voice" \
-    --ref-audio /path/to/reference.wav \
-    --ref-text "Original transcript of the reference audio"
-```
-
-### 3. Using curl
-
-```bash
-# Simple TTS request
-curl -X POST http://localhost:8000/v1/audio/speech \
-    -H "Content-Type: application/json" \
-    -d '{
-        "input": "Hello, how are you?",
-        "voice": "Vivian",
-        "language": "English"
-    }' --output output.wav
-
-# With style instruction
-curl -X POST http://localhost:8000/v1/audio/speech \
-    -H "Content-Type: application/json" \
-    -d '{
-        "input": "I am so excited!",
-        "voice": "Vivian",
-        "instructions": "Speak with great enthusiasm"
-    }' --output excited.wav
-
-# List available voices in CustomVoice models
-curl http://localhost:8000/v1/audio/voices
-```
-
-## API Reference
-
-### Endpoint
-
-```
-POST /v1/audio/speech
-```
-
-This endpoint follows the [OpenAI Audio Speech API](https://platform.openai.com/docs/api-reference/audio/createSpeech) format with additional Qwen3-TTS parameters.
-
-### Request Body
-
-```json
-{
-    "input": "Text to synthesize",
-    "voice": "Vivian",
-    "response_format": "wav",
-    "task_type": "CustomVoice",
-    "language": "Auto",
-    "instructions": "Optional style instructions",
-    "ref_audio": "URL or base64 for voice cloning",
-    "ref_text": "Reference audio transcript",
-    "x_vector_only_mode": false,
-    "max_new_tokens": 2048
-}
-```
-
-> **Note:** The `model` field is optional when serving a single model, as the server already knows which model is loaded.
-
-### Response
-
-Returns audio data in the requested format (default: WAV).
-
-## Parameters
-
-### Standard OpenAI Parameters
-
-| Parameter | Type | Default | Description |
-|-----------|------|---------|-------------|
-| `input` | string | required | Text to synthesize |
-| `voice` | string | "Vivian" | Speaker/voice name |
-| `response_format` | string | "wav" | Audio format: wav, mp3, flac, pcm, aac, opus |
-| `speed` | float | 1.0 | Playback speed (0.25-4.0) |
-| `model` | string | optional | Model name (optional when serving single model) |
-
-### Qwen3-TTS Parameters
-
-| Parameter | Type | Default | Description |
-|-----------|------|---------|-------------|
-| `task_type` | string | "CustomVoice" | Task: CustomVoice, VoiceDesign, or Base |
-| `language` | string | "Auto" | Language: Auto, Chinese, English, Japanese, Korean |
-| `instructions` | string | "" | Voice style/emotion instructions |
-| `max_new_tokens` | int | 2048 | Maximum tokens to generate |
-
-### Voice Clone Parameters (Base task)
-
-| Parameter | Type | Required | Description |
-|-----------|------|----------|-------------|
-| `ref_audio` | string | Yes* | Reference audio (file path, URL, or base64) |
-| `ref_text` | string | No | Transcript of reference audio (for ICL mode) |
-| `x_vector_only_mode` | bool | false | Use speaker embedding only (no ICL) |
-
-## Python Usage
-
-```python
-import httpx
-
-# Simple request
-response = httpx.post(
-    "http://localhost:8000/v1/audio/speech",
-    json={
-        "model": "Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice",
-        "input": "Hello world",
-        "voice": "Vivian",
-    },
-    timeout=300.0,
-)
-
-with open("output.wav", "wb") as f:
-    f.write(response.content)
-```
-
-## Limitations
-
-- **No streaming**: Audio is generated completely before being returned. Streaming will be supported after the pipeline is disaggregated (see RFC #938).
-- **Single request**: Batch processing is not yet optimized for online serving.
-
-## Troubleshooting
-
-1. **Connection refused**: Make sure the server is running on the correct port
-2. **Out of memory**: Reduce `--gpu-memory-utilization` in run_server.sh
-3. **Unsupported speaker**: Check supported speakers via model documentation
-4. **Voice clone fails**: Ensure you're using the Base model variant for voice cloning

examples/online_serving/qwen3_tts/openai_speech_client.py

@@ -1,240 +0,0 @@
-"""OpenAI-compatible client for Qwen3-TTS via /v1/audio/speech endpoint.
-
-This script demonstrates how to use the OpenAI-compatible speech API
-to generate audio from text using Qwen3-TTS models.
-
-Examples:
-    # CustomVoice task (predefined speaker)
-    python openai_speech_client.py --text "Hello, how are you?" --voice Vivian
-
-    # CustomVoice with emotion instruction
-    python openai_speech_client.py --text "I'm so happy!" --voice Vivian \
-        --instructions "Speak with excitement"
-
-    # VoiceDesign task (voice from description)
-    python openai_speech_client.py --text "Hello world" \
-        --task-type VoiceDesign \
-        --instructions "A warm, friendly female voice"
-
-    # Base task (voice cloning)
-    python openai_speech_client.py --text "Hello world" \
-        --task-type Base \
-        --ref-audio "https://example.com/reference.wav" \
-        --ref-text "This is the reference transcript"
-"""
-
-import argparse
-import base64
-import os
-
-import httpx
-
-# Default server configuration
-DEFAULT_API_BASE = "http://localhost:8000"
-DEFAULT_API_KEY = "EMPTY"
-
-
-def encode_audio_to_base64(audio_path: str) -> str:
-    """Encode a local audio file to base64 data URL."""
-    if not os.path.exists(audio_path):
-        raise FileNotFoundError(f"Audio file not found: {audio_path}")
-
-    # Detect MIME type from extension
-    audio_path_lower = audio_path.lower()
-    if audio_path_lower.endswith(".wav"):
-        mime_type = "audio/wav"
-    elif audio_path_lower.endswith((".mp3", ".mpeg")):
-        mime_type = "audio/mpeg"
-    elif audio_path_lower.endswith(".flac"):
-        mime_type = "audio/flac"
-    elif audio_path_lower.endswith(".ogg"):
-        mime_type = "audio/ogg"
-    else:
-        mime_type = "audio/wav"  # Default
-
-    with open(audio_path, "rb") as f:
-        audio_bytes = f.read()
-    audio_b64 = base64.b64encode(audio_bytes).decode("utf-8")
-    return f"data:{mime_type};base64,{audio_b64}"
-
-
-def run_tts_generation(args) -> None:
-    """Run TTS generation via OpenAI-compatible /v1/audio/speech API."""
-
-    # Build request payload
-    payload = {
-        "model": args.model,
-        "input": args.text,
-        "voice": args.voice,
-        "response_format": args.response_format,
-    }
-
-    # Add optional parameters
-    if args.instructions:
-        payload["instructions"] = args.instructions
-    if args.task_type:
-        payload["task_type"] = args.task_type
-    if args.language:
-        payload["language"] = args.language
-    if args.max_new_tokens:
-        payload["max_new_tokens"] = args.max_new_tokens
-
-    # Voice clone parameters (Base task)
-    if args.ref_audio:
-        if args.ref_audio.startswith(("http://", "https://")):
-            payload["ref_audio"] = args.ref_audio
-        else:
-            payload["ref_audio"] = encode_audio_to_base64(args.ref_audio)
-    if args.ref_text:
-        payload["ref_text"] = args.ref_text
-    if args.x_vector_only:
-        payload["x_vector_only_mode"] = True
-
-    print(f"Model: {args.model}")
-    print(f"Task type: {args.task_type or 'CustomVoice'}")
-    print(f"Text: {args.text}")
-    print(f"Voice: {args.voice}")
-    print("Generating audio...")
-
-    # Make the API call
-    api_url = f"{args.api_base}/v1/audio/speech"
-    headers = {
-        "Content-Type": "application/json",
-        "Authorization": f"Bearer {args.api_key}",
-    }
-
-    with httpx.Client(timeout=300.0) as client:
-        response = client.post(api_url, json=payload, headers=headers)
-
-    if response.status_code != 200:
-        print(f"Error: {response.status_code}")
-        print(response.text)
-        return
-
-    if response.content.decode("utf-8").startswith('{"error"'):
-        print(f"Error: {response.content.decode('utf-8')}")
-        return
-
-    # Save audio response
-    output_path = args.output or "tts_output.wav"
-    with open(output_path, "wb") as f:
-        f.write(response.content)
-    print(f"Audio saved to: {output_path}")
-
-
-def parse_args():
-    """Parse command line arguments."""
-    parser = argparse.ArgumentParser(
-        description="OpenAI-compatible client for Qwen3-TTS via /v1/audio/speech",
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-        epilog=__doc__,
-    )
-
-    # Server configuration
-    parser.add_argument(
-        "--api-base",
-        type=str,
-        default=DEFAULT_API_BASE,
-        help=f"API base URL (default: {DEFAULT_API_BASE})",
-    )
-    parser.add_argument(
-        "--api-key",
-        type=str,
-        default=DEFAULT_API_KEY,
-        help="API key (default: EMPTY)",
-    )
-    parser.add_argument(
-        "--model",
-        "-m",
-        type=str,
-        default="Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice",
-        help="Model name/path",
-    )
-
-    # Task configuration
-    parser.add_argument(
-        "--task-type",
-        "-t",
-        type=str,
-        default=None,
-        choices=["CustomVoice", "VoiceDesign", "Base"],
-        help="TTS task type (default: CustomVoice)",
-    )
-
-    # Input text
-    parser.add_argument(
-        "--text",
-        type=str,
-        required=True,
-        help="Text to synthesize",
-    )
-
-    # Voice/speaker
-    parser.add_argument(
-        "--voice",
-        type=str,
-        default="Vivian",
-        help="Speaker/voice name (default: Vivian). Options: Vivian, Ryan, etc.",
-    )
-    parser.add_argument(
-        "--language",
-        type=str,
-        default=None,
-        help="Language: Auto, Chinese, English, etc.",
-    )
-    parser.add_argument(
-        "--instructions",
-        type=str,
-        default=None,
-        help="Voice style/emotion instructions",
-    )
-
-    # Base (voice clone) parameters
-    parser.add_argument(
-        "--ref-audio",
-        type=str,
-        default=None,
-        help="Reference audio file path or URL for voice cloning (Base task)",
-    )
-    parser.add_argument(
-        "--ref-text",
-        type=str,
-        default=None,
-        help="Reference audio transcript for voice cloning (Base task)",
-    )
-    parser.add_argument(
-        "--x-vector-only",
-        action="store_true",
-        help="Use x-vector only mode for voice cloning (no ICL)",
-    )
-
-    # Generation parameters
-    parser.add_argument(
-        "--max-new-tokens",
-        type=int,
-        default=None,
-        help="Maximum new tokens to generate",
-    )
-
-    # Output
-    parser.add_argument(
-        "--response-format",
-        type=str,
-        default="wav",
-        choices=["wav", "mp3", "flac", "pcm", "aac", "opus"],
-        help="Audio output format (default: wav)",
-    )
-    parser.add_argument(
-        "--output",
-        "-o",
-        type=str,
-        default=None,
-        help="Output audio file path (default: tts_output.wav)",
-    )
-
-    return parser.parse_args()
-
-
-if __name__ == "__main__":
-    args = parse_args()
-    run_tts_generation(args)