comment

Signed-off-by: Roger Wang <hey@rogerw.io>

.buildkite/bootstrap-amd-omni.sh

@@ -1,238 +0,0 @@
-#!/bin/bash
-# vllm-omni customized version
-# Based on: https://github.com/vllm-project/ci-infra/blob/main/buildkite/bootstrap-amd.sh
-# Last synced: 2025-12-15
-# Modifications: Use local template file instead of downloading from ci-infra
-
-set -euo pipefail
-
-if [[ -z "${RUN_ALL:-}" ]]; then
-    RUN_ALL=0
-fi
-
-if [[ -z "${NIGHTLY:-}" ]]; then
-    NIGHTLY=0
-fi
-
-if [[ -z "${VLLM_CI_BRANCH:-}" ]]; then
-    VLLM_CI_BRANCH="main"
-fi
-
-if [[ -z "${AMD_MIRROR_HW:-}" ]]; then
-    AMD_MIRROR_HW="amdproduction"
-fi
-
-if [[ -z "${DOCS_ONLY_DISABLE:-}" ]]; then
-    DOCS_ONLY_DISABLE=0
-fi
-
-fail_fast() {
-    DISABLE_LABEL="ci-no-fail-fast"
-    # If BUILDKITE_PULL_REQUEST != "false", then we check the PR labels using curl and jq
-    if [ "$BUILDKITE_PULL_REQUEST" != "false" ]; then
-        PR_LABELS=$(curl -s "https://api.github.com/repos/vllm-project/vllm-omni/pulls/$BUILDKITE_PULL_REQUEST" | jq -r '.labels[].name')
-        if [[ $PR_LABELS == *"$DISABLE_LABEL"* ]]; then
-            echo false
-        else
-            echo true
-        fi
-    else
-        echo false  # not a PR or BUILDKITE_PULL_REQUEST not set
-    fi
-}
-
-check_run_all_label() {
-    RUN_ALL_LABEL="ready-run-all-tests"
-    # If BUILDKITE_PULL_REQUEST != "false", then we check the PR labels using curl and jq
-    if [ "$BUILDKITE_PULL_REQUEST" != "false" ]; then
-        PR_LABELS=$(curl -s "https://api.github.com/repos/vllm-project/vllm-omni/pulls/$BUILDKITE_PULL_REQUEST" | jq -r '.labels[].name')
-        if [[ $PR_LABELS == *"$RUN_ALL_LABEL"* ]]; then
-            echo true
-        else
-            echo false
-        fi
-    else
-        echo false  # not a PR or BUILDKITE_PULL_REQUEST not set
-    fi
-}
-
-if [[ -z "${COV_ENABLED:-}" ]]; then
-    COV_ENABLED=0
-fi
-
-upload_pipeline() {
-    echo "Uploading pipeline..."
-    # Install minijinja
-    ls .buildkite || buildkite-agent annotate --style error 'Please merge upstream main branch for buildkite CI'
-    curl -sSfL https://github.com/mitsuhiko/minijinja/releases/download/2.3.1/minijinja-cli-installer.sh | sh
-    source /var/lib/buildkite-agent/.cargo/env
-
-    if [[ $BUILDKITE_PIPELINE_SLUG == "fastcheck" ]]; then
-        AMD_MIRROR_HW="amdtentative"
-    fi
-
-    # Use local template file for vllm-omni
-    cp .buildkite/test-template-amd-omni.j2 .buildkite/test-template.j2
-
-
-    # (WIP) Use pipeline generator instead of jinja template
-    if [ -e ".buildkite/pipeline_generator/pipeline_generator.py" ]; then
-        python -m pip install click pydantic
-        python .buildkite/pipeline_generator/pipeline_generator.py --run_all=$RUN_ALL --list_file_diff="$LIST_FILE_DIFF" --nightly="$NIGHTLY" --mirror_hw="$AMD_MIRROR_HW"
-        buildkite-agent pipeline upload .buildkite/pipeline.yaml
-        exit 0
-    fi
-    echo "List file diff: $LIST_FILE_DIFF"
-    echo "Run all: $RUN_ALL"
-    echo "Nightly: $NIGHTLY"
-    echo "AMD Mirror HW: $AMD_MIRROR_HW"
-
-    FAIL_FAST=$(fail_fast)
-
-    cd .buildkite
-    (
-        set -x
-        # Output pipeline.yaml with all blank lines removed
-        minijinja-cli test-template.j2 test-amd.yaml \
-            -D branch="$BUILDKITE_BRANCH" \
-            -D list_file_diff="$LIST_FILE_DIFF" \
-            -D run_all="$RUN_ALL" \
-            -D nightly="$NIGHTLY" \
-            -D mirror_hw="$AMD_MIRROR_HW" \
-            -D fail_fast="$FAIL_FAST" \
-            -D vllm_use_precompiled="$VLLM_USE_PRECOMPILED" \
-            -D vllm_merge_base_commit="$(git merge-base origin/main HEAD)" \
-            -D cov_enabled="$COV_ENABLED" \
-            -D vllm_ci_branch="$VLLM_CI_BRANCH" \
-            | sed '/^[[:space:]]*$/d' \
-            > pipeline.yaml
-    )
-    cat pipeline.yaml
-    buildkite-agent artifact upload pipeline.yaml
-    buildkite-agent pipeline upload pipeline.yaml
-    exit 0
-}
-
-get_diff() {
-    $(git add .)
-    echo $(git diff --name-only --diff-filter=ACMDR $(git merge-base origin/main HEAD))
-}
-
-get_diff_main() {
-    $(git add .)
-    echo $(git diff --name-only --diff-filter=ACMDR HEAD~1)
-}
-
-file_diff=$(get_diff)
-if [[ $BUILDKITE_BRANCH == "main" ]]; then
-    file_diff=$(get_diff_main)
-fi
-
-# ----------------------------------------------------------------------
-# Early exit start: skip pipeline if conditions are met
-# ----------------------------------------------------------------------
-
-# skip pipeline if all changed files are under docs/
-if [[ "${DOCS_ONLY_DISABLE}" != "1" ]]; then
-  if [[ -n "${file_diff:-}" ]]; then
-    docs_only=1
-    # Robust iteration over newline-separated file_diff
-    while IFS= read -r f; do
-      [[ -z "$f" ]] && continue
-      # **Policy:** only skip if *every* path starts with docs/
-      if [[ "$f" != docs/* ]]; then
-        docs_only=0
-        break
-      fi
-    done < <(printf '%s\n' "$file_diff" | tr ' ' '\n' | tr -d '\r')
-
-    if [[ "$docs_only" -eq 1 ]]; then
-      buildkite-agent annotate ":memo: CI skipped — docs/** only changes detected
-
-\`\`\`
-${file_diff}
-\`\`\`" --style "info" || true
-      echo "[docs-only] All changes are under docs/. Exiting before pipeline upload."
-      exit 0
-    fi
-  fi
-fi
-
-# ----------------------------------------------------------------------
-# Early exit end
-# ----------------------------------------------------------------------
-
-patterns=(
-    "docker/Dockerfile"
-    "CMakeLists.txt"
-    "requirements/common.txt"
-    "requirements/cuda.txt"
-    "requirements/build.txt"
-    "requirements/test.txt"
-    "setup.py"
-    "csrc/"
-    "cmake/"
-)
-
-ignore_patterns=(
-    "docker/Dockerfile."
-    "csrc/cpu"
-    "csrc/rocm"
-    "cmake/hipify.py"
-    "cmake/cpu_extension.cmake"
-)
-
-for file in $file_diff; do
-    # First check if file matches any pattern
-    matches_pattern=0
-    for pattern in "${patterns[@]}"; do
-        if [[ $file == $pattern* ]] || [[ $file == $pattern ]]; then
-            matches_pattern=1
-            break
-        fi
-    done
-
-    # If file matches pattern, check it's not in ignore patterns
-    if [[ $matches_pattern -eq 1 ]]; then
-        matches_ignore=0
-        for ignore in "${ignore_patterns[@]}"; do
-            if [[ $file == $ignore* ]] || [[ $file == $ignore ]]; then
-                matches_ignore=1
-                break
-            fi
-        done
-
-        if [[ $matches_ignore -eq 0 ]]; then
-            RUN_ALL=1
-            echo "Found changes: $file. Run all tests"
-            break
-        fi
-    fi
-done
-
-# Check for ready-run-all-tests label
-LABEL_RUN_ALL=$(check_run_all_label)
-if [[ $LABEL_RUN_ALL == true ]]; then
-    RUN_ALL=1
-    NIGHTLY=1
-    echo "Found 'ready-run-all-tests' label. Running all tests including optional tests."
-fi
-
-# Decide whether to use precompiled wheels
-# Relies on existing patterns array as a basis.
-if [[ -n "${VLLM_USE_PRECOMPILED:-}" ]]; then
-    echo "VLLM_USE_PRECOMPILED is already set to: $VLLM_USE_PRECOMPILED"
-elif [[ $RUN_ALL -eq 1 ]]; then
-    export VLLM_USE_PRECOMPILED=0
-    echo "Detected critical changes, building wheels from source"
-else
-    export VLLM_USE_PRECOMPILED=1
-    echo "No critical changes, using precompiled wheels"
-fi
-
-
-LIST_FILE_DIFF=$(get_diff | tr ' ' '|')
-if [[ $BUILDKITE_BRANCH == "main" ]]; then
-    LIST_FILE_DIFF=$(get_diff_main | tr ' ' '|')
-fi
-upload_pipeline

.buildkite/pipeline.yml

@@ -3,38 +3,85 @@ steps:
     key: image-build
     commands:
       - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7"
-      - "docker build --progress=plain --file docker/Dockerfile.ci -t vllm-omni-ci ."
+      - "docker build --file docker/Dockerfile.ci -t vllm-omni-ci ."
       - "docker tag vllm-omni-ci public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT"
       - "docker push public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT"
     agents:
-      queue: "cpu_queue_premerge"
+      queue: "cpu_queue_premerge_us_east_1"
 
-  # L2 Test
-  - label: "Upload Ready Pipeline"
-    depends_on: image-build
-    key: upload-ready-pipeline
-    if: build.branch != "main"
+  - label: "Simple Unit Test"
     commands:
-      - buildkite-agent pipeline upload .buildkite/test-ready.yml
+      - ".buildkite/scripts/simple_test.sh"
     agents:
       queue: "cpu_queue_premerge"
 
-  # L3 Test
-  - label: "Upload Merge Pipeline"
+  - label: "Diffusion Model Test"
+    timeout_in_minutes: 15
     depends_on: image-build
-    key: upload-merge-pipeline
-    if: build.branch == "main"
     commands:
-      - buildkite-agent pipeline upload .buildkite/test-merge.yml
+      - pytest -s -v tests/e2e/offline_inference/test_t2i_model.py
     agents:
-      queue: "cpu_queue_premerge"
+      queue: "gpu_1_queue" # g6.4xlarge instance on AWS, has 1 L4 GPU
+    plugins:
+      - docker#v5.2.0:
+          image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
+          always-pull: true
+          propagate-environment: true
+          environment:
+            - "HF_HOME=/fsx/hf_cache"
+          volumes:
+            - "/fsx/hf_cache:/fsx/hf_cache"
 
-  # L4 Test
-  - label: "Upload Nightly Pipeline"
+  - label: "Omni Model Test"
+    timeout_in_minutes: 15
     depends_on: image-build
-    key: upload-nightly-pipeline
-    if: build.env("NIGHTLY") == "1"
     commands:
-      - buildkite-agent pipeline upload .buildkite/test-nightly.yml
+      - export VLLM_LOGGING_LEVEL=DEBUG
+      - export VLLM_WORKER_MULTIPROC_METHOD=spawn
+      - pytest -s -v tests/e2e/offline_inference/test_qwen2_5_omni.py
     agents:
-      queue: "cpu_queue_premerge"
+      queue: "gpu_4_queue" # g6.12xlarge instance on AWS, has 4 L4 GPU
+    plugins:
+      - docker#v5.2.0:
+          image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
+          always-pull: true
+          propagate-environment: true
+          environment:
+            - "HF_HOME=/fsx/hf_cache"
+          volumes:
+            - "/fsx/hf_cache:/fsx/hf_cache"
+
+  - label: "Omni Model Test with H100"
+    timeout_in_minutes: 20
+    depends_on: image-build
+    commands:
+      - export VLLM_WORKER_MULTIPROC_METHOD=spawn
+      - pytest -s -v tests/e2e/offline_inference/test_qwen3_omni.py tests/e2e/online_serving/test_qwen3_omni.py
+    agents:
+      queue: "mithril-h100-pool"
+    plugins:
+      - kubernetes:
+          podSpec:
+            containers:
+              - image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
+                resources:
+                  limits:
+                    nvidia.com/gpu: 2
+                volumeMounts:
+                  - name: devshm
+                    mountPath: /dev/shm
+                  - name: hf-cache
+                    mountPath: /root/.cache/huggingface
+                env:
+                  - name: HF_HOME
+                    value: /root/.cache/huggingface
+            nodeSelector:
+              node.kubernetes.io/instance-type: gpu-h100-sxm
+            volumes:
+              - name: devshm
+                emptyDir:
+                  medium: Memory
+              - name: hf-cache
+                hostPath:
+                  path: /mnt/hf-cache
+                  type: DirectoryOrCreate

.buildkite/scripts/docker_login_ecr_public.sh

@@ -1,73 +0,0 @@
-#!/bin/bash
-# Helper function to safely login to ECR Public with per-job config isolation
-# Uses DOCKER_CONFIG environment variable to prevent race conditions
-#
-# This script prevents the "device or resource busy" error by giving each
-# Buildkite job its own isolated Docker config directory.
-#
-# Usage:
-#   source docker_login_ecr_public.sh && safe_docker_login_ecr_public
-
-set -euo pipefail
-
-# Configuration
-ECR_REGISTRY="public.ecr.aws"
-
-setup_isolated_docker_config() {
-    # Use BUILDKITE_JOB_ID for job-specific isolation
-    # Fallback to PID if running outside Buildkite
-    local job_id="${BUILDKITE_JOB_ID:-$$}"
-
-    # Set Docker config to job-specific directory
-    export DOCKER_CONFIG="/tmp/docker-config-${job_id}"
-
-    # Create directory if it doesn't exist
-    mkdir -p "$DOCKER_CONFIG"
-
-    echo "[docker-config] Using isolated Docker config: $DOCKER_CONFIG"
-}
-
-check_docker_auth() {
-    # Check if already authenticated to the given registry
-    # Returns 0 if authenticated, 1 if not
-    local registry="$1"
-
-    # Check if credentials exist in the isolated config
-    if [[ -f "$DOCKER_CONFIG/config.json" ]]; then
-        # Check if registry is present in config
-        if grep -q "$registry" "$DOCKER_CONFIG/config.json" 2>/dev/null; then
-            return 0
-        fi
-    fi
-
-    return 1
-}
-
-safe_docker_login_ecr_public() {
-    # Setup isolated config first
-    setup_isolated_docker_config
-
-    local registry="$ECR_REGISTRY"
-
-    # Check if already authenticated (within this job)
-    if check_docker_auth "$registry"; then
-        echo "[docker-login] Already authenticated to $registry in this job"
-        return 0
-    fi
-
-    # Perform login to isolated config directory
-    echo "[docker-login] Logging in to $ECR_REGISTRY (isolated config)..."
-    if aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin "$ECR_REGISTRY"; then
-        echo "[docker-login] Login successful (config: $DOCKER_CONFIG)"
-        return 0
-    else
-        local exit_code=$?
-        echo "[docker-login] ERROR: Login failed with exit code $exit_code" >&2
-        return $exit_code
-    fi
-}
-
-# Execute if run as script (not sourced)
-if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then
-    safe_docker_login_ecr_public
-fi

.buildkite/scripts/hardware_ci/run-amd-test.sh

@@ -1,166 +0,0 @@
-#!/bin/bash
-# vllm-omni customized version
-# Based on: vllm/.buildkite/scripts/hardware_ci/run-amd-test.sh
-# Last synced: 2025-12-15
-# Modifications: docker image name for vllm-omni
-
-# This script runs test inside the corresponding ROCm docker container.
-set -o pipefail
-
-# Export Python path
-export PYTHONPATH=".."
-
-# Print ROCm version
-echo "--- Confirming Clean Initial State"
-while true; do
-        sleep 3
-        if grep -q clean /opt/amdgpu/etc/gpu_state; then
-                echo "GPUs state is \"clean\""
-                break
-        fi
-done
-
-echo "--- ROCm info"
-rocminfo
-
-# cleanup older docker images
-cleanup_docker() {
-  # Get Docker's root directory
-  docker_root=$(docker info -f '{{.DockerRootDir}}')
-  if [ -z "$docker_root" ]; then
-    echo "Failed to determine Docker root directory."
-    exit 1
-  fi
-  echo "Docker root directory: $docker_root"
-  # Check disk usage of the filesystem where Docker's root directory is located
-  disk_usage=$(df "$docker_root" | tail -1 | awk '{print $5}' | sed 's/%//')
-  # Define the threshold
-  threshold=70
-  if [ "$disk_usage" -gt "$threshold" ]; then
-    echo "Disk usage is above $threshold%. Cleaning up Docker images and volumes..."
-    # Remove dangling images (those that are not tagged and not used by any container)
-    docker image prune -f
-    # Remove unused volumes / force the system prune for old images as well.
-    docker volume prune -f && docker system prune --force --filter "until=72h" --all
-    echo "Docker images and volumes cleanup completed."
-  else
-    echo "Disk usage is below $threshold%. No cleanup needed."
-  fi
-}
-
-# Call the cleanup docker function
-cleanup_docker
-
-echo "--- Resetting GPUs"
-
-echo "reset" > /opt/amdgpu/etc/gpu_state
-
-while true; do
-        sleep 3
-        if grep -q clean /opt/amdgpu/etc/gpu_state; then
-                echo "GPUs state is \"clean\""
-                break
-        fi
-done
-
-echo "--- Pulling container"
-image_name="public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:${BUILDKITE_COMMIT}-rocm-omni"
-container_name="rocm_${BUILDKITE_COMMIT}_$(tr -dc A-Za-z0-9 < /dev/urandom | head -c 10; echo)"
-
-# Install AWS CLI to authenticate to ECR Public Gallery to get higher rate limit for pulling images
-sudo apt-get update && sudo apt-get install -y awscli
-# Use safe docker login helper to prevent race conditions
-source "$(dirname "${BASH_SOURCE[0]}")/../docker_login_ecr_public.sh"
-safe_docker_login_ecr_public
-# Pull the container from ECR Public Gallery
-
-docker pull "${image_name}"
-
-remove_docker_container() {
-   docker rm -f "${container_name}" || docker image rm -f "${image_name}" || true
-}
-trap remove_docker_container EXIT
-
-echo "--- Running container"
-
-HF_CACHE="$(realpath ~)/huggingface"
-mkdir -p "${HF_CACHE}"
-HF_MOUNT="/root/.cache/huggingface"
-
-commands=$@
-echo "Commands:$commands"
-
-PARALLEL_JOB_COUNT=8
-MYPYTHONPATH=".."
-
-# Test that we're launching on the machine that has
-# proper access to GPUs
-render_gid=$(getent group render | cut -d: -f3)
-if [[ -z "$render_gid" ]]; then
-  echo "Error: 'render' group not found. This is required for GPU access." >&2
-  exit 1
-fi
-
-# check if the command contains shard flag, we will run all shards in parallel because the host have 8 GPUs.
-if [[ $commands == *"--shard-id="* ]]; then
-  # assign job count as the number of shards used
-  commands=$(echo "$commands" | sed -E "s/--num-shards[[:blank:]]*=[[:blank:]]*[0-9]*/--num-shards=${PARALLEL_JOB_COUNT} /g" | sed 's/ \\ / /g')
-  for GPU in $(seq 0 $(($PARALLEL_JOB_COUNT-1))); do
-    # assign shard-id for each shard
-    commands_gpu=$(echo "$commands" | sed -E "s/--shard-id[[:blank:]]*=[[:blank:]]*[0-9]*/--shard-id=${GPU} /g" | sed 's/ \\ / /g')
-    echo "Shard ${GPU} commands:$commands_gpu"
-    echo "Render devices: $BUILDKITE_AGENT_META_DATA_RENDER_DEVICES"
-    docker run \
-        --device /dev/kfd $BUILDKITE_AGENT_META_DATA_RENDER_DEVICES \
-        --network=host \
-        --shm-size=16gb \
-        --group-add "$render_gid" \
-        --rm \
-        -e MIOPEN_DEBUG_CONV_DIRECT=0 \
-        -e MIOPEN_DEBUG_CONV_GEMM=0 \
-        -e VLLM_ROCM_USE_AITER=1 \
-        -e HIP_VISIBLE_DEVICES="${GPU}" \
-        -e HF_TOKEN \
-        -e AWS_ACCESS_KEY_ID \
-        -e AWS_SECRET_ACCESS_KEY \
-        -v "${HF_CACHE}:${HF_MOUNT}" \
-        -e "HF_HOME=${HF_MOUNT}" \
-        -e "PYTHONPATH=${MYPYTHONPATH}" \
-        --name "${container_name}_${GPU}" \
-        "${image_name}" \
-        /bin/bash -c "${commands_gpu}" \
-        |& while read -r line; do echo ">>Shard $GPU: $line"; done &
-    PIDS+=($!)
-  done
-  #wait for all processes to finish and collect exit codes
-  for pid in "${PIDS[@]}"; do
-    wait "${pid}"
-    STATUS+=($?)
-  done
-  for st in "${STATUS[@]}"; do
-    if [[ ${st} -ne 0 ]]; then
-      echo "One of the processes failed with $st"
-      exit "${st}"
-    fi
-  done
-else
-  echo "Render devices: $BUILDKITE_AGENT_META_DATA_RENDER_DEVICES"
-  docker run \
-          --device /dev/kfd $BUILDKITE_AGENT_META_DATA_RENDER_DEVICES \
-          --network=host \
-          --shm-size=16gb \
-          --group-add "$render_gid" \
-          --rm \
-          -e MIOPEN_DEBUG_CONV_DIRECT=0 \
-          -e MIOPEN_DEBUG_CONV_GEMM=0 \
-          -e VLLM_ROCM_USE_AITER=1 \
-          -e HF_TOKEN \
-          -e AWS_ACCESS_KEY_ID \
-          -e AWS_SECRET_ACCESS_KEY \
-          -v "${HF_CACHE}:${HF_MOUNT}" \
-          -e "HF_HOME=${HF_MOUNT}" \
-          -e "PYTHONPATH=${MYPYTHONPATH}" \
-          --name "${container_name}" \
-          "${image_name}" \
-          /bin/bash -c "${commands}"
-fi

.buildkite/scripts/hardware_ci/run_npu_test.sh

@@ -1,145 +0,0 @@
-#!/bin/bash
-
-# This script build the Ascend NPU docker image and run the offline inference inside the container.
-# It serves a sanity check for compilation and basic model usage.
-set -ex
-
-# Base ubuntu image with basic ascend development libraries and python installed
-VLLM_OMNI_REPO="https://github.com/vllm-project/vllm-omni.git"
-BASE_IMAGE_NAME="quay.nju.edu.cn/ascend/vllm-ascend:v0.11.0rc2"
-image_name="npu/vllm-omni-ci:${BUILDKITE_COMMIT}_${EPOCHSECONDS}"
-# image_name="npu/vllm-ci:${BUILDKITE_COMMIT}_${EPOCHSECONDS}"
-container_name="npu_${BUILDKITE_COMMIT}_$(tr -dc A-Za-z0-9 < /dev/urandom | head -c 10; echo)"
-
-# BUILDKITE_AGENT_NAME format is {hostname}-{agent_idx}-{npu_card_num}cards
-agent_idx=$(echo "${BUILDKITE_AGENT_NAME}" | awk -F'-' '{print $(NF-1)}')
-echo "agent_idx: ${agent_idx}"
-builder_name="cachebuilder${agent_idx}"
-builder_cache_dir="/mnt/docker-cache${agent_idx}"
-mkdir -p ${builder_cache_dir}
-
-# Try building the docker image
-cat <<EOF | DOCKER_BUILDKIT=1 docker build \
-    --add-host pypi-cache:${PYPI_CACHE_HOST} \
-    --builder ${builder_name} --cache-from type=local,src=${builder_cache_dir} \
-                           --cache-to type=local,dest=${builder_cache_dir},mode=max \
-    --build-arg BUILDKITE_PULL_REQUEST="${BUILDKITE_PULL_REQUEST}" \
-    --build-arg BUILDKITE_PULL_REQUEST_REPO="${BUILDKITE_PULL_REQUEST_REPO}" \
-    --progress=plain --load -t ${image_name} -f - .
-FROM ${BASE_IMAGE_NAME}
-
-# Define environments
-ENV DEBIAN_FRONTEND=noninteractive
-
-RUN pip config set global.index-url http://pypi-cache:${PYPI_CACHE_PORT}/pypi/simple && \
-    pip config set global.trusted-host pypi-cache && \
-    apt-get update -y && \
-    apt-get install -y python3-pip git vim wget net-tools gcc g++ cmake libnuma-dev && \
-    rm -rf /var/cache/apt/* && \
-    rm -rf /var/lib/apt/lists/*
-
-# Install for pytest to make the docker build cache layer always valid
-RUN --mount=type=cache,target=/root/.cache/pip \
-    pip install pytest>=6.0  pytest-cov modelscope
-
-COPY . .
-
-# Install vllm-omni
-WORKDIR /workspace
-ARG VLLM_OMNI_REPO=https://github.com/vllm-project/vllm-omni.git
-ARG VLLM_OMNI_TAG=main
-ARG BUILDKITE_PULL_REQUEST
-ARG BUILDKITE_PULL_REQUEST_REPO
-RUN git config --global url."https://gh-proxy.test.osinfra.cn/https://github.com/".insteadOf "https://github.com/" && \
-    if [ "\$BUILDKITE_PULL_REQUEST" != "false" ] && [ -n "\$BUILDKITE_PULL_REQUEST" ]; then \
-        echo "Cloning and checking out PR #\$BUILDKITE_PULL_REQUEST..." && \
-        git clone \$VLLM_OMNI_REPO /workspace/vllm-omni && \
-        cd /workspace/vllm-omni && \
-        git fetch origin pull/\$BUILDKITE_PULL_REQUEST/head:pr-\$BUILDKITE_PULL_REQUEST && \
-        git checkout pr-\$BUILDKITE_PULL_REQUEST; \
-    else \
-        echo "Not a PR build, using main branch" && \
-        git clone --depth 1 \$VLLM_OMNI_REPO /workspace/vllm-omni; \
-    fi
-
-RUN --mount=type=cache,target=/root/.cache/pip \
-    export PIP_EXTRA_INDEX_URL=https://mirrors.huaweicloud.com/ascend/repos/pypi && \
-    source /usr/local/Ascend/ascend-toolkit/set_env.sh && \
-    source /usr/local/Ascend/nnal/atb/set_env.sh && \
-    export LD_LIBRARY_PATH=\$LD_LIBRARY_PATH:/usr/local/Ascend/ascend-toolkit/latest/`uname -i`-linux/devlib && \
-    python3 -m pip install -v -e /workspace/vllm-omni/
-
-ENV VLLM_WORKER_MULTIPROC_METHOD=spawn
-
-WORKDIR /workspace/vllm-omni
-CMD ["/bin/bash"]
-
-EOF
-
-# Setup cleanup
-remove_docker_container() {
-  docker rm -f "${container_name}" || true;
-  docker image rm -f "${image_name}" || true;
-  docker system prune -f || true;
-}
-trap remove_docker_container EXIT
-
-# Generate corresponding --device args based on BUILDKITE_AGENT_NAME
-# Ascend NPU BUILDKITE_AGENT_NAME format is {hostname}-{agent_idx}-{npu_card_num}cards, and agent_idx starts from 1.
-#   e.g. atlas-a2-001-1-2cards means this is the 1-th agent on atlas-a2-001 host, and it has 2 NPU cards.
-#   returns --device /dev/davinci0 --device /dev/davinci1
-parse_and_gen_devices() {
-    local input="$1"
-    local index cards_num
-    if [[ "$input" =~ ([0-9]+)-([0-9]+)cards$ ]]; then
-        index="${BASH_REMATCH[1]}"
-        cards_num="${BASH_REMATCH[2]}"
-    else
-        echo "parse error" >&2
-        return 1
-    fi
-
-    local devices=""
-    local i=0
-    while (( i < cards_num )); do
-        local dev_idx=$(((index - 1)*cards_num + i ))
-        devices="$devices --device /dev/davinci${dev_idx}"
-        ((i++))
-    done
-
-    # trim leading space
-    devices="${devices#"${devices%%[![:space:]]*}"}"
-    # Output devices: assigned to the caller variable
-    printf '%s' "$devices"
-}
-
-devices=$(parse_and_gen_devices "${BUILDKITE_AGENT_NAME}") || exit 1
-
-# Run the image and execute the Out-Of-Tree (OOT) platform interface test case on Ascend NPU hardware.
-# This test checks whether the OOT platform interface is functioning properly in conjunction with
-# the hardware plugin vllm-ascend.
-hf_model_cache_dir=/mnt/hf_cache${agent_idx}
-ms_model_cache_dir=/mnt/modelscope${agent_idx}
-mkdir -p ${hf_model_cache_dir}
-mkdir -p ${ms_model_cache_dir}
-docker run \
-    --init \
-    ${devices} \
-    --device /dev/davinci_manager \
-    --device /dev/devmm_svm \
-    --device /dev/hisi_hdc \
-    -v /usr/local/dcmi:/usr/local/dcmi \
-    -v /usr/local/bin/npu-smi:/usr/local/bin/npu-smi \
-    -v /usr/local/Ascend/driver/lib64/:/usr/local/Ascend/driver/lib64/ \
-    -v /usr/local/Ascend/driver/version.info:/usr/local/Ascend/driver/version.info \
-    -v /etc/ascend_install.info:/etc/ascend_install.info \
-    -v ${hf_model_cache_dir}:/root/.cache/huggingface \
-    -v ${ms_model_cache_dir}:/root/.cache/modelscope \
-    --network host \
-    --entrypoint="" \
-    --name "${container_name}" \
-    "${image_name}" \
-    bash -c '
-    set -e
-    VLLM_USE_MODELSCOPE=True pytest -s -v tests/e2e/offline_inference/test_qwen2_5_omni.py
-'

.buildkite/scripts/simple_test.sh

@@ -0,0 +1,52 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="$(cd -- "${SCRIPT_DIR}/../.." && pwd)"
+UV_BIN="uv"
+PYTHON_BOOTSTRAP="${PYTHON:-python3}"
+VENV_DIR="${VENV_DIR:-${REPO_ROOT}/.venv-simple-test}"
+TARGET_PY_VERSION="3.12"
+
+if ! command -v "${PYTHON_BOOTSTRAP}" >/dev/null 2>&1; then
+  PYTHON_BOOTSTRAP="python"
+fi
+
+cd "${REPO_ROOT}"
+
+# Ensure pip-installed scripts (including uv) are on PATH
+PYTHON_SCRIPTS="$(${PYTHON_BOOTSTRAP} -c 'import sysconfig; print(sysconfig.get_path("scripts"))')"
+USER_BASE="$(${PYTHON_BOOTSTRAP} -m site --user-base 2>/dev/null || true)"
+PATH="${PYTHON_SCRIPTS}:${PATH}"
+if [[ -n "${USER_BASE}" ]]; then
+  PATH="${USER_BASE}/bin:${PATH}"
+fi
+
+if ! command -v "${UV_BIN}" >/dev/null 2>&1; then
+  "${PYTHON_BOOTSTRAP}" -m pip install --upgrade pip
+  "${PYTHON_BOOTSTRAP}" -m pip install uv
+  hash -r
+fi
+
+TARGET_PY_PATH="$(${UV_BIN} python find "${TARGET_PY_VERSION}" 2>/dev/null | head -n1 || true)"
+if [[ -z "${TARGET_PY_PATH}" ]]; then
+  "${UV_BIN}" python install "${TARGET_PY_VERSION}"
+  TARGET_PY_PATH="$(${UV_BIN} python find "${TARGET_PY_VERSION}" 2>/dev/null | head -n1 || true)"
+fi
+
+if [[ -z "${TARGET_PY_PATH}" ]]; then
+  echo "Failed to locate python ${TARGET_PY_VERSION}" >&2
+  exit 1
+fi
+
+if [[ ! -d "${VENV_DIR}" ]]; then
+  "${UV_BIN}" venv --python "${TARGET_PY_PATH}" "${VENV_DIR}"
+fi
+
+VENV_PYTHON="${VENV_DIR}/bin/python"
+[[ -x "${VENV_PYTHON}" ]] || { echo "Python not found in ${VENV_DIR}"; exit 1; }
+
+"${UV_BIN}" pip install --python "${VENV_PYTHON}" vllm==0.11.0
+"${UV_BIN}" pip install --python "${VENV_PYTHON}" -e ".[dev]"
+"${VENV_PYTHON}" -m pytest -v -s tests/test_omni_llm.py

.buildkite/test-amd.yaml

@@ -1,118 +0,0 @@
-steps:
-
-- label: "Diffusion Model Test"
-  timeout_in_minutes: 20
-  agent_pool: mi325_2
-  depends_on: amd-build
-  mirror_hardwares: [amdproduction]
-  grade: Blocking
-  commands:
-    - export GPU_ARCHS=gfx942
-    - pytest -s -v tests/e2e/offline_inference/test_t2i_model.py
-
-- label: "Diffusion Images API LoRA E2E"
-  timeout_in_minutes: 20
-  agent_pool: mi325_1
-  depends_on: amd-build
-  mirror_hardwares: [amdproduction]
-  grade: Blocking
-  commands:
-    - export GPU_ARCHS=gfx942
-    - export VLLM_LOGGING_LEVEL=DEBUG
-    - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-    - pytest -s -v tests/e2e/online_serving/test_images_generations_lora.py
-
-- label: "Diffusion Model CPU offloading Test"
-  timeout_in_minutes: 20
-  agent_pool: mi325_1
-  depends_on: amd-build
-  mirror_hardwares: [amdproduction]
-  grade: Blocking
-  commands:
-    - export GPU_ARCHS=gfx942
-    - export VLLM_LOGGING_LEVEL=DEBUG
-    - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-    - pytest -s -v tests/e2e/offline_inference/test_diffusion_cpu_offload.py
-    - pytest -s -v tests/e2e/offline_inference/test_diffusion_layerwise_offload.py
-
-- label: "Diffusion Cache Backend Test"
-  timeout_in_minutes: 15
-  agent_pool: mi325_1
-  depends_on: amd-build
-  mirror_hardwares: [amdproduction]
-  grade: Blocking
-  commands:
-    - export GPU_ARCHS=gfx942
-    - export VLLM_LOGGING_LEVEL=DEBUG
-    - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-    - pytest -s -v -m 'core_model and cache and diffusion and not distributed_rocm and MI325'
-
-- label: "Diffusion Sequence Parallelism Test"
-  timeout_in_minutes: 20
-  agent_pool: mi325_2
-  depends_on: amd-build
-  mirror_hardwares: [amdproduction]
-  grade: Blocking
-  commands:
-    - export GPU_ARCHS=gfx942
-    - export VLLM_LOGGING_LEVEL=DEBUG
-    - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-    - pytest -s -v tests/e2e/offline_inference/test_sequence_parallel.py -m core_model
-
-- label: "Diffusion Tensor Parallelism Test"
-  timeout_in_minutes: 20
-  agent_pool: mi325_2
-  depends_on: amd-build
-  commands:
-    - export GPU_ARCHS=gfx942
-    - export VLLM_LOGGING_LEVEL=DEBUG
-    - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-    - pytest -s -v tests/e2e/offline_inference/test_zimage_parallelism.py
-
-- label: "Diffusion GPU Worker Test"
-  timeout_in_minutes: 20
-  agent_pool: mi325_2
-  depends_on: amd-build
-  mirror_hardwares: [amdproduction]
-  grade: Blocking
-  commands:
-    - pytest -s -v tests/diffusion/test_diffusion_worker.py
-
-- label: "Omni Model Test Qwen2-5-Omni"
-  timeout_in_minutes: 15
-  agent_pool: mi325_2
-  depends_on: amd-build
-  mirror_hardwares: [amdproduction]
-  grade: Blocking
-  commands:
-    - export GPU_ARCHS=gfx942
-    - export VLLM_LOGGING_LEVEL=DEBUG
-    - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-    - pytest -s -v tests/e2e/offline_inference/test_qwen2_5_omni.py
-    - pytest -s -v tests/engine/test_async_omni_engine_abort.py
-
-- label: "Omni Model Test Qwen3-Omni"
-  timeout_in_minutes: 15
-  agent_pool: mi325_2
-  depends_on: amd-build
-  mirror_hardwares: [amdproduction]
-  grade: Blocking
-  commands:
-    - export VLLM_LOGGING_LEVEL=DEBUG
-    - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-    - export VLLM_TEST_CLEAN_GPU_MEMORY="1"
-    - pytest -s -v tests/e2e/offline_inference/test_qwen3_omni.py
-    - pytest -s -v tests/e2e/online_serving/test_qwen3_omni.py
-
-
-- label: "Diffusion Image Edit Test"
-  timeout_in_minutes: 15
-  agent_pool: mi325_1
-  depends_on: amd-build
-  mirror_hardwares: [amdproduction]
-  grade: Blocking
-  commands:
-    - export GPU_ARCHS=gfx942
-    - export VLLM_LOGGING_LEVEL=DEBUG
-    - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-    - pytest -s -v tests/e2e/online_serving/test_image_gen_edit.py

.buildkite/test-merge.yml

@@ -1,308 +0,0 @@
-steps:
-  - label: "Simple Unit Test"
-    depends_on: upload-merge-pipeline
-    commands:
-      - "pytest -v -s -m 'core_model and cpu' --cov=vllm_omni --cov-branch --cov-report=term-missing --cov-report=html --cov-report=xml"
-    agents:
-      queue: "gpu_1_queue"
-    plugins:
-      - docker#v5.2.0:
-          image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-          always-pull: true
-          propagate-environment: true
-          environment:
-            - "HF_HOME=/fsx/hf_cache"
-          volumes:
-            - "/fsx/hf_cache:/fsx/hf_cache"
-
-  - label: "Diffusion Model Test"
-    timeout_in_minutes: 20
-    depends_on: upload-merge-pipeline
-    commands:
-      - pytest -s -v tests/e2e/offline_inference/test_t2i_model.py -m "advanced_model and diffusion" --run-level "advanced_model"
-    agents:
-      queue: "gpu_1_queue" # g6.4xlarge instance on AWS, has 1 L4 GPU
-    plugins:
-      - docker#v5.2.0:
-          image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-          always-pull: true
-          propagate-environment: true
-          environment:
-            - "HF_HOME=/fsx/hf_cache"
-          volumes:
-            - "/fsx/hf_cache:/fsx/hf_cache"
-
-  - label: "Diffusion Images API LoRA E2E"
-    timeout_in_minutes: 20
-    depends_on: upload-merge-pipeline
-    commands:
-      - pytest -s -v tests/e2e/online_serving/test_images_generations_lora.py
-    agents:
-      queue: "gpu_1_queue" # g6.4xlarge instance on AWS, has 1 L4 GPU
-    plugins:
-      - docker#v5.2.0:
-          image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-          always-pull: true
-          propagate-environment: true
-          environment:
-            - "HF_HOME=/fsx/hf_cache"
-          volumes:
-            - "/fsx/hf_cache:/fsx/hf_cache"
-
-  - label: "Diffusion Model CPU offloading Test"
-    timeout_in_minutes: 20
-    depends_on: upload-merge-pipeline
-    commands:
-      - pytest -s -v tests/e2e/offline_inference/test_diffusion_cpu_offload.py
-      - pytest -s -v tests/e2e/offline_inference/test_diffusion_layerwise_offload.py
-    agents:
-      queue: "gpu_1_queue" # g6.4xlarge instance on AWS, has 1 L4 GPU
-    plugins:
-      - docker#v5.2.0:
-          image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-          always-pull: true
-          propagate-environment: true
-          environment:
-            - "HF_HOME=/fsx/hf_cache"
-          volumes:
-            - "/fsx/hf_cache:/fsx/hf_cache"
-
-  - label: "Audio Generation Model Test"
-    timeout_in_minutes: 20
-    depends_on: upload-merge-pipeline
-    commands:
-      - pytest -s -v tests/e2e/offline_inference/test_stable_audio_model.py
-    agents:
-      queue: "gpu_1_queue" # g6.4xlarge instance on AWS, has 1 L4 GPU
-    plugins:
-      - docker#v5.2.0:
-          image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-          always-pull: true
-          propagate-environment: true
-          environment:
-            - "HF_HOME=/fsx/hf_cache"
-          volumes:
-            - "/fsx/hf_cache:/fsx/hf_cache"
-
-  - label: "Diffusion Cache Backend Test"
-    timeout_in_minutes: 15
-    depends_on: upload-merge-pipeline
-    commands:
-      - pytest -s -v -m 'core_model and cache and diffusion and not distributed_cuda and L4'
-    agents:
-      queue: "gpu_1_queue" # g6.4xlarge instance on AWS, has 1 L4 GPU
-    plugins:
-      - docker#v5.2.0:
-          image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-          always-pull: true
-          propagate-environment: true
-          environment:
-            - "HF_HOME=/fsx/hf_cache"
-          volumes:
-            - "/fsx/hf_cache:/fsx/hf_cache"
-
-  - label: "Diffusion Sequence Parallelism Test"
-    timeout_in_minutes: 20
-    depends_on: upload-merge-pipeline
-    commands:
-      - pytest -s -v tests/e2e/offline_inference/test_sequence_parallel.py
-    agents:
-      queue: "gpu_4_queue" # g6.12xlarge instance on AWS, has 4 L4 GPU
-    plugins:
-      - docker#v5.2.0:
-          image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-          always-pull: true
-          propagate-environment: true
-          shm-size: "8gb"
-          environment:
-            - "HF_HOME=/fsx/hf_cache"
-          volumes:
-            - "/fsx/hf_cache:/fsx/hf_cache"
-
-  - label: "Diffusion Tensor Parallelism Test"
-    timeout_in_minutes: 20
-    depends_on: upload-merge-pipeline
-    commands:
-      - pytest -s -v tests/e2e/offline_inference/test_zimage_parallelism.py
-    agents:
-      queue: "gpu_4_queue" # g6.12xlarge instance on AWS, has 4 L4 GPU
-    plugins:
-      - docker#v5.2.0:
-          image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-          always-pull: true
-          propagate-environment: true
-          shm-size: "8gb"
-          environment:
-            - "HF_HOME=/fsx/hf_cache"
-          volumes:
-            - "/fsx/hf_cache:/fsx/hf_cache"
-
-  - label: "Diffusion GPU Worker Test"
-    timeout_in_minutes: 20
-    depends_on: upload-merge-pipeline
-    commands:
-      - pytest -s -v tests/diffusion/test_diffusion_worker.py
-    agents:
-      queue: "gpu_4_queue" # g6.12xlarge instance on AWS, has 4 L4 GPU
-    plugins:
-      - docker#v5.2.0:
-          image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-          always-pull: true
-          propagate-environment: true
-          shm-size: "8gb"
-          environment:
-            - "HF_HOME=/fsx/hf_cache"
-          volumes:
-            - "/fsx/hf_cache:/fsx/hf_cache"
-
-  - label: "Benchmark & Engine Test"
-    depends_on: upload-merge-pipeline
-    commands:
-      - |
-        timeout 15m bash -c '
-                export VLLM_WORKER_MULTIPROC_METHOD=spawn
-                set +e
-                pytest -s -v tests/benchmarks/test_serve_cli.py
-                EXIT1=$$?
-                pytest -s -v tests/engine/test_async_omni_engine_abort.py
-                EXIT2=$$?
-                exit $$((EXIT1 | EXIT2))
-        '
-    agents:
-      queue: "gpu_4_queue" # g6.12xlarge instance on AWS, has 4 L4 GPU
-    plugins:
-      - docker#v5.2.0:
-          image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-          always-pull: true
-          propagate-environment: true
-          shm-size: "8gb"
-          environment:
-            - "HF_HOME=/fsx/hf_cache"
-          volumes:
-            - "/fsx/hf_cache:/fsx/hf_cache"
-
-  - label: "Omni Model Test"
-    timeout_in_minutes: 15
-    depends_on: upload-merge-pipeline
-    commands:
-      - export VLLM_LOGGING_LEVEL=DEBUG
-      - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-      - pytest -s -v tests/e2e/offline_inference/test_qwen2_5_omni.py
-    agents:
-      queue: "gpu_4_queue" # g6.12xlarge instance on AWS, has 4 L4 GPU
-    plugins:
-      - docker#v5.2.0:
-          image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-          always-pull: true
-          propagate-environment: true
-          environment:
-            - "HF_HOME=/fsx/hf_cache"
-          volumes:
-            - "/fsx/hf_cache:/fsx/hf_cache"
-
-  - label: "Omni Model Test with H100"
-    timeout_in_minutes: 30
-    depends_on: upload-merge-pipeline
-    commands:
-      - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-      - export VLLM_TEST_CLEAN_GPU_MEMORY="1"
-      - pytest -s -v tests/e2e/offline_inference/test_qwen3_omni.py
-      - pytest -s -v tests/e2e/online_serving/test_qwen3_omni.py -m "advanced_model" --run-level "advanced_model"
-    agents:
-      queue: "mithril-h100-pool"
-    plugins:
-      - kubernetes:
-          podSpec:
-            containers:
-              - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-                resources:
-                  limits:
-                    nvidia.com/gpu: 2
-                volumeMounts:
-                  - name: devshm
-                    mountPath: /dev/shm
-                  - name: hf-cache
-                    mountPath: /root/.cache/huggingface
-                env:
-                  - name: HF_HOME
-                    value: /root/.cache/huggingface
-            nodeSelector:
-              node.kubernetes.io/instance-type: gpu-h100-sxm
-            volumes:
-              - name: devshm
-                emptyDir:
-                  medium: Memory
-              - name: hf-cache
-                hostPath:
-                  path: /mnt/hf-cache
-                  type: DirectoryOrCreate
-
-  - label: "Diffusion Image Edit Test with H100 (1 GPU)"
-    timeout_in_minutes: 20
-    depends_on: upload-merge-pipeline
-    commands:
-      - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-      - pytest -s -v tests/e2e/online_serving/test_image_gen_edit.py
-    agents:
-      queue: "mithril-h100-pool"
-    plugins:
-      - kubernetes:
-          podSpec:
-            containers:
-              - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-                resources:
-                  limits:
-                    nvidia.com/gpu: 1
-                volumeMounts:
-                  - name: devshm
-                    mountPath: /dev/shm
-                  - name: hf-cache
-                    mountPath: /root/.cache/huggingface
-                env:
-                  - name: HF_HOME
-                    value: /root/.cache/huggingface
-            nodeSelector:
-              node.kubernetes.io/instance-type: gpu-h100-sxm
-            volumes:
-              - name: devshm
-                emptyDir:
-                  medium: Memory
-              - name: hf-cache
-                hostPath:
-                  path: /mnt/hf-cache
-                  type: DirectoryOrCreate
-
-  # - label: "Bagel Text2Img Model Test with H100"
-  #   timeout_in_minutes: 30
-  #   depends_on: upload-merge-pipeline
-  #   commands:
-  #     - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-  #     - pytest -s -v tests/e2e/offline_inference/test_bagel_text2img.py
-  #   agents:
-  #     queue: "mithril-h100-pool"
-  #   plugins:
-  #     - kubernetes:
-  #         podSpec:
-  #           containers:
-  #             - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-  #               resources:
-  #                 limits:
-  #                   nvidia.com/gpu: 1
-  #               volumeMounts:
-  #                 - name: devshm
-  #                   mountPath: /dev/shm
-  #                 - name: hf-cache
-  #                   mountPath: /root/.cache/huggingface
-  #               env:
-  #                 - name: HF_HOME
-  #                   value: /root/.cache/huggingface
-  #           nodeSelector:
-  #             node.kubernetes.io/instance-type: gpu-h100-sxm
-  #           volumes:
-  #             - name: devshm
-  #               emptyDir:
-  #                 medium: Memory
-  #             - name: hf-cache
-  #               hostPath:
-  #                 path: /mnt/hf-cache
-  #                 type: DirectoryOrCreate

.buildkite/test-nightly.yml

@@ -1,98 +0,0 @@
-steps:
-  - label: "Omni Model Test with H100"
-    timeout_in_minutes: 90
-    depends_on: upload-nightly-pipeline
-    if: build.env("NIGHTLY") == "1"
-    commands:
-      - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-      - |
-        set +e
-        pytest -s -v tests/e2e/online_serving/test_qwen3_omni_expansion.py -m "advanced_model" --run-level "advanced_model"
-        EXIT1=$$?
-        pytest -s -v tests/examples/online_serving/test_qwen3_omni.py -m "advanced_model" --run-level "advanced_model"
-        EXIT2=$$?
-        exit $$((EXIT1 | EXIT2))
-    agents:
-      queue: "mithril-h100-pool"
-    plugins:
-      - kubernetes:
-          podSpec:
-            containers:
-              - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-                resources:
-                  limits:
-                    nvidia.com/gpu: 2
-                volumeMounts:
-                  - name: devshm
-                    mountPath: /dev/shm
-                  - name: hf-cache
-                    mountPath: /root/.cache/huggingface
-                env:
-                  - name: HF_HOME
-                    value: /root/.cache/huggingface
-            nodeSelector:
-              node.kubernetes.io/instance-type: gpu-h100-sxm
-            volumes:
-              - name: devshm
-                emptyDir:
-                  medium: Memory
-              - name: hf-cache
-                hostPath:
-                  path: /mnt/hf-cache
-                  type: DirectoryOrCreate
-
-  - label: "Omni Model Test"
-    timeout_in_minutes: 60
-    depends_on: upload-nightly-pipeline
-    if: build.env("NIGHTLY") == "1"
-    commands:
-      - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-      - pytest -s -v tests/examples/online_serving/test_qwen2_5_omni.py -m "advanced_model" --run-level "advanced_model"
-    agents:
-      queue: "gpu_4_queue" # g6.12xlarge instance on AWS, has 4 L4 GPU
-    plugins:
-      - docker#v5.2.0:
-          image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-          always-pull: true
-          shm-size: "8gb"
-          propagate-environment: true
-          environment:
-            - "HF_HOME=/fsx/hf_cache"
-          volumes:
-            - "/fsx/hf_cache:/fsx/hf_cache"
-
-  - label: "Omni Model Perf Test"
-    timeout_in_minutes: 120
-    depends_on: upload-nightly-pipeline
-    if: build.env("NIGHTLY") == "1"
-    commands:
-      - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-      - pytest -s -v tests/perf/scripts/run_benchmark.py
-    agents:
-      queue: "mithril-h100-pool"
-    plugins:
-      - kubernetes:
-          podSpec:
-            containers:
-              - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-                resources:
-                  limits:
-                    nvidia.com/gpu: 2
-                volumeMounts:
-                  - name: devshm
-                    mountPath: /dev/shm
-                  - name: hf-cache
-                    mountPath: /root/.cache/huggingface
-                env:
-                  - name: HF_HOME
-                    value: /root/.cache/huggingface
-            nodeSelector:
-              node.kubernetes.io/instance-type: gpu-h100-sxm
-            volumes:
-              - name: devshm
-                emptyDir:
-                  medium: Memory
-              - name: hf-cache
-                hostPath:
-                  path: /mnt/hf-cache
-                  type: DirectoryOrCreate

.buildkite/test-ready.yml

@@ -1,306 +0,0 @@
-steps:
-  - label: "Simple Unit Test"
-    depends_on: upload-ready-pipeline
-    commands:
-      - "timeout 20m pytest -v -s -m 'core_model and cpu' --cov=vllm_omni --cov-branch --cov-report=term-missing --cov-report=html --cov-report=xml"
-    agents:
-      queue: "gpu_1_queue"
-    plugins:
-      - docker#v5.2.0:
-          image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-          always-pull: true
-          propagate-environment: true
-          environment:
-            - "HF_HOME=/fsx/hf_cache"
-          volumes:
-            - "/fsx/hf_cache:/fsx/hf_cache"
-
-  - label: "Diffusion Model Test"
-    depends_on: upload-ready-pipeline
-    commands:
-      - timeout 20m pytest -s -v tests/e2e/offline_inference/test_t2i_model.py -m "core_model and diffusion" --run-level "core_model"
-    agents:
-      queue: "gpu_1_queue"
-    plugins:
-      - docker#v5.2.0:
-          image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-          always-pull: true
-          propagate-environment: true
-          environment:
-            - "HF_HOME=/fsx/hf_cache"
-          volumes:
-            - "/fsx/hf_cache:/fsx/hf_cache"
-
-  - label: "Diffusion Model CPU offloading Test"
-    depends_on: upload-ready-pipeline
-    commands:
-      - |
-        timeout 20m bash -c '
-          set +e
-          pytest -s -v tests/e2e/offline_inference/test_diffusion_cpu_offload.py
-          EXIT1=$$?
-          pytest -s -v tests/e2e/offline_inference/test_diffusion_layerwise_offload.py
-          EXIT2=$$?
-          exit $$((EXIT1 | EXIT2))
-        '
-    agents:
-      queue: "gpu_1_queue" # g6.4xlarge instance on AWS, has 1 L4 GPU
-    plugins:
-      - docker#v5.2.0:
-          image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-          always-pull: true
-          propagate-environment: true
-          environment:
-            - "HF_HOME=/fsx/hf_cache"
-          volumes:
-            - "/fsx/hf_cache:/fsx/hf_cache"
-
-  - label: "Audio Generation Model Test"
-    depends_on: upload-ready-pipeline
-    commands:
-      - timeout 20m pytest -s -v tests/e2e/offline_inference/test_stable_audio_model.py
-    agents:
-      queue: "gpu_1_queue" # g6.4xlarge instance on AWS, has 1 L4 GPU
-    plugins:
-      - docker#v5.2.0:
-          image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-          always-pull: true
-          propagate-environment: true
-          environment:
-            - "HF_HOME=/fsx/hf_cache"
-          volumes:
-            - "/fsx/hf_cache:/fsx/hf_cache"
-
-  - label: "Diffusion Cache Backend Test"
-    depends_on: upload-ready-pipeline
-    commands:
-      - timeout 15m pytest -s -v -m 'core_model and cache and diffusion and not distributed_cuda and L4'
-    agents:
-      queue: "gpu_1_queue" # g6.4xlarge instance on AWS, has 1 L4 GPU
-    plugins:
-      - docker#v5.2.0:
-          image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-          always-pull: true
-          propagate-environment: true
-          environment:
-            - "HF_HOME=/fsx/hf_cache"
-          volumes:
-            - "/fsx/hf_cache:/fsx/hf_cache"
-
-  - label: "Diffusion Sequence Parallelism Test"
-    depends_on: upload-ready-pipeline
-    commands:
-      - timeout 20m pytest -s -v tests/e2e/offline_inference/test_sequence_parallel.py -m core_model
-    agents:
-      queue: "gpu_4_queue" # g6.12xlarge instance on AWS, has 4 L4 GPU
-    plugins:
-      - docker#v5.2.0:
-          image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-          always-pull: true
-          propagate-environment: true
-          shm-size: "8gb"
-          environment:
-            - "HF_HOME=/fsx/hf_cache"
-          volumes:
-            - "/fsx/hf_cache:/fsx/hf_cache"
-
-  - label: "Diffusion GPU Worker Test"
-    depends_on: upload-ready-pipeline
-    commands:
-      - timeout 20m pytest -s -v tests/diffusion/test_diffusion_worker.py
-    agents:
-      queue: "gpu_4_queue" # g6.12xlarge instance on AWS, has 4 L4 GPU
-    plugins:
-      - docker#v5.2.0:
-          image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-          always-pull: true
-          propagate-environment: true
-          shm-size: "8gb"
-          environment:
-            - "HF_HOME=/fsx/hf_cache"
-          volumes:
-            - "/fsx/hf_cache:/fsx/hf_cache"
-
-
-  - label: "Benchmark & Engine Test"
-    depends_on: upload-ready-pipeline
-    commands:
-      - |
-        timeout 15m bash -c '
-                export VLLM_WORKER_MULTIPROC_METHOD=spawn
-                set +e
-                pytest -s -v tests/benchmarks/test_serve_cli.py
-                EXIT1=$$?
-                pytest -s -v tests/engine/test_async_omni_engine_abort.py
-                EXIT2=$$?
-                exit $$((EXIT1 | EXIT2))
-        '
-    agents:
-      queue: "gpu_4_queue" # g6.12xlarge instance on AWS, has 4 L4 GPU
-    plugins:
-      - docker#v5.2.0:
-          image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-          always-pull: true
-          propagate-environment: true
-          shm-size: "8gb"
-          environment:
-            - "HF_HOME=/fsx/hf_cache"
-          volumes:
-            - "/fsx/hf_cache:/fsx/hf_cache"
-
-
-  - label: "Omni Model Test"
-    depends_on: upload-ready-pipeline
-    commands:
-      - |
-        timeout 17m bash -c '
-          export VLLM_LOGGING_LEVEL=DEBUG
-          export VLLM_WORKER_MULTIPROC_METHOD=spawn
-          pytest -s -v tests/e2e/offline_inference/test_qwen2_5_omni.py
-        '
-    agents:
-      queue: "gpu_4_queue" # g6.12xlarge instance on AWS, has 4 L4 GPU
-    plugins:
-      - docker#v5.2.0:
-          image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-          always-pull: true
-          propagate-environment: true
-          environment:
-            - "HF_HOME=/fsx/hf_cache"
-          volumes:
-            - "/fsx/hf_cache:/fsx/hf_cache"
-
-  # - label: "Omni Model Test with H100"
-  #   depends_on: upload-ready-pipeline
-  #   commands:
-  #     - |
-  #       timeout 20m bash -c '
-  #         export VLLM_WORKER_MULTIPROC_METHOD=spawn
-  #         export VLLM_TEST_CLEAN_GPU_MEMORY="1"
-  #         # - pytest -s -v tests/e2e/offline_inference/test_qwen3_omni.py
-  #         pytest -s -v tests/e2e/online_serving/test_qwen3_omni.py -m "core_model" --run-level "core_model"
-  #       '
-  #   agents:
-  #     queue: "mithril-h100-pool"
-  #   plugins:
-  #     - kubernetes:
-  #         podSpec:
-  #           containers:
-  #             - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-  #               resources:
-  #                 limits:
-  #                   nvidia.com/gpu: 2
-  #               volumeMounts:
-  #                 - name: devshm
-  #                   mountPath: /dev/shm
-  #                 - name: hf-cache
-  #                   mountPath: /root/.cache/huggingface
-  #               env:
-  #                 - name: HF_HOME
-  #                   value: /root/.cache/huggingface
-  #           nodeSelector:
-  #             node.kubernetes.io/instance-type: gpu-h100-sxm
-  #           volumes:
-  #             - name: devshm
-  #               emptyDir:
-  #                 medium: Memory
-  #             - name: hf-cache
-  #               hostPath:
-  #                 path: /mnt/hf-cache
-  #                 type: DirectoryOrCreate
-
-  - label: "Qwen3-TTS E2E Test"
-    depends_on: upload-ready-pipeline
-    commands:
-      - |
-        timeout 20m bash -c '
-          export VLLM_LOGGING_LEVEL=DEBUG
-          export VLLM_WORKER_MULTIPROC_METHOD=spawn
-          pytest -s -v tests/e2e/online_serving/test_qwen3_tts.py
-        '
-    agents:
-      queue: "gpu_4_queue"
-    plugins:
-      - docker#v5.2.0:
-          image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-          always-pull: true
-          propagate-environment: true
-          shm-size: "8gb"
-          environment:
-            - "HF_HOME=/fsx/hf_cache"
-          volumes:
-            - "/fsx/hf_cache:/fsx/hf_cache"
-
-  # - label: "Diffusion Image Edit Test with H100 (1 GPU)"
-  #   depends_on: upload-ready-pipeline
-  #   commands:
-  #     - |
-  #       timeout 20m bash -c '
-  #         export VLLM_WORKER_MULTIPROC_METHOD=spawn
-  #         pytest -s -v tests/e2e/online_serving/test_image_gen_edit.py
-  #       '
-  #   agents:
-  #     queue: "mithril-h100-pool"
-  #   plugins:
-  #     - kubernetes:
-  #         podSpec:
-  #           containers:
-  #             - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-  #               resources:
-  #                 limits:
-  #                   nvidia.com/gpu: 1
-  #               volumeMounts:
-  #                 - name: devshm
-  #                   mountPath: /dev/shm
-  #                 - name: hf-cache
-  #                   mountPath: /root/.cache/huggingface
-  #               env:
-  #                 - name: HF_HOME
-  #                   value: /root/.cache/huggingface
-  #           nodeSelector:
-  #             node.kubernetes.io/instance-type: gpu-h100-sxm
-  #           volumes:
-  #             - name: devshm
-  #               emptyDir:
-  #                 medium: Memory
-  #             - name: hf-cache
-  #               hostPath:
-  #                 path: /mnt/hf-cache
-  #                 type: DirectoryOrCreate
-
-  # - label: "Bagel Text2Img Model Test with H100"
-  #   depends_on: upload-ready-pipeline
-  #   commands:
-  #     - |
-  #       timeout 30m bash -c '
-  #         export VLLM_WORKER_MULTIPROC_METHOD=spawn
-  #         pytest -s -v tests/e2e/offline_inference/test_bagel_text2img.py
-  #       '
-  #   agents:
-  #     queue: "mithril-h100-pool"
-  #   plugins:
-  #     - kubernetes:
-  #         podSpec:
-  #           containers:
-  #             - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-  #               resources:
-  #                 limits:
-  #                   nvidia.com/gpu: 1
-  #               volumeMounts:
-  #                 - name: devshm
-  #                   mountPath: /dev/shm
-  #                 - name: hf-cache
-  #                   mountPath: /root/.cache/huggingface
-  #               env:
-  #                 - name: HF_HOME
-  #                   value: /root/.cache/huggingface
-  #           nodeSelector:
-  #             node.kubernetes.io/instance-type: gpu-h100-sxm
-  #           volumes:
-  #             - name: devshm
-  #               emptyDir:
-  #                 medium: Memory
-  #             - name: hf-cache
-  #               hostPath:
-  #                 path: /mnt/hf-cache
-  #                 type: DirectoryOrCreate

.buildkite/test-template-amd-omni.j2

@@ -1,53 +0,0 @@
-{# vllm-omni customized version
-   Based on: https://github.com/vllm-project/ci-infra/blob/main/buildkite/test-template-amd.j2
-   Last synced: 2025-12-15
-   Modifications: Removed unused CUDA/NVIDIA logic, keeping only AMD tests
-#}
-{% set docker_image_amd = "public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT-rocm-omni" %}
-{% set default_working_dir = "/app/vllm-omni" %}
-
-  - group: "AMD Tests"
-    depends_on: ~
-    steps:
-      - label: "AMD: :docker: build image"
-        depends_on: ~
-        soft_fail: false
-        commands:
-          - "source .buildkite/scripts/docker_login_ecr_public.sh && safe_docker_login_ecr_public"
-          - "docker build -f docker/Dockerfile.rocm -t {{ docker_image_amd }} --target test --build-arg ARG_PYTORCH_ROCM_ARCH=gfx942 --progress plain ."
-          - "docker push {{ docker_image_amd }}"
-        key: "amd-build"
-        env:
-          DOCKER_BUILDKIT: "1"
-        retry:
-          automatic:
-            - exit_status: -1  # Agent was lost
-              limit: 1
-            - exit_status: -10  # Agent was lost
-              limit: 1
-            - exit_status: 1  # Machine occasionally fail
-              limit: 1
-        agents:
-          queue: cpu_queue_premerge
-
-    {% for step in steps %}
-    {% if step.mirror_hardwares and mirror_hw in step.mirror_hardwares %}
-      - label: "{{ step.agent_pool }}: {{ step.label }}"
-        depends_on: amd-build
-        agents:
-          {% if step.agent_pool %}
-          queue: amd_{{ step.agent_pool }}
-          {% else %}
-          queue: amd_mi325_1
-          {% endif %}
-        command: bash .buildkite/scripts/hardware_ci/run-amd-test.sh "(command rocm-smi || true) && cd {{ (step.working_dir or default_working_dir) | safe  }} ; {{ step.command  or (step.commands | join(" && ")) | safe }}"
-        env:
-          DOCKER_BUILDKIT: "1"
-        priority: 100
-        {% if step.grade and step.grade == "Blocking" %}
-        soft_fail: false
-        {% else %}
-        soft_fail: true
-        {% endif%}
-    {% endif %}
-    {% endfor %}

.github/ISSUE_TEMPLATE/400-bug-report.yml

@@ -24,7 +24,7 @@ body:
     description: |
       Please run the following and paste the output below.
       ```sh
-      wget https://raw.githubusercontent.com/vllm-project/vllm-omni/main/collect_env.py
+      wget https://raw.githubusercontent.com/vllm-project/vllm-omni/main/vllm_omni/collect_env.py
       # For security purposes, please feel free to check the contents of collect_env.py before running it.
       python collect_env.py
       ```
@@ -40,31 +40,6 @@ body:
       </details>
   validations:
     required: true
-
-- type: textarea
-  attributes:
-    label: Your code version
-    description: |
-      Please provide a description of which commit id or version the code is.
-
-    value: |
-      <details>
-      <summary>The commit id or version of vllm</summary>
-
-      ```text
-
-      ```
-      </details>
-      <details>
-      <summary>The commit id or version of vllm-omni</summary>
-
-      ```text
-
-      ```
-      </details>
-  validations:
-    required: true
-
 - type: textarea
   attributes:
     label: 🐛 Describe the bug

.github/ISSUE_TEMPLATE/750-RFC.yml

@@ -20,8 +20,6 @@ body:
     label: Proposed Change.
     description: >
       The proposed change of the RFC.
-    value: |
-      Please provide the detailed design document of the RFC using the [template](https://docs.google.com/document/d/12YxSsVeD1jvL-InClkeAEnZyWFDndz_65JmXvsamuV4/edit?tab=t.0#heading=h.4ef4szrsgspp).
   validations:
     required: true
 - type: textarea

.github/PULL_REQUEST_TEMPLATE.md

@@ -12,10 +12,10 @@ PLEASE FILL IN THE PR DESCRIPTION HERE ENSURING ALL CHECKLIST ITEMS (AT THE BOTT
 <summary> Essential Elements of an Effective PR Description Checklist </summary>
 
 - [ ] The purpose of the PR, such as "Fix some issue (link existing issues this PR will resolve)".
-- [ ] The test plan. Please provide the test scripts & test commands. Please state the reasons if your codes don't require additional test scripts. For test file guidelines, please check the [test style doc](https://docs.vllm.ai/projects/vllm-omni/en/latest/contributing/ci/tests_style/)
-- [ ] The test results. Please paste the results comparison before and after, or the e2e results.
-- [ ] (Optional) The necessary documentation update, such as updating `supported_models.md` and `examples` for a new model. **Please run `mkdocs serve` to sync the documentation editions to `./docs`.**
-- [ ] (Optional) Release notes update. If your change is user-facing, please update the release notes draft.
+- [ ] The test plan, such as providing test command.
+- [ ] The test results, such as pasting the results comparison before and after, or e2e results
+- [ ] (Optional) The necessary documentation update, such as updating `supported_models.md` and `examples` for a new model.
+- [ ] (Optional) Release notes update. If your change is user facing, please update the release notes draft.
 </details>
 
 **BEFORE SUBMITTING, PLEASE READ <https://github.com/vllm-project/vllm-omni/blob/main/CONTRIBUTING.md>** (anything written below this line will be removed by GitHub Actions)

.github/scripts/pr_reviewer.py

@@ -1,629 +0,0 @@
-#!/usr/bin/env python3
-"""
-PR Reviewer using GLM API for vllm-omni project.
-"""
-
-import json
-import logging
-import os
-import sys
-import time
-from dataclasses import dataclass
-from typing import Any, TypedDict
-
-import requests
-
-
-# Type definitions for API responses
-class PRDetails(TypedDict):
-    """Type definition for GitHub PR details response."""
-
-    title: str
-    body: str
-    number: int
-    state: str
-    user: dict[str, Any]
-
-
-class GLMMessage(TypedDict):
-    """Type definition for GLM API message."""
-
-    role: str
-    content: str
-
-
-class GLMChoice(TypedDict):
-    """Type definition for GLM API choice."""
-
-    message: GLMMessage
-    finish_reason: str
-
-
-class GLMResponse(TypedDict):
-    """Type definition for GLM API response."""
-
-    choices: list[GLMChoice]
-    usage: dict[str, int] | None
-
-
-class GitHubComment(TypedDict):
-    """Type definition for GitHub comment."""
-
-    id: int
-    body: str
-    created_at: str
-    user: dict[str, Any]
-
-
-# Configuration
-TRIGGER_PHRASE: str = "@vllm-omni-reviewer"
-DEFAULT_GLM_API_URL: str = "https://open.bigmodel.cn/api/paas/v4/chat/completions"  # noqa: E501
-DEFAULT_GLM_MODEL: str = "glm-5"
-DEFAULT_COOLDOWN_MINUTES: int = 5
-DEFAULT_MAX_RETRIES: int = 3
-DEFAULT_RETRY_DELAY: float = 1.0
-MAX_DIFF_SIZE: int = 100_000  # Maximum diff size in characters
-
-
-@dataclass
-class Config:
-    """Configuration for the PR reviewer."""
-
-    glm_api_url: str
-    glm_model: str
-    cooldown_minutes: int
-    max_retries: int
-    retry_delay: float
-    max_diff_size: int
-
-
-# Setup logging
-logging.basicConfig(
-    level=logging.INFO,
-    format="[PR Reviewer] %(asctime)s - %(levelname)s - %(message)s",
-    datefmt="%Y-%m-%d %H:%M:%S",
-)
-logger: logging.Logger = logging.getLogger(__name__)
-
-
-def get_config() -> Config:
-    """Load configuration from environment variables with defaults."""
-    return Config(
-        glm_api_url=os.getenv("GLM_API_URL", DEFAULT_GLM_API_URL),
-        glm_model=os.getenv("GLM_MODEL", DEFAULT_GLM_MODEL),
-        cooldown_minutes=int(
-            os.getenv(
-                "PR_REVIEWER_COOLDOWN_MINUTES",
-                str(DEFAULT_COOLDOWN_MINUTES),
-            )
-        ),
-        max_retries=int(
-            os.getenv(
-                "PR_REVIEWER_MAX_RETRIES",
-                str(DEFAULT_MAX_RETRIES),
-            )
-        ),
-        retry_delay=float(os.getenv("PR_REVIEWER_RETRY_DELAY", str(DEFAULT_RETRY_DELAY))),
-        max_diff_size=int(os.getenv("PR_REVIEWER_MAX_DIFF_SIZE", str(MAX_DIFF_SIZE))),  # noqa: E501
-    )
-
-
-def get_env_var(name: str) -> str:
-    """
-    Get an environment variable or raise an error.
-
-    Args:
-        name: Name of the environment variable.
-
-    Returns:
-        The value of the environment variable.
-
-    Raises:
-        SystemExit: If the environment variable is not set.
-    """
-    value = os.environ.get(name)
-    if not value:
-        logger.error(f"Environment variable {name} is not set")
-        sys.exit(1)
-    return value
-
-
-def check_trigger(comment_body: str) -> bool:
-    """
-    Check if the comment contains the trigger phrase.
-
-    Args:
-        comment_body: The body of the comment to check.
-
-    Returns:
-        True if the trigger phrase is found, False otherwise.
-    """
-    return TRIGGER_PHRASE in comment_body
-
-
-def fetch_pr_diff(
-    repo_name: str,
-    pr_number: int,
-    token: str,
-    max_size: int = MAX_DIFF_SIZE,
-) -> str | None:
-    """
-    Fetch the diff for a pull request.
-
-    Args:
-        repo_name: The repository name in format "owner/repo".
-        pr_number: The pull request number.
-        token: GitHub authentication token.
-        max_size: Maximum diff size in characters.
-
-    Returns:
-        The diff content as a string, or None if fetching failed.
-        Returns empty string if diff is larger than max_size.
-    """
-    url: str = f"https://api.github.com/repos/{repo_name}/pulls/{pr_number}"
-    headers: dict[str, str] = {
-        "Authorization": f"Bearer {token}",
-        "Accept": "application/vnd.github.v3.diff",
-    }
-
-    logger.info(f"Fetching PR diff from {url}")
-    response = requests.get(url, headers=headers, timeout=30)
-
-    if response.status_code == 200:
-        diff: str = response.text
-        if len(diff) > max_size:
-            logger.warning(
-                f"Diff size ({len(diff)} bytes) exceeds maximum "
-                f"({max_size} bytes), truncating to first "
-                f"{max_size} bytes"
-            )
-            return diff[:max_size] + "\n\n... [Diff truncated due to size] ..."
-        logger.info(f"Successfully fetched diff ({len(diff)} bytes)")
-        return diff
-    else:
-        logger.error(f"Failed to fetch PR diff: {response.status_code}")
-        logger.error(f"Response: {response.text}")
-        return None
-
-
-def fetch_pr_details(
-    repo_name: str,
-    pr_number: int,
-    token: str,
-) -> PRDetails | None:
-    """
-    Fetch PR details including title and description.
-
-    Args:
-        repo_name: The repository name in format "owner/repo".
-        pr_number: The pull request number.
-        token: GitHub authentication token.
-
-    Returns:
-        A dictionary containing PR details, or None if fetching failed.
-    """
-    url: str = f"https://api.github.com/repos/{repo_name}/pulls/{pr_number}"
-    headers: dict[str, str] = {
-        "Authorization": f"Bearer {token}",
-        "Accept": "application/vnd.github.v3+json",
-    }
-
-    logger.info(f"Fetching PR details from {url}")
-    response = requests.get(url, headers=headers, timeout=30)
-
-    if response.status_code == 200:
-        return response.json()
-    else:
-        logger.error(f"Failed to fetch PR details: {response.status_code}")
-        return None
-
-
-def build_review_prompt(pr_title: str, pr_description: str, diff: str) -> str:
-    """
-    Build the prompt for the GLM-4.7 API.
-
-    Args:
-        pr_title: The title of the pull request.
-        pr_description: The description/body of the pull request.
-        diff: The diff content of the pull request.
-
-    Returns:
-        The formatted prompt string for the API.
-    """
-    return f"""You are an expert code reviewer for the VLLM-Omni project. \
-Please review the following pull request:
-
-## Pull Request Details
-**Title:** {pr_title}
-
-**Description:**
-{pr_description if pr_description else "No description provided."}
-
-## Code Changes (Diff)
-{diff}
-
-## Review Guidelines
-
-Please provide a comprehensive code review with the following sections:
-
-### 1. Overview
-- Brief summary of the changes
-- Overall assessment (positive, neutral, or concerns)
-
-### 2. Code Quality
-- Code style and consistency
-- Potential bugs or edge cases
-- Performance considerations
-- Error handling
-
-### 3. Architecture & Design
-- Integration with existing codebase
-- Design patterns and best practices
-- Potential improvements
-
-### 4. Security & Safety
-- Security concerns (if any)
-- Resource management
-- Input validation
-
-### 5. Testing & Documentation
-- Test coverage considerations
-- Documentation completeness
-- Examples and usage clarity
-
-### 6. Specific Suggestions
-- Line-by-line specific feedback (use `file:line` format)
-- Concrete actionable suggestions
-- Code examples for improvements (if applicable)
-
-### 7. Approval Status
-- **LGTM** (Looks Good To Me) if the PR is ready to merge
-- **LGTM with suggestions** if the PR is good but has minor suggestions
-- **Changes requested** if significant changes are needed
-
-## Important Notes
-- Be constructive and helpful
-- Focus on objective technical feedback
-- Acknowledge good practices when you see them
-- Prioritize critical issues over nitpicks
-- If the diff is empty or minimal, acknowledge this and provide
-  any relevant context-specific guidance
-
-Please format your response in Markdown with clear section headers.
-"""
-
-
-def validate_glm_response(data: dict[str, Any]) -> str | None:
-    """
-    Validate and extract content from GLM API response.
-
-    Args:
-        data: The response data from GLM API.
-
-    Returns:
-        The review content string if valid, None otherwise.
-    """
-    # Check if choices exists and is a non-empty list
-    if "choices" not in data:
-        logger.error("GLM API response missing 'choices' field")
-        logger.error(f"Response structure: {json.dumps(data, indent=2)}")
-        return None
-
-    choices = data["choices"]
-    if not isinstance(choices, list):
-        logger.error(f"GLM API 'choices' is not a list: {type(choices)}")
-        return None
-
-    if len(choices) == 0:
-        logger.error("GLM API 'choices' is an empty list")
-        return None
-
-    # Check if first choice has message
-    try:
-        first_choice = choices[0]
-        if not isinstance(first_choice, dict):
-            logger.error(f"GLM API choice is not a dict: {type(first_choice)}")
-            return None
-
-        if "message" not in first_choice:
-            logger.error("GLM API choice missing 'message' field")
-            logger.error(f"Choice structure: {json.dumps(first_choice, indent=2)}")  # noqa: E501
-            return None
-
-        message = first_choice["message"]
-        if not isinstance(message, dict):
-            logger.error(f"GLM API message is not a dict: {type(message)}")
-            return None
-
-        if "content" not in message:
-            logger.error("GLM API message missing 'content' field")
-            logger.error(f"Message structure: {json.dumps(message, indent=2)}")
-            return None
-
-        content = message["content"]
-        if not isinstance(content, str):
-            logger.error(f"GLM API content is not a string: {type(content)}")
-            return None
-
-        return content
-
-    except (KeyError, IndexError, TypeError) as e:
-        logger.error(f"Failed to parse GLM API response: {e}")
-        logger.error(f"Response: {json.dumps(data, indent=2)}")
-        return None
-
-
-def call_glm_api(prompt: str, api_key: str, config: Config) -> str | None:
-    """
-    Call the GLM-4.7 API to get code review with retry logic.
-
-    Args:
-        prompt: The prompt to send to the API.
-        api_key: The GLM API key.
-        config: Configuration object.
-
-    Returns:
-        The review content as a string, or None if all retries failed.
-    """
-    headers: dict[str, str] = {
-        "Authorization": f"Bearer {api_key}",
-        "Content-Type": "application/json",
-    }
-
-    payload: dict[str, Any] = {
-        "model": config.glm_model,
-        "messages": [{"role": "user", "content": prompt}],
-        "temperature": 0.3,
-        "max_tokens": 32000,
-        "top_p": 0.9,
-    }
-
-    last_error: str | None = None
-
-    for attempt in range(config.max_retries):
-        try:
-            logger.info(f"Calling GLM API ({config.glm_model}) - Attempt {attempt + 1}/{config.max_retries}")
-            response = requests.post(
-                config.glm_api_url,
-                headers=headers,
-                json=payload,
-                timeout=120,
-            )
-
-            if response.status_code == 200:
-                data = response.json()
-                review = validate_glm_response(data)
-                if review:
-                    logger.info(f"Successfully received review ({len(review)} chars)")  # noqa: E501
-                    return review
-                else:
-                    last_error = "Failed to validate API response structure"
-                    logger.error(last_error)
-            else:
-                last_error = f"GLM API request failed: {response.status_code} - {response.text}"
-                logger.error(last_error)
-
-        except requests.exceptions.Timeout:
-            last_error = f"GLM API request timed out (attempt {attempt + 1})"
-            logger.error(last_error)
-        except requests.exceptions.RequestException as e:
-            last_error = f"GLM API request exception: {e}"
-            logger.error(last_error)
-        except json.JSONDecodeError as e:
-            last_error = f"Failed to decode GLM API response as JSON: {e}"
-            logger.error(last_error)
-
-        # Exponential backoff before retry
-        if attempt < config.max_retries - 1:
-            wait_time: float = config.retry_delay * (2**attempt)
-            logger.info(f"Waiting {wait_time}s before retry...")  # noqa: E501
-            time.sleep(wait_time)
-
-    logger.error(
-        f"All {config.max_retries} attempts failed. Last error: {last_error}"  # noqa: E501
-    )
-    return None
-
-
-def check_cooldown(  # noqa: E501
-    repo_name: str,
-    pr_number: int,
-    token: str,
-    cooldown_minutes: int,
-) -> bool:
-    """
-    Check if the PR is within the cooldown period.
-
-    Args:
-        repo_name: The repository name in format "owner/repo".
-        pr_number: The pull request number.
-        token: GitHub authentication token.
-        cooldown_minutes: Cooldown period in minutes.
-
-    Returns:
-        True if within cooldown period (should skip), False otherwise.
-    """
-    from datetime import datetime, timedelta
-
-    url: str = (
-        f"https://api.github.com/repos/{repo_name}/issues/"
-        f"{pr_number}/comments"  # noqa: E501
-    )
-    headers: dict[str, str] = {
-        "Authorization": f"Bearer {token}",
-        "Accept": "application/vnd.github.v3+json",
-    }
-
-    logger.info(f"Checking cooldown period ({cooldown_minutes} minutes)")
-    response = requests.get(url, headers=headers, timeout=30)
-
-    if response.status_code != 200:
-        logger.warning(f"Failed to check cooldown: {response.status_code}, proceeding with review")
-        return False
-
-    comments: list[dict[str, Any]] = response.json()
-    cutoff_time: datetime = datetime.utcnow() - timedelta(minutes=cooldown_minutes)  # noqa: E501
-
-    for comment in reversed(comments):
-        # Check if this is a bot comment
-        body: str = comment.get("body", "")
-        if "VLLM-Omni PR Review" in body or "PR Reviewer Bot" in body:
-            created_at_str: str = comment.get("created_at", "")
-            try:
-                # Parse GitHub timestamp format
-                created_at = datetime.fromisoformat(created_at_str.replace("Z", "+00:00"))
-                created_at = created_at.replace(tzinfo=None)
-                if created_at > cutoff_time:
-                    logger.info(f"PR is within cooldown period (last review: {created_at_str})")
-                    return True
-            except ValueError:
-                logger.warning(f"Failed to parse comment timestamp: {created_at_str}")  # noqa: E501
-                continue
-
-    logger.info("PR is outside cooldown period, proceeding with review")
-    return False
-
-
-def post_review_comment(  # noqa: E501
-    repo_name: str,
-    pr_number: int,
-    token: str,
-    review: str,
-) -> bool:
-    """
-    Post the review as a comment on the PR.
-
-    Args:
-        repo_name: The repository name in format "owner/repo".
-        pr_number: The pull request number.
-        token: GitHub authentication token.
-        review: The review content to post.
-
-    Returns:
-        True if posting succeeded, False otherwise.
-    """
-    url: str = (
-        f"https://api.github.com/repos/{repo_name}/issues/"
-        f"{pr_number}/comments"  # noqa: E501
-    )
-    headers: dict[str, str] = {
-        "Authorization": f"Bearer {token}",
-        "Accept": "application/vnd.github.v3+json",
-    }
-
-    # Format the review comment
-    comment_body: str = f"""## 🤖 VLLM-Omni PR Review
-
-{review}
-
----
-*This review was generated automatically by the VLLM-Omni PR Reviewer Bot
-using {os.getenv("GLM_MODEL", DEFAULT_GLM_MODEL)}.*
-"""
-
-    payload: dict[str, str] = {"body": comment_body}
-
-    logger.info(f"Posting review comment to PR #{pr_number}")
-    response = requests.post(url, headers=headers, json=payload, timeout=30)
-
-    if response.status_code == 201:
-        logger.info("Successfully posted review comment")
-        return True
-    else:
-        logger.error(f"Failed to post comment: {response.status_code}")
-        logger.error(f"Response: {response.text}")
-        return False
-
-
-def main() -> int:
-    """
-    Main entry point for the PR reviewer bot.
-
-    Returns:
-        0 on success, 1 on error.
-    """
-    logger.info("VLLM-Omni PR Reviewer Bot starting...")
-
-    # Load configuration
-    config: Config = get_config()
-    logger.info(
-        f"Configuration: model={config.glm_model}, "
-        f"cooldown={config.cooldown_minutes}min, "
-        f"max_retries={config.max_retries}"
-    )
-
-    # Get environment variables
-    token: str = get_env_var("GITHUB_TOKEN")
-    api_key: str = get_env_var("GLM_API_KEY")
-    repo_name: str = get_env_var("REPO_NAME")
-    pr_number_str: str = get_env_var("PR_NUMBER")
-    comment_body: str = get_env_var("COMMENT_BODY")
-
-    try:
-        pr_number: int = int(pr_number_str)
-    except ValueError:
-        logger.error(f"Invalid PR number: {pr_number_str}")
-        return 1
-
-    logger.info(f"Repository: {repo_name}")
-    logger.info(f"PR Number: {pr_number}")
-
-    # Check if the comment contains the trigger phrase
-    if not check_trigger(comment_body):
-        logger.info(
-            f"Comment does not contain trigger phrase '{TRIGGER_PHRASE}', exiting"  # noqa: E501
-        )
-        return 0
-
-    logger.info("Trigger phrase detected! Starting review process...")
-
-    # Check cooldown period
-    if check_cooldown(repo_name, pr_number, token, config.cooldown_minutes):
-        logger.info("Skipping review due to cooldown period")
-        return 0
-
-    # Fetch PR details
-    logger.info("Step 1/4: Fetching PR details...")
-    pr_details: PRDetails | None = fetch_pr_details(repo_name, pr_number, token)  # noqa: E501
-    if not pr_details:
-        logger.error("Failed to fetch PR details")
-        return 1
-
-    pr_title: str = pr_details.get("title", "Unknown")
-    pr_description: str = pr_details.get("body", "")
-
-    logger.info(f"PR Title: {pr_title}")
-
-    # Fetch PR diff
-    logger.info("Step 2/4: Fetching PR diff...")
-    diff: str | None = fetch_pr_diff(repo_name, pr_number, token, config.max_diff_size)
-    if diff is None:
-        logger.error("Failed to fetch PR diff")
-        return 1
-
-    if not diff:
-        logger.warning("Warning: Empty diff - this might be a draft PR or no code changes")
-
-    # Build prompt
-    logger.info("Step 3/4: Building review prompt...")
-    prompt: str = build_review_prompt(pr_title, pr_description, diff)
-
-    # Call GLM API
-    logger.info("Step 4/4: Calling GLM API...")
-    review: str | None = call_glm_api(prompt, api_key, config)
-    if not review:
-        logger.error("Failed to get review from GLM API")
-        return 1
-
-    # Post review comment
-    logger.info("Posting review comment...")
-    if not post_review_comment(repo_name, pr_number, token, review):
-        logger.error("Failed to post review comment")
-        return 1
-
-    logger.info("PR review completed successfully!")
-    return 0
-
-
-if __name__ == "__main__":
-    sys.exit(main())

.github/workflows/build_wheel.yml

@@ -1,63 +0,0 @@
-name: Build Wheel
-
-on:
-  push:
-    branches:
-      - main
-    paths-ignore:
-      - 'docs/**'
-      - 'examples/**'
-      - 'apps/**'
-      - '.github/**'
-      - '.buildkite/**'
-      - '**.md'
-      - '*.yml'
-      - '.gitignore'
-      - '.gitattributes'
-      - 'LICENSE'
-  pull_request:
-    branches:
-      - main
-    paths-ignore:
-      - 'docs/**'
-      - 'examples/**'
-      - 'apps/**'
-      - '.github/**'
-      - '.buildkite/**'
-      - '*.yml'
-      - '**.md'
-      - '.gitignore'
-      - '.gitattributes'
-      - 'LICENSE'
-  workflow_dispatch:
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        python-version: ["3.11", "3.12"]
-
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v5
-        with:
-          python-version: ${{ matrix.python-version }}
-
-      - name: Install uv
-        run: |
-          pip install uv
-
-      - name: Build Wheel
-        env:
-          UV_SYSTEM_PYTHON: 1
-        run: |
-          bash scripts/build_wheel.sh --python python
-
-      - name: Upload Wheel Artifact
-        uses: actions/upload-artifact@v4
-        with:
-          name: vllm-omni-wheel-py${{ matrix.python-version }}
-          path: dist/*.whl

.github/workflows/pr-reviewer.yml

@@ -1,62 +0,0 @@
-name: VLLM-Omni PR Reviewer
-
-on:
-  issue_comment:
-    types: [created]
-
-permissions:
-  contents: read
-  pull-requests: write
-  issues: write
-
-jobs:
-  pr-reviewer:
-    name: Review Pull Request
-    runs-on: ubuntu-latest
-    timeout-minutes: 10
-    # Only run when the comment is from a collaborator/owner/member
-    if: |
-      github.event_name == 'issue_comment' &&
-      github.event.issue.pull_request != null &&
-      contains(github.event.comment.body, '@vllm-omni-reviewer') &&
-      (github.event.comment.author_association == 'MEMBER' ||
-       github.event.comment.author_association == 'COLLABORATOR' ||
-       github.event.comment.author_association == 'OWNER')
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4.2.2
-        with:
-          fetch-depth: 0
-
-      - name: Set up Python
-        uses: actions/setup-python@v5.3.0
-        with:
-          python-version: '3.11'
-          cache: 'pip'
-
-      - name: Install dependencies
-        run: |
-          python -m pip install --upgrade pip==24.0
-          pip install requests==2.31.0 pyyaml==6.0.1
-
-      - name: Run PR Reviewer
-        id: reviewer
-        env:
-          GLM_API_KEY: ${{ secrets.GLM_API_KEY }}
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          PR_NUMBER: ${{ github.event.issue.number || github.event.pull_request.number }}
-          COMMENT_BODY: ${{ github.event.comment.body }}
-          REPO_NAME: ${{ github.repository }}
-          PR_REVIEWER_COOLDOWN_MINUTES: 5
-          PR_REVIEWER_MAX_RETRIES: 3
-        run: |
-          python .github/scripts/pr_reviewer.py 2>&1 | tee "/tmp/pr_review_${PR_NUMBER}.log"
-
-      - name: Upload review logs
-        if: always()
-        uses: actions/upload-artifact@v4.4.3
-        with:
-          name: pr-review-logs-${{ github.event.issue.number || github.event.pull_request.number }}
-          path: /tmp/pr_review_*.log
-          retention-days: 7
-          if-no-files-found: ignore

.gitignore

@@ -153,14 +153,6 @@ dmypy.json
 # Cython debug symbols
 cython_debug/
 
-# Claude
-CLAUDE.md
-.claude/
-
-# Codex
-AGENTS.md
-.codex/
-
 # cursor
 .cursor/
 
@@ -222,7 +214,6 @@ data/
 datasets/
 *.csv
 *.json
-!apps/ComfyUI-vLLM-Omni/example_workflows/*.json
 *.jsonl
 *.parquet
 
@@ -230,7 +221,6 @@ datasets/
 outputs/
 results/
 generated/
-output_*/
 
 # Configuration overrides
 configs/local.yaml
@@ -242,8 +232,3 @@ configs/development.yaml
 Dockerfile.dev
 discussion
 tmp_test
-
-# Auto-generated version file (created by setuptools_scm during build)
-vllm_omni/_version.py
-# output files
-*.wav

.pre-commit-config.yaml

@@ -3,7 +3,7 @@ default_install_hook_types:
   - commit-msg
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v6.0.0
+    rev: v4.6.0
     hooks:
       # list of supported hooks: https://pre-commit.com/hooks.html
       - id: check-yaml
@@ -16,22 +16,19 @@ repos:
         args: ["--markdown-linebreak-ext=md"]
 
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.14.10
+    rev: v0.14.0
     hooks:
     - id: ruff-check
       args: [--output-format, github, --fix]
     - id: ruff-format
 
   - repo: https://github.com/crate-ci/typos
-    rev: typos-dict-v0.13.13
+    rev: v1.35.5
     hooks:
       - id: typos
         # only for staged files
 
   - repo: https://github.com/rhysd/actionlint
-    # v1.7.8+ sets `go 1.24.0` in go.mod, which older Go toolchains (and most
-    # current CI images) cannot parse. Pin to v1.7.7 until actionlint fixes the
-    # go.mod directive.
     rev: v1.7.7
     hooks:
       - id: actionlint
@@ -61,10 +58,3 @@ repos:
       verbose: true
       pass_filenames: false
     # Insert new entries above the `suggestion` entry
-
-    - id: check-pickle-imports
-      name: Prevent new pickle/cloudpickle imports
-      entry: python tools/pre_commit/check_pickle_imports.py
-      language: python
-      types: [python]
-      additional_dependencies: [regex]

README.md

@@ -9,16 +9,13 @@ Easy, fast, and cheap omni-modality model serving for everyone
 </h3>
 
 <p align="center">
-| <a href="https://vllm-omni.readthedocs.io/en/latest/"><b>Documentation</b></a> | <a href="https://discuss.vllm.ai"><b>User Forum</b></a> | <a href="https://slack.vllm.ai"><b>Developer Slack</b></a> | <a href="docs/assets/WeChat.jpg"><b>WeChat</b></a> | <a href="https://arxiv.org/abs/2602.02204"><b>Paper</b></a> | <a href="https://docs.google.com/presentation/d/1qv4qMW1rKAqDREMXiUDLIgqqHQe7TDPj/edit?usp=sharing&ouid=110473603432222024453&rtpof=true&sd=true"><b>Slides</b></a> |
+| <a href="https://vllm-omni.readthedocs.io/en/latest/"><b>Documentation</b></a> | <a href="https://discuss.vllm.ai"><b>User Forum</b></a> | <a href="https://slack.vllm.ai"><b>Developer Slack</b></a> |
 </p>
 
-
 ---
 
 *Latest News* 🔥
 
-- [2026/02] We released [0.14.0](https://github.com/vllm-project/vllm-omni/releases/tag/v0.14.0) - This is the first **stable release** of vLLM-Omni that expands Omni’s diffusion / image-video generation and audio / TTS stack, improves distributed execution and memory efficiency, and broadens platform/backend coverage (GPU/ROCm/NPU/XPU). It also brings meaningful upgrades to serving APIs, profiling & benchmarking, and overall stability. Please check our latest [paper](https://arxiv.org/abs/2602.02204) for architecture design and performance results.
-- [2026/01] We released [0.12.0rc1](https://github.com/vllm-project/vllm-omni/releases/tag/v0.12.0rc1) - a major RC milestone focused on maturing the diffusion stack, strengthening OpenAI-compatible serving, expanding omni-model coverage, and improving stability across platforms (GPU/NPU/ROCm).
 - [2025/11] vLLM community officially released [vllm-project/vllm-omni](https://github.com/vllm-project/vllm-omni) in order to support omni-modality models serving.
 
 ---
@@ -69,26 +66,9 @@ Visit our [documentation](https://vllm-omni.readthedocs.io/en/latest/) to learn
 We welcome and value any contributions and collaborations.
 Please check out [Contributing to vLLM-Omni](https://vllm-omni.readthedocs.io/en/latest/contributing/) for how to get involved.
 
-## Citation
-
-If you use vLLM-Omni for your research, please cite our [paper](https://arxiv.org/abs/2602.02204):
-
-```bibtex
-@article{yin2026vllmomni,
-  title={vLLM-Omni: Fully Disaggregated Serving for Any-to-Any Multimodal Models},
-  author={Peiqi Yin, Jiangyun Zhu, Han Gao, Chenguang Zheng, Yongxiang Huang, Taichang Zhou, Ruirui Yang, Weizhi Liu, Weiqing Chen, Canlin Guo, Didan Deng, Zifeng Mo, Cong Wang, James Cheng, Roger Wang, Hongsheng Liu},
-  journal={arXiv preprint arXiv:2602.02204},
-  year={2026}
-}
-```
-
 ## Join the Community
 Feel free to ask questions, provide feedbacks and discuss with fellow users of vLLM-Omni in `#sig-omni` slack channel at [slack.vllm.ai](https://slack.vllm.ai) or vLLM user forum at [discuss.vllm.ai](https://discuss.vllm.ai).
 
-## Star History
-
-[![Star History Chart](https://api.star-history.com/svg?repos=vllm-project/vllm-omni&type=date&legend=top-left)](https://www.star-history.com/#vllm-project/vllm-omni&type=date&legend=top-left)
-
 ## License
 
 Apache License 2.0, as found in the [LICENSE](./LICENSE) file.

apps/ComfyUI-vLLM-Omni/.gitignore

@@ -1,115 +0,0 @@
-# Byte-compiled / optimized / DLL files
-__pycache__/
-*.py[cod]
-*$py.class
-
-# OSX useful to ignore
-*.DS_Store
-.AppleDouble
-.LSOverride
-
-# Thumbnails
-._*
-
-# Files that might appear in the root of a volume
-.DocumentRevisions-V100
-.fseventsd
-.Spotlight-V100
-.TemporaryItems
-.Trashes
-.VolumeIcon.icns
-.com.apple.timemachine.donotpresent
-
-# Directories potentially created on remote AFP share
-.AppleDB
-.AppleDesktop
-Network Trash Folder
-Temporary Items
-.apdisk
-
-# C extensions
-*.so
-
-# Distribution / packaging
-.Python
-env/
-venv/
-build/
-develop-eggs/
-dist/
-downloads/
-eggs/
-.eggs/
-lib/
-lib64/
-parts/
-sdist/
-var/
-*.egg-info/
-.installed.cfg
-*.egg
-
-# PyInstaller
-#  Usually these files are written by a python script from a template
-#  before PyInstaller builds the exe, so as to inject date/other infos into it.
-*.manifest
-*.spec
-
-# Installer logs
-pip-log.txt
-pip-delete-this-directory.txt
-
-# Unit test / coverage reports
-htmlcov/
-.tox/
-.coverage
-.coverage.*
-.cache
-nosetests.xml
-coverage.xml
-*,cover
-.hypothesis/
-.pytest_cache/
-
-# Translations
-*.mo
-*.pot
-
-# Django stuff:
-*.log
-
-# Sphinx documentation
-docs/_build/
-
-# IntelliJ Idea
-.idea
-*.iml
-*.ipr
-*.iws
-
-# PyBuilder
-target/
-
-# Cookiecutter
-output/
-python_boilerplate/
-cookiecutter-pypackage-env/
-
-# vscode settings
-.history/
-*.code-workspace
-
-# Frontend extension
-node_modules/
-.env
-.env.local
-.env.development.local
-.env.test.local
-.env.production.local
-npm-debug.log*
-yarn-debug.log*
-yarn-error.log*
-node.zip
-.vscode/
-.claude/
-.codemate/

apps/ComfyUI-vLLM-Omni/LICENSE

@@ -1,15 +0,0 @@
-Apache Software License 2.0
-
-Copyright (c) 2026, Zeyu Huang
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.

apps/ComfyUI-vLLM-Omni/README.md

@@ -1,169 +0,0 @@
-# vLLM-Omni
-
-vLLM-Omni offers a ComfyUI integration on top of its online serving API.
-It can send model inference requests to either a locally running vLLM-Omni service or a remote one.
-
-## Requirement
-
-- Python 3.12 or above
-- [ComfyUI installed](https://docs.comfy.org/installation/system_requirements)
-- [vLLM-Omni installed](https://docs.vllm.ai/projects/vllm-omni/en/latest/getting_started/installation/) on either the same device or another device discoverable via the internet.
-- No need to install additional packages apart from those already required by ComfyUI.
-
-> [!TIP]
-> If you run both ComfyUI and vLLM-Omni on the same device, you can create separate virtual environments and use different Python versions for them.
-
-
-## Installation
-
-Copy this folder to the `custom_nodes` subfolder of your ComfyUI installation. Your directory should look like `ComfyUI/custom_nodes/ComfyUI-vLLM-Omni`.
-
-If you are running ComfyUI during copying, you should restart ComfyUI to load this extension.
-
-> [!TIP]
-> You can use utility websites such as https://download-directory.github.io/ to download a subdirectory of a repo. Also checkout community discussions (e.g., https://stackoverflow.com/questions/7106012/download-a-single-folder-or-directory-from-a-github-repository) for more info.
-
-On the device and virtual environment you run ComfyUI, launch ComfyUI with
-```bash
-cd ComfyUI
-
-# The regular way
-python main.py
-
-# If you are mainly using this node, launch it faster with
-python main.py --cpu
-```
-
-On the device and virtual environment you run vLLM-Omni, start a model service with
-```bash
-vllm serve The_Model_ID_to_Serve --omni --port 8000
-```
-
-Check **ComfyUI's sidebar -> Node Library**. There should be a new folder named **vLLM-Omni**.
-If no, check your shell running the ComfyUI process. There may be some error messages before the line `Import times for custom nodes:` and the line `To see the GUI go to: http://127.0.0.1:8188`.
-
-## Quickstart
-
-This extension offers the following nodes based on the output modalities (at **ComfyUI sidebar -> Node Library**):
-
-- **Generate Image** for text-to-image and image-to-image tasks
-- **Multimodality Understanding** for multimodality-to-text and multimodality-to-audio tasks
-- **TTS** and **TTS Voice Clone** for TTS tasks
-
-This extension also offers example workflows (at **ComfyUI sidebar -> Templates -> vLLM-Omni**)
-
-> [!NOTE]
-> The node UI and feature designs are intended to match vLLM-Omni online serving interfaces. It cannot offer more than what the interfaces support.
-
-To build a simple workflow yourself,
-
-- Drag a generation node onto the canvas.
-- Depending on your need, grab built-in multimedia file loader nodes, such as **image->Load Image**, **image->video->Load Video**, **audio->Load Audio**
-- Depending on your need, grab built-in multimedia file preview nodes, such as **image->Preview Image**, **image->video->Save Video**, **audio->Preview Audio**. For text output, you can install [ComfyUI-Custom-Scripts plugin](https://github.com/pythongosssss/ComfyUI-Custom-Scripts/) and grab its **utils->Show Text 🐍** node.
-- If you want to tune sampling parameters, grab corresponding nodes from **vLLM-Omni-> Sampling Params**.
-    - For multi-stage models, you can connect multiple **AR Sampling Params** and **Diffusion Sampling Params** nodes to a **Multi-Stage Sampling Params List** node, and connect this node to the generation node.
-    - For some multi-stage models like BAGEL, [only one stage's sampling parameters are exposed and tunable via vLLM-Omni's online serving API](https://docs.vllm.ai/projects/vllm-omni/en/latest/user_guide/examples/online_serving/bagel/). Thus, these models are treated as single-stage ones. Please check the vLLM-Omni documentation on how to correctly set each model's sampling parameters.
-    - For multi-stage models where all stages are either autoregression or diffusion, you can also connect only a single Sampling Params node, indicating that this set of sampling parameters will be used for all stages.
-
-## Screenshots and Examples
-
-### Multimodal understanding (e.g., Qwen Omni series, BAGEL)
-
-(Also available at **ComfyUI sidebar->Template->vLLM-Omni->vLLM-Omni Multimodal Understanding**)
-
-<p align="center">
-  <picture>
-    <source media="(prefers-color-scheme: dark)" src="https://raw.githubusercontent.com/vllm-project/vllm-omni/refs/heads/main/apps/ComfyUI-vLLM-Omni/docs/images/comfyui-understanding.jpg">
-    <img alt="vLLM-Omni multimodal understanding" src="https://raw.githubusercontent.com/vllm-project/vllm-omni/refs/heads/main/apps/ComfyUI-vLLM-Omni/docs/images/comfyui-understanding.jpg" width=55%>
-  </picture>
-</p>
-
-> [!TIP]
-> Although this node enables all-modality input, you should check whether the specific model you host and request for supports the modalities you connect to the node.
-
-You can configure per-stage sampling parameters for multi-stage models.
-
-<p align="center">
-  <picture>
-    <source media="(prefers-color-scheme: dark)" src="https://raw.githubusercontent.com/vllm-project/vllm-omni/refs/heads/main/apps/ComfyUI-vLLM-Omni/docs/images/comfyui-multi-stage.jpg">
-    <img alt="vLLM-Omni multiple stages" src="https://raw.githubusercontent.com/vllm-project/vllm-omni/refs/heads/main/apps/ComfyUI-vLLM-Omni/docs/images/comfyui-multi-stage.jpg" width=55%>
-  </picture>
-</p>
-
-### Text-to-image and image-to-image generation (e.g., Z-Image-Turbo, Qwen-Image-Edit, BAGEL)
-
-(Also available at **ComfyUI sidebar->Template->vLLM-Omni->vLLM-Omni Image Generation**)
-
-<p align="center">
-  <picture>
-    <source media="(prefers-color-scheme: dark)" src="https://raw.githubusercontent.com/vllm-project/vllm-omni/refs/heads/main/apps/ComfyUI-vLLM-Omni/docs/images/comfyui-image-generation.jpg">
-    <img alt="vLLM-Omni image generation" src="https://raw.githubusercontent.com/vllm-project/vllm-omni/refs/heads/main/apps/ComfyUI-vLLM-Omni/docs/images/comfyui-image-generation.jpg" width=55%>
-  </picture>
-</p>
-
-> [!TIP]
-> The node automatically choose text-to-image or image-to-image API endpoints depending on whether you connect an image input or not.
-
-### TTS (e.g., Qwen TTS series)
-
-(Also available at **ComfyUI sidebar->Template->vLLM-Omni->vLLM-Omni TTS**)
-
-<p align="center">
-  <picture>
-    <source media="(prefers-color-scheme: dark)" src="https://raw.githubusercontent.com/vllm-project/vllm-omni/refs/heads/main/apps/ComfyUI-vLLM-Omni/docs/images/comfyui-tts.jpg">
-    <img alt="vLLM-Omni TTS" src="https://raw.githubusercontent.com/vllm-project/vllm-omni/refs/heads/main/apps/ComfyUI-vLLM-Omni/docs/images/comfyui-tts.jpg" width=55%>
-  </picture>
-</p>
-
-> [!TIP]
-> There is a dedicated node for VoiceClone tasks with reference audio input. Other simple text-to-speech tasks should use the regular TTS node.
-
-### Chaining multiple model services
-
-(Also available at **ComfyUI sidebar->Template->vLLM-Omni->vLLM-Omni Chaining Services**)
-
-<p align="center">
-  <picture>
-    <source media="(prefers-color-scheme: dark)" src="https://raw.githubusercontent.com/vllm-project/vllm-omni/refs/heads/main/apps/ComfyUI-vLLM-Omni/docs/images/comfyui-chaining-services.jpg">
-    <img alt="vLLM-Omni TTS" src="https://raw.githubusercontent.com/vllm-project/vllm-omni/refs/heads/main/apps/ComfyUI-vLLM-Omni/docs/images/comfyui-chaining-services.jpg" width=55%>
-  </picture>
-</p>
-
-## Develop
-
-Follow the [development convention and rules of vLLM-Omni](https://docs.vllm.ai/projects/vllm-omni/en/latest/contributing/).
-
-## Limitation and Non-Goals
-
-- Single server mode only. No automatic load balancing or failover.
-- Features set is bounded to vLLM-Omni's online service capability, including
-    - The types of models supported in online mode,
-    - The types of sampling parameters supported in the online mode,
-    - The ways to send files (primarily through full-length base64 in JSON payload),
-    - Figuring out errors in the payload (such as unsupported fields by a specific model) if the endpoint does not explicitly return an error,
-    - (The lack of) Authentication
-    - (The lack of) Progress indicator
-
-## Support
-
-If you are new to ComfyUI, please check out [its documentation](https://docs.comfy.org/) for usage instructions.
-
-If you are new to vLLM-Omni, please also check out [its documentation](https://docs.vllm.ai/projects/vllm-omni/en/latest/) for usage instructions.
-
-Whenever you find an issue or problem, please
-
-- First find out if this is an upstream limitation of vLLM-Omni's online serving mode, by [checking their documentation](https://docs.vllm.ai/projects/vllm-omni/en/latest/examples/).
-- [Open an issue](https://github.com/vllm-project/vllm-omni/issues) that clearly describes this ComfyUI or online service problem.
-
-## Acknowledgements
-
-Features
-
-- https://github.com/dougbtv/comfyui-vllm-omni/ The official reference implementation for ComfyUI integration with vLLM-Omni's DALL-E compatible image generation API.
-- https://github.com/Comfy-Org/ComfyUI/tree/master/comfy_extras ComfyUI's built-in node implementations.
-
-UI/UX design references
-
-- https://github.com/sgl-project/sglang/pull/15271 SGLang Diffusion's official ComfyUI integration for image and video generation.
-- https://github.com/SXQBW/ComfyUI-Qwen-Omni A third party ComfyUI integration for Qwen Omni series.
-- https://github.com/flybirdxx/ComfyUI-Qwen-TTS https://github.com/DarioFT/ComfyUI-Qwen3-TTS Third  party ComfyUI integrations for Qwen TTS series.

apps/ComfyUI-vLLM-Omni/__init__.py

@@ -1,52 +0,0 @@
-"""Top-level package for comfyui_vllm_omni."""  # noqa: N999  # This is not a python library intended to be imported
-
-__all__ = [
-    "NODE_CLASS_MAPPINGS",
-    "NODE_DISPLAY_NAME_MAPPINGS",
-    "WEB_DIRECTORY",
-]
-
-__author__ = """vLLM-Omni Team"""
-__email__ = "vllm-omni@vllm.ai"
-__version__ = "0.0.1"
-
-from .comfyui_vllm_omni.nodes import (
-    VLLMOmniARSampling,
-    VLLMOmniDiffusionSampling,
-    VLLMOmniGenerateImage,
-    VLLMOmniQwenTTSParams,
-    VLLMOmniSamplingParamsList,
-    VLLMOmniTTS,
-    VLLMOmniUnderstanding,
-    VLLMOmniVoiceClone,
-)
-
-# A dictionary that contains all nodes you want to export with their names
-NODE_CLASS_MAPPINGS = {
-    # === Generation ===
-    "VLLMOmniGenerateImage": VLLMOmniGenerateImage,
-    "VLLMOmniUnderstanding": VLLMOmniUnderstanding,
-    "VLLMOmniTTS": VLLMOmniTTS,
-    "VLLMOmniVoiceClone": VLLMOmniVoiceClone,
-    # === Params ===
-    "VLLMOmniARSampling": VLLMOmniARSampling,
-    "VLLMOmniDiffusionSampling": VLLMOmniDiffusionSampling,
-    "VLLMOmniSamplingParamsList": VLLMOmniSamplingParamsList,
-    "VLLMOmniQwenTTSParams": VLLMOmniQwenTTSParams,
-}
-
-# A dictionary that contains the friendly/humanly readable titles for the nodes
-NODE_DISPLAY_NAME_MAPPINGS = {
-    # === Generation ===
-    "VLLMOmniGenerateImage": "Generate Image",
-    "VLLMOmniUnderstanding": "Multimodality Understanding",
-    "VLLMOmniTTS": "TTS (Text to Speech)",
-    "VLLMOmniVoiceClone": "TTS Voice Cloning",
-    # === Params ===
-    "VLLMOmniARSampling": "AR Sampling Params",
-    "VLLMOmniDiffusionSampling": "Diffusion Sampling Params",
-    "VLLMOmniSamplingParamsList": "Multi-Stage Sampling Params List",
-    "VLLMOmniQwenTTSParams": "Qwen TTS Params",
-}
-
-WEB_DIRECTORY = "./web"

apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/__init__.py

@@ -1 +0,0 @@
-# noqa: N999   # This is not a python library intended to be imported

apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/nodes.py

@@ -1,574 +0,0 @@
-from typing import Literal
-
-import torch
-from comfy_api.input import AudioInput, VideoInput
-
-from .utils.api_client import VLLMOmniClient
-from .utils.logger import get_logger
-from .utils.models import lookup_model_spec
-from .utils.types import (
-    AudioFormat,
-    AutoregressionSamplingParams,
-    DiffusionSamplingParams,
-    QwenTTSModelSpecificParams,
-)
-from .utils.validators import (
-    add_sampling_parameters_to_stage,
-    validate_model_and_sampling_params_types,
-)
-
-logger = get_logger(__name__)
-
-
-class _VLLMOmniGenerateBase:
-    """Base class for vLLM-Omni generation nodes with shared functionality."""
-
-    CATEGORY = "vLLM-Omni"
-
-    @classmethod
-    def VALIDATE_INPUTS(cls, url, model) -> str | Literal[True]:
-        """
-        Can only validate this model's own input. Cannot check inputs from other nodes.
-        See: https://docs.comfy.org/custom-nodes/backend/server_overview#validate_inputs
-        """
-        if not url:
-            return "URL must not be empty"
-        if not model:
-            return "Model must not be empty"
-        return True
-
-
-class VLLMOmniGenerateImage(_VLLMOmniGenerateBase):
-    @classmethod
-    def INPUT_TYPES(cls):
-        return {
-            "required": {
-                "url": ("STRING", {"default": "http://localhost:8000/v1"}),
-                "model": ("STRING", {"default": "Tongyi-MAI/Z-Image-Turbo"}),
-                "prompt": ("STRING", {"multiline": True}),
-                "negative_prompt": ("STRING", {"multiline": True, "default": ""}),
-                "width": ("INT", {"default": 512, "min": 64, "max": 2048}),
-                "height": ("INT", {"default": 512, "min": 64, "max": 2048}),
-            },
-            "optional": {
-                "image": ("IMAGE",),
-                "mask": ("MASK",),
-                # "video": ("VIDEO",),
-                # "audio": ("AUDIO",),
-                "sampling_params": ("SAMPLING_PARAMS",),
-            },
-        }
-
-    RETURN_TYPES = ("IMAGE",)
-    RETURN_NAMES = ("image",)
-    FUNCTION = "generate"
-
-    async def generate(
-        self,
-        url: str,
-        model: str,
-        prompt: str,
-        width: int,
-        height: int,
-        negative_prompt: str | None = None,
-        image: torch.Tensor | None = None,
-        mask: torch.Tensor | None = None,
-        audio: AudioInput | None = None,  # Hidden & unused
-        video: VideoInput | None = None,  # Hidden & unused
-        sampling_params: dict | list[dict] | None = None,
-        **kwargs,
-    ):
-        logger.info("Uncaught kwargs: %s", kwargs)
-        logger.debug("Got sampling params: %s", sampling_params)
-        validate_model_and_sampling_params_types(model, sampling_params)
-        if image is None and mask is not None:
-            raise ValueError("Mask input provided without an image input.")
-
-        client = VLLMOmniClient(url)
-
-        spec, pattern = lookup_model_spec(model)
-        is_bagel = pattern is not None and "bagel" in pattern.lower()
-
-        # Prefer DALL-E compatible API for simple (one-stage) diffusion models
-        if (spec is None or spec["stages"] == ["diffusion"]) and not is_bagel:
-            # The number of sampling parameter groups should have been validated.
-            # Now, simply convert single-item list to dict.
-            if isinstance(sampling_params, list):
-                sampling_params = sampling_params[0]
-            if audio is None and image is None and video is None:
-                # No multimodal input --- use DALL-E image generation
-                logger.info("Using DALL-E image generation endpoint")
-                output = await client.generate_image(
-                    model=model,
-                    prompt=prompt,
-                    width=width,
-                    height=height,
-                    negative_prompt=negative_prompt,
-                    sampling_params=sampling_params,
-                )
-                return (output,)
-            elif image is not None and audio is None and video is None:
-                # Image and text input --- use DALL-E image edit
-                logger.info("Using DALL-E image edit endpoint")
-                output = await client.edit_image(
-                    model=model,
-                    prompt=prompt,
-                    image=image,
-                    width=width,
-                    height=height,
-                    negative_prompt=negative_prompt,
-                    mask=mask,
-                    sampling_params=sampling_params,
-                )
-                return (output,)
-
-        logger.info("Using chat completion endpoint")
-        sampling_params = add_sampling_parameters_to_stage(
-            model, sampling_params, "diffusion", width=width, height=height
-        )
-        logger.debug("Edited sampling params: %s", sampling_params)
-
-        output = await client.generate_image_chat_completion(
-            model=model,
-            prompt=prompt,
-            negative_prompt=negative_prompt,
-            image=image,
-            audio=audio,
-            video=video,
-            sampling_params=sampling_params,
-        )
-
-        return (output,)
-
-
-class VLLMOmniUnderstanding(_VLLMOmniGenerateBase):
-    @classmethod
-    def INPUT_TYPES(cls):
-        return {
-            "required": {
-                "url": ("STRING", {"default": "http://localhost:8000/v1"}),
-                "model": ("STRING", {"default": "Qwen/Qwen2.5-Omni-7B"}),
-                "prompt": ("STRING", {"multiline": True}),
-                "output_text": ("BOOLEAN", {"default": True}),
-                "output_audio": ("BOOLEAN", {"default": True}),
-                "use_audio_in_video": ("BOOLEAN", {"default": True}),
-            },
-            "optional": {
-                "image": ("IMAGE",),
-                "video": ("VIDEO",),
-                "audio": ("AUDIO",),
-                "sampling_params": ("SAMPLING_PARAMS",),
-            },
-        }
-
-    RETURN_TYPES = ("STRING", "AUDIO")
-    RETURN_NAMES = ("text_response", "audio_response")
-    FUNCTION = "generate"
-
-    @classmethod
-    def VALIDATE_INPUTS(cls, url, model, output_text, output_audio) -> str | Literal[True]:  # type: ignore[reportIncompatibleMethodOverride]
-        super_validation = super().VALIDATE_INPUTS(url, model)
-        if isinstance(super_validation, str):
-            return super_validation
-        if not output_text and not output_audio:
-            return "At least one of output_text or output_audio must be True."
-        return True
-
-    async def generate(
-        self,
-        url: str,
-        model: str,
-        prompt: str,
-        image: torch.Tensor | None = None,
-        audio: AudioInput | None = None,
-        video: VideoInput | None = None,
-        sampling_params: dict | list[dict] | None = None,
-        output_text: bool = True,
-        output_audio: bool = True,
-        use_audio_in_video: bool = True,
-        **kwargs,
-    ) -> tuple[str, AudioInput]:
-        logger.info("Uncaught kwargs: %s", kwargs)
-        logger.debug("Got sampling params: %s", sampling_params)
-        validate_model_and_sampling_params_types(model, sampling_params)
-
-        client = VLLMOmniClient(url)
-        spec, pattern = lookup_model_spec(model)
-        is_bagel = pattern is not None and "bagel" in pattern.lower()
-
-        if is_bagel:
-            # A lot of special handlings here...
-            if output_audio:
-                raise ValueError("BAGEL models do not support audio output.")
-            if audio is not None or video is not None:
-                raise ValueError("BAGEL models do not support audio or video input.")
-            (
-                text_response,
-                _,
-            ) = await client.generate_understanding_chat_completion(
-                model=model,
-                prompt=prompt,
-                image=image,
-                audio=None,
-                video=None,
-                sampling_params=sampling_params,
-                modalities=["text"],
-            )
-        else:
-            modalities = []
-            if output_text:
-                modalities.append("text")
-            if output_audio:
-                modalities.append("audio")
-
-            if use_audio_in_video and video is not None:
-                use_audio_in_video = True
-            else:
-                use_audio_in_video = False
-
-            (
-                text_response,
-                audio,
-            ) = await client.generate_understanding_chat_completion(
-                model=model,
-                prompt=prompt,
-                image=image,
-                audio=audio,
-                video=video,
-                sampling_params=sampling_params,
-                modalities=modalities,
-                # == extra kwargs ==
-                mm_processor_kwargs={"use_audio_in_video": use_audio_in_video},
-            )
-
-        if text_response is None:
-            text_response = ""
-        if audio is None:
-            channels = 1
-            duration = 1
-            sample_rate = 44100
-            num_samples = int(round(duration * sample_rate))
-            waveform = torch.zeros((1, channels, num_samples), dtype=torch.float32)
-            audio = {"waveform": waveform, "sample_rate": sample_rate}
-
-        return (text_response, audio)
-
-
-class VLLMOmniTTS(_VLLMOmniGenerateBase):
-    @classmethod
-    def INPUT_TYPES(cls):
-        return {
-            "required": {
-                "url": ("STRING", {"default": "http://localhost:8000/v1"}),
-                "model": (
-                    "STRING",
-                    {"default": "Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice"},
-                ),
-                "input": ("STRING", {"multiline": True}),
-                "voice": ("STRING", {"default": "Vivian"}),
-                "response_format": (["mp3", "opus", "aac", "flac", "wav", "pcm"],),
-                "speed": (
-                    "FLOAT",
-                    {"default": 1.0, "min": 0.25, "max": 4.0, "step": 0.01},
-                ),
-            },
-            "optional": {
-                "model_specific_params": ("TTS_PARAMS",),
-            },
-        }
-
-    RETURN_TYPES = ("AUDIO",)
-    RETURN_NAMES = ("audio",)
-    FUNCTION = "generate"
-
-    async def generate(
-        self,
-        url: str,
-        model: str,
-        input: str,
-        voice: str,
-        response_format: AudioFormat,
-        speed: float,
-        model_specific_params: dict | None,
-        **kwargs,
-    ) -> tuple[AudioInput]:
-        logger.info("Got extra kwargs in TTS: %s", kwargs)
-
-        is_qwen_tts = "qwen3-tts" in model.lower()
-        if not is_qwen_tts and isinstance(model_specific_params, QwenTTSModelSpecificParams):
-            raise ValueError(
-                "You have provided Qwen-specific TTS params."
-                "However, the model appears to not be a Qwen TTS model (no 'Qwen3-TTS' in model name)."
-            )
-
-        combined_params = {**kwargs, **(model_specific_params or {})}
-
-        client = VLLMOmniClient(url)
-
-        audio = await client.generate_speech(
-            model=model,
-            input=input,
-            voice=voice,
-            response_format=response_format,
-            speed=speed,
-            **combined_params,
-        )
-        return (audio,)
-
-
-class VLLMOmniVoiceClone(_VLLMOmniGenerateBase):
-    @classmethod
-    def INPUT_TYPES(cls):
-        return {
-            "required": {
-                "url": ("STRING", {"default": "http://localhost:8000/v1"}),
-                "model": ("STRING", {"default": "Qwen/Qwen3-TTS-12Hz-1.7B-Base"}),
-                "input": ("STRING", {"multiline": True}),
-                "voice": ("STRING", {"default": "Vivian"}),
-                "response_format": (["mp3", "opus", "aac", "flac", "wav", "pcm"],),
-                "speed": (
-                    "FLOAT",
-                    {"default": 1.0, "min": 0.25, "max": 4.0, "step": 0.01},
-                ),
-                "ref_audio": ("AUDIO",),
-                "ref_text": ("STRING", {"multiline": True}),
-                "x_vector_only_mode": ("BOOLEAN", {"default": False}),
-            },
-            "optional": {
-                "model_specific_params": ("TTS_PARAMS",),
-            },
-        }
-
-    RETURN_TYPES = ("AUDIO",)
-    RETURN_NAMES = ("audio",)
-    FUNCTION = "generate"
-
-    async def generate(
-        self,
-        url: str,
-        model: str,
-        input: str,
-        voice: str,
-        response_format: AudioFormat,
-        speed: float,
-        ref_audio: AudioInput,
-        ref_text: str,
-        x_vector_only_mode: bool,
-        model_specific_params: dict | None,
-        **kwargs,
-    ):
-        is_qwen_tts = "qwen3-tts" in model.lower()
-        if not is_qwen_tts and isinstance(model_specific_params, QwenTTSModelSpecificParams):
-            raise ValueError(
-                "You have provided Qwen-specific TTS params."
-                "However, the model appears to not be a Qwen TTS model (no 'Qwen3-TTS' in model name)."
-            )
-
-        combined_params = {
-            "ref_audio": ref_audio,
-            "ref_text": ref_text,
-            "x_vector_only_mode": x_vector_only_mode,
-            **kwargs,
-            **(model_specific_params or {}),
-        }
-
-        client = VLLMOmniClient(url)
-
-        audio = await client.generate_speech(
-            model=model,
-            input=input,
-            voice=voice,
-            response_format=response_format,
-            speed=speed,
-            **combined_params,
-        )
-        return (audio,)
-
-
-class VLLMOmniARSampling:
-    @classmethod
-    def INPUT_TYPES(cls):
-        return {
-            "required": {
-                "max_tokens": ("INT", {"default": 100, "min": 1, "max": 10000}),
-                "temperature": (
-                    "FLOAT",
-                    {"default": 1.0, "min": 0.0, "max": 2.0, "step": 0.01},
-                ),
-                "top_p": (
-                    "FLOAT",
-                    {"default": 1.0, "min": 0.0, "max": 1.0, "step": 0.01},
-                ),
-                "repetition_penalty": (
-                    "FLOAT",
-                    {"default": 1.0, "min": 0.0, "max": 5.0, "step": 0.01},
-                ),
-                # === Put seed at last. ===
-                # Whenever a field named "seed" is present, ComfyUI adds another field called "control after generate"
-                "seed": (
-                    "INT",
-                    {
-                        "default": -1,
-                        "min": -1,
-                        "step": 1,
-                        "tooltip": "-1 means to not provide a seed.",
-                    },
-                ),
-            }
-        }
-
-    RETURN_TYPES = ("SAMPLING_PARAMS",)
-    RETURN_NAMES = ("AR sampling params",)
-    FUNCTION = "get_params"
-    CATEGORY = "vLLM-Omni/Sampling Params"
-
-    def get_params(self, seed, **kwargs):
-        params = AutoregressionSamplingParams(kwargs)
-        if seed >= 0:
-            params["seed"] = seed
-        return (params,)
-
-
-class VLLMOmniDiffusionSampling:
-    @classmethod
-    def INPUT_TYPES(cls):
-        return {
-            "required": {
-                "n": (
-                    "INT",
-                    {
-                        "default": 1,
-                        "min": 0,
-                        "max": 10,
-                        "step": 1,
-                        "tooltip": "Number of images to generate",
-                    },
-                ),
-                "num_inference_steps": (
-                    "INT",
-                    {
-                        "default": 50,
-                        "min": 1,
-                        "max": 1000,
-                        "tooltip": "Number of denoising steps (higher = better quality, slower).",
-                    },
-                ),
-                "guidance_scale": (
-                    "FLOAT",
-                    {
-                        "default": 7.5,
-                        "min": 0.0,
-                        "max": 20.0,
-                        "step": 0.1,
-                        "tooltip": "Classifier-free guidance scale (higher = more prompt adherence).",
-                    },
-                ),
-                "true_cfg_scale": (
-                    "FLOAT",
-                    {
-                        "default": 1.0,
-                        "min": 0.0,
-                        "max": 20.0,
-                        "step": 0.5,
-                        "tooltip": "True CFG scale for advanced control (model-specific).",
-                    },
-                ),
-                "vae_use_slicing": (
-                    "BOOLEAN",
-                    {
-                        "default": False,
-                        "tooltip": "Enable VAE slicing for reduced memory usage (slight quality trade-off)",
-                    },
-                ),
-                "vae_use_tiling": (
-                    "BOOLEAN",
-                    {
-                        "default": False,
-                        "tooltip": "Enable VAE tiling for reduced memory usage (slight quality trade-off)",
-                    },
-                ),
-                # === Put seed at last. ===
-                # Whenever a field named "seed" is present, ComfyUI adds another field called "control after generate"
-                "seed": (
-                    "INT",
-                    {
-                        "default": -1,
-                        "min": -1,
-                        "step": 1,
-                        "tooltip": "-1 means to not provide a seed.",
-                    },
-                ),
-            }
-        }
-
-    RETURN_TYPES = ("SAMPLING_PARAMS",)
-    RETURN_NAMES = ("diffusion sampling params",)
-    FUNCTION = "get_params"
-    CATEGORY = "vLLM-Omni/Sampling Params"
-
-    def get_params(self, seed, **kwargs):
-        params = DiffusionSamplingParams(kwargs)
-        if seed >= 0:
-            params["seed"] = seed
-        return (params,)
-
-
-class VLLMOmniSamplingParamsList:
-    @classmethod
-    def INPUT_TYPES(cls):
-        return {
-            "required": {
-                "param1": ("SAMPLING_PARAMS",),
-            },
-            "optional": {
-                "param2": ("SAMPLING_PARAMS",),
-                "param3": ("SAMPLING_PARAMS",),
-            },
-        }
-
-    RETURN_TYPES = ("SAMPLING_PARAMS",)
-    RETURN_NAMES = ("param list",)
-    FUNCTION = "aggregate"
-    CATEGORY = "vLLM-Omni/Sampling Params"
-
-    def aggregate(self, param1: dict, param2: dict | None = None, param3: dict | None = None):
-        for i, p in enumerate((param1, param2, param3)):
-            if isinstance(p, list):
-                raise ValueError(
-                    f"Input {i} is a Multi-Stage Sampling Params List. "
-                    f"Expected a single sampling parameters node (either AR or Diffusion)."
-                )
-
-        params = [param1]
-        if param2 is not None:
-            params.append(param2)
-        if param3 is not None:
-            params.append(param3)
-        return (params,)
-
-
-class VLLMOmniQwenTTSParams:
-    @classmethod
-    def INPUT_TYPES(cls):
-        return {
-            "required": {
-                "task_type": (
-                    ["CustomVoice", "VoiceDesign", "Base"],
-                    {"default": "CustomVoice"},
-                ),
-                "language": (
-                    ["Auto", "Chinese", "English", "Japanese", "Korean"],
-                    {"default": "Auto"},
-                ),
-                "instructions": ("STRING", {"multiline": True}),
-                "max_new_tokens": ("INT", {"default": 2048, "min": 1}),
-            }
-        }
-
-    RETURN_TYPES = ("TTS_PARAMS",)
-    RETURN_NAMES = ("Qwen TTS params",)
-    FUNCTION = "get_params"
-    CATEGORY = "vLLM-Omni/TTS Params"
-
-    def get_params(self, **kwargs):
-        return (QwenTTSModelSpecificParams(kwargs),)

apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/api_client.py

@@ -1,459 +0,0 @@
-"""
-An high-level API client adapter that forwards ComfyUI inputs to vLLM-Omni's REST API,
-and transforms the API responses back to ComfyUI formats.
-
-The image generation part is derived from dougbtv/comfyui-vllm-omni by Doug (@dougbtv).
-Original source at https://github.com/dougbtv/comfyui-vllm-omni, distributed under the MIT License.
-"""
-
-from typing import Any
-
-import aiohttp
-import av.error
-import torch
-from comfy_api.input import AudioInput, VideoInput
-
-from .format import (
-    audio_to_base64,
-    base64_to_audio,
-    base64_to_image_tensor,
-    bytes_to_audio,
-    image_tensor_to_base64,
-    image_tensor_to_png_bytes,
-    video_to_base64,
-)
-from .logger import get_logger, pretty_printer
-from .models import lookup_model_spec
-from .types import AudioFormat
-
-logger = get_logger(__name__)
-
-
-class VLLMOmniClient:
-    def __init__(self, base_url: str, timeout: float = 300.0):
-        self.base_url = base_url
-        self.timeout = aiohttp.ClientTimeout(total=timeout)
-
-    async def generate_image(
-        self,
-        *,
-        model: str,
-        prompt: str,
-        width: int,
-        height: int,
-        negative_prompt: str | None = None,
-        sampling_params: dict | None = None,
-    ) -> torch.Tensor:
-        """Run text-to-image generation via DALLE API"""
-        await self._check_model_exist(model)
-
-        size = f"{width}x{height}"
-        payload = {
-            "model": model,
-            "prompt": prompt,
-            "size": size,
-            "response_format": "b64_json",
-        }
-        if negative_prompt:
-            payload["negative_prompt"] = negative_prompt
-        if sampling_params is not None:
-            payload.update(sampling_params)
-        logger.debug("img gen payload: %s", payload)
-
-        url = self.base_url + "/images/generations"
-        async with aiohttp.ClientSession(timeout=self.timeout) as session:
-            try:
-                async with session.post(
-                    url,
-                    json=payload,
-                    headers={"Content-Type": "application/json"},
-                ) as response:
-                    if not response.ok:
-                        error_text = await response.text()
-                        raise (ValueError if response.status < 500 else RuntimeError)(
-                            f"vLLM-Omni API returned status {response.status}: {error_text}"
-                        )
-
-                    try:
-                        data = await response.json()
-                    except aiohttp.ContentTypeError as e:
-                        raise RuntimeError(f"Invalid JSON response from vLLM-Omni: {e}")
-                    if "data" not in data:
-                        raise RuntimeError("API response missing 'data' field - expected OpenAI DALL-E format")
-                    if not data["data"]:
-                        raise RuntimeError("API returned empty data array")
-
-                    image_tensors = []
-                    for idx, img in enumerate(data["data"]):
-                        if "b64_json" not in img:
-                            raise RuntimeError(f"API returned image #{idx} without 'b64_json' field")
-                        base64_str = img["b64_json"]
-                        tensor = base64_to_image_tensor(base64_str)
-                        image_tensors.append(tensor)
-                        logger.debug("Image #%d has shape %s", idx, tensor.shape)
-
-                    batch_tensor = torch.stack(image_tensors, dim=0)
-                    logger.debug("batch_tensor output has shape: %s", batch_tensor.shape)
-                    return batch_tensor
-
-            except aiohttp.ClientError as e:
-                raise RuntimeError(f"Network error connecting to vLLM-Omni at {url}: {e}")
-
-    async def edit_image(
-        self,
-        *,
-        model: str,
-        prompt: str,
-        image: torch.Tensor,
-        width: int,
-        height: int,
-        negative_prompt: str | None = None,
-        mask: torch.Tensor | None = None,
-        sampling_params: dict | None = None,
-    ) -> torch.Tensor:
-        """Run image editing via DALLE API"""
-        await self._check_model_exist(model)
-
-        size = f"{width}x{height}"
-        image_filename = "image.png"  # Required for multipart form
-        form = aiohttp.FormData()
-        form.add_field("model", model)
-        form.add_field(
-            "image",
-            image_tensor_to_png_bytes(image, image_filename),
-            filename=image_filename,
-            content_type="image/png",
-        )
-        form.add_field("prompt", prompt)
-        form.add_field("size", size)
-        if negative_prompt:
-            form.add_field("negative_prompt", negative_prompt)
-        if sampling_params is not None:
-            for k, v in sampling_params.items():
-                form.add_field(k, str(v))
-        if mask is not None:
-            mask_filename = "mask.png"
-            form.add_field(
-                "mask",
-                image_tensor_to_png_bytes(mask, mask_filename),
-                filename=mask_filename,
-                content_type="image/png",
-            )
-
-        url = self.base_url + "/images/edits"
-        async with aiohttp.ClientSession(timeout=self.timeout) as session:
-            try:
-                async with session.post(url, data=form) as response:
-                    if not response.ok:
-                        error_text = await response.text()
-                        raise (ValueError if response.status < 500 else RuntimeError)(
-                            f"vLLM-Omni API returned status {response.status}: {error_text}"
-                        )
-
-                    try:
-                        data = await response.json()
-                    except aiohttp.ContentTypeError as e:
-                        raise RuntimeError(f"Invalid JSON response from vLLM-Omni: {e}")
-
-                    if "data" not in data:
-                        raise RuntimeError("API response missing 'data' field - expected OpenAI DALL-E format")
-                    if not data["data"]:
-                        raise RuntimeError("API returned empty data array")
-
-                    image_tensors = []
-                    for idx, img in enumerate(data["data"]):
-                        if "b64_json" not in img:
-                            raise RuntimeError(f"API returned image #{idx} without 'b64_json' field")
-                        base64_str = img["b64_json"]
-                        tensor = base64_to_image_tensor(base64_str)
-                        image_tensors.append(tensor)
-
-                    return torch.stack(image_tensors, dim=0)
-
-            except aiohttp.ClientError as e:
-                raise RuntimeError(f"Network error connecting to vLLM-Omni at {url}: {e}")
-
-    async def generate_image_chat_completion(
-        self,
-        *,
-        model: str,
-        prompt: str,
-        negative_prompt: str | None = None,
-        image: torch.Tensor | None = None,
-        audio: AudioInput | None = None,
-        video: VideoInput | None = None,
-        sampling_params: dict | list[dict] | None = None,
-    ) -> torch.Tensor:
-        payload = VLLMOmniClient._prepare_chat_completion_messages(
-            model=model,
-            prompt=prompt,
-            negative_prompt=negative_prompt,
-            image=image,
-            audio=audio,
-            video=video,
-            sampling_params=sampling_params,
-            modalities=["image"],
-        )
-        choices = await self._generate_base_chat_completion(model, payload)
-
-        image_tensors = []
-        for idx, img_content in enumerate(choices[0]["message"]["content"]):
-            base64_str = img_content.get("image_url", {}).get("url", "")
-            if not base64_str:
-                raise RuntimeError(f"API returned image #{idx} without image url")
-            tensor = base64_to_image_tensor(base64_str)
-            image_tensors.append(tensor)
-
-        return torch.stack(image_tensors, dim=0)
-
-    async def generate_understanding_chat_completion(
-        self,
-        *,
-        model: str,
-        prompt: str,
-        image: torch.Tensor | None = None,
-        audio: AudioInput | None = None,
-        video: VideoInput | None = None,
-        sampling_params: dict | list[dict] | None = None,
-        modalities: list[str] = ["text", "audio"],
-        **extra_body,
-    ) -> tuple[str | None, AudioInput | None]:
-        # Response may contain two choices: one with text, one with audio
-        payload = VLLMOmniClient._prepare_chat_completion_messages(
-            model=model,
-            prompt=prompt,
-            negative_prompt=None,
-            image=image,
-            audio=audio,
-            video=video,
-            sampling_params=sampling_params,
-            modalities=modalities,
-            **extra_body,
-        )
-
-        choices = await self._generate_base_chat_completion(model, payload)
-        text_response = None
-        audio_base64 = None
-        for choice in choices:
-            try:
-                text_response = choice["message"]["content"]
-            except (KeyError, TypeError):
-                # Either this case (text response) or the audio response case will be hit. Checking None's later.
-                pass
-            try:
-                audio_base64 = choice["message"]["audio"]["data"]
-            except (KeyError, TypeError):
-                # Either this case (text response) or the audio response case will be hit. Checking None's later.
-                pass
-        if audio_base64 is None and text_response is None:
-            raise RuntimeError(
-                "API response missing both '.message.audio' and 'message.content' fields."
-                f"The choices object is {choices}"
-            )
-        if audio_base64 is not None:
-            audio = base64_to_audio(audio_base64)
-            logger.debug(
-                "audio sample rate %d, audio shape %s, duration in second %f",
-                audio["sample_rate"],
-                audio["waveform"].shape,
-                audio["waveform"].shape[2] / audio["sample_rate"],
-            )
-        else:
-            audio = None
-        return text_response, audio
-
-    async def generate_speech(
-        self,
-        *,
-        model: str,
-        input: str,
-        voice: str,
-        response_format: AudioFormat,
-        speed: float,
-        **extra_params,
-    ) -> AudioInput:
-        await self._check_model_exist(model)
-
-        ref_audio: AudioInput | None = extra_params.pop("ref_audio", None)
-
-        payload = {
-            "model": model,
-            "input": input,
-            "voice": voice,
-            "response_format": response_format,
-            "speed": speed,
-            **extra_params,
-        }
-
-        if ref_audio is not None:
-            audio_base64 = audio_to_base64(ref_audio)
-            payload["ref_audio"] = audio_base64
-
-        logger.debug("Omni TTS payload: %s", pretty_printer.pformat(payload))
-
-        url = self.base_url + "/audio/speech"
-        async with aiohttp.ClientSession(timeout=self.timeout) as session:
-            try:
-                async with session.post(
-                    url,
-                    json=payload,
-                    headers={"Content-Type": "application/json"},
-                ) as response:
-                    if not response.ok:
-                        error_text = await response.text()
-                        raise (ValueError if response.status < 500 else RuntimeError)(
-                            f"vLLM-Omni API returned status {response.status}: {error_text}"
-                        )
-
-                    try:
-                        audio_bytes = await response.read()
-                    except aiohttp.ContentTypeError as e:
-                        raise RuntimeError(f"Invalid JSON response from vLLM-Omni: {e}")
-
-                    try:
-                        audio = bytes_to_audio(audio_bytes)
-                    except av.error.InvalidDataError as e:
-                        raise ValueError(
-                            f"Invalid audio data received from vLLM-Omni: {e}"
-                            "Check if you have input unsupported arguments (such as 'voice')"
-                        )
-                    return audio
-
-            except aiohttp.ClientError as e:
-                raise RuntimeError(f"Network error connecting to vLLM-Omni at {url}: {e}")
-
-    async def _generate_base_chat_completion(self, model: str, payload: dict[str, Any]) -> list[dict[str, Any]]:
-        logger.debug("Omni payload: %s", pretty_printer.pformat(payload))
-        await self._check_model_exist(model)
-
-        url = self.base_url + "/chat/completions"
-        async with aiohttp.ClientSession(timeout=self.timeout) as session:
-            try:
-                async with session.post(
-                    url,
-                    json=payload,
-                    headers={"Content-Type": "application/json"},
-                ) as response:
-                    if not response.ok:
-                        error_text = await response.text()
-                        raise (ValueError if response.status < 500 else RuntimeError)(
-                            f"vLLM-Omni API returned status {response.status}: {error_text}"
-                        )
-
-                    try:
-                        data = await response.json()
-                    except aiohttp.ContentTypeError as e:
-                        raise RuntimeError(f"Invalid JSON response from vLLM-Omni: {e}")
-
-                    logger.debug(
-                        "chat completion response: %s",
-                        pretty_printer.pformat(data),
-                    )
-
-                    try:
-                        return data["choices"]
-                    except (KeyError, TypeError):
-                        raise RuntimeError("Invalid JSON response from vLLM-Omni: missing 'choices' field")
-
-            except aiohttp.ClientError as e:
-                raise RuntimeError(f"Network error connecting to vLLM-Omni at {self.base_url}: {e}")
-
-    async def _check_model_exist(self, model: str):
-        url = self.base_url + "/models"
-        async with aiohttp.ClientSession(timeout=self.timeout) as session:
-            try:
-                async with session.get(
-                    url,
-                    headers={"Content-Type": "application/json"},
-                ) as response:
-                    if not response.ok:
-                        error_text = await response.text()
-                        raise (ValueError if response.status < 500 else RuntimeError)(
-                            f"vLLM-Omni API returned status {response.status} "
-                            f"when getting hosted model list: {error_text}"
-                        )
-
-                    try:
-                        data = await response.json()
-                    except aiohttp.ContentTypeError as e:
-                        raise RuntimeError(f"Invalid JSON response when getting hosted model list from vLLM-Omni: {e}")
-
-            except aiohttp.ClientError as e:
-                raise RuntimeError(f"Network error connecting to vLLM-Omni at {self.base_url}: {e}")
-        try:
-            model_list = data["data"]
-            model_found = next((True for m in model_list if m["id"] == model), False)
-        except (KeyError, TypeError):
-            raise RuntimeError(f"Invalid JSON response of the hosted model list: {data}")
-
-        if not model_found:
-            raise ValueError(f"Model {model} not served at {self.base_url}.")
-
-    @staticmethod
-    def _prepare_chat_completion_messages(
-        *,
-        model: str,
-        prompt: str,
-        negative_prompt: str | None,
-        image: torch.Tensor | None = None,
-        audio: AudioInput | None = None,
-        video: VideoInput | None = None,
-        sampling_params: dict | list[dict] | None = None,
-        modalities: list[str] | None = None,  # diffusion don't have this field
-        **extra_body,
-    ):
-        message_content: list[dict] = [{"type": "text", "text": prompt}]
-        if image is not None:
-            message_content.append(
-                {
-                    "type": "image_url",
-                    "image_url": {"url": image_tensor_to_base64(image)},
-                }
-            )
-        if audio is not None:
-            message_content.append({"type": "audio_url", "audio_url": {"url": audio_to_base64(audio)}})
-        if video is not None:
-            message_content.append({"type": "video_url", "video_url": {"url": video_to_base64(video)}})
-        messages = [{"role": "user", "content": message_content}]
-
-        payload: dict[str, Any] = {"messages": messages, "model": model}
-        if modalities:
-            payload["modalities"] = modalities
-
-        combined_extra_body: dict[str, Any] = {}
-        if sampling_params is not None:
-            spec, _ = lookup_model_spec(model)
-            is_single_sampling_param = isinstance(sampling_params, dict) or len(sampling_params) == 1
-
-            if (spec is None and is_single_sampling_param) or (spec is not None and spec["stages"] == ["diffusion"]):
-                # Diffusion format: extra_body directly contains sampling params.
-                # Validation has already taken care of matching sampling params' types and length. Safe to take [0].
-                # * Use this mode if the model is a simple one-stage diffusion model.
-                # * Fallback to this mode if model is not registered and a single sampling param is provided.
-                sampling_params = sampling_params if isinstance(sampling_params, dict) else sampling_params[0]
-                combined_extra_body: dict[str, Any] = sampling_params.copy()
-                if "n" in combined_extra_body:
-                    combined_extra_body["num_outputs_per_prompt"] = combined_extra_body.pop("n")
-            else:
-                # AR format: the payload has a sampling_params_list field, containing a list.
-                sampling_params_list = sampling_params if isinstance(sampling_params, list) else [sampling_params]
-                payload["sampling_params_list"] = sampling_params_list
-
-        if negative_prompt:
-            combined_extra_body["negative_prompt"] = negative_prompt
-
-        if extra_body:
-            combined_extra_body.update(extra_body)
-
-        # Add extra_body only if it has any content.
-        if combined_extra_body:
-            payload["extra_body"] = combined_extra_body
-
-        # Place to inject any model-specific payload adjustment
-        spec, _ = lookup_model_spec(model)
-        if spec:
-            preprocessor = spec.get("payload_preprocessor", None)
-            if preprocessor is not None:
-                payload = preprocessor(payload)
-
-        return payload

apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/format.py

@@ -1,230 +0,0 @@
-"""Image/tensor format helpers.
-
-The image generation part is derived from dougbtv/comfyui-vllm-omni by Doug (@dougbtv).
-Original source at https://github.com/dougbtv/comfyui-vllm-omni, distributed under the MIT License.
-"""
-
-import base64
-import mimetypes
-from io import BytesIO
-
-import av
-import numpy as np
-import torch
-from comfy_api.input import AudioInput, VideoInput
-from comfy_extras import nodes_audio
-from PIL import Image
-
-from .logger import get_logger
-
-logger = get_logger(__name__)
-
-
-def base64_to_image_tensor(base64_str: str, mode: str = "RGB") -> torch.Tensor:
-    """
-    Convert base64-encoded image to ComfyUI image tensor.
-
-    Args:
-        base64_str: Base64-encoded image string
-        mode: PIL image mode (default RGB for transparency support)
-
-    Returns:
-        torch.Tensor with shape (1, H, W, C) in float32 [0, 1] range
-
-    Raises:
-        ValueError: If base64 string is invalid or image cannot be decoded
-    """
-    if base64_str.startswith("data:image"):
-        _, base64_str = base64_str.split(",", 1)
-
-    try:
-        # Decode base64 to bytes
-        image_bytes = base64.b64decode(base64_str)
-    except Exception as e:
-        raise ValueError(f"Invalid base64 string: {e}")
-
-    # Create BytesIO object for PIL
-    image_bytesio = BytesIO(image_bytes)
-
-    # Open with PIL and convert to desired mode
-    try:
-        pil_image = Image.open(image_bytesio)
-        pil_image = pil_image.convert(mode)
-    except Exception as e:
-        raise RuntimeError(f"Failed to open image: {e}")
-
-    image_array = np.asarray(pil_image).astype(np.float32) / 255.0
-    image_tensor = torch.from_numpy(image_array)
-    return image_tensor
-
-
-def image_tensor_to_png_bytes(tensor: torch.Tensor, filename: str = "image.png") -> BytesIO:
-    """
-    Convert ComfyUI image tensor to PNG BytesIO for multipart upload.
-
-    This function converts a ComfyUI IMAGE tensor to a PNG-encoded BytesIO object
-    suitable for multipart/form-data upload. The BytesIO object has its .name
-    attribute set, which is required by aiohttp for file uploads.
-
-    Args:
-        tensor: ComfyUI IMAGE tensor with shape (B, H, W, C), dtype float32, range [0, 1]
-        filename: Name attribute to set on BytesIO (default: "image.png")
-
-    Returns:
-        BytesIO object containing PNG-encoded image with .name attribute set
-
-    Raises:
-        ValueError: If tensor format is invalid (not 4D, wrong dtype, etc.)
-    """
-    if tensor.ndim != 4:
-        raise ValueError(f"Expected 4D tensor with shape (B, H, W, C), got {tensor.ndim}D tensor")
-
-    image_tensor = tensor[0]  # Shape: (H, W, C)
-    image_np = (image_tensor.cpu().numpy() * 255.0).astype(np.uint8)
-    pil_image = Image.fromarray(image_np)
-
-    # Save to BytesIO as image file
-    img_bytes = BytesIO()
-    # Set name attribute (required for multipart upload and mimetype detection)
-    img_bytes.name = filename
-    try:
-        pil_image.save(img_bytes)
-    except Exception as e:
-        raise RuntimeError(f"Failed to save image as file: {e}")
-
-    # Reset position to beginning
-    img_bytes.seek(0)
-
-    return img_bytes
-
-
-def image_tensor_to_base64(tensor: torch.Tensor, filename: str = "image.png") -> str:
-    """
-    Convert ComfyUI image tensor to base64-encoded image string.
-
-    Args:
-        tensor: ComfyUI IMAGE tensor with shape (B, H, W, C), dtype float32, range [0, 1]
-        filename: Name attribute to set on BytesIO (default: "image.png")
-        format: File format of the output image file buffer (default: "PNG")
-
-    Returns:
-        Base64-encoded image string
-
-    Raises:
-        ValueError: If tensor format is invalid (not 4D, wrong dtype, etc.)
-    """
-    img_bytes = image_tensor_to_png_bytes(tensor, filename)
-    img_bytes.seek(0)
-    byte_data = img_bytes.read()
-    base64_str = base64.b64encode(byte_data).decode("utf-8")
-    mime_type = mimetypes.guess_type(filename)[0] or "application/octet-stream"
-    return f"data:{mime_type};base64,{base64_str}"
-
-
-def video_to_bytes(video: VideoInput, filename: str = "video.mp4") -> BytesIO:
-    output_buffer = BytesIO()
-    output_buffer.name = filename
-    video.save_to(output_buffer)
-    output_buffer.seek(0)
-    return output_buffer
-
-
-def video_to_base64(video: VideoInput, filename: str = "video.mp4") -> str:
-    video_buffer = video_to_bytes(video, filename)
-    video_buffer.seek(0)
-    byte_data = video_buffer.read()
-    base64_str = base64.b64encode(byte_data).decode("utf-8")
-    mime_type = mimetypes.guess_type(filename)[0] or "application/octet-stream"
-    return f"data:{mime_type};base64,{base64_str}"
-
-
-def audio_to_bytes(audio: AudioInput, filename: str = "audio.mp3", quality: str = "128k") -> BytesIO:
-    waveform = audio["waveform"][0]  # Shape: (C, T)
-    sample_rate = audio["sample_rate"]
-    format = filename.rsplit(".", maxsplit=1)[1]
-    layout = "mono" if waveform.shape[0] == 1 else "stereo"
-
-    output_buffer = BytesIO()
-    output_buffer.name = filename
-    output_container = av.open(output_buffer, mode="w", format=format)
-    if format == "opus":
-        out_stream = output_container.add_stream("libopus", rate=sample_rate, layout=layout)
-        if quality == "64k":
-            out_stream.bit_rate = 64000  # type: ignore # copy from ComfyUI comfy_api/latest/_ui.py
-        elif quality == "96k":
-            out_stream.bit_rate = 96000  # type: ignore # copy from ComfyUI comfy_api/latest/_ui.py
-        elif quality == "128k":
-            out_stream.bit_rate = 128000  # type: ignore # copy from ComfyUI comfy_api/latest/_ui.py
-        elif quality == "192k":
-            out_stream.bit_rate = 192000  # type: ignore # copy from ComfyUI comfy_api/latest/_ui.py
-        elif quality == "320k":
-            out_stream.bit_rate = 320000  # type: ignore # copy from ComfyUI comfy_api/latest/_ui.py
-    elif format == "mp3":
-        out_stream = output_container.add_stream("libmp3lame", rate=sample_rate, layout=layout)
-        if quality == "V0":
-            out_stream.codec_context.qscale = 1  # type: ignore # copy from ComfyUI comfy_api/latest/_ui.py
-        elif quality == "128k":
-            out_stream.bit_rate = 128000  # type: ignore # copy from ComfyUI comfy_api/latest/_ui.py
-        elif quality == "320k":
-            out_stream.bit_rate = 320000  # type: ignore # copy from ComfyUI comfy_api/latest/_ui.py
-    else:  # format == "flac":
-        out_stream = output_container.add_stream("flac", rate=sample_rate, layout=layout)
-
-    frame = av.AudioFrame.from_ndarray(
-        waveform.movedim(0, 1).reshape(1, -1).float().numpy(),
-        format="flt",
-        layout=layout,
-    )
-    frame.sample_rate = sample_rate
-    frame.pts = 0
-    output_container.mux(out_stream.encode(frame))  # type: ignore # copy from ComfyUI comfy_api/latest/_ui.py
-    # Flush encoder
-    output_container.mux(out_stream.encode(None))  # type: ignore # copy from ComfyUI comfy_api/latest/_ui.py
-    output_container.close()
-    output_buffer.seek(0)
-
-    return output_buffer
-
-
-def audio_to_base64(audio: AudioInput, filename: str = "audio.mp3", quality: str = "128k") -> str:
-    audio_buffer = audio_to_bytes(audio, filename, quality)
-    audio_buffer.seek(0)
-    byte_data = audio_buffer.read()
-    base64_str = base64.b64encode(byte_data).decode("utf-8")
-    mime_type = mimetypes.guess_type(filename)[0] or "application/octet-stream"
-    return f"data:{mime_type};base64,{base64_str}"
-
-
-def bytes_to_audio(audio_bytes: bytes) -> AudioInput:
-    """
-    Convert audio bytes to ComfyUI audio tensor.
-
-    Args:
-        audio_bytes: Audio file bytes
-    Returns:
-        torch.Tensor with shape (B, C, T) in float32 range [-1, 1]
-    """
-    audio_buffer = BytesIO(audio_bytes)
-    waveform, sample_rate = nodes_audio.load(audio_buffer)  # type: ignore # Although expect string argument, it calls av.open underneath, which supports BytesIO (file-like)
-    return {"waveform": waveform.unsqueeze(0), "sample_rate": sample_rate}
-
-
-def base64_to_audio(base64_str: str) -> AudioInput:
-    """
-    Convert base64-encoded audio to ComfyUI audio tensor.
-
-    Args:
-        base64_str: Base64-encoded audio string
-    Returns:
-        torch.Tensor with shape (B, C, T) in float32 range [-1, 1]
-    """
-    if base64_str.startswith("data:audio"):
-        _, base64_str = base64_str.split(",", 1)
-
-    try:
-        # Decode base64 to bytes
-        audio_bytes = base64.b64decode(base64_str)
-    except Exception as e:
-        raise ValueError(f"Invalid base64 string: {e}")
-
-    return bytes_to_audio(audio_bytes)

apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/logger.py

@@ -1,125 +0,0 @@
-"""Centralized logger configuration for vLLM-Omni ComfyUI."""
-
-import logging
-import pprint
-import sys
-from typing import Any
-
-
-def get_logger(name: str) -> logging.Logger:
-    """
-    Get or create a logger with proper formatting.
-
-    Args:
-        name: Logger name (typically __name__ of the calling module)
-
-    Returns:
-        Configured logger instance
-    """
-    logger = logging.getLogger(name)
-
-    # Only configure if not already configured
-    if not logger.handlers:
-        logger.setLevel(logging.DEBUG)
-
-        # Create console handler
-        handler = logging.StreamHandler(sys.stdout)
-        handler.setLevel(logging.INFO)
-
-        # Create formatter
-        formatter = logging.Formatter(
-            fmt="(ComfyUI-vLLM-Omni) [%(levelname)s] %(asctime)s [%(filename)s:%(lineno)s] %(message)s",
-            datefmt="%Y-%m-%d %H:%M:%S",
-        )
-        handler.setFormatter(formatter)
-
-        # Add handler to logger
-        logger.addHandler(handler)
-
-        # Prevent propagation to root logger
-        logger.propagate = False
-
-    return logger
-
-
-class OmitBase64PrettyPrinter(pprint.PrettyPrinter):
-    """
-    A PrettyPrinter that redacts specific field names with '...'
-    wherever they appear in nested structures.
-    """
-
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-
-    def _format(self, obj: Any, stream, indent: int, allowance: int, context, level: int) -> None:
-        # Check if this is a dict with redacted keys
-        if isinstance(obj, dict):
-            # Create a copy with redacted values
-            display_obj = {}
-            for key, value in obj.items():
-                if key == "data" or key == "url":
-                    if value.startswith("data:"):
-                        base64_header = value.split(",", 1)[0]
-                        display_obj[key] = f"{base64_header},***"
-                    elif value.startswith("http://") or value.startswith("https://"):
-                        display_obj[key] = value
-                    elif len(value) > 10:
-                        display_obj[key] = f"{value[:10]}***"
-                    else:
-                        display_obj[key] = value
-                else:
-                    display_obj[key] = value
-            obj = display_obj
-
-        # Handle list/tuple/set containing dicts that might have redacted keys
-        # (pprint will recursively call _format on nested items, so this
-        # handles arbitrary nesting automatically)
-
-        super()._format(obj, stream, indent, allowance, context, level)
-
-
-pretty_printer = OmitBase64PrettyPrinter()
-
-
-# ========== PPrint EXAMPLES ==========
-
-if __name__ == "__main__":
-    data = {
-        "messages": [
-            {
-                "role": "system",
-                "content": (
-                    "You are Qwen, a virtual human developed by the Qwen Team, Alibaba Group,"
-                    "capable of perceiving auditory and visual inputs, as well as generating text and speech."
-                ),
-            },
-            {
-                "role": "user",
-                "content": [
-                    {
-                        "type": "text",
-                        "text": "What sound is it, and what is the drawing about?",
-                    },
-                    {
-                        "type": "text",
-                        "text": "What sound is it, and what is the drawing about?",
-                    },
-                    {
-                        "type": "image_url",
-                        "image_url": {"url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAwAAAA"},
-                    },
-                    {
-                        "type": "audio_url",
-                        "audio_url": {"url": "data:audio/mpeg;base64,SUQzBAAAAAAAIlRTU0UAAAAOAAADT="},
-                    },
-                ],
-            },
-        ],
-        "extra_body": {"mm_processor_kwargs": {"use_audio_in_video": False}},
-        "modalities": ["text"],
-    }
-
-    # Create printer that redacts 'password' and 'token' fields
-
-    print("\nRedactingPrettyPrinter (hides secrets):")
-    pretty_printer.pprint(data)

apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/models.py

@@ -1,118 +0,0 @@
-import re
-
-from .types import Modality, ModelMode, Spec
-
-
-def _bagel_payload_preprocessor(payload: dict) -> dict:
-    try:
-        for message in payload["messages"]:
-            for content in message["content"]:
-                if content["type"] == "text":
-                    content["text"] = "<|im_start|>" + content["text"] + "<|im_end|>"
-    except (KeyError, TypeError):
-        raise RuntimeError("Internal Error: malformatted BAGEL payload")
-    return payload
-
-
-def _qwen25_payload_preprocessor(payload: dict) -> dict:
-    if payload["messages"][0]["role"] != "system":
-        payload["messages"] = [
-            {
-                "role": "system",
-                "content": (
-                    "You are Qwen, a virtual human developed by the Qwen Team, Alibaba Group,"
-                    "capable of perceiving auditory and visual inputs, as well as generating text and speech."
-                ),
-            },
-            *payload["messages"],
-        ]
-    return payload
-
-
-_MODEL_PIPELINE_SPECS: dict[str, Spec] = {
-    r"BAGEL-7B-MoT": {
-        "stages": [
-            "diffusion"  # The vLLM-Omni interface treats it as a single-stage diffusion model
-        ],
-        "modes": [
-            {
-                "mode": ModelMode.UNDERSTANDING,
-                "input_modalities": [Modality.TEXT, Modality.IMAGE],
-            }
-        ],
-        "payload_preprocessor": _bagel_payload_preprocessor,
-    },
-    r"Qwen2.5-Omni*": {
-        "stages": ["autoregression", "autoregression", "autoregression"],
-        "payload_preprocessor": _qwen25_payload_preprocessor,
-        "modes": [
-            {
-                "mode": ModelMode.UNDERSTANDING,
-                "input_modalities": [
-                    Modality.TEXT,
-                    Modality.IMAGE,
-                    Modality.VIDEO,
-                    Modality.AUDIO,
-                ],
-            }
-        ],
-    },
-    r"Qwen3-Omni*": {
-        "stages": ["autoregression", "autoregression", "autoregression"],
-        "modes": [
-            {
-                "mode": ModelMode.UNDERSTANDING,
-                "input_modalities": [
-                    Modality.TEXT,
-                    Modality.IMAGE,
-                    Modality.VIDEO,
-                    Modality.AUDIO,
-                ],
-            }
-        ],
-    },
-}
-# Convert dict keys to regex patterns
-MODEL_PIPELINE_SPECS: dict[re.Pattern, Spec] = {}
-for k, v in _MODEL_PIPELINE_SPECS.items():
-    MODEL_PIPELINE_SPECS[re.compile(k)] = v
-del _MODEL_PIPELINE_SPECS
-
-
-def lookup_model_spec(model: str) -> tuple[Spec | None, str | None]:
-    try:
-        last_component = model.rstrip("/").rsplit("/", 1)[-1]
-    except IndexError:
-        last_component = model
-    for pattern, spec in MODEL_PIPELINE_SPECS.items():
-        if pattern.search(last_component):
-            return spec, pattern.pattern
-    return None, None
-
-
-# ============== DEMONSTRATION ==============
-
-if __name__ == "__main__":
-    test_paths = [
-        "Qwen/Qwen2.5-Omni-7B",
-        "MyModels/Qwen2.5-Omni-3B",
-        "/root/home/Qwen2.5-Omni-7B",
-        "Qwen/Qwen3-Omni",
-        "Qwen/Qwen3-Omni-30B-A3B-Instruct",
-        "Custom/Path/UnknownModel-Instruct",
-        "Not/Matching/Anything",
-    ]
-
-    test_payload = {"messages": [{"role": "user", "content": "prompt"}]}
-
-    print("Testing registry lookups:\n")
-    for path in test_paths:
-        spec, _ = lookup_model_spec(path)
-        if spec:
-            if preprocessor := spec.get("payload_preprocessor"):
-                result = preprocessor(test_payload)
-                print(f"✓ {path:<40} → {result}")
-            else:
-                print(f"✓ {path:<40} → No preprocessor")
-        else:
-            print(f"✗ {path:<40} → No match")

apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/types.py

@@ -1,51 +0,0 @@
-from collections.abc import Callable
-from enum import Enum, auto
-from typing import (
-    Any,
-    Literal,
-    NotRequired,
-    TypeAlias,
-    TypedDict,
-)
-
-AudioFormat: TypeAlias = Literal["mp3", "opus", "aac", "flac", "wav", "pcm"]
-
-
-class AutoregressionSamplingParams(dict):
-    pass
-
-
-class DiffusionSamplingParams(dict):
-    pass
-
-
-class QwenTTSModelSpecificParams(dict):
-    pass
-
-
-class ModelMode(Enum):
-    IMAGE_GENERATION = auto()
-    VIDEO_GENERATION = auto()
-    AUDIO_GENERATION = auto()
-    UNDERSTANDING = auto()
-
-
-class Modality(Enum):
-    TEXT = auto()  # maybe not useful. Prompt is always required
-    IMAGE = auto()
-    VIDEO = auto()
-    AUDIO = auto()
-
-
-class ModelModeSpec(TypedDict):
-    mode: ModelMode
-    input_modalities: list[Modality]
-
-
-PayloadPreprocessor: TypeAlias = Callable[[dict[str, Any]], dict[str, Any]]
-
-
-class Spec(TypedDict):
-    stages: list[Literal["diffusion", "autoregression"]]
-    modes: list[ModelModeSpec]
-    payload_preprocessor: NotRequired[PayloadPreprocessor]

apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/validators.py

@@ -1,89 +0,0 @@
-from .logger import get_logger
-from .models import lookup_model_spec
-from .types import AutoregressionSamplingParams, DiffusionSamplingParams
-
-logger = get_logger(__name__)
-
-
-def validate_model_and_sampling_params_types(
-    model_name: str,
-    sampling_param_list: dict | list[dict] | None = None,
-):
-    # Check if model name exists
-    if not model_name:
-        raise ValueError("Model name must not be empty.")
-
-    # Skip if no spec or no sampling params
-    pipeline_spec, _ = lookup_model_spec(model_name)
-    if pipeline_spec is None:
-        logger.info("skipping sampling params check because spec is not found")
-        return
-    if sampling_param_list is None:
-        return
-
-    # Check the number of stages and their data types
-    stages = pipeline_spec["stages"]
-    if isinstance(sampling_param_list, list):
-        # Check that the lengths match
-        if len(stages) != len(sampling_param_list):
-            raise ValueError(
-                f"Sampling parameter list length {len(sampling_param_list)} does not match "
-                f"number of stages {len(stages)} for model {model_name}."
-            )
-        # Check that each stage's type match
-        for i, sp in enumerate(sampling_param_list):
-            if not _check_sampling_param_matches_stage(sp, stages[i]):
-                raise ValueError(
-                    f"Sampling parameter type ({sp.__class__.__name__}) does not match "
-                    f"stage type ({stages[i]}) at index {i} for model {model_name}."
-                )
-    elif isinstance(sampling_param_list, dict):
-        # Check that the provided single sampling param matches all stages
-        for i, stage in enumerate(stages):
-            if not _check_sampling_param_matches_stage(sampling_param_list, stage):
-                raise ValueError(
-                    f"Provided single sampling parameter type ({sampling_param_list.__class__.__name__}) must match "
-                    f"the types of all stages of the model. "
-                    f"However, stage {i} of model {model_name} is of type {stage}."
-                )
-
-
-def add_sampling_parameters_to_stage(
-    model_name: str,
-    sampling_param_list: dict | list[dict] | None,
-    stage_type: str,
-    /,
-    **params_to_add,
-) -> dict | list[dict]:
-    """
-    Given a model's name and the sampling parameter list to query this model,
-    add arbitrary additional parameters to the sampling parameters of all stages of the given type.
-    """
-    pipeline_spec, _ = lookup_model_spec(model_name)
-    if not pipeline_spec:
-        logger.warning(
-            f"Since the model {model_name} is not in our list, we cannot ensure if "
-            f"the fields ({tuple(params_to_add.keys())}) are added to the correct stage's sampling params. "
-            f"We will do it heuristically."
-        )
-        pipeline_spec = {"stages": ["diffusion"]}
-
-    stages = pipeline_spec["stages"]
-    if isinstance(sampling_param_list, dict):
-        sampling_param_list.update(params_to_add)
-    elif sampling_param_list is None:
-        sampling_param_list = params_to_add.copy()
-    else:
-        for i, stage in enumerate(stages):
-            if stage == stage_type:
-                sampling_param_list[i].update(params_to_add)
-
-    return sampling_param_list
-
-
-def _check_sampling_param_matches_stage(sampling_param: dict, stage_type: str) -> bool:
-    if stage_type == "autoregression":
-        return isinstance(sampling_param, AutoregressionSamplingParams)
-    if stage_type == "diffusion":
-        return isinstance(sampling_param, DiffusionSamplingParams)
-    raise RuntimeError(f"Internal error: unknown stage type {stage_type}.")

apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni Chaining Services.json

@@ -1,552 +0,0 @@
-{
-    "id": "6643e5fd-fa2a-4f25-935a-483173a8097c",
-    "revision": 0,
-    "last_node_id": 8,
-    "last_link_id": 6,
-    "nodes": [
-        {
-            "id": 1,
-            "type": "PreviewImage",
-            "pos": [
-                1446.2005205859512,
-                -316.7359686049902
-            ],
-            "size": [
-                305.6628685610174,
-                307.0336884172169
-            ],
-            "flags": {},
-            "order": 5,
-            "mode": 0,
-            "inputs": [
-                {
-                    "localized_name": "images",
-                    "name": "images",
-                    "type": "IMAGE",
-                    "link": 1
-                }
-            ],
-            "outputs": [],
-            "properties": {
-                "Node name for S&R": "PreviewImage"
-            },
-            "widgets_values": []
-        },
-        {
-            "id": 8,
-            "type": "PreviewImage",
-            "pos": [
-                1010.1848219273943,
-                -311.54216706114363
-            ],
-            "size": [
-                305.6628685610174,
-                307.0336884172169
-            ],
-            "flags": {},
-            "order": 4,
-            "mode": 0,
-            "inputs": [
-                {
-                    "localized_name": "images",
-                    "name": "images",
-                    "type": "IMAGE",
-                    "link": 6
-                }
-            ],
-            "outputs": [],
-            "properties": {
-                "Node name for S&R": "PreviewImage"
-            },
-            "widgets_values": []
-        },
-        {
-            "id": 7,
-            "type": "VLLMOmniDiffusionSampling",
-            "pos": [
-                234.61958293819296,
-                9.405681270043619
-            ],
-            "size": [
-                284.205078125,
-                226
-            ],
-            "flags": {},
-            "order": 0,
-            "mode": 0,
-            "inputs": [
-                {
-                    "localized_name": "n",
-                    "name": "n",
-                    "type": "INT",
-                    "widget": {
-                        "name": "n"
-                    },
-                    "link": null
-                },
-                {
-                    "localized_name": "num_inference_steps",
-                    "name": "num_inference_steps",
-                    "type": "INT",
-                    "widget": {
-                        "name": "num_inference_steps"
-                    },
-                    "link": null
-                },
-                {
-                    "localized_name": "guidance_scale",
-                    "name": "guidance_scale",
-                    "type": "FLOAT",
-                    "widget": {
-                        "name": "guidance_scale"
-                    },
-                    "link": null
-                },
-                {
-                    "localized_name": "true_cfg_scale",
-                    "name": "true_cfg_scale",
-                    "type": "FLOAT",
-                    "widget": {
-                        "name": "true_cfg_scale"
-                    },
-                    "link": null
-                },
-                {
-                    "localized_name": "vae_use_slicing",
-                    "name": "vae_use_slicing",
-                    "type": "BOOLEAN",
-                    "widget": {
-                        "name": "vae_use_slicing"
-                    },
-                    "link": null
-                },
-                {
-                    "localized_name": "vae_use_tiling",
-                    "name": "vae_use_tiling",
-                    "type": "BOOLEAN",
-                    "widget": {
-                        "name": "vae_use_tiling"
-                    },
-                    "link": null
-                },
-                {
-                    "localized_name": "seed",
-                    "name": "seed",
-                    "type": "INT",
-                    "widget": {
-                        "name": "seed"
-                    },
-                    "link": null
-                }
-            ],
-            "outputs": [
-                {
-                    "localized_name": "diffusion sampling params",
-                    "name": "diffusion sampling params",
-                    "type": "SAMPLING_PARAMS",
-                    "links": [
-                        5
-                    ]
-                }
-            ],
-            "properties": {
-                "Node name for S&R": "VLLMOmniDiffusionSampling"
-            },
-            "widgets_values": [
-                1,
-                50,
-                1,
-                1,
-                false,
-                false,
-                1525,
-                "randomize"
-            ]
-        },
-        {
-            "id": 4,
-            "type": "VLLMOmniDiffusionSampling",
-            "pos": [
-                666.8380026154548,
-                268.86271068330126
-            ],
-            "size": [
-                284.205078125,
-                226
-            ],
-            "flags": {},
-            "order": 1,
-            "mode": 0,
-            "inputs": [
-                {
-                    "localized_name": "n",
-                    "name": "n",
-                    "type": "INT",
-                    "widget": {
-                        "name": "n"
-                    },
-                    "link": null
-                },
-                {
-                    "localized_name": "num_inference_steps",
-                    "name": "num_inference_steps",
-                    "type": "INT",
-                    "widget": {
-                        "name": "num_inference_steps"
-                    },
-                    "link": null
-                },
-                {
-                    "localized_name": "guidance_scale",
-                    "name": "guidance_scale",
-                    "type": "FLOAT",
-                    "widget": {
-                        "name": "guidance_scale"
-                    },
-                    "link": null
-                },
-                {
-                    "localized_name": "true_cfg_scale",
-                    "name": "true_cfg_scale",
-                    "type": "FLOAT",
-                    "widget": {
-                        "name": "true_cfg_scale"
-                    },
-                    "link": null
-                },
-                {
-                    "localized_name": "vae_use_slicing",
-                    "name": "vae_use_slicing",
-                    "type": "BOOLEAN",
-                    "widget": {
-                        "name": "vae_use_slicing"
-                    },
-                    "link": null
-                },
-                {
-                    "localized_name": "vae_use_tiling",
-                    "name": "vae_use_tiling",
-                    "type": "BOOLEAN",
-                    "widget": {
-                        "name": "vae_use_tiling"
-                    },
-                    "link": null
-                },
-                {
-                    "localized_name": "seed",
-                    "name": "seed",
-                    "type": "INT",
-                    "widget": {
-                        "name": "seed"
-                    },
-                    "link": null
-                }
-            ],
-            "outputs": [
-                {
-                    "localized_name": "diffusion sampling params",
-                    "name": "diffusion sampling params",
-                    "type": "SAMPLING_PARAMS",
-                    "links": [
-                        2
-                    ]
-                }
-            ],
-            "properties": {
-                "Node name for S&R": "VLLMOmniDiffusionSampling"
-            },
-            "widgets_values": [
-                4,
-                50,
-                7,
-                1,
-                false,
-                false,
-                42,
-                "fixed"
-            ]
-        },
-        {
-            "id": 5,
-            "type": "VLLMOmniGenerateImage",
-            "pos": [
-                984.723613585788,
-                63.376900027553276
-            ],
-            "size": [
-                416.56628685610167,
-                372.1662621294205
-            ],
-            "flags": {},
-            "order": 3,
-            "mode": 0,
-            "inputs": [
-                {
-                    "localized_name": "image",
-                    "name": "image",
-                    "shape": 7,
-                    "type": "IMAGE",
-                    "link": 4
-                },
-                {
-                    "localized_name": "mask",
-                    "name": "mask",
-                    "shape": 7,
-                    "type": "MASK",
-                    "link": null
-                },
-                {
-                    "localized_name": "sampling_params",
-                    "name": "sampling_params",
-                    "shape": 7,
-                    "type": "SAMPLING_PARAMS",
-                    "link": 2
-                },
-                {
-                    "localized_name": "url",
-                    "name": "url",
-                    "type": "STRING",
-                    "widget": {
-                        "name": "url"
-                    },
-                    "link": null
-                },
-                {
-                    "localized_name": "model",
-                    "name": "model",
-                    "type": "STRING",
-                    "widget": {
-                        "name": "model"
-                    },
-                    "link": null
-                },
-                {
-                    "localized_name": "prompt",
-                    "name": "prompt",
-                    "type": "STRING",
-                    "widget": {
-                        "name": "prompt"
-                    },
-                    "link": null
-                },
-                {
-                    "localized_name": "negative_prompt",
-                    "name": "negative_prompt",
-                    "type": "STRING",
-                    "widget": {
-                        "name": "negative_prompt"
-                    },
-                    "link": null
-                },
-                {
-                    "localized_name": "width",
-                    "name": "width",
-                    "type": "INT",
-                    "widget": {
-                        "name": "width"
-                    },
-                    "link": null
-                },
-                {
-                    "localized_name": "height",
-                    "name": "height",
-                    "type": "INT",
-                    "widget": {
-                        "name": "height"
-                    },
-                    "link": null
-                }
-            ],
-            "outputs": [
-                {
-                    "localized_name": "image",
-                    "name": "image",
-                    "type": "IMAGE",
-                    "links": [
-                        1
-                    ]
-                }
-            ],
-            "properties": {
-                "Node name for S&R": "VLLMOmniGenerateImage"
-            },
-            "widgets_values": [
-                "http://localhost:8001/v1",
-                "/home/models/Qwen/Qwen-Image-Edit",
-                "A high-quality, high contrast, stylized portrait of the object in the uploaded reference image. Pop art and doodle style, with abundant scribbling patterns, such as a teal crown, orange lightning bolts, colorful handwritten scripts.",
-                "Realistic",
-                800,
-                800
-            ]
-        },
-        {
-            "id": 6,
-            "type": "VLLMOmniGenerateImage",
-            "pos": [
-                541.2702717429818,
-                -160.70392744708548
-            ],
-            "size": [
-                416.56628685610167,
-                372.1662621294205
-            ],
-            "flags": {},
-            "order": 2,
-            "mode": 0,
-            "inputs": [
-                {
-                    "localized_name": "image",
-                    "name": "image",
-                    "shape": 7,
-                    "type": "IMAGE",
-                    "link": null
-                },
-                {
-                    "localized_name": "mask",
-                    "name": "mask",
-                    "shape": 7,
-                    "type": "MASK",
-                    "link": null
-                },
-                {
-                    "localized_name": "sampling_params",
-                    "name": "sampling_params",
-                    "shape": 7,
-                    "type": "SAMPLING_PARAMS",
-                    "link": 5
-                },
-                {
-                    "localized_name": "url",
-                    "name": "url",
-                    "type": "STRING",
-                    "widget": {
-                        "name": "url"
-                    },
-                    "link": null
-                },
-                {
-                    "localized_name": "model",
-                    "name": "model",
-                    "type": "STRING",
-                    "widget": {
-                        "name": "model"
-                    },
-                    "link": null
-                },
-                {
-                    "localized_name": "prompt",
-                    "name": "prompt",
-                    "type": "STRING",
-                    "widget": {
-                        "name": "prompt"
-                    },
-                    "link": null
-                },
-                {
-                    "localized_name": "negative_prompt",
-                    "name": "negative_prompt",
-                    "type": "STRING",
-                    "widget": {
-                        "name": "negative_prompt"
-                    },
-                    "link": null
-                },
-                {
-                    "localized_name": "width",
-                    "name": "width",
-                    "type": "INT",
-                    "widget": {
-                        "name": "width"
-                    },
-                    "link": null
-                },
-                {
-                    "localized_name": "height",
-                    "name": "height",
-                    "type": "INT",
-                    "widget": {
-                        "name": "height"
-                    },
-                    "link": null
-                }
-            ],
-            "outputs": [
-                {
-                    "localized_name": "image",
-                    "name": "image",
-                    "type": "IMAGE",
-                    "links": [
-                        4,
-                        6
-                    ]
-                }
-            ],
-            "properties": {
-                "Node name for S&R": "VLLMOmniGenerateImage"
-            },
-            "widgets_values": [
-                "http://localhost:8000/v1",
-                "/home/models/Tongyi-MAI/Z-Image-Turbo",
-                "A headshot of a cute Siamese kitty. Blurred background due to wide aperture. Close-up look. Realistic.",
-                "Cartoonish.",
-                800,
-                800
-            ]
-        }
-    ],
-    "links": [
-        [
-            1,
-            5,
-            0,
-            1,
-            0,
-            "IMAGE"
-        ],
-        [
-            2,
-            4,
-            0,
-            5,
-            2,
-            "SAMPLING_PARAMS"
-        ],
-        [
-            4,
-            6,
-            0,
-            5,
-            0,
-            "IMAGE"
-        ],
-        [
-            5,
-            7,
-            0,
-            6,
-            2,
-            "SAMPLING_PARAMS"
-        ],
-        [
-            6,
-            6,
-            0,
-            8,
-            0,
-            "IMAGE"
-        ]
-    ],
-    "groups": [],
-    "config": {},
-    "extra": {
-        "workflowRendererVersion": "LG",
-        "ds": {
-            "scale": 1.0426432563169903,
-            "offset": [
-                -90.4071374799346,
-                575.2948920069742
-            ]
-        }
-    },
-    "version": 0.4
-}

apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni Image Generation.json

@@ -1 +0,0 @@
-{"id":"91f75acc-8040-40f6-865a-2e8a7cfd6672","revision":0,"last_node_id":11,"last_link_id":23,"nodes":[{"id":3,"type":"PreviewImage","pos":[1281.8455167767304,-69.02638461454333],"size":[305.6628685610174,307.0336884172169],"flags":{},"order":4,"mode":0,"inputs":[{"localized_name":"图像","name":"images","type":"IMAGE","link":22}],"outputs":[],"properties":{"Node name for S&R":"PreviewImage"},"widgets_values":[]},{"id":11,"type":"VLLMOmniGenerateImage","pos":[816.0962515873642,-137.51112584854445],"size":[400,278],"flags":{},"order":3,"mode":0,"inputs":[{"localized_name":"image","name":"image","shape":7,"type":"IMAGE","link":21},{"localized_name":"mask","name":"mask","shape":7,"type":"MASK","link":null},{"localized_name":"sampling_params","name":"sampling_params","shape":7,"type":"SAMPLING_PARAMS","link":23},{"localized_name":"url","name":"url","type":"STRING","widget":{"name":"url"},"link":null},{"localized_name":"model","name":"model","type":"STRING","widget":{"name":"model"},"link":null},{"localized_name":"prompt","name":"prompt","type":"STRING","widget":{"name":"prompt"},"link":null},{"localized_name":"negative_prompt","name":"negative_prompt","type":"STRING","widget":{"name":"negative_prompt"},"link":null},{"localized_name":"width","name":"width","type":"INT","widget":{"name":"width"},"link":null},{"localized_name":"height","name":"height","type":"INT","widget":{"name":"height"},"link":null}],"outputs":[{"localized_name":"image","name":"image","type":"IMAGE","links":[22]}],"properties":{"Node name for S&R":"VLLMOmniGenerateImage"},"widgets_values":["http://localhost:8000/v1","Qwen/Qwen-Image-Edit","Put this figure in a realistic mountain view","",512,512]},{"id":10,"type":"MarkdownNote","pos":[227.99306819575278,-231.24143306069843],"size":[240.3326478922474,136.3505820791881],"flags":{},"order":1,"mode":0,"inputs":[],"outputs":[],"title":"Note: Task and Input","properties":{},"widgets_values":["vLLM-Omni nodes are categorized based on the output modality. The \"Generate Image\" node supports both text-to-image generation or image-to-image generation (a.k.a. image editing). The node will route to the correct endpoint depending on whether an input image is present or not."],"color":"#432","bgcolor":"#000"},{"id":4,"type":"LoadImage","pos":[496.31859627609606,-229.71277089860084],"size":[270,314],"flags":{},"order":2,"mode":0,"inputs":[{"localized_name":"图像","name":"image","type":"COMBO","widget":{"name":"image"},"link":null},{"localized_name":"选择文件上传","name":"upload","type":"IMAGEUPLOAD","widget":{"name":"upload"},"link":null}],"outputs":[{"localized_name":"图像","name":"IMAGE","type":"IMAGE","links":[21]},{"localized_name":"遮罩","name":"MASK","type":"MASK","links":null}],"properties":{"Node name for S&R":"LoadImage"},"widgets_values":["example.png","image"]},{"id":8,"type":"VLLMOmniDiffusionSampling","pos":[478.59266934006774,183.67711984955648],"size":[284.205078125,226],"flags":{},"order":0,"mode":0,"inputs":[{"localized_name":"n","name":"n","type":"INT","widget":{"name":"n"},"link":null},{"localized_name":"num_inference_steps","name":"num_inference_steps","type":"INT","widget":{"name":"num_inference_steps"},"link":null},{"localized_name":"guidance_scale","name":"guidance_scale","type":"FLOAT","widget":{"name":"guidance_scale"},"link":null},{"localized_name":"true_cfg_scale","name":"true_cfg_scale","type":"FLOAT","widget":{"name":"true_cfg_scale"},"link":null},{"localized_name":"vae_use_slicing","name":"vae_use_slicing","type":"BOOLEAN","widget":{"name":"vae_use_slicing"},"link":null},{"localized_name":"vae_use_tiling","name":"vae_use_tiling","type":"BOOLEAN","widget":{"name":"vae_use_tiling"},"link":null},{"localized_name":"seed","name":"seed","type":"INT","widget":{"name":"seed"},"link":null}],"outputs":[{"localized_name":"diffusion sampling params","name":"diffusion sampling params","type":"SAMPLING_PARAMS","links":[23]}],"properties":{"Node name for S&R":"VLLMOmniDiffusionSampling"},"widgets_values":[4,50,1,1,false,false,42,"randomize"]}],"links":[[21,4,0,11,0,"IMAGE"],[22,11,0,3,0,"IMAGE"],[23,8,0,11,2,"SAMPLING_PARAMS"]],"groups":[{"id":1,"title":"Input","bounding":[213.1706010087147,-313.20095750667554,560.8246471437027,407.4246532472182],"color":"#3f789e","font_size":24,"flags":{}}],"config":{},"extra":{"workflowRendererVersion":"LG","ds":{"scale":1.1469075819486894,"offset":[-55.18047513099182,220.2553505195962]}},"version":0.4}

apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni Multimodal Understanding.json

@@ -1,761 +0,0 @@
-{
-    "id": "1c99f525-0a37-45ba-a28a-7df7c3af66b4",
-    "revision": 0,
-    "last_node_id": 12,
-    "last_link_id": 14,
-    "nodes": [
-        {
-            "id": 1,
-            "type": "VLLMOmniUnderstanding",
-            "pos": [
-                1191.2177053682556,
-                144.66829928181377
-            ],
-            "size": [
-                400,
-                268
-            ],
-            "flags": {},
-            "order": 8,
-            "mode": 0,
-            "inputs": [
-                {
-                    "localized_name": "image",
-                    "name": "image",
-                    "shape": 7,
-                    "type": "IMAGE",
-                    "link": 1
-                },
-                {
-                    "localized_name": "video",
-                    "name": "video",
-                    "shape": 7,
-                    "type": "VIDEO",
-                    "link": 2
-                },
-                {
-                    "localized_name": "audio",
-                    "name": "audio",
-                    "shape": 7,
-                    "type": "AUDIO",
-                    "link": 3
-                },
-                {
-                    "localized_name": "sampling_params",
-                    "name": "sampling_params",
-                    "shape": 7,
-                    "type": "SAMPLING_PARAMS",
-                    "link": 14
-                },
-                {
-                    "localized_name": "url",
-                    "name": "url",
-                    "type": "STRING",
-                    "widget": {
-                        "name": "url"
-                    },
-                    "link": null
-                },
-                {
-                    "localized_name": "model",
-                    "name": "model",
-                    "type": "STRING",
-                    "widget": {
-                        "name": "model"
-                    },
-                    "link": null
-                },
-                {
-                    "localized_name": "prompt",
-                    "name": "prompt",
-                    "type": "STRING",
-                    "widget": {
-                        "name": "prompt"
-                    },
-                    "link": null
-                },
-                {
-                    "localized_name": "output_text",
-                    "name": "output_text",
-                    "type": "BOOLEAN",
-                    "widget": {
-                        "name": "output_text"
-                    },
-                    "link": null
-                },
-                {
-                    "localized_name": "output_audio",
-                    "name": "output_audio",
-                    "type": "BOOLEAN",
-                    "widget": {
-                        "name": "output_audio"
-                    },
-                    "link": null
-                },
-                {
-                    "localized_name": "use_audio_in_video",
-                    "name": "use_audio_in_video",
-                    "type": "BOOLEAN",
-                    "widget": {
-                        "name": "use_audio_in_video"
-                    },
-                    "link": null
-                }
-            ],
-            "outputs": [
-                {
-                    "localized_name": "text_response",
-                    "name": "text_response",
-                    "type": "STRING",
-                    "links": [
-                        8
-                    ]
-                },
-                {
-                    "localized_name": "audio_response",
-                    "name": "audio_response",
-                    "type": "AUDIO",
-                    "links": [
-                        9
-                    ]
-                }
-            ],
-            "properties": {
-                "Node name for S&R": "VLLMOmniUnderstanding"
-            },
-            "widgets_values": [
-                "http://localhost:8000/v1",
-                "Qwen/Qwen2.5-Omni-7B",
-                "",
-                true,
-                true,
-                true
-            ]
-        },
-        {
-            "id": 3,
-            "type": "LoadVideo",
-            "pos": [
-                729.5984141255855,
-                -198.631920454299
-            ],
-            "size": [
-                282.798828125,
-                233.0743408203125
-            ],
-            "flags": {},
-            "order": 0,
-            "mode": 0,
-            "inputs": [
-                {
-                    "localized_name": "file",
-                    "name": "file",
-                    "type": "COMBO",
-                    "widget": {
-                        "name": "file"
-                    },
-                    "link": null
-                },
-                {
-                    "localized_name": "choose file to upload",
-                    "name": "upload",
-                    "type": "IMAGEUPLOAD",
-                    "widget": {
-                        "name": "upload"
-                    },
-                    "link": null
-                }
-            ],
-            "outputs": [
-                {
-                    "localized_name": "VIDEO",
-                    "name": "VIDEO",
-                    "type": "VIDEO",
-                    "links": [
-                        2
-                    ]
-                }
-            ],
-            "properties": {
-                "Node name for S&R": "LoadVideo"
-            },
-            "widgets_values": [
-                "draw.mp4",
-                "image"
-            ]
-        },
-        {
-            "id": 4,
-            "type": "LoadAudio",
-            "pos": [
-                729.8037086965753,
-                99.86963519703949
-            ],
-            "size": [
-                282.798828125,
-                136
-            ],
-            "flags": {},
-            "order": 1,
-            "mode": 0,
-            "inputs": [
-                {
-                    "localized_name": "audio",
-                    "name": "audio",
-                    "type": "COMBO",
-                    "widget": {
-                        "name": "audio"
-                    },
-                    "link": null
-                },
-                {
-                    "localized_name": "audioUI",
-                    "name": "audioUI",
-                    "type": "AUDIO_UI",
-                    "widget": {
-                        "name": "audioUI"
-                    },
-                    "link": null
-                },
-                {
-                    "localized_name": "choose file to upload",
-                    "name": "upload",
-                    "type": "AUDIOUPLOAD",
-                    "widget": {
-                        "name": "upload"
-                    },
-                    "link": null
-                }
-            ],
-            "outputs": [
-                {
-                    "localized_name": "AUDIO",
-                    "name": "AUDIO",
-                    "type": "AUDIO",
-                    "links": [
-                        3
-                    ]
-                }
-            ],
-            "properties": {
-                "Node name for S&R": "LoadAudio"
-            },
-            "widgets_values": [
-                "Megan-Fox.mp3",
-                null,
-                null
-            ]
-        },
-        {
-            "id": 5,
-            "type": "VLLMOmniARSampling",
-            "pos": [
-                510.3517536642828,
-                658.073751009259
-            ],
-            "size": [
-                270,
-                178
-            ],
-            "flags": {},
-            "order": 2,
-            "mode": 0,
-            "inputs": [
-                {
-                    "localized_name": "max_tokens",
-                    "name": "max_tokens",
-                    "type": "INT",
-                    "widget": {
-                        "name": "max_tokens"
-                    },
-                    "link": null
-                },
-                {
-                    "localized_name": "temperature",
-                    "name": "temperature",
-                    "type": "FLOAT",
-                    "widget": {
-                        "name": "temperature"
-                    },
-                    "link": null
-                },
-                {
-                    "localized_name": "top_p",
-                    "name": "top_p",
-                    "type": "FLOAT",
-                    "widget": {
-                        "name": "top_p"
-                    },
-                    "link": null
-                },
-                {
-                    "localized_name": "repetition_penalty",
-                    "name": "repetition_penalty",
-                    "type": "FLOAT",
-                    "widget": {
-                        "name": "repetition_penalty"
-                    },
-                    "link": null
-                },
-                {
-                    "localized_name": "seed",
-                    "name": "seed",
-                    "type": "INT",
-                    "widget": {
-                        "name": "seed"
-                    },
-                    "link": null
-                }
-            ],
-            "outputs": [
-                {
-                    "localized_name": "AR sampling params",
-                    "name": "AR sampling params",
-                    "type": "SAMPLING_PARAMS",
-                    "links": [
-                        12
-                    ]
-                }
-            ],
-            "properties": {
-                "Node name for S&R": "VLLMOmniARSampling"
-            },
-            "widgets_values": [
-                100,
-                1,
-                1,
-                1,
-                -1,
-                "randomize"
-            ]
-        },
-        {
-            "id": 7,
-            "type": "VLLMOmniARSampling",
-            "pos": [
-                503.33235181647115,
-                419.34158016181806
-            ],
-            "size": [
-                270,
-                178
-            ],
-            "flags": {},
-            "order": 3,
-            "mode": 0,
-            "inputs": [
-                {
-                    "localized_name": "max_tokens",
-                    "name": "max_tokens",
-                    "type": "INT",
-                    "widget": {
-                        "name": "max_tokens"
-                    },
-                    "link": null
-                },
-                {
-                    "localized_name": "temperature",
-                    "name": "temperature",
-                    "type": "FLOAT",
-                    "widget": {
-                        "name": "temperature"
-                    },
-                    "link": null
-                },
-                {
-                    "localized_name": "top_p",
-                    "name": "top_p",
-                    "type": "FLOAT",
-                    "widget": {
-                        "name": "top_p"
-                    },
-                    "link": null
-                },
-                {
-                    "localized_name": "repetition_penalty",
-                    "name": "repetition_penalty",
-                    "type": "FLOAT",
-                    "widget": {
-                        "name": "repetition_penalty"
-                    },
-                    "link": null
-                },
-                {
-                    "localized_name": "seed",
-                    "name": "seed",
-                    "type": "INT",
-                    "widget": {
-                        "name": "seed"
-                    },
-                    "link": null
-                }
-            ],
-            "outputs": [
-                {
-                    "localized_name": "AR sampling params",
-                    "name": "AR sampling params",
-                    "type": "SAMPLING_PARAMS",
-                    "links": [
-                        5,
-                        13
-                    ]
-                }
-            ],
-            "properties": {
-                "Node name for S&R": "VLLMOmniARSampling"
-            },
-            "widgets_values": [
-                100,
-                1,
-                1,
-                1,
-                -1,
-                "randomize"
-            ]
-        },
-        {
-            "id": 8,
-            "type": "VLLMOmniSamplingParamsList",
-            "pos": [
-                820.6056617389042,
-                426.38372037182273
-            ],
-            "size": [
-                263.066015625,
-                66
-            ],
-            "flags": {},
-            "order": 7,
-            "mode": 0,
-            "inputs": [
-                {
-                    "localized_name": "param1",
-                    "name": "param1",
-                    "type": "SAMPLING_PARAMS",
-                    "link": 5
-                },
-                {
-                    "localized_name": "param2",
-                    "name": "param2",
-                    "shape": 7,
-                    "type": "SAMPLING_PARAMS",
-                    "link": 12
-                },
-                {
-                    "localized_name": "param3",
-                    "name": "param3",
-                    "shape": 7,
-                    "type": "SAMPLING_PARAMS",
-                    "link": 13
-                }
-            ],
-            "outputs": [
-                {
-                    "localized_name": "param list",
-                    "name": "param list",
-                    "type": "SAMPLING_PARAMS",
-                    "links": [
-                        14
-                    ]
-                }
-            ],
-            "properties": {
-                "Node name for S&R": "VLLMOmniSamplingParamsList"
-            },
-            "widgets_values": []
-        },
-        {
-            "id": 11,
-            "type": "MarkdownNote",
-            "pos": [
-                826.2328280438272,
-                569.1890318701705
-            ],
-            "size": [
-                333.8220435590464,
-                261.63596728060656
-            ],
-            "flags": {},
-            "order": 4,
-            "mode": 0,
-            "inputs": [],
-            "outputs": [],
-            "title": "Note: Sampling Parameters",
-            "properties": {},
-            "widgets_values": [
-                "## Sampling Parameter Types\n\nThere are two types of sampling parameters: one for autoregression and one for diffusion.\nYou should ensure that you have chosen the correct type of sampling parameters for the model you request.\n\n## Stages & Shorthand\n\nFor multi-stage models such as Qwen Omni, you can either\n- connect one sampling parameter node, which is applied to all stages.\n- connect exactly the same number of sampling parameter nodes to a \"Multi-Stage Sampling Parameter List\" node, then connect this node to the primary request node.\n\nNote that this shorthand is intended to stay consistent with the [online serving API](https://docs.vllm.ai/projects/vllm-omni/en/latest/user_guide/examples/online_serving/qwen2_5_omni/)"
-            ],
-            "color": "#432",
-            "bgcolor": "#000"
-        },
-        {
-            "id": 12,
-            "type": "MarkdownNote",
-            "pos": [
-                378.9866207003777,
-                152.59550252215752
-            ],
-            "size": [
-                319.7287574247016,
-                107.15904081906785
-            ],
-            "flags": {},
-            "order": 6,
-            "mode": 0,
-            "inputs": [],
-            "outputs": [],
-            "title": "Note: Input",
-            "properties": {},
-            "widgets_values": [
-                "Note that not all models support every modality as input. For example, `ByteDance-Seed/BAGEL-7B-MoT` in Multimodality Understanding mode only support text and image input.\n\nYou should ensure that the input are supported by the model. You can check the corresponding [online serving documentation](https://docs.vllm.ai/projects/vllm-omni/en/latest/user_guide/examples/online_serving/bagel/) for confirmation."
-            ],
-            "color": "#432",
-            "bgcolor": "#000"
-        },
-        {
-            "id": 2,
-            "type": "LoadImage",
-            "pos": [
-                394.4674804308822,
-                -207.6987397548834
-            ],
-            "size": [
-                282.798828125,
-                314
-            ],
-            "flags": {},
-            "order": 5,
-            "mode": 0,
-            "inputs": [
-                {
-                    "localized_name": "image",
-                    "name": "image",
-                    "type": "COMBO",
-                    "widget": {
-                        "name": "image"
-                    },
-                    "link": null
-                },
-                {
-                    "localized_name": "choose file to upload",
-                    "name": "upload",
-                    "type": "IMAGEUPLOAD",
-                    "widget": {
-                        "name": "upload"
-                    },
-                    "link": null
-                }
-            ],
-            "outputs": [
-                {
-                    "localized_name": "IMAGE",
-                    "name": "IMAGE",
-                    "type": "IMAGE",
-                    "links": [
-                        1
-                    ]
-                },
-                {
-                    "localized_name": "MASK",
-                    "name": "MASK",
-                    "type": "MASK",
-                    "links": null
-                }
-            ],
-            "properties": {
-                "Node name for S&R": "LoadImage"
-            },
-            "widgets_values": [
-                "example.png",
-                "image"
-            ]
-        },
-        {
-            "id": 10,
-            "type": "PreviewAudio",
-            "pos": [
-                1664.548345556043,
-                297.5921292054054
-            ],
-            "size": [
-                270,
-                88
-            ],
-            "flags": {},
-            "order": 10,
-            "mode": 0,
-            "inputs": [
-                {
-                    "localized_name": "audio",
-                    "name": "audio",
-                    "type": "AUDIO",
-                    "link": 9
-                },
-                {
-                    "localized_name": "audioUI",
-                    "name": "audioUI",
-                    "type": "AUDIO_UI",
-                    "widget": {
-                        "name": "audioUI"
-                    },
-                    "link": null
-                }
-            ],
-            "outputs": [],
-            "properties": {
-                "Node name for S&R": "PreviewAudio"
-            },
-            "widgets_values": []
-        },
-        {
-            "id": 9,
-            "type": "ShowText|pysssss",
-            "pos": [
-                1649.2506875091847,
-                66.22823888292349
-            ],
-            "size": [
-                318.7188464232943,
-                173.38502269972975
-            ],
-            "flags": {},
-            "order": 9,
-            "mode": 0,
-            "inputs": [
-                {
-                    "localized_name": "text",
-                    "name": "text",
-                    "type": "STRING",
-                    "link": 8
-                }
-            ],
-            "outputs": [
-                {
-                    "localized_name": "STRING",
-                    "name": "STRING",
-                    "shape": 6,
-                    "type": "STRING",
-                    "links": null
-                }
-            ],
-            "properties": {
-                "Node name for S&R": "ShowText|pysssss"
-            },
-            "widgets_values": []
-        }
-    ],
-    "links": [
-        [
-            1,
-            2,
-            0,
-            1,
-            0,
-            "IMAGE"
-        ],
-        [
-            2,
-            3,
-            0,
-            1,
-            1,
-            "VIDEO"
-        ],
-        [
-            3,
-            4,
-            0,
-            1,
-            2,
-            "AUDIO"
-        ],
-        [
-            5,
-            7,
-            0,
-            8,
-            0,
-            "SAMPLING_PARAMS"
-        ],
-        [
-            8,
-            1,
-            0,
-            9,
-            0,
-            "STRING"
-        ],
-        [
-            9,
-            1,
-            1,
-            10,
-            0,
-            "AUDIO"
-        ],
-        [
-            12,
-            5,
-            0,
-            8,
-            1,
-            "SAMPLING_PARAMS"
-        ],
-        [
-            13,
-            7,
-            0,
-            8,
-            2,
-            "SAMPLING_PARAMS"
-        ],
-        [
-            14,
-            8,
-            0,
-            1,
-            3,
-            "SAMPLING_PARAMS"
-        ]
-    ],
-    "groups": [
-        {
-            "id": 1,
-            "title": "Sampling Parameters",
-            "bounding": [
-                480.1649301181556,
-                341.08402513937995,
-                692.4113568972277,
-                510.48648853403665
-            ],
-            "color": "#3f789e",
-            "font_size": 24,
-            "flags": {}
-        },
-        {
-            "id": 2,
-            "title": "Input",
-            "bounding": [
-                344.2364817813528,
-                -287.5850484183313,
-                704.8832238768634,
-                559.5124009832894
-            ],
-            "color": "#3f789e",
-            "font_size": 24,
-            "flags": {}
-        }
-    ],
-    "config": {},
-    "extra": {
-        "workflowRendererVersion": "LG",
-        "ds": {
-            "scale": 0.9478575057427204,
-            "offset": [
-                33.81037136029557,
-                307.1974296197726
-            ]
-        }
-    },
-    "version": 0.4
-}

apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni TTS.json

@@ -1 +0,0 @@
-{"id":"c0a11241-f0e6-4f5c-8488-b3d344a9df3d","revision":0,"last_node_id":7,"last_link_id":3,"nodes":[{"id":4,"type":"PreviewAudio","pos":[690.1759063763212,226.76444957981545],"size":[270,88],"flags":{},"order":5,"mode":0,"inputs":[{"localized_name":"audio","name":"audio","type":"AUDIO","link":2},{"localized_name":"audioUI","name":"audioUI","type":"AUDIO_UI","widget":{"name":"audioUI"},"link":null}],"outputs":[],"properties":{"Node name for S&R":"PreviewAudio"},"widgets_values":[]},{"id":5,"type":"LoadAudio","pos":[-83.61093919682321,103.80510898465587],"size":[282.798828125,136],"flags":{},"order":0,"mode":0,"inputs":[{"localized_name":"audio","name":"audio","type":"COMBO","widget":{"name":"audio"},"link":null},{"localized_name":"audioUI","name":"audioUI","type":"AUDIO_UI","widget":{"name":"audioUI"},"link":null},{"localized_name":"choose file to upload","name":"upload","type":"AUDIOUPLOAD","widget":{"name":"upload"},"link":null}],"outputs":[{"localized_name":"AUDIO","name":"AUDIO","type":"AUDIO","links":[3]}],"properties":{"Node name for S&R":"LoadAudio"},"widgets_values":["Megan-Fox.mp3",null,null]},{"id":3,"type":"VLLMOmniVoiceClone","pos":[248.63466280064262,168.13112893706375],"size":[400,306],"flags":{},"order":4,"mode":0,"inputs":[{"localized_name":"ref_audio","name":"ref_audio","type":"AUDIO","link":3},{"localized_name":"model_specific_params","name":"model_specific_params","shape":7,"type":"TTS_PARAMS","link":1},{"localized_name":"url","name":"url","type":"STRING","widget":{"name":"url"},"link":null},{"localized_name":"model","name":"model","type":"STRING","widget":{"name":"model"},"link":null},{"localized_name":"input","name":"input","type":"STRING","widget":{"name":"input"},"link":null},{"localized_name":"voice","name":"voice","type":"STRING","widget":{"name":"voice"},"link":null},{"localized_name":"response_format","name":"response_format","type":"COMBO","widget":{"name":"response_format"},"link":null},{"localized_name":"speed","name":"speed","type":"FLOAT","widget":{"name":"speed"},"link":null},{"localized_name":"ref_text","name":"ref_text","type":"STRING","widget":{"name":"ref_text"},"link":null},{"localized_name":"x_vector_only_mode","name":"x_vector_only_mode","type":"BOOLEAN","widget":{"name":"x_vector_only_mode"},"link":null}],"outputs":[{"localized_name":"audio","name":"audio","type":"AUDIO","links":[2]}],"properties":{"Node name for S&R":"VLLMOmniVoiceClone"},"widgets_values":["http://localhost:8000/v1","Qwen/Qwen3-TTS-12Hz-1.7B-Base","Someone just spilled a cup of coffee on my jacket this morning in the subway. Now I have to wear this stained jacket for a whole day!","","mp3",1,"",true]},{"id":6,"type":"MarkdownNote","pos":[255.52667408769662,27.26842775454861],"size":[381.58204108637847,88],"flags":{},"order":2,"mode":0,"inputs":[],"outputs":[],"title":"Note: TTS Nodes","properties":{},"widgets_values":["Apart from the Voice Cloning node, there is also another TTS node for simple text-to-speech tasks (without extra inputs for reference audio)."],"color":"#432","bgcolor":"#000"},{"id":7,"type":"MarkdownNote","pos":[-196.99453003423977,598.3693693727904],"size":[382.4539509209102,88],"flags":{},"order":3,"mode":0,"inputs":[],"outputs":[],"title":"Note: Model-Specific Parameters","properties":{},"widgets_values":["TTS models often require some tailor-made parameters. If you need to customize these parameters, grab one that matches the model you are requesting from \"TTS Params\" subfolder."],"color":"#432","bgcolor":"#000"},{"id":2,"type":"VLLMOmniQwenTTSParams","pos":[-207.8925310560497,342.7942867650348],"size":[400,200],"flags":{},"order":1,"mode":0,"inputs":[{"localized_name":"task_type","name":"task_type","type":"COMBO","widget":{"name":"task_type"},"link":null},{"localized_name":"language","name":"language","type":"COMBO","widget":{"name":"language"},"link":null},{"localized_name":"instructions","name":"instructions","type":"STRING","widget":{"name":"instructions"},"link":null},{"localized_name":"max_new_tokens","name":"max_new_tokens","type":"INT","widget":{"name":"max_new_tokens"},"link":null}],"outputs":[{"localized_name":"Qwen TTS params","name":"Qwen TTS params","type":"TTS_PARAMS","links":[1]}],"properties":{"Node name for S&R":"VLLMOmniQwenTTSParams"},"widgets_values":["Base","Auto","Super angry",2048]}],"links":[[1,2,0,3,1,"TTS_PARAMS"],[2,3,0,4,0,"AUDIO"],[3,5,0,3,0,"AUDIO"]],"groups":[{"id":1,"title":"Model-Specific Parameters","bounding":[-228.38328407738007,264.42790274716276,443.9887293164586,439.6853171248897],"color":"#3f789e","font_size":24,"flags":{}}],"config":{},"extra":{"workflowRendererVersion":"LG","ds":{"scale":1.1469075819486911,"offset":[898.8819468322333,258.7179979718389]}},"version":0.4}

apps/ComfyUI-vLLM-Omni/web/main.js

@@ -1,21 +0,0 @@
-/**
- * @file This file is intended to add dynamic fields to vLLM-Omni nodes
- * based on widget (in-node form fields) and input (connection link) values and changes.
- * However, this functionality is currently disabled/commented out.
- * Because it introduces too much complexity,
- * and it may even conflict with the current backend (Python) validation for unknown reasons (pending ComfyUI upstream fixes).
- */
-
-import { app } from "../../scripts/app.js";
-app.registerExtension({
-    name: "vllm.vllm_omni",
-    async beforeRegisterNodeDef(nodeType, nodeData, app) {
-        if (!nodeData.name.startsWith("VLLMOmni")) {
-            return
-        }
-        // Stub frontend plugin for now
-    },
-    async setup() {
-        console.info("vLLM-Omni Setup complete!")
-    },
-})

benchmarks/README.md

@@ -1,42 +0,0 @@
-# Benchmarks Overview and Architecture
-
-This document explains the benchmark architecture across all benchmark assets in this repo. It describes what we measure, and where to find or plug in new scenarios. Per-task details remain in subfolder READMEs (e.g., `benchmarks/<model>/README.md`).
-
-## Scope and goals
-- Establish repeatable latency/throughput measurements for multimodal LLM pipelines.
-- Provide both HF Transformers (offline) and vLLM-Omni (multi-stage/pipeline) baselines.
-- Make it easy to plug in new datasets and models with minimal changes to the runner scripts.
-
-## Dataset and inputs
-- Default example: SeedTTS top-100 prompts (`benchmarks/build_dataset/top100.txt`) via `benchmarks/build_dataset/`.
-- Extensible: drop in new prompt files or modality-aligned payloads; keep the expected format for the consuming scripts (e.g., one prompt per line).
-- If you add a new dataset, document it under `benchmarks/<model>/README.md` and point scripts to your data path.
-
-## Directory layout
-- `benchmarks/build_dataset/` — dataset prep utilities (e.g., SeedTTS top100).
-- `benchmarks/<model>/vllm_omni/` — vLLM-Omni pipeline benchmarks, logs, outputs.
-- Add new tasks under `benchmarks/<model>/...` with the same pattern: `transformers/`, `vllm_omni/`, task-specific README, and (optionally) dataset prep notes.
-
-## Reference workflows
-- **HF Transformers (offline, single process)**  
-  Script (example): `benchmarks/<model>/transformers/eval_qwen3_moe_omni_transformers.sh`  
-  Outputs: `benchmark_results/perf_stats.json`, `benchmark_results/results.json`, `benchmark_results/audio/` (if audio is produced).
-
-- **vLLM-Omni end-to-end pipeline**  
-  Script (example): `benchmarks/<model>/vllm_omni/eval_qwen3_moe_omni.sh`  
-  Outputs: `vllm_omni/logs/*.stats.jsonl` (per-stage/overall latency & TPS), `vllm_omni/logs/stage*.log`, `vllm_omni/outputs/` (text/audio artifacts).
-
-- **Adding a new task/model**  
-  1) Create `benchmarks/<model>/transformers/` and/or `benchmarks/<model>/vllm_omni/` with scripts referencing your model and dataset.  
-  2) Add a task README describing dataset, configs, and expected outputs.  
-  3) Keep the output/log structure similar for easy comparison (perf_stats/results/audio or text outputs; stats.jsonl/logs for pipeline).
-
-## Metrics to watch
-- **Throughput**: `overall_tps`, `*_tps_avg` per stage.
-- **Latency distribution**: look for long tails in `*.stats.jsonl`.
-- **Quality/completeness**: missing outputs or errors in stage logs indicate pipeline failures or misconfigurations.
-
-## Troubleshooting
-- Verify GPU/driver/FlashAttention2 requirements for your chosen model/config.
-- Ensure network access for dataset/model downloads (Google Drive, Hugging Face, etc.).
-- If outputs are missing or slow, inspect per-stage logs and `*.stats.jsonl` for errors, stragglers, or contention.

benchmarks/build_dataset/download_process_data_seedtts.md

@@ -1,82 +0,0 @@
-# Benchmark Dataset Preparation Guide
-
-This guide describes how to download and prepare the SeedTTS test dataset for benchmarking Qwen-Omni models.
-
-## Prerequisites
-
-- Python 3.8+
-- `gdown` for downloading from Google Drive
-- Access to the benchmark scripts
-
-## Steps
-
-### 1. Navigate to the Dataset Directory
-
-```bash
-cd benchmarks/build_dataset
-```
-
-### 2. Install Dependencies
-
-```bash
-pip install gdown
-```
-
-### 3. Download the SeedTTS Test Dataset
-
-Download the dataset from Google Drive:
-
-```bash
-gdown --id 1GlSjVfSHkW3-leKKBlfrjuuTGqQ_xaLP
-```
-
-### 4. Extract the Dataset
-
-```bash
-tar -xf seedtts_testset.tar
-```
-
-### 5. Prepare the Metadata File
-
-Copy the English metadata file to the working directory:
-
-```bash
-cp seedtts_testset/en/meta.lst meta.lst
-```
-
-### 6. Extract Prompts
-
-Extract the first N prompts from the metadata file:
-
-```bash
-# Extract top 100 prompts (adjust -n for different amounts)
-python extract_tts_prompts.py -i meta.lst -o top100.txt -n 100
-```
-
-**Options:**
-- `-i, --input`: Input metadata file (default: `meta.lst`)
-- `-o, --output`: Output prompts file (default: `prompts.txt`)
-- `-n, --num-lines`: Number of prompts to extract (required)
-
-### 7. Clean Up (Optional)
-
-Remove temporary files to save disk space:
-
-```bash
-rm -rf seedtts_testset
-rm seedtts_testset.tar
-rm meta.lst
-```
-
-## Quick Start (All-in-One)
-
-```bash
-# Full setup and benchmark
-cd benchmarks/build_dataset
-pip install gdown
-gdown --id 1GlSjVfSHkW3-leKKBlfrjuuTGqQ_xaLP
-tar -xf seedtts_testset.tar
-cp seedtts_testset/en/meta.lst meta.lst
-python extract_tts_prompts.py -i meta.lst -o top100.txt -n 100
-rm -rf seedtts_testset seedtts_testset.tar meta.lst
-```

benchmarks/build_dataset/extract_tts_prompts.py

@@ -1,73 +0,0 @@
-#!/usr/bin/env python3
-"""
-Extract prompts from meta.lst and save them to a txt file.
-
-Each line in meta.lst has the format:
-ID|prompt_text|audio_path|target_text
-
-This script extracts the prompt_text (second field) from the first N lines.
-"""
-
-import argparse
-from pathlib import Path
-
-
-def extract_prompts(input_file: str, output_file: str, num_lines: int) -> None:
-    """
-    Extract prompts from meta.lst and save to output file.
-
-    Args:
-        input_file: Path to the meta.lst file
-        output_file: Path to the output txt file
-        num_lines: Number of lines to process
-    """
-    prompts = []
-
-    with open(input_file, encoding="utf-8") as f:
-        for i, line in enumerate(f):
-            if i >= num_lines:
-                break
-
-            line = line.strip()
-            if not line:  # Skip empty lines
-                continue
-
-            parts = line.split("|")
-            if len(parts) >= 2:
-                prompt = parts[1]  # The prompt is the second field
-                prompts.append(prompt)
-
-    # Write prompts to output file
-    with open(output_file, "w", encoding="utf-8") as f:
-        for prompt in prompts:
-            f.write(prompt + "\n")
-
-    # Print result stats
-    print(f"Extracted {len(prompts)} prompts from first {num_lines} lines")
-    print(f"Saved to: {output_file}")
-
-
-def main():
-    parser = argparse.ArgumentParser(description="Extract prompts from meta.lst file")
-    parser.add_argument(
-        "-i", "--input", type=str, default="meta.lst", help="Input meta.lst file path (default: meta.lst)"
-    )
-    parser.add_argument(
-        "-o", "--output", type=str, default="prompts.txt", help="Output txt file path (default: prompts.txt)"
-    )
-    parser.add_argument(
-        "-n", "--num-lines", type=int, required=True, help="Number of lines to extract from the beginning"
-    )
-
-    args = parser.parse_args()
-
-    # Check if input file exists
-    if not Path(args.input).exists():
-        print(f"Error: Input file '{args.input}' not found")
-        return
-
-    extract_prompts(args.input, args.output, args.num_lines)
-
-
-if __name__ == "__main__":
-    main()

benchmarks/diffusion/README.md

@@ -1,117 +0,0 @@
-
-# Diffusion Serving Benchmark (Image/Video)
-
-This folder contains an online-serving benchmark script for diffusion models.
-It sends requests to a vLLM OpenAI-compatible endpoint and reports throughput,
-latency percentiles, and optional SLO attainment.
-
-The main entrypoint is:
-
-- `benchmarks/diffusion/diffusion_benchmark_serving.py`
-
-## 1. Quick Start
-
-1. Start the server:
-
-```bash
-vllm serve Qwen/Qwen-Image --omni --port 8099
-```
-
-2. Run a minimal benchmark:
-
-```bash
-python3 benchmarks/diffusion/diffusion_benchmark_serving.py \
-	--base-url http://localhost:8099 \
-	--model Qwen/Qwen-Image \
-	--task t2i \
-	--dataset vbench \
-	--num-prompts 5
-```
-
-**Notes**
-
-- The benchmark talks to `http://<host>:<port>/v1/chat/completions`.
-- If you run the server on another host or port, pass `--base-url` accordingly.
-
-## 2. Supported Datasets
-
-The benchmark supports three dataset modes via `--dataset`:
-
-- `vbench`: Built-in prompt/data loader.
-- `trace`: Heterogeneous request traces (each request can have different resolution/frames/steps).
-- `random`: Synthetic prompts for quick smoke tests.
-
-### VBench dataset
-
-If you use i2v/i2i bench datasets and need auto-download support, you may need:
-
-```bash
-uv pip install gdown
-```
-
-### Trace dataset
-
-Use `--dataset trace` to replay a trace file. The trace can specify per-request fields such as:
-
-- `width`, `height`
-- `num_frames` (video)
-- `num_inference_steps`
-- `seed`, `fps`
-- optional `slo_ms` (per-request SLO target)
-
-By default (when `--dataset-path` is not provided), the script downloads a default trace from
-the HuggingFace dataset repo `asukaqaqzz/Dit_Trace`. The default filename can depend on `--task`
-(e.g., `t2v` uses a video trace).
-
-Current defaults:
-
-- `--task t2i` -> `sd3_trace.txt`
-- `--task t2v` -> `cogvideox_trace.txt`
-
-You can point to your own trace using `--dataset-path`.
-
-## 3. Benchmark Parameters
-
-### Basic flags
-
-- `--base-url`: Server address (the script calls `.../v1/chat/completions`).
-- `--model`: The OpenAI-compatible `model` field.
-- `--task`: Task type (e.g., `t2i`, `t2v`, `i2i`, `i2v`).
-- `--dataset`: Dataset mode (`vbench` / `trace` / `random`).
-- `--num-prompts`: Number of requests to send.
-
-Common optional flags:
-
-- `--output-file`: Write metrics to a JSON file.
-- `--disable-tqdm`: Disable the progress bar.
-
-### Resolution / frames / steps: CLI defaults vs dataset fields
-
-Related flags: `--width`, `--height`, `--num-frames`, `--fps`, `--num-inference-steps`.
-
-- For `vbench` / `random`: these CLI flags act as global defaults for all generated requests.
-- For `trace`: each request can carry its own fields (e.g., `width/height/num_frames/num_inference_steps`).
-
-Precedence rules for `trace` (i.e., what actually gets sent):
-
-- `width/height`: if either `--width` or `--height` is explicitly set, it overrides per-request values from the trace; otherwise per-request values are used when present.
-- `num_frames`: per-request `num_frames` takes precedence; otherwise fall back to `--num-frames`.
-- `num_inference_steps`: per-request `num_inference_steps` takes precedence; otherwise fall back to `--num-inference-steps`.
-
-### SLO, warmup, and max concurrency
-
-Enable SLO evaluation with `--slo`.
-
-- If a request in the trace already has `slo_ms`, that value is used.
-- Otherwise, the script runs warmup requests to infer a base unit time, estimates `expected_ms` by linearly scaling with area/frames/steps, and then sets `slo_ms = expected_ms * --slo-scale`.
-
-Warmup flags:
-
-- `--warmup-requests`: Number of warmup requests.
-- `--warmup-num-inference-steps`: Steps used during warmup.
-- For `--task t2v`: warmup requests are forced to use `num_frames=1` to make warmup faster and less noisy.
-
-Traffic / concurrency flags:
-
-- `--request-rate`: Target request rate (requests/second). If set to `inf`, the script sends all requests immediately.
-- `--max-concurrency`: Max number of in-flight requests (default: `1`). This can hard-cap the achieved QPS: if it is too small, requests will queue behind the semaphore, and both achieved throughput and observed SLO attainment can be skewed.

benchmarks/diffusion/backends.py

@@ -1,198 +0,0 @@
-import base64
-import mimetypes
-import os
-import time
-import uuid
-from dataclasses import dataclass, field
-from typing import Any
-
-import aiohttp
-from tqdm import tqdm
-
-
-@dataclass
-class RequestFuncInput:
-    prompt: str
-    api_url: str
-    model: str
-    width: int | None = None
-    height: int | None = None
-    num_frames: int | None = None
-    num_inference_steps: int | None = None
-    seed: int | None = None
-    fps: int | None = None
-    timestamp: float | None = None
-    slo_ms: float | None = None
-    extra_body: dict[str, Any] = field(default_factory=dict)
-    image_paths: list[str] | None = None
-    request_id: str = field(default_factory=lambda: str(uuid.uuid4()))
-
-
-@dataclass
-class RequestFuncOutput:
-    success: bool = False
-    latency: float = 0.0
-    error: str = ""
-    start_time: float = 0.0
-    response_body: dict[str, Any] = field(default_factory=dict)
-    peak_memory_mb: float = 0.0
-    slo_achieved: bool | None = None
-
-
-def _guess_mime_type(path: str) -> str:
-    mime, _ = mimetypes.guess_type(path)
-    return mime or "application/octet-stream"
-
-
-def _encode_image_as_data_url(path: str) -> str:
-    with open(path, "rb") as f:
-        encoded = base64.b64encode(f.read()).decode("utf-8")
-    mime = _guess_mime_type(path)
-    return f"data:{mime};base64,{encoded}"
-
-
-async def async_request_chat_completions(
-    input: RequestFuncInput,
-    session: aiohttp.ClientSession,
-    pbar: tqdm | None = None,
-) -> RequestFuncOutput:
-    output = RequestFuncOutput()
-    output.start_time = time.perf_counter()
-
-    extra_body = dict(input.extra_body)
-    if input.width and input.height:
-        extra_body.setdefault("height", input.height)
-        extra_body.setdefault("width", input.width)
-    if input.num_frames:
-        extra_body.setdefault("num_frames", input.num_frames)
-    if input.num_inference_steps:
-        extra_body.setdefault("num_inference_steps", input.num_inference_steps)
-    if input.seed is not None:
-        extra_body.setdefault("seed", input.seed)
-    if input.fps:
-        extra_body.setdefault("fps", input.fps)
-
-    if input.image_paths and len(input.image_paths) > 0:
-        content = []
-        if input.prompt:
-            content.append({"type": "text", "text": input.prompt})
-        for img_path in input.image_paths:
-            if not os.path.exists(img_path):
-                output.error = f"Image file not found: {img_path}"
-                output.success = False
-                if pbar:
-                    pbar.update(1)
-                return output
-            content.append(
-                {
-                    "type": "image_url",
-                    "image_url": {"url": _encode_image_as_data_url(img_path)},
-                }
-            )
-        messages = [{"role": "user", "content": content}]
-    else:
-        messages = [{"role": "user", "content": input.prompt}]
-
-    payload = {
-        "model": input.model,
-        "messages": messages,
-    }
-    if extra_body:
-        payload["extra_body"] = extra_body
-
-    try:
-        async with session.post(input.api_url, json=payload) as response:
-            if response.status == 200:
-                resp_json = await response.json()
-                output.response_body = resp_json
-                output.success = True
-                if "peak_memory_mb" in resp_json:
-                    output.peak_memory_mb = resp_json["peak_memory_mb"]
-            else:
-                output.error = f"HTTP {response.status}: {await response.text()}"
-                output.success = False
-    except Exception as e:
-        output.error = str(e)
-        output.success = False
-
-    output.latency = time.perf_counter() - output.start_time
-
-    if output.success and input.slo_ms is not None:
-        output.slo_achieved = (output.latency * 1000.0) <= float(input.slo_ms)
-
-    if pbar:
-        pbar.update(1)
-    return output
-
-
-async def async_request_openai_images(
-    input: RequestFuncInput,
-    session: aiohttp.ClientSession,
-    pbar: tqdm | None = None,
-) -> RequestFuncOutput:
-    """
-    Send request to OpenAI's /v1/images/generations endpoint.
-    """
-    output = RequestFuncOutput()
-    output.start_time = time.perf_counter()
-
-    # Build size string from width/height
-    width = input.width or 1024
-    height = input.height or 1024
-    size = f"{width}x{height}"
-
-    payload: dict[str, Any] = {
-        "model": input.model,
-        "prompt": input.prompt,
-        "n": 1,
-        "size": size,
-        "response_format": "b64_json",
-    }
-
-    # Add optional parameters
-    if input.seed is not None:
-        payload["seed"] = input.seed
-    if input.num_inference_steps is not None:
-        payload["num_inference_steps"] = input.num_inference_steps
-
-    # Add any extra body parameters
-    if input.extra_body:
-        for key, value in input.extra_body.items():
-            if key not in payload:
-                payload[key] = value
-
-    headers = {
-        "Content-Type": "application/json",
-        "Authorization": "Bearer EMPTY",
-    }
-
-    try:
-        async with session.post(input.api_url, json=payload, headers=headers) as response:
-            if response.status == 200:
-                resp_json = await response.json()
-                output.response_body = resp_json
-                output.success = True
-                # Check for usage/memory info if available
-                if "usage" in resp_json and "peak_memory_mb" in resp_json.get("usage", {}):
-                    output.peak_memory_mb = resp_json["usage"]["peak_memory_mb"]
-            else:
-                output.error = f"HTTP {response.status}: {await response.text()}"
-                output.success = False
-    except Exception as e:
-        output.error = str(e)
-        output.success = False
-
-    output.latency = time.perf_counter() - output.start_time
-
-    if output.success and input.slo_ms is not None:
-        output.slo_achieved = (output.latency * 1000.0) <= float(input.slo_ms)
-
-    if pbar:
-        pbar.update(1)
-    return output
-
-
-backends_function_mapping = {
-    "vllm-omni": (async_request_chat_completions, "/v1/chat/completions"),
-    "openai": (async_request_openai_images, "/v1/images/generations"),
-}

benchmarks/diffusion/diffusion_benchmark_serving.py

@@ -1,978 +0,0 @@
-# adapted from sglang and fastvideo
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-
-"""
-Benchmark online serving for diffusion models (Image/Video Generation).
-If you want to use i2v, i2i dataset, you should `uv pip install gdown` first
-
-Supports multiple backends:
-    - vllm-omni: Uses /v1/chat/completions endpoint (default)
-    - openai: Uses /v1/images/generations endpoint
-
-Usage:
-    # Video (vllm-omni backend)
-    t2v:
-    python3 benchmarks/diffusion/diffusion_benchmark_serving.py \
-        --backend vllm-omni --dataset vbench --task t2v --num-prompts 10 \
-        --height 480 --width 640 --fps 16 --num-frames 80
-
-    i2v:
-    python3 benchmarks/diffusion/diffusion_benchmark_serving.py \
-        --backend vllm-omni --dataset vbench --task i2v --num-prompts 10
-
-
-    # Image (vllm-omni backend)
-    t2i:
-    python3 benchmarks/diffusion/diffusion_benchmark_serving.py \
-        --backend vllm-omni --dataset vbench --task t2i --num-prompts 10 \
-        --height 1024 --width 1024
-
-    i2i:
-    python3 benchmarks/diffusion/diffusion_benchmark_serving.py \
-        --backend vllm-omni --dataset vbench --task i2i --num-prompts 10
-
-    # Image (openai backend)
-    t2i:
-    python3 benchmarks/diffusion/diffusion_benchmark_serving.py \
-        --backend openai --dataset vbench --task t2i --num-prompts 10 \
-        --height 1024 --width 1024 --port 3000
-
-"""
-
-import argparse
-import ast
-import asyncio
-import glob
-import json
-import os
-import time
-import uuid
-from abc import ABC, abstractmethod
-from collections.abc import AsyncGenerator
-from dataclasses import replace
-from typing import Any
-
-import aiohttp
-import numpy as np
-import requests
-from backends import RequestFuncInput, RequestFuncOutput, backends_function_mapping
-from tqdm.asyncio import tqdm
-
-
-class BaseDataset(ABC):
-    def __init__(self, args, api_url: str, model: str):
-        self.args = args
-        self.api_url = api_url
-        self.model = model
-
-    @abstractmethod
-    def __len__(self) -> int:
-        pass
-
-    @abstractmethod
-    def __getitem__(self, idx: int) -> RequestFuncInput:
-        pass
-
-    @abstractmethod
-    def get_requests(self) -> list[RequestFuncInput]:
-        pass
-
-
-class VBenchDataset(BaseDataset):
-    """
-    Dataset loader for VBench prompts.
-    Supports t2v, i2v.
-    """
-
-    T2V_PROMPT_URL = (
-        "https://raw.githubusercontent.com/Vchitect/VBench/master/prompts/prompts_per_dimension/subject_consistency.txt"
-    )
-    I2V_DOWNLOAD_SCRIPT_URL = (
-        "https://raw.githubusercontent.com/Vchitect/VBench/master/vbench2_beta_i2v/download_data.sh"
-    )
-
-    def __init__(self, args, api_url: str, model: str):
-        super().__init__(args, api_url, model)
-        self.cache_dir = os.path.join(os.path.expanduser("~"), ".cache", "vllm-omni")
-        self.items = self._load_data()
-
-    def _load_data(self) -> list[dict[str, Any]]:
-        if self.args.task == "t2v":
-            return self._load_t2v_prompts()
-        elif self.args.task in ["i2v", "ti2v", "ti2i", "i2i"]:
-            return self._load_i2v_data()
-        else:
-            return self._load_t2v_prompts()
-
-    def _download_file(self, url: str, dest_path: str) -> None:
-        """Download a file from URL to destination path."""
-        os.makedirs(os.path.dirname(dest_path), exist_ok=True)
-        resp = requests.get(url)
-        resp.raise_for_status()
-        with open(dest_path, "w") as f:
-            f.write(resp.text)
-
-    def _load_t2v_prompts(self) -> list[dict[str, Any]]:
-        path = self.args.dataset_path
-
-        if not path:
-            path = os.path.join(self.cache_dir, "vbench_subject_consistency.txt")
-            if not os.path.exists(path):
-                print(f"Downloading VBench T2V prompts to {path}...")
-                try:
-                    self._download_file(self.T2V_PROMPT_URL, path)
-                except Exception as e:
-                    print(f"Failed to download VBench prompts: {e}")
-                    return [{"prompt": "A cat sitting on a bench"}] * 50
-
-        prompts = []
-        with open(path) as f:
-            for line in f:
-                line = line.strip()
-                if line:
-                    prompts.append({"prompt": line})
-
-        return self._resize_data(prompts)
-
-    def _auto_download_i2v_dataset(self) -> str:
-        """Auto-download VBench I2V dataset and return the dataset directory."""
-        vbench_i2v_dir = os.path.join(self.cache_dir, "vbench_i2v", "vbench2_beta_i2v")
-        info_json_path = os.path.join(vbench_i2v_dir, "data", "i2v-bench-info.json")
-
-        if os.path.exists(info_json_path):
-            return vbench_i2v_dir
-
-        print(f"Downloading VBench I2V dataset to {vbench_i2v_dir}...")
-        try:
-            cache_root = os.path.join(self.cache_dir, "vbench_i2v")
-            script_path = os.path.join(cache_root, "download_data.sh")
-
-            self._download_file(self.I2V_DOWNLOAD_SCRIPT_URL, script_path)
-            os.chmod(script_path, 0o755)
-
-            print("Executing download_data.sh (this may take a while)...")
-            import subprocess
-
-            result = subprocess.run(
-                ["bash", script_path],
-                cwd=cache_root,
-                capture_output=True,
-                text=True,
-            )
-
-            if result.returncode != 0:
-                raise RuntimeError(f"Download script failed: {result.stderr}")
-
-            print(f"Successfully downloaded VBench I2V dataset to {vbench_i2v_dir}")
-        except Exception as e:
-            print(f"Failed to download VBench I2V dataset: {e}")
-            print("Please manually download following instructions at:")
-            print("https://github.com/Vchitect/VBench/tree/master/vbench2_beta_i2v#22-download")
-            return None
-
-        return vbench_i2v_dir if os.path.exists(info_json_path) else None
-
-    def _load_from_i2v_json(self, json_path: str) -> list[dict[str, Any]]:
-        """Load I2V data from i2v-bench-info.json format."""
-        with open(json_path) as f:
-            items = json.load(f)
-
-        base_dir = os.path.dirname(os.path.dirname(json_path))  # Go up to vbench2_beta_i2v
-        origin_dir = os.path.join(base_dir, "data", "origin")
-
-        data = []
-        for item in items:
-            img_path = os.path.join(origin_dir, item.get("file_name", ""))
-            if os.path.exists(img_path):
-                data.append({"prompt": item.get("caption", ""), "image_path": img_path})
-            else:
-                print(f"Warning: Image not found: {img_path}")
-
-        print(f"Loaded {len(data)} I2V samples from VBench I2V dataset")
-        return data
-
-    def _scan_directory_for_images(self, path: str) -> list[dict[str, Any]]:
-        """Scan directory for image files."""
-        exts = ["*.jpg", "*.jpeg", "*.png", "*.webp"]
-        files = []
-
-        for ext in exts:
-            files.extend(glob.glob(os.path.join(path, ext)))
-            files.extend(glob.glob(os.path.join(path, ext.upper())))
-
-            # Also check in data/origin subdirectory
-            origin_dir = os.path.join(path, "data", "origin")
-            if os.path.exists(origin_dir):
-                files.extend(glob.glob(os.path.join(origin_dir, ext)))
-                files.extend(glob.glob(os.path.join(origin_dir, ext.upper())))
-
-        return [{"prompt": os.path.splitext(os.path.basename(f))[0], "image_path": f} for f in files]
-
-    def _create_dummy_data(self) -> list[dict[str, Any]]:
-        """Create dummy data with a placeholder image in cache directory."""
-        print("No I2V data found. Using dummy placeholders.")
-
-        dummy_image = os.path.join(self.cache_dir, "dummy_image.jpg")
-        if not os.path.exists(dummy_image):
-            try:
-                from PIL import Image
-
-                os.makedirs(self.cache_dir, exist_ok=True)
-                img = Image.new("RGB", (100, 100), color="red")
-                img.save(dummy_image)
-                print(f"Created dummy image at {dummy_image}")
-            except ImportError:
-                print("PIL not installed, cannot create dummy image.")
-                return []
-
-        return [{"prompt": "A moving cat", "image_path": dummy_image}] * 10
-
-    def _load_i2v_data(self) -> list[dict[str, Any]]:
-        """Load I2V data from VBench I2V dataset or user-provided path."""
-        path = self.args.dataset_path
-
-        # Auto-download if no path provided
-        if not path:
-            path = self._auto_download_i2v_dataset()
-            if not path:
-                return self._resize_data(self._create_dummy_data())
-
-        # Try to load from i2v-bench-info.json
-        info_json_candidates = [
-            os.path.join(path, "data", "i2v-bench-info.json"),
-            path if path.endswith(".json") else None,
-        ]
-
-        for json_path in info_json_candidates:
-            if json_path and os.path.exists(json_path):
-                try:
-                    return self._resize_data(self._load_from_i2v_json(json_path))
-                except Exception as e:
-                    print(f"Failed to load {json_path}: {e}")
-
-        # Fallback: scan directory for images
-        if os.path.isdir(path):
-            data = self._scan_directory_for_images(path)
-            if data:
-                return self._resize_data(data)
-
-        # Last resort: dummy data
-        return self._resize_data(self._create_dummy_data())
-
-    def _resize_data(self, data: list[dict[str, Any]]) -> list[dict[str, Any]]:
-        """Resize data to match num_prompts."""
-        if not self.args.num_prompts:
-            return data
-
-        if len(data) < self.args.num_prompts:
-            factor = (self.args.num_prompts // len(data)) + 1
-            data = data * factor
-
-        return data[: self.args.num_prompts]
-
-    def __len__(self) -> int:
-        return len(self.items)
-
-    def __getitem__(self, idx: int) -> RequestFuncInput:
-        item = self.items[idx]
-        image_paths = [item["image_path"]] if "image_path" in item else None
-
-        return RequestFuncInput(
-            prompt=item.get("prompt", ""),
-            api_url=self.api_url,
-            model=self.model,
-            width=self.args.width,
-            height=self.args.height,
-            num_frames=self.args.num_frames,
-            num_inference_steps=self.args.num_inference_steps,
-            seed=self.args.seed,
-            fps=self.args.fps,
-            image_paths=image_paths,
-        )
-
-    def get_requests(self) -> list[RequestFuncInput]:
-        return [self[i] for i in range(len(self))]
-
-
-class TraceDataset(BaseDataset):
-    """Trace-based dataset loader for heterogeneous diffusion requests."""
-
-    DEFAULT_REPO_ID = "asukaqaqzz/Dit_Trace"
-    DEFAULT_FILENAME = "sd3_trace.txt"
-    DEFAULT_FILENAME_BY_TASK: dict[str, str] = {
-        # Text-to-image traces (e.g., SD3)
-        "t2i": "sd3_trace.txt",
-        # Text-to-video traces (e.g., CogVideoX)
-        "t2v": "cogvideox_trace.txt",
-    }
-
-    def __init__(self, args, api_url: str, model: str):
-        super().__init__(args, api_url, model)
-        self.cache_dir = os.path.join(os.path.expanduser("~"), ".cache", "vllm-omni", "trace")
-        self.default_filename = self.DEFAULT_FILENAME_BY_TASK.get(getattr(args, "task", ""), self.DEFAULT_FILENAME)
-        dataset_root = args.dataset_path
-        if not dataset_root:
-            dataset_root = self._download_default_trace()
-        self.items = self._load_items(dataset_root)
-
-    @staticmethod
-    def _coerce_int(x: Any) -> int | None:
-        if x is None:
-            return None
-        if isinstance(x, bool):
-            return None
-        if isinstance(x, int):
-            return x
-        try:
-            s = str(x).strip()
-            if not s:
-                return None
-            return int(float(s))
-        except Exception:
-            return None
-
-    @staticmethod
-    def _coerce_float(x: Any) -> float | None:
-        if x is None:
-            return None
-        if isinstance(x, float):
-            return x
-        if isinstance(x, int):
-            return float(x)
-        try:
-            s = str(x).strip()
-            if not s:
-                return None
-            return float(s)
-        except Exception:
-            return None
-
-    def _download_default_trace(self) -> str:
-        """Download default trace file from HuggingFace Hub if not provided."""
-
-        try:
-            from huggingface_hub import hf_hub_download
-        except ImportError as exc:
-            raise ImportError(
-                "huggingface_hub is required to download the default trace dataset. "
-                "Install via `pip install huggingface_hub`."
-            ) from exc
-
-        os.makedirs(self.cache_dir, exist_ok=True)
-        return hf_hub_download(
-            repo_id=self.DEFAULT_REPO_ID,
-            filename=self.default_filename,
-            repo_type="dataset",
-            local_dir=self.cache_dir,
-            local_dir_use_symlinks=False,
-        )
-
-    def _expand_paths(self, dataset_path: str | None) -> list[str]:
-        if not dataset_path:
-            return []
-
-        parts = [p.strip() for p in str(dataset_path).split(",") if p.strip()]
-        paths: list[str] = []
-        for p in parts:
-            if any(ch in p for ch in ["*", "?", "["]):
-                paths.extend(sorted(glob.glob(p)))
-            elif os.path.isdir(p):
-                paths.extend(sorted(glob.glob(os.path.join(p, "**", "*.txt"), recursive=True)))
-            else:
-                paths.append(p)
-
-        seen = set()
-        unique_paths = []
-        for p in paths:
-            if p not in seen:
-                seen.add(p)
-                unique_paths.append(p)
-        return unique_paths
-
-    def _parse_trace_file(self, path: str) -> list[dict[str, Any]]:
-        rows: list[dict[str, Any]] = []
-
-        def parse_request_repr_line(line: str) -> dict[str, Any] | None:
-            text = line.strip()
-            if not text:
-                return None
-            if not (text.startswith("Request(") and text.endswith(")")):
-                return None
-            inner = text[len("Request(") : -1]
-            try:
-                expr = ast.parse(f"f({inner})", mode="eval")
-                if not isinstance(expr.body, ast.Call):
-                    return None
-                call = expr.body
-                out: dict[str, Any] = {}
-                for kw in call.keywords:
-                    if kw.arg is None:
-                        continue
-                    out[kw.arg] = ast.literal_eval(kw.value)
-                return out
-            except Exception:
-                return None
-
-        # detect first non-empty line to pick parser
-        first_non_empty = None
-        with open(path, encoding="utf-8") as f:
-            for _ in range(50):
-                pos = f.tell()
-                line = f.readline()
-                if not line:
-                    break
-                if line.strip():
-                    first_non_empty = line.strip()
-                    f.seek(pos)
-                    break
-
-        if first_non_empty is None:
-            return rows
-
-        if first_non_empty.startswith("Request("):
-            with open(path, encoding="utf-8") as f:
-                for line in f:
-                    parsed = parse_request_repr_line(line)
-                    if isinstance(parsed, dict):
-                        rows.append(parsed)
-            return rows
-
-        # txt fallback: parse Request(...) lines only
-        with open(path, encoding="utf-8") as f:
-            for line in f:
-                parsed = parse_request_repr_line(line)
-                if isinstance(parsed, dict):
-                    rows.append(parsed)
-        return rows
-
-    def _load_items(self, dataset_root: str) -> list[dict[str, Any]]:
-        paths = self._expand_paths(dataset_root)
-        if not paths:
-            raise ValueError("No trace files found. Provide --dataset-path or rely on default HuggingFace download.")
-
-        items: list[dict[str, Any]] = []
-        for p in paths:
-            if not os.path.exists(p):
-                continue
-            for row in self._parse_trace_file(p):
-                if isinstance(row, dict):
-                    row = dict(row)
-                    row.setdefault("_source", p)
-                    items.append(row)
-
-        if not items:
-            raise ValueError("Trace dataset is empty after parsing provided paths.")
-
-        if self.args.num_prompts is not None:
-            items = items[: self.args.num_prompts]
-
-        return items
-
-    def __len__(self) -> int:
-        return len(self.items)
-
-    def __getitem__(self, idx: int) -> RequestFuncInput:
-        row = self.items[idx]
-        prompt = row.get("prompt") or row.get("text") or ""
-
-        row_height = self._coerce_int(row.get("height"))
-        row_width = self._coerce_int(row.get("width"))
-        num_frames = self._coerce_int(row.get("num_frames"))
-        num_steps = self._coerce_int(row.get("num_inference_steps"))
-        seed = self._coerce_int(row.get("seed"))
-        fps = self._coerce_int(row.get("fps"))
-        timestamp = self._coerce_float(row.get("timestamp"))
-        slo_ms = self._coerce_float(row.get("slo_ms"))
-        image_paths = row.get("image_paths")
-
-        override_w = self.args.width
-        override_h = self.args.height
-        if override_w is not None or override_h is not None:
-            width = override_w
-            height = override_h
-        else:
-            width = row_width
-            height = row_height
-
-        return RequestFuncInput(
-            prompt=str(prompt),
-            api_url=self.api_url,
-            model=self.model,
-            width=width,
-            height=height,
-            num_frames=num_frames if num_frames is not None else self.args.num_frames,
-            num_inference_steps=num_steps if num_steps is not None else self.args.num_inference_steps,
-            seed=seed if seed is not None else self.args.seed,
-            fps=fps if fps is not None else self.args.fps,
-            timestamp=timestamp,
-            slo_ms=slo_ms,
-            image_paths=image_paths,
-            request_id=str(row.get("request_id")) if row.get("request_id") is not None else str(uuid.uuid4()),
-        )
-
-    def get_requests(self) -> list[RequestFuncInput]:
-        return [self[i] for i in range(len(self))]
-
-
-class RandomDataset(BaseDataset):
-    def __init__(self, args, api_url: str, model: str):
-        self.args = args
-        self.api_url = api_url
-        self.model = model
-        self.num_prompts = args.num_prompts
-
-    def __len__(self) -> int:
-        return self.num_prompts
-
-    def __getitem__(self, idx: int) -> RequestFuncInput:
-        return RequestFuncInput(
-            prompt=f"Random prompt {idx} for benchmarking diffusion models",
-            api_url=self.api_url,
-            model=self.model,
-            width=self.args.width,
-            height=self.args.height,
-            num_frames=self.args.num_frames,
-            num_inference_steps=self.args.num_inference_steps,
-            seed=self.args.seed,
-            fps=self.args.fps,
-        )
-
-    def get_requests(self) -> list[RequestFuncInput]:
-        return [self[i] for i in range(len(self))]
-
-
-def _compute_expected_latency_ms_from_base(req: RequestFuncInput, args, base_time_ms: float | None) -> float | None:
-    """Compute expected execution time (ms) based on a base per-step-per-frame unit time.
-
-    Assumes linear scaling with pixel area, frame count, and num_inference_steps.
-    The base unit represents latency for a 16x16 resolution, single frame, single step.
-    """
-
-    if base_time_ms is None:
-        return None
-
-    width = req.width if req.width is not None else args.width
-    height = req.height if req.height is not None else args.height
-    if width is None or height is None:
-        return None
-
-    frames = req.num_frames if req.num_frames is not None else args.num_frames
-    steps = req.num_inference_steps if req.num_inference_steps is not None else args.num_inference_steps
-
-    frame_scale = frames if isinstance(frames, int) and frames > 0 else 1
-    step_scale = steps if isinstance(steps, int) and steps > 0 else 1
-
-    area_units = max((float(width) * float(height)) / float(16 * 16), 1.0)
-    return float(base_time_ms) * area_units * frame_scale * step_scale
-
-
-def _infer_slo_base_time_ms_from_warmups(
-    warmup_pairs: list[tuple[RequestFuncInput, RequestFuncOutput]],
-    args,
-) -> float | None:
-    """Infer base SLO unit time from warmup requests.
-
-    Returns the median base latency (ms) for a 16x16 resolution, single-frame,
-    single-step request. Only uses warmups that succeeded and have resolvable
-    width/height.
-    """
-
-    candidates_ms: list[float] = []
-    for req, out in warmup_pairs:
-        if not out.success or out.latency <= 0:
-            continue
-
-        width = req.width if req.width is not None else args.width
-        height = req.height if req.height is not None else args.height
-        if width is None or height is None:
-            continue
-
-        frames = req.num_frames if req.num_frames is not None else args.num_frames
-        steps = req.num_inference_steps if req.num_inference_steps is not None else args.num_inference_steps
-
-        frame_scale = int(frames) if isinstance(frames, int) and frames > 0 else 1
-        step_scale = int(steps) if isinstance(steps, int) and steps > 0 else 1
-
-        area_units = max((float(width) * float(height)) / float(16 * 16), 1.0)
-        denom = area_units * float(frame_scale) * float(step_scale)
-        if denom <= 0:
-            continue
-
-        candidates_ms.append((out.latency * 1000.0) / denom)
-
-    if not candidates_ms:
-        return None
-    return float(np.median(candidates_ms))
-
-
-def _populate_slo_ms_from_warmups(
-    requests_list: list[RequestFuncInput],
-    warmup_pairs: list[tuple[RequestFuncInput, RequestFuncOutput]],
-    args,
-) -> list[RequestFuncInput]:
-    """Populate missing RequestFuncInput.slo_ms using warmup outputs.
-
-    - If a request already has slo_ms (e.g., trace-provided), it is kept as-is.
-    - If any request has slo_ms is None and we can infer base time from warmups,
-      we estimate each missing request's expected execution time and set:
-        req.slo_ms = expected_latency_ms * args.slo_scale
-
-    Returns updated requests_list.
-    """
-
-    if not any(req.slo_ms is None for req in requests_list):
-        return requests_list
-
-    base_time_ms = _infer_slo_base_time_ms_from_warmups(warmup_pairs, args)
-    if base_time_ms is None:
-        return requests_list
-
-    slo_scale = float(getattr(args, "slo_scale", 3.0))
-    if slo_scale <= 0:
-        raise ValueError(f"slo_scale must be positive, got {slo_scale}.")
-
-    updated: list[RequestFuncInput] = []
-    for req in requests_list:
-        if req.slo_ms is not None:
-            updated.append(req)
-            continue
-        expected_ms = _compute_expected_latency_ms_from_base(req, args, base_time_ms)
-        updated.append(replace(req, slo_ms=(expected_ms * slo_scale) if expected_ms is not None else None))
-
-    return updated
-
-
-async def iter_requests(
-    requests_list: list[RequestFuncInput],
-    request_rate: float,
-) -> AsyncGenerator[RequestFuncInput, None]:
-    """Yield requests using a fixed interval if request_rate is set.
-
-    - If request_rate is inf, all requests are yielded immediately (no sleep).
-    - Otherwise, requests are emitted at a fixed cadence of 1 / request_rate seconds.
-    """
-
-    if request_rate != float("inf"):
-        if request_rate <= 0:
-            raise ValueError(f"request_rate must be positive or inf, got {request_rate}.")
-        interval_s = 1.0 / float(request_rate)
-
-    for i, req in enumerate(requests_list):
-        if request_rate != float("inf") and i > 0:
-            await asyncio.sleep(interval_s)
-        yield req
-
-
-def calculate_metrics(
-    outputs: list[RequestFuncOutput],
-    total_duration: float,
-    requests_list: list[RequestFuncInput],
-    args,
-    slo_enabled: bool,
-):
-    success_outputs = [o for o in outputs if o.success]
-    error_outputs = [o for o in outputs if not o.success]
-
-    num_success = len(success_outputs)
-    latencies = [o.latency for o in success_outputs]
-    peak_memories = [o.peak_memory_mb for o in success_outputs if o.peak_memory_mb > 0]
-
-    metrics = {
-        "duration": total_duration,
-        "completed_requests": num_success,
-        "failed_requests": len(error_outputs),
-        "throughput_qps": num_success / total_duration if total_duration > 0 else 0,
-        "latency_mean": np.mean(latencies) if latencies else 0,
-        "latency_median": np.median(latencies) if latencies else 0,
-        "latency_p99": np.percentile(latencies, 99) if latencies else 0,
-        "latency_p50": np.percentile(latencies, 50) if latencies else 0,
-        "peak_memory_mb_max": max(peak_memories) if peak_memories else 0,
-        "peak_memory_mb_mean": np.mean(peak_memories) if peak_memories else 0,
-        "peak_memory_mb_median": np.median(peak_memories) if peak_memories else 0,
-    }
-
-    if slo_enabled:
-        slo_defined_total = 0
-        slo_met_success = 0
-
-        for req, out in zip(requests_list, outputs):
-            if req.slo_ms is None:
-                continue
-            slo_defined_total += 1
-            if out.slo_achieved is None:
-                continue
-            if out.slo_achieved:
-                slo_met_success += 1
-
-        slo_attain_all = (slo_met_success / slo_defined_total) if slo_defined_total > 0 else 0.0
-
-        metrics.update(
-            {
-                "slo_attainment_rate": slo_attain_all,
-                "slo_met_success": slo_met_success,
-                "slo_scale": getattr(args, "slo_scale", 3.0),
-            }
-        )
-
-    return metrics
-
-
-def wait_for_service(base_url: str, timeout: int = 120) -> None:
-    print(f"Waiting for service at {base_url}...")
-    start_time = time.time()
-    while True:
-        try:
-            # Try /health endpoint first
-            resp = requests.get(f"{base_url}/health", timeout=1)
-            if resp.status_code == 200:
-                print("Service is ready.")
-                break
-        except requests.exceptions.RequestException:
-            pass
-
-        if time.time() - start_time > timeout:
-            raise TimeoutError(f"Service at {base_url} did not start within {timeout} seconds.")
-
-        time.sleep(1)
-
-
-async def benchmark(args):
-    # Construct base_url if not provided
-    if args.base_url is None:
-        args.base_url = f"http://{args.host}:{args.port}"
-
-    # Setup API URL and request function based on backend
-    request_func, api_url = backends_function_mapping[args.backend]
-    api_url = f"{args.base_url}{api_url}"
-
-    if args.dataset == "vbench":
-        dataset = VBenchDataset(args, api_url, args.model)
-    elif args.dataset == "trace":
-        dataset = TraceDataset(args, api_url, args.model)
-    elif args.dataset == "random":
-        dataset = RandomDataset(args, api_url, args.model)
-    else:
-        raise ValueError(f"Unknown dataset: {args.dataset}")
-
-    print("Loading requests...")
-    requests_list = dataset.get_requests()
-    print(f"Prepared {len(requests_list)} requests from {args.dataset} dataset.")
-
-    # Limit concurrency
-    if args.max_concurrency is not None:
-        semaphore = asyncio.Semaphore(args.max_concurrency)
-    else:
-        semaphore = None
-
-    async def limited_request_func(req, session, pbar):
-        if semaphore:
-            async with semaphore:
-                return await request_func(req, session, pbar)
-        else:
-            return await request_func(req, session, pbar)
-
-    # Run benchmark
-    pbar = tqdm(total=len(requests_list), disable=args.disable_tqdm)
-
-    async with aiohttp.ClientSession() as session:
-        warmup_pairs: list[tuple[RequestFuncInput, RequestFuncOutput]] = []
-        if args.warmup_requests and requests_list:
-            print(
-                f"Running {args.warmup_requests} warmup request(s) \
-                with num_inference_steps={args.warmup_num_inference_steps}..."
-            )
-            for i in range(args.warmup_requests):
-                warm_req = requests_list[i % len(requests_list)]
-                if args.warmup_num_inference_steps is not None:
-                    warm_req = replace(
-                        warm_req,
-                        num_inference_steps=args.warmup_num_inference_steps,
-                    )
-                warm_out = await limited_request_func(warm_req, session, None)
-                warmup_pairs.append((warm_req, warm_out))
-
-        if args.slo:
-            # Prefer trace-provided per-request slo_ms. Only populate when missing.
-            requests_list = _populate_slo_ms_from_warmups(
-                requests_list=requests_list,
-                warmup_pairs=warmup_pairs,
-                args=args,
-            )
-
-        start_time = time.perf_counter()
-        tasks = []
-        async for req in iter_requests(requests_list=requests_list, request_rate=args.request_rate):
-            task = asyncio.create_task(limited_request_func(req, session, pbar))
-            tasks.append(task)
-
-        outputs = await asyncio.gather(*tasks)
-        total_duration = time.perf_counter() - start_time
-
-    pbar.close()
-
-    # Calculate metrics
-    metrics = calculate_metrics(outputs, total_duration, requests_list, args, args.slo)
-
-    # Add configuration info to metrics for JSON output
-    metrics["backend"] = args.backend
-    metrics["model"] = args.model
-    metrics["dataset"] = args.dataset
-    metrics["task"] = args.task
-
-    print("\n{s:{c}^{n}}".format(s=" Serving Benchmark Result ", n=60, c="="))
-
-    # Section 1: Configuration
-    print("{:<40} {:<15}".format("Backend:", args.backend))
-    print("{:<40} {:<15}".format("Model:", args.model))
-    print("{:<40} {:<15}".format("Dataset:", args.dataset))
-    print("{:<40} {:<15}".format("Task:", args.task))
-
-    # Section 2: Execution & Traffic
-    print(f"{'-' * 50}")
-    print("{:<40} {:<15.2f}".format("Benchmark duration (s):", metrics["duration"]))
-    print("{:<40} {:<15}".format("Request rate:", str(args.request_rate)))
-    print(
-        "{:<40} {:<15}".format(
-            "Max request concurrency:",
-            str(args.max_concurrency) if args.max_concurrency else "not set",
-        )
-    )
-    print("{:<40} {}/{:<15}".format("Successful requests:", metrics["completed_requests"], len(requests_list)))
-
-    # Section 3: Performance Metrics
-    print(f"{'-' * 50}")
-
-    print("{:<40} {:<15.2f}".format("Request throughput (req/s):", metrics["throughput_qps"]))
-    print("{:<40} {:<15.4f}".format("Latency Mean (s):", metrics["latency_mean"]))
-    print("{:<40} {:<15.4f}".format("Latency Median (s):", metrics["latency_median"]))
-    print("{:<40} {:<15.4f}".format("Latency P99 (s):", metrics["latency_p99"]))
-
-    if args.slo:
-        print(f"{'-' * 50}")
-        print("{:<40} {:<15.2%}".format("SLO Attainment Rate (all):", metrics.get("slo_attainment_rate", 0.0)))
-        print("{:<40} {:<15}".format("SLO Met (success count):", str(metrics.get("slo_met_success", 0))))
-        print("{:<40} {:<15}".format("SLO Scale:", str(metrics.get("slo_scale", 3.0))))
-
-    if metrics["peak_memory_mb_max"] > 0:
-        print(f"{'-' * 50}")
-        print("{:<40} {:<15.2f}".format("Peak Memory Max (MB):", metrics["peak_memory_mb_max"]))
-        print("{:<40} {:<15.2f}".format("Peak Memory Mean (MB):", metrics["peak_memory_mb_mean"]))
-        print("{:<40} {:<15.2f}".format("Peak Memory Median (MB):", metrics["peak_memory_mb_median"]))
-
-    print("\n" + "=" * 60)
-
-    if args.output_file:
-        with open(args.output_file, "w") as f:
-            json.dump(metrics, f, indent=2)
-        print(f"Metrics saved to {args.output_file}")
-
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser(description="Benchmark serving for diffusion models.")
-    parser.add_argument(
-        "--base-url",
-        type=str,
-        default=None,
-        help="Base URL of the server (e.g., http://localhost:8091). Overrides host/port.",
-    )
-    parser.add_argument("--host", type=str, default="localhost", help="Server host.")
-    parser.add_argument("--port", type=int, default=8091, help="Server port.")
-    parser.add_argument("--model", type=str, default="default", help="Model name.")
-    parser.add_argument(
-        "--backend",
-        type=str,
-        default="vllm-omni",
-        choices=["vllm-omni", "openai"],
-        help="Backend to target the benchmark to.",
-    )
-    parser.add_argument(
-        "--dataset",
-        type=str,
-        default="vbench",
-        choices=["vbench", "trace", "random"],
-        help="Dataset to use.",
-    )
-    parser.add_argument(
-        "--task",
-        type=str,
-        default="t2v",
-        choices=["t2v", "i2v", "ti2v", "ti2i", "i2i", "t2i"],
-        help="Task type.",
-    )
-    parser.add_argument(
-        "--dataset-path",
-        type=str,
-        default=None,
-        help="Path to local dataset file (optional).",
-    )
-    parser.add_argument("--num-prompts", type=int, default=10, help="Number of prompts to benchmark.")
-    parser.add_argument(
-        "--max-concurrency",
-        type=int,
-        default=1,
-        help="Maximum number of concurrent requests, default to `1`. This can be used "
-        "to help simulate an environment where a higher level component "
-        "is enforcing a maximum number of concurrent requests. While the "
-        "--request-rate argument controls the rate at which requests are "
-        "initiated, this argument will control how many are actually allowed "
-        "to execute at a time. This means that when used in combination, the "
-        "actual request rate may be lower than specified with --request-rate, "
-        "if the server is not processing requests fast enough to keep up.",
-    )
-    parser.add_argument(
-        "--request-rate",
-        type=float,
-        default=float("inf"),
-        help="Number of requests per second. If this is inf, then all the requests are sent at time 0. "
-        "Otherwise, we use Poisson process to synthesize the request arrival times. Default is inf.",
-    )
-    parser.add_argument(
-        "--warmup-requests",
-        type=int,
-        default=1,
-        help="Number of warmup requests to run before measurement.",
-    )
-    parser.add_argument(
-        "--warmup-num-inference-steps",
-        type=int,
-        default=1,
-        help="num_inference_steps used for warmup requests.",
-    )
-    parser.add_argument("--width", type=int, default=None, help="Image/Video width.")
-    parser.add_argument("--height", type=int, default=None, help="Image/Video height.")
-    parser.add_argument("--num-frames", type=int, default=None, help="Number of frames (for video).")
-    parser.add_argument(
-        "--num-inference-steps",
-        type=int,
-        default=50,
-        help="Number of inference steps (for diffusion models).",
-    )
-    parser.add_argument(
-        "--seed",
-        type=int,
-        default=None,
-        help="Random seed (for diffusion models).",
-    )
-    parser.add_argument("--fps", type=int, default=None, help="FPS (for video).")
-    parser.add_argument("--output-file", type=str, default=None, help="Output JSON file for metrics.")
-    parser.add_argument(
-        "--slo",
-        action="store_true",
-        help=(
-            "Enable SLO calculation and reporting. If trace provides per-request slo_ms, it is used. "
-            "Otherwise, warmup request(s) are used to infer expected execution time assuming linear "
-            "scaling by resolution, frames, and steps, then slo_ms = expected_time * --slo-scale."
-        ),
-    )
-    parser.add_argument(
-        "--slo-scale",
-        type=float,
-        default=3.0,
-        help="SLO target multiplier: slo_ms = estimated_exec_time_ms * slo_scale (default: 3).",
-    )
-    parser.add_argument("--disable-tqdm", action="store_true", help="Disable progress bar.")
-
-    args = parser.parse_args()
-
-    asyncio.run(benchmark(args))

benchmarks/distributed/omni_connectors/README.md

@@ -1,397 +0,0 @@
-# RDMA Test Configuration Guide
-
-This document explains how to configure the RDMA environment and run tests for `MooncakeTransferEngineConnector`.
-
-## Table of Contents
-
-- [Docker Container Permissions](#docker-container-permissions)
-- [Single-Node Testing](#single-node-testing)
-- [Multi-Node Testing](#multi-node-testing)
-- [Running Tests](#running-tests)
-- [Cross-Node Testing](#cross-node-testing)
-- [Troubleshooting](#troubleshooting)
-
----
-
-## Docker Container Permissions
-
-RDMA tests require access to InfiniBand/RoCE devices and system topology. Add the following permissions when running `docker run`.
-
-### Option 1: Minimal Permissions (Recommended)
-
-```bash
-docker run -it \
-    --cap-add=SYS_PTRACE \
-    --cap-add=IPC_LOCK \
-    --security-opt seccomp=unconfined \
-    --network=host \
-    --device=/dev/infiniband \
-    -v /sys/class/infiniband:/sys/class/infiniband:ro \
-    your-image:tag
-```
-
-Parameter explanation:
-- `--cap-add=SYS_PTRACE`: Allow reading system topology information
-- `--cap-add=IPC_LOCK`: Allow memory locking (required for RDMA memory registration)
-- `--security-opt seccomp=unconfined`: Disable seccomp restrictions
-- `--network=host`: Use host network (required for RDMA)
-- `--device=/dev/infiniband`: Mount InfiniBand devices
-- `-v /sys/class/infiniband`: Mount IB device info (read-only)
-
-### Option 2: Full Permissions (Quick but not recommended for production)
-
-```bash
-docker run -it \
-    --privileged \
-    --network=host \
-    your-image:tag
-```
-
-`--privileged` grants full host permissions. Suitable for quick testing but not recommended for production.
-
----
-
-## Single-Node Testing
-
-When running single-node tests (producer and consumer on the same machine), ensure they use the **same RDMA device**.
-
-### Problem Background
-
-InfiniBand devices use LID (Local Identifier) for routing. Different devices have different LIDs and cannot communicate directly. If no device is specified, Mooncake may assign different devices to connectors, causing handshake failures.
-
-Common error:
-```
-[Handshake] Failed to modify QP to RTR, check mtu, gid, peer lid, peer qp num: Invalid argument [22]
-```
-
-### Solution
-
-**Method 1: Set Environment Variable (Recommended)**
-
-```bash
-# List available RDMA devices
-ibstat
-
-# Select a device (e.g., mlx5_0)
-export RDMA_DEVICE_NAME='mlx5_0'
-
-# Run tests
-pytest test_mooncake_transfer_engine_rdma.py -v -s
-```
-
-**Method 2: Use RoCE Devices**
-
-If the system has RoCE devices (using IPv4 routing), the test code will automatically detect and prefer them. RoCE device GIDs start with `00:00:00:00:00:00:00:00:00:00:ff:ff` (IPv4-mapped).
-
-**Method 3: Ensure MTU Consistency**
-
-Make sure both endpoints use the same MTU:
-
-```bash
-# Check device MTU
-ibstatus mlx5_0
-```
-
----
-
-## Multi-Node Testing
-
-For multi-node tests, producer and consumer run on different machines connected via InfiniBand switch.
-
-### Prerequisites
-
-1. Both machines have Mooncake and RDMA drivers installed
-2. Both machines are in the same InfiniBand subnet
-3. Switch is properly configured
-
-### Configuration
-
-**Machine A (Producer):**
-
-```bash
-# Set RDMA host IP (InfiniBand interface IP)
-export RDMA_TEST_HOST='10.0.0.1'
-
-# Optional: Specify device
-export RDMA_DEVICE_NAME='mlx5_0'
-```
-
-**Machine B (Consumer):**
-
-```bash
-# Set RDMA host IP
-export RDMA_TEST_HOST='10.0.0.2'
-
-# Optional: Specify device
-export RDMA_DEVICE_NAME='mlx5_0'
-```
-
-### Verify Connectivity
-
-```bash
-# Ping IB interface
-ping 10.0.0.2
-
-# Test RDMA connectivity with ibping
-# On Machine B (server)
-ibping -S
-
-# On Machine A (client)
-ibping -G <Machine_B_GID>
-```
-
----
-
-## Running Tests
-
-### Run All RDMA Tests (Single-Node, fast suite)
-
-Slow tests (large payloads, stress, concurrency integrity) are marked `@pytest.mark.slow`. Use `-m "not slow"` to skip them in quick CI or local fast iteration.
-
-```bash
-cd tests/distributed/omni_connectors
-
-# Fast suite only (excludes slow/stress tests)
-pytest test_mooncake_transfer_engine_rdma.py test_mooncake_transfer_engine_buffer.py -v -s -m "not slow"
-```
-
-### Run Including Slow Tests
-
-```bash
-# Run ALL tests including slow/stress tests
-pytest test_mooncake_transfer_engine_rdma.py test_mooncake_transfer_engine_buffer.py -v -s
-
-# Run ONLY the slow/stress tests
-pytest test_mooncake_transfer_engine_rdma.py test_mooncake_transfer_engine_buffer.py -v -s -m slow
-```
-
-### Run Buffer Management Tests
-
-```bash
-# Fast only
-pytest test_mooncake_transfer_engine_buffer.py -v -s -m "not slow"
-
-# Including allocator invariant tests (double-free, overlap, merge)
-pytest test_mooncake_transfer_engine_buffer.py -v -s
-```
-
-### Run Specific Test Classes
-
-```bash
-# Basic connector tests
-pytest test_mooncake_transfer_engine_rdma.py::TestBasicConnector -v -s
-
-# End-to-end RDMA transfer tests
-pytest test_mooncake_transfer_engine_rdma.py::TestEndToEnd -v -s
-
-# Lifecycle & resource management tests
-pytest test_mooncake_transfer_engine_rdma.py::TestLifecycle -v -s
-
-# GPU memory pool tests (requires CUDA)
-pytest test_mooncake_transfer_engine_rdma.py::TestGPUPool -v -s
-
-# Stress / correctness tests (slow)
-pytest test_mooncake_transfer_engine_rdma.py::TestStressCorrectness -v -s
-```
-
-### RDMA Environment Diagnostics
-
-For quick diagnostics (device status, Mooncake availability, env vars, etc.),
-see the [Troubleshooting section](../../../docs/design/feature/omni_connectors/mooncake_transfer_engine_connector.md#troubleshooting)
-in the connector documentation.
-
----
-
-## Cross-Node Testing
-
-The `cross_node_mooncake_transfer_engine.py` script enables testing RDMA transfers between two separate physical machines. This script is **not** auto-discovered by `pytest` (it does not start with `test_`) — it must be run manually on each node.
-
-### Prerequisites
-
-1. Both machines have Mooncake installed
-2. Both machines are connected via InfiniBand/RoCE switch
-3. Firewall allows ZMQ ports (default: 15500, 15501)
-4. Same RDMA device name on both nodes (if multiple devices exist)
-
-### Running Cross-Node Tests
-
-**On Machine A (Producer) — start first:**
-
-```bash
-cd benchmarks/distributed/omni_connectors/
-
-# Optional: specify device if multiple exist
-export RDMA_DEVICE_NAME='mlx5_0'
-
-python cross_node_mooncake_transfer_engine.py \
-    --role producer \
-    --local-host <PRODUCER_IP> \
-    --remote-host <CONSUMER_IP> \
-    --tensor-size-mb 100 \
-    --num-transfers 3
-```
-
-**On Machine B (Consumer) — start after producer:**
-
-```bash
-cd benchmarks/distributed/omni_connectors/
-
-export RDMA_DEVICE_NAME='mlx5_0'
-
-python cross_node_mooncake_transfer_engine.py \
-    --role consumer \
-    --local-host <CONSUMER_IP> \
-    --remote-host <PRODUCER_IP> \
-    --tensor-size-mb 100 \
-    --num-transfers 3
-```
-
-### Transfer Modes
-
-| Mode | Description | Example |
-|------|-------------|---------|
-| `copy` | Normal path — tensor copied to RDMA pool (default) | `--mode copy` |
-| `zerocopy` | Zero-copy path — data created directly in RDMA pool | `--mode zerocopy` |
-| `gpu` | GPU transfer — RDMA pool on GPU, uses GPUDirect | `--mode gpu --gpu-id 0` |
-
-### Benchmark Mode
-
-Skip MD5 verification and measure pure RDMA throughput:
-
-```bash
-# Producer
-python cross_node_mooncake_transfer_engine.py \
-    --role producer \
-    --local-host <PRODUCER_IP> \
-    --remote-host <CONSUMER_IP> \
-    --tensor-size-mb 1024 \
-    --num-transfers 20 \
-    --benchmark
-
-# Consumer
-python cross_node_mooncake_transfer_engine.py \
-    --role consumer \
-    --local-host <CONSUMER_IP> \
-    --remote-host <PRODUCER_IP> \
-    --tensor-size-mb 1024 \
-    --num-transfers 20 \
-    --benchmark
-```
-
-### Cross-Node Test Options
-
-| Option | Description | Default |
-|--------|-------------|---------|
-| `--role` | `producer` or `consumer` | Required |
-| `--local-host` | Local RDMA IP address | Required |
-| `--remote-host` | Remote RDMA IP address | Required |
-| `--local-port` | Local ZMQ port for RDMA data | 15500 |
-| `--remote-port` | Remote ZMQ port for RDMA data | 15500 |
-| `--ctrl-port` | Control channel port | 15501 |
-| `--tensor-size-mb` | Tensor size in MB | 100 |
-| `--num-transfers` | Number of transfers | 3 |
-| `--mode` | `copy`, `zerocopy`, or `gpu` | `copy` |
-| `--gpu-id` | GPU ID for GPU mode | 0 |
-| `--benchmark` | Skip MD5, pure performance test | off |
-
----
-
-## Troubleshooting
-
-### 1. "Failed to modify QP to RTR" Error
-
-**Cause**: QP handshake failed, usually due to device configuration mismatch.
-
-**Solution**:
-```bash
-# Force using the same device
-export RDMA_DEVICE_NAME='mlx5_0'
-```
-
-### 2. "Mooncake TransferEngine is not available"
-
-**Cause**: Mooncake not installed or import failed.
-
-**Solution**:
-```bash
-# Check Mooncake installation
-python -c "from mooncake.engine import TransferEngine; print('OK')"
-
-# Reinstall if needed
-pip install mooncake-transfer-engine
-# Or using uv
-uv pip install mooncake-transfer-engine
-
-```
-
-### 3. "Permission denied" accessing /dev/infiniband
-
-**Cause**: Container lacks IB device access permissions.
-
-**Solution**:
-```bash
-docker run --device=/dev/infiniband --cap-add=IPC_LOCK ...
-```
-
-### 4. Test Timeout
-
-**Cause**: RDMA connection establishment failed or network latency.
-
-**Solution**:
-```bash
-# Check network status
-ibstat
-ibstatus
-```
-
-### 5. GPU Test Failed "CUDA is not available"
-
-**Cause**: CUDA environment not configured or GPU unavailable.
-
-**Solution**:
-```bash
-# Check CUDA
-python -c "import torch; print(torch.cuda.is_available())"
-
-# Docker needs NVIDIA runtime
-docker run --gpus all ...
-```
-
----
-
-## Environment Variables Reference
-
-| Variable | Description | Example |
-|----------|-------------|---------|
-| `RDMA_DEVICE_NAME` | Specify RDMA device name | `mlx5_0` |
-| `RDMA_TEST_HOST` | Specify test host IP | `10.0.0.1` |
-| `MC_TE_METRIC` | Enable Mooncake metrics | `1` |
-| `MC_IB_PCI_RELAXED_ORDERING` | Enable PCIe relaxed ordering | `1` |
-
----
-
-## Test Files Overview
-
-| File | Description | Auto-discovered by pytest |
-|------|-------------|--------------------------|
-| `test_mooncake_transfer_engine_rdma.py` | Integration tests for MooncakeTransferEngineConnector (basic, E2E, lifecycle, GPU) | Yes |
-| `test_mooncake_transfer_engine_buffer.py` | Memory pool and buffer management unit tests | Yes |
-| `cross_node_mooncake_transfer_engine.py` | Cross-node (multi-machine) testing script — run manually | No (filename does not start with `test_`) |
-
-### test_mooncake_transfer_engine_rdma.py — Test Classes
-
-| Test Class | Memory Pool | Marker | Description |
-|------------|-------------|--------|-------------|
-| `TestBasicConnector` | CPU | — | Initialization, put tensor/bytes/object, cleanup, pool exhaustion |
-| `TestEndToEnd` | CPU | — | E2E RDMA transfer: tensor, bytes, object, zero-copy, large payload (100MB), mixed types, concurrency |
-| `TestLifecycle` | CPU | — | Close, context manager, double-close safety |
-| `TestGPUPool` | GPU | — | GPU pool init, put CPU/GPU tensor, GPU E2E transfer |
-| `TestStressCorrectness` | CPU | `slow` | Concurrent put+get with MD5 integrity, bidirectional concurrency, edge cases (1-element tensor, empty bytes), 500MB payload, rapid alloc/free cycles |
-
-### test_mooncake_transfer_engine_buffer.py — Test Classes
-
-| Test Class | Marker | Description |
-|------------|--------|-------------|
-| `TestBufferAllocator` | — | Basic alloc/free, alignment, exhaustion/recovery, thread safety |
-| `TestAllocatorInvariants` | `slow` | Double-free safety, overlap corruption detection, adjacent-block merging, fragmentation/defrag |
-| `TestManagedBuffer` | — | Tensor views, context manager |

benchmarks/distributed/omni_connectors/cross_node_mooncake_transfer_engine.py

@@ -1,644 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-
-"""
-Cross-Node RDMA Test Script (Automated Version)
-
-This script enables testing RDMA transfers between two separate machines.
-Supports three transfer modes:
-  - copy:      Normal path - tensor copied to RDMA pool (default)
-  - zerocopy:  Zero-copy path - data created directly in RDMA pool
-  - gpu:       GPU transfer - RDMA pool on GPU, uses GPUDirect
-
-Usage:
-    # On Machine A (Producer) - start first:
-    python cross_node_mooncake_transfer_engine.py --role producer --local-host hostname_A --remote-host hostname_B
-
-    # On Machine B (Consumer) - start after producer:
-    python cross_node_mooncake_transfer_engine.py --role consumer --local-host hostname_B --remote-host hostname_A
-
-    # Zero-copy mode:
-    python cross_node_mooncake_transfer_engine.py --role producer ... --mode zerocopy
-
-    # GPU mode (requires GPUDirect RDMA support):
-    python cross_node_mooncake_transfer_engine.py --role producer ... --mode gpu --gpu-id 0
-
-    # Benchmark mode (skip random data generation and MD5 verification,
-    # measures pure RDMA throughput):
-    python cross_node_mooncake_transfer_engine.py --role producer ... --benchmark
-
-Environment Variables:
-    RDMA_DEVICE_NAME:              Specify RDMA device (e.g., mlx5_0)
-    MC_IB_PCI_RELAXED_ORDERING:    Set to 1 to enable PCIe relaxed ordering
-                                   for higher RDMA throughput
-"""
-
-import argparse
-import hashlib
-import os
-import sys
-import time
-from abc import ABC, abstractmethod
-from dataclasses import dataclass
-from typing import Any
-
-import msgspec
-import torch
-import zmq
-
-# Add parent path for imports
-sys.path.insert(
-    0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))))
-)
-
-from vllm_omni.distributed.omni_connectors.connectors.mooncake_transfer_engine_connector import (
-    ManagedBuffer,
-    MooncakeTransferEngineConnector,
-    TransferEngine,
-)
-
-
-def compute_md5(tensor: torch.Tensor) -> str:
-    """Compute MD5 checksum of a tensor."""
-    if tensor.is_cuda:
-        tensor = tensor.cpu()
-    data = tensor.contiguous().view(torch.uint8).numpy().tobytes()
-    return hashlib.md5(data).hexdigest()
-
-
-# Control channel message types
-class CtrlMsg(msgspec.Struct):
-    """Control channel message."""
-
-    msg_type: str  # "READY", "TRANSFER", "ACK", "DONE", "ERROR"
-    request_id: str = ""
-    md5: str = ""
-    data_size: int = 0
-    error: str = ""
-
-
-@dataclass
-class TransferConfig:
-    """Configuration for cross-node transfer test."""
-
-    local_host: str
-    remote_host: str
-    local_port: int
-    remote_port: int
-    ctrl_port: int
-    num_transfers: int
-    tensor_size_mb: int
-    mode: str  # "copy", "zerocopy", "gpu"
-    gpu_id: int = 0
-    benchmark: bool = False  # Skip MD5 verification for pure performance test
-
-
-@dataclass
-class TransferStats:
-    """Statistics for transfer operations."""
-
-    success_count: int = 0
-    fail_count: int = 0
-    total_bytes: int = 0
-    elapsed_time: float = 0.0
-
-    @property
-    def throughput_mbps(self) -> float:
-        if self.elapsed_time > 0:
-            return (self.total_bytes / (1024 * 1024)) / self.elapsed_time
-        return 0.0
-
-    def print_summary(self, role: str):
-        print(f"\n{'=' * 60}")
-        print(f" {role.upper()} SUMMARY")
-        print(f"  Successful: {self.success_count}/{self.success_count + self.fail_count}")
-        print(f"  Failed:     {self.fail_count}/{self.success_count + self.fail_count}")
-        print(f"  Total:      {self.total_bytes / (1024 * 1024):.2f} MB")
-        print(f"  Time:       {self.elapsed_time:.2f} s")
-        print(f"  Throughput: {self.throughput_mbps:.2f} MB/s")
-        print(f"{'=' * 60}")
-
-
-class CrossNodeTester(ABC):
-    """Abstract base class for cross-node RDMA testing."""
-
-    def __init__(self, config: TransferConfig):
-        self.config = config
-        self.connector: MooncakeTransferEngineConnector | None = None
-        self.zmq_ctx: zmq.Context | None = None
-        self.ctrl_socket: zmq.Socket | None = None
-        self.stats = TransferStats()
-
-    def get_connector_config(self) -> dict:
-        """Get connector configuration based on mode."""
-        pool_size = int(self.config.tensor_size_mb * 1.5) * 1024 * 1024
-        pool_size = max(pool_size, 128 * 1024 * 1024)
-
-        conn_config = {
-            "host": self.config.local_host,
-            "zmq_port": self.config.local_port,
-            "protocol": "rdma",
-            "memory_pool_size": pool_size,
-        }
-
-        # Set device based on mode
-        if self.config.mode == "gpu":
-            conn_config["memory_pool_device"] = f"cuda:{self.config.gpu_id}"
-        else:
-            conn_config["memory_pool_device"] = "cpu"
-
-        # RDMA device name from environment
-        device_name = os.environ.get("RDMA_DEVICE_NAME")
-        if device_name:
-            conn_config["device_name"] = device_name
-            print(f"[CONFIG] Using RDMA device: {device_name}")
-
-        return conn_config
-
-    def initialize(self):
-        """Initialize connector and ZMQ context."""
-        print(f"[{self.role}] Initializing connector...")
-        conn_config = self.get_connector_config()
-        self.connector = MooncakeTransferEngineConnector(conn_config)
-        self.zmq_ctx = zmq.Context()
-        print(f"[{self.role}] Ready at {self.config.local_host}:{self.config.local_port}")
-
-    def cleanup(self):
-        """Cleanup resources."""
-        if self.ctrl_socket:
-            self.ctrl_socket.close()
-        if self.zmq_ctx:
-            self.zmq_ctx.term()
-        if self.connector:
-            self.connector.close()
-        print(f"[{self.role}] Closed.")
-
-    @property
-    @abstractmethod
-    def role(self) -> str:
-        pass
-
-    @abstractmethod
-    def run(self):
-        pass
-
-
-class Producer(CrossNodeTester):
-    """Producer node - sends data to consumer."""
-
-    @property
-    def role(self) -> str:
-        return "PRODUCER"
-
-    def print_header(self):
-        print(f"\n{'=' * 60}")
-        print(f" PRODUCER MODE ({self.config.mode.upper()})")
-        print(f" Local:  {self.config.local_host}:{self.config.local_port}")
-        print(f" Remote: {self.config.remote_host}:{self.config.remote_port}")
-        print(f" Control Port: {self.config.ctrl_port}")
-        print(f" Transfer Mode: {self.config.mode}")
-        if self.config.mode == "gpu":
-            print(f" GPU ID: {self.config.gpu_id}")
-        print(f"{'=' * 60}\n")
-
-    def setup_control_channel(self):
-        """Setup ZMQ control channel as server (REP socket)."""
-        self.ctrl_socket = self.zmq_ctx.socket(zmq.REP)
-        self.ctrl_socket.bind(f"tcp://*:{self.config.ctrl_port}")
-        print(f"[PRODUCER] Control channel listening on port {self.config.ctrl_port}")
-        print("[PRODUCER] Waiting for consumer to connect...")
-
-    def wait_for_consumer(self) -> bool:
-        """Wait for consumer READY signal."""
-        msg_data = self.ctrl_socket.recv()
-        msg = msgspec.msgpack.decode(msg_data, type=CtrlMsg)
-        if msg.msg_type != "READY":
-            print(f"[PRODUCER] Unexpected message: {msg.msg_type}")
-            return False
-        print("[PRODUCER] Consumer connected!")
-        self.ctrl_socket.send(msgspec.msgpack.encode(CtrlMsg(msg_type="ACK")))
-        return True
-
-    def create_test_data(self, transfer_idx: int) -> tuple[Any, str, int]:
-        """
-        Create test data based on transfer mode.
-
-        Returns:
-            (data, md5, size) tuple
-        """
-        num_elements = (self.config.tensor_size_mb * 1024 * 1024) // 4
-        data_size = num_elements * 4
-
-        if self.config.mode == "zerocopy":
-            # Zero-Copy Path: Allocate directly from connector's pool
-            offset = self.connector.allocator.alloc(data_size)
-            managed_buf = ManagedBuffer(self.connector.allocator, offset, data_size, self.connector.pool)
-
-            if self.config.benchmark:
-                # In benchmark mode, skip random data generation (use uninitialized memory)
-                return managed_buf, "", data_size
-            else:
-                # Fill buffer with random data using tensor view
-                tensor_view = managed_buf.as_tensor(dtype=torch.float32, shape=(num_elements,))
-                random_data = torch.randn(num_elements, dtype=torch.float32)
-                if tensor_view.is_cuda:
-                    tensor_view.copy_(random_data.to(tensor_view.device))
-                else:
-                    tensor_view.copy_(random_data)
-                md5 = compute_md5(tensor_view)
-                return managed_buf, md5, data_size
-
-        elif self.config.mode == "gpu":
-            # GPU Path: Create tensor on GPU
-            device = f"cuda:{self.config.gpu_id}"
-            if self.config.benchmark:
-                # In benchmark mode, use empty tensor (no random generation)
-                gpu_tensor = torch.empty(num_elements, dtype=torch.float32, device=device)
-                return gpu_tensor, "", data_size
-            else:
-                cpu_tensor = torch.randn(num_elements, dtype=torch.float32)
-                md5 = compute_md5(cpu_tensor)
-                gpu_tensor = cpu_tensor.to(device)
-                return gpu_tensor, md5, data_size
-
-        else:
-            # Copy Path (default): Create regular CPU tensor
-            if self.config.benchmark:
-                # In benchmark mode, use empty tensor (no random generation)
-                tensor = torch.empty(num_elements, dtype=torch.float32)
-                return tensor, "", data_size
-            else:
-                tensor = torch.randn(num_elements, dtype=torch.float32)
-                md5 = compute_md5(tensor)
-                return tensor, md5, data_size
-
-    def do_transfer(self, transfer_idx: int) -> bool:
-        """Perform a single transfer."""
-        req_id = f"cross_node_transfer_{transfer_idx}"
-
-        if not self.config.benchmark:
-            print(f"\n[PRODUCER] Transfer {transfer_idx + 1}/{self.config.num_transfers}")
-
-        # Create test data
-        t0 = time.time()
-        data, md5, data_size = self.create_test_data(transfer_idx)
-        t_create = time.time() - t0
-
-        if not self.config.benchmark:
-            print(f"  Mode: {self.config.mode}")
-            print(f"  Size: {self.config.tensor_size_mb} MB")
-            if md5:
-                print(f"  MD5:  {md5[:16]}...")
-            print(f"  Create time: {t_create * 1000:.1f} ms")
-
-        # Put data
-        t1 = time.time()
-        success, size, metadata = self.connector.put("producer", "consumer", req_id, data)
-        t_put = time.time() - t1
-
-        if not success:
-            print("  [FAIL] Put failed")
-            return False
-
-        if not self.config.benchmark:
-            print(f"  [OK] Put successful, {size} bytes ({t_put * 1000:.1f} ms)")
-
-        # Wait for consumer to request transfer info
-        msg_data = self.ctrl_socket.recv()
-        msg = msgspec.msgpack.decode(msg_data, type=CtrlMsg)
-
-        if msg.msg_type != "READY":
-            print(f"  [ERROR] Unexpected message: {msg.msg_type}")
-            return False
-
-        # Send transfer metadata to consumer
-        transfer_msg = CtrlMsg(
-            msg_type="TRANSFER",
-            request_id=req_id,
-            md5=md5,
-            data_size=data_size,
-        )
-        self.ctrl_socket.send(msgspec.msgpack.encode(transfer_msg))
-
-        # Wait for consumer ACK (this includes RDMA transfer time)
-        t2 = time.time()
-        msg_data = self.ctrl_socket.recv()
-        t_rdma = time.time() - t2
-        msg = msgspec.msgpack.decode(msg_data, type=CtrlMsg)
-
-        success = msg.msg_type == "ACK"
-        if success:
-            if not self.config.benchmark:
-                print(f"  [OK] RDMA transfer complete ({t_rdma * 1000:.1f} ms)")
-            self.stats.success_count += 1
-            self.stats.total_bytes += size
-        else:
-            print(f"  [WARN] Consumer reported error: {msg.error}")
-            self.stats.fail_count += 1
-
-        # Send ACK to allow consumer to continue
-        self.ctrl_socket.send(msgspec.msgpack.encode(CtrlMsg(msg_type="ACK")))
-
-        # Cleanup buffer
-        self.connector.cleanup(req_id)
-
-        return success
-
-    def run(self):
-        """Run the producer."""
-        self.print_header()
-        self.initialize()
-        self.setup_control_channel()
-
-        try:
-            if not self.wait_for_consumer():
-                return
-
-            if self.config.benchmark:
-                print(
-                    f"[BENCHMARK] Running {self.config.num_transfers} "
-                    f"transfers of {self.config.tensor_size_mb} MB each..."
-                )
-
-            start_time = time.time()
-
-            for i in range(self.config.num_transfers):
-                self.do_transfer(i)
-                if self.config.benchmark and (i + 1) % 10 == 0:
-                    elapsed = time.time() - start_time
-                    current_throughput = (self.stats.total_bytes / (1024 * 1024)) / elapsed
-                    print(f"  Progress: {i + 1}/{self.config.num_transfers}, Throughput: {current_throughput:.2f} MB/s")
-
-            self.stats.elapsed_time = time.time() - start_time
-            self.stats.print_summary("PRODUCER")
-
-            # Wait for final consumer message and send DONE
-            self.ctrl_socket.recv()
-            self.ctrl_socket.send(msgspec.msgpack.encode(CtrlMsg(msg_type="DONE")))
-
-        finally:
-            self.cleanup()
-
-
-class Consumer(CrossNodeTester):
-    """Consumer node - receives data from producer."""
-
-    @property
-    def role(self) -> str:
-        return "CONSUMER"
-
-    def print_header(self):
-        print(f"\n{'=' * 60}")
-        print(f" CONSUMER MODE ({self.config.mode.upper()})")
-        print(f" Local:  {self.config.local_host}:{self.config.local_port}")
-        print(f" Remote: {self.config.remote_host}:{self.config.remote_port}")
-        print(f" Control Port: {self.config.ctrl_port}")
-        print(f" Transfer Mode: {self.config.mode}")
-        if self.config.mode == "gpu":
-            print(f" GPU ID: {self.config.gpu_id}")
-        print(f"{'=' * 60}\n")
-
-    def setup_control_channel(self):
-        """Setup ZMQ control channel as client (REQ socket)."""
-        self.ctrl_socket = self.zmq_ctx.socket(zmq.REQ)
-        ctrl_addr = f"tcp://{self.config.remote_host}:{self.config.ctrl_port}"
-        print(f"[CONSUMER] Connecting to producer control channel at {ctrl_addr}...")
-        self.ctrl_socket.connect(ctrl_addr)
-
-    def connect_to_producer(self) -> bool:
-        """Connect to producer and send READY signal."""
-        self.ctrl_socket.send(msgspec.msgpack.encode(CtrlMsg(msg_type="READY")))
-        msg_data = self.ctrl_socket.recv()
-        msg = msgspec.msgpack.decode(msg_data, type=CtrlMsg)
-        if msg.msg_type != "ACK":
-            print(f"[CONSUMER] Unexpected response: {msg.msg_type}")
-            return False
-        print("[CONSUMER] Connected to producer! Starting transfers...")
-        return True
-
-    def do_transfer(self, transfer_idx: int) -> bool:
-        """Perform a single transfer."""
-        if not self.config.benchmark:
-            print(f"\n[CONSUMER] Transfer {transfer_idx + 1}/{self.config.num_transfers}")
-
-        # Request next transfer info
-        self.ctrl_socket.send(msgspec.msgpack.encode(CtrlMsg(msg_type="READY")))
-        msg_data = self.ctrl_socket.recv()
-        msg = msgspec.msgpack.decode(msg_data, type=CtrlMsg)
-
-        if msg.msg_type == "DONE":
-            print("[CONSUMER] Producer signaled completion")
-            return False
-
-        if msg.msg_type != "TRANSFER":
-            print(f"[CONSUMER] Unexpected message: {msg.msg_type}")
-            return False
-
-        req_id = msg.request_id
-        expected_md5 = msg.md5
-        data_size = msg.data_size
-        num_elements = data_size // 4
-
-        if not self.config.benchmark:
-            print(f"  Request ID: {req_id}")
-            if expected_md5:
-                print(f"  Expected MD5: {expected_md5[:16]}...")
-            print(f"  Data Size: {data_size / (1024 * 1024):.2f} MB")
-
-        # Build metadata for get
-        metadata = {
-            "request_id": req_id,
-            "source_host": self.config.remote_host,
-            "source_port": self.config.remote_port,
-            "data_size": data_size,
-            "dtype": "float32",
-            "shape": [num_elements],
-            "is_fast_path": True,
-        }
-
-        if not self.config.benchmark:
-            print(f"  [INFO] Requesting from {self.config.remote_host}:{self.config.remote_port}")
-
-        # Get data with timing
-        t0 = time.time()
-        result = self.connector.get("producer", "consumer", req_id, metadata)
-        t_get = time.time() - t0
-
-        response_msg = CtrlMsg(msg_type="ERROR", error="Get failed")
-
-        if result is not None:
-            recv_buffer, recv_size = result
-            if not self.config.benchmark:
-                print(f"  [OK] Get successful, {recv_size} bytes ({t_get * 1000:.1f} ms)")
-
-            if isinstance(recv_buffer, ManagedBuffer):
-                # In benchmark mode, skip MD5 verification
-                if self.config.benchmark or not expected_md5:
-                    response_msg = CtrlMsg(msg_type="ACK")
-                    self.stats.success_count += 1
-                    self.stats.total_bytes += recv_size
-                else:
-                    # Verify data
-                    t1 = time.time()
-                    reconstructed = recv_buffer.as_tensor(dtype=torch.float32, shape=(num_elements,))
-                    recv_md5 = compute_md5(reconstructed)
-                    t_md5 = time.time() - t1
-                    print(f"  MD5: {recv_md5[:16]}... ({t_md5 * 1000:.1f} ms)")
-
-                    if recv_md5 == expected_md5:
-                        print("  [PASS] MD5 checksum verified!")
-                        response_msg = CtrlMsg(msg_type="ACK")
-                        self.stats.success_count += 1
-                        self.stats.total_bytes += recv_size
-                    else:
-                        print("  [FAIL] MD5 mismatch!")
-                        response_msg = CtrlMsg(msg_type="ERROR", error="MD5 mismatch")
-                        self.stats.fail_count += 1
-
-                recv_buffer.release()
-            else:
-                response_msg = CtrlMsg(msg_type="ACK")
-                self.stats.success_count += 1
-                self.stats.total_bytes += recv_size
-        else:
-            print("  [FAIL] Get failed")
-            self.stats.fail_count += 1
-
-        # Send response to producer
-        self.ctrl_socket.send(msgspec.msgpack.encode(response_msg))
-        # Wait for ACK
-        self.ctrl_socket.recv()
-
-        return response_msg.msg_type == "ACK"
-
-    def run(self):
-        """Run the consumer."""
-        self.print_header()
-        self.initialize()
-        self.setup_control_channel()
-
-        try:
-            if not self.connect_to_producer():
-                return
-
-            if self.config.benchmark:
-                print(
-                    f"[BENCHMARK] Running {self.config.num_transfers} "
-                    f"transfers of {self.config.tensor_size_mb} MB each..."
-                )
-
-            start_time = time.time()
-
-            for i in range(self.config.num_transfers):
-                if not self.do_transfer(i):
-                    break
-                if self.config.benchmark and (i + 1) % 10 == 0:
-                    elapsed = time.time() - start_time
-                    current_throughput = (self.stats.total_bytes / (1024 * 1024)) / elapsed
-                    print(f"  Progress: {i + 1}/{self.config.num_transfers}, Throughput: {current_throughput:.2f} MB/s")
-
-            self.stats.elapsed_time = time.time() - start_time
-            self.stats.print_summary("CONSUMER")
-
-            # Send final READY and wait for DONE
-            self.ctrl_socket.send(msgspec.msgpack.encode(CtrlMsg(msg_type="READY")))
-            self.ctrl_socket.recv()
-
-        finally:
-            self.cleanup()
-
-
-def main():
-    parser = argparse.ArgumentParser(
-        description="Cross-Node RDMA Test (Automated)",
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-        epilog="""
-Transfer Modes:
-  copy      - Normal path: tensor copied to RDMA pool (default)
-  zerocopy  - Zero-copy path: data created directly in RDMA pool
-  gpu       - GPU transfer: RDMA pool on GPU, uses GPUDirect
-
-Examples:
-  # Copy mode (default):
-  python cross_node_mooncake_transfer_engine.py --role producer \
-  --local-host hostA --remote-host hostB
-
-  # Zero-copy mode:
-  python cross_node_mooncake_transfer_engine.py --role producer \
-  --local-host hostA --remote-host hostB --mode zerocopy
-
-  # GPU mode:
-  python cross_node_mooncake_transfer_engine.py --role producer \
-  --local-host hostA --remote-host hostB --mode gpu --gpu-id 0
-
-  # Benchmark mode (skip MD5, measure pure RDMA performance):
-  python cross_node_mooncake_transfer_engine.py --role producer \
-  --local-host hostA --remote-host hostB --benchmark
-
-  # With specific RDMA device:
-  RDMA_DEVICE_NAME=mlx5_0 python cross_node_mooncake_transfer_engine.py --role producer ...
-        """,
-    )
-
-    parser.add_argument(
-        "--role", required=True, choices=["producer", "consumer"], help="Role: producer (sends) or consumer (receives)"
-    )
-    parser.add_argument("--local-host", required=True, help="Local hostname or IP address")
-    parser.add_argument("--remote-host", required=True, help="Remote hostname or IP address")
-    parser.add_argument("--local-port", type=int, default=15500, help="Local ZMQ port for RDMA data (default: 15500)")
-    parser.add_argument("--remote-port", type=int, default=15500, help="Remote ZMQ port for RDMA data (default: 15500)")
-    parser.add_argument("--ctrl-port", type=int, default=15501, help="Control channel port (default: 15501)")
-    parser.add_argument("--num-transfers", type=int, default=20, help="Number of transfers to perform (default: 3)")
-    parser.add_argument("--tensor-size-mb", type=int, default=100, help="Tensor size in MB (default: 100)")
-    parser.add_argument(
-        "--mode",
-        choices=["copy", "zerocopy", "gpu"],
-        default="copy",
-        help="Transfer mode: copy, zerocopy, or gpu (default: copy)",
-    )
-    parser.add_argument("--gpu-id", type=int, default=0, help="GPU ID for GPU mode (default: 0)")
-    parser.add_argument(
-        "--benchmark", action="store_true", help="Benchmark mode: skip MD5 verification for pure performance test"
-    )
-
-    args = parser.parse_args()
-
-    # Check Mooncake
-    if TransferEngine is None:
-        print("[ERROR] Mooncake TransferEngine is not available.")
-        print("Install with: pip install mooncake")
-        sys.exit(1)
-
-    # Check CUDA for GPU mode
-    if args.mode == "gpu":
-        if not torch.cuda.is_available():
-            print("[ERROR] CUDA is not available but GPU mode was requested.")
-            sys.exit(1)
-        if args.gpu_id >= torch.cuda.device_count():
-            print(f"[ERROR] GPU {args.gpu_id} not available. Found {torch.cuda.device_count()} GPUs.")
-            sys.exit(1)
-        print(f"[INFO] Using GPU {args.gpu_id}: {torch.cuda.get_device_name(args.gpu_id)}")
-
-    config = TransferConfig(
-        local_host=args.local_host,
-        remote_host=args.remote_host,
-        local_port=args.local_port,
-        remote_port=args.remote_port,
-        ctrl_port=args.ctrl_port,
-        num_transfers=args.num_transfers,
-        tensor_size_mb=args.tensor_size_mb,
-        mode=args.mode,
-        gpu_id=args.gpu_id,
-        benchmark=args.benchmark,
-    )
-
-    if args.role == "producer":
-        tester = Producer(config)
-    else:
-        tester = Consumer(config)
-
-    tester.run()
-
-
-if __name__ == "__main__":
-    main()

benchmarks/qwen3-omni/README.md

@@ -1,86 +0,0 @@
-# Benchmarks Guide
-
-This README explains how to (1) prepare benchmark datasets and (2) run the provided Qwen3-Omni benchmarks.
-
-## 1) Prepare the dataset (SeedTTS top100)
-
-```bash
-cd benchmarks/build_dataset
-pip install gdown
-
-# Download SeedTTS test set from Google Drive
-gdown --id 1GlSjVfSHkW3-leKKBlfrjuuTGqQ_xaLP
-
-# Extract
-tar -xf seedtts_testset.tar
-
-# Copy metadata and extract top-100 prompts
-cp seedtts_testset/en/meta.lst meta.lst
-python extract_prompts.py -i meta.lst -o top100.txt -n 100
-
-# (Optional) clean up to save space
-rm -rf seedtts_testset seedtts_testset.tar meta.lst
-```
-
-Artifacts:
-- `benchmarks/build_dataset/top100.txt` — 100 text prompts (one per line).
-
-## 2) Run benchmarks
-
-All commands assume repo root (`vllm-omni`).
-
-### A. Transformers benchmark (offline, HF Transformers)
-
-```
-bash benchmarks/qwen3-omni/transformers/eval_qwen3_moe_omni_transformers.sh
-```
-
-What it does:
-- Runs `qwen3_omni_moe_transformers.py` over `top100.txt` with `--num-prompts 100`.
-- Outputs to `benchmarks/qwen3-omni/transformers/benchmark_results/`:
-  - `perf_stats.json` — aggregated & per-prompt TPS/latency (thinker/talker/code2wav/overall).
-  - `results.json` — per-prompt outputs and audio paths.
-  - `audio/` — ~100 generated `.wav` files.
-
-Key checks:
-- `overall_tps` and `*_tps_avg` should be non-zero and reasonably stable.
-- Investigate any 0/NaN or unusually low TPS / long-tail latency.
-
-### B. vLLM Omni end-to-end benchmark (pipeline)
-
-```
-bash benchmarks/qwen3-omni/vllm_omni/eval_qwen3_moe_omni.sh
-```
-
-What it does:
-- Runs `examples/offline_inference/qwen3_omni/end2end.py` with `--log-stats`.
-- Uses `benchmarks/build_dataset/top100.txt` and writes to:
-  - Logs: `benchmarks/qwen3-omni/vllm_omni/logs/`
-    - `omni_llm_pipeline_text.orchestrator.stats.jsonl` — per-stage latency stats.
-    - `omni_llm_pipeline_text.overall.stats.jsonl` — end-to-end latency/TPS.
-    - `omni_llm_pipeline_text.stage{0,1,2}.log` — per-stage detailed logs/errors.
-  - Outputs: `benchmarks/qwen3-omni/vllm_omni/outputs/` — ~100 text and `.wav` files.
-
-Key checks:
-- Overall stats: end-to-end latency/TPS should be reasonable.
-- Orchestrator stats: per-stage latency should be stable; investigate long tails.
-- Stage logs: ensure no errors and no unusually slow stages.
-
-
-## Performance snapshot
-
-The chart below summarizes our measured Qwen3-Omni MoE end-to-end benchmark, comparing vLLM-Omni against HF Transformers. It shows the overall throughput advantage for vLLM-Omni. These are actual experiment results—please refer to this performance when evaluating or reproducing the benchmark.
-
-![vLLM-Omni vs HF](./vllm-omni-vs-hf.png)
-
-## Directory layout
-- `benchmarks/build_dataset/` — dataset prep utilities (e.g., SeedTTS top100).
-- `benchmarks/<model>/vllm_omni/` — vLLM-Omni pipeline benchmarks, logs, outputs.
-- Add new tasks under `benchmarks/<model>/...` with the same pattern: `transformers/`, `vllm_omni/`, task-specific README, and (optionally) dataset prep notes.
-- `benchmarks/<model>/vllm-omni-vs-hf.png` — current performance snapshot (overall throughput comparison).
-- `benchmarks/<model>/transformers/` — HF Transformers benchmarks (offline reference).
-
-## Troubleshooting
-- Make sure GPU/driver/FlashAttention2 requirements are met for the chosen model.
-- If downloads fail, confirm network access to Google Drive (`gdown`) and Hugging Face.
-- If audio files are missing, check for errors in stage logs or model generation.***

benchmarks/qwen3-omni/transformers/eval_qwen3_moe_omni_transformers.sh

@@ -1,26 +0,0 @@
-#!/bin/bash
-# Qwen3-Omni Transformers Benchmark Evaluation Script
-# This script must be run from the vllm-omni root directory
-
-# Get the directory where this script is located
-SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-
-# Navigate to vllm-omni root directory (4 levels up from script location)
-VLLM_OMNI_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)"
-cd "$VLLM_OMNI_ROOT" || { echo "Error: Failed to navigate to vllm-omni directory"; exit 1; }
-
-echo "Working directory: $(pwd)"
-# Verify we're in the correct directory and run benchmark
-if [[ ! -f "benchmarks/qwen3-omni/transformers/qwen3_omni_moe_transformers.py" ]]; then
-    echo "Error: Not in vllm-omni root directory. Please run from vllm-omni folder."
-else
-    cd benchmarks/qwen3-omni/transformers
-
-    python qwen3_omni_moe_transformers.py --prompts-file ../../build_dataset/top100.txt --num-prompts 100
-
-    echo "Logs and outputs are saved to $(pwd)/benchmark_results:"
-    echo "  - perf_stats.json    Aggregated/per-prompt TPS and latency (thinker/talker/code2wav/overall)"
-    echo "  - results.json       Per-prompt outputs and audio paths"
-    echo "  - audio/             Generated wav files, there should be 100 wav file generated"
-    echo "Key checks: overall_tps and *_tps_avg should be non-zero and stable; investigate 0/NaN or unusually low TPS/long-tail latency."
-fi

benchmarks/qwen3-omni/transformers/qwen3_omni_moe_model.py

@@ -1,265 +0,0 @@
-import time
-
-import torch
-from transformers import Qwen3OmniMoeForConditionalGeneration
-
-
-class Qwen3OmniMoeForConditionalGenerationWithLogging(Qwen3OmniMoeForConditionalGeneration):
-    @torch.no_grad()
-    def generate(
-        self,
-        input_ids: torch.Tensor | None = None,
-        speaker: str = "Ethan",
-        use_audio_in_video: bool = False,
-        return_audio: bool | None = None,
-        thinker_max_new_tokens: int = 1024,
-        thinker_eos_token_id: int = 151645,
-        talker_max_new_tokens: int = 4096,
-        talker_do_sample: bool = True,
-        talker_top_k: int = 50,
-        talker_top_p: float = 1.0,
-        talker_temperature: float = 0.9,
-        talker_repetition_penalty: float = 1.05,
-        **kwargs,
-    ):
-        total_t0 = time.time()
-        perf_stats = {
-            "thinker_tokens": 0,
-            "thinker_time_s": 0.0,
-            "thinker_tps": 0.0,
-            "talker_tokens": 0,
-            "talker_time_s": 0.0,
-            "talker_tps": 0.0,
-            "code2wav_tokens": 0,
-            "code2wav_time_s": 0.0,
-            "code2wav_tps": 0.0,
-            "total_tokens": 0,
-            "total_time_s": 0.0,
-            "total_tps": 0.0,
-        }
-        if return_audio and not self.has_talker:
-            raise ValueError(
-                "Cannot use talker when talker module not initialized. "
-                "Use `enable_talker` method or set enable_talker in config "
-                "to enable talker."
-            )
-        if return_audio is None:
-            return_audio = self.has_talker
-
-        shared_kwargs = {"use_audio_in_video": use_audio_in_video}
-        thinker_kwargs = {
-            "max_new_tokens": thinker_max_new_tokens,
-            "eos_token_id": thinker_eos_token_id,
-        }
-
-        talker_kwargs = {}
-        token2wav_kwargs = {}
-        if return_audio:
-            speaker_id = self.config.talker_config.speaker_id.get(speaker.lower())
-            if speaker_id is None:
-                raise NotImplementedError(f"Speaker {speaker} not implemented")
-            if input_ids.shape[0] != 1:
-                raise NotImplementedError("Qwen3-Omni currently does not support batched inference with audio output")
-            talker_suppressed_tokens = [
-                i
-                for i in range(
-                    self.config.talker_config.text_config.vocab_size - 1024,
-                    self.config.talker_config.text_config.vocab_size,
-                )
-                if i != self.config.talker_config.codec_eos_token_id
-            ]  # Suppress additional special tokens, should not be predicted
-            talker_kwargs = {
-                "max_new_tokens": talker_max_new_tokens,
-                "do_sample": talker_do_sample,
-                "top_k": talker_top_k,
-                "top_p": talker_top_p,
-                "temperature": talker_temperature,
-                "eos_token_id": self.config.talker_config.codec_eos_token_id,
-                "repetition_penalty": talker_repetition_penalty,
-                "suppress_tokens": talker_suppressed_tokens,
-                "output_hidden_states": True,
-                "return_dict_in_generate": True,
-            }
-            token2wav_kwargs = {}
-
-        for key, value in kwargs.items():
-            if key.startswith("thinker_"):
-                thinker_kwargs[key[len("thinker_") :]] = value
-            elif key.startswith("talker_"):
-                talker_kwargs[key[len("talker_") :]] = value
-            elif key.startswith("token2wav_"):
-                token2wav_kwargs[key[len("token2wav_") :]] = value
-            # Process special input values
-            elif key == "feature_attention_mask":
-                thinker_kwargs[key] = value
-                talker_kwargs["audio_feature_lengths"] = torch.sum(value, dim=1)
-            elif key in ("input_features", "attention_mask"):
-                thinker_kwargs[key] = value
-            # Put other key to shared kwargs
-            else:
-                shared_kwargs[key] = value
-
-        # Merge kwargs
-        for key, value in shared_kwargs.items():
-            if key not in thinker_kwargs:
-                thinker_kwargs[key] = value
-            if key not in talker_kwargs and key in ["image_grid_thw", "video_grid_thw", "video_second_per_grid"]:
-                talker_kwargs[key] = value
-            if key not in token2wav_kwargs:
-                token2wav_kwargs[key] = value
-
-        # 1. Generate from thinker module
-        generate_audio = return_audio and self.has_talker
-        if generate_audio:
-            thinker_kwargs["output_hidden_states"] = True
-            thinker_kwargs["return_dict_in_generate"] = True
-
-        t0 = time.time()
-        thinker_result = self.thinker.generate(input_ids=input_ids, **thinker_kwargs)
-        t1 = time.time()
-        perf_stats["thinker_time_s"] = max(0.0, t1 - t0)
-        try:
-            prompt_len = int(input_ids.shape[1]) if input_ids is not None else 0
-            total_len = int(thinker_result.sequences.shape[-1])
-            thinker_out_len = max(0, total_len - prompt_len)
-        except Exception:
-            thinker_out_len = 0
-        perf_stats["thinker_tokens"] = thinker_out_len
-        perf_stats["thinker_tps"] = (
-            (thinker_out_len / perf_stats["thinker_time_s"]) if perf_stats["thinker_time_s"] > 0 else 0.0
-        )
-
-        if not generate_audio:
-            perf_stats["total_tokens"] = perf_stats["thinker_tokens"]
-            perf_stats["total_time_s"] = time.time() - total_t0
-            perf_stats["total_tps"] = (
-                (perf_stats["total_tokens"] / perf_stats["total_time_s"]) if perf_stats["total_time_s"] > 0 else 0.0
-            )
-            # attach stats to self
-            setattr(self, "_perf_stats_last", perf_stats)
-            if not hasattr(self, "_perf_stats_history"):
-                setattr(self, "_perf_stats_history", [])
-            self._perf_stats_history.append(perf_stats)
-            return thinker_result, None
-
-        # 2. Prepare talker input
-        thinker_embed = torch.cat([hidden_states[0] for hidden_states in thinker_result.hidden_states], dim=1).to(
-            self.talker.device
-        )  # [1 t d]
-        thinker_hidden = torch.cat(
-            [
-                hidden_states[self.config.talker_config.accept_hidden_layer]
-                for hidden_states in thinker_result.hidden_states
-            ],
-            dim=1,
-        ).to(self.talker.device)  # [1 t d]
-
-        im_start_indexes = torch.cat(
-            (
-                torch.nonzero(input_ids[0] == self.config.im_start_token_id).squeeze(),
-                torch.tensor([thinker_result.sequences.shape[-1]], device=input_ids.device, dtype=input_ids.dtype),
-            ),
-            dim=-1,
-        ).to(self.talker.device)  # Shape [n_starts + 1]; Take batch 0 since batched inference is not supported here.
-        multimodal_mask = (
-            (thinker_result.sequences == self.config.thinker_config.audio_token_id) |
-            (thinker_result.sequences == self.config.thinker_config.image_token_id) |
-            (thinker_result.sequences == self.config.thinker_config.video_token_id)
-        ).to(self.talker.device)  # [1 t] # fmt: skip
-
-        talker_special_tokens = torch.tensor(
-            [[self.config.tts_bos_token_id, self.config.tts_eos_token_id, self.config.tts_pad_token_id]],
-            device=self.thinker.device,
-            dtype=input_ids.dtype,
-        )
-        tts_bos_embed, tts_eos_embed, tts_pad_embed = (
-            self.talker.text_projection(self.thinker.get_input_embeddings()(talker_special_tokens))
-            .to(self.talker.device)
-            .chunk(3, dim=1)
-        )  # 3 * [1 1 d]
-
-        talker_input_embeds = []  # [1 t d]
-        talker_input_ids = []
-        # For every chatml parts
-        for i in range(len(im_start_indexes) - 1):
-            im_start_index = im_start_indexes[i]
-            segment_end_index = im_start_indexes[i + 1]
-            role_token = input_ids[0][im_start_index + 1]
-            # Talker should ignore thinker system prompt
-            if role_token == self.config.system_token_id:
-                continue
-            # Talker takes word embeddings for tokens and hidden state from `accept_hidden_layer` for multimodal inputs
-            elif role_token == self.config.user_token_id:
-                talker_user_part = self._get_talker_user_parts(
-                    im_start_index, segment_end_index, multimodal_mask, thinker_hidden, thinker_embed
-                )
-                talker_input_embeds.append(talker_user_part)
-                talker_input_ids.append(thinker_result.sequences[:, im_start_index:segment_end_index])
-            # Take assistant output (for now)
-            elif role_token == self.config.assistant_token_id and i == len(im_start_indexes) - 2:
-                talker_assistant_embeds, talker_assistant_ids, trailing_text_hidden = self._get_talker_assistant_parts(
-                    im_start_index,
-                    segment_end_index,
-                    speaker_id,
-                    thinker_embed,
-                    tts_pad_embed,
-                    tts_bos_embed,
-                    tts_eos_embed,
-                )
-                talker_input_embeds.append(talker_assistant_embeds)
-                talker_input_ids.append(talker_assistant_ids)
-            # History assistant output (ignore for now)
-            elif role_token == self.config.assistant_token_id and i != len(im_start_indexes) - 2:
-                continue
-            else:
-                raise AssertionError("Expect role id after <|im_start|> (assistant, user, system)")
-        talker_input_embed = torch.cat([embed.to(self.talker.device) for embed in talker_input_embeds], dim=1)
-        talker_input_id = torch.cat([embed.to(self.talker.device) for embed in talker_input_ids], dim=1)
-        t2 = time.time()
-        talker_result = self.talker.generate(
-            inputs_embeds=talker_input_embed,
-            trailing_text_hidden=trailing_text_hidden,
-            tts_pad_embed=tts_pad_embed,
-            talker_input_ids=talker_input_id,  # Not use input_ids to prevent repetition penalty out of bound
-            **talker_kwargs,
-        )
-        t3 = time.time()
-        perf_stats["talker_time_s"] = max(0.0, t3 - t2)
-        talker_codes = (
-            torch.stack([hid[-1] for hid in talker_result.hidden_states if hid[-1] is not None], dim=1)
-            .transpose(1, 2)
-            .to(self.code2wav.device)
-        )
-        try:
-            # codes shape: (B, num_quantizers, T). We log T as token length.
-            perf_stats["talker_tokens"] = int(talker_codes.shape[-1])
-        except Exception:
-            perf_stats["talker_tokens"] = 0
-        perf_stats["talker_tps"] = (
-            (perf_stats["talker_tokens"] / perf_stats["talker_time_s"]) if perf_stats["talker_time_s"] > 0 else 0.0
-        )
-        t4 = time.time()
-        talker_wavs = self.code2wav.chunked_decode(talker_codes, chunk_size=300, left_context_size=25).float()
-        t5 = time.time()
-        perf_stats["code2wav_time_s"] = max(0.0, t5 - t4)
-        perf_stats["code2wav_tokens"] = perf_stats["talker_tokens"]  # same T, not times 16
-        perf_stats["code2wav_tps"] = (
-            (perf_stats["code2wav_tokens"] / perf_stats["code2wav_time_s"])
-            if perf_stats["code2wav_time_s"] > 0
-            else 0.0
-        )
-        perf_stats["total_tokens"] = perf_stats["thinker_tokens"] + perf_stats["talker_tokens"]
-        perf_stats["total_time_s"] = time.time() - total_t0
-        perf_stats["total_tps"] = (
-            (perf_stats["total_tokens"] / perf_stats["total_time_s"]) if perf_stats["total_time_s"] > 0 else 0.0
-        )
-        setattr(self, "_perf_stats_last", perf_stats)
-        if not hasattr(self, "_perf_stats_history"):
-            setattr(self, "_perf_stats_history", [])
-        self._perf_stats_history.append(perf_stats)
-        return thinker_result, talker_wavs.float()
-
-
-__all__ = [
-    "Qwen3OmniMoeForConditionalGenerationWithLogging",
-]

benchmarks/qwen3-omni/transformers/qwen3_omni_moe_transformers.py

@@ -1,275 +0,0 @@
-import argparse
-import json
-import os
-
-import soundfile as sf
-from qwen3_omni_moe_model import Qwen3OmniMoeForConditionalGenerationWithLogging
-from qwen_omni_utils import process_mm_info
-from tqdm import tqdm
-from transformers import Qwen3OmniMoeProcessor
-
-MODEL_PATH = "Qwen/Qwen3-Omni-30B-A3B-Instruct"
-# MODEL_PATH = "Qwen/Qwen3-Omni-30B-A3B-Thinking"
-
-
-def load_prompts(prompts_file: str) -> list[str]:
-    """Load prompts from a text file, one prompt per line."""
-    prompts = []
-    with open(prompts_file, encoding="utf-8") as f:
-        for line in f:
-            line = line.strip()
-            if line:
-                prompts.append(line)
-    return prompts
-
-
-def run_benchmark(
-    model,
-    processor,
-    prompts: list[str],
-    output_dir: str = "benchmark_results",
-    speaker: str = "Ethan",
-    use_audio_in_video: bool = True,
-):
-    """
-    Run benchmark on a list of prompts and collect performance stats.
-
-    Args:
-        model: The Qwen3OmniMoe model
-        processor: The Qwen3OmniMoe processor
-        prompts: List of text prompts to process
-        output_dir: Directory to save results
-        speaker: Speaker voice for audio output
-        use_audio_in_video: Whether to use audio in video
-
-    Returns:
-        tuple: (aggregated_stats, results, audio_outputs)
-            - aggregated_stats: dict with aggregated performance statistics
-            - results: list of dicts with per-prompt results
-            - audio_outputs: list of audio tensors/arrays (or None if no audio)
-    """
-    os.makedirs(output_dir, exist_ok=True)
-    audio_dir = os.path.join(output_dir, "audio")
-    os.makedirs(audio_dir, exist_ok=True)
-
-    all_stats = []
-    results = []
-    audio_outputs = []
-
-    for idx, prompt in enumerate(tqdm(prompts, desc="Processing prompts")):
-        conversation = [
-            {
-                "role": "user",
-                "content": [{"type": "text", "text": prompt}],
-            },
-        ]
-
-        # Preparation for inference
-        text = processor.apply_chat_template(conversation, add_generation_prompt=True, tokenize=False)
-        audios, images, videos = process_mm_info(conversation, use_audio_in_video=use_audio_in_video)
-        inputs = processor(
-            text=text,
-            audio=audios,
-            images=images,
-            videos=videos,
-            return_tensors="pt",
-            padding=True,
-            use_audio_in_video=use_audio_in_video,
-        )
-        inputs = inputs.to(model.device).to(model.dtype)
-
-        # Inference: Generation of the output text and audio
-        text_ids, audio = model.generate(
-            **inputs, speaker=speaker, thinker_return_dict_in_generate=True, use_audio_in_video=use_audio_in_video
-        )
-
-        # Decode output text
-        output_text = processor.batch_decode(
-            text_ids.sequences[:, inputs["input_ids"].shape[1] :],
-            skip_special_tokens=True,
-            clean_up_tokenization_spaces=False,
-        )[0]
-
-        # Collect performance stats
-        perf_stats = None
-        if hasattr(model, "_perf_stats_last"):
-            perf_stats = model._perf_stats_last.copy()
-            perf_stats["prompt_idx"] = idx
-            perf_stats["prompt"] = prompt
-            all_stats.append(perf_stats)
-
-        # Save audio and collect audio output
-        audio_path = None
-        audio_data = None
-        if audio is not None:
-            audio_data = audio.reshape(-1).detach().cpu().numpy()
-            audio_path = os.path.join(audio_dir, f"output_{idx:04d}.wav")
-            sf.write(
-                audio_path,
-                audio_data,
-                samplerate=24000,
-            )
-            audio_outputs.append(audio_data)
-        else:
-            audio_outputs.append(None)
-
-        # Save result
-        result = {
-            "idx": idx,
-            "prompt": prompt,
-            "output": output_text,
-            "audio_path": audio_path,
-            "perf_stats": perf_stats,
-        }
-        results.append(result)
-
-    # Aggregate statistics
-    aggregated_stats = aggregate_stats(all_stats)
-
-    # Save all results
-    results_path = os.path.join(output_dir, "results.json")
-    with open(results_path, "w", encoding="utf-8") as f:
-        json.dump(results, f, ensure_ascii=False, indent=2)
-
-    # Save aggregated stats
-    stats_path = os.path.join(output_dir, "perf_stats.json")
-    with open(stats_path, "w", encoding="utf-8") as f:
-        json.dump({"aggregated": aggregated_stats, "per_prompt": all_stats}, f, ensure_ascii=False, indent=2)
-
-    # Count saved audio files
-    num_audio_saved = sum(1 for a in audio_outputs if a is not None)
-    print(f"\nSaved {num_audio_saved} audio files to {audio_dir}/")
-
-    return aggregated_stats, results, audio_outputs
-
-
-def aggregate_stats(all_stats: list[dict]) -> dict:
-    """Aggregate performance statistics from multiple runs."""
-    if not all_stats:
-        return {}
-
-    keys = [
-        "thinker_tokens",
-        "thinker_time_s",
-        "thinker_tps",
-        "talker_tokens",
-        "talker_time_s",
-        "talker_tps",
-        "code2wav_tokens",
-        "code2wav_time_s",
-        "code2wav_tps",
-        "total_tokens",
-        "total_time_s",
-        "total_tps",
-    ]
-
-    aggregated = {
-        "num_samples": len(all_stats),
-    }
-
-    for key in keys:
-        values = [s.get(key, 0) for s in all_stats if key in s]
-        if values:
-            aggregated[f"{key}_sum"] = sum(values)
-            aggregated[f"{key}_avg"] = sum(values) / len(values)
-            aggregated[f"{key}_min"] = min(values)
-            aggregated[f"{key}_max"] = max(values)
-
-    # Calculate overall throughput
-    total_tokens = aggregated.get("total_tokens_sum", 0)
-    total_time = aggregated.get("total_time_s_sum", 0)
-    if total_time > 0:
-        aggregated["overall_tps"] = total_tokens / total_time
-
-    return aggregated
-
-
-def print_stats(stats: dict):
-    """Print performance statistics in a formatted way."""
-    print("\n" + "=" * 60)
-    print("Performance Statistics Summary")
-    print("=" * 60)
-
-    print(f"\nNumber of samples: {stats.get('num_samples', 0)}")
-
-    print("\n--- Thinker ---")
-    print(f"  Total tokens:  {stats.get('thinker_tokens_sum', 0):.0f}")
-    print(f"  Total time:    {stats.get('thinker_time_s_sum', 0):.2f}s")
-    print(f"  Avg TPS:       {stats.get('thinker_tps_avg', 0):.2f}")
-    print(f"  Min TPS:       {stats.get('thinker_tps_min', 0):.2f}")
-    print(f"  Max TPS:       {stats.get('thinker_tps_max', 0):.2f}")
-
-    print("\n--- Talker ---")
-    print(f"  Total tokens:  {stats.get('talker_tokens_sum', 0):.0f}")
-    print(f"  Total time:    {stats.get('talker_time_s_sum', 0):.2f}s")
-    print(f"  Avg TPS:       {stats.get('talker_tps_avg', 0):.2f}")
-    print(f"  Min TPS:       {stats.get('talker_tps_min', 0):.2f}")
-    print(f"  Max TPS:       {stats.get('talker_tps_max', 0):.2f}")
-
-    print("\n--- Code2Wav ---")
-    print(f"  Total tokens:  {stats.get('code2wav_tokens_sum', 0):.0f}")
-    print(f"  Total time:    {stats.get('code2wav_time_s_sum', 0):.2f}s")
-    print(f"  Avg TPS:       {stats.get('code2wav_tps_avg', 0):.2f}")
-    print(f"  Min TPS:       {stats.get('code2wav_tps_min', 0):.2f}")
-    print(f"  Max TPS:       {stats.get('code2wav_tps_max', 0):.2f}")
-
-    print("\n--- Overall ---")
-    print(f"  Total tokens:  {stats.get('total_tokens_sum', 0):.0f}")
-    print(f"  Total time:    {stats.get('total_time_s_sum', 0):.2f}s")
-    print(f"  Overall TPS:   {stats.get('overall_tps', 0):.2f}")
-    print(f"  Avg TPS:       {stats.get('total_tps_avg', 0):.2f}")
-    print(f"  Min TPS:       {stats.get('total_tps_min', 0):.2f}")
-    print(f"  Max TPS:       {stats.get('total_tps_max', 0):.2f}")
-
-    print("=" * 60 + "\n")
-
-
-def main():
-    parser = argparse.ArgumentParser(description="Qwen3-Omni Benchmark Script")
-    parser.add_argument(
-        "--prompts-file",
-        type=str,
-        default="benchmark/build_dataset/top100.txt",
-        help="Path to the prompts file (one prompt per line)",
-    )
-    parser.add_argument(
-        "--output-dir", type=str, default="benchmark_results", help="Directory to save benchmark results"
-    )
-    parser.add_argument("--model-path", type=str, default=MODEL_PATH, help="Path to the model")
-    parser.add_argument("--speaker", type=str, default="Ethan", help="Speaker voice for audio output")
-    parser.add_argument("--num-prompts", type=int, default=None, help="Number of prompts to process (default: all)")
-    args = parser.parse_args()
-
-    # Load model and processor
-    print(f"Loading model from {args.model_path}...")
-    model = Qwen3OmniMoeForConditionalGenerationWithLogging.from_pretrained(
-        args.model_path,
-        dtype="auto",
-        device_map="auto",
-        attn_implementation="flash_attention_2",
-    )
-    processor = Qwen3OmniMoeProcessor.from_pretrained(args.model_path)
-
-    # Benchmark mode
-    print(f"Loading prompts from {args.prompts_file}...")
-    prompts = load_prompts(args.prompts_file)
-
-    if args.num_prompts:
-        prompts = prompts[: args.num_prompts]
-
-    print(f"Running benchmark on {len(prompts)} prompts...")
-
-    aggregated_stats, results, audio_outputs = run_benchmark(
-        model=model,
-        processor=processor,
-        prompts=prompts,
-        output_dir=args.output_dir,
-        speaker=args.speaker,
-    )
-
-    print_stats(aggregated_stats)
-    print(f"\nResults saved to {args.output_dir}/")
-
-
-if __name__ == "__main__":
-    main()

benchmarks/qwen3-omni/vllm_omni/eval_qwen3_moe_omni.sh

@@ -1,37 +0,0 @@
-#!/bin/bash
-# Qwen3-Omni Benchmark Evaluation Script
-# This script must be run from the vllm-omni root directory
-
-# Get the directory where this script is located
-SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-
-# Navigate to vllm-omni root directory (4 levels up from script location)
-VLLM_OMNI_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)"
-cd "$VLLM_OMNI_ROOT" || { echo "Error: Failed to navigate to vllm-omni directory"; exit 1; }
-
-echo "Working directory: $(pwd)"
-
-# Verify we're in the correct directory and run benchmark
-if [[ ! -d "benchmarks/qwen3-omni/vllm_omni" ]]; then
-    echo "Error: Not in vllm-omni root directory. Please run from vllm-omni folder."
-else
-    log_dir=benchmarks/qwen3-omni/vllm_omni/logs
-    outputs_dir=benchmarks/qwen3-omni/vllm_omni/outputs
-    end2end_script_path=examples/offline_inference/qwen3_omni/end2end.py
-    build_dataset_path=benchmarks/build_dataset/top100.txt
-
-    python $end2end_script_path --output-wav $outputs_dir \
-                      --query-type text \
-                      --txt-prompts $build_dataset_path \
-                      --log-stats \
-                      --log-dir $log_dir
-    echo "Logs and outputs are saved in ${log_dir} and ${outputs_dir} respectively:"
-    echo "  - omni_llm_pipeline_text                       run dir/base name"
-    echo "  - omni_llm_pipeline_text.orchestrator.stats.jsonl  orchestrator-stage latency stats"
-    echo "  - omni_llm_pipeline_text.overall.stats.jsonl       overall latency/TPS stats"
-    echo "  - omni_llm_pipeline_text.stage0.log                per-stage detailed logs"
-    echo "  - omni_llm_pipeline_text.stage1.log"
-    echo "  - omni_llm_pipeline_text.stage2.log"
-    echo "Key checks: overall.stats.jsonl for end-to-end latency/TPS; orchestrator.stats.jsonl for stable per-stage latency; stage*.log for errors or long tails."
-    echo "  - outputs/             Generated txt and wav files, there should be 100 text and wav files generated respectively"
-fi

collect_env.py

@@ -1,760 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-
-# ruff: noqa
-# code borrowed from https://github.com/pytorch/pytorch/blob/main/torch/utils/collect_env.py
-
-import datetime
-import locale
-import os
-import subprocess
-import sys
-
-# Unlike the rest of the PyTorch this file must be python2 compliant.
-# This script outputs relevant system environment info
-# Run it with `python collect_env.py` or `python -m torch.utils.collect_env`
-from collections import namedtuple
-
-import regex as re
-
-from vllm.envs import environment_variables
-
-try:
-    import torch
-
-    TORCH_AVAILABLE = True
-except (ImportError, NameError, AttributeError, OSError):
-    TORCH_AVAILABLE = False
-
-# System Environment Information
-SystemEnv = namedtuple(
-    "SystemEnv",
-    [
-        "torch_version",
-        "is_debug_build",
-        "cuda_compiled_version",
-        "gcc_version",
-        "clang_version",
-        "cmake_version",
-        "os",
-        "libc_version",
-        "python_version",
-        "python_platform",
-        "is_cuda_available",
-        "cuda_runtime_version",
-        "cuda_module_loading",
-        "nvidia_driver_version",
-        "nvidia_gpu_models",
-        "cudnn_version",
-        "pip_version",  # 'pip' or 'pip3'
-        "pip_packages",
-        "conda_packages",
-        "hip_compiled_version",
-        "hip_runtime_version",
-        "miopen_runtime_version",
-        "caching_allocator_config",
-        "is_xnnpack_available",
-        "cpu_info",
-        "rocm_version",  # vllm specific field
-        "vllm_version",  # vllm specific field
-        "vllm_omni_version",  # vllm-omni specific field
-        "vllm_build_flags",  # vllm specific field
-        "gpu_topo",  # vllm specific field
-        "env_vars",
-    ],
-)
-
-DEFAULT_CONDA_PATTERNS = {
-    "torch",
-    "numpy",
-    "cudatoolkit",
-    "soumith",
-    "mkl",
-    "magma",
-    "triton",
-    "optree",
-    "nccl",
-    "transformers",
-    "zmq",
-    "nvidia",
-    "pynvml",
-    "flashinfer-python",
-}
-
-DEFAULT_PIP_PATTERNS = {
-    "torch",
-    "numpy",
-    "mypy",
-    "flake8",
-    "triton",
-    "optree",
-    "onnx",
-    "nccl",
-    "transformers",
-    "zmq",
-    "nvidia",
-    "pynvml",
-    "flashinfer-python",
-}
-
-
-def run(command):
-    """Return (return-code, stdout, stderr)."""
-    shell = True if type(command) is str else False
-    try:
-        p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=shell)
-        raw_output, raw_err = p.communicate()
-        rc = p.returncode
-        if get_platform() == "win32":
-            enc = "oem"
-        else:
-            enc = locale.getpreferredencoding()
-        output = raw_output.decode(enc)
-        if command == "nvidia-smi topo -m":
-            # don't remove the leading whitespace of `nvidia-smi topo -m`
-            #   because they are meaningful
-            output = output.rstrip()
-        else:
-            output = output.strip()
-        err = raw_err.decode(enc)
-        return rc, output, err.strip()
-
-    except FileNotFoundError:
-        cmd_str = command if isinstance(command, str) else command[0]
-        return 127, "", f"Command not found: {cmd_str}"
-
-
-def run_and_read_all(run_lambda, command):
-    """Run command using run_lambda; reads and returns entire output if rc is 0."""
-    rc, out, _ = run_lambda(command)
-    if rc != 0:
-        return None
-    return out
-
-
-def run_and_parse_first_match(run_lambda, command, regex):
-    """Run command using run_lambda, returns the first regex match if it exists."""
-    rc, out, _ = run_lambda(command)
-    if rc != 0:
-        return None
-    match = re.search(regex, out)
-    if match is None:
-        return None
-    return match.group(1)
-
-
-def run_and_return_first_line(run_lambda, command):
-    """Run command using run_lambda and returns first line if output is not empty."""
-    rc, out, _ = run_lambda(command)
-    if rc != 0:
-        return None
-    return out.split("\n")[0]
-
-
-def get_conda_packages(run_lambda, patterns=None):
-    if patterns is None:
-        patterns = DEFAULT_CONDA_PATTERNS
-    conda = os.environ.get("CONDA_EXE", "conda")
-    out = run_and_read_all(run_lambda, [conda, "list"])
-    if out is None:
-        return out
-
-    return "\n".join(
-        line for line in out.splitlines() if not line.startswith("#") and any(name in line for name in patterns)
-    )
-
-
-def get_gcc_version(run_lambda):
-    return run_and_parse_first_match(run_lambda, "gcc --version", r"gcc (.*)")
-
-
-def get_clang_version(run_lambda):
-    return run_and_parse_first_match(run_lambda, "clang --version", r"clang version (.*)")
-
-
-def get_cmake_version(run_lambda):
-    return run_and_parse_first_match(run_lambda, "cmake --version", r"cmake (.*)")
-
-
-def get_nvidia_driver_version(run_lambda):
-    if get_platform() == "darwin":
-        cmd = "kextstat | grep -i cuda"
-        return run_and_parse_first_match(run_lambda, cmd, r"com[.]nvidia[.]CUDA [(](.*?)[)]")
-    smi = get_nvidia_smi()
-    return run_and_parse_first_match(run_lambda, smi, r"Driver Version: (.*?) ")
-
-
-def get_gpu_info(run_lambda):
-    if get_platform() == "darwin" or (
-        TORCH_AVAILABLE and hasattr(torch.version, "hip") and torch.version.hip is not None
-    ):
-        if TORCH_AVAILABLE and torch.cuda.is_available():
-            if torch.version.hip is not None:
-                prop = torch.cuda.get_device_properties(0)
-                if hasattr(prop, "gcnArchName"):
-                    gcnArch = " ({})".format(prop.gcnArchName)
-                else:
-                    gcnArch = "NoGCNArchNameOnOldPyTorch"
-            else:
-                gcnArch = ""
-            return torch.cuda.get_device_name(None) + gcnArch
-        return None
-    smi = get_nvidia_smi()
-    uuid_regex = re.compile(r" \(UUID: .+?\)")
-    rc, out, _ = run_lambda(smi + " -L")
-    if rc != 0:
-        return None
-    # Anonymize GPUs by removing their UUID
-    return re.sub(uuid_regex, "", out)
-
-
-def get_running_cuda_version(run_lambda):
-    return run_and_parse_first_match(run_lambda, "nvcc --version", r"release .+ V(.*)")
-
-
-def get_cudnn_version(run_lambda):
-    """Return a list of libcudnn.so; it's hard to tell which one is being used."""
-    if get_platform() == "win32":
-        system_root = os.environ.get("SYSTEMROOT", "C:\\Windows")
-        cuda_path = os.environ.get("CUDA_PATH", "%CUDA_PATH%")
-        where_cmd = os.path.join(system_root, "System32", "where")
-        cudnn_cmd = '{} /R "{}\\bin" cudnn*.dll'.format(where_cmd, cuda_path)
-    elif get_platform() == "darwin":
-        # CUDA libraries and drivers can be found in /usr/local/cuda/. See
-        # https://docs.nvidia.com/cuda/cuda-installation-guide-mac-os-x/index.html#install
-        # https://docs.nvidia.com/deeplearning/sdk/cudnn-install/index.html#installmac
-        # Use CUDNN_LIBRARY when cudnn library is installed elsewhere.
-        cudnn_cmd = "ls /usr/local/cuda/lib/libcudnn*"
-    else:
-        cudnn_cmd = 'ldconfig -p | grep libcudnn | rev | cut -d" " -f1 | rev'
-    rc, out, _ = run_lambda(cudnn_cmd)
-    # find will return 1 if there are permission errors or if not found
-    if len(out) == 0 or (rc != 1 and rc != 0):
-        l = os.environ.get("CUDNN_LIBRARY")
-        if l is not None and os.path.isfile(l):
-            return os.path.realpath(l)
-        return None
-    files_set = set()
-    for fn in out.split("\n"):
-        fn = os.path.realpath(fn)  # eliminate symbolic links
-        if os.path.isfile(fn):
-            files_set.add(fn)
-    if not files_set:
-        return None
-    # Alphabetize the result because the order is non-deterministic otherwise
-    files = sorted(files_set)
-    if len(files) == 1:
-        return files[0]
-    result = "\n".join(files)
-    return "Probably one of the following:\n{}".format(result)
-
-
-def get_nvidia_smi():
-    # Note: nvidia-smi is currently available only on Windows and Linux
-    smi = "nvidia-smi"
-    if get_platform() == "win32":
-        system_root = os.environ.get("SYSTEMROOT", "C:\\Windows")
-        program_files_root = os.environ.get("PROGRAMFILES", "C:\\Program Files")
-        legacy_path = os.path.join(program_files_root, "NVIDIA Corporation", "NVSMI", smi)
-        new_path = os.path.join(system_root, "System32", smi)
-        smis = [new_path, legacy_path]
-        for candidate_smi in smis:
-            if os.path.exists(candidate_smi):
-                smi = '"{}"'.format(candidate_smi)
-                break
-    return smi
-
-
-def get_rocm_version(run_lambda):
-    """Returns the ROCm version if available, otherwise 'N/A'."""
-    return run_and_parse_first_match(run_lambda, "hipcc --version", r"HIP version: (\S+)")
-
-
-def get_vllm_version():
-    from vllm import __version__, __version_tuple__
-
-    if __version__ == "dev":
-        return "N/A (dev)"
-    version_str = __version_tuple__[-1]
-    if isinstance(version_str, str) and version_str.startswith("g"):
-        # it's a dev build
-        if "." in version_str:
-            # it's a dev build containing local changes
-            git_sha = version_str.split(".")[0][1:]
-            date = version_str.split(".")[-1][1:]
-            return f"{__version__} (git sha: {git_sha}, date: {date})"
-        else:
-            # it's a dev build without local changes
-            git_sha = version_str[1:]  # type: ignore
-            return f"{__version__} (git sha: {git_sha})"
-    return __version__
-
-
-def get_vllm_omni_version(run_lambda):
-    try:
-        import vllm_omni
-        from vllm_omni import __version__, __version_tuple__
-
-        version_str = __version_tuple__[-1]
-        if isinstance(version_str, str) and version_str.startswith("g"):
-            if "." in version_str:
-                git_sha = version_str.split(".")[0][1:]
-                date = version_str.split(".")[-1][1:]
-                return f"{__version__} (git sha: {git_sha}, date: {date})"
-            else:
-                git_sha = version_str[1:]
-                return f"{__version__} (git sha: {git_sha})"
-
-        package_dir = os.path.dirname(os.path.abspath(vllm_omni.__file__))
-        git_sha = run_and_read_all(run_lambda, f"git -C {package_dir} rev-parse --short HEAD")
-        if git_sha:
-            return f"{__version__} (git sha: {git_sha})"
-
-        return __version__
-    except ImportError:
-        return "N/A (vllm_omni not installed)"
-
-
-def summarize_vllm_build_flags():
-    # This could be a static method if the flags are constant, or dynamic if you need to check environment variables, etc.
-    return "CUDA Archs: {}; ROCm: {}".format(
-        os.environ.get("TORCH_CUDA_ARCH_LIST", "Not Set"),
-        "Enabled" if os.environ.get("ROCM_HOME") else "Disabled",
-    )
-
-
-def get_gpu_topo(run_lambda):
-    output = None
-
-    if get_platform() == "linux":
-        output = run_and_read_all(run_lambda, "nvidia-smi topo -m")
-        if output is None:
-            output = run_and_read_all(run_lambda, "rocm-smi --showtopo")
-
-    return output
-
-
-def get_cpu_info(run_lambda):
-    rc, out, err = 0, "", ""
-    if get_platform() == "linux":
-        rc, out, err = run_lambda("lscpu")
-    elif get_platform() == "win32":
-        rc, out, err = run_lambda(
-            "wmic cpu get Name,Manufacturer,Family,Architecture,ProcessorType,DeviceID, \
-        CurrentClockSpeed,MaxClockSpeed,L2CacheSize,L2CacheSpeed,Revision /VALUE"
-        )
-    elif get_platform() == "darwin":
-        rc, out, err = run_lambda("sysctl -n machdep.cpu.brand_string")
-    cpu_info = "None"
-    if rc == 0:
-        cpu_info = out
-    else:
-        cpu_info = err
-    return cpu_info
-
-
-def get_platform():
-    if sys.platform.startswith("linux"):
-        return "linux"
-    elif sys.platform.startswith("win32"):
-        return "win32"
-    elif sys.platform.startswith("cygwin"):
-        return "cygwin"
-    elif sys.platform.startswith("darwin"):
-        return "darwin"
-    else:
-        return sys.platform
-
-
-def get_mac_version(run_lambda):
-    return run_and_parse_first_match(run_lambda, "sw_vers -productVersion", r"(.*)")
-
-
-def get_windows_version(run_lambda):
-    system_root = os.environ.get("SYSTEMROOT", "C:\\Windows")
-    wmic_cmd = os.path.join(system_root, "System32", "Wbem", "wmic")
-    findstr_cmd = os.path.join(system_root, "System32", "findstr")
-    return run_and_read_all(run_lambda, "{} os get Caption | {} /v Caption".format(wmic_cmd, findstr_cmd))
-
-
-def get_lsb_version(run_lambda):
-    return run_and_parse_first_match(run_lambda, "lsb_release -a", r"Description:\t(.*)")
-
-
-def check_release_file(run_lambda):
-    return run_and_parse_first_match(run_lambda, "cat /etc/*-release", r'PRETTY_NAME="(.*)"')
-
-
-def get_os(run_lambda):
-    from platform import machine
-
-    platform = get_platform()
-
-    if platform == "win32" or platform == "cygwin":
-        return get_windows_version(run_lambda)
-
-    if platform == "darwin":
-        version = get_mac_version(run_lambda)
-        if version is None:
-            return None
-        return "macOS {} ({})".format(version, machine())
-
-    if platform == "linux":
-        # Ubuntu/Debian based
-        desc = get_lsb_version(run_lambda)
-        if desc is not None:
-            return "{} ({})".format(desc, machine())
-
-        # Try reading /etc/*-release
-        desc = check_release_file(run_lambda)
-        if desc is not None:
-            return "{} ({})".format(desc, machine())
-
-        return "{} ({})".format(platform, machine())
-
-    # Unknown platform
-    return platform
-
-
-def get_python_platform():
-    import platform
-
-    return platform.platform()
-
-
-def get_libc_version():
-    import platform
-
-    if get_platform() != "linux":
-        return "N/A"
-    return "-".join(platform.libc_ver())
-
-
-def is_uv_venv():
-    if os.environ.get("UV"):
-        return True
-    pyvenv_cfg_path = os.path.join(sys.prefix, "pyvenv.cfg")
-    if os.path.exists(pyvenv_cfg_path):
-        with open(pyvenv_cfg_path, "r") as f:
-            return any(line.startswith("uv = ") for line in f)
-    return False
-
-
-def get_pip_packages(run_lambda, patterns=None):
-    """Return `pip list` output. Note: will also find conda-installed pytorch and numpy packages."""
-    if patterns is None:
-        patterns = DEFAULT_PIP_PATTERNS
-
-    def run_with_pip():
-        try:
-            import importlib.util
-
-            pip_spec = importlib.util.find_spec("pip")
-            pip_available = pip_spec is not None
-        except ImportError:
-            pip_available = False
-
-        if pip_available:
-            cmd = [sys.executable, "-mpip", "list", "--format=freeze"]
-        elif is_uv_venv():
-            print("uv is set")
-            cmd = ["uv", "pip", "list", "--format=freeze"]
-        else:
-            raise RuntimeError("Could not collect pip list output (pip or uv module not available)")
-
-        out = run_and_read_all(run_lambda, cmd)
-        return "\n".join(line for line in out.splitlines() if any(name in line for name in patterns))
-
-    pip_version = "pip3" if sys.version[0] == "3" else "pip"
-    out = run_with_pip()
-    return pip_version, out
-
-
-def get_cachingallocator_config():
-    ca_config = os.environ.get("PYTORCH_CUDA_ALLOC_CONF", "")
-    return ca_config
-
-
-def get_cuda_module_loading_config():
-    if TORCH_AVAILABLE and torch.cuda.is_available():
-        torch.cuda.init()
-        config = os.environ.get("CUDA_MODULE_LOADING", "")
-        return config
-    else:
-        return "N/A"
-
-
-def is_xnnpack_available():
-    if TORCH_AVAILABLE:
-        import torch.backends.xnnpack
-
-        return str(torch.backends.xnnpack.enabled)  # type: ignore[attr-defined]
-    else:
-        return "N/A"
-
-
-def get_env_vars():
-    env_vars = ""
-    secret_terms = ("secret", "token", "api", "access", "password")
-    report_prefix = (
-        "TORCH",
-        "NCCL",
-        "PYTORCH",
-        "CUDA",
-        "CUBLAS",
-        "CUDNN",
-        "OMP_",
-        "MKL_",
-        "NVIDIA",
-    )
-    for k, v in os.environ.items():
-        if any(term in k.lower() for term in secret_terms):
-            continue
-        if k in environment_variables:
-            env_vars = env_vars + "{}={}".format(k, v) + "\n"
-        if k.startswith(report_prefix):
-            env_vars = env_vars + "{}={}".format(k, v) + "\n"
-
-    return env_vars
-
-
-def get_env_info():
-    run_lambda = run
-    pip_version, pip_list_output = get_pip_packages(run_lambda)
-
-    if TORCH_AVAILABLE:
-        version_str = torch.__version__
-        debug_mode_str = str(torch.version.debug)
-        cuda_available_str = str(torch.cuda.is_available())
-        cuda_version_str = torch.version.cuda
-        if not hasattr(torch.version, "hip") or torch.version.hip is None:  # cuda version
-            hip_compiled_version = hip_runtime_version = miopen_runtime_version = "N/A"
-        else:  # HIP version
-
-            def get_version_or_na(cfg, prefix):
-                _lst = [s.rsplit(None, 1)[-1] for s in cfg if prefix in s]
-                return _lst[0] if _lst else "N/A"
-
-            cfg = torch._C._show_config().split("\n")
-            hip_runtime_version = get_version_or_na(cfg, "HIP Runtime")
-            miopen_runtime_version = get_version_or_na(cfg, "MIOpen")
-            cuda_version_str = "N/A"
-            hip_compiled_version = torch.version.hip
-    else:
-        version_str = debug_mode_str = cuda_available_str = cuda_version_str = "N/A"
-        hip_compiled_version = hip_runtime_version = miopen_runtime_version = "N/A"
-
-    sys_version = sys.version.replace("\n", " ")
-
-    conda_packages = get_conda_packages(run_lambda)
-
-    rocm_version = get_rocm_version(run_lambda)
-    vllm_version = get_vllm_version()
-    vllm_omni_version = get_vllm_omni_version(run_lambda)
-    vllm_build_flags = summarize_vllm_build_flags()
-    gpu_topo = get_gpu_topo(run_lambda)
-
-    return SystemEnv(
-        torch_version=version_str,
-        is_debug_build=debug_mode_str,
-        python_version="{} ({}-bit runtime)".format(sys_version, sys.maxsize.bit_length() + 1),
-        python_platform=get_python_platform(),
-        is_cuda_available=cuda_available_str,
-        cuda_compiled_version=cuda_version_str,
-        cuda_runtime_version=get_running_cuda_version(run_lambda),
-        cuda_module_loading=get_cuda_module_loading_config(),
-        nvidia_gpu_models=get_gpu_info(run_lambda),
-        nvidia_driver_version=get_nvidia_driver_version(run_lambda),
-        cudnn_version=get_cudnn_version(run_lambda),
-        hip_compiled_version=hip_compiled_version,
-        hip_runtime_version=hip_runtime_version,
-        miopen_runtime_version=miopen_runtime_version,
-        pip_version=pip_version,
-        pip_packages=pip_list_output,
-        conda_packages=conda_packages,
-        os=get_os(run_lambda),
-        libc_version=get_libc_version(),
-        gcc_version=get_gcc_version(run_lambda),
-        clang_version=get_clang_version(run_lambda),
-        cmake_version=get_cmake_version(run_lambda),
-        caching_allocator_config=get_cachingallocator_config(),
-        is_xnnpack_available=is_xnnpack_available(),
-        cpu_info=get_cpu_info(run_lambda),
-        rocm_version=rocm_version,
-        vllm_version=vllm_version,
-        vllm_omni_version=vllm_omni_version,
-        vllm_build_flags=vllm_build_flags,
-        gpu_topo=gpu_topo,
-        env_vars=get_env_vars(),
-    )
-
-
-env_info_fmt = """
-==============================
-        System Info
-==============================
-OS                           : {os}
-GCC version                  : {gcc_version}
-Clang version                : {clang_version}
-CMake version                : {cmake_version}
-Libc version                 : {libc_version}
-
-==============================
-       PyTorch Info
-==============================
-PyTorch version              : {torch_version}
-Is debug build               : {is_debug_build}
-CUDA used to build PyTorch   : {cuda_compiled_version}
-ROCM used to build PyTorch   : {hip_compiled_version}
-
-==============================
-      Python Environment
-==============================
-Python version               : {python_version}
-Python platform              : {python_platform}
-
-==============================
-       CUDA / GPU Info
-==============================
-Is CUDA available            : {is_cuda_available}
-CUDA runtime version         : {cuda_runtime_version}
-CUDA_MODULE_LOADING set to   : {cuda_module_loading}
-GPU models and configuration : {nvidia_gpu_models}
-Nvidia driver version        : {nvidia_driver_version}
-cuDNN version                : {cudnn_version}
-HIP runtime version          : {hip_runtime_version}
-MIOpen runtime version       : {miopen_runtime_version}
-Is XNNPACK available         : {is_xnnpack_available}
-
-==============================
-          CPU Info
-==============================
-{cpu_info}
-
-==============================
-Versions of relevant libraries
-==============================
-{pip_packages}
-{conda_packages}
-""".strip()
-
-# both the above code and the following code use `strip()` to
-# remove leading/trailing whitespaces, so we need to add a newline
-# in between to separate the two sections
-env_info_fmt += "\n\n"
-
-env_info_fmt += """
-==============================
-         vLLM Info
-==============================
-ROCM Version                 : {rocm_version}
-vLLM Version                 : {vllm_version}
-vLLM-Omni Version            : {vllm_omni_version}
-vLLM Build Flags:
-  {vllm_build_flags}
-GPU Topology:
-  {gpu_topo}
-
-==============================
-     Environment Variables
-==============================
-{env_vars}
-""".strip()
-
-
-def pretty_str(envinfo):
-    def replace_nones(dct, replacement="Could not collect"):
-        for key in dct.keys():
-            if dct[key] is not None:
-                continue
-            dct[key] = replacement
-        return dct
-
-    def replace_bools(dct, true="Yes", false="No"):
-        for key in dct.keys():
-            if dct[key] is True:
-                dct[key] = true
-            elif dct[key] is False:
-                dct[key] = false
-        return dct
-
-    def prepend(text, tag="[prepend]"):
-        lines = text.split("\n")
-        updated_lines = [tag + line for line in lines]
-        return "\n".join(updated_lines)
-
-    def replace_if_empty(text, replacement="No relevant packages"):
-        if text is not None and len(text) == 0:
-            return replacement
-        return text
-
-    def maybe_start_on_next_line(string):
-        # If `string` is multiline, prepend a \n to it.
-        if string is not None and len(string.split("\n")) > 1:
-            return "\n{}\n".format(string)
-        return string
-
-    mutable_dict = envinfo._asdict()
-
-    # If nvidia_gpu_models is multiline, start on the next line
-    mutable_dict["nvidia_gpu_models"] = maybe_start_on_next_line(envinfo.nvidia_gpu_models)
-
-    # If the machine doesn't have CUDA, report some fields as 'No CUDA'
-    dynamic_cuda_fields = [
-        "cuda_runtime_version",
-        "nvidia_gpu_models",
-        "nvidia_driver_version",
-    ]
-    all_cuda_fields = dynamic_cuda_fields + ["cudnn_version"]
-    all_dynamic_cuda_fields_missing = all(mutable_dict[field] is None for field in dynamic_cuda_fields)
-    if TORCH_AVAILABLE and not torch.cuda.is_available() and all_dynamic_cuda_fields_missing:
-        for field in all_cuda_fields:
-            mutable_dict[field] = "No CUDA"
-        if envinfo.cuda_compiled_version is None:
-            mutable_dict["cuda_compiled_version"] = "None"
-
-    # Replace True with Yes, False with No
-    mutable_dict = replace_bools(mutable_dict)
-
-    # Replace all None objects with 'Could not collect'
-    mutable_dict = replace_nones(mutable_dict)
-
-    # If either of these are '', replace with 'No relevant packages'
-    mutable_dict["pip_packages"] = replace_if_empty(mutable_dict["pip_packages"])
-    mutable_dict["conda_packages"] = replace_if_empty(mutable_dict["conda_packages"])
-
-    # Tag conda and pip packages with a prefix
-    # If they were previously None, they'll show up as ie '[conda] Could not collect'
-    if mutable_dict["pip_packages"]:
-        mutable_dict["pip_packages"] = prepend(mutable_dict["pip_packages"], "[{}] ".format(envinfo.pip_version))
-    if mutable_dict["conda_packages"]:
-        mutable_dict["conda_packages"] = prepend(mutable_dict["conda_packages"], "[conda] ")
-    mutable_dict["cpu_info"] = envinfo.cpu_info
-    return env_info_fmt.format(**mutable_dict)
-
-
-def get_pretty_env_info():
-    return pretty_str(get_env_info())
-
-
-def main():
-    print("Collecting environment information...")
-    output = get_pretty_env_info()
-    print(output)
-
-    if TORCH_AVAILABLE and hasattr(torch, "utils") and hasattr(torch.utils, "_crash_handler"):
-        minidump_dir = torch.utils._crash_handler.DEFAULT_MINIDUMP_DIR
-        if sys.platform == "linux" and os.path.exists(minidump_dir):
-            dumps = [os.path.join(minidump_dir, dump) for dump in os.listdir(minidump_dir)]
-            latest = max(dumps, key=os.path.getctime)
-            ctime = os.path.getctime(latest)
-            creation_time = datetime.datetime.fromtimestamp(ctime).strftime("%Y-%m-%d %H:%M:%S")
-            msg = (
-                "\n*** Detected a minidump at {} created on {}, ".format(latest, creation_time)
-                + "if this is related to your bug please include it when you file a report ***"
-            )
-            print(msg, file=sys.stderr)
-
-
-if __name__ == "__main__":
-    main()

docker/Dockerfile.ci

@@ -1,20 +1,13 @@
 ARG VLLM_BASE_IMAGE=vllm/vllm-openai
-ARG VLLM_BASE_TAG=v0.16.0
+ARG VLLM_BASE_TAG=v0.11.0
 FROM ${VLLM_BASE_IMAGE}:${VLLM_BASE_TAG}
 ARG APP_DIR=/workspace/vllm-omni
 WORKDIR ${APP_DIR}
 
 COPY . .
 
-# Install system dependencies
-RUN apt-get update && \
-    apt-get install -y ffmpeg git sox libsox-fmt-all jq && \
-    apt-get clean && \
-    rm -rf /var/lib/apt/lists/*
-
 # Install vllm-omni into the same uv-managed Python environment used by the base image.
-# Use bash -c so that $(python3 -c ...) is expanded inside the container.
-RUN uv pip install --system --no-cache-dir ".[dev]"
+RUN uv pip install --python "$(python3 -c 'import sys; print(sys.executable)')" --no-cache-dir ".[dev]"
 
 RUN ln -sf /usr/bin/python3 /usr/bin/python
 

docker/Dockerfile.npu

@@ -1,18 +0,0 @@
-ARG VLLM_ASCEND_IMAGE=quay.io/ascend/vllm-ascend
-ARG VLLM_ASCEND_TAG=v0.14.0rc1
-FROM ${VLLM_ASCEND_IMAGE}:${VLLM_ASCEND_TAG}
-
-WORKDIR /vllm-workspace/vllm-ascend
-RUN git checkout e2175d9c7e62b437391dfee996b1375674ba7c18
-RUN pip install -v -e .
-
-ARG APP_DIR=/vllm-workspace/vllm-omni
-WORKDIR ${APP_DIR}
-
-COPY . .
-
-RUN pip install -v -e .
-
-ENV VLLM_WORKER_MULTIPROC_METHOD=spawn
-
-ENTRYPOINT []

docker/Dockerfile.npu.a3

@@ -1,18 +0,0 @@
-ARG VLLM_ASCEND_IMAGE=quay.io/ascend/vllm-ascend
-ARG VLLM_ASCEND_TAG=v0.14.0rc1-a3
-FROM ${VLLM_ASCEND_IMAGE}:${VLLM_ASCEND_TAG}
-
-WORKDIR /vllm-workspace/vllm-ascend
-RUN git checkout e2175d9c7e62b437391dfee996b1375674ba7c18
-RUN pip install -v -e .
-
-ARG APP_DIR=/vllm-workspace/vllm-omni
-WORKDIR ${APP_DIR}
-
-COPY . .
-
-RUN pip install -v -e .
-
-ENV VLLM_WORKER_MULTIPROC_METHOD=spawn
-
-ENTRYPOINT []

docker/Dockerfile.rocm

@@ -1,53 +0,0 @@
-ARG BASE_IMAGE=vllm/vllm-openai-rocm:v0.16.0
-FROM ${BASE_IMAGE} AS base
-
-# Declare a variable to know if we want to use the nightly build or the stable build.
-# NOTE: REMINDER to vLLM-Omni rebase maintainer
-# Remember to set `USE_NIGHTLY_BUILD` to 0, when switching back to stable vLLM docker image
-# 1. If vLLM-omni maintainer is forced to use custom commits
-#    during rebasing, they can change the two variables
-# 2. Whenever vLLM upstream has released a stable version,
-#    we should swap over to use stable release ASAP.
-#    We should avoid relying on custom commits.
-ARG USE_NIGHTLY_BUILD=0
-ARG VLLM_VERSION_OR_COMMIT_HASH=2d5be1dd5ce2e44dfea53ea03ff61143da5137eb
-ARG ARG_PYTORCH_ROCM_ARCH
-ENV PYTORCH_ROCM_ARCH=${ARG_PYTORCH_ROCM_ARCH:-${PYTORCH_ROCM_ARCH}}
-
-ARG COMMON_WORKDIR=/app
-WORKDIR ${COMMON_WORKDIR}
-
-# Step 1: Setup - Install system dependencies
-RUN apt-get update && \
-    apt-get install -y ffmpeg git sox libsox-fmt-all jq && \
-    apt-get clean && \
-    rm -rf /var/lib/apt/lists/*
-
-# Step 2: Conditionally reinstall vllm from source for nightly builds
-RUN if [ "${USE_NIGHTLY_BUILD}" = "1" ]; then \
-        python3 -m pip uninstall -y vllm && rm -rf vllm && \
-        git clone https://github.com/vllm-project/vllm.git && \
-        cd vllm && \
-        git checkout ${VLLM_VERSION_OR_COMMIT_HASH} && \
-        python3 -m pip install -r requirements/rocm.txt && \
-        python3 setup.py clean --all && \
-        PYTORCH_ROCM_ARCH=${PYTORCH_ROCM_ARCH} python3 setup.py develop && \
-        cd ../ && \
-        rm -rf vllm/.git; \
-    fi
-
-# Step 3: Copy vllm-omni code and install without uv
-RUN mkdir -p ${COMMON_WORKDIR}/vllm-omni
-COPY . ${COMMON_WORKDIR}/vllm-omni
-RUN cd ${COMMON_WORKDIR}/vllm-omni && uv pip install --python "$(python3 -c 'import sys; print(sys.executable)')" --no-cache-dir ".[dev]" --no-build-isolation
-
-RUN ln -sf /usr/bin/python3 /usr/bin/python
-
-FROM base AS test
-
-CMD ["/bin/bash"]
-ENTRYPOINT []
-
-# Set entrypoint for vllm-openai official images
-FROM base AS vllm-openai
-ENTRYPOINT ["vllm", "serve", "--omni"]

docker/Dockerfile.xpu

@@ -1,139 +0,0 @@
-# Argument to configure vllm base image if pre-built
-ARG VLLM_BASE=vllm-base
-
-FROM intel/deep-learning-essentials:2025.3.2-0-devel-ubuntu24.04 AS vllm-base
-
-WORKDIR /workspace/
-
-ARG PYTHON_VERSION=3.12
-ARG PIP_EXTRA_INDEX_URL="https://download.pytorch.org/whl/xpu"
-
-RUN wget -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB | gpg --dearmor | tee /usr/share/keyrings/oneapi-archive-keyring.gpg > /dev/null && \
-    echo "deb [signed-by=/usr/share/keyrings/oneapi-archive-keyring.gpg] https://apt.repos.intel.com/oneapi all main" | tee /etc/apt/sources.list.d/oneAPI.list && \
-    add-apt-repository -y ppa:kobuk-team/intel-graphics
-
-RUN apt clean && apt-get update -y && \
-    apt-get install -y --no-install-recommends --fix-missing \
-    curl \
-    ffmpeg \
-    git \
-    libsndfile1 \
-    libsm6 \
-    libxext6 \
-    libgl1 \
-    lsb-release \
-    libaio-dev \
-    numactl \
-    wget \
-    vim \
-    python3.12 \
-    python3.12-dev \
-    python3-pip
-
-RUN apt update && apt upgrade -y && \
-    apt install -y libze1 libze-dev libze-intel-gpu1 intel-opencl-icd libze-intel-gpu-raytracing intel-ocloc && \
-    apt install -y intel-oneapi-compiler-dpcpp-cpp-2025.3
-
-ENV PATH="/root/.local/bin:$PATH"
-ENV VIRTUAL_ENV="/opt/venv"
-ENV UV_PYTHON_INSTALL_DIR=/opt/uv/python
-RUN curl -LsSf https://astral.sh/uv/install.sh | sh
-RUN uv venv --python ${PYTHON_VERSION} --seed ${VIRTUAL_ENV}
-ENV PATH="$VIRTUAL_ENV/bin:$PATH"
-
-# This oneccl contains the BMG support which is not the case for default version of oneapi 2025.2.
-ARG ONECCL_INSTALLER="intel-oneccl-2021.15.7.8_offline.sh"
-RUN wget "https://github.com/uxlfoundation/oneCCL/releases/download/2021.15.7/${ONECCL_INSTALLER}" && \
-    bash "${ONECCL_INSTALLER}" -a --silent --eula accept && \
-    rm "${ONECCL_INSTALLER}" && \
-    echo "source /opt/intel/oneapi/setvars.sh --force" >> /root/.bashrc && \
-    echo "source /opt/intel/oneapi/ccl/2021.15/env/vars.sh --force" >> /root/.bashrc
-RUN rm -f /opt/intel/oneapi/ccl/latest && \
-    ln -s /opt/intel/oneapi/ccl/2021.15 /opt/intel/oneapi/ccl/latest
-
-SHELL ["bash", "-c"]
-CMD ["bash", "-c", "source /root/.bashrc && exec bash"]
-
-WORKDIR /workspace/
-ENV UV_HTTP_TIMEOUT=500
-
-# Configure package index for XPU
-ENV PIP_EXTRA_INDEX_URL=${PIP_EXTRA_INDEX_URL}
-ENV UV_EXTRA_INDEX_URL=${PIP_EXTRA_INDEX_URL}
-ENV UV_INDEX_STRATEGY="unsafe-best-match"
-ENV UV_LINK_MODE="copy"
-
-ARG VLLM_VERSION=v0.16.0
-RUN git clone -b ${VLLM_VERSION} https://github.com/vllm-project/vllm
-WORKDIR /workspace/vllm
-
-RUN --mount=type=cache,target=/root/.cache/uv \
-    uv pip install --upgrade pip && \
-    uv pip install -r requirements/xpu.txt
-
- # used for suffix method speculative decoding
- # build deps for proto + nanobind-based extensions to set up the build environment
-RUN --mount=type=cache,target=/root/.cache/uv \
-    uv pip install grpcio-tools protobuf nanobind
- # arctic-inference is built from source which needs torch-xpu properly installed first
-RUN --mount=type=cache,target=/root/.cache/uv \
-    source /opt/intel/oneapi/setvars.sh --force && \
-    source /opt/intel/oneapi/ccl/2021.15/env/vars.sh --force && \
-    export CMAKE_PREFIX_PATH="$(python -c 'import site; print(site.getsitepackages()[0])'):${CMAKE_PREFIX_PATH}" && \
-    uv pip install --no-build-isolation arctic-inference==0.1.1
-
-ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/local/lib/"
-
-ENV VLLM_TARGET_DEVICE=xpu
-ENV VLLM_WORKER_MULTIPROC_METHOD=spawn
-
-RUN --mount=type=cache,target=/root/.cache/uv \
-    uv pip install --no-build-isolation .
-
-CMD ["/bin/bash"]
-
-FROM vllm-base AS vllm-openai
-
-# install additional dependencies for openai api server
-RUN --mount=type=cache,target=/root/.cache/uv \
-    uv pip install accelerate hf_transfer pytest pytest_asyncio lm_eval[api] modelscope
-
-# install development dependencies (for testing)
-RUN uv pip install -e tests/vllm_test_utils
-
-# install nixl from source code
-ENV NIXL_VERSION=0.7.0
-RUN python /workspace/vllm/tools/install_nixl_from_source_ubuntu.py
-
-# ensure vllm is properly installed
-RUN python -c "import vllm, inspect; print(vllm.__file__)"
-RUN uv pip show vllm
-
-CMD ["/bin/bash"]
-
-ENTRYPOINT []
-
-FROM ${VLLM_BASE} AS vllm-omni
-
-WORKDIR /workspace/vllm-omni
-COPY . .
-
-ENV VLLM_OMNI_TARGET_DEVICE=xpu
-RUN uv pip install --no-cache-dir ".[dev]" --no-build-isolation
-
-# FIX triton
-RUN --mount=type=cache,target=/root/.cache/uv \
-    uv pip uninstall triton triton-xpu && \
-    uv pip install triton-xpu==3.6.0
-
-# remove torch bundled oneccl to avoid conflicts
-RUN --mount=type=cache,target=/root/.cache/uv \
-    uv pip uninstall oneccl oneccl-devel
-
-FROM vllm-omni AS vllm-omni-openai
-
-RUN ln -sf /usr/bin/python3 /usr/bin/python
-
-ENV VLLM_WORKER_MULTIPROC_METHOD=spawn
-
-ENTRYPOINT ["vllm", "serve", "--omni"]

docs/.nav.yml

@@ -3,88 +3,46 @@ nav:
 - User Guide:
   - Getting Started:
     - getting_started/quickstart.md
-    - getting_started/installation/*
-  - Serving:
-    - OpenAI-Compatible API:
-      - Image Generation: serving/image_generation_api.md
-      - Image Edit: serving/image_edit_api.md
-      - Text to Speech: serving/speech_api.md
+    - getting_started/installation
   - Examples:
     - examples/README.md
     - Offline Inference:
-      - BAGEL-7B-MoT: user_guide/examples/offline_inference/bagel.md
-      - GLM-Image Multistage End-to-End Inference: user_guide/examples/offline_inference/glm_image.md
-      - Image-To-Image: user_guide/examples/offline_inference/image_to_image.md
-      - Image-To-Video: user_guide/examples/offline_inference/image_to_video.md
-      - MiMo-Audio Offline Inference: user_guide/examples/offline_inference/mimo_audio.md
       - Qwen2.5-Omni: user_guide/examples/offline_inference/qwen2_5_omni.md
       - Qwen3-Omni: user_guide/examples/offline_inference/qwen3_omni.md
-      - Qwen3-TTS: user_guide/examples/offline_inference/qwen3_tts.md
-      - Text-To-Audio: user_guide/examples/offline_inference/text_to_audio.md
       - Text-To-Image: user_guide/examples/offline_inference/text_to_image.md
+      - Image-To-Image: user_guide/examples/offline_inference/image_to_image.md
       - Text-To-Video: user_guide/examples/offline_inference/text_to_video.md
     - Online Serving:
-      - BAGEL-7B-MoT: user_guide/examples/online_serving/bagel.md
-      - GLM-Image Online Serving: user_guide/examples/online_serving/glm_image.md
-      - Image-To-Image: user_guide/examples/online_serving/image_to_image.md
-      - Image-To-Video: user_guide/examples/online_serving/image_to_video.md
-      - Online serving Example of vLLM-Omni for MiMo-Audio: user_guide/examples/online_serving/mimo_audio.md
       - Qwen2.5-Omni: user_guide/examples/online_serving/qwen2_5_omni.md
       - Qwen3-Omni: user_guide/examples/online_serving/qwen3_omni.md
-      - Qwen3-TTS: user_guide/examples/online_serving/qwen3_tts.md
-      - Text-To-Image: user_guide/examples/online_serving/text_to_image.md
-      - Text-To-Video: user_guide/examples/online_serving/text_to_video.md
   - General:
     - usage/*
   - Configuration:
     - configuration/README.md
     - configuration/*
+  - Diffusion Acceleration:
+     - TeaCache: user_guide/teacache.md
   - Models:
     - models/supported_models.md
-  - Features:
-    - Sleep Mode: features/sleep_mode.md
-    - Diffusion Features:
-      - Acceleration Overview: user_guide/diffusion_acceleration.md
-      - TeaCache: user_guide/diffusion/teacache.md
-      - Cache-DiT: user_guide/diffusion/cache_dit_acceleration.md
-      - Quantization:
-        - Overview: user_guide/diffusion/quantization/overview.md
-        - FP8: user_guide/diffusion/quantization/fp8.md
-        - GGUF: user_guide/diffusion/quantization/gguf.md
-      - Parallelism Acceleration: user_guide/diffusion/parallelism_acceleration.md
-      - CPU Offloading: user_guide/diffusion/cpu_offload_diffusion.md
-      - LoRA: user_guide/diffusion/lora.md
-      - Hybrid Sharded Data Parallel: design/feature/hsdp.md
-      - Custom Pipeline: features/custom_pipeline.md
-    - ComfyUI: features/comfyui.md
 - Developer Guide:
   - General:
     - contributing/README.md
-    - pr_reviewer.md
     - glob: contributing/*
       flatten_single_child_sections: true
   - Model Implementation:
     - contributing/model/README.md
-    - contributing/model/adding_omni_model.md
-    - contributing/model/adding_diffusion_model.md
   - CI: contributing/ci
   - Design Documents:
     - design/index.md
     - design/architecture_overview.md
     - Feature Design:
-      - design/feature/disaggregated_inference.md
+      - design/feature/multi_request_streaming_design.md
+      - design/feature/omni_connector_design.md
       - design/feature/ray_based_execution.md
-      - design/feature/omni_connectors/
-      - design/feature/cfg_parallel.md
-      - design/feature/sequence_parallel.md
-      - design/feature/tensor_parallel.md
-      - design/feature/cache_dit.md
-      - design/feature/teacache.md
-      - design/feature/async_chunk_design.md
     - Module Design:
-      - design/module/ar_module.md
-      - design/module/dit_module.md
-      - design/module/entrypoint_module.md
+      - design/module/ar_module_design.md
+      - design/module/dit_module_design.md
+      - design/module/entrypoint_module_design.md
   - Docs Guide: contributing/DOCS_GUIDE.md
 - API Reference:
   - api/README.md

docs/README.md

@@ -60,5 +60,5 @@ vLLM-Omni seamlessly supports most popular open-source models on HuggingFace, in
 
 For more information, checkout the following:
 
-- [vllm-omni architecture design and recent roadmaps](https://docs.google.com/presentation/d/1qv4qMW1rKAqDREMXiUDLIgqqHQe7TDPj/edit?usp=sharing&ouid=110473603432222024453&rtpof=true&sd=true)
+- [vllm-omni architecture design and recent roadmaps](https://docs.google.com/presentation/d/1Y7t2Zm3BIISPN-_X_sDpxZMNCWir2rAx/edit?usp=drive_link&ouid=110473603432222024453&rtpof=true&sd=true)
 - [vllm-omni announcement blogpost](https://blog.vllm.ai/2025/11/30/vllm-omni.html)

docs/api/README.md

@@ -5,30 +5,26 @@
 Main entry points for vLLM-Omni inference and serving.
 
 - [vllm_omni.entrypoints.async_omni.AsyncOmni][]
-- [vllm_omni.entrypoints.async_omni_diffusion.AsyncOmniDiffusion][]
-- [vllm_omni.entrypoints.async_omni_llm.AsyncOmniLLM][]
-- [vllm_omni.entrypoints.cli.benchmark.base.OmniBenchmarkSubcommandBase][]
-- [vllm_omni.entrypoints.cli.benchmark.main.OmniBenchmarkSubcommand][]
-- [vllm_omni.entrypoints.cli.benchmark.serve.OmniBenchmarkServingSubcommand][]
+- [vllm_omni.entrypoints.async_omni.AsyncOmniStageLLM][]
+- [vllm_omni.entrypoints.chat_utils.OmniAsyncMultiModalContentParser][]
+- [vllm_omni.entrypoints.chat_utils.OmniAsyncMultiModalItemTracker][]
+- [vllm_omni.entrypoints.chat_utils.parse_chat_messages_futures][]
 - [vllm_omni.entrypoints.cli.serve.OmniServeCommand][]
-- [vllm_omni.entrypoints.client_request_state.ClientRequestState][]
+- [vllm_omni.entrypoints.log_utils.OrchestratorMetrics][]
 - [vllm_omni.entrypoints.omni.Omni][]
-- [vllm_omni.entrypoints.omni.OmniBase][]
-- [vllm_omni.entrypoints.omni_diffusion.OmniDiffusion][]
 - [vllm_omni.entrypoints.omni_llm.OmniLLM][]
+- [vllm_omni.entrypoints.omni_llm.OmniStageLLM][]
 - [vllm_omni.entrypoints.omni_stage.OmniStage][]
-- [vllm_omni.entrypoints.stage_utils.OmniStageTaskType][]
-- [vllm_omni.entrypoints.zmq_utils.ZmqQueue][]
+- [vllm_omni.entrypoints.openai.serving_chat.OmniOpenAIServingChat][]
 
 ## Inputs
 
 Input data structures for multi-modal inputs.
 
-- [vllm_omni.inputs.data.OmniDiffusionSamplingParams][]
 - [vllm_omni.inputs.data.OmniEmbedsPrompt][]
-- [vllm_omni.inputs.data.OmniTextPrompt][]
 - [vllm_omni.inputs.data.OmniTokenInputs][]
 - [vllm_omni.inputs.data.OmniTokensPrompt][]
+- [vllm_omni.inputs.parse.parse_singleton_prompt_omni][]
 - [vllm_omni.inputs.preprocess.OmniInputPreprocessor][]
 
 ## Engine
@@ -36,12 +32,6 @@ Input data structures for multi-modal inputs.
 Engine classes for offline and online inference.
 
 - [vllm_omni.diffusion.diffusion_engine.DiffusionEngine][]
-- [vllm_omni.distributed.omni_connectors.connectors.mooncake_transfer_engine_connector.BufferAllocator][]
-- [vllm_omni.distributed.omni_connectors.connectors.mooncake_transfer_engine_connector.ManagedBuffer][]
-- [vllm_omni.distributed.omni_connectors.connectors.mooncake_transfer_engine_connector.MooncakeAgentMetadata][]
-- [vllm_omni.distributed.omni_connectors.connectors.mooncake_transfer_engine_connector.MooncakeTransferEngineConnector][]
-- [vllm_omni.distributed.omni_connectors.connectors.mooncake_transfer_engine_connector.QueryRequest][]
-- [vllm_omni.distributed.omni_connectors.connectors.mooncake_transfer_engine_connector.QueryResponse][]
 - [vllm_omni.engine.AdditionalInformationEntry][]
 - [vllm_omni.engine.AdditionalInformationPayload][]
 - [vllm_omni.engine.OmniEngineCoreOutput][]
@@ -50,25 +40,56 @@ Engine classes for offline and online inference.
 - [vllm_omni.engine.PromptEmbedsPayload][]
 - [vllm_omni.engine.arg_utils.AsyncOmniEngineArgs][]
 - [vllm_omni.engine.arg_utils.OmniEngineArgs][]
-- [vllm_omni.engine.input_processor.OmniInputProcessor][]
 - [vllm_omni.engine.output_processor.MultimodalOutputProcessor][]
 - [vllm_omni.engine.output_processor.OmniRequestState][]
+- [vllm_omni.engine.processor.OmniProcessor][]
 
 ## Core
 
 Core scheduling and caching components.
 
-- [vllm_omni.core.sched.omni_ar_scheduler.KVCacheTransferData][]
+- [vllm_omni.core.dit_cache_manager.DiTCacheManager][]
 - [vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler][]
 - [vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler][]
-- [vllm_omni.core.sched.output.OmniCachedRequestData][]
 - [vllm_omni.core.sched.output.OmniNewRequestData][]
-- [vllm_omni.core.sched.output.OmniSchedulerOutput][]
-- [vllm_omni.model_executor.models.qwen3_tts.tokenizer_25hz.vq.core_vq.DistributedGroupResidualVectorQuantization][]
-- [vllm_omni.model_executor.models.qwen3_tts.tokenizer_25hz.vq.core_vq.DistributedResidualVectorQuantization][]
-- [vllm_omni.model_executor.models.qwen3_tts.tokenizer_25hz.vq.core_vq.EuclideanCodebook][]
-- [vllm_omni.model_executor.models.qwen3_tts.tokenizer_25hz.vq.core_vq.VectorQuantization][]
-- [vllm_omni.model_executor.models.qwen3_tts.tokenizer_25hz.vq.core_vq.preprocess][]
+
+## Model Executor
+
+Model execution components.
+
+- [vllm_omni.model_executor.models.output_templates.OmniOutput][]
+- [vllm_omni.model_executor.models.qwen2_5_omni.qwen2_5_omni.Qwen2_5OmniForConditionalGeneration][]
+- [vllm_omni.model_executor.models.qwen2_5_omni.qwen2_5_omni_talker.Qwen2_5OmniTalkerForConditionalGeneration][]
+- [vllm_omni.model_executor.models.qwen2_5_omni.qwen2_5_omni_thinker.Qwen2_5OmniAudioFeatureInputs][]
+- [vllm_omni.model_executor.models.qwen2_5_omni.qwen2_5_omni_thinker.Qwen2_5OmniConditionalGenerationMixin][]
+- [vllm_omni.model_executor.models.qwen2_5_omni.qwen2_5_omni_thinker.Qwen2_5OmniThinkerDummyInputsBuilder][]
+- [vllm_omni.model_executor.models.qwen2_5_omni.qwen2_5_omni_thinker.Qwen2_5OmniThinkerForConditionalGeneration][]
+- [vllm_omni.model_executor.models.qwen2_5_omni.qwen2_5_omni_thinker.Qwen2_5OmniThinkerMultiModalDataParser][]
+- [vllm_omni.model_executor.models.qwen2_5_omni.qwen2_5_omni_thinker.Qwen2_5OmniThinkerMultiModalProcessor][]
+- [vllm_omni.model_executor.models.qwen2_5_omni.qwen2_5_omni_thinker.Qwen2_5OmniThinkerProcessingInfo][]
+- [vllm_omni.model_executor.models.qwen2_5_omni.qwen2_5_omni_token2wav.Qwen2_5OmniToken2WavBigVGANModel][]
+- [vllm_omni.model_executor.models.qwen2_5_omni.qwen2_5_omni_token2wav.Qwen2_5OmniToken2WavDiTModel][]
+- [vllm_omni.model_executor.models.qwen2_5_omni.qwen2_5_omni_token2wav.Qwen2_5OmniToken2WavForConditionalGenerationVLLM][]
+- [vllm_omni.model_executor.models.qwen2_5_omni.qwen2_5_omni_token2wav.Qwen2_5OmniToken2WavModel][]
+- [vllm_omni.model_executor.models.qwen2_5_omni.qwen2_old.Qwen2EmbeddingModel][]
+- [vllm_omni.model_executor.models.qwen2_5_omni.qwen2_old.Qwen2ForCausalLM][]
+- [vllm_omni.model_executor.models.qwen2_5_omni.qwen2_old.Qwen2Model][]
+- [vllm_omni.model_executor.models.qwen3_omni.qwen3_moe.Qwen3MoeForCausalLM][]
+- [vllm_omni.model_executor.models.qwen3_omni.qwen3_omni.Qwen3OmniMoeForConditionalGeneration][]
+- [vllm_omni.model_executor.models.qwen3_omni.qwen3_omni_code2wav.Qwen3OmniMoeCode2Wav][]
+- [vllm_omni.model_executor.models.qwen3_omni.qwen3_omni_moe_code_predictor_mtp.Qwen3OmniCodePredictorBaseModel][]
+- [vllm_omni.model_executor.models.qwen3_omni.qwen3_omni_moe_code_predictor_mtp.Qwen3OmniMoeTalkerCodePredictor][]
+- [vllm_omni.model_executor.models.qwen3_omni.qwen3_omni_moe_talker.Qwen3OmniMoeModel][]
+- [vllm_omni.model_executor.models.qwen3_omni.qwen3_omni_moe_talker.Qwen3OmniMoeTalkerForConditionalGeneration][]
+- [vllm_omni.model_executor.models.qwen3_omni.qwen3_omni_moe_thinker.Qwen3MoeLLMForCausalLM][]
+- [vllm_omni.model_executor.models.qwen3_omni.qwen3_omni_moe_thinker.Qwen3MoeLLMModel][]
+- [vllm_omni.model_executor.models.qwen3_omni.qwen3_omni_moe_thinker.Qwen3OmniMoeConditionalGenerationMixin][]
+- [vllm_omni.model_executor.models.qwen3_omni.qwen3_omni_moe_thinker.Qwen3OmniMoeThinkerForConditionalGeneration][]
+- [vllm_omni.model_executor.models.qwen3_omni.qwen3_omni_moe_thinker.Qwen3OmniMoeThinkerMultiModalProcessor][]
+- [vllm_omni.model_executor.models.qwen3_omni.qwen3_omni_moe_thinker.Qwen3OmniMoeThinkerProcessingInfo][]
+- [vllm_omni.model_executor.models.qwen3_omni.qwen3_omni_moe_thinker.Qwen3Omni_VisionTransformer][]
+- [vllm_omni.model_executor.models.qwen3_omni.qwen3_omni_moe_thinker.Qwen3_VisionPatchEmbed][]
+- [vllm_omni.model_executor.models.qwen3_omni.qwen3_omni_moe_thinker.Qwen3_VisionPatchMerger][]
 
 ## Configuration
 
@@ -76,47 +97,22 @@ Configuration classes.
 
 - [vllm_omni.config.model.OmniModelConfig][]
 - [vllm_omni.diffusion.cache.teacache.config.TeaCacheConfig][]
-- [vllm_omni.distributed.omni_connectors.utils.config.ConnectorSpec][]
-- [vllm_omni.distributed.omni_connectors.utils.config.OmniTransferConfig][]
-- [vllm_omni.model_executor.models.mimo_audio.config_mimo_audio.MiMoAudioConfig][]
-- [vllm_omni.model_executor.models.mimo_audio.config_mimo_audio.MiMoAudioTokenizerConfig][]
-- [vllm_omni.model_executor.models.qwen3_tts.configuration_qwen3_tts.Qwen3TTSConfig][]
-- [vllm_omni.model_executor.models.qwen3_tts.configuration_qwen3_tts.Qwen3TTSSpeakerEncoderConfig][]
-- [vllm_omni.model_executor.models.qwen3_tts.configuration_qwen3_tts.Qwen3TTSTalkerCodePredictorConfig][]
-- [vllm_omni.model_executor.models.qwen3_tts.configuration_qwen3_tts.Qwen3TTSTalkerConfig][]
-- [vllm_omni.model_executor.models.qwen3_tts.tokenizer_12hz.configuration_qwen3_tts_tokenizer_v2.Qwen3TTSTokenizerV2Config][]
-- [vllm_omni.model_executor.models.qwen3_tts.tokenizer_12hz.configuration_qwen3_tts_tokenizer_v2.Qwen3TTSTokenizerV2DecoderConfig][]
-- [vllm_omni.model_executor.models.qwen3_tts.tokenizer_25hz.configuration_qwen3_tts_tokenizer_v1.Qwen3TTSTokenizerV1Config][]
-- [vllm_omni.model_executor.models.qwen3_tts.tokenizer_25hz.configuration_qwen3_tts_tokenizer_v1.Qwen3TTSTokenizerV1DecoderBigVGANConfig][]
-- [vllm_omni.model_executor.models.qwen3_tts.tokenizer_25hz.configuration_qwen3_tts_tokenizer_v1.Qwen3TTSTokenizerV1DecoderConfig][]
-- [vllm_omni.model_executor.models.qwen3_tts.tokenizer_25hz.configuration_qwen3_tts_tokenizer_v1.Qwen3TTSTokenizerV1DecoderDiTConfig][]
-- [vllm_omni.model_executor.models.qwen3_tts.tokenizer_25hz.configuration_qwen3_tts_tokenizer_v1.Qwen3TTSTokenizerV1EncoderConfig][]
 
 ## Workers
 
 Worker classes and model runners for distributed inference.
 
-- [vllm_omni.diffusion.worker.diffusion_model_runner.DiffusionModelRunner][]
-- [vllm_omni.diffusion.worker.diffusion_worker.CustomPipelineWorkerExtension][]
-- [vllm_omni.diffusion.worker.diffusion_worker.DiffusionWorker][]
-- [vllm_omni.diffusion.worker.diffusion_worker.WorkerProc][]
-- [vllm_omni.diffusion.worker.diffusion_worker.WorkerWrapperBase][]
-- [vllm_omni.platforms.npu.worker.npu_ar_model_runner.ExecuteModelState][]
-- [vllm_omni.platforms.npu.worker.npu_ar_model_runner.NPUARModelRunner][]
-- [vllm_omni.platforms.npu.worker.npu_ar_worker.NPUARWorker][]
-- [vllm_omni.platforms.npu.worker.npu_generation_model_runner.NPUGenerationModelRunner][]
-- [vllm_omni.platforms.npu.worker.npu_generation_worker.NPUGenerationWorker][]
-- [vllm_omni.platforms.npu.worker.npu_model_runner.OmniNPUModelRunner][]
-- [vllm_omni.platforms.xpu.worker.xpu_ar_model_runner.XPUARModelRunner][]
-- [vllm_omni.platforms.xpu.worker.xpu_ar_worker.XPUARWorker][]
-- [vllm_omni.platforms.xpu.worker.xpu_generation_model_runner.XPUGenerationModelRunner][]
-- [vllm_omni.platforms.xpu.worker.xpu_generation_worker.XPUGenerationWorker][]
-- [vllm_omni.worker.base.OmniGPUWorkerBase][]
-- [vllm_omni.worker.gpu_ar_model_runner.ExecuteModelState][]
+- [vllm_omni.diffusion.worker.gpu_worker.GPUWorker][]
+- [vllm_omni.diffusion.worker.gpu_worker.WorkerProc][]
+- [vllm_omni.diffusion.worker.npu.npu_worker.NPUWorker][]
+- [vllm_omni.diffusion.worker.npu.npu_worker.NPUWorkerProc][]
 - [vllm_omni.worker.gpu_ar_model_runner.GPUARModelRunner][]
 - [vllm_omni.worker.gpu_ar_worker.GPUARWorker][]
 - [vllm_omni.worker.gpu_generation_model_runner.GPUGenerationModelRunner][]
 - [vllm_omni.worker.gpu_generation_worker.GPUGenerationWorker][]
-- [vllm_omni.worker.gpu_memory_utils.parse_cuda_visible_devices][]
 - [vllm_omni.worker.gpu_model_runner.OmniGPUModelRunner][]
-- [vllm_omni.worker.mixins.OmniWorkerMixin][]
+- [vllm_omni.worker.npu.npu_ar_model_runner.NPUARModelRunner][]
+- [vllm_omni.worker.npu.npu_ar_worker.NPUARWorker][]
+- [vllm_omni.worker.npu.npu_generation_model_runner.NPUGenerationModelRunner][]
+- [vllm_omni.worker.npu.npu_generation_worker.NPUGenerationWorker][]
+- [vllm_omni.worker.npu.npu_model_runner.OmniNPUModelRunner][]

docs/cli/README.md

@@ -22,21 +22,3 @@ If you have custom stage configs file, launch the server with command below
 ```bash
 vllm serve Qwen/Qwen2.5-Omni-7B --omni --stage-configs-path /path/to/stage_configs_file
 ```
-
-
-## bench
-
-Run benchmark tests for online serving throughput.
-Available Commands:
-
-```bash
-vllm bench serve --omni \
-    --model Qwen/Qwen2.5-Omni-7B \
-    --host server-host \
-    --port server-port \
-    --random-input-len 32 \
-    --random-output-len 4  \
-    --num-prompts  5
-```
-
-See [vllm bench serve](./bench/serve.md) for the full reference of all available arguments.

docs/cli/bench/serve.md

@@ -1,359 +0,0 @@
-# vLLM-Omni Benchmark CLI Guide
-The vllm bench command launches the vLLM-Omni benchmark to evaluate the performance of multimodal models.
-
-## Notes
-We currently only support using the "openai-chat-omni" backend.
-
-## Basic Parameter Description
-You can use `vllm bench serve --omni --help=all` to get descriptions of all parameters. The commonly used parameters are described below:
-- `--omni`  
-  Enable Omni (multimodal) mode, supporting multimodal inputs and outputs such as images, videos, and audio.
-
-- `--backend`  
-  Specify the backend adapter as openai-chat-omni, using OpenAI Chat compatible API behavior as the protocol. Currently only openai-chat-omni is supported.
-
-- `--model`  
-  The model identifier to load, filled according to the models supported by vLLM-Omni.
-
-- `--endpoint`  
-  The API endpoint exposed externally, to which clients send their requests.
-
-- `--dataset-name`  
-  The name of the dataset used; random-mm indicates generating random multimodal inputs (images, videos, audio).
-
-- `--num-prompts`  
-  The total number of requests to send, an integer.
-
-- `--max-concurrency`  
-  "Maximum number of concurrent requests. This can be used "
-        "to help simulate an environment where a higher level component "
-        "is enforcing a maximum number of concurrent requests. While the "
-        "--request-rate argument controls the rate at which requests are "
-        "initiated, this argument will control how many are actually allowed "
-        "to execute at a time. This means that when used in combination, the "
-        "actual request rate may be lower than specified with --request-rate, "
-        "if the server is not processing requests fast enough to keep up."
-
-- `--request-rate`  
-  "Number of requests per second. If this is inf, "
-        "then all the requests are sent at time 0. "
-        "Otherwise, we use Poisson process or gamma distribution "
-        "to synthesize the request arrival times."
-
-- `--ignore-eos`  
-  "Set ignore_eos flag when sending the benchmark request."
-
-- `--metric-percentiles`  
-  Comma-separated list of percentiles for selected metrics. "
-        "To report 25-th, 50-th, and 75-th percentiles, use \"25,50,75\". "
-        "Default value is \"99\"."
-        "Use \"--percentile-metrics\" to select metrics.
-
-- `--percentile-metrics`  
-        "Comma-separated list of selected metrics to report percentiles."
-                    "This argument specifies the metrics to report percentiles."
-                    'Allowed metric names are "ttft", "tpot", "itl", "e2el", "audio_ttfp", "audio_rtf", "audio_duration". '
-
-- `--save-result`  
-Specify to save benchmark results to a json file
-
-- `--save-detailed`  
-"When saving the results, whether to include per request "
-        "information such as response, error, ttfs, tpots, etc."
-
-- `--result-dir`  
- "Specify directory to save benchmark json results."
-        "If not specified, results are saved in the current directory."
-
-- `--result-filename`  
-"Specify the filename to save benchmark json results."
-        "If not specified, results will be saved in "
-        "{label}-{args.request_rate}qps-{base_model_id}-{current_dt}.json"
-
-- `--random-prefix-len`  
-  Number of fixed prefix tokens before the random context in a request.
-  The total input length is the sum of random-prefix-len and a random
-  context length sampled from [input_len * (1 - range_ratio),
-  input_len * (1 + range_ratio)].Only the random and random-mm modes
-  support this parameter.
-
-- `--random-input-len`  
-  Number of input tokens per request.Only the random and random-mm modes support this parameter.
-
-- `--random-output-len`  
-  Number of output tokens per request.Only the random and random-mm modes support this parameter.
-
-- `--random-range-ratio`  
-  Range ratio for sampling input/output length,
-  used only for random sampling. Must be in the range [0, 1) to define
-  a symmetric sampling range
-  [length * (1 - range_ratio), length * (1 + range_ratio)].
-  Only the random and random-mm modes support this parameter.
-
-- `--random-mm-base-items-per-request`  
-  Base number of multimodal items per request for random-mm.
-  Actual per-request count is sampled around this base using
-  --random-mm-num-mm-items-range-ratio.
-  Only the random-mm mode supports this parameter.
-
-- `--random-mm-limit-mm-per-prompt`  
-  Per-modality hard caps for items attached per request, e.g.
-  '{"image": 3, "video": 1, "audio": 1}'. The sampled per-request item
-  count is clamped to the sum of these limits. When a modality
-  reaches its cap, its buckets are excluded and probabilities are
-  renormalized.
-  Only the random-mm mode supports this parameter.
-
-- `--random-mm-num-mm-items-range-ratio`  
-  Range ratio r in [0, 1] for sampling items per request.
-  We sample uniformly from the closed integer range
-  [floor(n*(1-r)), ceil(n*(1+r))]
-  where n is the base items per request.
-  r=0 keeps it fixed; r=1 allows 0 items. The maximum is clamped
-  to the sum of per-modality limits from
-  --random-mm-limit-mm-per-prompt.
-  An error is raised if the computed min exceeds the max.
-  Only the random-mm mode supports this parameter.
-
-- `--random-mm-bucket-config`  
-  The bucket config is a dictionary mapping a multimodal item
-  sampling configuration to a probability.
-  Currently allows for 3 modalities: audio, images and videos.
-  A bucket key is a tuple of (height, width, num_frames)
-  The value is the probability of sampling that specific item.
-  Example:
-  --random-mm-bucket-config
-  "{(256, 256, 1): 0.5, (720, 1280, 16): 0.4, (0, 1, 5): 0.10}"
-  First item: images with resolution 256x256 w.p. 0.5
-  Second item: videos with resolution 720x1280 and 16 frames
-  Third item: audios with 1s duration and 5 channels w.p. 0.1
-  OBS.: If the probabilities do not sum to 1, they are normalized.
-  Only the random-mm mode supports this parameter
-
-## Usage Examples
-
-### Online Benchmark
-<details class="admonition abstract" markdown="1">
-<summary>Show more</summary>
-
-First start serving your model:
-
-```bash
-vllm serve Qwen/Qwen2.5-Omni-7B --omni
-```
-
-Then run the benchmarking for sharegpt:
-
-```bash
-# download dataset
-# wget https://huggingface.co/datasets/anon8231489123/ShareGPT_Vicuna_unfiltered/resolve/main/ShareGPT_V3_unfiltered_cleaned_split.json
-vllm bench serve \
-  --omni \
-  --port 43845 \
-  --model /home/models/Qwen/Qwen3-Omni-30B-A3B-Instruct \
-  --endpoint /v1/chat/completions \
-  --backend openai-chat-omni \
-  --num-prompts 2 \
-  --dataset-name sharegpt \
-  --dataset-path ShareGPT_V3_unfiltered_cleaned_split.json \
-  --percentile-metrics ttft,tpot,itl,e2el
-```
-If successful, you will see the following output:
-```text
-============ Serving Benchmark Result ============
-Successful requests:                     2
-Failed requests:                         0
-Benchmark duration (s):                  81.63
-Request throughput (req/s):              0.02
-Peak concurrent requests:                2.00
-----------------End-to-end Latency----------------
-Mean E2EL (ms):                          56966.13
-Median E2EL (ms):                        56966.13
-P99 E2EL (ms):                           81016.80
-================== Text Result ===================
-Total input tokens:                      36
-Total generated tokens:                  5926
-Output token throughput (tok/s):         72.60
-Peak output token throughput (tok/s):    103.00
-Peak concurrent requests:                2.00
-Total Token throughput (tok/s):          73.04
----------------Time to First Token----------------
-Mean TTFT (ms):                          124.76
-Median TTFT (ms):                        124.76
-P99 TTFT (ms):                           156.10
------Time per Output Token (excl. 1st token)------
-Mean TPOT (ms):                          481.30
-Median TPOT (ms):                        481.30
-P99 TPOT (ms):                           947.55
----------------Inter-token Latency----------------
-Mean ITL (ms):                           25.11
-Median ITL (ms):                         0.33
-P99 ITL (ms):                            25.17
-================== Audio Result ==================
-Total audio duration generated(s):       3.95
-Total audio frames generated:            94890
-Audio throughput(audio duration/s):      0.05
-==================================================
-```
-
-Or run the benchmarking for random:
-
-```bash
-vllm bench serve \
-  --omni \
-  --port 43845 \
-  --endpoint /v1/chat/completions \
-  --backend openai-chat-omni \
-  --model /home/models/Qwen/Qwen3-Omni-30B-A3B-Instruct \
-  --dataset-name random \
-  --num-prompts 2 \
-  --random-prefix-len 5 \
-  --random-input-len 10 \
-  --random-output-len 100 \
-  --percentile-metrics ttft,tpot,itl,e2el,audio_ttfp,audio_rtf \
-  --ignore-eos
-```
-
-If successful, you will see the following output:
-
-```text
-============ Serving Benchmark Result ============
-Successful requests:                     2
-Failed requests:                         0
-Benchmark duration (s):                  3.89
-Request throughput (req/s):              0.51
-Peak concurrent requests:                2.00
-----------------End-to-end Latency----------------
-Mean E2EL (ms):                          3824.76
-Median E2EL (ms):                        3824.76
-P99 E2EL (ms):                           3888.54
-================== Text Result ===================
-Total input tokens:                      30
-Total generated tokens:                  10101
-Output token throughput (tok/s):         2595.57
-Peak output token throughput (tok/s):    111.00
-Peak concurrent requests:                2.00
-Total Token throughput (tok/s):          2603.28
----------------Time to First Token----------------
-Mean TTFT (ms):                          117.15
-Median TTFT (ms):                        117.15
-P99 TTFT (ms):                           142.69
------Time per Output Token (excl. 1st token)------
-Mean TPOT (ms):                          0.73
-Median TPOT (ms):                        0.73
-P99 TPOT (ms):                           0.74
----------------Inter-token Latency----------------
-Mean ITL (ms):                           16.47
-Median ITL (ms):                         16.19
-P99 ITL (ms):                            52.55
-================== Audio Result ==================
-Total audio duration generated(s):       15.79
-Total audio frames generated:            379050
-Audio throughput(audio duration/s):      4.06
----------------Time to First Packet---------------
-Mean AUDIO_TTFP (ms):                    3701.37
-Median AUDIO_TTFP (ms):                  3701.37
-P99 AUDIO_TTFP (ms):                     3762.25
------------------Real Time Factor-----------------
-Mean AUDIO_RTF:                          0.47
-Median AUDIO_RTF:                        0.47
-P99 AUDIO_RTF:                           0.48
-==================================================
-```
-Notes:
-We use audio generation time / audio duration to calculate RTF.
-
-</details>
-
-### Multi-Modal Benchmark
-
-<details class="admonition abstract" markdown="1">
-<summary>Show more</summary>
-
-Benchmark the performance of multi-modal requests in vLLM-Omni.
-
-Generate synthetic image、video、audio inputs alongside random text prompts to stress-test vision models without external datasets.
-
-Notes:
-
-- Works only with online benchmark via the OpenAI backend (`--backend openai-chat-omni`) and endpoint `/v1/chat/completions`.
-
-Start the server (example):
-
-```bash
-vllm serve Qwen/Qwen2.5-Omni-7B --omni
-```
-
-It is recommended to use the flag `--ignore-eos` to simulate real responses. You can set the size of the output via the arg `random-output-len`.
-
-Then run the benchmarking script:
-```bash
-vllm bench serve \
-    --omni \
-  --dataset-name random-mm \
-  --port 40849 \
-  --model /home/models/Qwen/Qwen3-Omni-30B-A3B-Instruct \
-  --endpoint /v1/chat/completions \
-  --backend openai-chat-omni \
-  --request-rate 1 \
-  --num-prompts 1 \
-  --random-input-len 10 \
-  --random-range-ratio 0.0 \
-  --random-mm-base-items-per-request 2 \
-  --random-mm-num-mm-items-range-ratio 0 \
-  --random-mm-limit-mm-per-prompt '{"image":1,"video":1, "audio": 1}' \
-  --random-mm-bucket-config '{"(32, 32, 1)": 0.5, "(0, 1, 1)": 0.1, "(32, 32, 2)":0.4}' \
-  --ignore-eos \
-  --percentile-metrics ttft,tpot,itl \
-  --random-output-len 2 \
-  --extra_body '{"modalities": ["text"]}'
-```
-
-If successful, you will see the following output:
-
-```text
-============ Serving Benchmark Result ============
-Successful requests:                     1
-Failed requests:                         0
-Request rate configured (RPS):           1.00
-Benchmark duration (s):                  1.21
-Request throughput (req/s):              0.83
-Peak concurrent requests:                1.00
-================== Text Result ===================
-Total input tokens:                      10
-Total generated tokens:                  3
-Output token throughput (tok/s):         2.49
-Peak output token throughput (tok/s):    3.00
-Peak concurrent requests:                1.00
-Total Token throughput (tok/s):          10.77
----------------Time to First Token----------------
-Mean TTFT (ms):                          179.74
-Median TTFT (ms):                        179.74
-P99 TTFT (ms):                           179.74
------Time per Output Token (excl. 1st token)------
-Mean TPOT (ms):                          12.76
-Median TPOT (ms):                        12.76
-P99 TPOT (ms):                           12.76
----------------Inter-token Latency----------------
-Mean ITL (ms):                           12.76
-Median ITL (ms):                         12.76
-P99 ITL (ms):                            25.24
-================== Audio Result ==================
-Total audio duration generated(s):       0.00
-Total audio frames generated:            0
-Audio throughput(audio duration/s):      0.00
-==================================================
-```
-
-Behavioral notes:
-
-- If the requested base item count cannot be satisfied under the provided per-prompt limits, the tool raises an error rather than silently clamping.
-
-How sampling works:
-
-- Determine per-request item count k by sampling uniformly from the integer range defined by `--random-mm-base-items-per-request` and `--random-mm-num-mm-items-range-ratio`, then clamp k to at most the sum of per-modality limits.
-- For each of the k items, sample a bucket (H, W, T) according to the normalized probabilities in `--random-mm-bucket-config`, while tracking how many items of each modality have been added.
-- If a modality (e.g., image) reaches its limit from `--random-mm-limit-mm-per-prompt`, all buckets of that modality are excluded and the remaining bucket probabilities are renormalized before continuing.
-This should be seen as an edge case, and if this behavior can be avoided by setting `--random-mm-limit-mm-per-prompt` to a large number. Note that this might result in errors due to engine config `--limit-mm-per-prompt`.
-- The resulting request contains synthetic image data in `multi_modal_data` (OpenAI Chat format). When `random-mm` is used with the OpenAI Chat backend, prompts remain text and MM content is attached via `multi_modal_data`.
-</details>

docs/cli/serve.md

@@ -1,9 +0,0 @@
-# vllm-omni serve
-
-## JSON CLI Arguments
-
---8<-- "docs/cli/json_tip.inc.md"
-
-## Arguments
-
---8<-- "docs/generated/argparse_omni/omni_serve.inc.md"

docs/configuration/README.md

@@ -2,7 +2,7 @@
 
 This section lists the most common options for running vLLM-Omni.
 
-For options within a vLLM Engine. Please refer to [vLLM Configuration](https://docs.vllm.ai/en/v0.14.0/configuration/index.html)
+For options within a vLLM Engine. Please refer to [vLLM Configuration](https://docs.vllm.ai/en/v0.11.0/configuration/index.html)
 
 Currently, the main options are maintained by stage configs for each model.
 
@@ -10,12 +10,6 @@ For specific example, please refer to [Qwen2.5-omni stage config](stage_configs/
 
 For introduction, please check [Introduction for stage config](./stage_configs.md)
 
-## Memory Configuration
-
-- **[GPU Memory Calculation and Configuration](./gpu_memory_utilization.md)** - Guide on how to calculate memory requirements and set up `gpu_memory_utilization` for optimal performance
-
 ## Optimization Features
 
-- **[TeaCache Configuration](../user_guide/diffusion/teacache.md)** - Enable TeaCache adaptive caching for DiT models to achieve 1.5x-2.0x speedup with minimal quality loss
-- **[Cache-DiT Configuration](../user_guide/diffusion/cache_dit_acceleration.md)** - Enable Cache-DiT as cache acceleration backends for DiT models
-- **[Parallelism Configuration](../user_guide/diffusion/parallelism_acceleration.md)** - Enable parallelism (e.g., sequence parallelism) for for DiT models
+- **[TeaCache Configuration](../user_guide/teacache.md)** - Enable TeaCache adaptive caching for DiT models to achieve 1.5x-2.0x speedup with minimal quality loss

docs/configuration/gpu_memory_utilization.md

@@ -1,207 +0,0 @@
-# GPU Memory Calculation and Configuration
-
-This guide explains how to calculate GPU memory requirements and properly configure `gpu_memory_utilization` for vLLM-Omni stages.
-
-## Overview
-
-`gpu_memory_utilization` is a critical parameter that controls how much GPU memory each stage can use. It's specified as a fraction between 0.0 and 1.0, where:
-- `0.8` means 80% of the GPU's total memory
-- `1.0` means 100% of the GPU's total memory (not recommended, leaves no buffer)
-
-## How Memory is Calculated
-
-### Memory Allocation Formula
-
-For each stage, vLLM-Omni calculates the requested memory as:
-
-```
-requested_memory = total_gpu_memory × gpu_memory_utilization
-```
-
-The system checks that:
-```
-free_memory ≥ requested_memory
-```
-
-If this condition is not met, the stage will fail to initialize with an error message showing the memory requirements.
-
-### Memory Components
-
-The total memory used by a stage includes:
-
-1. **Model Weights**: The size of the model parameters loaded on the GPU
-2. **KV Cache**: Memory for storing key-value cache during generation
-3. **Activation Memory**: Temporary memory for intermediate computations
-4. **System Overhead**: Memory used by CUDA, PyTorch, and other system components
-5. **Non-Torch Memory**: Memory allocated outside of PyTorch (e.g., CUDA graphs)
-
-### Example Calculation
-
-For a GPU with 80GB total memory:
-- `gpu_memory_utilization: 0.8` → 64GB available for the stage
-- `gpu_memory_utilization: 0.6` → 48GB available for the stage
-- `gpu_memory_utilization: 0.15` → 12GB available for the stage
-
-## Setting Up `gpu_memory_utilization`
-
-### Step 1: Determine GPU Memory
-
-First, check your GPU's total memory:
-
-```bash
-# Using nvidia-smi
-nvidia-smi --query-gpu=memory.total --format=csv
-
-# Or using Python
-python -c "import torch; print(f'{torch.cuda.get_device_properties(0).total_memory / 1e9:.1f} GB')"
-```
-
-### Step 2: Estimate Model Memory Requirements
-
-#### For Autoregressive (AR) Stages
-
-AR stages typically need more memory due to:
-- Large model weights
-- KV cache for attention
-- Activation buffers
-
-#### For Diffusion/Generation Stages
-
-Diffusion stages (like code2wav) typically need less memory:
-- Smaller model components
-- Different memory access patterns
-
-**Typical values:**
-- `0.1 - 0.3` for most diffusion stages
-
-### Step 3: Consider Multi-Stage Scenarios
-
-When multiple stages share the same GPU, you must ensure the sum of their `gpu_memory_utilization` values doesn't exceed 1.0.
-
-**Example: Two stages on GPU 0**
-```yaml
-stage_args:
-  - stage_id: 0
-    runtime:
-      devices: "0"
-    engine_args:
-      gpu_memory_utilization: 0.6  # Uses 60% of GPU 0
-
-  - stage_id: 1
-    runtime:
-      devices: "0"
-    engine_args:
-      gpu_memory_utilization: 0.3  # Uses 30% of GPU 0
-      # Total: 90% of GPU 0 (safe, leaves 10% buffer)
-```
-
-**Important:** If stages run on different GPUs, each can use up to 1.0 independently.
-
-### Step 4: Account for Tensor Parallelism
-
-When using `tensor_parallel_size > 1`, the model is split across multiple GPUs, so each GPU needs less memory.
-
-**Example: 2-way tensor parallelism**
-```yaml
-stage_args:
-  - stage_id: 0
-    runtime:
-      devices: "0,1"  # Uses both GPUs
-    engine_args:
-      tensor_parallel_size: 2
-      gpu_memory_utilization: 0.6  # 60% per GPU
-      # Model is split, so each GPU uses ~30% of model memory
-```
-
-## Examples
-
-### Qwen3-Omni-MoE on 2x H100-80GB
-
-```yaml
-stage_args:
-  - stage_id: 0  # Thinker stage with TP=2
-    runtime:
-      devices: "0,1"
-    engine_args:
-      tensor_parallel_size: 2
-      gpu_memory_utilization: 0.6  # 48GB per GPU
-
-  - stage_id: 1  # Talker stage
-    runtime:
-      devices: "1"
-    engine_args:
-      gpu_memory_utilization: 0.3  # 24GB on GPU 1
-
-  - stage_id: 2  # Code2Wav stage
-    runtime:
-      devices: "0"
-    engine_args:
-      gpu_memory_utilization: 0.1  # 8GB on GPU 0
-```
-**Note:** In this configuration, stages 0 and 2 share GPU 0, but they run at different times in the pipeline, so their memory usage doesn't overlap.
-
-## Troubleshooting
-
-### Error: "Free memory is less than desired GPU memory utilization"
-
-This means the GPU doesn't have enough free memory when the stage starts.
-
-**Solutions:**
-1. Free up memory by closing other processes
-2. Reduce `gpu_memory_utilization` for this stage
-3. Use a GPU with more memory
-4. Move the stage to a different GPU
-
-### Error: OOM during inference
-
-The stage initialized but ran out of memory during processing.
-
-**Solutions:**
-1. Reduce `max_num_batched_tokens`
-2. Reduce `max_batch_size` in runtime config
-3. Lower `gpu_memory_utilization` slightly
-4. Enable quantization if supported
-
-### Memory Not Fully Utilized
-
-If you see low memory usage, you can:
-1. Increase `gpu_memory_utilization` to allow larger KV cache
-2. Increase `max_num_batched_tokens` for better batching
-3. Check if other stages are limiting throughput
-
-## Useful formula for Memory Calculation
-
-### KV Cache Memory
-
-The KV cache size depends on:
-- Number of sequences in batch
-- Sequence length (prompt + generation)
-- Model hidden size
-- Number of attention heads
-- Number of layers
-
-approximate Formula:
-```
-kv_cache_memory ≈ batch_size × seq_len × hidden_size × num_layers × 2 × dtype_size
-```
-2 for k & v
-
-### Model Weight Memory
-
-```
-model_memory ≈ num_parameters × dtype_size
-```
-
-For example:
-- 7B parameters in FP16: ~14GB
-- 7B parameters in FP32: ~28GB
-- 7B parameters in INT8: ~7GB
-
-### Activation Memory
-
-Activation memory is typically smaller but varies with:
-- Batch size
-- Sequence length
-- Model architecture
-
-It's usually 10-30% of model weight memory during inference.

docs/configuration/stage_configs.md

@@ -40,8 +40,8 @@ stage_args:
     engine_args: # Engine arguments for a certain engine
       model_stage: thinker
       model_arch: Qwen2_5OmniForConditionalGeneration # The model implementation registered in model_executor/models/registry.py
-      worker_type: ar # The specific worker used
-      scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler # The specific scehduler used
+      worker_cls: vllm_omni.worker.gpu_ar_worker.GPUARWorker # The specific worker used
+      scheduler_cls: vllm_omni.core.sched.scheduler.OmniScheduler # The specific scehduler used
       gpu_memory_utilization: 0.8 # The gpu memory allocation for the stage within a single chip
       enforce_eager: true  # Now we only support eager mode
       trust_remote_code: true # Needed by huggingface config parsing
@@ -66,8 +66,8 @@ stage_args:
     engine_args:
       model_stage: talker
       model_arch: Qwen2_5OmniForConditionalGeneration
-      worker_type: ar
-      scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
+      worker_cls: vllm_omni.worker.gpu_ar_worker.GPUARWorker
+      scheduler_cls: vllm_omni.core.sched.scheduler.OmniScheduler
       gpu_memory_utilization: 0.8
       enforce_eager: true
       trust_remote_code: true
@@ -92,8 +92,8 @@ stage_args:
     engine_args:
       model_stage: code2wav
       model_arch: Qwen2_5OmniForConditionalGeneration
-      worker_type: generation
-      scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler
+      worker_cls: vllm_omni.worker.gpu_diffusion_worker.GPUDiffusionWorker
+      scheduler_cls: vllm_omni.core.sched.diffusion_scheduler.DiffusionScheduler
       gpu_memory_utilization: 0.15
       enforce_eager: true
       trust_remote_code: true
@@ -171,11 +171,11 @@ The model architecture class name that is registered in `model_executor/models/r
 
 #### `engine_args.worker_cls`
 
-The specific worker class to use for this stage. This determines how the model computations are executed. Examples include `vllm_omni.worker.gpu_ar_worker.GPUARWorker` for autoregressive stages and `vllm_omni.worker.gpu_generation_worker.GPUGenerationWorker` for diffusion-based stages.
+The specific worker class to use for this stage. This determines how the model computations are executed. Examples include `vllm_omni.worker.gpu_ar_worker.GPUARWorker` for autoregressive stages and `vllm_omni.worker.gpu_diffusion_worker.GPUDiffusionWorker` for diffusion-based stages.
 
 #### `engine_args.scheduler_cls`
 
-The scheduler class to use for this stage. The scheduler manages request queuing, batching, and execution order. Examples include `vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler` for standard stages and `vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler` for diffusion stages.
+The scheduler class to use for this stage. The scheduler manages request queuing, batching, and execution order. Examples include `vllm_omni.core.sched.scheduler.OmniScheduler` for standard stages and `vllm_omni.core.sched.diffusion_scheduler.DiffusionScheduler` for diffusion stages.
 
 #### `engine_args.gpu_memory_utilization`
 
@@ -183,9 +183,6 @@ The fraction of GPU memory to allocate for this stage within a single GPU chip.
 
 Default: `0.8`
 
-!!! tip "Memory Configuration Guide"
-    For detailed information on how to calculate memory requirements and properly configure `gpu_memory_utilization`, see the [GPU Memory Calculation and Configuration Guide](./gpu_memory_utilization.md).
-
 #### `engine_args.enforce_eager`
 
 Whether to enforce eager execution mode. When set to `true`, the engine will run in eager mode without using CUDA graphs or other compilation optimizations. Currently, vLLM-Omni only supports eager mode.
@@ -273,15 +270,3 @@ Default: `True`
 Penalty applied to tokens that have already appeared in the generated sequence. Values greater than 1.0 discourage repetition, while values less than 1.0 encourage it. A value of 1.0 applies no penalty.
 
 Default: `1.1`
-
-### `tts_args` (TTS stages only)
-
-Configuration for Text-to-Speech specific parameters. This section is only applicable to TTS model stages (e.g., `qwen3_tts`).
-
-#### `tts_args.max_instructions_length`
-
-Maximum character length for voice style/emotion instructions. Instructions exceeding this limit will be rejected with a validation error.
-
-Default: `500`
-
-This value can be overridden at runtime using the `--tts-max-instructions-length` CLI parameter when starting the server.

docs/configuration/stage_configs/qwen2_5_omni.yaml

@@ -8,7 +8,7 @@ stage_args:
     engine_args:
       model_stage: thinker
       model_arch: Qwen2_5OmniForConditionalGeneration
-      worker_type: ar
+      worker_cls: vllm_omni.worker.gpu_ar_worker.GPUARWorker
       scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
       gpu_memory_utilization: 0.8
       enforce_eager: true  # Now we only support eager mode
@@ -34,7 +34,7 @@ stage_args:
     engine_args:
       model_stage: talker
       model_arch: Qwen2_5OmniForConditionalGeneration
-      worker_type: ar
+      worker_cls: vllm_omni.worker.gpu_ar_worker.GPUARWorker
       scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
       gpu_memory_utilization: 0.8
       enforce_eager: true
@@ -60,8 +60,8 @@ stage_args:
     engine_args:
       model_stage: code2wav
       model_arch: Qwen2_5OmniForConditionalGeneration
-      worker_type: generation
-      scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler
+      worker_cls: vllm_omni.worker.gpu_diffusion_worker.GPUDiffusionWorker
+      scheduler_cls: vllm_omni.core.sched.diffusion_scheduler.DiffusionScheduler
       gpu_memory_utilization: 0.15
       enforce_eager: true
       trust_remote_code: true

docs/contributing/README.md

@@ -13,18 +13,23 @@ vLLM-Omni uses `uv` as the environment manager, to create and manage Python envi
 uv venv --python 3.12 --seed
 source .venv/bin/activate
 ```
+### Development Environment for vLLM
 
-### Development Environment for vLLM and vLLM-Omni
+vLLM-Omni is built on top of the stable release 0.11.0 of vLLM. You may install `vllm==0.11.0` directly or install the library from source if you need to check, modify or debug with source code of vLLM.
 
-vLLM-Omni is quickly evolving, please see the [installation guide](../getting_started/installation/README.md) for details. It's recommended to build from source to provide the latest development environment.
+### Development Environment for vLLM-Omni
+
+Install vLLM-Omni from source with development dependencies:
+
+```bash
+git clone https://github.com/vllm-project/vllm-omni.git
+cd vllm-omni
+uv pip install -e ".[dev]"
+```
 
 !!! tip
     vLLM-Omni is compatible with Python versions 3.10 to 3.12. However, we recommend developing with Python 3.12 to minimize the chance of your local environment clashing with our CI environment.
 
-### Adding a new model to vLLM-Omni
-
-Please check [model implementation](model/README.md) for how to add diffusion and omni-modality models to vLLM-Omni.
-
 ### Linting
 
 vLLM-Omni uses `pre-commit` to lint and format the codebase. See [pre-commit documentation](https://pre-commit.com/#usage) if `pre-commit` is new to you. Setting up `pre-commit` is as easy as:
@@ -116,7 +121,7 @@ Only specific types of PRs will be reviewed. The PR title is prefixed appropriat
 - `[Model]` for adding a new model or improving an existing model. Model name should appear in the title.
 - `[Frontend]` For changes on the vLLM-Omni frontend (e.g., OpenAI API server, `OmniLLM` class, etc.)
 - `[Kernel]` for changes affecting CUDA kernels or other compute kernels.
-- `[Core]` for changes in the core vLLM-Omni logic (e.g., `OmniProcessor`, `OmniARScheduler`, etc.)
+- `[Core]` for changes in the core vLLM-Omni logic (e.g., `OmniProcessor`, `OmniScheduler`, etc.)
 - `[Hardware][Vendor]` for hardware-specific changes. Vendor name should appear in the prefix, such as [Ascend] for Ascend NPUs.
 - `[Misc]` for PRs that do not fit the above categories. Please use this sparingly.
 

docs/contributing/ci/CI_5levels.md

@@ -1,678 +0,0 @@
-# Multi-Level Automated Testing System Documentation
-
-## Document Overview
-
-This testing system aims to build a complete, efficient, and well-structured quality assurance framework for the development, integration, and release of model services. It draws on the concept of the test pyramid from modern software engineering, progressively expanding testing activities from basic code logic verification to complex end-to-end (E2E) functionality, performance, accuracy, and even long-term stability validation.
-
-Through five levels (L1-L5) and common (Common) specifications, the system clarifies the testing objectives, scope, execution frequency, and required resources for different development stages (e.g., each commit, PR merge, daily build, pre-release). This ensures that models meet high standards for functionality, performance, and reliability across various deployment scenarios (online serving and offline inference).
-
-<table>
-  <thead>
-    <tr>
-      <th>Level</th>
-      <th>Scope & Focus</th>
-      <th>Time Cost</th>
-      <th>Test Dir</th>
-      <th>Doc</th>
-      <th>Frequency</th>
-      <th>Hardware</th>
-    </tr>
-  </thead>
-  <tbody>
-    <tr>
-      <td rowspan="2"><strong>Common</strong></td>
-      <td>Contribution Guideline & PR checklist</td>
-      <td>/</td>
-      <td>/</td>
-      <td>.github/PULL_REQUEST_TEMPLATE.md <a href="../tests_style/"> Test Style (PR Checklist)</a></td>
-      <td>/</td>
-      <td>/</td>
-    </tr>
-    <tr>
-      <td>CI Failure Description</td>
-      <td>/</td>
-      <td>/</td>
-      <td><a href="../failures/"> CI Failures</a></td>
-      <td>/</td>
-      <td>/</td>
-    </tr>
-    <tr>
-      <td><strong>L1</strong><br>(Unit & Logic)</td>
-      <td>Unit tests for components like entrypoints, models</td>
-      <td rowspan="2">&lt;15min</td>
-      <td>/tests/{component_name}/test_xxx</td>
-      <td>
-        <a href="#chapter-1-l1-l2-level-testing-unit-testing-and-basic-end-to-end-verification">Chapter 1</a><br>
-        Section 1 L1&amp;L2: Purpose, Test Content, Directory Location, Example
-      </td>
-      <td>PR with ready label (also can run locally)</td>
-      <td>CPU</td>
-    </tr>
-    <tr>
-      <td><strong>L2</strong><br>(E2E across models & GPU-required UT)</td>
-      <td>Online & Offline (basic deployment scenarios):<br>dummy, normal inference function (output format, stream), some instance startup UT</td>
-      <td>
-        /tests/e2e/online_serving/test_{model_name}.py<br>
-        /tests/e2e/offline_inference/test_{model_name}.py
-      </td>
-      <td>
-        <a href="#chapter-1-l1-l2-level-testing-unit-testing-and-basic-end-to-end-verification">Chapter 1</a><br>
-        L1&amp;L2: Purpose, Test Content, Directory Location, Example
-      </td>
-      <td>PR with ready label</td>
-      <td>GPU</td>
-    </tr>
-    <tr>
-      <td><strong>L3</strong><br>(Important Perf & Integration & Accuracy)</td>
-      <td>Online & Offline (multiple deployment scenarios):<br>real model, normal inference function, normal accuracy</td>
-      <td>&lt;30min</td>
-      <td>
-        /tests/e2e/online_serving/test_{model_name}_expansion.py<br>
-        /tests/e2e/offline_inference/test_{model_name}_expansion.py
-      </td>
-      <td>
-        <a href="#chapter-2-l3-level-testing-core-integration-performance-and-accuracy-verification">Chapter 2</a><br>
-        L3: Purpose, Test Content, Directory Location, Example
-      </td>
-      <td>PR Merged (Also run L1&L2 Tests)</td>
-      <td>GPU</td>
-    </tr>
-    <tr>
-      <td><strong>L4</strong><br>(Perf & Integration & Accuracy)</td>
-      <td>Online & Offline: full functional scenarios + performance test + doc test</td>
-      <td>&lt;3 hour</td>
-      <td>
-        <strong>Full Function:</strong><br>
-        /tests/e2e/online_serving/test_{model_name}_expansion.py<br>
-        /tests/e2e/offline_inference/test_{model_name}_expansion.py<br>
-        <strong>Performance:</strong><br>
-        /tests/e2e/perf/nightly.json<br>
-        <strong>Doc Test:</strong><br>
-        tests/example/online_serving/test_{model_name}.py<br>
-        tests/example/offline_inference/test_{model_name}.py
-      </td>
-      <td>
-        <a href="#chapter-3-l4-level-testing-full-functionality-performance-and-documentation-testing">Chapter 3</a><br>
-        L4: Purpose, Test Content, Directory Location, Example
-      </td>
-      <td>Nightly</td>
-      <td>GPU</td>
-    </tr>
-    <tr>
-      <td><strong>L5</strong><br>(Stability & Reliability)</td>
-      <td>Online & Offline: long-term stability test + reliability test</td>
-      <td> Depends on reality </td>
-      <td>
-        <strong>Stability:</strong><br>
-        tests/e2e/stability/weekly.json<br>
-        <strong>Reliability:</strong><br>
-        tests/e2e/reliability/test_{model_name}.py
-      </td>
-      <td>
-        <a href="#chapter-4-l5-level-testing-stability-and-reliability-testing">Chapter 4</a><br>
-        L5: Purpose, Test Content, Directory Location, Example
-      </td>
-      <td>Weekly / Days before Release</td>
-      <td>GPU</td>
-    </tr>
-  </tbody>
-</table>
-
-
----
-<details>
-<summary> The folder structure for tests file based on the 5 levels design</summary>
-Legend: `✅` = test exists, `⬜` = suggested to add.
-```
-vllm_omni/                                    tests/
-├── config/                             →     ├── config/
-│   ├── model.py                              │   └── test_model.py                    ⬜
-│   └── lora.py                               │   └── test_lora.py                      ⬜
-│
-├── core/                               →     ├── core/
-│   └── sched/                                 │   └── sched/
-│       ├── omni_ar_scheduler.py               │       ├── test_omni_ar_scheduler.py    ⬜
-│       ├── omni_generation_scheduler.py       │       ├── test_omni_generation_scheduler.py  ⬜
-│       └── output.py                          │       └── test_output.py               ✅ currently in entrypoints/test_omni_new_request_data.py (tests output.OmniNewRequestData)
-│
-├── diffusion/                          →     ├── diffusion/
-│   ├── diffusion_engine.py                    │   ├── test_diffusion_engine.py          ⬜
-│   ├── attention/                             │   ├── attention/
-│   │   ├── layer.py                            │   │   ├── test_attention_sp.py         ✅
-│   │   └── backends/                           │   │   └── test_flash_attn.py           ✅
-│   ├── distributed/                           │   ├── distributed/
-│   │   └── ...                                 │   │   ├── test_comm.py                 ✅
-│   │                                            │   │   ├── test_cfg_parallel.py        ✅
-│   │                                            │   │   └── test_sp_plan_hooks.py       ✅
-│   ├── lora/                                   │   ├── lora/
-│   │   └── ...                                 │   │   ├── test_base_linear.py          ✅
-│   │                                            │   │   └── test_lora_manager.py        ✅
-│   ├── models/                                 │   ├── models/
-│   │   ├── qwen_image/                         │   │   ├── qwen_image/                 (e2e coverage)
-│   │   ├── z_image/                            │   │   └── z_image/
-│   │   └── ...                                 │   │       └── test_zimage_tp_constraints.py  ✅
-│   └── worker/                                 │   └── worker/
-│       ├── diffusion_worker.py                 │       └── test_diffusion_worker.py   ✅ file at diffusion/test_diffusion_worker.py
-│       └── diffusion_model_runner.py            │
-│
-├── distributed/                         →     ├── distributed/
-│   └── omni_connectors/                         │   └── omni_connectors/
-│       ├── adapter.py                           │       ├── test_adapter_and_flow.py   ✅
-│       ├── kv_transfer_manager.py               │       ├── test_basic_connectors.py   ✅
-│       ├── connectors/                           │       ├── test_kv_flow.py             ✅
-│       └── utils/                               │       └── test_omni_connector_configs.py  ✅
-│
-├── engine/                             →     ├── engine/
-│   ├── input_processor.py                      │   ├── test_input_processor.py         ⬜  (no processor.py in source)
-│   ├── output_processor.py                     │   └── test_output_processor.py         ⬜
-│   └── arg_utils.py                            │   └── test_arg_utils.py               ⬜
-│
-├── entrypoints/                        →     ├── entrypoints/
-│   ├── omni.py                                 │   ├── test_omni.py                    ⬜  (E2E covered by e2e/offline, e2e/online)
-│   ├── omni_llm.py                             │   ├── test_omni_llm.py                ✅
-│   ├── omni_stage.py                            │   ├── test_omni_stage.py              ⬜  (partial in test_omni_stage_diffusion_config.py)
-│   ├── omni_diffusion.py                       │   ├── test_omni_diffusion.py          ✅
-│   ├── async_omni.py                            │   ├── test_async_omni.py              ✅ actually in e2e/online_serving/test_async_omni.py
-│   ├── async_omni_diffusion.py                 │   ├── test_async_omni_diffusion_config.py  ✅
-│   ├── stage_utils.py                          │   ├── test_stage_utils.py            ✅
-│   ├── cli/                                     │   ├── cli/                           (benchmarks/test_serve_cli.py covers CLI serve)
-│   │   └── ...                                  │   │   └── test_*.py                  ⬜
-│   └── openai/                                  │   └── openai_api/                    # maps to entrypoints/openai/
-│       ├── api_server.py                        │       ├── test_api_server.py         ⬜  (e2e indirect coverage)
-│       ├── serving_chat.py                       │       ├── test_serving_chat_sampling_params.py  ✅
-│       ├── serving_speech.py                     │       ├── test_serving_speech.py     ✅
-│       └── image_api_utils.py                   │       └── test_image_server.py      ✅
-│
-├── inputs/                             →     ├── inputs/
-│   ├── data.py                                 │   ├── test_data.py                   ⬜
-│   ├── parse.py                                │   ├── test_parse.py                 ⬜
-│   └── preprocess.py                            │   └── test_preprocess.py            ✅ currently in entrypoints/test_omni_input_preprocessor.py
-│
-├── model_executor/                     →     ├── model_executor/
-│   ├── layers/                                  │   ├── layers/
-│   │   └── mrope.py                             │   │   └── test_mrope.py              ⬜
-│   ├── model_loader/                            │   ├── model_loader/
-│   │   └── weight_utils.py                      │   │   └── test_weight_utils.py      ⬜
-│   ├── models/                                  │   ├── models/
-│   │   ├── qwen2_5_omni/                         │   │   ├── qwen2_5_omni/
-│   │   │   ├── qwen2_5_omni_thinker.py           │   │   │   ├── test_audio_length.py  ✅
-│   │   │   ├── qwen2_5_omni_talker.py            │   │   │   ├── test_qwen2_5_omni_thinker.py  ⬜
-│   │   │   └── qwen2_5_omni_token2wav.py         │   │   │   ├── test_qwen2_5_omni_talker.py  ⬜
-│   │   └── qwen3_omni/                          │   │   │   └── test_qwen2_5_omni_token2wav.py  ⬜
-│   │       └── ...                               │   │   └── qwen3_omni/
-│   ├── stage_configs/                           │   │       └── test_*.py              ⬜
-│   │   └── *.yaml                               │   └── stage_configs/                 (used by e2e, test_*.py can be added)  ⬜
-│   └── stage_input_processors/                  │   └── stage_input_processors/
-│       └── ...                                  │       └── test_*.py                 ⬜
-│
-├── sample/                             →     ├── sample/
-│   └── __init__.py                             │   └── test_*.py                      ⬜
-│
-├── utils/                              →     ├── utils/
-│   └── __init__.py                             │   └── test_*.py                       ⬜  (no platform_utils.py currently)
-│
-├── worker/                             →     ├── worker/
-│   ├── gpu_ar_model_runner.py                  │   ├── test_gpu_ar_model_runner.py    ⬜
-│   ├── gpu_ar_worker.py                        │   ├── test_gpu_ar_worker.py           ⬜
-│   ├── gpu_generation_model_runner.py          │   ├── test_gpu_generation_model_runner.py  ✅
-│   ├── gpu_generation_worker.py                │   ├── test_gpu_generation_worker.py  ⬜
-│   ├── gpu_model_runner.py                     │   ├── test_omni_gpu_model_runner.py   ✅
-│   └── mixins.py                               │   └── (npu under platforms/npu/worker/)  # not worker/npu/
-│
-├── platforms/                          →     (no tests/platforms/, e2e and stage_configs provide indirect coverage)
-│   ├── cuda/
-│   ├── npu/worker/                             # NPU worker here, not vllm_omni/worker/npu/
-│   ├── rocm/
-│   └── xpu/worker/
-│
-├── outputs.py                          →     test_outputs.py                         ✅ (at tests root)
-├── (logger, patch, request, version)    →     (no corresponding unit test)
-│
-└── e2e (tests side only)               →     ├── e2e/
-                                               ├── online_serving/                     ✅ non-empty
-                                               │   ├── test_async_omni.py
-                                               │   ├── test_qwen3_omni.py
-                                               │   ├── test_qwen3_omni_expansion.py
-                                               │   ├── test_image_gen_edit.py
-                                               │   ├── test_images_generations_lora.py
-                                               │   └── stage_configs/
-                                               └── offline_inference/                  ✅
-                                                   ├── test_qwen2_5_omni.py
-                                                   ├── test_qwen3_omni.py
-                                                   ├── test_bagel_text2img.py
-                                                   ├── test_t2i_model.py
-                                                   ├── test_t2v_model.py
-                                                   ├── test_ovis_image.py
-                                                   ├── test_zimage_tensor_parallel.py
-                                                   ├── test_cache_dit.py
-                                                   ├── test_teacache.py
-                                                   ├── test_stable_audio_model.py
-                                                   ├── test_diffusion_cpu_offload.py
-                                                   ├── test_diffusion_layerwise_offload.py
-                                                   ├── test_diffusion_lora.py
-                                                   ├── test_sequence_parallel.py
-                                                   └── stage_configs/
-                                                       ├── qwen2_5_omni_ci.yaml
-                                                       ├── qwen3_omni_ci.yaml
-                                                       ├── bagel_*.yaml
-                                                       └── npu/, rocm/, etc.
-```
-
-
-</details>
-
-
-## Common Specifications
-
-Before entering specific testing levels, the project establishes two common specifications aimed at standardizing the development process and quickly locating issues.
-
-1.  ***PR Checklist ([Tests Style](../ci/tests_style.md))***: This template defines the self-check items that must be completed before submitting a code review (Pull Request). It ensures that each code change meets basic requirements such as code style, dependency updates, and documentation synchronization before entering the automated testing pipeline, serving as the first manual line of defense for quality assurance.
-2.  ***CI Failure Explanation ([CI Failures](../ci/failures.md))***: This document archives and explains common failure patterns in the Continuous Integration (CI) pipeline, error log interpretation, and preliminary troubleshooting steps. It helps developers and testers quickly diagnose the causes of automated test failures, improving problem-solving efficiency.
-
-## Chapter 1: L1 & L2 Level Testing - Unit Testing and Basic End-to-End Verification
-
-### 1.1 Testing Purpose
-
-L1 and L2 level testing form the foundation of the quality assurance system. L1 level testing focuses on verifying the internal logic correctness of code units (e.g., functions, classes), ensuring each independent component behaves as designed.
-
-L2 level testing builds upon L1 by introducing GPU resources and verifying that the end-to-end (E2E) process of the model in basic deployment scenarios is smooth. For example, it uses dummy models to confirm that core interfaces like the inference pipeline, output format, and streaming response work properly. The common goal of these two levels is to provide developers with rapid feedback, discovering and fixing issues early in the development cycle  .
-
-
-
-### 1.2 Testing Content and Scope
-
--   ***L1 (Unit & Logic Testing)***:
--   -   ***Scope***: Tests internal functions and methods of core components such as `entrypoints`, `models`.
-    -   ***Focus***: Branch coverage, exception handling, algorithm logic correctness. Does not involve external dependencies or the complete service stack.
-    -   ***Time Cost***: Execution time is controlled within ***15 minutes*** to ensure fast feedback.
--   ***L2 (Basic End-to-End Testing)***:
--   -   ***Scope***: Covers two basic deployment scenarios: `online` (serving) and `offline` (inference).
-    -   ***Focus***: Uses `dummy` models or lightweight real models to verify that the entire chain from request input to result output works normally, including output data structure, streaming (stream) support, etc. Also includes some unit tests that require launching independent service instances.
-    -   ***Characteristic***: Requires ***GPU*** resources to perform model computations.
-
-### 1.3 Test Directory and Execution Files
-
-A clear directory structure is key to managing test cases efficiently.
-
--   ***L1 Test Directory***: `/tests/{component_name}/test_xxx.py`
--   -   Here, `{component_name}` corresponds to modules in the source code, such as `distributed`, `entrypoints`, etc., and `test_xxx.py` is the specific test file.
--   ***L2 Test Directory***:
--   -   Online Serving: `/tests/e2e/online_serving/test_{model_name}.py`
-    -   Offline Inference: `/tests/e2e/offline_inference/test_{model_name}.py`
-
-### 1.4 Execution Method and Example
-
--   ***Trigger Timing***: **`PR with ready label`**. That is, when a developer adds a "ready for review" or similar label to a PR on platforms like GitHub, L1 and L2 tests are automatically triggered.
--   ***Execution Environment***: L1 uses ***CPU*** environment; L2 requires ***GPU*** environment.
--   ***Script Example***:
-
-<details>
-<summary> L1 Test Examples</summary>
-
-Examples from `tests/model_executor/models/qwen2_5_omni/test_audio_length.py`
-```python
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-
-import pytest
-
-pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
-
-def test_resolve_max_mel_frames_default():
-    from vllm_omni.model_executor.models.qwen2_5_omni.audio_length import resolve_max_mel_frames
-
-    assert resolve_max_mel_frames(None, default=30000) == 30000
-    assert resolve_max_mel_frames(None, default=6000) == 6000
-
-
-def test_resolve_max_mel_frames_explicit():
-    from vllm_omni.model_executor.models.qwen2_5_omni.audio_length import resolve_max_mel_frames
-
-    # Explicit argument always wins over default
-    assert resolve_max_mel_frames(123, default=30000) == 123
-    assert resolve_max_mel_frames(6000, default=30000) == 6000
-    assert resolve_max_mel_frames(0, default=30000) == 0
-
-
-@pytest.mark.parametrize("repeats", [2, 4])
-@pytest.mark.parametrize("code_len", [0, 1, 32768])
-@pytest.mark.parametrize("max_mel_frames", [None, -1, 0, 1, 6000, 30000])
-def test_cap_and_align_mel_length_no_mismatch(repeats, code_len, max_mel_frames):
-    """Guard that any max_mel_frames yields a mel length aligned to repeats, and
-    consistent with the truncated code length (prevents concat mismatch).
-    """
-    from vllm_omni.model_executor.models.qwen2_5_omni.audio_length import cap_and_align_mel_length
-
-    target_code_len, target_mel_len = cap_and_align_mel_length(
-        code_len=code_len,
-        repeats=repeats,
-        max_mel_frames=max_mel_frames,
-    )
-
-    assert isinstance(target_code_len, int)
-    assert isinstance(target_mel_len, int)
-
-    if code_len == 0:
-        assert target_code_len == 0
-        assert target_mel_len == 0
-        return
-
-    assert target_code_len >= 1
-    assert target_mel_len >= repeats
-    assert target_mel_len % repeats == 0
-    assert target_mel_len == target_code_len * repeats
-    assert target_code_len <= code_len
-
-    if max_mel_frames is not None and int(max_mel_frames) > 0 and int(max_mel_frames) >= repeats:
-        assert target_mel_len <= int(max_mel_frames)
-```
-</details>
-
-<details>
-<summary> L2 Test Examples</summary>
-You can refer to Test Examples in Chapter 2 to see example test cases that incorporate both L2 and L3 testing logic.
-</details>
-
--   -   ***Run Command***:
-
-    `pytest -s -v /tests/e2e/online_serving/test_{model_name}.py`
-    `pytest -s -v -m 'core_model and cpu' --run-level=core_model`
-
-## Chapter 2: L3 Level Testing - Core Integration, Performance, and Accuracy Verification
-
-### 2.1 Testing Purpose
-
-L3 level testing executes after code is merged into the main branch. Its core purpose is to verify the integration effect, key performance indicators, and output accuracy of ***real models*** in ***multiple deployment scenarios***
-
-. It acts as the "quality gatekeeper" for the main branch, ensuring that no merge breaks the core capabilities of the model service. Testing needs to provide clear conclusions within a relatively short time (<30min), balancing test depth with feedback speed.
-
-
-
-### 2.2 Testing Content and Scope
-
--   ***Deployment Scenarios***: Covers richer `online` and `offline` deployment configurations, which may include different hardware configurations, batch sizes, concurrency levels, etc.
--   ***Core Verification***:
--   1.  ***Inference Functionality***: Ensures real models can perform forward computation normally and return results.
-    2.  ***Accuracy Compliance***: Verifies that the model's evaluation metrics (e.g., accuracy) meet the expected baseline, preventing code changes from introducing accuracy issues.
-    3.  ***Important Performance***: Verifies whether performance (e.g., P99 latency, throughput) in core scenarios meets preset thresholds.
-
-### 2.3 Test Directory and Execution Files
-
--   ***Functional Testing***:
--   -   Online Serving: `/tests/e2e/online_serving/test_{model_name}_expansion.py`
-    -   Offline Inference: `/tests/e2e/offline_inference/test_{model_name}_expansion.py`
-    -   (Note: `_expansion.py` likely means it contains more comprehensive scenario cases compared to L2 tests).
-
-### 2.4 Execution Method and Example
-
--   ***Trigger Timing***: **`PR Merged`**. Automatically triggered after code review is approved and merged into the main branch.
--   ***Execution Environment***: ***GPU*** servers.
--   ***Script Example***:
-
-???+ example "Test Examples"
-
-    **2.4.1 Mark Declaration Section**
-
-    ```python
-    @pytest.mark.advanced_model
-    @pytest.mark.core_model
-    @pytest.mark.parametrize("omni_server", test_params, indirect=True)
-    ```
-
-    **Explanation**:
-
-    @pytest.mark.advanced_model: Marks the test as L3 or L4 level, indicating that this test case performs deep validation, using real models for performance, integration, and accuracy testing. This forms a "basic-advanced" correspondence with the core_model mark at the L2 level.
-
-    @pytest.mark.core_model: Marks the test as L1 or L2 level, indicating that this test case validates the basic functionality of the core model. It uses mock weights and only checks if the relevant interface functions correctly.
-
-    @pytest.mark.parametrize: A parameterization decorator that allows abstracting test data into parameters, enabling reuse of the same test logic across different data configurations. indirect=True indicates that parameters will be passed to the fixture for processing.
-
-    **Notes**: If you believe the test case only needs to execute basic run logic at the PR-level CI, you can mark it only with @pytest.mark.core_model. If you believe it only needs to execute deep validation run logic at the merge or nightly level, you can mark it only with @pytest.mark.advanced_model. If you believe the test case needs to accommodate both basic run and deep validation test logic, you should mark it with both @pytest.mark.core_model and @pytest.mark.advanced_model.
-
-    **2.4.2 Test Function Definition and Documentation**
-
-    ```python
-    def test_mix_to_text_audio_001(omni_server, openai_client) -> None:
-        """
-        Test multi-modal input processing and text/audio output generation via OpenAI API.
-        Deploy Setting: default yaml
-        Input Modal: text + audio + video + image
-        Output Modal: text + audio
-        Input Setting: stream=True
-        Datasets: single request
-        """
-    ```
-
-    **Explanation**:
-
-    **Function Naming Convention**: Uses the test_ prefix, describes the test scenario mix_to_text_audio, and the number 001 indicates the first test case for this scenario.
-
-    **Parameter Explanation**:
-
-    omni_server: Omni server instance obtained via fixture, containing model information and configuration.
-
-    openai_client: Unified OpenAI client processing instance, encapsulating request sending and response validation logic.
-
-    Docstring: Describes the test purpose, deployment settings, input/output modalities, streaming settings, and dataset type in detail, providing clear context for test maintenance.
-
-    **2.4.3 Multimodal Data Preparation**
-
-    ```python
-    video_data_url = f"data:video/mp4;base64,{generate_synthetic_video(224, 224, 300)['base64']}"
-    image_data_url = f"data:image/jpeg;base64,{generate_synthetic_image(224, 224)['base64']}"
-    audio_data_url = f"data:audio/wav;base64,{generate_synthetic_audio(5, 1)['base64']}"
-    ```
-
-    **Explanation**:
-
-    **Data Generation Functions**: Use the generate_synthetic_* series of functions to generate synthetic test data, avoiding reliance on external resources and ensuring test reproducibility and stability.
-
-    **Parameter Explanation**:
-
-    Video: width, height, duration_frames
-
-    Image: width, height
-
-    Audio: duration_seconds, channels
-
-    **2.4.4 Request Configuration and Keyword Validation**
-
-    ```python
-    request_config = {
-        "model": omni_server.model,
-        "messages": messages,
-        "stream": True,
-        "key_words": {
-            "audio": ["water", "cricket"],
-            "video": ["sphere", "globe", "circle", "round"],
-            "image": ["square", "quadrate"],
-            "text": ["beijing"]
-        },
-    }
-    ```
-
-    **Explanation**:
-
-    **Model Specification**: Uses omni_server.model to ensure the test aligns with the model configured on the server.
-
-    **Keyword Validation Mechanism**: This is an innovative design of the template to address the specific needs of multimodal testing:
-
-    Audio Keywords: Validate whether the generated text's description of audio content contains expected elements (e.g., "water" for water sounds, "cricket" for cricket sounds). If you provide multiple keywords, the validation is considered successful if at least one keyword is present.
-
-    **Video Keywords**: Validate whether the generated text's description of video content contains expected elements. If you provide multiple keywords, the validation is considered successful if at least one keyword is present.
-
-    Image Keywords: Validate whether the generated text's description of image content contains expected elements. If you provide multiple keywords, the validation is considered successful if at least one keyword is present.
-
-    Text Keywords: Validate whether the generated text contains expected elements. If you provide multiple keywords, the validation is considered successful if at least one keyword is present.
-
-    **2.4.5 Request Execution**
-
-    ```python
-    openai_client.send_request(request_config, request_num=1)
-    ```
-
-    **Explanation**:
-
-    **Unified Client**: Uses the OpenAIClientHandler instance to send requests. This client encapsulates error handling, retry mechanisms, and response validation logic.
-
-    **Single Request**: The comment clearly states this is a single-request completion test. For concurrent testing, it can be extended to multiple requests using request_num = n.
-
-    **Implicit Validation**: The send_request method internally includes validation logic dynamically selected based on the --run-level parameter: core_model performs basic validation, while advanced_model performs deep validation.
-
--   ***Run Command***: `pytest -s -v /tests/e2e/online_serving/test_{model_name}.py -m advanced_model --run-level=advanced_model`
-
-## Chapter 3: L4 Level Testing - Full Functionality, Performance, and Documentation Testing
-
-### 3.1 Testing Purpose
-
-L4 level testing is a comprehensive quality audit before a version release. It expands upon L3, executing ***full*** functional scenarios, conducting systematic ***performance stress tests***, and simultaneously verifying the correctness of accompanying ***example documentation***. Its purpose is to perform deep validation of the system during off-peak nighttime hours, providing quality trend reports for daytime development and data support for release decisions.
-
-
-
-### 3.2 Testing Content and Scope
-
--   ***Full Functionality Testing***: Executes all test cases defined in `test_{model_name}_expansion.py`, covering all implemented features, positive flows, boundary conditions, and exception handling.
--   ***Performance Testing***: Uses the `/tests/e2e/perf/nightly.json` configuration file to drive performance testing tools for stress, load, and endurance tests, collecting metrics like throughput, response time, and resource utilization.
--   ***Documentation Testing***: Verifies whether the example code provided to users is runnable and its results match the description.
-
-### 3.3 Test Directory and Execution Files
-
--   ***Functional Testing***: Same directories as L3.
--   ***Performance Test Configuration***: `/tests/e2e/perf/nightly.json`
--   ***Documentation Example Tests***:
--   -   `tests/example/online_serving/test_{model_name}.py`
-    -   `tests/example/offline_inference/test_{model_name}.py`
-
-### 3.4 Execution Method and Example
-
--   ***Trigger Timing***: **`Nightly`**, automatically executed every night.
--   ***Execution Environment***: ***GPU*** server clusters to meet the resource demands of performance testing.
--   ***Script Example***:
-<details>
-<summary> Test Examples</summary>
-When you want to add L4-level performance test cases, you can refer to the following format for case addition in tests/perf/tests/test.json:
-
-```JSON
-{
-    "test_name": "test_qwen3_omni",
-    "server_params": {
-        "model": "Qwen/Qwen3-Omni-30B-A3B-Instruct",
-        "stage_config_name": "qwen3_omni.yaml"
-    },
-    "benchmark_params": [
-        {
-            "dataset_name": "random",
-            "num_prompts": [10, 20],
-            "request_rate": [0.5, 1],
-            "random_input_len": 2500,
-            "random_output_len": 900,
-            "ignore_eos": true,
-            "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
-            "baseline": {
-                "mean_ttft_ms": 100000,
-                "mean_audio_ttfp_ms": 100000,
-                "mean_audio_rtf": 100000
-            }
-        }
-    ]
-}
-```
-
-#### Parameter Explanation
-
-***Overview***
-
-| Field            | Required | Description                                                     |
-| ---------------- | -------- | --------------------------------------------------------------- |
-| test_name        | Yes      | Unique identifier for the test case                             |
-| server_params    | Yes      | Server-side configuration parameters                            |
-| benchmark_params | Yes      | Benchmark running parameters (supports multiple configurations) |
-
-#### server_params Configuration
-
-##### Basic Parameters
-
-| Parameter         | Required | Example                            | Description                   |
-| ----------------- | -------- | ---------------------------------- | ----------------------------- |
-| model             | Yes      | "Qwen/Qwen3-Omni-30B-A3B-Instruct" | Model name or path            |
-| stage_config_name | Yes      | "qwen3_omni.yaml"                  | Stage configuration file name |
-
-##### Dynamic Configuration (update/delete)
-
-Supports incremental modifications based on the basic configuration:
-
-| Operation | Description                          |
-| --------- | ------------------------------------ |
-| update    | Update or add configuration items    |
-| delete    | Delete specified configuration items |
-
-***Example***:
-```
-"update": {
-    "async_chunk": true,  // Enable asynchronous chunk processing
-    "stage_args": {
-        "0": {
-            "engine_args.custom_process_next_stage_input_func": "vllm_omni.model_executor.stage_input_processors.qwen3_omni.thinker2talker_async_chunk"
-        }
-    }
-},
-"delete": {
-    "stage_args": {
-        "2": ["custom_process_input_func"]  // Delete this configuration for stage 2
-    }
-}
-```
-#### benchmark_params Configuration
-
-You can add any benchmark running parameters you need here. For all optional parameters, refer to the [benchmark documentation](https://github.com/vllm-project/vllm-omni/blob/main/docs/cli/bench/serve.md). General modifications are as follows:
-
-1.  Change the ---xxx-xx-xx running parameters to xxx_xx_xx format and fill them as keys in the JSON file.
-2.  For boolean variables in the running parameters, modify them to forms such as ignore_eos: true/false and fill them into the JSON file.
-3.  Add the baseline parameter to specify the required validation values, ensuring the validation metric names match those in the result.json generated by the benchmark.
-4.  The qps and concurrency modes are mutually exclusive. For detailed explanations, see the table below:
-
-| Parameter       | Type        | Required | Example/Values  | Description                                                                                                                                                                                                                                                          |
-| --------------- | ----------- | -------- | --------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| num_prompts     | int / array | Yes      | 10,[10, 20, 30] | Number of requests. Supports single values or arrays. If a single value is used, it will be automatically expanded to match the number of qps or max_concurrency, e.g., [10,10,10]. If an array is used, its length must match the number of qps or max_concurrency. |
-| request_rate    | int / array | No       | 1, [1, 2, 3]    | Queries per second. Supports single values or arrays. If a single value is used, it will be automatically expanded to match the number of num_prompts, e.g., [1,1,1]. If an array is used, its length must match the number of num_prompts.                          |
-| max_concurrency | int / array | No       | 1, [1, 2, 3]    | Queries per second. Supports single values or arrays. If a single value is used, it will be automatically expanded to match the number of num_prompts, e.g., [1,1,1]. If an array is used, its length must match the number of num_prompts.                          |
-</details>
-
--   -   ***Run Command***: (Specific commands would depend on the performance testing tool and configuration defined in `nightly.json`).
-
-## Chapter 4: L5 Level Testing - Stability and Reliability Testing
-
-### 4.1 Testing Purpose
-
-L5 level testing focuses on the performance of model services under ***long-running*** and ***abnormal fault*** scenarios. It aims to uncover deep-seated issues that only manifest under sustained pressure or extreme conditions, such as memory leaks, resource contention, gradual performance degradation, and lack of fault tolerance mechanisms. This is the final, yet crucial, line of defense for ensuring service high availability and production environment robustness.
-
-
-
-### 4.2 Testing Content and Scope
-
--   ***Long-term Stability (Stability) Testing***: Uses the `tests/e2e/stability/weekly.json` configuration to run the service under moderate load for an extended period (e.g., over 12 hours), monitoring whether metrics like memory/VRAM usage, response time, and throughput degrade over time, and whether the service process remains stable.
--   ***Reliability Testing***: Uses `tests/e2e/reliability/test_{model_name}.py` to actively simulate various fault and abnormal scenarios, such as: dependent service interruption, abnormal input data, network flicker, hardware resource preemption, etc., to verify the system's fault tolerance, self-healing, and graceful degradation capabilities.
-
-### 4.3 Test Directory and Execution Files
-
--   ***Stability Test Configuration***: `tests/e2e/stability/weekly.json`
--   ***Reliability Test Suite***: `tests/e2e/reliability/test_{model_name}.py`
-
-### 4.4 Execution Method and Example
-
--   ***Trigger Timing***: **`Weekly`** (weekly) or **`Days before Release`** (several days before a major release). Due to long execution times, the frequency is lower.
--   ***Execution Environment***: ***GPU*** servers, requiring a stable and exclusive testing environment.
--   ***Script Example***:
-<details>
-<summary> Test Examples</summary>
-```python
-# WIP
-```
-</details>
-
--   -   ***Stability***: (Execution would be driven by the configuration in `weekly.json`).
-    -   ***Reliability***: `pytest -s -v tests/e2e/reliability/test_{model_name}.py`
-
-## Summary
-
-This multi-level testing system achieves continuous, progressive validation of model service quality by tightly integrating testing activities with the development workflow (commit, review, merge, release). From rapid unit testing to comprehensive end-to-end testing, and further to in-depth performance, stability, and reliability verification, each level has clear objectives, collectively building a robust quality protection net. By following this system, teams can deliver high-quality, highly reliable model services more efficiently.

docs/contributing/ci/failures.md

@@ -1,10 +1,4 @@
 # CI Failures
 
-What should I do when a CI job fails on my PR, but I don't think my PR caused the failure?
-
-
-## Common Case 1:
-
-🚨 Error: The command was interrupted by a signal: signal: terminated
-
-Reason: The test is terminated due to exceed the time limits. Sometimes rebuild the test will pass.
+What should I do when a CI job fails on my PR, but I don't think my PR caused
+the failure?

docs/contributing/ci/tests_markers.md

@@ -1,157 +0,0 @@
-# Markers for Tests
-
-By adding markers before test functions, tests can later be executed uniformly by simply declaring the corresponding marker type.
-
-## Current Markers
-Defined in `pyproject.toml`:
-
-| Marker             | Description                                               |
-| ------------------ | --------------------------------------------------------- |
-| `core_model`       | L1&L2 tests (run in each PR)                              |
-| `advanced_model`   | L3&L4 level tests (run in each merge or nightly)          |
-| `diffusion`        | Diffusion model tests                                     |
-| `omni`             | Omni model tests                                          |
-| `cache`            | Cache backend tests                                       |
-| `parallel`         | Parallelism/distributed tests                             |
-| `cpu`              | Tests that run on CPU                                     |
-| `gpu`              | Tests that run on GPU *                                   |
-| `cuda`             | Tests that run on CUDA *                                  |
-| `rocm`             | Tests that run on AMD/ROCm *                              |
-| `npu`              | Tests that run on NPU/Ascend *                            |
-| `H100`             | Tests that require H100 GPU  *                            |
-| `L4`               | Tests that require L4 GPU *                               |
-| `MI325`            | Tests that require MI325 GPU (AMD/ROCm) *                 |
-| `A2`               | Tests that require A2 NPU *                               |
-| `A3`               | Tests that require A3 NPU *                               |
-| `distributed_cuda` | Tests that require multi cards on CUDA platform *         |
-| `distributed_rocm` | Tests that require multi cards on ROCm platform  *        |
-| `distributed_npu`  | Tests that require multi cards on NPU platform  *         |
-| `skipif_cuda`      | Skip if the num of CUDA cards is less than the required * |
-| `skipif_rocm`      | Skip if the num of ROCm cards is less than the required * |
-| `skipif_npu`       | Skip if the num of NPU cards is less than the required *  |
-| `slow`             | Slow tests (may skip in quick CI)                         |
-| `benchmark`        | Benchmark tests                                           |
-
-\* Means those markers are auto-added, and they will be added by the `@hardware_test` decorator.
-
-### Example usage for markers
-
-```python
-from tests.utils import hardware_test
-
-@pytest.mark.core_model
-@pytest.mark.omni
-@hardware_test(
-   res={"cuda": "L4", "rocm": "MI325", "npu": "A2"},
-   num_cards=2,
-)
-@pytest.mark.parametrize("omni_server", test_params, indirect=True)
-def test_video_to_audio()
-    ...
-```
-### Decorator: `@hardware_test`
-
-This decorator is intended to make hardware-aware, cross-platform test authoring easier and more robust for CI/CD environments. The `hardware_test` decorator in `vllm-omni/tests/utils.py` performs the following actions:
-
-1. **Applies platform and resource markers**  
-   Adds the appropriate pytest markers for each specified hardware platform (e.g., `cuda`, `rocm`, `npu`) and resource type (e.g., `L4`, `H100`, `MI325`, `A2`, `A3`).
-   ```
-   @pytest.mark.cuda
-   @pytest.mark.L4
-   ```
-2. **Handles multi-card (distributed) scenarios**  
-   For tests requiring multiple cards, it automatically adds distributed markers such as `distributed_cuda`, `distributed_rocm`, or `distributed_npu`.
-   ```
-   @pytest.mark.distributed_cuda(num_cards=num_cards)
-   ```
-3. **Supports flexible card requirements**  
-   Accepts `num_cards` as either a single integer for all platforms or as a dictionary with per-platform values. If not specified, defaults to 1 card per platform.
-
-4. **Integrates resource validation**  
-   On CUDA, adds a skip marker (`skipif_cuda`) if the system does not have the required number of devices.
-   Support for `skipif_rocm` and `skipif_npu` will be implemented later.
-
-
-5. **Works with pytest filtering**  
-   Allows tests to be filtered and selected at runtime using standard pytest marker expressions (e.g., `-m "distributed_cuda and L4"`).
-
-#### Example usage for decorator
-- Single call for multiple platforms:
-    ```python
-    @hardware_test(
-        res={"cuda": "L4", "rocm": "MI325", "npu": "A2"},
-        num_cards={"cuda": 2, "rocm": 2, "npu": 2},
-    )
-    ```
-    or
-    ```python
-    @hardware_test(
-        res={"cuda": "L4", "rocm": "MI325", "npu": "A2"},
-        num_cards=2,
-    )
-    ```
-- `res` must be a dict; supported resources: CUDA (L4/H100), ROCm (MI325), NPU (A2/A3)
-- `num_cards` can be int (all platforms) or dict (per platform); defaults to 1 when missing
-- Distributed markers (`distributed_cuda`, `distributed_rocm`, `distributed_npu`) are auto-added for multi-card cases
-- Filtering examples:
-    - CUDA only: `pytest -m "distributed_cuda and L4"`
-    - ROCm only: `pytest -m "distributed_rocm and MI325"`
-    - NPU only: `pytest -m "distributed_npu"`
-
-## Add Support for a New Platform
-
-If you want to add support for a new platform (e.g., "tpu" for a new accelerator), follow these steps:
-
-1. **Extend the marker list in your pytest config** so that platform/resource markers are defined:
-   ```toml
-   # In pyproject.toml or pytest.ini
-   [tool.pytest.ini_options]
-   markers = [
-       # ... existing markers ...
-       "tpu: Tests that require TPU device",
-       "TPU_V3: Tests that require TPU v3 hardware",
-       "distributed_tpu: Tests that require multiple TPU devices",
-   ]
-   ```
-2. **Implement a marker construction function for your platform** in `vllm-omni/tests/utils.py`:
-   ```python
-   # In vllm-omni/tests/utils.py
-
-   def tpu_marks(*, res: str, num_cards: int):
-       test_platform = pytest.mark.tpu
-       if res == "TPU_V3":
-           test_resource = pytest.mark.TPU_V3
-       else:
-           raise ValueError(
-               f"Invalid TPU resource type: {res}. Supported: TPU_V3")
-
-       if num_cards == 1:
-           return [test_platform, test_resource]
-       else:
-           test_distributed = pytest.mark.distributed_tpu(num_cards=num_cards)
-           # Optionally: add skipif_tpu when implemented
-           return [test_platform, test_resource, test_distributed]
-   ```
-3. **Update `hardware_test` to recognize your new platform**:
-    In the relevant place (see the `hardware_test` implementation), add:
-    ```python
-    if platform == "tpu":
-        marks = tpu_marks(res=resource, num_cards=cards)
-    ```
-4. **(Recommended) Add a test using your new markers**:
-   ```python
-   @hardware_test(
-       res={"tpu": "TPU_V3"},
-       num_cards=2,
-   )
-   def test_my_tpu_feature():
-       ...
-   ```
-
-**Summary**:  
-- Add pytest markers for your new platform/resources  
-- Implement a marker function (`xxx_marks`)  
-- Plug into `hardware_test`  
-- You're done: tests decorated with `@hardware_test` using your platform now automatically get the correct markers, distribution, and isolation!
-
-See code in `vllm-omni/tests/utils.py` for existing examples (`cuda_marks`, `rocm_marks`, `npu_marks`).

docs/contributing/ci/tests_style.md

@@ -1,467 +0,0 @@
-# Test File Structure and Style Guide
-
-To ensure project maintainability and sustainable development, we encourage contributors to submit test code (unit tests, system tests, or end-to-end tests) alongside their code changes. This document outlines the guidelines for organizing and naming test files.
-
-## Checklist before submitting your test files
-
-1. The file is saved in an appropriate place and the file name is clear.
-2. The coding style follows the requirements outlined below.
-3. All test functions have appropriate pytest markers.
-4. For tests that need run in CI, please ensure it labeled as ``@pytest.mark.core_model` the test is configured under the `./buildkite/` folder.
-
-
-## Test Types
-
-For more details about our [Five Levels Tests design](../ci/CI_5levels.md).
-
-### Unit Tests and System Tests
-For unit tests and system tests, we strongly recommend placing test files in the same directory structure as the source code being tested, using the naming convention `test_*.py`.
-
-### End-to-End (E2E) Tests for Models
-End-to-end tests verify the complete functionality of a system or component. For our project, the E2E tests for different omni models are organized into two subdirectories:
-
-- **`tests/e2e/offline_inference/`**: Tests for offline inference modes (e.g., Qwen3Omni offline inference)
-
-- **`tests/e2e/online_serving/`**: Tests for online serving scenarios (e.g., API server tests)
-
-**Example:** The test file for `vllm_omni/entrypoints/omni_llm.py` should be located at `tests/entrypoints/test_omni_llm.py`.
-
-## Test Directory Structure
-
-The ideal directory structure mirrors the source code organization. Legend: `✅` = test exists, `⬜` = suggested to add.
-
-```
-vllm_omni/                                    tests/
-├── config/                             →     ├── config/
-│   ├── model.py                              │   └── test_model.py                    ⬜
-│   └── lora.py                               │   └── test_lora.py                      ⬜
-│
-├── core/                               →     ├── core/
-│   └── sched/                                 │   └── sched/
-│       ├── omni_ar_scheduler.py               │       ├── test_omni_ar_scheduler.py    ⬜
-│       ├── omni_generation_scheduler.py       │       ├── test_omni_generation_scheduler.py  ⬜
-│       └── output.py                          │       └── test_output.py               ✅ currently in entrypoints/test_omni_new_request_data.py (tests output.OmniNewRequestData)
-│
-├── diffusion/                          →     ├── diffusion/
-│   ├── diffusion_engine.py                    │   ├── test_diffusion_engine.py          ⬜
-│   ├── attention/                             │   ├── attention/
-│   │   ├── layer.py                            │   │   ├── test_attention_sp.py         ✅
-│   │   └── backends/                           │   │   └── test_flash_attn.py           ✅
-│   ├── distributed/                           │   ├── distributed/
-│   │   └── ...                                 │   │   ├── test_comm.py                 ✅
-│   │                                            │   │   ├── test_cfg_parallel.py        ✅
-│   │                                            │   │   └── test_sp_plan_hooks.py       ✅
-│   ├── lora/                                   │   ├── lora/
-│   │   └── ...                                 │   │   ├── test_base_linear.py          ✅
-│   │                                            │   │   └── test_lora_manager.py        ✅
-│   ├── models/                                 │   ├── models/
-│   │   ├── qwen_image/                         │   │   ├── qwen_image/                 (e2e coverage)
-│   │   ├── z_image/                            │   │   └── z_image/
-│   │   └── ...                                 │   │       └── test_zimage_tp_constraints.py  ✅
-│   └── worker/                                 │   └── worker/
-│       ├── diffusion_worker.py                 │       └── test_diffusion_worker.py   ✅ file at diffusion/test_diffusion_worker.py
-│       └── diffusion_model_runner.py            │
-│
-├── distributed/                         →     ├── distributed/
-│   └── omni_connectors/                         │   └── omni_connectors/
-│       ├── adapter.py                           │       ├── test_adapter_and_flow.py   ✅
-│       ├── kv_transfer_manager.py               │       ├── test_basic_connectors.py   ✅
-│       ├── connectors/                           │       ├── test_kv_flow.py             ✅
-│       └── utils/                               │       └── test_omni_connector_configs.py  ✅
-│
-├── engine/                             →     ├── engine/
-│   ├── input_processor.py                      │   ├── test_input_processor.py         ⬜  (no processor.py in source)
-│   ├── output_processor.py                     │   └── test_output_processor.py         ⬜
-│   └── arg_utils.py                            │   └── test_arg_utils.py               ⬜
-│
-├── entrypoints/                        →     ├── entrypoints/
-│   ├── omni.py                                 │   ├── test_omni.py                    ⬜  (E2E covered by e2e/offline, e2e/online)
-│   ├── omni_llm.py                             │   ├── test_omni_llm.py                ✅
-│   ├── omni_stage.py                            │   ├── test_omni_stage.py              ⬜  (partial in test_omni_stage_diffusion_config.py)
-│   ├── omni_diffusion.py                       │   ├── test_omni_diffusion.py          ✅
-│   ├── async_omni.py                            │   ├── test_async_omni.py              ✅ actually in e2e/online_serving/test_async_omni.py
-│   ├── async_omni_diffusion.py                 │   ├── test_async_omni_diffusion_config.py  ✅
-│   ├── stage_utils.py                          │   ├── test_stage_utils.py            ✅
-│   ├── cli/                                     │   ├── cli/                           (benchmarks/test_serve_cli.py covers CLI serve)
-│   │   └── ...                                  │   │   └── test_*.py                  ⬜
-│   └── openai/                                  │   └── openai_api/                    # maps to entrypoints/openai/
-│       ├── api_server.py                        │       ├── test_api_server.py         ⬜  (e2e indirect coverage)
-│       ├── serving_chat.py                       │       ├── test_serving_chat_sampling_params.py  ✅
-│       ├── serving_speech.py                     │       ├── test_serving_speech.py     ✅
-│       └── image_api_utils.py                   │       └── test_image_server.py      ✅
-│
-├── inputs/                             →     ├── inputs/
-│   ├── data.py                                 │   ├── test_data.py                   ⬜
-│   ├── parse.py                                │   ├── test_parse.py                 ⬜
-│   └── preprocess.py                            │   └── test_preprocess.py            ✅ currently in entrypoints/test_omni_input_preprocessor.py
-│
-├── model_executor/                     →     ├── model_executor/
-│   ├── layers/                                  │   ├── layers/
-│   │   └── mrope.py                             │   │   └── test_mrope.py              ⬜
-│   ├── model_loader/                            │   ├── model_loader/
-│   │   └── weight_utils.py                      │   │   └── test_weight_utils.py      ⬜
-│   ├── models/                                  │   ├── models/
-│   │   ├── qwen2_5_omni/                         │   │   ├── qwen2_5_omni/
-│   │   │   ├── qwen2_5_omni_thinker.py           │   │   │   ├── test_audio_length.py  ✅
-│   │   │   ├── qwen2_5_omni_talker.py            │   │   │   ├── test_qwen2_5_omni_thinker.py  ⬜
-│   │   │   └── qwen2_5_omni_token2wav.py         │   │   │   ├── test_qwen2_5_omni_talker.py  ⬜
-│   │   └── qwen3_omni/                          │   │   │   └── test_qwen2_5_omni_token2wav.py  ⬜
-│   │       └── ...                               │   │   └── qwen3_omni/
-│   ├── stage_configs/                           │   │       └── test_*.py              ⬜
-│   │   └── *.yaml                               │   └── stage_configs/                 (used by e2e, test_*.py can be added)  ⬜
-│   └── stage_input_processors/                  │   └── stage_input_processors/
-│       └── ...                                  │       └── test_*.py                 ⬜
-│
-├── sample/                             →     ├── sample/
-│   └── __init__.py                             │   └── test_*.py                      ⬜
-│
-├── utils/                              →     ├── utils/
-│   └── __init__.py                             │   └── test_*.py                       ⬜  (no platform_utils.py currently)
-│
-├── worker/                             →     ├── worker/
-│   ├── gpu_ar_model_runner.py                  │   ├── test_gpu_ar_model_runner.py    ⬜
-│   ├── gpu_ar_worker.py                        │   ├── test_gpu_ar_worker.py           ⬜
-│   ├── gpu_generation_model_runner.py          │   ├── test_gpu_generation_model_runner.py  ✅
-│   ├── gpu_generation_worker.py                │   ├── test_gpu_generation_worker.py  ⬜
-│   ├── gpu_model_runner.py                     │   ├── test_omni_gpu_model_runner.py   ✅
-│   └── mixins.py                               │   └── (npu under platforms/npu/worker/)  # not worker/npu/
-│
-├── platforms/                          →     (no tests/platforms/, e2e and stage_configs provide indirect coverage)
-│   ├── cuda/
-│   ├── npu/worker/                             # NPU worker here, not vllm_omni/worker/npu/
-│   ├── rocm/
-│   └── xpu/worker/
-│
-├── outputs.py                          →     test_outputs.py                         ✅ (at tests root)
-├── (logger, patch, request, version)    →     (no corresponding unit test)
-│
-└── e2e (tests side only)               →     ├── e2e/
-                                               ├── online_serving/                     ✅ non-empty
-                                               │   ├── test_async_omni.py
-                                               │   ├── test_qwen3_omni.py
-                                               │   ├── test_qwen3_omni_expansion.py
-                                               │   ├── test_image_gen_edit.py
-                                               │   ├── test_images_generations_lora.py
-                                               │   └── stage_configs/
-                                               └── offline_inference/                  ✅
-                                                   ├── test_qwen2_5_omni.py
-                                                   ├── test_qwen3_omni.py
-                                                   ├── test_bagel_text2img.py
-                                                   ├── test_t2i_model.py
-                                                   ├── test_t2v_model.py
-                                                   ├── test_ovis_image.py
-                                                   ├── test_zimage_tensor_parallel.py
-                                                   ├── test_cache_dit.py
-                                                   ├── test_teacache.py
-                                                   ├── test_stable_audio_model.py
-                                                   ├── test_diffusion_cpu_offload.py
-                                                   ├── test_diffusion_layerwise_offload.py
-                                                   ├── test_diffusion_lora.py
-                                                   ├── test_sequence_parallel.py
-                                                   └── stage_configs/
-                                                       ├── qwen2_5_omni_ci.yaml
-                                                       ├── qwen3_omni_ci.yaml
-                                                       ├── bagel_*.yaml
-                                                       └── npu/, rocm/, etc.
-```
-
-
-
-### Naming Conventions
-
-- **Unit Tests**: Use `test_<module_name>.py` format. Example: `omni_llm.py` → `test_omni_llm.py`
-
-- **E2E Tests**: Place in `tests/e2e/offline_inference/` or `tests/e2e/online_serving/` with descriptive names. Example: `tests/e2e/offline_inference/test_qwen3_omni.py`, `tests/e2e/offline_inference/test_diffusion_model.py`
-
-- **Expansion Tests**
-
-### Best Practices
-
-1. **Mirror Source Structure**: Test directories should mirror the source code structure
-2. **Test Type Indicators**: Use comments to indicate test types (UT for unit tests, E2E for end-to-end tests)
-3. **Shared Resources**: Place shared test configurations (e.g., CI configs) in appropriate subdirectories
-4. **Consistent Naming**: Follow the `test_*.py` naming convention consistently across all test files
-
-
-## Test codes requirements
-
-### Coding style
-
-1. **File header**: Add SPDX license header to all test files
-2. **Imports**: Pls don't use manual `sys.path` modifications, use standard imports instead.
-3. **Test type differentiation**:
-
-      - Unit tests: Maintain mock style
-      - E2E tests for models: Consider using OmniRunner uniformly, avoid decorators
-
-4. **Documentation**: Add docstrings to all test functions
-5. **Environment variables**: Set uniformly in `conftest.py` or at the top of files
-6. **Type annotations**: Add type annotations to all test function parameters
-7. **Pytest Markers**: Add necessary markers like `@pytest.mark.core_model` and use `@hardware_test` to declare hardware requirements (check detailed in [Markers for Tests](../ci/tests_markers.md)).
-
-### Template
-#### E2E - Online serving
-
-E2E Online tests for Qwen3-Omni model with mix input and audio+text output. Based on `tests/e2e/online_serving/test_qwen3_omni.py`.
-
-```python
-"""
-E2E Online tests for Qwen3-Omni model with mix input and audio+text output.
-"""
-
-import os
-
-os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
-os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
-
-import threading
-from pathlib import Path
-
-import openai
-import pytest
-
-from tests.conftest import (
-    OmniServer,
-    convert_audio_to_text,
-    cosine_similarity_text,
-    dummy_messages_from_mix_data,
-    generate_synthetic_video,
-    merge_base64_and_convert_to_text,
-)
-from vllm_omni.platforms import current_omni_platform
-
-# Edit: model name and stage config path
-models = ["Qwen/Qwen3-Omni-30B-A3B-Instruct"]
-
-#If you use the default configuration file, you can directly use the following address.
-def get_default_config():
-    return str(Path(__file__).parent.parent / "stage_configs" / "qwen3_omni_ci.yaml")
-
-#If you need to modify the configuration file, you can use modify_stage_config.
-def get_chunk_config():
-    path = modify_stage_config(
-        get_default_config(),
-        updates={
-            "async_chunk": True,
-            "stage_args": {
-                0: {
-                    "engine_args.custom_process_next_stage_input_func": "vllm_omni.model_executor.stage_input_processors.qwen3_omni.thinker2talker_async_chunk"
-                },
-                1: {
-                    "engine_args.custom_process_next_stage_input_func": "vllm_omni.model_executor.stage_input_processors.qwen3_omni.talker2code2wav_async_chunk"
-                },
-            },
-        },
-        deletes={"stage_args": {2: ["custom_process_input_func"]}},
-    )
-    return path
-
-stage_configs = [get_default_config(), CHUNK_CONFIG_PATH]
-
-test_params = [(model, stage_config) for model in models for stage_config in stage_configs]
-
-
-#Please use this method to launch the online instance.
-_omni_server_lock = threading.Lock()
-
-@pytest.fixture(scope="module")
-def omni_server(request):
-    """Start vLLM-Omni server as a subprocess. Use module scope for multi-stage init (10-20+ min)."""
-    with _omni_server_lock:
-        model, stage_config_path = request.param
-        with OmniServer(
-            model,
-            ["--stage-configs-path", stage_config_path, "--stage-init-timeout", "120"],
-        ) as server:
-            yield server
-
-
-@pytest.fixture
-def client(omni_server):
-    """OpenAI client for the running vLLM-Omni server."""
-    return openai.OpenAI(
-        base_url=f"http://{omni_server.host}:{omni_server.port}/v1",
-        api_key="EMPTY",
-    )
-
-#Please use function definitions above the test function to define the prompts and other parameters you need.
-def get_system_prompt():
-    return {
-        "role": "system",
-        "content": [
-            {
-                "type": "text",
-                "text": (
-                    "You are Qwen, a virtual human developed by the Qwen Team, "
-                    "Alibaba Group, capable of perceiving auditory and visual inputs, "
-                    "as well as generating text and speech."
-                ),
-            }
-        ],
-    }
-
-...
-
-#Please define test case tags according to the instructions in the marker documentation.
-@pytest.mark.core_model
-@pytest.mark.omni
-@pytest.mark.parametrize("omni_server", test_params, indirect=True)
-def test_mix_to_text_audio_001(client: openai.OpenAI, omni_server, request) -> None:
-    # PLEASE FOLLOW THESE TEMPLATE INSTRUCTIONS:
-    # ============================================================================
-    # TEMPLATE USAGE GUIDE:
-    # 1. Copy this entire function as a starting point for multi-modal tests
-    # 2. Update the test name to reflect your specific test scenario
-    # 3. Modify input/output modalities as needed (see OPTIONS section below)
-    # 4. Adjust assertions based on your expected outcomes
-    # 5. Add custom validation logic for your specific use case
-    # ============================================================================
-
-    #Please list the relevant test points.
-    """
-    Test multi-modal input processing and text/audio output generation via OpenAI API.
-    Deploy Setting: default yaml
-    Input Modal: text + audio + video + image
-    Output Modal: text + audio
-    Input Setting: stream=True
-    Datasets: single request
-    """
-    # SECTION 1: TEST SETUP AND INITIALIZATION
-    # =========================================
-    # INSTRUCTIONS: Initialize test variables and prepare test environment
-    # MODIFY: Add any additional test setup required for your scenario
-    e2e_list = list()
-    # SECTION 2: TEST DATA GENERATION
-    # ================================
-    # INSTRUCTIONS: Generate or load test data for each input modality
-    # MODIFY: Replace synthetic generators with your actual data sources
-    # VIDEO DATA - Generate synthetic video for testing
-    # FORMAT: data:video/mp4;base64,{base64_encoded_video}
-    # PARAMETERS: width, height, duration_frames
-    video_data_url = f"data:video/mp4;base64,{generate_synthetic_video(224, 224, 300)['base64']}"
-    # IMAGE DATA - Generate synthetic image for testing
-    # FORMAT: data:image/jpeg;base64,{base64_encoded_image}
-    # PARAMETERS: width, height
-    image_data_url = f"data:image/jpeg;base64,{generate_synthetic_image(224, 224)['base64']}"
-    # AUDIO DATA - Generate synthetic audio for testing
-    # FORMAT: data:audio/wav;base64,{base64_encoded_audio}
-    # PARAMETERS: duration_seconds, channels
-    audio_data_url = f"data:audio/wav;base64,{generate_synthetic_audio(5, 1)['base64']}"
-
-    # SECTION 3: MESSAGE CONSTRUCTION
-    # ================================
-    # INSTRUCTIONS: Assemble the complete message payload for API request
-    # MODIFY: Add/remove modalities or change prompt structure as needed
-
-    # USAGE: Construct a message containing all input modalities
-    # IMPORTANT: Ensure the message structure matches OpenAI API expectations
-    # CUSTOMIZATION POINTS:
-    #   - system_prompt: Controls the assistant's behavior
-    #   - content_text: The user's text prompt/question
-    #   - *_data_url: URLs for media content (video/image/audio)
-    messages = dummy_messages_from_mix_data(
-        system_prompt=get_system_prompt(),
-        video_data_url=video_data_url,
-        image_data_url=image_data_url,
-        audio_data_url=audio_data_url,
-        content_text=get_prompt("mix"),
-    )
-
-    # SECTION 4: API REQUEST EXECUTION
-    # =================================
-    # INSTRUCTIONS: Make the API call and measure performance
-    # MODIFY: Add timeout, retry logic, or additional parameters
-    start_time = time.perf_counter()
-    chat_completion = client.chat.completions.create(model=omni_server.model, messages=messages, stream=True)
-
-    #Call using your preferred method and obtain the final audio and text outputs.
-    ...
-
-    # SECTION 5: OUTPUT VALIDATION
-    # =============================
-    # INSTRUCTIONS: Verify that outputs meet expected criteria
-    # MODIFY: Adjust validation logic for your specific requirements
-
-    # ASSERTION 1: E2E Validation
-    # PURPOSE: Verify that the E2E latency is less than the baseline.
-    current_e2e = time.perf_counter() - start_time
-    print(f"the request e2e is: {current_e2e}")
-    e2e_list.append(current_e2e)
-
-    print(f"the avg e2e is: {sum(e2e_list) / len(e2e_list)}")
-
-
-
-    # ASSERTION 2: Text Output Validation
-    # PURPOSE: Verify that text output was generated with keyword content
-    assert text_content is not None and len(text_content) >= 2, "No text output is generated"
-    assert any(
-        keyword in text_content.lower() for keyword in ["square", "quadrate", "sphere", "globe", "circle", "round"]
-    ), "The output does not contain any of the keywords."
-
-
-    # ASSERTION 3: Cross-Modal Consistency
-    # PURPOSE: Verify text and audio outputs convey the same information
-    # CUSTOMIZATION: Adjust similarity threshold (0.9) based on accuracy requirements
-    assert audio_data is not None, "No audio output is generated"
-    audio_content = merge_base64_and_convert_to_text(audio_data)
-    print(f"text content is: {text_content}")
-    print(f"audio content is: {audio_content}")
-    similarity = cosine_similarity_text(audio_content.lower(), text_content.lower())
-    print(f"similarity is: {similarity}")
-    assert similarity > 0.9, "The audio content is not same as the text"
-```
-
-
-#### E2E - Offline inference
-```python
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-"""
-Offline E2E smoke test for an omni model (video → audio).
-"""
-
-import os
-from pathlib import Path
-
-import pytest
-from vllm.assets.video import VideoAsset
-
-from tests.utils import hardware_test
-from ..multi_stages.conftest import OmniRunner
-
-# Optional: set process start method for workers
-os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
-
-models = ["{your model name}"] #Edit here to load your model
-stage_configs = [str(Path(__file__).parent / "stage_configs" / {your model yaml})] #Edit here to load your model yaml
-
-# Create parameter combinations for model and stage config
-test_params = [(model, stage_config) for model in models for stage_config in stage_configs]
-
-# function name: test_{input_modality}_to_{output_modality}
-# modality candidate: text, image, audio, video, mixed_modalities
-@pytest.mark.core_model
-@pytest.mark.omni
-@hardware_test(
-    res={"cuda": "L4", "rocm": "MI325", "npu": "A2"},
-    num_cards=2,
-)
-@pytest.mark.parametrize("test_config", test_params)
-def test_video_to_audio(omni_runner: type[OmniRunner], model: str) -> None:
-    """Offline inference: video input, audio output."""
-    model, stage_config_path = test_config
-    with omni_runner(model, seed=42, stage_configs_path=stage_config_path) as runner:
-        # Prepare inputs
-        video = VideoAsset(name="sample", num_frames=4).np_ndarrays
-
-        outputs = runner.generate_multimodal(
-            prompts="Describe this video briefly.",
-            videos=video,
-        )
-
-        # Minimal assertions: got outputs and at least one audio result
-        assert outputs
-        has_audio = any(o.final_output_type == "audio" for o in outputs)
-        assert has_audio
-```

docs/contributing/metrics.md

@@ -1,173 +0,0 @@
-
-# Metrics
-
-You can use these metrics in production to monitor the health and performance of the vLLM-omni system. Typical scenarios include:
-
-- **Performance Monitoring**: Track throughput (e.g., `e2e_avg_tokens_per_s`), latency (e.g., `e2e_total_ms`), and resource utilization to verify that the system meets expected standards.
-
-- **Debugging and Troubleshooting**: Use detailed per-request metrics to diagnose issues, such as high transfer times or unexpected token counts.
-
-## How to Enable and View Metrics
-
-### Start the Service with Metrics Logging
-
-```bash
-vllm serve /workspace/models/Qwen3-Omni-30B-A3B-Instruct --omni --port 8014 --log-stats
-```
-
-### Send a Request
-
-```bash
-python openai_chat_completion_client_for_multimodal_generation.py --query-type use_image
-```
-
-### What You Will See
-
-With `--log-stats` enabled, the server will output detailed metrics logs after each request. Example output:
-
-
-#### Overall Summary
-
-| Field                       | Value        |
-|-----------------------------|--------------|
-| e2e_requests                | 1            |
-| e2e_wall_time_ms            | 41,299.190   |
-| e2e_total_tokens            | 5,202        |
-| e2e_avg_time_per_request_ms | 41,299.190   |
-| e2e_avg_tokens_per_s        | 125.959      |
-| e2e_stage_0_wall_time_ms    | 10,192.289   |
-| e2e_stage_1_wall_time_ms    | 30,541.409   |
-| e2e_stage_2_wall_time_ms    |    207.496   |
-
-#### RequestE2EStats
-
-| Field                   | Value      |
-|-------------------------|------------|
-| e2e_total_ms            | 41,299.133 |
-| e2e_total_tokens        | 5,202      |
-| transfers_total_time_ms | 245.895    |
-| transfers_total_kbytes  | 138,089.939|
-
-#### StageRequestStats
-
-| Field                  | 0      | 1      | 2      |
-|------------------------|--------|--------|--------|
-| audio_generated_frames | 0      | 0      | 525,525|
-| batch_id               | 38     | 274    | 0      |
-| batch_size             | 1      | 1      | 1      |
-| num_tokens_in          | 4,860  | 4,826  | 4,384  |
-| num_tokens_out         | 67     | 275    | 0      |
-| postprocess_time_ms    | 256.158| 0.491  | 0.000  |
-| stage_gen_time_ms      | 9,910.007|30,379.198|160.745|
-
-#### TransferEdgeStats
-
-| Field               | 0->1        | 1->2       |
-|---------------------|-------------|------------|
-| size_kbytes         | 109,277.349 | 28,812.591 |
-| tx_time_ms          | 78.701      | 18.790     |
-| rx_decode_time_ms   | 111.865     | 31.706     |
-| in_flight_time_ms   | 2.015       | 2.819      |
-
-
-These logs include:
-
-- **Overall summary**: total requests, wall time, average tokens/sec, etc.
-
-- **E2E table**: per-request latency and token counts.
-
-- **Stage table**: per-stage batch and timing details.
-
-- **Transfer table**: data transfer and timing for each edge.
-
-You can use these logs to monitor system health, debug performance, and analyze request-level metrics as described above.
-
-
-## Metrics Scope: Offline vs Online Inference
-
-For **offline inference** (batch mode), the summary includes both system-level metrics (aggregated across all requests) and per-request metrics. In this case, `e2e_requests` can be greater than 1, reflecting multiple completed requests in a batch.
-
-For **online inference** (serving mode), the summary is always per-request. `e2e_requests` is always 1, and only request-level metrics are reported for each completion.
-
----
-
-## Parameter Details
-
-### Summary Metrics
-
-| Field                     | Meaning                                                                                       |
-|---------------------------|----------------------------------------------------------------------------------------------|
-| `e2e_requests`            | Number of completed requests.                                                                |
-| `e2e_wall_time_ms`        | Wall-clock time span from run start to last completion, in ms.                               |
-| `e2e_total_tokens`        | Total tokens counted across all completed requests (stage0 input + all stage outputs).       |
-| `e2e_avg_time_per_request_ms` | Average wall time per request: `e2e_wall_time_ms / e2e_requests`.                        |
-| `e2e_avg_tokens_per_s`    | Average token throughput over wall time: `e2e_total_tokens * 1000 / e2e_wall_time_ms`.      |
-| `e2e_stage_{i}_wall_time_ms` | Wall-clock time span for stage i, in ms. Each stage's wall time is reported as a separate field, e.g., `e2e_stage_0_wall_time_ms`, `e2e_stage_1_wall_time_ms`, etc. |
-
----
-
-### E2E Table (per request)
-
-| Field                     | Meaning                                                               |
-|---------------------------|-----------------------------------------------------------------------|
-| `e2e_total_ms`            | End-to-end latency in ms.                                             |
-| `e2e_total_tokens`        | Total tokens for the request (stage0 input + all stage outputs).      |
-| `transfers_total_time_ms` | Sum of transfer edge `total_time_ms` for this request.                |
-| `transfers_total_kbytes`  | Sum of transfer kbytes for this request.                              |
-
-
----
-
-### Stage Table (per stage event / request)
-
-| Field                     | Meaning                                                                                         |
-|---------------------------|-------------------------------------------------------------------------------------------------|
-| `batch_id`                | Batch index.                                                                                    |
-| `batch_size`              | Batch size.                                                                                     |
-| `num_tokens_in`           | Input tokens to the stage.                                                                      |
-| `num_tokens_out`          | Output tokens from the stage.                                                                   |
-| `stage_gen_time_ms`       | Stage compute time in ms, excluding postprocessing time (reported separately as `postprocess_time_ms`). |
-| `image_num`               | Number of images generated (for diffusion/image stages).                                        |
-| `resolution`              | Image resolution (for diffusion/image stages).                                                                  |
-| `postprocess_time_ms` | Diffusion/image: post-processing time in ms.                                                    |
-
----
-
-### Transfer Table (per edge / request)
-
-| Field                | Meaning                                                                   |
-|----------------------|---------------------------------------------------------------------------|
-| `size_kbytes`        | Total kbytes transferred.                                                 |
-| `tx_time_ms`         | Sender transfer time in ms.                                               |
-| `rx_decode_time_ms`  | Receiver decode time in ms.                                               |
-| `in_flight_time_ms`  | In-flight time in ms.                                                     |
-
-
-### Expectation of the Numbers (Verification)
-
-**Formulas:**
-
-- `e2e_total_tokens = Stage0's num_tokens_in + sum(all stages' num_tokens_out)`
-
-- `transfers_total_time_ms = sum(tx_time_ms + rx_decode_time_ms + in_flight_time_ms)` for every edge
-
-**Using the example above:**
-
-**e2e_total_tokens**
-
-- Stage0's `num_tokens_in`: **4,860**
-- Stage0's `num_tokens_out`: **67**
-- Stage1's `num_tokens_out`: **275**
-- Stage2's `num_tokens_out`: **0**
-
-so `e2e_total_tokens = 4,860 + 67 + 275 + 0 = 5,202`, which matches the table value `e2e_total_tokens`.
-
-**transfers_total_time_ms**
-
-For each edge:
-
-- 0->1: tx_time_ms (**78.701**) + rx_decode_time_ms (**111.865**) + in_flight_time_ms (**2.015**) = **192.581**
-
-- 1->2: tx_time_ms (**18.790**) + rx_decode_time_ms (**31.706**) + in_flight_time_ms (**2.819**) = **53.315**
-
-192.581 + 53.315 = **245.896** = transfers_total_time_ms, which matches the calculation (difference is due to rounding)

docs/contributing/model/README.md

@@ -1,15 +1,2 @@
 # Adding a New Model
-
-This section provides comprehensive guidance on how to add a new model to vLLM-Omni.
-
-## Documentation
-
-- **[Adding an Omni-Modality Model](adding_omni_model.md)**: Complete step-by-step guide using Qwen3-Omni as an example.
-
-- **[Adding a Diffusion Model](adding_diffusion_model.md)**: Complete step-by-step guide using Qwen/Qwen-Image-Edit as an example.
-
-
-
-## Quick Start
-
-For a quick reference, see the [Adding a New Multi-Stage Model Guide](adding_omni_model.md) and [Adding a New Diffusion Model Guide](adding_diffusion_model.md).
+Stay tuned for guidance on how to add a new model on vLLM-Omni!

docs/contributing/model/adding_diffusion_model.md

@@ -1,951 +0,0 @@
-# Adding a Diffusion Model
-
-This guide walks you through adding a new diffusion model to vLLM-Omni. We use **Qwen-Image** as the primary example, with references to other models (LongCat, Flux, Wan2.2) to illustrate different patterns.
-
-
----
-
-## Table of Contents
-
-1. [Overview](#overview)
-2. [Directory Structure](#directory-structure)
-3. [Basic Implementation](#basic-implementation)
-4. [Advanced Features](#advanced-features)
-5. [Troubleshooting](#troubleshooting)
-6. [Pull Request Checklist](#pull-request-checklist)
-7. [Reference Implementations](#reference-implementations)
-8. [Summary](#summary)
-
----
-
-## Overview
-
-vLLM-Omni's diffusion inference follows this architecture:
-
-<p align="center">
-  <picture>
-    <source media="(prefers-color-scheme: dark)" src="https://raw.githubusercontent.com/vllm-project/vllm-omni/refs/heads/main/docs/source/architecture/vllm-omni-diffusion-flow.png">
-    <img alt="Diffusion Flow" src="https://raw.githubusercontent.com/vllm-project/vllm-omni/refs/heads/main/docs/source/architecture/vllm-omni-diffusion-flow.png" width=55%>
-  </picture>
-</p>
-
-**Key Components:**
-
-1. **Request Handling:** User prompts → `OmniDiffusionRequest`
-2. **Diffusion Engine:**  Request →  Preprocessing (Optional) → Pipeline execution -> Post-processing
-3. **Pipeline Execution:** Request → Encode prompt → Diffusion steps → Vae decode
-
-
-## Directory Structure
-
-Organize your model files following this structure:
-
-```
-vllm_omni/
-└── diffusion/
-    ├── registry.py                          # ← Register your model here
-    ├── request.py                           # Request data structures
-    └── models/
-        └── your_model_name/                 # ← Create this directory
-            ├── __init__.py                  # Export pipeline and transformer
-            ├── pipeline_xxx.py              # Pipeline implementation
-            └── xxx_transformer.py           # Transformer implementation
-```
-
-**Naming Conventions:**
-
-- **Model directory:** `your_model_name` (lowercase, underscores),  e.g., `qwen_image`, `flux`, `longcat_image`, `wan2_2`
-- **Pipeline file:** `pipeline_xxx.py` where `xxx` describes the task, e.g., `pipeline_qwen_image.py`, `pipeline_qwen_image_edit.py`
-- **Transformer file:** `xxx_transformer.py` matching transformer class name, e.g.,  `qwen_image_transformer.py`, `flux_transformer.py`
-
----
-
-## Basic Implementation
-
-This section covers the minimal steps to get a model working in vLLM-Omni with basic features (online/offline serving, batch requests).
-
-### Step 1: Adapt Transformer Model
-
-The transformer is the core denoising network. Start by copying the transformer implementation from Diffusers and making these adaptations.
-
-
-#### 1.1: Remove Diffusers Mixins
-
-Diffusers' `Mixin` classes are not needed in vLLM-Omni. Remove them:
-
-```diff
-# Before (Diffusers)
-- from diffusers.models.modeling_utils import ModelMixin
-- from diffusers.models.attention_processor import AttentionModuleMixin
-
-- class YourModelTransformer2DModel(ModelMixin, AttentionModuleMixin):
-+ class YourModelTransformer2DModel(nn.Module):
-    """Your transformer model."""
-```
-
-**Example mixins to remove:**
-
-- `ModelMixin` - Weight loading utilities (vLLM-Omni has its own weight loader)
-- `AttentionModuleMixin` - Attention processors (using vLLM-Omni's Attention layer instead)
-- `ConfigMixin` - Config management (not needed)
-- `PeftAdapterMixin` - Parameter efficient finetune utilities (not needed)
-
-#### 1.2: Replace Attention Implementation
-
-**The most important adaptation:** Replace Diffusers' attention with vLLM-Omni's optimized `Attention` layer.
-
-**Before (Diffusers):**
-```python
-from diffusers.models.attention_processor import dispatch_attention_fn
-
-class YourAttentionBlock(nn.Module):
-    def forward(self, hidden_states, encoder_hidden_states=None, ...):
-        ...
-        hidden_states = dispatch_attention_fn(
-            query, key, value,
-            attn_mask=attention_mask,
-            dropout_p=0.0,
-            is_causal=False,
-            backend=self._attention_backend,
-        )
-```
-
-**After (vLLM-Omni):**
-```python
-from vllm_omni.diffusion.attention.layer import Attention
-from vllm_omni.diffusion.attention.backends.abstract import AttentionMetadata
-
-class YourAttentionBlock(nn.Module):
-    def __init__(self, ...):
-        super().__init__()
-
-        # Initialize vLLM-Omni's Attention layer
-        self.attn = Attention(
-            num_heads=self.num_heads,
-            head_size=self.head_dim,
-            softmax_scale=1.0 / (self.head_dim ** 0.5),
-            causal=False,  # Diffusion models typically use bidirectional attention
-            num_kv_heads=self.num_kv_heads,
-        )
-
-    def forward(self, hidden_states, encoder_hidden_states=None, attention_mask=None, ...):
-        ...
-        # Create attention metadata
-        attn_metadata = AttentionMetadata(attn_mask=attention_mask)
-        hidden_states = self.attn(query, key, value, attn_metadata=attn_metadata)
-
-```
-
-**Key Points:**
-
-- **Attention layer initialization:** Done in `__init__`, not per-forward
-- **Tensor shapes:** vLLM-Omni `Attention` expects QKV to have `[B, seq, num_heads, head_dim]` shape
-- **AttentionMetadata:** Wraps attention mask and other metadata
-
-**Attention backends:** vLLM-Omni automatically selects the attention backend given the environmental variable `DIFFUSION_ATTENTION_BACKEND`. The default attention backend is `FLASH_ATTN` for diffusion models.
-
-#### 1.3: Replace Imports and Utilities
-
-**Logger:**
-```diff
-- from diffusers.utils import logging
-- logger = logging.get_logger(__name__)
-
-+ from vllm.logger import init_logger
-+ logger = init_logger(__name__)
-```
-
-**Custom layers from vLLM and vLLM-Omni (if needed):**
-
-```python
-from vllm.model_executor.layers.layernorm import RMSNorm
-from vllm_omni.diffusion.layers.rope import RotaryEmbedding
-from vllm_omni.diffusion.layers.adalayernorm import AdaLayerNorm
-```
-
-#### 1.4: Remove Training-Only Code
-
-Remove code that's only needed for training:
-
-```diff
-# Remove gradient checkpointing
-- if torch.is_grad_enabled() and self.gradient_checkpointing:
--     hidden_states = torch.utils.checkpoint.checkpoint(
--         self._forward_block, hidden_states, ...
--     )
-- else:
--     hidden_states = self._forward_block(hidden_states, ...)
-+ hidden_states = self._forward_block(hidden_states, ...)
-
-# Remove training-specific attributes
-- self.gradient_checkpointing = False
-
-# Remove dropout (set to 0 or remove)
-- self.dropout = nn.Dropout(dropout_prob)
-+ # Removed dropout for inference
-```
-
-#### 1.5: Add Configuration Support
-
-Add support for vLLM-Omni's `OmniDiffusionConfig`:
-
-```python
-from vllm_omni.diffusion.data import OmniDiffusionConfig
-
-class YourModelTransformer2DModel(nn.Module):
-    def __init__(
-        self,
-        *,
-        od_config: OmniDiffusionConfig | None = None,  # ← Add vLLM-Omni config
-        # ... other model-specific parameters
-        num_layers: int = 28,
-        hidden_size: int = 3072,
-        num_heads: int = 24,
-        **kwargs,
-    ):
-        super().__init__()
-
-        # Store config
-        self.od_config = od_config
-        self.parallel_config = od_config.parallel_config if od_config else None
-
-        # Model architecture
-        self.num_layers = num_layers
-        self.hidden_size = hidden_size
-        # ... initialize layers
-```
-
-### Step 2: Adapt Pipeline
-
-The pipeline orchestrates the full generation process (text encoding, denoising loop, VAE decoding). Adapt it from Diffusers format to vLLM-Omni's interface.
-
-#### 2.1: Remove Diffusers Inheritance
-
-**Remove Diffusers base classes:**
-```diff
-- from diffusers import DiffusionPipeline
-- from diffusers.loaders import LoraLoaderMixin
-
-- class YourModelPipeline(DiffusionPipeline, LoraLoaderMixin):
-+ class YourModelPipeline(nn.Module):
-    """Your model pipeline for vLLM-Omni."""
-```
-
-#### 2.2: Adapt `__init__` Method
-
-**Before (Diffusers):**
-```python
-class YourModelPipeline(DiffusionPipeline):
-    def __init__(
-        self,
-        vae: AutoencoderKL,
-        text_encoder: CLIPTextModel,
-        tokenizer: CLIPTokenizer,
-        transformer: YourTransformer,
-        scheduler: FlowMatchScheduler,
-    ):
-        super().__init__()
-        self.register_modules(
-            vae=vae,
-            text_encoder=text_encoder,
-            tokenizer=tokenizer,
-            transformer=transformer,
-            scheduler=scheduler,
-        )
-```
-
-**After (vLLM-Omni):**
-```python
-import os
-from diffusers import AutoencoderKL
-from diffusers.schedulers import FlowMatchEulerDiscreteScheduler
-from transformers import CLIPTextModel, CLIPTokenizer
-
-from vllm_omni.diffusion.data import OmniDiffusionConfig
-from vllm_omni.diffusion.distributed.utils import get_local_device
-from vllm_omni.diffusion.utils.tf_utils import get_transformer_config_kwargs
-from vllm_omni.diffusion.models.your_model_name.your_model_transformer import (
-    YourModelTransformer2DModel,
-)
-
-
-class YourModelPipeline(nn.Module):
-    def __init__(
-        self,
-        *,
-        od_config: OmniDiffusionConfig,
-        prefix: str = "",
-    ):
-        super().__init__()
-        self.od_config = od_config
-        self.parallel_config = od_config.parallel_config
-        self.device = get_local_device()
-        model = od_config.model
-        local_files_only = os.path.exists(model)
-
-        # Load components from checkpoint
-        self.scheduler = FlowMatchEulerDiscreteScheduler.from_pretrained(
-            model, subfolder="scheduler", local_files_only=local_files_only)
-        self.text_encoder = CLIPTextModel.from_pretrained(
-            model, subfolder="text_encoder", local_files_only=local_files_only).to(self.device)
-        self.tokenizer = CLIPTokenizer.from_pretrained(
-            model, subfolder="tokenizer", local_files_only=local_files_only)
-        self.vae = AutoencoderKL.from_pretrained(
-            model, subfolder="vae", local_files_only=local_files_only).to(self.device)
-
-        # Initialize transformer with vLLM-Omni config
-        transformer_kwargs = get_transformer_config_kwargs(
-            od_config.tf_model_config, YourModelTransformer2DModel)
-        self.transformer = YourModelTransformer2DModel(
-            od_config=od_config, **transformer_kwargs)
-
-        # Store VAE scale factor for latent space conversions
-        self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1)
-        self.default_sample_size = 128  # Default latent size
-```
-
-**Key Changes:**
-
-1. **`od_config` parameter:** All configuration through `OmniDiffusionConfig`
-2. **Manual component loading:** No `register_modules()`, load each component explicitly
-3. **Local files support:** Check `os.path.exists(model)` for local checkpoints
-4. **Transformer with config:** Pass `od_config` to transformer constructor
-
-#### 2.3: Adapt `__call__` → `forward` Method
-
-**Change signature:**
-
-```diff
-- @torch.no_grad()
-- def __call__(
-+ def forward(
-    self,
-+   req: OmniDiffusionRequest,  # ← Add request parameter here
-- ):
-+ ) -> DiffusionOutput:  # ← Add return type
-```
-
-[`OmniDiffusionRequest`](https://docs.vllm.ai/projects/vllm-omni/en/latest/api/vllm_omni/diffusion/request/#vllm_omni.diffusion.request.OmniDiffusionRequest) is a dataclass that contains the **prompts** and **sampling parameters** [`OmniDiffusionSamplingParams`](https://docs.vllm.ai/projects/vllm-omni/en/latest/api/vllm_omni/inputs/data/#vllm_omni.inputs.data.OmniDiffusionSamplingParams) for the diffusion pipeline execution. It also contains a request_id for other components to trace this request and its outputs.
-
-See some parameters in `OmniDiffusionSamplingParams` as follows:
-
-| parameters | type |value | function |
-|:---:|:---:|:---:|:---:|
-| `num_inference_steps` | `int` | 50 |  The number of diffusion steps during inference|
-| `guidance_scale` |  `float` | 0.0 |  The classifier free guidance scale |
-| `width` and `height` | `int` | None | The width and height of the generated image |
-
-**Extract parameters from request:**
-
-```python
-from vllm_omni.diffusion.request import OmniDiffusionRequest
-from vllm_omni.diffusion.data import DiffusionOutput
-
-def forward(
-    self,
-    req: OmniDiffusionRequest,
-) -> DiffusionOutput:
-    # Extract prompts from request
-    if req.prompts is not None:
-        prompt = [
-            p if isinstance(p, str) else (p.get("prompt") or "")
-            for p in req.prompts
-        ]
-
-    # Extract sampling parameters
-    sampling_params = req.sampling_params
-    num_inference_steps = sampling_params.num_inference_steps or 50
-    guidance_scale = sampling_params.guidance_scale or 7.5
-    height = sampling_params.height or (self.default_sample_size * self.vae_scale_factor)
-    width = sampling_params.width or (self.default_sample_size * self.vae_scale_factor)
-
-    # For image editing pipelines, extract images from multi_modal_data
-    if hasattr(req, 'multi_modal_data') and req.multi_modal_data:
-        input_images = req.multi_modal_data.get('image', [])
-
-    # ... rest of generation logic
-```
-
-For an image editing model, an example `OmniDiffusionRequest` is like:
-```python
-{
-    "prompt": "turn this cat to a dog",
-    "multi_modal_data": {"image": input_image}
-},
-```
-
-**Wrap output:**
-
-```diff
-    # Generate images
-    images = self.vae.decode(latents)[0]
-
--   return {"images": images}
-+   return DiffusionOutput(output=images)
-```
-
-#### 2.4: Extract Pre/Post-Processing Functions
-
-vLLM-Omni separates image processing from the main pipeline for better modularity.
-
-**Post-processing function (required):**
-```python
-def get_your_model_post_process_func(
-    od_config: OmniDiffusionConfig,
-):
-    """
-    Create post-processing function for your model.
-
-    Returns a function that converts latents to images.
-    """
-    from diffusers.image_processor import VaeImageProcessor
-    import json
-
-    # Load VAE config to get scale factor
-    model_path = od_config.model
-    if not os.path.exists(model_path):
-        from vllm_omni.diffusion.model_loader.utils import download_weights_from_hf_specific
-        model_path = download_weights_from_hf_specific(model_path, None, ["*"])
-
-    vae_config_path = os.path.join(model_path, "vae/config.json")
-    with open(vae_config_path) as f:
-        vae_config = json.load(f)
-        vae_scale_factor = 2 ** (len(vae_config["block_out_channels"]) - 1)
-
-    # Create image processor
-    image_processor = VaeImageProcessor(vae_scale_factor=vae_scale_factor)
-
-    def post_process_func(images: torch.Tensor):
-        return image_processor.postprocess(images, output_type="pil")
-
-    return post_process_func
-```
-
-**Pre-processing function (for image editing pipelines):**
-
-```python
-def get_your_model_pre_process_func(
-    od_config: OmniDiffusionConfig,
-):
-    """
-    Create pre-processing function for image editing.
-
-    Returns a function that prepares input images.
-    """
-    from PIL import Image
-    from diffusers.image_processor import VaeImageProcessor
-
-    # Load VAE config
-    # ... (similar to post_process_func)
-
-    image_processor = VaeImageProcessor(vae_scale_factor=vae_scale_factor)
-
-    def pre_process_func(
-        request: OmniDiffusionRequest,
-        ):
-        for i, prompt in enumerate(request.prompts):
-            multi_modal_data = prompt.get("multi_modal_data", {}) if not isinstance(prompt, str) else None
-            raw_image = multi_modal_data.get("image", None) if multi_modal_data is not None else None
-            # image pre-processing
-            # after pre-processing, update the request attributes
-            ...
-        return request
-
-    return pre_process_func
-```
-
-#### 2.5: Add Weight Loading Support
-
-Add methods for automatic weight downloading and loading:
-
-```python
-from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader
-from vllm.model_executor.models.utils import AutoWeightsLoader
-
-class YourModelPipeline(nn.Module):
-    def __init__(self, *, od_config: OmniDiffusionConfig, prefix: str = ""):
-        super().__init__()
-        # ... initialization code
-
-        # Define weight sources for automatic loading
-        self.weights_sources = [
-            DiffusersPipelineLoader.ComponentSource(
-                model_or_path=od_config.model,
-                subfolder="transformer",
-                revision=None,
-                prefix="transformer.",
-                fall_back_to_pt=True,
-            )
-        ]
-
-    def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
-        """
-        Customize the weight loading behavior, such as filter weights name.
-
-        Args:
-            weights: Iterable of (param_name, param_tensor) tuples
-
-        Returns:
-            Set of loaded parameter names
-        """
-        loader = AutoWeightsLoader(self)
-        return loader.load_weights(weights)
-```
-
-### Step 3: Register Model
-
-Register your model in `vllm_omni/diffusion/registry.py` so vLLM-Omni can discover and load it.
-
-#### 3.1: Register Pipeline Class
-
-```python
-# vllm_omni/diffusion/registry.py
-
-_DIFFUSION_MODELS = {
-    # Format: "PipelineClassName": (module_folder, module_file, class_name)
-
-    # Existing models
-    "QwenImagePipeline": ("qwen_image", "pipeline_qwen_image", "QwenImagePipeline"),
-    "FluxPipeline": ("flux", "pipeline_flux", "FluxPipeline"),
-
-    # Add your model
-    "YourModelPipeline": (
-        "your_model_name",           # Module folder name
-        "pipeline_your_model",       # Python file name (without .py)
-        "YourModelPipeline",         # Pipeline class name
-    ),
-}
-```
-
-#### 3.2: Register Pre/Post-Processing Function
-
-```python
-# vllm_omni/diffusion/registry.py
-_DIFFUSION_PRE_PROCESS_FUNCS = {
-    # arch: pre_process_func
-    # `pre_process_func` function must be placed in {mod_folder}/{mod_relname}.py,
-    # where mod_folder and mod_relname are  defined and mapped using `_DIFFUSION_MODELS` via the `arch` key
-    "GlmImagePipeline": "get_glm_image_pre_process_func",
-    "QwenImageEditPipeline": "get_qwen_image_edit_pre_process_func",
-
-    # Add your model
-    "YourModelPipeline": "get_your_model_pre_process_func", # Optional
-}
-_DIFFUSION_POST_PROCESS_FUNCS = {
-    # Format: "PipelineClassName": "function_name"
-
-    # Existing models
-    "QwenImagePipeline": "get_qwen_image_post_process_func",
-    "FluxPipeline": "get_flux_post_process_func",
-
-    # Add your model
-    "YourModelPipeline": "get_your_model_post_process_func",
-}
-```
-
-
-#### 3.3: Export from Module
-
-Create/update `__init__.py` to export your classes:
-
-```python
-# vllm_omni/diffusion/models/your_model_name/__init__.py
-
-from .pipeline_your_model import (
-    YourModelPipeline,
-    get_your_model_post_process_func,
-)
-from .your_model_transformer import YourModelTransformer2DModel
-
-__all__ = [
-    "YourModelPipeline",
-    "YourModelTransformer2DModel",
-    "get_your_model_post_process_func",
-]
-```
-
----
-
-### Step 4: Add Example Script
-
-
-If your model is one of Text-to-Image, Text-to-Audio, Text-to-Video, Image-to-Image, Image-to-Video models, you can simply try one of the following offline inference scripts to run your model:
-
-| Model Category | Offline Inference Script |
-|---|---|
-| Image-to-Image | `examples/offline_inference/image_to_image/image_edit.py` |
-| Image-to-Video | `examples/offline_inference/image_to_video/image_to_video.py` |
-| Text-to-Image | `examples/offline_inference/text_to_image/text_to_image.py` |
-| Text-to-Audio | `examples/offline_inference/text_to_audio/text_to_audio.py` |
-| Text-to-Video | `examples/offline_inference/text_to_video/text_to_video.py` |
-
-
-If new CLI arguments need to be added, please edit the offline inference script corresponding to your model category from the table above, and update the example inference script in its corresponding document file (e.g., `examples/offline_inference/text_to_video/text_to_video.md`).
-
-For online inference, all the supported tasks are listed in `docs/user_guide/examples/online_serving/`. If your model falls into these categories, please check the corresponding documentation in this folder and the example at `examples/online_serving/TASK_NAME`. Update them accordingly if needed.
-
----
-
-If your model is an Omni (understanding and generation) model, please follow the steps below.
-
-#### 4.1: Create Example File
-
-Taking **BAGEL** model as examples for both offline and online:
-
-- Offline: `examples/offline_inference/bagel/`
-- Online: `examples/online_serving/bagel/`
-
-Add **two example folders** for your model:
-
-```bash
-mkdir -p examples/offline_inference/your_model_name
-mkdir -p examples/online_serving/your_model_name
-```
-
-**Offline (recommended minimum):** create `examples/offline_inference/your_model_name/end2end.py` and a README.
-
-- Script: `examples/offline_inference/your_model_name/end2end.py`
-  - Parse args like BAGEL (`--model`, `--modality`, optional `--image-path`, `--steps`, etc.)
-  - Use `from vllm_omni.entrypoints.omni import Omni` (or `OmniDiffusion` if your model is diffusion-only)
-  - Save outputs (images/audio/video/text) with deterministic filenames (e.g., `output_0_0.png`)
-- Doc: `examples/offline_inference/your_model_name/README.md`
-  - Include at least one runnable command, e.g.:
-
-```bash
-cd examples/offline_inference/your_model_name
-python end2end.py --model your-org/your-model-name --modality text2img --prompts "A cute cat"
-```
-
-#### 4.2: Add Online Serving Example (OpenAI-Compatible)
-
-Mirror BAGEL’s online serving setup:
-
-- Server launcher: `examples/online_serving/your_model_name/run_server.sh`
-  - Wrap `vllm serve ... --omni --port ...` (and `--stage-configs-path ...` if needed)
-- Client: `examples/online_serving/your_model_name/openai_chat_client.py`
-  - Send requests to `POST /v1/chat/completions`
-  - Support multimodal inputs (e.g., base64 image) if your model needs it
-- Doc: `examples/online_serving/your_model_name/README.md`
-  - Include both “launch server” and “send request”:
-
-```bash
-# Terminal 1: launch server
-cd examples/online_serving/your_model_name
-bash run_server.sh
-
-# Terminal 2: send request
-python openai_chat_client.py --prompt "A cute cat" --modality text2img
-```
-
-
-### Step 5: Test Your Implementation
-
-Before submitting, thoroughly test your implementation.
-
-#### 5.1: Performance/Speed Check
-
-Manually compare **latency/throughput** and **output quality** against a Diffusers baseline.
-
-For a fair comparison, keep the same **prompt**, **seed**, **resolution**, **num_inference_steps**, and **guidance settings**, and run multiple trials to reduce randomness. Record the results (and your hardware / driver / CUDA versions) in your PR description.
-
-
-#### 5.2 Functionality Check in CI
-
-To ensure project maintainability and sustainable development, we encourage contributors to submit test code (unit tests, system tests, or end-to-end tests) alongside their code changes.
-For comprehensive testing guidelines, please refer to the [Test File Structure and Style Guide](../ci/tests_style.md).
-
----
-
-## Advanced Features
-
-Once basic implementation works, add advanced features for better performance.
-
-### torch.compile Support
-
-Enable automatic compilation for repeated blocks:
-
-```python
-# In your_model_transformer.py
-
-class YourModelTransformer2DModel(nn.Module):
-    # Specify which blocks can be compiled
-    _repeated_blocks = ["YourTransformerBlock"]  # List of block class names
-
-    def __init__(self, ...):
-        super().__init__()
-        # ... initialization
-```
-
-vLLM-Omni automatically compiles blocks in `_repeated_blocks` when `torch.compile` is available.
-
-### Tensor Parallelism
-
-See detailed guide: [How to add Tensor Parallel support](../../design/feature/tensor_parallel.md)
-
-**Quick setup:**
-
-1. Replace Linear layers by various parallel linear layers (e.g., `ColumnParallelLinear`) in vLLM
-2. Check `tp_size` validity: `hidden_dim`, `num_heads`, and `num_kv_heads` must be divisible by `tp_size`
-
-**Usage:** Set `tensor_parallel_size` when initializing:
-```python
-omni = Omni(model="your-model", tensor_parallel_size=2)
-```
-
-### CFG Parallelism
-
-See detailed guide: [How to add CFG-Parallel support](../../design/feature/cfg_parallel.md)
-
-**Quick setup:**
-
-1. Implement `diffuse()` method
-2. Inherit `CFGParallelMixin` in your pipeline class
-
-**Usage:** Set `cfg_parallel_size` when initializing:
-```python
-omni = Omni(model="your-model", cfg_parallel_size=2)
-```
-
-### Sequence Parallelism
-
-See detailed guide: [How to add Sequence Parallel support](../../design/feature/sequence_parallel.md)
-
-**Quick setup:**
-
-1. Add `_sp_plan` class attribute to transformer
-2. Specify where to shard/gather tensors
-
-**Usage:** Set `ulysses_degree` and `ring_degree` when initializing:
-```python
-omni = Omni(model="your-model", ulysses_degree=2, ring_degree=2)
-```
-
-### Cache Acceleration
-
-#### TeaCache
-
-See detailed guide: [How to add TeaCache support](../../design/feature/teacache.md)
-
-**Quick setup:**
-
-1. Write extractor function
-2. Register in `EXTRACTOR_REGISTRY`
-3. Add polynomial coefficients
-
-**Usage:** Set `cache_backend` and `cache_config` when initializing:
-```python
-omni = Omni(model="your-model",
-    cache_backend="tea_cache",
-    cache_config={"rel_l1_thresh": 0.2}
-)
-
-```
-
-
-#### Cache-DiT
-
-See detailed guide: [How to add Cache-DiT support](../../design/feature/cache_dit.md)
-
-**Quick setup:**
-
-- For standard models: Works automatically
-- For complex architectures: Write custom cache config
-
-**Usage:** Set `cache_backend` and `cache_config` when initializing:
-```python
-omni = Omni(model="your-model",  
-    cache_backend="cache_dit",
-    cache_config={
-        "Fn_compute_blocks": 1,
-        "Bn_compute_blocks": 0,
-        "max_warmup_steps": 4,
-    }
-)
-```
-
-### CPU Offload
-
-See detailed guide: [CPU Offloading for Diffusion Models](../../user_guide/diffusion/cpu_offload_diffusion.md)
-
-vLLM-Omni provides two offloading strategies to reduce GPU memory usage:
-
-1. **Model-level offload**: Mutual exclusion between DiT and encoders (only one on GPU at a time)
-2. **Layerwise (Blockwise) offload**: Keeps only a single transformer block on GPU at a time with compute-memory overlap
-
-**Usage:** Enable offload when initializing:
-```python
-# Model-level offload
-omni = Omni(model="your-model", enable_cpu_offload=True)
-
-# Layerwise offload
-omni = Omni(model="your-model", enable_layerwise_offload=True)
-```
-
-**To support layerwise offloading:** Define the blocks attribute name in your transformer:
-
-```python
-class WanTransformer3DModel(nn.Module):
-    _layerwise_offload_blocks_attr = "blocks"  # Attribute name containing transformer blocks
-
-    def __init__(self):
-        self.blocks = nn.ModuleList([...])  # Transformer blocks
-```
-
-**Note:** Layerwise offloading is primarily recommended for large **video generation models** where the compute cost per block is high enough to effectively overlap with memory prefetch operations.
-
-
----
-
-## Troubleshooting
-
-
-**Issue: ImportError when loading model**
-
-**Symptoms:** `ModuleNotFoundError` or `ImportError` when calling `Omni(model="your-model")`
-
-**Causes:**
-
-1. Model not registered in `registry.py`
-2. Wrong class name in registry
-3. Missing `__init__.py` exports
-
-
-**Issue: Shape mismatch in attention**
-
-**Symptoms:** `RuntimeError: shape mismatch` in attention forward
-
-**Cause:** Incorrect tensor reshaping for vLLM-Omni's attention interface
-
-**Solution:** Ensure correct shapes:
-
-```python
-# vLLM-Omni expects: [batch, seq_len, num_heads, head_dim]
-query = query.view(batch_size, seq_len, self.num_heads, self.head_dim)
-key = key.view(batch_size, kv_seq_len, self.num_kv_heads, self.head_dim)
-value = value.view(batch_size, kv_seq_len, self.num_kv_heads, self.head_dim)
-
-hidden_states = self.attn(query, key, value, attn_metadata=attn_metadata)
-
-# Reshape back: [batch, seq_len, num_heads, head_dim] → [batch, seq_len, hidden_size]
-hidden_states = hidden_states.reshape(batch_size, seq_len, -1)
-```
-
-**Issue: Different outputs compared to Diffusers**
-
-**Symptoms:** Generated images look different from Diffusers
-
-**Causes:**
-
-1. Attention backend differences (FlashAttention vs PyTorch SDPA)
-2. Missing normalization or scaling
-
-**4. Issue: Out of memory (OOM)**
-
-**Symptoms:** CUDA out of memory errors
-
-**Solutions:**
-
-1. **Reduce batch size:**
-   ```python
-   omni.generate(prompts=[...], max_batch_size=2)
-   ```
-
-2. **Use smaller image size:**
-   ```python
-   sampling_params = OmniDiffusionSamplingParams(height=512, width=512)
-   ```
-
-3. **Enable model offloading:**
-   ```python
-   omni = Omni(model="...", enable_cpu_offload=True)
-   ```
-
-4. **Apply vae tiling and slicing**
-   ```python
-   omni = Omni(model="...", vae_use_slicing=True, vae_use_tiling=True,)
-   ```
-
----
-
-## Pull Request Checklist
-
-When submitting a PR to add your model, include:
-
-**1. Implementation Files**
-
-- ✅ Transformer model (`xxx_transformer.py`)
-- ✅ Pipeline (`pipeline_xxx.py`)
-- ✅ Registry entries in `registry.py`
-- ✅ `__init__.py` with proper exports
-
-**2. Example and Tests**
-
-- ✅ Example script in `examples/`
-- ✅ Test file in `tests/e2e/`
-- ✅ Documentation (`docs/`) creation or updates
-
-_Note: End-to-end test files in `tests/e2e/` are optional but strongly recommended. README updates are required for all new models._
-
-**3. Documentation Updates**
-
-- ✅ Add model to supported models table in `docs/models/supported_models.md`
-- ✅ If supporting acceleration features (e.g., sequence parallelism, CFG parallel), update acceleration feature tables in:
-  - `docs/user_guide/diffusion_acceleration.md`
-  - `docs/user_guide/diffusion/parallelism_acceleration.md`
-
----
-
-## Model Recipe
-
-After implementing and testing your model, please add a model recipe to the [vllm-project/recipes](https://github.com/vllm-project/recipes) repository. This helps other users understand how to use your model with vLLM-Omni.
-
-**What to Include**
-
-Your recipe should include:
-
-1. **Model Overview**: Brief description of the model and its capabilities
-2. **Installation Instructions**: Step-by-step setup instructions including:
-   - Installing vllm-omni and dependencies
-   - Installing any additional required packages (e.g., xformers, diffusers)
-   - Any version requirements
-3. **Usage Examples**: Command-line examples demonstrating how to run the model
-4. **Configuration Details**: Important configuration parameters and their meanings
-
-**Example**
-
-For reference, see the [LongCat recipe example](https://github.com/vllm-project/recipes/pull/179) which demonstrates the expected format and structure.
-
-**Recipe Location**
-
-Create your recipe file in the appropriate directory structure:
-- For organization-specific models: `OrganizationName/ModelName.md`
-- For general models: `ModelName.md`
-
-The recipe should be a Markdown file that provides clear, reproducible instructions for users to get started with your model.
-
----
-
-## Reference Implementations
-
-Study these complete examples:
-
-| Model | Architecture | Key Features | Files |
-|-------|--------------|--------------|-------|
-| **Qwen-Image** | Dual-stream transformer | CFG-Parallel, SP, TP, Cache | `vllm_omni/diffusion/models/qwen_image/` |
-| **Wan2.2** | Video transformer | Dual transformers, SP, CFG-Parallel | `vllm_omni/diffusion/models/wan2_2/` |
-
----
-
-## Summary
-
-Adding a diffusion model to vLLM-Omni involves:
-
-1. ✅ **Adapt transformer** - Replace attention, remove mixins, add config support
-2. ✅ **Adapt pipeline** - Change interface, add request handling, extract processing
-3. ✅ **Register model** - Add entries to `registry.py`
-4. ✅ **Add examples** - Provide runnable scripts
-5. ✅ **Test thoroughly** - Verify correctness and performance
-6. ✅ **Add advanced features** - Enable parallelism and acceleration (optional)
-7. ✅ **Submit PR** - Include verification results and documentation
-
-**Need help?** Check reference implementations or ask in [slack.vllm.ai](https://slack.vllm.ai) or vLLM user forum at [discuss.vllm.ai](https://discuss.vllm.ai).

docs/contributing/model/adding_omni_model.md

@@ -1,633 +0,0 @@
-# Adding an Omni-Modality Model
-
-This guide walks through the process of adding a new multi-stage model to vLLM-Omni, using **Qwen3-Omni** as a comprehensive example. Qwen3-Omni is a multi-stage omni-modality model that demonstrates the full capabilities of vLLM-Omni's architecture.
-
-## Table of Contents
-
-1. [Overview](#overview)
-2. [Directory Structure](#directory-structure)
-3. [Step-by-Step Implementation](#step-by-step-implementation)
-4. [Key Components](#key-components)
-5. [Model Registration](#model-registration)
-6. [Stage Configuration](#stage-configuration)
-7. [Stage Input Processors](#stage-input-processors)
-8. [Testing](#testing)
-9. [Adding a Model Recipe](#adding-a-model-recipe)
-10. [Summary](#summary)
-
-## Overview
-
-vLLM-Omni supports multi-stage model architectures where different stages can run on different devices and process different modalities. The Qwen3-Omni model exemplifies this with three stages:
-
-1. **Thinker Stage**: Multimodal understanding (text + audio + video) → text generation
-2. **Talker Stage**: Text embeddings → RVQ codec codes
-3. **Code2Wav Stage**: RVQ codes → audio waveform
-
-Each stage is implemented as a separate model class that can be configured independently.
-
-## Directory Structure
-
-When adding a new model, you'll need to create the following structure:
-
-```
-vllm_omni/model_executor/models/
-└── your_model_name/              # Model directory (e.g., qwen3_omni)
-    ├── __init__.py               # Exports main model class
-    ├── your_model.py             # Main unified model class
-    ├── your_model_stage1_implementation.py      # Stage 1 implementation (e.g., thinker)
-    ├── your_model_stage2_implementation.py      # Stage 2 implementation (e.g., talker)
-    └── your_model_stage3_implementation.py      # Stage 3 implementation (e.g., code2wav)
-    └── ... maybe other stage implementations
-
-vllm_omni/model_executor/stage_input_processors/
-└── your_model_name.py            # Stage transition processors
-
-vllm_omni/model_executor/stage_configs/
-└── your_model_name.yaml          # Stage configuration file
-```
-
-## Step-by-Step Implementation
-
-### Step 1: Create the Model Directory
-
-Create a new directory under `vllm_omni/model_executor/models/`
-
-### Step 2: Implement Stage Components
-
-For Qwen3-Omni, we have three stage components:
-
-#### 2.1 Thinker Stage (`qwen3_omni_moe_thinker.py`)
-
-The thinker stage handles multimodal understanding. Key features:
-
-- Inherits from base Qwen3 MoE model in vLLM, using vLLM fused ops & page attn to accelerate
-- Implements multimodal processing interfaces
-- Handles audio, video, and image inputs
-- Generates text outputs
-
-```python
-from vllm.model_executor.models.interfaces import SupportsMultiModal, SupportsPP
-from vllm.model_executor.models.qwen3_moe import Qwen3MoeForCausalLM
-
-class Qwen3OmniMoeThinkerForConditionalGeneration(
-    Qwen3MoeForCausalLM,
-    SupportsMultiModal,
-    SupportsPP
-):
-    """Thinker stage: multimodal understanding → text generation."""
-
-    def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
-        # Initialize base model
-        # Set up multimodal processors
-        # Configure audio/video/image encoders
-        pass
-```
-
-#### 2.2 Talker Stage (`qwen3_omni_moe_talker.py`)
-
-The talker stage converts text embeddings to codec codes:
-
-```python
-class Qwen3OmniMoeTalkerForConditionalGeneration(
-    Qwen3MoeForCausalLM,
-    SupportsPP
-):
-    """Talker stage: text embeddings → RVQ codec codes."""
-
-    def __init__(self, vllm_config, talker_config, prefix):
-        # Initialize base model
-        # Replace LM head with codec head
-        # Set up text projection from thinker
-        pass
-```
-
-#### 2.3 Code2Wav Stage (`qwen3_omni_code2wav.py`)
-
-The code2wav stage generates audio waveforms:
-
-```python
-class Qwen3OmniMoeCode2Wav(nn.Module):
-    """Code2Wav stage: RVQ codes → audio waveform."""
-
-    def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
-        # Initialize audio decoder
-        # Set up codec processing
-        pass
-```
-
-### Step 3: Implement the Unified Model Class
-
-The main model class (`qwen3_omni.py`) orchestrates all stages:
-
-```python
-@MULTIMODAL_REGISTRY.register_processor(
-    Qwen3OmniMoeThinkerMultiModalProcessor,
-    info=Qwen3OmniMoeThinkerProcessingInfo,
-    dummy_inputs=Qwen3OmniMoeThinkerDummyInputsBuilder,
-)
-class Qwen3OmniMoeForConditionalGeneration(
-    nn.Module, SupportsMultiModal, SupportsPP, Qwen3OmniMoeConditionalGenerationMixin
-):
-    """
-    Unified Qwen3 Omni MoE model combining thinker, talker, and code2wav.
-
-    Architecture:
-    - Thinker: Multimodal understanding (text + audio + video) → text generation
-    - Talker: Text embeddings → RVQ codec codes
-    - Code2Wav: RVQ codes → audio waveform
-
-    Usage:
-        Set `model_stage` in vllm_config to one of: "thinker", "talker", "code2wav"
-    """
-
-    def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
-        super().__init__()
-        self.have_multimodal_outputs = True
-        config: Qwen3OmniMoeConfig = vllm_config.model_config.hf_config
-
-        # Determine which stage to initialize
-        self.model_stage = vllm_config.model_config.model_stage
-
-        if self.model_stage == "thinker":
-            # Initialize thinker model
-            thinker_vllm_config = vllm_config.with_hf_config(
-                config.thinker_config,
-                architectures=["Qwen3OmniMoeThinkerForConditionalGeneration"]
-            )
-            self.thinker = init_vllm_registered_model(
-                vllm_config=thinker_vllm_config,
-                prefix=maybe_prefix(prefix, "thinker"),
-                hf_config=config.thinker_config,
-                architectures=["Qwen3OmniMoeThinkerForConditionalGeneration"],
-            )
-            self.model = self.thinker
-
-        elif self.model_stage == "talker":
-            # Initialize talker model
-            talker_vllm_config = vllm_config.with_hf_config(
-                config.talker_config,
-                architectures=["Qwen3OmniMoeTalkerForConditionalGeneration"]
-            )
-            self.talker = init_vllm_registered_model(
-                vllm_config=talker_vllm_config,
-                prefix=maybe_prefix(prefix, "talker"),
-                hf_config=config.talker_config,
-                architectures=["Qwen3OmniMoeTalkerForConditionalGeneration"],
-            )
-            self.model = self.talker
-
-        elif self.model_stage == "code2wav":
-            # Initialize code2wav model
-            code2wav_vllm_config = vllm_config.with_hf_config(
-                config.code2wav_config,
-                architectures=["Qwen3OmniMoeCode2Wav"]
-            )
-            self.code2wav = init_vllm_registered_model(
-                vllm_config=code2wav_vllm_config,
-                prefix=maybe_prefix(prefix, "code2wav"),
-                hf_config=config.code2wav_config,
-                architectures=["Qwen3OmniMoeCode2Wav"],
-            )
-            self.model = self.code2wav
-        else:
-            raise ValueError(
-                f"Invalid model_stage: {self.model_stage}. "
-                f"Must be one of: 'thinker', 'talker', 'code2wav'"
-            )
-```
-
-#### Key Methods to Implement
-
-1. **`forward()`**: Handles the forward pass for each stage
-2. **`embed_input_ids()`**: Embeds input token IDs
-3. **`embed_multimodal()`**: Processes multimodal inputs (if applicable)
-4. **`compute_logits()`**: Computes logits from hidden states
-5. **`load_weights()`**: Loads model weights with proper prefixing of different stages
-
-### Step 4: Create `__init__.py`
-
-Export the main model class:
-
-```python
-# vllm_omni/model_executor/models/qwen3_omni/__init__.py
-from .qwen3_omni import Qwen3OmniMoeForConditionalGeneration
-
-__all__ = ["Qwen3OmniMoeForConditionalGeneration"]
-```
-
-## Key Components
-
-### 1. Model Interfaces
-
-Your model should implement the appropriate interfaces:
-
-- **`SupportsMultiModal`**: For models that process multimodal inputs
-- **`SupportsPP`**: For models that support pipeline parallelism
-- **`SupportsMRoPE`**: For models using multi-dimensional RoPE (if applicable)
-
-### 2. Multimodal Registration
-
-If your model processes multimodal inputs, register it with the multimodal registry:
-
-```python
-@MULTIMODAL_REGISTRY.register_processor(
-    YourMultiModalProcessor,
-    info=YourProcessingInfo,
-    dummy_inputs=YourDummyInputsBuilder,
-)
-class YourModel(nn.Module, SupportsMultiModal):
-    pass
-```
-
-### 3. Weight Loading
-
-Implement `load_weights()` to handle weight loading with proper prefixing:
-
-```python
-def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
-    """Load weights for all components of the omni model."""
-    loaded_weights = set()
-    thinker_weights = []
-    talker_weights = []
-    code2wav_weights = []
-
-    # Separate weights by component
-    for k, v in weights:
-        if k.startswith("thinker."):
-            thinker_weights.append((k, v))
-        elif k.startswith("talker."):
-            talker_weights.append((k, v))
-        elif k.startswith("code2wav."):
-            code2wav_weights.append((k, v))
-
-    # Load each component's weights
-    if self.thinker and thinker_weights:
-        thinker_loaded = self.thinker.load_weights(thinker_weights)
-        thinker_loaded = add_prefix_to_loaded_weights(thinker_loaded, "thinker")
-        loaded_weights.update(thinker_loaded)
-
-    # Similar for talker and code2wav...
-
-    return loaded_weights
-```
-
-### 4. Output Format
-
-Use `OmniOutput` for stage outputs:
-
-```python
-from vllm_omni.model_executor.models.output_templates import OmniOutput
-
-# In forward method
-return OmniOutput(
-    text_hidden_states=hidden_states,
-    multimodal_outputs={"additional_data": data},
-    next_token_id=next_token_id,
-)
-```
-
-## Model Registration
-
-Register your model in `vllm_omni/model_executor/models/registry.py`:
-
-```python
-_OMNI_MODELS = {
-    # ... existing models ...
-
-    # Your new model
-    "YourModelForConditionalGeneration": (
-        "your_model_name",        # Module folder name
-        "your_model",             # Module file name (without .py)
-        "YourModelForConditionalGeneration",  # Class name
-    ),
-    "YourModelThinkerForConditionalGeneration": (
-        "your_model_name",
-        "your_model_thinker",
-        "YourModelThinkerForConditionalGeneration",
-    ),
-    # ... other stages ...
-}
-```
-
-The registry uses lazy loading, so the model class is imported only when needed.
-
-## Stage Configuration
-
-Create a YAML configuration file in `vllm_omni/model_executor/stage_configs/`. For a complete example, see the [Qwen3-Omni configuration file](gh-file:vllm_omni/model_executor/stage_configs/qwen3_omni_moe.yaml).
-
-### Key Configuration Fields
-
-- **`model_stage`**: Which stage to run ("thinker", "talker", "code2wav", etc.)
-- **`model_arch`**: The model architecture name (must match registry)
-- **`engine_input_source`**: List of stage IDs that provide input to this stage
-- **`custom_process_input_func`**: Function to process inputs from previous stages
-- **`final_output`**: Whether this stage produces the final output (True/False)
-- **`final_output_type`**: Type of final output ("text", "audio", "image", etc.)
-
-## Stage Input Processors
-
-Stage transitions are the mechanism by which outputs from one stage are converted into inputs for the next stage. This section explains where and how stage transitions occur.
-
-### Where Stage Transitions Are Called
-
-Stage transitions happen automatically in the orchestrator (`OmniLLM` class) during the generation loop. Here's the detailed flow:
-
-1. **Location**: `vllm_omni/entrypoints/omni_llm.py` in the `_run_generation()` method
-2. **Trigger**: When a stage completes processing and produces outputs
-3. **Execution Flow**:
-   ```python
-   # In omni_llm.py, _run_generation() method (around line 345-460)
-
-   # Main orchestrator loop polls each stage for completed requests
-   for stage_id, stage in enumerate(self.stage_list):
-       result = stage.try_collect()  # Get completed request
-       if result is None:
-           continue
-
-       # Store outputs from this stage
-       engine_outputs = _load(result, obj_key="engine_outputs", shm_key="engine_outputs_shm")
-       stage.set_engine_outputs(engine_outputs)
-
-       # Check if there's a next stage to forward to
-       next_stage_id = stage_id + 1
-       if next_stage_id < len(self.stage_list):
-           next_stage: OmniStage = self.stage_list[next_stage_id]
-
-           # THIS IS WHERE STAGE TRANSITION HAPPENS
-           next_inputs = next_stage.process_engine_inputs(
-               self.stage_list,
-               [request_id_to_prompt[req_id]]
-           )
-
-           # Submit to next stage
-           task = {
-               "type": OmniStageTaskType.GENERATE,
-               "request_id": req_id,
-               "engine_inputs": next_inputs[0],
-               "sampling_params": sampling_params_list[next_stage_id],
-           }
-           next_stage.submit(task)
-   ```
-
-### How Stage Transitions Work
-
-The stage transition process follows these steps:
-
-1. **Stage Completion**: When a stage finishes processing a request, it stores outputs via `stage.set_engine_outputs(engine_outputs)`
-
-2. **Transition Detection**: The orchestrator checks if there's a next stage and calls `process_engine_inputs()` on it
-
-3. **Input Processing**: The `process_engine_inputs()` method in `OmniStage` (`omni_stage.py`) handles the transition:
-   ```python
-   def process_engine_inputs(
-       self, stage_list: list[Any], prompt: OmniTokensPrompt | TextPrompt = None
-   ) -> list[OmniTokensPrompt | TextPrompt]:
-       """Process engine inputs for this stage from upstream stage outputs."""
-
-       if self.custom_process_input_func is None:
-           # Default behavior: pass token IDs directly
-           # Extract outputs from source stage
-           source_stage_id = self.engine_input_source[0]
-           source_outputs = stage_list[source_stage_id].engine_outputs
-           # ... create OmniTokensPrompt from token_ids ...
-       else:
-           # Custom transition function (YOUR CODE HERE)
-           return self.custom_process_input_func(
-               stage_list,
-               self.engine_input_source,
-               prompt,
-               self.requires_multimodal_data
-           )
-   ```
-   - If `custom_process_input_func` is configured, it calls that function
-   - Otherwise, it uses default behavior (passing token IDs directly)
-
-4. **Custom Function Execution**: Your custom function receives:
-   - `stage_list`: List of all stage objects (to access upstream stage outputs)
-   - `engine_input_source`: List of source stage IDs (e.g., `[0]` for stage 0)
-   - `prompt`: Original prompt data (for preserving multimodal data)
-   - `requires_multimodal_data`: Whether multimodal data is required
-
-5. **Output Format**: The function must return a list of `OmniTokensPrompt` objects ready for the next stage
-
-### Data Structures in Stage Transitions
-
-Understanding the data structures is crucial for implementing stage transitions:
-
-**Input to your function:**
-- `stage_list[source_stage_id].engine_outputs`: List of `EngineCoreOutput` objects
-  - Each contains `outputs`: List of `RequestOutput` objects
-  - Each `RequestOutput` has:
-    - `token_ids`: Generated token IDs
-    - `multimodal_output`: Dict with keys like `"code_predictor_codes"`, etc.
-      - These are the hidden states or intermediate outputs from the model's forward pass
-    - `prompt_token_ids`: Original prompt token IDs
-
-**Output from your function:**
-- Must return `list[OmniTokensPrompt]` where each `OmniTokensPrompt` contains:
-  - `prompt_token_ids`: List[int] - Token IDs for the next stage
-  - `additional_information`: Dict[str, Any] - Optional metadata (e.g., embeddings, hidden states)
-  - `multi_modal_data`: Optional multimodal data if needed
-
-### How Model Outputs Are Stored
-
-The model's `forward()` method returns an `OmniOutput` object that contains:
-- `text_hidden_states`: Final hidden states for text generation
-- `multimodal_outputs`: Dict containing intermediate outputs
-
-These outputs are captured during the forward pass and stored in `multimodal_output` with specific keys:
-
-```python
-# In your model's forward() method (e.g., qwen3_omni.py)
-def forward(self, ...):
-    # ... processing ...
-
-    # For thinker stage: capture embeddings and hidden states
-    multimodal_outputs = {
-        "0": captured_embeddings,      # Layer 0 embeddings
-        "24": captured_hidden_states,  # Layer 24 hidden states
-        "tts_bos_embed": tts_bos_embed,
-        "tts_eos_embed": tts_eos_embed,
-        # ... other intermediate outputs ...
-    }
-
-    return OmniOutput(
-        text_hidden_states=hidden_states,
-        multimodal_outputs=multimodal_outputs,
-    )
-```
-
-These keys are then accessible in your stage transition function:
-```python
-# In stage_input_processors/qwen3_omni.py
-thinker_embeddings = output.multimodal_output["0"]  # Access by key
-thinker_hidden_states = output.multimodal_output["24"]
-```
-
-### Key Points
-
-1. **Accessing Upstream Outputs**: Use `stage_list[source_stage_id].engine_outputs` to get outputs from the source stage
-2. **Extracting Data**: Access `output.multimodal_output[key]` to get specific hidden states or intermediate results
-   - Keys are defined by your model's `forward()` method when it creates `multimodal_outputs`
-3. **Device Management**: Move tensors to appropriate devices (CPU for serialization, GPU for processing)
-4. **Shape Transformations**: Reshape tensors as needed for the next stage (e.g., flattening codec codes)
-5. **Batch Handling**: Process each request in the batch separately and return a list
-
-### Complete Flow Diagram
-
-<p align="center">
-  <picture>
-    <source media="(prefers-color-scheme: dark)" src="https://raw.githubusercontent.com/vllm-project/vllm-omni/refs/heads/main/docs/source/architecture/vllm-omni-dataflow-between-stages.png">
-    <img alt="Data Flow between stages" src="https://raw.githubusercontent.com/vllm-project/vllm-omni/refs/heads/main/docs/source/architecture/vllm-omni-dataflow-between-stages.png" width=55%>
-  </picture>
-</p>
-
-### Implementation Example
-
-Create stage transition processors in `vllm_omni/model_executor/stage_input_processors/your_model_name.py`:
-
-```python
-# qwen3_omni.py
-
-def thinker2talker(
-    stage_list: list[Any],
-    engine_input_source: list[int],
-    prompt: OmniTokensPrompt | TextPrompt | None = None,
-    requires_multimodal_data: bool = False,
-) -> list[OmniTokensPrompt]:
-    """
-    Process thinker outputs to create talker inputs.
-
-    Args:
-        stage_list: List of stage objects
-        engine_input_source: Source stage IDs (typically [0] for thinker)
-        prompt: Original prompt data
-
-    Returns:
-        List of OmniTokensPrompt for talker stage
-    """
-    source_stage_id = engine_input_source[0]
-    thinker_outputs = stage_list[source_stage_id].engine_outputs
-    talker_inputs = []
-
-    for thinker_output in thinker_outputs:
-        output = thinker_output.outputs[0]
-        # Extract thinker embeddings and hidden states
-        thinker_embeddings = output.multimodal_output["0"].float().clone().detach().cuda()
-        thinker_hidden_states = output.multimodal_output["24"].float().clone().detach().cuda()
-
-        info = {
-            "thinker_embeddings": thinker_embeddings,
-            "thinker_hidden_states": thinker_hidden_states,
-            "thinker_sequences": thinker_output.prompt_token_ids + output.token_ids,
-            "thinker_input_ids": thinker_output.prompt_token_ids,
-        }
-
-        talker_inputs.append(
-            OmniTokensPrompt(
-                prompt_token_ids=[0] * computed_length,
-                additional_information=info,
-                multi_modal_data=None,
-            )
-        )
-
-    return talker_inputs
-
-
-def talker2code2wav(
-    stage_list: list[Any],
-    engine_input_source: list[int],
-    prompt: OmniTokensPrompt | TextPrompt | None = None,
-    requires_multimodal_data: bool = False,
-) -> list[OmniTokensPrompt]:
-    """
-    Process talker outputs to create code2wav inputs.
-    """
-    source_stage_id = engine_input_source[0]
-    talker_outputs = stage_list[source_stage_id].engine_outputs
-    code2wav_inputs = []
-
-    for talker_output in talker_outputs:
-        output = talker_output.outputs[0]
-        # Extract codec codes
-        codec_codes = (
-            output.multimodal_output["code_predictor_codes"]
-            .to(torch.long)
-            .transpose(0, 1)
-            .cpu()
-            .to(torch.long)
-            .reshape(-1)
-            .tolist()
-        )
-
-        code2wav_inputs.append(
-            OmniTokensPrompt(
-                prompt_token_ids=codec_codes,
-                multi_modal_data=None,
-            )
-        )
-
-    return code2wav_inputs
-```
-
-## Testing
-
-For comprehensive testing guidelines, please refer to the [Test File Structure and Style Guide](../ci/tests_style.md).
-
-## Adding a Model Recipe
-
-After implementing and testing your model, please add a model recipe to the [vllm-project/recipes](https://github.com/vllm-project/recipes) repository. This helps other users understand how to use your model with vLLM-Omni.
-
-### What to Include
-
-Your recipe should include:
-
-1. **Model Overview**: Brief description of the model and its capabilities
-2. **Installation Instructions**: Step-by-step setup instructions including:
-   - Installing vllm-omni and dependencies
-   - Installing any additional required packages (e.g., xformers, diffusers)
-   - Any version requirements
-3. **Usage Examples**: Command-line examples demonstrating how to run the model
-4. **Configuration Details**: Important configuration parameters and their meanings
-
-### Example
-
-For reference, see the [LongCat recipe example](https://github.com/vllm-project/recipes/pull/179) which demonstrates the expected format and structure.
-
-### Recipe Location
-
-Create your recipe file in the appropriate directory structure:
-- For organization-specific models: `OrganizationName/ModelName.md`
-- For general models: `ModelName.md`
-
-The recipe should be a Markdown file that provides clear, reproducible instructions for users to get started with your model.
-
-## Summary
-
-Adding a new model to vLLM-Omni involves:
-
-1. **Create model directory structure** with stage implementations
-2. **Implement unified model class** that orchestrates stages
-3. **Register model** in `registry.py`
-4. **Create stage configuration** YAML file
-5. **Implement stage input processors** for stage transitions
-6. **Write tests** to verify functionality
-7. **Add model recipe** to the [vllm-project/recipes](https://github.com/vllm-project/recipes) repository (see [Adding a Model Recipe](#adding-a-model-recipe) section)
-
-### Qwen3-Omni Reference Files
-
-For a complete reference implementation, see:
-
-- **Main model**: `vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py`
-- **Thinker**: `vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_thinker.py`
-- **Talker**: `vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_talker.py`
-- **Code2Wav**: `vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_code2wav.py`
-- **Stage config**: `vllm_omni/model_executor/stage_configs/qwen3_omni_moe.yaml`
-- **Input processors**: `vllm_omni/model_executor/stage_input_processors/qwen3_omni.py`
-- **Registry**: `vllm_omni/model_executor/models/registry.py`
-- **Testing**: `vllm_omni/tests/e2e/offline_inference/test_qwen3_omni.py`
-
-For more information, see:
-- [Architecture Overview](../../design/architecture_overview.md)
-- [Supported Models](../../models/supported_models.md)
-- [Stage Configuration Guide](../../configuration/stage_configs.md)

docs/contributing/profiling.md

@@ -1,146 +0,0 @@
-# Profiling vLLM-Omni
-
-> **Warning:** Profiling incurs significant overhead. Use only for development and debugging, never in production.
-
-vLLM-Omni uses the PyTorch Profiler to analyze performance across both **multi-stage omni-modality models** and **diffusion models**.
-
-### 1. Set the Output Directory
-Before running any script, set this environment variable. The system detects this and automatically saves traces here.
-
-```bash
-export VLLM_TORCH_PROFILER_DIR=./profiles
-```
-
-### 2. Profiling Omni-Modality Models
-
-It is best to limit profiling to one iteration to keep trace files manageable.
-
-```bash
-export VLLM_PROFILER_MAX_ITERS=1
-```
-
-**Selective Stage Profiling**
-The profiler is default to function across all stages. But It is highly recommended to profile specific stages by passing the stages list, preventing from producing too large trace files:
-```python
-# Profile all stages
-omni_llm.start_profile()
-
-# Only profile Stage 1
-omni_llm.start_profile(stages=[1])
-```
-
-```python
-# Stage 0 (Thinker) and Stage 2 (Audio Decoder) for qwen omni
-omni_llm.start_profile(stages=[0, 2])
-```
-
-**Python Usage**: Wrap your generation logic with `start_profile()` and `stop_profile()`.
-
-```python
-from vllm_omni import omni_llm
-
-profiler_enabled = bool(os.getenv("VLLM_TORCH_PROFILER_DIR"))
-
-# 1. Start profiling if enabled
-if profiler_enabled:
-    omni_llm.start_profile(stages=[0])
-
-# Initialize generator
-omni_generator = omni_llm.generate(prompts, sampling_params_list, py_generator=args.py_generator)
-
-total_requests = len(prompts)
-processed_count = 0
-
-# Main Processing Loop
-for stage_outputs in omni_generator:
-
-    # ... [Output processing logic for text/audio would go here] ...
-
-    # Update count to track when to stop profiling
-    processed_count += len(stage_outputs.request_output)
-
-    # 2. Check if all requests are done to stop the profiler safely
-    if profiler_enabled and processed_count >= total_requests:
-        print(f"[Info] Processed {processed_count}/{total_requests}. Stopping profiler inside active loop...")
-
-        # Stop the profiler while workers are still active
-        omni_llm.stop_profile()
-
-        # Wait for traces to flush to disk
-        print("[Info] Waiting 30s for workers to write trace files to disk...")
-        time.sleep(30)
-        print("[Info] Trace export wait time finished.")
-
-omni_llm.close()
-```
-
-
-**Examples**:
-
-1. **Qwen2.5-Omni**:  [https://github.com/vllm-project/vllm-omni/blob/main/examples/offline_inference/qwen2_5_omni/end2end.py](https://github.com/vllm-project/vllm-omni/blob/main/examples/offline_inference/qwen2_5_omni/end2end.py)
-
-2. **Qwen3-Omni**:   [https://github.com/vllm-project/vllm-omni/blob/main/examples/offline_inference/qwen3_omni/end2end.py](https://github.com/vllm-project/vllm-omni/blob/main/examples/offline_inference/qwen3_omni/end2end.py)
-
-
-### 3. Profiling diffusion models
-
-Diffusion profiling is End-to-End, capturing encoding, denoising loops, and decoding.
-
-**CLI Usage:**
-```python
-
-python image_to_video.py \
-    --model Wan-AI/Wan2.2-I2V-A14B-Diffusers \
-    --image qwen-bear.png \
-    --prompt "A cat playing with yarn, smooth motion" \
-    \
-    # Minimize Spatial Dimensions (Optional but helpful):
-    #    Drastically reduces memory usage so the profiler doesn't
-    #    crash due to overhead, though for accurate performance
-    #    tuning you often want target resolutions.
-    --height 48 \
-    --width 64 \
-    \
-    # Minimize Temporal Dimension (Frames):
-    #    Video models process 3D tensors (Time, Height, Width).
-    #    Reducing frames to the absolute minimum (2) keeps the
-    #    tensor size small, ensuring the trace file doesn't become
-    #    multi-gigabytes in size.
-    --num-frames 2 \
-    \
-    # Minimize Iteration Loop (Steps):
-    #    This is the most critical setting for profiling.
-    #    Diffusion models run the same loop X times.
-    #    Profiling 2 steps gives you the exact same performance
-    #    data as 50 steps, but saves minutes of runtime and
-    #    prevents the trace viewer from freezing.
-    --num-inference-steps 2 \
-    \
-    --guidance-scale 5.0 \
-    --guidance-scale-high 6.0 \
-    --boundary-ratio 0.875 \
-    --flow-shift 12.0 \
-    --fps 16 \
-    --output i2v_output.mp4
-
-```
-
-**Examples**:
-
-1. **Qwen image edit**:  [https://github.com/vllm-project/vllm-omni/blob/main/examples/offline_inference/image_to_image/image_edit.py](https://github.com/vllm-project/vllm-omni/blob/main/examples/offline_inference/image_to_image/image_edit.py)
-
-2. **Wan-AI/Wan2.2-I2V-A14B-Diffusers**:   [https://github.com/vllm-project/vllm-omni/tree/main/examples/offline_inference/image_to_video](https://github.com/vllm-project/vllm-omni/tree/main/examples/offline_inference/image_to_video)
-
-### 4. Analyzing Omni Traces
-
-Output files are saved to your configured ```VLLM_TORCH_PROFILER_DIR```.
-
-**Output**
-**Chrome Trace** (```.json.gz```): Visual timeline of kernels and stages. Open in Perfetto UI.
-
-**Viewing Tools:**
-
-- [Perfetto](https://ui.perfetto.dev/)(recommended)
-- ```chrome://tracing```(Chrome only)
-
-**Note**: vLLM-Omni reuses the PyTorch Profiler infrastructure from vLLM. See the official vLLM profiler documentation:  [vLLM Profiling Guide](https://docs.vllm.ai/en/stable/contributing/profiling/)

docs/design/architecture_overview.md

@@ -28,8 +28,8 @@ According to analysis for current popular open-source models, most of them have
 
 <p align="center">
   <picture>
-    <source media="(prefers-color-scheme: dark)" src="https://raw.githubusercontent.com/vllm-project/vllm-omni/refs/heads/main/docs/source/architecture/dit-main-architecture.png">
-    <img alt="Qwen-Image" src="https://raw.githubusercontent.com/vllm-project/vllm-omni/refs/heads/main/docs/source/architecture/dit-main-architecture.png" width=30%>
+    <source media="(prefers-color-scheme: dark)" src="https://raw.githubusercontent.com/vllm-project/vllm-omni/refs/heads/main/docs/source/architecture/ar-main-architecture.png">
+    <img alt="Qwen-Image" src="https://raw.githubusercontent.com/vllm-project/vllm-omni/refs/heads/main/docs/source/architecture/ar-main-architecture.png" width=30%>
   </picture>
 </p>
 
@@ -38,8 +38,8 @@ According to analysis for current popular open-source models, most of them have
 
 <p align="center">
   <picture>
-    <source media="(prefers-color-scheme: dark)" src="https://raw.githubusercontent.com/vllm-project/vllm-omni/refs/heads/main/docs/source/architecture/ar-main-architecture.png">
-    <img alt="Bagel" src="https://raw.githubusercontent.com/vllm-project/vllm-omni/refs/heads/main/docs/source/architecture/ar-main-architecture.png" width=30%>
+    <source media="(prefers-color-scheme: dark)" src="https://raw.githubusercontent.com/vllm-project/vllm-omni/refs/heads/main/docs/source/architecture/dit-main-architecture.png">
+    <img alt="Bagel" src="https://raw.githubusercontent.com/vllm-project/vllm-omni/refs/heads/main/docs/source/architecture/dit-main-architecture.png" width=30%>
   </picture>
 </p>
 
@@ -64,13 +64,13 @@ According to analysis for current popular open-source models, most of them have
 
 ## Key Components
 
-| Component         | Description                                                                                                                              |
-| ----------------- | ---------------------------------------------------------------------------------------------------------------------------------------- |
-| **OmniRouter**    | provide an intelligent router for Omni-modality requests dispatch                                                                        |
-| **EntryPoints**   | define the APIs for offline/online serving (APIServer, Omni/AsyncOmni) and provide the OmniStage abstraction for different AR/DiT stages |
-| **AR**            | adapted for omni-modality models while inheriting efficient features from vLLM, such as cache management                                 |
-| **Diffusion**     | natively implemented and optimized using acceleration components                                                                         |
-| **OmniConnector** | supports fully disaggregation based on E/P/D/G (Encoding/Processing/Decoding/Generation) disaggregation across stages                    |
+| Component | Description |
+|-----------|-------------|
+| **OmniRouter** | provide an intelligent router for Omni-modality requests dispatch |
+| **EntryPoints** | define the APIs for offline/online serving (APIServer, Omni/AsyncOmni) and provide the OmniStage abstraction for different AR/DiT stages |
+| **AR** | adapted for omni-modality models while inheriting efficient features from vLLM, such as cache management |
+| **Diffusion** | natively implemented and optimized using acceleration components |
+| **OmniConnector** | supports fully disaggregation based on E/P/D/G (Encoding/Processing/Decoding/Generation) disaggregation across stages |
 
 Disaggregated stages are managed through configuration, such as in the Qwen3-Omni example, where stages like Thinker, Talker, and Code2wav are defined as separate OmniStage instances with specific resources and input/output type.
 
@@ -192,4 +192,4 @@ curl -sS -X POST http://localhost:8091/v1/chat/completions \
 }
 ```
 
-For more usages, please refer to [examples](https://github.com/vllm-project/vllm-omni/tree/main/examples).
+For more usages, please refer to [examples](../user_guide/examples/).

docs/design/feature/async_chunk_design.md

@@ -1,209 +0,0 @@
-# Async Chunk Design
-
-## Table of Contents
-
-1. [Overview](#overview)
-2. [Performance](#performance)
-3. [Architecture](#architecture)
-4. [Configuration](#configuration)
-5. [Related Files](#related-files)
-
-## Overview
-
-The `async_chunk` feature enables asynchronous, chunked processing of data across multiple stages in a multi-stage pipeline (e.g., Qwen3-Omni with Thinker → Talker → Code2Wav stages). Instead of waiting for a complete stage output before forwarding to the next stage, this feature allows stages to process and forward data in chunks as it becomes available, significantly reducing latency and improving throughput.
-
-**Chunk Size Definition**
-
-- **Prefill Phase**: `chunk_size = num_scheduled_tokens` for chunked prefill processing
-- **Decode Phase**: `chunk_size = num_scheduled_tokens = 1 ` for per-token streaming
-
-For qwen3-omni:
-- **Thinker → Talker**: Per decode step (typically chunk_size=1)
-- **Talker → Code2Wav**: Accumulated to code2wav chunk_size(default=25, currently only support default, will support chunk_size soon) before sending
-- **Code2Wav**: Streaming decode with code2wav chunk_size
-
-With `async_chunk`:
-- Stages can start processing as soon as chunks are available
-- Overlapping execution across stages
-- Reduced latency and improved throughput
-- Better resource utilization
-- Async scheduling: Chunk IO (get/put) overlaps with compute via background threads so the scheduler is not blocked waiting for chunks
-
-## Performance
-1. **Reduced Latency**: Next stage can start processing immediately
-2. **Streaming Support**: Enables streaming for audio generation
-3. **IO-Compute Overlap**: Chunk retrieval happens asynchronously while other requests compute
-4. **Non-blocking Scheduler**: Requests waiting for chunks don't block the entire scheduler
-5. **Code2Wav Batch Inference**: Supports batched processing in code2wav stage
-
-| Input     | Output           | Async_chunk enabled | Code2Wav batch size | Max_Concurrency | Prompts | Mean E2E   | Mean TTFT  | Mean TPOT | Mean TTFP   | Mean RTF | Mean ITL |
-|-----------|------------------|---------------------|---------------------|----------------|---------|------------|------------|-----------|-------------|----------|----------|
-| text 100  | text 100+audio   | False               | 1                   | 1              | 50      | 6581.80    | 43.22      | 8.31      | 6459.34     | 0.24     | 8.22     |
-| text 100  | text 100+audio   | False               | 1                   | 4              | 50      | 7398.63    | 67.57      | 9.14      | 7285.35     | 0.27     | 9.05     |
-| text 100  | text 100+audio   | False               | 1                   | 10             | 50      | 13522.99   | 131.82     | 12.72     | 13410.44    | 0.49     | 12.60    |
-| text 100  | text 100+audio   | False               | 64                  | 1              | 50      | 6505.13    | 43.14      | 8.52      | 6395.40     | 0.24     | 8.44     |
-| text 100  | text 100+audio   | False               | 64                  | 4              | 50      | 7668.15    | 51.15      | 9.36      | 7562.37     | 0.28     | 9.27     |
-| text 100  | text 100+audio   | False               | 64                  | 10             | 50      | 9516.18    | 138.06     | 14.75     | 9409.26     | 0.34     | 14.60    |
-| text 100  | text 100+audio   | True                | 1                   | 1              | 50      | 6179.79    | 44.58      | 8.69      | 522.99      | 0.22     | 8.60     |
-| text 100  | text 100+audio   | True                | 1                   | 4              | 50      | 7692.69    | 103.96     | 10.22     | 785.85      | 0.29     | 10.12    |
-| text 100  | text 100+audio   | True                | 1                   | 10             | 50      | 11152.71   | 685.60     | 17.64     | 1628.88     | 0.41     | 17.62    |
-
-
-Performance data collected on H800 GPUs through comprehensive benchmarking with cudagraph enabled. text input uses random dataset.
-
-Enabling **async_chunk** (False→True) sharply reduces time-to-first-audio (TTFP)—e.g. ~92% at concurrency 1 (6.5s→0.52s)—and improves E2E latency (e.g. ~6% at conc 1, ~17% at conc 10). RTF (Real Time Factor) also improves with async_chunk on (e.g. ~8% at conc 1: 0.24→0.22, ~16% at conc 10: 0.49→0.41). Enabling **Code2Wav batch size 64** (vs 1) improves E2E and TTFP at higher concurrency when async_chunk is off (e.g. ~30% at conc 10: 13.5s→9.5s E2E, 13.4s→9.4s TTFP).
-
-<p align="center">
-  <picture>
-    <source media="(prefers-color-scheme: dark)" src="https://raw.githubusercontent.com/vllm-project/vllm-omni/refs/heads/main/docs/source/performance/qwen3-omni_ttfp_performance.png">
-    <img alt="TTFP Performance Data Comparison" src="https://raw.githubusercontent.com/vllm-project/vllm-omni/refs/heads/main/docs/source/performance/qwen3-omni_ttfp_performance.png" width=100%>
-  </picture>
-</p>
-
-<p align="center">
-  <picture>
-    <source media="(prefers-color-scheme: dark)" src="https://raw.githubusercontent.com/vllm-project/vllm-omni/refs/heads/main/docs/source/performance/qwen3-omni_rtf_performance.png">
-    <img alt="RTF Performance Data Comparison" src="https://raw.githubusercontent.com/vllm-project/vllm-omni/refs/heads/main/docs/source/performance/qwen3-omni_rtf_performance.png" width=100%>
-  </picture>
-</p>
-
-## Architecture
-### Data Flow
-
-#### Sequential Flow
-<p align="center">
-  <picture>
-    <source media="(prefers-color-scheme: dark)" src="https://raw.githubusercontent.com/vllm-project/vllm-omni/refs/heads/main/docs/source/architecture/qwen3-omni-non-async-chunk.png">
-    <img alt="Data Flow between stages" src="https://raw.githubusercontent.com/vllm-project/vllm-omni/refs/heads/main/docs/source/architecture/qwen3-omni-non-async-chunk.png" width=100%>
-  </picture>
-</p>
-
-#### Async Chunk Flow
-
-<p align="center">
-  <picture>
-    <source media="(prefers-color-scheme: dark)" src="https://raw.githubusercontent.com/vllm-project/vllm-omni/refs/heads/main/docs/source/architecture/qwen3-omni-async-chunk.png">
-    <img alt="Data Flow between stages" src="https://raw.githubusercontent.com/vllm-project/vllm-omni/refs/heads/main/docs/source/architecture/qwen3-omni-async-chunk.png" width=100%>
-  </picture>
-</p>
-
-### Async Chunk architecture
-<p align="center">
-  <picture>
-    <source media="(prefers-color-scheme: dark)" src="https://raw.githubusercontent.com/vllm-project/vllm-omni/refs/heads/main/docs/source/architecture/async-chunk-architecture.png">
-    <img alt="Async Chunk Architecture" src="https://raw.githubusercontent.com/vllm-project/vllm-omni/refs/heads/main/docs/source/architecture/async-chunk-architecture.png" width=100%>
-  </picture>
-</p>
-
-
-### Key Components
-
-1. **OmniConnector**: Inter-stage data transport only
-   - Shared memory or other IPC mechanisms
-   - **Transport-only API**: `put(from_stage, to_stage, put_key, data)` and `get(from_stage, to_stage, get_key)` (optionally with timeout)
-   - **No request-specific state**: Connector does not track put_requests, get_requests, request_payload, finished_requests, or other request-bound metadata; it only performs put/get operations
-   - Chunk keys and request/chunk lifecycle are managed by **OmniChunkTransferAdapter**
-
-2. **Transfer Adapter Layer**: Extensible abstraction for managing data transfer via connectors
-   - **OmniTransferAdapterBase**: Base class with background **recv_loop** and **save_loop** threads;
-   - **OmniChunkTransferAdapter**: Chunk-specific implementation that owns the full chunk lifecycle when async_chunk is enabled
-     - **Chunk ID and key construction**: Builds keys like `{req_id}_{stage_id}_{chunk_id}` for put/get
-     - **Async get**: `load_async(request)` enqueues the request; background **recv_loop** polls the connector (non-blocking); when data is available, updates the request and marks it in `_finished_load_reqs`; scheduler calls `get_finished_requests()` to learn which requests have chunks ready
-     - **Async put**: `save_async(pooling_output, request)` invokes `custom_process_next_stage_input_func` in the main thread to build the payload, then enqueues a save task; background **save_loop** performs `connector.put()`; payload processing and chunk accumulation (e.g. code2wav chunk_size) remain in the main thread
-
-3. **Stage Input Processors**: Custom functions that process stage outputs into chunks for different models
-   - Receive **transfer_manager** (OmniChunkTransferAdapter)
-   - Qwen3-omni reference: `thinker2talker_async_chunk`, `talker2code2wav_async_chunk`
-
-4. **Schedulers**: Modified to handle chunk-based scheduling with async IO-compute overlap
-   - `OmniARScheduler`: For autoregressive stages
-   - `OmniGenerationScheduler`: For generation stages
-   - Both schedulers use **OmniChunkTransferAdapter** and **before/after** hooks around `super().schedule()`:
-     - **Before** `super().schedule()`: `process_pending_chunks(waiting, running)` moves requests waiting for chunks to `WAITING_FOR_CHUNK`, enqueues load tasks for background polling
-     - **After** `super().schedule()`: `restore_queues(waiting, running)` restores requests with ready chunks back to waiting/running, `postprocess_scheduler_output(scheduler_output)` attaches cached additional_information, clears chunk-ready flags
-   - **put_chunk** `save_async(pooler_output, request)`; **get_chunk** / **get_chunk_for_generation** `load_async(request)`
-
-5. **Model Runners**: Handle chunk processing
-   - `OmniGPUModelRunner`: Processes chunks in AR stages
-   - `GPUGenerationModelRunner`: Processes chunks in generation stages
-     - Uses `ubatch_slices` from `get_forward_context()` to track per-request sequence lengths in batched inference
-     - Reuses `ubatch_slices_padded` for code2wav batching to properly split batch outputs
-     - Handles list-type multimodal outputs: iterates through requests and assigns corresponding tensor to each
-     - Improved request state management: removes unscheduled and finished requests from input batch
-
-6. **Model Implementation**: Model-specific chunk handling
-   - `Qwen3OmniMoeForConditionalGeneration`: Main model with async_chunk support
-     - **Code2Wav stage batching**: Uses `ubatch_slices` to construct batched codec codes tensor `[batch_size, 16, max_seq_len]`
-     - **Batch output handling**: `generate_audio()` returns `list[torch.Tensor]`, one audio tensor per request
-     - **Multimodal outputs**: Returns list of audio tensors for batch processing instead of single concatenated tensor
-   - `Qwen3OmniCode2WavDecoder`: Audio generation model
-     - `chunked_decode()` and `chunked_decode_streaming()`: Return `list[torch.Tensor]` (one per request)
-     - Uses `ubatch_slices` to split batched waveform output into per-request audio chunks
-     - Each request gets correctly sized audio based on its code sequence length: `waveform[:, :, :code_seq_len * total_upsample]`
-
-7. **Request status**: `RequestStatus.WAITING_FOR_CHUNK` is added via patch (e.g. in `vllm_omni/patch.py`) so requests waiting for a chunk are not scheduled by the base vLLM scheduler until the chunk is ready.
-
-## Configuration
-
-Enable async_chunk in stage configuration YAML:
-
-```yaml
-async_chunk: true
-stage_args:
-  - stage_id: 0
-    engine_args:
-      custom_process_next_stage_input_func: vllm_omni.model_executor.stage_input_processors.qwen3_omni.thinker2talker_async_chunk
-  - stage_id: 1
-    engine_args:
-      custom_process_next_stage_input_func: vllm_omni.model_executor.stage_input_processors.qwen3_omni.talker2code2wav_async_chunk
-```
-
-### Stage Configuration
-
-- `async_chunk: bool`: Enable/disable async chunk mode
-- `custom_process_next_stage_input_func: str`: Path to custom chunk processing function; receives `(transfer_manager, pooling_output, request)`. For qwen3-omni: `thinker2talker_async_chunk`, `talker2code2wav_async_chunk`
-- `stage_connector_config: dict`: Connector configuration
-- `worker_type: str`: Model type, e.g. `"ar"` or `"generation"` (used by OmniChunkTransferAdapter for mode-specific payload handling)
-- `max_batch_size: int`: Maximum batch size for the stage
-
-
-### Connector Configuration
-
-```yaml
-connectors:
-  - from_stage: 0
-    to_stage: 1
-    spec:
-      name: SharedMemoryConnector
-      extra:
-        stage_id: 0
-```
-
-### Code2Wav Batch Configuration
-
-For optimal performance with async_chunk, the code2wav stage should be configured with batching:
-
-```yaml
-stage_args:
-  - stage_id: 2  # code2wav stage
-    runtime:
-      devices: "1"
-      max_batch_size: 64  # Enables batched audio generation
-    engine_args:
-      model_stage: code2wav
-```
-
-## Related Files
-
-- `vllm_omni/model_executor/stage_input_processors/qwen3_omni.py`: Chunk processing functions (receive `transfer_manager` as first param)
-- `vllm_omni/distributed/omni_connectors/transfer_adapter/base.py`: OmniTransferAdapterBase (recv_loop, save_loop, load_async, save_async)
-- `vllm_omni/distributed/omni_connectors/transfer_adapter/chunk_transfer_adapter.py`: OmniChunkTransferAdapter (process_pending_chunks, restore_queues, postprocess_scheduler_output)
-- `vllm_omni/distributed/omni_connectors/connectors/shm_connector.py`: SharedMemoryConnector (transport-only put/get)
-- `vllm_omni/core/sched/omni_ar_scheduler.py`: AR scheduler with chunk_transfer_adapter
-- `vllm_omni/core/sched/omni_generation_scheduler.py`: Generation scheduler with same async chunk pattern
-- `vllm_omni/worker/gpu_model_runner.py`: Model runner with chunk handling
-- `vllm_omni/worker/gpu_generation_model_runner.py`: Generation model runner with batch output handling and ubatch_slices support
-- `vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py`: Model implementation with code2wav batching
-- `vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_code2wav.py`: Code2wav decoder with batch support
-- `vllm_omni/engine/arg_utils.py`: Configuration definitions (async_chunk, worker_type)
-- `vllm_omni/config/model.py`: Model config with async_chunk field

docs/design/feature/cache_dit.md

@@ -1,286 +0,0 @@
-# Cache-DiT
-
-This section describes how to add cache-dit acceleration to a new diffusion pipeline. We use the Qwen-Image pipeline and LongCat-Image pipeline as reference implementations.
-
----
-
-## Table of Contents
-
-- [Overview](#overview)
-- [Standard Models: Automatic Support](#standard-models-automatic-support)
-- [Custom Architectures: Writing Custom Implementation](#custom-architectures-writing-custom-implementation)
-- [Testing](#testing)
-- [Troubleshooting](#troubleshooting)
-- [Reference Implementations](#reference-implementations)
-- [Summary](#summary)
-
----
-
-## Overview
-
-### What is Cache-DiT?
-
-Cache-DiT is an acceleration library for Diffusion Transformers (DiT) that caches intermediate computation results across denoising steps. The core insight is that adjacent denoising steps often produce similar intermediate features, so we can skip redundant computations by reusing cached results.
-
-The library supports three main caching strategies:
-
-- **DBCache:** Dynamic block-level caching that selectively computes or caches transformer blocks based on residual differences
-- **TaylorSeer:** Calibration-based prediction that estimates block outputs using Taylor expansion
-- **SCM (Step Computation Masking):** Dynamic step skipping based on configurable policies
-
-### Architecture
-
-vLLM-omni integrates cache-dit through the `CacheDiTBackend` class, which provides a unified interface for managing cache-dit acceleration on diffusion models.
-
-| Method/Class | Purpose | Behavior |
-|--------------|---------|----------|
-| [`CacheDiTBackend`](https://docs.vllm.ai/projects/vllm-omni/en/latest/api/vllm_omni/diffusion/cache/#vllm_omni.diffusion.cache.CacheBackend) | Unified backend interface | Automatically handles enabler selection and cache refresh |
-| [`enable_cache_for_dit()`](https://docs.vllm.ai/projects/vllm-omni/en/latest/api/vllm_omni/diffusion/cache/cache_dit_backend/#vllm_omni.diffusion.cache.cache_dit_backend.enable_cache_for_dit) | Apply caching to transformer | Configures DBCache on transformer blocks |
-
-**Key APIs from Cache-DiT:**
-
-[Cache-DiT API Reference](https://cache-dit.readthedocs.io/en/latest/user_guide/CACHE_API/)
-
-| API | Description |
-|-----|-------------|
-| `BlockAdapter` | Core abstraction for applying cache-dit to transformers. Specifies transformer module(s), block list(s), and forward signature pattern(s). |
-| `ForwardPattern` | Defines block forward signature patterns: `Pattern_0`, `Pattern_1`, `Pattern_2` |
-| `ParamsModifier` | Per-transformer or per-block-list cache configuration customization |
-| `DBCacheConfig` | Configuration for DBCache parameters (warmup steps, cached steps, thresholds) |
-| `refresh_context()` | Update cache context | Called when `num_inference_steps` changes |
-
----
-
-## Standard Models: Automatic Support
-
-Most DiT models follow this pattern:
-- Single transformer with one `ModuleList` of blocks
-- Standard forward signature
-- Compatible with cache-dit's automatic detection
-
-**Examples:** Qwen-Image, Z-Image
-
-For standard single-transformer models, **no code changes are needed**. The `CacheDiTBackend` automatically uses `enable_cache_for_dit()`:
-
-```python
-from vllm_omni import Omni
-
-# Works automatically for standard models
-omni = Omni(
-    model="Qwen/Qwen-Image",  # Standard single-transformer model
-    cache_backend="cache_dit",
-    cache_config={
-        "Fn_compute_blocks": 1,
-        "Bn_compute_blocks": 0,
-        "max_warmup_steps": 4,
-    }
-)
-```
-
-**What happens automatically:**
-
-```python
-def enable_cache_for_dit(pipeline: Any, cache_config: Any) -> Callable[[int], None]:
-    """Default enabler for standard single-transformer DiT models."""
-
-    # Build cache configuration
-    db_cache_config = DBCacheConfig(
-        num_inference_steps=None,  # Will be set during first inference
-        Fn_compute_blocks=cache_config.Fn_compute_blocks,
-        Bn_compute_blocks=cache_config.Bn_compute_blocks,
-        max_warmup_steps=cache_config.max_warmup_steps,
-        max_cached_steps=cache_config.max_cached_steps,
-        max_continuous_cached_steps=cache_config.max_continuous_cached_steps,
-        residual_diff_threshold=cache_config.residual_diff_threshold,
-    )
-
-    # Enable cache-dit on transformer
-    cache_dit.enable_cache(
-        pipeline.transformer,
-        cache_config=db_cache_config,
-    )
-
-    # Return refresh function for dynamic num_inference_steps updates
-    def refresh_cache_context(pipeline: Any, num_inference_steps: int, verbose: bool = True):
-        cache_dit.refresh_context(pipeline.transformer, num_inference_steps=num_inference_steps, verbose=verbose)
-
-    return refresh_cache_context
-```
-
----
-
-## Custom Architectures: Writing Custom Implementation
-
-Some models require custom handling:
-
-- **Dual-transformer:** Models with separate high-noise and low-noise transformers (e.g., Wan2.2)
-- **Multi-block-list:** Models with multiple block lists in one transformer (e.g., LongCatImage with `transformer_blocks` + `single_transformer_blocks`)
-- **Special forward patterns:** Models with non-standard block execution patterns
-
-### Example 1: Dual-Transformer Model (Wan2.2)
-
-Wan2.2 uses two transformers: one for high-noise steps and one for low-noise steps.
-
-**Key difference:** Use `BlockAdapter` to wrap multiple transformers with separate configurations.
-
-```python
-# Standard: cache_dit.enable_cache(pipeline.transformer, ...)
-# Custom: Use BlockAdapter to handle multiple transformers
-cache_dit.enable_cache(
-    BlockAdapter(
-        transformer=[pipeline.transformer, pipeline.transformer_2],  # Multiple transformers
-        blocks=[pipeline.transformer.blocks, pipeline.transformer_2.blocks],
-        forward_pattern=[ForwardPattern.Pattern_2, ForwardPattern.Pattern_2],
-        params_modifiers=[
-            ParamsModifier(...),  # Config for high-noise transformer
-            ParamsModifier(...),  # Config for low-noise transformer (different params)
-        ],
-    ),
-    cache_config=db_cache_config,
-)
-```
-
-**Key difference:** `refresh_context` must be called on each transformer separately.
-
-```python
-# Standard: cache_dit.refresh_context(pipeline.transformer, num_inference_steps=N)
-# Custom: Refresh each transformer with its own step count
-def refresh_cache_context(pipeline, num_inference_steps, verbose=True):
-    high_steps, low_steps = _split_inference_steps(num_inference_steps)
-    cache_dit.refresh_context(pipeline.transformer, num_inference_steps=high_steps, ...)
-    cache_dit.refresh_context(pipeline.transformer_2, num_inference_steps=low_steps, ...)
-```
-
-### Example 2: Multi-Block-List Model (LongCatImage)
-
-LongCatImage has a single transformer with two block lists: `transformer_blocks` and `single_transformer_blocks`.
-
-**Key difference:** Use `BlockAdapter` to specify multiple block lists within one transformer.
-
-```python
-# Standard: cache_dit.enable_cache(pipeline.transformer, ...)
-#   - Automatically detects single block list
-# Custom: Use BlockAdapter to specify multiple block lists
-cache_dit.enable_cache(
-    BlockAdapter(
-        transformer=pipeline.transformer,  # Single transformer
-        blocks=[
-            pipeline.transformer.transformer_blocks,        # Block list 1
-            pipeline.transformer.single_transformer_blocks, # Block list 2
-        ],
-        forward_pattern=[ForwardPattern.Pattern_1, ForwardPattern.Pattern_1],
-        params_modifiers=[modifier],
-    ),
-    cache_config=db_cache_config,
-)
-```
-
-> **Note:** For single transformer with multiple block lists, `refresh_context` works the same as standard models.
-
-### Registering Custom Implementations
-
-After writing your custom enabler, register it in `CUSTOM_DIT_ENABLERS` in `vllm_omni/diffusion/cache/cache_dit_backend.py`:
-
-```python
-CUSTOM_DIT_ENABLERS = {
-    "Wan22Pipeline": enable_cache_for_wan22,
-    "LongCatImagePipeline": enable_cache_for_longcat_image,
-    "YourCustomPipeline": enable_cache_for_your_model,  # Add here
-}
-```
-
----
-
-## Testing
-
-After adding cache-dit support, test with:
-
-```python
-from vllm_omni import Omni
-from vllm_omni.inputs.data import OmniDiffusionSamplingParams
-
-# Test your custom model
-omni = Omni(
-    model="your-model-name",
-    cache_backend="cache_dit",
-    cache_config={
-        "Fn_compute_blocks": 1,
-        "Bn_compute_blocks": 0,
-        "max_warmup_steps": 4,
-        "residual_diff_threshold": 0.24,
-    }
-)
-
-images = omni.generate(
-    "a beautiful landscape",
-    OmniDiffusionSamplingParams(num_inference_steps=50),
-)
-```
-
-**Verify:**
-
-1. Cache is applied (check logs for "Cache-dit enabled successfully on xxx")
-2. Performance improvement (should be around 1.5x-2x faster)
-3. Image quality (compare with `cache_backend=None`)
-
----
-
-## Troubleshooting
-
-### Issue: Cache not applied
-
-**Symptoms:** No speedup observed, no cache-related log messages.
-
-**Causes & Solutions:**
-
-- **Enabler not registered:**
-
-**Problem:** Pipeline name not in `CUSTOM_DIT_ENABLERS` registry.
-
-**Solution:** Verify `pipeline.__class__.__name__` matches the registry key and add your enabler to `CUSTOM_DIT_ENABLERS`.
-
-### Issue: Quality degradation
-
-**Symptoms:** Generated images have artifacts or lower quality compared to non-cached inference.
-
-**Causes & Solutions:**
-
-- **Cache parameters too aggressive:**
-
-**Solution:**
-```python
-cache_config={
-    "residual_diff_threshold": 0.12,  # Lower from 0.24 (try 0.12-0.18)
-    "max_warmup_steps": 6,            # Increase from 4 (try 6-8)
-    "max_continuous_cached_steps": 2, # Reduce if higher
-}
-```
-
-Check the [user guide for cache_dit](../../user_guide/diffusion/cache_dit_acceleration.md) for more adjustable parameters.
-
----
-
-## Reference Implementations
-
-Complete examples in the codebase:
-
-| Model | Path | Pattern | Notes |
-|-------|------|---------|-------|
-| **Standard DiT** | [`cache_dit_backend.py::enable_cache_for_dit`](https://docs.vllm.ai/projects/vllm-omni/en/latest/api/vllm_omni/diffusion/cache/cache_dit_backend/#vllm_omni.diffusion.cache.cache_dit_backend.enable_cache_for_dit) | Default enabler | Single transformer, automatic |
-| **Wan2.2** | [`cache_dit_backend.py::enable_cache_for_wan22`](https://docs.vllm.ai/projects/vllm-omni/en/latest/api/vllm_omni/diffusion/cache/cache_dit_backend/#vllm_omni.diffusion.cache.cache_dit_backend.enable_cache_for_wan22) | Dual-transformer | Separate high/low noise transformers |
-| **LongCat** | [`cache_dit_backend.py::enable_cache_for_longcat_image`](https://docs.vllm.ai/projects/vllm-omni/en/latest/api/vllm_omni/diffusion/cache/cache_dit_backend/#vllm_omni.diffusion.cache.cache_dit_backend.enable_cache_for_longcat_image) | Multi-block-list | Two block lists in one transformer |
-| **BAGEL** | [`cache_dit_backend.py::enable_cache_for_bagel`](https://docs.vllm.ai/projects/vllm-omni/en/latest/api/vllm_omni/diffusion/cache/cache_dit_backend/#vllm_omni.diffusion.cache.cache_dit_backend.enable_cache_for_bagel) | Omni model | Complex architecture |
-
----
-
-## Summary
-
-Adding cache-dit support:
-
-1. ✅ **Check model type** - Standard models work automatically, custom architectures need enablers
-2. ✅ **Write enabler** (if needed) - Use `BlockAdapter` for complex architectures
-3. ✅ **Register enabler** (if needed) - Add to `CUSTOM_DIT_ENABLERS` dictionary
-4. ✅ **Return refresh function** (if needed) - Handle `num_inference_steps` changes
-5. ✅ **Test** - Verify with `cache_backend="cache_dit"`
-
-For most models, the default enabler is sufficient. Only write custom enablers for complex architectures!

docs/design/feature/cfg_parallel.md

@@ -1,249 +0,0 @@
-# CFG-Parallel
-
-This section describes how to add CFG-Parallel (Classifier-Free Guidance Parallel) to a diffusion pipeline. We use the Qwen-Image pipeline as the reference implementation.
-
----
-
-## Table of Contents
-
-- [Overview](#overview)
-- [Step-by-Step Implementation](#step-by-step-implementation)
-- [Customization](#customization)
-- [Testing](#testing)
-- [Troubleshooting](#troubleshooting)
-- [Reference Implementations](#reference-implementations)
-- [Summary](#summary)
-
----
-
-## Overview
-
-### What is CFG-Parallel?
-
-In standard Classifier-Free Guidance, each diffusion step requires two forward passes through the transformer:
-
-1. **Positive/Conditional**: Guided by the text prompt
-2. **Negative/Unconditional**: Typically using empty or negative prompt
-
-CFG-Parallel eliminates this bottleneck by distributing the two forward passes across different GPU ranks, allowing them to execute simultaneously rather than sequentially.
-
-### Architecture
-
-vLLM-omni provides `CFGParallelMixin` that encapsulates all CFG parallel logic. Pipelines inherit from this mixin and implement a `diffuse()` method that orchestrates the denoising loop.
-
-| Method | Purpose | Automatic Behavior |
-|--------|---------|-------------------|
-| [`predict_noise_maybe_with_cfg()`](https://docs.vllm.ai/projects/vllm-omni/en/latest/api/vllm_omni/diffusion/distributed/cfg_parallel/) | Predict noise with CFG | Detects parallel mode, distributes computation, gathers results |
-| [`scheduler_step_maybe_with_cfg()`](https://docs.vllm.ai/projects/vllm-omni/en/latest/api/vllm_omni/diffusion/distributed/cfg_parallel/) | Step scheduler with sync | Rank 0 steps, broadcasts latents to all ranks |
-| [`combine_cfg_noise()`](https://docs.vllm.ai/projects/vllm-omni/en/latest/api/vllm_omni/diffusion/distributed/cfg_parallel/) | Combine positive/negative | Applies CFG formula with optional normalization |
-| [`predict_noise()`](https://docs.vllm.ai/projects/vllm-omni/en/latest/api/vllm_omni/diffusion/distributed/cfg_parallel/) | Forward pass wrapper | Override for custom transformer calls |
-| [`cfg_normalize_function()`](https://docs.vllm.ai/projects/vllm-omni/en/latest/api/vllm_omni/diffusion/distributed/cfg_parallel/) | Normalize CFG output | Override for custom normalization |
-
-### How It Works
-
-`predict_noise_maybe_with_cfg()` automatically detects and switches between two execution modes:
-
-- **CFG-Parallel mode** (when `cfg_world_size > 1`):
-  - Rank 0 computes positive prompt prediction
-  - Rank 1 computes negative prompt prediction
-  - Results are gathered via `all_gather()`
-  - Combined on rank 0 using CFG formula
-
-- **Sequential mode** (when `cfg_world_size == 1`):
-  - Single rank computes both positive and negative predictions
-  - Directly combines them with CFG formula
-
-`scheduler_step_maybe_with_cfg()` ensures consistent latent states across all ranks:
-
-- **CFG-Parallel mode**:
-  - Only rank 0 performs the scheduler step (applies noise prediction to update latents)
-  - Updated latents are broadcast to all other ranks via `broadcast()`
-  - All ranks maintain synchronized latent states for the next iteration
-
-- **Sequential mode**:
-  - Single rank directly performs the scheduler step
-  - No synchronization needed
-
----
-
-## Step-by-Step Implementation
-
-### Step 1: Inherit `CFGParallelMixin`
-
-Allow your pipeline to inherit from `CFGParallelMixin` and implements the `diffuse()` method for your specific model.
-
-**Example (Qwen-Image):**
-
-```python
-from vllm_omni.diffusion.distributed.cfg_parallel import CFGParallelMixin
-import torch.nn as nn
-class YourModelPipeline(nn.Module, CFGParallelMixin):
-    def diffuse(self, ...) -> torch.Tensor:
-        for i, t in enumerate(timesteps):
-            # Prepare positive_kwargs (conditional) and negative_kwargs (unconditional)
-            positive_kwargs = {...}  # hidden_states, encoder_hidden_states, etc.
-            negative_kwargs = {...} if do_true_cfg else None
-
-            # Key method 1: Predict noise with automatic CFG parallel handling
-            noise_pred = self.predict_noise_maybe_with_cfg(
-                do_true_cfg=do_true_cfg,
-                true_cfg_scale=true_cfg_scale,
-                positive_kwargs=positive_kwargs,
-                negative_kwargs=negative_kwargs,
-            )
-
-            # Key method 2: Step scheduler with automatic CFG synchronization
-            latents = self.scheduler_step_maybe_with_cfg(
-                noise_pred, t, latents, do_true_cfg
-            )
-
-        return latents
-```
-
-**Key Points:**
-
-- `positive_kwargs`: transformer arguments for conditional (text-guided) prediction
-- `negative_kwargs`: transformer arguments for unconditional prediction (set to `None` if CFG disabled)
-- For image editing pipelines, add `output_slice=image_seq_len` to extract the generative image portion
-
-### Step 2: Call `diffuse`
-
-Call `self.diffuse` in your pipeline's forward function:
-
-```python
-import torch.nn as nn
-class YourModelPipeline(nn.Module, CFGParallelMixin):
-    def forward(
-        self,
-        prompt: str,
-        negative_prompt: str | None = None,
-        guidance_scale: float = 3.5,
-        num_inference_steps: int = 50,
-        **kwargs,
-    ):
-        # Encode prompts, Initialize latents, Get timesteps
-        ...
-        # Run diffusion loop (calls the mixin's diffuse method)
-        latents = self.diffuse(
-            prompt_embeds=prompt_embeds,
-            prompt_embeds_mask=prompt_embeds_mask,
-            negative_prompt_embeds=negative_embeds,
-            negative_prompt_embeds_mask=negative_mask,
-            latents=latents,
-            timesteps=timesteps,
-            do_true_cfg=do_true_cfg,
-            true_cfg_scale=guidance_scale,
-            ...
-        )
-```
-
----
-
-## Customization
-
-### Override `predict_noise()` for Custom Transformer Calls
-
-If your transformer requires custom prediction function, you can rewrite `predict_noise` function. Taking Wan2.2 as an example, which has two transformer models. The actual transformer to be called is determined by `self.transformer`.
-
-```python
-class Wan22Pipeline(nn.Module, CFGParallelMixin):
-    def predict_noise(self, current_model: nn.Module | None = None, **kwargs: Any) -> torch.Tensor:
-        if current_model is None:
-            current_model = self.transformer
-        return current_model(**kwargs)[0]
-```
-
-
-### Override `cfg_normalize_function()` for Custom Normalization
-
-Some models have their own normalization function. Taking LongCat Image model as an example:
-
-```python
-class LongCatImagePipeline(nn.Module, CFGParallelMixin):
-    def cfg_normalize_function(self, noise_pred, comb_pred, cfg_renorm_min=0.0):
-        """
-        Normalize the combined noise prediction.
-        """
-        cond_norm = torch.norm(noise_pred, dim=-1, keepdim=True)
-        noise_norm = torch.norm(comb_pred, dim=-1, keepdim=True)
-        scale = (cond_norm / (noise_norm + 1e-8)).clamp(min=cfg_renorm_min, max=1.0)
-        noise_pred = comb_pred * scale
-        return noise_pred
-
-        # The original cfg_normalize_function function in CFGParallelMixin
-        # cond_norm = torch.norm(noise_pred, dim=-1, keepdim=True)
-        # noise_norm = torch.norm(comb_pred, dim=-1, keepdim=True)
-        # noise_pred = comb_pred * (cond_norm / noise_norm)
-        # return noise_pred
-```
-
----
-
-## Testing
-
-After adding CFG-Parallel support, test with:
-
-```bash
-cd examples/offline_inference/text_to_image
-python text_to_image.py \
-    --model Your-org/your-model \
-    --prompt "a cup of coffee on the table" \
-    --negative-prompt "ugly, unclear" \
-    --cfg-scale 4.0 \
-    --num-inference-steps 50 \
-    --output "cfg_enabled.png" \
-    --cfg-parallel-size 2
-```
-
-**Verify:**
-
-1. Check logs for CFG parallel being activated
-2. Record the `e2e_time_ms` in the log and compare with CFG-Parallel disabled
-3. Compare the generated result quality with baseline
-4. Record comparison results in your PR
-
----
-
-## Troubleshooting
-
-### Issue: CFG parallel not activating
-
-**Symptoms:** Generation still slow, logs don't show CFG parallel being used.
-
-**Causes & Solutions:**
-
-- **CFG is not enabled:**
-
-**Problem:** Guidance scale too low or negative prompt not provided.
-
-**Solution:** Ensure `guidance_scale > 1.0` and negative prompt is provided:
-```python
-images = pipeline(
-    prompt="a cat",
-    negative_prompt="",  # Must provide (even if empty)
-    guidance_scale=3.5,   # Must be > 1.0
-)
-```
-
----
-
-## Reference Implementations
-
-Complete examples in the codebase:
-
-| Model | Path | Pattern | Notes |
-|-------|------|---------|-------|
-| **Qwen-Image** | `vllm_omni/diffusion/models/qwen_image/cfg_parallel.py` | Mixin | Dual-stream transformer |
-| **Qwen-Image-Edit** | `vllm_omni/diffusion/models/qwen_image/pipeline_qwen_image_edit.py` | Mixin | Image editing with `output_slice` |
-| **Wan2.2** | `vllm_omni/diffusion/models/wan2_2/pipeline_wan2_2.py` | Mixin | Dual-transformer architecture |
-| **CFGParallelMixin** | `vllm_omni/diffusion/distributed/cfg_parallel.py` | Base implementation | Core mixin class |
-
----
-
-## Summary
-
-Adding CFG-Parallel support:
-
-1. ✅ **Create mixin** - Inherit from `CFGParallelMixin` and implement `diffuse()` method
-2. ✅ **(Optional) Customize** - Override `predict_noise()` or `cfg_normalize_function()` for custom behavior
-3. ✅ **Test** - Verify with `--cfg-parallel-size 2` and compare performance

docs/design/feature/disaggregated_inference.md

@@ -1,108 +0,0 @@
-# Disaggregated Inference for Omni-Modality Models
-
-This guide explains how to configure and use distributed connectors
-(`vllm_omni/distributed/omni_connectors`) in vllm-omni for multi-stage pipelines.
-
-Backend-specific setup lives in separate docs:
-
-- [SharedMemoryConnector](omni_connectors/shared_memory_connector.md)
-- [MooncakeStoreConnector](omni_connectors/mooncake_store_connector.md)
-- [MooncakeTransferEngineConnector](omni_connectors/mooncake_transfer_engine_connector.md)
-- [YuanrongConnector](omni_connectors/yuanrong_connector.md)
-
-## Overview
-
-Connectors enable data transfer between pipeline stages (e.g., Thinker -> Talker).
-Current connectors operate in D2H2D (device to host to device) mode.
-
-## Connector Choices
-
-| Use Case | Recommended Connector | Notes |
-| :--- | :--- | :--- |
-| Single node | SharedMemoryConnector | Auto-configured if no connector is specified. |
-| Multi node (Mooncake Store) | MooncakeStoreConnector | TCP-based, requires Mooncake Master + metadata server. |
-| Multi node (Mooncake RDMA) | MooncakeTransferEngineConnector | RDMA/TCP direct transfer with managed memory pool. Fastest. |
-| Multi node (Yuanrong) | YuanrongConnector | Requires Yuanrong Datasystem + etcd. |
-
-## Core API
-
-The connector system is built around `OmniConnectorBase`.
-
-```python
-class OmniConnectorBase(ABC):
-    @abstractmethod
-    def put(self, from_stage: str, to_stage: str, put_key: str, data: Any) -> tuple[bool, int, Optional[dict]]:
-        """
-        Store data.
-        Returns: (success, serialized_size, metadata)
-        """
-        pass
-
-    @abstractmethod
-    def get(self, from_stage: str, to_stage: str, get_key: str, metadata: Optional[dict] = None) -> Optional[tuple[Any, int]]:
-        """
-        Retrieve data.
-        Args: metadata - transport-specific handles returned by put() (e.g., SHM name).
-        Returns: (object, serialized_size)
-        """
-        pass
-```
-
-### Metadata Passing
-
-Some connectors (e.g., SharedMemoryConnector) generate transient resources during `put()`.
-This `metadata` must be passed through the control plane so `get()` can locate the data.
-
-## Configuration Model
-
-Define connectors in runtime:
-
-```yaml
-runtime:
-  connectors:
-    connector_of_shared_memory:
-      name: SharedMemoryConnector
-      extra:
-        shm_threshold_bytes: 65536
-```
-
-Wire stages to connectors:
-
-```yaml
-stage_args:
-  - stage_id: 0
-    output_connectors:
-      to_stage_1: connector_of_shared_memory
-
-  - stage_id: 1
-    input_connectors:
-      from_stage_0: connector_of_shared_memory
-```
-
-If a pipeline edge has no explicit connector, the system auto-creates a
-SharedMemoryConnector for that edge.
-
-## Relationship with vLLM
-
-vLLM provides specialized distributed mechanisms for specific artifacts:
-
-- KV Transfer (`vllm.distributed.kv_transfer`): optimized for KV caches.
-- EC Transfer (`vllm.distributed.ec_transfer`): optimized for encoder embeddings.
-- Device Communicators (`vllm.distributed.device_communicators`): low-level primitives (NCCL, SHM).
-
-vllm-omni complements this with a generalized connector abstraction:
-
-1. Unifies transport via a single `put`/`get` API for any stage artifact.
-2. Enables DAG-style pipelines across processes or nodes with per-edge transports.
-3. Can wrap vLLM-specific transfers for KV paths while keeping a consistent interface.
-
-## Operational Notes
-
-- Fail-fast config validation: missing expected edges cause startup failures.
-- Missing payloads halt stages: verify connector wiring and metadata propagation.
-
-## Future Roadmap: D2D Transport
-
-Current connectors use D2H2D paths. Future versions will introduce direct
-device-to-device connectors (NCCL, UCX, IPC) to reduce latency for large
-tensor payloads.

docs/design/feature/hsdp.md

@@ -1,146 +0,0 @@
-# HSDP
-
-This section describes how to add HSDP (Hybrid Sharded Data Parallel) support to a diffusion transformer model. We use the Wan2.2 transformer as the reference implementation.
-
----
-
-## Table of Contents
-
-- [Overview](#overview)
-- [Step-by-Step Implementation](#step-by-step-implementation)
-- [Testing](#testing)
-- [Reference Implementations](#reference-implementations)
-
----
-
-## Overview
-
-### What is HSDP?
-
-HSDP (Hybrid Sharded Data Parallel) is a memory optimization technique that **shards model weights** across multiple GPUs using PyTorch's FSDP2. Unlike Tensor Parallelism which splits computation, HSDP:
-
-- Shards weights across GPUs to reduce per-GPU memory usage
-- Gathers weights on-demand during forward passes
-- Can work standalone or combined with other parallelism (e.g., Sequence Parallel)
-
-This enables inference of large models (e.g., Wan2.2 14B) on GPUs with limited memory.
-
-**Important constraints:**
-- HSDP cannot be used with Tensor Parallelism
-- For standalone HSDP (no other parallelism), `hsdp_shard_size` must be specified explicitly
-
-### Architecture
-
-HSDP implementation relies on:
-
-1. **`_hsdp_shard_conditions`**: Model attribute specifying which modules to shard
-2. **`apply_hsdp_to_model`**: Function that applies FSDP2 sharding
-3. **`HSDPInferenceConfig`**: Runtime configuration for HSDP
-
----
-
-## Step-by-Step Implementation
-
-### Step 1: Identify Modules to Shard
-
-Determine which modules in your transformer should be sharded. Typically, these are:
-
-- Transformer blocks (e.g., `blocks.0`, `blocks.1`, ...)
-- Large submodules with significant weight memory
-
-**Key questions:**
-- Which modules have the largest weights?
-- Which modules are repeated (like transformer blocks)?
-
-### Step 2: Define Shard Conditions
-
-Add `_hsdp_shard_conditions` to your model class. This is a list of functions that return `True` for modules that should be sharded.
-
-**Example (Transformer Blocks):**
-
-```python
-class MyTransformerModel(nn.Module):
-
-    @staticmethod
-    def _is_transformer_block(name: str, module) -> bool:
-        """Match transformer blocks for HSDP sharding.
-
-        Args:
-            name: Module name from named_modules() (e.g., "blocks.0", "blocks.0.attn")
-            module: The module instance
-
-        Returns:
-            True if this module should be sharded
-        """
-        return "blocks" in name and name.split(".")[-1].isdigit()
-
-    _hsdp_shard_conditions = [_is_transformer_block]
-```
-
-**Multiple Conditions Example:**
-
-```python
-class MyModel(nn.Module):
-
-    @staticmethod
-    def _is_transformer_block(name: str, module) -> bool:
-        return "blocks" in name and name.split(".")[-1].isdigit()
-
-    @staticmethod
-    def _is_moe_expert(name: str, module) -> bool:
-        # Also shard MoE expert layers
-        return "experts" in name and name.split(".")[-1].isdigit()
-
-    # Module is sharded if ANY condition returns True
-    _hsdp_shard_conditions = [_is_transformer_block, _is_moe_expert]
-```
-
----
-
-## Testing
-
-After adding HSDP support, test with:
-
-```python
-from vllm_omni import Omni
-from vllm_omni.diffusion.data import DiffusionParallelConfig
-from vllm_omni.inputs.data import OmniDiffusionSamplingParams
-
-parallel_config = DiffusionParallelConfig(
-    use_hsdp=True,
-    hsdp_shard_size=8,  # Shard across 8 GPUs
-)
-omni = Omni(model="your-model-name", parallel_config=parallel_config)
-
-output = omni.generate(
-    "a cup of coffee on the table",
-    OmniDiffusionSamplingParams(num_inference_steps=50),
-)
-```
-
-**Or via command line:**
-
-```bash
-vllm serve Your-org/your-model --omni --port 8091 --use-hsdp
-```
-
-**Verify:**
-
-1. Check logs for "HSDP Inference: replicate_size=..., shard_size=..."
-2. Check logs for "Sharded N modules + root"
-3. Verify memory usage is reduced proportionally
-4. Compare generated output quality with HSDP disabled
-
----
-
-## Reference Implementations
-
-Complete examples in the codebase:
-
-| Model | Path | Notes |
-|-------|------|-------|
-| **Wan2.2** | `vllm_omni/diffusion/models/wan2_2/wan2_2_transformer.py` | Reference implementation |
-| **HSDP Core** | `vllm_omni/diffusion/distributed/hsdp.py` | `apply_hsdp_to_model`, `shard_model` |
-| **HSDP Tests** | `tests/diffusion/distributed/test_hsdp.py` | Unit tests |
-
----

docs/design/feature/multi_request_streaming_design.md

@@ -0,0 +1,188 @@
+## Multi-Request Streaming (MRS) on a Single Machine
+
+### 1. Background & Scope
+- All processing runs on a single physical machine with multi-process, per-stage workers. No proxy or network transport involved.
+- Current alignment with vllm-omni: `OmniLLM` supports multiple stages (`OmniStage`). GPU runners already expose streamable steps (prefill/decoding/diffusion), but the entry layer still collects lists and lacks intra-stage streaming and window scheduling.
+- Goal: implement multi-stage, multi-request streaming (MRS) locally. Each stage outputs segments; downstream stages stitch and trigger compute based on configured windows. Shared memory and zero-copy strategies reduce data movement overhead.
+
+### 2. Key Constraints
+- Multi-process per stage: each stage is an independent process with a while loop; device visibility can be configured (`CUDA_VISIBLE_DEVICES`/`torch.cuda.set_device`).
+- Simple IPC (copy-based): use `multiprocessing.Queue`/Pipe for inter-process communication with CPU copies/serialization; do not rely on CUDA IPC/SHM zero-copy in this version.
+- Cross-stage pipeline: different stages can process different requests concurrently (e.g., stage A handles request 1 while stage B handles request 0).
+
+### 3. Architecture Overview
+- Processes & IPC queues
+  - Each "sub-stage" is an OS process (worker). The loop: take from input_queue → compute → put to output_queue.
+  - Inter-stage connection via IPC: copy-based `multiprocessing.Queue` passing dict payloads; use shared memory for large objects.
+  - Each link is SPSC (single-producer/single-consumer): the upstream is the orchestrator and the downstream is a single stage process; queues are unbounded (maxsize=0) on the orchestrator side.
+- Device visibility
+  - Each stage sets `CUDA_VISIBLE_DEVICES` or calls `torch.cuda.set_device` to bind to GPU sets.
+  - A stage may use multiple GPUs internally (TP/PP/DP) but presents as a single stage unit.
+- Simplified IPC: copy-based queues/pipes for data transfer; zero-copy is future work.
+- Pipeline progression: when a stage finishes a request, it enqueues outputs to the downstream stage; if downstream is idle, it starts immediately.
+- Scheduling
+  - A downstream stage triggers only after the upstream completes the request.
+  - Windowed segmentation/stitched triggering is not implemented; intra-stage streaming is not provided.
+
+### 7. IPC Implementation (simplified: copy-based)
+- Use `multiprocessing.Queue`/Pipe for inter-process communication (control + data).
+- Data is serialized/copied via CPU; no CUDA IPC/SHM zero-copy in this version.
+- Backpressure: queues are unbounded; pressure manifests as compute-rate differences. Optional SHM reduces large-object transfer cost; RX/decoding overhead is recorded for observability.
+
+### 8. Scheduling & Cancellation (simplified)
+- Pipeline: when a stage finishes a request, it enqueues to the next stage; that stage immediately pulls the next request from its input queue, enabling cross-stage concurrency.
+- Cancellation/timeout: explicit cancellation/timeouts are not provided; graceful shutdown uses a `None` sentinel sent to each stage input queue.
+
+#### Short sequence example (req0/req1, stage A→B)
+1) t0: stage A handles req0
+2) t1: req0 completes on A → enters B; A immediately starts req1
+3) t2: B handles req0 while A handles req1 (parallel across stages)
+
+### 9. Integration Points (by file)
+- `vllm_omni/entrypoints/omni_llm.py` (Orchestrator)
+  - Class `OmniLLM` orchestrates multi-process stages; constructs `OmniStage` instances in parallel and spawns per-stage workers.
+  - Spawns stage processes per config (set `CUDA_VISIBLE_DEVICES`/`torch.cuda.set_device`), creates control/data channels, builds simple full-trigger flow.
+  - Stats/logging are disabled by default; per-stage and orchestrator stats are only written when explicitly enabled.
+  - Manages process lifecycle: start/wait for readiness, graceful shutdown; forwards results between stages using copy-based IPC and optional SHM.
+  - Stage readiness: each stage emits `{"type": "stage_ready"}` after initialization; the orchestrator waits for all stages or times out and logs diagnostic suggestions.
+
+### 9.1 Process Device Visibility
+- Device binding: set `CUDA_VISIBLE_DEVICES` before process start, or call `torch.cuda.set_device` early in init; configs can be single/multi-GPU.
+- Cross-device transfer (simplified): via CPU copies; zero-copy paths are out of scope for now.
+
+### 10. Worker Template (pseudo-code)
+```python
+def stage_worker(input_q, output_q, runtime, shm_threshold):
+    # Device binding
+    set_stage_gpu_devices(runtime.devices)
+    engine = OmniStageLLM(model, **engine_args)
+    batch_seq = 0
+    while True:
+        first = input_q.get()
+        if first is None:
+            break
+        # Batch intake (up to runtime.max_batch_size)
+        batch = [first]
+        while len(batch) < int(runtime.max_batch_size or 1) and not input_q.empty():
+            nxt = input_q.get()
+            if nxt is None:
+                input_q.put(None)
+                break
+            batch.append(nxt)
+        # Decode IPC payload and sampling params
+        request_ids = []
+        engine_inputs = []
+        rx_bytes = {}
+        rx_decode_ms = {}
+        for t in batch:
+            ein, rx = maybe_load_from_ipc_with_metrics(t, "engine_inputs", "engine_inputs_shm")
+            request_ids.append(t["request_id"])
+            engine_inputs.extend(ein if isinstance(ein, list) else [ein])
+            rx_bytes[t["request_id"]] = rx["rx_transfer_bytes"]
+            rx_decode_ms[t["request_id"]] = rx["rx_decode_time_ms"]
+        # Generate and dispatch (grouped by request_id)
+        batch_seq += 1
+        outputs = list(engine.generate(engine_inputs, batch[0]["sampling_params"], use_tqdm=False))
+        grouped = group_by_request_id(outputs, request_ids)
+        for rid in request_ids:
+            r_out = grouped.get(rid, [])
+            use_shm, payload = maybe_dump_to_shm(r_out, shm_threshold)
+            msg = {
+                "request_id": rid,
+                "metrics": {
+                    "batch_id": batch_seq,
+                    "rx_transfer_bytes": int(rx_bytes.get(rid, 0)),
+                    "rx_decode_time_ms": float(rx_decode_ms.get(rid, 0.0)),
+                    # omitted: generation latency and token stats
+                },
+            }
+            if use_shm:
+                msg["engine_outputs_shm"] = payload
+            else:
+                msg["engine_outputs"] = payload
+            output_q.put(msg)
+```
+
+### 12. Observability
+- Metrics (as implemented):
+  - per-request (emitted by stages): `num_tokens_out`, `stage_gen_time_ms`, `rx_transfer_bytes`, `rx_decode_time_ms`, `rx_in_flight_time_ms`, `batch_id`
+  - orchestrator aggregates: E2E latency, tokens/s (written only when stats are enabled)
+  - optional per-stage JSONL: `{log_file}.stage{stage_id}.stats.jsonl`
+- Logs/tracing:
+  - optional per-stage log files: `{log_file}.stage{stage_id}.log`
+  - the orchestrator can log readiness, forward size/time, and summary information
+
+### 15. Risks & Mitigations
+— End-to-end latency: full-trigger sacrifices interactivity; start simple/correct, add windowed streaming later.
+— CPU copy overhead: copy-based IPC for maintainability; add SHM/zero-copy later as an optimization path.
+
+### 16. Refactor: Sink LLM init and process into Stage (diagram)
+
+Objective: without changing external call patterns, encapsulate per-stage LLM init, process creation, and worker logic inside `Stage`, so `PipelinedOmniLLM` focuses on orchestration (seeding, polling, forwarding, collecting) while keeping multi-process, shared memory, and device mapping capabilities.
+
+Key changes (vs. current):
+- Before: Orchestrator created processes directly, passing `_stage_worker`; device setup, LLM init, batching, and SHM were implemented there.
+- After: enhanced `OmniStage` owns its subprocess (`init_stage_worker`), with `_stage_worker` as the entry; device setup, LLM init, batching, and SHM live in `OmniStage`.
+
+Classes & responsibilities
+- OmniLLM (orchestrator)
+  - Build `OmniStage` list in parallel (preserve `process_engine_inputs` wiring)
+  - Connect adjacent `Stage` input/output queues; seed requests to stage 0
+  - Poll stage outputs, decode results, call `process_engine_inputs`, then encode and forward to next stage
+  - Termination & cleanup: distribute shutdown signals; join/terminate subprocesses
+- Stage (stage unit)
+  - Members: `stage_config` (with `engine_args` and `runtime`), `in_q/out_q`, subprocess handle, stats
+  - API:
+    - `init_stage_worker(...)`: spawn subprocess
+    - `stop_stage_worker()`: graceful exit
+    - `submit(ipc_payload)`: submit to input queue (may use shared memory)
+    - `try_collect() -> Optional[payload]`: non-blocking get from output queue
+    - `process_engine_inputs(stages, prompts)`: reuse/delegate existing logic
+  - Subprocess entry `worker_main()`:
+    - `set_stage_gpu_devices` (`CUDA_VISIBLE_DEVICES` & logical index mapping)
+    - build `OmniStageLLM(model, **engine_args)`
+    - while-loop:
+      - take first task + up to `runtime.max_batch_size-1` non-blocking; support `max_batch_size>=1`
+      - `maybe_load_from_ipc_with_metrics` per `engine_inputs` in batch
+      - `stage_engine.generate(batched_inputs, sampling_params)` (window `-1`)
+      - write per-request aggregated outputs to output queue after `maybe_dump_to_shm`
+      - log start/finish; report `error` per request on exceptions
+
+Data flow (sketch)
+```
+OmniLLM
+  ├─ Stage[0].start_process(in0, out0)
+  ├─ Stage[1].start_process(in1, out1)
+  ├─ ...
+  └─ Orchestrator loop:
+       - stage0.submit(encode(engine_inputs))
+       - for each stage:
+           res = stage.try_collect()
+           outputs = decode(res)
+           stage.set_engine_outputs(outputs)
+           if !final: next_inputs = next_stage.process_engine_inputs(stages, original_prompt)
+                    next_stage.submit(encode(next_inputs))
+```
+
+Config (preserve & extend)
+- `runtime.devices`:
+  - Comma-separated visibility list (e.g., "2,5,7"): set visible set; default logical index 0 (first GPU)
+  - Number/string: treat as logical index (0/1-based per implementation) mapping to physical GPUs
+- `runtime.max_batch_size`: max requests taken per intake (default 1)
+- `shm_threshold_bytes`: objects larger than this go via shared memory (default 64KB; injected into stages)
+- Stage link and inputs:
+  - `engine_input_source`: list of upstream stage_id(s) from which the downstream stage takes inputs (typically a single source)
+  - `custom_process_input_func`: function to construct downstream engine_inputs from upstream outputs
+- Output flags:
+  - `final_output`, `final_output_type`: marks that a stage produces the final external output and its type
+
+Kept & strengthened
+- Multi-process: each stage in its own process; parallel across stages; serial/batching inside a stage controlled by `max_batch_size`
+- Shared memory: large objects via SHM with strict `memoryview -> close -> unlink`
+- Device mapping: unified `set_stage_gpu_devices`, support comma lists and logical indices
+
+Migration (execution order)
+1) Enhanced `OmniStage` with `init_stage_worker/stop_stage_worker/submit/try_collect/_stage_worker`.
+2) `OmniLLM` holds `OmniStage` instances, replacing the older pipelined class; uses Stage APIs.
+3) Reuse `entrypoints/stage_utils`: device/SHM/encode-decode utilities live here.
+4) Regression: single/multi-request, various `max_batch_size`, SHM threshold, device mapping, multi-stage parallelism, error paths, and shutdown flows.

docs/design/feature/omni_connector_design.md

@@ -0,0 +1,200 @@
+# OmniConnector: a unified connector interface of disaggregated inference for omni-modality models
+
+This guide explains how to configure and use distributed connectors (vllm_omni/distributed/connectors) in vllm-omni for multi-stage pipelines.
+
+## 1. Overview
+
+Connectors enable data transfer between pipeline stages (e.g., Thinker -> Talker).
+Currently supported connectors operate in **D2H2D (Device-to-Host-to-Device)** mode:
+1. **SharedMemoryConnector**: Uses system shared memory.
+2. **MooncakeConnector**: Uses [Mooncake](https://github.com/kvcache-ai/Mooncake).
+
+*   **SharedMemoryConnector (Default)**: Zero-copy (on host), lowest latency. Best for **single-node** deployments. Auto-configured if no connectors are specified.
+*   **MooncakeConnector**: TCP/RDMA based. Best for **multi-node** distributed deployments. Requires a Mooncake Master service.
+
+## 2. API Design
+
+The connector system is built around the `OmniConnectorBase` abstraction, which decouples data transport from stage logic.
+
+### Core Interface
+
+```python
+class OmniConnectorBase(ABC):
+    @abstractmethod
+    def put(self, from_stage: str, to_stage: str, request_id: str, data: Any) -> tuple[bool, int, Optional[dict]]:
+        """
+        Store data.
+        Returns: (success, serialized_size, metadata)
+        """
+        pass
+
+    @abstractmethod
+    def get(self, from_stage: str, to_stage: str, request_id: str, metadata: Optional[dict] = None) -> Optional[tuple[Any, int]]:
+        """
+        Retrieve data.
+        Args: metadata - Transport-specific handles returned by put() (e.g., SHM name).
+        Returns: (object, serialized_size)
+        """
+        pass
+```
+
+### Key Concept: Metadata Passing
+Unlike a pure key-value store, some connectors (like `SharedMemoryConnector`) generate transient resources (e.g., a shared memory block name) during `put()`. This `metadata` **must be passed via the control plane** (e.g., HTTP headers, queue messages) from the producer stage to the consumer stage so `get()` can locate the data.
+
+## 3. Backends & Use Cases
+
+### 3.1 SharedMemoryConnector
+**Best for:** Single-node, high-performance IPC.
+
+*   **Mechanism:**
+    *   **Small Payloads (< Threshold)**: Data is serialized and passed directly "inline" within the `metadata` dictionary. This avoids the overhead of creating SHM blocks for tiny messages.
+    *   **Large Payloads (>= Threshold)**: Data is written to a named System Shared Memory block. The block name is returned in `metadata`.
+*   **Configuration:**
+    *   `shm_threshold_bytes`: Size in bytes to switch from inline to SHM (default: 64KB).
+
+### 3.2 MooncakeConnector
+**Best for:** Multi-node distributed inference.
+
+*   **Mechanism:** Uses Mooncake's distributed KVCache store.
+    *   **Data Plane**: TCP or RDMA for high-bandwidth transfer.
+    *   **Control Plane**: Uses a centralized Mooncake Master and Metadata Server.
+    *   **Keying**: Deterministic keys based on `request_id/from_stage_to_stage`.
+*   **Requirements**: Requires a running Mooncake Master service.
+
+## 4. Relationship with vLLM
+
+vLLM provides specialized distributed mechanisms for specific artifacts:
+*   **KV Transfer** (`vllm.distributed.kv_transfer`): Optimized for transferring KV caches between prefill and decode instances (using NCCL, Mooncake, etc.).
+*   **EC Transfer** (`vllm.distributed.ec_transfer`): Optimized for sharing encoder embeddings.
+*   **Device Communicators** (`vllm.distributed.device_communicators`): Low-level primitives (NCCL, SHM) for Tensor/Pipeline Parallelism.
+
+`vllm-omni` complements this by introducing a **Generalized Connector Abstraction** (`OmniConnector`) for multimodal pipelines. While vLLM's connectors are artifact-specific, `vllm-omni`:
+
+1.  **Unifies Transport**: Provides a single API (`put`/`get`) to transport *any* stage artifact (Input Embeddings, Hidden States, Audio/Image Tensors, KV Cache, Final Output) between arbitrary pipeline stages (e.g., AudioEncoder -> LLM -> AudioGenerator).
+2.  **Extends Connectivity**: Enables flexible construction of complex DAGs (Directed Acyclic Graphs) where stages can run in the same process, same node, or across nodes, using the most appropriate backend (SHM, Mooncake, etc.) for each edge.
+3.  **Wraps & Adapts**: Can internally utilize vLLM's specialized `kv_transfer` for KV paths while using generic transports (SHM/Mooncake) for other data types, presenting a consistent interface to the application layer.
+
+## 5. Installation (Mooncake)
+
+If using `MooncakeConnector`, install the library first:
+
+```bash
+# For CUDA-enabled systems (Recommended)
+pip install mooncake-transfer-engine
+
+# For non-CUDA systems
+pip install mooncake-transfer-engine-non-cuda
+```
+
+## 6. Using MooncakeConnector
+
+### 6.1 Start Mooncake Master
+
+Start the master service on your primary node:
+
+```bash
+# if you use mooncake SSD storage
+mkdir -p ./mc_storage
+
+mooncake_master \
+  --rpc_port=50051 \
+  --enable_http_metadata_server=true \
+  --http_metadata_server_host=0.0.0.0 \
+  --http_metadata_server_port=8080 \
+  --metrics_port=9003 \
+  --root_fs_dir=./mc_storage/ \
+  --cluster_id=mc-local-1 &
+```
+
+### 6.2 Configuration (YAML)
+
+Edit your stage config (e.g., `qwen2_5_omni.yaml`).
+
+**Step 1: Define Connector in Global Runtime**
+
+```yaml
+runtime:
+  connectors:
+    connector_of_mooncake:
+      name: MooncakeConnector
+      extra:
+        host: "127.0.0.1"           # Local Worker IP
+        metadata_server: "http://<MASTER_IP>:8080/metadata"
+        master: "<MASTER_IP>:50051"
+        segment: 512000000          # 512MB segment
+        localbuf: 64000000          # 64MB buffer
+        proto: "tcp"                # "tcp" or "rdma"
+    ```
+
+**Mooncake Configuration Parameters:**
+
+*   **host**: The hostname or IP address of the local machine (worker). Mooncake uses this to register itself in the metadata server so other nodes can find it.
+*   **metadata_server**: The URL of the metadata server. This is used for service discovery and connection establishment (e.g., exchanging QP information for RDMA).
+*   **master**: The address of the Mooncake Master Server (e.g., `<MASTER_IP>:50051`). This is used for global state management and control plane operations.
+*   **segment**: The size of the global memory segment in bytes (default: ~512MB). This defines the shared memory region accessible by Mooncake for data transfer.
+*   **localbuf**: The size of the local buffer in bytes (default: ~64MB). Used for local data buffering during transfer operations.
+*   **proto**: The transport protocol to use. Options:
+    *   `tcp`: Standard TCP/IP (easier setup, universal compatibility).
+    *   `rdma`: Remote Direct Memory Access (higher performance, requires RDMA-capable hardware).
+
+For more details, refer to the [Mooncake Repository](https://github.com/kvcache-ai/Mooncake).
+
+    **Step 2: Reference in Stages**
+
+Explicitly link stages using `input_connectors` and `output_connectors`:
+
+```yaml
+stage_args:
+  - stage_id: 0
+    # ...
+    output_connectors:
+      to_stage_1: connector_of_mooncake
+
+  - stage_id: 1
+    # ...
+    input_connectors:
+      from_stage_0: connector_of_mooncake
+```
+
+## 7. Using SharedMemoryConnector (Auto-Mode)
+
+**Best for single-node.**
+
+The system will automatically create `SharedMemoryConnector`s for any pipeline edge that does not have an explicit connector defined. This is inferred from:
+1.  `runtime.edges` list in the config.
+2.  `engine_input_source` dependencies defined in `stage_args`.
+
+### Threshold Configuration
+By default, payloads larger than **64KB** (default threshold) are transferred via shared memory, while smaller ones use the control queue (inline).
+
+To adjust this threshold (e.g., to 1GB), add the following to your `runtime.connectors`:
+
+```yaml
+runtime:
+  connectors:
+    connector_of_shared_memory:
+      name: SharedMemoryConnector
+      extra:
+        shm_threshold_bytes: 1024 # 1KB threshold
+```
+
+## 8. Summary
+
+| Use Case | Recommended Connector | Configuration |
+| :--- | :--- | :--- |
+| **Single Node** | `SharedMemoryConnector` | **None** (Automatic) or Custom Threshold |
+| **Multi Node** | `MooncakeConnector` | Explicit YAML + Mooncake Master |
+
+## 9. Operational Notes (important)
+
+- **Fail-fast config validation**: the loader raises if any expected edge is missing a connector. Define `input_connectors`/`output_connectors` or rely on auto-SHM filling; otherwise startup aborts.
+- **Missing payloads halt the stage**: workers expect connector payloads; if metadata or connector config is missing, the stage raises and stops. Verify connector wiring and metadata propagation before production.
+
+## 10. Future Roadmap: Device-to-Device (D2D) Transport
+
+The current implementations (`SharedMemoryConnector`, `MooncakeConnector`) utilize a **D2H2D (Device-to-Host-to-Device)** data path. Tensors are moved to CPU memory (Host) for transport, which incurs PCIe overhead.
+
+As outlined in the design RFC, future versions will introduce **D2D (Device-to-Device)** connectors:
+
+*   **Goal**: Direct GPU-to-GPU transfer (via NCCL, UCX, or IPC) to minimize latency for large tensor payloads.
+*   **Mechanism**: The `OmniConnector` API allows `put()` to initiate a transfer and return a lightweight handle (metadata) via the control plane, while the heavy payload flows directly between devices.

docs/design/feature/omni_connectors/mooncake_store_connector.md

@@ -1,74 +0,0 @@
-# MooncakeStoreConnector
-
-## When to Use
-
-Best for multi-node distributed inference using Mooncake.
-
-## Installation
-
-```bash
-# For CUDA-enabled systems (recommended)
-pip install mooncake-transfer-engine
-
-# For non-CUDA systems
-pip install mooncake-transfer-engine-non-cuda
-```
-
-## Start Mooncake Master
-
-```bash
-# If you use Mooncake SSD storage
-mkdir -p ./mc_storage
-
-mooncake_master \
-  --rpc_port=50051 \
-  --enable_http_metadata_server=true \
-  --http_metadata_server_host=0.0.0.0 \
-  --http_metadata_server_port=8080 \
-  --metrics_port=9003 \
-  --root_fs_dir=./mc_storage/ \
-  --cluster_id=mc-local-1 &
-```
-
-## Configuration
-
-Define the connector in runtime:
-
-```yaml
-runtime:
-  connectors:
-    connector_of_mooncake:
-      name: MooncakeStoreConnector
-      extra:
-        host: "127.0.0.1"
-        metadata_server: "http://<MASTER_IP>:8080/metadata"
-        master: "<MASTER_IP>:50051"
-        segment: 512000000
-        localbuf: 64000000
-        proto: "tcp"
-```
-
-Wire stages to the connector:
-
-```yaml
-stage_args:
-  - stage_id: 0
-    output_connectors:
-      to_stage_1: connector_of_mooncake
-
-  - stage_id: 1
-    input_connectors:
-      from_stage_0: connector_of_mooncake
-```
-
-Parameters:
-
-- host: local worker IP registered in the metadata server.
-- metadata_server: metadata server URL for discovery and setup.
-- master: Mooncake Master address.
-- segment: global memory segment size in bytes.
-- localbuf: local buffer size in bytes.
-- proto: transport protocol ("tcp" or "rdma").
-
-For more details, refer to the
-[Mooncake repository](https://github.com/kvcache-ai/Mooncake).

docs/design/feature/omni_connectors/mooncake_transfer_engine_connector.md

@@ -1,207 +0,0 @@
-# MooncakeTransferEngineConnector
-
-## When to Use
-
-Best for high-performance multi-node data transfer between stages using Mooncake
-Transfer Engine. Supports both RDMA and TCP protocols with a managed memory pool,
-zero-copy deserialization, and optional GPUDirect RDMA. Applicable to any
-inter-stage data (KV caches, request payloads, etc.), not limited to KV cache transfer.
-
-Compared to `MooncakeStoreConnector` (TCP key-value store), this connector
-provides **~60x faster** data transfer via RDMA direct memory access.
-
-## Installation
-
-```bash
-pip install mooncake-transfer-engine
-```
-
-Ensure RDMA drivers are installed on all nodes (e.g., Mellanox OFED for
-InfiniBand/RoCE NICs).
-
-## Configuration
-
-Define the connector in runtime:
-
-```yaml
-runtime:
-  connectors:
-    rdma_connector:
-      name: MooncakeTransferEngineConnector
-      extra:
-        host: "auto"                  # Auto-detect local RDMA IP
-        zmq_port: 50051               # ZMQ base port (see "Port Offset Scheme" below)
-        protocol: "rdma"              # "rdma" or "tcp"
-        device_name: ""               # RDMA device (e.g., "mlx5_0"), empty for auto-detect
-        memory_pool_size: 2147483648  # 2GB memory pool
-        memory_pool_device: "cpu"     # "cpu" for pinned memory, "cuda" for GPUDirect RDMA
-```
-
-Wire stages to the connector:
-
-```yaml
-stage_args:
-  - stage_id: 0
-    output_connectors:
-      to_stage_1: rdma_connector
-
-  - stage_id: 1
-    input_connectors:
-      from_stage_0: rdma_connector
-```
-
-## Parameters
-
-### Required
-
-| Parameter | Description |
-|---|---|
-| `role` | **Internal, do not set manually.** Auto-injected by the orchestration layer (`"sender"` for `output_connectors`, `"receiver"` for `input_connectors`). Defaults to `"sender"` if omitted. |
-| `host` | Local IP address for RDMA. `"auto"` detects from network interfaces. |
-| `protocol` | Transport protocol: `"rdma"` (InfiniBand/RoCE) or `"tcp"`. |
-
-### Memory Pool
-
-| Parameter | Default | Description |
-|---|---|---|
-| `memory_pool_size` | 1 GB | Total size of the RDMA-registered memory pool in bytes. |
-| `memory_pool_device` | `"cpu"` | `"cpu"`: pinned host memory (recommended). `"cuda"`: GPU VRAM for GPUDirect RDMA (requires NIC-GPU direct PCIe connectivity). |
-
-### Networking
-
-| Parameter | Default | Description |
-|---|---|---|
-| `zmq_port` | 50051 | ZMQ **base** port. The orchestration layer computes the actual port as `base + purpose_offset + stage_offset` (see table below). Users only set this base value. |
-| `sender_host` | `None` | **Internal.** Receiver-side only — dynamically resolved via `update_sender_info()`. Not needed in YAML. |
-| `sender_zmq_port` | `None` | **Internal.** Receiver-side only — defaults to the sender's adjusted port. Not needed in YAML. |
-| `device_name` | `""` | RDMA device name (e.g., `"mlx5_0"`). Empty for auto-detect. Can also be set via `RDMA_DEVICE_NAME` env var. |
-
-#### ZMQ Port Offset Scheme
-
-To avoid port conflicts when multiple edges, purposes, DP replicas, or TP ranks share the same node, the actual ZMQ port is computed as:
-
-```
-side_channel_port = zmq_port + purpose_offset + stage_offset + dp_index * tp_size
-sender_listen     = side_channel_port + tp_rank
-receiver_connect  = remote_side_channel_port + tp_rank
-```
-
-| Component | Value | Description |
-|---|---|---|
-| `zmq_port` | 50051 (default) | Base port from YAML config |
-| `purpose_offset` | `request_forwarding` = 0, `kv_transfer` = 100 | Separates control-plane vs KV-cache connections |
-| `stage_offset` | `int(from_stage)` (0, 1, 2...) | Separates edges from different source stages |
-| `dp_index * tp_size` | e.g., DP1 × TP2 = 2 | Each DP replica reserves a port range of size `tp_size` (following vLLM convention: `VLLM_MOONCAKE_BOOTSTRAP_PORT + dp_index * tp_size`) |
-| `tp_rank` | 0, 1, 2... | Each TP rank within a DP replica uses its own port |
-| orchestrator | +200 | Extra offset when caller is the orchestrator (avoids collision with stage workers on the same node) |
-
-**Example** (base=50051, stage 0→1, DP=2, TP=2, kv_transfer):
-
-| Caller | DP | TP rank | Port |
-|---|---|---|---|
-| Stage worker | DP0 | rank 0 | `50051 + 100 + 0 + 0×2 + 0 = 50151` |
-| Stage worker | DP0 | rank 1 | `50051 + 100 + 0 + 0×2 + 1 = 50152` |
-| Stage worker | DP1 | rank 0 | `50051 + 100 + 0 + 1×2 + 0 = 50153` |
-| Stage worker | DP1 | rank 1 | `50051 + 100 + 0 + 1×2 + 1 = 50154` |
-| Orchestrator | — | — | `50051 + 200 + 0 = 50251` |
-
-## Memory Pool Modes
-
-| Mode | Config | Data Flow | Best For |
-|---|---|---|---|
-| CPU Pinned | `memory_pool_device: "cpu"` | GPU → CPU pool → RDMA → CPU pool → GPU | Most hardware topologies (recommended) |
-| GPUDirect | `memory_pool_device: "cuda"` | GPU → GPU pool → RDMA (NIC reads GPU BAR1) → GPU pool | NIC-GPU direct PCIe (PIX topology) |
-
-> **Note**: GPUDirect RDMA requires the NIC and GPU to share a direct PCIe
-> switch (PIX topology). On systems where they are connected via PXB or NODE,
-> CPU pinned memory is faster due to GPU BAR1 bandwidth limitations.
-
-## Environment Variables
-
-| Variable | Description |
-|---|---|
-| `RDMA_DEVICE_NAME` | Override RDMA device name (e.g., `mlx5_0`). |
-| `MC_IB_PCI_RELAXED_ORDERING` | Set to `1` to enable PCIe relaxed ordering for GPUDirect. |
-
-## Docker / Container Setup
-
-RDMA requires host-level device access:
-
-```bash
-docker run -it \
-    --cap-add=SYS_PTRACE \
-    --cap-add=IPC_LOCK \
-    --security-opt seccomp=unconfined \
-    --network=host \
-    --device=/dev/infiniband \
-    -v /sys/class/infiniband:/sys/class/infiniband:ro \
-    your-image:tag
-```
-
-## Performance
-
-Benchmark results on H800 GPUs with mlx5_0 RDMA NIC (~186 MB KV cache):
-
-| Metric | MooncakeStoreConnector | MooncakeTransferEngineConnector (CPU) |
-|---|---|---|
-| KV transfer wall time | ~810 ms | **~14 ms** |
-| RDMA throughput | N/A (TCP) | ~22 GB/s |
-| Speedup | 1x | **58x** |
-
-## Troubleshooting
-
-### Quick Diagnostics
-
-```bash
-# 1. Check RDMA devices and link status
-ibdev2netdev
-# Expected: "mlx5_X port 1 ==> <iface> (Up)"
-# RoCE devices map to Ethernet interfaces (e.g., enp75s0f0)
-# IB devices map to ib0, ib1, etc.
-
-# 2. Check InfiniBand device details
-ibstat
-
-# 3. Verify /dev/infiniband is accessible (required in containers)
-ls /dev/infiniband/
-
-# 4. Check Mooncake installation
-python -c "from mooncake.engine import TransferEngine; print('OK')"
-
-# 5. Check environment variables
-echo "RDMA_DEVICE_NAME=${RDMA_DEVICE_NAME:-<not set>}"
-echo "MC_IB_PCI_RELAXED_ORDERING=${MC_IB_PCI_RELAXED_ORDERING:-<not set>}"
-```
-
-### Common Issues
-
-| Symptom | Cause | Fix |
-|---|---|---|
-| `Failed to modify QP to RTR` | Cross-NIC QP handshake failure (multi-NIC DGX) | Set `device_name` to a single RoCE NIC (e.g., `mlx5_2`) or set `RDMA_DEVICE_NAME` env var |
-| `transport retry counter exceeded` | RDMA path between incompatible NICs | Same as above — restrict to one NIC |
-| `zmq.error.Again: Resource temporarily unavailable` | ZMQ recv timeout (transfer took too long) | Check NIC selection; increase data may need longer timeout |
-| `Mooncake Engine initialization failed` | Missing RDMA drivers or `/dev/infiniband` | Install Mellanox OFED; in Docker add `--device=/dev/infiniband` |
-| `MemoryError` in allocator | Memory pool too small for payload | Increase `memory_pool_size` |
-| GPU transfer slower than CPU | GPU BAR1 bandwidth limitation (PXB/NODE topology) | Use `memory_pool_device: "cpu"` instead of `"cuda"` |
-
-### Multi-NIC Environments (DGX)
-
-On DGX machines with 12+ RDMA NICs, only RoCE NICs (with a bound network
-interface) reliably support loopback. IB-only NICs may fail cross-NIC QP
-handshakes. To identify RoCE NICs:
-
-```bash
-ibdev2netdev | grep -v "ib[0-9]"
-# RoCE devices show Ethernet interface names like enp75s0f0
-```
-
-Then configure the connector:
-```yaml
-device_name: "mlx5_2"  # or set RDMA_DEVICE_NAME=mlx5_2
-```
-
-See the RDMA Test README in tests/distributed/omni_connectors/README.md
-for test-specific setup instructions.
-
-For more details on the underlying engine, refer to the
-[Mooncake repository](https://github.com/kvcache-ai/Mooncake).

docs/design/feature/omni_connectors/shared_memory_connector.md

@@ -1,27 +0,0 @@
-# SharedMemoryConnector
-
-## When to Use
-
-Best for single-node deployments where stages run on the same host. It is
-auto-configured when no explicit connector is specified for an edge.
-
-## How It Works
-
-- Small payloads (< threshold): serialized and passed inline in metadata.
-- Large payloads (>= threshold): stored in shared memory; the SHM name is
-  returned in metadata.
-
-## Configuration
-
-```yaml
-runtime:
-  connectors:
-    connector_of_shared_memory:
-      name: SharedMemoryConnector
-      extra:
-        shm_threshold_bytes: 65536
-```
-
-## Notes
-
-- Auto-mode uses SharedMemoryConnector if no connector is declared for an edge.

docs/design/feature/omni_connectors/yuanrong_connector.md

@@ -1,101 +0,0 @@
-# YuanrongConnector
-
-## When to Use
-
-Best for multi-node distributed inference using Yuanrong Datasystem.
-
-## Mechanism
-
-Uses Yuanrong Datasystem's distributed KV store (`datasystem.kv_client`).
-
-- Data Plane: TCP or RDMA for high-bandwidth transfer.
-- Control Plane: Yuanrong Datasystem workers and etcd.
-- Keying: deterministic keys based on `put_key` (often composed as `request_id:fromStage_toStage`).
-
-## Installation
-
-```bash
-pip install openyuanrong-datasystem
-```
-
-## Start etcd
-
-```bash
-# Download and install etcd (v3.5.12 or higher)
-ETCD_VERSION="v3.5.12"
-ETCD_ARCH="linux-arm64"
-wget https://github.com/etcd-io/etcd/releases/download/${ETCD_VERSION}/etcd-${ETCD_VERSION}-${ETCD_ARCH}.tar.gz
-tar -xvf etcd-${ETCD_VERSION}-${ETCD_ARCH}.tar.gz
-cd etcd-${ETCD_VERSION}-${ETCD_ARCH}
-sudo cp etcd etcdctl /usr/local/bin/
-
-# Start etcd
-etcd \
-  --name etcd-single \
-  --data-dir /tmp/etcd-data \
-  --listen-client-urls http://0.0.0.0:2379 \
-  --advertise-client-urls http://0.0.0.0:2379 \
-  --listen-peer-urls http://0.0.0.0:2380 \
-  --initial-advertise-peer-urls http://0.0.0.0:2380 \
-  --initial-cluster etcd-single=http://0.0.0.0:2380 &
-
-# Verify etcd is running
-etcdctl --endpoints "127.0.0.1:2379" put key "value"
-etcdctl --endpoints "127.0.0.1:2379" get key
-```
-
-For production environments, refer to the
-[official etcd clustering documentation](https://etcd.io/docs/current/op-guide/clustering/).
-
-## Start Datasystem Worker
-
-```bash
-# Replace ${ETCD_IP} with etcd node IP, ${WORKER_IP} with local node IP
-dscli start -w \
-  --worker_address "${WORKER_IP}:31501" \
-  --etcd_address "${ETCD_IP}:2379" \
-  --shared_memory_size_mb 20480
-```
-
-To stop the worker:
-
-```bash
-dscli stop --worker_address "${WORKER_IP}:31501"
-```
-
-## Configuration
-
-Define the connector in runtime:
-
-```yaml
-runtime:
-  connectors:
-    connector_of_yuanrong:
-      name: YuanrongConnector
-      extra:
-        host: "127.0.0.1"
-        port: 31501
-        get_sub_timeout_ms: 1000
-```
-
-Wire stages to the connector:
-
-```yaml
-stage_args:
-  - stage_id: 0
-    output_connectors:
-      to_stage_1: connector_of_yuanrong
-
-  - stage_id: 1
-    input_connectors:
-      from_stage_0: connector_of_yuanrong
-```
-
-Parameters:
-
-- host: datasystem worker host.
-- port: datasystem worker port.
-- get_sub_timeout_ms: get timeout in milliseconds (0 for no timeout).
-
-For more details, refer to the
-[Yuanrong Datasystem repository](https://atomgit.com/openeuler/yuanrong-datasystem).

docs/design/feature/ray_based_execution.md

@@ -1,17 +1,14 @@
-# Distributed utils
+# VLLM-Omni Distributed Utilities
 
 This directory (vllm_omni/distributed/ray_utils) contains utilities for distributed execution in vllm-omni, supporting both **Ray** and **Multiprocessing** backends.
-## 1. Installation
-```bash
-pip install "ray[default]"
-```
-## 2. Ray Utils
+
+## 1. Ray Utils
 
 The `ray_utils` module provides helper functions for managing Ray clusters and actors, which is essential for:
 *   **Multi-node deployment**: Running pipeline stages across different physical machines.
 *   **Resource management**: Efficient GPU/CPU allocation.
 
-### 2.1 Basic Usage
+### 1.1 Basic Usage
 
 To use the Ray backend, specify `worker_backend="ray"` when initializing the engine.
 
@@ -24,7 +21,7 @@ vllm serve Qwen/Qwen2.5-Omni-7B \
   --ray-address auto
 ```
 
-### 2.2 Cluster Setup
+### 1.2 Cluster Setup
 
 **Step 1: Start Head Node**
 Run this on your primary machine:
@@ -41,19 +38,24 @@ ray start --address=<HEAD_NODE_IP>:6399
 > **Tip**: For a complete cluster setup script, refer to the vLLM example:
 > [run_cluster.sh](https://github.com/vllm-project/vllm/blob/main/examples/online_serving/run_cluster.sh)
 
-### 2.3 Distributed Connector Support
+### 1.3 Distributed Connector Support
 
 When running on Ray, the system automatically adapts its communication strategy:
 
-*   **Cross-Node**: Recommended to use `MooncakeTransferEngineConnector` (RDMA, fastest) or `MooncakeStoreConnector` (TCP fallback).
+*   **Cross-Node**: Recommended to use `MooncakeConnector` (requires separate configuration).
 *   **Same-Node**: Can still use `SharedMemoryConnector` for efficiency, or Ray's native object store (plasma).
 *   **SHM threshold default differs**: when `worker_backend="ray"`, the SharedMemoryConnector default threshold is set to `sys.maxsize`, which forces payloads to go inline (no SHM). Override `shm_threshold_bytes` in the connector config if you want SHM for Ray runs.
 
-### 2.4 Internal Helpers
+### 1.4 Internal Helpers
 
 *   **`initialize_ray_cluster`**: Connects to an existing Ray cluster or starts a local one.
 
-## 3. Troubleshooting
+## 2. Troubleshooting
 
 *   **Connection Issues**: Ensure the Ray head node is accessible and ports (default 6399 in this example) are open.
 *   **Version Mismatch**: Ensure all nodes run the same version of Ray and Python.
+
+### Installation
+```bash
+pip install "ray[default]"
+```

docs/design/feature/sequence_parallel.md

@@ -1,456 +0,0 @@
-# Sequence Parallel
-
-This section describes how to add Sequence Parallel (SP) to a diffusion transformer model. We use the Qwen-Image transformer and Wan2.2 transformer as reference implementations.
-
----
-
-## Table of Contents
-
-- [Overview](#overview)
-- [Approach 1: Non-Intrusive `_sp_plan` (Recommended)](#approach-1-non-intrusive-_sp_plan-recommended)
-- [Approach 2: Intrusive Modification (For Complex Cases)](#approach-2-intrusive-modification-for-complex-cases)
-- [Testing](#testing)
-- [Troubleshooting](#troubleshooting)
-- [Reference Implementations](#reference-implementations)
-- [Summary](#summary)
-
----
-
-## Overview
-
-
-### What is Sequence Parallel?
-
-**Terminology Note:** Our "Sequence Parallelism" (SP) corresponds to "Context Parallelism" (CP) in the [diffusers library](https://github.com/huggingface/diffusers/blob/main/src/diffusers/models/_modeling_parallel.py). We use "Sequence Parallelism" to align with vLLM-Omni's terminology.
-
-Diffusion transformers process long sequences of image patches or video frames. For high-resolution generation, these sequences can become very large. Enabling SP allows each GPU to process only a portion of the sequence, with attention mechanisms (Ulysses/Ring) handling cross-GPU communication transparently.
-
-### Architecture
-
-The major APIs for Sequence Parallel:
-
-```python
-from vllm_omni.diffusion.distributed.sp_plan import (
-    SequenceParallelInput,   # For sharding (splitting) tensors
-    SequenceParallelOutput,  # For gathering tensors
-)
-from vllm_omni.diffusion.distributed.sp_sharding import sp_shard, sp_gather
-```
-
-| Method/Class | Purpose | Behavior |
-|--------------|---------|----------|
-| `SequenceParallelInput` | Declare input sharding in `_sp_plan` | Auto-shards tensors at module input |
-| `SequenceParallelOutput` | Declare output gathering in `_sp_plan` | Auto-gathers tensors at module output |
-| `sp_shard()` | Manual tensor sharding | Splits tensor across SP workers |
-| `sp_gather()` | Manual tensor gathering | Gathers sharded tensors from all workers |
-
----
-
-## Approach 1: Non-Intrusive `_sp_plan` (Recommended)
-
-The `_sp_plan` mechanism allows SP **without modifying `forward()` logic**. The framework automatically registers hooks to shard inputs and gather outputs at module boundaries.
-
-**When to use:**
-- Standard transformer architectures
-- Tensor operations happen at `nn.Module` boundaries
-- Predictable sharding/gathering patterns
-
-**How it works:**
-1. Declare `_sp_plan` dict in your transformer class
-2. Framework automatically applies hooks when `sequence_parallel_size > 1`
-3. Hooks shard/gather tensors at specified module boundaries
-4. Attention layers handle cross-GPU communication internally
-
-```python
-class StandardTransformer(nn.Module):
-    _sp_plan = {
-        # Shard hidden_states at first transformer block input
-        "blocks.0": {
-            "hidden_states": SequenceParallelInput(split_dim=1, expected_dims=3),
-        },
-        # Gather at final output projection
-        "proj_out": SequenceParallelOutput(gather_dim=1, expected_dims=3),
-    }
-```
-
-`StandardTransformer` has a transformer blocks list `self.blocks = nn.ModuleList([...])`, and a projection output layer `self.proj_out`. The `_sp_plan` above defines that when SP is enabled, sharding the input tensor to the first transformer block, and gathering the sharded tensor at the final output projection layer.
-
-**Requirements:**
-- Tensor operations that need sharding/gathering must happen at **`nn.Module` boundaries**
-- Inline Python operations (e.g., `torch.cat`, `pad_sequence`) **cannot be hooked**
-
-**Solution for inline operations:** Extract into a submodule (see Step 2 below).
-
-### Step 1: Understand Module Boundaries
-
-Identify where tensors need to be sharded or gathered in your model's forward pass:
-
-```python
-class MyTransformer(nn.Module):
-    def __init__(self):
-        self.patch_embed = PatchEmbed()      # ← Boundary 1
-        self.pos_embed = RoPE()              # ← Boundary 2
-        self.blocks = nn.ModuleList([...])   # ← Boundary 3
-        self.norm_out = LayerNorm()
-        self.proj_out = Linear()             # ← Boundary 4
-
-    def forward(self, x):
-        x = self.patch_embed(x)              # ← Shard before this?
-        pos = self.pos_embed(x)              # ← Shard RoPE outputs?
-        for block in self.blocks:
-            x = block(x, pos)                # ← Blocks process sharded x
-        x = self.norm_out(x)
-        output = self.proj_out(x)            # ← Gather after this?
-        return output
-```
-
-### Step 2: Handle Inline Operations
-
-If your `forward()` contains inline tensor operations, **extract them into submodules**.
-
-**Example: Z-Image concatenates image + text features inline**
-
-```python
-# ❌ BAD: Inline operation - hooks cannot intercept
-class ZImageTransformer(nn.Module):
-    def forward(self, x, cap_feats):
-        # This concatenation happens inline - _sp_plan can't shard it!
-        unified = torch.cat([x, cap_feats], dim=1)
-
-        for layer in self.layers:
-            unified = layer(unified)
-
-        return unified
-
-# ✅ GOOD: Extract into submodule
-class UnifiedPrepare(nn.Module):
-    """Submodule to concatenate image and text features."""
-    def forward(self, x, cap_feats):
-        return torch.cat([x, cap_feats], dim=1)
-
-class ZImageTransformer(nn.Module):
-    def __init__(self):
-        super().__init__()
-        self.unified_prepare = UnifiedPrepare()  # Now a module!
-        self.layers = nn.ModuleList([...])
-
-    def forward(self, x, cap_feats):
-        # Now _sp_plan can shard the output of unified_prepare!
-        unified = self.unified_prepare(x, cap_feats)
-
-        for layer in self.layers:
-            unified = layer(unified)
-
-        return unified
-```
-
-**Other common cases:**
-- `pad_sequence()` → `PadSequenceModule`
-- `torch.cat()` → `ConcatModule`
-- `tensor.reshape()` → `ReshapeModule`
-- Complex preprocessing → `PreprocessModule`
-
-### Step 3: Write `_sp_plan` for Your Model
-
-Create a class-level `_sp_plan` dictionary specifying where to shard/gather tensors.
-
-Typically, there are two patterns for diffusion models:
-
-**Pattern 1: Shard at first block, gather at output projection**
-
-Most common pattern for standard transformers:
-
-```python
-from vllm_omni.diffusion.distributed.sp_plan import (
-    SequenceParallelInput,   # For sharding (splitting) tensors
-    SequenceParallelOutput,  # For gathering tensors
-)
-class StandardTransformer(nn.Module):
-    _sp_plan = {
-        # Shard hidden_states at first transformer block input
-        "blocks.0": {
-            "hidden_states": SequenceParallelInput(split_dim=1, expected_dims=3),
-        },
-        # Gather at final output projection
-        "proj_out": SequenceParallelOutput(gather_dim=1, expected_dims=3),
-    }
-```
-
-**Pattern 2: Shard RoPE embeddings separately**
-
-When RoPE is computed in a separate module:
-
-```python
-from vllm_omni.diffusion.distributed.sp_plan import (
-    SequenceParallelInput,   # For sharding (splitting) tensors
-    SequenceParallelOutput,  # For gathering tensors
-)
-class TransformerWithRoPE(nn.Module):
-    _sp_plan = {
-        # Shard RoPE module OUTPUTS (returns tuple of cos, sin)
-        "rope": {
-            0: SequenceParallelInput(split_dim=1, expected_dims=4, split_output=True),  # cos
-            1: SequenceParallelInput(split_dim=1, expected_dims=4, split_output=True),  # sin
-        },
-        # Shard transformer block INPUT
-        "blocks.0": {
-            "hidden_states": SequenceParallelInput(split_dim=1, expected_dims=3),
-        },
-        # Gather at output
-        "proj_out": SequenceParallelOutput(gather_dim=1, expected_dims=3),
-    }
-```
-
-### API Reference
-
-**SequenceParallelInput Parameters:**
-
-| Parameter | Type | Description |
-|-----------|------|-------------|
-| `split_dim` | int | Dimension to split (usually `1` for sequence) |
-| `expected_dims` | int \| None | Expected tensor rank for validation (optional) |
-| `split_output` | bool | `False`: shard **input** params; `True`: shard **output** tensors |
-| `auto_pad` | bool | Auto-pad if sequence not divisible by world_size (default: `False`) |
-
-**SequenceParallelOutput Parameters:**
-
-| Parameter | Type | Description |
-|-----------|------|-------------|
-| `gather_dim` | int | Dimension to gather (usually `1` for sequence) |
-| `expected_dims` | int \| None | Expected tensor rank for validation (optional) |
-
-**Module Naming Conventions:**
-
-| Key | Meaning | Python equivalent |
-|-----|---------|-------------------|
-| `""` | Root model | `model` |
-| `"blocks.0"` | First element of ModuleList | `model.blocks[0]` |
-| `"blocks.*"` | All elements of ModuleList | `for b in model.blocks` |
-| `"rope"` | Named submodule | `model.rope` |
-| `"outputs.main"` | ModuleDict entry | `model.outputs["main"]` |
-
-**Dictionary Value Types:**
-
-| Key type | `split_output` | Description |
-|----------|----------------|-------------|
-| `"param_name"` (str) | `False` | Shard **input parameter** by name |
-| `0`, `1`, ... (int) | `True` | Shard **output tuple** by index |
-
----
-
-## Approach 2: Intrusive Modification (For Complex Cases)
-
-For models with dynamic sharding logic that cannot be expressed via `_sp_plan`, manually insert shard/gather calls.
-
-**When to use:**
-- Dynamic/conditional sharding logic
-- Complex tensor manipulations that can't be encapsulated
-- Temporary workaround during development
-
-```python
-from vllm_omni.diffusion.distributed.sp_sharding import sp_shard, sp_gather
-
-def forward(self, hidden_states, ...):
-    if self.parallel_config.sequence_parallel_size > 1:
-        hidden_states = sp_shard(hidden_states, dim=1)
-
-    # ... computation ...
-
-    if self.parallel_config.sequence_parallel_size > 1:
-        output = sp_gather(output, dim=1)
-
-    return output
-```
-
----
-
-## Testing
-
-After implementing Sequence Parallel support, thoroughly test your implementation to ensure correctness and performance across different configurations.
-
-**Test Different `sp_size`:**
-
-Test your model with various sequence parallel world sizes to verify correctness and identify optimal configurations:
-
-```bash
-cd examples/offline_inference/text_to_image
-python text_to_image.py \
-    --model Your-org/your-model \
-    --prompt "a cup of coffee on the table" \
-    --num-inference-steps 50 \
-    --ulysses-degree 2 \
-    --ring-degree 2 \
-    --output sp_test_image_ulysses=2_ring=2.png
-```
-
-**Verify:**
-
-1. **Correctness:** Output should be identical across all `sp_size` values
-2. **Speed:** Throughput should remain stable or improve (especially for large sequences)
-3. **Logs:** Check for any shape mismatch or communication errors
-
-**Test with Tensor Parallel:**
-
-Sequence Parallel can be combined with other parallelism strategies:
-
-```bash
-cd examples/offline_inference/text_to_image
-python text_to_image.py \
-    --model Your-org/your-model \
-    --prompt "a cup of coffee on the table" \
-    --num-inference-steps 50 \
-    --ulysses-degree 2 \
-    --tensor-parallel-size 2 \
-    --output sp_test_image_ulysses=2_tp=2.png
-```
-
----
-
-## Troubleshooting
-
-### Issue: Shape mismatch errors
-
-**Symptoms:** `RuntimeError: shape mismatch` during forward pass.
-
-**Causes & Solutions:**
-
-- **RoPE dimension mismatch:**
-
-**Problem:** RoPE embeddings not sharded, but hidden_states is sharded.
-
-**Solution:** Shard RoPE outputs in `_sp_plan`:
-```python
-_sp_plan = {
-    "rope": {
-        0: SequenceParallelInput(split_dim=1, expected_dims=4, split_output=True),
-        1: SequenceParallelInput(split_dim=1, expected_dims=4, split_output=True),
-    },
-    ...
-}
-```
-
-- **Sequence Length not divisible by sp_size:**
-
-**Problem:** `SequenceParallelInput(auto_pad=False)` - auto_pad should be True to enable automatic sequence padding.
-
-**Solution:** In `SequenceParallelInput`, set `auto_pad=True` and add attention mask support.
-
-> **Experimental Feature:** `auto_pad=True` is an experimental feature and may be changed in the future. We plan to improve this solution to involve minimal changes to model files. More details are [here](https://github.com/vllm-project/vllm-omni/issues/1324).
-
-**Constraints of auto_pad:**
-
-| Constraint | Description |
-|------------|-------------|
-| **Attention Backend Compatibility** | The attention backends must support `attention_mask`. Currently only `TORCH_SDPA` and `FLASH_ATTN` (default for diffusion models) are supported. |
-| **Ring Attention Limitation** | Ring attention does not support `attention_mask`. Therefore, when using `auto_pad=True`, the combination of Ulysses + Ring attention is not feasible. |
-
-1. Enable `auto_pad=True` for all sequence-dimension inputs in `_sp_plan`:
-```python
-_sp_plan = {
-    "rope": {
-        0: SequenceParallelInput(split_dim=1, expected_dims=4, split_output=True, auto_pad=True),
-        1: SequenceParallelInput(split_dim=1, expected_dims=4, split_output=True, auto_pad=True),
-    },
-    "blocks.0": {
-        "hidden_states": SequenceParallelInput(split_dim=1, expected_dims=3, auto_pad=True)
-    },
-    ...
-}
-```
-
-2. Create attention mask dynamically when padding is applied:
-```python
-from vllm_omni.diffusion.forward_context import get_forward_context
-from vllm_omni.diffusion.attention.backends.abstract import AttentionMetadata
-
-# In model forward(), before transformer blocks:
-hidden_states_mask = None
-ctx = get_forward_context()
-if ctx.sp_original_seq_len is not None and ctx.sp_padding_size > 0:
-    batch_size = hidden_states.shape[0]
-    padded_seq_len = ctx.sp_original_seq_len + ctx.sp_padding_size
-    hidden_states_mask = torch.ones(batch_size, padded_seq_len, dtype=torch.bool, device=hidden_states.device)
-    hidden_states_mask[:, ctx.sp_original_seq_len:] = False
-
-# Pass mask to attention layers
-attn_metadata = AttentionMetadata(attn_mask=hidden_states_mask) if hidden_states_mask is not None else None
-output = self.attn(query, key, value, attn_metadata)
-```
-
-**Important Quality Considerations:**
-
-While `auto_pad` enables generation for irregular resolutions, be aware of potential quality impacts:
-
-| Aspect | Impact |
-|--------|--------|
-| **Training Distribution** | Models perform best on aspect ratios seen during training (e.g., 1:1, 16:9, 4:3). Unusual ratios like 700x400 (1.75:1) may produce lower quality results. |
-| **Padding Overhead** | Padded positions consume compute even when masked. For best efficiency, prefer resolutions divisible by `sp_size`. |
-
-**Recommendations for users:**
-- Use standard aspect ratios when possible (e.g., 768x432 for 16:9 instead of 700x400)
-- Ensure post-patch dimensions are divisible by `sp_size` for optimal quality
-- Test generation quality when using unusual resolutions
-
-### Issue: Inline operations not sharded
-
-**Symptoms:** Some tensors remain full-sized, not sharded.
-
-**Causes & Solutions:**
-
-- **Operations happen inline in `forward()`, not at module boundaries:**
-
-**Problem:**
-```python
-def forward(self, x, cap):
-    unified = torch.cat([x, cap], dim=1)  # ← Inline operation!
-    # _sp_plan can't hook this
-```
-
-**Solution:** Extract into submodule:
-```python
-class ConcatModule(nn.Module):
-    def forward(self, x, cap):
-        return torch.cat([x, cap], dim=1)
-
-class MyModel(nn.Module):
-    _sp_plan = {
-        "concat": {
-            0: SequenceParallelInput(split_dim=1, expected_dims=4, split_output=True),
-            1: SequenceParallelInput(split_dim=1, expected_dims=4, split_output=True),
-        },
-        ...
-    }
-    def __init__(self):
-        self.concat = ConcatModule()  # Now hookable!
-
-    def forward(self, x, cap):
-        unified = self.concat(x, cap)  # ← Can be sharded via _sp_plan
-```
-
----
-
-## Reference Implementations
-
-Complete examples in the codebase:
-
-| Model | Path | Pattern | Notes |
-|-------|------|---------|-------|
-| **Qwen-Image** | `vllm_omni/diffusion/models/qwen_image/qwen_image_transformer.py` | Dual-stream + preprocessing | auto_pad, separate RoPE |
-| **Wan2.2** | `vllm_omni/diffusion/models/wan2_2/wan2_2_transformer.py` | Dual-Transformer + RoPE | Video transformer |
-| **Z-Image** | `vllm_omni/diffusion/models/z_image/z_image_transformer.py` | Unified sequence | Concatenated input |
-| **SP Plan Types** | `vllm_omni/diffusion/distributed/sp_plan.py` | Type definitions | SequenceParallelInput/Output |
-| **Hook Implementation** | `vllm_omni/diffusion/hooks/sequence_parallel.py` | Hook mechanics | How hooks work |
-| **Tests** | `tests/diffusion/distributed/test_sp_plan_hooks.py` | Test examples | Validation patterns |
-
----
-
-## Summary
-
-Adding Sequence Parallel support to a transformer:
-
-1. ✅ **Choose approach** - Use `_sp_plan` for standard cases, intrusive modification for complex cases
-2. ✅ **Identify sharding boundaries** - Where should tensors be split/gathered?
-3. ✅ **Extract inline operations** - Move `torch.cat`, `pad_sequence`, etc. to submodules
-4. ✅ **Define `_sp_plan`** - Declare shard/gather points as class attribute
-5. ✅ **Use `auto_pad` for variable lengths** - Support non-uniform sequences
-6. ✅ **Test** - Verify with different `ulysses_degree` and `ring_degree` combinations

docs/design/feature/teacache.md

@@ -1,459 +0,0 @@
-# TeaCache
-
-This section describes how to add TeaCache to a diffusion transformer model. We use the Qwen-Image transformer as the reference implementation.
-
----
-
-## Table of Contents
-
-- [Overview](#overview)
-- [Step-by-Step Implementation](#step-by-step-implementation)
-- [Customization](#customization)
-- [Testing](#testing)
-- [Troubleshooting](#troubleshooting)
-- [Reference Implementations](#reference-implementations)
-- [Summary](#summary)
-
----
-
-## Overview
-
-### What is TeaCache?
-
-TeaCache speeds up diffusion inference by caching transformer block computations when consecutive timesteps are similar. It provides **1.5x-2.0x speedup** with minimal quality loss.
-
-The core insight is that the modulated input (after normalization and timestep conditioning) changes gradually across timesteps. By measuring the L1 distance between consecutive modulated inputs and comparing it to a threshold, TeaCache decides whether to execute the full transformer blocks or reuse the cached residual from the previous step.
-
-vLLM-omni provides a **hook-based** TeaCache system that requires **zero changes to model code**. The hook completely intercepts the transformer's forward pass and implements adaptive caching transparently. This design allows easy integration with any transformer model by simply writing an extractor function.
-
-### Architecture
-
-The TeaCache system consists of three main components:
-
-| Component | Purpose | Location |
-|-----------|---------|----------|
-| [`CacheContext`](https://docs.vllm.ai/projects/vllm-omni/en/latest/api/vllm_omni/diffusion/cache/#vllm_omni.diffusion.cache.CacheContext) | Dataclass containing model-specific information for caching | `vllm_omni/diffusion/cache/teacache/context.py` |
-| [`EXTRACTOR_REGISTRY`](https://docs.vllm.ai/projects/vllm-omni/en/latest/api/vllm_omni/diffusion/cache/teacache/extractors/#vllm_omni.diffusion.cache.teacache.extractors.EXTRACTOR_REGISTRY) | Maps transformer class names to extractor functions | `vllm_omni/diffusion/cache/teacache/extractors.py` |
-| [`TeaCacheConfig`](https://docs.vllm.ai/projects/vllm-omni/en/latest/api/vllm_omni/diffusion/cache/#vllm_omni.diffusion.cache.TeaCacheConfig) | Configuration including thresholds and polynomial coefficients | `vllm_omni/diffusion/cache/teacache/config.py` |
-
-The hook handles all caching logic automatically, including:
-
-- CFG-aware state management (separate states for positive/negative branches)
-- CFG-parallel compatibility
-- L1 distance computation with polynomial rescaling
-- Residual caching and reuse
-
-
----
-
-## Step-by-Step Implementation
-
-To add TeaCache support for a new model, you need to:
-
-1. Write an **extractor function** that returns a `CacheContext` object
-2. Register the extractor in the `EXTRACTOR_REGISTRY`
-3. Add model-specific polynomial coefficients to `TeaCacheConfig`
-
-### Step 1: Model-Specific Preprocessing
-
-Extract and process model inputs. This typically involves:
-- Embedding image/latent inputs
-- Processing text encoder outputs (if dual-stream)
-- Creating timestep embeddings
-- Computing positional embeddings
-
-**Example (Qwen-Image):**
-
-```python
-def extract_qwen_context(
-    module: nn.Module,
-    hidden_states: torch.Tensor,
-    encoder_hidden_states: torch.Tensor,
-    encoder_hidden_states_mask: torch.Tensor,
-    timestep: torch.Tensor,
-    img_shapes: torch.Tensor,
-    txt_seq_lens: torch.Tensor,
-    guidance: torch.Tensor | None = None,
-    **kwargs: Any,
-) -> CacheContext:
-    # Validate model structure
-    if not hasattr(module, "transformer_blocks") or len(module.transformer_blocks) == 0:
-        raise ValueError("Module must have transformer_blocks")
-
-    # Preprocessing: embed inputs
-    hidden_states = module.img_in(hidden_states)
-    timestep = timestep.to(device=hidden_states.device, dtype=hidden_states.dtype)
-    encoder_hidden_states = module.txt_norm(encoder_hidden_states)
-    encoder_hidden_states = module.txt_in(encoder_hidden_states)
-
-    # Create timestep embedding
-    if guidance is not None:
-        guidance = guidance.to(hidden_states.dtype) * 1000
-    temb = (
-        module.time_text_embed(timestep, hidden_states)
-        if guidance is None
-        else module.time_text_embed(timestep, guidance, hidden_states)
-    )
-
-    # Compute position embeddings
-    image_rotary_emb = module.pos_embed(img_shapes, txt_seq_lens, device=hidden_states.device)
-```
-
-### Step 2: Extract Modulated Input
-
-The modulated input is used for cache decisions. Extract it from the **first transformer block** after normalization and modulation.
-
-**Example (Qwen-Image):**
-
-```python
-    # Extract modulated input from first transformer block
-    block = module.transformer_blocks[0]
-    img_mod_params = block.img_mod(temb)
-    img_mod1, _ = img_mod_params.chunk(2, dim=-1)
-    img_modulated, _ = block.img_norm1(hidden_states, img_mod1)
-```
-
-**Key Points:**
-
-- Use the **first block** to extract modulated input early
-- Apply the same normalization and modulation as the actual forward pass
-- The tensor should represent the processed features that will change across timesteps
-
-### Step 3: Define Transformer Execution
-
-Create a callable that executes all transformer blocks. This encapsulates the main computation loop.
-
-**Example (Qwen-Image dual-stream):**
-
-```python
-    def run_transformer_blocks():
-        """Execute all Qwen transformer blocks."""
-        h = hidden_states
-        e = encoder_hidden_states
-
-        for block in module.transformer_blocks:
-            e, h = block(
-                hidden_states=h,
-                encoder_hidden_states=e,
-                encoder_hidden_states_mask=encoder_hidden_states_mask,
-                temb=temb,
-                image_rotary_emb=image_rotary_emb,
-            )
-        return (h, e)  # Return both image and text hidden states
-```
-
-**Example (Single-stream model like Flux):**
-
-```python
-    def run_transformer_blocks():
-        """Execute all Flux transformer blocks."""
-        h = hidden_states
-
-        for block in module.transformer_blocks:
-            h = block(h, temb=temb)
-        return (h,)  # Return only image hidden states
-```
-
-**Key Points:**
-
-- Return format:
-- For single-stream models: return `(hidden_states,)`
-- For dual-stream models: return `(hidden_states, encoder_hidden_states)`
-
-### Step 4: Define Postprocessing
-
-Create a callable that applies final transformations to produce the model output.
-
-**Example (Qwen-Image):**
-
-```python
-    return_dict = kwargs.get("return_dict", True)
-
-    def postprocess(h):
-        """Apply Qwen-specific output postprocessing."""
-        h = module.norm_out(h, temb)
-        output = module.proj_out(h)
-        if not return_dict:
-            return (output,)
-        return Transformer2DModelOutput(sample=output)
-```
-
-### Step 5: Return CacheContext
-
-Package all information into a `CacheContext` object.
-
-```python
-    return CacheContext(
-        modulated_input=img_modulated,
-        hidden_states=hidden_states,
-        encoder_hidden_states=encoder_hidden_states,  # or None for single-stream
-        temb=temb,
-        run_transformer_blocks=run_transformer_blocks,
-        postprocess=postprocess,
-    )
-```
-
-**CacheContext Fields:**
-
-| Field | Type | Purpose |
-|-------|------|---------|
-| `modulated_input` | `torch.Tensor` | Tensor used for cache decision (similarity comparison) |
-| `hidden_states` | `torch.Tensor` | Current hidden states (will be modified by caching) |
-| `encoder_hidden_states` | `torch.Tensor | None` | Encoder states for dual-stream models, `None` for single-stream |
-| `temb` | `torch.Tensor` | Timestep embedding tensor |
-| `run_transformer_blocks` | `Callable[[], tuple]` | Executes transformer blocks, returns `(hidden_states, [encoder_hidden_states])` |
-| `postprocess` | `Callable[[torch.Tensor], Any]` | Applies final transformations to produce model output |
-| `extra_states` | `dict | None` | Optional dict for additional model-specific state |
-
-### Step 6: Register the Extractor
-
-Add your extractor to the `EXTRACTOR_REGISTRY` in `vllm_omni/diffusion/cache/teacache/extractors.py`:
-
-```python
-EXTRACTOR_REGISTRY: dict[str, Callable] = {
-    "QwenImageTransformer2DModel": extract_qwen_context,
-    "Bagel": extract_bagel_context,
-    "ZImageTransformer2DModel": extract_zimage_context,
-    "YourModelTransformer2DModel": extract_your_model_context,  # Add here
-}
-```
-
-**Key:** Use the transformer class name (`module.__class__.__name__`)
-
-### Step 7: Add Model Coefficients
-
-Add polynomial rescaling coefficients to `vllm_omni/diffusion/cache/teacache/config.py`:
-
-```python
-_MODEL_COEFFICIENTS = {
-    "QwenImageTransformer2DModel": [
-        -4.50000000e02,
-        2.80000000e02,
-        -4.50000000e01,
-        3.20000000e00,
-        -2.00000000e-02,
-    ],
-    "YourModelTransformer2DModel": [  # Add your model's coefficients
-        # 5 polynomial coefficients (can reuse similar model's coefficients initially)
-    ],
-}
-```
-
-
-**Initial approach:** Start with coefficients from a similar model architecture, then tune empirically following [Customization](#customization) section.
-
----
-
-## Customization
-
-### Coefficient Estimation
-
-While you can start with coefficients from a similar model architecture, estimating custom coefficients for your specific model typically improves TeaCache performance.
-
-**Why Estimate Coefficients?**
-
-The polynomial coefficients rescale L1 distances between consecutive modulated inputs to better predict when cached residuals can be reused. Model-specific coefficients account for:
-
-- Architecture differences (layer count, hidden size, attention patterns)
-- Training data characteristics
-- Noise prediction behavior across timesteps
-
-| Approach | Performance | Effort |
-|----------|-------------|--------|
-| Using defaults from similar model | Within 5-10% of optimal | Low |
-| Estimating custom coefficients | Best performance | Medium |
-
-#### Implement Data Collection Adapter
-
-Add an adapter in `vllm_omni/diffusion/cache/teacache/coefficient_estimator.py`:
-
-```python
-class YourModelAdapter:
-    """Adapter for coefficient estimation on your model."""
-
-    @staticmethod
-    def load_pipeline(model_path: str, device: str, dtype: torch.dtype) -> Any:
-        """Load your diffusion pipeline."""
-        from your_model_package import YourModelPipeline
-
-        pipeline = YourModelPipeline.from_pretrained(
-            model_path,
-            torch_dtype=dtype,
-        )
-        pipeline = pipeline.to(device)
-        return pipeline
-
-    @staticmethod
-    def get_transformer(pipeline: Any) -> tuple[Any, str]:
-        """Extract transformer from pipeline."""
-        return pipeline.transformer, "YourTransformer2DModel"
-
-    @staticmethod
-    def install_hook(transformer: Any, hook: DataCollectionHook) -> None:
-        """Install data collection hook on transformer."""
-        from vllm_omni.diffusion.hooks import HookRegistry
-
-        registry = HookRegistry.get_or_create(transformer)
-        registry.register_hook(hook._HOOK_NAME, hook)
-
-
-# Register your adapter
-_MODEL_ADAPTERS["YourModel"] = YourModelAdapter
-```
-
-#### Collect Data and Estimate
-
-```python
-from vllm_omni.diffusion.cache.teacache.coefficient_estimator import (
-    TeaCacheCoefficientEstimator,
-)
-from datasets import load_dataset
-from tqdm import tqdm
-
-# Initialize estimator
-estimator = TeaCacheCoefficientEstimator(
-    model_path="/path/to/your/model",
-    model_type="YourModel",
-)
-
-# Load diverse prompts (paper recommends ~70 prompts)
-dataset = load_dataset("nateraw/parti-prompts", split="train")
-prompts = dataset["Prompt"][:70]
-
-# Collect data
-for prompt in tqdm(prompts, desc="Collecting data"):
-    estimator.collect_from_prompt(prompt=prompt, num_inference_steps=50)
-
-# Estimate coefficients
-coeffs = estimator.estimate(poly_order=4)
-print(f"Estimated coefficients: {coeffs.tolist()}")
-```
-
-**Data Statistics Guide:**
-
-| Metric | Good Range | Warning Signs |
-|--------|------------|---------------|
-| **Count** | 2000-5000+ | < 1000: too few prompts |
-| **Input Diffs (x)** | 0.01-0.10 | Very small (<0.001): model may not modulate properly |
-| **Output Diffs (y)** | Should correlate with x | No correlation: check extractor |
-| **Coefficient magnitude** | -1e6 to 1e6 | > 1e8: numerical instability |
-
----
-
-## Testing
-
-After adding TeaCache support, test with:
-
-```python
-from vllm_omni import Omni
-from vllm_omni.inputs.data import OmniDiffusionSamplingParams
-
-omni = Omni(
-    model="your-model-name",
-    cache_backend="tea_cache",
-    cache_config={
-        "rel_l1_thresh": 0.2,
-        "coefficients": [1.33e6, -1.69e5, 7.95e3, -1.64e2, 1.26],  # Your coefficients
-    }
-)
-
-images = omni.generate(
-    "a beautiful landscape",
-    OmniDiffusionSamplingParams(num_inference_steps=50),
-)
-```
-
-**Verify:**
-
-1. **Check logs** - Look for TeaCache initialization messages
-2. **Compare performance** - Measure speedup vs baseline (expect 1.5x-2.0x)
-3. **Verify output quality** - Visually compare cached vs uncached outputs (should be nearly identical)
-
-See more detailed examples in [user guide for teacache](../../user_guide/diffusion/teacache.md).
-
----
-
-## Troubleshooting
-
-### Issue: "Unknown model type"
-
-**Symptoms:** Error message indicating the model type is not recognized when enabling TeaCache.
-
-**Causes & Solutions:**
-
-- **Extractor not registered:**
-
-**Problem:** The transformer class name doesn't exist in `EXTRACTOR_REGISTRY`.
-
-**Solution:** Check the class name and add to registry:
-```python
-# Check transformer class name
-print(pipeline.transformer.__class__.__name__)
-
-# Add to EXTRACTOR_REGISTRY
-EXTRACTOR_REGISTRY["YourTransformer2DModel"] = extract_your_context
-```
-
-- **Transformer class name mismatch:**
-
-**Solution:** Ensure the registry key matches exactly with `module.__class__.__name__`.
-
-### Issue: "Cannot find coefficients"
-
-**Symptoms:** Error when initializing TeaCache about missing model coefficients.
-
-**Causes & Solutions:**
-
-- **Missing coefficients in config:**
-
-**Solution:** Add coefficients to `_MODEL_COEFFICIENTS` in `config.py`, or pass custom coefficients:
-```python
-omni = Omni(
-    model="your-model",
-    cache_backend="tea_cache",
-    cache_config={"coefficients": [1.0, -0.5, 0.1, -0.01, 0.001]}
-)
-```
-
-### Issue: Quality Degradation
-
-**Symptoms:** Output images look noticeably different or have artifacts compared to baseline.
-
-**Causes & Solutions:**
-
-- **Threshold too high:**
-
-**Problem:** `rel_l1_thresh` is too aggressive, causing cache reuse when outputs differ significantly.
-
-**Solution:** Lower the threshold:
-```python
-cache_config={"rel_l1_thresh": 0.1}  # Try 0.1-0.2
-```
-
-- **Coefficients not tuned:**
-
-**Solution:** Estimate model-specific coefficients using the coefficient estimation process described above.
-
----
-
-## Reference Implementations
-
-Complete examples in the codebase:
-
-| Model | Path | Pattern | Notes |
-|-------|------|---------|-------|
-| **Qwen-Image** | `vllm_omni/diffusion/cache/teacache/extractors.py` | Dual-stream | `extract_qwen_context` |
-| **Bagel** | `vllm_omni/diffusion/cache/teacache/extractors.py` | Omni model | `extract_bagel_context` |
-| **TeaCache Core** | `vllm_omni/diffusion/cache/teacache/` | Base implementation | Hook and config |
-| **Coefficient Estimator** | `vllm_omni/diffusion/cache/teacache/coefficient_estimator.py` | Estimation tool | Adapter pattern |
-
----
-
-## Summary
-
-Adding TeaCache support:
-
-1. ✅ **Write extractor** - Create function returning `CacheContext` with model-specific preprocessing
-2. ✅ **Register extractor** - Add to `EXTRACTOR_REGISTRY` with transformer class name
-3. ✅ **Add coefficients** - Add polynomial coefficients to `_MODEL_COEFFICIENTS`
-4. ✅ **Test** - Verify with `cache_backend="tea_cache"`

docs/design/feature/tensor_parallel.md

@@ -1,279 +0,0 @@
-# Tensor Parallel
-
-This section describes how to add Tensor Parallel (TP) to a diffusion transformer model. We use the Z-Image transformer as the reference implementation.
-
----
-
-## Table of Contents
-
-- [Overview](#overview)
-- [Step-by-Step Implementation](#step-by-step-implementation)
-- [Testing](#testing)
-- [Troubleshooting](#troubleshooting)
-- [Reference Implementations](#reference-implementations)
-- [Summary](#summary)
-
----
-
-## Overview
-
-### What is Tensor Parallel?
-
-Tensor Parallel (TP) is a model parallelism technique that **shards model weights** across multiple GPUs. Each GPU holds only a portion of the model's parameters and computes only part of each layer's output.
-
-Diffusion transformers contain large attention and MLP layers. We can use Tensor Parallel to shard the model dimension across multiple GPUs, allowing larger models to fit in memory while achieving near-linear speedup.
-
-### Architecture
-
-The Tensor Parallel implementation relies vLLM's Parallel Layers:
-
-[vLLM Parallel Layers API Reference](https://docs.vllm.ai/en/latest/contributing/model/basic/?h=column#3-optional-implement-tensor-parallelism-and-quantization-support)
-
-**Parallel Layer Types:**
-
-| Layer Type | Purpose | Weight Partitioning |
-|------------|---------|---------------------|
-| `ColumnParallelLinear` | First FFN layer, separated QKV | Columns (output dimension) |
-| `RowParallelLinear` | Second FFN layer, attention output | Rows (input dimension) |
-| `QKVParallelLinear` | Multi-head/grouped-query attention QKV | Handles head replication automatically |
-| `ReplicatedLinear` | Layers that shouldn't be sharded | No partitioning (replicated) |
-
----
-
-## Step-by-Step Implementation
-
-
-### Step 1: Identify Linear Layers
-
-Find all `nn.Linear` layers in your transformer that need to be sharded.
-
-**Key questions:**
-- Which layers should be column parallel (weight split by columns)?
-- Which layers should be row parallel (weight split by rows)?
-
-### Step 2: Replace Linear Layers with Parallel Equivalents
-
-Replace `nn.Linear` with parallel layers from `vllm.model_executor.layers.linear`.
-
-**Example (MLP Block - Up-Down Pattern):**
-
-```python
-class FeedForward(nn.Module):
-    def __init__(self, dim: int, hidden_dim: int):
-        super().__init__()
-        # Column parallel: weight split by columns [hidden_dim/N, dim]
-        self.w1 = ColumnParallelLinear(
-            dim,
-            hidden_dim,
-            bias=False,
-            return_bias=False,
-        )
-        self.act = nn.GELU()
-
-        self.w2 = RowParallelLinear(
-            hidden_dim,
-            dim,
-            bias=False,
-            input_is_parallel=True,  # Input already sharded from w1
-            return_bias=False,
-        )
-
-    def forward(self, x):
-        # x: [batch, seq, dim] (replicated on all GPUs)
-        # w1 outputs sharded [batch, seq, hidden_dim/N]
-        x = self.w1(x)
-        # act operates on sharded tensors (no communication)
-        x = self.act(x)
-        # w2 outputs full dim [batch, seq, dim] via all-reduce
-        x = self.w2(x)
-        return x
-```
-
-**Example (Attention - QKV-Out Pattern):**
-
-```python
-from vllm_omni.diffusion.attention.layer import Attention
-class YourModelAttention(nn.Module):
-    def __init__(self, dim: int, num_heads: int, num_kv_heads: int):
-        super().__init__()
-        self.head_dim = dim // num_heads
-
-        # Column parallel: QKV weight split by columns
-        # Each GPU gets num_heads/N heads
-        self.to_qkv = QKVParallelLinear(
-            hidden_size=dim,
-            head_size=self.head_dim,
-            total_num_heads=num_heads,
-            total_num_kv_heads=num_kv_heads,
-            bias=False,
-            return_bias=False,
-        )
-
-        # Row parallel: output weight split by rows
-        self.to_out = RowParallelLinear(
-            dim,
-            dim,
-            bias=False,
-            input_is_parallel=True,  # Input sharded from attention
-            return_bias=False,
-        )
-
-        self.attn = Attention(
-            num_heads=self.to_qkv.num_heads, # Each GPU gets num_heads/N heads
-            head_size=self.head_dim,
-            softmax_scale=1.0 / (self.head_dim**0.5),
-            causal=False,
-            num_kv_heads=self.to_qkv.num_kv_heads,
-        )
-
-    def forward(self, x):
-        # x: [batch, seq, dim] (replicated)
-        # to_qkv outputs sharded [batch, seq, (q+k+v) * head_dim/N]
-        qkv = self.to_qkv(x)
-        # Split into Q, K, V (each sharded on heads)
-        q, k, v = qkv.split([...], dim=-1)
-        # Attention computed independently on each GPU
-        out = self.attn(q, k, v)
-        # to_out all-reduces to full dim
-        out = self.to_out(out)
-        return out
-```
-
-**Key Points:**
-
-- `ColumnParallelLinear` → `RowParallelLinear` is the standard pairing
-- Set `input_is_parallel=True` on `RowParallelLinear` when input comes from `ColumnParallelLinear`
-- Use `QKVParallelLinear` for attention projections (handles head replication automatically)
-
-### Step 3: Validate TP Constraints
-
-For correct TP operation, these dimensions **must be divisible** by `tensor_parallel_size`:
-
-| Dimension | Reason | Example Error |
-|-----------|--------|---------------|
-| `num_heads` | Heads sharded by QKVParallelLinear | `num_heads=30, tp=4` ❌ (30 % 4 ≠ 0) |
-| `num_kv_heads` | KV heads sharded by QKVParallelLinear | `num_kv_heads=30, tp=4` ❌ (30 % 4 ≠ 0) |
-
----
-
-## Testing
-
-After adding Tensor Parallel support, test with:
-
-```python
-from vllm_omni import Omni
-from vllm_omni.diffusion.data import DiffusionParallelConfig
-from vllm_omni.inputs.data import OmniDiffusionSamplingParams
-
-parallel_config = DiffusionParallelConfig(tensor_parallel_size=2)
-omni = Omni(model="your-model-name", parallel_config=parallel_config)
-
-output = omni.generate(
-    "a cup of coffee on the table",
-    OmniDiffusionSamplingParams(num_inference_steps=50),
-)
-```
-
-**Or via command line:**
-
-```bash
-cd examples/offline_inference/text_to_image
-python text_to_image.py \
-    --model Your-org/your-model \
-    --prompt "a cup of coffee on the table" \
-    --negative-prompt "ugly, unclear" \
-    --cfg-scale 4.0 \
-    --num-inference-steps 50 \
-    --output "tp_enabled.png" \
-    --tensor-parallel-size 2
-```
-
-**Verify:**
-
-1. Check the `e2e_time_ms` in the log for speedup
-2. Compare generated image quality with TP disabled
-3. Verify memory usage is reduced proportionally
-4. Record comparison results in your PR
-
----
-
-## Troubleshooting
-
-### Issue: TP not activating
-
-**Symptoms:** Model runs on single GPU, no memory savings or speedup.
-
-**Causes & Solutions:**
-
-- **Still using `nn.Linear`:**
-
-**Problem:** Linear layers not replaced with parallel equivalents.
-
-**Solution:** Replace with parallel layers:
-```python
-# ❌ BAD
-self.proj = nn.Linear(dim, dim)
-
-# ✅ GOOD
-self.proj = RowParallelLinear(dim, dim, input_is_parallel=True)
-```
-
-### Issue: Dimension mismatch errors
-
-**Symptoms:** `RuntimeError: shape mismatch` during forward pass.
-
-**Causes & Solutions:**
-
-- **Missing `input_is_parallel=True`:**
-
-**Problem:** RowParallelLinear expects sharded input but receives full tensor.
-
-**Solution:** Set `input_is_parallel=True` when input comes from ColumnParallelLinear:
-```python
-# ✅ GOOD: Correct pairing
-self.w1 = ColumnParallelLinear(dim, hidden_dim, return_bias=False,)
-self.w2 = RowParallelLinear(
-    hidden_dim,
-    dim,
-    input_is_parallel=True,  # Input sharded from w1
-    return_bias=False,
-)
-```
-
-- **Incorrect split dimensions:**
-
-**Problem:** QKV split sizes don't match sharded dimensions.
-
-**Solution:** Use `self.to_qkv.num_heads` (local heads per GPU):
-```python
-# ❌ BAD: Uses total heads
-q_size = self.total_num_heads * self.head_dim
-
-# ✅ GOOD: Uses local heads
-q_size = self.to_qkv.num_heads * self.head_dim
-```
-
----
-
-## Reference Implementations
-
-Complete examples in the codebase:
-
-| Model | Path | Pattern | Notes |
-|-------|------|---------|-------|
-| **Z-Image** | `vllm_omni/diffusion/models/z_image/z_image_transformer.py` | Standard TP | Full implementation with validation |
-| **FLUX** | `vllm_omni/diffusion/models/flux/flux_transformer.py` | Dual-stream | Image + text streams |
-| **Qwen-Image** | `vllm_omni/diffusion/models/qwen_image/qwen_image_transformer.py` | Standard TP | With RoPE |
-| **TP Tests** | `tests/e2e/offline_inference/test_zimage_parallelism.py` | E2E testing | TP correctness and performance |
-| **Constraint Tests** | `tests/diffusion/models/z_image/test_zimage_tp_constraints.py` | Unit testing | Validation logic |
-
----
-
-## Summary
-
-Adding Tensor Parallel support to a transformer:
-
-1. ✅ **Identify linear layers** - Which layers should be sharded?
-2. ✅ **Replace with parallel layers** - Use QKVParallelLinear, ColumnParallelLinear, RowParallelLinear
-3. ✅ **Validate TP constraints** - Ensure dimensions divisible by TP size
-4. ✅ **Test** - Verify with `tensor_parallel_size=N`, check memory, speed, and quality

docs/design/index.md

@@ -8,11 +8,12 @@ This section contains design documents and architecture specifications for vLLM-
 
 ## Feature Design Documents
 
-- [Disaggregated Inference](feature/disaggregated_inference.md)
+- [Multi-Request Streaming](feature/multi_request_streaming_design.md)
+- [Omni Connector Design](feature/omni_connector_design.md)
 - [Ray-based Execution](feature/ray_based_execution.md)
 
 ## Module Design Documents
 
-- [AR Module](module/ar_module.md)
-- [DIT Module](module/dit_module.md)
-- [Entrypoint Module](module/entrypoint_module.md)
+- [AR Module Design](module/ar_module_design.md)
+- [DIT Module Design](module/dit_module_design.md)
+- [Entrypoint Module Design](module/entrypoint_module_design.md)