Revert "[Config Refactor] Derive deploy override fields from stage config (#3162)"

This reverts commit 01ebc0cdf5.

Signed-off-by: GitHub <noreply@github.com>

Co-authored-by: Gaohan123 <20148503+Gaohan123@users.noreply.github.com>

.buildkite/bootstrap-amd-omni.sh

@@ -1,238 +0,0 @@
-#!/bin/bash
-# vllm-omni customized version
-# Based on: https://github.com/vllm-project/ci-infra/blob/main/buildkite/bootstrap-amd.sh
-# Last synced: 2025-12-15
-# Modifications: Use local template file instead of downloading from ci-infra
-
-set -euo pipefail
-
-if [[ -z "${RUN_ALL:-}" ]]; then
-    RUN_ALL=0
-fi
-
-if [[ -z "${NIGHTLY:-}" ]]; then
-    NIGHTLY=0
-fi
-
-if [[ -z "${VLLM_CI_BRANCH:-}" ]]; then
-    VLLM_CI_BRANCH="main"
-fi
-
-if [[ -z "${AMD_MIRROR_HW:-}" ]]; then
-    AMD_MIRROR_HW="amdproduction"
-fi
-
-if [[ -z "${DOCS_ONLY_DISABLE:-}" ]]; then
-    DOCS_ONLY_DISABLE=0
-fi
-
-fail_fast() {
-    DISABLE_LABEL="ci-no-fail-fast"
-    # If BUILDKITE_PULL_REQUEST != "false", then we check the PR labels using curl and jq
-    if [ "$BUILDKITE_PULL_REQUEST" != "false" ]; then
-        PR_LABELS=$(curl -s "https://api.github.com/repos/vllm-project/vllm-omni/pulls/$BUILDKITE_PULL_REQUEST" | jq -r '.labels[].name')
-        if [[ $PR_LABELS == *"$DISABLE_LABEL"* ]]; then
-            echo false
-        else
-            echo true
-        fi
-    else
-        echo false  # not a PR or BUILDKITE_PULL_REQUEST not set
-    fi
-}
-
-check_run_all_label() {
-    RUN_ALL_LABEL="ready-run-all-tests"
-    # If BUILDKITE_PULL_REQUEST != "false", then we check the PR labels using curl and jq
-    if [ "$BUILDKITE_PULL_REQUEST" != "false" ]; then
-        PR_LABELS=$(curl -s "https://api.github.com/repos/vllm-project/vllm-omni/pulls/$BUILDKITE_PULL_REQUEST" | jq -r '.labels[].name')
-        if [[ $PR_LABELS == *"$RUN_ALL_LABEL"* ]]; then
-            echo true
-        else
-            echo false
-        fi
-    else
-        echo false  # not a PR or BUILDKITE_PULL_REQUEST not set
-    fi
-}
-
-if [[ -z "${COV_ENABLED:-}" ]]; then
-    COV_ENABLED=0
-fi
-
-upload_pipeline() {
-    echo "Uploading pipeline..."
-    # Install minijinja
-    ls .buildkite || buildkite-agent annotate --style error 'Please merge upstream main branch for buildkite CI'
-    curl -sSfL https://github.com/mitsuhiko/minijinja/releases/download/2.3.1/minijinja-cli-installer.sh | sh
-    source /var/lib/buildkite-agent/.cargo/env
-
-    if [[ $BUILDKITE_PIPELINE_SLUG == "fastcheck" ]]; then
-        AMD_MIRROR_HW="amdtentative"
-    fi
-
-    # Use local template file for vllm-omni
-    cp .buildkite/test-template-amd-omni.j2 .buildkite/test-template.j2
-
-
-    # (WIP) Use pipeline generator instead of jinja template
-    if [ -e ".buildkite/pipeline_generator/pipeline_generator.py" ]; then
-        python -m pip install click pydantic
-        python .buildkite/pipeline_generator/pipeline_generator.py --run_all=$RUN_ALL --list_file_diff="$LIST_FILE_DIFF" --nightly="$NIGHTLY" --mirror_hw="$AMD_MIRROR_HW"
-        buildkite-agent pipeline upload .buildkite/pipeline.yaml
-        exit 0
-    fi
-    echo "List file diff: $LIST_FILE_DIFF"
-    echo "Run all: $RUN_ALL"
-    echo "Nightly: $NIGHTLY"
-    echo "AMD Mirror HW: $AMD_MIRROR_HW"
-
-    FAIL_FAST=$(fail_fast)
-
-    cd .buildkite
-    (
-        set -x
-        # Output pipeline.yaml with all blank lines removed
-        minijinja-cli test-template.j2 test-amd.yaml \
-            -D branch="$BUILDKITE_BRANCH" \
-            -D list_file_diff="$LIST_FILE_DIFF" \
-            -D run_all="$RUN_ALL" \
-            -D nightly="$NIGHTLY" \
-            -D mirror_hw="$AMD_MIRROR_HW" \
-            -D fail_fast="$FAIL_FAST" \
-            -D vllm_use_precompiled="$VLLM_USE_PRECOMPILED" \
-            -D vllm_merge_base_commit="$(git merge-base origin/main HEAD)" \
-            -D cov_enabled="$COV_ENABLED" \
-            -D vllm_ci_branch="$VLLM_CI_BRANCH" \
-            | sed '/^[[:space:]]*$/d' \
-            > pipeline.yaml
-    )
-    cat pipeline.yaml
-    buildkite-agent artifact upload pipeline.yaml
-    buildkite-agent pipeline upload pipeline.yaml
-    exit 0
-}
-
-get_diff() {
-    $(git add .)
-    echo $(git diff --name-only --diff-filter=ACMDR $(git merge-base origin/main HEAD))
-}
-
-get_diff_main() {
-    $(git add .)
-    echo $(git diff --name-only --diff-filter=ACMDR HEAD~1)
-}
-
-file_diff=$(get_diff)
-if [[ $BUILDKITE_BRANCH == "main" ]]; then
-    file_diff=$(get_diff_main)
-fi
-
-# ----------------------------------------------------------------------
-# Early exit start: skip pipeline if conditions are met
-# ----------------------------------------------------------------------
-
-# skip pipeline if all changed files are under docs/
-if [[ "${DOCS_ONLY_DISABLE}" != "1" ]]; then
-  if [[ -n "${file_diff:-}" ]]; then
-    docs_only=1
-    # Robust iteration over newline-separated file_diff
-    while IFS= read -r f; do
-      [[ -z "$f" ]] && continue
-      # **Policy:** only skip if *every* path starts with docs/
-      if [[ "$f" != docs/* ]]; then
-        docs_only=0
-        break
-      fi
-    done < <(printf '%s\n' "$file_diff" | tr ' ' '\n' | tr -d '\r')
-
-    if [[ "$docs_only" -eq 1 ]]; then
-      buildkite-agent annotate ":memo: CI skipped — docs/** only changes detected
-
-\`\`\`
-${file_diff}
-\`\`\`" --style "info" || true
-      echo "[docs-only] All changes are under docs/. Exiting before pipeline upload."
-      exit 0
-    fi
-  fi
-fi
-
-# ----------------------------------------------------------------------
-# Early exit end
-# ----------------------------------------------------------------------
-
-patterns=(
-    "docker/Dockerfile"
-    "CMakeLists.txt"
-    "requirements/common.txt"
-    "requirements/cuda.txt"
-    "requirements/build.txt"
-    "requirements/test.txt"
-    "setup.py"
-    "csrc/"
-    "cmake/"
-)
-
-ignore_patterns=(
-    "docker/Dockerfile."
-    "csrc/cpu"
-    "csrc/rocm"
-    "cmake/hipify.py"
-    "cmake/cpu_extension.cmake"
-)
-
-for file in $file_diff; do
-    # First check if file matches any pattern
-    matches_pattern=0
-    for pattern in "${patterns[@]}"; do
-        if [[ $file == $pattern* ]] || [[ $file == $pattern ]]; then
-            matches_pattern=1
-            break
-        fi
-    done
-
-    # If file matches pattern, check it's not in ignore patterns
-    if [[ $matches_pattern -eq 1 ]]; then
-        matches_ignore=0
-        for ignore in "${ignore_patterns[@]}"; do
-            if [[ $file == $ignore* ]] || [[ $file == $ignore ]]; then
-                matches_ignore=1
-                break
-            fi
-        done
-
-        if [[ $matches_ignore -eq 0 ]]; then
-            RUN_ALL=1
-            echo "Found changes: $file. Run all tests"
-            break
-        fi
-    fi
-done
-
-# Check for ready-run-all-tests label
-LABEL_RUN_ALL=$(check_run_all_label)
-if [[ $LABEL_RUN_ALL == true ]]; then
-    RUN_ALL=1
-    NIGHTLY=1
-    echo "Found 'ready-run-all-tests' label. Running all tests including optional tests."
-fi
-
-# Decide whether to use precompiled wheels
-# Relies on existing patterns array as a basis.
-if [[ -n "${VLLM_USE_PRECOMPILED:-}" ]]; then
-    echo "VLLM_USE_PRECOMPILED is already set to: $VLLM_USE_PRECOMPILED"
-elif [[ $RUN_ALL -eq 1 ]]; then
-    export VLLM_USE_PRECOMPILED=0
-    echo "Detected critical changes, building wheels from source"
-else
-    export VLLM_USE_PRECOMPILED=1
-    echo "No critical changes, using precompiled wheels"
-fi
-
-
-LIST_FILE_DIFF=$(get_diff | tr ' ' '|')
-if [[ $BUILDKITE_BRANCH == "main" ]]; then
-    LIST_FILE_DIFF=$(get_diff_main | tr ' ' '|')
-fi
-upload_pipeline

.buildkite/pipeline.yml

@@ -1,40 +0,0 @@
-steps:
-  - label: ":docker: Build image"
-    key: image-build
-    commands:
-      - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7"
-      - "docker build --file docker/Dockerfile.ci -t vllm-omni-ci ."
-      - "docker tag vllm-omni-ci public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT"
-      - "docker push public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT"
-    agents:
-      queue: "cpu_queue_premerge"
-
-  # L2 Test
-  - label: "Upload Ready Pipeline"
-    depends_on: image-build
-    key: upload-ready-pipeline
-    if: build.branch != "main"
-    commands:
-      - buildkite-agent pipeline upload .buildkite/test-ready.yml
-    agents:
-      queue: "cpu_queue_premerge"
-
-  # L3 Test
-  - label: "Upload Merge Pipeline"
-    depends_on: image-build
-    key: upload-merge-pipeline
-    if: build.branch == "main"
-    commands:
-      - buildkite-agent pipeline upload .buildkite/test-merge.yml
-    agents:
-      queue: "cpu_queue_premerge"
-
-  # L4 Test
-  - label: "Upload Nightly Pipeline"
-    depends_on: image-build
-    key: upload-nightly-pipeline
-    if: build.env("NIGHTLY") == "1"
-    commands:
-      - buildkite-agent pipeline upload .buildkite/test-nightly.yml
-    agents:
-      queue: "cpu_queue_premerge"

.buildkite/scripts/docker_login_ecr_public.sh

@@ -1,73 +0,0 @@
-#!/bin/bash
-# Helper function to safely login to ECR Public with per-job config isolation
-# Uses DOCKER_CONFIG environment variable to prevent race conditions
-#
-# This script prevents the "device or resource busy" error by giving each
-# Buildkite job its own isolated Docker config directory.
-#
-# Usage:
-#   source docker_login_ecr_public.sh && safe_docker_login_ecr_public
-
-set -euo pipefail
-
-# Configuration
-ECR_REGISTRY="public.ecr.aws"
-
-setup_isolated_docker_config() {
-    # Use BUILDKITE_JOB_ID for job-specific isolation
-    # Fallback to PID if running outside Buildkite
-    local job_id="${BUILDKITE_JOB_ID:-$$}"
-
-    # Set Docker config to job-specific directory
-    export DOCKER_CONFIG="/tmp/docker-config-${job_id}"
-
-    # Create directory if it doesn't exist
-    mkdir -p "$DOCKER_CONFIG"
-
-    echo "[docker-config] Using isolated Docker config: $DOCKER_CONFIG"
-}
-
-check_docker_auth() {
-    # Check if already authenticated to the given registry
-    # Returns 0 if authenticated, 1 if not
-    local registry="$1"
-
-    # Check if credentials exist in the isolated config
-    if [[ -f "$DOCKER_CONFIG/config.json" ]]; then
-        # Check if registry is present in config
-        if grep -q "$registry" "$DOCKER_CONFIG/config.json" 2>/dev/null; then
-            return 0
-        fi
-    fi
-
-    return 1
-}
-
-safe_docker_login_ecr_public() {
-    # Setup isolated config first
-    setup_isolated_docker_config
-
-    local registry="$ECR_REGISTRY"
-
-    # Check if already authenticated (within this job)
-    if check_docker_auth "$registry"; then
-        echo "[docker-login] Already authenticated to $registry in this job"
-        return 0
-    fi
-
-    # Perform login to isolated config directory
-    echo "[docker-login] Logging in to $ECR_REGISTRY (isolated config)..."
-    if aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin "$ECR_REGISTRY"; then
-        echo "[docker-login] Login successful (config: $DOCKER_CONFIG)"
-        return 0
-    else
-        local exit_code=$?
-        echo "[docker-login] ERROR: Login failed with exit code $exit_code" >&2
-        return $exit_code
-    fi
-}
-
-# Execute if run as script (not sourced)
-if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then
-    safe_docker_login_ecr_public
-fi

.buildkite/scripts/hardware_ci/run-amd-test.sh

@@ -1,166 +0,0 @@
-#!/bin/bash
-# vllm-omni customized version
-# Based on: vllm/.buildkite/scripts/hardware_ci/run-amd-test.sh
-# Last synced: 2025-12-15
-# Modifications: docker image name for vllm-omni
-
-# This script runs test inside the corresponding ROCm docker container.
-set -o pipefail
-
-# Export Python path
-export PYTHONPATH=".."
-
-# Print ROCm version
-echo "--- Confirming Clean Initial State"
-while true; do
-        sleep 3
-        if grep -q clean /opt/amdgpu/etc/gpu_state; then
-                echo "GPUs state is \"clean\""
-                break
-        fi
-done
-
-echo "--- ROCm info"
-rocminfo
-
-# cleanup older docker images
-cleanup_docker() {
-  # Get Docker's root directory
-  docker_root=$(docker info -f '{{.DockerRootDir}}')
-  if [ -z "$docker_root" ]; then
-    echo "Failed to determine Docker root directory."
-    exit 1
-  fi
-  echo "Docker root directory: $docker_root"
-  # Check disk usage of the filesystem where Docker's root directory is located
-  disk_usage=$(df "$docker_root" | tail -1 | awk '{print $5}' | sed 's/%//')
-  # Define the threshold
-  threshold=70
-  if [ "$disk_usage" -gt "$threshold" ]; then
-    echo "Disk usage is above $threshold%. Cleaning up Docker images and volumes..."
-    # Remove dangling images (those that are not tagged and not used by any container)
-    docker image prune -f
-    # Remove unused volumes / force the system prune for old images as well.
-    docker volume prune -f && docker system prune --force --filter "until=72h" --all
-    echo "Docker images and volumes cleanup completed."
-  else
-    echo "Disk usage is below $threshold%. No cleanup needed."
-  fi
-}
-
-# Call the cleanup docker function
-cleanup_docker
-
-echo "--- Resetting GPUs"
-
-echo "reset" > /opt/amdgpu/etc/gpu_state
-
-while true; do
-        sleep 3
-        if grep -q clean /opt/amdgpu/etc/gpu_state; then
-                echo "GPUs state is \"clean\""
-                break
-        fi
-done
-
-echo "--- Pulling container"
-image_name="public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:${BUILDKITE_COMMIT}-rocm-omni"
-container_name="rocm_${BUILDKITE_COMMIT}_$(tr -dc A-Za-z0-9 < /dev/urandom | head -c 10; echo)"
-
-# Install AWS CLI to authenticate to ECR Public Gallery to get higher rate limit for pulling images
-sudo apt-get update && sudo apt-get install -y awscli
-# Use safe docker login helper to prevent race conditions
-source "$(dirname "${BASH_SOURCE[0]}")/../docker_login_ecr_public.sh"
-safe_docker_login_ecr_public
-# Pull the container from ECR Public Gallery
-
-docker pull "${image_name}"
-
-remove_docker_container() {
-   docker rm -f "${container_name}" || docker image rm -f "${image_name}" || true
-}
-trap remove_docker_container EXIT
-
-echo "--- Running container"
-
-HF_CACHE="$(realpath ~)/huggingface"
-mkdir -p "${HF_CACHE}"
-HF_MOUNT="/root/.cache/huggingface"
-
-commands=$@
-echo "Commands:$commands"
-
-PARALLEL_JOB_COUNT=8
-MYPYTHONPATH=".."
-
-# Test that we're launching on the machine that has
-# proper access to GPUs
-render_gid=$(getent group render | cut -d: -f3)
-if [[ -z "$render_gid" ]]; then
-  echo "Error: 'render' group not found. This is required for GPU access." >&2
-  exit 1
-fi
-
-# check if the command contains shard flag, we will run all shards in parallel because the host have 8 GPUs.
-if [[ $commands == *"--shard-id="* ]]; then
-  # assign job count as the number of shards used
-  commands=$(echo "$commands" | sed -E "s/--num-shards[[:blank:]]*=[[:blank:]]*[0-9]*/--num-shards=${PARALLEL_JOB_COUNT} /g" | sed 's/ \\ / /g')
-  for GPU in $(seq 0 $(($PARALLEL_JOB_COUNT-1))); do
-    # assign shard-id for each shard
-    commands_gpu=$(echo "$commands" | sed -E "s/--shard-id[[:blank:]]*=[[:blank:]]*[0-9]*/--shard-id=${GPU} /g" | sed 's/ \\ / /g')
-    echo "Shard ${GPU} commands:$commands_gpu"
-    echo "Render devices: $BUILDKITE_AGENT_META_DATA_RENDER_DEVICES"
-    docker run \
-        --device /dev/kfd $BUILDKITE_AGENT_META_DATA_RENDER_DEVICES \
-        --network=host \
-        --shm-size=16gb \
-        --group-add "$render_gid" \
-        --rm \
-        -e MIOPEN_DEBUG_CONV_DIRECT=0 \
-        -e MIOPEN_DEBUG_CONV_GEMM=0 \
-        -e VLLM_ROCM_USE_AITER=1 \
-        -e HIP_VISIBLE_DEVICES="${GPU}" \
-        -e HF_TOKEN \
-        -e AWS_ACCESS_KEY_ID \
-        -e AWS_SECRET_ACCESS_KEY \
-        -v "${HF_CACHE}:${HF_MOUNT}" \
-        -e "HF_HOME=${HF_MOUNT}" \
-        -e "PYTHONPATH=${MYPYTHONPATH}" \
-        --name "${container_name}_${GPU}" \
-        "${image_name}" \
-        /bin/bash -c "${commands_gpu}" \
-        |& while read -r line; do echo ">>Shard $GPU: $line"; done &
-    PIDS+=($!)
-  done
-  #wait for all processes to finish and collect exit codes
-  for pid in "${PIDS[@]}"; do
-    wait "${pid}"
-    STATUS+=($?)
-  done
-  for st in "${STATUS[@]}"; do
-    if [[ ${st} -ne 0 ]]; then
-      echo "One of the processes failed with $st"
-      exit "${st}"
-    fi
-  done
-else
-  echo "Render devices: $BUILDKITE_AGENT_META_DATA_RENDER_DEVICES"
-  docker run \
-          --device /dev/kfd $BUILDKITE_AGENT_META_DATA_RENDER_DEVICES \
-          --network=host \
-          --shm-size=16gb \
-          --group-add "$render_gid" \
-          --rm \
-          -e MIOPEN_DEBUG_CONV_DIRECT=0 \
-          -e MIOPEN_DEBUG_CONV_GEMM=0 \
-          -e VLLM_ROCM_USE_AITER=1 \
-          -e HF_TOKEN \
-          -e AWS_ACCESS_KEY_ID \
-          -e AWS_SECRET_ACCESS_KEY \
-          -v "${HF_CACHE}:${HF_MOUNT}" \
-          -e "HF_HOME=${HF_MOUNT}" \
-          -e "PYTHONPATH=${MYPYTHONPATH}" \
-          --name "${container_name}" \
-          "${image_name}" \
-          /bin/bash -c "${commands}"
-fi

.buildkite/scripts/hardware_ci/run_npu_test.sh

@@ -1,145 +0,0 @@
-#!/bin/bash
-
-# This script build the Ascend NPU docker image and run the offline inference inside the container.
-# It serves a sanity check for compilation and basic model usage.
-set -ex
-
-# Base ubuntu image with basic ascend development libraries and python installed
-VLLM_OMNI_REPO="https://github.com/vllm-project/vllm-omni.git"
-BASE_IMAGE_NAME="quay.nju.edu.cn/ascend/vllm-ascend:v0.11.0rc2"
-image_name="npu/vllm-omni-ci:${BUILDKITE_COMMIT}_${EPOCHSECONDS}"
-# image_name="npu/vllm-ci:${BUILDKITE_COMMIT}_${EPOCHSECONDS}"
-container_name="npu_${BUILDKITE_COMMIT}_$(tr -dc A-Za-z0-9 < /dev/urandom | head -c 10; echo)"
-
-# BUILDKITE_AGENT_NAME format is {hostname}-{agent_idx}-{npu_card_num}cards
-agent_idx=$(echo "${BUILDKITE_AGENT_NAME}" | awk -F'-' '{print $(NF-1)}')
-echo "agent_idx: ${agent_idx}"
-builder_name="cachebuilder${agent_idx}"
-builder_cache_dir="/mnt/docker-cache${agent_idx}"
-mkdir -p ${builder_cache_dir}
-
-# Try building the docker image
-cat <<EOF | DOCKER_BUILDKIT=1 docker build \
-    --add-host pypi-cache:${PYPI_CACHE_HOST} \
-    --builder ${builder_name} --cache-from type=local,src=${builder_cache_dir} \
-                           --cache-to type=local,dest=${builder_cache_dir},mode=max \
-    --build-arg BUILDKITE_PULL_REQUEST="${BUILDKITE_PULL_REQUEST}" \
-    --build-arg BUILDKITE_PULL_REQUEST_REPO="${BUILDKITE_PULL_REQUEST_REPO}" \
-    --progress=plain --load -t ${image_name} -f - .
-FROM ${BASE_IMAGE_NAME}
-
-# Define environments
-ENV DEBIAN_FRONTEND=noninteractive
-
-RUN pip config set global.index-url http://pypi-cache:${PYPI_CACHE_PORT}/pypi/simple && \
-    pip config set global.trusted-host pypi-cache && \
-    apt-get update -y && \
-    apt-get install -y python3-pip git vim wget net-tools gcc g++ cmake libnuma-dev && \
-    rm -rf /var/cache/apt/* && \
-    rm -rf /var/lib/apt/lists/*
-
-# Install for pytest to make the docker build cache layer always valid
-RUN --mount=type=cache,target=/root/.cache/pip \
-    pip install pytest>=6.0  pytest-cov modelscope
-
-COPY . .
-
-# Install vllm-omni
-WORKDIR /workspace
-ARG VLLM_OMNI_REPO=https://github.com/vllm-project/vllm-omni.git
-ARG VLLM_OMNI_TAG=main
-ARG BUILDKITE_PULL_REQUEST
-ARG BUILDKITE_PULL_REQUEST_REPO
-RUN git config --global url."https://gh-proxy.test.osinfra.cn/https://github.com/".insteadOf "https://github.com/" && \
-    if [ "\$BUILDKITE_PULL_REQUEST" != "false" ] && [ -n "\$BUILDKITE_PULL_REQUEST" ]; then \
-        echo "Cloning and checking out PR #\$BUILDKITE_PULL_REQUEST..." && \
-        git clone \$VLLM_OMNI_REPO /workspace/vllm-omni && \
-        cd /workspace/vllm-omni && \
-        git fetch origin pull/\$BUILDKITE_PULL_REQUEST/head:pr-\$BUILDKITE_PULL_REQUEST && \
-        git checkout pr-\$BUILDKITE_PULL_REQUEST; \
-    else \
-        echo "Not a PR build, using main branch" && \
-        git clone --depth 1 \$VLLM_OMNI_REPO /workspace/vllm-omni; \
-    fi
-
-RUN --mount=type=cache,target=/root/.cache/pip \
-    export PIP_EXTRA_INDEX_URL=https://mirrors.huaweicloud.com/ascend/repos/pypi && \
-    source /usr/local/Ascend/ascend-toolkit/set_env.sh && \
-    source /usr/local/Ascend/nnal/atb/set_env.sh && \
-    export LD_LIBRARY_PATH=\$LD_LIBRARY_PATH:/usr/local/Ascend/ascend-toolkit/latest/`uname -i`-linux/devlib && \
-    python3 -m pip install -v -e /workspace/vllm-omni/
-
-ENV VLLM_WORKER_MULTIPROC_METHOD=spawn
-
-WORKDIR /workspace/vllm-omni
-CMD ["/bin/bash"]
-
-EOF
-
-# Setup cleanup
-remove_docker_container() {
-  docker rm -f "${container_name}" || true;
-  docker image rm -f "${image_name}" || true;
-  docker system prune -f || true;
-}
-trap remove_docker_container EXIT
-
-# Generate corresponding --device args based on BUILDKITE_AGENT_NAME
-# Ascend NPU BUILDKITE_AGENT_NAME format is {hostname}-{agent_idx}-{npu_card_num}cards, and agent_idx starts from 1.
-#   e.g. atlas-a2-001-1-2cards means this is the 1-th agent on atlas-a2-001 host, and it has 2 NPU cards.
-#   returns --device /dev/davinci0 --device /dev/davinci1
-parse_and_gen_devices() {
-    local input="$1"
-    local index cards_num
-    if [[ "$input" =~ ([0-9]+)-([0-9]+)cards$ ]]; then
-        index="${BASH_REMATCH[1]}"
-        cards_num="${BASH_REMATCH[2]}"
-    else
-        echo "parse error" >&2
-        return 1
-    fi
-
-    local devices=""
-    local i=0
-    while (( i < cards_num )); do
-        local dev_idx=$(((index - 1)*cards_num + i ))
-        devices="$devices --device /dev/davinci${dev_idx}"
-        ((i++))
-    done
-
-    # trim leading space
-    devices="${devices#"${devices%%[![:space:]]*}"}"
-    # Output devices: assigned to the caller variable
-    printf '%s' "$devices"
-}
-
-devices=$(parse_and_gen_devices "${BUILDKITE_AGENT_NAME}") || exit 1
-
-# Run the image and execute the Out-Of-Tree (OOT) platform interface test case on Ascend NPU hardware.
-# This test checks whether the OOT platform interface is functioning properly in conjunction with
-# the hardware plugin vllm-ascend.
-hf_model_cache_dir=/mnt/hf_cache${agent_idx}
-ms_model_cache_dir=/mnt/modelscope${agent_idx}
-mkdir -p ${hf_model_cache_dir}
-mkdir -p ${ms_model_cache_dir}
-docker run \
-    --init \
-    ${devices} \
-    --device /dev/davinci_manager \
-    --device /dev/devmm_svm \
-    --device /dev/hisi_hdc \
-    -v /usr/local/dcmi:/usr/local/dcmi \
-    -v /usr/local/bin/npu-smi:/usr/local/bin/npu-smi \
-    -v /usr/local/Ascend/driver/lib64/:/usr/local/Ascend/driver/lib64/ \
-    -v /usr/local/Ascend/driver/version.info:/usr/local/Ascend/driver/version.info \
-    -v /etc/ascend_install.info:/etc/ascend_install.info \
-    -v ${hf_model_cache_dir}:/root/.cache/huggingface \
-    -v ${ms_model_cache_dir}:/root/.cache/modelscope \
-    --network host \
-    --entrypoint="" \
-    --name "${container_name}" \
-    "${image_name}" \
-    bash -c '
-    set -e
-    VLLM_USE_MODELSCOPE=True pytest -s -v tests/e2e/offline_inference/test_qwen2_5_omni.py
-'

.buildkite/test-amd.yaml

@@ -1,116 +0,0 @@
-steps:
-
-- label: "Diffusion Model Test"
-  timeout_in_minutes: 20
-  agent_pool: mi325_2
-  depends_on: amd-build
-  mirror_hardwares: [amdproduction]
-  grade: Blocking
-  commands:
-    - export GPU_ARCHS=gfx942
-    - pytest -s -v tests/e2e/offline_inference/test_t2i_model.py
-
-- label: "Diffusion Images API LoRA E2E"
-  timeout_in_minutes: 20
-  agent_pool: mi325_1
-  depends_on: amd-build
-  mirror_hardwares: [amdproduction]
-  grade: Blocking
-  commands:
-    - export GPU_ARCHS=gfx942
-    - export VLLM_LOGGING_LEVEL=DEBUG
-    - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-    - pytest -s -v tests/e2e/online_serving/test_images_generations_lora.py
-
-- label: "Diffusion Model CPU offloading Test"
-  timeout_in_minutes: 20
-  agent_pool: mi325_1
-  depends_on: amd-build
-  mirror_hardwares: [amdproduction]
-  grade: Blocking
-  commands:
-    - export GPU_ARCHS=gfx942
-    - export VLLM_LOGGING_LEVEL=DEBUG
-    - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-    - pytest -s -v tests/e2e/offline_inference/test_diffusion_cpu_offload.py
-
-- label: "Diffusion Cache Backend Test"
-  timeout_in_minutes: 15
-  agent_pool: mi325_1
-  depends_on: amd-build
-  mirror_hardwares: [amdproduction]
-  grade: Blocking
-  commands:
-    - export GPU_ARCHS=gfx942
-    - export VLLM_LOGGING_LEVEL=DEBUG
-    - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-    - pytest -s -v -m 'core_model and cache and diffusion and not distributed_rocm and MI325'
-
-- label: "Diffusion Sequence Parallelism Test"
-  timeout_in_minutes: 20
-  agent_pool: mi325_2
-  depends_on: amd-build
-  mirror_hardwares: [amdproduction]
-  grade: Blocking
-  commands:
-    - export GPU_ARCHS=gfx942
-    - export VLLM_LOGGING_LEVEL=DEBUG
-    - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-    - pytest -s -v tests/e2e/offline_inference/test_sequence_parallel.py -m core_model
-
-- label: "Diffusion Tensor Parallelism Test"
-  timeout_in_minutes: 20
-  agent_pool: mi325_2
-  depends_on: amd-build
-  commands:
-    - export GPU_ARCHS=gfx942
-    - export VLLM_LOGGING_LEVEL=DEBUG
-    - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-    - pytest -s -v tests/e2e/offline_inference/test_zimage_parallelism.py
-
-- label: "Diffusion GPU Worker Test"
-  timeout_in_minutes: 20
-  agent_pool: mi325_2
-  depends_on: amd-build
-  mirror_hardwares: [amdproduction]
-  grade: Blocking
-  commands:
-    - pytest -s -v tests/diffusion/test_diffusion_worker.py
-
-- label: "Omni Model Test Qwen2-5-Omni"
-  timeout_in_minutes: 15
-  agent_pool: mi325_2
-  depends_on: amd-build
-  mirror_hardwares: [amdproduction]
-  grade: Blocking
-  commands:
-    - export GPU_ARCHS=gfx942
-    - export VLLM_LOGGING_LEVEL=DEBUG
-    - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-    - pytest -s -v tests/e2e/offline_inference/test_qwen2_5_omni.py
-
-- label: "Omni Model Test Qwen3-Omni"
-  timeout_in_minutes: 15
-  agent_pool: mi325_2
-  depends_on: amd-build
-  mirror_hardwares: [amdproduction]
-  grade: Blocking
-  commands:
-    - export VLLM_LOGGING_LEVEL=DEBUG
-    - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-    - export VLLM_TEST_CLEAN_GPU_MEMORY="1"
-    - pytest -s -v tests/e2e/offline_inference/test_qwen3_omni.py
-    - pytest -s -v tests/e2e/online_serving/test_qwen3_omni.py
-    - pytest -s -v tests/engine/test_async_omni_engine_abort.py
-
-- label: "Diffusion Image Edit Test"
-  timeout_in_minutes: 15
-  agent_pool: mi325_1
-  depends_on: amd-build
-  mirror_hardwares: [amdproduction]
-  grade: Blocking
-  commands:
-    - export GPU_ARCHS=gfx942
-    - export VLLM_LOGGING_LEVEL=DEBUG
-    - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-    - pytest -s -v tests/e2e/online_serving/test_image_gen_edit.py

.buildkite/test-merge.yml

@@ -1,318 +0,0 @@
-steps:
-  - label: "Simple Unit Test"
-    depends_on: upload-merge-pipeline
-    commands:
-      - "pytest -v -s -m 'core_model and cpu' --cov=vllm_omni --cov-branch --cov-report=term-missing --cov-report=html --cov-report=xml"
-    agents:
-      queue: "gpu_1_queue"
-    plugins:
-      - docker#v5.2.0:
-          image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-          always-pull: true
-          propagate-environment: true
-          environment:
-            - "HF_HOME=/fsx/hf_cache"
-          volumes:
-            - "/fsx/hf_cache:/fsx/hf_cache"
-
-  - label: "Diffusion Model Test"
-    timeout_in_minutes: 20
-    depends_on: upload-merge-pipeline
-    commands:
-      - pytest -s -v tests/e2e/offline_inference/test_t2i_model.py -m "advanced_model and diffusion" --run-level "advanced_model"
-    agents:
-      queue: "gpu_1_queue" # g6.4xlarge instance on AWS, has 1 L4 GPU
-    plugins:
-      - docker#v5.2.0:
-          image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-          always-pull: true
-          propagate-environment: true
-          environment:
-            - "HF_HOME=/fsx/hf_cache"
-          volumes:
-            - "/fsx/hf_cache:/fsx/hf_cache"
-
-  - label: "Diffusion Images API LoRA E2E"
-    timeout_in_minutes: 20
-    depends_on: upload-merge-pipeline
-    commands:
-      - pytest -s -v tests/e2e/online_serving/test_images_generations_lora.py
-    agents:
-      queue: "gpu_1_queue" # g6.4xlarge instance on AWS, has 1 L4 GPU
-    plugins:
-      - docker#v5.2.0:
-          image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-          always-pull: true
-          propagate-environment: true
-          environment:
-            - "HF_HOME=/fsx/hf_cache"
-          volumes:
-            - "/fsx/hf_cache:/fsx/hf_cache"
-
-  - label: "Diffusion Model CPU offloading Test"
-    timeout_in_minutes: 20
-    depends_on: upload-merge-pipeline
-    commands:
-      - pytest -s -v tests/e2e/offline_inference/test_diffusion_cpu_offload.py
-      - pytest -s -v tests/e2e/offline_inference/test_diffusion_layerwise_offload.py
-    agents:
-      queue: "gpu_1_queue" # g6.4xlarge instance on AWS, has 1 L4 GPU
-    plugins:
-      - docker#v5.2.0:
-          image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-          always-pull: true
-          propagate-environment: true
-          environment:
-            - "HF_HOME=/fsx/hf_cache"
-          volumes:
-            - "/fsx/hf_cache:/fsx/hf_cache"
-
-  - label: "Audio Generation Model Test"
-    timeout_in_minutes: 20
-    depends_on: upload-merge-pipeline
-    commands:
-      - pytest -s -v tests/e2e/offline_inference/test_stable_audio_model.py
-    agents:
-      queue: "gpu_1_queue" # g6.4xlarge instance on AWS, has 1 L4 GPU
-    plugins:
-      - docker#v5.2.0:
-          image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-          always-pull: true
-          propagate-environment: true
-          environment:
-            - "HF_HOME=/fsx/hf_cache"
-          volumes:
-            - "/fsx/hf_cache:/fsx/hf_cache"
-
-  - label: "Diffusion Cache Backend Test"
-    timeout_in_minutes: 15
-    depends_on: upload-merge-pipeline
-    commands:
-      - pytest -s -v -m 'core_model and cache and diffusion and not distributed_cuda and L4'
-    agents:
-      queue: "gpu_1_queue" # g6.4xlarge instance on AWS, has 1 L4 GPU
-    plugins:
-      - docker#v5.2.0:
-          image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-          always-pull: true
-          propagate-environment: true
-          environment:
-            - "HF_HOME=/fsx/hf_cache"
-          volumes:
-            - "/fsx/hf_cache:/fsx/hf_cache"
-
-  - label: "Diffusion Sequence Parallelism Test"
-    timeout_in_minutes: 20
-    depends_on: upload-merge-pipeline
-    commands:
-      - pytest -s -v tests/e2e/offline_inference/test_sequence_parallel.py
-    agents:
-      queue: "gpu_4_queue" # g6.12xlarge instance on AWS, has 4 L4 GPU
-    plugins:
-      - docker#v5.2.0:
-          image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-          always-pull: true
-          propagate-environment: true
-          shm-size: "8gb"
-          environment:
-            - "HF_HOME=/fsx/hf_cache"
-          volumes:
-            - "/fsx/hf_cache:/fsx/hf_cache"
-
-  - label: "Diffusion Tensor Parallelism Test"
-    timeout_in_minutes: 20
-    depends_on: upload-merge-pipeline
-    commands:
-      - pytest -s -v tests/e2e/offline_inference/test_zimage_parallelism.py
-    agents:
-      queue: "gpu_4_queue" # g6.12xlarge instance on AWS, has 4 L4 GPU
-    plugins:
-      - docker#v5.2.0:
-          image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-          always-pull: true
-          propagate-environment: true
-          shm-size: "8gb"
-          environment:
-            - "HF_HOME=/fsx/hf_cache"
-          volumes:
-            - "/fsx/hf_cache:/fsx/hf_cache"
-
-  - label: "Diffusion GPU Worker Test"
-    timeout_in_minutes: 20
-    depends_on: upload-merge-pipeline
-    commands:
-      - pytest -s -v tests/diffusion/test_diffusion_worker.py
-    agents:
-      queue: "gpu_4_queue" # g6.12xlarge instance on AWS, has 4 L4 GPU
-    plugins:
-      - docker#v5.2.0:
-          image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-          always-pull: true
-          propagate-environment: true
-          shm-size: "8gb"
-          environment:
-            - "HF_HOME=/fsx/hf_cache"
-          volumes:
-            - "/fsx/hf_cache:/fsx/hf_cache"
-
-  - label: "Benchmark & Engine Test with H100"
-    timeout_in_minutes: 15
-    depends_on: upload-merge-pipeline
-    commands:
-      - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-      - pytest -s -v tests/benchmarks/test_serve_cli.py
-      - pytest -s -v tests/engine/test_async_omni_engine_abort.py
-    agents:
-      queue: "mithril-h100-pool"
-    plugins:
-      - kubernetes:
-          podSpec:
-            containers:
-              - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-                resources:
-                  limits:
-                    nvidia.com/gpu: 2
-                volumeMounts:
-                  - name: devshm
-                    mountPath: /dev/shm
-                  - name: hf-cache
-                    mountPath: /root/.cache/huggingface
-                env:
-                  - name: HF_HOME
-                    value: /root/.cache/huggingface
-            nodeSelector:
-              node.kubernetes.io/instance-type: gpu-h100-sxm
-            volumes:
-              - name: devshm
-                emptyDir:
-                  medium: Memory
-              - name: hf-cache
-                hostPath:
-                  path: /mnt/hf-cache
-                  type: DirectoryOrCreate
-
-  - label: "Omni Model Test"
-    timeout_in_minutes: 15
-    depends_on: upload-merge-pipeline
-    commands:
-      - export VLLM_LOGGING_LEVEL=DEBUG
-      - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-      - pytest -s -v tests/e2e/offline_inference/test_qwen2_5_omni.py
-    agents:
-      queue: "gpu_4_queue" # g6.12xlarge instance on AWS, has 4 L4 GPU
-    plugins:
-      - docker#v5.2.0:
-          image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-          always-pull: true
-          propagate-environment: true
-          environment:
-            - "HF_HOME=/fsx/hf_cache"
-          volumes:
-            - "/fsx/hf_cache:/fsx/hf_cache"
-
-  - label: "Omni Model Test with H100"
-    timeout_in_minutes: 30
-    depends_on: upload-merge-pipeline
-    commands:
-      - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-      - export VLLM_TEST_CLEAN_GPU_MEMORY="1"
-      - pytest -s -v tests/e2e/offline_inference/test_qwen3_omni.py
-      - pytest -s -v tests/e2e/online_serving/test_qwen3_omni.py -m "advanced_model" --run-level "advanced_model"
-    agents:
-      queue: "mithril-h100-pool"
-    plugins:
-      - kubernetes:
-          podSpec:
-            containers:
-              - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-                resources:
-                  limits:
-                    nvidia.com/gpu: 2
-                volumeMounts:
-                  - name: devshm
-                    mountPath: /dev/shm
-                  - name: hf-cache
-                    mountPath: /root/.cache/huggingface
-                env:
-                  - name: HF_HOME
-                    value: /root/.cache/huggingface
-            nodeSelector:
-              node.kubernetes.io/instance-type: gpu-h100-sxm
-            volumes:
-              - name: devshm
-                emptyDir:
-                  medium: Memory
-              - name: hf-cache
-                hostPath:
-                  path: /mnt/hf-cache
-                  type: DirectoryOrCreate
-
-  - label: "Diffusion Image Edit Test with H100 (1 GPU)"
-    timeout_in_minutes: 20
-    depends_on: upload-merge-pipeline
-    commands:
-      - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-      - pytest -s -v tests/e2e/online_serving/test_image_gen_edit.py
-    agents:
-      queue: "mithril-h100-pool"
-    plugins:
-      - kubernetes:
-          podSpec:
-            containers:
-              - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-                resources:
-                  limits:
-                    nvidia.com/gpu: 1
-                volumeMounts:
-                  - name: devshm
-                    mountPath: /dev/shm
-                  - name: hf-cache
-                    mountPath: /root/.cache/huggingface
-                env:
-                  - name: HF_HOME
-                    value: /root/.cache/huggingface
-            nodeSelector:
-              node.kubernetes.io/instance-type: gpu-h100-sxm
-            volumes:
-              - name: devshm
-                emptyDir:
-                  medium: Memory
-              - name: hf-cache
-                hostPath:
-                  path: /mnt/hf-cache
-                  type: DirectoryOrCreate
-
-  # - label: "Bagel Text2Img Model Test with H100"
-  #   timeout_in_minutes: 30
-  #   depends_on: upload-merge-pipeline
-  #   commands:
-  #     - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-  #     - pytest -s -v tests/e2e/offline_inference/test_bagel_text2img.py
-  #   agents:
-  #     queue: "mithril-h100-pool"
-  #   plugins:
-  #     - kubernetes:
-  #         podSpec:
-  #           containers:
-  #             - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-  #               resources:
-  #                 limits:
-  #                   nvidia.com/gpu: 1
-  #               volumeMounts:
-  #                 - name: devshm
-  #                   mountPath: /dev/shm
-  #                 - name: hf-cache
-  #                   mountPath: /root/.cache/huggingface
-  #               env:
-  #                 - name: HF_HOME
-  #                   value: /root/.cache/huggingface
-  #           nodeSelector:
-  #             node.kubernetes.io/instance-type: gpu-h100-sxm
-  #           volumes:
-  #             - name: devshm
-  #               emptyDir:
-  #                 medium: Memory
-  #             - name: hf-cache
-  #               hostPath:
-  #                 path: /mnt/hf-cache
-  #                 type: DirectoryOrCreate

.buildkite/test-nightly.yml

@@ -1,93 +0,0 @@
-steps:
-  - label: "Omni Model Test with H100"
-    timeout_in_minutes: 90
-    depends_on: upload-nightly-pipeline
-    if: build.env("NIGHTLY") == "1"
-    commands:
-      - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-      - pytest -s -v tests/e2e/online_serving/test_qwen3_omni_expansion.py -m "advanced_model" --run-level "advanced_model"
-      - pytest -s -v tests/examples/online_serving/test_qwen3_omni.py -m "advanced_model" --run-level "advanced_model"
-    agents:
-      queue: "mithril-h100-pool"
-    plugins:
-      - kubernetes:
-          podSpec:
-            containers:
-              - image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-                resources:
-                  limits:
-                    nvidia.com/gpu: 2
-                volumeMounts:
-                  - name: devshm
-                    mountPath: /dev/shm
-                  - name: hf-cache
-                    mountPath: /root/.cache/huggingface
-                env:
-                  - name: HF_HOME
-                    value: /root/.cache/huggingface
-            nodeSelector:
-              node.kubernetes.io/instance-type: gpu-h100-sxm
-            volumes:
-              - name: devshm
-                emptyDir:
-                  medium: Memory
-              - name: hf-cache
-                hostPath:
-                  path: /mnt/hf-cache
-                  type: DirectoryOrCreate
-
-  - label: "Omni Model Test"
-    timeout_in_minutes: 60
-    depends_on: upload-nightly-pipeline
-    if: build.env("NIGHTLY") == "1"
-    commands:
-      - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-      - pytest -s -v tests/examples/online_serving/test_qwen2_5_omni.py -m "advanced_model" --run-level "advanced_model"
-    agents:
-      queue: "gpu_4_queue" # g6.12xlarge instance on AWS, has 4 L4 GPU
-    plugins:
-      - docker#v5.2.0:
-          image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-          always-pull: true
-          shm-size: "8gb"
-          propagate-environment: true
-          environment:
-            - "HF_HOME=/fsx/hf_cache"
-          volumes:
-            - "/fsx/hf_cache:/fsx/hf_cache"
-
-  - label: "Omni Model Perf Test"
-    timeout_in_minutes: 120
-    depends_on: upload-nightly-pipeline
-    if: build.env("NIGHTLY") == "1"
-    commands:
-      - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-      - pytest -s -v tests/perf/scripts/run_benchmark.py
-    agents:
-      queue: "mithril-h100-pool"
-    plugins:
-      - kubernetes:
-          podSpec:
-            containers:
-              - image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-                resources:
-                  limits:
-                    nvidia.com/gpu: 2
-                volumeMounts:
-                  - name: devshm
-                    mountPath: /dev/shm
-                  - name: hf-cache
-                    mountPath: /root/.cache/huggingface
-                env:
-                  - name: HF_HOME
-                    value: /root/.cache/huggingface
-            nodeSelector:
-              node.kubernetes.io/instance-type: gpu-h100-sxm
-            volumes:
-              - name: devshm
-                emptyDir:
-                  medium: Memory
-              - name: hf-cache
-                hostPath:
-                  path: /mnt/hf-cache
-                  type: DirectoryOrCreate

.buildkite/test-ready.yml

@@ -1,303 +0,0 @@
-steps:
-  - label: "Simple Unit Test"
-    depends_on: upload-ready-pipeline
-    commands:
-      - "pytest -v -s -m 'core_model and cpu' --cov=vllm_omni --cov-branch --cov-report=term-missing --cov-report=html --cov-report=xml"
-    agents:
-      queue: "gpu_1_queue"
-    plugins:
-      - docker#v5.2.0:
-          image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-          always-pull: true
-          propagate-environment: true
-          environment:
-            - "HF_HOME=/fsx/hf_cache"
-          volumes:
-            - "/fsx/hf_cache:/fsx/hf_cache"
-
-  - label: "Diffusion Model Test"
-    timeout_in_minutes: 20
-    depends_on: upload-ready-pipeline
-    commands:
-      - pytest -s -v tests/e2e/offline_inference/test_t2i_model.py -m "core_model and diffusion" --run-level "core_model"
-    agents:
-      queue: "gpu_1_queue" # g6.4xlarge instance on AWS, has 1 L4 GPU
-    plugins:
-      - docker#v5.2.0:
-          image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-          always-pull: true
-          propagate-environment: true
-          environment:
-            - "HF_HOME=/fsx/hf_cache"
-          volumes:
-            - "/fsx/hf_cache:/fsx/hf_cache"
-
-  - label: "Diffusion Model CPU offloading Test"
-    timeout_in_minutes: 20
-    depends_on: upload-ready-pipeline
-    commands:
-      - pytest -s -v tests/e2e/offline_inference/test_diffusion_cpu_offload.py
-      - pytest -s -v tests/e2e/offline_inference/test_diffusion_layerwise_offload.py
-    agents:
-      queue: "gpu_1_queue" # g6.4xlarge instance on AWS, has 1 L4 GPU
-    plugins:
-      - docker#v5.2.0:
-          image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-          always-pull: true
-          propagate-environment: true
-          environment:
-            - "HF_HOME=/fsx/hf_cache"
-          volumes:
-            - "/fsx/hf_cache:/fsx/hf_cache"
-
-  - label: "Audio Generation Model Test"
-    timeout_in_minutes: 20
-    depends_on: upload-ready-pipeline
-    commands:
-      - pytest -s -v tests/e2e/offline_inference/test_stable_audio_model.py
-    agents:
-      queue: "gpu_1_queue" # g6.4xlarge instance on AWS, has 1 L4 GPU
-    plugins:
-      - docker#v5.2.0:
-          image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-          always-pull: true
-          propagate-environment: true
-          environment:
-            - "HF_HOME=/fsx/hf_cache"
-          volumes:
-            - "/fsx/hf_cache:/fsx/hf_cache"
-
-  - label: "Diffusion Cache Backend Test"
-    timeout_in_minutes: 15
-    depends_on: upload-ready-pipeline
-    commands:
-      - pytest -s -v -m 'core_model and cache and diffusion and not distributed_cuda and L4'
-    agents:
-      queue: "gpu_1_queue" # g6.4xlarge instance on AWS, has 1 L4 GPU
-    plugins:
-      - docker#v5.2.0:
-          image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-          always-pull: true
-          propagate-environment: true
-          environment:
-            - "HF_HOME=/fsx/hf_cache"
-          volumes:
-            - "/fsx/hf_cache:/fsx/hf_cache"
-
-  - label: "Diffusion Sequence Parallelism Test"
-    timeout_in_minutes: 20
-    depends_on: upload-ready-pipeline
-    commands:
-      - pytest -s -v tests/e2e/offline_inference/test_sequence_parallel.py -m core_model
-    agents:
-      queue: "gpu_4_queue" # g6.12xlarge instance on AWS, has 4 L4 GPU
-    plugins:
-      - docker#v5.2.0:
-          image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-          always-pull: true
-          propagate-environment: true
-          shm-size: "8gb"
-          environment:
-            - "HF_HOME=/fsx/hf_cache"
-          volumes:
-            - "/fsx/hf_cache:/fsx/hf_cache"
-
-  - label: "Diffusion GPU Worker Test"
-    timeout_in_minutes: 20
-    depends_on: upload-ready-pipeline
-    commands:
-      - pytest -s -v tests/diffusion/test_diffusion_worker.py
-    agents:
-      queue: "gpu_4_queue" # g6.12xlarge instance on AWS, has 4 L4 GPU
-    plugins:
-      - docker#v5.2.0:
-          image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-          always-pull: true
-          propagate-environment: true
-          shm-size: "8gb"
-          environment:
-            - "HF_HOME=/fsx/hf_cache"
-          volumes:
-            - "/fsx/hf_cache:/fsx/hf_cache"
-
-  # - label: "Benchmark & Engine Test with H100"
-  #   timeout_in_minutes: 15
-  #   depends_on: upload-ready-pipeline
-  #   commands:
-  #     - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-  #     - pytest -s -v tests/benchmarks/test_serve_cli.py
-  #     - pytest -s -v tests/engine/test_async_omni_engine_abort.py
-  #   agents:
-  #     queue: "mithril-h100-pool"
-  #   plugins:
-  #     - kubernetes:
-  #         podSpec:
-  #           containers:
-  #             - image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-  #               resources:
-  #                 limits:
-  #                   nvidia.com/gpu: 2
-  #               volumeMounts:
-  #                 - name: devshm
-  #                   mountPath: /dev/shm
-  #                 - name: hf-cache
-  #                   mountPath: /root/.cache/huggingface
-  #               env:
-  #                 - name: HF_HOME
-  #                   value: /root/.cache/huggingface
-  #           nodeSelector:
-  #             node.kubernetes.io/instance-type: gpu-h100-sxm
-  #           volumes:
-  #             - name: devshm
-  #               emptyDir:
-  #                 medium: Memory
-  #             - name: hf-cache
-  #               hostPath:
-  #                 path: /mnt/hf-cache
-  #                 type: DirectoryOrCreate
-
-  - label: "Omni Model Test"
-    timeout_in_minutes: 17
-    depends_on: upload-ready-pipeline
-    commands:
-      - export VLLM_LOGGING_LEVEL=DEBUG
-      - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-      - pytest -s -v tests/e2e/offline_inference/test_qwen2_5_omni.py
-    agents:
-      queue: "gpu_4_queue" # g6.12xlarge instance on AWS, has 4 L4 GPU
-    plugins:
-      - docker#v5.2.0:
-          image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-          always-pull: true
-          propagate-environment: true
-          environment:
-            - "HF_HOME=/fsx/hf_cache"
-          volumes:
-            - "/fsx/hf_cache:/fsx/hf_cache"
-
-  # - label: "Omni Model Test with H100"
-  #   timeout_in_minutes: 20
-  #   depends_on: upload-ready-pipeline
-  #   commands:
-  #     - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-  #     - export VLLM_TEST_CLEAN_GPU_MEMORY="1"
-  #     # - pytest -s -v tests/e2e/offline_inference/test_qwen3_omni.py
-  #     - pytest -s -v tests/e2e/online_serving/test_qwen3_omni.py -m "core_model" --run-level "core_model"
-  #   agents:
-  #     queue: "mithril-h100-pool"
-  #   plugins:
-  #     - kubernetes:
-  #         podSpec:
-  #           containers:
-  #             - image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-  #               resources:
-  #                 limits:
-  #                   nvidia.com/gpu: 2
-  #               volumeMounts:
-  #                 - name: devshm
-  #                   mountPath: /dev/shm
-  #                 - name: hf-cache
-  #                   mountPath: /root/.cache/huggingface
-  #               env:
-  #                 - name: HF_HOME
-  #                   value: /root/.cache/huggingface
-  #           nodeSelector:
-  #             node.kubernetes.io/instance-type: gpu-h100-sxm
-  #           volumes:
-  #             - name: devshm
-  #               emptyDir:
-  #                 medium: Memory
-  #             - name: hf-cache
-  #               hostPath:
-  #                 path: /mnt/hf-cache
-  #                 type: DirectoryOrCreate
-
-  - label: "Qwen3-TTS E2E Test"
-    timeout_in_minutes: 20
-    depends_on: upload-ready-pipeline
-    commands:
-      - export VLLM_LOGGING_LEVEL=DEBUG
-      - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-      - pytest -s -v tests/e2e/online_serving/test_qwen3_tts.py
-    agents:
-      queue: "gpu_4_queue"
-    plugins:
-      - docker#v5.2.0:
-          image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-          always-pull: true
-          propagate-environment: true
-          shm-size: "8gb"
-          environment:
-            - "HF_HOME=/fsx/hf_cache"
-          volumes:
-            - "/fsx/hf_cache:/fsx/hf_cache"
-
-  # - label: "Diffusion Image Edit Test with H100 (1 GPU)"
-  #   timeout_in_minutes: 20
-  #   depends_on: upload-ready-pipeline
-  #   commands:
-  #     - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-  #     - pytest -s -v tests/e2e/online_serving/test_image_gen_edit.py
-  #   agents:
-  #     queue: "mithril-h100-pool"
-  #   plugins:
-  #     - kubernetes:
-  #         podSpec:
-  #           containers:
-  #             - image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-  #               resources:
-  #                 limits:
-  #                   nvidia.com/gpu: 1
-  #               volumeMounts:
-  #                 - name: devshm
-  #                   mountPath: /dev/shm
-  #                 - name: hf-cache
-  #                   mountPath: /root/.cache/huggingface
-  #               env:
-  #                 - name: HF_HOME
-  #                   value: /root/.cache/huggingface
-  #           nodeSelector:
-  #             node.kubernetes.io/instance-type: gpu-h100-sxm
-  #           volumes:
-  #             - name: devshm
-  #               emptyDir:
-  #                 medium: Memory
-  #             - name: hf-cache
-  #               hostPath:
-  #                 path: /mnt/hf-cache
-  #                 type: DirectoryOrCreate
-
-  # - label: "Bagel Text2Img Model Test with H100"
-  #   timeout_in_minutes: 30
-  #   depends_on: upload-ready-pipeline
-  #   commands:
-  #     - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-  #     - pytest -s -v tests/e2e/offline_inference/test_bagel_text2img.py
-  #   agents:
-  #     queue: "mithril-h100-pool"
-  #   plugins:
-  #     - kubernetes:
-  #         podSpec:
-  #           containers:
-  #             - image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-  #               resources:
-  #                 limits:
-  #                   nvidia.com/gpu: 1
-  #               volumeMounts:
-  #                 - name: devshm
-  #                   mountPath: /dev/shm
-  #                 - name: hf-cache
-  #                   mountPath: /root/.cache/huggingface
-  #               env:
-  #                 - name: HF_HOME
-  #                   value: /root/.cache/huggingface
-  #           nodeSelector:
-  #             node.kubernetes.io/instance-type: gpu-h100-sxm
-  #           volumes:
-  #             - name: devshm
-  #               emptyDir:
-  #                 medium: Memory
-  #             - name: hf-cache
-  #               hostPath:
-  #                 path: /mnt/hf-cache
-  #                 type: DirectoryOrCreate

.buildkite/test-template-amd-omni.j2

@@ -1,53 +0,0 @@
-{# vllm-omni customized version
-   Based on: https://github.com/vllm-project/ci-infra/blob/main/buildkite/test-template-amd.j2
-   Last synced: 2025-12-15
-   Modifications: Removed unused CUDA/NVIDIA logic, keeping only AMD tests
-#}
-{% set docker_image_amd = "public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT-rocm-omni" %}
-{% set default_working_dir = "/app/vllm-omni" %}
-
-  - group: "AMD Tests"
-    depends_on: ~
-    steps:
-      - label: "AMD: :docker: build image"
-        depends_on: ~
-        soft_fail: false
-        commands:
-          - "source .buildkite/scripts/docker_login_ecr_public.sh && safe_docker_login_ecr_public"
-          - "docker build -f docker/Dockerfile.rocm -t {{ docker_image_amd }} --target test --build-arg ARG_PYTORCH_ROCM_ARCH=gfx942 --progress plain ."
-          - "docker push {{ docker_image_amd }}"
-        key: "amd-build"
-        env:
-          DOCKER_BUILDKIT: "1"
-        retry:
-          automatic:
-            - exit_status: -1  # Agent was lost
-              limit: 1
-            - exit_status: -10  # Agent was lost
-              limit: 1
-            - exit_status: 1  # Machine occasionally fail
-              limit: 1
-        agents:
-          queue: cpu_queue_premerge
-
-    {% for step in steps %}
-    {% if step.mirror_hardwares and mirror_hw in step.mirror_hardwares %}
-      - label: "{{ step.agent_pool }}: {{ step.label }}"
-        depends_on: amd-build
-        agents:
-          {% if step.agent_pool %}
-          queue: amd_{{ step.agent_pool }}
-          {% else %}
-          queue: amd_mi325_1
-          {% endif %}
-        command: bash .buildkite/scripts/hardware_ci/run-amd-test.sh "(command rocm-smi || true) && cd {{ (step.working_dir or default_working_dir) | safe  }} ; {{ step.command  or (step.commands | join(" && ")) | safe }}"
-        env:
-          DOCKER_BUILDKIT: "1"
-        priority: 100
-        {% if step.grade and step.grade == "Blocking" %}
-        soft_fail: false
-        {% else %}
-        soft_fail: true
-        {% endif%}
-    {% endif %}
-    {% endfor %}

.gitattributes

@@ -1,44 +0,0 @@
-# Set default behavior to automatically normalize line endings
-* text=auto
-
-# Explicitly declare text files you want to always be normalized and converted
-# to native line endings on checkout
-*.py text eol=lf
-*.yaml text eol=lf
-*.yml text eol=lf
-*.md text eol=lf
-*.txt text eol=lf
-*.json text eol=lf
-*.toml text eol=lf
-*.sh text eol=lf
-*.bat text eol=crlf
-*.cmd text eol=crlf
-
-# Declare files that will always have CRLF line endings on checkout
-*.bat text eol=crlf
-*.cmd text eol=crlf
-
-# Denote all files that are truly binary and should not be modified
-*.png binary
-*.jpg binary
-*.jpeg binary
-*.gif binary
-*.ico binary
-*.mov binary
-*.mp4 binary
-*.mp3 binary
-*.flv binary
-*.fla binary
-*.swf binary
-*.gz binary
-*.zip binary
-*.7z binary
-*.ttf binary
-*.eot binary
-*.woff binary
-*.woff2 binary
-*.pyc binary
-*.pdf binary
-*.ai binary
-*.psd binary
-*.xcf binary

.github/CODEOWNERS

@@ -1,17 +0,0 @@
-# CODEOWNERS is used to automatically assign PR reviewers
-# Format: path pattern @username or @team
-# More info: https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/customizing-your-repository/about-code-owners
-
-# Default owners (fallback reviewers for all files)
-# later, we can add more reviewers here
-* @hsliuustc0106
-
-# Assign reviewers by directory
-/.github/ @owner1
-/tests/ @owner1
-/docs/ @owner1
-
-# Example: reviewers for specific directories
-# /examples/offline_inference/ @offline-inference-team
-# /vllm_omni/core/ @core-team
-# /vllm_omni/model_executor/ @model-team

.github/ISSUE_TEMPLATE/100-documentation.yml

@@ -1,29 +0,0 @@
-name: 📚 Documentation
-description: Report an issue related to documentation
-title: "[Doc]: "
-labels: ["documentation"]
-
-body:
-- type: textarea
-  attributes:
-    label: 📚 The doc issue
-    description: >
-      A clear and concise description of what content in the documentation is an issue.
-  validations:
-    required: true
-- type: textarea
-  attributes:
-    label: Suggest a potential alternative/fix
-    description: >
-      Tell us how we could improve the documentation in this regard.
-- type: markdown
-  attributes:
-    value: >
-      Thanks for contributing 🎉!
-- type: checkboxes
-  id: askllm
-  attributes:
-    label: Before submitting a new issue...
-    options:
-      - label: Make sure you already searched for relevant issues, and asked the chatbot living at the bottom right corner of the [documentation page](https://vllm-omni.readthedocs.io), which can answer lots of frequently asked questions.
-        required: true

.github/ISSUE_TEMPLATE/200-installation.yml

@@ -1,47 +0,0 @@
-name: 🛠️ Installation
-description: Report an issue here when you hit errors during installation.
-title: "[Installation]: "
-labels: ["installation"]
-
-body:
-- type: markdown
-  attributes:
-    value: >
-      #### Before submitting an issue, please make sure the issue hasn't been already addressed by searching through [the existing and past issues](https://github.com/vllm-project/vllm-omni/issues?q=is%3Aissue+sort%3Acreated-desc+).
-- type: textarea
-  attributes:
-    label: Your current environment
-    description: |
-      Please run the following and paste the output below.
-      ```sh
-      wget https://raw.githubusercontent.com/vllm-project/vllm-omni/main/vllm_omni/collect_env.py
-      # For security purposes, please feel free to check the contents of collect_env.py before running it.
-      python collect_env.py
-      ```
-      It is suggested to download and execute the latest script, as vllm-omni might frequently update the diagnosis information needed for accurately and quickly responding to issues.
-    value: |
-      ```text
-      The output of `python collect_env.py`
-      ```
-  validations:
-    required: true
-- type: textarea
-  attributes:
-    label: How you are installing vllm-omni
-    description: |
-      Paste the full command you are trying to execute.
-    value: |
-      ```sh
-      pip install -vvv vllm-omni
-      ```
-- type: markdown
-  attributes:
-    value: >
-      Thanks for contributing 🎉!
-- type: checkboxes
-  id: askllm
-  attributes:
-    label: Before submitting a new issue...
-    options:
-      - label: Make sure you already searched for relevant issues, and asked the chatbot living at the bottom right corner of the [documentation page](https://vllm-omni.readthedocs.io), which can answer lots of frequently asked questions.
-        required: true

.github/ISSUE_TEMPLATE/400-bug-report.yml

@@ -1,136 +0,0 @@
-name: 🐛 Bug report
-description: Raise an issue here if you find a bug.
-title: "[Bug]: "
-labels: ["bug"]
-
-body:
-- type: markdown
-  attributes:
-    value: >
-      #### Before submitting an issue, please make sure the issue hasn't been already addressed by searching through [the existing and past issues](https://github.com/vllm-project/vllm-omni/issues?q=is%3Aissue+sort%3Acreated-desc+).
-- type: markdown
-  attributes:
-    value: |
-      ⚠️ **SECURITY WARNING:** Please review any text you paste to ensure it does not contain sensitive information such as:
-      - API tokens or keys (e.g., Hugging Face tokens, OpenAI API keys)
-      - Passwords or authentication credentials
-      - Private URLs or endpoints
-      - Personal or confidential data
-
-      Consider redacting or replacing sensitive values with placeholders like `<YOUR_TOKEN_HERE>` when sharing configuration or code examples.
-- type: textarea
-  attributes:
-    label: Your current environment
-    description: |
-      Please run the following and paste the output below.
-      ```sh
-      wget https://raw.githubusercontent.com/vllm-project/vllm-omni/main/collect_env.py
-      # For security purposes, please feel free to check the contents of collect_env.py before running it.
-      python collect_env.py
-      ```
-      It is suggested to download and execute the latest script, as vllm-omni might frequently update the diagnosis information needed for accurately and quickly responding to issues.
-    value: |
-      <details>
-      <summary>The output of <code>python collect_env.py</code></summary>
-
-      ```text
-      Your output of `python collect_env.py` here
-      ```
-
-      </details>
-  validations:
-    required: true
-
-- type: textarea
-  attributes:
-    label: Your code version
-    description: |
-      Please provide a description of which commit id or version the code is.
-
-    value: |
-      <details>
-      <summary>The commit id or version of vllm</summary>
-
-      ```text
-
-      ```
-      </details>
-      <details>
-      <summary>The commit id or version of vllm-omni</summary>
-
-      ```text
-
-      ```
-      </details>
-  validations:
-    required: true
-
-- type: textarea
-  attributes:
-    label: 🐛 Describe the bug
-    description: |
-      Please provide a clear and concise description of what the bug is.
-
-      If relevant, add a minimal example so that we can reproduce the error by running the code. It is very important for the snippet to be as succinct (minimal) as possible, so please take time to trim down any irrelevant code to help us debug efficiently. We are going to copy-paste your code and we expect to get the same result as you did: avoid any external data, and include the relevant imports, etc. For example:
-
-      ```python
-      from vllm_omni import OmniLLM, create_ar_stage_config, create_dit_stage_config
-
-      # Create stage configurations
-      ar_config = create_ar_stage_config(
-          stage_id=0,
-          model_path="Qwen/Qwen3-0.6B",
-          input_modalities=["text"],
-          output_modalities=["text"]
-      )
-
-      dit_config = create_dit_stage_config(
-          stage_id=1,
-          model_path="stabilityai/stable-diffusion-2-1",
-          input_modalities=["text"],
-          output_modalities=["image"]
-      )
-
-      # Initialize OmniLLM
-      omni = OmniLLM([ar_config, dit_config])
-
-      # Generate
-      outputs = omni.generate(prompt="A scenic watercolor painting of a lighthouse at sunset")
-      ```
-
-      If the code is too long (hopefully, it isn't), feel free to put it in a public gist and link it in the issue: https://gist.github.com.
-
-      Please also paste or describe the results you observe instead of the expected results. If you observe an error, please paste the error message including the **full** traceback of the exception. It may be relevant to wrap error messages in ```` ```triple quotes blocks``` ````.
-
-      Please set the environment variable `export VLLM_OMNI_LOGGING_LEVEL=DEBUG` to turn on more logging to help debugging potential issues.
-
-      If you experienced crashes or hangs, it would be helpful to run vllm-omni with `export VLLM_OMNI_TRACE_FUNCTION=1` . All the function calls in vllm-omni will be recorded. Inspect these log files, and tell which function crashes or hangs.
-    placeholder: |
-      A clear and concise description of what the bug is.
-
-      ```python
-      # Sample code to reproduce the problem
-      ```
-
-      ```
-      The error message you got, with the full traceback and the error logs with [dump_input.py:##] if present.
-      ```
-  validations:
-    required: true
-- type: markdown
-  attributes:
-    value: |
-      ⚠️ Please separate bugs of `transformers` implementation or usage from bugs of `vllm-omni`. If you think anything is wrong with the model's output:
-
-      - Try the counterpart of `transformers` first. If the error appears, please go to [their issues](https://github.com/huggingface/transformers/issues?q=is%3Aissue+is%3Aopen+sort%3Aupdated-desc).
-
-      - If the error only appears in vllm-omni, please provide the detailed script of how you run `transformers` and `vllm-omni`, also highlight the difference and what you expect.
-
-      Thanks for reporting 🙏!
-- type: checkboxes
-  id: askllm
-  attributes:
-    label: Before submitting a new issue...
-    options:
-      - label: Make sure you already searched for relevant issues, and asked the chatbot living at the bottom right corner of the [documentation page](https://vllm-omni.readthedocs.io), which can answer lots of frequently asked questions.
-        required: true

.github/ISSUE_TEMPLATE/500-feature-request.yml

@@ -1,38 +0,0 @@
-name: 🚀 Feature request
-description: Submit a proposal/request for a new vllm-omni feature
-title: "[Feature]: "
-labels: ["feature request"]
-
-body:
-- type: markdown
-  attributes:
-    value: >
-      #### Before submitting an issue, please make sure the issue hasn't been already addressed by searching through [the existing and past issues](https://github.com/vllm-project/vllm-omni/issues?q=is%3Aissue+sort%3Acreated-desc+).
-- type: textarea
-  attributes:
-    label: 🚀 The feature, motivation and pitch
-    description: >
-      A clear and concise description of the feature proposal. Please outline the motivation for the proposal. Is your feature request related to a specific problem? e.g., *"I'm working on X and would like Y to be possible"*. If this is related to another GitHub issue, please link here too.
-  validations:
-    required: true
-- type: textarea
-  attributes:
-    label: Alternatives
-    description: >
-      A description of any alternative solutions or features you've considered, if any.
-- type: textarea
-  attributes:
-    label: Additional context
-    description: >
-      Add any other context or screenshots about the feature request.
-- type: markdown
-  attributes:
-    value: >
-      Thanks for contributing 🎉!
-- type: checkboxes
-  id: askllm
-  attributes:
-    label: Before submitting a new issue...
-    options:
-      - label: Make sure you already searched for relevant issues, and asked the chatbot living at the bottom right corner of the [documentation page](https://vllm-omni.readthedocs.io), which can answer lots of frequently asked questions.
-        required: true

.github/ISSUE_TEMPLATE/600-new-model.yml

@@ -1,45 +0,0 @@
-name: 🤗 Support request for a new model
-description: Submit a proposal/request for a new model support
-title: "[New Model]: "
-labels: ["new model"]
-
-body:
-- type: markdown
-  attributes:
-    value: >
-      #### Before submitting an issue, please make sure the issue hasn't been already addressed by searching through [the existing and past issues](https://github.com/vllm-project/vllm-omni/issues?q=is%3Aissue+sort%3Acreated-desc+).
-
-      #### We also highly recommend you read the contributing guidelines first to understand how to add a new model.
-- type: textarea
-  attributes:
-    label: The model to consider.
-    description: >
-      A huggingface url, pointing to the model, e.g. https://huggingface.co/openai-community/gpt2 .
-  validations:
-    required: true
-- type: textarea
-  attributes:
-    label: The closest model vllm-omni already supports.
-    description: >
-      Here is the list of models already supported by vllm-omni. Which model is the most similar to the model you want to add support for?
-- type: textarea
-  attributes:
-    label: What's your difficulty of supporting the model you want?
-    description: >
-      For example, any new operators or new architecture? Is it a multimodal model? Does it require special preprocessing?
-- type: textarea
-  attributes:
-    label: Use case and motivation
-    description: >
-      What is the primary use case for this model? Why is it important to support in vllm-omni?
-- type: markdown
-  attributes:
-    value: >
-      Thanks for contributing 🎉!
-- type: checkboxes
-  id: askllm
-  attributes:
-    label: Before submitting a new issue...
-    options:
-      - label: Make sure you already searched for relevant issues, and asked the chatbot living at the bottom right corner of the [documentation page](https://vllm-omni.readthedocs.io), which can answer lots of frequently asked questions.
-        required: true

.github/ISSUE_TEMPLATE/700-performance-discussion.yml

@@ -1,59 +0,0 @@
-name: ⚡ Discussion on the performance of vllm-omni
-description: Submit a proposal/discussion about the performance of vllm-omni
-title: "[Performance]: "
-labels: ["performance"]
-
-body:
-- type: markdown
-  attributes:
-    value: >
-      #### Before submitting an issue, please make sure the issue hasn't been already addressed by searching through [the existing and past issues](https://github.com/vllm-project/vllm-omni/issues?q=is%3Aissue+sort%3Acreated-desc+).
-- type: textarea
-  attributes:
-    label: Proposal to improve performance
-    description: >
-      How do you plan to improve vllm-omni's performance?
-  validations:
-    required: false
-- type: textarea
-  attributes:
-    label: Report of performance regression
-    description: >
-      Please provide detailed description of performance comparison to confirm the regression. You may want to run the benchmark script at https://github.com/vllm-project/vllm-omni/tree/main/tests/benchmarks .
-  validations:
-    required: false
-- type: textarea
-  attributes:
-    label: Misc discussion on performance
-    description: >
-      Anything about the performance.
-  validations:
-    required: false
-- type: textarea
-  attributes:
-    label: Your current environment (if you think it is necessary)
-    description: |
-      Please run the following and paste the output below.
-      ```sh
-      wget https://raw.githubusercontent.com/vllm-project/vllm-omni/main/vllm_omni/collect_env.py
-      # For security purposes, please feel free to check the contents of collect_env.py before running it.
-      python collect_env.py
-      ```
-      It is suggested to download and execute the latest script, as vllm-omni might frequently update the diagnosis information needed for accurately and quickly responding to issues.
-    value: |
-      ```text
-      The output of `python collect_env.py`
-      ```
-  validations:
-    required: false
-- type: markdown
-  attributes:
-    value: >
-      Thanks for contributing 🎉!
-- type: checkboxes
-  id: askllm
-  attributes:
-    label: Before submitting a new issue...
-    options:
-      - label: Make sure you already searched for relevant issues, and asked the chatbot living at the bottom right corner of the [documentation page](https://vllm-omni.readthedocs.io), which can answer lots of frequently asked questions.
-        required: true

.github/ISSUE_TEMPLATE/750-RFC.yml

@@ -1,54 +0,0 @@
-name: 💬 Request for comments (RFC).
-description: Ask for feedback on major architectural changes or design choices.
-title: "[RFC]: "
-labels: ["RFC"]
-
-body:
-- type: markdown
-  attributes:
-    value: >
-      #### Please take a look at previous [RFCs](https://github.com/vllm-project/vllm-omni/issues?q=label%3ARFC+sort%3Aupdated-desc) for reference.
-- type: textarea
-  attributes:
-    label: Motivation.
-    description: >
-      The motivation of the RFC.
-  validations:
-    required: true
-- type: textarea
-  attributes:
-    label: Proposed Change.
-    description: >
-      The proposed change of the RFC.
-    value: |
-      Please provide the detailed design document of the RFC using the [template](https://docs.google.com/document/d/12YxSsVeD1jvL-InClkeAEnZyWFDndz_65JmXvsamuV4/edit?tab=t.0#heading=h.4ef4szrsgspp).
-  validations:
-    required: true
-- type: textarea
-  attributes:
-    label: Feedback Period.
-    description: >
-      The feedback period of the RFC. Usually at least one week.
-  validations:
-    required: false
-- type: textarea
-  attributes:
-    label: CC List.
-    description: >
-      The list of people you want to CC.
-  validations:
-    required: false
-- type: textarea
-  attributes:
-    label: Any Other Things.
-    description: >
-      Any other things you would like to mention.
-  validations:
-    required: false
-- type: checkboxes
-  id: askllm
-  attributes:
-    label: Before submitting a new issue...
-    options:
-      - label: Make sure you already searched for relevant issues, and asked the chatbot living at the bottom right corner of the [documentation page](https://vllm-omni.readthedocs.io), which can answer lots of frequently asked questions.
-        required: true

.github/ISSUE_TEMPLATE/config.yml

@@ -1,5 +0,0 @@
-blank_issues_enabled: false
-contact_links:
-  - name: Questions
-    url: https://slack.vllm.ai
-    about: Ask questions and discuss with other vLLM-Omni community members in our #sig-omni slack channel.

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,21 +0,0 @@
-<!-- markdownlint-disable -->
-PLEASE FILL IN THE PR DESCRIPTION HERE ENSURING ALL CHECKLIST ITEMS (AT THE BOTTOM) HAVE BEEN CONSIDERED.
-
-## Purpose
-
-## Test Plan
-
-## Test Result
-
----
-<details>
-<summary> Essential Elements of an Effective PR Description Checklist </summary>
-
-- [ ] The purpose of the PR, such as "Fix some issue (link existing issues this PR will resolve)".
-- [ ] The test plan. Please providing the test scripts & test commands. Please state the reasons if your codes don't require additional test scripts. For test file guidelines, please check the [test style doc](https://docs.vllm.ai/projects/vllm-omni/en/latest/contributing/ci/tests_style/)
-- [ ] The test results. Please pasting the results comparison before and after, or e2e results.
-- [ ] (Optional) The necessary documentation update, such as updating `supported_models.md` and `examples` for a new model. **Please run `mkdocs serve` to sync the documentation editions to `./docs`.**
-- [ ] (Optional) Release notes update. If your change is user facing, please update the release notes draft.
-</details>
-
-**BEFORE SUBMITTING, PLEASE READ <https://github.com/vllm-project/vllm-omni/blob/main/CONTRIBUTING.md>** (anything written below this line will be removed by GitHub Actions)

.github/scripts/detect_changed_tests.sh

@@ -1,89 +0,0 @@
-#!/bin/bash
-# Script to detect changed files and find corresponding test files
-# Usage: ./detect_changed_tests.sh [base_sha] [head_sha] [event_type]
-#
-# Outputs to GITHUB_OUTPUT:
-#   test_files: Space-separated list of test files to run
-#   has_tests: 'true' if test files found, 'false' otherwise
-
-set -e
-
-BASE_SHA="${1:-}"
-HEAD_SHA="${2:-}"
-EVENT_TYPE="${3:-push}"
-
-# Determine base and head SHA based on event type
-if [ "$EVENT_TYPE" = "pull_request" ]; then
-    BASE_SHA="${BASE_SHA:-$GITHUB_BASE_REF}"
-    HEAD_SHA="${HEAD_SHA:-$GITHUB_SHA}"
-elif [ -z "$BASE_SHA" ] || [ -z "$HEAD_SHA" ]; then
-    # For push events, use git to find the previous commit
-    BASE_SHA="${BASE_SHA:-$(git rev-parse HEAD~1 2>/dev/null || echo HEAD)}"
-    HEAD_SHA="${HEAD_SHA:-HEAD}"
-fi
-
-echo "Event type: $EVENT_TYPE"
-echo "Base SHA: $BASE_SHA"
-echo "Head SHA: $HEAD_SHA"
-
-# Get changed Python files
-CHANGED_FILES=$(git diff --name-only --diff-filter=ACMRT "$BASE_SHA" "$HEAD_SHA" | grep -E '\.(py)$' || true)
-
-if [ -z "$CHANGED_FILES" ]; then
-    echo "No Python files changed"
-    echo "test_files=" >> "$GITHUB_OUTPUT"
-    echo "has_tests=false" >> "$GITHUB_OUTPUT"
-    exit 0
-fi
-
-echo "Changed Python files:"
-echo "$CHANGED_FILES"
-
-# Extract related test files
-TEST_FILES=""
-for file in $CHANGED_FILES; do
-    # If changed file is a test file, add it directly
-    if [[ "$file" == tests/* ]]; then
-        TEST_FILES="$TEST_FILES $file"
-    # If changed file is source code, find corresponding test file
-    elif [[ "$file" == vllm_omni/* ]]; then
-        # Convert vllm_omni/path/to/module.py to possible test file paths
-        REL_PATH=$(echo "$file" | sed 's|^vllm_omni/||' | sed 's|\.py$||')
-        MODULE_NAME=$(basename "$REL_PATH")
-        DIR_PATH=$(dirname "$REL_PATH")
-
-        # Try multiple possible test file naming patterns
-        POSSIBLE_TESTS=(
-            "tests/test_${REL_PATH//\//_}.py"
-            "tests/${REL_PATH//\//_}_test.py"
-        )
-
-        # If multi-level directory, also try directory structure
-        if [ "$DIR_PATH" != "." ]; then
-            POSSIBLE_TESTS+=(
-                "tests/${DIR_PATH}/test_${MODULE_NAME}.py"
-                "tests/${DIR_PATH}/${MODULE_NAME}_test.py"
-            )
-        fi
-
-        for test_file in "${POSSIBLE_TESTS[@]}"; do
-            if [ -f "$test_file" ]; then
-                TEST_FILES="$TEST_FILES $test_file"
-                break
-            fi
-        done
-    fi
-done
-
-# Deduplicate and format
-TEST_FILES=$(echo "$TEST_FILES" | tr ' ' '\n' | grep -v '^$' | sort -u | tr '\n' ' ' | sed 's/^ *//;s/ *$//')
-
-if [ -z "$TEST_FILES" ]; then
-    echo "No related test files found, will run all tests"
-    echo "test_files=" >> "$GITHUB_OUTPUT"
-    echo "has_tests=false" >> "$GITHUB_OUTPUT"
-else
-    echo "Found test files: $TEST_FILES"
-    echo "test_files=$TEST_FILES" >> "$GITHUB_OUTPUT"
-    echo "has_tests=true" >> "$GITHUB_OUTPUT"
-fi

.github/scripts/pr_reviewer.py

@@ -1,629 +0,0 @@
-#!/usr/bin/env python3
-"""
-PR Reviewer using GLM API for vllm-omni project.
-"""
-
-import json
-import logging
-import os
-import sys
-import time
-from dataclasses import dataclass
-from typing import Any, TypedDict
-
-import requests
-
-
-# Type definitions for API responses
-class PRDetails(TypedDict):
-    """Type definition for GitHub PR details response."""
-
-    title: str
-    body: str
-    number: int
-    state: str
-    user: dict[str, Any]
-
-
-class GLMMessage(TypedDict):
-    """Type definition for GLM API message."""
-
-    role: str
-    content: str
-
-
-class GLMChoice(TypedDict):
-    """Type definition for GLM API choice."""
-
-    message: GLMMessage
-    finish_reason: str
-
-
-class GLMResponse(TypedDict):
-    """Type definition for GLM API response."""
-
-    choices: list[GLMChoice]
-    usage: dict[str, int] | None
-
-
-class GitHubComment(TypedDict):
-    """Type definition for GitHub comment."""
-
-    id: int
-    body: str
-    created_at: str
-    user: dict[str, Any]
-
-
-# Configuration
-TRIGGER_PHRASE: str = "@vllm-omni-reviewer"
-DEFAULT_GLM_API_URL: str = "https://open.bigmodel.cn/api/paas/v4/chat/completions"  # noqa: E501
-DEFAULT_GLM_MODEL: str = "glm-5"
-DEFAULT_COOLDOWN_MINUTES: int = 5
-DEFAULT_MAX_RETRIES: int = 3
-DEFAULT_RETRY_DELAY: float = 1.0
-MAX_DIFF_SIZE: int = 100_000  # Maximum diff size in characters
-
-
-@dataclass
-class Config:
-    """Configuration for the PR reviewer."""
-
-    glm_api_url: str
-    glm_model: str
-    cooldown_minutes: int
-    max_retries: int
-    retry_delay: float
-    max_diff_size: int
-
-
-# Setup logging
-logging.basicConfig(
-    level=logging.INFO,
-    format="[PR Reviewer] %(asctime)s - %(levelname)s - %(message)s",
-    datefmt="%Y-%m-%d %H:%M:%S",
-)
-logger: logging.Logger = logging.getLogger(__name__)
-
-
-def get_config() -> Config:
-    """Load configuration from environment variables with defaults."""
-    return Config(
-        glm_api_url=os.getenv("GLM_API_URL", DEFAULT_GLM_API_URL),
-        glm_model=os.getenv("GLM_MODEL", DEFAULT_GLM_MODEL),
-        cooldown_minutes=int(
-            os.getenv(
-                "PR_REVIEWER_COOLDOWN_MINUTES",
-                str(DEFAULT_COOLDOWN_MINUTES),
-            )
-        ),
-        max_retries=int(
-            os.getenv(
-                "PR_REVIEWER_MAX_RETRIES",
-                str(DEFAULT_MAX_RETRIES),
-            )
-        ),
-        retry_delay=float(os.getenv("PR_REVIEWER_RETRY_DELAY", str(DEFAULT_RETRY_DELAY))),
-        max_diff_size=int(os.getenv("PR_REVIEWER_MAX_DIFF_SIZE", str(MAX_DIFF_SIZE))),  # noqa: E501
-    )
-
-
-def get_env_var(name: str) -> str:
-    """
-    Get an environment variable or raise an error.
-
-    Args:
-        name: Name of the environment variable.
-
-    Returns:
-        The value of the environment variable.
-
-    Raises:
-        SystemExit: If the environment variable is not set.
-    """
-    value = os.environ.get(name)
-    if not value:
-        logger.error(f"Environment variable {name} is not set")
-        sys.exit(1)
-    return value
-
-
-def check_trigger(comment_body: str) -> bool:
-    """
-    Check if the comment contains the trigger phrase.
-
-    Args:
-        comment_body: The body of the comment to check.
-
-    Returns:
-        True if the trigger phrase is found, False otherwise.
-    """
-    return TRIGGER_PHRASE in comment_body
-
-
-def fetch_pr_diff(
-    repo_name: str,
-    pr_number: int,
-    token: str,
-    max_size: int = MAX_DIFF_SIZE,
-) -> str | None:
-    """
-    Fetch the diff for a pull request.
-
-    Args:
-        repo_name: The repository name in format "owner/repo".
-        pr_number: The pull request number.
-        token: GitHub authentication token.
-        max_size: Maximum diff size in characters.
-
-    Returns:
-        The diff content as a string, or None if fetching failed.
-        Returns empty string if diff is larger than max_size.
-    """
-    url: str = f"https://api.github.com/repos/{repo_name}/pulls/{pr_number}"
-    headers: dict[str, str] = {
-        "Authorization": f"Bearer {token}",
-        "Accept": "application/vnd.github.v3.diff",
-    }
-
-    logger.info(f"Fetching PR diff from {url}")
-    response = requests.get(url, headers=headers, timeout=30)
-
-    if response.status_code == 200:
-        diff: str = response.text
-        if len(diff) > max_size:
-            logger.warning(
-                f"Diff size ({len(diff)} bytes) exceeds maximum "
-                f"({max_size} bytes), truncating to first "
-                f"{max_size} bytes"
-            )
-            return diff[:max_size] + "\n\n... [Diff truncated due to size] ..."
-        logger.info(f"Successfully fetched diff ({len(diff)} bytes)")
-        return diff
-    else:
-        logger.error(f"Failed to fetch PR diff: {response.status_code}")
-        logger.error(f"Response: {response.text}")
-        return None
-
-
-def fetch_pr_details(
-    repo_name: str,
-    pr_number: int,
-    token: str,
-) -> PRDetails | None:
-    """
-    Fetch PR details including title and description.
-
-    Args:
-        repo_name: The repository name in format "owner/repo".
-        pr_number: The pull request number.
-        token: GitHub authentication token.
-
-    Returns:
-        A dictionary containing PR details, or None if fetching failed.
-    """
-    url: str = f"https://api.github.com/repos/{repo_name}/pulls/{pr_number}"
-    headers: dict[str, str] = {
-        "Authorization": f"Bearer {token}",
-        "Accept": "application/vnd.github.v3+json",
-    }
-
-    logger.info(f"Fetching PR details from {url}")
-    response = requests.get(url, headers=headers, timeout=30)
-
-    if response.status_code == 200:
-        return response.json()
-    else:
-        logger.error(f"Failed to fetch PR details: {response.status_code}")
-        return None
-
-
-def build_review_prompt(pr_title: str, pr_description: str, diff: str) -> str:
-    """
-    Build the prompt for the GLM-4.7 API.
-
-    Args:
-        pr_title: The title of the pull request.
-        pr_description: The description/body of the pull request.
-        diff: The diff content of the pull request.
-
-    Returns:
-        The formatted prompt string for the API.
-    """
-    return f"""You are an expert code reviewer for the VLLM-Omni project. \
-Please review the following pull request:
-
-## Pull Request Details
-**Title:** {pr_title}
-
-**Description:**
-{pr_description if pr_description else "No description provided."}
-
-## Code Changes (Diff)
-{diff}
-
-## Review Guidelines
-
-Please provide a comprehensive code review with the following sections:
-
-### 1. Overview
-- Brief summary of the changes
-- Overall assessment (positive, neutral, or concerns)
-
-### 2. Code Quality
-- Code style and consistency
-- Potential bugs or edge cases
-- Performance considerations
-- Error handling
-
-### 3. Architecture & Design
-- Integration with existing codebase
-- Design patterns and best practices
-- Potential improvements
-
-### 4. Security & Safety
-- Security concerns (if any)
-- Resource management
-- Input validation
-
-### 5. Testing & Documentation
-- Test coverage considerations
-- Documentation completeness
-- Examples and usage clarity
-
-### 6. Specific Suggestions
-- Line-by-line specific feedback (use `file:line` format)
-- Concrete actionable suggestions
-- Code examples for improvements (if applicable)
-
-### 7. Approval Status
-- **LGTM** (Looks Good To Me) if the PR is ready to merge
-- **LGTM with suggestions** if the PR is good but has minor suggestions
-- **Changes requested** if significant changes are needed
-
-## Important Notes
-- Be constructive and helpful
-- Focus on objective technical feedback
-- Acknowledge good practices when you see them
-- Prioritize critical issues over nitpicks
-- If the diff is empty or minimal, acknowledge this and provide
-  any relevant context-specific guidance
-
-Please format your response in Markdown with clear section headers.
-"""
-
-
-def validate_glm_response(data: dict[str, Any]) -> str | None:
-    """
-    Validate and extract content from GLM API response.
-
-    Args:
-        data: The response data from GLM API.
-
-    Returns:
-        The review content string if valid, None otherwise.
-    """
-    # Check if choices exists and is a non-empty list
-    if "choices" not in data:
-        logger.error("GLM API response missing 'choices' field")
-        logger.error(f"Response structure: {json.dumps(data, indent=2)}")
-        return None
-
-    choices = data["choices"]
-    if not isinstance(choices, list):
-        logger.error(f"GLM API 'choices' is not a list: {type(choices)}")
-        return None
-
-    if len(choices) == 0:
-        logger.error("GLM API 'choices' is an empty list")
-        return None
-
-    # Check if first choice has message
-    try:
-        first_choice = choices[0]
-        if not isinstance(first_choice, dict):
-            logger.error(f"GLM API choice is not a dict: {type(first_choice)}")
-            return None
-
-        if "message" not in first_choice:
-            logger.error("GLM API choice missing 'message' field")
-            logger.error(f"Choice structure: {json.dumps(first_choice, indent=2)}")  # noqa: E501
-            return None
-
-        message = first_choice["message"]
-        if not isinstance(message, dict):
-            logger.error(f"GLM API message is not a dict: {type(message)}")
-            return None
-
-        if "content" not in message:
-            logger.error("GLM API message missing 'content' field")
-            logger.error(f"Message structure: {json.dumps(message, indent=2)}")
-            return None
-
-        content = message["content"]
-        if not isinstance(content, str):
-            logger.error(f"GLM API content is not a string: {type(content)}")
-            return None
-
-        return content
-
-    except (KeyError, IndexError, TypeError) as e:
-        logger.error(f"Failed to parse GLM API response: {e}")
-        logger.error(f"Response: {json.dumps(data, indent=2)}")
-        return None
-
-
-def call_glm_api(prompt: str, api_key: str, config: Config) -> str | None:
-    """
-    Call the GLM-4.7 API to get code review with retry logic.
-
-    Args:
-        prompt: The prompt to send to the API.
-        api_key: The GLM API key.
-        config: Configuration object.
-
-    Returns:
-        The review content as a string, or None if all retries failed.
-    """
-    headers: dict[str, str] = {
-        "Authorization": f"Bearer {api_key}",
-        "Content-Type": "application/json",
-    }
-
-    payload: dict[str, Any] = {
-        "model": config.glm_model,
-        "messages": [{"role": "user", "content": prompt}],
-        "temperature": 0.3,
-        "max_tokens": 32000,
-        "top_p": 0.9,
-    }
-
-    last_error: str | None = None
-
-    for attempt in range(config.max_retries):
-        try:
-            logger.info(f"Calling GLM API ({config.glm_model}) - Attempt {attempt + 1}/{config.max_retries}")
-            response = requests.post(
-                config.glm_api_url,
-                headers=headers,
-                json=payload,
-                timeout=120,
-            )
-
-            if response.status_code == 200:
-                data = response.json()
-                review = validate_glm_response(data)
-                if review:
-                    logger.info(f"Successfully received review ({len(review)} chars)")  # noqa: E501
-                    return review
-                else:
-                    last_error = "Failed to validate API response structure"
-                    logger.error(last_error)
-            else:
-                last_error = f"GLM API request failed: {response.status_code} - {response.text}"
-                logger.error(last_error)
-
-        except requests.exceptions.Timeout:
-            last_error = f"GLM API request timed out (attempt {attempt + 1})"
-            logger.error(last_error)
-        except requests.exceptions.RequestException as e:
-            last_error = f"GLM API request exception: {e}"
-            logger.error(last_error)
-        except json.JSONDecodeError as e:
-            last_error = f"Failed to decode GLM API response as JSON: {e}"
-            logger.error(last_error)
-
-        # Exponential backoff before retry
-        if attempt < config.max_retries - 1:
-            wait_time: float = config.retry_delay * (2**attempt)
-            logger.info(f"Waiting {wait_time}s before retry...")  # noqa: E501
-            time.sleep(wait_time)
-
-    logger.error(
-        f"All {config.max_retries} attempts failed. Last error: {last_error}"  # noqa: E501
-    )
-    return None
-
-
-def check_cooldown(  # noqa: E501
-    repo_name: str,
-    pr_number: int,
-    token: str,
-    cooldown_minutes: int,
-) -> bool:
-    """
-    Check if the PR is within the cooldown period.
-
-    Args:
-        repo_name: The repository name in format "owner/repo".
-        pr_number: The pull request number.
-        token: GitHub authentication token.
-        cooldown_minutes: Cooldown period in minutes.
-
-    Returns:
-        True if within cooldown period (should skip), False otherwise.
-    """
-    from datetime import datetime, timedelta
-
-    url: str = (
-        f"https://api.github.com/repos/{repo_name}/issues/"
-        f"{pr_number}/comments"  # noqa: E501
-    )
-    headers: dict[str, str] = {
-        "Authorization": f"Bearer {token}",
-        "Accept": "application/vnd.github.v3+json",
-    }
-
-    logger.info(f"Checking cooldown period ({cooldown_minutes} minutes)")
-    response = requests.get(url, headers=headers, timeout=30)
-
-    if response.status_code != 200:
-        logger.warning(f"Failed to check cooldown: {response.status_code}, proceeding with review")
-        return False
-
-    comments: list[dict[str, Any]] = response.json()
-    cutoff_time: datetime = datetime.utcnow() - timedelta(minutes=cooldown_minutes)  # noqa: E501
-
-    for comment in reversed(comments):
-        # Check if this is a bot comment
-        body: str = comment.get("body", "")
-        if "VLLM-Omni PR Review" in body or "PR Reviewer Bot" in body:
-            created_at_str: str = comment.get("created_at", "")
-            try:
-                # Parse GitHub timestamp format
-                created_at = datetime.fromisoformat(created_at_str.replace("Z", "+00:00"))
-                created_at = created_at.replace(tzinfo=None)
-                if created_at > cutoff_time:
-                    logger.info(f"PR is within cooldown period (last review: {created_at_str})")
-                    return True
-            except ValueError:
-                logger.warning(f"Failed to parse comment timestamp: {created_at_str}")  # noqa: E501
-                continue
-
-    logger.info("PR is outside cooldown period, proceeding with review")
-    return False
-
-
-def post_review_comment(  # noqa: E501
-    repo_name: str,
-    pr_number: int,
-    token: str,
-    review: str,
-) -> bool:
-    """
-    Post the review as a comment on the PR.
-
-    Args:
-        repo_name: The repository name in format "owner/repo".
-        pr_number: The pull request number.
-        token: GitHub authentication token.
-        review: The review content to post.
-
-    Returns:
-        True if posting succeeded, False otherwise.
-    """
-    url: str = (
-        f"https://api.github.com/repos/{repo_name}/issues/"
-        f"{pr_number}/comments"  # noqa: E501
-    )
-    headers: dict[str, str] = {
-        "Authorization": f"Bearer {token}",
-        "Accept": "application/vnd.github.v3+json",
-    }
-
-    # Format the review comment
-    comment_body: str = f"""## 🤖 VLLM-Omni PR Review
-
-{review}
-
----
-*This review was generated automatically by the VLLM-Omni PR Reviewer Bot
-using {os.getenv("GLM_MODEL", DEFAULT_GLM_MODEL)}.*
-"""
-
-    payload: dict[str, str] = {"body": comment_body}
-
-    logger.info(f"Posting review comment to PR #{pr_number}")
-    response = requests.post(url, headers=headers, json=payload, timeout=30)
-
-    if response.status_code == 201:
-        logger.info("Successfully posted review comment")
-        return True
-    else:
-        logger.error(f"Failed to post comment: {response.status_code}")
-        logger.error(f"Response: {response.text}")
-        return False
-
-
-def main() -> int:
-    """
-    Main entry point for the PR reviewer bot.
-
-    Returns:
-        0 on success, 1 on error.
-    """
-    logger.info("VLLM-Omni PR Reviewer Bot starting...")
-
-    # Load configuration
-    config: Config = get_config()
-    logger.info(
-        f"Configuration: model={config.glm_model}, "
-        f"cooldown={config.cooldown_minutes}min, "
-        f"max_retries={config.max_retries}"
-    )
-
-    # Get environment variables
-    token: str = get_env_var("GITHUB_TOKEN")
-    api_key: str = get_env_var("GLM_API_KEY")
-    repo_name: str = get_env_var("REPO_NAME")
-    pr_number_str: str = get_env_var("PR_NUMBER")
-    comment_body: str = get_env_var("COMMENT_BODY")
-
-    try:
-        pr_number: int = int(pr_number_str)
-    except ValueError:
-        logger.error(f"Invalid PR number: {pr_number_str}")
-        return 1
-
-    logger.info(f"Repository: {repo_name}")
-    logger.info(f"PR Number: {pr_number}")
-
-    # Check if the comment contains the trigger phrase
-    if not check_trigger(comment_body):
-        logger.info(
-            f"Comment does not contain trigger phrase '{TRIGGER_PHRASE}', exiting"  # noqa: E501
-        )
-        return 0
-
-    logger.info("Trigger phrase detected! Starting review process...")
-
-    # Check cooldown period
-    if check_cooldown(repo_name, pr_number, token, config.cooldown_minutes):
-        logger.info("Skipping review due to cooldown period")
-        return 0
-
-    # Fetch PR details
-    logger.info("Step 1/4: Fetching PR details...")
-    pr_details: PRDetails | None = fetch_pr_details(repo_name, pr_number, token)  # noqa: E501
-    if not pr_details:
-        logger.error("Failed to fetch PR details")
-        return 1
-
-    pr_title: str = pr_details.get("title", "Unknown")
-    pr_description: str = pr_details.get("body", "")
-
-    logger.info(f"PR Title: {pr_title}")
-
-    # Fetch PR diff
-    logger.info("Step 2/4: Fetching PR diff...")
-    diff: str | None = fetch_pr_diff(repo_name, pr_number, token, config.max_diff_size)
-    if diff is None:
-        logger.error("Failed to fetch PR diff")
-        return 1
-
-    if not diff:
-        logger.warning("Warning: Empty diff - this might be a draft PR or no code changes")
-
-    # Build prompt
-    logger.info("Step 3/4: Building review prompt...")
-    prompt: str = build_review_prompt(pr_title, pr_description, diff)
-
-    # Call GLM API
-    logger.info("Step 4/4: Calling GLM API...")
-    review: str | None = call_glm_api(prompt, api_key, config)
-    if not review:
-        logger.error("Failed to get review from GLM API")
-        return 1
-
-    # Post review comment
-    logger.info("Posting review comment...")
-    if not post_review_comment(repo_name, pr_number, token, review):
-        logger.error("Failed to post review comment")
-        return 1
-
-    logger.info("PR review completed successfully!")
-    return 0
-
-
-if __name__ == "__main__":
-    sys.exit(main())

.github/workflows/build_wheel.yml

@@ -1,63 +0,0 @@
-name: Build Wheel
-
-on:
-  push:
-    branches:
-      - main
-    paths-ignore:
-      - 'docs/**'
-      - 'examples/**'
-      - 'apps/**'
-      - '.github/**'
-      - '.buildkite/**'
-      - '**.md'
-      - '*.yml'
-      - '.gitignore'
-      - '.gitattributes'
-      - 'LICENSE'
-  pull_request:
-    branches:
-      - main
-    paths-ignore:
-      - 'docs/**'
-      - 'examples/**'
-      - 'apps/**'
-      - '.github/**'
-      - '.buildkite/**'
-      - '*.yml'
-      - '**.md'
-      - '.gitignore'
-      - '.gitattributes'
-      - 'LICENSE'
-  workflow_dispatch:
-
-jobs:
-  build:
-    runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        python-version: ["3.11", "3.12"]
-
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: Set up Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v5
-        with:
-          python-version: ${{ matrix.python-version }}
-
-      - name: Install uv
-        run: |
-          pip install uv
-
-      - name: Build Wheel
-        env:
-          UV_SYSTEM_PYTHON: 1
-        run: |
-          bash scripts/build_wheel.sh --python python
-
-      - name: Upload Wheel Artifact
-        uses: actions/upload-artifact@v4
-        with:
-          name: vllm-omni-wheel-py${{ matrix.python-version }}
-          path: dist/*.whl

.github/workflows/comfyui-publish.yml

@@ -1,27 +0,0 @@
-name: ComfyUI Integration -- Publish to Comfy registry
-on:
-  workflow_dispatch:
-  push:
-    tags:
-      - "*"
-    paths:
-      - "apps/ComfyUI-vLLM-Omni/**"
-
-permissions:
-  issues: write
-
-defaults:
-  run:
-    working-directory: apps/ComfyUI-vLLM-Omni
-
-jobs:
-  publish-node:
-    name: Publish Custom Node to registry
-    runs-on: ubuntu-latest
-    steps:
-      - name: ♻️ Check out code
-        uses: actions/checkout@v4
-      - name: 📦 Publish Custom Node
-        uses: Comfy-Org/publish-node-action@main
-        with:
-          personal_access_token: ${{ secrets.REGISTRY_ACCESS_TOKEN }}

.github/workflows/comfyui-validate.yml

@@ -1,40 +0,0 @@
-name: ComfyUI Integration -- Validate backwards compatibility
-
-on:
-  pull_request:
-    branches:
-      - master
-      - main
-    paths:
-      - "apps/ComfyUI-vLLM-Omni/**"
-
-defaults:
-  run:
-    working-directory: apps/ComfyUI-vLLM-Omni
-
-jobs:
-  check-base-path:
-    runs-on: ubuntu-latest
-    outputs:
-      exists: ${{ steps.check.outputs.exists }}
-    defaults:
-      run:
-        working-directory: .
-    steps:
-      - uses: actions/checkout@v4
-        with:
-          ref: ${{ github.event.pull_request.base.sha }}
-          fetch-depth: 1
-      - id: check
-        run: |
-          if [ -d "apps/ComfyUI-vLLM-Omni" ]; then
-            echo "exists=true" >> "$GITHUB_OUTPUT"
-          else
-            echo "exists=false" >> "$GITHUB_OUTPUT"
-          fi
-  validate:
-    needs: check-base-path
-    if: needs.check-base-path.outputs.exists == 'true'
-    runs-on: ubuntu-latest
-    steps:
-      - uses: comfy-org/node-diff@main

.github/workflows/pr-reviewer.yml

@@ -1,62 +0,0 @@
-name: VLLM-Omni PR Reviewer
-
-on:
-  issue_comment:
-    types: [created]
-
-permissions:
-  contents: read
-  pull-requests: write
-  issues: write
-
-jobs:
-  pr-reviewer:
-    name: Review Pull Request
-    runs-on: ubuntu-latest
-    timeout-minutes: 10
-    # Only run when the comment is from a collaborator/owner/member
-    if: |
-      github.event_name == 'issue_comment' &&
-      github.event.issue.pull_request != null &&
-      contains(github.event.comment.body, '@vllm-omni-reviewer') &&
-      (github.event.comment.author_association == 'MEMBER' ||
-       github.event.comment.author_association == 'COLLABORATOR' ||
-       github.event.comment.author_association == 'OWNER')
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4.2.2
-        with:
-          fetch-depth: 0
-
-      - name: Set up Python
-        uses: actions/setup-python@v5.3.0
-        with:
-          python-version: '3.11'
-          cache: 'pip'
-
-      - name: Install dependencies
-        run: |
-          python -m pip install --upgrade pip==24.0
-          pip install requests==2.31.0 pyyaml==6.0.1
-
-      - name: Run PR Reviewer
-        id: reviewer
-        env:
-          GLM_API_KEY: ${{ secrets.GLM_API_KEY }}
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          PR_NUMBER: ${{ github.event.issue.number || github.event.pull_request.number }}
-          COMMENT_BODY: ${{ github.event.comment.body }}
-          REPO_NAME: ${{ github.repository }}
-          PR_REVIEWER_COOLDOWN_MINUTES: 5
-          PR_REVIEWER_MAX_RETRIES: 3
-        run: |
-          python .github/scripts/pr_reviewer.py 2>&1 | tee "/tmp/pr_review_${PR_NUMBER}.log"
-
-      - name: Upload review logs
-        if: always()
-        uses: actions/upload-artifact@v4.4.3
-        with:
-          name: pr-review-logs-${{ github.event.issue.number || github.event.pull_request.number }}
-          path: /tmp/pr_review_*.log
-          retention-days: 7
-          if-no-files-found: ignore

.github/workflows/pre-commit.yml

@@ -1,26 +0,0 @@
-name: pre-commit
-
-on:
-  pull_request:
-    branches: [main]
-  push:
-    branches: [main]
-  workflow_dispatch: # for manual trigger
-
-concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
-  cancel-in-progress: ${{ github.event_name == 'pull_request' }}
-
-permissions:
-  contents: read
-
-jobs:
-  pre-commit:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-      - uses: actions/setup-python@v5
-        with:
-          python-version: "3.12"
-          cache: 'pip'
-      - uses: pre-commit/action@v3.0.1

.gitignore

@@ -1,247 +0,0 @@
-# Byte-compiled / optimized / DLL files
-__pycache__/
-*.py[cod]
-*$py.class
-
-# C extensions
-*.so
-
-# Distribution / packaging
-.Python
-build/
-develop-eggs/
-dist/
-downloads/
-eggs/
-.eggs/
-lib/
-lib64/
-parts/
-sdist/
-var/
-wheels/
-vllm/
-share/python-wheels/
-*.egg-info/
-.installed.cfg
-*.egg
-MANIFEST
-
-# PyInstaller
-#  Usually these files are written by a python script from a template
-#  before PyInstaller builds the exe, so as to inject date/other infos into it.
-*.manifest
-*.spec
-
-# Installer logs
-pip-log.txt
-pip-delete-this-directory.txt
-
-# Unit test / coverage reports
-htmlcov/
-.tox/
-.nox/
-.coverage
-.coverage.*
-.cache
-nosetests.xml
-coverage.xml
-*.cover
-*.py,cover
-.hypothesis/
-.pytest_cache/
-cover/
-
-# Translations
-*.mo
-*.pot
-
-# Django stuff:
-*.log
-local_settings.py
-db.sqlite3
-db.sqlite3-journal
-
-# Flask stuff:
-instance/
-.webassets-cache
-
-# Scrapy stuff:
-.scrapy
-
-# Sphinx documentation
-docs/_build/
-
-# PyBuilder
-.pybuilder/
-target/
-
-# Jupyter Notebook
-.ipynb_checkpoints
-
-# IPython
-profile_default/
-ipython_config.py
-
-# pyenv
-#   For a library or package, you might want to ignore these files since the code is
-#   intended to run in multiple environments; otherwise, check them in:
-# .python-version
-
-# pipenv
-#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
-#   However, in case of collaboration, if having platform-specific dependencies or dependencies
-#   having no cross-platform support, pipenv may install dependencies that don't work, or not
-#   install all needed dependencies.
-#Pipfile.lock
-
-# poetry
-#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
-#   This is especially recommended for binary packages to ensure reproducibility, and is more
-#   commonly ignored for libraries.
-#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
-#poetry.lock
-
-# pdm
-#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
-#pdm.lock
-#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
-#   in version control.
-#   https://pdm.fming.dev/#use-with-ide
-.pdm.toml
-
-# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
-__pypackages__/
-
-# Celery stuff
-celerybeat-schedule
-celerybeat.pid
-
-# SageMath parsed files
-*.sage.py
-
-# Environments
-.env
-.venv
-env/
-venv/
-ENV/
-env.bak/
-venv.bak/
-
-# Spyder project settings
-.spyderproject
-.spyproject
-
-# Rope project settings
-.ropeproject
-
-# mkdocs documentation
-/site
-
-# mypy
-.mypy_cache/
-.dmypy.json
-dmypy.json
-
-# Pyre type checker
-.pyre/
-
-# pytype static type analyzer
-.pytype/
-
-# Cython debug symbols
-cython_debug/
-
-# Claude
-CLAUDE.md
-.claude/
-
-# Codex
-AGENTS.md
-.codex/
-
-# cursor
-.cursor/
-
-# scripts
-/scripts/
-
-# PyCharm
-#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
-#  be added to the global gitignore or merged into this project gitignore.  For a PyCharm
-#  project, it is recommended to include the following files:
-#  .idea/
-#  *.iml
-#  *.ipr
-#  *.iws
-#  .idea/
-
-# vLLM-Omni specific
-# Model files and checkpoints
-checkpoints/
-*.bin
-*.safetensors
-*.pt
-*.pth
-
-# Cache directories
-cache/
-!vllm_omni/diffusion/cache/
-.cache/
-diffusion_cache/
-kv_cache/
-
-# Logs
-logs/
-*.log
-
-# Temporary files
-tmp/
-temp/
-.tmp/
-
-# IDE files
-.vscode/
-.idea/
-*.swp
-*.swo
-*~
-
-# OS files
-.DS_Store
-.DS_Store?
-._*
-.Spotlight-V100
-.Trashes
-ehthumbs.db
-Thumbs.db
-
-# Data files
-data/
-datasets/
-*.csv
-*.json
-!apps/ComfyUI-vLLM-Omni/example_workflows/*.json
-*.jsonl
-*.parquet
-
-# Output files
-outputs/
-results/
-generated/
-output_*/
-
-# Configuration overrides
-configs/local.yaml
-configs/production.yaml
-configs/development.yaml
-
-# Docker
-.dockerignore
-Dockerfile.dev
-discussion
-tmp_test
-
-# output files
-*.wav

.pre-commit-config.yaml

@@ -1,70 +0,0 @@
-default_install_hook_types:
-  - pre-commit
-  - commit-msg
-repos:
-  - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v6.0.0
-    hooks:
-      # list of supported hooks: https://pre-commit.com/hooks.html
-      - id: check-yaml
-        args: ["--unsafe"]
-      - id: debug-statements
-      - id: end-of-file-fixer
-      - id: mixed-line-ending
-        args: ["--fix=lf"]
-      - id: trailing-whitespace
-        args: ["--markdown-linebreak-ext=md"]
-
-  - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.14.10
-    hooks:
-    - id: ruff-check
-      args: [--output-format, github, --fix]
-    - id: ruff-format
-
-  - repo: https://github.com/crate-ci/typos
-    rev: typos-dict-v0.13.13
-    hooks:
-      - id: typos
-        # only for staged files
-
-  - repo: https://github.com/rhysd/actionlint
-    # v1.7.8+ sets `go 1.24.0` in go.mod, which older Go toolchains (and most
-    # current CI images) cannot parse. Pin to v1.7.7 until actionlint fixes the
-    # go.mod directive.
-    rev: v1.7.7
-    hooks:
-      - id: actionlint
-        files: ^\.github/workflows/.*\.ya?ml$
-
-
-  - repo: local
-    hooks:
-    - id: signoff-commit
-      name: Sign-off Commit
-      entry: bash
-      args:
-        - -c
-        - |
-          if ! grep -q "^Signed-off-by: $(git config user.name) <$(git config user.email)>" "$(git rev-parse --git-path COMMIT_EDITMSG)"; then
-            printf "\nSigned-off-by: $(git config user.name) <$(git config user.email)>\n" >> "$(git rev-parse --git-path COMMIT_EDITMSG)"
-          fi
-      language: system
-      verbose: true
-      stages: [commit-msg]
-
-    # Keep `suggestion` last
-    - id: suggestion
-      name: Suggestion
-      entry: bash -c 'echo "To bypass all the pre-commit hooks, add --no-verify to git commit. To skip a specific hook, prefix the commit command with SKIP=<hook-id>."'
-      language: system
-      verbose: true
-      pass_filenames: false
-    # Insert new entries above the `suggestion` entry
-
-    - id: check-pickle-imports
-      name: Prevent new pickle/cloudpickle imports
-      entry: python tools/pre_commit/check_pickle_imports.py
-      language: python
-      types: [python]
-      additional_dependencies: [regex]

.readthedocs.yml

@@ -1,24 +0,0 @@
-# Read the Docs configuration file
-# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
-
-version: 2
-
-build:
-  os: ubuntu-22.04
-  tools:
-    python: "3.12"
-  jobs:
-    post_checkout:
-      - git fetch --unshallow || true
-
-mkdocs:
-  configuration: mkdocs.yml
-  fail_on_warning: true
-
-# Optionally declare the Python requirements required to build your docs
-python:
-  install:
-    - method: pip
-      path: .
-      extra_requirements:
-        - docs

CONTRIBUTING.md

@@ -1,3 +0,0 @@
-# Contributing to vLLM-Omni
-
-You may find information about contributing to vLLM-Omni on [Contributing](https://vllm-omni.readthedocs.io/en/latest/contributing/)

LICENSE

@@ -1,201 +0,0 @@
-                                 Apache License
-                           Version 2.0, January 2004
-                        http://www.apache.org/licenses/
-
-   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
-   1. Definitions.
-
-      "License" shall mean the terms and conditions for use, reproduction,
-      and distribution as defined by Sections 1 through 9 of this document.
-
-      "Licensor" shall mean the copyright owner or entity authorized by
-      the copyright owner that is granting the License.
-
-      "Legal Entity" shall mean the union of the acting entity and all
-      other entities that control, are controlled by, or are under common
-      control with that entity. For the purposes of this definition,
-      "control" means (i) the power, direct or indirect, to cause the
-      direction or management of such entity, whether by contract or
-      otherwise, or (ii) ownership of fifty percent (50%) or more of the
-      outstanding shares, or (iii) beneficial ownership of such entity.
-
-      "You" (or "Your") shall mean an individual or Legal Entity
-      exercising permissions granted by this License.
-
-      "Source" form shall mean the preferred form for making modifications,
-      including but not limited to software source code, documentation
-      source, and configuration files.
-
-      "Object" form shall mean any form resulting from mechanical
-      transformation or translation of a Source form, including but
-      not limited to compiled object code, generated documentation,
-      and conversions to other media types.
-
-      "Work" shall mean the work of authorship, whether in Source or
-      Object form, made available under the License, as indicated by a
-      copyright notice that is included in or attached to the work
-      (an example is provided in the Appendix below).
-
-      "Derivative Works" shall mean any work, whether in Source or Object
-      form, that is based on (or derived from) the Work and for which the
-      editorial revisions, annotations, elaborations, or other modifications
-      represent, as a whole, an original work of authorship. For the purposes
-      of this License, Derivative Works shall not include works that remain
-      separable from, or merely link (or bind by name) to the interfaces of,
-      the Work and Derivative Works thereof.
-
-      "Contribution" shall mean any work of authorship, including
-      the original version of the Work and any modifications or additions
-      to that Work or Derivative Works thereof, that is intentionally
-      submitted to Licensor for inclusion in the Work by the copyright owner
-      or by an individual or Legal Entity authorized to submit on behalf of
-      the copyright owner. For the purposes of this definition, "submitted"
-      means any form of electronic, verbal, or written communication sent
-      to the Licensor or its representatives, including but not limited to
-      communication on electronic mailing lists, source code control systems,
-      and issue tracking systems that are managed by, or on behalf of, the
-      Licensor for the purpose of discussing and improving the Work, but
-      excluding communication that is conspicuously marked or otherwise
-      designated in writing by the copyright owner as "Not a Contribution."
-
-      "Contributor" shall mean Licensor and any individual or Legal Entity
-      on behalf of whom a Contribution has been received by Licensor and
-      subsequently incorporated within the Work.
-
-   2. Grant of Copyright License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      copyright license to reproduce, prepare Derivative Works of,
-      publicly display, publicly perform, sublicense, and distribute the
-      Work and such Derivative Works in Source or Object form.
-
-   3. Grant of Patent License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      (except as stated in this section) patent license to make, have made,
-      use, offer to sell, sell, import, and otherwise transfer the Work,
-      where such license applies only to those patent claims licensable
-      by such Contributor that are necessarily infringed by their
-      Contribution(s) alone or by combination of their Contribution(s)
-      with the Work to which such Contribution(s) was submitted. If You
-      institute patent litigation against any entity (including a
-      cross-claim or counterclaim in a lawsuit) alleging that the Work
-      or a Contribution incorporated within the Work constitutes direct
-      or contributory patent infringement, then any patent licenses
-      granted to You under this License for that Work shall terminate
-      as of the date such litigation is filed.
-
-   4. Redistribution. You may reproduce and distribute copies of the
-      Work or Derivative Works thereof in any medium, with or without
-      modifications, and in Source or Object form, provided that You
-      meet the following conditions:
-
-      (a) You must give any other recipients of the Work or
-          Derivative Works a copy of this License; and
-
-      (b) You must cause any modified files to carry prominent notices
-          stating that You changed the files; and
-
-      (c) You must retain, in the Source form of any Derivative Works
-          that You distribute, all copyright, patent, trademark, and
-          attribution notices from the Source form of the Work,
-          excluding those notices that do not pertain to any part of
-          the Derivative Works; and
-
-      (d) If the Work includes a "NOTICE" text file as part of its
-          distribution, then any Derivative Works that You distribute must
-          include a readable copy of the attribution notices contained
-          within such NOTICE file, excluding those notices that do not
-          pertain to any part of the Derivative Works, in at least one
-          of the following places: within a NOTICE text file distributed
-          as part of the Derivative Works; within the Source form or
-          documentation, if provided along with the Derivative Works; or,
-          within a display generated by the Derivative Works, if and
-          wherever such third-party notices normally appear. The contents
-          of the NOTICE file are for informational purposes only and
-          do not modify the License. You may add Your own attribution
-          notices within Derivative Works that You distribute, alongside
-          or as an addendum to the NOTICE text from the Work, provided
-          that such additional attribution notices cannot be construed
-          as modifying the License.
-
-      You may add Your own copyright statement to Your modifications and
-      may provide additional or different license terms and conditions
-      for use, reproduction, or distribution of Your modifications, or
-      for any such Derivative Works as a whole, provided Your use,
-      reproduction, and distribution of the Work otherwise complies with
-      the conditions stated in this License.
-
-   5. Submission of Contributions. Unless You explicitly state otherwise,
-      any Contribution intentionally submitted for inclusion in the Work
-      by You to the Licensor shall be under the terms and conditions of
-      this License, without any additional terms or conditions.
-      Notwithstanding the above, nothing herein shall supersede or modify
-      the terms of any separate license agreement you may have executed
-      with Licensor regarding such Contributions.
-
-   6. Trademarks. This License does not grant permission to use the trade
-      names, trademarks, service marks, or product names of the Licensor,
-      except as required for reasonable and customary use in describing the
-      origin of the Work and reproducing the content of the NOTICE file.
-
-   7. Disclaimer of Warranty. Unless required by applicable law or
-      agreed to in writing, Licensor provides the Work (and each
-      Contributor provides its Contributions) on an "AS IS" BASIS,
-      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-      implied, including, without limitation, any warranties or conditions
-      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
-      PARTICULAR PURPOSE. You are solely responsible for determining the
-      appropriateness of using or redistributing the Work and assume any
-      risks associated with Your exercise of permissions under this License.
-
-   8. Limitation of Liability. In no event and under no legal theory,
-      whether in tort (including negligence), contract, or otherwise,
-      unless required by applicable law (such as deliberate and grossly
-      negligent acts) or agreed to in writing, shall any Contributor be
-      liable to You for damages, including any direct, indirect, special,
-      incidental, or consequential damages of any character arising as a
-      result of this License or out of the use or inability to use the
-      Work (including but not limited to damages for loss of goodwill,
-      work stoppage, computer failure or malfunction, or any and all
-      other commercial damages or losses), even if such Contributor
-      has been advised of the possibility of such damages.
-
-   9. Accepting Warranty or Additional Liability. While redistributing
-      the Work or Derivative Works thereof, You may choose to offer,
-      and charge a fee for, acceptance of support, warranty, indemnity,
-      or other liability obligations and/or rights consistent with this
-      License. However, in accepting such obligations, You may act only
-      on Your own behalf and on Your sole responsibility, not on behalf
-      of any other Contributor, and only if You agree to indemnify,
-      defend, and hold each Contributor harmless for any liability
-      incurred by, or claims asserted against, such Contributor by reason
-      of your accepting any such warranty or additional liability.
-
-   END OF TERMS AND CONDITIONS
-
-   APPENDIX: How to apply the Apache License to your work.
-
-      To apply the Apache License to your work, attach the following
-      boilerplate notice, with the fields enclosed by brackets "[]"
-      replaced with your own identifying information. (Don't include
-      the brackets!)  The text should be enclosed in the appropriate
-      comment syntax for the file format. We also recommend that a
-      file or class name and description of purpose be included on the
-      same "printed page" as the copyright notice for easier
-      identification within third-party archives.
-
-   Copyright [yyyy] [name of copyright owner]
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.

README.md

@@ -1,94 +0,0 @@
-<p align="center">
-  <picture>
-    <source media="(prefers-color-scheme: dark)" src="https://raw.githubusercontent.com/vllm-project/vllm-omni/refs/heads/main/docs/source/logos/vllm-omni-logo.png">
-    <img alt="vllm-omni" src="https://raw.githubusercontent.com/vllm-project/vllm-omni/refs/heads/main/docs/source/logos/vllm-omni-logo.png" width=55%>
-  </picture>
-</p>
-<h3 align="center">
-Easy, fast, and cheap omni-modality model serving for everyone
-</h3>
-
-<p align="center">
-| <a href="https://vllm-omni.readthedocs.io/en/latest/"><b>Documentation</b></a> | <a href="https://discuss.vllm.ai"><b>User Forum</b></a> | <a href="https://slack.vllm.ai"><b>Developer Slack</b></a> | <a href="docs/assets/WeChat.jpg"><b>WeChat</b></a> | <a href="https://arxiv.org/abs/2602.02204"><b>Paper</b></a> | <a href="https://docs.google.com/presentation/d/1qv4qMW1rKAqDREMXiUDLIgqqHQe7TDPj/edit?usp=sharing&ouid=110473603432222024453&rtpof=true&sd=true"><b>Slides</b></a> |
-</p>
-
-
----
-
-*Latest News* 🔥
-
-- [2026/02] We released [0.14.0](https://github.com/vllm-project/vllm-omni/releases/tag/v0.14.0) - This is the first **stable release** of vLLM-Omni that expands Omni’s diffusion / image-video generation and audio / TTS stack, improves distributed execution and memory efficiency, and broadens platform/backend coverage (GPU/ROCm/NPU/XPU). It also brings meaningful upgrades to serving APIs, profiling & benchmarking, and overall stability. Please check our latest [paper](https://arxiv.org/abs/2602.02204) for architecture design and performance results.
-- [2026/01] We released [0.12.0rc1](https://github.com/vllm-project/vllm-omni/releases/tag/v0.12.0rc1) - a major RC milestone focused on maturing the diffusion stack, strengthening OpenAI-compatible serving, expanding omni-model coverage, and improving stability across platforms (GPU/NPU/ROCm).
-- [2025/11] vLLM community officially released [vllm-project/vllm-omni](https://github.com/vllm-project/vllm-omni) in order to support omni-modality models serving.
-
----
-
-## About
-
-[vLLM](https://github.com/vllm-project/vllm) was originally designed to support large language models for text-based autoregressive generation tasks. vLLM-Omni is a framework that extends its support for omni-modality model inference and serving:
-
-- **Omni-modality**: Text, image, video, and audio data processing
-- **Non-autoregressive Architectures**: extend the AR support of vLLM to Diffusion Transformers (DiT) and other parallel generation models
-- **Heterogeneous outputs**: from traditional text generation to multimodal outputs
-
-<p align="center">
-  <picture>
-    <img alt="vllm-omni" src="https://raw.githubusercontent.com/vllm-project/vllm-omni/refs/heads/main/docs/source/architecture/omni-modality-model-architecture.png" width=55%>
-  </picture>
-</p>
-
-vLLM-Omni is fast with:
-
-- State-of-the-art AR support by leveraging efficient KV cache management from vLLM
-- Pipelined stage execution overlapping for high throughput performance
-- Fully disaggregation based on OmniConnector and dynamic resource allocation across stages
-
-vLLM-Omni is flexible and easy to use with:
-
-- Heterogeneous pipeline abstraction to manage complex model workflows
-- Seamless integration with popular Hugging Face models
-- Tensor, pipeline, data and expert parallelism support for distributed inference
-- Streaming outputs
-- OpenAI-compatible API server
-
-vLLM-Omni seamlessly supports most popular open-source models on HuggingFace, including:
-
-- Omni-modality models (e.g. Qwen-Omni)
-- Multi-modality generation models (e.g. Qwen-Image)
-
-## Getting Started
-
-Visit our [documentation](https://vllm-omni.readthedocs.io/en/latest/) to learn more.
-
-- [Installation](https://vllm-omni.readthedocs.io/en/latest/getting_started/installation/)
-- [Quickstart](https://vllm-omni.readthedocs.io/en/latest/getting_started/quickstart/)
-- [List of Supported Models](https://vllm-omni.readthedocs.io/en/latest/models/supported_models/)
-
-## Contributing
-
-We welcome and value any contributions and collaborations.
-Please check out [Contributing to vLLM-Omni](https://vllm-omni.readthedocs.io/en/latest/contributing/) for how to get involved.
-
-## Citation
-
-If you use vLLM-Omni for your research, please cite our [paper](https://arxiv.org/abs/2602.02204):
-
-```bibtex
-@article{yin2026vllmomni,
-  title={vLLM-Omni: Fully Disaggregated Serving for Any-to-Any Multimodal Models},
-  author={Peiqi Yin, Jiangyun Zhu, Han Gao, Chenguang Zheng, Yongxiang Huang, Taichang Zhou, Ruirui Yang, Weizhi Liu, Weiqing Chen, Canlin Guo, Didan Deng, Zifeng Mo, Cong Wang, James Cheng, Roger Wang, Hongsheng Liu},
-  journal={arXiv preprint arXiv:2602.02204},
-  year={2026}
-}
-```
-
-## Join the Community
-Feel free to ask questions, provide feedbacks and discuss with fellow users of vLLM-Omni in `#sig-omni` slack channel at [slack.vllm.ai](https://slack.vllm.ai) or vLLM user forum at [discuss.vllm.ai](https://discuss.vllm.ai).
-
-## Star History
-
-[![Star History Chart](https://api.star-history.com/svg?repos=vllm-project/vllm-omni&type=date&legend=top-left)](https://www.star-history.com/#vllm-project/vllm-omni&type=date&legend=top-left)
-
-## License
-
-Apache License 2.0, as found in the [LICENSE](./LICENSE) file.

apps/ComfyUI-vLLM-Omni/.gitignore

@@ -1,115 +0,0 @@
-# Byte-compiled / optimized / DLL files
-__pycache__/
-*.py[cod]
-*$py.class
-
-# OSX useful to ignore
-*.DS_Store
-.AppleDouble
-.LSOverride
-
-# Thumbnails
-._*
-
-# Files that might appear in the root of a volume
-.DocumentRevisions-V100
-.fseventsd
-.Spotlight-V100
-.TemporaryItems
-.Trashes
-.VolumeIcon.icns
-.com.apple.timemachine.donotpresent
-
-# Directories potentially created on remote AFP share
-.AppleDB
-.AppleDesktop
-Network Trash Folder
-Temporary Items
-.apdisk
-
-# C extensions
-*.so
-
-# Distribution / packaging
-.Python
-env/
-venv/
-build/
-develop-eggs/
-dist/
-downloads/
-eggs/
-.eggs/
-lib/
-lib64/
-parts/
-sdist/
-var/
-*.egg-info/
-.installed.cfg
-*.egg
-
-# PyInstaller
-#  Usually these files are written by a python script from a template
-#  before PyInstaller builds the exe, so as to inject date/other infos into it.
-*.manifest
-*.spec
-
-# Installer logs
-pip-log.txt
-pip-delete-this-directory.txt
-
-# Unit test / coverage reports
-htmlcov/
-.tox/
-.coverage
-.coverage.*
-.cache
-nosetests.xml
-coverage.xml
-*,cover
-.hypothesis/
-.pytest_cache/
-
-# Translations
-*.mo
-*.pot
-
-# Django stuff:
-*.log
-
-# Sphinx documentation
-docs/_build/
-
-# IntelliJ Idea
-.idea
-*.iml
-*.ipr
-*.iws
-
-# PyBuilder
-target/
-
-# Cookiecutter
-output/
-python_boilerplate/
-cookiecutter-pypackage-env/
-
-# vscode settings
-.history/
-*.code-workspace
-
-# Frontend extension
-node_modules/
-.env
-.env.local
-.env.development.local
-.env.test.local
-.env.production.local
-npm-debug.log*
-yarn-debug.log*
-yarn-error.log*
-node.zip
-.vscode/
-.claude/
-.codemate/

apps/ComfyUI-vLLM-Omni/LICENSE

@@ -1,15 +0,0 @@
-Apache Software License 2.0
-
-Copyright (c) 2026, Zeyu Huang
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.

apps/ComfyUI-vLLM-Omni/README.md

@@ -1,170 +0,0 @@
-# vLLM-Omni
-
-vLLM-Omni offers a ComfyUI integration on top of its online serving API.
-It can send model inference requests to either a locally running vLLM-Omni service or a remote one.
-
-## Requirement
-
-- Python 3.12 or above
-- [ComfyUI installed](https://docs.comfy.org/installation/system_requirements)
-- [vLLM-Omni installed](https://docs.vllm.ai/projects/vllm-omni/en/latest/getting_started/installation/) on either the same device or another device discoverable via the internet.
-- No need to install additional packages apart from those already required by ComfyUI.
-
-> [!TIP]
-> If you run both ComfyUI and vLLM-Omni on the same device, you can create separate virtual environments and use different Python versions for them.
-
-
-## Installation
-
-Copy this folder to the `custom_nodes` subfolder of your ComfyUI installation. Your directory should look like `ComfyUI/custom_nodes/ComfyUI-vLLM-Omni`.
-
-If you are running ComfyUI during copying, you should restart ComfyUI to load this extension.
-
-> [!TIP]
-> You can use utility websites such as https://download-directory.github.io/ to download a subdirectory of a repo. Also checkout community discussions (e.g., https://stackoverflow.com/questions/7106012/download-a-single-folder-or-directory-from-a-github-repository) for more info.
-
-On the device and virtual environment you run ComfyUI, launch ComfyUI with
-```bash
-cd ComfyUI
-
-# The regular way
-python main.py
-
-# If you are mainly using this node, launch it faster with
-python main.py --cpu
-```
-
-On the device and virtual environment you run vLLM-Omni, start a model service with
-```bash
-vllm serve The_Model_ID_to_Serve --omni --port 8000
-```
-
-Check **ComfyUI's sidebar -> Node Library**. There should be a new folder named **vLLM-Omni**.
-If no, check your shell running the ComfyUI process. There may be some error messages before the line `Import times for custom nodes:` and the line `To see the GUI go to: http://127.0.0.1:8188`.
-
-## Quickstart
-
-This extension offers the following nodes based on the output modalities (at **ComfyUI sidebar -> Node Library**):
-
-- **Generate Image** for text-to-image and image-to-image tasks
-- **Multimodality Comprehension** for multimodality-to-text and multimodality-to-audio tasks
-- **TTS** and **TTS Voice Clone** for TTS tasks
-
-This extension also offers example workflows (at **ComfyUI sidebar -> Templates -> vLLM-Omni**)
-
-> [!INFO]
-> The node UI and feature designs are intended to match vLLM-Omni online serving interfaces. It cannot offer more than what the interfaces support.
-
-To build a simple workflow yourself,
-
-- Drag a generation node onto the canvas.
-- Depending on your need, grab built-in multimedia file loader nodes, such as **image->Load Image**, **image->video->Load Video**, **audio->Load Audio**
-- Depending on your need, grab built-in multimedia file preview nodes, such as **image->Preview Image**, **image->video->Save Video**, **audio->Preview Audio**. For text output, you can install [ComfyUI-Custom-Scripts plugin](https://github.com/pythongosssss/ComfyUI-Custom-Scripts/) and grab its **utils->Show Text 🐍** node.
-- If you want to tune sampling parameters, grab corresponding nodes from **vLLM-Omni-> Sampling Params**.
-    - For multi-stage models, you can connect multiple **AR Sampling Params** and **Diffusion Sampling Params** nodes to a **Multi-Stage Sampling Params List** node, and connect this node to the generation node.
-    - For some multi-stage models like BAGEL, [only one stage's sampling parameters are exposed and tunable via vLLM-Omni's online serving API](https://docs.vllm.ai/projects/vllm-omni/en/latest/user_guide/examples/online_serving/bagel/). Thus, these models are treated as single-stage ones. Please check the vLLM-Omni documentation on how to correctly set each model's sampling parameters.
-    - For multi-stage models where all stages are either autoregression or diffusion, you can also connect only a single Sampling Params node, indicating that this set of sampling parameters will be used for all stages.
-
-**The following features are tested**:
-
-- Single-node workflows for
-    - Multimodal Comprehension (e.g., Qwen Omni, BAGEL)
-    - Text-to-Image Generation (e.g., Qwen-Image)
-    - Image-to-Image Generation (e.g., Qwen-Image-Edit)
-    - TTS (e.g., Qwen TTS, including VoiceDesign, VoiceClone, CustomVoice)
-
-**The following features are not currently tested**. They may work or break. You are welcomed to test it out and offer comments.
-
-- Multi-node workflow that connects multiple model services together.
-
-## Screenshots and Examples
-
-### Multimodal comprehension (e.g., Qwen Omni series, BAGEL)
-
-(Also available at **ComfyUI sidebar->Template->vLLM-Omni->vLLM-Omni Annotated Example**)
-
-<p align="center">
-  <picture>
-    <source media="(prefers-color-scheme: dark)" src="https://raw.githubusercontent.com/vllm-project/vllm-omni/refs/heads/main/apps/ComfyUI-vLLM-Omni/docs/images/comfyui-comprehension.jpg">
-    <img alt="vLLM-Omni Main Architecture" src="https://raw.githubusercontent.com/vllm-project/vllm-omni/refs/heads/main/apps/ComfyUI-vLLM-Omni/docs/images/comfyui-comprehension.jpg" width=55%>
-  </picture>
-</p>
-
-> [!TIP]
-> Although this node enables all-modality input, you should check whether the specific model you host and request for supports the modalities you connect to the node.
-
-You can configure per-stage sampling parameters for multi-stage models.
-
-<p align="center">
-  <picture>
-    <source media="(prefers-color-scheme: dark)" src="https://raw.githubusercontent.com/vllm-project/vllm-omni/refs/heads/main/apps/ComfyUI-vLLM-Omni/docs/images/comfyui-multi-stage.jpg">
-    <img alt="vLLM-Omni Main Architecture" src="https://raw.githubusercontent.com/vllm-project/vllm-omni/refs/heads/main/apps/ComfyUI-vLLM-Omni/docs/images/comfyui-multi-stage.jpg" width=55%>
-  </picture>
-</p>
-
-### Text-to-image and image-to-image generation (e.g., Z-Image-Turbo, Qwen-Image-Edit, BAGEL)
-
-(Also available at **ComfyUI sidebar->Template->vLLM-Omni->vLLM-Omni Image Generation**)
-
-<p align="center">
-  <picture>
-    <source media="(prefers-color-scheme: dark)" src="https://raw.githubusercontent.com/vllm-project/vllm-omni/refs/heads/main/apps/ComfyUI-vLLM-Omni/docs/images/comfyui-image-generation.jpg">
-    <img alt="vLLM-Omni Main Architecture" src="https://raw.githubusercontent.com/vllm-project/vllm-omni/refs/heads/main/apps/ComfyUI-vLLM-Omni/docs/images/comfyui-image-generation.jpg" width=55%>
-  </picture>
-</p>
-
-> [!TIP]
-> The node automatically choose text-to-image or image-to-image API endpoints depending on whether you connect an image input or not.
-
-### TTS (e.g., Qwen TTS series)
-
-(Also available at **ComfyUI sidebar->Template->vLLM-Omni->vLLM-Omni TTS**)
-
-<p align="center">
-  <picture>
-    <source media="(prefers-color-scheme: dark)" src="https://raw.githubusercontent.com/vllm-project/vllm-omni/refs/heads/main/apps/ComfyUI-vLLM-Omni/docs/images/comfyui-tts.jpg">
-    <img alt="vLLM-Omni Main Architecture" src="https://raw.githubusercontent.com/vllm-project/vllm-omni/refs/heads/main/apps/ComfyUI-vLLM-Omni/docs/images/comfyui-tts.jpg" width=55%>
-  </picture>
-</p>
-
-> [!TIP]
-> There is a dedicated node for VoiceClone tasks with reference audio input. Other simple text-to-speech tasks should use the regular TTS node.
-
-## Develop
-
-Follow the [development convention and rules of vLLM-Omni](https://docs.vllm.ai/projects/vllm-omni/en/latest/contributing/).
-
-## Limitation and Non-Goals
-
-- Single server mode only. No automatic load balancing or failover.
-- Features set is bounded to vLLM-Omni's online service capability, including
-    - The types of models supported in online mode,
-    - The types of sampling parameters supported in the online mode,
-    - The ways to send files (primarily through full-length base64 in JSON payload),
-    - Figuring out errors in the payload (such as unsupported fields by a specific model) if the endpoint does not explicitly return an error,
-    - (The lack of) Authentication
-    - (The lack of) Progress indicator
-
-## Support
-
-If you are new to ComfyUI, please check out [its documentation](https://docs.comfy.org/) for usage instructions.
-
-If you are new to vLLM-Omni, please also check out [its documentation](https://docs.vllm.ai/projects/vllm-omni/en/latest/) for usage instructions.
-
-Whenever you find an issue or problem, please
-
-- First find out if this is an upstream limitation of vLLM-Omni's online serving mode, by [checking their documentation](https://docs.vllm.ai/projects/vllm-omni/en/latest/examples/).
-- [Open an issue](https://github.com/vllm-project/vllm-omni/issues) that clearly describes this ComfyUI or online service problem.
-
-## Acknowledgements
-
-Features
-
-- https://github.com/dougbtv/comfyui-vllm-omni/ The official reference implementation for ComfyUI integration with vLLM-Omni's DALL-E compatible image generation API.
-- https://github.com/Comfy-Org/ComfyUI/tree/master/comfy_extras ComfyUI's built-in node implementations.
-
-UI/UX design references
-
-- https://github.com/sgl-project/sglang/pull/15271 SGLang Diffusion's official ComfyUI integration for image and video generation.
-- https://github.com/SXQBW/ComfyUI-Qwen-Omni A third party ComfyUI integration for Qwen Omni series.
-- https://github.com/flybirdxx/ComfyUI-Qwen-TTS https://github.com/DarioFT/ComfyUI-Qwen3-TTS Third  party ComfyUI integrations for Qwen TTS series.

apps/ComfyUI-vLLM-Omni/__init__.py

@@ -1,52 +0,0 @@
-"""Top-level package for comfyui_vllm_omni."""  # noqa: N999  # This is not a python library intended to be imported
-
-__all__ = [
-    "NODE_CLASS_MAPPINGS",
-    "NODE_DISPLAY_NAME_MAPPINGS",
-    "WEB_DIRECTORY",
-]
-
-__author__ = """Zeyu Huang"""
-__email__ = "11222265+fhfuih@users.noreply.github.com"
-__version__ = "0.0.1"
-
-from .comfyui_vllm_omni.nodes import (
-    VLLMOmniARSampling,
-    VLLMOmniComprehension,
-    VLLMOmniDiffusionSampling,
-    VLLMOmniGenerateImage,
-    VLLMOmniQwenTTSParams,
-    VLLMOmniSamplingParamsList,
-    VLLMOmniTTS,
-    VLLMOmniVoiceClone,
-)
-
-# A dictionary that contains all nodes you want to export with their names
-NODE_CLASS_MAPPINGS = {
-    # === Generation ===
-    "VLLMOmniGenerateImage": VLLMOmniGenerateImage,
-    "VLLMOmniComprehension": VLLMOmniComprehension,
-    "VLLMOmniTTS": VLLMOmniTTS,
-    "VLLMOmniVoiceClone": VLLMOmniVoiceClone,
-    # === Params ===
-    "VLLMOmniARSampling": VLLMOmniARSampling,
-    "VLLMOmniDiffusionSampling": VLLMOmniDiffusionSampling,
-    "VLLMOmniSamplingParamsList": VLLMOmniSamplingParamsList,
-    "VLLMOmniQwenTTSParams": VLLMOmniQwenTTSParams,
-}
-
-# A dictionary that contains the friendly/humanly readable titles for the nodes
-NODE_DISPLAY_NAME_MAPPINGS = {
-    # === Generation ===
-    "VLLMOmniGenerateImage": "Generate Image",
-    "VLLMOmniComprehension": "Multimodality Comprehension",
-    "VLLMOmniTTS": "TTS (Text to Speech)",
-    "VLLMOmniVoiceClone": "TTS Voice Cloning",
-    # === Params ===
-    "VLLMOmniARSampling": "AR Sampling Params",
-    "VLLMOmniDiffusionSampling": "Diffusion Sampling Params",
-    "VLLMOmniSamplingParamsList": "Multi-Stage Sampling Params List",
-    "VLLMOmniQwenTTSParams": "Qwen TTS Params",
-}
-
-WEB_DIRECTORY = "./web"

apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/__init__.py

@@ -1 +0,0 @@
-# noqa: N999   # This is not a python library intended to be imported

apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/nodes.py

@@ -1,569 +0,0 @@
-from typing import Literal, cast
-
-import torch
-from comfy_api.input import AudioInput, VideoInput
-
-from .utils.api_client import VLLMOmniClient
-from .utils.logger import get_logger
-from .utils.models import lookup_model_spec
-from .utils.types import AudioFormat
-from .utils.validators import (
-    add_sampling_parameters_to_stage,
-    validate_model_and_sampling_params_types,
-)
-
-logger = get_logger(__name__)
-
-
-class _VLLMOmniGenerateBase:
-    """Base class for vLLM-Omni generation nodes with shared functionality."""
-
-    CATEGORY = "vLLM-Omni"
-
-    @classmethod
-    def VALIDATE_INPUTS(cls, url, model) -> str | Literal[True]:
-        """
-        Can only validate this model's own input. Cannot check inputs from other nodes.
-        See: https://docs.comfy.org/custom-nodes/backend/server_overview#validate_inputs
-        """
-        if not url:
-            return "URL must not be empty"
-        if not model:
-            return "Model must not be empty"
-        return True
-
-
-class VLLMOmniGenerateImage(_VLLMOmniGenerateBase):
-    @classmethod
-    def INPUT_TYPES(cls):
-        return {
-            "required": {
-                "url": ("STRING", {"default": "http://localhost:8000/v1"}),
-                "model": ("STRING", {"default": "Tongyi-MAI/Z-Image-Turbo"}),
-                "prompt": ("STRING", {"multiline": True}),
-                "negative_prompt": ("STRING", {"multiline": True, "default": ""}),
-                "width": ("INT", {"default": 512, "min": 64, "max": 2048}),
-                "height": ("INT", {"default": 512, "min": 64, "max": 2048}),
-            },
-            "optional": {
-                "image": ("IMAGE",),
-                "mask": ("MASK",),
-                # "video": ("VIDEO",),
-                # "audio": ("AUDIO",),
-                "sampling_params": ("SAMPLING_PARAMS",),
-            },
-        }
-
-    RETURN_TYPES = ("IMAGE",)
-    RETURN_NAMES = ("image",)
-    FUNCTION = "generate"
-
-    async def generate(
-        self,
-        url: str,
-        model: str,
-        prompt: str,
-        width: int,
-        height: int,
-        negative_prompt: str | None = None,
-        image: torch.Tensor | None = None,
-        mask: torch.Tensor | None = None,
-        audio: AudioInput | None = None,  # Hidden & unused
-        video: VideoInput | None = None,  # Hidden & unused
-        sampling_params: dict | list[dict] | None = None,
-        **kwargs,
-    ):
-        logger.info("Uncaught kwargs: %s", kwargs)
-        logger.debug("Got sampling params: %s", sampling_params)
-        validate_model_and_sampling_params_types(model, sampling_params)
-        if image is None and mask is not None:
-            raise ValueError("Mask input provided without an image input.")
-
-        client = VLLMOmniClient(url)
-
-        spec, pattern = lookup_model_spec(model)
-        is_bagel = pattern is not None and "bagel" in pattern.lower()
-
-        # Prefer DALL-E compatible API for simple (one-stage) diffusion models
-        if (spec is None or spec["stages"] == ["diffusion"]) and not is_bagel:
-            sampling_params = cast(dict | None, sampling_params)
-            if audio is None and image is None and video is None:
-                # No multimodal input --- use DALL-E image generation
-                logger.info("Using DALL-E image generation endpoint")
-                output = await client.generate_image(
-                    model=model,
-                    prompt=prompt,
-                    width=width,
-                    height=height,
-                    negative_prompt=negative_prompt,
-                    sampling_params=sampling_params,
-                )
-                return (output,)
-            elif image is not None and audio is None and video is None:
-                # Image and text input --- use DALL-E image edit
-                logger.info("Using DALL-E image edit endpoint")
-                output = await client.edit_image(
-                    model=model,
-                    prompt=prompt,
-                    image=image,
-                    width=width,
-                    height=height,
-                    negative_prompt=negative_prompt,
-                    mask=mask,
-                    sampling_params=sampling_params,
-                )
-                return (output,)
-
-        logger.info("Using chat completion endpoint")
-        sampling_params = add_sampling_parameters_to_stage(
-            model, sampling_params, "diffusion", width=width, height=height
-        )
-        logger.debug("Edited sampling params: %s", sampling_params)
-
-        output = await client.generate_image_chat_completion(
-            model=model,
-            prompt=prompt,
-            negative_prompt=negative_prompt,
-            image=image,
-            audio=audio,
-            video=video,
-            sampling_params=sampling_params,
-        )
-
-        return (output,)
-
-
-class VLLMOmniComprehension(_VLLMOmniGenerateBase):
-    @classmethod
-    def INPUT_TYPES(cls):
-        return {
-            "required": {
-                "url": ("STRING", {"default": "http://localhost:8000/v1"}),
-                "model": ("STRING", {"default": "Qwen/Qwen2.5-Omni-7B"}),
-                "prompt": ("STRING", {"multiline": True}),
-                "output_text": ("BOOLEAN", {"default": True}),
-                "output_audio": ("BOOLEAN", {"default": True}),
-                "use_audio_in_video": ("BOOLEAN", {"default": True}),
-            },
-            "optional": {
-                "image": ("IMAGE",),
-                "video": ("VIDEO",),
-                "audio": ("AUDIO",),
-                "sampling_params": ("SAMPLING_PARAMS",),
-            },
-        }
-
-    RETURN_TYPES = ("STRING", "AUDIO")
-    RETURN_NAMES = ("text_response", "audio_response")
-    FUNCTION = "generate"
-
-    @classmethod
-    def VALIDATE_INPUTS(cls, url, model, output_text, output_audio) -> str | Literal[True]:  # type: ignore[reportIncompatibleMethodOverride]
-        super_validation = super().VALIDATE_INPUTS(url, model)
-        if isinstance(super_validation, str):
-            return super_validation
-        if not output_text and not output_audio:
-            return "At least one of output_text or output_audio must be True."
-        return True
-
-    async def generate(
-        self,
-        url: str,
-        model: str,
-        prompt: str,
-        image: torch.Tensor | None = None,
-        audio: AudioInput | None = None,
-        video: VideoInput | None = None,
-        sampling_params: dict | list[dict] | None = None,
-        output_text: bool = True,
-        output_audio: bool = True,
-        use_audio_in_video: bool = True,
-        **kwargs,
-    ) -> tuple[str, AudioInput]:
-        logger.info("Uncaught kwargs: %s", kwargs)
-        logger.debug("Got sampling params: %s", sampling_params)
-        validate_model_and_sampling_params_types(model, sampling_params)
-
-        client = VLLMOmniClient(url)
-        spec, pattern = lookup_model_spec(model)
-        is_bagel = pattern is not None and "bagel" in pattern.lower()
-
-        if is_bagel:
-            # A lot of special handlings here...
-            if output_audio:
-                raise ValueError("BAGEL models do not support audio output.")
-            if audio is not None or video is not None:
-                raise ValueError("BAGEL models do not support audio or video input.")
-            (
-                text_response,
-                _,
-            ) = await client.generate_comprehension_chat_completion(
-                model=model,
-                prompt=prompt,
-                image=image,
-                audio=None,
-                video=None,
-                sampling_params=sampling_params,
-                modalities=["text"],
-            )
-        else:
-            modalities = []
-            if output_text:
-                modalities.append("text")
-            if output_audio:
-                modalities.append("audio")
-
-            if use_audio_in_video and video is not None:
-                use_audio_in_video = True
-            else:
-                use_audio_in_video = False
-
-            (
-                text_response,
-                audio,
-            ) = await client.generate_comprehension_chat_completion(
-                model=model,
-                prompt=prompt,
-                image=image,
-                audio=audio,
-                video=video,
-                sampling_params=sampling_params,
-                modalities=modalities,
-                # == extra kwargs ==
-                mm_processor_kwargs={"use_audio_in_video": use_audio_in_video},
-            )
-
-        if text_response is None:
-            text_response = ""
-        if audio is None:
-            channels = 1
-            duration = 1
-            sample_rate = 44100
-            num_samples = int(round(duration * sample_rate))
-            waveform = torch.zeros((1, channels, num_samples), dtype=torch.float32)
-            audio = {"waveform": waveform, "sample_rate": sample_rate}
-
-        return (text_response, audio)
-
-
-class VLLMOmniTTS(_VLLMOmniGenerateBase):
-    @classmethod
-    def INPUT_TYPES(cls):
-        return {
-            "required": {
-                "url": ("STRING", {"default": "http://localhost:8000/v1"}),
-                "model": (
-                    "STRING",
-                    {"default": "Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice"},
-                ),
-                "input": ("STRING", {"multiline": True}),
-                "voice": ("STRING", {"default": "Vivian"}),
-                "response_format": (["mp3", "opus", "aac", "flac", "wav", "pcm"],),
-                "speed": (
-                    "FLOAT",
-                    {"default": 1.0, "min": 0.25, "max": 4.0, "step": 0.01},
-                ),
-            },
-            "optional": {
-                "model_specific_params": ("TTS_PARAMS",),
-            },
-        }
-
-    RETURN_TYPES = ("AUDIO",)
-    RETURN_NAMES = ("audio",)
-    FUNCTION = "generate"
-
-    async def generate(
-        self,
-        url: str,
-        model: str,
-        input: str,
-        voice: str,
-        response_format: AudioFormat,
-        speed: float,
-        model_specific_params: dict | None,
-        **kwargs,
-    ) -> tuple[AudioInput]:
-        logger.info("Got extra kwargs in TTS: %s", kwargs)
-
-        is_qwen_tts = "qwen3-tts" in model.lower()
-        extra_params_type = None if model_specific_params is None else model_specific_params["type"]
-        if not is_qwen_tts and extra_params_type == "qwen-tts":
-            raise ValueError(
-                "You have provided Qwen-specific TTS params."
-                "However, the model appears to not be a Qwen TTS model (no 'Qwen3-TTS' in model name)."
-            )
-
-        combined_params = {**kwargs, **(model_specific_params or {})}
-        combined_params.pop("type", None)  # Internal fields in model_specific_params
-
-        client = VLLMOmniClient(url)
-
-        audio = await client.generate_speech(
-            model=model,
-            input=input,
-            voice=voice,
-            response_format=response_format,
-            speed=speed,
-            **combined_params,
-        )
-        return (audio,)
-
-
-class VLLMOmniVoiceClone(_VLLMOmniGenerateBase):
-    @classmethod
-    def INPUT_TYPES(cls):
-        return {
-            "required": {
-                "url": ("STRING", {"default": "http://localhost:8000/v1"}),
-                "model": ("STRING", {"default": "Qwen/Qwen3-TTS-12Hz-1.7B-Base"}),
-                "input": ("STRING", {"multiline": True}),
-                "voice": ("STRING", {"default": "Vivian"}),
-                "response_format": (["mp3", "opus", "aac", "flac", "wav", "pcm"],),
-                "speed": (
-                    "FLOAT",
-                    {"default": 1.0, "min": 0.25, "max": 4.0, "step": 0.01},
-                ),
-                "ref_audio": ("AUDIO",),
-                "ref_text": ("STRING", {"multiline": True}),
-                "x_vector_only_mode": ("BOOLEAN", {"default": False}),
-            },
-            "optional": {
-                "model_specific_params": ("TTS_PARAMS",),
-            },
-        }
-
-    RETURN_TYPES = ("AUDIO",)
-    RETURN_NAMES = ("audio",)
-    FUNCTION = "generate"
-
-    async def generate(
-        self,
-        url: str,
-        model: str,
-        input: str,
-        voice: str,
-        response_format: AudioFormat,
-        speed: float,
-        ref_audio: AudioInput,
-        ref_text: str,
-        x_vector_only_mode: bool,
-        model_specific_params: dict | None,
-        **kwargs,
-    ):
-        is_qwen_tts = "qwen3-tts" in model.lower()
-        extra_params_type = None if model_specific_params is None else model_specific_params["type"]
-        if not is_qwen_tts and extra_params_type == "qwen-tts":
-            raise ValueError(
-                "You have provided Qwen-specific TTS params."
-                "However, the model appears to not be a Qwen TTS model (no 'Qwen3-TTS' in model name)."
-            )
-
-        combined_params = {
-            "ref_audio": ref_audio,
-            "ref_text": ref_text,
-            "x_vector_only_mode": x_vector_only_mode,
-            **kwargs,
-            **(model_specific_params or {}),
-        }
-        combined_params.pop("type", None)  # Internal fields in model_specific_params
-
-        client = VLLMOmniClient(url)
-
-        audio = await client.generate_speech(
-            model=model,
-            input=input,
-            voice=voice,
-            response_format=response_format,
-            speed=speed,
-            **combined_params,
-        )
-        return (audio,)
-
-
-class VLLMOmniARSampling:
-    @classmethod
-    def INPUT_TYPES(cls):
-        return {
-            "required": {
-                "max_tokens": ("INT", {"default": 100, "min": 1, "max": 10000}),
-                "temperature": (
-                    "FLOAT",
-                    {"default": 1.0, "min": 0.0, "max": 2.0, "step": 0.01},
-                ),
-                "top_p": (
-                    "FLOAT",
-                    {"default": 1.0, "min": 0.0, "max": 1.0, "step": 0.01},
-                ),
-                "repetition_penalty": (
-                    "FLOAT",
-                    {"default": 1.0, "min": 0.0, "max": 5.0, "step": 0.01},
-                ),
-                # === Put seed at last. ===
-                # Whenever a field named "seed" is present, ComfyUI adds another field called "control after generate"
-                "seed": (
-                    "INT",
-                    {
-                        "default": -1,
-                        "min": -1,
-                        "step": 1,
-                        "tooltip": "-1 means to not provide a seed.",
-                    },
-                ),
-            }
-        }
-
-    RETURN_TYPES = ("SAMPLING_PARAMS",)
-    RETURN_NAMES = ("AR sampling params",)
-    FUNCTION = "get_params"
-    CATEGORY = "vLLM-Omni/Sampling Params"
-
-    def get_params(self, seed, **kwargs):
-        params = {
-            "type": "autoregression",  # for internal use, removed before sending the request
-            **kwargs,
-        }
-        if seed >= 0:
-            params["seed"] = seed
-        return (params,)
-
-
-class VLLMOmniDiffusionSampling:
-    @classmethod
-    def INPUT_TYPES(cls):
-        return {
-            "required": {
-                "n": (
-                    "INT",
-                    {
-                        "default": 1,
-                        "min": 0,
-                        "max": 10,
-                        "step": 1,
-                        "tooltip": "Number of images to generate",
-                    },
-                ),
-                "num_inference_steps": (
-                    "INT",
-                    {
-                        "default": 50,
-                        "min": 1,
-                        "max": 1000,
-                        "tooltip": "Number of denoising steps (higher = better quality, slower).",
-                    },
-                ),
-                "guidance_scale": (
-                    "FLOAT",
-                    {
-                        "default": 7.5,
-                        "min": 0.0,
-                        "max": 20.0,
-                        "step": 0.1,
-                        "tooltip": "Classifier-free guidance scale (higher = more prompt adherence).",
-                    },
-                ),
-                "true_cfg_scale": (
-                    "FLOAT",
-                    {
-                        "default": 1.0,
-                        "min": 0.0,
-                        "max": 20.0,
-                        "step": 0.5,
-                        "tooltip": "True CFG scale for advanced control (model-specific).",
-                    },
-                ),
-                "vae_use_slicing": (
-                    "BOOLEAN",
-                    {
-                        "default": False,
-                        "tooltip": "Enable VAE slicing for reduced memory usage (slight quality trade-off)",
-                    },
-                ),
-                # === Put seed at last. ===
-                # Whenever a field named "seed" is present, ComfyUI adds another field called "control after generate"
-                "seed": (
-                    "INT",
-                    {
-                        "default": -1,
-                        "min": -1,
-                        "step": 1,
-                        "tooltip": "-1 means to not provide a seed.",
-                    },
-                ),
-            }
-        }
-
-    RETURN_TYPES = ("SAMPLING_PARAMS",)
-    RETURN_NAMES = ("diffusion sampling params",)
-    FUNCTION = "get_params"
-    CATEGORY = "vLLM-Omni/Sampling Params"
-
-    def get_params(self, seed, **kwargs):
-        params = {
-            "type": "diffusion",  # for internal use, removed before sending the request
-            **kwargs,
-        }
-        if seed >= 0:
-            params["seed"] = seed
-        return (params,)
-
-
-class VLLMOmniSamplingParamsList:
-    @classmethod
-    def INPUT_TYPES(cls):
-        return {
-            "required": {
-                "param1": ("SAMPLING_PARAMS",),
-            },
-            "optional": {
-                "param2": ("SAMPLING_PARAMS",),
-                "param3": ("SAMPLING_PARAMS",),
-            },
-        }
-
-    RETURN_TYPES = ("SAMPLING_PARAMS",)
-    RETURN_NAMES = ("param list",)
-    FUNCTION = "aggregate"
-    CATEGORY = "vLLM-Omni/Sampling Params"
-
-    def aggregate(self, param1: dict, param2: dict | None = None, param3: dict | None = None):
-        for i, p in enumerate((param1, param2, param3)):
-            if isinstance(p, list):
-                raise ValueError(
-                    f"Input {i} is a Multi-Stage Sampling Params List. "
-                    f"Expected a single sampling parameters node (either AR or Diffusion)."
-                )
-
-        params = [param1]
-        if param2 is not None:
-            params.append(param2)
-        if param3 is not None:
-            params.append(param3)
-        return (params,)
-
-
-class VLLMOmniQwenTTSParams:
-    @classmethod
-    def INPUT_TYPES(cls):
-        return {
-            "required": {
-                "task_type": (
-                    ["CustomVoice", "VoiceDesign", "Base"],
-                    {"default": "CustomVoice"},
-                ),
-                "language": (
-                    ["Auto", "Chinese", "English", "Japanese", "Korean"],
-                    {"default": "Auto"},
-                ),
-                "instructions": ("STRING", {"multiline": True}),
-                "max_new_tokens": ("INT", {"default": 2048, "min": 1}),
-            }
-        }
-
-    RETURN_TYPES = ("TTS_PARAMS",)
-    RETURN_NAMES = ("Qwen TTS params",)
-    FUNCTION = "get_params"
-    CATEGORY = "vLLM-Omni/TTS Params"
-
-    def get_params(self, **kwargs):
-        return ({"type": "qwen-tts", **kwargs},)

apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/api_client.py

@@ -1,473 +0,0 @@
-"""
-An high-level API client adapter that forwards ComfyUI inputs to vLLM-Omni's REST API,
-and transforms the API responses back to ComfyUI formats.
-
-The image generation part is derived from dougbtv/comfyui-vllm-omni by Doug (@dougbtv).
-Original source at https://github.com/dougbtv/comfyui-vllm-omni, distributed under the MIT License.
-"""
-
-from typing import Any
-
-import aiohttp
-import av.error
-import torch
-from comfy_api.input import AudioInput, VideoInput
-
-from .format import (
-    audio_to_base64,
-    base64_to_audio,
-    base64_to_image_tensor,
-    bytes_to_audio,
-    image_tensor_to_base64,
-    image_tensor_to_png_bytes,
-    video_to_base64,
-)
-from .logger import get_logger, pretty_printer
-from .models import lookup_model_spec
-from .types import AudioFormat
-
-logger = get_logger(__name__)
-
-
-class VLLMOmniClient:
-    def __init__(self, base_url: str, timeout: float = 300.0):
-        self.base_url = base_url
-        self.timeout = aiohttp.ClientTimeout(total=timeout)
-
-    async def generate_image(
-        self,
-        *,
-        model: str,
-        prompt: str,
-        width: int,
-        height: int,
-        negative_prompt: str | None = None,
-        sampling_params: dict | None = None,
-    ) -> torch.Tensor:
-        """Run text-to-image generation via DALLE API"""
-        await self._check_model_exist(model)
-
-        size = f"{width}x{height}"
-        payload = {
-            "model": model,
-            "prompt": prompt,
-            "size": size,
-            "response_format": "b64_json",
-        }
-        if negative_prompt:
-            payload["negative_prompt"] = negative_prompt
-        if sampling_params is not None:
-            # Only select specific sampling params
-            for k in (
-                "n",
-                "num_inference_steps",
-                "guidance_scale",
-                "true_cfg_scale",
-                "vae_use_slicing",
-            ):
-                if k in sampling_params and sampling_params[k] is not None:
-                    payload[k] = sampling_params[k]
-        logger.debug("img gen payload: %s", payload)
-
-        url = self.base_url + "/images/generations"
-        async with aiohttp.ClientSession(timeout=self.timeout) as session:
-            try:
-                async with session.post(
-                    url,
-                    json=payload,
-                    headers={"Content-Type": "application/json"},
-                ) as response:
-                    if not response.ok:
-                        error_text = await response.text()
-                        raise (ValueError if response.status < 500 else RuntimeError)(
-                            f"vLLM-Omni API returned status {response.status}: {error_text}"
-                        )
-
-                    try:
-                        data = await response.json()
-                    except aiohttp.ContentTypeError as e:
-                        raise RuntimeError(f"Invalid JSON response from vLLM-Omni: {e}")
-                    if "data" not in data:
-                        raise RuntimeError("API response missing 'data' field - expected OpenAI DALL-E format")
-                    if not data["data"]:
-                        raise RuntimeError("API returned empty data array")
-
-                    image_tensors = []
-                    for idx, img in enumerate(data["data"]):
-                        if "b64_json" not in img:
-                            raise RuntimeError(f"API returned image #{idx} without 'b64_json' field")
-                        base64_str = img["b64_json"]
-                        tensor = base64_to_image_tensor(base64_str)
-                        image_tensors.append(tensor)
-                        logger.debug("Image #%d has shape %s", idx, tensor.shape)
-
-                    batch_tensor = torch.stack(image_tensors, dim=0)
-                    logger.debug("batch_tensor output has shape: %s", batch_tensor.shape)
-                    return batch_tensor
-
-            except aiohttp.ClientError as e:
-                raise RuntimeError(f"Network error connecting to vLLM-Omni at {url}: {e}")
-
-    async def edit_image(
-        self,
-        *,
-        model: str,
-        prompt: str,
-        image: torch.Tensor,
-        width: int,
-        height: int,
-        negative_prompt: str | None = None,
-        mask: torch.Tensor | None = None,
-        sampling_params: dict | None = None,
-    ) -> torch.Tensor:
-        """Run image editing via DALLE API"""
-        await self._check_model_exist(model)
-
-        size = f"{width}x{height}"
-        image_filename = "image.png"  # Required for multipart form
-        form = aiohttp.FormData()
-        form.add_field("model", model)
-        form.add_field(
-            "image",
-            image_tensor_to_png_bytes(image, image_filename),
-            filename=image_filename,
-            content_type="image/png",
-        )
-        form.add_field("prompt", prompt)
-        form.add_field("size", size)
-        if negative_prompt:
-            form.add_field("negative_prompt", negative_prompt)
-        if sampling_params is not None:
-            # Only select specific sampling params
-            for k in ("n", "num_inference_steps", "guidance_scale", "true_cfg_scale"):
-                if k in sampling_params and sampling_params[k] is not None:
-                    form.add_field(k, str(sampling_params[k]))
-        if mask is not None:
-            mask_filename = "mask.png"
-            form.add_field(
-                "mask",
-                image_tensor_to_png_bytes(mask, mask_filename),
-                filename=mask_filename,
-                content_type="image/png",
-            )
-
-        url = self.base_url + "/images/edits"
-        async with aiohttp.ClientSession(timeout=self.timeout) as session:
-            try:
-                async with session.post(url, data=form) as response:
-                    if not response.ok:
-                        error_text = await response.text()
-                        raise (ValueError if response.status < 500 else RuntimeError)(
-                            f"vLLM-Omni API returned status {response.status}: {error_text}"
-                        )
-
-                    try:
-                        data = await response.json()
-                    except aiohttp.ContentTypeError as e:
-                        raise RuntimeError(f"Invalid JSON response from vLLM-Omni: {e}")
-
-                    if "data" not in data:
-                        raise RuntimeError("API response missing 'data' field - expected OpenAI DALL-E format")
-                    if not data["data"]:
-                        raise RuntimeError("API returned empty data array")
-
-                    image_tensors = []
-                    for idx, img in enumerate(data["data"]):
-                        if "b64_json" not in img:
-                            raise RuntimeError(f"API returned image #{idx} without 'b64_json' field")
-                        base64_str = img["b64_json"]
-                        tensor = base64_to_image_tensor(base64_str)
-                        image_tensors.append(tensor)
-
-                    return torch.stack(image_tensors, dim=0)
-
-            except aiohttp.ClientError as e:
-                raise RuntimeError(f"Network error connecting to vLLM-Omni at {url}: {e}")
-
-    async def generate_image_chat_completion(
-        self,
-        *,
-        model: str,
-        prompt: str,
-        negative_prompt: str | None = None,
-        image: torch.Tensor | None = None,
-        audio: AudioInput | None = None,
-        video: VideoInput | None = None,
-        sampling_params: dict | list[dict] | None = None,
-    ) -> torch.Tensor:
-        payload = VLLMOmniClient._prepare_chat_completion_messages(
-            model=model,
-            prompt=prompt,
-            negative_prompt=negative_prompt,
-            image=image,
-            audio=audio,
-            video=video,
-            sampling_params=sampling_params,
-            modalities=["image"],
-        )
-        choices = await self._generate_base_chat_completion(model, payload)
-
-        image_tensors = []
-        for idx, img_content in enumerate(choices[0]["message"]["content"]):
-            base64_str = img_content.get("image_url", {}).get("url", "")
-            if not base64_str:
-                raise RuntimeError(f"API returned image #{idx} without image url")
-            tensor = base64_to_image_tensor(base64_str)
-            image_tensors.append(tensor)
-
-        return torch.stack(image_tensors, dim=0)
-
-    async def generate_comprehension_chat_completion(
-        self,
-        *,
-        model: str,
-        prompt: str,
-        image: torch.Tensor | None = None,
-        audio: AudioInput | None = None,
-        video: VideoInput | None = None,
-        sampling_params: dict | list[dict] | None = None,
-        modalities: list[str] = ["text", "audio"],
-        **extra_body,
-    ) -> tuple[str | None, AudioInput | None]:
-        # Response may contain two choices: one with text, one with audio
-        payload = VLLMOmniClient._prepare_chat_completion_messages(
-            model=model,
-            prompt=prompt,
-            negative_prompt=None,
-            image=image,
-            audio=audio,
-            video=video,
-            sampling_params=sampling_params,
-            modalities=modalities,
-            **extra_body,
-        )
-
-        choices = await self._generate_base_chat_completion(model, payload)
-        text_response = None
-        audio_base64 = None
-        for choice in choices:
-            try:
-                text_response = choice["message"]["content"]
-            except (KeyError, TypeError):
-                # Either this case (text response) or the audio response case will be hit. Checking None's later.
-                pass
-            try:
-                audio_base64 = choice["message"]["audio"]["data"]
-            except (KeyError, TypeError):
-                # Either this case (text response) or the audio response case will be hit. Checking None's later.
-                pass
-        if audio_base64 is None and text_response is None:
-            raise RuntimeError(
-                "API response missing both '.message.audio' and 'message.content' fields."
-                f"The choices object is {choices}"
-            )
-        if audio_base64 is not None:
-            audio = base64_to_audio(audio_base64)
-            logger.debug(
-                "audio sample rate %d, audio shape %s, duration in second %f",
-                audio["sample_rate"],
-                audio["waveform"].shape,
-                audio["waveform"].shape[2] / audio["sample_rate"],
-            )
-        else:
-            audio = None
-        return text_response, audio
-
-    async def generate_speech(
-        self,
-        *,
-        model: str,
-        input: str,
-        voice: str,
-        response_format: AudioFormat,
-        speed: float,
-        **extra_params,
-    ) -> AudioInput:
-        await self._check_model_exist(model)
-
-        ref_audio: AudioInput | None = extra_params.pop("ref_audio", None)
-
-        payload = {
-            "model": model,
-            "input": input,
-            "voice": voice,
-            "response_format": response_format,
-            "speed": speed,
-            **extra_params,
-        }
-
-        if ref_audio is not None:
-            audio_base64 = audio_to_base64(ref_audio)
-            payload["ref_audio"] = audio_base64
-
-        logger.debug("Omni TTS payload: %s", pretty_printer.pformat(payload))
-
-        url = self.base_url + "/audio/speech"
-        async with aiohttp.ClientSession(timeout=self.timeout) as session:
-            try:
-                async with session.post(
-                    url,
-                    json=payload,
-                    headers={"Content-Type": "application/json"},
-                ) as response:
-                    if not response.ok:
-                        error_text = await response.text()
-                        raise (ValueError if response.status < 500 else RuntimeError)(
-                            f"vLLM-Omni API returned status {response.status}: {error_text}"
-                        )
-
-                    try:
-                        audio_bytes = await response.read()
-                    except aiohttp.ContentTypeError as e:
-                        raise RuntimeError(f"Invalid JSON response from vLLM-Omni: {e}")
-
-                    try:
-                        audio = bytes_to_audio(audio_bytes)
-                    except av.error.InvalidDataError as e:
-                        raise ValueError(
-                            f"Invalid audio data received from vLLM-Omni: {e}"
-                            "Check if you have input unsupported arguments (such as 'voice')"
-                        )
-                    return audio
-
-            except aiohttp.ClientError as e:
-                raise RuntimeError(f"Network error connecting to vLLM-Omni at {url}: {e}")
-
-    async def _generate_base_chat_completion(self, model: str, payload: dict[str, Any]) -> list[dict[str, Any]]:
-        logger.debug("Omni payload: %s", pretty_printer.pformat(payload))
-        await self._check_model_exist(model)
-
-        url = self.base_url + "/chat/completions"
-        async with aiohttp.ClientSession(timeout=self.timeout) as session:
-            try:
-                async with session.post(
-                    url,
-                    json=payload,
-                    headers={"Content-Type": "application/json"},
-                ) as response:
-                    if not response.ok:
-                        error_text = await response.text()
-                        raise (ValueError if response.status < 500 else RuntimeError)(
-                            f"vLLM-Omni API returned status {response.status}: {error_text}"
-                        )
-
-                    try:
-                        data = await response.json()
-                    except aiohttp.ContentTypeError as e:
-                        raise RuntimeError(f"Invalid JSON response from vLLM-Omni: {e}")
-
-                    logger.debug(
-                        "chat completion response: %s",
-                        pretty_printer.pformat(data),
-                    )
-
-                    try:
-                        return data["choices"]
-                    except (KeyError, TypeError):
-                        raise RuntimeError("Invalid JSON response from vLLM-Omni: missing 'choices' field")
-
-            except aiohttp.ClientError as e:
-                raise RuntimeError(f"Network error connecting to vLLM-Omni at {self.base_url}: {e}")
-
-    async def _check_model_exist(self, model: str):
-        url = self.base_url + "/models"
-        async with aiohttp.ClientSession(timeout=self.timeout) as session:
-            try:
-                async with session.get(
-                    url,
-                    headers={"Content-Type": "application/json"},
-                ) as response:
-                    if not response.ok:
-                        error_text = await response.text()
-                        raise (ValueError if response.status < 500 else RuntimeError)(
-                            f"vLLM-Omni API returned status {response.status} "
-                            f"when getting hosted model list: {error_text}"
-                        )
-
-                    try:
-                        data = await response.json()
-                    except aiohttp.ContentTypeError as e:
-                        raise RuntimeError(f"Invalid JSON response when getting hosted model list from vLLM-Omni: {e}")
-
-            except aiohttp.ClientError as e:
-                raise RuntimeError(f"Network error connecting to vLLM-Omni at {self.base_url}: {e}")
-        try:
-            model_list = data["data"]
-            model_found = next((True for m in model_list if m["id"] == model), False)
-        except (KeyError, TypeError):
-            raise RuntimeError(f"Invalid JSON response of the hosted model list: {data}")
-
-        if not model_found:
-            raise ValueError(f"Model {model} not served at {self.base_url}.")
-
-    @staticmethod
-    def _prepare_chat_completion_messages(
-        *,
-        model: str,
-        prompt: str,
-        negative_prompt: str | None,
-        image: torch.Tensor | None = None,
-        audio: AudioInput | None = None,
-        video: VideoInput | None = None,
-        sampling_params: dict | list[dict] | None = None,
-        modalities: list[str] | None = None,  # diffusion don't have this field
-        **extra_body,
-    ):
-        message_content: list[dict] = [{"type": "text", "text": prompt}]
-        if image is not None:
-            message_content.append(
-                {
-                    "type": "image_url",
-                    "image_url": {"url": image_tensor_to_base64(image)},
-                }
-            )
-        if audio is not None:
-            message_content.append({"type": "audio_url", "audio_url": {"url": audio_to_base64(audio)}})
-        if video is not None:
-            message_content.append({"type": "video_url", "video_url": {"url": video_to_base64(video)}})
-        messages = [{"role": "user", "content": message_content}]
-
-        combined_extra_body: dict[str, Any] = {}
-        if sampling_params is not None:
-            spec, _ = lookup_model_spec(model)
-            is_single_sampling_param = isinstance(sampling_params, dict) or len(sampling_params) == 1
-
-            # Exclude internal key
-            if isinstance(sampling_params, dict):
-                sampling_params = {k: v for k, v in sampling_params.items() if k != "type"}
-            else:
-                sampling_params = [{k: v for k, v in sp.items() if k != "type"} for sp in sampling_params]
-
-            if (spec is None and is_single_sampling_param) or (spec is not None and spec["stages"] == ["diffusion"]):
-                # Diffusion format: extra_body directly contains sampling params.
-                # Validation should have taken care of matching sampling params' types.
-                # * Use this mode if the model is a simple one-stage diffusion model.
-                # * Fallback to this mode if model is not registered and a single sampling param is provided.
-                sampling_params = sampling_params if isinstance(sampling_params, dict) else sampling_params[0]
-                combined_extra_body: dict[str, Any] = {**sampling_params}
-                if "n" in combined_extra_body:
-                    combined_extra_body["num_outputs_per_prompt"] = combined_extra_body["n"]
-                    del combined_extra_body["n"]
-            else:
-                # Use AR style payload, extra_body has a sampling_params_list field
-                combined_extra_body: dict[str, Any] = {"sampling_params_list": sampling_params}
-
-        if negative_prompt:
-            combined_extra_body["negative_prompt"] = negative_prompt
-
-        if extra_body:
-            combined_extra_body.update(extra_body)
-
-        payload: dict[str, Any] = {"messages": messages, "model": model}
-        if combined_extra_body:
-            payload["extra_body"] = combined_extra_body
-        if modalities:
-            payload["modalities"] = modalities
-
-        spec, _ = lookup_model_spec(model)
-        if spec:
-            preprocessor = spec.get("payload_preprocessor", None)
-            if preprocessor is not None:
-                payload = preprocessor(payload)
-
-        return payload

apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/format.py

@@ -1,230 +0,0 @@
-"""Image/tensor format helpers.
-
-The image generation part is derived from dougbtv/comfyui-vllm-omni by Doug (@dougbtv).
-Original source at https://github.com/dougbtv/comfyui-vllm-omni, distributed under the MIT License.
-"""
-
-import base64
-import mimetypes
-from io import BytesIO
-
-import av
-import numpy as np
-import torch
-from comfy_api.input import AudioInput, VideoInput
-from comfy_extras import nodes_audio
-from PIL import Image
-
-from .logger import get_logger
-
-logger = get_logger(__name__)
-
-
-def base64_to_image_tensor(base64_str: str, mode: str = "RGB") -> torch.Tensor:
-    """
-    Convert base64-encoded image to ComfyUI image tensor.
-
-    Args:
-        base64_str: Base64-encoded image string
-        mode: PIL image mode (default RGB for transparency support)
-
-    Returns:
-        torch.Tensor with shape (1, H, W, C) in float32 [0, 1] range
-
-    Raises:
-        ValueError: If base64 string is invalid or image cannot be decoded
-    """
-    if base64_str.startswith("data:image"):
-        _, base64_str = base64_str.split(",", 1)
-
-    try:
-        # Decode base64 to bytes
-        image_bytes = base64.b64decode(base64_str)
-    except Exception as e:
-        raise ValueError(f"Invalid base64 string: {e}")
-
-    # Create BytesIO object for PIL
-    image_bytesio = BytesIO(image_bytes)
-
-    # Open with PIL and convert to desired mode
-    try:
-        pil_image = Image.open(image_bytesio)
-        pil_image = pil_image.convert(mode)
-    except Exception as e:
-        raise RuntimeError(f"Failed to open image: {e}")
-
-    image_array = np.asarray(pil_image).astype(np.float32) / 255.0
-    image_tensor = torch.from_numpy(image_array)
-    return image_tensor
-
-
-def image_tensor_to_png_bytes(tensor: torch.Tensor, filename: str = "image.png") -> BytesIO:
-    """
-    Convert ComfyUI image tensor to PNG BytesIO for multipart upload.
-
-    This function converts a ComfyUI IMAGE tensor to a PNG-encoded BytesIO object
-    suitable for multipart/form-data upload. The BytesIO object has its .name
-    attribute set, which is required by aiohttp for file uploads.
-
-    Args:
-        tensor: ComfyUI IMAGE tensor with shape (B, H, W, C), dtype float32, range [0, 1]
-        filename: Name attribute to set on BytesIO (default: "image.png")
-
-    Returns:
-        BytesIO object containing PNG-encoded image with .name attribute set
-
-    Raises:
-        ValueError: If tensor format is invalid (not 4D, wrong dtype, etc.)
-    """
-    if tensor.ndim != 4:
-        raise ValueError(f"Expected 4D tensor with shape (B, H, W, C), got {tensor.ndim}D tensor")
-
-    image_tensor = tensor[0]  # Shape: (H, W, C)
-    image_np = (image_tensor.cpu().numpy() * 255.0).astype(np.uint8)
-    pil_image = Image.fromarray(image_np)
-
-    # Save to BytesIO as image file
-    img_bytes = BytesIO()
-    # Set name attribute (required for multipart upload and mimetype detection)
-    img_bytes.name = filename
-    try:
-        pil_image.save(img_bytes)
-    except Exception as e:
-        raise RuntimeError(f"Failed to save image as file: {e}")
-
-    # Reset position to beginning
-    img_bytes.seek(0)
-
-    return img_bytes
-
-
-def image_tensor_to_base64(tensor: torch.Tensor, filename: str = "image.png") -> str:
-    """
-    Convert ComfyUI image tensor to base64-encoded image string.
-
-    Args:
-        tensor: ComfyUI IMAGE tensor with shape (B, H, W, C), dtype float32, range [0, 1]
-        filename: Name attribute to set on BytesIO (default: "image.png")
-        format: File format of the output image file buffer (default: "PNG")
-
-    Returns:
-        Base64-encoded image string
-
-    Raises:
-        ValueError: If tensor format is invalid (not 4D, wrong dtype, etc.)
-    """
-    img_bytes = image_tensor_to_png_bytes(tensor, filename)
-    img_bytes.seek(0)
-    byte_data = img_bytes.read()
-    base64_str = base64.b64encode(byte_data).decode("utf-8")
-    mime_type = mimetypes.guess_type(filename)[0] or "application/octet-stream"
-    return f"data:{mime_type};base64,{base64_str}"
-
-
-def video_to_bytes(video: VideoInput, filename: str = "video.mp4") -> BytesIO:
-    output_buffer = BytesIO()
-    output_buffer.name = filename
-    video.save_to(output_buffer)
-    output_buffer.seek(0)
-    return output_buffer
-
-
-def video_to_base64(video: VideoInput, filename: str = "video.mp4") -> str:
-    video_buffer = video_to_bytes(video, filename)
-    video_buffer.seek(0)
-    byte_data = video_buffer.read()
-    base64_str = base64.b64encode(byte_data).decode("utf-8")
-    mime_type = mimetypes.guess_type(filename)[0] or "application/octet-stream"
-    return f"data:{mime_type};base64,{base64_str}"
-
-
-def audio_to_bytes(audio: AudioInput, filename: str = "audio.mp3", quality: str = "128k") -> BytesIO:
-    waveform = audio["waveform"][0]  # Shape: (C, T)
-    sample_rate = audio["sample_rate"]
-    format = filename.rsplit(".", maxsplit=1)[1]
-    layout = "mono" if waveform.shape[0] == 1 else "stereo"
-
-    output_buffer = BytesIO()
-    output_buffer.name = filename
-    output_container = av.open(output_buffer, mode="w", format=format)
-    if format == "opus":
-        out_stream = output_container.add_stream("libopus", rate=sample_rate, layout=layout)
-        if quality == "64k":
-            out_stream.bit_rate = 64000  # type: ignore # copy from ComfyUI comfy_api/latest/_ui.py
-        elif quality == "96k":
-            out_stream.bit_rate = 96000  # type: ignore # copy from ComfyUI comfy_api/latest/_ui.py
-        elif quality == "128k":
-            out_stream.bit_rate = 128000  # type: ignore # copy from ComfyUI comfy_api/latest/_ui.py
-        elif quality == "192k":
-            out_stream.bit_rate = 192000  # type: ignore # copy from ComfyUI comfy_api/latest/_ui.py
-        elif quality == "320k":
-            out_stream.bit_rate = 320000  # type: ignore # copy from ComfyUI comfy_api/latest/_ui.py
-    elif format == "mp3":
-        out_stream = output_container.add_stream("libmp3lame", rate=sample_rate, layout=layout)
-        if quality == "V0":
-            out_stream.codec_context.qscale = 1  # type: ignore # copy from ComfyUI comfy_api/latest/_ui.py
-        elif quality == "128k":
-            out_stream.bit_rate = 128000  # type: ignore # copy from ComfyUI comfy_api/latest/_ui.py
-        elif quality == "320k":
-            out_stream.bit_rate = 320000  # type: ignore # copy from ComfyUI comfy_api/latest/_ui.py
-    else:  # format == "flac":
-        out_stream = output_container.add_stream("flac", rate=sample_rate, layout=layout)
-
-    frame = av.AudioFrame.from_ndarray(
-        waveform.movedim(0, 1).reshape(1, -1).float().numpy(),
-        format="flt",
-        layout=layout,
-    )
-    frame.sample_rate = sample_rate
-    frame.pts = 0
-    output_container.mux(out_stream.encode(frame))  # type: ignore # copy from ComfyUI comfy_api/latest/_ui.py
-    # Flush encoder
-    output_container.mux(out_stream.encode(None))  # type: ignore # copy from ComfyUI comfy_api/latest/_ui.py
-    output_container.close()
-    output_buffer.seek(0)
-
-    return output_buffer
-
-
-def audio_to_base64(audio: AudioInput, filename: str = "audio.mp3", quality: str = "128k") -> str:
-    audio_buffer = audio_to_bytes(audio, filename, quality)
-    audio_buffer.seek(0)
-    byte_data = audio_buffer.read()
-    base64_str = base64.b64encode(byte_data).decode("utf-8")
-    mime_type = mimetypes.guess_type(filename)[0] or "application/octet-stream"
-    return f"data:{mime_type};base64,{base64_str}"
-
-
-def bytes_to_audio(audio_bytes: bytes) -> AudioInput:
-    """
-    Convert audio bytes to ComfyUI audio tensor.
-
-    Args:
-        audio_bytes: Audio file bytes
-    Returns:
-        torch.Tensor with shape (B, C, T) in float32 range [-1, 1]
-    """
-    audio_buffer = BytesIO(audio_bytes)
-    waveform, sample_rate = nodes_audio.load(audio_buffer)  # type: ignore # Although expect string argument, it calls av.open underneath, which supports BytesIO (file-like)
-    return {"waveform": waveform.unsqueeze(0), "sample_rate": sample_rate}
-
-
-def base64_to_audio(base64_str: str) -> AudioInput:
-    """
-    Convert base64-encoded audio to ComfyUI audio tensor.
-
-    Args:
-        base64_str: Base64-encoded audio string
-    Returns:
-        torch.Tensor with shape (B, C, T) in float32 range [-1, 1]
-    """
-    if base64_str.startswith("data:audio"):
-        _, base64_str = base64_str.split(",", 1)
-
-    try:
-        # Decode base64 to bytes
-        audio_bytes = base64.b64decode(base64_str)
-    except Exception as e:
-        raise ValueError(f"Invalid base64 string: {e}")
-
-    return bytes_to_audio(audio_bytes)

apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/logger.py

@@ -1,125 +0,0 @@
-"""Centralized logger configuration for vLLM-Omni ComfyUI."""
-
-import logging
-import pprint
-import sys
-from typing import Any
-
-
-def get_logger(name: str) -> logging.Logger:
-    """
-    Get or create a logger with proper formatting.
-
-    Args:
-        name: Logger name (typically __name__ of the calling module)
-
-    Returns:
-        Configured logger instance
-    """
-    logger = logging.getLogger(name)
-
-    # Only configure if not already configured
-    if not logger.handlers:
-        logger.setLevel(logging.DEBUG)
-
-        # Create console handler
-        handler = logging.StreamHandler(sys.stdout)
-        handler.setLevel(logging.INFO)
-
-        # Create formatter
-        formatter = logging.Formatter(
-            fmt="(ComfyUI-vLLM-Omni) [%(levelname)s] %(asctime)s [%(filename)s:%(lineno)s] %(message)s",
-            datefmt="%Y-%m-%d %H:%M:%S",
-        )
-        handler.setFormatter(formatter)
-
-        # Add handler to logger
-        logger.addHandler(handler)
-
-        # Prevent propagation to root logger
-        logger.propagate = False
-
-    return logger
-
-
-class OmitBase64PrettyPrinter(pprint.PrettyPrinter):
-    """
-    A PrettyPrinter that redacts specific field names with '...'
-    wherever they appear in nested structures.
-    """
-
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-
-    def _format(self, obj: Any, stream, indent: int, allowance: int, context, level: int) -> None:
-        # Check if this is a dict with redacted keys
-        if isinstance(obj, dict):
-            # Create a copy with redacted values
-            display_obj = {}
-            for key, value in obj.items():
-                if key == "data" or key == "url":
-                    if value.startswith("data:"):
-                        base64_header = value.split(",", 1)[0]
-                        display_obj[key] = f"{base64_header},***"
-                    elif value.startswith("http://") or value.startswith("https://"):
-                        display_obj[key] = value
-                    elif len(value) > 10:
-                        display_obj[key] = f"{value[:10]}***"
-                    else:
-                        display_obj[key] = value
-                else:
-                    display_obj[key] = value
-            obj = display_obj
-
-        # Handle list/tuple/set containing dicts that might have redacted keys
-        # (pprint will recursively call _format on nested items, so this
-        # handles arbitrary nesting automatically)
-
-        super()._format(obj, stream, indent, allowance, context, level)
-
-
-pretty_printer = OmitBase64PrettyPrinter()
-
-
-# ========== PPrint EXAMPLES ==========
-
-if __name__ == "__main__":
-    data = {
-        "messages": [
-            {
-                "role": "system",
-                "content": (
-                    "You are Qwen, a virtual human developed by the Qwen Team, Alibaba Group,"
-                    "capable of perceiving auditory and visual inputs, as well as generating text and speech."
-                ),
-            },
-            {
-                "role": "user",
-                "content": [
-                    {
-                        "type": "text",
-                        "text": "What sound is it, and what is the drawing about?",
-                    },
-                    {
-                        "type": "text",
-                        "text": "What sound is it, and what is the drawing about?",
-                    },
-                    {
-                        "type": "image_url",
-                        "image_url": {"url": "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAwAAAA"},
-                    },
-                    {
-                        "type": "audio_url",
-                        "audio_url": {"url": "data:audio/mpeg;base64,SUQzBAAAAAAAIlRTU0UAAAAOAAADT="},
-                    },
-                ],
-            },
-        ],
-        "extra_body": {"mm_processor_kwargs": {"use_audio_in_video": False}},
-        "modalities": ["text"],
-    }
-
-    # Create printer that redacts 'password' and 'token' fields
-
-    print("\nRedactingPrettyPrinter (hides secrets):")
-    pretty_printer.pprint(data)

apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/models.py

@@ -1,119 +0,0 @@
-import re
-
-from .types import Modality, ModelMode, Spec
-
-
-def _bagel_payload_preprocessor(payload: dict) -> dict:
-    try:
-        for message in payload["messages"]:
-            for content in message["content"]:
-                if content["type"] == "text":
-                    content["text"] = "<|im_start|>" + content["text"] + "<|im_end|>"
-    except (KeyError, TypeError):
-        raise RuntimeError("Internal Error: malformatted BAGEL payload")
-    extra_body = payload.pop("extra_body", {})
-    return {**payload, **extra_body}
-
-
-def _qwen25_payload_preprocessor(payload: dict) -> dict:
-    if payload["messages"][0]["role"] != "system":
-        payload["messages"] = [
-            {
-                "role": "system",
-                "content": (
-                    "You are Qwen, a virtual human developed by the Qwen Team, Alibaba Group,"
-                    "capable of perceiving auditory and visual inputs, as well as generating text and speech."
-                ),
-            },
-            *payload["messages"],
-        ]
-    return payload
-
-
-_MODEL_PIPELINE_SPECS: dict[str, Spec] = {
-    r"BAGEL-7B-MoT": {
-        "stages": [
-            "diffusion"  # The vLLM-Omni interface treats it as a single-stage diffusion model
-        ],
-        "modes": [
-            {
-                "mode": ModelMode.COMPREHENSION,
-                "input_modalities": [Modality.TEXT, Modality.IMAGE],
-            }
-        ],
-        "payload_preprocessor": _bagel_payload_preprocessor,
-    },
-    r"Qwen2.5-Omni*": {
-        "stages": ["autoregression", "autoregression", "autoregression"],
-        "payload_preprocessor": _qwen25_payload_preprocessor,
-        "modes": [
-            {
-                "mode": ModelMode.COMPREHENSION,
-                "input_modalities": [
-                    Modality.TEXT,
-                    Modality.IMAGE,
-                    Modality.VIDEO,
-                    Modality.AUDIO,
-                ],
-            }
-        ],
-    },
-    r"Qwen3-Omni*": {
-        "stages": ["autoregression", "autoregression", "autoregression"],
-        "modes": [
-            {
-                "mode": ModelMode.COMPREHENSION,
-                "input_modalities": [
-                    Modality.TEXT,
-                    Modality.IMAGE,
-                    Modality.VIDEO,
-                    Modality.AUDIO,
-                ],
-            }
-        ],
-    },
-}
-# Convert dict keys to regex patterns
-MODEL_PIPELINE_SPECS: dict[re.Pattern, Spec] = {}
-for k, v in _MODEL_PIPELINE_SPECS.items():
-    MODEL_PIPELINE_SPECS[re.compile(k)] = v
-del _MODEL_PIPELINE_SPECS
-
-
-def lookup_model_spec(model: str) -> tuple[Spec | None, str | None]:
-    try:
-        last_component = model.rstrip("/").rsplit("/", 1)[-1]
-    except IndexError:
-        last_component = model
-    for pattern, spec in MODEL_PIPELINE_SPECS.items():
-        if pattern.search(last_component):
-            return spec, pattern.pattern
-    return None, None
-
-
-# ============== DEMONSTRATION ==============
-
-if __name__ == "__main__":
-    test_paths = [
-        "Qwen/Qwen2.5-Omni-7B",
-        "MyModels/Qwen2.5-Omni-3B",
-        "/root/home/Qwen2.5-Omni-7B",
-        "Qwen/Qwen3-Omni",
-        "Qwen/Qwen3-Omni-30B-A3B-Instruct",
-        "Custom/Path/UnknownModel-Instruct",
-        "Not/Matching/Anything",
-    ]
-
-    test_payload = {"messages": [{"role": "user", "content": "prompt"}]}
-
-    print("Testing registry lookups:\n")
-    for path in test_paths:
-        spec, _ = lookup_model_spec(path)
-        if spec:
-            if preprocessor := spec.get("payload_preprocessor"):
-                result = preprocessor(test_payload)
-                print(f"✓ {path:<40} → {result}")
-            else:
-                print(f"✓ {path:<40} → No preprocessor")
-        else:
-            print(f"✗ {path:<40} → No match")

apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/types.py

@@ -1,39 +0,0 @@
-from collections.abc import Callable
-from enum import Enum, auto
-from typing import (
-    Any,
-    Literal,
-    NotRequired,
-    TypeAlias,
-    TypedDict,
-)
-
-AudioFormat: TypeAlias = Literal["mp3", "opus", "aac", "flac", "wav", "pcm"]
-
-
-class ModelMode(Enum):
-    IMAGE_GENERATION = auto()
-    VIDEO_GENERATION = auto()
-    AUDIO_GENERATION = auto()
-    COMPREHENSION = auto()
-
-
-class Modality(Enum):
-    TEXT = auto()  # maybe not useful. Prompt is always required
-    IMAGE = auto()
-    VIDEO = auto()
-    AUDIO = auto()
-
-
-class ModelModeSpec(TypedDict):
-    mode: ModelMode
-    input_modalities: list[Modality]
-
-
-PayloadPreprocessor: TypeAlias = Callable[[dict[str, Any]], dict[str, Any]]
-
-
-class Spec(TypedDict):
-    stages: list[Literal["diffusion", "autoregression"]]
-    modes: list[ModelModeSpec]
-    payload_preprocessor: NotRequired[PayloadPreprocessor]

apps/ComfyUI-vLLM-Omni/comfyui_vllm_omni/utils/validators.py

@@ -1,84 +0,0 @@
-from .logger import get_logger
-from .models import lookup_model_spec
-
-logger = get_logger(__name__)
-
-
-def validate_model_and_sampling_params_types(
-    model_name: str,
-    sampling_param_list: dict | list[dict] | None = None,
-):
-    # Check if model name exists
-    if not model_name:
-        raise ValueError("Model name must not be empty.")
-
-    # Skip if no spec or no sampling params
-    pipeline_spec, _ = lookup_model_spec(model_name)
-    if pipeline_spec is None:
-        logger.info("skipping sampling params check because spec is not found")
-        return
-    if sampling_param_list is None:
-        return
-
-    # Check the number of stages and their data types
-    stages = pipeline_spec["stages"]
-    if isinstance(sampling_param_list, list):
-        # Check that the lengths match
-        if len(stages) != len(sampling_param_list):
-            raise ValueError(
-                f"Sampling parameter list length {len(sampling_param_list)} does not match "
-                f"number of stages {len(stages)} for model {model_name}."
-            )
-        # Check that each stage's type match
-        for i, sp in enumerate(sampling_param_list):
-            if "type" not in sp:
-                raise RuntimeError("Internal error: unknown sampling parameter type")
-            if sp["type"] != stages[i]:
-                raise ValueError(
-                    f"Sampling parameter type ({sp['type']}) does not match "
-                    f"stage type ({stages[i]}) at index {i} for model {model_name}."
-                )
-    elif isinstance(sampling_param_list, dict):
-        if "type" not in sampling_param_list:
-            raise RuntimeError("Internal error: unknown sampling parameter type")
-        # Check that the provided single sampling param matches all stages
-        elif any(stage != sampling_param_list["type"] for stage in stages):
-            raise ValueError(
-                f"When passing a single sampling parameter node, all stages of the model must match "
-                f"the provided sampling parameter's type. "
-                f"However, the stages of model {model_name} are: {stages}. "
-                f"The provided sampling parameter is {sampling_param_list['type']}"
-            )
-
-
-def add_sampling_parameters_to_stage(
-    model_name: str,
-    sampling_param_list: dict | list[dict] | None,
-    stage_type: str,
-    /,
-    **params_to_add,
-) -> dict | list[dict]:
-    """
-    Given a model's name and the sampling parameter list to query this model,
-    add arbitrary additional parameters to the sampling parameters of all stages of the given type.
-    """
-    pipeline_spec, _ = lookup_model_spec(model_name)
-    if not pipeline_spec:
-        logger.warning(
-            f"Since the model {model_name} is not in our list, we cannot ensure if "
-            f"the fields ({tuple(params_to_add.keys())}) are added to the correct stage's sampling params. "
-            f"We will do it heuristically."
-        )
-        pipeline_spec = {"stages": ["diffusion"]}
-
-    stages = pipeline_spec["stages"]
-    if isinstance(sampling_param_list, dict):
-        sampling_param_list.update(params_to_add)
-    elif sampling_param_list is None:
-        sampling_param_list = params_to_add.copy()
-    else:
-        for i, stage in enumerate(stages):
-            if stage == stage_type:
-                sampling_param_list[i].update(params_to_add)
-
-    return sampling_param_list

apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni Annotated Example.json

@@ -1 +0,0 @@
-{"id":"1c99f525-0a37-45ba-a28a-7df7c3af66b4","revision":0,"last_node_id":12,"last_link_id":14,"nodes":[{"id":1,"type":"VLLMOmniComprehension","pos":[1191.2177053682556,144.66829928181377],"size":[400,268],"flags":{},"order":8,"mode":0,"inputs":[{"localized_name":"image","name":"image","shape":7,"type":"IMAGE","link":1},{"localized_name":"video","name":"video","shape":7,"type":"VIDEO","link":2},{"localized_name":"audio","name":"audio","shape":7,"type":"AUDIO","link":3},{"localized_name":"sampling_params","name":"sampling_params","shape":7,"type":"SAMPLING_PARAMS","link":14},{"localized_name":"url","name":"url","type":"STRING","widget":{"name":"url"},"link":null},{"localized_name":"model","name":"model","type":"STRING","widget":{"name":"model"},"link":null},{"localized_name":"prompt","name":"prompt","type":"STRING","widget":{"name":"prompt"},"link":null},{"localized_name":"output_text","name":"output_text","type":"BOOLEAN","widget":{"name":"output_text"},"link":null},{"localized_name":"output_audio","name":"output_audio","type":"BOOLEAN","widget":{"name":"output_audio"},"link":null},{"localized_name":"use_audio_in_video","name":"use_audio_in_video","type":"BOOLEAN","widget":{"name":"use_audio_in_video"},"link":null}],"outputs":[{"localized_name":"text_response","name":"text_response","type":"STRING","links":[8]},{"localized_name":"audio_response","name":"audio_response","type":"AUDIO","links":[9]}],"properties":{"Node name for S&R":"VLLMOmniComprehension"},"widgets_values":["http://localhost:8000/v1","Qwen/Qwen2.5-Omni-7B","",true,true,true]},{"id":3,"type":"LoadVideo","pos":[729.5984141255855,-198.631920454299],"size":[282.798828125,233.0743408203125],"flags":{},"order":0,"mode":0,"inputs":[{"localized_name":"file","name":"file","type":"COMBO","widget":{"name":"file"},"link":null},{"localized_name":"choose file to upload","name":"upload","type":"IMAGEUPLOAD","widget":{"name":"upload"},"link":null}],"outputs":[{"localized_name":"VIDEO","name":"VIDEO","type":"VIDEO","links":[2]}],"properties":{"Node name for S&R":"LoadVideo"},"widgets_values":["draw.mp4","image"]},{"id":4,"type":"LoadAudio","pos":[729.8037086965753,99.86963519703949],"size":[282.798828125,136],"flags":{},"order":1,"mode":0,"inputs":[{"localized_name":"audio","name":"audio","type":"COMBO","widget":{"name":"audio"},"link":null},{"localized_name":"audioUI","name":"audioUI","type":"AUDIO_UI","widget":{"name":"audioUI"},"link":null},{"localized_name":"choose file to upload","name":"upload","type":"AUDIOUPLOAD","widget":{"name":"upload"},"link":null}],"outputs":[{"localized_name":"AUDIO","name":"AUDIO","type":"AUDIO","links":[3]}],"properties":{"Node name for S&R":"LoadAudio"},"widgets_values":["Megan-Fox.mp3",null,null]},{"id":5,"type":"VLLMOmniARSampling","pos":[510.3517536642828,658.073751009259],"size":[270,178],"flags":{},"order":2,"mode":0,"inputs":[{"localized_name":"max_tokens","name":"max_tokens","type":"INT","widget":{"name":"max_tokens"},"link":null},{"localized_name":"temperature","name":"temperature","type":"FLOAT","widget":{"name":"temperature"},"link":null},{"localized_name":"top_p","name":"top_p","type":"FLOAT","widget":{"name":"top_p"},"link":null},{"localized_name":"repetition_penalty","name":"repetition_penalty","type":"FLOAT","widget":{"name":"repetition_penalty"},"link":null},{"localized_name":"seed","name":"seed","type":"INT","widget":{"name":"seed"},"link":null}],"outputs":[{"localized_name":"AR sampling params","name":"AR sampling params","type":"SAMPLING_PARAMS","links":[12]}],"properties":{"Node name for S&R":"VLLMOmniARSampling"},"widgets_values":[100,1,1,1,-1,"randomize"]},{"id":7,"type":"VLLMOmniARSampling","pos":[503.33235181647115,419.34158016181806],"size":[270,178],"flags":{},"order":3,"mode":0,"inputs":[{"localized_name":"max_tokens","name":"max_tokens","type":"INT","widget":{"name":"max_tokens"},"link":null},{"localized_name":"temperature","name":"temperature","type":"FLOAT","widget":{"name":"temperature"},"link":null},{"localized_name":"top_p","name":"top_p","type":"FLOAT","widget":{"name":"top_p"},"link":null},{"localized_name":"repetition_penalty","name":"repetition_penalty","type":"FLOAT","widget":{"name":"repetition_penalty"},"link":null},{"localized_name":"seed","name":"seed","type":"INT","widget":{"name":"seed"},"link":null}],"outputs":[{"localized_name":"AR sampling params","name":"AR sampling params","type":"SAMPLING_PARAMS","links":[5,13]}],"properties":{"Node name for S&R":"VLLMOmniARSampling"},"widgets_values":[100,1,1,1,-1,"randomize"]},{"id":8,"type":"VLLMOmniSamplingParamsList","pos":[820.6056617389042,426.38372037182273],"size":[263.066015625,66],"flags":{},"order":7,"mode":0,"inputs":[{"localized_name":"param1","name":"param1","type":"SAMPLING_PARAMS","link":5},{"localized_name":"param2","name":"param2","shape":7,"type":"SAMPLING_PARAMS","link":12},{"localized_name":"param3","name":"param3","shape":7,"type":"SAMPLING_PARAMS","link":13}],"outputs":[{"localized_name":"param list","name":"param list"Line truncated

apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni Image Generation.json

@@ -1 +0,0 @@
-{"id":"91f75acc-8040-40f6-865a-2e8a7cfd6672","revision":0,"last_node_id":11,"last_link_id":23,"nodes":[{"id":3,"type":"PreviewImage","pos":[1281.8455167767304,-69.02638461454333],"size":[305.6628685610174,307.0336884172169],"flags":{},"order":4,"mode":0,"inputs":[{"localized_name":"images","name":"images","type":"IMAGE","link":22}],"outputs":[],"properties":{"Node name for S&R":"PreviewImage"},"widgets_values":[]},{"id":11,"type":"VLLMOmniGenerateImage","pos":[816.0962515873642,-137.51112584854445],"size":[400,278],"flags":{},"order":3,"mode":0,"inputs":[{"localized_name":"image","name":"image","shape":7,"type":"IMAGE","link":21},{"localized_name":"mask","name":"mask","shape":7,"type":"MASK","link":null},{"localized_name":"sampling_params","name":"sampling_params","shape":7,"type":"SAMPLING_PARAMS","link":23},{"localized_name":"url","name":"url","type":"STRING","widget":{"name":"url"},"link":null},{"localized_name":"model","name":"model","type":"STRING","widget":{"name":"model"},"link":null},{"localized_name":"prompt","name":"prompt","type":"STRING","widget":{"name":"prompt"},"link":null},{"localized_name":"negative_prompt","name":"negative_prompt","type":"STRING","widget":{"name":"negative_prompt"},"link":null},{"localized_name":"width","name":"width","type":"INT","widget":{"name":"width"},"link":null},{"localized_name":"height","name":"height","type":"INT","widget":{"name":"height"},"link":null}],"outputs":[{"localized_name":"image","name":"image","type":"IMAGE","links":[22]}],"properties":{"Node name for S&R":"VLLMOmniGenerateImage"},"widgets_values":["http://localhost:8000/v1","Qwen/Qwen-Image-Edit","Put this figure in a realistic mountain view","",512,512]},{"id":8,"type":"VLLMOmniDiffusionSampling","pos":[478.59266934006774,183.67711984955648],"size":[284.205078125,202],"flags":{},"order":0,"mode":0,"inputs":[{"localized_name":"n","name":"n","type":"INT","widget":{"name":"n"},"link":null},{"localized_name":"num_inference_steps","name":"num_inference_steps","type":"INT","widget":{"name":"num_inference_steps"},"link":null},{"localized_name":"guidance_scale","name":"guidance_scale","type":"FLOAT","widget":{"name":"guidance_scale"},"link":null},{"localized_name":"true_cfg_scale","name":"true_cfg_scale","type":"FLOAT","widget":{"name":"true_cfg_scale"},"link":null},{"localized_name":"vae_use_slicing","name":"vae_use_slicing","type":"BOOLEAN","widget":{"name":"vae_use_slicing"},"link":null},{"localized_name":"seed","name":"seed","type":"INT","widget":{"name":"seed"},"link":null}],"outputs":[{"localized_name":"diffusion sampling params","name":"diffusion sampling params","type":"SAMPLING_PARAMS","links":[23]}],"properties":{"Node name for S&R":"VLLMOmniDiffusionSampling"},"widgets_values":[4,50,1,1,false,42,"fixed"]},{"id":10,"type":"MarkdownNote","pos":[227.99306819575278,-231.24143306069843],"size":[240.3326478922474,136.3505820791881],"flags":{},"order":2,"mode":0,"inputs":[],"outputs":[],"title":"Note: Task and Input","properties":{},"widgets_values":["vLLM-Omni nodes are categorized based on the output modality. The \"Generate Image\" node supports both text-to-image generation or image-to-image generation (a.k.a. image editing). The node will route to the correct endpoint depending on whether an input image is present or not."],"color":"#432","bgcolor":"#000"},{"id":4,"type":"LoadImage","pos":[496.31859627609606,-229.71277089860084],"size":[270,314],"flags":{},"order":1,"mode":0,"inputs":[{"localized_name":"image","name":"image","type":"COMBO","widget":{"name":"image"},"link":null},{"localized_name":"choose file to upload","name":"upload","type":"IMAGEUPLOAD","widget":{"name":"upload"},"link":null}],"outputs":[{"localized_name":"IMAGE","name":"IMAGE","type":"IMAGE","links":[21]},{"localized_name":"MASK","name":"MASK","type":"MASK","links":null}],"properties":{"Node name for S&R":"LoadImage"},"widgets_values":["example.png","image"]}],"links":[[21,4,0,11,0,"IMAGE"],[22,11,0,3,0,"IMAGE"],[23,8,0,11,2,"SAMPLING_PARAMS"]],"groups":[{"id":1,"title":"Input","bounding":[213.1706010087147,-313.20095750667554,560.8246471437027,407.4246532472182],"color":"#3f789e","font_size":24,"flags":{}}],"config":{},"extra":{"workflowRendererVersion":"LG","ds":{"scale":1.1469075819486894,"offset":[276.8427259980847,549.4885226647582]}},"version":0.4}

apps/ComfyUI-vLLM-Omni/example_workflows/vLLM-Omni TTS.json

@@ -1 +0,0 @@
-{"id":"c0a11241-f0e6-4f5c-8488-b3d344a9df3d","revision":0,"last_node_id":7,"last_link_id":3,"nodes":[{"id":4,"type":"PreviewAudio","pos":[690.1759063763212,226.76444957981545],"size":[270,88],"flags":{},"order":5,"mode":0,"inputs":[{"localized_name":"audio","name":"audio","type":"AUDIO","link":2},{"localized_name":"audioUI","name":"audioUI","type":"AUDIO_UI","widget":{"name":"audioUI"},"link":null}],"outputs":[],"properties":{"Node name for S&R":"PreviewAudio"},"widgets_values":[]},{"id":5,"type":"LoadAudio","pos":[-83.61093919682321,103.80510898465587],"size":[282.798828125,136],"flags":{},"order":0,"mode":0,"inputs":[{"localized_name":"audio","name":"audio","type":"COMBO","widget":{"name":"audio"},"link":null},{"localized_name":"audioUI","name":"audioUI","type":"AUDIO_UI","widget":{"name":"audioUI"},"link":null},{"localized_name":"choose file to upload","name":"upload","type":"AUDIOUPLOAD","widget":{"name":"upload"},"link":null}],"outputs":[{"localized_name":"AUDIO","name":"AUDIO","type":"AUDIO","links":[3]}],"properties":{"Node name for S&R":"LoadAudio"},"widgets_values":["Megan-Fox.mp3",null,null]},{"id":3,"type":"VLLMOmniVoiceClone","pos":[248.63466280064262,168.13112893706375],"size":[400,306],"flags":{},"order":4,"mode":0,"inputs":[{"localized_name":"ref_audio","name":"ref_audio","type":"AUDIO","link":3},{"localized_name":"model_specific_params","name":"model_specific_params","shape":7,"type":"TTS_PARAMS","link":1},{"localized_name":"url","name":"url","type":"STRING","widget":{"name":"url"},"link":null},{"localized_name":"model","name":"model","type":"STRING","widget":{"name":"model"},"link":null},{"localized_name":"input","name":"input","type":"STRING","widget":{"name":"input"},"link":null},{"localized_name":"voice","name":"voice","type":"STRING","widget":{"name":"voice"},"link":null},{"localized_name":"response_format","name":"response_format","type":"COMBO","widget":{"name":"response_format"},"link":null},{"localized_name":"speed","name":"speed","type":"FLOAT","widget":{"name":"speed"},"link":null},{"localized_name":"ref_text","name":"ref_text","type":"STRING","widget":{"name":"ref_text"},"link":null},{"localized_name":"x_vector_only_mode","name":"x_vector_only_mode","type":"BOOLEAN","widget":{"name":"x_vector_only_mode"},"link":null}],"outputs":[{"localized_name":"audio","name":"audio","type":"AUDIO","links":[2]}],"properties":{"Node name for S&R":"VLLMOmniVoiceClone"},"widgets_values":["http://localhost:8000/v1","Qwen/Qwen3-TTS-12Hz-1.7B-Base","Someone just spilled a cup of coffee on my jacket this morning in the subway. Now I have to wear this stained jacket for a whole day!","","mp3",1,"",true]},{"id":6,"type":"MarkdownNote","pos":[255.52667408769662,27.26842775454861],"size":[381.58204108637847,88],"flags":{},"order":2,"mode":0,"inputs":[],"outputs":[],"title":"Note: TTS Nodes","properties":{},"widgets_values":["Apart from the Voice Cloning node, there is also another TTS node for simple text-to-speech tasks (without extra inputs for reference audio)."],"color":"#432","bgcolor":"#000"},{"id":7,"type":"MarkdownNote","pos":[-196.99453003423977,598.3693693727904],"size":[382.4539509209102,88],"flags":{},"order":3,"mode":0,"inputs":[],"outputs":[],"title":"Note: Model-Specific Parameters","properties":{},"widgets_values":["TTS models often require some tailor-made parameters. If you need to customize these parameters, grab one that matches the model you are requesting from \"TTS Params\" subfolder."],"color":"#432","bgcolor":"#000"},{"id":2,"type":"VLLMOmniQwenTTSParams","pos":[-207.8925310560497,342.7942867650348],"size":[400,200],"flags":{},"order":1,"mode":0,"inputs":[{"localized_name":"task_type","name":"task_type","type":"COMBO","widget":{"name":"task_type"},"link":null},{"localized_name":"language","name":"language","type":"COMBO","widget":{"name":"language"},"link":null},{"localized_name":"instructions","name":"instructions","type":"STRING","widget":{"name":"instructions"},"link":null},{"localized_name":"max_new_tokens","name":"max_new_tokens","type":"INT","widget":{"name":"max_new_tokens"},"link":null}],"outputs":[{"localized_name":"Qwen TTS params","name":"Qwen TTS params","type":"TTS_PARAMS","links":[1]}],"properties":{"Node name for S&R":"VLLMOmniQwenTTSParams"},"widgets_values":["Base","Auto","Super angry",2048]}],"links":[[1,2,0,3,1,"TTS_PARAMS"],[2,3,0,4,0,"AUDIO"],[3,5,0,3,0,"AUDIO"]],"groups":[{"id":1,"title":"Model-Specific Parameters","bounding":[-228.38328407738007,264.42790274716276,443.9887293164586,439.6853171248897],"color":"#3f789e","font_size":24,"flags":{}}],"config":{},"extra":{"workflowRendererVersion":"LG","ds":{"scale":1.1469075819486911,"offset":[898.8819468322333,258.7179979718389]}},"version":0.4}

apps/ComfyUI-vLLM-Omni/web/main.js

@@ -1,21 +0,0 @@
-/**
- * @file This file is intended to add dynamic fields to vLLM-Omni nodes
- * based on widget (in-node form fields) and input (connection link) values and changes.
- * However, this functionality is currently disabled/commented out.
- * Because it introduces too much complexity,
- * and it may even conflict with the current backend (Python) validation for unknown reasons (pending ComfyUI upstream fixes).
- */
-
-import { app } from "../../scripts/app.js";
-app.registerExtension({
-    name: "vllm.vllm_omni",
-    async beforeRegisterNodeDef(nodeType, nodeData, app) {
-        if (!nodeData.name.startsWith("VLLMOmni")) {
-            return
-        }
-        // Stub frontend plugin for now
-    },
-    async setup() {
-        console.info("vLLM-Omni Setup complete!")
-    },
-})

benchmarks/README.md

@@ -1,42 +0,0 @@
-# Benchmarks Overview and Architecture
-
-This document explains the benchmark architecture across all benchmark assets in this repo. It describes what we measure, and where to find or plug in new scenarios. Per-task details remain in subfolder READMEs (e.g., `benchmarks/<model>/README.md`).
-
-## Scope and goals
-- Establish repeatable latency/throughput measurements for multimodal LLM pipelines.
-- Provide both HF Transformers (offline) and vLLM-Omni (multi-stage/pipeline) baselines.
-- Make it easy to plug in new datasets and models with minimal changes to the runner scripts.
-
-## Dataset and inputs
-- Default example: SeedTTS top-100 prompts (`benchmarks/build_dataset/top100.txt`) via `benchmarks/build_dataset/`.
-- Extensible: drop in new prompt files or modality-aligned payloads; keep the expected format for the consuming scripts (e.g., one prompt per line).
-- If you add a new dataset, document it under `benchmarks/<model>/README.md` and point scripts to your data path.
-
-## Directory layout
-- `benchmarks/build_dataset/` — dataset prep utilities (e.g., SeedTTS top100).
-- `benchmarks/<model>/vllm_omni/` — vLLM-Omni pipeline benchmarks, logs, outputs.
-- Add new tasks under `benchmarks/<model>/...` with the same pattern: `transformers/`, `vllm_omni/`, task-specific README, and (optionally) dataset prep notes.
-
-## Reference workflows
-- **HF Transformers (offline, single process)**  
-  Script (example): `benchmarks/<model>/transformers/eval_qwen3_moe_omni_transformers.sh`  
-  Outputs: `benchmark_results/perf_stats.json`, `benchmark_results/results.json`, `benchmark_results/audio/` (if audio is produced).
-
-- **vLLM-Omni end-to-end pipeline**  
-  Script (example): `benchmarks/<model>/vllm_omni/eval_qwen3_moe_omni.sh`  
-  Outputs: `vllm_omni/logs/*.stats.jsonl` (per-stage/overall latency & TPS), `vllm_omni/logs/stage*.log`, `vllm_omni/outputs/` (text/audio artifacts).
-
-- **Adding a new task/model**  
-  1) Create `benchmarks/<model>/transformers/` and/or `benchmarks/<model>/vllm_omni/` with scripts referencing your model and dataset.  
-  2) Add a task README describing dataset, configs, and expected outputs.  
-  3) Keep the output/log structure similar for easy comparison (perf_stats/results/audio or text outputs; stats.jsonl/logs for pipeline).
-
-## Metrics to watch
-- **Throughput**: `overall_tps`, `*_tps_avg` per stage.
-- **Latency distribution**: look for long tails in `*.stats.jsonl`.
-- **Quality/completeness**: missing outputs or errors in stage logs indicate pipeline failures or misconfigurations.
-
-## Troubleshooting
-- Verify GPU/driver/FlashAttention2 requirements for your chosen model/config.
-- Ensure network access for dataset/model downloads (Google Drive, Hugging Face, etc.).
-- If outputs are missing or slow, inspect per-stage logs and `*.stats.jsonl` for errors, stragglers, or contention.

benchmarks/build_dataset/download_process_data_seedtts.md

@@ -1,82 +0,0 @@
-# Benchmark Dataset Preparation Guide
-
-This guide describes how to download and prepare the SeedTTS test dataset for benchmarking Qwen-Omni models.
-
-## Prerequisites
-
-- Python 3.8+
-- `gdown` for downloading from Google Drive
-- Access to the benchmark scripts
-
-## Steps
-
-### 1. Navigate to the Dataset Directory
-
-```bash
-cd benchmarks/build_dataset
-```
-
-### 2. Install Dependencies
-
-```bash
-pip install gdown
-```
-
-### 3. Download the SeedTTS Test Dataset
-
-Download the dataset from Google Drive:
-
-```bash
-gdown --id 1GlSjVfSHkW3-leKKBlfrjuuTGqQ_xaLP
-```
-
-### 4. Extract the Dataset
-
-```bash
-tar -xf seedtts_testset.tar
-```
-
-### 5. Prepare the Metadata File
-
-Copy the English metadata file to the working directory:
-
-```bash
-cp seedtts_testset/en/meta.lst meta.lst
-```
-
-### 6. Extract Prompts
-
-Extract the first N prompts from the metadata file:
-
-```bash
-# Extract top 100 prompts (adjust -n for different amounts)
-python extract_tts_prompts.py -i meta.lst -o top100.txt -n 100
-```
-
-**Options:**
-- `-i, --input`: Input metadata file (default: `meta.lst`)
-- `-o, --output`: Output prompts file (default: `prompts.txt`)
-- `-n, --num-lines`: Number of prompts to extract (required)
-
-### 7. Clean Up (Optional)
-
-Remove temporary files to save disk space:
-
-```bash
-rm -rf seedtts_testset
-rm seedtts_testset.tar
-rm meta.lst
-```
-
-## Quick Start (All-in-One)
-
-```bash
-# Full setup and benchmark
-cd benchmarks/build_dataset
-pip install gdown
-gdown --id 1GlSjVfSHkW3-leKKBlfrjuuTGqQ_xaLP
-tar -xf seedtts_testset.tar
-cp seedtts_testset/en/meta.lst meta.lst
-python extract_tts_prompts.py -i meta.lst -o top100.txt -n 100
-rm -rf seedtts_testset seedtts_testset.tar meta.lst
-```

benchmarks/build_dataset/extract_tts_prompts.py

@@ -1,73 +0,0 @@
-#!/usr/bin/env python3
-"""
-Extract prompts from meta.lst and save them to a txt file.
-
-Each line in meta.lst has the format:
-ID|prompt_text|audio_path|target_text
-
-This script extracts the prompt_text (second field) from the first N lines.
-"""
-
-import argparse
-from pathlib import Path
-
-
-def extract_prompts(input_file: str, output_file: str, num_lines: int) -> None:
-    """
-    Extract prompts from meta.lst and save to output file.
-
-    Args:
-        input_file: Path to the meta.lst file
-        output_file: Path to the output txt file
-        num_lines: Number of lines to process
-    """
-    prompts = []
-
-    with open(input_file, encoding="utf-8") as f:
-        for i, line in enumerate(f):
-            if i >= num_lines:
-                break
-
-            line = line.strip()
-            if not line:  # Skip empty lines
-                continue
-
-            parts = line.split("|")
-            if len(parts) >= 2:
-                prompt = parts[1]  # The prompt is the second field
-                prompts.append(prompt)
-
-    # Write prompts to output file
-    with open(output_file, "w", encoding="utf-8") as f:
-        for prompt in prompts:
-            f.write(prompt + "\n")
-
-    # Print result stats
-    print(f"Extracted {len(prompts)} prompts from first {num_lines} lines")
-    print(f"Saved to: {output_file}")
-
-
-def main():
-    parser = argparse.ArgumentParser(description="Extract prompts from meta.lst file")
-    parser.add_argument(
-        "-i", "--input", type=str, default="meta.lst", help="Input meta.lst file path (default: meta.lst)"
-    )
-    parser.add_argument(
-        "-o", "--output", type=str, default="prompts.txt", help="Output txt file path (default: prompts.txt)"
-    )
-    parser.add_argument(
-        "-n", "--num-lines", type=int, required=True, help="Number of lines to extract from the beginning"
-    )
-
-    args = parser.parse_args()
-
-    # Check if input file exists
-    if not Path(args.input).exists():
-        print(f"Error: Input file '{args.input}' not found")
-        return
-
-    extract_prompts(args.input, args.output, args.num_lines)
-
-
-if __name__ == "__main__":
-    main()

benchmarks/diffusion/README.md

@@ -1,117 +0,0 @@
-
-# Diffusion Serving Benchmark (Image/Video)
-
-This folder contains an online-serving benchmark script for diffusion models.
-It sends requests to a vLLM OpenAI-compatible endpoint and reports throughput,
-latency percentiles, and optional SLO attainment.
-
-The main entrypoint is:
-
-- `benchmarks/diffusion/diffusion_benchmark_serving.py`
-
-## 1. Quick Start
-
-1. Start the server:
-
-```bash
-vllm serve Qwen/Qwen-Image --omni --port 8099
-```
-
-2. Run a minimal benchmark:
-
-```bash
-python3 benchmarks/diffusion/diffusion_benchmark_serving.py \
-	--base-url http://localhost:8099 \
-	--model Qwen/Qwen-Image \
-	--task t2i \
-	--dataset vbench \
-	--num-prompts 5
-```
-
-**Notes**
-
-- The benchmark talks to `http://<host>:<port>/v1/chat/completions`.
-- If you run the server on another host or port, pass `--base-url` accordingly.
-
-## 2. Supported Datasets
-
-The benchmark supports three dataset modes via `--dataset`:
-
-- `vbench`: Built-in prompt/data loader.
-- `trace`: Heterogeneous request traces (each request can have different resolution/frames/steps).
-- `random`: Synthetic prompts for quick smoke tests.
-
-### VBench dataset
-
-If you use i2v/i2i bench datasets and need auto-download support, you may need:
-
-```bash
-uv pip install gdown
-```
-
-### Trace dataset
-
-Use `--dataset trace` to replay a trace file. The trace can specify per-request fields such as:
-
-- `width`, `height`
-- `num_frames` (video)
-- `num_inference_steps`
-- `seed`, `fps`
-- optional `slo_ms` (per-request SLO target)
-
-By default (when `--dataset-path` is not provided), the script downloads a default trace from
-the HuggingFace dataset repo `asukaqaqzz/Dit_Trace`. The default filename can depend on `--task`
-(e.g., `t2v` uses a video trace).
-
-Current defaults:
-
-- `--task t2i` -> `sd3_trace.txt`
-- `--task t2v` -> `cogvideox_trace.txt`
-
-You can point to your own trace using `--dataset-path`.
-
-## 3. Benchmark Parameters
-
-### Basic flags
-
-- `--base-url`: Server address (the script calls `.../v1/chat/completions`).
-- `--model`: The OpenAI-compatible `model` field.
-- `--task`: Task type (e.g., `t2i`, `t2v`, `i2i`, `i2v`).
-- `--dataset`: Dataset mode (`vbench` / `trace` / `random`).
-- `--num-prompts`: Number of requests to send.
-
-Common optional flags:
-
-- `--output-file`: Write metrics to a JSON file.
-- `--disable-tqdm`: Disable the progress bar.
-
-### Resolution / frames / steps: CLI defaults vs dataset fields
-
-Related flags: `--width`, `--height`, `--num-frames`, `--fps`, `--num-inference-steps`.
-
-- For `vbench` / `random`: these CLI flags act as global defaults for all generated requests.
-- For `trace`: each request can carry its own fields (e.g., `width/height/num_frames/num_inference_steps`).
-
-Precedence rules for `trace` (i.e., what actually gets sent):
-
-- `width/height`: if either `--width` or `--height` is explicitly set, it overrides per-request values from the trace; otherwise per-request values are used when present.
-- `num_frames`: per-request `num_frames` takes precedence; otherwise fall back to `--num-frames`.
-- `num_inference_steps`: per-request `num_inference_steps` takes precedence; otherwise fall back to `--num-inference-steps`.
-
-### SLO, warmup, and max concurrency
-
-Enable SLO evaluation with `--slo`.
-
-- If a request in the trace already has `slo_ms`, that value is used.
-- Otherwise, the script runs warmup requests to infer a base unit time, estimates `expected_ms` by linearly scaling with area/frames/steps, and then sets `slo_ms = expected_ms * --slo-scale`.
-
-Warmup flags:
-
-- `--warmup-requests`: Number of warmup requests.
-- `--warmup-num-inference-steps`: Steps used during warmup.
-- For `--task t2v`: warmup requests are forced to use `num_frames=1` to make warmup faster and less noisy.
-
-Traffic / concurrency flags:
-
-- `--request-rate`: Target request rate (requests/second). If set to `inf`, the script sends all requests immediately.
-- `--max-concurrency`: Max number of in-flight requests (default: `1`). This can hard-cap the achieved QPS: if it is too small, requests will queue behind the semaphore, and both achieved throughput and observed SLO attainment can be skewed.

benchmarks/diffusion/backends.py

@@ -1,198 +0,0 @@
-import base64
-import mimetypes
-import os
-import time
-import uuid
-from dataclasses import dataclass, field
-from typing import Any
-
-import aiohttp
-from tqdm import tqdm
-
-
-@dataclass
-class RequestFuncInput:
-    prompt: str
-    api_url: str
-    model: str
-    width: int | None = None
-    height: int | None = None
-    num_frames: int | None = None
-    num_inference_steps: int | None = None
-    seed: int | None = None
-    fps: int | None = None
-    timestamp: float | None = None
-    slo_ms: float | None = None
-    extra_body: dict[str, Any] = field(default_factory=dict)
-    image_paths: list[str] | None = None
-    request_id: str = field(default_factory=lambda: str(uuid.uuid4()))
-
-
-@dataclass
-class RequestFuncOutput:
-    success: bool = False
-    latency: float = 0.0
-    error: str = ""
-    start_time: float = 0.0
-    response_body: dict[str, Any] = field(default_factory=dict)
-    peak_memory_mb: float = 0.0
-    slo_achieved: bool | None = None
-
-
-def _guess_mime_type(path: str) -> str:
-    mime, _ = mimetypes.guess_type(path)
-    return mime or "application/octet-stream"
-
-
-def _encode_image_as_data_url(path: str) -> str:
-    with open(path, "rb") as f:
-        encoded = base64.b64encode(f.read()).decode("utf-8")
-    mime = _guess_mime_type(path)
-    return f"data:{mime};base64,{encoded}"
-
-
-async def async_request_chat_completions(
-    input: RequestFuncInput,
-    session: aiohttp.ClientSession,
-    pbar: tqdm | None = None,
-) -> RequestFuncOutput:
-    output = RequestFuncOutput()
-    output.start_time = time.perf_counter()
-
-    extra_body = dict(input.extra_body)
-    if input.width and input.height:
-        extra_body.setdefault("height", input.height)
-        extra_body.setdefault("width", input.width)
-    if input.num_frames:
-        extra_body.setdefault("num_frames", input.num_frames)
-    if input.num_inference_steps:
-        extra_body.setdefault("num_inference_steps", input.num_inference_steps)
-    if input.seed is not None:
-        extra_body.setdefault("seed", input.seed)
-    if input.fps:
-        extra_body.setdefault("fps", input.fps)
-
-    if input.image_paths and len(input.image_paths) > 0:
-        content = []
-        if input.prompt:
-            content.append({"type": "text", "text": input.prompt})
-        for img_path in input.image_paths:
-            if not os.path.exists(img_path):
-                output.error = f"Image file not found: {img_path}"
-                output.success = False
-                if pbar:
-                    pbar.update(1)
-                return output
-            content.append(
-                {
-                    "type": "image_url",
-                    "image_url": {"url": _encode_image_as_data_url(img_path)},
-                }
-            )
-        messages = [{"role": "user", "content": content}]
-    else:
-        messages = [{"role": "user", "content": input.prompt}]
-
-    payload = {
-        "model": input.model,
-        "messages": messages,
-    }
-    if extra_body:
-        payload["extra_body"] = extra_body
-
-    try:
-        async with session.post(input.api_url, json=payload) as response:
-            if response.status == 200:
-                resp_json = await response.json()
-                output.response_body = resp_json
-                output.success = True
-                if "peak_memory_mb" in resp_json:
-                    output.peak_memory_mb = resp_json["peak_memory_mb"]
-            else:
-                output.error = f"HTTP {response.status}: {await response.text()}"
-                output.success = False
-    except Exception as e:
-        output.error = str(e)
-        output.success = False
-
-    output.latency = time.perf_counter() - output.start_time
-
-    if output.success and input.slo_ms is not None:
-        output.slo_achieved = (output.latency * 1000.0) <= float(input.slo_ms)
-
-    if pbar:
-        pbar.update(1)
-    return output
-
-
-async def async_request_openai_images(
-    input: RequestFuncInput,
-    session: aiohttp.ClientSession,
-    pbar: tqdm | None = None,
-) -> RequestFuncOutput:
-    """
-    Send request to OpenAI's /v1/images/generations endpoint.
-    """
-    output = RequestFuncOutput()
-    output.start_time = time.perf_counter()
-
-    # Build size string from width/height
-    width = input.width or 1024
-    height = input.height or 1024
-    size = f"{width}x{height}"
-
-    payload: dict[str, Any] = {
-        "model": input.model,
-        "prompt": input.prompt,
-        "n": 1,
-        "size": size,
-        "response_format": "b64_json",
-    }
-
-    # Add optional parameters
-    if input.seed is not None:
-        payload["seed"] = input.seed
-    if input.num_inference_steps is not None:
-        payload["num_inference_steps"] = input.num_inference_steps
-
-    # Add any extra body parameters
-    if input.extra_body:
-        for key, value in input.extra_body.items():
-            if key not in payload:
-                payload[key] = value
-
-    headers = {
-        "Content-Type": "application/json",
-        "Authorization": "Bearer EMPTY",
-    }
-
-    try:
-        async with session.post(input.api_url, json=payload, headers=headers) as response:
-            if response.status == 200:
-                resp_json = await response.json()
-                output.response_body = resp_json
-                output.success = True
-                # Check for usage/memory info if available
-                if "usage" in resp_json and "peak_memory_mb" in resp_json.get("usage", {}):
-                    output.peak_memory_mb = resp_json["usage"]["peak_memory_mb"]
-            else:
-                output.error = f"HTTP {response.status}: {await response.text()}"
-                output.success = False
-    except Exception as e:
-        output.error = str(e)
-        output.success = False
-
-    output.latency = time.perf_counter() - output.start_time
-
-    if output.success and input.slo_ms is not None:
-        output.slo_achieved = (output.latency * 1000.0) <= float(input.slo_ms)
-
-    if pbar:
-        pbar.update(1)
-    return output
-
-
-backends_function_mapping = {
-    "vllm-omni": (async_request_chat_completions, "/v1/chat/completions"),
-    "openai": (async_request_openai_images, "/v1/images/generations"),
-}

benchmarks/diffusion/diffusion_benchmark_serving.py

@@ -1,978 +0,0 @@
-# adapted from sglang and fastvideo
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-
-"""
-Benchmark online serving for diffusion models (Image/Video Generation).
-If you want to use i2v, i2i dataset, you should `uv pip install gdown` first
-
-Supports multiple backends:
-    - vllm-omni: Uses /v1/chat/completions endpoint (default)
-    - openai: Uses /v1/images/generations endpoint
-
-Usage:
-    # Video (vllm-omni backend)
-    t2v:
-    python3 benchmarks/diffusion/diffusion_benchmark_serving.py \
-        --backend vllm-omni --dataset vbench --task t2v --num-prompts 10 \
-        --height 480 --width 640 --fps 16 --num-frames 80
-
-    i2v:
-    python3 benchmarks/diffusion/diffusion_benchmark_serving.py \
-        --backend vllm-omni --dataset vbench --task i2v --num-prompts 10
-
-
-    # Image (vllm-omni backend)
-    t2i:
-    python3 benchmarks/diffusion/diffusion_benchmark_serving.py \
-        --backend vllm-omni --dataset vbench --task t2i --num-prompts 10 \
-        --height 1024 --width 1024
-
-    i2i:
-    python3 benchmarks/diffusion/diffusion_benchmark_serving.py \
-        --backend vllm-omni --dataset vbench --task i2i --num-prompts 10
-
-    # Image (openai backend)
-    t2i:
-    python3 benchmarks/diffusion/diffusion_benchmark_serving.py \
-        --backend openai --dataset vbench --task t2i --num-prompts 10 \
-        --height 1024 --width 1024 --port 3000
-
-"""
-
-import argparse
-import ast
-import asyncio
-import glob
-import json
-import os
-import time
-import uuid
-from abc import ABC, abstractmethod
-from collections.abc import AsyncGenerator
-from dataclasses import replace
-from typing import Any
-
-import aiohttp
-import numpy as np
-import requests
-from backends import RequestFuncInput, RequestFuncOutput, backends_function_mapping
-from tqdm.asyncio import tqdm
-
-
-class BaseDataset(ABC):
-    def __init__(self, args, api_url: str, model: str):
-        self.args = args
-        self.api_url = api_url
-        self.model = model
-
-    @abstractmethod
-    def __len__(self) -> int:
-        pass
-
-    @abstractmethod
-    def __getitem__(self, idx: int) -> RequestFuncInput:
-        pass
-
-    @abstractmethod
-    def get_requests(self) -> list[RequestFuncInput]:
-        pass
-
-
-class VBenchDataset(BaseDataset):
-    """
-    Dataset loader for VBench prompts.
-    Supports t2v, i2v.
-    """
-
-    T2V_PROMPT_URL = (
-        "https://raw.githubusercontent.com/Vchitect/VBench/master/prompts/prompts_per_dimension/subject_consistency.txt"
-    )
-    I2V_DOWNLOAD_SCRIPT_URL = (
-        "https://raw.githubusercontent.com/Vchitect/VBench/master/vbench2_beta_i2v/download_data.sh"
-    )
-
-    def __init__(self, args, api_url: str, model: str):
-        super().__init__(args, api_url, model)
-        self.cache_dir = os.path.join(os.path.expanduser("~"), ".cache", "vllm-omni")
-        self.items = self._load_data()
-
-    def _load_data(self) -> list[dict[str, Any]]:
-        if self.args.task == "t2v":
-            return self._load_t2v_prompts()
-        elif self.args.task in ["i2v", "ti2v", "ti2i", "i2i"]:
-            return self._load_i2v_data()
-        else:
-            return self._load_t2v_prompts()
-
-    def _download_file(self, url: str, dest_path: str) -> None:
-        """Download a file from URL to destination path."""
-        os.makedirs(os.path.dirname(dest_path), exist_ok=True)
-        resp = requests.get(url)
-        resp.raise_for_status()
-        with open(dest_path, "w") as f:
-            f.write(resp.text)
-
-    def _load_t2v_prompts(self) -> list[dict[str, Any]]:
-        path = self.args.dataset_path
-
-        if not path:
-            path = os.path.join(self.cache_dir, "vbench_subject_consistency.txt")
-            if not os.path.exists(path):
-                print(f"Downloading VBench T2V prompts to {path}...")
-                try:
-                    self._download_file(self.T2V_PROMPT_URL, path)
-                except Exception as e:
-                    print(f"Failed to download VBench prompts: {e}")
-                    return [{"prompt": "A cat sitting on a bench"}] * 50
-
-        prompts = []
-        with open(path) as f:
-            for line in f:
-                line = line.strip()
-                if line:
-                    prompts.append({"prompt": line})
-
-        return self._resize_data(prompts)
-
-    def _auto_download_i2v_dataset(self) -> str:
-        """Auto-download VBench I2V dataset and return the dataset directory."""
-        vbench_i2v_dir = os.path.join(self.cache_dir, "vbench_i2v", "vbench2_beta_i2v")
-        info_json_path = os.path.join(vbench_i2v_dir, "data", "i2v-bench-info.json")
-
-        if os.path.exists(info_json_path):
-            return vbench_i2v_dir
-
-        print(f"Downloading VBench I2V dataset to {vbench_i2v_dir}...")
-        try:
-            cache_root = os.path.join(self.cache_dir, "vbench_i2v")
-            script_path = os.path.join(cache_root, "download_data.sh")
-
-            self._download_file(self.I2V_DOWNLOAD_SCRIPT_URL, script_path)
-            os.chmod(script_path, 0o755)
-
-            print("Executing download_data.sh (this may take a while)...")
-            import subprocess
-
-            result = subprocess.run(
-                ["bash", script_path],
-                cwd=cache_root,
-                capture_output=True,
-                text=True,
-            )
-
-            if result.returncode != 0:
-                raise RuntimeError(f"Download script failed: {result.stderr}")
-
-            print(f"Successfully downloaded VBench I2V dataset to {vbench_i2v_dir}")
-        except Exception as e:
-            print(f"Failed to download VBench I2V dataset: {e}")
-            print("Please manually download following instructions at:")
-            print("https://github.com/Vchitect/VBench/tree/master/vbench2_beta_i2v#22-download")
-            return None
-
-        return vbench_i2v_dir if os.path.exists(info_json_path) else None
-
-    def _load_from_i2v_json(self, json_path: str) -> list[dict[str, Any]]:
-        """Load I2V data from i2v-bench-info.json format."""
-        with open(json_path) as f:
-            items = json.load(f)
-
-        base_dir = os.path.dirname(os.path.dirname(json_path))  # Go up to vbench2_beta_i2v
-        origin_dir = os.path.join(base_dir, "data", "origin")
-
-        data = []
-        for item in items:
-            img_path = os.path.join(origin_dir, item.get("file_name", ""))
-            if os.path.exists(img_path):
-                data.append({"prompt": item.get("caption", ""), "image_path": img_path})
-            else:
-                print(f"Warning: Image not found: {img_path}")
-
-        print(f"Loaded {len(data)} I2V samples from VBench I2V dataset")
-        return data
-
-    def _scan_directory_for_images(self, path: str) -> list[dict[str, Any]]:
-        """Scan directory for image files."""
-        exts = ["*.jpg", "*.jpeg", "*.png", "*.webp"]
-        files = []
-
-        for ext in exts:
-            files.extend(glob.glob(os.path.join(path, ext)))
-            files.extend(glob.glob(os.path.join(path, ext.upper())))
-
-            # Also check in data/origin subdirectory
-            origin_dir = os.path.join(path, "data", "origin")
-            if os.path.exists(origin_dir):
-                files.extend(glob.glob(os.path.join(origin_dir, ext)))
-                files.extend(glob.glob(os.path.join(origin_dir, ext.upper())))
-
-        return [{"prompt": os.path.splitext(os.path.basename(f))[0], "image_path": f} for f in files]
-
-    def _create_dummy_data(self) -> list[dict[str, Any]]:
-        """Create dummy data with a placeholder image in cache directory."""
-        print("No I2V data found. Using dummy placeholders.")
-
-        dummy_image = os.path.join(self.cache_dir, "dummy_image.jpg")
-        if not os.path.exists(dummy_image):
-            try:
-                from PIL import Image
-
-                os.makedirs(self.cache_dir, exist_ok=True)
-                img = Image.new("RGB", (100, 100), color="red")
-                img.save(dummy_image)
-                print(f"Created dummy image at {dummy_image}")
-            except ImportError:
-                print("PIL not installed, cannot create dummy image.")
-                return []
-
-        return [{"prompt": "A moving cat", "image_path": dummy_image}] * 10
-
-    def _load_i2v_data(self) -> list[dict[str, Any]]:
-        """Load I2V data from VBench I2V dataset or user-provided path."""
-        path = self.args.dataset_path
-
-        # Auto-download if no path provided
-        if not path:
-            path = self._auto_download_i2v_dataset()
-            if not path:
-                return self._resize_data(self._create_dummy_data())
-
-        # Try to load from i2v-bench-info.json
-        info_json_candidates = [
-            os.path.join(path, "data", "i2v-bench-info.json"),
-            path if path.endswith(".json") else None,
-        ]
-
-        for json_path in info_json_candidates:
-            if json_path and os.path.exists(json_path):
-                try:
-                    return self._resize_data(self._load_from_i2v_json(json_path))
-                except Exception as e:
-                    print(f"Failed to load {json_path}: {e}")
-
-        # Fallback: scan directory for images
-        if os.path.isdir(path):
-            data = self._scan_directory_for_images(path)
-            if data:
-                return self._resize_data(data)
-
-        # Last resort: dummy data
-        return self._resize_data(self._create_dummy_data())
-
-    def _resize_data(self, data: list[dict[str, Any]]) -> list[dict[str, Any]]:
-        """Resize data to match num_prompts."""
-        if not self.args.num_prompts:
-            return data
-
-        if len(data) < self.args.num_prompts:
-            factor = (self.args.num_prompts // len(data)) + 1
-            data = data * factor
-
-        return data[: self.args.num_prompts]
-
-    def __len__(self) -> int:
-        return len(self.items)
-
-    def __getitem__(self, idx: int) -> RequestFuncInput:
-        item = self.items[idx]
-        image_paths = [item["image_path"]] if "image_path" in item else None
-
-        return RequestFuncInput(
-            prompt=item.get("prompt", ""),
-            api_url=self.api_url,
-            model=self.model,
-            width=self.args.width,
-            height=self.args.height,
-            num_frames=self.args.num_frames,
-            num_inference_steps=self.args.num_inference_steps,
-            seed=self.args.seed,
-            fps=self.args.fps,
-            image_paths=image_paths,
-        )
-
-    def get_requests(self) -> list[RequestFuncInput]:
-        return [self[i] for i in range(len(self))]
-
-
-class TraceDataset(BaseDataset):
-    """Trace-based dataset loader for heterogeneous diffusion requests."""
-
-    DEFAULT_REPO_ID = "asukaqaqzz/Dit_Trace"
-    DEFAULT_FILENAME = "sd3_trace.txt"
-    DEFAULT_FILENAME_BY_TASK: dict[str, str] = {
-        # Text-to-image traces (e.g., SD3)
-        "t2i": "sd3_trace.txt",
-        # Text-to-video traces (e.g., CogVideoX)
-        "t2v": "cogvideox_trace.txt",
-    }
-
-    def __init__(self, args, api_url: str, model: str):
-        super().__init__(args, api_url, model)
-        self.cache_dir = os.path.join(os.path.expanduser("~"), ".cache", "vllm-omni", "trace")
-        self.default_filename = self.DEFAULT_FILENAME_BY_TASK.get(getattr(args, "task", ""), self.DEFAULT_FILENAME)
-        dataset_root = args.dataset_path
-        if not dataset_root:
-            dataset_root = self._download_default_trace()
-        self.items = self._load_items(dataset_root)
-
-    @staticmethod
-    def _coerce_int(x: Any) -> int | None:
-        if x is None:
-            return None
-        if isinstance(x, bool):
-            return None
-        if isinstance(x, int):
-            return x
-        try:
-            s = str(x).strip()
-            if not s:
-                return None
-            return int(float(s))
-        except Exception:
-            return None
-
-    @staticmethod
-    def _coerce_float(x: Any) -> float | None:
-        if x is None:
-            return None
-        if isinstance(x, float):
-            return x
-        if isinstance(x, int):
-            return float(x)
-        try:
-            s = str(x).strip()
-            if not s:
-                return None
-            return float(s)
-        except Exception:
-            return None
-
-    def _download_default_trace(self) -> str:
-        """Download default trace file from HuggingFace Hub if not provided."""
-
-        try:
-            from huggingface_hub import hf_hub_download
-        except ImportError as exc:
-            raise ImportError(
-                "huggingface_hub is required to download the default trace dataset. "
-                "Install via `pip install huggingface_hub`."
-            ) from exc
-
-        os.makedirs(self.cache_dir, exist_ok=True)
-        return hf_hub_download(
-            repo_id=self.DEFAULT_REPO_ID,
-            filename=self.default_filename,
-            repo_type="dataset",
-            local_dir=self.cache_dir,
-            local_dir_use_symlinks=False,
-        )
-
-    def _expand_paths(self, dataset_path: str | None) -> list[str]:
-        if not dataset_path:
-            return []
-
-        parts = [p.strip() for p in str(dataset_path).split(",") if p.strip()]
-        paths: list[str] = []
-        for p in parts:
-            if any(ch in p for ch in ["*", "?", "["]):
-                paths.extend(sorted(glob.glob(p)))
-            elif os.path.isdir(p):
-                paths.extend(sorted(glob.glob(os.path.join(p, "**", "*.txt"), recursive=True)))
-            else:
-                paths.append(p)
-
-        seen = set()
-        unique_paths = []
-        for p in paths:
-            if p not in seen:
-                seen.add(p)
-                unique_paths.append(p)
-        return unique_paths
-
-    def _parse_trace_file(self, path: str) -> list[dict[str, Any]]:
-        rows: list[dict[str, Any]] = []
-
-        def parse_request_repr_line(line: str) -> dict[str, Any] | None:
-            text = line.strip()
-            if not text:
-                return None
-            if not (text.startswith("Request(") and text.endswith(")")):
-                return None
-            inner = text[len("Request(") : -1]
-            try:
-                expr = ast.parse(f"f({inner})", mode="eval")
-                if not isinstance(expr.body, ast.Call):
-                    return None
-                call = expr.body
-                out: dict[str, Any] = {}
-                for kw in call.keywords:
-                    if kw.arg is None:
-                        continue
-                    out[kw.arg] = ast.literal_eval(kw.value)
-                return out
-            except Exception:
-                return None
-
-        # detect first non-empty line to pick parser
-        first_non_empty = None
-        with open(path, encoding="utf-8") as f:
-            for _ in range(50):
-                pos = f.tell()
-                line = f.readline()
-                if not line:
-                    break
-                if line.strip():
-                    first_non_empty = line.strip()
-                    f.seek(pos)
-                    break
-
-        if first_non_empty is None:
-            return rows
-
-        if first_non_empty.startswith("Request("):
-            with open(path, encoding="utf-8") as f:
-                for line in f:
-                    parsed = parse_request_repr_line(line)
-                    if isinstance(parsed, dict):
-                        rows.append(parsed)
-            return rows
-
-        # txt fallback: parse Request(...) lines only
-        with open(path, encoding="utf-8") as f:
-            for line in f:
-                parsed = parse_request_repr_line(line)
-                if isinstance(parsed, dict):
-                    rows.append(parsed)
-        return rows
-
-    def _load_items(self, dataset_root: str) -> list[dict[str, Any]]:
-        paths = self._expand_paths(dataset_root)
-        if not paths:
-            raise ValueError("No trace files found. Provide --dataset-path or rely on default HuggingFace download.")
-
-        items: list[dict[str, Any]] = []
-        for p in paths:
-            if not os.path.exists(p):
-                continue
-            for row in self._parse_trace_file(p):
-                if isinstance(row, dict):
-                    row = dict(row)
-                    row.setdefault("_source", p)
-                    items.append(row)
-
-        if not items:
-            raise ValueError("Trace dataset is empty after parsing provided paths.")
-
-        if self.args.num_prompts is not None:
-            items = items[: self.args.num_prompts]
-
-        return items
-
-    def __len__(self) -> int:
-        return len(self.items)
-
-    def __getitem__(self, idx: int) -> RequestFuncInput:
-        row = self.items[idx]
-        prompt = row.get("prompt") or row.get("text") or ""
-
-        row_height = self._coerce_int(row.get("height"))
-        row_width = self._coerce_int(row.get("width"))
-        num_frames = self._coerce_int(row.get("num_frames"))
-        num_steps = self._coerce_int(row.get("num_inference_steps"))
-        seed = self._coerce_int(row.get("seed"))
-        fps = self._coerce_int(row.get("fps"))
-        timestamp = self._coerce_float(row.get("timestamp"))
-        slo_ms = self._coerce_float(row.get("slo_ms"))
-        image_paths = row.get("image_paths")
-
-        override_w = self.args.width
-        override_h = self.args.height
-        if override_w is not None or override_h is not None:
-            width = override_w
-            height = override_h
-        else:
-            width = row_width
-            height = row_height
-
-        return RequestFuncInput(
-            prompt=str(prompt),
-            api_url=self.api_url,
-            model=self.model,
-            width=width,
-            height=height,
-            num_frames=num_frames if num_frames is not None else self.args.num_frames,
-            num_inference_steps=num_steps if num_steps is not None else self.args.num_inference_steps,
-            seed=seed if seed is not None else self.args.seed,
-            fps=fps if fps is not None else self.args.fps,
-            timestamp=timestamp,
-            slo_ms=slo_ms,
-            image_paths=image_paths,
-            request_id=str(row.get("request_id")) if row.get("request_id") is not None else str(uuid.uuid4()),
-        )
-
-    def get_requests(self) -> list[RequestFuncInput]:
-        return [self[i] for i in range(len(self))]
-
-
-class RandomDataset(BaseDataset):
-    def __init__(self, args, api_url: str, model: str):
-        self.args = args
-        self.api_url = api_url
-        self.model = model
-        self.num_prompts = args.num_prompts
-
-    def __len__(self) -> int:
-        return self.num_prompts
-
-    def __getitem__(self, idx: int) -> RequestFuncInput:
-        return RequestFuncInput(
-            prompt=f"Random prompt {idx} for benchmarking diffusion models",
-            api_url=self.api_url,
-            model=self.model,
-            width=self.args.width,
-            height=self.args.height,
-            num_frames=self.args.num_frames,
-            num_inference_steps=self.args.num_inference_steps,
-            seed=self.args.seed,
-            fps=self.args.fps,
-        )
-
-    def get_requests(self) -> list[RequestFuncInput]:
-        return [self[i] for i in range(len(self))]
-
-
-def _compute_expected_latency_ms_from_base(req: RequestFuncInput, args, base_time_ms: float | None) -> float | None:
-    """Compute expected execution time (ms) based on a base per-step-per-frame unit time.
-
-    Assumes linear scaling with pixel area, frame count, and num_inference_steps.
-    The base unit represents latency for a 16x16 resolution, single frame, single step.
-    """
-
-    if base_time_ms is None:
-        return None
-
-    width = req.width if req.width is not None else args.width
-    height = req.height if req.height is not None else args.height
-    if width is None or height is None:
-        return None
-
-    frames = req.num_frames if req.num_frames is not None else args.num_frames
-    steps = req.num_inference_steps if req.num_inference_steps is not None else args.num_inference_steps
-
-    frame_scale = frames if isinstance(frames, int) and frames > 0 else 1
-    step_scale = steps if isinstance(steps, int) and steps > 0 else 1
-
-    area_units = max((float(width) * float(height)) / float(16 * 16), 1.0)
-    return float(base_time_ms) * area_units * frame_scale * step_scale
-
-
-def _infer_slo_base_time_ms_from_warmups(
-    warmup_pairs: list[tuple[RequestFuncInput, RequestFuncOutput]],
-    args,
-) -> float | None:
-    """Infer base SLO unit time from warmup requests.
-
-    Returns the median base latency (ms) for a 16x16 resolution, single-frame,
-    single-step request. Only uses warmups that succeeded and have resolvable
-    width/height.
-    """
-
-    candidates_ms: list[float] = []
-    for req, out in warmup_pairs:
-        if not out.success or out.latency <= 0:
-            continue
-
-        width = req.width if req.width is not None else args.width
-        height = req.height if req.height is not None else args.height
-        if width is None or height is None:
-            continue
-
-        frames = req.num_frames if req.num_frames is not None else args.num_frames
-        steps = req.num_inference_steps if req.num_inference_steps is not None else args.num_inference_steps
-
-        frame_scale = int(frames) if isinstance(frames, int) and frames > 0 else 1
-        step_scale = int(steps) if isinstance(steps, int) and steps > 0 else 1
-
-        area_units = max((float(width) * float(height)) / float(16 * 16), 1.0)
-        denom = area_units * float(frame_scale) * float(step_scale)
-        if denom <= 0:
-            continue
-
-        candidates_ms.append((out.latency * 1000.0) / denom)
-
-    if not candidates_ms:
-        return None
-    return float(np.median(candidates_ms))
-
-
-def _populate_slo_ms_from_warmups(
-    requests_list: list[RequestFuncInput],
-    warmup_pairs: list[tuple[RequestFuncInput, RequestFuncOutput]],
-    args,
-) -> list[RequestFuncInput]:
-    """Populate missing RequestFuncInput.slo_ms using warmup outputs.
-
-    - If a request already has slo_ms (e.g., trace-provided), it is kept as-is.
-    - If any request has slo_ms is None and we can infer base time from warmups,
-      we estimate each missing request's expected execution time and set:
-        req.slo_ms = expected_latency_ms * args.slo_scale
-
-    Returns updated requests_list.
-    """
-
-    if not any(req.slo_ms is None for req in requests_list):
-        return requests_list
-
-    base_time_ms = _infer_slo_base_time_ms_from_warmups(warmup_pairs, args)
-    if base_time_ms is None:
-        return requests_list
-
-    slo_scale = float(getattr(args, "slo_scale", 3.0))
-    if slo_scale <= 0:
-        raise ValueError(f"slo_scale must be positive, got {slo_scale}.")
-
-    updated: list[RequestFuncInput] = []
-    for req in requests_list:
-        if req.slo_ms is not None:
-            updated.append(req)
-            continue
-        expected_ms = _compute_expected_latency_ms_from_base(req, args, base_time_ms)
-        updated.append(replace(req, slo_ms=(expected_ms * slo_scale) if expected_ms is not None else None))
-
-    return updated
-
-
-async def iter_requests(
-    requests_list: list[RequestFuncInput],
-    request_rate: float,
-) -> AsyncGenerator[RequestFuncInput, None]:
-    """Yield requests using a fixed interval if request_rate is set.
-
-    - If request_rate is inf, all requests are yielded immediately (no sleep).
-    - Otherwise, requests are emitted at a fixed cadence of 1 / request_rate seconds.
-    """
-
-    if request_rate != float("inf"):
-        if request_rate <= 0:
-            raise ValueError(f"request_rate must be positive or inf, got {request_rate}.")
-        interval_s = 1.0 / float(request_rate)
-
-    for i, req in enumerate(requests_list):
-        if request_rate != float("inf") and i > 0:
-            await asyncio.sleep(interval_s)
-        yield req
-
-
-def calculate_metrics(
-    outputs: list[RequestFuncOutput],
-    total_duration: float,
-    requests_list: list[RequestFuncInput],
-    args,
-    slo_enabled: bool,
-):
-    success_outputs = [o for o in outputs if o.success]
-    error_outputs = [o for o in outputs if not o.success]
-
-    num_success = len(success_outputs)
-    latencies = [o.latency for o in success_outputs]
-    peak_memories = [o.peak_memory_mb for o in success_outputs if o.peak_memory_mb > 0]
-
-    metrics = {
-        "duration": total_duration,
-        "completed_requests": num_success,
-        "failed_requests": len(error_outputs),
-        "throughput_qps": num_success / total_duration if total_duration > 0 else 0,
-        "latency_mean": np.mean(latencies) if latencies else 0,
-        "latency_median": np.median(latencies) if latencies else 0,
-        "latency_p99": np.percentile(latencies, 99) if latencies else 0,
-        "latency_p50": np.percentile(latencies, 50) if latencies else 0,
-        "peak_memory_mb_max": max(peak_memories) if peak_memories else 0,
-        "peak_memory_mb_mean": np.mean(peak_memories) if peak_memories else 0,
-        "peak_memory_mb_median": np.median(peak_memories) if peak_memories else 0,
-    }
-
-    if slo_enabled:
-        slo_defined_total = 0
-        slo_met_success = 0
-
-        for req, out in zip(requests_list, outputs):
-            if req.slo_ms is None:
-                continue
-            slo_defined_total += 1
-            if out.slo_achieved is None:
-                continue
-            if out.slo_achieved:
-                slo_met_success += 1
-
-        slo_attain_all = (slo_met_success / slo_defined_total) if slo_defined_total > 0 else 0.0
-
-        metrics.update(
-            {
-                "slo_attainment_rate": slo_attain_all,
-                "slo_met_success": slo_met_success,
-                "slo_scale": getattr(args, "slo_scale", 3.0),
-            }
-        )
-
-    return metrics
-
-
-def wait_for_service(base_url: str, timeout: int = 120) -> None:
-    print(f"Waiting for service at {base_url}...")
-    start_time = time.time()
-    while True:
-        try:
-            # Try /health endpoint first
-            resp = requests.get(f"{base_url}/health", timeout=1)
-            if resp.status_code == 200:
-                print("Service is ready.")
-                break
-        except requests.exceptions.RequestException:
-            pass
-
-        if time.time() - start_time > timeout:
-            raise TimeoutError(f"Service at {base_url} did not start within {timeout} seconds.")
-
-        time.sleep(1)
-
-
-async def benchmark(args):
-    # Construct base_url if not provided
-    if args.base_url is None:
-        args.base_url = f"http://{args.host}:{args.port}"
-
-    # Setup API URL and request function based on backend
-    request_func, api_url = backends_function_mapping[args.backend]
-    api_url = f"{args.base_url}{api_url}"
-
-    if args.dataset == "vbench":
-        dataset = VBenchDataset(args, api_url, args.model)
-    elif args.dataset == "trace":
-        dataset = TraceDataset(args, api_url, args.model)
-    elif args.dataset == "random":
-        dataset = RandomDataset(args, api_url, args.model)
-    else:
-        raise ValueError(f"Unknown dataset: {args.dataset}")
-
-    print("Loading requests...")
-    requests_list = dataset.get_requests()
-    print(f"Prepared {len(requests_list)} requests from {args.dataset} dataset.")
-
-    # Limit concurrency
-    if args.max_concurrency is not None:
-        semaphore = asyncio.Semaphore(args.max_concurrency)
-    else:
-        semaphore = None
-
-    async def limited_request_func(req, session, pbar):
-        if semaphore:
-            async with semaphore:
-                return await request_func(req, session, pbar)
-        else:
-            return await request_func(req, session, pbar)
-
-    # Run benchmark
-    pbar = tqdm(total=len(requests_list), disable=args.disable_tqdm)
-
-    async with aiohttp.ClientSession() as session:
-        warmup_pairs: list[tuple[RequestFuncInput, RequestFuncOutput]] = []
-        if args.warmup_requests and requests_list:
-            print(
-                f"Running {args.warmup_requests} warmup request(s) \
-                with num_inference_steps={args.warmup_num_inference_steps}..."
-            )
-            for i in range(args.warmup_requests):
-                warm_req = requests_list[i % len(requests_list)]
-                if args.warmup_num_inference_steps is not None:
-                    warm_req = replace(
-                        warm_req,
-                        num_inference_steps=args.warmup_num_inference_steps,
-                    )
-                warm_out = await limited_request_func(warm_req, session, None)
-                warmup_pairs.append((warm_req, warm_out))
-
-        if args.slo:
-            # Prefer trace-provided per-request slo_ms. Only populate when missing.
-            requests_list = _populate_slo_ms_from_warmups(
-                requests_list=requests_list,
-                warmup_pairs=warmup_pairs,
-                args=args,
-            )
-
-        start_time = time.perf_counter()
-        tasks = []
-        async for req in iter_requests(requests_list=requests_list, request_rate=args.request_rate):
-            task = asyncio.create_task(limited_request_func(req, session, pbar))
-            tasks.append(task)
-
-        outputs = await asyncio.gather(*tasks)
-        total_duration = time.perf_counter() - start_time
-
-    pbar.close()
-
-    # Calculate metrics
-    metrics = calculate_metrics(outputs, total_duration, requests_list, args, args.slo)
-
-    # Add configuration info to metrics for JSON output
-    metrics["backend"] = args.backend
-    metrics["model"] = args.model
-    metrics["dataset"] = args.dataset
-    metrics["task"] = args.task
-
-    print("\n{s:{c}^{n}}".format(s=" Serving Benchmark Result ", n=60, c="="))
-
-    # Section 1: Configuration
-    print("{:<40} {:<15}".format("Backend:", args.backend))
-    print("{:<40} {:<15}".format("Model:", args.model))
-    print("{:<40} {:<15}".format("Dataset:", args.dataset))
-    print("{:<40} {:<15}".format("Task:", args.task))
-
-    # Section 2: Execution & Traffic
-    print(f"{'-' * 50}")
-    print("{:<40} {:<15.2f}".format("Benchmark duration (s):", metrics["duration"]))
-    print("{:<40} {:<15}".format("Request rate:", str(args.request_rate)))
-    print(
-        "{:<40} {:<15}".format(
-            "Max request concurrency:",
-            str(args.max_concurrency) if args.max_concurrency else "not set",
-        )
-    )
-    print("{:<40} {}/{:<15}".format("Successful requests:", metrics["completed_requests"], len(requests_list)))
-
-    # Section 3: Performance Metrics
-    print(f"{'-' * 50}")
-
-    print("{:<40} {:<15.2f}".format("Request throughput (req/s):", metrics["throughput_qps"]))
-    print("{:<40} {:<15.4f}".format("Latency Mean (s):", metrics["latency_mean"]))
-    print("{:<40} {:<15.4f}".format("Latency Median (s):", metrics["latency_median"]))
-    print("{:<40} {:<15.4f}".format("Latency P99 (s):", metrics["latency_p99"]))
-
-    if args.slo:
-        print(f"{'-' * 50}")
-        print("{:<40} {:<15.2%}".format("SLO Attainment Rate (all):", metrics.get("slo_attainment_rate", 0.0)))
-        print("{:<40} {:<15}".format("SLO Met (success count):", str(metrics.get("slo_met_success", 0))))
-        print("{:<40} {:<15}".format("SLO Scale:", str(metrics.get("slo_scale", 3.0))))
-
-    if metrics["peak_memory_mb_max"] > 0:
-        print(f"{'-' * 50}")
-        print("{:<40} {:<15.2f}".format("Peak Memory Max (MB):", metrics["peak_memory_mb_max"]))
-        print("{:<40} {:<15.2f}".format("Peak Memory Mean (MB):", metrics["peak_memory_mb_mean"]))
-        print("{:<40} {:<15.2f}".format("Peak Memory Median (MB):", metrics["peak_memory_mb_median"]))
-
-    print("\n" + "=" * 60)
-
-    if args.output_file:
-        with open(args.output_file, "w") as f:
-            json.dump(metrics, f, indent=2)
-        print(f"Metrics saved to {args.output_file}")
-
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser(description="Benchmark serving for diffusion models.")
-    parser.add_argument(
-        "--base-url",
-        type=str,
-        default=None,
-        help="Base URL of the server (e.g., http://localhost:8091). Overrides host/port.",
-    )
-    parser.add_argument("--host", type=str, default="localhost", help="Server host.")
-    parser.add_argument("--port", type=int, default=8091, help="Server port.")
-    parser.add_argument("--model", type=str, default="default", help="Model name.")
-    parser.add_argument(
-        "--backend",
-        type=str,
-        default="vllm-omni",
-        choices=["vllm-omni", "openai"],
-        help="Backend to target the benchmark to.",
-    )
-    parser.add_argument(
-        "--dataset",
-        type=str,
-        default="vbench",
-        choices=["vbench", "trace", "random"],
-        help="Dataset to use.",
-    )
-    parser.add_argument(
-        "--task",
-        type=str,
-        default="t2v",
-        choices=["t2v", "i2v", "ti2v", "ti2i", "i2i", "t2i"],
-        help="Task type.",
-    )
-    parser.add_argument(
-        "--dataset-path",
-        type=str,
-        default=None,
-        help="Path to local dataset file (optional).",
-    )
-    parser.add_argument("--num-prompts", type=int, default=10, help="Number of prompts to benchmark.")
-    parser.add_argument(
-        "--max-concurrency",
-        type=int,
-        default=1,
-        help="Maximum number of concurrent requests, default to `1`. This can be used "
-        "to help simulate an environment where a higher level component "
-        "is enforcing a maximum number of concurrent requests. While the "
-        "--request-rate argument controls the rate at which requests are "
-        "initiated, this argument will control how many are actually allowed "
-        "to execute at a time. This means that when used in combination, the "
-        "actual request rate may be lower than specified with --request-rate, "
-        "if the server is not processing requests fast enough to keep up.",
-    )
-    parser.add_argument(
-        "--request-rate",
-        type=float,
-        default=float("inf"),
-        help="Number of requests per second. If this is inf, then all the requests are sent at time 0. "
-        "Otherwise, we use Poisson process to synthesize the request arrival times. Default is inf.",
-    )
-    parser.add_argument(
-        "--warmup-requests",
-        type=int,
-        default=1,
-        help="Number of warmup requests to run before measurement.",
-    )
-    parser.add_argument(
-        "--warmup-num-inference-steps",
-        type=int,
-        default=1,
-        help="num_inference_steps used for warmup requests.",
-    )
-    parser.add_argument("--width", type=int, default=None, help="Image/Video width.")
-    parser.add_argument("--height", type=int, default=None, help="Image/Video height.")
-    parser.add_argument("--num-frames", type=int, default=None, help="Number of frames (for video).")
-    parser.add_argument(
-        "--num-inference-steps",
-        type=int,
-        default=50,
-        help="Number of inference steps (for diffusion models).",
-    )
-    parser.add_argument(
-        "--seed",
-        type=int,
-        default=None,
-        help="Random seed (for diffusion models).",
-    )
-    parser.add_argument("--fps", type=int, default=None, help="FPS (for video).")
-    parser.add_argument("--output-file", type=str, default=None, help="Output JSON file for metrics.")
-    parser.add_argument(
-        "--slo",
-        action="store_true",
-        help=(
-            "Enable SLO calculation and reporting. If trace provides per-request slo_ms, it is used. "
-            "Otherwise, warmup request(s) are used to infer expected execution time assuming linear "
-            "scaling by resolution, frames, and steps, then slo_ms = expected_time * --slo-scale."
-        ),
-    )
-    parser.add_argument(
-        "--slo-scale",
-        type=float,
-        default=3.0,
-        help="SLO target multiplier: slo_ms = estimated_exec_time_ms * slo_scale (default: 3).",
-    )
-    parser.add_argument("--disable-tqdm", action="store_true", help="Disable progress bar.")
-
-    args = parser.parse_args()
-
-    asyncio.run(benchmark(args))

benchmarks/distributed/omni_connectors/README.md

@@ -1,397 +0,0 @@
-# RDMA Test Configuration Guide
-
-This document explains how to configure the RDMA environment and run tests for `MooncakeTransferEngineConnector`.
-
-## Table of Contents
-
-- [Docker Container Permissions](#docker-container-permissions)
-- [Single-Node Testing](#single-node-testing)
-- [Multi-Node Testing](#multi-node-testing)
-- [Running Tests](#running-tests)
-- [Cross-Node Testing](#cross-node-testing)
-- [Troubleshooting](#troubleshooting)
-
----
-
-## Docker Container Permissions
-
-RDMA tests require access to InfiniBand/RoCE devices and system topology. Add the following permissions when running `docker run`.
-
-### Option 1: Minimal Permissions (Recommended)
-
-```bash
-docker run -it \
-    --cap-add=SYS_PTRACE \
-    --cap-add=IPC_LOCK \
-    --security-opt seccomp=unconfined \
-    --network=host \
-    --device=/dev/infiniband \
-    -v /sys/class/infiniband:/sys/class/infiniband:ro \
-    your-image:tag
-```
-
-Parameter explanation:
-- `--cap-add=SYS_PTRACE`: Allow reading system topology information
-- `--cap-add=IPC_LOCK`: Allow memory locking (required for RDMA memory registration)
-- `--security-opt seccomp=unconfined`: Disable seccomp restrictions
-- `--network=host`: Use host network (required for RDMA)
-- `--device=/dev/infiniband`: Mount InfiniBand devices
-- `-v /sys/class/infiniband`: Mount IB device info (read-only)
-
-### Option 2: Full Permissions (Quick but not recommended for production)
-
-```bash
-docker run -it \
-    --privileged \
-    --network=host \
-    your-image:tag
-```
-
-`--privileged` grants full host permissions. Suitable for quick testing but not recommended for production.
-
----
-
-## Single-Node Testing
-
-When running single-node tests (producer and consumer on the same machine), ensure they use the **same RDMA device**.
-
-### Problem Background
-
-InfiniBand devices use LID (Local Identifier) for routing. Different devices have different LIDs and cannot communicate directly. If no device is specified, Mooncake may assign different devices to connectors, causing handshake failures.
-
-Common error:
-```
-[Handshake] Failed to modify QP to RTR, check mtu, gid, peer lid, peer qp num: Invalid argument [22]
-```
-
-### Solution
-
-**Method 1: Set Environment Variable (Recommended)**
-
-```bash
-# List available RDMA devices
-ibstat
-
-# Select a device (e.g., mlx5_0)
-export RDMA_DEVICE_NAME='mlx5_0'
-
-# Run tests
-pytest test_mooncake_transfer_engine_rdma.py -v -s
-```
-
-**Method 2: Use RoCE Devices**
-
-If the system has RoCE devices (using IPv4 routing), the test code will automatically detect and prefer them. RoCE device GIDs start with `00:00:00:00:00:00:00:00:00:00:ff:ff` (IPv4-mapped).
-
-**Method 3: Ensure MTU Consistency**
-
-Make sure both endpoints use the same MTU:
-
-```bash
-# Check device MTU
-ibstatus mlx5_0
-```
-
----
-
-## Multi-Node Testing
-
-For multi-node tests, producer and consumer run on different machines connected via InfiniBand switch.
-
-### Prerequisites
-
-1. Both machines have Mooncake and RDMA drivers installed
-2. Both machines are in the same InfiniBand subnet
-3. Switch is properly configured
-
-### Configuration
-
-**Machine A (Producer):**
-
-```bash
-# Set RDMA host IP (InfiniBand interface IP)
-export RDMA_TEST_HOST='10.0.0.1'
-
-# Optional: Specify device
-export RDMA_DEVICE_NAME='mlx5_0'
-```
-
-**Machine B (Consumer):**
-
-```bash
-# Set RDMA host IP
-export RDMA_TEST_HOST='10.0.0.2'
-
-# Optional: Specify device
-export RDMA_DEVICE_NAME='mlx5_0'
-```
-
-### Verify Connectivity
-
-```bash
-# Ping IB interface
-ping 10.0.0.2
-
-# Test RDMA connectivity with ibping
-# On Machine B (server)
-ibping -S
-
-# On Machine A (client)
-ibping -G <Machine_B_GID>
-```
-
----
-
-## Running Tests
-
-### Run All RDMA Tests (Single-Node, fast suite)
-
-Slow tests (large payloads, stress, concurrency integrity) are marked `@pytest.mark.slow`. Use `-m "not slow"` to skip them in quick CI or local fast iteration.
-
-```bash
-cd tests/distributed/omni_connectors
-
-# Fast suite only (excludes slow/stress tests)
-pytest test_mooncake_transfer_engine_rdma.py test_mooncake_transfer_engine_buffer.py -v -s -m "not slow"
-```
-
-### Run Including Slow Tests
-
-```bash
-# Run ALL tests including slow/stress tests
-pytest test_mooncake_transfer_engine_rdma.py test_mooncake_transfer_engine_buffer.py -v -s
-
-# Run ONLY the slow/stress tests
-pytest test_mooncake_transfer_engine_rdma.py test_mooncake_transfer_engine_buffer.py -v -s -m slow
-```
-
-### Run Buffer Management Tests
-
-```bash
-# Fast only
-pytest test_mooncake_transfer_engine_buffer.py -v -s -m "not slow"
-
-# Including allocator invariant tests (double-free, overlap, merge)
-pytest test_mooncake_transfer_engine_buffer.py -v -s
-```
-
-### Run Specific Test Classes
-
-```bash
-# Basic connector tests
-pytest test_mooncake_transfer_engine_rdma.py::TestBasicConnector -v -s
-
-# End-to-end RDMA transfer tests
-pytest test_mooncake_transfer_engine_rdma.py::TestEndToEnd -v -s
-
-# Lifecycle & resource management tests
-pytest test_mooncake_transfer_engine_rdma.py::TestLifecycle -v -s
-
-# GPU memory pool tests (requires CUDA)
-pytest test_mooncake_transfer_engine_rdma.py::TestGPUPool -v -s
-
-# Stress / correctness tests (slow)
-pytest test_mooncake_transfer_engine_rdma.py::TestStressCorrectness -v -s
-```
-
-### RDMA Environment Diagnostics
-
-For quick diagnostics (device status, Mooncake availability, env vars, etc.),
-see the [Troubleshooting section](../../../docs/design/feature/omni_connectors/mooncake_transfer_engine_connector.md#troubleshooting)
-in the connector documentation.
-
----
-
-## Cross-Node Testing
-
-The `cross_node_mooncake_transfer_engine.py` script enables testing RDMA transfers between two separate physical machines. This script is **not** auto-discovered by `pytest` (it does not start with `test_`) — it must be run manually on each node.
-
-### Prerequisites
-
-1. Both machines have Mooncake installed
-2. Both machines are connected via InfiniBand/RoCE switch
-3. Firewall allows ZMQ ports (default: 15500, 15501)
-4. Same RDMA device name on both nodes (if multiple devices exist)
-
-### Running Cross-Node Tests
-
-**On Machine A (Producer) — start first:**
-
-```bash
-cd benchmarks/distributed/omni_connectors/
-
-# Optional: specify device if multiple exist
-export RDMA_DEVICE_NAME='mlx5_0'
-
-python cross_node_mooncake_transfer_engine.py \
-    --role producer \
-    --local-host <PRODUCER_IP> \
-    --remote-host <CONSUMER_IP> \
-    --tensor-size-mb 100 \
-    --num-transfers 3
-```
-
-**On Machine B (Consumer) — start after producer:**
-
-```bash
-cd benchmarks/distributed/omni_connectors/
-
-export RDMA_DEVICE_NAME='mlx5_0'
-
-python cross_node_mooncake_transfer_engine.py \
-    --role consumer \
-    --local-host <CONSUMER_IP> \
-    --remote-host <PRODUCER_IP> \
-    --tensor-size-mb 100 \
-    --num-transfers 3
-```
-
-### Transfer Modes
-
-| Mode | Description | Example |
-|------|-------------|---------|
-| `copy` | Normal path — tensor copied to RDMA pool (default) | `--mode copy` |
-| `zerocopy` | Zero-copy path — data created directly in RDMA pool | `--mode zerocopy` |
-| `gpu` | GPU transfer — RDMA pool on GPU, uses GPUDirect | `--mode gpu --gpu-id 0` |
-
-### Benchmark Mode
-
-Skip MD5 verification and measure pure RDMA throughput:
-
-```bash
-# Producer
-python cross_node_mooncake_transfer_engine.py \
-    --role producer \
-    --local-host <PRODUCER_IP> \
-    --remote-host <CONSUMER_IP> \
-    --tensor-size-mb 1024 \
-    --num-transfers 20 \
-    --benchmark
-
-# Consumer
-python cross_node_mooncake_transfer_engine.py \
-    --role consumer \
-    --local-host <CONSUMER_IP> \
-    --remote-host <PRODUCER_IP> \
-    --tensor-size-mb 1024 \
-    --num-transfers 20 \
-    --benchmark
-```
-
-### Cross-Node Test Options
-
-| Option | Description | Default |
-|--------|-------------|---------|
-| `--role` | `producer` or `consumer` | Required |
-| `--local-host` | Local RDMA IP address | Required |
-| `--remote-host` | Remote RDMA IP address | Required |
-| `--local-port` | Local ZMQ port for RDMA data | 15500 |
-| `--remote-port` | Remote ZMQ port for RDMA data | 15500 |
-| `--ctrl-port` | Control channel port | 15501 |
-| `--tensor-size-mb` | Tensor size in MB | 100 |
-| `--num-transfers` | Number of transfers | 3 |
-| `--mode` | `copy`, `zerocopy`, or `gpu` | `copy` |
-| `--gpu-id` | GPU ID for GPU mode | 0 |
-| `--benchmark` | Skip MD5, pure performance test | off |
-
----
-
-## Troubleshooting
-
-### 1. "Failed to modify QP to RTR" Error
-
-**Cause**: QP handshake failed, usually due to device configuration mismatch.
-
-**Solution**:
-```bash
-# Force using the same device
-export RDMA_DEVICE_NAME='mlx5_0'
-```
-
-### 2. "Mooncake TransferEngine is not available"
-
-**Cause**: Mooncake not installed or import failed.
-
-**Solution**:
-```bash
-# Check Mooncake installation
-python -c "from mooncake.engine import TransferEngine; print('OK')"
-
-# Reinstall if needed
-pip install mooncake-transfer-engine
-# Or using uv
-uv pip install mooncake-transfer-engine
-
-```
-
-### 3. "Permission denied" accessing /dev/infiniband
-
-**Cause**: Container lacks IB device access permissions.
-
-**Solution**:
-```bash
-docker run --device=/dev/infiniband --cap-add=IPC_LOCK ...
-```
-
-### 4. Test Timeout
-
-**Cause**: RDMA connection establishment failed or network latency.
-
-**Solution**:
-```bash
-# Check network status
-ibstat
-ibstatus
-```
-
-### 5. GPU Test Failed "CUDA is not available"
-
-**Cause**: CUDA environment not configured or GPU unavailable.
-
-**Solution**:
-```bash
-# Check CUDA
-python -c "import torch; print(torch.cuda.is_available())"
-
-# Docker needs NVIDIA runtime
-docker run --gpus all ...
-```
-
----
-
-## Environment Variables Reference
-
-| Variable | Description | Example |
-|----------|-------------|---------|
-| `RDMA_DEVICE_NAME` | Specify RDMA device name | `mlx5_0` |
-| `RDMA_TEST_HOST` | Specify test host IP | `10.0.0.1` |
-| `MC_TE_METRIC` | Enable Mooncake metrics | `1` |
-| `MC_IB_PCI_RELAXED_ORDERING` | Enable PCIe relaxed ordering | `1` |
-
----
-
-## Test Files Overview
-
-| File | Description | Auto-discovered by pytest |
-|------|-------------|--------------------------|
-| `test_mooncake_transfer_engine_rdma.py` | Integration tests for MooncakeTransferEngineConnector (basic, E2E, lifecycle, GPU) | Yes |
-| `test_mooncake_transfer_engine_buffer.py` | Memory pool and buffer management unit tests | Yes |
-| `cross_node_mooncake_transfer_engine.py` | Cross-node (multi-machine) testing script — run manually | No (filename does not start with `test_`) |
-
-### test_mooncake_transfer_engine_rdma.py — Test Classes
-
-| Test Class | Memory Pool | Marker | Description |
-|------------|-------------|--------|-------------|
-| `TestBasicConnector` | CPU | — | Initialization, put tensor/bytes/object, cleanup, pool exhaustion |
-| `TestEndToEnd` | CPU | — | E2E RDMA transfer: tensor, bytes, object, zero-copy, large payload (100MB), mixed types, concurrency |
-| `TestLifecycle` | CPU | — | Close, context manager, double-close safety |
-| `TestGPUPool` | GPU | — | GPU pool init, put CPU/GPU tensor, GPU E2E transfer |
-| `TestStressCorrectness` | CPU | `slow` | Concurrent put+get with MD5 integrity, bidirectional concurrency, edge cases (1-element tensor, empty bytes), 500MB payload, rapid alloc/free cycles |
-
-### test_mooncake_transfer_engine_buffer.py — Test Classes
-
-| Test Class | Marker | Description |
-|------------|--------|-------------|
-| `TestBufferAllocator` | — | Basic alloc/free, alignment, exhaustion/recovery, thread safety |
-| `TestAllocatorInvariants` | `slow` | Double-free safety, overlap corruption detection, adjacent-block merging, fragmentation/defrag |
-| `TestManagedBuffer` | — | Tensor views, context manager |

benchmarks/distributed/omni_connectors/cross_node_mooncake_transfer_engine.py

@@ -1,644 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-
-"""
-Cross-Node RDMA Test Script (Automated Version)
-
-This script enables testing RDMA transfers between two separate machines.
-Supports three transfer modes:
-  - copy:      Normal path - tensor copied to RDMA pool (default)
-  - zerocopy:  Zero-copy path - data created directly in RDMA pool
-  - gpu:       GPU transfer - RDMA pool on GPU, uses GPUDirect
-
-Usage:
-    # On Machine A (Producer) - start first:
-    python cross_node_mooncake_transfer_engine.py --role producer --local-host hostname_A --remote-host hostname_B
-
-    # On Machine B (Consumer) - start after producer:
-    python cross_node_mooncake_transfer_engine.py --role consumer --local-host hostname_B --remote-host hostname_A
-
-    # Zero-copy mode:
-    python cross_node_mooncake_transfer_engine.py --role producer ... --mode zerocopy
-
-    # GPU mode (requires GPUDirect RDMA support):
-    python cross_node_mooncake_transfer_engine.py --role producer ... --mode gpu --gpu-id 0
-
-    # Benchmark mode (skip random data generation and MD5 verification,
-    # measures pure RDMA throughput):
-    python cross_node_mooncake_transfer_engine.py --role producer ... --benchmark
-
-Environment Variables:
-    RDMA_DEVICE_NAME:              Specify RDMA device (e.g., mlx5_0)
-    MC_IB_PCI_RELAXED_ORDERING:    Set to 1 to enable PCIe relaxed ordering
-                                   for higher RDMA throughput
-"""
-
-import argparse
-import hashlib
-import os
-import sys
-import time
-from abc import ABC, abstractmethod
-from dataclasses import dataclass
-from typing import Any
-
-import msgspec
-import torch
-import zmq
-
-# Add parent path for imports
-sys.path.insert(
-    0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))))
-)
-
-from vllm_omni.distributed.omni_connectors.connectors.mooncake_transfer_engine_connector import (
-    ManagedBuffer,
-    MooncakeTransferEngineConnector,
-    TransferEngine,
-)
-
-
-def compute_md5(tensor: torch.Tensor) -> str:
-    """Compute MD5 checksum of a tensor."""
-    if tensor.is_cuda:
-        tensor = tensor.cpu()
-    data = tensor.contiguous().view(torch.uint8).numpy().tobytes()
-    return hashlib.md5(data).hexdigest()
-
-
-# Control channel message types
-class CtrlMsg(msgspec.Struct):
-    """Control channel message."""
-
-    msg_type: str  # "READY", "TRANSFER", "ACK", "DONE", "ERROR"
-    request_id: str = ""
-    md5: str = ""
-    data_size: int = 0
-    error: str = ""
-
-
-@dataclass
-class TransferConfig:
-    """Configuration for cross-node transfer test."""
-
-    local_host: str
-    remote_host: str
-    local_port: int
-    remote_port: int
-    ctrl_port: int
-    num_transfers: int
-    tensor_size_mb: int
-    mode: str  # "copy", "zerocopy", "gpu"
-    gpu_id: int = 0
-    benchmark: bool = False  # Skip MD5 verification for pure performance test
-
-
-@dataclass
-class TransferStats:
-    """Statistics for transfer operations."""
-
-    success_count: int = 0
-    fail_count: int = 0
-    total_bytes: int = 0
-    elapsed_time: float = 0.0
-
-    @property
-    def throughput_mbps(self) -> float:
-        if self.elapsed_time > 0:
-            return (self.total_bytes / (1024 * 1024)) / self.elapsed_time
-        return 0.0
-
-    def print_summary(self, role: str):
-        print(f"\n{'=' * 60}")
-        print(f" {role.upper()} SUMMARY")
-        print(f"  Successful: {self.success_count}/{self.success_count + self.fail_count}")
-        print(f"  Failed:     {self.fail_count}/{self.success_count + self.fail_count}")
-        print(f"  Total:      {self.total_bytes / (1024 * 1024):.2f} MB")
-        print(f"  Time:       {self.elapsed_time:.2f} s")
-        print(f"  Throughput: {self.throughput_mbps:.2f} MB/s")
-        print(f"{'=' * 60}")
-
-
-class CrossNodeTester(ABC):
-    """Abstract base class for cross-node RDMA testing."""
-
-    def __init__(self, config: TransferConfig):
-        self.config = config
-        self.connector: MooncakeTransferEngineConnector | None = None
-        self.zmq_ctx: zmq.Context | None = None
-        self.ctrl_socket: zmq.Socket | None = None
-        self.stats = TransferStats()
-
-    def get_connector_config(self) -> dict:
-        """Get connector configuration based on mode."""
-        pool_size = int(self.config.tensor_size_mb * 1.5) * 1024 * 1024
-        pool_size = max(pool_size, 128 * 1024 * 1024)
-
-        conn_config = {
-            "host": self.config.local_host,
-            "zmq_port": self.config.local_port,
-            "protocol": "rdma",
-            "memory_pool_size": pool_size,
-        }
-
-        # Set device based on mode
-        if self.config.mode == "gpu":
-            conn_config["memory_pool_device"] = f"cuda:{self.config.gpu_id}"
-        else:
-            conn_config["memory_pool_device"] = "cpu"
-
-        # RDMA device name from environment
-        device_name = os.environ.get("RDMA_DEVICE_NAME")
-        if device_name:
-            conn_config["device_name"] = device_name
-            print(f"[CONFIG] Using RDMA device: {device_name}")
-
-        return conn_config
-
-    def initialize(self):
-        """Initialize connector and ZMQ context."""
-        print(f"[{self.role}] Initializing connector...")
-        conn_config = self.get_connector_config()
-        self.connector = MooncakeTransferEngineConnector(conn_config)
-        self.zmq_ctx = zmq.Context()
-        print(f"[{self.role}] Ready at {self.config.local_host}:{self.config.local_port}")
-
-    def cleanup(self):
-        """Cleanup resources."""
-        if self.ctrl_socket:
-            self.ctrl_socket.close()
-        if self.zmq_ctx:
-            self.zmq_ctx.term()
-        if self.connector:
-            self.connector.close()
-        print(f"[{self.role}] Closed.")
-
-    @property
-    @abstractmethod
-    def role(self) -> str:
-        pass
-
-    @abstractmethod
-    def run(self):
-        pass
-
-
-class Producer(CrossNodeTester):
-    """Producer node - sends data to consumer."""
-
-    @property
-    def role(self) -> str:
-        return "PRODUCER"
-
-    def print_header(self):
-        print(f"\n{'=' * 60}")
-        print(f" PRODUCER MODE ({self.config.mode.upper()})")
-        print(f" Local:  {self.config.local_host}:{self.config.local_port}")
-        print(f" Remote: {self.config.remote_host}:{self.config.remote_port}")
-        print(f" Control Port: {self.config.ctrl_port}")
-        print(f" Transfer Mode: {self.config.mode}")
-        if self.config.mode == "gpu":
-            print(f" GPU ID: {self.config.gpu_id}")
-        print(f"{'=' * 60}\n")
-
-    def setup_control_channel(self):
-        """Setup ZMQ control channel as server (REP socket)."""
-        self.ctrl_socket = self.zmq_ctx.socket(zmq.REP)
-        self.ctrl_socket.bind(f"tcp://*:{self.config.ctrl_port}")
-        print(f"[PRODUCER] Control channel listening on port {self.config.ctrl_port}")
-        print("[PRODUCER] Waiting for consumer to connect...")
-
-    def wait_for_consumer(self) -> bool:
-        """Wait for consumer READY signal."""
-        msg_data = self.ctrl_socket.recv()
-        msg = msgspec.msgpack.decode(msg_data, type=CtrlMsg)
-        if msg.msg_type != "READY":
-            print(f"[PRODUCER] Unexpected message: {msg.msg_type}")
-            return False
-        print("[PRODUCER] Consumer connected!")
-        self.ctrl_socket.send(msgspec.msgpack.encode(CtrlMsg(msg_type="ACK")))
-        return True
-
-    def create_test_data(self, transfer_idx: int) -> tuple[Any, str, int]:
-        """
-        Create test data based on transfer mode.
-
-        Returns:
-            (data, md5, size) tuple
-        """
-        num_elements = (self.config.tensor_size_mb * 1024 * 1024) // 4
-        data_size = num_elements * 4
-
-        if self.config.mode == "zerocopy":
-            # Zero-Copy Path: Allocate directly from connector's pool
-            offset = self.connector.allocator.alloc(data_size)
-            managed_buf = ManagedBuffer(self.connector.allocator, offset, data_size, self.connector.pool)
-
-            if self.config.benchmark:
-                # In benchmark mode, skip random data generation (use uninitialized memory)
-                return managed_buf, "", data_size
-            else:
-                # Fill buffer with random data using tensor view
-                tensor_view = managed_buf.as_tensor(dtype=torch.float32, shape=(num_elements,))
-                random_data = torch.randn(num_elements, dtype=torch.float32)
-                if tensor_view.is_cuda:
-                    tensor_view.copy_(random_data.to(tensor_view.device))
-                else:
-                    tensor_view.copy_(random_data)
-                md5 = compute_md5(tensor_view)
-                return managed_buf, md5, data_size
-
-        elif self.config.mode == "gpu":
-            # GPU Path: Create tensor on GPU
-            device = f"cuda:{self.config.gpu_id}"
-            if self.config.benchmark:
-                # In benchmark mode, use empty tensor (no random generation)
-                gpu_tensor = torch.empty(num_elements, dtype=torch.float32, device=device)
-                return gpu_tensor, "", data_size
-            else:
-                cpu_tensor = torch.randn(num_elements, dtype=torch.float32)
-                md5 = compute_md5(cpu_tensor)
-                gpu_tensor = cpu_tensor.to(device)
-                return gpu_tensor, md5, data_size
-
-        else:
-            # Copy Path (default): Create regular CPU tensor
-            if self.config.benchmark:
-                # In benchmark mode, use empty tensor (no random generation)
-                tensor = torch.empty(num_elements, dtype=torch.float32)
-                return tensor, "", data_size
-            else:
-                tensor = torch.randn(num_elements, dtype=torch.float32)
-                md5 = compute_md5(tensor)
-                return tensor, md5, data_size
-
-    def do_transfer(self, transfer_idx: int) -> bool:
-        """Perform a single transfer."""
-        req_id = f"cross_node_transfer_{transfer_idx}"
-
-        if not self.config.benchmark:
-            print(f"\n[PRODUCER] Transfer {transfer_idx + 1}/{self.config.num_transfers}")
-
-        # Create test data
-        t0 = time.time()
-        data, md5, data_size = self.create_test_data(transfer_idx)
-        t_create = time.time() - t0
-
-        if not self.config.benchmark:
-            print(f"  Mode: {self.config.mode}")
-            print(f"  Size: {self.config.tensor_size_mb} MB")
-            if md5:
-                print(f"  MD5:  {md5[:16]}...")
-            print(f"  Create time: {t_create * 1000:.1f} ms")
-
-        # Put data
-        t1 = time.time()
-        success, size, metadata = self.connector.put("producer", "consumer", req_id, data)
-        t_put = time.time() - t1
-
-        if not success:
-            print("  [FAIL] Put failed")
-            return False
-
-        if not self.config.benchmark:
-            print(f"  [OK] Put successful, {size} bytes ({t_put * 1000:.1f} ms)")
-
-        # Wait for consumer to request transfer info
-        msg_data = self.ctrl_socket.recv()
-        msg = msgspec.msgpack.decode(msg_data, type=CtrlMsg)
-
-        if msg.msg_type != "READY":
-            print(f"  [ERROR] Unexpected message: {msg.msg_type}")
-            return False
-
-        # Send transfer metadata to consumer
-        transfer_msg = CtrlMsg(
-            msg_type="TRANSFER",
-            request_id=req_id,
-            md5=md5,
-            data_size=data_size,
-        )
-        self.ctrl_socket.send(msgspec.msgpack.encode(transfer_msg))
-
-        # Wait for consumer ACK (this includes RDMA transfer time)
-        t2 = time.time()
-        msg_data = self.ctrl_socket.recv()
-        t_rdma = time.time() - t2
-        msg = msgspec.msgpack.decode(msg_data, type=CtrlMsg)
-
-        success = msg.msg_type == "ACK"
-        if success:
-            if not self.config.benchmark:
-                print(f"  [OK] RDMA transfer complete ({t_rdma * 1000:.1f} ms)")
-            self.stats.success_count += 1
-            self.stats.total_bytes += size
-        else:
-            print(f"  [WARN] Consumer reported error: {msg.error}")
-            self.stats.fail_count += 1
-
-        # Send ACK to allow consumer to continue
-        self.ctrl_socket.send(msgspec.msgpack.encode(CtrlMsg(msg_type="ACK")))
-
-        # Cleanup buffer
-        self.connector.cleanup(req_id)
-
-        return success
-
-    def run(self):
-        """Run the producer."""
-        self.print_header()
-        self.initialize()
-        self.setup_control_channel()
-
-        try:
-            if not self.wait_for_consumer():
-                return
-
-            if self.config.benchmark:
-                print(
-                    f"[BENCHMARK] Running {self.config.num_transfers} "
-                    f"transfers of {self.config.tensor_size_mb} MB each..."
-                )
-
-            start_time = time.time()
-
-            for i in range(self.config.num_transfers):
-                self.do_transfer(i)
-                if self.config.benchmark and (i + 1) % 10 == 0:
-                    elapsed = time.time() - start_time
-                    current_throughput = (self.stats.total_bytes / (1024 * 1024)) / elapsed
-                    print(f"  Progress: {i + 1}/{self.config.num_transfers}, Throughput: {current_throughput:.2f} MB/s")
-
-            self.stats.elapsed_time = time.time() - start_time
-            self.stats.print_summary("PRODUCER")
-
-            # Wait for final consumer message and send DONE
-            self.ctrl_socket.recv()
-            self.ctrl_socket.send(msgspec.msgpack.encode(CtrlMsg(msg_type="DONE")))
-
-        finally:
-            self.cleanup()
-
-
-class Consumer(CrossNodeTester):
-    """Consumer node - receives data from producer."""
-
-    @property
-    def role(self) -> str:
-        return "CONSUMER"
-
-    def print_header(self):
-        print(f"\n{'=' * 60}")
-        print(f" CONSUMER MODE ({self.config.mode.upper()})")
-        print(f" Local:  {self.config.local_host}:{self.config.local_port}")
-        print(f" Remote: {self.config.remote_host}:{self.config.remote_port}")
-        print(f" Control Port: {self.config.ctrl_port}")
-        print(f" Transfer Mode: {self.config.mode}")
-        if self.config.mode == "gpu":
-            print(f" GPU ID: {self.config.gpu_id}")
-        print(f"{'=' * 60}\n")
-
-    def setup_control_channel(self):
-        """Setup ZMQ control channel as client (REQ socket)."""
-        self.ctrl_socket = self.zmq_ctx.socket(zmq.REQ)
-        ctrl_addr = f"tcp://{self.config.remote_host}:{self.config.ctrl_port}"
-        print(f"[CONSUMER] Connecting to producer control channel at {ctrl_addr}...")
-        self.ctrl_socket.connect(ctrl_addr)
-
-    def connect_to_producer(self) -> bool:
-        """Connect to producer and send READY signal."""
-        self.ctrl_socket.send(msgspec.msgpack.encode(CtrlMsg(msg_type="READY")))
-        msg_data = self.ctrl_socket.recv()
-        msg = msgspec.msgpack.decode(msg_data, type=CtrlMsg)
-        if msg.msg_type != "ACK":
-            print(f"[CONSUMER] Unexpected response: {msg.msg_type}")
-            return False
-        print("[CONSUMER] Connected to producer! Starting transfers...")
-        return True
-
-    def do_transfer(self, transfer_idx: int) -> bool:
-        """Perform a single transfer."""
-        if not self.config.benchmark:
-            print(f"\n[CONSUMER] Transfer {transfer_idx + 1}/{self.config.num_transfers}")
-
-        # Request next transfer info
-        self.ctrl_socket.send(msgspec.msgpack.encode(CtrlMsg(msg_type="READY")))
-        msg_data = self.ctrl_socket.recv()
-        msg = msgspec.msgpack.decode(msg_data, type=CtrlMsg)
-
-        if msg.msg_type == "DONE":
-            print("[CONSUMER] Producer signaled completion")
-            return False
-
-        if msg.msg_type != "TRANSFER":
-            print(f"[CONSUMER] Unexpected message: {msg.msg_type}")
-            return False
-
-        req_id = msg.request_id
-        expected_md5 = msg.md5
-        data_size = msg.data_size
-        num_elements = data_size // 4
-
-        if not self.config.benchmark:
-            print(f"  Request ID: {req_id}")
-            if expected_md5:
-                print(f"  Expected MD5: {expected_md5[:16]}...")
-            print(f"  Data Size: {data_size / (1024 * 1024):.2f} MB")
-
-        # Build metadata for get
-        metadata = {
-            "request_id": req_id,
-            "source_host": self.config.remote_host,
-            "source_port": self.config.remote_port,
-            "data_size": data_size,
-            "dtype": "float32",
-            "shape": [num_elements],
-            "is_fast_path": True,
-        }
-
-        if not self.config.benchmark:
-            print(f"  [INFO] Requesting from {self.config.remote_host}:{self.config.remote_port}")
-
-        # Get data with timing
-        t0 = time.time()
-        result = self.connector.get("producer", "consumer", req_id, metadata)
-        t_get = time.time() - t0
-
-        response_msg = CtrlMsg(msg_type="ERROR", error="Get failed")
-
-        if result is not None:
-            recv_buffer, recv_size = result
-            if not self.config.benchmark:
-                print(f"  [OK] Get successful, {recv_size} bytes ({t_get * 1000:.1f} ms)")
-
-            if isinstance(recv_buffer, ManagedBuffer):
-                # In benchmark mode, skip MD5 verification
-                if self.config.benchmark or not expected_md5:
-                    response_msg = CtrlMsg(msg_type="ACK")
-                    self.stats.success_count += 1
-                    self.stats.total_bytes += recv_size
-                else:
-                    # Verify data
-                    t1 = time.time()
-                    reconstructed = recv_buffer.as_tensor(dtype=torch.float32, shape=(num_elements,))
-                    recv_md5 = compute_md5(reconstructed)
-                    t_md5 = time.time() - t1
-                    print(f"  MD5: {recv_md5[:16]}... ({t_md5 * 1000:.1f} ms)")
-
-                    if recv_md5 == expected_md5:
-                        print("  [PASS] MD5 checksum verified!")
-                        response_msg = CtrlMsg(msg_type="ACK")
-                        self.stats.success_count += 1
-                        self.stats.total_bytes += recv_size
-                    else:
-                        print("  [FAIL] MD5 mismatch!")
-                        response_msg = CtrlMsg(msg_type="ERROR", error="MD5 mismatch")
-                        self.stats.fail_count += 1
-
-                recv_buffer.release()
-            else:
-                response_msg = CtrlMsg(msg_type="ACK")
-                self.stats.success_count += 1
-                self.stats.total_bytes += recv_size
-        else:
-            print("  [FAIL] Get failed")
-            self.stats.fail_count += 1
-
-        # Send response to producer
-        self.ctrl_socket.send(msgspec.msgpack.encode(response_msg))
-        # Wait for ACK
-        self.ctrl_socket.recv()
-
-        return response_msg.msg_type == "ACK"
-
-    def run(self):
-        """Run the consumer."""
-        self.print_header()
-        self.initialize()
-        self.setup_control_channel()
-
-        try:
-            if not self.connect_to_producer():
-                return
-
-            if self.config.benchmark:
-                print(
-                    f"[BENCHMARK] Running {self.config.num_transfers} "
-                    f"transfers of {self.config.tensor_size_mb} MB each..."
-                )
-
-            start_time = time.time()
-
-            for i in range(self.config.num_transfers):
-                if not self.do_transfer(i):
-                    break
-                if self.config.benchmark and (i + 1) % 10 == 0:
-                    elapsed = time.time() - start_time
-                    current_throughput = (self.stats.total_bytes / (1024 * 1024)) / elapsed
-                    print(f"  Progress: {i + 1}/{self.config.num_transfers}, Throughput: {current_throughput:.2f} MB/s")
-
-            self.stats.elapsed_time = time.time() - start_time
-            self.stats.print_summary("CONSUMER")
-
-            # Send final READY and wait for DONE
-            self.ctrl_socket.send(msgspec.msgpack.encode(CtrlMsg(msg_type="READY")))
-            self.ctrl_socket.recv()
-
-        finally:
-            self.cleanup()
-
-
-def main():
-    parser = argparse.ArgumentParser(
-        description="Cross-Node RDMA Test (Automated)",
-        formatter_class=argparse.RawDescriptionHelpFormatter,
-        epilog="""
-Transfer Modes:
-  copy      - Normal path: tensor copied to RDMA pool (default)
-  zerocopy  - Zero-copy path: data created directly in RDMA pool
-  gpu       - GPU transfer: RDMA pool on GPU, uses GPUDirect
-
-Examples:
-  # Copy mode (default):
-  python cross_node_mooncake_transfer_engine.py --role producer \
-  --local-host hostA --remote-host hostB
-
-  # Zero-copy mode:
-  python cross_node_mooncake_transfer_engine.py --role producer \
-  --local-host hostA --remote-host hostB --mode zerocopy
-
-  # GPU mode:
-  python cross_node_mooncake_transfer_engine.py --role producer \
-  --local-host hostA --remote-host hostB --mode gpu --gpu-id 0
-
-  # Benchmark mode (skip MD5, measure pure RDMA performance):
-  python cross_node_mooncake_transfer_engine.py --role producer \
-  --local-host hostA --remote-host hostB --benchmark
-
-  # With specific RDMA device:
-  RDMA_DEVICE_NAME=mlx5_0 python cross_node_mooncake_transfer_engine.py --role producer ...
-        """,
-    )
-
-    parser.add_argument(
-        "--role", required=True, choices=["producer", "consumer"], help="Role: producer (sends) or consumer (receives)"
-    )
-    parser.add_argument("--local-host", required=True, help="Local hostname or IP address")
-    parser.add_argument("--remote-host", required=True, help="Remote hostname or IP address")
-    parser.add_argument("--local-port", type=int, default=15500, help="Local ZMQ port for RDMA data (default: 15500)")
-    parser.add_argument("--remote-port", type=int, default=15500, help="Remote ZMQ port for RDMA data (default: 15500)")
-    parser.add_argument("--ctrl-port", type=int, default=15501, help="Control channel port (default: 15501)")
-    parser.add_argument("--num-transfers", type=int, default=20, help="Number of transfers to perform (default: 3)")
-    parser.add_argument("--tensor-size-mb", type=int, default=100, help="Tensor size in MB (default: 100)")
-    parser.add_argument(
-        "--mode",
-        choices=["copy", "zerocopy", "gpu"],
-        default="copy",
-        help="Transfer mode: copy, zerocopy, or gpu (default: copy)",
-    )
-    parser.add_argument("--gpu-id", type=int, default=0, help="GPU ID for GPU mode (default: 0)")
-    parser.add_argument(
-        "--benchmark", action="store_true", help="Benchmark mode: skip MD5 verification for pure performance test"
-    )
-
-    args = parser.parse_args()
-
-    # Check Mooncake
-    if TransferEngine is None:
-        print("[ERROR] Mooncake TransferEngine is not available.")
-        print("Install with: pip install mooncake")
-        sys.exit(1)
-
-    # Check CUDA for GPU mode
-    if args.mode == "gpu":
-        if not torch.cuda.is_available():
-            print("[ERROR] CUDA is not available but GPU mode was requested.")
-            sys.exit(1)
-        if args.gpu_id >= torch.cuda.device_count():
-            print(f"[ERROR] GPU {args.gpu_id} not available. Found {torch.cuda.device_count()} GPUs.")
-            sys.exit(1)
-        print(f"[INFO] Using GPU {args.gpu_id}: {torch.cuda.get_device_name(args.gpu_id)}")
-
-    config = TransferConfig(
-        local_host=args.local_host,
-        remote_host=args.remote_host,
-        local_port=args.local_port,
-        remote_port=args.remote_port,
-        ctrl_port=args.ctrl_port,
-        num_transfers=args.num_transfers,
-        tensor_size_mb=args.tensor_size_mb,
-        mode=args.mode,
-        gpu_id=args.gpu_id,
-        benchmark=args.benchmark,
-    )
-
-    if args.role == "producer":
-        tester = Producer(config)
-    else:
-        tester = Consumer(config)
-
-    tester.run()
-
-
-if __name__ == "__main__":
-    main()

benchmarks/qwen3-omni/README.md

@@ -1,86 +0,0 @@
-# Benchmarks Guide
-
-This README explains how to (1) prepare benchmark datasets and (2) run the provided Qwen3-Omni benchmarks.
-
-## 1) Prepare the dataset (SeedTTS top100)
-
-```bash
-cd benchmarks/build_dataset
-pip install gdown
-
-# Download SeedTTS test set from Google Drive
-gdown --id 1GlSjVfSHkW3-leKKBlfrjuuTGqQ_xaLP
-
-# Extract
-tar -xf seedtts_testset.tar
-
-# Copy metadata and extract top-100 prompts
-cp seedtts_testset/en/meta.lst meta.lst
-python extract_prompts.py -i meta.lst -o top100.txt -n 100
-
-# (Optional) clean up to save space
-rm -rf seedtts_testset seedtts_testset.tar meta.lst
-```
-
-Artifacts:
-- `benchmarks/build_dataset/top100.txt` — 100 text prompts (one per line).
-
-## 2) Run benchmarks
-
-All commands assume repo root (`vllm-omni`).
-
-### A. Transformers benchmark (offline, HF Transformers)
-
-```
-bash benchmarks/qwen3-omni/transformers/eval_qwen3_moe_omni_transformers.sh
-```
-
-What it does:
-- Runs `qwen3_omni_moe_transformers.py` over `top100.txt` with `--num-prompts 100`.
-- Outputs to `benchmarks/qwen3-omni/transformers/benchmark_results/`:
-  - `perf_stats.json` — aggregated & per-prompt TPS/latency (thinker/talker/code2wav/overall).
-  - `results.json` — per-prompt outputs and audio paths.
-  - `audio/` — ~100 generated `.wav` files.
-
-Key checks:
-- `overall_tps` and `*_tps_avg` should be non-zero and reasonably stable.
-- Investigate any 0/NaN or unusually low TPS / long-tail latency.
-
-### B. vLLM Omni end-to-end benchmark (pipeline)
-
-```
-bash benchmarks/qwen3-omni/vllm_omni/eval_qwen3_moe_omni.sh
-```
-
-What it does:
-- Runs `examples/offline_inference/qwen3_omni/end2end.py` with `--log-stats`.
-- Uses `benchmarks/build_dataset/top100.txt` and writes to:
-  - Logs: `benchmarks/qwen3-omni/vllm_omni/logs/`
-    - `omni_llm_pipeline_text.orchestrator.stats.jsonl` — per-stage latency stats.
-    - `omni_llm_pipeline_text.overall.stats.jsonl` — end-to-end latency/TPS.
-    - `omni_llm_pipeline_text.stage{0,1,2}.log` — per-stage detailed logs/errors.
-  - Outputs: `benchmarks/qwen3-omni/vllm_omni/outputs/` — ~100 text and `.wav` files.
-
-Key checks:
-- Overall stats: end-to-end latency/TPS should be reasonable.
-- Orchestrator stats: per-stage latency should be stable; investigate long tails.
-- Stage logs: ensure no errors and no unusually slow stages.
-
-
-## Performance snapshot
-
-The chart below summarizes our measured Qwen3-Omni MoE end-to-end benchmark, comparing vLLM-Omni against HF Transformers. It shows the overall throughput advantage for vLLM-Omni. These are actual experiment results—please refer to this performance when evaluating or reproducing the benchmark.
-
-![vLLM-Omni vs HF](./vllm-omni-vs-hf.png)
-
-## Directory layout
-- `benchmarks/build_dataset/` — dataset prep utilities (e.g., SeedTTS top100).
-- `benchmarks/<model>/vllm_omni/` — vLLM-Omni pipeline benchmarks, logs, outputs.
-- Add new tasks under `benchmarks/<model>/...` with the same pattern: `transformers/`, `vllm_omni/`, task-specific README, and (optionally) dataset prep notes.
-- `benchmarks/<model>/vllm-omni-vs-hf.png` — current performance snapshot (overall throughput comparison).
-- `benchmarks/<model>/transformers/` — HF Transformers benchmarks (offline reference).
-
-## Troubleshooting
-- Make sure GPU/driver/FlashAttention2 requirements are met for the chosen model.
-- If downloads fail, confirm network access to Google Drive (`gdown`) and Hugging Face.
-- If audio files are missing, check for errors in stage logs or model generation.***

benchmarks/qwen3-omni/transformers/eval_qwen3_moe_omni_transformers.sh

@@ -1,26 +0,0 @@
-#!/bin/bash
-# Qwen3-Omni Transformers Benchmark Evaluation Script
-# This script must be run from the vllm-omni root directory
-
-# Get the directory where this script is located
-SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-
-# Navigate to vllm-omni root directory (4 levels up from script location)
-VLLM_OMNI_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)"
-cd "$VLLM_OMNI_ROOT" || { echo "Error: Failed to navigate to vllm-omni directory"; exit 1; }
-
-echo "Working directory: $(pwd)"
-# Verify we're in the correct directory and run benchmark
-if [[ ! -f "benchmarks/qwen3-omni/transformers/qwen3_omni_moe_transformers.py" ]]; then
-    echo "Error: Not in vllm-omni root directory. Please run from vllm-omni folder."
-else
-    cd benchmarks/qwen3-omni/transformers
-
-    python qwen3_omni_moe_transformers.py --prompts-file ../../build_dataset/top100.txt --num-prompts 100
-
-    echo "Logs and outputs are saved to $(pwd)/benchmark_results:"
-    echo "  - perf_stats.json    Aggregated/per-prompt TPS and latency (thinker/talker/code2wav/overall)"
-    echo "  - results.json       Per-prompt outputs and audio paths"
-    echo "  - audio/             Generated wav files, there should be 100 wav file generated"
-    echo "Key checks: overall_tps and *_tps_avg should be non-zero and stable; investigate 0/NaN or unusually low TPS/long-tail latency."
-fi

benchmarks/qwen3-omni/transformers/qwen3_omni_moe_model.py

@@ -1,265 +0,0 @@
-import time
-
-import torch
-from transformers import Qwen3OmniMoeForConditionalGeneration
-
-
-class Qwen3OmniMoeForConditionalGenerationWithLogging(Qwen3OmniMoeForConditionalGeneration):
-    @torch.no_grad()
-    def generate(
-        self,
-        input_ids: torch.Tensor | None = None,
-        speaker: str = "Ethan",
-        use_audio_in_video: bool = False,
-        return_audio: bool | None = None,
-        thinker_max_new_tokens: int = 1024,
-        thinker_eos_token_id: int = 151645,
-        talker_max_new_tokens: int = 4096,
-        talker_do_sample: bool = True,
-        talker_top_k: int = 50,
-        talker_top_p: float = 1.0,
-        talker_temperature: float = 0.9,
-        talker_repetition_penalty: float = 1.05,
-        **kwargs,
-    ):
-        total_t0 = time.time()
-        perf_stats = {
-            "thinker_tokens": 0,
-            "thinker_time_s": 0.0,
-            "thinker_tps": 0.0,
-            "talker_tokens": 0,
-            "talker_time_s": 0.0,
-            "talker_tps": 0.0,
-            "code2wav_tokens": 0,
-            "code2wav_time_s": 0.0,
-            "code2wav_tps": 0.0,
-            "total_tokens": 0,
-            "total_time_s": 0.0,
-            "total_tps": 0.0,
-        }
-        if return_audio and not self.has_talker:
-            raise ValueError(
-                "Cannot use talker when talker module not initialized. "
-                "Use `enable_talker` method or set enable_talker in config "
-                "to enable talker."
-            )
-        if return_audio is None:
-            return_audio = self.has_talker
-
-        shared_kwargs = {"use_audio_in_video": use_audio_in_video}
-        thinker_kwargs = {
-            "max_new_tokens": thinker_max_new_tokens,
-            "eos_token_id": thinker_eos_token_id,
-        }
-
-        talker_kwargs = {}
-        token2wav_kwargs = {}
-        if return_audio:
-            speaker_id = self.config.talker_config.speaker_id.get(speaker.lower())
-            if speaker_id is None:
-                raise NotImplementedError(f"Speaker {speaker} not implemented")
-            if input_ids.shape[0] != 1:
-                raise NotImplementedError("Qwen3-Omni currently does not support batched inference with audio output")
-            talker_suppressed_tokens = [
-                i
-                for i in range(
-                    self.config.talker_config.text_config.vocab_size - 1024,
-                    self.config.talker_config.text_config.vocab_size,
-                )
-                if i != self.config.talker_config.codec_eos_token_id
-            ]  # Suppress additional special tokens, should not be predicted
-            talker_kwargs = {
-                "max_new_tokens": talker_max_new_tokens,
-                "do_sample": talker_do_sample,
-                "top_k": talker_top_k,
-                "top_p": talker_top_p,
-                "temperature": talker_temperature,
-                "eos_token_id": self.config.talker_config.codec_eos_token_id,
-                "repetition_penalty": talker_repetition_penalty,
-                "suppress_tokens": talker_suppressed_tokens,
-                "output_hidden_states": True,
-                "return_dict_in_generate": True,
-            }
-            token2wav_kwargs = {}
-
-        for key, value in kwargs.items():
-            if key.startswith("thinker_"):
-                thinker_kwargs[key[len("thinker_") :]] = value
-            elif key.startswith("talker_"):
-                talker_kwargs[key[len("talker_") :]] = value
-            elif key.startswith("token2wav_"):
-                token2wav_kwargs[key[len("token2wav_") :]] = value
-            # Process special input values
-            elif key == "feature_attention_mask":
-                thinker_kwargs[key] = value
-                talker_kwargs["audio_feature_lengths"] = torch.sum(value, dim=1)
-            elif key in ("input_features", "attention_mask"):
-                thinker_kwargs[key] = value
-            # Put other key to shared kwargs
-            else:
-                shared_kwargs[key] = value
-
-        # Merge kwargs
-        for key, value in shared_kwargs.items():
-            if key not in thinker_kwargs:
-                thinker_kwargs[key] = value
-            if key not in talker_kwargs and key in ["image_grid_thw", "video_grid_thw", "video_second_per_grid"]:
-                talker_kwargs[key] = value
-            if key not in token2wav_kwargs:
-                token2wav_kwargs[key] = value
-
-        # 1. Generate from thinker module
-        generate_audio = return_audio and self.has_talker
-        if generate_audio:
-            thinker_kwargs["output_hidden_states"] = True
-            thinker_kwargs["return_dict_in_generate"] = True
-
-        t0 = time.time()
-        thinker_result = self.thinker.generate(input_ids=input_ids, **thinker_kwargs)
-        t1 = time.time()
-        perf_stats["thinker_time_s"] = max(0.0, t1 - t0)
-        try:
-            prompt_len = int(input_ids.shape[1]) if input_ids is not None else 0
-            total_len = int(thinker_result.sequences.shape[-1])
-            thinker_out_len = max(0, total_len - prompt_len)
-        except Exception:
-            thinker_out_len = 0
-        perf_stats["thinker_tokens"] = thinker_out_len
-        perf_stats["thinker_tps"] = (
-            (thinker_out_len / perf_stats["thinker_time_s"]) if perf_stats["thinker_time_s"] > 0 else 0.0
-        )
-
-        if not generate_audio:
-            perf_stats["total_tokens"] = perf_stats["thinker_tokens"]
-            perf_stats["total_time_s"] = time.time() - total_t0
-            perf_stats["total_tps"] = (
-                (perf_stats["total_tokens"] / perf_stats["total_time_s"]) if perf_stats["total_time_s"] > 0 else 0.0
-            )
-            # attach stats to self
-            setattr(self, "_perf_stats_last", perf_stats)
-            if not hasattr(self, "_perf_stats_history"):
-                setattr(self, "_perf_stats_history", [])
-            self._perf_stats_history.append(perf_stats)
-            return thinker_result, None
-
-        # 2. Prepare talker input
-        thinker_embed = torch.cat([hidden_states[0] for hidden_states in thinker_result.hidden_states], dim=1).to(
-            self.talker.device
-        )  # [1 t d]
-        thinker_hidden = torch.cat(
-            [
-                hidden_states[self.config.talker_config.accept_hidden_layer]
-                for hidden_states in thinker_result.hidden_states
-            ],
-            dim=1,
-        ).to(self.talker.device)  # [1 t d]
-
-        im_start_indexes = torch.cat(
-            (
-                torch.nonzero(input_ids[0] == self.config.im_start_token_id).squeeze(),
-                torch.tensor([thinker_result.sequences.shape[-1]], device=input_ids.device, dtype=input_ids.dtype),
-            ),
-            dim=-1,
-        ).to(self.talker.device)  # Shape [n_starts + 1]; Take batch 0 since batched inference is not supported here.
-        multimodal_mask = (
-            (thinker_result.sequences == self.config.thinker_config.audio_token_id) |
-            (thinker_result.sequences == self.config.thinker_config.image_token_id) |
-            (thinker_result.sequences == self.config.thinker_config.video_token_id)
-        ).to(self.talker.device)  # [1 t] # fmt: skip
-
-        talker_special_tokens = torch.tensor(
-            [[self.config.tts_bos_token_id, self.config.tts_eos_token_id, self.config.tts_pad_token_id]],
-            device=self.thinker.device,
-            dtype=input_ids.dtype,
-        )
-        tts_bos_embed, tts_eos_embed, tts_pad_embed = (
-            self.talker.text_projection(self.thinker.get_input_embeddings()(talker_special_tokens))
-            .to(self.talker.device)
-            .chunk(3, dim=1)
-        )  # 3 * [1 1 d]
-
-        talker_input_embeds = []  # [1 t d]
-        talker_input_ids = []
-        # For every chatml parts
-        for i in range(len(im_start_indexes) - 1):
-            im_start_index = im_start_indexes[i]
-            segment_end_index = im_start_indexes[i + 1]
-            role_token = input_ids[0][im_start_index + 1]
-            # Talker should ignore thinker system prompt
-            if role_token == self.config.system_token_id:
-                continue
-            # Talker takes word embeddings for tokens and hidden state from `accept_hidden_layer` for multimodal inputs
-            elif role_token == self.config.user_token_id:
-                talker_user_part = self._get_talker_user_parts(
-                    im_start_index, segment_end_index, multimodal_mask, thinker_hidden, thinker_embed
-                )
-                talker_input_embeds.append(talker_user_part)
-                talker_input_ids.append(thinker_result.sequences[:, im_start_index:segment_end_index])
-            # Take assistant output (for now)
-            elif role_token == self.config.assistant_token_id and i == len(im_start_indexes) - 2:
-                talker_assistant_embeds, talker_assistant_ids, trailing_text_hidden = self._get_talker_assistant_parts(
-                    im_start_index,
-                    segment_end_index,
-                    speaker_id,
-                    thinker_embed,
-                    tts_pad_embed,
-                    tts_bos_embed,
-                    tts_eos_embed,
-                )
-                talker_input_embeds.append(talker_assistant_embeds)
-                talker_input_ids.append(talker_assistant_ids)
-            # History assistant output (ignore for now)
-            elif role_token == self.config.assistant_token_id and i != len(im_start_indexes) - 2:
-                continue
-            else:
-                raise AssertionError("Expect role id after <|im_start|> (assistant, user, system)")
-        talker_input_embed = torch.cat([embed.to(self.talker.device) for embed in talker_input_embeds], dim=1)
-        talker_input_id = torch.cat([embed.to(self.talker.device) for embed in talker_input_ids], dim=1)
-        t2 = time.time()
-        talker_result = self.talker.generate(
-            inputs_embeds=talker_input_embed,
-            trailing_text_hidden=trailing_text_hidden,
-            tts_pad_embed=tts_pad_embed,
-            talker_input_ids=talker_input_id,  # Not use input_ids to prevent repetition penalty out of bound
-            **talker_kwargs,
-        )
-        t3 = time.time()
-        perf_stats["talker_time_s"] = max(0.0, t3 - t2)
-        talker_codes = (
-            torch.stack([hid[-1] for hid in talker_result.hidden_states if hid[-1] is not None], dim=1)
-            .transpose(1, 2)
-            .to(self.code2wav.device)
-        )
-        try:
-            # codes shape: (B, num_quantizers, T). We log T as token length.
-            perf_stats["talker_tokens"] = int(talker_codes.shape[-1])
-        except Exception:
-            perf_stats["talker_tokens"] = 0
-        perf_stats["talker_tps"] = (
-            (perf_stats["talker_tokens"] / perf_stats["talker_time_s"]) if perf_stats["talker_time_s"] > 0 else 0.0
-        )
-        t4 = time.time()
-        talker_wavs = self.code2wav.chunked_decode(talker_codes, chunk_size=300, left_context_size=25).float()
-        t5 = time.time()
-        perf_stats["code2wav_time_s"] = max(0.0, t5 - t4)
-        perf_stats["code2wav_tokens"] = perf_stats["talker_tokens"]  # same T, not times 16
-        perf_stats["code2wav_tps"] = (
-            (perf_stats["code2wav_tokens"] / perf_stats["code2wav_time_s"])
-            if perf_stats["code2wav_time_s"] > 0
-            else 0.0
-        )
-        perf_stats["total_tokens"] = perf_stats["thinker_tokens"] + perf_stats["talker_tokens"]
-        perf_stats["total_time_s"] = time.time() - total_t0
-        perf_stats["total_tps"] = (
-            (perf_stats["total_tokens"] / perf_stats["total_time_s"]) if perf_stats["total_time_s"] > 0 else 0.0
-        )
-        setattr(self, "_perf_stats_last", perf_stats)
-        if not hasattr(self, "_perf_stats_history"):
-            setattr(self, "_perf_stats_history", [])
-        self._perf_stats_history.append(perf_stats)
-        return thinker_result, talker_wavs.float()
-
-
-__all__ = [
-    "Qwen3OmniMoeForConditionalGenerationWithLogging",
-]

benchmarks/qwen3-omni/transformers/qwen3_omni_moe_transformers.py

@@ -1,275 +0,0 @@
-import argparse
-import json
-import os
-
-import soundfile as sf
-from qwen3_omni_moe_model import Qwen3OmniMoeForConditionalGenerationWithLogging
-from qwen_omni_utils import process_mm_info
-from tqdm import tqdm
-from transformers import Qwen3OmniMoeProcessor
-
-MODEL_PATH = "Qwen/Qwen3-Omni-30B-A3B-Instruct"
-# MODEL_PATH = "Qwen/Qwen3-Omni-30B-A3B-Thinking"
-
-
-def load_prompts(prompts_file: str) -> list[str]:
-    """Load prompts from a text file, one prompt per line."""
-    prompts = []
-    with open(prompts_file, encoding="utf-8") as f:
-        for line in f:
-            line = line.strip()
-            if line:
-                prompts.append(line)
-    return prompts
-
-
-def run_benchmark(
-    model,
-    processor,
-    prompts: list[str],
-    output_dir: str = "benchmark_results",
-    speaker: str = "Ethan",
-    use_audio_in_video: bool = True,
-):
-    """
-    Run benchmark on a list of prompts and collect performance stats.
-
-    Args:
-        model: The Qwen3OmniMoe model
-        processor: The Qwen3OmniMoe processor
-        prompts: List of text prompts to process
-        output_dir: Directory to save results
-        speaker: Speaker voice for audio output
-        use_audio_in_video: Whether to use audio in video
-
-    Returns:
-        tuple: (aggregated_stats, results, audio_outputs)
-            - aggregated_stats: dict with aggregated performance statistics
-            - results: list of dicts with per-prompt results
-            - audio_outputs: list of audio tensors/arrays (or None if no audio)
-    """
-    os.makedirs(output_dir, exist_ok=True)
-    audio_dir = os.path.join(output_dir, "audio")
-    os.makedirs(audio_dir, exist_ok=True)
-
-    all_stats = []
-    results = []
-    audio_outputs = []
-
-    for idx, prompt in enumerate(tqdm(prompts, desc="Processing prompts")):
-        conversation = [
-            {
-                "role": "user",
-                "content": [{"type": "text", "text": prompt}],
-            },
-        ]
-
-        # Preparation for inference
-        text = processor.apply_chat_template(conversation, add_generation_prompt=True, tokenize=False)
-        audios, images, videos = process_mm_info(conversation, use_audio_in_video=use_audio_in_video)
-        inputs = processor(
-            text=text,
-            audio=audios,
-            images=images,
-            videos=videos,
-            return_tensors="pt",
-            padding=True,
-            use_audio_in_video=use_audio_in_video,
-        )
-        inputs = inputs.to(model.device).to(model.dtype)
-
-        # Inference: Generation of the output text and audio
-        text_ids, audio = model.generate(
-            **inputs, speaker=speaker, thinker_return_dict_in_generate=True, use_audio_in_video=use_audio_in_video
-        )
-
-        # Decode output text
-        output_text = processor.batch_decode(
-            text_ids.sequences[:, inputs["input_ids"].shape[1] :],
-            skip_special_tokens=True,
-            clean_up_tokenization_spaces=False,
-        )[0]
-
-        # Collect performance stats
-        perf_stats = None
-        if hasattr(model, "_perf_stats_last"):
-            perf_stats = model._perf_stats_last.copy()
-            perf_stats["prompt_idx"] = idx
-            perf_stats["prompt"] = prompt
-            all_stats.append(perf_stats)
-
-        # Save audio and collect audio output
-        audio_path = None
-        audio_data = None
-        if audio is not None:
-            audio_data = audio.reshape(-1).detach().cpu().numpy()
-            audio_path = os.path.join(audio_dir, f"output_{idx:04d}.wav")
-            sf.write(
-                audio_path,
-                audio_data,
-                samplerate=24000,
-            )
-            audio_outputs.append(audio_data)
-        else:
-            audio_outputs.append(None)
-
-        # Save result
-        result = {
-            "idx": idx,
-            "prompt": prompt,
-            "output": output_text,
-            "audio_path": audio_path,
-            "perf_stats": perf_stats,
-        }
-        results.append(result)
-
-    # Aggregate statistics
-    aggregated_stats = aggregate_stats(all_stats)
-
-    # Save all results
-    results_path = os.path.join(output_dir, "results.json")
-    with open(results_path, "w", encoding="utf-8") as f:
-        json.dump(results, f, ensure_ascii=False, indent=2)
-
-    # Save aggregated stats
-    stats_path = os.path.join(output_dir, "perf_stats.json")
-    with open(stats_path, "w", encoding="utf-8") as f:
-        json.dump({"aggregated": aggregated_stats, "per_prompt": all_stats}, f, ensure_ascii=False, indent=2)
-
-    # Count saved audio files
-    num_audio_saved = sum(1 for a in audio_outputs if a is not None)
-    print(f"\nSaved {num_audio_saved} audio files to {audio_dir}/")
-
-    return aggregated_stats, results, audio_outputs
-
-
-def aggregate_stats(all_stats: list[dict]) -> dict:
-    """Aggregate performance statistics from multiple runs."""
-    if not all_stats:
-        return {}
-
-    keys = [
-        "thinker_tokens",
-        "thinker_time_s",
-        "thinker_tps",
-        "talker_tokens",
-        "talker_time_s",
-        "talker_tps",
-        "code2wav_tokens",
-        "code2wav_time_s",
-        "code2wav_tps",
-        "total_tokens",
-        "total_time_s",
-        "total_tps",
-    ]
-
-    aggregated = {
-        "num_samples": len(all_stats),
-    }
-
-    for key in keys:
-        values = [s.get(key, 0) for s in all_stats if key in s]
-        if values:
-            aggregated[f"{key}_sum"] = sum(values)
-            aggregated[f"{key}_avg"] = sum(values) / len(values)
-            aggregated[f"{key}_min"] = min(values)
-            aggregated[f"{key}_max"] = max(values)
-
-    # Calculate overall throughput
-    total_tokens = aggregated.get("total_tokens_sum", 0)
-    total_time = aggregated.get("total_time_s_sum", 0)
-    if total_time > 0:
-        aggregated["overall_tps"] = total_tokens / total_time
-
-    return aggregated
-
-
-def print_stats(stats: dict):
-    """Print performance statistics in a formatted way."""
-    print("\n" + "=" * 60)
-    print("Performance Statistics Summary")
-    print("=" * 60)
-
-    print(f"\nNumber of samples: {stats.get('num_samples', 0)}")
-
-    print("\n--- Thinker ---")
-    print(f"  Total tokens:  {stats.get('thinker_tokens_sum', 0):.0f}")
-    print(f"  Total time:    {stats.get('thinker_time_s_sum', 0):.2f}s")
-    print(f"  Avg TPS:       {stats.get('thinker_tps_avg', 0):.2f}")
-    print(f"  Min TPS:       {stats.get('thinker_tps_min', 0):.2f}")
-    print(f"  Max TPS:       {stats.get('thinker_tps_max', 0):.2f}")
-
-    print("\n--- Talker ---")
-    print(f"  Total tokens:  {stats.get('talker_tokens_sum', 0):.0f}")
-    print(f"  Total time:    {stats.get('talker_time_s_sum', 0):.2f}s")
-    print(f"  Avg TPS:       {stats.get('talker_tps_avg', 0):.2f}")
-    print(f"  Min TPS:       {stats.get('talker_tps_min', 0):.2f}")
-    print(f"  Max TPS:       {stats.get('talker_tps_max', 0):.2f}")
-
-    print("\n--- Code2Wav ---")
-    print(f"  Total tokens:  {stats.get('code2wav_tokens_sum', 0):.0f}")
-    print(f"  Total time:    {stats.get('code2wav_time_s_sum', 0):.2f}s")
-    print(f"  Avg TPS:       {stats.get('code2wav_tps_avg', 0):.2f}")
-    print(f"  Min TPS:       {stats.get('code2wav_tps_min', 0):.2f}")
-    print(f"  Max TPS:       {stats.get('code2wav_tps_max', 0):.2f}")
-
-    print("\n--- Overall ---")
-    print(f"  Total tokens:  {stats.get('total_tokens_sum', 0):.0f}")
-    print(f"  Total time:    {stats.get('total_time_s_sum', 0):.2f}s")
-    print(f"  Overall TPS:   {stats.get('overall_tps', 0):.2f}")
-    print(f"  Avg TPS:       {stats.get('total_tps_avg', 0):.2f}")
-    print(f"  Min TPS:       {stats.get('total_tps_min', 0):.2f}")
-    print(f"  Max TPS:       {stats.get('total_tps_max', 0):.2f}")
-
-    print("=" * 60 + "\n")
-
-
-def main():
-    parser = argparse.ArgumentParser(description="Qwen3-Omni Benchmark Script")
-    parser.add_argument(
-        "--prompts-file",
-        type=str,
-        default="benchmark/build_dataset/top100.txt",
-        help="Path to the prompts file (one prompt per line)",
-    )
-    parser.add_argument(
-        "--output-dir", type=str, default="benchmark_results", help="Directory to save benchmark results"
-    )
-    parser.add_argument("--model-path", type=str, default=MODEL_PATH, help="Path to the model")
-    parser.add_argument("--speaker", type=str, default="Ethan", help="Speaker voice for audio output")
-    parser.add_argument("--num-prompts", type=int, default=None, help="Number of prompts to process (default: all)")
-    args = parser.parse_args()
-
-    # Load model and processor
-    print(f"Loading model from {args.model_path}...")
-    model = Qwen3OmniMoeForConditionalGenerationWithLogging.from_pretrained(
-        args.model_path,
-        dtype="auto",
-        device_map="auto",
-        attn_implementation="flash_attention_2",
-    )
-    processor = Qwen3OmniMoeProcessor.from_pretrained(args.model_path)
-
-    # Benchmark mode
-    print(f"Loading prompts from {args.prompts_file}...")
-    prompts = load_prompts(args.prompts_file)
-
-    if args.num_prompts:
-        prompts = prompts[: args.num_prompts]
-
-    print(f"Running benchmark on {len(prompts)} prompts...")
-
-    aggregated_stats, results, audio_outputs = run_benchmark(
-        model=model,
-        processor=processor,
-        prompts=prompts,
-        output_dir=args.output_dir,
-        speaker=args.speaker,
-    )
-
-    print_stats(aggregated_stats)
-    print(f"\nResults saved to {args.output_dir}/")
-
-
-if __name__ == "__main__":
-    main()

benchmarks/qwen3-omni/vllm_omni/eval_qwen3_moe_omni.sh

@@ -1,37 +0,0 @@
-#!/bin/bash
-# Qwen3-Omni Benchmark Evaluation Script
-# This script must be run from the vllm-omni root directory
-
-# Get the directory where this script is located
-SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-
-# Navigate to vllm-omni root directory (4 levels up from script location)
-VLLM_OMNI_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)"
-cd "$VLLM_OMNI_ROOT" || { echo "Error: Failed to navigate to vllm-omni directory"; exit 1; }
-
-echo "Working directory: $(pwd)"
-
-# Verify we're in the correct directory and run benchmark
-if [[ ! -d "benchmarks/qwen3-omni/vllm_omni" ]]; then
-    echo "Error: Not in vllm-omni root directory. Please run from vllm-omni folder."
-else
-    log_dir=benchmarks/qwen3-omni/vllm_omni/logs
-    outputs_dir=benchmarks/qwen3-omni/vllm_omni/outputs
-    end2end_script_path=examples/offline_inference/qwen3_omni/end2end.py
-    build_dataset_path=benchmarks/build_dataset/top100.txt
-
-    python $end2end_script_path --output-wav $outputs_dir \
-                      --query-type text \
-                      --txt-prompts $build_dataset_path \
-                      --log-stats \
-                      --log-dir $log_dir
-    echo "Logs and outputs are saved in ${log_dir} and ${outputs_dir} respectively:"
-    echo "  - omni_llm_pipeline_text                       run dir/base name"
-    echo "  - omni_llm_pipeline_text.orchestrator.stats.jsonl  orchestrator-stage latency stats"
-    echo "  - omni_llm_pipeline_text.overall.stats.jsonl       overall latency/TPS stats"
-    echo "  - omni_llm_pipeline_text.stage0.log                per-stage detailed logs"
-    echo "  - omni_llm_pipeline_text.stage1.log"
-    echo "  - omni_llm_pipeline_text.stage2.log"
-    echo "Key checks: overall.stats.jsonl for end-to-end latency/TPS; orchestrator.stats.jsonl for stable per-stage latency; stage*.log for errors or long tails."
-    echo "  - outputs/             Generated txt and wav files, there should be 100 text and wav files generated respectively"
-fi

collect_env.py

@@ -1,760 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-
-# ruff: noqa
-# code borrowed from https://github.com/pytorch/pytorch/blob/main/torch/utils/collect_env.py
-
-import datetime
-import locale
-import os
-import subprocess
-import sys
-
-# Unlike the rest of the PyTorch this file must be python2 compliant.
-# This script outputs relevant system environment info
-# Run it with `python collect_env.py` or `python -m torch.utils.collect_env`
-from collections import namedtuple
-
-import regex as re
-
-from vllm.envs import environment_variables
-
-try:
-    import torch
-
-    TORCH_AVAILABLE = True
-except (ImportError, NameError, AttributeError, OSError):
-    TORCH_AVAILABLE = False
-
-# System Environment Information
-SystemEnv = namedtuple(
-    "SystemEnv",
-    [
-        "torch_version",
-        "is_debug_build",
-        "cuda_compiled_version",
-        "gcc_version",
-        "clang_version",
-        "cmake_version",
-        "os",
-        "libc_version",
-        "python_version",
-        "python_platform",
-        "is_cuda_available",
-        "cuda_runtime_version",
-        "cuda_module_loading",
-        "nvidia_driver_version",
-        "nvidia_gpu_models",
-        "cudnn_version",
-        "pip_version",  # 'pip' or 'pip3'
-        "pip_packages",
-        "conda_packages",
-        "hip_compiled_version",
-        "hip_runtime_version",
-        "miopen_runtime_version",
-        "caching_allocator_config",
-        "is_xnnpack_available",
-        "cpu_info",
-        "rocm_version",  # vllm specific field
-        "vllm_version",  # vllm specific field
-        "vllm_omni_version",  # vllm-omni specific field
-        "vllm_build_flags",  # vllm specific field
-        "gpu_topo",  # vllm specific field
-        "env_vars",
-    ],
-)
-
-DEFAULT_CONDA_PATTERNS = {
-    "torch",
-    "numpy",
-    "cudatoolkit",
-    "soumith",
-    "mkl",
-    "magma",
-    "triton",
-    "optree",
-    "nccl",
-    "transformers",
-    "zmq",
-    "nvidia",
-    "pynvml",
-    "flashinfer-python",
-}
-
-DEFAULT_PIP_PATTERNS = {
-    "torch",
-    "numpy",
-    "mypy",
-    "flake8",
-    "triton",
-    "optree",
-    "onnx",
-    "nccl",
-    "transformers",
-    "zmq",
-    "nvidia",
-    "pynvml",
-    "flashinfer-python",
-}
-
-
-def run(command):
-    """Return (return-code, stdout, stderr)."""
-    shell = True if type(command) is str else False
-    try:
-        p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=shell)
-        raw_output, raw_err = p.communicate()
-        rc = p.returncode
-        if get_platform() == "win32":
-            enc = "oem"
-        else:
-            enc = locale.getpreferredencoding()
-        output = raw_output.decode(enc)
-        if command == "nvidia-smi topo -m":
-            # don't remove the leading whitespace of `nvidia-smi topo -m`
-            #   because they are meaningful
-            output = output.rstrip()
-        else:
-            output = output.strip()
-        err = raw_err.decode(enc)
-        return rc, output, err.strip()
-
-    except FileNotFoundError:
-        cmd_str = command if isinstance(command, str) else command[0]
-        return 127, "", f"Command not found: {cmd_str}"
-
-
-def run_and_read_all(run_lambda, command):
-    """Run command using run_lambda; reads and returns entire output if rc is 0."""
-    rc, out, _ = run_lambda(command)
-    if rc != 0:
-        return None
-    return out
-
-
-def run_and_parse_first_match(run_lambda, command, regex):
-    """Run command using run_lambda, returns the first regex match if it exists."""
-    rc, out, _ = run_lambda(command)
-    if rc != 0:
-        return None
-    match = re.search(regex, out)
-    if match is None:
-        return None
-    return match.group(1)
-
-
-def run_and_return_first_line(run_lambda, command):
-    """Run command using run_lambda and returns first line if output is not empty."""
-    rc, out, _ = run_lambda(command)
-    if rc != 0:
-        return None
-    return out.split("\n")[0]
-
-
-def get_conda_packages(run_lambda, patterns=None):
-    if patterns is None:
-        patterns = DEFAULT_CONDA_PATTERNS
-    conda = os.environ.get("CONDA_EXE", "conda")
-    out = run_and_read_all(run_lambda, [conda, "list"])
-    if out is None:
-        return out
-
-    return "\n".join(
-        line for line in out.splitlines() if not line.startswith("#") and any(name in line for name in patterns)
-    )
-
-
-def get_gcc_version(run_lambda):
-    return run_and_parse_first_match(run_lambda, "gcc --version", r"gcc (.*)")
-
-
-def get_clang_version(run_lambda):
-    return run_and_parse_first_match(run_lambda, "clang --version", r"clang version (.*)")
-
-
-def get_cmake_version(run_lambda):
-    return run_and_parse_first_match(run_lambda, "cmake --version", r"cmake (.*)")
-
-
-def get_nvidia_driver_version(run_lambda):
-    if get_platform() == "darwin":
-        cmd = "kextstat | grep -i cuda"
-        return run_and_parse_first_match(run_lambda, cmd, r"com[.]nvidia[.]CUDA [(](.*?)[)]")
-    smi = get_nvidia_smi()
-    return run_and_parse_first_match(run_lambda, smi, r"Driver Version: (.*?) ")
-
-
-def get_gpu_info(run_lambda):
-    if get_platform() == "darwin" or (
-        TORCH_AVAILABLE and hasattr(torch.version, "hip") and torch.version.hip is not None
-    ):
-        if TORCH_AVAILABLE and torch.cuda.is_available():
-            if torch.version.hip is not None:
-                prop = torch.cuda.get_device_properties(0)
-                if hasattr(prop, "gcnArchName"):
-                    gcnArch = " ({})".format(prop.gcnArchName)
-                else:
-                    gcnArch = "NoGCNArchNameOnOldPyTorch"
-            else:
-                gcnArch = ""
-            return torch.cuda.get_device_name(None) + gcnArch
-        return None
-    smi = get_nvidia_smi()
-    uuid_regex = re.compile(r" \(UUID: .+?\)")
-    rc, out, _ = run_lambda(smi + " -L")
-    if rc != 0:
-        return None
-    # Anonymize GPUs by removing their UUID
-    return re.sub(uuid_regex, "", out)
-
-
-def get_running_cuda_version(run_lambda):
-    return run_and_parse_first_match(run_lambda, "nvcc --version", r"release .+ V(.*)")
-
-
-def get_cudnn_version(run_lambda):
-    """Return a list of libcudnn.so; it's hard to tell which one is being used."""
-    if get_platform() == "win32":
-        system_root = os.environ.get("SYSTEMROOT", "C:\\Windows")
-        cuda_path = os.environ.get("CUDA_PATH", "%CUDA_PATH%")
-        where_cmd = os.path.join(system_root, "System32", "where")
-        cudnn_cmd = '{} /R "{}\\bin" cudnn*.dll'.format(where_cmd, cuda_path)
-    elif get_platform() == "darwin":
-        # CUDA libraries and drivers can be found in /usr/local/cuda/. See
-        # https://docs.nvidia.com/cuda/cuda-installation-guide-mac-os-x/index.html#install
-        # https://docs.nvidia.com/deeplearning/sdk/cudnn-install/index.html#installmac
-        # Use CUDNN_LIBRARY when cudnn library is installed elsewhere.
-        cudnn_cmd = "ls /usr/local/cuda/lib/libcudnn*"
-    else:
-        cudnn_cmd = 'ldconfig -p | grep libcudnn | rev | cut -d" " -f1 | rev'
-    rc, out, _ = run_lambda(cudnn_cmd)
-    # find will return 1 if there are permission errors or if not found
-    if len(out) == 0 or (rc != 1 and rc != 0):
-        l = os.environ.get("CUDNN_LIBRARY")
-        if l is not None and os.path.isfile(l):
-            return os.path.realpath(l)
-        return None
-    files_set = set()
-    for fn in out.split("\n"):
-        fn = os.path.realpath(fn)  # eliminate symbolic links
-        if os.path.isfile(fn):
-            files_set.add(fn)
-    if not files_set:
-        return None
-    # Alphabetize the result because the order is non-deterministic otherwise
-    files = sorted(files_set)
-    if len(files) == 1:
-        return files[0]
-    result = "\n".join(files)
-    return "Probably one of the following:\n{}".format(result)
-
-
-def get_nvidia_smi():
-    # Note: nvidia-smi is currently available only on Windows and Linux
-    smi = "nvidia-smi"
-    if get_platform() == "win32":
-        system_root = os.environ.get("SYSTEMROOT", "C:\\Windows")
-        program_files_root = os.environ.get("PROGRAMFILES", "C:\\Program Files")
-        legacy_path = os.path.join(program_files_root, "NVIDIA Corporation", "NVSMI", smi)
-        new_path = os.path.join(system_root, "System32", smi)
-        smis = [new_path, legacy_path]
-        for candidate_smi in smis:
-            if os.path.exists(candidate_smi):
-                smi = '"{}"'.format(candidate_smi)
-                break
-    return smi
-
-
-def get_rocm_version(run_lambda):
-    """Returns the ROCm version if available, otherwise 'N/A'."""
-    return run_and_parse_first_match(run_lambda, "hipcc --version", r"HIP version: (\S+)")
-
-
-def get_vllm_version():
-    from vllm import __version__, __version_tuple__
-
-    if __version__ == "dev":
-        return "N/A (dev)"
-    version_str = __version_tuple__[-1]
-    if isinstance(version_str, str) and version_str.startswith("g"):
-        # it's a dev build
-        if "." in version_str:
-            # it's a dev build containing local changes
-            git_sha = version_str.split(".")[0][1:]
-            date = version_str.split(".")[-1][1:]
-            return f"{__version__} (git sha: {git_sha}, date: {date})"
-        else:
-            # it's a dev build without local changes
-            git_sha = version_str[1:]  # type: ignore
-            return f"{__version__} (git sha: {git_sha})"
-    return __version__
-
-
-def get_vllm_omni_version(run_lambda):
-    try:
-        import vllm_omni
-        from vllm_omni import __version__, __version_tuple__
-
-        version_str = __version_tuple__[-1]
-        if isinstance(version_str, str) and version_str.startswith("g"):
-            if "." in version_str:
-                git_sha = version_str.split(".")[0][1:]
-                date = version_str.split(".")[-1][1:]
-                return f"{__version__} (git sha: {git_sha}, date: {date})"
-            else:
-                git_sha = version_str[1:]
-                return f"{__version__} (git sha: {git_sha})"
-
-        package_dir = os.path.dirname(os.path.abspath(vllm_omni.__file__))
-        git_sha = run_and_read_all(run_lambda, f"git -C {package_dir} rev-parse --short HEAD")
-        if git_sha:
-            return f"{__version__} (git sha: {git_sha})"
-
-        return __version__
-    except ImportError:
-        return "N/A (vllm_omni not installed)"
-
-
-def summarize_vllm_build_flags():
-    # This could be a static method if the flags are constant, or dynamic if you need to check environment variables, etc.
-    return "CUDA Archs: {}; ROCm: {}".format(
-        os.environ.get("TORCH_CUDA_ARCH_LIST", "Not Set"),
-        "Enabled" if os.environ.get("ROCM_HOME") else "Disabled",
-    )
-
-
-def get_gpu_topo(run_lambda):
-    output = None
-
-    if get_platform() == "linux":
-        output = run_and_read_all(run_lambda, "nvidia-smi topo -m")
-        if output is None:
-            output = run_and_read_all(run_lambda, "rocm-smi --showtopo")
-
-    return output
-
-
-def get_cpu_info(run_lambda):
-    rc, out, err = 0, "", ""
-    if get_platform() == "linux":
-        rc, out, err = run_lambda("lscpu")
-    elif get_platform() == "win32":
-        rc, out, err = run_lambda(
-            "wmic cpu get Name,Manufacturer,Family,Architecture,ProcessorType,DeviceID, \
-        CurrentClockSpeed,MaxClockSpeed,L2CacheSize,L2CacheSpeed,Revision /VALUE"
-        )
-    elif get_platform() == "darwin":
-        rc, out, err = run_lambda("sysctl -n machdep.cpu.brand_string")
-    cpu_info = "None"
-    if rc == 0:
-        cpu_info = out
-    else:
-        cpu_info = err
-    return cpu_info
-
-
-def get_platform():
-    if sys.platform.startswith("linux"):
-        return "linux"
-    elif sys.platform.startswith("win32"):
-        return "win32"
-    elif sys.platform.startswith("cygwin"):
-        return "cygwin"
-    elif sys.platform.startswith("darwin"):
-        return "darwin"
-    else:
-        return sys.platform
-
-
-def get_mac_version(run_lambda):
-    return run_and_parse_first_match(run_lambda, "sw_vers -productVersion", r"(.*)")
-
-
-def get_windows_version(run_lambda):
-    system_root = os.environ.get("SYSTEMROOT", "C:\\Windows")
-    wmic_cmd = os.path.join(system_root, "System32", "Wbem", "wmic")
-    findstr_cmd = os.path.join(system_root, "System32", "findstr")
-    return run_and_read_all(run_lambda, "{} os get Caption | {} /v Caption".format(wmic_cmd, findstr_cmd))
-
-
-def get_lsb_version(run_lambda):
-    return run_and_parse_first_match(run_lambda, "lsb_release -a", r"Description:\t(.*)")
-
-
-def check_release_file(run_lambda):
-    return run_and_parse_first_match(run_lambda, "cat /etc/*-release", r'PRETTY_NAME="(.*)"')
-
-
-def get_os(run_lambda):
-    from platform import machine
-
-    platform = get_platform()
-
-    if platform == "win32" or platform == "cygwin":
-        return get_windows_version(run_lambda)
-
-    if platform == "darwin":
-        version = get_mac_version(run_lambda)
-        if version is None:
-            return None
-        return "macOS {} ({})".format(version, machine())
-
-    if platform == "linux":
-        # Ubuntu/Debian based
-        desc = get_lsb_version(run_lambda)
-        if desc is not None:
-            return "{} ({})".format(desc, machine())
-
-        # Try reading /etc/*-release
-        desc = check_release_file(run_lambda)
-        if desc is not None:
-            return "{} ({})".format(desc, machine())
-
-        return "{} ({})".format(platform, machine())
-
-    # Unknown platform
-    return platform
-
-
-def get_python_platform():
-    import platform
-
-    return platform.platform()
-
-
-def get_libc_version():
-    import platform
-
-    if get_platform() != "linux":
-        return "N/A"
-    return "-".join(platform.libc_ver())
-
-
-def is_uv_venv():
-    if os.environ.get("UV"):
-        return True
-    pyvenv_cfg_path = os.path.join(sys.prefix, "pyvenv.cfg")
-    if os.path.exists(pyvenv_cfg_path):
-        with open(pyvenv_cfg_path, "r") as f:
-            return any(line.startswith("uv = ") for line in f)
-    return False
-
-
-def get_pip_packages(run_lambda, patterns=None):
-    """Return `pip list` output. Note: will also find conda-installed pytorch and numpy packages."""
-    if patterns is None:
-        patterns = DEFAULT_PIP_PATTERNS
-
-    def run_with_pip():
-        try:
-            import importlib.util
-
-            pip_spec = importlib.util.find_spec("pip")
-            pip_available = pip_spec is not None
-        except ImportError:
-            pip_available = False
-
-        if pip_available:
-            cmd = [sys.executable, "-mpip", "list", "--format=freeze"]
-        elif is_uv_venv():
-            print("uv is set")
-            cmd = ["uv", "pip", "list", "--format=freeze"]
-        else:
-            raise RuntimeError("Could not collect pip list output (pip or uv module not available)")
-
-        out = run_and_read_all(run_lambda, cmd)
-        return "\n".join(line for line in out.splitlines() if any(name in line for name in patterns))
-
-    pip_version = "pip3" if sys.version[0] == "3" else "pip"
-    out = run_with_pip()
-    return pip_version, out
-
-
-def get_cachingallocator_config():
-    ca_config = os.environ.get("PYTORCH_CUDA_ALLOC_CONF", "")
-    return ca_config
-
-
-def get_cuda_module_loading_config():
-    if TORCH_AVAILABLE and torch.cuda.is_available():
-        torch.cuda.init()
-        config = os.environ.get("CUDA_MODULE_LOADING", "")
-        return config
-    else:
-        return "N/A"
-
-
-def is_xnnpack_available():
-    if TORCH_AVAILABLE:
-        import torch.backends.xnnpack
-
-        return str(torch.backends.xnnpack.enabled)  # type: ignore[attr-defined]
-    else:
-        return "N/A"
-
-
-def get_env_vars():
-    env_vars = ""
-    secret_terms = ("secret", "token", "api", "access", "password")
-    report_prefix = (
-        "TORCH",
-        "NCCL",
-        "PYTORCH",
-        "CUDA",
-        "CUBLAS",
-        "CUDNN",
-        "OMP_",
-        "MKL_",
-        "NVIDIA",
-    )
-    for k, v in os.environ.items():
-        if any(term in k.lower() for term in secret_terms):
-            continue
-        if k in environment_variables:
-            env_vars = env_vars + "{}={}".format(k, v) + "\n"
-        if k.startswith(report_prefix):
-            env_vars = env_vars + "{}={}".format(k, v) + "\n"
-
-    return env_vars
-
-
-def get_env_info():
-    run_lambda = run
-    pip_version, pip_list_output = get_pip_packages(run_lambda)
-
-    if TORCH_AVAILABLE:
-        version_str = torch.__version__
-        debug_mode_str = str(torch.version.debug)
-        cuda_available_str = str(torch.cuda.is_available())
-        cuda_version_str = torch.version.cuda
-        if not hasattr(torch.version, "hip") or torch.version.hip is None:  # cuda version
-            hip_compiled_version = hip_runtime_version = miopen_runtime_version = "N/A"
-        else:  # HIP version
-
-            def get_version_or_na(cfg, prefix):
-                _lst = [s.rsplit(None, 1)[-1] for s in cfg if prefix in s]
-                return _lst[0] if _lst else "N/A"
-
-            cfg = torch._C._show_config().split("\n")
-            hip_runtime_version = get_version_or_na(cfg, "HIP Runtime")
-            miopen_runtime_version = get_version_or_na(cfg, "MIOpen")
-            cuda_version_str = "N/A"
-            hip_compiled_version = torch.version.hip
-    else:
-        version_str = debug_mode_str = cuda_available_str = cuda_version_str = "N/A"
-        hip_compiled_version = hip_runtime_version = miopen_runtime_version = "N/A"
-
-    sys_version = sys.version.replace("\n", " ")
-
-    conda_packages = get_conda_packages(run_lambda)
-
-    rocm_version = get_rocm_version(run_lambda)
-    vllm_version = get_vllm_version()
-    vllm_omni_version = get_vllm_omni_version(run_lambda)
-    vllm_build_flags = summarize_vllm_build_flags()
-    gpu_topo = get_gpu_topo(run_lambda)
-
-    return SystemEnv(
-        torch_version=version_str,
-        is_debug_build=debug_mode_str,
-        python_version="{} ({}-bit runtime)".format(sys_version, sys.maxsize.bit_length() + 1),
-        python_platform=get_python_platform(),
-        is_cuda_available=cuda_available_str,
-        cuda_compiled_version=cuda_version_str,
-        cuda_runtime_version=get_running_cuda_version(run_lambda),
-        cuda_module_loading=get_cuda_module_loading_config(),
-        nvidia_gpu_models=get_gpu_info(run_lambda),
-        nvidia_driver_version=get_nvidia_driver_version(run_lambda),
-        cudnn_version=get_cudnn_version(run_lambda),
-        hip_compiled_version=hip_compiled_version,
-        hip_runtime_version=hip_runtime_version,
-        miopen_runtime_version=miopen_runtime_version,
-        pip_version=pip_version,
-        pip_packages=pip_list_output,
-        conda_packages=conda_packages,
-        os=get_os(run_lambda),
-        libc_version=get_libc_version(),
-        gcc_version=get_gcc_version(run_lambda),
-        clang_version=get_clang_version(run_lambda),
-        cmake_version=get_cmake_version(run_lambda),
-        caching_allocator_config=get_cachingallocator_config(),
-        is_xnnpack_available=is_xnnpack_available(),
-        cpu_info=get_cpu_info(run_lambda),
-        rocm_version=rocm_version,
-        vllm_version=vllm_version,
-        vllm_omni_version=vllm_omni_version,
-        vllm_build_flags=vllm_build_flags,
-        gpu_topo=gpu_topo,
-        env_vars=get_env_vars(),
-    )
-
-
-env_info_fmt = """
-==============================
-        System Info
-==============================
-OS                           : {os}
-GCC version                  : {gcc_version}
-Clang version                : {clang_version}
-CMake version                : {cmake_version}
-Libc version                 : {libc_version}
-
-==============================
-       PyTorch Info
-==============================
-PyTorch version              : {torch_version}
-Is debug build               : {is_debug_build}
-CUDA used to build PyTorch   : {cuda_compiled_version}
-ROCM used to build PyTorch   : {hip_compiled_version}
-
-==============================
-      Python Environment
-==============================
-Python version               : {python_version}
-Python platform              : {python_platform}
-
-==============================
-       CUDA / GPU Info
-==============================
-Is CUDA available            : {is_cuda_available}
-CUDA runtime version         : {cuda_runtime_version}
-CUDA_MODULE_LOADING set to   : {cuda_module_loading}
-GPU models and configuration : {nvidia_gpu_models}
-Nvidia driver version        : {nvidia_driver_version}
-cuDNN version                : {cudnn_version}
-HIP runtime version          : {hip_runtime_version}
-MIOpen runtime version       : {miopen_runtime_version}
-Is XNNPACK available         : {is_xnnpack_available}
-
-==============================
-          CPU Info
-==============================
-{cpu_info}
-
-==============================
-Versions of relevant libraries
-==============================
-{pip_packages}
-{conda_packages}
-""".strip()
-
-# both the above code and the following code use `strip()` to
-# remove leading/trailing whitespaces, so we need to add a newline
-# in between to separate the two sections
-env_info_fmt += "\n\n"
-
-env_info_fmt += """
-==============================
-         vLLM Info
-==============================
-ROCM Version                 : {rocm_version}
-vLLM Version                 : {vllm_version}
-vLLM-Omni Version            : {vllm_omni_version}
-vLLM Build Flags:
-  {vllm_build_flags}
-GPU Topology:
-  {gpu_topo}
-
-==============================
-     Environment Variables
-==============================
-{env_vars}
-""".strip()
-
-
-def pretty_str(envinfo):
-    def replace_nones(dct, replacement="Could not collect"):
-        for key in dct.keys():
-            if dct[key] is not None:
-                continue
-            dct[key] = replacement
-        return dct
-
-    def replace_bools(dct, true="Yes", false="No"):
-        for key in dct.keys():
-            if dct[key] is True:
-                dct[key] = true
-            elif dct[key] is False:
-                dct[key] = false
-        return dct
-
-    def prepend(text, tag="[prepend]"):
-        lines = text.split("\n")
-        updated_lines = [tag + line for line in lines]
-        return "\n".join(updated_lines)
-
-    def replace_if_empty(text, replacement="No relevant packages"):
-        if text is not None and len(text) == 0:
-            return replacement
-        return text
-
-    def maybe_start_on_next_line(string):
-        # If `string` is multiline, prepend a \n to it.
-        if string is not None and len(string.split("\n")) > 1:
-            return "\n{}\n".format(string)
-        return string
-
-    mutable_dict = envinfo._asdict()
-
-    # If nvidia_gpu_models is multiline, start on the next line
-    mutable_dict["nvidia_gpu_models"] = maybe_start_on_next_line(envinfo.nvidia_gpu_models)
-
-    # If the machine doesn't have CUDA, report some fields as 'No CUDA'
-    dynamic_cuda_fields = [
-        "cuda_runtime_version",
-        "nvidia_gpu_models",
-        "nvidia_driver_version",
-    ]
-    all_cuda_fields = dynamic_cuda_fields + ["cudnn_version"]
-    all_dynamic_cuda_fields_missing = all(mutable_dict[field] is None for field in dynamic_cuda_fields)
-    if TORCH_AVAILABLE and not torch.cuda.is_available() and all_dynamic_cuda_fields_missing:
-        for field in all_cuda_fields:
-            mutable_dict[field] = "No CUDA"
-        if envinfo.cuda_compiled_version is None:
-            mutable_dict["cuda_compiled_version"] = "None"
-
-    # Replace True with Yes, False with No
-    mutable_dict = replace_bools(mutable_dict)
-
-    # Replace all None objects with 'Could not collect'
-    mutable_dict = replace_nones(mutable_dict)
-
-    # If either of these are '', replace with 'No relevant packages'
-    mutable_dict["pip_packages"] = replace_if_empty(mutable_dict["pip_packages"])
-    mutable_dict["conda_packages"] = replace_if_empty(mutable_dict["conda_packages"])
-
-    # Tag conda and pip packages with a prefix
-    # If they were previously None, they'll show up as ie '[conda] Could not collect'
-    if mutable_dict["pip_packages"]:
-        mutable_dict["pip_packages"] = prepend(mutable_dict["pip_packages"], "[{}] ".format(envinfo.pip_version))
-    if mutable_dict["conda_packages"]:
-        mutable_dict["conda_packages"] = prepend(mutable_dict["conda_packages"], "[conda] ")
-    mutable_dict["cpu_info"] = envinfo.cpu_info
-    return env_info_fmt.format(**mutable_dict)
-
-
-def get_pretty_env_info():
-    return pretty_str(get_env_info())
-
-
-def main():
-    print("Collecting environment information...")
-    output = get_pretty_env_info()
-    print(output)
-
-    if TORCH_AVAILABLE and hasattr(torch, "utils") and hasattr(torch.utils, "_crash_handler"):
-        minidump_dir = torch.utils._crash_handler.DEFAULT_MINIDUMP_DIR
-        if sys.platform == "linux" and os.path.exists(minidump_dir):
-            dumps = [os.path.join(minidump_dir, dump) for dump in os.listdir(minidump_dir)]
-            latest = max(dumps, key=os.path.getctime)
-            ctime = os.path.getctime(latest)
-            creation_time = datetime.datetime.fromtimestamp(ctime).strftime("%Y-%m-%d %H:%M:%S")
-            msg = (
-                "\n*** Detected a minidump at {} created on {}, ".format(latest, creation_time)
-                + "if this is related to your bug please include it when you file a report ***"
-            )
-            print(msg, file=sys.stderr)
-
-
-if __name__ == "__main__":
-    main()

docker/Dockerfile.ci

@@ -1,21 +0,0 @@
-ARG VLLM_BASE_IMAGE=vllm/vllm-openai
-ARG VLLM_BASE_TAG=v0.16.0
-FROM ${VLLM_BASE_IMAGE}:${VLLM_BASE_TAG}
-ARG APP_DIR=/workspace/vllm-omni
-WORKDIR ${APP_DIR}
-
-COPY . .
-
-# Install system dependencies
-RUN apt-get update && \
-    apt-get install -y ffmpeg sox libsox-fmt-all jq && \
-    apt-get clean && \
-    rm -rf /var/lib/apt/lists/*
-
-# Install vllm-omni into the same uv-managed Python environment used by the base image.
-# Use bash -c so that $(python3 -c ...) is expanded inside the container.
-RUN uv pip install --system --no-cache-dir ".[dev]"
-
-RUN ln -sf /usr/bin/python3 /usr/bin/python
-
-ENTRYPOINT []

docker/Dockerfile.npu

@@ -1,16 +0,0 @@
-ARG VLLM_ASCEND_IMAGE=quay.io/ascend/vllm-ascend
-ARG VLLM_ASCEND_TAG=v0.14.0rc1
-FROM ${VLLM_ASCEND_IMAGE}:${VLLM_ASCEND_TAG}
-
-ARG APP_DIR=/vllm-workspace/vllm-omni
-WORKDIR ${APP_DIR}
-
-COPY . .
-
-RUN sed -i -E 's/^([[:space:]]*)"fa3-fwd==0\.0\.1",/\1# "fa3-fwd==0.0.1",/' pyproject.toml \
- && sed -i -E 's/\bonnxruntime\b/onnxruntime-cann/g' pyproject.toml
-RUN pip install -v -e .
-
-ENV VLLM_WORKER_MULTIPROC_METHOD=spawn
-
-ENTRYPOINT []

docker/Dockerfile.npu.a3

@@ -1,16 +0,0 @@
-ARG VLLM_ASCEND_IMAGE=quay.io/ascend/vllm-ascend
-ARG VLLM_ASCEND_TAG=v0.14.0rc1-a3
-FROM ${VLLM_ASCEND_IMAGE}:${VLLM_ASCEND_TAG}
-
-ARG APP_DIR=/vllm-workspace/vllm-omni
-WORKDIR ${APP_DIR}
-
-COPY . .
-
-RUN sed -i -E 's/^([[:space:]]*)"fa3-fwd==0\.0\.1",/\1# "fa3-fwd==0.0.1",/' pyproject.toml \
- && sed -i -E 's/\bonnxruntime\b/onnxruntime-cann/g' pyproject.toml
-RUN pip install -v -e .
-
-ENV VLLM_WORKER_MULTIPROC_METHOD=spawn
-
-ENTRYPOINT []

docker/Dockerfile.rocm

@@ -1,53 +0,0 @@
-ARG BASE_IMAGE=vllm/vllm-openai-rocm:v0.16.0
-FROM ${BASE_IMAGE} AS base
-
-# Declare a variable to know if we want to use the nightly build or the stable build.
-# NOTE: REMINDER to vLLM-Omni rebase maintainer
-# Remember to set `USE_NIGHTLY_BUILD` to 0, when switching back to stable vLLM docker image
-# 1. If vLLM-omni maintainer is forced to use custom commits
-#    during rebasing, they can change the two variables
-# 2. Whenever vLLM upstream has released a stable version,
-#    we should swap over to use stable release ASAP.
-#    We should avoid relying on custom commits.
-ARG USE_NIGHTLY_BUILD=0
-ARG VLLM_VERSION_OR_COMMIT_HASH=2d5be1dd5ce2e44dfea53ea03ff61143da5137eb
-ARG ARG_PYTORCH_ROCM_ARCH
-ENV PYTORCH_ROCM_ARCH=${ARG_PYTORCH_ROCM_ARCH:-${PYTORCH_ROCM_ARCH}}
-
-ARG COMMON_WORKDIR=/app
-WORKDIR ${COMMON_WORKDIR}
-
-# Step 1: Setup - Install system dependencies
-RUN apt-get update && \
-    apt-get install -y ffmpeg && \
-    apt-get clean && \
-    rm -rf /var/lib/apt/lists/*
-
-# Step 2: Conditionally reinstall vllm from source for nightly builds
-RUN if [ "${USE_NIGHTLY_BUILD}" = "1" ]; then \
-        python3 -m pip uninstall -y vllm && rm -rf vllm && \
-        git clone https://github.com/vllm-project/vllm.git && \
-        cd vllm && \
-        git checkout ${VLLM_VERSION_OR_COMMIT_HASH} && \
-        python3 -m pip install -r requirements/rocm.txt && \
-        python3 setup.py clean --all && \
-        PYTORCH_ROCM_ARCH=${PYTORCH_ROCM_ARCH} python3 setup.py develop && \
-        cd ../ && \
-        rm -rf vllm/.git; \
-    fi
-
-# Step 3: Copy vllm-omni code and install without uv
-RUN mkdir -p ${COMMON_WORKDIR}/vllm-omni
-COPY . ${COMMON_WORKDIR}/vllm-omni
-RUN cd ${COMMON_WORKDIR}/vllm-omni && uv pip install --python "$(python3 -c 'import sys; print(sys.executable)')" --no-cache-dir ".[dev]" --no-build-isolation
-
-RUN ln -sf /usr/bin/python3 /usr/bin/python
-
-FROM base AS test
-
-CMD ["/bin/bash"]
-ENTRYPOINT []
-
-# Set entrypoint for vllm-openai official images
-FROM base AS vllm-openai
-ENTRYPOINT ["vllm", "serve", "--omni"]

docker/Dockerfile.xpu

@@ -1,139 +0,0 @@
-# Argument to configure vllm base image if pre-built
-ARG VLLM_BASE=vllm-base
-
-FROM intel/deep-learning-essentials:2025.3.2-0-devel-ubuntu24.04 AS vllm-base
-
-WORKDIR /workspace/
-
-ARG PYTHON_VERSION=3.12
-ARG PIP_EXTRA_INDEX_URL="https://download.pytorch.org/whl/xpu"
-
-RUN wget -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB | gpg --dearmor | tee /usr/share/keyrings/oneapi-archive-keyring.gpg > /dev/null && \
-    echo "deb [signed-by=/usr/share/keyrings/oneapi-archive-keyring.gpg] https://apt.repos.intel.com/oneapi all main" | tee /etc/apt/sources.list.d/oneAPI.list && \
-    add-apt-repository -y ppa:kobuk-team/intel-graphics
-
-RUN apt clean && apt-get update -y && \
-    apt-get install -y --no-install-recommends --fix-missing \
-    curl \
-    ffmpeg \
-    git \
-    libsndfile1 \
-    libsm6 \
-    libxext6 \
-    libgl1 \
-    lsb-release \
-    libaio-dev \
-    numactl \
-    wget \
-    vim \
-    python3.12 \
-    python3.12-dev \
-    python3-pip
-
-RUN apt update && apt upgrade -y && \
-    apt install -y libze1 libze-dev libze-intel-gpu1 intel-opencl-icd libze-intel-gpu-raytracing intel-ocloc && \
-    apt install -y intel-oneapi-compiler-dpcpp-cpp-2025.3
-
-ENV PATH="/root/.local/bin:$PATH"
-ENV VIRTUAL_ENV="/opt/venv"
-ENV UV_PYTHON_INSTALL_DIR=/opt/uv/python
-RUN curl -LsSf https://astral.sh/uv/install.sh | sh
-RUN uv venv --python ${PYTHON_VERSION} --seed ${VIRTUAL_ENV}
-ENV PATH="$VIRTUAL_ENV/bin:$PATH"
-
-# This oneccl contains the BMG support which is not the case for default version of oneapi 2025.2.
-ARG ONECCL_INSTALLER="intel-oneccl-2021.15.7.8_offline.sh"
-RUN wget "https://github.com/uxlfoundation/oneCCL/releases/download/2021.15.7/${ONECCL_INSTALLER}" && \
-    bash "${ONECCL_INSTALLER}" -a --silent --eula accept && \
-    rm "${ONECCL_INSTALLER}" && \
-    echo "source /opt/intel/oneapi/setvars.sh --force" >> /root/.bashrc && \
-    echo "source /opt/intel/oneapi/ccl/2021.15/env/vars.sh --force" >> /root/.bashrc
-RUN rm -f /opt/intel/oneapi/ccl/latest && \
-    ln -s /opt/intel/oneapi/ccl/2021.15 /opt/intel/oneapi/ccl/latest
-
-SHELL ["bash", "-c"]
-CMD ["bash", "-c", "source /root/.bashrc && exec bash"]
-
-WORKDIR /workspace/
-ENV UV_HTTP_TIMEOUT=500
-
-# Configure package index for XPU
-ENV PIP_EXTRA_INDEX_URL=${PIP_EXTRA_INDEX_URL}
-ENV UV_EXTRA_INDEX_URL=${PIP_EXTRA_INDEX_URL}
-ENV UV_INDEX_STRATEGY="unsafe-best-match"
-ENV UV_LINK_MODE="copy"
-
-ARG VLLM_VERSION=v0.16.0
-RUN git clone -b ${VLLM_VERSION} https://github.com/vllm-project/vllm
-WORKDIR /workspace/vllm
-
-RUN --mount=type=cache,target=/root/.cache/uv \
-    uv pip install --upgrade pip && \
-    uv pip install -r requirements/xpu.txt
-
- # used for suffix method speculative decoding
- # build deps for proto + nanobind-based extensions to set up the build environment
-RUN --mount=type=cache,target=/root/.cache/uv \
-    uv pip install grpcio-tools protobuf nanobind
- # arctic-inference is built from source which needs torch-xpu properly installed first
-RUN --mount=type=cache,target=/root/.cache/uv \
-    source /opt/intel/oneapi/setvars.sh --force && \
-    source /opt/intel/oneapi/ccl/2021.15/env/vars.sh --force && \
-    export CMAKE_PREFIX_PATH="$(python -c 'import site; print(site.getsitepackages()[0])'):${CMAKE_PREFIX_PATH}" && \
-    uv pip install --no-build-isolation arctic-inference==0.1.1
-
-ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/local/lib/"
-
-ENV VLLM_TARGET_DEVICE=xpu
-ENV VLLM_WORKER_MULTIPROC_METHOD=spawn
-
-RUN --mount=type=cache,target=/root/.cache/uv \
-    uv pip install --no-build-isolation .
-
-CMD ["/bin/bash"]
-
-FROM vllm-base AS vllm-openai
-
-# install additional dependencies for openai api server
-RUN --mount=type=cache,target=/root/.cache/uv \
-    uv pip install accelerate hf_transfer pytest pytest_asyncio lm_eval[api] modelscope
-
-# install development dependencies (for testing)
-RUN uv pip install -e tests/vllm_test_utils
-
-# install nixl from source code
-ENV NIXL_VERSION=0.7.0
-RUN python /workspace/vllm/tools/install_nixl_from_source_ubuntu.py
-
-# ensure vllm is properly installed
-RUN python -c "import vllm, inspect; print(vllm.__file__)"
-RUN uv pip show vllm
-
-CMD ["/bin/bash"]
-
-ENTRYPOINT []
-
-FROM ${VLLM_BASE} AS vllm-omni
-
-WORKDIR /workspace/vllm-omni
-COPY . .
-
-ENV VLLM_OMNI_TARGET_DEVICE=xpu
-RUN uv pip install --no-cache-dir ".[dev]" --no-build-isolation
-
-# FIX triton
-RUN --mount=type=cache,target=/root/.cache/uv \
-    uv pip uninstall triton triton-xpu && \
-    uv pip install triton-xpu==3.6.0
-
-# remove torch bundled oneccl to avoid conflicts
-RUN --mount=type=cache,target=/root/.cache/uv \
-    uv pip uninstall oneccl oneccl-devel
-
-FROM vllm-omni AS vllm-omni-openai
-
-RUN ln -sf /usr/bin/python3 /usr/bin/python
-
-ENV VLLM_WORKER_MULTIPROC_METHOD=spawn
-
-ENTRYPOINT ["vllm", "serve", "--omni"]

docs/.nav.yml

@@ -1,88 +0,0 @@
-nav:
-- Home: README.md
-- User Guide:
-  - Getting Started:
-    - getting_started/quickstart.md
-    - getting_started/installation/*
-  - Serving:
-    - OpenAI-Compatible API:
-      - Image Generation: serving/image_generation_api.md
-      - Image Edit: serving/image_edit_api.md
-      - Text to Speech: serving/speech_api.md
-  - Examples:
-    - examples/README.md
-    - Offline Inference:
-      - BAGEL-7B-MoT: user_guide/examples/offline_inference/bagel.md
-      - Image-To-Image: user_guide/examples/offline_inference/image_to_image.md
-      - Image-To-Video: user_guide/examples/offline_inference/image_to_video.md
-      - LoRA Inference(Diffusion): user_guide/examples/offline_inference/lora_inference.md
-      - Qwen2.5-Omni: user_guide/examples/offline_inference/qwen2_5_omni.md
-      - Qwen3-Omni: user_guide/examples/offline_inference/qwen3_omni.md
-      - Qwen3-TTS: user_guide/examples/offline_inference/qwen3_tts.md
-      - Text-To-Audio: user_guide/examples/offline_inference/text_to_audio.md
-      - Text-To-Image: user_guide/examples/offline_inference/text_to_image.md
-      - Text-To-Video: user_guide/examples/offline_inference/text_to_video.md
-    - Online Serving:
-      - BAGEL-7B-MoT: user_guide/examples/online_serving/bagel.md
-      - Image-To-Image: user_guide/examples/online_serving/image_to_image.md
-      - LoRA Inference(Diffusion): user_guide/examples/online_serving/lora_inference.md
-      - Qwen2.5-Omni: user_guide/examples/online_serving/qwen2_5_omni.md
-      - Qwen3-Omni: user_guide/examples/online_serving/qwen3_omni.md
-      - Qwen3-TTS: user_guide/examples/online_serving/qwen3_tts.md
-      - Text-To-Image: user_guide/examples/online_serving/text_to_image.md
-  - General:
-    - usage/*
-  - Configuration:
-    - configuration/README.md
-    - configuration/*
-  - Models:
-    - models/supported_models.md
-  - Features:
-    - Sleep Mode: features/sleep_mode.md
-    - Diffusion Features:
-      - Acceleration Overview: user_guide/diffusion_acceleration.md
-      - TeaCache: user_guide/diffusion/teacache.md
-      - Cache-DiT: user_guide/diffusion/cache_dit_acceleration.md
-      - Quantization:
-        - Overview: user_guide/diffusion/quantization/overview.md
-        - FP8: user_guide/diffusion/quantization/fp8.md
-      - Parallelism Acceleration: user_guide/diffusion/parallelism_acceleration.md
-      - CPU Offloading: user_guide/diffusion/cpu_offload_diffusion.md
-      - LoRA: user_guide/diffusion/lora.md
-    - ComfyUI: features/comfyui.md
-- Developer Guide:
-  - General:
-    - contributing/README.md
-    - glob: contributing/*
-      flatten_single_child_sections: true
-  - Model Implementation:
-    - contributing/model/README.md
-    - contributing/model/adding_omni_model.md
-    - contributing/model/adding_diffusion_model.md
-  - CI: contributing/ci
-  - Design Documents:
-    - design/index.md
-    - design/architecture_overview.md
-    - Feature Design:
-      - design/feature/disaggregated_inference.md
-      - design/feature/ray_based_execution.md
-      - design/feature/omni_connectors/
-      - design/feature/cfg_parallel.md
-      - design/feature/sequence_parallel.md
-      - design/feature/tensor_parallel.md
-      - design/feature/cache_dit.md
-      - design/feature/teacache.md
-    - Module Design:
-      - design/module/ar_module.md
-      - design/module/dit_module.md
-      - design/module/entrypoint_module.md
-  - Docs Guide: contributing/DOCS_GUIDE.md
-- API Reference:
-  - api/README.md
-  - api/vllm_omni
-- CLI Reference: cli
-- Community:
-  - community/*
-  - Slack: https://slack.vllm.ai
-  - Blog: https://blog.vllm.ai
-  - Forum: https://discuss.vllm.ai

docs/README.md

@@ -1,64 +0,0 @@
----
-hide:
-  - navigation
-  - toc
----
-
-# Welcome to vLLM-Omni
-
-<p align="center">
-  <picture>
-    <source media="(prefers-color-scheme: dark)" src="./source/logos/vllm-omni-logo.png">
-    <img alt="vllm-omni" src="./source/logos/vllm-omni-logo.png" width=55%>
-  </picture>
-</p>
-<h3 align="center">
-Easy, fast, and cheap omni-modality model serving for everyone
-</h3>
-
-<p style="text-align:center">
-<script async defer src="https://buttons.github.io/buttons.js"></script>
-<a class="github-button" href="https://github.com/vllm-project/vllm-omni" data-show-count="true" data-size="large" aria-label="Star">Star</a>
-<a class="github-button" href="https://github.com/vllm-project/vllm-omni/subscription" data-show-count="true" data-icon="octicon-eye" data-size="large" aria-label="Watch">Watch</a>
-<a class="github-button" href="https://github.com/vllm-project/vllm-omni/fork" data-show-count="true" data-icon="octicon-repo-forked" data-size="large" aria-label="Fork">Fork</a>
-</p>
-
-
-## About
-
-[vLLM](https://github.com/vllm-project/vllm) was originally designed to support large language models for text-based autoregressive generation tasks. vLLM-Omni is a framework that extends its support for omni-modality model inference and serving:
-
-- **Omni-modality**: Text, image, video, and audio data processing
-- **Non-autoregressive Architectures**: extend the AR support of vLLM to Diffusion Transformers (DiT) and other parallel generation models
-- **Heterogeneous outputs**: from traditional text generation to multimodal outputs
-
-<p align="center">
-  <picture>
-    <source media="(prefers-color-scheme: dark)" src="./source/architecture/omni-modality-model-architecture.png">
-    <img alt="vllm-omni-arch" src="./source/architecture/omni-modality-model-architecture.png" width=55%>
-  </picture>
-</p>
-
-vLLM-Omni is fast with:
-
-- State-of-the-art AR support by leveraging efficient KV cache management from vLLM
-- Pipelined stage execution overlapping for high throughput performance
-- Fully disaggregation based on OmniConnector and dynamic resource allocation across stages
-
-vLLM-Omni is flexible and easy to use with:
-
-- Heterogeneous pipeline abstraction to manage complex model workflows
-- Seamless integration with popular Hugging Face models
-- Tensor, pipeline, data and expert parallelism support for distributed inference
-- Streaming outputs
-- OpenAI-compatible API server
-
-vLLM-Omni seamlessly supports most popular open-source models on HuggingFace, including:
-
-- Omni-modality models (e.g. Qwen2.5-Omni, Qwen3-Omni)
-- Multi-modality generation models (e.g. Qwen-Image)
-
-For more information, checkout the following:
-
-- [vllm-omni architecture design and recent roadmaps](https://docs.google.com/presentation/d/1qv4qMW1rKAqDREMXiUDLIgqqHQe7TDPj/edit?usp=sharing&ouid=110473603432222024453&rtpof=true&sd=true)
-- [vllm-omni announcement blogpost](https://blog.vllm.ai/2025/11/30/vllm-omni.html)

docs/api/README.md

@@ -1,112 +0,0 @@
-# Summary
-
-## Entry Points
-
-Main entry points for vLLM-Omni inference and serving.
-
-- [vllm_omni.entrypoints.async_omni.AsyncOmni][]
-- [vllm_omni.entrypoints.async_omni_diffusion.AsyncOmniDiffusion][]
-- [vllm_omni.entrypoints.async_omni_llm.AsyncOmniLLM][]
-- [vllm_omni.entrypoints.chat_utils.extract_audio_from_video_async][]
-- [vllm_omni.entrypoints.cli.benchmark.base.OmniBenchmarkSubcommandBase][]
-- [vllm_omni.entrypoints.cli.benchmark.main.OmniBenchmarkSubcommand][]
-- [vllm_omni.entrypoints.cli.benchmark.serve.OmniBenchmarkServingSubcommand][]
-- [vllm_omni.entrypoints.cli.serve.OmniServeCommand][]
-- [vllm_omni.entrypoints.client_request_state.ClientRequestState][]
-- [vllm_omni.entrypoints.omni.Omni][]
-- [vllm_omni.entrypoints.omni.OmniBase][]
-- [vllm_omni.entrypoints.omni_diffusion.OmniDiffusion][]
-- [vllm_omni.entrypoints.omni_llm.OmniLLM][]
-- [vllm_omni.entrypoints.omni_stage.OmniStage][]
-- [vllm_omni.entrypoints.stage_utils.OmniStageTaskType][]
-
-## Inputs
-
-Input data structures for multi-modal inputs.
-
-- [vllm_omni.inputs.data.OmniDiffusionSamplingParams][]
-- [vllm_omni.inputs.data.OmniEmbedsPrompt][]
-- [vllm_omni.inputs.data.OmniTextPrompt][]
-- [vllm_omni.inputs.data.OmniTokenInputs][]
-- [vllm_omni.inputs.data.OmniTokensPrompt][]
-- [vllm_omni.inputs.preprocess.OmniInputPreprocessor][]
-
-## Engine
-
-Engine classes for offline and online inference.
-
-- [vllm_omni.diffusion.diffusion_engine.DiffusionEngine][]
-- [vllm_omni.engine.AdditionalInformationEntry][]
-- [vllm_omni.engine.AdditionalInformationPayload][]
-- [vllm_omni.engine.OmniEngineCoreOutput][]
-- [vllm_omni.engine.OmniEngineCoreOutputs][]
-- [vllm_omni.engine.OmniEngineCoreRequest][]
-- [vllm_omni.engine.PromptEmbedsPayload][]
-- [vllm_omni.engine.arg_utils.AsyncOmniEngineArgs][]
-- [vllm_omni.engine.arg_utils.OmniEngineArgs][]
-- [vllm_omni.engine.input_processor.OmniInputProcessor][]
-- [vllm_omni.engine.output_processor.MultimodalOutputProcessor][]
-- [vllm_omni.engine.output_processor.OmniRequestState][]
-
-## Core
-
-Core scheduling and caching components.
-
-- [vllm_omni.core.sched.omni_ar_scheduler.KVCacheTransferData][]
-- [vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler][]
-- [vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler][]
-- [vllm_omni.core.sched.output.OmniCachedRequestData][]
-- [vllm_omni.core.sched.output.OmniNewRequestData][]
-- [vllm_omni.core.sched.output.OmniSchedulerOutput][]
-- [vllm_omni.model_executor.models.qwen3_tts.tokenizer_25hz.vq.core_vq.DistributedGroupResidualVectorQuantization][]
-- [vllm_omni.model_executor.models.qwen3_tts.tokenizer_25hz.vq.core_vq.DistributedResidualVectorQuantization][]
-- [vllm_omni.model_executor.models.qwen3_tts.tokenizer_25hz.vq.core_vq.EuclideanCodebook][]
-- [vllm_omni.model_executor.models.qwen3_tts.tokenizer_25hz.vq.core_vq.VectorQuantization][]
-- [vllm_omni.model_executor.models.qwen3_tts.tokenizer_25hz.vq.core_vq.preprocess][]
-
-## Configuration
-
-Configuration classes.
-
-- [vllm_omni.config.model.OmniModelConfig][]
-- [vllm_omni.diffusion.cache.teacache.config.TeaCacheConfig][]
-- [vllm_omni.distributed.omni_connectors.utils.config.ConnectorSpec][]
-- [vllm_omni.distributed.omni_connectors.utils.config.OmniTransferConfig][]
-- [vllm_omni.model_executor.models.qwen3_tts.configuration_qwen3_tts.Qwen3TTSConfig][]
-- [vllm_omni.model_executor.models.qwen3_tts.configuration_qwen3_tts.Qwen3TTSSpeakerEncoderConfig][]
-- [vllm_omni.model_executor.models.qwen3_tts.configuration_qwen3_tts.Qwen3TTSTalkerCodePredictorConfig][]
-- [vllm_omni.model_executor.models.qwen3_tts.configuration_qwen3_tts.Qwen3TTSTalkerConfig][]
-- [vllm_omni.model_executor.models.qwen3_tts.tokenizer_12hz.configuration_qwen3_tts_tokenizer_v2.Qwen3TTSTokenizerV2Config][]
-- [vllm_omni.model_executor.models.qwen3_tts.tokenizer_12hz.configuration_qwen3_tts_tokenizer_v2.Qwen3TTSTokenizerV2DecoderConfig][]
-- [vllm_omni.model_executor.models.qwen3_tts.tokenizer_25hz.configuration_qwen3_tts_tokenizer_v1.Qwen3TTSTokenizerV1Config][]
-- [vllm_omni.model_executor.models.qwen3_tts.tokenizer_25hz.configuration_qwen3_tts_tokenizer_v1.Qwen3TTSTokenizerV1DecoderBigVGANConfig][]
-- [vllm_omni.model_executor.models.qwen3_tts.tokenizer_25hz.configuration_qwen3_tts_tokenizer_v1.Qwen3TTSTokenizerV1DecoderConfig][]
-- [vllm_omni.model_executor.models.qwen3_tts.tokenizer_25hz.configuration_qwen3_tts_tokenizer_v1.Qwen3TTSTokenizerV1DecoderDiTConfig][]
-- [vllm_omni.model_executor.models.qwen3_tts.tokenizer_25hz.configuration_qwen3_tts_tokenizer_v1.Qwen3TTSTokenizerV1EncoderConfig][]
-
-## Workers
-
-Worker classes and model runners for distributed inference.
-
-- [vllm_omni.diffusion.worker.diffusion_model_runner.DiffusionModelRunner][]
-- [vllm_omni.diffusion.worker.diffusion_worker.DiffusionWorker][]
-- [vllm_omni.diffusion.worker.diffusion_worker.WorkerProc][]
-- [vllm_omni.platforms.npu.worker.npu_ar_model_runner.ExecuteModelState][]
-- [vllm_omni.platforms.npu.worker.npu_ar_model_runner.NPUARModelRunner][]
-- [vllm_omni.platforms.npu.worker.npu_ar_worker.NPUARWorker][]
-- [vllm_omni.platforms.npu.worker.npu_generation_model_runner.NPUGenerationModelRunner][]
-- [vllm_omni.platforms.npu.worker.npu_generation_worker.NPUGenerationWorker][]
-- [vllm_omni.platforms.npu.worker.npu_model_runner.OmniNPUModelRunner][]
-- [vllm_omni.platforms.xpu.worker.xpu_ar_model_runner.XPUARModelRunner][]
-- [vllm_omni.platforms.xpu.worker.xpu_ar_worker.XPUARWorker][]
-- [vllm_omni.platforms.xpu.worker.xpu_generation_model_runner.XPUGenerationModelRunner][]
-- [vllm_omni.platforms.xpu.worker.xpu_generation_worker.XPUGenerationWorker][]
-- [vllm_omni.worker.base.OmniGPUWorkerBase][]
-- [vllm_omni.worker.gpu_ar_model_runner.ExecuteModelState][]
-- [vllm_omni.worker.gpu_ar_model_runner.GPUARModelRunner][]
-- [vllm_omni.worker.gpu_ar_worker.GPUARWorker][]
-- [vllm_omni.worker.gpu_generation_model_runner.GPUGenerationModelRunner][]
-- [vllm_omni.worker.gpu_generation_worker.GPUGenerationWorker][]
-- [vllm_omni.worker.gpu_memory_utils.parse_cuda_visible_devices][]
-- [vllm_omni.worker.gpu_model_runner.OmniGPUModelRunner][]
-- [vllm_omni.worker.mixins.OmniWorkerMixin][]

docs/cli/README.md

@@ -1,42 +0,0 @@
-# vLLM-Omni CLI Guide
-
-The CLI for vLLM-Omni inherits from vllm with some additional arguments.
-
-## serve
-
-Starts the vLLM-Omni OpenAI Compatible API server.
-
-Start with a model:
-
-```bash
-vllm serve Qwen/Qwen2.5-Omni-7B --omni
-```
-
-Specify the port:
-
-```bash
-vllm serve Qwen/Qwen2.5-Omni-7B --omni --port 8091
-```
-
-If you have custom stage configs file, launch the server with command below
-```bash
-vllm serve Qwen/Qwen2.5-Omni-7B --omni --stage-configs-path /path/to/stage_configs_file
-```
-
-
-## bench
-
-Run benchmark tests for online serving throughput.
-Available Commands:
-
-```bash
-vllm bench serve --omni \
-    --model Qwen/Qwen2.5-Omni-7B \
-    --host server-host \
-    --port server-port \
-    --random-input-len 32 \
-    --random-output-len 4  \
-    --num-prompts  5
-```
-
-See [vllm bench serve](./bench/serve.md) for the full reference of all available arguments.

docs/cli/bench/serve.md

@@ -1,359 +0,0 @@
-# vLLM-Omni Benchmark CLI Guide
-The vllm bench command launches the vLLM-Omni benchmark to evaluate the performance of multimodal models.
-
-## Notes
-We currently only support using the "openai-chat-omni" backend.
-
-## Basic Parameter Description
-You can use `vllm bench serve --omni --help=all` to get descriptions of all parameters. The commonly used parameters are described below:
-- `--omni`  
-  Enable Omni (multimodal) mode, supporting multimodal inputs and outputs such as images, videos, and audio.
-
-- `--backend`  
-  Specify the backend adapter as openai-chat-omni, using OpenAI Chat compatible API behavior as the protocol. Currently only openai-chat-omni is supported.
-
-- `--model`  
-  The model identifier to load, filled according to the models supported by vLLM-Omni.
-
-- `--endpoint`  
-  The API endpoint exposed externally, to which clients send their requests.
-
-- `--dataset-name`  
-  The name of the dataset used; random-mm indicates generating random multimodal inputs (images, videos, audio).
-
-- `--num-prompts`  
-  The total number of requests to send, an integer.
-
-- `--max-concurrency`  
-  "Maximum number of concurrent requests. This can be used "
-        "to help simulate an environment where a higher level component "
-        "is enforcing a maximum number of concurrent requests. While the "
-        "--request-rate argument controls the rate at which requests are "
-        "initiated, this argument will control how many are actually allowed "
-        "to execute at a time. This means that when used in combination, the "
-        "actual request rate may be lower than specified with --request-rate, "
-        "if the server is not processing requests fast enough to keep up."
-
-- `--request-rate`  
-  "Number of requests per second. If this is inf, "
-        "then all the requests are sent at time 0. "
-        "Otherwise, we use Poisson process or gamma distribution "
-        "to synthesize the request arrival times."
-
-- `--ignore-eos`  
-  "Set ignore_eos flag when sending the benchmark request."
-
-- `--metric-percentiles`  
-  Comma-separated list of percentiles for selected metrics. "
-        "To report 25-th, 50-th, and 75-th percentiles, use \"25,50,75\". "
-        "Default value is \"99\"."
-        "Use \"--percentile-metrics\" to select metrics.
-
-- `--percentile-metrics`  
-        "Comma-separated list of selected metrics to report percentiles."
-                    "This argument specifies the metrics to report percentiles."
-                    'Allowed metric names are "ttft", "tpot", "itl", "e2el", "audio_ttfp", "audio_rtf", "audio_duration". '
-
-- `--save-result`  
-Specify to save benchmark results to a json file
-
-- `--save-detailed`  
-"When saving the results, whether to include per request "
-        "information such as response, error, ttfs, tpots, etc."
-
-- `--result-dir`  
- "Specify directory to save benchmark json results."
-        "If not specified, results are saved in the current directory."
-
-- `--result-filename`  
-"Specify the filename to save benchmark json results."
-        "If not specified, results will be saved in "
-        "{label}-{args.request_rate}qps-{base_model_id}-{current_dt}.json"
-
-- `--random-prefix-len`  
-  Number of fixed prefix tokens before the random context in a request.
-  The total input length is the sum of random-prefix-len and a random
-  context length sampled from [input_len * (1 - range_ratio),
-  input_len * (1 + range_ratio)].Only the random and random-mm modes
-  support this parameter.
-
-- `--random-input-len`  
-  Number of input tokens per request.Only the random and random-mm modes support this parameter.
-
-- `--random-output-len`  
-  Number of output tokens per request.Only the random and random-mm modes support this parameter.
-
-- `--random-range-ratio`  
-  Range ratio for sampling input/output length,
-  used only for random sampling. Must be in the range [0, 1) to define
-  a symmetric sampling range
-  [length * (1 - range_ratio), length * (1 + range_ratio)].
-  Only the random and random-mm modes support this parameter.
-
-- `--random-mm-base-items-per-request`  
-  Base number of multimodal items per request for random-mm.
-  Actual per-request count is sampled around this base using
-  --random-mm-num-mm-items-range-ratio.
-  Only the random-mm mode supports this parameter.
-
-- `--random-mm-limit-mm-per-prompt`  
-  Per-modality hard caps for items attached per request, e.g.
-  '{"image": 3, "video": 1, "audio": 1}'. The sampled per-request item
-  count is clamped to the sum of these limits. When a modality
-  reaches its cap, its buckets are excluded and probabilities are
-  renormalized.
-  Only the random-mm mode supports this parameter.
-
-- `--random-mm-num-mm-items-range-ratio`  
-  Range ratio r in [0, 1] for sampling items per request.
-  We sample uniformly from the closed integer range
-  [floor(n*(1-r)), ceil(n*(1+r))]
-  where n is the base items per request.
-  r=0 keeps it fixed; r=1 allows 0 items. The maximum is clamped
-  to the sum of per-modality limits from
-  --random-mm-limit-mm-per-prompt.
-  An error is raised if the computed min exceeds the max.
-  Only the random-mm mode supports this parameter.
-
-- `--random-mm-bucket-config`  
-  The bucket config is a dictionary mapping a multimodal item
-  sampling configuration to a probability.
-  Currently allows for 3 modalities: audio, images and videos.
-  A bucket key is a tuple of (height, width, num_frames)
-  The value is the probability of sampling that specific item.
-  Example:
-  --random-mm-bucket-config
-  "{(256, 256, 1): 0.5, (720, 1280, 16): 0.4, (0, 1, 5): 0.10}"
-  First item: images with resolution 256x256 w.p. 0.5
-  Second item: videos with resolution 720x1280 and 16 frames
-  Third item: audios with 1s duration and 5 channels w.p. 0.1
-  OBS.: If the probabilities do not sum to 1, they are normalized.
-  Only the random-mm mode supports this parameter
-
-## Usage Examples
-
-### Online Benchmark
-<details class="admonition abstract" markdown="1">
-<summary>Show more</summary>
-
-First start serving your model:
-
-```bash
-vllm serve Qwen/Qwen2.5-Omni-7B --omni
-```
-
-Then run the benchmarking for sharegpt:
-
-```bash
-# download dataset
-# wget https://huggingface.co/datasets/anon8231489123/ShareGPT_Vicuna_unfiltered/resolve/main/ShareGPT_V3_unfiltered_cleaned_split.json
-vllm bench serve \
-  --omni \
-  --port 43845 \
-  --model /home/models/Qwen/Qwen3-Omni-30B-A3B-Instruct \
-  --endpoint /v1/chat/completions \
-  --backend openai-chat-omni \
-  --num-prompts 2 \
-  --dataset-name sharegpt \
-  --dataset-path ShareGPT_V3_unfiltered_cleaned_split.json \
-  --percentile-metrics ttft,tpot,itl,e2el
-```
-If successful, you will see the following output:
-```text
-============ Serving Benchmark Result ============
-Successful requests:                     2
-Failed requests:                         0
-Benchmark duration (s):                  81.63
-Request throughput (req/s):              0.02
-Peak concurrent requests:                2.00
-----------------End-to-end Latency----------------
-Mean E2EL (ms):                          56966.13
-Median E2EL (ms):                        56966.13
-P99 E2EL (ms):                           81016.80
-================== Text Result ===================
-Total input tokens:                      36
-Total generated tokens:                  5926
-Output token throughput (tok/s):         72.60
-Peak output token throughput (tok/s):    103.00
-Peak concurrent requests:                2.00
-Total Token throughput (tok/s):          73.04
----------------Time to First Token----------------
-Mean TTFT (ms):                          124.76
-Median TTFT (ms):                        124.76
-P99 TTFT (ms):                           156.10
------Time per Output Token (excl. 1st token)------
-Mean TPOT (ms):                          481.30
-Median TPOT (ms):                        481.30
-P99 TPOT (ms):                           947.55
----------------Inter-token Latency----------------
-Mean ITL (ms):                           25.11
-Median ITL (ms):                         0.33
-P99 ITL (ms):                            25.17
-================== Audio Result ==================
-Total audio duration generated(s):       3.95
-Total audio frames generated:            94890
-Audio throughput(audio duration/s):      0.05
-==================================================
-```
-
-Or run the benchmarking for random:
-
-```bash
-vllm bench serve \
-  --omni \
-  --port 43845 \
-  --endpoint /v1/chat/completions \
-  --backend openai-chat-omni \
-  --model /home/models/Qwen/Qwen3-Omni-30B-A3B-Instruct \
-  --dataset-name random \
-  --num-prompts 2 \
-  --random-prefix-len 5 \
-  --random-input-len 10 \
-  --random-output-len 100 \
-  --percentile-metrics ttft,tpot,itl,e2el,audio_ttfp,audio_rtf \
-  --ignore-eos
-```
-
-If successful, you will see the following output:
-
-```text
-============ Serving Benchmark Result ============
-Successful requests:                     2
-Failed requests:                         0
-Benchmark duration (s):                  3.89
-Request throughput (req/s):              0.51
-Peak concurrent requests:                2.00
-----------------End-to-end Latency----------------
-Mean E2EL (ms):                          3824.76
-Median E2EL (ms):                        3824.76
-P99 E2EL (ms):                           3888.54
-================== Text Result ===================
-Total input tokens:                      30
-Total generated tokens:                  10101
-Output token throughput (tok/s):         2595.57
-Peak output token throughput (tok/s):    111.00
-Peak concurrent requests:                2.00
-Total Token throughput (tok/s):          2603.28
----------------Time to First Token----------------
-Mean TTFT (ms):                          117.15
-Median TTFT (ms):                        117.15
-P99 TTFT (ms):                           142.69
------Time per Output Token (excl. 1st token)------
-Mean TPOT (ms):                          0.73
-Median TPOT (ms):                        0.73
-P99 TPOT (ms):                           0.74
----------------Inter-token Latency----------------
-Mean ITL (ms):                           16.47
-Median ITL (ms):                         16.19
-P99 ITL (ms):                            52.55
-================== Audio Result ==================
-Total audio duration generated(s):       15.79
-Total audio frames generated:            379050
-Audio throughput(audio duration/s):      4.06
----------------Time to First Packet---------------
-Mean AUDIO_TTFP (ms):                    3701.37
-Median AUDIO_TTFP (ms):                  3701.37
-P99 AUDIO_TTFP (ms):                     3762.25
------------------Real Time Factor-----------------
-Mean AUDIO_RTF:                          0.47
-Median AUDIO_RTF:                        0.47
-P99 AUDIO_RTF:                           0.48
-==================================================
-```
-Notes:
-We use audio generation time / audio duration to calculate RTF.
-
-</details>
-
-### Multi-Modal Benchmark
-
-<details class="admonition abstract" markdown="1">
-<summary>Show more</summary>
-
-Benchmark the performance of multi-modal requests in vLLM-Omni.
-
-Generate synthetic image、video、audio inputs alongside random text prompts to stress-test vision models without external datasets.
-
-Notes:
-
-- Works only with online benchmark via the OpenAI backend (`--backend openai-chat-omni`) and endpoint `/v1/chat/completions`.
-
-Start the server (example):
-
-```bash
-vllm serve Qwen/Qwen2.5-Omni-7B --omni
-```
-
-It is recommended to use the flag `--ignore-eos` to simulate real responses. You can set the size of the output via the arg `random-output-len`.
-
-Then run the benchmarking script:
-```bash
-vllm bench serve \
-    --omni \
-  --dataset-name random-mm \
-  --port 40849 \
-  --model /home/models/Qwen/Qwen3-Omni-30B-A3B-Instruct \
-  --endpoint /v1/chat/completions \
-  --backend openai-chat-omni \
-  --request-rate 1 \
-  --num-prompts 1 \
-  --random-input-len 10 \
-  --random-range-ratio 0.0 \
-  --random-mm-base-items-per-request 2 \
-  --random-mm-num-mm-items-range-ratio 0 \
-  --random-mm-limit-mm-per-prompt '{"image":1,"video":1, "audio": 1}' \
-  --random-mm-bucket-config '{"(32, 32, 1)": 0.5, "(0, 1, 1)": 0.1, "(32, 32, 2)":0.4}' \
-  --ignore-eos \
-  --percentile-metrics ttft,tpot,itl \
-  --random-output-len 2 \
-  --extra_body '{"modalities": ["text"]}'
-```
-
-If successful, you will see the following output:
-
-```text
-============ Serving Benchmark Result ============
-Successful requests:                     1
-Failed requests:                         0
-Request rate configured (RPS):           1.00
-Benchmark duration (s):                  1.21
-Request throughput (req/s):              0.83
-Peak concurrent requests:                1.00
-================== Text Result ===================
-Total input tokens:                      10
-Total generated tokens:                  3
-Output token throughput (tok/s):         2.49
-Peak output token throughput (tok/s):    3.00
-Peak concurrent requests:                1.00
-Total Token throughput (tok/s):          10.77
----------------Time to First Token----------------
-Mean TTFT (ms):                          179.74
-Median TTFT (ms):                        179.74
-P99 TTFT (ms):                           179.74
------Time per Output Token (excl. 1st token)------
-Mean TPOT (ms):                          12.76
-Median TPOT (ms):                        12.76
-P99 TPOT (ms):                           12.76
----------------Inter-token Latency----------------
-Mean ITL (ms):                           12.76
-Median ITL (ms):                         12.76
-P99 ITL (ms):                            25.24
-================== Audio Result ==================
-Total audio duration generated(s):       0.00
-Total audio frames generated:            0
-Audio throughput(audio duration/s):      0.00
-==================================================
-```
-
-Behavioral notes:
-
-- If the requested base item count cannot be satisfied under the provided per-prompt limits, the tool raises an error rather than silently clamping.
-
-How sampling works:
-
-- Determine per-request item count k by sampling uniformly from the integer range defined by `--random-mm-base-items-per-request` and `--random-mm-num-mm-items-range-ratio`, then clamp k to at most the sum of per-modality limits.
-- For each of the k items, sample a bucket (H, W, T) according to the normalized probabilities in `--random-mm-bucket-config`, while tracking how many items of each modality have been added.
-- If a modality (e.g., image) reaches its limit from `--random-mm-limit-mm-per-prompt`, all buckets of that modality are excluded and the remaining bucket probabilities are renormalized before continuing.
-This should be seen as an edge case, and if this behavior can be avoided by setting `--random-mm-limit-mm-per-prompt` to a large number. Note that this might result in errors due to engine config `--limit-mm-per-prompt`.
-- The resulting request contains synthetic image data in `multi_modal_data` (OpenAI Chat format). When `random-mm` is used with the OpenAI Chat backend, prompts remain text and MM content is attached via `multi_modal_data`.
-</details>

docs/cli/serve.md

@@ -1,9 +0,0 @@
-# vllm-omni serve
-
-## JSON CLI Arguments
-
---8<-- "docs/cli/json_tip.inc.md"
-
-## Arguments
-
---8<-- "docs/generated/argparse_omni/omni_serve.inc.md"

docs/community/contact_us.md

@@ -1,5 +0,0 @@
-# Contact Us
-
-- For technical questions and feature requests, please use GitHub [Issues](https://github.com/vllm-project/vllm-omni/issues)
-- For coordinating contributions and development and discussing with other users and developers, please join `sig-omni` channel in our [Slack](https://slack.vllm.ai/) or use the [vLLM Forum](https://discuss.vllm.ai/)
-- For security disclosures, please use GitHub's [Security Advisories](https://github.com/vllm-project/vllm-omni/security/advisories) feature

docs/community/meetups.md

@@ -1 +0,0 @@
-# Meetups

docs/community/volunteers.md

@@ -1,12 +0,0 @@
-# Volunteers for Bugfix and CI
-
-We encourage you to check current docs and [issues](https://github.com/vllm-project/vllm-omni/issues) to find possible solutions for your questions. If non of these can solve it, please propose an issue to describe your questions about bug or CI problems for developing.
-
-If you have urgent need for locating and solving bugfix or CI problems, please find community volunteers below.
-
-| Dec 4-Dec 12 | Dec 15-Dec 19 | Dec 22-Dec 26 | Dec 29- Jan 2, 2026| Jan 5-Jan 9 | Jan 12-Jan 16 |
-|----------|----------|----------|----------|----------|----------|
-| <a href="https://github.com/congw729">Conw729</a> | <a href="https://github.com/yinpeiqi">yinpeiqi</a> | <a href="https://github.com/tzhouam">tzhouam</a> | <a href="https://github.com/SamitHuang">SamitHuang</a> | <a href="https://github.com/gcanlin">gcanlin</a> | <a href="https://github.com/natureofnature">natureofnature</a> |
-| <a href="https://github.com/david6666666">david6666666</a> | <a href="https://github.com/R2-Y">R2-Y</a> | <a href="https://github.com/hsliuustc0106">hsliuustc0106</a> | <a href="https://github.com/Gaohan123">Gaohan123</a> | <a href="https://github.com/ZJY0516">ZJY0516</a> | <a href="https://github.com/qibaoyuan">qibaoyuan</a> |
-
-We kindly welcome more contributors to fix bugs and contribute new features!

docs/configuration/README.md

@@ -1,21 +0,0 @@
-# Configuration Options
-
-This section lists the most common options for running vLLM-Omni.
-
-For options within a vLLM Engine. Please refer to [vLLM Configuration](https://docs.vllm.ai/en/v0.14.0/configuration/index.html)
-
-Currently, the main options are maintained by stage configs for each model.
-
-For specific example, please refer to [Qwen2.5-omni stage config](stage_configs/qwen2_5_omni.yaml)
-
-For introduction, please check [Introduction for stage config](./stage_configs.md)
-
-## Memory Configuration
-
-- **[GPU Memory Calculation and Configuration](./gpu_memory_utilization.md)** - Guide on how to calculate memory requirements and set up `gpu_memory_utilization` for optimal performance
-
-## Optimization Features
-
-- **[TeaCache Configuration](../user_guide/diffusion/teacache.md)** - Enable TeaCache adaptive caching for DiT models to achieve 1.5x-2.0x speedup with minimal quality loss
-- **[Cache-DiT Configuration](../user_guide/diffusion/cache_dit_acceleration.md)** - Enable Cache-DiT as cache acceleration backends for DiT models
-- **[Parallelism Configuration](../user_guide/diffusion/parallelism_acceleration.md)** - Enable parallelism (e.g., sequence parallelism) for for DiT models

docs/configuration/gpu_memory_utilization.md

@@ -1,207 +0,0 @@
-# GPU Memory Calculation and Configuration
-
-This guide explains how to calculate GPU memory requirements and properly configure `gpu_memory_utilization` for vLLM-Omni stages.
-
-## Overview
-
-`gpu_memory_utilization` is a critical parameter that controls how much GPU memory each stage can use. It's specified as a fraction between 0.0 and 1.0, where:
-- `0.8` means 80% of the GPU's total memory
-- `1.0` means 100% of the GPU's total memory (not recommended, leaves no buffer)
-
-## How Memory is Calculated
-
-### Memory Allocation Formula
-
-For each stage, vLLM-Omni calculates the requested memory as:
-
-```
-requested_memory = total_gpu_memory × gpu_memory_utilization
-```
-
-The system checks that:
-```
-free_memory ≥ requested_memory
-```
-
-If this condition is not met, the stage will fail to initialize with an error message showing the memory requirements.
-
-### Memory Components
-
-The total memory used by a stage includes:
-
-1. **Model Weights**: The size of the model parameters loaded on the GPU
-2. **KV Cache**: Memory for storing key-value cache during generation
-3. **Activation Memory**: Temporary memory for intermediate computations
-4. **System Overhead**: Memory used by CUDA, PyTorch, and other system components
-5. **Non-Torch Memory**: Memory allocated outside of PyTorch (e.g., CUDA graphs)
-
-### Example Calculation
-
-For a GPU with 80GB total memory:
-- `gpu_memory_utilization: 0.8` → 64GB available for the stage
-- `gpu_memory_utilization: 0.6` → 48GB available for the stage
-- `gpu_memory_utilization: 0.15` → 12GB available for the stage
-
-## Setting Up `gpu_memory_utilization`
-
-### Step 1: Determine GPU Memory
-
-First, check your GPU's total memory:
-
-```bash
-# Using nvidia-smi
-nvidia-smi --query-gpu=memory.total --format=csv
-
-# Or using Python
-python -c "import torch; print(f'{torch.cuda.get_device_properties(0).total_memory / 1e9:.1f} GB')"
-```
-
-### Step 2: Estimate Model Memory Requirements
-
-#### For Autoregressive (AR) Stages
-
-AR stages typically need more memory due to:
-- Large model weights
-- KV cache for attention
-- Activation buffers
-
-#### For Diffusion/Generation Stages
-
-Diffusion stages (like code2wav) typically need less memory:
-- Smaller model components
-- Different memory access patterns
-
-**Typical values:**
-- `0.1 - 0.3` for most diffusion stages
-
-### Step 3: Consider Multi-Stage Scenarios
-
-When multiple stages share the same GPU, you must ensure the sum of their `gpu_memory_utilization` values doesn't exceed 1.0.
-
-**Example: Two stages on GPU 0**
-```yaml
-stage_args:
-  - stage_id: 0
-    runtime:
-      devices: "0"
-    engine_args:
-      gpu_memory_utilization: 0.6  # Uses 60% of GPU 0
-
-  - stage_id: 1
-    runtime:
-      devices: "0"
-    engine_args:
-      gpu_memory_utilization: 0.3  # Uses 30% of GPU 0
-      # Total: 90% of GPU 0 (safe, leaves 10% buffer)
-```
-
-**Important:** If stages run on different GPUs, each can use up to 1.0 independently.
-
-### Step 4: Account for Tensor Parallelism
-
-When using `tensor_parallel_size > 1`, the model is split across multiple GPUs, so each GPU needs less memory.
-
-**Example: 2-way tensor parallelism**
-```yaml
-stage_args:
-  - stage_id: 0
-    runtime:
-      devices: "0,1"  # Uses both GPUs
-    engine_args:
-      tensor_parallel_size: 2
-      gpu_memory_utilization: 0.6  # 60% per GPU
-      # Model is split, so each GPU uses ~30% of model memory
-```
-
-## Examples
-
-### Qwen3-Omni-MoE on 2x H100-80GB
-
-```yaml
-stage_args:
-  - stage_id: 0  # Thinker stage with TP=2
-    runtime:
-      devices: "0,1"
-    engine_args:
-      tensor_parallel_size: 2
-      gpu_memory_utilization: 0.6  # 48GB per GPU
-
-  - stage_id: 1  # Talker stage
-    runtime:
-      devices: "1"
-    engine_args:
-      gpu_memory_utilization: 0.3  # 24GB on GPU 1
-
-  - stage_id: 2  # Code2Wav stage
-    runtime:
-      devices: "0"
-    engine_args:
-      gpu_memory_utilization: 0.1  # 8GB on GPU 0
-```
-**Note:** In this configuration, stages 0 and 2 share GPU 0, but they run at different times in the pipeline, so their memory usage doesn't overlap.
-
-## Troubleshooting
-
-### Error: "Free memory is less than desired GPU memory utilization"
-
-This means the GPU doesn't have enough free memory when the stage starts.
-
-**Solutions:**
-1. Free up memory by closing other processes
-2. Reduce `gpu_memory_utilization` for this stage
-3. Use a GPU with more memory
-4. Move the stage to a different GPU
-
-### Error: OOM during inference
-
-The stage initialized but ran out of memory during processing.
-
-**Solutions:**
-1. Reduce `max_num_batched_tokens`
-2. Reduce `max_batch_size` in runtime config
-3. Lower `gpu_memory_utilization` slightly
-4. Enable quantization if supported
-
-### Memory Not Fully Utilized
-
-If you see low memory usage, you can:
-1. Increase `gpu_memory_utilization` to allow larger KV cache
-2. Increase `max_num_batched_tokens` for better batching
-3. Check if other stages are limiting throughput
-
-## Useful formula for Memory Calculation
-
-### KV Cache Memory
-
-The KV cache size depends on:
-- Number of sequences in batch
-- Sequence length (prompt + generation)
-- Model hidden size
-- Number of attention heads
-- Number of layers
-
-approximate Formula:
-```
-kv_cache_memory ≈ batch_size × seq_len × hidden_size × num_layers × 2 × dtype_size
-```
-2 for k & v
-
-### Model Weight Memory
-
-```
-model_memory ≈ num_parameters × dtype_size
-```
-
-For example:
-- 7B parameters in FP16: ~14GB
-- 7B parameters in FP32: ~28GB
-- 7B parameters in INT8: ~7GB
-
-### Activation Memory
-
-Activation memory is typically smaller but varies with:
-- Batch size
-- Sequence length
-- Model architecture
-
-It's usually 10-30% of model weight memory during inference.

docs/configuration/stage_configs.md

@@ -1,287 +0,0 @@
-# Stage configs for vLLM-Omni
-
-In vLLM-Omni, the target model is separated into multiple stages, which are processed by different LLMEngines, DiffusionEngines or other types of engines. Depending on different types of stages, such as Autoregressive (AR) stage or Diffusion transformer (DiT) stage, each can choose corresponding schedulers, model workers to load with the Engines in a plug-in fashion.
-
-!!! note
-    Default stage config YAMLs (for example, `vllm_omni/model_executor/stage_configs/qwen2_5_omni.yaml` and `vllm_omni/model_executor/stage_configs/qwen3_omni_moe.yaml`) are bundled and loaded automatically when `stage_configs_path` is not provided. They have been verified to work on 1xH100 for Qwen2.5-Omni and 2xH100 for Qwen3-Omni.
-
-Therefore, as a core part of vLLM-Omni, the stage configs for a model have several main functions:
-
-- Claim partition of stages and their corresponding class implementation in `model_executor/models`.
-- The disaggregated configuration for each stage and the communication topology among them.
-- Engine arguments for each engine within the stage.
-- Input and output dependencies for each stage.
-- Default input parameters.
-
-If users want to modify some part of it. The custom stage_configs file can be input as input argument in both online and offline. Just like examples below:
-
-For offline (Assume necessary dependencies have ben imported):
-```python
-model_name = "Qwen/Qwen2.5-Omni-7B"
-omni_llm = OmniLLM(model=model_name, stage_configs_path="/path/to/custom_stage_configs.yaml")
-```
-
-For online serving:
-```bash
-vllm serve Qwen/Qwen2.5-Omni-7B --omni --port 8091 --stage-configs-path /path/to/stage_configs_file
-```
-!!! important
-    We are actively iterating on the definition of stage configs, and we welcome all feedbacks from both community users and developers to help us shape the development!
-
-Below is a specific example of stage_configs.yaml in Qwen2.5-omni.
-```python
-# stage config for running qwen2.5-omni with architecture of OmniLLM.
-stage_args:
-  - stage_id: 0 # mark the unique id for each stage
-    runtime: # The disaggregated configuration
-      process: true  # Run this stage in a separate process
-      devices: "0" # Visible devices for this stage (CUDA_VISIBLE_DEVICES/torch.cuda.set_device)
-      max_batch_size: 1 # the batch_size for offline inference
-    engine_args: # Engine arguments for a certain engine
-      model_stage: thinker
-      model_arch: Qwen2_5OmniForConditionalGeneration # The model implementation registered in model_executor/models/registry.py
-      worker_type: ar # The specific worker used
-      scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler # The specific scehduler used
-      gpu_memory_utilization: 0.8 # The gpu memory allocation for the stage within a single chip
-      enforce_eager: true  # Now we only support eager mode
-      trust_remote_code: true # Needed by huggingface config parsing
-      engine_output_type: latent  # It claims that the stage will input latent hiddenstates besides token ids
-      enable_prefix_caching: false # For request with hiddenstates output, the prefix caching is not supported now
-    is_comprehension: true # If the stage is a text or multimodal comprehension module. If it is, the AsyncOmni will use its tokenizer as default
-    final_output: true # If the stage has output as part of final outputs. If it is false, which means that the stage only works as a intermediate role.
-    final_output_type: text # What is the final output type. It can be text and audio now.
-    default_sampling_params: # sampling parameters for the stage. Their meaning aligns with vLLM.
-      temperature: 0.0
-      top_p: 1.0
-      top_k: -1
-      max_tokens: 2048
-      seed: 42
-      detokenize: True
-      repetition_penalty: 1.1
-  - stage_id: 1
-    runtime:
-      process: true
-      devices: "1"
-      max_batch_size: 3
-    engine_args:
-      model_stage: talker
-      model_arch: Qwen2_5OmniForConditionalGeneration
-      worker_type: ar
-      scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
-      gpu_memory_utilization: 0.8
-      enforce_eager: true
-      trust_remote_code: true
-      enable_prefix_caching: false
-      engine_output_type: latent
-    engine_input_source: [0]
-    custom_process_input_func: vllm_omni.model_executor.stage_input_processors.qwen2_5_omni.thinker2talker
-    default_sampling_params:
-      temperature: 0.9
-      top_p: 0.8
-      top_k: 40
-      max_tokens: 2048
-      seed: 42
-      detokenize: True
-      repetition_penalty: 1.05
-      stop_token_ids: [8294]
-  - stage_id: 2
-    runtime:
-      process: true
-      devices: "0"            # Example: use a different GPU than the previous stage; use "0" if single GPU
-      max_batch_size: 1
-    engine_args:
-      model_stage: code2wav
-      model_arch: Qwen2_5OmniForConditionalGeneration
-      worker_type: generation
-      scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler
-      gpu_memory_utilization: 0.15
-      enforce_eager: true
-      trust_remote_code: true
-      enable_prefix_caching: false
-      engine_output_type: audio
-    engine_input_source: [1]
-    final_output: true
-    final_output_type: audio
-    default_sampling_params:
-      temperature: 0.0
-      top_p: 1.0
-      top_k: -1
-      max_tokens: 2048
-      seed: 42
-      detokenize: True
-      repetition_penalty: 1.1
-
-# Top-level runtime config (concise): default windows and stage edges
-runtime:
-  enabled: true
-  defaults:
-    window_size: -1             # Simplified: trigger downstream only after full upstream completion
-    max_inflight: 1             # Simplified: process serially within each stage
-  edges:
-    - from: 0                   # thinker → talker: trigger only after receiving full input (-1)
-      to: 1
-      window_size: -1
-    - from: 1                   # talker → code2wav: trigger only after receiving full input (-1)
-      to: 2
-      window_size: -1
-
-```
-
-## Stage Configuration Arguments
-
-Each stage in the `stage_args` list contains the following configuration options:
-
-### `stage_id`
-
-A unique identifier for each stage in the multi-stage pipeline. Stages are numbered sequentially starting from 0, and this ID is used to reference stages in inter-stage dependencies (e.g., `engine_input_source`).
-
-### `runtime`
-
-Configuration for disaggregated execution of the stage, controlling how the stage is deployed and executed.
-
-#### `runtime.process`
-
-Whether to run this stage in a separate process. When set to `true`, the stage will be executed in an isolated process, enabling better resource isolation and parallel execution across different stages. This is essential for multi-GPU deployments where different stages run on different devices.
-
-Default: `true`
-
-#### `runtime.devices`
-
-Visible devices for this stage, specified as a string. This controls which GPU devices are available to the stage process, similar to setting `CUDA_VISIBLE_DEVICES` or using `torch.cuda.set_device()`. For example, `"0"` uses GPU 0, `"1"` uses GPU 1, and `"0,1"` makes both GPUs 0 and 1 visible.
-
-Default: `"0"`
-
-#### `runtime.max_batch_size`
-
-The maximum batch size for offline inference in this stage. This limits how many sequences can be processed together in a single batch during offline inference operations.
-
-Default: `1`
-
-### `engine_args`
-
-Engine arguments for configuring the LLM engine, diffusion engine, or other engine types used by this stage.
-
-#### `engine_args.model_stage`
-
-The name identifier for this model stage within the multi-stage architecture. This is used internally to distinguish different stages of the same model (e.g., "thinker", "talker", "code2wav" in Qwen2.5-Omni).
-
-#### `engine_args.model_arch`
-
-The model architecture class name that is registered in `model_executor/models/registry.py`. This specifies which model implementation to use for this stage. The class must be registered in the model registry for vLLM-Omni to locate and instantiate it.
-
-#### `engine_args.worker_cls`
-
-The specific worker class to use for this stage. This determines how the model computations are executed. Examples include `vllm_omni.worker.gpu_ar_worker.GPUARWorker` for autoregressive stages and `vllm_omni.worker.gpu_generation_worker.GPUGenerationWorker` for diffusion-based stages.
-
-#### `engine_args.scheduler_cls`
-
-The scheduler class to use for this stage. The scheduler manages request queuing, batching, and execution order. Examples include `vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler` for standard stages and `vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler` for diffusion stages.
-
-#### `engine_args.gpu_memory_utilization`
-
-The fraction of GPU memory to allocate for this stage within a single GPU chip. This is a value between 0.0 and 1.0, where 0.8 means 80% of the GPU memory will be used by this stage. This allows fine-grained control over memory allocation when multiple stages share the same GPU or when reserving memory for other operations.
-
-Default: `0.8`
-
-!!! tip "Memory Configuration Guide"
-    For detailed information on how to calculate memory requirements and properly configure `gpu_memory_utilization`, see the [GPU Memory Calculation and Configuration Guide](./gpu_memory_utilization.md).
-
-#### `engine_args.enforce_eager`
-
-Whether to enforce eager execution mode. When set to `true`, the engine will run in eager mode without using CUDA graphs or other compilation optimizations. Currently, vLLM-Omni only supports eager mode.
-
-Default: `true`
-
-#### `engine_args.trust_remote_code`
-
-Whether to trust remote code when loading models from Hugging Face. This is required for models that use custom code in their configuration files. Set to `true` when loading models that require custom model implementations.
-
-Default: `true`
-
-#### `engine_args.engine_output_type`
-
-Specifies the type of output produced by this stage's engine. This determines what kind of data flows to downstream stages. Possible values include `latent` (hidden states), `text` (tokenized text), and `audio` (audio waveforms). When set to `latent`, the stage outputs latent hidden states in addition to token IDs, which are consumed by downstream stages.
-
-Default: `latent`
-
-#### `engine_args.enable_prefix_caching`
-
-Whether to enable prefix caching for this stage. Prefix caching can improve performance by caching KV cache for common prompt prefixes. However, for requests that output hidden states (when `engine_output_type` is `latent`), prefix caching is not currently supported and should be set to `false`.
-
-Default: `false`
-
-### `is_comprehension`
-
-Whether this stage is a text or multimodal comprehension module. When set to `true`, the stage acts as a comprehension module that processes input text or multimodal content. If this is the first comprehension stage, `AsyncOmni` will use its tokenizer as the default tokenizer for the entire pipeline.
-
-Default: `true`
-
-### `final_output`
-
-Whether this stage produces output that is part of the final outputs returned to the user. When set to `false`, the stage only works as an intermediate stage, processing data that flows to downstream stages but not contributing directly to the final response.
-
-Default: `true`
-
-### `final_output_type`
-
-The type of final output produced by this stage. This specifies what format the output will be in when returned to the user. Currently supported values are `text` (for text generation) and `audio` (for audio generation).
-
-Default: `text`
-
-### `default_sampling_params`
-
-Default sampling parameters for this stage. These parameters control the generation behavior and align with vLLM's sampling parameter semantics. These defaults are used when no explicit sampling parameters are provided in the request.
-
-#### `default_sampling_params.temperature`
-
-Sampling temperature for controlling randomness. Lower values (e.g., 0.0) make the output more deterministic and focused, while higher values increase randomness.
-
-Default: `0.0`
-
-#### `default_sampling_params.top_p`
-
-Nucleus sampling parameter. Only tokens with cumulative probability mass up to `top_p` are considered. This helps filter out low-probability tokens while maintaining diversity.
-
-Default: `1.0`
-
-#### `default_sampling_params.top_k`
-
-Top-k sampling parameter. Only the top `k` most likely tokens are considered. Set to `-1` to disable top-k filtering and consider all tokens.
-
-Default: `-1`
-
-#### `default_sampling_params.max_tokens`
-
-Maximum number of tokens to generate in this stage. This limits the length of the output sequence.
-
-Default: `2048`
-
-#### `default_sampling_params.seed`
-
-Random seed for reproducible generation. When set, the random number generator will be initialized with this seed to ensure consistent outputs across runs.
-
-Default: `42`
-
-#### `default_sampling_params.detokenize`
-
-Whether to detokenize the output tokens into text. When set to `true`, token IDs are converted back to readable text strings.
-
-Default: `True`
-
-#### `default_sampling_params.repetition_penalty`
-
-Penalty applied to tokens that have already appeared in the generated sequence. Values greater than 1.0 discourage repetition, while values less than 1.0 encourage it. A value of 1.0 applies no penalty.
-
-Default: `1.1`
-
-### `tts_args` (TTS stages only)
-
-Configuration for Text-to-Speech specific parameters. This section is only applicable to TTS model stages (e.g., `qwen3_tts`).
-
-#### `tts_args.max_instructions_length`
-
-Maximum character length for voice style/emotion instructions. Instructions exceeding this limit will be rejected with a validation error.
-
-Default: `500`
-
-This value can be overridden at runtime using the `--tts-max-instructions-length` CLI parameter when starting the server.

docs/configuration/stage_configs/qwen2_5_omni.yaml

@@ -1,94 +0,0 @@
-# stage config for running qwen2.5-omni with architecture of OmniLLM.
-stage_args:
-  - stage_id: 0
-    runtime:
-      process: true            # Run this stage in a separate process
-      devices: "0"            # Visible devices for this stage (CUDA_VISIBLE_DEVICES/torch.cuda.set_device)
-      max_batch_size: 1
-    engine_args:
-      model_stage: thinker
-      model_arch: Qwen2_5OmniForConditionalGeneration
-      worker_type: ar
-      scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
-      gpu_memory_utilization: 0.8
-      enforce_eager: true  # Now we only support eager mode
-      trust_remote_code: true
-      engine_output_type: latent
-      enable_prefix_caching: false
-    is_comprehension: true
-    final_output: true
-    final_output_type: text
-    default_sampling_params:
-      temperature: 0.0
-      top_p: 1.0
-      top_k: -1
-      max_tokens: 2048
-      seed: 42
-      detokenize: True
-      repetition_penalty: 1.1
-  - stage_id: 1
-    runtime:
-      process: true
-      devices: "1"
-      max_batch_size: 1
-    engine_args:
-      model_stage: talker
-      model_arch: Qwen2_5OmniForConditionalGeneration
-      worker_type: ar
-      scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
-      gpu_memory_utilization: 0.8
-      enforce_eager: true
-      trust_remote_code: true
-      enable_prefix_caching: false
-      engine_output_type: latent
-    engine_input_source: [0]
-    custom_process_input_func: vllm_omni.model_executor.stage_input_processors.qwen2_5_omni.thinker2talker
-    default_sampling_params:
-      temperature: 0.9
-      top_p: 0.8
-      top_k: 40
-      max_tokens: 2048
-      seed: 42
-      detokenize: True
-      repetition_penalty: 1.05
-      stop_token_ids: [8294]
-  - stage_id: 2
-    runtime:
-      process: true
-      devices: "0"            # Example: use a different GPU than the previous stage; use "0" if single GPU
-      max_batch_size: 1
-    engine_args:
-      model_stage: code2wav
-      model_arch: Qwen2_5OmniForConditionalGeneration
-      worker_type: generation
-      scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler
-      gpu_memory_utilization: 0.15
-      enforce_eager: true
-      trust_remote_code: true
-      enable_prefix_caching: false
-      engine_output_type: audio
-    engine_input_source: [1]
-    final_output: true
-    final_output_type: audio
-    default_sampling_params:
-      temperature: 0.0
-      top_p: 1.0
-      top_k: -1
-      max_tokens: 2048
-      seed: 42
-      detokenize: True
-      repetition_penalty: 1.1
-
-# Top-level runtime config (concise): default windows and stage edges
-runtime:
-  enabled: true
-  defaults:
-    window_size: -1             # Simplified: trigger downstream only after full upstream completion
-    max_inflight: 1             # Simplified: process serially within each stage
-  edges:
-    - from: 0                   # thinker → talker: trigger only after receiving full input (-1)
-      to: 1
-      window_size: -1
-    - from: 1                   # talker → code2wav: trigger only after receiving full input (-1)
-      to: 2
-      window_size: -1

docs/contributing/DOCS_GUIDE.md

@@ -1,139 +0,0 @@
-# Documentation Build Guide
-
-This directory contains the source files for the vLLM-Omni documentation.
-
-## Building Documentation Locally
-
-### Prerequisites
-
-Install documentation dependencies:
-
-```bash
-uv pip install -e ".[docs]"
-```
-
-### Build and Serve Documentation
-
-From the project root:
-
-```bash
-# Serve documentation locally (auto-reload on changes)
-# This starts a local web server at http://127.0.0.1:8000
-mkdocs serve
-
-# Build static site (generates HTML files in site/ directory)
-mkdocs build
-```
-
-When using `mkdocs serve`, the documentation will be automatically available at `http://127.0.0.1:8000`. The server will automatically reload when you make changes to the documentation files.
-
-## Auto-generating API Documentation
-
-The documentation automatically extracts docstrings from the code using mkdocstrings. To ensure your code is documented:
-
-1. Add docstrings to all public classes, functions, and methods
-2. Use Google or NumPy style docstrings (both are supported)
-3. Rebuild the documentation to see changes
-
-Example docstring:
-
-```python
-class Omni:
-    """Main entry point for vLLM-Omni inference.
-
-    This class provides a high-level interface for running multi-modal
-    inference with non-autoregressive models.
-
-    Args:
-        model: Model name or path
-        stage_configs: Optional stage configurations
-        **kwargs: Additional arguments passed to the engine
-
-    Example:
-        >>> llm = Omni(model="Qwen/Qwen2.5-Omni")
-        >>> outputs = llm.generate(prompts="Hello")
-    """
-```
-
-## Documentation Structure
-
-```
-docs/
-├── index.md              # Main documentation page
-├── getting_started/      # Getting started guides
-├── architecture/        # Architecture documentation
-├── api/                 # API reference (auto-generated from code)
-├── examples/            # Code examples
-└── stylesheets/         # Custom CSS
-```
-
-## Publishing Documentation
-
-### GitHub Pages (Recommended)
-
-The documentation is automatically deployed to GitHub Pages using GitHub Actions.
-
-1. **Enable GitHub Pages**:
-   - Go to repository `Settings` → `Pages`
-   - Set `Source` to `GitHub Actions`
-   - Save settings
-
-2. **Push changes**:
-   ```bash
-   git push origin main
-   ```
-
-3. **Documentation will be available at**:
-   - `https://vllm-omni.readthedocs.io`
-
-The GitHub Actions workflow (`.github/workflows/docs.yml`) will automatically:
-- Build the documentation when you push to `main` branch
-- Deploy it to GitHub Pages
-- Update the documentation whenever you make changes
-
-
-### Read the Docs (Alternative)
-
-You can also use Read the Docs for hosting:
-
-1. Sign up at https://readthedocs.org/
-2. Import the `vllm-project/vllm-omni` repository
-3. Read the Docs will automatically build using `.readthedocs.yml`
-4. Documentation will be available at: `https://vllm-omni.readthedocs.io/`
-
-## Configuration
-
-The documentation configuration is in `mkdocs.yml` at the project root.
-
-## Tips
-
-- **API Documentation**: API docs are automatically generated using `mkdocs-api-autonav` and `mkdocstrings`
-  - No need to manually create API pages - they're generated automatically
-  - Use `[module.name.ClassName][]` syntax for cross-references in Summary pages
-- **Code Snippets**: Use `--8<-- "path/to/file.py"` for including code snippets
-- **Markdown**: Use Markdown for all documentation (no need for RST)
-- **Material Theme**: Use Material theme features like:
-  - Admonitions: `!!! note`, `!!! warning`, etc.
-  - Code blocks with syntax highlighting
-  - Tabs for organizing content
-  - Math formulas using `pymdownx.arithmatex`
-
-## Troubleshooting
-
-### Documentation not updating
-
-- Make sure you've saved all files
-- If using `mkdocs serve`, it should auto-reload
-- Check for syntax errors in `mkdocs.yml`
-
-### API links not working
-
-- Ensure class names match exactly (case-sensitive)
-- Check that the module is imported correctly
-- Run `mkdocs build --strict` to check for errors
-
-### Build errors
-
-- Check Python version (requires 3.9+)
-- Ensure all dependencies are installed: `pip install -e ".[docs]"`
-- Check `mkdocs.yml` syntax with `mkdocs build --strict`

docs/contributing/README.md

@@ -1,155 +0,0 @@
-# Contributing to vLLM-Omni
-
-Thank you for your interest in contributing to vLLM-Omni! This document provides guidelines and instructions for contributing.
-
-!!! note
-    We host weekly developer-facing online meetings to discuss milestones and updates **every Tuesday at 19:30 PDT**. Meeting link as well as the past meeting notes can be found [here](https://tinyurl.com/vllm-omni-meeting).
-
-## Getting Started
-
-vLLM-Omni uses `uv` as the environment manager, to create and manage Python environments. Please follow the documentation to install `uv`. After installing `uv`, you can create a new Python environment using the following commands:
-
-```bash
-uv venv --python 3.12 --seed
-source .venv/bin/activate
-```
-
-### Development Environment for vLLM and vLLM-Omni
-
-vLLM-Omni is quickly evolving, please see the [installation guide](../getting_started/installation/README.md) for details. It's recommended to build from source to provide the latest development environment.
-
-!!! tip
-    vLLM-Omni is compatible with Python versions 3.10 to 3.12. However, we recommend developing with Python 3.12 to minimize the chance of your local environment clashing with our CI environment.
-
-### Adding a new model to vLLM-Omni
-
-Please check [model implementation](model/README.md) for how to add diffusion and omni-modality models to vLLM-Omni.
-
-### Linting
-
-vLLM-Omni uses `pre-commit` to lint and format the codebase. See [pre-commit documentation](https://pre-commit.com/#usage) if `pre-commit` is new to you. Setting up `pre-commit` is as easy as:
-
-```bash
-uv pip install pre-commit
-pre-commit install
-```
-
-vLLM-Omni's `pre-commit` hooks will now run automatically every time you commit.
-
-!!! tip
-    You can manually run the `pre-commit` hooks using:
-
-    ```bash
-    pre-commit run     # runs on staged files
-    pre-commit run --show-diff-on-failure --color=always --all-files  # runs on all files (short for --all-files)
-    ```
-
-### Documentation
-
-MkDocs is a fast, simple and downright gorgeous static site generator that's geared towards building project documentation. Documentation source files are written in Markdown, and configured with a single YAML configuration file, `mkdocs.yml`.
-
-Get started with:
-
-```bash
-uv pip install -e ".[docs]"
-```
-
-MkDocs comes with a built-in dev-server that lets you preview your documentation as you work on it. From the root of the repository, run:
-
-```bash
-mkdocs serve                           # with API ref (~10 minutes)
-API_AUTONAV_EXCLUDE=vllm_omni mkdocs serve  # API ref off (~15 seconds)
-```
-
-Once you see `Serving on http://127.0.0.1:8000/` in the logs, the live preview is ready! Open <http://127.0.0.1:8000/> in your browser to see it.
-
-For additional features and advanced configurations, refer to the:
-
-- [MkDocs documentation](https://www.mkdocs.org/)
-- [Material for MkDocs documentation](https://squidfunk.github.io/mkdocs-material/) (the MkDocs theme we use)
-
-### Testing
-
-vLLM-Omni uses `pytest` to test the codebase.
-
-```bash
-# Run all tests
-pytest tests/
-
-# Run tests for a single test file with detailed output
-pytest -s -v tests/test_omni_llm.py
-```
-
-!!! warning
-    Currently, not all unit tests pass when run on CPU platforms. If you don't have access to a GPU platform to run unit tests locally, rely on the continuous integration system to run the tests for now.
-
-## Issues
-
-If you encounter a bug or have a feature request, please search existing issues first to see if it has already been reported. If not, please file a new issue, providing as much relevant information as possible.
-
-!!! important
-    If you discover a security vulnerability, please report it by creating a GitHub issue with the `security` label.
-
-## Pull Requests & Code Reviews
-
-Thank you for your contribution to vLLM-Omni! Before submitting the pull request, please ensure the PR meets the following criteria. This helps vLLM-Omni maintain the code quality and improve the efficiency of the review process.
-
-### DCO and Signed-off-by
-
-When contributing changes to this project, you must agree to the [DCO](https://developercertificate.org/). Commits must include a `Signed-off-by:` header which certifies agreement with the terms of the DCO.
-
-Using `-s` with `git commit` will automatically add this header.
-
-!!! tip
-    You can enable automatic sign-off via your IDE:
-
-    - **PyCharm**: Click on the `Show Commit Options` icon to the right of the `Commit and Push...` button in the `Commit` window. It will bring up a `git` window where you can modify the `Author` and enable `Sign-off commit`.
-    - **VSCode**: Open the Settings editor and enable the `Git: Always Sign Off` (`git.alwaysSignOff`) field.
-
-### PR Title and Classification
-
-Only specific types of PRs will be reviewed. The PR title is prefixed appropriately to indicate the type of change. Please use one of the following:
-
-- `[Bugfix]` for bug fixes.
-- `[CI/Build]` for build or continuous integration improvements.
-- `[Doc]` for documentation fixes and improvements.
-- `[Model]` for adding a new model or improving an existing model. Model name should appear in the title.
-- `[Frontend]` For changes on the vLLM-Omni frontend (e.g., OpenAI API server, `OmniLLM` class, etc.)
-- `[Kernel]` for changes affecting CUDA kernels or other compute kernels.
-- `[Core]` for changes in the core vLLM-Omni logic (e.g., `OmniProcessor`, `OmniARScheduler`, etc.)
-- `[Hardware][Vendor]` for hardware-specific changes. Vendor name should appear in the prefix, such as [Ascend] for Ascend NPUs.
-- `[Misc]` for PRs that do not fit the above categories. Please use this sparingly.
-
-!!! note
-    If the PR spans more than one category, please include all relevant prefixes.
-
-### Code Quality
-
-The PR needs to meet the following code quality standards:
-
-- We adhere to Google Python style guide and Google C++ style guide.
-- Pass all linter checks.
-- The code needs to be well-documented to ensure future contributors can easily understand the code.
-- Include sufficient tests to ensure the project stays correct and robust. This includes both unit tests and integration tests.
-- Please add documentation to `docs/` if the PR modifies the user-facing behaviors of vLLM-Omni. It helps vLLM-Omni users understand and utilize the new features or changes.
-
-### Notes for Large Changes
-
-Please keep the changes as concise as possible. For major architectural changes (>500 LOC excluding kernel/data/config/test), we would expect a GitHub issue (RFC) discussing the technical design and justification. Otherwise, we will tag it with `rfc-required` and might not go through the PR.
-
-### What to Expect for the Reviews
-
-The goal of the vLLM-Omni team is to be a _transparent reviewing machine_. We would like to make the review process transparent and efficient and make sure no contributor feels confused or frustrated. However, the vLLM-Omni team is small, so we need to prioritize some PRs over others. Here is what you can expect from the review process:
-
-- After the PR is submitted, the PR will be assigned to a reviewer. Every reviewer will pick up the PRs based on their expertise and availability.
-- After the PR is assigned, the reviewer will provide status updates every 2-3 days. If the PR is not reviewed within 7 days, please feel free to ping the reviewer or the vLLM-Omni team.
-- After the review, the reviewer will put an `action-required` label on the PR if there are changes required. The contributor should address the comments and ping the reviewer to re-review the PR.
-- Please respond to all comments within a reasonable time frame. If a comment isn't clear or you disagree with a suggestion, feel free to ask for clarification or discuss the suggestion.
-
-## Additional Resources
-
-- [Design Documents](../design/index.md) - Architecture and design documentation
-
-## Thank You
-
-Finally, thank you for taking the time to read these guidelines and for your interest in contributing to vLLM-Omni. All of your contributions help make vLLM-Omni a great tool and community for everyone!

docs/contributing/ci/CI_5levels.md

@@ -1,678 +0,0 @@
-# Multi-Level Automated Testing System Documentation
-
-## Document Overview
-
-This testing system aims to build a complete, efficient, and well-structured quality assurance framework for the development, integration, and release of model services. It draws on the concept of the test pyramid from modern software engineering, progressively expanding testing activities from basic code logic verification to complex end-to-end (E2E) functionality, performance, accuracy, and even long-term stability validation.
-
-Through five levels (L1-L5) and common (Common) specifications, the system clarifies the testing objectives, scope, execution frequency, and required resources for different development stages (e.g., each commit, PR merge, daily build, pre-release). This ensures that models meet high standards for functionality, performance, and reliability across various deployment scenarios (online serving and offline inference).
-
-<table>
-  <thead>
-    <tr>
-      <th>Level</th>
-      <th>Scope & Focus</th>
-      <th>Time Cost</th>
-      <th>Test Dir</th>
-      <th>Doc</th>
-      <th>Frequency</th>
-      <th>Hardware</th>
-    </tr>
-  </thead>
-  <tbody>
-    <tr>
-      <td rowspan="2"><strong>Common</strong></td>
-      <td>Contribution Guideline & PR checklist</td>
-      <td>/</td>
-      <td>/</td>
-      <td>.github/PULL_REQUEST_TEMPLATE.md <a href="../tests_style/"> Test Style (PR Checklist)</a></td>
-      <td>/</td>
-      <td>/</td>
-    </tr>
-    <tr>
-      <td>CI Failure Description</td>
-      <td>/</td>
-      <td>/</td>
-      <td><a href="../failures/"> CI Failures</a></td>
-      <td>/</td>
-      <td>/</td>
-    </tr>
-    <tr>
-      <td><strong>L1</strong><br>(Unit & Logic)</td>
-      <td>Unit tests for components like entrypoints, models</td>
-      <td rowspan="2">&lt;15min</td>
-      <td>/tests/{component_name}/test_xxx</td>
-      <td>
-        <a href="#chapter-1-l1-l2-level-testing-unit-testing-and-basic-end-to-end-verification">Chapter 1</a><br>
-        Section 1 L1&amp;L2: Purpose, Test Content, Directory Location, Example
-      </td>
-      <td>PR with ready label (also can run locally)</td>
-      <td>CPU</td>
-    </tr>
-    <tr>
-      <td><strong>L2</strong><br>(E2E across models & GPU-required UT)</td>
-      <td>Online & Offline (basic deployment scenarios):<br>dummy, normal inference function (output format, stream), some instance startup UT</td>
-      <td>
-        /tests/e2e/online_serving/test_{model_name}.py<br>
-        /tests/e2e/offline_inference/test_{model_name}.py
-      </td>
-      <td>
-        <a href="#chapter-1-l1-l2-level-testing-unit-testing-and-basic-end-to-end-verification">Chapter 1</a><br>
-        L1&amp;L2: Purpose, Test Content, Directory Location, Example
-      </td>
-      <td>PR with ready label</td>
-      <td>GPU</td>
-    </tr>
-    <tr>
-      <td><strong>L3</strong><br>(Important Perf & Integration & Accuracy)</td>
-      <td>Online & Offline (multiple deployment scenarios):<br>real model, normal inference function, normal accuracy</td>
-      <td>&lt;30min</td>
-      <td>
-        /tests/e2e/online_serving/test_{model_name}_expansion.py<br>
-        /tests/e2e/offline_inference/test_{model_name}_expansion.py
-      </td>
-      <td>
-        <a href="#chapter-2-l3-level-testing-core-integration-performance-and-accuracy-verification">Chapter 2</a><br>
-        L3: Purpose, Test Content, Directory Location, Example
-      </td>
-      <td>PR Merged (Also run L1&L2 Tests)</td>
-      <td>GPU</td>
-    </tr>
-    <tr>
-      <td><strong>L4</strong><br>(Perf & Integration & Accuracy)</td>
-      <td>Online & Offline: full functional scenarios + performance test + doc test</td>
-      <td>&lt;3 hour</td>
-      <td>
-        <strong>Full Function:</strong><br>
-        /tests/e2e/online_serving/test_{model_name}_expansion.py<br>
-        /tests/e2e/offline_inference/test_{model_name}_expansion.py<br>
-        <strong>Performance:</strong><br>
-        /tests/e2e/perf/nightly.json<br>
-        <strong>Doc Test:</strong><br>
-        tests/example/online_serving/test_{model_name}.py<br>
-        tests/example/offline_inference/test_{model_name}.py
-      </td>
-      <td>
-        <a href="#chapter-3-l4-level-testing-full-functionality-performance-and-documentation-testing">Chapter 3</a><br>
-        L4: Purpose, Test Content, Directory Location, Example
-      </td>
-      <td>Nightly</td>
-      <td>GPU</td>
-    </tr>
-    <tr>
-      <td><strong>L5</strong><br>(Stability & Reliability)</td>
-      <td>Online & Offline: long-term stability test + reliability test</td>
-      <td> Depends on reality </td>
-      <td>
-        <strong>Stability:</strong><br>
-        tests/e2e/stability/weekly.json<br>
-        <strong>Reliability:</strong><br>
-        tests/e2e/reliability/test_{model_name}.py
-      </td>
-      <td>
-        <a href="#chapter-4-l5-level-testing-stability-and-reliability-testing">Chapter 4</a><br>
-        L5: Purpose, Test Content, Directory Location, Example
-      </td>
-      <td>Weekly / Days before Release</td>
-      <td>GPU</td>
-    </tr>
-  </tbody>
-</table>
-
-
----
-<details>
-<summary> The folder structure for tests file based on the 5 levels design</summary>
-Legend: `✅` = test exists, `⬜` = suggested to add.
-```
-vllm_omni/                                    tests/
-├── config/                             →     ├── config/
-│   ├── model.py                              │   └── test_model.py                    ⬜
-│   └── lora.py                               │   └── test_lora.py                      ⬜
-│
-├── core/                               →     ├── core/
-│   └── sched/                                 │   └── sched/
-│       ├── omni_ar_scheduler.py               │       ├── test_omni_ar_scheduler.py    ⬜
-│       ├── omni_generation_scheduler.py       │       ├── test_omni_generation_scheduler.py  ⬜
-│       └── output.py                          │       └── test_output.py               ✅ currently in entrypoints/test_omni_new_request_data.py (tests output.OmniNewRequestData)
-│
-├── diffusion/                          →     ├── diffusion/
-│   ├── diffusion_engine.py                    │   ├── test_diffusion_engine.py          ⬜
-│   ├── attention/                             │   ├── attention/
-│   │   ├── layer.py                            │   │   ├── test_attention_sp.py         ✅
-│   │   └── backends/                           │   │   └── test_flash_attn.py           ✅
-│   ├── distributed/                           │   ├── distributed/
-│   │   └── ...                                 │   │   ├── test_comm.py                 ✅
-│   │                                            │   │   ├── test_cfg_parallel.py        ✅
-│   │                                            │   │   └── test_sp_plan_hooks.py       ✅
-│   ├── lora/                                   │   ├── lora/
-│   │   └── ...                                 │   │   ├── test_base_linear.py          ✅
-│   │                                            │   │   └── test_lora_manager.py        ✅
-│   ├── models/                                 │   ├── models/
-│   │   ├── qwen_image/                         │   │   ├── qwen_image/                 (e2e coverage)
-│   │   ├── z_image/                            │   │   └── z_image/
-│   │   └── ...                                 │   │       └── test_zimage_tp_constraints.py  ✅
-│   └── worker/                                 │   └── worker/
-│       ├── diffusion_worker.py                 │       └── test_diffusion_worker.py   ✅ file at diffusion/test_diffusion_worker.py
-│       └── diffusion_model_runner.py            │
-│
-├── distributed/                         →     ├── distributed/
-│   └── omni_connectors/                         │   └── omni_connectors/
-│       ├── adapter.py                           │       ├── test_adapter_and_flow.py   ✅
-│       ├── kv_transfer_manager.py               │       ├── test_basic_connectors.py   ✅
-│       ├── connectors/                           │       ├── test_kv_flow.py             ✅
-│       └── utils/                               │       └── test_omni_connector_configs.py  ✅
-│
-├── engine/                             →     ├── engine/
-│   ├── input_processor.py                      │   ├── test_input_processor.py         ⬜  (no processor.py in source)
-│   ├── output_processor.py                     │   └── test_output_processor.py         ⬜
-│   └── arg_utils.py                            │   └── test_arg_utils.py               ⬜
-│
-├── entrypoints/                        →     ├── entrypoints/
-│   ├── omni.py                                 │   ├── test_omni.py                    ⬜  (E2E covered by e2e/offline, e2e/online)
-│   ├── omni_llm.py                             │   ├── test_omni_llm.py                ✅
-│   ├── omni_stage.py                            │   ├── test_omni_stage.py              ⬜  (partial in test_omni_stage_diffusion_config.py)
-│   ├── omni_diffusion.py                       │   ├── test_omni_diffusion.py          ✅
-│   ├── async_omni.py                            │   ├── test_async_omni.py              ✅ actually in e2e/online_serving/test_async_omni.py
-│   ├── async_omni_diffusion.py                 │   ├── test_async_omni_diffusion_config.py  ✅
-│   ├── stage_utils.py                          │   ├── test_stage_utils.py            ✅
-│   ├── cli/                                     │   ├── cli/                           (benchmarks/test_serve_cli.py covers CLI serve)
-│   │   └── ...                                  │   │   └── test_*.py                  ⬜
-│   └── openai/                                  │   └── openai_api/                    # maps to entrypoints/openai/
-│       ├── api_server.py                        │       ├── test_api_server.py         ⬜  (e2e indirect coverage)
-│       ├── serving_chat.py                       │       ├── test_serving_chat_sampling_params.py  ✅
-│       ├── serving_speech.py                     │       ├── test_serving_speech.py     ✅
-│       └── image_api_utils.py                   │       └── test_image_server.py      ✅
-│
-├── inputs/                             →     ├── inputs/
-│   ├── data.py                                 │   ├── test_data.py                   ⬜
-│   ├── parse.py                                │   ├── test_parse.py                 ⬜
-│   └── preprocess.py                            │   └── test_preprocess.py            ✅ currently in entrypoints/test_omni_input_preprocessor.py
-│
-├── model_executor/                     →     ├── model_executor/
-│   ├── layers/                                  │   ├── layers/
-│   │   └── mrope.py                             │   │   └── test_mrope.py              ⬜
-│   ├── model_loader/                            │   ├── model_loader/
-│   │   └── weight_utils.py                      │   │   └── test_weight_utils.py      ⬜
-│   ├── models/                                  │   ├── models/
-│   │   ├── qwen2_5_omni/                         │   │   ├── qwen2_5_omni/
-│   │   │   ├── qwen2_5_omni_thinker.py           │   │   │   ├── test_audio_length.py  ✅
-│   │   │   ├── qwen2_5_omni_talker.py            │   │   │   ├── test_qwen2_5_omni_thinker.py  ⬜
-│   │   │   └── qwen2_5_omni_token2wav.py         │   │   │   ├── test_qwen2_5_omni_talker.py  ⬜
-│   │   └── qwen3_omni/                          │   │   │   └── test_qwen2_5_omni_token2wav.py  ⬜
-│   │       └── ...                               │   │   └── qwen3_omni/
-│   ├── stage_configs/                           │   │       └── test_*.py              ⬜
-│   │   └── *.yaml                               │   └── stage_configs/                 (used by e2e, test_*.py can be added)  ⬜
-│   └── stage_input_processors/                  │   └── stage_input_processors/
-│       └── ...                                  │       └── test_*.py                 ⬜
-│
-├── sample/                             →     ├── sample/
-│   └── __init__.py                             │   └── test_*.py                      ⬜
-│
-├── utils/                              →     ├── utils/
-│   └── __init__.py                             │   └── test_*.py                       ⬜  (no platform_utils.py currently)
-│
-├── worker/                             →     ├── worker/
-│   ├── gpu_ar_model_runner.py                  │   ├── test_gpu_ar_model_runner.py    ⬜
-│   ├── gpu_ar_worker.py                        │   ├── test_gpu_ar_worker.py           ⬜
-│   ├── gpu_generation_model_runner.py          │   ├── test_gpu_generation_model_runner.py  ✅
-│   ├── gpu_generation_worker.py                │   ├── test_gpu_generation_worker.py  ⬜
-│   ├── gpu_model_runner.py                     │   ├── test_omni_gpu_model_runner.py   ✅
-│   └── mixins.py                               │   └── (npu under platforms/npu/worker/)  # not worker/npu/
-│
-├── platforms/                          →     (no tests/platforms/, e2e and stage_configs provide indirect coverage)
-│   ├── cuda/
-│   ├── npu/worker/                             # NPU worker here, not vllm_omni/worker/npu/
-│   ├── rocm/
-│   └── xpu/worker/
-│
-├── outputs.py                          →     test_outputs.py                         ✅ (at tests root)
-├── (logger, patch, request, version)    →     (no corresponding unit test)
-│
-└── e2e (tests side only)               →     ├── e2e/
-                                               ├── online_serving/                     ✅ non-empty
-                                               │   ├── test_async_omni.py
-                                               │   ├── test_qwen3_omni.py
-                                               │   ├── test_qwen3_omni_expansion.py
-                                               │   ├── test_image_gen_edit.py
-                                               │   ├── test_images_generations_lora.py
-                                               │   └── stage_configs/
-                                               └── offline_inference/                  ✅
-                                                   ├── test_qwen2_5_omni.py
-                                                   ├── test_qwen3_omni.py
-                                                   ├── test_bagel_text2img.py
-                                                   ├── test_t2i_model.py
-                                                   ├── test_t2v_model.py
-                                                   ├── test_ovis_image.py
-                                                   ├── test_zimage_tensor_parallel.py
-                                                   ├── test_cache_dit.py
-                                                   ├── test_teacache.py
-                                                   ├── test_stable_audio_model.py
-                                                   ├── test_diffusion_cpu_offload.py
-                                                   ├── test_diffusion_layerwise_offload.py
-                                                   ├── test_diffusion_lora.py
-                                                   ├── test_sequence_parallel.py
-                                                   └── stage_configs/
-                                                       ├── qwen2_5_omni_ci.yaml
-                                                       ├── qwen3_omni_ci.yaml
-                                                       ├── bagel_*.yaml
-                                                       └── npu/, rocm/, etc.
-```
-
-
-</details>
-
-
-## Common Specifications
-
-Before entering specific testing levels, the project establishes two common specifications aimed at standardizing the development process and quickly locating issues.
-
-1.  ***PR Checklist ([Tests Style](../ci/tests_style.md))***: This template defines the self-check items that must be completed before submitting a code review (Pull Request). It ensures that each code change meets basic requirements such as code style, dependency updates, and documentation synchronization before entering the automated testing pipeline, serving as the first manual line of defense for quality assurance.
-2.  ***CI Failure Explanation ([CI Failures](../ci/failures.md))***: This document archives and explains common failure patterns in the Continuous Integration (CI) pipeline, error log interpretation, and preliminary troubleshooting steps. It helps developers and testers quickly diagnose the causes of automated test failures, improving problem-solving efficiency.
-
-## Chapter 1: L1 & L2 Level Testing - Unit Testing and Basic End-to-End Verification
-
-### 1.1 Testing Purpose
-
-L1 and L2 level testing form the foundation of the quality assurance system. L1 level testing focuses on verifying the internal logic correctness of code units (e.g., functions, classes), ensuring each independent component behaves as designed.
-
-L2 level testing builds upon L1 by introducing GPU resources and verifying that the end-to-end (E2E) process of the model in basic deployment scenarios is smooth. For example, it uses dummy models to confirm that core interfaces like the inference pipeline, output format, and streaming response work properly. The common goal of these two levels is to provide developers with rapid feedback, discovering and fixing issues early in the development cycle  .
-
-
-
-### 1.2 Testing Content and Scope
-
--   ***L1 (Unit & Logic Testing)***:
--   -   ***Scope***: Tests internal functions and methods of core components such as `entrypoints`, `models`.
-    -   ***Focus***: Branch coverage, exception handling, algorithm logic correctness. Does not involve external dependencies or the complete service stack.
-    -   ***Time Cost***: Execution time is controlled within ***15 minutes*** to ensure fast feedback.
--   ***L2 (Basic End-to-End Testing)***:
--   -   ***Scope***: Covers two basic deployment scenarios: `online` (serving) and `offline` (inference).
-    -   ***Focus***: Uses `dummy` models or lightweight real models to verify that the entire chain from request input to result output works normally, including output data structure, streaming (stream) support, etc. Also includes some unit tests that require launching independent service instances.
-    -   ***Characteristic***: Requires ***GPU*** resources to perform model computations.
-
-### 1.3 Test Directory and Execution Files
-
-A clear directory structure is key to managing test cases efficiently.
-
--   ***L1 Test Directory***: `/tests/{component_name}/test_xxx.py`
--   -   Here, `{component_name}` corresponds to modules in the source code, such as `distributed`, `entrypoints`, etc., and `test_xxx.py` is the specific test file.
--   ***L2 Test Directory***:
--   -   Online Serving: `/tests/e2e/online_serving/test_{model_name}.py`
-    -   Offline Inference: `/tests/e2e/offline_inference/test_{model_name}.py`
-
-### 1.4 Execution Method and Example
-
--   ***Trigger Timing***: **`PR with ready label`**. That is, when a developer adds a "ready for review" or similar label to a PR on platforms like GitHub, L1 and L2 tests are automatically triggered.
--   ***Execution Environment***: L1 uses ***CPU*** environment; L2 requires ***GPU*** environment.
--   ***Script Example***:
-
-<details>
-<summary> L1 Test Examples</summary>
-
-Examples from `tests/model_executor/models/qwen2_5_omni/test_audio_length.py`
-```python
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-
-import pytest
-
-pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
-
-def test_resolve_max_mel_frames_default():
-    from vllm_omni.model_executor.models.qwen2_5_omni.audio_length import resolve_max_mel_frames
-
-    assert resolve_max_mel_frames(None, default=30000) == 30000
-    assert resolve_max_mel_frames(None, default=6000) == 6000
-
-
-def test_resolve_max_mel_frames_explicit():
-    from vllm_omni.model_executor.models.qwen2_5_omni.audio_length import resolve_max_mel_frames
-
-    # Explicit argument always wins over default
-    assert resolve_max_mel_frames(123, default=30000) == 123
-    assert resolve_max_mel_frames(6000, default=30000) == 6000
-    assert resolve_max_mel_frames(0, default=30000) == 0
-
-
-@pytest.mark.parametrize("repeats", [2, 4])
-@pytest.mark.parametrize("code_len", [0, 1, 32768])
-@pytest.mark.parametrize("max_mel_frames", [None, -1, 0, 1, 6000, 30000])
-def test_cap_and_align_mel_length_no_mismatch(repeats, code_len, max_mel_frames):
-    """Guard that any max_mel_frames yields a mel length aligned to repeats, and
-    consistent with the truncated code length (prevents concat mismatch).
-    """
-    from vllm_omni.model_executor.models.qwen2_5_omni.audio_length import cap_and_align_mel_length
-
-    target_code_len, target_mel_len = cap_and_align_mel_length(
-        code_len=code_len,
-        repeats=repeats,
-        max_mel_frames=max_mel_frames,
-    )
-
-    assert isinstance(target_code_len, int)
-    assert isinstance(target_mel_len, int)
-
-    if code_len == 0:
-        assert target_code_len == 0
-        assert target_mel_len == 0
-        return
-
-    assert target_code_len >= 1
-    assert target_mel_len >= repeats
-    assert target_mel_len % repeats == 0
-    assert target_mel_len == target_code_len * repeats
-    assert target_code_len <= code_len
-
-    if max_mel_frames is not None and int(max_mel_frames) > 0 and int(max_mel_frames) >= repeats:
-        assert target_mel_len <= int(max_mel_frames)
-```
-</details>
-
-<details>
-<summary> L2 Test Examples</summary>
-You can refer to Test Examples in Chapter 2 to see example test cases that incorporate both L2 and L3 testing logic.
-</details>
-
--   -   ***Run Command***:
-
-    `pytest -s -v /tests/e2e/online_serving/test_{model_name}.py`
-    `pytest -s -v -m 'core_model and cpu' --run-level=core_model`
-
-## Chapter 2: L3 Level Testing - Core Integration, Performance, and Accuracy Verification
-
-### 2.1 Testing Purpose
-
-L3 level testing executes after code is merged into the main branch. Its core purpose is to verify the integration effect, key performance indicators, and output accuracy of ***real models*** in ***multiple deployment scenarios***
-
-. It acts as the "quality gatekeeper" for the main branch, ensuring that no merge breaks the core capabilities of the model service. Testing needs to provide clear conclusions within a relatively short time (<30min), balancing test depth with feedback speed.
-
-
-
-### 2.2 Testing Content and Scope
-
--   ***Deployment Scenarios***: Covers richer `online` and `offline` deployment configurations, which may include different hardware configurations, batch sizes, concurrency levels, etc.
--   ***Core Verification***:
--   1.  ***Inference Functionality***: Ensures real models can perform forward computation normally and return results.
-    2.  ***Accuracy Compliance***: Verifies that the model's evaluation metrics (e.g., accuracy) meet the expected baseline, preventing code changes from introducing accuracy issues.
-    3.  ***Important Performance***: Verifies whether performance (e.g., P99 latency, throughput) in core scenarios meets preset thresholds.
-
-### 2.3 Test Directory and Execution Files
-
--   ***Functional Testing***:
--   -   Online Serving: `/tests/e2e/online_serving/test_{model_name}_expansion.py`
-    -   Offline Inference: `/tests/e2e/offline_inference/test_{model_name}_expansion.py`
-    -   (Note: `_expansion.py` likely means it contains more comprehensive scenario cases compared to L2 tests).
-
-### 2.4 Execution Method and Example
-
--   ***Trigger Timing***: **`PR Merged`**. Automatically triggered after code review is approved and merged into the main branch.
--   ***Execution Environment***: ***GPU*** servers.
--   ***Script Example***:
-
-???+ example "Test Examples"
-
-    **2.4.1 Mark Declaration Section**
-
-    ```python
-    @pytest.mark.advanced_model
-    @pytest.mark.core_model
-    @pytest.mark.parametrize("omni_server", test_params, indirect=True)
-    ```
-
-    **Explanation**:
-
-    @pytest.mark.advanced_model: Marks the test as L3 or L4 level, indicating that this test case performs deep validation, using real models for performance, integration, and accuracy testing. This forms a "basic-advanced" correspondence with the core_model mark at the L2 level.
-
-    @pytest.mark.core_model: Marks the test as L1 or L2 level, indicating that this test case validates the basic functionality of the core model. It uses mock weights and only checks if the relevant interface functions correctly.
-
-    @pytest.mark.parametrize: A parameterization decorator that allows abstracting test data into parameters, enabling reuse of the same test logic across different data configurations. indirect=True indicates that parameters will be passed to the fixture for processing.
-
-    **Notes**: If you believe the test case only needs to execute basic run logic at the PR-level CI, you can mark it only with @pytest.mark.core_model. If you believe it only needs to execute deep validation run logic at the merge or nightly level, you can mark it only with @pytest.mark.advanced_model. If you believe the test case needs to accommodate both basic run and deep validation test logic, you should mark it with both @pytest.mark.core_model and @pytest.mark.advanced_model.
-
-    **2.4.2 Test Function Definition and Documentation**
-
-    ```python
-    def test_mix_to_text_audio_001(omni_server, openai_client) -> None:
-        """
-        Test multi-modal input processing and text/audio output generation via OpenAI API.
-        Deploy Setting: default yaml
-        Input Modal: text + audio + video + image
-        Output Modal: text + audio
-        Input Setting: stream=True
-        Datasets: single request
-        """
-    ```
-
-    **Explanation**:
-
-    **Function Naming Convention**: Uses the test_ prefix, describes the test scenario mix_to_text_audio, and the number 001 indicates the first test case for this scenario.
-
-    **Parameter Explanation**:
-
-    omni_server: Omni server instance obtained via fixture, containing model information and configuration.
-
-    openai_client: Unified OpenAI client processing instance, encapsulating request sending and response validation logic.
-
-    Docstring: Describes the test purpose, deployment settings, input/output modalities, streaming settings, and dataset type in detail, providing clear context for test maintenance.
-
-    **2.4.3 Multimodal Data Preparation**
-
-    ```python
-    video_data_url = f"data:video/mp4;base64,{generate_synthetic_video(224, 224, 300)['base64']}"
-    image_data_url = f"data:image/jpeg;base64,{generate_synthetic_image(224, 224)['base64']}"
-    audio_data_url = f"data:audio/wav;base64,{generate_synthetic_audio(5, 1)['base64']}"
-    ```
-
-    **Explanation**:
-
-    **Data Generation Functions**: Use the generate_synthetic_* series of functions to generate synthetic test data, avoiding reliance on external resources and ensuring test reproducibility and stability.
-
-    **Parameter Explanation**:
-
-    Video: width, height, duration_frames
-
-    Image: width, height
-
-    Audio: duration_seconds, channels
-
-    **2.4.4 Request Configuration and Keyword Validation**
-
-    ```python
-    request_config = {
-        "model": omni_server.model,
-        "messages": messages,
-        "stream": True,
-        "key_words": {
-            "audio": ["water", "cricket"],
-            "video": ["sphere", "globe", "circle", "round"],
-            "image": ["square", "quadrate"],
-            "text": ["beijing"]
-        },
-    }
-    ```
-
-    **Explanation**:
-
-    **Model Specification**: Uses omni_server.model to ensure the test aligns with the model configured on the server.
-
-    **Keyword Validation Mechanism**: This is an innovative design of the template to address the specific needs of multimodal testing:
-
-    Audio Keywords: Validate whether the generated text's description of audio content contains expected elements (e.g., "water" for water sounds, "cricket" for cricket sounds). If you provide multiple keywords, the validation is considered successful if at least one keyword is present.
-
-    **Video Keywords**: Validate whether the generated text's description of video content contains expected elements. If you provide multiple keywords, the validation is considered successful if at least one keyword is present.
-
-    Image Keywords: Validate whether the generated text's description of image content contains expected elements. If you provide multiple keywords, the validation is considered successful if at least one keyword is present.
-
-    Text Keywords: Validate whether the generated text contains expected elements. If you provide multiple keywords, the validation is considered successful if at least one keyword is present.
-
-    **2.4.5 Request Execution**
-
-    ```python
-    openai_client.send_request(request_config, request_num=1)
-    ```
-
-    **Explanation**:
-
-    **Unified Client**: Uses the OpenAIClientHandler instance to send requests. This client encapsulates error handling, retry mechanisms, and response validation logic.
-
-    **Single Request**: The comment clearly states this is a single-request completion test. For concurrent testing, it can be extended to multiple requests using request_num = n.
-
-    **Implicit Validation**: The send_request method internally includes validation logic dynamically selected based on the --run-level parameter: core_model performs basic validation, while advanced_model performs deep validation.
-
--   ***Run Command***: `pytest -s -v /tests/e2e/online_serving/test_{model_name}.py -m advanced_model --run-level=advanced_model`
-
-## Chapter 3: L4 Level Testing - Full Functionality, Performance, and Documentation Testing
-
-### 3.1 Testing Purpose
-
-L4 level testing is a comprehensive quality audit before a version release. It expands upon L3, executing ***full*** functional scenarios, conducting systematic ***performance stress tests***, and simultaneously verifying the correctness of accompanying ***example documentation***. Its purpose is to perform deep validation of the system during off-peak nighttime hours, providing quality trend reports for daytime development and data support for release decisions.
-
-
-
-### 3.2 Testing Content and Scope
-
--   ***Full Functionality Testing***: Executes all test cases defined in `test_{model_name}_expansion.py`, covering all implemented features, positive flows, boundary conditions, and exception handling.
--   ***Performance Testing***: Uses the `/tests/e2e/perf/nightly.json` configuration file to drive performance testing tools for stress, load, and endurance tests, collecting metrics like throughput, response time, and resource utilization.
--   ***Documentation Testing***: Verifies whether the example code provided to users is runnable and its results match the description.
-
-### 3.3 Test Directory and Execution Files
-
--   ***Functional Testing***: Same directories as L3.
--   ***Performance Test Configuration***: `/tests/e2e/perf/nightly.json`
--   ***Documentation Example Tests***:
--   -   `tests/example/online_serving/test_{model_name}.py`
-    -   `tests/example/offline_inference/test_{model_name}.py`
-
-### 3.4 Execution Method and Example
-
--   ***Trigger Timing***: **`Nightly`**, automatically executed every night.
--   ***Execution Environment***: ***GPU*** server clusters to meet the resource demands of performance testing.
--   ***Script Example***:
-<details>
-<summary> Test Examples</summary>
-When you want to add L4-level performance test cases, you can refer to the following format for case addition in tests/perf/tests/test.json:
-
-```JSON
-{
-    "test_name": "test_qwen3_omni",
-    "server_params": {
-        "model": "Qwen/Qwen3-Omni-30B-A3B-Instruct",
-        "stage_config_name": "qwen3_omni.yaml"
-    },
-    "benchmark_params": [
-        {
-            "dataset_name": "random",
-            "num_prompts": [10, 20],
-            "request_rate": [0.5, 1],
-            "random_input_len": 2500,
-            "random_output_len": 900,
-            "ignore_eos": true,
-            "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
-            "baseline": {
-                "mean_ttft_ms": 100000,
-                "mean_audio_ttfp_ms": 100000,
-                "mean_audio_rtf": 100000
-            }
-        }
-    ]
-}
-```
-
-#### Parameter Explanation
-
-***Overview***
-
-| Field            | Required | Description                                                     |
-| ---------------- | -------- | --------------------------------------------------------------- |
-| test_name        | Yes      | Unique identifier for the test case                             |
-| server_params    | Yes      | Server-side configuration parameters                            |
-| benchmark_params | Yes      | Benchmark running parameters (supports multiple configurations) |
-
-#### server_params Configuration
-
-##### Basic Parameters
-
-| Parameter         | Required | Example                            | Description                   |
-| ----------------- | -------- | ---------------------------------- | ----------------------------- |
-| model             | Yes      | "Qwen/Qwen3-Omni-30B-A3B-Instruct" | Model name or path            |
-| stage_config_name | Yes      | "qwen3_omni.yaml"                  | Stage configuration file name |
-
-##### Dynamic Configuration (update/delete)
-
-Supports incremental modifications based on the basic configuration:
-
-| Operation | Description                          |
-| --------- | ------------------------------------ |
-| update    | Update or add configuration items    |
-| delete    | Delete specified configuration items |
-
-***Example***:
-```
-"update": {
-    "async_chunk": true,  // Enable asynchronous chunk processing
-    "stage_args": {
-        "0": {
-            "engine_args.custom_process_next_stage_input_func": "vllm_omni.model_executor.stage_input_processors.qwen3_omni.thinker2talker_async_chunk"
-        }
-    }
-},
-"delete": {
-    "stage_args": {
-        "2": ["custom_process_input_func"]  // Delete this configuration for stage 2
-    }
-}
-```
-#### benchmark_params Configuration
-
-You can add any benchmark running parameters you need here. For all optional parameters, refer to the [benchmark documentation](https://github.com/vllm-project/vllm-omni/blob/main/docs/cli/bench/serve.md). General modifications are as follows:
-
-1.  Change the ---xxx-xx-xx running parameters to xxx_xx_xx format and fill them as keys in the JSON file.
-2.  For boolean variables in the running parameters, modify them to forms such as ignore_eos: true/false and fill them into the JSON file.
-3.  Add the baseline parameter to specify the required validation values, ensuring the validation metric names match those in the result.json generated by the benchmark.
-4.  The qps and concurrency modes are mutually exclusive. For detailed explanations, see the table below:
-
-| Parameter       | Type        | Required | Example/Values  | Description                                                                                                                                                                                                                                                          |
-| --------------- | ----------- | -------- | --------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| num_prompts     | int / array | Yes      | 10,[10, 20, 30] | Number of requests. Supports single values or arrays. If a single value is used, it will be automatically expanded to match the number of qps or max_concurrency, e.g., [10,10,10]. If an array is used, its length must match the number of qps or max_concurrency. |
-| request_rate    | int / array | No       | 1, [1, 2, 3]    | Queries per second. Supports single values or arrays. If a single value is used, it will be automatically expanded to match the number of num_prompts, e.g., [1,1,1]. If an array is used, its length must match the number of num_prompts.                          |
-| max_concurrency | int / array | No       | 1, [1, 2, 3]    | Queries per second. Supports single values or arrays. If a single value is used, it will be automatically expanded to match the number of num_prompts, e.g., [1,1,1]. If an array is used, its length must match the number of num_prompts.                          |
-</details>
-
--   -   ***Run Command***: (Specific commands would depend on the performance testing tool and configuration defined in `nightly.json`).
-
-## Chapter 4: L5 Level Testing - Stability and Reliability Testing
-
-### 4.1 Testing Purpose
-
-L5 level testing focuses on the performance of model services under ***long-running*** and ***abnormal fault*** scenarios. It aims to uncover deep-seated issues that only manifest under sustained pressure or extreme conditions, such as memory leaks, resource contention, gradual performance degradation, and lack of fault tolerance mechanisms. This is the final, yet crucial, line of defense for ensuring service high availability and production environment robustness.
-
-
-
-### 4.2 Testing Content and Scope
-
--   ***Long-term Stability (Stability) Testing***: Uses the `tests/e2e/stability/weekly.json` configuration to run the service under moderate load for an extended period (e.g., over 12 hours), monitoring whether metrics like memory/VRAM usage, response time, and throughput degrade over time, and whether the service process remains stable.
--   ***Reliability Testing***: Uses `tests/e2e/reliability/test_{model_name}.py` to actively simulate various fault and abnormal scenarios, such as: dependent service interruption, abnormal input data, network flicker, hardware resource preemption, etc., to verify the system's fault tolerance, self-healing, and graceful degradation capabilities.
-
-### 4.3 Test Directory and Execution Files
-
--   ***Stability Test Configuration***: `tests/e2e/stability/weekly.json`
--   ***Reliability Test Suite***: `tests/e2e/reliability/test_{model_name}.py`
-
-### 4.4 Execution Method and Example
-
--   ***Trigger Timing***: **`Weekly`** (weekly) or **`Days before Release`** (several days before a major release). Due to long execution times, the frequency is lower.
--   ***Execution Environment***: ***GPU*** servers, requiring a stable and exclusive testing environment.
--   ***Script Example***:
-<details>
-<summary> Test Examples</summary>
-```python
-# WIP
-```
-</details>
-
--   -   ***Stability***: (Execution would be driven by the configuration in `weekly.json`).
-    -   ***Reliability***: `pytest -s -v tests/e2e/reliability/test_{model_name}.py`
-
-## Summary
-
-This multi-level testing system achieves continuous, progressive validation of model service quality by tightly integrating testing activities with the development workflow (commit, review, merge, release). From rapid unit testing to comprehensive end-to-end testing, and further to in-depth performance, stability, and reliability verification, each level has clear objectives, collectively building a robust quality protection net. By following this system, teams can deliver high-quality, highly reliable model services more efficiently.

docs/contributing/ci/failures.md

@@ -1,10 +0,0 @@
-# CI Failures
-
-What should I do when a CI job fails on my PR, but I don't think my PR caused the failure?
-
-
-## Common Case 1:
-
-🚨 Error: The command was interrupted by a signal: signal: terminated
-
-Reason: The test is terminated due to exceed the time limits. Sometimes rebuild the test will pass.

docs/contributing/ci/tests_markers.md

@@ -1,157 +0,0 @@
-# Markers for Tests
-
-By adding markers before test functions, tests can later be executed uniformly by simply declaring the corresponding marker type.
-
-## Current Markers
-Defined in `pyproject.toml`:
-
-| Marker             | Description                                               |
-| ------------------ | --------------------------------------------------------- |
-| `core_model`       | L1&L2 tests (run in each PR)                              |
-| `advanced_model`   | L3&L4 level tests (run in each merge or nightly)          |
-| `diffusion`        | Diffusion model tests                                     |
-| `omni`             | Omni model tests                                          |
-| `cache`            | Cache backend tests                                       |
-| `parallel`         | Parallelism/distributed tests                             |
-| `cpu`              | Tests that run on CPU                                     |
-| `gpu`              | Tests that run on GPU *                                   |
-| `cuda`             | Tests that run on CUDA *                                  |
-| `rocm`             | Tests that run on AMD/ROCm *                              |
-| `npu`              | Tests that run on NPU/Ascend *                            |
-| `H100`             | Tests that require H100 GPU  *                            |
-| `L4`               | Tests that require L4 GPU *                               |
-| `MI325`            | Tests that require MI325 GPU (AMD/ROCm) *                 |
-| `A2`               | Tests that require A2 NPU *                               |
-| `A3`               | Tests that require A3 NPU *                               |
-| `distributed_cuda` | Tests that require multi cards on CUDA platform *         |
-| `distributed_rocm` | Tests that require multi cards on ROCm platform  *        |
-| `distributed_npu`  | Tests that require multi cards on NPU platform  *         |
-| `skipif_cuda`      | Skip if the num of CUDA cards is less than the required * |
-| `skipif_rocm`      | Skip if the num of ROCm cards is less than the required * |
-| `skipif_npu`       | Skip if the num of NPU cards is less than the required *  |
-| `slow`             | Slow tests (may skip in quick CI)                         |
-| `benchmark`        | Benchmark tests                                           |
-
-\* Means those markers are auto-added, and they will be added by the `@hardware_test` decorator.
-
-### Example usage for markers
-
-```python
-from tests.utils import hardware_test
-
-@pytest.mark.core_model
-@pytest.mark.omni
-@hardware_test(
-   res={"cuda": "L4", "rocm": "MI325", "npu": "A2"},
-   num_cards=2,
-)
-@pytest.mark.parametrize("omni_server", test_params, indirect=True)
-def test_video_to_audio()
-    ...
-```
-### Decorator: `@hardware_test`
-
-This decorator is intended to make hardware-aware, cross-platform test authoring easier and more robust for CI/CD environments. The `hardware_test` decorator in `vllm-omni/tests/utils.py` performs the following actions:
-
-1. **Applies platform and resource markers**  
-   Adds the appropriate pytest markers for each specified hardware platform (e.g., `cuda`, `rocm`, `npu`) and resource type (e.g., `L4`, `H100`, `MI325`, `A2`, `A3`).
-   ```
-   @pytest.mark.cuda
-   @pytest.mark.L4
-   ```
-2. **Handles multi-card (distributed) scenarios**  
-   For tests requiring multiple cards, it automatically adds distributed markers such as `distributed_cuda`, `distributed_rocm`, or `distributed_npu`.
-   ```
-   @pytest.mark.distributed_cuda(num_cards=num_cards)
-   ```
-3. **Supports flexible card requirements**  
-   Accepts `num_cards` as either a single integer for all platforms or as a dictionary with per-platform values. If not specified, defaults to 1 card per platform.
-
-4. **Integrates resource validation**  
-   On CUDA, adds a skip marker (`skipif_cuda`) if the system does not have the required number of devices.
-   Support for `skipif_rocm` and `skipif_npu` will be implemented later.
-
-
-5. **Works with pytest filtering**  
-   Allows tests to be filtered and selected at runtime using standard pytest marker expressions (e.g., `-m "distributed_cuda and L4"`).
-
-#### Example usage for decorator
-- Single call for multiple platforms:
-    ```python
-    @hardware_test(
-        res={"cuda": "L4", "rocm": "MI325", "npu": "A2"},
-        num_cards={"cuda": 2, "rocm": 2, "npu": 2},
-    )
-    ```
-    or
-    ```python
-    @hardware_test(
-        res={"cuda": "L4", "rocm": "MI325", "npu": "A2"},
-        num_cards=2,
-    )
-    ```
-- `res` must be a dict; supported resources: CUDA (L4/H100), ROCm (MI325), NPU (A2/A3)
-- `num_cards` can be int (all platforms) or dict (per platform); defaults to 1 when missing
-- Distributed markers (`distributed_cuda`, `distributed_rocm`, `distributed_npu`) are auto-added for multi-card cases
-- Filtering examples:
-    - CUDA only: `pytest -m "distributed_cuda and L4"`
-    - ROCm only: `pytest -m "distributed_rocm and MI325"`
-    - NPU only: `pytest -m "distributed_npu"`
-
-## Add Support for a New Platform
-
-If you want to add support for a new platform (e.g., "tpu" for a new accelerator), follow these steps:
-
-1. **Extend the marker list in your pytest config** so that platform/resource markers are defined:
-   ```toml
-   # In pyproject.toml or pytest.ini
-   [tool.pytest.ini_options]
-   markers = [
-       # ... existing markers ...
-       "tpu: Tests that require TPU device",
-       "TPU_V3: Tests that require TPU v3 hardware",
-       "distributed_tpu: Tests that require multiple TPU devices",
-   ]
-   ```
-2. **Implement a marker construction function for your platform** in `vllm-omni/tests/utils.py`:
-   ```python
-   # In vllm-omni/tests/utils.py
-
-   def tpu_marks(*, res: str, num_cards: int):
-       test_platform = pytest.mark.tpu
-       if res == "TPU_V3":
-           test_resource = pytest.mark.TPU_V3
-       else:
-           raise ValueError(
-               f"Invalid TPU resource type: {res}. Supported: TPU_V3")
-
-       if num_cards == 1:
-           return [test_platform, test_resource]
-       else:
-           test_distributed = pytest.mark.distributed_tpu(num_cards=num_cards)
-           # Optionally: add skipif_tpu when implemented
-           return [test_platform, test_resource, test_distributed]
-   ```
-3. **Update `hardware_test` to recognize your new platform**:
-    In the relevant place (see the `hardware_test` implementation), add:
-    ```python
-    if platform == "tpu":
-        marks = tpu_marks(res=resource, num_cards=cards)
-    ```
-4. **(Recommended) Add a test using your new markers**:
-   ```python
-   @hardware_test(
-       res={"tpu": "TPU_V3"},
-       num_cards=2,
-   )
-   def test_my_tpu_feature():
-       ...
-   ```
-
-**Summary**:  
-- Add pytest markers for your new platform/resources  
-- Implement a marker function (`xxx_marks`)  
-- Plug into `hardware_test`  
-- You're done: tests decorated with `@hardware_test` using your platform now automatically get the correct markers, distribution, and isolation!
-
-See code in `vllm-omni/tests/utils.py` for existing examples (`cuda_marks`, `rocm_marks`, `npu_marks`).

docs/contributing/ci/tests_style.md

@@ -1,467 +0,0 @@
-# Test File Structure and Style Guide
-
-To ensure project maintainability and sustainable development, we encourage contributors to submit test code (unit tests, system tests, or end-to-end tests) alongside their code changes. This document outlines the guidelines for organizing and naming test files.
-
-## Checklist before submitting your test files
-
-1. The file is saved in an appropriate place and the file name is clear.
-2. The coding style follows the requirements outlined below.
-3. All test functions have appropriate pytest markers.
-4. For tests that need run in CI, please ensure it labeled as ``@pytest.mark.core_model` the test is configured under the `./buildkite/` folder.
-
-
-## Test Types
-
-For more details about our [Five Levels Tests design](../ci/CI_5levels.md).
-
-### Unit Tests and System Tests
-For unit tests and system tests, we strongly recommend placing test files in the same directory structure as the source code being tested, using the naming convention `test_*.py`.
-
-### End-to-End (E2E) Tests for Models
-End-to-end tests verify the complete functionality of a system or component. For our project, the E2E tests for different omni models are organized into two subdirectories:
-
-- **`tests/e2e/offline_inference/`**: Tests for offline inference modes (e.g., Qwen3Omni offline inference)
-
-- **`tests/e2e/online_serving/`**: Tests for online serving scenarios (e.g., API server tests)
-
-**Example:** The test file for `vllm_omni/entrypoints/omni_llm.py` should be located at `tests/entrypoints/test_omni_llm.py`.
-
-## Test Directory Structure
-
-The ideal directory structure mirrors the source code organization. Legend: `✅` = test exists, `⬜` = suggested to add.
-
-```
-vllm_omni/                                    tests/
-├── config/                             →     ├── config/
-│   ├── model.py                              │   └── test_model.py                    ⬜
-│   └── lora.py                               │   └── test_lora.py                      ⬜
-│
-├── core/                               →     ├── core/
-│   └── sched/                                 │   └── sched/
-│       ├── omni_ar_scheduler.py               │       ├── test_omni_ar_scheduler.py    ⬜
-│       ├── omni_generation_scheduler.py       │       ├── test_omni_generation_scheduler.py  ⬜
-│       └── output.py                          │       └── test_output.py               ✅ currently in entrypoints/test_omni_new_request_data.py (tests output.OmniNewRequestData)
-│
-├── diffusion/                          →     ├── diffusion/
-│   ├── diffusion_engine.py                    │   ├── test_diffusion_engine.py          ⬜
-│   ├── attention/                             │   ├── attention/
-│   │   ├── layer.py                            │   │   ├── test_attention_sp.py         ✅
-│   │   └── backends/                           │   │   └── test_flash_attn.py           ✅
-│   ├── distributed/                           │   ├── distributed/
-│   │   └── ...                                 │   │   ├── test_comm.py                 ✅
-│   │                                            │   │   ├── test_cfg_parallel.py        ✅
-│   │                                            │   │   └── test_sp_plan_hooks.py       ✅
-│   ├── lora/                                   │   ├── lora/
-│   │   └── ...                                 │   │   ├── test_base_linear.py          ✅
-│   │                                            │   │   └── test_lora_manager.py        ✅
-│   ├── models/                                 │   ├── models/
-│   │   ├── qwen_image/                         │   │   ├── qwen_image/                 (e2e coverage)
-│   │   ├── z_image/                            │   │   └── z_image/
-│   │   └── ...                                 │   │       └── test_zimage_tp_constraints.py  ✅
-│   └── worker/                                 │   └── worker/
-│       ├── diffusion_worker.py                 │       └── test_diffusion_worker.py   ✅ file at diffusion/test_diffusion_worker.py
-│       └── diffusion_model_runner.py            │
-│
-├── distributed/                         →     ├── distributed/
-│   └── omni_connectors/                         │   └── omni_connectors/
-│       ├── adapter.py                           │       ├── test_adapter_and_flow.py   ✅
-│       ├── kv_transfer_manager.py               │       ├── test_basic_connectors.py   ✅
-│       ├── connectors/                           │       ├── test_kv_flow.py             ✅
-│       └── utils/                               │       └── test_omni_connector_configs.py  ✅
-│
-├── engine/                             →     ├── engine/
-│   ├── input_processor.py                      │   ├── test_input_processor.py         ⬜  (no processor.py in source)
-│   ├── output_processor.py                     │   └── test_output_processor.py         ⬜
-│   └── arg_utils.py                            │   └── test_arg_utils.py               ⬜
-│
-├── entrypoints/                        →     ├── entrypoints/
-│   ├── omni.py                                 │   ├── test_omni.py                    ⬜  (E2E covered by e2e/offline, e2e/online)
-│   ├── omni_llm.py                             │   ├── test_omni_llm.py                ✅
-│   ├── omni_stage.py                            │   ├── test_omni_stage.py              ⬜  (partial in test_omni_stage_diffusion_config.py)
-│   ├── omni_diffusion.py                       │   ├── test_omni_diffusion.py          ✅
-│   ├── async_omni.py                            │   ├── test_async_omni.py              ✅ actually in e2e/online_serving/test_async_omni.py
-│   ├── async_omni_diffusion.py                 │   ├── test_async_omni_diffusion_config.py  ✅
-│   ├── stage_utils.py                          │   ├── test_stage_utils.py            ✅
-│   ├── cli/                                     │   ├── cli/                           (benchmarks/test_serve_cli.py covers CLI serve)
-│   │   └── ...                                  │   │   └── test_*.py                  ⬜
-│   └── openai/                                  │   └── openai_api/                    # maps to entrypoints/openai/
-│       ├── api_server.py                        │       ├── test_api_server.py         ⬜  (e2e indirect coverage)
-│       ├── serving_chat.py                       │       ├── test_serving_chat_sampling_params.py  ✅
-│       ├── serving_speech.py                     │       ├── test_serving_speech.py     ✅
-│       └── image_api_utils.py                   │       └── test_image_server.py      ✅
-│
-├── inputs/                             →     ├── inputs/
-│   ├── data.py                                 │   ├── test_data.py                   ⬜
-│   ├── parse.py                                │   ├── test_parse.py                 ⬜
-│   └── preprocess.py                            │   └── test_preprocess.py            ✅ currently in entrypoints/test_omni_input_preprocessor.py
-│
-├── model_executor/                     →     ├── model_executor/
-│   ├── layers/                                  │   ├── layers/
-│   │   └── mrope.py                             │   │   └── test_mrope.py              ⬜
-│   ├── model_loader/                            │   ├── model_loader/
-│   │   └── weight_utils.py                      │   │   └── test_weight_utils.py      ⬜
-│   ├── models/                                  │   ├── models/
-│   │   ├── qwen2_5_omni/                         │   │   ├── qwen2_5_omni/
-│   │   │   ├── qwen2_5_omni_thinker.py           │   │   │   ├── test_audio_length.py  ✅
-│   │   │   ├── qwen2_5_omni_talker.py            │   │   │   ├── test_qwen2_5_omni_thinker.py  ⬜
-│   │   │   └── qwen2_5_omni_token2wav.py         │   │   │   ├── test_qwen2_5_omni_talker.py  ⬜
-│   │   └── qwen3_omni/                          │   │   │   └── test_qwen2_5_omni_token2wav.py  ⬜
-│   │       └── ...                               │   │   └── qwen3_omni/
-│   ├── stage_configs/                           │   │       └── test_*.py              ⬜
-│   │   └── *.yaml                               │   └── stage_configs/                 (used by e2e, test_*.py can be added)  ⬜
-│   └── stage_input_processors/                  │   └── stage_input_processors/
-│       └── ...                                  │       └── test_*.py                 ⬜
-│
-├── sample/                             →     ├── sample/
-│   └── __init__.py                             │   └── test_*.py                      ⬜
-│
-├── utils/                              →     ├── utils/
-│   └── __init__.py                             │   └── test_*.py                       ⬜  (no platform_utils.py currently)
-│
-├── worker/                             →     ├── worker/
-│   ├── gpu_ar_model_runner.py                  │   ├── test_gpu_ar_model_runner.py    ⬜
-│   ├── gpu_ar_worker.py                        │   ├── test_gpu_ar_worker.py           ⬜
-│   ├── gpu_generation_model_runner.py          │   ├── test_gpu_generation_model_runner.py  ✅
-│   ├── gpu_generation_worker.py                │   ├── test_gpu_generation_worker.py  ⬜
-│   ├── gpu_model_runner.py                     │   ├── test_omni_gpu_model_runner.py   ✅
-│   └── mixins.py                               │   └── (npu under platforms/npu/worker/)  # not worker/npu/
-│
-├── platforms/                          →     (no tests/platforms/, e2e and stage_configs provide indirect coverage)
-│   ├── cuda/
-│   ├── npu/worker/                             # NPU worker here, not vllm_omni/worker/npu/
-│   ├── rocm/
-│   └── xpu/worker/
-│
-├── outputs.py                          →     test_outputs.py                         ✅ (at tests root)
-├── (logger, patch, request, version)    →     (no corresponding unit test)
-│
-└── e2e (tests side only)               →     ├── e2e/
-                                               ├── online_serving/                     ✅ non-empty
-                                               │   ├── test_async_omni.py
-                                               │   ├── test_qwen3_omni.py
-                                               │   ├── test_qwen3_omni_expansion.py
-                                               │   ├── test_image_gen_edit.py
-                                               │   ├── test_images_generations_lora.py
-                                               │   └── stage_configs/
-                                               └── offline_inference/                  ✅
-                                                   ├── test_qwen2_5_omni.py
-                                                   ├── test_qwen3_omni.py
-                                                   ├── test_bagel_text2img.py
-                                                   ├── test_t2i_model.py
-                                                   ├── test_t2v_model.py
-                                                   ├── test_ovis_image.py
-                                                   ├── test_zimage_tensor_parallel.py
-                                                   ├── test_cache_dit.py
-                                                   ├── test_teacache.py
-                                                   ├── test_stable_audio_model.py
-                                                   ├── test_diffusion_cpu_offload.py
-                                                   ├── test_diffusion_layerwise_offload.py
-                                                   ├── test_diffusion_lora.py
-                                                   ├── test_sequence_parallel.py
-                                                   └── stage_configs/
-                                                       ├── qwen2_5_omni_ci.yaml
-                                                       ├── qwen3_omni_ci.yaml
-                                                       ├── bagel_*.yaml
-                                                       └── npu/, rocm/, etc.
-```
-
-
-
-### Naming Conventions
-
-- **Unit Tests**: Use `test_<module_name>.py` format. Example: `omni_llm.py` → `test_omni_llm.py`
-
-- **E2E Tests**: Place in `tests/e2e/offline_inference/` or `tests/e2e/online_serving/` with descriptive names. Example: `tests/e2e/offline_inference/test_qwen3_omni.py`, `tests/e2e/offline_inference/test_diffusion_model.py`
-
-- **Expansion Tests**
-
-### Best Practices
-
-1. **Mirror Source Structure**: Test directories should mirror the source code structure
-2. **Test Type Indicators**: Use comments to indicate test types (UT for unit tests, E2E for end-to-end tests)
-3. **Shared Resources**: Place shared test configurations (e.g., CI configs) in appropriate subdirectories
-4. **Consistent Naming**: Follow the `test_*.py` naming convention consistently across all test files
-
-
-## Test codes requirements
-
-### Coding style
-
-1. **File header**: Add SPDX license header to all test files
-2. **Imports**: Pls don't use manual `sys.path` modifications, use standard imports instead.
-3. **Test type differentiation**:
-
-      - Unit tests: Maintain mock style
-      - E2E tests for models: Consider using OmniRunner uniformly, avoid decorators
-
-4. **Documentation**: Add docstrings to all test functions
-5. **Environment variables**: Set uniformly in `conftest.py` or at the top of files
-6. **Type annotations**: Add type annotations to all test function parameters
-7. **Pytest Markers**: Add necessary markers like `@pytest.mark.core_model` and use `@hardware_test` to declare hardware requirements (check detailed in [Markers for Tests](../ci/tests_markers.md)).
-
-### Template
-#### E2E - Online serving
-
-E2E Online tests for Qwen3-Omni model with mix input and audio+text output. Based on `tests/e2e/online_serving/test_qwen3_omni.py`.
-
-```python
-"""
-E2E Online tests for Qwen3-Omni model with mix input and audio+text output.
-"""
-
-import os
-
-os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
-os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
-
-import threading
-from pathlib import Path
-
-import openai
-import pytest
-
-from tests.conftest import (
-    OmniServer,
-    convert_audio_to_text,
-    cosine_similarity_text,
-    dummy_messages_from_mix_data,
-    generate_synthetic_video,
-    merge_base64_and_convert_to_text,
-)
-from vllm_omni.platforms import current_omni_platform
-
-# Edit: model name and stage config path
-models = ["Qwen/Qwen3-Omni-30B-A3B-Instruct"]
-
-#If you use the default configuration file, you can directly use the following address.
-def get_default_config():
-    return str(Path(__file__).parent.parent / "stage_configs" / "qwen3_omni_ci.yaml")
-
-#If you need to modify the configuration file, you can use modify_stage_config.
-def get_chunk_config():
-    path = modify_stage_config(
-        get_default_config(),
-        updates={
-            "async_chunk": True,
-            "stage_args": {
-                0: {
-                    "engine_args.custom_process_next_stage_input_func": "vllm_omni.model_executor.stage_input_processors.qwen3_omni.thinker2talker_async_chunk"
-                },
-                1: {
-                    "engine_args.custom_process_next_stage_input_func": "vllm_omni.model_executor.stage_input_processors.qwen3_omni.talker2code2wav_async_chunk"
-                },
-            },
-        },
-        deletes={"stage_args": {2: ["custom_process_input_func"]}},
-    )
-    return path
-
-stage_configs = [get_default_config(), CHUNK_CONFIG_PATH]
-
-test_params = [(model, stage_config) for model in models for stage_config in stage_configs]
-
-
-#Please use this method to launch the online instance.
-_omni_server_lock = threading.Lock()
-
-@pytest.fixture(scope="module")
-def omni_server(request):
-    """Start vLLM-Omni server as a subprocess. Use module scope for multi-stage init (10-20+ min)."""
-    with _omni_server_lock:
-        model, stage_config_path = request.param
-        with OmniServer(
-            model,
-            ["--stage-configs-path", stage_config_path, "--stage-init-timeout", "120"],
-        ) as server:
-            yield server
-
-
-@pytest.fixture
-def client(omni_server):
-    """OpenAI client for the running vLLM-Omni server."""
-    return openai.OpenAI(
-        base_url=f"http://{omni_server.host}:{omni_server.port}/v1",
-        api_key="EMPTY",
-    )
-
-#Please use function definitions above the test function to define the prompts and other parameters you need.
-def get_system_prompt():
-    return {
-        "role": "system",
-        "content": [
-            {
-                "type": "text",
-                "text": (
-                    "You are Qwen, a virtual human developed by the Qwen Team, "
-                    "Alibaba Group, capable of perceiving auditory and visual inputs, "
-                    "as well as generating text and speech."
-                ),
-            }
-        ],
-    }
-
-...
-
-#Please define test case tags according to the instructions in the marker documentation.
-@pytest.mark.core_model
-@pytest.mark.omni
-@pytest.mark.parametrize("omni_server", test_params, indirect=True)
-def test_mix_to_text_audio_001(client: openai.OpenAI, omni_server, request) -> None:
-    # PLEASE FOLLOW THESE TEMPLATE INSTRUCTIONS:
-    # ============================================================================
-    # TEMPLATE USAGE GUIDE:
-    # 1. Copy this entire function as a starting point for multi-modal tests
-    # 2. Update the test name to reflect your specific test scenario
-    # 3. Modify input/output modalities as needed (see OPTIONS section below)
-    # 4. Adjust assertions based on your expected outcomes
-    # 5. Add custom validation logic for your specific use case
-    # ============================================================================
-
-    #Please list the relevant test points.
-    """
-    Test multi-modal input processing and text/audio output generation via OpenAI API.
-    Deploy Setting: default yaml
-    Input Modal: text + audio + video + image
-    Output Modal: text + audio
-    Input Setting: stream=True
-    Datasets: single request
-    """
-    # SECTION 1: TEST SETUP AND INITIALIZATION
-    # =========================================
-    # INSTRUCTIONS: Initialize test variables and prepare test environment
-    # MODIFY: Add any additional test setup required for your scenario
-    e2e_list = list()
-    # SECTION 2: TEST DATA GENERATION
-    # ================================
-    # INSTRUCTIONS: Generate or load test data for each input modality
-    # MODIFY: Replace synthetic generators with your actual data sources
-    # VIDEO DATA - Generate synthetic video for testing
-    # FORMAT: data:video/mp4;base64,{base64_encoded_video}
-    # PARAMETERS: width, height, duration_frames
-    video_data_url = f"data:video/mp4;base64,{generate_synthetic_video(224, 224, 300)['base64']}"
-    # IMAGE DATA - Generate synthetic image for testing
-    # FORMAT: data:image/jpeg;base64,{base64_encoded_image}
-    # PARAMETERS: width, height
-    image_data_url = f"data:image/jpeg;base64,{generate_synthetic_image(224, 224)['base64']}"
-    # AUDIO DATA - Generate synthetic audio for testing
-    # FORMAT: data:audio/wav;base64,{base64_encoded_audio}
-    # PARAMETERS: duration_seconds, channels
-    audio_data_url = f"data:audio/wav;base64,{generate_synthetic_audio(5, 1)['base64']}"
-
-    # SECTION 3: MESSAGE CONSTRUCTION
-    # ================================
-    # INSTRUCTIONS: Assemble the complete message payload for API request
-    # MODIFY: Add/remove modalities or change prompt structure as needed
-
-    # USAGE: Construct a message containing all input modalities
-    # IMPORTANT: Ensure the message structure matches OpenAI API expectations
-    # CUSTOMIZATION POINTS:
-    #   - system_prompt: Controls the assistant's behavior
-    #   - content_text: The user's text prompt/question
-    #   - *_data_url: URLs for media content (video/image/audio)
-    messages = dummy_messages_from_mix_data(
-        system_prompt=get_system_prompt(),
-        video_data_url=video_data_url,
-        image_data_url=image_data_url,
-        audio_data_url=audio_data_url,
-        content_text=get_prompt("mix"),
-    )
-
-    # SECTION 4: API REQUEST EXECUTION
-    # =================================
-    # INSTRUCTIONS: Make the API call and measure performance
-    # MODIFY: Add timeout, retry logic, or additional parameters
-    start_time = time.perf_counter()
-    chat_completion = client.chat.completions.create(model=omni_server.model, messages=messages, stream=True)
-
-    #Call using your preferred method and obtain the final audio and text outputs.
-    ...
-
-    # SECTION 5: OUTPUT VALIDATION
-    # =============================
-    # INSTRUCTIONS: Verify that outputs meet expected criteria
-    # MODIFY: Adjust validation logic for your specific requirements
-
-    # ASSERTION 1: E2E Validation
-    # PURPOSE: Verify that the E2E latency is less than the baseline.
-    current_e2e = time.perf_counter() - start_time
-    print(f"the request e2e is: {current_e2e}")
-    e2e_list.append(current_e2e)
-
-    print(f"the avg e2e is: {sum(e2e_list) / len(e2e_list)}")
-
-
-
-    # ASSERTION 2: Text Output Validation
-    # PURPOSE: Verify that text output was generated with keyword content
-    assert text_content is not None and len(text_content) >= 2, "No text output is generated"
-    assert any(
-        keyword in text_content.lower() for keyword in ["square", "quadrate", "sphere", "globe", "circle", "round"]
-    ), "The output does not contain any of the keywords."
-
-
-    # ASSERTION 3: Cross-Modal Consistency
-    # PURPOSE: Verify text and audio outputs convey the same information
-    # CUSTOMIZATION: Adjust similarity threshold (0.9) based on accuracy requirements
-    assert audio_data is not None, "No audio output is generated"
-    audio_content = merge_base64_and_convert_to_text(audio_data)
-    print(f"text content is: {text_content}")
-    print(f"audio content is: {audio_content}")
-    similarity = cosine_similarity_text(audio_content.lower(), text_content.lower())
-    print(f"similarity is: {similarity}")
-    assert similarity > 0.9, "The audio content is not same as the text"
-```
-
-
-#### E2E - Offline inference
-```python
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-"""
-Offline E2E smoke test for an omni model (video → audio).
-"""
-
-import os
-from pathlib import Path
-
-import pytest
-from vllm.assets.video import VideoAsset
-
-from tests.utils import hardware_test
-from ..multi_stages.conftest import OmniRunner
-
-# Optional: set process start method for workers
-os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
-
-models = ["{your model name}"] #Edit here to load your model
-stage_configs = [str(Path(__file__).parent / "stage_configs" / {your model yaml})] #Edit here to load your model yaml
-
-# Create parameter combinations for model and stage config
-test_params = [(model, stage_config) for model in models for stage_config in stage_configs]
-
-# function name: test_{input_modality}_to_{output_modality}
-# modality candidate: text, image, audio, video, mixed_modalities
-@pytest.mark.core_model
-@pytest.mark.omni
-@hardware_test(
-    res={"cuda": "L4", "rocm": "MI325", "npu": "A2"},
-    num_cards=2,
-)
-@pytest.mark.parametrize("test_config", test_params)
-def test_video_to_audio(omni_runner: type[OmniRunner], model: str) -> None:
-    """Offline inference: video input, audio output."""
-    model, stage_config_path = test_config
-    with omni_runner(model, seed=42, stage_configs_path=stage_config_path) as runner:
-        # Prepare inputs
-        video = VideoAsset(name="sample", num_frames=4).np_ndarrays
-
-        outputs = runner.generate_multimodal(
-            prompts="Describe this video briefly.",
-            videos=video,
-        )
-
-        # Minimal assertions: got outputs and at least one audio result
-        assert outputs
-        has_audio = any(o.final_output_type == "audio" for o in outputs)
-        assert has_audio
-```

docs/contributing/metrics.md

@@ -1,173 +0,0 @@
-
-# Metrics
-
-You can use these metrics in production to monitor the health and performance of the vLLM-omni system. Typical scenarios include:
-
-- **Performance Monitoring**: Track throughput (e.g., `e2e_avg_tokens_per_s`), latency (e.g., `e2e_total_ms`), and resource utilization to verify that the system meets expected standards.
-
-- **Debugging and Troubleshooting**: Use detailed per-request metrics to diagnose issues, such as high transfer times or unexpected token counts.
-
-## How to Enable and View Metrics
-
-### Start the Service with Metrics Logging
-
-```bash
-vllm serve /workspace/models/Qwen3-Omni-30B-A3B-Instruct --omni --port 8014 --log-stats
-```
-
-### Send a Request
-
-```bash
-python openai_chat_completion_client_for_multimodal_generation.py --query-type use_image
-```
-
-### What You Will See
-
-With `--log-stats` enabled, the server will output detailed metrics logs after each request. Example output:
-
-
-#### Overall Summary
-
-| Field                       | Value        |
-|-----------------------------|--------------|
-| e2e_requests                | 1            |
-| e2e_wall_time_ms            | 41,299.190   |
-| e2e_total_tokens            | 5,202        |
-| e2e_avg_time_per_request_ms | 41,299.190   |
-| e2e_avg_tokens_per_s        | 125.959      |
-| e2e_stage_0_wall_time_ms    | 10,192.289   |
-| e2e_stage_1_wall_time_ms    | 30,541.409   |
-| e2e_stage_2_wall_time_ms    |    207.496   |
-
-#### RequestE2EStats
-
-| Field                   | Value      |
-|-------------------------|------------|
-| e2e_total_ms            | 41,299.133 |
-| e2e_total_tokens        | 5,202      |
-| transfers_total_time_ms | 245.895    |
-| transfers_total_kbytes  | 138,089.939|
-
-#### StageRequestStats
-
-| Field                  | 0      | 1      | 2      |
-|------------------------|--------|--------|--------|
-| audio_generated_frames | 0      | 0      | 525,525|
-| batch_id               | 38     | 274    | 0      |
-| batch_size             | 1      | 1      | 1      |
-| num_tokens_in          | 4,860  | 4,826  | 4,384  |
-| num_tokens_out         | 67     | 275    | 0      |
-| postprocess_time_ms    | 256.158| 0.491  | 0.000  |
-| stage_gen_time_ms      | 9,910.007|30,379.198|160.745|
-
-#### TransferEdgeStats
-
-| Field               | 0->1        | 1->2       |
-|---------------------|-------------|------------|
-| size_kbytes         | 109,277.349 | 28,812.591 |
-| tx_time_ms          | 78.701      | 18.790     |
-| rx_decode_time_ms   | 111.865     | 31.706     |
-| in_flight_time_ms   | 2.015       | 2.819      |
-
-
-These logs include:
-
-- **Overall summary**: total requests, wall time, average tokens/sec, etc.
-
-- **E2E table**: per-request latency and token counts.
-
-- **Stage table**: per-stage batch and timing details.
-
-- **Transfer table**: data transfer and timing for each edge.
-
-You can use these logs to monitor system health, debug performance, and analyze request-level metrics as described above.
-
-
-## Metrics Scope: Offline vs Online Inference
-
-For **offline inference** (batch mode), the summary includes both system-level metrics (aggregated across all requests) and per-request metrics. In this case, `e2e_requests` can be greater than 1, reflecting multiple completed requests in a batch.
-
-For **online inference** (serving mode), the summary is always per-request. `e2e_requests` is always 1, and only request-level metrics are reported for each completion.
-
----
-
-## Parameter Details
-
-### Summary Metrics
-
-| Field                     | Meaning                                                                                       |
-|---------------------------|----------------------------------------------------------------------------------------------|
-| `e2e_requests`            | Number of completed requests.                                                                |
-| `e2e_wall_time_ms`        | Wall-clock time span from run start to last completion, in ms.                               |
-| `e2e_total_tokens`        | Total tokens counted across all completed requests (stage0 input + all stage outputs).       |
-| `e2e_avg_time_per_request_ms` | Average wall time per request: `e2e_wall_time_ms / e2e_requests`.                        |
-| `e2e_avg_tokens_per_s`    | Average token throughput over wall time: `e2e_total_tokens * 1000 / e2e_wall_time_ms`.      |
-| `e2e_stage_{i}_wall_time_ms` | Wall-clock time span for stage i, in ms. Each stage's wall time is reported as a separate field, e.g., `e2e_stage_0_wall_time_ms`, `e2e_stage_1_wall_time_ms`, etc. |
-
----
-
-### E2E Table (per request)
-
-| Field                     | Meaning                                                               |
-|---------------------------|-----------------------------------------------------------------------|
-| `e2e_total_ms`            | End-to-end latency in ms.                                             |
-| `e2e_total_tokens`        | Total tokens for the request (stage0 input + all stage outputs).      |
-| `transfers_total_time_ms` | Sum of transfer edge `total_time_ms` for this request.                |
-| `transfers_total_kbytes`  | Sum of transfer kbytes for this request.                              |
-
-
----
-
-### Stage Table (per stage event / request)
-
-| Field                     | Meaning                                                                                         |
-|---------------------------|-------------------------------------------------------------------------------------------------|
-| `batch_id`                | Batch index.                                                                                    |
-| `batch_size`              | Batch size.                                                                                     |
-| `num_tokens_in`           | Input tokens to the stage.                                                                      |
-| `num_tokens_out`          | Output tokens from the stage.                                                                   |
-| `stage_gen_time_ms`       | Stage compute time in ms, excluding postprocessing time (reported separately as `postprocess_time_ms`). |
-| `image_num`               | Number of images generated (for diffusion/image stages).                                        |
-| `resolution`              | Image resolution (for diffusion/image stages).                                                                  |
-| `postprocess_time_ms` | Diffusion/image: post-processing time in ms.                                                    |
-
----
-
-### Transfer Table (per edge / request)
-
-| Field                | Meaning                                                                   |
-|----------------------|---------------------------------------------------------------------------|
-| `size_kbytes`        | Total kbytes transferred.                                                 |
-| `tx_time_ms`         | Sender transfer time in ms.                                               |
-| `rx_decode_time_ms`  | Receiver decode time in ms.                                               |
-| `in_flight_time_ms`  | In-flight time in ms.                                                     |
-
-
-### Expectation of the Numbers (Verification)
-
-**Formulas:**
-
-- `e2e_total_tokens = Stage0's num_tokens_in + sum(all stages' num_tokens_out)`
-
-- `transfers_total_time_ms = sum(tx_time_ms + rx_decode_time_ms + in_flight_time_ms)` for every edge
-
-**Using the example above:**
-
-**e2e_total_tokens**
-
-- Stage0's `num_tokens_in`: **4,860**
-- Stage0's `num_tokens_out`: **67**
-- Stage1's `num_tokens_out`: **275**
-- Stage2's `num_tokens_out`: **0**
-
-so `e2e_total_tokens = 4,860 + 67 + 275 + 0 = 5,202`, which matches the table value `e2e_total_tokens`.
-
-**transfers_total_time_ms**
-
-For each edge:
-
-- 0->1: tx_time_ms (**78.701**) + rx_decode_time_ms (**111.865**) + in_flight_time_ms (**2.015**) = **192.581**
-
-- 1->2: tx_time_ms (**18.790**) + rx_decode_time_ms (**31.706**) + in_flight_time_ms (**2.819**) = **53.315**
-
-192.581 + 53.315 = **245.896** = transfers_total_time_ms, which matches the calculation (difference is due to rounding)

docs/contributing/model/README.md

@@ -1,15 +0,0 @@
-# Adding a New Model
-
-This section provides comprehensive guidance on how to add a new model to vLLM-Omni.
-
-## Documentation
-
-- **[Adding an Omni-Modality Model](adding_omni_model.md)**: Complete step-by-step guide using Qwen3-Omni as an example.
-
-- **[Adding a Diffusion Model](adding_diffusion_model.md)**: Complete step-by-step guide using Qwen/Qwen-Image-Edit as an example.
-
-
-
-## Quick Start
-
-For a quick reference, see the [Adding a New Multi-Stage Model Guide](adding_omni_model.md) and [Adding a New Diffusion Model Guide](adding_diffusion_model.md).

docs/contributing/model/adding_diffusion_model.md

@@ -1,951 +0,0 @@
-# Adding a Diffusion Model
-
-This guide walks you through adding a new diffusion model to vLLM-Omni. We use **Qwen-Image** as the primary example, with references to other models (LongCat, Flux, Wan2.2) to illustrate different patterns.
-
-
----
-
-## Table of Contents
-
-1. [Overview](#overview)
-2. [Directory Structure](#directory-structure)
-3. [Basic Implementation](#basic-implementation)
-4. [Advanced Features](#advanced-features)
-5. [Troubleshooting](#troubleshooting)
-6. [Pull Request Checklist](#pull-request-checklist)
-7. [Reference Implementations](#reference-implementations)
-8. [Summary](#summary)
-
----
-
-## Overview
-
-vLLM-Omni's diffusion inference follows this architecture:
-
-<p align="center">
-  <picture>
-    <source media="(prefers-color-scheme: dark)" src="https://raw.githubusercontent.com/vllm-project/vllm-omni/refs/heads/main/docs/source/architecture/vllm-omni-diffusion-flow.png">
-    <img alt="Diffusion Flow" src="https://raw.githubusercontent.com/vllm-project/vllm-omni/refs/heads/main/docs/source/architecture/vllm-omni-diffusion-flow.png" width=55%>
-  </picture>
-</p>
-
-**Key Components:**
-
-1. **Request Handling:** User prompts → `OmniDiffusionRequest`
-2. **Diffusion Engine:**  Request →  Preprocessing (Optional) → Pipeline execution -> Post-processing
-3. **Pipeline Execution:** Request → Encode prompt → Diffusion steps → Vae decode
-
-
-## Directory Structure
-
-Organize your model files following this structure:
-
-```
-vllm_omni/
-└── diffusion/
-    ├── registry.py                          # ← Register your model here
-    ├── request.py                           # Request data structures
-    └── models/
-        └── your_model_name/                 # ← Create this directory
-            ├── __init__.py                  # Export pipeline and transformer
-            ├── pipeline_xxx.py              # Pipeline implementation
-            └── xxx_transformer.py           # Transformer implementation
-```
-
-**Naming Conventions:**
-
-- **Model directory:** `your_model_name` (lowercase, underscores),  e.g., `qwen_image`, `flux`, `longcat_image`, `wan2_2`
-- **Pipeline file:** `pipeline_xxx.py` where `xxx` describes the task, e.g., `pipeline_qwen_image.py`, `pipeline_qwen_image_edit.py`
-- **Transformer file:** `xxx_transformer.py` matching transformer class name, e.g.,  `qwen_image_transformer.py`, `flux_transformer.py`
-
----
-
-## Basic Implementation
-
-This section covers the minimal steps to get a model working in vLLM-Omni with basic features (online/offline serving, batch requests).
-
-### Step 1: Adapt Transformer Model
-
-The transformer is the core denoising network. Start by copying the transformer implementation from Diffusers and making these adaptations.
-
-
-#### 1.1: Remove Diffusers Mixins
-
-Diffusers' `Mixin` classes are not needed in vLLM-Omni. Remove them:
-
-```diff
-# Before (Diffusers)
-- from diffusers.models.modeling_utils import ModelMixin
-- from diffusers.models.attention_processor import AttentionModuleMixin
-
-- class YourModelTransformer2DModel(ModelMixin, AttentionModuleMixin):
-+ class YourModelTransformer2DModel(nn.Module):
-    """Your transformer model."""
-```
-
-**Example mixins to remove:**
-
-- `ModelMixin` - Weight loading utilities (vLLM-Omni has its own weight loader)
-- `AttentionModuleMixin` - Attention processors (using vLLM-Omni's Attention layer instead)
-- `ConfigMixin` - Config management (not needed)
-- `PeftAdapterMixin` - Parameter efficient finetune utilities (not needed)
-
-#### 1.2: Replace Attention Implementation
-
-**The most important adaptation:** Replace Diffusers' attention with vLLM-Omni's optimized `Attention` layer.
-
-**Before (Diffusers):**
-```python
-from diffusers.models.attention_processor import dispatch_attention_fn
-
-class YourAttentionBlock(nn.Module):
-    def forward(self, hidden_states, encoder_hidden_states=None, ...):
-        ...
-        hidden_states = dispatch_attention_fn(
-            query, key, value,
-            attn_mask=attention_mask,
-            dropout_p=0.0,
-            is_causal=False,
-            backend=self._attention_backend,
-        )
-```
-
-**After (vLLM-Omni):**
-```python
-from vllm_omni.diffusion.attention.layer import Attention
-from vllm_omni.diffusion.attention.backends.abstract import AttentionMetadata
-
-class YourAttentionBlock(nn.Module):
-    def __init__(self, ...):
-        super().__init__()
-
-        # Initialize vLLM-Omni's Attention layer
-        self.attn = Attention(
-            num_heads=self.num_heads,
-            head_size=self.head_dim,
-            softmax_scale=1.0 / (self.head_dim ** 0.5),
-            causal=False,  # Diffusion models typically use bidirectional attention
-            num_kv_heads=self.num_kv_heads,
-        )
-
-    def forward(self, hidden_states, encoder_hidden_states=None, attention_mask=None, ...):
-        ...
-        # Create attention metadata
-        attn_metadata = AttentionMetadata(attn_mask=attention_mask)
-        hidden_states = self.attn(query, key, value, attn_metadata=attn_metadata)
-
-```
-
-**Key Points:**
-
-- **Attention layer initialization:** Done in `__init__`, not per-forward
-- **Tensor shapes:** vLLM-Omni `Attention` expects QKV to have `[B, seq, num_heads, head_dim]` shape
-- **AttentionMetadata:** Wraps attention mask and other metadata
-
-**Attention backends:** vLLM-Omni automatically selects the attention backend given the environmental variable `DIFFUSION_ATTENTION_BACKEND`. The default attention backend is `FLASH_ATTN` for diffusion models.
-
-#### 1.3: Replace Imports and Utilities
-
-**Logger:**
-```diff
-- from diffusers.utils import logging
-- logger = logging.get_logger(__name__)
-
-+ from vllm.logger import init_logger
-+ logger = init_logger(__name__)
-```
-
-**Custom layers from vLLM and vLLM-Omni (if needed):**
-
-```python
-from vllm.model_executor.layers.layernorm import RMSNorm
-from vllm_omni.diffusion.layers.rope import RotaryEmbedding
-from vllm_omni.diffusion.layers.adalayernorm import AdaLayerNorm
-```
-
-#### 1.4: Remove Training-Only Code
-
-Remove code that's only needed for training:
-
-```diff
-# Remove gradient checkpointing
-- if torch.is_grad_enabled() and self.gradient_checkpointing:
--     hidden_states = torch.utils.checkpoint.checkpoint(
--         self._forward_block, hidden_states, ...
--     )
-- else:
--     hidden_states = self._forward_block(hidden_states, ...)
-+ hidden_states = self._forward_block(hidden_states, ...)
-
-# Remove training-specific attributes
-- self.gradient_checkpointing = False
-
-# Remove dropout (set to 0 or remove)
-- self.dropout = nn.Dropout(dropout_prob)
-+ # Removed dropout for inference
-```
-
-#### 1.5: Add Configuration Support
-
-Add support for vLLM-Omni's `OmniDiffusionConfig`:
-
-```python
-from vllm_omni.diffusion.data import OmniDiffusionConfig
-
-class YourModelTransformer2DModel(nn.Module):
-    def __init__(
-        self,
-        *,
-        od_config: OmniDiffusionConfig | None = None,  # ← Add vLLM-Omni config
-        # ... other model-specific parameters
-        num_layers: int = 28,
-        hidden_size: int = 3072,
-        num_heads: int = 24,
-        **kwargs,
-    ):
-        super().__init__()
-
-        # Store config
-        self.od_config = od_config
-        self.parallel_config = od_config.parallel_config if od_config else None
-
-        # Model architecture
-        self.num_layers = num_layers
-        self.hidden_size = hidden_size
-        # ... initialize layers
-```
-
-### Step 2: Adapt Pipeline
-
-The pipeline orchestrates the full generation process (text encoding, denoising loop, VAE decoding). Adapt it from Diffusers format to vLLM-Omni's interface.
-
-#### 2.1: Remove Diffusers Inheritance
-
-**Remove Diffusers base classes:**
-```diff
-- from diffusers import DiffusionPipeline
-- from diffusers.loaders import LoraLoaderMixin
-
-- class YourModelPipeline(DiffusionPipeline, LoraLoaderMixin):
-+ class YourModelPipeline(nn.Module):
-    """Your model pipeline for vLLM-Omni."""
-```
-
-#### 2.2: Adapt `__init__` Method
-
-**Before (Diffusers):**
-```python
-class YourModelPipeline(DiffusionPipeline):
-    def __init__(
-        self,
-        vae: AutoencoderKL,
-        text_encoder: CLIPTextModel,
-        tokenizer: CLIPTokenizer,
-        transformer: YourTransformer,
-        scheduler: FlowMatchScheduler,
-    ):
-        super().__init__()
-        self.register_modules(
-            vae=vae,
-            text_encoder=text_encoder,
-            tokenizer=tokenizer,
-            transformer=transformer,
-            scheduler=scheduler,
-        )
-```
-
-**After (vLLM-Omni):**
-```python
-import os
-from diffusers import AutoencoderKL
-from diffusers.schedulers import FlowMatchEulerDiscreteScheduler
-from transformers import CLIPTextModel, CLIPTokenizer
-
-from vllm_omni.diffusion.data import OmniDiffusionConfig
-from vllm_omni.diffusion.distributed.utils import get_local_device
-from vllm_omni.diffusion.utils.tf_utils import get_transformer_config_kwargs
-from vllm_omni.diffusion.models.your_model_name.your_model_transformer import (
-    YourModelTransformer2DModel,
-)
-
-
-class YourModelPipeline(nn.Module):
-    def __init__(
-        self,
-        *,
-        od_config: OmniDiffusionConfig,
-        prefix: str = "",
-    ):
-        super().__init__()
-        self.od_config = od_config
-        self.parallel_config = od_config.parallel_config
-        self.device = get_local_device()
-        model = od_config.model
-        local_files_only = os.path.exists(model)
-
-        # Load components from checkpoint
-        self.scheduler = FlowMatchEulerDiscreteScheduler.from_pretrained(
-            model, subfolder="scheduler", local_files_only=local_files_only)
-        self.text_encoder = CLIPTextModel.from_pretrained(
-            model, subfolder="text_encoder", local_files_only=local_files_only).to(self.device)
-        self.tokenizer = CLIPTokenizer.from_pretrained(
-            model, subfolder="tokenizer", local_files_only=local_files_only)
-        self.vae = AutoencoderKL.from_pretrained(
-            model, subfolder="vae", local_files_only=local_files_only).to(self.device)
-
-        # Initialize transformer with vLLM-Omni config
-        transformer_kwargs = get_transformer_config_kwargs(
-            od_config.tf_model_config, YourModelTransformer2DModel)
-        self.transformer = YourModelTransformer2DModel(
-            od_config=od_config, **transformer_kwargs)
-
-        # Store VAE scale factor for latent space conversions
-        self.vae_scale_factor = 2 ** (len(self.vae.config.block_out_channels) - 1)
-        self.default_sample_size = 128  # Default latent size
-```
-
-**Key Changes:**
-
-1. **`od_config` parameter:** All configuration through `OmniDiffusionConfig`
-2. **Manual component loading:** No `register_modules()`, load each component explicitly
-3. **Local files support:** Check `os.path.exists(model)` for local checkpoints
-4. **Transformer with config:** Pass `od_config` to transformer constructor
-
-#### 2.3: Adapt `__call__` → `forward` Method
-
-**Change signature:**
-
-```diff
-- @torch.no_grad()
-- def __call__(
-+ def forward(
-    self,
-+   req: OmniDiffusionRequest,  # ← Add request parameter here
-- ):
-+ ) -> DiffusionOutput:  # ← Add return type
-```
-
-[`OmniDiffusionRequest`](https://docs.vllm.ai/projects/vllm-omni/en/latest/api/vllm_omni/diffusion/request/#vllm_omni.diffusion.request.OmniDiffusionRequest) is a dataclass that contains the **prompts** and **sampling parameters** [`OmniDiffusionSamplingParams`](https://docs.vllm.ai/projects/vllm-omni/en/latest/api/vllm_omni/inputs/data/#vllm_omni.inputs.data.OmniDiffusionSamplingParams) for the diffusion pipeline execution. It also contains a request_id for other components to trace this request and its outputs.
-
-See some parameters in `OmniDiffusionSamplingParams` as follows:
-
-| parameters | type |value | function |
-|:---:|:---:|:---:|:---:|
-| `num_inference_steps` | `int` | 50 |  The number of diffusion steps during inference|
-| `guidance_scale` |  `float` | 0.0 |  The classifier free guidance scale |
-| `width` and `height` | `int` | None | The width and height of the generated image |
-
-**Extract parameters from request:**
-
-```python
-from vllm_omni.diffusion.request import OmniDiffusionRequest
-from vllm_omni.diffusion.data import DiffusionOutput
-
-def forward(
-    self,
-    req: OmniDiffusionRequest,
-) -> DiffusionOutput:
-    # Extract prompts from request
-    if req.prompts is not None:
-        prompt = [
-            p if isinstance(p, str) else (p.get("prompt") or "")
-            for p in req.prompts
-        ]
-
-    # Extract sampling parameters
-    sampling_params = req.sampling_params
-    num_inference_steps = sampling_params.num_inference_steps or 50
-    guidance_scale = sampling_params.guidance_scale or 7.5
-    height = sampling_params.height or (self.default_sample_size * self.vae_scale_factor)
-    width = sampling_params.width or (self.default_sample_size * self.vae_scale_factor)
-
-    # For image editing pipelines, extract images from multi_modal_data
-    if hasattr(req, 'multi_modal_data') and req.multi_modal_data:
-        input_images = req.multi_modal_data.get('image', [])
-
-    # ... rest of generation logic
-```
-
-For an image editing model, an example `OmniDiffusionRequest` is like:
-```python
-{
-    "prompt": "turn this cat to a dog",
-    "multi_modal_data": {"image": input_image}
-},
-```
-
-**Wrap output:**
-
-```diff
-    # Generate images
-    images = self.vae.decode(latents)[0]
-
--   return {"images": images}
-+   return DiffusionOutput(output=images)
-```
-
-#### 2.4: Extract Pre/Post-Processing Functions
-
-vLLM-Omni separates image processing from the main pipeline for better modularity.
-
-**Post-processing function (required):**
-```python
-def get_your_model_post_process_func(
-    od_config: OmniDiffusionConfig,
-):
-    """
-    Create post-processing function for your model.
-
-    Returns a function that converts latents to images.
-    """
-    from diffusers.image_processor import VaeImageProcessor
-    import json
-
-    # Load VAE config to get scale factor
-    model_path = od_config.model
-    if not os.path.exists(model_path):
-        from vllm_omni.diffusion.model_loader.utils import download_weights_from_hf_specific
-        model_path = download_weights_from_hf_specific(model_path, None, ["*"])
-
-    vae_config_path = os.path.join(model_path, "vae/config.json")
-    with open(vae_config_path) as f:
-        vae_config = json.load(f)
-        vae_scale_factor = 2 ** (len(vae_config["block_out_channels"]) - 1)
-
-    # Create image processor
-    image_processor = VaeImageProcessor(vae_scale_factor=vae_scale_factor)
-
-    def post_process_func(images: torch.Tensor):
-        return image_processor.postprocess(images, output_type="pil")
-
-    return post_process_func
-```
-
-**Pre-processing function (for image editing pipelines):**
-
-```python
-def get_your_model_pre_process_func(
-    od_config: OmniDiffusionConfig,
-):
-    """
-    Create pre-processing function for image editing.
-
-    Returns a function that prepares input images.
-    """
-    from PIL import Image
-    from diffusers.image_processor import VaeImageProcessor
-
-    # Load VAE config
-    # ... (similar to post_process_func)
-
-    image_processor = VaeImageProcessor(vae_scale_factor=vae_scale_factor)
-
-    def pre_process_func(
-        request: OmniDiffusionRequest,
-        ):
-        for i, prompt in enumerate(request.prompts):
-            multi_modal_data = prompt.get("multi_modal_data", {}) if not isinstance(prompt, str) else None
-            raw_image = multi_modal_data.get("image", None) if multi_modal_data is not None else None
-            # image pre-processing
-            # after pre-processing, update the request attributes
-            ...
-        return request
-
-    return pre_process_func
-```
-
-#### 2.5: Add Weight Loading Support
-
-Add methods for automatic weight downloading and loading:
-
-```python
-from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader
-from vllm.model_executor.models.utils import AutoWeightsLoader
-
-class YourModelPipeline(nn.Module):
-    def __init__(self, *, od_config: OmniDiffusionConfig, prefix: str = ""):
-        super().__init__()
-        # ... initialization code
-
-        # Define weight sources for automatic loading
-        self.weights_sources = [
-            DiffusersPipelineLoader.ComponentSource(
-                model_or_path=od_config.model,
-                subfolder="transformer",
-                revision=None,
-                prefix="transformer.",
-                fall_back_to_pt=True,
-            )
-        ]
-
-    def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
-        """
-        Customize the weight loading behavior, such as filter weights name.
-
-        Args:
-            weights: Iterable of (param_name, param_tensor) tuples
-
-        Returns:
-            Set of loaded parameter names
-        """
-        loader = AutoWeightsLoader(self)
-        return loader.load_weights(weights)
-```
-
-### Step 3: Register Model
-
-Register your model in `vllm_omni/diffusion/registry.py` so vLLM-Omni can discover and load it.
-
-#### 3.1: Register Pipeline Class
-
-```python
-# vllm_omni/diffusion/registry.py
-
-_DIFFUSION_MODELS = {
-    # Format: "PipelineClassName": (module_folder, module_file, class_name)
-
-    # Existing models
-    "QwenImagePipeline": ("qwen_image", "pipeline_qwen_image", "QwenImagePipeline"),
-    "FluxPipeline": ("flux", "pipeline_flux", "FluxPipeline"),
-
-    # Add your model
-    "YourModelPipeline": (
-        "your_model_name",           # Module folder name
-        "pipeline_your_model",       # Python file name (without .py)
-        "YourModelPipeline",         # Pipeline class name
-    ),
-}
-```
-
-#### 3.2: Register Pre/Post-Processing Function
-
-```python
-# vllm_omni/diffusion/registry.py
-_DIFFUSION_PRE_PROCESS_FUNCS = {
-    # arch: pre_process_func
-    # `pre_process_func` function must be placed in {mod_folder}/{mod_relname}.py,
-    # where mod_folder and mod_relname are  defined and mapped using `_DIFFUSION_MODELS` via the `arch` key
-    "GlmImagePipeline": "get_glm_image_pre_process_func",
-    "QwenImageEditPipeline": "get_qwen_image_edit_pre_process_func",
-
-    # Add your model
-    "YourModelPipeline": "get_your_model_pre_process_func", # Optional
-}
-_DIFFUSION_POST_PROCESS_FUNCS = {
-    # Format: "PipelineClassName": "function_name"
-
-    # Existing models
-    "QwenImagePipeline": "get_qwen_image_post_process_func",
-    "FluxPipeline": "get_flux_post_process_func",
-
-    # Add your model
-    "YourModelPipeline": "get_your_model_post_process_func",
-}
-```
-
-
-#### 3.3: Export from Module
-
-Create/update `__init__.py` to export your classes:
-
-```python
-# vllm_omni/diffusion/models/your_model_name/__init__.py
-
-from .pipeline_your_model import (
-    YourModelPipeline,
-    get_your_model_post_process_func,
-)
-from .your_model_transformer import YourModelTransformer2DModel
-
-__all__ = [
-    "YourModelPipeline",
-    "YourModelTransformer2DModel",
-    "get_your_model_post_process_func",
-]
-```
-
----
-
-### Step 4: Add Example Script
-
-
-If your model is one of Text-to-Image, Text-to-Audio, Text-to-Video, Image-to-Image, Image-to-Video models, you can simply try one of the following offline inference scripts to run your model:
-
-| Model Category | Offline Inference Script |
-|---|---|
-| Image-to-Image | `examples/offline_inference/image_to_image/image_edit.py` |
-| Image-to-Video | `examples/offline_inference/image_to_video/image_to_video.py` |
-| Text-to-Image | `examples/offline_inference/text_to_image/text_to_image.py` |
-| Text-to-Audio | `examples/offline_inference/text_to_audio/text_to_audio.py` |
-| Text-to-Video | `examples/offline_inference/text_to_video/text_to_video.py` |
-
-
-If new CLI arguments need to be added, please edit the offline inference script corresponding to your model category from the table above, and update the example inference script in its corresponding document file (e.g., `examples/offline_inference/text_to_video/text_to_video.md`).
-
-For online inference, all the supported tasks are listed in `docs/user_guide/examples/online_serving/`. If your model falls into these categories, please check the corresponding documentation in this folder and the example at `examples/online_serving/TASK_NAME`. Update them accordingly if needed.
-
----
-
-If your model is an Omni (understanding and generation) model, please follow the steps below.
-
-#### 4.1: Create Example File
-
-Taking **BAGEL** model as examples for both offline and online:
-
-- Offline: `examples/offline_inference/bagel/`
-- Online: `examples/online_serving/bagel/`
-
-Add **two example folders** for your model:
-
-```bash
-mkdir -p examples/offline_inference/your_model_name
-mkdir -p examples/online_serving/your_model_name
-```
-
-**Offline (recommended minimum):** create `examples/offline_inference/your_model_name/end2end.py` and a README.
-
-- Script: `examples/offline_inference/your_model_name/end2end.py`
-  - Parse args like BAGEL (`--model`, `--modality`, optional `--image-path`, `--steps`, etc.)
-  - Use `from vllm_omni.entrypoints.omni import Omni` (or `OmniDiffusion` if your model is diffusion-only)
-  - Save outputs (images/audio/video/text) with deterministic filenames (e.g., `output_0_0.png`)
-- Doc: `examples/offline_inference/your_model_name/README.md`
-  - Include at least one runnable command, e.g.:
-
-```bash
-cd examples/offline_inference/your_model_name
-python end2end.py --model your-org/your-model-name --modality text2img --prompts "A cute cat"
-```
-
-#### 4.2: Add Online Serving Example (OpenAI-Compatible)
-
-Mirror BAGEL’s online serving setup:
-
-- Server launcher: `examples/online_serving/your_model_name/run_server.sh`
-  - Wrap `vllm serve ... --omni --port ...` (and `--stage-configs-path ...` if needed)
-- Client: `examples/online_serving/your_model_name/openai_chat_client.py`
-  - Send requests to `POST /v1/chat/completions`
-  - Support multimodal inputs (e.g., base64 image) if your model needs it
-- Doc: `examples/online_serving/your_model_name/README.md`
-  - Include both “launch server” and “send request”:
-
-```bash
-# Terminal 1: launch server
-cd examples/online_serving/your_model_name
-bash run_server.sh
-
-# Terminal 2: send request
-python openai_chat_client.py --prompt "A cute cat" --modality text2img
-```
-
-
-### Step 5: Test Your Implementation
-
-Before submitting, thoroughly test your implementation.
-
-#### 5.1: Performance/Speed Check
-
-Manually compare **latency/throughput** and **output quality** against a Diffusers baseline.
-
-For a fair comparison, keep the same **prompt**, **seed**, **resolution**, **num_inference_steps**, and **guidance settings**, and run multiple trials to reduce randomness. Record the results (and your hardware / driver / CUDA versions) in your PR description.
-
-
-#### 5.2 Functionality Check in CI
-
-To ensure project maintainability and sustainable development, we encourage contributors to submit test code (unit tests, system tests, or end-to-end tests) alongside their code changes.
-For comprehensive testing guidelines, please refer to the [Test File Structure and Style Guide](../ci/tests_style.md).
-
----
-
-## Advanced Features
-
-Once basic implementation works, add advanced features for better performance.
-
-### torch.compile Support
-
-Enable automatic compilation for repeated blocks:
-
-```python
-# In your_model_transformer.py
-
-class YourModelTransformer2DModel(nn.Module):
-    # Specify which blocks can be compiled
-    _repeated_blocks = ["YourTransformerBlock"]  # List of block class names
-
-    def __init__(self, ...):
-        super().__init__()
-        # ... initialization
-```
-
-vLLM-Omni automatically compiles blocks in `_repeated_blocks` when `torch.compile` is available.
-
-### Tensor Parallelism
-
-See detailed guide: [How to add Tensor Parallel support](../../design/feature/tensor_parallel.md)
-
-**Quick setup:**
-
-1. Replace Linear layers by various parallel linear layers (e.g., `ColumnParallelLinear`) in vLLM
-2. Check `tp_size` validity: `hidden_dim`, `num_heads`, and `num_kv_heads` must be divisible by `tp_size`
-
-**Usage:** Set `tensor_parallel_size` when initializing:
-```python
-omni = Omni(model="your-model", tensor_parallel_size=2)
-```
-
-### CFG Parallelism
-
-See detailed guide: [How to add CFG-Parallel support](../../design/feature/cfg_parallel.md)
-
-**Quick setup:**
-
-1. Implement `diffuse()` method
-2. Inherit `CFGParallelMixin` in your pipeline class
-
-**Usage:** Set `cfg_parallel_size` when initializing:
-```python
-omni = Omni(model="your-model", cfg_parallel_size=2)
-```
-
-### Sequence Parallelism
-
-See detailed guide: [How to add Sequence Parallel support](../../design/feature/sequence_parallel.md)
-
-**Quick setup:**
-
-1. Add `_sp_plan` class attribute to transformer
-2. Specify where to shard/gather tensors
-
-**Usage:** Set `ulysses_degree` and `ring_degree` when initializing:
-```python
-omni = Omni(model="your-model", ulysses_degree=2, ring_degree=2)
-```
-
-### Cache Acceleration
-
-#### TeaCache
-
-See detailed guide: [How to add TeaCache support](../../design/feature/teacache.md)
-
-**Quick setup:**
-
-1. Write extractor function
-2. Register in `EXTRACTOR_REGISTRY`
-3. Add polynomial coefficients
-
-**Usage:** Set `cache_backend` and `cache_config` when initializing:
-```python
-omni = Omni(model="your-model",
-    cache_backend="tea_cache",
-    cache_config={"rel_l1_thresh": 0.2}
-)
-
-```
-
-
-#### Cache-DiT
-
-See detailed guide: [How to add Cache-DiT support](../../design/feature/cache_dit.md)
-
-**Quick setup:**
-
-- For standard models: Works automatically
-- For complex architectures: Write custom cache config
-
-**Usage:** Set `cache_backend` and `cache_config` when initializing:
-```python
-omni = Omni(model="your-model",  
-    cache_backend="cache_dit",
-    cache_config={
-        "Fn_compute_blocks": 1,
-        "Bn_compute_blocks": 0,
-        "max_warmup_steps": 4,
-    }
-)
-```
-
-### CPU Offload
-
-See detailed guide: [CPU Offloading for Diffusion Models](../../user_guide/diffusion/cpu_offload_diffusion.md)
-
-vLLM-Omni provides two offloading strategies to reduce GPU memory usage:
-
-1. **Model-level offload**: Mutual exclusion between DiT and encoders (only one on GPU at a time)
-2. **Layerwise (Blockwise) offload**: Keeps only a single transformer block on GPU at a time with compute-memory overlap
-
-**Usage:** Enable offload when initializing:
-```python
-# Model-level offload
-omni = Omni(model="your-model", enable_cpu_offload=True)
-
-# Layerwise offload
-omni = Omni(model="your-model", enable_layerwise_offload=True)
-```
-
-**To support layerwise offloading:** Define the blocks attribute name in your transformer:
-
-```python
-class WanTransformer3DModel(nn.Module):
-    _layerwise_offload_blocks_attr = "blocks"  # Attribute name containing transformer blocks
-
-    def __init__(self):
-        self.blocks = nn.ModuleList([...])  # Transformer blocks
-```
-
-**Note:** Layerwise offloading is primarily recommended for large **video generation models** where the compute cost per block is high enough to effectively overlap with memory prefetch operations.
-
-
----
-
-## Troubleshooting
-
-
-**Issue: ImportError when loading model**
-
-**Symptoms:** `ModuleNotFoundError` or `ImportError` when calling `Omni(model="your-model")`
-
-**Causes:**
-
-1. Model not registered in `registry.py`
-2. Wrong class name in registry
-3. Missing `__init__.py` exports
-
-
-**Issue: Shape mismatch in attention**
-
-**Symptoms:** `RuntimeError: shape mismatch` in attention forward
-
-**Cause:** Incorrect tensor reshaping for vLLM-Omni's attention interface
-
-**Solution:** Ensure correct shapes:
-
-```python
-# vLLM-Omni expects: [batch, seq_len, num_heads, head_dim]
-query = query.view(batch_size, seq_len, self.num_heads, self.head_dim)
-key = key.view(batch_size, kv_seq_len, self.num_kv_heads, self.head_dim)
-value = value.view(batch_size, kv_seq_len, self.num_kv_heads, self.head_dim)
-
-hidden_states = self.attn(query, key, value, attn_metadata=attn_metadata)
-
-# Reshape back: [batch, seq_len, num_heads, head_dim] → [batch, seq_len, hidden_size]
-hidden_states = hidden_states.reshape(batch_size, seq_len, -1)
-```
-
-**Issue: Different outputs compared to Diffusers**
-
-**Symptoms:** Generated images look different from Diffusers
-
-**Causes:**
-
-1. Attention backend differences (FlashAttention vs PyTorch SDPA)
-2. Missing normalization or scaling
-
-**4. Issue: Out of memory (OOM)**
-
-**Symptoms:** CUDA out of memory errors
-
-**Solutions:**
-
-1. **Reduce batch size:**
-   ```python
-   omni.generate(prompts=[...], max_batch_size=2)
-   ```
-
-2. **Use smaller image size:**
-   ```python
-   sampling_params = OmniDiffusionSamplingParams(height=512, width=512)
-   ```
-
-3. **Enable model offloading:**
-   ```python
-   omni = Omni(model="...", enable_cpu_offload=True)
-   ```
-
-4. **Apply vae tiling and slicing**
-   ```python
-   omni = Omni(model="...", vae_use_slicing=True, vae_use_tiling=True,)
-   ```
-
----
-
-## Pull Request Checklist
-
-When submitting a PR to add your model, include:
-
-**1. Implementation Files**
-
-- ✅ Transformer model (`xxx_transformer.py`)
-- ✅ Pipeline (`pipeline_xxx.py`)
-- ✅ Registry entries in `registry.py`
-- ✅ `__init__.py` with proper exports
-
-**2. Example and Tests**
-
-- ✅ Example script in `examples/`
-- ✅ Test file in `tests/e2e/`
-- ✅ Documentation (`docs/`) creation or updates
-
-_Note: End-to-end test files in `tests/e2e/` are optional but strongly recommended. README updates are required for all new models._
-
-**3. Documentation Updates**
-
-- ✅ Add model to supported models table in `docs/models/supported_models.md`
-- ✅ If supporting acceleration features (e.g., sequence parallelism, CFG parallel), update acceleration feature tables in:
-  - `docs/user_guide/diffusion_acceleration.md`
-  - `docs/user_guide/diffusion/parallelism_acceleration.md`
-
----
-
-## Model Recipe
-
-After implementing and testing your model, please add a model recipe to the [vllm-project/recipes](https://github.com/vllm-project/recipes) repository. This helps other users understand how to use your model with vLLM-Omni.
-
-**What to Include**
-
-Your recipe should include:
-
-1. **Model Overview**: Brief description of the model and its capabilities
-2. **Installation Instructions**: Step-by-step setup instructions including:
-   - Installing vllm-omni and dependencies
-   - Installing any additional required packages (e.g., xformers, diffusers)
-   - Any version requirements
-3. **Usage Examples**: Command-line examples demonstrating how to run the model
-4. **Configuration Details**: Important configuration parameters and their meanings
-
-**Example**
-
-For reference, see the [LongCat recipe example](https://github.com/vllm-project/recipes/pull/179) which demonstrates the expected format and structure.
-
-**Recipe Location**
-
-Create your recipe file in the appropriate directory structure:
-- For organization-specific models: `OrganizationName/ModelName.md`
-- For general models: `ModelName.md`
-
-The recipe should be a Markdown file that provides clear, reproducible instructions for users to get started with your model.
-
----
-
-## Reference Implementations
-
-Study these complete examples:
-
-| Model | Architecture | Key Features | Files |
-|-------|--------------|--------------|-------|
-| **Qwen-Image** | Dual-stream transformer | CFG-Parallel, SP, TP, Cache | `vllm_omni/diffusion/models/qwen_image/` |
-| **Wan2.2** | Video transformer | Dual transformers, SP, CFG-Parallel | `vllm_omni/diffusion/models/wan2_2/` |
-
----
-
-## Summary
-
-Adding a diffusion model to vLLM-Omni involves:
-
-1. ✅ **Adapt transformer** - Replace attention, remove mixins, add config support
-2. ✅ **Adapt pipeline** - Change interface, add request handling, extract processing
-3. ✅ **Register model** - Add entries to `registry.py`
-4. ✅ **Add examples** - Provide runnable scripts
-5. ✅ **Test thoroughly** - Verify correctness and performance
-6. ✅ **Add advanced features** - Enable parallelism and acceleration (optional)
-7. ✅ **Submit PR** - Include verification results and documentation
-
-**Need help?** Check reference implementations or ask in [slack.vllm.ai](https://slack.vllm.ai) or vLLM user forum at [discuss.vllm.ai](https://discuss.vllm.ai).

docs/contributing/model/adding_omni_model.md

@@ -1,633 +0,0 @@
-# Adding an Omni-Modality Model
-
-This guide walks through the process of adding a new multi-stage model to vLLM-Omni, using **Qwen3-Omni** as a comprehensive example. Qwen3-Omni is a multi-stage omni-modality model that demonstrates the full capabilities of vLLM-Omni's architecture.
-
-## Table of Contents
-
-1. [Overview](#overview)
-2. [Directory Structure](#directory-structure)
-3. [Step-by-Step Implementation](#step-by-step-implementation)
-4. [Key Components](#key-components)
-5. [Model Registration](#model-registration)
-6. [Stage Configuration](#stage-configuration)
-7. [Stage Input Processors](#stage-input-processors)
-8. [Testing](#testing)
-9. [Adding a Model Recipe](#adding-a-model-recipe)
-10. [Summary](#summary)
-
-## Overview
-
-vLLM-Omni supports multi-stage model architectures where different stages can run on different devices and process different modalities. The Qwen3-Omni model exemplifies this with three stages:
-
-1. **Thinker Stage**: Multimodal understanding (text + audio + video) → text generation
-2. **Talker Stage**: Text embeddings → RVQ codec codes
-3. **Code2Wav Stage**: RVQ codes → audio waveform
-
-Each stage is implemented as a separate model class that can be configured independently.
-
-## Directory Structure
-
-When adding a new model, you'll need to create the following structure:
-
-```
-vllm_omni/model_executor/models/
-└── your_model_name/              # Model directory (e.g., qwen3_omni)
-    ├── __init__.py               # Exports main model class
-    ├── your_model.py             # Main unified model class
-    ├── your_model_stage1_implementation.py      # Stage 1 implementation (e.g., thinker)
-    ├── your_model_stage2_implementation.py      # Stage 2 implementation (e.g., talker)
-    └── your_model_stage3_implementation.py      # Stage 3 implementation (e.g., code2wav)
-    └── ... maybe other stage implementations
-
-vllm_omni/model_executor/stage_input_processors/
-└── your_model_name.py            # Stage transition processors
-
-vllm_omni/model_executor/stage_configs/
-└── your_model_name.yaml          # Stage configuration file
-```
-
-## Step-by-Step Implementation
-
-### Step 1: Create the Model Directory
-
-Create a new directory under `vllm_omni/model_executor/models/`
-
-### Step 2: Implement Stage Components
-
-For Qwen3-Omni, we have three stage components:
-
-#### 2.1 Thinker Stage (`qwen3_omni_moe_thinker.py`)
-
-The thinker stage handles multimodal understanding. Key features:
-
-- Inherits from base Qwen3 MoE model in vLLM, using vLLM fused ops & page attn to accelerate
-- Implements multimodal processing interfaces
-- Handles audio, video, and image inputs
-- Generates text outputs
-
-```python
-from vllm.model_executor.models.interfaces import SupportsMultiModal, SupportsPP
-from vllm.model_executor.models.qwen3_moe import Qwen3MoeForCausalLM
-
-class Qwen3OmniMoeThinkerForConditionalGeneration(
-    Qwen3MoeForCausalLM,
-    SupportsMultiModal,
-    SupportsPP
-):
-    """Thinker stage: multimodal understanding → text generation."""
-
-    def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
-        # Initialize base model
-        # Set up multimodal processors
-        # Configure audio/video/image encoders
-        pass
-```
-
-#### 2.2 Talker Stage (`qwen3_omni_moe_talker.py`)
-
-The talker stage converts text embeddings to codec codes:
-
-```python
-class Qwen3OmniMoeTalkerForConditionalGeneration(
-    Qwen3MoeForCausalLM,
-    SupportsPP
-):
-    """Talker stage: text embeddings → RVQ codec codes."""
-
-    def __init__(self, vllm_config, talker_config, prefix):
-        # Initialize base model
-        # Replace LM head with codec head
-        # Set up text projection from thinker
-        pass
-```
-
-#### 2.3 Code2Wav Stage (`qwen3_omni_code2wav.py`)
-
-The code2wav stage generates audio waveforms:
-
-```python
-class Qwen3OmniMoeCode2Wav(nn.Module):
-    """Code2Wav stage: RVQ codes → audio waveform."""
-
-    def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
-        # Initialize audio decoder
-        # Set up codec processing
-        pass
-```
-
-### Step 3: Implement the Unified Model Class
-
-The main model class (`qwen3_omni.py`) orchestrates all stages:
-
-```python
-@MULTIMODAL_REGISTRY.register_processor(
-    Qwen3OmniMoeThinkerMultiModalProcessor,
-    info=Qwen3OmniMoeThinkerProcessingInfo,
-    dummy_inputs=Qwen3OmniMoeThinkerDummyInputsBuilder,
-)
-class Qwen3OmniMoeForConditionalGeneration(
-    nn.Module, SupportsMultiModal, SupportsPP, Qwen3OmniMoeConditionalGenerationMixin
-):
-    """
-    Unified Qwen3 Omni MoE model combining thinker, talker, and code2wav.
-
-    Architecture:
-    - Thinker: Multimodal understanding (text + audio + video) → text generation
-    - Talker: Text embeddings → RVQ codec codes
-    - Code2Wav: RVQ codes → audio waveform
-
-    Usage:
-        Set `model_stage` in vllm_config to one of: "thinker", "talker", "code2wav"
-    """
-
-    def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
-        super().__init__()
-        self.have_multimodal_outputs = True
-        config: Qwen3OmniMoeConfig = vllm_config.model_config.hf_config
-
-        # Determine which stage to initialize
-        self.model_stage = vllm_config.model_config.model_stage
-
-        if self.model_stage == "thinker":
-            # Initialize thinker model
-            thinker_vllm_config = vllm_config.with_hf_config(
-                config.thinker_config,
-                architectures=["Qwen3OmniMoeThinkerForConditionalGeneration"]
-            )
-            self.thinker = init_vllm_registered_model(
-                vllm_config=thinker_vllm_config,
-                prefix=maybe_prefix(prefix, "thinker"),
-                hf_config=config.thinker_config,
-                architectures=["Qwen3OmniMoeThinkerForConditionalGeneration"],
-            )
-            self.model = self.thinker
-
-        elif self.model_stage == "talker":
-            # Initialize talker model
-            talker_vllm_config = vllm_config.with_hf_config(
-                config.talker_config,
-                architectures=["Qwen3OmniMoeTalkerForConditionalGeneration"]
-            )
-            self.talker = init_vllm_registered_model(
-                vllm_config=talker_vllm_config,
-                prefix=maybe_prefix(prefix, "talker"),
-                hf_config=config.talker_config,
-                architectures=["Qwen3OmniMoeTalkerForConditionalGeneration"],
-            )
-            self.model = self.talker
-
-        elif self.model_stage == "code2wav":
-            # Initialize code2wav model
-            code2wav_vllm_config = vllm_config.with_hf_config(
-                config.code2wav_config,
-                architectures=["Qwen3OmniMoeCode2Wav"]
-            )
-            self.code2wav = init_vllm_registered_model(
-                vllm_config=code2wav_vllm_config,
-                prefix=maybe_prefix(prefix, "code2wav"),
-                hf_config=config.code2wav_config,
-                architectures=["Qwen3OmniMoeCode2Wav"],
-            )
-            self.model = self.code2wav
-        else:
-            raise ValueError(
-                f"Invalid model_stage: {self.model_stage}. "
-                f"Must be one of: 'thinker', 'talker', 'code2wav'"
-            )
-```
-
-#### Key Methods to Implement
-
-1. **`forward()`**: Handles the forward pass for each stage
-2. **`embed_input_ids()`**: Embeds input token IDs
-3. **`embed_multimodal()`**: Processes multimodal inputs (if applicable)
-4. **`compute_logits()`**: Computes logits from hidden states
-5. **`load_weights()`**: Loads model weights with proper prefixing of different stages
-
-### Step 4: Create `__init__.py`
-
-Export the main model class:
-
-```python
-# vllm_omni/model_executor/models/qwen3_omni/__init__.py
-from .qwen3_omni import Qwen3OmniMoeForConditionalGeneration
-
-__all__ = ["Qwen3OmniMoeForConditionalGeneration"]
-```
-
-## Key Components
-
-### 1. Model Interfaces
-
-Your model should implement the appropriate interfaces:
-
-- **`SupportsMultiModal`**: For models that process multimodal inputs
-- **`SupportsPP`**: For models that support pipeline parallelism
-- **`SupportsMRoPE`**: For models using multi-dimensional RoPE (if applicable)
-
-### 2. Multimodal Registration
-
-If your model processes multimodal inputs, register it with the multimodal registry:
-
-```python
-@MULTIMODAL_REGISTRY.register_processor(
-    YourMultiModalProcessor,
-    info=YourProcessingInfo,
-    dummy_inputs=YourDummyInputsBuilder,
-)
-class YourModel(nn.Module, SupportsMultiModal):
-    pass
-```
-
-### 3. Weight Loading
-
-Implement `load_weights()` to handle weight loading with proper prefixing:
-
-```python
-def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
-    """Load weights for all components of the omni model."""
-    loaded_weights = set()
-    thinker_weights = []
-    talker_weights = []
-    code2wav_weights = []
-
-    # Separate weights by component
-    for k, v in weights:
-        if k.startswith("thinker."):
-            thinker_weights.append((k, v))
-        elif k.startswith("talker."):
-            talker_weights.append((k, v))
-        elif k.startswith("code2wav."):
-            code2wav_weights.append((k, v))
-
-    # Load each component's weights
-    if self.thinker and thinker_weights:
-        thinker_loaded = self.thinker.load_weights(thinker_weights)
-        thinker_loaded = add_prefix_to_loaded_weights(thinker_loaded, "thinker")
-        loaded_weights.update(thinker_loaded)
-
-    # Similar for talker and code2wav...
-
-    return loaded_weights
-```
-
-### 4. Output Format
-
-Use `OmniOutput` for stage outputs:
-
-```python
-from vllm_omni.model_executor.models.output_templates import OmniOutput
-
-# In forward method
-return OmniOutput(
-    text_hidden_states=hidden_states,
-    multimodal_outputs={"additional_data": data},
-    next_token_id=next_token_id,
-)
-```
-
-## Model Registration
-
-Register your model in `vllm_omni/model_executor/models/registry.py`:
-
-```python
-_OMNI_MODELS = {
-    # ... existing models ...
-
-    # Your new model
-    "YourModelForConditionalGeneration": (
-        "your_model_name",        # Module folder name
-        "your_model",             # Module file name (without .py)
-        "YourModelForConditionalGeneration",  # Class name
-    ),
-    "YourModelThinkerForConditionalGeneration": (
-        "your_model_name",
-        "your_model_thinker",
-        "YourModelThinkerForConditionalGeneration",
-    ),
-    # ... other stages ...
-}
-```
-
-The registry uses lazy loading, so the model class is imported only when needed.
-
-## Stage Configuration
-
-Create a YAML configuration file in `vllm_omni/model_executor/stage_configs/`. For a complete example, see the [Qwen3-Omni configuration file](gh-file:vllm_omni/model_executor/stage_configs/qwen3_omni_moe.yaml).
-
-### Key Configuration Fields
-
-- **`model_stage`**: Which stage to run ("thinker", "talker", "code2wav", etc.)
-- **`model_arch`**: The model architecture name (must match registry)
-- **`engine_input_source`**: List of stage IDs that provide input to this stage
-- **`custom_process_input_func`**: Function to process inputs from previous stages
-- **`final_output`**: Whether this stage produces the final output (True/False)
-- **`final_output_type`**: Type of final output ("text", "audio", "image", etc.)
-
-## Stage Input Processors
-
-Stage transitions are the mechanism by which outputs from one stage are converted into inputs for the next stage. This section explains where and how stage transitions occur.
-
-### Where Stage Transitions Are Called
-
-Stage transitions happen automatically in the orchestrator (`OmniLLM` class) during the generation loop. Here's the detailed flow:
-
-1. **Location**: `vllm_omni/entrypoints/omni_llm.py` in the `_run_generation()` method
-2. **Trigger**: When a stage completes processing and produces outputs
-3. **Execution Flow**:
-   ```python
-   # In omni_llm.py, _run_generation() method (around line 345-460)
-
-   # Main orchestrator loop polls each stage for completed requests
-   for stage_id, stage in enumerate(self.stage_list):
-       result = stage.try_collect()  # Get completed request
-       if result is None:
-           continue
-
-       # Store outputs from this stage
-       engine_outputs = _load(result, obj_key="engine_outputs", shm_key="engine_outputs_shm")
-       stage.set_engine_outputs(engine_outputs)
-
-       # Check if there's a next stage to forward to
-       next_stage_id = stage_id + 1
-       if next_stage_id < len(self.stage_list):
-           next_stage: OmniStage = self.stage_list[next_stage_id]
-
-           # THIS IS WHERE STAGE TRANSITION HAPPENS
-           next_inputs = next_stage.process_engine_inputs(
-               self.stage_list,
-               [request_id_to_prompt[req_id]]
-           )
-
-           # Submit to next stage
-           task = {
-               "type": OmniStageTaskType.GENERATE,
-               "request_id": req_id,
-               "engine_inputs": next_inputs[0],
-               "sampling_params": sampling_params_list[next_stage_id],
-           }
-           next_stage.submit(task)
-   ```
-
-### How Stage Transitions Work
-
-The stage transition process follows these steps:
-
-1. **Stage Completion**: When a stage finishes processing a request, it stores outputs via `stage.set_engine_outputs(engine_outputs)`
-
-2. **Transition Detection**: The orchestrator checks if there's a next stage and calls `process_engine_inputs()` on it
-
-3. **Input Processing**: The `process_engine_inputs()` method in `OmniStage` (`omni_stage.py`) handles the transition:
-   ```python
-   def process_engine_inputs(
-       self, stage_list: list[Any], prompt: OmniTokensPrompt | TextPrompt = None
-   ) -> list[OmniTokensPrompt | TextPrompt]:
-       """Process engine inputs for this stage from upstream stage outputs."""
-
-       if self.custom_process_input_func is None:
-           # Default behavior: pass token IDs directly
-           # Extract outputs from source stage
-           source_stage_id = self.engine_input_source[0]
-           source_outputs = stage_list[source_stage_id].engine_outputs
-           # ... create OmniTokensPrompt from token_ids ...
-       else:
-           # Custom transition function (YOUR CODE HERE)
-           return self.custom_process_input_func(
-               stage_list,
-               self.engine_input_source,
-               prompt,
-               self.requires_multimodal_data
-           )
-   ```
-   - If `custom_process_input_func` is configured, it calls that function
-   - Otherwise, it uses default behavior (passing token IDs directly)
-
-4. **Custom Function Execution**: Your custom function receives:
-   - `stage_list`: List of all stage objects (to access upstream stage outputs)
-   - `engine_input_source`: List of source stage IDs (e.g., `[0]` for stage 0)
-   - `prompt`: Original prompt data (for preserving multimodal data)
-   - `requires_multimodal_data`: Whether multimodal data is required
-
-5. **Output Format**: The function must return a list of `OmniTokensPrompt` objects ready for the next stage
-
-### Data Structures in Stage Transitions
-
-Understanding the data structures is crucial for implementing stage transitions:
-
-**Input to your function:**
-- `stage_list[source_stage_id].engine_outputs`: List of `EngineCoreOutput` objects
-  - Each contains `outputs`: List of `RequestOutput` objects
-  - Each `RequestOutput` has:
-    - `token_ids`: Generated token IDs
-    - `multimodal_output`: Dict with keys like `"code_predictor_codes"`, etc.
-      - These are the hidden states or intermediate outputs from the model's forward pass
-    - `prompt_token_ids`: Original prompt token IDs
-
-**Output from your function:**
-- Must return `list[OmniTokensPrompt]` where each `OmniTokensPrompt` contains:
-  - `prompt_token_ids`: List[int] - Token IDs for the next stage
-  - `additional_information`: Dict[str, Any] - Optional metadata (e.g., embeddings, hidden states)
-  - `multi_modal_data`: Optional multimodal data if needed
-
-### How Model Outputs Are Stored
-
-The model's `forward()` method returns an `OmniOutput` object that contains:
-- `text_hidden_states`: Final hidden states for text generation
-- `multimodal_outputs`: Dict containing intermediate outputs
-
-These outputs are captured during the forward pass and stored in `multimodal_output` with specific keys:
-
-```python
-# In your model's forward() method (e.g., qwen3_omni.py)
-def forward(self, ...):
-    # ... processing ...
-
-    # For thinker stage: capture embeddings and hidden states
-    multimodal_outputs = {
-        "0": captured_embeddings,      # Layer 0 embeddings
-        "24": captured_hidden_states,  # Layer 24 hidden states
-        "tts_bos_embed": tts_bos_embed,
-        "tts_eos_embed": tts_eos_embed,
-        # ... other intermediate outputs ...
-    }
-
-    return OmniOutput(
-        text_hidden_states=hidden_states,
-        multimodal_outputs=multimodal_outputs,
-    )
-```
-
-These keys are then accessible in your stage transition function:
-```python
-# In stage_input_processors/qwen3_omni.py
-thinker_embeddings = output.multimodal_output["0"]  # Access by key
-thinker_hidden_states = output.multimodal_output["24"]
-```
-
-### Key Points
-
-1. **Accessing Upstream Outputs**: Use `stage_list[source_stage_id].engine_outputs` to get outputs from the source stage
-2. **Extracting Data**: Access `output.multimodal_output[key]` to get specific hidden states or intermediate results
-   - Keys are defined by your model's `forward()` method when it creates `multimodal_outputs`
-3. **Device Management**: Move tensors to appropriate devices (CPU for serialization, GPU for processing)
-4. **Shape Transformations**: Reshape tensors as needed for the next stage (e.g., flattening codec codes)
-5. **Batch Handling**: Process each request in the batch separately and return a list
-
-### Complete Flow Diagram
-
-<p align="center">
-  <picture>
-    <source media="(prefers-color-scheme: dark)" src="https://raw.githubusercontent.com/vllm-project/vllm-omni/refs/heads/main/docs/source/architecture/vllm-omni-dataflow-between-stages.png">
-    <img alt="Data Flow between stages" src="https://raw.githubusercontent.com/vllm-project/vllm-omni/refs/heads/main/docs/source/architecture/vllm-omni-dataflow-between-stages.png" width=55%>
-  </picture>
-</p>
-
-### Implementation Example
-
-Create stage transition processors in `vllm_omni/model_executor/stage_input_processors/your_model_name.py`:
-
-```python
-# qwen3_omni.py
-
-def thinker2talker(
-    stage_list: list[Any],
-    engine_input_source: list[int],
-    prompt: OmniTokensPrompt | TextPrompt | None = None,
-    requires_multimodal_data: bool = False,
-) -> list[OmniTokensPrompt]:
-    """
-    Process thinker outputs to create talker inputs.
-
-    Args:
-        stage_list: List of stage objects
-        engine_input_source: Source stage IDs (typically [0] for thinker)
-        prompt: Original prompt data
-
-    Returns:
-        List of OmniTokensPrompt for talker stage
-    """
-    source_stage_id = engine_input_source[0]
-    thinker_outputs = stage_list[source_stage_id].engine_outputs
-    talker_inputs = []
-
-    for thinker_output in thinker_outputs:
-        output = thinker_output.outputs[0]
-        # Extract thinker embeddings and hidden states
-        thinker_embeddings = output.multimodal_output["0"].float().clone().detach().cuda()
-        thinker_hidden_states = output.multimodal_output["24"].float().clone().detach().cuda()
-
-        info = {
-            "thinker_embeddings": thinker_embeddings,
-            "thinker_hidden_states": thinker_hidden_states,
-            "thinker_sequences": thinker_output.prompt_token_ids + output.token_ids,
-            "thinker_input_ids": thinker_output.prompt_token_ids,
-        }
-
-        talker_inputs.append(
-            OmniTokensPrompt(
-                prompt_token_ids=[0] * computed_length,
-                additional_information=info,
-                multi_modal_data=None,
-            )
-        )
-
-    return talker_inputs
-
-
-def talker2code2wav(
-    stage_list: list[Any],
-    engine_input_source: list[int],
-    prompt: OmniTokensPrompt | TextPrompt | None = None,
-    requires_multimodal_data: bool = False,
-) -> list[OmniTokensPrompt]:
-    """
-    Process talker outputs to create code2wav inputs.
-    """
-    source_stage_id = engine_input_source[0]
-    talker_outputs = stage_list[source_stage_id].engine_outputs
-    code2wav_inputs = []
-
-    for talker_output in talker_outputs:
-        output = talker_output.outputs[0]
-        # Extract codec codes
-        codec_codes = (
-            output.multimodal_output["code_predictor_codes"]
-            .to(torch.long)
-            .transpose(0, 1)
-            .cpu()
-            .to(torch.long)
-            .reshape(-1)
-            .tolist()
-        )
-
-        code2wav_inputs.append(
-            OmniTokensPrompt(
-                prompt_token_ids=codec_codes,
-                multi_modal_data=None,
-            )
-        )
-
-    return code2wav_inputs
-```
-
-## Testing
-
-For comprehensive testing guidelines, please refer to the [Test File Structure and Style Guide](../ci/tests_style.md).
-
-## Adding a Model Recipe
-
-After implementing and testing your model, please add a model recipe to the [vllm-project/recipes](https://github.com/vllm-project/recipes) repository. This helps other users understand how to use your model with vLLM-Omni.
-
-### What to Include
-
-Your recipe should include:
-
-1. **Model Overview**: Brief description of the model and its capabilities
-2. **Installation Instructions**: Step-by-step setup instructions including:
-   - Installing vllm-omni and dependencies
-   - Installing any additional required packages (e.g., xformers, diffusers)
-   - Any version requirements
-3. **Usage Examples**: Command-line examples demonstrating how to run the model
-4. **Configuration Details**: Important configuration parameters and their meanings
-
-### Example
-
-For reference, see the [LongCat recipe example](https://github.com/vllm-project/recipes/pull/179) which demonstrates the expected format and structure.
-
-### Recipe Location
-
-Create your recipe file in the appropriate directory structure:
-- For organization-specific models: `OrganizationName/ModelName.md`
-- For general models: `ModelName.md`
-
-The recipe should be a Markdown file that provides clear, reproducible instructions for users to get started with your model.
-
-## Summary
-
-Adding a new model to vLLM-Omni involves:
-
-1. **Create model directory structure** with stage implementations
-2. **Implement unified model class** that orchestrates stages
-3. **Register model** in `registry.py`
-4. **Create stage configuration** YAML file
-5. **Implement stage input processors** for stage transitions
-6. **Write tests** to verify functionality
-7. **Add model recipe** to the [vllm-project/recipes](https://github.com/vllm-project/recipes) repository (see [Adding a Model Recipe](#adding-a-model-recipe) section)
-
-### Qwen3-Omni Reference Files
-
-For a complete reference implementation, see:
-
-- **Main model**: `vllm_omni/model_executor/models/qwen3_omni/qwen3_omni.py`
-- **Thinker**: `vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_thinker.py`
-- **Talker**: `vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_moe_talker.py`
-- **Code2Wav**: `vllm_omni/model_executor/models/qwen3_omni/qwen3_omni_code2wav.py`
-- **Stage config**: `vllm_omni/model_executor/stage_configs/qwen3_omni_moe.yaml`
-- **Input processors**: `vllm_omni/model_executor/stage_input_processors/qwen3_omni.py`
-- **Registry**: `vllm_omni/model_executor/models/registry.py`
-- **Testing**: `vllm_omni/tests/e2e/offline_inference/test_qwen3_omni.py`
-
-For more information, see:
-- [Architecture Overview](../../design/architecture_overview.md)
-- [Supported Models](../../models/supported_models.md)
-- [Stage Configuration Guide](../../configuration/stage_configs.md)

docs/contributing/profiling.md

@@ -1,149 +0,0 @@
-# Profiling vLLM-Omni
-
-> **Warning:** Profiling incurs significant overhead. Use only for development and debugging, never in production.
-
-vLLM-Omni uses the PyTorch Profiler to analyze performance across both **multi-stage omni-modality models** and **diffusion models**.
-
-### 1. Set the Output Directory
-Before running any script, set this environment variable. The system detects this and automatically saves traces here.
-
-```bash
-export VLLM_TORCH_PROFILER_DIR=./profiles
-```
-
-### 2. Profiling Omni-Modality Models
-
-It is best to limit profiling to one iteration to keep trace files manageable.
-
-```bash
-export VLLM_PROFILER_MAX_ITERS=1
-```
-
-**Selective Stage Profiling**
-The profiler is default to function across all stages. But It is highly recommended to profile specific stages by passing the stages list, preventing from producing too large trace files:
-```python
-# Profile all stages
-omni_llm.start_profile()
-
-# Only profile Stage 1
-omni_llm.start_profile(stages=[1])
-```
-
-```python
-# Stage 0 (Thinker) and Stage 2 (Audio Decoder) for qwen omni
-omni_llm.start_profile(stages=[0, 2])
-```
-
-**Python Usage**: Wrap your generation logic with `start_profile()` and `stop_profile()`.
-
-```python
-from vllm_omni import omni_llm
-
-profiler_enabled = bool(os.getenv("VLLM_TORCH_PROFILER_DIR"))
-
-# 1. Start profiling if enabled
-if profiler_enabled:
-    omni_llm.start_profile(stages=[0])
-
-# Initialize generator
-omni_generator = omni_llm.generate(prompts, sampling_params_list, py_generator=args.py_generator)
-
-total_requests = len(prompts)
-processed_count = 0
-
-# Main Processing Loop
-for stage_outputs in omni_generator:
-
-    # ... [Output processing logic for text/audio would go here] ...
-
-    # Update count to track when to stop profiling
-    processed_count += len(stage_outputs.request_output)
-
-    # 2. Check if all requests are done to stop the profiler safely
-    if profiler_enabled and processed_count >= total_requests:
-        print(f"[Info] Processed {processed_count}/{total_requests}. Stopping profiler inside active loop...")
-
-        # Stop the profiler while workers are still active
-        omni_llm.stop_profile()
-
-        # Wait for traces to flush to disk
-        print("[Info] Waiting 30s for workers to write trace files to disk...")
-        time.sleep(30)
-        print("[Info] Trace export wait time finished.")
-
-omni_llm.close()
-```
-
-
-**Examples**:
-
-1. **Qwen2.5-Omni**:  [https://github.com/vllm-project/vllm-omni/blob/main/examples/offline_inference/qwen2_5_omni/end2end.py](https://github.com/vllm-project/vllm-omni/blob/main/examples/offline_inference/qwen2_5_omni/end2end.py)
-
-2. **Qwen3-Omni**:   [https://github.com/vllm-project/vllm-omni/blob/main/examples/offline_inference/qwen3_omni/end2end.py](https://github.com/vllm-project/vllm-omni/blob/main/examples/offline_inference/qwen3_omni/end2end.py)
-
-
-### 3. Profiling diffusion models
-
-Diffusion profiling is End-to-End, capturing encoding, denoising loops, and decoding.
-
-**CLI Usage:**
-```python
-
-python image_to_video.py \
-    --model Wan-AI/Wan2.2-I2V-A14B-Diffusers \
-    --image qwen-bear.png \
-    --prompt "A cat playing with yarn, smooth motion" \
-    \
-    # Minimize Spatial Dimensions (Optional but helpful):
-    #    Drastically reduces memory usage so the profiler doesn't
-    #    crash due to overhead, though for accurate performance
-    #    tuning you often want target resolutions.
-    --height 48 \
-    --width 64 \
-    \
-    # Minimize Temporal Dimension (Frames):
-    #    Video models process 3D tensors (Time, Height, Width).
-    #    Reducing frames to the absolute minimum (2) keeps the
-    #    tensor size small, ensuring the trace file doesn't become
-    #    multi-gigabytes in size.
-    --num-frames 2 \
-    \
-    # Minimize Iteration Loop (Steps):
-    #    This is the most critical setting for profiling.
-    #    Diffusion models run the same loop X times.
-    #    Profiling 2 steps gives you the exact same performance
-    #    data as 50 steps, but saves minutes of runtime and
-    #    prevents the trace viewer from freezing.
-    --num-inference-steps 2 \
-    \
-    --guidance-scale 5.0 \
-    --guidance-scale-high 6.0 \
-    --boundary-ratio 0.875 \
-    --flow-shift 12.0 \
-    --fps 16 \
-    --output i2v_output.mp4
-
-```
-
-**Examples**:
-
-1. **Qwen image edit**:  [https://github.com/vllm-project/vllm-omni/blob/main/examples/offline_inference/image_to_image/image_edit.py](https://github.com/vllm-project/vllm-omni/blob/main/examples/offline_inference/image_to_image/image_edit.py)
-
-2. **Wan-AI/Wan2.2-I2V-A14B-Diffusers**:   [https://github.com/vllm-project/vllm-omni/tree/main/examples/offline_inference/image_to_video](https://github.com/vllm-project/vllm-omni/tree/main/examples/offline_inference/image_to_video)
-
-> **Note:**
-As of now, asynchronous (online) profiling is not fully supported in vLLM-Omni. While start_profile() and stop_profile() methods exist, they are only reliable in offline inference scripts (e.g., the provided end2end.py examples). Do not use them in server-mode or streaming scenarios—traces may be incomplete or fail to flush.
-
-### 4. Analyzing Omni Traces
-
-Output files are saved to your configured ```VLLM_TORCH_PROFILER_DIR```.
-
-**Output**
-**Chrome Trace** (```.json.gz```): Visual timeline of kernels and stages. Open in Perfetto UI.
-
-**Viewing Tools:**
-
-- [Perfetto](https://ui.perfetto.dev/)(recommended)
-- ```chrome://tracing```(Chrome only)
-
-**Note**: vLLM-Omni reuses the PyTorch Profiler infrastructure from vLLM. See the official vLLM profiler documentation:  [vLLM Profiling Guide](https://docs.vllm.ai/en/stable/contributing/profiling/)

docs/design/architecture_overview.md

@@ -1,195 +0,0 @@
-# Architecture Overview
-
-This document outlines the architectural design for vLLM-Omni.
-
-<p align="center">
-  <picture>
-    <source media="(prefers-color-scheme: dark)" src="https://raw.githubusercontent.com/vllm-project/vllm-omni/refs/heads/main/docs/source/architecture/omni-modality-model-architecture.png">
-    <img alt="Omni-Modality Model Architecture" src="https://raw.githubusercontent.com/vllm-project/vllm-omni/refs/heads/main/docs/source/architecture/omni-modality-model-architecture.png" width=55%>
-  </picture>
-</p>
-
-# Goals
-
-The primary goal of the vLLM-Omni project is to build the fastest and easiest-to-use open-source Omni-Modality model inference & serving engine. vLLM-Omni extends the original vLLM, which was created to support large language models for text-based autoregressive (AR) generation tasks. vLLM-Omni is designed to support:
-
-* **Non-textual Output:** Enables the integration, efficient processing and output of various data types, including but not limited to, images, audio, and video, alongside text.
-* **Non-Autoregressive Structure:** Support model structure beyond autoregressive, especially Diffusion Transformer (DiT), which is widely used in visual and audio generation.
-* **Integration with vLLM Core:** Maintain compatibility and leverage existing vLLM key modules and optimizations where applicable.
-* **Extensibility:** Design a modular and flexible architecture that can easily accommodate new modalities, model architectures, and output formats.
-
-
-# Representative omni-modality models
-
-According to analysis for current popular open-source models, most of them have the combination of AR+DiT. Specifically, they can be further categorized into 3 types below:
-
-**DiT as a main structure, with AR as text encoder (e.g.: Qwen-Image)**
-  A powerful image generation foundation model capable of complex text rendering and precise image editing.
-
-<p align="center">
-  <picture>
-    <source media="(prefers-color-scheme: dark)" src="https://raw.githubusercontent.com/vllm-project/vllm-omni/refs/heads/main/docs/source/architecture/dit-main-architecture.png">
-    <img alt="Qwen-Image" src="https://raw.githubusercontent.com/vllm-project/vllm-omni/refs/heads/main/docs/source/architecture/dit-main-architecture.png" width=30%>
-  </picture>
-</p>
-
-**AR as a main structure, with DiT as multi-modal generator (e.g. BAGEL)**
-  A unified multimodal comprehension and generation model, with cot text output and visual generation.
-
-<p align="center">
-  <picture>
-    <source media="(prefers-color-scheme: dark)" src="https://raw.githubusercontent.com/vllm-project/vllm-omni/refs/heads/main/docs/source/architecture/ar-main-architecture.png">
-    <img alt="Bagel" src="https://raw.githubusercontent.com/vllm-project/vllm-omni/refs/heads/main/docs/source/architecture/ar-main-architecture.png" width=30%>
-  </picture>
-</p>
-
-**AR+DiT (e.g. Qwen-Omni)**
-  A natively end-to-end omni-modal LLM for multimodal inputs (text/image/audio/video...) and outputs (text/audio...).
-
-<p align="center">
-  <picture>
-    <source media="(prefers-color-scheme: dark)" src="https://raw.githubusercontent.com/vllm-project/vllm-omni/refs/heads/main/docs/source/architecture/ar-dit-main-architecture.png">
-    <img alt="Qwen-Omni" src="https://raw.githubusercontent.com/vllm-project/vllm-omni/refs/heads/main/docs/source/architecture/ar-dit-main-architecture.png" width=30%>
-  </picture>
-</p>
-
-# vLLM-Omni main architecture
-
-<p align="center">
-  <picture>
-    <source media="(prefers-color-scheme: dark)" src="https://raw.githubusercontent.com/vllm-project/vllm-omni/refs/heads/main/docs/source/architecture/vllm-omni-main-architecture.png">
-    <img alt="vLLM-Omni Main Architecture" src="https://raw.githubusercontent.com/vllm-project/vllm-omni/refs/heads/main/docs/source/architecture/vllm-omni-main-architecture.png" width=55%>
-  </picture>
-</p>
-
-## Key Components
-
-| Component         | Description                                                                                                                              |
-| ----------------- | ---------------------------------------------------------------------------------------------------------------------------------------- |
-| **OmniRouter**    | provide an intelligent router for Omni-modality requests dispatch                                                                        |
-| **EntryPoints**   | define the APIs for offline/online serving (APIServer, Omni/AsyncOmni) and provide the OmniStage abstraction for different AR/DiT stages |
-| **AR**            | adapted for omni-modality models while inheriting efficient features from vLLM, such as cache management                                 |
-| **Diffusion**     | natively implemented and optimized using acceleration components                                                                         |
-| **OmniConnector** | supports fully disaggregation based on E/P/D/G (Encoding/Processing/Decoding/Generation) disaggregation across stages                    |
-
-Disaggregated stages are managed through configuration, such as in the Qwen3-Omni example, where stages like Thinker, Talker, and Code2wav are defined as separate OmniStage instances with specific resources and input/output type.
-
-## Main features
-
-vLLM-Omni aims to be fast, flexible, and easy to use with the following features:
-
-### Performance and Acceleration
-
-The framework achieves high performance through several optimization techniques:
-
-* **Efficient AR Support:** Leverages efficient KV cache management inherited from vLLM.
-* **Pipelined Execution:** Uses pipelined stage execution overlapping to ensure high throughput.
-* **Full Disaggregation:** Relies on the OmniConnector and dynamic resource allocation across stages.
-* **Diffusion Acceleration:** Includes integrated support for diffusion acceleration. This is managed by the acceleration layer, which handles:
-    * **Cache:** Includes DBCache, TeaCache and third-party integration(e.g., [cache-dit](https://github.com/vipshop/cache-dit)).
-    * **Parallelism:** Supports TP, CP, USP, and CFG.
-    * **Attention:** Provides an interface for third-party integration (e.g., FA3, SAGE, MindIE-SD).
-    * **Quantization:** Supports various quantization implementations including FP8 and AWQ.
-    * **FusedOps:** Allows for custom and third-party integration.
-
-### Flexibility and Usability
-
-vLLM-Omni is designed to be flexible and straightforward for users:
-
-* **Heterogeneous Pipeline Abstraction:** Manages complex model workflows effectively.
-* **Hugging Face Integration:** Offers seamless integration with popular Hugging Face models.
-* **Distributed Inference:** Supports tensor, pipeline, data, and expert parallelism.
-* **Streaming Outputs:** Supports streaming outputs.
-* **Unified API:** Provides a consistent and unified API interface compatible with vLLM.
-* **OpenAI-compatible API Server:** Includes a FastAPI-based server for online serving that is compatible with the OpenAI API.
-
-# Interface design
-
-If you use vLLM, then you know how to use vLLM-Omni from Day 0:
-
-<p align="center">
-  <picture>
-    <source media="(prefers-color-scheme: dark)" src="https://raw.githubusercontent.com/vllm-project/vllm-omni/refs/heads/main/docs/source/architecture/vllm-omni-user-interface.png">
-    <img alt="vLLM-Omni interface design" src="https://raw.githubusercontent.com/vllm-project/vllm-omni/refs/heads/main/docs/source/architecture/vllm-omni-user-interface.png" width=55%>
-  </picture>
-</p>
-
-Taking **Qwen3-Omni** as an example:
-
-## Offline Inference
-The **Omni** class provides a Python interface for offline batched inference. Users initialize the Omni class with a Hugging Face model name and use the generate method, passing inputs that include both text prompts and multi-modal data:
-
-```
-# Create an omni_lm with HF model name.
-from vllm_omni.entrypoints.omni import Omni
-
-omni_lm = Omni(model="Qwen/Qwen3-Omni-30B-A3B-Instruct")
-
-# Example prompts.
-om_inputs = {"prompt": prompt,
-             "multi_modal_data": {
-                 "video": video_frames,
-                 "audio": audio_signal,
-             }}
-
-# Generate texts and audio from the multi-modality inputs.
-outputs = omni_lm.generate(om_inputs, sampling_params_list)
-```
-
-## Online Serving
-Similar to vLLM, vLLM-Omni also provides a FastAPI-based server for online serving. Users can launch the server using the vllm serve command with the `--omni` flag:
-
-```
-vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091
-```
-
-Users can send requests to the server using curl:
-
-```
-# prepare user content
-user_content='[
-        {
-          "type": "video_url",
-          "video_url": {
-            "url": "'"$SAMPLE_VIDEO_URL"'"
-          }
-        },
-        {
-          "type": "text",
-          "text": "Why is this video funny?"
-        }
-      ]'
-    sampling_params_list='[
-      '"$thinker_sampling_params"',
-      '"$talker_sampling_params"',
-      '"$code2wav_sampling_params"'
-    ]'
-    mm_processor_kwargs="{}"
-
-# send the request
-curl -sS -X POST http://localhost:8091/v1/chat/completions \
-    -H "Content-Type: application/json" \
-    -d @- <<EOF
-{
-  "model": "Qwen/Qwen3-Omni-30B-A3B-Instruct",
-  "sampling_params_list": $sampling_params_list,
-  "mm_processor_kwargs": $mm_processor_kwargs,
-  "messages": [
-    {
-      "role": "system",
-      "content": [
-        {
-          "type": "text",
-          "text": "You are Qwen, a virtual human developed by the Qwen Team, Alibaba Group, capable of perceiving auditory and visual inputs, as well as generating text and speech."
-        }
-      ]
-    },
-    {
-      "role": "user",
-      "content": $user_content
-    }
-  ]
-}
-```
-
-For more usages, please refer to [examples](https://github.com/vllm-project/vllm-omni/tree/main/examples).