add

.buildkite/pipeline-intel.yaml

@@ -10,7 +10,7 @@ steps:
           DOCKER_BUILDKIT: "1"
           # Buildkite will automatically replace this with the actual commit hash
           VLLM_IMAGE_TAG: "${BUILDKITE_COMMIT}"
-          VLLM_VERSION: "v0.19.0"
+          VLLM_VERSION: "v0.18.0"
         priority: 100
         timeout_in_minutes: 60
         soft_fail: true

.buildkite/pipeline.yml

@@ -1,21 +1,6 @@
-# Document 1: Buildkite loads only this block on first parse. The next step resolves docs-only skip-ci
-# from git diff, then uploads document 2. When docs-only skip applies, image-build still runs if nightly-test
-# / main NIGHTLY so upload-nightly is not skipped together with test-ready/test-merge.
-#
-# Document 2: appended after `---`; same file, read by upload_pipeline_with_skip_ci.sh (not evaluated as a second pipeline by Buildkite).
-steps:
-  - label: ":github: Resolve skip-ci & upload pipeline"
-    key: upload-ci-pipeline
-    commands:
-      - "bash .buildkite/scripts/upload_pipeline_with_skip_ci.sh"
-    agents:
-      queue: "cpu_queue_premerge"
-
----
 steps:
   - label: ":docker: Build image"
     key: image-build
-    if: __IMAGE_BUILD_IF__
     commands:
       - "aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/q9t5s3a7"
       - "docker build --progress=plain --file docker/Dockerfile.ci -t vllm-omni-ci ."
@@ -28,7 +13,7 @@ steps:
   - label: "Upload Ready Pipeline"
     depends_on: image-build
     key: upload-ready-pipeline
-    if: __UPLOAD_READY_IF__
+    if: build.branch != "main" && build.pull_request.labels includes "ready"
     commands:
       - buildkite-agent pipeline upload .buildkite/test-ready.yml
     agents:
@@ -38,7 +23,7 @@ steps:
   - label: "Upload Merge Pipeline"
     depends_on: image-build
     key: upload-merge-pipeline
-    if: __UPLOAD_MERGE_IF__
+    if: build.branch == "main" && build.env("NIGHTLY") != "1"
     commands:
       - buildkite-agent pipeline upload .buildkite/test-merge.yml
     agents:

.buildkite/scripts/generate-and-upload-nightly-index.sh

@@ -19,7 +19,7 @@ has_new_python=$($PYTHON -c "print(1 if __import__('sys').version_info >= (3,12)
 if [[ "$has_new_python" -eq 0 ]]; then
     # use new python from docker
     docker pull python:3-slim
-    PYTHON="docker run --rm --user $(id -u):$(id -g) -v $(pwd):/app -w /app python:3-slim python3"
+    PYTHON="docker run --rm -v $(pwd):/app -w /app python:3-slim python3"
 fi
 
 echo "Using python interpreter: $PYTHON"
@@ -36,7 +36,7 @@ mkdir -p "$INDICES_OUTPUT_DIR"
 
 # HACK: we do not need regex module here, but it is required by pre-commit hook
 # To avoid any external dependency, we simply replace it back to the stdlib re module
-sed -i.bak 's/import regex as re/import re/g' .buildkite/scripts/generate-nightly-index.py && rm -f .buildkite/scripts/generate-nightly-index.py.bak
+sed -i 's/import regex as re/import re/g' .buildkite/scripts/generate-nightly-index.py
 
 # Generate indices -- the version is just the commit hash (not omni/{commit})
 # because relative paths are computed between the index and wheel directories,
@@ -73,16 +73,15 @@ echo "Pure version (without variant): $pure_version"
 
 # re-generate and copy to /omni/{version}/ only if it does not have "dev" in the version
 if [[ "$version" != *"dev"* ]]; then
-    s3_version="v$pure_version"
+    echo "Re-generating indices for /omni/$pure_version/"
-    echo "Re-generating indices for /omni/$s3_version/"
     rm -rf "${INDICES_OUTPUT_DIR:?}"
     mkdir -p "$INDICES_OUTPUT_DIR"
     # wheel-dir is overridden to be the commit directory, so that the indices point to the correct wheel path
     $PYTHON .buildkite/scripts/generate-nightly-index.py \
-        --version "$s3_version" \
+        --version "$pure_version" \
         --wheel-dir "$BUILDKITE_COMMIT" \
         --current-objects "$obj_json" \
         --output-dir "$INDICES_OUTPUT_DIR" \
         --comment "version $pure_version"
-    aws s3 cp --recursive "$INDICES_OUTPUT_DIR/" "s3://$BUCKET/omni/$s3_version/"
+    aws s3 cp --recursive "$INDICES_OUTPUT_DIR/" "s3://$BUCKET/omni/$pure_version/"
 fi

.buildkite/scripts/generate-nightly-index.py

@@ -11,7 +11,7 @@ from pathlib import Path
 from typing import Any
 from urllib.parse import quote
 
-import re
+import regex as re
 
 
 def normalize_package_name(name: str) -> str:

.buildkite/scripts/upload_pipeline_with_skip_ci.sh

@@ -1,137 +0,0 @@
-#!/usr/bin/env bash
-# Evaluate docs-only skip-ci and upload continuation steps from the same `.buildkite/pipeline.yml`
-# (YAML document after the first `---`). Buildkite `if` is evaluated at upload time.
-set -euo pipefail
-
-ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
-PIPELINE_YML="${ROOT}/.buildkite/pipeline.yml"
-
-# Prints a single digit to stdout: 1 = skip image CI, 0 = run. Logs go to stderr.
-is_docs_only_change() {
-  local file_path
-  local has_any=0
-
-  while IFS= read -r file_path; do
-    [[ -z "${file_path}" ]] && continue
-    has_any=1
-
-    if [[ "${file_path}" == docs/* ]]; then
-      continue
-    fi
-    if [[ "${file_path}" == *.md ]]; then
-      continue
-    fi
-    if [[ "${file_path}" == "mkdocs.yaml" ]]; then
-      continue
-    fi
-    return 1
-  done
-
-  [[ "${has_any}" -eq 1 ]]
-}
-
-resolve_skip_ci() {
-  local is_pr_build=0
-  local files
-  local base_branch base_ref
-
-  if [[ "${BUILDKITE_PULL_REQUEST:-false}" != "false" && -n "${BUILDKITE_PULL_REQUEST:-}" ]]; then
-    is_pr_build=1
-  fi
-
-  if [[ "${is_pr_build}" -eq 1 ]]; then
-    base_branch="${BUILDKITE_PULL_REQUEST_BASE_BRANCH:-main}"
-    if ! git rev-parse --verify "origin/${base_branch}" >/dev/null 2>&1; then
-      echo "resolve_skip_ci: origin/${base_branch} not found locally; trying fetch" >&2
-      git fetch --depth=200 origin "${base_branch}" >/dev/null 2>&1 || true
-    fi
-
-    base_ref=""
-    if git rev-parse --verify "origin/${base_branch}" >/dev/null 2>&1; then
-      base_ref="origin/${base_branch}"
-    elif git rev-parse --verify "${base_branch}" >/dev/null 2>&1; then
-      base_ref="${base_branch}"
-    else
-      echo "resolve_skip_ci: cannot resolve PR base ${base_branch}; skip-ci=0" >&2
-      echo -n 0
-      return 0
-    fi
-
-    if ! files="$(git diff --name-only "${base_ref}...${BUILDKITE_COMMIT}" 2>/dev/null)"; then
-      echo "resolve_skip_ci: failed to compute PR changed files; skip-ci=0" >&2
-      echo -n 0
-      return 0
-    fi
-  elif [[ "${BUILDKITE_BRANCH:-}" == "main" ]]; then
-    if ! git rev-parse --verify "${BUILDKITE_COMMIT}^" >/dev/null 2>&1; then
-      echo "resolve_skip_ci: commit has no parent on main; skip-ci=0" >&2
-      echo -n 0
-      return 0
-    fi
-    if ! files="$(git diff --name-only "${BUILDKITE_COMMIT}^..${BUILDKITE_COMMIT}" 2>/dev/null)"; then
-      echo "resolve_skip_ci: failed to compute main changed files; skip-ci=0" >&2
-      echo -n 0
-      return 0
-    fi
-  else
-    echo "resolve_skip_ci: not PR/main build; skip-ci=0" >&2
-    echo -n 0
-    return 0
-  fi
-
-  if is_docs_only_change <<< "${files}"; then
-    echo "resolve_skip_ci: docs-only change detected; skip-ci=1" >&2
-    echo -n 1
-    return 0
-  fi
-
-  echo "resolve_skip_ci: non-doc changes detected; skip-ci=0" >&2
-  echo -n 0
-}
-
-SKIP_CI="$(resolve_skip_ci)"
-
-if [[ ! -f "${PIPELINE_YML}" ]]; then
-  echo "upload_pipeline_with_skip_ci: missing ${PIPELINE_YML}" >&2
-  exit 1
-fi
-
-export ROOT SKIP_CI PIPELINE_YML
-python3 <<'PY' | buildkite-agent pipeline upload
-import os
-import pathlib
-
-path = pathlib.Path(os.environ["PIPELINE_YML"])
-text = path.read_text(encoding="utf-8")
-sep = "\n---\n"
-if sep not in text:
-    raise SystemExit(
-        "upload_pipeline_with_skip_ci: .buildkite/pipeline.yml must contain a '\\n---\\n' separator "
-        "(document 1 = bootstrap, document 2 = uploaded steps)"
-    )
-_, continuation = text.split(sep, 1)
-
-skip = os.environ.get("SKIP_CI") == "1"
-# When docs-only skip-ci: skip default CI image, but still build for L4 nightly (PR label nightly-test or
-# main NIGHTLY=1), otherwise upload-nightly (depends_on image-build) would be skipped too.
-nightly_only = (
-    '(build.pull_request.labels includes "nightly-test") '
-    '|| (build.branch == "main" && build.env("NIGHTLY") == "1")'
-)
-# Placeholder in pipeline.yml is `if: __IMAGE_BUILD_IF__` (valid YAML); replace value only.
-if skip:
-    rep = f"'{nightly_only}'"
-    ready_rep = "'false'"
-    merge_rep = "'false'"
-else:
-    rep = "'true'"
-    ready_rep = "'build.branch != \"main\" && build.pull_request.labels includes \"ready\"'"
-    merge_rep = "'build.branch == \"main\" && build.env(\"NIGHTLY\") != \"1\"'"
-rendered = (
-    continuation
-    .replace("__IMAGE_BUILD_IF__", rep)
-    .replace("__UPLOAD_READY_IF__", ready_rep)
-    .replace("__UPLOAD_MERGE_IF__", merge_rep)
-)
-print(rendered, end="")
-PY

.buildkite/test-merge.yml

@@ -56,8 +56,8 @@ steps:
     timeout_in_minutes: 20
     depends_on: upload-merge-pipeline
     commands:
-      # Single pytest session for one combined summary at end of log.
+      - pytest -s -v tests/e2e/offline_inference/test_diffusion_cpu_offload.py
-      - pytest -s -v tests/e2e/offline_inference/test_diffusion_cpu_offload.py tests/e2e/offline_inference/test_diffusion_layerwise_offload.py
+      - pytest -s -v tests/e2e/offline_inference/test_diffusion_layerwise_offload.py
     agents:
       queue: "gpu_1_queue" # g6.4xlarge instance on AWS, has 1 L4 GPU
     plugins:
@@ -111,7 +111,8 @@ steps:
     timeout_in_minutes: 20
     depends_on: upload-merge-pipeline
     commands:
-      - pytest -s -v tests/e2e/offline_inference/test_sequence_parallel.py tests/diffusion/distributed/test_ulysses_uaa_perf.py
+      - pytest -s -v tests/e2e/offline_inference/test_sequence_parallel.py
+      - pytest -s -v tests/diffusion/distributed/test_ulysses_uaa_perf.py
     agents:
       queue: "gpu_4_queue" # g6.12xlarge instance on AWS, has 4 L4 GPU
     plugins:
@@ -173,7 +174,7 @@ steps:
                 pytest -s -v tests/engine/test_async_omni_engine_abort.py
         '
     agents:
-      queue: "gpu_1_queue" # g6.12xlarge instance on AWS, has 4 L4 GPU
+      queue: "gpu_4_queue" # g6.12xlarge instance on AWS, has 4 L4 GPU
     plugins:
       - docker#v5.2.0:
           image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
@@ -192,7 +193,8 @@ steps:
     commands:
       - export VLLM_LOGGING_LEVEL=DEBUG
       - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-      - pytest -s -v tests/e2e/offline_inference/test_qwen2_5_omni.py tests/e2e/online_serving/test_qwen2_5_omni.py -m "advanced_model" --run-level "advanced_model"
+      - pytest -s -v tests/e2e/offline_inference/test_qwen2_5_omni.py
+      - pytest -s -v tests/e2e/online_serving/test_qwen2_5_omni.py -m "advanced_model" --run-level "advanced_model"
     agents:
       queue: "gpu_4_queue" # g6.12xlarge instance on AWS, has 4 L4 GPU
     plugins:
@@ -214,7 +216,7 @@ steps:
           export VLLM_LOGGING_LEVEL=DEBUG
           export VLLM_WORKER_MULTIPROC_METHOD=spawn
           export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
-          pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py tests/e2e/offline_inference/test_qwen3_tts_customvoice.py -m "advanced_model" --run-level "advanced_model"
+          pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py -m "advanced_model" --run-level "advanced_model" && pytest -s -v tests/e2e/offline_inference/test_qwen3_tts_customvoice.py
         '
     agents:
       queue: "gpu_1_queue"
@@ -237,7 +239,7 @@ steps:
           export VLLM_LOGGING_LEVEL=DEBUG
           export VLLM_WORKER_MULTIPROC_METHOD=spawn
           export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
-          pytest -s -v tests/e2e/online_serving/test_qwen3_tts_base.py tests/e2e/offline_inference/test_qwen3_tts_base.py -m "advanced_model" --run-level "advanced_model"
+          pytest -s -v tests/e2e/online_serving/test_qwen3_tts_base.py -m "advanced_model" --run-level "advanced_model" && pytest -s -v tests/e2e/offline_inference/test_qwen3_tts_base.py
         '
     agents:
       queue: "gpu_1_queue"
@@ -257,8 +259,9 @@ steps:
     depends_on: upload-merge-pipeline
     commands:
       - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-      - export VLLM_TEST_CLEAN_GPU_MEMORY="1"
+      - pytest -s -v tests/e2e/offline_inference/test_qwen3_omni.py
-      - pytest -s -v tests/e2e/offline_inference/test_qwen3_omni.py tests/e2e/online_serving/test_qwen3_omni.py tests/e2e/online_serving/test_mimo_audio.py -m "advanced_model" --run-level "advanced_model"
+      - pytest -s -v tests/e2e/online_serving/test_qwen3_omni.py -m "advanced_model" --run-level "advanced_model"
+      - pytest -s -v tests/e2e/online_serving/test_mimo_audio.py -m "advanced_model" --run-level "advanced_model"
     agents:
       queue: "mithril-h100-pool"
     plugins:
@@ -344,7 +347,8 @@ steps:
           export VLLM_TEST_CLEAN_GPU_MEMORY=1
           export VLLM_IMAGE_FETCH_TIMEOUT=60
           pytest -s -v tests/e2e/offline_inference/test_bagel_text2img.py -m "advanced_model" --run-level "advanced_model" -k "shared_memory"
-          pytest -s -v tests/e2e/offline_inference/test_bagel_img2img.py tests/e2e/online_serving/test_bagel_online.py -m "advanced_model" --run-level "advanced_model"
+          pytest -s -v tests/e2e/offline_inference/test_bagel_img2img.py -m "advanced_model" --run-level "advanced_model"
+          pytest -s -v tests/e2e/online_serving/test_bagel_online.py -m "advanced_model" --run-level "advanced_model"
         '
     agents:
       queue: "mithril-h100-pool"
@@ -388,7 +392,8 @@ steps:
         timeout 20m bash -c '
           export VLLM_LOGGING_LEVEL=DEBUG
           export VLLM_WORKER_MULTIPROC_METHOD=spawn
-          pytest -s -v tests/e2e/online_serving/test_voxtral_tts.py tests/e2e/offline_inference/test_voxtral_tts.py -m "advanced_model" --run-level "advanced_model"
+          pytest -s -v tests/e2e/online_serving/test_voxtral_tts.py -m "advanced_model" --run-level "advanced_model"
+          pytest -s -v tests/e2e/offline_inference/test_voxtral_tts.py -m "advanced_model" --run-level "advanced_model"
         '
     agents:
       queue: "mithril-h100-pool"
@@ -408,11 +413,6 @@ steps:
                 env:
                   - name: HF_HOME
                     value: /root/.cache/huggingface
-                  - name: HF_TOKEN
-                    valueFrom:
-                      secretKeyRef:
-                        name: hf-token-secret
-                        key: token
             nodeSelector:
               node.kubernetes.io/instance-type: gpu-h100-sxm
             volumes:

.buildkite/test-nightly.yml

@@ -6,10 +6,16 @@ steps:
     commands:
       - export VLLM_WORKER_MULTIPROC_METHOD=spawn
       - |
-        pytest -s -v \
+        set +e
-          tests/examples/ \
+        pytest -s -v tests/e2e/online_serving/test_qwen3_omni_expansion.py -m "advanced_model" --run-level "advanced_model"
-          tests/e2e/online_serving/test_*_expansion.py \
+        EXIT1=$$?
-          -m "advanced_model and H100 and omni" --run-level "advanced_model"
+        pytest -s -v tests/examples/online_serving/test_qwen3_omni.py -m "advanced_model" --run-level "advanced_model"
+        EXIT2=$$?
+        pytest -s -v tests/e2e/online_serving/test_voxtral_tts.py -m "advanced_model" --run-level "advanced_model"
+        EXIT3=$$?
+        pytest -s -v tests/e2e/offline_inference/test_voxtral_tts.py -m "advanced_model" --run-level "advanced_model"
+        EXIT4=$$?
+        exit $$((EXIT1 | EXIT2 | EXIT3 | EXIT4))
     agents:
       queue: "mithril-h100-pool"
     plugins:
@@ -51,7 +57,8 @@ steps:
     commands:
       - export VLLM_WORKER_MULTIPROC_METHOD=spawn
       - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
-      - pytest -s -v tests/examples/ tests/e2e/online_serving/test_*_expansion.py -m "advanced_model and L4 and omni" --run-level "advanced_model"
+      - pytest -s -v tests/examples/ -m "advanced_model and L4 and omni" --run-level "advanced_model"
+      - pytest -s -v tests/e2e/online_serving/test_*_expansion.py -m "advanced_model and L4 and omni" --run-level "advanced_model"
     agents:
       queue: "gpu_4_queue" # g6.12xlarge instance on AWS, has 4 L4 GPU
     plugins:
@@ -72,7 +79,7 @@ steps:
     if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test"
     commands:
       - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-      - pytest -s -v tests/e2e/online_serving/test_*_expansion.py -k "not test_wan22_expansion and not test_wan_2_1_vace_expansion" -m "advanced_model and diffusion and H100" --run-level "advanced_model"
+      - pytest -s -v tests/e2e/online_serving/test_*_expansion.py -k "not test_wan22_expansion" -m "advanced_model and diffusion and H100" --run-level "advanced_model"
     agents:
       queue: "mithril-h100-pool"
     plugins:
@@ -107,13 +114,13 @@ steps:
                   path: /mnt/hf-cache
                   type: DirectoryOrCreate
 
-  - label: ":full_moon: Diffusion Model (Wan) Test with H100"
+  - label: ":full_moon: Diffusion Model Wan22 completed Test with H100"
     timeout_in_minutes: 90
     depends_on: upload-nightly-pipeline
     if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test"
     commands:
       - export VLLM_WORKER_MULTIPROC_METHOD=spawn
-      - pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py tests/e2e/online_serving/test_wan_2_1_vace_expansion.py -m "advanced_model" --run-level "advanced_model"
+      - pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py -m "advanced_model" --run-level "advanced_model"
     agents:
       queue: "mithril-h100-pool"
     plugins:
@@ -148,7 +155,7 @@ steps:
                   path: /mnt/hf-cache
                   type: DirectoryOrCreate
 
-  - label: ":full_moon: Diffusion Model Test"
+  - label: ":full_moon: Diffusion Model Test with L4"
     timeout_in_minutes: 60
     depends_on: upload-nightly-pipeline
     if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test"
@@ -170,7 +177,7 @@ steps:
             - "/fsx/hf_cache:/fsx/hf_cache"
 
 
-  - label: ":full_moon: Doc Example Code Test with H100"
+  - label: ":full_moon: Documentation Example Code Test with H100"
     timeout_in_minutes: 60
     depends_on: upload-nightly-pipeline
     if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test"
@@ -212,7 +219,7 @@ steps:
                   path: /mnt/hf-cache
                   type: DirectoryOrCreate
 
-  - label: ":full_moon: Omni Model Perf Test & Testcase Statistics with H100"
+  - label: ":full_moon: Omni Model Perf Test & Test Case Statistics"
     key: nightly-omni-performance
     timeout_in_minutes: 180
     depends_on: upload-nightly-pipeline
@@ -362,7 +369,7 @@ steps:
       - kubernetes:
           podSpec:
             containers:
-              - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
+              - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:c392ce21e9cf9ea65c52b866447793db10e0261c
                 resources:
                   limits:
                     nvidia.com/gpu: 2
@@ -390,9 +397,9 @@ steps:
                   path: /mnt/hf-cache
                   type: DirectoryOrCreate
 
-  - label: ":full_moon: Diffusion Perf Test with H100"
+  - label: ":full_moon: Qwen-Image Diffusion Perf Test with H100"
     key: nightly-qwen-image-performance
-    timeout_in_minutes: 180
+    timeout_in_minutes: 300
     depends_on: upload-nightly-pipeline
     if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test"
     commands:

.buildkite/test-ready.yml

@@ -180,7 +180,7 @@ steps:
                 pytest -s -v tests/engine/test_async_omni_engine_abort.py
         '
     agents:
-      queue: "gpu_1_queue" # g6.12xlarge instance on AWS, has 4 L4 GPU
+      queue: "gpu_4_queue" # g6.12xlarge instance on AWS, has 4 L4 GPU
     plugins:
       - docker#v5.2.0:
           image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
@@ -271,7 +271,7 @@ steps:
               - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
                 resources:
                   limits:
-                    nvidia.com/gpu: 1
+                    nvidia.com/gpu: 2
                 volumeMounts:
                   - name: devshm
                     mountPath: /dev/shm
@@ -320,29 +320,6 @@ steps:
           volumes:
             - "/fsx/hf_cache:/fsx/hf_cache"
 
-  - label: "OmniVoice E2E Test"
-    timeout_in_minutes: 20
-    depends_on: upload-ready-pipeline
-    commands:
-      - |
-        timeout 20m bash -c '
-          export VLLM_LOGGING_LEVEL=DEBUG
-          export VLLM_WORKER_MULTIPROC_METHOD=spawn
-          pytest -s -v tests/e2e/online_serving/test_omnivoice.py -m "core_model" --run-level "core_model"
-        '
-    agents:
-      queue: "gpu_1_queue"
-    plugins:
-      - docker#v5.2.0:
-          image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-          always-pull: true
-          propagate-environment: true
-          shm-size: "8gb"
-          environment:
-            - "HF_HOME=/fsx/hf_cache"
-          volumes:
-            - "/fsx/hf_cache:/fsx/hf_cache"
-
   - label: "Voxtral-TTS E2E Test"
     timeout_in_minutes: 20
     depends_on: upload-ready-pipeline
@@ -351,7 +328,8 @@ steps:
         timeout 20m bash -c '
           export VLLM_LOGGING_LEVEL=DEBUG
           export VLLM_WORKER_MULTIPROC_METHOD=spawn
-          pytest -s -v tests/e2e/online_serving/test_voxtral_tts.py -m "core_model" --run-level "core_model"
+          pytest -s -v tests/e2e/online_serving/test_voxtral_tts.py -m "advanced_model" --run-level "advanced_model"
+          pytest -s -v tests/e2e/offline_inference/test_voxtral_tts.py -m "advanced_model" --run-level "advanced_model"
         '
     agents:
       queue: "mithril-h100-pool"
@@ -548,46 +526,3 @@ steps:
                 hostPath:
                   path: /mnt/hf-cache
                   type: DirectoryOrCreate
-
-  - label: "CosyVoice3-TTS E2E Test"
-    timeout_in_minutes: 20
-    depends_on: upload-ready-pipeline
-    commands:
-      - |
-        timeout 20m bash -c '
-          export VLLM_WORKER_MULTIPROC_METHOD=spawn
-          pytest -s -v tests/e2e/online_serving/test_cosyvoice3_tts.py -m "core_model" --run-level "core_model"
-        '
-    agents:
-      queue: "mithril-h100-pool"
-    plugins:
-      - kubernetes:
-          podSpec:
-            containers:
-              - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
-                resources:
-                  limits:
-                    nvidia.com/gpu: 1
-                volumeMounts:
-                  - name: devshm
-                    mountPath: /dev/shm
-                  - name: hf-cache
-                    mountPath: /root/.cache/huggingface
-                env:
-                  - name: HF_HOME
-                    value: /root/.cache/huggingface
-                  - name: HF_TOKEN
-                    valueFrom:
-                      secretKeyRef:
-                        name: hf-token-secret
-                        key: token
-            nodeSelector:
-              node.kubernetes.io/instance-type: gpu-h100-sxm
-            volumes:
-              - name: devshm
-                emptyDir:
-                  medium: Memory
-              - name: hf-cache
-                hostPath:
-                  path: /mnt/hf-cache
-                  type: DirectoryOrCreate

.github/ISSUE_TEMPLATE/750-RFC.yml

@@ -7,7 +7,7 @@ body:
 - type: markdown
   attributes:
     value: >
-      #### Please take a look at previous [RFCs](https://github.com/vllm-project/vllm-omni/issues?q=in%3Atitle%20RFC%20sort%3Aupdated-desc) for reference.
+      #### Please take a look at previous [RFCs](https://github.com/vllm-project/vllm-omni/issues?q=label%3ARFC+sort%3Aupdated-desc) for reference.
 - type: textarea
   attributes:
     label: Motivation.
@@ -21,7 +21,7 @@ body:
     description: >
       The proposed change of the RFC.
     value: |
-      Please provide the detailed design document of the RFC using the [template](https://docs.google.com/document/d/1jcgR3cDaUQH3VczD4ZcKaJAoYWHjCmnYzHkCNyz-9fk/edit?usp=sharing).
+      Please provide the detailed design document of the RFC using the [template](https://docs.google.com/document/d/12YxSsVeD1jvL-InClkeAEnZyWFDndz_65JmXvsamuV4/edit?tab=t.0#heading=h.4ef4szrsgspp).
   validations:
     required: true
 - type: textarea

.gitignore

@@ -83,9 +83,6 @@ target/
 profile_default/
 ipython_config.py
 
-# uv
-uv.lock
-
 # pyenv
 #   For a library or package, you might want to ignore these files since the code is
 #   intended to run in multiple environments; otherwise, check them in:

benchmarks/qwen3-tts/README.md

@@ -32,9 +32,6 @@ bash run_benchmark.sh --hf-only
 # Use a different model (e.g. 1.7B)
 MODEL=Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice bash run_benchmark.sh --async-only
 
-# Use a Voice Clone model
-MODEL=Qwen/Qwen3-TTS-12Hz-1.7B-Base TASK_TYPE=Base bash run_benchmark.sh --async-only
-
 # Use bs16 config for higher throughput
 STAGE_CONFIG=vllm_omni/configs/qwen3_tts_bs16.yaml bash run_benchmark.sh --async-only
 

benchmarks/qwen3-tts/run_benchmark.sh

@@ -23,9 +23,6 @@
 #   # Use 1.7B model:
 #   MODEL=Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice bash run_benchmark.sh --async-only
 #
-#   # Use Voice Clone model
-#   MODEL=Qwen/Qwen3-TTS-12Hz-1.7B-Base TASK_TYPE=Base bash run_benchmark.sh --async-only
-#
 #   # Use batch_size=4 config:
 #   STAGE_CONFIG=vllm_omni/configs/qwen3_tts_bs4.yaml bash run_benchmark.sh --async-only
 #
@@ -38,7 +35,6 @@
 #   GPU_MEM_TALKER   - gpu_memory_utilization for talker stage (default: 0.3)
 #   GPU_MEM_CODE2WAV - gpu_memory_utilization for code2wav stage (default: 0.2)
 #   STAGE_CONFIG     - Path to stage config YAML (default: configs/qwen3_tts_bs1.yaml)
-#   TASK_TYPE        - Task type: CustomVoice, VoiceDesign, Base (default: CustomVoice)
 
 set -euo pipefail
 
@@ -57,7 +53,6 @@ NUM_WARMUPS="${NUM_WARMUPS:-3}"
 STAGE_CONFIG="${STAGE_CONFIG:-vllm_omni/configs/qwen3_tts_bs1.yaml}"
 RESULT_DIR="${SCRIPT_DIR}/results"
 TIMESTAMP="$(date +%Y%m%d_%H%M%S)"
-TASK_TYPE="${TASK_TYPE:-CustomVoice}"
 
 # Parse args
 RUN_ASYNC=true
@@ -82,7 +77,6 @@ echo " Concurrency:  ${CONCURRENCY}"
 echo " Port:         ${PORT}"
 echo " Stage config: ${STAGE_CONFIG}"
 echo " Results:      ${RESULT_DIR}"
-echo " Task type:    ${TASK_TYPE}"
 echo "============================================================"
 
 # Prepare stage config with correct GPU device and memory settings
@@ -201,8 +195,7 @@ run_bench() {
         --max-concurrency ${conc_args} \
         --num-warmups "${NUM_WARMUPS}" \
         --config-name "${config_name}" \
-        --result-dir "${RESULT_DIR}" \
+        --result-dir "${RESULT_DIR}"
-        --task-type "${TASK_TYPE}"
 
     stop_server
 
@@ -229,8 +222,7 @@ if [ "${RUN_HF}" = true ]; then
         --num-warmups "${NUM_WARMUPS}" \
         --gpu-device "${GPU_DEVICE}" \
         --config-name "hf_transformers" \
-        --result-dir "${RESULT_DIR}" \
+        --result-dir "${RESULT_DIR}"
-        --task-type "${TASK_TYPE}"
 
     # Allow GPU memory to settle
     sleep 5

benchmarks/qwen3-tts/transformers/bench_tts_hf.py

@@ -38,10 +38,6 @@ PROMPTS = [
     "It was a dark and stormy night when the old lighthouse keeper heard a knock at the door.",
 ]
 
-REF_AUDIO = "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen3-TTS-Repo/clone_2.wav"
-REF_TEXT = "Okay. Yeah. I resent you. I love you. I respect you. But you know what? You blew it! And thanks to you."
-INSTRUCT = "Speak in an incredulous tone, but with a hint of panic beginning to creep into your voice."
-
 
 @dataclass
 class BenchmarkResult:
@@ -79,29 +75,6 @@ class BenchmarkResult:
     per_request: list = field(default_factory=list)
 
 
-def generate_audio(model, prompt: str, args):
-    if args.task_type == "Base":
-        return model.generate_voice_clone(
-            text=prompt,
-            language=args.language,
-            ref_audio=REF_AUDIO,
-            ref_text=REF_TEXT,
-        )
-
-    if args.task_type == "VoiceDesign":
-        return model.generate_voice_design(
-            text=prompt,
-            language=args.language,
-            instruct=INSTRUCT,
-        )
-
-    return model.generate_custom_voice(
-        text=prompt,
-        language=args.language,
-        speaker=args.voice,
-    )
-
-
 def run_benchmark(args):
     from qwen_tts import Qwen3TTSModel
 
@@ -122,7 +95,11 @@ def run_benchmark(args):
         print(f"Warming up with {args.num_warmups} requests...")
         for i in range(args.num_warmups):
             p = PROMPTS[i % len(PROMPTS)]
-            wavs, sr = generate_audio(model, p, args)
+            wavs, sr = model.generate_custom_voice(
+                text=p,
+                language=args.language,
+                speaker=args.voice,
+            )
         # Sync GPU
         torch.cuda.synchronize(device)
         print("Warmup done.")
@@ -147,7 +124,11 @@ def run_benchmark(args):
             torch.cuda.synchronize(device)
             st = time.perf_counter()
 
-            wavs, sr = generate_audio(model, prompt, args)
+            wavs, sr = model.generate_custom_voice(
+                text=prompt,
+                language=args.language,
+                speaker=args.voice,
+            )
 
             torch.cuda.synchronize(device)
             elapsed = time.perf_counter() - st
@@ -287,7 +268,6 @@ def parse_args():
     parser.add_argument("--gpu-device", type=int, default=0)
     parser.add_argument("--voice", type=str, default="Vivian")
     parser.add_argument("--language", type=str, default="English")
-    parser.add_argument("--task-type", type=str, default="CustomVoice", choices=["CustomVoice", "VoiceDesign", "Base"])
     parser.add_argument(
         "--config-name", type=str, default="hf_transformers", help="Label for this config (used in filenames)"
     )

benchmarks/qwen3-tts/vllm_omni/bench_tts_serve.py

@@ -37,9 +37,6 @@ PROMPTS = [
     "Could you please turn down the music a little bit, I'm trying to concentrate on my work.",
     "It was a dark and stormy night when the old lighthouse keeper heard a knock at the door.",
 ]
-REF_AUDIO = "https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen3-TTS-Repo/clone_2.wav"
-REF_TEXT = "Okay. Yeah. I resent you. I love you. I respect you. But you know what? You blew it! And thanks to you."
-INSTRUCT = "Speak in an incredulous tone, but with a hint of panic beginning to creep into your voice."
 
 
 @dataclass
@@ -96,39 +93,22 @@ def pcm_bytes_to_duration(num_bytes: int, sample_rate: int = 24000, sample_width
     return num_samples / sample_rate
 
 
-def create_payload(
-    prompt: str, task_type: str = "CustomVoice", voice: str = "vivian", language: str = "English"
-) -> dict:
-    payload = {
-        "input": prompt,
-        "language": language,
-        "stream": True,
-        "response_format": "pcm",
-        "task_type": task_type,
-    }
-
-    if task_type == "Base":
-        payload["ref_audio"] = REF_AUDIO
-        payload["ref_text"] = REF_TEXT
-    elif task_type == "CustomVoice":
-        payload["voice"] = voice
-    elif task_type == "VoiceDesign":
-        payload["instructions"] = INSTRUCT
-
-    return payload
-
-
 async def send_tts_request(
     session: aiohttp.ClientSession,
     api_url: str,
     prompt: str,
-    task_type: str = "CustomVoice",
     voice: str = "vivian",
     language: str = "English",
     pbar: tqdm | None = None,
 ) -> RequestResult:
     """Send a streaming TTS request and measure latency metrics."""
-    payload = create_payload(prompt, task_type, voice, language)
+    payload = {
+        "input": prompt,
+        "voice": voice,
+        "language": language,
+        "stream": True,
+        "response_format": "pcm",
+    }
 
     result = RequestResult(prompt=prompt)
     st = time.perf_counter()
@@ -173,7 +153,6 @@ async def run_benchmark(
     num_prompts: int,
     max_concurrency: int,
     num_warmups: int = 3,
-    task_type: str = "CustomVoice",
     voice: str = "vivian",
     language: str = "English",
 ) -> BenchmarkResult:
@@ -196,7 +175,7 @@ async def run_benchmark(
         warmup_tasks = []
         for i in range(num_warmups):
             prompt = PROMPTS[i % len(PROMPTS)]
-            warmup_tasks.append(send_tts_request(session, api_url, prompt, task_type, voice, language))
+            warmup_tasks.append(send_tts_request(session, api_url, prompt, voice, language))
         await asyncio.gather(*warmup_tasks)
         print("  Warmup done.")
 
@@ -210,7 +189,7 @@ async def run_benchmark(
 
     async def limited_request(prompt):
         async with semaphore:
-            return await send_tts_request(session, api_url, prompt, task_type, voice, language, pbar)
+            return await send_tts_request(session, api_url, prompt, voice, language, pbar)
 
     start_time = time.perf_counter()
     tasks = [asyncio.create_task(limited_request(p)) for p in request_prompts]
@@ -327,7 +306,6 @@ async def main(args):
             num_prompts=args.num_prompts,
             max_concurrency=concurrency,
             num_warmups=args.num_warmups,
-            task_type=args.task_type,
             voice=args.voice,
             language=args.language,
         )
@@ -356,7 +334,6 @@ def parse_args():
         "--max-concurrency", type=int, nargs="+", default=[1, 4, 10], help="Concurrency levels to test"
     )
     parser.add_argument("--num-warmups", type=int, default=3)
-    parser.add_argument("--task-type", type=str, default="CustomVoice", choices=["CustomVoice", "VoiceDesign", "Base"])
     parser.add_argument("--voice", type=str, default="vivian")
     parser.add_argument("--language", type=str, default="English")
     parser.add_argument(

docker/Dockerfile.ci

@@ -1,5 +1,5 @@
 ARG VLLM_BASE_IMAGE=vllm/vllm-openai
-ARG VLLM_BASE_TAG=v0.19.0
+ARG VLLM_BASE_TAG=v0.18.0
 FROM ${VLLM_BASE_IMAGE}:${VLLM_BASE_TAG}
 ARG APP_DIR=/workspace/vllm-omni
 WORKDIR ${APP_DIR}
@@ -11,7 +11,9 @@ RUN apt-get update && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/*
 
-RUN uv pip install --system ".[dev]"
+# Install vllm-omni into the same uv-managed Python environment used by the base image.
+# Use bash -c so that $(python3 -c ...) is expanded inside the container.
+RUN uv pip install --system --no-cache-dir ".[dev]"
 
 RUN ln -sf /usr/bin/python3 /usr/bin/python
 

docker/Dockerfile.cuda

@@ -1,22 +0,0 @@
-ARG BASE_IMAGE=vllm/vllm-openai:v0.19.0
-FROM ${BASE_IMAGE}
-
-ARG COMMON_WORKDIR=/app
-
-WORKDIR ${COMMON_WORKDIR}
-
-# Step 1: Setup - Install system dependencies
-RUN apt-get update && \
-    apt-get install -y ffmpeg git sox libsox-fmt-all jq && \
-    apt-get clean && \
-    rm -rf /var/lib/apt/lists/*
-
-RUN mkdir -p ${COMMON_WORKDIR}/vllm-omni
-
-# Step 2: Copy vllm-omni code and install
-COPY . ${COMMON_WORKDIR}/vllm-omni
-RUN cd ${COMMON_WORKDIR}/vllm-omni && uv pip install --python "$(python3 -c 'import sys; print(sys.executable)')" --no-cache-dir "."
-
-RUN ln -sf /usr/bin/python3 /usr/bin/python
-
-ENTRYPOINT []

docker/Dockerfile.npu

@@ -1,20 +1,20 @@
 ARG VLLM_ASCEND_IMAGE=quay.io/ascend/vllm-ascend
-ARG VLLM_ASCEND_TAG=v0.18.0rc1
+ARG VLLM_ASCEND_TAG=v0.17.0rc1
 FROM ${VLLM_ASCEND_IMAGE}:${VLLM_ASCEND_TAG}
 
-# WORKDIR /vllm-workspace/vllm
+WORKDIR /vllm-workspace/vllm
-# RUN git fetch origin --tags && git checkout v0.18.0
+RUN git fetch origin --tags && git checkout v0.18.0
 
-# WORKDIR /vllm-workspace/vllm-ascend
+WORKDIR /vllm-workspace/vllm-ascend
-# RUN git fetch origin releases/v0.18.0 && git checkout d781902ce9dbda8ab1e11bb0f2f0c1bc508fee7a
+RUN git fetch origin releases/v0.18.0 && git checkout d781902ce9dbda8ab1e11bb0f2f0c1bc508fee7a
-# # Install vllm-ascend
+# Install vllm-ascend
-# # Append `libascend_hal.so` path (devlib) to LD_LIBRARY_PATH
+# Append `libascend_hal.so` path (devlib) to LD_LIBRARY_PATH
-# RUN export PIP_EXTRA_INDEX_URL=https://mirrors.huaweicloud.com/ascend/repos/pypi && \
+RUN export PIP_EXTRA_INDEX_URL=https://mirrors.huaweicloud.com/ascend/repos/pypi && \
-#     source /usr/local/Ascend/ascend-toolkit/set_env.sh && \
+    source /usr/local/Ascend/ascend-toolkit/set_env.sh && \
-#     source /usr/local/Ascend/nnal/atb/set_env.sh && \
+    source /usr/local/Ascend/nnal/atb/set_env.sh && \
-#     export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/Ascend/ascend-toolkit/latest/`uname -i`-linux/devlib && \
+    export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/Ascend/ascend-toolkit/latest/`uname -i`-linux/devlib && \
-#     python3 -m pip install -v -e /vllm-workspace/vllm-ascend/ --extra-index https://download.pytorch.org/whl/cpu/ && \
+    python3 -m pip install -v -e /vllm-workspace/vllm-ascend/ --extra-index https://download.pytorch.org/whl/cpu/ && \
-#     python3 -m pip cache purge
+    python3 -m pip cache purge
 
 ARG APP_DIR=/vllm-workspace/vllm-omni
 WORKDIR ${APP_DIR}

docker/Dockerfile.npu.a3

@@ -1,20 +1,20 @@
 ARG VLLM_ASCEND_IMAGE=quay.io/ascend/vllm-ascend
-ARG VLLM_ASCEND_TAG=v0.18.0rc1-a3
+ARG VLLM_ASCEND_TAG=v0.17.0rc1-a3
 FROM ${VLLM_ASCEND_IMAGE}:${VLLM_ASCEND_TAG}
 
-# WORKDIR /vllm-workspace/vllm
+WORKDIR /vllm-workspace/vllm
-# RUN git fetch origin --tags && git checkout v0.18.0
+RUN git fetch origin --tags && git checkout v0.18.0
 
-# WORKDIR /vllm-workspace/vllm-ascend
+WORKDIR /vllm-workspace/vllm-ascend
-# RUN git fetch origin releases/v0.18.0 && git checkout d781902ce9dbda8ab1e11bb0f2f0c1bc508fee7a
+RUN git fetch origin releases/v0.18.0 && git checkout d781902ce9dbda8ab1e11bb0f2f0c1bc508fee7a
-# # Install vllm-ascend
+# Install vllm-ascend
-# # Append `libascend_hal.so` path (devlib) to LD_LIBRARY_PATH
+# Append `libascend_hal.so` path (devlib) to LD_LIBRARY_PATH
-# RUN export PIP_EXTRA_INDEX_URL=https://mirrors.huaweicloud.com/ascend/repos/pypi && \
+RUN export PIP_EXTRA_INDEX_URL=https://mirrors.huaweicloud.com/ascend/repos/pypi && \
-#     source /usr/local/Ascend/ascend-toolkit/set_env.sh && \
+    source /usr/local/Ascend/ascend-toolkit/set_env.sh && \
-#     source /usr/local/Ascend/nnal/atb/set_env.sh && \
+    source /usr/local/Ascend/nnal/atb/set_env.sh && \
-#     export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/Ascend/ascend-toolkit/latest/`uname -i`-linux/devlib && \
+    export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/Ascend/ascend-toolkit/latest/`uname -i`-linux/devlib && \
-#     python3 -m pip install -v -e /vllm-workspace/vllm-ascend/ --extra-index https://download.pytorch.org/whl/cpu/ && \
+    python3 -m pip install -v -e /vllm-workspace/vllm-ascend/ --extra-index https://download.pytorch.org/whl/cpu/ && \
-#     python3 -m pip cache purge
+    python3 -m pip cache purge
 
 ARG APP_DIR=/vllm-workspace/vllm-omni
 WORKDIR ${APP_DIR}

docker/Dockerfile.rocm

@@ -1,4 +1,4 @@
-ARG BASE_IMAGE=vllm/vllm-openai-rocm:v0.19.0
+ARG BASE_IMAGE=vllm/vllm-openai-rocm:v0.18.0
 FROM ${BASE_IMAGE} AS base
 
 # Declare a variable to know if we want to use the nightly build or the stable build.

docker/Dockerfile.xpu

@@ -76,7 +76,7 @@ ENV UV_EXTRA_INDEX_URL=${PIP_EXTRA_INDEX_URL}
 ENV UV_INDEX_STRATEGY="unsafe-best-match"
 ENV UV_LINK_MODE="copy"
 
-ARG VLLM_VERSION=v0.19.0
+ARG VLLM_VERSION=v0.18.0
 RUN git clone -b ${VLLM_VERSION} https://github.com/vllm-project/vllm
 WORKDIR /workspace/vllm
 

docs/api/README.md

@@ -5,6 +5,7 @@
 Main entry points for vLLM-Omni inference and serving.
 
 - [vllm_omni.entrypoints.async_omni.AsyncOmni][]
+- [vllm_omni.entrypoints.async_omni_diffusion.AsyncOmniDiffusion][]
 - [vllm_omni.entrypoints.cfg_companion_tracker.CfgCompanionTracker][]
 - [vllm_omni.entrypoints.cli.benchmark.base.OmniBenchmarkSubcommandBase][]
 - [vllm_omni.entrypoints.cli.benchmark.main.OmniBenchmarkSubcommand][]

docs/contributing/ci/CI_5levels.md

@@ -168,6 +168,7 @@ vllm_omni/                                    tests/
 │   └── arg_utils.py                            │   └── test_arg_utils.py               ⬜
 │
 ├── entrypoints/                        →     ├── entrypoints/
+│   ├── async_omni_diffusion.py                 │   ├── test_async_omni_diffusion_config.py  ✅
 │   ├── stage_utils.py                          │   ├── test_stage_utils.py            ✅
 │   ├── cli/                                     │   ├── cli/                           (benchmarks/test_serve_cli.py covers CLI serve)
 │   │   └── ...                                  │   │   └── test_*.py                  ⬜

docs/contributing/ci/tests_style.md

@@ -73,6 +73,7 @@ vllm_omni/                                    tests/
 │   └── arg_utils.py                            │   └── test_arg_utils.py               ⬜
 │
 ├── entrypoints/                        →     ├── entrypoints/
+│   ├── async_omni_diffusion.py                 │   ├── test_async_omni_diffusion_config.py  ✅
 │   ├── stage_utils.py                          │   ├── test_stage_utils.py            ✅
 │   ├── cli/                                     │   ├── cli/                           (benchmarks/test_serve_cli.py covers CLI serve)
 │   │   └── ...                                  │   │   └── test_*.py                  ⬜

docs/contributing/model/adding_diffusion_model.md

@@ -802,7 +802,7 @@ omni = Omni(model="your-model", enable_layerwise_offload=True)
 
 ```python
 class WanTransformer3DModel(nn.Module):
-    _layerwise_offload_blocks_attrs = ["blocks"]  # Attribute name containing transformer blocks
+    _layerwise_offload_blocks_attr = "blocks"  # Attribute name containing transformer blocks
 
     def __init__(self):
         self.blocks = nn.ModuleList([...])  # Transformer blocks

docs/design/feature/vae_parallel.md

@@ -1,15 +1,14 @@
 # VAE Patch Parallelism
 
 This document describes how to add **VAE Patch Parallelism** support to a diffusion model.
-We use **Qwen-Image** as the reference implementation for decode parallel, and **Wan2.2** for encode parallel.
+We use **Qwen-Image** as the reference implementation.
 
 ---
 
 ## Table of Contents
 
 - [Overview](#overview)
-- [Step-by-Step Implementation (Decode)](#step-by-step-implementation-decode)
+- [Step-by-Step Implementation](#step-by-step-implementation)
-- [Encode Parallel Implementation](#encode-parallel-implementation)
 - [Testing](#testing)
 - [Reference Implementations](#reference-implementations)
 - [Summary](#summary)
@@ -20,13 +19,13 @@ We use **Qwen-Image** as the reference implementation for decode parallel, and *
 
 ### What is Vae Patch parallel?
 
-**VAE Patch Parallelism** is an acceleration technique for both **encoding** and **decoding**. Instead of processing the entire tensor at once, the tensor is:
+**VAE Patch Parallelism** is a decoding acceleration technique. Instead of decoding the entire latent tensor at once, the latent tensor is:
 
 + Split into multiple spatial tiles
 
 + Distributed across multiple ranks
 
-+ Encoded/Decoded in parallel
++ Decoded in parallel
 
 + Merged to reconstruct the final output
 
@@ -36,17 +35,10 @@ This approach:
 
 + Reduces peak memory usage per device
 
-+ Accelerates encoding/decoding latency
++ Accelerates decoding latency
-
-### When to Use Encode vs Decode Parallel
-
-| Operation | Use Case | Example |
-|-----------|----------|---------|
-| **Decode Parallel** | Text-to-Image, Text-to-Video | Latent → Image/Video |
-| **Encode Parallel** | Image-to-Video (I2V) | Image → Latent (for conditioning) |
 
 ### Architecture
-We introduce **DistributedVaeExecutor** as the core component responsible for distributed VAE encoding/decoding.
+We introduce **DistributedVaeExecutor** as the core component responsible for distributed VAE decoding.
 
 The executor is model-agnostic and accepts three function parameters:
 
@@ -92,7 +84,7 @@ Therefore:
 
 + Merge must perform blending to avoid seams
 
-## Step-by-Step Implementation (Decode)
+## Step-by-Step Implementation
 
 ### Step 1: Implement DistributedAutoencoderKLQwenImage
 `QwenImagePipeline` use `AutoencoderKLQwenImage` for vae, so implement a distributed version:
@@ -213,14 +205,14 @@ def tile_merge(self, coord_tensor_map: dict[tuple[int, ...], torch.Tensor], grid
 We need to override tiled_decode, the main logic is:
 + check distributed is enabled
 + select split/exec/merge
-+ Invoke self.distributed_executor.execute to decode
++ Invoke self.distributed_decoder.execute to decode
 ```
 def tiled_decode(self, z: torch.Tensor, return_dict: bool = True):
     if not self.is_distributed_enabled():
         return super().tiled_decode(z, return_dict=return_dict)
 
     logger.info("Decode run with distributed executor")
-    result = self.distributed_executor.execute(
+    result = self.distributed_decoder.execute(
         z,
         DistributedOperator(split=self.tile_split, exec=self.tile_exec, merge=self.tile_merge),
         broadcast_result=True,
@@ -251,166 +243,6 @@ class YourModelPipeline(nn.Module):
 +       ).to(self.device)
 ```
 
-## Encode Parallel Implementation
-
-For models that require VAE encoding (e.g., Image-to-Video), you can also parallelize the encode operation. We use **Wan2.2** as the reference implementation.
-
-### Step 1: Implement encode_tile_split
-
-Similar to decode, split the input tensor into tiles. Key considerations:
-
-+ **Patchify handling**: If the model uses `patch_size`, scale tile parameters accordingly
-+ **Temporal chunking**: Video VAEs may have temporal compression (e.g., 4x)
-
-```python
-def encode_tile_split(self, x: torch.Tensor) -> tuple[list[TileTask], GridSpec]:
-    _, _, num_frames, height, width = x.shape
-    encode_spatial_compression_ratio = self.spatial_compression_ratio
-
-    # Scale tile parameters for patchified coordinate system
-    tile_sample_min_height = self.tile_sample_min_height
-    tile_sample_min_width = self.tile_sample_min_width
-    tile_sample_stride_height = self.tile_sample_stride_height
-    tile_sample_stride_width = self.tile_sample_stride_width
-
-    if self.config.patch_size is not None:
-        # When input is patchified, scale tile parameters accordingly
-        encode_spatial_compression_ratio = self.spatial_compression_ratio // self.config.patch_size
-        tile_sample_min_height = tile_sample_min_height // self.config.patch_size
-        tile_sample_min_width = tile_sample_min_width // self.config.patch_size
-        tile_sample_stride_height = tile_sample_stride_height // self.config.patch_size
-        tile_sample_stride_width = tile_sample_stride_width // self.config.patch_size
-
-    latent_height = height // encode_spatial_compression_ratio
-    latent_width = width // encode_spatial_compression_ratio
-
-    tile_latent_min_height = tile_sample_min_height // encode_spatial_compression_ratio
-    tile_latent_min_width = tile_sample_min_width // encode_spatial_compression_ratio
-    tile_latent_stride_height = tile_sample_stride_height // encode_spatial_compression_ratio
-    tile_latent_stride_width = tile_sample_stride_width // encode_spatial_compression_ratio
-
-    blend_height = tile_latent_min_height - tile_latent_stride_height
-    blend_width = tile_latent_min_width - tile_latent_stride_width
-
-    tiletask_list = []
-    # Use temporal compression ratio from config instead of hardcoding
-    temporal_compression = self.config.scale_factor_temporal
-
-    for i in range(0, height, tile_sample_stride_height):
-        for j in range(0, width, tile_sample_stride_width):
-            time_list = []
-            frame_range = 1 + (num_frames - 1) // temporal_compression
-            for k in range(frame_range):
-                if k == 0:
-                    tile = x[:, :, :1, i : i + tile_sample_min_height, j : j + tile_sample_min_width]
-                else:
-                    tile = x[
-                        :, :,
-                        1 + temporal_compression * (k - 1) : 1 + temporal_compression * k,
-                        i : i + tile_sample_min_height,
-                        j : j + tile_sample_min_width,
-                    ]
-                time_list.append(tile)
-            tiletask_list.append(
-                TileTask(len(tiletask_list), (i // tile_sample_stride_height, j // tile_sample_stride_width),
-                         time_list, workload=time_list[0].shape[3] * time_list[0].shape[4])
-            )
-
-    grid_spec = GridSpec(
-        split_dims=(3, 4),
-        grid_shape=(tiletask_list[-1].grid_coord[0] + 1, tiletask_list[-1].grid_coord[1] + 1),
-        tile_spec={
-            "latent_height": latent_height, "latent_width": latent_width,
-            "blend_height": blend_height, "blend_width": blend_width,
-            "tile_latent_stride_height": tile_latent_stride_height,
-            "tile_latent_stride_width": tile_latent_stride_width,
-        },
-        output_dtype=self.dtype,
-    )
-    return tiletask_list, grid_spec
-```
-
-### Step 2: Implement encode_tile_exec
-
-```python
-def encode_tile_exec(self, task: TileTask) -> torch.Tensor:
-    """Encode a single sample tile into latent space."""
-    self.clear_cache()
-    time = []
-    for k, tile in enumerate(task.tensor):
-        self._enc_conv_idx = [0]
-        encoded = self.encoder(tile, feat_cache=self._enc_feat_map, feat_idx=self._enc_conv_idx)
-        encoded = self.quant_conv(encoded)
-        time.append(encoded)
-    result = torch.cat(time, dim=2)
-    self.clear_cache()
-    return result
-```
-
-### Step 3: Implement encode_tile_merge
-
-```python
-def encode_tile_merge(
-    self, coord_tensor_map: dict[tuple[int, ...], torch.Tensor], grid_spec: GridSpec
-) -> torch.Tensor:
-    """Merge encoded tiles into a full latent tensor."""
-    grid_h, grid_w = grid_spec.grid_shape
-    result_rows = []
-    for i in range(grid_h):
-        result_row = []
-        for j in range(grid_w):
-            tile = coord_tensor_map[(i, j)]
-            if i > 0:
-                tile = self.blend_v(coord_tensor_map[(i - 1, j)], tile, grid_spec.tile_spec["blend_height"])
-            if j > 0:
-                tile = self.blend_h(coord_tensor_map[(i, j - 1)], tile, grid_spec.tile_spec["blend_width"])
-            result_row.append(tile[:, :, :,
-                : grid_spec.tile_spec["tile_latent_stride_height"],
-                : grid_spec.tile_spec["tile_latent_stride_width"]])
-        result_rows.append(torch.cat(result_row, dim=-1))
-
-    enc = torch.cat(result_rows, dim=3)[
-        :, :, :, : grid_spec.tile_spec["latent_height"], : grid_spec.tile_spec["latent_width"]
-    ]
-    return enc
-```
-
-### Step 4: Override tiled_encode method
-
-Override `tiled_encode` instead of `encode`. The parent's `_encode()` handles patchify before calling `tiled_encode()`, so input `x` is already patchified.
-
-```python
-def tiled_encode(self, x: torch.Tensor) -> torch.Tensor:
-    """
-    Encode using distributed VAE executor.
-
-    Note: x is already patchified by parent's _encode() before calling this method.
-    """
-    if not self.is_distributed_enabled():
-        return super().tiled_encode(x)
-
-    self.clear_cache()
-    result = self.distributed_executor.execute(
-        x,
-        DistributedOperator(
-            split=self.encode_tile_split,
-            exec=self.encode_tile_exec,
-            merge=self.encode_tile_merge,
-        ),
-        broadcast_result=True,  # Latents needed by all ranks for diffusion
-    )
-    self.clear_cache()
-    return result
-```
-
-**Key differences from decode parallel:**
-
-| Aspect | Decode Parallel | Encode Parallel |
-|--------|-----------------|-----------------|
-| `broadcast_result` | Often `False` (only rank 0 needs output) | `True` (all ranks need latents for diffusion) |
-| Patchify | Applied in merge (unpatchify) | Handled by parent `_encode()` before `tiled_encode()` |
-| Temporal chunking | Frame-by-frame | Chunk-based (e.g., 1 + 4n frames) |
-
 ## Testing
 Verify numerical consistency between:
 + vae_patch_parallel_size = 1
@@ -440,20 +272,18 @@ When vae_patch_parallel_size is larger than the DiT world size, it will automati
 
 Complete examples in the codebase:
 
-| Model | Path | Decode Parallel | Encode Parallel |
+| Model | Path | Notes |
-|-------|------|-----------------|-----------------|
+|-------|------|-------|
-| **Z-Image** | `vllm_omni/diffusion/distributed/autoencoders/autoencoder_kl.py` | ✅ | ❌ |
+| **Z-Image** | `vllm_omni/diffusion/distributed/autoencoders/autoencoder_kl.py` | Distributed AutoencoderKL |
-| **Wan2.2** | `vllm_omni/diffusion/distributed/autoencoders/autoencoder_kl_wan.py` | ✅ | ✅ |
+| **Wan2.2** | `vllm_omni/diffusion/distributed/autoencoders/autoencoder_kl_wan.py` | Distributed AutoencoderKLWan |
-| **Qwen-Image** | `vllm_omni/diffusion/distributed/autoencoders/autoencoder_kl_qwenimage.py` | ✅ | ❌ |
+| **Qwen-Image** | `vllm_omni/diffusion/distributed/autoencoders/autoencoder_kl_qwenimage.py` | Distributed AutoencoderKLQwenImage |
 
 ---
 
 ## Summary
 
-Adding VAE Patch Parallel support to diffusion model:
+Adding Vae Patch Parallel support to diffusion model:
 
-1. **Implement Distributed VAE** - Inherit from base VAE class and `DistributedVaeMixin`
+1. **Implement Distributed Vae** - mainly copy from `diffusers` tiled_decode, and refactor into split/exec/merge
-2. **Decode Parallel** - Refactor `tiled_decode` into `tile_split`/`tile_exec`/`tile_merge`
+2. **Change vae model in pipeline to Distributed Vae**
-3. **Encode Parallel** (optional) - Implement `encode_tile_split`/`encode_tile_exec`/`encode_tile_merge` for I2V models
+3. **Test** - Verify with `tensor_parallel_size=N` quality
-4. **Change VAE model in pipeline** - Use the distributed version
-5. **Test** - Verify numerical consistency with `vae_patch_parallel_size=1` vs `N`

docs/design/module/async_omni_architecture.md

@@ -69,7 +69,7 @@
 [5] Orchestrator._orchestration_loop (loop)
     -> poll stage output
        - llm stage: await get_output_async()
-       - diffusion stage: get_diffusion_output_nowait()
+       - diffusion stage: get_diffusion_output_async()
     -> (llm stage) output_processors[i].process_outputs(...)
     -> _route_output(...)
     -> if finished and not final_stage and non-async-chunk:
@@ -112,7 +112,7 @@ sequenceDiagram
     ORCH->>S0: add_request_async
 
     loop poll route forward
-        ORCH->>S0: get_output_async / get_diffusion_output_nowait
+        ORCH->>S0: get_output_async / get_diffusion_output_async
         ORCH->>ORCH: _route_output
         alt need forward to next stage
             ORCH->>SN: add_request_async

docs/getting_started/installation/README.md

@@ -6,5 +6,4 @@ vLLM-Omni supports the following hardware platforms:
     - [NVIDIA CUDA](gpu.md)
     - [AMD ROCm](gpu.md)
     - [Intel XPU](gpu.md)
-    - [MThreads MUSA](gpu.md)
 - [NPU](npu.md)

docs/getting_started/installation/gpu.md

@@ -22,10 +22,6 @@ vLLM-Omni is a Python library that supports the following GPU variants. The libr
 
     --8<-- "docs/getting_started/installation/gpu/xpu.inc.md:requirements"
 
-=== "MThreads MUSA"
-
-    --8<-- "docs/getting_started/installation/gpu/musa.inc.md:requirements"
-
 ## Set up using Python
 
 ### Create a new Python environment
@@ -48,10 +44,6 @@ Note: Pre-built wheels are currently available for vLLM-Omni 0.11.0rc1, 0.12.0rc
 
     --8<-- "docs/getting_started/installation/gpu/xpu.inc.md:pre-built-wheels"
 
-=== "MThreads MUSA"
-
-    --8<-- "docs/getting_started/installation/gpu/musa.inc.md:pre-built-wheels"
-
 [](){ #build-from-source }
 
 ### Build wheel from source
@@ -68,10 +60,6 @@ Note: Pre-built wheels are currently available for vLLM-Omni 0.11.0rc1, 0.12.0rc
 
     --8<-- "docs/getting_started/installation/gpu/xpu.inc.md:build-wheel-from-source"
 
-=== "MThreads MUSA"
-
-    --8<-- "docs/getting_started/installation/gpu/musa.inc.md:build-wheel-from-source"
-
 ## Set up using Docker
 
 ### Pre-built images
@@ -88,10 +76,6 @@ Note: Pre-built wheels are currently available for vLLM-Omni 0.11.0rc1, 0.12.0rc
 
     --8<-- "docs/getting_started/installation/gpu/xpu.inc.md:pre-built-images"
 
-=== "MThreads MUSA"
-
-    --8<-- "docs/getting_started/installation/gpu/musa.inc.md:pre-built-images"
-
 ### Build your own docker image
 
 === "AMD ROCm"
@@ -101,7 +85,3 @@ Note: Pre-built wheels are currently available for vLLM-Omni 0.11.0rc1, 0.12.0rc
 === "Intel XPU"
 
     --8<-- "docs/getting_started/installation/gpu/xpu.inc.md:build-docker"
-
-=== "MThreads MUSA"
-
-    --8<-- "docs/getting_started/installation/gpu/musa.inc.md:build-docker"

docs/getting_started/installation/gpu/cuda.inc.md

@@ -39,13 +39,13 @@ uv pip install 'vllm-omni[demo]'
 # --8<-- [start:build-wheel-from-source]
 
 #### Installation of vLLM
-If you do not need to modify source code of vLLM, you can directly install the stable 0.19.0 release version of the library
+If you do not need to modify source code of vLLM, you can directly install the stable 0.18.0 release version of the library
 
 ```bash
-uv pip install vllm==0.19.0 --torch-backend=auto
+uv pip install vllm==0.18.0 --torch-backend=auto
 ```
 
-The 0.19.0 release of vLLM ships CUDA 13.0-compatible binaries by default. If you need a different CUDA variant or want to reuse an existing PyTorch installation, build vLLM from source instead.
+The 0.18.0 release of vLLM ships CUDA 12.9-compatible binaries by default. If you need a different CUDA variant or want to reuse an existing PyTorch installation, build vLLM from source instead.
 
 #### Installation of vLLM-Omni
 Since vllm-omni is rapidly evolving, it's recommended to install it from source
@@ -66,12 +66,11 @@ If you want to check, modify or debug with source code of vLLM, install the libr
 ```bash
 git clone https://github.com/vllm-project/vllm.git
 cd vllm
-git checkout v0.19.0
+git checkout v0.18.0
 ```
 Set up environment variables to get pre-built wheels. If there are internet problems, just download the whl file manually. And set `VLLM_PRECOMPILED_WHEEL_LOCATION` as your local absolute path of whl file.
 ```bash
-#For CUDA 13.0
+export VLLM_PRECOMPILED_WHEEL_LOCATION=https://github.com/vllm-project/vllm/releases/download/v0.18.0/vllm-0.18.0+cu129-cp38-abi3-manylinux_2_35_x86_64.whl
-export VLLM_PRECOMPILED_WHEEL_LOCATION=https://github.com/vllm-project/vllm/releases/download/v0.19.0/vllm-0.19.0+cu130-cp38-abi3-manylinux_2_35_x86_64.whl
 ```
 Install vllm with command below (If you have no existing PyTorch).
 ```bash

docs/getting_started/installation/gpu/musa.inc.md

@@ -1,65 +0,0 @@
-# --8<-- [start:requirements]
-
-- GPU: Moore Threads GPU with MUSA SDK installed (validated on MTT S5000)
-
-# --8<-- [end:requirements]
-# --8<-- [start:set-up-using-python]
-
-vLLM-Omni for MUSA requires building from source. Pre-built wheels are not currently available.
-
-!!! note
-    MUSA platform requires vLLM-MUSA to be installed first.
-
-# --8<-- [start:pre-built-wheels]
-
-# --8<-- [end:pre-built-wheels]
-
-# --8<-- [start:build-wheel-from-source]
-
-#### Prerequisites
-
-- **MUSA SDK**: Download from [MUSA SDK Download](https://developer.mthreads.com/sdk/download/musa)
-- **torchada**: CUDA→MUSA compatibility layer for PyTorch (`pip install torchada`)
-- **mthreads-ml-py**: MTML Python bindings (`pip install mthreads-ml-py`)
-- **MATE**: MUSA AI Tensor Engine ([GitHub](https://github.com/MooreThreads/mate))
-
-#### Installation of vLLM-MUSA
-
-```bash
-git clone https://github.com/MooreThreads/vllm-musa.git
-cd vllm-musa
-git checkout v0.18.0-dev
-pip install . --no-build-isolation -v
-```
-
-#### Installation of vLLM-Omni
-
-```bash
-git clone https://github.com/vllm-project/vllm-omni.git
-cd vllm-omni
-VLLM_OMNI_TARGET_DEVICE=musa pip install -e . --no-build-isolation
-```
-
-For Gradio demos:
-
-```bash
-pip install -e '.[demo]' --no-build-isolation
-```
-
-#### Environment Variables
-
-```bash
-export MUSA_VISIBLE_DEVICES=0,1
-export VLLM_WORKER_MULTIPROC_METHOD=spawn
-export VLLM_MUSA_CUSTOM_OP_USE_NATIVE=false
-```
-
-# --8<-- [end:build-wheel-from-source]
-
-# --8<-- [start:build-docker]
-
-# --8<-- [end:build-docker]
-
-# --8<-- [start:pre-built-images]
-
-# --8<-- [end:pre-built-images]

docs/getting_started/installation/gpu/rocm.inc.md

@@ -13,7 +13,7 @@ vLLM-Omni current recommends the steps in under setup through Docker Images.
 
 vLLM-Omni is built based on vLLM. Please install it with command below.
 ```bash
-uv pip install vllm==0.19.0+rocm700 --extra-index-url https://wheels.vllm.ai/rocm/0.19.0/rocm700
+uv pip install vllm==0.18.0+rocm700 --extra-index-url https://wheels.vllm.ai/rocm/0.18.0/rocm700
 ```
 
 #### Installation of vLLM-Omni
@@ -34,13 +34,13 @@ uv pip install onnxruntime-rocm sox
 # --8<-- [start:build-wheel-from-source]
 
 #### Installation of vLLM
-If you do not need to modify source code of vLLM, you can directly install the stable 0.19.0 release version of the library
+If you do not need to modify source code of vLLM, you can directly install the stable 0.18.0 release version of the library
 
 ```bash
-uv pip install vllm==0.19.0+rocm700 --extra-index-url https://wheels.vllm.ai/rocm/0.19.0/rocm700
+uv pip install vllm==0.18.0+rocm700 --extra-index-url https://wheels.vllm.ai/rocm/0.18.0/rocm700
 ```
 
-The pre-built 0.19.0 vLLM wheel targets ROCm 7.0. If you need a different ROCm stack or want to reuse an existing PyTorch installation, build vLLM from source instead.
+The pre-built 0.18.0 vLLM wheel targets ROCm 7.0. If you need a different ROCm stack or want to reuse an existing PyTorch installation, build vLLM from source instead.
 
 #### Installation of vLLM-Omni
 Since vllm-omni is rapidly evolving, it's recommended to install it from source
@@ -58,7 +58,7 @@ If you want to check, modify or debug with source code of vLLM, install the libr
 ```bash
 git clone https://github.com/vllm-project/vllm.git
 cd vllm
-git checkout v0.19.0
+git checkout v0.18.0
 python3 -m pip install -r requirements/rocm.txt
 python3 setup.py develop
 ```
@@ -130,7 +130,7 @@ docker run --rm \
   -v ~/.cache/huggingface:/root/.cache/huggingface \
   --env "HF_TOKEN=$HF_TOKEN" \
   -p 8091:8091 \
-  vllm/vllm-omni-rocm:v0.19.0 \
+  vllm/vllm-omni-rocm:v0.18.0 \
   --model Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091
 ```
 
@@ -149,7 +149,7 @@ docker run --rm -it \
   -v ~/.cache/huggingface:/root/.cache/huggingface \
   --env "HF_TOKEN=$HF_TOKEN" \
   --entrypoint bash \
-  vllm/vllm-omni-rocm:v0.19.0
+  vllm/vllm-omni-rocm:v0.18.0
 ```
 
 # --8<-- [end:pre-built-images]

docs/getting_started/installation/npu/npu.inc.md

@@ -10,10 +10,10 @@ The recommended way to use vLLM-Omni on NPU is through the vllm-ascend pre-built
 ```bash
 # Update the vllm-ascend image
 # Atlas A2:
-# export IMAGE=quay.io/ascend/vllm-ascend:v0.18.0rc1
+# export IMAGE=quay.io/ascend/vllm-ascend:v0.17.0rc1
 # Atlas A3:
-# export IMAGE=quay.io/ascend/vllm-ascend:v0.18.0rc1-a3
+# export IMAGE=quay.io/ascend/vllm-ascend:v0.17.0rc1-a3
-export IMAGE=quay.io/ascend/vllm-ascend:v0.18.0rc1
+export IMAGE=quay.io/ascend/vllm-ascend:v0.17.0rc1
 docker run --rm \
     --name vllm-omni-npu \
     --shm-size=1g \
@@ -33,6 +33,17 @@ docker run --rm \
     -p 8000:8000 \
     -it $IMAGE bash
 
+cd /vllm-workspace/vllm
+git fetch origin --tags
+git checkout v0.18.0
+
+# Because vllm-ascend will release v0.18.0rc1 after vllm-omni 0.16.0,
+# we have to pin vllm-ascend at the current commit.
+cd /vllm-workspace/vllm-ascend
+git pull origin main
+git checkout d781902ce9dbda8ab1e11bb0f2f0c1bc508fee7a
+pip install -v -e .
+
 # Inside the container, install vLLM-Omni from source
 cd /vllm-workspace
 git clone -b v0.18.0 https://github.com/vllm-project/vllm-omni.git
@@ -57,10 +68,15 @@ You can also build vLLM-Omni from the latest main branch if you want to use the
 
 ```bash
 # Pin vLLM version to 0.18.0
-git clone -b v0.18.0 https://github.com/vllm-project/vllm.git
+cd /vllm-workspace/vllm
-VLLM_TARGET_DEVICE=empty pip install -v -e .
+git fetch origin --tags
+git checkout v0.18.0
 
-git clone -b v0.18.0rc1 https://github.com/vllm-project/vllm-ascend.git
+# Because vllm-ascend has not yet entered continuous development and has not been officially released, we need to pin it to a specific commit. Please note that this commit may change over time.
+cd /vllm-workspace/vllm-ascend
+git pull origin main
+git fetch origin --tags
+git checkout d781902ce9dbda8ab1e11bb0f2f0c1bc508fee7a
 pip install -v -e .
 
 # Install vLLM-Omni from the latest main branch

docs/getting_started/quickstart.md

@@ -19,10 +19,10 @@ uv venv --python 3.12 --seed
 source .venv/bin/activate
 
 # On CUDA
-uv pip install vllm==0.19.0 --torch-backend=auto
+uv pip install vllm==0.18.0 --torch-backend=auto
 
 # On ROCm
-uv pip install vllm==0.19.0+rocm700 --extra-index-url https://wheels.vllm.ai/rocm/0.19.0/rocm700
+uv pip install vllm==0.18.0+rocm700 --extra-index-url https://wheels.vllm.ai/rocm/0.18.0/rocm700
 
 git clone https://github.com/vllm-project/vllm-omni.git
 cd vllm-omni

docs/mkdocs/hooks/generate_argparse.py

@@ -120,7 +120,6 @@ def extract_omni_serve_subparser_init():
                         "_FlexibleArgumentParser": _FlexibleArgumentParser,
                         "FlexibleArgumentParser": _FlexibleArgumentParser,
                         "make_arg_parser": lambda parser: parser,  # no-op for doc
-                        "_ensure_vllm_platform": lambda: None,  # no-op for doc
                         "VLLM_SUBCMD_PARSER_EPILOG": "",
                         "logger": logger,
                         "DummySubparsers": DummySubparsers,

docs/models/supported_models.md

@@ -31,13 +31,9 @@ th {
 | `ZImagePipeline` | Z-Image | `Tongyi-MAI/Z-Image-Turbo` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
 | `WanPipeline` | Wan2.1-T2V, Wan2.2-T2V, Wan2.2-TI2V | `Wan-AI/Wan2.1-T2V-1.3B-Diffusers`, `Wan-AI/Wan2.1-T2V-14B-Diffusers`, `Wan-AI/Wan2.2-T2V-A14B-Diffusers`, `Wan-AI/Wan2.2-TI2V-5B-Diffusers` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
 | `WanImageToVideoPipeline` | Wan2.2-I2V | `Wan-AI/Wan2.2-I2V-A14B-Diffusers` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
-| `Wan22VACEPipeline` | Wan2.1-VACE | `Wan-AI/Wan2.1-VACE-1.3B-diffusers`, `Wan-AI/Wan2.1-VACE-14B-diffusers` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
 | `LTX2Pipeline` | LTX-2-T2V | `Lightricks/LTX-2` | ✅︎ | ✅︎ | | |
 | `LTX2ImageToVideoPipeline` | LTX-2-I2V | `Lightricks/LTX-2` | ✅︎ | ✅︎ | | |
-| `LTX2TwoStagesPipeline` | LTX-2-T2V | `rootonchair/LTX-2-19b-distilled` | ✅︎ | ✅︎ | | |
-| `LTX2ImageToVideoTwoStagesPipeline` | LTX-2-I2V | `rootonchair/LTX-2-19b-distilled` | ✅︎ | ✅︎ | | |
 | `HeliosPipeline`, `HeliosPyramidPipeline` | Helios | `BestWishYsh/Helios-Base`, `BestWishYsh/Helios-Mid`, `BestWishYsh/Helios-Distilled` | ✅︎ | ✅︎ | ✅︎ | |
-| `MagiHumanPipeline` | MagiHuman | `SII-GAIR/daVinci-MagiHuman-Base-1080p` | ✅︎ | ✅︎ | | |
 | `OvisImagePipeline` | Ovis-Image | `OvisAI/Ovis-Image` | ✅︎ | ✅︎ | | ✅︎ |
 | `LongcatImagePipeline` | LongCat-Image | `meituan-longcat/LongCat-Image` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
 | `LongCatImageEditPipeline` | LongCat-Image-Edit | `meituan-longcat/LongCat-Image-Edit` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |

docs/serving/speech_api.md

@@ -118,7 +118,6 @@ Content-Type: application/json
 | `instructions` | string | "" | Voice style/emotion instructions |
 | `max_new_tokens` | integer | 2048 | Maximum tokens to generate |
 | `initial_codec_chunk_frames` | integer | null | Per-request initial chunk size override for TTFA tuning. When null, IC is computed dynamically based on server load. |
-| `stream` | bool | false | Stream raw PCM chunks as they are decoded (requires `response_format="pcm"`) |
 
 **Supported languages:** Auto, Chinese, English, Japanese, Korean, German, French, Russian, Portuguese, Spanish, Italian
 
@@ -144,23 +143,9 @@ Lists available voices for the loaded model.
 
 ```json
 {
-    "voices": ["aiden", "dylan", "eric", "ono_anna", "ryan", "serena", "sohee", "uncle_fu", "vivian", "custom_voice_1"],
+    "voices": ["aiden", "dylan", "eric", "ono_anna", "ryan", "serena", "sohee", "uncle_fu", "vivian"]
-    "uploaded_voices": [
-        {
-            "name": "custom_voice_1",
-            "consent": "user_consent_id",
-            "created_at": 1738660000,
-            "file_size": 1024000,
-            "mime_type": "audio/wav",
-            "ref_text": "The exact transcript of the audio sample.",
-            "speaker_description": "warm narrator"
-        }
-    ]
 }
 ```
-
-`uploaded_voices` is always present (empty list when no custom voices have been uploaded). Fields `ref_text` and `speaker_description` are omitted per-entry when not provided at upload time.
-
 ```
 POST /v1/audio/voices
 Content-Type: multipart/form-data
@@ -176,7 +161,6 @@ Upload a new voice sample for voice cloning in Base task TTS requests.
 | `consent` | string | Yes | Consent recording ID |
 | `name` | string | Yes | Name for the new voice |
 | `ref_text` | string | No | Transcript of the audio. When provided, enables in-context voice cloning (higher quality). Without it, only the speaker embedding is extracted. |
-| `speaker_description` | string | No | Free-form description of the voice (e.g. "warm narrator", "energetic presenter"). Stored as metadata and returned in `GET /v1/audio/voices`. |
 
 **Response Example:**
 
@@ -188,15 +172,11 @@ Upload a new voice sample for voice cloning in Base task TTS requests.
     "consent": "user_consent_id",
     "created_at": 1738660000,
     "mime_type": "audio/wav",
-    "file_size": 1024000,
+    "file_size": 1024000
-    "ref_text": "The exact transcript of the audio sample.",
-    "speaker_description": "warm narrator"
   }
 }
 ```
 
-Fields `ref_text` and `speaker_description` are omitted when not provided at upload time.
-
 **Usage Example:**
 
 ```bash
@@ -204,8 +184,7 @@ curl -X POST http://localhost:8091/v1/audio/voices \
   -F "audio_sample=@/path/to/voice_sample.wav" \
   -F "consent=user_consent_id" \
   -F "name=custom_voice_1" \
-  -F "ref_text=The exact transcript of the audio sample." \
+  -F "ref_text=The exact transcript of the audio sample."
-  -F "speaker_description=warm narrator"
 ```
 
 ## Streaming Text Input (WebSocket)

docs/user_guide/diffusion/cache_acceleration/cache_dit.md

@@ -164,7 +164,7 @@ cache_config={
 
 **Performance Tips**:
 
-- Default `Fn_compute_blocks=1` works well for most cases. Some models (e.g., FLUX.2-klein) use a larger value for `Fn_compute_blocks` for a balanced performance.
+- Default `Fn_compute_blocks=1` works well for most cases. Some models (e.g., [FLUX.2-klein](https://github.com/wtomin/vllm-omni/blob/main/vllm_omni/diffusion/cache/cache_dit_backend.py#L363)) use a larger value for `Fn_compute_blocks` for a balanced performance.
 - Increase `residual_diff_threshold` (e.g., 0.12-0.15) for faster inference with slight quality trade-off, or decrease from default 0.24 for higher quality.
 - Default `max_warmup_steps=4` is optimized for few-step models. Increase to 6-8 for more steps if needed.
 

docs/user_guide/diffusion/cpu_offload_diffusion.md

@@ -91,19 +91,12 @@ Models must define the blocks attribute name for layerwise offloading:
 
 ```python
 class WanTransformer3DModel(nn.Module):
-    _layerwise_offload_blocks_attrs = ["blocks"]  # Attribute names containing transformer blocks
+    _layerwise_offload_blocks_attr = "blocks"  # Attribute name containing transformer blocks
 
     def __init__(self):
         self.blocks = nn.ModuleList([...])  # Transformer blocks
 ```
 
-For models with multiple block types:
-
-```python
-class Flux2Transformer2DModel(nn.Module):
-    _layerwise_offload_blocks_attrs = ["transformer_blocks", "single_transformer_blocks"]
-```
-
 ### Limitations
 - Cold start latency increases because of
     1) components are loaded to CPU first at the very first during initialization,
@@ -147,4 +140,4 @@ Factory function `get_offload_backend()` selects the appropriate backend based o
 
 **Notes:**
 - Model-Level Offloading is expected to be supported by all common diffusion models (DiT and encoders) naturally
-- Layerwise Offloading requires DiT class to define `_layerwise_offload_blocks_attrs` pointing to transformer blocks
+- Layerwise Offloading requires DiT class to define `_layerwise_offload_blocks_attr` pointing to transformer blocks

docs/user_guide/diffusion/quantization/autoround.md

@@ -1,91 +0,0 @@
-# AutoRound Quantization
-
-## Overview
-
-[AutoRound](https://github.com/intel/auto-round) is an advanced quantization toolkit designed for Large Language Models (LLMs), Vision-Language Models (VLMs), and diffusion models. It achieves high accuracy at ultra-low bit widths (2–4 bits) with minimal tuning by leveraging sign-gradient descent, while providing broad hardware compatibility with multi-datatype support.
-
-The quantization config is auto-detected from the checkpoint's `config.json` (`quantization_config.quant_method = "auto-round"`). No extra CLI flags are needed.
-
-### Supported Schemes
-
-| Scheme | Bits | Status |
-|--------|------|--------|
-| W4A16 | 4 | ✅ Supported |
-| W8A16 | 8 | Planned |
-
-W4A16 is the first supported scheme. Additional schemes will be added in future releases.
-
-## Configuration
-
-1. **Python API**: point `model` at a pre-quantized checkpoint. The quantization is detected automatically.
-
-```python
-from vllm_omni import Omni
-from vllm_omni.inputs.data import OmniDiffusionSamplingParams
-
-omni = Omni(model="vllm-project-org/FLUX.1-dev-AutoRound-w4a16")
-
-outputs = omni.generate(
-    "A cat sitting on a windowsill",
-    OmniDiffusionSamplingParams(num_inference_steps=28),
-)
-outputs[0].save_images("output.png")
-```
-
-2. **CLI**: pass the quantized model path directly.
-
-```bash
-python examples/offline_inference/text_to_image/text_to_image.py \
-  --model vllm-project-org/FLUX.1-dev-AutoRound-w4a16 \
-  --prompt "A cat sitting on a windowsill" \
-  --num-inference-steps 28 \
-  --output outputs/flux_w4a16.png
-```
-
-No `--quantization` flag is needed — the quantization method is read from the checkpoint.
-
-## How It Works
-
-The checkpoint's `config.json` contains:
-
-```json
-{
-  "quantization_config": {
-    "quant_method": "auto-round",
-    "bits": 4,
-    "group_size": 128,
-    "sym": true,
-    "packing_format": "auto_round:auto_gptq",
-    "block_name_to_quantize": "transformer_blocks,single_transformer_blocks"
-  }
-}
-```
-
-At load time:
-
-1. `TransformerConfig.from_dict()` parses the `quantization_config` section and builds a vLLM `INCConfig` via `build_quant_config("auto-round", ...)`.
-2. `OmniDiffusionConfig.set_tf_model_config()` propagates the detected config to the engine.
-3. The appropriate compute kernel (e.g. GPTQ-Marlin for W4A16) is selected automatically based on the checkpoint's bit-width and packing format.
-
-## Supported Models
-
-| Model | HF Checkpoint | Scheme | Group Size | Backend |
-|-------|--------------|--------|------------|---------|
-| FLUX.1-dev | `vllm-project-org/FLUX.1-dev-AutoRound-w4a16` | W4A16 | 128 | GPTQ-Marlin |
-
-## Creating a Quantized Checkpoint
-
-Use [AutoRound](https://github.com/intel/auto-round) to quantize a BF16 model. The `--scheme` flag selects the quantization scheme:
-
-```bash
-# W4A16 (4-bit weight, 16-bit activation)
-auto-round \
-    --model black-forest-labs/FLUX.1-dev \
-    --scheme W4A16 \
-    --batch_size 1 \
-    --disable_opt_rtn \
-    --dataset coco2014 \
-    --iters 0
-```
-
-The output directory can be used directly as the `model` argument. See the [AutoRound documentation](https://github.com/intel/auto-round) for all available schemes and options.

docs/user_guide/diffusion/quantization/overview.md

@@ -11,7 +11,6 @@ vLLM-Omni provides a unified quantization framework that supports both diffusion
 | FP8 | [FP8](fp8.md) | FP8 W8A8, dynamic or static | Z-Image, Qwen-Image, Flux, Bagel | SM 89 (Ada) |
 | Int8 | [Int8](int8.md) | Int8 W8A8 | Z-Image, Qwen-Image | SM 89 (Ada) / Ascend NPU |
 | GGUF | [GGUF](gguf.md) | GGUF format, dequant+GEMM for N-D tensors | Z-Image, Flux | SM 60 |
-| AutoRound | [AutoRound](autoround.md) | W4A16 (pre-quantized) | Flux | SM 80 (Ampere) |
 
 ### Multi-stage Omni Models (Pre-quantized Checkpoints)
 
@@ -103,9 +102,6 @@ config = build_quant_config("fp8")
 # Dict with parameters
 config = build_quant_config({"method": "fp8", "activation_scheme": "static"})
 
-# AutoRound / INC (auto-detected from checkpoint, or explicit)
-config = build_quant_config("auto-round", bits=4, group_size=128)
-
 # Per-component dict
 config = build_quant_config({
     "transformer": {"method": "fp8"},

docs/user_guide/diffusion_features.md

@@ -105,36 +105,34 @@ The following tables show which models support each feature:
 | **FLUX.2-klein** | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ |
 | **FLUX.1-Kontext-dev** | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ |
 | **FLUX.2-dev** | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ |
-| **GLM-Image** | ❌ | ❌ | ❌ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ |
+| **GLM-Image** | ❌ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ |
-| **HunyuanImage3** | ❌ | ✅ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ✅ | ❌ |
+| **HunyuanImage3** | ❌ | ✅ | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | ❌ | ❌ |
 | **LongCat-Image** | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ |
 | **LongCat-Image-Edit** | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ |
-| **MagiHuman** | ❌ | ❌ | ❌ | ❓ | ✅ | ❌ | ✅ | ❌ | ❌ | ❌ |
 | **MammothModa2(T2I)** | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ |
 | **Nextstep_1(T2I)** | ❓ | ❓ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ |
 | **OmniGen2** | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ |
 | **Ovis-Image** | ❌ | ✅ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ |
-| **Qwen-Image** | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ (decode) | ✅ | ✅ |
+| **Qwen-Image** | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ✅ |
-| **Qwen-Image-2512** | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ (decode) | ✅ | ✅ |
+| **Qwen-Image-2512** | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ | ✅ |
-| **Qwen-Image-Edit** | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ (decode) | ❌ | ❌ |
+| **Qwen-Image-Edit** | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ |
-| **Qwen-Image-Edit-2509** | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ (decode) | ✅ | ❌ | ❌ |
+| **Qwen-Image-Edit-2509** | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ |
-| **Qwen-Image-Layered** | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ (decode) | ❌ | ❌ |
+| **Qwen-Image-Layered** | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ |
-| **Stable-Diffusion3.5** | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ✅ (decode) | ❌ | ❌ |
+| **Stable-Diffusion3.5** | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ |
-| **Z-Image** | ✅ | ✅ | ✅ | ❓ | ✅ (TP=2 only) | ✅ | ❌ | ✅ (decode) | ✅ | ❌ |
+| **Z-Image** | ✅ | ✅ | ✅ | ❓ | ✅ (TP=2 only) | ❌ | ❌ | ✅ | ✅ | ❌ |
 
 > Notes:
 > 1. Nextstep_1(T2I) does not support cache acceleration methods such as TeaCache or Cache-DiT.
-> 2. `Tongyi-MAI/Z-Image-Turbo` and `SII-GAIR/daVinci-MagiHuman-Base-1080p` are distilled models with minimal NFEs; CFG-Parallel is not necessary.
+> 2. `Tongyi-MAI/Z-Image-Turbo` is a distilled model with minimal NFEs; CFG-Parallel is not necessary.
 
 ### VideoGen
 
 | Model | ⚡TeaCache | ⚡Cache-DiT | 🔀SP (Ulysses & Ring) | 🔀CFG-Parallel | 🔀Tensor-Parallel | 🔀HSDP | 💾CPU Offload (Layerwise) | 💾VAE-Patch-Parallel | 💾Quantization | 🔄Step Execution |
 |-------|:----------:|:-----------:|:---------------------:|:--------------:|:-----------------:|:------:|:------------------------:|:--------------------:|:--------------:|:----------------:|
-| **Wan2.2** | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ (encode/decode) | ❌ | ❌ |
+| **Wan2.2** | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ |
-| **Wan2.1-VACE** | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ (decode) | ❌ | ❌ |
 | **LTX-2** | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ |
 | **Helios** | ❌ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ❌ |
-| **HunyuanVideo-1.5 T2V I2V** | ❌ | ✅ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ (decode) | ✅ | ❌ |
+| **HunyuanVideo-1.5 T2V I2V** | ❌ | ✅ | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ |
 | **DreamID-Omni** | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ |
 
 ### AudioGen

docs/user_guide/examples/online_serving/qwen3_tts.md

@@ -159,7 +159,7 @@ curl -X POST http://localhost:8091/v1/audio/speech \
     -H "Content-Type: application/json" \
     -d '{
         "input": "Hello, how are you?",
-        "voice": "vivian",
+        "speaker": "vivian",
         "language": "English"
     }' --output output.wav
 
@@ -168,7 +168,7 @@ curl -X POST http://localhost:8091/v1/audio/speech \
     -H "Content-Type: application/json" \
     -d '{
         "input": "I am so excited!",
-        "voice": "vivian",
+        "speaker": "vivian",
         "instructions": "Speak with great enthusiasm"
     }' --output excited.wav
 
@@ -185,7 +185,7 @@ client = OpenAI(base_url="http://localhost:8091/v1", api_key="none")
 
 response = client.audio.speech.create(
     model="Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice",
-    voice="vivian",
+    speaker="vivian",
     input="Hello, how are you?",
 )
 
@@ -201,7 +201,7 @@ response = httpx.post(
     "http://localhost:8091/v1/audio/speech",
     json={
         "input": "Hello, how are you?",
-        "voice": "vivian",
+        "speaker": "vivian",
         "language": "English",
     },
     timeout=300.0,
@@ -237,16 +237,12 @@ List all available voices/speakers from the loaded model, including both built-i
       "consent": "user_consent_id",
       "created_at": 1738660000,
       "file_size": 1024000,
-      "mime_type": "audio/wav",
+      "mime_type": "audio/wav"
-      "ref_text": "The exact transcript of the audio sample.",
-      "speaker_description": "warm narrator"
     }
   ]
 }
 ```
 
-Fields `ref_text` and `speaker_description` are omitted per-entry when not provided at upload time.
-
 #### POST /v1/audio/voices
 
 Upload a new voice sample for voice cloning in Base task TTS requests.
@@ -256,7 +252,6 @@ Upload a new voice sample for voice cloning in Base task TTS requests.
 - `consent` (required): Consent recording ID
 - `name` (required): Name for the new voice
 - `ref_text` (optional): Transcript of the audio. Enables in-context voice cloning (higher quality).
-- `speaker_description` (optional): Free-form description of the voice (e.g. "warm narrator", "energetic presenter"). Stored as metadata.
 
 **Response Example:**
 ```json
@@ -267,23 +262,18 @@ Upload a new voice sample for voice cloning in Base task TTS requests.
     "consent": "user_consent_id",
     "created_at": 1738660000,
     "mime_type": "audio/wav",
-    "file_size": 1024000,
+    "file_size": 1024000
-    "ref_text": "The exact transcript of the audio sample.",
-    "speaker_description": "warm narrator"
   }
 }
 ```
 
-Fields `ref_text` and `speaker_description` are omitted when not provided at upload time.
-
 **Usage Example:**
 ```bash
-curl -X POST http://localhost:8091/v1/audio/voices \
+curl -X POST http://localhost:8000/v1/audio/voices \
   -F "audio_sample=@/path/to/voice_sample.wav" \
   -F "consent=user_consent_id" \
   -F "name=custom_voice_1" \
-  -F "ref_text=The exact transcript of the audio sample." \
+  -F "ref_text=The exact transcript of the audio sample."
-  -F "speaker_description=warm narrator"
 ```
 
 ### Endpoint
@@ -300,7 +290,7 @@ This endpoint follows the [OpenAI Audio Speech API](https://platform.openai.com/
 ```json
 {
     "input": "Text to synthesize",
-    "voice": "vivian",
+    "speaker": "vivian",
     "response_format": "wav",
     "task_type": "CustomVoice",
     "language": "Auto",
@@ -320,7 +310,7 @@ Returns binary audio data with appropriate `Content-Type` header (e.g., `audio/w
 
 ### Voice and language (summary)
 
-- **Speaker**: Use the `voice` request field to select the speaker (e.g., `vivian`, `ryan`, `aiden`). List available speakers with `GET /v1/audio/voices`.
+- **Speaker**: Use the `speaker` request field to select the speaker (e.g., `vivian`, `ryan`, `aiden`). List available speakers with `GET /v1/audio/voices`.
 - **Language**: Use the `language` field for the codec language tag (`Auto`, `Chinese`, `English`, etc.). Default is `Auto` for automatic detection.
 - **CustomVoice**: Requires a valid `voice` from the model’s speaker set. **VoiceDesign**: Use `instructions` to describe the voice. **Base**: Use `ref_audio` and `ref_text` for voice cloning.
 
@@ -332,7 +322,7 @@ Returns binary audio data with appropriate `Content-Type` header (e.g., `audio/w
 | ----------------- | ------ | -------------- | ----------------------------------------------------------- |
 | `input`           | string | **required**   | Text to synthesize                                          |
 | `model`           | string | server's model | Model to use (optional, should match server if specified)   |
-| `voice`           | string | "vivian"       | Speaker name (e.g., vivian, ryan, aiden)                    |
+| `speaker`         | string | "vivian"       | Speaker name (e.g., vivian, ryan, aiden)                    |
 | `response_format` | string | "wav"          | Audio format: wav, mp3, flac, pcm, aac, opus                |
 | `speed`           | float  | 1.0            | Playback speed (0.25-4.0, not supported with `stream=true`) |
 
@@ -367,7 +357,7 @@ curl -X POST http://localhost:8091/v1/audio/speech \
     -H "Content-Type: application/json" \
     -d '{
         "input": "Hello, how are you?",
-        "voice": "vivian",
+        "speaker": "vivian",
         "language": "English",
         "stream": true,
         "response_format": "pcm"

examples/offline_inference/bagel/end2end.py

@@ -2,10 +2,6 @@ import argparse
 import os
 
 from vllm_omni.inputs.data import OmniPromptType
-from vllm_omni.model_executor.stage_input_processors.bagel import (
-    GEN_THINK_SYSTEM_PROMPT,
-    VLM_THINK_SYSTEM_PROMPT,
-)
 
 
 def parse_args():
@@ -69,17 +65,6 @@ def parse_args():
         help="CFG parallel size: 1=batched (single GPU), 2=parallel with 2 branches (text CFG only), 3=parallel (3 GPUs).",
     )
     parser.add_argument("--seed", type=int, default=None, help="Random seed for generation.")
-    parser.add_argument(
-        "--cfg-interval",
-        type=float,
-        nargs=2,
-        default=None,
-        help="CFG interval [start, end] (default: pipeline default)",
-    )
-    parser.add_argument(
-        "--cfg-renorm-type", type=str, default=None, help="CFG renorm type: global, text_channel, channel"
-    )
-    parser.add_argument("--cfg-renorm-min", type=float, default=None, help="CFG renorm min")
     parser.add_argument(
         "--enable-diffusion-pipeline-profiler",
         action="store_true",
@@ -91,12 +76,6 @@ def parse_args():
         default=None,
         help="Quantization method (e.g. 'fp8').",
     )
-    parser.add_argument(
-        "--think",
-        action="store_true",
-        default=False,
-        help="Enable thinking mode: AR stage decodes <think>...</think> planning tokens before image generation.",
-    )
 
     args = parser.parse_args()
     return args
@@ -131,12 +110,8 @@ def main():
     from vllm_omni.entrypoints.omni import Omni
 
     omni_kwargs = {}
-    stage_configs_path = args.stage_configs_path
+    if args.stage_configs_path:
-    if args.think and stage_configs_path is None:
+        omni_kwargs["stage_configs_path"] = args.stage_configs_path
-        stage_configs_path = "vllm_omni/model_executor/stage_configs/bagel_think.yaml"
-        print(f"[Info] Think mode enabled, using stage config: {stage_configs_path}")
-    if stage_configs_path:
-        omni_kwargs["stage_configs_path"] = stage_configs_path
 
     omni_kwargs.update(
         {
@@ -161,8 +136,7 @@ def main():
             if not args.image_path or not os.path.exists(args.image_path):
                 raise ValueError(f"img2img requires --image-path pointing to an existing file, got: {args.image_path}")
             loaded_image = Image.open(args.image_path).convert("RGB")
-            think_prefix = f"<|im_start|>{GEN_THINK_SYSTEM_PROMPT}<|im_end|>" if args.think else ""
+            final_prompt_text = f"<|fim_middle|><|im_start|>{p}<|im_end|>"
-            final_prompt_text = f"{think_prefix}<|fim_middle|><|im_start|>{p}<|im_end|>"
             prompt_dict = {
                 "prompt": final_prompt_text,
                 "multi_modal_data": {"img2img": loaded_image},
@@ -174,10 +148,7 @@ def main():
         elif args.modality == "img2text":
             if args.image_path:
                 loaded_image = Image.open(args.image_path).convert("RGB")
-                think_prefix = f"<|im_start|>system\n{VLM_THINK_SYSTEM_PROMPT}<|im_end|>\n" if args.think else ""
+                final_prompt_text = f"<|im_start|>user\n<|image_pad|>\n{p}<|im_end|>\n<|im_start|>assistant\n"
-                final_prompt_text = (
-                    f"{think_prefix}<|im_start|>user\n<|image_pad|>\n{p}<|im_end|>\n<|im_start|>assistant\n"
-                )
                 prompt_dict = {
                     "prompt": final_prompt_text,
                     "multi_modal_data": {"image": loaded_image},
@@ -185,13 +156,11 @@ def main():
                 }
                 formatted_prompts.append(prompt_dict)
         elif args.modality == "text2text":
-            think_prefix = f"<|im_start|>{VLM_THINK_SYSTEM_PROMPT}<|im_end|>" if args.think else ""
+            final_prompt_text = f"<|im_start|>user\n{p}<|im_end|>\n<|im_start|>assistant\n"
-            final_prompt_text = f"{think_prefix}<|im_start|>{p}<|im_end|><|im_start|>"
             prompt_dict = {"prompt": final_prompt_text, "modalities": ["text"]}
             formatted_prompts.append(prompt_dict)
         else:
-            think_prefix = f"<|im_start|>{GEN_THINK_SYSTEM_PROMPT}<|im_end|>" if args.think else ""
+            final_prompt_text = f"<|im_start|>{p}<|im_end|>"
-            final_prompt_text = f"{think_prefix}<|im_start|>{p}<|im_end|>"
             prompt_dict = {"prompt": final_prompt_text, "modalities": ["image"]}
             if args.negative_prompt is not None:
                 prompt_dict["negative_prompt"] = args.negative_prompt
@@ -209,12 +178,6 @@ def main():
                 "cfg_text_scale": args.cfg_text_scale,
                 "cfg_img_scale": args.cfg_img_scale,
             }
-            if args.cfg_interval is not None:
-                extra["cfg_interval"] = tuple(args.cfg_interval)
-            if args.cfg_renorm_type is not None:
-                extra["cfg_renorm_type"] = args.cfg_renorm_type
-            if args.cfg_renorm_min is not None:
-                extra["cfg_renorm_min"] = args.cfg_renorm_min
             if args.negative_prompt is not None:
                 extra["negative_prompt"] = args.negative_prompt
             diffusion_params.extra_args = extra  # type: ignore
@@ -223,13 +186,6 @@ def main():
 
     img_idx = 0
     for req_output in omni_outputs:
-        if args.think:
-            ro = getattr(req_output, "request_output", None)
-            if ro and getattr(ro, "outputs", None):
-                txt = "".join(getattr(o, "text", "") or "" for o in ro.outputs)
-                if txt:
-                    print(txt)
-
         images = getattr(req_output, "images", None)
 
         if not images:
@@ -238,7 +194,6 @@ def main():
         for j, img in enumerate(images):
             save_path = os.path.join(args.output, f"output_{img_idx}_{j}.png")
             img.save(save_path)
-            print(f"[Output] Saved image to {save_path}")
         img_idx += 1
 
     print(omni_outputs)

examples/offline_inference/image_to_video/image_to_video.py

@@ -490,6 +490,10 @@ def main():
         if frames_np.ndim == 4 and frames_np.shape[-1] == 4:
             frames_np = frames_np[..., :3]
 
+        frames_np = np.clip(frames_np, 0.0, 1.0)
+        frames_u8 = (frames_np * 255).round().clip(0, 255).astype("uint8")
+        video_tensor = torch.from_numpy(frames_u8)
+
         audio_out = None
         if audio is not None:
             if isinstance(audio, list):
@@ -503,7 +507,7 @@ def main():
                 audio_out = audio_out.float().cpu()
 
         encode_video(
-            frames_np,
+            video_tensor,
             fps=fps,
             audio=audio_out,
             audio_sample_rate=args.audio_sample_rate if audio_out is not None else None,

examples/offline_inference/magi_human/README.md

@@ -1,72 +0,0 @@
-# MagiHuman Generation
-
-MagiHuman is an advanced, omni-modality model that generates both high-quality video and lip-synced audio from a text prompt.
-
-Because MagiHuman is a very large model featuring a powerful DiT MoE backbone and a ~9B parameter T5Gemma text encoder, it natively supports **Tensor Parallelism (TP)** in vLLM-Omni to run efficiently across multi-GPU setups, reducing device memory bottlenecks.
-
-## Setup
-
-### Install MagiCompiler (recommended)
-
-MagiHuman relies on [MagiCompiler](https://github.com/SandAI-org/MagiCompiler) for custom-op registration used by the DiT attention kernels. While the pipeline can fall back to stub implementations, installing MagiCompiler is **strongly recommended** for correct behaviour.
-
-```bash
-# Clone the repo
-git clone https://github.com/SandAI-org/MagiCompiler.git
-cd MagiCompiler
-
-# System dependencies (optional, for FX graph visualization; Debian/Ubuntu)
-sudo apt update && sudo apt install -y graphviz
-
-# Python dependencies
-pip install -r requirements.txt
-
-# Install MagiCompiler
-pip install .          # end users (recommended)
-# pip install -e .     # developers (editable install)
-```
-
-### Hardware requirements
-
-Ensure your hardware has enough VRAM. For a standard node with 80GB GPUs, running with `--tensor-parallel-size 4` is recommended to shard both the MoE weights and the T5Gemma text encoder across 4 GPUs, reducing the per-GPU peak VRAM overhead significantly (by roughly ~13.5GB per GPU compared to single-device inference).
-
-Please refer to the [stage configuration documentation](https://docs.vllm.ai/projects/vllm-omni/en/latest/configuration/stage_configs/) for further details on allocating memory.
-
-## Run Examples
-
-Get into the example folder:
-```bash
-cd examples/offline_inference/magi_human
-```
-
-### End-to-End Generation (Text to Video+Audio)
-
-Generate a video with synchronized speech natively generated by the model.
-
-```bash
-python end2end.py \
-    --model /proj-tango-pvc/users/zhipeng.wang/workspace/models/daVinci-MagiHuman \
-    --prompt "A young woman with long, wavy golden blonde hair..." \
-    --tensor-parallel-size 4 \
-    --output output_magihuman.mp4
-```
-
-## Common Parameters
-
-| Parameter | Default | Description |
-|-----------|---------|-------------|
-| `--model` | *(Required)* | Local model path or HuggingFace ID |
-| `--prompt` | *(built-in demo prompt)* | Highly detailed text prompt dictating visual look and dialogue text |
-| `--tensor-parallel-size` | `4` | Tensor parallelism size (Number of GPUs) |
-| `--height` | `256` | Initial resolution height |
-| `--width` | `448` | Initial resolution width |
-| `--num-inference-steps` | `8` | Denoising steps |
-| `--seed` | `52` | Random seed |
-| `--output` | `output_magihuman.mp4` | Output video with audio path |
-
-## Example materials
-
-??? abstract "end2end.py"
-    ``````py
-    --8<-- "examples/offline_inference/magi_human/end2end.py"
-    ``````

examples/offline_inference/magi_human/end2end.py

@@ -1,117 +0,0 @@
-import argparse
-
-from vllm_omni.diffusion.utils.media_utils import mux_video_audio_bytes
-from vllm_omni.entrypoints.omni import Omni
-from vllm_omni.inputs.data import OmniDiffusionSamplingParams
-
-
-def parse_args():
-    parser = argparse.ArgumentParser(description="End-to-end inference script for MagiHuman.")
-    parser.add_argument("--model", type=str, required=True, help="Path or ID of the MagiHuman model.")
-    parser.add_argument(
-        "--prompt",
-        type=str,
-        default="",
-        help="Text prompt containing visual description, dialogue, and background sound.",
-    )
-    parser.add_argument(
-        "--tensor-parallel-size", "-tp", type=int, default=4, help="Tensor parallel size (number of GPUs)."
-    )
-    parser.add_argument(
-        "--output", type=str, default="output_magihuman.mp4", help="Path to save the generated mp4 file."
-    )
-    parser.add_argument("--height", type=int, default=256, help="Video height.")
-    parser.add_argument("--width", type=int, default=448, help="Video width.")
-    parser.add_argument("--num-inference-steps", type=int, default=8, help="Number of denoising steps.")
-    parser.add_argument("--seed", type=int, default=52, help="Random seed for generation.")
-    return parser.parse_args()
-
-
-def main():
-    args = parse_args()
-
-    print(f"Initializing MagiHuman pipeline with TP={args.tensor_parallel_size}...")
-    omni = Omni(
-        model=args.model,
-        init_timeout=1200,
-        tensor_parallel_size=args.tensor_parallel_size,
-        devices=list(range(args.tensor_parallel_size)),
-    )
-
-    prompt = args.prompt
-    if not prompt:
-        prompt = (
-            "A young woman with long, wavy golden blonde hair and bright blue eyes, "
-            "wearing a fitted ivory silk blouse with a delicate lace collar, sits "
-            "stationary in front of a softly lit, blurred warm-toned interior. Her "
-            "overall disposition is warm, composed, and gently confident. The camera "
-            "holds a static medium close-up, framing her from the shoulders up, "
-            "with shallow depth of field keeping her face in sharp focus. Soft "
-            "directional key light falls from the upper left, casting a gentle "
-            "highlight along her cheekbone and nose bridge. She draws a quiet breath, "
-            "the levator labii superiors relaxing as her lips part. She speaks in "
-            "clear, warm, unhurried American English: "
-            "\"The most beautiful things in life aren't things at all — "
-            "they're moments, feelings, and the people who make you feel truly alive.\" "
-            "Her jaw descends smoothly on each stressed syllable; the orbicularis oris "
-            "shapes each vowel with precision. A faint, genuine smile engages the "
-            "zygomaticus major, lifting her lip corners fractionally. Her brows rest "
-            "in a soft, neutral arch throughout. She maintains steady, forward-facing "
-            "eye contact. Head position remains level; no torso displacement occurs.\n\n"
-            "Dialogue:\n"
-            "<Young blonde woman, American English>: "
-            "\"The most beautiful things in life aren't things at all — "
-            "they're moments, feelings, and the people who make you feel truly alive.\"\n\n"
-            "Background Sound:\n"
-            "<Soft, warm indoor ambience with a faint distant piano melody>"
-        )
-
-    sampling_params = OmniDiffusionSamplingParams(
-        height=args.height,
-        width=args.width,
-        num_inference_steps=args.num_inference_steps,
-        seed=args.seed,
-        extra_args={
-            "seconds": 5,
-            "sr_height": 1080,
-            "sr_width": 1920,
-            "sr_num_inference_steps": 5,
-        },
-    )
-
-    print(f"Generating with prompt: {prompt[:80]}...")
-    outputs = omni.generate(
-        prompts=[prompt],
-        sampling_params_list=[sampling_params],
-    )
-
-    print(f"Generation complete. Output type: {type(outputs)}")
-    if outputs:
-        first = outputs[0]
-
-        if hasattr(first, "images") and first.images:
-            video_frames = first.images[0]
-            print(f"Video frames: shape={video_frames.shape}, dtype={video_frames.dtype}")
-
-            audio_waveform = None
-            if hasattr(first, "multimodal_output") and first.multimodal_output:
-                audio_waveform = first.multimodal_output.get("audio")
-                if audio_waveform is not None:
-                    print(f"Audio waveform: shape={audio_waveform.shape}, dtype={audio_waveform.dtype}")
-
-            video_bytes = mux_video_audio_bytes(
-                video_frames,
-                audio_waveform,
-                fps=25.0,
-                audio_sample_rate=44100,
-            )
-            with open(args.output, "wb") as f:
-                f.write(video_bytes)
-            print(f"Saved MP4 ({len(video_bytes)} bytes) to {args.output}")
-        print("SUCCESS: MagiHuman pipeline generation completed.")
-    else:
-        print("WARNING: No outputs returned.")
-
-
-if __name__ == "__main__":
-    main()

examples/offline_inference/omnivoice/README.md

@@ -1,73 +0,0 @@
-# OmniVoice
-
-This directory contains an offline demo for running OmniVoice TTS models with vLLM Omni. It generates speech from text and saves WAV files locally.
-
-## Model Overview
-
-[OmniVoice](https://huggingface.co/k2-fsa/OmniVoice) is a zero-shot multilingual TTS model supporting 600+ languages. It uses a diffusion language model (Qwen3-0.6B backbone) with iterative masked unmasking to generate speech.
-
-Three inference modes are supported:
-
-- **Auto Voice**: Generate speech without any reference — the model picks a voice automatically.
-- **Voice Clone**: Clone a voice from a reference audio + transcription.
-- **Voice Design**: Control voice style via natural language instruction (e.g., "female, low pitch, british accent").
-
-## Setup
-
-Ensure the model is downloaded:
-
-```bash
-huggingface-cli download k2-fsa/OmniVoice
-```
-
-> **Note:** Voice cloning requires `transformers>=5.3.0` for `HiggsAudioV2TokenizerModel`. Auto voice and voice design modes work with `transformers>=4.57.0`.
-
-## Quick Start
-
-Auto voice (text only):
-
-```bash
-python end2end.py --model k2-fsa/OmniVoice --text "Hello, this is a test."
-```
-
-Voice design (with style instruction):
-
-```bash
-python end2end.py --model k2-fsa/OmniVoice \
-    --text "Hello, this is a test." \
-    --instruct "female, low pitch, british accent"
-```
-
-Voice clone (with reference audio):
-
-```bash
-python end2end.py --model k2-fsa/OmniVoice \
-    --text "Hello, this is a test." \
-    --ref-audio ref.wav \
-    --ref-text "This is the reference transcription."
-```
-
-## Language Support
-
-Specify a language for improved quality:
-
-```bash
-python end2end.py --model k2-fsa/OmniVoice \
-    --text "你好，这是一个测试。" \
-    --lang zh
-```
-
-## Architecture
-
-OmniVoice uses a two-stage pipeline:
-
-- **Stage 0 (Generator)**: Qwen3-0.6B transformer with 32-step iterative unmasking and classifier-free guidance. Generates 8-codebook audio tokens from text.
-- **Stage 1 (Decoder)**: HiggsAudioV2 RVQ quantizer + DAC acoustic decoder. Converts tokens to 24kHz waveform.
-
-Both stages use `GPUGenerationWorker` with `OmniGenerationScheduler`.
-
-## Notes
-
-- Output audio is saved to `output.wav` by default. Use `--output` to change the path.
-- The model estimates duration from text automatically via `RuleDurationEstimator`.
-- Use `--stage-init-timeout` to increase the stage initialization timeout for first-time model downloads.

examples/offline_inference/omnivoice/end2end.py

@@ -1,164 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-"""End-to-end OmniVoice TTS inference via vLLM-Omni.
-
-Supports:
-- Auto voice mode: text only → generated speech
-- Voice cloning mode: text + reference audio → cloned voice speech
-
-Usage:
-    # Auto voice
-    python end2end.py --model k2-fsa/OmniVoice --text "Hello world"
-
-    # Voice cloning
-    python end2end.py --model k2-fsa/OmniVoice --text "Hello" \
-        --ref-audio ref.wav --ref-text "reference transcription"
-"""
-
-import argparse
-import os
-
-import numpy as np
-import soundfile as sf
-
-from vllm_omni.entrypoints.omni import Omni
-from vllm_omni.inputs.data import OmniDiffusionSamplingParams
-
-
-def run_e2e():
-    parser = argparse.ArgumentParser(description="OmniVoice E2E TTS inference")
-    parser.add_argument(
-        "--model",
-        type=str,
-        default="k2-fsa/OmniVoice",
-        help="Model name or path (HuggingFace or local)",
-    )
-    parser.add_argument(
-        "--stage-config",
-        type=str,
-        default="vllm_omni/model_executor/stage_configs/omnivoice.yaml",
-    )
-    parser.add_argument(
-        "--text",
-        type=str,
-        default="Hello, this is a test of the OmniVoice text to speech system.",
-    )
-    parser.add_argument(
-        "--ref-audio",
-        type=str,
-        default=None,
-        help="Reference audio for voice cloning (WAV file)",
-    )
-    parser.add_argument(
-        "--ref-text",
-        type=str,
-        default=None,
-        help="Transcription of reference audio",
-    )
-    parser.add_argument(
-        "--lang",
-        type=str,
-        default=None,
-        help="Language code (e.g., 'en', 'zh')",
-    )
-    parser.add_argument(
-        "--instruct",
-        type=str,
-        default=None,
-        help="Voice design instruction (e.g., 'female, low pitch, british accent')",
-    )
-    parser.add_argument(
-        "--output",
-        type=str,
-        default="output.wav",
-        help="Output audio file path",
-    )
-    parser.add_argument(
-        "--stage-init-timeout",
-        type=int,
-        default=600,
-        help="Stage initialization timeout in seconds",
-    )
-    args = parser.parse_args()
-
-    if not os.path.exists(args.stage_config):
-        raise FileNotFoundError(f"Stage config not found: {args.stage_config}")
-
-    print(f"Initializing OmniVoice with model={args.model}")
-
-    omni = Omni(
-        model=args.model,
-        stage_configs_path=args.stage_config,
-        trust_remote_code=True,
-        log_stats=True,
-    )
-
-    print("Model initialized. Preparing inputs...")
-
-    # Build prompt
-    mm_processor_kwargs = {}
-    multi_modal_data = {}
-
-    if args.ref_audio:
-        if not os.path.exists(args.ref_audio):
-            raise FileNotFoundError(f"Reference audio not found: {args.ref_audio}")
-
-        import librosa
-
-        audio_signal, sr = librosa.load(args.ref_audio, sr=None)
-        multi_modal_data["audio"] = (audio_signal.astype(np.float32), sr)
-        mm_processor_kwargs["ref_text"] = args.ref_text or ""
-        mm_processor_kwargs["sample_rate"] = sr
-
-    if args.lang:
-        mm_processor_kwargs["lang"] = args.lang
-    if args.instruct:
-        mm_processor_kwargs["instruct"] = args.instruct
-
-    prompts = {"prompt": args.text}
-    if multi_modal_data:
-        prompts["multi_modal_data"] = multi_modal_data
-    if mm_processor_kwargs:
-        prompts["mm_processor_kwargs"] = mm_processor_kwargs
-
-    sampling_params_list = [OmniDiffusionSamplingParams()]
-
-    print(f"Generating speech for: {args.text}")
-
-    outputs = list(omni.generate(prompts, sampling_params_list=sampling_params_list))
-
-    print(f"Received {len(outputs)} outputs.")
-    for i, output in enumerate(outputs):
-        try:
-            ro = output.request_output
-            if ro is None:
-                print("No request_output found.")
-                continue
-
-            mm = getattr(ro, "multimodal_output", None)
-            if not mm and ro.outputs:
-                mm = getattr(ro.outputs[0], "multimodal_output", None)
-
-            if mm:
-                print(f"Multimodal output keys: {mm.keys()}")
-                if "audio" in mm:
-                    audio_out = mm["audio"]
-                    sr = mm.get("sr", 24000)
-                    if isinstance(audio_out, np.ndarray):
-                        audio_np = audio_out
-                    else:
-                        audio_np = audio_out.cpu().numpy().squeeze()
-                    out_path = args.output if i == 0 else f"output_{i}.wav"
-                    sf.write(out_path, audio_np, sr)
-                    print(f"Saved audio to {out_path} ({sr}Hz, {len(audio_np) / sr:.2f}s)")
-            else:
-                print("No multimodal output found.")
-        except Exception as e:
-            print(f"Error inspecting output: {e}")
-
-    omni.close()
-    print("Done.")
-
-
-if __name__ == "__main__":
-    run_e2e()

examples/offline_inference/text_to_image/README.md

@@ -33,7 +33,6 @@ This folder provides several entrypoints for experimenting with text-to-image di
 | `black-forest-labs/FLUX.2-klein-4B` | 1024 x 1024 | 72.7 | 14.9 |
 | `black-forest-labs/FLUX.2-klein-9B` | 1024 x 1024 | 37.1 | 32.3 |
 | `black-forest-labs/FLUX.2-dev` | 1024 x 1024 | 65.7 | >80 (CPU offload required) |
-| `HunyuanImage-3.0` | 1024 x 1024 | 80.0 (TP≥3)  | 160 |
 
 !!! info
 *Peak VRAM:  based on basic single-card usage, batch size =1, without any acceleration/optimization features. FLUX.2-dev requires `--enable-cpu-offload` on a single 80 GiB GPU.
@@ -91,8 +90,6 @@ python text_to_image.py \
 | `--enable-cpu-offload` | flag | off | Enable CPU offloading for diffusion models |
 | `--lora-path` | str | — | Path to PEFT LoRA adapter folder |
 | `--lora-scale` | float | `1.0` | Scale factor for LoRA weights |
-| `--use-system-prompt` | str | `None` | System prompt preset: `en_unified`, `en_vanilla`, `en_recaption`, `en_think_recaption`, `dynamic`, `None`, or custom text. Recommended: `en_unified`. Only for HunyuanImage-3.0.|
-| `--system-prompt` | str | `None` | Custom system prompt text. Only used when `--use-system-prompt` is set to `custom`. Only for HunyuanImage-3.0.|
 
 **NextStep-1.1 specific arguments:**
 

examples/offline_inference/text_to_image/text_to_image.py

@@ -242,19 +242,6 @@ def parse_args() -> argparse.Namespace:
         action="store_true",
         help="Enable logging of diffusion pipeline stats.",
     )
-    parser.add_argument(
-        "--use-system-prompt",
-        type=str,
-        default=None,
-        choices=["None", "dynamic", "en_vanilla", "en_recaption", "en_think_recaption", "en_unified", "custom"],
-        help="System prompt preset for generation. Recommended: en_unified.",
-    )
-    parser.add_argument(
-        "--system-prompt",
-        type=str,
-        default=None,
-        help=("Custom system prompt. Used when --use-system-prompt is custom. "),
-    )
     return parser.parse_args()
 
 
@@ -395,13 +382,13 @@ def main():
         )
 
     generation_start = time.perf_counter()
+
     extra_args = {
         "timesteps_shift": args.timesteps_shift,
         "cfg_schedule": args.cfg_schedule,
         "use_norm": args.use_norm,
-        "use_system_prompt": args.use_system_prompt,
-        "system_prompt": args.system_prompt,
     }
+
     if lora_request:
         extra_args["lora_request"] = lora_request
         extra_args["lora_scale"] = args.lora_scale

examples/offline_inference/text_to_video/text_to_video.py

@@ -56,13 +56,8 @@ def parse_args() -> argparse.Namespace:
         "Examples: Wan-AI/Wan2.2-T2V-A14B-Diffusers, "
         "hunyuanvideo-community/HunyuanVideo-1.5-480p_t2v",
     )
-    parser.add_argument(
-        "--model-class-name",
-        default=None,
-        help="Override model class name (e.g., LTX2TwoStagesVideoPipeline).",
-    )
     parser.add_argument("--prompt", default="A serene lakeside sunrise with mist over the water.", help="Text prompt.")
-    parser.add_argument("--negative-prompt", default="", help="Negative prompt.")
+    parser.add_argument("--negative-prompt", default="", help="Negative prompt (Wan2.2 only).")
     parser.add_argument("--seed", type=int, default=42, help="Random seed.")
     parser.add_argument("--guidance-scale", type=float, default=None, help="CFG scale. Default: model-specific.")
     parser.add_argument(
@@ -190,7 +185,6 @@ def parse_args() -> argparse.Namespace:
 
 def main():
     args = parse_args()
-    model_class_name = args.model_class_name
 
     preset = _detect_preset(args.model)
     for key, default_val in preset.items():
@@ -235,7 +229,6 @@ def main():
         enable_cpu_offload=args.enable_cpu_offload,
         parallel_config=parallel_config,
         enforce_eager=args.enforce_eager,
-        model_class_name=model_class_name,
         cache_backend=args.cache_backend,
         cache_config=cache_config,
         enable_diffusion_pipeline_profiler=args.enable_diffusion_pipeline_profiler,

examples/offline_inference/vace/vace_video_generation.md

@@ -1,88 +0,0 @@
-# VACE Video Generation
-
-[VACE](https://github.com/ali-vilab/VACE) (Video All-in-one Creation Engine) supports multiple video tasks through a single model.
-
-| Model | Architecture | Model Weights (bf16) | HuggingFace |
-|-------|-------------|----------------------|-------------|
-| Wan2.1-VACE (1.3B) | Wan2.1 | ~10 GB | [Wan-AI/Wan2.1-VACE-1.3B-diffusers](https://huggingface.co/Wan-AI/Wan2.1-VACE-1.3B-diffusers) |
-| Wan2.1-VACE (14B) | Wan2.1 | ~38 GB | [Wan-AI/Wan2.1-VACE-14B-diffusers](https://huggingface.co/Wan-AI/Wan2.1-VACE-14B-diffusers) |
-
-## Text-to-Video (T2V)
-
-```bash
-python vace_video_generation.py \
-  --mode t2v \
-  --prompt "A sleek robot stands in a vast warehouse filled with boxes" \
-  --height 480 --width 832 --num-frames 81 \
-  --num-inference-steps 30 --guidance-scale 5.0 --flow-shift 5.0 \
-  --output t2v_output.mp4
-```
-
-## Image-to-Video (I2V)
-
-First frame is kept, remaining frames are generated:
-
-```bash
-python vace_video_generation.py \
-  --mode i2v \
-  --image astronaut.jpg \
-  --prompt "An astronaut emerging from a cracked egg on the moon" \
-  --height 480 --width 832 --num-frames 81 \
-  --output i2v_output.mp4
-```
-
-## First-Last-Frame Interpolation (FLF2V)
-
-```bash
-python vace_video_generation.py \
-  --mode flf2v \
-  --image first_frame.jpg --last-image last_frame.jpg \
-  --prompt "A bird takes off from a branch and lands on another" \
-  --height 512 --width 512 --num-frames 81 \
-  --output flf2v_output.mp4
-```
-
-## Inpainting
-
-Center vertical stripe is masked and regenerated:
-
-```bash
-python vace_video_generation.py \
-  --mode inpaint \
-  --image scene.jpg \
-  --prompt "Shrek walks out of a building" \
-  --height 480 --width 832 --num-frames 81 \
-  --output inpaint_output.mp4
-```
-
-## Reference Image-guided (R2V)
-
-```bash
-python vace_video_generation.py \
-  --mode r2v \
-  --image reference.jpg \
-  --prompt "Camera slowly zooms out from the character" \
-  --height 480 --width 832 --num-frames 81 \
-  --output r2v_output.mp4
-```
-
-## Key Arguments
-
-- `--mode`: VACE task mode (`t2v`, `i2v`, `flf2v`, `inpaint`, `r2v`).
-- `--model`: Model ID (default: `Wan-AI/Wan2.1-VACE-1.3B-diffusers`).
-- `--image`: Input image for I2V, inpainting, and R2V modes.
-- `--last-image`: Last frame image for FLF2V mode.
-- `--prompt`: Text description of desired video.
-- `--height/--width`: Output resolution (default 480x832). Dimensions should be multiples of 16.
-- `--num-frames`: Number of frames (default 81).
-- `--guidance-scale`: CFG scale (default 5.0).
-- `--flow-shift`: Scheduler flow shift (default 5.0).
-- `--num-inference-steps`: Number of denoising steps (default 30).
-- `--fps`: Frames per second for the saved MP4 (default 16).
-- `--output`: Path to save the generated video.
-- `--vae-use-tiling`: Enable VAE tiling for memory optimization.
-- `--ulysses-degree`: Ulysses sequence parallelism degree for multi-GPU.
-- `--cfg-parallel-size`: CFG parallel size for multi-GPU.
-- `--tensor-parallel-size`: Tensor parallel size.
-
-> If you encounter OOM errors, try `--vae-use-tiling` or multi-GPU parallelism options.

examples/offline_inference/vace/vace_video_generation.py

@@ -1,209 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-
-"""VACE video generation example.
-
-VACE (Video All-in-one Creation Engine) supports multiple video tasks:
-  - T2V:        Text-to-Video
-  - I2V:        Image-to-Video (first frame conditioning)
-  - V2LF:       Video-to-Last-Frame
-  - FLF2V:      First-Last-Frame interpolation
-  - Inpainting:  Masked region generation
-  - R2V:        Reference image-guided generation
-
-Usage examples:
-  # T2V (text-to-video)
-  python vace_video_generation.py --mode t2v --prompt "A robot in a warehouse"
-
-  # I2V (image-to-video, first frame kept)
-  python vace_video_generation.py --mode i2v --image input.jpg --prompt "..."
-
-  # FLF2V (first-last frame interpolation)
-  python vace_video_generation.py --mode flf2v --image first.jpg --last-image last.jpg
-
-  # R2V (reference image guided)
-  python vace_video_generation.py --mode r2v --image ref.jpg --prompt "..."
-"""
-
-import argparse
-import time
-from pathlib import Path
-
-import numpy as np
-import PIL.Image
-import torch
-
-from vllm_omni.diffusion.data import DiffusionParallelConfig
-from vllm_omni.entrypoints.omni import Omni
-from vllm_omni.inputs.data import OmniDiffusionSamplingParams
-from vllm_omni.platforms import current_omni_platform
-
-
-def parse_args() -> argparse.Namespace:
-    parser = argparse.ArgumentParser(description="VACE video generation.")
-    parser.add_argument(
-        "--model",
-        default="Wan-AI/Wan2.1-VACE-14B-diffusers",
-        help="VACE model ID or local path.",
-    )
-    parser.add_argument(
-        "--mode",
-        default="t2v",
-        choices=["t2v", "i2v", "v2lf", "flf2v", "inpaint", "r2v"],
-        help="Generation mode.",
-    )
-    parser.add_argument("--prompt", default="A cat walking in a garden", help="Text prompt.")
-    parser.add_argument("--negative-prompt", default="", help="Negative prompt.")
-    parser.add_argument("--image", type=str, default=None, help="Input image path (for I2V, R2V, FLF2V, inpaint).")
-    parser.add_argument("--last-image", type=str, default=None, help="Last frame image path (for FLF2V).")
-    parser.add_argument("--video-dir", type=str, default=None, help="Directory of video frames (for inpaint).")
-    parser.add_argument("--seed", type=int, default=42, help="Random seed.")
-    parser.add_argument("--guidance-scale", type=float, default=5.0, help="CFG guidance scale.")
-    parser.add_argument("--height", type=int, default=480, help="Video height.")
-    parser.add_argument("--width", type=int, default=832, help="Video width.")
-    parser.add_argument("--num-frames", type=int, default=81, help="Number of frames.")
-    parser.add_argument("--num-inference-steps", type=int, default=30, help="Sampling steps.")
-    parser.add_argument("--flow-shift", type=float, default=5.0, help="Scheduler flow_shift.")
-    parser.add_argument("--output", type=str, default="vace_output.mp4", help="Output video path.")
-    parser.add_argument("--fps", type=int, default=16, help="Output video FPS.")
-    parser.add_argument("--vae-use-tiling", action="store_true", default=True, help="Enable VAE tiling.")
-    parser.add_argument("--enforce-eager", action="store_true", help="Disable torch.compile.")
-    parser.add_argument("--ulysses-degree", type=int, default=1, help="Ulysses SP degree.")
-    parser.add_argument("--ring-degree", type=int, default=1, help="Ring attention degree.")
-    parser.add_argument("--cfg-parallel-size", type=int, default=1, choices=[1, 2], help="CFG parallel size.")
-    return parser.parse_args()
-
-
-def build_prompts(args):
-    """Build prompt dict with multi_modal_data based on mode."""
-    h, w, nf = args.height, args.width, args.num_frames
-
-    gray = PIL.Image.new("RGB", (w, h), (128, 128, 128))
-    mask_black = PIL.Image.new("L", (w, h), 0)
-    mask_white = PIL.Image.new("L", (w, h), 255)
-
-    prompt_data = {
-        "prompt": args.prompt,
-        "negative_prompt": args.negative_prompt,
-    }
-
-    if args.mode == "t2v":
-        return prompt_data
-
-    if args.mode == "r2v":
-        assert args.image, "--image required for R2V mode"
-        ref_img = PIL.Image.open(args.image).convert("RGB").resize((w, h))
-        prompt_data["multi_modal_data"] = {"reference_images": [ref_img]}
-        return prompt_data
-
-    if args.mode == "i2v":
-        assert args.image, "--image required for I2V mode"
-        img = PIL.Image.open(args.image).convert("RGB").resize((w, h))
-        prompt_data["multi_modal_data"] = {
-            "video": [img] + [gray] * (nf - 1),
-            "mask": [mask_black] + [mask_white] * (nf - 1),
-        }
-        return prompt_data
-
-    if args.mode == "v2lf":
-        assert args.image, "--image required for V2LF mode"
-        img = PIL.Image.open(args.image).convert("RGB").resize((w, h))
-        prompt_data["multi_modal_data"] = {
-            "video": [gray] * (nf - 1) + [img],
-            "mask": [mask_white] * (nf - 1) + [mask_black],
-        }
-        return prompt_data
-
-    if args.mode == "flf2v":
-        assert args.image and args.last_image, "--image and --last-image required for FLF2V"
-        first = PIL.Image.open(args.image).convert("RGB").resize((w, h))
-        last = PIL.Image.open(args.last_image).convert("RGB").resize((w, h))
-        prompt_data["multi_modal_data"] = {
-            "video": [first] + [gray] * (nf - 2) + [last],
-            "mask": [mask_black] + [mask_white] * (nf - 2) + [mask_black],
-        }
-        return prompt_data
-
-    if args.mode == "inpaint":
-        assert args.image, "--image required for inpaint mode"
-        img = PIL.Image.open(args.image).convert("RGB").resize((w, h))
-        d = 80
-        frames, masks = [], []
-        for _ in range(nf):
-            base = np.array(img).copy()
-            mask = PIL.Image.new("L", (w, h), 0)
-            stripe = PIL.Image.new("L", (2 * d, h), 255)
-            mask.paste(stripe, (w // 2 - d, 0))
-            base[np.array(mask) > 128] = 128
-            frames.append(PIL.Image.fromarray(base))
-            masks.append(mask)
-        prompt_data["multi_modal_data"] = {"video": frames, "mask": masks}
-        return prompt_data
-
-    raise ValueError(f"Unknown mode: {args.mode}")
-
-
-def main():
-    args = parse_args()
-    generator = torch.Generator(device=current_omni_platform.device_type).manual_seed(args.seed)
-
-    parallel_config = DiffusionParallelConfig(
-        ulysses_degree=args.ulysses_degree,
-        ring_degree=args.ring_degree,
-        cfg_parallel_size=args.cfg_parallel_size,
-    )
-
-    omni = Omni(
-        model=args.model,
-        vae_use_tiling=args.vae_use_tiling,
-        flow_shift=args.flow_shift,
-        enforce_eager=args.enforce_eager,
-        parallel_config=parallel_config,
-    )
-
-    prompt_data = build_prompts(args)
-
-    print(f"\n{'=' * 60}")
-    print(f"VACE {args.mode.upper()} Generation")
-    print(f"  Model: {args.model}")
-    print(f"  Size: {args.width}x{args.height}, {args.num_frames} frames, {args.num_inference_steps} steps")
-    print(f"{'=' * 60}\n")
-
-    start = time.perf_counter()
-    outputs = omni.generate(
-        prompt_data,
-        OmniDiffusionSamplingParams(
-            height=args.height,
-            width=args.width,
-            num_frames=args.num_frames,
-            num_inference_steps=args.num_inference_steps,
-            guidance_scale=args.guidance_scale,
-            generator=generator,
-        ),
-    )
-    elapsed = time.perf_counter() - start
-
-    video = outputs[0].images
-    if isinstance(video, list):
-        video = video[0]
-    if isinstance(video, torch.Tensor):
-        video = video.cpu().numpy()
-    if video.ndim == 5:
-        video = video[0]
-    print(f"Output shape: {video.shape}, Time: {elapsed:.1f}s")
-
-    output_path = Path(args.output)
-    output_path.parent.mkdir(parents=True, exist_ok=True)
-
-    from diffusers.utils import export_to_video
-
-    if np.issubdtype(video.dtype, np.integer):
-        video = video.astype(np.float32) / 255.0
-    export_to_video(list(video), str(output_path), fps=args.fps)
-    print(f"Saved to {output_path}")
-
-    omni.close()
-
-
-if __name__ == "__main__":
-    main()

examples/offline_inference/x_to_video_audio/download_dreamid_omni.py

@@ -82,6 +82,7 @@ def main(output_dir: str):
 
     data = {
         "_class_name": "DreamIDOmniPipeline",
+        "fusion": "DreamID-Omni/dreamid_omni.safetensors",
     }
 
     with open(os.path.join(output_dir, "model_index.json"), "w", encoding="utf-8") as f:
@@ -89,12 +90,6 @@ def main(output_dir: str):
 
     print(f"model_index.json created at {os.path.join(output_dir, 'model_index.json')}")
 
-    transformer_dir = os.path.join(output_dir, "transformer")
-    os.makedirs(transformer_dir, exist_ok=True)
-    with open(os.path.join(transformer_dir, "config.json"), "w", encoding="utf-8") as f:
-        json.dump({"fusion": "DreamID-Omni/dreamid_omni.safetensors"}, f)
-    print(f"transformer/config.json created at {os.path.join(transformer_dir, 'config.json')}")
-
     # now we download the dependency code
     download_dependency()
 

examples/offline_inference/x_to_video_audio/x_to_video_audio.md

@@ -24,9 +24,7 @@ dreamid_omni/
 │   ├── models_t5_umt5-xxl-enc-bf16.pth
 │   ├── Wan2.2_VAE.pth
 │
-├── model_index.json
+├── model_index.json # create by download_dreamid_omni.py
-└── transformer/
-    └── config.json   # create by download_dreamid_omni.py
 ```
 
 ### Run the Inference

examples/offline_inference/x_to_video_audio/x_to_video_audio.py

@@ -132,8 +132,8 @@ def main() -> None:
     if not outputs:
         raise RuntimeError("No output returned from DreamID-Omni.")
     output = outputs[0].request_output
-    generated_video = output.images[0][0]
+    generated_video = output[0].images[0][0]
-    generated_audio = output.images[0][1]
+    generated_audio = output[0].images[0][1]
     try:
         from dreamid_omni.utils.io_utils import save_video
     except Exception as e:

examples/online_serving/omnivoice/README.md

@@ -1,131 +0,0 @@
-# OmniVoice
-
-## Model Overview
-
-| Model | Description |
-|-------|-------------|
-| `k2-fsa/OmniVoice` | Zero-shot multilingual TTS (600+ languages) with diffusion language model (Qwen3-0.6B backbone) |
-
-> **Note:** Requires `transformers>=5.3.0` for voice cloning (HiggsAudioV2 tokenizer). Auto voice and voice design work with `transformers>=4.57.0`.
-
-## Launch the Server
-
-```bash
-vllm serve k2-fsa/OmniVoice \
-    --omni \
-    --port 8091 \
-    --trust-remote-code
-```
-
-Or use the convenience script:
-
-```bash
-./run_server.sh
-```
-
-## Send TTS Request
-
-### Using curl
-
-```bash
-# Basic TTS (auto voice)
-curl -X POST http://localhost:8091/v1/audio/speech \
-    -H "Content-Type: application/json" \
-    -d '{
-        "input": "Hello, how are you?",
-        "voice": "default",
-        "response_format": "wav"
-    }' --output output.wav
-```
-
-### Using Python
-
-```python
-import httpx
-
-response = httpx.post(
-    "http://localhost:8091/v1/audio/speech",
-    json={
-        "input": "Hello, how are you?",
-        "voice": "default",
-        "response_format": "wav",
-    },
-    timeout=300.0,
-)
-
-with open("output.wav", "wb") as f:
-    f.write(response.content)
-```
-
-### Using OpenAI SDK
-
-```python
-from openai import OpenAI
-
-client = OpenAI(base_url="http://localhost:8091/v1", api_key="none")
-
-response = client.audio.speech.create(
-    model="k2-fsa/OmniVoice",
-    voice="default",
-    input="Hello, how are you?",
-)
-
-response.stream_to_file("output.wav")
-```
-
-### Using the CLI Client
-
-```bash
-cd examples/online_serving/omnivoice
-
-# Basic TTS
-python speech_client.py --text "Hello, how are you?"
-
-# Specify language for improved quality
-python speech_client.py --text "Bonjour, comment allez-vous?" --language French
-```
-
-The CLI client supports:
-
-- `--api-base`: API base URL (default: `http://localhost:8091`)
-- `--model` (or `-m`): Model name (default: `k2-fsa/OmniVoice`)
-- `--text`: Text to synthesize (required)
-- `--response-format`: Audio format: wav, mp3, flac, pcm, aac, opus (default: wav)
-- `--language`: Language hint (default: Auto)
-- `--output` (or `-o`): Output file path (default: `omnivoice_output.wav`)
-
-## Inference Modes
-
-OmniVoice supports three inference modes. Currently, **auto voice** is supported through the online Speech API. Voice cloning and voice design are available via offline inference (see `examples/offline_inference/omnivoice/`).
-
-| Mode | Description | Online API | Offline |
-|------|-------------|:----------:|:-------:|
-| Auto Voice | Generate speech without reference | Yes | Yes |
-| Voice Clone | Clone from reference audio + transcript | - | Yes |
-| Voice Design | Control style via natural language instruction | - | Yes |
-
-## Architecture
-
-OmniVoice uses a single-stage diffusion pipeline:
-
-- **Stage 0 (Generator)**: Qwen3-0.6B transformer with 32-step iterative masked unmasking and classifier-free guidance. Generates 8-codebook audio tokens from text, then decodes to 24kHz waveform via HiggsAudioV2 RVQ quantizer + DAC acoustic decoder.
-
-## API Parameters
-
-OmniVoice uses the standard `/v1/audio/speech` endpoint. See the [Speech API reference](https://docs.vllm.ai/projects/vllm-omni/en/latest/serving/speech_api/) for full documentation.
-
-Key parameters:
-
-| Parameter | Type | Default | Description |
-|-----------|------|---------|-------------|
-| `input` | string | **required** | Text to synthesize |
-| `voice` | string | "default" | Voice name |
-| `response_format` | string | "wav" | Audio format: wav, mp3, flac, pcm, aac, opus |
-| `speed` | float | 1.0 | Playback speed (0.25-4.0) |
-
-## Troubleshooting
-
-1. **TTS model did not produce audio output**: Ensure the model is fully downloaded (`huggingface-cli download k2-fsa/OmniVoice`)
-2. **Connection refused**: Make sure the server is running on the correct port
-3. **Out of memory**: Reduce `--gpu-memory-utilization` (default stage config uses 0.5)
-4. **Slow first request**: The model performs warmup on first inference; subsequent requests are faster

examples/online_serving/omnivoice/run_server.sh

@@ -1,19 +0,0 @@
-#!/bin/bash
-# Launch vLLM-Omni server for OmniVoice TTS
-#
-# Usage:
-#   ./run_server.sh
-#   CUDA_VISIBLE_DEVICES=0 ./run_server.sh
-
-set -e
-
-MODEL="${MODEL:-k2-fsa/OmniVoice}"
-PORT="${PORT:-8091}"
-
-echo "Starting OmniVoice server with model: $MODEL"
-
-vllm serve "$MODEL" \
-    --host 0.0.0.0 \
-    --port "$PORT" \
-    --trust-remote-code \
-    --omni

examples/online_serving/omnivoice/speech_client.py

@@ -1,84 +0,0 @@
-"""Client for OmniVoice TTS via /v1/audio/speech endpoint.
-
-Examples:
-    # Basic TTS (auto voice)
-    python speech_client.py --text "Hello, how are you?"
-
-    # Specify language
-    python speech_client.py --text "Bonjour, comment allez-vous?" --language French
-"""
-
-import argparse
-
-import httpx
-
-DEFAULT_API_BASE = "http://localhost:8091"
-DEFAULT_API_KEY = "EMPTY"
-
-
-def run_tts(args) -> None:
-    """Generate speech via /v1/audio/speech API."""
-    payload = {
-        "model": args.model,
-        "input": args.text,
-        "voice": "default",
-        "response_format": args.response_format,
-    }
-
-    if args.language:
-        payload["language"] = args.language
-
-    print(f"Model: {args.model}")
-    print(f"Text: {args.text}")
-    if args.language:
-        print(f"Language: {args.language}")
-    print("Generating audio...")
-
-    api_url = f"{args.api_base}/v1/audio/speech"
-    headers = {
-        "Content-Type": "application/json",
-        "Authorization": f"Bearer {args.api_key}",
-    }
-
-    with httpx.Client(timeout=300.0) as client:
-        response = client.post(api_url, json=payload, headers=headers)
-
-    if response.status_code != 200:
-        print(f"Error: {response.status_code}")
-        print(response.text)
-        return
-
-    try:
-        text = response.content.decode("utf-8")
-        if text.startswith('{"error"'):
-            print(f"Error: {text}")
-            return
-    except UnicodeDecodeError:
-        pass
-
-    output_path = args.output or "omnivoice_output.wav"
-    with open(output_path, "wb") as f:
-        f.write(response.content)
-    print(f"Audio saved to: {output_path}")
-
-
-def main():
-    parser = argparse.ArgumentParser(description="OmniVoice TTS client")
-    parser.add_argument("--api-base", default=DEFAULT_API_BASE, help="API base URL")
-    parser.add_argument("--api-key", default=DEFAULT_API_KEY, help="API key")
-    parser.add_argument("--model", "-m", default="k2-fsa/OmniVoice", help="Model name")
-    parser.add_argument("--text", required=True, help="Text to synthesize")
-    parser.add_argument("--language", default=None, help="Language hint (e.g., English, Chinese, French)")
-    parser.add_argument(
-        "--response-format",
-        default="wav",
-        choices=["wav", "mp3", "flac", "pcm", "aac", "opus"],
-        help="Audio format (default: wav)",
-    )
-    parser.add_argument("--output", "-o", default=None, help="Output file path")
-    args = parser.parse_args()
-    run_tts(args)
-
-
-if __name__ == "__main__":
-    main()

examples/online_serving/qwen3_omni/README.md

@@ -36,45 +36,6 @@ cd examples/online_serving/qwen3_omni
 python examples/online_serving/openai_chat_completion_client_for_multimodal_generation.py --model Qwen/Qwen3-Omni-30B-A3B-Instruct --query-type use_image --port 8091 --host "localhost"
 ```
 
-#### Realtime WebSocket client (`openai_realtime_client.py`)
-
-[`openai_realtime_client.py`](./openai_realtime_client.py) connects to **`ws://<host>:<port>/v1/realtime`**, uploads a local audio file as **PCM16 mono @ 16 kHz** chunks (OpenAI-style `input_audio_buffer.append` / `commit`), and prints **streaming transcription** (`transcription.delta` / `transcription.done`).
-
-**Dependencies:**
-
-```bash
-pip install websockets librosa numpy
-```
-
-(ffmpeg may be required by `librosa` for some formats; see the FAQ below.)
-
-**From this directory** (`examples/online_serving/qwen3_omni`):
-
-```bash
-python openai_realtime_client.py \
-  --host localhost \
-  --port 8091 \
-  --model Qwen/Qwen3-Omni-30B-A3B-Instruct \
-  --audio_path /path/to/your.wav
-```
-
-If `--audio_path` is omitted, the script uses a bundled default clip (`mary_had_lamb` via vLLM assets).
-
-**Arguments:**
-
-| Flag | Default | Description |
-|------|---------|-------------|
-| `--host` | `localhost` | API server host |
-| `--port` | `8000` | API server port (match your `vllm serve` port, e.g. `8091`) |
-| `--model` | `Qwen/Qwen3-Omni-30B-A3B-Instruct` | Must match the served model (also sent in `session.update`) |
-| `--audio_path` | *(optional)* | Path to input audio; resampled to 16 kHz mono inside the client |
-
-Ensure the vLLM-Omni server is running with realtime support for this endpoint, for example:
-
-```bash
-vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091
-```
-
 The Python client supports the following command-line arguments:
 
 - `--query-type` (or `-q`): Query type (default: `use_video`). Options: `text`, `use_audio`, `use_image`, `use_video`

examples/online_serving/qwen3_omni/openai_realtime_client.py

@@ -1,146 +0,0 @@
-"""
-This script demonstrates how to use the vLLM-Omni Realtime WebSocket API to perform
-audio transcription by uploading an audio file.
-
-Before running this script, you must start the vLLM-Omni server with a realtime-capable
-model, for example:
-
-    vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni
-
-Requirements:
-- vllm with audio support
-- websockets
-- librosa
-- numpy
-
-The script:
-1. Connects to the Realtime WebSocket endpoint
-2. Converts an audio file to PCM16 @ 16kHz
-3. Sends audio chunks to the server
-4. Receives and prints transcription as it streams
-"""
-
-import argparse
-import asyncio
-import base64
-import json
-
-import librosa
-import numpy as np
-import websockets
-from vllm.assets.audio import AudioAsset
-
-
-def audio_to_pcm16_base64(audio_path: str) -> str:
-    """
-    Load an audio file and convert it to base64-encoded PCM16 @ 16kHz.
-    """
-    # Load audio and resample to 16kHz mono
-    audio, _ = librosa.load(audio_path, sr=16000, mono=True)
-    # Convert to PCM16
-    pcm16 = (audio * 32767).astype(np.int16)
-    # Encode as base64
-    return base64.b64encode(pcm16.tobytes()).decode("utf-8")
-
-
-async def realtime_transcribe(audio_path: str, host: str, port: int, model: str):
-    """
-    Connect to the Realtime API and transcribe an audio file.
-    """
-    uri = f"ws://{host}:{port}/v1/realtime"
-
-    async with websockets.connect(uri) as ws:
-        # Wait for session.created
-        response = json.loads(await ws.recv())
-        if response["type"] == "session.created":
-            print(f"Session created: {response['id']}")
-        else:
-            print(f"Unexpected response: {response}")
-            return
-
-        # Validate model
-        await ws.send(json.dumps({"type": "session.update", "model": model}))
-
-        # Signal ready to start
-        await ws.send(json.dumps({"type": "input_audio_buffer.commit"}))
-
-        # Convert audio file to base64 PCM16
-        print(f"Loading audio from: {audio_path}")
-        audio_base64 = audio_to_pcm16_base64(audio_path)
-
-        # Send audio in chunks (4KB of raw audio = ~8KB base64)
-        chunk_size = 4096
-        audio_bytes = base64.b64decode(audio_base64)
-        total_chunks = (len(audio_bytes) + chunk_size - 1) // chunk_size
-
-        print(f"Sending {total_chunks} audio chunks...")
-        for i in range(0, len(audio_bytes), chunk_size):
-            chunk = audio_bytes[i : i + chunk_size]
-            await ws.send(
-                json.dumps(
-                    {
-                        "type": "input_audio_buffer.append",
-                        "audio": base64.b64encode(chunk).decode("utf-8"),
-                    }
-                )
-            )
-
-        # Signal all audio is sent
-        await ws.send(json.dumps({"type": "input_audio_buffer.commit", "final": True}))
-        print("Audio sent. Waiting for transcription...\n")
-
-        # Receive transcription
-        print("Transcription: ", end="", flush=True)
-        while True:
-            response = json.loads(await ws.recv())
-            if response["type"] == "transcription.delta":
-                print(response["delta"], end="", flush=True)
-            elif response["type"] == "transcription.done":
-                print(f"\n\nFinal transcription: {response['text']}")
-                if response.get("usage"):
-                    print(f"Usage: {response['usage']}")
-                break
-            elif response["type"] == "error":
-                print(f"\nError: {response['error']}")
-                break
-
-
-def main(args):
-    if args.audio_path:
-        audio_path = args.audio_path
-    else:
-        # Use default audio asset
-        audio_path = str(AudioAsset("mary_had_lamb").get_local_path())
-        print(f"No audio path provided, using default: {audio_path}")
-
-    asyncio.run(realtime_transcribe(audio_path, args.host, args.port, args.model))
-
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser(description="Realtime WebSocket Transcription Client")
-    parser.add_argument(
-        "--model",
-        type=str,
-        default="Qwen/Qwen3-Omni-30B-A3B-Instruct",
-        help="Model that is served and should be pinged.",
-    )
-    parser.add_argument(
-        "--audio_path",
-        type=str,
-        default=None,
-        help="Path to the audio file to transcribe.",
-    )
-    parser.add_argument(
-        "--host",
-        type=str,
-        default="localhost",
-        help="vLLM-Omni server host (default: localhost)",
-    )
-    parser.add_argument(
-        "--port",
-        type=int,
-        default=8000,
-        help="vLLM-Omni server port (default: 8000)",
-    )
-    args = parser.parse_args()
-    main(args)

examples/online_serving/qwen3_tts/README.md

@@ -233,7 +233,6 @@ Upload a new voice sample for voice cloning in Base task TTS requests.
 - `consent` (required): Consent recording ID
 - `name` (required): Name for the new voice
 - `ref_text` (optional): Transcript of the audio. Enables in-context voice cloning (higher quality).
-- `speaker_description` (optional): Free-form description of the voice (e.g. "warm narrator", "energetic presenter").
 
 **Response Example:**
 ```json
@@ -244,23 +243,18 @@ Upload a new voice sample for voice cloning in Base task TTS requests.
     "consent": "user_consent_id",
     "created_at": 1738660000,
     "mime_type": "audio/wav",
-    "file_size": 1024000,
+    "file_size": 1024000
-    "ref_text": "The exact transcript of the audio sample.",
-    "speaker_description": "warm narrator"
   }
 }
 ```
 
-Fields `ref_text` and `speaker_description` are omitted when not provided at upload time.
-
 **Usage Example:**
 ```bash
 curl -X POST http://localhost:8000/v1/audio/voices \
   -F "audio_sample=@/path/to/voice_sample.wav" \
   -F "consent=user_consent_id" \
   -F "name=custom_voice_1" \
-  -F "ref_text=The exact transcript of the audio sample." \
+  -F "ref_text=The exact transcript of the audio sample."
-  -F "speaker_description=warm narrator"
 ```
 
 ### Endpoint

examples/online_serving/qwen3_tts/openai_speech_client.py

@@ -71,7 +71,7 @@ def run_tts_generation(args) -> None:
     payload = {
         "model": args.model,
         "input": args.text,
-        "voice": args.speaker,
+        "speaker": args.speaker,
         "response_format": args.response_format,
     }
 

examples/online_serving/text_to_image/README.md

@@ -231,8 +231,6 @@ count, use `size` and `n` rather than `height`, `width`, or
 | `seed`                   | int   | None    | Random seed (reproducible)     |
 | `negative_prompt`        | str   | None    | Negative prompt                |
 | `num_outputs_per_prompt` | int   | 1       | Number of images to generate   |
-| `use_system_prompt` | str | None | System prompt preset: `en_unified`, `en_vanilla`, `en_recaption`, `en_think_recaption`, `dynamic`, `None`, or custom text string. Only for HunyuanImage-3.0. |
-| `system_prompt` | str | None | Custom system prompt text. Only used when `use_system_prompt` is set to `custom`. Only for HunyuanImage-3.0. |
 
 ## Response Format
 

examples/online_serving/text_to_image/openai_chat_client.py

@@ -28,8 +28,6 @@ def generate_image(
     lora_name: str | None = None,
     lora_scale: float | None = None,
     lora_int_id: int | None = None,
-    use_system_prompt: str | None = None,
-    system_prompt: str | None = None,
 ) -> bytes | None:
     """Generate an image using the images generation API.
 
@@ -47,8 +45,6 @@ def generate_image(
         lora_name: LoRA name (optional, defaults to path stem)
         lora_scale: LoRA scale factor (default: 1.0)
         lora_int_id: LoRA integer ID (optional, derived from path if not provided)
-        use_system_prompt: System prompt for generation.
-        system_prompt: Custom system prompt.
 
     Returns:
         Image bytes or None if failed
@@ -74,10 +70,7 @@ def generate_image(
         payload["negative_prompt"] = negative_prompt
     if seed is not None:
         payload["seed"] = seed
-    if use_system_prompt is not None:
+
-        payload["use_system_prompt"] = use_system_prompt
-    if system_prompt is not None:
-        payload["system_prompt"] = system_prompt
     # Add LoRA if provided
     if lora_path:
         lora_body: dict = {
@@ -135,21 +128,9 @@ def main():
         default=None,
         help="LoRA integer id (cache key). If omitted, the server derives a stable id from lora_path.",
     )
-    parser.add_argument(
+
-        "--use-system-prompt",
-        type=str,
-        default=None,
-        help=(
-            "System prompt for generation. Use predefined types: 'en_unified', 'en_vanilla', 'en_recaption', 'en_think_recaption', 'dynamic', or 'None'; Or provide custom text string directly. Recommended en_unified. "
-        ),
-    )
-    parser.add_argument(
-        "--system-prompt",
-        type=str,
-        default=None,
-        help=("Custom system prompt. Used when --use-system-prompt is custom. "),
-    )
     args = parser.parse_args()
+
     print(f"Generating image for: {args.prompt}")
 
     image_bytes = generate_image(
@@ -165,8 +146,6 @@ def main():
         lora_name=args.lora_name,
         lora_scale=args.lora_scale if args.lora_path else None,
         lora_int_id=args.lora_int_id if args.lora_path else None,
-        use_system_prompt=args.use_system_prompt,
-        system_prompt=args.system_prompt,
     )
 
     if image_bytes:

pyproject.toml

@@ -223,4 +223,3 @@ extend-ignore-identifiers-re = [
 ue = "ue"
 semantics = "semantics"
 fullset = "fullset"
-Vai = "Vai"

requirements/common.txt

@@ -1,5 +1,4 @@
 # Common dependencies for all platforms
-av>=14.0.0
 omegaconf>=2.3.0
 librosa>=0.11.0
 resampy>=0.4.3

tests/conftest.py

@@ -1146,6 +1146,18 @@ def convert_audio_bytes_to_text(raw_bytes: bytes) -> str:
     return text
 
 
+def merge_base64_and_convert_to_text(base64_list):
+    """
+    Merge a list of base64 encoded audio data and convert to text.
+    """
+    merged_audio = _merge_base64_audio_to_segment(base64_list)
+    output_path = f"./test_{uuid.uuid4().hex}.wav"
+    merged_audio.export(output_path, format="wav")
+    print(f"audio data is saved: {output_path}")
+    text = convert_audio_file_to_text(output_path)
+    return text
+
+
 def modify_stage_config(
     yaml_path: str,
     updates: dict[str, Any] = None,
@@ -1730,7 +1742,7 @@ def _estimate_voice_gender_from_audio(audio_bytes: bytes) -> str:
         label = str(top.get("label", "")).lower()
         conf = float(top.get("score", 0.0))
 
-        if conf < 0.5:
+        if conf < 0.6:
             gender = "unknown"
         # Some models use non-English labels (e.g., Russian). Normalize to 'male'/'female'.
         elif ("female" in label) or ("жен" in label):
@@ -1759,34 +1771,6 @@ def _estimate_voice_gender_from_audio(audio_bytes: bytes) -> str:
         return "unknown"
 
 
-_PRESET_VOICE_GENDER_MAP: dict[str, str] = {
-    "serena": "female",
-    "uncle_fu": "male",
-    "chelsie": "female",
-    "clone": "female",
-    "ethan": "male",
-}
-
-
-def _assert_preset_voice_gender_from_audio(
-    audio_bytes: bytes | None,
-    voice_name: str | None,
-) -> None:
-    """If ``voice_name`` matches a known preset, assert classifier gender matches (skip when unknown)."""
-    if not voice_name or not audio_bytes:
-        return
-    key = str(voice_name).lower()
-    expected_gender = _PRESET_VOICE_GENDER_MAP.get(key)
-    if expected_gender is None:
-        return
-    estimated_gender = _estimate_voice_gender_from_audio(audio_bytes)
-    print(f"Preset voice gender check: preset={key!r}, estimated={estimated_gender!r}, expected={expected_gender!r}")
-    if estimated_gender != "unknown":
-        assert estimated_gender == expected_gender, (
-            f"{voice_name!r} is expected {expected_gender}, but estimated gender is {estimated_gender!r}"
-        )
-
-
 # Threshold aligned with _compute_pcm_hnr_db docstring (clean clone vs distorted).
 _MIN_PCM_SPEECH_HNR_DB = 1.0
 
@@ -1853,12 +1837,6 @@ def assert_omni_response(response: OmniResponse, request_config: dict[str, Any],
         if "audio" in modalities:
             assert response.audio_content is not None, "No audio output is generated"
             print(f"audio content is: {response.audio_content}")
-            speaker = request_config.get("speaker")
-            if speaker:
-                _assert_preset_voice_gender_from_audio(
-                    response.audio_bytes,
-                    speaker,
-                )
 
         if "text" in modalities:
             assert response.text_content is not None, "No text output is generated"
@@ -1871,14 +1849,12 @@ def assert_omni_response(response: OmniResponse, request_config: dict[str, Any],
             keywords = keywords_dict.get(word_type)
             if "text" in modalities:
                 if keywords:
-                    text_lower = response.text_content.lower()
+                    assert any(keyword in response.text_content.lower() for keyword in keywords), (
-                    assert any(str(kw).lower() in text_lower for kw in keywords), (
                         "The output does not contain any of the keywords."
                     )
             else:
                 if keywords:
-                    audio_lower = response.audio_content.lower()
+                    assert any(keyword in response.audio_content.lower() for keyword in keywords), (
-                    assert any(str(kw).lower() in audio_lower for kw in keywords), (
                         "The output does not contain any of the keywords."
                     )
 
@@ -1932,11 +1908,23 @@ def assert_audio_speech_response(
                 f"Transcript doesn't match input: similarity={similarity:.2f}, transcript='{transcript}'"
             )
 
-        # Voice gender consistency check (preset names in ``_PRESET_VOICE_GENDER_MAP``).
+        # Voice gender consistency check:
         # When the estimator returns 'unknown', we treat it as inconclusive and do NOT fail the test.
-        _assert_preset_voice_gender_from_audio(
+        voice = (request_config.get("voice") or "").lower()
-            response.audio_bytes,
+        if voice and response.audio_bytes:
-            request_config.get("voice"),
+            estimated_gender = _estimate_voice_gender_from_audio(response.audio_bytes)
+            voice_gender_map = {
+                # adjust this mapping to your actual voice names
+                "serena": "female",
+                "uncle_fu": "male",
+                "clone": "female",
+            }
+            expected_gender = voice_gender_map.get(voice)
+            if expected_gender is not None:
+                print(f"Estimated voice gender from audio: {estimated_gender} (voice='{voice}')")
+                if estimated_gender != "unknown":
+                    assert estimated_gender == expected_gender, (
+                        f"Voice '{voice}' is expected {expected_gender}, but estimated gender is '{estimated_gender}'"
                     )
 
 
@@ -2053,11 +2041,7 @@ class OpenAIClientHandler:
 
             if audio_data or text_content:
                 if audio_data:
-                    merged_seg = _merge_base64_audio_to_segment(audio_data)
+                    audio_content = merge_base64_and_convert_to_text(audio_data)
-                    wav_buf = BytesIO()
-                    merged_seg.export(wav_buf, format="wav")
-                    result.audio_bytes = wav_buf.getvalue()
-                    audio_content = convert_audio_bytes_to_text(result.audio_bytes)
                 if audio_content and text_content:
                     similarity = cosine_similarity_text(audio_content.lower(), text_content.lower())
 
@@ -2112,8 +2096,7 @@ class OpenAIClientHandler:
 
             if audio_data or text_content:
                 if audio_data:
-                    result.audio_bytes = base64.b64decode(audio_data)
+                    audio_content = convert_audio_to_text(audio_data)
-                    audio_content = convert_audio_bytes_to_text(result.audio_bytes)
                 if audio_content and text_content:
                     similarity = cosine_similarity_text(audio_content.lower(), text_content.lower())
 
@@ -2282,9 +2265,8 @@ class OpenAIClientHandler:
             request_config: Request configuration dictionary containing parameters like model, messages, stream.
                 Optional ``use_audio_in_video`` (bool): when true, sets
                 ``extra_body["mm_processor_kwargs"] = {"use_audio_in_video": True}`` for Qwen-Omni video+audio
-                extraction.
+                extraction (merged with any existing ``extra_body`` / ``mm_processor_kwargs``).
-                Optional top-level ``speaker`` (str): Qwen3-Omni preset TTS speaker name; sent as
+                Optional ``extra_body`` (dict): passed through to ``chat.completions.create`` after merge.
-                ``extra_body["speaker"]`` to ``chat.completions.create``.
             request_num: Number of requests, defaults to 1 (single request)
 
         Returns:
@@ -2296,8 +2278,9 @@ class OpenAIClientHandler:
         modalities = request_config.get("modalities", ["text", "audio"])
 
         extra_body: dict[str, Any] = {}
-        if "speaker" in request_config:
+        raw_extra = request_config.get("extra_body")
-            extra_body["speaker"] = request_config["speaker"]
+        if raw_extra:
+            extra_body.update(raw_extra)
         if request_config.get("use_audio_in_video"):
             mm = dict(extra_body.get("mm_processor_kwargs") or {})
             mm["use_audio_in_video"] = True
@@ -2329,15 +2312,12 @@ class OpenAIClientHandler:
             # Send concurrent requests: run create + process in worker so e2e_latency includes full round-trip.
             def _one_omni_request():
                 start = time.perf_counter()
-                worker_kwargs: dict[str, Any] = {
+                chat_completion = self.client.chat.completions.create(
-                    "model": request_config.get("model"),
+                    model=request_config.get("model"),
-                    "messages": request_config.get("messages"),
+                    messages=request_config.get("messages"),
-                    "modalities": modalities,
+                    modalities=modalities,
-                    "stream": stream,
+                    stream=stream,
-                }
+                )
-                if extra_body_arg is not None:
-                    worker_kwargs["extra_body"] = extra_body_arg
-                chat_completion = self.client.chat.completions.create(**worker_kwargs)
                 if stream:
                     response = self._process_stream_omni_response(chat_completion)
                 else:

tests/dfx/perf/stage_configs/qwen3_tts.yaml

@@ -88,7 +88,7 @@ runtime:
         connector_get_max_wait_first_chunk: 3000
         connector_get_max_wait: 300
         codec_chunk_frames: 25
-        codec_left_context_frames: 72
+        codec_left_context_frames: 25
 
   edges:
     - from: 0

tests/diffusion/distributed/test_autoencoder_kl_wan_encode.py

@@ -1,273 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-
-"""Unit tests for DistributedAutoencoderKLWan encode parallel (CPU-only)."""
-
-import pytest
-import torch
-
-pytestmark = [pytest.mark.cpu, pytest.mark.core_model]
-
-
-class _DummyConfig:
-    def __init__(self, patch_size=None, scale_factor_temporal=4):
-        self.patch_size = patch_size
-        self.scale_factor_temporal = scale_factor_temporal
-
-
-class _DummyWanVae:
-    """Minimal mock of DistributedAutoencoderKLWan for testing encode_tile_split."""
-
-    def __init__(
-        self,
-        config=None,
-        spatial_compression_ratio=8,
-        tile_sample_min_height=256,
-        tile_sample_min_width=256,
-        tile_sample_stride_height=192,
-        tile_sample_stride_width=192,
-    ):
-        self.config = config or _DummyConfig()
-        self.spatial_compression_ratio = spatial_compression_ratio
-        self.tile_sample_min_height = tile_sample_min_height
-        self.tile_sample_min_width = tile_sample_min_width
-        self.tile_sample_stride_height = tile_sample_stride_height
-        self.tile_sample_stride_width = tile_sample_stride_width
-        self.dtype = torch.float32
-
-        # Mock caches
-        self._enc_feat_map = None
-        self._enc_conv_idx = [0]
-
-    def clear_cache(self):
-        self._enc_feat_map = None
-        self._enc_conv_idx = [0]
-
-    def encoder(self, x, feat_cache=None, feat_idx=None):  # noqa: ARG002
-        # Simple mock: just return the input
-        return x
-
-    def quant_conv(self, x):
-        return x
-
-    def blend_v(self, _a, b, _blend_extent):
-        return b
-
-    def blend_h(self, _a, b, _blend_extent):
-        return b
-
-
-def _import_encode_tile_split():
-    """Import the encode_tile_split method from the module."""
-    from vllm_omni.diffusion.distributed.autoencoders.autoencoder_kl_wan import (
-        DistributedAutoencoderKLWan,
-    )
-
-    return DistributedAutoencoderKLWan.encode_tile_split
-
-
-def _import_encode_tile_exec():
-    from vllm_omni.diffusion.distributed.autoencoders.autoencoder_kl_wan import (
-        DistributedAutoencoderKLWan,
-    )
-
-    return DistributedAutoencoderKLWan.encode_tile_exec
-
-
-def _import_encode_tile_merge():
-    from vllm_omni.diffusion.distributed.autoencoders.autoencoder_kl_wan import (
-        DistributedAutoencoderKLWan,
-    )
-
-    return DistributedAutoencoderKLWan.encode_tile_merge
-
-
-class TestEncodeTileSplit:
-    """Tests for encode_tile_split method."""
-
-    def test_basic_split_without_patch_size(self):
-        """Test basic tile splitting without patch_size."""
-        encode_tile_split = _import_encode_tile_split()
-
-        vae = _DummyWanVae(
-            config=_DummyConfig(patch_size=None, scale_factor_temporal=4),
-            spatial_compression_ratio=8,
-            tile_sample_min_height=256,
-            tile_sample_min_width=256,
-            tile_sample_stride_height=192,
-            tile_sample_stride_width=192,
-        )
-
-        # Input: (B, C, T, H, W) = (1, 3, 5, 256, 256)
-        x = torch.randn(1, 3, 5, 256, 256)
-
-        tiletask_list, grid_spec = encode_tile_split(vae, x)
-
-        # With stride 192 and input size 256, we should get:
-        # Height: ceil(256/192) = 2 positions (0, 192) but 192+256 > 256, so only 1
-        # Actually for i in range(0, 256, 192): i = 0, 192 but 192 is out of bounds
-        # So we get 1x1 grid
-        assert len(tiletask_list) >= 1
-        assert grid_spec.grid_shape[0] >= 1
-        assert grid_spec.grid_shape[1] >= 1
-
-        # Check temporal chunking: 5 frames -> 1 + (5-1)//4 = 2 chunks
-        first_task = tiletask_list[0]
-        assert len(first_task.tensor) == 2  # 2 temporal chunks
-
-    def test_split_with_patch_size_scales_coordinates(self):
-        """Test that patch_size properly scales tile coordinates."""
-        encode_tile_split = _import_encode_tile_split()
-
-        # Without patch_size
-        vae_no_patch = _DummyWanVae(
-            config=_DummyConfig(patch_size=None, scale_factor_temporal=4),
-            spatial_compression_ratio=8,
-            tile_sample_min_height=256,
-            tile_sample_min_width=256,
-            tile_sample_stride_height=128,
-            tile_sample_stride_width=128,
-        )
-
-        # With patch_size=2 (simulating patchified input)
-        vae_with_patch = _DummyWanVae(
-            config=_DummyConfig(patch_size=2, scale_factor_temporal=4),
-            spatial_compression_ratio=8,
-            tile_sample_min_height=256,
-            tile_sample_min_width=256,
-            tile_sample_stride_height=128,
-            tile_sample_stride_width=128,
-        )
-
-        # Same patchified input size
-        x = torch.randn(1, 3, 5, 256, 256)
-
-        tasks_no_patch, _ = encode_tile_split(vae_no_patch, x)
-        tasks_with_patch, _ = encode_tile_split(vae_with_patch, x)
-
-        # With patch_size=2, stride becomes 128//2=64, so more tiles
-        assert len(tasks_with_patch) >= len(tasks_no_patch)
-
-    def test_temporal_compression_from_config(self):
-        """Test that temporal compression ratio is read from config."""
-        encode_tile_split = _import_encode_tile_split()
-
-        # temporal_compression=4 (default)
-        vae_4x = _DummyWanVae(
-            config=_DummyConfig(scale_factor_temporal=4),
-            tile_sample_min_height=512,
-            tile_sample_min_width=512,
-            tile_sample_stride_height=512,
-            tile_sample_stride_width=512,
-        )
-
-        # temporal_compression=2
-        vae_2x = _DummyWanVae(
-            config=_DummyConfig(scale_factor_temporal=2),
-            tile_sample_min_height=512,
-            tile_sample_min_width=512,
-            tile_sample_stride_height=512,
-            tile_sample_stride_width=512,
-        )
-
-        # 9 frames input
-        x = torch.randn(1, 3, 9, 512, 512)
-
-        tasks_4x, _ = encode_tile_split(vae_4x, x)
-        tasks_2x, _ = encode_tile_split(vae_2x, x)
-
-        # With 4x compression: 1 + (9-1)//4 = 3 chunks
-        assert len(tasks_4x[0].tensor) == 3
-
-        # With 2x compression: 1 + (9-1)//2 = 5 chunks
-        assert len(tasks_2x[0].tensor) == 5
-
-    def test_grid_spec_latent_dimensions(self):
-        """Test that grid_spec contains correct latent dimensions."""
-        encode_tile_split = _import_encode_tile_split()
-
-        vae = _DummyWanVae(
-            config=_DummyConfig(patch_size=None),
-            spatial_compression_ratio=8,
-            tile_sample_min_height=512,
-            tile_sample_min_width=512,
-            tile_sample_stride_height=512,
-            tile_sample_stride_width=512,
-        )
-
-        # Input: 512x512 with compression 8 -> 64x64 latent
-        x = torch.randn(1, 3, 5, 512, 512)
-
-        _, grid_spec = encode_tile_split(vae, x)
-
-        assert grid_spec.tile_spec["latent_height"] == 64
-        assert grid_spec.tile_spec["latent_width"] == 64
-
-
-class TestEncodeTileExec:
-    """Tests for encode_tile_exec method."""
-
-    def test_basic_exec(self):
-        """Test basic tile execution."""
-        encode_tile_exec = _import_encode_tile_exec()
-
-        vae = _DummyWanVae()
-
-        from vllm_omni.diffusion.distributed.autoencoders.distributed_vae_executor import (
-            TileTask,
-        )
-
-        # Create a simple task with 2 temporal chunks
-        tile1 = torch.randn(1, 3, 1, 32, 32)
-        tile2 = torch.randn(1, 3, 4, 32, 32)
-        task = TileTask(tile_id=0, grid_coord=(0, 0), tensor=[tile1, tile2])
-
-        result = encode_tile_exec(vae, task)
-
-        # Result should concatenate temporal dimension
-        assert result.shape[2] == 5  # 1 + 4 frames
-
-
-class TestEncodeTileMerge:
-    """Tests for encode_tile_merge method."""
-
-    def test_basic_merge(self):
-        """Test basic tile merging."""
-        encode_tile_merge = _import_encode_tile_merge()
-
-        vae = _DummyWanVae()
-
-        from vllm_omni.diffusion.distributed.autoencoders.distributed_vae_executor import (
-            GridSpec,
-        )
-
-        # Create 2x2 grid of tiles
-        tile_00 = torch.ones(1, 16, 2, 32, 32) * 0
-        tile_01 = torch.ones(1, 16, 2, 32, 32) * 1
-        tile_10 = torch.ones(1, 16, 2, 32, 32) * 2
-        tile_11 = torch.ones(1, 16, 2, 32, 32) * 3
-
-        coord_tensor_map = {
-            (0, 0): tile_00,
-            (0, 1): tile_01,
-            (1, 0): tile_10,
-            (1, 1): tile_11,
-        }
-
-        grid_spec = GridSpec(
-            split_dims=(3, 4),
-            grid_shape=(2, 2),
-            tile_spec={
-                "latent_height": 48,
-                "latent_width": 48,
-                "blend_height": 8,
-                "blend_width": 8,
-                "tile_latent_stride_height": 24,
-                "tile_latent_stride_width": 24,
-            },
-        )
-
-        result = encode_tile_merge(vae, coord_tensor_map, grid_spec)
-
-        # Output should be (1, 16, 2, 48, 48)
-        assert result.shape == (1, 16, 2, 48, 48)

tests/diffusion/distributed/test_distributed_vae_executor.py

@@ -59,9 +59,9 @@ class E2EOperator:
 class DummyMixin(DistributedVaeMixin):
     def __init__(self):
         self.use_tiling = True
-        self.distributed_executor = MagicMock()
+        self.distributed_decoder = MagicMock()
-        self.distributed_executor.parallel_size = 2
+        self.distributed_decoder.parallel_size = 2
-        self.distributed_executor.group = None
+        self.distributed_decoder.group = None
 
 
 @pytest.fixture(autouse=True)

tests/diffusion/layers/test_adalayernorm.py

@@ -1,237 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-"""Unit tests for shared AdaLayerNorm layers used by FLUX and other models."""
-
-import os
-
-import pytest
-import torch
-
-pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cpu]
-
-
-@pytest.fixture(autouse=True)
-def _init_distributed():
-    """Initialize the minimal distributed environment required by
-    ReplicatedLinear (tensor-parallel group must exist)."""
-    from vllm.distributed.parallel_state import (
-        cleanup_dist_env_and_memory,
-        init_distributed_environment,
-        initialize_model_parallel,
-    )
-
-    os.environ.setdefault("MASTER_ADDR", "localhost")
-    os.environ.setdefault("MASTER_PORT", "29501")
-    init_distributed_environment(
-        world_size=1,
-        rank=0,
-        local_rank=0,
-        distributed_init_method="env://",
-    )
-    initialize_model_parallel()
-    yield
-    cleanup_dist_env_and_memory()
-
-
-@pytest.fixture(autouse=True)
-def _force_default_gemm(monkeypatch):
-    """Force CPU-compatible GEMM dispatch for tests using CPU tensors.
-
-    vLLM's dispatch_unquantized_gemm() selects the backend by platform
-    (e.g. rocm_unquantized_gemm on AMD machines), not by tensor device.
-    CPU test tensors crash with NotImplementedError on ROCm.  Monkeypatch
-    the dispatcher to always return the default (torch.nn.functional.linear)
-    implementation which works on any device."""
-    from vllm.model_executor.layers.utils import default_unquantized_gemm
-
-    monkeypatch.setattr(
-        "vllm.model_executor.layers.linear.dispatch_unquantized_gemm",
-        lambda: default_unquantized_gemm,
-    )
-
-
-def test_adalayernorm_import_from_shared_module():
-    """Verify imports work from the shared adalayernorm module."""
-    from vllm_omni.diffusion.layers.adalayernorm import (  # noqa: F401
-        AdaLayerNormContinuous,
-        AdaLayerNormZero,
-        AdaLayerNormZeroSingle,
-    )
-
-
-def test_adalayernorm_zero_forward_shape():
-    """AdaLayerNormZero produces correct output shapes (x, gate, shift, scale, gate)."""
-    from vllm_omni.diffusion.layers.adalayernorm import AdaLayerNormZero
-
-    dim = 64
-    batch = 2
-    seq_len = 4
-    norm = AdaLayerNormZero(dim)
-
-    x = torch.randn(batch, seq_len, dim)
-    emb = torch.randn(batch, dim)
-
-    out_x, gate_msa, shift_mlp, scale_mlp, gate_mlp = norm(x, emb)
-
-    assert out_x.shape == (batch, seq_len, dim)
-    assert gate_msa.shape == (batch, dim)
-    assert shift_mlp.shape == (batch, dim)
-    assert scale_mlp.shape == (batch, dim)
-    assert gate_mlp.shape == (batch, dim)
-
-
-def test_adalayernorm_zero_single_forward_shape():
-    """AdaLayerNormZeroSingle produces (x, gate) with correct shapes."""
-    from vllm_omni.diffusion.layers.adalayernorm import AdaLayerNormZeroSingle
-
-    dim = 64
-    batch = 2
-    seq_len = 4
-    norm = AdaLayerNormZeroSingle(dim)
-
-    x = torch.randn(batch, seq_len, dim)
-    emb = torch.randn(batch, dim)
-
-    out_x, gate = norm(x, emb)
-
-    assert out_x.shape == (batch, seq_len, dim)
-    assert gate.shape == (batch, dim)
-
-
-def test_adalayernorm_continuous_forward_shape():
-    """AdaLayerNormContinuous produces correct output shape."""
-    from vllm_omni.diffusion.layers.adalayernorm import AdaLayerNormContinuous
-
-    dim = 64
-    cond_dim = 64
-    batch = 2
-    seq_len = 4
-    norm = AdaLayerNormContinuous(dim, cond_dim)
-
-    x = torch.randn(batch, seq_len, dim)
-    conditioning = torch.randn(batch, cond_dim)
-
-    out = norm(x, conditioning)
-
-    assert out.shape == (batch, seq_len, dim)
-
-
-def test_adalayernorm_zero_accepts_quant_config():
-    """Constructor accepts quant_config=None and prefix='test' without error."""
-    from vllm_omni.diffusion.layers.adalayernorm import (
-        AdaLayerNormContinuous,
-        AdaLayerNormZero,
-        AdaLayerNormZeroSingle,
-    )
-
-    # Should not raise with quant_config=None and prefix
-    AdaLayerNormZero(64, quant_config=None, prefix="test.norm1")
-    AdaLayerNormZeroSingle(64, quant_config=None, prefix="test.norm")
-    AdaLayerNormContinuous(64, 64, quant_config=None, prefix="test.norm_out")
-
-
-def test_adalayernorm_uses_replicated_linear():
-    """Verify .linear is a ReplicatedLinear instance (not nn.Linear)."""
-    from vllm.model_executor.layers.linear import ReplicatedLinear
-
-    from vllm_omni.diffusion.layers.adalayernorm import (
-        AdaLayerNormContinuous,
-        AdaLayerNormZero,
-        AdaLayerNormZeroSingle,
-    )
-
-    norm_zero = AdaLayerNormZero(64)
-    assert isinstance(norm_zero.linear, ReplicatedLinear)
-
-    norm_zero_single = AdaLayerNormZeroSingle(64)
-    assert isinstance(norm_zero_single.linear, ReplicatedLinear)
-
-    norm_continuous = AdaLayerNormContinuous(64, 64)
-    assert isinstance(norm_continuous.linear, ReplicatedLinear)
-
-
-# ── Numerical equivalence tests against diffusers originals ──
-
-
-def _copy_weights(src_linear, dst_replicated_linear):
-    """Copy weights from nn.Linear to ReplicatedLinear for comparison."""
-    dst_replicated_linear.weight.data.copy_(src_linear.weight.data)
-    if src_linear.bias is not None and dst_replicated_linear.bias is not None:
-        dst_replicated_linear.bias.data.copy_(src_linear.bias.data)
-
-
-def test_adalayernorm_zero_matches_diffusers():
-    """Verify AdaLayerNormZero produces identical output to diffusers original."""
-    from diffusers.models.normalization import (
-        AdaLayerNormZero as DiffusersAdaLayerNormZero,
-    )
-
-    from vllm_omni.diffusion.layers.adalayernorm import AdaLayerNormZero
-
-    dim = 64
-    torch.manual_seed(42)
-    ours = AdaLayerNormZero(dim)
-    ref = DiffusersAdaLayerNormZero(dim)
-
-    # Copy weights: nn.Linear -> ReplicatedLinear
-    _copy_weights(ref.linear, ours.linear)
-
-    x = torch.randn(2, 4, dim)
-    emb = torch.randn(2, dim)
-
-    out_ours = ours(x, emb)
-    out_ref = ref(x, emb=emb)
-
-    for o, r in zip(out_ours, out_ref):
-        torch.testing.assert_close(o, r, atol=1e-5, rtol=1e-5)
-
-
-def test_adalayernorm_zero_single_matches_diffusers():
-    """Verify AdaLayerNormZeroSingle produces identical output to diffusers original."""
-    from diffusers.models.normalization import (
-        AdaLayerNormZeroSingle as DiffusersAdaLayerNormZeroSingle,
-    )
-
-    from vllm_omni.diffusion.layers.adalayernorm import AdaLayerNormZeroSingle
-
-    dim = 64
-    torch.manual_seed(42)
-    ours = AdaLayerNormZeroSingle(dim)
-    ref = DiffusersAdaLayerNormZeroSingle(dim)
-
-    _copy_weights(ref.linear, ours.linear)
-
-    x = torch.randn(2, 4, dim)
-    emb = torch.randn(2, dim)
-
-    out_ours = ours(x, emb)
-    out_ref = ref(x, emb=emb)
-
-    for o, r in zip(out_ours, out_ref):
-        torch.testing.assert_close(o, r, atol=1e-5, rtol=1e-5)
-
-
-def test_adalayernorm_continuous_matches_diffusers():
-    """Verify AdaLayerNormContinuous produces identical output to diffusers original."""
-    from diffusers.models.normalization import (
-        AdaLayerNormContinuous as DiffusersAdaLayerNormContinuous,
-    )
-
-    from vllm_omni.diffusion.layers.adalayernorm import AdaLayerNormContinuous
-
-    dim = 64
-    cond_dim = 64
-    torch.manual_seed(42)
-    # Match constructor args: diffusers defaults elementwise_affine=True, eps=1e-5
-    ours = AdaLayerNormContinuous(dim, cond_dim, elementwise_affine=False, eps=1e-6)
-    ref = DiffusersAdaLayerNormContinuous(dim, cond_dim, elementwise_affine=False, eps=1e-6)
-
-    _copy_weights(ref.linear, ours.linear)
-
-    x = torch.randn(2, 4, dim)
-    cond = torch.randn(2, cond_dim)
-
-    out_ours = ours(x, cond)
-    out_ref = ref(x, cond)
-
-    torch.testing.assert_close(out_ours, out_ref, atol=1e-5, rtol=1e-5)

tests/diffusion/lora/conftest.py

@@ -1,56 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-"""Shared test helpers for diffusion LoRA tests."""
-
-from __future__ import annotations
-
-import torch
-from vllm.model_executor.layers.linear import LinearBase
-
-
-class FakeLinearBase(LinearBase):
-    """Minimal LinearBase stub for LoRA layer discovery."""
-
-    def __init__(self):
-        torch.nn.Module.__init__(self)
-
-
-class DummyBaseLayerWithLoRA(torch.nn.Module):
-    """Fake LoRA wrapper that records set/reset/create calls."""
-
-    def __init__(self, base_layer: torch.nn.Module):
-        super().__init__()
-        self.base_layer = base_layer
-
-        self.set_calls: list[
-            tuple[list[torch.Tensor | None] | torch.Tensor, list[torch.Tensor | None] | torch.Tensor]
-        ] = []
-        self.reset_calls: int = 0
-        self.create_calls: int = 0
-
-    def set_lora(self, index: int, lora_a, lora_b):
-        assert index == 0
-        self.set_calls.append((lora_a, lora_b))
-
-    def reset_lora(self, index: int):
-        assert index == 0
-        self.reset_calls += 1
-
-    def create_lora_weights(self, max_loras, lora_config, model_config):
-        self.create_calls += 1
-
-
-def fake_replace_submodule(
-    root: torch.nn.Module,
-    module_name: str,
-    submodule: torch.nn.Module,
-    replace_calls: list[str] | None = None,
-) -> None:
-    """Replace a submodule by traversing dotted paths correctly."""
-    if replace_calls is not None:
-        replace_calls.append(module_name)
-    parts = module_name.split(".")
-    parent = root
-    for attr in parts[:-1]:
-        parent = getattr(parent, attr)
-    setattr(parent, parts[-1], submodule)

tests/diffusion/lora/test_lora_manager.py

@@ -7,12 +7,8 @@ import pytest
 import torch
 from vllm.lora.lora_weights import LoRALayerWeights
 from vllm.lora.utils import get_supported_lora_modules
+from vllm.model_executor.layers.linear import LinearBase
 
-from tests.diffusion.lora.conftest import (
-    DummyBaseLayerWithLoRA,
-    FakeLinearBase,
-    fake_replace_submodule,
-)
 from vllm_omni.diffusion.lora.manager import DiffusionLoRAManager
 from vllm_omni.lora.request import LoRARequest
 
@@ -37,9 +33,35 @@ class _DummyLoRALayer:
         self.reset_calls += 1
 
 
-# Aliases for backward compatibility within this file
+class _FakeLinearBase(LinearBase):
-_FakeLinearBase = FakeLinearBase
+    def __init__(self):
-_DummyBaseLayerWithLoRA = DummyBaseLayerWithLoRA
+        torch.nn.Module.__init__(self)
+
+
+class _DummyBaseLayerWithLoRA(torch.nn.Module):
+    def __init__(self, base_layer: torch.nn.Module):
+        super().__init__()
+        self.base_layer = base_layer
+
+        self.set_calls: list[
+            tuple[list[torch.Tensor | None] | torch.Tensor, list[torch.Tensor | None] | torch.Tensor]
+        ] = []
+        self.reset_calls: int = 0
+        self.create_calls: int = 0
+
+    def set_lora(self, index: int, lora_a, lora_b):
+        assert index == 0
+        self.set_calls.append((lora_a, lora_b))
+
+    def reset_lora(self, index: int):
+        assert index == 0
+        self.reset_calls += 1
+
+    def create_lora_weights(self, max_loras, lora_config, model_config):
+        # Needs to be callable for scale test when rank changes, but not
+        # actually used since we mock everything and check everything based
+        # on set calls.
+        self.create_calls += 1
 
 
 class _DummyPipeline(torch.nn.Module):
@@ -533,45 +555,3 @@ def test_lora_manager_max_rank_validation(monkeypatch, rank):
     req1 = _dummy_lora_request(1)
     with pytest.raises(ValueError):
         manager.add_adapter(req1)
-
-
-def test_lora_manager_discovers_bagel_component(monkeypatch):
-    """Verify that _replace_layers_with_lora finds layers under 'bagel'."""
-    import vllm_omni.diffusion.lora.manager as manager_mod
-
-    monkeypatch.setattr(manager_mod, "BaseLayerWithLoRA", _DummyBaseLayerWithLoRA)
-
-    def _fake_from_layer_diffusion(*, layer: torch.nn.Module, **_kwargs):
-        if isinstance(layer, _FakeLinearBase):
-            return _DummyBaseLayerWithLoRA(layer)
-        return layer
-
-    replace_calls: list[str] = []
-
-    monkeypatch.setattr(manager_mod, "from_layer_diffusion", _fake_from_layer_diffusion)
-    monkeypatch.setattr(
-        manager_mod,
-        "replace_submodule",
-        lambda root, name, sub: fake_replace_submodule(root, name, sub, replace_calls),
-    )
-
-    # Pipeline with a 'bagel' component (no 'transformer')
-    pipeline = torch.nn.Module()
-    pipeline.bagel = torch.nn.Module()
-    pipeline.bagel.language_model = torch.nn.Module()
-    pipeline.bagel.language_model.qkv_proj = _FakeLinearBase()
-
-    manager = DiffusionLoRAManager(
-        pipeline=pipeline,
-        device=torch.device("cpu"),
-        dtype=torch.bfloat16,
-        max_cached_adapters=1,
-    )
-
-    peft_helper = type("_PH", (), {"r": 1})()
-    manager._replace_layers_with_lora(peft_helper)
-
-    assert "language_model.qkv_proj" in replace_calls
-    assert "bagel.language_model.qkv_proj" in manager._lora_modules
-    # Verify the module was actually replaced in the tree (not just recorded)
-    assert isinstance(pipeline.bagel.language_model.qkv_proj, _DummyBaseLayerWithLoRA)

tests/diffusion/models/bagel/test_bagel_lora.py

@@ -1,248 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-"""Unit tests for BAGEL LoRA support across Stage 0 (Thinker) and Stage 1 (DiT)."""
-
-from __future__ import annotations
-
-import json
-from pathlib import Path
-
-import pytest
-import torch
-from safetensors.torch import save_file
-
-from tests.diffusion.lora.conftest import (
-    DummyBaseLayerWithLoRA,
-    FakeLinearBase,
-    fake_replace_submodule,
-)
-from vllm_omni.diffusion.lora.manager import DiffusionLoRAManager
-from vllm_omni.lora.request import LoRARequest
-
-pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
-
-_FakeLinearBase = FakeLinearBase
-
-
-# ---------------------------------------------------------------------------
-# Stage 0 (Thinker / AR) -- packed_modules_mapping on the AR model class
-# ---------------------------------------------------------------------------
-
-
-class TestStage0ThinkerLoRA:
-    """Validate that OmniBagelForConditionalGeneration declares correct LoRA metadata."""
-
-    def test_omni_bagel_supports_lora(self):
-        from vllm_omni.model_executor.models.bagel.bagel import (
-            OmniBagelForConditionalGeneration,
-        )
-
-        assert getattr(OmniBagelForConditionalGeneration, "supports_lora", False) is True
-
-    def test_omni_bagel_packed_modules_mapping_complete(self):
-        from vllm_omni.model_executor.models.bagel.bagel import (
-            OmniBagelForConditionalGeneration,
-        )
-
-        mapping = OmniBagelForConditionalGeneration.packed_modules_mapping
-        # Standard Qwen2 projections
-        assert mapping["qkv_proj"] == ["q_proj", "k_proj", "v_proj"]
-        assert mapping["gate_up_proj"] == ["gate_proj", "up_proj"]
-        # MoE generation-mode projections
-        assert mapping["qkv_proj_moe_gen"] == [
-            "q_proj_moe_gen",
-            "k_proj_moe_gen",
-            "v_proj_moe_gen",
-        ]
-        assert mapping["mlp_moe_gen.gate_up_proj"] == [
-            "mlp_moe_gen.gate_proj",
-            "mlp_moe_gen.up_proj",
-        ]
-
-
-# ---------------------------------------------------------------------------
-# Stage 1 (DiT / Diffusion) -- DiffusionLoRAManager with bagel component
-# ---------------------------------------------------------------------------
-
-
-class TestStage1DiTLoRA:
-    """Validate DiffusionLoRAManager discovers BAGEL's packed modules."""
-
-    def test_diffusion_lora_manager_discovers_bagel_packed_modules(self):
-        """Manager should derive packed→sublayer mapping from stacked_params_mapping."""
-        pipeline = torch.nn.Module()
-        pipeline.bagel = torch.nn.Module()
-
-        # Simulate a submodule that exposes stacked_params_mapping
-        # (as Bagel does after load_weights())
-        language_model = torch.nn.Module()
-        language_model.stacked_params_mapping = [
-            (".qkv_proj_moe_gen", ".q_proj_moe_gen", "q"),
-            (".qkv_proj_moe_gen", ".k_proj_moe_gen", "k"),
-            (".qkv_proj_moe_gen", ".v_proj_moe_gen", "v"),
-            (".qkv_proj", ".q_proj", "q"),
-            (".qkv_proj", ".k_proj", "k"),
-            (".qkv_proj", ".v_proj", "v"),
-            (".gate_up_proj", ".gate_proj", 0),
-            (".gate_up_proj", ".up_proj", 1),
-        ]
-        pipeline.bagel.language_model = language_model
-
-        manager = DiffusionLoRAManager(
-            pipeline=pipeline,
-            device=torch.device("cpu"),
-            dtype=torch.bfloat16,
-            max_cached_adapters=1,
-        )
-
-        mapping = manager._packed_modules_mapping
-        assert mapping["qkv_proj"] == ["q_proj", "k_proj", "v_proj"]
-        assert mapping["qkv_proj_moe_gen"] == [
-            "q_proj_moe_gen",
-            "k_proj_moe_gen",
-            "v_proj_moe_gen",
-        ]
-        assert mapping["gate_up_proj"] == ["gate_proj", "up_proj"]
-
-    def test_diffusion_lora_manager_replaces_bagel_packed_layer_via_sublayer_target(self, monkeypatch):
-        """Targeting sublayer 'q_proj' should replace the fused 'qkv_proj' under bagel."""
-        import vllm_omni.diffusion.lora.manager as manager_mod
-
-        monkeypatch.setattr(manager_mod, "BaseLayerWithLoRA", DummyBaseLayerWithLoRA)
-
-        def _fake_from_layer_diffusion(*, layer, **_kwargs):
-            return DummyBaseLayerWithLoRA(layer)
-
-        replace_calls: list[str] = []
-
-        monkeypatch.setattr(manager_mod, "from_layer_diffusion", _fake_from_layer_diffusion)
-        monkeypatch.setattr(
-            manager_mod,
-            "replace_submodule",
-            lambda root, name, sub: fake_replace_submodule(root, name, sub, replace_calls),
-        )
-
-        # Build pipeline with bagel component
-        pipeline = torch.nn.Module()
-        pipeline.bagel = torch.nn.Module()
-        lm = torch.nn.Module()
-        lm.stacked_params_mapping = [
-            (".qkv_proj", ".q_proj", "q"),
-            (".qkv_proj", ".k_proj", "k"),
-            (".qkv_proj", ".v_proj", "v"),
-        ]
-        lm.attn = torch.nn.Module()
-        lm.attn.qkv_proj = _FakeLinearBase()
-        pipeline.bagel.language_model = lm
-
-        manager = DiffusionLoRAManager(
-            pipeline=pipeline,
-            device=torch.device("cpu"),
-            dtype=torch.bfloat16,
-            max_cached_adapters=1,
-        )
-
-        # Treat qkv_proj as 3-slice packed layer
-        monkeypatch.setattr(manager, "_get_packed_modules_list", lambda _module: ["q", "k", "v"])
-
-        # Target sublayer "q_proj" -- manager should replace the packed "qkv_proj"
-        peft_helper = type("_PH", (), {"r": 1, "target_modules": ["q_proj"]})()
-        manager._replace_layers_with_lora(peft_helper)
-
-        assert "language_model.attn.qkv_proj" in replace_calls
-        assert "bagel.language_model.attn.qkv_proj" in manager._lora_modules
-        # Verify the module was actually replaced in the tree (not just recorded)
-        assert isinstance(pipeline.bagel.language_model.attn.qkv_proj, DummyBaseLayerWithLoRA)
-
-
-# ---------------------------------------------------------------------------
-# Round-trip: synthetic checkpoint → set_active_adapter → verify weights
-# ---------------------------------------------------------------------------
-
-
-def _write_synthetic_lora(
-    adapter_dir: Path,
-    module_name: str,
-    rank: int,
-    in_dim: int,
-    out_dim: int,
-) -> str:
-    """Write a minimal LoRA adapter (safetensors + config) to *adapter_dir*."""
-    adapter_dir.mkdir(parents=True, exist_ok=True)
-    lora_a = torch.ones((rank, in_dim), dtype=torch.float32)
-    lora_b = torch.ones((out_dim, rank), dtype=torch.float32) * 2.0
-    save_file(
-        {
-            f"base_model.model.{module_name}.lora_A.weight": lora_a,
-            f"base_model.model.{module_name}.lora_B.weight": lora_b,
-        },
-        str(adapter_dir / "adapter_model.safetensors"),
-    )
-    (adapter_dir / "adapter_config.json").write_text(
-        json.dumps({"r": rank, "lora_alpha": rank, "target_modules": [module_name]}),
-        encoding="utf-8",
-    )
-    return str(adapter_dir)
-
-
-class TestBagelLoRARoundTrip:
-    """End-to-end: synthetic checkpoint → load → activate → verify weights in fused layer."""
-
-    def test_set_active_adapter_loads_and_activates_bagel_lora(self, tmp_path, monkeypatch):
-        """Full round-trip through set_active_adapter for a bagel component module."""
-        import vllm_omni.diffusion.lora.manager as manager_mod
-
-        monkeypatch.setattr(manager_mod, "BaseLayerWithLoRA", DummyBaseLayerWithLoRA)
-
-        # Build pipeline with bagel.language_model.foo (simple non-packed layer)
-        pipeline = torch.nn.Module()
-        pipeline.bagel = torch.nn.Module()
-        lm = torch.nn.Module()
-        lm.foo = _FakeLinearBase()
-        pipeline.bagel.language_model = lm
-
-        def _fake_from_layer(*, layer, **_kwargs):
-            if isinstance(layer, FakeLinearBase):
-                return DummyBaseLayerWithLoRA(layer)
-            return layer
-
-        monkeypatch.setattr(manager_mod, "from_layer_diffusion", _fake_from_layer)
-        monkeypatch.setattr(
-            manager_mod,
-            "replace_submodule",
-            lambda root, name, sub: fake_replace_submodule(root, name, sub),
-        )
-
-        manager = DiffusionLoRAManager(
-            pipeline=pipeline,
-            device=torch.device("cpu"),
-            dtype=torch.bfloat16,
-            max_cached_adapters=1,
-        )
-
-        # Write synthetic adapter targeting bagel.language_model.foo
-        module_name = "bagel.language_model.foo"
-        rank = 2
-        in_dim = 4
-        out_dim = 4
-        lora_dir = _write_synthetic_lora(tmp_path / "lora", module_name, rank, in_dim, out_dim)
-
-        lora_request = LoRARequest(
-            lora_name="test_bagel",
-            lora_int_id=42,
-            lora_path=lora_dir,
-        )
-
-        # Full round-trip: load from disk → replace layer → activate weights
-        manager.set_active_adapter(lora_request, lora_scale=0.5)
-
-        # Verify the layer was replaced and weights were set
-        replaced_layer = pipeline.bagel.language_model.foo
-        assert isinstance(replaced_layer, DummyBaseLayerWithLoRA), "Layer should be wrapped with LoRA"
-        assert len(replaced_layer.set_calls) == 1, "set_lora should have been called once"
-
-        lora_a, lora_b = replaced_layer.set_calls[0]
-        # A weights should be ones (as written)
-        assert torch.all(lora_a == 1.0), f"lora_a should be all ones, got {lora_a}"
-        # B weights should be 2.0 * scale(0.5) = 1.0
-        assert torch.allclose(lora_b, torch.ones_like(lora_b)), f"lora_b should be 2.0 * 0.5 = 1.0, got {lora_b}"

tests/diffusion/models/flux/test_flux_prefix_propagation.py

@@ -1,134 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-"""Tests that FLUX transformer blocks correctly propagate `quant_config` and
-`prefix` through all sub-layers.
-
-The tests instantiate blocks with a known prefix and verify that all quantization-
-aware sub-layers (AdaLayerNorm, FeedForward, Attention projections) receive the
-prefix rooted at the block prefix.  This is critical for quantized weight loading
-to match checkpoint keys to the correct model parameters.
-"""
-
-import os
-
-import pytest
-
-pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cpu]
-
-# Standard dimensions for a minimal FLUX block
-_DIM = 64
-_HEADS = 2
-_HEAD_DIM = _DIM // _HEADS
-
-
-@pytest.fixture(autouse=True)
-def _init_distributed():
-    """Initialize the minimal distributed environment required by
-    vLLM parallel linear layers (tensor-parallel group must exist)."""
-    from vllm.distributed.parallel_state import (
-        cleanup_dist_env_and_memory,
-        init_distributed_environment,
-        initialize_model_parallel,
-    )
-
-    os.environ.setdefault("MASTER_ADDR", "localhost")
-    os.environ.setdefault("MASTER_PORT", "29502")
-    init_distributed_environment(
-        world_size=1,
-        rank=0,
-        local_rank=0,
-        distributed_init_method="env://",
-    )
-    initialize_model_parallel()
-    yield
-    cleanup_dist_env_and_memory()
-
-
-def _param_names(module) -> set[str]:
-    """Return the set of all parameter names in a module."""
-    return {name for name, _ in module.named_parameters()}
-
-
-def test_flux_transformer_block_passes_prefix():
-    """FluxTransformerBlock propagates prefix to norm1, norm1_context, attn, ff, ff_context."""
-    from vllm_omni.diffusion.models.flux.flux_transformer import FluxTransformerBlock
-
-    prefix = "transformer_blocks.0"
-    block = FluxTransformerBlock(
-        dim=_DIM,
-        num_attention_heads=_HEADS,
-        attention_head_dim=_HEAD_DIM,
-        quant_config=None,
-        prefix=prefix,
-    )
-
-    params = _param_names(block)
-
-    # norm1 and norm1_context (AdaLayerNormZero) should have linear weights
-    assert any(name.startswith("norm1.linear.") for name in params), (
-        f"norm1.linear.* not found in params: {sorted(params)}"
-    )
-    assert any(name.startswith("norm1_context.linear.") for name in params), (
-        f"norm1_context.linear.* not found in params: {sorted(params)}"
-    )
-
-    # attn should have QKV projections
-    assert any(name.startswith("attn.to_qkv.") for name in params), (
-        f"attn.to_qkv.* not found in params: {sorted(params)}"
-    )
-
-    # ff and ff_context should have net layers
-    assert any(name.startswith("ff.net.") for name in params), f"ff.net.* not found in params: {sorted(params)}"
-    assert any(name.startswith("ff_context.net.") for name in params), (
-        f"ff_context.net.* not found in params: {sorted(params)}"
-    )
-
-
-def test_flux_single_transformer_block_passes_prefix():
-    """FluxSingleTransformerBlock propagates prefix to norm, proj_mlp, attn."""
-    from vllm_omni.diffusion.models.flux.flux_transformer import FluxSingleTransformerBlock
-
-    prefix = "single_transformer_blocks.0"
-    block = FluxSingleTransformerBlock(
-        dim=_DIM,
-        num_attention_heads=_HEADS,
-        attention_head_dim=_HEAD_DIM,
-        quant_config=None,
-        prefix=prefix,
-    )
-
-    params = _param_names(block)
-
-    # norm (AdaLayerNormZeroSingle) should have linear weights
-    assert any(name.startswith("norm.linear.") for name in params), (
-        f"norm.linear.* not found in params: {sorted(params)}"
-    )
-
-    # proj_mlp (ReplicatedLinear) should have weight
-    assert any(name.startswith("proj_mlp.") for name in params), f"proj_mlp.* not found in params: {sorted(params)}"
-
-    # attn should have QKV projection
-    assert any(name.startswith("attn.to_qkv.") for name in params), (
-        f"attn.to_qkv.* not found in params: {sorted(params)}"
-    )
-
-
-def test_flux_feedforward_passes_prefix():
-    """FeedForward propagates prefix to net.0 (GELU proj) and net.2 (output proj)."""
-    from vllm_omni.diffusion.models.flux.flux_transformer import FeedForward
-
-    prefix = "transformer_blocks.0.ff"
-    ff = FeedForward(
-        dim=_DIM,
-        dim_out=_DIM,
-        quant_config=None,
-        prefix=prefix,
-    )
-
-    params = _param_names(ff)
-
-    # net.0 is ColumnParallelApproxGELU which wraps a ColumnParallelLinear
-    assert any("net.0" in name for name in params), f"net.0 not found in params: {sorted(params)}"
-
-    # net.2 is RowParallelLinear
-    assert any("net.2" in name for name in params), f"net.2 not found in params: {sorted(params)}"

tests/diffusion/offloader/test_layerwise_backend.py

@@ -1,7 +1,7 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
 
-"""Unit tests for LayerwiseOffloadHook and LayerWiseOffloadBackend utilities."""
+"""Unit tests for LayerwiseOffloadHook."""
 
 import gc
 import os
@@ -15,7 +15,7 @@ from torch import nn
 from torch.distributed.tensor import DeviceMesh, DTensor, Replicate
 
 import vllm_omni.diffusion.offloader.layerwise_backend as layerwise_backend_module
-from vllm_omni.diffusion.offloader.layerwise_backend import LayerWiseOffloadBackend, LayerwiseOffloadHook
+from vllm_omni.diffusion.offloader.layerwise_backend import LayerwiseOffloadHook
 from vllm_omni.platforms import current_omni_platform
 
 pytestmark = [pytest.mark.diffusion, pytest.mark.cpu, pytest.mark.core_model]
@@ -127,116 +127,3 @@ class TestLayerwiseOffloadHook:
         assert current_block.weight.to_local().is_meta
         assert current_block.weight.to_local().shape == torch.Size([4])
         assert not hook.is_materialized
-
-
-class _DummyBlock(nn.Module):
-    def __init__(self):
-        super().__init__()
-        self.weight = nn.Parameter(torch.randn(10, 10))
-
-
-class _SingleBlockModel(nn.Module):
-    _layerwise_offload_blocks_attrs = ["blocks"]
-
-    def __init__(self, num_blocks: int = 3):
-        super().__init__()
-        self.blocks = nn.ModuleList([_DummyBlock() for _ in range(num_blocks)])
-
-
-class _MultiBlockModel(nn.Module):
-    _layerwise_offload_blocks_attrs = ["transformer_blocks", "single_transformer_blocks"]
-
-    def __init__(self, num_transformer: int = 2, num_single: int = 2):
-        super().__init__()
-        self.transformer_blocks = nn.ModuleList([_DummyBlock() for _ in range(num_transformer)])
-        self.single_transformer_blocks = nn.ModuleList([_DummyBlock() for _ in range(num_single)])
-
-
-class _EmptyBlocksModel(nn.Module):
-    _layerwise_offload_blocks_attrs = ["blocks"]
-
-    def __init__(self):
-        super().__init__()
-        self.blocks = nn.ModuleList([])
-
-
-class _InvalidAttrModel(nn.Module):
-    _layerwise_offload_blocks_attrs = ["nonexistent_blocks", "blocks"]
-
-    def __init__(self, num_blocks: int = 2):
-        super().__init__()
-        self.blocks = nn.ModuleList([_DummyBlock() for _ in range(num_blocks)])
-
-
-class _DeprecatedSingleAttrModel(nn.Module):
-    _layerwise_offload_blocks_attr = "blocks"
-
-    def __init__(self, num_blocks: int = 2):
-        super().__init__()
-        self.blocks = nn.ModuleList([_DummyBlock() for _ in range(num_blocks)])
-
-
-class _NoAttrsModel(nn.Module):
-    def __init__(self, num_blocks: int = 2):
-        super().__init__()
-        self.blocks = nn.ModuleList([_DummyBlock() for _ in range(num_blocks)])
-
-
-class TestGetBlocksFromDit:
-    def test_get_blocks_from_dit_single_block_attr(self):
-        model = _SingleBlockModel(num_blocks=3)
-        attr_names, blocks = LayerWiseOffloadBackend.get_blocks_from_dit(model)
-        assert attr_names == ["blocks"]
-        assert len(blocks) == 3
-        assert all(isinstance(b, _DummyBlock) for b in blocks)
-
-    def test_get_blocks_from_dit_multi_block_attrs(self):
-        model = _MultiBlockModel(num_transformer=2, num_single=3)
-        attr_names, blocks = LayerWiseOffloadBackend.get_blocks_from_dit(model)
-        assert set(attr_names) == {"transformer_blocks", "single_transformer_blocks"}
-        assert len(blocks) == 5
-        assert all(isinstance(b, _DummyBlock) for b in blocks)
-
-    def test_get_blocks_from_dit_empty_blocks(self):
-        model = _EmptyBlocksModel()
-        attr_names, blocks = LayerWiseOffloadBackend.get_blocks_from_dit(model)
-        assert attr_names == []
-        assert blocks == []
-
-    def test_get_blocks_from_dit_invalid_attr_name(self):
-        model = _InvalidAttrModel(num_blocks=2)
-        with pytest.raises(
-            AttributeError,
-            match="Attribute 'nonexistent_blocks' declared in _layerwise_offload_blocks_attrs does not exist",
-        ):
-            LayerWiseOffloadBackend.get_blocks_from_dit(model)
-
-    def test_get_blocks_from_dit_no_attrs_defined(self):
-        model = _NoAttrsModel(num_blocks=3)
-        attr_names, blocks = LayerWiseOffloadBackend.get_blocks_from_dit(model)
-        assert attr_names == []
-        assert blocks == []
-
-    def test_get_blocks_from_dit_deprecated_single_attr(self):
-        model = _DeprecatedSingleAttrModel(num_blocks=2)
-        attr_names, blocks = LayerWiseOffloadBackend.get_blocks_from_dit(model)
-        assert attr_names == ["blocks"]
-        assert len(blocks) == 2
-
-
-class TestGetBlocksAttrNames:
-    def test_get_blocks_attr_names_new_format(self):
-        model = _MultiBlockModel()
-        attrs = LayerWiseOffloadBackend.get_blocks_attr_names(model)
-        assert attrs == ["transformer_blocks", "single_transformer_blocks"]
-
-    def test_get_blocks_attr_names_no_attrs(self):
-        model = _NoAttrsModel()
-        attrs = LayerWiseOffloadBackend.get_blocks_attr_names(model)
-        assert attrs == []
-
-    def test_set_blocks_attr_names(self):
-        model = _NoAttrsModel()
-        LayerWiseOffloadBackend.set_blocks_attr_names(model, ["new_blocks"])
-        assert hasattr(model.__class__, "_layerwise_offload_blocks_attrs")
-        assert model.__class__._layerwise_offload_blocks_attrs == ["new_blocks"]

tests/diffusion/quantization/test_inc_config.py

@@ -1,147 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-"""Tests for INC/AutoRound quantization via the unified framework."""
-
-import pytest
-
-pytestmark = [pytest.mark.core_model, pytest.mark.diffusion]
-
-
-def test_build_quant_config_autoround():
-    """build_quant_config("auto-round", ...) should produce an INCConfig."""
-    from vllm.model_executor.layers.quantization.inc import INCConfig
-
-    from vllm_omni.quantization import build_quant_config
-
-    config = build_quant_config(
-        "auto-round",
-        bits=4,
-        group_size=128,
-        sym=True,
-        packing_format="auto_round:auto_gptq",
-    )
-    assert config is not None
-    assert isinstance(config, INCConfig)
-    assert config.weight_bits == 4
-    assert config.group_size == 128
-
-
-def test_build_quant_config_inc():
-    """build_quant_config("inc", ...) should also produce an INCConfig."""
-    from vllm.model_executor.layers.quantization.inc import INCConfig
-
-    from vllm_omni.quantization import build_quant_config
-
-    config = build_quant_config("inc", bits=4, group_size=128)
-    assert isinstance(config, INCConfig)
-    assert config.weight_bits == 4
-
-
-def test_build_quant_config_autoround_dict():
-    """Dict-style config with method=auto-round should work."""
-    from vllm.model_executor.layers.quantization.inc import INCConfig
-
-    from vllm_omni.quantization import build_quant_config
-
-    config = build_quant_config(
-        {
-            "method": "auto-round",
-            "bits": 4,
-            "group_size": 128,
-            "sym": True,
-            "packing_format": "auto_round:auto_gptq",
-        }
-    )
-    assert isinstance(config, INCConfig)
-    assert config.weight_bits == 4
-
-
-def test_build_quant_config_autoround_filters_metadata():
-    """Checkpoint metadata keys (autoround_version, batch_size, iters)
-    should be silently filtered out instead of causing TypeError."""
-    from vllm.model_executor.layers.quantization.inc import INCConfig
-
-    from vllm_omni.quantization import build_quant_config
-
-    config = build_quant_config(
-        "auto-round",
-        bits=4,
-        group_size=128,
-        sym=True,
-        packing_format="auto_round:auto_gptq",
-        block_name_to_quantize="transformer_blocks,single_transformer_blocks",
-        autoround_version="0.12.0",  # metadata — must be filtered
-        batch_size=1,  # metadata — must be filtered
-        iters=0,  # metadata — must be filtered
-    )
-    assert isinstance(config, INCConfig)
-    assert config.weight_bits == 4
-    assert config.group_size == 128
-
-
-def test_build_quant_config_bits_to_weight_bits_mapping():
-    """The 'bits' key from checkpoints should be mapped to 'weight_bits'."""
-    from vllm.model_executor.layers.quantization.inc import INCConfig
-
-    from vllm_omni.quantization import build_quant_config
-
-    # If weight_bits is already provided, bits should be ignored
-    config = build_quant_config("auto-round", weight_bits=4, group_size=128)
-    assert isinstance(config, INCConfig)
-    assert config.weight_bits == 4
-
-
-def test_autoround_in_supported_methods():
-    """auto-round and inc should appear in SUPPORTED_QUANTIZATION_METHODS."""
-    from vllm_omni.quantization import SUPPORTED_QUANTIZATION_METHODS
-
-    assert "auto-round" in SUPPORTED_QUANTIZATION_METHODS
-    assert "inc" in SUPPORTED_QUANTIZATION_METHODS
-
-
-def test_integration_autoround_via_omni_diffusion_config():
-    """OmniDiffusionConfig with auto-round quantization dict should resolve."""
-    from vllm.model_executor.layers.quantization.inc import INCConfig
-
-    from vllm_omni.diffusion.data import OmniDiffusionConfig
-
-    config = OmniDiffusionConfig(
-        model="test",
-        quantization_config={
-            "method": "auto-round",
-            "bits": 4,
-            "group_size": 128,
-            "sym": True,
-        },
-    )
-    assert isinstance(config.quantization_config, INCConfig)
-    assert config.quantization_config.weight_bits == 4
-
-
-def test_integration_autodetect_from_transformer_config():
-    """When TransformerConfig has quant_config, OmniDiffusionConfig should
-    auto-detect it even without explicit quantization_config."""
-    from vllm.model_executor.layers.quantization.inc import INCConfig
-
-    from vllm_omni.diffusion.data import OmniDiffusionConfig, TransformerConfig
-
-    tf_config = TransformerConfig.from_dict(
-        {
-            "quantization_config": {
-                "quant_method": "auto-round",
-                "bits": 4,
-                "group_size": 128,
-                "sym": True,
-                "packing_format": "auto_round:auto_gptq",
-                "autoround_version": "0.12.0",
-                "batch_size": 1,
-                "iters": 0,
-            }
-        }
-    )
-    assert tf_config.quant_method == "auto-round"
-    assert isinstance(tf_config.quant_config, INCConfig)
-
-    od_config = OmniDiffusionConfig(model="test", tf_model_config=tf_config)
-    assert isinstance(od_config.quantization_config, INCConfig)
-    assert od_config.quantization_config.weight_bits == 4

tests/diffusion/quantization/test_quantization_quality.py

@@ -118,12 +118,8 @@ def _generate_image(omni, config: QualityTestConfig):
 
     peak_mem = torch.cuda.max_memory_allocated() / (1024**3)
     first = outputs[0]
-    if hasattr(first, "images") and first.images:
+    req_out = first.request_output[0] if hasattr(first, "request_output") else first
-        return first.images[0], peak_mem
+    return req_out.images[0], peak_mem
-    inner = first.request_output
-    if inner is not None and hasattr(inner, "images") and inner.images:
-        return inner.images[0], peak_mem
-    raise ValueError("Could not extract image from output.")
 
 
 def _generate_video(omni, config: QualityTestConfig):

tests/distributed/omni_connectors/test_chunk_transfer_adapter.py

@@ -133,22 +133,6 @@ def test_save_async(build_adapter):
     assert task["is_finished"] is False
 
 
-def test_send_single_request_cleans_up_after_finished_payload(build_adapter, monkeypatch):
-    adapter, _ = build_adapter(stage_id=1)
-    request = _req("req-finished", RequestStatus.FINISHED_STOPPED, external_req_id="ext-finished")
-
-    adapter.custom_process_next_stage_input_func = lambda **kwargs: {"x": [1], "finished": True}
-    cleanup_calls = []
-    monkeypatch.setattr(adapter, "cleanup", lambda *a, **kw: cleanup_calls.append((a, kw)))
-
-    adapter._send_single_request({"pooling_output": None, "request": request, "is_finished": True})
-
-    assert len(cleanup_calls) == 1
-    args, _ = cleanup_calls[0]
-    assert args[0] == "req-finished"
-    assert args[1] == "ext-finished"
-
-
 def test_update_request_payload(build_adapter):
     adapter, _ = build_adapter()
 
@@ -425,86 +409,3 @@ def test_generation_scheduler_calls_cleanup_on_finished(monkeypatch, mocker: Moc
     args, _ = cleanup_calls[0]
     assert args[0] == "req-s1"
     assert args[1] == "ext-s1"
-
-
-def test_ar_scheduler_defers_cleanup_and_queues_save_on_finished(mocker: MockerFixture):
-    """OmniARScheduler should enqueue save; adapter cleanup is handled in save thread."""
-    cleanup_calls = []
-    save_calls = []
-
-    adapter_mock = mocker.MagicMock()
-    adapter_mock.cleanup = lambda *a, **kw: cleanup_calls.append((a, kw))
-    adapter_mock.save_async = lambda *a, **kw: save_calls.append((a, kw))
-
-    from vllm_omni.core.sched.omni_ar_scheduler import OmniARScheduler
-
-    scheduler = mocker.MagicMock()
-    scheduler.chunk_transfer_adapter = adapter_mock
-    scheduler.connector = None
-    scheduler.perf_metrics = None
-    scheduler.log_stats = False
-    scheduler.recompute_kv_load_failures = False
-    scheduler.structured_output_manager = mocker.MagicMock()
-    scheduler.structured_output_manager.should_advance.return_value = False
-    scheduler.finished_req_ids_dict = {}
-    scheduler.kv_cache_manager = mocker.MagicMock()
-    scheduler.kv_cache_manager.take_events.return_value = None
-    scheduler.kv_event_publisher = mocker.MagicMock()
-    scheduler.waiting_for_transfer_free = set()
-    scheduler.transfer_triggered_requests = set()
-    scheduler.active_kv_transfers = set()
-
-    request = _HashableRequest(
-        request_id="req-ar",
-        external_req_id="ext-ar",
-        status=RequestStatus.RUNNING,
-        is_finished=lambda: False,
-        num_computed_tokens=1,
-        num_prompt_tokens=1,
-        prompt_token_ids=[1],
-        num_output_placeholders=0,
-        sampling_params=None,
-        pooling_params=None,
-        stop_reason=None,
-        client_index=0,
-        take_events=lambda: [],
-        trace_headers=None,
-        num_cached_tokens=0,
-        num_external_computed_tokens=0,
-        num_nans_in_logits=0,
-        get_finished_reason=lambda: "stop",
-    )
-    scheduler.requests = {"req-ar": request}
-
-    scheduler._update_request_with_output = mocker.MagicMock(return_value=([], True))
-    scheduler._process_kv_transfer_trigger = mocker.MagicMock(return_value=False)
-    scheduler._handle_stopped_request = mocker.MagicMock(return_value=True)
-    scheduler._free_request = mocker.MagicMock(return_value=None)
-    scheduler._get_routed_experts = mocker.MagicMock(return_value=None)
-    scheduler.running = [request]
-    scheduler.waiting = mocker.MagicMock()
-    scheduler.waiting.remove_requests = mocker.MagicMock()
-    scheduler.make_spec_decoding_stats = mocker.MagicMock(return_value=None)
-    scheduler.make_stats = mocker.MagicMock(return_value=None)
-
-    scheduler_output = SimpleNamespace(
-        num_scheduled_tokens={"req-ar": 1},
-        scheduled_spec_decode_tokens={},
-        num_invalid_spec_tokens=0,
-    )
-    model_runner_output = SimpleNamespace(
-        sampled_token_ids=[[123]],
-        logprobs=None,
-        prompt_logprobs_dict={},
-        pooler_output=None,
-        num_nans_in_logits=None,
-        kv_connector_output=None,
-        cudagraph_stats=None,
-        req_id_to_index={"req-ar": 0},
-        kv_extracted_req_ids=None,
-    )
-
-    OmniARScheduler.update_from_output(scheduler, scheduler_output, model_runner_output)
-
-    assert len(cleanup_calls) == 0
-    assert len(save_calls) == 1

tests/distributed/omni_coordinator/test_omni_coord_client_for_stage.py

@@ -2,20 +2,13 @@
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
 
 import json
-import threading
 
-import pytest
 import zmq
 
 from vllm_omni.distributed.omni_coordinator import (
     OmniCoordClientForStage,
     StageStatus,
 )
-from vllm_omni.distributed.omni_coordinator import (
-    omni_coord_client_for_stage as stage_client_module,
-)
-
-pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
 
 
 def _bind_router() -> tuple[zmq.Context, zmq.Socket, str]:
@@ -26,8 +19,7 @@ def _bind_router() -> tuple[zmq.Context, zmq.Socket, str]:
     return ctx, router, endpoint
 
 
-def _recv_event(router: zmq.Socket, timeout_ms: int = 2000) -> dict:
+def _recv_event(router: zmq.Socket) -> dict:
-    assert router.poll(timeout=timeout_ms) != 0, "Timed out waiting for coordinator event"
     frames = router.recv_multipart()
     # ROUTER adds identity frame; the last frame is the payload.
     payload = frames[-1]
@@ -116,197 +108,3 @@ def test_stage_client_close_sends_down_status():
 
     router.close(0)
     ctx.term()
-
-
-def test_stage_client_reconnects_after_send_failure(mocker):
-    """Verify send failure path invokes reconnect before retrying send."""
-    ctx, router, endpoint = _bind_router()
-
-    client = OmniCoordClientForStage(
-        endpoint,
-        "tcp://stage:reconnect-in",
-        "tcp://stage:reconnect-out",
-        0,
-    )
-
-    # Discard initial registration event from the real socket.
-    _recv_event(router)
-
-    class _FlakySocket:
-        def __init__(self):
-            self.send_calls = 0
-            self.closed = False
-
-        def send(self, *_args, **_kwargs):
-            self.send_calls += 1
-            if self.send_calls == 1:
-                raise RuntimeError("simulated send failure")
-
-        def close(self, *_args, **_kwargs):
-            self.closed = True
-
-    flaky_socket = _FlakySocket()
-    client._socket = flaky_socket
-    client._reconnect = mocker.Mock(return_value=True)
-
-    client.update_info(queue_length=1)
-
-    client._reconnect.assert_called_once_with(max_retries=3)
-    assert flaky_socket.send_calls == 2
-
-    client.close()
-    router.close(0)
-    ctx.term()
-
-
-def test_stage_client_raises_when_reconnect_fails(mocker):
-    """Verify send failure is propagated when reconnect cannot recover."""
-    ctx, router, endpoint = _bind_router()
-
-    client = OmniCoordClientForStage(
-        endpoint,
-        "tcp://stage:reconnect-fail-in",
-        "tcp://stage:reconnect-fail-out",
-        0,
-    )
-
-    # Discard initial registration event from the real socket.
-    _recv_event(router)
-
-    class _AlwaysFailSocket:
-        def send(self, *_args, **_kwargs):
-            raise RuntimeError("simulated send failure")
-
-        def close(self, *_args, **_kwargs):
-            pass
-
-    client._socket = _AlwaysFailSocket()
-    client._reconnect = mocker.Mock(return_value=False)
-
-    with pytest.raises(RuntimeError, match="simulated send failure"):
-        client.update_info(queue_length=2)
-
-    client._reconnect.assert_called_once_with(max_retries=3)
-    client.close()
-    router.close(0)
-    ctx.term()
-
-
-def test_stage_client_close_handles_runtime_error_in_final_update(mocker):
-    """Verify close() still releases resources when final update raises RuntimeError."""
-    ctx, router, endpoint = _bind_router()
-
-    client = OmniCoordClientForStage(
-        endpoint,
-        "tcp://stage:close-runtime-in",
-        "tcp://stage:close-runtime-out",
-        0,
-    )
-
-    # Discard initial registration event from the real socket.
-    _recv_event(router)
-
-    client._send_event = mocker.Mock(side_effect=RuntimeError("simulated close-time failure"))
-    client.close()
-
-    assert client._closed
-    assert client._socket.closed
-
-    router.close(0)
-    ctx.term()
-
-
-def test_reconnect_respects_retry_limit(monkeypatch):
-    """Verify _reconnect stops after max_retries on repeated failures."""
-    attempts = {"connect": 0}
-
-    class _FailSocket:
-        def close(self, *_args, **_kwargs):
-            pass
-
-        def connect(self, *_args, **_kwargs):
-            attempts["connect"] += 1
-            raise zmq.ZMQError("simulated reconnect failure")
-
-    class _FailContext:
-        def socket(self, *_args, **_kwargs):
-            return _FailSocket()
-
-        def term(self):
-            pass
-
-    client = OmniCoordClientForStage.__new__(OmniCoordClientForStage)
-    client._closed = False
-    client._coord_zmq_addr = "tcp://127.0.0.1:9999"
-    client._stop_event = threading.Event()
-    client._send_lock = threading.RLock()
-    client._socket = _FailSocket()
-    client._ctx = _FailContext()
-
-    monkeypatch.setattr(stage_client_module.zmq, "Context", lambda: _FailContext())
-    monkeypatch.setattr(stage_client_module.time, "sleep", lambda *_args, **_kwargs: None)
-
-    assert client._reconnect(max_retries=3, retry_interval=5.0) is False
-    assert attempts["connect"] == 3
-
-
-def test_heartbeat_loop_retries_after_transient_send_failure():
-    """Verify heartbeat loop continues after one transient send failure."""
-
-    class _FakeStopEvent:
-        def __init__(self):
-            self.wait_calls = 0
-            self._set = False
-
-        def wait(self, timeout=None):
-            _ = timeout
-            self.wait_calls += 1
-            # Run two loop iterations, then stop.
-            return self._set or self.wait_calls >= 3
-
-        def is_set(self):
-            return self._set
-
-        def set(self):
-            self._set = True
-
-    client = OmniCoordClientForStage.__new__(OmniCoordClientForStage)
-    client._closed = False
-    client._heartbeat_interval = 0.0
-    client._stop_event = _FakeStopEvent()
-
-    calls = {"count": 0}
-
-    def _fake_send(event_type):
-        assert event_type == "heartbeat"
-        calls["count"] += 1
-        if calls["count"] == 1:
-            raise RuntimeError("transient heartbeat failure")
-
-    client._send_event = _fake_send
-
-    client._heartbeat_loop()
-
-    assert calls["count"] == 2
-
-
-def test_update_info_rejected_while_closing():
-    """Verify update_info is rejected once client enters closing state."""
-    ctx, router, endpoint = _bind_router()
-
-    client = OmniCoordClientForStage(
-        endpoint,
-        "tcp://stage:closing-in",
-        "tcp://stage:closing-out",
-        0,
-    )
-    _recv_event(router)
-
-    client._closing = True
-    with pytest.raises(RuntimeError, match="closing"):
-        client.update_info(queue_length=3)
-
-    client._closing = False
-    client.close()
-    router.close(0)
-    ctx.term()

tests/e2e/offline_inference/test_bagel_text2img.py

@@ -229,24 +229,6 @@ def _wait_for_port(host: str, port: int, timeout: int = 30) -> bool:
     return False
 
 
-def _is_mooncake_master_available() -> bool:
-    """Check if mooncake_master binary is present and can actually execute."""
-    import shutil
-
-    binary = shutil.which("mooncake_master")
-    if binary is None:
-        return False
-    try:
-        result = subprocess.run(
-            [binary, "--help"],
-            capture_output=True,
-            timeout=5,
-        )
-        return result.returncode != 127
-    except (subprocess.TimeoutExpired, OSError):
-        return True
-
-
 def _cleanup_mooncake_processes(timeout_secs: int = 5) -> None:
     """Clean up any existing mooncake_master processes.
 
@@ -310,8 +292,6 @@ def _load_mooncake_config(host: str, rpc_port: int, http_port: int) -> str:
 @hardware_test(res={"cuda": "H100"})
 def test_bagel_text2img_mooncake_connector(run_level):
     """Test Bagel text2img with Mooncake connector for inter-stage communication."""
-    if not _is_mooncake_master_available():
-        pytest.skip("mooncake_master is not available or cannot execute (missing shared libraries like libibverbs)")
     MOONCAKE_HOST = "127.0.0.1"
     MOONCAKE_RPC_PORT = _find_free_port()
     MOONCAKE_HTTP_PORT = _find_free_port()

tests/e2e/offline_inference/test_bagel_understanding.py

@@ -1,144 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-
-"""
-End-to-end tests for Bagel text2text and img2text (understanding) tasks.
-
-These tests validate that the Bagel multistage pipeline correctly generates
-text output for understanding tasks, matching reference results.
-
-Equivalent to running:
-    python3 examples/offline_inference/bagel/end2end.py \
-        --modality text2text \
-        --prompts "Where is the capital of France?"
-
-    python3 examples/offline_inference/bagel/end2end.py \
-        --modality img2text \
-        --prompts "Please describe this image" \
-        --image-path 2560px-Gfp-wisconsin-madison-the-nature-boardwalk.jpg
-"""
-
-import os
-
-os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
-os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "1"
-from pathlib import Path
-
-import pytest
-from vllm.assets.image import ImageAsset
-
-from tests.conftest import modify_stage_config
-from tests.utils import hardware_test
-from vllm_omni.entrypoints.omni import Omni
-
-MODEL_NAME = "ByteDance-Seed/BAGEL-7B-MoT"
-STAGE_CONFIG = str(Path(__file__).parent / "stage_configs" / "bagel_sharedmemory_ci.yaml")
-
-REFERENCE_TEXT_TEXT2TEXT = "The capital of France is Paris."
-
-REFERENCE_TEXT_IMG2TEXT = (
-    "This is a photo of a wooden boardwalk or pathway that leads through "
-    "tall green grass. The path appears to be in a natural setting, possibly "
-    "a wetland or marsh area. The sky above is blue with some scattered "
-    "clouds, suggesting it might be a sunny day. The overall scene looks "
-    "peaceful and serene."
-)
-
-
-def _resolve_stage_config(config_path: str, run_level: str) -> str:
-    """Strip load_format: dummy for advanced_model (real weights)."""
-    if run_level == "advanced_model":
-        return modify_stage_config(
-            config_path,
-            deletes={
-                "stage_args": {
-                    0: ["engine_args.load_format"],
-                    1: ["engine_args.load_format"],
-                }
-            },
-        )
-    return config_path
-
-
-def _extract_text(omni_outputs: list) -> str:
-    """Extract generated text from OmniRequestOutput list."""
-    for req_output in omni_outputs:
-        ro = getattr(req_output, "request_output", None)
-        if ro and getattr(ro, "outputs", None):
-            return "".join(getattr(o, "text", "") or "" for o in ro.outputs)
-    return ""
-
-
-@pytest.mark.core_model
-@pytest.mark.advanced_model
-@pytest.mark.diffusion
-@hardware_test(res={"cuda": "H100", "rocm": "MI325"})
-def test_bagel_text2text(run_level):
-    """Test Bagel text2text produces correct text output."""
-    config_path = _resolve_stage_config(STAGE_CONFIG, run_level)
-    omni = Omni(
-        model=MODEL_NAME,
-        stage_configs_path=config_path,
-        stage_init_timeout=300,
-    )
-
-    try:
-        prompt = "<|im_start|>user\nWhere is the capital of France?<|im_end|>\n<|im_start|>assistant\n"
-        params_list = omni.default_sampling_params_list
-        omni_outputs = list(
-            omni.generate(
-                prompts=[{"prompt": prompt, "modalities": ["text"]}],
-                sampling_params_list=params_list,
-            )
-        )
-
-        assert len(omni_outputs) > 0, "No outputs returned"
-        text = _extract_text(omni_outputs)
-        assert len(text) > 0, "Generated text is empty"
-
-        if run_level == "advanced_model":
-            assert text == REFERENCE_TEXT_TEXT2TEXT, (
-                f"Text mismatch: expected {REFERENCE_TEXT_TEXT2TEXT!r}, got {text!r}"
-            )
-    finally:
-        omni.close()
-
-
-@pytest.mark.core_model
-@pytest.mark.advanced_model
-@pytest.mark.diffusion
-@hardware_test(res={"cuda": "H100", "rocm": "MI325"})
-def test_bagel_img2text(run_level):
-    """Test Bagel img2text produces correct text output."""
-    input_image = ImageAsset("2560px-Gfp-wisconsin-madison-the-nature-boardwalk").pil_image.convert("RGB")
-    config_path = _resolve_stage_config(STAGE_CONFIG, run_level)
-    omni = Omni(
-        model=MODEL_NAME,
-        stage_configs_path=config_path,
-        stage_init_timeout=300,
-    )
-
-    try:
-        prompt = "<|im_start|>user\n<|image_pad|>\nPlease describe this image<|im_end|>\n<|im_start|>assistant\n"
-        params_list = omni.default_sampling_params_list
-        omni_outputs = list(
-            omni.generate(
-                prompts=[
-                    {
-                        "prompt": prompt,
-                        "multi_modal_data": {"image": input_image},
-                        "modalities": ["text"],
-                    }
-                ],
-                sampling_params_list=params_list,
-            )
-        )
-
-        assert len(omni_outputs) > 0, "No outputs returned"
-        text = _extract_text(omni_outputs)
-        assert len(text) > 0, "Generated text is empty"
-
-        if run_level == "advanced_model":
-            assert text == REFERENCE_TEXT_IMG2TEXT, f"Text mismatch: expected {REFERENCE_TEXT_IMG2TEXT!r}, got {text!r}"
-    finally:
-        omni.close()

tests/e2e/offline_inference/test_cosyvoice3.py

@@ -1,218 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-"""
-Offline E2E smoke test for CosyVoice3 zero-shot reference inference.
-
-This test uses the official upstream zero-shot prompt text/audio pair and
-verifies a stable reference recipe:
-- config-derived top_p/top_k and token-length ratios
-- model EOS token as the stop token
-- a conservative repetition penalty to avoid degenerate loops
-"""
-
-from __future__ import annotations
-
-import functools
-import io
-import os
-import tempfile
-from pathlib import Path
-from urllib.request import urlopen
-
-import numpy as np
-import pytest
-import soundfile as sf
-import yaml
-from huggingface_hub import snapshot_download
-from vllm.sampling_params import SamplingParams
-
-from tests.conftest import OmniRunner
-from tests.utils import hardware_test
-from vllm_omni.model_executor.models.cosyvoice3.config import CosyVoice3Config
-from vllm_omni.model_executor.models.cosyvoice3.tokenizer import get_qwen_tokenizer
-
-MODEL = "FunAudioLLM/Fun-CosyVoice3-0.5B-2512"
-MODEL_DIR_ENV = "VLLM_OMNI_COSYVOICE3_MODEL_DIR"
-
-REFERENCE_PROMPT_WAV_URL = "https://raw.githubusercontent.com/FunAudioLLM/CosyVoice/main/asset/zero_shot_prompt.wav"
-REFERENCE_PROMPT_TEXT = "You are a helpful assistant.<|endofprompt|>希望你以后能够做的比我还好呦。"
-REFERENCE_SYNTH_TEXT = (
-    "CosyVoice is undergoing a comprehensive upgrade, providing more accurate, "
-    "stable, faster, and better voice generation capabilities."
-)
-REFERENCE_STAGE0_TEMPERATURE = 1.0
-REFERENCE_STAGE0_REPETITION_PENALTY = 2.0
-
-
-def _stage_config(name: str) -> str:
-    return str(Path(__file__).parent.parent.parent.parent / "vllm_omni" / "model_executor" / "stage_configs" / name)
-
-
-STAGE_CONFIGS = [
-    _stage_config("cosyvoice3.yaml"),
-    _stage_config("cosyvoice3_async_chunk.yaml"),
-]
-
-
-@functools.lru_cache(maxsize=1)
-def _load_reference_prompt_wav() -> tuple[np.ndarray, int]:
-    with urlopen(REFERENCE_PROMPT_WAV_URL, timeout=30) as resp:
-        data = resp.read()
-    audio, sr = sf.read(io.BytesIO(data), dtype="float32", always_2d=False)
-    if isinstance(audio, np.ndarray) and audio.ndim > 1:
-        audio = np.mean(audio, axis=-1)
-    return np.asarray(audio, dtype=np.float32), int(sr)
-
-
-@functools.lru_cache(maxsize=1)
-def _resolve_model_dir() -> Path:
-    override = os.environ.get(MODEL_DIR_ENV)
-    if override:
-        return Path(override).expanduser().resolve()
-    return Path(snapshot_download(MODEL, allow_patterns=["*"]))
-
-
-def _reference_zero_shot_stage0_sampling(*, text: str) -> SamplingParams:
-    config = CosyVoice3Config()
-    sampling_cfg = config.llm.get("sampling", {})
-    eos_token_id = int(config.llm["eos_token_id"])
-    model_dir = _resolve_model_dir()
-    tokenizer = get_qwen_tokenizer(
-        token_path=str(model_dir / config.qwen_pretrain_path),
-        skip_special_tokens=config.skip_special_tokens,
-        version=config.version,
-    )
-    text_len = max(1, len(tokenizer.encode(text, allowed_special=config.allowed_special)))
-    return SamplingParams(
-        temperature=REFERENCE_STAGE0_TEMPERATURE,
-        top_p=float(sampling_cfg.get("top_p", 0.8)),
-        top_k=int(sampling_cfg.get("top_k", 25)),
-        repetition_penalty=REFERENCE_STAGE0_REPETITION_PENALTY,
-        stop_token_ids=[eos_token_id],
-        min_tokens=int(text_len * config.min_token_text_ratio),
-        max_tokens=int(text_len * config.max_token_text_ratio),
-    )
-
-
-def _concat_audio(audio_val) -> np.ndarray:
-    import torch
-
-    if isinstance(audio_val, list):
-        tensors = []
-        for t in audio_val:
-            if t is None:
-                continue
-            if hasattr(t, "detach"):
-                t = t.detach()
-            if hasattr(t, "cpu"):
-                t = t.cpu()
-            if hasattr(t, "float"):
-                t = t.float()
-            if isinstance(t, torch.Tensor):
-                tensors.append(t.reshape(-1))
-        if not tensors:
-            return np.zeros((0,), dtype=np.float32)
-        return torch.cat(tensors, dim=-1).numpy().astype(np.float32, copy=False)
-
-    if hasattr(audio_val, "detach"):
-        audio_val = audio_val.detach()
-    if hasattr(audio_val, "cpu"):
-        audio_val = audio_val.cpu()
-    if hasattr(audio_val, "float"):
-        audio_val = audio_val.float()
-    if hasattr(audio_val, "numpy"):
-        audio_val = audio_val.numpy()
-    audio_np = np.asarray(audio_val, dtype=np.float32)
-    return audio_np.reshape(-1)
-
-
-def _get_stage_engine_outputs(omni_runner: OmniRunner, stage_id: int):
-    stage_list = getattr(omni_runner.omni, "stage_list", None)
-    if stage_list is not None:
-        return getattr(stage_list[stage_id], "engine_outputs", None) or []
-
-    stage_clients = getattr(getattr(omni_runner.omni, "engine", None), "stage_clients", None)
-    if stage_clients is not None:
-        return getattr(stage_clients[stage_id], "engine_outputs", None) or []
-
-    raise AttributeError("Unable to locate stage outputs on Omni runner")
-
-
-def _patched_stage_config(base_stage_config: str, model_dir: Path, tmp_dir: Path) -> str:
-    cfg = yaml.safe_load(Path(base_stage_config).read_text(encoding="utf-8"))
-    tokenizer_path = str(model_dir / "CosyVoice-BlankEN")
-    for stage in cfg.get("stage_args", []):
-        engine_args = stage.setdefault("engine_args", {})
-        engine_args["tokenizer"] = tokenizer_path
-        engine_args["enforce_eager"] = True
-        engine_args["hf_overrides"] = {"architectures": ["CosyVoice3Model"]}
-    out_path = tmp_dir / Path(base_stage_config).name
-    out_path.write_text(yaml.safe_dump(cfg, sort_keys=False), encoding="utf-8")
-    return str(out_path)
-
-
-def _build_reference_inputs(prompt_audio: tuple[np.ndarray, int]) -> list[dict[str, object]]:
-    return [
-        {
-            "prompt": REFERENCE_SYNTH_TEXT,
-            "multi_modal_data": {"audio": prompt_audio},
-            "modalities": ["audio"],
-            "mm_processor_kwargs": {"prompt_text": REFERENCE_PROMPT_TEXT},
-        }
-    ]
-
-
-@pytest.mark.core_model
-@pytest.mark.omni
-@hardware_test(res={"cuda": "L4"}, num_cards=1)
-@pytest.mark.parametrize("base_stage_config", STAGE_CONFIGS)
-def test_cosyvoice3_offline_reference_zero_shot(base_stage_config: str) -> None:
-    """CosyVoice3 zero-shot reference inference should stop cleanly and produce sane audio."""
-    prompt_audio, prompt_sr = _load_reference_prompt_wav()
-    model_dir = _resolve_model_dir()
-    expected_stop_token = int(CosyVoice3Config().llm["eos_token_id"])
-
-    with tempfile.TemporaryDirectory(prefix="cv3-e2e-") as tmp:
-        stage_config = _patched_stage_config(base_stage_config, model_dir, Path(tmp))
-        with OmniRunner(
-            str(model_dir), seed=42, stage_configs_path=stage_config, stage_init_timeout=300
-        ) as omni_runner:
-            sampling_params_list = omni_runner.get_default_sampling_params_list()
-            sampling_params_list[0] = _reference_zero_shot_stage0_sampling(text=REFERENCE_SYNTH_TEXT)
-
-            outputs = omni_runner.omni.generate(
-                _build_reference_inputs((prompt_audio, prompt_sr)), sampling_params_list
-            )
-
-            assert outputs, "No outputs returned"
-            audio_mm = outputs[0].multimodal_output
-            assert "audio" in audio_mm, "No audio output found"
-
-            audio = _concat_audio(audio_mm["audio"])
-            assert audio.size > 0, "Generated audio is empty"
-
-            sr_val = audio_mm.get("sr", 24000)
-            if isinstance(sr_val, list) and sr_val:
-                sr_val = sr_val[-1]
-            if hasattr(sr_val, "item"):
-                sr_val = sr_val.item()
-            sr = int(sr_val)
-            assert sr == 24000, f"Unexpected sample_rate={sr}"
-
-            duration_s = audio.size / sr
-            assert 2.8 <= duration_s <= 8.8, f"Unexpected duration={duration_s:.3f}s (samples={audio.size}, sr={sr})"
-
-            stage0_outputs = _get_stage_engine_outputs(omni_runner, 0)
-            if stage0_outputs:
-                completion = stage0_outputs[0].outputs[0]
-                finish_reason = getattr(completion, "finish_reason", None)
-                stop_reason = getattr(completion, "stop_reason", None)
-                num_tokens = len(getattr(completion, "token_ids", []) or [])
-
-                assert finish_reason == "stop", f"Stage-0 finish_reason={finish_reason}, expected 'stop'"
-                assert int(stop_reason) == expected_stop_token, (
-                    f"Stage-0 stop_reason={stop_reason}, expected {expected_stop_token}"
-                )
-                assert 80 <= num_tokens <= 220, f"Stage-0 num_tokens={num_tokens}, expected sane stop-bound range"
-            else:
-                assert "async_chunk" in Path(base_stage_config).name, "Stage-0 produced no engine outputs"

tests/e2e/offline_inference/test_flux_autoround_w4a16.py

@@ -1,127 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-"""E2E tests for FLUX AutoRound W4A16 quantized inference.
-
-These tests require:
-  - A CUDA GPU
-  - The quantized model checkpoint (vllm-project-org/FLUX.1-dev-AutoRound-w4a16)
-"""
-
-import gc
-import sys
-from pathlib import Path
-
-import pytest
-import torch
-from vllm.distributed.parallel_state import cleanup_dist_env_and_memory
-
-from tests.utils import DeviceMemoryMonitor, hardware_test
-from vllm_omni.inputs.data import OmniDiffusionSamplingParams
-from vllm_omni.outputs import OmniRequestOutput
-from vllm_omni.platforms import current_omni_platform
-
-# ruff: noqa: E402
-REPO_ROOT = Path(__file__).resolve().parents[2]
-if str(REPO_ROOT) not in sys.path:
-    sys.path.insert(0, str(REPO_ROOT))
-
-from vllm_omni import Omni
-
-QUANTIZED_MODEL = "vllm-project-org/FLUX.1-dev-AutoRound-w4a16"
-BASELINE_MODEL = "black-forest-labs/FLUX.1-dev"
-
-# Allow overriding via environment for local testing
-import os as _os
-
-QUANTIZED_MODEL = _os.environ.get("FLUX_AUTOROUND_MODEL", QUANTIZED_MODEL)
-BASELINE_MODEL = _os.environ.get("FLUX_BASELINE_MODEL", BASELINE_MODEL)
-
-# Small resolution to keep GPU memory & time manageable
-HEIGHT = 256
-WIDTH = 256
-NUM_STEPS = 2  # minimal for smoke-test
-
-
-def _generate_image(model_name: str, **extra_kwargs) -> tuple[list, float]:
-    """Load a FLUX model, generate one image, return (images, peak_memory_mb)."""
-    gc.collect()
-    current_omni_platform.empty_cache()
-    device_index = current_omni_platform.current_device()
-    current_omni_platform.reset_peak_memory_stats()
-    monitor = DeviceMemoryMonitor(device_index=device_index, interval=0.02)
-    monitor.start()
-
-    m = Omni(model=model_name, enforce_eager=True, **extra_kwargs)
-
-    current_omni_platform.reset_peak_memory_stats()
-    outputs = m.generate(
-        "a photo of a cat sitting on a laptop keyboard",
-        OmniDiffusionSamplingParams(
-            height=HEIGHT,
-            width=WIDTH,
-            num_inference_steps=NUM_STEPS,
-            guidance_scale=0.0,
-            generator=torch.Generator(device=current_omni_platform.device_type).manual_seed(42),
-        ),
-    )
-
-    peak = monitor.peak_used_mb
-    monitor.stop()
-
-    first_output = outputs[0]
-    assert first_output.final_output_type == "image"
-    req_out = first_output.request_output
-    assert isinstance(req_out, OmniRequestOutput) and hasattr(req_out, "images")
-    images = req_out.images
-
-    del m
-    gc.collect()
-    current_omni_platform.empty_cache()
-
-    return images, peak
-
-
-@pytest.mark.advanced_model
-@pytest.mark.diffusion
-@hardware_test(res={"cuda": "L4"})
-def test_flux_autoround_w4a16_generates_image():
-    """Load the W4A16 quantized FLUX model and verify it produces a valid image."""
-    images, _ = _generate_image(QUANTIZED_MODEL)
-
-    assert len(images) >= 1, "Expected at least one generated image"
-    img = images[0]
-    assert img.width == WIDTH, f"Expected width {WIDTH}, got {img.width}"
-    assert img.height == HEIGHT, f"Expected height {HEIGHT}, got {img.height}"
-
-    # Sanity: image should not be blank (all-zero)
-    import numpy as np
-
-    arr = np.array(img)
-    assert arr.std() > 1.0, "Generated image appears blank (std ≈ 0)"
-
-
-@pytest.mark.advanced_model
-@pytest.mark.diffusion
-@hardware_test(res={"cuda": "L4"})
-def test_flux_autoround_w4a16_memory_savings():
-    """Compare peak GPU memory of quantized vs FP16 baseline.
-
-    The W4A16 model should use meaningfully less memory than the
-    BF16/FP16 baseline since weights are 4-bit instead of 16-bit.
-    """
-    quant_images, quant_peak = _generate_image(QUANTIZED_MODEL)
-    cleanup_dist_env_and_memory()
-    _, baseline_peak = _generate_image(BASELINE_MODEL)
-
-    print(f"Quantized (W4A16) peak memory: {quant_peak:.0f} MB")
-    print(f"Baseline (BF16) peak memory:   {baseline_peak:.0f} MB")
-    print(f"Savings:                        {baseline_peak - quant_peak:.0f} MB")
-
-    # W4A16 weights are 4x smaller than BF16/FP16.  FLUX.1-dev transformer
-    # is ~12 GB in BF16, so we expect ~9 GB savings on weights alone.
-    # Use a conservative threshold to account for activations and overhead.
-    min_savings_mb = 2000
-    assert quant_peak + min_savings_mb < baseline_peak, (
-        f"Quantized model ({quant_peak:.0f} MB) should use at least "
-        f"{min_savings_mb} MB less than baseline ({baseline_peak:.0f} MB)"
-    )

tests/e2e/offline_inference/test_hunyuanimage3_text2img.py

@@ -1,347 +0,0 @@
-# ruff: noqa: E501
-from collections.abc import Generator
-from pathlib import Path
-
-import pytest
-import torch
-import torch.nn.functional as F
-from PIL import Image
-from transformers import CLIPModel, CLIPProcessor
-
-from vllm_omni import Omni
-from vllm_omni.inputs.data import OmniDiffusionSamplingParams
-from vllm_omni.platforms import current_omni_platform
-
-PROMPT = "A brown and white dog is running on the grass"
-MODEL_NAME = "tencent/HunyuanImage-3.0"
-LOCAL_CLIP_PATH = "openai/clip-vit-base-patch32"
-REPO_ROOT = Path(__file__).resolve().parents[3]
-STAGE_CONFIG_PATH = REPO_ROOT / "vllm_omni" / "model_executor" / "stage_configs" / "hunyuan_image_3_moe.yaml"
-
-pytestmark = [pytest.mark.advanced_model, pytest.mark.diffusion]
-
-# System prompt type. Options: None, dynamic, en_vanilla, en_recaption, en_think_recaption, en_unified
-# Below are the CLIP embedding tensors from the official HunyuanImage model (seed=1234, prompt: "A brown and white dog is running on the grass").
-# SEED_1234 denotes the output without system prompt, while the remaining entries correspond to outputs generated with different system prompts.
-# fmt: off
-SEED_1234 = torch.tensor(
-    [
-        0.027797, 0.028964, -0.005051, 0.001059, 0.017021, -0.034029, 0.021989, 0.033318, -0.000308, 0.016179, 0.010504, -0.034201, 0.050230, -0.021170, 0.083530, -0.003621,
-        0.040758, 0.039913, 0.044305, -0.019285, -0.058387, -0.001099, 0.042782, -0.036136, -0.014955, 0.002147, 0.009439, 0.012943, -0.028732, -0.018349, 0.002861, 0.013019,
-        0.014362, -0.038833, 0.029413, 0.020724, 0.002714, 0.010416, -0.020527, 0.050266, -0.081026, -0.006814, -0.007457, -0.032333, 0.008417, -0.122455, -0.006085, -0.025610,
-        0.012614, 0.025817, -0.005419, 0.038657, 0.000789, 0.067111, 0.002818, 0.028696, 0.047305, -0.009993, -0.019508, 0.038604, 0.099657, 0.026728, 0.012361, 0.013626,
-        0.023164, -0.037186, 0.007535, 0.054645, -0.009012, -0.019383, -0.005234, -0.018715, -0.000346, 0.051317, -0.028744, 0.029933, -0.006382, -0.018414, -0.033906, -0.028892,
-        -0.015301, -0.004276, 0.014626, -0.008505, 0.013717, -0.027323, -0.001332, -0.040227, 0.047021, -0.019082, -0.037260, -0.029780, -0.594026, 0.016573, -0.010523, 0.042616,
-        -0.013136, 0.030540, -0.151685, -0.005367, 0.016209, -0.034183, 0.009852, 0.038452, 0.005494, -0.017887, -0.007167, 0.017262, -0.038980, 0.011995, 0.021952, -0.031660,
-        0.020507, -0.035880, 0.035183, -0.026975, -0.050788, -0.002553, 0.037774, -0.020082, -0.015403, 0.045022, 0.072167, -0.029237, 0.003895, -0.051250, 0.008581, 0.023545,
-        -0.026827, 0.020895, 0.041780, -0.040766, -0.008146, 0.080630, 0.000404, 0.032003, -0.005279, -0.090707, -0.013813, 0.010204, -0.001513, 0.016394, -0.001321, 0.020535,
-        -0.038645, 0.024858, 0.024378, 0.018717, -0.056314, 0.024402, 0.018694, 0.029009, -0.008502, -0.014694, -0.028345, 0.005202, 0.046116, -0.032166, -0.030706, -0.038738,
-        -0.031356, -0.009683, 0.040069, 0.001596, -0.012621, 0.018590, -0.024138, 0.035330, 0.011546, 0.015791, -0.026932, 0.004531, 0.022455, -0.012871, 0.013915, -0.009567,
-        -0.010976, 0.013497, 0.042590, 0.002072, -0.052718, -0.045494, 0.013036, -0.005403, -0.005947, -0.003437, 0.016653, -0.016805, -0.040291, 0.007927, 0.001296, -0.008319,
-        0.021514, -0.001452, -0.121998, 0.015396, -0.022594, -0.006977, -0.040108, -0.035550, -0.021872, -0.014721, 0.019799, 0.036556, 0.015072, -0.057988, -0.011684, -0.045220,
-        -0.026295, 0.052647, 0.013741, -0.013428, 0.061794, 0.021431, -0.011316, -0.009963, 0.008198, 0.027746, 0.074219, -0.019499, 0.042673, 0.016028, 0.007214, -0.010650,
-        -0.019682, 0.001902, 0.038867, -0.007333, 0.031749, 0.004391, 0.018688, 0.044654, 0.030615, -0.027816, 0.031711, -0.056952, -0.033499, -0.039368, 0.025801, -0.027610,
-        -0.009329, -0.001799, 0.024061, -0.012593, -0.050266, -0.012512, 0.019528, -0.083434, 0.018238, 0.034138, -0.020120, -0.009910, -0.002280, 0.035325, 0.034440, -0.055205,
-        -0.017698, -0.000439, -0.034703, 0.013356, -0.037287, 0.048494, -0.018570, 0.028069, 0.019269, -0.007263, -0.008521, 0.000426, -0.016677, 0.056162, -0.011944, 0.017322,
-        0.022219, -0.014266, -0.009292, -0.009979, 0.014973, 0.011623, -0.017799, 0.032925, -0.024668, 0.007312, -0.025035, -0.008967, -0.026827, 0.011889, -0.138517, -0.009608,
-        -0.020592, -0.001272, 0.015676, -0.025706, 0.031775, -0.004195, 0.026876, -0.014748, -0.025966, -0.008741, 0.035437, 0.017139, -0.005140, -0.007101, -0.012510, -0.023600,
-        0.032969, -0.005510, 0.020010, 0.032567, 0.015558, 0.004265, -0.036300, 0.048210, 0.080424, -0.052820, -0.002063, -0.020875, 0.052530, -0.001638, -0.020299, -0.035202,
-        0.087818, 0.034614, -0.032735, 0.033201, -0.001751, 0.029574, 0.009926, 0.011619, -0.001267, -0.020149, -0.003826, -0.029860, 0.011437, -0.051276, 0.024344, 0.003096,
-        -0.011573, 0.038228, -0.005730, -0.052328, 0.001909, -0.025877, 0.019976, -0.010160, 0.023892, 0.049161, -0.028978, 0.018700, -0.026460, 0.001090, -0.072128, -0.008406,
-        0.010828, 0.020621, -0.005706, 0.023797, 0.036231, -0.112069, 0.017601, 0.007496, 0.045999, 0.016771, 0.021977, 0.022305, 0.018377, 0.002036, -0.029815, -0.082922,
-        -0.012710, -0.026355, 0.003790, 0.017472, -0.023148, -0.002901, -0.057854, 0.028393, 0.230866, -0.023486, 0.051094, 0.047508, 0.018957, -0.037130, 0.001054, -0.026126,
-        0.021970, -0.046915, -0.019419, -0.014077, 0.002502, -0.079454, -0.057149, -0.081701, 0.041979, -0.043074, -0.009425, -0.035776, -0.021794, -0.004826, -0.057263, -0.072940,
-        0.037651, -0.013991, -0.043863, -0.020581, 0.034319, -0.052566, -0.010355, -0.022963, 0.027144, -0.017339, 0.088930, -0.000670, -0.026547, -0.026586, -0.032531, 0.040314,
-        0.010148, 0.021104, 0.009228, -0.073227, 0.036650, -0.019337, 0.010211, -0.089620, -0.024676, -0.020729, -0.004070, 0.000784, -0.110561, 0.015390, 0.027151, -0.003228,
-        -0.066704, -0.004797, -0.026117, -0.018131, -0.090114, 0.020659, -0.007157, 0.013608, -0.022324, 0.027487, 0.018873, 0.027854, 0.045085, -0.039992, -0.017829, 0.011071,
-        -0.011393, -0.004454, -0.037189, -0.030299, 0.059668, 0.005064, 0.024655, -0.037239, 0.046882, -0.010356, -0.009690, 0.061909, -0.024736, 0.016849, 0.000784, 0.000201,
-        0.066165, 0.010234, -0.012134, -0.002823, -0.060847, 0.008953, 0.010348, 0.022292, -0.044602, -0.020981, 0.038839, 0.006616, -0.016836, -0.043995, -0.005463, -0.036413,
-        0.034895, -0.018008, -0.009543, -0.025080, -0.035243, 0.042696, -0.028911, -0.030676, -0.038542, -0.027798, -0.026607, 0.019467, 0.070629, -0.037356, -0.042648, -0.000284,
-        0.033095, 0.077781, -0.052930, 0.022515, -0.029926, -0.033821, -0.003277, -0.000038, -0.026871, 0.018223, -0.004221, 0.023454, -0.030611, -0.006396, -0.009873, -0.008402,
-    ],
-    dtype=torch.float32,
-)
-SYSTEM_PROMPT_DYNAMIC = torch.tensor(
-    [
-        0.010809, 0.021177, -0.017600, -0.016814, 0.012351, -0.024554, 0.018299, 0.039305, 0.003331, 0.030473, 0.005557, -0.040898, 0.047294, -0.016136, 0.076989, -0.002723,
-        0.017622, 0.042330, 0.058266, -0.016232, -0.029502, 0.004529, 0.033543, -0.041481, -0.017631, 0.002727, 0.018874, 0.019932, -0.030052, -0.009997, 0.004582, 0.002135,
-        -0.003720, -0.030923, 0.021174, 0.034033, -0.007096, 0.011522, -0.009518, 0.055688, -0.092351, -0.003914, 0.004589, -0.032635, 0.012479, -0.140607, -0.014141, -0.031821,
-        0.001396, 0.026780, -0.007623, 0.039957, 0.006434, 0.047516, 0.014377, 0.015237, 0.034212, 0.003576, -0.027357, 0.038888, 0.087272, 0.020248, 0.015165, 0.016002,
-        0.020781, -0.040509, -0.008929, 0.080857, -0.002642, -0.009738, -0.005683, -0.000615, -0.012801, 0.046457, -0.045004, 0.024689, 0.002498, -0.017333, -0.027366, -0.023231,
-        -0.006064, -0.021505, 0.007405, -0.021249, 0.026252, -0.018690, 0.020093, -0.036954, 0.037510, -0.032027, -0.030871, -0.011173, -0.618627, 0.021213, -0.004366, 0.029555,
-        -0.004324, 0.020221, -0.143832, -0.021386, 0.010482, -0.042113, 0.016164, 0.040350, 0.014627, -0.011778, -0.018102, 0.035380, -0.020305, 0.010590, 0.009227, -0.011415,
-        0.018623, -0.036384, 0.031003, -0.017073, -0.056456, -0.010423, 0.033029, -0.023511, -0.008717, 0.045716, 0.068273, -0.027886, 0.009665, -0.039801, 0.001465, 0.024361,
-        -0.015039, 0.022903, 0.033362, -0.022804, 0.008631, 0.076518, 0.000619, 0.022786, -0.015435, -0.095242, -0.006092, 0.015496, -0.009081, 0.015740, 0.004280, 0.013103,
-        -0.031836, 0.034241, 0.031836, 0.032636, -0.053721, 0.034370, 0.019172, 0.018383, 0.006907, -0.036039, -0.027927, 0.008646, 0.040496, -0.060314, -0.039116, -0.021488,
-        -0.031682, -0.005077, 0.034920, 0.002148, -0.008087, 0.002024, -0.008480, 0.041096, 0.011401, 0.020380, -0.025078, 0.005002, 0.022252, -0.014577, 0.008051, -0.014476,
-        -0.007078, 0.021075, 0.036965, 0.005343, -0.038671, -0.037222, 0.014052, -0.009952, -0.003958, -0.001878, 0.017848, -0.016608, -0.030813, 0.010921, 0.001068, 0.003095,
-        0.007076, -0.001936, -0.102996, 0.006838, -0.005243, -0.009140, -0.043796, -0.027227, -0.008426, -0.013177, 0.015602, 0.021036, 0.025484, -0.064836, -0.003593, -0.038036,
-        -0.023102, 0.064053, 0.007850, 0.000771, 0.039297, 0.011903, -0.015866, -0.017612, 0.006308, 0.024342, 0.086761, -0.016705, 0.039239, 0.025079, -0.006452, 0.003174,
-        -0.010146, 0.010787, 0.035932, -0.015346, 0.037191, 0.010990, 0.011573, 0.044958, 0.035560, -0.017339, 0.018878, -0.025394, -0.044339, -0.029852, 0.015951, -0.032248,
-        -0.012019, 0.013497, 0.012224, -0.001284, -0.034041, -0.015768, 0.000230, -0.086076, 0.024878, 0.031929, -0.016668, -0.019815, -0.001325, 0.007944, 0.017674, -0.036097,
-        -0.019651, -0.001272, -0.032842, 0.002056, -0.037140, 0.043191, -0.003710, 0.011767, 0.020313, -0.018396, -0.015935, 0.010228, -0.017349, 0.049363, -0.010007, 0.019533,
-        0.018076, 0.016608, -0.005523, -0.007793, 0.016868, 0.019341, -0.008236, 0.026765, -0.025324, -0.007849, -0.023648, -0.007791, -0.018508, 0.015357, -0.166499, -0.003718,
-        -0.035447, -0.005229, 0.019327, -0.014207, 0.028433, -0.002619, 0.013888, -0.033146, -0.017015, 0.004677, 0.039554, 0.003803, -0.014592, -0.018886, -0.023868, -0.022708,
-        0.033661, 0.008626, 0.015687, 0.046395, 0.014173, 0.015083, -0.025994, 0.039120, 0.076334, -0.061165, 0.001791, -0.017579, 0.067567, -0.002415, -0.032495, -0.025576,
-        0.079027, 0.036370, -0.013303, 0.030510, -0.009061, 0.019135, 0.015627, 0.024864, 0.015093, -0.017066, -0.014075, -0.021907, 0.017388, -0.033492, 0.013317, -0.000040,
-        0.003396, 0.044030, -0.009194, -0.049524, -0.005015, -0.040007, 0.009104, 0.000580, 0.005603, 0.035891, -0.038913, 0.023239, -0.017022, -0.002695, -0.095759, 0.018503,
-        0.017365, 0.011104, -0.003433, 0.024113, 0.052609, -0.085274, 0.027565, -0.005833, 0.020700, 0.015842, 0.019148, 0.020203, -0.000698, -0.005337, -0.037400, -0.060144,
-        -0.031893, -0.038396, -0.001949, 0.018901, -0.014268, -0.004721, -0.055913, 0.013814, 0.215024, -0.011357, 0.057530, 0.050092, 0.016513, -0.059254, 0.001494, -0.031472,
-        0.032190, -0.047512, -0.020501, -0.002571, 0.007844, -0.063630, -0.043938, -0.079595, 0.032820, -0.021659, -0.003738, -0.035267, -0.013794, -0.021172, -0.046356, -0.077079,
-        0.021526, -0.007447, -0.050276, -0.029743, 0.022208, -0.039137, -0.021426, -0.029825, 0.029390, -0.002943, 0.073158, -0.000435, -0.032029, -0.038524, -0.029886, 0.017473,
-        0.013513, 0.022738, 0.000632, -0.073718, 0.029219, -0.018896, 0.007302, -0.116122, -0.013324, -0.012214, -0.005960, -0.003720, -0.155869, 0.019896, 0.016919, -0.021133,
-        -0.066911, -0.000926, -0.020871, -0.015295, -0.086108, 0.014918, -0.009284, 0.001689, -0.038155, 0.039163, 0.015988, 0.014413, 0.034205, -0.053273, 0.001687, 0.012227,
-        -0.007341, -0.006123, -0.005731, -0.026863, 0.060196, 0.028929, 0.019328, -0.033709, 0.038789, -0.015624, 0.013323, 0.053821, -0.015538, -0.001610, 0.012959, -0.013897,
-        0.082010, 0.012866, -0.017269, 0.000017, -0.059458, 0.015870, 0.028455, 0.025234, -0.051163, -0.022976, 0.011866, -0.005613, -0.008738, -0.047658, -0.002155, -0.029432,
-        0.039242, -0.013491, -0.001641, -0.024210, -0.019187, 0.026716, -0.025698, -0.027591, -0.034678, -0.002473, -0.019391, 0.017597, 0.064385, -0.029104, -0.034501, -0.004955,
-        0.015008, 0.060749, -0.051693, 0.020279, -0.027170, -0.027003, 0.000254, 0.011352, -0.028116, 0.028938, -0.007224, 0.019978, -0.025379, -0.004874, -0.019361, -0.020278,
-    ],
-    dtype=torch.float32,
-)
-SYSTEM_EN_RECAPTION = torch.tensor(
-    [
-        0.007721, 0.015421, -0.019305, -0.000920, 0.016031, -0.019730, 0.029683, 0.026810, -0.010510, 0.021463, 0.008833, -0.040851, 0.043260, -0.007042, 0.057224, 0.011995,
-        0.007818, 0.046369, 0.059838, -0.028548, -0.047399, -0.000983, 0.024343, -0.052259, -0.013638, 0.006856, 0.009186, 0.014235, -0.031497, -0.008644, -0.009349, 0.018900,
-        0.002913, -0.022475, 0.039518, 0.019052, -0.007600, 0.010634, -0.011830, 0.075675, -0.071738, -0.014947, 0.004995, -0.025804, -0.002553, -0.093262, 0.002881, -0.033744,
-        -0.007234, 0.013659, 0.009897, 0.039185, -0.005366, 0.041534, -0.005924, 0.019786, 0.048566, -0.009356, -0.027360, 0.042557, 0.091286, 0.009286, 0.015410, 0.028166,
-        0.022476, -0.025162, 0.012144, 0.084603, -0.003150, -0.008549, -0.002099, -0.014987, -0.019480, 0.046843, -0.030613, 0.015557, -0.008965, -0.008798, -0.027032, -0.014112,
-        0.018703, -0.014749, -0.000928, -0.024660, 0.024004, 0.004560, 0.028156, -0.028467, 0.025444, -0.038699, -0.014927, -0.031593, -0.648498, 0.018529, 0.003378, 0.030188,
-        -0.002314, 0.014950, -0.146615, -0.009005, 0.016579, -0.037867, 0.020907, 0.033160, 0.007877, -0.026345, -0.056428, 0.031255, -0.018404, 0.013334, 0.009988, -0.022790,
-        0.020803, -0.036862, 0.036222, -0.006646, -0.058084, -0.012036, 0.044199, -0.027665, -0.015779, 0.051554, 0.059970, -0.025977, 0.003967, -0.035247, -0.000488, 0.023182,
-        0.000468, 0.019190, 0.047268, -0.032279, -0.005302, 0.078669, -0.001915, 0.024918, -0.014952, -0.078905, -0.018333, 0.001362, -0.015115, 0.005435, 0.002313, 0.018766,
-        -0.032773, 0.037344, 0.024061, 0.012143, -0.057106, 0.029490, 0.019537, 0.009099, 0.026064, -0.015927, -0.037047, 0.006002, 0.025191, -0.035318, -0.032245, -0.047822,
-        -0.023568, -0.004533, 0.025100, 0.002758, -0.002649, -0.012287, -0.012139, 0.043080, 0.003295, 0.024667, -0.021050, 0.006752, 0.025315, -0.011127, 0.009800, -0.021343,
-        -0.024866, 0.010098, 0.026954, 0.012467, -0.035866, -0.031780, 0.007479, -0.003388, -0.012619, -0.012099, 0.014974, -0.001908, -0.032700, 0.004703, 0.003238, -0.007498,
-        0.023241, 0.002715, -0.111739, 0.003317, 0.006475, -0.019792, -0.046558, -0.032593, -0.020762, -0.005059, 0.016934, 0.029195, 0.028744, -0.050633, 0.001907, -0.028791,
-        -0.016695, 0.052143, 0.010439, 0.007204, 0.028502, 0.012607, -0.012414, -0.031238, 0.007305, 0.032309, 0.087924, -0.010530, 0.029925, 0.032666, -0.002202, 0.017539,
-        -0.009091, -0.001631, 0.024906, -0.013102, 0.031772, 0.018465, 0.012035, 0.031460, 0.030193, 0.005289, 0.025859, -0.038971, -0.046577, -0.025852, 0.035235, -0.038514,
-        0.001042, 0.013012, 0.023701, -0.014630, -0.029269, -0.011981, 0.008219, -0.067347, -0.003456, 0.028198, -0.008657, -0.017773, 0.010540, 0.023964, 0.021012, -0.034465,
-        -0.023748, 0.004065, -0.021598, 0.008440, -0.031533, 0.038390, -0.007680, -0.003852, 0.016136, -0.017906, -0.008927, 0.006300, -0.001251, 0.029337, -0.008632, 0.020568,
-        0.021560, -0.007222, 0.005313, -0.013089, 0.012299, 0.031303, -0.013951, 0.016547, -0.024771, -0.008753, -0.030908, -0.014421, -0.017656, 0.014044, -0.114986, 0.000956,
-        -0.035588, 0.003756, 0.015383, -0.013358, 0.009385, -0.001359, 0.012623, -0.028724, 0.001607, 0.012809, 0.032668, 0.011834, -0.015587, -0.007170, -0.021344, -0.019664,
-        0.017690, -0.014538, 0.016511, 0.038037, 0.029919, 0.020907, -0.018565, 0.032964, 0.078548, -0.050386, -0.003012, -0.016965, 0.064131, 0.008077, -0.025879, -0.035820,
-        0.095075, 0.019901, -0.019114, 0.022832, 0.003741, 0.027148, 0.018231, 0.027741, 0.020328, 0.001700, -0.006939, -0.024154, 0.018523, -0.029819, 0.008050, -0.004477,
-        0.006087, 0.056878, -0.009083, -0.061537, -0.011531, -0.037551, 0.000434, -0.005843, 0.024739, 0.032020, -0.053119, 0.020704, -0.012385, -0.002726, -0.082489, 0.009072,
-        0.013341, 0.000316, 0.001899, 0.022868, 0.034407, -0.066857, 0.020589, 0.012195, 0.023211, -0.001520, 0.000897, 0.029670, -0.015930, 0.006509, -0.035172, -0.061215,
-        -0.014099, -0.038584, -0.012213, 0.018613, -0.012365, -0.002777, -0.055184, 0.017146, 0.214358, -0.015750, 0.052488, 0.045205, 0.025334, -0.054615, 0.002117, -0.038122,
-        0.012402, -0.053418, -0.025405, 0.007235, 0.013208, -0.092481, -0.048700, -0.085186, 0.029039, -0.036767, -0.000777, -0.017625, -0.012556, -0.004887, -0.033660, -0.082310,
-        0.013387, -0.003256, -0.062981, -0.019886, 0.017624, -0.037421, -0.020743, -0.020894, 0.041974, -0.008502, 0.088413, -0.018697, -0.029398, -0.029389, -0.043721, 0.013872,
-        0.003944, 0.030361, 0.005355, -0.081355, 0.041843, -0.016395, 0.011954, -0.060440, -0.000966, -0.019101, 0.006803, -0.011310, -0.148581, 0.020342, 0.012795, -0.016473,
-        -0.053300, -0.012340, -0.016640, -0.029834, -0.082405, 0.011859, -0.004255, -0.004396, -0.012515, 0.031962, 0.030438, 0.013792, 0.031557, -0.047200, 0.006485, 0.024815,
-        -0.019376, -0.011454, -0.034184, -0.021329, 0.050115, 0.021720, 0.002874, -0.047163, 0.044031, -0.014663, 0.020534, 0.056017, 0.007017, 0.003323, 0.005734, -0.002777,
-        0.082836, 0.012048, -0.023236, -0.007401, -0.071598, 0.016760, 0.017282, 0.028306, -0.026220, -0.008016, -0.000202, -0.020271, -0.019828, -0.046986, -0.005805, -0.039647,
-        0.042879, -0.004463, 0.007753, -0.028916, -0.020612, 0.028833, -0.039839, -0.052447, -0.013275, -0.002407, -0.018937, 0.033216, 0.075535, -0.045026, -0.009901, 0.016637,
-        -0.000322, 0.073925, -0.055701, 0.014912, -0.045671, -0.021189, 0.006761, -0.002015, -0.027410, 0.018250, -0.015916, 0.016254, -0.044964, 0.029261, -0.029319, -0.005222,
-    ],
-    dtype=torch.float32,
-)
-SYSTEM_EN_THINK_RECAPTION = torch.tensor(
-    [
-        0.011004, 0.017341, -0.019959, -0.018314, 0.016520, -0.027395, 0.017946, 0.039665, 0.000645, 0.035903, 0.002499, -0.045664, 0.039472, -0.013479, 0.081302, 0.000182,
-        0.006947, 0.042845, 0.059741, -0.010796, -0.035240, 0.004176, 0.029557, -0.043467, -0.017271, 0.006896, 0.010997, 0.022498, -0.023308, -0.013046, -0.000742, 0.016209,
-        -0.007152, -0.029868, 0.028747, 0.033743, -0.000227, 0.018419, -0.015023, 0.050376, -0.098475, -0.002375, 0.007897, -0.023936, 0.007843, -0.122463, -0.011680, -0.027267,
-        -0.007270, 0.021869, -0.011415, 0.043770, 0.000551, 0.048573, 0.003132, 0.014233, 0.037080, -0.004818, -0.028738, 0.044468, 0.073843, 0.016947, 0.014484, 0.021931,
-        0.020110, -0.032309, -0.003811, 0.095704, -0.006950, -0.007237, -0.005529, -0.020573, -0.016259, 0.041909, -0.038748, 0.018029, 0.005066, -0.021186, -0.020102, -0.019719,
-        0.006239, -0.021284, 0.004213, -0.024963, 0.032345, -0.012557, 0.037268, -0.038075, 0.040998, -0.032766, -0.023509, -0.016426, -0.627412, 0.022675, 0.000101, 0.023162,
-        -0.002081, 0.015922, -0.138671, -0.027995, 0.011579, -0.042859, 0.019935, 0.038077, 0.012640, -0.017377, -0.027456, 0.035151, -0.015756, 0.018530, 0.004646, -0.002589,
-        0.019645, -0.043736, 0.034947, -0.010166, -0.061165, -0.019195, 0.028909, -0.019415, -0.009485, 0.049566, 0.068621, -0.038644, 0.011278, -0.036133, 0.000564, 0.022611,
-        -0.013612, 0.020854, 0.030614, -0.025578, 0.005673, 0.076526, -0.004887, 0.027769, -0.022605, -0.092657, -0.013218, 0.008081, -0.015227, 0.018031, -0.005145, 0.015028,
-        -0.027193, 0.034767, 0.028710, 0.032007, -0.053175, 0.033528, 0.019437, 0.011517, 0.012107, -0.027679, -0.026937, 0.008612, 0.036909, -0.051484, -0.039971, -0.034372,
-        -0.023825, -0.003025, 0.033648, -0.001852, 0.007309, 0.000714, -0.001075, 0.038534, 0.007586, 0.016213, -0.025223, -0.001099, 0.015852, -0.011477, 0.020635, -0.010696,
-        -0.019634, 0.025613, 0.034374, 0.007169, -0.035000, -0.032268, 0.015114, -0.014217, -0.005229, -0.005495, 0.018189, -0.011360, -0.026755, 0.007036, -0.002333, -0.001174,
-        0.014729, 0.001739, -0.108591, 0.004699, 0.002048, -0.014801, -0.042855, -0.028846, -0.009609, -0.004500, 0.019466, 0.021848, 0.022140, -0.063035, -0.004272, -0.030798,
-        -0.018452, 0.055169, 0.012240, -0.003555, 0.038293, 0.008503, -0.016608, -0.021309, 0.000690, 0.027093, 0.088054, -0.008881, 0.034087, 0.030647, 0.003284, 0.005038,
-        -0.008359, 0.006311, 0.032462, -0.009699, 0.035283, 0.015261, 0.012827, 0.038169, 0.033959, -0.018048, 0.018122, -0.025259, -0.040084, -0.030879, 0.019853, -0.042558,
-        -0.011938, 0.019602, 0.016537, -0.003378, -0.027890, -0.014909, -0.005464, -0.071862, 0.012335, 0.021899, -0.017008, -0.023228, 0.003263, 0.004571, 0.016447, -0.029446,
-        -0.022645, -0.001261, -0.018573, 0.007431, -0.027587, 0.035362, -0.006785, -0.000614, 0.026044, -0.009056, -0.009843, 0.010467, -0.011929, 0.042025, -0.014068, 0.023113,
-        0.023880, 0.014948, 0.004370, -0.005262, 0.012587, 0.021608, -0.001783, 0.023697, -0.024945, -0.011533, -0.020953, -0.007205, -0.024693, 0.012961, -0.168760, 0.001767,
-        -0.041265, -0.007044, 0.015021, -0.008407, 0.029642, -0.000956, 0.008607, -0.035365, -0.012187, 0.011744, 0.032612, 0.006226, -0.015891, -0.017747, -0.022565, -0.024505,
-        0.031279, 0.004188, 0.011939, 0.038032, 0.008798, 0.012314, -0.024830, 0.034484, 0.076395, -0.060108, 0.001019, -0.016138, 0.067729, 0.003899, -0.029845, -0.019960,
-        0.086663, 0.040965, -0.010458, 0.027808, -0.006394, 0.017343, 0.014788, 0.024756, 0.016446, -0.012537, -0.008406, -0.028109, 0.013369, -0.033571, 0.012170, -0.002199,
-        0.005263, 0.052280, -0.018171, -0.047898, -0.010087, -0.038632, 0.006773, -0.000838, 0.011197, 0.038187, -0.049525, 0.021689, -0.007385, -0.005987, -0.094551, 0.019019,
-        0.012760, 0.009617, -0.002262, 0.030228, 0.047823, -0.079764, 0.023391, -0.005561, 0.018866, 0.012817, 0.020878, 0.027037, -0.013905, -0.002874, -0.035522, -0.046266,
-        -0.032448, -0.036010, -0.007776, 0.016512, -0.012279, -0.005665, -0.057974, 0.016967, 0.202836, -0.009066, 0.066093, 0.045689, 0.018319, -0.048465, 0.000242, -0.040874,
-        0.027824, -0.049045, -0.015616, -0.000307, 0.009163, -0.072975, -0.042979, -0.082254, 0.040549, -0.027049, 0.000725, -0.034118, -0.019604, -0.019097, -0.042483, -0.075446,
-        0.019387, -0.005218, -0.053573, -0.029975, 0.008195, -0.036608, -0.018920, -0.025610, 0.028426, -0.002688, 0.074996, -0.003423, -0.032505, -0.030565, -0.028142, 0.014437,
-        0.013359, 0.019376, 0.008356, -0.069731, 0.031824, -0.011103, 0.019327, -0.117090, -0.009352, -0.010290, -0.002129, -0.009198, -0.172915, 0.021232, 0.017274, -0.030060,
-        -0.061449, -0.006598, -0.013069, -0.012857, -0.081220, 0.019058, -0.004841, 0.003066, -0.037741, 0.041806, 0.018281, 0.009458, 0.036761, -0.044987, 0.003557, 0.008890,
-        -0.008011, -0.004063, -0.013474, -0.022090, 0.055398, 0.037475, 0.006991, -0.035962, 0.045503, -0.017162, 0.022391, 0.052754, -0.005924, -0.005936, 0.012673, -0.017922,
-        0.084548, 0.014695, -0.013817, 0.000421, -0.065167, 0.018269, 0.023317, 0.023523, -0.034229, -0.019588, 0.007911, -0.002426, -0.017109, -0.050870, 0.002848, -0.033077,
-        0.043451, -0.010609, -0.000375, -0.023206, -0.018155, 0.027102, -0.036006, -0.035115, -0.023922, 0.005989, -0.015372, 0.027123, 0.075210, -0.035302, -0.029799, 0.003642,
-        0.007714, 0.063498, -0.053234, 0.015699, -0.040459, -0.027354, -0.002433, 0.010923, -0.020134, 0.029292, -0.010176, 0.013508, -0.032403, 0.004323, -0.017504, -0.015237,
-    ],
-    dtype=torch.float32,
-)
-SYSTEM_EN_VANILLA = torch.tensor(
-    [
-        0.010809, 0.021177, -0.017600, -0.016814, 0.012351, -0.024554, 0.018299, 0.039305, 0.003331, 0.030473, 0.005557, -0.040898, 0.047294, -0.016136, 0.076989, -0.002723,
-        0.017622, 0.042330, 0.058266, -0.016232, -0.029502, 0.004529, 0.033543, -0.041481, -0.017631, 0.002727, 0.018874, 0.019932, -0.030052, -0.009997, 0.004582, 0.002135,
-        -0.003720, -0.030923, 0.021174, 0.034033, -0.007096, 0.011522, -0.009518, 0.055688, -0.092351, -0.003914, 0.004589, -0.032635, 0.012479, -0.140607, -0.014141, -0.031821,
-        0.001396, 0.026780, -0.007623, 0.039957, 0.006434, 0.047516, 0.014377, 0.015237, 0.034212, 0.003576, -0.027357, 0.038888, 0.087272, 0.020248, 0.015165, 0.016002,
-        0.020781, -0.040509, -0.008929, 0.080857, -0.002642, -0.009738, -0.005683, -0.000615, -0.012801, 0.046457, -0.045004, 0.024689, 0.002498, -0.017333, -0.027366, -0.023231,
-        -0.006064, -0.021505, 0.007405, -0.021249, 0.026252, -0.018690, 0.020093, -0.036954, 0.037510, -0.032027, -0.030871, -0.011173, -0.618627, 0.021213, -0.004366, 0.029555,
-        -0.004324, 0.020221, -0.143832, -0.021386, 0.010482, -0.042113, 0.016164, 0.040350, 0.014627, -0.011778, -0.018102, 0.035380, -0.020305, 0.010590, 0.009227, -0.011415,
-        0.018623, -0.036384, 0.031003, -0.017073, -0.056456, -0.010423, 0.033029, -0.023511, -0.008717, 0.045716, 0.068273, -0.027886, 0.009665, -0.039801, 0.001465, 0.024361,
-        -0.015039, 0.022903, 0.033362, -0.022804, 0.008631, 0.076518, 0.000619, 0.022786, -0.015435, -0.095242, -0.006092, 0.015496, -0.009081, 0.015740, 0.004280, 0.013103,
-        -0.031836, 0.034241, 0.031836, 0.032636, -0.053721, 0.034370, 0.019172, 0.018383, 0.006907, -0.036039, -0.027927, 0.008646, 0.040496, -0.060314, -0.039116, -0.021488,
-        -0.031682, -0.005077, 0.034920, 0.002148, -0.008087, 0.002024, -0.008480, 0.041096, 0.011401, 0.020380, -0.025078, 0.005002, 0.022252, -0.014577, 0.008051, -0.014476,
-        -0.007078, 0.021075, 0.036965, 0.005343, -0.038671, -0.037222, 0.014052, -0.009952, -0.003958, -0.001878, 0.017848, -0.016608, -0.030813, 0.010921, 0.001068, 0.003095,
-        0.007076, -0.001936, -0.102996, 0.006838, -0.005243, -0.009140, -0.043796, -0.027227, -0.008426, -0.013177, 0.015602, 0.021036, 0.025484, -0.064836, -0.003593, -0.038036,
-        -0.023102, 0.064053, 0.007850, 0.000771, 0.039297, 0.011903, -0.015866, -0.017612, 0.006308, 0.024342, 0.086761, -0.016705, 0.039239, 0.025079, -0.006452, 0.003174,
-        -0.010146, 0.010787, 0.035932, -0.015346, 0.037191, 0.010990, 0.011573, 0.044958, 0.035560, -0.017339, 0.018878, -0.025394, -0.044339, -0.029852, 0.015951, -0.032248,
-        -0.012019, 0.013497, 0.012224, -0.001284, -0.034041, -0.015768, 0.000230, -0.086076, 0.024878, 0.031929, -0.016668, -0.019815, -0.001325, 0.007944, 0.017674, -0.036097,
-        -0.019651, -0.001272, -0.032842, 0.002056, -0.037140, 0.043191, -0.003710, 0.011767, 0.020313, -0.018396, -0.015935, 0.010228, -0.017349, 0.049363, -0.010007, 0.019533,
-        0.018076, 0.016608, -0.005523, -0.007793, 0.016868, 0.019341, -0.008236, 0.026765, -0.025324, -0.007849, -0.023648, -0.007791, -0.018508, 0.015357, -0.166499, -0.003718,
-        -0.035447, -0.005229, 0.019327, -0.014207, 0.028433, -0.002619, 0.013888, -0.033146, -0.017015, 0.004677, 0.039554, 0.003803, -0.014592, -0.018886, -0.023868, -0.022708,
-        0.033661, 0.008626, 0.015687, 0.046395, 0.014173, 0.015083, -0.025994, 0.039120, 0.076334, -0.061165, 0.001791, -0.017579, 0.067567, -0.002415, -0.032495, -0.025576,
-        0.079027, 0.036370, -0.013303, 0.030510, -0.009061, 0.019135, 0.015627, 0.024864, 0.015093, -0.017066, -0.014075, -0.021907, 0.017388, -0.033492, 0.013317, -0.000040,
-        0.003396, 0.044030, -0.009194, -0.049524, -0.005015, -0.040007, 0.009104, 0.000580, 0.005603, 0.035891, -0.038913, 0.023239, -0.017022, -0.002695, -0.095759, 0.018503,
-        0.017365, 0.011104, -0.003433, 0.024113, 0.052609, -0.085274, 0.027565, -0.005833, 0.020700, 0.015842, 0.019148, 0.020203, -0.000698, -0.005337, -0.037400, -0.060144,
-        -0.031893, -0.038396, -0.001949, 0.018901, -0.014268, -0.004721, -0.055913, 0.013814, 0.215024, -0.011357, 0.057530, 0.050092, 0.016513, -0.059254, 0.001494, -0.031472,
-        0.032190, -0.047512, -0.020501, -0.002571, 0.007844, -0.063630, -0.043938, -0.079595, 0.032820, -0.021659, -0.003738, -0.035267, -0.013794, -0.021172, -0.046356, -0.077079,
-        0.021526, -0.007447, -0.050276, -0.029743, 0.022208, -0.039137, -0.021426, -0.029825, 0.029390, -0.002943, 0.073158, -0.000435, -0.032029, -0.038524, -0.029886, 0.017473,
-        0.013513, 0.022738, 0.000632, -0.073718, 0.029219, -0.018896, 0.007302, -0.116122, -0.013324, -0.012214, -0.005960, -0.003720, -0.155869, 0.019896, 0.016919, -0.021133,
-        -0.066911, -0.000926, -0.020871, -0.015295, -0.086108, 0.014918, -0.009284, 0.001689, -0.038155, 0.039163, 0.015988, 0.014413, 0.034205, -0.053273, 0.001687, 0.012227,
-        -0.007341, -0.006123, -0.005731, -0.026863, 0.060196, 0.028929, 0.019328, -0.033709, 0.038789, -0.015624, 0.013323, 0.053821, -0.015538, -0.001610, 0.012959, -0.013897,
-        0.082010, 0.012866, -0.017269, 0.000017, -0.059458, 0.015870, 0.028455, 0.025234, -0.051163, -0.022976, 0.011866, -0.005613, -0.008738, -0.047658, -0.002155, -0.029432,
-        0.039242, -0.013491, -0.001641, -0.024210, -0.019187, 0.026716, -0.025698, -0.027591, -0.034678, -0.002473, -0.019391, 0.017597, 0.064385, -0.029104, -0.034501, -0.004955,
-        0.015008, 0.060749, -0.051693, 0.020279, -0.027170, -0.027003, 0.000254, 0.011352, -0.028116, 0.028938, -0.007224, 0.019978, -0.025379, -0.004874, -0.019361, -0.020278,
-    ],
-    dtype=torch.float32,
-)
-SYSTEM_EN_UNIFIED = torch.tensor(
-    [
-        0.011409, 0.014191, -0.023163, -0.020119, 0.019190, -0.029559, 0.019616, 0.035872, 0.010434, 0.028709, 0.011616, -0.039422, 0.038369, -0.004631, 0.081177, 0.007400,
-        0.008903, 0.040408, 0.055323, -0.011950, -0.026940, 0.004916, 0.028101, -0.046200, -0.016732, 0.005115, 0.012100, 0.016136, -0.026057, -0.013827, -0.004914, 0.015261,
-        -0.010824, -0.028188, 0.022934, 0.026204, -0.003855, 0.013797, -0.014518, 0.050289, -0.100077, -0.002962, 0.009050, -0.028205, 0.016294, -0.128956, -0.012730, -0.023647,
-        -0.009306, 0.020066, 0.000033, 0.043619, 0.003250, 0.053425, 0.005889, 0.021529, 0.036032, -0.003254, -0.029715, 0.048345, 0.077978, 0.010674, 0.019296, 0.018721,
-        0.019244, -0.040115, -0.004245, 0.085214, -0.005280, -0.010746, -0.000164, -0.023405, -0.015641, 0.040193, -0.038735, 0.018966, -0.004031, -0.017879, -0.023017, -0.030379,
-        0.006468, -0.015959, 0.000532, -0.026530, 0.042640, -0.006095, 0.037899, -0.043658, 0.040965, -0.034682, -0.023729, -0.019291, -0.630840, 0.029658, 0.005462, 0.026650,
-        -0.000292, 0.013954, -0.149594, -0.019405, 0.015321, -0.045104, 0.030332, 0.031727, 0.012349, -0.009553, -0.022371, 0.034043, -0.014838, 0.015398, -0.003657, 0.000477,
-        0.021084, -0.041406, 0.029946, -0.013832, -0.057358, -0.018086, 0.031598, -0.031835, -0.006697, 0.040866, 0.068602, -0.042203, 0.007362, -0.036959, 0.003794, 0.026533,
-        -0.011873, 0.017343, 0.028333, -0.021804, 0.004007, 0.075133, 0.003340, 0.025326, -0.015068, -0.092280, -0.011514, 0.006827, -0.008254, 0.021181, -0.005035, 0.022263,
-        -0.022443, 0.043919, 0.026637, 0.028568, -0.056881, 0.036740, 0.024430, 0.015891, 0.012257, -0.031126, -0.030108, 0.007229, 0.026998, -0.051685, -0.033003, -0.031170,
-        -0.024021, 0.004235, 0.030164, 0.002674, 0.008018, 0.005532, 0.001621, 0.044790, 0.006413, 0.027160, -0.015022, 0.000911, 0.019723, -0.016244, 0.020077, -0.006847,
-        -0.014110, 0.022461, 0.031656, 0.002760, -0.039078, -0.026893, 0.006628, -0.011775, -0.000240, -0.005908, 0.014943, -0.012131, -0.021755, 0.004732, -0.005297, -0.002922,
-        0.014631, -0.002010, -0.112400, 0.000842, -0.002732, -0.014861, -0.052099, -0.034167, -0.011613, -0.006101, 0.013278, 0.018867, 0.026530, -0.068150, -0.003306, -0.032801,
-        -0.018523, 0.050875, 0.005488, -0.007241, 0.045707, 0.023119, -0.021519, -0.022683, 0.004806, 0.024827, 0.091371, -0.014424, 0.043836, 0.033094, 0.002390, 0.005450,
-        -0.004893, 0.013608, 0.031272, -0.002449, 0.031607, 0.014646, 0.014146, 0.043995, 0.028826, -0.012219, 0.021008, -0.020911, -0.036967, -0.036256, 0.013328, -0.038382,
-        -0.012084, 0.018183, 0.018782, -0.004697, -0.024284, -0.015474, -0.001463, -0.076015, 0.013923, 0.022125, -0.018765, -0.010793, 0.008409, 0.002067, 0.017961, -0.029716,
-        -0.020915, -0.001779, -0.009217, -0.001933, -0.036081, 0.042577, 0.000118, -0.013920, 0.014901, -0.016486, -0.010278, -0.000449, -0.017234, 0.042453, -0.009893, 0.021087,
-        0.017671, 0.009861, -0.004210, 0.004944, 0.015627, 0.014370, -0.001128, 0.030247, -0.019552, -0.014017, -0.020859, -0.002614, -0.024405, 0.016532, -0.173204, -0.001196,
-        -0.037415, -0.010990, 0.010449, -0.006124, 0.019211, 0.003695, 0.011679, -0.031852, -0.009764, 0.005773, 0.035793, 0.003455, -0.011772, -0.020532, -0.027434, -0.024761,
-        0.027483, -0.001554, 0.010411, 0.037888, 0.015619, 0.019186, -0.021204, 0.038158, 0.074991, -0.064521, -0.002503, -0.014499, 0.068165, 0.006145, -0.032891, -0.021540,
-        0.091385, 0.047584, -0.009590, 0.028004, -0.002962, 0.021061, 0.014854, 0.025840, 0.016068, -0.014364, -0.016418, -0.033454, 0.011734, -0.036518, 0.013015, -0.003966,
-        0.000855, 0.051373, -0.010960, -0.047078, -0.011048, -0.042015, 0.006818, 0.005483, 0.010251, 0.034951, -0.046162, 0.021258, -0.013397, -0.005259, -0.093775, 0.019974,
-        0.014992, 0.004043, -0.005931, 0.035662, 0.050723, -0.083293, 0.028047, -0.008042, 0.020763, 0.016763, 0.022913, 0.027129, -0.014314, -0.009854, -0.039019, -0.044870,
-        -0.028101, -0.038026, -0.006294, 0.018265, -0.015425, -0.007866, -0.052784, 0.010470, 0.200260, -0.007798, 0.064482, 0.046612, 0.025353, -0.059695, -0.001831, -0.039643,
-        0.025148, -0.042752, -0.014928, -0.010216, 0.014195, -0.069149, -0.041424, -0.078360, 0.036999, -0.021357, 0.011032, -0.026564, -0.016214, -0.023440, -0.044723, -0.064498,
-        0.018283, -0.007165, -0.051802, -0.026299, 0.005867, -0.034691, -0.020621, -0.030512, 0.024458, -0.011330, 0.066558, -0.004069, -0.031624, -0.030639, -0.037451, 0.013079,
-        0.015152, 0.008058, 0.009223, -0.069514, 0.030702, -0.009681, 0.014826, -0.115441, -0.005514, -0.011925, 0.001046, -0.007148, -0.164128, 0.018043, 0.017001, -0.026352,
-        -0.049691, -0.011637, -0.013045, -0.014851, -0.079469, 0.017692, -0.006575, 0.001063, -0.028299, 0.038777, 0.019930, 0.010641, 0.036955, -0.039004, -0.006477, 0.004278,
-        -0.001006, -0.002514, -0.017242, -0.023927, 0.049113, 0.038393, 0.011633, -0.031537, 0.041725, -0.012146, 0.023445, 0.049999, -0.008538, 0.001319, 0.012732, -0.021170,
-        0.082096, 0.009610, -0.025717, 0.002566, -0.060849, 0.017403, 0.032650, 0.018658, -0.030629, -0.025032, 0.005555, 0.000522, -0.009667, -0.043099, 0.005939, -0.027156,
-        0.045634, -0.011986, 0.002713, -0.032225, -0.015494, 0.028734, -0.036528, -0.033101, -0.027174, 0.009490, -0.016537, 0.029435, 0.065709, -0.037711, -0.020497, -0.005578,
-        0.011768, 0.061035, -0.044676, 0.016113, -0.042945, -0.022579, 0.002430, 0.012474, -0.018198, 0.030468, -0.016646, 0.019020, -0.035804, 0.001175, -0.018312, -0.010760,
-    ],
-    dtype=torch.float32,
-)
-# fmt: on
-SYSTEM_PROMPT_CASES = [
-    pytest.param("none", None, SEED_1234, id="none"),
-    pytest.param("dynamic", "dynamic", SYSTEM_PROMPT_DYNAMIC, id="dynamic"),
-    pytest.param("en_vanilla", "en_vanilla", SYSTEM_EN_VANILLA, id="en_vanilla"),
-    pytest.param("en_recaption", "en_recaption", SYSTEM_EN_RECAPTION, id="en_recaption"),
-    pytest.param("en_think_recaption", "en_think_recaption", SYSTEM_EN_THINK_RECAPTION, id="en_think_recaption"),
-    pytest.param("en_unified", "en_unified", SYSTEM_EN_UNIFIED, id="en_unified"),
-]
-
-
-@pytest.fixture(scope="session")
-def clip_bundle() -> tuple[CLIPModel, CLIPProcessor]:
-    try:
-        model = CLIPModel.from_pretrained(LOCAL_CLIP_PATH, local_files_only=True)
-        processor = CLIPProcessor.from_pretrained(LOCAL_CLIP_PATH, local_files_only=True)
-    except OSError as exc:
-        pytest.skip(f"Could not load CLIP model from local cache ({LOCAL_CLIP_PATH}): {exc}")
-
-    model.eval()
-    return model, processor
-
-
-@pytest.fixture(scope="module")
-def omni() -> Generator[Omni, None, None]:
-    engine = Omni(
-        model=MODEL_NAME,
-        stage_configs_path=str(STAGE_CONFIG_PATH),
-        stage_init_timeout=600,
-        init_timeout=900,
-    )
-    try:
-        yield engine
-    finally:
-        engine.close()
-
-
-def _extract_generated_image(outputs: list[object]) -> Image.Image:
-    if not outputs:
-        raise AssertionError("No outputs were returned from Omni.generate()")
-
-    first_output = outputs[0]
-    if images := getattr(first_output, "images", None):
-        return images[0]
-
-    request_output = getattr(first_output, "request_output", None)
-    if request_output is not None and (images := getattr(request_output, "images", None)):
-        return images[0]
-
-    raise AssertionError("No generated image found in Omni output")
-
-
-def extract_embedding(image: Image.Image, clip_model: CLIPModel, clip_processor: CLIPProcessor) -> torch.Tensor:
-    inputs = clip_processor(images=image.convert("RGB"), return_tensors="pt")
-    with torch.inference_mode():
-        features = clip_model.get_image_features(**inputs)
-        features = F.normalize(features, p=2, dim=-1)
-    return features.squeeze(0)
-
-
-def compare_semantic(
-    expected_embedding: torch.Tensor,
-    image: Image.Image,
-    clip_model: CLIPModel,
-    clip_processor: CLIPProcessor,
-) -> float:
-    features = extract_embedding(image, clip_model, clip_processor)
-    expected = F.normalize(expected_embedding, p=2, dim=-1)
-    return torch.dot(expected, features).item()
-
-
-def _generate_image(omni: Omni, use_system_prompt: str | None) -> Image.Image:
-    generator_device = current_omni_platform.device_type or "cuda"
-    sampling_params = OmniDiffusionSamplingParams(
-        seed=1234,
-        generator=torch.Generator(device=generator_device).manual_seed(1234),
-        num_outputs_per_prompt=1,
-    )
-    if use_system_prompt is not None:
-        sampling_params.extra_args = {"use_system_prompt": use_system_prompt}
-
-    outputs = omni.generate({"prompt": PROMPT}, sampling_params)
-    return _extract_generated_image(outputs)
-
-
-@pytest.mark.skipif(torch.cuda.device_count() < 8, reason="Need at least 8 CUDA GPUs for this test.")
-@pytest.mark.parametrize("system_prompt_name,use_system_prompt,expected_embedding", SYSTEM_PROMPT_CASES)
-def test_system_prompt_scores(
-    omni: Omni,
-    clip_bundle: tuple[CLIPModel, CLIPProcessor],
-    system_prompt_name: str,
-    use_system_prompt: str | None,
-    expected_embedding: torch.Tensor,
-) -> None:
-    clip_model, clip_processor = clip_bundle
-    generated_image = _generate_image(omni, use_system_prompt)
-    score = compare_semantic(expected_embedding, generated_image, clip_model, clip_processor)
-
-    print(f"{system_prompt_name}: CLIP cosine similarity = {score:.6f}")

tests/e2e/offline_inference/test_magi_human.py

@@ -1,129 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-"""End-to-end tests for MagiHuman pipeline via vLLM-Omni."""
-
-import io
-
-import av
-import numpy as np
-import pytest
-
-from tests.utils import hardware_test
-from vllm_omni.diffusion.utils.media_utils import mux_video_audio_bytes
-from vllm_omni.entrypoints.omni import Omni
-from vllm_omni.inputs.data import OmniDiffusionSamplingParams
-
-
-def _validate_mp4(video_bytes: bytes, min_frames: int = 10) -> None:
-    """Validate that the MP4 contains meaningful video and audio tracks."""
-    container = av.open(io.BytesIO(video_bytes))
-
-    v_streams = [s for s in container.streams if s.type == "video"]
-    assert len(v_streams) >= 1, "No video stream found in MP4"
-
-    a_streams = [s for s in container.streams if s.type == "audio"]
-    assert len(a_streams) >= 1, "No audio stream found in MP4"
-
-    v_stream = v_streams[0]
-    assert v_stream.width >= 1080, f"Unexpected video width: {v_stream.width}"
-    assert v_stream.height >= 1056, f"Unexpected video height: {v_stream.height}"
-
-    frame_count = 0
-    for frame in container.decode(video=0):
-        frame_count += 1
-        if frame_count >= min_frames:
-            break
-    assert frame_count >= min_frames, f"Video has only {frame_count} frames (expected >= {min_frames})"
-
-    container.close()
-
-
-@pytest.mark.core_model
-@pytest.mark.advanced_model
-@pytest.mark.diffusion
-@hardware_test(res={"cuda": "H100"}, num_cards=2)
-def test_magi_human_e2e(run_level):
-    """End-to-end test for MagiHuman generating video and audio."""
-    if run_level != "advanced_model":
-        pytest.skip("MagiHuman e2e test requires advanced_model run level with real weights.")
-
-    model_path = "SII-GAIR/daVinci-MagiHuman-Base-1080p"
-
-    omni = Omni(
-        model=model_path,
-        init_timeout=1200,
-        tensor_parallel_size=2,
-    )
-
-    prompt = (
-        "A young woman with long, wavy golden blonde hair and bright blue eyes, "
-        "wearing a fitted ivory silk blouse with a delicate lace collar, sits "
-        "stationary in front of a softly lit, blurred warm-toned interior. Her "
-        "overall disposition is warm, composed, and gently confident. The camera "
-        "holds a static medium close-up, framing her from the shoulders up, "
-        "with shallow depth of field keeping her face in sharp focus. Soft "
-        "directional key light falls from the upper left, casting a gentle "
-        "highlight along her cheekbone and nose bridge. She draws a quiet breath, "
-        "the levator labii superiors relaxing as her lips part. She speaks in "
-        "clear, warm, unhurried American English: "
-        "\"The most beautiful things in life aren't things at all — "
-        "they're moments, feelings, and the people who make you feel truly alive.\" "
-        "Her jaw descends smoothly on each stressed syllable; the orbicularis oris "
-        "shapes each vowel with precision. A faint, genuine smile engages the "
-        "zygomaticus major, lifting her lip corners fractionally. Her brows rest "
-        "in a soft, neutral arch throughout. She maintains steady, forward-facing "
-        "eye contact. Head position remains level; no torso displacement occurs.\n\n"
-        "Dialogue:\n"
-        "<Young blonde woman, American English>: "
-        "\"The most beautiful things in life aren't things at all — "
-        "they're moments, feelings, and the people who make you feel truly alive.\"\n\n"
-        "Background Sound:\n"
-        "<Soft, warm indoor ambience with a faint distant piano melody>"
-    )
-
-    sampling_params = OmniDiffusionSamplingParams(
-        height=256,
-        width=448,
-        num_inference_steps=8,
-        seed=52,
-        extra_args={
-            "seconds": 5,
-            "sr_height": 1080,
-            "sr_width": 1920,
-            "sr_num_inference_steps": 5,
-        },
-    )
-
-    try:
-        outputs = list(
-            omni.generate(
-                prompts=[prompt],
-                sampling_params_list=[sampling_params],
-            )
-        )
-
-        assert len(outputs) > 0, "No outputs returned"
-        first = outputs[0]
-
-        assert hasattr(first, "images") and first.images, "No video frames in output"
-        video_frames = first.images[0]
-        assert isinstance(video_frames, np.ndarray), f"Expected numpy array, got {type(video_frames)}"
-        assert video_frames.ndim == 4, f"Expected 4D array (T,H,W,3), got shape {video_frames.shape}"
-
-        audio_waveform = None
-        if hasattr(first, "multimodal_output") and first.multimodal_output:
-            audio_waveform = first.multimodal_output.get("audio")
-        assert audio_waveform is not None, "No audio waveform in multimodal_output"
-
-        video_bytes = mux_video_audio_bytes(
-            video_frames,
-            audio_waveform,
-            fps=25.0,
-            audio_sample_rate=44100,
-        )
-        assert isinstance(video_bytes, bytes), f"Expected MP4 bytes, got {type(video_bytes)}"
-        assert len(video_bytes) > 1000, f"MP4 too small ({len(video_bytes)} bytes)"
-
-        _validate_mp4(video_bytes)
-    finally:
-        omni.close()

tests/e2e/offline_inference/test_omnivoice.py

@@ -1,84 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-"""
-E2E offline tests for OmniVoice TTS model with text input and audio output.
-
-Uses GPUGenerationWorker for both stages (iterative unmasking + DAC decoder).
-"""
-
-import os
-
-os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
-os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
-
-from pathlib import Path
-
-import numpy as np
-import pytest
-
-from tests.utils import hardware_test
-
-MODEL = "k2-fsa/OmniVoice"
-
-
-def get_stage_config():
-    return str(
-        Path(__file__).parent.parent.parent.parent / "vllm_omni" / "model_executor" / "stage_configs" / "omnivoice.yaml"
-    )
-
-
-@pytest.mark.advanced_model
-@pytest.mark.omni
-@hardware_test(res={"cuda": "L4"}, num_cards=1)
-def test_omnivoice_text_to_audio() -> None:
-    """
-    Test OmniVoice text-to-audio generation via offline Omni runner.
-    Deploy Setting: omnivoice.yaml (enforce_eager=true)
-    Input Modal: text
-    Output Modal: audio
-    """
-    from vllm_omni.entrypoints.omni import Omni
-
-    omni = Omni(
-        model=MODEL,
-        stage_configs_path=get_stage_config(),
-        trust_remote_code=True,
-        log_stats=True,
-    )
-
-    try:
-        prompts = {"prompt": "Hello, this is a test for text to audio."}
-
-        from vllm_omni.inputs.data import OmniDiffusionSamplingParams
-
-        sampling_params_list = [OmniDiffusionSamplingParams()]
-
-        outputs = list(omni.generate(prompts, sampling_params_list=sampling_params_list))
-
-        assert len(outputs) > 0, "No outputs generated"
-
-        # Check final output has audio
-        final_output = outputs[-1]
-        ro = final_output.request_output
-        assert ro is not None, "No request_output"
-
-        mm = getattr(ro, "multimodal_output", None)
-        if not mm and ro.outputs:
-            mm = getattr(ro.outputs[0], "multimodal_output", None)
-
-        assert mm is not None, "No multimodal_output"
-        assert "audio" in mm, f"No 'audio' key in multimodal_output: {mm.keys()}"
-
-        audio = mm["audio"]
-        if isinstance(audio, np.ndarray):
-            audio_np = audio
-        else:
-            audio_np = audio.cpu().numpy().squeeze()
-
-        assert audio_np.size > 0, "Audio output is empty"
-        rms = np.sqrt(np.mean(audio_np**2))
-        assert rms > 0.01, f"Audio RMS too low ({rms:.4f}), likely silence"
-
-        print(f"Generated audio: {len(audio_np) / 24000:.2f}s, rms={rms:.4f}")
-    finally:
-        omni.close()

tests/e2e/offline_inference/test_qwen2_5_omni.py

@@ -57,7 +57,7 @@ def get_question(prompt_type="mix"):
     return prompts.get(prompt_type, prompts["mix"])
 
 
-@pytest.mark.advanced_model
+@pytest.mark.core_model
 @pytest.mark.omni
 @hardware_test(res={"cuda": "L4", "rocm": "MI325", "xpu": "B60"}, num_cards={"cuda": 4, "rocm": 2, "xpu": 3})
 @pytest.mark.parametrize("omni_runner", test_params, indirect=True)
@@ -88,7 +88,7 @@ def test_mix_to_audio(omni_runner, omni_runner_handler) -> None:
     omni_runner_handler.send_request(request_config)
 
 
-@pytest.mark.advanced_model
+@pytest.mark.core_model
 @pytest.mark.omni
 @hardware_test(res={"cuda": "L4", "rocm": "MI325", "xpu": "B60"}, num_cards={"cuda": 4, "rocm": 2, "xpu": 3})
 @pytest.mark.parametrize("omni_runner", test_params, indirect=True)

tests/e2e/offline_inference/test_qwen3_omni.py

@@ -56,7 +56,7 @@ def get_question(prompt_type="video"):
     return prompts.get(prompt_type, prompts["video"])
 
 
-@pytest.mark.advanced_model
+@pytest.mark.core_model
 @pytest.mark.omni
 @hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
 @pytest.mark.parametrize("omni_runner", test_params, indirect=True)

tests/e2e/offline_inference/test_qwen_image_diffusion_batching.py

@@ -509,7 +509,7 @@ def test_diffusion_batching_async_explicit_batch(model_name: str):
     all prompts in a single engine call and returns a single combined result.
 
     The list-prompt path routes through the orchestrator's
-    ``add_batch_request_async`` → ``AsyncOmni.generate_batch``
+    ``add_batch_request_async`` → ``AsyncOmniDiffusion.generate_batch``
     and yields ONE ``OmniRequestOutput`` with ALL images combined.
     """
 

tests/e2e/online_serving/test_cosyvoice3_tts.py

@@ -1,134 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-"""
-E2E Online tests for CosyVoice3 TTS model with voice cloning.
-
-These tests verify the /v1/audio/speech endpoint works correctly with
-the CosyVoice3 model, which requires reference audio for voice cloning.
-"""
-
-import os
-
-os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
-os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
-
-from pathlib import Path
-
-import pytest
-
-from tests.conftest import OmniServerParams
-from tests.utils import hardware_test
-
-MODEL = "FunAudioLLM/Fun-CosyVoice3-0.5B-2512"
-
-# Official CosyVoice zero-shot prompt audio and its transcript
-REF_AUDIO_URL = "https://raw.githubusercontent.com/FunAudioLLM/CosyVoice/main/asset/zero_shot_prompt.wav"
-REF_TEXT = "希望你以后能够做的比我还好呦。"
-
-
-def get_stage_config(name: str = "cosyvoice3.yaml"):
-    """Get the stage config path from vllm_omni model_executor stage_configs."""
-    return str(Path(__file__).parent.parent.parent.parent / "vllm_omni" / "model_executor" / "stage_configs" / name)
-
-
-def get_prompt(prompt_type="zh"):
-    prompts = {
-        "zh": "收到好友从远方寄来的生日礼物，那份意外的惊喜与深深的感动让我热泪盈眶。",
-        "en": "Hello, this is a voice cloning test with English text.",
-    }
-    return prompts.get(prompt_type, prompts["zh"])
-
-
-tts_server_params = [
-    pytest.param(
-        OmniServerParams(
-            model=MODEL,
-            stage_config_path=get_stage_config(),
-            server_args=["--trust-remote-code", "--disable-log-stats"],
-        ),
-        id="cosyvoice3",
-    )
-]
-
-tts_async_chunk_server_params = [
-    pytest.param(
-        OmniServerParams(
-            model=MODEL,
-            stage_config_path=get_stage_config("cosyvoice3_async_chunk.yaml"),
-            server_args=["--trust-remote-code", "--disable-log-stats"],
-        ),
-        id="cosyvoice3_async_chunk",
-    )
-]
-
-
-@pytest.mark.core_model
-@pytest.mark.omni
-@hardware_test(res={"cuda": "H100"}, num_cards=1)
-@pytest.mark.parametrize("omni_server", tts_server_params, indirect=True)
-def test_voice_clone_zh_001(omni_server, openai_client) -> None:
-    """
-    Test voice cloning TTS with Chinese text via OpenAI API.
-    Deploy Setting: default yaml
-    Input Modal: text + ref_audio + ref_text
-    Output Modal: audio
-    Input Setting: stream=False
-    Datasets: single request
-    """
-    request_config = {
-        "model": omni_server.model,
-        "input": get_prompt("zh"),
-        "stream": False,
-        "response_format": "wav",
-        "ref_audio": REF_AUDIO_URL,
-        "ref_text": REF_TEXT,
-    }
-    openai_client.send_audio_speech_request(request_config)
-
-
-@pytest.mark.core_model
-@pytest.mark.omni
-@hardware_test(res={"cuda": "H100"}, num_cards=1)
-@pytest.mark.parametrize("omni_server", tts_async_chunk_server_params, indirect=True)
-def test_voice_clone_zh_002(omni_server, openai_client) -> None:
-    """
-    Test voice cloning TTS with Chinese text via async_chunk streaming.
-    Deploy Setting: cosyvoice3_async_chunk.yaml
-    Input Modal: text + ref_audio + ref_text
-    Output Modal: audio (streamed)
-    Input Setting: stream=True
-    Datasets: single request
-    """
-    request_config = {
-        "model": omni_server.model,
-        "input": get_prompt("zh"),
-        "stream": True,
-        "response_format": "wav",
-        "ref_audio": REF_AUDIO_URL,
-        "ref_text": REF_TEXT,
-    }
-    openai_client.send_audio_speech_request(request_config)
-
-
-@pytest.mark.core_model
-@pytest.mark.omni
-@hardware_test(res={"cuda": "H100"}, num_cards=1)
-@pytest.mark.parametrize("omni_server", tts_server_params, indirect=True)
-def test_voice_clone_en_001(omni_server, openai_client) -> None:
-    """
-    Test voice cloning TTS with English text via OpenAI API.
-    Deploy Setting: default yaml
-    Input Modal: text + ref_audio + ref_text
-    Output Modal: audio
-    Input Setting: stream=False
-    Datasets: single request
-    """
-    request_config = {
-        "model": omni_server.model,
-        "input": get_prompt("en"),
-        "stream": False,
-        "response_format": "wav",
-        "ref_audio": REF_AUDIO_URL,
-        "ref_text": REF_TEXT,
-    }
-    openai_client.send_audio_speech_request(request_config)

tests/e2e/online_serving/test_flux2_expansion.py

@@ -1,12 +1,6 @@
 """
 Tests for Flux2 Klein; currently Dev is implemented separately,
 but ideally these models will fold together in the future.
-
-Coverage:
-- FP8 + CacheDiT + Ulysses=2 + TP=2
-- Layerwise CPU offload + Ulysses=2 + Ring=2
-- Layerwise CPU offload + TP=2
-- Layerwise CPU offload + HSDP
 """
 
 import pytest
@@ -48,45 +42,6 @@ def _get_diffusion_feature_cases(model: str):
             ),
             marks=FOUR_CARD_FEATURE_MARKS,
         ),
-        pytest.param(
-            OmniServerParams(
-                model=model,
-                server_args=[
-                    "--enable-layerwise-offload",
-                    "--ulysses-degree",
-                    "2",
-                    "--ring",
-                    "2",
-                ],
-            ),
-            id="layerwise_ulysses2_ring2",
-            marks=FOUR_CARD_FEATURE_MARKS,
-        ),
-        pytest.param(
-            OmniServerParams(
-                model=model,
-                server_args=[
-                    "--enable-layerwise-offload",
-                    "--tensor-parallel-size",
-                    "2",
-                ],
-            ),
-            id="layerwise_tp2",
-            marks=FOUR_CARD_FEATURE_MARKS,
-        ),
-        pytest.param(
-            OmniServerParams(
-                model=model,
-                server_args=[
-                    "--enable-layerwise-offload",
-                    "--use-hsdp",
-                    "--hsdp-shard-size",
-                    "2",
-                ],
-            ),
-            id="layerwise_hsdp",
-            marks=FOUR_CARD_FEATURE_MARKS,
-        ),
     ]
 
 

tests/e2e/online_serving/test_mimo_audio.py

@@ -63,13 +63,11 @@ def download_tokenizer():
 
 
 # CI stage config for H100 / MI325
-# Guard module-level setup so test collection doesn't fail in environments
-# where the model cache is read-only or models aren't available.
-try:
 stage_configs = [get_chunk_config()]
 tokenizer_path = download_tokenizer()
 os.environ["MIMO_AUDIO_TOKENIZER_PATH"] = tokenizer_path
 
+# Create parameter combinations for model and stage config
 test_params = [
     OmniServerParams(
         model=model,
@@ -79,11 +77,6 @@ try:
     for model in models
     for stage_config in stage_configs
 ]
-except Exception as exc:
-    pytest.skip(
-        f"MiMo-Audio online serving tests skipped: module setup failed ({type(exc).__name__}: {exc})",
-        allow_module_level=True,
-    )
 
 
 def get_prompt(prompt_type="text_only"):
@@ -102,7 +95,7 @@ def get_max_batch_size(size_type="few"):
 @pytest.mark.advanced_model
 @pytest.mark.core_model
 @pytest.mark.omni
-@hardware_test(res={"cuda": "L4", "rocm": "MI325"}, num_cards=1)
+@hardware_test(res={"cuda": "L4", "rocm": "MI325"}, num_cards=2)
 @pytest.mark.parametrize("omni_server", test_params, indirect=True)
 def test_audio_to_text_audio_001(omni_server, openai_client) -> None:
     """
@@ -135,7 +128,7 @@ def test_audio_to_text_audio_001(omni_server, openai_client) -> None:
 
 @pytest.mark.advanced_model
 @pytest.mark.omni
-@hardware_test(res={"cuda": "L4", "rocm": "MI325"}, num_cards=1)
+@hardware_test(res={"cuda": "L4", "rocm": "MI325"}, num_cards=2)
 @pytest.mark.parametrize("omni_server", test_params, indirect=True)
 def test_text_to_text_001(omni_server, openai_client) -> None:
     """

tests/e2e/online_serving/test_omnivoice.py

@@ -1,84 +0,0 @@
-# SPDX-License-Identifier: Apache-2.0
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
-"""
-E2E Online tests for OmniVoice TTS model via /v1/audio/speech endpoint.
-
-Tests verify that the OmniVoice model generates valid audio when
-accessed through the standard OpenAI-compatible speech API.
-"""
-
-import os
-
-os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
-os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
-
-from pathlib import Path
-
-import httpx
-import pytest
-
-from tests.conftest import OmniServerParams
-from tests.utils import hardware_test
-
-MODEL = "k2-fsa/OmniVoice"
-
-STAGE_CONFIG = str(
-    Path(__file__).parent.parent.parent.parent / "vllm_omni" / "model_executor" / "stage_configs" / "omnivoice.yaml"
-)
-EXTRA_ARGS = [
-    "--trust-remote-code",
-    "--disable-log-stats",
-]
-TEST_PARAMS = [
-    OmniServerParams(
-        model=MODEL,
-        stage_config_path=STAGE_CONFIG,
-        server_args=EXTRA_ARGS,
-    )
-]
-
-MIN_AUDIO_BYTES = 5000
-
-
-def make_speech_request(
-    host: str,
-    port: int,
-    text: str,
-    timeout: float = 180.0,
-) -> httpx.Response:
-    """Make a request to the /v1/audio/speech endpoint for OmniVoice."""
-    url = f"http://{host}:{port}/v1/audio/speech"
-    payload = {"input": text}
-
-    with httpx.Client(timeout=timeout) as client:
-        return client.post(url, json=payload)
-
-
-def verify_wav_audio(content: bytes) -> bool:
-    """Verify that content is valid WAV audio data."""
-    if len(content) < 44:
-        return False
-    return content[:4] == b"RIFF" and content[8:12] == b"WAVE"
-
-
-@pytest.mark.parametrize("omni_server", TEST_PARAMS, indirect=True)
-class TestOmniVoiceTTS:
-    """E2E tests for OmniVoice TTS model."""
-
-    @pytest.mark.core_model
-    @pytest.mark.omni
-    @hardware_test(res={"cuda": "L4"}, num_cards=1)
-    def test_speech_auto_voice(self, omni_server) -> None:
-        """Test auto voice TTS generation (text only, no reference audio)."""
-        response = make_speech_request(
-            host=omni_server.host,
-            port=omni_server.port,
-            text="Hello, this is a test of the OmniVoice text to speech system.",
-        )
-
-        assert response.status_code == 200, f"Request failed: {response.text}"
-        assert response.headers.get("content-type") == "audio/wav"
-        assert verify_wav_audio(response.content), "Response is not valid WAV audio"
-        assert len(response.content) > MIN_AUDIO_BYTES, (
-            f"Audio too small ({len(response.content)} bytes), expected > {MIN_AUDIO_BYTES}"
-        )

tests/e2e/online_serving/test_qwen3_omni_expansion.py

@@ -26,7 +26,7 @@ from tests.utils import hardware_test
 model = "Qwen/Qwen3-Omni-30B-A3B-Instruct"
 
 AUDIO_KEY = ["test"]
-IMAGE_KEY = ["square", "quadrate", "rectangle"]
+IMAGE_KEY = ["square", "quadrate"]
 VIDEO_KEY = ["sphere", "globe", "circle", "round", "ball"]
 
 
@@ -103,7 +103,6 @@ def get_prompt(prompt_type="text_only"):
         "text_audio": "What is in this audio? ",
         "text_audio_video": "First, what is in this audio? Then, what is in this video? ",
         "one_word": "What is the capital of UK? Answer in one word",
-        "text_chinese": "北京，中国的首都，是一座融合了长城等历史地点与现代建筑的国际化大都市，充满了独特的文化与活力。请重复这句话。",
     }
     return prompts.get(prompt_type, prompts["text_only"])
 
@@ -465,10 +464,20 @@ def test_audio_in_video_002(omni_server, openai_client) -> None:
         "messages": messages,
         "stream": True,
         "use_audio_in_video": True,
-        "key_words": {"video": VIDEO_KEY},
+        "key_words": {"video": VIDEO_KEY, "audio": AUDIO_KEY + ["beep", "electronic"]},
     }
 
+    # Retry when assert_omni_response fails on key_words (see tests/conftest.py).
+    _keyword_assert_msg = "The output does not contain any of the keywords."
+    _max_retries = 3
+    for attempt in range(_max_retries):
+        try:
             openai_client.send_omni_request(request_config, request_num=get_max_batch_size())
+            break
+        except AssertionError as e:
+            if _keyword_assert_msg not in str(e) or attempt == _max_retries - 1:
+                raise
+            print(f"Keyword assertion failed, retrying {attempt + 2}/{_max_retries}: {e!r}")
 
 
 @pytest.mark.advanced_model
@@ -505,120 +514,3 @@ def test_one_word_prompt_001(omni_server, openai_client) -> None:
             if _similarity_assert_msg not in str(e) or attempt == _max_retries - 1:
                 raise
             print(f"Similarity assertion failed, retrying {attempt + 2}/{_max_retries}: {e!r}")
-
-
-@pytest.mark.advanced_model
-@pytest.mark.omni
-@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
-@pytest.mark.parametrize("omni_server", test_params, indirect=True)
-def test_speaker_001(omni_server, openai_client) -> None:
-    """
-    Input Modal: text only (one-word answer constraint).
-    Output Modal: text, audio (default ``modalities``); ``key_words`` only assert on text.
-    Input Setting: stream=True
-    Datasets: single request
-    """
-    messages = dummy_messages_from_mix_data(
-        system_prompt=get_system_prompt(),
-        content_text=get_prompt("text"),
-    )
-
-    request_config = {
-        "model": omni_server.model,
-        "messages": messages,
-        "stream": True,
-        "speaker": "Chelsie",
-        "key_words": {"text": ["beijing"]},
-    }
-
-    openai_client.send_omni_request(request_config)
-
-
-@pytest.mark.advanced_model
-@pytest.mark.omni
-@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
-@pytest.mark.parametrize("omni_server", test_params, indirect=True)
-def test_speaker_002(omni_server, openai_client) -> None:
-    """
-    Input Modal: text only (one-word answer constraint).
-    Output Modal: text, audio (default ``modalities``); ``key_words`` only assert on text.
-    Input Setting: stream=True
-    Datasets: single request
-    """
-    messages = dummy_messages_from_mix_data(
-        system_prompt=get_system_prompt(),
-        content_text=get_prompt("text"),
-    )
-
-    request_config = {
-        "model": omni_server.model,
-        "messages": messages,
-        "stream": True,
-        "speaker": "Ethan",
-        "key_words": {"text": ["beijing"]},
-    }
-
-    # Retry only when assert_omni_response fails on preset voice gender (see tests/conftest.py).
-    _gender_assert_substr = "estimated gender"
-    _max_retries = 3
-    for attempt in range(_max_retries):
-        try:
-            openai_client.send_omni_request(request_config, request_num=get_max_batch_size())
-            break
-        except AssertionError as e:
-            if _gender_assert_substr not in str(e) or attempt == _max_retries - 1:
-                raise
-            print(f"Gender assertion failed, retrying {attempt + 2}/{_max_retries}: {e!r}")
-
-
-@pytest.mark.advanced_model
-@pytest.mark.omni
-@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
-@pytest.mark.parametrize("omni_server", test_params, indirect=True)
-def test_speaker_003(omni_server, openai_client) -> None:
-    """
-    Input Modal: text only (one-word answer constraint).
-    Output Modal: text, audio (default ``modalities``); ``key_words`` only assert on text.
-    Input Setting: stream=True
-    Datasets: single request
-    """
-    messages = dummy_messages_from_mix_data(
-        system_prompt=get_system_prompt(),
-        content_text=get_prompt("text"),
-    )
-
-    request_config = {
-        "model": omni_server.model,
-        "messages": messages,
-        "stream": True,
-        "speaker": "CHELSIE",
-        "key_words": {"text": ["beijing"]},
-    }
-
-    openai_client.send_omni_request(request_config)
-
-
-@pytest.mark.advanced_model
-@pytest.mark.omni
-@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
-@pytest.mark.parametrize("omni_server", test_params, indirect=True)
-def test_language_001(omni_server, openai_client) -> None:
-    """
-    Input Modal: text only (one-word answer constraint).
-    Output Modal: text, audio (default ``modalities``); ``key_words`` only assert on text.
-    Input Setting: stream=True
-    Datasets: single request
-    """
-    messages = dummy_messages_from_mix_data(
-        system_prompt=get_system_prompt(),
-        content_text=get_prompt("text_chinese"),
-    )
-
-    request_config = {
-        "model": omni_server.model,
-        "messages": messages,
-        "stream": True,
-        "key_words": {"text": ["北京"]},
-    }
-
-    openai_client.send_omni_request(request_config)

tests/e2e/online_serving/test_qwen3_tts_customvoice_expansion.py

@@ -120,31 +120,6 @@ def test_voice_002(omni_server, openai_client) -> None:
     openai_client.send_audio_speech_request(request_config)
 
 
-@pytest.mark.advanced_model
-@pytest.mark.omni
-@hardware_test(res={"cuda": "L4"}, num_cards=1)
-@pytest.mark.parametrize("omni_server", tts_server_params, indirect=True)
-def test_voice_003(omni_server, openai_client) -> None:
-    """
-    Test text input processing and audio output via OpenAI API.
-    Deploy Setting: default yaml
-    Input Modal: text
-    Output Modal: audio
-    Input Setting: stream=False, language=chinese
-    Datasets: few requests
-    """
-    request_config = {
-        "model": omni_server.model,
-        "input": get_prompt(),
-        "stream": False,
-        "response_format": "wav",
-        "task_type": "CustomVoice",
-        "voice": "SERENA",
-    }
-
-    openai_client.send_audio_speech_request(request_config)
-
-
 @pytest.mark.advanced_model
 @pytest.mark.omni
 @hardware_test(res={"cuda": "L4"}, num_cards=1)