Compare commits

...
Author SHA1 Message Date
Haco 01ebc0cdf5 [Config Refactor] Derive deploy override fields from stage config (#3162)
Signed-off-by: xiaohajiayou <923390377@qq.com>
2026-05-01 01:04:03 +08:00
汪志鹏 77e1e5c01c [BugFix][Bagel]: Fix vLLM-Omni as rollout bug: number of trajectory_latents count less (#3258)
Signed-off-by: princepride <wangzhipeng628@gmail.com>
2026-05-01 00:15:16 +08:00
TJian e637726f94 [AMD][CI][Bugfix] Fix "simple unit test" (#3225)
Signed-off-by: tjtanaa <tunjian.tan@embeddedllm.com>
2026-04-30 10:38:06 -04:00
Alex BrooksandSYLAR bede8e02d8 [Misc] Remove Entrypoint Hijack for vLLM / 0.20.0 Changes (#3082)
Signed-off-by: Alex Brooks <albrooks@redhat.com>
Co-authored-by: SYLAR <125541396+lishunyang12@users.noreply.github.com>
2026-04-30 12:00:21 +00:00
rongfu.leng da60663f78 add check_health in inline client (#3052)
Signed-off-by: rongfu.leng <lenronfu@gmail.com>
2026-04-30 11:45:09 +00:00
Yuanheng Zhao 0309b3bd19 [Bugfix] Prevent silent failure of get_config when trust_remote_code passed as None (#3241)
Signed-off-by: Yuanheng Zhao <jonathan.zhaoyh@gmail.com>
2026-04-30 10:25:25 +00:00
wangyu aa0587c407 [CI] remove skip in test_z_image_expansion and test_qwen3_omni_expansion (#3211)
Signed-off-by: wangyu <410167048@qq.com>
2026-04-30 17:54:56 +08:00
Haco 16310c8ef3 [BugFix] Fix nullify regressions in pure diffusion offline examples (#3181)
Signed-off-by: xiaohajiayou <923390377@qq.com>
2026-04-30 17:03:09 +08:00
Canlin Guo 15a1143695 [Mergify] Init config (#812)
Signed-off-by: gcanlin <canlinguosdu@gmail.com>
2026-04-30 08:48:53 +00:00
Hongsheng Liu 7fc528f7a9 Revert "[Quantization] Enable FP8 online quantization for Z-image text encoder" (#3272) 2026-04-30 15:08:17 +08:00
50 changed files with 533 additions and 340 deletions

No files matched your search

+1 -1
View File
@@ -10,7 +10,7 @@ steps:
DOCKER_BUILDKIT: "1" DOCKER_BUILDKIT: "1"
# Buildkite will automatically replace this with the actual commit hash # Buildkite will automatically replace this with the actual commit hash
VLLM_IMAGE_TAG: "${BUILDKITE_COMMIT}" VLLM_IMAGE_TAG: "${BUILDKITE_COMMIT}"
VLLM_VERSION: "v0.19.0" VLLM_VERSION: "v0.20.0"
priority: 100 priority: 100
timeout_in_minutes: 60 timeout_in_minutes: 60
soft_fail: true soft_fail: true
+3 -1
View File
@@ -7,7 +7,9 @@ steps:
grade: Blocking grade: Blocking
commands: commands:
- export VLLM_ROCM_USE_AITER=0 - export VLLM_ROCM_USE_AITER=0
- "timeout 20m pytest -v -s -m 'core_model and cpu' --cov=vllm_omni --cov-branch --cov-report=term-missing --cov-report=html --cov-report=xml" # ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM
# that is not supported on CPU
- "timeout 20m pytest -v -s -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-branch --cov-report=term-missing --cov-report=html --cov-report=xml"
- label: "Diffusion Model Test" - label: "Diffusion Model Test"
agent_pool: mi325_2 agent_pool: mi325_2
+3 -1
View File
@@ -7,7 +7,9 @@ steps:
grade: Blocking grade: Blocking
commands: commands:
- export VLLM_ROCM_USE_AITER=0 - export VLLM_ROCM_USE_AITER=0
- "timeout 20m pytest -v -s -m 'core_model and cpu' --cov=vllm_omni --cov-branch --cov-report=term-missing --cov-report=html --cov-report=xml" # ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM
# that is not supported on CPU
- "timeout 20m pytest -vvvv -s -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-branch --cov-report=term-missing --cov-report=html --cov-report=xml"
- label: "Voxtral TTS CUDA Unit Test" - label: "Voxtral TTS CUDA Unit Test"
agent_pool: mi325_1 agent_pool: mi325_1
+223
View File
@@ -0,0 +1,223 @@
pull_request_rules:
- name: label-documentation
description: Automatically apply documentation label
conditions:
- label != stale
- or:
- files~=^[^/]+\.md$
- files~=^docs/
- files~=^examples/
actions:
label:
add:
- documentation
comment:
message: "Documentation preview: https://vllm--{{number}}.org.readthedocs.build/en/{{number}}/"
- name: comment-pre-commit-failure
description: Comment on PR when pre-commit check fails
conditions:
- status-failure=pre-commit
- -closed
- -draft
actions:
comment:
message: |
Hi @{{author}}, the pre-commit checks have failed. Please run:
```bash
uv pip install pre-commit
pre-commit install
pre-commit run --all-files
```
Then, commit the changes and push to your branch.
For future commits, `pre-commit` will run automatically on changed files before each commit.
> [!TIP]
> <details>
> <summary>Is <code>mypy</code> or <code>markdownlint</code> failing?</summary>
> <br/>
> <code>mypy</code> and <code>markdownlint</code> are run differently in CI. If the failure is related to either of these checks, please use the following commands to run them locally:
>
> ```bash
> # For mypy (substitute "3.10" with the failing version if needed)
> pre-commit run --hook-stage manual mypy-3.10
> # For markdownlint
> pre-commit run --hook-stage manual markdownlint
> ```
> </details>
- name: comment-dco-failure
description: Comment on PR when DCO check fails
conditions:
- status-failure=dco
- -closed
- -draft
actions:
comment:
message: |
Hi @{{author}}, the DCO check has failed. Please click on DCO in the Checks section for instructions on how to resolve this.
- name: label-ci-build
description: Automatically apply ci/build label
conditions:
- label != stale
- or:
- files~=^\.github/
- files~=\.buildkite/
- files~=^cmake/
- files=CMakeLists.txt
- files~=^docker/Dockerfile
- files~=^requirements.*\.txt
- files=setup.py
actions:
label:
add:
- ci/build
- name: label-frontend
description: Automatically apply frontend label
conditions:
- label != stale
- files~=^vllm_omni/entrypoints/
actions:
label:
add:
- frontend
- name: label-diffusion
description: Automatically apply diffusion label
conditions:
- label != stale
- files~=^vllm_omni/diffusion/
actions:
label:
add:
- diffusion
- name: label-new-model
description: Automatically apply new-model label
conditions:
- label != stale
- or:
- files=vllm_omni/model_executor/models/registry.py
- files=vllm_omni/diffusion/registry.py
actions:
label:
add:
- new-model
- name: label-npu
description: Automatically apply npu label
conditions:
- label != stale
- or:
- files~=^vllm_omni/platforms/npu/
- title~=(?i)NPU
- title~=(?i)Ascend
actions:
label:
add:
- npu
assign:
users:
- "gcanlin"
- name: label-rocm
description: Automatically apply rocm label
conditions:
- label != stale
- or:
- files~=^vllm_omni/platforms/rocm/
- title~=(?i)AMD
- title~=(?i)ROCm
actions:
label:
add:
- rocm
assign:
users:
- "tjtanaa"
- name: label-xpu
description: Automatically apply xpu label
conditions:
- label != stale
- or:
- files~=^vllm_omni/platforms/xpu/
- title~=(?i)Intel
- title~=(?i)XPU
actions:
label:
add:
- xpu
assign:
users:
- "yma11"
- name: label-modelrunner
description: Automatically apply modelrunner label
conditions:
- label != stale
- or:
- files~=^vllm_omni/worker/
- title~=(?i)ModelRunner
- title~=(?i)worker
actions:
label:
add:
- ModelRunner
assign:
users:
- "gcanlin" # To check whether need to sync to NPU
- name: auto-rebase if approved, ready, and 30 commits behind main
conditions:
- base = main
- label=ready
- "#approved-reviews-by >= 1"
- "#commits-behind >= 30"
- -closed
- -draft
- -conflict
actions:
rebase: {}
- name: ping author on conflicts and add 'needs-rebase' label
conditions:
- label != stale
- conflict
- -closed
actions:
label:
add:
- needs-rebase
comment:
message: |
This pull request has merge conflicts that must be resolved before it can be
merged. Please rebase the PR, @{{author}}.
https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/working-with-forks/syncing-a-fork
- name: remove 'needs-rebase' label when conflict is resolved
conditions:
- -conflict
- -closed
actions:
label:
remove:
- needs-rebase
- name: label-bug
description: Automatically apply bug label
conditions:
- label != stale
- or:
- title~=(?i)\bbug\b
- title~=(?i)\bbugfix\b
actions:
label:
add:
- bug
+1 -1
View File
@@ -1,5 +1,5 @@
ARG VLLM_BASE_IMAGE=vllm/vllm-openai ARG VLLM_BASE_IMAGE=vllm/vllm-openai
ARG VLLM_BASE_TAG=v0.20.0-cu130 ARG VLLM_BASE_TAG=v0.20.0
FROM ${VLLM_BASE_IMAGE}:${VLLM_BASE_TAG} FROM ${VLLM_BASE_IMAGE}:${VLLM_BASE_TAG}
ARG APP_DIR=/workspace/vllm-omni ARG APP_DIR=/workspace/vllm-omni
WORKDIR ${APP_DIR} WORKDIR ${APP_DIR}
+1 -1
View File
@@ -1,4 +1,4 @@
ARG BASE_IMAGE=vllm/vllm-openai:v0.19.0 ARG BASE_IMAGE=vllm/vllm-openai:v0.20.0
FROM ${BASE_IMAGE} FROM ${BASE_IMAGE}
ARG COMMON_WORKDIR=/app ARG COMMON_WORKDIR=/app
+1 -1
View File
@@ -1,4 +1,4 @@
ARG BASE_IMAGE=vllm/vllm-openai-rocm:v0.19.0 ARG BASE_IMAGE=vllm/vllm-openai-rocm:v0.20.0
FROM ${BASE_IMAGE} AS base FROM ${BASE_IMAGE} AS base
# Declare a variable to know if we want to use the nightly build or the stable build. # Declare a variable to know if we want to use the nightly build or the stable build.
+1 -1
View File
@@ -74,7 +74,7 @@ ENV UV_EXTRA_INDEX_URL=${PIP_EXTRA_INDEX_URL}
ENV UV_INDEX_STRATEGY="unsafe-best-match" ENV UV_INDEX_STRATEGY="unsafe-best-match"
ENV UV_LINK_MODE="copy" ENV UV_LINK_MODE="copy"
ARG VLLM_VERSION=v0.19.0 ARG VLLM_VERSION=v0.20.0
RUN git clone -b ${VLLM_VERSION} https://github.com/vllm-project/vllm RUN git clone -b ${VLLM_VERSION} https://github.com/vllm-project/vllm
WORKDIR /workspace/vllm WORKDIR /workspace/vllm
@@ -13,7 +13,7 @@ vLLM-Omni current recommends the steps in under setup through Docker Images.
vLLM-Omni is built based on vLLM. Please install it with command below. vLLM-Omni is built based on vLLM. Please install it with command below.
```bash ```bash
uv pip install vllm==0.20.0+rocm700 --extra-index-url https://wheels.vllm.ai/rocm/0.20.0/rocm700 uv pip install vllm==0.20.0+rocm721 --extra-index-url https://wheels.vllm.ai/rocm/0.20.0/rocm721
``` ```
#### Installation of vLLM-Omni #### Installation of vLLM-Omni
@@ -37,10 +37,10 @@ uv pip install onnxruntime-rocm
If you do not need to modify source code of vLLM, you can directly install the stable 0.20.0 release version of the library If you do not need to modify source code of vLLM, you can directly install the stable 0.20.0 release version of the library
```bash ```bash
uv pip install vllm==0.20.0+rocm700 --extra-index-url https://wheels.vllm.ai/rocm/0.20.0/rocm700 uv pip install vllm==0.20.0+rocm721 --extra-index-url https://wheels.vllm.ai/rocm/0.20.0/rocm721
``` ```
The pre-built 0.20.0 vLLM wheel targets ROCm 7.0. If you need a different ROCm stack or want to reuse an existing PyTorch installation, build vLLM from source instead. The pre-built 0.20.0 vLLM wheel targets ROCm 7.2.1. If you need a different ROCm stack or want to reuse an existing PyTorch installation, build vLLM from source instead.
#### Installation of vLLM-Omni #### Installation of vLLM-Omni
Since vllm-omni is rapidly evolving, it's recommended to install it from source Since vllm-omni is rapidly evolving, it's recommended to install it from source
+7 -1
View File
@@ -22,7 +22,7 @@ source .venv/bin/activate
uv pip install vllm==0.20.0 --torch-backend=auto uv pip install vllm==0.20.0 --torch-backend=auto
# On ROCm # On ROCm
uv pip install vllm==0.20.0+rocm700 --extra-index-url https://wheels.vllm.ai/rocm/0.20.0/rocm700 uv pip install vllm==0.20.0+rocm721 --extra-index-url https://wheels.vllm.ai/rocm/0.20.0/rocm721
git clone https://github.com/vllm-project/vllm-omni.git git clone https://github.com/vllm-project/vllm-omni.git
cd vllm-omni cd vllm-omni
@@ -31,6 +31,12 @@ uv pip install -e .
For additional installation methods — please see the [installation guide](installation/README.md). For additional installation methods — please see the [installation guide](installation/README.md).
!!! note
It is important to install the same major & minor version of vLLM and vLLM Omni, otherwise things may not work as expected. If the versions are misaligned, you will see a warning when you import vLLM Omni.
If you are seeing strange behavior with the `vllm` command not handling the `--omni` flag correctly, you most likely have a version mismatch with vLLM < `0.20.0` and vLLM Omni `0.20.0`, as vLLM Omni no longer hijacks the vLLM entrypoint. Updating vLLM should resolve this issue.
## Offline Inference ## Offline Inference
Text-to-image generation quickstart with vLLM-Omni: Text-to-image generation quickstart with vLLM-Omni:
+9 -9
View File
@@ -32,15 +32,15 @@ guide. FP8 on Ampere may use a weight-only path where available.
### Diffusion Model (Qwen-Image, Wan2.2) ### Diffusion Model (Qwen-Image, Wan2.2)
| Model | HF models | Online | Pre-calibrated | Recommendation | `ignored_layers` | Text-Encoder quantization | | Model | HF models | Online | Pre-calibrated | Recommendation | `ignored_layers` |
|-------|-----------|:-------:|:------:|----------------|------------------|------------------| |-------|-----------|:-------:|:------:|----------------|------------------|
| Qwen-Image | `Qwen/Qwen-Image`, `Qwen/Qwen-Image-2512` | Yes | Yes | Skip sensitive image-stream MLPs when quality regresses | `img_mlp` | | | Qwen-Image | `Qwen/Qwen-Image`, `Qwen/Qwen-Image-2512` | Yes | Yes | Skip sensitive image-stream MLPs when quality regresses | `img_mlp` |
| Wan2.2 | Wan2.2 diffusion pipelines | Not validated | Not validated | Validate against BF16 before documenting as supported | TBD | | | Wan2.2 | Wan2.2 diffusion pipelines | Not validated | Not validated | Validate against BF16 before documenting as supported | TBD |
| Z-Image | `Tongyi-MAI/Z-Image-Turbo` | Yes | Yes | All layers | None | ✅︎ | | Z-Image | `Tongyi-MAI/Z-Image-Turbo` | Yes | Yes | All layers | None |
| FLUX.1 | `black-forest-labs/FLUX.1-dev`, `black-forest-labs/FLUX.1-schnell` | Yes | Yes | All layers | None | | | FLUX.1 | `black-forest-labs/FLUX.1-dev`, `black-forest-labs/FLUX.1-schnell` | Yes | Yes | All layers | None |
| FLUX.2-klein | `black-forest-labs/FLUX.2-klein-4B` | Yes | Yes | All layers | None | | | FLUX.2-klein | `black-forest-labs/FLUX.2-klein-4B` | Yes | Yes | All layers | None |
| HunyuanImage-3.0 | `tencent/HunyuanImage-3.0`, `tencent/HunyuanImage-3.0-Instruct` | Yes | Yes | All layers; use the Hunyuan stage config for multi-stage runs | None | | | HunyuanImage-3.0 | `tencent/HunyuanImage-3.0`, `tencent/HunyuanImage-3.0-Instruct` | Yes | Yes | All layers; use the Hunyuan stage config for multi-stage runs | None |
| HunyuanVideo-1.5 | `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v`, `720p_t2v`, `480p_i2v` | Yes | Yes | All layers | None | | | HunyuanVideo-1.5 | `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v`, `720p_t2v`, `480p_i2v` | Yes | Yes | All layers | None |
### Multi-Stage Omni/TTS Model (Qwen3-Omni, Qwen3-TTS) ### Multi-Stage Omni/TTS Model (Qwen3-Omni, Qwen3-TTS)
@@ -118,9 +118,6 @@ def parse_args() -> argparse.Namespace:
help='JSON profiler config for torch/cuda profiling, e.g. \'{"profiler":"torch","torch_profiler_dir":"./perf"}\'.', help='JSON profiler config for torch/cuda profiling, e.g. \'{"profiler":"torch","torch_profiler_dir":"./perf"}\'.',
) )
from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults
nullify_stage_engine_defaults(parser)
return parser.parse_args() return parser.parse_args()
@@ -196,9 +196,6 @@ def parse_args() -> argparse.Namespace:
parser.add_argument("--cfg-parallel-size", type=int, default=1, choices=[1, 2], help="CFG parallel size.") parser.add_argument("--cfg-parallel-size", type=int, default=1, choices=[1, 2], help="CFG parallel size.")
parser.add_argument("--tensor-parallel-size", type=int, default=1, help="Tensor parallelism size.") parser.add_argument("--tensor-parallel-size", type=int, default=1, help="Tensor parallelism size.")
from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults
nullify_stage_engine_defaults(parser)
return parser.parse_args() return parser.parse_args()
@@ -378,9 +378,6 @@ def parse_args() -> argparse.Namespace:
default=None, default=None,
help='JSON profiler config for torch/cuda profiling, e.g. \'{"profiler":"torch","torch_profiler_dir":"./perf"}\'.', help='JSON profiler config for torch/cuda profiling, e.g. \'{"profiler":"torch","torch_profiler_dir":"./perf"}\'.',
) )
from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults
nullify_stage_engine_defaults(parser)
return parser.parse_args() return parser.parse_args()
@@ -211,9 +211,6 @@ def parse_args() -> argparse.Namespace:
default=None, default=None,
help='JSON profiler config for torch/cuda profiling, e.g. \'{"profiler":"torch","torch_profiler_dir":"./perf"}\'.', help='JSON profiler config for torch/cuda profiling, e.g. \'{"profiler":"torch","torch_profiler_dir":"./perf"}\'.',
) )
from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults
nullify_stage_engine_defaults(parser)
return parser.parse_args() return parser.parse_args()
@@ -24,9 +24,6 @@ def parse_args():
parser.add_argument("--width", type=int, default=448, help="Video width.") parser.add_argument("--width", type=int, default=448, help="Video width.")
parser.add_argument("--num-inference-steps", type=int, default=8, help="Number of denoising steps.") parser.add_argument("--num-inference-steps", type=int, default=8, help="Number of denoising steps.")
parser.add_argument("--seed", type=int, default=52, help="Random seed for generation.") parser.add_argument("--seed", type=int, default=52, help="Random seed for generation.")
from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults
nullify_stage_engine_defaults(parser)
return parser.parse_args() return parser.parse_args()
@@ -9,7 +9,7 @@ from typing import Any
import torch import torch
from vllm_omni.diffusion.data import DiffusionParallelConfig, logger from vllm_omni.diffusion.data import logger
from vllm_omni.entrypoints.omni import Omni from vllm_omni.entrypoints.omni import Omni
from vllm_omni.inputs.data import OmniDiffusionSamplingParams from vllm_omni.inputs.data import OmniDiffusionSamplingParams
from vllm_omni.lora.request import LoRARequest from vllm_omni.lora.request import LoRARequest
@@ -342,17 +342,6 @@ def main():
# (e.g., QwenImagePipeline or FluxPipeline) # (e.g., QwenImagePipeline or FluxPipeline)
} }
# assert args.ring_degree == 1, "Ring attention is not supported yet"
parallel_config = DiffusionParallelConfig(
ulysses_degree=args.ulysses_degree,
ring_degree=args.ring_degree,
ulysses_mode=args.ulysses_mode,
cfg_parallel_size=args.cfg_parallel_size,
tensor_parallel_size=args.tensor_parallel_size,
vae_patch_parallel_size=args.vae_patch_parallel_size,
enable_expert_parallel=args.enable_expert_parallel,
)
profiler_enabled = args.profiler_config is not None profiler_enabled = args.profiler_config is not None
# Prepare LoRA kwargs for Omni initialization # Prepare LoRA kwargs for Omni initialization
@@ -388,7 +377,13 @@ def main():
"cache_backend": args.cache_backend, "cache_backend": args.cache_backend,
"cache_config": cache_config, "cache_config": cache_config,
"enable_cache_dit_summary": args.enable_cache_dit_summary, "enable_cache_dit_summary": args.enable_cache_dit_summary,
"parallel_config": parallel_config, "ulysses_degree": args.ulysses_degree,
"ring_degree": args.ring_degree,
"ulysses_mode": args.ulysses_mode,
"cfg_parallel_size": args.cfg_parallel_size,
"tensor_parallel_size": args.tensor_parallel_size,
"vae_patch_parallel_size": args.vae_patch_parallel_size,
"enable_expert_parallel": args.enable_expert_parallel,
"enforce_eager": args.enforce_eager, "enforce_eager": args.enforce_eager,
"enable_cpu_offload": args.enable_cpu_offload, "enable_cpu_offload": args.enable_cpu_offload,
"mode": "text-to-image", "mode": "text-to-image",
@@ -226,9 +226,6 @@ def parse_args() -> argparse.Namespace:
default=1, default=1,
help="Number of HSDP replica groups.", help="Number of HSDP replica groups.",
) )
from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults
nullify_stage_engine_defaults(parser)
return parser.parse_args() return parser.parse_args()
@@ -71,9 +71,6 @@ def parse_args() -> argparse.Namespace:
parser.add_argument("--ulysses-degree", type=int, default=1, help="Ulysses SP degree.") parser.add_argument("--ulysses-degree", type=int, default=1, help="Ulysses SP degree.")
parser.add_argument("--ring-degree", type=int, default=1, help="Ring attention degree.") parser.add_argument("--ring-degree", type=int, default=1, help="Ring attention degree.")
parser.add_argument("--cfg-parallel-size", type=int, default=1, choices=[1, 2], help="CFG parallel size.") parser.add_argument("--cfg-parallel-size", type=int, default=1, choices=[1, 2], help="CFG parallel size.")
from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults
nullify_stage_engine_defaults(parser)
return parser.parse_args() return parser.parse_args()
-2
View File
@@ -32,7 +32,6 @@ classifiers = [
# Dependencies are now managed dynamically via setup.py based on detected hardware platform. # Dependencies are now managed dynamically via setup.py based on detected hardware platform.
# This allows automatic installation of the correct platform-specific dependencies (CUDA/ROCm/CPU/XPU/NPU) # This allows automatic installation of the correct platform-specific dependencies (CUDA/ROCm/CPU/XPU/NPU)
# without requiring extras like [cuda]. See requirements/ directory for platform-specific dependencies. # without requiring extras like [cuda]. See requirements/ directory for platform-specific dependencies.
# Note: vllm is intentionally excluded due to entrypoints overwrite issue.
[project.optional-dependencies] [project.optional-dependencies]
@@ -90,7 +89,6 @@ Documentation = "https://vllm-omni.readthedocs.io"
"Bug Tracker" = "https://github.com/vllm-project/vllm-omni/issues" "Bug Tracker" = "https://github.com/vllm-project/vllm-omni/issues"
[project.scripts] [project.scripts]
vllm = "vllm_omni.entrypoints.cli.main:main"
vllm-omni = "vllm_omni.entrypoints.cli.main:main" vllm-omni = "vllm_omni.entrypoints.cli.main:main"
@@ -121,7 +121,8 @@ class TestTrajectoryRecording:
assert trajectory_latents is not None assert trajectory_latents is not None
assert trajectory_timesteps is not None assert trajectory_timesteps is not None
assert len(trajectory_latents) == EXPECTED_STEPS # initial latent + one per denoising step
assert len(trajectory_latents) == EXPECTED_STEPS + 1
assert len(trajectory_timesteps) == EXPECTED_STEPS assert len(trajectory_timesteps) == EXPECTED_STEPS
# log_probs is None without a scheduler (default ODE path) # log_probs is None without a scheduler (default ODE path)
assert trajectory_log_probs is None assert trajectory_log_probs is None
@@ -167,6 +168,54 @@ class TestTrajectoryRecording:
"Last trajectory latent should match the final output" "Last trajectory latent should match the final output"
) )
def test_initial_latent_matches_input_noise(self, bagel_and_args):
"""Regression: the first trajectory entry must be the initial noise (pre-denoising)."""
bagel, args = bagel_and_args
init_noise = args["packed_init_noises"].clone()
_, trajectory_latents, *_ = bagel.generate_image(**args, return_trajectory_latents=True)
assert torch.allclose(trajectory_latents[0], init_noise, atol=1e-6), (
"trajectory_latents[0] should be the initial noise before any denoising step"
)
def test_trajectory_timesteps_match_expected_schedule(self, bagel_and_args):
"""Regression: trajectory_timesteps must record raw timesteps[i], not timesteps[i] - dts[i]."""
bagel, args = bagel_and_args
# Recompute the expected timestep schedule (mirrors generate_image logic)
ts = torch.linspace(1, 0, args["num_timesteps"])
shift = args.get("timestep_shift", 1.0)
ts = shift * ts / (1 + (shift - 1) * ts)
expected_timesteps = ts[:-1] # last element is dropped
_, _, trajectory_timesteps, _ = bagel.generate_image(**args, return_trajectory_latents=True)
assert len(trajectory_timesteps) == len(expected_timesteps)
for i, (actual, expected) in enumerate(zip(trajectory_timesteps, expected_timesteps)):
assert abs(float(actual) - float(expected)) < 1e-6, (
f"Step {i}: trajectory_timestep={float(actual):.6f}, expected={float(expected):.6f}"
)
def test_latent_count_equals_timesteps_plus_one(self, bagel_and_args):
"""Regression: len(trajectory_latents) == len(timesteps) + 1 (initial + one per step)."""
bagel, args = bagel_and_args
# Compute the number of denoising steps the same way generate_image does
ts = torch.linspace(1, 0, args["num_timesteps"])
shift = args.get("timestep_shift", 1.0)
ts = shift * ts / (1 + (shift - 1) * ts)
num_steps = len(ts) - 1 # timesteps = ts[:-1]
_, trajectory_latents, trajectory_timesteps, _ = bagel.generate_image(**args, return_trajectory_latents=True)
assert len(trajectory_latents) == num_steps + 1, (
f"Expected {num_steps + 1} latents (initial + {num_steps} steps), got {len(trajectory_latents)}"
)
assert len(trajectory_timesteps) == num_steps, (
f"Expected {num_steps} timesteps, got {len(trajectory_timesteps)}"
)
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
# Mock scheduler for log-prob tests # Mock scheduler for log-prob tests
@@ -356,7 +356,6 @@ def test_mix_to_text_audio_001(omni_server, openai_client) -> None:
@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2) @hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
@pytest.mark.parametrize("omni_server", test_params, indirect=True) @pytest.mark.parametrize("omni_server", test_params, indirect=True)
@pytest.mark.skip(reason="issue: #2827")
def test_audio_in_video_001(omni_server, openai_client) -> None: def test_audio_in_video_001(omni_server, openai_client) -> None:
""" """
Input Modal: text + video (synthetic MP4 with embedded audio; ``use_audio_in_video`` uses audio from the video). Input Modal: text + video (synthetic MP4 with embedded audio; ``use_audio_in_video`` uses audio from the video).
@@ -376,7 +375,7 @@ def test_audio_in_video_001(omni_server, openai_client) -> None:
"messages": messages, "messages": messages,
"stream": False, "stream": False,
"use_audio_in_video": True, "use_audio_in_video": True,
"key_words": {"video": VIDEO_KEY, "audio": AUDIO_KEY + ["beep", "electronic"]}, "key_words": {"video": VIDEO_KEY},
} }
openai_client.send_omni_request(request_config) openai_client.send_omni_request(request_config)
@@ -98,7 +98,7 @@ def _get_diffusion_feature_cases():
], ],
), ),
id="layerwise_hsdp", id="layerwise_hsdp",
marks=[*FOUR_CARD_MARKS, pytest.mark.skip(reason="issue #2435")], marks=[*FOUR_CARD_MARKS],
), ),
] ]
@@ -4,6 +4,7 @@
import pytest import pytest
from vllm.utils.argparse_utils import FlexibleArgumentParser from vllm.utils.argparse_utils import FlexibleArgumentParser
from vllm_omni.config.stage_config import deploy_override_field_names
from vllm_omni.engine.async_omni_engine import AsyncOmniEngine from vllm_omni.engine.async_omni_engine import AsyncOmniEngine
from vllm_omni.entrypoints.cli.serve import OmniServeCommand, _create_default_diffusion_stage_cfg from vllm_omni.entrypoints.cli.serve import OmniServeCommand, _create_default_diffusion_stage_cfg
@@ -30,6 +31,15 @@ def test_default_stage_config_includes_cache_backend():
assert engine_args["model_stage"] == "diffusion" assert engine_args["model_stage"] == "diffusion"
def test_default_stage_config_ignores_none_deploy_overrides():
"""Ensure nullified deploy override defaults do not alter diffusion defaults."""
baseline = AsyncOmniEngine._create_default_diffusion_stage_cfg({})[0]
nullified_overrides = {name: None for name in deploy_override_field_names()}
stage_cfg = AsyncOmniEngine._create_default_diffusion_stage_cfg(nullified_overrides)[0]
assert stage_cfg == baseline
def test_default_cache_config_used_when_missing(): def test_default_cache_config_used_when_missing():
"""Ensure default cache_config is synthesized when only backend is given.""" """Ensure default cache_config is synthesized when only backend is given."""
stage_cfg = AsyncOmniEngine._create_default_diffusion_stage_cfg( stage_cfg = AsyncOmniEngine._create_default_diffusion_stage_cfg(
@@ -55,6 +65,44 @@ def test_default_stage_devices_from_sequence_parallel():
assert stage_cfg["runtime"]["devices"] == "0,1,2,3" assert stage_cfg["runtime"]["devices"] == "0,1,2,3"
def test_default_stage_config_uses_parallel_size_kwargs():
"""Ensure default diffusion parallel_config uses CLI/API parallel sizes."""
stage_cfg = AsyncOmniEngine._create_default_diffusion_stage_cfg(
{
"pipeline_parallel_size": 2,
"data_parallel_size": 3,
"tensor_parallel_size": 4,
"enable_expert_parallel": True,
}
)[0]
parallel_config = stage_cfg["engine_args"]["parallel_config"]
assert parallel_config.pipeline_parallel_size == 2
assert parallel_config.data_parallel_size == 3
assert parallel_config.tensor_parallel_size == 4
assert parallel_config.enable_expert_parallel is True
def test_default_stage_config_defaults_nullified_parallel_size_kwargs():
"""Ensure nullified diffusion parallel-size kwargs fall back to defaults."""
stage_cfg = AsyncOmniEngine._create_default_diffusion_stage_cfg(
{
"pipeline_parallel_size": None,
"data_parallel_size": None,
"tensor_parallel_size": None,
"enable_expert_parallel": None,
"enforce_eager": None,
}
)[0]
parallel_config = stage_cfg["engine_args"]["parallel_config"]
assert parallel_config.pipeline_parallel_size == 1
assert parallel_config.data_parallel_size == 1
assert parallel_config.tensor_parallel_size == 1
assert parallel_config.enable_expert_parallel is False
assert stage_cfg["engine_args"]["enforce_eager"] is False
def test_default_stage_config_propagates_ulysses_mode(): def test_default_stage_config_propagates_ulysses_mode():
"""Ensure UAA mode survives default diffusion-stage creation.""" """Ensure UAA mode survives default diffusion-stage creation."""
stage_cfg = AsyncOmniEngine._create_default_diffusion_stage_cfg( stage_cfg = AsyncOmniEngine._create_default_diffusion_stage_cfg(
@@ -26,27 +26,9 @@ def _skip_readme_snippet(language: str, code: str, h2_title: str) -> tuple[bool,
README_SNIPPETS = ReadmeSnippet.extract_readme_snippets(README_PATH, skipif=_skip_readme_snippet) README_SNIPPETS = ReadmeSnippet.extract_readme_snippets(README_PATH, skipif=_skip_readme_snippet)
# Track: https://github.com/vllm-project/vllm-omni/issues/3123 — CLI example subprocess exit 1 in CI; re-enable when fixed.
_SKIP_T2I_SNIPPET_IDS = frozenset(
{
"quick_start_002",
"more_cli_examples_001",
"more_cli_examples_002",
"more_cli_examples_003",
"more_cli_examples_007",
"more_cli_examples_008",
}
)
_SKIP_T2I_REASON = (
"Skipped: text_to_image README examples failing in CI (subprocess exit 1). "
"See https://github.com/vllm-project/vllm-omni/issues/3123"
)
@pytest.mark.parametrize("snippet", README_SNIPPETS, ids=lambda snippet: snippet.test_id) @pytest.mark.parametrize("snippet", README_SNIPPETS, ids=lambda snippet: snippet.test_id)
def test_text_to_image(snippet: ReadmeSnippet, example_runner: ExampleRunner): def test_text_to_image(snippet: ReadmeSnippet, example_runner: ExampleRunner):
if snippet.test_id in _SKIP_T2I_SNIPPET_IDS:
pytest.skip(_SKIP_T2I_REASON)
should_skip, reason = snippet.skip should_skip, reason = snippet.skip
if should_skip: if should_skip:
pytest.skip(reason) pytest.skip(reason)
+3 -9
View File
@@ -3,14 +3,7 @@
"""Pytest marks and decorators for hardware / resource selection (CUDA, ROCm, …).""" """Pytest marks and decorators for hardware / resource selection (CUDA, ROCm, …)."""
import pytest import pytest
from vllm.platforms import current_platform
try:
from vllm.utils.torch_utils import cuda_device_count_stateless
except ImportError:
import torch
def cuda_device_count_stateless() -> int:
return torch.cuda.device_count()
# Re-exported from tests.helpers.env (GPU wait + DeviceMemoryMonitor). # Re-exported from tests.helpers.env (GPU wait + DeviceMemoryMonitor).
@@ -27,8 +20,9 @@ def cuda_marks(*, res: str, num_cards: int):
if num_cards == 1: if num_cards == 1:
return marks return marks
test_distributed = pytest.mark.distributed_cuda(num_cards=num_cards) test_distributed = pytest.mark.distributed_cuda(num_cards=num_cards)
test_skipif = pytest.mark.skipif_cuda( test_skipif = pytest.mark.skipif_cuda(
cuda_device_count_stateless() < num_cards, not current_platform.is_cuda() or (current_platform.device_count() < num_cards),
reason=f"Need at least {num_cards} CUDA GPUs to run the test.", reason=f"Need at least {num_cards} CUDA GPUs to run the test.",
) )
return marks + [test_distributed, test_skipif] return marks + [test_distributed, test_skipif]
+1
View File
@@ -494,6 +494,7 @@ _CI_OVERLAYS: dict[str, dict[str, Any]] = {
"max_num_seqs": 1, "max_num_seqs": 1,
"gpu_memory_utilization": 0.9, "gpu_memory_utilization": 0.9,
"enforce_eager": True, "enforce_eager": True,
"enable_prefix_caching": False,
"max_num_batched_tokens": 16384, "max_num_batched_tokens": 16384,
"max_model_len": 16384, "max_model_len": 16384,
"skip_mm_profiling": True, "skip_mm_profiling": True,
+1 -1
View File
@@ -369,8 +369,8 @@ def _build_full_serve_parser():
def test_nullify_stage_engine_defaults_resets_inherited_defaults(): def test_nullify_stage_engine_defaults_resets_inherited_defaults():
import argparse import argparse
from vllm_omni.config.stage_config import deploy_override_field_names
from vllm_omni.engine.arg_utils import ( from vllm_omni.engine.arg_utils import (
deploy_override_field_names,
nullify_stage_engine_defaults, nullify_stage_engine_defaults,
) )
+67 -2
View File
@@ -93,6 +93,7 @@ class TestStageConfig:
assert omega_config.engine_args.worker_type == "ar" assert omega_config.engine_args.worker_type == "ar"
assert omega_config.final_output is True assert omega_config.final_output is True
assert omega_config.final_output_type == "text" assert omega_config.final_output_type == "text"
assert "max_num_seqs" not in omega_config.engine_args
# Legacy field name for backward compatibility # Legacy field name for backward compatibility
assert omega_config.engine_input_source == [] assert omega_config.engine_input_source == []
@@ -146,6 +147,24 @@ class TestStageConfig:
omega_config = config.to_omegaconf() omega_config = config.to_omegaconf()
assert omega_config.engine_args.max_num_seqs == 32 assert omega_config.engine_args.max_num_seqs == 32
def test_to_omegaconf_omits_none_deploy_overrides_for_engine_args(self):
"""None deploy overrides must fall through to EngineArgs defaults."""
from vllm_omni.config.stage_config import deploy_override_field_names
config = StageConfig(
stage_id=0,
model_stage="thinker",
runtime_overrides={name: None for name in deploy_override_field_names()},
)
omega_config = config.to_omegaconf()
engine_args = dict(omega_config.engine_args)
assert "devices" not in engine_args
assert "max_batch_size" not in engine_args
for name in deploy_override_field_names() - {"devices"}:
assert name not in engine_args
class TestModelPipeline: class TestModelPipeline:
"""Tests for ModelPipeline class.""" """Tests for ModelPipeline class."""
@@ -802,6 +821,40 @@ class TestPipelineRegistry:
class TestDeployConfigLoading: class TestDeployConfigLoading:
def test_deploy_override_fields_include_deploy_schema_fields(self):
from vllm_omni.config.stage_config import deploy_override_field_names
expected_fields = {
"async_chunk",
"async_scheduling",
"config_format",
"data_parallel_size",
"devices",
"disable_hybrid_kv_cache_manager",
"distributed_executor_backend",
"dtype",
"enable_chunked_prefill",
"enable_flashinfer_autotune",
"enable_prefix_caching",
"enforce_eager",
"gpu_memory_utilization",
"load_format",
"max_model_len",
"max_num_batched_tokens",
"max_num_seqs",
"mm_processor_cache_gb",
"pipeline_parallel_size",
"profiler_config",
"quantization",
"skip_mm_profiling",
"subtalker_sampling_params",
"tensor_parallel_size",
"tokenizer_mode",
"trust_remote_code",
}
assert expected_fields == deploy_override_field_names()
def test_load_deploy_config(self): def test_load_deploy_config(self):
from pathlib import Path from pathlib import Path
@@ -817,6 +870,17 @@ class TestDeployConfigLoading:
assert deploy.connectors is not None assert deploy.connectors is not None
assert deploy.platforms is not None assert deploy.platforms is not None
voxtral_path = Path(__file__).parent.parent / "vllm_omni" / "deploy" / "voxtral_tts.yaml"
if voxtral_path.exists():
voxtral_deploy = load_deploy_config(voxtral_path)
assert voxtral_deploy.stages[0].config_format == "mistral"
assert voxtral_deploy.stages[0].load_format == "mistral"
assert voxtral_deploy.stages[0].tokenizer_mode == "mistral"
assert not any(
name in voxtral_deploy.stages[0].engine_extras
for name in ("config_format", "load_format", "tokenizer_mode")
)
def test_merge_pipeline_deploy(self): def test_merge_pipeline_deploy(self):
from pathlib import Path from pathlib import Path
@@ -1011,7 +1075,8 @@ class TestBaseConfigInheritance:
deploy = load_deploy_config(ci_path) deploy = load_deploy_config(ci_path)
assert len(deploy.stages) == 3 assert len(deploy.stages) == 3
# CI overrides # CI overrides
assert deploy.stages[0].engine_extras.get("load_format") == "dummy" assert deploy.stages[0].load_format == "dummy"
assert "load_format" not in deploy.stages[0].engine_extras
assert deploy.stages[0].max_num_seqs == 5 assert deploy.stages[0].max_num_seqs == 5
# Inherited from base # Inherited from base
assert deploy.stages[0].gpu_memory_utilization == 0.9 assert deploy.stages[0].gpu_memory_utilization == 0.9
@@ -1216,7 +1281,7 @@ class TestSentinelDefaultPrecedence:
def test_none_value_skipped_yaml_wins(self): def test_none_value_skipped_yaml_wins(self):
stages = self._stages({"max_num_seqs": None}) stages = self._stages({"max_num_seqs": None})
assert stages[2].runtime_overrides.get("max_num_seqs") is None assert stages[2].runtime_overrides.get("max_num_seqs") is None
assert stages[2].yaml_engine_args.get("max_num_seqs") == 1 assert "max_num_seqs" not in stages[2].yaml_engine_args
def test_empty_kwargs_yaml_only(self): def test_empty_kwargs_yaml_only(self):
stages = self._stages({}) stages = self._stages({})
+38 -16
View File
@@ -402,11 +402,11 @@ class StageDeployConfig:
""" """
stage_id: int stage_id: int
max_num_seqs: int = 64 max_num_seqs: int | None = None
gpu_memory_utilization: float = 0.9 gpu_memory_utilization: float | None = None
tensor_parallel_size: int = 1 tensor_parallel_size: int | None = None
enforce_eager: bool = False enforce_eager: bool | None = None
max_num_batched_tokens: int = 32768 max_num_batched_tokens: int | None = None
max_model_len: int | None = None max_model_len: int | None = None
async_scheduling: bool | None = None async_scheduling: bool | None = None
devices: str = "0" devices: str = "0"
@@ -414,6 +414,14 @@ class StageDeployConfig:
input_connectors: dict[str, str] | None = None input_connectors: dict[str, str] | None = None
default_sampling_params: dict[str, Any] | None = None default_sampling_params: dict[str, Any] | None = None
subtalker_sampling_params: dict[str, Any] | None = None subtalker_sampling_params: dict[str, Any] | None = None
profiler_config: dict[str, Any] | None = None
disable_hybrid_kv_cache_manager: bool | None = None
mm_processor_cache_gb: float | None = None
skip_mm_profiling: bool | None = None
enable_flashinfer_autotune: bool | None = None
config_format: str | None = None
load_format: str | None = None
tokenizer_mode: str | None = None
engine_extras: dict[str, Any] = field(default_factory=dict) engine_extras: dict[str, Any] = field(default_factory=dict)
@@ -438,14 +446,14 @@ class DeployConfig:
pipeline: str | None = None pipeline: str | None = None
# === Pipeline-wide engine settings (applied uniformly to every stage) === # === Pipeline-wide engine settings (applied uniformly to every stage) ===
trust_remote_code: bool = True trust_remote_code: bool | None = None
distributed_executor_backend: str | None = None distributed_executor_backend: str | None = None
dtype: str | None = None dtype: str | None = None
quantization: str | None = None quantization: str | None = None
enable_prefix_caching: bool = False enable_prefix_caching: bool | None = None
enable_chunked_prefill: bool | None = None enable_chunked_prefill: bool | None = None
data_parallel_size: int = 1 data_parallel_size: int | None = None
pipeline_parallel_size: int = 1 pipeline_parallel_size: int | None = None
_STAGE_NON_ENGINE_KEYS = frozenset( _STAGE_NON_ENGINE_KEYS = frozenset(
@@ -689,6 +697,18 @@ _PIPELINE_WIDE_ENGINE_FIELDS: tuple[str, ...] = (
) )
def deploy_override_field_names() -> frozenset[str]:
"""Return deploy-schema fields whose CLI defaults must not override YAML."""
return (
frozenset(_STAGE_DEPLOY_FIELDS)
| frozenset(_PIPELINE_WIDE_ENGINE_FIELDS)
| {
"async_chunk",
"devices",
}
)
def _build_engine_args( def _build_engine_args(
ps: StagePipelineConfig, ps: StagePipelineConfig,
ds: StageDeployConfig | None, ds: StageDeployConfig | None,
@@ -861,13 +881,15 @@ class StageConfig:
# CLI overrides take precedence over YAML defaults # CLI overrides take precedence over YAML defaults
for key, value in self.runtime_overrides.items(): for key, value in self.runtime_overrides.items():
if value is None:
continue
if key not in ("devices", "max_batch_size"): if key not in ("devices", "max_batch_size"):
engine_args[key] = value engine_args[key] = value
# Build runtime config from YAML defaults + CLI overrides # Build runtime config from YAML defaults + CLI overrides
runtime: dict[str, Any] = dict(self.yaml_runtime) runtime: dict[str, Any] = dict(self.yaml_runtime)
runtime.setdefault("process", True) runtime.setdefault("process", True)
if "devices" in self.runtime_overrides: if self.runtime_overrides.get("devices") is not None:
runtime["devices"] = self.runtime_overrides["devices"] runtime["devices"] = self.runtime_overrides["devices"]
# Legacy compat: migrate runtime.max_batch_size → engine_args.max_num_seqs # Legacy compat: migrate runtime.max_batch_size → engine_args.max_num_seqs
@@ -883,8 +905,6 @@ class StageConfig:
effective_mbs = int(cli_mbs or legacy_mbs or 1) effective_mbs = int(cli_mbs or legacy_mbs or 1)
engine_args.setdefault("max_num_seqs", effective_mbs) engine_args.setdefault("max_num_seqs", effective_mbs)
engine_args.setdefault("max_num_seqs", 1)
# Build full config dict # Build full config dict
config_dict: dict[str, Any] = { config_dict: dict[str, Any] = {
"stage_id": self.stage_id, "stage_id": self.stage_id,
@@ -1009,6 +1029,8 @@ class StageConfigFactory:
cli_overrides = {} cli_overrides = {}
trust_remote_code = cli_overrides.get("trust_remote_code", True) trust_remote_code = cli_overrides.get("trust_remote_code", True)
if trust_remote_code is None:
trust_remote_code = False
# --- New path: check pipeline registry by model_type first --- # --- New path: check pipeline registry by model_type first ---
model_type, hf_config = cls._auto_detect_model_type(model, trust_remote_code=trust_remote_code) model_type, hf_config = cls._auto_detect_model_type(model, trust_remote_code=trust_remote_code)
@@ -1352,8 +1374,8 @@ class StageConfigFactory:
hf_config = get_config(model, trust_remote_code=trust_remote_code) hf_config = get_config(model, trust_remote_code=trust_remote_code)
return hf_config.model_type, hf_config return hf_config.model_type, hf_config
except Exception: except Exception as e:
pass logger.debug(f"`get_config` failed for {e}; Falling back to raw config.json path")
# Fallback: read config.json directly for custom model types that # Fallback: read config.json directly for custom model types that
# are not registered with transformers (e.g. qwen3_tts). # are not registered with transformers (e.g. qwen3_tts).
@@ -1390,8 +1412,8 @@ class StageConfigFactory:
class_name, class_name,
) )
return pipeline_cfg.model_type, None return pipeline_cfg.model_type, None
except Exception: except Exception as e:
pass logger.debug(f"Failed to detect model type for diffusers-style models: {e}")
# Final fallback: some models (e.g. CosyVoice3) ship an empty # Final fallback: some models (e.g. CosyVoice3) ship an empty
# config.json and rely on naming conventions. Match the model path # config.json and rely on naming conventions. Match the model path
+2 -2
View File
@@ -40,7 +40,7 @@ stages:
# near-identity repetition penalty forces vLLM to track # near-identity repetition penalty forces vLLM to track
# output_token_ids for RAS (stop-token logit logsumexp). # output_token_ids for RAS (stop-token logit logsumexp).
repetition_penalty: 1.0001 repetition_penalty: 1.0001
disable_hybrid_kv_cache_manager: true enable_prefix_caching: false
mm_processor_cache_gb: 0 mm_processor_cache_gb: 0
skip_mm_profiling: true skip_mm_profiling: true
@@ -54,5 +54,5 @@ stages:
from_stage_0: connector_of_shared_memory from_stage_0: connector_of_shared_memory
default_sampling_params: default_sampling_params:
max_tokens: 2048 max_tokens: 2048
disable_hybrid_kv_cache_manager: true enable_prefix_caching: false
skip_mm_profiling: true skip_mm_profiling: true
+1
View File
@@ -42,6 +42,7 @@ stages:
max_num_seqs: 1 max_num_seqs: 1
gpu_memory_utilization: 0.2 gpu_memory_utilization: 0.2
enforce_eager: true enforce_eager: true
enable_prefix_caching: false
async_scheduling: false async_scheduling: false
max_num_batched_tokens: 8192 max_num_batched_tokens: 8192
max_model_len: 8192 max_model_len: 8192
+2
View File
@@ -22,6 +22,7 @@ stages:
max_num_seqs: 1 max_num_seqs: 1
gpu_memory_utilization: 0.8 gpu_memory_utilization: 0.8
enforce_eager: true enforce_eager: true
enable_prefix_caching: false
mm_processor_cache_gb: 0 mm_processor_cache_gb: 0
devices: "0" devices: "0"
default_sampling_params: default_sampling_params:
@@ -49,6 +50,7 @@ stages:
max_num_seqs: 1 max_num_seqs: 1
gpu_memory_utilization: 0.15 gpu_memory_utilization: 0.15
enforce_eager: true enforce_eager: true
enable_prefix_caching: false
enable_flashinfer_autotune: false enable_flashinfer_autotune: false
async_scheduling: false async_scheduling: false
devices: "0" devices: "0"
+2
View File
@@ -23,6 +23,7 @@ connectors:
stages: stages:
- stage_id: 0 - stage_id: 0
gpu_memory_utilization: 0.9 gpu_memory_utilization: 0.9
enable_prefix_caching: false
devices: "0" devices: "0"
default_sampling_params: default_sampling_params:
temperature: 0.4 temperature: 0.4
@@ -47,6 +48,7 @@ stages:
- stage_id: 2 - stage_id: 2
gpu_memory_utilization: 0.1 gpu_memory_utilization: 0.1
enforce_eager: true enforce_eager: true
enable_prefix_caching: false
async_scheduling: false async_scheduling: false
max_num_batched_tokens: 51200 max_num_batched_tokens: 51200
devices: "1" devices: "1"
+1
View File
@@ -53,6 +53,7 @@ stages:
max_num_seqs: 1 max_num_seqs: 1
gpu_memory_utilization: 0.3 gpu_memory_utilization: 0.3
enforce_eager: true enforce_eager: true
enable_prefix_caching: false
async_scheduling: true async_scheduling: true
# Must be divisible by num_code_groups and cover (left_context + chunk). # Must be divisible by num_code_groups and cover (left_context + chunk).
# Prefill length is Q * num_frames (e.g. 16 * 2148 = 34368); keep # Prefill length is Q * num_frames (e.g. 16 * 2148 = 34368); keep
+1
View File
@@ -20,6 +20,7 @@ stages:
max_num_batched_tokens: 4096 max_num_batched_tokens: 4096
max_model_len: 4096 max_model_len: 4096
devices: "0" devices: "0"
trust_remote_code: true
default_sampling_params: default_sampling_params:
temperature: 0.0 temperature: 0.0
top_p: 1.0 top_p: 1.0
+2
View File
@@ -24,6 +24,7 @@ stages:
max_num_seqs: 32 max_num_seqs: 32
gpu_memory_utilization: 0.8 gpu_memory_utilization: 0.8
enforce_eager: false enforce_eager: false
enable_prefix_caching: false
async_scheduling: true async_scheduling: true
max_model_len: 4096 max_model_len: 4096
devices: "0" devices: "0"
@@ -48,6 +49,7 @@ stages:
max_num_seqs: 32 max_num_seqs: 32
gpu_memory_utilization: 0.1 gpu_memory_utilization: 0.1
enforce_eager: true enforce_eager: true
enable_prefix_caching: false
async_scheduling: false async_scheduling: false
max_num_batched_tokens: 65536 max_num_batched_tokens: 65536
max_model_len: 65536 max_model_len: 65536
@@ -28,12 +28,6 @@ class DistributedAutoencoderKL_base(DistributedVaeMixin):
model.init_distributed() model.init_distributed()
return model return model
@classmethod
def from_config(cls, *args: Any, **kwargs: Any):
model = super().from_config(*args, **kwargs)
model.init_distributed()
return model
def tile_split(self, z: torch.Tensor) -> tuple[list[TileTask], GridSpec]: def tile_split(self, z: torch.Tensor) -> tuple[list[TileTask], GridSpec]:
# mostly copy from AutoencoderKL # mostly copy from AutoencoderKL
overlap_size = int(self.tile_latent_min_size * (1 - self.tile_overlap_factor)) overlap_size = int(self.tile_latent_min_size * (1 - self.tile_overlap_factor))
@@ -385,9 +385,9 @@ class MultiprocDiffusionExecutor(DiffusionExecutor):
raise raise
def check_health(self) -> None: def check_health(self) -> None:
self._ensure_open()
if self.is_failed: if self.is_failed:
raise EngineDeadError() raise EngineDeadError()
self._ensure_open()
for p in self._processes: for p in self._processes:
if not p.is_alive(): if not p.is_alive():
self.is_failed = True self.is_failed = True
@@ -15,6 +15,7 @@ from typing import TYPE_CHECKING, Any
import torch import torch
from PIL import Image from PIL import Image
from vllm.logger import init_logger from vllm.logger import init_logger
from vllm.v1.engine.exceptions import EngineDeadError
from vllm_omni.diffusion.data import DiffusionRequestAbortedError from vllm_omni.diffusion.data import DiffusionRequestAbortedError
from vllm_omni.diffusion.diffusion_engine import DiffusionEngine from vllm_omni.diffusion.diffusion_engine import DiffusionEngine
@@ -329,6 +330,12 @@ class InlineStageDiffusionClient:
None, None,
) )
def check_health(self) -> None:
"""Check if the inline diffusion engine and its workers are healthy."""
if self._shutting_down:
raise EngineDeadError("InlineStageDiffusionClient is shutting down")
self._engine.executor.check_health()
def shutdown(self) -> None: def shutdown(self) -> None:
self._shutting_down = True self._shutting_down = True
@@ -26,7 +26,6 @@ from vllm.model_executor.model_loader.weight_utils import (
multi_thread_safetensors_weights_iterator, multi_thread_safetensors_weights_iterator,
safetensors_weights_iterator, safetensors_weights_iterator,
) )
from vllm.transformers_utils.repo_utils import file_exists
from vllm.utils.import_utils import resolve_obj_by_qualname from vllm.utils.import_utils import resolve_obj_by_qualname
from vllm.utils.torch_utils import set_default_torch_dtype from vllm.utils.torch_utils import set_default_torch_dtype
@@ -50,8 +49,6 @@ def _natural_sort_key(filepath: str) -> list:
MODEL_INDEX = "model_index.json" MODEL_INDEX = "model_index.json"
DIFFUSION_MODEL_WEIGHTS_INDEX = "diffusion_pytorch_model.safetensors.index.json" DIFFUSION_MODEL_WEIGHTS_INDEX = "diffusion_pytorch_model.safetensors.index.json"
TRANSFORMER_WEIGHTS_INDEX = "model.safetensors.index.json"
INDEX_FILES = [DIFFUSION_MODEL_WEIGHTS_INDEX, TRANSFORMER_WEIGHTS_INDEX]
class DiffusersPipelineLoader: class DiffusersPipelineLoader:
@@ -102,22 +99,8 @@ class DiffusersPipelineLoader:
is_local = os.path.isdir(model_name_or_path) is_local = os.path.isdir(model_name_or_path)
load_format = self.load_config.load_format load_format = self.load_config.load_format
use_safetensors = False use_safetensors = False
possible_index_files = [ index_file = DIFFUSION_MODEL_WEIGHTS_INDEX
f"{subfolder}/{index_file}" if subfolder is not None else index_file for index_file in INDEX_FILES index_file_with_subfolder = f"{subfolder}/{index_file}" if subfolder else index_file
]
available_index_file = list(
filter(lambda f: file_exists(model_name_or_path, f, revision=revision), possible_index_files)
)
if len(available_index_file) > 1:
raise ValueError(
f"Multiple index files found in {model_name_or_path} with subfolder {subfolder}: {available_index_file}"
)
index_file_with_subfolder = available_index_file[0] if len(available_index_file) == 1 else None
index_file = (
index_file_with_subfolder.split("/")[-1]
if index_file_with_subfolder and subfolder is not None
else index_file_with_subfolder
)
# only hf is supported currently # only hf is supported currently
if load_format == "auto": if load_format == "auto":
@@ -180,7 +163,7 @@ class DiffusersPipelineLoader:
hf_weights_files = filter_duplicate_safetensors_files( hf_weights_files = filter_duplicate_safetensors_files(
hf_weights_files, hf_weights_files,
filter_folder, filter_folder,
index_file or "", index_file,
) )
else: else:
hf_weights_files = filter_files_not_needed_for_inference(hf_weights_files) hf_weights_files = filter_files_not_needed_for_inference(hf_weights_files)
@@ -1842,6 +1842,8 @@ class Bagel(nn.Module):
# ── SP + CFG: sequential single-branch forwards ── # ── SP + CFG: sequential single-branch forwards ──
use_sp = self._sp_size > 1 use_sp = self._sp_size > 1
if use_sp and use_cfg_text: if use_sp and use_cfg_text:
if return_trajectory_latents and len(timesteps) > 0:
trajectory_latents.append(x_t.clone())
for i, t in enumerate(timesteps): for i, t in enumerate(timesteps):
timestep = torch.tensor([t] * x_t.shape[0], device=x_t.device) timestep = torch.tensor([t] * x_t.shape[0], device=x_t.device)
in_cfg_window = t > cfg_interval[0] and t <= cfg_interval[1] in_cfg_window = t > cfg_interval[0] and t <= cfg_interval[1]
@@ -1905,13 +1907,15 @@ class Bagel(nn.Module):
x_t = x_t - v_t.to(x_t.device) * dts[i] x_t = x_t - v_t.to(x_t.device) * dts[i]
if return_trajectory_latents: if return_trajectory_latents:
trajectory_latents.append(x_t.clone()) trajectory_latents.append(x_t.clone())
trajectory_timesteps.append(timesteps[i] - dts[i]) trajectory_timesteps.append(timesteps[i])
unpacked_latent = x_t.split((packed_seqlens - 2).tolist()) unpacked_latent = x_t.split((packed_seqlens - 2).tolist())
return unpacked_latent, trajectory_latents, trajectory_timesteps, trajectory_log_probs return unpacked_latent, trajectory_latents, trajectory_timesteps, trajectory_log_probs
# ── SP without CFG: direct single-branch loop ── # ── SP without CFG: direct single-branch loop ──
if use_sp: if use_sp:
if return_trajectory_latents and len(timesteps) > 0:
trajectory_latents.append(x_t.clone())
for i, t in enumerate(timesteps): for i, t in enumerate(timesteps):
timestep = torch.tensor([t] * x_t.shape[0], device=x_t.device) timestep = torch.tensor([t] * x_t.shape[0], device=x_t.device)
v_t = self.forward_single_branch( v_t = self.forward_single_branch(
@@ -1938,7 +1942,7 @@ class Bagel(nn.Module):
x_t = x_t - v_t.to(x_t.device) * dts[i] x_t = x_t - v_t.to(x_t.device) * dts[i]
if return_trajectory_latents: if return_trajectory_latents:
trajectory_latents.append(x_t.clone()) trajectory_latents.append(x_t.clone())
trajectory_timesteps.append(timesteps[i] - dts[i]) trajectory_timesteps.append(timesteps[i])
unpacked_latent = x_t.split((packed_seqlens - 2).tolist()) unpacked_latent = x_t.split((packed_seqlens - 2).tolist())
return unpacked_latent, trajectory_latents, trajectory_timesteps, trajectory_log_probs return unpacked_latent, trajectory_latents, trajectory_timesteps, trajectory_log_probs
@@ -1999,6 +2003,9 @@ class Bagel(nn.Module):
"merged_cache": self._merge_naive_caches(branches_cache), "merged_cache": self._merge_naive_caches(branches_cache),
} }
if return_trajectory_latents and len(timesteps) > 0:
trajectory_latents.append(x_t.clone())
for i, t in enumerate(timesteps): for i, t in enumerate(timesteps):
timestep = torch.tensor([t] * x_t.shape[0], device=x_t.device) timestep = torch.tensor([t] * x_t.shape[0], device=x_t.device)
if t > cfg_interval[0] and t <= cfg_interval[1]: if t > cfg_interval[0] and t <= cfg_interval[1]:
@@ -2036,7 +2043,7 @@ class Bagel(nn.Module):
x_t = x_t - v_t.to(x_t.device) * dts[i] # velocity pointing from data to noise x_t = x_t - v_t.to(x_t.device) * dts[i] # velocity pointing from data to noise
if return_trajectory_latents: if return_trajectory_latents:
trajectory_latents.append(x_t.clone()) trajectory_latents.append(x_t.clone())
trajectory_timesteps.append(timesteps[i] - dts[i]) trajectory_timesteps.append(timesteps[i])
unpacked_latent = x_t.split((packed_seqlens - 2).tolist()) unpacked_latent = x_t.split((packed_seqlens - 2).tolist())
return unpacked_latent, trajectory_latents, trajectory_timesteps, trajectory_log_probs return unpacked_latent, trajectory_latents, trajectory_timesteps, trajectory_log_probs
@@ -2137,6 +2144,9 @@ class Bagel(nn.Module):
) )
_sched_kw = scheduler_kwargs or {} _sched_kw = scheduler_kwargs or {}
if return_trajectory_latents and len(timesteps) > 0:
trajectory_latents.append(x_t.clone())
for i, t in enumerate(timesteps): for i, t in enumerate(timesteps):
timestep = torch.tensor([t] * x_t.shape[0], device=x_t.device) timestep = torch.tensor([t] * x_t.shape[0], device=x_t.device)
use_cfg_this_step = t > cfg_interval[0] and t <= cfg_interval[1] and cfg_text_scale > 1.0 use_cfg_this_step = t > cfg_interval[0] and t <= cfg_interval[1] and cfg_text_scale > 1.0
@@ -2194,7 +2204,7 @@ class Bagel(nn.Module):
x_t = x_t - v_t.to(x_t.device) * dts[i] x_t = x_t - v_t.to(x_t.device) * dts[i]
if return_trajectory_latents: if return_trajectory_latents:
trajectory_latents.append(x_t.clone()) trajectory_latents.append(x_t.clone())
trajectory_timesteps.append(timesteps[i] - dts[i]) trajectory_timesteps.append(timesteps[i])
unpacked_latent = x_t.split((packed_seqlens - 2).tolist()) unpacked_latent = x_t.split((packed_seqlens - 2).tolist())
return unpacked_latent, trajectory_latents, trajectory_timesteps, trajectory_log_probs return unpacked_latent, trajectory_latents, trajectory_timesteps, trajectory_log_probs
-126
View File
@@ -5,132 +5,6 @@ from __future__ import annotations
import json import json
import os import os
from typing import TYPE_CHECKING, Literal
import torch
import torch.nn as nn
from vllm.model_executor.layers.linear import (
ColumnParallelLinear,
ReplicatedLinear,
RowParallelLinear,
)
from vllm.model_executor.models.transformers.utils import init_on_device_without_buffers
from vllm.model_executor.models.utils import maybe_prefix
from vllm_omni.diffusion.data import OmniDiffusionConfig
from vllm_omni.diffusion.quantization import get_vllm_quant_config_for_layers
if TYPE_CHECKING:
from transformers import PretrainedConfig, PreTrainedModel
from transformers.models.auto.auto_factory import _BaseAutoModelClass
from vllm.model_executor.layers.quantization.base_config import (
QuantizationConfig,
)
Style = Literal["colwise", "colwise_rep", "rowwise", "rowwise_rep", "replicate"]
def replace_linear_class(
linear: nn.Linear,
style: Style = "replicate",
quant_config: QuantizationConfig | None = None,
*,
prefix: str = "",
) -> ColumnParallelLinear | RowParallelLinear | ReplicatedLinear:
"""
Replace nn.Linear with one of vLLM's tensor parallel linear classes.
Args:
linear: `nn.Linear` to be replaced.
style: Tensor parallel style of the new linear, e.g. "colwise".
quant_config: Quantization config for the new linear.
Returns:
The new linear.
"""
if not isinstance(style, str):
raise ValueError(f"Unsupported parallel style type {type(style)}, expected str")
vllm_linear_maps = {
"colwise": (ColumnParallelLinear, {}),
"colwise_rep": (ColumnParallelLinear, {"gather_output": True}),
"rowwise": (RowParallelLinear, {}),
"rowwise_rep": (RowParallelLinear, {"input_is_parallel": False}),
"replicate": (ReplicatedLinear, {}),
}
vllm_linear_cls, vllm_linear_kwargs = vllm_linear_maps[style]
return vllm_linear_cls(
input_size=linear.in_features,
output_size=linear.out_features,
bias=linear.bias is not None,
quant_config=quant_config,
prefix=prefix,
return_bias=False,
**vllm_linear_kwargs,
)
def recursive_replace_linear(model: nn.Module, od_config: OmniDiffusionConfig):
"""Recursively replace modules in the model as needed.
Currently, this replaces:
- `nn.Linear` with vLLM's tensor parallel linear classes
"""
# Prefix the patterns because we always start from `self.model`
quant_config = get_vllm_quant_config_for_layers(od_config.quantization_config)
def _recursive_replace(module: nn.Module, prefix: str):
for child_name, child_module in module.named_children():
new_module = child_module
qual_name = maybe_prefix(prefix, child_name)
# Replace modules as needed
if isinstance(child_module, nn.Linear):
style = "replicate"
new_module = replace_linear_class(child_module, style, quant_config, prefix=qual_name)
else:
_recursive_replace(child_module, prefix=qual_name)
if new_module is not child_module:
setattr(module, child_name, new_module)
_recursive_replace(model, prefix="")
def init_parameters(
module: nn.Module,
dtype: torch.dtype | None,
device: torch.device | None = None,
):
for name, param in module.named_parameters(recurse=False):
if param.device == torch.device("meta"):
new_param = nn.Parameter(
torch.empty_like(
param.data,
dtype=dtype,
device=device,
),
requires_grad=param.requires_grad,
)
setattr(module, name, new_param)
for child in module.children():
init_parameters(child, dtype, device)
def create_transformers_model(
auto_cls: _BaseAutoModelClass,
od_config: OmniDiffusionConfig,
hf_config: PretrainedConfig,
dtype: torch.dtype | None = None,
device: torch.device | None = None,
) -> PreTrainedModel:
"""Create a HuggingFace model using the given auto class and model name."""
dtype = dtype or od_config.dtype
device = device or torch.get_default_device()
with init_on_device_without_buffers("meta"):
model = auto_cls.from_config(hf_config)
recursive_replace_linear(model, od_config)
init_parameters(model, dtype=dtype, device=device)
return model
def _load_json(model_path: str, filename: str, local_files_only: bool = True) -> dict: def _load_json(model_path: str, filename: str, local_files_only: bool = True) -> dict:
@@ -28,7 +28,7 @@ from diffusers.image_processor import PipelineImageInput, VaeImageProcessor
from diffusers.schedulers import FlowMatchEulerDiscreteScheduler from diffusers.schedulers import FlowMatchEulerDiscreteScheduler
from diffusers.utils import logging from diffusers.utils import logging
from diffusers.utils.torch_utils import randn_tensor from diffusers.utils.torch_utils import randn_tensor
from transformers import AutoConfig, AutoModelForCausalLM, AutoTokenizer from transformers import AutoModel, AutoTokenizer
from vllm.model_executor.models.utils import AutoWeightsLoader from vllm.model_executor.models.utils import AutoWeightsLoader
from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig
@@ -36,7 +36,6 @@ from vllm_omni.diffusion.distributed.autoencoders.autoencoder_kl import Distribu
from vllm_omni.diffusion.distributed.utils import get_local_device from vllm_omni.diffusion.distributed.utils import get_local_device
from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader
from vllm_omni.diffusion.model_loader.hub_prefetch import prefetch_subfolders from vllm_omni.diffusion.model_loader.hub_prefetch import prefetch_subfolders
from vllm_omni.diffusion.models.utils import create_transformers_model
from vllm_omni.diffusion.models.z_image.z_image_transformer import ( from vllm_omni.diffusion.models.z_image.z_image_transformer import (
ZImageTransformer2DModel, ZImageTransformer2DModel,
) )
@@ -170,25 +169,13 @@ class ZImagePipeline(nn.Module, DiffusionPipelineProfilerMixin):
super().__init__() super().__init__()
self.od_config = od_config self.od_config = od_config
self.weights_sources = [ self.weights_sources = [
DiffusersPipelineLoader.ComponentSource(
model_or_path=od_config.model,
subfolder="text_encoder",
revision=od_config.revision,
prefix="text_encoder.",
),
DiffusersPipelineLoader.ComponentSource( DiffusersPipelineLoader.ComponentSource(
model_or_path=od_config.model, model_or_path=od_config.model,
subfolder="transformer", subfolder="transformer",
revision=od_config.revision, revision=None,
prefix="transformer.", prefix="transformer.",
fall_back_to_pt=True, fall_back_to_pt=True,
), )
DiffusersPipelineLoader.ComponentSource(
model_or_path=od_config.model,
subfolder="vae",
revision=od_config.revision,
prefix="vae.",
),
] ]
self._execution_device = get_local_device() self._execution_device = get_local_device()
model = od_config.model model = od_config.model
@@ -205,19 +192,12 @@ class ZImagePipeline(nn.Module, DiffusionPipelineProfilerMixin):
model, subfolder="scheduler", local_files_only=local_files_only model, subfolder="scheduler", local_files_only=local_files_only
) )
text_encoder_config = AutoConfig.from_pretrained( self.text_encoder = AutoModel.from_pretrained(
model, subfolder="text_encoder", local_files_only=local_files_only model, subfolder="text_encoder", local_files_only=local_files_only
)
self.text_encoder = create_transformers_model(
AutoModelForCausalLM,
od_config,
hf_config=text_encoder_config,
).to(self._execution_device) ).to(self._execution_device)
if text_encoder_config.tie_word_embeddings: self.vae = DistributedAutoencoderKL.from_pretrained(
self.text_encoder.lm_head.weight = self.text_encoder.get_input_embeddings().weight model, subfolder="vae", local_files_only=local_files_only
).to(self._execution_device)
vae_config = DistributedAutoencoderKL.load_config(model, subfolder="vae", local_files_only=local_files_only)
self.vae = DistributedAutoencoderKL.from_config(vae_config).to(self._execution_device)
self.transformer = ZImageTransformer2DModel(quant_config=od_config.quantization_config) self.transformer = ZImageTransformer2DModel(quant_config=od_config.quantization_config)
self.tokenizer = AutoTokenizer.from_pretrained(model, subfolder="tokenizer", local_files_only=local_files_only) self.tokenizer = AutoTokenizer.from_pretrained(model, subfolder="tokenizer", local_files_only=local_files_only)
+2 -38
View File
@@ -456,50 +456,12 @@ SHARED_FIELDS: frozenset[str] = frozenset(
} }
) )
_DEPLOY_ENGINE_ARG_OVERRIDE_FIELDS: frozenset[str] = frozenset(
{
# Capacity / scheduling.
"async_scheduling",
"max_model_len",
"max_num_batched_tokens",
"max_num_seqs",
# Memory / parallelism.
"data_parallel_size",
"gpu_memory_utilization",
"pipeline_parallel_size",
"tensor_parallel_size",
# Execution / loading.
"enforce_eager",
"distributed_executor_backend",
"dtype",
"quantization",
"trust_remote_code",
# Caching / chunking.
"async_chunk",
"enable_prefix_caching",
"enable_chunked_prefill",
# Model-specific engine extras.
"subtalker_sampling_params",
}
)
_DEPLOY_RUNTIME_OVERRIDE_FIELDS: frozenset[str] = frozenset(
{
"devices",
}
)
def orchestrator_field_names() -> frozenset[str]: def orchestrator_field_names() -> frozenset[str]:
"""Return the names of every field on OrchestratorArgs.""" """Return the names of every field on OrchestratorArgs."""
return frozenset(f.name for f in fields(OrchestratorArgs)) return frozenset(f.name for f in fields(OrchestratorArgs))
def deploy_override_field_names() -> frozenset[str]:
"""Return kwargs whose parser defaults must not override deploy YAML."""
return _DEPLOY_ENGINE_ARG_OVERRIDE_FIELDS | _DEPLOY_RUNTIME_OVERRIDE_FIELDS
def internal_blacklist_keys() -> frozenset[str]: def internal_blacklist_keys() -> frozenset[str]:
"""Return the set of CLI keys that must never be forwarded as per-stage """Return the set of CLI keys that must never be forwarded as per-stage
engine overrides. engine overrides.
@@ -653,6 +615,8 @@ def nullify_stage_engine_defaults(parser: argparse.ArgumentParser) -> None:
"""Reset stage-level engine flag defaults to ``None``; preserve real """Reset stage-level engine flag defaults to ``None``; preserve real
default in help text. Only deploy-YAML override fields are touched. default in help text. Only deploy-YAML override fields are touched.
Idempotent.""" Idempotent."""
from vllm_omni.config.stage_config import deploy_override_field_names
override_dests = deploy_override_field_names() override_dests = deploy_override_field_names()
for action in parser._actions: for action in parser._actions:
+10 -3
View File
@@ -1294,9 +1294,12 @@ class AsyncOmniEngine:
ring_degree = normalized_kwargs.get("ring_degree") or 1 ring_degree = normalized_kwargs.get("ring_degree") or 1
ulysses_mode = normalized_kwargs.get("ulysses_mode") or "strict" ulysses_mode = normalized_kwargs.get("ulysses_mode") or "strict"
sequence_parallel_size = normalized_kwargs.get("sequence_parallel_size") sequence_parallel_size = normalized_kwargs.get("sequence_parallel_size")
pipeline_parallel_size = normalized_kwargs.get("pipeline_parallel_size") or 1
data_parallel_size = normalized_kwargs.get("data_parallel_size") or 1
tensor_parallel_size = normalized_kwargs.get("tensor_parallel_size") or 1 tensor_parallel_size = normalized_kwargs.get("tensor_parallel_size") or 1
cfg_parallel_size = normalized_kwargs.get("cfg_parallel_size") or 1 cfg_parallel_size = normalized_kwargs.get("cfg_parallel_size") or 1
vae_patch_parallel_size = normalized_kwargs.get("vae_patch_parallel_size") or 1 vae_patch_parallel_size = normalized_kwargs.get("vae_patch_parallel_size") or 1
enable_expert_parallel = normalized_kwargs.get("enable_expert_parallel") or False
use_hsdp = normalized_kwargs.get("use_hsdp", False) use_hsdp = normalized_kwargs.get("use_hsdp", False)
hsdp_shard_size = normalized_kwargs.get("hsdp_shard_size", -1) hsdp_shard_size = normalized_kwargs.get("hsdp_shard_size", -1)
hsdp_replicate_size = normalized_kwargs.get("hsdp_replicate_size", 1) hsdp_replicate_size = normalized_kwargs.get("hsdp_replicate_size", 1)
@@ -1304,9 +1307,10 @@ class AsyncOmniEngine:
sequence_parallel_size = ulysses_degree * ring_degree sequence_parallel_size = ulysses_degree * ring_degree
parallel_config = DiffusionParallelConfig( parallel_config = DiffusionParallelConfig(
pipeline_parallel_size=1, pipeline_parallel_size=pipeline_parallel_size,
data_parallel_size=1, data_parallel_size=data_parallel_size,
tensor_parallel_size=tensor_parallel_size, tensor_parallel_size=tensor_parallel_size,
enable_expert_parallel=enable_expert_parallel,
sequence_parallel_size=sequence_parallel_size, sequence_parallel_size=sequence_parallel_size,
ulysses_degree=ulysses_degree, ulysses_degree=ulysses_degree,
ring_degree=ring_degree, ring_degree=ring_degree,
@@ -1320,6 +1324,9 @@ class AsyncOmniEngine:
num_devices = max(1, int(parallel_config.world_size)) num_devices = max(1, int(parallel_config.world_size))
devices = ",".join(str(i) for i in range(num_devices)) devices = ",".join(str(i) for i in range(num_devices))
enforce_eager = kwargs.get("enforce_eager")
if enforce_eager is None:
enforce_eager = False
stage_engine_args = { stage_engine_args = {
"max_num_seqs": 1, "max_num_seqs": 1,
@@ -1333,7 +1340,7 @@ class AsyncOmniEngine:
"enable_cache_dit_summary": kwargs.get("enable_cache_dit_summary", False), "enable_cache_dit_summary": kwargs.get("enable_cache_dit_summary", False),
"enable_cpu_offload": kwargs.get("enable_cpu_offload", False), "enable_cpu_offload": kwargs.get("enable_cpu_offload", False),
"enable_layerwise_offload": kwargs.get("enable_layerwise_offload", False), "enable_layerwise_offload": kwargs.get("enable_layerwise_offload", False),
"enforce_eager": kwargs.get("enforce_eager", False), "enforce_eager": enforce_eager,
"boundary_ratio": kwargs.get("boundary_ratio", None), "boundary_ratio": kwargs.get("boundary_ratio", None),
"flow_shift": kwargs.get("flow_shift", None), "flow_shift": kwargs.get("flow_shift", None),
"diffusion_load_format": kwargs.get("diffusion_load_format", "default"), "diffusion_load_format": kwargs.get("diffusion_load_format", "default"),
+1 -3
View File
@@ -100,9 +100,7 @@ class OmniBase(PDDisaggregationMixin):
kwargs: dict[str, Any] = {k: v for k, v in vars(args).items() if not k.startswith("_")} kwargs: dict[str, Any] = {k: v for k, v in vars(args).items() if not k.startswith("_")}
if parser is not None and not getattr(parser, "_omni_nullified", False): if parser is not None and not getattr(parser, "_omni_nullified", False):
from vllm_omni.engine.arg_utils import ( from vllm_omni.config.stage_config import deploy_override_field_names
deploy_override_field_names,
)
from vllm_omni.entrypoints.utils import detect_explicit_cli_keys from vllm_omni.entrypoints.utils import detect_explicit_cli_keys
explicit = detect_explicit_cli_keys(sys.argv[1:], parser) or set() explicit = detect_explicit_cli_keys(sys.argv[1:], parser) or set()
+2 -3
View File
@@ -65,7 +65,7 @@ from vllm.entrypoints.openai.speech_to_text.serving import (
from vllm.entrypoints.openai.utils import validate_json_request from vllm.entrypoints.openai.utils import validate_json_request
from vllm.entrypoints.pooling.classify.serving import ServingClassification from vllm.entrypoints.pooling.classify.serving import ServingClassification
from vllm.entrypoints.pooling.embed.serving import ServingEmbedding as OpenAIServingEmbedding from vllm.entrypoints.pooling.embed.serving import ServingEmbedding as OpenAIServingEmbedding
from vllm.entrypoints.pooling.pooling.serving import ServingPooling as OpenAIServingPooling from vllm.entrypoints.pooling.pooling.serving import ServingPooling
from vllm.entrypoints.pooling.scoring.serving import ServingScores from vllm.entrypoints.pooling.scoring.serving import ServingScores
from vllm.entrypoints.serve.disagg.serving import ServingTokens from vllm.entrypoints.serve.disagg.serving import ServingTokens
@@ -800,10 +800,9 @@ async def omni_init_app_state(
else None else None
) )
state.openai_serving_pooling = ( state.openai_serving_pooling = (
OpenAIServingPooling( ServingPooling(
engine_client, engine_client,
state.openai_serving_models, state.openai_serving_models,
state.openai_serving_render,
supported_tasks=tuple(supported_tasks), supported_tasks=tuple(supported_tasks),
request_logger=request_logger, request_logger=request_logger,
chat_template=resolved_chat_template, chat_template=resolved_chat_template,
@@ -489,7 +489,7 @@ class Qwen3OmniMoeModel(Qwen3MoeLLMForCausalLM):
""" """
Qwen3 Omni MoE Talker language model. Qwen3 Omni MoE Talker language model.
Extends Qwen3MoeLLMForCausalLM (which already uses SharedFusedMoE with Extends Qwen3MoeLLMForCausalLM (which already uses FusedMoE with
shared-expert support) and replaces the text embedding / LM head with a shared-expert support) and replaces the text embedding / LM head with a
codec embedding so the talker operates over audio-codec tokens instead codec embedding so the talker operates over audio-codec tokens instead
of text tokens. of text tokens.
@@ -103,6 +103,9 @@ def prepare_hunyuan_fused_moe_runtime() -> None:
_ensure_forward_context_attr("flash_comm_v1_enabled", bool, False) _ensure_forward_context_attr("flash_comm_v1_enabled", bool, False)
# NOTE: As of v0.20,0 vLLM has folded SharedFusedMoE -> FusedMoE and removed the class.
# Since AscendSharedFusedMoE is a subclass of SharedFusedMoE, we should be careful
# to ensure that this is updated correctly.
class AscendHunyuanFusedMoE(AscendSharedFusedMoE): class AscendHunyuanFusedMoE(AscendSharedFusedMoE):
def __init__(self, *, prefix: str = "", **kwargs: Any) -> None: def __init__(self, *, prefix: str = "", **kwargs: Any) -> None:
super().__init__(prefix=prefix, **kwargs) super().__init__(prefix=prefix, **kwargs)