Compare commits

..
Author SHA1 Message Date
Hongsheng Liu cf301fe75b Revert "[Quantization] Enable FP8 online quantization for Z-image text encode…"
This reverts commit ac66282746.
2026-04-30 15:05:26 +08:00
45 changed files with 152 additions and 514 deletions

No files matched your search

+1 -1
View File
@@ -10,7 +10,7 @@ steps:
DOCKER_BUILDKIT: "1" DOCKER_BUILDKIT: "1"
# Buildkite will automatically replace this with the actual commit hash # Buildkite will automatically replace this with the actual commit hash
VLLM_IMAGE_TAG: "${BUILDKITE_COMMIT}" VLLM_IMAGE_TAG: "${BUILDKITE_COMMIT}"
VLLM_VERSION: "v0.20.0" VLLM_VERSION: "v0.19.0"
priority: 100 priority: 100
timeout_in_minutes: 60 timeout_in_minutes: 60
soft_fail: true soft_fail: true
+1 -3
View File
@@ -7,9 +7,7 @@ steps:
grade: Blocking grade: Blocking
commands: commands:
- export VLLM_ROCM_USE_AITER=0 - export VLLM_ROCM_USE_AITER=0
# ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM - "timeout 20m pytest -v -s -m 'core_model and cpu' --cov=vllm_omni --cov-branch --cov-report=term-missing --cov-report=html --cov-report=xml"
# that is not supported on CPU
- "timeout 20m pytest -v -s -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-branch --cov-report=term-missing --cov-report=html --cov-report=xml"
- label: "Diffusion Model Test" - label: "Diffusion Model Test"
agent_pool: mi325_2 agent_pool: mi325_2
+1 -3
View File
@@ -7,9 +7,7 @@ steps:
grade: Blocking grade: Blocking
commands: commands:
- export VLLM_ROCM_USE_AITER=0 - export VLLM_ROCM_USE_AITER=0
# ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM - "timeout 20m pytest -v -s -m 'core_model and cpu' --cov=vllm_omni --cov-branch --cov-report=term-missing --cov-report=html --cov-report=xml"
# that is not supported on CPU
- "timeout 20m pytest -vvvv -s -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-branch --cov-report=term-missing --cov-report=html --cov-report=xml"
- label: "Voxtral TTS CUDA Unit Test" - label: "Voxtral TTS CUDA Unit Test"
agent_pool: mi325_1 agent_pool: mi325_1
-223
View File
@@ -1,223 +0,0 @@
pull_request_rules:
- name: label-documentation
description: Automatically apply documentation label
conditions:
- label != stale
- or:
- files~=^[^/]+\.md$
- files~=^docs/
- files~=^examples/
actions:
label:
add:
- documentation
comment:
message: "Documentation preview: https://vllm--{{number}}.org.readthedocs.build/en/{{number}}/"
- name: comment-pre-commit-failure
description: Comment on PR when pre-commit check fails
conditions:
- status-failure=pre-commit
- -closed
- -draft
actions:
comment:
message: |
Hi @{{author}}, the pre-commit checks have failed. Please run:
```bash
uv pip install pre-commit
pre-commit install
pre-commit run --all-files
```
Then, commit the changes and push to your branch.
For future commits, `pre-commit` will run automatically on changed files before each commit.
> [!TIP]
> <details>
> <summary>Is <code>mypy</code> or <code>markdownlint</code> failing?</summary>
> <br/>
> <code>mypy</code> and <code>markdownlint</code> are run differently in CI. If the failure is related to either of these checks, please use the following commands to run them locally:
>
> ```bash
> # For mypy (substitute "3.10" with the failing version if needed)
> pre-commit run --hook-stage manual mypy-3.10
> # For markdownlint
> pre-commit run --hook-stage manual markdownlint
> ```
> </details>
- name: comment-dco-failure
description: Comment on PR when DCO check fails
conditions:
- status-failure=dco
- -closed
- -draft
actions:
comment:
message: |
Hi @{{author}}, the DCO check has failed. Please click on DCO in the Checks section for instructions on how to resolve this.
- name: label-ci-build
description: Automatically apply ci/build label
conditions:
- label != stale
- or:
- files~=^\.github/
- files~=\.buildkite/
- files~=^cmake/
- files=CMakeLists.txt
- files~=^docker/Dockerfile
- files~=^requirements.*\.txt
- files=setup.py
actions:
label:
add:
- ci/build
- name: label-frontend
description: Automatically apply frontend label
conditions:
- label != stale
- files~=^vllm_omni/entrypoints/
actions:
label:
add:
- frontend
- name: label-diffusion
description: Automatically apply diffusion label
conditions:
- label != stale
- files~=^vllm_omni/diffusion/
actions:
label:
add:
- diffusion
- name: label-new-model
description: Automatically apply new-model label
conditions:
- label != stale
- or:
- files=vllm_omni/model_executor/models/registry.py
- files=vllm_omni/diffusion/registry.py
actions:
label:
add:
- new-model
- name: label-npu
description: Automatically apply npu label
conditions:
- label != stale
- or:
- files~=^vllm_omni/platforms/npu/
- title~=(?i)NPU
- title~=(?i)Ascend
actions:
label:
add:
- npu
assign:
users:
- "gcanlin"
- name: label-rocm
description: Automatically apply rocm label
conditions:
- label != stale
- or:
- files~=^vllm_omni/platforms/rocm/
- title~=(?i)AMD
- title~=(?i)ROCm
actions:
label:
add:
- rocm
assign:
users:
- "tjtanaa"
- name: label-xpu
description: Automatically apply xpu label
conditions:
- label != stale
- or:
- files~=^vllm_omni/platforms/xpu/
- title~=(?i)Intel
- title~=(?i)XPU
actions:
label:
add:
- xpu
assign:
users:
- "yma11"
- name: label-modelrunner
description: Automatically apply modelrunner label
conditions:
- label != stale
- or:
- files~=^vllm_omni/worker/
- title~=(?i)ModelRunner
- title~=(?i)worker
actions:
label:
add:
- ModelRunner
assign:
users:
- "gcanlin" # To check whether need to sync to NPU
- name: auto-rebase if approved, ready, and 30 commits behind main
conditions:
- base = main
- label=ready
- "#approved-reviews-by >= 1"
- "#commits-behind >= 30"
- -closed
- -draft
- -conflict
actions:
rebase: {}
- name: ping author on conflicts and add 'needs-rebase' label
conditions:
- label != stale
- conflict
- -closed
actions:
label:
add:
- needs-rebase
comment:
message: |
This pull request has merge conflicts that must be resolved before it can be
merged. Please rebase the PR, @{{author}}.
https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/working-with-forks/syncing-a-fork
- name: remove 'needs-rebase' label when conflict is resolved
conditions:
- -conflict
- -closed
actions:
label:
remove:
- needs-rebase
- name: label-bug
description: Automatically apply bug label
conditions:
- label != stale
- or:
- title~=(?i)\bbug\b
- title~=(?i)\bbugfix\b
actions:
label:
add:
- bug
+1 -1
View File
@@ -1,5 +1,5 @@
ARG VLLM_BASE_IMAGE=vllm/vllm-openai ARG VLLM_BASE_IMAGE=vllm/vllm-openai
ARG VLLM_BASE_TAG=v0.20.0 ARG VLLM_BASE_TAG=v0.20.0-cu130
FROM ${VLLM_BASE_IMAGE}:${VLLM_BASE_TAG} FROM ${VLLM_BASE_IMAGE}:${VLLM_BASE_TAG}
ARG APP_DIR=/workspace/vllm-omni ARG APP_DIR=/workspace/vllm-omni
WORKDIR ${APP_DIR} WORKDIR ${APP_DIR}
+1 -1
View File
@@ -1,4 +1,4 @@
ARG BASE_IMAGE=vllm/vllm-openai:v0.20.0 ARG BASE_IMAGE=vllm/vllm-openai:v0.19.0
FROM ${BASE_IMAGE} FROM ${BASE_IMAGE}
ARG COMMON_WORKDIR=/app ARG COMMON_WORKDIR=/app
+1 -1
View File
@@ -1,4 +1,4 @@
ARG BASE_IMAGE=vllm/vllm-openai-rocm:v0.20.0 ARG BASE_IMAGE=vllm/vllm-openai-rocm:v0.19.0
FROM ${BASE_IMAGE} AS base FROM ${BASE_IMAGE} AS base
# Declare a variable to know if we want to use the nightly build or the stable build. # Declare a variable to know if we want to use the nightly build or the stable build.
+1 -1
View File
@@ -74,7 +74,7 @@ ENV UV_EXTRA_INDEX_URL=${PIP_EXTRA_INDEX_URL}
ENV UV_INDEX_STRATEGY="unsafe-best-match" ENV UV_INDEX_STRATEGY="unsafe-best-match"
ENV UV_LINK_MODE="copy" ENV UV_LINK_MODE="copy"
ARG VLLM_VERSION=v0.20.0 ARG VLLM_VERSION=v0.19.0
RUN git clone -b ${VLLM_VERSION} https://github.com/vllm-project/vllm RUN git clone -b ${VLLM_VERSION} https://github.com/vllm-project/vllm
WORKDIR /workspace/vllm WORKDIR /workspace/vllm
@@ -13,7 +13,7 @@ vLLM-Omni current recommends the steps in under setup through Docker Images.
vLLM-Omni is built based on vLLM. Please install it with command below. vLLM-Omni is built based on vLLM. Please install it with command below.
```bash ```bash
uv pip install vllm==0.20.0+rocm721 --extra-index-url https://wheels.vllm.ai/rocm/0.20.0/rocm721 uv pip install vllm==0.20.0+rocm700 --extra-index-url https://wheels.vllm.ai/rocm/0.20.0/rocm700
``` ```
#### Installation of vLLM-Omni #### Installation of vLLM-Omni
@@ -37,10 +37,10 @@ uv pip install onnxruntime-rocm
If you do not need to modify source code of vLLM, you can directly install the stable 0.20.0 release version of the library If you do not need to modify source code of vLLM, you can directly install the stable 0.20.0 release version of the library
```bash ```bash
uv pip install vllm==0.20.0+rocm721 --extra-index-url https://wheels.vllm.ai/rocm/0.20.0/rocm721 uv pip install vllm==0.20.0+rocm700 --extra-index-url https://wheels.vllm.ai/rocm/0.20.0/rocm700
``` ```
The pre-built 0.20.0 vLLM wheel targets ROCm 7.2.1. If you need a different ROCm stack or want to reuse an existing PyTorch installation, build vLLM from source instead. The pre-built 0.20.0 vLLM wheel targets ROCm 7.0. If you need a different ROCm stack or want to reuse an existing PyTorch installation, build vLLM from source instead.
#### Installation of vLLM-Omni #### Installation of vLLM-Omni
Since vllm-omni is rapidly evolving, it's recommended to install it from source Since vllm-omni is rapidly evolving, it's recommended to install it from source
+1 -7
View File
@@ -22,7 +22,7 @@ source .venv/bin/activate
uv pip install vllm==0.20.0 --torch-backend=auto uv pip install vllm==0.20.0 --torch-backend=auto
# On ROCm # On ROCm
uv pip install vllm==0.20.0+rocm721 --extra-index-url https://wheels.vllm.ai/rocm/0.20.0/rocm721 uv pip install vllm==0.20.0+rocm700 --extra-index-url https://wheels.vllm.ai/rocm/0.20.0/rocm700
git clone https://github.com/vllm-project/vllm-omni.git git clone https://github.com/vllm-project/vllm-omni.git
cd vllm-omni cd vllm-omni
@@ -31,12 +31,6 @@ uv pip install -e .
For additional installation methods — please see the [installation guide](installation/README.md). For additional installation methods — please see the [installation guide](installation/README.md).
!!! note
It is important to install the same major & minor version of vLLM and vLLM Omni, otherwise things may not work as expected. If the versions are misaligned, you will see a warning when you import vLLM Omni.
If you are seeing strange behavior with the `vllm` command not handling the `--omni` flag correctly, you most likely have a version mismatch with vLLM < `0.20.0` and vLLM Omni `0.20.0`, as vLLM Omni no longer hijacks the vLLM entrypoint. Updating vLLM should resolve this issue.
## Offline Inference ## Offline Inference
Text-to-image generation quickstart with vLLM-Omni: Text-to-image generation quickstart with vLLM-Omni:
@@ -118,6 +118,9 @@ def parse_args() -> argparse.Namespace:
help='JSON profiler config for torch/cuda profiling, e.g. \'{"profiler":"torch","torch_profiler_dir":"./perf"}\'.', help='JSON profiler config for torch/cuda profiling, e.g. \'{"profiler":"torch","torch_profiler_dir":"./perf"}\'.',
) )
from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults
nullify_stage_engine_defaults(parser)
return parser.parse_args() return parser.parse_args()
@@ -196,6 +196,9 @@ def parse_args() -> argparse.Namespace:
parser.add_argument("--cfg-parallel-size", type=int, default=1, choices=[1, 2], help="CFG parallel size.") parser.add_argument("--cfg-parallel-size", type=int, default=1, choices=[1, 2], help="CFG parallel size.")
parser.add_argument("--tensor-parallel-size", type=int, default=1, help="Tensor parallelism size.") parser.add_argument("--tensor-parallel-size", type=int, default=1, help="Tensor parallelism size.")
from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults
nullify_stage_engine_defaults(parser)
return parser.parse_args() return parser.parse_args()
@@ -378,6 +378,9 @@ def parse_args() -> argparse.Namespace:
default=None, default=None,
help='JSON profiler config for torch/cuda profiling, e.g. \'{"profiler":"torch","torch_profiler_dir":"./perf"}\'.', help='JSON profiler config for torch/cuda profiling, e.g. \'{"profiler":"torch","torch_profiler_dir":"./perf"}\'.',
) )
from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults
nullify_stage_engine_defaults(parser)
return parser.parse_args() return parser.parse_args()
@@ -211,6 +211,9 @@ def parse_args() -> argparse.Namespace:
default=None, default=None,
help='JSON profiler config for torch/cuda profiling, e.g. \'{"profiler":"torch","torch_profiler_dir":"./perf"}\'.', help='JSON profiler config for torch/cuda profiling, e.g. \'{"profiler":"torch","torch_profiler_dir":"./perf"}\'.',
) )
from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults
nullify_stage_engine_defaults(parser)
return parser.parse_args() return parser.parse_args()
@@ -24,6 +24,9 @@ def parse_args():
parser.add_argument("--width", type=int, default=448, help="Video width.") parser.add_argument("--width", type=int, default=448, help="Video width.")
parser.add_argument("--num-inference-steps", type=int, default=8, help="Number of denoising steps.") parser.add_argument("--num-inference-steps", type=int, default=8, help="Number of denoising steps.")
parser.add_argument("--seed", type=int, default=52, help="Random seed for generation.") parser.add_argument("--seed", type=int, default=52, help="Random seed for generation.")
from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults
nullify_stage_engine_defaults(parser)
return parser.parse_args() return parser.parse_args()
@@ -9,7 +9,7 @@ from typing import Any
import torch import torch
from vllm_omni.diffusion.data import logger from vllm_omni.diffusion.data import DiffusionParallelConfig, logger
from vllm_omni.entrypoints.omni import Omni from vllm_omni.entrypoints.omni import Omni
from vllm_omni.inputs.data import OmniDiffusionSamplingParams from vllm_omni.inputs.data import OmniDiffusionSamplingParams
from vllm_omni.lora.request import LoRARequest from vllm_omni.lora.request import LoRARequest
@@ -342,6 +342,17 @@ def main():
# (e.g., QwenImagePipeline or FluxPipeline) # (e.g., QwenImagePipeline or FluxPipeline)
} }
# assert args.ring_degree == 1, "Ring attention is not supported yet"
parallel_config = DiffusionParallelConfig(
ulysses_degree=args.ulysses_degree,
ring_degree=args.ring_degree,
ulysses_mode=args.ulysses_mode,
cfg_parallel_size=args.cfg_parallel_size,
tensor_parallel_size=args.tensor_parallel_size,
vae_patch_parallel_size=args.vae_patch_parallel_size,
enable_expert_parallel=args.enable_expert_parallel,
)
profiler_enabled = args.profiler_config is not None profiler_enabled = args.profiler_config is not None
# Prepare LoRA kwargs for Omni initialization # Prepare LoRA kwargs for Omni initialization
@@ -377,13 +388,7 @@ def main():
"cache_backend": args.cache_backend, "cache_backend": args.cache_backend,
"cache_config": cache_config, "cache_config": cache_config,
"enable_cache_dit_summary": args.enable_cache_dit_summary, "enable_cache_dit_summary": args.enable_cache_dit_summary,
"ulysses_degree": args.ulysses_degree, "parallel_config": parallel_config,
"ring_degree": args.ring_degree,
"ulysses_mode": args.ulysses_mode,
"cfg_parallel_size": args.cfg_parallel_size,
"tensor_parallel_size": args.tensor_parallel_size,
"vae_patch_parallel_size": args.vae_patch_parallel_size,
"enable_expert_parallel": args.enable_expert_parallel,
"enforce_eager": args.enforce_eager, "enforce_eager": args.enforce_eager,
"enable_cpu_offload": args.enable_cpu_offload, "enable_cpu_offload": args.enable_cpu_offload,
"mode": "text-to-image", "mode": "text-to-image",
@@ -226,6 +226,9 @@ def parse_args() -> argparse.Namespace:
default=1, default=1,
help="Number of HSDP replica groups.", help="Number of HSDP replica groups.",
) )
from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults
nullify_stage_engine_defaults(parser)
return parser.parse_args() return parser.parse_args()
@@ -71,6 +71,9 @@ def parse_args() -> argparse.Namespace:
parser.add_argument("--ulysses-degree", type=int, default=1, help="Ulysses SP degree.") parser.add_argument("--ulysses-degree", type=int, default=1, help="Ulysses SP degree.")
parser.add_argument("--ring-degree", type=int, default=1, help="Ring attention degree.") parser.add_argument("--ring-degree", type=int, default=1, help="Ring attention degree.")
parser.add_argument("--cfg-parallel-size", type=int, default=1, choices=[1, 2], help="CFG parallel size.") parser.add_argument("--cfg-parallel-size", type=int, default=1, choices=[1, 2], help="CFG parallel size.")
from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults
nullify_stage_engine_defaults(parser)
return parser.parse_args() return parser.parse_args()
+2
View File
@@ -32,6 +32,7 @@ classifiers = [
# Dependencies are now managed dynamically via setup.py based on detected hardware platform. # Dependencies are now managed dynamically via setup.py based on detected hardware platform.
# This allows automatic installation of the correct platform-specific dependencies (CUDA/ROCm/CPU/XPU/NPU) # This allows automatic installation of the correct platform-specific dependencies (CUDA/ROCm/CPU/XPU/NPU)
# without requiring extras like [cuda]. See requirements/ directory for platform-specific dependencies. # without requiring extras like [cuda]. See requirements/ directory for platform-specific dependencies.
# Note: vllm is intentionally excluded due to entrypoints overwrite issue.
[project.optional-dependencies] [project.optional-dependencies]
@@ -89,6 +90,7 @@ Documentation = "https://vllm-omni.readthedocs.io"
"Bug Tracker" = "https://github.com/vllm-project/vllm-omni/issues" "Bug Tracker" = "https://github.com/vllm-project/vllm-omni/issues"
[project.scripts] [project.scripts]
vllm = "vllm_omni.entrypoints.cli.main:main"
vllm-omni = "vllm_omni.entrypoints.cli.main:main" vllm-omni = "vllm_omni.entrypoints.cli.main:main"
@@ -121,8 +121,7 @@ class TestTrajectoryRecording:
assert trajectory_latents is not None assert trajectory_latents is not None
assert trajectory_timesteps is not None assert trajectory_timesteps is not None
# initial latent + one per denoising step assert len(trajectory_latents) == EXPECTED_STEPS
assert len(trajectory_latents) == EXPECTED_STEPS + 1
assert len(trajectory_timesteps) == EXPECTED_STEPS assert len(trajectory_timesteps) == EXPECTED_STEPS
# log_probs is None without a scheduler (default ODE path) # log_probs is None without a scheduler (default ODE path)
assert trajectory_log_probs is None assert trajectory_log_probs is None
@@ -168,54 +167,6 @@ class TestTrajectoryRecording:
"Last trajectory latent should match the final output" "Last trajectory latent should match the final output"
) )
def test_initial_latent_matches_input_noise(self, bagel_and_args):
"""Regression: the first trajectory entry must be the initial noise (pre-denoising)."""
bagel, args = bagel_and_args
init_noise = args["packed_init_noises"].clone()
_, trajectory_latents, *_ = bagel.generate_image(**args, return_trajectory_latents=True)
assert torch.allclose(trajectory_latents[0], init_noise, atol=1e-6), (
"trajectory_latents[0] should be the initial noise before any denoising step"
)
def test_trajectory_timesteps_match_expected_schedule(self, bagel_and_args):
"""Regression: trajectory_timesteps must record raw timesteps[i], not timesteps[i] - dts[i]."""
bagel, args = bagel_and_args
# Recompute the expected timestep schedule (mirrors generate_image logic)
ts = torch.linspace(1, 0, args["num_timesteps"])
shift = args.get("timestep_shift", 1.0)
ts = shift * ts / (1 + (shift - 1) * ts)
expected_timesteps = ts[:-1] # last element is dropped
_, _, trajectory_timesteps, _ = bagel.generate_image(**args, return_trajectory_latents=True)
assert len(trajectory_timesteps) == len(expected_timesteps)
for i, (actual, expected) in enumerate(zip(trajectory_timesteps, expected_timesteps)):
assert abs(float(actual) - float(expected)) < 1e-6, (
f"Step {i}: trajectory_timestep={float(actual):.6f}, expected={float(expected):.6f}"
)
def test_latent_count_equals_timesteps_plus_one(self, bagel_and_args):
"""Regression: len(trajectory_latents) == len(timesteps) + 1 (initial + one per step)."""
bagel, args = bagel_and_args
# Compute the number of denoising steps the same way generate_image does
ts = torch.linspace(1, 0, args["num_timesteps"])
shift = args.get("timestep_shift", 1.0)
ts = shift * ts / (1 + (shift - 1) * ts)
num_steps = len(ts) - 1 # timesteps = ts[:-1]
_, trajectory_latents, trajectory_timesteps, _ = bagel.generate_image(**args, return_trajectory_latents=True)
assert len(trajectory_latents) == num_steps + 1, (
f"Expected {num_steps + 1} latents (initial + {num_steps} steps), got {len(trajectory_latents)}"
)
assert len(trajectory_timesteps) == num_steps, (
f"Expected {num_steps} timesteps, got {len(trajectory_timesteps)}"
)
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
# Mock scheduler for log-prob tests # Mock scheduler for log-prob tests
@@ -356,6 +356,7 @@ def test_mix_to_text_audio_001(omni_server, openai_client) -> None:
@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2) @hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
@pytest.mark.parametrize("omni_server", test_params, indirect=True) @pytest.mark.parametrize("omni_server", test_params, indirect=True)
@pytest.mark.skip(reason="issue: #2827")
def test_audio_in_video_001(omni_server, openai_client) -> None: def test_audio_in_video_001(omni_server, openai_client) -> None:
""" """
Input Modal: text + video (synthetic MP4 with embedded audio; ``use_audio_in_video`` uses audio from the video). Input Modal: text + video (synthetic MP4 with embedded audio; ``use_audio_in_video`` uses audio from the video).
@@ -375,7 +376,7 @@ def test_audio_in_video_001(omni_server, openai_client) -> None:
"messages": messages, "messages": messages,
"stream": False, "stream": False,
"use_audio_in_video": True, "use_audio_in_video": True,
"key_words": {"video": VIDEO_KEY}, "key_words": {"video": VIDEO_KEY, "audio": AUDIO_KEY + ["beep", "electronic"]},
} }
openai_client.send_omni_request(request_config) openai_client.send_omni_request(request_config)
@@ -98,7 +98,7 @@ def _get_diffusion_feature_cases():
], ],
), ),
id="layerwise_hsdp", id="layerwise_hsdp",
marks=[*FOUR_CARD_MARKS], marks=[*FOUR_CARD_MARKS, pytest.mark.skip(reason="issue #2435")],
), ),
] ]
@@ -4,7 +4,6 @@
import pytest import pytest
from vllm.utils.argparse_utils import FlexibleArgumentParser from vllm.utils.argparse_utils import FlexibleArgumentParser
from vllm_omni.config.stage_config import deploy_override_field_names
from vllm_omni.engine.async_omni_engine import AsyncOmniEngine from vllm_omni.engine.async_omni_engine import AsyncOmniEngine
from vllm_omni.entrypoints.cli.serve import OmniServeCommand, _create_default_diffusion_stage_cfg from vllm_omni.entrypoints.cli.serve import OmniServeCommand, _create_default_diffusion_stage_cfg
@@ -31,15 +30,6 @@ def test_default_stage_config_includes_cache_backend():
assert engine_args["model_stage"] == "diffusion" assert engine_args["model_stage"] == "diffusion"
def test_default_stage_config_ignores_none_deploy_overrides():
"""Ensure nullified deploy override defaults do not alter diffusion defaults."""
baseline = AsyncOmniEngine._create_default_diffusion_stage_cfg({})[0]
nullified_overrides = {name: None for name in deploy_override_field_names()}
stage_cfg = AsyncOmniEngine._create_default_diffusion_stage_cfg(nullified_overrides)[0]
assert stage_cfg == baseline
def test_default_cache_config_used_when_missing(): def test_default_cache_config_used_when_missing():
"""Ensure default cache_config is synthesized when only backend is given.""" """Ensure default cache_config is synthesized when only backend is given."""
stage_cfg = AsyncOmniEngine._create_default_diffusion_stage_cfg( stage_cfg = AsyncOmniEngine._create_default_diffusion_stage_cfg(
@@ -65,44 +55,6 @@ def test_default_stage_devices_from_sequence_parallel():
assert stage_cfg["runtime"]["devices"] == "0,1,2,3" assert stage_cfg["runtime"]["devices"] == "0,1,2,3"
def test_default_stage_config_uses_parallel_size_kwargs():
"""Ensure default diffusion parallel_config uses CLI/API parallel sizes."""
stage_cfg = AsyncOmniEngine._create_default_diffusion_stage_cfg(
{
"pipeline_parallel_size": 2,
"data_parallel_size": 3,
"tensor_parallel_size": 4,
"enable_expert_parallel": True,
}
)[0]
parallel_config = stage_cfg["engine_args"]["parallel_config"]
assert parallel_config.pipeline_parallel_size == 2
assert parallel_config.data_parallel_size == 3
assert parallel_config.tensor_parallel_size == 4
assert parallel_config.enable_expert_parallel is True
def test_default_stage_config_defaults_nullified_parallel_size_kwargs():
"""Ensure nullified diffusion parallel-size kwargs fall back to defaults."""
stage_cfg = AsyncOmniEngine._create_default_diffusion_stage_cfg(
{
"pipeline_parallel_size": None,
"data_parallel_size": None,
"tensor_parallel_size": None,
"enable_expert_parallel": None,
"enforce_eager": None,
}
)[0]
parallel_config = stage_cfg["engine_args"]["parallel_config"]
assert parallel_config.pipeline_parallel_size == 1
assert parallel_config.data_parallel_size == 1
assert parallel_config.tensor_parallel_size == 1
assert parallel_config.enable_expert_parallel is False
assert stage_cfg["engine_args"]["enforce_eager"] is False
def test_default_stage_config_propagates_ulysses_mode(): def test_default_stage_config_propagates_ulysses_mode():
"""Ensure UAA mode survives default diffusion-stage creation.""" """Ensure UAA mode survives default diffusion-stage creation."""
stage_cfg = AsyncOmniEngine._create_default_diffusion_stage_cfg( stage_cfg = AsyncOmniEngine._create_default_diffusion_stage_cfg(
@@ -26,9 +26,27 @@ def _skip_readme_snippet(language: str, code: str, h2_title: str) -> tuple[bool,
README_SNIPPETS = ReadmeSnippet.extract_readme_snippets(README_PATH, skipif=_skip_readme_snippet) README_SNIPPETS = ReadmeSnippet.extract_readme_snippets(README_PATH, skipif=_skip_readme_snippet)
# Track: https://github.com/vllm-project/vllm-omni/issues/3123 — CLI example subprocess exit 1 in CI; re-enable when fixed.
_SKIP_T2I_SNIPPET_IDS = frozenset(
{
"quick_start_002",
"more_cli_examples_001",
"more_cli_examples_002",
"more_cli_examples_003",
"more_cli_examples_007",
"more_cli_examples_008",
}
)
_SKIP_T2I_REASON = (
"Skipped: text_to_image README examples failing in CI (subprocess exit 1). "
"See https://github.com/vllm-project/vllm-omni/issues/3123"
)
@pytest.mark.parametrize("snippet", README_SNIPPETS, ids=lambda snippet: snippet.test_id) @pytest.mark.parametrize("snippet", README_SNIPPETS, ids=lambda snippet: snippet.test_id)
def test_text_to_image(snippet: ReadmeSnippet, example_runner: ExampleRunner): def test_text_to_image(snippet: ReadmeSnippet, example_runner: ExampleRunner):
if snippet.test_id in _SKIP_T2I_SNIPPET_IDS:
pytest.skip(_SKIP_T2I_REASON)
should_skip, reason = snippet.skip should_skip, reason = snippet.skip
if should_skip: if should_skip:
pytest.skip(reason) pytest.skip(reason)
+9 -3
View File
@@ -3,7 +3,14 @@
"""Pytest marks and decorators for hardware / resource selection (CUDA, ROCm, …).""" """Pytest marks and decorators for hardware / resource selection (CUDA, ROCm, …)."""
import pytest import pytest
from vllm.platforms import current_platform
try:
from vllm.utils.torch_utils import cuda_device_count_stateless
except ImportError:
import torch
def cuda_device_count_stateless() -> int:
return torch.cuda.device_count()
# Re-exported from tests.helpers.env (GPU wait + DeviceMemoryMonitor). # Re-exported from tests.helpers.env (GPU wait + DeviceMemoryMonitor).
@@ -20,9 +27,8 @@ def cuda_marks(*, res: str, num_cards: int):
if num_cards == 1: if num_cards == 1:
return marks return marks
test_distributed = pytest.mark.distributed_cuda(num_cards=num_cards) test_distributed = pytest.mark.distributed_cuda(num_cards=num_cards)
test_skipif = pytest.mark.skipif_cuda( test_skipif = pytest.mark.skipif_cuda(
not current_platform.is_cuda() or (current_platform.device_count() < num_cards), cuda_device_count_stateless() < num_cards,
reason=f"Need at least {num_cards} CUDA GPUs to run the test.", reason=f"Need at least {num_cards} CUDA GPUs to run the test.",
) )
return marks + [test_distributed, test_skipif] return marks + [test_distributed, test_skipif]
-1
View File
@@ -494,7 +494,6 @@ _CI_OVERLAYS: dict[str, dict[str, Any]] = {
"max_num_seqs": 1, "max_num_seqs": 1,
"gpu_memory_utilization": 0.9, "gpu_memory_utilization": 0.9,
"enforce_eager": True, "enforce_eager": True,
"enable_prefix_caching": False,
"max_num_batched_tokens": 16384, "max_num_batched_tokens": 16384,
"max_model_len": 16384, "max_model_len": 16384,
"skip_mm_profiling": True, "skip_mm_profiling": True,
+1 -1
View File
@@ -369,8 +369,8 @@ def _build_full_serve_parser():
def test_nullify_stage_engine_defaults_resets_inherited_defaults(): def test_nullify_stage_engine_defaults_resets_inherited_defaults():
import argparse import argparse
from vllm_omni.config.stage_config import deploy_override_field_names
from vllm_omni.engine.arg_utils import ( from vllm_omni.engine.arg_utils import (
deploy_override_field_names,
nullify_stage_engine_defaults, nullify_stage_engine_defaults,
) )
+2 -67
View File
@@ -93,7 +93,6 @@ class TestStageConfig:
assert omega_config.engine_args.worker_type == "ar" assert omega_config.engine_args.worker_type == "ar"
assert omega_config.final_output is True assert omega_config.final_output is True
assert omega_config.final_output_type == "text" assert omega_config.final_output_type == "text"
assert "max_num_seqs" not in omega_config.engine_args
# Legacy field name for backward compatibility # Legacy field name for backward compatibility
assert omega_config.engine_input_source == [] assert omega_config.engine_input_source == []
@@ -147,24 +146,6 @@ class TestStageConfig:
omega_config = config.to_omegaconf() omega_config = config.to_omegaconf()
assert omega_config.engine_args.max_num_seqs == 32 assert omega_config.engine_args.max_num_seqs == 32
def test_to_omegaconf_omits_none_deploy_overrides_for_engine_args(self):
"""None deploy overrides must fall through to EngineArgs defaults."""
from vllm_omni.config.stage_config import deploy_override_field_names
config = StageConfig(
stage_id=0,
model_stage="thinker",
runtime_overrides={name: None for name in deploy_override_field_names()},
)
omega_config = config.to_omegaconf()
engine_args = dict(omega_config.engine_args)
assert "devices" not in engine_args
assert "max_batch_size" not in engine_args
for name in deploy_override_field_names() - {"devices"}:
assert name not in engine_args
class TestModelPipeline: class TestModelPipeline:
"""Tests for ModelPipeline class.""" """Tests for ModelPipeline class."""
@@ -821,40 +802,6 @@ class TestPipelineRegistry:
class TestDeployConfigLoading: class TestDeployConfigLoading:
def test_deploy_override_fields_include_deploy_schema_fields(self):
from vllm_omni.config.stage_config import deploy_override_field_names
expected_fields = {
"async_chunk",
"async_scheduling",
"config_format",
"data_parallel_size",
"devices",
"disable_hybrid_kv_cache_manager",
"distributed_executor_backend",
"dtype",
"enable_chunked_prefill",
"enable_flashinfer_autotune",
"enable_prefix_caching",
"enforce_eager",
"gpu_memory_utilization",
"load_format",
"max_model_len",
"max_num_batched_tokens",
"max_num_seqs",
"mm_processor_cache_gb",
"pipeline_parallel_size",
"profiler_config",
"quantization",
"skip_mm_profiling",
"subtalker_sampling_params",
"tensor_parallel_size",
"tokenizer_mode",
"trust_remote_code",
}
assert expected_fields == deploy_override_field_names()
def test_load_deploy_config(self): def test_load_deploy_config(self):
from pathlib import Path from pathlib import Path
@@ -870,17 +817,6 @@ class TestDeployConfigLoading:
assert deploy.connectors is not None assert deploy.connectors is not None
assert deploy.platforms is not None assert deploy.platforms is not None
voxtral_path = Path(__file__).parent.parent / "vllm_omni" / "deploy" / "voxtral_tts.yaml"
if voxtral_path.exists():
voxtral_deploy = load_deploy_config(voxtral_path)
assert voxtral_deploy.stages[0].config_format == "mistral"
assert voxtral_deploy.stages[0].load_format == "mistral"
assert voxtral_deploy.stages[0].tokenizer_mode == "mistral"
assert not any(
name in voxtral_deploy.stages[0].engine_extras
for name in ("config_format", "load_format", "tokenizer_mode")
)
def test_merge_pipeline_deploy(self): def test_merge_pipeline_deploy(self):
from pathlib import Path from pathlib import Path
@@ -1075,8 +1011,7 @@ class TestBaseConfigInheritance:
deploy = load_deploy_config(ci_path) deploy = load_deploy_config(ci_path)
assert len(deploy.stages) == 3 assert len(deploy.stages) == 3
# CI overrides # CI overrides
assert deploy.stages[0].load_format == "dummy" assert deploy.stages[0].engine_extras.get("load_format") == "dummy"
assert "load_format" not in deploy.stages[0].engine_extras
assert deploy.stages[0].max_num_seqs == 5 assert deploy.stages[0].max_num_seqs == 5
# Inherited from base # Inherited from base
assert deploy.stages[0].gpu_memory_utilization == 0.9 assert deploy.stages[0].gpu_memory_utilization == 0.9
@@ -1281,7 +1216,7 @@ class TestSentinelDefaultPrecedence:
def test_none_value_skipped_yaml_wins(self): def test_none_value_skipped_yaml_wins(self):
stages = self._stages({"max_num_seqs": None}) stages = self._stages({"max_num_seqs": None})
assert stages[2].runtime_overrides.get("max_num_seqs") is None assert stages[2].runtime_overrides.get("max_num_seqs") is None
assert "max_num_seqs" not in stages[2].yaml_engine_args assert stages[2].yaml_engine_args.get("max_num_seqs") == 1
def test_empty_kwargs_yaml_only(self): def test_empty_kwargs_yaml_only(self):
stages = self._stages({}) stages = self._stages({})
+16 -38
View File
@@ -402,11 +402,11 @@ class StageDeployConfig:
""" """
stage_id: int stage_id: int
max_num_seqs: int | None = None max_num_seqs: int = 64
gpu_memory_utilization: float | None = None gpu_memory_utilization: float = 0.9
tensor_parallel_size: int | None = None tensor_parallel_size: int = 1
enforce_eager: bool | None = None enforce_eager: bool = False
max_num_batched_tokens: int | None = None max_num_batched_tokens: int = 32768
max_model_len: int | None = None max_model_len: int | None = None
async_scheduling: bool | None = None async_scheduling: bool | None = None
devices: str = "0" devices: str = "0"
@@ -414,14 +414,6 @@ class StageDeployConfig:
input_connectors: dict[str, str] | None = None input_connectors: dict[str, str] | None = None
default_sampling_params: dict[str, Any] | None = None default_sampling_params: dict[str, Any] | None = None
subtalker_sampling_params: dict[str, Any] | None = None subtalker_sampling_params: dict[str, Any] | None = None
profiler_config: dict[str, Any] | None = None
disable_hybrid_kv_cache_manager: bool | None = None
mm_processor_cache_gb: float | None = None
skip_mm_profiling: bool | None = None
enable_flashinfer_autotune: bool | None = None
config_format: str | None = None
load_format: str | None = None
tokenizer_mode: str | None = None
engine_extras: dict[str, Any] = field(default_factory=dict) engine_extras: dict[str, Any] = field(default_factory=dict)
@@ -446,14 +438,14 @@ class DeployConfig:
pipeline: str | None = None pipeline: str | None = None
# === Pipeline-wide engine settings (applied uniformly to every stage) === # === Pipeline-wide engine settings (applied uniformly to every stage) ===
trust_remote_code: bool | None = None trust_remote_code: bool = True
distributed_executor_backend: str | None = None distributed_executor_backend: str | None = None
dtype: str | None = None dtype: str | None = None
quantization: str | None = None quantization: str | None = None
enable_prefix_caching: bool | None = None enable_prefix_caching: bool = False
enable_chunked_prefill: bool | None = None enable_chunked_prefill: bool | None = None
data_parallel_size: int | None = None data_parallel_size: int = 1
pipeline_parallel_size: int | None = None pipeline_parallel_size: int = 1
_STAGE_NON_ENGINE_KEYS = frozenset( _STAGE_NON_ENGINE_KEYS = frozenset(
@@ -697,18 +689,6 @@ _PIPELINE_WIDE_ENGINE_FIELDS: tuple[str, ...] = (
) )
def deploy_override_field_names() -> frozenset[str]:
"""Return deploy-schema fields whose CLI defaults must not override YAML."""
return (
frozenset(_STAGE_DEPLOY_FIELDS)
| frozenset(_PIPELINE_WIDE_ENGINE_FIELDS)
| {
"async_chunk",
"devices",
}
)
def _build_engine_args( def _build_engine_args(
ps: StagePipelineConfig, ps: StagePipelineConfig,
ds: StageDeployConfig | None, ds: StageDeployConfig | None,
@@ -881,15 +861,13 @@ class StageConfig:
# CLI overrides take precedence over YAML defaults # CLI overrides take precedence over YAML defaults
for key, value in self.runtime_overrides.items(): for key, value in self.runtime_overrides.items():
if value is None:
continue
if key not in ("devices", "max_batch_size"): if key not in ("devices", "max_batch_size"):
engine_args[key] = value engine_args[key] = value
# Build runtime config from YAML defaults + CLI overrides # Build runtime config from YAML defaults + CLI overrides
runtime: dict[str, Any] = dict(self.yaml_runtime) runtime: dict[str, Any] = dict(self.yaml_runtime)
runtime.setdefault("process", True) runtime.setdefault("process", True)
if self.runtime_overrides.get("devices") is not None: if "devices" in self.runtime_overrides:
runtime["devices"] = self.runtime_overrides["devices"] runtime["devices"] = self.runtime_overrides["devices"]
# Legacy compat: migrate runtime.max_batch_size → engine_args.max_num_seqs # Legacy compat: migrate runtime.max_batch_size → engine_args.max_num_seqs
@@ -905,6 +883,8 @@ class StageConfig:
effective_mbs = int(cli_mbs or legacy_mbs or 1) effective_mbs = int(cli_mbs or legacy_mbs or 1)
engine_args.setdefault("max_num_seqs", effective_mbs) engine_args.setdefault("max_num_seqs", effective_mbs)
engine_args.setdefault("max_num_seqs", 1)
# Build full config dict # Build full config dict
config_dict: dict[str, Any] = { config_dict: dict[str, Any] = {
"stage_id": self.stage_id, "stage_id": self.stage_id,
@@ -1029,8 +1009,6 @@ class StageConfigFactory:
cli_overrides = {} cli_overrides = {}
trust_remote_code = cli_overrides.get("trust_remote_code", True) trust_remote_code = cli_overrides.get("trust_remote_code", True)
if trust_remote_code is None:
trust_remote_code = False
# --- New path: check pipeline registry by model_type first --- # --- New path: check pipeline registry by model_type first ---
model_type, hf_config = cls._auto_detect_model_type(model, trust_remote_code=trust_remote_code) model_type, hf_config = cls._auto_detect_model_type(model, trust_remote_code=trust_remote_code)
@@ -1374,8 +1352,8 @@ class StageConfigFactory:
hf_config = get_config(model, trust_remote_code=trust_remote_code) hf_config = get_config(model, trust_remote_code=trust_remote_code)
return hf_config.model_type, hf_config return hf_config.model_type, hf_config
except Exception as e: except Exception:
logger.debug(f"`get_config` failed for {e}; Falling back to raw config.json path") pass
# Fallback: read config.json directly for custom model types that # Fallback: read config.json directly for custom model types that
# are not registered with transformers (e.g. qwen3_tts). # are not registered with transformers (e.g. qwen3_tts).
@@ -1412,8 +1390,8 @@ class StageConfigFactory:
class_name, class_name,
) )
return pipeline_cfg.model_type, None return pipeline_cfg.model_type, None
except Exception as e: except Exception:
logger.debug(f"Failed to detect model type for diffusers-style models: {e}") pass
# Final fallback: some models (e.g. CosyVoice3) ship an empty # Final fallback: some models (e.g. CosyVoice3) ship an empty
# config.json and rely on naming conventions. Match the model path # config.json and rely on naming conventions. Match the model path
+2 -2
View File
@@ -40,7 +40,7 @@ stages:
# near-identity repetition penalty forces vLLM to track # near-identity repetition penalty forces vLLM to track
# output_token_ids for RAS (stop-token logit logsumexp). # output_token_ids for RAS (stop-token logit logsumexp).
repetition_penalty: 1.0001 repetition_penalty: 1.0001
enable_prefix_caching: false disable_hybrid_kv_cache_manager: true
mm_processor_cache_gb: 0 mm_processor_cache_gb: 0
skip_mm_profiling: true skip_mm_profiling: true
@@ -54,5 +54,5 @@ stages:
from_stage_0: connector_of_shared_memory from_stage_0: connector_of_shared_memory
default_sampling_params: default_sampling_params:
max_tokens: 2048 max_tokens: 2048
enable_prefix_caching: false disable_hybrid_kv_cache_manager: true
skip_mm_profiling: true skip_mm_profiling: true
-1
View File
@@ -42,7 +42,6 @@ stages:
max_num_seqs: 1 max_num_seqs: 1
gpu_memory_utilization: 0.2 gpu_memory_utilization: 0.2
enforce_eager: true enforce_eager: true
enable_prefix_caching: false
async_scheduling: false async_scheduling: false
max_num_batched_tokens: 8192 max_num_batched_tokens: 8192
max_model_len: 8192 max_model_len: 8192
-2
View File
@@ -22,7 +22,6 @@ stages:
max_num_seqs: 1 max_num_seqs: 1
gpu_memory_utilization: 0.8 gpu_memory_utilization: 0.8
enforce_eager: true enforce_eager: true
enable_prefix_caching: false
mm_processor_cache_gb: 0 mm_processor_cache_gb: 0
devices: "0" devices: "0"
default_sampling_params: default_sampling_params:
@@ -50,7 +49,6 @@ stages:
max_num_seqs: 1 max_num_seqs: 1
gpu_memory_utilization: 0.15 gpu_memory_utilization: 0.15
enforce_eager: true enforce_eager: true
enable_prefix_caching: false
enable_flashinfer_autotune: false enable_flashinfer_autotune: false
async_scheduling: false async_scheduling: false
devices: "0" devices: "0"
-2
View File
@@ -23,7 +23,6 @@ connectors:
stages: stages:
- stage_id: 0 - stage_id: 0
gpu_memory_utilization: 0.9 gpu_memory_utilization: 0.9
enable_prefix_caching: false
devices: "0" devices: "0"
default_sampling_params: default_sampling_params:
temperature: 0.4 temperature: 0.4
@@ -48,7 +47,6 @@ stages:
- stage_id: 2 - stage_id: 2
gpu_memory_utilization: 0.1 gpu_memory_utilization: 0.1
enforce_eager: true enforce_eager: true
enable_prefix_caching: false
async_scheduling: false async_scheduling: false
max_num_batched_tokens: 51200 max_num_batched_tokens: 51200
devices: "1" devices: "1"
-1
View File
@@ -53,7 +53,6 @@ stages:
max_num_seqs: 1 max_num_seqs: 1
gpu_memory_utilization: 0.3 gpu_memory_utilization: 0.3
enforce_eager: true enforce_eager: true
enable_prefix_caching: false
async_scheduling: true async_scheduling: true
# Must be divisible by num_code_groups and cover (left_context + chunk). # Must be divisible by num_code_groups and cover (left_context + chunk).
# Prefill length is Q * num_frames (e.g. 16 * 2148 = 34368); keep # Prefill length is Q * num_frames (e.g. 16 * 2148 = 34368); keep
-1
View File
@@ -20,7 +20,6 @@ stages:
max_num_batched_tokens: 4096 max_num_batched_tokens: 4096
max_model_len: 4096 max_model_len: 4096
devices: "0" devices: "0"
trust_remote_code: true
default_sampling_params: default_sampling_params:
temperature: 0.0 temperature: 0.0
top_p: 1.0 top_p: 1.0
-2
View File
@@ -24,7 +24,6 @@ stages:
max_num_seqs: 32 max_num_seqs: 32
gpu_memory_utilization: 0.8 gpu_memory_utilization: 0.8
enforce_eager: false enforce_eager: false
enable_prefix_caching: false
async_scheduling: true async_scheduling: true
max_model_len: 4096 max_model_len: 4096
devices: "0" devices: "0"
@@ -49,7 +48,6 @@ stages:
max_num_seqs: 32 max_num_seqs: 32
gpu_memory_utilization: 0.1 gpu_memory_utilization: 0.1
enforce_eager: true enforce_eager: true
enable_prefix_caching: false
async_scheduling: false async_scheduling: false
max_num_batched_tokens: 65536 max_num_batched_tokens: 65536
max_model_len: 65536 max_model_len: 65536
@@ -385,9 +385,9 @@ class MultiprocDiffusionExecutor(DiffusionExecutor):
raise raise
def check_health(self) -> None: def check_health(self) -> None:
self._ensure_open()
if self.is_failed: if self.is_failed:
raise EngineDeadError() raise EngineDeadError()
self._ensure_open()
for p in self._processes: for p in self._processes:
if not p.is_alive(): if not p.is_alive():
self.is_failed = True self.is_failed = True
@@ -15,7 +15,6 @@ from typing import TYPE_CHECKING, Any
import torch import torch
from PIL import Image from PIL import Image
from vllm.logger import init_logger from vllm.logger import init_logger
from vllm.v1.engine.exceptions import EngineDeadError
from vllm_omni.diffusion.data import DiffusionRequestAbortedError from vllm_omni.diffusion.data import DiffusionRequestAbortedError
from vllm_omni.diffusion.diffusion_engine import DiffusionEngine from vllm_omni.diffusion.diffusion_engine import DiffusionEngine
@@ -330,12 +329,6 @@ class InlineStageDiffusionClient:
None, None,
) )
def check_health(self) -> None:
"""Check if the inline diffusion engine and its workers are healthy."""
if self._shutting_down:
raise EngineDeadError("InlineStageDiffusionClient is shutting down")
self._engine.executor.check_health()
def shutdown(self) -> None: def shutdown(self) -> None:
self._shutting_down = True self._shutting_down = True
@@ -1842,8 +1842,6 @@ class Bagel(nn.Module):
# ── SP + CFG: sequential single-branch forwards ── # ── SP + CFG: sequential single-branch forwards ──
use_sp = self._sp_size > 1 use_sp = self._sp_size > 1
if use_sp and use_cfg_text: if use_sp and use_cfg_text:
if return_trajectory_latents and len(timesteps) > 0:
trajectory_latents.append(x_t.clone())
for i, t in enumerate(timesteps): for i, t in enumerate(timesteps):
timestep = torch.tensor([t] * x_t.shape[0], device=x_t.device) timestep = torch.tensor([t] * x_t.shape[0], device=x_t.device)
in_cfg_window = t > cfg_interval[0] and t <= cfg_interval[1] in_cfg_window = t > cfg_interval[0] and t <= cfg_interval[1]
@@ -1907,15 +1905,13 @@ class Bagel(nn.Module):
x_t = x_t - v_t.to(x_t.device) * dts[i] x_t = x_t - v_t.to(x_t.device) * dts[i]
if return_trajectory_latents: if return_trajectory_latents:
trajectory_latents.append(x_t.clone()) trajectory_latents.append(x_t.clone())
trajectory_timesteps.append(timesteps[i]) trajectory_timesteps.append(timesteps[i] - dts[i])
unpacked_latent = x_t.split((packed_seqlens - 2).tolist()) unpacked_latent = x_t.split((packed_seqlens - 2).tolist())
return unpacked_latent, trajectory_latents, trajectory_timesteps, trajectory_log_probs return unpacked_latent, trajectory_latents, trajectory_timesteps, trajectory_log_probs
# ── SP without CFG: direct single-branch loop ── # ── SP without CFG: direct single-branch loop ──
if use_sp: if use_sp:
if return_trajectory_latents and len(timesteps) > 0:
trajectory_latents.append(x_t.clone())
for i, t in enumerate(timesteps): for i, t in enumerate(timesteps):
timestep = torch.tensor([t] * x_t.shape[0], device=x_t.device) timestep = torch.tensor([t] * x_t.shape[0], device=x_t.device)
v_t = self.forward_single_branch( v_t = self.forward_single_branch(
@@ -1942,7 +1938,7 @@ class Bagel(nn.Module):
x_t = x_t - v_t.to(x_t.device) * dts[i] x_t = x_t - v_t.to(x_t.device) * dts[i]
if return_trajectory_latents: if return_trajectory_latents:
trajectory_latents.append(x_t.clone()) trajectory_latents.append(x_t.clone())
trajectory_timesteps.append(timesteps[i]) trajectory_timesteps.append(timesteps[i] - dts[i])
unpacked_latent = x_t.split((packed_seqlens - 2).tolist()) unpacked_latent = x_t.split((packed_seqlens - 2).tolist())
return unpacked_latent, trajectory_latents, trajectory_timesteps, trajectory_log_probs return unpacked_latent, trajectory_latents, trajectory_timesteps, trajectory_log_probs
@@ -2003,9 +1999,6 @@ class Bagel(nn.Module):
"merged_cache": self._merge_naive_caches(branches_cache), "merged_cache": self._merge_naive_caches(branches_cache),
} }
if return_trajectory_latents and len(timesteps) > 0:
trajectory_latents.append(x_t.clone())
for i, t in enumerate(timesteps): for i, t in enumerate(timesteps):
timestep = torch.tensor([t] * x_t.shape[0], device=x_t.device) timestep = torch.tensor([t] * x_t.shape[0], device=x_t.device)
if t > cfg_interval[0] and t <= cfg_interval[1]: if t > cfg_interval[0] and t <= cfg_interval[1]:
@@ -2043,7 +2036,7 @@ class Bagel(nn.Module):
x_t = x_t - v_t.to(x_t.device) * dts[i] # velocity pointing from data to noise x_t = x_t - v_t.to(x_t.device) * dts[i] # velocity pointing from data to noise
if return_trajectory_latents: if return_trajectory_latents:
trajectory_latents.append(x_t.clone()) trajectory_latents.append(x_t.clone())
trajectory_timesteps.append(timesteps[i]) trajectory_timesteps.append(timesteps[i] - dts[i])
unpacked_latent = x_t.split((packed_seqlens - 2).tolist()) unpacked_latent = x_t.split((packed_seqlens - 2).tolist())
return unpacked_latent, trajectory_latents, trajectory_timesteps, trajectory_log_probs return unpacked_latent, trajectory_latents, trajectory_timesteps, trajectory_log_probs
@@ -2144,9 +2137,6 @@ class Bagel(nn.Module):
) )
_sched_kw = scheduler_kwargs or {} _sched_kw = scheduler_kwargs or {}
if return_trajectory_latents and len(timesteps) > 0:
trajectory_latents.append(x_t.clone())
for i, t in enumerate(timesteps): for i, t in enumerate(timesteps):
timestep = torch.tensor([t] * x_t.shape[0], device=x_t.device) timestep = torch.tensor([t] * x_t.shape[0], device=x_t.device)
use_cfg_this_step = t > cfg_interval[0] and t <= cfg_interval[1] and cfg_text_scale > 1.0 use_cfg_this_step = t > cfg_interval[0] and t <= cfg_interval[1] and cfg_text_scale > 1.0
@@ -2204,7 +2194,7 @@ class Bagel(nn.Module):
x_t = x_t - v_t.to(x_t.device) * dts[i] x_t = x_t - v_t.to(x_t.device) * dts[i]
if return_trajectory_latents: if return_trajectory_latents:
trajectory_latents.append(x_t.clone()) trajectory_latents.append(x_t.clone())
trajectory_timesteps.append(timesteps[i]) trajectory_timesteps.append(timesteps[i] - dts[i])
unpacked_latent = x_t.split((packed_seqlens - 2).tolist()) unpacked_latent = x_t.split((packed_seqlens - 2).tolist())
return unpacked_latent, trajectory_latents, trajectory_timesteps, trajectory_log_probs return unpacked_latent, trajectory_latents, trajectory_timesteps, trajectory_log_probs
+38 -2
View File
@@ -456,12 +456,50 @@ SHARED_FIELDS: frozenset[str] = frozenset(
} }
) )
_DEPLOY_ENGINE_ARG_OVERRIDE_FIELDS: frozenset[str] = frozenset(
{
# Capacity / scheduling.
"async_scheduling",
"max_model_len",
"max_num_batched_tokens",
"max_num_seqs",
# Memory / parallelism.
"data_parallel_size",
"gpu_memory_utilization",
"pipeline_parallel_size",
"tensor_parallel_size",
# Execution / loading.
"enforce_eager",
"distributed_executor_backend",
"dtype",
"quantization",
"trust_remote_code",
# Caching / chunking.
"async_chunk",
"enable_prefix_caching",
"enable_chunked_prefill",
# Model-specific engine extras.
"subtalker_sampling_params",
}
)
_DEPLOY_RUNTIME_OVERRIDE_FIELDS: frozenset[str] = frozenset(
{
"devices",
}
)
def orchestrator_field_names() -> frozenset[str]: def orchestrator_field_names() -> frozenset[str]:
"""Return the names of every field on OrchestratorArgs.""" """Return the names of every field on OrchestratorArgs."""
return frozenset(f.name for f in fields(OrchestratorArgs)) return frozenset(f.name for f in fields(OrchestratorArgs))
def deploy_override_field_names() -> frozenset[str]:
"""Return kwargs whose parser defaults must not override deploy YAML."""
return _DEPLOY_ENGINE_ARG_OVERRIDE_FIELDS | _DEPLOY_RUNTIME_OVERRIDE_FIELDS
def internal_blacklist_keys() -> frozenset[str]: def internal_blacklist_keys() -> frozenset[str]:
"""Return the set of CLI keys that must never be forwarded as per-stage """Return the set of CLI keys that must never be forwarded as per-stage
engine overrides. engine overrides.
@@ -615,8 +653,6 @@ def nullify_stage_engine_defaults(parser: argparse.ArgumentParser) -> None:
"""Reset stage-level engine flag defaults to ``None``; preserve real """Reset stage-level engine flag defaults to ``None``; preserve real
default in help text. Only deploy-YAML override fields are touched. default in help text. Only deploy-YAML override fields are touched.
Idempotent.""" Idempotent."""
from vllm_omni.config.stage_config import deploy_override_field_names
override_dests = deploy_override_field_names() override_dests = deploy_override_field_names()
for action in parser._actions: for action in parser._actions:
+3 -10
View File
@@ -1294,12 +1294,9 @@ class AsyncOmniEngine:
ring_degree = normalized_kwargs.get("ring_degree") or 1 ring_degree = normalized_kwargs.get("ring_degree") or 1
ulysses_mode = normalized_kwargs.get("ulysses_mode") or "strict" ulysses_mode = normalized_kwargs.get("ulysses_mode") or "strict"
sequence_parallel_size = normalized_kwargs.get("sequence_parallel_size") sequence_parallel_size = normalized_kwargs.get("sequence_parallel_size")
pipeline_parallel_size = normalized_kwargs.get("pipeline_parallel_size") or 1
data_parallel_size = normalized_kwargs.get("data_parallel_size") or 1
tensor_parallel_size = normalized_kwargs.get("tensor_parallel_size") or 1 tensor_parallel_size = normalized_kwargs.get("tensor_parallel_size") or 1
cfg_parallel_size = normalized_kwargs.get("cfg_parallel_size") or 1 cfg_parallel_size = normalized_kwargs.get("cfg_parallel_size") or 1
vae_patch_parallel_size = normalized_kwargs.get("vae_patch_parallel_size") or 1 vae_patch_parallel_size = normalized_kwargs.get("vae_patch_parallel_size") or 1
enable_expert_parallel = normalized_kwargs.get("enable_expert_parallel") or False
use_hsdp = normalized_kwargs.get("use_hsdp", False) use_hsdp = normalized_kwargs.get("use_hsdp", False)
hsdp_shard_size = normalized_kwargs.get("hsdp_shard_size", -1) hsdp_shard_size = normalized_kwargs.get("hsdp_shard_size", -1)
hsdp_replicate_size = normalized_kwargs.get("hsdp_replicate_size", 1) hsdp_replicate_size = normalized_kwargs.get("hsdp_replicate_size", 1)
@@ -1307,10 +1304,9 @@ class AsyncOmniEngine:
sequence_parallel_size = ulysses_degree * ring_degree sequence_parallel_size = ulysses_degree * ring_degree
parallel_config = DiffusionParallelConfig( parallel_config = DiffusionParallelConfig(
pipeline_parallel_size=pipeline_parallel_size, pipeline_parallel_size=1,
data_parallel_size=data_parallel_size, data_parallel_size=1,
tensor_parallel_size=tensor_parallel_size, tensor_parallel_size=tensor_parallel_size,
enable_expert_parallel=enable_expert_parallel,
sequence_parallel_size=sequence_parallel_size, sequence_parallel_size=sequence_parallel_size,
ulysses_degree=ulysses_degree, ulysses_degree=ulysses_degree,
ring_degree=ring_degree, ring_degree=ring_degree,
@@ -1324,9 +1320,6 @@ class AsyncOmniEngine:
num_devices = max(1, int(parallel_config.world_size)) num_devices = max(1, int(parallel_config.world_size))
devices = ",".join(str(i) for i in range(num_devices)) devices = ",".join(str(i) for i in range(num_devices))
enforce_eager = kwargs.get("enforce_eager")
if enforce_eager is None:
enforce_eager = False
stage_engine_args = { stage_engine_args = {
"max_num_seqs": 1, "max_num_seqs": 1,
@@ -1340,7 +1333,7 @@ class AsyncOmniEngine:
"enable_cache_dit_summary": kwargs.get("enable_cache_dit_summary", False), "enable_cache_dit_summary": kwargs.get("enable_cache_dit_summary", False),
"enable_cpu_offload": kwargs.get("enable_cpu_offload", False), "enable_cpu_offload": kwargs.get("enable_cpu_offload", False),
"enable_layerwise_offload": kwargs.get("enable_layerwise_offload", False), "enable_layerwise_offload": kwargs.get("enable_layerwise_offload", False),
"enforce_eager": enforce_eager, "enforce_eager": kwargs.get("enforce_eager", False),
"boundary_ratio": kwargs.get("boundary_ratio", None), "boundary_ratio": kwargs.get("boundary_ratio", None),
"flow_shift": kwargs.get("flow_shift", None), "flow_shift": kwargs.get("flow_shift", None),
"diffusion_load_format": kwargs.get("diffusion_load_format", "default"), "diffusion_load_format": kwargs.get("diffusion_load_format", "default"),
+3 -1
View File
@@ -100,7 +100,9 @@ class OmniBase(PDDisaggregationMixin):
kwargs: dict[str, Any] = {k: v for k, v in vars(args).items() if not k.startswith("_")} kwargs: dict[str, Any] = {k: v for k, v in vars(args).items() if not k.startswith("_")}
if parser is not None and not getattr(parser, "_omni_nullified", False): if parser is not None and not getattr(parser, "_omni_nullified", False):
from vllm_omni.config.stage_config import deploy_override_field_names from vllm_omni.engine.arg_utils import (
deploy_override_field_names,
)
from vllm_omni.entrypoints.utils import detect_explicit_cli_keys from vllm_omni.entrypoints.utils import detect_explicit_cli_keys
explicit = detect_explicit_cli_keys(sys.argv[1:], parser) or set() explicit = detect_explicit_cli_keys(sys.argv[1:], parser) or set()
+3 -2
View File
@@ -65,7 +65,7 @@ from vllm.entrypoints.openai.speech_to_text.serving import (
from vllm.entrypoints.openai.utils import validate_json_request from vllm.entrypoints.openai.utils import validate_json_request
from vllm.entrypoints.pooling.classify.serving import ServingClassification from vllm.entrypoints.pooling.classify.serving import ServingClassification
from vllm.entrypoints.pooling.embed.serving import ServingEmbedding as OpenAIServingEmbedding from vllm.entrypoints.pooling.embed.serving import ServingEmbedding as OpenAIServingEmbedding
from vllm.entrypoints.pooling.pooling.serving import ServingPooling from vllm.entrypoints.pooling.pooling.serving import ServingPooling as OpenAIServingPooling
from vllm.entrypoints.pooling.scoring.serving import ServingScores from vllm.entrypoints.pooling.scoring.serving import ServingScores
from vllm.entrypoints.serve.disagg.serving import ServingTokens from vllm.entrypoints.serve.disagg.serving import ServingTokens
@@ -800,9 +800,10 @@ async def omni_init_app_state(
else None else None
) )
state.openai_serving_pooling = ( state.openai_serving_pooling = (
ServingPooling( OpenAIServingPooling(
engine_client, engine_client,
state.openai_serving_models, state.openai_serving_models,
state.openai_serving_render,
supported_tasks=tuple(supported_tasks), supported_tasks=tuple(supported_tasks),
request_logger=request_logger, request_logger=request_logger,
chat_template=resolved_chat_template, chat_template=resolved_chat_template,
@@ -489,7 +489,7 @@ class Qwen3OmniMoeModel(Qwen3MoeLLMForCausalLM):
""" """
Qwen3 Omni MoE Talker language model. Qwen3 Omni MoE Talker language model.
Extends Qwen3MoeLLMForCausalLM (which already uses FusedMoE with Extends Qwen3MoeLLMForCausalLM (which already uses SharedFusedMoE with
shared-expert support) and replaces the text embedding / LM head with a shared-expert support) and replaces the text embedding / LM head with a
codec embedding so the talker operates over audio-codec tokens instead codec embedding so the talker operates over audio-codec tokens instead
of text tokens. of text tokens.
@@ -103,9 +103,6 @@ def prepare_hunyuan_fused_moe_runtime() -> None:
_ensure_forward_context_attr("flash_comm_v1_enabled", bool, False) _ensure_forward_context_attr("flash_comm_v1_enabled", bool, False)
# NOTE: As of v0.20,0 vLLM has folded SharedFusedMoE -> FusedMoE and removed the class.
# Since AscendSharedFusedMoE is a subclass of SharedFusedMoE, we should be careful
# to ensure that this is updated correctly.
class AscendHunyuanFusedMoE(AscendSharedFusedMoE): class AscendHunyuanFusedMoE(AscendSharedFusedMoE):
def __init__(self, *, prefix: str = "", **kwargs: Any) -> None: def __init__(self, *, prefix: str = "", **kwargs: Any) -> None:
super().__init__(prefix=prefix, **kwargs) super().__init__(prefix=prefix, **kwargs)