Compare commits

..
Author SHA1 Message Date
Hongsheng Liu cf301fe75b Revert "[Quantization] Enable FP8 online quantization for Z-image text encode…"
This reverts commit ac66282746.
2026-04-30 15:05:26 +08:00
45 changed files with 152 additions and 514 deletions

No files matched your search

+1 -1
View File
@@ -10,7 +10,7 @@ steps:
DOCKER_BUILDKIT: "1"
# Buildkite will automatically replace this with the actual commit hash
VLLM_IMAGE_TAG: "${BUILDKITE_COMMIT}"
VLLM_VERSION: "v0.20.0"
VLLM_VERSION: "v0.19.0"
priority: 100
timeout_in_minutes: 60
soft_fail: true
+1 -3
View File
@@ -7,9 +7,7 @@ steps:
grade: Blocking
commands:
- export VLLM_ROCM_USE_AITER=0
# ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM
# that is not supported on CPU
- "timeout 20m pytest -v -s -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-branch --cov-report=term-missing --cov-report=html --cov-report=xml"
- "timeout 20m pytest -v -s -m 'core_model and cpu' --cov=vllm_omni --cov-branch --cov-report=term-missing --cov-report=html --cov-report=xml"
- label: "Diffusion Model Test"
agent_pool: mi325_2
+1 -3
View File
@@ -7,9 +7,7 @@ steps:
grade: Blocking
commands:
- export VLLM_ROCM_USE_AITER=0
# ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM
# that is not supported on CPU
- "timeout 20m pytest -vvvv -s -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-branch --cov-report=term-missing --cov-report=html --cov-report=xml"
- "timeout 20m pytest -v -s -m 'core_model and cpu' --cov=vllm_omni --cov-branch --cov-report=term-missing --cov-report=html --cov-report=xml"
- label: "Voxtral TTS CUDA Unit Test"
agent_pool: mi325_1
-223
View File
@@ -1,223 +0,0 @@
pull_request_rules:
- name: label-documentation
description: Automatically apply documentation label
conditions:
- label != stale
- or:
- files~=^[^/]+\.md$
- files~=^docs/
- files~=^examples/
actions:
label:
add:
- documentation
comment:
message: "Documentation preview: https://vllm--{{number}}.org.readthedocs.build/en/{{number}}/"
- name: comment-pre-commit-failure
description: Comment on PR when pre-commit check fails
conditions:
- status-failure=pre-commit
- -closed
- -draft
actions:
comment:
message: |
Hi @{{author}}, the pre-commit checks have failed. Please run:
```bash
uv pip install pre-commit
pre-commit install
pre-commit run --all-files
```
Then, commit the changes and push to your branch.
For future commits, `pre-commit` will run automatically on changed files before each commit.
> [!TIP]
> <details>
> <summary>Is <code>mypy</code> or <code>markdownlint</code> failing?</summary>
> <br/>
> <code>mypy</code> and <code>markdownlint</code> are run differently in CI. If the failure is related to either of these checks, please use the following commands to run them locally:
>
> ```bash
> # For mypy (substitute "3.10" with the failing version if needed)
> pre-commit run --hook-stage manual mypy-3.10
> # For markdownlint
> pre-commit run --hook-stage manual markdownlint
> ```
> </details>
- name: comment-dco-failure
description: Comment on PR when DCO check fails
conditions:
- status-failure=dco
- -closed
- -draft
actions:
comment:
message: |
Hi @{{author}}, the DCO check has failed. Please click on DCO in the Checks section for instructions on how to resolve this.
- name: label-ci-build
description: Automatically apply ci/build label
conditions:
- label != stale
- or:
- files~=^\.github/
- files~=\.buildkite/
- files~=^cmake/
- files=CMakeLists.txt
- files~=^docker/Dockerfile
- files~=^requirements.*\.txt
- files=setup.py
actions:
label:
add:
- ci/build
- name: label-frontend
description: Automatically apply frontend label
conditions:
- label != stale
- files~=^vllm_omni/entrypoints/
actions:
label:
add:
- frontend
- name: label-diffusion
description: Automatically apply diffusion label
conditions:
- label != stale
- files~=^vllm_omni/diffusion/
actions:
label:
add:
- diffusion
- name: label-new-model
description: Automatically apply new-model label
conditions:
- label != stale
- or:
- files=vllm_omni/model_executor/models/registry.py
- files=vllm_omni/diffusion/registry.py
actions:
label:
add:
- new-model
- name: label-npu
description: Automatically apply npu label
conditions:
- label != stale
- or:
- files~=^vllm_omni/platforms/npu/
- title~=(?i)NPU
- title~=(?i)Ascend
actions:
label:
add:
- npu
assign:
users:
- "gcanlin"
- name: label-rocm
description: Automatically apply rocm label
conditions:
- label != stale
- or:
- files~=^vllm_omni/platforms/rocm/
- title~=(?i)AMD
- title~=(?i)ROCm
actions:
label:
add:
- rocm
assign:
users:
- "tjtanaa"
- name: label-xpu
description: Automatically apply xpu label
conditions:
- label != stale
- or:
- files~=^vllm_omni/platforms/xpu/
- title~=(?i)Intel
- title~=(?i)XPU
actions:
label:
add:
- xpu
assign:
users:
- "yma11"
- name: label-modelrunner
description: Automatically apply modelrunner label
conditions:
- label != stale
- or:
- files~=^vllm_omni/worker/
- title~=(?i)ModelRunner
- title~=(?i)worker
actions:
label:
add:
- ModelRunner
assign:
users:
- "gcanlin" # To check whether need to sync to NPU
- name: auto-rebase if approved, ready, and 30 commits behind main
conditions:
- base = main
- label=ready
- "#approved-reviews-by >= 1"
- "#commits-behind >= 30"
- -closed
- -draft
- -conflict
actions:
rebase: {}
- name: ping author on conflicts and add 'needs-rebase' label
conditions:
- label != stale
- conflict
- -closed
actions:
label:
add:
- needs-rebase
comment:
message: |
This pull request has merge conflicts that must be resolved before it can be
merged. Please rebase the PR, @{{author}}.
https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/working-with-forks/syncing-a-fork
- name: remove 'needs-rebase' label when conflict is resolved
conditions:
- -conflict
- -closed
actions:
label:
remove:
- needs-rebase
- name: label-bug
description: Automatically apply bug label
conditions:
- label != stale
- or:
- title~=(?i)\bbug\b
- title~=(?i)\bbugfix\b
actions:
label:
add:
- bug
+1 -1
View File
@@ -1,5 +1,5 @@
ARG VLLM_BASE_IMAGE=vllm/vllm-openai
ARG VLLM_BASE_TAG=v0.20.0
ARG VLLM_BASE_TAG=v0.20.0-cu130
FROM ${VLLM_BASE_IMAGE}:${VLLM_BASE_TAG}
ARG APP_DIR=/workspace/vllm-omni
WORKDIR ${APP_DIR}
+1 -1
View File
@@ -1,4 +1,4 @@
ARG BASE_IMAGE=vllm/vllm-openai:v0.20.0
ARG BASE_IMAGE=vllm/vllm-openai:v0.19.0
FROM ${BASE_IMAGE}
ARG COMMON_WORKDIR=/app
+1 -1
View File
@@ -1,4 +1,4 @@
ARG BASE_IMAGE=vllm/vllm-openai-rocm:v0.20.0
ARG BASE_IMAGE=vllm/vllm-openai-rocm:v0.19.0
FROM ${BASE_IMAGE} AS base
# Declare a variable to know if we want to use the nightly build or the stable build.
+1 -1
View File
@@ -74,7 +74,7 @@ ENV UV_EXTRA_INDEX_URL=${PIP_EXTRA_INDEX_URL}
ENV UV_INDEX_STRATEGY="unsafe-best-match"
ENV UV_LINK_MODE="copy"
ARG VLLM_VERSION=v0.20.0
ARG VLLM_VERSION=v0.19.0
RUN git clone -b ${VLLM_VERSION} https://github.com/vllm-project/vllm
WORKDIR /workspace/vllm
@@ -13,7 +13,7 @@ vLLM-Omni current recommends the steps in under setup through Docker Images.
vLLM-Omni is built based on vLLM. Please install it with command below.
```bash
uv pip install vllm==0.20.0+rocm721 --extra-index-url https://wheels.vllm.ai/rocm/0.20.0/rocm721
uv pip install vllm==0.20.0+rocm700 --extra-index-url https://wheels.vllm.ai/rocm/0.20.0/rocm700
```
#### Installation of vLLM-Omni
@@ -37,10 +37,10 @@ uv pip install onnxruntime-rocm
If you do not need to modify source code of vLLM, you can directly install the stable 0.20.0 release version of the library
```bash
uv pip install vllm==0.20.0+rocm721 --extra-index-url https://wheels.vllm.ai/rocm/0.20.0/rocm721
uv pip install vllm==0.20.0+rocm700 --extra-index-url https://wheels.vllm.ai/rocm/0.20.0/rocm700
```
The pre-built 0.20.0 vLLM wheel targets ROCm 7.2.1. If you need a different ROCm stack or want to reuse an existing PyTorch installation, build vLLM from source instead.
The pre-built 0.20.0 vLLM wheel targets ROCm 7.0. If you need a different ROCm stack or want to reuse an existing PyTorch installation, build vLLM from source instead.
#### Installation of vLLM-Omni
Since vllm-omni is rapidly evolving, it's recommended to install it from source
+1 -7
View File
@@ -22,7 +22,7 @@ source .venv/bin/activate
uv pip install vllm==0.20.0 --torch-backend=auto
# On ROCm
uv pip install vllm==0.20.0+rocm721 --extra-index-url https://wheels.vllm.ai/rocm/0.20.0/rocm721
uv pip install vllm==0.20.0+rocm700 --extra-index-url https://wheels.vllm.ai/rocm/0.20.0/rocm700
git clone https://github.com/vllm-project/vllm-omni.git
cd vllm-omni
@@ -31,12 +31,6 @@ uv pip install -e .
For additional installation methods — please see the [installation guide](installation/README.md).
!!! note
It is important to install the same major & minor version of vLLM and vLLM Omni, otherwise things may not work as expected. If the versions are misaligned, you will see a warning when you import vLLM Omni.
If you are seeing strange behavior with the `vllm` command not handling the `--omni` flag correctly, you most likely have a version mismatch with vLLM < `0.20.0` and vLLM Omni `0.20.0`, as vLLM Omni no longer hijacks the vLLM entrypoint. Updating vLLM should resolve this issue.
## Offline Inference
Text-to-image generation quickstart with vLLM-Omni:
@@ -118,6 +118,9 @@ def parse_args() -> argparse.Namespace:
help='JSON profiler config for torch/cuda profiling, e.g. \'{"profiler":"torch","torch_profiler_dir":"./perf"}\'.',
)
from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults
nullify_stage_engine_defaults(parser)
return parser.parse_args()
@@ -196,6 +196,9 @@ def parse_args() -> argparse.Namespace:
parser.add_argument("--cfg-parallel-size", type=int, default=1, choices=[1, 2], help="CFG parallel size.")
parser.add_argument("--tensor-parallel-size", type=int, default=1, help="Tensor parallelism size.")
from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults
nullify_stage_engine_defaults(parser)
return parser.parse_args()
@@ -378,6 +378,9 @@ def parse_args() -> argparse.Namespace:
default=None,
help='JSON profiler config for torch/cuda profiling, e.g. \'{"profiler":"torch","torch_profiler_dir":"./perf"}\'.',
)
from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults
nullify_stage_engine_defaults(parser)
return parser.parse_args()
@@ -211,6 +211,9 @@ def parse_args() -> argparse.Namespace:
default=None,
help='JSON profiler config for torch/cuda profiling, e.g. \'{"profiler":"torch","torch_profiler_dir":"./perf"}\'.',
)
from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults
nullify_stage_engine_defaults(parser)
return parser.parse_args()
@@ -24,6 +24,9 @@ def parse_args():
parser.add_argument("--width", type=int, default=448, help="Video width.")
parser.add_argument("--num-inference-steps", type=int, default=8, help="Number of denoising steps.")
parser.add_argument("--seed", type=int, default=52, help="Random seed for generation.")
from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults
nullify_stage_engine_defaults(parser)
return parser.parse_args()
@@ -9,7 +9,7 @@ from typing import Any
import torch
from vllm_omni.diffusion.data import logger
from vllm_omni.diffusion.data import DiffusionParallelConfig, logger
from vllm_omni.entrypoints.omni import Omni
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
from vllm_omni.lora.request import LoRARequest
@@ -342,6 +342,17 @@ def main():
# (e.g., QwenImagePipeline or FluxPipeline)
}
# assert args.ring_degree == 1, "Ring attention is not supported yet"
parallel_config = DiffusionParallelConfig(
ulysses_degree=args.ulysses_degree,
ring_degree=args.ring_degree,
ulysses_mode=args.ulysses_mode,
cfg_parallel_size=args.cfg_parallel_size,
tensor_parallel_size=args.tensor_parallel_size,
vae_patch_parallel_size=args.vae_patch_parallel_size,
enable_expert_parallel=args.enable_expert_parallel,
)
profiler_enabled = args.profiler_config is not None
# Prepare LoRA kwargs for Omni initialization
@@ -377,13 +388,7 @@ def main():
"cache_backend": args.cache_backend,
"cache_config": cache_config,
"enable_cache_dit_summary": args.enable_cache_dit_summary,
"ulysses_degree": args.ulysses_degree,
"ring_degree": args.ring_degree,
"ulysses_mode": args.ulysses_mode,
"cfg_parallel_size": args.cfg_parallel_size,
"tensor_parallel_size": args.tensor_parallel_size,
"vae_patch_parallel_size": args.vae_patch_parallel_size,
"enable_expert_parallel": args.enable_expert_parallel,
"parallel_config": parallel_config,
"enforce_eager": args.enforce_eager,
"enable_cpu_offload": args.enable_cpu_offload,
"mode": "text-to-image",
@@ -226,6 +226,9 @@ def parse_args() -> argparse.Namespace:
default=1,
help="Number of HSDP replica groups.",
)
from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults
nullify_stage_engine_defaults(parser)
return parser.parse_args()
@@ -71,6 +71,9 @@ def parse_args() -> argparse.Namespace:
parser.add_argument("--ulysses-degree", type=int, default=1, help="Ulysses SP degree.")
parser.add_argument("--ring-degree", type=int, default=1, help="Ring attention degree.")
parser.add_argument("--cfg-parallel-size", type=int, default=1, choices=[1, 2], help="CFG parallel size.")
from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults
nullify_stage_engine_defaults(parser)
return parser.parse_args()
+2
View File
@@ -32,6 +32,7 @@ classifiers = [
# Dependencies are now managed dynamically via setup.py based on detected hardware platform.
# This allows automatic installation of the correct platform-specific dependencies (CUDA/ROCm/CPU/XPU/NPU)
# without requiring extras like [cuda]. See requirements/ directory for platform-specific dependencies.
# Note: vllm is intentionally excluded due to entrypoints overwrite issue.
[project.optional-dependencies]
@@ -89,6 +90,7 @@ Documentation = "https://vllm-omni.readthedocs.io"
"Bug Tracker" = "https://github.com/vllm-project/vllm-omni/issues"
[project.scripts]
vllm = "vllm_omni.entrypoints.cli.main:main"
vllm-omni = "vllm_omni.entrypoints.cli.main:main"
@@ -121,8 +121,7 @@ class TestTrajectoryRecording:
assert trajectory_latents is not None
assert trajectory_timesteps is not None
# initial latent + one per denoising step
assert len(trajectory_latents) == EXPECTED_STEPS + 1
assert len(trajectory_latents) == EXPECTED_STEPS
assert len(trajectory_timesteps) == EXPECTED_STEPS
# log_probs is None without a scheduler (default ODE path)
assert trajectory_log_probs is None
@@ -168,54 +167,6 @@ class TestTrajectoryRecording:
"Last trajectory latent should match the final output"
)
def test_initial_latent_matches_input_noise(self, bagel_and_args):
"""Regression: the first trajectory entry must be the initial noise (pre-denoising)."""
bagel, args = bagel_and_args
init_noise = args["packed_init_noises"].clone()
_, trajectory_latents, *_ = bagel.generate_image(**args, return_trajectory_latents=True)
assert torch.allclose(trajectory_latents[0], init_noise, atol=1e-6), (
"trajectory_latents[0] should be the initial noise before any denoising step"
)
def test_trajectory_timesteps_match_expected_schedule(self, bagel_and_args):
"""Regression: trajectory_timesteps must record raw timesteps[i], not timesteps[i] - dts[i]."""
bagel, args = bagel_and_args
# Recompute the expected timestep schedule (mirrors generate_image logic)
ts = torch.linspace(1, 0, args["num_timesteps"])
shift = args.get("timestep_shift", 1.0)
ts = shift * ts / (1 + (shift - 1) * ts)
expected_timesteps = ts[:-1] # last element is dropped
_, _, trajectory_timesteps, _ = bagel.generate_image(**args, return_trajectory_latents=True)
assert len(trajectory_timesteps) == len(expected_timesteps)
for i, (actual, expected) in enumerate(zip(trajectory_timesteps, expected_timesteps)):
assert abs(float(actual) - float(expected)) < 1e-6, (
f"Step {i}: trajectory_timestep={float(actual):.6f}, expected={float(expected):.6f}"
)
def test_latent_count_equals_timesteps_plus_one(self, bagel_and_args):
"""Regression: len(trajectory_latents) == len(timesteps) + 1 (initial + one per step)."""
bagel, args = bagel_and_args
# Compute the number of denoising steps the same way generate_image does
ts = torch.linspace(1, 0, args["num_timesteps"])
shift = args.get("timestep_shift", 1.0)
ts = shift * ts / (1 + (shift - 1) * ts)
num_steps = len(ts) - 1 # timesteps = ts[:-1]
_, trajectory_latents, trajectory_timesteps, _ = bagel.generate_image(**args, return_trajectory_latents=True)
assert len(trajectory_latents) == num_steps + 1, (
f"Expected {num_steps + 1} latents (initial + {num_steps} steps), got {len(trajectory_latents)}"
)
assert len(trajectory_timesteps) == num_steps, (
f"Expected {num_steps} timesteps, got {len(trajectory_timesteps)}"
)
# ---------------------------------------------------------------------------
# Mock scheduler for log-prob tests
@@ -356,6 +356,7 @@ def test_mix_to_text_audio_001(omni_server, openai_client) -> None:
@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
@pytest.mark.parametrize("omni_server", test_params, indirect=True)
@pytest.mark.skip(reason="issue: #2827")
def test_audio_in_video_001(omni_server, openai_client) -> None:
"""
Input Modal: text + video (synthetic MP4 with embedded audio; ``use_audio_in_video`` uses audio from the video).
@@ -375,7 +376,7 @@ def test_audio_in_video_001(omni_server, openai_client) -> None:
"messages": messages,
"stream": False,
"use_audio_in_video": True,
"key_words": {"video": VIDEO_KEY},
"key_words": {"video": VIDEO_KEY, "audio": AUDIO_KEY + ["beep", "electronic"]},
}
openai_client.send_omni_request(request_config)
@@ -98,7 +98,7 @@ def _get_diffusion_feature_cases():
],
),
id="layerwise_hsdp",
marks=[*FOUR_CARD_MARKS],
marks=[*FOUR_CARD_MARKS, pytest.mark.skip(reason="issue #2435")],
),
]
@@ -4,7 +4,6 @@
import pytest
from vllm.utils.argparse_utils import FlexibleArgumentParser
from vllm_omni.config.stage_config import deploy_override_field_names
from vllm_omni.engine.async_omni_engine import AsyncOmniEngine
from vllm_omni.entrypoints.cli.serve import OmniServeCommand, _create_default_diffusion_stage_cfg
@@ -31,15 +30,6 @@ def test_default_stage_config_includes_cache_backend():
assert engine_args["model_stage"] == "diffusion"
def test_default_stage_config_ignores_none_deploy_overrides():
"""Ensure nullified deploy override defaults do not alter diffusion defaults."""
baseline = AsyncOmniEngine._create_default_diffusion_stage_cfg({})[0]
nullified_overrides = {name: None for name in deploy_override_field_names()}
stage_cfg = AsyncOmniEngine._create_default_diffusion_stage_cfg(nullified_overrides)[0]
assert stage_cfg == baseline
def test_default_cache_config_used_when_missing():
"""Ensure default cache_config is synthesized when only backend is given."""
stage_cfg = AsyncOmniEngine._create_default_diffusion_stage_cfg(
@@ -65,44 +55,6 @@ def test_default_stage_devices_from_sequence_parallel():
assert stage_cfg["runtime"]["devices"] == "0,1,2,3"
def test_default_stage_config_uses_parallel_size_kwargs():
"""Ensure default diffusion parallel_config uses CLI/API parallel sizes."""
stage_cfg = AsyncOmniEngine._create_default_diffusion_stage_cfg(
{
"pipeline_parallel_size": 2,
"data_parallel_size": 3,
"tensor_parallel_size": 4,
"enable_expert_parallel": True,
}
)[0]
parallel_config = stage_cfg["engine_args"]["parallel_config"]
assert parallel_config.pipeline_parallel_size == 2
assert parallel_config.data_parallel_size == 3
assert parallel_config.tensor_parallel_size == 4
assert parallel_config.enable_expert_parallel is True
def test_default_stage_config_defaults_nullified_parallel_size_kwargs():
"""Ensure nullified diffusion parallel-size kwargs fall back to defaults."""
stage_cfg = AsyncOmniEngine._create_default_diffusion_stage_cfg(
{
"pipeline_parallel_size": None,
"data_parallel_size": None,
"tensor_parallel_size": None,
"enable_expert_parallel": None,
"enforce_eager": None,
}
)[0]
parallel_config = stage_cfg["engine_args"]["parallel_config"]
assert parallel_config.pipeline_parallel_size == 1
assert parallel_config.data_parallel_size == 1
assert parallel_config.tensor_parallel_size == 1
assert parallel_config.enable_expert_parallel is False
assert stage_cfg["engine_args"]["enforce_eager"] is False
def test_default_stage_config_propagates_ulysses_mode():
"""Ensure UAA mode survives default diffusion-stage creation."""
stage_cfg = AsyncOmniEngine._create_default_diffusion_stage_cfg(
@@ -26,9 +26,27 @@ def _skip_readme_snippet(language: str, code: str, h2_title: str) -> tuple[bool,
README_SNIPPETS = ReadmeSnippet.extract_readme_snippets(README_PATH, skipif=_skip_readme_snippet)
# Track: https://github.com/vllm-project/vllm-omni/issues/3123 — CLI example subprocess exit 1 in CI; re-enable when fixed.
_SKIP_T2I_SNIPPET_IDS = frozenset(
{
"quick_start_002",
"more_cli_examples_001",
"more_cli_examples_002",
"more_cli_examples_003",
"more_cli_examples_007",
"more_cli_examples_008",
}
)
_SKIP_T2I_REASON = (
"Skipped: text_to_image README examples failing in CI (subprocess exit 1). "
"See https://github.com/vllm-project/vllm-omni/issues/3123"
)
@pytest.mark.parametrize("snippet", README_SNIPPETS, ids=lambda snippet: snippet.test_id)
def test_text_to_image(snippet: ReadmeSnippet, example_runner: ExampleRunner):
if snippet.test_id in _SKIP_T2I_SNIPPET_IDS:
pytest.skip(_SKIP_T2I_REASON)
should_skip, reason = snippet.skip
if should_skip:
pytest.skip(reason)
+9 -3
View File
@@ -3,7 +3,14 @@
"""Pytest marks and decorators for hardware / resource selection (CUDA, ROCm, …)."""
import pytest
from vllm.platforms import current_platform
try:
from vllm.utils.torch_utils import cuda_device_count_stateless
except ImportError:
import torch
def cuda_device_count_stateless() -> int:
return torch.cuda.device_count()
# Re-exported from tests.helpers.env (GPU wait + DeviceMemoryMonitor).
@@ -20,9 +27,8 @@ def cuda_marks(*, res: str, num_cards: int):
if num_cards == 1:
return marks
test_distributed = pytest.mark.distributed_cuda(num_cards=num_cards)
test_skipif = pytest.mark.skipif_cuda(
not current_platform.is_cuda() or (current_platform.device_count() < num_cards),
cuda_device_count_stateless() < num_cards,
reason=f"Need at least {num_cards} CUDA GPUs to run the test.",
)
return marks + [test_distributed, test_skipif]
-1
View File
@@ -494,7 +494,6 @@ _CI_OVERLAYS: dict[str, dict[str, Any]] = {
"max_num_seqs": 1,
"gpu_memory_utilization": 0.9,
"enforce_eager": True,
"enable_prefix_caching": False,
"max_num_batched_tokens": 16384,
"max_model_len": 16384,
"skip_mm_profiling": True,
+1 -1
View File
@@ -369,8 +369,8 @@ def _build_full_serve_parser():
def test_nullify_stage_engine_defaults_resets_inherited_defaults():
import argparse
from vllm_omni.config.stage_config import deploy_override_field_names
from vllm_omni.engine.arg_utils import (
deploy_override_field_names,
nullify_stage_engine_defaults,
)
+2 -67
View File
@@ -93,7 +93,6 @@ class TestStageConfig:
assert omega_config.engine_args.worker_type == "ar"
assert omega_config.final_output is True
assert omega_config.final_output_type == "text"
assert "max_num_seqs" not in omega_config.engine_args
# Legacy field name for backward compatibility
assert omega_config.engine_input_source == []
@@ -147,24 +146,6 @@ class TestStageConfig:
omega_config = config.to_omegaconf()
assert omega_config.engine_args.max_num_seqs == 32
def test_to_omegaconf_omits_none_deploy_overrides_for_engine_args(self):
"""None deploy overrides must fall through to EngineArgs defaults."""
from vllm_omni.config.stage_config import deploy_override_field_names
config = StageConfig(
stage_id=0,
model_stage="thinker",
runtime_overrides={name: None for name in deploy_override_field_names()},
)
omega_config = config.to_omegaconf()
engine_args = dict(omega_config.engine_args)
assert "devices" not in engine_args
assert "max_batch_size" not in engine_args
for name in deploy_override_field_names() - {"devices"}:
assert name not in engine_args
class TestModelPipeline:
"""Tests for ModelPipeline class."""
@@ -821,40 +802,6 @@ class TestPipelineRegistry:
class TestDeployConfigLoading:
def test_deploy_override_fields_include_deploy_schema_fields(self):
from vllm_omni.config.stage_config import deploy_override_field_names
expected_fields = {
"async_chunk",
"async_scheduling",
"config_format",
"data_parallel_size",
"devices",
"disable_hybrid_kv_cache_manager",
"distributed_executor_backend",
"dtype",
"enable_chunked_prefill",
"enable_flashinfer_autotune",
"enable_prefix_caching",
"enforce_eager",
"gpu_memory_utilization",
"load_format",
"max_model_len",
"max_num_batched_tokens",
"max_num_seqs",
"mm_processor_cache_gb",
"pipeline_parallel_size",
"profiler_config",
"quantization",
"skip_mm_profiling",
"subtalker_sampling_params",
"tensor_parallel_size",
"tokenizer_mode",
"trust_remote_code",
}
assert expected_fields == deploy_override_field_names()
def test_load_deploy_config(self):
from pathlib import Path
@@ -870,17 +817,6 @@ class TestDeployConfigLoading:
assert deploy.connectors is not None
assert deploy.platforms is not None
voxtral_path = Path(__file__).parent.parent / "vllm_omni" / "deploy" / "voxtral_tts.yaml"
if voxtral_path.exists():
voxtral_deploy = load_deploy_config(voxtral_path)
assert voxtral_deploy.stages[0].config_format == "mistral"
assert voxtral_deploy.stages[0].load_format == "mistral"
assert voxtral_deploy.stages[0].tokenizer_mode == "mistral"
assert not any(
name in voxtral_deploy.stages[0].engine_extras
for name in ("config_format", "load_format", "tokenizer_mode")
)
def test_merge_pipeline_deploy(self):
from pathlib import Path
@@ -1075,8 +1011,7 @@ class TestBaseConfigInheritance:
deploy = load_deploy_config(ci_path)
assert len(deploy.stages) == 3
# CI overrides
assert deploy.stages[0].load_format == "dummy"
assert "load_format" not in deploy.stages[0].engine_extras
assert deploy.stages[0].engine_extras.get("load_format") == "dummy"
assert deploy.stages[0].max_num_seqs == 5
# Inherited from base
assert deploy.stages[0].gpu_memory_utilization == 0.9
@@ -1281,7 +1216,7 @@ class TestSentinelDefaultPrecedence:
def test_none_value_skipped_yaml_wins(self):
stages = self._stages({"max_num_seqs": None})
assert stages[2].runtime_overrides.get("max_num_seqs") is None
assert "max_num_seqs" not in stages[2].yaml_engine_args
assert stages[2].yaml_engine_args.get("max_num_seqs") == 1
def test_empty_kwargs_yaml_only(self):
stages = self._stages({})
+16 -38
View File
@@ -402,11 +402,11 @@ class StageDeployConfig:
"""
stage_id: int
max_num_seqs: int | None = None
gpu_memory_utilization: float | None = None
tensor_parallel_size: int | None = None
enforce_eager: bool | None = None
max_num_batched_tokens: int | None = None
max_num_seqs: int = 64
gpu_memory_utilization: float = 0.9
tensor_parallel_size: int = 1
enforce_eager: bool = False
max_num_batched_tokens: int = 32768
max_model_len: int | None = None
async_scheduling: bool | None = None
devices: str = "0"
@@ -414,14 +414,6 @@ class StageDeployConfig:
input_connectors: dict[str, str] | None = None
default_sampling_params: dict[str, Any] | None = None
subtalker_sampling_params: dict[str, Any] | None = None
profiler_config: dict[str, Any] | None = None
disable_hybrid_kv_cache_manager: bool | None = None
mm_processor_cache_gb: float | None = None
skip_mm_profiling: bool | None = None
enable_flashinfer_autotune: bool | None = None
config_format: str | None = None
load_format: str | None = None
tokenizer_mode: str | None = None
engine_extras: dict[str, Any] = field(default_factory=dict)
@@ -446,14 +438,14 @@ class DeployConfig:
pipeline: str | None = None
# === Pipeline-wide engine settings (applied uniformly to every stage) ===
trust_remote_code: bool | None = None
trust_remote_code: bool = True
distributed_executor_backend: str | None = None
dtype: str | None = None
quantization: str | None = None
enable_prefix_caching: bool | None = None
enable_prefix_caching: bool = False
enable_chunked_prefill: bool | None = None
data_parallel_size: int | None = None
pipeline_parallel_size: int | None = None
data_parallel_size: int = 1
pipeline_parallel_size: int = 1
_STAGE_NON_ENGINE_KEYS = frozenset(
@@ -697,18 +689,6 @@ _PIPELINE_WIDE_ENGINE_FIELDS: tuple[str, ...] = (
)
def deploy_override_field_names() -> frozenset[str]:
"""Return deploy-schema fields whose CLI defaults must not override YAML."""
return (
frozenset(_STAGE_DEPLOY_FIELDS)
| frozenset(_PIPELINE_WIDE_ENGINE_FIELDS)
| {
"async_chunk",
"devices",
}
)
def _build_engine_args(
ps: StagePipelineConfig,
ds: StageDeployConfig | None,
@@ -881,15 +861,13 @@ class StageConfig:
# CLI overrides take precedence over YAML defaults
for key, value in self.runtime_overrides.items():
if value is None:
continue
if key not in ("devices", "max_batch_size"):
engine_args[key] = value
# Build runtime config from YAML defaults + CLI overrides
runtime: dict[str, Any] = dict(self.yaml_runtime)
runtime.setdefault("process", True)
if self.runtime_overrides.get("devices") is not None:
if "devices" in self.runtime_overrides:
runtime["devices"] = self.runtime_overrides["devices"]
# Legacy compat: migrate runtime.max_batch_size → engine_args.max_num_seqs
@@ -905,6 +883,8 @@ class StageConfig:
effective_mbs = int(cli_mbs or legacy_mbs or 1)
engine_args.setdefault("max_num_seqs", effective_mbs)
engine_args.setdefault("max_num_seqs", 1)
# Build full config dict
config_dict: dict[str, Any] = {
"stage_id": self.stage_id,
@@ -1029,8 +1009,6 @@ class StageConfigFactory:
cli_overrides = {}
trust_remote_code = cli_overrides.get("trust_remote_code", True)
if trust_remote_code is None:
trust_remote_code = False
# --- New path: check pipeline registry by model_type first ---
model_type, hf_config = cls._auto_detect_model_type(model, trust_remote_code=trust_remote_code)
@@ -1374,8 +1352,8 @@ class StageConfigFactory:
hf_config = get_config(model, trust_remote_code=trust_remote_code)
return hf_config.model_type, hf_config
except Exception as e:
logger.debug(f"`get_config` failed for {e}; Falling back to raw config.json path")
except Exception:
pass
# Fallback: read config.json directly for custom model types that
# are not registered with transformers (e.g. qwen3_tts).
@@ -1412,8 +1390,8 @@ class StageConfigFactory:
class_name,
)
return pipeline_cfg.model_type, None
except Exception as e:
logger.debug(f"Failed to detect model type for diffusers-style models: {e}")
except Exception:
pass
# Final fallback: some models (e.g. CosyVoice3) ship an empty
# config.json and rely on naming conventions. Match the model path
+2 -2
View File
@@ -40,7 +40,7 @@ stages:
# near-identity repetition penalty forces vLLM to track
# output_token_ids for RAS (stop-token logit logsumexp).
repetition_penalty: 1.0001
enable_prefix_caching: false
disable_hybrid_kv_cache_manager: true
mm_processor_cache_gb: 0
skip_mm_profiling: true
@@ -54,5 +54,5 @@ stages:
from_stage_0: connector_of_shared_memory
default_sampling_params:
max_tokens: 2048
enable_prefix_caching: false
disable_hybrid_kv_cache_manager: true
skip_mm_profiling: true
-1
View File
@@ -42,7 +42,6 @@ stages:
max_num_seqs: 1
gpu_memory_utilization: 0.2
enforce_eager: true
enable_prefix_caching: false
async_scheduling: false
max_num_batched_tokens: 8192
max_model_len: 8192
-2
View File
@@ -22,7 +22,6 @@ stages:
max_num_seqs: 1
gpu_memory_utilization: 0.8
enforce_eager: true
enable_prefix_caching: false
mm_processor_cache_gb: 0
devices: "0"
default_sampling_params:
@@ -50,7 +49,6 @@ stages:
max_num_seqs: 1
gpu_memory_utilization: 0.15
enforce_eager: true
enable_prefix_caching: false
enable_flashinfer_autotune: false
async_scheduling: false
devices: "0"
-2
View File
@@ -23,7 +23,6 @@ connectors:
stages:
- stage_id: 0
gpu_memory_utilization: 0.9
enable_prefix_caching: false
devices: "0"
default_sampling_params:
temperature: 0.4
@@ -48,7 +47,6 @@ stages:
- stage_id: 2
gpu_memory_utilization: 0.1
enforce_eager: true
enable_prefix_caching: false
async_scheduling: false
max_num_batched_tokens: 51200
devices: "1"
-1
View File
@@ -53,7 +53,6 @@ stages:
max_num_seqs: 1
gpu_memory_utilization: 0.3
enforce_eager: true
enable_prefix_caching: false
async_scheduling: true
# Must be divisible by num_code_groups and cover (left_context + chunk).
# Prefill length is Q * num_frames (e.g. 16 * 2148 = 34368); keep
-1
View File
@@ -20,7 +20,6 @@ stages:
max_num_batched_tokens: 4096
max_model_len: 4096
devices: "0"
trust_remote_code: true
default_sampling_params:
temperature: 0.0
top_p: 1.0
-2
View File
@@ -24,7 +24,6 @@ stages:
max_num_seqs: 32
gpu_memory_utilization: 0.8
enforce_eager: false
enable_prefix_caching: false
async_scheduling: true
max_model_len: 4096
devices: "0"
@@ -49,7 +48,6 @@ stages:
max_num_seqs: 32
gpu_memory_utilization: 0.1
enforce_eager: true
enable_prefix_caching: false
async_scheduling: false
max_num_batched_tokens: 65536
max_model_len: 65536
@@ -385,9 +385,9 @@ class MultiprocDiffusionExecutor(DiffusionExecutor):
raise
def check_health(self) -> None:
self._ensure_open()
if self.is_failed:
raise EngineDeadError()
self._ensure_open()
for p in self._processes:
if not p.is_alive():
self.is_failed = True
@@ -15,7 +15,6 @@ from typing import TYPE_CHECKING, Any
import torch
from PIL import Image
from vllm.logger import init_logger
from vllm.v1.engine.exceptions import EngineDeadError
from vllm_omni.diffusion.data import DiffusionRequestAbortedError
from vllm_omni.diffusion.diffusion_engine import DiffusionEngine
@@ -330,12 +329,6 @@ class InlineStageDiffusionClient:
None,
)
def check_health(self) -> None:
"""Check if the inline diffusion engine and its workers are healthy."""
if self._shutting_down:
raise EngineDeadError("InlineStageDiffusionClient is shutting down")
self._engine.executor.check_health()
def shutdown(self) -> None:
self._shutting_down = True
@@ -1842,8 +1842,6 @@ class Bagel(nn.Module):
# ── SP + CFG: sequential single-branch forwards ──
use_sp = self._sp_size > 1
if use_sp and use_cfg_text:
if return_trajectory_latents and len(timesteps) > 0:
trajectory_latents.append(x_t.clone())
for i, t in enumerate(timesteps):
timestep = torch.tensor([t] * x_t.shape[0], device=x_t.device)
in_cfg_window = t > cfg_interval[0] and t <= cfg_interval[1]
@@ -1907,15 +1905,13 @@ class Bagel(nn.Module):
x_t = x_t - v_t.to(x_t.device) * dts[i]
if return_trajectory_latents:
trajectory_latents.append(x_t.clone())
trajectory_timesteps.append(timesteps[i])
trajectory_timesteps.append(timesteps[i] - dts[i])
unpacked_latent = x_t.split((packed_seqlens - 2).tolist())
return unpacked_latent, trajectory_latents, trajectory_timesteps, trajectory_log_probs
# ── SP without CFG: direct single-branch loop ──
if use_sp:
if return_trajectory_latents and len(timesteps) > 0:
trajectory_latents.append(x_t.clone())
for i, t in enumerate(timesteps):
timestep = torch.tensor([t] * x_t.shape[0], device=x_t.device)
v_t = self.forward_single_branch(
@@ -1942,7 +1938,7 @@ class Bagel(nn.Module):
x_t = x_t - v_t.to(x_t.device) * dts[i]
if return_trajectory_latents:
trajectory_latents.append(x_t.clone())
trajectory_timesteps.append(timesteps[i])
trajectory_timesteps.append(timesteps[i] - dts[i])
unpacked_latent = x_t.split((packed_seqlens - 2).tolist())
return unpacked_latent, trajectory_latents, trajectory_timesteps, trajectory_log_probs
@@ -2003,9 +1999,6 @@ class Bagel(nn.Module):
"merged_cache": self._merge_naive_caches(branches_cache),
}
if return_trajectory_latents and len(timesteps) > 0:
trajectory_latents.append(x_t.clone())
for i, t in enumerate(timesteps):
timestep = torch.tensor([t] * x_t.shape[0], device=x_t.device)
if t > cfg_interval[0] and t <= cfg_interval[1]:
@@ -2043,7 +2036,7 @@ class Bagel(nn.Module):
x_t = x_t - v_t.to(x_t.device) * dts[i] # velocity pointing from data to noise
if return_trajectory_latents:
trajectory_latents.append(x_t.clone())
trajectory_timesteps.append(timesteps[i])
trajectory_timesteps.append(timesteps[i] - dts[i])
unpacked_latent = x_t.split((packed_seqlens - 2).tolist())
return unpacked_latent, trajectory_latents, trajectory_timesteps, trajectory_log_probs
@@ -2144,9 +2137,6 @@ class Bagel(nn.Module):
)
_sched_kw = scheduler_kwargs or {}
if return_trajectory_latents and len(timesteps) > 0:
trajectory_latents.append(x_t.clone())
for i, t in enumerate(timesteps):
timestep = torch.tensor([t] * x_t.shape[0], device=x_t.device)
use_cfg_this_step = t > cfg_interval[0] and t <= cfg_interval[1] and cfg_text_scale > 1.0
@@ -2204,7 +2194,7 @@ class Bagel(nn.Module):
x_t = x_t - v_t.to(x_t.device) * dts[i]
if return_trajectory_latents:
trajectory_latents.append(x_t.clone())
trajectory_timesteps.append(timesteps[i])
trajectory_timesteps.append(timesteps[i] - dts[i])
unpacked_latent = x_t.split((packed_seqlens - 2).tolist())
return unpacked_latent, trajectory_latents, trajectory_timesteps, trajectory_log_probs
+38 -2
View File
@@ -456,12 +456,50 @@ SHARED_FIELDS: frozenset[str] = frozenset(
}
)
_DEPLOY_ENGINE_ARG_OVERRIDE_FIELDS: frozenset[str] = frozenset(
{
# Capacity / scheduling.
"async_scheduling",
"max_model_len",
"max_num_batched_tokens",
"max_num_seqs",
# Memory / parallelism.
"data_parallel_size",
"gpu_memory_utilization",
"pipeline_parallel_size",
"tensor_parallel_size",
# Execution / loading.
"enforce_eager",
"distributed_executor_backend",
"dtype",
"quantization",
"trust_remote_code",
# Caching / chunking.
"async_chunk",
"enable_prefix_caching",
"enable_chunked_prefill",
# Model-specific engine extras.
"subtalker_sampling_params",
}
)
_DEPLOY_RUNTIME_OVERRIDE_FIELDS: frozenset[str] = frozenset(
{
"devices",
}
)
def orchestrator_field_names() -> frozenset[str]:
"""Return the names of every field on OrchestratorArgs."""
return frozenset(f.name for f in fields(OrchestratorArgs))
def deploy_override_field_names() -> frozenset[str]:
"""Return kwargs whose parser defaults must not override deploy YAML."""
return _DEPLOY_ENGINE_ARG_OVERRIDE_FIELDS | _DEPLOY_RUNTIME_OVERRIDE_FIELDS
def internal_blacklist_keys() -> frozenset[str]:
"""Return the set of CLI keys that must never be forwarded as per-stage
engine overrides.
@@ -615,8 +653,6 @@ def nullify_stage_engine_defaults(parser: argparse.ArgumentParser) -> None:
"""Reset stage-level engine flag defaults to ``None``; preserve real
default in help text. Only deploy-YAML override fields are touched.
Idempotent."""
from vllm_omni.config.stage_config import deploy_override_field_names
override_dests = deploy_override_field_names()
for action in parser._actions:
+3 -10
View File
@@ -1294,12 +1294,9 @@ class AsyncOmniEngine:
ring_degree = normalized_kwargs.get("ring_degree") or 1
ulysses_mode = normalized_kwargs.get("ulysses_mode") or "strict"
sequence_parallel_size = normalized_kwargs.get("sequence_parallel_size")
pipeline_parallel_size = normalized_kwargs.get("pipeline_parallel_size") or 1
data_parallel_size = normalized_kwargs.get("data_parallel_size") or 1
tensor_parallel_size = normalized_kwargs.get("tensor_parallel_size") or 1
cfg_parallel_size = normalized_kwargs.get("cfg_parallel_size") or 1
vae_patch_parallel_size = normalized_kwargs.get("vae_patch_parallel_size") or 1
enable_expert_parallel = normalized_kwargs.get("enable_expert_parallel") or False
use_hsdp = normalized_kwargs.get("use_hsdp", False)
hsdp_shard_size = normalized_kwargs.get("hsdp_shard_size", -1)
hsdp_replicate_size = normalized_kwargs.get("hsdp_replicate_size", 1)
@@ -1307,10 +1304,9 @@ class AsyncOmniEngine:
sequence_parallel_size = ulysses_degree * ring_degree
parallel_config = DiffusionParallelConfig(
pipeline_parallel_size=pipeline_parallel_size,
data_parallel_size=data_parallel_size,
pipeline_parallel_size=1,
data_parallel_size=1,
tensor_parallel_size=tensor_parallel_size,
enable_expert_parallel=enable_expert_parallel,
sequence_parallel_size=sequence_parallel_size,
ulysses_degree=ulysses_degree,
ring_degree=ring_degree,
@@ -1324,9 +1320,6 @@ class AsyncOmniEngine:
num_devices = max(1, int(parallel_config.world_size))
devices = ",".join(str(i) for i in range(num_devices))
enforce_eager = kwargs.get("enforce_eager")
if enforce_eager is None:
enforce_eager = False
stage_engine_args = {
"max_num_seqs": 1,
@@ -1340,7 +1333,7 @@ class AsyncOmniEngine:
"enable_cache_dit_summary": kwargs.get("enable_cache_dit_summary", False),
"enable_cpu_offload": kwargs.get("enable_cpu_offload", False),
"enable_layerwise_offload": kwargs.get("enable_layerwise_offload", False),
"enforce_eager": enforce_eager,
"enforce_eager": kwargs.get("enforce_eager", False),
"boundary_ratio": kwargs.get("boundary_ratio", None),
"flow_shift": kwargs.get("flow_shift", None),
"diffusion_load_format": kwargs.get("diffusion_load_format", "default"),
+3 -1
View File
@@ -100,7 +100,9 @@ class OmniBase(PDDisaggregationMixin):
kwargs: dict[str, Any] = {k: v for k, v in vars(args).items() if not k.startswith("_")}
if parser is not None and not getattr(parser, "_omni_nullified", False):
from vllm_omni.config.stage_config import deploy_override_field_names
from vllm_omni.engine.arg_utils import (
deploy_override_field_names,
)
from vllm_omni.entrypoints.utils import detect_explicit_cli_keys
explicit = detect_explicit_cli_keys(sys.argv[1:], parser) or set()
+3 -2
View File
@@ -65,7 +65,7 @@ from vllm.entrypoints.openai.speech_to_text.serving import (
from vllm.entrypoints.openai.utils import validate_json_request
from vllm.entrypoints.pooling.classify.serving import ServingClassification
from vllm.entrypoints.pooling.embed.serving import ServingEmbedding as OpenAIServingEmbedding
from vllm.entrypoints.pooling.pooling.serving import ServingPooling
from vllm.entrypoints.pooling.pooling.serving import ServingPooling as OpenAIServingPooling
from vllm.entrypoints.pooling.scoring.serving import ServingScores
from vllm.entrypoints.serve.disagg.serving import ServingTokens
@@ -800,9 +800,10 @@ async def omni_init_app_state(
else None
)
state.openai_serving_pooling = (
ServingPooling(
OpenAIServingPooling(
engine_client,
state.openai_serving_models,
state.openai_serving_render,
supported_tasks=tuple(supported_tasks),
request_logger=request_logger,
chat_template=resolved_chat_template,
@@ -489,7 +489,7 @@ class Qwen3OmniMoeModel(Qwen3MoeLLMForCausalLM):
"""
Qwen3 Omni MoE Talker language model.
Extends Qwen3MoeLLMForCausalLM (which already uses FusedMoE with
Extends Qwen3MoeLLMForCausalLM (which already uses SharedFusedMoE with
shared-expert support) and replaces the text embedding / LM head with a
codec embedding so the talker operates over audio-codec tokens instead
of text tokens.
@@ -103,9 +103,6 @@ def prepare_hunyuan_fused_moe_runtime() -> None:
_ensure_forward_context_attr("flash_comm_v1_enabled", bool, False)
# NOTE: As of v0.20,0 vLLM has folded SharedFusedMoE -> FusedMoE and removed the class.
# Since AscendSharedFusedMoE is a subclass of SharedFusedMoE, we should be careful
# to ensure that this is updated correctly.
class AscendHunyuanFusedMoE(AscendSharedFusedMoE):
def __init__(self, *, prefix: str = "", **kwargs: Any) -> None:
super().__init__(prefix=prefix, **kwargs)