Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
cf301fe75b |
No files matched your search
@@ -10,7 +10,7 @@ steps:
|
||||
DOCKER_BUILDKIT: "1"
|
||||
# Buildkite will automatically replace this with the actual commit hash
|
||||
VLLM_IMAGE_TAG: "${BUILDKITE_COMMIT}"
|
||||
VLLM_VERSION: "v0.20.0"
|
||||
VLLM_VERSION: "v0.19.0"
|
||||
priority: 100
|
||||
timeout_in_minutes: 60
|
||||
soft_fail: true
|
||||
@@ -7,9 +7,7 @@ steps:
|
||||
grade: Blocking
|
||||
commands:
|
||||
- export VLLM_ROCM_USE_AITER=0
|
||||
# ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM
|
||||
# that is not supported on CPU
|
||||
- "timeout 20m pytest -v -s -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-branch --cov-report=term-missing --cov-report=html --cov-report=xml"
|
||||
- "timeout 20m pytest -v -s -m 'core_model and cpu' --cov=vllm_omni --cov-branch --cov-report=term-missing --cov-report=html --cov-report=xml"
|
||||
|
||||
- label: "Diffusion Model Test"
|
||||
agent_pool: mi325_2
|
||||
|
||||
@@ -7,9 +7,7 @@ steps:
|
||||
grade: Blocking
|
||||
commands:
|
||||
- export VLLM_ROCM_USE_AITER=0
|
||||
# ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM
|
||||
# that is not supported on CPU
|
||||
- "timeout 20m pytest -vvvv -s -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-branch --cov-report=term-missing --cov-report=html --cov-report=xml"
|
||||
- "timeout 20m pytest -v -s -m 'core_model and cpu' --cov=vllm_omni --cov-branch --cov-report=term-missing --cov-report=html --cov-report=xml"
|
||||
|
||||
- label: "Voxtral TTS CUDA Unit Test"
|
||||
agent_pool: mi325_1
|
||||
|
||||
@@ -1,223 +0,0 @@
|
||||
pull_request_rules:
|
||||
- name: label-documentation
|
||||
description: Automatically apply documentation label
|
||||
conditions:
|
||||
- label != stale
|
||||
- or:
|
||||
- files~=^[^/]+\.md$
|
||||
- files~=^docs/
|
||||
- files~=^examples/
|
||||
actions:
|
||||
label:
|
||||
add:
|
||||
- documentation
|
||||
comment:
|
||||
message: "Documentation preview: https://vllm--{{number}}.org.readthedocs.build/en/{{number}}/"
|
||||
|
||||
- name: comment-pre-commit-failure
|
||||
description: Comment on PR when pre-commit check fails
|
||||
conditions:
|
||||
- status-failure=pre-commit
|
||||
- -closed
|
||||
- -draft
|
||||
actions:
|
||||
comment:
|
||||
message: |
|
||||
Hi @{{author}}, the pre-commit checks have failed. Please run:
|
||||
|
||||
```bash
|
||||
uv pip install pre-commit
|
||||
pre-commit install
|
||||
pre-commit run --all-files
|
||||
```
|
||||
|
||||
Then, commit the changes and push to your branch.
|
||||
|
||||
For future commits, `pre-commit` will run automatically on changed files before each commit.
|
||||
|
||||
> [!TIP]
|
||||
> <details>
|
||||
> <summary>Is <code>mypy</code> or <code>markdownlint</code> failing?</summary>
|
||||
> <br/>
|
||||
> <code>mypy</code> and <code>markdownlint</code> are run differently in CI. If the failure is related to either of these checks, please use the following commands to run them locally:
|
||||
>
|
||||
> ```bash
|
||||
> # For mypy (substitute "3.10" with the failing version if needed)
|
||||
> pre-commit run --hook-stage manual mypy-3.10
|
||||
> # For markdownlint
|
||||
> pre-commit run --hook-stage manual markdownlint
|
||||
> ```
|
||||
> </details>
|
||||
|
||||
- name: comment-dco-failure
|
||||
description: Comment on PR when DCO check fails
|
||||
conditions:
|
||||
- status-failure=dco
|
||||
- -closed
|
||||
- -draft
|
||||
actions:
|
||||
comment:
|
||||
message: |
|
||||
Hi @{{author}}, the DCO check has failed. Please click on DCO in the Checks section for instructions on how to resolve this.
|
||||
|
||||
- name: label-ci-build
|
||||
description: Automatically apply ci/build label
|
||||
conditions:
|
||||
- label != stale
|
||||
- or:
|
||||
- files~=^\.github/
|
||||
- files~=\.buildkite/
|
||||
- files~=^cmake/
|
||||
- files=CMakeLists.txt
|
||||
- files~=^docker/Dockerfile
|
||||
- files~=^requirements.*\.txt
|
||||
- files=setup.py
|
||||
actions:
|
||||
label:
|
||||
add:
|
||||
- ci/build
|
||||
|
||||
- name: label-frontend
|
||||
description: Automatically apply frontend label
|
||||
conditions:
|
||||
- label != stale
|
||||
- files~=^vllm_omni/entrypoints/
|
||||
actions:
|
||||
label:
|
||||
add:
|
||||
- frontend
|
||||
|
||||
- name: label-diffusion
|
||||
description: Automatically apply diffusion label
|
||||
conditions:
|
||||
- label != stale
|
||||
- files~=^vllm_omni/diffusion/
|
||||
actions:
|
||||
label:
|
||||
add:
|
||||
- diffusion
|
||||
|
||||
- name: label-new-model
|
||||
description: Automatically apply new-model label
|
||||
conditions:
|
||||
- label != stale
|
||||
- or:
|
||||
- files=vllm_omni/model_executor/models/registry.py
|
||||
- files=vllm_omni/diffusion/registry.py
|
||||
actions:
|
||||
label:
|
||||
add:
|
||||
- new-model
|
||||
|
||||
- name: label-npu
|
||||
description: Automatically apply npu label
|
||||
conditions:
|
||||
- label != stale
|
||||
- or:
|
||||
- files~=^vllm_omni/platforms/npu/
|
||||
- title~=(?i)NPU
|
||||
- title~=(?i)Ascend
|
||||
actions:
|
||||
label:
|
||||
add:
|
||||
- npu
|
||||
assign:
|
||||
users:
|
||||
- "gcanlin"
|
||||
|
||||
- name: label-rocm
|
||||
description: Automatically apply rocm label
|
||||
conditions:
|
||||
- label != stale
|
||||
- or:
|
||||
- files~=^vllm_omni/platforms/rocm/
|
||||
- title~=(?i)AMD
|
||||
- title~=(?i)ROCm
|
||||
actions:
|
||||
label:
|
||||
add:
|
||||
- rocm
|
||||
assign:
|
||||
users:
|
||||
- "tjtanaa"
|
||||
|
||||
- name: label-xpu
|
||||
description: Automatically apply xpu label
|
||||
conditions:
|
||||
- label != stale
|
||||
- or:
|
||||
- files~=^vllm_omni/platforms/xpu/
|
||||
- title~=(?i)Intel
|
||||
- title~=(?i)XPU
|
||||
actions:
|
||||
label:
|
||||
add:
|
||||
- xpu
|
||||
assign:
|
||||
users:
|
||||
- "yma11"
|
||||
|
||||
- name: label-modelrunner
|
||||
description: Automatically apply modelrunner label
|
||||
conditions:
|
||||
- label != stale
|
||||
- or:
|
||||
- files~=^vllm_omni/worker/
|
||||
- title~=(?i)ModelRunner
|
||||
- title~=(?i)worker
|
||||
actions:
|
||||
label:
|
||||
add:
|
||||
- ModelRunner
|
||||
assign:
|
||||
users:
|
||||
- "gcanlin" # To check whether need to sync to NPU
|
||||
|
||||
- name: auto-rebase if approved, ready, and 30 commits behind main
|
||||
conditions:
|
||||
- base = main
|
||||
- label=ready
|
||||
- "#approved-reviews-by >= 1"
|
||||
- "#commits-behind >= 30"
|
||||
- -closed
|
||||
- -draft
|
||||
- -conflict
|
||||
actions:
|
||||
rebase: {}
|
||||
|
||||
- name: ping author on conflicts and add 'needs-rebase' label
|
||||
conditions:
|
||||
- label != stale
|
||||
- conflict
|
||||
- -closed
|
||||
actions:
|
||||
label:
|
||||
add:
|
||||
- needs-rebase
|
||||
comment:
|
||||
message: |
|
||||
This pull request has merge conflicts that must be resolved before it can be
|
||||
merged. Please rebase the PR, @{{author}}.
|
||||
|
||||
https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/working-with-forks/syncing-a-fork
|
||||
|
||||
- name: remove 'needs-rebase' label when conflict is resolved
|
||||
conditions:
|
||||
- -conflict
|
||||
- -closed
|
||||
actions:
|
||||
label:
|
||||
remove:
|
||||
- needs-rebase
|
||||
|
||||
- name: label-bug
|
||||
description: Automatically apply bug label
|
||||
conditions:
|
||||
- label != stale
|
||||
- or:
|
||||
- title~=(?i)\bbug\b
|
||||
- title~=(?i)\bbugfix\b
|
||||
actions:
|
||||
label:
|
||||
add:
|
||||
- bug
|
||||
@@ -1,5 +1,5 @@
|
||||
ARG VLLM_BASE_IMAGE=vllm/vllm-openai
|
||||
ARG VLLM_BASE_TAG=v0.20.0
|
||||
ARG VLLM_BASE_TAG=v0.20.0-cu130
|
||||
FROM ${VLLM_BASE_IMAGE}:${VLLM_BASE_TAG}
|
||||
ARG APP_DIR=/workspace/vllm-omni
|
||||
WORKDIR ${APP_DIR}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
ARG BASE_IMAGE=vllm/vllm-openai:v0.20.0
|
||||
ARG BASE_IMAGE=vllm/vllm-openai:v0.19.0
|
||||
FROM ${BASE_IMAGE}
|
||||
|
||||
ARG COMMON_WORKDIR=/app
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
ARG BASE_IMAGE=vllm/vllm-openai-rocm:v0.20.0
|
||||
ARG BASE_IMAGE=vllm/vllm-openai-rocm:v0.19.0
|
||||
FROM ${BASE_IMAGE} AS base
|
||||
|
||||
# Declare a variable to know if we want to use the nightly build or the stable build.
|
||||
|
||||
@@ -74,7 +74,7 @@ ENV UV_EXTRA_INDEX_URL=${PIP_EXTRA_INDEX_URL}
|
||||
ENV UV_INDEX_STRATEGY="unsafe-best-match"
|
||||
ENV UV_LINK_MODE="copy"
|
||||
|
||||
ARG VLLM_VERSION=v0.20.0
|
||||
ARG VLLM_VERSION=v0.19.0
|
||||
RUN git clone -b ${VLLM_VERSION} https://github.com/vllm-project/vllm
|
||||
WORKDIR /workspace/vllm
|
||||
|
||||
|
||||
@@ -13,7 +13,7 @@ vLLM-Omni current recommends the steps in under setup through Docker Images.
|
||||
|
||||
vLLM-Omni is built based on vLLM. Please install it with command below.
|
||||
```bash
|
||||
uv pip install vllm==0.20.0+rocm721 --extra-index-url https://wheels.vllm.ai/rocm/0.20.0/rocm721
|
||||
uv pip install vllm==0.20.0+rocm700 --extra-index-url https://wheels.vllm.ai/rocm/0.20.0/rocm700
|
||||
```
|
||||
|
||||
#### Installation of vLLM-Omni
|
||||
@@ -37,10 +37,10 @@ uv pip install onnxruntime-rocm
|
||||
If you do not need to modify source code of vLLM, you can directly install the stable 0.20.0 release version of the library
|
||||
|
||||
```bash
|
||||
uv pip install vllm==0.20.0+rocm721 --extra-index-url https://wheels.vllm.ai/rocm/0.20.0/rocm721
|
||||
uv pip install vllm==0.20.0+rocm700 --extra-index-url https://wheels.vllm.ai/rocm/0.20.0/rocm700
|
||||
```
|
||||
|
||||
The pre-built 0.20.0 vLLM wheel targets ROCm 7.2.1. If you need a different ROCm stack or want to reuse an existing PyTorch installation, build vLLM from source instead.
|
||||
The pre-built 0.20.0 vLLM wheel targets ROCm 7.0. If you need a different ROCm stack or want to reuse an existing PyTorch installation, build vLLM from source instead.
|
||||
|
||||
#### Installation of vLLM-Omni
|
||||
Since vllm-omni is rapidly evolving, it's recommended to install it from source
|
||||
|
||||
@@ -22,7 +22,7 @@ source .venv/bin/activate
|
||||
uv pip install vllm==0.20.0 --torch-backend=auto
|
||||
|
||||
# On ROCm
|
||||
uv pip install vllm==0.20.0+rocm721 --extra-index-url https://wheels.vllm.ai/rocm/0.20.0/rocm721
|
||||
uv pip install vllm==0.20.0+rocm700 --extra-index-url https://wheels.vllm.ai/rocm/0.20.0/rocm700
|
||||
|
||||
git clone https://github.com/vllm-project/vllm-omni.git
|
||||
cd vllm-omni
|
||||
@@ -31,12 +31,6 @@ uv pip install -e .
|
||||
|
||||
For additional installation methods — please see the [installation guide](installation/README.md).
|
||||
|
||||
|
||||
!!! note
|
||||
It is important to install the same major & minor version of vLLM and vLLM Omni, otherwise things may not work as expected. If the versions are misaligned, you will see a warning when you import vLLM Omni.
|
||||
|
||||
If you are seeing strange behavior with the `vllm` command not handling the `--omni` flag correctly, you most likely have a version mismatch with vLLM < `0.20.0` and vLLM Omni `0.20.0`, as vLLM Omni no longer hijacks the vLLM entrypoint. Updating vLLM should resolve this issue.
|
||||
|
||||
## Offline Inference
|
||||
|
||||
Text-to-image generation quickstart with vLLM-Omni:
|
||||
|
||||
@@ -118,6 +118,9 @@ def parse_args() -> argparse.Namespace:
|
||||
help='JSON profiler config for torch/cuda profiling, e.g. \'{"profiler":"torch","torch_profiler_dir":"./perf"}\'.',
|
||||
)
|
||||
|
||||
from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults
|
||||
|
||||
nullify_stage_engine_defaults(parser)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
|
||||
@@ -196,6 +196,9 @@ def parse_args() -> argparse.Namespace:
|
||||
parser.add_argument("--cfg-parallel-size", type=int, default=1, choices=[1, 2], help="CFG parallel size.")
|
||||
parser.add_argument("--tensor-parallel-size", type=int, default=1, help="Tensor parallelism size.")
|
||||
|
||||
from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults
|
||||
|
||||
nullify_stage_engine_defaults(parser)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
|
||||
@@ -378,6 +378,9 @@ def parse_args() -> argparse.Namespace:
|
||||
default=None,
|
||||
help='JSON profiler config for torch/cuda profiling, e.g. \'{"profiler":"torch","torch_profiler_dir":"./perf"}\'.',
|
||||
)
|
||||
from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults
|
||||
|
||||
nullify_stage_engine_defaults(parser)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
|
||||
@@ -211,6 +211,9 @@ def parse_args() -> argparse.Namespace:
|
||||
default=None,
|
||||
help='JSON profiler config for torch/cuda profiling, e.g. \'{"profiler":"torch","torch_profiler_dir":"./perf"}\'.',
|
||||
)
|
||||
from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults
|
||||
|
||||
nullify_stage_engine_defaults(parser)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
|
||||
@@ -24,6 +24,9 @@ def parse_args():
|
||||
parser.add_argument("--width", type=int, default=448, help="Video width.")
|
||||
parser.add_argument("--num-inference-steps", type=int, default=8, help="Number of denoising steps.")
|
||||
parser.add_argument("--seed", type=int, default=52, help="Random seed for generation.")
|
||||
from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults
|
||||
|
||||
nullify_stage_engine_defaults(parser)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
|
||||
@@ -9,7 +9,7 @@ from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
from vllm_omni.diffusion.data import logger
|
||||
from vllm_omni.diffusion.data import DiffusionParallelConfig, logger
|
||||
from vllm_omni.entrypoints.omni import Omni
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
from vllm_omni.lora.request import LoRARequest
|
||||
@@ -342,6 +342,17 @@ def main():
|
||||
# (e.g., QwenImagePipeline or FluxPipeline)
|
||||
}
|
||||
|
||||
# assert args.ring_degree == 1, "Ring attention is not supported yet"
|
||||
parallel_config = DiffusionParallelConfig(
|
||||
ulysses_degree=args.ulysses_degree,
|
||||
ring_degree=args.ring_degree,
|
||||
ulysses_mode=args.ulysses_mode,
|
||||
cfg_parallel_size=args.cfg_parallel_size,
|
||||
tensor_parallel_size=args.tensor_parallel_size,
|
||||
vae_patch_parallel_size=args.vae_patch_parallel_size,
|
||||
enable_expert_parallel=args.enable_expert_parallel,
|
||||
)
|
||||
|
||||
profiler_enabled = args.profiler_config is not None
|
||||
|
||||
# Prepare LoRA kwargs for Omni initialization
|
||||
@@ -377,13 +388,7 @@ def main():
|
||||
"cache_backend": args.cache_backend,
|
||||
"cache_config": cache_config,
|
||||
"enable_cache_dit_summary": args.enable_cache_dit_summary,
|
||||
"ulysses_degree": args.ulysses_degree,
|
||||
"ring_degree": args.ring_degree,
|
||||
"ulysses_mode": args.ulysses_mode,
|
||||
"cfg_parallel_size": args.cfg_parallel_size,
|
||||
"tensor_parallel_size": args.tensor_parallel_size,
|
||||
"vae_patch_parallel_size": args.vae_patch_parallel_size,
|
||||
"enable_expert_parallel": args.enable_expert_parallel,
|
||||
"parallel_config": parallel_config,
|
||||
"enforce_eager": args.enforce_eager,
|
||||
"enable_cpu_offload": args.enable_cpu_offload,
|
||||
"mode": "text-to-image",
|
||||
|
||||
@@ -226,6 +226,9 @@ def parse_args() -> argparse.Namespace:
|
||||
default=1,
|
||||
help="Number of HSDP replica groups.",
|
||||
)
|
||||
from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults
|
||||
|
||||
nullify_stage_engine_defaults(parser)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
|
||||
@@ -71,6 +71,9 @@ def parse_args() -> argparse.Namespace:
|
||||
parser.add_argument("--ulysses-degree", type=int, default=1, help="Ulysses SP degree.")
|
||||
parser.add_argument("--ring-degree", type=int, default=1, help="Ring attention degree.")
|
||||
parser.add_argument("--cfg-parallel-size", type=int, default=1, choices=[1, 2], help="CFG parallel size.")
|
||||
from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults
|
||||
|
||||
nullify_stage_engine_defaults(parser)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
|
||||
@@ -32,6 +32,7 @@ classifiers = [
|
||||
# Dependencies are now managed dynamically via setup.py based on detected hardware platform.
|
||||
# This allows automatic installation of the correct platform-specific dependencies (CUDA/ROCm/CPU/XPU/NPU)
|
||||
# without requiring extras like [cuda]. See requirements/ directory for platform-specific dependencies.
|
||||
# Note: vllm is intentionally excluded due to entrypoints overwrite issue.
|
||||
|
||||
[project.optional-dependencies]
|
||||
|
||||
@@ -89,6 +90,7 @@ Documentation = "https://vllm-omni.readthedocs.io"
|
||||
"Bug Tracker" = "https://github.com/vllm-project/vllm-omni/issues"
|
||||
|
||||
[project.scripts]
|
||||
vllm = "vllm_omni.entrypoints.cli.main:main"
|
||||
vllm-omni = "vllm_omni.entrypoints.cli.main:main"
|
||||
|
||||
|
||||
|
||||
@@ -121,8 +121,7 @@ class TestTrajectoryRecording:
|
||||
|
||||
assert trajectory_latents is not None
|
||||
assert trajectory_timesteps is not None
|
||||
# initial latent + one per denoising step
|
||||
assert len(trajectory_latents) == EXPECTED_STEPS + 1
|
||||
assert len(trajectory_latents) == EXPECTED_STEPS
|
||||
assert len(trajectory_timesteps) == EXPECTED_STEPS
|
||||
# log_probs is None without a scheduler (default ODE path)
|
||||
assert trajectory_log_probs is None
|
||||
@@ -168,54 +167,6 @@ class TestTrajectoryRecording:
|
||||
"Last trajectory latent should match the final output"
|
||||
)
|
||||
|
||||
def test_initial_latent_matches_input_noise(self, bagel_and_args):
|
||||
"""Regression: the first trajectory entry must be the initial noise (pre-denoising)."""
|
||||
bagel, args = bagel_and_args
|
||||
init_noise = args["packed_init_noises"].clone()
|
||||
|
||||
_, trajectory_latents, *_ = bagel.generate_image(**args, return_trajectory_latents=True)
|
||||
|
||||
assert torch.allclose(trajectory_latents[0], init_noise, atol=1e-6), (
|
||||
"trajectory_latents[0] should be the initial noise before any denoising step"
|
||||
)
|
||||
|
||||
def test_trajectory_timesteps_match_expected_schedule(self, bagel_and_args):
|
||||
"""Regression: trajectory_timesteps must record raw timesteps[i], not timesteps[i] - dts[i]."""
|
||||
bagel, args = bagel_and_args
|
||||
|
||||
# Recompute the expected timestep schedule (mirrors generate_image logic)
|
||||
ts = torch.linspace(1, 0, args["num_timesteps"])
|
||||
shift = args.get("timestep_shift", 1.0)
|
||||
ts = shift * ts / (1 + (shift - 1) * ts)
|
||||
expected_timesteps = ts[:-1] # last element is dropped
|
||||
|
||||
_, _, trajectory_timesteps, _ = bagel.generate_image(**args, return_trajectory_latents=True)
|
||||
|
||||
assert len(trajectory_timesteps) == len(expected_timesteps)
|
||||
for i, (actual, expected) in enumerate(zip(trajectory_timesteps, expected_timesteps)):
|
||||
assert abs(float(actual) - float(expected)) < 1e-6, (
|
||||
f"Step {i}: trajectory_timestep={float(actual):.6f}, expected={float(expected):.6f}"
|
||||
)
|
||||
|
||||
def test_latent_count_equals_timesteps_plus_one(self, bagel_and_args):
|
||||
"""Regression: len(trajectory_latents) == len(timesteps) + 1 (initial + one per step)."""
|
||||
bagel, args = bagel_and_args
|
||||
|
||||
# Compute the number of denoising steps the same way generate_image does
|
||||
ts = torch.linspace(1, 0, args["num_timesteps"])
|
||||
shift = args.get("timestep_shift", 1.0)
|
||||
ts = shift * ts / (1 + (shift - 1) * ts)
|
||||
num_steps = len(ts) - 1 # timesteps = ts[:-1]
|
||||
|
||||
_, trajectory_latents, trajectory_timesteps, _ = bagel.generate_image(**args, return_trajectory_latents=True)
|
||||
|
||||
assert len(trajectory_latents) == num_steps + 1, (
|
||||
f"Expected {num_steps + 1} latents (initial + {num_steps} steps), got {len(trajectory_latents)}"
|
||||
)
|
||||
assert len(trajectory_timesteps) == num_steps, (
|
||||
f"Expected {num_steps} timesteps, got {len(trajectory_timesteps)}"
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Mock scheduler for log-prob tests
|
||||
|
||||
@@ -356,6 +356,7 @@ def test_mix_to_text_audio_001(omni_server, openai_client) -> None:
|
||||
|
||||
@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
|
||||
@pytest.mark.parametrize("omni_server", test_params, indirect=True)
|
||||
@pytest.mark.skip(reason="issue: #2827")
|
||||
def test_audio_in_video_001(omni_server, openai_client) -> None:
|
||||
"""
|
||||
Input Modal: text + video (synthetic MP4 with embedded audio; ``use_audio_in_video`` uses audio from the video).
|
||||
@@ -375,7 +376,7 @@ def test_audio_in_video_001(omni_server, openai_client) -> None:
|
||||
"messages": messages,
|
||||
"stream": False,
|
||||
"use_audio_in_video": True,
|
||||
"key_words": {"video": VIDEO_KEY},
|
||||
"key_words": {"video": VIDEO_KEY, "audio": AUDIO_KEY + ["beep", "electronic"]},
|
||||
}
|
||||
openai_client.send_omni_request(request_config)
|
||||
|
||||
|
||||
@@ -98,7 +98,7 @@ def _get_diffusion_feature_cases():
|
||||
],
|
||||
),
|
||||
id="layerwise_hsdp",
|
||||
marks=[*FOUR_CARD_MARKS],
|
||||
marks=[*FOUR_CARD_MARKS, pytest.mark.skip(reason="issue #2435")],
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
@@ -4,7 +4,6 @@
|
||||
import pytest
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
|
||||
from vllm_omni.config.stage_config import deploy_override_field_names
|
||||
from vllm_omni.engine.async_omni_engine import AsyncOmniEngine
|
||||
from vllm_omni.entrypoints.cli.serve import OmniServeCommand, _create_default_diffusion_stage_cfg
|
||||
|
||||
@@ -31,15 +30,6 @@ def test_default_stage_config_includes_cache_backend():
|
||||
assert engine_args["model_stage"] == "diffusion"
|
||||
|
||||
|
||||
def test_default_stage_config_ignores_none_deploy_overrides():
|
||||
"""Ensure nullified deploy override defaults do not alter diffusion defaults."""
|
||||
baseline = AsyncOmniEngine._create_default_diffusion_stage_cfg({})[0]
|
||||
nullified_overrides = {name: None for name in deploy_override_field_names()}
|
||||
stage_cfg = AsyncOmniEngine._create_default_diffusion_stage_cfg(nullified_overrides)[0]
|
||||
|
||||
assert stage_cfg == baseline
|
||||
|
||||
|
||||
def test_default_cache_config_used_when_missing():
|
||||
"""Ensure default cache_config is synthesized when only backend is given."""
|
||||
stage_cfg = AsyncOmniEngine._create_default_diffusion_stage_cfg(
|
||||
@@ -65,44 +55,6 @@ def test_default_stage_devices_from_sequence_parallel():
|
||||
assert stage_cfg["runtime"]["devices"] == "0,1,2,3"
|
||||
|
||||
|
||||
def test_default_stage_config_uses_parallel_size_kwargs():
|
||||
"""Ensure default diffusion parallel_config uses CLI/API parallel sizes."""
|
||||
stage_cfg = AsyncOmniEngine._create_default_diffusion_stage_cfg(
|
||||
{
|
||||
"pipeline_parallel_size": 2,
|
||||
"data_parallel_size": 3,
|
||||
"tensor_parallel_size": 4,
|
||||
"enable_expert_parallel": True,
|
||||
}
|
||||
)[0]
|
||||
|
||||
parallel_config = stage_cfg["engine_args"]["parallel_config"]
|
||||
assert parallel_config.pipeline_parallel_size == 2
|
||||
assert parallel_config.data_parallel_size == 3
|
||||
assert parallel_config.tensor_parallel_size == 4
|
||||
assert parallel_config.enable_expert_parallel is True
|
||||
|
||||
|
||||
def test_default_stage_config_defaults_nullified_parallel_size_kwargs():
|
||||
"""Ensure nullified diffusion parallel-size kwargs fall back to defaults."""
|
||||
stage_cfg = AsyncOmniEngine._create_default_diffusion_stage_cfg(
|
||||
{
|
||||
"pipeline_parallel_size": None,
|
||||
"data_parallel_size": None,
|
||||
"tensor_parallel_size": None,
|
||||
"enable_expert_parallel": None,
|
||||
"enforce_eager": None,
|
||||
}
|
||||
)[0]
|
||||
|
||||
parallel_config = stage_cfg["engine_args"]["parallel_config"]
|
||||
assert parallel_config.pipeline_parallel_size == 1
|
||||
assert parallel_config.data_parallel_size == 1
|
||||
assert parallel_config.tensor_parallel_size == 1
|
||||
assert parallel_config.enable_expert_parallel is False
|
||||
assert stage_cfg["engine_args"]["enforce_eager"] is False
|
||||
|
||||
|
||||
def test_default_stage_config_propagates_ulysses_mode():
|
||||
"""Ensure UAA mode survives default diffusion-stage creation."""
|
||||
stage_cfg = AsyncOmniEngine._create_default_diffusion_stage_cfg(
|
||||
|
||||
@@ -26,9 +26,27 @@ def _skip_readme_snippet(language: str, code: str, h2_title: str) -> tuple[bool,
|
||||
|
||||
README_SNIPPETS = ReadmeSnippet.extract_readme_snippets(README_PATH, skipif=_skip_readme_snippet)
|
||||
|
||||
# Track: https://github.com/vllm-project/vllm-omni/issues/3123 — CLI example subprocess exit 1 in CI; re-enable when fixed.
|
||||
_SKIP_T2I_SNIPPET_IDS = frozenset(
|
||||
{
|
||||
"quick_start_002",
|
||||
"more_cli_examples_001",
|
||||
"more_cli_examples_002",
|
||||
"more_cli_examples_003",
|
||||
"more_cli_examples_007",
|
||||
"more_cli_examples_008",
|
||||
}
|
||||
)
|
||||
_SKIP_T2I_REASON = (
|
||||
"Skipped: text_to_image README examples failing in CI (subprocess exit 1). "
|
||||
"See https://github.com/vllm-project/vllm-omni/issues/3123"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("snippet", README_SNIPPETS, ids=lambda snippet: snippet.test_id)
|
||||
def test_text_to_image(snippet: ReadmeSnippet, example_runner: ExampleRunner):
|
||||
if snippet.test_id in _SKIP_T2I_SNIPPET_IDS:
|
||||
pytest.skip(_SKIP_T2I_REASON)
|
||||
should_skip, reason = snippet.skip
|
||||
if should_skip:
|
||||
pytest.skip(reason)
|
||||
|
||||
@@ -3,7 +3,14 @@
|
||||
"""Pytest marks and decorators for hardware / resource selection (CUDA, ROCm, …)."""
|
||||
|
||||
import pytest
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
try:
|
||||
from vllm.utils.torch_utils import cuda_device_count_stateless
|
||||
except ImportError:
|
||||
import torch
|
||||
|
||||
def cuda_device_count_stateless() -> int:
|
||||
return torch.cuda.device_count()
|
||||
|
||||
# Re-exported from tests.helpers.env (GPU wait + DeviceMemoryMonitor).
|
||||
|
||||
@@ -20,9 +27,8 @@ def cuda_marks(*, res: str, num_cards: int):
|
||||
if num_cards == 1:
|
||||
return marks
|
||||
test_distributed = pytest.mark.distributed_cuda(num_cards=num_cards)
|
||||
|
||||
test_skipif = pytest.mark.skipif_cuda(
|
||||
not current_platform.is_cuda() or (current_platform.device_count() < num_cards),
|
||||
cuda_device_count_stateless() < num_cards,
|
||||
reason=f"Need at least {num_cards} CUDA GPUs to run the test.",
|
||||
)
|
||||
return marks + [test_distributed, test_skipif]
|
||||
|
||||
@@ -494,7 +494,6 @@ _CI_OVERLAYS: dict[str, dict[str, Any]] = {
|
||||
"max_num_seqs": 1,
|
||||
"gpu_memory_utilization": 0.9,
|
||||
"enforce_eager": True,
|
||||
"enable_prefix_caching": False,
|
||||
"max_num_batched_tokens": 16384,
|
||||
"max_model_len": 16384,
|
||||
"skip_mm_profiling": True,
|
||||
|
||||
@@ -369,8 +369,8 @@ def _build_full_serve_parser():
|
||||
def test_nullify_stage_engine_defaults_resets_inherited_defaults():
|
||||
import argparse
|
||||
|
||||
from vllm_omni.config.stage_config import deploy_override_field_names
|
||||
from vllm_omni.engine.arg_utils import (
|
||||
deploy_override_field_names,
|
||||
nullify_stage_engine_defaults,
|
||||
)
|
||||
|
||||
|
||||
@@ -93,7 +93,6 @@ class TestStageConfig:
|
||||
assert omega_config.engine_args.worker_type == "ar"
|
||||
assert omega_config.final_output is True
|
||||
assert omega_config.final_output_type == "text"
|
||||
assert "max_num_seqs" not in omega_config.engine_args
|
||||
# Legacy field name for backward compatibility
|
||||
assert omega_config.engine_input_source == []
|
||||
|
||||
@@ -147,24 +146,6 @@ class TestStageConfig:
|
||||
omega_config = config.to_omegaconf()
|
||||
assert omega_config.engine_args.max_num_seqs == 32
|
||||
|
||||
def test_to_omegaconf_omits_none_deploy_overrides_for_engine_args(self):
|
||||
"""None deploy overrides must fall through to EngineArgs defaults."""
|
||||
from vllm_omni.config.stage_config import deploy_override_field_names
|
||||
|
||||
config = StageConfig(
|
||||
stage_id=0,
|
||||
model_stage="thinker",
|
||||
runtime_overrides={name: None for name in deploy_override_field_names()},
|
||||
)
|
||||
|
||||
omega_config = config.to_omegaconf()
|
||||
engine_args = dict(omega_config.engine_args)
|
||||
|
||||
assert "devices" not in engine_args
|
||||
assert "max_batch_size" not in engine_args
|
||||
for name in deploy_override_field_names() - {"devices"}:
|
||||
assert name not in engine_args
|
||||
|
||||
|
||||
class TestModelPipeline:
|
||||
"""Tests for ModelPipeline class."""
|
||||
@@ -821,40 +802,6 @@ class TestPipelineRegistry:
|
||||
|
||||
|
||||
class TestDeployConfigLoading:
|
||||
def test_deploy_override_fields_include_deploy_schema_fields(self):
|
||||
from vllm_omni.config.stage_config import deploy_override_field_names
|
||||
|
||||
expected_fields = {
|
||||
"async_chunk",
|
||||
"async_scheduling",
|
||||
"config_format",
|
||||
"data_parallel_size",
|
||||
"devices",
|
||||
"disable_hybrid_kv_cache_manager",
|
||||
"distributed_executor_backend",
|
||||
"dtype",
|
||||
"enable_chunked_prefill",
|
||||
"enable_flashinfer_autotune",
|
||||
"enable_prefix_caching",
|
||||
"enforce_eager",
|
||||
"gpu_memory_utilization",
|
||||
"load_format",
|
||||
"max_model_len",
|
||||
"max_num_batched_tokens",
|
||||
"max_num_seqs",
|
||||
"mm_processor_cache_gb",
|
||||
"pipeline_parallel_size",
|
||||
"profiler_config",
|
||||
"quantization",
|
||||
"skip_mm_profiling",
|
||||
"subtalker_sampling_params",
|
||||
"tensor_parallel_size",
|
||||
"tokenizer_mode",
|
||||
"trust_remote_code",
|
||||
}
|
||||
|
||||
assert expected_fields == deploy_override_field_names()
|
||||
|
||||
def test_load_deploy_config(self):
|
||||
from pathlib import Path
|
||||
|
||||
@@ -870,17 +817,6 @@ class TestDeployConfigLoading:
|
||||
assert deploy.connectors is not None
|
||||
assert deploy.platforms is not None
|
||||
|
||||
voxtral_path = Path(__file__).parent.parent / "vllm_omni" / "deploy" / "voxtral_tts.yaml"
|
||||
if voxtral_path.exists():
|
||||
voxtral_deploy = load_deploy_config(voxtral_path)
|
||||
assert voxtral_deploy.stages[0].config_format == "mistral"
|
||||
assert voxtral_deploy.stages[0].load_format == "mistral"
|
||||
assert voxtral_deploy.stages[0].tokenizer_mode == "mistral"
|
||||
assert not any(
|
||||
name in voxtral_deploy.stages[0].engine_extras
|
||||
for name in ("config_format", "load_format", "tokenizer_mode")
|
||||
)
|
||||
|
||||
def test_merge_pipeline_deploy(self):
|
||||
from pathlib import Path
|
||||
|
||||
@@ -1075,8 +1011,7 @@ class TestBaseConfigInheritance:
|
||||
deploy = load_deploy_config(ci_path)
|
||||
assert len(deploy.stages) == 3
|
||||
# CI overrides
|
||||
assert deploy.stages[0].load_format == "dummy"
|
||||
assert "load_format" not in deploy.stages[0].engine_extras
|
||||
assert deploy.stages[0].engine_extras.get("load_format") == "dummy"
|
||||
assert deploy.stages[0].max_num_seqs == 5
|
||||
# Inherited from base
|
||||
assert deploy.stages[0].gpu_memory_utilization == 0.9
|
||||
@@ -1281,7 +1216,7 @@ class TestSentinelDefaultPrecedence:
|
||||
def test_none_value_skipped_yaml_wins(self):
|
||||
stages = self._stages({"max_num_seqs": None})
|
||||
assert stages[2].runtime_overrides.get("max_num_seqs") is None
|
||||
assert "max_num_seqs" not in stages[2].yaml_engine_args
|
||||
assert stages[2].yaml_engine_args.get("max_num_seqs") == 1
|
||||
|
||||
def test_empty_kwargs_yaml_only(self):
|
||||
stages = self._stages({})
|
||||
|
||||
@@ -402,11 +402,11 @@ class StageDeployConfig:
|
||||
"""
|
||||
|
||||
stage_id: int
|
||||
max_num_seqs: int | None = None
|
||||
gpu_memory_utilization: float | None = None
|
||||
tensor_parallel_size: int | None = None
|
||||
enforce_eager: bool | None = None
|
||||
max_num_batched_tokens: int | None = None
|
||||
max_num_seqs: int = 64
|
||||
gpu_memory_utilization: float = 0.9
|
||||
tensor_parallel_size: int = 1
|
||||
enforce_eager: bool = False
|
||||
max_num_batched_tokens: int = 32768
|
||||
max_model_len: int | None = None
|
||||
async_scheduling: bool | None = None
|
||||
devices: str = "0"
|
||||
@@ -414,14 +414,6 @@ class StageDeployConfig:
|
||||
input_connectors: dict[str, str] | None = None
|
||||
default_sampling_params: dict[str, Any] | None = None
|
||||
subtalker_sampling_params: dict[str, Any] | None = None
|
||||
profiler_config: dict[str, Any] | None = None
|
||||
disable_hybrid_kv_cache_manager: bool | None = None
|
||||
mm_processor_cache_gb: float | None = None
|
||||
skip_mm_profiling: bool | None = None
|
||||
enable_flashinfer_autotune: bool | None = None
|
||||
config_format: str | None = None
|
||||
load_format: str | None = None
|
||||
tokenizer_mode: str | None = None
|
||||
engine_extras: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
|
||||
@@ -446,14 +438,14 @@ class DeployConfig:
|
||||
pipeline: str | None = None
|
||||
|
||||
# === Pipeline-wide engine settings (applied uniformly to every stage) ===
|
||||
trust_remote_code: bool | None = None
|
||||
trust_remote_code: bool = True
|
||||
distributed_executor_backend: str | None = None
|
||||
dtype: str | None = None
|
||||
quantization: str | None = None
|
||||
enable_prefix_caching: bool | None = None
|
||||
enable_prefix_caching: bool = False
|
||||
enable_chunked_prefill: bool | None = None
|
||||
data_parallel_size: int | None = None
|
||||
pipeline_parallel_size: int | None = None
|
||||
data_parallel_size: int = 1
|
||||
pipeline_parallel_size: int = 1
|
||||
|
||||
|
||||
_STAGE_NON_ENGINE_KEYS = frozenset(
|
||||
@@ -697,18 +689,6 @@ _PIPELINE_WIDE_ENGINE_FIELDS: tuple[str, ...] = (
|
||||
)
|
||||
|
||||
|
||||
def deploy_override_field_names() -> frozenset[str]:
|
||||
"""Return deploy-schema fields whose CLI defaults must not override YAML."""
|
||||
return (
|
||||
frozenset(_STAGE_DEPLOY_FIELDS)
|
||||
| frozenset(_PIPELINE_WIDE_ENGINE_FIELDS)
|
||||
| {
|
||||
"async_chunk",
|
||||
"devices",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _build_engine_args(
|
||||
ps: StagePipelineConfig,
|
||||
ds: StageDeployConfig | None,
|
||||
@@ -881,15 +861,13 @@ class StageConfig:
|
||||
|
||||
# CLI overrides take precedence over YAML defaults
|
||||
for key, value in self.runtime_overrides.items():
|
||||
if value is None:
|
||||
continue
|
||||
if key not in ("devices", "max_batch_size"):
|
||||
engine_args[key] = value
|
||||
|
||||
# Build runtime config from YAML defaults + CLI overrides
|
||||
runtime: dict[str, Any] = dict(self.yaml_runtime)
|
||||
runtime.setdefault("process", True)
|
||||
if self.runtime_overrides.get("devices") is not None:
|
||||
if "devices" in self.runtime_overrides:
|
||||
runtime["devices"] = self.runtime_overrides["devices"]
|
||||
|
||||
# Legacy compat: migrate runtime.max_batch_size → engine_args.max_num_seqs
|
||||
@@ -905,6 +883,8 @@ class StageConfig:
|
||||
effective_mbs = int(cli_mbs or legacy_mbs or 1)
|
||||
engine_args.setdefault("max_num_seqs", effective_mbs)
|
||||
|
||||
engine_args.setdefault("max_num_seqs", 1)
|
||||
|
||||
# Build full config dict
|
||||
config_dict: dict[str, Any] = {
|
||||
"stage_id": self.stage_id,
|
||||
@@ -1029,8 +1009,6 @@ class StageConfigFactory:
|
||||
cli_overrides = {}
|
||||
|
||||
trust_remote_code = cli_overrides.get("trust_remote_code", True)
|
||||
if trust_remote_code is None:
|
||||
trust_remote_code = False
|
||||
|
||||
# --- New path: check pipeline registry by model_type first ---
|
||||
model_type, hf_config = cls._auto_detect_model_type(model, trust_remote_code=trust_remote_code)
|
||||
@@ -1374,8 +1352,8 @@ class StageConfigFactory:
|
||||
|
||||
hf_config = get_config(model, trust_remote_code=trust_remote_code)
|
||||
return hf_config.model_type, hf_config
|
||||
except Exception as e:
|
||||
logger.debug(f"`get_config` failed for {e}; Falling back to raw config.json path")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Fallback: read config.json directly for custom model types that
|
||||
# are not registered with transformers (e.g. qwen3_tts).
|
||||
@@ -1412,8 +1390,8 @@ class StageConfigFactory:
|
||||
class_name,
|
||||
)
|
||||
return pipeline_cfg.model_type, None
|
||||
except Exception as e:
|
||||
logger.debug(f"Failed to detect model type for diffusers-style models: {e}")
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Final fallback: some models (e.g. CosyVoice3) ship an empty
|
||||
# config.json and rely on naming conventions. Match the model path
|
||||
|
||||
@@ -40,7 +40,7 @@ stages:
|
||||
# near-identity repetition penalty forces vLLM to track
|
||||
# output_token_ids for RAS (stop-token logit logsumexp).
|
||||
repetition_penalty: 1.0001
|
||||
enable_prefix_caching: false
|
||||
disable_hybrid_kv_cache_manager: true
|
||||
mm_processor_cache_gb: 0
|
||||
skip_mm_profiling: true
|
||||
|
||||
@@ -54,5 +54,5 @@ stages:
|
||||
from_stage_0: connector_of_shared_memory
|
||||
default_sampling_params:
|
||||
max_tokens: 2048
|
||||
enable_prefix_caching: false
|
||||
disable_hybrid_kv_cache_manager: true
|
||||
skip_mm_profiling: true
|
||||
@@ -42,7 +42,6 @@ stages:
|
||||
max_num_seqs: 1
|
||||
gpu_memory_utilization: 0.2
|
||||
enforce_eager: true
|
||||
enable_prefix_caching: false
|
||||
async_scheduling: false
|
||||
max_num_batched_tokens: 8192
|
||||
max_model_len: 8192
|
||||
|
||||
@@ -22,7 +22,6 @@ stages:
|
||||
max_num_seqs: 1
|
||||
gpu_memory_utilization: 0.8
|
||||
enforce_eager: true
|
||||
enable_prefix_caching: false
|
||||
mm_processor_cache_gb: 0
|
||||
devices: "0"
|
||||
default_sampling_params:
|
||||
@@ -50,7 +49,6 @@ stages:
|
||||
max_num_seqs: 1
|
||||
gpu_memory_utilization: 0.15
|
||||
enforce_eager: true
|
||||
enable_prefix_caching: false
|
||||
enable_flashinfer_autotune: false
|
||||
async_scheduling: false
|
||||
devices: "0"
|
||||
|
||||
@@ -23,7 +23,6 @@ connectors:
|
||||
stages:
|
||||
- stage_id: 0
|
||||
gpu_memory_utilization: 0.9
|
||||
enable_prefix_caching: false
|
||||
devices: "0"
|
||||
default_sampling_params:
|
||||
temperature: 0.4
|
||||
@@ -48,7 +47,6 @@ stages:
|
||||
- stage_id: 2
|
||||
gpu_memory_utilization: 0.1
|
||||
enforce_eager: true
|
||||
enable_prefix_caching: false
|
||||
async_scheduling: false
|
||||
max_num_batched_tokens: 51200
|
||||
devices: "1"
|
||||
|
||||
@@ -53,7 +53,6 @@ stages:
|
||||
max_num_seqs: 1
|
||||
gpu_memory_utilization: 0.3
|
||||
enforce_eager: true
|
||||
enable_prefix_caching: false
|
||||
async_scheduling: true
|
||||
# Must be divisible by num_code_groups and cover (left_context + chunk).
|
||||
# Prefill length is Q * num_frames (e.g. 16 * 2148 = 34368); keep
|
||||
|
||||
@@ -20,7 +20,6 @@ stages:
|
||||
max_num_batched_tokens: 4096
|
||||
max_model_len: 4096
|
||||
devices: "0"
|
||||
trust_remote_code: true
|
||||
default_sampling_params:
|
||||
temperature: 0.0
|
||||
top_p: 1.0
|
||||
|
||||
@@ -24,7 +24,6 @@ stages:
|
||||
max_num_seqs: 32
|
||||
gpu_memory_utilization: 0.8
|
||||
enforce_eager: false
|
||||
enable_prefix_caching: false
|
||||
async_scheduling: true
|
||||
max_model_len: 4096
|
||||
devices: "0"
|
||||
@@ -49,7 +48,6 @@ stages:
|
||||
max_num_seqs: 32
|
||||
gpu_memory_utilization: 0.1
|
||||
enforce_eager: true
|
||||
enable_prefix_caching: false
|
||||
async_scheduling: false
|
||||
max_num_batched_tokens: 65536
|
||||
max_model_len: 65536
|
||||
|
||||
@@ -385,9 +385,9 @@ class MultiprocDiffusionExecutor(DiffusionExecutor):
|
||||
raise
|
||||
|
||||
def check_health(self) -> None:
|
||||
self._ensure_open()
|
||||
if self.is_failed:
|
||||
raise EngineDeadError()
|
||||
self._ensure_open()
|
||||
for p in self._processes:
|
||||
if not p.is_alive():
|
||||
self.is_failed = True
|
||||
|
||||
@@ -15,7 +15,6 @@ from typing import TYPE_CHECKING, Any
|
||||
import torch
|
||||
from PIL import Image
|
||||
from vllm.logger import init_logger
|
||||
from vllm.v1.engine.exceptions import EngineDeadError
|
||||
|
||||
from vllm_omni.diffusion.data import DiffusionRequestAbortedError
|
||||
from vllm_omni.diffusion.diffusion_engine import DiffusionEngine
|
||||
@@ -330,12 +329,6 @@ class InlineStageDiffusionClient:
|
||||
None,
|
||||
)
|
||||
|
||||
def check_health(self) -> None:
|
||||
"""Check if the inline diffusion engine and its workers are healthy."""
|
||||
if self._shutting_down:
|
||||
raise EngineDeadError("InlineStageDiffusionClient is shutting down")
|
||||
self._engine.executor.check_health()
|
||||
|
||||
def shutdown(self) -> None:
|
||||
self._shutting_down = True
|
||||
|
||||
|
||||
@@ -1842,8 +1842,6 @@ class Bagel(nn.Module):
|
||||
# ── SP + CFG: sequential single-branch forwards ──
|
||||
use_sp = self._sp_size > 1
|
||||
if use_sp and use_cfg_text:
|
||||
if return_trajectory_latents and len(timesteps) > 0:
|
||||
trajectory_latents.append(x_t.clone())
|
||||
for i, t in enumerate(timesteps):
|
||||
timestep = torch.tensor([t] * x_t.shape[0], device=x_t.device)
|
||||
in_cfg_window = t > cfg_interval[0] and t <= cfg_interval[1]
|
||||
@@ -1907,15 +1905,13 @@ class Bagel(nn.Module):
|
||||
x_t = x_t - v_t.to(x_t.device) * dts[i]
|
||||
if return_trajectory_latents:
|
||||
trajectory_latents.append(x_t.clone())
|
||||
trajectory_timesteps.append(timesteps[i])
|
||||
trajectory_timesteps.append(timesteps[i] - dts[i])
|
||||
|
||||
unpacked_latent = x_t.split((packed_seqlens - 2).tolist())
|
||||
return unpacked_latent, trajectory_latents, trajectory_timesteps, trajectory_log_probs
|
||||
|
||||
# ── SP without CFG: direct single-branch loop ──
|
||||
if use_sp:
|
||||
if return_trajectory_latents and len(timesteps) > 0:
|
||||
trajectory_latents.append(x_t.clone())
|
||||
for i, t in enumerate(timesteps):
|
||||
timestep = torch.tensor([t] * x_t.shape[0], device=x_t.device)
|
||||
v_t = self.forward_single_branch(
|
||||
@@ -1942,7 +1938,7 @@ class Bagel(nn.Module):
|
||||
x_t = x_t - v_t.to(x_t.device) * dts[i]
|
||||
if return_trajectory_latents:
|
||||
trajectory_latents.append(x_t.clone())
|
||||
trajectory_timesteps.append(timesteps[i])
|
||||
trajectory_timesteps.append(timesteps[i] - dts[i])
|
||||
|
||||
unpacked_latent = x_t.split((packed_seqlens - 2).tolist())
|
||||
return unpacked_latent, trajectory_latents, trajectory_timesteps, trajectory_log_probs
|
||||
@@ -2003,9 +1999,6 @@ class Bagel(nn.Module):
|
||||
"merged_cache": self._merge_naive_caches(branches_cache),
|
||||
}
|
||||
|
||||
if return_trajectory_latents and len(timesteps) > 0:
|
||||
trajectory_latents.append(x_t.clone())
|
||||
|
||||
for i, t in enumerate(timesteps):
|
||||
timestep = torch.tensor([t] * x_t.shape[0], device=x_t.device)
|
||||
if t > cfg_interval[0] and t <= cfg_interval[1]:
|
||||
@@ -2043,7 +2036,7 @@ class Bagel(nn.Module):
|
||||
x_t = x_t - v_t.to(x_t.device) * dts[i] # velocity pointing from data to noise
|
||||
if return_trajectory_latents:
|
||||
trajectory_latents.append(x_t.clone())
|
||||
trajectory_timesteps.append(timesteps[i])
|
||||
trajectory_timesteps.append(timesteps[i] - dts[i])
|
||||
|
||||
unpacked_latent = x_t.split((packed_seqlens - 2).tolist())
|
||||
return unpacked_latent, trajectory_latents, trajectory_timesteps, trajectory_log_probs
|
||||
@@ -2144,9 +2137,6 @@ class Bagel(nn.Module):
|
||||
)
|
||||
_sched_kw = scheduler_kwargs or {}
|
||||
|
||||
if return_trajectory_latents and len(timesteps) > 0:
|
||||
trajectory_latents.append(x_t.clone())
|
||||
|
||||
for i, t in enumerate(timesteps):
|
||||
timestep = torch.tensor([t] * x_t.shape[0], device=x_t.device)
|
||||
use_cfg_this_step = t > cfg_interval[0] and t <= cfg_interval[1] and cfg_text_scale > 1.0
|
||||
@@ -2204,7 +2194,7 @@ class Bagel(nn.Module):
|
||||
x_t = x_t - v_t.to(x_t.device) * dts[i]
|
||||
if return_trajectory_latents:
|
||||
trajectory_latents.append(x_t.clone())
|
||||
trajectory_timesteps.append(timesteps[i])
|
||||
trajectory_timesteps.append(timesteps[i] - dts[i])
|
||||
|
||||
unpacked_latent = x_t.split((packed_seqlens - 2).tolist())
|
||||
return unpacked_latent, trajectory_latents, trajectory_timesteps, trajectory_log_probs
|
||||
|
||||
@@ -456,12 +456,50 @@ SHARED_FIELDS: frozenset[str] = frozenset(
|
||||
}
|
||||
)
|
||||
|
||||
_DEPLOY_ENGINE_ARG_OVERRIDE_FIELDS: frozenset[str] = frozenset(
|
||||
{
|
||||
# Capacity / scheduling.
|
||||
"async_scheduling",
|
||||
"max_model_len",
|
||||
"max_num_batched_tokens",
|
||||
"max_num_seqs",
|
||||
# Memory / parallelism.
|
||||
"data_parallel_size",
|
||||
"gpu_memory_utilization",
|
||||
"pipeline_parallel_size",
|
||||
"tensor_parallel_size",
|
||||
# Execution / loading.
|
||||
"enforce_eager",
|
||||
"distributed_executor_backend",
|
||||
"dtype",
|
||||
"quantization",
|
||||
"trust_remote_code",
|
||||
# Caching / chunking.
|
||||
"async_chunk",
|
||||
"enable_prefix_caching",
|
||||
"enable_chunked_prefill",
|
||||
# Model-specific engine extras.
|
||||
"subtalker_sampling_params",
|
||||
}
|
||||
)
|
||||
|
||||
_DEPLOY_RUNTIME_OVERRIDE_FIELDS: frozenset[str] = frozenset(
|
||||
{
|
||||
"devices",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def orchestrator_field_names() -> frozenset[str]:
|
||||
"""Return the names of every field on OrchestratorArgs."""
|
||||
return frozenset(f.name for f in fields(OrchestratorArgs))
|
||||
|
||||
|
||||
def deploy_override_field_names() -> frozenset[str]:
|
||||
"""Return kwargs whose parser defaults must not override deploy YAML."""
|
||||
return _DEPLOY_ENGINE_ARG_OVERRIDE_FIELDS | _DEPLOY_RUNTIME_OVERRIDE_FIELDS
|
||||
|
||||
|
||||
def internal_blacklist_keys() -> frozenset[str]:
|
||||
"""Return the set of CLI keys that must never be forwarded as per-stage
|
||||
engine overrides.
|
||||
@@ -615,8 +653,6 @@ def nullify_stage_engine_defaults(parser: argparse.ArgumentParser) -> None:
|
||||
"""Reset stage-level engine flag defaults to ``None``; preserve real
|
||||
default in help text. Only deploy-YAML override fields are touched.
|
||||
Idempotent."""
|
||||
from vllm_omni.config.stage_config import deploy_override_field_names
|
||||
|
||||
override_dests = deploy_override_field_names()
|
||||
|
||||
for action in parser._actions:
|
||||
|
||||
@@ -1294,12 +1294,9 @@ class AsyncOmniEngine:
|
||||
ring_degree = normalized_kwargs.get("ring_degree") or 1
|
||||
ulysses_mode = normalized_kwargs.get("ulysses_mode") or "strict"
|
||||
sequence_parallel_size = normalized_kwargs.get("sequence_parallel_size")
|
||||
pipeline_parallel_size = normalized_kwargs.get("pipeline_parallel_size") or 1
|
||||
data_parallel_size = normalized_kwargs.get("data_parallel_size") or 1
|
||||
tensor_parallel_size = normalized_kwargs.get("tensor_parallel_size") or 1
|
||||
cfg_parallel_size = normalized_kwargs.get("cfg_parallel_size") or 1
|
||||
vae_patch_parallel_size = normalized_kwargs.get("vae_patch_parallel_size") or 1
|
||||
enable_expert_parallel = normalized_kwargs.get("enable_expert_parallel") or False
|
||||
use_hsdp = normalized_kwargs.get("use_hsdp", False)
|
||||
hsdp_shard_size = normalized_kwargs.get("hsdp_shard_size", -1)
|
||||
hsdp_replicate_size = normalized_kwargs.get("hsdp_replicate_size", 1)
|
||||
@@ -1307,10 +1304,9 @@ class AsyncOmniEngine:
|
||||
sequence_parallel_size = ulysses_degree * ring_degree
|
||||
|
||||
parallel_config = DiffusionParallelConfig(
|
||||
pipeline_parallel_size=pipeline_parallel_size,
|
||||
data_parallel_size=data_parallel_size,
|
||||
pipeline_parallel_size=1,
|
||||
data_parallel_size=1,
|
||||
tensor_parallel_size=tensor_parallel_size,
|
||||
enable_expert_parallel=enable_expert_parallel,
|
||||
sequence_parallel_size=sequence_parallel_size,
|
||||
ulysses_degree=ulysses_degree,
|
||||
ring_degree=ring_degree,
|
||||
@@ -1324,9 +1320,6 @@ class AsyncOmniEngine:
|
||||
|
||||
num_devices = max(1, int(parallel_config.world_size))
|
||||
devices = ",".join(str(i) for i in range(num_devices))
|
||||
enforce_eager = kwargs.get("enforce_eager")
|
||||
if enforce_eager is None:
|
||||
enforce_eager = False
|
||||
|
||||
stage_engine_args = {
|
||||
"max_num_seqs": 1,
|
||||
@@ -1340,7 +1333,7 @@ class AsyncOmniEngine:
|
||||
"enable_cache_dit_summary": kwargs.get("enable_cache_dit_summary", False),
|
||||
"enable_cpu_offload": kwargs.get("enable_cpu_offload", False),
|
||||
"enable_layerwise_offload": kwargs.get("enable_layerwise_offload", False),
|
||||
"enforce_eager": enforce_eager,
|
||||
"enforce_eager": kwargs.get("enforce_eager", False),
|
||||
"boundary_ratio": kwargs.get("boundary_ratio", None),
|
||||
"flow_shift": kwargs.get("flow_shift", None),
|
||||
"diffusion_load_format": kwargs.get("diffusion_load_format", "default"),
|
||||
|
||||
@@ -100,7 +100,9 @@ class OmniBase(PDDisaggregationMixin):
|
||||
kwargs: dict[str, Any] = {k: v for k, v in vars(args).items() if not k.startswith("_")}
|
||||
|
||||
if parser is not None and not getattr(parser, "_omni_nullified", False):
|
||||
from vllm_omni.config.stage_config import deploy_override_field_names
|
||||
from vllm_omni.engine.arg_utils import (
|
||||
deploy_override_field_names,
|
||||
)
|
||||
from vllm_omni.entrypoints.utils import detect_explicit_cli_keys
|
||||
|
||||
explicit = detect_explicit_cli_keys(sys.argv[1:], parser) or set()
|
||||
|
||||
@@ -65,7 +65,7 @@ from vllm.entrypoints.openai.speech_to_text.serving import (
|
||||
from vllm.entrypoints.openai.utils import validate_json_request
|
||||
from vllm.entrypoints.pooling.classify.serving import ServingClassification
|
||||
from vllm.entrypoints.pooling.embed.serving import ServingEmbedding as OpenAIServingEmbedding
|
||||
from vllm.entrypoints.pooling.pooling.serving import ServingPooling
|
||||
from vllm.entrypoints.pooling.pooling.serving import ServingPooling as OpenAIServingPooling
|
||||
from vllm.entrypoints.pooling.scoring.serving import ServingScores
|
||||
from vllm.entrypoints.serve.disagg.serving import ServingTokens
|
||||
|
||||
@@ -800,9 +800,10 @@ async def omni_init_app_state(
|
||||
else None
|
||||
)
|
||||
state.openai_serving_pooling = (
|
||||
ServingPooling(
|
||||
OpenAIServingPooling(
|
||||
engine_client,
|
||||
state.openai_serving_models,
|
||||
state.openai_serving_render,
|
||||
supported_tasks=tuple(supported_tasks),
|
||||
request_logger=request_logger,
|
||||
chat_template=resolved_chat_template,
|
||||
|
||||
@@ -489,7 +489,7 @@ class Qwen3OmniMoeModel(Qwen3MoeLLMForCausalLM):
|
||||
"""
|
||||
Qwen3 Omni MoE Talker language model.
|
||||
|
||||
Extends Qwen3MoeLLMForCausalLM (which already uses FusedMoE with
|
||||
Extends Qwen3MoeLLMForCausalLM (which already uses SharedFusedMoE with
|
||||
shared-expert support) and replaces the text embedding / LM head with a
|
||||
codec embedding so the talker operates over audio-codec tokens instead
|
||||
of text tokens.
|
||||
|
||||
@@ -103,9 +103,6 @@ def prepare_hunyuan_fused_moe_runtime() -> None:
|
||||
_ensure_forward_context_attr("flash_comm_v1_enabled", bool, False)
|
||||
|
||||
|
||||
# NOTE: As of v0.20,0 vLLM has folded SharedFusedMoE -> FusedMoE and removed the class.
|
||||
# Since AscendSharedFusedMoE is a subclass of SharedFusedMoE, we should be careful
|
||||
# to ensure that this is updated correctly.
|
||||
class AscendHunyuanFusedMoE(AscendSharedFusedMoE):
|
||||
def __init__(self, *, prefix: str = "", **kwargs: Any) -> None:
|
||||
super().__init__(prefix=prefix, **kwargs)
|
||||
|
||||
Reference in new issue
Block a user