Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
cf301fe75b |
No files matched your search
@@ -10,7 +10,7 @@ steps:
|
|||||||
DOCKER_BUILDKIT: "1"
|
DOCKER_BUILDKIT: "1"
|
||||||
# Buildkite will automatically replace this with the actual commit hash
|
# Buildkite will automatically replace this with the actual commit hash
|
||||||
VLLM_IMAGE_TAG: "${BUILDKITE_COMMIT}"
|
VLLM_IMAGE_TAG: "${BUILDKITE_COMMIT}"
|
||||||
VLLM_VERSION: "v0.20.0"
|
VLLM_VERSION: "v0.19.0"
|
||||||
priority: 100
|
priority: 100
|
||||||
timeout_in_minutes: 60
|
timeout_in_minutes: 60
|
||||||
soft_fail: true
|
soft_fail: true
|
||||||
@@ -7,9 +7,7 @@ steps:
|
|||||||
grade: Blocking
|
grade: Blocking
|
||||||
commands:
|
commands:
|
||||||
- export VLLM_ROCM_USE_AITER=0
|
- export VLLM_ROCM_USE_AITER=0
|
||||||
# ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM
|
- "timeout 20m pytest -v -s -m 'core_model and cpu' --cov=vllm_omni --cov-branch --cov-report=term-missing --cov-report=html --cov-report=xml"
|
||||||
# that is not supported on CPU
|
|
||||||
- "timeout 20m pytest -v -s -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-branch --cov-report=term-missing --cov-report=html --cov-report=xml"
|
|
||||||
|
|
||||||
- label: "Diffusion Model Test"
|
- label: "Diffusion Model Test"
|
||||||
agent_pool: mi325_2
|
agent_pool: mi325_2
|
||||||
|
|||||||
@@ -7,9 +7,7 @@ steps:
|
|||||||
grade: Blocking
|
grade: Blocking
|
||||||
commands:
|
commands:
|
||||||
- export VLLM_ROCM_USE_AITER=0
|
- export VLLM_ROCM_USE_AITER=0
|
||||||
# ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM
|
- "timeout 20m pytest -v -s -m 'core_model and cpu' --cov=vllm_omni --cov-branch --cov-report=term-missing --cov-report=html --cov-report=xml"
|
||||||
# that is not supported on CPU
|
|
||||||
- "timeout 20m pytest -vvvv -s -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-branch --cov-report=term-missing --cov-report=html --cov-report=xml"
|
|
||||||
|
|
||||||
- label: "Voxtral TTS CUDA Unit Test"
|
- label: "Voxtral TTS CUDA Unit Test"
|
||||||
agent_pool: mi325_1
|
agent_pool: mi325_1
|
||||||
|
|||||||
@@ -1,223 +0,0 @@
|
|||||||
pull_request_rules:
|
|
||||||
- name: label-documentation
|
|
||||||
description: Automatically apply documentation label
|
|
||||||
conditions:
|
|
||||||
- label != stale
|
|
||||||
- or:
|
|
||||||
- files~=^[^/]+\.md$
|
|
||||||
- files~=^docs/
|
|
||||||
- files~=^examples/
|
|
||||||
actions:
|
|
||||||
label:
|
|
||||||
add:
|
|
||||||
- documentation
|
|
||||||
comment:
|
|
||||||
message: "Documentation preview: https://vllm--{{number}}.org.readthedocs.build/en/{{number}}/"
|
|
||||||
|
|
||||||
- name: comment-pre-commit-failure
|
|
||||||
description: Comment on PR when pre-commit check fails
|
|
||||||
conditions:
|
|
||||||
- status-failure=pre-commit
|
|
||||||
- -closed
|
|
||||||
- -draft
|
|
||||||
actions:
|
|
||||||
comment:
|
|
||||||
message: |
|
|
||||||
Hi @{{author}}, the pre-commit checks have failed. Please run:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
uv pip install pre-commit
|
|
||||||
pre-commit install
|
|
||||||
pre-commit run --all-files
|
|
||||||
```
|
|
||||||
|
|
||||||
Then, commit the changes and push to your branch.
|
|
||||||
|
|
||||||
For future commits, `pre-commit` will run automatically on changed files before each commit.
|
|
||||||
|
|
||||||
> [!TIP]
|
|
||||||
> <details>
|
|
||||||
> <summary>Is <code>mypy</code> or <code>markdownlint</code> failing?</summary>
|
|
||||||
> <br/>
|
|
||||||
> <code>mypy</code> and <code>markdownlint</code> are run differently in CI. If the failure is related to either of these checks, please use the following commands to run them locally:
|
|
||||||
>
|
|
||||||
> ```bash
|
|
||||||
> # For mypy (substitute "3.10" with the failing version if needed)
|
|
||||||
> pre-commit run --hook-stage manual mypy-3.10
|
|
||||||
> # For markdownlint
|
|
||||||
> pre-commit run --hook-stage manual markdownlint
|
|
||||||
> ```
|
|
||||||
> </details>
|
|
||||||
|
|
||||||
- name: comment-dco-failure
|
|
||||||
description: Comment on PR when DCO check fails
|
|
||||||
conditions:
|
|
||||||
- status-failure=dco
|
|
||||||
- -closed
|
|
||||||
- -draft
|
|
||||||
actions:
|
|
||||||
comment:
|
|
||||||
message: |
|
|
||||||
Hi @{{author}}, the DCO check has failed. Please click on DCO in the Checks section for instructions on how to resolve this.
|
|
||||||
|
|
||||||
- name: label-ci-build
|
|
||||||
description: Automatically apply ci/build label
|
|
||||||
conditions:
|
|
||||||
- label != stale
|
|
||||||
- or:
|
|
||||||
- files~=^\.github/
|
|
||||||
- files~=\.buildkite/
|
|
||||||
- files~=^cmake/
|
|
||||||
- files=CMakeLists.txt
|
|
||||||
- files~=^docker/Dockerfile
|
|
||||||
- files~=^requirements.*\.txt
|
|
||||||
- files=setup.py
|
|
||||||
actions:
|
|
||||||
label:
|
|
||||||
add:
|
|
||||||
- ci/build
|
|
||||||
|
|
||||||
- name: label-frontend
|
|
||||||
description: Automatically apply frontend label
|
|
||||||
conditions:
|
|
||||||
- label != stale
|
|
||||||
- files~=^vllm_omni/entrypoints/
|
|
||||||
actions:
|
|
||||||
label:
|
|
||||||
add:
|
|
||||||
- frontend
|
|
||||||
|
|
||||||
- name: label-diffusion
|
|
||||||
description: Automatically apply diffusion label
|
|
||||||
conditions:
|
|
||||||
- label != stale
|
|
||||||
- files~=^vllm_omni/diffusion/
|
|
||||||
actions:
|
|
||||||
label:
|
|
||||||
add:
|
|
||||||
- diffusion
|
|
||||||
|
|
||||||
- name: label-new-model
|
|
||||||
description: Automatically apply new-model label
|
|
||||||
conditions:
|
|
||||||
- label != stale
|
|
||||||
- or:
|
|
||||||
- files=vllm_omni/model_executor/models/registry.py
|
|
||||||
- files=vllm_omni/diffusion/registry.py
|
|
||||||
actions:
|
|
||||||
label:
|
|
||||||
add:
|
|
||||||
- new-model
|
|
||||||
|
|
||||||
- name: label-npu
|
|
||||||
description: Automatically apply npu label
|
|
||||||
conditions:
|
|
||||||
- label != stale
|
|
||||||
- or:
|
|
||||||
- files~=^vllm_omni/platforms/npu/
|
|
||||||
- title~=(?i)NPU
|
|
||||||
- title~=(?i)Ascend
|
|
||||||
actions:
|
|
||||||
label:
|
|
||||||
add:
|
|
||||||
- npu
|
|
||||||
assign:
|
|
||||||
users:
|
|
||||||
- "gcanlin"
|
|
||||||
|
|
||||||
- name: label-rocm
|
|
||||||
description: Automatically apply rocm label
|
|
||||||
conditions:
|
|
||||||
- label != stale
|
|
||||||
- or:
|
|
||||||
- files~=^vllm_omni/platforms/rocm/
|
|
||||||
- title~=(?i)AMD
|
|
||||||
- title~=(?i)ROCm
|
|
||||||
actions:
|
|
||||||
label:
|
|
||||||
add:
|
|
||||||
- rocm
|
|
||||||
assign:
|
|
||||||
users:
|
|
||||||
- "tjtanaa"
|
|
||||||
|
|
||||||
- name: label-xpu
|
|
||||||
description: Automatically apply xpu label
|
|
||||||
conditions:
|
|
||||||
- label != stale
|
|
||||||
- or:
|
|
||||||
- files~=^vllm_omni/platforms/xpu/
|
|
||||||
- title~=(?i)Intel
|
|
||||||
- title~=(?i)XPU
|
|
||||||
actions:
|
|
||||||
label:
|
|
||||||
add:
|
|
||||||
- xpu
|
|
||||||
assign:
|
|
||||||
users:
|
|
||||||
- "yma11"
|
|
||||||
|
|
||||||
- name: label-modelrunner
|
|
||||||
description: Automatically apply modelrunner label
|
|
||||||
conditions:
|
|
||||||
- label != stale
|
|
||||||
- or:
|
|
||||||
- files~=^vllm_omni/worker/
|
|
||||||
- title~=(?i)ModelRunner
|
|
||||||
- title~=(?i)worker
|
|
||||||
actions:
|
|
||||||
label:
|
|
||||||
add:
|
|
||||||
- ModelRunner
|
|
||||||
assign:
|
|
||||||
users:
|
|
||||||
- "gcanlin" # To check whether need to sync to NPU
|
|
||||||
|
|
||||||
- name: auto-rebase if approved, ready, and 30 commits behind main
|
|
||||||
conditions:
|
|
||||||
- base = main
|
|
||||||
- label=ready
|
|
||||||
- "#approved-reviews-by >= 1"
|
|
||||||
- "#commits-behind >= 30"
|
|
||||||
- -closed
|
|
||||||
- -draft
|
|
||||||
- -conflict
|
|
||||||
actions:
|
|
||||||
rebase: {}
|
|
||||||
|
|
||||||
- name: ping author on conflicts and add 'needs-rebase' label
|
|
||||||
conditions:
|
|
||||||
- label != stale
|
|
||||||
- conflict
|
|
||||||
- -closed
|
|
||||||
actions:
|
|
||||||
label:
|
|
||||||
add:
|
|
||||||
- needs-rebase
|
|
||||||
comment:
|
|
||||||
message: |
|
|
||||||
This pull request has merge conflicts that must be resolved before it can be
|
|
||||||
merged. Please rebase the PR, @{{author}}.
|
|
||||||
|
|
||||||
https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/working-with-forks/syncing-a-fork
|
|
||||||
|
|
||||||
- name: remove 'needs-rebase' label when conflict is resolved
|
|
||||||
conditions:
|
|
||||||
- -conflict
|
|
||||||
- -closed
|
|
||||||
actions:
|
|
||||||
label:
|
|
||||||
remove:
|
|
||||||
- needs-rebase
|
|
||||||
|
|
||||||
- name: label-bug
|
|
||||||
description: Automatically apply bug label
|
|
||||||
conditions:
|
|
||||||
- label != stale
|
|
||||||
- or:
|
|
||||||
- title~=(?i)\bbug\b
|
|
||||||
- title~=(?i)\bbugfix\b
|
|
||||||
actions:
|
|
||||||
label:
|
|
||||||
add:
|
|
||||||
- bug
|
|
||||||
@@ -1,5 +1,5 @@
|
|||||||
ARG VLLM_BASE_IMAGE=vllm/vllm-openai
|
ARG VLLM_BASE_IMAGE=vllm/vllm-openai
|
||||||
ARG VLLM_BASE_TAG=v0.20.0
|
ARG VLLM_BASE_TAG=v0.20.0-cu130
|
||||||
FROM ${VLLM_BASE_IMAGE}:${VLLM_BASE_TAG}
|
FROM ${VLLM_BASE_IMAGE}:${VLLM_BASE_TAG}
|
||||||
ARG APP_DIR=/workspace/vllm-omni
|
ARG APP_DIR=/workspace/vllm-omni
|
||||||
WORKDIR ${APP_DIR}
|
WORKDIR ${APP_DIR}
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
ARG BASE_IMAGE=vllm/vllm-openai:v0.20.0
|
ARG BASE_IMAGE=vllm/vllm-openai:v0.19.0
|
||||||
FROM ${BASE_IMAGE}
|
FROM ${BASE_IMAGE}
|
||||||
|
|
||||||
ARG COMMON_WORKDIR=/app
|
ARG COMMON_WORKDIR=/app
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
ARG BASE_IMAGE=vllm/vllm-openai-rocm:v0.20.0
|
ARG BASE_IMAGE=vllm/vllm-openai-rocm:v0.19.0
|
||||||
FROM ${BASE_IMAGE} AS base
|
FROM ${BASE_IMAGE} AS base
|
||||||
|
|
||||||
# Declare a variable to know if we want to use the nightly build or the stable build.
|
# Declare a variable to know if we want to use the nightly build or the stable build.
|
||||||
|
|||||||
@@ -74,7 +74,7 @@ ENV UV_EXTRA_INDEX_URL=${PIP_EXTRA_INDEX_URL}
|
|||||||
ENV UV_INDEX_STRATEGY="unsafe-best-match"
|
ENV UV_INDEX_STRATEGY="unsafe-best-match"
|
||||||
ENV UV_LINK_MODE="copy"
|
ENV UV_LINK_MODE="copy"
|
||||||
|
|
||||||
ARG VLLM_VERSION=v0.20.0
|
ARG VLLM_VERSION=v0.19.0
|
||||||
RUN git clone -b ${VLLM_VERSION} https://github.com/vllm-project/vllm
|
RUN git clone -b ${VLLM_VERSION} https://github.com/vllm-project/vllm
|
||||||
WORKDIR /workspace/vllm
|
WORKDIR /workspace/vllm
|
||||||
|
|
||||||
|
|||||||
@@ -13,7 +13,7 @@ vLLM-Omni current recommends the steps in under setup through Docker Images.
|
|||||||
|
|
||||||
vLLM-Omni is built based on vLLM. Please install it with command below.
|
vLLM-Omni is built based on vLLM. Please install it with command below.
|
||||||
```bash
|
```bash
|
||||||
uv pip install vllm==0.20.0+rocm721 --extra-index-url https://wheels.vllm.ai/rocm/0.20.0/rocm721
|
uv pip install vllm==0.20.0+rocm700 --extra-index-url https://wheels.vllm.ai/rocm/0.20.0/rocm700
|
||||||
```
|
```
|
||||||
|
|
||||||
#### Installation of vLLM-Omni
|
#### Installation of vLLM-Omni
|
||||||
@@ -37,10 +37,10 @@ uv pip install onnxruntime-rocm
|
|||||||
If you do not need to modify source code of vLLM, you can directly install the stable 0.20.0 release version of the library
|
If you do not need to modify source code of vLLM, you can directly install the stable 0.20.0 release version of the library
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
uv pip install vllm==0.20.0+rocm721 --extra-index-url https://wheels.vllm.ai/rocm/0.20.0/rocm721
|
uv pip install vllm==0.20.0+rocm700 --extra-index-url https://wheels.vllm.ai/rocm/0.20.0/rocm700
|
||||||
```
|
```
|
||||||
|
|
||||||
The pre-built 0.20.0 vLLM wheel targets ROCm 7.2.1. If you need a different ROCm stack or want to reuse an existing PyTorch installation, build vLLM from source instead.
|
The pre-built 0.20.0 vLLM wheel targets ROCm 7.0. If you need a different ROCm stack or want to reuse an existing PyTorch installation, build vLLM from source instead.
|
||||||
|
|
||||||
#### Installation of vLLM-Omni
|
#### Installation of vLLM-Omni
|
||||||
Since vllm-omni is rapidly evolving, it's recommended to install it from source
|
Since vllm-omni is rapidly evolving, it's recommended to install it from source
|
||||||
|
|||||||
@@ -22,7 +22,7 @@ source .venv/bin/activate
|
|||||||
uv pip install vllm==0.20.0 --torch-backend=auto
|
uv pip install vllm==0.20.0 --torch-backend=auto
|
||||||
|
|
||||||
# On ROCm
|
# On ROCm
|
||||||
uv pip install vllm==0.20.0+rocm721 --extra-index-url https://wheels.vllm.ai/rocm/0.20.0/rocm721
|
uv pip install vllm==0.20.0+rocm700 --extra-index-url https://wheels.vllm.ai/rocm/0.20.0/rocm700
|
||||||
|
|
||||||
git clone https://github.com/vllm-project/vllm-omni.git
|
git clone https://github.com/vllm-project/vllm-omni.git
|
||||||
cd vllm-omni
|
cd vllm-omni
|
||||||
@@ -31,12 +31,6 @@ uv pip install -e .
|
|||||||
|
|
||||||
For additional installation methods — please see the [installation guide](installation/README.md).
|
For additional installation methods — please see the [installation guide](installation/README.md).
|
||||||
|
|
||||||
|
|
||||||
!!! note
|
|
||||||
It is important to install the same major & minor version of vLLM and vLLM Omni, otherwise things may not work as expected. If the versions are misaligned, you will see a warning when you import vLLM Omni.
|
|
||||||
|
|
||||||
If you are seeing strange behavior with the `vllm` command not handling the `--omni` flag correctly, you most likely have a version mismatch with vLLM < `0.20.0` and vLLM Omni `0.20.0`, as vLLM Omni no longer hijacks the vLLM entrypoint. Updating vLLM should resolve this issue.
|
|
||||||
|
|
||||||
## Offline Inference
|
## Offline Inference
|
||||||
|
|
||||||
Text-to-image generation quickstart with vLLM-Omni:
|
Text-to-image generation quickstart with vLLM-Omni:
|
||||||
|
|||||||
@@ -118,6 +118,9 @@ def parse_args() -> argparse.Namespace:
|
|||||||
help='JSON profiler config for torch/cuda profiling, e.g. \'{"profiler":"torch","torch_profiler_dir":"./perf"}\'.',
|
help='JSON profiler config for torch/cuda profiling, e.g. \'{"profiler":"torch","torch_profiler_dir":"./perf"}\'.',
|
||||||
)
|
)
|
||||||
|
|
||||||
|
from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults
|
||||||
|
|
||||||
|
nullify_stage_engine_defaults(parser)
|
||||||
return parser.parse_args()
|
return parser.parse_args()
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -196,6 +196,9 @@ def parse_args() -> argparse.Namespace:
|
|||||||
parser.add_argument("--cfg-parallel-size", type=int, default=1, choices=[1, 2], help="CFG parallel size.")
|
parser.add_argument("--cfg-parallel-size", type=int, default=1, choices=[1, 2], help="CFG parallel size.")
|
||||||
parser.add_argument("--tensor-parallel-size", type=int, default=1, help="Tensor parallelism size.")
|
parser.add_argument("--tensor-parallel-size", type=int, default=1, help="Tensor parallelism size.")
|
||||||
|
|
||||||
|
from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults
|
||||||
|
|
||||||
|
nullify_stage_engine_defaults(parser)
|
||||||
return parser.parse_args()
|
return parser.parse_args()
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -378,6 +378,9 @@ def parse_args() -> argparse.Namespace:
|
|||||||
default=None,
|
default=None,
|
||||||
help='JSON profiler config for torch/cuda profiling, e.g. \'{"profiler":"torch","torch_profiler_dir":"./perf"}\'.',
|
help='JSON profiler config for torch/cuda profiling, e.g. \'{"profiler":"torch","torch_profiler_dir":"./perf"}\'.',
|
||||||
)
|
)
|
||||||
|
from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults
|
||||||
|
|
||||||
|
nullify_stage_engine_defaults(parser)
|
||||||
return parser.parse_args()
|
return parser.parse_args()
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -211,6 +211,9 @@ def parse_args() -> argparse.Namespace:
|
|||||||
default=None,
|
default=None,
|
||||||
help='JSON profiler config for torch/cuda profiling, e.g. \'{"profiler":"torch","torch_profiler_dir":"./perf"}\'.',
|
help='JSON profiler config for torch/cuda profiling, e.g. \'{"profiler":"torch","torch_profiler_dir":"./perf"}\'.',
|
||||||
)
|
)
|
||||||
|
from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults
|
||||||
|
|
||||||
|
nullify_stage_engine_defaults(parser)
|
||||||
return parser.parse_args()
|
return parser.parse_args()
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -24,6 +24,9 @@ def parse_args():
|
|||||||
parser.add_argument("--width", type=int, default=448, help="Video width.")
|
parser.add_argument("--width", type=int, default=448, help="Video width.")
|
||||||
parser.add_argument("--num-inference-steps", type=int, default=8, help="Number of denoising steps.")
|
parser.add_argument("--num-inference-steps", type=int, default=8, help="Number of denoising steps.")
|
||||||
parser.add_argument("--seed", type=int, default=52, help="Random seed for generation.")
|
parser.add_argument("--seed", type=int, default=52, help="Random seed for generation.")
|
||||||
|
from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults
|
||||||
|
|
||||||
|
nullify_stage_engine_defaults(parser)
|
||||||
return parser.parse_args()
|
return parser.parse_args()
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -9,7 +9,7 @@ from typing import Any
|
|||||||
|
|
||||||
import torch
|
import torch
|
||||||
|
|
||||||
from vllm_omni.diffusion.data import logger
|
from vllm_omni.diffusion.data import DiffusionParallelConfig, logger
|
||||||
from vllm_omni.entrypoints.omni import Omni
|
from vllm_omni.entrypoints.omni import Omni
|
||||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||||
from vllm_omni.lora.request import LoRARequest
|
from vllm_omni.lora.request import LoRARequest
|
||||||
@@ -342,6 +342,17 @@ def main():
|
|||||||
# (e.g., QwenImagePipeline or FluxPipeline)
|
# (e.g., QwenImagePipeline or FluxPipeline)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# assert args.ring_degree == 1, "Ring attention is not supported yet"
|
||||||
|
parallel_config = DiffusionParallelConfig(
|
||||||
|
ulysses_degree=args.ulysses_degree,
|
||||||
|
ring_degree=args.ring_degree,
|
||||||
|
ulysses_mode=args.ulysses_mode,
|
||||||
|
cfg_parallel_size=args.cfg_parallel_size,
|
||||||
|
tensor_parallel_size=args.tensor_parallel_size,
|
||||||
|
vae_patch_parallel_size=args.vae_patch_parallel_size,
|
||||||
|
enable_expert_parallel=args.enable_expert_parallel,
|
||||||
|
)
|
||||||
|
|
||||||
profiler_enabled = args.profiler_config is not None
|
profiler_enabled = args.profiler_config is not None
|
||||||
|
|
||||||
# Prepare LoRA kwargs for Omni initialization
|
# Prepare LoRA kwargs for Omni initialization
|
||||||
@@ -377,13 +388,7 @@ def main():
|
|||||||
"cache_backend": args.cache_backend,
|
"cache_backend": args.cache_backend,
|
||||||
"cache_config": cache_config,
|
"cache_config": cache_config,
|
||||||
"enable_cache_dit_summary": args.enable_cache_dit_summary,
|
"enable_cache_dit_summary": args.enable_cache_dit_summary,
|
||||||
"ulysses_degree": args.ulysses_degree,
|
"parallel_config": parallel_config,
|
||||||
"ring_degree": args.ring_degree,
|
|
||||||
"ulysses_mode": args.ulysses_mode,
|
|
||||||
"cfg_parallel_size": args.cfg_parallel_size,
|
|
||||||
"tensor_parallel_size": args.tensor_parallel_size,
|
|
||||||
"vae_patch_parallel_size": args.vae_patch_parallel_size,
|
|
||||||
"enable_expert_parallel": args.enable_expert_parallel,
|
|
||||||
"enforce_eager": args.enforce_eager,
|
"enforce_eager": args.enforce_eager,
|
||||||
"enable_cpu_offload": args.enable_cpu_offload,
|
"enable_cpu_offload": args.enable_cpu_offload,
|
||||||
"mode": "text-to-image",
|
"mode": "text-to-image",
|
||||||
|
|||||||
@@ -226,6 +226,9 @@ def parse_args() -> argparse.Namespace:
|
|||||||
default=1,
|
default=1,
|
||||||
help="Number of HSDP replica groups.",
|
help="Number of HSDP replica groups.",
|
||||||
)
|
)
|
||||||
|
from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults
|
||||||
|
|
||||||
|
nullify_stage_engine_defaults(parser)
|
||||||
return parser.parse_args()
|
return parser.parse_args()
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -71,6 +71,9 @@ def parse_args() -> argparse.Namespace:
|
|||||||
parser.add_argument("--ulysses-degree", type=int, default=1, help="Ulysses SP degree.")
|
parser.add_argument("--ulysses-degree", type=int, default=1, help="Ulysses SP degree.")
|
||||||
parser.add_argument("--ring-degree", type=int, default=1, help="Ring attention degree.")
|
parser.add_argument("--ring-degree", type=int, default=1, help="Ring attention degree.")
|
||||||
parser.add_argument("--cfg-parallel-size", type=int, default=1, choices=[1, 2], help="CFG parallel size.")
|
parser.add_argument("--cfg-parallel-size", type=int, default=1, choices=[1, 2], help="CFG parallel size.")
|
||||||
|
from vllm_omni.engine.arg_utils import nullify_stage_engine_defaults
|
||||||
|
|
||||||
|
nullify_stage_engine_defaults(parser)
|
||||||
return parser.parse_args()
|
return parser.parse_args()
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -32,6 +32,7 @@ classifiers = [
|
|||||||
# Dependencies are now managed dynamically via setup.py based on detected hardware platform.
|
# Dependencies are now managed dynamically via setup.py based on detected hardware platform.
|
||||||
# This allows automatic installation of the correct platform-specific dependencies (CUDA/ROCm/CPU/XPU/NPU)
|
# This allows automatic installation of the correct platform-specific dependencies (CUDA/ROCm/CPU/XPU/NPU)
|
||||||
# without requiring extras like [cuda]. See requirements/ directory for platform-specific dependencies.
|
# without requiring extras like [cuda]. See requirements/ directory for platform-specific dependencies.
|
||||||
|
# Note: vllm is intentionally excluded due to entrypoints overwrite issue.
|
||||||
|
|
||||||
[project.optional-dependencies]
|
[project.optional-dependencies]
|
||||||
|
|
||||||
@@ -89,6 +90,7 @@ Documentation = "https://vllm-omni.readthedocs.io"
|
|||||||
"Bug Tracker" = "https://github.com/vllm-project/vllm-omni/issues"
|
"Bug Tracker" = "https://github.com/vllm-project/vllm-omni/issues"
|
||||||
|
|
||||||
[project.scripts]
|
[project.scripts]
|
||||||
|
vllm = "vllm_omni.entrypoints.cli.main:main"
|
||||||
vllm-omni = "vllm_omni.entrypoints.cli.main:main"
|
vllm-omni = "vllm_omni.entrypoints.cli.main:main"
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -121,8 +121,7 @@ class TestTrajectoryRecording:
|
|||||||
|
|
||||||
assert trajectory_latents is not None
|
assert trajectory_latents is not None
|
||||||
assert trajectory_timesteps is not None
|
assert trajectory_timesteps is not None
|
||||||
# initial latent + one per denoising step
|
assert len(trajectory_latents) == EXPECTED_STEPS
|
||||||
assert len(trajectory_latents) == EXPECTED_STEPS + 1
|
|
||||||
assert len(trajectory_timesteps) == EXPECTED_STEPS
|
assert len(trajectory_timesteps) == EXPECTED_STEPS
|
||||||
# log_probs is None without a scheduler (default ODE path)
|
# log_probs is None without a scheduler (default ODE path)
|
||||||
assert trajectory_log_probs is None
|
assert trajectory_log_probs is None
|
||||||
@@ -168,54 +167,6 @@ class TestTrajectoryRecording:
|
|||||||
"Last trajectory latent should match the final output"
|
"Last trajectory latent should match the final output"
|
||||||
)
|
)
|
||||||
|
|
||||||
def test_initial_latent_matches_input_noise(self, bagel_and_args):
|
|
||||||
"""Regression: the first trajectory entry must be the initial noise (pre-denoising)."""
|
|
||||||
bagel, args = bagel_and_args
|
|
||||||
init_noise = args["packed_init_noises"].clone()
|
|
||||||
|
|
||||||
_, trajectory_latents, *_ = bagel.generate_image(**args, return_trajectory_latents=True)
|
|
||||||
|
|
||||||
assert torch.allclose(trajectory_latents[0], init_noise, atol=1e-6), (
|
|
||||||
"trajectory_latents[0] should be the initial noise before any denoising step"
|
|
||||||
)
|
|
||||||
|
|
||||||
def test_trajectory_timesteps_match_expected_schedule(self, bagel_and_args):
|
|
||||||
"""Regression: trajectory_timesteps must record raw timesteps[i], not timesteps[i] - dts[i]."""
|
|
||||||
bagel, args = bagel_and_args
|
|
||||||
|
|
||||||
# Recompute the expected timestep schedule (mirrors generate_image logic)
|
|
||||||
ts = torch.linspace(1, 0, args["num_timesteps"])
|
|
||||||
shift = args.get("timestep_shift", 1.0)
|
|
||||||
ts = shift * ts / (1 + (shift - 1) * ts)
|
|
||||||
expected_timesteps = ts[:-1] # last element is dropped
|
|
||||||
|
|
||||||
_, _, trajectory_timesteps, _ = bagel.generate_image(**args, return_trajectory_latents=True)
|
|
||||||
|
|
||||||
assert len(trajectory_timesteps) == len(expected_timesteps)
|
|
||||||
for i, (actual, expected) in enumerate(zip(trajectory_timesteps, expected_timesteps)):
|
|
||||||
assert abs(float(actual) - float(expected)) < 1e-6, (
|
|
||||||
f"Step {i}: trajectory_timestep={float(actual):.6f}, expected={float(expected):.6f}"
|
|
||||||
)
|
|
||||||
|
|
||||||
def test_latent_count_equals_timesteps_plus_one(self, bagel_and_args):
|
|
||||||
"""Regression: len(trajectory_latents) == len(timesteps) + 1 (initial + one per step)."""
|
|
||||||
bagel, args = bagel_and_args
|
|
||||||
|
|
||||||
# Compute the number of denoising steps the same way generate_image does
|
|
||||||
ts = torch.linspace(1, 0, args["num_timesteps"])
|
|
||||||
shift = args.get("timestep_shift", 1.0)
|
|
||||||
ts = shift * ts / (1 + (shift - 1) * ts)
|
|
||||||
num_steps = len(ts) - 1 # timesteps = ts[:-1]
|
|
||||||
|
|
||||||
_, trajectory_latents, trajectory_timesteps, _ = bagel.generate_image(**args, return_trajectory_latents=True)
|
|
||||||
|
|
||||||
assert len(trajectory_latents) == num_steps + 1, (
|
|
||||||
f"Expected {num_steps + 1} latents (initial + {num_steps} steps), got {len(trajectory_latents)}"
|
|
||||||
)
|
|
||||||
assert len(trajectory_timesteps) == num_steps, (
|
|
||||||
f"Expected {num_steps} timesteps, got {len(trajectory_timesteps)}"
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# Mock scheduler for log-prob tests
|
# Mock scheduler for log-prob tests
|
||||||
|
|||||||
@@ -356,6 +356,7 @@ def test_mix_to_text_audio_001(omni_server, openai_client) -> None:
|
|||||||
|
|
||||||
@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
|
@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
|
||||||
@pytest.mark.parametrize("omni_server", test_params, indirect=True)
|
@pytest.mark.parametrize("omni_server", test_params, indirect=True)
|
||||||
|
@pytest.mark.skip(reason="issue: #2827")
|
||||||
def test_audio_in_video_001(omni_server, openai_client) -> None:
|
def test_audio_in_video_001(omni_server, openai_client) -> None:
|
||||||
"""
|
"""
|
||||||
Input Modal: text + video (synthetic MP4 with embedded audio; ``use_audio_in_video`` uses audio from the video).
|
Input Modal: text + video (synthetic MP4 with embedded audio; ``use_audio_in_video`` uses audio from the video).
|
||||||
@@ -375,7 +376,7 @@ def test_audio_in_video_001(omni_server, openai_client) -> None:
|
|||||||
"messages": messages,
|
"messages": messages,
|
||||||
"stream": False,
|
"stream": False,
|
||||||
"use_audio_in_video": True,
|
"use_audio_in_video": True,
|
||||||
"key_words": {"video": VIDEO_KEY},
|
"key_words": {"video": VIDEO_KEY, "audio": AUDIO_KEY + ["beep", "electronic"]},
|
||||||
}
|
}
|
||||||
openai_client.send_omni_request(request_config)
|
openai_client.send_omni_request(request_config)
|
||||||
|
|
||||||
|
|||||||
@@ -98,7 +98,7 @@ def _get_diffusion_feature_cases():
|
|||||||
],
|
],
|
||||||
),
|
),
|
||||||
id="layerwise_hsdp",
|
id="layerwise_hsdp",
|
||||||
marks=[*FOUR_CARD_MARKS],
|
marks=[*FOUR_CARD_MARKS, pytest.mark.skip(reason="issue #2435")],
|
||||||
),
|
),
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|||||||
@@ -4,7 +4,6 @@
|
|||||||
import pytest
|
import pytest
|
||||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||||
|
|
||||||
from vllm_omni.config.stage_config import deploy_override_field_names
|
|
||||||
from vllm_omni.engine.async_omni_engine import AsyncOmniEngine
|
from vllm_omni.engine.async_omni_engine import AsyncOmniEngine
|
||||||
from vllm_omni.entrypoints.cli.serve import OmniServeCommand, _create_default_diffusion_stage_cfg
|
from vllm_omni.entrypoints.cli.serve import OmniServeCommand, _create_default_diffusion_stage_cfg
|
||||||
|
|
||||||
@@ -31,15 +30,6 @@ def test_default_stage_config_includes_cache_backend():
|
|||||||
assert engine_args["model_stage"] == "diffusion"
|
assert engine_args["model_stage"] == "diffusion"
|
||||||
|
|
||||||
|
|
||||||
def test_default_stage_config_ignores_none_deploy_overrides():
|
|
||||||
"""Ensure nullified deploy override defaults do not alter diffusion defaults."""
|
|
||||||
baseline = AsyncOmniEngine._create_default_diffusion_stage_cfg({})[0]
|
|
||||||
nullified_overrides = {name: None for name in deploy_override_field_names()}
|
|
||||||
stage_cfg = AsyncOmniEngine._create_default_diffusion_stage_cfg(nullified_overrides)[0]
|
|
||||||
|
|
||||||
assert stage_cfg == baseline
|
|
||||||
|
|
||||||
|
|
||||||
def test_default_cache_config_used_when_missing():
|
def test_default_cache_config_used_when_missing():
|
||||||
"""Ensure default cache_config is synthesized when only backend is given."""
|
"""Ensure default cache_config is synthesized when only backend is given."""
|
||||||
stage_cfg = AsyncOmniEngine._create_default_diffusion_stage_cfg(
|
stage_cfg = AsyncOmniEngine._create_default_diffusion_stage_cfg(
|
||||||
@@ -65,44 +55,6 @@ def test_default_stage_devices_from_sequence_parallel():
|
|||||||
assert stage_cfg["runtime"]["devices"] == "0,1,2,3"
|
assert stage_cfg["runtime"]["devices"] == "0,1,2,3"
|
||||||
|
|
||||||
|
|
||||||
def test_default_stage_config_uses_parallel_size_kwargs():
|
|
||||||
"""Ensure default diffusion parallel_config uses CLI/API parallel sizes."""
|
|
||||||
stage_cfg = AsyncOmniEngine._create_default_diffusion_stage_cfg(
|
|
||||||
{
|
|
||||||
"pipeline_parallel_size": 2,
|
|
||||||
"data_parallel_size": 3,
|
|
||||||
"tensor_parallel_size": 4,
|
|
||||||
"enable_expert_parallel": True,
|
|
||||||
}
|
|
||||||
)[0]
|
|
||||||
|
|
||||||
parallel_config = stage_cfg["engine_args"]["parallel_config"]
|
|
||||||
assert parallel_config.pipeline_parallel_size == 2
|
|
||||||
assert parallel_config.data_parallel_size == 3
|
|
||||||
assert parallel_config.tensor_parallel_size == 4
|
|
||||||
assert parallel_config.enable_expert_parallel is True
|
|
||||||
|
|
||||||
|
|
||||||
def test_default_stage_config_defaults_nullified_parallel_size_kwargs():
|
|
||||||
"""Ensure nullified diffusion parallel-size kwargs fall back to defaults."""
|
|
||||||
stage_cfg = AsyncOmniEngine._create_default_diffusion_stage_cfg(
|
|
||||||
{
|
|
||||||
"pipeline_parallel_size": None,
|
|
||||||
"data_parallel_size": None,
|
|
||||||
"tensor_parallel_size": None,
|
|
||||||
"enable_expert_parallel": None,
|
|
||||||
"enforce_eager": None,
|
|
||||||
}
|
|
||||||
)[0]
|
|
||||||
|
|
||||||
parallel_config = stage_cfg["engine_args"]["parallel_config"]
|
|
||||||
assert parallel_config.pipeline_parallel_size == 1
|
|
||||||
assert parallel_config.data_parallel_size == 1
|
|
||||||
assert parallel_config.tensor_parallel_size == 1
|
|
||||||
assert parallel_config.enable_expert_parallel is False
|
|
||||||
assert stage_cfg["engine_args"]["enforce_eager"] is False
|
|
||||||
|
|
||||||
|
|
||||||
def test_default_stage_config_propagates_ulysses_mode():
|
def test_default_stage_config_propagates_ulysses_mode():
|
||||||
"""Ensure UAA mode survives default diffusion-stage creation."""
|
"""Ensure UAA mode survives default diffusion-stage creation."""
|
||||||
stage_cfg = AsyncOmniEngine._create_default_diffusion_stage_cfg(
|
stage_cfg = AsyncOmniEngine._create_default_diffusion_stage_cfg(
|
||||||
|
|||||||
@@ -26,9 +26,27 @@ def _skip_readme_snippet(language: str, code: str, h2_title: str) -> tuple[bool,
|
|||||||
|
|
||||||
README_SNIPPETS = ReadmeSnippet.extract_readme_snippets(README_PATH, skipif=_skip_readme_snippet)
|
README_SNIPPETS = ReadmeSnippet.extract_readme_snippets(README_PATH, skipif=_skip_readme_snippet)
|
||||||
|
|
||||||
|
# Track: https://github.com/vllm-project/vllm-omni/issues/3123 — CLI example subprocess exit 1 in CI; re-enable when fixed.
|
||||||
|
_SKIP_T2I_SNIPPET_IDS = frozenset(
|
||||||
|
{
|
||||||
|
"quick_start_002",
|
||||||
|
"more_cli_examples_001",
|
||||||
|
"more_cli_examples_002",
|
||||||
|
"more_cli_examples_003",
|
||||||
|
"more_cli_examples_007",
|
||||||
|
"more_cli_examples_008",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
_SKIP_T2I_REASON = (
|
||||||
|
"Skipped: text_to_image README examples failing in CI (subprocess exit 1). "
|
||||||
|
"See https://github.com/vllm-project/vllm-omni/issues/3123"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize("snippet", README_SNIPPETS, ids=lambda snippet: snippet.test_id)
|
@pytest.mark.parametrize("snippet", README_SNIPPETS, ids=lambda snippet: snippet.test_id)
|
||||||
def test_text_to_image(snippet: ReadmeSnippet, example_runner: ExampleRunner):
|
def test_text_to_image(snippet: ReadmeSnippet, example_runner: ExampleRunner):
|
||||||
|
if snippet.test_id in _SKIP_T2I_SNIPPET_IDS:
|
||||||
|
pytest.skip(_SKIP_T2I_REASON)
|
||||||
should_skip, reason = snippet.skip
|
should_skip, reason = snippet.skip
|
||||||
if should_skip:
|
if should_skip:
|
||||||
pytest.skip(reason)
|
pytest.skip(reason)
|
||||||
|
|||||||
@@ -3,7 +3,14 @@
|
|||||||
"""Pytest marks and decorators for hardware / resource selection (CUDA, ROCm, …)."""
|
"""Pytest marks and decorators for hardware / resource selection (CUDA, ROCm, …)."""
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
from vllm.platforms import current_platform
|
|
||||||
|
try:
|
||||||
|
from vllm.utils.torch_utils import cuda_device_count_stateless
|
||||||
|
except ImportError:
|
||||||
|
import torch
|
||||||
|
|
||||||
|
def cuda_device_count_stateless() -> int:
|
||||||
|
return torch.cuda.device_count()
|
||||||
|
|
||||||
# Re-exported from tests.helpers.env (GPU wait + DeviceMemoryMonitor).
|
# Re-exported from tests.helpers.env (GPU wait + DeviceMemoryMonitor).
|
||||||
|
|
||||||
@@ -20,9 +27,8 @@ def cuda_marks(*, res: str, num_cards: int):
|
|||||||
if num_cards == 1:
|
if num_cards == 1:
|
||||||
return marks
|
return marks
|
||||||
test_distributed = pytest.mark.distributed_cuda(num_cards=num_cards)
|
test_distributed = pytest.mark.distributed_cuda(num_cards=num_cards)
|
||||||
|
|
||||||
test_skipif = pytest.mark.skipif_cuda(
|
test_skipif = pytest.mark.skipif_cuda(
|
||||||
not current_platform.is_cuda() or (current_platform.device_count() < num_cards),
|
cuda_device_count_stateless() < num_cards,
|
||||||
reason=f"Need at least {num_cards} CUDA GPUs to run the test.",
|
reason=f"Need at least {num_cards} CUDA GPUs to run the test.",
|
||||||
)
|
)
|
||||||
return marks + [test_distributed, test_skipif]
|
return marks + [test_distributed, test_skipif]
|
||||||
|
|||||||
@@ -494,7 +494,6 @@ _CI_OVERLAYS: dict[str, dict[str, Any]] = {
|
|||||||
"max_num_seqs": 1,
|
"max_num_seqs": 1,
|
||||||
"gpu_memory_utilization": 0.9,
|
"gpu_memory_utilization": 0.9,
|
||||||
"enforce_eager": True,
|
"enforce_eager": True,
|
||||||
"enable_prefix_caching": False,
|
|
||||||
"max_num_batched_tokens": 16384,
|
"max_num_batched_tokens": 16384,
|
||||||
"max_model_len": 16384,
|
"max_model_len": 16384,
|
||||||
"skip_mm_profiling": True,
|
"skip_mm_profiling": True,
|
||||||
|
|||||||
@@ -369,8 +369,8 @@ def _build_full_serve_parser():
|
|||||||
def test_nullify_stage_engine_defaults_resets_inherited_defaults():
|
def test_nullify_stage_engine_defaults_resets_inherited_defaults():
|
||||||
import argparse
|
import argparse
|
||||||
|
|
||||||
from vllm_omni.config.stage_config import deploy_override_field_names
|
|
||||||
from vllm_omni.engine.arg_utils import (
|
from vllm_omni.engine.arg_utils import (
|
||||||
|
deploy_override_field_names,
|
||||||
nullify_stage_engine_defaults,
|
nullify_stage_engine_defaults,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
@@ -93,7 +93,6 @@ class TestStageConfig:
|
|||||||
assert omega_config.engine_args.worker_type == "ar"
|
assert omega_config.engine_args.worker_type == "ar"
|
||||||
assert omega_config.final_output is True
|
assert omega_config.final_output is True
|
||||||
assert omega_config.final_output_type == "text"
|
assert omega_config.final_output_type == "text"
|
||||||
assert "max_num_seqs" not in omega_config.engine_args
|
|
||||||
# Legacy field name for backward compatibility
|
# Legacy field name for backward compatibility
|
||||||
assert omega_config.engine_input_source == []
|
assert omega_config.engine_input_source == []
|
||||||
|
|
||||||
@@ -147,24 +146,6 @@ class TestStageConfig:
|
|||||||
omega_config = config.to_omegaconf()
|
omega_config = config.to_omegaconf()
|
||||||
assert omega_config.engine_args.max_num_seqs == 32
|
assert omega_config.engine_args.max_num_seqs == 32
|
||||||
|
|
||||||
def test_to_omegaconf_omits_none_deploy_overrides_for_engine_args(self):
|
|
||||||
"""None deploy overrides must fall through to EngineArgs defaults."""
|
|
||||||
from vllm_omni.config.stage_config import deploy_override_field_names
|
|
||||||
|
|
||||||
config = StageConfig(
|
|
||||||
stage_id=0,
|
|
||||||
model_stage="thinker",
|
|
||||||
runtime_overrides={name: None for name in deploy_override_field_names()},
|
|
||||||
)
|
|
||||||
|
|
||||||
omega_config = config.to_omegaconf()
|
|
||||||
engine_args = dict(omega_config.engine_args)
|
|
||||||
|
|
||||||
assert "devices" not in engine_args
|
|
||||||
assert "max_batch_size" not in engine_args
|
|
||||||
for name in deploy_override_field_names() - {"devices"}:
|
|
||||||
assert name not in engine_args
|
|
||||||
|
|
||||||
|
|
||||||
class TestModelPipeline:
|
class TestModelPipeline:
|
||||||
"""Tests for ModelPipeline class."""
|
"""Tests for ModelPipeline class."""
|
||||||
@@ -821,40 +802,6 @@ class TestPipelineRegistry:
|
|||||||
|
|
||||||
|
|
||||||
class TestDeployConfigLoading:
|
class TestDeployConfigLoading:
|
||||||
def test_deploy_override_fields_include_deploy_schema_fields(self):
|
|
||||||
from vllm_omni.config.stage_config import deploy_override_field_names
|
|
||||||
|
|
||||||
expected_fields = {
|
|
||||||
"async_chunk",
|
|
||||||
"async_scheduling",
|
|
||||||
"config_format",
|
|
||||||
"data_parallel_size",
|
|
||||||
"devices",
|
|
||||||
"disable_hybrid_kv_cache_manager",
|
|
||||||
"distributed_executor_backend",
|
|
||||||
"dtype",
|
|
||||||
"enable_chunked_prefill",
|
|
||||||
"enable_flashinfer_autotune",
|
|
||||||
"enable_prefix_caching",
|
|
||||||
"enforce_eager",
|
|
||||||
"gpu_memory_utilization",
|
|
||||||
"load_format",
|
|
||||||
"max_model_len",
|
|
||||||
"max_num_batched_tokens",
|
|
||||||
"max_num_seqs",
|
|
||||||
"mm_processor_cache_gb",
|
|
||||||
"pipeline_parallel_size",
|
|
||||||
"profiler_config",
|
|
||||||
"quantization",
|
|
||||||
"skip_mm_profiling",
|
|
||||||
"subtalker_sampling_params",
|
|
||||||
"tensor_parallel_size",
|
|
||||||
"tokenizer_mode",
|
|
||||||
"trust_remote_code",
|
|
||||||
}
|
|
||||||
|
|
||||||
assert expected_fields == deploy_override_field_names()
|
|
||||||
|
|
||||||
def test_load_deploy_config(self):
|
def test_load_deploy_config(self):
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
@@ -870,17 +817,6 @@ class TestDeployConfigLoading:
|
|||||||
assert deploy.connectors is not None
|
assert deploy.connectors is not None
|
||||||
assert deploy.platforms is not None
|
assert deploy.platforms is not None
|
||||||
|
|
||||||
voxtral_path = Path(__file__).parent.parent / "vllm_omni" / "deploy" / "voxtral_tts.yaml"
|
|
||||||
if voxtral_path.exists():
|
|
||||||
voxtral_deploy = load_deploy_config(voxtral_path)
|
|
||||||
assert voxtral_deploy.stages[0].config_format == "mistral"
|
|
||||||
assert voxtral_deploy.stages[0].load_format == "mistral"
|
|
||||||
assert voxtral_deploy.stages[0].tokenizer_mode == "mistral"
|
|
||||||
assert not any(
|
|
||||||
name in voxtral_deploy.stages[0].engine_extras
|
|
||||||
for name in ("config_format", "load_format", "tokenizer_mode")
|
|
||||||
)
|
|
||||||
|
|
||||||
def test_merge_pipeline_deploy(self):
|
def test_merge_pipeline_deploy(self):
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
@@ -1075,8 +1011,7 @@ class TestBaseConfigInheritance:
|
|||||||
deploy = load_deploy_config(ci_path)
|
deploy = load_deploy_config(ci_path)
|
||||||
assert len(deploy.stages) == 3
|
assert len(deploy.stages) == 3
|
||||||
# CI overrides
|
# CI overrides
|
||||||
assert deploy.stages[0].load_format == "dummy"
|
assert deploy.stages[0].engine_extras.get("load_format") == "dummy"
|
||||||
assert "load_format" not in deploy.stages[0].engine_extras
|
|
||||||
assert deploy.stages[0].max_num_seqs == 5
|
assert deploy.stages[0].max_num_seqs == 5
|
||||||
# Inherited from base
|
# Inherited from base
|
||||||
assert deploy.stages[0].gpu_memory_utilization == 0.9
|
assert deploy.stages[0].gpu_memory_utilization == 0.9
|
||||||
@@ -1281,7 +1216,7 @@ class TestSentinelDefaultPrecedence:
|
|||||||
def test_none_value_skipped_yaml_wins(self):
|
def test_none_value_skipped_yaml_wins(self):
|
||||||
stages = self._stages({"max_num_seqs": None})
|
stages = self._stages({"max_num_seqs": None})
|
||||||
assert stages[2].runtime_overrides.get("max_num_seqs") is None
|
assert stages[2].runtime_overrides.get("max_num_seqs") is None
|
||||||
assert "max_num_seqs" not in stages[2].yaml_engine_args
|
assert stages[2].yaml_engine_args.get("max_num_seqs") == 1
|
||||||
|
|
||||||
def test_empty_kwargs_yaml_only(self):
|
def test_empty_kwargs_yaml_only(self):
|
||||||
stages = self._stages({})
|
stages = self._stages({})
|
||||||
|
|||||||
@@ -402,11 +402,11 @@ class StageDeployConfig:
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
stage_id: int
|
stage_id: int
|
||||||
max_num_seqs: int | None = None
|
max_num_seqs: int = 64
|
||||||
gpu_memory_utilization: float | None = None
|
gpu_memory_utilization: float = 0.9
|
||||||
tensor_parallel_size: int | None = None
|
tensor_parallel_size: int = 1
|
||||||
enforce_eager: bool | None = None
|
enforce_eager: bool = False
|
||||||
max_num_batched_tokens: int | None = None
|
max_num_batched_tokens: int = 32768
|
||||||
max_model_len: int | None = None
|
max_model_len: int | None = None
|
||||||
async_scheduling: bool | None = None
|
async_scheduling: bool | None = None
|
||||||
devices: str = "0"
|
devices: str = "0"
|
||||||
@@ -414,14 +414,6 @@ class StageDeployConfig:
|
|||||||
input_connectors: dict[str, str] | None = None
|
input_connectors: dict[str, str] | None = None
|
||||||
default_sampling_params: dict[str, Any] | None = None
|
default_sampling_params: dict[str, Any] | None = None
|
||||||
subtalker_sampling_params: dict[str, Any] | None = None
|
subtalker_sampling_params: dict[str, Any] | None = None
|
||||||
profiler_config: dict[str, Any] | None = None
|
|
||||||
disable_hybrid_kv_cache_manager: bool | None = None
|
|
||||||
mm_processor_cache_gb: float | None = None
|
|
||||||
skip_mm_profiling: bool | None = None
|
|
||||||
enable_flashinfer_autotune: bool | None = None
|
|
||||||
config_format: str | None = None
|
|
||||||
load_format: str | None = None
|
|
||||||
tokenizer_mode: str | None = None
|
|
||||||
engine_extras: dict[str, Any] = field(default_factory=dict)
|
engine_extras: dict[str, Any] = field(default_factory=dict)
|
||||||
|
|
||||||
|
|
||||||
@@ -446,14 +438,14 @@ class DeployConfig:
|
|||||||
pipeline: str | None = None
|
pipeline: str | None = None
|
||||||
|
|
||||||
# === Pipeline-wide engine settings (applied uniformly to every stage) ===
|
# === Pipeline-wide engine settings (applied uniformly to every stage) ===
|
||||||
trust_remote_code: bool | None = None
|
trust_remote_code: bool = True
|
||||||
distributed_executor_backend: str | None = None
|
distributed_executor_backend: str | None = None
|
||||||
dtype: str | None = None
|
dtype: str | None = None
|
||||||
quantization: str | None = None
|
quantization: str | None = None
|
||||||
enable_prefix_caching: bool | None = None
|
enable_prefix_caching: bool = False
|
||||||
enable_chunked_prefill: bool | None = None
|
enable_chunked_prefill: bool | None = None
|
||||||
data_parallel_size: int | None = None
|
data_parallel_size: int = 1
|
||||||
pipeline_parallel_size: int | None = None
|
pipeline_parallel_size: int = 1
|
||||||
|
|
||||||
|
|
||||||
_STAGE_NON_ENGINE_KEYS = frozenset(
|
_STAGE_NON_ENGINE_KEYS = frozenset(
|
||||||
@@ -697,18 +689,6 @@ _PIPELINE_WIDE_ENGINE_FIELDS: tuple[str, ...] = (
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
def deploy_override_field_names() -> frozenset[str]:
|
|
||||||
"""Return deploy-schema fields whose CLI defaults must not override YAML."""
|
|
||||||
return (
|
|
||||||
frozenset(_STAGE_DEPLOY_FIELDS)
|
|
||||||
| frozenset(_PIPELINE_WIDE_ENGINE_FIELDS)
|
|
||||||
| {
|
|
||||||
"async_chunk",
|
|
||||||
"devices",
|
|
||||||
}
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
def _build_engine_args(
|
def _build_engine_args(
|
||||||
ps: StagePipelineConfig,
|
ps: StagePipelineConfig,
|
||||||
ds: StageDeployConfig | None,
|
ds: StageDeployConfig | None,
|
||||||
@@ -881,15 +861,13 @@ class StageConfig:
|
|||||||
|
|
||||||
# CLI overrides take precedence over YAML defaults
|
# CLI overrides take precedence over YAML defaults
|
||||||
for key, value in self.runtime_overrides.items():
|
for key, value in self.runtime_overrides.items():
|
||||||
if value is None:
|
|
||||||
continue
|
|
||||||
if key not in ("devices", "max_batch_size"):
|
if key not in ("devices", "max_batch_size"):
|
||||||
engine_args[key] = value
|
engine_args[key] = value
|
||||||
|
|
||||||
# Build runtime config from YAML defaults + CLI overrides
|
# Build runtime config from YAML defaults + CLI overrides
|
||||||
runtime: dict[str, Any] = dict(self.yaml_runtime)
|
runtime: dict[str, Any] = dict(self.yaml_runtime)
|
||||||
runtime.setdefault("process", True)
|
runtime.setdefault("process", True)
|
||||||
if self.runtime_overrides.get("devices") is not None:
|
if "devices" in self.runtime_overrides:
|
||||||
runtime["devices"] = self.runtime_overrides["devices"]
|
runtime["devices"] = self.runtime_overrides["devices"]
|
||||||
|
|
||||||
# Legacy compat: migrate runtime.max_batch_size → engine_args.max_num_seqs
|
# Legacy compat: migrate runtime.max_batch_size → engine_args.max_num_seqs
|
||||||
@@ -905,6 +883,8 @@ class StageConfig:
|
|||||||
effective_mbs = int(cli_mbs or legacy_mbs or 1)
|
effective_mbs = int(cli_mbs or legacy_mbs or 1)
|
||||||
engine_args.setdefault("max_num_seqs", effective_mbs)
|
engine_args.setdefault("max_num_seqs", effective_mbs)
|
||||||
|
|
||||||
|
engine_args.setdefault("max_num_seqs", 1)
|
||||||
|
|
||||||
# Build full config dict
|
# Build full config dict
|
||||||
config_dict: dict[str, Any] = {
|
config_dict: dict[str, Any] = {
|
||||||
"stage_id": self.stage_id,
|
"stage_id": self.stage_id,
|
||||||
@@ -1029,8 +1009,6 @@ class StageConfigFactory:
|
|||||||
cli_overrides = {}
|
cli_overrides = {}
|
||||||
|
|
||||||
trust_remote_code = cli_overrides.get("trust_remote_code", True)
|
trust_remote_code = cli_overrides.get("trust_remote_code", True)
|
||||||
if trust_remote_code is None:
|
|
||||||
trust_remote_code = False
|
|
||||||
|
|
||||||
# --- New path: check pipeline registry by model_type first ---
|
# --- New path: check pipeline registry by model_type first ---
|
||||||
model_type, hf_config = cls._auto_detect_model_type(model, trust_remote_code=trust_remote_code)
|
model_type, hf_config = cls._auto_detect_model_type(model, trust_remote_code=trust_remote_code)
|
||||||
@@ -1374,8 +1352,8 @@ class StageConfigFactory:
|
|||||||
|
|
||||||
hf_config = get_config(model, trust_remote_code=trust_remote_code)
|
hf_config = get_config(model, trust_remote_code=trust_remote_code)
|
||||||
return hf_config.model_type, hf_config
|
return hf_config.model_type, hf_config
|
||||||
except Exception as e:
|
except Exception:
|
||||||
logger.debug(f"`get_config` failed for {e}; Falling back to raw config.json path")
|
pass
|
||||||
|
|
||||||
# Fallback: read config.json directly for custom model types that
|
# Fallback: read config.json directly for custom model types that
|
||||||
# are not registered with transformers (e.g. qwen3_tts).
|
# are not registered with transformers (e.g. qwen3_tts).
|
||||||
@@ -1412,8 +1390,8 @@ class StageConfigFactory:
|
|||||||
class_name,
|
class_name,
|
||||||
)
|
)
|
||||||
return pipeline_cfg.model_type, None
|
return pipeline_cfg.model_type, None
|
||||||
except Exception as e:
|
except Exception:
|
||||||
logger.debug(f"Failed to detect model type for diffusers-style models: {e}")
|
pass
|
||||||
|
|
||||||
# Final fallback: some models (e.g. CosyVoice3) ship an empty
|
# Final fallback: some models (e.g. CosyVoice3) ship an empty
|
||||||
# config.json and rely on naming conventions. Match the model path
|
# config.json and rely on naming conventions. Match the model path
|
||||||
|
|||||||
@@ -40,7 +40,7 @@ stages:
|
|||||||
# near-identity repetition penalty forces vLLM to track
|
# near-identity repetition penalty forces vLLM to track
|
||||||
# output_token_ids for RAS (stop-token logit logsumexp).
|
# output_token_ids for RAS (stop-token logit logsumexp).
|
||||||
repetition_penalty: 1.0001
|
repetition_penalty: 1.0001
|
||||||
enable_prefix_caching: false
|
disable_hybrid_kv_cache_manager: true
|
||||||
mm_processor_cache_gb: 0
|
mm_processor_cache_gb: 0
|
||||||
skip_mm_profiling: true
|
skip_mm_profiling: true
|
||||||
|
|
||||||
@@ -54,5 +54,5 @@ stages:
|
|||||||
from_stage_0: connector_of_shared_memory
|
from_stage_0: connector_of_shared_memory
|
||||||
default_sampling_params:
|
default_sampling_params:
|
||||||
max_tokens: 2048
|
max_tokens: 2048
|
||||||
enable_prefix_caching: false
|
disable_hybrid_kv_cache_manager: true
|
||||||
skip_mm_profiling: true
|
skip_mm_profiling: true
|
||||||
@@ -42,7 +42,6 @@ stages:
|
|||||||
max_num_seqs: 1
|
max_num_seqs: 1
|
||||||
gpu_memory_utilization: 0.2
|
gpu_memory_utilization: 0.2
|
||||||
enforce_eager: true
|
enforce_eager: true
|
||||||
enable_prefix_caching: false
|
|
||||||
async_scheduling: false
|
async_scheduling: false
|
||||||
max_num_batched_tokens: 8192
|
max_num_batched_tokens: 8192
|
||||||
max_model_len: 8192
|
max_model_len: 8192
|
||||||
|
|||||||
@@ -22,7 +22,6 @@ stages:
|
|||||||
max_num_seqs: 1
|
max_num_seqs: 1
|
||||||
gpu_memory_utilization: 0.8
|
gpu_memory_utilization: 0.8
|
||||||
enforce_eager: true
|
enforce_eager: true
|
||||||
enable_prefix_caching: false
|
|
||||||
mm_processor_cache_gb: 0
|
mm_processor_cache_gb: 0
|
||||||
devices: "0"
|
devices: "0"
|
||||||
default_sampling_params:
|
default_sampling_params:
|
||||||
@@ -50,7 +49,6 @@ stages:
|
|||||||
max_num_seqs: 1
|
max_num_seqs: 1
|
||||||
gpu_memory_utilization: 0.15
|
gpu_memory_utilization: 0.15
|
||||||
enforce_eager: true
|
enforce_eager: true
|
||||||
enable_prefix_caching: false
|
|
||||||
enable_flashinfer_autotune: false
|
enable_flashinfer_autotune: false
|
||||||
async_scheduling: false
|
async_scheduling: false
|
||||||
devices: "0"
|
devices: "0"
|
||||||
|
|||||||
@@ -23,7 +23,6 @@ connectors:
|
|||||||
stages:
|
stages:
|
||||||
- stage_id: 0
|
- stage_id: 0
|
||||||
gpu_memory_utilization: 0.9
|
gpu_memory_utilization: 0.9
|
||||||
enable_prefix_caching: false
|
|
||||||
devices: "0"
|
devices: "0"
|
||||||
default_sampling_params:
|
default_sampling_params:
|
||||||
temperature: 0.4
|
temperature: 0.4
|
||||||
@@ -48,7 +47,6 @@ stages:
|
|||||||
- stage_id: 2
|
- stage_id: 2
|
||||||
gpu_memory_utilization: 0.1
|
gpu_memory_utilization: 0.1
|
||||||
enforce_eager: true
|
enforce_eager: true
|
||||||
enable_prefix_caching: false
|
|
||||||
async_scheduling: false
|
async_scheduling: false
|
||||||
max_num_batched_tokens: 51200
|
max_num_batched_tokens: 51200
|
||||||
devices: "1"
|
devices: "1"
|
||||||
|
|||||||
@@ -53,7 +53,6 @@ stages:
|
|||||||
max_num_seqs: 1
|
max_num_seqs: 1
|
||||||
gpu_memory_utilization: 0.3
|
gpu_memory_utilization: 0.3
|
||||||
enforce_eager: true
|
enforce_eager: true
|
||||||
enable_prefix_caching: false
|
|
||||||
async_scheduling: true
|
async_scheduling: true
|
||||||
# Must be divisible by num_code_groups and cover (left_context + chunk).
|
# Must be divisible by num_code_groups and cover (left_context + chunk).
|
||||||
# Prefill length is Q * num_frames (e.g. 16 * 2148 = 34368); keep
|
# Prefill length is Q * num_frames (e.g. 16 * 2148 = 34368); keep
|
||||||
|
|||||||
@@ -20,7 +20,6 @@ stages:
|
|||||||
max_num_batched_tokens: 4096
|
max_num_batched_tokens: 4096
|
||||||
max_model_len: 4096
|
max_model_len: 4096
|
||||||
devices: "0"
|
devices: "0"
|
||||||
trust_remote_code: true
|
|
||||||
default_sampling_params:
|
default_sampling_params:
|
||||||
temperature: 0.0
|
temperature: 0.0
|
||||||
top_p: 1.0
|
top_p: 1.0
|
||||||
|
|||||||
@@ -24,7 +24,6 @@ stages:
|
|||||||
max_num_seqs: 32
|
max_num_seqs: 32
|
||||||
gpu_memory_utilization: 0.8
|
gpu_memory_utilization: 0.8
|
||||||
enforce_eager: false
|
enforce_eager: false
|
||||||
enable_prefix_caching: false
|
|
||||||
async_scheduling: true
|
async_scheduling: true
|
||||||
max_model_len: 4096
|
max_model_len: 4096
|
||||||
devices: "0"
|
devices: "0"
|
||||||
@@ -49,7 +48,6 @@ stages:
|
|||||||
max_num_seqs: 32
|
max_num_seqs: 32
|
||||||
gpu_memory_utilization: 0.1
|
gpu_memory_utilization: 0.1
|
||||||
enforce_eager: true
|
enforce_eager: true
|
||||||
enable_prefix_caching: false
|
|
||||||
async_scheduling: false
|
async_scheduling: false
|
||||||
max_num_batched_tokens: 65536
|
max_num_batched_tokens: 65536
|
||||||
max_model_len: 65536
|
max_model_len: 65536
|
||||||
|
|||||||
@@ -385,9 +385,9 @@ class MultiprocDiffusionExecutor(DiffusionExecutor):
|
|||||||
raise
|
raise
|
||||||
|
|
||||||
def check_health(self) -> None:
|
def check_health(self) -> None:
|
||||||
|
self._ensure_open()
|
||||||
if self.is_failed:
|
if self.is_failed:
|
||||||
raise EngineDeadError()
|
raise EngineDeadError()
|
||||||
self._ensure_open()
|
|
||||||
for p in self._processes:
|
for p in self._processes:
|
||||||
if not p.is_alive():
|
if not p.is_alive():
|
||||||
self.is_failed = True
|
self.is_failed = True
|
||||||
|
|||||||
@@ -15,7 +15,6 @@ from typing import TYPE_CHECKING, Any
|
|||||||
import torch
|
import torch
|
||||||
from PIL import Image
|
from PIL import Image
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
from vllm.v1.engine.exceptions import EngineDeadError
|
|
||||||
|
|
||||||
from vllm_omni.diffusion.data import DiffusionRequestAbortedError
|
from vllm_omni.diffusion.data import DiffusionRequestAbortedError
|
||||||
from vllm_omni.diffusion.diffusion_engine import DiffusionEngine
|
from vllm_omni.diffusion.diffusion_engine import DiffusionEngine
|
||||||
@@ -330,12 +329,6 @@ class InlineStageDiffusionClient:
|
|||||||
None,
|
None,
|
||||||
)
|
)
|
||||||
|
|
||||||
def check_health(self) -> None:
|
|
||||||
"""Check if the inline diffusion engine and its workers are healthy."""
|
|
||||||
if self._shutting_down:
|
|
||||||
raise EngineDeadError("InlineStageDiffusionClient is shutting down")
|
|
||||||
self._engine.executor.check_health()
|
|
||||||
|
|
||||||
def shutdown(self) -> None:
|
def shutdown(self) -> None:
|
||||||
self._shutting_down = True
|
self._shutting_down = True
|
||||||
|
|
||||||
|
|||||||
@@ -1842,8 +1842,6 @@ class Bagel(nn.Module):
|
|||||||
# ── SP + CFG: sequential single-branch forwards ──
|
# ── SP + CFG: sequential single-branch forwards ──
|
||||||
use_sp = self._sp_size > 1
|
use_sp = self._sp_size > 1
|
||||||
if use_sp and use_cfg_text:
|
if use_sp and use_cfg_text:
|
||||||
if return_trajectory_latents and len(timesteps) > 0:
|
|
||||||
trajectory_latents.append(x_t.clone())
|
|
||||||
for i, t in enumerate(timesteps):
|
for i, t in enumerate(timesteps):
|
||||||
timestep = torch.tensor([t] * x_t.shape[0], device=x_t.device)
|
timestep = torch.tensor([t] * x_t.shape[0], device=x_t.device)
|
||||||
in_cfg_window = t > cfg_interval[0] and t <= cfg_interval[1]
|
in_cfg_window = t > cfg_interval[0] and t <= cfg_interval[1]
|
||||||
@@ -1907,15 +1905,13 @@ class Bagel(nn.Module):
|
|||||||
x_t = x_t - v_t.to(x_t.device) * dts[i]
|
x_t = x_t - v_t.to(x_t.device) * dts[i]
|
||||||
if return_trajectory_latents:
|
if return_trajectory_latents:
|
||||||
trajectory_latents.append(x_t.clone())
|
trajectory_latents.append(x_t.clone())
|
||||||
trajectory_timesteps.append(timesteps[i])
|
trajectory_timesteps.append(timesteps[i] - dts[i])
|
||||||
|
|
||||||
unpacked_latent = x_t.split((packed_seqlens - 2).tolist())
|
unpacked_latent = x_t.split((packed_seqlens - 2).tolist())
|
||||||
return unpacked_latent, trajectory_latents, trajectory_timesteps, trajectory_log_probs
|
return unpacked_latent, trajectory_latents, trajectory_timesteps, trajectory_log_probs
|
||||||
|
|
||||||
# ── SP without CFG: direct single-branch loop ──
|
# ── SP without CFG: direct single-branch loop ──
|
||||||
if use_sp:
|
if use_sp:
|
||||||
if return_trajectory_latents and len(timesteps) > 0:
|
|
||||||
trajectory_latents.append(x_t.clone())
|
|
||||||
for i, t in enumerate(timesteps):
|
for i, t in enumerate(timesteps):
|
||||||
timestep = torch.tensor([t] * x_t.shape[0], device=x_t.device)
|
timestep = torch.tensor([t] * x_t.shape[0], device=x_t.device)
|
||||||
v_t = self.forward_single_branch(
|
v_t = self.forward_single_branch(
|
||||||
@@ -1942,7 +1938,7 @@ class Bagel(nn.Module):
|
|||||||
x_t = x_t - v_t.to(x_t.device) * dts[i]
|
x_t = x_t - v_t.to(x_t.device) * dts[i]
|
||||||
if return_trajectory_latents:
|
if return_trajectory_latents:
|
||||||
trajectory_latents.append(x_t.clone())
|
trajectory_latents.append(x_t.clone())
|
||||||
trajectory_timesteps.append(timesteps[i])
|
trajectory_timesteps.append(timesteps[i] - dts[i])
|
||||||
|
|
||||||
unpacked_latent = x_t.split((packed_seqlens - 2).tolist())
|
unpacked_latent = x_t.split((packed_seqlens - 2).tolist())
|
||||||
return unpacked_latent, trajectory_latents, trajectory_timesteps, trajectory_log_probs
|
return unpacked_latent, trajectory_latents, trajectory_timesteps, trajectory_log_probs
|
||||||
@@ -2003,9 +1999,6 @@ class Bagel(nn.Module):
|
|||||||
"merged_cache": self._merge_naive_caches(branches_cache),
|
"merged_cache": self._merge_naive_caches(branches_cache),
|
||||||
}
|
}
|
||||||
|
|
||||||
if return_trajectory_latents and len(timesteps) > 0:
|
|
||||||
trajectory_latents.append(x_t.clone())
|
|
||||||
|
|
||||||
for i, t in enumerate(timesteps):
|
for i, t in enumerate(timesteps):
|
||||||
timestep = torch.tensor([t] * x_t.shape[0], device=x_t.device)
|
timestep = torch.tensor([t] * x_t.shape[0], device=x_t.device)
|
||||||
if t > cfg_interval[0] and t <= cfg_interval[1]:
|
if t > cfg_interval[0] and t <= cfg_interval[1]:
|
||||||
@@ -2043,7 +2036,7 @@ class Bagel(nn.Module):
|
|||||||
x_t = x_t - v_t.to(x_t.device) * dts[i] # velocity pointing from data to noise
|
x_t = x_t - v_t.to(x_t.device) * dts[i] # velocity pointing from data to noise
|
||||||
if return_trajectory_latents:
|
if return_trajectory_latents:
|
||||||
trajectory_latents.append(x_t.clone())
|
trajectory_latents.append(x_t.clone())
|
||||||
trajectory_timesteps.append(timesteps[i])
|
trajectory_timesteps.append(timesteps[i] - dts[i])
|
||||||
|
|
||||||
unpacked_latent = x_t.split((packed_seqlens - 2).tolist())
|
unpacked_latent = x_t.split((packed_seqlens - 2).tolist())
|
||||||
return unpacked_latent, trajectory_latents, trajectory_timesteps, trajectory_log_probs
|
return unpacked_latent, trajectory_latents, trajectory_timesteps, trajectory_log_probs
|
||||||
@@ -2144,9 +2137,6 @@ class Bagel(nn.Module):
|
|||||||
)
|
)
|
||||||
_sched_kw = scheduler_kwargs or {}
|
_sched_kw = scheduler_kwargs or {}
|
||||||
|
|
||||||
if return_trajectory_latents and len(timesteps) > 0:
|
|
||||||
trajectory_latents.append(x_t.clone())
|
|
||||||
|
|
||||||
for i, t in enumerate(timesteps):
|
for i, t in enumerate(timesteps):
|
||||||
timestep = torch.tensor([t] * x_t.shape[0], device=x_t.device)
|
timestep = torch.tensor([t] * x_t.shape[0], device=x_t.device)
|
||||||
use_cfg_this_step = t > cfg_interval[0] and t <= cfg_interval[1] and cfg_text_scale > 1.0
|
use_cfg_this_step = t > cfg_interval[0] and t <= cfg_interval[1] and cfg_text_scale > 1.0
|
||||||
@@ -2204,7 +2194,7 @@ class Bagel(nn.Module):
|
|||||||
x_t = x_t - v_t.to(x_t.device) * dts[i]
|
x_t = x_t - v_t.to(x_t.device) * dts[i]
|
||||||
if return_trajectory_latents:
|
if return_trajectory_latents:
|
||||||
trajectory_latents.append(x_t.clone())
|
trajectory_latents.append(x_t.clone())
|
||||||
trajectory_timesteps.append(timesteps[i])
|
trajectory_timesteps.append(timesteps[i] - dts[i])
|
||||||
|
|
||||||
unpacked_latent = x_t.split((packed_seqlens - 2).tolist())
|
unpacked_latent = x_t.split((packed_seqlens - 2).tolist())
|
||||||
return unpacked_latent, trajectory_latents, trajectory_timesteps, trajectory_log_probs
|
return unpacked_latent, trajectory_latents, trajectory_timesteps, trajectory_log_probs
|
||||||
|
|||||||
@@ -456,12 +456,50 @@ SHARED_FIELDS: frozenset[str] = frozenset(
|
|||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
|
_DEPLOY_ENGINE_ARG_OVERRIDE_FIELDS: frozenset[str] = frozenset(
|
||||||
|
{
|
||||||
|
# Capacity / scheduling.
|
||||||
|
"async_scheduling",
|
||||||
|
"max_model_len",
|
||||||
|
"max_num_batched_tokens",
|
||||||
|
"max_num_seqs",
|
||||||
|
# Memory / parallelism.
|
||||||
|
"data_parallel_size",
|
||||||
|
"gpu_memory_utilization",
|
||||||
|
"pipeline_parallel_size",
|
||||||
|
"tensor_parallel_size",
|
||||||
|
# Execution / loading.
|
||||||
|
"enforce_eager",
|
||||||
|
"distributed_executor_backend",
|
||||||
|
"dtype",
|
||||||
|
"quantization",
|
||||||
|
"trust_remote_code",
|
||||||
|
# Caching / chunking.
|
||||||
|
"async_chunk",
|
||||||
|
"enable_prefix_caching",
|
||||||
|
"enable_chunked_prefill",
|
||||||
|
# Model-specific engine extras.
|
||||||
|
"subtalker_sampling_params",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
_DEPLOY_RUNTIME_OVERRIDE_FIELDS: frozenset[str] = frozenset(
|
||||||
|
{
|
||||||
|
"devices",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def orchestrator_field_names() -> frozenset[str]:
|
def orchestrator_field_names() -> frozenset[str]:
|
||||||
"""Return the names of every field on OrchestratorArgs."""
|
"""Return the names of every field on OrchestratorArgs."""
|
||||||
return frozenset(f.name for f in fields(OrchestratorArgs))
|
return frozenset(f.name for f in fields(OrchestratorArgs))
|
||||||
|
|
||||||
|
|
||||||
|
def deploy_override_field_names() -> frozenset[str]:
|
||||||
|
"""Return kwargs whose parser defaults must not override deploy YAML."""
|
||||||
|
return _DEPLOY_ENGINE_ARG_OVERRIDE_FIELDS | _DEPLOY_RUNTIME_OVERRIDE_FIELDS
|
||||||
|
|
||||||
|
|
||||||
def internal_blacklist_keys() -> frozenset[str]:
|
def internal_blacklist_keys() -> frozenset[str]:
|
||||||
"""Return the set of CLI keys that must never be forwarded as per-stage
|
"""Return the set of CLI keys that must never be forwarded as per-stage
|
||||||
engine overrides.
|
engine overrides.
|
||||||
@@ -615,8 +653,6 @@ def nullify_stage_engine_defaults(parser: argparse.ArgumentParser) -> None:
|
|||||||
"""Reset stage-level engine flag defaults to ``None``; preserve real
|
"""Reset stage-level engine flag defaults to ``None``; preserve real
|
||||||
default in help text. Only deploy-YAML override fields are touched.
|
default in help text. Only deploy-YAML override fields are touched.
|
||||||
Idempotent."""
|
Idempotent."""
|
||||||
from vllm_omni.config.stage_config import deploy_override_field_names
|
|
||||||
|
|
||||||
override_dests = deploy_override_field_names()
|
override_dests = deploy_override_field_names()
|
||||||
|
|
||||||
for action in parser._actions:
|
for action in parser._actions:
|
||||||
|
|||||||
@@ -1294,12 +1294,9 @@ class AsyncOmniEngine:
|
|||||||
ring_degree = normalized_kwargs.get("ring_degree") or 1
|
ring_degree = normalized_kwargs.get("ring_degree") or 1
|
||||||
ulysses_mode = normalized_kwargs.get("ulysses_mode") or "strict"
|
ulysses_mode = normalized_kwargs.get("ulysses_mode") or "strict"
|
||||||
sequence_parallel_size = normalized_kwargs.get("sequence_parallel_size")
|
sequence_parallel_size = normalized_kwargs.get("sequence_parallel_size")
|
||||||
pipeline_parallel_size = normalized_kwargs.get("pipeline_parallel_size") or 1
|
|
||||||
data_parallel_size = normalized_kwargs.get("data_parallel_size") or 1
|
|
||||||
tensor_parallel_size = normalized_kwargs.get("tensor_parallel_size") or 1
|
tensor_parallel_size = normalized_kwargs.get("tensor_parallel_size") or 1
|
||||||
cfg_parallel_size = normalized_kwargs.get("cfg_parallel_size") or 1
|
cfg_parallel_size = normalized_kwargs.get("cfg_parallel_size") or 1
|
||||||
vae_patch_parallel_size = normalized_kwargs.get("vae_patch_parallel_size") or 1
|
vae_patch_parallel_size = normalized_kwargs.get("vae_patch_parallel_size") or 1
|
||||||
enable_expert_parallel = normalized_kwargs.get("enable_expert_parallel") or False
|
|
||||||
use_hsdp = normalized_kwargs.get("use_hsdp", False)
|
use_hsdp = normalized_kwargs.get("use_hsdp", False)
|
||||||
hsdp_shard_size = normalized_kwargs.get("hsdp_shard_size", -1)
|
hsdp_shard_size = normalized_kwargs.get("hsdp_shard_size", -1)
|
||||||
hsdp_replicate_size = normalized_kwargs.get("hsdp_replicate_size", 1)
|
hsdp_replicate_size = normalized_kwargs.get("hsdp_replicate_size", 1)
|
||||||
@@ -1307,10 +1304,9 @@ class AsyncOmniEngine:
|
|||||||
sequence_parallel_size = ulysses_degree * ring_degree
|
sequence_parallel_size = ulysses_degree * ring_degree
|
||||||
|
|
||||||
parallel_config = DiffusionParallelConfig(
|
parallel_config = DiffusionParallelConfig(
|
||||||
pipeline_parallel_size=pipeline_parallel_size,
|
pipeline_parallel_size=1,
|
||||||
data_parallel_size=data_parallel_size,
|
data_parallel_size=1,
|
||||||
tensor_parallel_size=tensor_parallel_size,
|
tensor_parallel_size=tensor_parallel_size,
|
||||||
enable_expert_parallel=enable_expert_parallel,
|
|
||||||
sequence_parallel_size=sequence_parallel_size,
|
sequence_parallel_size=sequence_parallel_size,
|
||||||
ulysses_degree=ulysses_degree,
|
ulysses_degree=ulysses_degree,
|
||||||
ring_degree=ring_degree,
|
ring_degree=ring_degree,
|
||||||
@@ -1324,9 +1320,6 @@ class AsyncOmniEngine:
|
|||||||
|
|
||||||
num_devices = max(1, int(parallel_config.world_size))
|
num_devices = max(1, int(parallel_config.world_size))
|
||||||
devices = ",".join(str(i) for i in range(num_devices))
|
devices = ",".join(str(i) for i in range(num_devices))
|
||||||
enforce_eager = kwargs.get("enforce_eager")
|
|
||||||
if enforce_eager is None:
|
|
||||||
enforce_eager = False
|
|
||||||
|
|
||||||
stage_engine_args = {
|
stage_engine_args = {
|
||||||
"max_num_seqs": 1,
|
"max_num_seqs": 1,
|
||||||
@@ -1340,7 +1333,7 @@ class AsyncOmniEngine:
|
|||||||
"enable_cache_dit_summary": kwargs.get("enable_cache_dit_summary", False),
|
"enable_cache_dit_summary": kwargs.get("enable_cache_dit_summary", False),
|
||||||
"enable_cpu_offload": kwargs.get("enable_cpu_offload", False),
|
"enable_cpu_offload": kwargs.get("enable_cpu_offload", False),
|
||||||
"enable_layerwise_offload": kwargs.get("enable_layerwise_offload", False),
|
"enable_layerwise_offload": kwargs.get("enable_layerwise_offload", False),
|
||||||
"enforce_eager": enforce_eager,
|
"enforce_eager": kwargs.get("enforce_eager", False),
|
||||||
"boundary_ratio": kwargs.get("boundary_ratio", None),
|
"boundary_ratio": kwargs.get("boundary_ratio", None),
|
||||||
"flow_shift": kwargs.get("flow_shift", None),
|
"flow_shift": kwargs.get("flow_shift", None),
|
||||||
"diffusion_load_format": kwargs.get("diffusion_load_format", "default"),
|
"diffusion_load_format": kwargs.get("diffusion_load_format", "default"),
|
||||||
|
|||||||
@@ -100,7 +100,9 @@ class OmniBase(PDDisaggregationMixin):
|
|||||||
kwargs: dict[str, Any] = {k: v for k, v in vars(args).items() if not k.startswith("_")}
|
kwargs: dict[str, Any] = {k: v for k, v in vars(args).items() if not k.startswith("_")}
|
||||||
|
|
||||||
if parser is not None and not getattr(parser, "_omni_nullified", False):
|
if parser is not None and not getattr(parser, "_omni_nullified", False):
|
||||||
from vllm_omni.config.stage_config import deploy_override_field_names
|
from vllm_omni.engine.arg_utils import (
|
||||||
|
deploy_override_field_names,
|
||||||
|
)
|
||||||
from vllm_omni.entrypoints.utils import detect_explicit_cli_keys
|
from vllm_omni.entrypoints.utils import detect_explicit_cli_keys
|
||||||
|
|
||||||
explicit = detect_explicit_cli_keys(sys.argv[1:], parser) or set()
|
explicit = detect_explicit_cli_keys(sys.argv[1:], parser) or set()
|
||||||
|
|||||||
@@ -65,7 +65,7 @@ from vllm.entrypoints.openai.speech_to_text.serving import (
|
|||||||
from vllm.entrypoints.openai.utils import validate_json_request
|
from vllm.entrypoints.openai.utils import validate_json_request
|
||||||
from vllm.entrypoints.pooling.classify.serving import ServingClassification
|
from vllm.entrypoints.pooling.classify.serving import ServingClassification
|
||||||
from vllm.entrypoints.pooling.embed.serving import ServingEmbedding as OpenAIServingEmbedding
|
from vllm.entrypoints.pooling.embed.serving import ServingEmbedding as OpenAIServingEmbedding
|
||||||
from vllm.entrypoints.pooling.pooling.serving import ServingPooling
|
from vllm.entrypoints.pooling.pooling.serving import ServingPooling as OpenAIServingPooling
|
||||||
from vllm.entrypoints.pooling.scoring.serving import ServingScores
|
from vllm.entrypoints.pooling.scoring.serving import ServingScores
|
||||||
from vllm.entrypoints.serve.disagg.serving import ServingTokens
|
from vllm.entrypoints.serve.disagg.serving import ServingTokens
|
||||||
|
|
||||||
@@ -800,9 +800,10 @@ async def omni_init_app_state(
|
|||||||
else None
|
else None
|
||||||
)
|
)
|
||||||
state.openai_serving_pooling = (
|
state.openai_serving_pooling = (
|
||||||
ServingPooling(
|
OpenAIServingPooling(
|
||||||
engine_client,
|
engine_client,
|
||||||
state.openai_serving_models,
|
state.openai_serving_models,
|
||||||
|
state.openai_serving_render,
|
||||||
supported_tasks=tuple(supported_tasks),
|
supported_tasks=tuple(supported_tasks),
|
||||||
request_logger=request_logger,
|
request_logger=request_logger,
|
||||||
chat_template=resolved_chat_template,
|
chat_template=resolved_chat_template,
|
||||||
|
|||||||
@@ -489,7 +489,7 @@ class Qwen3OmniMoeModel(Qwen3MoeLLMForCausalLM):
|
|||||||
"""
|
"""
|
||||||
Qwen3 Omni MoE Talker language model.
|
Qwen3 Omni MoE Talker language model.
|
||||||
|
|
||||||
Extends Qwen3MoeLLMForCausalLM (which already uses FusedMoE with
|
Extends Qwen3MoeLLMForCausalLM (which already uses SharedFusedMoE with
|
||||||
shared-expert support) and replaces the text embedding / LM head with a
|
shared-expert support) and replaces the text embedding / LM head with a
|
||||||
codec embedding so the talker operates over audio-codec tokens instead
|
codec embedding so the talker operates over audio-codec tokens instead
|
||||||
of text tokens.
|
of text tokens.
|
||||||
|
|||||||
@@ -103,9 +103,6 @@ def prepare_hunyuan_fused_moe_runtime() -> None:
|
|||||||
_ensure_forward_context_attr("flash_comm_v1_enabled", bool, False)
|
_ensure_forward_context_attr("flash_comm_v1_enabled", bool, False)
|
||||||
|
|
||||||
|
|
||||||
# NOTE: As of v0.20,0 vLLM has folded SharedFusedMoE -> FusedMoE and removed the class.
|
|
||||||
# Since AscendSharedFusedMoE is a subclass of SharedFusedMoE, we should be careful
|
|
||||||
# to ensure that this is updated correctly.
|
|
||||||
class AscendHunyuanFusedMoE(AscendSharedFusedMoE):
|
class AscendHunyuanFusedMoE(AscendSharedFusedMoE):
|
||||||
def __init__(self, *, prefix: str = "", **kwargs: Any) -> None:
|
def __init__(self, *, prefix: str = "", **kwargs: Any) -> None:
|
||||||
super().__init__(prefix=prefix, **kwargs)
|
super().__init__(prefix=prefix, **kwargs)
|
||||||
|
|||||||
Reference in new issue
Block a user