Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
 hsliuustc0106
|
7db3a0cd5e |
No files matched your search
@@ -1,7 +1,4 @@
|
||||
#!/bin/bash
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
# vllm-omni customized version
|
||||
# Based on: https://github.com/vllm-project/ci-infra/blob/main/buildkite/bootstrap-amd.sh
|
||||
# Last synced: 2025-12-15
|
||||
@@ -9,9 +6,6 @@
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# The bootstrap runs from the repository root; ShellCheck does not follow
|
||||
# sourced files unless invoked with -x.
|
||||
# shellcheck disable=SC1091
|
||||
source .buildkite/common/scripts/resolve_skip_ci.sh
|
||||
|
||||
if [[ -z "${RUN_ALL:-}" ]]; then
|
||||
@@ -69,8 +63,6 @@ upload_pipeline() {
|
||||
# Install minijinja
|
||||
ls .buildkite || buildkite-agent annotate --style error 'Please merge upstream main branch for buildkite CI'
|
||||
curl -sSfL https://github.com/mitsuhiko/minijinja/releases/download/2.3.1/minijinja-cli-installer.sh | sh
|
||||
# Installed by the minijinja bootstrap above and only present on the CI agent.
|
||||
# shellcheck disable=SC1091
|
||||
source /var/lib/buildkite-agent/.cargo/env
|
||||
|
||||
if [[ $BUILDKITE_PIPELINE_SLUG == "fastcheck" ]]; then
|
||||
@@ -98,82 +90,22 @@ upload_pipeline() {
|
||||
cd .buildkite/amd
|
||||
|
||||
# Select test definition file: merge suite for main, ready suite for PRs.
|
||||
# For debugging, DEBUG_TEST_YAML accepts a comma-separated list containing
|
||||
# "merge" and/or "ready" (case-insensitive). Multiple suites are combined
|
||||
# before rendering so they share one amd-build step.
|
||||
# For debugging, DEBUG_TEST_YAML can override the selection — accepts
|
||||
# "merge" or "ready" (case-insensitive).
|
||||
if [[ -n "${DEBUG_TEST_YAML:-}" ]]; then
|
||||
declare -a DEBUG_TEST_SPECS=()
|
||||
declare -A SEEN_DEBUG_TESTS=()
|
||||
IFS=',' read -ra REQUESTED_DEBUG_TESTS <<< "${DEBUG_TEST_YAML,,}"
|
||||
|
||||
for requested_test in "${REQUESTED_DEBUG_TESTS[@]}"; do
|
||||
# Permit readable values such as "ready, merge," and ignore the
|
||||
# empty item produced by a trailing comma.
|
||||
requested_test="${requested_test//[[:space:]]/}"
|
||||
[[ -z "$requested_test" ]] && continue
|
||||
|
||||
if [[ -n "${SEEN_DEBUG_TESTS[$requested_test]:-}" ]]; then
|
||||
echo "ERROR: duplicate DEBUG_TEST_YAML suite '$requested_test'" >&2
|
||||
case "${DEBUG_TEST_YAML,,}" in
|
||||
merge)
|
||||
TEST_YAML="test-amd-merge.yml"
|
||||
;;
|
||||
ready)
|
||||
TEST_YAML="test-amd-ready.yml"
|
||||
;;
|
||||
*)
|
||||
echo "ERROR: DEBUG_TEST_YAML must be 'merge' or 'ready', got '$DEBUG_TEST_YAML'" >&2
|
||||
exit 1
|
||||
fi
|
||||
SEEN_DEBUG_TESTS[$requested_test]=1
|
||||
|
||||
case "$requested_test" in
|
||||
ready)
|
||||
DEBUG_TEST_SPECS+=("READY_TESTS:test-amd-ready.yml")
|
||||
;;
|
||||
merge)
|
||||
DEBUG_TEST_SPECS+=("MERGE_TESTS:test-amd-merge.yml")
|
||||
;;
|
||||
*)
|
||||
echo "ERROR: DEBUG_TEST_YAML entries must be 'merge' or 'ready', got '$requested_test'" >&2
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [[ ${#DEBUG_TEST_SPECS[@]} -eq 0 ]]; then
|
||||
echo "ERROR: DEBUG_TEST_YAML did not contain a test suite" >&2
|
||||
exit 1
|
||||
elif [[ ${#DEBUG_TEST_SPECS[@]} -eq 1 ]]; then
|
||||
TEST_YAML="${DEBUG_TEST_SPECS[0]#*:}"
|
||||
else
|
||||
TEST_YAML=$(mktemp "${TMPDIR:-/tmp}/amd-debug-tests.XXXXXX.yml")
|
||||
python - "$TEST_YAML" "${DEBUG_TEST_SPECS[@]}" <<'PY'
|
||||
import sys
|
||||
|
||||
import yaml
|
||||
|
||||
|
||||
output_path, *suite_specs = sys.argv[1:]
|
||||
combined = {"env": {}, "steps": []}
|
||||
|
||||
for suite_spec in suite_specs:
|
||||
group_name, input_path = suite_spec.split(":", 1)
|
||||
with open(input_path, encoding="utf-8") as test_file:
|
||||
suite = yaml.safe_load(test_file)
|
||||
|
||||
for name, value in (suite.get("env") or {}).items():
|
||||
previous = combined["env"].get(name, value)
|
||||
if previous != value:
|
||||
raise ValueError(
|
||||
f"Conflicting environment value for {name}: {previous!r} != {value!r}"
|
||||
)
|
||||
combined["env"][name] = value
|
||||
|
||||
suite_steps = []
|
||||
for entry in suite.get("steps") or []:
|
||||
if "group" in entry:
|
||||
suite_steps.extend(entry.get("steps") or [])
|
||||
else:
|
||||
suite_steps.append(entry)
|
||||
combined["steps"].append({"group": group_name, "steps": suite_steps})
|
||||
|
||||
with open(output_path, "w", encoding="utf-8") as output_file:
|
||||
yaml.safe_dump(combined, output_file, sort_keys=False)
|
||||
PY
|
||||
fi
|
||||
echo "DEBUG_TEST_YAML override: using ${DEBUG_TEST_SPECS[*]}"
|
||||
;;
|
||||
esac
|
||||
echo "DEBUG_TEST_YAML override: using $TEST_YAML"
|
||||
elif [[ $BUILDKITE_BRANCH == "main" ]]; then
|
||||
TEST_YAML="test-amd-merge.yml"
|
||||
else
|
||||
@@ -198,20 +130,19 @@ PY
|
||||
> pipeline.yaml
|
||||
)
|
||||
cat pipeline.yaml
|
||||
if [[ "$TEST_YAML" == "${TMPDIR:-/tmp}/amd-debug-tests."*.yml ]]; then
|
||||
rm -f -- "$TEST_YAML"
|
||||
fi
|
||||
buildkite-agent artifact upload pipeline.yaml
|
||||
buildkite-agent pipeline upload pipeline.yaml
|
||||
exit 0
|
||||
}
|
||||
|
||||
get_diff() {
|
||||
git diff --name-only --diff-filter=ACMDR "$(git merge-base origin/main HEAD)"
|
||||
$(git add .)
|
||||
echo $(git diff --name-only --diff-filter=ACMDR $(git merge-base origin/main HEAD))
|
||||
}
|
||||
|
||||
get_diff_main() {
|
||||
git diff --name-only --diff-filter=ACMDR HEAD~1
|
||||
$(git add .)
|
||||
echo $(git diff --name-only --diff-filter=ACMDR HEAD~1)
|
||||
}
|
||||
|
||||
file_diff=$(get_diff)
|
||||
@@ -250,7 +181,7 @@ for file in $file_diff; do
|
||||
# First check if file matches any pattern
|
||||
matches_pattern=0
|
||||
for pattern in "${patterns[@]}"; do
|
||||
if [[ $file == "$pattern"* ]] || [[ $file == "$pattern" ]]; then
|
||||
if [[ $file == $pattern* ]] || [[ $file == $pattern ]]; then
|
||||
matches_pattern=1
|
||||
break
|
||||
fi
|
||||
@@ -260,7 +191,7 @@ for file in $file_diff; do
|
||||
if [[ $matches_pattern -eq 1 ]]; then
|
||||
matches_ignore=0
|
||||
for ignore in "${ignore_patterns[@]}"; do
|
||||
if [[ $file == "$ignore"* ]] || [[ $file == "$ignore" ]]; then
|
||||
if [[ $file == $ignore* ]] || [[ $file == $ignore ]]; then
|
||||
matches_ignore=1
|
||||
break
|
||||
fi
|
||||
|
||||
@@ -21,7 +21,7 @@ steps:
|
||||
|
||||
- label: "Simple · Diffusion Test · Shard %N/%t"
|
||||
agent_pool: mi300_1
|
||||
parallelism: 4
|
||||
parallelism: 2
|
||||
depends_on: amd-build
|
||||
mirror_hardwares: [amdproduction]
|
||||
grade: Blocking
|
||||
@@ -71,45 +71,26 @@ steps:
|
||||
commands:
|
||||
- export VLLM_LOGGING_LEVEL=DEBUG
|
||||
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
- pytest -s -v tests/diffusion/offloader/test_diffusion_cpu_offload.py
|
||||
- >-
|
||||
pytest -s -v tests/diffusion/offloader/test_diffusion_layerwise_offload.py
|
||||
--deselect='tests/diffusion/offloader/test_diffusion_layerwise_offload.py::test_layerwise_offload_diffusion_model[stabilityai/stable-audio-open-1.0]'
|
||||
- |
|
||||
bash -c '
|
||||
set +e
|
||||
pytest -s -v tests/diffusion/offloader/test_diffusion_cpu_offload.py
|
||||
EXIT1=\$?
|
||||
pytest -s -v tests/diffusion/offloader/test_diffusion_layerwise_offload.py
|
||||
EXIT2=\$?
|
||||
exit \$((EXIT1 | EXIT2))
|
||||
'
|
||||
|
||||
# TODO: Diagnose unstable Stable Audio layerwise-offload memory savings on ROCm.
|
||||
- label: "Diffusion Stable Audio Layerwise Offload Test (quarantined)"
|
||||
agent_pool: mi300_1
|
||||
depends_on: amd-build
|
||||
mirror_hardwares: [amdproduction]
|
||||
grade: NonBlocking
|
||||
commands:
|
||||
- export VLLM_LOGGING_LEVEL=DEBUG
|
||||
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
- >-
|
||||
pytest -s -v
|
||||
'tests/diffusion/offloader/test_diffusion_layerwise_offload.py::test_layerwise_offload_diffusion_model[stabilityai/stable-audio-open-1.0]'
|
||||
|
||||
- label: "Diffusion Sequence Parallelism Test (Need 4 GPUs) · Shard 1/2"
|
||||
- label: "Diffusion Sequence Parallelism Test (Need 4 GPUs)"
|
||||
agent_pool: mi300_4
|
||||
depends_on: amd-build
|
||||
mirror_hardwares: [amdproduction]
|
||||
timeout_in_minutes: 40
|
||||
grade: Blocking
|
||||
commands:
|
||||
- export VLLM_LOGGING_LEVEL=DEBUG
|
||||
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
- pytest -s -v tests/diffusion/distributed/test_sequence_parallel.py
|
||||
|
||||
- label: "Diffusion Sequence Parallelism Test (Need 4 GPUs) · Shard 2/2"
|
||||
agent_pool: mi300_4
|
||||
depends_on: amd-build
|
||||
mirror_hardwares: [amdproduction]
|
||||
timeout_in_minutes: 40
|
||||
grade: Blocking
|
||||
commands:
|
||||
- export VLLM_LOGGING_LEVEL=DEBUG
|
||||
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
- pytest -s -v tests/diffusion/distributed/test_ulysses_uaa_perf.py
|
||||
- timeout 20m pytest -s -v tests/diffusion/distributed/test_sequence_parallel.py
|
||||
- timeout 20m pytest -s -v tests/diffusion/distributed/test_ulysses_uaa_perf.py
|
||||
|
||||
# merge-only tests
|
||||
- label: "Diffusion Tensor Parallelism Test"
|
||||
@@ -190,45 +171,21 @@ steps:
|
||||
grade: Blocking
|
||||
commands:
|
||||
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
- pytest -s -v tests/e2e/offline_inference/test_qwen3_omni.py tests/e2e/online_serving/test_qwen3_omni.py tests/e2e/offline_inference/test_qwen3_omni_colocate_async.py -m 'advanced_model and cuda' --run-level 'advanced_model'
|
||||
- pytest -s -v tests/e2e/offline_inference/test_qwen3_omni.py tests/e2e/online_serving/test_qwen3_omni.py -m 'advanced_model and cuda' --run-level 'advanced_model'
|
||||
|
||||
- label: "Qwen3-TTS CustomVoice E2E Test · Shard %N/%t"
|
||||
agent_pool: mi300_1
|
||||
parallelism: 3
|
||||
parallelism: 2
|
||||
depends_on: amd-build
|
||||
mirror_hardwares: [amdproduction]
|
||||
grade: Blocking
|
||||
timeout_in_minutes: 80
|
||||
commands:
|
||||
- |
|
||||
timeout --signal=TERM --kill-after=2m 80m bash -c '
|
||||
bash -c '
|
||||
export VLLM_LOGGING_LEVEL=DEBUG
|
||||
export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
|
||||
TESTS=(
|
||||
"tests/e2e/online_serving/test_qwen3_tts_customvoice.py::test_default_cuda_graph_startup[async_chunk]"
|
||||
"tests/e2e/online_serving/test_qwen3_tts_customvoice.py::test_text_to_audio_002[async_chunk]"
|
||||
"tests/e2e/offline_inference/test_qwen3_tts_customvoice.py::test_text_to_audio_001[no_cuda_graph]"
|
||||
)
|
||||
TEST="$${TESTS[$$BUILDKITE_PARALLEL_JOB]}"
|
||||
pytest -s -v "$$TEST" -m "advanced_model" --run-level "advanced_model"
|
||||
'
|
||||
|
||||
# TODO: Restore this case to the blocking sharded step once its long-running
|
||||
# Whisper validation on AMD CI is resolved.
|
||||
- label: "Qwen3-TTS CustomVoice async_chunk Test (quarantined)"
|
||||
agent_pool: mi300_1
|
||||
depends_on: amd-build
|
||||
mirror_hardwares: [amdproduction]
|
||||
grade: NonBlocking
|
||||
timeout_in_minutes: 80
|
||||
commands:
|
||||
- |
|
||||
timeout --signal=TERM --kill-after=2m 75m bash -c '
|
||||
export VLLM_LOGGING_LEVEL=DEBUG
|
||||
export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
|
||||
pytest -s -v "tests/e2e/online_serving/test_qwen3_tts_customvoice.py::test_text_to_audio_001[async_chunk]" -m "advanced_model" --run-level "advanced_model"
|
||||
pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py tests/e2e/offline_inference/test_qwen3_tts_customvoice.py -m "advanced_model" --run-level "advanced_model" --num-shards=$$BUILDKITE_PARALLEL_JOB_COUNT --shard-id=$$BUILDKITE_PARALLEL_JOB
|
||||
'
|
||||
|
||||
- label: "Qwen3-TTS Base E2E Test"
|
||||
@@ -238,10 +195,13 @@ steps:
|
||||
grade: Blocking
|
||||
timeout_in_minutes: 50
|
||||
commands:
|
||||
- export VLLM_LOGGING_LEVEL=DEBUG
|
||||
- export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
- export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
|
||||
- timeout 40m pytest -s -v tests/e2e/online_serving/test_qwen3_tts_base.py tests/e2e/offline_inference/test_qwen3_tts_base.py -m "advanced_model and cuda" --run-level "advanced_model"
|
||||
- |
|
||||
timeout 40m bash -c '
|
||||
export VLLM_LOGGING_LEVEL=DEBUG
|
||||
export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
|
||||
pytest -s -v tests/e2e/online_serving/test_qwen3_tts_base.py tests/e2e/offline_inference/test_qwen3_tts_base.py -m 'advanced_model and cuda' --run-level 'advanced_model'
|
||||
'
|
||||
|
||||
# TODO: Bagel test on ROCm is very unstable. @tjtanaa
|
||||
# Need to debug before reneable numerical changes across large PRs
|
||||
|
||||
@@ -191,7 +191,10 @@ steps:
|
||||
mirror_hardwares: [amdproduction]
|
||||
grade: Blocking
|
||||
commands:
|
||||
- pytest -s -v tests/e2e/online_serving/test_cosyvoice3_tts_expansion.py -m "slow" --run-level "core_model"
|
||||
- |
|
||||
timeout 20m bash -c '
|
||||
pytest -s -v tests/e2e/online_serving/test_cosyvoice3_tts_expansion.py -m "slow" --run-level "core_model"
|
||||
'
|
||||
|
||||
# TODO: Bagel test on ROCm is very unstable. @tjtanaa
|
||||
# Need to debug before reneable numerical changes across large PRs
|
||||
|
||||
@@ -85,9 +85,6 @@ steps:
|
||||
- vllm_omni/model_executor/models/common/alias_free_activation.py
|
||||
- vllm_omni/model_executor/stage_input_processors/qwen3_tts.py
|
||||
- vllm_omni/deploy/qwen3_tts.yaml
|
||||
- vllm_omni/entrypoints/openai/serving_speech.py
|
||||
- vllm_omni/entrypoints/openai/tts_adapters/
|
||||
- vllm_omni/engine/stage_pool.py
|
||||
commands:
|
||||
- |
|
||||
timeout 40m bash -c "
|
||||
@@ -107,9 +104,6 @@ steps:
|
||||
- vllm_omni/model_executor/models/common/alias_free_activation.py
|
||||
- vllm_omni/model_executor/stage_input_processors/qwen3_tts.py
|
||||
- vllm_omni/deploy/qwen3_tts.yaml
|
||||
- vllm_omni/entrypoints/openai/serving_speech.py
|
||||
- vllm_omni/entrypoints/openai/tts_adapters/
|
||||
- vllm_omni/engine/stage_pool.py
|
||||
- .buildkite/common/scripts/run_cov_split.sh
|
||||
- pyproject.toml
|
||||
# 40 -> 60: the coverage split runs the model twice.
|
||||
@@ -128,7 +122,6 @@ steps:
|
||||
- label: "Omni · Qwen3-Omni Test"
|
||||
source_file_dependencies:
|
||||
- tests/e2e/offline_inference/test_qwen3_omni.py
|
||||
- tests/e2e/offline_inference/test_qwen3_omni_colocate_async.py
|
||||
- tests/e2e/online_serving/test_qwen3_omni.py
|
||||
- tests/helpers/runtime.py
|
||||
- vllm_omni/entrypoints/openai/serving_chat.py
|
||||
@@ -139,10 +132,9 @@ steps:
|
||||
- vllm_omni/model_executor/models/qwen3_tts/cuda_graph_decoder_wrapper.py
|
||||
- vllm_omni/model_executor/stage_input_processors/qwen3_omni.py
|
||||
- vllm_omni/deploy/qwen3_omni_moe.yaml
|
||||
- vllm_omni/deploy/qwen3_omni_moe_thinking.yaml
|
||||
timeout_in_minutes: 50
|
||||
commands:
|
||||
- pytest -s -v tests/e2e/offline_inference/test_qwen3_omni.py tests/e2e/online_serving/test_qwen3_omni.py tests/e2e/offline_inference/test_qwen3_omni_colocate_async.py -m 'advanced_model and cuda' --run-level 'advanced_model'
|
||||
- pytest -s -v tests/e2e/offline_inference/test_qwen3_omni.py tests/e2e/online_serving/test_qwen3_omni.py -m 'advanced_model and cuda' --run-level 'advanced_model'
|
||||
mirror_hardwares: h100_2
|
||||
|
||||
- label: "Omni · MiniCPM-o 4.5 Test"
|
||||
@@ -174,10 +166,10 @@ steps:
|
||||
- tests/e2e/online_serving/test_minicpmo_realtime_duplex_drivers.py
|
||||
- tests/e2e/online_serving/helpers/minicpmo_realtime_duplex_scenarios.py
|
||||
- tests/e2e/online_serving/run_minicpmo_realtime_duplex_multi_session.py
|
||||
- tests/e2e/online_serving/run_minicpmo_realtime_duplex_server_vad.py
|
||||
- tests/e2e/online_serving/run_minicpmo_realtime_duplex_soft_interrupt.py
|
||||
- vllm_omni/deploy/minicpmo_4_5.yaml
|
||||
- tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py
|
||||
- vllm_omni/deploy/minicpmo_4_5_duplex.yaml
|
||||
- vllm_omni/experimental/fullduplex/
|
||||
- vllm_omni/model_executor/models/minicpmo_4_5/
|
||||
- vllm_omni/model_executor/models/step_audio2/
|
||||
@@ -225,50 +217,3 @@ steps:
|
||||
pytest -s -v tests/e2e/offline_inference/test_wan22_t2v.py tests/e2e/online_serving/test_wan22_t2v.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model'
|
||||
"
|
||||
mirror_hardwares: h100_1
|
||||
|
||||
- label: "Diffusion · HunyuanVideo-1.5 Test"
|
||||
source_file_dependencies:
|
||||
- tests/e2e/online_serving/test_hunyuan_video_15_expansion.py
|
||||
- vllm_omni/diffusion/models/hunyuan_video/
|
||||
commands:
|
||||
- |
|
||||
timeout 40m bash -c "
|
||||
export VLLM_IMAGE_FETCH_TIMEOUT=60
|
||||
pytest -s -v tests/e2e/online_serving/test_hunyuan_video_15_expansion.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model'
|
||||
"
|
||||
mirror_hardwares: h100_1
|
||||
|
||||
- label: "Diffusion · MiniMax-H3 L3 DLO DP2 and FastH3"
|
||||
source_file_dependencies:
|
||||
- tests/e2e/online_serving/minimax_h3/
|
||||
- tests/helpers/assertions.py
|
||||
- tests/helpers/mark.py
|
||||
- tests/helpers/media.py
|
||||
- tests/helpers/runtime.py
|
||||
- vllm_omni/diffusion/models/minimax_h3/
|
||||
- vllm_omni/diffusion/lora/
|
||||
- vllm_omni/diffusion/offloader/
|
||||
- vllm_omni/diffusion/distributed/
|
||||
- vllm_omni/entrypoints/openai/
|
||||
# Keep four server lifecycles in one Buildkite job, but never in one
|
||||
# pytest process. DLO uses multiprocessing shared-memory queues; the
|
||||
# parent's resource_tracker can retain a stale /psm_* entry during
|
||||
# teardown and strand the next Ref2VA collective.
|
||||
# A per-file process plus a hard timeout also makes a regression fail
|
||||
# instead of leaving the merge queue stuck indefinitely.
|
||||
timeout_in_minutes: 150
|
||||
commands:
|
||||
- |
|
||||
set -eu
|
||||
export VLLM_IMAGE_FETCH_TIMEOUT=60
|
||||
if [ -f vllm_omni/diffusion/models/minimax_h3/lora.py ]; then
|
||||
export VLLM_TEST_MINIMAX_H3_TURBO_LORA=$(python3 -c 'from huggingface_hub import hf_hub_download; print(hf_hub_download(repo_id="lightx2v/Minimax-h3-Turbo", filename="minimax_h3_fl2v_turbo_4step_v1.0_768p_bf16.safetensors"))')
|
||||
fi
|
||||
if [ -f vllm_omni/diffusion/models/minimax_h3/fasth3.py ]; then
|
||||
export VLLM_TEST_MINIMAX_H3_FASTH3_LORA=$(python3 -c 'from huggingface_hub import hf_hub_download; print(hf_hub_download(repo_id="FastVideo/FastVideo-FastH3-4-step-Preview-v1-LoRA", filename="dense-datafree/adapter_model.safetensors"))')
|
||||
fi
|
||||
timeout 35m pytest -s -v tests/e2e/online_serving/minimax_h3/test_minimax_h3_dlo_dp2_fl2va.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model'
|
||||
timeout 35m pytest -s -v tests/e2e/online_serving/minimax_h3/test_minimax_h3_dlo_dp2_ref2va.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model'
|
||||
timeout 35m pytest -s -v tests/e2e/online_serving/minimax_h3/test_minimax_h3_turbo_lora.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model'
|
||||
timeout 35m pytest -s -v tests/e2e/online_serving/minimax_h3/test_minimax_h3_fasth3_lora.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model'
|
||||
mirror_hardwares: h100_2
|
||||
@@ -12,22 +12,16 @@ steps:
|
||||
if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test" || build.pull_request.labels includes "omni-test"
|
||||
steps:
|
||||
|
||||
- label: ":full_moon: Omni · Function Test with H100 · Single-GPU"
|
||||
timeout_in_minutes: 90
|
||||
- label: ":full_moon: Omni · Function Test with H100"
|
||||
timeout_in_minutes: 120
|
||||
commands:
|
||||
- pytest -sv tests/e2e --run-level "full_model" -m "full_model and H100 and omni and cards_1" --ignore=tests/e2e/accuracy --ignore=tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py
|
||||
mirror_hardwares: h100_1
|
||||
|
||||
- label: ":full_moon: Omni · Function Test with H100 · 2-GPU"
|
||||
timeout_in_minutes: 90
|
||||
commands:
|
||||
- pytest -sv tests/e2e --run-level "full_model" -m "full_model and H100 and omni and cards_2" --ignore=tests/e2e/accuracy --ignore=tests/e2e/online_serving/test_qwen3_omni_multi_replicas.py
|
||||
- pytest -sv tests/e2e --run-level "full_model" -m "full_model and H100 and omni" --ignore=tests/e2e/accuracy --ignore=tests/e2e/online_serving/test_qwen3_omni_multi_replicas.py --ignore=tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py
|
||||
mirror_hardwares: h100_2
|
||||
|
||||
- label: ":full_moon: Omni · MiniCPM-o 4.5 Duplex Test"
|
||||
timeout_in_minutes: 50
|
||||
commands:
|
||||
- pytest -s -v tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py -m "full_model and cuda and H100 and omni and cards_1" --run-level "full_model"
|
||||
- pytest -s -v tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py -m "full_model and cuda and H100 and omni" --run-level "full_model"
|
||||
mirror_hardwares: h100_1
|
||||
|
||||
- label: ":full_moon: Omni · Doc Test with L4"
|
||||
@@ -37,10 +31,10 @@ steps:
|
||||
- pytest -s -v tests/examples/ -m "full_model and omni and L4" --run-level "full_model"
|
||||
mirror_hardwares: l4_4
|
||||
|
||||
- label: ":full_moon: Omni · Doc Test with H100 · 2-GPU"
|
||||
- label: ":full_moon: Omni · Doc Test with H100"
|
||||
timeout_in_minutes: 90
|
||||
commands:
|
||||
- pytest -s -v tests/examples/ -m "full_model and omni and H100 and cards_2" --run-level "full_model"
|
||||
- pytest -s -v tests/examples/ -m "full_model and omni and H100" --run-level "full_model"
|
||||
mirror_hardwares: h100_2
|
||||
|
||||
- label: ":full_moon: Omni · Accuracy Test"
|
||||
@@ -142,29 +136,15 @@ steps:
|
||||
- pytest -s -v tests/e2e/ -m "full_model and L4 and tts" --run-level "full_model" --ignore=tests/e2e/accuracy
|
||||
mirror_hardwares: l4_1
|
||||
|
||||
- label: ":full_moon: TTS · Perf Test · Single-GPU"
|
||||
key: nightly-tts-performance-single
|
||||
- label: ":full_moon: TTS · Perf Test"
|
||||
key: nightly-tts-performance
|
||||
timeout_in_minutes: 180
|
||||
commands:
|
||||
- export BENCHMARK_DIR=tests/dfx/perf/results
|
||||
- export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
|
||||
- |
|
||||
set +e
|
||||
pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_tts.json -m "H100 and cards_1"
|
||||
EXIT=$$?
|
||||
buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
|
||||
exit $$EXIT
|
||||
mirror_hardwares: h100_1
|
||||
|
||||
- label: ":full_moon: TTS · Perf Test · 2-GPU"
|
||||
key: nightly-tts-performance-2gpu
|
||||
timeout_in_minutes: 180
|
||||
commands:
|
||||
- export BENCHMARK_DIR=tests/dfx/perf/results
|
||||
- export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
|
||||
- |
|
||||
set +e
|
||||
pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_tts.json -m "H100 and cards_2"
|
||||
pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_tts.json
|
||||
EXIT=$$?
|
||||
buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
|
||||
exit $$EXIT
|
||||
@@ -199,6 +179,7 @@ steps:
|
||||
build.pull_request.labels includes "nightly-test" ||
|
||||
build.pull_request.labels includes "diffusion-x2iat-test"
|
||||
steps:
|
||||
# Use explicit file shards so single-card cases do not reserve 2+ GPUs.
|
||||
|
||||
- label: ":full_moon: Diffusion X2I(&A&T) · Function Test with H100 · Single-GPU"
|
||||
timeout_in_minutes: 120
|
||||
@@ -206,24 +187,24 @@ steps:
|
||||
- >-
|
||||
pytest -sv
|
||||
tests/e2e/online_serving/test_qwen_image_expansion.py
|
||||
-m "full_model and diffusion and H100 and cards_1"
|
||||
-m "full_model and diffusion and H100 and not distributed_cuda"
|
||||
--run-level "full_model"
|
||||
mirror_hardwares: h100_1
|
||||
|
||||
- label: ":full_moon: Diffusion X2I(&A&T) · Function Test with H100 · 2-GPU"
|
||||
- label: ":full_moon: Diffusion X2I(&A&T) · Function Test with H100 · Multi-GPU Qwen-Image"
|
||||
timeout_in_minutes: 120
|
||||
commands:
|
||||
- >-
|
||||
pytest -sv
|
||||
tests/e2e/online_serving/test_qwen_image_expansion.py
|
||||
-m "full_model and diffusion and H100 and cards_2"
|
||||
-m "full_model and diffusion and H100 and distributed_cuda"
|
||||
--run-level "full_model"
|
||||
mirror_hardwares: h100_2
|
||||
|
||||
- label: ":full_moon: Diffusion X2I(&A&T) · Function Test with L4 · Multi-GPU"
|
||||
- label: ":full_moon: Diffusion X2I(&A&T) · Function Test with L4"
|
||||
timeout_in_minutes: 120
|
||||
commands:
|
||||
- pytest -sv tests/e2e/ -k "not test_wan and not hunyuan" -m "full_model and diffusion and L4 and not cards_1" --run-level "full_model" --ignore=tests/e2e/accuracy
|
||||
- pytest -sv tests/e2e/ -k "not test_wan and not hunyuan" -m "full_model and diffusion and L4" --run-level "full_model" --ignore=tests/e2e/accuracy
|
||||
mirror_hardwares: l4_4
|
||||
|
||||
- label: ":full_moon: Diffusion X2I(&A&T) · Doc Test"
|
||||
@@ -254,8 +235,8 @@ steps:
|
||||
- pytest -s -v tests/e2e/accuracy/test_hidream_o1_image.py -m full_model --run-level full_model
|
||||
mirror_hardwares: h100_2
|
||||
|
||||
- label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · Qwen-Image · Single-GPU"
|
||||
key: nightly-diffusion-x2iat-performance-qwen-image-single
|
||||
- label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · Qwen-Image"
|
||||
key: nightly-diffusion-x2iat-performance-qwen-image
|
||||
timeout_in_minutes: 180
|
||||
commands:
|
||||
- export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
|
||||
@@ -263,23 +244,7 @@ steps:
|
||||
- export CACHE_DIT_VERSION=1.5.0
|
||||
- |
|
||||
set +e
|
||||
pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_vllm_omni.json -m "H100 and cards_1"
|
||||
EXIT=$$?
|
||||
buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
|
||||
buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
|
||||
exit $$EXIT
|
||||
mirror_hardwares: h100_1
|
||||
|
||||
- label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · Qwen-Image · 4-GPU"
|
||||
key: nightly-diffusion-x2iat-performance-qwen-image-4gpu
|
||||
timeout_in_minutes: 180
|
||||
commands:
|
||||
- export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
|
||||
- export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN
|
||||
- export CACHE_DIT_VERSION=1.5.0
|
||||
- |
|
||||
set +e
|
||||
pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_vllm_omni.json -m "H100 and cards_4"
|
||||
pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_vllm_omni.json
|
||||
EXIT=$$?
|
||||
buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
|
||||
buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
|
||||
@@ -295,34 +260,16 @@ steps:
|
||||
build.pull_request.labels includes "diffusion-x2v-test"
|
||||
steps:
|
||||
|
||||
- label: ":full_moon: Diffusion X2V · Wan2.2 T2V Function Test · Single-GPU"
|
||||
- label: ":full_moon: Diffusion X2V · Wan2.2 T2V Function Test"
|
||||
timeout_in_minutes: 90
|
||||
commands:
|
||||
- pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py -k "t2v" -m "full_model and cuda and H100 and cards_1" --run-level "full_model"
|
||||
mirror_hardwares: h100_1
|
||||
|
||||
- label: ":full_moon: Diffusion X2V · Wan2.2 T2V Function Test · 2-GPU"
|
||||
timeout_in_minutes: 90
|
||||
commands:
|
||||
- pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py -k "t2v" -m "full_model and cuda and H100 and cards_2" --run-level "full_model"
|
||||
- pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py -k "t2v" -m "full_model and cuda" --run-level "full_model"
|
||||
mirror_hardwares: h100_2
|
||||
|
||||
- label: ":full_moon: Diffusion X2V · Wan2.2 I2V/TI2V Function Test · Single-GPU"
|
||||
- label: ":full_moon: Diffusion X2V · Wan2.2 I2V/TI2V Function Test"
|
||||
timeout_in_minutes: 90
|
||||
commands:
|
||||
- pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py -k "i2v" -m "full_model and cuda and H100 and cards_1" --run-level "full_model"
|
||||
mirror_hardwares: h100_1
|
||||
|
||||
- label: ":full_moon: Diffusion X2V · Wan2.2 I2V/TI2V Function Test · 2-GPU"
|
||||
timeout_in_minutes: 90
|
||||
commands:
|
||||
- pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py -k "i2v" -m "full_model and cuda and H100 and cards_2" --run-level "full_model"
|
||||
mirror_hardwares: h100_2
|
||||
|
||||
- label: ":full_moon: Diffusion X2V · HunyuanVideo-1.5 Function Test"
|
||||
timeout_in_minutes: 90
|
||||
commands:
|
||||
- pytest -s -v tests/e2e/online_serving/test_hunyuan_video_15_expansion.py -m "full_model and cuda" --run-level "full_model"
|
||||
- pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py -k "i2v" -m "full_model and cuda" --run-level "full_model"
|
||||
mirror_hardwares: h100_2
|
||||
|
||||
- label: ":full_moon: Diffusion X2V · Doc Test"
|
||||
@@ -357,49 +304,17 @@ steps:
|
||||
- pytest -s -v tests/e2e/accuracy/hunyuanvideo15_i2v/test_hunyuanvideo15_i2v_video_similarity.py -m full_model --run-level full_model
|
||||
mirror_hardwares: h100_2
|
||||
|
||||
- label: ":full_moon: Diffusion X2V · Perf Test · Single-GPU"
|
||||
key: nightly-diffusion-x2v-performance-single
|
||||
- label: ":full_moon: Diffusion X2V · Perf Test"
|
||||
key: nightly-diffusion-x2v-performance
|
||||
timeout_in_minutes: 180
|
||||
commands:
|
||||
- export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
|
||||
- export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN
|
||||
- |
|
||||
set +e
|
||||
pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json -m "H100 and cards_1"
|
||||
EXIT=$$?
|
||||
buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
|
||||
buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
|
||||
exit $$EXIT
|
||||
mirror_hardwares: h100_1
|
||||
|
||||
- label: ":full_moon: Diffusion X2V · Perf Test · 2-GPU"
|
||||
key: nightly-diffusion-x2v-performance-2gpu
|
||||
timeout_in_minutes: 180
|
||||
commands:
|
||||
- export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
|
||||
- export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN
|
||||
- |
|
||||
set +e
|
||||
pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json -m "H100 and cards_2"
|
||||
pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json
|
||||
EXIT1=$$?
|
||||
pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_cosmos3_vllm_omni.json -m "H100 and cards_2"
|
||||
EXIT2=$$?
|
||||
buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
|
||||
buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
|
||||
if [ $$EXIT1 -ne 0 ]; then exit $$EXIT1; fi
|
||||
exit $$EXIT2
|
||||
mirror_hardwares: h100_2
|
||||
|
||||
- label: ":full_moon: Diffusion X2V · Perf Test · HunyuanVideo-1.5"
|
||||
key: nightly-diffusion-x2v-performance-hunyuanvideo15
|
||||
timeout_in_minutes: 180
|
||||
commands:
|
||||
- export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
|
||||
- |
|
||||
set +e
|
||||
pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_hunyuanvideo15_t2v_vllm_omni.json
|
||||
EXIT1=$$?
|
||||
pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_hunyuanvideo15_i2v_vllm_omni.json
|
||||
pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_cosmos3_vllm_omni.json
|
||||
EXIT2=$$?
|
||||
buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
|
||||
buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
|
||||
@@ -433,16 +348,10 @@ steps:
|
||||
- pytest -sv tests/diffusion/distributed/ -m 'full_model and cuda and L4' --run-level "full_model"
|
||||
mirror_hardwares: l4_4
|
||||
|
||||
- label: ":full_moon: Diffusion · Distributed Test with H100 · 2-GPU"
|
||||
- label: ":full_moon: Diffusion · Distributed Test with H100"
|
||||
timeout_in_minutes: 60
|
||||
commands:
|
||||
- pytest -sv tests/diffusion/distributed/ -m 'full_model and cuda and H100 and cards_2' --run-level "full_model"
|
||||
mirror_hardwares: h100_2
|
||||
|
||||
- label: ":full_moon: Diffusion · Distributed Test with H100 · 4-GPU"
|
||||
timeout_in_minutes: 60
|
||||
commands:
|
||||
- pytest -sv tests/diffusion/distributed/ -m 'full_model and cuda and H100 and cards_4' --run-level "full_model"
|
||||
- pytest -sv tests/diffusion/distributed/ -m 'full_model and cuda and H100' --run-level "full_model"
|
||||
mirror_hardwares: h100_4
|
||||
|
||||
- label: ":full_moon: Diffusion · Offloader Test"
|
||||
@@ -477,25 +386,19 @@ steps:
|
||||
depends_on:
|
||||
- nightly-omni-performance-no-async-chunk
|
||||
- nightly-omni-performance-async-chunk
|
||||
- nightly-tts-performance-single
|
||||
- nightly-tts-performance-2gpu
|
||||
- nightly-diffusion-x2iat-performance-qwen-image-single
|
||||
- nightly-diffusion-x2iat-performance-qwen-image-4gpu
|
||||
- nightly-diffusion-x2v-performance-single
|
||||
- nightly-diffusion-x2v-performance-2gpu
|
||||
- nightly-tts-performance
|
||||
- nightly-diffusion-x2iat-performance-qwen-image
|
||||
- nightly-diffusion-x2v-performance
|
||||
if: build.env("NIGHTLY") == "1" && build.env("EMAIL_DISTRIBUTION") == "1"
|
||||
commands:
|
||||
- pip install openpyxl
|
||||
- export DEFAULT_INPUT_DIR=tests/dfx/perf/results
|
||||
- export DEFAULT_OUTPUT_DIR=tests/dfx/perf/results
|
||||
- buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-tts-performance-single
|
||||
- buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-tts-performance-2gpu
|
||||
- buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-tts-performance
|
||||
- buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-omni-performance-no-async-chunk
|
||||
- buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-omni-performance-async-chunk
|
||||
- buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-qwen-image-single
|
||||
- buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-qwen-image-4gpu
|
||||
- buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2v-performance-single
|
||||
- buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2v-performance-2gpu
|
||||
- buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-qwen-image
|
||||
- buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2v-performance
|
||||
- python tools/nightly/generate_nightly_perf_excel.py
|
||||
- python tools/nightly/generate_nightly_perf_html.py
|
||||
- python tools/nightly/send_nightly_email.py --report-file "tests/dfx/perf/results/*.xlsx, tests/dfx/perf/results/*.html"
|
||||
|
||||
@@ -205,10 +205,10 @@ steps:
|
||||
- tests/e2e/online_serving/test_minicpmo_realtime_duplex_drivers.py
|
||||
- tests/e2e/online_serving/helpers/minicpmo_realtime_duplex_scenarios.py
|
||||
- tests/e2e/online_serving/run_minicpmo_realtime_duplex_multi_session.py
|
||||
- tests/e2e/online_serving/run_minicpmo_realtime_duplex_server_vad.py
|
||||
- tests/e2e/online_serving/run_minicpmo_realtime_duplex_soft_interrupt.py
|
||||
- vllm_omni/deploy/minicpmo_4_5.yaml
|
||||
- tests/e2e/online_serving/helpers/minicpmo_4_5_duplex.py
|
||||
- vllm_omni/deploy/minicpmo_4_5_duplex.yaml
|
||||
- vllm_omni/experimental/fullduplex/
|
||||
- vllm_omni/model_executor/models/minicpmo_4_5/
|
||||
- vllm_omni/model_executor/models/step_audio2/
|
||||
@@ -252,9 +252,6 @@ steps:
|
||||
- vllm_omni/model_executor/models/common/alias_free_activation.py
|
||||
- vllm_omni/model_executor/stage_input_processors/qwen3_tts.py
|
||||
- vllm_omni/deploy/qwen3_tts.yaml
|
||||
- vllm_omni/entrypoints/openai/serving_speech.py
|
||||
- vllm_omni/entrypoints/openai/tts_adapters/
|
||||
- vllm_omni/engine/stage_pool.py
|
||||
commands:
|
||||
- |
|
||||
timeout 20m bash -c "
|
||||
@@ -263,6 +260,25 @@ steps:
|
||||
"
|
||||
mirror_hardwares: l4_1
|
||||
|
||||
- label: "TTS · Qwen3-TTS Base Test"
|
||||
source_file_dependencies:
|
||||
- tests/e2e/online_serving/test_qwen3_tts_base.py
|
||||
- vllm_omni/model_executor/models/qwen3_tts/
|
||||
- vllm_omni/model_executor/models/common/qwen3_code_predictor.py
|
||||
- vllm_omni/model_executor/models/common/snake_activation.py
|
||||
- vllm_omni/model_executor/models/common/alias_free_activation.py
|
||||
- vllm_omni/model_executor/stage_input_processors/qwen3_tts.py
|
||||
- vllm_omni/deploy/qwen3_tts.yaml
|
||||
commands:
|
||||
- |
|
||||
timeout 20m bash -c "
|
||||
export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
export VLLM_ALLOW_LONG_MAX_MODEL_LEN=1
|
||||
export VLLM_OMNI_USE_V2_RUNNER=1
|
||||
pytest -s -v tests/e2e/online_serving/test_qwen3_tts_base.py -m 'core_model' --run-level 'core_model'
|
||||
"
|
||||
mirror_hardwares: l4_1
|
||||
|
||||
- label: "Diffusion · Wan22 Test"
|
||||
source_file_dependencies:
|
||||
- tests/e2e/online_serving/test_wan22_t2v.py
|
||||
@@ -291,24 +307,6 @@ steps:
|
||||
"
|
||||
mirror_hardwares: h100_1
|
||||
|
||||
- label: "Diffusion · MiniMax-H3 DLO DP2 T2VA Test"
|
||||
timeout_in_minutes: 90
|
||||
source_file_dependencies:
|
||||
- tests/e2e/online_serving/test_minimax_h3_dlo_dp2_t2va.py
|
||||
- tests/helpers/assertions.py
|
||||
- tests/helpers/runtime.py
|
||||
- vllm_omni/diffusion/models/minimax_h3/
|
||||
- vllm_omni/diffusion/offloader/
|
||||
- vllm_omni/diffusion/distributed/
|
||||
- vllm_omni/entrypoints/openai/
|
||||
commands:
|
||||
- |
|
||||
timeout 80m bash -c "
|
||||
export VLLM_IMAGE_FETCH_TIMEOUT=60
|
||||
pytest -s -v tests/e2e/online_serving/test_minimax_h3_dlo_dp2_t2va.py -m 'core_model and cuda and diffusion' --run-level 'core_model'
|
||||
"
|
||||
mirror_hardwares: h100_2
|
||||
|
||||
- label: "Diffusion · Qwen Image Test"
|
||||
source_file_dependencies:
|
||||
- tests/e2e/online_serving/test_qwen_image.py
|
||||
@@ -322,15 +320,3 @@ steps:
|
||||
pytest -s -v tests/e2e/online_serving/test_qwen_image.py -m 'core_model' --run-level 'core_model'
|
||||
"
|
||||
mirror_hardwares: h100_1
|
||||
|
||||
- label: "Diffusion · HunyuanVideo-1.5 Test"
|
||||
source_file_dependencies:
|
||||
- tests/e2e/online_serving/test_hunyuan_video_15_expansion.py
|
||||
- vllm_omni/diffusion/models/hunyuan_video/
|
||||
commands:
|
||||
- |
|
||||
timeout 40m bash -c "
|
||||
export VLLM_IMAGE_FETCH_TIMEOUT=60
|
||||
pytest -s -v tests/e2e/online_serving/test_hunyuan_video_15_expansion.py -m 'core_model and diffusion' --run-level 'core_model'
|
||||
"
|
||||
mirror_hardwares: h100_1
|
||||
@@ -44,16 +44,10 @@ steps:
|
||||
- pytest -s -v tests/dfx/reliability/test_reliability_hunyuan_image.py -m "slow"
|
||||
mirror_hardwares: h100_4
|
||||
|
||||
- label: "Reliability Test · Invalid parameters · H100 · Single-GPU"
|
||||
timeout_in_minutes: 90
|
||||
- label: "Reliability Test · Invalid parameters · H100"
|
||||
timeout_in_minutes: 120
|
||||
commands:
|
||||
- pytest -s -v tests/dfx/reliability/invalid_param_test/ -m "slow and H100 and cards_1"
|
||||
mirror_hardwares: h100_1
|
||||
|
||||
- label: "Reliability Test · Invalid parameters · H100 · 2-GPU"
|
||||
timeout_in_minutes: 90
|
||||
commands:
|
||||
- pytest -s -v tests/dfx/reliability/invalid_param_test/ -m "slow and H100 and cards_2"
|
||||
- pytest -s -v tests/dfx/reliability/invalid_param_test/ -m "slow and H100"
|
||||
mirror_hardwares: h100_2
|
||||
|
||||
- label: "Reliability Test · Invalid parameters · L4"
|
||||
@@ -111,37 +105,27 @@ steps:
|
||||
if: build.env("NON_CRITICAL") == "1"
|
||||
steps:
|
||||
|
||||
# Qwen2.5 expansion (CUDA 4-card / online 2-card) stays on this l4_4 job.
|
||||
- label: "Omni · L4 · Multi-GPU"
|
||||
- label: "Omni · L4"
|
||||
timeout_in_minutes: 90
|
||||
commands:
|
||||
- export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
|
||||
- pytest -s -v tests/e2e/*/test_qwen2_5_omni_expansion.py -m "slow and L4 and omni and not cards_1" --run-level "core_model"
|
||||
- pytest -s -v tests/e2e/*/test_qwen2_5_omni_expansion.py --run-level "core_model"
|
||||
- >-
|
||||
pytest -s -v tests/e2e/
|
||||
--ignore=tests/e2e/online_serving/test_qwen2_5_omni_expansion.py
|
||||
--ignore=tests/e2e/offline_inference/test_qwen2_5_omni_expansion.py
|
||||
-m "slow and L4 and omni and not cards_1"
|
||||
-m "slow and L4 and omni"
|
||||
--run-level "full_model"
|
||||
mirror_hardwares: l4_4
|
||||
|
||||
- label: "Omni · H100 · Single-GPU"
|
||||
- label: "Omni · H100"
|
||||
timeout_in_minutes: 90
|
||||
commands:
|
||||
- >-
|
||||
pytest -sv
|
||||
tests/e2e/ -m "slow and H100 and omni and cards_1"
|
||||
tests/e2e/ -m "slow and H100 and omni"
|
||||
--run-level "full_model"
|
||||
mirror_hardwares: h100_1
|
||||
|
||||
- label: "Omni · H100 · 4-GPU"
|
||||
timeout_in_minutes: 90
|
||||
commands:
|
||||
- >-
|
||||
pytest -sv
|
||||
tests/e2e/ -m "slow and H100 and omni and cards_4"
|
||||
--run-level "full_model"
|
||||
mirror_hardwares: h100_4
|
||||
mirror_hardwares: h100_2
|
||||
|
||||
- label: "TTS · L4"
|
||||
timeout_in_minutes: 120
|
||||
@@ -185,21 +169,12 @@ steps:
|
||||
--run-level "full_model"
|
||||
mirror_hardwares: l4_1
|
||||
|
||||
- label: "Diffusion · L4 · Single-GPU"
|
||||
- label: "Diffusion · L4"
|
||||
timeout_in_minutes: 60
|
||||
commands:
|
||||
- >-
|
||||
pytest -sv
|
||||
tests/e2e/ -m "slow and diffusion and L4 and cards_1"
|
||||
--run-level "full_model"
|
||||
mirror_hardwares: l4_1
|
||||
|
||||
- label: "Diffusion · L4 · Multi-GPU"
|
||||
timeout_in_minutes: 60
|
||||
commands:
|
||||
- >-
|
||||
pytest -sv
|
||||
tests/e2e/ -m "slow and diffusion and L4 and not cards_1"
|
||||
tests/e2e/ -m "slow and diffusion and L4"
|
||||
--run-level "full_model"
|
||||
mirror_hardwares: l4_4
|
||||
|
||||
@@ -208,24 +183,15 @@ steps:
|
||||
commands:
|
||||
- >-
|
||||
pytest -sv
|
||||
tests/e2e/ -m "slow and diffusion and H100 and cards_1"
|
||||
tests/e2e/ -m "slow and diffusion and H100 and not distributed_cuda"
|
||||
--run-level "full_model"
|
||||
mirror_hardwares: h100_1
|
||||
|
||||
- label: "Diffusion · H100 · 2-GPU"
|
||||
- label: "Diffusion · H100 · Multi-GPU"
|
||||
timeout_in_minutes: 90
|
||||
commands:
|
||||
- >-
|
||||
pytest -sv
|
||||
tests/e2e/ -m "slow and diffusion and H100 and cards_2"
|
||||
--run-level "full_model"
|
||||
mirror_hardwares: h100_2
|
||||
|
||||
- label: "Diffusion · H100 · 4-GPU"
|
||||
timeout_in_minutes: 90
|
||||
commands:
|
||||
- >-
|
||||
pytest -sv
|
||||
tests/e2e/ -m "slow and diffusion and H100 and cards_4"
|
||||
tests/e2e/ -m "slow and diffusion and H100 and distributed_cuda"
|
||||
--run-level "full_model"
|
||||
mirror_hardwares: h100_4
|
||||
@@ -10,7 +10,7 @@ steps:
|
||||
DOCKER_BUILDKIT: "1"
|
||||
# Buildkite will automatically replace this with the actual commit hash
|
||||
VLLM_IMAGE_TAG: "${BUILDKITE_COMMIT}"
|
||||
VLLM_VERSION: "v0.28.0"
|
||||
VLLM_VERSION: "v0.27.0"
|
||||
priority: 100
|
||||
timeout_in_minutes: 60
|
||||
soft_fail: false
|
||||
@@ -5,12 +5,7 @@ set -ex
|
||||
|
||||
omni_source_dir=$(git rev-parse --show-toplevel)
|
||||
|
||||
: "${VLLM_VERSION:?VLLM_VERSION must be set}"
|
||||
# Official upstream XPU image. It is published from vllm's own
|
||||
# docker/Dockerfile.xpu, which is exactly what build_vllm_base() clones and
|
||||
# builds below, so pulling it is equivalent to building the fallback base.
|
||||
upstream_base_image="vllm/vllm-openai-xpu:${VLLM_VERSION}"
|
||||
local_base_image="xpu/vllm-omni-ci-base:${VLLM_VERSION}"
|
||||
base_image_name="xpu/vllm-omni-ci-base:${VLLM_VERSION:?VLLM_VERSION must be set}"
|
||||
image_name="xpu/vllm-omni-ci:${BUILDKITE_COMMIT:?BUILDKITE_COMMIT must be set}"
|
||||
container_name="xpu_${BUILDKITE_COMMIT}_$(
|
||||
tr -dc A-Za-z0-9 </dev/urandom | head -c 10
|
||||
@@ -54,36 +49,9 @@ build_vllm_base() (
|
||||
"${vllm_source_dir}"
|
||||
)
|
||||
|
||||
# Resolve the vLLM base image, in priority order:
|
||||
# 1. the published upstream image for VLLM_VERSION, pulled fresh (so moving
|
||||
# tags like nightly are refreshed rather than served stale from disk);
|
||||
# 2. a local copy of that same image, when the registry is unreachable;
|
||||
# 3. VLLM_BASE, if that image is already on disk;
|
||||
# 4. otherwise build VLLM_BASE from the matching vLLM source tag (~30min).
|
||||
# VLLM_BASE names the fallback base and defaults to ${local_base_image}; set it
|
||||
# to reuse or produce a base under a different tag.
|
||||
resolve_vllm_base() {
|
||||
if docker pull "${upstream_base_image}"; then
|
||||
base_image_name="${upstream_base_image}"
|
||||
return
|
||||
fi
|
||||
|
||||
# Registry unreachable, but a previous run may have left the image behind.
|
||||
if [ -n "$(docker images -q "${upstream_base_image}")" ]; then
|
||||
echo "WARNING: could not pull ${upstream_base_image}; using local copy" >&2
|
||||
base_image_name="${upstream_base_image}"
|
||||
return
|
||||
fi
|
||||
|
||||
echo "WARNING: ${upstream_base_image} unavailable remotely and locally" >&2
|
||||
base_image_name="${VLLM_BASE:-${local_base_image}}"
|
||||
if [ -z "$(docker images -q "${base_image_name}")" ]; then
|
||||
echo "WARNING: building ${base_image_name} from vLLM ${VLLM_VERSION} source" >&2
|
||||
build_vllm_base
|
||||
fi
|
||||
}
|
||||
|
||||
resolve_vllm_base
|
||||
if [ -z "$(docker images -q "${base_image_name}")" ]; then
|
||||
build_vllm_base
|
||||
fi
|
||||
|
||||
# Try building the docker image
|
||||
docker_build "${image_name}" --build-arg "VLLM_BASE=${base_image_name}" --build-arg "VLLM_VERSION=${VLLM_VERSION}"
|
||||
|
||||
@@ -10,18 +10,6 @@ steps:
|
||||
-m 'core_model and npu and A3 and omni'
|
||||
--run-level 'core_model'
|
||||
|
||||
- label: "Diffusion NPU layer tests"
|
||||
key: ready-diffusion-npu-layer-tests
|
||||
depends_on: upload-ready-pipeline
|
||||
mirror_hardwares: a3_npu_2
|
||||
commands:
|
||||
- >-
|
||||
pytest -s -v
|
||||
tests/diffusion/layers/test_activation.py
|
||||
tests/diffusion/layers/test_fused_qk_norm_rope_npu.py
|
||||
-m 'core_model and npu and A3'
|
||||
--run-level 'core_model'
|
||||
|
||||
- group: ":card_index_dividers: Omni Model Test"
|
||||
key: ready-omni-group
|
||||
depends_on: upload-ready-pipeline
|
||||
|
||||
@@ -105,7 +105,7 @@ For custom loop structures, follow `vllm_omni/diffusion/models/progress_bar.py`
|
||||
|
||||
## Path B: Custom/Private Repo Model
|
||||
|
||||
For models without a diffusers pipeline — weights in custom format, model code in a private repo. Real examples: BAGEL, HunyuanImage3. See [references/custom-model-patterns.md](references/custom-model-patterns.md) for concrete custom-model integration patterns.
|
||||
For models without a diffusers pipeline — weights in custom format, model code in a private repo. Real examples: DreamID-Omni, BAGEL, HunyuanImage3. See [references/custom-model-patterns.md](references/custom-model-patterns.md) for concrete custom-model integration patterns.
|
||||
|
||||
### B1. Understand the reference repo
|
||||
|
||||
@@ -126,9 +126,9 @@ This is the key design decision for custom models. Follow these placement rules:
|
||||
| **Pipeline orchestration** (init, forward, denoise loop) | `vllm_omni/diffusion/models/<name>/pipeline_<name>.py` | Always required |
|
||||
| **Custom transformer/backbone** (ported and adapted to vllm-omni) | `vllm_omni/diffusion/models/<name>/<name>_transformer.py` or similar | `wan2_2.py`, `fusion.py`, `bagel_transformer.py` |
|
||||
| **Custom sub-models** (VAE, fusion, autoencoder) | `vllm_omni/diffusion/models/<name>/` as separate files | `autoencoder.py`, `fusion.py` |
|
||||
| **External dependency code** (original repo utilities) | **External repo**, installed via download script or pip | original-repo package via git clone |
|
||||
| **External dependency code** (original repo utilities) | **External repo**, installed via download script or pip | `dreamid_omni` package via git clone |
|
||||
| **Hardcoded model configs** | Module-level dicts in pipeline file | `VIDEO_CONFIG`, `AUDIO_CONFIG` dicts |
|
||||
| **Download/setup script** | `examples/offline_inference/<name>/download_<name>.py` | `download_<name>.py` |
|
||||
| **Download/setup script** | `examples/offline_inference/<name>/download_<name>.py` | `download_dreamid_omni.py` |
|
||||
| **Custom `model_index.json`** | Generated by download script, placed at model root | Minimal: `{"_class_name": "YourPipeline", ...}` |
|
||||
|
||||
### B3. Handle external dependencies
|
||||
@@ -157,7 +157,7 @@ Copy the essential model files into `vllm_omni/diffusion/models/<name>/` and ada
|
||||
|
||||
Custom models have two common patterns for weight loading:
|
||||
|
||||
**Pattern 1: Bypass standard loader** (eager custom init)
|
||||
**Pattern 1: Bypass standard loader** (DreamID-Omni style)
|
||||
|
||||
When the original model has complex custom init functions that load weights in `__init__`:
|
||||
|
||||
@@ -338,7 +338,7 @@ Classify the model's **CI priority** first:
|
||||
|
||||
**L2 & L3 online — same file, dual marks on the baseline smoke:** The **first / simplest** case in `test_{slug}.py` (default deploy, minimal steps, single prompt) should carry **both** `@pytest.mark.core_model` and `@pytest.mark.advanced_model` on the **same** function so L2 (`test-ready.yml`) and L3 (`test-merge.yml`) share one smoke test. Heavier deploy variants or API paths in the same file use **`advanced_model` only**. When L3 moves to nightly, migrate those heavier cases into `test_{slug}_expansion.py` with `full_model` and remove the dedicated `test-merge.yml` job (see `test_longcat_image_expansion.py`, `test_qwen_image_expansion.py`).
|
||||
|
||||
**L4 design (high priority):** Combine multiple supported features (Cache-DiT, TP, USP, CFG, HSDP, CPU offload, quantization) into **few parametrized** `OmniServerParams` rows so each feature appears in at least one case without exploding GPU jobs. Shard single-GPU vs multi-GPU cases across the nightly X2I/X2V function steps (`cards_1` vs `not cards_1`).
|
||||
**L4 design (high priority):** Combine multiple supported features (Cache-DiT, TP, USP, CFG, HSDP, CPU offload, quantization) into **few parametrized** `OmniServerParams` rows so each feature appears in at least one case without exploding GPU jobs. Shard single-GPU vs multi-GPU cases across the nightly X2I/X2V function steps (1-GPU vs `distributed_cuda`).
|
||||
|
||||
**L4 design (medium / low):** One or two parametrized rows covering the best quality/perf trade-off; skip perf JSON unless the model is high priority.
|
||||
|
||||
|
||||
@@ -16,19 +16,20 @@ vllm_omni/diffusion/models/wan2_2/
|
||||
|
||||
The transformer is loaded separately via `weights_sources` + `load_weights()`. Non-transformer components (VAE, text encoder) are loaded in `__init__` via `from_pretrained()`.
|
||||
|
||||
### Custom model with external deps
|
||||
### Custom model with external deps (e.g., DreamID-Omni)
|
||||
|
||||
```
|
||||
vllm_omni/diffusion/models/<name>/
|
||||
vllm_omni/diffusion/models/dreamid_omni/
|
||||
├── __init__.py # Exports pipeline only
|
||||
├── pipeline_<name>.py # Pipeline: loads ALL weights in __init__ via custom helpers
|
||||
└── <backbone>.py # Custom transformer / fusion architecture
|
||||
├── pipeline_dreamid_omni.py # Pipeline: loads ALL weights in __init__ via custom helpers
|
||||
├── fusion.py # Custom fusion architecture (video + audio cross-attention)
|
||||
└── wan2_2.py # Re-implemented Wan backbone with split API
|
||||
|
||||
examples/offline_inference/<task>/
|
||||
└── download_<name>.py # Downloads weights and/or clones code repo
|
||||
examples/offline_inference/x_to_video_audio/
|
||||
└── download_dreamid_omni.py # Downloads weights from 3 HF repos + clones code repo
|
||||
```
|
||||
|
||||
All weights loaded eagerly in `__init__`. `load_weights()` is a no-op. External dependency imported with try/except.
|
||||
All weights loaded eagerly in `__init__`. `load_weights()` is a no-op. External dependency (`dreamid_omni` package) imported with try/except.
|
||||
|
||||
### Custom model with ported code (e.g., BAGEL)
|
||||
|
||||
@@ -75,7 +76,7 @@ self.weights_sources = [
|
||||
]
|
||||
```
|
||||
|
||||
### Pattern 3: Fully custom loading
|
||||
### Pattern 3: Fully custom loading (DreamID-Omni)
|
||||
|
||||
```
|
||||
init → load ALL weights eagerly via custom helpers → load_weights() = no-op
|
||||
@@ -107,8 +108,8 @@ Standard diffusers `model_index.json`:
|
||||
Custom model `model_index.json` (minimal):
|
||||
```json
|
||||
{
|
||||
"_class_name": "YourCustomPipeline",
|
||||
"fusion": "weights/model.safetensors"
|
||||
"_class_name": "DreamIDOmniPipeline",
|
||||
"fusion": "DreamID-Omni/dreamid_omni.safetensors"
|
||||
}
|
||||
```
|
||||
|
||||
@@ -116,7 +117,7 @@ The only **required** field is `_class_name` — it must match a key in `_DIFFUS
|
||||
|
||||
## External Dependency Management
|
||||
|
||||
### Git clone + .pth injection
|
||||
### Git clone + .pth injection (DreamID-Omni pattern)
|
||||
|
||||
```python
|
||||
def download_dependency():
|
||||
@@ -162,7 +163,7 @@ The engine checks `isinstance(pipeline, SupportImageInput)` at startup to config
|
||||
|
||||
Diffusers models use `config.json` in each subfolder. Custom models often use:
|
||||
|
||||
**Module-level config dicts**:
|
||||
**Module-level config dicts** (DreamID-Omni):
|
||||
```python
|
||||
VIDEO_CONFIG = {
|
||||
"patch_size": [1, 2, 2], "model_type": "ti2v",
|
||||
@@ -180,7 +181,7 @@ vae_cfg = bagel_cfg.get("vae_config", {})
|
||||
|
||||
## Custom Architecture Patterns
|
||||
|
||||
### Split forward API
|
||||
### Split forward API (DreamID-Omni)
|
||||
|
||||
When a fusion model needs to interleave blocks from two backbones:
|
||||
|
||||
|
||||
@@ -34,7 +34,7 @@ but the changes are correct — stage the modified files and re-commit.
|
||||
| `check-forbidden-imports` | Stdlib `re`/`base64`, pickle, Hugging Face Hub API, or direct Triton/TileLang | `import regex as re` and `pybase64`; use `vllm.transformers_utils.repo_utils`. Do **not** add the file to `allowed_files` without review |
|
||||
| `check-torch-cuda-call` | New `torch.cuda.*` call site | Use `current_omni_platform` / `OmniPlatform`; do not grow `ALLOWED_FILES` without review |
|
||||
| `check-tts-adapter-migration` | New `self._tts_model_type` branch in `serving_speech.py` | Put per-model logic in `tts_adapters/`. Lower `MAX_MODEL_TYPE_BRANCHES` when removing branches; do not raise it without review |
|
||||
| `check-mark` | New `tests/**/test_*.py` missing level/hardware mark, or direct `pytest.mark.H100` | Add `core_model`/`advanced_model`/… plus `cpu`/`cuda`/`hardware_test(`; SKU marks only via helpers |
|
||||
| `check-test-ci-coverage` | New `tests/**/test_*.py` missing level or hardware mark | Add `core_model`/`advanced_model`/… plus `cpu`/`cuda`/`hardware_test(` |
|
||||
| `mypy-3.10` | Type error on changed `vllm_omni/` files | Fix the types; model trees are excluded. Extra versions: `pre-commit run --hook-stage manual mypy-3.12` |
|
||||
| `markdownlint-cli2` | Docs/README markdown lint | Accept auto-fix under `docs/` / `recipes/` / root README |
|
||||
| `check-buildkite` | Invalid `.buildkite/*.yml` | Fix the YAML; do not grow `SKIP_FILES` without review |
|
||||
|
||||
@@ -52,8 +52,8 @@ Always attach markers deliberately:
|
||||
- `tts` — speech synthesis / TTS-only models (`/v1/audio/speech`, voice clone, etc.)
|
||||
- `diffusion` — generative diffusion models (image / audio / text / video from noise)
|
||||
- **Cross-cutting area** (when relevant): `parallel`, `cache`, `example`, `benchmark`
|
||||
- **Hardware**: `cpu`, `gpu`, `cuda`, `rocm`, `npu`, …; SKU (`H100`, `L4`, …) and `cards_{n}` via `@hardware_test(...)` / `hardware_marks(...)` only — never `pytest.mark.H100` (`check-mark` rejects it)
|
||||
- Optional: `slow`. Multi-card: `num_cards=N` on the helper (filter with `cards_N` / `not cards_1`)
|
||||
- **Hardware**: `cpu`, `gpu`, `cuda`, `rocm`, `npu`, `L4`, `H100`, `distributed_cuda`, …
|
||||
- Optional: `slow`, distributed markers when multi-card is required
|
||||
|
||||
**Baseline smoke (L2 + L3):** The simplest e2e case per model — default deploy, minimal request — should usually carry **both** `@pytest.mark.core_model` and `@pytest.mark.advanced_model` on the **same** test function so `test-ready.yml` and `test-merge.yml` share one test. `send_*_request` picks validation depth from `--run-level`. References: `test_voxcpm2_tts.py::test_text_to_audio_001`, `test_qwen3_tts_customvoice.py::test_text_to_audio_001`. Heavier scenarios use **`advanced_model` only**; L4 expansion uses **`full_model`**.
|
||||
|
||||
@@ -274,7 +274,7 @@ def test_text_to_video_001(omni_server, online_client) -> None:
|
||||
|
||||
**3. Cross-cutting rules**
|
||||
|
||||
- New `.py` files need the Omni SPDX header (`Copyright contributors to the vLLM-Omni project`, not `vLLM project`). `tests/` may keep stdlib `re`/`base64`; pickle is still banned unless the file is already on the pickle allowlist. Every collected `tests/**/test_*.py` needs a CI **level** mark and a **hardware** platform mark/helper (`check-mark` runs locally; GHA skips it). Do not write `pytest.mark.H100` / `pytest.mark.L4`; use the helpers so `cards_{n}` is attached. Do not grow `allowed_files` to land a test. Policy: [docs/contributing/README.md](../../../docs/contributing/README.md#linting).
|
||||
- New `.py` files need the Omni SPDX header (`Copyright contributors to the vLLM-Omni project`, not `vLLM project`). `tests/` may keep stdlib `re`/`base64`; pickle is still banned unless the file is already on the pickle allowlist. Every collected `tests/**/test_*.py` needs a CI **level** mark and a **hardware** mark/helper (`check-test-ci-coverage` runs locally; GHA skips it). Do not grow `allowed_files` to land a test. Policy: [docs/contributing/README.md](../../../docs/contributing/README.md#linting).
|
||||
- Reuse existing fixtures for the chosen scenario; do not mix “online client” assumptions into offline `OmniRunner` tests without a clear reason.
|
||||
- Avoid external network dependency in assertions unless the scenario is explicitly “online serving” or doc examples that require a model hub (then align with CI secrets/cache).
|
||||
- Keep **one test function = one intent** (one modality combo, one endpoint contract, or one acceleration combo).
|
||||
|
||||
@@ -79,7 +79,7 @@ E2E (L2+) should not rely on mocks unless documenting a rare exception; prefer r
|
||||
- `full_model` — L4 nightly (`test-nightly.yml`); some expansion tests still carry both `advanced_model` and `full_model` during migration
|
||||
2. **Model type** (pick one): `omni`, `tts`, or `diffusion`
|
||||
3. **Cross-cutting** when relevant: `parallel`, `cache`, `example`, `benchmark`
|
||||
4. **Hardware**: `cpu`, `cuda`, `rocm`, `npu`, `L4`, `H100`, `cards_1`, …
|
||||
4. **Hardware**: `cpu`, `cuda`, `rocm`, `npu`, `L4`, `H100`, `distributed_cuda`, …
|
||||
5. Multi-card: `@hardware_test(...)` in `tests/helpers/mark.py`
|
||||
|
||||
## Command Templates
|
||||
@@ -244,7 +244,7 @@ pytest -s -v dfx/reliability/invalid_param_test/test_invalid_image_generation.py
|
||||
|
||||
```bash
|
||||
cd tests
|
||||
pytest -s -v -m "core_model and L4 and not cards_1" --run-level=core_model
|
||||
pytest -s -v -m "core_model and distributed_cuda and L4" --run-level=core_model
|
||||
```
|
||||
|
||||
### Concrete e2e paths (common in-tree)
|
||||
|
||||
@@ -27,4 +27,4 @@ jobs:
|
||||
# Don't run historical-debt hooks against the whole tree in CI yet.
|
||||
# They still run locally on changed files.
|
||||
env:
|
||||
SKIP: check-mark,markdownlint-cli2,shellcheck,check-spdx-header,mypy-3.10
|
||||
SKIP: check-test-ci-coverage,markdownlint-cli2,shellcheck,check-spdx-header,mypy-3.10
|
||||
@@ -105,8 +105,8 @@ repos:
|
||||
<<: *mypy_common
|
||||
stages: [manual]
|
||||
|
||||
- id: check-mark
|
||||
name: Ensure test files have CI marks (no direct SKU)
|
||||
- id: check-test-ci-coverage
|
||||
name: Ensure test files have pytestmarks
|
||||
entry: python tools/pre_commit/check_test_marks.py
|
||||
language: python
|
||||
types: [python]
|
||||
|
||||
@@ -16,7 +16,6 @@ Easy, fast, and cheap omni-modality model serving for everyone
|
||||
|
||||
*Latest News* 🔥
|
||||
|
||||
- [2026/08] We released [0.28.0](https://github.com/vllm-project/vllm-omni/releases/tag/v0.28.0), featuring production-ready [MiniMax H3](recipes/MiniMaxAI/MiniMax-H3.md) serving on GPU and NPU, a unified AR/DiT paged KV cache runtime, and enhanced realtime full-duplex serving for the [MiniCPM-o series](recipes/OpenBMB/MiniCPM-o-4_5.md).
|
||||
- [2026/08] [VeRL-Omni](https://github.com/verl-project/verl-omni) `v0.2.0` is released: faster diffusion RL powered by vLLM-Omni (request-level/step-wise batching with FA3), rebuilt Qwen3-Omni multimodal training (DPO & GSPO), plus LTX-2.3, Qwen-Image-Edit support and more. See the [release notes](https://github.com/verl-project/verl-omni/releases/tag/v0.2.0).
|
||||
- [2026/08] We released [0.26.0](https://github.com/vllm-project/vllm-omni/releases/tag/v0.26.0) - aligned with the vLLM 0.26 release line, featuring [MiniMax H3](recipes/MiniMaxAI/MiniMax-H3.md) joint video/audio generation, an experimental full-duplex realtime runtime for [MiniCPM-o 4.5](recipes/OpenBMB/MiniCPM-o-4_5.md), distributed layerwise diffusion offload, and broader model, hardware, streaming, TTS, and quantization support.
|
||||
- [2026/07] We released [0.24.0](https://github.com/vllm-project/vllm-omni/releases/tag/v0.24.0) - aligned with the vLLM 0.24 release line, expanding production-ready coverage across TTS, speech, diffusion, image/video generation, and robot-policy serving, with major Omni stage runtime refactoring, diffusion request-level batching, async output materialization, quantization/cache/memory improvements, and broad CUDA/ROCm/XPU/NPU support.
|
||||
@@ -58,9 +57,9 @@ vLLM-Omni is flexible and easy to use with:
|
||||
vLLM-Omni seamlessly supports most popular open-source models on HuggingFace, including:
|
||||
|
||||
- **Omni-modality models** (e.g. Qwen3-Omni, MiniCPM-o 4.5, Cosmos3, HunyuanImage, BAGEL)
|
||||
- **TTS models** (e.g. Qwen3-TTS, IndexTTS 2.5, dots.tts, CosyVoice3)
|
||||
- **Diffusion models** — image, video, and audio generation (e.g. MiniMax H3, LTX-2.5, SANA-Video, Wan2.2)
|
||||
- **Robot-policy and action models** (e.g. π0, GR00T-N1.7, DreamZero-DROID, InternVLA-A1)
|
||||
- **TTS models** (e.g. Qwen3-TTS, VoxCPM2, Ming-Omni-TTS, CosyVoice3)
|
||||
- **Diffusion models** — image, video, and audio generation (e.g. MiniMax H3, Qwen-Image, Wan2.2, FLUX)
|
||||
- **Robot-policy and action models** (e.g. GR00T-N1.7, DreamZero-DROID, InternVLA-A1, Cosmos3 action policy)
|
||||
|
||||
## Getting Started
|
||||
|
||||
@@ -68,7 +67,6 @@ Visit our [documentation](https://vllm-omni.readthedocs.io/en/latest/) to learn
|
||||
|
||||
- [Installation](https://vllm-omni.readthedocs.io/en/latest/getting_started/installation/)
|
||||
- [Quickstart](https://vllm-omni.readthedocs.io/en/latest/getting_started/quickstart/)
|
||||
- [Nightly CUDA Docker images](https://hub.docker.com/r/vllm/vllm-omni/tags?name=nightly) built from the latest `main`
|
||||
- [List of Supported Models](https://vllm-omni.readthedocs.io/en/latest/models/supported_models/)
|
||||
- [Deployment Recipes](https://recipes.vllm.ai) for vLLM-Omni model serving
|
||||
|
||||
|
||||
@@ -1,151 +0,0 @@
|
||||
"""Benchmark CosyVoice3-style CUDA stream handoff strategies."""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import statistics
|
||||
import time
|
||||
|
||||
import torch
|
||||
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
|
||||
def run_case(
|
||||
*,
|
||||
device: torch.device,
|
||||
steps: int,
|
||||
warmup_steps: int,
|
||||
producer_cycles: int,
|
||||
estimator_cycles: int,
|
||||
use_stream_dependencies: bool,
|
||||
) -> dict[str, float | int | str]:
|
||||
caller_stream = torch.cuda.Stream(device=device)
|
||||
estimator_stream = torch.cuda.Stream(device=device)
|
||||
input_tensor = torch.zeros(1024, device=device)
|
||||
output_tensor = torch.empty_like(input_tensor)
|
||||
submission_ms = []
|
||||
|
||||
def step(measure: bool) -> None:
|
||||
with torch.cuda.stream(caller_stream):
|
||||
torch.cuda._sleep(producer_cycles)
|
||||
input_tensor.add_(1)
|
||||
|
||||
start = time.perf_counter_ns()
|
||||
if use_stream_dependencies:
|
||||
estimator_stream.wait_stream(caller_stream)
|
||||
else:
|
||||
caller_stream.synchronize()
|
||||
|
||||
with torch.cuda.stream(estimator_stream):
|
||||
torch.cuda._sleep(estimator_cycles)
|
||||
output_tensor.copy_(input_tensor)
|
||||
|
||||
if use_stream_dependencies:
|
||||
caller_stream.wait_stream(estimator_stream)
|
||||
else:
|
||||
estimator_stream.synchronize()
|
||||
|
||||
if measure:
|
||||
submission_ms.append((time.perf_counter_ns() - start) / 1e6)
|
||||
|
||||
for _ in range(warmup_steps):
|
||||
step(measure=False)
|
||||
caller_stream.synchronize()
|
||||
estimator_stream.synchronize()
|
||||
input_tensor.zero_()
|
||||
torch.accelerator.reset_peak_memory_stats(device)
|
||||
initial_allocated = torch.accelerator.memory_allocated(device)
|
||||
|
||||
start_event = torch.cuda.Event(enable_timing=True)
|
||||
end_event = torch.cuda.Event(enable_timing=True)
|
||||
wall_start = time.perf_counter()
|
||||
with torch.cuda.stream(caller_stream):
|
||||
start_event.record()
|
||||
for _ in range(steps):
|
||||
step(measure=True)
|
||||
with torch.cuda.stream(caller_stream):
|
||||
end_event.record()
|
||||
end_event.synchronize()
|
||||
wall_ms = (time.perf_counter() - wall_start) * 1e3
|
||||
peak_delta_bytes = torch.accelerator.max_memory_allocated(device) - initial_allocated
|
||||
|
||||
expected = torch.full_like(output_tensor, steps)
|
||||
torch.testing.assert_close(output_tensor, expected, rtol=0, atol=0)
|
||||
return {
|
||||
"mode": "wait_stream" if use_stream_dependencies else "synchronize",
|
||||
"steps": steps,
|
||||
"submission_median_ms": statistics.median(submission_ms),
|
||||
"submission_p95_ms": statistics.quantiles(submission_ms, n=20)[18],
|
||||
"submission_mean_ms": statistics.mean(submission_ms),
|
||||
"gpu_elapsed_ms": start_event.elapsed_time(end_event),
|
||||
"wall_elapsed_ms": wall_ms,
|
||||
"peak_allocation_delta_bytes": peak_delta_bytes,
|
||||
"final_value": int(output_tensor[0].item()),
|
||||
}
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--device", default="cuda:0")
|
||||
parser.add_argument("--steps", type=int, default=100)
|
||||
parser.add_argument("--warmup-steps", type=int, default=10)
|
||||
parser.add_argument("--repeats", type=int, default=7)
|
||||
parser.add_argument("--producer-cycles", type=int, default=1_000_000)
|
||||
parser.add_argument("--estimator-cycles", type=int, default=2_000_000)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
args = parse_args()
|
||||
device = torch.device(args.device)
|
||||
current_omni_platform.set_device(device)
|
||||
torch.manual_seed(0)
|
||||
|
||||
old_results = []
|
||||
new_results = []
|
||||
for repeat in range(args.repeats):
|
||||
common_args = {
|
||||
"device": device,
|
||||
"steps": args.steps,
|
||||
"warmup_steps": args.warmup_steps,
|
||||
"producer_cycles": args.producer_cycles,
|
||||
"estimator_cycles": args.estimator_cycles,
|
||||
}
|
||||
if repeat % 2 == 0:
|
||||
old_results.append(run_case(**common_args, use_stream_dependencies=False))
|
||||
new_results.append(run_case(**common_args, use_stream_dependencies=True))
|
||||
else:
|
||||
new_results.append(run_case(**common_args, use_stream_dependencies=True))
|
||||
old_results.append(run_case(**common_args, use_stream_dependencies=False))
|
||||
|
||||
old_submission_ms = statistics.median(result["submission_median_ms"] for result in old_results)
|
||||
new_submission_ms = statistics.median(result["submission_median_ms"] for result in new_results)
|
||||
old_gpu_ms = statistics.median(result["gpu_elapsed_ms"] for result in old_results)
|
||||
new_gpu_ms = statistics.median(result["gpu_elapsed_ms"] for result in new_results)
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"device": torch.cuda.get_device_name(device),
|
||||
"torch": torch.__version__,
|
||||
"cuda": torch.version.cuda,
|
||||
"repeats": args.repeats,
|
||||
"warmup_steps": args.warmup_steps,
|
||||
"producer_cycles": args.producer_cycles,
|
||||
"estimator_cycles": args.estimator_cycles,
|
||||
"old_submission_median_ms": old_submission_ms,
|
||||
"new_submission_median_ms": new_submission_ms,
|
||||
"submission_median_speedup": old_submission_ms / new_submission_ms,
|
||||
"old_gpu_elapsed_median_ms": old_gpu_ms,
|
||||
"new_gpu_elapsed_median_ms": new_gpu_ms,
|
||||
"gpu_elapsed_change_percent": (new_gpu_ms - old_gpu_ms) / old_gpu_ms * 100,
|
||||
"parity": all(result["final_value"] == args.steps for result in old_results + new_results),
|
||||
"old_runs": old_results,
|
||||
"new_runs": new_results,
|
||||
},
|
||||
indent=2,
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,4 +1,4 @@
|
||||
# Target vllm version: v0.28.0 (tag v0.28.0 @ 2cf0a6915ce5).
|
||||
# Target vllm version: v0.27.0 (releases/v0.27.0 branch).
|
||||
# Keep VLLM_BASE_TAG aligned with the target vLLM release on every rebase.
|
||||
#
|
||||
# Release-boundary form: there is no precompiled-wheel pin here, because the
|
||||
@@ -14,14 +14,11 @@
|
||||
# core init), registering libnvJitLink.so.13 with ldconfig, and re-pinning numpy
|
||||
# under numba's ceiling. All of it was compensation for a base image and a vLLM
|
||||
# that disagreed. At a real release boundary they agree, so none of it is
|
||||
# needed. The 0.28 rebase repeated the cycle: the v0.28.0rc / final-tag wheels
|
||||
# rode on the v0.27.0 base image with the full reinstall block until the
|
||||
# official v0.28.0 image shipped, at which point this file dropped back to the
|
||||
# form you are reading. If you pin an untagged commit again, restore the
|
||||
# reinstall block — agent/tasks/20_update_wheel_url.sh fails the run rather
|
||||
# than letting the Dockerfile silently ship the wrong vLLM.
|
||||
# needed. If you pin an untagged commit again, restore the reinstall block —
|
||||
# agent/tasks/20_update_wheel_url.sh fails the run rather than letting the
|
||||
# Dockerfile silently ship the wrong vLLM.
|
||||
ARG VLLM_BASE_IMAGE=vllm/vllm-openai
|
||||
ARG VLLM_BASE_TAG=v0.28.0
|
||||
ARG VLLM_BASE_TAG=v0.27.0
|
||||
|
||||
# We use registry-based layer caching for the dependencies,
|
||||
# where the tag is determined based on the hash of the contents
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
ARG BASE_IMAGE=vllm/vllm-openai:v0.28.0
|
||||
ARG BASE_IMAGE=vllm/vllm-openai:v0.26.0
|
||||
FROM ${BASE_IMAGE}
|
||||
|
||||
ARG COMMON_WORKDIR=/app
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
ARG VLLM_ASCEND_IMAGE=quay.io/atlas-ci/vllm-ascend
|
||||
ARG VLLM_ASCEND_TAG=v0.28.0
|
||||
ARG VLLM_ASCEND_TAG=v0.27.0
|
||||
FROM ${VLLM_ASCEND_IMAGE}:${VLLM_ASCEND_TAG}
|
||||
|
||||
# WORKDIR /vllm-workspace/vllm
|
||||
@@ -30,7 +30,6 @@ RUN python3 -m pip install -v --no-deps \
|
||||
"stepaudio2-minicpmo>=0.1.1" \
|
||||
"step-audio2>=1.0.0"
|
||||
|
||||
ENV VLLM_TARGET_DEVICE=npu
|
||||
ENV VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
|
||||
ENTRYPOINT []
|
||||
@@ -1,5 +1,5 @@
|
||||
ARG VLLM_ASCEND_IMAGE=quay.io/atlas-ci/vllm-ascend
|
||||
ARG VLLM_ASCEND_TAG=v0.28.0-a3
|
||||
ARG VLLM_ASCEND_TAG=v0.27.0-a3
|
||||
FROM ${VLLM_ASCEND_IMAGE}:${VLLM_ASCEND_TAG}
|
||||
|
||||
# WORKDIR /vllm-workspace/vllm
|
||||
@@ -30,7 +30,6 @@ RUN python3 -m pip install -v --no-deps \
|
||||
"stepaudio2-minicpmo>=0.1.1" \
|
||||
"step-audio2>=1.0.0"
|
||||
|
||||
ENV VLLM_TARGET_DEVICE=npu
|
||||
ENV VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
|
||||
ENTRYPOINT []
|
||||
@@ -1,5 +1,5 @@
|
||||
ARG VLLM_ASCEND_IMAGE=quay.io/atlas-ci/vllm-ascend
|
||||
ARG VLLM_ASCEND_TAG=v0.28.0
|
||||
ARG VLLM_ASCEND_TAG=v0.27.0
|
||||
FROM ${VLLM_ASCEND_IMAGE}:${VLLM_ASCEND_TAG}
|
||||
|
||||
ARG APP_DIR=/vllm-workspace/vllm-omni
|
||||
@@ -65,7 +65,6 @@ RUN python3 -m pip install -v --no-deps \
|
||||
--trusted-host cache-service.nginx-pypi-cache.svc.cluster.local \
|
||||
--default-timeout=60
|
||||
|
||||
ENV VLLM_TARGET_DEVICE=npu
|
||||
ENV VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
|
||||
ENTRYPOINT []
|
||||
@@ -1,5 +1,5 @@
|
||||
ARG VLLM_ASCEND_IMAGE=quay.io/atlas-ci/vllm-ascend
|
||||
ARG VLLM_ASCEND_TAG=v0.28.0-a3
|
||||
ARG VLLM_ASCEND_TAG=v0.27.0-a3
|
||||
FROM ${VLLM_ASCEND_IMAGE}:${VLLM_ASCEND_TAG}
|
||||
|
||||
ARG APP_DIR=/vllm-workspace/vllm-omni
|
||||
@@ -66,7 +66,6 @@ RUN python3 -m pip install -v --no-deps \
|
||||
--trusted-host cache-service.nginx-pypi-cache.svc.cluster.local \
|
||||
--default-timeout=60
|
||||
|
||||
ENV VLLM_TARGET_DEVICE=npu
|
||||
ENV VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
|
||||
ENTRYPOINT []
|
||||
@@ -1,4 +1,4 @@
|
||||
ARG BASE_IMAGE=vllm/vllm-openai-rocm:v0.28.0
|
||||
ARG BASE_IMAGE=vllm/vllm-openai-rocm:v0.27.0
|
||||
FROM ${BASE_IMAGE} AS base
|
||||
|
||||
# Declare a variable to know if we want to use the nightly build or the stable build.
|
||||
@@ -10,7 +10,7 @@ FROM ${BASE_IMAGE} AS base
|
||||
# we should swap over to use stable release ASAP.
|
||||
# We should avoid relying on custom commits.
|
||||
ARG USE_NIGHTLY_BUILD=0
|
||||
ARG VLLM_VERSION_OR_COMMIT_HASH=v0.28.0
|
||||
ARG VLLM_VERSION_OR_COMMIT_HASH=v0.27.0
|
||||
ARG ARG_PYTORCH_ROCM_ARCH
|
||||
ENV PYTORCH_ROCM_ARCH=${ARG_PYTORCH_ROCM_ARCH:-${PYTORCH_ROCM_ARCH}}
|
||||
|
||||
|
||||
+134
-27
@@ -1,33 +1,140 @@
|
||||
# Target vLLM version: v0.28.0. Keep VLLM_VERSION aligned with the target vLLM
|
||||
# release on every rebase.
|
||||
#
|
||||
# This file layers vLLM-Omni on top of the published upstream XPU image, which
|
||||
# already ships a self-consistent XPU runtime (torch-xpu, oneCCL/UCX+NIXL, the
|
||||
# pinned UMD/compute-runtime stack, and vLLM itself) built from
|
||||
# https://github.com/vllm-project/vllm/blob/main/docker/Dockerfile.xpu. Docker
|
||||
# pulls that base automatically, so there is nothing to build here beyond the
|
||||
# Omni layers. Mirrors how docker/Dockerfile.ci consumes vllm/vllm-openai.
|
||||
#
|
||||
# If the tag does not exist yet (targeting an unreleased vLLM commit), build the
|
||||
# base from upstream's own Dockerfile and point VLLM_BASE at the result:
|
||||
#
|
||||
# git clone --depth 1 --branch $VLLM_VERSION https://github.com/vllm-project/vllm /tmp/vllm
|
||||
# docker build -t vllm-openai-xpu:local --target vllm-openai \
|
||||
# -f /tmp/vllm/docker/Dockerfile.xpu /tmp/vllm
|
||||
# docker build -f docker/Dockerfile.xpu --build-arg VLLM_BASE=vllm-openai-xpu:local .
|
||||
#
|
||||
# .buildkite/intel/scripts/run-xpu-test.sh automates that pull-then-build fallback.
|
||||
ARG VLLM_BASE_IMAGE=vllm/vllm-openai-xpu
|
||||
ARG VLLM_VERSION=v0.28.0
|
||||
ARG VLLM_BASE=${VLLM_BASE_IMAGE}:${VLLM_VERSION}
|
||||
# Argument to configure vllm base image if pre-built
|
||||
ARG VLLM_BASE=vllm-base
|
||||
|
||||
FROM intel/deep-learning-essentials:2025.3.2-0-devel-ubuntu24.04 AS vllm-base
|
||||
|
||||
WORKDIR /workspace/
|
||||
|
||||
ARG PYTHON_VERSION=3.12
|
||||
ARG PIP_EXTRA_INDEX_URL="https://download.pytorch.org/whl/xpu"
|
||||
|
||||
RUN wget -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB | gpg --dearmor | tee /usr/share/keyrings/oneapi-archive-keyring.gpg > /dev/null && \
|
||||
echo "deb [signed-by=/usr/share/keyrings/oneapi-archive-keyring.gpg] https://apt.repos.intel.com/oneapi all main" | tee /etc/apt/sources.list.d/oneAPI.list
|
||||
|
||||
RUN apt clean && apt-get update -y && \
|
||||
apt-get install -y --no-install-recommends --fix-missing \
|
||||
curl \
|
||||
ffmpeg \
|
||||
git \
|
||||
libsndfile1 \
|
||||
libsm6 \
|
||||
libxext6 \
|
||||
libgl1 \
|
||||
lsb-release \
|
||||
libaio-dev \
|
||||
numactl \
|
||||
wget \
|
||||
vim \
|
||||
python3.12 \
|
||||
python3.12-dev \
|
||||
python3-pip
|
||||
|
||||
# Add oneAPI repo, pin oneAPI to 2025.3, then install pinned packages in one layer.
|
||||
RUN wget -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB | gpg --dearmor | tee /usr/share/keyrings/oneapi-archive-keyring.gpg > /dev/null && \
|
||||
echo "deb [signed-by=/usr/share/keyrings/oneapi-archive-keyring.gpg] https://apt.repos.intel.com/oneapi all main" | tee /etc/apt/sources.list.d/oneAPI.list && \
|
||||
printf '%s\n' \
|
||||
'Package: intel-oneapi-* intel-deep-learning-essentials* intel-pti*' \
|
||||
'Pin: version 2025.3*' \
|
||||
'Pin-Priority: 1001' \
|
||||
> /etc/apt/preferences.d/oneapi-2025.3.pref && \
|
||||
apt-get update -y && \
|
||||
apt-get install -y --no-install-recommends \
|
||||
intel-oneapi-compiler-dpcpp-cpp-2025.3 \
|
||||
intel-oneapi-mkl-devel-2025.3 \
|
||||
intel-oneapi-dnnl-devel-2025.3 && \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Install UMD with fixed version
|
||||
RUN mkdir neo && \
|
||||
cd neo && \
|
||||
wget https://github.com/intel/intel-graphics-compiler/releases/download/v2.24.8/intel-igc-core-2_2.24.8+20344_amd64.deb && \
|
||||
wget https://github.com/intel/intel-graphics-compiler/releases/download/v2.24.8/intel-igc-opencl-2_2.24.8+20344_amd64.deb && \
|
||||
wget https://github.com/intel/compute-runtime/releases/download/25.48.36300.8/intel-ocloc_25.48.36300.8-0_amd64.deb && \
|
||||
wget https://github.com/intel/compute-runtime/releases/download/25.48.36300.8/intel-opencl-icd_25.48.36300.8-0_amd64.deb && \
|
||||
wget https://github.com/intel/compute-runtime/releases/download/25.48.36300.8/libigdgmm12_22.8.2_amd64.deb && \
|
||||
wget https://github.com/intel/compute-runtime/releases/download/25.48.36300.8/libze-intel-gpu1_25.48.36300.8-0_amd64.deb && \
|
||||
wget https://github.com/oneapi-src/level-zero/releases/download/v1.26.0/level-zero_1.26.0+u24.04_amd64.deb && \
|
||||
dpkg -i *.deb && \
|
||||
cd .. && \
|
||||
rm -rf neo
|
||||
|
||||
ENV PATH="/root/.local/bin:$PATH"
|
||||
ENV VIRTUAL_ENV="/opt/venv"
|
||||
ENV UV_PYTHON_INSTALL_DIR=/opt/uv/python
|
||||
RUN curl -LsSf https://astral.sh/uv/install.sh | sh
|
||||
RUN uv venv --python ${PYTHON_VERSION} --seed ${VIRTUAL_ENV}
|
||||
ENV PATH="$VIRTUAL_ENV/bin:$PATH"
|
||||
|
||||
# This oneccl contains the BMG support which is not the case for default version of oneapi 2025.3.
|
||||
ARG ONECCL_INSTALLER="intel-oneccl-2021.15.9.14_offline.sh"
|
||||
RUN wget "https://github.com/uxlfoundation/oneCCL/releases/download/2021.15.9/${ONECCL_INSTALLER}" && \
|
||||
bash "${ONECCL_INSTALLER}" -a --silent --eula accept && \
|
||||
rm "${ONECCL_INSTALLER}" && \
|
||||
echo "source /opt/intel/oneapi/setvars.sh --force" >> /root/.bashrc && \
|
||||
echo "source /opt/intel/oneapi/ccl/2021.15/env/vars.sh --force" >> /root/.bashrc
|
||||
RUN rm -f /opt/intel/oneapi/ccl/latest && \
|
||||
ln -s /opt/intel/oneapi/ccl/2021.15 /opt/intel/oneapi/ccl/latest
|
||||
|
||||
SHELL ["bash", "-c"]
|
||||
CMD ["bash", "-c", "source /root/.bashrc && exec bash"]
|
||||
|
||||
WORKDIR /workspace/
|
||||
|
||||
ENV UV_HTTP_TIMEOUT=500
|
||||
|
||||
# Configure package index for XPU
|
||||
ENV PIP_EXTRA_INDEX_URL=${PIP_EXTRA_INDEX_URL}
|
||||
ENV UV_EXTRA_INDEX_URL=${PIP_EXTRA_INDEX_URL}
|
||||
ENV UV_INDEX_STRATEGY="unsafe-best-match"
|
||||
ENV UV_LINK_MODE="copy"
|
||||
|
||||
ARG VLLM_VERSION=v0.27.0
|
||||
RUN git clone -b ${VLLM_VERSION} https://github.com/vllm-project/vllm
|
||||
WORKDIR /workspace/vllm
|
||||
|
||||
RUN --mount=type=cache,target=/root/.cache/uv \
|
||||
uv pip install --upgrade pip && \
|
||||
uv pip install -r requirements/xpu.txt && \
|
||||
uv pip install setuptools_scm &&\
|
||||
uv pip install grpcio-tools protobuf nanobind && \
|
||||
source /opt/intel/oneapi/setvars.sh --force && \
|
||||
source /opt/intel/oneapi/ccl/2021.15/env/vars.sh --force && \
|
||||
export CMAKE_PREFIX_PATH="$(python3 -c 'import site; print(site.getsitepackages()[0])'):${CMAKE_PREFIX_PATH}"
|
||||
|
||||
ENV LD_LIBRARY_PATH="/opt/intel/oneapi/ccl/2021.15/lib:$LD_LIBRARY_PATH:/usr/local/lib/"
|
||||
|
||||
ENV VLLM_TARGET_DEVICE=xpu
|
||||
ENV VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
|
||||
RUN --mount=type=cache,target=/root/.cache/uv \
|
||||
uv pip uninstall triton triton-xpu && \
|
||||
uv pip install triton-xpu==3.7.2 && \
|
||||
uv pip install --no-build-isolation --no-deps .
|
||||
|
||||
CMD ["/bin/bash"]
|
||||
|
||||
FROM vllm-base AS vllm-openai
|
||||
|
||||
# install nixl from source code
|
||||
RUN uv pip install nixl
|
||||
|
||||
# install additional dependencies for openai api server
|
||||
RUN --mount=type=cache,target=/root/.cache/uv \
|
||||
uv pip install accelerate hf_transfer pytest pytest_asyncio lm_eval[api] modelscope
|
||||
|
||||
# install development dependencies (for testing)
|
||||
RUN uv pip install -e tests/vllm_test_utils
|
||||
|
||||
# ensure vllm is properly installed
|
||||
RUN python -c "import vllm, inspect; print(vllm.__file__)"
|
||||
RUN uv pip show vllm
|
||||
|
||||
CMD ["/bin/bash"]
|
||||
|
||||
ENTRYPOINT []
|
||||
|
||||
FROM ${VLLM_BASE} AS vllm-omni
|
||||
|
||||
# The base image ships ENTRYPOINT ["vllm", "serve"]; clear it so this stage stays
|
||||
# usable as a plain shell/test container regardless of which VLLM_BASE was used.
|
||||
ENTRYPOINT []
|
||||
CMD ["/bin/bash"]
|
||||
|
||||
RUN apt-get update && \
|
||||
apt-get install -y espeak-ng ffmpeg git jq && \
|
||||
apt-get clean && \
|
||||
|
||||
+10
-20
@@ -5,22 +5,14 @@ nav:
|
||||
- getting_started/quickstart.md
|
||||
- getting_started/installation/*
|
||||
- Serving:
|
||||
- API Server:
|
||||
- Overview: serving/README.md
|
||||
- Core APIs:
|
||||
- Chat Completions: serving/chat_completions_api.md
|
||||
- Text to Speech: serving/speech_api.md
|
||||
- Audio Generation: serving/audio_generate_api.md
|
||||
- Image Generation: serving/image_generation_api.md
|
||||
- Image Edit: serving/image_edit_api.md
|
||||
- Videos: serving/videos_api.md
|
||||
- Streaming and Realtime:
|
||||
- Realtime Audio: serving/realtime_api.md
|
||||
- Full Duplex: serving/full_duplex_api.md
|
||||
- Streaming Video Input: serving/video_stream_api.md
|
||||
- Streaming Video Output: serving/streaming_video_output_api.md
|
||||
- OpenPI Robot Policy: serving/openpi_api.md
|
||||
- Standalone Experimental Servers: serving/standalone_servers.md
|
||||
- OpenAI-Compatible API:
|
||||
- Chat Completions API: serving/chat_completions_api.md
|
||||
- Image Generation: serving/image_generation_api.md
|
||||
- Image Edit: serving/image_edit_api.md
|
||||
- Text to Speech: serving/speech_api.md
|
||||
- Videos: serving/videos_api.md
|
||||
- Audio Generation: serving/audio_generate_api.md
|
||||
- Streaming Video Input: serving/video_stream_api.md
|
||||
- Examples:
|
||||
- examples/README.md
|
||||
- Offline Inference:
|
||||
@@ -32,12 +24,13 @@ nav:
|
||||
- Text-To-Speech: user_guide/examples/offline_inference/text_to_speech.md
|
||||
- Text-To-Video: user_guide/examples/offline_inference/text_to_video.md
|
||||
- X-To-Text: user_guide/examples/offline_inference/x_to_text.md
|
||||
- X-To-Video-Audio: user_guide/examples/offline_inference/x_to_video_audio.md
|
||||
- Online Serving:
|
||||
- Image-To-Image: user_guide/examples/online_serving/image_to_image.md
|
||||
- Image-To-Video: user_guide/examples/online_serving/image_to_video.md
|
||||
- Speech-To-Video: user_guide/examples/online_serving/speech_to_video.md
|
||||
- Text-To-Image: user_guide/examples/online_serving/text_to_image.md
|
||||
- Text-To-Speech (Online Serving): user_guide/examples/online_serving/text_to_speech.md
|
||||
- Text-To-Speech: user_guide/examples/online_serving/text_to_speech.md
|
||||
- Text-To-Video: user_guide/examples/online_serving/text_to_video.md
|
||||
- General:
|
||||
- usage/*
|
||||
@@ -61,7 +54,6 @@ nav:
|
||||
- BitsAndBytes W4: user_guide/quantization/bitsandbytes.md
|
||||
- MXFP8 W8A8: user_guide/quantization/mxfp8.md
|
||||
- MXFP4 W4A4: user_guide/quantization/mxfp4.md
|
||||
- SVDQuant W4A4: user_guide/quantization/svdquant.md
|
||||
- ModelOpt: user_guide/quantization/modelopt.md
|
||||
- AutoRound: user_guide/quantization/autoround.md
|
||||
- msModelSlim: user_guide/quantization/msmodelslim.md
|
||||
@@ -127,8 +119,6 @@ nav:
|
||||
- Async Omni Output Materialization: design/feature/omni_async_output_materialization.md
|
||||
- Automatic Prefix Caching: design/feature/prefix_caching.md
|
||||
- Realtime AR-Diffusion: design/feature/realtime_ar_diffusion.md
|
||||
- Model-Specific Capabilities:
|
||||
- MiniMax H3 Temporal VAE Chunks: design/feature/minimax_h3_temporal_vae_chunks.md
|
||||
- Communication:
|
||||
- OmniConnector Implementations:
|
||||
- Mooncake Store Connector: design/feature/omni_connectors/mooncake_store_connector.md
|
||||
|
||||
+3
-3
@@ -57,9 +57,9 @@ vLLM-Omni is flexible and easy to use with:
|
||||
vLLM-Omni seamlessly supports most popular open-source models on HuggingFace, including:
|
||||
|
||||
- Omni-modality models (e.g. Qwen3-Omni, MiniCPM-o 4.5, Cosmos3, HunyuanImage, BAGEL)
|
||||
- TTS models (e.g. Qwen3-TTS, IndexTTS 2.5, dots.tts, CosyVoice3)
|
||||
- Diffusion models — image, video, and audio generation (e.g. MiniMax H3, LTX-2.5, SANA-Video, Wan2.2)
|
||||
- Robot-policy and action models (e.g. π0, GR00T-N1.7, DreamZero-DROID, InternVLA-A1)
|
||||
- TTS models (e.g. Qwen3-TTS, VoxCPM2, Ming-Omni-TTS, CosyVoice3)
|
||||
- Diffusion models — image, video, and audio generation (e.g. MiniMax H3, Qwen-Image, Wan2.2, FLUX)
|
||||
- Robot-policy and action models (e.g. GR00T-N1.7, DreamZero-DROID, InternVLA-A1, Cosmos3 action policy)
|
||||
|
||||
For more information, checkout the following:
|
||||
|
||||
|
||||
@@ -114,10 +114,6 @@ Configuration classes.
|
||||
- [vllm_omni.model_executor.models.qwen3_tts.tokenizer_25hz.configuration_qwen3_tts_tokenizer_v1.Qwen3TTSTokenizerV1DecoderDiTConfig][]
|
||||
- [vllm_omni.model_executor.models.qwen3_tts.tokenizer_25hz.configuration_qwen3_tts_tokenizer_v1.Qwen3TTSTokenizerV1EncoderConfig][]
|
||||
- [vllm_omni.transformers_utils.configs.cosyvoice3.CosyVoice3Config][]
|
||||
- [vllm_omni.transformers_utils.configs.mammoth_moda2.Mammothmoda2Config][]
|
||||
- [vllm_omni.transformers_utils.configs.mammoth_moda2.Mammothmoda2Qwen2_5_VLConfig][]
|
||||
- [vllm_omni.transformers_utils.configs.mammoth_moda2.Mammothmoda2Qwen2_5_VLTextConfig][]
|
||||
- [vllm_omni.transformers_utils.configs.mammoth_moda2.Mammothmoda2Qwen2_5_VLVisionConfig][]
|
||||
- [vllm_omni.transformers_utils.configs.omnivoice.OmniVoiceConfig][]
|
||||
- [vllm_omni.transformers_utils.configs.voxcpm2.VoxCPM2Config][]
|
||||
|
||||
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 274 KiB After Width: | Height: | Size: 269 KiB |
@@ -2,7 +2,7 @@
|
||||
|
||||
This section lists the most common options for running vLLM-Omni.
|
||||
|
||||
For options within a vLLM Engine, please refer to the [vLLM 0.28 configuration guide](https://docs.vllm.ai/en/v0.28.0/configuration/index.html).
|
||||
For options within a vLLM Engine, please refer to the [vLLM 0.26 configuration guide](https://docs.vllm.ai/en/v0.26.0/configuration/index.html).
|
||||
|
||||
Each model defines fixed topology in a registered `PipelineConfig` and runtime
|
||||
overrides in a deploy YAML.
|
||||
|
||||
@@ -82,8 +82,6 @@ depends on the installed kernels and model path.
|
||||
| --- | --- | --- | --- | --- |
|
||||
| `SPEAKER_SAMPLES_DIR` | Filesystem path; default `~/.cache/vllm-omni/speakers` | Speech server; read when speaker storage initializes | Environment-only setting. The directory is created; filesystem errors propagate. | Stable |
|
||||
| `SPEAKER_MAX_UPLOADED` | Integer; default `1000` | Speech server; read when speaker storage initializes | Environment-only setting. A non-integer logs a warning and uses `1000`; range is not otherwise validated. | Stable |
|
||||
| `VLLM_OMNI_ASYNC_OUTPUT_TIMEOUT` | Float seconds; default `600` | Diffusion engine async-output wait in `step_streaming`; resolved per call on the request path, not at import | Environment-only setting. A non-float or `<=0` value warns once and uses the default. | Experimental |
|
||||
| `VLLM_OMNI_EVENT_DRIVEN_ORCH` | `1`, `true`, `yes` or `on` enables; default `0` (off) | Orchestration loop and the serving-side final-output drain; read once when the `Orchestrator` is constructed | Environment-only setting. Values are stripped and case-normalized; any unrecognized value leaves the legacy poll loop selected. | Experimental |
|
||||
| `VLLM_OMNI_INPUT_WAIT_TIMEOUT_S` | Float seconds; default `600`; `<=0` disables | Full-payload input coordinator, not async-chunk transfer; read when the scheduler module imports in each worker | Environment-only setting. A non-float logs a warning and uses `600`. | Stable operational control |
|
||||
| `VLLM_OMNI_ORCH_MONITOR_PATH` | Filesystem path; default `<current-working-directory>/vllm_omni_orch_monitor_<timestamp>.json` | Orchestrator monitor enabled by `--enable-orch-monitor`; read when the monitor is created | Environment-only path override. Parent directories are created; write errors are logged. | Diagnostic |
|
||||
| `VLLM_OMNI_VIDEO_SYNC_TIMEOUT` | Float seconds; default `600` | Synchronous Videos API; read when the API server module imports | Environment-only setting. A non-float raises `ValueError` during import. | Experimental |
|
||||
@@ -151,7 +149,7 @@ their keys only.
|
||||
## Inherited vLLM variables
|
||||
|
||||
vLLM-Omni also reads variables through its aligned vLLM dependency. Refer to
|
||||
the [vLLM 0.28 environment-variable reference](https://docs.vllm.ai/en/v0.28.0/configuration/env_vars.html)
|
||||
the [vLLM 0.26 environment-variable reference](https://docs.vllm.ai/en/v0.26.0/configuration/env_vars.html)
|
||||
for their definitions. This includes vLLM launch, cache, logging, plugin, ROCm,
|
||||
XPU, ModelScope, and FlashInfer workspace settings.
|
||||
|
||||
@@ -179,7 +177,7 @@ collection, examples, bug reports, or logs.
|
||||
|
||||
## Model-specific variables
|
||||
|
||||
The audit found 56 variables read by a single model or pipeline family. They are
|
||||
The audit found 57 variables read by a single model or pipeline family. They are
|
||||
not listed as public usage options here because doing so would turn implementation
|
||||
escape hatches into an accidental compatibility contract.
|
||||
|
||||
@@ -192,7 +190,7 @@ Every audited model-specific name has a migration disposition in the
|
||||
| Request scope | 6 | Move request-varying behavior into a declared request-option schema. |
|
||||
| External | 0 | Retain only when a supported third-party library owns the contract. |
|
||||
| Internalize | 11 | Keep a debug or diagnostic switch out of public documentation and configuration. |
|
||||
| Deprecate/remove | 6 | Remove a compatibility escape hatch that has no continuing contract. |
|
||||
| Deprecate/remove | 7 | Remove a compatibility escape hatch that has no continuing contract. |
|
||||
|
||||
The disposition is a migration target, not a statement that the existing
|
||||
environment switch is stable. Promote or request-scope work should land in
|
||||
|
||||
@@ -14,7 +14,7 @@ describe fixed model topology and are not accepted in deploy YAMLs.
|
||||
Common `PipelineConfig` fields include:
|
||||
|
||||
| Field | Description |
|
||||
| ------- | ------------- |
|
||||
|-------|-------------|
|
||||
| `model_type` | Pipeline identifier used during model and config resolution. |
|
||||
| `default_deploy_config_name` | Bundled deploy YAML loaded when the user does not pass `deploy_config`. |
|
||||
| `model_arch` | Default Hugging Face architecture for the pipeline. |
|
||||
@@ -26,7 +26,7 @@ Common `PipelineConfig` fields include:
|
||||
Common `StagePipelineConfig` fields include:
|
||||
|
||||
| Field | Description |
|
||||
| ------- | ------------- |
|
||||
|-------|-------------|
|
||||
| `stage_id` | Stable stage identifier. |
|
||||
| `model_stage` | Logical stage name used by runtime and strategy resolution. |
|
||||
| `execution_type` | `LLM_AR`, `LLM_GENERATION`, or `DIFFUSION`. |
|
||||
@@ -49,17 +49,17 @@ deployment overrides.
|
||||
The new deploy schema lives under `vllm_omni/deploy/` and is paired with a frozen `PipelineConfig` registered by the model's `pipeline.py`. Each deploy YAML has these top-level fields:
|
||||
|
||||
| Field | Type | Required | Default | Description |
|
||||
| ------- | ------ | ---------- | --------- | ------------- |
|
||||
|-------|------|----------|---------|-------------|
|
||||
| `base_config` | str (path) | optional | — | Overlay parent (relative or absolute). `stages:` / `platforms:` deep-merged by stage_id; other scalars overlay-wins. Intended for user-authored overlays; prod yamls stay flat. |
|
||||
| `async_chunk` | bool | optional | `true` | Enable chunked streaming between stages. Pin to `false` if the pipeline runs end-to-end. |
|
||||
| `session_mode` | str | optional | `"turn"` | Session behavior. MiniCPM-o 4.5 deploy YAMLs set `"duplex"` so `/v1/realtime` and chat share the same profile. |
|
||||
| `session_mode` | str | optional | `"turn"` | Session behavior. Use `"duplex"` only with a pipeline that enables duplex control. |
|
||||
| `active_stream_window` | int | optional | `0` | Number of active downstream stream slots; `0` preserves all-stream cycling. |
|
||||
| `duplex_session` | dict | optional | runtime defaults | Full-duplex session lifecycle, buffering, replay, and capacity limits. |
|
||||
| `connectors` | dict | optional | `null` | Named connector specs (`{name, extra}`). Referenced by each stage's `input_connectors` / `output_connectors`. See [Connector schema](#connector-schema). |
|
||||
| `edges` | list | optional | `null` | Explicit edge list for the KV transfer graph. Auto-derived from stage inputs if omitted. |
|
||||
| `stages` | list | optional | `[]` | Per-stage runtime overrides matched by `stage_id`. Pipeline stages are still created from `PipelineConfig` when this list is empty. |
|
||||
| `platforms` | dict | optional | `null` | Keyed by `npu` / `rocm` / `xpu`, each contains a `stages:` list with per-platform overrides applied on top of the CUDA defaults. |
|
||||
| `pipeline` | str | optional | `null` | Override the auto-detected pipeline registry key (used for structural variants like `qwen2_5_omni_thinker_only` / `qwen3_omni_moe_thinker_only`). |
|
||||
| `pipeline` | str | optional | `null` | Override the auto-detected pipeline registry key (used for structural variants like `qwen2_5_omni_thinker_only`). |
|
||||
| `trust_remote_code` | bool \| null | optional | `null` | **Pipeline-wide.** Trust HF remote code on model load; applies to every stage when specified. |
|
||||
| `distributed_executor_backend` | str \| null | optional | `null` | **Pipeline-wide.** Distributed executor backend forwarded to vLLM (`"mp"`, `"ray"`, `"external_launcher"`). If omitted, vLLM auto-selects backend from runtime topology. |
|
||||
| `dtype` | str \| null | optional | `null` | **Pipeline-wide.** Model dtype for every stage. |
|
||||
@@ -84,7 +84,7 @@ subprocess on one GPU. `ray` / `external_launcher` are not fully supported yet.
|
||||
Each entry under `stages:` accepts any `StageDeployConfig` field directly (no nested `engine_args:`). Only fields whose value legitimately varies across stages live here; pipeline-wide settings (trust_remote_code, distributed_executor_backend, dtype, quantization, prefix/chunked prefill, DP/PP sizes) are declared at the top level and applied to every stage. Unknown keys fall through to `engine_extras:` and are forwarded to the engine. Frequently used fields are listed below; the source-of-truth schema is `StageDeployConfig` in `vllm_omni/config/stage_config.py`.
|
||||
|
||||
| Field | Type | Required | Default | Description |
|
||||
| ------- | ------ | ---------- | --------- | ------------- |
|
||||
|-------|------|----------|---------|-------------|
|
||||
| `stage_id` | int | required | — | Stage identity; matched against `PipelineConfig.stages[*].stage_id`. |
|
||||
| `max_num_seqs` | int \| null | optional | `null` | Max concurrent sequences per stage. |
|
||||
| `gpu_memory_utilization` | float \| null | optional | `null` | Per-stage total memory target; used for automatic KV-cache sizing. |
|
||||
@@ -94,14 +94,12 @@ Each entry under `stages:` accepts any `StageDeployConfig` field directly (no ne
|
||||
| `max_num_batched_tokens` | int \| null | optional | `null` | Per-stage prefill/token budget; also contributes to the native maximum in-flight token limit. |
|
||||
| `max_model_len` | int \| null | optional | `null` | Per-sequence context or KV length; `-1` enables native cache-capacity auto-fitting, while values above the HF default auto-set `VLLM_ALLOW_LONG_MAX_MODEL_LEN=1`. |
|
||||
| `async_scheduling` | bool \| null | optional | `null` | Per-stage async scheduling toggle. |
|
||||
| `devices` | str \| null | optional | `null` | Device list assigned to this stage. The number of device ids must equal this stage's local world size (`tensor_parallel_size` × local data-parallel size × `pipeline_parallel_size`, or `num_replicas` × that product for a replica pool); a mismatch fails early — see the note below. |
|
||||
| `devices` | str \| null | optional | `null` | Device list assigned to this stage. |
|
||||
| `output_connectors` | dict \| null | optional | `null` | Keyed by `to_stage_<n>`; values are names registered under top-level `connectors:`. |
|
||||
| `input_connectors` | dict \| null | optional | `null` | Keyed by `from_stage_<n>`; values are names registered under top-level `connectors:`. |
|
||||
| `default_sampling_params` | dict \| null | optional | `null` | Baseline sampling params. Deep-merged with pipeline `sampling_constraints` (pipeline wins). |
|
||||
| `engine_extras` | dict | optional | `{}` | Catch-all for engine fields not listed above; deep-merged across overlays and forwarded to the stage engine. |
|
||||
|
||||
**Note:** a stage's `devices` count must equal its local world size (`tensor_parallel_size` × `data_parallel_size_local` × `pipeline_parallel_size`, falling back to global `data_parallel_size` when the local size is unset), or `num_replicas` × that product for a replica pool. A mismatch fails early and names the offending stage. A top-level `--tensor-parallel-size` is broadcast to every stage, so it can make a single-GPU stage violate this contract ([issue #5003](gh-issue:5003)); fix that case with `--stage-overrides` (set `tensor_parallel_size` and `devices` together per stage) or set TP only on the multi-GPU stage.
|
||||
|
||||
### Connector schema
|
||||
|
||||
Each entry under top-level `connectors:` follows this shape:
|
||||
@@ -115,9 +113,9 @@ connectors:
|
||||
# Additional connector-specific options
|
||||
```
|
||||
|
||||
| Connector class | Use case | `extra` keys |
|
||||
|--------------------------|-----------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------|
|
||||
| `SharedMemoryConnector` | Same-host KV transfer between stages (default for bundled YAMLs). | None. All payloads use shared memory. |
|
||||
| Connector class | Use case | `extra` keys |
|
||||
|-----------------|----------|--------------|
|
||||
| `SharedMemoryConnector` | Same-host KV transfer between stages (default for bundled YAMLs). | None. All payloads use shared memory. |
|
||||
| `MooncakeStoreConnector` | Cross-host KV transfer over TCP. Required for multi-node deployments. | `host`, `metadata_server`, `master`, `segment` (int bytes), `localbuf` (int bytes), `proto` (`"tcp"` / `"rdma"`). |
|
||||
|
||||
A stage references a connector by name in its `input_connectors` / `output_connectors`:
|
||||
@@ -137,7 +135,7 @@ stages:
|
||||
### CLI flags
|
||||
|
||||
| Flag | Description |
|
||||
| ------ | ------------- |
|
||||
|------|-------------|
|
||||
| `--deploy-config PATH` | Load a deploy YAML. **Optional** — when omitted, the bundled `vllm_omni/deploy/<model_type>.yaml` is auto-loaded by the model registry. |
|
||||
| `--stage-overrides JSON` | Per-stage JSON overrides, e.g. `'{"0":{"gpu_memory_utilization":0.5}}'`. Per-stage values always win over global flags. |
|
||||
| `--async-chunk` / `--no-async-chunk` | Flip the deploy YAML's `async_chunk:` bool. Unset (default) leaves the YAML value in force. |
|
||||
@@ -251,7 +249,7 @@ CUDA_VISIBLE_DEVICES=0 vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni \
|
||||
Effective config per stage after the merge:
|
||||
|
||||
| Stage | Field | Final value | Source |
|
||||
| ------- | ------- | ------------- | -------- |
|
||||
|-------|-------|-------------|--------|
|
||||
| 0 | `gpu_memory_utilization` | `0.9` | base YAML (overlay didn't touch stage 0) |
|
||||
| 0 | `max_num_seqs` | `8` | per-stage CLI (`--stage-overrides`) — wins over base `32` |
|
||||
| 0 | `max_model_len` | `16384` | global CLI |
|
||||
@@ -275,14 +273,12 @@ when loading a deploy configuration.
|
||||
Examples:
|
||||
|
||||
For offline inference (assuming the necessary dependencies have been imported):
|
||||
|
||||
```python
|
||||
model_name = "Qwen/Qwen2.5-Omni-7B"
|
||||
omni = Omni(model=model_name, deploy_config="/path/to/deploy_config.yaml")
|
||||
```
|
||||
|
||||
For online serving:
|
||||
|
||||
```bash
|
||||
vllm serve Qwen/Qwen2.5-Omni-7B --omni --port 8091 --deploy-config /path/to/deploy_config.yaml
|
||||
```
|
||||
|
||||
@@ -49,7 +49,7 @@ vLLM-Omni's `pre-commit` hooks will now run automatically every time you commit.
|
||||
`--all-files` run stays green while historical debt is cleaned up. The
|
||||
current `SKIP` list in
|
||||
[`.github/workflows/pre-commit.yml`](https://github.com/vllm-project/vllm-omni/blob/main/.github/workflows/pre-commit.yml)
|
||||
is `check-mark`, `markdownlint-cli2`, `shellcheck`,
|
||||
is `check-test-ci-coverage`, `markdownlint-cli2`, `shellcheck`,
|
||||
`check-spdx-header`, and `mypy-3.10`. Those hooks still run on **your
|
||||
commit** for changed files. A passing GitHub pre-commit check does not mean
|
||||
they passed locally. Hooks **not** on that list (forbidden imports,
|
||||
@@ -66,7 +66,7 @@ already there). `check-pickle-imports` is gone: pickle is now one rule inside
|
||||
| `markdownlint-cli2` | Markdown in `docs/`, `recipes/`, `README.md`, `CONTRIBUTING.md` (not `.claude/` / `.cursor/` / `CLAUDE.md`) | skipped |
|
||||
| `mypy-3.10` | Type-check changed `vllm_omni/` files (model trees excluded). `tests/` uses `--follow-imports skip` | skipped |
|
||||
| `mypy-3.11` / `3.12` / `3.13` | Same checker, extra Python versions | not installed; `pre-commit run --hook-stage manual mypy-3.12` |
|
||||
| `check-mark` | Every `tests/**/test_*.py` has a CI level mark and a hardware platform mark or helper; no direct `pytest.mark.H100` / SKU | skipped |
|
||||
| `check-test-ci-coverage` | Every `tests/**/test_*.py` has a CI level mark and a hardware mark/helper | skipped |
|
||||
| `check-tts-adapter-migration` | `self._tts_model_type` branches in `serving_speech.py` must not increase | runs |
|
||||
| `shellcheck` | `*.sh` quoting / undefined vars | skipped |
|
||||
| `check-spdx-header` | Omni SPDX header on `.py` / `.pyi` / `.sh` / `.rs` / `.proto` | skipped |
|
||||
@@ -144,13 +144,11 @@ pre-commit run --hook-stage manual mypy-3.12
|
||||
|
||||
#### Test CI marks
|
||||
|
||||
`check-mark` requires each collected test module under `tests/` to
|
||||
`check-test-ci-coverage` requires each collected test module under `tests/` to
|
||||
have at least one CI **level** mark (`core_model`, `advanced_model`,
|
||||
`full_model`, `local_model`, or `slow`) and a **hardware** platform mark
|
||||
(`cpu`, `cuda`, …) or helper (`hardware_test(` / `hardware_marks(`). SKU marks
|
||||
(`H100`, `L4`, …) must come from those helpers so `cards_{n}` is attached;
|
||||
direct `pytest.mark.H100` is rejected. GitHub Actions skips this hook; local
|
||||
commit does not. See the
|
||||
`full_model`, `local_model`, or `slow`) and a **hardware** mark (`cpu`, `cuda`,
|
||||
`H100`, …) or helper (`hardware_test(` / `hardware_marks(`). GitHub Actions
|
||||
skips this hook; local commit does not. See the
|
||||
[test writing guide](./ci/test_writing_guide.md).
|
||||
|
||||
#### TTS adapter ratchet
|
||||
|
||||
@@ -402,7 +402,7 @@ loads the model once per mode.
|
||||
Before splitting, confirm each half actually collects at least one test under the job's forwarded filters — pytest exits with code 5 ("no tests collected") if a half is empty, which the script reports as a failure, red-ing a job that used to pass as one combined invocation.
|
||||
|
||||
Also check the tests' `num_cards` against the job's `mirror_hardwares` preset.
|
||||
`_cuda_marks` turns `num_cards > 1` into `skipif(device_count() < num_cards)`, so a
|
||||
`cuda_marks` turns `num_cards > 1` into `skipif(device_count() < num_cards)`, so a
|
||||
test asking for more GPUs than the preset provides is silently skipped and its
|
||||
coverage artifact comes back empty. Splitting such a job produces a file that
|
||||
looks fine and measures nothing — "Diffusion · Z Image Test" is the current
|
||||
|
||||
@@ -81,8 +81,8 @@ Supported form:
|
||||
|
||||
| Form | Example | Effect |
|
||||
| ---- | ------- | ------ |
|
||||
| Array (single platform) | `"mark": [{"hardware_marks": {"res": {"cuda": "H100"}, "num_cards": 2}}, "full_model", "diffusion"]` | Hardware marks from `hardware_marks(...)`; `num_cards` adds `cards_{n}` (including `cards_1` when omitted). String entries become extra pytest marks. |
|
||||
| Array (multi-platform) | `"mark": [{"hardware_marks": {"res": {"cuda": "H100", "rocm": "MI325"}, "num_cards": 2}}, "full_model", "omni"]` | Same as above, but declares **multiple platforms** on one item. All platforms must share the same `num_cards` (int or identical dict values). Different counts need `@hardware_test` or one `hardware_marks()` per `pytest.param`. Filter examples: `-m "full_model and H100 and cards_2 and cuda"`, `-m "full_model and MI325 and cards_2 and rocm"`. |
|
||||
| Array (single platform) | `"mark": [{"hardware_marks": {"res": {"cuda": "H100"}, "num_cards": 2}}, "full_model", "diffusion"]` | Hardware marks from `hardware_marks(...)`; `num_cards` > 1 adds `distributed_*` (+ CUDA `skipif_cuda` when applicable). String entries become extra pytest marks. |
|
||||
| Array (multi-platform) | `"mark": [{"hardware_marks": {"res": {"cuda": "H100", "rocm": "MI325", "npu": "A2"}, "num_cards": {"cuda": 2, "rocm": 2, "npu": 1}}}, "full_model", "omni"]` | Same as above, but declares **multiple platforms** in one case. Filter examples: `-m "full_model and H100 and cuda"`, `-m "full_model and MI325 and rocm"`. |
|
||||
|
||||
Recommended for L4 perf cases:
|
||||
|
||||
@@ -157,7 +157,7 @@ pytest -s -v tests/dfx/perf/scripts/run_benchmark.py \
|
||||
```
|
||||
|
||||
<!-- markdownlint-disable-next-line MD051 -->
|
||||
See also [Markers for Tests](#markers-for-tests) for registered hardware markers (`H100`, `L4`, `cuda`, `cards_1`, …).
|
||||
See also [Markers for Tests](#markers-for-tests) for registered hardware markers (`H100`, `L4`, `cuda`, `distributed_cuda`, …).
|
||||
|
||||
#### `server_params` Configuration
|
||||
|
||||
|
||||
@@ -88,7 +88,7 @@ Failed jobs: 1/2
|
||||
```bash
|
||||
cd tests
|
||||
pytest -s -v test_xxxx.py --run-level=core_model
|
||||
pytest -s -v -m "core_model and L4 and not cards_1" --run-level=core_model
|
||||
pytest -s -v -m "core_model and distributed_cuda and L4" --run-level=core_model
|
||||
```
|
||||
|
||||
=== "L3 level"
|
||||
@@ -119,7 +119,7 @@ Failed jobs: 1/2
|
||||
|
||||
```bash
|
||||
pytest -s -v test_xxxx.py --run-level=advanced_model
|
||||
pytest -s -v -m "advanced_model and L4 and not cards_1" --run-level=advanced_model
|
||||
pytest -s -v -m "advanced_model and distributed_cuda and L4" --run-level=advanced_model
|
||||
```
|
||||
|
||||
=== "L4 level"
|
||||
@@ -153,12 +153,12 @@ Failed jobs: 1/2
|
||||
```bash
|
||||
cd tests
|
||||
pytest -s -v test_xxxx.py --run-level=full_model
|
||||
pytest -s -v -m "full_model and L4 and not cards_1" --run-level=full_model
|
||||
pytest -s -v -m "full_model and distributed_cuda and L4" --run-level=full_model
|
||||
pytest -s -v -m "full_model and (omni or tts) and H100" --run-level=full_model
|
||||
```
|
||||
If you only want to run specific test cases on a particular platform, you can use:
|
||||
```bash
|
||||
pytest -s -v -m "full_model and L4 and not cards_1" --run-level=full_model
|
||||
pytest -s -v -m "full_model and distributed_cuda and L4" --run-level=full_model
|
||||
```
|
||||
Note: ``run_benchmark.py`` and ``run_diffusion_benchmark.py`` accept an optional ``--test-config-file``. If omitted, each loads every ``*.json`` under ``tests/dfx/perf/tests/`` (omni/tts vs diffusion split by ``is_diffusion_perf_config``) and pytest ``-m`` filters by each case's JSON ``mark``:
|
||||
```bash
|
||||
|
||||
@@ -48,12 +48,17 @@ Defined in `pyproject.toml`:
|
||||
| `MI325` | Tests that require MI325 GPU (AMD/ROCm) * |
|
||||
| `A2` | Tests that require A2 NPU * |
|
||||
| `A3` | Tests that require A3 NPU * |
|
||||
| `cards_{n}` | Tests that require *n* accelerator cards (`cards_1` … `cards_8`); auto-added from `num_cards` * |
|
||||
| `distributed_cuda` | Tests that require multi cards on CUDA platform * |
|
||||
| `distributed_rocm` | Tests that require multi cards on ROCm platform * |
|
||||
| `distributed_npu` | Tests that require multi cards on NPU platform * |
|
||||
| `skipif_cuda` | Skip if the num of CUDA cards is less than the required * |
|
||||
| `skipif_rocm` | Skip if the num of ROCm cards is less than the required * |
|
||||
| `skipif_npu` | Skip if the num of NPU cards is less than the required * |
|
||||
| `slow` | Slow tests (may skip in quick CI) |
|
||||
| `benchmark` | Benchmark tests (decorator on runner test functions; perf JSON uses `full_model` + type marker instead) |
|
||||
| `local_model` | Tests requiring local / non-HF-hub model weights |
|
||||
|
||||
\* Means those markers are auto-added by `@hardware_test` (parametrization decorator) or `hardware_marks` (only returning the list of marks for flexibility). Do **not** write `pytest.mark.H100` / `pytest.mark.L4` (or other SKU names) by hand: `check-mark` rejects that so `cards_{n}` cannot be skipped.
|
||||
\* Means those markers are auto-added by `@hardware_test` (parametrization decorator) or `hardware_marks` (only returning the list of marks for flexibility).
|
||||
|
||||
#### Example usage for markers
|
||||
|
||||
@@ -76,23 +81,25 @@ def test_video_to_audio()
|
||||
This decorator is intended to make hardware-aware, cross-platform test authoring easier and more robust for CI/CD environments. The `hardware_test` decorator in `vllm-omni/tests/helpers/mark.py` performs the following actions:
|
||||
|
||||
1. **Applies platform and resource markers**
|
||||
Adds the appropriate pytest markers for each specified hardware platform (e.g., `cuda`, `rocm`, `xpu`, `npu`) and resource type (e.g., `L4`, `H100`, `H200`, `B200`, `MI325`, `B60`, `A2`, `A3`).
|
||||
Adds the appropriate pytest markers for each specified hardware platform (e.g., `cuda`, `rocm`, `xpu`, `npu`) and resource type (e.g., `L4`, `H100`, `MI325`, `B60`, `A2`, `A3`).
|
||||
```python
|
||||
@pytest.mark.cuda
|
||||
@pytest.mark.L4
|
||||
```
|
||||
2. **Handles card-count filtering**
|
||||
Adds a filterable marker `cards_{n}` (including `cards_1` when `num_cards` is omitted / 1). Use this with the SKU and platform, e.g. `-m "H100 and cards_4 and cuda"` vs `-m "H100 and cards_2 and cuda"`. Any multi-card CUDA test is `-m "cuda and not cards_1"`.
|
||||
|
||||
When `res` lists more than one platform **and** `num_cards` differs by platform, `@hardware_test` collects **one variant per platform** so those `cards_{n}` marks do not stack. `-m "H100 and cards_4 and cuda"` then selects only the CUDA/H100 4-card variant, not a ROCm 2-card sibling. The same count on every platform stays a single item.
|
||||
2. **Handles multi-card (distributed) scenarios**
|
||||
For tests requiring multiple cards, it automatically adds distributed markers such as `distributed_cuda`, `distributed_rocm`, or `distributed_npu`.
|
||||
```python
|
||||
@pytest.mark.distributed_cuda(num_cards=num_cards)
|
||||
```
|
||||
3. **Supports flexible card requirements**
|
||||
Accepts `num_cards` as either a single integer for all platforms or as a dictionary with per-platform values. If not specified, defaults to 1 card per platform.
|
||||
|
||||
4. **Integrates resource validation**
|
||||
On CUDA and XPU, when `num_cards > 1`, attaches pytest's built-in `skipif` if the machine has fewer devices than requested. This is not a named `skipif_*` marker and is not used for `-m` filtering.
|
||||
On CUDA, adds a skip marker (`skipif_cuda`) if the system does not have the required number of devices.
|
||||
Support for `skipif_rocm` and `skipif_npu` will be implemented later.
|
||||
|
||||
5. **Works with pytest filtering**
|
||||
Allows tests to be filtered and selected at runtime using standard pytest marker expressions (e.g., `-m "H100 and cards_2"`, `-m "L4 and not cards_1"`).
|
||||
Allows tests to be filtered and selected at runtime using standard pytest marker expressions (e.g., `-m "distributed_cuda and L4"`).
|
||||
|
||||
##### Example usage for decorator
|
||||
|
||||
@@ -110,14 +117,13 @@ This decorator is intended to make hardware-aware, cross-platform test authoring
|
||||
num_cards=2,
|
||||
)
|
||||
```
|
||||
- `res` must be a dict. SKU values come from `pyproject.toml` markers tagged `[hardware-resource]` plus the platform key (`[cuda]`, `[rocm]`, `[xpu]`, `[npu]`, `[musa]`).
|
||||
- `res` must be a dict; supported resources: CUDA (L4/H100), ROCm (MI325), XPU (B60), MUSA (S5000), NPU (A2/A3)
|
||||
- `num_cards` can be int (all platforms) or dict (per platform); defaults to 1 when missing
|
||||
- Distributed markers (`distributed_cuda`, `distributed_rocm`, `distributed_npu`) are auto-added for multi-card cases
|
||||
- Filtering examples:
|
||||
- CUDA H100 4-card: `pytest -m "H100 and cards_4 and cuda"`
|
||||
- CUDA H100 2-card: `pytest -m "H100 and cards_2 and cuda"`
|
||||
- CUDA multi-card (any count): `pytest -m "L4 and not cards_1"`
|
||||
- ROCm multi-card: `pytest -m "MI325 and not cards_1"`
|
||||
- NPU multi-card: `pytest -m "npu and not cards_1"`
|
||||
- CUDA only: `pytest -m "distributed_cuda and L4"`
|
||||
- ROCm only: `pytest -m "distributed_rocm and MI325"`
|
||||
- NPU only: `pytest -m "distributed_npu"`
|
||||
|
||||
#### Function: `hardware_marks`
|
||||
|
||||
@@ -173,30 +179,37 @@ If you want to add support for a new platform (e.g., "tpu" for a new accelerator
|
||||
[tool.pytest.ini_options]
|
||||
markers = [
|
||||
# ... existing markers ...
|
||||
"tpu: [hardware-platform] Tests that require TPU device",
|
||||
# Add [gpu] next to [hardware-platform] if this platform should also get pytest.mark.gpu.
|
||||
"TPU_V3: [hardware-resource] [tpu] Tests that require TPU v3 hardware",
|
||||
"cards_1: [hardware-cards] Tests that require 1 accelerator card",
|
||||
"cards_2: [hardware-cards] Tests that require 2 accelerator cards",
|
||||
"tpu: Tests that require TPU device",
|
||||
"TPU_V3: Tests that require TPU v3 hardware",
|
||||
"distributed_tpu: Tests that require multiple TPU devices",
|
||||
]
|
||||
```
|
||||
`check-mark` treats `[hardware-platform]` names (including `cpu`/`gpu`) as allowed `pytest.mark.<name>` values. `hardware_test(res=...)` still rejects `cpu`/`gpu` as `res` keys.
|
||||
2. **Implement a marker construction function for your platform** in `vllm-omni/tests/helpers/mark.py`:
|
||||
```python
|
||||
# In vllm-omni/tests/helpers/mark.py
|
||||
|
||||
def _tpu_marks(*, res: str, num_cards: int):
|
||||
_require_sku("tpu", res)
|
||||
return [pytest.mark.tpu, getattr(pytest.mark, res), _cards_mark(num_cards)]
|
||||
def tpu_marks(*, res: str, num_cards: int):
|
||||
test_platform = pytest.mark.tpu
|
||||
if res == "TPU_V3":
|
||||
test_resource = pytest.mark.TPU_V3
|
||||
else:
|
||||
raise ValueError(
|
||||
f"Invalid TPU resource type: {res}. Supported: TPU_V3")
|
||||
|
||||
if num_cards == 1:
|
||||
return [test_platform, test_resource]
|
||||
else:
|
||||
test_distributed = pytest.mark.distributed_tpu(num_cards=num_cards)
|
||||
# Optionally: add skipif_tpu when implemented
|
||||
return [test_platform, test_resource, test_distributed]
|
||||
```
|
||||
3. **Register the builder** in `_marks_for_platform`:
|
||||
3. **Update `hardware_marks` to recognize your new platform**:
|
||||
In the relevant place (see the `hardware_marks` implementation), add:
|
||||
```python
|
||||
builders = {
|
||||
# ... existing platforms ...
|
||||
"tpu": _tpu_marks,
|
||||
}
|
||||
if platform == "tpu":
|
||||
marks = tpu_marks(res=resource, num_cards=cards)
|
||||
```
|
||||
(`hardware_test` / `hardware_marks` both go through `_marks_for_platform`.)
|
||||
(`hardware_test` calls `hardware_marks` internally, so both will pick up the change.)
|
||||
4. **(Recommended) Add a test using your new markers**:
|
||||
```python
|
||||
@hardware_test(
|
||||
@@ -210,11 +223,11 @@ If you want to add support for a new platform (e.g., "tpu" for a new accelerator
|
||||
**Summary**:
|
||||
|
||||
- Add pytest markers for your new platform/resources
|
||||
- Implement a private marker function (`_xxx_marks`)
|
||||
- Register it in `_marks_for_platform`
|
||||
- You're done: tests using `@hardware_test` or `hardware_marks` with your platform now automatically get the correct markers and `cards_{n}` filtering.
|
||||
- Implement a marker function (`xxx_marks`)
|
||||
- Plug into `hardware_marks`
|
||||
- You're done: tests using `@hardware_test` or `hardware_marks` with your platform now automatically get the correct markers, distribution, and isolation!
|
||||
|
||||
See code in `vllm-omni/tests/helpers/mark.py` for existing examples (`_cuda_marks`, `_rocm_marks`, `_npu_marks`).
|
||||
See code in `vllm-omni/tests/helpers/mark.py` for existing examples (`cuda_marks`, `rocm_marks`, `npu_marks`).
|
||||
|
||||
## Test case style
|
||||
|
||||
|
||||
@@ -438,7 +438,7 @@ selects the scheduler and worker family:
|
||||
Key topology fields belong to `StagePipelineConfig`:
|
||||
|
||||
| Field | Description |
|
||||
| ------- | ------------- |
|
||||
|-------|-------------|
|
||||
| `model_stage` | Logical stage name (`ar_stage`, `decoder`, etc.). |
|
||||
| `execution_type` | Runtime family for this stage. |
|
||||
| `input_sources` | Upstream stage IDs that provide input. |
|
||||
@@ -960,7 +960,7 @@ Common failures and fixes:
|
||||
| `check-forbidden-imports` | Stdlib `re`/`base64`, pickle, Hugging Face Hub API, or direct Triton/TileLang | `import regex as re` and `pybase64`; do not grow the allowlist without review |
|
||||
| `check-torch-cuda-call` | New `torch.cuda.*` call site | Use `current_omni_platform`; do not grow `ALLOWED_FILES` without review |
|
||||
| `check-tts-adapter-migration` | New `_tts_model_type` branch in `serving_speech.py` | Put logic in `tts_adapters/`; lower `MAX_MODEL_TYPE_BRANCHES` when removing branches |
|
||||
| `check-mark` | Test file missing level/hardware mark, or direct `pytest.mark.H100` | Add `core_model`/… plus `cpu`/`cuda`/`hardware_test(`; SKU marks only via helpers |
|
||||
| `check-test-ci-coverage` | Test file missing level or hardware mark | Add `core_model`/… plus `cpu`/`cuda`/`hardware_test(` |
|
||||
| `shellcheck` | Native `shellcheck` missing, or script warning | Install via apt/dnf/brew (or `shellcheck.exe` on Windows). See [Linting](../README.md#linting) |
|
||||
| `mypy-3.10` / `markdownlint-cli2` / `check-buildkite` | Types, docs markdown, or Buildkite YAML | See [Linting](../README.md#linting) |
|
||||
|
||||
|
||||
@@ -322,15 +322,6 @@ Each 1-second window records:
|
||||
|
||||
On shutdown the server also logs a short summary (`loop_active_pct`, per-replica queue averages/maxima).
|
||||
|
||||
`loop_idle` and `loop_active` count loop iterations, so their absolute scale is
|
||||
tied to which orchestration loop is running. Under the default poll loop an idle
|
||||
orchestrator records roughly one iteration per millisecond. Under the
|
||||
event-driven loop (`VLLM_OMNI_EVENT_DRIVEN_ORCH=1`, see
|
||||
[Speech API](../serving/speech_api.md#orchestration-loop-experimental)) an idle
|
||||
orchestrator wakes only once per 0.5 s reconcile timeout, while a busy one still
|
||||
records one iteration per routed output. Window counts and the `loop_active_pct`
|
||||
summary are therefore not comparable across the two modes.
|
||||
|
||||
### Relationship to other diagnostics
|
||||
|
||||
This monitor is intentionally separate from the existing profiling tools:
|
||||
|
||||
@@ -170,10 +170,6 @@ because the user-visible output is diffusion media rather than a token stream.
|
||||
The
|
||||
[MiniMax-H3 recipe](https://github.com/vllm-project/vllm-omni/blob/main/recipes/MiniMaxAI/MiniMax-H3.md)
|
||||
contains the hardware-specific profiles and warmup requirements.
|
||||
The internal, opt-in
|
||||
[temporal VAE chunk callback](feature/minimax_h3_temporal_vae_chunks.md)
|
||||
documents the narrower released-source contract for publishing committed video
|
||||
frames during decode. It does not change the complete-output serving topology.
|
||||
|
||||
### Cosmos3: unified MoT reasoner and generator
|
||||
|
||||
|
||||
@@ -63,7 +63,7 @@ vLLM-omni provides `CFGParallelMixin` that encapsulates all CFG parallel logic.
|
||||
|
||||
### N-Branch CFG (3+ branches)
|
||||
|
||||
Some models require more than 2 CFG branches. For example, Bagel and OmniGen2 use 3 branches.
|
||||
Some models require more than 2 CFG branches. For example, Bagel and OmniGen2 use 3 branches, DreamID Omni uses 4 branches.
|
||||
|
||||
`predict_noise_with_multi_branch_cfg()` handles these by automatically dispatching N branches across M GPUs using round-robin (rule: branch `i` → rank `i % M`):
|
||||
|
||||
|
||||
@@ -14,12 +14,12 @@ specified in the [Host Weight Runtime module design](../module/host_weight_runti
|
||||
|
||||
## Status
|
||||
|
||||
The implementation provides contracts and a CPU local-filesystem store. The
|
||||
diffusion consumer additionally defines typed,
|
||||
The first implementation provides contracts and a CPU local-filesystem store.
|
||||
The initial diffusion consumer contract additionally defines typed,
|
||||
representation-independent final-layout identity/restoration mechanics plus a
|
||||
concrete BF16-with-preserved-FP32 policy for MiniMax H3 and
|
||||
`black-forest-labs/FLUX.2-klein-4B`. The opt-in no-AllGather DLO integration
|
||||
selects, publishes, restores, and transfers these artifacts.
|
||||
concrete BF16-with-preserved-FP32 policy for MiniMax H3. It remains
|
||||
library-only: no loader or DLO path selects, publishes, restores, or transports
|
||||
that artifact yet.
|
||||
|
||||
V1 includes:
|
||||
|
||||
@@ -33,12 +33,13 @@ V1 includes:
|
||||
|
||||
V1 does not include:
|
||||
|
||||
- default-on activation or consumers outside no-AllGather DLO;
|
||||
- online FP8, quantized, merged-adaptation, or additional model producers;
|
||||
- HWR interaction with DLO AllGather;
|
||||
- a public CLI, loader activation, or default-on model integration;
|
||||
- lease handoff to DLO or another transport consumer;
|
||||
- FP8, quantized, merged-adaptation, or additional model producers;
|
||||
- CUDA registration, pinned staging, H2D scheduling, or GPU kernels;
|
||||
- a remote artifact provider or cross-node coordination;
|
||||
- automatic eviction; or
|
||||
- a change to DLO collective or execution behavior.
|
||||
- a change to DLO AllGather or no-AllGather behavior.
|
||||
|
||||
## Motivation and use cases
|
||||
|
||||
@@ -254,8 +255,7 @@ The contract is intentionally separate from loader activation:
|
||||
- `FinalLayoutBF16Producer` accepts only the matching identity context and a
|
||||
finalized CPU model. It is `POST_LOAD_ONLY` and `SINGLE_PROCESS` per exact TP
|
||||
coordinate. Its BF16 policy preserves model-declared FP32 parameters and
|
||||
buffers, revalidates MiniMax H3 mixed-precision invariants, and revalidates
|
||||
FLUX.2-klein's two block stacks, packed QKV mapping, and BF16 base layout.
|
||||
buffers and revalidates MiniMax H3 mixed-precision invariants.
|
||||
|
||||
Other representations reuse source identity, typed parallel identity, tensor
|
||||
ownership, and exact restoration only when their policy proves those semantics.
|
||||
|
||||
@@ -1,260 +0,0 @@
|
||||
# MiniMax H3 temporal VAE chunk callback
|
||||
|
||||
!!! warning "Experimental internal capability"
|
||||
|
||||
This feature is an opt-in model integration seam. The current MiniMax H3
|
||||
pipeline still uses complete-output VAE decode, and no CLI option, serving
|
||||
path, transport, or MP4 sink enables temporal chunks by default.
|
||||
|
||||
## Status and scope
|
||||
|
||||
MiniMax H3 decodes video latents in temporal windows, but the released VAE
|
||||
normally exposes only the complete reconstructed video. The temporal callback
|
||||
publishes frames after the released decoder has completed overlap blending and
|
||||
padding removal. This gives a future runtime consumer a stable point at which
|
||||
it can schedule representation conversion, D2H transfer, and encoding while
|
||||
the VAE continues decoding later windows.
|
||||
|
||||
The implementation remains model-local:
|
||||
|
||||
- `vae.py` owns VAE topology and the thin
|
||||
`decode_latent_with_chunks()` facade;
|
||||
- `chunked_decode.py` owns the normalized callback contract, metadata
|
||||
validation, and fixed VAE-group rendezvous points; and
|
||||
- `temporal_chunks.py` owns the source-gated compatibility planner and the
|
||||
released temporal assembly control flow.
|
||||
|
||||
Crop policy, typed media, transport, backpressure, encoding, and public output
|
||||
remain runtime responsibilities. The model code does not schedule requests or
|
||||
select a presentation format.
|
||||
|
||||
## Execution flow
|
||||
|
||||
The default and opt-in paths are intentionally separate:
|
||||
|
||||
```text
|
||||
Default:
|
||||
latent -> MiniMaxH3VideoVAE.decode_latent() -> complete RGB video
|
||||
|
||||
Opt-in producer seam:
|
||||
latent
|
||||
-> MiniMaxH3VideoVAE.decode_latent_with_chunks()
|
||||
-> group-local preflight and callback-owner validation
|
||||
-> released temporal decode / overlap blending / padding removal
|
||||
-> borrowed decoder-domain chunk
|
||||
-> normalize + clone into owned RGB float32 BCTHW [0, 1]
|
||||
-> synchronous callback
|
||||
-> complete RGB video is still returned
|
||||
```
|
||||
|
||||
Calling `decode_latent_with_chunks()` is the opt-in. Supplying no callback on a
|
||||
VAE peer does not select the default decoder; every rank in a VAE patch group
|
||||
must enter the chunked method, and only the output rank receives the callback.
|
||||
|
||||
## Producer contract
|
||||
|
||||
The normalized callback has this structural signature:
|
||||
|
||||
```python
|
||||
def callback(
|
||||
frames: torch.Tensor,
|
||||
/,
|
||||
*,
|
||||
chunk_index: int,
|
||||
total_chunks: int,
|
||||
frame_start: int,
|
||||
is_final: bool,
|
||||
) -> None:
|
||||
...
|
||||
```
|
||||
|
||||
For a successful decode:
|
||||
|
||||
- `frames` is owned, contiguous RGB float32 in `BCTHW` layout and `[0, 1]`;
|
||||
- callback storage does not alias the returned complete video;
|
||||
- the consumer may retain or mutate a chunk;
|
||||
- `chunk_index` is zero-based, contiguous, and strictly increasing;
|
||||
- `total_chunks` is positive and stable for the decode;
|
||||
- `frame_start` equals the end of the previous published range;
|
||||
- chunks are nonempty and contain only temporally committed frames;
|
||||
- exactly one chunk has `is_final=True`; and
|
||||
- concatenating chunks along time exactly reconstructs the returned video.
|
||||
|
||||
The callback runs synchronously on the producer's current device stream. A
|
||||
consumer that reads the tensor on another stream must establish ordering, for
|
||||
example with an event, and retain the allocation for that stream's lifetime.
|
||||
A slow callback also blocks VAE production. An overlap-oriented runtime
|
||||
consumer should therefore acquire bounded capacity, enqueue side-stream work,
|
||||
and return promptly instead of encoding frames inside the callback.
|
||||
|
||||
Chunks retain the decoded VAE canvas dimensions. A future runtime integration
|
||||
must apply the same requested-height and requested-width crop as the complete
|
||||
MiniMax H3 pipeline before declaring the media transport-ready.
|
||||
|
||||
The raw compatibility callback is deliberately narrower. It receives borrowed
|
||||
decoder-domain tensors and must not retain or mutate them. The coordinator is
|
||||
the ownership boundary that normalizes and clones raw chunks before invoking
|
||||
the public callback.
|
||||
|
||||
## Temporal planning and terminal publication
|
||||
|
||||
The released configuration advances five latent tokens per temporal window and
|
||||
includes a two-token overlap, so each `_adaptive_decode` input normally contains
|
||||
seven tokens. After temporal scaling, pre-padding removal, and five-frame
|
||||
overlap blending, the normal committed unit is 17 output frames. The final
|
||||
remainder depends on the requested frame plan.
|
||||
|
||||
Representative released frame plans include:
|
||||
|
||||
| Output frames | Published chunks |
|
||||
| ---: | --- |
|
||||
| 124 | `7 x 17 + 5` |
|
||||
| 209 | `12 x 17 + 5` |
|
||||
| 243 | `14 x 17 + 5` |
|
||||
| 362 | `21 x 17 + 5` |
|
||||
|
||||
The terminal callback is held until the complete frame plan passes its logical
|
||||
frame, padding, written-frame, chunk-count, and finality checks. Earlier chunks
|
||||
are semantically stable, but they do not imply that the overall request has
|
||||
succeeded; a consumer must discard or abort partial output if a later decoder
|
||||
failure terminates the request.
|
||||
|
||||
## Source and configuration gate
|
||||
|
||||
The released VAE has no public temporal-callback API. vLLM-Omni therefore owns
|
||||
a compatibility implementation for one exact released code contract. The
|
||||
reference revision identifies its provenance, but runtime acceptance is based
|
||||
on source content, configuration, and structure rather than repository or
|
||||
weight metadata:
|
||||
|
||||
- reference Hugging Face revision, not independently checked at runtime:
|
||||
`42ed227ee7df40d41602854ae760620d6eb651fe`;
|
||||
- sorted 15-file top-level `video_vae/*.py` manifest (excluding
|
||||
`__init__.py`) SHA-256:
|
||||
`30e64afbaa940696bf8bfe2c86003a4b1666c84b22fa3aa12403a3e6a03f705d`;
|
||||
- clip length `17`;
|
||||
- token chunk, overlap, and drop `5`, `2`, and `3`;
|
||||
- temporal ratio `4`;
|
||||
- frame pre-padding and overlap `3` and `5`; and
|
||||
- no isolated first or last frame.
|
||||
|
||||
The adapter does not verify Hugging Face revision metadata, repository identity,
|
||||
or model weights. The manifest pin detects Python compatibility drift; it is
|
||||
not a security boundary for remote-code execution. The source manifest, listed
|
||||
temporal attributes, structural contract, plan, and dtype are validated before
|
||||
the first `_adaptive_decode` call. A mismatch disables only the opt-in chunk path.
|
||||
`decode_latent()` remains available and performs no source fingerprint or chunk
|
||||
rendezvous.
|
||||
|
||||
## Supported models
|
||||
|
||||
Support in this table means the internal producer contract above, not public
|
||||
serving or end-to-end transport support.
|
||||
|
||||
| Model or component | Source contract | Status | Validation |
|
||||
| --- | --- | --- | --- |
|
||||
| MiniMax H3 FL2VA video VAE | Released manifest/listed temporal attributes above (reference `42ed227e`) | Supported, opt-in | Same-environment full-output parity and L20X/B300 feasibility measured on the implementation through PR commit `60e9bbad` |
|
||||
| MiniMax H3 Ref2VA video VAE | Same released manifest/listed temporal attributes | Source-compatible; not separately benchmarked | Ref2VA and FL2VA use byte-identical video-VAE source and configuration in the reference snapshot |
|
||||
| MiniMax H3 with a different Python manifest or listed temporal attribute | Different enforced contract | Unsupported for chunk mode | Fails closed before decoder collectives; complete decode remains available |
|
||||
| Same manifest/temporal attributes with different weights or other component metadata | Not checked by this gate | Not independently qualified | The adapter does not inspect weights, repository identity, revision metadata, or arbitrary component configuration |
|
||||
| Other video models | Any | Unsupported | No generic temporal VAE producer ABI is declared by this feature |
|
||||
|
||||
## Feature compatibility
|
||||
|
||||
| Dimension or combination | Status | Constraint |
|
||||
| --- | --- | --- |
|
||||
| Existing complete-output decode | Supported and default | No source check, callback object, or callback rendezvous |
|
||||
| Explicit chunk decode | Supported internally | Caller invokes `decode_latent_with_chunks()` directly |
|
||||
| VAE patch parallel size 1 | Supported by the model seam | Caller supplies the callback only on the request output owner; no WORLD rendezvous is added |
|
||||
| VAE patch parallel size equal to the full DiT group | Supported when data parallel size is 1 | Every VAE-group rank calls; exactly group rank 0 owns the callback |
|
||||
| Partial VAE patch subgroup | Unsupported | MiniMax H3 currently accepts size 1 or the full DiT group only |
|
||||
| Data parallel size greater than 1 with VAE patch parallel size 1 | Model seam only | Per-replica callback ownership is possible, but no serving/runtime consumer is wired |
|
||||
| Data parallel size greater than 1 with VAE patch parallel size greater than 1 | Unsupported | Fails before decoder collectives because no replica-local VAE patch group is defined |
|
||||
| Too few spatial tiles for the VAE group | Correctness fallback | Decodes rank-locally and restores native tiling state; slower than patch parallel decode |
|
||||
| Device memory and full-output assembly | Not reduced or bounded | The compatibility path still assembles and returns the full decode while allocating owned callback chunks; no peak-HBM reduction is claimed |
|
||||
| DiT cache, attention, TP, SP, HSDP, LoRA, or quantization features | No direct callback dependency | This seam runs after denoising and does not independently qualify combinations beyond the MiniMax H3 model's own matrix |
|
||||
| Layerwise or distributed component offload | Not separately validated | Component residency must place the video VAE before decode |
|
||||
| Step execution or request batching | Not integrated | No pipeline caller binds a runtime chunk consumer |
|
||||
| Requested-size crop and device uint8 preparation | Not implemented here | Runtime-owned follow-up |
|
||||
| Pinned D2H, SHM/ZMQ, or cross-process transport | Not implemented here | Runtime-owned follow-up |
|
||||
| Complete MP4 or fragmented MP4 sink | Not implemented here | Existing serving behavior still waits for complete output |
|
||||
| Audio VAE chunks and incremental A/V composition | Not supported | Video chunks do not imply audio or request finality |
|
||||
| Cancellation and multi-output lease ownership | Not implemented here | Requires a bounded runtime lifecycle |
|
||||
| NVIDIA CUDA | Feasibility evidence | L20X and B300 measurements apply to the implementation through `60e9bbad`; the current coordination refactor was not reprofiled |
|
||||
| AMD ROCm | Not validated for this feature | Base MiniMax H3 support is documented separately and does not imply temporal-chunk validation |
|
||||
| Ascend, Intel, or other accelerators | No feature support claim | Consult the model support table independently; this design adds no platform qualification |
|
||||
|
||||
## Distributed and failure semantics
|
||||
|
||||
When VAE patch parallelism is active, all VAE-group ranks run the same
|
||||
preflight before entering checkpoint collectives. Source/planner errors and the
|
||||
resolved temporal concat dtype are reconciled at fixed group rendezvous points.
|
||||
|
||||
Callback ownership is also a group decision. Rank `r` contributes `r + 1` when
|
||||
it owns a callback and zero otherwise. A group SUM equals one only when rank 0
|
||||
is the sole owner; every invalid owner configuration therefore fails
|
||||
identically on all ranks before decode.
|
||||
|
||||
Callback exceptions are retained on the output rank. Further publication stops,
|
||||
but every rank completes the same remaining native decoder collective schedule.
|
||||
The original exception is then raised on the owner and
|
||||
`MiniMaxH3ChunkCallbackPeerError` on peers.
|
||||
|
||||
Exceptions raised inside the checkpoint's native distributed decode cannot in
|
||||
general be reconciled by the adapter: a peer may already be blocked inside a
|
||||
checkpoint-owned collective. Resolving such failures requires native collective
|
||||
handling or supervisor-level timeout and worker-group teardown. The process
|
||||
group must not be assumed reusable after such a failure.
|
||||
|
||||
## Validation evidence
|
||||
|
||||
The contract suite covers ordinary and padded frame plans, blend boundaries,
|
||||
owned storage and mutation isolation, source-manifest/listed-temporal-attribute
|
||||
drift, callback failure, and healthy subsequent requests. Real Gloo processes cover
|
||||
rank-asymmetric preflight failures, dtype disagreement, two-rank callback
|
||||
failure propagation, three-rank wrong-owner rejection, insufficient-tile
|
||||
fallback, and group recovery for failures that occur at adapter-owned
|
||||
rendezvous points.
|
||||
|
||||
Four-device L20X and independent four- and eight-device B300 experiments on the
|
||||
implementation through PR commit `60e9bbad` showed exact serial-versus-chunked
|
||||
frame/MP4 parity within each environment and lower VAE-to-complete-MP4 latency
|
||||
when an experimental consumer was attached. Hashes are not claimed to match
|
||||
across different hardware/software environments. The later coordination and
|
||||
module-boundary refactors were revalidated with CPU and real Gloo tests but were
|
||||
not reprofiled on GPUs. These are feasibility measurements, not a promise for
|
||||
default serving, because this change does not contain the transport or encoder
|
||||
consumer. See [RFC #6872](https://github.com/vllm-project/vllm-omni/issues/6872)
|
||||
and [PR #6885](https://github.com/vllm-project/vllm-omni/pull/6885) for the
|
||||
measurement provenance and limitations.
|
||||
|
||||
## Generalization boundary and follow-ups
|
||||
|
||||
This feature does not define a repository-wide `SupportsVideoChunkDecode`
|
||||
interface. H3 temporal blending, Wan causal caches, LTX temporal/spatial halos,
|
||||
and pipeline-level generation chunks have different commit and failure
|
||||
boundaries. A generic producer contract should be extracted only after a second
|
||||
model proves the same semantic event without `Any` arguments or model-name
|
||||
dispatch.
|
||||
|
||||
The reusable future boundary is the emitted, committed video-media chunk, not a
|
||||
universal latent-to-VAE method. It should extend the typed media work in
|
||||
[RFC #6541](https://github.com/vllm-project/vllm-omni/issues/6541) and
|
||||
[PR #6615](https://github.com/vllm-project/vllm-omni/pull/6615), while keeping
|
||||
model-specific halo, cache, padding, blending, distributed, and source-gating
|
||||
logic inside each model adapter.
|
||||
|
||||
Runtime follow-ups own:
|
||||
|
||||
- typed video-plus-audio media and request/output identity;
|
||||
- requested-size crop and device preparation;
|
||||
- bounded leases, backpressure, cancellation, and cleanup;
|
||||
- D2H/IPC transport and observability; and
|
||||
- complete-MP4 and fragmented-MP4 consumers.
|
||||
|
||||
## Related documentation
|
||||
|
||||
- [Diffusion model integration](../module/diffusion/diffusion_model_integration.md)
|
||||
- [VAE patch parallelism](vae_parallel.md)
|
||||
- [Async diffusion output](async_diffusion_output.md)
|
||||
- [MiniMax H3 recipe](https://github.com/vllm-project/vllm-omni/blob/main/recipes/MiniMaxAI/MiniMax-H3.md)
|
||||
@@ -26,11 +26,10 @@ Legend: ✅ supported, ⚠️ compatibility path or limited validation, ❌ unsu
|
||||
| **TP > 1** | ⚠️ Ordinary TP-aware loader only; no direct checkpoint mmap. | ⚠️ Ordinary TP-aware loader only; no direct checkpoint mmap. |
|
||||
| **HSDP** | ❌ Rejected to avoid double-sharding parameters. | ⚠️ Limited end-to-end coverage. |
|
||||
| **Per-tensor online FP8 linears** | ✅ Ordinary loader finalizes weights and scales before DLO sharding. | ✅ Ordinary loader retains complete rank-local tensors. |
|
||||
| **Online INT8 linears** | ✅ Ordinary loader finalizes the int8 weight (contiguous or transposed view) and fp32 scale before DLO sharding. | ✅ Ordinary loader retains complete rank-local tensors. |
|
||||
| **Online MXFP8 linears** | ✅ Ordinary loader finalizes the fp8 weight (contiguous) and e8m0 block scale (contiguous or transposed view) before DLO sharding. | ✅ Ordinary loader retains complete rank-local tensors. |
|
||||
| **Other online quantization methods** | ❌ Rejected until runtime packing and scale layouts are validated. | ⚠️ Allowed through the ordinary loader; validation is method-specific. |
|
||||
| **Model-level or standard layerwise CPU offload** | ❌ Disabled because DLO takes priority. | ❌ Disabled because DLO takes priority. |
|
||||
| **Resident leading layers** | ❌ Rejected. | ✅ Requires eligible resident paths in the model's `OffloadPlan`. |
|
||||
| **Auxiliary component policy** | ✅ Encoders/VAEs may remain resident. | ✅ Encoders/VAEs may remain resident. |
|
||||
|
||||
See [Parallelism compatibility](#parallelism-compatibility) and
|
||||
[Request and loading constraints](#request-and-loading-constraints) for the
|
||||
@@ -222,15 +221,9 @@ model revision and parallel layout with `required`. TP coordinates have distinct
|
||||
identities, while equivalent DP replicas share them.
|
||||
|
||||
The current producer/consumer boundary is the model-declared final-layout BF16
|
||||
contract (MiniMax H3 and `black-forest-labs/FLUX.2-klein-4B`). The FLUX.2-klein
|
||||
contract covers both transformer block stacks, constructor-stable packed QKV
|
||||
mapping state, BF16 parameters, and any persistent loader-owned `beta`/`eps`
|
||||
buffers. The shared HWR machinery supports ordinary-loader final layouts for
|
||||
TP1 and TP2 rank identities plus SP layout identities. FLUX.2-klein evidence
|
||||
in this change is TP1-only; its TP2/SP layouts rely on the existing
|
||||
per-coordinate identity mechanics and remain unmeasured. Online quantization,
|
||||
HSDP, LoRA/adapted weights, and non-default load formats remain ineligible.
|
||||
HWR mode
|
||||
contract (MiniMax H3 today). It supports ordinary-loader final layouts for TP1
|
||||
and TP2 rank identities plus SP layout identities; online quantization, HSDP,
|
||||
LoRA/adapted weights, and non-default load formats remain ineligible. HWR mode
|
||||
`disabled`, DLO-disabled, and DLO AllGather configurations stop before source
|
||||
identity or store construction and retain the existing checkpoint-mmap or
|
||||
ordinary-loader path.
|
||||
@@ -427,6 +420,22 @@ failure.
|
||||
- **HSDP + DP or TP:** rejected independently by the diffusion parallel
|
||||
configuration.
|
||||
|
||||
### Auxiliary component policy
|
||||
|
||||
`dlo_offload_components` maps discovered encoder/VAE paths to booleans. A
|
||||
component set to `False` is materialized on the target device once and is not
|
||||
entered through the model's manual offload lifecycle. A `True` value permits
|
||||
the existing `OffloadPlan.on_demand_component_paths` and encoder block-hook
|
||||
behavior; it does not add support where the model declares none. The optional
|
||||
`default` key handles unmatched auxiliary components, and an empty map
|
||||
preserves the current policy.
|
||||
|
||||
The policy is resolved after component discovery and before DLO mutates DiT
|
||||
storage. Unknown names, including DiT paths, are rejected. DLO continues to
|
||||
own DiT streaming regardless of this map, which avoids an invalid state where
|
||||
the loader has skipped ordinary DiT materialization but the backend is asked
|
||||
to keep that DiT resident.
|
||||
|
||||
## Request and loading constraints
|
||||
|
||||
AllGather DP multi-concurrency requires:
|
||||
@@ -438,17 +447,22 @@ AllGather DP multi-concurrency requires:
|
||||
The no-AllGather path does not impose these DLO-specific synchronized-wave
|
||||
requirements.
|
||||
|
||||
MiniMax-H3 currently builds one global encoder TP group and broadcasts the
|
||||
resulting conditioning from global rank 0. Therefore an encoder TP group that
|
||||
spans multiple DP replicas is not valid for independent-prompt AllGather
|
||||
waves. Exact-recipe DP2/TP2 and DP2/SP2 experiments with encoder TP4 completed
|
||||
the collective schedule, but two distinct prompts with the same seed produced
|
||||
byte-identical video and audio in every measured wave. Enabling or disabling
|
||||
encoder offload produced paired-identical results, so this is a pre-existing
|
||||
model topology limitation rather than component-policy behavior.
|
||||
|
||||
Direct checkpoint mmap can back either transfer path. It is currently limited
|
||||
to proven TP1, non-HSDP, non-online-quantized layouts. Other layouts use the
|
||||
ordinary loader. Per-tensor online FP8, online INT8, and online MXFP8 linears
|
||||
can use DLO AllGather after the ordinary loader finalizes their runtime
|
||||
weights and scales; DLO then shards and reconstructs those tensors with their
|
||||
recorded layouts (they all keep plain 1-byte dtypes over ordinary strided
|
||||
views: online INT8 has an int8 weight plus fp32 scale, online MXFP8 has an
|
||||
fp8 weight plus e8m0 block scale, each with a contiguous or transposed-view
|
||||
finalized tensor, all covered by the same physical-order packing as online
|
||||
FP8). Other online methods must use `--dlo-no-use-allgather` or disable
|
||||
online quantization until their runtime layouts are validated.
|
||||
ordinary loader. Per-tensor online FP8 linears can use DLO AllGather after the
|
||||
ordinary loader finalizes their runtime weights and scales; DLO then shards and
|
||||
reconstructs those tensors with their recorded layouts. Other online methods
|
||||
must use `--dlo-no-use-allgather` or disable online quantization until their
|
||||
runtime layouts are validated.
|
||||
|
||||
The Host Weight Runtime representation, publication, and no-AllGather consumer
|
||||
contracts are merged; see
|
||||
@@ -465,8 +479,8 @@ Current source-level validation includes:
|
||||
- HSDP + DLO without AllGather acceptance at configuration level;
|
||||
- loader preflight fallback for TP, HSDP, online quantization, unknown custom
|
||||
loaders, missing keys, and shape/dtype mismatches;
|
||||
- ordinary-loader fallback for per-tensor online FP8, online INT8, and online
|
||||
MXFP8 linears followed by DLO sharding of finalized weights and scales;
|
||||
- ordinary-loader fallback for per-tensor online FP8 linears followed by DLO
|
||||
sharding of finalized weights and scales;
|
||||
- exact loader-to-backend plan transfer and ordinary-loader fallback;
|
||||
- ordered publication hashing, overlapped durability, unordered parallel
|
||||
checksum fallback, and node-local source-digest reuse/invalidation;
|
||||
@@ -506,6 +520,9 @@ AllGather, because DP replicas did not share checkpoint-backed runtime
|
||||
weights. This is a functional and memory smoke test, not a production-quality
|
||||
performance or output-quality benchmark.
|
||||
|
||||
These B300 DP waves intentionally reused the same prompt and seed, so they do
|
||||
not establish independent-prompt isolation across DP replicas.
|
||||
|
||||
### Host-memory measurement
|
||||
|
||||
A two-worker MiniMax-H3 FL2VA measurement on one L20X node compared the
|
||||
|
||||
@@ -22,60 +22,59 @@ We use **Qwen-Image** as the reference implementation for decode parallel, and *
|
||||
|
||||
**VAE Patch Parallelism** is an acceleration technique for both **encoding** and **decoding**. Instead of processing the entire tensor at once, the tensor is:
|
||||
|
||||
- Split into multiple spatial tiles
|
||||
+ Split into multiple spatial tiles
|
||||
|
||||
- Distributed across multiple ranks
|
||||
+ Distributed across multiple ranks
|
||||
|
||||
- Encoded/Decoded in parallel
|
||||
+ Encoded/Decoded in parallel
|
||||
|
||||
- Merged to reconstruct the final output
|
||||
+ Merged to reconstruct the final output
|
||||
|
||||
This approach:
|
||||
|
||||
- Distributes computation across multiple devices
|
||||
+ Distributes computation across multiple devices
|
||||
|
||||
- Reduces peak memory usage per device
|
||||
+ Reduces peak memory usage per device
|
||||
|
||||
- Accelerates encoding/decoding latency
|
||||
+ Accelerates encoding/decoding latency
|
||||
|
||||
### When to Use Encode vs Decode Parallel
|
||||
|
||||
| Operation | Use Case | Example |
|
||||
| ------------------- | ---------------------------- | --------------------------------- |
|
||||
| **Decode Parallel** | Text-to-Image, Text-to-Video | Latent → Image/Video |
|
||||
| **Encode Parallel** | Image-to-Video (I2V) | Image → Latent (for conditioning) |
|
||||
| Operation | Use Case | Example |
|
||||
|-----------|----------|---------|
|
||||
| **Decode Parallel** | Text-to-Image, Text-to-Video | Latent → Image/Video |
|
||||
| **Encode Parallel** | Image-to-Video (I2V) | Image → Latent (for conditioning) |
|
||||
|
||||
### Architecture
|
||||
|
||||
We introduce **DistributedVaeExecutor** as the core component responsible for distributed VAE encoding/decoding.
|
||||
|
||||
The executor is model-agnostic and accepts three function parameters:
|
||||
|
||||
- split – Partition the latent into tiles
|
||||
+ split – Partition the latent into tiles
|
||||
|
||||
- exec – Decode a single tile
|
||||
+ exec – Decode a single tile
|
||||
|
||||
- merge – Combine decoded tiles into the final output
|
||||
+ merge – Combine decoded tiles into the final output
|
||||
|
||||
#### Execution Flow
|
||||
|
||||
- Call split(z) to generate a list of TileTask and a GridSpec
|
||||
+ Call split(z) to generate a list of TileTask and a GridSpec
|
||||
|
||||
- Dispatch tasks across ranks using workload-based balancing
|
||||
+ Dispatch tasks across ranks using workload-based balancing
|
||||
|
||||
- Each rank executes exec(task) on its assigned tiles
|
||||
+ Each rank executes exec(task) on its assigned tiles
|
||||
|
||||
- Gather decoded tile results to rank 0
|
||||
+ Gather decoded tile results to rank 0
|
||||
|
||||
- Rank 0 performs merge(...)
|
||||
+ Rank 0 performs merge(...)
|
||||
|
||||
- (Optional) Broadcast final result to all ranks
|
||||
+ (Optional) Broadcast final result to all ranks
|
||||
|
||||
This design separates:
|
||||
|
||||
- Distributed execution logic
|
||||
+ Distributed execution logic
|
||||
|
||||
- Model-specific tiling and merging logic
|
||||
+ Model-specific tiling and merging logic
|
||||
|
||||
#### Why split / exec / merge is necessary?
|
||||
|
||||
@@ -83,23 +82,23 @@ The latent tensor cannot be arbitrarily partitioned.
|
||||
|
||||
During decoding:
|
||||
|
||||
- Each output pixel may depend on neighboring pixels
|
||||
+ Each output pixel may depend on neighboring pixels
|
||||
|
||||
- The receptive field is model-dependent
|
||||
+ The receptive field is model-dependent
|
||||
|
||||
Therefore:
|
||||
|
||||
- Tiles must include overlap
|
||||
+ Tiles must include overlap
|
||||
|
||||
- Merge must perform blending to avoid seams
|
||||
+ Merge must perform blending to avoid seams
|
||||
|
||||
## Step-by-Step Implementation (Decode)
|
||||
|
||||
### Step 1: Implement DistributedAutoencoderKLQwenImage
|
||||
|
||||
`QwenImagePipeline` use `AutoencoderKLQwenImage` for vae, so implement a distributed version:
|
||||
|
||||
```python
|
||||
|
||||
```
|
||||
class DistributedAutoencoderKLQwenImage(AutoencoderKLQwenImage, DistributedVaeMixin):
|
||||
@classmethod
|
||||
def from_pretrained(cls, *args: Any, **kwargs: Any):
|
||||
@@ -107,35 +106,28 @@ class DistributedAutoencoderKLQwenImage(AutoencoderKLQwenImage, DistributedVaeMi
|
||||
model.init_distributed()
|
||||
return model
|
||||
```
|
||||
|
||||
**Key points**:
|
||||
|
||||
- Inherit both AutoencoderKLQwenImage and DistributedVaeMixin
|
||||
- Call init_distributed() after loading weights
|
||||
+ Inherit both AutoencoderKLQwenImage and DistributedVaeMixin
|
||||
+ Call init_distributed() after loading weights
|
||||
|
||||
### Step 2: Implement split/exec/merge
|
||||
|
||||
Reuse `AutoencoderKLQwenImage.tiled_decode` logic and divide it into three stages. And we need return tiles with `GridSpec` and `TileTask`:
|
||||
|
||||
```python
|
||||
```
|
||||
class GridSpec:
|
||||
split_dims: tuple[int, ...] # Tensor dimensions being split (e.g., (2, 3) for (B, C, H, W))
|
||||
grid_shape: tuple[int, ...] # Tile grid layout (num_rows, num_cols)
|
||||
tile_spec: dict = field(default_factory=dict) # Metadata required for merging
|
||||
output_dtype: torch.dtype | None = None # Final output dtype
|
||||
```
|
||||
|
||||
```python
|
||||
```
|
||||
class TileTask:
|
||||
tile_id: int # task id
|
||||
grid_coord: tuple[int, ...] # Tile position in grid
|
||||
tensor: torch.Tensor | list[torch.Tensor] # The tile tensor
|
||||
workload: int | float = 1 # Used for load balancing (e.g., tile area)
|
||||
```
|
||||
|
||||
And tiled base split/exec/merge as follow:
|
||||
|
||||
```python
|
||||
```
|
||||
def tile_split(self, z: torch.Tensor) -> tuple[list[TileTask], GridSpec]:
|
||||
# mostly copy from AutoencoderKL
|
||||
_, _, num_frames, height, width = z.shape
|
||||
@@ -218,14 +210,11 @@ def tile_merge(self, coord_tensor_map: dict[tuple[int, ...], torch.Tensor], grid
|
||||
```
|
||||
|
||||
### Step 3: Override tiled_decode
|
||||
|
||||
We need to override tiled_decode, the main logic is:
|
||||
|
||||
- check distributed is enabled
|
||||
- select split/exec/merge
|
||||
- Invoke self.distributed_executor.execute to decode
|
||||
|
||||
```python
|
||||
+ check distributed is enabled
|
||||
+ select split/exec/merge
|
||||
+ Invoke self.distributed_executor.execute to decode
|
||||
```
|
||||
def tiled_decode(self, z: torch.Tensor, return_dict: bool = True):
|
||||
if not self.is_distributed_enabled():
|
||||
return super().tiled_decode(z, return_dict=return_dict)
|
||||
@@ -241,14 +230,11 @@ def tiled_decode(self, z: torch.Tensor, return_dict: bool = True):
|
||||
|
||||
return DecoderOutput(sample=result)
|
||||
```
|
||||
|
||||
`broadcast_result` is set to True or False depending on the model; when enabled, the result will be used even on ranks other than 0.
|
||||
|
||||
### Step 4: Modify Pipeline
|
||||
|
||||
Change vae model from AutoencoderKLQwenImage to DistributedAutoencoderKLQwenImage
|
||||
|
||||
```diff
|
||||
```
|
||||
class YourModelPipeline(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
@@ -273,8 +259,8 @@ For models that require VAE encoding (e.g., Image-to-Video), you can also parall
|
||||
|
||||
Similar to decode, split the input tensor into tiles. Key considerations:
|
||||
|
||||
- **Patchify handling**: If the model uses `patch_size`, scale tile parameters accordingly
|
||||
- **Temporal chunking**: Video VAEs may have temporal compression (e.g., 4x)
|
||||
+ **Patchify handling**: If the model uses `patch_size`, scale tile parameters accordingly
|
||||
+ **Temporal chunking**: Video VAEs may have temporal compression (e.g., 4x)
|
||||
|
||||
```python
|
||||
def encode_tile_split(self, x: torch.Tensor) -> tuple[list[TileTask], GridSpec]:
|
||||
@@ -420,28 +406,23 @@ def tiled_encode(self, x: torch.Tensor) -> torch.Tensor:
|
||||
**Key differences from decode parallel:**
|
||||
|
||||
| Aspect | Decode Parallel | Encode Parallel |
|
||||
| -------- | ----------------- | ----------------- |
|
||||
|--------|-----------------|-----------------|
|
||||
| `broadcast_result` | Often `False` (only rank 0 needs output) | `True` (all ranks need latents for diffusion) |
|
||||
| Patchify | Applied in merge (unpatchify) | Handled by parent `_encode()` before `tiled_encode()` |
|
||||
| Temporal chunking | Frame-by-frame | Chunk-based (e.g., 1 + 4n frames) |
|
||||
|
||||
## Testing
|
||||
|
||||
Verify numerical consistency between:
|
||||
+ vae_patch_parallel_size = 1
|
||||
|
||||
- vae_patch_parallel_size = 1
|
||||
- vae_patch_parallel_size = N
|
||||
+ vae_patch_parallel_size = N
|
||||
|
||||
Example:
|
||||
|
||||
```python
|
||||
torch.allclose(output_1, output_n, atol=1e-5)
|
||||
```
|
||||
|
||||
Testing requirements:
|
||||
|
||||
- Fix random seed
|
||||
- Use identical tiling strategy
|
||||
+ Fix random seed
|
||||
+ Use identical tiling strategy
|
||||
|
||||
```python
|
||||
m = Omni(
|
||||
@@ -453,7 +434,6 @@ m = Omni(
|
||||
),
|
||||
)
|
||||
```
|
||||
|
||||
When vae_patch_parallel_size is larger than the DiT world size, it will automatically fall back to using the DiT world size instead.
|
||||
|
||||
## Reference Implementations
|
||||
@@ -461,7 +441,7 @@ When vae_patch_parallel_size is larger than the DiT world size, it will automati
|
||||
Complete examples in the codebase:
|
||||
|
||||
| Model | Path | Decode Parallel | Encode Parallel |
|
||||
| ------- | ------ | ----------------- | ----------------- |
|
||||
|-------|------|-----------------|-----------------|
|
||||
| **Z-Image** | `vllm_omni/diffusion/distributed/autoencoders/autoencoder_kl.py` | ✅ | ❌ |
|
||||
| **Wan2.2** | `vllm_omni/diffusion/distributed/autoencoders/autoencoder_kl_wan.py` | ✅ | ✅ |
|
||||
| **Qwen-Image** | `vllm_omni/diffusion/distributed/autoencoders/autoencoder_kl_qwenimage.py` | ✅ | ❌ |
|
||||
@@ -476,7 +456,7 @@ The tile-parallel executor above assigns independent spatial **tiles** to ranks.
|
||||
### How it differs from tile parallel
|
||||
|
||||
| Aspect | Tile parallel (`"tile"`) | Spatially-sharded (`"spatial_shard_height"`/`"spatial_shard_width"`) |
|
||||
| -------- | -------------------------- | ----------------------------------------------- |
|
||||
|--------|--------------------------|-----------------------------------------------|
|
||||
| Unit of work | Independent overlapping tiles | A single global feature map sharded along H or W |
|
||||
| Cross-rank communication | Gather tiles to rank 0, stitch + blend | Per-conv **halo exchange** of boundary rows/cols (P2P) |
|
||||
| Output assembly | Blend overlapping tiles | All-gather shards on rank 0, trim padding (matches `broadcast_result=False`) |
|
||||
@@ -501,7 +481,7 @@ serve.py (--vae-parallel-mode) / OmniEngineArgs
|
||||
### Notes
|
||||
|
||||
- The decoder is patched **in place** the first time spatial-shard decode runs and is bound to a single split dimension for the lifetime of the VAE instance.
|
||||
- Numerical correctness vs. single-GPU decode is covered by `tests/diffusion/distributed/test_wan_spatial_shard.py::test_spatial_shard_decode_matches_reference` (multi-GPU, nightly `full_model` + `cards_2`).
|
||||
- Numerical correctness vs. single-GPU decode is covered by `tests/diffusion/distributed/test_wan_spatial_shard.py::test_spatial_shard_decode_matches_reference` (multi-GPU, nightly `full_model` + `distributed_cuda`).
|
||||
|
||||
---
|
||||
|
||||
|
||||
@@ -25,10 +25,6 @@ implementation contract; it is not, by itself, a general support claim.
|
||||
- [Automatic Prefix Caching in Omni Models](feature/prefix_caching.md)
|
||||
- [Realtime AR-Diffusion Sessions](feature/realtime_ar_diffusion.md)
|
||||
|
||||
### Model-specific capabilities
|
||||
|
||||
- [MiniMax H3 Temporal VAE Chunks](feature/minimax_h3_temporal_vae_chunks.md)
|
||||
|
||||
### Communication
|
||||
|
||||
#### OmniConnector implementations
|
||||
|
||||
+30
-76
@@ -6,7 +6,7 @@ This document describes how vLLM-Omni exposes Prometheus metrics for multi-stage
|
||||
|
||||
- Expose pipeline-level request and latency metrics that span the full multi-stage execution (orchestrator scope).
|
||||
- Preserve all upstream vLLM per-engine metrics (`vllm:*`) for stages backed by an AR LLM engine, and reshape their `engine` label into `stage` + `replica` so multi-replica deployments gain per-replica visibility automatically.
|
||||
- Expose per-modality SLO metrics that the upstream `vllm:*` families do not capture — audio TTFP / RTF / duration / frames / streaming continuity / silent-loss, plus image and diffusion workload, queueing, execution, and memory signals.
|
||||
- Expose per-modality SLO metrics that the upstream `vllm:*` families do not capture — audio TTFP / RTF / duration / frames / streaming continuity / silent-loss.
|
||||
- Expose per-replica-edge cross-stage transfer metrics so the slack between E2E latency and the sum of per-stage `gen_time` (queueing, serialization, network) becomes attributable.
|
||||
- Keep the metrics collection overhead low enough that it does not regress TTFA or throughput.
|
||||
|
||||
@@ -20,7 +20,7 @@ vLLM's `unregister_vllm_metrics()` function strips every `prometheus_client` col
|
||||
|
||||
### The Problem
|
||||
|
||||
vLLM-Omni runs multiple engine instances (stages × replicas) within a single process, coordinated by an Orchestrator. The pipeline needs its own metrics — aggregate request counts, end-to-end latency across all stages, per-modality SLO signals, and cross-stage transfer attribution — that do not exist in upstream vLLM. All pipeline-level metrics use the `vllm_omni:` prefix to distinguish them from upstream per-engine metrics. At import time (see `vllm_omni/patch.py`) we replace `unregister_vllm_metrics()` with a scoped version that still strips upstream `vllm:*` collectors before each new `PrometheusStatLogger` registers (so multi-engine processes don't crash on duplicate registration), but preserves anything prefixed `vllm_omni:` / `vllm_omni`.
|
||||
vLLM-Omni runs multiple engine instances (stages × replicas) within a single process, coordinated by an Orchestrator. The pipeline needs its own metrics — aggregate request counts, end-to-end latency across all stages, per-modality SLO signals, and cross-stage transfer attribution — that do not exist in upstream vLLM. All pipeline-level metrics use the `vllm:omni_` prefix to distinguish them from upstream per-engine metrics. At import time (see `vllm_omni/patch.py`) we replace `unregister_vllm_metrics()` with a scoped version that still strips upstream `vllm:*` collectors before each new `PrometheusStatLogger` registers (so multi-engine processes don't crash on duplicate registration), but preserves anything prefixed `vllm:omni_` / `vllm_omni`.
|
||||
|
||||
Upstream per-engine metrics retain the `vllm:` prefix but are now registered by `OmniPrometheusStatLogger`, a thin subclass of upstream's `PrometheusStatLogger` that reshapes the single `engine` label into a `stage` + `replica` pair (see "OmniPrometheusStatLogger wrap" below).
|
||||
|
||||
@@ -38,7 +38,7 @@ Upstream per-engine metrics retain the `vllm:` prefix but are now registered by
|
||||
|
|
||||
+--------+--------+--------+--------+--------+
|
||||
| |
|
||||
vllm_omni:* vllm:*
|
||||
vllm:omni_* vllm:*
|
||||
collectors collectors
|
||||
| |
|
||||
+----+--------+ +-----------+ +----------+ +-----------+
|
||||
@@ -58,30 +58,17 @@ Upstream per-engine metrics retain the `vllm:` prefix but are now registered by
|
||||
|
||||
### Data Flow
|
||||
|
||||
There are five independent paths for metric collection.
|
||||
There are four independent paths for metric collection.
|
||||
|
||||
**Path 1: Pipeline-level metrics (`vllm_omni:*`)**
|
||||
**Path 1: Pipeline-level metrics (`vllm:omni_*`)**
|
||||
|
||||
`OmniPrometheusMetrics` registers the Gauge / Counter / Histogram collectors at import time. It is instantiated once per entrypoint, labeled with the model name. The entrypoint calls its methods as requests progress:
|
||||
|
||||
- `set_running(n)` / `set_waiting(n)` — updated after each request completes. The running count comes from `OmniRequestCounter`, a simple counter incremented/decremented by the Orchestrator as it tracks requests. Waiting is derived as `total - running`.
|
||||
- `request_succeeded(e2e_seconds, finished_reason="stop")` — recorded when a request finishes at the final stage. `finished_reason` is extracted from `engine_outputs.outputs[0].finish_reason` (vLLM `CompletionOutput` convention) and increments `vllm_omni:requests_success_total{finished_reason}`.
|
||||
- `request_failed()` — recorded by the cleanup path when a request exits without natural completion. Internally maps to `finished_reason="abort"` on the existing completion Counter.
|
||||
- `inc_requests_failed(reason)` — increments the dedicated failure-attribution Counter once per request. Reasons are normalized to the bounded set `client_abort`, `client_disconnect`, `stage_error`, and `unknown`.
|
||||
- `request_succeeded(e2e_seconds, finished_reason="stop")` — recorded when a request finishes at the final stage. `finished_reason` is extracted from `engine_outputs.outputs[0].finish_reason` (vLLM `CompletionOutput` convention) and increments `vllm:omni_requests_success_total{finished_reason}`.
|
||||
- `request_failed()` — recorded by the cleanup path when a request exits without natural completion. Internally maps to `finished_reason="abort"` so a single Counter family covers both natural and aborted completion.
|
||||
|
||||
**Path 2: Image and diffusion metrics**
|
||||
|
||||
Image and diffusion metrics use two cooperating collectors. `OmniPrometheusMetrics` owns pipeline/stage workload families, while `OmniModalityMetrics` owns per-replica diffusion breakdowns. Finished stage messages are consumed once in `OmniBase._process_single_result`, which prevents replayed terminal messages from incrementing Counters or Histograms twice.
|
||||
|
||||
- `stage_gen_time_s`, `stage_in_queue_s`, image workload, inference-step, and per-stage peak-memory values are emitted from `StageRequestStats` when a stage finishes. `image_pixels` and `image_count_total` are restricted to image outputs; `num_inference_steps` is restricted to diffusion stages.
|
||||
- Diffusion engine timings are accumulated per request by `OrchestratorAggregator.accumulate_diffusion_metrics`. Engine-side millisecond values are converted to seconds before `observe_modality_at_finalize(...)` dispatches the per-replica Histogram observations.
|
||||
- `request_queue_wait_s` is emitted once at final request completion from `pipeline_timings["queue_wait_ms"]`. A present value of `0` is a valid observation; a missing key produces no sample.
|
||||
- `stage_waiting_requests` is the sum of the latest scheduler snapshots for all live replicas in a stage. AR snapshots arrive through `SchedulerStats`; diffusion snapshots ride normal diffusion output metrics. A dead replica's snapshot is removed. Because this deliberately reuses existing message paths, the gauge can retain its last value while a replica produces no output.
|
||||
- `kv_wait_s` measures the AR scheduler interval from entering the KV-transfer-free wait state until extraction acknowledgement. The sample is carried to the Orchestrator through the existing engine output and labeled by connector type. Abort/error/removal paths discard incomplete timers rather than emitting partial waits.
|
||||
|
||||
Zero and missing values have different meanings. Valid zero-duration queue observations are recorded, while unavailable optional measurements are omitted instead of being exported as synthetic zeroes. `peak_memory_mb`, `diffusion_forward_s`, `diffusion_kv_load_s`, `vae_decode_s`, and `kv_wait_s` therefore appear only when their data source is active.
|
||||
|
||||
**Path 3: Audio modality metrics (`vllm_omni:audio_*`)**
|
||||
**Path 2: Audio modality metrics (`vllm:omni_audio_*`)**
|
||||
|
||||
`OmniModalityMetrics` registers seven audio families with `{model_name, stage, replica}` (plus an extra `threshold_ms` / `reason` label on the two extra-cardinality Counters). Three observation sites:
|
||||
|
||||
@@ -89,7 +76,7 @@ Zero and missing values have different meanings. Valid zero-duration queue obser
|
||||
- `observe_audio_first_packet(...)` — called from the OpenAI SSE audio branch in `serving_chat.py` on the first audio packet for a request. The once-per-request guard is held by `ClientRequestState.first_audio_ts`. The `request_arrival_ts` anchor is stored in `ClientRequestState` by `async_omni.generate()`, computed at request entry.
|
||||
- `observe_audio_streaming_finalize(...)` — called from `serving_chat.py` after the streaming chunk loop exhausts. It runs the per-chunk player simulation from `vllm_omni/benchmarks/audio_continuity.py` to compute the worst-case underrun and emits `audio_underrun_s` plus (when the request stayed below the threshold) `audio_continuity_ok_total{threshold_ms}`. Per-chunk PCM byte counts and arrival timestamps are recorded by the same audio branch that updates `first_audio_ts`.
|
||||
|
||||
**Path 4: Cross-stage transfer metrics (`vllm_omni:transfer_*`)**
|
||||
**Path 3: Cross-stage transfer metrics (`vllm:omni_transfer_*`)**
|
||||
|
||||
`OmniTransferMetrics` registers four Histogram families with `{model_name, from_stage, from_replica, to_stage, to_replica}` labels. Each observation corresponds to one physical transfer hop (one chunk between adjacent stages), not the per-request accumulated total — so the histograms track per-transfer distribution.
|
||||
|
||||
@@ -100,7 +87,7 @@ The hook lives in `OrchestratorAggregator.record_transfer_tx` and `record_transf
|
||||
|
||||
Defensive fail-safe: if `transfer_emitter` or `replica_resolver` is missing, or the resolver returns `None` for either side, the emit is skipped silently (the underlying `TransferEdgeStats` accumulation is unaffected).
|
||||
|
||||
**Path 5: Per-engine metrics (`vllm:*`, stage/replica wrap)**
|
||||
**Path 4: Per-engine metrics (`vllm:*`, stage/replica wrap)**
|
||||
|
||||
The Orchestrator instantiates `OmniPrometheusStatLogger` (a thin subclass of upstream `vllm.v1.metrics.loggers.PrometheusStatLogger`) and feeds it scheduler stats and iteration stats after processing each batch of engine outputs. This populates the standard ~37 vLLM metric families (TTFT, ITL, TPOT, KV cache usage, etc.) using the same upstream code path — but with the `engine` label reshaped into `stage` + `replica` so multi-replica deployments produce distinct series per replica. See the next section for the wrap mechanics.
|
||||
|
||||
@@ -110,9 +97,9 @@ The Orchestrator runs in a background thread. The API server (OmniBase) runs in
|
||||
|
||||
### Metric Registration and Lifecycle
|
||||
|
||||
All `vllm_omni:*` collectors are registered once when their owning class (`OmniPrometheusMetrics` / `OmniModalityMetrics` / `OmniTransferMetrics`) is imported. Per-`(stage, replica)` labels are bound lazily on first observation to avoid registering label sets for combinations that never produce data.
|
||||
All `vllm:omni_*` collectors are registered once when their owning class (`OmniPrometheusMetrics` / `OmniModalityMetrics` / `OmniTransferMetrics`) is imported. Per-`(stage, replica)` labels are bound lazily on first observation to avoid registering label sets for combinations that never produce data.
|
||||
|
||||
The `prometheus_client` default registry holds all collectors. FastAPI's `/metrics` endpoint serves the default registry, so `vllm_omni:*` and the wrapped `vllm:*` metrics appear in the same scrape response alongside `http_*` and `process_*` metrics from the instrumentator and the Python client runtime.
|
||||
The `prometheus_client` default registry holds all collectors. FastAPI's `/metrics` endpoint serves the default registry, so `vllm:omni_*` and the wrapped `vllm:*` metrics appear in the same scrape response alongside `http_*` and `process_*` metrics from the instrumentator and the Python client runtime.
|
||||
|
||||
## OmniPrometheusStatLogger Wrap
|
||||
|
||||
@@ -143,11 +130,11 @@ A reverse map `(stage_id, replica_id) -> flat_idx` is maintained on the Orchestr
|
||||
|
||||
## Gating: `--log-stats`
|
||||
|
||||
All metrics — both the omni-specific `vllm_omni:*` families and the upstream `vllm:*` wrap families — are gated by the user's `--log-stats` CLI flag (default off). The flag is plumbed from `OmniBase.__init__(log_stats=...)` through `AsyncOmniEngine` to the stage-spawn helpers and to the three Prometheus metric classes (`OmniPrometheusMetrics` / `OmniModalityMetrics` / `OmniTransferMetrics`), and is forwarded to `Orchestrator._init_metrics_state(...)`.
|
||||
All metrics — both the 15 `vllm:omni_*` families and the ~65 upstream `vllm:*` wrap families — are gated by the user's `--log-stats` CLI flag (default off). The flag is plumbed from `OmniBase.__init__(log_stats=...)` through `AsyncOmniEngine` to the stage-spawn helpers and to the three Prometheus metric classes (`OmniPrometheusMetrics` / `OmniModalityMetrics` / `OmniTransferMetrics`), and is forwarded to `Orchestrator._init_metrics_state(...)`.
|
||||
|
||||
Behavior with `--log-stats=off` (default):
|
||||
|
||||
- The three `Omni*Metrics` classes register their module-level Gauge / Counter / Histogram families at import time (prometheus_client requires up-front registration), but each `observe / inc / set` method early-returns. The per-label child series for `vllm_omni:*` stay materialized but never have data written to them.
|
||||
- The three `Omni*Metrics` classes register their module-level Gauge / Counter / Histogram families at import time (prometheus_client requires up-front registration), but each `observe / inc / set` method early-returns. The per-label child series for `vllm:omni_*` stay materialized but never have data written to them.
|
||||
- `OmniPrometheusStatLogger` is not constructed in `_init_metrics_state`, so the ~65 upstream `vllm:*` wrap families are not registered in the default registry at all.
|
||||
- The engine core's `Scheduler.make_stats()` also short-circuits inside upstream (`if not self.log_stats: return None`), so no `SchedulerStats` is produced per step — the per-iteration cost is bounded by the existing upstream gate.
|
||||
|
||||
@@ -175,43 +162,10 @@ second.
|
||||
|
||||
| Metric | Type | Labels | Description |
|
||||
|--------|------|--------|-------------|
|
||||
| `vllm_omni:num_requests_running` | Gauge | `model_name` | Requests currently executing across all stages |
|
||||
| `vllm_omni:num_requests_waiting` | Gauge | `model_name` | Requests queued but not yet scheduled |
|
||||
| `vllm_omni:requests_success_total` | Counter | `model_name`, `finished_reason` | Total requests by completion reason ({stop, length, abort, ...}); aborts cover client-disconnect / cancellation paths in addition to upstream `FinishReason.ABORT` |
|
||||
| `vllm_omni:e2e_request_latency_s` | Histogram | `model_name` | Pipeline-global end-to-end latency in seconds |
|
||||
|
||||
### Image and diffusion service-level metrics
|
||||
|
||||
| Metric | Type | Labels | Description |
|
||||
|--------|------|--------|-------------|
|
||||
| `vllm_omni:stage_gen_time_s` | Histogram | `model_name`, `stage`, `stage_type` | Stage submit to finished output; includes in-stage queueing |
|
||||
| `vllm_omni:request_queue_wait_s` | Histogram | `model_name` | Orchestration-layer queue wait; a present zero is recorded |
|
||||
| `vllm_omni:stage_waiting_requests` | Gauge | `model_name`, `stage` | Sum of the latest waiting snapshots across live replicas in the stage |
|
||||
| `vllm_omni:stage_in_queue_s` | Histogram | `model_name`, `stage` | Diffusion scheduler wait from enqueue to initial execution admission |
|
||||
| `vllm_omni:num_inference_steps` | Histogram | `model_name` | Denoising-step distribution for diffusion stages |
|
||||
| `vllm_omni:image_count_total` | Counter | `model_name` | Cumulative number of generated image outputs |
|
||||
| `vllm_omni:image_pixels` | Histogram | `model_name` | Pixel-count distribution for image outputs |
|
||||
| `vllm_omni:peak_memory_mb` | Gauge | `model_name`, `stage` | Positive per-stage peak-memory samples reported by engine outputs |
|
||||
| `vllm_omni:requests_failed_total` | Counter | `model_name`, `reason` | Request failures with bounded reason values (`client_abort`, `client_disconnect`, `stage_error`, `unknown`) |
|
||||
| `vllm_omni:kv_wait_s` | Histogram | `model_name`, `connector_type` | Completed AR KV-transfer wait intervals, grouped by connector backend |
|
||||
|
||||
### Diffusion execution breakdown
|
||||
|
||||
Labels: `{model_name, stage, replica}`.
|
||||
|
||||
| Metric | Type | Description |
|
||||
|--------|------|-------------|
|
||||
| `vllm_omni:diffusion_exec_s` | Histogram | Core diffusion-step execution time per request |
|
||||
| `vllm_omni:diffusion_exec_per_step_s` | Histogram | Core diffusion execution divided by inference-step count |
|
||||
| `vllm_omni:diffusion_preprocess_s` | Histogram | Diffusion input preprocessing time |
|
||||
| `vllm_omni:diffusion_postprocess_s` | Histogram | Diffusion output postprocessing time |
|
||||
| `vllm_omni:vae_decode_s` | Histogram | VAE decode time when the engine reports it |
|
||||
| `vllm_omni:diffusion_forward_s` | Histogram | Forward-only denoise-loop time when pipeline profiling is enabled |
|
||||
| `vllm_omni:diffusion_kv_load_s` | Histogram | Diffusion-side KV receive/load time when upstream KV is used |
|
||||
| `vllm_omni:image_ttfp_s` | Histogram | Image-stage submit to first materialized image output |
|
||||
| `vllm_omni:denoise_step_latency_s` | Histogram | Mean diffusion forward time per denoise step for image output stages |
|
||||
|
||||
The optional breakdown and memory families are sparse by design: when an engine does not produce the source measurement, no sample is emitted. This differs from queue timings, where an explicitly measured zero is meaningful and is retained.
|
||||
| `vllm:omni_num_requests_running` | Gauge | `model_name` | Requests currently executing across all stages |
|
||||
| `vllm:omni_num_requests_waiting` | Gauge | `model_name` | Requests queued but not yet scheduled |
|
||||
| `vllm:omni_requests_success_total` | Counter | `model_name`, `finished_reason` | Total requests by completion reason ({stop, length, abort, ...}); aborts cover client-disconnect / cancellation paths in addition to upstream `FinishReason.ABORT` |
|
||||
| `vllm:omni_e2e_request_latency_s` | Histogram | `model_name` | Pipeline-global end-to-end latency in seconds |
|
||||
|
||||
### Audio (7)
|
||||
|
||||
@@ -219,13 +173,13 @@ Labels: `{model_name, stage, replica}` plus the listed extra label.
|
||||
|
||||
| Metric | Type | Extra label | Description |
|
||||
|--------|------|-------------|-------------|
|
||||
| `vllm_omni:audio_ttfp_s` | Histogram | — | Time from request arrival to first audio packet/frame |
|
||||
| `vllm_omni:audio_duration_s` | Histogram | — | Audio content duration (`audio_frames / sample_rate`) |
|
||||
| `vllm_omni:audio_rtf` | Histogram | — | Real-time factor `stage_gen_time_s / audio_duration_s` (SLO `< 1`); uses `RTF_BUCKETS` |
|
||||
| `vllm_omni:audio_frames_total` | Counter | — | Cumulative audio frames generated |
|
||||
| `vllm_omni:audio_underrun_s` | Histogram | — | Per-request worst-case player deficit; `> 0` indicates listener heard silent gaps |
|
||||
| `vllm_omni:audio_continuity_ok_total` | Counter | `threshold_ms` | Incremented when the request's worst underrun stayed below `threshold_ms` |
|
||||
| `vllm_omni:audio_skipped_requests_total` | Counter | `reason` | Silent-loss counter — code2wav rejected malformed codec input and returned `200 OK` with empty audio |
|
||||
| `vllm:omni_audio_ttfp_s` | Histogram | — | Time from request arrival to first audio packet/frame |
|
||||
| `vllm:omni_audio_duration_s` | Histogram | — | Audio content duration (`audio_frames / sample_rate`) |
|
||||
| `vllm:omni_audio_rtf` | Histogram | — | Real-time factor `stage_gen_time_s / audio_duration_s` (SLO `< 1`); uses `RTF_BUCKETS` |
|
||||
| `vllm:omni_audio_frames_total` | Counter | — | Cumulative audio frames generated |
|
||||
| `vllm:omni_audio_underrun_s` | Histogram | — | Per-request worst-case player deficit; `> 0` indicates listener heard silent gaps |
|
||||
| `vllm:omni_audio_continuity_ok_total` | Counter | `threshold_ms` | Incremented when the request's worst underrun stayed below `threshold_ms` |
|
||||
| `vllm:omni_audio_skipped_requests_total` | Counter | `reason` | Silent-loss counter — code2wav rejected malformed codec input and returned `200 OK` with empty audio |
|
||||
|
||||
### Cross-stage transfer (4)
|
||||
|
||||
@@ -233,10 +187,10 @@ Labels: `{model_name, from_stage, from_replica, to_stage, to_replica}`.
|
||||
|
||||
| Metric | Type | Description |
|
||||
|--------|------|-------------|
|
||||
| `vllm_omni:transfer_size_bytes` | Histogram | Per-transfer payload size in bytes |
|
||||
| `vllm_omni:transfer_tx_s` | Histogram | Sender-side time (serialize + submit to connector) |
|
||||
| `vllm_omni:transfer_rx_s` | Histogram | Receiver-side time (recv + deserialize) |
|
||||
| `vllm_omni:transfer_in_flight_s` | Histogram | Network in-flight time (TX done → RX recv start) |
|
||||
| `vllm:omni_transfer_size_bytes` | Histogram | Per-transfer payload size in bytes |
|
||||
| `vllm:omni_transfer_tx_s` | Histogram | Sender-side time (serialize + submit to connector) |
|
||||
| `vllm:omni_transfer_rx_s` | Histogram | Receiver-side time (recv + deserialize) |
|
||||
| `vllm:omni_transfer_in_flight_s` | Histogram | Network in-flight time (TX done → RX recv start) |
|
||||
|
||||
### LLM stage-level (wrapped `vllm:*`)
|
||||
|
||||
@@ -249,7 +203,7 @@ After the wrap, every upstream `vllm:*` family — TTFT, ITL, TPOT, e2e latency,
|
||||
- `SECONDS_FAST_BUCKETS` (0.001 s – 60 s) for fine-grained cross-stage transfer and audio-underrun values that need millisecond-level resolution.
|
||||
- Counters use the `_total` suffix (auto-appended by `prometheus_client`).
|
||||
- Sizes use the `_bytes` suffix.
|
||||
- All omni-specific families are prefixed `vllm_omni:`. The upstream `unregister_vllm_metrics()` function is monkey-patched to a scoped version that still strips upstream `vllm:*` collectors (so multi-engine init within one process does not crash on duplicate registration) but preserves anything prefixed `vllm_omni:` / `vllm_omni`.
|
||||
- All omni-specific families are prefixed `vllm:omni_`. The upstream `unregister_vllm_metrics()` function is monkey-patched to a scoped version that still strips upstream `vllm:*` collectors (so multi-engine init within one process does not crash on duplicate registration) but preserves anything prefixed `vllm:omni_` / `vllm_omni`.
|
||||
|
||||
## Logging vs. Prometheus
|
||||
|
||||
|
||||
@@ -57,10 +57,3 @@ cross-stage routing.
|
||||
|
||||
Test registry selection, checkpoint loading, minimal inference, input and output
|
||||
contracts, and every declared optional capability.
|
||||
|
||||
## Model-specific capability documents
|
||||
|
||||
- [MiniMax H3 temporal VAE chunk callback](../../feature/minimax_h3_temporal_vae_chunks.md)
|
||||
defines the source-gated producer seam and its boundary with future runtime
|
||||
media transport. It is an internal opt-in capability, not a public serving
|
||||
feature.
|
||||
@@ -37,7 +37,6 @@ validation_paths:
|
||||
- tests/engine/test_async_omni_engine_input.py
|
||||
- tests/engine/test_async_omni_engine_outputs.py
|
||||
- tests/engine/test_async_omni_engine_abort.py
|
||||
- tests/e2e/offline_inference/test_qwen3_omni.py
|
||||
- tests/engine/test_async_omni_engine_stage_init.py
|
||||
- tests/engine/test_orchestrator.py
|
||||
- tests/engine/test_orchestrator_error_handling.py
|
||||
@@ -67,12 +66,6 @@ the boundary affected by the in-flight stage client/process refactor in
|
||||
[#5441](https://github.com/vllm-project/vllm-omni/pull/5441). Names and
|
||||
responsibilities proposed only by that PR are not current contracts.
|
||||
|
||||
The orchestration loop also has an opt-in event-driven mode
|
||||
(`VLLM_OMNI_EVENT_DRIVEN_ORCH=1`, default off) proposed in
|
||||
[#5221](https://github.com/vllm-project/vllm-omni/pull/5221). It changes poll
|
||||
cadence only: the routing, ordering, and terminal-state contracts below hold
|
||||
identically on both loops.
|
||||
|
||||
## Ownership boundary
|
||||
|
||||
This document owns `AsyncOmniEngine`, `Orchestrator`, request-state creation,
|
||||
@@ -114,15 +107,6 @@ Numbers become append-only after normative promotion.
|
||||
|
||||
Test routing, output ordering, queue correlation, cancellation, failure
|
||||
propagation, shutdown ordering, and representative multi-stage execution.
|
||||
AR abort of a final-stage LLM request must deliver a terminal output with
|
||||
`finish_reason="abort"` and the generated prefix so collocated training can
|
||||
resume; diffusion abort remains whole-sample retry. Aborting a parallel-sampling
|
||||
child must drop `parent_requests` once no children remain. EngineCore control
|
||||
RPCs (`sleep` / `wake_up` / `pause_scheduler` / `resume_scheduler`) must
|
||||
propagate worker exceptions rather than returning `{"supported": False}`.
|
||||
When a request has multiple final output stages, only the last abort message
|
||||
is request-terminal. Output-processor abort state is committed only after the
|
||||
physical EngineCore abort succeeds.
|
||||
Changes that cross into stage runtime or public error behavior require review
|
||||
from that contract's owners.
|
||||
|
||||
|
||||
@@ -43,7 +43,6 @@ depends_on:
|
||||
validation_paths:
|
||||
- tests/entrypoints/test_omni_entrypoints.py
|
||||
- tests/entrypoints/test_async_omni.py
|
||||
- tests/entrypoints/test_async_omni_pause_sleep_routing.py
|
||||
- tests/entrypoints/test_async_omni_duplex.py
|
||||
- tests/entrypoints/test_serve.py
|
||||
- tests/entrypoints/test_stream_finish_reason.py
|
||||
@@ -120,13 +119,7 @@ promotion.
|
||||
|
||||
Test request validation, protocol conversion, model-adapter routing,
|
||||
streaming, session identity, disconnect/cancellation, and error mapping for
|
||||
each affected offline or serving entrypoint. Sleep must wait for in-flight
|
||||
`generate()` admission before EngineCore offload; `wake_up` does not resume
|
||||
admission — callers must `resume_generation()`. Sleeping tags are tracked per
|
||||
stage so `wake_up(stage_ids=[0])` does not skip a later `wake_up(stage_ids=[1])`.
|
||||
Streaming input pumps take an admission slot immediately before each EngineCore
|
||||
ADD or update, not while waiting for the next client chunk. Frontend abort
|
||||
keeps `request_states` until `generate()` consumes the terminal output.
|
||||
each affected offline or serving entrypoint.
|
||||
|
||||
## Promotion gate
|
||||
|
||||
|
||||
@@ -257,53 +257,6 @@ Lookup checks the marker both before validation and after constructing the
|
||||
lease so a weaker overlapping lookup cannot return a lease after a stronger
|
||||
validation has denied the artifact.
|
||||
|
||||
If publishing the external deny marker fails, lookup returns a retryable
|
||||
storage failure and marks the artifact directory with a persistent invalidation
|
||||
mode before releasing the shared lock. Cooperating readers check that mode both
|
||||
while opening and immediately before returning a lease. Lookup then makes a
|
||||
bounded attempt to take the exclusive artifact lock and quarantine the entry;
|
||||
the quarantine transition removes the internal mode marker. If a cooperating
|
||||
builder replaces the marked inode while the locks are exchanged, containment
|
||||
preserves that new publication instead of quarantining it. If neither the
|
||||
external marker nor the directory-mode marker can be persisted, cross-process
|
||||
exclusion cannot be guaranteed on an unavailable filesystem and the domain
|
||||
must remain failed rather than treating the artifact as an ordinary invalid
|
||||
cache entry.
|
||||
|
||||
New publications carry the internal invalidation mode through the staging
|
||||
rename. Hardening clears it and is the logical authority transition for a
|
||||
process crash; the temporary- and artifact-parent syncs then make the rename
|
||||
durable, with an artifact-inode sync first making the hardened mode durable. A
|
||||
process that dies before hardening leaves a persistently denied
|
||||
entry, while one that dies after successful hardening leaves an authoritative
|
||||
ready entry. Any synchronous failure after rename is marked invalid and
|
||||
contained in quarantine before `PUBLICATION_FAILED` is returned, so that
|
||||
nonretryable result cannot later resolve as a ready hit. If the filesystem
|
||||
refuses both the inode marker and the quarantine move, the store instead returns
|
||||
a retryable storage failure with `outcome_uncertain` details; as with deny-marker
|
||||
double failure, cross-process exclusion cannot be guaranteed until storage
|
||||
recovers. Once the artifact-parent sync succeeds, a later artifact- or
|
||||
build-lock release error is logged and lookup resolves the committed entry
|
||||
instead of reporting a contradictory publication failure. The same authority
|
||||
rule applies when a competing publisher already installed byte-identical
|
||||
semantic content; staging-cleanup errors are retried without downgrading that
|
||||
validated ready entry to publication failure.
|
||||
|
||||
Before widening a hardened artifact for a lifecycle move, the store durably
|
||||
sets the same invalidation mode. A process crash before rename therefore leaves
|
||||
the writable source denied. After rename, the store re-hardens the destination
|
||||
but retains the marker until both parent-directory syncs make the move durable;
|
||||
only then does it durably clear the marker. An interrupted move therefore leaves
|
||||
a marked transition that the next exact-key lifecycle operation re-hardens.
|
||||
Explicit cleanup uses the locked move and retries exact-key
|
||||
`.cleanup.*` tombstones left by an interrupted removal; a later build performs
|
||||
the same tombstone reconciliation before producing replacement content.
|
||||
|
||||
An operational failure while inspecting a noncooperative competing publication
|
||||
is a retryable storage failure. The competing entry remains authoritative and
|
||||
is not mislabeled as corrupt or quarantined merely because its manifest could
|
||||
not be read transiently.
|
||||
|
||||
`inspect_domain()` and `inspect_artifact()` return typed, read-only snapshots.
|
||||
Inspection validation does not create a deny marker. Reported lock activity is
|
||||
point-in-time evidence only; kernel locks are not a persistent owner registry.
|
||||
|
||||
@@ -3,6 +3,4 @@
|
||||
vLLM-Omni's examples are split into two categories:
|
||||
|
||||
- If you are using vLLM-Omni from within Python code, see the *Offline Inference* section.
|
||||
- If you are using vLLM-Omni from an HTTP application or client, start with
|
||||
the [API Server guide](../serving/README.md) to choose an endpoint, then use
|
||||
the *Online Serving* section for model-specific examples.
|
||||
- If you are using vLLM-Omni from an HTTP application or client, see the *Online Serving* section.
|
||||
@@ -1,7 +1,7 @@
|
||||
# Installation
|
||||
|
||||
!!! important
|
||||
vLLM-Omni is released against the matching upstream vLLM major/minor version. vLLM-Omni 0.28.x requires vLLM 0.28.x; the stable v0.28.0 instructions pin vLLM 0.28.0.
|
||||
vLLM-Omni is released against the matching upstream vLLM major/minor version. vLLM-Omni 0.26.x requires vLLM 0.26.x; the stable v0.26.0 instructions pin vLLM 0.26.0.
|
||||
|
||||
vLLM-Omni supports the following hardware platforms:
|
||||
|
||||
|
||||
@@ -34,7 +34,7 @@ vLLM-Omni is a Python library that supports the following GPU variants. The libr
|
||||
|
||||
### Pre-built wheels
|
||||
|
||||
Note: Pre-built wheels are currently available for vLLM-Omni 0.11.0rc1, 0.12.0rc1, 0.14.0rc1, 0.14.0, 0.16.0, 0.18.0, 0.20.0, 0.21.0, 0.22.0, 0.23.0, 0.24.0, 0.25.0, 0.26.0, and 0.28.0. If you need a newer unreleased revision, please [build from source](https://docs.vllm.ai/projects/vllm-omni/en/latest/getting_started/installation/gpu/#build-wheel-from-source).
|
||||
Note: Pre-built wheels are currently available for vLLM-Omni 0.11.0rc1, 0.12.0rc1, 0.14.0rc1, 0.14.0, 0.16.0, 0.18.0, 0.20.0, 0.21.0, 0.22.0, 0.23.0, 0.24.0, 0.25.0, and 0.26.0. If you need a newer unreleased revision, please [build from source](https://docs.vllm.ai/projects/vllm-omni/en/latest/getting_started/installation/gpu/#build-wheel-from-source).
|
||||
|
||||
=== "NVIDIA CUDA"
|
||||
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
# --8<-- [end:requirements]
|
||||
# --8<-- [start:set-up-using-python]
|
||||
|
||||
vLLM-Omni depends on the matching major/minor release of vLLM. The vLLM-Omni 0.28.x release line uses vLLM 0.28.x.
|
||||
vLLM-Omni depends on the matching major/minor release of vLLM. The vLLM-Omni 0.26.x release line uses vLLM 0.26.x.
|
||||
|
||||
!!! note
|
||||
PyTorch installed via `conda` will statically link `NCCL` library, which can cause issues when vLLM tries to use `NCCL`. See <gh-issue:8420> for more details.
|
||||
@@ -20,7 +20,7 @@ Therefore, it is recommended to install vLLM and vLLM-Omni with a **fresh new**
|
||||
|
||||
vLLM-Omni is built based on vLLM. Please install it with command below.
|
||||
```bash
|
||||
uv pip install vllm==0.28.0 --torch-backend=auto
|
||||
uv pip install vllm==0.26.0 --torch-backend=auto
|
||||
```
|
||||
|
||||
#### Installation of vLLM-Omni
|
||||
@@ -39,13 +39,13 @@ uv pip install 'vllm-omni[demo]'
|
||||
# --8<-- [start:build-wheel-from-source]
|
||||
|
||||
#### Installation of vLLM
|
||||
If you do not need to modify source code of vLLM, you can directly install the stable 0.28.0 release version of the library
|
||||
If you do not need to modify source code of vLLM, you can directly install the stable 0.26.0 release version of the library
|
||||
|
||||
```bash
|
||||
uv pip install vllm==0.28.0 --torch-backend=auto
|
||||
uv pip install vllm==0.26.0 --torch-backend=auto
|
||||
```
|
||||
|
||||
The 0.28.0 release of vLLM ships CUDA 13.0-compatible binaries by default. If you need a different CUDA variant or want to reuse an existing PyTorch installation, build vLLM from source instead.
|
||||
The 0.26.0 release of vLLM ships CUDA 13.0-compatible binaries by default. If you need a different CUDA variant or want to reuse an existing PyTorch installation, build vLLM from source instead.
|
||||
|
||||
#### Installation of vLLM-Omni
|
||||
Since vllm-omni is rapidly evolving, it's recommended to install it from source
|
||||
@@ -66,12 +66,12 @@ If you want to check, modify or debug with source code of vLLM, install the libr
|
||||
```bash
|
||||
git clone https://github.com/vllm-project/vllm.git
|
||||
cd vllm
|
||||
git checkout v0.28.0
|
||||
git checkout v0.26.0
|
||||
```
|
||||
Set up environment variables to get pre-built wheels. If there are internet problems, just download the whl file manually. And set `VLLM_PRECOMPILED_WHEEL_LOCATION` as your local absolute path of whl file.
|
||||
```bash
|
||||
#For CUDA 13.0 (the default for v0.28.0; the wheel filename has no `+cu130` suffix)
|
||||
export VLLM_PRECOMPILED_WHEEL_LOCATION=https://github.com/vllm-project/vllm/releases/download/v0.28.0/vllm-0.28.0-cp38-abi3-manylinux_2_28_x86_64.whl
|
||||
#For CUDA 13.0 (the default for v0.26.0; the wheel filename has no `+cu130` suffix)
|
||||
export VLLM_PRECOMPILED_WHEEL_LOCATION=https://github.com/vllm-project/vllm/releases/download/v0.26.0/vllm-0.26.0-cp38-abi3-manylinux_2_28_x86_64.whl
|
||||
```
|
||||
Install vllm with command below (If you have no existing PyTorch).
|
||||
```bash
|
||||
@@ -95,13 +95,6 @@ uv pip install --no-build-isolation --editable .
|
||||
|
||||
vLLM-Omni offers an official docker image for deployment. These images are built on top of vLLM docker images and available on Docker Hub as [vllm/vllm-omni](https://hub.docker.com/r/vllm/vllm-omni/tags). The version of vLLM-Omni indicates which release of vLLM it is based on.
|
||||
|
||||
Nightly CUDA images built from the latest `main` are also available:
|
||||
|
||||
- [`vllm/vllm-omni:nightly`](https://hub.docker.com/r/vllm/vllm-omni/tags?name=nightly) is the rolling multi-architecture tag.
|
||||
- [`vllm/vllm-omni:nightly-<commit>`](https://hub.docker.com/r/vllm/vllm-omni/tags?name=nightly-) provides commit-pinned multi-architecture tags. The newest 14 are retained.
|
||||
|
||||
Use a commit-pinned tag when you need a reproducible nightly environment.
|
||||
|
||||
Here's an example deployment command that has been verified on 2 x H100's:
|
||||
```bash
|
||||
docker run --runtime nvidia --gpus 2 \
|
||||
@@ -109,7 +102,7 @@ docker run --runtime nvidia --gpus 2 \
|
||||
--env "HF_TOKEN=$HF_TOKEN" \
|
||||
-p 8091:8091 \
|
||||
--ipc=host \
|
||||
vllm/vllm-omni:v0.28.0 \
|
||||
vllm/vllm-omni:v0.26.0 \
|
||||
vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091
|
||||
```
|
||||
|
||||
@@ -131,7 +124,7 @@ If you want to specify the base vLLM version:
|
||||
```bash
|
||||
DOCKER_BUILDKIT=1 docker build \
|
||||
-f docker/Dockerfile.cuda \
|
||||
--build-arg BASE_IMAGE=vllm/vllm-openai:v0.28.0 \
|
||||
--build-arg BASE_IMAGE=vllm/vllm-openai:v0.26.0 \
|
||||
-t vllm-omni-cuda .
|
||||
```
|
||||
|
||||
|
||||
@@ -13,7 +13,7 @@ vLLM-Omni current recommends the steps in under setup through Docker Images.
|
||||
|
||||
vLLM-Omni is built based on vLLM. Please install it with command below.
|
||||
```bash
|
||||
uv pip install vllm==0.28.0+rocm723 --extra-index-url https://wheels.vllm.ai/rocm/0.28.0/rocm723
|
||||
uv pip install vllm==0.26.0+rocm723 --extra-index-url https://wheels.vllm.ai/rocm/0.26.0/rocm723
|
||||
```
|
||||
|
||||
#### Installation of vLLM-Omni
|
||||
@@ -34,13 +34,13 @@ uv pip install onnxruntime-rocm
|
||||
# --8<-- [start:build-wheel-from-source]
|
||||
|
||||
#### Installation of vLLM
|
||||
If you do not need to modify source code of vLLM, you can directly install the stable 0.28.0 release version of the library
|
||||
If you do not need to modify source code of vLLM, you can directly install the stable 0.26.0 release version of the library
|
||||
|
||||
```bash
|
||||
uv pip install vllm==0.28.0+rocm723 --extra-index-url https://wheels.vllm.ai/rocm/0.28.0/rocm723
|
||||
uv pip install vllm==0.26.0+rocm723 --extra-index-url https://wheels.vllm.ai/rocm/0.26.0/rocm723
|
||||
```
|
||||
|
||||
The pre-built 0.28.0 vLLM wheel targets ROCm 7.2.3. If you need a different ROCm stack or want to reuse an existing PyTorch installation, build vLLM from source instead.
|
||||
The pre-built 0.26.0 vLLM wheel targets ROCm 7.2.3. If you need a different ROCm stack or want to reuse an existing PyTorch installation, build vLLM from source instead.
|
||||
|
||||
#### Installation of vLLM-Omni
|
||||
Since vllm-omni is rapidly evolving, it's recommended to install it from source
|
||||
@@ -58,7 +58,7 @@ If you want to check, modify or debug with source code of vLLM, install the libr
|
||||
```bash
|
||||
git clone https://github.com/vllm-project/vllm.git
|
||||
cd vllm
|
||||
git checkout v0.28.0
|
||||
git checkout v0.26.0
|
||||
python3 -m pip install -r requirements/rocm.txt
|
||||
python3 setup.py develop
|
||||
```
|
||||
@@ -130,7 +130,7 @@ docker run --rm \
|
||||
-v ~/.cache/huggingface:/root/.cache/huggingface \
|
||||
--env "HF_TOKEN=$HF_TOKEN" \
|
||||
-p 8091:8091 \
|
||||
vllm/vllm-omni-rocm:v0.28.0 \
|
||||
vllm/vllm-omni-rocm:v0.26.0 \
|
||||
--model Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091
|
||||
```
|
||||
|
||||
@@ -149,7 +149,7 @@ docker run --rm -it \
|
||||
-v ~/.cache/huggingface:/root/.cache/huggingface \
|
||||
--env "HF_TOKEN=$HF_TOKEN" \
|
||||
--entrypoint bash \
|
||||
vllm/vllm-omni-rocm:v0.28.0
|
||||
vllm/vllm-omni-rocm:v0.26.0
|
||||
```
|
||||
|
||||
# --8<-- [end:pre-built-images]
|
||||
@@ -23,19 +23,6 @@ vLLM-Omni currently recommends using the Docker image setup steps below.
|
||||
DOCKER_BUILDKIT=1 docker build -f docker/Dockerfile.xpu -t vllm-omni-xpu --shm-size=4g .
|
||||
```
|
||||
|
||||
This layers vLLM-Omni on top of the published `vllm/vllm-openai-xpu:<VLLM_VERSION>`
|
||||
base image, which Docker pulls automatically. To target a different vLLM release,
|
||||
pass `--build-arg VLLM_VERSION=<tag>`. If that tag has not been published yet, build
|
||||
the base from upstream's own Dockerfile first and point the build at it:
|
||||
|
||||
```bash
|
||||
git clone --depth 1 --branch "$VLLM_VERSION" https://github.com/vllm-project/vllm /tmp/vllm
|
||||
DOCKER_BUILDKIT=1 docker build -t vllm-openai-xpu:local --target vllm-openai \
|
||||
-f /tmp/vllm/docker/Dockerfile.xpu /tmp/vllm
|
||||
DOCKER_BUILDKIT=1 docker build -f docker/Dockerfile.xpu -t vllm-omni-xpu --shm-size=4g \
|
||||
--build-arg VLLM_BASE=vllm-openai-xpu:local .
|
||||
```
|
||||
|
||||
#### Launch the docker image
|
||||
|
||||
##### Launch with OpenAI API Server
|
||||
|
||||
@@ -9,8 +9,8 @@ The recommended way to use vLLM-Omni on NPU is through the vLLM-Ascend pre-built
|
||||
|
||||
```bash
|
||||
# vLLM-Ascend image
|
||||
# Atlas A2: quay.io/atlas-ci/vllm-ascend:v0.28.0
|
||||
# Atlas A3: quay.io/atlas-ci/vllm-ascend:v0.28.0-a3
|
||||
# Atlas A2: quay.io/atlas-ci/vllm-ascend:v0.26.0
|
||||
# Atlas A3: quay.io/atlas-ci/vllm-ascend:v0.26.0-a3
|
||||
docker run --rm \
|
||||
--name vllm-omni-npu \
|
||||
--shm-size=1g \
|
||||
@@ -28,11 +28,11 @@ docker run --rm \
|
||||
-v /etc/ascend_install.info:/etc/ascend_install.info \
|
||||
-v /root/.cache:/root/.cache \
|
||||
-p 8000:8000 \
|
||||
-it quay.io/atlas-ci/vllm-ascend:v0.28.0-a3 bash
|
||||
-it quay.io/atlas-ci/vllm-ascend:v0.26.0-a3 bash
|
||||
|
||||
# Inside the container, install vLLM-Omni from source
|
||||
cd /vllm-workspace
|
||||
git clone -b v0.28.0 https://github.com/vllm-project/vllm-omni.git
|
||||
git clone -b v0.26.0 https://github.com/vllm-project/vllm-omni.git
|
||||
cd vllm-omni
|
||||
pip install -v -e . --no-build-isolation
|
||||
# or VLLM_OMNI_TARGET_DEVICE=npu pip install -v -e .
|
||||
@@ -50,11 +50,21 @@ We are keeping [issue #886](https://github.com/vllm-project/vllm-omni/issues/886
|
||||
|
||||
# --8<-- [start:installation-main]
|
||||
|
||||
You can also install vLLM-Omni from the latest main branch if you want to use the latest features or bug fixes. Use the aligned vLLM-Ascend v0.28.0 image described above, and check [issue #886](https://github.com/vllm-project/vllm-omni/issues/886) for the status of the latest vLLM-Omni commit on NPU.
|
||||
You can also build vLLM-Omni from the latest main branch if you want to use the latest features or bug fixes. (But sometimes it will break for a while. You can check [issue #886](https://github.com/vllm-project/vllm-omni/issues/886) for the status of the latest commit of vLLM-Omni main branch on NPU.)
|
||||
|
||||
```bash
|
||||
# Inside the vLLM-Ascend v0.28.0 container, install vLLM-Omni from main
|
||||
cd /vllm-workspace
|
||||
# Pin vLLM and vLLM-Ascend to the v0.26 release line
|
||||
git clone -b v0.26.0 https://github.com/vllm-project/vllm.git
|
||||
cd vllm
|
||||
VLLM_TARGET_DEVICE=empty pip install -v -e .
|
||||
cd ..
|
||||
|
||||
git clone -b releases/v0.26.0rc https://github.com/vllm-project/vllm-ascend.git
|
||||
cd vllm-ascend
|
||||
pip install -v -e .
|
||||
cd ..
|
||||
|
||||
# Install vLLM-Omni from the latest main branch
|
||||
git clone https://github.com/vllm-project/vllm-omni.git
|
||||
cd vllm-omni
|
||||
pip install -v -e . --no-build-isolation
|
||||
@@ -78,8 +88,8 @@ Supported images as following.
|
||||
Here's an example deployment command that has been verified on 4 x NPUs:
|
||||
|
||||
```bash
|
||||
# Atlas A2: quay.io/ascend/vllm-omni:v0.28.0
|
||||
# Atlas A3: quay.io/ascend/vllm-omni:v0.28.0-a3
|
||||
# Atlas A2: quay.io/ascend/vllm-omni:v0.26.0
|
||||
# Atlas A3: quay.io/ascend/vllm-omni:v0.26.0-a3
|
||||
docker run --rm \
|
||||
--name vllm-omni-a3 \
|
||||
--shm-size=4g \
|
||||
@@ -104,7 +114,7 @@ docker run --rm \
|
||||
-v /etc/ascend_install.info:/etc/ascend_install.info \
|
||||
-v ~/.cache:/root/.cache \
|
||||
-p 8091:8091 \
|
||||
-it quay.io/ascend/vllm-omni:v0.28.0-a3 bash
|
||||
-it quay.io/ascend/vllm-omni:v0.26.0-a3 bash
|
||||
```
|
||||
|
||||
!!! tip
|
||||
|
||||
@@ -19,10 +19,10 @@ uv venv --python 3.12 --seed
|
||||
source .venv/bin/activate
|
||||
|
||||
# On CUDA
|
||||
uv pip install vllm==0.28.0 --torch-backend=auto
|
||||
uv pip install vllm==0.26.0 --torch-backend=auto
|
||||
|
||||
# On ROCm
|
||||
uv pip install vllm==0.28.0+rocm723 --extra-index-url https://wheels.vllm.ai/rocm/0.28.0/rocm723
|
||||
uv pip install vllm==0.26.0+rocm723 --extra-index-url https://wheels.vllm.ai/rocm/0.26.0/rocm723
|
||||
|
||||
git clone https://github.com/vllm-project/vllm-omni.git
|
||||
cd vllm-omni
|
||||
@@ -34,7 +34,7 @@ For additional installation methods — please see the [installation guide](inst
|
||||
!!! note
|
||||
It is important to install the same major & minor version of vLLM and vLLM Omni, otherwise things may not work as expected. If the versions are misaligned, you will see a warning when you import vLLM Omni.
|
||||
|
||||
If you are seeing strange behavior with the `vllm` command not handling the `--omni` flag correctly, you most likely have a version mismatch with vLLM < `0.28.0` and vLLM Omni `0.28.0`, as vLLM Omni no longer hijacks the vLLM entrypoint. Updating vLLM should resolve this issue.
|
||||
If you are seeing strange behavior with the `vllm` command not handling the `--omni` flag correctly, you most likely have a version mismatch with vLLM < `0.26.0` and vLLM Omni `0.26.0`, as vLLM Omni no longer hijacks the vLLM entrypoint. Updating vLLM should resolve this issue.
|
||||
|
||||
## Offline Inference
|
||||
|
||||
@@ -91,7 +91,7 @@ if __name__ == "__main__":
|
||||
|
||||
For more usages, please refer to [offline inference](../user_guide/examples/offline_inference/qwen2_5_omni.md)
|
||||
|
||||
## Online Serving with the API Server
|
||||
## Online Serving with OpenAI-Completions API
|
||||
|
||||
Text-to-image generation quickstart with vLLM-Omni:
|
||||
|
||||
@@ -100,16 +100,20 @@ vllm serve Tongyi-MAI/Z-Image-Turbo --omni --port 8091
|
||||
```
|
||||
|
||||
```bash
|
||||
curl -s http://localhost:8091/v1/images/generations \
|
||||
curl -s http://localhost:8091/v1/chat/completions \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"prompt": "a cup of coffee on the table",
|
||||
"size": "1024x1024",
|
||||
"response_format": "b64_json",
|
||||
"seed": 42
|
||||
}' | jq -r '.data[0].b64_json' | base64 -d > coffee.png
|
||||
"messages": [
|
||||
{"role": "user", "content": "a cup of coffee on the table"}
|
||||
],
|
||||
"extra_body": {
|
||||
"height": 1024,
|
||||
"width": 1024,
|
||||
"num_inference_steps": 50,
|
||||
"guidance_scale": 4.0,
|
||||
"seed": 42
|
||||
}
|
||||
}' | jq -r '.choices[0].message.content[0].image_url.url' | cut -d',' -f2 | base64 -d > coffee.png
|
||||
```
|
||||
|
||||
See the [API Server guide](../serving/README.md) to choose an endpoint. For
|
||||
model-specific details, refer to the [text-to-image online serving
|
||||
example](../user_guide/examples/online_serving/text_to_image.md).
|
||||
For more details, please refer to [online serving](../user_guide/examples/online_serving/text_to_image.md).
|
||||
@@ -37,6 +37,7 @@ GENERAL_EXAMPLE_SLUGS = frozenset(
|
||||
"text_to_speech",
|
||||
"text_to_video",
|
||||
"x_to_text",
|
||||
"x_to_video_audio",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@@ -21,20 +21,20 @@ th {
|
||||
|--------------|--------|-------------------|------------|---------|-----|-----------|--------|
|
||||
| `Qwen3OmniMoeForConditionalGeneration` | Qwen3-Omni | `Qwen/Qwen3-Omni-30B-A3B-Instruct` | ✅︎ | | | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/Qwen/Qwen3-Omni.md) |
|
||||
| `Qwen2_5OmniForConditionalGeneration` | Qwen2.5-Omni | `Qwen/Qwen2.5-Omni-7B`, `Qwen/Qwen2.5-Omni-3B` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
|
||||
| `MingFlashOmniForConditionalGeneration` + `MingImagePipeline` | Ming-flash-omni-2.0 (omni-speech + imagegen<sup>1</sup>) | `Jonathan1909/Ming-flash-omni-2.0` | ✅︎ | | | | — |
|
||||
| `MingFlashOmniForConditionalGeneration` + `MingImagePipeline` | Ming-flash-omni-2.0 (omni-speech + imagegen<sup>1</sup>) | `Jonathan1909/Ming-flash-omni-2.0` | ✅︎ | | | | — |
|
||||
| `BagelForConditionalGeneration` | BAGEL (DiT-only) | `ByteDance-Seed/BAGEL-7B-MoT` | ✅︎ | ✅︎ | | ✅︎ | — |
|
||||
| `InternVLAA1Pipeline` | InternVLA-A1 | `InternRobotics/InternVLA-A1-3B` | ✅︎ | ✅︎ | | | — |
|
||||
| `Gr00tN1d7Pipeline` | GR00T N1.7 | `nvidia/GR00T-N1.7-3B` | ✅︎ | | | | — |
|
||||
| `HunyuanImage3ForCausalMM` | HunyuanImage3.0 (DiT-only) | `tencent/HunyuanImage-3.0`, `tencent/HunyuanImage-3.0-Instruct` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
|
||||
| `QwenImagePipeline` | Qwen-Image | `Qwen/Qwen-Image` | ✅︎ | | | ✅︎ | [Published](https://recipes.vllm.ai/Qwen/Qwen-Image) |
|
||||
| `QwenImagePipeline` | Qwen-Image-2512 | `Qwen/Qwen-Image-2512` | ✅︎ | | | ✅︎ | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/Qwen/Qwen-Image-2512.md) |
|
||||
| `QwenImagePipeline` | Qwen-Image | `Qwen/Qwen-Image` | ✅︎ | | | | [Published](https://recipes.vllm.ai/Qwen/Qwen-Image) |
|
||||
| `QwenImagePipeline` | Qwen-Image-2512 | `Qwen/Qwen-Image-2512` | ✅︎ | | | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/Qwen/Qwen-Image-2512.md) |
|
||||
| `QwenImageEditPipeline` | Qwen-Image-Edit | `Qwen/Qwen-Image-Edit` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
|
||||
| `QwenImageEditPlusPipeline` | Qwen-Image-Edit-2509 | `Qwen/Qwen-Image-Edit-2509` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
|
||||
| `QwenImageLayeredPipeline` | Qwen-Image-Layered | `Qwen/Qwen-Image-Layered` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
|
||||
| `QwenImageEditPlusPipeline` | Qwen-Image-Edit-2511 | `Qwen/Qwen-Image-Edit-2511` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
|
||||
| `GlmImagePipeline` | GLM-Image | `zai-org/GLM-Image` | ✅︎ | ✅︎ | | | — |
|
||||
| `ZImagePipeline` | Z-Image | `Tongyi-MAI/Z-Image-Turbo` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
|
||||
| `Krea2Pipeline` | Krea 2 (Raw + Turbo) | `krea/Krea-2-Raw`, `krea/Krea-2-Turbo` | ✅︎ | | | ✅︎ | — |
|
||||
| `Krea2Pipeline` | Krea 2 (Raw + Turbo) | `krea/Krea-2-Raw`, `krea/Krea-2-Turbo` | ✅︎ | | | | — |
|
||||
| `WanPipeline` | Wan2.1-T2V, Wan2.2-T2V, Wan2.2-TI2V | `Wan-AI/Wan2.1-T2V-1.3B-Diffusers`, `Wan-AI/Wan2.1-T2V-14B-Diffusers`, `Wan-AI/Wan2.2-T2V-A14B-Diffusers`, `Wan-AI/Wan2.2-TI2V-5B-Diffusers` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
|
||||
| `WanImageToVideoPipeline` | Wan2.2-I2V | `Wan-AI/Wan2.2-I2V-A14B-Diffusers` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
|
||||
| `LingBotWorldCausalDMDPipeline` | LingBot-World 2.0 v1 (experimental) | `robbyant/lingbot-world-v2-14b-causal-fast-diffusers` | Experimental | | | | — |
|
||||
@@ -42,7 +42,7 @@ th {
|
||||
| `SanaWmPipeline` | SANA-WM | `BBBBruce/SANA-WM_bidirectional-stage1-diffusers` | ✅︎ | | | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/NVIDIA/SANA-WM.md) |
|
||||
| `Wan22S2VPipeline` | Wan2.2-S2V | `Wan-AI/Wan2.2-S2V-14B` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
|
||||
| `Wan22VACEPipeline` | Wan2.1-VACE | `Wan-AI/Wan2.1-VACE-1.3B-diffusers`, `Wan-AI/Wan2.1-VACE-14B-diffusers` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
|
||||
| `Wan22VACEPipeline` | Wan2.2-VACE | `Pyros13/Wan2.2-VACE-Fun-A14B-Diffusers` | ✅︎ | | | ✅︎ | — |
|
||||
| `Wan22VACEPipeline` | Wan2.2-VACE | `Pyros13/Wan2.2-VACE-Fun-A14B-Diffusers` | ✅︎ | | | | — |
|
||||
| `LTX2Pipeline` | LTX-2 / LTX-2.3 one-stage T2V and I2V | `Lightricks/LTX-2`, `diffusers/LTX-2.3-Diffusers` | ✅︎ | ✅︎ | | | — |
|
||||
| `LTX2TwoStagePipeline` | LTX-2 / LTX-2.3 ordinary two-stage T2V and I2V | `Lightricks/LTX-2`, `diffusers/LTX-2.3-Diffusers` + matching Lightricks LoRA and upsampler | ✅︎ | ✅︎ | | | — |
|
||||
| `LTX2DistilledOneStagePipeline` | LTX-2 / LTX-2.3 merged-distilled one-stage T2V and I2V | `rootonchair/LTX-2-19b-distilled`, `diffusers/LTX-2.3-Distilled-Diffusers` | ✅︎ | ✅︎ | | | — |
|
||||
@@ -58,19 +58,18 @@ th {
|
||||
| `DreamZeroPipeline` | DreamZero-DROID | `GEAR-Dreams/DreamZero-DROID` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
|
||||
| `Pi0Pipeline` | π0 (Pi-Zero) VLA | `lerobot/pi0_base` | ✅︎ | | | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/lerobot/Pi0.md) |
|
||||
| `HeliosPipeline`, `HeliosPyramidPipeline` | Helios | `BestWishYsh/Helios-Base`, `BestWishYsh/Helios-Mid`, `BestWishYsh/Helios-Distilled` | ✅︎ | ✅︎ | ✅︎ | | — |
|
||||
| `MagiHumanPipeline` | MagiHuman | `SII-GAIR/daVinci-MagiHuman-Base-1080p` | ✅︎ | ✅︎ | | | — |
|
||||
| `OvisImagePipeline` | Ovis-Image | `OvisAI/Ovis-Image` | ✅︎ | ✅︎ | | ✅︎ | — |
|
||||
| `LongcatImagePipeline` | LongCat-Image | `meituan-longcat/LongCat-Image` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
|
||||
| `LongCatImageEditPipeline` | LongCat-Image-Edit | `meituan-longcat/LongCat-Image-Edit` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
|
||||
| `LongCatVideoAvatarPipeline` | LongCat-Video-Avatar-1.5 A2V/AI2V (native single-speaker and multi-speaker AI2V/AVC) | `meituan-longcat/LongCat-Video-Avatar-1.5` | ✅︎ | | | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/meituan-longcat/LongCat-Video-Avatar-1.5.md) |
|
||||
| `BooguImagePipeline` | Boogu-Image | `Boogu/Boogu-Image-0.1-Base`, `Boogu/Boogu-Image-0.1-Edit`, `Boogu/Boogu-Image-0.1-Edit-Turbo` | ✅︎ | | | ✅︎ | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/Boogu/Boogu-Image.md) |
|
||||
| `BooguImagePipeline` | Boogu-Image | `Boogu/Boogu-Image-0.1-Base`, `Boogu/Boogu-Image-0.1-Edit` | ✅︎ | | | | — |
|
||||
| `StableDiffusionXLPipeline` | Stable-Diffusion-XL | `stabilityai/stable-diffusion-xl-base-1.0` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
|
||||
| `StableDiffusion3Pipeline` | Stable-Diffusion-3 | `stabilityai/stable-diffusion-3.5-medium` | ✅︎ | ✅︎ | | ✅︎ | — |
|
||||
| `CosyVoice3Model` | CosyVoice3 | `FunAudioLLM/Fun-CosyVoice3-0.5B-2512` | ✅︎ | ✅︎ | | ✅︎ | — |
|
||||
| `OmniVoiceModel` | OmniVoice | `k2-fsa/OmniVoice` | ✅︎ | | | ✅︎ | — |
|
||||
| `VoxCPM2TalkerForConditionalGeneration` | VoxCPM2 | `openbmb/VoxCPM2` | ✅︎ | | | ✅︎ | — |
|
||||
| `OmniVoiceModel` | OmniVoice | `k2-fsa/OmniVoice` | ✅︎ | | | | — |
|
||||
| `VoxCPM2TalkerForConditionalGeneration` | VoxCPM2 | `openbmb/VoxCPM2` | ✅︎ | | | | — |
|
||||
| `DotsTTSForConditionalGeneration` | dots.tts | `rednote-hilab/dots.tts-soar` | ✅︎ | | | | — |
|
||||
| `MammothModa2ForConditionalGeneration` | MammothModa2-Preview | `bytedance-research/MammothModa2-Preview` | ✅︎ | ✅︎ | | | — |
|
||||
| `MammothModa2ForConditionalGeneration` | MammothModa2-Dev (AR-only image understanding) | `bytedance-research/MammothModa2-Dev` | ✅︎ | | | | — |
|
||||
| `Flux2KleinPipeline` | FLUX.2-klein | `black-forest-labs/FLUX.2-klein-4B`, `black-forest-labs/FLUX.2-klein-9B` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
|
||||
| `FluxKontextPipeline` | FLUX.1-Kontext-dev | `black-forest-labs/FLUX.1-Kontext-dev` | ✅︎ | ✅︎ | | | — |
|
||||
| `FluxPipeline` | FLUX.1-dev | `black-forest-labs/FLUX.1-dev` | ✅︎ | ✅︎ | | ✅︎ | — |
|
||||
@@ -78,10 +77,10 @@ th {
|
||||
| `OmniGen2Pipeline` | OmniGen2 | `OmniGen2/OmniGen2` | ✅︎ | ✅︎ | | ✅︎ | — |
|
||||
| `StableAudioPipeline` | Stable-Audio-Open | `stabilityai/stable-audio-open-1.0` | ✅︎ | ✅︎ | | ✅︎ | — |
|
||||
| `MiniMaxMusic3ForConditionalGeneration` | MiniMax Music 3 (text-to-music) | `MiniMaxAI/MiniMax-Music3` | ✅︎ | | | | — |
|
||||
| `Qwen3TTSForConditionalGeneration` | Qwen3-TTS-12Hz-1.7B-CustomVoice | `Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice` | ✅︎ | | | ✅︎ | [Published](https://recipes.vllm.ai/Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice) |
|
||||
| `Qwen3TTSForConditionalGeneration` | Qwen3-TTS-12Hz-1.7B-CustomVoice | `Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice` | ✅︎ | | | | [Published](https://recipes.vllm.ai/Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice) |
|
||||
| `Qwen3TTSForConditionalGeneration` | Qwen3-TTS-12Hz-1.7B-VoiceDesign | `Qwen/Qwen3-TTS-12Hz-1.7B-VoiceDesign` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
|
||||
| `Qwen3TTSForConditionalGeneration` | Qwen3-TTS-12Hz-1.7B-Base | `Qwen/Qwen3-TTS-12Hz-1.7B-Base` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
|
||||
| `MingTTSForConditionalGeneration` | Ming-omni-tts dense 0.5B | `inclusionAI/Ming-omni-tts-0.5B` | ✅︎ | | | ✅︎ | — |
|
||||
| `MingTTSForConditionalGeneration` | Ming-omni-tts dense 0.5B | `inclusionAI/Ming-omni-tts-0.5B` | ✅︎ | | | | — |
|
||||
| `GLMTTSForConditionalGeneration` | GLM-TTS | `zai-org/GLM-TTS` | ✅︎ | | | | — |
|
||||
| `MossTTSNanoForCausalLM` | MOSS-TTS-Nano | `OpenMOSS-Team/MOSS-TTS-Nano` | ✅︎ | | | | — |
|
||||
| `MossTTSDelayModel` | MOSS-TTS, MOSS-TTSD, MOSS-SoundEffect, MOSS-VoiceGenerator | `OpenMOSS-Team/MOSS-VoiceGenerator` | ✅︎ | | | | — |
|
||||
@@ -95,16 +94,17 @@ th {
|
||||
| `MiMoV2ASRForCausalLM` | MiMo-V2.5-ASR | `XiaomiMiMo/MiMo-V2.5-ASR` | ✅︎ | ✅︎ | | | — |
|
||||
| `Flux2Pipeline` | FLUX.2-dev | `black-forest-labs/FLUX.2-dev` | ✅︎ | ✅︎ | | | — |
|
||||
| `FishSpeechSlowARForConditionalGeneration` | Fish Speech S2 Pro | `fishaudio/s2-pro` | ✅︎ | ✅︎ | | ✅︎ | — |
|
||||
| `SenseNovaU1Pipeline` | SenseNova-U1 (DiT-only), SenseNova-U1.5 | `SenseNova/SenseNova-U1-8B-MoT`, `sensenova/SenseNova-U1.5-8B-MoT` | ✅︎ | | | | [Repository](https://github.com/vllm-project/vllm-omni/blob/main/recipes/SenseNova/SenseNova-U1.5.md) |
|
||||
| `DreamIDOmniPipeline` | DreamID-Omni | `XuGuo699/DreamID-Omni` | ✅︎ | ✅︎ | | | — |
|
||||
| `SenseNovaU1Pipeline` | SenseNova-U1 (DiT-only) | `SenseNova/SenseNova-U1-8B-MoT` | ✅︎ | | | | — |
|
||||
| `LancePipeline` | Lance | `bytedance-research/Lance` | ✅︎ | | | | — |
|
||||
| `HunyuanVideo15Pipeline` | HunyuanVideo-1.5-T2V | `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v`, `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-720p_t2v` | ✅︎ | ✅︎ | | | — |
|
||||
| `HunyuanVideo15ImageToVideoPipeline` | HunyuanVideo-1.5-I2V | `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_i2v`, `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-720p_i2v` | ✅︎ | ✅︎ | | | — |
|
||||
| `VoxtralTTSForConditionalGeneration` | Voxtral TTS | `mistralai/Voxtral-4B-TTS-2603` | ✅︎ | ✅︎ | | | — |
|
||||
| `CovoAudioForConditionalGeneration` | Covo-Audio-Chat | `tencent/Covo-Audio-Chat` | ✅︎ | | | | — |
|
||||
|`DyninOmniForConditionalGeneration` | Dynin-Omni | `snu-aidas/Dynin-Omni` | ✅︎ | | | ✅︎ | — |
|
||||
|`DyninOmniForConditionalGeneration` | Dynin-Omni | `snu-aidas/Dynin-Omni` | ✅︎ | | | | — |
|
||||
| `MiniCPMO45OmniForConditionalGeneration` | MiniCPM-o 4.5 | `openbmb/MiniCPM-o-4_5` | ✅︎ | | ✅︎ | | — |
|
||||
| `ErnieImagePipeline` | ERNIE-Image | `baidu/ERNIE-Image`, `baidu/ERNIE-Image-Turbo` | ✅︎ | ✅︎ | ✅︎ | ✅︎ | — |
|
||||
| `GepardTalkerForConditionalGeneration` | Gepard-1.0 | `nineninesix/gepard-1.0` | ✅︎ | | | ✅︎ | — |
|
||||
| `GepardTalkerForConditionalGeneration` | Gepard-1.0 | `nineninesix/gepard-1.0` | ✅︎ | | | | — |
|
||||
|`HiDreamImagePipeline` | HiDream-I1-Full | `HiDream-ai/HiDream-I1-Full` | ✅︎ | ✅︎ | | | — |
|
||||
| `HiDreamO1ImagePipeline` | HiDream-O1-Image | `HiDream-ai/HiDream-O1-Image` | ✅︎ | | | | — |
|
||||
|
||||
|
||||
@@ -1,187 +0,0 @@
|
||||
# API Server
|
||||
|
||||
vLLM-Omni exposes OpenAI-compatible HTTP APIs plus vLLM-Omni extensions for
|
||||
image, audio, video, realtime, and robot-policy workloads. Use this page to
|
||||
choose an endpoint; follow the linked reference page for model-specific fields,
|
||||
wire protocols, and response details.
|
||||
|
||||
Each server instance hosts one model. An endpoint is usable only when the
|
||||
loaded model supports that task.
|
||||
|
||||
This guide focuses on vLLM-Omni's primary task endpoints. Compatible models
|
||||
may also expose standard routes inherited from vLLM, such as
|
||||
`POST /v1/completions`; those routes remain model- and task-dependent.
|
||||
|
||||
## Start and Verify the Server
|
||||
|
||||
Start a model with the shared serving command:
|
||||
|
||||
```bash
|
||||
vllm serve <model> --omni --port 8091
|
||||
```
|
||||
|
||||
Some models require additional flags or a deployment configuration. Use the
|
||||
model-specific guide when one is provided.
|
||||
|
||||
After the server starts, check its health and served model name:
|
||||
|
||||
```bash
|
||||
export VLLM_OMNI_BASE_URL=http://localhost:8091
|
||||
|
||||
curl "$VLLM_OMNI_BASE_URL/health"
|
||||
curl "$VLLM_OMNI_BASE_URL/v1/models" | jq .
|
||||
```
|
||||
|
||||
Use `http://localhost:8091/v1` as the `base_url` for an OpenAI SDK client. If
|
||||
the server was started with `--api-key`, include an `Authorization` header with
|
||||
`Bearer <api-key>` in requests.
|
||||
|
||||
## Choose a Core HTTP Endpoint
|
||||
|
||||
Prefer a task-specific endpoint when one matches your workload. Use Chat
|
||||
Completions for conversational or heterogeneous omni pipelines, rather than as
|
||||
the default wrapper for every generation task.
|
||||
|
||||
| Task | Endpoint | Request | Response | Details |
|
||||
|------|----------|---------|----------|---------|
|
||||
| Conversation or multimodal understanding/generation | `POST /v1/chat/completions` | JSON | OpenAI-style JSON or SSE | [Chat Completions](chat_completions_api.md) |
|
||||
| Text-to-speech | `POST /v1/audio/speech` | JSON | Audio bytes or SSE | [Speech](speech_api.md) |
|
||||
| Sound, music, or ambient audio generation | `POST /v1/audio/generate` | JSON | Audio bytes | [Audio Generation](audio_generate_api.md) |
|
||||
| Text-to-image | `POST /v1/images/generations` | JSON | Base64 JSON or an image file | [Image Generation](image_generation_api.md) |
|
||||
| Image editing | `POST /v1/images/edits` | Multipart form | Base64 JSON or SSE | [Image Edit](image_edit_api.md) |
|
||||
| Video generation (recommended) | `POST /v1/videos` | Multipart form | Asynchronous job JSON | [Videos](videos_api.md) |
|
||||
| Video generation (blocking) | `POST /v1/videos/sync` | Multipart form | Video bytes | [Videos](videos_api.md#synchronous-response) |
|
||||
|
||||
`POST /v1/videos/sync` is intended for tests and simple scripts. Use the
|
||||
asynchronous job API for production requests so clients can poll, download,
|
||||
and delete generated videos independently.
|
||||
|
||||
## Minimal Requests
|
||||
|
||||
The examples below assume a compatible model is already running. TTS voices,
|
||||
media inputs, generation controls, and output capabilities vary by model.
|
||||
|
||||
=== "Chat"
|
||||
|
||||
```bash
|
||||
curl "$VLLM_OMNI_BASE_URL/v1/chat/completions" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"messages": [{"role": "user", "content": "Describe vLLM-Omni briefly."}],
|
||||
"modalities": ["text"]
|
||||
}'
|
||||
```
|
||||
|
||||
=== "Speech"
|
||||
|
||||
```bash
|
||||
curl "$VLLM_OMNI_BASE_URL/v1/audio/speech" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"input": "Hello from vLLM-Omni.",
|
||||
"voice": "vivian"
|
||||
}' \
|
||||
--output speech.wav
|
||||
```
|
||||
|
||||
=== "Audio"
|
||||
|
||||
```bash
|
||||
curl "$VLLM_OMNI_BASE_URL/v1/audio/generate" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"input": "Ocean waves on a quiet beach",
|
||||
"audio_length": 5.0
|
||||
}' \
|
||||
--output audio.wav
|
||||
```
|
||||
|
||||
=== "Image"
|
||||
|
||||
```bash
|
||||
curl "$VLLM_OMNI_BASE_URL/v1/images/generations" \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"prompt": "A small robot reading beside a window",
|
||||
"size": "1024x1024",
|
||||
"response_format": "file"
|
||||
}' \
|
||||
--output image.png
|
||||
```
|
||||
|
||||
=== "Image edit"
|
||||
|
||||
```bash
|
||||
curl "$VLLM_OMNI_BASE_URL/v1/images/edits" \
|
||||
-F "image=@input.png" \
|
||||
-F "prompt=Turn the background into a snowy mountain" \
|
||||
| jq -r '.data[0].b64_json' \
|
||||
| base64 --decode > edited.png
|
||||
```
|
||||
|
||||
=== "Video"
|
||||
|
||||
```bash
|
||||
curl "$VLLM_OMNI_BASE_URL/v1/videos" \
|
||||
-F "prompt=A cinematic tracking shot of a mountain lake at sunrise" \
|
||||
-F "size=1280x720" \
|
||||
-F "seconds=5"
|
||||
```
|
||||
|
||||
The response contains a video job `id`. Poll and download it with:
|
||||
|
||||
```bash
|
||||
curl "$VLLM_OMNI_BASE_URL/v1/videos/<video_id>"
|
||||
curl -L "$VLLM_OMNI_BASE_URL/v1/videos/<video_id>/content" --output video.mp4
|
||||
```
|
||||
|
||||
## Choose a Streaming or Realtime Endpoint
|
||||
|
||||
These WebSocket APIs have different event schemas and cannot be used
|
||||
interchangeably.
|
||||
|
||||
| Workload | Endpoint | Interaction model | Details |
|
||||
|----------|----------|-------------------|---------|
|
||||
| Incremental text input for speech synthesis | `WS /v1/audio/speech/stream` | Send text events and receive audio | [Streaming Text to Speech](speech_api.md#streaming-text-input-websocket) |
|
||||
| Live video understanding | `WS /v1/video/chat/stream` | Send video frames and receive text/audio | [Streaming Video Input](video_stream_api.md) |
|
||||
| Turn-based realtime audio | `WS /v1/realtime` | Stream one audio input and receive transcript/audio events | [Realtime Audio](realtime_api.md) |
|
||||
| Continuous speech-to-speech interaction | `WS /v1/realtime?duplex=1` or `WS /v1/duplex` | Listen and speak concurrently with session control | [Full Duplex](full_duplex_api.md) |
|
||||
| Generated video chunks | `WS /v1/realtime/video` | Start a diffusion request and receive fragmented MP4 | [Streaming Video Output](streaming_video_output_api.md) |
|
||||
| Robot policy inference | `WS /v1/realtime/robot/openpi` | Send MessagePack observations and receive action arrays | [OpenPI Robot Policy](openpi_api.md) |
|
||||
|
||||
All six routes are model- or configuration-dependent. In particular,
|
||||
`/v1/realtime` is not full duplex unless the client sets `duplex=1` and the
|
||||
deployment explicitly enables duplex sessions. Clients should also verify the
|
||||
duplex capability payload because the query-parameter form falls back to the
|
||||
ordinary realtime handler when duplex is unavailable.
|
||||
|
||||
## Related Endpoints
|
||||
|
||||
| Purpose | Endpoints | Reference |
|
||||
|---------|-----------|-----------|
|
||||
| Discovery and readiness | `GET /health`, `GET /v1/models` | This page |
|
||||
| Batched conversations | `POST /v1/chat/completions/batch` | [Batch requests](chat_completions_api.md#batch-requests) |
|
||||
| Batched speech | `POST /v1/audio/speech/batch` | [Batch speech generation](speech_api.md#batch-speech-generation) |
|
||||
| TTS voice management | `GET/POST /v1/audio/voices`, `DELETE /v1/audio/voices/{name}` | [Voices](speech_api.md#voices-endpoint) |
|
||||
| Video job lifecycle | `GET /v1/videos`, `GET/DELETE /v1/videos/{video_id}`, `GET /v1/videos/{video_id}/content` | [Video endpoints](videos_api.md#endpoints) |
|
||||
| Release and restore stage memory | `POST /v1/omni/sleep`, `POST /v1/omni/wakeup` | [Sleep Mode](../features/sleep_mode.md) |
|
||||
|
||||
## Standalone Experimental Servers
|
||||
|
||||
PersonaPlex also provides a Moshi-compatible WebSocket server, while JoyVL
|
||||
provides a stateful interaction orchestrator in front of another model server.
|
||||
These are separate processes with their own routes; they are not additional
|
||||
paths on every `vllm serve --omni` instance. See [Standalone Experimental
|
||||
Servers](standalone_servers.md) before deploying either one.
|
||||
|
||||
## Compatibility Notes
|
||||
|
||||
- A registered path does not mean every model supports it. Unsupported tasks
|
||||
return an OpenAI-style error or a WebSocket `error` event.
|
||||
- Use JSON for Chat, Speech, Audio Generation, and Image Generation. Use
|
||||
`multipart/form-data` when uploading images, videos, or other files.
|
||||
- Generation can take minutes for large diffusion models. Configure client
|
||||
timeouts accordingly; prefer asynchronous video jobs over long blocking
|
||||
requests.
|
||||
- Model-specific parameters are vLLM-Omni extensions. Check the endpoint
|
||||
reference before assuming an OpenAI SDK exposes them directly.
|
||||
@@ -4,14 +4,14 @@ vLLM-Omni provides an API for text-to-audio generation using diffusion-based mod
|
||||
|
||||
Unlike the [Speech API](speech_api.md) which targets text-to-speech synthesis, the Audio Generate API is designed for general-purpose audio generation from text descriptions (sound effects, music, ambient soundscapes, etc.).
|
||||
|
||||
Each server instance runs a single model (specified at startup via `vllm serve <model> --omni`).
|
||||
Each server instance runs a single model (specified at startup via `vllm-omni serve <model> --omni`).
|
||||
|
||||
## Quick Start
|
||||
|
||||
### Start the Server
|
||||
|
||||
```bash
|
||||
vllm serve stabilityai/stable-audio-open-1.0 \
|
||||
vllm-omni serve stabilityai/stable-audio-open-1.0 \
|
||||
--host 0.0.0.0 \
|
||||
--port 8091 \
|
||||
--gpu-memory-utilization 0.9 \
|
||||
@@ -326,7 +326,7 @@ curl http://localhost:8091/health
|
||||
Enable debug logging:
|
||||
|
||||
```bash
|
||||
vllm serve stabilityai/stable-audio-open-1.0 \
|
||||
vllm-omni serve stabilityai/stable-audio-open-1.0 \
|
||||
--host 0.0.0.0 \
|
||||
--port 8091 \
|
||||
--gpu-memory-utilization 0.9 \
|
||||
|
||||
@@ -1,110 +1,78 @@
|
||||
# Chat Completions API
|
||||
|
||||
Use `/v1/chat/completions` for conversational and multimodal pipelines that
|
||||
follow the OpenAI Chat API schema. Depending on the loaded model, a request can
|
||||
contain text, image, audio, or video input and can return text, audio, images,
|
||||
or other model-specific output.
|
||||
|
||||
Some diffusion pipelines also support image generation and editing through
|
||||
Chat Completions. Prefer the dedicated [Image Generation
|
||||
API](image_generation_api.md) or [Image Edit API](image_edit_api.md) for
|
||||
task-specific clients; their request fields and response contracts are more
|
||||
direct.
|
||||
|
||||
## Basic Request
|
||||
|
||||
```bash
|
||||
curl http://localhost:8091/v1/chat/completions \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"messages": [
|
||||
{"role": "user", "content": "Describe vLLM-Omni briefly."}
|
||||
],
|
||||
"modalities": ["text"]
|
||||
}'
|
||||
```
|
||||
|
||||
Use `stream: true` for Server-Sent Events. Input media syntax, supported output
|
||||
`modalities`, and response choices depend on the model; see the
|
||||
[model-specific online serving examples](../user_guide/examples/online_serving/qwen3_omni.md).
|
||||
vLLM-Omni supports generating and editing images via the `/v1/chat/completions`
|
||||
endpoint using diffusion models. This page explains how to pass generation
|
||||
parameters (such as `num_inference_steps`, `height`, `width`) to diffusion
|
||||
models through this endpoint.
|
||||
|
||||
!!! tip
|
||||
Each server hosts one model. Query `GET /v1/models` for its served model
|
||||
name when your client requires the `model` field.
|
||||
For dedicated endpoints that accept generation parameters as top-level
|
||||
fields, see [Image Generation API](image_generation_api.md) and
|
||||
[Image Edit API](image_edit_api.md).
|
||||
|
||||
## vLLM Extension Parameters
|
||||
## Passing Generation Parameters
|
||||
|
||||
Send standard and vLLM-specific Chat parameters as top-level fields in a
|
||||
direct HTTP request. When using the OpenAI Python SDK, pass vLLM-specific
|
||||
fields through the SDK's `extra_body` keyword; the SDK merges them into the
|
||||
top-level JSON sent to the server.
|
||||
The `/v1/chat/completions` endpoint follows the OpenAI Chat API schema, which
|
||||
does not natively include diffusion-specific fields like `num_inference_steps`
|
||||
or `height`. How you pass these extra fields depends on your client.
|
||||
|
||||
=== "curl"
|
||||
### curl / Python `requests`
|
||||
|
||||
```bash
|
||||
curl http://localhost:8091/v1/chat/completions \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"messages": [{"role": "user", "content": "Write a short haiku."}],
|
||||
"top_k": 40
|
||||
}'
|
||||
```
|
||||
|
||||
=== "OpenAI Python SDK"
|
||||
|
||||
```python
|
||||
from openai import OpenAI
|
||||
|
||||
client = OpenAI(base_url="http://localhost:8091/v1", api_key="none")
|
||||
|
||||
response = client.chat.completions.create(
|
||||
model="your-served-model",
|
||||
messages=[{"role": "user", "content": "Write a short haiku."}],
|
||||
extra_body={"top_k": 40},
|
||||
)
|
||||
```
|
||||
|
||||
??? note "Diffusion compatibility"
|
||||
|
||||
Diffusion models exposed through Chat Completions can accept supported
|
||||
fields such as `num_inference_steps`, `seed`, `height`, and `width`. Send
|
||||
them at the top level with direct HTTP, or in the SDK's `extra_body`
|
||||
keyword argument.
|
||||
|
||||
A literal nested `"extra_body"` JSON object is accepted for compatibility,
|
||||
but it is not recommended for new direct HTTP clients. Do not provide the
|
||||
same parameter in multiple locations; the server returns a `400` error for
|
||||
duplicate diffusion parameters. Prefer the dedicated [Image Generation
|
||||
API](image_generation_api.md) or [Image Edit API](image_edit_api.md) when
|
||||
either endpoint matches the task.
|
||||
|
||||
## Batch Requests
|
||||
|
||||
`POST /v1/chat/completions/batch` accepts the same shared generation fields,
|
||||
but `messages` is a list of conversations. The response contains one choice
|
||||
per conversation in input order.
|
||||
Wrap generation parameters inside an `"extra_body"` key in the JSON body:
|
||||
|
||||
```bash
|
||||
curl http://localhost:8091/v1/chat/completions/batch \
|
||||
curl -s http://localhost:8091/v1/chat/completions \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"messages": [
|
||||
[{"role": "user", "content": "Summarize vLLM in one sentence."}],
|
||||
[{"role": "user", "content": "Summarize vLLM-Omni in one sentence."}]
|
||||
{"role": "user", "content": "A beautiful landscape painting"}
|
||||
],
|
||||
"max_tokens": 64
|
||||
"extra_body": {
|
||||
"num_inference_steps": 50,
|
||||
"seed": 42
|
||||
}
|
||||
}'
|
||||
```
|
||||
|
||||
Batch Chat Completions does not support streaming, tools, beam search, or
|
||||
`n > 1`.
|
||||
### OpenAI Python SDK
|
||||
|
||||
Use the `extra_body` **keyword argument**. The SDK automatically merges these
|
||||
fields into the top-level request body:
|
||||
|
||||
```python
|
||||
response = client.chat.completions.create(
|
||||
model="Qwen/Qwen-Image",
|
||||
messages=[{"role": "user", "content": "A beautiful landscape painting"}],
|
||||
extra_body={
|
||||
"num_inference_steps": 50,
|
||||
"seed": 42,
|
||||
},
|
||||
)
|
||||
```
|
||||
|
||||
!!! note "SDK `extra_body` vs. JSON `extra_body`"
|
||||
These two `extra_body` usages look similar but work differently under the
|
||||
hood. The SDK flattens the dict into the top-level request JSON, while the
|
||||
curl/requests approach sends it as a nested `"extra_body"` key. Both are
|
||||
handled correctly by the server.
|
||||
|
||||
!!! note "About the `ignored fields` warning"
|
||||
You may see a log message like:
|
||||
|
||||
```
|
||||
WARNING: The following fields were present in the request but ignored: {'height', 'width', ...}
|
||||
```
|
||||
|
||||
This is **harmless**. It is emitted by vLLM's request validation layer
|
||||
because these fields are not part of the standard OpenAI
|
||||
`ChatCompletionRequest` schema. The fields are still stored internally
|
||||
and correctly forwarded to the diffusion pipeline.
|
||||
|
||||
## Model-Specific Examples
|
||||
|
||||
For complete examples with model-specific inputs and outputs, see:
|
||||
For complete examples with full request/response details, see the model-specific
|
||||
guides:
|
||||
|
||||
- [Qwen3-Omni](../user_guide/examples/online_serving/qwen3_omni.md)
|
||||
- [Qwen2.5-Omni](../user_guide/examples/online_serving/qwen2_5_omni.md)
|
||||
- [Text-to-Image (Qwen-Image)](../user_guide/examples/online_serving/text_to_image.md)
|
||||
- [Image-to-Image (Qwen-Image-Edit, Qwen-Image-Layered)](../user_guide/examples/online_serving/image_to_image.md)
|
||||
- [GLM-Image](../user_guide/examples/online_serving/glm_image.md)
|
||||
@@ -1,116 +0,0 @@
|
||||
# Full-Duplex WebSocket API
|
||||
|
||||
vLLM-Omni provides an experimental full-duplex runtime for models that can
|
||||
continue receiving speech while producing speech. It adds persistent session
|
||||
state, model-specific turn policy, overlap handling, playback acknowledgement,
|
||||
and optional session resume.
|
||||
|
||||
Full duplex is distinct from the turn-based [Realtime Audio API](realtime_api.md).
|
||||
|
||||
## Choose an Endpoint
|
||||
|
||||
| Endpoint | Protocol | Recommended use |
|
||||
|----------|----------|-----------------|
|
||||
| `WS /v1/realtime?duplex=1` | OpenAI Realtime-style event projection | Applications and browser clients |
|
||||
| `WS /v1/duplex` | Native vLLM-Omni duplex events | Runtime integration and low-level testing |
|
||||
|
||||
Both endpoints use the same duplex engine and require the same model-side
|
||||
adapter. Prefer the Realtime projection unless the native lifecycle events are
|
||||
specifically required.
|
||||
|
||||
## Enable Full Duplex
|
||||
|
||||
The route becomes usable only when the deployment configuration explicitly
|
||||
sets:
|
||||
|
||||
```yaml
|
||||
session_mode: duplex
|
||||
```
|
||||
|
||||
The selected model pipeline must also provide a duplex serving adapter.
|
||||
Model-native deployments configure the corresponding engine runtime extension
|
||||
and control plane as part of their registered pipeline.
|
||||
|
||||
!!! warning
|
||||
|
||||
`WS /v1/duplex` fails with `Duplex API is not available` when duplex is not
|
||||
enabled. By contrast, `/v1/realtime?duplex=1` falls back to the ordinary
|
||||
turn-based realtime handler when no duplex handler exists. Confirm that
|
||||
`session.created.session.capabilities` is present before treating the
|
||||
connection as full duplex.
|
||||
|
||||
The current unified-runtime integrations are:
|
||||
|
||||
- MiniCPM-o 4.5, using `vllm_omni/deploy/minicpmo_4_5_duplex.yaml`;
|
||||
- PersonaPlex, whose default `vllm_omni/deploy/personaplex.yaml` enables duplex.
|
||||
|
||||
JoyVL is a separate HTTP interaction orchestrator and does not use these
|
||||
WebSocket endpoints. See [Standalone Experimental Servers](standalone_servers.md).
|
||||
|
||||
## MiniCPM-o Quick Start
|
||||
|
||||
Start the duplex deployment:
|
||||
|
||||
```bash
|
||||
vllm serve openbmb/MiniCPM-o-4_5 --omni \
|
||||
--deploy-config vllm_omni/deploy/minicpmo_4_5_duplex.yaml \
|
||||
--trust-remote-code \
|
||||
--port 8091
|
||||
```
|
||||
|
||||
Stream a mono, PCM16, 16 kHz WAV file with the provided client:
|
||||
|
||||
```bash
|
||||
python examples/online_serving/minicpmo/realtime_duplex_demo.py \
|
||||
--url 'ws://localhost:8091/v1/realtime?duplex=1' \
|
||||
--model openbmb/MiniCPM-o-4_5 \
|
||||
--input-wav input_16k_mono.wav \
|
||||
--ref-audio reference_voice.wav \
|
||||
--output-dir /tmp/minicpmo-duplex
|
||||
```
|
||||
|
||||
## Realtime Event Lifecycle
|
||||
|
||||
A typical `/v1/realtime?duplex=1` session follows this lifecycle:
|
||||
|
||||
1. Send `session.update` with the model, modalities, audio formats, and session
|
||||
options.
|
||||
2. Wait for `session.created`; inspect `session.capabilities` instead of
|
||||
assuming every model supports the same controls.
|
||||
3. Send `input_audio_buffer.append` events while microphone audio arrives.
|
||||
4. Send `input_audio_buffer.commit` at a user-turn boundary when required by
|
||||
the model policy.
|
||||
5. Consume `response.created`, transcript deltas, `response.audio.delta`, and
|
||||
`response.done` or `response.listen` events.
|
||||
6. Send `playback.ack` after audio has been played when the session advertises
|
||||
playback acknowledgement support.
|
||||
7. Send `session.close` and wait for `session.closed`.
|
||||
|
||||
Unlike the turn-based realtime endpoint, input may continue while a response
|
||||
is active. The server can emit `overlap.decision` to describe whether input was
|
||||
deferred, treated as a short acknowledgement, or used to interrupt output.
|
||||
|
||||
## Capabilities and Model Differences
|
||||
|
||||
The `session.created` payload includes capability fields such as
|
||||
`supports_barge_in`, `supports_playback_ack`, `supports_multi_session`,
|
||||
`supports_session_resume`, and `chunk_period_ms`. Treat this payload as the
|
||||
runtime contract.
|
||||
|
||||
For example, PersonaPlex supports native overlapping speech but currently
|
||||
advertises `supports_barge_in=false`; destructive output interruption and
|
||||
model-state rewind have not been validated for that integration. Capacity and
|
||||
session-resume behavior also depend on the selected deployment configuration.
|
||||
|
||||
## Native Protocol
|
||||
|
||||
`WS /v1/duplex` exposes lower-level lifecycle names including
|
||||
`session.create`, `input_audio_buffer.append`, `turn.signal`, `playback.ack`,
|
||||
and `session.close`. It returns native session, input, response, overlap, and
|
||||
error events. This protocol is experimental and may evolve with the runtime;
|
||||
applications should use the provided Realtime client where possible.
|
||||
|
||||
See the [MiniCPM-o example](https://github.com/vllm-project/vllm-omni/tree/main/examples/online_serving/minicpmo),
|
||||
[PersonaPlex example](https://github.com/vllm-project/vllm-omni/tree/main/examples/online_serving/personaplex),
|
||||
and [full-duplex runtime design](https://github.com/vllm-project/vllm-omni/blob/main/vllm_omni/experimental/fullduplex/DESIGN.md)
|
||||
for model-specific validation and architecture details.
|
||||
@@ -1,109 +0,0 @@
|
||||
# OpenPI Robot Policy WebSocket API
|
||||
|
||||
Use `WS /v1/realtime/robot/openpi` for low-latency robot-policy inference with
|
||||
OpenPI-compatible clients. A persistent WebSocket carries observations to the
|
||||
loaded policy model and returns action arrays.
|
||||
|
||||
Despite the `/realtime` prefix, this endpoint does not use the OpenAI Realtime
|
||||
event schema. It uses binary MessagePack frames with NumPy extensions.
|
||||
|
||||
## Availability
|
||||
|
||||
The endpoint is available only when the loaded diffusion policy configuration
|
||||
contains `policy_server_config`. The server sends that model-specific
|
||||
configuration to the client as the first binary frame.
|
||||
|
||||
For example, DreamZero declares:
|
||||
|
||||
```yaml
|
||||
policy_server_config:
|
||||
image_resolution: [180, 320]
|
||||
n_external_cameras: 2
|
||||
needs_wrist_camera: true
|
||||
needs_stereo_camera: false
|
||||
needs_session_id: true
|
||||
action_space: joint_position
|
||||
```
|
||||
|
||||
Without this configuration, the WebSocket returns `Robot policy not available`
|
||||
and closes.
|
||||
|
||||
## DreamZero Quick Start
|
||||
|
||||
Start the bundled DreamZero deployment:
|
||||
|
||||
```bash
|
||||
vllm serve GEAR-Dreams/DreamZero-DROID --omni --port 8091 \
|
||||
--served-model-name dreamzero-droid \
|
||||
--deploy-config vllm_omni/deploy/dreamzero_tp1_cfg2.yaml \
|
||||
--enforce-eager --disable-log-stats
|
||||
```
|
||||
|
||||
Install the optional client dependencies, download the sample camera inputs,
|
||||
and run the client:
|
||||
|
||||
```bash
|
||||
pip install openpi-client websockets opencv-python
|
||||
|
||||
hf download YangshenDeng/vllm-omni-dreamzero-assets \
|
||||
--repo-type dataset \
|
||||
--local-dir outputs/dreamzero/assets
|
||||
|
||||
python examples/online_serving/dreamzero/openpi_client.py \
|
||||
--host 127.0.0.1 \
|
||||
--port 8091 \
|
||||
--video-dir outputs/dreamzero/assets
|
||||
```
|
||||
|
||||
## Protocol
|
||||
|
||||
The connection is request-response after an initial server handshake:
|
||||
|
||||
```text
|
||||
connect
|
||||
<- msgpack(policy_server_config)
|
||||
|
||||
infer
|
||||
-> msgpack({"endpoint": "infer", "session_id": "...", ...observation})
|
||||
<- msgpack(ndarray | dict[str, ndarray])
|
||||
|
||||
reset
|
||||
-> msgpack({"endpoint": "reset"})
|
||||
<- msgpack({"status": "reset successful"})
|
||||
```
|
||||
|
||||
If `endpoint` is omitted, the server treats the message as `infer`. Each
|
||||
inference message produces one action response after the engine request
|
||||
completes; action tokens or intermediate tensors are not streamed.
|
||||
|
||||
NumPy arrays use the marker format implemented by `openpi-client`. The server
|
||||
also accepts the legacy vLLM NumPy marker representation on input. JSON text
|
||||
frames are not observation messages.
|
||||
|
||||
## Sessions and Reset
|
||||
|
||||
- The API layer's current-session and first-call counters are scoped to each
|
||||
WebSocket connection.
|
||||
- `session_id` identifies model-side state across observations on that
|
||||
connection. If omitted, it defaults to `default`.
|
||||
- The first inference for a session is sent to the model with `reset=true`.
|
||||
- Changing `session_id`, or sending the `reset` command, causes the next
|
||||
inference to start with `reset=true`.
|
||||
- The policy pipeline owns observation transforms and persistent model state,
|
||||
normally keyed by `session_id`; the API layer forwards the raw observation
|
||||
dictionary.
|
||||
|
||||
## Limits and Errors
|
||||
|
||||
- Maximum inbound payload size is 64 MiB.
|
||||
- The server closes an idle connection after 30 seconds.
|
||||
- Invalid binary input returns a MessagePack
|
||||
`{"type":"error","message":"Invalid request payload"}` response.
|
||||
- Inference failures return a generic `Internal inference error` without
|
||||
exposing an internal traceback.
|
||||
- Unsupported NumPy object, structured, and complex dtypes are rejected.
|
||||
|
||||
Observation keys, camera layout, state tensors, and action shapes are defined
|
||||
by the loaded policy rather than by this transport. Read the handshake before
|
||||
constructing observations. See the [DreamZero example](https://github.com/vllm-project/vllm-omni/tree/main/examples/online_serving/dreamzero)
|
||||
for a complete OpenPI client and DROID simulation loop.
|
||||
@@ -1,82 +0,0 @@
|
||||
# Realtime Audio WebSocket API
|
||||
|
||||
Use `WS /v1/realtime` to stream PCM audio into a compatible omni model and
|
||||
receive incremental transcription and synthesized response audio. This is a
|
||||
turn-based streaming API: it starts one generation over an audio stream and
|
||||
finishes after the client closes that input stream.
|
||||
|
||||
For a model that listens while it is already speaking, use the experimental
|
||||
[Full-Duplex API](full_duplex_api.md) instead.
|
||||
|
||||
## Quick Start
|
||||
|
||||
Start a compatible model. The currently documented example uses Qwen3-Omni:
|
||||
|
||||
```bash
|
||||
vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091
|
||||
```
|
||||
|
||||
Run the provided client with a mono, 16-bit PCM, 16 kHz WAV file:
|
||||
|
||||
```bash
|
||||
python examples/online_serving/qwen3_omni/openai_realtime_client.py \
|
||||
--url ws://localhost:8091/v1/realtime \
|
||||
--model Qwen/Qwen3-Omni-30B-A3B-Instruct \
|
||||
--input-wav input_16k_mono.wav \
|
||||
--output-wav response.wav \
|
||||
--output-text response.txt
|
||||
```
|
||||
|
||||
The client dependency is `websockets`.
|
||||
|
||||
## Protocol
|
||||
|
||||
Messages in both directions are JSON text frames. Audio carried inside an
|
||||
event is base64-encoded raw PCM16.
|
||||
|
||||
| Direction | Event | Purpose |
|
||||
|-----------|-------|---------|
|
||||
| Server to client | `session.created` | Confirms that the WebSocket connection is ready |
|
||||
| Client to server | `session.update` | Selects and validates the served model |
|
||||
| Client to server | `input_audio_buffer.commit` with `final: false` | Starts generation over the incoming stream |
|
||||
| Client to server | `input_audio_buffer.append` | Appends base64 PCM16 audio |
|
||||
| Client to server | `input_audio_buffer.commit` with `final: true` | Marks the end of input |
|
||||
| Server to client | `transcription.delta` | Carries incremental response text |
|
||||
| Server to client | `transcription.done` | Carries final text and token usage |
|
||||
| Server to client | `response.audio.delta` | Carries incremental PCM16 response audio |
|
||||
| Server to client | `response.audio.done` | Marks the end of response audio |
|
||||
| Server to client | `error` | Reports an invalid event, model, or audio payload |
|
||||
|
||||
A minimal client sends events in this order:
|
||||
|
||||
```json
|
||||
{"type":"session.update","model":"Qwen/Qwen3-Omni-30B-A3B-Instruct"}
|
||||
{"type":"input_audio_buffer.commit","final":false}
|
||||
{"type":"input_audio_buffer.append","audio":"<base64-pcm16>"}
|
||||
{"type":"input_audio_buffer.commit","final":true}
|
||||
```
|
||||
|
||||
The initial non-final commit intentionally starts generation before all audio
|
||||
has arrived. Continue sending `input_audio_buffer.append` events while the
|
||||
engine is consuming the stream.
|
||||
|
||||
## Audio Handling
|
||||
|
||||
- Input is mono PCM16 at 16 kHz for the Qwen3-Omni example.
|
||||
- `response.audio.delta.audio` contains base64-encoded PCM16 bytes.
|
||||
- Read `sample_rate_hz` from each audio event instead of assuming an output
|
||||
rate. Qwen3-Omni output is typically 24 kHz.
|
||||
- Concatenate audio deltas in receive order to construct the output waveform.
|
||||
- Set `input_audio_buffer.commit.final` to `true` after the last input chunk so
|
||||
the server can terminate the streaming request.
|
||||
|
||||
## Availability and Limitations
|
||||
|
||||
The path is registered on the unified API server, but it is usable only when
|
||||
the loaded pipeline implements realtime audio input and produces compatible
|
||||
audio output. Unsupported deployments return an `error` event. The endpoint
|
||||
does not provide duplex session resume, playback acknowledgement, overlap
|
||||
policy, or barge-in controls.
|
||||
|
||||
See the [Qwen3-Omni online serving example](https://github.com/vllm-project/vllm-omni/tree/main/examples/online_serving/qwen3_omni)
|
||||
for concurrency options, chunk pacing, and per-delta audio debugging.
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
vLLM-Omni provides an OpenAI-compatible API for text-to-speech (TTS) generation. Supported TTS models include:
|
||||
|
||||
- **Qwen3-TTS** (`Qwen/Qwen3-TTS-12Hz-*`) -- Qwen3-based TTS with CustomVoice, VoiceDesign, and Base (voice cloning) task types. Native output: 24 kHz; API output can be resampled to 8 kHz.
|
||||
- **Qwen3-TTS** (`Qwen/Qwen3-TTS-12Hz-*`) -- Qwen3-based TTS with CustomVoice, VoiceDesign, and Base (voice cloning) task types. Output: 24 kHz.
|
||||
- **Fish Speech S2 Pro** (`fishaudio/s2-pro`) -- Dual-AR TTS with DAC codec. Supports text-to-speech and voice cloning via reference audio. Output: 44.1 kHz.
|
||||
- **Voxtral TTS** (`mistralai/Voxtral-4B-TTS-2603`) -- AR + FlowMatching TTS with preset voices. Output: 24 kHz.
|
||||
- **CosyVoice3** (`FunAudioLLM/Fun-CosyVoice3-0.5B-2512`) -- 2-stage talker + flow-matching code2wav. Voice cloning via `ref_audio` + `ref_text` (no presets). Output: 24 kHz.
|
||||
@@ -116,7 +116,6 @@ Content-Type: application/json
|
||||
|
||||
| Parameter | Type | Default | Description |
|
||||
|-----------|------|---------|-------------|
|
||||
| `sample_rate` | integer | model native | Target output sample rate. Qwen3-TTS supports 8000 or 24000 Hz; the model remains native 24 kHz internally and is resampled before encoding. |
|
||||
| `task_type` | string | "CustomVoice" | TTS task type: CustomVoice, VoiceDesign, or Base |
|
||||
| `language` | string | "Auto" | Language (see supported languages below) |
|
||||
| `instructions` | string | "" | Voice style/emotion instructions |
|
||||
@@ -812,51 +811,6 @@ If you encounter OOM errors:
|
||||
|
||||
Use `/v1/audio/voices` to list available voices for the loaded model.
|
||||
|
||||
## Orchestration Loop (experimental)
|
||||
|
||||
Multi-stage omni deployments route stage outputs through a single orchestrator
|
||||
loop. By default that loop polls every stage replica on a 1 ms cadence. An
|
||||
opt-in event-driven mode replaces the poll with one reader task per live stage
|
||||
replica awaiting its client directly, and switches the serving-side
|
||||
final-output drain to a condition-variable wakeup at the same time.
|
||||
|
||||
**Configuration (environment variables):**
|
||||
|
||||
| Variable | Default | Description |
|
||||
|----------|---------|-------------|
|
||||
| `VLLM_OMNI_EVENT_DRIVEN_ORCH` | `0` (off) | Switches the orchestration loop and the final-output drain from the legacy 1 ms poll to event-driven wakeups. Enabled by `1`, `true`, `yes`, or `on`, matched case-insensitively after surrounding whitespace is stripped; any other value leaves it off. |
|
||||
|
||||
Set it on the process that runs the orchestrator (stage 0 of an omni
|
||||
deployment) before starting the server:
|
||||
|
||||
```bash
|
||||
export VLLM_OMNI_EVENT_DRIVEN_ORCH=1
|
||||
vllm serve Qwen/Qwen3-TTS-12Hz-1.7B-Base \
|
||||
--omni \
|
||||
--port 8091
|
||||
```
|
||||
|
||||
The server logs the selected loop mode and its reader/poller counts once at
|
||||
startup, so you can confirm which loop is live.
|
||||
|
||||
Routing, output ordering, and terminal-state behavior are identical on both
|
||||
loops; only the poll cadence changes. Leaving the variable unset keeps the
|
||||
legacy poll loop, which is the supported default.
|
||||
|
||||
**Known limitations:**
|
||||
|
||||
- The measured serving A/B (idle CPU 2.43% to 0.07%; TTFP p99 -32% at
|
||||
concurrency 8) predates the rebuild on the per-replica fault-isolation work
|
||||
in [#4285](https://github.com/vllm-project/vllm-omni/pull/4285). That work
|
||||
changed dead-replica handling and reader/poller lifecycle rather than the
|
||||
steady-state output path, and the parity suite covers it, but the serving
|
||||
A/B has not been re-run on the current head.
|
||||
- The diffusion-poller branch is covered by unit tests only. Deployments whose
|
||||
stages all run as standard engine cores never exercise it, including GLM-TTS,
|
||||
which deploys its DiT without `stage_type: diffusion`.
|
||||
- Concurrency 1 and 32 measured at parity with the legacy loop. At 32 the
|
||||
latency is admission-bound, which this mode does not address.
|
||||
|
||||
## Development
|
||||
|
||||
Enable debug logging:
|
||||
|
||||
@@ -1,109 +0,0 @@
|
||||
# Standalone Experimental Servers
|
||||
|
||||
Most vLLM-Omni endpoints run on the unified server started with
|
||||
`vllm serve <model> --omni`. The experimental full-duplex package also contains
|
||||
two separate user-facing server processes. Their routes are not added to the
|
||||
unified API server.
|
||||
|
||||
| Server | Transport | Role |
|
||||
|--------|-----------|------|
|
||||
| PersonaPlex compatibility server | WebSocket | Hosts the official PersonaPlex browser protocol and a raw-PCM alternative |
|
||||
| JoyVL interaction server | HTTP | Adds state, memory, proactive decisions, and delegation in front of a separate OpenAI-compatible model backend |
|
||||
|
||||
## PersonaPlex Compatibility Server
|
||||
|
||||
Use this server when a client requires the official Moshi/PersonaPlex wire
|
||||
protocol or the bundled browser UI. For the engine-owned unified duplex path,
|
||||
use [`WS /v1/realtime?duplex=1`](full_duplex_api.md) instead.
|
||||
|
||||
Start it with:
|
||||
|
||||
```bash
|
||||
HF_TOKEN=... CUDA_VISIBLE_DEVICES=0 python -m \
|
||||
vllm_omni.experimental.fullduplex.personaplex.serving.server \
|
||||
--port 8091
|
||||
```
|
||||
|
||||
| Route | Purpose |
|
||||
|-------|---------|
|
||||
| `GET /` | Official PersonaPlex browser client, when its assets are available |
|
||||
| `GET /health` | Readiness check |
|
||||
| `WS /api/chat` | Moshi-compatible tagged binary protocol with Opus audio |
|
||||
| `WS /v1/audio/duplex` | Simpler float32 raw-PCM protocol |
|
||||
|
||||
### `/api/chat` Protocol
|
||||
|
||||
The first byte of every binary frame is a message tag:
|
||||
|
||||
| Direction | Tag | Payload |
|
||||
|-----------|-----|---------|
|
||||
| Server to client | `0x00` | Ready handshake |
|
||||
| Client to server | `0x01` | Opus microphone audio at 24 kHz |
|
||||
| Server to client | `0x01` | Opus response audio |
|
||||
| Server to client | `0x02` | UTF-8 inner-monologue text |
|
||||
|
||||
The optional `text_prompt` and `voice_prompt` query parameters select a persona
|
||||
and voice. The default server admits one conversation; use `--batch-size N` to
|
||||
configure multiple lockstep slots. Connections beyond capacity close with
|
||||
WebSocket code 1013.
|
||||
|
||||
### `/v1/audio/duplex` Protocol
|
||||
|
||||
Send `{"type":"open","persona":"...","voice":"..."}` as a JSON text frame,
|
||||
then exchange binary float32 PCM frames. The server returns generated float32
|
||||
audio as binary frames and text as `{"type":"text","text":"..."}`. Send
|
||||
`{"type":"close"}` and wait for `{"type":"done"}` to flush and finish.
|
||||
|
||||
See the [PersonaPlex example](https://github.com/vllm-project/vllm-omni/tree/main/examples/online_serving/personaplex)
|
||||
for the browser and headless clients.
|
||||
|
||||
## JoyVL Interaction Server
|
||||
|
||||
JoyVL is an orchestration layer, not a model-serving engine. It calls a
|
||||
separate OpenAI-compatible backend and adds per-session frame history, memory,
|
||||
persona policy, and optional delegation.
|
||||
|
||||
Start the model backend and orchestrator separately:
|
||||
|
||||
```bash
|
||||
vllm serve jdopensource/JoyAI-VL-Interaction-Preview \
|
||||
--served-model-name JoyAI-VL-Interaction-Preview \
|
||||
--port 8092 \
|
||||
--max-model-len 131072 \
|
||||
--enable-prefix-caching \
|
||||
--limit-mm-per-prompt '{"image":256,"video":1}'
|
||||
|
||||
python -m vllm_omni.experimental.fullduplex.joyvl.serving.server \
|
||||
--port 8091 \
|
||||
--main-backend-url http://127.0.0.1:8092/v1 \
|
||||
--main-model JoyAI-VL-Interaction-Preview
|
||||
```
|
||||
|
||||
| Method and route | Purpose |
|
||||
|------------------|---------|
|
||||
| `GET /health` | Readiness check |
|
||||
| `GET /v1/models` | Reports the configured interaction model |
|
||||
| `POST /v1/chat/completions` | Processes one frame or interaction tick |
|
||||
| `POST /reset` | Resets a session |
|
||||
| `POST /v1/streaming/reset` | Alias for `/reset` |
|
||||
| `POST /v1/streaming/persona` | Changes the session persona |
|
||||
|
||||
Send one multimodal Chat Completions request per video frame, normally around
|
||||
one frame per second, and identify the session with `x-session-id`. Responses
|
||||
include an `interaction` block whose action is `silence`, `response`, or
|
||||
`delegate`.
|
||||
|
||||
```bash
|
||||
curl http://127.0.0.1:8091/v1/chat/completions \
|
||||
-H 'content-type: application/json' \
|
||||
-H 'x-session-id: demo' \
|
||||
-d '{
|
||||
"messages": [{"role": "user", "content": [
|
||||
{"type": "text", "text": "Alert me if a fire appears"},
|
||||
{"type": "image_url", "image_url": {"url": "data:image/jpeg;base64,..."}}
|
||||
]}]
|
||||
}'
|
||||
```
|
||||
|
||||
See the [JoyVL recipe](https://github.com/vllm-project/vllm-omni/blob/main/recipes/JD/JoyAI-VL-Interaction.md)
|
||||
for memory, personas, delegation backends, and the browser UI.
|
||||
@@ -1,92 +0,0 @@
|
||||
# Streaming Video Output WebSocket API
|
||||
|
||||
Use `WS /v1/realtime/video` to receive fragmented MP4 chunks while a compatible
|
||||
diffusion video model is generating. This endpoint streams generated video
|
||||
output; it is unrelated to [`WS /v1/video/chat/stream`](video_stream_api.md),
|
||||
which streams live video input into a conversational model.
|
||||
|
||||
## Quick Start
|
||||
|
||||
Start a model that supports step execution and enable diffusion streaming
|
||||
output:
|
||||
|
||||
```bash
|
||||
vllm serve BestWishYsh/Helios-Distilled --omni \
|
||||
--diffusion-streaming-output \
|
||||
--port 8091
|
||||
```
|
||||
|
||||
Install the client dependency and run the provided client:
|
||||
|
||||
```bash
|
||||
pip install av websockets
|
||||
|
||||
python examples/online_serving/streaming_video_generation/streaming_video_client.py \
|
||||
--host 127.0.0.1 \
|
||||
--port 8091 \
|
||||
--model BestWishYsh/Helios-Distilled \
|
||||
--prompt "A serene lakeside sunrise with mist over the water" \
|
||||
--output output.mp4
|
||||
```
|
||||
|
||||
## Protocol
|
||||
|
||||
Control events are JSON text frames. Each `video.chunk_metadata` event is
|
||||
immediately followed by one binary WebSocket frame containing fragmented MP4
|
||||
bytes.
|
||||
|
||||
| Direction | Event or frame | Purpose |
|
||||
|-----------|----------------|---------|
|
||||
| Client to server | `session.start` | Starts generation with a prompt and video parameters |
|
||||
| Server to client | `video.start` | Confirms the request ID, format, and accepted configuration |
|
||||
| Server to client | `video.chunk_metadata` | Describes the next binary media frame |
|
||||
| Server to client | Binary frame | Carries one fragmented MP4 chunk |
|
||||
| Client to server | `session.interaction` | Queues an optional prompt update during generation |
|
||||
| Server to client | `session.interaction.queued` | Confirms that an interaction was queued |
|
||||
| Client to server | `session.ping` | Refreshes the stall clock |
|
||||
| Server to client | `session.pong` | Acknowledges a ping |
|
||||
| Client to server | `session.stop` | Cancels the active generation |
|
||||
| Server to client | `session.done` | Reports completion or cancellation and the chunk count |
|
||||
| Server to client | `error` | Reports validation, generation, or timeout failures |
|
||||
|
||||
Start a minimal session with:
|
||||
|
||||
```json
|
||||
{
|
||||
"type": "session.start",
|
||||
"model": "BestWishYsh/Helios-Distilled",
|
||||
"prompt": "A serene lakeside sunrise",
|
||||
"format": "m4s",
|
||||
"width": 640,
|
||||
"height": 384,
|
||||
"fps": 16,
|
||||
"num_frames": 99
|
||||
}
|
||||
```
|
||||
|
||||
Only `m4s` output is currently accepted. Concatenate the binary frames in
|
||||
receive order, then remux the fragmented stream if the target player requires
|
||||
a progressive MP4 file.
|
||||
|
||||
## Mid-Generation Interaction
|
||||
|
||||
Models that implement interaction updates can change the active prompt:
|
||||
|
||||
```json
|
||||
{
|
||||
"type": "session.interaction",
|
||||
"interaction": {
|
||||
"event_id": "sunlight-update",
|
||||
"event": {"prompt": "Sunlight breaks through the mist"},
|
||||
"transition_chunks": 2
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
This feature is pipeline-dependent. An enabled endpoint does not imply that
|
||||
the loaded model supports midway prompt updates.
|
||||
|
||||
The default start timeout is 10 seconds and the server reports a stall after
|
||||
about 60 seconds without generation progress or a `session.ping`. See the
|
||||
[streaming video generation example](https://github.com/vllm-project/vllm-omni/tree/main/examples/online_serving/streaming_video_generation)
|
||||
for prompt-update scheduling and browser playback.
|
||||
@@ -17,8 +17,6 @@ globally.
|
||||
| --- | --- | --- |
|
||||
| `sparsity` | finite, `[0, 1]` | Nominal dropped-key-block fraction; default `0.8`; `0` disables sparsity |
|
||||
| `start_step` | integer, `>= 0` | Number of early denoise steps kept dense |
|
||||
| `end_step` | integer, `>= 0` | Number of final denoise steps kept dense (tail fallback) |
|
||||
| `precision` | `"bf16"`, `"fp8"`, `"mix"` | Kernel precision mode; default `"bf16"`. Requires MindIE-SD with `sparse_attention(precision=...)` |
|
||||
| `skip_layers` | selector such as `"0-3,38"` | DiT blocks kept dense |
|
||||
|
||||
```bash
|
||||
@@ -54,12 +52,6 @@ platform raises. It is incompatible with ring sequence parallelism because
|
||||
the kernel needs the complete key sequence for ranking; use Ulysses sequence
|
||||
parallelism with `ring_degree=1`.
|
||||
|
||||
The `precision` knob requires a MindIE-SD release whose `sparse_attention`
|
||||
accepts `precision=`; older releases accept it through `**kwargs` but
|
||||
silently ignore it and run the BF16 path. `RAINFUSION_ATTN` raises at runtime
|
||||
when a non-`bf16` precision is requested without that support, so pin a
|
||||
compatible MindIE-SD release when using `precision="mix"` or `"fp8"`.
|
||||
|
||||
## Geometry handling
|
||||
|
||||
RainFusion handles arbitrary video grids by rearranging video spatially and,
|
||||
|
||||
@@ -12,17 +12,15 @@ LoRA adapters are lightweight, model-specific fine-tuning weights that can be ap
|
||||
## LoRA Adapter Format
|
||||
|
||||
### PEFT (Parameter-Efficient Fine-Tuning) format (default)
|
||||
|
||||
A typical PEFT-format LoRA adapter directory structure:
|
||||
|
||||
```text
|
||||
```
|
||||
lora_adapter/
|
||||
├── adapter_config.json
|
||||
└── adapter_model.safetensors
|
||||
```
|
||||
|
||||
The `adapter_config.json` file contains metadata about the LoRA adapter, including:
|
||||
|
||||
- `r`: LoRA rank
|
||||
- `lora_alpha`: LoRA alpha scaling factor
|
||||
- `target_modules`: List of module names to apply LoRA to
|
||||
@@ -69,7 +67,7 @@ For distilled few-step LoRAs, pass `lora_backend="distill"` together with one or
|
||||
For Qwen-Image and Wan pipelines, the distill backend calls `pipeline.load_lora_weights(...)` during worker initialization.
|
||||
|
||||
| Pipeline | Supported distilled LoRA repo | Notes |
|
||||
| ---------- | -------------------------------- | ------- |
|
||||
|----------|--------------------------------|-------|
|
||||
| `QwenImagePipeline` | `lightx2v/Qwen-Image-2512-Lightning` | Used with Qwen-Image-2512 Lightning-style few-step inference. |
|
||||
| `Wan22Pipeline` | `lightx2v/Wan2.1-Distill-Loras`, `lightx2v/Wan2.2-Distill-Loras` | Wan2.1 uses one LoRA file. For dual-transformer Wan2.2 MoE, pass high-noise then low-noise LoRA files. |
|
||||
| `Wan22I2VPipeline` | `lightx2v/Wan2.2-Distill-Loras` | For dual-transformer Wan2.2 MoE, pass high-noise then low-noise LoRA files. |
|
||||
@@ -242,37 +240,6 @@ Notes:
|
||||
- Output quality and speed depend on the replacement checkpoints and sampling params you choose.
|
||||
- If you only need to fuse distilled LoRAs into a Wan2.2 checkpoint at load time (without the full LightX2V convert + assemble pipeline), you can instead pass them directly via `--lora-backend distill --lora-path <high>.safetensors <low>.safetensors`. See the [Distill backend](#distill-backend-fuse-distilled-lora-at-init) section above.
|
||||
|
||||
## MiniMax-H3 adapter-declared schedules
|
||||
|
||||
MiniMax-H3 supports two few-step mechanisms that must not be conflated:
|
||||
|
||||
- **Checkpoint-pinned schedule**: a merged release writes `base_schedule` into
|
||||
`model_index.json`. Requests must pass `num_inference_steps` as the interval
|
||||
count (for example `4` for `[1.0, 0.7, 0.4, 0.15, 0.0]`).
|
||||
- **Runtime Turbo LoRA**: LightX2V Turbo artifacts request five sigma points and
|
||||
enforce `flow_shift=6`, `audio_flow_shift=3`.
|
||||
- **Runtime native LoRA**: FlashGen-style artifacts declare
|
||||
`key_format=minimax-h3-native` and embed `base_schedule` in safetensors
|
||||
metadata. When active, the adapter schedule overrides the base checkpoint and
|
||||
requests must use the interval-count contract (`num_inference_steps=4`).
|
||||
Request-mode generation may omit the field and take the count from the adapter
|
||||
schedule; step execution requires it explicitly, because the step scheduler
|
||||
derives the total step count from the request before the adapter schedule is
|
||||
known.
|
||||
|
||||
Native artifacts also declare `qkv_layout=grouped`. The H3 loader reorders fused
|
||||
`qkv_proj` LoRA rows with the same `_reorder_grouped_qkv_to_qkv` helper used for
|
||||
base-weight loading, then binds the packed Q/K/V slices through the legacy PEFT
|
||||
manager without modifying `DiffusionLoRAManager`.
|
||||
|
||||
Both runtime adapters run with distributed layerwise offload in request-mode
|
||||
generation, where the manager keeps the LoRA A/B buffers resident on the compute
|
||||
device while DLO streams the base blocks. MiniMax-H3 step execution still
|
||||
rejects DLO, so `--step-execution` cannot be combined with
|
||||
`--enable-distributed-layerwise-offload`. Model-level CPU offload and standard
|
||||
layerwise offload are rejected in every mode, because the dynamic LoRA tensors
|
||||
are neither parameters nor registered buffers and therefore do not participate
|
||||
in those weight lifecycles.
|
||||
|
||||
## See Also
|
||||
|
||||
|
||||
@@ -36,7 +36,8 @@ vllm serve /path/to/model --omni \
|
||||
vllm serve /path/to/model --omni \
|
||||
--enable-distributed-layerwise-offload \
|
||||
--data-parallel-size 4 \
|
||||
--dlo-no-use-allgather
|
||||
--dlo-no-use-allgather \
|
||||
--dlo-offload-components '{"text_encoder": false}'
|
||||
|
||||
# Sequence parallel deployment
|
||||
vllm serve /path/to/model --omni \
|
||||
@@ -51,6 +52,7 @@ omni = Omni(
|
||||
model="/path/to/model",
|
||||
enable_distributed_layerwise_offload=True,
|
||||
dlo_use_allgather=True,
|
||||
dlo_offload_components={"text_encoder": False},
|
||||
)
|
||||
```
|
||||
|
||||
@@ -63,6 +65,7 @@ omni = Omni(
|
||||
| `--dlo-use-allgather` | Shard host weights and reconstruct with AllGather | `true` |
|
||||
| `--dlo-no-use-allgather` | Stream complete rank-local blocks without a DLO weight collective | `false` |
|
||||
| `--dlo-resident-layers N` | Keep N leading main-DiT blocks on device; requires no-AllGather and model-declared resident paths | `0` |
|
||||
| `--dlo-offload-components JSON` | Permit or disable offload for discovered encoder/VAE components | `{}` |
|
||||
| `--host-weight-runtime-mode {disabled,preferred,required}` | HWR policy: no interaction, populate on a miss, or require an exact hit | `disabled` |
|
||||
| `--host-weight-runtime-root PATH` | Writable node-local HWR store shared by workers in one storage domain; required for `preferred` and `required` | unset |
|
||||
| `--dlo-host-registration-limit-gib N` | Optional per-worker ceiling for registering an HWR mmap; zero adds no ceiling | `0` |
|
||||
@@ -112,12 +115,6 @@ perform a collective and uses the same rank-local transfer behavior.
|
||||
HWR is an opt-in startup optimization for models that declare the final-layout
|
||||
BF16 restore contract. Use it only with no-AllGather DLO:
|
||||
|
||||
The validated BF16 model contracts currently cover MiniMax H3 and
|
||||
`black-forest-labs/FLUX.2-klein-4B`. FLUX.2-klein-9B shares the same model
|
||||
class but has not been validated against this contract. FLUX.2-dev, online
|
||||
FP8, HSDP, LoRA/adapted weights, and non-default load formats remain outside
|
||||
the validated scope.
|
||||
|
||||
```bash
|
||||
vllm serve /path/to/model --omni \
|
||||
--enable-distributed-layerwise-offload \
|
||||
@@ -250,6 +247,25 @@ class MyPipeline(nn.Module):
|
||||
When no plan exists, discovery falls back to
|
||||
`_layerwise_offload_blocks_attrs` and then heuristic attribute lookup.
|
||||
|
||||
## Auxiliary component policy
|
||||
|
||||
`dlo_offload_components` follows the same component-map style as diffusion
|
||||
quantization. Keys are discovered encoder or VAE paths, values are booleans,
|
||||
and `default` is the optional fallback:
|
||||
|
||||
```python
|
||||
dlo_offload_components = {
|
||||
"text_encoder": False,
|
||||
"default": True,
|
||||
}
|
||||
```
|
||||
|
||||
`True` permits the model's existing DLO behavior for that component. `False`
|
||||
keeps the whole component resident, including its child blocks and non-block
|
||||
modules. An empty map preserves current behavior. Unknown keys are rejected;
|
||||
the map does not control DiT blocks because enabling DLO gives the backend
|
||||
ownership of DiT streaming.
|
||||
|
||||
## Data-parallel concurrency
|
||||
|
||||
With `data_parallel_size > 1` and AllGather enabled, the scheduler can process
|
||||
@@ -274,6 +290,13 @@ must enter each collective.
|
||||
are validated.
|
||||
- Resident leading layers require `--dlo-no-use-allgather` and a model
|
||||
`OffloadPlan` that declares eligible `resident_dit_paths`.
|
||||
- Component policy applies only to discovered auxiliary encoders and VAEs;
|
||||
it cannot disable DLO for a DiT.
|
||||
- MiniMax-H3's encoder TP group is global rather than DP-replica-local. An
|
||||
encoder TP size that spans multiple DP replicas (for example TP4 with
|
||||
DP2/TP2 or DP2/SP2 on four ranks) is not supported for independent-prompt
|
||||
AllGather waves: the current pipeline broadcasts rank 0's conditioning to
|
||||
both replicas. This limitation is independent of the component policy.
|
||||
- DP concurrency requires an explicit, identical inference-step count.
|
||||
|
||||
Sharing quantized or otherwise unvalidated transformed layouts through a
|
||||
|
||||
@@ -129,7 +129,7 @@ The following tables show which models support each feature:
|
||||
| **Krea 2** | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ✅ | ✅ | ✅ (decode) | ❌ | ❌ |
|
||||
| **LongCat-Image** | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ |
|
||||
| **LongCat-Image-Edit** | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ |
|
||||
| **MammothModa2(T2I)** | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ |
|
||||
| **MagiHuman** | ❌ | ❌ | ❌ | ❓ | ✅ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ |
|
||||
| **Nextstep_1(T2I)** | ❓ | ❓ | ❌ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ |
|
||||
| **OmniGen2** | ❌ | ✅ | ✅ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ |
|
||||
| **Ovis-Image** | ❌ | ✅ | ❌ | ✅ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ |
|
||||
@@ -138,7 +138,7 @@ The following tables show which models support each feature:
|
||||
| **Qwen-Image-Edit** | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ (decode) | ❌ | ❌ |
|
||||
| **Qwen-Image-Edit-2509** | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ (decode) | ✅ | ❌ | ❌ |
|
||||
| **Qwen-Image-Layered** | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ (decode) | ❌ | ❌ |
|
||||
| **SenseNova-U1 / U1.5** | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ |
|
||||
| **SenseNova-U1** | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ |
|
||||
| **Stable-Diffusion-XL** | ❌ | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ (decode) | ❌ | ❌ |
|
||||
| **Stable-Diffusion3.5** | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ✅ | ✅ (decode) | ❌ | ❌ |
|
||||
| **Z-Image** | ✅ | ✅ | ✅ | ❓ | ✅ (TP=2 only) | ❌ | ✅ | ❌ | ✅ (decode) | ✅ | ❌ |
|
||||
@@ -147,7 +147,7 @@ The following tables show which models support each feature:
|
||||
|
||||
> Notes:
|
||||
> 1. Nextstep_1(T2I) does not support cache acceleration methods such as TeaCache or Cache-DiT.
|
||||
> 2. `Tongyi-MAI/Z-Image-Turbo` is a distilled model with minimal NFEs; CFG-Parallel is not necessary.
|
||||
> 2. `Tongyi-MAI/Z-Image-Turbo` and `SII-GAIR/daVinci-MagiHuman-Base-1080p` are distilled models with minimal NFEs; CFG-Parallel is not necessary.
|
||||
> 3. Cosmos3 T2I uses `Cosmos3OmniDiffusersPipeline` with `modalities=["image"]`. Model-level CPU offload swaps the nested UND reasoner and GEN generator pathways; layerwise offload remains available for blockwise GEN/UND offload.
|
||||
> 4. Krea 2 currently supports single-GPU inference plus LoRA, Cache-DiT, HSDP, CPU/layerwise offload, and VAE-patch-parallel (decode). TP/SP/CFG-Parallel are not yet wired. The few-step distilled (Turbo) checkpoint uses `is_distilled=true` (fixed timestep shift `mu=1.15`); generate at 2048x2048 by default with `num_inference_steps≈8` and `guidance_scale=0`. The Raw checkpoint uses 1024x1024, `num_inference_steps=28`, and `guidance_scale=4.5`.
|
||||
> 5. HunyuanImage3 supports step execution. Multi-request step execution requires `TORCH_SDPA`; see [Diffusion Execution Modes](diffusion/execution_modes.md#step-execution).
|
||||
@@ -165,6 +165,7 @@ The following tables show which models support each feature:
|
||||
| **SANA-Video-2B T2V I2V** | ❓ | ❓ | ❓ | ❓ | ❓ | ❓ | ❓ | ❓ | ❓ | ❓ | ❌ |
|
||||
| **Helios** | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ✅* |
|
||||
| **HunyuanVideo-1.5 T2V I2V** | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ (encode/decode) | ✅ | ❌ |
|
||||
| **DreamID-Omni** | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| **Cosmos3** | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ (encode/decode) | ✅ | ❌ |
|
||||
| **LongCat-Video-Avatar-1.5** | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ |
|
||||
| **MiniMax-H3** | ✅ (FL2VA) | ✅ | ✅ | ❌ | ✅ (DiT/TE) | ❌ | ✅ | ✅ | ✅ (tile) | ✅ (DiT) | ❌ |
|
||||
|
||||
@@ -0,0 +1,115 @@
|
||||
# X-To-Video-Audio
|
||||
|
||||
Source <https://github.com/vllm-project/vllm-omni/tree/main/examples/offline_inference/x_to_video_audio>.
|
||||
|
||||
|
||||
The `DreamID-Omni` pipeline generates short videos from text, image and video.
|
||||
|
||||
## Local CLI Usage
|
||||
### Download the Model locally
|
||||
Since DreamID-Omni combine multiple models, and without any config, so we need to download them locally.
|
||||
|
||||
```bash
|
||||
python download_dreamid_omni.py --output-dir ./dreamid_omni
|
||||
```
|
||||
After download, the model directory will look like this:
|
||||
|
||||
```
|
||||
dreamid_omni/
|
||||
├── DreamID-Omni/
|
||||
│ ├── dreamid_omni.safetensors
|
||||
├── MMAudio/
|
||||
│ ├── ext_weights/
|
||||
│ │ ├── best_netG.pt
|
||||
│ │ ├── v1-16.pth
|
||||
├── Wan2.2-TI2V-5B/
|
||||
│ ├── google/*
|
||||
│ ├── models_t5_umt5-xxl-enc-bf16.pth
|
||||
│ ├── Wan2.2_VAE.pth
|
||||
│
|
||||
├── model_index.json # create by download_dreamid_omni.py
|
||||
```
|
||||
|
||||
### Run the Inference
|
||||
```python
|
||||
python x_to_video_audio.py \
|
||||
--model /path/to/dreamid_omni \
|
||||
--prompt "Two people walking together and singing happily" \
|
||||
--image-path ./example0.png ./example1.png \
|
||||
--audio-path ./example0.wav ./example1.wav \
|
||||
--video-negative-prompt "jitter, bad hands, blur, distortion" \
|
||||
--audio-negative-prompt "robotic, muffled, echo, distorted" \
|
||||
--cfg-parallel-size 2 \
|
||||
--num-inference-steps 45 \
|
||||
--height 704 \
|
||||
--width 1280 \
|
||||
--output out_dreamid_omni_twoip.mp4
|
||||
```
|
||||
In the current test scenario (2 images + 2 audio inputs), the VRAM requirement is 72GB, regardless of whether cfg-parallel is enabled or disabled.
|
||||
The VRAM usage can be reduced by enabling CPU offload via --enable-cpu-offload.
|
||||
For multi-GPU memory reduction on the fused DreamID-Omni transformer, you can also enable HSDP:
|
||||
|
||||
```python
|
||||
python x_to_video_audio.py \
|
||||
--model /path/to/dreamid_omni \
|
||||
--prompt "Two people walking together and singing happily" \
|
||||
--image-path ./example0.png ./example1.png \
|
||||
--audio-path ./example0.wav ./example1.wav \
|
||||
--video-negative-prompt "jitter, bad hands, blur, distortion" \
|
||||
--audio-negative-prompt "robotic, muffled, echo, distorted" \
|
||||
--cfg-parallel-size 2 \
|
||||
--num-inference-steps 45 \
|
||||
--height 704 \
|
||||
--width 1280 \
|
||||
--use-hsdp \
|
||||
--hsdp-shard-size 2 \
|
||||
--output out_dreamid_omni_twoip.mp4
|
||||
```
|
||||
|
||||
|
||||
You could take reference images/audios from the test cases in the official repo: https://github.com/Guoxu1233/DreamID-Omni
|
||||
|
||||
For example, single IP ref resources can be found under https://github.com/Guoxu1233/DreamID-Omni/tree/main/test_case/oneip, you could download them correspondingly to your local and use them for testing.
|
||||
|
||||
```python
|
||||
# Example usage for oneip, ref media from the official repo DreamID-Omni
|
||||
python x_to_video_audio.py \
|
||||
--model /path/to/dreamid_omni \
|
||||
--prompt "<img1>: In the frame, a woman with black long hair is identified as <sub1>.\n**Overall Environment/Scene**: A lively open-kitchen café at night; stove flames flare, steam rises, and warm pendant lights swing slightly as staff move behind her. The shot is an upper-body close-up.\n**Main Characters/Subjects Appearance**: <sub1> is a young woman with thick dark wavy hair and a side part. She wears a fitted black top under a light apron, a thin gold chain necklace, and small stud earrings.\n**Main Characters/Subjects Actions**: <sub1> tastes the sauce with a spoon, then turns her face toward the camera while still holding the spoon, her expression shifting from focused to conflicted.\n<sub1> maintains eye contact, swallows as if choosing her words, and says, <S>I keep telling myself I’m fine,but some nights it feels like I’m just performing calm.<E>" \
|
||||
--image-path 9.png \
|
||||
--audio-path 9.wav \
|
||||
--video-negative-prompt "jitter, bad hands, blur, distortion" \
|
||||
--audio-negative-prompt "robotic, muffled, echo, distorted" \
|
||||
--cfg-parallel-size 2 \
|
||||
--num-inference-steps 45 \
|
||||
--height 704 \
|
||||
--width 1280 \
|
||||
--output out_dreamid_omni_oneip.mp4
|
||||
```
|
||||
|
||||
|
||||
Key arguments:
|
||||
- `--prompt`: text description (string).
|
||||
- `--model`: path to the model local directory.
|
||||
- `--height/--width`: output resolution (defaults 704 * 1024).
|
||||
- `--image-path`: path to the input image list.
|
||||
- `--audio-path`: path to the input audio list, indicate the timbre of the output video.
|
||||
- `--cfg-parallel-size`: number of parallel cfg parallel (defaults 1).
|
||||
- `--use-hsdp`: enable HSDP weight sharding for DreamID-Omni fused blocks.
|
||||
- `--hsdp-shard-size`: number of GPUs used for HSDP sharding.
|
||||
- `--hsdp-replicate-size`: number of HSDP replica groups.
|
||||
- `--num-inference-steps`: number of denoising steps (defaults 45).
|
||||
- `--video-negative-prompt`: negative prompt for video generation.
|
||||
- `--audio-negative-prompt`: negative prompt for audio generation.
|
||||
- `--enable-cpu-offload`: enable CPU offload (defaults False).
|
||||
|
||||
## Example materials
|
||||
|
||||
??? abstract "download_dreamid_omni.py"
|
||||
``````py
|
||||
--8<-- "examples/offline_inference/x_to_video_audio/download_dreamid_omni.py"
|
||||
``````
|
||||
??? abstract "x_to_video_audio.py"
|
||||
``````py
|
||||
--8<-- "examples/offline_inference/x_to_video_audio/x_to_video_audio.py"
|
||||
``````
|
||||
@@ -28,18 +28,6 @@ vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091 \
|
||||
--deploy-config /path/to/deploy_config_file
|
||||
```
|
||||
|
||||
To serve **thinker-only** (text output, no talker / code2wav loaded) with Instruct
|
||||
weights, pass the bundled thinker-only deploy YAML. The `pipeline:` key
|
||||
`qwen3_omni_moe_thinker_only` overrides the HF `enable_audio_output` resolver:
|
||||
|
||||
```bash
|
||||
vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091 \
|
||||
--deploy-config vllm_omni/deploy/qwen3_omni_moe_thinking.yaml
|
||||
```
|
||||
|
||||
Captioner / Thinking checkpoints (`enable_audio_output=false`) still auto-select
|
||||
the same single-stage pipeline without `--deploy-config`.
|
||||
|
||||
### Launch individual stages (stage-based CLI)
|
||||
|
||||
Adopt the stage-based CLI architecture to independently instantiate execution processes per functional stage.
|
||||
|
||||
@@ -1,50 +0,0 @@
|
||||
# SVDQuant W4A4
|
||||
|
||||
## Overview
|
||||
|
||||
[SVDQuant](https://arxiv.org/abs/2411.05007) combines four-bit weights
|
||||
and activations with a small low-rank branch that corrects part of the
|
||||
quantization error. vLLM-Omni consumes an offline-quantized checkpoint;
|
||||
it does not calibrate the model while loading.
|
||||
|
||||
## Checkpoint contract
|
||||
|
||||
Place the following entry in the diffusion transformer's `config.json`:
|
||||
|
||||
```json
|
||||
{
|
||||
"quantization_config": {
|
||||
"quant_method": "svdquant",
|
||||
"rank": 32,
|
||||
"precision": "nvfp4",
|
||||
"act_unsigned": false,
|
||||
"modules_to_not_convert": []
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
For a quantized linear with input size `K`, output size `N`, and correction
|
||||
rank `R`, the checkpoint stores:
|
||||
|
||||
| Suffix | Shape | dtype |
|
||||
| --- | --- | --- |
|
||||
| `qweight` | `(N, K / 2)` | `int8` (two packed FP4 values per byte) |
|
||||
| `wscales` | `(K / 16, N)` | `float8_e4m3fn` |
|
||||
| `proj_down` | `(K, R)` | `bfloat16` |
|
||||
| `proj_up` | `(N, R)` | `bfloat16` |
|
||||
| `smooth_factor` | `(K,)` | `bfloat16` |
|
||||
| `wcscales` | `(N,)` | `bfloat16` |
|
||||
| `wtscale` | `(1,)` | `bfloat16` |
|
||||
|
||||
`K` must be divisible by 16 on every tensor-parallel rank. Set `wcscales` to
|
||||
ones when no per-output correction is needed. Modules listed in
|
||||
`modules_to_not_convert` keep their checkpoint precision.
|
||||
|
||||
## Runtime support
|
||||
|
||||
The compatibility path accepts BF16 inputs and executes an NVFP4 GEMM followed
|
||||
by the BF16 rank correction. It supports vLLM's FlashInfer, CUTLASS, and FBGEMM
|
||||
NVFP4 tensor layouts; incompatible forced backends fail during model loading.
|
||||
SM103 is the currently validated and enabled hardware target. Native fusion of
|
||||
the NVFP4 GEMM and rank correction is separate from this checkpoint-loading
|
||||
contract.
|
||||
@@ -1,9 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
"""vLLM-Omni example scripts, importable as a package.
|
||||
|
||||
Keep this ``__init__.py``: with the upstream vLLM installed editable (which also
|
||||
places an ``examples`` package on ``sys.path``), a regular package at this repo
|
||||
root wins over the upstream ``examples`` namespace during ``import examples``.
|
||||
"""
|
||||
@@ -1,5 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""
|
||||
Image-to-Video generation example using Wan2.2 I2V/TI2V models, LTX2/LTX-2.3,
|
||||
@@ -111,13 +111,6 @@ def build_image_to_video_prompt(
|
||||
return result
|
||||
|
||||
|
||||
def _validate_video_output_type(output_type: str) -> None:
|
||||
if output_type not in {"image", "video"}:
|
||||
raise ValueError(
|
||||
f"Unexpected output type '{output_type}', expected 'video' or legacy 'image' for video generation."
|
||||
)
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(
|
||||
description=(
|
||||
@@ -672,7 +665,10 @@ def main():
|
||||
frames = frames[0] if frames else None
|
||||
|
||||
if isinstance(frames, OmniRequestOutput):
|
||||
_validate_video_output_type(frames.final_output_type)
|
||||
if frames.final_output_type != "image":
|
||||
raise ValueError(
|
||||
f"Unexpected output type '{frames.final_output_type}', expected 'image' for video generation."
|
||||
)
|
||||
if frames.multimodal_output and "audio" in frames.multimodal_output:
|
||||
audio = frames.multimodal_output["audio"]
|
||||
audio_sample_rate = frames.multimodal_output.get("audio_sample_rate", audio_sample_rate)
|
||||
|
||||
@@ -1,117 +0,0 @@
|
||||
"""
|
||||
Offline inference example: MammothModa2 image summarization (single AR stage).
|
||||
|
||||
Example:
|
||||
uv run python examples/offline_inference/mammothmodal2_preview/run_mammothmoda2_image_summarize.py \
|
||||
--model path/to/MammothModa2-Preview \
|
||||
--deploy-config vllm_omni/deploy/mammoth_moda2_ar.yaml \
|
||||
--image /path/to/input.jpg \
|
||||
--question "Please summarize the content of this image."
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import os
|
||||
|
||||
from PIL import Image
|
||||
from vllm import SamplingParams
|
||||
from vllm.multimodal.image import convert_image_mode
|
||||
|
||||
from vllm_omni import Omni
|
||||
|
||||
DEFAULT_SYSTEM = "You are a helpful assistant."
|
||||
DEFAULT_QUESTION = "Please summarize the content of this image."
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description="MammothModa2 image summarization (offline, AR only).")
|
||||
parser.add_argument("--model", type=str, required=True, help="Path to model directory or model id.")
|
||||
parser.add_argument("--deploy-config", type=str, required=True, help="Path to deploy config yaml.")
|
||||
parser.add_argument("--image", type=str, required=True, help="Path to input image.")
|
||||
parser.add_argument("--question", type=str, default=DEFAULT_QUESTION, help="Question/instruction for the model.")
|
||||
parser.add_argument("--system", type=str, default=DEFAULT_SYSTEM, help="System prompt.")
|
||||
parser.add_argument(
|
||||
"--max-tokens",
|
||||
type=int,
|
||||
default=512,
|
||||
help="Max new tokens to generate.",
|
||||
)
|
||||
parser.add_argument("--temperature", type=float, default=0.2)
|
||||
parser.add_argument("--top-p", type=float, default=0.9)
|
||||
parser.add_argument("--seed", type=int, default=42)
|
||||
parser.add_argument("--trust-remote-code", action="store_true")
|
||||
parser.add_argument(
|
||||
"--enable-diffusion-pipeline-profiler",
|
||||
action="store_true",
|
||||
help="Enable diffusion pipeline profiler to display stage durations.",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def build_prompt(system: str, question: str) -> str:
|
||||
return (
|
||||
f"<|im_start|>system\n{system}<|im_end|>\n"
|
||||
"<|im_start|>user\n"
|
||||
"<|vision_start|><|image_pad|><|vision_end|>"
|
||||
f"{question}<|im_end|>\n"
|
||||
"<|im_start|>assistant\n"
|
||||
)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
args = parse_args()
|
||||
|
||||
if not os.path.exists(args.image):
|
||||
raise FileNotFoundError(f"Image file not found: {args.image}")
|
||||
|
||||
pil_image = Image.open(args.image)
|
||||
image_data = convert_image_mode(pil_image, "RGB")
|
||||
prompt = build_prompt(args.system, args.question)
|
||||
|
||||
omni = Omni(
|
||||
model=args.model,
|
||||
deploy_config=args.deploy_config,
|
||||
trust_remote_code=args.trust_remote_code,
|
||||
enable_diffusion_pipeline_profiler=args.enable_diffusion_pipeline_profiler,
|
||||
)
|
||||
try:
|
||||
sp = SamplingParams(
|
||||
temperature=float(args.temperature),
|
||||
top_p=float(args.top_p),
|
||||
top_k=-1,
|
||||
max_tokens=int(args.max_tokens),
|
||||
seed=int(args.seed),
|
||||
detokenize=True,
|
||||
)
|
||||
# NOTE: omni.generate() returns a Generator[OmniRequestOutput, None, None].
|
||||
# Consume it inside the try block so the worker isn't closed early.
|
||||
outputs = list(
|
||||
omni.generate(
|
||||
[
|
||||
{
|
||||
"prompt": prompt,
|
||||
"multi_modal_data": {"image": image_data},
|
||||
"additional_information": {"omni_task": ["chat"]},
|
||||
}
|
||||
],
|
||||
[sp],
|
||||
)
|
||||
)
|
||||
finally:
|
||||
omni.close()
|
||||
|
||||
lines: list[str] = []
|
||||
for stage_outputs in outputs:
|
||||
ro = stage_outputs
|
||||
text = ro.outputs[0].text if getattr(ro, "outputs", None) else str(ro)
|
||||
lines.append(f"request_id: {getattr(ro, 'request_id', 'unknown')}\n")
|
||||
lines.append("answer:\n")
|
||||
lines.append(text.strip() + "\n")
|
||||
lines.append("\n")
|
||||
|
||||
print("\n".join(lines))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,5 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""Validate MiniMax-H3 request-mode and DLO lifecycle behavior.
|
||||
|
||||
The DLO modes accept any positive data-parallel size. With AllGather enabled,
|
||||
@@ -85,12 +85,20 @@ def parse_args() -> argparse.Namespace:
|
||||
parser.add_argument("--height", type=int, default=768)
|
||||
parser.add_argument("--batch-wait-ms", type=float, default=500.0)
|
||||
parser.add_argument("--init-timeout", type=float, default=1800.0)
|
||||
parser.add_argument(
|
||||
"--dlo-offload-components",
|
||||
type=json.loads,
|
||||
default=None,
|
||||
help="Component policy JSON, e.g. '{\"text_encoder\": false}'",
|
||||
)
|
||||
parser.add_argument("--output", type=Path)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def engine_kwargs(args: argparse.Namespace) -> dict[str, Any]:
|
||||
is_dlo = args.mode in {"dlo", "dlo-no-allgather"}
|
||||
if args.dlo_offload_components and not is_dlo:
|
||||
raise ValueError("--dlo-offload-components requires a DLO mode")
|
||||
common: dict[str, Any] = {
|
||||
"model": args.model,
|
||||
"trust_remote_code": True,
|
||||
@@ -114,6 +122,7 @@ def engine_kwargs(args: argparse.Namespace) -> dict[str, Any]:
|
||||
enable_distributed_layerwise_offload=True,
|
||||
dlo_use_allgather=args.mode == "dlo",
|
||||
dlo_resident_layers=0,
|
||||
dlo_offload_components=({} if args.dlo_offload_components is None else args.dlo_offload_components),
|
||||
)
|
||||
else:
|
||||
common.update(
|
||||
|
||||
@@ -599,17 +599,6 @@ def main():
|
||||
if args.seed is not None and hasattr(params, "seed"):
|
||||
params.seed = args.seed
|
||||
|
||||
# MammothModa2's AR stage emits one visual token per grid cell,
|
||||
# one EOL token per row, and one final look-ahead token whose hidden
|
||||
# state is unavailable. Size the first stage from the prompt metadata
|
||||
# instead of SamplingParams' default of 16 tokens.
|
||||
prompt_info = prompt_dict.get("additional_information", {})
|
||||
if idx == 0 and prompt_info.get("omni_task") == ["t2i"]:
|
||||
ar_width = int(prompt_info.get("ar_width", [0])[0])
|
||||
ar_height = int(prompt_info.get("ar_height", [0])[0])
|
||||
if ar_width > 0 and ar_height > 0:
|
||||
params.max_tokens = ar_height * (ar_width + 1) + 1
|
||||
|
||||
if not diffusion_replaced and len(sampling_params_list) == 1:
|
||||
sampling_params_list = [diffusion_params]
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import argparse
|
||||
import json
|
||||
@@ -174,13 +174,6 @@ def build_text_to_video_prompt(prompt: str, negative_prompt: str | None) -> dict
|
||||
return result
|
||||
|
||||
|
||||
def _validate_video_output_type(output_type: str) -> None:
|
||||
if output_type not in {"image", "video"}:
|
||||
raise ValueError(
|
||||
f"Unexpected output type '{output_type}', expected 'video' or legacy 'image' for video generation."
|
||||
)
|
||||
|
||||
|
||||
def _normalize_float_tensor(tensor: torch.Tensor, source_range: str) -> torch.Tensor:
|
||||
"""Normalize decoded frames according to the pipeline's output contract."""
|
||||
if not tensor.is_floating_point():
|
||||
@@ -627,7 +620,10 @@ def main():
|
||||
frames = frames[0] if frames else None
|
||||
|
||||
if isinstance(frames, OmniRequestOutput):
|
||||
_validate_video_output_type(frames.final_output_type)
|
||||
if frames.final_output_type != "image":
|
||||
raise ValueError(
|
||||
f"Unexpected output type '{frames.final_output_type}', expected 'image' for video generation."
|
||||
)
|
||||
if frames.multimodal_output and "audio" in frames.multimodal_output:
|
||||
audio = frames.multimodal_output["audio"]
|
||||
audio_sample_rate = frames.multimodal_output.get("audio_sample_rate", audio_sample_rate)
|
||||
|
||||
@@ -10,12 +10,10 @@ Generate text from text or image inputs with vLLM-Omni's shared offline entrypoi
|
||||
| --- | --- | --- | --- |
|
||||
| `ByteDance-Seed/BAGEL-7B-MoT` | Yes | Yes | `vllm_omni/deploy/bagel.yaml` |
|
||||
| `tencent/HunyuanImage-3.0-Instruct` | Yes | Yes | `vllm_omni/deploy/hunyuan_image3_ar.yaml` |
|
||||
| `bytedance-research/MammothModa2-Preview` | Yes | Yes | `vllm_omni/deploy/mammoth_moda2_ar.yaml` |
|
||||
| `bytedance-research/MammothModa2-Dev` | Yes | Yes | `vllm_omni/deploy/mammoth_moda2_ar.yaml` |
|
||||
|
||||
The script recognizes these model families from `config.json`, applies the
|
||||
model-specific prompt format, and selects an AR-only deploy for HunyuanImage-3
|
||||
and MammothModa2. BAGEL uses its registered default deploy.
|
||||
model-specific prompt format, and selects an AR-only deploy for HunyuanImage-3.
|
||||
BAGEL uses its registered default deploy.
|
||||
|
||||
## Text-To-Text
|
||||
|
||||
@@ -35,22 +33,6 @@ python examples/offline_inference/x_to_text/x_to_text.py \
|
||||
--prompt "Explain multimodal inference in three concise sentences."
|
||||
```
|
||||
|
||||
### MammothModa2
|
||||
|
||||
```bash
|
||||
python examples/offline_inference/x_to_text/x_to_text.py \
|
||||
--model bytedance-research/MammothModa2-Preview \
|
||||
--prompt "Explain multimodal inference in three concise sentences."
|
||||
```
|
||||
|
||||
The same command supports the Dev checkpoint:
|
||||
|
||||
```bash
|
||||
python examples/offline_inference/x_to_text/x_to_text.py \
|
||||
--model bytedance-research/MammothModa2-Dev \
|
||||
--prompt "Explain multimodal inference in three concise sentences."
|
||||
```
|
||||
|
||||
## Image-To-Text
|
||||
|
||||
Pass one image with `--image`. The default question can be replaced with any
|
||||
@@ -72,16 +54,6 @@ python examples/offline_inference/x_to_text/x_to_text.py \
|
||||
--prompt "Describe this image in detail."
|
||||
```
|
||||
|
||||
```bash
|
||||
python examples/offline_inference/x_to_text/x_to_text.py \
|
||||
--model bytedance-research/MammothModa2-Preview \
|
||||
--image image.png \
|
||||
--prompt "Describe this image in detail."
|
||||
```
|
||||
|
||||
Use `bytedance-research/MammothModa2-Dev` in the same command to run I2T with
|
||||
the Dev checkpoint.
|
||||
|
||||
## Key Arguments
|
||||
|
||||
| Argument | Default | Description |
|
||||
@@ -102,4 +74,4 @@ the Dev checkpoint.
|
||||
|
||||
The committed HunyuanImage-3 AR-only default uses four GPUs with TP=4. To use a
|
||||
different topology, pass a custom AR-only YAML through `--deploy-config`.
|
||||
BAGEL and MammothModa2 use their repository deploy defaults unless overridden.
|
||||
BAGEL uses its repository deploy default unless overridden.
|
||||
@@ -15,7 +15,6 @@ from vllm_omni.model_extras import build_x_to_text_prompt, get_x_to_text_model_f
|
||||
|
||||
_REPO_ROOT = Path(__file__).resolve().parents[3]
|
||||
_HUNYUAN_AR_DEPLOY_CONFIG = _REPO_ROOT / "vllm_omni" / "deploy" / "hunyuan_image3_ar.yaml"
|
||||
_MAMMOTH_AR_DEPLOY_CONFIG = _REPO_ROOT / "vllm_omni" / "deploy" / "mammoth_moda2_ar.yaml"
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
@@ -62,7 +61,7 @@ def main() -> None:
|
||||
omni_kwargs: dict[str, Any] = {
|
||||
"model": args.model,
|
||||
"mode": "image-to-text" if image is not None else "text-to-text",
|
||||
"trust_remote_code": args.trust_remote_code or family in {"hunyuan_image3", "mammoth_moda2"},
|
||||
"trust_remote_code": args.trust_remote_code or family == "hunyuan_image3",
|
||||
"log_stats": args.log_stats,
|
||||
}
|
||||
if args.enforce_eager is not None:
|
||||
@@ -71,8 +70,6 @@ def main() -> None:
|
||||
omni_kwargs["deploy_config"] = args.deploy_config
|
||||
elif family == "hunyuan_image3":
|
||||
omni_kwargs["deploy_config"] = str(_HUNYUAN_AR_DEPLOY_CONFIG)
|
||||
elif family == "mammoth_moda2":
|
||||
omni_kwargs["deploy_config"] = str(_MAMMOTH_AR_DEPLOY_CONFIG)
|
||||
|
||||
omni = Omni(**omni_kwargs)
|
||||
try:
|
||||
|
||||
@@ -0,0 +1,108 @@
|
||||
import argparse
|
||||
import fcntl
|
||||
import os
|
||||
import site
|
||||
import subprocess
|
||||
import tempfile
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
from huggingface_hub import snapshot_download
|
||||
|
||||
DEPENDENCY_REPO = "https://github.com/bytedance/DreamID-V.git"
|
||||
DEPENDENCY_BRANCH = "omni"
|
||||
CACHE_DIR = Path(tempfile.gettempdir()) / "vllm-omni-dependency"
|
||||
LOCK_FILE = CACHE_DIR / ".install.lock"
|
||||
DEPENDENCY_DIR = CACHE_DIR / "DreamID-Omni"
|
||||
|
||||
|
||||
def download_dependency():
|
||||
CACHE_DIR.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
with open(LOCK_FILE, "w") as f:
|
||||
fcntl.flock(f, fcntl.LOCK_EX)
|
||||
if not DEPENDENCY_DIR.exists():
|
||||
print(f"Downloading DreamID-Omni to {DEPENDENCY_DIR} ...")
|
||||
subprocess.run(
|
||||
["git", "clone", "--depth", "1", DEPENDENCY_REPO, "--branch", DEPENDENCY_BRANCH, str(DEPENDENCY_DIR)],
|
||||
check=True,
|
||||
)
|
||||
print("Download finished.")
|
||||
fcntl.flock(f, fcntl.LOCK_UN)
|
||||
|
||||
# write .pth to site-packages
|
||||
site_packages = Path(site.getsitepackages()[0])
|
||||
pth_file = site_packages / "vllm_omni_dependency.pth"
|
||||
pth_file.write_text(str(DEPENDENCY_DIR))
|
||||
print(f"Added {DEPENDENCY_DIR} to site-packages via {pth_file}")
|
||||
|
||||
|
||||
def timed_download(repo_id: str, local_dir: str, allow_patterns: list | None = None):
|
||||
"""Download files from HF repo and log time + destination."""
|
||||
if os.path.exists(local_dir):
|
||||
print(f"Directory {local_dir} already exists. Skipping download.")
|
||||
return
|
||||
print(f"Starting download from {repo_id} into {local_dir}")
|
||||
start_time = time.time()
|
||||
|
||||
snapshot_download(
|
||||
repo_id=repo_id,
|
||||
local_dir=local_dir,
|
||||
local_dir_use_symlinks=False,
|
||||
allow_patterns=allow_patterns,
|
||||
)
|
||||
|
||||
elapsed = time.time() - start_time
|
||||
print(f"✅ Finished downloading {repo_id} in {elapsed:.2f} seconds. Files saved at: {local_dir}")
|
||||
|
||||
|
||||
def main(output_dir: str):
|
||||
# Wan2.2
|
||||
wan_dir = os.path.join(output_dir, "Wan2.2-TI2V-5B")
|
||||
timed_download(
|
||||
repo_id="Wan-AI/Wan2.2-TI2V-5B",
|
||||
local_dir=wan_dir,
|
||||
allow_patterns=["google/*", "models_t5_umt5-xxl-enc-bf16.pth", "Wan2.2_VAE.pth"],
|
||||
)
|
||||
|
||||
# MMAudio
|
||||
mm_audio_dir = os.path.join(output_dir, "MMAudio")
|
||||
timed_download(
|
||||
repo_id="hkchengrex/MMAudio",
|
||||
local_dir=mm_audio_dir,
|
||||
allow_patterns=["ext_weights/best_netG.pt", "ext_weights/v1-16.pth"],
|
||||
)
|
||||
|
||||
dreamid_dir = os.path.join(output_dir, "DreamID-Omni")
|
||||
|
||||
timed_download(repo_id="XuGuo699/DreamID-Omni", local_dir=dreamid_dir)
|
||||
|
||||
# Now we construct the config file
|
||||
import json
|
||||
|
||||
data = {
|
||||
"_class_name": "DreamIDOmniPipeline",
|
||||
}
|
||||
|
||||
with open(os.path.join(output_dir, "model_index.json"), "w", encoding="utf-8") as f:
|
||||
json.dump(data, f, indent=2)
|
||||
|
||||
print(f"model_index.json created at {os.path.join(output_dir, 'model_index.json')}")
|
||||
|
||||
transformer_dir = os.path.join(output_dir, "transformer")
|
||||
os.makedirs(transformer_dir, exist_ok=True)
|
||||
with open(os.path.join(transformer_dir, "config.json"), "w", encoding="utf-8") as f:
|
||||
json.dump({"fusion": "DreamID-Omni/dreamid_omni.safetensors"}, f)
|
||||
print(f"transformer/config.json created at {os.path.join(transformer_dir, 'config.json')}")
|
||||
|
||||
# now we download the dependency code
|
||||
download_dependency()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(description="Download models from Hugging Face")
|
||||
parser.add_argument(
|
||||
"--output-dir", type=str, default="./dreamid_omni", help="Base directory to save downloaded models"
|
||||
)
|
||||
args = parser.parse_args()
|
||||
main(args.output_dir)
|
||||
@@ -0,0 +1,163 @@
|
||||
# X-To-Video-Audio
|
||||
|
||||
The `DreamID-Omni` pipeline generates short videos from text, image and video.
|
||||
|
||||
## Local CLI Usage
|
||||
### Download the Model locally
|
||||
Since DreamID-Omni combine multiple models, and without any config, so we need to download them locally.
|
||||
|
||||
```bash
|
||||
python download_dreamid_omni.py --output-dir ./dreamid_omni
|
||||
```
|
||||
After download, the model directory will look like this:
|
||||
|
||||
```
|
||||
dreamid_omni/
|
||||
├── DreamID-Omni/
|
||||
│ ├── dreamid_omni.safetensors
|
||||
├── MMAudio/
|
||||
│ ├── ext_weights/
|
||||
│ │ ├── best_netG.pt
|
||||
│ │ ├── v1-16.pth
|
||||
├── Wan2.2-TI2V-5B/
|
||||
│ ├── google/*
|
||||
│ ├── models_t5_umt5-xxl-enc-bf16.pth
|
||||
│ ├── Wan2.2_VAE.pth
|
||||
│
|
||||
├── model_index.json
|
||||
└── transformer/
|
||||
└── config.json # create by download_dreamid_omni.py
|
||||
```
|
||||
|
||||
### Run the Inference
|
||||
```python
|
||||
python x_to_video_audio.py \
|
||||
--model /path/to/dreamid_omni \
|
||||
--prompt "Two people walking together and singing happily" \
|
||||
--image-path ./example0.png ./example1.png \
|
||||
--audio-path ./example0.wav ./example1.wav \
|
||||
--video-negative-prompt "jitter, bad hands, blur, distortion" \
|
||||
--audio-negative-prompt "robotic, muffled, echo, distorted" \
|
||||
--cfg-parallel-size 2 \
|
||||
--num-inference-steps 45 \
|
||||
--height 704 \
|
||||
--width 1280 \
|
||||
--output out_dreamid_omni_twoip.mp4
|
||||
```
|
||||
In the current test scenario (2 images + 2 audio inputs), the VRAM requirement is 72GB, regardless of whether cfg-parallel is enabled or disabled.
|
||||
The VRAM usage can be reduced by enabling CPU offload via --enable-cpu-offload.
|
||||
For multi-GPU memory reduction on the fused DreamID-Omni transformer, you can also enable HSDP:
|
||||
|
||||
```python
|
||||
python x_to_video_audio.py \
|
||||
--model /path/to/dreamid_omni \
|
||||
--prompt "Two people walking together and singing happily" \
|
||||
--image-path ./example0.png ./example1.png \
|
||||
--audio-path ./example0.wav ./example1.wav \
|
||||
--video-negative-prompt "jitter, bad hands, blur, distortion" \
|
||||
--audio-negative-prompt "robotic, muffled, echo, distorted" \
|
||||
--cfg-parallel-size 2 \
|
||||
--num-inference-steps 45 \
|
||||
--height 704 \
|
||||
--width 1280 \
|
||||
--use-hsdp \
|
||||
--hsdp-shard-size 2 \
|
||||
--output out_dreamid_omni_twoip.mp4
|
||||
```
|
||||
|
||||
|
||||
You could take reference images/audios from the test cases in the official repo: https://github.com/Guoxu1233/DreamID-Omni
|
||||
|
||||
For example, single IP ref resources can be found under https://github.com/Guoxu1233/DreamID-Omni/tree/main/test_case/oneip, you could download them correspondingly to your local and use them for testing.
|
||||
|
||||
```python
|
||||
# Example usage for oneip, ref media from the official repo DreamID-Omni
|
||||
python x_to_video_audio.py \
|
||||
--model /path/to/dreamid_omni \
|
||||
--prompt "<img1>: In the frame, a woman with black long hair is identified as <sub1>.\n**Overall Environment/Scene**: A lively open-kitchen café at night; stove flames flare, steam rises, and warm pendant lights swing slightly as staff move behind her. The shot is an upper-body close-up.\n**Main Characters/Subjects Appearance**: <sub1> is a young woman with thick dark wavy hair and a side part. She wears a fitted black top under a light apron, a thin gold chain necklace, and small stud earrings.\n**Main Characters/Subjects Actions**: <sub1> tastes the sauce with a spoon, then turns her face toward the camera while still holding the spoon, her expression shifting from focused to conflicted.\n<sub1> maintains eye contact, swallows as if choosing her words, and says, <S>I keep telling myself I’m fine,but some nights it feels like I’m just performing calm.<E>" \
|
||||
--image-path 9.png \
|
||||
--audio-path 9.wav \
|
||||
--video-negative-prompt "jitter, bad hands, blur, distortion" \
|
||||
--audio-negative-prompt "robotic, muffled, echo, distorted" \
|
||||
--cfg-parallel-size 2 \
|
||||
--num-inference-steps 45 \
|
||||
--height 704 \
|
||||
--width 1280 \
|
||||
--output out_dreamid_omni_oneip.mp4
|
||||
```
|
||||
|
||||
|
||||
## MagiHuman (`--model-type magi-human`)
|
||||
|
||||
MagiHuman is a text → video+audio model with a DiT MoE backbone and a ~9B-param
|
||||
T5Gemma text encoder. A detailed text prompt is the only required input; an optional
|
||||
image and/or audio file may be supplied for conditioning. Natively supports Tensor
|
||||
Parallelism. For an 80GB node, `--tensor-parallel-size 4` is recommended to shard
|
||||
the MoE weights and the text encoder.
|
||||
|
||||
> Install [MagiCompiler](https://github.com/SandAI-org/MagiCompiler) for correct
|
||||
> attention-kernel behaviour (the pipeline otherwise falls back to stubs).
|
||||
|
||||
Text-only generation:
|
||||
|
||||
```bash
|
||||
python x_to_video_audio.py \
|
||||
--model-type magi-human \
|
||||
--model /path/to/daVinci-MagiHuman \
|
||||
--prompt "A young woman with long, wavy golden blonde hair... <dialogue and background sound>" \
|
||||
--tensor-parallel-size 4 \
|
||||
--height 256 --width 448 \
|
||||
--num-inference-steps 8 \
|
||||
--seed 52 \
|
||||
--extra-body '{"seconds": 5, "sr_height": 1080, "sr_width": 1920, "sr_num_inference_steps": 5}' \
|
||||
--output output_magihuman.mp4
|
||||
```
|
||||
|
||||
With optional image and audio conditioning:
|
||||
|
||||
```bash
|
||||
python x_to_video_audio.py \
|
||||
--model-type magi-human \
|
||||
--model /path/to/daVinci-MagiHuman \
|
||||
--prompt "A young woman..." \
|
||||
--tensor-parallel-size 4 \
|
||||
--height 256 --width 448 \
|
||||
--num-inference-steps 8 \
|
||||
--seed 52 \
|
||||
--extra-body '{"image_path": "/path/to/ref.jpg", "audio_path": "/path/to/ref.wav", "sr_height": 1080, "sr_width": 1920, "sr_num_inference_steps": 5}' \
|
||||
--output output_magihuman.mp4
|
||||
```
|
||||
|
||||
MagiHuman-specific arguments are passed as a JSON dict via `--extra-body` (declared
|
||||
in `vllm_omni/model_extras/magi_human.py`, routed via `extra_args`):
|
||||
|
||||
- `seconds`: output duration in seconds (default 10; ignored when `audio_path` is set,
|
||||
because the audio length then determines the number of frames).
|
||||
- `audio_path`: path to an audio file for audio-to-video conditioning. When provided,
|
||||
the audio drives the frame count and is encoded as a latent condition into the DiT
|
||||
(`is_a2v` mode). Omit for pure text-to-video+audio generation.
|
||||
- `image_path`: path to an image file for visual conditioning. Applied at both the
|
||||
base-resolution (BR) and super-resolution (SR) stages when SR is enabled.
|
||||
- `sr_height` / `sr_width`: super-resolution output resolution. SR stage is skipped
|
||||
when these are omitted.
|
||||
- `sr_num_inference_steps`: denoising steps for the SR stage.
|
||||
|
||||
## DreamID-Omni arguments
|
||||
|
||||
Key arguments:
|
||||
- `--prompt`: text description (string).
|
||||
- `--model`: path to the model local directory.
|
||||
- `--height/--width`: output resolution (defaults 704 * 1024).
|
||||
- `--image-path`: path to the input image list.
|
||||
- `--audio-path`: path to the input audio list, indicate the timbre of the output video.
|
||||
- `--cfg-parallel-size`: number of parallel cfg parallel (defaults 1).
|
||||
- `--use-hsdp`: enable HSDP weight sharding for DreamID-Omni fused blocks.
|
||||
- `--hsdp-shard-size`: number of GPUs used for HSDP sharding.
|
||||
- `--hsdp-replicate-size`: number of HSDP replica groups.
|
||||
- `--num-inference-steps`: number of denoising steps (defaults 45).
|
||||
- `--video-negative-prompt`: negative prompt for video generation.
|
||||
- `--audio-negative-prompt`: negative prompt for audio generation.
|
||||
- `--enable-cpu-offload`: enable CPU offload (defaults False).
|
||||
- `--cache-backend`: enable `cache_dit` for acceleration.
|
||||
- `--quantization`: online (dynamic) quantization method — `fp8` or `int8`.
|
||||
(VAEs, the T5 text encoder, norms and modulation stay bf16.)
|
||||
@@ -0,0 +1,279 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import time
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
from PIL import Image
|
||||
from vllm.multimodal.media.audio import load_audio
|
||||
|
||||
from vllm_omni.diffusion.utils.media_utils import mux_video_audio_bytes
|
||||
from vllm_omni.entrypoints.omni import Omni
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
|
||||
IMAGE_EXTENSIONS = {".jpg", ".jpeg", ".png", ".bmp", ".webp", ".gif"}
|
||||
AUDIO_EXTENSIONS = {".wav", ".mp3", ".flac", ".m4a", ".aac", ".ogg"}
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description="Offline inference for X -> video+audio models.")
|
||||
parser.add_argument("--model", required=True, help="Model ckpt root directory.")
|
||||
parser.add_argument(
|
||||
"--model-type",
|
||||
default="dreamid-omni",
|
||||
choices=["dreamid-omni", "magi-human"],
|
||||
help="Model type.",
|
||||
)
|
||||
parser.add_argument("--prompt", default=None, help="Text prompt.")
|
||||
|
||||
parser.add_argument("--image-path", type=str, nargs="+", help="list of image-path")
|
||||
parser.add_argument("--audio-path", type=str, nargs="+", help="list of audio-path")
|
||||
parser.add_argument("--prompt-file", type=str, default=None, help="Text prompt in json format.")
|
||||
parser.add_argument(
|
||||
"--extra-body",
|
||||
type=str,
|
||||
default=None,
|
||||
help="[magi-human] JSON dict of model-specific extra params (declared in vllm_omni/model_extras/), "
|
||||
'merged into sampling extra_args. Example: \'{"image_path": "/path/to/img.jpg", "seconds": 5}\'.',
|
||||
)
|
||||
|
||||
parser.add_argument("--height", type=int, default=704, help="Video height.")
|
||||
parser.add_argument("--width", type=int, default=1280, help="Video width.")
|
||||
parser.add_argument("--num-inference-steps", type=int, default=45, help="Sampling steps.")
|
||||
parser.add_argument("--solver-name", default="unipc", help="Solver name: unipc|dpm++|euler.")
|
||||
parser.add_argument("--shift", type=float, default=5.0, help="Scheduler shift.")
|
||||
parser.add_argument("--seed", type=int, default=103, help="Random seed for reproducible generation.")
|
||||
parser.add_argument(
|
||||
"--cfg-parallel-size",
|
||||
type=int,
|
||||
default=1,
|
||||
choices=[1, 2, 3, 4],
|
||||
help="Number of GPUs used for classifier free guidance parallel size (max 4 branches).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--tensor-parallel-size",
|
||||
type=int,
|
||||
default=1,
|
||||
help=("Number of GPUs used for tensor parallelism (TP) inside the DiT."),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--video-negative-prompt",
|
||||
default="jitter, bad hands, blur, distortion",
|
||||
help="Negative prompt for video.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--audio-negative-prompt",
|
||||
default="robotic, muffled, echo, distorted",
|
||||
help="Negative prompt for audio.",
|
||||
)
|
||||
parser.add_argument("--output", default="dreamid_output.mp4", help="Output video path.")
|
||||
parser.add_argument(
|
||||
"--quantization",
|
||||
type=str,
|
||||
default=None,
|
||||
choices=["fp8", "int8"],
|
||||
help=("Online (dynamic) quantization method for the DreamID-Omni transformer. "),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--enable-cpu-offload",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Enable CPU offloading for diffusion models.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--enable-layerwise-offload",
|
||||
action="store_true",
|
||||
help="Enable layerwise (blockwise) offloading on DiT modules.",
|
||||
)
|
||||
parser.add_argument("--cache-backend", type=str, default=None, choices=["cache_dit"], help="Cache backend.")
|
||||
parser.add_argument(
|
||||
"--use-hsdp",
|
||||
action="store_true",
|
||||
help="Enable HSDP for DreamID-Omni fused transformer blocks.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--hsdp-shard-size",
|
||||
type=int,
|
||||
default=1,
|
||||
help="Number of GPUs used for HSDP sharding when HSDP is enabled.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--hsdp-replicate-size",
|
||||
type=int,
|
||||
default=1,
|
||||
help="Number of HSDP replica groups. Default 1 means pure sharding.",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def load_image_and_audio(image_paths, audio_paths):
|
||||
image = []
|
||||
audio = []
|
||||
|
||||
for path in image_paths:
|
||||
with Image.open(path) as img:
|
||||
img = img.convert("RGB")
|
||||
image.append(img)
|
||||
|
||||
for path in audio_paths:
|
||||
audio_array, sr = load_audio(path, sr=16000)
|
||||
audio_array = audio_array[int(sr * 1) : int(sr * 3)]
|
||||
audio.append(audio_array)
|
||||
return image, audio
|
||||
|
||||
|
||||
def _extract_peak_memory_mb(result: Any) -> float:
|
||||
"""Pull worker-reported peak VRAM (MiB) generation result.
|
||||
|
||||
Mirrors vllm_omni/entrypoints/openai/serving_video.py:_extract_peak_memory_mb.
|
||||
"""
|
||||
if isinstance(result, list):
|
||||
result = result[0] if result else None
|
||||
if result is None:
|
||||
return 0.0
|
||||
val = getattr(result, "peak_memory_mb", 0.0)
|
||||
if not val:
|
||||
inner = result
|
||||
if isinstance(inner, list):
|
||||
inner = inner[0] if inner else None
|
||||
val = getattr(inner, "peak_memory_mb", 0.0)
|
||||
try:
|
||||
return float(val or 0.0)
|
||||
except (TypeError, ValueError):
|
||||
return 0.0
|
||||
|
||||
|
||||
def main() -> None:
|
||||
args = parse_args()
|
||||
if args.prompt is None and args.prompt_file is None:
|
||||
raise ValueError("Either --prompt or --prompt-file must be provided.")
|
||||
|
||||
text_prompt = args.prompt
|
||||
if args.prompt_file:
|
||||
with open(args.prompt_file) as f:
|
||||
text_prompt = json.load(f)
|
||||
text_prompt = re.sub(
|
||||
r"\[SPEAKER_TIMESTAMPS_START\].*?\[SPEAKER_TIMESTAMPS_END\]", "", text_prompt, flags=re.DOTALL
|
||||
).strip()
|
||||
text_prompt = re.sub(
|
||||
r"\[AUDIO_DESCRIPTION_START].*?\[AUDIO_DESCRIPTION_END]", "", text_prompt, flags=re.DOTALL
|
||||
).strip()
|
||||
text_prompt = re.sub(r"\[[A-Z_]+\]", "", text_prompt)
|
||||
text_prompt = re.sub(r"\n\s*\n", "\n", text_prompt).strip()
|
||||
|
||||
if args.model_type == "magi-human":
|
||||
prompt = text_prompt
|
||||
extra_args = json.loads(args.extra_body) if args.extra_body else {}
|
||||
sampling_params = OmniDiffusionSamplingParams(
|
||||
height=args.height,
|
||||
width=args.width,
|
||||
num_inference_steps=args.num_inference_steps,
|
||||
seed=args.seed,
|
||||
extra_args=extra_args,
|
||||
)
|
||||
else:
|
||||
image, audio = load_image_and_audio(args.image_path, args.audio_path)
|
||||
|
||||
prompt = {
|
||||
"prompt": text_prompt,
|
||||
"video_negative_prompt": args.video_negative_prompt,
|
||||
"audio_negative_prompt": args.audio_negative_prompt,
|
||||
"multi_modal_data": {"image": image, "audio": audio},
|
||||
}
|
||||
|
||||
sampling_params = OmniDiffusionSamplingParams(
|
||||
height=args.height,
|
||||
width=args.width,
|
||||
num_inference_steps=args.num_inference_steps,
|
||||
seed=args.seed,
|
||||
extra_args={
|
||||
"solver_name": args.solver_name,
|
||||
"shift": args.shift,
|
||||
},
|
||||
)
|
||||
|
||||
cache_config = None
|
||||
if args.cache_backend == "cache_dit":
|
||||
cache_config = {
|
||||
"Fn_compute_blocks": 1,
|
||||
"Bn_compute_blocks": 0,
|
||||
"max_warmup_steps": 4,
|
||||
"max_cached_steps": 20,
|
||||
"residual_diff_threshold": 0.24,
|
||||
"max_continuous_cached_steps": 3,
|
||||
"enable_taylorseer": False,
|
||||
"taylorseer_order": 1,
|
||||
"scm_steps_mask_policy": None,
|
||||
"scm_steps_policy": "dynamic",
|
||||
}
|
||||
|
||||
omni_kwargs = dict(
|
||||
model=args.model,
|
||||
cfg_parallel_size=args.cfg_parallel_size,
|
||||
tensor_parallel_size=args.tensor_parallel_size,
|
||||
use_hsdp=args.use_hsdp,
|
||||
hsdp_shard_size=args.hsdp_shard_size,
|
||||
hsdp_replicate_size=args.hsdp_replicate_size,
|
||||
model_type=args.model_type,
|
||||
enable_cpu_offload=args.enable_cpu_offload,
|
||||
enable_layerwise_offload=args.enable_layerwise_offload,
|
||||
cache_backend=args.cache_backend,
|
||||
cache_config=cache_config,
|
||||
)
|
||||
if args.quantization is not None:
|
||||
omni_kwargs["quantization"] = args.quantization
|
||||
omni = Omni(**omni_kwargs)
|
||||
start = time.perf_counter()
|
||||
outputs = omni.generate(prompt, sampling_params)
|
||||
elapsed = time.perf_counter() - start
|
||||
|
||||
if not outputs:
|
||||
raise RuntimeError("No output returned from the model.")
|
||||
result = outputs[0]
|
||||
if not result.images:
|
||||
raise RuntimeError("No video frames found in model output.")
|
||||
generated_video = result.images[0]
|
||||
mm = result.multimodal_output or {}
|
||||
generated_audio = mm.get("audio")
|
||||
|
||||
if args.model_type == "magi-human":
|
||||
# MagiHuman returns frames already as (F, H, W, C) uint8, ready to mux.
|
||||
fps = float(mm.get("fps", 25))
|
||||
sample_rate = int(mm.get("audio_sample_rate", 24000))
|
||||
frames = generated_video
|
||||
else:
|
||||
fps = float(mm.get("fps", 24))
|
||||
sample_rate = int(mm.get("audio_sample_rate", 16000))
|
||||
# DreamID-Omni returns video as (C, F, H, W) float32 in [-1, 1].
|
||||
# mux_video_audio_bytes expects (F, H, W, C) uint8.
|
||||
if not isinstance(generated_video, np.ndarray) or generated_video.ndim != 4:
|
||||
raise RuntimeError(f"Unexpected video shape: {getattr(generated_video, 'shape', None)}")
|
||||
frames = generated_video.transpose(1, 2, 3, 0)
|
||||
frames = (np.clip((frames + 1.0) / 2.0, 0.0, 1.0) * 255.0).round().astype(np.uint8)
|
||||
|
||||
audio_np = None
|
||||
if generated_audio is not None:
|
||||
audio_np = np.squeeze(np.asarray(generated_audio)).astype(np.float32)
|
||||
|
||||
output_path = args.output
|
||||
video_bytes = mux_video_audio_bytes(
|
||||
frames,
|
||||
audio_np,
|
||||
fps=float(fps),
|
||||
audio_sample_rate=sample_rate,
|
||||
)
|
||||
with open(output_path, "wb") as f:
|
||||
f.write(video_bytes)
|
||||
print(f"Saved generated video to {output_path}")
|
||||
print(f"Total time: {elapsed:.2f}s")
|
||||
peak_mb = _extract_peak_memory_mb(outputs)
|
||||
if peak_mb:
|
||||
print(f"Worker peak GPU memory (reserved): {peak_mb:.2f} MiB ({peak_mb / 1024:.2f} GiB)")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -32,14 +32,12 @@ GPU 0 (90%) and colocates the Talker (55%) and Code2Wav (35%) on GPU 1. That
|
||||
profile admits at most four concurrent sequences per stage.
|
||||
|
||||
| deploy config | GPUs | Notes |
|
||||
| --- | --- | --- |
|
||||
|---|---|---|
|
||||
| `minicpmo_4_5.yaml` (default) | 1 | Memory-constrained compatibility layout. |
|
||||
| `minicpmo_4_5_2gpu.yaml` | 2 | Recommended continuous-batching layout; Talker and Code2Wav share GPU 1. |
|
||||
| `minicpmo_4_5_3gpu.yaml` | 3 | One GPU per stage. |
|
||||
| `minicpmo_4_5_8x4090.yaml` | 8 | Full 8x4090 layout. |
|
||||
|
||||
Every profile sets `session_mode: duplex`, so native full-duplex is served
|
||||
from the same process as `/v1/chat/completions`.
|
||||
| `minicpmo_4_5_duplex.yaml` | 1 | Experimental native full-duplex overlay. |
|
||||
|
||||
The split pipeline preserves native-duplex epoch/turn identity, segment text,
|
||||
turn completion, reference voice, and terminal-audio metadata through
|
||||
@@ -49,7 +47,7 @@ scenario below for live barge-in validation on the target GPU.
|
||||
Default:
|
||||
|
||||
```bash
|
||||
vllm serve openbmb/MiniCPM-o-4_5 \
|
||||
vllm-omni serve openbmb/MiniCPM-o-4_5 \
|
||||
--omni \
|
||||
--deploy-config vllm_omni/deploy/minicpmo_4_5.yaml \
|
||||
--trust-remote-code \
|
||||
@@ -57,8 +55,8 @@ vllm serve openbmb/MiniCPM-o-4_5 \
|
||||
```
|
||||
|
||||
For local ModelScope checkpoints, replace `openbmb/MiniCPM-o-4_5` with the
|
||||
checkpoint path. For native full-duplex, connect `/v1/realtime?duplex=1` on
|
||||
this same server.
|
||||
checkpoint path. To start the experimental native duplex backend, use
|
||||
`vllm_omni/deploy/minicpmo_4_5_duplex.yaml`.
|
||||
|
||||
### Per-stage overrides
|
||||
|
||||
@@ -147,7 +145,7 @@ assistant template, so it is not an apples-to-apples accuracy run.
|
||||
|
||||
## Run the Realtime duplex CLI demo
|
||||
|
||||
After the server is running, stream one WAV through the Realtime
|
||||
After the duplex backend is running, stream one WAV through the Realtime
|
||||
WebSocket endpoint:
|
||||
|
||||
```bash
|
||||
@@ -159,33 +157,6 @@ python examples/online_serving/minicpmo/realtime_duplex_demo.py \
|
||||
--output-dir /tmp/minicpmo_realtime_duplex_demo
|
||||
```
|
||||
|
||||
Video input uses the same session. PyAV demuxes JPEG frames at `--video-fps`
|
||||
(default 1.0) and vLLM `load_audio` extracts a 16 kHz mono WAV unless
|
||||
`--input-wav` overrides the soundtrack. Frames keep their capture size
|
||||
(`--frame-max-side 0`); the server `process_image` normalizes at 448.
|
||||
|
||||
`--stack-frames N` raises the visual refresh rate the way official duplex does:
|
||||
each 1 s unit also samples the `N-1` sub-frames captured inside it and tiles
|
||||
them into a single composite image sent next to that unit's base frame. The
|
||||
audio cadence never changes — a unit is always one second — and the wire always
|
||||
carries 2 images per unit however large `N` is, because the sub-frames share one
|
||||
composite. Official uses 5 for high refresh rate mode:
|
||||
|
||||
```bash
|
||||
python examples/online_serving/minicpmo/realtime_duplex_demo.py \
|
||||
--url ws://localhost:8099/v1/realtime?duplex=1 \
|
||||
--model openbmb/MiniCPM-o-4_5 \
|
||||
--input-video /path/to/clip.mp4 \
|
||||
--stack-frames 5 \
|
||||
--ref-audio /path/to/MiniCPM-o-Demo/assets/ref_audio/ref_minicpm_signature.wav \
|
||||
--output-dir /tmp/minicpmo_realtime_duplex_video_demo
|
||||
```
|
||||
|
||||
Detail inside a composite is capped by `scale_resolution=448` at
|
||||
`max_slice_nums=1`: official suggests HD slicing (`max_slice_nums=[2, 1]`) for
|
||||
stacked frames, which the duplex adapter does not implement yet. Reading small
|
||||
text or digits out of a wide scene is limited by that, not by frame timing.
|
||||
|
||||
## Open the experimental browser client
|
||||
|
||||
The browser UI serves the page and proxies the same-origin Realtime WebSocket to
|
||||
@@ -219,64 +190,6 @@ stronger `response-required` mode is diagnostic: it requires a purpose-built
|
||||
two-response WAV, its `--input-sha256`, and an
|
||||
`--expect-second-response-substring` value.
|
||||
|
||||
## Run Omni-DuplexEval
|
||||
|
||||
Omni-DuplexEval generation uses the vLLM-Omni native MiniCPM-o duplex
|
||||
endpoint. Evaluation uses a separate OpenAI-compatible multimodal judge
|
||||
served from the same vLLM-Omni environment. This validates the vLLM-Omni
|
||||
duplex client and local judge integration; it does not reproduce the paper's
|
||||
original MiniCPM-o inference implementation.
|
||||
|
||||
Start the duplex generation server:
|
||||
|
||||
```bash
|
||||
vllm serve openbmb/MiniCPM-o-4_5 --omni --trust-remote-code \
|
||||
--deploy-config vllm_omni/deploy/minicpmo_4_5_duplex.yaml \
|
||||
--served-model-name openbmb/MiniCPM-o-4_5 \
|
||||
--host 0.0.0.0 --port 8099
|
||||
```
|
||||
|
||||
Start a separate multimodal judge. The allowed path must contain any local
|
||||
videos passed with `--judge-video-mode video_url`:
|
||||
|
||||
```bash
|
||||
vllm serve Qwen/Qwen2.5-VL-7B-Instruct \
|
||||
--served-model-name Qwen/Qwen2.5-VL-7B-Instruct \
|
||||
--allowed-local-media-path /data/omni-duplex-eval \
|
||||
--host 0.0.0.0 --port 8000
|
||||
```
|
||||
|
||||
Generate, evaluate, and summarize:
|
||||
|
||||
```bash
|
||||
vllm bench omni-duplex-eval --omni generate \
|
||||
--url ws://127.0.0.1:8099/v1/realtime?duplex=1 \
|
||||
--model openbmb/MiniCPM-o-4_5 \
|
||||
--ref-audio /data/ref.wav \
|
||||
--dataset Hothan/Omni-DuplexEval \
|
||||
--concurrency 2 \
|
||||
--response-root /data/omni-duplex-eval/responses
|
||||
|
||||
vllm bench omni-duplex-eval --omni evaluate \
|
||||
--dataset Hothan/Omni-DuplexEval \
|
||||
--response-root /data/omni-duplex-eval/responses \
|
||||
--score-root /data/omni-duplex-eval/scores \
|
||||
--judge-base-url http://127.0.0.1:8000 \
|
||||
--judge-model Qwen/Qwen2.5-VL-7B-Instruct \
|
||||
--judge-video-mode video_url \
|
||||
--eval-workers 4
|
||||
|
||||
vllm bench omni-duplex-eval --omni summarize \
|
||||
--score-root /data/omni-duplex-eval/scores
|
||||
```
|
||||
|
||||
The dataset names such as `RTD_OCR` and `PR_correction` are Hugging Face
|
||||
splits, not dataset configurations. Pass one with `--split`, or omit it to
|
||||
run all nine splits. `--limit 1` is useful for a smoke test. Realtime
|
||||
generation records `clock=media`; artifacts generated with
|
||||
`--pace as-fast-as-possible` record `clock=invalid` and evaluation rejects
|
||||
them unless `--allow-invalid-clock` is explicit.
|
||||
|
||||
## Related examples
|
||||
|
||||
- [Offline MiniCPM-o inference](../../offline_inference/minicpmo/)
|
||||
|
||||
@@ -12,9 +12,7 @@ from __future__ import annotations
|
||||
import argparse
|
||||
import asyncio
|
||||
import base64
|
||||
import io
|
||||
import json
|
||||
import math
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
@@ -24,8 +22,8 @@ if str(REPO_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(REPO_ROOT))
|
||||
|
||||
from vllm_omni.experimental.fullduplex.client import ( # noqa: E402
|
||||
PCM16_BYTES_PER_SAMPLE,
|
||||
PCM16_SAMPLE_RATE,
|
||||
PCM16_BYTES_PER_SAMPLE, # noqa: F401 - compatibility export
|
||||
PCM16_SAMPLE_RATE, # noqa: F401 - compatibility export
|
||||
RealtimeDuplexClient,
|
||||
RealtimeEventCollector,
|
||||
build_realtime_url,
|
||||
@@ -40,10 +38,6 @@ from vllm_omni.experimental.fullduplex.client import ( # noqa: E402
|
||||
from vllm_omni.experimental.fullduplex.client import ( # noqa: E402
|
||||
reference_audio_data_url as _ref_audio_data_url,
|
||||
)
|
||||
from vllm_omni.experimental.fullduplex.video_stacking import ( # noqa: E402
|
||||
concat_frames_b64,
|
||||
unit_subframe_offsets,
|
||||
)
|
||||
|
||||
|
||||
class _StreamingOutputWriter:
|
||||
@@ -155,23 +149,6 @@ def _response_in_progress(events: list[dict[str, object]]) -> bool:
|
||||
)
|
||||
|
||||
|
||||
async def _commit_input_after_playback_checkpoint(client: RealtimeDuplexClient) -> float:
|
||||
"""Commit input only after preserving already-played assistant history.
|
||||
|
||||
Native duplex can finish several responses while the user is still
|
||||
streaming. Under the ``ack_only`` playback policy, acknowledging those
|
||||
responses after the final input commit is intentionally rejected: it could
|
||||
append an old assistant message after the newly committed user message.
|
||||
Checkpoint playback first so the server commits each assistant item at its
|
||||
original history position. A second acknowledgement after the commit can
|
||||
then safely advance response-local playback that drained in the meantime.
|
||||
"""
|
||||
await client.acknowledge_playback()
|
||||
commit_sent_at_s = time.monotonic()
|
||||
await client.commit()
|
||||
return commit_sent_at_s
|
||||
|
||||
|
||||
def _event_count_after(
|
||||
events: list[dict[str, object]],
|
||||
event_type: str,
|
||||
@@ -182,258 +159,14 @@ def _event_count_after(
|
||||
return sum(event.get("type") == event_type for event in events[index + 1 :])
|
||||
|
||||
|
||||
def _probe_video_frames_and_fps(video_path: Path) -> tuple[int, float]:
|
||||
"""Frame count and average FPS via PyAV (same approach as Daily-Omni)."""
|
||||
import av
|
||||
|
||||
with av.open(str(video_path)) as container:
|
||||
stream = container.streams.video[0]
|
||||
avg_fps = float(stream.average_rate) if stream.average_rate else 0.0
|
||||
num_frames = stream.frames or 0
|
||||
if num_frames <= 0:
|
||||
duration: float | None = None
|
||||
if stream.duration is not None and stream.time_base is not None:
|
||||
duration = float(stream.duration * stream.time_base)
|
||||
elif container.duration is not None:
|
||||
duration = container.duration / av.time_base
|
||||
if duration and avg_fps:
|
||||
num_frames = int(math.floor(duration * avg_fps))
|
||||
if num_frames <= 0:
|
||||
stream.thread_type = "AUTO"
|
||||
num_frames = sum(1 for _ in container.decode(stream))
|
||||
if num_frames <= 0:
|
||||
raise ValueError(f"No decodable video frames in {video_path}")
|
||||
return num_frames, avg_fps or 1.0
|
||||
|
||||
|
||||
def _upright(image, rotation: int):
|
||||
"""Apply a container display-matrix rotation, in degrees, to ``image``.
|
||||
|
||||
PyAV hands back the stored raster and leaves the display matrix for the
|
||||
caller, unlike the ffmpeg CLI which autorotates. Phone clips routinely
|
||||
carry +/-90, so skipping this feeds the model a sideways frame: arrows and
|
||||
digits stop being readable and the model mirrors the motion it reports.
|
||||
"""
|
||||
from PIL import Image
|
||||
|
||||
quarter_turns = {
|
||||
90: Image.Transpose.ROTATE_90,
|
||||
180: Image.Transpose.ROTATE_180,
|
||||
270: Image.Transpose.ROTATE_270,
|
||||
}
|
||||
turn = quarter_turns.get(int(rotation) % 360)
|
||||
return image.transpose(turn) if turn is not None else image
|
||||
|
||||
|
||||
def _decode_video_frames_rgb(video_path: Path, frame_idx: list[int]) -> list:
|
||||
"""Decode presentation-order ``frame_idx`` into upright RGB ``PIL.Image``s."""
|
||||
import av
|
||||
|
||||
wanted = sorted({int(i) for i in frame_idx})
|
||||
if not wanted:
|
||||
return []
|
||||
|
||||
decoded: dict[int, object] = {}
|
||||
with av.open(str(video_path)) as container:
|
||||
stream = container.streams.video[0]
|
||||
stream.thread_type = "AUTO"
|
||||
cursor = 0
|
||||
for position, frame in enumerate(container.decode(stream)):
|
||||
if wanted[cursor] > position:
|
||||
continue
|
||||
image = _upright(frame.to_image(), getattr(frame, "rotation", 0) or 0)
|
||||
while cursor < len(wanted) and wanted[cursor] <= position:
|
||||
decoded[wanted[cursor]] = image
|
||||
cursor += 1
|
||||
if cursor >= len(wanted):
|
||||
break
|
||||
|
||||
if not decoded:
|
||||
raise ValueError(f"Decoded no frames from {video_path} for indices {wanted[:8]}")
|
||||
last_image = decoded[max(decoded)]
|
||||
return [decoded.get(int(i), last_image) for i in frame_idx]
|
||||
|
||||
|
||||
def _resize_max_side(image, max_side: int):
|
||||
width, height = image.size
|
||||
longest = max(width, height)
|
||||
if longest <= max_side or max_side <= 0:
|
||||
return image
|
||||
scale = max_side / float(longest)
|
||||
new_size = (max(1, int(round(width * scale))), max(1, int(round(height * scale))))
|
||||
return image.resize(new_size)
|
||||
|
||||
|
||||
def _jpeg_bytes(image, *, quality: int = 95) -> bytes:
|
||||
buf = io.BytesIO()
|
||||
image.convert("RGB").save(buf, format="JPEG", quality=quality)
|
||||
return buf.getvalue()
|
||||
|
||||
|
||||
def _to_pcm16_bytes(audio_np) -> bytes:
|
||||
import numpy as np
|
||||
|
||||
arr = np.asarray(audio_np).reshape(-1)
|
||||
if arr.dtype != np.int16:
|
||||
arr = (np.clip(arr, -1.0, 1.0) * 32767.0).astype(np.int16)
|
||||
return arr.tobytes()
|
||||
|
||||
|
||||
def _pcm16_wav_duration_s(path: Path) -> float:
|
||||
pcm16 = read_pcm16_wav(path)
|
||||
return len(pcm16) / float(PCM16_SAMPLE_RATE * PCM16_BYTES_PER_SAMPLE)
|
||||
|
||||
|
||||
def _extract_video_frames(
|
||||
video_path: Path,
|
||||
work_dir: Path,
|
||||
*,
|
||||
fps: float,
|
||||
max_side: int,
|
||||
duration_s: float,
|
||||
stack_frames: int = 1,
|
||||
) -> tuple[list[str], list[str | None]]:
|
||||
"""Demux JPEG frames only. Frame count follows ``duration_s``, not soundtrack.
|
||||
|
||||
Returns the base frames (one per model unit) and the parallel stacked track.
|
||||
With ``stack_frames=N`` each unit also samples its ``N-1`` interior
|
||||
sub-frames and tiles them into one composite, which is how official duplex
|
||||
raises the visual refresh rate while keeping one second of audio per unit.
|
||||
A unit whose interior falls past the end of the clip stacks nothing.
|
||||
"""
|
||||
num_frames, avg_fps = _probe_video_frames_and_fps(video_path)
|
||||
sample_fps = max(float(fps), 1e-6)
|
||||
unit_s = 1.0 / sample_fps
|
||||
timeline_s = max(float(duration_s), unit_s)
|
||||
num_samples = max(1, int(math.ceil(timeline_s * sample_fps - 1e-9)))
|
||||
offsets = unit_subframe_offsets(stack_frames, unit_s=unit_s)
|
||||
|
||||
def frame_at(seconds: float) -> int:
|
||||
# Official get_video_frame_audio_segments uses truncating int(), not
|
||||
# round(), so 29.97 fps lands on the same presentation index.
|
||||
return min(int(seconds * avg_fps), num_frames - 1)
|
||||
|
||||
base_idx = [frame_at(unit * unit_s) for unit in range(num_samples)]
|
||||
sub_times = [
|
||||
[unit * unit_s + offset for offset in offsets if unit * unit_s + offset < timeline_s]
|
||||
for unit in range(num_samples)
|
||||
]
|
||||
sub_idx = [frame_at(seconds) for times in sub_times for seconds in times]
|
||||
images = _decode_video_frames_rgb(video_path, base_idx + sub_idx)
|
||||
sub_images = iter(images[len(base_idx) :])
|
||||
|
||||
frame_dir = work_dir / "frames"
|
||||
frame_dir.mkdir(parents=True, exist_ok=True)
|
||||
for stale in frame_dir.glob("*.jpg"):
|
||||
stale.unlink()
|
||||
frames_b64: list[str] = []
|
||||
stacked_b64: list[str | None] = []
|
||||
for index, (image, times) in enumerate(zip(images[: len(base_idx)], sub_times), start=1):
|
||||
jpeg = _jpeg_bytes(_resize_max_side(image, max_side))
|
||||
(frame_dir / f"frame_{index:05d}.jpg").write_bytes(jpeg)
|
||||
frames_b64.append(base64.b64encode(jpeg).decode("ascii"))
|
||||
interior = [_resize_max_side(next(sub_images), max_side) for _ in times]
|
||||
if not interior:
|
||||
stacked_b64.append(None)
|
||||
continue
|
||||
composite = concat_frames_b64(interior)
|
||||
(frame_dir / f"stack_{index:05d}.jpg").write_bytes(base64.b64decode(composite))
|
||||
stacked_b64.append(composite)
|
||||
return frames_b64, stacked_b64
|
||||
|
||||
|
||||
def _extract_video_soundtrack(video_path: Path, work_dir: Path) -> Path:
|
||||
"""Write a 16 kHz mono WAV from the video soundtrack."""
|
||||
try:
|
||||
from vllm.multimodal.media.audio import load_audio
|
||||
except ImportError as exc: # pragma: no cover
|
||||
raise SystemExit("--input-video needs vLLM audio helpers (load_audio)") from exc
|
||||
|
||||
audio_np, sr = load_audio(str(video_path), sr=PCM16_SAMPLE_RATE, mono=True)
|
||||
work_dir.mkdir(parents=True, exist_ok=True)
|
||||
audio_path = work_dir / "input_16k.wav"
|
||||
write_pcm16_wav(audio_path, _to_pcm16_bytes(audio_np), sample_rate_hz=int(sr))
|
||||
return audio_path
|
||||
|
||||
|
||||
def _extract_video_input(
|
||||
video_path: Path,
|
||||
work_dir: Path,
|
||||
*,
|
||||
fps: float,
|
||||
max_side: int,
|
||||
stack_frames: int,
|
||||
) -> tuple[Path, list[str], list[str | None]]:
|
||||
"""Demux a video into duplex inputs: 16 kHz mono WAV plus JPEG frames.
|
||||
|
||||
Use this only when the video soundtrack is the audio source. When the
|
||||
caller already has ``--input-wav``, extract frames against that WAV
|
||||
duration instead so a silent clip cannot fail ``load_audio``.
|
||||
"""
|
||||
audio_path = _extract_video_soundtrack(video_path, work_dir)
|
||||
frames_b64, stacked_b64 = _extract_video_frames(
|
||||
video_path,
|
||||
work_dir,
|
||||
fps=fps,
|
||||
max_side=max_side,
|
||||
duration_s=_pcm16_wav_duration_s(audio_path),
|
||||
stack_frames=stack_frames,
|
||||
)
|
||||
return audio_path, frames_b64, stacked_b64
|
||||
|
||||
|
||||
def _resolve_duplex_av_inputs(
|
||||
*,
|
||||
input_wav: str | None,
|
||||
input_video: str | None,
|
||||
work_dir: Path,
|
||||
fps: float,
|
||||
max_side: int,
|
||||
stack_frames: int = 1,
|
||||
) -> tuple[str, list[str], list[str | None]]:
|
||||
"""Return ``(wav_path, frames_b64, stacked_b64)``. External WAV wins over video audio."""
|
||||
if input_video:
|
||||
video_path = Path(input_video)
|
||||
if input_wav:
|
||||
frames, stacked = _extract_video_frames(
|
||||
video_path,
|
||||
work_dir,
|
||||
fps=fps,
|
||||
max_side=max_side,
|
||||
duration_s=_pcm16_wav_duration_s(Path(input_wav)),
|
||||
stack_frames=stack_frames,
|
||||
)
|
||||
return input_wav, frames, stacked
|
||||
extracted_wav, frames, stacked = _extract_video_input(
|
||||
video_path,
|
||||
work_dir,
|
||||
fps=fps,
|
||||
max_side=max_side,
|
||||
stack_frames=stack_frames,
|
||||
)
|
||||
return str(extracted_wav), frames, stacked
|
||||
if not input_wav:
|
||||
raise SystemExit("provide --input-wav, --input-video, or both")
|
||||
return input_wav, [], []
|
||||
|
||||
|
||||
async def run_demo(args: argparse.Namespace) -> dict[str, object]:
|
||||
output_dir = Path(args.output_dir)
|
||||
stream_writer = _StreamingOutputWriter(output_dir)
|
||||
|
||||
input_wav, video_frames, stacked_video_frames = _resolve_duplex_av_inputs(
|
||||
input_wav=args.input_wav,
|
||||
input_video=args.input_video,
|
||||
work_dir=output_dir / "video_input",
|
||||
fps=args.video_fps,
|
||||
max_side=args.frame_max_side,
|
||||
stack_frames=args.stack_frames,
|
||||
)
|
||||
|
||||
input_pcm16 = read_pcm16_wav(Path(input_wav))
|
||||
input_pcm16 = read_pcm16_wav(Path(args.input_wav))
|
||||
if not input_pcm16:
|
||||
raise ValueError("input WAV has no audio")
|
||||
|
||||
output_dir = Path(args.output_dir)
|
||||
stream_writer = _StreamingOutputWriter(output_dir)
|
||||
|
||||
url = build_realtime_url(
|
||||
args.url,
|
||||
args.model,
|
||||
@@ -451,12 +184,10 @@ async def run_demo(args: argparse.Namespace) -> dict[str, object]:
|
||||
timeout_s=args.timeout_s,
|
||||
)
|
||||
stream_event_cursor = len(client.events.events)
|
||||
frames_sent = await client.stream_pcm16(
|
||||
await client.stream_pcm16(
|
||||
input_pcm16,
|
||||
chunk_ms=args.chunk_ms,
|
||||
realtime=not args.no_realtime_pacing,
|
||||
video_frames=video_frames,
|
||||
stacked_video_frames=stacked_video_frames,
|
||||
)
|
||||
commit_event_cursor = len(client.events.events)
|
||||
stream_decision = _latest_model_decision(client.events.events, stream_event_cursor)
|
||||
@@ -465,7 +196,8 @@ async def run_demo(args: argparse.Namespace) -> dict[str, object]:
|
||||
chunk_period_ms=_chunk_period_ms(client.events.events),
|
||||
)
|
||||
wait_for_post_commit_decision = False
|
||||
commit_sent_at_s = await _commit_input_after_playback_checkpoint(client)
|
||||
commit_sent_at_s = time.monotonic()
|
||||
await client.commit()
|
||||
wait_error: str | None = None
|
||||
committed_index: int | None = None
|
||||
post_commit_decision: str | None = None
|
||||
@@ -578,10 +310,6 @@ async def run_demo(args: argparse.Namespace) -> dict[str, object]:
|
||||
committed_index,
|
||||
),
|
||||
},
|
||||
"input_video": args.input_video,
|
||||
"video_frames_extracted": len(video_frames),
|
||||
"video_frames_sent": frames_sent,
|
||||
"video_stacked_frames_extracted": sum(frame is not None for frame in stacked_video_frames),
|
||||
"audio_bytes": len(audio),
|
||||
"audio_chunk_count": len(stream_writer.audio_chunk_paths),
|
||||
"audio_chunk_files": [str(path) for path in stream_writer.audio_chunk_paths],
|
||||
@@ -628,48 +356,7 @@ def parse_args() -> argparse.Namespace:
|
||||
parser.add_argument("--url", default="ws://localhost:8099/v1/realtime?duplex=1")
|
||||
parser.add_argument("--model", default="openbmb/MiniCPM-o-4_5")
|
||||
parser.add_argument("--session-id")
|
||||
parser.add_argument(
|
||||
"--input-wav",
|
||||
help="Mono 16 kHz PCM16 clip to stream. Optional when --input-video carries the audio track.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--input-video",
|
||||
help=(
|
||||
"Video to stream as omni duplex input. PyAV demuxes frames and "
|
||||
"vLLM load_audio extracts a 16 kHz mono WAV (unless --input-wav overrides the audio)."
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--video-fps",
|
||||
type=float,
|
||||
default=1.0,
|
||||
help="Frame extraction rate. The wire contract carries ~1 frame per second of audio.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--frame-max-side",
|
||||
type=int,
|
||||
default=0,
|
||||
help=(
|
||||
"Optional client-side downscale of the longest side, in pixels. "
|
||||
"0 (default) keeps the capture resolution so the server "
|
||||
"process_image can normalize at scale_resolution=448 from a "
|
||||
"higher-quality source. The web demo never resizes on the client."
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--stack-frames",
|
||||
type=int,
|
||||
default=1,
|
||||
help=(
|
||||
"Frames sampled per 1 s audio unit. 1 (default) sends the unit's "
|
||||
"frame alone. N > 1 also samples the N-1 sub-frames inside the unit "
|
||||
"and tiles them into one composite sent next to the base frame, so "
|
||||
"motion (arrows, doors, digits changing) is visible at N fps while "
|
||||
"the audio stays 1 s per unit. Official uses 5 for high refresh "
|
||||
"rate mode; the wire carries 2 images per unit at any N because the "
|
||||
"sub-frames share one composite."
|
||||
),
|
||||
)
|
||||
parser.add_argument("--input-wav", required=True)
|
||||
parser.add_argument(
|
||||
"--ref-audio",
|
||||
required=True,
|
||||
@@ -685,7 +372,7 @@ def parse_args() -> argparse.Namespace:
|
||||
"--temperature",
|
||||
type=float,
|
||||
default=None,
|
||||
help="Stage 0 sampling temperature; omit to preserve the model default.",
|
||||
help="Stage0 sampling temperature; response-required soft-interrupt uses 0.0.",
|
||||
)
|
||||
parser.add_argument("--no-realtime-pacing", action="store_true")
|
||||
parser.add_argument("--require-audio", action="store_true")
|
||||
|
||||
@@ -24,18 +24,6 @@ vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091 \
|
||||
--deploy-config /path/to/your_deploy_config.yaml
|
||||
```
|
||||
|
||||
To serve **thinker-only** (text output, no talker / code2wav loaded) with Instruct
|
||||
weights, pass the bundled thinker-only deploy YAML. The `pipeline:` key
|
||||
`qwen3_omni_moe_thinker_only` overrides the HF `enable_audio_output` resolver:
|
||||
|
||||
```bash
|
||||
vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091 \
|
||||
--deploy-config vllm_omni/deploy/qwen3_omni_moe_thinking.yaml
|
||||
```
|
||||
|
||||
Captioner / Thinking checkpoints (`enable_audio_output=false`) still auto-select
|
||||
the same single-stage pipeline without `--deploy-config`.
|
||||
|
||||
For a 3x-GPU multi-replica layout (talker/code2wav scale-out on cuda:1,2),
|
||||
use `--stage-overrides` on top of the default config:
|
||||
|
||||
|
||||
@@ -37,7 +37,7 @@ During generation the client normally sends only `session.start` and then receiv
|
||||
## Install Client Dependency
|
||||
|
||||
```bash
|
||||
pip install av websockets
|
||||
pip install websockets
|
||||
# For the Gradio demo:
|
||||
pip install 'vllm-omni[demo]' websockets
|
||||
```
|
||||
|
||||
@@ -8,7 +8,7 @@ the received bytes to disk when the session finishes or is interrupted.
|
||||
Image/reference input is intentionally not supported in this example yet.
|
||||
|
||||
Requirements:
|
||||
pip install av websockets
|
||||
pip install websockets
|
||||
"""
|
||||
|
||||
import argparse
|
||||
|
||||
@@ -1,6 +1,3 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
"""OpenAI-compatible client for Qwen3-TTS via /v1/audio/speech endpoint.
|
||||
|
||||
This script demonstrates how to use the OpenAI-compatible speech API
|
||||
@@ -77,8 +74,6 @@ def run_tts_generation(args) -> None:
|
||||
"voice": args.speaker,
|
||||
"response_format": args.response_format,
|
||||
}
|
||||
if args.sample_rate is not None:
|
||||
payload["sample_rate"] = args.sample_rate
|
||||
|
||||
# Add optional parameters
|
||||
if args.instructions:
|
||||
@@ -251,13 +246,6 @@ def parse_args():
|
||||
choices=["wav", "mp3", "flac", "pcm", "aac", "opus"],
|
||||
help="Audio output format (default: wav)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--sample-rate",
|
||||
type=int,
|
||||
default=None,
|
||||
choices=[8000, 24000],
|
||||
help="Output sample rate in Hz (default: model native)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output",
|
||||
"-o",
|
||||
|
||||
+35
-42
@@ -71,11 +71,11 @@ dev = [
|
||||
"voxcpm==2.0.3", # for VoxCPM2 path-locating in tests
|
||||
"torchdiffeq==0.2.5", # for covo_audio Test
|
||||
"FlagEmbedding==1.4.0", # for hunyuan_image3 tests
|
||||
"kernels==0.14.1", # for accuracy tests comparison with diffusers output (0.15.0 incompatible with diffusers 0.38.0)
|
||||
"s3tokenizer==0.3.0", # for tts test
|
||||
# Provides top-level cosyvoice2; do not co-install stepaudio2-minicpmo,
|
||||
# whose top-level stepaudio2 package conflicts with this distribution.
|
||||
"step-audio2==1.0.0", # Step-Audio2 / MiniCPM-o 4.5 Token2Wav (flashcosyvoice / hyperpyyaml)
|
||||
"silero-vad==6.1", # Realtime server-VAD duplex E2E
|
||||
]
|
||||
|
||||
demo = [
|
||||
@@ -85,10 +85,6 @@ demo = [
|
||||
"cn2an>=0.5.22", # Chinese number normalization for TTS (e.g. "1次" -> "一次")
|
||||
]
|
||||
|
||||
server-vad = [
|
||||
"silero-vad==6.1",
|
||||
]
|
||||
|
||||
forced-aligner = [
|
||||
"qwen-asr>=0.0.6", # faithful Japanese/Korean word segmentation for the TTS forced aligner
|
||||
]
|
||||
@@ -248,11 +244,11 @@ addopts = [
|
||||
"--dist=loadgroup"
|
||||
]
|
||||
markers = [
|
||||
# ci/cd required — tag with [ci-level] for tests.helpers.mark.get_level_markers().
|
||||
"core_model: [ci-level] L1&L2 tests (run in each PR)",
|
||||
"advanced_model: [ci-level] L3 level tests (run on each merge)",
|
||||
"full_model: [ci-level] L4 level tests (run nightly)",
|
||||
"local_model: [ci-level] tests (run locally)",
|
||||
# ci/cd required
|
||||
"core_model: L1&L2 tests (run in each PR)",
|
||||
"advanced_model: L3 level tests (run on each merge)",
|
||||
"full_model: L4 level tests (run nightly)",
|
||||
"local_model: tests (run locally)",
|
||||
# function module markers
|
||||
"diffusion: Diffusion model tests",
|
||||
"omni: Omni model tests",
|
||||
@@ -262,41 +258,38 @@ markers = [
|
||||
"sp: Sequence parallelism tests",
|
||||
"example: Doc example code tests",
|
||||
# platform markers
|
||||
# Tag with [hardware-platform] for tests.helpers.mark.get_supported_platforms().
|
||||
# Add [gpu] on accelerator platforms that should also get pytest.mark.gpu.
|
||||
# cpu/gpu are pytest marks; hardware_test(res=...) rejects them as keys.
|
||||
"cpu: [hardware-platform] Tests that run on CPU",
|
||||
"gpu: [hardware-platform] Tests that run on GPU (auto-added)",
|
||||
"cuda: [hardware-platform] [gpu] Tests that run on CUDA (auto-added)",
|
||||
"rocm: [hardware-platform] [gpu] Tests that run on AMD/ROCm (auto-added)",
|
||||
"xpu: [hardware-platform] [gpu] Tests that run on XPU (auto-added)",
|
||||
"npu: [hardware-platform] Tests that run on NPU/Ascend (auto-added)",
|
||||
"musa: [hardware-platform] [gpu] Tests that run on MUSA/Moore Threads (auto-added)",
|
||||
"cpu: Tests that run on CPU",
|
||||
"gpu: Tests that run on GPU (auto-added)",
|
||||
"cuda: Tests that run on CUDA (auto-added)",
|
||||
"rocm: Tests that run on AMD/ROCm (auto-added)",
|
||||
"xpu: Tests that run on XPU (auto-added)",
|
||||
"npu: Tests that run on NPU/Ascend (auto-added)",
|
||||
"musa: Tests that run on MUSA/Moore Threads (auto-added)",
|
||||
# specified computation resources marks (auto-added)
|
||||
# [hardware-resource] + [cuda]/[rocm]/[xpu]/[npu]/[musa] for hardware_test(res=...).
|
||||
# Do not put [gpu] on SKU lines: that would make gpu a res key.
|
||||
"H100: [hardware-resource] [cuda] Tests that require H100 GPU",
|
||||
"H800: [hardware-resource] [cuda] Tests that require H800 GPU",
|
||||
"H200: [hardware-resource] [cuda] Tests that require H200 GPU",
|
||||
"L4: [hardware-resource] [cuda] Tests that require L4 GPU",
|
||||
"B200: [hardware-resource] [cuda] Tests that require B200 GPU",
|
||||
"MI325: [hardware-resource] [rocm] Tests that require MI325 GPU (AMD/ROCm)",
|
||||
"B60: [hardware-resource] [xpu] Tests that require Intel Arc Pro B60 XPU",
|
||||
"S5000: [hardware-resource] [musa] Tests that require S5000 GPU (Moore Threads/MUSA)",
|
||||
"A2: [hardware-resource] [npu] Tests that require A2 NPU",
|
||||
"A3: [hardware-resource] [npu] Tests that require A3 NPU",
|
||||
# Tag with [hardware-cards] for tests.helpers.mark.get_supported_card_counts().
|
||||
"cards_1: [hardware-cards] Tests that require 1 accelerator card",
|
||||
"cards_2: [hardware-cards] Tests that require 2 accelerator cards",
|
||||
"cards_3: [hardware-cards] Tests that require 3 accelerator cards",
|
||||
"cards_4: [hardware-cards] Tests that require 4 accelerator cards",
|
||||
"cards_5: [hardware-cards] Tests that require 5 accelerator cards",
|
||||
"cards_6: [hardware-cards] Tests that require 6 accelerator cards",
|
||||
"cards_7: [hardware-cards] Tests that require 7 accelerator cards",
|
||||
"cards_8: [hardware-cards] Tests that require 8 accelerator cards",
|
||||
# Tag with [hardware-resource] for tests.helpers.mark.get_hardware_mark_list().
|
||||
"H100: [hardware-resource] Tests that require H100 GPU",
|
||||
"H800: [hardware-resource] Tests that require H800 GPU",
|
||||
"H200: [hardware-resource] Tests that require H200 GPU",
|
||||
"L4: [hardware-resource] Tests that require L4 GPU",
|
||||
"B200: [hardware-resource] Tests that require B200 GPU",
|
||||
"MI325: [hardware-resource] Tests that require MI325 GPU (AMD/ROCm)",
|
||||
"B60: [hardware-resource] Tests that require Intel Arc Pro B60 XPU",
|
||||
"S5000: [hardware-resource] Tests that require S5000 GPU (Moore Threads/MUSA)",
|
||||
"A2: [hardware-resource] Tests that require A2 NPU",
|
||||
"A3: [hardware-resource] Tests that require A3 NPU",
|
||||
"distributed_cuda: Tests that require multi cards on CUDA platform",
|
||||
"distributed_rocm: Tests that require multi cards on ROCm platform",
|
||||
"distributed_xpu: Tests that require multi cards on XPU platform",
|
||||
"distributed_npu: Tests that require multi cards on NPU platform",
|
||||
"distributed_musa: Tests that require multi cards on MUSA platform",
|
||||
"skipif_cuda: Skip if the num of CUDA cards is less than the required",
|
||||
"skipif_rocm: Skip if the num of ROCm cards is less than the required",
|
||||
"skipif_xpu: Skip if the num of XPU cards is less than the required",
|
||||
"skipif_npu: Skip if the num of NPU cards is less than the required",
|
||||
"skipif_musa: Skip if the num of MUSA cards is less than the required",
|
||||
# more detailed markers
|
||||
"sp: Sequence parallelism tests (multi-GPU)",
|
||||
"slow: [ci-level] Slow tests (may skip in quick CI)",
|
||||
"slow: Slow tests (may skip in quick CI)",
|
||||
"benchmark: Benchmark tests",
|
||||
"xdist: xdist-parallelizable tests",
|
||||
]
|
||||
|
||||
@@ -1,14 +1,11 @@
|
||||
# Boogu-Image
|
||||
|
||||
> Text-to-image and image-editing online serving
|
||||
(Boogu-Image-0.1-Base / -Edit / -Edit-Turbo)
|
||||
> Text-to-image and image-editing online serving (Boogu-Image-0.1-Base / -Edit)
|
||||
|
||||
## Summary
|
||||
|
||||
- Vendor: Boogu
|
||||
- Model: `Boogu/Boogu-Image-0.1-Base` (text-to-image),
|
||||
`Boogu/Boogu-Image-0.1-Edit` (image editing), and
|
||||
`Boogu/Boogu-Image-0.1-Edit-Turbo` (four-step image editing)
|
||||
- Model: `Boogu/Boogu-Image-0.1-Base` (text-to-image), `Boogu/Boogu-Image-0.1-Edit` (image editing)
|
||||
- Task: Text-to-image generation and text-guided image editing (TI2I)
|
||||
- Mode: Online serving with the OpenAI-compatible API
|
||||
- Maintainer: Community
|
||||
@@ -16,23 +13,19 @@
|
||||
## When to use this recipe
|
||||
|
||||
Use this recipe when you want a known-good starting point for serving
|
||||
`Boogu/Boogu-Image-0.1-Base`, `Boogu/Boogu-Image-0.1-Edit`, or
|
||||
`Boogu/Boogu-Image-0.1-Edit-Turbo` with vLLM-Omni's native pipeline (no
|
||||
`Boogu/Boogu-Image-0.1-Base` with vLLM-Omni's native pipeline (no
|
||||
`--diffusion-load-format diffusers`, and the upstream `boogu` package is not
|
||||
required).
|
||||
|
||||
Boogu-Image-0.1 is an Apache-2.0 unified image generation and editing model
|
||||
family. The Base text-to-image checkpoint pairs a Qwen3-VL multimodal encoder
|
||||
with a Diffusion Transformer (DiT) and a flow-match Euler scheduler with
|
||||
time-shift. It handles photorealistic generation and Chinese/English text
|
||||
rendering. The Edit and Edit-Turbo checkpoints use the same native
|
||||
`BooguImagePipeline` for image editing.
|
||||
family. This recipe covers the Base text-to-image checkpoint, which pairs a
|
||||
Qwen3-VL multimodal encoder with a Diffusion Transformer (DiT) and a flow-match
|
||||
Euler scheduler with time-shift. It handles photorealistic generation and
|
||||
Chinese/English text rendering.
|
||||
|
||||
## References
|
||||
|
||||
- Upstream model card: <https://huggingface.co/Boogu/Boogu-Image-0.1-Base>
|
||||
- Edit model card: <https://huggingface.co/Boogu/Boogu-Image-0.1-Edit>
|
||||
- Edit-Turbo 1K hotfix: <https://huggingface.co/Boogu/Boogu-Image-0.1-Edit-Turbo/tree/hotfix-1k-20260708>
|
||||
- Project page: <https://boogu.org>
|
||||
- GitHub: <https://github.com/boogu-project/Boogu-Image>
|
||||
- Related example: [`examples/online_serving/text_to_image/`](../../examples/online_serving/text_to_image/README.md)
|
||||
@@ -100,7 +93,7 @@ curl -s http://localhost:8091/v1/chat/completions \
|
||||
- **Memory usage:** ~34.6 GiB on GPU. Use `--vae-use-slicing --vae-use-tiling`
|
||||
to trim peak VRAM if needed.
|
||||
- **Key flags:**
|
||||
- `--omni` — enables vLLM-Omni diffusion serving.
|
||||
- `--omni` — enables vLLM-Omni diffusion serving.
|
||||
- **Guidance:** Boogu-Image uses `guidance_scale` (mapped to the upstream
|
||||
`text_guidance_scale`); the default is `4.0`. Classifier-free guidance is
|
||||
active whenever `guidance_scale > 1.0`.
|
||||
@@ -118,13 +111,13 @@ the image-editing (TI2I) path activates automatically when a request carries a
|
||||
reference image. The Base text-to-image path is unaffected (no reference image
|
||||
is sent).
|
||||
|
||||
### Command
|
||||
#### Command
|
||||
|
||||
```bash
|
||||
vllm serve Boogu/Boogu-Image-0.1-Edit --omni --port 8091
|
||||
```
|
||||
|
||||
### Verification
|
||||
#### Verification
|
||||
|
||||
Edit an image with `/v1/images/edits` (the model-card example — change a photo
|
||||
to a colored-pencil drawing). Diffusion parameters are plain multipart form
|
||||
@@ -164,14 +157,14 @@ curl -s http://localhost:8091/v1/chat/completions \
|
||||
}' | jq -r '.choices[0].message.content[0].image_url.url' | cut -d',' -f2- | base64 -d > edited.png
|
||||
```
|
||||
|
||||
### Notes
|
||||
#### Notes
|
||||
|
||||
- **Single reference image:** only one input image is supported for now (the
|
||||
upstream "Only support 1 reference image for now" limit).
|
||||
- **Guidance semantics:**
|
||||
- `guidance_scale` = text guidance (upstream `text_guidance_scale`, default
|
||||
- `guidance_scale` = text guidance (upstream `text_guidance_scale`, default
|
||||
`4.0`); `> 1.0` enables text CFG. Editing typically uses `5.0`.
|
||||
- `guidance_scale_2` = image guidance (upstream `image_guidance_scale`,
|
||||
- `guidance_scale_2` = image guidance (upstream `image_guidance_scale`,
|
||||
default `1.0` = off). Setting it `> 1.0` enables the double-guidance path
|
||||
(3 model predictions per step), steering more strongly toward the reference
|
||||
image.
|
||||
@@ -185,69 +178,3 @@ curl -s http://localhost:8091/v1/chat/completions \
|
||||
supports Boogu-Image-Edit directly
|
||||
(`--model Boogu/Boogu-Image-0.1-Edit --guidance-scale 5.0`, optional
|
||||
`--guidance-scale-2`).
|
||||
|
||||
## Fast image editing (Boogu-Image-0.1-Edit-Turbo)
|
||||
|
||||
Edit-Turbo is the distilled image-editing checkpoint. Its `model_index.json`
|
||||
declares `BooguImagePipeline`, so it uses the same native TI2I path as Edit.
|
||||
The upstream 1K hotfix is recommended for stable results and is pinned below
|
||||
to avoid silently loading the older checkpoint from the repository's default
|
||||
revision.
|
||||
|
||||
### Command
|
||||
|
||||
```bash
|
||||
vllm serve Boogu/Boogu-Image-0.1-Edit-Turbo \
|
||||
--omni \
|
||||
--revision hotfix-1k-20260708 \
|
||||
--port 8091
|
||||
```
|
||||
|
||||
### Verification
|
||||
|
||||
Edit an image with the distilled four-step settings:
|
||||
|
||||
```bash
|
||||
curl -s http://localhost:8091/v1/images/edits \
|
||||
-F model="Boogu/Boogu-Image-0.1-Edit-Turbo" \
|
||||
-F image="@input.png" \
|
||||
-F prompt="Change the style to a colored pencil drawing." \
|
||||
-F num_inference_steps=4 \
|
||||
-F guidance_scale=1.0 \
|
||||
-F seed=42 \
|
||||
| jq -r '.data[0].b64_json' | base64 -d > edited-turbo.png
|
||||
```
|
||||
|
||||
Or via chat completions:
|
||||
|
||||
```bash
|
||||
curl -s http://localhost:8091/v1/chat/completions \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"messages": [
|
||||
{"role": "user", "content": [
|
||||
{"type": "image_url", "image_url": {"url": "data:image/png;base64,<BASE64>"}},
|
||||
{"type": "text", "text": "Change the style to a colored pencil drawing."}
|
||||
]}
|
||||
],
|
||||
"extra_body": {
|
||||
"num_inference_steps": 4,
|
||||
"guidance_scale": 1.0,
|
||||
"seed": 42
|
||||
}
|
||||
}' | jq -r '.choices[0].message.content[0].image_url.url' | cut -d',' -f2- | base64 -d > edited-turbo.png
|
||||
```
|
||||
|
||||
### Notes
|
||||
|
||||
- **Pinned revision:** use `hotfix-1k-20260708`; upstream recommends the 1K
|
||||
hotfix over the 1.5K variant for more stable results.
|
||||
- **Recommended settings:** `num_inference_steps=4` and
|
||||
`guidance_scale=1.0`. Edit-Turbo is guidance-distilled, meaning the guidance
|
||||
behavior is already baked into the distilled checkpoint. A scale of `1.0`
|
||||
avoids applying additional classifier-free guidance; unlike the regular Edit
|
||||
checkpoint, it should not be increased to `5.0`.
|
||||
- **Reference images:** the same single-reference and `align_res` behavior as
|
||||
the regular Edit checkpoint applies.
|
||||
- **Known limitations:** the same single-GPU limitations as Base and Edit apply;
|
||||
CPU offload, Cache-DiT, and multi-GPU parallelism are not yet validated.
|
||||
+5
-47
@@ -10,22 +10,14 @@ checkpoint for T2V and first-frame I2V. Every output includes synchronized
|
||||
48 kHz stereo audio.
|
||||
|
||||
The raw [`Lightricks/LTX-2.5`](https://huggingface.co/Lightricks/LTX-2.5)
|
||||
repository is not directly loadable with `--model`. It supplies the canonical
|
||||
Diffusion VAE checkpoint and the official upsampler and LoRA sidecars used by
|
||||
the Full/SFT two-stage pipeline, so accept both model licenses and authenticate
|
||||
before first use.
|
||||
|
||||
The vLLM-Omni integration code is provided under the project's
|
||||
[Apache-2.0 license](../../LICENSE). The LTX-2.5 model weights are separate
|
||||
artifacts covered by the
|
||||
[LTX-2.x Community License](https://github.com/Lightricks/LTX-2/blob/main/LICENSE.md),
|
||||
not the vLLM-Omni license. Users are responsible for reviewing and complying
|
||||
with the model license before downloading or using the weights.
|
||||
repository is not directly loadable with `--model`. It supplies the official
|
||||
upsampler and LoRA sidecars used by the Full/SFT two-stage pipeline, so accept
|
||||
both model licenses and authenticate before first use.
|
||||
|
||||
## Choose a pipeline
|
||||
|
||||
| Pipeline | Mode | Default size | Schedule |
|
||||
| --- | --- | ---: | ---: |
|
||||
|---|---|---:|---:|
|
||||
| `LTX2Pipeline` | Full/SFT one-stage | 960x544 | 30 steps |
|
||||
| `LTX2TwoStagePipeline` | Full/SFT two-stage | 1920x1088 | 30 + 3 steps |
|
||||
| `LTX2DistilledOneStagePipeline` | Distilled one-stage | 960x544 | 8 steps |
|
||||
@@ -36,40 +28,6 @@ latent upsampler, and run a three-step refinement stage. Select the class with
|
||||
`--model-class-name`; no `--task-type` flag is required. Supplying one initial
|
||||
image selects I2V, while omitting it selects T2V.
|
||||
|
||||
## Video decoder
|
||||
|
||||
LTX-2.5 uses the canonical Native Diffusion VAE decoder by default. To opt in
|
||||
to the legacy convolutional VAE, set the startup-only stage-0 model extra
|
||||
`ltx2_use_conv_vae: true`. The choice applies to all requests and all four
|
||||
pipeline classes above. DiffVAE loads its NATTEN kernel from Hugging Face Hub
|
||||
during startup. `decode_timestep` and `decode_noise_scale` condition only the
|
||||
legacy ConvVAE and have no effect when DiffVAE is selected.
|
||||
|
||||
For offline Python usage:
|
||||
|
||||
```python
|
||||
omni = Omni(
|
||||
model="Lightricks/LTX-2.5-Diffusers",
|
||||
model_class_name="LTX2Pipeline",
|
||||
stage_overrides='{"0":{"extras":{"ltx2_use_conv_vae":true}}}',
|
||||
)
|
||||
```
|
||||
|
||||
For online serving:
|
||||
|
||||
```bash
|
||||
vllm serve Lightricks/LTX-2.5-Diffusers \
|
||||
--omni \
|
||||
--model-class-name LTX2Pipeline \
|
||||
--stage-overrides '{"0":{"extras":{"ltx2_use_conv_vae":true}}}'
|
||||
```
|
||||
|
||||
Both decoders are untiled by default. Set `vae_use_tiling` for memory-saving
|
||||
serial tiling; DiffVAE tiles only above 80 frames or 768 pixels in either spatial
|
||||
dimension. DiffVAE also supports distributed VAE decode; see the
|
||||
[VAE Parallelism Guide](../../docs/user_guide/diffusion/parallelism/vae_parallelism.md).
|
||||
DiffVAE is decoder-only, so I2V still uses the convolutional VAE encoder.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
```bash
|
||||
@@ -84,7 +42,7 @@ Install matching vLLM and vLLM-Omni versions, and ensure `ffmpeg` and
|
||||
## Hardware
|
||||
|
||||
| GPU | Status | Recommended scope |
|
||||
| --- | --- | --- |
|
||||
|---|---|---|
|
||||
| NVIDIA B300 | Verified | All four canonical pipelines |
|
||||
| NVIDIA B200 or H200 | Capacity-based recommendation; not yet verified | All four canonical pipelines |
|
||||
| NVIDIA GB200 or GB300 | Capacity-based recommendation; not yet verified | All four canonical pipelines |
|
||||
|
||||
@@ -1,212 +0,0 @@
|
||||
# MammothModa2
|
||||
|
||||
> MammothModa2-Preview and MammothModa2-Dev unified understanding and generation
|
||||
|
||||
## Summary
|
||||
|
||||
- Vendor: ByteDance Research
|
||||
- Models: `bytedance-research/MammothModa2-Preview`, `bytedance-research/MammothModa2-Dev`
|
||||
- Tasks: Preview and Dev text-to-image (AR → DiT); Dev text/image understanding
|
||||
- Mode: Offline inference
|
||||
- Maintainer: Community
|
||||
|
||||
## When to use this recipe
|
||||
|
||||
Use this recipe to run MammothModa2-Preview through shared task-oriented
|
||||
offline examples. Text-to-image uses the shared image example (`text_to_image.py`) instead of a model-specific script.
|
||||
The generic example formats the AR prompt, drives the AR → DiT stage pipeline,
|
||||
and forwards MammothModa2-specific generation parameters through the
|
||||
pipeline-declared `extra_body` contract.
|
||||
|
||||
MammothModa2's DiT stage consumes its inputs through the multi-stage kwargs
|
||||
interface (not `OmniDiffusionRequest`), so its generation knobs
|
||||
(`text_guidance_scale`, `cfg_range`, `num_inference_steps`) are passed via
|
||||
`--extra-body` rather than the standard `--num-inference-steps` / `--cfg-scale`
|
||||
flags. Image size uses the standard `--height` / `--width` flags.
|
||||
|
||||
## References
|
||||
|
||||
- Upstream model:
|
||||
[`bytedance-research/MammothModa2-Preview`](https://huggingface.co/bytedance-research/MammothModa2-Preview)
|
||||
- Dev model:
|
||||
[`bytedance-research/MammothModa2-Dev`](https://huggingface.co/bytedance-research/MammothModa2-Dev)
|
||||
- Related offline example:
|
||||
[`examples/offline_inference/text_to_image/text_to_image.py`](../../examples/offline_inference/text_to_image/text_to_image.py)
|
||||
- Related T2T/I2T example:
|
||||
[`examples/offline_inference/x_to_text/x_to_text.py`](../../examples/offline_inference/x_to_text/x_to_text.py)
|
||||
- Declared parameters:
|
||||
[`vllm_omni/model_extras/mammothmodal2_preview.py`](../../vllm_omni/model_extras/mammothmodal2_preview.py)
|
||||
- Deploy config:
|
||||
[`vllm_omni/deploy/mammoth_moda2.yaml`](../../vllm_omni/deploy/mammoth_moda2.yaml)
|
||||
|
||||
## Hardware Support
|
||||
|
||||
The default deploy config runs both the AR and DiT stages on a single GPU
|
||||
(`devices: "0"`). The committed `gpu_memory_utilization` split (stage-0 AR `0.5`,
|
||||
stage-1 DiT `0.3`) is sized for an ~80 GB GPU. The model also fits on a 48 GB GPU
|
||||
after rebalancing the split so the AR weights (~23 GB) leave room for the KV
|
||||
cache — see the note under *1x L40S 48GB*.
|
||||
|
||||
## GPU
|
||||
|
||||
### 1x L40S 48GB
|
||||
|
||||
> **48 GB config adjustment:** the committed
|
||||
> `vllm_omni/deploy/mammoth_moda2.yaml` uses
|
||||
> `gpu_memory_utilization` 0.5 / 0.3 (sized for ~80 GB). To fit on a 48 GB L40S,
|
||||
> set the stage-0 (AR) value to `0.8` and the stage-1 (DiT) value to `0.16`
|
||||
> before running. (On an ~80 GB GPU, leave the defaults unchanged.)
|
||||
|
||||
#### Environment
|
||||
|
||||
- OS: Linux
|
||||
- Python: Match the repository requirements for your checkout
|
||||
- Driver / runtime: NVIDIA CUDA environment with one L40S 48 GB (verified) or an
|
||||
~80 GB GPU for the default config
|
||||
- vLLM version: Match the repository requirements for your checkout
|
||||
- vLLM-Omni version or commit: Use the commit you are deploying from
|
||||
|
||||
#### Offline Commands
|
||||
|
||||
Download the model:
|
||||
|
||||
```bash
|
||||
hf download bytedance-research/MammothModa2-Preview --local-dir ./MammothModa2-Preview
|
||||
```
|
||||
|
||||
Run text-to-image with the shared offline example from the repository root. The
|
||||
deploy config sets `trust_remote_code`, so no extra flag is needed. Forward the
|
||||
MammothModa2 generation parameters as a JSON object through `--extra-body`:
|
||||
|
||||
```bash
|
||||
python examples/offline_inference/text_to_image/text_to_image.py \
|
||||
--model ./MammothModa2-Preview \
|
||||
--deploy-config vllm_omni/deploy/mammoth_moda2.yaml \
|
||||
--prompt "A stylish woman riding a motorcycle in NYC, movie poster style" \
|
||||
--height 1024 \
|
||||
--width 1024 \
|
||||
--extra-body '{"text_guidance_scale": 4.0, "cfg_range": [0.0, 1.0], "num_inference_steps": 50}' \
|
||||
--output mammoth_t2i.png
|
||||
```
|
||||
|
||||
The `--extra-body` JSON forwards MammothModa2-specific parameters into
|
||||
`OmniDiffusionSamplingParams.extra_args`. Keys are filtered against the model's
|
||||
declared `extra_body_params` (see
|
||||
[`vllm_omni/model_extras/mammothmodal2_preview.py`](../../vllm_omni/model_extras/mammothmodal2_preview.py)),
|
||||
so unknown keys for MammothModa2 are silently dropped:
|
||||
|
||||
- `text_guidance_scale` — classifier-free guidance scale for the DiT stage
|
||||
(default `9.0`; CFG is active only when `> 1.0`).
|
||||
- `cfg_range` — relative step range `[start, end]` over which CFG is applied
|
||||
(default `[0.0, 1.0]`).
|
||||
- `num_inference_steps` — number of DiT denoising steps (default `50`).
|
||||
|
||||
`--height` and `--width` must be multiples of 16.
|
||||
|
||||
Run text-to-text through the shared understanding example. It recognizes the
|
||||
MammothModa2 checkpoint and automatically selects `mammoth_moda2_ar.yaml`:
|
||||
|
||||
```bash
|
||||
python examples/offline_inference/x_to_text/x_to_text.py \
|
||||
--model ./MammothModa2-Preview \
|
||||
--prompt "Explain multimodal generation in three sentences."
|
||||
```
|
||||
|
||||
Add an image for image-to-text or image summarization. The shared example
|
||||
uses MammothModa2's chat and vision-token template:
|
||||
|
||||
```bash
|
||||
python examples/offline_inference/x_to_text/x_to_text.py \
|
||||
--model ./MammothModa2-Preview \
|
||||
--image /path/to/input.jpg \
|
||||
--prompt "Please summarize the content of this image."
|
||||
```
|
||||
|
||||
#### Verification
|
||||
|
||||
The example writes the generated image to the `--output` path. Confirm the file
|
||||
exists and is a valid image:
|
||||
|
||||
```bash
|
||||
ls -lh mammoth_t2i.png
|
||||
python -c "from PIL import Image; print(Image.open('mammoth_t2i.png').size)"
|
||||
```
|
||||
|
||||
### 1x AMD MI300X, MammothModa2 Preview
|
||||
|
||||
#### Environment
|
||||
|
||||
- OS: Linux 6.8.0-134-generic, x86_64
|
||||
- Container: official ROCm image built from `docker/Dockerfile.rocm`
|
||||
- Python: 3.12.13
|
||||
- PyTorch: 2.11.0+gitd0c8b1f
|
||||
- Driver / runtime: AMD 6.19.14.31400000 / ROCm 7.2.53211
|
||||
- GPU: one AMD Instinct MI300X, `gfx942:sramecc+:xnack-`, 191.69 GiB visible HBM
|
||||
- vLLM version: 0.27.0+rocm723
|
||||
- vLLM Omni version or commit: `73e1368c7bb940efe1a025859c9d6c8eeeb2e3f0`
|
||||
- Installed vLLM Omni package metadata: `0.27.0rc2.dev44+g55abdade9.rocm`
|
||||
|
||||
#### Offline Commands
|
||||
|
||||
The checked run used the committed stage split, with `gpu_memory_utilization` set to 0.5 for AR and 0.3 for DiT:
|
||||
|
||||
```bash
|
||||
python3 examples/offline_inference/text_to_image/text_to_image.py \
|
||||
--model bytedance-research/MammothModa2-Preview \
|
||||
--deploy-config vllm_omni/deploy/mammoth_moda2.yaml \
|
||||
--prompt "A stylish woman riding a motorcycle in NYC, movie poster style" \
|
||||
--height 1024 \
|
||||
--width 1024 \
|
||||
--seed 42 \
|
||||
--extra-body '{"text_guidance_scale": 4.0, "cfg_range": [0.0, 1.0], "num_inference_steps": 50}' \
|
||||
--enable-diffusion-pipeline-profiler \
|
||||
--log-stats \
|
||||
--output mammoth_t2i.png
|
||||
```
|
||||
|
||||
#### Verification
|
||||
|
||||
The first request took 85.224 seconds. The AR stage generated 4,161 visual tokens in 72.996 seconds, and the DiT stage took 12.163 seconds. AR weight loading used 21.4 GiB and took 8.250 seconds. DiT weight loading used 5.49 GiB and took 1.824 seconds. The largest one second whole device memory sample was 106.57 GiB, including the AR KV cache reserved by the 0.5 memory setting.
|
||||
|
||||
The output was a valid 1024 by 1024 RGB PNG.
|
||||
|
||||
## MammothModa2-Dev unified inference
|
||||
|
||||
MammothModa2-Dev uses a Qwen3-VL AR backbone, while MammothModa2-Preview uses
|
||||
Qwen2.5-VL. vLLM-Omni selects the matching implementation from the nested
|
||||
`llm_config.model_type`; no checkpoint edits or `trust_remote_code` flag are
|
||||
required.
|
||||
|
||||
Text-to-text and image-to-text use the AR-only deploy. Text-to-image loads the
|
||||
Qwen3 generation experts (`gen_mlp`), extra visual vocabulary and image head,
|
||||
then sends the generated visual tokens and hidden states to the DiT stage.
|
||||
|
||||
Download the checkpoint:
|
||||
|
||||
```bash
|
||||
hf download bytedance-research/MammothModa2-Dev --local-dir ./MammothModa2-Dev
|
||||
```
|
||||
|
||||
Run text-to-text through the shared understanding example. It recognizes the
|
||||
Dev checkpoint as MammothModa2 and automatically selects
|
||||
`mammoth_moda2_ar.yaml`:
|
||||
|
||||
```bash
|
||||
python examples/offline_inference/x_to_text/x_to_text.py \
|
||||
--model ./MammothModa2-Dev \
|
||||
--prompt "Explain multimodal generation in three sentences."
|
||||
```
|
||||
|
||||
Add an image for image-to-text or image summarization:
|
||||
|
||||
```bash
|
||||
python examples/offline_inference/x_to_text/x_to_text.py \
|
||||
--model ./MammothModa2-Dev \
|
||||
--image ./image.png \
|
||||
--prompt "Please summarize the content of this image."
|
||||
```
|
||||
|
||||
The Dev checkpoint is approximately 47.55 GiB on disk. In the verified AR-only
|
||||
run, loaded model weights used approximately 16.97 GiB of GPU memory before KV
|
||||
and encoder caches. Allow additional GPU memory for those caches and the input
|
||||
image.
|
||||
Loaded 100 of 645 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user