Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Gao Han
|
6e0bd92f6d | ||
|
tzhouamandClaude Opus 4.8
|
7e7b99b60f |
No files matched your search
@@ -10,7 +10,7 @@ steps:
|
||||
DOCKER_BUILDKIT: "1"
|
||||
# Buildkite will automatically replace this with the actual commit hash
|
||||
VLLM_IMAGE_TAG: "${BUILDKITE_COMMIT}"
|
||||
VLLM_VERSION: "v0.24.0"
|
||||
VLLM_VERSION: "v0.23.0"
|
||||
priority: 100
|
||||
timeout_in_minutes: 60
|
||||
soft_fail: false
|
||||
+4
-14
@@ -26,33 +26,23 @@ steps:
|
||||
agents:
|
||||
queue: "cpu_queue_premerge"
|
||||
|
||||
# L2 Test — PR ready label (diff-aware) or main+NIGHTLY=1 (--e2e, full E2E group)
|
||||
# L2 Test
|
||||
- label: "Upload Ready Pipeline"
|
||||
depends_on: image-build
|
||||
key: upload-ready-pipeline
|
||||
if: __UPLOAD_READY_IF__
|
||||
commands:
|
||||
- |
|
||||
EXTRA=""
|
||||
if [ "${NIGHTLY:-}" = "1" ] && [ "${BUILDKITE_BRANCH}" = "main" ]; then
|
||||
EXTRA="--e2e"
|
||||
fi
|
||||
python3 .buildkite/scripts/upload_pipeline.py --upload $$EXTRA .buildkite/test-ready.yml
|
||||
- python3 .buildkite/scripts/upload_pipeline.py --upload .buildkite/test-ready.yml
|
||||
agents:
|
||||
queue: "cpu_queue_premerge"
|
||||
|
||||
# L3 Test — PR merge-test label (diff-aware) or main+NIGHTLY=1 (--e2e, full E2E group)
|
||||
# L3 Test
|
||||
- label: "Upload Merge Pipeline"
|
||||
depends_on: image-build
|
||||
key: upload-merge-pipeline
|
||||
if: __UPLOAD_MERGE_IF__
|
||||
commands:
|
||||
- |
|
||||
EXTRA=""
|
||||
if [ "${NIGHTLY:-}" = "1" ] && [ "${BUILDKITE_BRANCH}" = "main" ]; then
|
||||
EXTRA="--e2e"
|
||||
fi
|
||||
python3 .buildkite/scripts/upload_pipeline.py --upload $$EXTRA .buildkite/test-merge.yml
|
||||
- python3 .buildkite/scripts/upload_pipeline.py --upload .buildkite/test-merge.yml
|
||||
agents:
|
||||
queue: "cpu_queue_premerge"
|
||||
|
||||
|
||||
@@ -13,7 +13,7 @@ Test pipeline mode (e.g. test-merge.yml):
|
||||
recognize this uploader-only key).
|
||||
|
||||
Usage:
|
||||
python3 upload_pipeline.py [--upload] [--all | --e2e] <pipeline.yml>
|
||||
python3 upload_pipeline.py [--upload] [--all] <pipeline.yml>
|
||||
|
||||
# Bootstrap (replaces upload_pipeline_with_skip_ci.sh):
|
||||
python3 upload_pipeline.py --upload .buildkite/pipeline.yml
|
||||
@@ -23,9 +23,6 @@ Usage:
|
||||
|
||||
# Full suite (rebase pipeline): keep all steps, still strip the uploader-only key:
|
||||
python3 upload_pipeline.py --upload --all .buildkite/test-merge.yml
|
||||
|
||||
# Nightly L2 E2E only: keep the E2E Test group, ignore source_file_dependencies:
|
||||
python3 upload_pipeline.py --upload --e2e .buildkite/test-ready.yml
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -50,7 +47,6 @@ LOG = "upload_pipeline"
|
||||
ROOT = Path(__file__).resolve().parent.parent.parent
|
||||
DOC_SEP = "\n---\n"
|
||||
BOOTSTRAP_MARKER = "__IMAGE_BUILD_IF__"
|
||||
E2E_GROUP_MARKER = "E2E Test"
|
||||
|
||||
|
||||
def _log(message: str) -> None:
|
||||
@@ -326,18 +322,18 @@ def render_bootstrap_pipeline(text: str, *, skip_ci: bool) -> str:
|
||||
nightly_only = (
|
||||
'(build.pull_request.labels includes "nightly-test") || (build.branch == "main" && build.env("NIGHTLY") == "1")'
|
||||
)
|
||||
nightly_main = 'build.branch == "main" && build.env("NIGHTLY") == "1"'
|
||||
ready_pr = 'build.branch != "main" && build.pull_request.labels includes "ready"'
|
||||
merge_main = 'build.branch == "main" && build.env("NIGHTLY") != "1" && build.env("WEEKLY") != "1"'
|
||||
merge_pr = 'build.branch != "main" && build.pull_request.labels includes "merge-test"'
|
||||
if skip_ci:
|
||||
image_if = f"'{nightly_only}'"
|
||||
ready_if = f"'({nightly_main})'"
|
||||
merge_if = f"'({nightly_main})'"
|
||||
ready_if = "'false'"
|
||||
merge_if = "'false'"
|
||||
else:
|
||||
image_if = "'true'"
|
||||
ready_if = f"'(({nightly_main}) || ({ready_pr}))'"
|
||||
merge_if = f"'(({nightly_main}) || (({merge_main}) || ({merge_pr})))'"
|
||||
ready_if = '\'build.branch != "main" && build.pull_request.labels includes "ready"\''
|
||||
merge_if = (
|
||||
'\'(build.branch == "main" && build.env("NIGHTLY") != "1" '
|
||||
'&& build.env("WEEKLY") != "1") || '
|
||||
'(build.branch != "main" && build.pull_request.labels includes "merge-test")\''
|
||||
)
|
||||
|
||||
return (
|
||||
continuation.replace("__IMAGE_BUILD_IF__", image_if)
|
||||
@@ -400,20 +396,6 @@ def _filter_steps(steps: list[Any], changed_files: list[str]) -> list[Any]:
|
||||
return filtered
|
||||
|
||||
|
||||
def _is_e2e_group(step: dict[str, Any]) -> bool:
|
||||
group = step.get("group")
|
||||
return isinstance(group, str) and E2E_GROUP_MARKER in group
|
||||
|
||||
|
||||
def _select_e2e_group_steps(steps: list[Any]) -> list[Any]:
|
||||
selected = [step for step in steps if isinstance(step, dict) and _is_e2e_group(step)]
|
||||
if not selected:
|
||||
_log(f"no group matching {E2E_GROUP_MARKER!r} found")
|
||||
else:
|
||||
_log(f"keep {len(selected)} group(s) matching {E2E_GROUP_MARKER!r}")
|
||||
return selected
|
||||
|
||||
|
||||
def _strip_uploader_metadata_from_steps(steps: list[Any]) -> list[Any]:
|
||||
"""Remove uploader-only keys while keeping all steps (no diff filtering)."""
|
||||
stripped: list[Any] = []
|
||||
@@ -440,16 +422,11 @@ def _strip_uploader_metadata_from_steps(steps: list[Any]) -> list[Any]:
|
||||
def render_test_pipeline(
|
||||
doc: dict[str, Any],
|
||||
changed_files: list[str] | None,
|
||||
*,
|
||||
e2e_only: bool = False,
|
||||
) -> dict[str, Any]:
|
||||
steps = doc.get("steps")
|
||||
if not isinstance(steps, list):
|
||||
return doc
|
||||
if e2e_only:
|
||||
steps = _select_e2e_group_steps(steps)
|
||||
steps = _strip_uploader_metadata_from_steps(steps)
|
||||
elif changed_files is not None:
|
||||
if changed_files is not None:
|
||||
steps = _filter_steps(steps, changed_files)
|
||||
else:
|
||||
steps = _strip_uploader_metadata_from_steps(steps)
|
||||
@@ -463,12 +440,7 @@ def resolve_pipeline_path(arg: str) -> Path:
|
||||
return ROOT / path
|
||||
|
||||
|
||||
def render_pipeline(
|
||||
path: Path,
|
||||
*,
|
||||
force_all: bool = False,
|
||||
e2e_only: bool = False,
|
||||
) -> str:
|
||||
def render_pipeline(path: Path, *, force_all: bool = False) -> str:
|
||||
text = path.read_text(encoding="utf-8")
|
||||
diff_range = resolve_diff_range()
|
||||
changed_files = resolve_changed_files()
|
||||
@@ -476,7 +448,7 @@ def render_pipeline(
|
||||
# ``--all`` forces the keep-all-steps path (no diff-aware skipping) while still
|
||||
# stripping ``source_file_dependencies``. Used by the rebase pipeline so main builds
|
||||
# run the full e2e suite (see .buildkite/rebase-pipeline.yaml).
|
||||
if force_all or e2e_only:
|
||||
if force_all:
|
||||
changed_files = None
|
||||
|
||||
if BOOTSTRAP_MARKER in text:
|
||||
@@ -490,7 +462,7 @@ def render_pipeline(
|
||||
if not isinstance(doc, dict):
|
||||
raise ValueError(f"invalid pipeline YAML: {path}")
|
||||
|
||||
doc = render_test_pipeline(doc, changed_files, e2e_only=e2e_only)
|
||||
doc = render_test_pipeline(doc, changed_files)
|
||||
|
||||
return yaml.safe_dump(doc, sort_keys=False)
|
||||
|
||||
@@ -517,17 +489,11 @@ def main() -> int:
|
||||
action="store_true",
|
||||
help="Pipe rendered YAML to buildkite-agent pipeline upload",
|
||||
)
|
||||
mode = parser.add_mutually_exclusive_group()
|
||||
mode.add_argument(
|
||||
parser.add_argument(
|
||||
"--all",
|
||||
action="store_true",
|
||||
help="Keep all steps (disable diff-aware skipping); still strips source_file_dependencies",
|
||||
)
|
||||
mode.add_argument(
|
||||
"--e2e",
|
||||
action="store_true",
|
||||
help="Keep only the E2E Test group; disable diff-aware skipping for those steps",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
path = resolve_pipeline_path(args.pipeline)
|
||||
@@ -535,7 +501,7 @@ def main() -> int:
|
||||
_log(f"missing pipeline file: {path}")
|
||||
return 1
|
||||
|
||||
rendered = render_pipeline(path, force_all=args.all, e2e_only=args.e2e)
|
||||
rendered = render_pipeline(path, force_all=args.all)
|
||||
if args.upload:
|
||||
upload_to_buildkite(rendered)
|
||||
else:
|
||||
|
||||
@@ -7,29 +7,17 @@ steps:
|
||||
|
||||
- group: ":card_index_dividers: Simple Test"
|
||||
steps:
|
||||
- label: "Simple · Diffusion & Model Executor Test (1/2)"
|
||||
- label: "Simple · Diffusion & Model Executor Test"
|
||||
agent_pool: mi325_1
|
||||
depends_on: amd-build
|
||||
timeout_in_minutes: 70
|
||||
timeout_in_minutes: 50
|
||||
mirror_hardwares: [amdproduction]
|
||||
grade: Blocking
|
||||
commands:
|
||||
- export VLLM_ROCM_USE_AITER=0
|
||||
# ignore test_teacache_extractors.py because it use rocm gemm kernel from vLLM
|
||||
# that is not supported on CPU
|
||||
# CPU tests on ROCm gpu machines are very slow, so we give them more time.
|
||||
- "timeout 60m pytest -sv tests/diffusion -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"
|
||||
|
||||
- label: "Simple · Diffusion & Model Executor Test (2/2)"
|
||||
agent_pool: mi325_1
|
||||
depends_on: amd-build
|
||||
timeout_in_minutes: 70
|
||||
mirror_hardwares: [amdproduction]
|
||||
grade: Blocking
|
||||
commands:
|
||||
- export VLLM_ROCM_USE_AITER=0
|
||||
# CPU tests on ROCm gpu machines are very slow, so we give them more time.
|
||||
- "timeout 60m pytest -sv tests/model_executor -m 'core_model and cpu' --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"
|
||||
- "timeout 40m pytest -sv tests/diffusion tests/model_executor -m 'core_model and cpu' --ignore=tests/diffusion/cache/test_teacache_extractors.py --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"
|
||||
|
||||
- label: "Simple · Other Test"
|
||||
agent_pool: mi325_1
|
||||
@@ -170,7 +158,7 @@ steps:
|
||||
grade: Blocking
|
||||
commands:
|
||||
- |
|
||||
timeout 50m bash -c '
|
||||
timeout 20m bash -c '
|
||||
export VLLM_LOGGING_LEVEL=DEBUG
|
||||
export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
|
||||
@@ -182,14 +170,39 @@ steps:
|
||||
depends_on: amd-build
|
||||
mirror_hardwares: [amdproduction]
|
||||
grade: Blocking
|
||||
timeout_in_minutes: 50
|
||||
commands:
|
||||
- |
|
||||
timeout 40m bash -c '
|
||||
timeout 30m bash -c '
|
||||
export VLLM_LOGGING_LEVEL=DEBUG
|
||||
export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
|
||||
pytest -s -v tests/e2e/online_serving/test_qwen3_tts_base.py tests/e2e/offline_inference/test_qwen3_tts_base.py -m 'advanced_model and cuda' --run-level 'advanced_model'
|
||||
pytest -s -v tests/e2e/online_serving/test_qwen3_tts_base.py tests/e2e/offline_inference/test_qwen3_tts_base.py -m "advanced_model" --run-level "advanced_model"
|
||||
'
|
||||
|
||||
# It is the whisper ASR validation step that is slow, not the Voxtral TTS generation.
|
||||
# whisper ASR runs on CPU.
|
||||
- label: "Voxtral-TTS E2E Test (Online Serving)"
|
||||
agent_pool: mi325_1
|
||||
depends_on: amd-build
|
||||
mirror_hardwares: [amdproduction]
|
||||
grade: Blocking
|
||||
commands:
|
||||
- |
|
||||
timeout 40m bash -c '
|
||||
export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
pytest -s -v tests/e2e/online_serving/test_voxtral_tts.py -m "advanced_model" --run-level "advanced_model"
|
||||
'
|
||||
|
||||
- label: "Voxtral-TTS E2E Test (Offline Inference)"
|
||||
agent_pool: mi325_1
|
||||
depends_on: amd-build
|
||||
mirror_hardwares: [amdproduction]
|
||||
grade: Blocking
|
||||
commands:
|
||||
- |
|
||||
timeout 30m bash -c '
|
||||
export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
pytest -s -v tests/e2e/offline_inference/test_voxtral_tts.py -m "advanced_model" --run-level "advanced_model"
|
||||
'
|
||||
|
||||
# TODO: Bagel test on ROCm is very unstable. @tjtanaa
|
||||
|
||||
@@ -161,6 +161,19 @@ steps:
|
||||
|
||||
- group: ":card_index_dividers: E2E Test"
|
||||
steps:
|
||||
- label: "MiMo-Audio E2E Test with H100"
|
||||
agent_pool: mi325_1
|
||||
depends_on: amd-build
|
||||
mirror_hardwares: [amdproduction]
|
||||
grade: Blocking
|
||||
commands:
|
||||
- |
|
||||
timeout 30m bash -c '
|
||||
export VLLM_LOGGING_LEVEL=DEBUG
|
||||
export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
pytest -s -v tests/e2e/online_serving/test_mimo_audio.py -m "core_model" --run-level "core_model"
|
||||
'
|
||||
|
||||
- label: "Qwen3-TTS E2E Test"
|
||||
agent_pool: mi325_1
|
||||
depends_on: amd-build
|
||||
@@ -180,7 +193,7 @@ steps:
|
||||
commands:
|
||||
- |
|
||||
timeout 20m bash -c '
|
||||
pytest -s -v tests/e2e/online_serving/test_cosyvoice3_tts_expansion.py -m "slow" --run-level "core_model"
|
||||
pytest -s -v tests/e2e/online_serving/test_cosyvoice3_tts_expansion.py -m "full_model" --run-level "core_model"
|
||||
'
|
||||
|
||||
# TODO: Bagel test on ROCm is very unstable. @tjtanaa
|
||||
|
||||
+350
-57
@@ -61,14 +61,13 @@ steps:
|
||||
- "/fsx/hf_cache:/fsx/hf_cache"
|
||||
|
||||
- label: "Distributed Test"
|
||||
depends_on: upload-merge-pipeline
|
||||
timeout_in_minutes: 60
|
||||
commands:
|
||||
- |
|
||||
timeout 55m bash -c "
|
||||
set -e
|
||||
export VLLM_IMAGE_FETCH_TIMEOUT=60
|
||||
pytest -s -v tests/distributed -m 'advanced_model and cuda and H100' --run-level 'advanced_model'
|
||||
pytest -s -v tests/distributed -m 'advanced_model and cuda' --run-level 'advanced_model'
|
||||
"
|
||||
agents:
|
||||
queue: "mithril-h100-pool"
|
||||
@@ -108,10 +107,7 @@ steps:
|
||||
depends_on: upload-merge-pipeline
|
||||
steps:
|
||||
- label: "Entrypoint Test with H100"
|
||||
# The suite relaunches an OmniServer per test class (~30 model loads,
|
||||
# incl. ten ~27.5 GiB Bagel loads); 60 min passed only ~1 in 3 runs
|
||||
# even on main, and vLLM 0.25 weight loading needs ~10 min more.
|
||||
timeout_in_minutes: 90
|
||||
timeout_in_minutes: 60
|
||||
commands:
|
||||
- pytest -s -v tests/entrypoints/ -m "advanced_model and cuda and H100" --run-level "advanced_model"
|
||||
agents:
|
||||
@@ -149,9 +145,7 @@ steps:
|
||||
type: DirectoryOrCreate
|
||||
|
||||
- label: "Entrypoint Test with L4"
|
||||
# 30 min left no headroom for a cold image pull (~11 min for new base
|
||||
# layers) plus per-class engine relaunches; main passed at 26/30.
|
||||
timeout_in_minutes: 45
|
||||
timeout_in_minutes: 30
|
||||
commands:
|
||||
- pytest -s -v tests/entrypoints/ -m "advanced_model and cuda and L4" --run-level "advanced_model"
|
||||
agents:
|
||||
@@ -170,6 +164,28 @@ steps:
|
||||
- group: ":card_index_dividers: E2E Test"
|
||||
depends_on: upload-merge-pipeline
|
||||
steps:
|
||||
- label: "TTS · Step-Audio2 E2E Test"
|
||||
timeout_in_minutes: 20
|
||||
commands:
|
||||
- |
|
||||
timeout 20m bash -c "
|
||||
export VLLM_LOGGING_LEVEL=DEBUG
|
||||
export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
pytest -s -v tests/e2e/offline_inference/test_step_audio2.py -m 'advanced_model' --run-level 'advanced_model'
|
||||
"
|
||||
agents:
|
||||
queue: "gpu_4_queue"
|
||||
plugins:
|
||||
- docker#v5.2.0:
|
||||
image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
|
||||
always-pull: true
|
||||
propagate-environment: true
|
||||
shm-size: "8gb"
|
||||
environment:
|
||||
- "HF_HOME=/fsx/hf_cache"
|
||||
volumes:
|
||||
- "/fsx/hf_cache:/fsx/hf_cache"
|
||||
|
||||
- label: "TTS · Qwen3-TTS CustomVoice Test"
|
||||
source_file_dependencies:
|
||||
- tests/e2e/online_serving/test_qwen3_tts_customvoice.py
|
||||
@@ -179,7 +195,7 @@ steps:
|
||||
- vllm_omni/deploy/qwen3_tts.yaml
|
||||
commands:
|
||||
- |
|
||||
timeout 40m bash -c "
|
||||
timeout 20m bash -c "
|
||||
export VLLM_LOGGING_LEVEL=DEBUG
|
||||
export VLLM_ALLOW_LONG_MAX_MODEL_LEN=1
|
||||
pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py tests/e2e/offline_inference/test_qwen3_tts_customvoice.py -m 'advanced_model and cuda' --run-level 'advanced_model'
|
||||
@@ -206,7 +222,7 @@ steps:
|
||||
- vllm_omni/deploy/qwen3_tts.yaml
|
||||
commands:
|
||||
- |
|
||||
timeout 40m bash -c "
|
||||
timeout 20m bash -c "
|
||||
export VLLM_LOGGING_LEVEL=DEBUG
|
||||
export VLLM_ALLOW_LONG_MAX_MODEL_LEN=1
|
||||
pytest -s -v tests/e2e/online_serving/test_qwen3_tts_base.py tests/e2e/offline_inference/test_qwen3_tts_base.py -m 'advanced_model and cuda' --run-level 'advanced_model'
|
||||
@@ -224,19 +240,19 @@ steps:
|
||||
volumes:
|
||||
- "/fsx/hf_cache:/fsx/hf_cache"
|
||||
|
||||
- label: "TTS · Higgs-Audio-V3 Test"
|
||||
- label: "TTS · Ming-Omni-TTS Test"
|
||||
source_file_dependencies:
|
||||
- tests/e2e/online_serving/test_higgs_audio_v3.py
|
||||
- vllm_omni/model_executor/models/higgs_audio_v3/
|
||||
- vllm_omni/model_executor/stage_input_processors/higgs_audio_v3.py
|
||||
- vllm_omni/deploy/higgs_multimodal_qwen3.yaml
|
||||
- tests/e2e/online_serving/test_ming_tts.py
|
||||
- tests/e2e/offline_inference/test_ming_tts.py
|
||||
- vllm_omni/model_executor/models/ming_tts/
|
||||
- vllm_omni/model_executor/stage_input_processors/ming_tts.py
|
||||
- vllm_omni/deploy/ming_tts.yaml
|
||||
timeout_in_minutes: 20
|
||||
commands:
|
||||
- |
|
||||
timeout 20m bash -c "
|
||||
export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
export VLLM_USE_DEEP_GEMM=0
|
||||
export VLLM_MOE_USE_DEEP_GEMM=0
|
||||
pytest -s -v tests/e2e/online_serving/test_higgs_audio_v3.py -m 'advanced_model' --run-level 'advanced_model'
|
||||
export VLLM_LOGGING_LEVEL=DEBUG
|
||||
pytest -s -v tests/e2e/online_serving/test_ming_tts.py tests/e2e/offline_inference/test_ming_tts.py -m 'advanced_model and cuda' --run-level 'advanced_model'
|
||||
"
|
||||
agents:
|
||||
queue: "gpu_1_queue"
|
||||
@@ -248,45 +264,104 @@ steps:
|
||||
shm-size: "8gb"
|
||||
environment:
|
||||
- "HF_HOME=/fsx/hf_cache"
|
||||
- "HF_TOKEN"
|
||||
volumes:
|
||||
- "/fsx/hf_cache:/fsx/hf_cache"
|
||||
|
||||
- label: "TTS · IndexTTS2 Test"
|
||||
- label: "TTS · Voxtral-TTS Test"
|
||||
source_file_dependencies:
|
||||
- tests/e2e/online_serving/test_indextts2.py
|
||||
- tests/e2e/offline_inference/test_indextts2.py
|
||||
- tests/model_executor/models/indextts2/
|
||||
- tests/model_executor/stage_input_processors/test_indextts2.py
|
||||
- tests/assets/indextts2/
|
||||
- vllm_omni/model_executor/models/indextts2/
|
||||
- vllm_omni/model_executor/stage_input_processors/indextts2.py
|
||||
- vllm_omni/deploy/indextts2.yaml
|
||||
- vllm_omni/deploy/indextts2_low_latency.yaml
|
||||
- vllm_omni/entrypoints/openai/serving_speech.py
|
||||
- vllm_omni/entrypoints/openai/tts_adapters/indextts2.py
|
||||
- vllm_omni/config/pipeline_registry.py
|
||||
- vllm_omni/model_executor/models/registry.py
|
||||
timeout_in_minutes: 60
|
||||
- tests/e2e/offline_inference/test_voxtral_tts.py
|
||||
- tests/e2e/online_serving/test_voxtral_tts.py
|
||||
- vllm_omni/model_executor/models/voxtral_tts/
|
||||
- vllm_omni/model_executor/stage_input_processors/voxtral_tts.py
|
||||
- vllm_omni/deploy/voxtral_tts.yaml
|
||||
timeout_in_minutes: 20
|
||||
commands:
|
||||
- |
|
||||
timeout 45m bash -c "
|
||||
timeout 20m bash -c "
|
||||
export VLLM_LOGGING_LEVEL=DEBUG
|
||||
pytest -s -v tests/e2e/online_serving/test_indextts2.py tests/e2e/offline_inference/test_indextts2.py -m 'advanced_model and cuda' --run-level 'advanced_model'
|
||||
pytest -s -v tests/e2e/offline_inference/test_voxtral_tts.py tests/e2e/online_serving/test_voxtral_tts.py -m 'advanced_model and cuda' --run-level 'advanced_model'
|
||||
"
|
||||
agents:
|
||||
queue: "gpu_4_queue"
|
||||
queue: "mithril-h100-pool"
|
||||
plugins:
|
||||
- docker#v5.2.0:
|
||||
image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
|
||||
always-pull: true
|
||||
propagate-environment: true
|
||||
environment:
|
||||
- "HF_HOME=/fsx/hf_cache"
|
||||
- "HF_TOKEN"
|
||||
gpus: all
|
||||
volumes:
|
||||
- "/fsx/hf_cache:/fsx/hf_cache"
|
||||
- kubernetes:
|
||||
podSpec:
|
||||
containers:
|
||||
- image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
|
||||
resources:
|
||||
limits:
|
||||
nvidia.com/gpu: 1
|
||||
volumeMounts:
|
||||
- name: devshm
|
||||
mountPath: /dev/shm
|
||||
- name: hf-cache
|
||||
mountPath: /root/.cache/huggingface
|
||||
env:
|
||||
- name: HF_HOME
|
||||
value: /root/.cache/huggingface
|
||||
- name: HF_TOKEN
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: hf-token-secret
|
||||
key: token
|
||||
nodeSelector:
|
||||
node.kubernetes.io/instance-type: gpu-h100-sxm
|
||||
volumes:
|
||||
- name: devshm
|
||||
emptyDir:
|
||||
medium: Memory
|
||||
- name: hf-cache
|
||||
hostPath:
|
||||
path: /mnt/hf-cache
|
||||
type: DirectoryOrCreate
|
||||
|
||||
- label: "TTS · GLM-TTS Test"
|
||||
source_file_dependencies:
|
||||
- tests/e2e/online_serving/test_glm_tts.py
|
||||
- tests/e2e/offline_inference/test_glm_tts.py
|
||||
- vllm_omni/model_executor/models/glm_tts/
|
||||
- vllm_omni/model_executor/stage_input_processors/glm_tts.py
|
||||
- vllm_omni/deploy/glm_tts.yaml
|
||||
timeout_in_minutes: 20
|
||||
commands:
|
||||
- |
|
||||
timeout 20m bash -c "
|
||||
export VLLM_LOGGING_LEVEL=DEBUG
|
||||
pytest -s -v tests/e2e/online_serving/test_glm_tts.py tests/e2e/offline_inference/test_glm_tts.py -m 'advanced_model and cuda' --run-level 'advanced_model'
|
||||
"
|
||||
agents:
|
||||
queue: "mithril-h100-pool"
|
||||
plugins:
|
||||
- kubernetes:
|
||||
podSpec:
|
||||
containers:
|
||||
- image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
|
||||
resources:
|
||||
limits:
|
||||
nvidia.com/gpu: 1
|
||||
volumeMounts:
|
||||
- name: devshm
|
||||
mountPath: /dev/shm
|
||||
- name: hf-cache
|
||||
mountPath: /root/.cache/huggingface
|
||||
env:
|
||||
- name: HF_HOME
|
||||
value: /root/.cache/huggingface
|
||||
- name: HF_TOKEN
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: hf-token-secret
|
||||
key: token
|
||||
nodeSelector:
|
||||
node.kubernetes.io/instance-type: gpu-h100-sxm
|
||||
volumes:
|
||||
- name: devshm
|
||||
emptyDir:
|
||||
medium: Memory
|
||||
- name: hf-cache
|
||||
hostPath:
|
||||
path: /mnt/hf-cache
|
||||
type: DirectoryOrCreate
|
||||
|
||||
- label: "Omni · Qwen3-Omni Test"
|
||||
source_file_dependencies:
|
||||
@@ -295,7 +370,7 @@ steps:
|
||||
- vllm_omni/model_executor/models/qwen3_omni/
|
||||
- vllm_omni/model_executor/stage_input_processors/qwen3_omni.py
|
||||
- vllm_omni/deploy/qwen3_omni_moe.yaml
|
||||
timeout_in_minutes: 50
|
||||
timeout_in_minutes: 30
|
||||
commands:
|
||||
- pytest -s -v tests/e2e/offline_inference/test_qwen3_omni.py tests/e2e/online_serving/test_qwen3_omni.py -m 'advanced_model and cuda' --run-level 'advanced_model'
|
||||
agents:
|
||||
@@ -332,6 +407,49 @@ steps:
|
||||
path: /mnt/hf-cache
|
||||
type: DirectoryOrCreate
|
||||
|
||||
- label: "Omni · Mimo Audio Test"
|
||||
source_file_dependencies:
|
||||
- tests/e2e/online_serving/test_mimo_audio.py
|
||||
- vllm_omni/model_executor/models/mimo_audio/
|
||||
- vllm_omni/model_executor/stage_input_processors/mimo_audio.py
|
||||
- vllm_omni/deploy/mimo_audio.yaml
|
||||
timeout_in_minutes: 30
|
||||
commands:
|
||||
- pytest -s -v tests/e2e/online_serving/test_mimo_audio.py -m 'advanced_model and cuda' --run-level 'advanced_model'
|
||||
agents:
|
||||
queue: "mithril-h100-pool"
|
||||
plugins:
|
||||
- kubernetes:
|
||||
podSpec:
|
||||
containers:
|
||||
- image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
|
||||
resources:
|
||||
limits:
|
||||
nvidia.com/gpu: 2
|
||||
volumeMounts:
|
||||
- name: devshm
|
||||
mountPath: /dev/shm
|
||||
- name: hf-cache
|
||||
mountPath: /root/.cache/huggingface
|
||||
env:
|
||||
- name: HF_HOME
|
||||
value: /root/.cache/huggingface
|
||||
- name: HF_TOKEN
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: hf-token-secret
|
||||
key: token
|
||||
nodeSelector:
|
||||
node.kubernetes.io/instance-type: gpu-h100-sxm
|
||||
volumes:
|
||||
- name: devshm
|
||||
emptyDir:
|
||||
medium: Memory
|
||||
- name: hf-cache
|
||||
hostPath:
|
||||
path: /mnt/hf-cache
|
||||
type: DirectoryOrCreate
|
||||
|
||||
- label: "TTS · VoxCPM2 Test"
|
||||
source_file_dependencies:
|
||||
- tests/e2e/offline_inference/test_voxcpm2_tts.py
|
||||
@@ -360,19 +478,22 @@ steps:
|
||||
volumes:
|
||||
- "/fsx/hf_cache:/fsx/hf_cache"
|
||||
|
||||
- label: "TTS · SoulX-Singer Test"
|
||||
- label: "Omni · Higgs-Audio V2 Test"
|
||||
source_file_dependencies:
|
||||
- tests/e2e/offline_inference/test_soulxsinger.py
|
||||
- vllm_omni/diffusion/models/soulx_singer/
|
||||
- vllm_omni/deploy/soulxsinger_svc.yaml
|
||||
- vllm_omni/deploy/soulxsinger_svs.yaml
|
||||
- tests/e2e/online_serving/test_higgs_audio_v2.py
|
||||
- tests/e2e/offline_inference/test_higgs_audio_v2.py
|
||||
- vllm_omni/model_executor/models/higgs_audio_v2/
|
||||
- vllm_omni/model_executor/stage_input_processors/higgs_audio_v2.py
|
||||
- vllm_omni/deploy/higgs_audio_v2.yaml
|
||||
timeout_in_minutes: 30
|
||||
commands:
|
||||
- |
|
||||
timeout 30m bash -c "
|
||||
export VLLM_LOGGING_LEVEL=DEBUG
|
||||
export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
pytest -s -v tests/e2e/offline_inference/test_soulxsinger.py -k 'svc or precomputed' -m 'advanced_model and cuda' --run-level 'advanced_model'
|
||||
export VLLM_USE_DEEP_GEMM=0
|
||||
export VLLM_MOE_USE_DEEP_GEMM=0
|
||||
pytest -s -v tests/e2e/online_serving/test_higgs_audio_v2.py tests/e2e/offline_inference/test_higgs_audio_v2.py -m 'advanced_model and cuda' --run-level 'advanced_model'
|
||||
"
|
||||
agents:
|
||||
queue: "gpu_1_queue"
|
||||
@@ -429,6 +550,88 @@ steps:
|
||||
path: /mnt/hf-cache
|
||||
type: DirectoryOrCreate
|
||||
|
||||
- label: "Diffusion · Longcat Image Edit Test"
|
||||
source_file_dependencies:
|
||||
- tests/e2e/online_serving/test_longcat_image_edit.py
|
||||
- vllm_omni/diffusion/models/longcat_image/
|
||||
timeout_in_minutes: 20
|
||||
commands:
|
||||
- pytest -s -v tests/e2e/online_serving/test_longcat_image_edit.py -m 'advanced_model and cuda' --run-level 'advanced_model'
|
||||
agents:
|
||||
queue: "mithril-h100-pool"
|
||||
plugins:
|
||||
- kubernetes:
|
||||
podSpec:
|
||||
containers:
|
||||
- image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
|
||||
resources:
|
||||
limits:
|
||||
nvidia.com/gpu: 1
|
||||
volumeMounts:
|
||||
- name: devshm
|
||||
mountPath: /dev/shm
|
||||
- name: hf-cache
|
||||
mountPath: /root/.cache/huggingface
|
||||
env:
|
||||
- name: HF_HOME
|
||||
value: /root/.cache/huggingface
|
||||
- name: HF_TOKEN
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: hf-token-secret
|
||||
key: token
|
||||
nodeSelector:
|
||||
node.kubernetes.io/instance-type: gpu-h100-sxm
|
||||
volumes:
|
||||
- name: devshm
|
||||
emptyDir:
|
||||
medium: Memory
|
||||
- name: hf-cache
|
||||
hostPath:
|
||||
path: /mnt/hf-cache
|
||||
type: DirectoryOrCreate
|
||||
|
||||
- label: "Diffusion · Longcat Image Test"
|
||||
source_file_dependencies:
|
||||
- tests/e2e/online_serving/test_longcat_image.py
|
||||
- vllm_omni/diffusion/models/longcat_image/
|
||||
timeout_in_minutes: 20
|
||||
commands:
|
||||
- pytest -s -v tests/e2e/online_serving/test_longcat_image.py -m 'advanced_model and cuda' --run-level 'advanced_model'
|
||||
agents:
|
||||
queue: "mithril-h100-pool"
|
||||
plugins:
|
||||
- kubernetes:
|
||||
podSpec:
|
||||
containers:
|
||||
- image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
|
||||
resources:
|
||||
limits:
|
||||
nvidia.com/gpu: 1
|
||||
volumeMounts:
|
||||
- name: devshm
|
||||
mountPath: /dev/shm
|
||||
- name: hf-cache
|
||||
mountPath: /root/.cache/huggingface
|
||||
env:
|
||||
- name: HF_HOME
|
||||
value: /root/.cache/huggingface
|
||||
- name: HF_TOKEN
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: hf-token-secret
|
||||
key: token
|
||||
nodeSelector:
|
||||
node.kubernetes.io/instance-type: gpu-h100-sxm
|
||||
volumes:
|
||||
- name: devshm
|
||||
emptyDir:
|
||||
medium: Memory
|
||||
- name: hf-cache
|
||||
hostPath:
|
||||
path: /mnt/hf-cache
|
||||
type: DirectoryOrCreate
|
||||
|
||||
- label: "Diffusion · Bagel Test"
|
||||
source_file_dependencies:
|
||||
- tests/e2e/offline_inference/test_bagel.py
|
||||
@@ -608,6 +811,50 @@ steps:
|
||||
path: /mnt/hf-cache
|
||||
type: DirectoryOrCreate
|
||||
|
||||
- label: "Diffusion · HunyuanVideo 1.5 Test"
|
||||
source_file_dependencies:
|
||||
- tests/e2e/online_serving/test_hunyuan_video_15.py
|
||||
- vllm_omni/diffusion/models/hunyuan_video/
|
||||
timeout_in_minutes: 40
|
||||
commands:
|
||||
- |
|
||||
timeout 40m bash -c "
|
||||
pytest -s -v tests/e2e/online_serving/test_hunyuan_video_15.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model'
|
||||
"
|
||||
agents:
|
||||
queue: "mithril-h100-pool"
|
||||
plugins:
|
||||
- kubernetes:
|
||||
podSpec:
|
||||
containers:
|
||||
- image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
|
||||
resources:
|
||||
limits:
|
||||
nvidia.com/gpu: 1
|
||||
volumeMounts:
|
||||
- name: devshm
|
||||
mountPath: /dev/shm
|
||||
- name: hf-cache
|
||||
mountPath: /root/.cache/huggingface
|
||||
env:
|
||||
- name: HF_HOME
|
||||
value: /root/.cache/huggingface
|
||||
- name: HF_TOKEN
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: hf-token-secret
|
||||
key: token
|
||||
nodeSelector:
|
||||
node.kubernetes.io/instance-type: gpu-h100-sxm
|
||||
volumes:
|
||||
- name: devshm
|
||||
emptyDir:
|
||||
medium: Memory
|
||||
- name: hf-cache
|
||||
hostPath:
|
||||
path: /mnt/hf-cache
|
||||
type: DirectoryOrCreate
|
||||
|
||||
- label: "Diffusion · Z Image Test"
|
||||
source_file_dependencies:
|
||||
- tests/e2e/offline_inference/test_z_image.py
|
||||
@@ -632,10 +879,11 @@ steps:
|
||||
- label: "Diffusion · FLUX.2 Klein Test"
|
||||
source_file_dependencies:
|
||||
- tests/e2e/offline_inference/test_flux2_klein.py
|
||||
- tests/e2e/online_serving/test_flux2_klein.py
|
||||
- vllm_omni/diffusion/models/flux2_klein/
|
||||
timeout_in_minutes: 30
|
||||
commands:
|
||||
- pytest -s -v tests/e2e/offline_inference/test_flux2_klein.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model'
|
||||
- pytest -s -v tests/e2e/offline_inference/test_flux2_klein.py tests/e2e/online_serving/test_flux2_klein.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model'
|
||||
agents:
|
||||
queue: "gpu_1_queue" # g6.4xlarge instance on AWS, has 1 L4 GPU
|
||||
plugins:
|
||||
@@ -648,3 +896,48 @@ steps:
|
||||
- "HF_TOKEN"
|
||||
volumes:
|
||||
- "/fsx/hf_cache:/fsx/hf_cache"
|
||||
|
||||
- label: "Diffusion · SD3 Test"
|
||||
source_file_dependencies:
|
||||
- tests/e2e/online_serving/test_sd3.py
|
||||
- vllm_omni/diffusion/models/sd3/
|
||||
timeout_in_minutes: 45
|
||||
commands:
|
||||
- |
|
||||
timeout 45m bash -c "
|
||||
export VLLM_IMAGE_FETCH_TIMEOUT=60
|
||||
pytest -s -v tests/e2e/online_serving/test_sd3.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model'
|
||||
"
|
||||
agents:
|
||||
queue: "mithril-h100-pool"
|
||||
plugins:
|
||||
- kubernetes:
|
||||
podSpec:
|
||||
containers:
|
||||
- image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
|
||||
resources:
|
||||
limits:
|
||||
nvidia.com/gpu: 4
|
||||
volumeMounts:
|
||||
- name: devshm
|
||||
mountPath: /dev/shm
|
||||
- name: hf-cache
|
||||
mountPath: /root/.cache/huggingface
|
||||
env:
|
||||
- name: HF_HOME
|
||||
value: /root/.cache/huggingface
|
||||
- name: HF_TOKEN
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: hf-token-secret
|
||||
key: token
|
||||
nodeSelector:
|
||||
node.kubernetes.io/instance-type: gpu-h100-sxm
|
||||
volumes:
|
||||
- name: devshm
|
||||
emptyDir:
|
||||
medium: Memory
|
||||
- name: hf-cache
|
||||
hostPath:
|
||||
path: /mnt/hf-cache
|
||||
type: DirectoryOrCreate
|
||||
+125
-245
@@ -12,7 +12,7 @@ steps:
|
||||
if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test" || build.pull_request.labels includes "omni-test"
|
||||
steps:
|
||||
- label: ":full_moon: Omni · Function Test with H100"
|
||||
timeout_in_minutes: 120
|
||||
timeout_in_minutes: 90
|
||||
commands:
|
||||
- pytest -sv tests/e2e --run-level "full_model" -m "full_model and H100 and omni" --ignore=tests/e2e/accuracy --ignore=tests/e2e/online_serving/test_qwen3_omni_multi_replicas.py
|
||||
agents:
|
||||
@@ -49,6 +49,25 @@ steps:
|
||||
path: /mnt/hf-cache
|
||||
type: DirectoryOrCreate
|
||||
|
||||
- label: ":full_moon: Omni · Function Test with L4"
|
||||
timeout_in_minutes: 90
|
||||
commands:
|
||||
- export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
|
||||
- pytest -s -v tests/e2e/*/test_qwen2_5_omni_expansion.py --run-level "core_model"
|
||||
agents:
|
||||
queue: "gpu_4_queue" # g6.12xlarge instance on AWS, has 4 L4 GPU
|
||||
plugins:
|
||||
- docker#v5.2.0:
|
||||
image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
|
||||
always-pull: true
|
||||
propagate-environment: true
|
||||
shm-size: "8gb"
|
||||
environment:
|
||||
- "HF_HOME=/fsx/hf_cache"
|
||||
- "HF_TOKEN"
|
||||
volumes:
|
||||
- "/fsx/hf_cache:/fsx/hf_cache"
|
||||
|
||||
- label: ":full_moon: Omni · Doc Test with L4"
|
||||
timeout_in_minutes: 90
|
||||
commands:
|
||||
@@ -153,7 +172,7 @@ steps:
|
||||
|
||||
- label: ":full_moon: Omni · Perf Test · No Async Chunk"
|
||||
key: nightly-omni-performance-no-async-chunk
|
||||
timeout_in_minutes: 300
|
||||
timeout_in_minutes: 180
|
||||
commands:
|
||||
- export BENCHMARK_DIR=tests/dfx/perf/results
|
||||
- |
|
||||
@@ -376,11 +395,9 @@ steps:
|
||||
if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test" || build.pull_request.labels includes "tts-test"
|
||||
steps:
|
||||
- label: ":full_moon: TTS · Function Test with L4"
|
||||
timeout_in_minutes: 120
|
||||
timeout_in_minutes: 90
|
||||
commands:
|
||||
- export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
|
||||
- export VLLM_USE_DEEP_GEMM="0"
|
||||
- export VLLM_MOE_USE_DEEP_GEMM="0"
|
||||
- pytest -s -v tests/e2e/ -m "full_model and L4 and tts" --run-level "full_model" --ignore=tests/e2e/accuracy --ignore=tests/e2e/online_serving/test_voxcpm2_tts_expansion.py
|
||||
agents:
|
||||
queue: "gpu_1_queue" # g6.12xlarge instance on AWS, has 4 L4 GPU
|
||||
@@ -551,84 +568,54 @@ steps:
|
||||
path: /mnt/hf-cache
|
||||
type: DirectoryOrCreate
|
||||
|
||||
- label: ":full_moon: TTS · Higgs-Audio-V3 · Perf Test"
|
||||
key: nightly-tts-performance-higgs-audio-v3
|
||||
timeout_in_minutes: 180
|
||||
commands:
|
||||
- export BENCHMARK_DIR=tests/dfx/perf/results
|
||||
- export VLLM_USE_DEEP_GEMM="0"
|
||||
- export VLLM_MOE_USE_DEEP_GEMM="0"
|
||||
- |
|
||||
set +e
|
||||
pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_higgs_audio_v3.json
|
||||
EXIT=$$?
|
||||
buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
|
||||
exit $$EXIT
|
||||
agents:
|
||||
queue: "mithril-h100-pool"
|
||||
plugins:
|
||||
- kubernetes:
|
||||
podSpec:
|
||||
containers:
|
||||
- image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
|
||||
resources:
|
||||
limits:
|
||||
nvidia.com/gpu: 1
|
||||
volumeMounts:
|
||||
- name: devshm
|
||||
mountPath: /dev/shm
|
||||
- name: hf-cache
|
||||
mountPath: /root/.cache/huggingface
|
||||
env:
|
||||
- name: HF_HOME
|
||||
value: /root/.cache/huggingface
|
||||
- name: HF_TOKEN
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: hf-token-secret
|
||||
key: token
|
||||
nodeSelector:
|
||||
node.kubernetes.io/instance-type: gpu-h100-sxm
|
||||
volumes:
|
||||
- name: devshm
|
||||
emptyDir:
|
||||
medium: Memory
|
||||
- name: hf-cache
|
||||
hostPath:
|
||||
path: /mnt/hf-cache
|
||||
type: DirectoryOrCreate
|
||||
|
||||
- group: ":card_index_dividers: Tiny Model Tests [Multi-GPU]"
|
||||
key: nightly-tiny-model-test-group
|
||||
- label: ":full_moon: TTS · Higgs-Audio-V3 · Perf Test"
|
||||
key: nightly-tts-performance-higgs-audio-v3
|
||||
depends_on: upload-nightly-pipeline
|
||||
if: >-
|
||||
build.env("NIGHTLY") == "1" ||
|
||||
build.pull_request.labels includes "nightly-test"
|
||||
steps:
|
||||
- label: ":full_moon: Tiny Model Tests · Diffusion (Multi-GPU accelerations)"
|
||||
timeout_in_minutes: 30
|
||||
# NOTE: Single-GPU (core_model) tiny model tests run on the ready label,
|
||||
# so nightly only adds the multi-GPU parallelism configurations.
|
||||
#
|
||||
# This is a bit confusing, but `full_model` is used to mark the multigpu tests, and
|
||||
# --run-level `core_model` is used to indicate that we should use tiny model weights
|
||||
# for the CI, since these tests should pass irrespective of whether or not we use
|
||||
# the full weights or not.
|
||||
commands:
|
||||
- timeout 25m pytest -sv tests/model_tests/diffusion -m 'full_model and cuda' --run-level "core_model"
|
||||
agents:
|
||||
queue: "gpu_4_queue" # g6.12xlarge instance on AWS, has 4 L4 GPU
|
||||
plugins:
|
||||
- docker#v5.2.0:
|
||||
image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
|
||||
always-pull: true
|
||||
propagate-environment: true
|
||||
shm-size: "8gb"
|
||||
environment:
|
||||
- "HF_HOME=/fsx/hf_cache"
|
||||
- "HF_TOKEN"
|
||||
volumes:
|
||||
- "/fsx/hf_cache:/fsx/hf_cache"
|
||||
if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test" || build.pull_request.labels includes "tts-test"
|
||||
timeout_in_minutes: 180
|
||||
commands:
|
||||
- export BENCHMARK_DIR=tests/dfx/perf/results
|
||||
- export VLLM_USE_DEEP_GEMM="0"
|
||||
- export VLLM_MOE_USE_DEEP_GEMM="0"
|
||||
- |
|
||||
set +e
|
||||
pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_higgs_audio_v3.json
|
||||
EXIT=$$?
|
||||
buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
|
||||
exit $$EXIT
|
||||
agents:
|
||||
queue: "mithril-h100-pool"
|
||||
plugins:
|
||||
- kubernetes:
|
||||
podSpec:
|
||||
containers:
|
||||
- image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
|
||||
resources:
|
||||
limits:
|
||||
nvidia.com/gpu: 1
|
||||
volumeMounts:
|
||||
- name: devshm
|
||||
mountPath: /dev/shm
|
||||
- name: hf-cache
|
||||
mountPath: /root/.cache/huggingface
|
||||
env:
|
||||
- name: HF_HOME
|
||||
value: /root/.cache/huggingface
|
||||
- name: HF_TOKEN
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: hf-token-secret
|
||||
key: token
|
||||
nodeSelector:
|
||||
node.kubernetes.io/instance-type: gpu-h100-sxm
|
||||
volumes:
|
||||
- name: devshm
|
||||
emptyDir:
|
||||
medium: Memory
|
||||
- name: hf-cache
|
||||
hostPath:
|
||||
path: /mnt/hf-cache
|
||||
type: DirectoryOrCreate
|
||||
|
||||
# Diffusion X2I suite: x2i / x2a / x2t and related non-video paths; x2v is only in "Diffusion X2V Model Test" below.
|
||||
- group: ":card_index_dividers: Diffusion X2I(&A&T) Model Test"
|
||||
@@ -649,7 +636,11 @@ steps:
|
||||
tests/e2e/online_serving/test_qwen_image_expansion.py
|
||||
tests/e2e/online_serving/test_qwen_image_edit_expansion.py
|
||||
tests/e2e/online_serving/test_qwen_image_layered_expansion.py
|
||||
tests/e2e/online_serving/test_longcat_image_expansion.py
|
||||
tests/e2e/online_serving/test_longcat_image_edit_expansion.py
|
||||
tests/e2e/online_serving/test_flux_2_dev_expansion.py
|
||||
tests/e2e/online_serving/test_bagel_expansion.py
|
||||
tests/e2e/offline_inference/test_sensenova_u1_expansion.py
|
||||
-m "full_model and diffusion and H100 and not distributed_cuda"
|
||||
--run-level "full_model"
|
||||
agents:
|
||||
@@ -735,6 +726,10 @@ steps:
|
||||
- >-
|
||||
pytest -sv
|
||||
tests/e2e/online_serving/test_qwen_image_layered_expansion.py
|
||||
tests/e2e/online_serving/test_longcat_image_expansion.py
|
||||
tests/e2e/online_serving/test_longcat_image_edit_expansion.py
|
||||
tests/e2e/online_serving/test_flux_2_dev_expansion.py
|
||||
tests/e2e/offline_inference/test_glm_image_autoround_w4a16_expansion.py
|
||||
-m "full_model and diffusion and H100 and distributed_cuda"
|
||||
--run-level "full_model"
|
||||
agents:
|
||||
@@ -813,14 +808,13 @@ steps:
|
||||
path: /mnt/hf-cache
|
||||
type: DirectoryOrCreate
|
||||
|
||||
- label: ":full_moon: Diffusion X2I(&A&T) · GLM-Image Function Test with H100"
|
||||
- label: ":full_moon: Diffusion X2I(&A&T) · Function Test with H100 · Multi-GPU DreamZero"
|
||||
timeout_in_minutes: 120
|
||||
commands:
|
||||
- >-
|
||||
pytest -sv
|
||||
tests/e2e/online_serving/test_glm_image_expansion.py
|
||||
tests/e2e/offline_inference/test_glm_image_autoround_w4a16_expansion.py
|
||||
-m "full_model and diffusion and H100"
|
||||
tests/e2e/online_serving/test_dreamzero_expansion.py
|
||||
-m "full_model and diffusion and H100 and distributed_cuda"
|
||||
--run-level "full_model"
|
||||
agents:
|
||||
queue: "mithril-h100-pool"
|
||||
@@ -857,8 +851,47 @@ steps:
|
||||
type: DirectoryOrCreate
|
||||
|
||||
|
||||
- label: ":full_moon: Diffusion X2I(&A&T) · Function Test with L4"
|
||||
- label: ":full_moon: Diffusion X2I(&A&T) · GLM-Image Function Test with H100"
|
||||
timeout_in_minutes: 120
|
||||
commands:
|
||||
- pytest -sv tests/e2e/online_serving/test_glm_image_expansion.py -m "full_model and diffusion and H100" --run-level "full_model"
|
||||
agents:
|
||||
queue: "mithril-h100-pool"
|
||||
plugins:
|
||||
- kubernetes:
|
||||
podSpec:
|
||||
containers:
|
||||
- image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
|
||||
resources:
|
||||
limits:
|
||||
nvidia.com/gpu: 2
|
||||
volumeMounts:
|
||||
- name: devshm
|
||||
mountPath: /dev/shm
|
||||
- name: hf-cache
|
||||
mountPath: /root/.cache/huggingface
|
||||
env:
|
||||
- name: HF_HOME
|
||||
value: /root/.cache/huggingface
|
||||
- name: HF_TOKEN
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: hf-token-secret
|
||||
key: token
|
||||
nodeSelector:
|
||||
node.kubernetes.io/instance-type: gpu-h100-sxm
|
||||
volumes:
|
||||
- name: devshm
|
||||
emptyDir:
|
||||
medium: Memory
|
||||
- name: hf-cache
|
||||
hostPath:
|
||||
path: /mnt/hf-cache
|
||||
type: DirectoryOrCreate
|
||||
|
||||
|
||||
- label: ":full_moon: Diffusion X2I(&A&T) · Function Test with L4"
|
||||
timeout_in_minutes: 60
|
||||
commands:
|
||||
- pytest -sv tests/e2e/ -k "not test_wan and not test_bagel_expansion and not hunyuan" -m "full_model and diffusion and L4" --run-level "full_model" --ignore=tests/e2e/accuracy
|
||||
agents:
|
||||
@@ -1243,10 +1276,10 @@ steps:
|
||||
build.pull_request.labels includes "nightly-test" ||
|
||||
build.pull_request.labels includes "diffusion-x2v-test"
|
||||
steps:
|
||||
- label: ":full_moon: Diffusion X2V · Wan2.2 T2V Function Test"
|
||||
- label: ":full_moon: Diffusion X2V · Function Test"
|
||||
timeout_in_minutes: 90
|
||||
commands:
|
||||
- pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py -k "t2v" -m "full_model and cuda" --run-level "full_model"
|
||||
- pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py tests/e2e/online_serving/test_wan_2_1_vace_expansion.py tests/e2e/online_serving/test_hunyuan_video_15_expansion.py tests/e2e/offline_inference/test_wan22_autoround_w4a16_expansion.py -m "full_model and cuda" --run-level "full_model"
|
||||
agents:
|
||||
queue: "mithril-h100-pool"
|
||||
plugins:
|
||||
@@ -1281,167 +1314,14 @@ steps:
|
||||
path: /mnt/hf-cache
|
||||
type: DirectoryOrCreate
|
||||
|
||||
- label: ":full_moon: Diffusion X2V · Wan2.2 I2V/TI2V Function Test"
|
||||
timeout_in_minutes: 90
|
||||
commands:
|
||||
- pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py -k "i2v" -m "full_model and cuda" --run-level "full_model"
|
||||
agents:
|
||||
queue: "mithril-h100-pool"
|
||||
plugins:
|
||||
- kubernetes:
|
||||
podSpec:
|
||||
containers:
|
||||
- image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
|
||||
resources:
|
||||
limits:
|
||||
nvidia.com/gpu: 2
|
||||
volumeMounts:
|
||||
- name: devshm
|
||||
mountPath: /dev/shm
|
||||
- name: hf-cache
|
||||
mountPath: /root/.cache/huggingface
|
||||
env:
|
||||
- name: HF_HOME
|
||||
value: /root/.cache/huggingface
|
||||
- name: HF_TOKEN
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: hf-token-secret
|
||||
key: token
|
||||
nodeSelector:
|
||||
node.kubernetes.io/instance-type: gpu-h100-sxm
|
||||
volumes:
|
||||
- name: devshm
|
||||
emptyDir:
|
||||
medium: Memory
|
||||
- name: hf-cache
|
||||
hostPath:
|
||||
path: /mnt/hf-cache
|
||||
type: DirectoryOrCreate
|
||||
|
||||
- label: ":full_moon: Diffusion X2V · Doc Test"
|
||||
timeout_in_minutes: 60
|
||||
commands:
|
||||
- pytest -s -v tests/examples/offline_inference/test_image_to_video.py -m "full_model and example and H100" --run-level "full_model"
|
||||
agents:
|
||||
queue: "mithril-h100-pool"
|
||||
plugins:
|
||||
- kubernetes:
|
||||
podSpec:
|
||||
containers:
|
||||
- image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
|
||||
resources:
|
||||
limits:
|
||||
nvidia.com/gpu: 1
|
||||
volumeMounts:
|
||||
- name: devshm
|
||||
mountPath: /dev/shm
|
||||
- name: hf-cache
|
||||
mountPath: /root/.cache/huggingface
|
||||
env:
|
||||
- name: HF_HOME
|
||||
value: /root/.cache/huggingface
|
||||
- name: HF_TOKEN
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: hf-token-secret
|
||||
key: token
|
||||
nodeSelector:
|
||||
node.kubernetes.io/instance-type: gpu-h100-sxm
|
||||
volumes:
|
||||
- name: devshm
|
||||
emptyDir:
|
||||
medium: Memory
|
||||
- name: hf-cache
|
||||
hostPath:
|
||||
path: /mnt/hf-cache
|
||||
type: DirectoryOrCreate
|
||||
|
||||
- label: ":full_moon: Diffusion X2V · Other Function Test"
|
||||
timeout_in_minutes: 90
|
||||
commands:
|
||||
- pytest -s -v tests/e2e/offline_inference/test_wan22_autoround_w4a16_expansion.py -m "full_model and cuda" --run-level "full_model"
|
||||
- pytest -s -v tests/e2e/online_serving/test_wan_2_1_vace_expansion.py tests/e2e/online_serving/test_ltx2_expansion.py -m "full_model and cuda" --run-level "full_model"
|
||||
agents:
|
||||
queue: "mithril-h100-pool"
|
||||
plugins:
|
||||
- kubernetes:
|
||||
podSpec:
|
||||
containers:
|
||||
- image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
|
||||
resources:
|
||||
limits:
|
||||
nvidia.com/gpu: 2
|
||||
volumeMounts:
|
||||
- name: devshm
|
||||
mountPath: /dev/shm
|
||||
- name: hf-cache
|
||||
mountPath: /root/.cache/huggingface
|
||||
env:
|
||||
- name: HF_HOME
|
||||
value: /root/.cache/huggingface
|
||||
- name: HF_TOKEN
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: hf-token-secret
|
||||
key: token
|
||||
nodeSelector:
|
||||
node.kubernetes.io/instance-type: gpu-h100-sxm
|
||||
volumes:
|
||||
- name: devshm
|
||||
emptyDir:
|
||||
medium: Memory
|
||||
- name: hf-cache
|
||||
hostPath:
|
||||
path: /mnt/hf-cache
|
||||
type: DirectoryOrCreate
|
||||
|
||||
- label: ":full_moon: Diffusion X2V · Wan2.2 Accuracy Test"
|
||||
- label: ":full_moon: Diffusion X2V · Accuracy Test"
|
||||
timeout_in_minutes: 180
|
||||
commands:
|
||||
- pytest -s -v tests/e2e/accuracy/wan22_i2v/test_wan22_i2v_video_similarity.py -m full_model --run-level full_model
|
||||
- pytest -s -v tests/e2e/accuracy/test_diffusers_backend_similarity.py -k '2v_matches_diffusers' -m full_model --run-level full_model
|
||||
agents:
|
||||
queue: "mithril-h100-pool"
|
||||
plugins:
|
||||
- kubernetes:
|
||||
podSpec:
|
||||
containers:
|
||||
- image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
|
||||
resources:
|
||||
limits:
|
||||
nvidia.com/gpu: 2
|
||||
volumeMounts:
|
||||
- name: devshm
|
||||
mountPath: /dev/shm
|
||||
- name: hf-cache
|
||||
mountPath: /root/.cache/huggingface
|
||||
env:
|
||||
- name: HF_HOME
|
||||
value: /root/.cache/huggingface
|
||||
- name: HF_TOKEN
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: hf-token-secret
|
||||
key: token
|
||||
nodeSelector:
|
||||
node.kubernetes.io/instance-type: gpu-h100-sxm
|
||||
volumes:
|
||||
- name: devshm
|
||||
emptyDir:
|
||||
medium: Memory
|
||||
- name: hf-cache
|
||||
hostPath:
|
||||
path: /mnt/hf-cache
|
||||
type: DirectoryOrCreate
|
||||
|
||||
- label: ":full_moon: Diffusion X2V · Other Accuracy Test"
|
||||
timeout_in_minutes: 180
|
||||
commands:
|
||||
- pytest -s -v tests/e2e/accuracy/hunyuanvideo15_t2v/test_hunyuanvideo15_t2v_video_similarity.py -m full_model --run-level full_model
|
||||
- pytest -s -v tests/e2e/accuracy/hunyuanvideo15_i2v/test_hunyuanvideo15_i2v_video_similarity.py -m full_model --run-level full_model
|
||||
- pytest -s -v tests/e2e/accuracy/test_ltx2_3_video_similarity.py -m full_model --run-level full_model
|
||||
- pytest -s -v tests/e2e/accuracy/test_video_streaming_output_similarity.py -m full_model --run-level full_model
|
||||
- pytest -s -v tests/e2e/accuracy/test_diffusers_backend_similarity.py -k '2v_matches_diffusers' -m full_model --run-level full_model
|
||||
agents:
|
||||
queue: "mithril-h100-pool"
|
||||
plugins:
|
||||
@@ -1585,7 +1465,7 @@ steps:
|
||||
key: nightly-testcase-statistics
|
||||
timeout_in_minutes: 120
|
||||
depends_on: upload-nightly-pipeline
|
||||
if: build.env("NIGHTLY") == "1"
|
||||
if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test"
|
||||
commands:
|
||||
- python tools/nightly/buildkite_testcase_statistics.py -o tests/dfx/perf/results/buildkite_testcase_statistics.html
|
||||
- buildkite-agent artifact upload "tests/dfx/perf/results/*.html"
|
||||
|
||||
+85
-97
@@ -157,27 +157,6 @@ steps:
|
||||
- "/fsx/hf_cache:/fsx/hf_cache"
|
||||
|
||||
|
||||
- group: ":card_index_dividers: Tiny Model Tests [Single GPU]"
|
||||
depends_on: upload-ready-pipeline
|
||||
steps:
|
||||
- label: "Tiny Model Tests · Diffusion (base)"
|
||||
commands:
|
||||
- timeout 15m pytest -sv tests/model_tests/diffusion -m 'core_model and cuda' --run-level "core_model"
|
||||
agents:
|
||||
# Ready label only runs the cases that don't need parallelism for tiny
|
||||
# model tests, which is usually just a smoke test for TTI and I2I
|
||||
queue: "gpu_1_queue" # g6.4xlarge instance on AWS, has 1 L4 GPU
|
||||
plugins:
|
||||
- docker#v5.2.0:
|
||||
image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
|
||||
always-pull: true
|
||||
propagate-environment: true
|
||||
environment:
|
||||
- "HF_HOME=/fsx/hf_cache"
|
||||
- "HF_TOKEN"
|
||||
volumes:
|
||||
- "/fsx/hf_cache:/fsx/hf_cache"
|
||||
|
||||
- label: "Engine&Model Executor Test"
|
||||
depends_on: upload-ready-pipeline
|
||||
commands:
|
||||
@@ -205,43 +184,96 @@ steps:
|
||||
commands:
|
||||
- |
|
||||
timeout 20m bash -c "
|
||||
pytest -sv tests/distributed/ -m 'core_model and cuda and L4' --run-level 'core_model'
|
||||
pytest -sv tests/distributed/ -m 'core_model and cuda' --run-level 'core_model'
|
||||
"
|
||||
agents:
|
||||
queue: "gpu_1_queue" # g6.4xlarge instance on AWS, has 1 L4 GPU
|
||||
queue: "mithril-h100-pool"
|
||||
plugins:
|
||||
- docker#v5.2.0:
|
||||
image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
|
||||
always-pull: true
|
||||
propagate-environment: true
|
||||
shm-size: "8gb"
|
||||
environment:
|
||||
- "HF_HOME=/fsx/hf_cache"
|
||||
- "HF_TOKEN"
|
||||
volumes:
|
||||
- "/fsx/hf_cache:/fsx/hf_cache"
|
||||
- kubernetes:
|
||||
podSpec:
|
||||
containers:
|
||||
- image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
|
||||
resources:
|
||||
limits:
|
||||
nvidia.com/gpu: 1
|
||||
volumeMounts:
|
||||
- name: devshm
|
||||
mountPath: /dev/shm
|
||||
- name: hf-cache
|
||||
mountPath: /root/.cache/huggingface
|
||||
env:
|
||||
- name: HF_HOME
|
||||
value: /root/.cache/huggingface
|
||||
nodeSelector:
|
||||
node.kubernetes.io/instance-type: gpu-h100-sxm
|
||||
volumes:
|
||||
- name: devshm
|
||||
emptyDir:
|
||||
medium: Memory
|
||||
- name: hf-cache
|
||||
hostPath:
|
||||
path: /mnt/hf-cache
|
||||
type: DirectoryOrCreate
|
||||
|
||||
- label: "Entrypoints Test"
|
||||
- group: ":card_index_dividers: Entrypoints Test"
|
||||
depends_on: upload-ready-pipeline
|
||||
commands:
|
||||
- |
|
||||
timeout 30m bash -c "
|
||||
export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
pytest -sv tests/entrypoints/ -m 'core_model and cuda' --run-level 'core_model'
|
||||
"
|
||||
agents:
|
||||
queue: "gpu_1_queue"
|
||||
plugins:
|
||||
- docker#v5.2.0:
|
||||
image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
|
||||
always-pull: true
|
||||
propagate-environment: true
|
||||
shm-size: "8gb"
|
||||
environment:
|
||||
- "HF_HOME=/fsx/hf_cache"
|
||||
- "HF_TOKEN"
|
||||
volumes:
|
||||
- "/fsx/hf_cache:/fsx/hf_cache"
|
||||
steps:
|
||||
- label: "Entrypoints Test with H100"
|
||||
commands:
|
||||
- |
|
||||
timeout 30m bash -c "
|
||||
export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
pytest -sv tests/entrypoints/ -m 'core_model and cuda and H100' --run-level 'core_model'
|
||||
"
|
||||
agents:
|
||||
queue: "mithril-h100-pool"
|
||||
plugins:
|
||||
- kubernetes:
|
||||
podSpec:
|
||||
containers:
|
||||
- image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
|
||||
resources:
|
||||
limits:
|
||||
nvidia.com/gpu: 2
|
||||
volumeMounts:
|
||||
- name: devshm
|
||||
mountPath: /dev/shm
|
||||
- name: hf-cache
|
||||
mountPath: /root/.cache/huggingface
|
||||
env:
|
||||
- name: HF_HOME
|
||||
value: /root/.cache/huggingface
|
||||
nodeSelector:
|
||||
node.kubernetes.io/instance-type: gpu-h100-sxm
|
||||
volumes:
|
||||
- name: devshm
|
||||
emptyDir:
|
||||
medium: Memory
|
||||
- name: hf-cache
|
||||
hostPath:
|
||||
path: /mnt/hf-cache
|
||||
type: DirectoryOrCreate
|
||||
|
||||
- label: "Entrypoints Test with L4"
|
||||
commands:
|
||||
- |
|
||||
timeout 30m bash -c "
|
||||
export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
pytest -sv tests/entrypoints/ -m 'core_model and cuda and L4' --run-level 'core_model'
|
||||
"
|
||||
agents:
|
||||
queue: "gpu_1_queue"
|
||||
plugins:
|
||||
- docker#v5.2.0:
|
||||
image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
|
||||
always-pull: true
|
||||
propagate-environment: true
|
||||
shm-size: "8gb"
|
||||
environment:
|
||||
- "HF_HOME=/fsx/hf_cache"
|
||||
- "HF_TOKEN"
|
||||
volumes:
|
||||
- "/fsx/hf_cache:/fsx/hf_cache"
|
||||
|
||||
- group: ":card_index_dividers: E2E Test"
|
||||
depends_on: upload-ready-pipeline
|
||||
@@ -552,50 +584,6 @@ steps:
|
||||
path: /mnt/hf-cache
|
||||
type: DirectoryOrCreate
|
||||
|
||||
- label: "Diffusion · Cosmos3 Test"
|
||||
source_file_dependencies:
|
||||
- tests/e2e/online_serving/test_cosmos3.py
|
||||
- vllm_omni/diffusion/models/cosmos3/
|
||||
commands:
|
||||
- |
|
||||
timeout 40m bash -c "
|
||||
export VLLM_IMAGE_FETCH_TIMEOUT=60
|
||||
pytest -s -v tests/e2e/online_serving/test_cosmos3.py -m 'core_model' --run-level 'core_model'
|
||||
"
|
||||
agents:
|
||||
queue: "mithril-h100-pool"
|
||||
plugins:
|
||||
- kubernetes:
|
||||
podSpec:
|
||||
containers:
|
||||
- image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
|
||||
resources:
|
||||
limits:
|
||||
nvidia.com/gpu: 1
|
||||
volumeMounts:
|
||||
- name: devshm
|
||||
mountPath: /dev/shm
|
||||
- name: hf-cache
|
||||
mountPath: /root/.cache/huggingface
|
||||
env:
|
||||
- name: HF_HOME
|
||||
value: /root/.cache/huggingface
|
||||
- name: HF_TOKEN
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: hf-token-secret
|
||||
key: token
|
||||
nodeSelector:
|
||||
node.kubernetes.io/instance-type: gpu-h100-sxm
|
||||
volumes:
|
||||
- name: devshm
|
||||
emptyDir:
|
||||
medium: Memory
|
||||
- name: hf-cache
|
||||
hostPath:
|
||||
path: /mnt/hf-cache
|
||||
type: DirectoryOrCreate
|
||||
|
||||
|
||||
- label: "Diffusion · Qwen Image Test"
|
||||
source_file_dependencies:
|
||||
|
||||
@@ -85,66 +85,6 @@ steps:
|
||||
path: /mnt/hf-cache
|
||||
type: DirectoryOrCreate
|
||||
|
||||
- label: "Reliability Test - hunyuan-image"
|
||||
timeout_in_minutes: 180
|
||||
depends_on: upload-weekly-pipeline
|
||||
if: build.env("WEEKLY") == "1" || build.pull_request.labels includes "weekly-test"
|
||||
commands:
|
||||
- pytest -s -v tests/dfx/reliability/test_reliability_hunyuan_image.py -m "slow"
|
||||
agents:
|
||||
queue: "mithril-h100-pool"
|
||||
plugins:
|
||||
- kubernetes:
|
||||
podSpec:
|
||||
containers:
|
||||
- image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
|
||||
resources:
|
||||
limits:
|
||||
nvidia.com/gpu: 4
|
||||
volumeMounts:
|
||||
- name: devshm
|
||||
mountPath: /dev/shm
|
||||
- name: hf-cache
|
||||
mountPath: /root/.cache/huggingface
|
||||
env:
|
||||
- name: HF_HOME
|
||||
value: /root/.cache/huggingface
|
||||
- name: HF_TOKEN
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: hf-token-secret
|
||||
key: token
|
||||
nodeSelector:
|
||||
node.kubernetes.io/instance-type: gpu-h100-sxm
|
||||
volumes:
|
||||
- name: devshm
|
||||
emptyDir:
|
||||
medium: Memory
|
||||
- name: hf-cache
|
||||
hostPath:
|
||||
path: /mnt/hf-cache
|
||||
type: DirectoryOrCreate
|
||||
|
||||
- label: "Reliability Test - voxcpm2"
|
||||
timeout_in_minutes: 180
|
||||
depends_on: upload-weekly-pipeline
|
||||
if: build.env("WEEKLY") == "1" || build.pull_request.labels includes "weekly-test"
|
||||
commands:
|
||||
- pytest -s -v tests/dfx/reliability/test_reliability_voxcpm2.py -m "slow"
|
||||
agents:
|
||||
queue: "gpu_1_queue"
|
||||
plugins:
|
||||
- docker#v5.2.0:
|
||||
image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
|
||||
always-pull: true
|
||||
propagate-environment: true
|
||||
shm-size: "8gb"
|
||||
environment:
|
||||
- "HF_HOME=/fsx/hf_cache"
|
||||
- "HF_TOKEN"
|
||||
volumes:
|
||||
- "/fsx/hf_cache:/fsx/hf_cache"
|
||||
|
||||
# Invalid HTTP / WS payloads vs live Omni servers (Issue #3649 tracks known skips).
|
||||
- group: "Reliability Test - Invalid parameters Test"
|
||||
depends_on: upload-weekly-pipeline
|
||||
@@ -206,246 +146,3 @@ steps:
|
||||
- "HF_TOKEN"
|
||||
volumes:
|
||||
- "/fsx/hf_cache:/fsx/hf_cache"
|
||||
|
||||
- group: ":card_index_dividers: E2E Tests"
|
||||
depends_on: upload-weekly-pipeline
|
||||
if: build.env("WEEKLY") == "1" || build.pull_request.labels includes "weekly-test"
|
||||
steps:
|
||||
- label: "Omni · L4"
|
||||
timeout_in_minutes: 90
|
||||
commands:
|
||||
- export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
|
||||
- pytest -s -v tests/e2e/*/test_qwen2_5_omni_expansion.py --run-level "core_model"
|
||||
- >-
|
||||
pytest -s -v tests/e2e/
|
||||
--ignore=tests/e2e/online_serving/test_qwen2_5_omni_expansion.py
|
||||
--ignore=tests/e2e/offline_inference/test_qwen2_5_omni_expansion.py
|
||||
-m "slow and L4 and omni"
|
||||
--run-level "full_model"
|
||||
agents:
|
||||
queue: "gpu_4_queue"
|
||||
plugins:
|
||||
- docker#v5.2.0:
|
||||
image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
|
||||
always-pull: true
|
||||
propagate-environment: true
|
||||
shm-size: "8gb"
|
||||
environment:
|
||||
- "HF_HOME=/fsx/hf_cache"
|
||||
- "HF_TOKEN"
|
||||
volumes:
|
||||
- "/fsx/hf_cache:/fsx/hf_cache"
|
||||
|
||||
- label: "Omni · H100"
|
||||
timeout_in_minutes: 90
|
||||
commands:
|
||||
- >-
|
||||
pytest -sv
|
||||
tests/e2e/ -m "slow and H100 and omni"
|
||||
--run-level "full_model"
|
||||
agents:
|
||||
queue: "mithril-h100-pool"
|
||||
plugins:
|
||||
- kubernetes:
|
||||
podSpec:
|
||||
containers:
|
||||
- image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
|
||||
resources:
|
||||
limits:
|
||||
nvidia.com/gpu: 2
|
||||
volumeMounts:
|
||||
- name: devshm
|
||||
mountPath: /dev/shm
|
||||
- name: hf-cache
|
||||
mountPath: /root/.cache/huggingface
|
||||
env:
|
||||
- name: HF_HOME
|
||||
value: /root/.cache/huggingface
|
||||
- name: HF_TOKEN
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: hf-token-secret
|
||||
key: token
|
||||
nodeSelector:
|
||||
node.kubernetes.io/instance-type: gpu-h100-sxm
|
||||
volumes:
|
||||
- name: devshm
|
||||
emptyDir:
|
||||
medium: Memory
|
||||
- name: hf-cache
|
||||
hostPath:
|
||||
path: /mnt/hf-cache
|
||||
type: DirectoryOrCreate
|
||||
|
||||
- label: "TTS · L4"
|
||||
timeout_in_minutes: 90
|
||||
commands:
|
||||
- export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
|
||||
- export VLLM_USE_DEEP_GEMM="0"
|
||||
- export VLLM_MOE_USE_DEEP_GEMM="0"
|
||||
- |
|
||||
failed=0
|
||||
for f in $(pytest --collect-only -q tests/e2e/ -m "slow and L4 and tts" --run-level "full_model" | grep '\.py::' | sed -E 's|::.*||' | sort -u); do
|
||||
pytest -s -v "$$f" -m "slow and L4 and tts" --run-level "full_model" || failed=1
|
||||
done
|
||||
exit $$failed
|
||||
agents:
|
||||
queue: "gpu_1_queue"
|
||||
plugins:
|
||||
- docker#v5.2.0:
|
||||
image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
|
||||
always-pull: true
|
||||
propagate-environment: true
|
||||
shm-size: "8gb"
|
||||
environment:
|
||||
- "HF_HOME=/fsx/hf_cache"
|
||||
- "HF_TOKEN"
|
||||
volumes:
|
||||
- "/fsx/hf_cache:/fsx/hf_cache"
|
||||
|
||||
- label: "TTS · H100"
|
||||
timeout_in_minutes: 90
|
||||
commands:
|
||||
- export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
|
||||
- export VLLM_USE_DEEP_GEMM="0"
|
||||
- export VLLM_MOE_USE_DEEP_GEMM="0"
|
||||
- >-
|
||||
pytest -sv
|
||||
tests/e2e/ -m "slow and H100 and tts"
|
||||
--run-level "full_model"
|
||||
agents:
|
||||
queue: "mithril-h100-pool"
|
||||
plugins:
|
||||
- kubernetes:
|
||||
podSpec:
|
||||
containers:
|
||||
- image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
|
||||
resources:
|
||||
limits:
|
||||
nvidia.com/gpu: 1
|
||||
volumeMounts:
|
||||
- name: devshm
|
||||
mountPath: /dev/shm
|
||||
- name: hf-cache
|
||||
mountPath: /root/.cache/huggingface
|
||||
env:
|
||||
- name: HF_HOME
|
||||
value: /root/.cache/huggingface
|
||||
- name: HF_TOKEN
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: hf-token-secret
|
||||
key: token
|
||||
nodeSelector:
|
||||
node.kubernetes.io/instance-type: gpu-h100-sxm
|
||||
volumes:
|
||||
- name: devshm
|
||||
emptyDir:
|
||||
medium: Memory
|
||||
- name: hf-cache
|
||||
hostPath:
|
||||
path: /mnt/hf-cache
|
||||
type: DirectoryOrCreate
|
||||
|
||||
- label: "Diffusion · L4"
|
||||
timeout_in_minutes: 60
|
||||
commands:
|
||||
- >-
|
||||
pytest -sv
|
||||
tests/e2e/ -m "slow and diffusion and L4"
|
||||
--run-level "full_model"
|
||||
agents:
|
||||
queue: "gpu_4_queue"
|
||||
plugins:
|
||||
- docker#v5.2.0:
|
||||
image: public.ecr.aws/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
|
||||
always-pull: true
|
||||
propagate-environment: true
|
||||
shm-size: "8gb"
|
||||
environment:
|
||||
- "HF_HOME=/fsx/hf_cache"
|
||||
- "HF_TOKEN"
|
||||
volumes:
|
||||
- "/fsx/hf_cache:/fsx/hf_cache"
|
||||
|
||||
- label: "Diffusion · H100 · Single-GPU"
|
||||
timeout_in_minutes: 120
|
||||
commands:
|
||||
- >-
|
||||
pytest -sv
|
||||
tests/e2e/ -m "slow and diffusion and H100 and not distributed_cuda"
|
||||
--run-level "full_model"
|
||||
agents:
|
||||
queue: "mithril-h100-pool"
|
||||
plugins:
|
||||
- kubernetes:
|
||||
podSpec:
|
||||
containers:
|
||||
- image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
|
||||
resources:
|
||||
limits:
|
||||
nvidia.com/gpu: 1
|
||||
volumeMounts:
|
||||
- name: devshm
|
||||
mountPath: /dev/shm
|
||||
- name: hf-cache
|
||||
mountPath: /root/.cache/huggingface
|
||||
env:
|
||||
- name: HF_HOME
|
||||
value: /root/.cache/huggingface
|
||||
- name: HF_TOKEN
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: hf-token-secret
|
||||
key: token
|
||||
nodeSelector:
|
||||
node.kubernetes.io/instance-type: gpu-h100-sxm
|
||||
volumes:
|
||||
- name: devshm
|
||||
emptyDir:
|
||||
medium: Memory
|
||||
- name: hf-cache
|
||||
hostPath:
|
||||
path: /mnt/hf-cache
|
||||
type: DirectoryOrCreate
|
||||
|
||||
- label: "Diffusion · H100 · Multi-GPU"
|
||||
timeout_in_minutes: 90
|
||||
commands:
|
||||
- >-
|
||||
pytest -sv
|
||||
tests/e2e/ -m "slow and diffusion and H100 and distributed_cuda"
|
||||
--run-level "full_model"
|
||||
agents:
|
||||
queue: "mithril-h100-pool"
|
||||
plugins:
|
||||
- kubernetes:
|
||||
podSpec:
|
||||
containers:
|
||||
- image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
|
||||
resources:
|
||||
limits:
|
||||
nvidia.com/gpu: 2
|
||||
volumeMounts:
|
||||
- name: devshm
|
||||
mountPath: /dev/shm
|
||||
- name: hf-cache
|
||||
mountPath: /root/.cache/huggingface
|
||||
env:
|
||||
- name: HF_HOME
|
||||
value: /root/.cache/huggingface
|
||||
- name: HF_TOKEN
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: hf-token-secret
|
||||
key: token
|
||||
nodeSelector:
|
||||
node.kubernetes.io/instance-type: gpu-h100-sxm
|
||||
volumes:
|
||||
- name: devshm
|
||||
emptyDir:
|
||||
medium: Memory
|
||||
- name: hf-cache
|
||||
hostPath:
|
||||
path: /mnt/hf-cache
|
||||
type: DirectoryOrCreate
|
||||
@@ -63,8 +63,6 @@ If multiple rows apply (e.g., a diffusion model is also a new model), union the
|
||||
|
||||
Ask: "Quick mode or full mode?" Then walk the checklist for the detected PR type from [references/checklists.md](references/checklists.md). Each item produces ✓, ✗, or ⚠.
|
||||
|
||||
**Also run the Code-Quality sweep on every PR**, regardless of type or mode: the five diff-scoped checks in [references/code-quality.md](references/code-quality.md) — kwargs fragility, broad-except swallow, `Any`/wrong type hints, hot-path `.clone()`/`deepcopy`, and event-loop blocking — plus the advisory conventions (log level, structured logging, synchronization, cleanup, dependencies, naming) in [checklists.md](references/checklists.md). These count only lines the PR *adds* — the pre-existing backlog across the repo is out of scope.
|
||||
|
||||
### Step 5: Print Report
|
||||
|
||||
```
|
||||
@@ -76,7 +74,6 @@ Pre-check report for <branch>
|
||||
Dimension Result
|
||||
───────────────── ──────
|
||||
PR title format ✓
|
||||
Code quality ⚠ 1 broad except, 2 Any hints
|
||||
PR desc integrity ✓
|
||||
Registry/config ✓
|
||||
Dead code ⚠ 2 warnings
|
||||
|
||||
@@ -35,31 +35,6 @@ PR title must follow the project convention documented in `docs/contributing/REA
|
||||
|
||||
---
|
||||
|
||||
## All PRs: Code Quality (run on every PR, quick & full)
|
||||
|
||||
Diff-scoped sweep for five fragility patterns that are pervasive in the existing codebase — **only lines the PR adds count**, not the pre-existing backlog. Detection commands, codebase examples, and full severity rules live in [code-quality.md](code-quality.md).
|
||||
|
||||
- [ ] **No new `**kwargs` string-lookup plumbing:** the diff adds no `**kwargs` + `kwargs["..."]` / `kwargs.get("...")` / `"..." in kwargs`. ⚠ any new instance; ✗ if a new string key is duplicated across ≥2 files without a shared constant, or unknown keys are silently dropped on a fail-fast path.
|
||||
- [ ] **No new broad exception swallow:** the diff adds no `except Exception:` / bare `except:`. ⚠ any new instance; ✗ for `except: pass` / `except Exception: return None` / `continue` on a fail-fast path (init, config validation, weight loading, request handling).
|
||||
- [ ] **No new `Any` / wrong type hints:** the diff adds no `: Any` / `-> Any` / untyped production signature, and no new `SimpleNamespace` in tests faking an object that already has a real typed stub. ⚠ any new instance; ✗ for a *wrong* (actively misleading) annotation or a `SimpleNamespace` test fake of a typed object.
|
||||
- [ ] **No new hot-path copy:** the diff adds no `.clone()` / `.copy_()` / `copy.deepcopy(...)` inside a per-step loop or per-request path. ⚠ any new instance; ✗ for `.clone()` of a full latent/activations tensor inside an AR or diffusion step loop without a comment, or `deepcopy` of a scheduler / sampling-config on the request path.
|
||||
- [ ] **No new event-loop blocking:** the diff adds no `time.sleep` / blocking HTTP (`requests`, `urllib`) / `await` inside a held lock in `async def` code. ⚠ any new instance; ✗ for blocking sleep or HTTP on the serving path (`engine/`, `entrypoints/`, async `connectors/`), or a lock held across an `await`. Blocking inside a dedicated worker thread is fine.
|
||||
|
||||
Roll the ⚠/✗ counts into the report as a single **Code quality** dimension row.
|
||||
|
||||
### Conventions (eyeball — no grep)
|
||||
|
||||
Judgment calls that don't grep cleanly — apply them while reading the diff, not as a mechanical check:
|
||||
|
||||
- **Right log level** — `info` is for end-user-visible events; routine internal events are `debug`; hot paths are `trace` or removed. Logging is not free.
|
||||
- **Structured fields over f-strings** — pass dynamic values as structured fields (`logger.info("msg", extra={...})`), not interpolated strings, so they stay greppable and parseable.
|
||||
- **No reflexive synchronization** — don't add `threading.Lock` / `asyncio.Lock` / `deepcopy` / re-wrap shared state unless there is real concurrent mutation; the GIL covers most Python code. Owners decide their own synchronization — don't pre-lock in a constructor.
|
||||
- **Context managers for cleanup** — use `with` / `contextlib` for files, locks, CUDA streams, and profilers; not raw `open()` + manual `.close()` or ad-hoc `finally`.
|
||||
- **No new dependencies without justification** — prefer the stdlib and existing vllm-omni deps; a new top-level dep needs a reason (check `pyproject.toml` / `requirements*.txt`).
|
||||
- **Naming / interface discipline** — don't rename or change interfaces just for taste in this PR, but do fix names that actively mislead (e.g. `serve` implying a long-running server, `Instance` where multiple instance kinds coexist).
|
||||
|
||||
---
|
||||
|
||||
## Bug Fix PRs
|
||||
|
||||
### Quick
|
||||
@@ -74,7 +49,7 @@ Judgment calls that don't grep cleanly — apply them while reading the diff, no
|
||||
- [ ] **Regression test exists:** `git diff --name-only` includes at least one `tests/` file
|
||||
- [ ] **Regression test reproduces the original error:** the test would fail on main, pass on this branch
|
||||
- [ ] **Fix matches root cause exactly:** no "fix the symptom + something else" — if the root cause is dtype, the fix is dtype, not dtype + reformatting
|
||||
- [ ] **No silent failure risk:** no bare `except: pass`, no `try: ... except: return None`, no empty fallback added. See the broad-except pattern in [code-quality.md](code-quality.md#2-broad-exception-swallow) for the full severity rule (⚠ any new broad catch; ✗ for a swallow on a fail-fast path).
|
||||
- [ ] **No silent failure risk:** no bare `except: pass`, no `try: ... except: return None`, no empty fallback added
|
||||
- [ ] **Upstream pattern match:** the fix follows the same pattern as existing code for similar cases (grep for analogous `from_pretrained` calls, etc.)
|
||||
- [ ] **Environment documented:** torch/transformers/vllm versions listed if the bug is version-dependent
|
||||
|
||||
|
||||
@@ -1,179 +0,0 @@
|
||||
# Code-Quality Patterns
|
||||
|
||||
Five fragility patterns to sweep on **every** PR, in both quick and full mode. These are pervasive in the existing codebase (hundreds of pre-existing sites) — so the checks are **diff-scoped**: only lines the PR *adds* count. Whole-repo greps light up every PR with the backlog and are useless.
|
||||
|
||||
All commands assume:
|
||||
```bash
|
||||
BASE="$(git merge-base HEAD origin/main)"
|
||||
```
|
||||
|
||||
**Severity:** ⚠ for any *new* instance introduced by the diff; ✗ for the acute sub-cases defined per pattern. Acute ✗ findings belong in the report's blocking count.
|
||||
|
||||
---
|
||||
|
||||
## 1. `**kwargs` string-lookup passthrough
|
||||
|
||||
### Anti-pattern
|
||||
```python
|
||||
def forward(self, **kwargs):
|
||||
# BLOCKER-prone: raw dict + string keys duplicated across files
|
||||
if "runtime_additional_information" in kwargs and \
|
||||
"model_intermediate_buffer" not in kwargs:
|
||||
...
|
||||
```
|
||||
Real instances of this exact guard are copy-pasted across `cosyvoice3.py`, `indextts2_talker.py` (×2), `indextts2_s2mel_decoder.py`, `qwen3_tts_talker.py`, `qwen3_omni.py`. A typo in any string key silently no-ops; there is no type check, no IDE completion, and unknown keys are silently dropped.
|
||||
|
||||
### Why dangerous
|
||||
Raw `dict` + `**kwargs` + lists of param-name strings is **fragile by construction**: the compiler can't see a missing or misspelled key, so failures are silent. Each duplicated copy is a new place for the strings to drift.
|
||||
|
||||
### Severity
|
||||
- ⚠ any *new* `**kwargs` signature whose body does `kwargs["..."]` / `kwargs.get("...")` / `"..." in kwargs`.
|
||||
- ✗ when the diff introduces a **new** string key that is duplicated across ≥2 files without a shared module constant, or when `**kwargs` plumbing **silently drops unknown keys** on a fail-fast path (init, config validation, weight loading).
|
||||
|
||||
### Detect (diff-scoped)
|
||||
```bash
|
||||
git diff ${BASE}...HEAD -- "*.py" | grep "^+[^+]" \
|
||||
| grep -E '\*\*kwargs|kwargs(\.get|\[)|"[a-z_]+"\s+in\s+kwargs'
|
||||
```
|
||||
|
||||
### Fix
|
||||
Prefer explicit typed params. The repo already has the right primitive: `msgspec.Struct` with `forbid_unknown_fields=True` rejects unknown keys instead of silently dropping them — see `_StructBase` in `vllm_omni/data_entry_keys.py:105` and `MoriPullRequest` in `vllm_omni/distributed/omni_connectors/connectors/mori_transfer_engine_connector.py:81`. A `@dataclass` or `TypedDict` works too.
|
||||
|
||||
If `**kwargs` is unavoidable (vLLM base-class compat), centralize the key strings as module-level constants behind **one** typed accessor, instead of repeating the literal across files.
|
||||
|
||||
---
|
||||
|
||||
## 2. Broad exception swallow
|
||||
|
||||
### Anti-pattern
|
||||
```python
|
||||
# BLOCKER-prone: catches everything, returns None on any failure
|
||||
try:
|
||||
evt = build_event(...)
|
||||
return evt
|
||||
except Exception:
|
||||
return None
|
||||
```
|
||||
Real clusters: `vllm_omni/metrics/stats.py:240,274,367,450` and `vllm_omni/metrics/{modality,utils}.py` — all `except Exception: return None`.
|
||||
|
||||
### Why dangerous
|
||||
A generic `except Exception:` is **too flexible**. If we catch an exception type we don't expect, it is almost always an unhandled edge case that needs to be *fixed* — not swallowed into `None`/`pass`/`continue`. Broad catches turn fixable bugs into silent wrong behavior and make debugging impossible.
|
||||
|
||||
The repo backs this with ruff `BLE001` (broad-exception catch); it is currently suppressed via `# noqa: BLE001` in `vllm_omni/patch.py:413`. Prefer fixing over suppressing.
|
||||
|
||||
### Severity
|
||||
- ⚠ any *new* `except Exception:` / `except BaseException:` / bare `except:`.
|
||||
- ✗ for `except:` / `except Exception: pass|return None|continue` on a **fail-fast path** (init, config validation, weight loading, request handling, connector setup). This extends the Bug-Fix "No silent failure risk" ✗ in [checklists.md](checklists.md).
|
||||
|
||||
### Detect (diff-scoped)
|
||||
```bash
|
||||
# Any new broad catch (⚠ baseline):
|
||||
git diff ${BASE}...HEAD -- "*.py" | grep "^+[^+]" \
|
||||
| grep -E 'except\s*(Exception|BaseException)?\s*:'
|
||||
# Note: this matches only the immediate-colon swallow forms (`except :`,
|
||||
# `except Exception:`) — it deliberately excludes specific catches like
|
||||
# `except ValueError:` and the `except Exception as e:` variant. Eyeball
|
||||
# the latter; `as e` usually means it is logged, but still verify.
|
||||
```
|
||||
|
||||
### Fix
|
||||
Catch the **specific** types you actually expect (`ValueError`, `KeyError`, `AttributeError`, `OSError`, `torch.cuda.OutOfMemoryError`, `msgspec.ValidationError`, …). At a genuine top-level / best-effort boundary (metrics, signal handlers), at minimum **log** the exception; never swallow into a bare `pass`/`return None` on a path that should fail loudly.
|
||||
|
||||
---
|
||||
|
||||
## 3. `Any` / wrong type annotations
|
||||
|
||||
### Anti-pattern
|
||||
```python
|
||||
# ⚠ Any param + Any return — no contract, no narrowing
|
||||
def _extract_mm_output(engine_outputs: Any) -> dict[str, Any]:
|
||||
...
|
||||
```
|
||||
|
||||
### Why dangerous (root cause)
|
||||
`Any` disables the type checker for that position, so typos and wrong shapes pass silently. The repo has 636 `: Any` params, 244 `-> Any` returns, and **1,173 `SimpleNamespace` in `tests/`**. The `Any` leak is largely test-driven: vibe-coded unit tests fake objects with `SimpleNamespace`, which forces the production code consuming them to be typed `Any`, and the `Any` then propagates outward into public signatures.
|
||||
|
||||
### Severity
|
||||
- ⚠ any *new* `: Any` / `-> Any` / fully-untyped signature in production code.
|
||||
- ✗ for a **wrong** annotation (actively misleading — e.g. `-> bool` that returns `Optional[bool]`), or a *new* `SimpleNamespace` in a test that mimics an object which already has a real typed stub / `@dataclass` / `TypedDict` / `Protocol`.
|
||||
|
||||
### Detect (diff-scoped)
|
||||
```bash
|
||||
# Any leaking into new signatures:
|
||||
git diff ${BASE}...HEAD -- "*.py" | grep "^+[^+]" \
|
||||
| grep -E ':\s*Any\b|->\s*Any\b'
|
||||
# SimpleNamespace newly added in tests (the Any-leak source):
|
||||
git diff ${BASE}...HEAD -- "tests/" | grep "^+[^+]" | grep "SimpleNamespace"
|
||||
```
|
||||
|
||||
### Fix
|
||||
Replace `Any` with the concrete type, a `Protocol`, or a `Union`. If a type is genuinely dynamic, prefer `object` + `isinstance` narrowing over `Any` — `object` keeps the checker engaged. In tests, replace `SimpleNamespace` with the real class, a small `@dataclass`/`TypedDict`, or a `Protocol` + fake, so the system under test stays fully typed.
|
||||
|
||||
---
|
||||
|
||||
## 4. Unnecessary copy — `.clone()` / `copy.deepcopy` in hot paths
|
||||
|
||||
### Anti-pattern
|
||||
```python
|
||||
# Per-step in a diffusion/AR loop: clones the full latent every iteration
|
||||
for t in timesteps:
|
||||
trajectory_latents.append(x_t.clone()) # O(N) GPU alloc + memcpy each step
|
||||
```
|
||||
Real clusters: `vllm_omni/diffusion/models/bagel/bagel_transformer.py` calls `x_t.clone()` ~9× across a trajectory loop (lines 1820–2186); the cache hooks `vllm_omni/diffusion/cache/teacache/hook.py:155` and `magcache/hook.py:182` clone `hidden_states` per step. Separately, `copy.deepcopy(self.scheduler)` / `copy.deepcopy(req.sampling_params)` is copy-pasted across diffusion pipelines — `pipeline_qwen_image.py:786`, `pipeline_hunyuan_image3.py:1904`, `pipeline_ltx2*.py` (×3), `pipeline_dreamzero.py:1085`, `diffusion_model_runner.py:398` — a full object-graph deepcopy on the request path. (Contrast `dreamzero/causal_wan_model.py:203`, which carries a `# No .clone() needed.` comment — the aware pattern.)
|
||||
|
||||
### Why dangerous
|
||||
A `.clone()` of a GPU tensor or a `copy.deepcopy` of a config object inside a per-step / per-request path is a hidden allocator: it reads as one Python line but triggers a GPU memcpy or a deep object-graph walk every iteration. Across 50 diffusion steps or hundreds of AR steps it dominates latency and VRAM bandwidth. `deepcopy` additionally ignores any custom device / `__copy__` logic and can drag tensors through CPU.
|
||||
|
||||
### Severity
|
||||
- ⚠ any *new* `.clone()` / `.copy_()` / `copy.deepcopy(...)` inside a per-step loop or per-request path.
|
||||
- ✗ for `.clone()` of a full latent/activations tensor inside an AR or diffusion step loop without an explanatory comment, or `copy.deepcopy` of a scheduler / sampling-config on the request hot path.
|
||||
|
||||
### Detect (diff-scoped)
|
||||
```bash
|
||||
git diff ${BASE}...HEAD -- "*.py" | grep "^+[^+]" \
|
||||
| grep -E '\.clone\(\)|\.copy_\(|copy\.deepcopy|deepcopy\('
|
||||
```
|
||||
|
||||
### Fix
|
||||
Tensors: prefer a view (`x[..., :k]`, `torch.narrow`, `as_strided`) or write into a pre-allocated buffer with `dst.copy_(src)` instead of `.clone()`; if the caller no longer needs the original, `move`/`del` it rather than copy. Schedulers / configs: build a fresh lightweight instance per request, or split out the mutable per-request bits so the shared immutable part isn't deepcopied. If a copy is genuinely required (e.g. the source is mutated in place later), keep it and add a one-line comment saying why.
|
||||
|
||||
---
|
||||
|
||||
## 5. Blocking the asyncio event loop / lock held across `await`
|
||||
|
||||
### Anti-pattern
|
||||
```python
|
||||
# BLOCKER-prone: blocking sleep / blocking HTTP / lock held across an await,
|
||||
# all inside an async (event-loop) function
|
||||
async def handle(self, req):
|
||||
with self._lock:
|
||||
resp = await self._client.call(req) # lock held across await -> serializes all requests
|
||||
time.sleep(0.1) # stalls the single loop for every concurrent request
|
||||
```
|
||||
|
||||
### Why dangerous
|
||||
vLLM-Omni serves over a single asyncio event loop. Any synchronous blocking work on an `async def` path — `time.sleep`, blocking HTTP (`requests`/`urllib`), heavy CPU work, or a lock held across an `await` — stalls the loop for *every* concurrent request, not just the blocking one. A lock held across `await` silently serializes the whole pipeline; blocking I/O tanks throughput and inflates tail latency.
|
||||
|
||||
### Severity
|
||||
- ⚠ any *new* `time.sleep(`, blocking HTTP call (`requests.get/post`, `urllib.request`, `urlopen`), or `await` inside a `with Lock` / `async with` critical section, in `async def` code.
|
||||
- ✗ for `time.sleep` or blocking HTTP on the serving path (`engine/`, `entrypoints/`, async `connectors/`), or a lock acquired and held across an `await` on the request path.
|
||||
|
||||
### Detect (diff-scoped)
|
||||
```bash
|
||||
# New blocking primitives introduced by the diff — then eyeball whether each
|
||||
# sits in an async function on the loop vs a dedicated worker thread:
|
||||
git diff ${BASE}...HEAD -- "*.py" | grep "^+[^+]" \
|
||||
| grep -E 'time\.sleep|requests\.(get|post|put|delete)|urllib\.request|urlopen'
|
||||
# Lock acquired with an await potentially inside the critical section (manual eyeball):
|
||||
git diff ${BASE}...HEAD -- "*.py" | grep "^+[^+]" | grep -E 'async with|\.acquire\(\)'
|
||||
```
|
||||
Note: blocking inside a **dedicated worker thread is correct** — the repo already does this (`diffusion/model_loader/hub_prefetch.py`, `distributed/omni_connectors/kv_transfer_manager.py`, the `mooncake`/`mori` connectors poll with `time.sleep` on their own threads). The check targets the *event-loop* path only.
|
||||
|
||||
### Fix
|
||||
Move blocking work off the loop with `asyncio.to_thread(...)` / `run_in_executor`, or use a native async client (`httpx.AsyncClient` / `aiohttp`). Never `await` while holding a lock — drop the lock before the `await` and re-acquire after, or restructure so the critical section contains no `await`.
|
||||
|
||||
---
|
||||
|
||||
## Running the sweep
|
||||
|
||||
Run all five detections against `${BASE}...HEAD`, then for each hit decide ⚠ vs ✗ using the severity rules above. Roll the results into the report as a single **Code quality** dimension row (count of ⚠ and ✗), alongside the type-specific checklist from [checklists.md](checklists.md).
|
||||
+10
-10
@@ -22,30 +22,30 @@
|
||||
/.buildkite/ @yenuo26 @congw729
|
||||
|
||||
# Engine and model_executor
|
||||
/vllm_omni/core/ @tzhouam @yinpeiqi @fake0fan @Sy0307
|
||||
/vllm_omni/engine/ @tzhouam @yinpeiqi @fake0fan @Sy0307
|
||||
/vllm_omni/entrypoints/ @tzhouam @yinpeiqi @fake0fan @alex-jw-brooks
|
||||
/vllm_omni/core/ @tzhouam @yinpeiqi @fake0fan
|
||||
/vllm_omni/engine/ @tzhouam @yinpeiqi @fake0fan
|
||||
/vllm_omni/entrypoints/ @tzhouam @yinpeiqi @fake0fan
|
||||
/vllm_omni/model_executor/ @tzhouam @gcanlin
|
||||
/vllm_omni/worker/ @tzhouam @gcanlin
|
||||
|
||||
# Diffusion
|
||||
/vllm_omni/diffusion/ @Isotr0py @princepride @SamitHuang @wtomin @ZJY0516 @RuixiangMa @david6666666 @xuechendi
|
||||
/vllm_omni/diffusion/ @Isotr0py @princepride @SamitHuang @wtomin @ZJY0516 @RuixiangMa @david6666666
|
||||
/vllm_omni/lora/ @princepride @SamitHuang @wtomin @ZJY0516 @RuixiangMa @yuanheng-zhao
|
||||
|
||||
# Diffusion CPU offloading
|
||||
/vllm_omni/diffusion/offloader/ @Isotr0py @princepride @SamitHuang @wtomin @ZJY0516 @RuixiangMa @yuanheng-zhao @xuechendi
|
||||
/vllm_omni/diffusion/offloader/ @Isotr0py @princepride @SamitHuang @wtomin @ZJY0516 @RuixiangMa @yuanheng-zhao
|
||||
|
||||
# Omni and TTS support
|
||||
/vllm_omni/transformers_utils/ @tzhouam @yuanheng-zhao
|
||||
/vllm_omni/model_executor/models/ @linyueqian @ZeldaHuang @princepride @yuanheng-zhao @amy-why-3459 @Bounty-hunter @Sy0307 @gcanlin
|
||||
/vllm_omni/model_executor/models/ @linyueqian @ZeldaHuang @princepride @yuanheng-zhao @amy-why-3459 @Bounty-hunter
|
||||
|
||||
# Benchmarks
|
||||
/benchmarks/accuracy/ @david6666666
|
||||
/benchmarks/build_dataset/ @linyueqian @yuanheng-zhao
|
||||
/benchmarks/diffusion/ @Isotr0py @princepride @SamitHuang @wtomin @ZJY0516 @RuixiangMa @xuechendi
|
||||
/benchmarks/diffusion/ @Isotr0py @princepride @SamitHuang @wtomin @ZJY0516 @RuixiangMa
|
||||
/benchmarks/distributed/ @princepride
|
||||
/benchmarks/glm_image/ @princepride @JaredforReal
|
||||
/benchmarks/tts/ @linyueqian @ZeldaHuang @princepride @Sy0307 @gcanlin
|
||||
/benchmarks/tts/ @linyueqian @ZeldaHuang @princepride
|
||||
|
||||
# Usability and Observability
|
||||
/vllm_omni/deploy/ @tzhouam @lishunyang12
|
||||
@@ -53,13 +53,13 @@
|
||||
/vllm_omni/profiler/ @gcanlin
|
||||
|
||||
# ComfyUI
|
||||
/apps/ @fhfuih @alex-jw-brooks
|
||||
/apps/ @fhfuih
|
||||
|
||||
# Large Scale Serving
|
||||
/vllm_omni/distributed/ @princepride @yuanheng-zhao @xuechendi @natureofnature @fake0fan
|
||||
|
||||
# Quantization and configuration
|
||||
/vllm_omni/config/ @lishunyang12 @alex-jw-brooks
|
||||
/vllm_omni/config/ @lishunyang12
|
||||
/vllm_omni/quantization/ @david6666666 @Isotr0py @lishunyang12
|
||||
|
||||
# Hardware plugins and accelerators
|
||||
|
||||
@@ -19,7 +19,6 @@ body:
|
||||
|
||||
Consider redacting or replacing sensitive values with placeholders like `<YOUR_TOKEN_HERE>` when sharing configuration or code examples.
|
||||
- type: textarea
|
||||
id: current-environment
|
||||
attributes:
|
||||
label: Your current environment
|
||||
description: |
|
||||
@@ -43,7 +42,6 @@ body:
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
id: code-version
|
||||
attributes:
|
||||
label: Your code version
|
||||
description: |
|
||||
@@ -68,7 +66,6 @@ body:
|
||||
required: true
|
||||
|
||||
- type: textarea
|
||||
id: bug-description
|
||||
attributes:
|
||||
label: 🐛 Describe the bug
|
||||
description: |
|
||||
|
||||
@@ -208,7 +208,6 @@ cache/
|
||||
.cache/
|
||||
diffusion_cache/
|
||||
kv_cache/
|
||||
!vllm_omni/experimental/ar_diffusion/kv_cache/
|
||||
|
||||
# Logs
|
||||
logs/
|
||||
@@ -242,7 +241,6 @@ datasets/
|
||||
*.json
|
||||
!tests/dfx/perf/tests/*.json
|
||||
!apps/ComfyUI-vLLM-Omni/example_workflows/*.json
|
||||
!tests/assets/soulxsinger/*.json
|
||||
*.jsonl
|
||||
*.parquet
|
||||
|
||||
@@ -275,8 +273,3 @@ tests/.ci_generated/
|
||||
|
||||
# Local Lance precision-matching dump scripts (not part of the repo)
|
||||
lance_compare/
|
||||
|
||||
# FP8 quantization test sample images: tests/diffusion/quantization/
|
||||
# test_quantization_fp8.py saves these into the repo root for manual
|
||||
# inspection; they are byproducts and must never be committed.
|
||||
test_*_fp8.png
|
||||
@@ -1,87 +0,0 @@
|
||||
# LVSA Long-Video Showcase
|
||||
|
||||
**Training-free block-sparse attention for long-video diffusion**, running on
|
||||
vLLM-Omni via the public plugin entry points (no core changes). LVSA speeds up
|
||||
extended-horizon generation and prevents the freeze/loop failure mode dense
|
||||
attention exhibits beyond a model's training length — with no fine-tuning and no
|
||||
weight changes.
|
||||
|
||||
Upstream: **[LongVideoSparseAttention](https://github.com/JiusiServe/LongVideoSparseAttention)**
|
||||
(algorithm, all model adapters, full docs).
|
||||
|
||||
## Results
|
||||
|
||||
Wan 2.1 1.3B, single A100 80 GB, mean over 5 prompts (Dense vs LVSA-FlashInfer):
|
||||
|
||||
| Horizon | 1× | 2× | 3× | 4× | 5× | 6× |
|
||||
|---|---|---|---|---|---|---|
|
||||
| **Speedup** | 0.9× | 1.4× | 1.9× | 2.5× | 3.0× | **3.5×** |
|
||||
| **VQeval composite** (Dense→LVSA) | 63→63 | 59→63 | 61→63 | 61→64 | 59→64 | 59→63 |
|
||||
|
||||
The speedup **grows with length** (LVSA is the dense regime at 1× by design, so
|
||||
the win starts at 2×), and quality *improves* at extension — LVSA's rotating
|
||||
keyframes prevent the looping/static output dense produces beyond its horizon
|
||||
(VQeval loop-quality +27 to +34 at ≥3×). At long horizons dense **OOMs on 80 GB**
|
||||
(HunyuanVideo ≥2×, Wan2.1-14B and Cosmos at 6×) where LVSA still generates.
|
||||
|
||||
→ Full sweep (5 models × 6 horizons, VQeval + VBench-Long) and the
|
||||
quality-metric methodology:
|
||||
**[benchmarks/](https://github.com/JiusiServe/LongVideoSparseAttention/tree/main/benchmarks)**.
|
||||
|
||||
## Quickstart
|
||||
|
||||
```bash
|
||||
# 1. vLLM-Omni 0.22.0 (stable) + matching vLLM
|
||||
pip install "vllm==0.22.0"
|
||||
pip install --no-build-isolation \
|
||||
"vllm-omni @ git+https://github.com/vllm-project/vllm-omni.git@v0.22.0"
|
||||
|
||||
# 2. LVSA core + plugin
|
||||
git clone https://github.com/JiusiServe/LongVideoSparseAttention.git
|
||||
( cd LongVideoSparseAttention && pip install -e . && pip install -e lvsa-vllm-omni/ )
|
||||
|
||||
# 3. this showcase's client dep
|
||||
pip install requests
|
||||
```
|
||||
|
||||
FlashInfer (used by the default `flashinfer` backend) **ships with vLLM** — step 1
|
||||
already pulls the pinned `flashinfer-python` + `flashinfer-cubin`, so the fused
|
||||
backend works with no extra install. Two optional extras:
|
||||
|
||||
```bash
|
||||
# (optional) faster startup — prebuilt JIT cache, matched to your CUDA + flashinfer
|
||||
CU=cu128 # match your CUDA: cu121 / cu124 / cu128 / …
|
||||
FIVER=$(python -c "import importlib.metadata as m; print(m.version('flashinfer-python'))")
|
||||
pip install --extra-index-url "https://flashinfer.ai/whl/${CU}" \
|
||||
"flashinfer-jit-cache==${FIVER}+${CU}"
|
||||
|
||||
# (fallback) no FlashInfer in your environment? run the SDPA backend instead:
|
||||
export LVSA_BACKEND=sdpa
|
||||
```
|
||||
|
||||
```bash
|
||||
# Serve an LVSA-enabled endpoint (Wan 2.1 1.3B at 2× horizon):
|
||||
MODEL=/path/to/Wan2.1-T2V-1.3B-Diffusers MODEL_FAMILY=wan FRAMES=161 \
|
||||
bash serve_lvsa.sh
|
||||
|
||||
# In another shell — generate:
|
||||
python generate.py --frames 161 --height 480 --width 832 \
|
||||
--prompt "A dog running through a sunlit forest." --out wan_lvsa.mp4
|
||||
```
|
||||
|
||||
`serve_lvsa.sh` sets the LVSA env vars for the chosen `MODEL_FAMILY`
|
||||
(`wan` / `hunyuan`). Wan and HunyuanVideo run through the LVSA **attention
|
||||
backend** by default — it stays sparse even under sequence-parallel, unlike the
|
||||
monkey-patch hooks.
|
||||
|
||||
## Configuration
|
||||
|
||||
The one knob you must get right is the model's training horizon
|
||||
(`LVSA_REFERENCE_LATENT_FRAMES`: Wan = 21, HunyuanVideo = 33); `serve_lvsa.sh`
|
||||
sets it per family. For the full environment-variable reference, the multi-GPU
|
||||
support matrix, and tuning (`sparsity_scale`, rotating keyframes, non-standard
|
||||
resolutions), see the upstream docs:
|
||||
|
||||
- **[lvsa-vllm-omni plugin reference](https://github.com/JiusiServe/LongVideoSparseAttention/blob/main/lvsa-vllm-omni/README.md)** — every `LVSA_*` env var
|
||||
- **[docs/parallelism.md](https://github.com/JiusiServe/LongVideoSparseAttention/blob/main/docs/parallelism.md)** — TP / Ulysses / HSDP support per model
|
||||
- **[docs/tuning.md](https://github.com/JiusiServe/LongVideoSparseAttention/blob/main/docs/tuning.md)** — picking knobs for your model and horizon
|
||||
@@ -1,93 +0,0 @@
|
||||
#!/usr/bin/env python
|
||||
"""Generate a long video through an LVSA-enabled vLLM-Omni server.
|
||||
|
||||
A thin client over the Omni video API — start the server first with
|
||||
``serve_lvsa.sh``. The sparse attention runs server-side; this just submits a
|
||||
job and saves the result.
|
||||
|
||||
Uses the **asynchronous** endpoint (``POST /v1/videos`` → poll
|
||||
``GET /v1/videos/{id}`` → download ``GET /v1/videos/{id}/content``) rather than
|
||||
``POST /v1/videos/sync``. The sync endpoint is capped by a fixed server-side
|
||||
timeout (``VIDEO_SYNC_TIMEOUT_S``, 600 s) and is documented as
|
||||
"for benchmark and testing scenarios" — long videos (LVSA's use case) routinely
|
||||
exceed it. The async job has no such cap.
|
||||
|
||||
python generate.py --frames 481 --height 480 --width 832 \
|
||||
--prompt "A golden retriever running through a sunlit forest." \
|
||||
--out dog_forest_lvsa.mp4
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import time
|
||||
|
||||
import requests
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--server", default=os.environ.get("OMNI_SERVER", "http://localhost:8000"))
|
||||
ap.add_argument("--prompt", default="A dog running through a sunlit forest.")
|
||||
ap.add_argument("--frames", type=int, default=161) # Wan 2x horizon
|
||||
ap.add_argument("--height", type=int, default=480)
|
||||
ap.add_argument("--width", type=int, default=832)
|
||||
ap.add_argument("--fps", type=int, default=16)
|
||||
ap.add_argument("--steps", type=int, default=50)
|
||||
ap.add_argument("--guidance", type=float, default=5.0)
|
||||
ap.add_argument("--out", default="lvsa_long_video.mp4")
|
||||
ap.add_argument("--poll-interval", type=float, default=5.0, help="Seconds between status polls.")
|
||||
ap.add_argument(
|
||||
"--max-wait", type=float, default=7200.0, help="Give up if the job hasn't finished after this many seconds."
|
||||
)
|
||||
args = ap.parse_args()
|
||||
|
||||
base = args.server.rstrip("/")
|
||||
form = {
|
||||
"prompt": args.prompt,
|
||||
"size": f"{args.width}x{args.height}",
|
||||
"num_frames": str(args.frames),
|
||||
"fps": str(args.fps),
|
||||
"num_inference_steps": str(args.steps),
|
||||
"guidance_scale": str(args.guidance),
|
||||
}
|
||||
|
||||
# ── 1. submit the async job ──────────────────────────────────────────────
|
||||
print(f"[lvsa-app] POST {base}/v1/videos {args.width}x{args.height} {args.frames}f steps={args.steps}")
|
||||
resp = requests.post(f"{base}/v1/videos", data=form, timeout=120)
|
||||
resp.raise_for_status()
|
||||
job = resp.json()
|
||||
vid = job["id"]
|
||||
print(f"[lvsa-app] job {vid} ({job.get('status', '?')})")
|
||||
|
||||
# ── 2. poll until completed / failed ─────────────────────────────────────
|
||||
deadline = time.time() + args.max_wait
|
||||
last = None
|
||||
while True:
|
||||
if time.time() > deadline:
|
||||
raise SystemExit(f"[lvsa-app] timed out after {args.max_wait:.0f}s (job {vid} still running)")
|
||||
time.sleep(args.poll_interval)
|
||||
r = requests.get(f"{base}/v1/videos/{vid}", timeout=60)
|
||||
if r.status_code != 200:
|
||||
# retrieve_video returns a non-200 JSON when the job has FAILED
|
||||
raise SystemExit(f"[lvsa-app] job failed: {r.text}")
|
||||
st = r.json()
|
||||
status = st.get("status")
|
||||
progress = st.get("progress", 0)
|
||||
if (status, progress) != last:
|
||||
print(f"[lvsa-app] {status} {progress}%")
|
||||
last = (status, progress)
|
||||
if status == "completed":
|
||||
break
|
||||
if status == "failed":
|
||||
raise SystemExit(f"[lvsa-app] job failed: {st.get('error')}")
|
||||
|
||||
# ── 3. download the rendered mp4 ─────────────────────────────────────────
|
||||
c = requests.get(f"{base}/v1/videos/{vid}/content", timeout=600)
|
||||
c.raise_for_status()
|
||||
with open(args.out, "wb") as f:
|
||||
f.write(c.content)
|
||||
print(f"[lvsa-app] saved {args.out}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,45 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
# Start an LVSA-enabled vLLM-Omni server for Wan or HunyuanVideo.
|
||||
#
|
||||
# LVSA is selected as the attention backend via --diffusion-attention-config (see
|
||||
# below); the `LVSA_*` env vars only tune it. The `lvsa-vllm-omni` plugin registers
|
||||
# itself through vLLM-Omni's `general_plugins` entry points — nothing in vLLM-Omni
|
||||
# core is modified. Watch the log for "[LVSA] Geometry detected" to confirm sparse
|
||||
# attention is active for your request geometry.
|
||||
#
|
||||
# MODEL=/path/to/Wan2.1-T2V-1.3B-Diffusers MODEL_FAMILY=wan FRAMES=161 bash serve_lvsa.sh
|
||||
# MODEL=/path/to/HunyuanVideo-1.5-... MODEL_FAMILY=hunyuan FRAMES=193 bash serve_lvsa.sh
|
||||
set -euo pipefail
|
||||
|
||||
MODEL=${MODEL:?set MODEL=/path/to/Wan-or-HunyuanVideo-Diffusers}
|
||||
MODEL_FAMILY=${MODEL_FAMILY:-wan} # wan | hunyuan
|
||||
PORT=${PORT:-8000}
|
||||
FRAMES=${FRAMES:-161} # target horizon in frames
|
||||
VAE_T=${VAE_T:-4} # VAE temporal compression (Wan/Hunyuan = 4)
|
||||
BACKEND=${LVSA_BACKEND:-flashinfer} # flashinfer (fused, fastest) | sdpa
|
||||
|
||||
T_LAT=$(( (FRAMES - 1) / VAE_T + 1 ))
|
||||
|
||||
# Per-family training reference horizon (Wan train ref = 21, Hunyuan = 33).
|
||||
case "$MODEL_FAMILY" in
|
||||
wan) REFERENCE=${REFERENCE:-21} ;;
|
||||
hunyuan) REFERENCE=${REFERENCE:-33} ;;
|
||||
*) echo "MODEL_FAMILY must be 'wan' or 'hunyuan'"; exit 1 ;;
|
||||
esac
|
||||
export LVSA_BACKEND="$BACKEND"
|
||||
export LVSA_TOTAL_LATENT_FRAMES="$T_LAT"
|
||||
export LVSA_REFERENCE_LATENT_FRAMES="$REFERENCE"
|
||||
export LVSA_ROTATE_KEYFRAMES=${LVSA_ROTATE_KEYFRAMES:-1}
|
||||
|
||||
echo "[lvsa-app] serving $MODEL ($MODEL_FAMILY) on :$PORT"
|
||||
echo "[lvsa-app] LVSA backend=$BACKEND T_lat=$T_LAT (frames=$FRAMES) reference=$REFERENCE"
|
||||
echo "[lvsa-app] confirm activation by grep '\\[LVSA\\] Geometry detected' in this log"
|
||||
|
||||
# vLLM-Omni 0.22 selects the LVSA attention backend per-role via
|
||||
# --diffusion-attention-config. The backend is what drives sparse attention
|
||||
# here; it engages under tensor-parallel too, unlike the LVSA_*_HOOK monkey-patch
|
||||
# path (which falls back to dense under sequence-parallel), so we deliberately do
|
||||
# NOT set a hook.
|
||||
exec vllm serve "$MODEL" --omni --host 0.0.0.0 --port "$PORT" \
|
||||
--diffusion-attention-config '{"per_role": {"self": {"backend": "LVSA"}}}' \
|
||||
--init-timeout 1800 ${EXTRA_ARGS:-}
|
||||
@@ -201,7 +201,6 @@ async def main():
|
||||
"input": prompt,
|
||||
"voice": "default",
|
||||
"stream": True,
|
||||
"stream_format": "audio",
|
||||
"response_format": "pcm",
|
||||
"ref_audio": ref_audio_b64,
|
||||
"ref_text": args.ref_text,
|
||||
@@ -241,7 +240,6 @@ async def main():
|
||||
"input": prompt,
|
||||
"voice": args.voice_name,
|
||||
"stream": True,
|
||||
"stream_format": "audio",
|
||||
"response_format": "pcm",
|
||||
"ref_text": args.ref_text,
|
||||
"max_new_tokens": 2048,
|
||||
|
||||
@@ -80,17 +80,6 @@ models:
|
||||
task_extra_body:
|
||||
voice_clone: {}
|
||||
|
||||
# MOSS-TTS-Local-Transformer-v1.5: Qwen3 backbone + 1-layer GPT2-style local
|
||||
# depth transformer (n_vq=12), paired with the 48 kHz stereo
|
||||
# MOSS-Audio-Tokenizer-v2 codec. Same voice-clone request shape as the
|
||||
# other MOSS-TTS variants.
|
||||
OpenMOSS-Team/MOSS-TTS-Local-Transformer-v1.5:
|
||||
supported_tasks: [voice_clone]
|
||||
backend: openai-audio-speech
|
||||
endpoint: /v1/audio/speech
|
||||
task_extra_body:
|
||||
voice_clone: {}
|
||||
|
||||
# MOSS-TTSD synthesizes [S1]/[S2] dialogue. The seed-tts voice_clone dataset
|
||||
# has single-speaker rows; bench piggybacks on it (both speakers share the
|
||||
# same ref_audio). For real dialogue evaluation use the ttsd dataset.
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
# Target vllm version: v0.24.0 (releases/v0.24.0 branch).
|
||||
# Target vllm version: v0.21.0 (releases/v0.21.0 branch).
|
||||
# The pinned VLLM_PRECOMPILED_WHEEL_COMMIT and VLLM_BASE_TAG below must
|
||||
# be updated on every rebase to match the new upstream vllm commit/tag.
|
||||
ARG VLLM_BASE_IMAGE=vllm/vllm-openai
|
||||
ARG VLLM_BASE_TAG=v0.25.0
|
||||
ARG VLLM_BASE_TAG=v0.23.0
|
||||
FROM ${VLLM_BASE_IMAGE}:${VLLM_BASE_TAG}
|
||||
ARG APP_DIR=/workspace/vllm-omni
|
||||
WORKDIR ${APP_DIR}
|
||||
@@ -19,4 +19,4 @@ RUN uv pip install --system ".[dev]"
|
||||
|
||||
RUN ln -sf /usr/bin/python3 /usr/bin/python
|
||||
|
||||
ENTRYPOINT []
|
||||
ENTRYPOINT []
|
||||
@@ -1,4 +1,4 @@
|
||||
ARG BASE_IMAGE=vllm/vllm-openai:v0.24.0
|
||||
ARG BASE_IMAGE=vllm/vllm-openai:v0.23.0
|
||||
FROM ${BASE_IMAGE}
|
||||
|
||||
ARG COMMON_WORKDIR=/app
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
ARG VLLM_ASCEND_IMAGE=quay.io/atlas-ci/vllm-ascend
|
||||
ARG VLLM_ASCEND_TAG=v0.24.0
|
||||
ARG VLLM_ASCEND_TAG=v0.23.0-4885d8e
|
||||
FROM ${VLLM_ASCEND_IMAGE}:${VLLM_ASCEND_TAG}
|
||||
|
||||
# WORKDIR /vllm-workspace/vllm
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
ARG VLLM_ASCEND_IMAGE=quay.io/atlas-ci/vllm-ascend
|
||||
ARG VLLM_ASCEND_TAG=v0.24.0-a3
|
||||
ARG VLLM_ASCEND_TAG=v0.23.0-4885d8e-a3
|
||||
FROM ${VLLM_ASCEND_IMAGE}:${VLLM_ASCEND_TAG}
|
||||
|
||||
# WORKDIR /vllm-workspace/vllm
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
ARG VLLM_ASCEND_IMAGE=quay.io/atlas-ci/vllm-ascend
|
||||
ARG VLLM_ASCEND_TAG=v0.24.0
|
||||
ARG VLLM_ASCEND_TAG=v0.23.0-4885d8e
|
||||
FROM ${VLLM_ASCEND_IMAGE}:${VLLM_ASCEND_TAG}
|
||||
|
||||
ARG APP_DIR=/vllm-workspace/vllm-omni
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
ARG VLLM_ASCEND_IMAGE=quay.io/atlas-ci/vllm-ascend
|
||||
ARG VLLM_ASCEND_TAG=v0.24.0-a3
|
||||
ARG VLLM_ASCEND_TAG=v0.23.0-4885d8e-a3
|
||||
FROM ${VLLM_ASCEND_IMAGE}:${VLLM_ASCEND_TAG}
|
||||
|
||||
ARG APP_DIR=/vllm-workspace/vllm-omni
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
ARG BASE_IMAGE=vllm/vllm-openai-rocm:v0.24.0
|
||||
ARG BASE_IMAGE=vllm/vllm-openai-rocm:v0.23.0
|
||||
FROM ${BASE_IMAGE} AS base
|
||||
|
||||
# Declare a variable to know if we want to use the nightly build or the stable build.
|
||||
|
||||
@@ -88,7 +88,7 @@ ENV UV_EXTRA_INDEX_URL=${PIP_EXTRA_INDEX_URL}
|
||||
ENV UV_INDEX_STRATEGY="unsafe-best-match"
|
||||
ENV UV_LINK_MODE="copy"
|
||||
|
||||
ARG VLLM_VERSION=v0.24.0
|
||||
ARG VLLM_VERSION=v0.23.0
|
||||
RUN git clone -b ${VLLM_VERSION} https://github.com/vllm-project/vllm
|
||||
WORKDIR /workspace/vllm
|
||||
|
||||
@@ -101,15 +101,13 @@ RUN --mount=type=cache,target=/root/.cache/uv \
|
||||
source /opt/intel/oneapi/ccl/2021.15/env/vars.sh --force && \
|
||||
export CMAKE_PREFIX_PATH="$(python3 -c 'import site; print(site.getsitepackages()[0])'):${CMAKE_PREFIX_PATH}"
|
||||
|
||||
ENV LD_LIBRARY_PATH="/opt/intel/oneapi/ccl/2021.15/lib:$LD_LIBRARY_PATH:/usr/local/lib/"
|
||||
ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/local/lib/"
|
||||
|
||||
ENV VLLM_TARGET_DEVICE=xpu
|
||||
ENV VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
|
||||
RUN --mount=type=cache,target=/root/.cache/uv \
|
||||
uv pip uninstall triton triton-xpu && \
|
||||
uv pip install triton-xpu==3.7.1 && \
|
||||
uv pip install --no-build-isolation --no-deps .
|
||||
uv pip install --no-build-isolation .
|
||||
|
||||
CMD ["/bin/bash"]
|
||||
|
||||
@@ -145,11 +143,12 @@ COPY . .
|
||||
|
||||
ENV VLLM_OMNI_TARGET_DEVICE=xpu
|
||||
RUN --mount=type=cache,target=/root/.cache/uv \
|
||||
echo "triton @ https://example.com/blocked" > /tmp/vllm_omni_constraints.txt && \
|
||||
PIP_CONSTRAINT=/tmp/vllm_omni_constraints.txt \
|
||||
uv pip install --no-build-isolation \
|
||||
--extra-index-url https://mirrors.ustc.edu.cn/pypi/simple/ \
|
||||
".[dev]" && \
|
||||
uv pip uninstall triton oneccl oneccl-devel && \
|
||||
uv pip install triton-xpu==3.7.1 --reinstall
|
||||
uv pip uninstall oneccl oneccl-devel
|
||||
|
||||
FROM vllm-omni AS vllm-omni-openai
|
||||
|
||||
|
||||
+1
-3
@@ -68,11 +68,10 @@ nav:
|
||||
- Hybrid Sharded Data Parallel: user_guide/diffusion/parallelism/hsdp.md
|
||||
- Sequence Parallel: user_guide/diffusion/parallelism/sequence_parallel.md
|
||||
- Tensor Parallel: user_guide/diffusion/parallelism/tensor_parallel.md
|
||||
- VAE Parallelism: user_guide/diffusion/parallelism/vae_parallelism.md
|
||||
- VAE Patch Parallel: user_guide/diffusion/parallelism/vae_patch_parallel.md
|
||||
- CPU Offloading: user_guide/diffusion/cpu_offload_diffusion.md
|
||||
- LoRA: user_guide/diffusion/lora.md
|
||||
- Custom Pipeline: features/custom_pipeline.md
|
||||
- Request-Level Batching: user_guide/diffusion/request_batching.md
|
||||
- Step Execution: user_guide/diffusion/step_execution.md
|
||||
- Quantization:
|
||||
- Overview: user_guide/quantization/overview.md
|
||||
@@ -119,7 +118,6 @@ nav:
|
||||
- design/feature/async_chunk.md
|
||||
- design/feature/vae_parallel.md
|
||||
- design/feature/diffusion_step_execution.md
|
||||
- design/feature/diffusion_request_level_batching.md
|
||||
- design/feature/diffusion_continuous_batching.md
|
||||
- Module Design:
|
||||
- design/module/ar_module.md
|
||||
|
||||
Binary file not shown.
|
Before Width: | Height: | Size: 266 KiB After Width: | Height: | Size: 410 KiB |
@@ -33,20 +33,16 @@ Lead maintainers are responsible for the overall direction and strategy of the p
|
||||
|
||||
Committers have write access and merge rights. They typically have deep expertise in specific areas of this project and shepherd the community contributions:
|
||||
|
||||
- [@alex-jw-brooks](https://github.com/alex-jw-brooks): Frontend, Configuration, and Entrypoint Support
|
||||
- [@amy-why-3459](https://github.com/amy-why-3459): Omni Modality Model and World Model Support
|
||||
- [@david6666666](https://github.com/david6666666): Quantization and Community Relationship
|
||||
- [@gcanlin](https://github.com/gcanlin): Hardware plugin, NPU integration, and TTS Model Support
|
||||
- [@gcanlin](https://github.com/gcanlin): Hardware plugin and NPU integration
|
||||
- [@Isotr0py](https://github.com/Isotr0py): Diffusion and Quantization
|
||||
- [@linyueqian](https://github.com/linyueqian): TTS and Omni Support
|
||||
- [@lishunyang12](https://github.com/lishunyang12): Quantization and Configuration
|
||||
- [@princepride](https://github.com/princepride): Diffusion and Omni Support
|
||||
- [@RuixiangMa](https://github.com/RuixiangMa): Diffusion models, parallel, cache, and docs
|
||||
- [@SamitHuang](https://github.com/SamitHuang): RL, Diffusion, and cache
|
||||
- [@Sy0307](https://github.com/Sy0307): TTS Model and AR Engine Support
|
||||
- [@tzhouam](https://github.com/tzhouam): Engine and New Model Support
|
||||
- [@wtomin](https://github.com/wtomin): Diffusion models, parallel, and docs
|
||||
- [@xuechendi](https://github.com/xuechendi): Diffusion and Intel GPU Support
|
||||
- [@ZeldaHuang](https://github.com/ZeldaHuang): Omni Support
|
||||
- [@ZJY0516](https://github.com/ZJY0516): Diffusion attention backend, kernel fusion, and CustomOp
|
||||
- [@yuanheng-zhao](https://github.com/yuanheng-zhao): Diffusion cache, offload, and Omni Support
|
||||
@@ -57,7 +53,7 @@ This section breaks down diffusion-related responsibilities. If you have PRs tou
|
||||
|
||||
### Runtime-related
|
||||
|
||||
- Diffusion models: @RuixiangMa, @wtomin, @xuechendi
|
||||
- Diffusion models: @RuixiangMa, @wtomin
|
||||
|
||||
### Optimization
|
||||
|
||||
@@ -65,7 +61,6 @@ This section breaks down diffusion-related responsibilities. If you have PRs tou
|
||||
- Attention backend: @ZJY0516
|
||||
- Cache: @yuanheng-zhao, @RuixiangMa, @SamitHuang
|
||||
- Offload: @yuanheng-zhao
|
||||
- Intel GPU support: @xuechendi
|
||||
- Quantization: @lishunyang12, @david6666666
|
||||
- Kernel fusion / communication-computation: @ZJY0516
|
||||
|
||||
|
||||
@@ -1,239 +0,0 @@
|
||||
# Composable parallel strategies
|
||||
|
||||
In vLLM-Omni, the `composable_parallel` system layers a declarative *parallel strategy* on top of the existing stage configs. A strategy YAML maps each pipeline stage (by its `model_stage` name) to a stack of axis declarations — one per parallelism dimension — that the engine translates into per-stage parallel sizing (`tensor_parallel_size`, `data_parallel_size`, `pipeline_parallel_size`, expert-parallel toggle, replica count) and, when relevant, an orchestrator-wide load-balancer policy.
|
||||
|
||||
Strategies are intentionally narrow: each one only writes the axes it declares, leaving every other engine arg untouched. This makes a strategy file a small, reusable overlay you can swap independently of the deploy YAML (see `configuration/stage_configs.md`).
|
||||
|
||||
!!! note
|
||||
Composable parallel is **opt-in**. Passing `--strategy-config <path/to/strategy.yaml>` overlays the derived sizing onto the registry-merged stages *before* any CLI overrides are applied. When `--strategy-config` is omitted, stages remain exactly as their deploy YAML produces them.
|
||||
|
||||
## Top-level schema reference
|
||||
|
||||
A strategy file has a single top-level mapping, `strategies`, keyed by the `model_stage` name of each stage you want to overlay. Each value is a list of axis declarations (see [StrategySpec fields](#strategyspec-fields) below).
|
||||
|
||||
| Field | Type | Required | Default | Description |
|
||||
|-------|------|----------|---------|-------------|
|
||||
| `strategies` | `dict[str, list]` | required | — | Maps `model_stage` name (e.g. `thinker`, `talker`, `code2wav`, `dit`) to a list of axis declarations. The key MUST be the stage name, not the integer `stage_id`. See `vllm_omni/config/composable_parallel/strategy_loader.py`. |
|
||||
|
||||
### StrategySpec fields
|
||||
|
||||
Each entry in a stage's list becomes one `StrategySpec` (see `vllm_omni/config/composable_parallel/spec.py`). The loader fills in `routing` and `aggregation` from per-kind defaults, so the file stays compact.
|
||||
|
||||
| Field | Type | Required | Default | Description |
|
||||
|-------|------|----------|---------|-------------|
|
||||
| `axis` | str | required | — | Mesh-axis kind. One of `tp`, `dp`, `pp`, `ep`, `stage_replica` (wired today) or `sp_ulysses`, `sp_ring`, `cfg`, `vae_pp`, `hsdp`, `stage_pp`, `cp` (reserved; declaring one raises `AxisTranslationError` at translate time). See `MeshAxisKind` in `vllm_omni/config/composable_parallel/spec.py`. |
|
||||
| `size` | int | required | — | Axis degree. Must be `> 0`. |
|
||||
| `routing` | str | optional | per-kind default | Stateless routing policy. Accepted only for `dp` / `stage_replica`; other kinds reject it. Values: `random`, `round_robin`, `least_queue`. (`hash` is recognised but currently raises `NotImplementedError`.) See `RouteByStage` in `vllm_omni/config/composable_parallel/routing.py`. |
|
||||
| `l1_owner` | str | optional | per-kind default | Overrides which layer owns request routing for this axis. `engine` for `tp`/`dp`/`pp`/`ep`, `delegated` for `stage_replica`. Recorded into `shard_extension`. |
|
||||
| `name` | str | optional | axis kind | Human-readable label for this spec, used in error messages. |
|
||||
|
||||
Two additional `StrategySpec` slots — `layer_hook_specs` and `kernel_specs` — let advanced authors bind layer-walker hook IDs (`LayerHookSpec`) or kernel IDs (`KernelSpec`) onto an axis. These are not exposed by the YAML loader today and are intended for programmatic construction. See `vllm_omni/config/composable_parallel/spec.py` for the dataclasses.
|
||||
|
||||
### CLI flags
|
||||
|
||||
| Flag | Description |
|
||||
|------|-------------|
|
||||
| `--strategy-config PATH` | Path to a `strategy.yaml`. Only takes effect on registry-based deploy paths (`--deploy-config` or the bundled default deploy YAML). Its derived sizing is overlaid onto the merged stages *before* CLI overrides. |
|
||||
| `--stage-configs-path PATH` | **Legacy.** Loads the old `stage_args` YAML schema for un-migrated models. Passing `--strategy-config` together with this path is silently inapplicable — vLLM-Omni emits a warning and the strategy is ignored (see `vllm_omni/entrypoints/utils.py`, lines 372–380). |
|
||||
| `--omni-lb-policy POLICY` | Orchestrator-wide load-balancer policy. Strategy-derived from any `stage_replica` axis; an explicit CLI value wins and must match the derived one or `AsyncOmniEngine` raises `Conflicting load-balancer policy` at construction. |
|
||||
|
||||
!!! warning
|
||||
`--strategy-config` cannot be used with `--stage-configs-path` (the legacy deploy path). If your model has not been migrated to the registry-based pipeline yet, the strategy is silently dropped with a warning. Migrate to a registry-based model or use `--deploy-config` to apply a strategy.
|
||||
|
||||
## Sub-schemas
|
||||
|
||||
### `axis`
|
||||
|
||||
The set of recognised axis kinds is the `MeshAxisKind` enum in `vllm_omni/config/composable_parallel/spec.py`:
|
||||
|
||||
```yaml
|
||||
# Translatable today (see `_SUPPORTED_KINDS` in translator.py).
|
||||
axis: tp # tensor parallel — sets tensor_parallel_size
|
||||
axis: dp # engine data parallel — sets data_parallel_size (vLLM's intra-engine DP LB)
|
||||
axis: pp # pipeline parallel — sets pipeline_parallel_size
|
||||
axis: ep # dense expert parallel — sets enable_expert_parallel=True; size must equal tp*dp
|
||||
axis: stage_replica # omni-coordinator-level replica fan-out — sets num_replicas + omni_lb_policy
|
||||
|
||||
# Reserved (raise AxisTranslationError today; declared in the type system for forward compat).
|
||||
axis: sp_ulysses
|
||||
axis: sp_ring
|
||||
axis: cfg
|
||||
axis: vae_pp
|
||||
axis: hsdp
|
||||
axis: stage_pp
|
||||
axis: cp
|
||||
```
|
||||
|
||||
### `routing`
|
||||
|
||||
The full closed hierarchy lives in `vllm_omni/config/composable_parallel/routing.py` (`Broadcast`, `PartitionByHash`, `ShardSequence`, `ShardSpatial`, `RouteByExpert`, `PipelineMicrobatch`, `DuplicateWithCondTag`, `RouteByStage`). For YAML authors only `RouteByStage` is exposed as a `routing:` string, and only for `dp` / `stage_replica` axes (other axes have fixed routing — `tp`/`ep` broadcast, `pp` microbatch). Recognised policies:
|
||||
|
||||
```yaml
|
||||
routing: random # uniform-random across replicas/DP ranks
|
||||
routing: round_robin # cycle through replicas/DP ranks (the default for dp/stage_replica)
|
||||
routing: least_queue # send to the replica with the smallest backlog
|
||||
# routing: hash # recognised but raises NotImplementedError; no key-stable balancer yet
|
||||
```
|
||||
|
||||
### `aggregation`
|
||||
|
||||
Aggregation is filled in automatically by the loader (`_default_aggregation` in `vllm_omni/config/composable_parallel/strategy_loader.py`) from the axis kind, so users do not author it:
|
||||
|
||||
| Axis kind | Aggregation pattern |
|
||||
|-----------|---------------------|
|
||||
| `tp`, `ep` | `TakeRank` (all ranks agree; pick one) |
|
||||
| `dp`, ep (in some flows) | `Union` (disjoint replica results) |
|
||||
| `pp` | `StitchPipeline` |
|
||||
| `stage_replica` | `FanInByStage` |
|
||||
|
||||
Full pattern hierarchy: `TakeRank`, `Union`, `GatherDim`, `AllGather`, `StitchSpatial`, `StitchPipeline`, `Combine`, `FanInByStage` (see `vllm_omni/config/composable_parallel/aggregation.py`).
|
||||
|
||||
## Precedence
|
||||
|
||||
From highest to lowest, when the same axis is touched by multiple layers:
|
||||
|
||||
1. **CLI overrides** (`--stage_N_tensor_parallel_size`, `--stage-overrides`, `--omni-lb-policy`, etc.). A CLI value always wins; a warning is emitted whenever it overrides a strategy-declared axis (see `_reconcile_strategy_with_cli` in `vllm_omni/config/config_factory.py`).
|
||||
2. **Strategy YAML** (`--strategy-config`). Derived sizing is written onto the merged stages with *conflict-on-explicit* semantics — if the deploy YAML already set a value to something different, application raises `StrategyApplyError`.
|
||||
3. **Deploy YAML** (`--deploy-config` or the bundled `vllm_omni/deploy/<model_type>.yaml`).
|
||||
4. **Parser defaults.**
|
||||
|
||||
After CLI overrides land, the device-layout guard re-runs (`check_device_layout`, called from `_reconcile_strategy_with_cli`) against the *effective* `tp * dp * pp * num_replicas` and the resolved `devices` string, so a `--devices` or `--tensor-parallel-size` that conflicts with the strategy's world size cannot slip past the pre-spawn check.
|
||||
|
||||
## Worked example
|
||||
|
||||
### Example 1 — `strategy_tp2.yaml`
|
||||
|
||||
Shard the `thinker` stage with TP=2 on top of the bundled Qwen2.5-Omni deploy YAML. Routing is fixed (broadcast) for TP, so no `routing:` field is needed.
|
||||
|
||||
```yaml
|
||||
# examples/offline_inference/qwen2_5_omni/strategy_tp2.yaml
|
||||
strategies:
|
||||
thinker:
|
||||
- axis: tp
|
||||
size: 2
|
||||
```
|
||||
|
||||
Launch:
|
||||
|
||||
```bash
|
||||
python3 end2end.py --query-type text --num-prompts 6 \
|
||||
--strategy-config strategy_tp2.yaml \
|
||||
--stage-overrides '{"0": {"devices": "0,1"}, "1": {"devices": "2"}, "2": {"devices": "2"}}'
|
||||
```
|
||||
|
||||
Effective `OmniParallelConfig` per stage after the overlay (see `OmniParallelConfig` in `vllm_omni/config/composable_parallel/translator.py`):
|
||||
|
||||
| Stage (`model_stage`) | `tensor_parallel_size` | `data_parallel_size` | `pipeline_parallel_size` | `stage_replica_size` | `omni_lb_policy` |
|
||||
|-----------------------|------------------------|----------------------|--------------------------|----------------------|------------------|
|
||||
| `thinker` | **2** (from strategy) | 1 | 1 | 1 | — |
|
||||
| `talker` | 1 | 1 | 1 | 1 | — |
|
||||
| `code2wav` | 1 | 1 | 1 | 1 | — |
|
||||
|
||||
### Example 2 — `strategy_stage_replica.yaml`
|
||||
|
||||
Replicate the `thinker` stage across two independent engines and let omni's StagePool round-robin requests across them.
|
||||
|
||||
```yaml
|
||||
# examples/offline_inference/qwen2_5_omni/strategy_stage_replica.yaml
|
||||
strategies:
|
||||
thinker:
|
||||
- axis: stage_replica
|
||||
size: 2
|
||||
routing: round_robin
|
||||
```
|
||||
|
||||
Launch:
|
||||
|
||||
```bash
|
||||
python3 end2end.py --query-type text --num-prompts 6 \
|
||||
--strategy-config strategy_stage_replica.yaml \
|
||||
--stage-overrides '{"0": {"devices": "0,1"}, "1": {"devices": "2"}, "2": {"devices": "2"}}'
|
||||
```
|
||||
|
||||
Effective per-stage config and the derived orchestrator knob:
|
||||
|
||||
| Stage (`model_stage`) | `tensor_parallel_size` | `stage_replica_size` | `omni_lb_policy` (derived) |
|
||||
|-----------------------|------------------------|----------------------|----------------------------|
|
||||
| `thinker` | 1 | **2** | **`round-robin`** |
|
||||
| `talker` | 1 | 1 | — |
|
||||
| `code2wav` | 1 | 1 | — |
|
||||
|
||||
The derived `omni_lb_policy = round-robin` is **not** written into any stage's engine args. Instead it is surfaced on `StrategyApplyResult.omni_lb_policy` and applied once at orchestrator construction by `AsyncOmniEngine._apply_strategy_lb_policy` (see `vllm_omni/engine/async_omni_engine.py`, lines 1107–1134). If you also pass `--omni-lb-policy round-robin` the values must match; passing a different value raises `Conflicting load-balancer policy`.
|
||||
|
||||
## Per-field deep-dive
|
||||
|
||||
### `name`
|
||||
|
||||
Default: the axis kind (e.g. `"tp"`).
|
||||
|
||||
Human-readable label that appears in `AxisTranslationError` and `StrategyApplyError` messages. Set it to disambiguate two stages declaring the same axis kind when scanning logs.
|
||||
|
||||
### `mesh_axis` (`axis` + `size`)
|
||||
|
||||
Default: no default — both fields are required.
|
||||
|
||||
`axis` chooses the parallelism dimension; `size` is its degree. Five kinds are wired end-to-end today (`tp`, `dp`, `pp`, `ep`, `stage_replica`); the remaining kinds in `MeshAxisKind` are reserved and raise `AxisTranslationError` at translate time so a typo cannot silently become a no-op (see `_SUPPORTED_KINDS` in `vllm_omni/config/composable_parallel/translator.py`). For `ep`, the size MUST equal `tensor_parallel_size * data_parallel_size`.
|
||||
|
||||
### `routing`
|
||||
|
||||
Default: per-kind — `RouteByStage(round_robin)` for `dp` and `stage_replica`; `PipelineMicrobatch()` for `pp`; `Broadcast()` for `tp`/`ep` and any other kind.
|
||||
|
||||
User-facing string only on `dp` and `stage_replica` axes. The four recognised values are `random`, `round_robin`, `least_queue`, `hash`; `hash` is rejected today with `NotImplementedError`. See `RouteByStage` in `vllm_omni/config/composable_parallel/routing.py`.
|
||||
|
||||
### `l1_owner`
|
||||
|
||||
Default: per-kind — `engine` for `tp`/`dp`/`pp`/`ep`, `delegated` for `stage_replica`.
|
||||
|
||||
Overrides which layer owns request routing along this axis. `engine` means vLLM realizes routing intra-engine; `delegated` means the omni coordinator owns routing across independent replicas. Set this only when overriding the per-kind default in `_DEFAULT_L1_OWNER` (`vllm_omni/config/composable_parallel/translator.py`); invalid values fail fast in `_resolve_l1_owner`.
|
||||
|
||||
### `stage_replica` and `omni_lb_policy`
|
||||
|
||||
`stage_replica` is *not* a vLLM world dimension; it spins up `size` independent engine replicas of one stage, coordinated by the omni `StagePool` load balancer. The axis writes both:
|
||||
|
||||
- The stage's `num_replicas` (a per-stage deploy knob).
|
||||
- The pipeline-wide `omni_lb_policy` string (an orchestrator-level knob).
|
||||
|
||||
Because `omni_lb_policy` is read once at `AsyncOmniEngine` construction — not from per-stage engine args — only one value is allowed across the whole pipeline. If two stages declare conflicting policies, application raises (see `apply_strategy_specs` in `vllm_omni/config/composable_parallel/apply.py`, lines 253–260). To set this knob manually instead, use `--omni-lb-policy` on the CLI; see [the model stage docs](stage_configs.md) for how stage replicas plug into the orchestrator at runtime.
|
||||
|
||||
### `model_stage` keys
|
||||
|
||||
Default: no default — required for every entry under `strategies:`.
|
||||
|
||||
Strategy keys MUST be `model_stage` *names* (strings), the same labels stage configs use for `engine_args.model_stage` (see [`model_stage` in `stage_configs.md`](stage_configs.md#engine_argsmodel_stage)). Integer `stage_id` values are not accepted; the resolver matches on `model_stage` exactly (`_resolve_stage` in `vllm_omni/config/composable_parallel/apply.py`).
|
||||
|
||||
## Common pitfalls
|
||||
|
||||
!!! warning "Using `stage_id` integers as keys"
|
||||
Strategy keys are `model_stage` names (str), not integer stage IDs. A file that uses an integer index will raise at apply time:
|
||||
|
||||
```text
|
||||
StrategyApplyError: strategy role model_stage=0 did not match any stage;
|
||||
available stages: 'thinker'(id=0), 'talker'(id=1), 'code2wav'(id=2)
|
||||
```
|
||||
|
||||
Use the `model_stage` name (e.g. `thinker`) instead.
|
||||
|
||||
!!! warning "Mixing CLI overrides with a strategy on the same axis"
|
||||
A strategy is meant to be the single writer for the axes it declares. If a CLI override (`--stage_0_tensor_parallel_size`, `--stage-overrides '{"0": {...}}'`) sets the same axis, the **CLI value wins** and a warning is emitted from `_reconcile_strategy_with_cli`. The device-layout guard then re-runs against the effective layout, so a CLI override that breaks `tp * dp * pp * num_replicas` will still fail fast at config time.
|
||||
|
||||
!!! warning "Passing `--strategy-config` against the legacy `--stage-configs-path`"
|
||||
Composable parallel only applies on the registry-based deploy path. If a model still resolves through the legacy `stage_args` YAML (loaded via `--stage-configs-path`), `--strategy-config` is silently inapplicable and emits:
|
||||
|
||||
```text
|
||||
--strategy-config (<path>) was provided but model '<model>' resolves via the
|
||||
legacy stage_configs YAML path, which does not support composable-parallel
|
||||
strategies; the strategy is ignored. Use a registry-based model to apply it.
|
||||
```
|
||||
|
||||
Migrate the model to the new deploy schema (see `configuration/stage_configs.md`) before applying a strategy.
|
||||
|
||||
!!! warning "Conflicting `omni_lb_policy` across stages"
|
||||
`omni_lb_policy` is a single pipeline-wide knob. If two `stage_replica` axes in different stages derive different policies, `apply_strategy_specs` raises:
|
||||
|
||||
```text
|
||||
StrategyApplyError: role 'talker' derives omni_lb_policy='least-queue-length'
|
||||
but role 'thinker' already derived 'round-robin'. The load-balancer policy is
|
||||
pipeline-wide; only one value is allowed.
|
||||
```
|
||||
|
||||
Make the policies match, drop one, or supply the value explicitly via `--omni-lb-policy`.
|
||||
@@ -123,7 +123,7 @@ After inserting the extra thinker stage, renumber the remaining stages:
|
||||
runtime:
|
||||
devices: "2"
|
||||
engine_input_source: [1]
|
||||
sync_process_input_func: vllm_omni.model_executor.stage_input_processors.qwen3_omni.thinker2talker_token_only
|
||||
custom_process_input_func: vllm_omni.model_executor.stage_input_processors.qwen3_omni.thinker2talker
|
||||
|
||||
- stage_id: 3
|
||||
runtime:
|
||||
@@ -131,7 +131,7 @@ After inserting the extra thinker stage, renumber the remaining stages:
|
||||
engine_args:
|
||||
max_num_seqs: 1
|
||||
engine_input_source: [2]
|
||||
sync_process_input_func: vllm_omni.model_executor.stage_input_processors.qwen3_omni.talker2code2wav_token_only
|
||||
custom_process_input_func: vllm_omni.model_executor.stage_input_processors.qwen3_omni.talker2code2wav
|
||||
```
|
||||
|
||||
Compared with the default Qwen3-Omni config:
|
||||
|
||||
@@ -290,6 +290,7 @@ stage_args:
|
||||
enable_prefix_caching: false
|
||||
engine_output_type: latent
|
||||
engine_input_source: [0]
|
||||
custom_process_input_func: vllm_omni.model_executor.stage_input_processors.qwen2_5_omni.thinker2talker
|
||||
default_sampling_params:
|
||||
temperature: 0.9
|
||||
top_p: 0.8
|
||||
@@ -378,20 +379,6 @@ The maximum number of sequences for concurrent processing in this stage. For LLM
|
||||
|
||||
Default: `1`
|
||||
|
||||
#### `engine_args.request_batch_max_wait_ms`
|
||||
|
||||
The maximum time, in milliseconds, that a diffusion request-mode stage may wait
|
||||
before the first `schedule()` of a new scheduler wave so compatible requests can
|
||||
accumulate for request-level batching. This only applies to diffusion pipelines
|
||||
that support request-level batching with `step_execution` disabled.
|
||||
|
||||
Use this together with `max_num_seqs > 1` for bursty serving traffic. `0`
|
||||
disables admission waiting and preserves the lowest first-request latency.
|
||||
For diffusion request-level batching tuning, see
|
||||
[Request-Level Batching](../user_guide/diffusion/request_batching.md).
|
||||
|
||||
Default: `0.0`
|
||||
|
||||
### `engine_args`
|
||||
|
||||
Engine arguments for configuring the LLM engine, diffusion engine, or other engine types used by this stage.
|
||||
|
||||
@@ -67,7 +67,7 @@ Through five levels (L1-L5) and common (Common) specifications, the system clari
|
||||
<tr>
|
||||
<td><strong>L2</strong><br>(E2E across models & GPU-required UT)</td>
|
||||
<td>Online (basic deployment scenarios):<br>dummy, normal inference function (output format, stream), some instance startup UT</td>
|
||||
<td>High-priority models + online basic scenarios; request success, non-empty output, format match (no Whisper/accuracy)</td>
|
||||
<td>High-priority models + online basic scenarios + request success validation</td>
|
||||
<td>High-priority features (using random lightweight models)</td>
|
||||
<td>High-priority interfaces (using random lightweight models)</td>
|
||||
<td><code>core_model and hardware_test(H100, L4, etc.) and omni/tts/diffusion</code></td>
|
||||
@@ -89,7 +89,7 @@ Through five levels (L1-L5) and common (Common) specifications, the system clari
|
||||
<tr>
|
||||
<td><strong>L3</strong><br>(Important Perf & Integration & Accuracy)</td>
|
||||
<td>Online & Offline (multiple deployment scenarios):<br>real model, normal inference function, normal accuracy</td>
|
||||
<td>High/medium-priority models + key online/offline scenarios; real weights, Whisper/similarity, preset voice gender, basic accuracy</td>
|
||||
<td>High/medium-priority models with real weights + online/offline key scenarios + basic accuracy validation</td>
|
||||
<td>Medium-priority features (using random lightweight models)</td>
|
||||
<td>Medium-priority interfaces (using random lightweight models)</td>
|
||||
<td><code>advanced_model and hardware_test(H100, L4, etc.) and omni/tts/diffusion</code></td>
|
||||
@@ -113,9 +113,9 @@ Through five levels (L1-L5) and common (Common) specifications, the system clari
|
||||
<tr>
|
||||
<td><strong>L4</strong><br>(Perf & Integration & Accuracy)</td>
|
||||
<td>Online: full functional scenarios + performance test + doc test + accuracy test</td>
|
||||
<td>High-priority models: function, performance, accuracy, and doc testing<br>Medium-priority models: function and doc testing</td>
|
||||
<td>Low-priority features (using real weights)</td>
|
||||
<td>Low-priority interfaces (using real weights)</td>
|
||||
<td>High-priority models: function, performance, accuracy, and doc testing<br>Medium/low-priority models: function and doc testing</td>
|
||||
<td>Low-priority features (using random lightweight models)</td>
|
||||
<td>Low-priority interfaces (using random lightweight models)</td>
|
||||
<td><code>full_model and hardware_test(H100, L4, etc.) and omni/tts/diffusion</code></td>
|
||||
<td><3 hour</td>
|
||||
<td>
|
||||
@@ -144,7 +144,7 @@ Through five levels (L1-L5) and common (Common) specifications, the system clari
|
||||
<tr>
|
||||
<td><strong>L5</strong><br>(Stability & Reliability)</td>
|
||||
<td>Online: long-term stability test + reliability test</td>
|
||||
<td>Long-term stability and reliability testing for high-priority models<br>Low-priority models: function and doc testing</td>
|
||||
<td>Long-term stability and reliability testing for high-priority models</td>
|
||||
<td>/</td>
|
||||
<td>Invalid-parameter validation for high-priority interfaces</td>
|
||||
<td><code>slow and hardware_test(H100, L4, etc.) and omni/tts/diffusion</code></td>
|
||||
@@ -155,7 +155,7 @@ Through five levels (L1-L5) and common (Common) specifications, the system clari
|
||||
/tests/dfx/stability/tests/test_wan22.json<br>
|
||||
<strong>Reliability:</strong><br>
|
||||
tests/dfx/reliability/test_reliability_{model_key}.py<br>
|
||||
(e.g. <code>test_reliability_qwen3_omni.py</code>, <code>test_reliability_wan22.py</code>, <code>test_reliability_hunyuan_image.py</code>, <code>test_reliability_voxcpm2.py</code>)
|
||||
(e.g. <code>test_reliability_qwen3_omni.py</code>, <code>test_reliability_wan22.py</code>)
|
||||
</td>
|
||||
<td>
|
||||
<a href="#chapter-4-l5-level-testing-stability-and-reliability-testing">Chapter 4</a><br>
|
||||
@@ -446,11 +446,7 @@ L2 level testing builds upon L1 by introducing GPU resources and verifying that
|
||||
- ***Time Cost***: Execution time is controlled within ***15 minutes*** to ensure fast feedback.
|
||||
- ***L2 (Basic End-to-End Testing)***:
|
||||
- - ***Scope***: Covers two basic deployment scenarios: `online` (serving) and `offline` (inference).
|
||||
- ***Focus***: Uses `dummy` weights (via deploy YAML patching at `core_model`) or lightweight real models to verify that the entire chain from request input to result output works normally, including output data structure, streaming (stream) support, and **cheap payload checks** at `--run-level core_model` (see below). Also includes some unit tests that require launching independent service instances.
|
||||
- ***L2 response validation (`core_model`)***: Implemented in `tests/helpers/assertions.py` and invoked by `OpenAIClientHandler` based on `--run-level`. At L2 we require **request success** plus minimal output sanity—not full accuracy:
|
||||
- **Speech / TTS** (`assert_audio_speech_response`): decoded audio must be present and non-empty (or exceed `min_audio_bytes` when set in `request_config`); `response_format` must match the returned content-type (e.g. `wav`, `pcm`). Whisper transcript similarity, PCM HNR, and preset-voice gender checks run only at L3+.
|
||||
- **Diffusion** (`assert_diffusion_response`): at least one non-empty image, video, or audio artifact. Resolution/frame-count parity and other deep checks run only at L3+.
|
||||
- **Omni multimodal** (`assert_omni_response`): L2 asserts successful completion; keyword, transcript, and cross-modal similarity checks run only at L3+.
|
||||
- ***Focus***: Uses `dummy` models or lightweight real models to verify that the entire chain from request input to result output works normally, including output data structure, streaming (stream) support, etc. Also includes some unit tests that require launching independent service instances.
|
||||
- ***Characteristic***: Requires ***GPU*** resources to perform model computations.
|
||||
|
||||
### 1.3 Test Directory and Execution Files
|
||||
@@ -559,11 +555,6 @@ L3 level testing executes after code is merged into the main branch. Its core pu
|
||||
- 1. ***Inference Functionality***: Ensures real models can perform forward computation normally and return results.
|
||||
2. ***Accuracy Compliance***: Verifies that the model's evaluation metrics (e.g., accuracy) meet the expected baseline, preventing code changes from introducing accuracy issues.
|
||||
3. ***Important Performance***: Verifies whether performance (e.g., P99 latency, throughput) in core scenarios meets preset thresholds.
|
||||
- ***L3 response validation (`advanced_model`)***: At `--run-level advanced_model`, `tests/helpers/assertions.py` adds semantic checks on top of L2 payload gates (see Chapter 1):
|
||||
- **Speech / TTS** (`assert_audio_speech_response`): Whisper transcript of returned audio vs `request_config["input"]` (cosine similarity > 0.9 when input text is set); optional `min_audio_bytes` floor; PCM harmonic-to-noise ratio when `response_format` is `pcm`; **preset voice gender** via `_assert_preset_voice_gender_from_audio` when `voice` matches a known preset in `_PRESET_VOICE_GENDER_MAP` (pitch/F0-based classifier; skipped for unknown voices or `pcm` output).
|
||||
- **Omni multimodal** (`assert_omni_response`): non-empty text/audio outputs; `key_words` in transcript or text; text–audio similarity / containment; preset **`speaker`** gender check (same helper as TTS).
|
||||
- **Diffusion** (`assert_*_diffusion_response`): image/video dimension and frame-count parity with request parameters where configured.
|
||||
- **Accuracy suites**: pixel/video similarity and metric baselines under `/tests/e2e/accuracy/` (separate from inline helper assertions).
|
||||
|
||||
### 2.3 Test Directory and Execution Files
|
||||
|
||||
@@ -591,9 +582,9 @@ L3 level testing executes after code is merged into the main branch. Its core pu
|
||||
|
||||
**Explanation**:
|
||||
|
||||
@pytest.mark.advanced_model: Marks the test as L3 merge level (`--run-level advanced_model`): real weights, Whisper/similarity/keyword checks, diffusion deep checks, and **preset voice/speaker gender** validation where applicable. @pytest.mark.full_model: Marks L4 nightly-only suites (e.g. `test_*_expansion.py`, doc examples).
|
||||
@pytest.mark.advanced_model: Marks the test as L3 merge level, indicating deep validation with real models. @pytest.mark.full_model: Marks L4 nightly-only suites (e.g. `test_*_expansion.py`, doc examples).
|
||||
|
||||
@pytest.mark.core_model: Marks the test as L1 or L2 level. At `--run-level core_model`, validation is limited to request success plus cheap payload checks (e.g. non-empty audio bytes, response format/content-type, non-empty diffusion outputs)—not Whisper, keyword, or accuracy gates. Deploy YAML may use `load_format: dummy` for fast PR feedback.
|
||||
@pytest.mark.core_model: Marks the test as L1 or L2 level, indicating that this test case validates the basic functionality of the core model. It uses mock weights and only checks if the relevant interface functions correctly.
|
||||
|
||||
@pytest.mark.parametrize: A parameterization decorator that allows abstracting test data into parameters, enabling reuse of the same test logic across different data configurations. indirect=True indicates that parameters will be passed to the fixture for processing.
|
||||
|
||||
@@ -689,7 +680,7 @@ L3 level testing executes after code is merged into the main branch. Its core pu
|
||||
|
||||
**Single Request**: The comment clearly states this is a single-request completion test. For concurrent testing, it can be extended to multiple requests using request_num = n.
|
||||
|
||||
**Implicit Validation**: `send_omni_request`, `send_audio_speech_request`, and `send_diffusion_request` call `assert_*_response` helpers with the session `--run-level`. At `core_model`, checks are limited to success plus cheap payload sanity (non-empty audio/media, format/content-type, optional `min_audio_bytes`). At `advanced_model` and `full_model`, deep validation adds Whisper transcripts, keyword/similarity, diffusion dimension checks, PCM HNR, etc.
|
||||
**Implicit Validation**: The `send_omni_request` and `send_diffusion_request` methods internally includes validation logic dynamically selected based on the --run-level parameter: core_model performs basic validation, while advanced_model and full_model perform deep validation.
|
||||
|
||||
**Audio output debugging**: Deep validation may transcribe returned audio via `convert_audio_bytes_to_text` (Whisper). If an audio keyword or text–audio similarity assertion fails, set `VLLM_OMNI_KEEP_REQUEST_MEDIA=1` before running pytest to keep the intermediate WAV files for inspection (see [Test helper environment variables](#test-helper-environment-variables)).
|
||||
|
||||
@@ -750,16 +741,14 @@ L5 level testing focuses on the performance of model services under ***long-runn
|
||||
### 4.2 Testing Content and Scope
|
||||
|
||||
- ***Long-term Stability (Stability) Testing***: Uses JSON under `tests/dfx/stability/tests/` (for example `test_qwen3_omni.json` and `test_wan22.json`) to run the service under moderate load for an extended period (e.g., over 12 hours), monitoring whether metrics like memory/VRAM usage, response time, and throughput degrade over time, and whether the service process remains stable.
|
||||
- ***Reliability Testing***: Uses pytest suites under `tests/dfx/reliability/` to inject controlled faults against a **live** `vllm_omni serve` instance (same **`omni_server` / `omni_server_function`** fixture style as E2E). Current suites emphasize **GPU memory pressure** (CUDA sidecar “memory hog”), **worker / runtime process kill** (`SIGKILL` on `VLLM::Worker` for Qwen3-Omni, `multiprocessing.spawn` for Wan2.2 video workers, or `vLLM-Omni::` for HunyuanImage DiT workers), **large multimodal chat**, **`/v1/videos`**, **`/v1/images/generations`**, or **`/v1/audio/speech`** jobs under OOM, **`/health` → 503** and **fast-fail / non-hanging concurrent** requests after kill, and **OpenAI-style 5xx error contracts** (e.g. text vs text+audio under OOM). **Post-fault recovery** checks exist where enabled (some cases may be `skip` while issues are tracked). See the Reliability `<details>` block in Section 4.4 for file-level responsibilities and CI markers (`slow`, `hardware_test`, POSIX-only kill).
|
||||
- ***Reliability Testing***: Uses pytest suites under `tests/dfx/reliability/` to inject controlled faults against a **live** `vllm_omni serve` instance (same **`omni_server` / `omni_server_function`** fixture style as E2E). Current suites emphasize **GPU memory pressure** (CUDA sidecar “memory hog”), **worker / runtime process kill** (`SIGKILL` on `VLLM::Worker` for Qwen3-Omni or `multiprocessing.spawn` for Wan2.2 video workers), **large multimodal chat** or **`/v1/videos`** jobs under OOM, **`/health` → 503** and **fast-fail / non-hanging concurrent** requests after kill, and **OpenAI-style 5xx error contracts** (e.g. text vs text+audio under OOM). **Post-fault recovery** checks exist where enabled (some cases may be `skip` while issues are tracked). See the Reliability `<details>` block in Section 4.4 for file-level responsibilities and CI markers (`slow`, `hardware_test`, POSIX-only kill).
|
||||
|
||||
### 4.3 Test Directory and Execution Files
|
||||
|
||||
- ***Stability Test Configuration***: `tests/dfx/stability/tests/test_qwen3_omni.json`, `tests/dfx/stability/tests/test_wan22.json` (one JSON per model / runner family)
|
||||
- ***Reliability Test Suite*** (`tests/dfx/reliability/`):
|
||||
- `test_reliability_qwen3_omni.py` — Qwen3-Omni chat / multimodal reliability (GPU OOM, process kill, recovery, error contract under `--async-chunk` vs default).
|
||||
- `test_reliability_wan22.py` — Wan2.2 I2V video API reliability (`/v1/videos` under OOM and process kill, recovery).
|
||||
- `test_reliability_hunyuan_image.py` — HunyuanImage-3.0-Instruct DiT-only reliability (`/v1/images/generations` under OOM and process kill; deploy `hunyuan_image3_dit.yaml`, H100 × 4).
|
||||
- `test_reliability_voxcpm2.py` — VoxCPM2 TTS reliability (`/v1/audio/speech` under OOM and process kill; deploy `voxcpm2.yaml`, L4 × 1).
|
||||
- `test_reliability_wan22.py` — Wan2.2 T2V video API reliability (`/v1/videos` under OOM and process kill, recovery).
|
||||
- `helpers.py` — Shared primitives used by current suites: raw HTTP probes for `/v1/chat/completions` and `/health`, OpenAI-style error parsing, GPU OOM sidecar (`inject_gpu_oom` / `stop_gpu_oom_hogs`), and `pgrep`-based process-kill injector construction (`make_process_kill_fault_injector`).
|
||||
- `conftest.py` — `fault_injector` and `omni_server_after_fault` / `omni_server_after_fault_function` fixtures to run a callable **after** the server is ready.
|
||||
- `README.md` — Short local run commands for this directory.
|
||||
@@ -857,33 +846,22 @@ Reliability tests are **short fault-injection** integration runs (L5 **(b)** in
|
||||
| `helpers.py` | Shared helpers used by current reliability suites: raw `POST`/`GET` probes (`/v1/chat/completions`, `/health`), OpenAI error parsing (`extract_openai_error_contract_from_bytes`), GPU OOM sidecar lifecycle (`inject_gpu_oom`, `stop_gpu_oom_hogs`), and process-kill injector builder (`make_process_kill_fault_injector`). |
|
||||
| `conftest.py` | Pytest fixtures: indirect `fault_injector`, `omni_server_after_fault` / `omni_server_after_fault_function` (run injector after server is ready, then yield server). |
|
||||
| `test_reliability_qwen3_omni.py` | Qwen3-Omni: OOM vs **text vs text+audio** error contract, large multimodal chat under OOM, concurrent pressure, **SIGKILL** on `VLLM::Worker`, `/health` → 503 + fast-fail + concurrent chat; optional OOM recovery scenario (may be skipped while tracked in issues). |
|
||||
| `test_reliability_wan22.py` | Wan2.2 I2V: large `/v1/videos` under OOM, **SIGKILL** on `multiprocessing.spawn` chain, health / fast-fail / concurrent video requests; optional recovery test (may be skipped). |
|
||||
| `test_reliability_hunyuan_image.py` | HunyuanImage DiT-only: large `/v1/images/generations` under OOM, **SIGKILL** on `vLLM-Omni::` workers and serve/tree targets, health / fast-fail / concurrent image requests; some OOM/recovery cases may be skipped while tracked in issues. |
|
||||
| `test_reliability_voxcpm2.py` | VoxCPM2: `/v1/audio/speech` under OOM (error contract), **SIGKILL** on `VLLM::` workers and serve/tree targets, health / fast-fail / concurrent speech requests; some OOM cases may be skipped while tracked in issues. |
|
||||
| `test_reliability_wan22.py` | Wan2.2 T2V: large `/v1/videos` under OOM, **SIGKILL** on `multiprocessing.spawn` chain, health / fast-fail / concurrent video requests; optional recovery test (may be skipped). |
|
||||
| `README.md` | Minimal run / collect examples. |
|
||||
|
||||
#### Parametrization and markers
|
||||
|
||||
- Each test module defines a **`RELIABILITY_SCENARIOS`** list (`test_name`, `server_params`: model, `stage_config_name` or diffusion `server_args`, etc.). **`create_reliability_omni_server_params()`** in `tests/dfx/conftest.py` resolves stage paths (including XPU substitutions where applicable) and builds **`OmniServerParams`** lists consumed by **`@pytest.mark.parametrize(..., indirect=True)`** on `omni_server` or `omni_server_function`.
|
||||
- Cases are tagged **`@pytest.mark.slow`** for weekly / selective CI. GPU-heavy suites use **`@hardware_test(res={"cuda": "H100"}, num_cards=...)`** or **`@hardware_test(res={"cuda": "L4"}, num_cards=1)`** (Qwen3-Omni **2**× H100; Wan2.2 **1**× H100; HunyuanImage DiT **4**× H100; VoxCPM2 **1**× L4).
|
||||
- Cases are tagged **`@pytest.mark.slow`** for weekly / selective CI. GPU-heavy suites use **`@hardware_test(res={"cuda": "H100"}, num_cards=...)`** (Qwen3-Omni paths often require **2** cards; Wan2.2 video paths **1** card).
|
||||
- **Process-kill** tests use **`@pytest.mark.skipif(os.name == "nt", ...)`** because injection uses POSIX **`pgrep` / `kill`**.
|
||||
|
||||
#### CI trigger
|
||||
|
||||
Weekly Buildkite (`.buildkite/test-weekly.yml`) runs one step per model suite (trigger: `WEEKLY=1` or PR label `weekly-test`), for example:
|
||||
|
||||
| Buildkite step | Test file | CI hardware |
|
||||
| -------------- | --------- | ----------- |
|
||||
| Reliability Test - qwen3-omni | `test_reliability_qwen3_omni.py` | H100 × 2 (`mithril-h100-pool`) |
|
||||
| Reliability Test - wan22 | `test_reliability_wan22.py` | H100 × 2 (`mithril-h100-pool`) |
|
||||
| Reliability Test - hunyuan-image | `test_reliability_hunyuan_image.py` | H100 × 4 (`mithril-h100-pool`) |
|
||||
| Reliability Test - voxcpm2 | `test_reliability_voxcpm2.py` | L4 × 1 (`gpu_1_queue`) |
|
||||
Weekly Buildkite (`.buildkite/test-weekly.yml`) runs, for example:
|
||||
|
||||
```bash
|
||||
pytest -s -v tests/dfx/reliability/test_reliability_qwen3_omni.py -m "slow"
|
||||
pytest -s -v tests/dfx/reliability/test_reliability_wan22.py -m "slow"
|
||||
pytest -s -v tests/dfx/reliability/test_reliability_hunyuan_image.py -m "slow"
|
||||
pytest -s -v tests/dfx/reliability/test_reliability_voxcpm2.py -m "slow"
|
||||
```
|
||||
|
||||
#### Local commands
|
||||
@@ -892,8 +870,6 @@ pytest -s -v tests/dfx/reliability/test_reliability_voxcpm2.py -m "slow"
|
||||
pytest --collect-only tests/dfx/reliability
|
||||
pytest -s -v tests/dfx/reliability/test_reliability_qwen3_omni.py -m slow
|
||||
pytest -s -v tests/dfx/reliability/test_reliability_wan22.py -m slow
|
||||
pytest -s -v tests/dfx/reliability/test_reliability_hunyuan_image.py -m slow
|
||||
pytest -s -v tests/dfx/reliability/test_reliability_voxcpm2.py -m slow
|
||||
```
|
||||
|
||||
#### Adding a new model suite
|
||||
@@ -906,7 +882,7 @@ pytest -s -v tests/dfx/reliability/test_reliability_voxcpm2.py -m slow
|
||||
</details>
|
||||
|
||||
- - ***Stability***: `pytest -s -v tests/dfx/stability/scripts/test_stability_qwen3_omni.py` or `pytest -s -v tests/dfx/stability/scripts/test_stability_wan22.py` (or add `test_stability_<model>.py` alongside a matching JSON config)
|
||||
- ***Reliability***: `pytest -s -v tests/dfx/reliability/test_reliability_<model>.py -m slow` (current suites: `qwen3_omni`, `wan22`, `hunyuan_image`, `voxcpm2`; add `test_reliability_<suite>.py` and a matching step in `.buildkite/test-weekly.yml` for new models)
|
||||
- ***Reliability***: `pytest -s -v tests/dfx/reliability/test_reliability_qwen3_omni.py -m slow` and/or `pytest -s -v tests/dfx/reliability/test_reliability_wan22.py -m slow` (add `test_reliability_<suite>.py` for new models)
|
||||
|
||||
## Summary
|
||||
|
||||
|
||||
@@ -96,21 +96,15 @@ Our test scripts use the pytest framework. First, please use `git clone https://
|
||||
# Stability: Wan2.2 (v1/videos diffusion benchmark loop)
|
||||
pytest -s -v dfx/stability/scripts/test_stability_wan22.py -m slow
|
||||
|
||||
# Reliability: Qwen3-Omni (H100 × 2)
|
||||
# Reliability: Qwen3-Omni
|
||||
pytest -s -v dfx/reliability/test_reliability_qwen3_omni.py -m slow
|
||||
|
||||
# Reliability: Wan2.2 (H100)
|
||||
# Reliability: Wan2.2
|
||||
pytest -s -v dfx/reliability/test_reliability_wan22.py -m slow
|
||||
|
||||
# Reliability: HunyuanImage DiT (H100 × 4)
|
||||
pytest -s -v dfx/reliability/test_reliability_hunyuan_image.py -m slow
|
||||
|
||||
# Reliability: VoxCPM2 (L4)
|
||||
pytest -s -v dfx/reliability/test_reliability_voxcpm2.py -m slow
|
||||
|
||||
```
|
||||
|
||||
The latest L5 CI jobs (reliability + invalid-parameter weekly steps) are in [test-weekly.yml](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/test-weekly.yml).
|
||||
The latest L5 commands for CI can be found in the [pipeline](https://github.com/vllm-project/vllm-omni/blob/main/.buildkite/test-ready.yml).
|
||||
|
||||
You can find more information about markers in the documentation: [marker doc](./tests_markers.md)
|
||||
|
||||
|
||||
@@ -349,14 +349,12 @@ class YourModelPipeline(nn.Module):
|
||||
- def __call__(
|
||||
+ def forward(
|
||||
self,
|
||||
+ req: DiffusionRequestBatch, # ← Add request-batch parameter here
|
||||
+ req: OmniDiffusionRequest, # ← Add request parameter here
|
||||
- ):
|
||||
+ ) -> list[DiffusionOutput]: # ← Add return type
|
||||
+ ) -> DiffusionOutput: # ← Add return type
|
||||
```
|
||||
|
||||
[`OmniDiffusionRequest`](https://docs.vllm.ai/projects/vllm-omni/en/latest/api/vllm_omni/diffusion/request/#vllm_omni.diffusion.request.OmniDiffusionRequest) is a dataclass that contains one **prompt** and the **sampling parameters** [`OmniDiffusionSamplingParams`](https://docs.vllm.ai/projects/vllm-omni/en/latest/api/vllm_omni/inputs/data/#vllm_omni.inputs.data.OmniDiffusionSamplingParams) for one logical diffusion request. It also contains a request_id for other components to trace this request and its outputs. Before pipeline execution, the runner wraps one or more independent requests into `DiffusionRequestBatch`.
|
||||
|
||||
[`DiffusionRequestBatch`](https://docs.vllm.ai/projects/vllm-omni/en/latest/api/vllm_omni/diffusion/worker/request_batch/#vllm_omni.diffusion.worker.request_batch.DiffusionRequestBatch) exposes compatibility properties such as `prompts`, `sampling_params`, and `request_id`. Pipelines that can execute the whole request batch in one forward pass should set `supports_request_batch = True`; other pipelines still receive a single-request batch and return a one-element output list.
|
||||
[`OmniDiffusionRequest`](https://docs.vllm.ai/projects/vllm-omni/en/latest/api/vllm_omni/diffusion/request/#vllm_omni.diffusion.request.OmniDiffusionRequest) is a dataclass that contains the **prompts** and **sampling parameters** [`OmniDiffusionSamplingParams`](https://docs.vllm.ai/projects/vllm-omni/en/latest/api/vllm_omni/inputs/data/#vllm_omni.inputs.data.OmniDiffusionSamplingParams) for the diffusion pipeline execution. It also contains a request_id for other components to trace this request and its outputs.
|
||||
|
||||
See some parameters in `OmniDiffusionSamplingParams` as follows:
|
||||
|
||||
@@ -369,36 +367,35 @@ See some parameters in `OmniDiffusionSamplingParams` as follows:
|
||||
**Extract parameters from request:**
|
||||
|
||||
```python
|
||||
from vllm_omni.diffusion.request import OmniDiffusionRequest
|
||||
from vllm_omni.diffusion.data import DiffusionOutput
|
||||
from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch
|
||||
|
||||
def forward(
|
||||
self,
|
||||
req: DiffusionRequestBatch,
|
||||
) -> list[DiffusionOutput]:
|
||||
# Extract prompts from the request batch
|
||||
prompts = [
|
||||
p if isinstance(p, str) else (p.get("prompt") or "")
|
||||
for p in req.prompts
|
||||
]
|
||||
req: OmniDiffusionRequest,
|
||||
) -> DiffusionOutput:
|
||||
# Extract prompts from request
|
||||
if req.prompts is not None:
|
||||
prompt = [
|
||||
p if isinstance(p, str) else (p.get("prompt") or "")
|
||||
for p in req.prompts
|
||||
]
|
||||
|
||||
# Extract common sampling parameters
|
||||
# Extract sampling parameters
|
||||
sampling_params = req.sampling_params
|
||||
num_inference_steps = sampling_params.num_inference_steps or 50
|
||||
guidance_scale = sampling_params.guidance_scale or 7.5
|
||||
height = sampling_params.height or (self.default_sample_size * self.vae_scale_factor)
|
||||
width = sampling_params.width or (self.default_sample_size * self.vae_scale_factor)
|
||||
|
||||
# For image editing pipelines, extract media from each prompt dict
|
||||
input_images = []
|
||||
for p in req.prompts:
|
||||
multi_modal_data = p.get("multi_modal_data", {}) if isinstance(p, dict) else {}
|
||||
input_images.append(multi_modal_data.get("image"))
|
||||
# For image editing pipelines, extract images from multi_modal_data
|
||||
if hasattr(req, 'multi_modal_data') and req.multi_modal_data:
|
||||
input_images = req.multi_modal_data.get('image', [])
|
||||
|
||||
# ... rest of generation logic
|
||||
```
|
||||
|
||||
For an image editing model, the request `prompt` can be a dict like:
|
||||
For an image editing model, an example `OmniDiffusionRequest` is like:
|
||||
```python
|
||||
{
|
||||
"prompt": "turn this cat to a dog",
|
||||
@@ -475,12 +472,12 @@ def get_your_model_pre_process_func(
|
||||
def pre_process_func(
|
||||
request: OmniDiffusionRequest,
|
||||
):
|
||||
prompt = request.prompt
|
||||
multi_modal_data = prompt.get("multi_modal_data", {}) if not isinstance(prompt, str) else None
|
||||
raw_image = multi_modal_data.get("image", None) if multi_modal_data is not None else None
|
||||
# image pre-processing
|
||||
# after pre-processing, update the request attributes
|
||||
...
|
||||
for i, prompt in enumerate(request.prompts):
|
||||
multi_modal_data = prompt.get("multi_modal_data", {}) if not isinstance(prompt, str) else None
|
||||
raw_image = multi_modal_data.get("image", None) if multi_modal_data is not None else None
|
||||
# image pre-processing
|
||||
# after pre-processing, update the request attributes
|
||||
...
|
||||
return request
|
||||
|
||||
return pre_process_func
|
||||
@@ -926,11 +923,11 @@ When implementing a new pipeline, avoid putting all logic inside a single functi
|
||||
|
||||
For example:
|
||||
```
|
||||
def forward(self, req: DiffusionRequestBatch) -> list[DiffusionOutput]:
|
||||
def forward(self, req: OmniDiffusionRequest):
|
||||
prompt_embeds = self.encode_prompt(req)
|
||||
latents = self.diffuse(prompt_embeds, req)
|
||||
images = self.vae.decode(latents)
|
||||
return [DiffusionOutput(output=images)]
|
||||
return DiffusionOutput(output=images)
|
||||
```
|
||||
This allows the timing utility to measure each stage (e.g., encode_prompt, diffuse, vae.decode) separately and helps identify performance bottlenecks more easily.
|
||||
|
||||
|
||||
@@ -475,64 +475,92 @@ thinker_hidden_states = output.multimodal_output["24"]
|
||||
|
||||
### Implementation Example
|
||||
|
||||
Create stage transition processors in `vllm_omni/model_executor/stage_input_processors/your_model_name.py`. Each inter-stage edge should provide a **coherent processor set** rather than a single monolithic function:
|
||||
|
||||
| Suffix | Role | Runs when |
|
||||
|--------|------|-----------|
|
||||
| `*_full_payload` | Worker-side payload producer | `async_chunk=false`; accumulates tensors and ships via connector |
|
||||
| `*_async_chunk` | Scheduler-side streaming producer | `async_chunk=true`; emits per-chunk payloads |
|
||||
| `*_token_only` | Orchestrator placeholder builder | `async_chunk=false`; allocates downstream prompt slots only |
|
||||
Create stage transition processors in `vllm_omni/model_executor/stage_input_processors/your_model_name.py`:
|
||||
|
||||
```python
|
||||
# qwen3_omni.py (Thinker → Talker, non-async path)
|
||||
# qwen3_omni.py
|
||||
|
||||
def thinker2talker_token_only(
|
||||
source_outputs: list[Any],
|
||||
def thinker2talker(
|
||||
stage_list: list[Any],
|
||||
engine_input_source: list[int],
|
||||
prompt: OmniTokensPrompt | TextPrompt | None = None,
|
||||
requires_multimodal_data: bool = False,
|
||||
streaming_context: Any | None = None,
|
||||
) -> list[OmniTokensPrompt]:
|
||||
"""Allocate talker prefill slots; bulk tensors arrive via the connector."""
|
||||
...
|
||||
"""
|
||||
Process thinker outputs to create talker inputs.
|
||||
|
||||
Args:
|
||||
stage_list: List of stage objects
|
||||
engine_input_source: Source stage IDs (typically [0] for thinker)
|
||||
prompt: Original prompt data
|
||||
|
||||
Returns:
|
||||
List of OmniTokensPrompt for talker stage
|
||||
"""
|
||||
source_stage_id = engine_input_source[0]
|
||||
thinker_outputs = stage_list[source_stage_id].engine_outputs
|
||||
talker_inputs = []
|
||||
|
||||
for thinker_output in thinker_outputs:
|
||||
output = thinker_output.outputs[0]
|
||||
# Extract thinker embeddings and hidden states
|
||||
thinker_prefill_embeddings = output.multimodal_output["0"].float().clone().detach().cuda()
|
||||
thinker_hidden_states = output.multimodal_output["24"].float().clone().detach().cuda()
|
||||
|
||||
info = {
|
||||
"thinker_prefill_embeddings": thinker_prefill_embeddings,
|
||||
"thinker_hidden_states": thinker_hidden_states,
|
||||
"thinker_sequences": thinker_output.prompt_token_ids + output.token_ids,
|
||||
"thinker_input_ids": thinker_output.prompt_token_ids,
|
||||
}
|
||||
|
||||
talker_inputs.append(
|
||||
OmniTokensPrompt(
|
||||
prompt_token_ids=[0] * computed_length,
|
||||
additional_information=info,
|
||||
multi_modal_data=None,
|
||||
)
|
||||
)
|
||||
|
||||
return talker_inputs
|
||||
|
||||
|
||||
def thinker2talker_full_payload(
|
||||
transfer_manager: Any,
|
||||
pooling_output: dict[str, Any],
|
||||
request: OmniEngineCoreRequest,
|
||||
) -> dict[str, Any] | None:
|
||||
"""Pack accumulated thinker hidden states into OmniPayload for stage-1."""
|
||||
...
|
||||
def talker2code2wav(
|
||||
stage_list: list[Any],
|
||||
engine_input_source: list[int],
|
||||
prompt: OmniTokensPrompt | TextPrompt | None = None,
|
||||
requires_multimodal_data: bool = False,
|
||||
) -> list[OmniTokensPrompt]:
|
||||
"""
|
||||
Process talker outputs to create code2wav inputs.
|
||||
"""
|
||||
source_stage_id = engine_input_source[0]
|
||||
talker_outputs = stage_list[source_stage_id].engine_outputs
|
||||
code2wav_inputs = []
|
||||
|
||||
for talker_output in talker_outputs:
|
||||
output = talker_output.outputs[0]
|
||||
# Extract codec codes
|
||||
codec_codes = (
|
||||
output.multimodal_output["code_predictor_codes"]
|
||||
.to(torch.long)
|
||||
.transpose(0, 1)
|
||||
.cpu()
|
||||
.to(torch.long)
|
||||
.reshape(-1)
|
||||
.tolist()
|
||||
)
|
||||
|
||||
def thinker2talker_async_chunk(
|
||||
transfer_manager: Any,
|
||||
multimodal_output: OmniPayload | dict[str, Any],
|
||||
request: OmniEngineCoreRequest,
|
||||
is_finished: bool = False,
|
||||
) -> OmniPayloadStruct | None:
|
||||
"""Stream thinker rows to talker while async_chunk is enabled."""
|
||||
...
|
||||
code2wav_inputs.append(
|
||||
OmniTokensPrompt(
|
||||
prompt_token_ids=codec_codes,
|
||||
multi_modal_data=None,
|
||||
)
|
||||
)
|
||||
|
||||
return code2wav_inputs
|
||||
```
|
||||
|
||||
Wire these in `pipeline.py`:
|
||||
|
||||
```python
|
||||
StagePipelineConfig(
|
||||
stage_id=0,
|
||||
custom_process_next_stage_input_func=f"{_PROC}.thinker2talker_full_payload",
|
||||
async_chunk_process_next_stage_input_func=f"{_PROC}.thinker2talker_async_chunk",
|
||||
...
|
||||
),
|
||||
StagePipelineConfig(
|
||||
stage_id=1,
|
||||
sync_process_input_func=f"{_PROC}.thinker2talker_token_only",
|
||||
...
|
||||
),
|
||||
```
|
||||
|
||||
Do **not** add a no-suffix `thinker2talker` when a `sync_process_input_func` is already declared — `_select_processor_funcs()` always prefers the `*_token_only` hook in non-async mode, so the bare function would never run. See `docs/design/rfc_stage_input_processors_refactor.md` for the full contract.
|
||||
|
||||
## Testing
|
||||
|
||||
For comprehensive testing guidelines, please refer to the [Test File Structure and Style Guide](../ci/tests_style.md).
|
||||
|
||||
@@ -286,54 +286,3 @@ Recommended viewers:
|
||||
- Nsight Systems GUI for CUDA kernel timelines
|
||||
|
||||
For upstream background on the underlying vLLM profiling infrastructure, see the [vLLM profiling guide](https://docs.vllm.ai/en/stable/contributing/profiling/).
|
||||
|
||||
## 7. Orchestrator Monitor
|
||||
|
||||
> **Warning:** Diagnostic only. Adds lightweight counters on the orchestrator poll loop and per-replica queue depth. Disable in production unless you are actively debugging orchestrator saturation.
|
||||
|
||||
Multi-stage omni pipelines route client-facing outputs and inter-stage connector traffic through a single-process orchestrator. When that loop is saturated (for example by large multimodal payloads on the poll path), TTFT and stability can degrade even when GPU stages are healthy. The orchestrator monitor records time-series signals to confirm whether the orchestrator or downstream stage queues are the bottleneck.
|
||||
|
||||
Enable on the stage-0 (orchestrator) process:
|
||||
|
||||
```bash
|
||||
vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091 \
|
||||
--stage-id 0 \
|
||||
--enable-orch-monitor
|
||||
```
|
||||
|
||||
Optional output path override:
|
||||
|
||||
```bash
|
||||
export VLLM_OMNI_ORCH_MONITOR_PATH=/tmp/vllm_omni_orch_monitor.json
|
||||
```
|
||||
|
||||
If unset, the monitor writes `./vllm_omni_orch_monitor_<MMDDHHMM>.json` at shutdown.
|
||||
|
||||
### Recorded metrics
|
||||
|
||||
Each 1-second window records:
|
||||
|
||||
| Series | Description |
|
||||
|---|---|
|
||||
| `windows.duration_s` | Wall time covered by the window |
|
||||
| `windows.loop_idle` / `windows.loop_active` | Orchestrator poll-loop iterations with no work vs. active forwarding |
|
||||
| `replicas.<stage,replica>.outputs_queue_size` | MP client `outputs_queue` backlog for that replica |
|
||||
| `replicas.<stage,replica>.inflight` | Requests currently bound/routed to the replica |
|
||||
|
||||
On shutdown the server also logs a short summary (`loop_active_pct`, per-replica queue averages/maxima).
|
||||
|
||||
### Relationship to other diagnostics
|
||||
|
||||
This monitor is intentionally separate from the existing profiling tools:
|
||||
|
||||
| Tool | Scope | Output |
|
||||
|---|---|---|
|
||||
| `--enable-diffusion-pipeline-profiler` | Diffusion stage functions (`vae.decode`, `diffuse`, …) | Per-request `stage_durations` / logs |
|
||||
| `--enable-ar-profiler` | AR stage generation time | Per-request `stage_durations` |
|
||||
| `profiler_config` (`torch` / `cuda`) | GPU/CPU kernels inside a stage worker | `trace.json`, Nsight, operator tables |
|
||||
| Prometheus `/metrics` (`vllm:omni_*`) | Pipeline SLOs and cross-stage transfer | Continuous scrape endpoint |
|
||||
| `--enable-orch-monitor` | Orchestrator poll loop + replica queue backlog | Single JSON file at shutdown |
|
||||
|
||||
The orchestrator monitor does not use `torch.profiler` because the bottleneck signal lives in the orchestrator process (poll-loop duty cycle and queue depth), not inside stage workers. It complements Prometheus metrics when you need a post-run time series for a benchmark or regression run.
|
||||
|
||||
Process liveness monitors (`monitor_engine_liveness`, worker/engine-core monitors) are unrelated: they detect crashed workers, not orchestrator load.
|
||||
@@ -126,16 +126,16 @@ Call `self.diffuse` in your pipeline's forward function:
|
||||
|
||||
```python
|
||||
import torch.nn as nn
|
||||
from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch
|
||||
|
||||
class YourModelPipeline(nn.Module, CFGParallelMixin):
|
||||
def forward(
|
||||
self,
|
||||
req: DiffusionRequestBatch,
|
||||
) -> list[DiffusionOutput]:
|
||||
# Read prompts and sampling params from the request
|
||||
...
|
||||
# Encode prompts, initialize latents, get timesteps
|
||||
prompt: str,
|
||||
negative_prompt: str | None = None,
|
||||
guidance_scale: float = 3.5,
|
||||
num_inference_steps: int = 50,
|
||||
**kwargs,
|
||||
):
|
||||
# Encode prompts, Initialize latents, Get timesteps
|
||||
...
|
||||
# Run diffusion loop (calls the mixin's diffuse method)
|
||||
latents = self.diffuse(
|
||||
|
||||
@@ -38,13 +38,9 @@ With continuous batching enabled:
|
||||
The current implementation is conservative:
|
||||
|
||||
- only compatible requests are batched together
|
||||
- request-mode diffusion still runs with `max_num_seqs=1`
|
||||
- per-request progress and completion remain independent
|
||||
|
||||
Here, "continuous batching" means the step-wise path enabled by
|
||||
`step_execution=True`. Request-mode `DiffusionRequestBatch` is static
|
||||
request-level batching for one full pipeline `forward()` call; it does not
|
||||
admit or remove requests between denoise steps.
|
||||
|
||||
## Enablement
|
||||
|
||||
Use `--step-execution` as the feature gate, then increase `--max-num-seqs`
|
||||
@@ -76,7 +72,7 @@ which is built from shape-sensitive and CFG-sensitive sampling fields. This is
|
||||
the core correctness rule for batching: requests are only co-batched when they
|
||||
share the same denoise tensor contract.
|
||||
|
||||
There are three important details:
|
||||
There are two important details:
|
||||
|
||||
- `num_inference_steps` is not part of the key, so requests with different
|
||||
total step counts can still share a batch
|
||||
@@ -92,10 +88,8 @@ key also covers LoRA identity (`lora_int_id`, `lora_scale`), so requests
|
||||
targeting different adapters or scales run in separate batches and the
|
||||
worker can activate exactly one adapter per step.
|
||||
|
||||
The scheduler batching unit is one logical `OmniDiffusionRequest`. In the
|
||||
step-wise path, runtime tensor batching is represented as `StepInputBatch`. For
|
||||
request-mode prompt semantics, see
|
||||
[Request-Level Batching](../../user_guide/diffusion/request_batching.md).
|
||||
The current batching unit is one `OmniDiffusionRequest`. Requests with
|
||||
multiple prompts do not participate in batching today.
|
||||
|
||||
## Runner
|
||||
|
||||
@@ -132,15 +126,17 @@ request-local scheduler state and outputs.
|
||||
the background loop and async add-request path needed for multiple requests to
|
||||
accumulate in the scheduler.
|
||||
|
||||
When `step_execution=True`, the engine routes work through the step-wise
|
||||
executor path. The continuous batching behavior is defined by scheduler-side
|
||||
compatibility gating and runner-side `StepInputBatch` packing.
|
||||
This is supporting infrastructure, not the main design point. The batching
|
||||
behavior is defined by scheduler-side compatibility gating and runner-side
|
||||
batch packing.
|
||||
|
||||
## Current Limitations
|
||||
|
||||
- Experimental feature; use `max_num_seqs=1` for the older conservative path.
|
||||
- Only native pipelines that already support `step_execution=True`.
|
||||
- Request-mode diffusion still clamps `max_num_seqs` back to `1`.
|
||||
- Only homogeneous batches keyed by `SamplingParamsKey` are supported.
|
||||
- Multi-prompt requests are not batched.
|
||||
- `cache_backend`, KV transfer, and other request-mode extras are not wired
|
||||
into the batched step-wise path yet.
|
||||
- Future work can relax the current same-shape restriction with richer
|
||||
|
||||
@@ -1,175 +0,0 @@
|
||||
# Request-Level Batching for Diffusion
|
||||
|
||||
This document describes the request-mode batching path for diffusion pipelines.
|
||||
For end-user enablement and tuning, see
|
||||
[Request-Level Batching](../../user_guide/diffusion/request_batching.md).
|
||||
|
||||
This is separate from
|
||||
[Continuous Batching for Step-Wise Diffusion](diffusion_continuous_batching.md).
|
||||
Request-level batching runs one full pipeline `forward()` over a static batch of
|
||||
compatible requests. Step-wise continuous batching admits work between denoise
|
||||
steps when `step_execution=True`.
|
||||
|
||||
## Why It Helps
|
||||
|
||||
The request-level design avoids coupling several logical requests to one request
|
||||
object. This keeps request identity, abort/error handling, and per-request
|
||||
metadata unambiguous while still allowing one fused pipeline forward pass for
|
||||
bursty or concurrent traffic.
|
||||
|
||||
## Overview
|
||||
|
||||
With request-level batching enabled:
|
||||
|
||||
- each `OmniDiffusionRequest` contains one `prompt` and one `request_id`
|
||||
- the scheduler groups compatible waiting requests into one scheduler wave
|
||||
- `DiffusionRequestBatch` wraps the scheduled requests for pipeline `forward()`
|
||||
- batch-capable pipelines return `list[DiffusionOutput]`, one output per
|
||||
request
|
||||
- `BatchRunnerOutput` maps each result back to its original `request_id`
|
||||
|
||||
Pipelines opt in with `supports_request_batch = True` and a `forward()` method
|
||||
that accepts `DiffusionRequestBatch` and returns `list[DiffusionOutput]`.
|
||||
Pipelines that do not opt in keep the existing per-request execution path.
|
||||
|
||||
## Enablement
|
||||
|
||||
Request-level batching is the request-mode path, so `step_execution` must remain
|
||||
disabled. Increase `max_num_seqs` above `1` to let the scheduler keep multiple
|
||||
compatible requests active:
|
||||
|
||||
```bash
|
||||
vllm serve Qwen/Qwen-Image --omni \
|
||||
--port 8091 \
|
||||
--max-num-seqs 4
|
||||
```
|
||||
|
||||
For bursty online ingress, `request_batch_max_wait_ms` can add a bounded
|
||||
admission wait before the first `schedule()` of a scheduler wave:
|
||||
|
||||
```bash
|
||||
vllm serve Qwen/Qwen-Image --omni \
|
||||
--port 8091 \
|
||||
--max-num-seqs 4 \
|
||||
--request-batch-max-wait-ms 20
|
||||
```
|
||||
|
||||
`request_batch_max_wait_ms=0` disables this wait and is the default.
|
||||
|
||||
## Request Contract
|
||||
|
||||
`OmniDiffusionRequest` represents one logical request. It owns one prompt,
|
||||
sampling parameters, request id, and request-local metadata. Runtime batches are
|
||||
formed by the scheduler and represented separately from the request payload.
|
||||
|
||||
Runtime batching is represented by:
|
||||
|
||||
- [`DiffusionSchedulerOutput`](gh-file:vllm_omni/diffusion/sched/interface.py)
|
||||
for scheduled request ids and request payloads
|
||||
- [`DiffusionRequestBatch`](gh-file:vllm_omni/diffusion/worker/request_batch.py)
|
||||
for the pipeline-facing request batch
|
||||
- [`BatchRunnerOutput`](gh-file:vllm_omni/diffusion/worker/utils.py) for
|
||||
per-request results
|
||||
|
||||
`DiffusionRequestBatch` intentionally exposes compatibility properties such as
|
||||
`prompts`, `sampling_params`, `request_id`, and `kv_sender_info` so migrated
|
||||
pipelines can stay close to upstream code while using a batch-aware contract.
|
||||
|
||||
## Scheduler
|
||||
|
||||
The scheduler derives its capacity from `max_num_seqs` through
|
||||
`max_num_running_reqs`. It exposes waiting/running queue counters so the engine
|
||||
can decide whether admission wait is useful before scheduling a new wave.
|
||||
|
||||
Batch compatibility is controlled by
|
||||
[`SamplingParamsKey`](gh-file:vllm_omni/diffusion/sched/interface.py). The key
|
||||
contains shape-sensitive and guidance-sensitive fields, including output count
|
||||
and LoRA identity. Requests with incompatible shapes, CFG settings, output
|
||||
counts, LoRA adapters, or LoRA scales are kept in separate batches.
|
||||
|
||||
Admission is conservative:
|
||||
|
||||
- the scheduler only batches compatible requests
|
||||
- FIFO ordering is preserved
|
||||
- an incompatible request at the head of the waiting queue blocks later
|
||||
compatible requests
|
||||
|
||||
## Engine
|
||||
|
||||
[`DiffusionEngine`](gh-file:vllm_omni/diffusion/diffusion_engine.py) resolves
|
||||
request-batch capability during initialization from the configured pipeline
|
||||
class, including custom pipeline classes.
|
||||
|
||||
The capability check uses the pipeline class attribute
|
||||
`supports_request_batch = True`. Pipelines that set this attribute must implement
|
||||
a request-batch-compatible `forward()` contract and return one
|
||||
`DiffusionOutput` per request; the runner validates that return shape at runtime.
|
||||
|
||||
When the selected pipeline is batch-capable and `step_execution=False`, request
|
||||
mode routes scheduler waves through the batch executor path. Otherwise it keeps
|
||||
the per-request executor path.
|
||||
|
||||
The optional admission wait runs only when:
|
||||
|
||||
- request batching is supported
|
||||
- `step_execution=False`
|
||||
- `request_batch_max_wait_ms > 0`
|
||||
- no requests are currently running
|
||||
|
||||
The wait exits early when the waiting queue reaches capacity, when the queue is
|
||||
stable for a short window, when the deadline expires, or when the engine stops.
|
||||
|
||||
## Executor And Runner
|
||||
|
||||
The executor exposes two request-mode entries:
|
||||
|
||||
- `execute_request`: one worker call per scheduled request
|
||||
- `execute_batch`: one worker call for the whole `DiffusionSchedulerOutput`
|
||||
|
||||
On the batch path, the worker builds a `DiffusionRequestBatch` and runs the
|
||||
pipeline once. Request-local setup remains per request:
|
||||
|
||||
- KV transfer metadata
|
||||
- random generator and seed handling
|
||||
- request output/error/abort mapping
|
||||
|
||||
Shared batch setup happens once per batch when possible:
|
||||
|
||||
- cache refresh
|
||||
- LoRA activation for the homogeneous adapter key
|
||||
- pipeline `forward(req_batch)`
|
||||
|
||||
Large tensor IPC still uses the shared-memory packing path. The packer traverses
|
||||
both normal `RunnerOutput.result` wrappers and nested batch results so batched
|
||||
outputs do not fall back to pickle IPC for tensor payloads.
|
||||
|
||||
## Current Limitations
|
||||
|
||||
- Only pipelines that declare the request-batch contract use fused batch
|
||||
execution.
|
||||
- Batches are homogeneous under `SamplingParamsKey`; heterogeneous resolution or
|
||||
incompatible guidance settings do not co-batch yet.
|
||||
- FIFO scheduling can reduce batching opportunities when an incompatible
|
||||
request is at the front of the queue.
|
||||
- `request_batch_max_wait_ms` improves burst coalescing but can add latency to
|
||||
the first request in a scheduler wave. Keep it small for latency-sensitive
|
||||
serving.
|
||||
- Step-wise continuous batching is documented separately and only applies when
|
||||
`step_execution=True`.
|
||||
|
||||
## Related Files
|
||||
|
||||
- Request object and request batch:
|
||||
[`vllm_omni/diffusion/request.py`](gh-file:vllm_omni/diffusion/request.py)
|
||||
- Scheduler interface:
|
||||
[`vllm_omni/diffusion/sched/interface.py`](gh-file:vllm_omni/diffusion/sched/interface.py)
|
||||
- Scheduler base:
|
||||
[`vllm_omni/diffusion/sched/base_scheduler.py`](gh-file:vllm_omni/diffusion/sched/base_scheduler.py)
|
||||
- Engine:
|
||||
[`vllm_omni/diffusion/diffusion_engine.py`](gh-file:vllm_omni/diffusion/diffusion_engine.py)
|
||||
- Worker runner:
|
||||
[`vllm_omni/diffusion/worker/diffusion_model_runner.py`](gh-file:vllm_omni/diffusion/worker/diffusion_model_runner.py)
|
||||
- Executor interface:
|
||||
[`vllm_omni/diffusion/executor/abstract.py`](gh-file:vllm_omni/diffusion/executor/abstract.py)
|
||||
- Tests:
|
||||
[`tests/diffusion/test_diffusion_engine.py`](gh-file:tests/diffusion/test_diffusion_engine.py)
|
||||
@@ -448,42 +448,6 @@ Complete examples in the codebase:
|
||||
|
||||
---
|
||||
|
||||
## Spatially-Sharded Decode (Wan)
|
||||
|
||||
The tile-parallel executor above assigns independent spatial **tiles** to ranks. The Wan VAE additionally supports a **spatially-sharded decode** backend, selected through `DiffusionParallelConfig.vae_parallel_mode` (`"spatial_shard_height"` or `"spatial_shard_width"`; default is `"tile"`).
|
||||
|
||||
### How it differs from tile parallel
|
||||
|
||||
| Aspect | Tile parallel (`"tile"`) | Spatially-sharded (`"spatial_shard_height"`/`"spatial_shard_width"`) |
|
||||
|--------|--------------------------|-----------------------------------------------|
|
||||
| Unit of work | Independent overlapping tiles | A single global feature map sharded along H or W |
|
||||
| Cross-rank communication | Gather tiles to rank 0, stitch + blend | Per-conv **halo exchange** of boundary rows/cols (P2P) |
|
||||
| Output assembly | Blend overlapping tiles | All-gather shards on rank 0, trim padding (matches `broadcast_result=False`) |
|
||||
| Scope | Decode + encode | Decode only |
|
||||
|
||||
Spatial-shard decode swaps the decoder's spatial convolutions/padding for halo-exchanging variants (`WanDistConv2d`, `WanDistCausalConv3d`, `WanDistZeroPad2d`) so each rank only holds a shard of the activations but still sees the correct receptive field at shard boundaries. Implementation lives in `vllm_omni/diffusion/distributed/autoencoders/wan_spatial_shard.py`.
|
||||
|
||||
### Wiring
|
||||
|
||||
`vae_parallel_mode` flows through the same path as `vae_patch_parallel_size`:
|
||||
|
||||
```text
|
||||
serve.py (--vae-parallel-mode) / OmniEngineArgs
|
||||
-> DiffusionParallelConfig.vae_parallel_mode
|
||||
-> registry.py: model.vae.set_parallel_size(vae_pp_size, mode=...)
|
||||
-> DistributedVaeExecutor.parallel_mode
|
||||
-> DistributedAutoencoderKLWan.tiled_decode dispatch
|
||||
```
|
||||
|
||||
`DistributedAutoencoderKLWan._spatial_shard_decode_enabled()` gates the path: it requires distributed decode to be enabled, a 5D latent, and `vae_patch_parallel_size == DiT group size`. Otherwise it logs a warning and falls back to tile-parallel decode.
|
||||
|
||||
### Notes
|
||||
|
||||
- The decoder is patched **in place** the first time spatial-shard decode runs and is bound to a single split dimension for the lifetime of the VAE instance.
|
||||
- Numerical correctness vs. single-GPU decode is covered by `tests/diffusion/distributed/test_wan_spatial_shard.py::test_spatial_shard_decode_matches_reference` (multi-GPU, nightly `full_model` + `distributed_cuda`).
|
||||
|
||||
---
|
||||
|
||||
## Summary
|
||||
|
||||
Adding VAE Patch Parallel support to diffusion model:
|
||||
|
||||
@@ -11,7 +11,6 @@ This section contains design documents and architecture specifications for vLLM-
|
||||
- [Disaggregated Inference](feature/disaggregated_inference.md)
|
||||
- [Ray-based Execution](feature/ray_based_execution.md)
|
||||
- [Adding Step Execution Support for Diffusion Pipelines](feature/diffusion_step_execution.md)
|
||||
- [Request-Level Batching for Diffusion](feature/diffusion_request_level_batching.md)
|
||||
- [Continuous Batching for Step-Wise Diffusion](feature/diffusion_continuous_batching.md)
|
||||
|
||||
## Infrastructure Design Documents
|
||||
|
||||
@@ -199,7 +199,7 @@ class _BaseScheduler(SchedulerInterface):
|
||||
- **Shared cleanup logic**: Duplicate-request checks, finish handling, and state removal are centralized instead of
|
||||
duplicated in each policy.
|
||||
|
||||
- **Current constraint boundary**: `_BaseScheduler` derives `max_num_running_reqs` from `max_num_seqs`. Request-mode diffusion can use that capacity for compatible independent requests when the configured pipeline declares `supports_request_batch = True`; step-wise diffusion uses the same scheduler capacity for compatible step batches.
|
||||
- **Current constraint boundary**: `_BaseScheduler` derives `max_num_running_reqs` from `max_num_seqs`, but request-mode diffusion is still clamped back to `1` by the engine. The step-wise path can keep this above `1` for compatible-request batching.
|
||||
|
||||
#### 2.4 Current `RequestScheduler` Policy
|
||||
|
||||
@@ -215,7 +215,7 @@ class RequestScheduler(_BaseScheduler):
|
||||
|
||||
- **FIFO request scheduling**: Waiting requests are promoted in queue order.
|
||||
|
||||
- **Compatible request admission**: `RequestScheduler` admits waiting requests while capacity remains and the request's `SamplingParamsKey` is compatible with the active batch. Request-mode execution keeps each logical request independent, while batch-capable pipelines receive the scheduled requests as a runner-side `DiffusionRequestBatch`.
|
||||
- **Single-request admission**: `RequestScheduler` still admits one active request at a time because request-mode execution completes a whole request per dispatch.
|
||||
|
||||
- **Executor result feedback**: `update_from_output()` converts executor output into `FINISHED_COMPLETED` or `FINISHED_ERROR` and returns finished request ids.
|
||||
|
||||
@@ -235,12 +235,7 @@ while True:
|
||||
|
||||
- **No scheduler-owned IPC**: Scheduler no longer talks to workers directly.
|
||||
|
||||
- **Split concurrency model**: Request-mode diffusion can schedule a static
|
||||
batch of compatible independent requests for one full pipeline forward when
|
||||
the pipeline supports request-level batching; the step-wise path can also
|
||||
admit or remove compatible requests between denoise steps. See
|
||||
[Request-Level Batching](../feature/diffusion_request_level_batching.md) and
|
||||
[Continuous Batching for Step-Wise Diffusion](../feature/diffusion_continuous_batching.md).
|
||||
- **Split concurrency model**: Request-mode diffusion remains single-active-request, while the step-wise path can keep multiple compatible requests running and advance them independently between denoise steps.
|
||||
|
||||
---
|
||||
|
||||
|
||||
@@ -54,24 +54,20 @@ Create a custom pipeline class that extends an existing pipeline. In this exampl
|
||||
|
||||
```python
|
||||
# custom_pipeline.py
|
||||
from vllm_omni.diffusion.data import OmniDiffusionConfig
|
||||
from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig
|
||||
from vllm_omni.diffusion.models.qwen_image.pipeline_qwen_image_edit import QwenImageEditPipeline
|
||||
from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch
|
||||
import torch
|
||||
|
||||
class CustomPipeline(QwenImageEditPipeline):
|
||||
def __init__(self, *, od_config: OmniDiffusionConfig, prefix: str = ""):
|
||||
super().__init__(od_config=od_config, prefix=prefix)
|
||||
|
||||
def forward(self, req: DiffusionRequestBatch):
|
||||
# Optionally customize sampling parameters
|
||||
actual_num_steps = req.sampling_params.num_inference_steps or 50
|
||||
req.sampling_params.num_inference_steps = actual_num_steps
|
||||
|
||||
def forward(self, req, prompt=None, negative_prompt=None, **kwargs):
|
||||
# Call parent's forward to get normal output
|
||||
output = super().forward(req=req)
|
||||
output = super().forward(req=req, prompt=prompt, negative_prompt=negative_prompt, **kwargs)
|
||||
|
||||
# Add custom trajectory data
|
||||
actual_num_steps = req.sampling_params.num_inference_steps or kwargs.get('num_inference_steps', 50)
|
||||
output.trajectory_timesteps = torch.linspace(1000, 0, actual_num_steps, dtype=torch.float32)
|
||||
output.trajectory_latents = torch.randn(actual_num_steps, 1, 16, 64, 64, dtype=torch.float32)
|
||||
|
||||
|
||||
@@ -34,7 +34,7 @@ vLLM-Omni is a Python library that supports the following GPU variants. The libr
|
||||
|
||||
### Pre-built wheels
|
||||
|
||||
Note: Pre-built wheels are currently available for vLLM-Omni 0.11.0rc1, 0.12.0rc1, 0.14.0rc1, 0.14.0, 0.16.0, 0.18.0, 0.20.0, 0.21.0, 0.22.0, 0.23.0, 0.24.0, and 0.25.0. If you need a newer unreleased revision, please [build from source](https://docs.vllm.ai/projects/vllm-omni/en/latest/getting_started/installation/gpu/#build-wheel-from-source).
|
||||
Note: Pre-built wheels are currently available for vLLM-Omni 0.11.0rc1, 0.12.0rc1, 0.14.0rc1, 0.14.0, 0.16.0, 0.18.0, 0.20.0, 0.21.0, 0.22.0, and 0.23.0. If you need a newer unreleased revision, please [build from source](https://docs.vllm.ai/projects/vllm-omni/en/latest/getting_started/installation/gpu/#build-wheel-from-source).
|
||||
|
||||
=== "NVIDIA CUDA"
|
||||
|
||||
|
||||
@@ -20,7 +20,7 @@ Therefore, it is recommended to install vLLM and vLLM-Omni with a **fresh new**
|
||||
|
||||
vLLM-Omni is built based on vLLM. Please install it with command below.
|
||||
```bash
|
||||
uv pip install vllm==0.25.0 --torch-backend=auto
|
||||
uv pip install vllm==0.23.0 --torch-backend=auto
|
||||
```
|
||||
|
||||
#### Installation of vLLM-Omni
|
||||
@@ -39,13 +39,13 @@ uv pip install 'vllm-omni[demo]'
|
||||
# --8<-- [start:build-wheel-from-source]
|
||||
|
||||
#### Installation of vLLM
|
||||
If you do not need to modify source code of vLLM, you can directly install the stable 0.25.0 release version of the library
|
||||
If you do not need to modify source code of vLLM, you can directly install the stable 0.23.0 release version of the library
|
||||
|
||||
```bash
|
||||
uv pip install vllm==0.25.0 --torch-backend=auto
|
||||
uv pip install vllm==0.23.0 --torch-backend=auto
|
||||
```
|
||||
|
||||
The 0.25.0 release of vLLM ships CUDA 13.0-compatible binaries by default. If you need a different CUDA variant or want to reuse an existing PyTorch installation, build vLLM from source instead.
|
||||
The 0.23.0 release of vLLM ships CUDA 13.0-compatible binaries by default. If you need a different CUDA variant or want to reuse an existing PyTorch installation, build vLLM from source instead.
|
||||
|
||||
#### Installation of vLLM-Omni
|
||||
Since vllm-omni is rapidly evolving, it's recommended to install it from source
|
||||
@@ -66,12 +66,12 @@ If you want to check, modify or debug with source code of vLLM, install the libr
|
||||
```bash
|
||||
git clone https://github.com/vllm-project/vllm.git
|
||||
cd vllm
|
||||
git checkout v0.25.0
|
||||
git checkout v0.23.0
|
||||
```
|
||||
Set up environment variables to get pre-built wheels. If there are internet problems, just download the whl file manually. And set `VLLM_PRECOMPILED_WHEEL_LOCATION` as your local absolute path of whl file.
|
||||
```bash
|
||||
#For CUDA 13.0 (the default for v0.25.0; the wheel filename has no `+cu130` suffix)
|
||||
export VLLM_PRECOMPILED_WHEEL_LOCATION=https://github.com/vllm-project/vllm/releases/download/v0.25.0/vllm-0.25.0-cp38-abi3-manylinux_2_28_x86_64.whl
|
||||
#For CUDA 13.0 (the default for v0.23.0; the wheel filename has no `+cu130` suffix)
|
||||
export VLLM_PRECOMPILED_WHEEL_LOCATION=https://github.com/vllm-project/vllm/releases/download/v0.23.0/vllm-0.23.0-cp38-abi3-manylinux_2_28_x86_64.whl
|
||||
```
|
||||
Install vllm with command below (If you have no existing PyTorch).
|
||||
```bash
|
||||
@@ -102,7 +102,7 @@ docker run --runtime nvidia --gpus 2 \
|
||||
--env "HF_TOKEN=$HF_TOKEN" \
|
||||
-p 8091:8091 \
|
||||
--ipc=host \
|
||||
vllm/vllm-omni:v0.25.0 \
|
||||
vllm/vllm-omni:v0.23.0 \
|
||||
vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091
|
||||
```
|
||||
|
||||
|
||||
@@ -13,7 +13,7 @@ vLLM-Omni current recommends the steps in under setup through Docker Images.
|
||||
|
||||
vLLM-Omni is built based on vLLM. Please install it with command below.
|
||||
```bash
|
||||
uv pip install vllm==0.25.0+rocm723 --extra-index-url https://wheels.vllm.ai/rocm/0.25.0/rocm723
|
||||
uv pip install vllm==0.23.0+rocm722 --extra-index-url https://wheels.vllm.ai/rocm/0.23.0/rocm722
|
||||
```
|
||||
|
||||
#### Installation of vLLM-Omni
|
||||
@@ -34,13 +34,13 @@ uv pip install onnxruntime-rocm
|
||||
# --8<-- [start:build-wheel-from-source]
|
||||
|
||||
#### Installation of vLLM
|
||||
If you do not need to modify source code of vLLM, you can directly install the stable 0.25.0 release version of the library
|
||||
If you do not need to modify source code of vLLM, you can directly install the stable 0.23.0 release version of the library
|
||||
|
||||
```bash
|
||||
uv pip install vllm==0.25.0+rocm723 --extra-index-url https://wheels.vllm.ai/rocm/0.25.0/rocm723
|
||||
uv pip install vllm==0.23.0+rocm722 --extra-index-url https://wheels.vllm.ai/rocm/0.23.0/rocm722
|
||||
```
|
||||
|
||||
The pre-built 0.25.0 vLLM wheel targets ROCm 7.2.3. If you need a different ROCm stack or want to reuse an existing PyTorch installation, build vLLM from source instead.
|
||||
The pre-built 0.23.0 vLLM wheel targets ROCm 7.2.2. If you need a different ROCm stack or want to reuse an existing PyTorch installation, build vLLM from source instead.
|
||||
|
||||
#### Installation of vLLM-Omni
|
||||
Since vllm-omni is rapidly evolving, it's recommended to install it from source
|
||||
@@ -58,7 +58,7 @@ If you want to check, modify or debug with source code of vLLM, install the libr
|
||||
```bash
|
||||
git clone https://github.com/vllm-project/vllm.git
|
||||
cd vllm
|
||||
git checkout v0.25.0
|
||||
git checkout v0.23.0
|
||||
python3 -m pip install -r requirements/rocm.txt
|
||||
python3 setup.py develop
|
||||
```
|
||||
@@ -130,7 +130,7 @@ docker run --rm \
|
||||
-v ~/.cache/huggingface:/root/.cache/huggingface \
|
||||
--env "HF_TOKEN=$HF_TOKEN" \
|
||||
-p 8091:8091 \
|
||||
vllm/vllm-omni-rocm:v0.25.0 \
|
||||
vllm/vllm-omni-rocm:v0.23.0 \
|
||||
--model Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091
|
||||
```
|
||||
|
||||
@@ -149,7 +149,7 @@ docker run --rm -it \
|
||||
-v ~/.cache/huggingface:/root/.cache/huggingface \
|
||||
--env "HF_TOKEN=$HF_TOKEN" \
|
||||
--entrypoint bash \
|
||||
vllm/vllm-omni-rocm:v0.25.0
|
||||
vllm/vllm-omni-rocm:v0.23.0
|
||||
```
|
||||
|
||||
# --8<-- [end:pre-built-images]
|
||||
@@ -1,16 +1,19 @@
|
||||
# --8<-- [start:requirements]
|
||||
|
||||
For detailed hardware and software requirements, please refer to the [vLLM-Ascend installation documentation](https://docs.vllm.ai/projects/ascend/en/latest/installation.html).
|
||||
For detailed hardware and software requirements, please refer to the [vllm-ascend installation documentation](https://docs.vllm.ai/projects/ascend/en/latest/installation.html).
|
||||
|
||||
# --8<-- [end:requirements]
|
||||
# --8<-- [start:installation-release]
|
||||
|
||||
The recommended way to use vLLM-Omni on NPU is through the vLLM-Ascend pre-built Docker images:
|
||||
The recommended way to use vLLM-Omni on NPU is through the vllm-ascend pre-built Docker images:
|
||||
|
||||
```bash
|
||||
# vLLM-Ascend image
|
||||
# Atlas A2: quay.io/atlas-ci/vllm-ascend:v0.24.0
|
||||
# Atlas A3: quay.io/atlas-ci/vllm-ascend:v0.24.0-a3
|
||||
# Update the vllm-ascend image
|
||||
# Atlas A2:
|
||||
# export IMAGE=quay.io/ascend/vllm-ascend:v0.18.0rc1
|
||||
# Atlas A3:
|
||||
# export IMAGE=quay.io/ascend/vllm-ascend:v0.18.0rc1-a3
|
||||
export IMAGE=quay.io/ascend/vllm-ascend:v0.18.0rc1
|
||||
docker run --rm \
|
||||
--name vllm-omni-npu \
|
||||
--shm-size=1g \
|
||||
@@ -28,11 +31,11 @@ docker run --rm \
|
||||
-v /etc/ascend_install.info:/etc/ascend_install.info \
|
||||
-v /root/.cache:/root/.cache \
|
||||
-p 8000:8000 \
|
||||
-it quay.io/atlas-ci/vllm-ascend:0.24.0-a3 bash
|
||||
-it $IMAGE bash
|
||||
|
||||
# Inside the container, install vLLM-Omni from source
|
||||
cd /vllm-workspace
|
||||
git clone -b v0.24.0 https://github.com/vllm-project/vllm-omni.git
|
||||
git clone -b v0.18.0 https://github.com/vllm-project/vllm-omni.git
|
||||
cd vllm-omni
|
||||
pip install -v -e . --no-build-isolation
|
||||
# or VLLM_OMNI_TARGET_DEVICE=npu pip install -v -e .
|
||||
@@ -84,11 +87,14 @@ Supported images as following.
|
||||
Here's an example deployment command that has been verified on 4 x NPUs:
|
||||
|
||||
```bash
|
||||
# Atlas A2: quay.io/ascend/vllm-omni:v0.24.0
|
||||
# Atlas A3: quay.io/ascend/vllm-omni:v0.24.0-a3
|
||||
# Atlas A2:
|
||||
# export IMAGE=quay.io/ascend/vllm-omni:v0.18.0
|
||||
# Atlas A3:
|
||||
# export IMAGE=quay.io/ascend/vllm-omni:v0.18.0-a3
|
||||
export IMAGE=quay.io/ascend/vllm-omni:v0.18.0
|
||||
docker run --rm \
|
||||
--name vllm-omni-a3 \
|
||||
--shm-size=4g \
|
||||
--name vllm-omni-npu \
|
||||
--shm-size=1g \
|
||||
--device /dev/davinci0 \
|
||||
--device /dev/davinci1 \
|
||||
--device /dev/davinci2 \
|
||||
@@ -100,17 +106,10 @@ docker run --rm \
|
||||
-v /usr/local/bin/npu-smi:/usr/local/bin/npu-smi \
|
||||
-v /usr/local/Ascend/driver/lib64/:/usr/local/Ascend/driver/lib64/ \
|
||||
-v /usr/local/Ascend/driver/version.info:/usr/local/Ascend/driver/version.info \
|
||||
--device /dev/davinci_manager \
|
||||
--device /dev/devmm_svm \
|
||||
--device /dev/hisi_hdc \
|
||||
-v /usr/local/dcmi:/usr/local/dcmi \
|
||||
-v /usr/local/bin/npu-smi:/usr/local/bin/npu-smi \
|
||||
-v /usr/local/Ascend/driver/lib64/:/usr/local/Ascend/driver/lib64/ \
|
||||
-v /usr/local/Ascend/driver/version.info:/usr/local/Ascend/driver/version.info \
|
||||
-v /etc/ascend_install.info:/etc/ascend_install.info \
|
||||
-v ~/.cache:/root/.cache \
|
||||
-p 8091:8091 \
|
||||
-it quay.io/ascend/vllm-omni:v0.24.0-a3 bash
|
||||
-v /root/.cache:/root/.cache \
|
||||
-p 8000:8000 \
|
||||
-it $IMAGE bash
|
||||
```
|
||||
|
||||
!!! tip
|
||||
|
||||
@@ -19,10 +19,10 @@ uv venv --python 3.12 --seed
|
||||
source .venv/bin/activate
|
||||
|
||||
# On CUDA
|
||||
uv pip install vllm==0.25.0 --torch-backend=auto
|
||||
uv pip install vllm==0.23.0 --torch-backend=auto
|
||||
|
||||
# On ROCm
|
||||
uv pip install vllm==0.25.0+rocm723 --extra-index-url https://wheels.vllm.ai/rocm/0.25.0/rocm723
|
||||
uv pip install vllm==0.23.0+rocm722 --extra-index-url https://wheels.vllm.ai/rocm/0.23.0/rocm722
|
||||
|
||||
git clone https://github.com/vllm-project/vllm-omni.git
|
||||
cd vllm-omni
|
||||
@@ -31,10 +31,11 @@ uv pip install -e .
|
||||
|
||||
For additional installation methods — please see the [installation guide](installation/README.md).
|
||||
|
||||
|
||||
!!! note
|
||||
It is important to install the same major & minor version of vLLM and vLLM Omni, otherwise things may not work as expected. If the versions are misaligned, you will see a warning when you import vLLM Omni.
|
||||
|
||||
If you are seeing strange behavior with the `vllm` command not handling the `--omni` flag correctly, you most likely have a version mismatch with vLLM < `0.25.0` and vLLM Omni `0.25.0`, as vLLM Omni no longer hijacks the vLLM entrypoint. Updating vLLM should resolve this issue.
|
||||
If you are seeing strange behavior with the `vllm` command not handling the `--omni` flag correctly, you most likely have a version mismatch with vLLM < `0.23.0` and vLLM Omni `0.23.0`, as vLLM Omni no longer hijacks the vLLM entrypoint. Updating vLLM should resolve this issue.
|
||||
|
||||
## Offline Inference
|
||||
|
||||
@@ -51,14 +52,14 @@ if __name__ == "__main__":
|
||||
images[0].save("coffee.png")
|
||||
```
|
||||
|
||||
You can pass a list of prompts and wait for the independent requests to finish,
|
||||
as shown below.
|
||||
You can pass a list of prompts and wait for them to process altogether, shown below.
|
||||
|
||||
!!! info
|
||||
|
||||
For diffusion pipelines, each prompt becomes a separate logical request.
|
||||
The runtime may automatically batch compatible in-flight requests through
|
||||
the scheduler and runner.
|
||||
However, it is not currently recommended to do so
|
||||
because not all models support batch inference,
|
||||
and batch requesting mostly does not provide significant performance improvement (despite the impression that it does).
|
||||
This feature is primarily for the sake of interface compatibility with vLLM and to allow for future improvements.
|
||||
|
||||
```python
|
||||
from vllm_omni.entrypoints.omni import Omni
|
||||
@@ -87,9 +88,9 @@ if __name__ == "__main__":
|
||||
|
||||
!!! info
|
||||
|
||||
For diffusion request-level batching controls such as `max_num_seqs` and
|
||||
`request_batch_max_wait_ms`, see
|
||||
[Request-Level Batching](../user_guide/diffusion/request_batching.md).
|
||||
For diffusion pipelines, the stage config field `stage_args.[].engine_args.max_num_seqs` is 1 by default, and the input
|
||||
list is sliced into single-item requests before feeding into the diffusion pipeline. For models that do internally support
|
||||
batched inputs, you can [modify this configuration](../configuration/stage_configs.md) to let the model accept a longer batch of prompts.
|
||||
|
||||
For more usages, please refer to [offline inference](../user_guide/examples/offline_inference/qwen2_5_omni.md)
|
||||
|
||||
|
||||
@@ -37,9 +37,8 @@ th {
|
||||
| `WanPipeline` | Wan2.1-T2V, Wan2.2-T2V, Wan2.2-TI2V | `Wan-AI/Wan2.1-T2V-1.3B-Diffusers`, `Wan-AI/Wan2.1-T2V-14B-Diffusers`, `Wan-AI/Wan2.2-T2V-A14B-Diffusers`, `Wan-AI/Wan2.2-TI2V-5B-Diffusers` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
|
||||
| `WanImageToVideoPipeline` | Wan2.2-I2V | `Wan-AI/Wan2.2-I2V-A14B-Diffusers` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
|
||||
| `Cosmos3OmniDiffusersPipeline` | Cosmos3 T2I, T2V, I2V, V2V, T2V with sound, action policy | `nvidia/Cosmos3-Nano`, `nvidia/Cosmos3-Super` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
|
||||
| `Wan22S2VPipeline` | Wan2.2-S2V | `Wan-AI/Wan2.2-S2V-14B` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
|
||||
| `WanSpeechToVideoPipeline` | Wan2.2-S2V | `Wan-AI/Wan2.2-S2V-14B` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
|
||||
| `Wan22VACEPipeline` | Wan2.1-VACE | `Wan-AI/Wan2.1-VACE-1.3B-diffusers`, `Wan-AI/Wan2.1-VACE-14B-diffusers` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
|
||||
| `Wan22VACEPipeline` | Wan2.2-VACE | `Pyros13/Wan2.2-VACE-Fun-A14B-Diffusers` | ✅︎ | | | |
|
||||
| `LTX2Pipeline` | LTX-2-T2V | `Lightricks/LTX-2` | ✅︎ | ✅︎ | | |
|
||||
| `LTX2ImageToVideoPipeline` | LTX-2-I2V | `Lightricks/LTX-2` | ✅︎ | ✅︎ | | |
|
||||
| `LTX2TwoStagesPipeline` | LTX-2-T2V | `rootonchair/LTX-2-19b-distilled` | ✅︎ | ✅︎ | | |
|
||||
@@ -64,8 +63,6 @@ th {
|
||||
| `FluxPipeline` | FLUX.1-schnell | `black-forest-labs/FLUX.1-schnell` | ✅︎ | ✅︎ | | ✅︎ |
|
||||
| `OmniGen2Pipeline` | OmniGen2 | `OmniGen2/OmniGen2` | ✅︎ | ✅︎ | | ✅︎ |
|
||||
| `StableAudioPipeline` | Stable-Audio-Open | `stabilityai/stable-audio-open-1.0` | ✅︎ | ✅︎ | | ✅︎ |
|
||||
| `SoulXSingerPipeline` | SoulX-Singer (SVS) | `Soul-AILab/SoulX-Singer` | ✅︎ | | | |
|
||||
| `SoulXSingerSVCPipeline` | SoulX-Singer-SVC | `Soul-AILab/SoulX-Singer` (`model-svc.pt`) | ✅︎ | | | |
|
||||
| `AudioXPipeline` | AudioX | `zhangj1an/AudioX` | ✅︎ | ✅︎ | | |
|
||||
| `Qwen3TTSForConditionalGeneration` | Qwen3-TTS-12Hz-1.7B-CustomVoice | `Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
|
||||
| `Qwen3TTSForConditionalGeneration` | Qwen3-TTS-12Hz-1.7B-VoiceDesign | `Qwen/Qwen3-TTS-12Hz-1.7B-VoiceDesign` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
|
||||
@@ -77,7 +74,6 @@ th {
|
||||
| `MossTTSRealtime` | MOSS-TTS-Realtime | `OpenMOSS-Team/MOSS-TTS-Realtime` | ✅︎ | | | |
|
||||
| `HiggsAudioV2ForConditionalGeneration` | Higgs-Audio v2 | `bosonai/higgs-audio-v2-generation-3B-base` | ✅︎ | | | |
|
||||
| `HiggsMultimodalQwen3ForConditionalGeneration` | Higgs-Audio v3 (TTS) | `bosonai/higgs-audio-v3-tts-4b` | ✅︎ | | | |
|
||||
| `IndexTTS2TalkerForConditionalGeneration` | IndexTTS-2 | `IndexTeam/IndexTTS-2` | ✅︎ | | | |
|
||||
| `NextStep11Pipeline` | NextStep-1.1 | `stepfun-ai/NextStep-1.1` | ✅︎ | ✅︎ | | ✅︎ |
|
||||
| `MiMoAudioModel` | MiMo-Audio-7B-Instruct | `XiaomiMiMo/MiMo-Audio-7B-Instruct` | ✅︎ | ✅︎ | | |
|
||||
| `MiMoV2ASRForCausalLM` | MiMo-V2.5-ASR | `XiaomiMiMo/MiMo-V2.5-ASR` | ✅︎ | ✅︎ | | |
|
||||
|
||||
@@ -32,20 +32,6 @@ curl -X POST http://localhost:8000/v1/images/generations \
|
||||
}' | jq -r '.data[0].b64_json' | base64 -d > dragon.png
|
||||
```
|
||||
|
||||
**Using curl save to file:**
|
||||
|
||||
```bash
|
||||
curl -o dragon.png -X POST http://localhost:8000/v1/images/generations \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"prompt": "a dragon laying over the spine of the Green Mountains of Vermont",
|
||||
"size": "1024x1024",
|
||||
"seed": 42,
|
||||
"response_format":"file"
|
||||
}'
|
||||
```
|
||||
|
||||
|
||||
**Using Python:**
|
||||
|
||||
```python
|
||||
@@ -71,36 +57,6 @@ img = Image.open(io.BytesIO(img_bytes))
|
||||
img.save("cat.png")
|
||||
```
|
||||
|
||||
**Using Python save to file:**
|
||||
|
||||
```python
|
||||
import requests
|
||||
import base64
|
||||
from PIL import Image
|
||||
import io
|
||||
import re
|
||||
|
||||
response = requests.post(
|
||||
"http://localhost:8000/v1/images/generations",
|
||||
json={
|
||||
"prompt": "a black and white cat wearing a princess tiara",
|
||||
"size": "1024x1024",
|
||||
"num_inference_steps": 50,
|
||||
"seed": 42,
|
||||
"response_format":"file"
|
||||
}
|
||||
)
|
||||
|
||||
# save to file
|
||||
content_disposition = response.headers.get("Content-Disposition", "")
|
||||
match = re.search(r'filename="?(.+)"?', content_disposition)
|
||||
filename = match.group(1) if match else "save.png"
|
||||
with open(filename, "wb") as f:
|
||||
for chunk in response.iter_content(8192):
|
||||
f.write(chunk)
|
||||
print("saved:", filename)
|
||||
```
|
||||
|
||||
**Using OpenAI SDK:**
|
||||
|
||||
```python
|
||||
|
||||
@@ -122,8 +122,7 @@ Content-Type: application/json
|
||||
| `max_new_tokens` | integer | 2048 | Maximum tokens to generate |
|
||||
| `initial_codec_chunk_frames` | integer | null | Per-request initial chunk size override for TTFA tuning. When null, IC is computed dynamically based on server load. |
|
||||
| `non_streaming_mode` | bool | null | Qwen3-TTS prompt construction mode override. Does not affect HTTP response streaming or async-chunk pipelining. When null, Qwen3-TTS uses model defaults: Base=false, CustomVoice/VoiceDesign=true. |
|
||||
| `stream` | bool | false | When true, stream OpenAI `speech.audio.*` SSE events (requires `response_format="pcm"` or `"wav"`). For raw PCM/WAV byte streaming, set `stream_format="audio"`. |
|
||||
| `stream_format` | string | null | Streaming output format. `"audio"` streams raw audio bytes as they are decoded; `"sse"` streams OpenAI `speech.audio.*` Server-Sent Events. If omitted, `stream=true` selects SSE and `stream=false` remains non-streaming. See [Response Format](#response-format). |
|
||||
| `stream` | bool | false | Stream raw PCM chunks as they are decoded (requires `response_format="pcm"`) |
|
||||
|
||||
**Supported languages:** Only applicable to Qwen3-TTS. Derived from the model configuration (`talker_config.codec_language_id` in the checkpoint's `config.json`), plus `Auto`, which is always accepted. Official Qwen3-TTS checkpoints support: Auto, Chinese, English, Japanese, Korean, German, French, Russian, Portuguese, Spanish, Italian.
|
||||
|
||||
@@ -137,57 +136,7 @@ Content-Type: application/json
|
||||
|
||||
### Response Format
|
||||
|
||||
The response shape depends on the streaming parameters:
|
||||
|
||||
**Non-streaming (default).** With `stream=false` and no `stream_format`, returns the
|
||||
complete clip as binary audio data with an appropriate `Content-Type` header (e.g.
|
||||
`audio/wav`). The raw-bytes body has no JSON carrier, so no `usage` is reported.
|
||||
|
||||
**Raw audio stream** (`stream_format="audio"`). Streams raw audio bytes (PCM or
|
||||
WAV) as they are decoded.
|
||||
|
||||
Both streaming modes (`stream_format="audio"` and `"sse"`) require
|
||||
`response_format="pcm"` or `"wav"`, and `speed` must be `1.0` (or omitted).
|
||||
|
||||
**SSE stream** (`stream=true` or `stream_format="sse"`). Streams [OpenAI
|
||||
`speech.audio.*` Server-Sent Events](https://platform.openai.com/docs/api-reference/audio-streaming).
|
||||
Each event has an `event:` line and a JSON `data:` line:
|
||||
|
||||
- `speech.audio.delta` — a base64 audio chunk:
|
||||
|
||||
```json
|
||||
{ "type": "speech.audio.delta", "audio": "<base64>", "response_format": "pcm" }
|
||||
```
|
||||
|
||||
- `speech.audio.done` — terminal event, carrying token `usage`:
|
||||
|
||||
```json
|
||||
{
|
||||
"type": "speech.audio.done",
|
||||
"usage": {
|
||||
"input_tokens": 119,
|
||||
"output_tokens": 77,
|
||||
"total_tokens": 196,
|
||||
"input_token_details": { "text_tokens": 18, "audio_tokens": 101 }
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
- `speech.audio.error` — emitted instead of `speech.audio.done` if generation fails:
|
||||
|
||||
```json
|
||||
{ "type": "speech.audio.error", "error": { "message": "...", "type": "server_error", "param": null, "code": 500 } }
|
||||
```
|
||||
|
||||
The `usage` object on `speech.audio.done` is the same shape returned per item by the
|
||||
[batch endpoint](#batch-speech-generation):
|
||||
|
||||
- `input_tokens` = `text_tokens` + `audio_tokens`
|
||||
- `text_tokens`: tokens of the synthesized text (`input` plus `instructions`)
|
||||
- `audio_tokens`: reference-audio codec frames, non-zero only for in-context
|
||||
voice cloning (Base task); `0` for CustomVoice/VoiceDesign or x-vector-only
|
||||
- `output_tokens`: generated codec tokens
|
||||
- `total_tokens` = `input_tokens` + `output_tokens`
|
||||
Returns binary audio data with appropriate `Content-Type` header (e.g., `audio/wav`).
|
||||
|
||||
### Voices Endpoint
|
||||
|
||||
@@ -284,9 +233,9 @@ Server -> Client:
|
||||
|
||||
| Message | Description |
|
||||
|---------|-------------|
|
||||
| `{"type": "audio.start", "sentence_index": 0, "sentence_text": "...", "format": "pcm", "sample_rate": 24000}` | Audio generation starting for the buffered input |
|
||||
| `{"type": "audio.start", "sentence_index": 0, "sentence_text": "...", "format": "pcm", "sample_rate": 24000}` | Audio generation starting for a sentence |
|
||||
| Binary frame | Raw audio bytes (one or more PCM chunks when `stream_audio=true`) |
|
||||
| `{"type": "audio.done", "sentence_index": 0, "total_bytes": 96000, "error": false}` | Audio complete for the buffered input |
|
||||
| `{"type": "audio.done", "sentence_index": 0, "total_bytes": 96000, "error": false}` | Audio complete for a sentence |
|
||||
| `{"type": "session.done", "total_sentences": N}` | Session complete |
|
||||
| `{"type": "error", "message": "..."}` | Non-fatal error |
|
||||
|
||||
@@ -296,7 +245,8 @@ All REST API parameters are supported, plus:
|
||||
|
||||
| Parameter | Type | Default | Description |
|
||||
|-----------|------|---------|-------------|
|
||||
| `stream_audio` | bool | false | Stream one or more PCM chunks for the buffered input over WebSocket |
|
||||
| `stream_audio` | bool | false | Stream one or more PCM chunks per sentence over WebSocket |
|
||||
| `split_granularity` | string | "sentence" | Text splitting granularity |
|
||||
|
||||
|
||||
```bash
|
||||
@@ -541,13 +491,7 @@ Each item in the `items` array requires only `input` (the text). All other field
|
||||
"index": 0,
|
||||
"status": "success",
|
||||
"audio_data": "<base64-encoded audio>",
|
||||
"media_type": "audio/wav",
|
||||
"usage": {
|
||||
"input_tokens": 119,
|
||||
"output_tokens": 77,
|
||||
"total_tokens": 196,
|
||||
"input_token_details": { "text_tokens": 18, "audio_tokens": 101 }
|
||||
}
|
||||
"media_type": "audio/wav"
|
||||
},
|
||||
{
|
||||
"index": 1,
|
||||
@@ -561,18 +505,6 @@ Each item in the `items` array requires only `input` (the text). All other field
|
||||
}
|
||||
```
|
||||
|
||||
Each successful item carries a `usage` object (errored items omit it):
|
||||
|
||||
- `input_tokens` = `text_tokens` + `audio_tokens`
|
||||
- `text_tokens`: tokens of the synthesized text (`input` plus `instructions`)
|
||||
- `audio_tokens`: reference-audio codec frames, non-zero only for in-context
|
||||
voice cloning (Base task); `0` for CustomVoice/VoiceDesign or x-vector-only
|
||||
- `output_tokens`: generated codec tokens
|
||||
- `total_tokens` = `input_tokens` + `output_tokens`
|
||||
|
||||
This is the same `usage` object emitted on the terminal `speech.audio.done` event
|
||||
of the single endpoint's [SSE stream](#response-format) (`stream_format="sse"`).
|
||||
|
||||
### Examples
|
||||
|
||||
**Basic batch with shared defaults:**
|
||||
|
||||
@@ -20,8 +20,6 @@ The full set of backends and their platform defaults is in the **Backend Options
|
||||
| `TORCH_SDPA` | PyTorch `scaled_dot_product_attention` with the default backend dispatcher. Most conservative; always available. |
|
||||
| `SAGE_ATTN` | SageAttention 2.2 — INT8-quantized attention with FP16 accumulation. Lossy but typically visually indistinguishable on diffusion outputs. Requires `sageattention`. |
|
||||
| `SAGE_ATTN_3` | Requires `sageattn3` from `SageAttention/sageattention3_blackwell`. CUDA only, intended for Blackwell GPUs, with GQA/MQA requests falling back to PyTorch SDPA. |
|
||||
| `FLASH_ATTN_HUB` | FlashAttention 2 from HuggingFace `kernels` library. Useful for train/rollout alignment. |
|
||||
| `FLASH_ATTN_3_HUB` | FlashAttention 3 from HuggingFace `kernels` library. CUDA Hopper (sm_90+) only; falls back to `FLASH_ATTN_HUB` on older GPUs. |
|
||||
|
||||
|
||||
## Configuration
|
||||
@@ -212,36 +210,6 @@ Notes:
|
||||
- `SAGE_ATTN_3` is only selected on CUDA when `sageattn3` is importable and the GPU is Blackwell-class.
|
||||
- SageAttention3's Blackwell kernel assumes `Hq == Hkv`. In vLLM-Omni, GQA/MQA diffusion requests fall back to PyTorch SDPA for correctness.
|
||||
|
||||
## HuggingFace Kernels Hub Backends
|
||||
|
||||
To achieve perfect numerical consistency between **training** (typically using HuggingFace Diffusers with Hub kernels) and **serving/rollout** (using vLLM-Omni), you can use the Hub-based attention backends. This eliminates numerical drift / sampling divergence caused by executing different local kernel versions during rollout.
|
||||
|
||||
The following backend options are supported:
|
||||
- `FLASH_ATTN_HUB` (HuggingFace `kernels-community/flash-attn2`)
|
||||
- `FLASH_ATTN_3_HUB` (HuggingFace `kernels-community/flash-attn3`, Hopper sm_90+ only)
|
||||
|
||||
### Installation
|
||||
|
||||
To use these backends, you must install the `kernels` library:
|
||||
|
||||
```bash
|
||||
pip install kernels==0.14.1
|
||||
```
|
||||
|
||||
If the `kernels` library is not available in the environment, vLLM-Omni will log a warning and fall back gracefully to the corresponding local backend implementations (`FLASH_ATTN`). On CUDA GPUs below Hopper (compute capability < 9.0), `FLASH_ATTN_3_HUB` falls back to `FLASH_ATTN_HUB`.
|
||||
|
||||
### Usage
|
||||
|
||||
Select a Hub backend using the global CLI flag or environment variables:
|
||||
|
||||
```bash
|
||||
# Environment variable
|
||||
export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN_HUB
|
||||
|
||||
# CLI flag
|
||||
vllm-omni serve <model> --diffusion-attention-backend FLASH_ATTN_HUB
|
||||
```
|
||||
|
||||
## Usage Examples
|
||||
|
||||
### Default (auto-route)
|
||||
|
||||
@@ -197,7 +197,6 @@ Factory function `get_offload_backend()` selects the appropriate backend based o
|
||||
| StableDiffusion3Pipeline | `stabilityai/stable-diffusion-3.5-medium` | `SD3Transformer2DModel` | - | ✓ | `"transformer_blocks"` |
|
||||
| Wan22I2VPipeline | `Wan-AI/Wan2.2-I2V-A14B-Diffusers` | `WanTransformer3DModel` | ✓ | ✓ | `"blocks"` |
|
||||
| Wan22Pipeline | `Wan-AI/Wan2.2-T2V-A14B-Diffusers` | `WanTransformer3DModel` | ✓ | ✓ | `"blocks"` |
|
||||
| SoulXSingerPipeline / SoulXSingerSVCPipeline | `Soul-AILab/SoulX-Singer` | `DiffLlama` (`cfm_decoder.model.diff_estimator`) | ✓ | ✓ | `"layers"` |
|
||||
| BagelPipeline | `ByteDance-Seed/BAGEL-7B-MoT` | `Qwen2MoTModel` | - | ✓ | `"layers"`, `"customized modules"` |
|
||||
|
||||
**Notes:**
|
||||
|
||||
@@ -10,7 +10,7 @@ This guide covers the parallelism methods in vLLM-Omni for speeding up diffusion
|
||||
| **[Sequence Parallelism](sequence_parallel.md)** | Splits sequence dimension across GPUs (Ulysses-SP, Ring-Attention, or hybrid) for high-resolution images and videos |
|
||||
| **[CFG-Parallel](cfg_parallel.md)** | Runs CFG positive/negative branches on separate GPUs for ~1.8x speedup on guided generation |
|
||||
| **[Pipeline Parallelism](pipeline_parallel.md)** | Splits the denoising transformer block-wise across sequential GPU stages to reduce per-GPU model memory |
|
||||
| **[VAE Parallelism](vae_parallelism.md)** | Distributes VAE decode spatially across GPUs to reduce peak VAE memory |
|
||||
| **[VAE Patch Parallelism](vae_patch_parallel.md)** | Distributes VAE decode spatially across GPUs to reduce peak VAE memory |
|
||||
| **[HSDP](hsdp.md)** | Shards full model weights via PyTorch FSDP2 to enable large-model inference on memory-constrained GPUs |
|
||||
| **[Expert Parallelism](expert_parallel.md)** | Shards MoE expert blocks across GPUs for MoE models (e.g. HunyuanImage3.0) |
|
||||
|
||||
|
||||
+3
-49
@@ -1,4 +1,4 @@
|
||||
# VAE Parallelism Guide
|
||||
# VAE Patch Parallelism Guide
|
||||
|
||||
|
||||
## Table of Content
|
||||
@@ -15,7 +15,7 @@
|
||||
|
||||
## Overview
|
||||
|
||||
VAE parallelism distributes VAE (Variational AutoEncoder) decode/encode work across multiple GPUs. This guide covers VAE patch/tile parallelism, which splits latent space into spatial tiles or patches, and Wan spatial-shard decode, which shards decoder feature maps along height or width.
|
||||
VAE Patch Parallelism distributes the VAE (Variational AutoEncoder) decode/encode computation across multiple GPUs by splitting the latent space into spatial tiles or patches. Each GPU processes a subset of tiles in parallel, significantly reducing peak memory consumption during the VAE decode stage while maintaining output quality.
|
||||
|
||||
This is particularly useful for:
|
||||
- **High-resolution image generation** where VAE decode can become a memory bottleneck
|
||||
@@ -25,7 +25,7 @@ This is particularly useful for:
|
||||
See supported models list in [Supported Models](../../diffusion_features.md#supported-models).
|
||||
|
||||
|
||||
VAE patch parallelism uses two strategies based on image size:
|
||||
VAE Patch Parallelism uses two strategies based on image size:
|
||||
|
||||
| Strategy | Use Case | How It Works | Overlap Handling | Output Quality |
|
||||
|----------|----------|--------------|------------------|----------------|
|
||||
@@ -113,7 +113,6 @@ In `DiffusionParallelConfig`:
|
||||
| Parameter | Type | Default | Description |
|
||||
|-----------|------|---------|-------------|
|
||||
| `vae_patch_parallel_size` | int | 1 | Number of GPUs for VAE patch/tile parallelism. Set to 2 or higher to enable. Should typically match `tensor_parallel_size` as they share the same process group. |
|
||||
| `vae_parallel_mode` | str | `"tile"` | VAE parallel decode strategy: `"tile"` (default tile/patch parallel decode), `"spatial_shard_height"`, or `"spatial_shard_width"` (spatially-sharded decode, Wan only). See [Spatially-Sharded Decode](#spatially-sharded-decode-wan). |
|
||||
|
||||
Additional requirements:
|
||||
|
||||
@@ -126,51 +125,6 @@ Additional requirements:
|
||||
|
||||
---
|
||||
|
||||
## Spatially-Sharded Decode (Wan)
|
||||
|
||||
The default `vae_parallel_mode="tile"` distributes whole tiles across ranks. For the **Wan** VAE there is an alternative decode strategy, **spatially-sharded decode**, selected via `vae_parallel_mode="spatial_shard_height"` or `vae_parallel_mode="spatial_shard_width"`.
|
||||
|
||||
Instead of assigning independent tiles to ranks, spatial-shard decode shards the decoder feature maps along the height (`spatial_shard_height`) or width (`spatial_shard_width`) dimension and exchanges halo rows/columns between neighboring ranks around the spatial convolutions. This keeps the receptive field correct across shard boundaries, so the result matches the single-GPU decode within numerical tolerance.
|
||||
|
||||
```python
|
||||
from vllm_omni import Omni
|
||||
from vllm_omni.diffusion.data import DiffusionParallelConfig
|
||||
|
||||
omni = Omni(
|
||||
model="Wan-AI/Wan2.1-T2V-1.3B-Diffusers",
|
||||
parallel_config=DiffusionParallelConfig(
|
||||
tensor_parallel_size=2,
|
||||
vae_patch_parallel_size=2, # must match the DiT group size
|
||||
vae_parallel_mode="spatial_shard_width", # or "spatial_shard_height"
|
||||
),
|
||||
)
|
||||
```
|
||||
|
||||
Or from the CLI / serving entrypoint:
|
||||
|
||||
```bash
|
||||
vllm serve Wan-AI/Wan2.1-T2V-1.3B-Diffusers --omni \
|
||||
--tensor-parallel-size 2 \
|
||||
--vae-patch-parallel-size 2 \
|
||||
--vae-parallel-mode spatial_shard_width
|
||||
```
|
||||
|
||||
**Constraints and behavior:**
|
||||
|
||||
- Spatial-shard decode is **decode-only** and currently implemented for the **Wan** VAE. Other models ignore `spatial_shard_*` modes.
|
||||
- It requires `vae_patch_parallel_size` to **match the DiT process group size**. If it does not, the VAE logs a warning and **falls back to tile-parallel decode** at runtime.
|
||||
- `spatial_shard_height` and `spatial_shard_width` are mutually exclusive for a given VAE instance (the decoder is patched in place for a single split dimension).
|
||||
|
||||
For end-to-end latency/throughput, launch serving with the desired `vae_parallel_mode` and use the existing diffusion serving benchmark:
|
||||
|
||||
```bash
|
||||
python3 benchmarks/diffusion/diffusion_benchmark_serving.py \
|
||||
--endpoint /v1/videos --dataset random --task t2v --num-prompts 1 \
|
||||
--height 480 --width 832 --num-frames 17 --max-concurrency 1
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Best Practices
|
||||
|
||||
### When to Use
|
||||
@@ -1,110 +0,0 @@
|
||||
# Request-Level Batching
|
||||
|
||||
Request-level batching lets diffusion serving combine multiple compatible
|
||||
logical requests into one pipeline `forward()` call. Each prompt remains a
|
||||
separate request with its own `request_id`, sampling parameters, seed, output,
|
||||
error, and abort state. The scheduler decides which requests can run together.
|
||||
|
||||
!!! warning "Prompt List Semantics"
|
||||
|
||||
Diffusion request-level batching does not support a top-level packed
|
||||
list-prompt request. Submit multiple prompts as independent requests and let
|
||||
the scheduler batch compatible in-flight requests. Multimodal payloads stay
|
||||
inside a single prompt dict, for example
|
||||
`{"prompt": "...", "multi_modal_data": {"image": image}}`.
|
||||
|
||||
## Enablement
|
||||
|
||||
Increase `max_num_seqs` above `1` to allow the request scheduler to keep more
|
||||
than one compatible request active:
|
||||
|
||||
```bash
|
||||
vllm serve Qwen/Qwen-Image --omni \
|
||||
--port 8091 \
|
||||
--max-num-seqs 4
|
||||
```
|
||||
|
||||
For bursty online traffic, you can also set a small admission wait window. This
|
||||
lets the engine wait briefly before the first `schedule()` of a new wave so
|
||||
nearby compatible requests can arrive and share the same fused forward pass:
|
||||
|
||||
```bash
|
||||
vllm serve Qwen/Qwen-Image --omni \
|
||||
--port 8091 \
|
||||
--max-num-seqs 4 \
|
||||
--request-batch-max-wait-ms 20
|
||||
```
|
||||
|
||||
`--request-batch-max-wait-ms 0` is the default and disables admission waiting,
|
||||
so there is no added wait latency.
|
||||
|
||||
For deploy YAMLs, configure the diffusion stage engine args:
|
||||
|
||||
```yaml
|
||||
stage_args:
|
||||
- stage_id: 0
|
||||
stage_type: diffusion
|
||||
engine_args:
|
||||
max_num_seqs: 4
|
||||
request_batch_max_wait_ms: 20
|
||||
```
|
||||
|
||||
## Compatibility
|
||||
|
||||
Only pipelines that declare request-batch support use the fused request-batch
|
||||
path. The engine validates that the pipeline `forward()` uses the request-batch
|
||||
contract and returns `list[DiffusionOutput]`. Pipelines that do not support this
|
||||
contract do not use fused `pipeline.forward(batch)`; scheduled requests are
|
||||
executed through per-request worker calls.
|
||||
|
||||
The scheduler batches only compatible requests. Compatibility is based on
|
||||
shape-sensitive and guidance-sensitive sampling fields, including resolution,
|
||||
frame count, CFG settings, output count, and LoRA identity. Requests with
|
||||
different LoRA adapters or scales are kept in separate batches so the worker
|
||||
activates one adapter per batch.
|
||||
|
||||
Request-level batching applies when `step_execution=False`. For the separate
|
||||
step-wise runtime, see [Step Execution](step_execution.md).
|
||||
|
||||
## Tuning
|
||||
|
||||
- `max_num_seqs` caps the number of active compatible requests in one scheduler
|
||||
wave.
|
||||
- `request_batch_max_wait_ms` is an upper bound on extra admission wait before a
|
||||
new wave starts. Keep it small for latency-sensitive serving; values such as
|
||||
`10` to `50` ms are a practical starting range for bursty HTTP ingress.
|
||||
- `0` disables admission waiting and preserves the lowest first-request latency.
|
||||
- FIFO ordering is conservative: an incompatible request at the front of the
|
||||
waiting queue can block later compatible requests from joining the current
|
||||
batch.
|
||||
|
||||
## Python API
|
||||
|
||||
When constructing `Omni`, pass the same engine arguments:
|
||||
|
||||
```python
|
||||
from vllm_omni.entrypoints.omni import Omni
|
||||
|
||||
omni = Omni(
|
||||
model="Qwen/Qwen-Image",
|
||||
max_num_seqs=4,
|
||||
request_batch_max_wait_ms=20.0,
|
||||
)
|
||||
|
||||
outputs = omni.generate(
|
||||
[
|
||||
"a cup of coffee on a table",
|
||||
"a toy dinosaur on a sandy beach",
|
||||
"a fox waking up in bed and yawning",
|
||||
]
|
||||
)
|
||||
```
|
||||
|
||||
`Omni.generate([...])` submits each list item as its own logical diffusion
|
||||
request. The runtime may batch those requests internally when their sampling
|
||||
parameters are compatible.
|
||||
|
||||
## For Contributors
|
||||
|
||||
For implementation details and model-author guidance, see
|
||||
[Request-Level Batching for Diffusion](../../design/feature/diffusion_request_level_batching.md).
|
||||
@@ -61,7 +61,7 @@ Memory optimization methods help reduce GPU memory usage, enabling inference on
|
||||
|--------|-------------|----------|
|
||||
| **[CPU Offload](diffusion/cpu_offload_diffusion.md)** | Offloads model components to CPU memory | Limited VRAM, large models on consumer GPUs |
|
||||
| **[Quantization](quantization/overview.md)** | Reduces transformer stages from BF16 to FP8/INT8/etc. | Limited VRAM, minimal accuracy loss |
|
||||
| **[VAE Parallelism](diffusion/parallelism/vae_parallelism.md)** | Distributes VAE decode work across GPUs | High-resolution generation with reduced VAE memory peak |
|
||||
| **[VAE Patch Parallelism](diffusion/parallelism/vae_patch_parallel.md)** | Distributes VAE decode tiling across GPUs | High-resolution generation with reduced VAE memory peak |
|
||||
|
||||
### Extensions
|
||||
|
||||
@@ -75,17 +75,13 @@ Extension methods add specialized capabilities to diffusion models beyond standa
|
||||
|
||||
### Execution Modes
|
||||
|
||||
Execution modes control how the diffusion pipeline processes requests and
|
||||
denoise steps.
|
||||
Execution modes control how the diffusion pipeline processes denoise steps.
|
||||
|
||||
| Method | Description | Best For |
|
||||
|--------|-------------|----------|
|
||||
| **[Request-Level Batching](diffusion/request_batching.md)** | Scheduler batches compatible independent diffusion requests into one pipeline forward pass | Bursty online serving and multi-request throughput |
|
||||
| **[Step Execution](diffusion/step_execution.md)** | Per-step denoise execution with mid-request abort support | Request cancellation between denoise steps, fine-grained execution control |
|
||||
|
||||
**Note:** Request-level batching is available for pipelines that declare the
|
||||
request-batch forward contract. Step execution is currently supported by
|
||||
QwenImagePipeline only. See [Supported Models](#supported-models) for details.
|
||||
**Note:** Step execution is currently supported by QwenImagePipeline only. See [Supported Models](#supported-models) for details.
|
||||
|
||||
### Quantization Methods
|
||||
|
||||
@@ -126,14 +122,14 @@ The following tables show which models support each feature:
|
||||
| **MagiHuman** | ❌ | ❌ | ❌ | ❓ | ✅ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ |
|
||||
| **MammothModa2(T2I)** | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ |
|
||||
| **Nextstep_1(T2I)** | ❓ | ❓ | ❌ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ |
|
||||
| **OmniGen2** | ❌ | ✅ | ✅ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ |
|
||||
| **OmniGen2** | ❌ | ✅ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ | ❌ | ❌ | ❌ |
|
||||
| **Ovis-Image** | ❌ | ✅ | ❌ | ✅ | ❌ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ |
|
||||
| **Qwen-Image** | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ (decode) | ✅ | ✅ |
|
||||
| **Qwen-Image-2512** | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ (decode) | ✅ | ✅ |
|
||||
| **Qwen-Image-Edit** | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ (decode) | ❌ | ❌ |
|
||||
| **Qwen-Image-Edit-2509** | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ (decode) | ✅ | ❌ | ❌ |
|
||||
| **Qwen-Image-Layered** | ✅ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ (decode) | ❌ | ❌ |
|
||||
| **SenseNova-U1** | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ |
|
||||
| **SenseNova-U1** | ❌ | ✅ | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | ❌ | ❌ | ❌ |
|
||||
| **Stable-Diffusion-XL** | ❌ | ❌ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ (decode) | ❌ | ❌ |
|
||||
| **Stable-Diffusion3.5** | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ✅ | ✅ (decode) | ❌ | ❌ |
|
||||
| **Z-Image** | ✅ | ✅ | ✅ | ❓ | ✅ (TP=2 only) | ❌ | ✅ | ❌ | ✅ (decode) | ✅ | ❌ |
|
||||
@@ -150,7 +146,7 @@ The following tables show which models support each feature:
|
||||
| Model | ⚡TeaCache | ⚡Cache-DiT | 🔀SP (Ulysses & Ring) | 🔀CFG-Parallel | 🔀Tensor-Parallel | Pipeline-Parallel | 🔀HSDP | 💾CPU Offload (Layerwise) | 💾VAE-Patch-Parallel | 💾Quantization | 🔄Step Execution |
|
||||
|------------------------------|:---------:|:----------:|:---------------------:|:--------------:|:-----------------:|:-----------------:|:------:|:-------------------------:|:--------------------:|:--------------:|:----------------:|
|
||||
| **Wan2.2** | ❌ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ (encode/decode) | ❌ | ❌ |
|
||||
| **Wan2.2-S2V** | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ (encode/decode) | ❌ | ❌ |
|
||||
| **Wan2.2-S2V** | ❌ | ✅ | ❌ | ❌ | ✅ | ❌ | ❌ | ✅ | ✅ (encode/decode) | ❌ | ❌ |
|
||||
| **Wan2.1-VACE** | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ✅ (decode) | ❌ | ❌ |
|
||||
| **LTX-2** | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ❌ |
|
||||
| **LTX-2.3** | ❌ | ✅ | ✅ | ✅ | ✅ | ❌ | ❌ | ✅ | ✅ (decode) | ❌ | ❌ |
|
||||
@@ -274,7 +270,7 @@ Measured on NVIDIA H800:
|
||||
**Memory Optimization:**
|
||||
|
||||
- **[CPU Offload Guide](diffusion/cpu_offload_diffusion.md)** - Offload model components to CPU, reduce GPU memory usage
|
||||
- **[VAE Parallelism Guide](diffusion/parallelism/vae_parallelism.md)** - Distribute VAE decode work across GPUs for high-resolution images and videos
|
||||
- **[VAE Patch Parallelism Guide](diffusion/parallelism/vae_patch_parallel.md)** - Distribute VAE decode tiling across GPUs for high-resolution images
|
||||
- **[Quantization Overview](quantization/overview.md)** - Overview of quantization methods for diffusion, multi-stage omni/TTS, and multi-stage diffusion models
|
||||
|
||||
**Extensions:**
|
||||
|
||||
@@ -3,9 +3,7 @@
|
||||
Source <https://github.com/vllm-project/vllm-omni/tree/main/examples/offline_inference/image_to_video>.
|
||||
|
||||
|
||||
This example demonstrates how to generate videos from images using Wan2.2,
|
||||
LTX-2/LTX-2.3, and HunyuanVideo-1.5 Image-to-Video models with vLLM-Omni's
|
||||
offline inference API.
|
||||
This example demonstrates how to generate videos from images using Wan2.2 Image-to-Video models with vLLM-Omni's offline inference API.
|
||||
|
||||
## Local CLI Usage
|
||||
|
||||
@@ -53,41 +51,13 @@ python image_to_video.py \
|
||||
--output i2v_output.mp4
|
||||
```
|
||||
|
||||
### LTX-2.3
|
||||
|
||||
```bash
|
||||
python image_to_video.py \
|
||||
--model dg845/LTX-2.3-Diffusers \
|
||||
--model-class-name LTX23ImageToVideoPipeline \
|
||||
--image cherry_blossom.jpg \
|
||||
--prompt "Cherry blossoms swaying gently in the breeze with synchronized ambient sound" \
|
||||
--negative-prompt "worst quality, inconsistent motion, blurry, jittery, distorted" \
|
||||
--height 384 \
|
||||
--width 512 \
|
||||
--num-frames 25 \
|
||||
--guidance-scale 4.0 \
|
||||
--num-inference-steps 20 \
|
||||
--frame-rate 24 \
|
||||
--fps 24 \
|
||||
--output ltx23_i2v_output.mp4
|
||||
```
|
||||
|
||||
Use a Diffusers-format checkpoint such as `dg845/LTX-2.3-Diffusers`; the
|
||||
upstream `Lightricks/LTX-2.3` raw safetensors repo is not directly loadable by
|
||||
this pipeline. Pass `--model-class-name LTX23ImageToVideoPipeline` to select
|
||||
the LTX-2.3 image-to-video pipeline.
|
||||
|
||||
Key arguments:
|
||||
|
||||
- `--model`: Model ID (I2V-A14B for MoE, TI2V-5B for unified T2V+I2V, or
|
||||
LTX-2/LTX-2.3).
|
||||
- `--model`: Model ID (I2V-A14B for MoE, TI2V-5B for unified T2V+I2V).
|
||||
- `--image`: Path to input image (required).
|
||||
- `--prompt`: Text description of desired motion/animation.
|
||||
- `--height/--width`: Output resolution (auto-calculated from image if not set).
|
||||
Wan dimensions should be multiples of 16; LTX dimensions should be multiples
|
||||
of 32.
|
||||
- `--num-frames`: Number of frames (model-specific default; LTX-style models
|
||||
work best with `8k + 1`).
|
||||
- `--height/--width`: Output resolution (auto-calculated from image if not set). Dimensions should be multiples of 16.
|
||||
- `--num-frames`: Number of frames (default 81).
|
||||
- `--guidance-scale` and `--guidance-scale-high`: CFG scale (applied to low/high-noise stages for MoE).
|
||||
- `--negative-prompt`: Optional list of artifacts to suppress.
|
||||
- `--boundary-ratio`: Boundary split ratio for two-stage MoE models.
|
||||
@@ -95,7 +65,6 @@ Key arguments:
|
||||
- `--sample-solver`: Wan2.2 sampling solver. Use `unipc` for the default multistep solver, or `euler` for Lightning/Distill checkpoints.
|
||||
- `--num-inference-steps`: Number of denoising steps (default 50).
|
||||
- `--fps`: Frames per second for the saved MP4 (requires `diffusers` export_to_video).
|
||||
- `--audio-sample-rate`: fallback audio sample rate for embedded audio.
|
||||
- `--output`: Path to save the generated video.
|
||||
- `--vae-use-slicing`: Enable VAE slicing for memory optimization.
|
||||
- `--vae-use-tiling`: Enable VAE tiling for memory optimization.
|
||||
|
||||
@@ -7,22 +7,29 @@ Source <https://github.com/vllm-project/vllm-omni/tree/main/examples/offline_inf
|
||||
|
||||
Download model
|
||||
```bash
|
||||
hf download bytedance-research/MammothModa2-Preview --local-dir ./MammothModa2-Preview
|
||||
hf bytedance-research/MammothModa2-Preview --local-dir ./MammothModa2-Preview
|
||||
```
|
||||
|
||||
### Text-to-Image (T2I)
|
||||
|
||||
Text-to-image now runs through the shared offline image example
|
||||
(`examples/offline_inference/text_to_image/text_to_image.py`). See the recipe
|
||||
`recipes/MammothModa2/MammothModa2-Preview.md` for the full command and the
|
||||
`extra_body` knobs (`text_guidance_scale`, `cfg_range`, `num_inference_steps`).
|
||||
```bash
|
||||
python examples/offline_inference/mammothmodal2_preview/run_mammothmoda2_t2i.py \
|
||||
--model ./MammothModa2-Preview \
|
||||
--stage-config ./vllm_omni/model_executor/stage_configs/mammoth_moda2.yaml \
|
||||
--prompt "A stylish woman riding a motorcycle in NYC, movie poster style" \
|
||||
--height 1024 \
|
||||
--width 1024 \
|
||||
--num-inference-steps 50 \
|
||||
--text-guidance-scale 4.0 \
|
||||
--out output.png
|
||||
```
|
||||
|
||||
### Image Summary
|
||||
|
||||
```bash
|
||||
python examples/offline_inference/mammothmodal2_preview/run_mammothmoda2_image_summarize.py \
|
||||
--model ./MammothModa2-Preview \
|
||||
--deploy-config ./vllm_omni/deploy/mammoth_moda2_ar.yaml \
|
||||
--stage-config ./vllm_omni/model_executor/stage_configs/mammoth_moda2_ar.yaml \
|
||||
--question "Summarize this image." \
|
||||
--image ./image.png
|
||||
```
|
||||
@@ -33,3 +40,7 @@ python examples/offline_inference/mammothmodal2_preview/run_mammothmoda2_image_s
|
||||
``````py
|
||||
--8<-- "examples/offline_inference/mammothmodal2_preview/run_mammothmoda2_image_summarize.py"
|
||||
``````
|
||||
??? abstract "run_mammothmoda2_t2i.py"
|
||||
``````py
|
||||
--8<-- "examples/offline_inference/mammothmodal2_preview/run_mammothmoda2_t2i.py"
|
||||
``````
|
||||
@@ -160,11 +160,9 @@ python examples/offline_inference/text_to_image/text_to_image.py \
|
||||
--output flux2-dev.png
|
||||
```
|
||||
|
||||
### Multiple Prompts
|
||||
### Batch Requests (Multiple Prompts)
|
||||
|
||||
You can pass multiple prompts in a single `generate` call. For diffusion
|
||||
pipelines, each prompt is submitted as a separate logical request; compatible
|
||||
requests may be automatically batched by the scheduler and runner.
|
||||
You can pass multiple prompts in a single `generate` call.
|
||||
|
||||
```python
|
||||
from vllm_omni.entrypoints.omni import Omni
|
||||
@@ -183,8 +181,17 @@ if __name__ == "__main__":
|
||||
|
||||
!!! info
|
||||
|
||||
For diffusion request-level batching controls such as `max_num_seqs`, see
|
||||
[Request-Level Batching](../../diffusion/request_batching.md).
|
||||
Not all models support batch inference, and batch requesting mostly does not provide significant
|
||||
performance improvement. This feature is primarily for interface compatibility with vLLM and to
|
||||
allow for future improvements.
|
||||
|
||||
!!! info
|
||||
|
||||
For diffusion pipelines, the stage config field `stage_args.[].runtime.max_batch_size` is 1 by
|
||||
default, and the input list is sliced into single-item requests before feeding into the diffusion
|
||||
pipeline. For models that do internally support batched inputs, you can
|
||||
[modify this configuration](https://github.com/vllm-project/vllm-omni/tree/main/configuration/stage_configs.md) to let the model accept a
|
||||
longer batch of prompts.
|
||||
|
||||
### Negative Prompts
|
||||
|
||||
|
||||
@@ -12,7 +12,6 @@ For backend selection and SageAttention usage, see the [Diffusion Attention Back
|
||||
| Model | Default Resolution | Default Frames | Default Steps | Guidance | VRAM (BF16) |
|
||||
|---|---|---|---|---|---|
|
||||
| `Wan-AI/Wan2.2-T2V-A14B-Diffusers` | 720x1280 | 81 | 40 | 4.0 | ~60 GiB |
|
||||
| `dg845/LTX-2.3-Diffusers` | 384x512 | 25 | 20 | 4.0 | 96GB-class GPU |
|
||||
| `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v` | 480x832 | 121 | 50 | 6.0 | 1×A100 80GB |
|
||||
| `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-720p_t2v` | 720x1280 | 121 | 50 | 6.0 | FP8 + VAE tiling required |
|
||||
|
||||
@@ -51,30 +50,6 @@ python text_to_video.py \
|
||||
--output ltx2_out.mp4
|
||||
```
|
||||
|
||||
### LTX-2.3
|
||||
|
||||
```bash
|
||||
python text_to_video.py \
|
||||
--model dg845/LTX-2.3-Diffusers \
|
||||
--model-class-name LTX23Pipeline \
|
||||
--prompt "Cherry blossoms swaying gently in the breeze with synchronized ambient sound" \
|
||||
--negative-prompt "worst quality, inconsistent motion, blurry, jittery, distorted" \
|
||||
--height 384 \
|
||||
--width 512 \
|
||||
--num-frames 25 \
|
||||
--num-inference-steps 20 \
|
||||
--guidance-scale 4.0 \
|
||||
--frame-rate 24 \
|
||||
--fps 24 \
|
||||
--audio-sample-rate 48000 \
|
||||
--output ltx23_t2v_output.mp4
|
||||
```
|
||||
|
||||
Use a Diffusers-format checkpoint such as `dg845/LTX-2.3-Diffusers`; the
|
||||
upstream `Lightricks/LTX-2.3` raw safetensors repo is not directly loadable by
|
||||
this pipeline. Pass `--model-class-name LTX23Pipeline` to select the LTX-2.3
|
||||
text-to-video pipeline explicitly.
|
||||
|
||||
### HunyuanVideo-1.5 (480p)
|
||||
|
||||
```bash
|
||||
@@ -135,8 +110,6 @@ python text_to_video.py \
|
||||
### Common
|
||||
|
||||
- `--model`: Diffusers model ID or local path.
|
||||
- `--model-class-name`: Optional explicit pipeline class. Use `LTX23Pipeline`
|
||||
for LTX-2.3 text-to-video.
|
||||
- `--prompt`: text description (string).
|
||||
- `--height/--width`: output resolution. Default depends on model.
|
||||
- `--num-frames`: number of frames. Default depends on model.
|
||||
@@ -151,8 +124,7 @@ python text_to_video.py \
|
||||
- `--enable-cpu-offload`: enable CPU offloading for diffusion models.
|
||||
- `--enable-layerwise-offload`: enable layerwise offloading on DiT modules.
|
||||
- `--frame-rate`: generation FPS for pipelines that require it (e.g., LTX2).
|
||||
- `--audio-sample-rate`: audio sample rate for embedded audio (when the
|
||||
pipeline returns audio; LTX-2.3 outputs 48kHz audio).
|
||||
- `--audio-sample-rate`: audio sample rate for embedded audio (when the pipeline returns audio).
|
||||
- `--quantization`: quantization method (`fp8` for FP8, `gguf` for GGUF).
|
||||
- `--flow-shift`: scheduler flow_shift parameter.
|
||||
|
||||
|
||||
@@ -9,8 +9,7 @@ vLLM-Omni supports running diffusion models with the diffusers backend, directly
|
||||
|
||||
The diffusers backend is a black-box adapter. Its primary focus is to serve diffusion models online.
|
||||
Currently, the following features are NOT yet supported.
|
||||
Community contributions are welcome to add these features when they can be
|
||||
implemented with clear Diffusers or vLLM-Omni behavior.
|
||||
It is not guaranteed whether they will be supported in the future.
|
||||
|
||||
- CFG parallel execution
|
||||
- Sequence parallel execution
|
||||
@@ -67,35 +66,6 @@ This is suitable for sampling parameters not available through the vLLM-Omni int
|
||||
When a parameter is available in the vLLM-Omni interface, it will be adapted here.
|
||||
But if that parameter is simultaneously set in both the vLLM-Omni interface and `diffusers_call_kwargs`, the **former** will take precedence (because it is set at request time).
|
||||
|
||||
### Quantization
|
||||
|
||||
Use `diffusers_load_kwargs` for Diffusers-native quantization options. If
|
||||
`diffusers_load_kwargs["quantization_config"]` is provided as a dictionary, the
|
||||
diffusers backend builds a Diffusers quantization config and lets Diffusers
|
||||
validate it before calling `DiffusionPipeline.from_pretrained()`.
|
||||
|
||||
The diffusers backend also provides a small compatibility shortcut for
|
||||
vLLM-Omni quantization configs when `diffusers_load_kwargs` does not already
|
||||
contain `quantization_config`. Currently this maps online/dynamic `fp8` and
|
||||
online/dynamic `int8` to Diffusers/TorchAO dynamic quantization for transformer
|
||||
components such as `transformer` or `transformer_2`. This path requires
|
||||
`torchao` to be installed.
|
||||
|
||||
For example, the CLI can request dynamic FP8 through the vLLM-Omni interface:
|
||||
|
||||
```bash
|
||||
vllm serve "Qwen/Qwen-Image" \
|
||||
--omni \
|
||||
--diffusion-load-format diffusers \
|
||||
--quantization-config '{"method": "fp8"}'
|
||||
```
|
||||
|
||||
Other vLLM-Omni quantization methods, such as `gguf`, `modelopt`, `mxfp4`,
|
||||
`mxfp8`, serialized checkpoints, static FP8 configs, and layer-name skip lists
|
||||
such as `ignored_layers`, are intentionally not translated by this compatibility
|
||||
shortcut. Use Diffusers-native configuration through `diffusers_load_kwargs` or
|
||||
a native vLLM-Omni pipeline for those cases.
|
||||
|
||||
### Attention Backends
|
||||
|
||||
The diffusers backend converts
|
||||
|
||||
@@ -79,7 +79,7 @@ response = client.audio.speech.create(
|
||||
response.stream_to_file("output.wav")
|
||||
```
|
||||
|
||||
Streaming PCM output (where supported) — set `stream=true`, `stream_format="audio"`, and `response_format="pcm"`:
|
||||
Streaming PCM output (where supported) — set `stream=true` with `response_format="pcm"`:
|
||||
|
||||
```bash
|
||||
curl -X POST http://localhost:8091/v1/audio/speech \
|
||||
@@ -88,7 +88,6 @@ curl -X POST http://localhost:8091/v1/audio/speech \
|
||||
"input": "Hello, how are you?",
|
||||
"voice": "default",
|
||||
"stream": true,
|
||||
"stream_format": "audio",
|
||||
"response_format": "pcm"
|
||||
}' --no-buffer | play -t raw -r 24000 -e signed -b 16 -c 1 -
|
||||
```
|
||||
@@ -415,11 +414,10 @@ curl -X POST http://localhost:8091/v1/audio/speech \
|
||||
"voice": "vivian",
|
||||
"language": "English",
|
||||
"stream": true,
|
||||
"stream_format": "audio",
|
||||
"response_format": "pcm"
|
||||
}' --no-buffer | play -t raw -r 24000 -e signed -b 16 -c 1 -
|
||||
```
|
||||
Raw PCM streaming requires `stream_format="audio"`, `response_format="pcm"`, and `async_chunk: true` on the stage config (default in `qwen3_tts.yaml`). `speed` is not supported when streaming.
|
||||
Streaming requires `response_format="pcm"` and `async_chunk: true` on the stage config (default in `qwen3_tts.yaml`). `speed` is not supported when streaming.
|
||||
|
||||
### Streaming WebSocket
|
||||
The `/v1/audio/speech/stream` endpoint accepts text incrementally, splits it at sentence boundaries, and emits one PCM stream per sentence:
|
||||
@@ -498,7 +496,7 @@ After startup, `/v1/audio/voices` lists `alice`, and `/v1/audio/speech` can use
|
||||
```bash
|
||||
python voxcpm2/gradio_demo.py
|
||||
```
|
||||
Uses an AudioWorklet-based player adapted from the Qwen3-TTS demo for gap-free playback. Raw PCM audio is streamed from the OpenAI Speech endpoint with `stream=true` and `stream_format="audio"`.
|
||||
Uses an AudioWorklet-based player adapted from the Qwen3-TTS demo for gap-free playback. Audio is streamed from the OpenAI Speech endpoint with `stream=true`.
|
||||
|
||||
---
|
||||
|
||||
|
||||
@@ -65,7 +65,7 @@ More failure modes (for example, network jitter and network interruption) will b
|
||||
|
||||
## Source of Conclusions
|
||||
|
||||
The behaviors and conclusions above are summarized from current fault injection validation results on **`Qwen3-Omni`**, **`Wan2.2`**, **`HunyuanImage-3.0-Instruct`** (DiT-only, `/v1/images/generations`), and **`VoxCPM2`** (`/v1/audio/speech`). Automated coverage lives under `tests/dfx/reliability/` (`test_reliability_qwen3_omni.py`, `test_reliability_wan22.py`, `test_reliability_hunyuan_image.py`, `test_reliability_voxcpm2.py`) and runs weekly via `.buildkite/test-weekly.yml`.
|
||||
The behaviors and conclusions above are summarized from current fault injection validation results on `Qwen3-Omni` and `Wan2.2`.
|
||||
|
||||
### Example: `SIGTERM` fault injection log (Qwen3-Omni)
|
||||
|
||||
|
||||
@@ -44,16 +44,10 @@ pip install vllm-omni[quack]
|
||||
pip install 'quack-kernels[cu13]' --extra-index-url https://download.pytorch.org/whl/cu130
|
||||
```
|
||||
|
||||
It is enabled automatically once installed (no flag needed) and is **datacenter
|
||||
Blackwell only** (`sm_100` / `sm_101` / `sm_103`, compute capability `10.x`, e.g.
|
||||
B200): quack's CuteDSL GEMM uses the 5th-gen `tcgen05` tensor-core MMA, which exists
|
||||
only on those parts. On Hopper/Ada the CUTLASS FP8 kernel already fuses bias, and on
|
||||
workstation/consumer Blackwell (`sm_120` / `sm_121`, compute capability `12.x`, e.g.
|
||||
RTX PRO 6000 / RTX 50-series) `tcgen05` is absent — so quack is **not** auto-enabled
|
||||
there and FlashInfer's native FP8 path is used instead. Set
|
||||
`VLLM_OMNI_USE_QUACK_FP8=1` to force quack on, or `VLLM_OMNI_USE_QUACK_FP8=0` to force
|
||||
the FlashInfer path. If `quack-kernels` is not installed, FP8 still works — it just
|
||||
keeps the unfused FlashInfer path.
|
||||
It is enabled automatically once installed (no flag needed) and is **Blackwell-only**:
|
||||
on Hopper/Ada the CUTLASS FP8 kernel already fuses bias, so quack is not used there.
|
||||
Set `VLLM_OMNI_USE_QUACK_FP8=0` to force the FlashInfer path. If `quack-kernels` is
|
||||
not installed, FP8 still works — it just keeps the unfused FlashInfer path.
|
||||
|
||||
#### Compile cache and warmup
|
||||
|
||||
|
||||
@@ -39,7 +39,7 @@ vLLM-Omni provides two quantization methods with different scale structures:
|
||||
| NVIDIA Blackwell GPU (SM 100+) | ⭕ |
|
||||
| NVIDIA Ada/Hopper GPU (SM 89+) | ⭕ |
|
||||
| NVIDIA Ampere GPU (SM 80+) | ⭕ |
|
||||
| AMD ROCm (gfx950 / MI355X) | ✅ |
|
||||
| AMD ROCm | ⭕ |
|
||||
| Intel XPU | ⭕ |
|
||||
| Ascend NPU (Atlas 950 A5) | ✅ |
|
||||
|
||||
|
||||
@@ -20,7 +20,7 @@ type has a different quantization scope.
|
||||
| NVIDIA Blackwell GPU (SM 100+) | ✅ | ✅ | ✅ | ⭕ | ⭕ | ✅ | ✅ | ❌ |
|
||||
| NVIDIA Ada/Hopper GPU (SM 89+) | ✅ | ✅ | ✅ | ⭕ | ⭕ | ✅ | ✅ | ❌ |
|
||||
| NVIDIA Ampere GPU (SM 80+) | ✅ | ✅ | ⭕ | ⭕ | ⭕ | ✅ | ✅ | ❌ |
|
||||
| AMD ROCm | ⭕ | ⭕ | ⭕ | ⭕ | ✅ | ⭕ | ⭕ | ❌ |
|
||||
| AMD ROCm | ⭕ | ⭕ | ⭕ | ⭕ | ⭕ | ⭕ | ⭕ | ❌ |
|
||||
| Intel XPU | ⭕ | ⭕ | ⭕ | ⭕ | ⭕ | ⭕ | ✅ | ❌ |
|
||||
| Ascend NPU | ❌ | ✅ | ❌ | ✅ | ✅ | ❌ | ❌ | ✅ |
|
||||
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
# AudioX offline inference
|
||||
|
||||
Generate audio with the [AudioX](https://zeyuet.github.io/AudioX/) MMDiT diffusion
|
||||
pipeline (`AudioXPipeline`). Six tasks: `t2a`, `t2m`, `v2a`, `v2m`, `tv2a`, `tv2m`.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
Download a vLLM-Omni weight bundle (component-sharded safetensors):
|
||||
|
||||
```bash
|
||||
huggingface-cli download zhangj1an/AudioX --local-dir ./audiox_weights
|
||||
```
|
||||
|
||||
The Hugging Face id `zhangj1an/AudioX` also works directly without prefetching.
|
||||
|
||||
## Usage
|
||||
|
||||
```bash
|
||||
# Text-to-audio only (default uses zhangj1an/AudioX from the Hub):
|
||||
python end2end.py --tasks t2a
|
||||
|
||||
# All six tasks against a local bundle and a sample video for v2*/tv2*:
|
||||
python end2end.py \
|
||||
--model ./audiox_weights \
|
||||
--video https://zeyuet.github.io/AudioX/static/samples/V2M/1XeBotOFqHA.mp4
|
||||
|
||||
# Subset of tasks, custom seed and steps:
|
||||
python end2end.py --tasks t2a tv2a --num-inference-steps 100 --seed 0
|
||||
```
|
||||
|
||||
## Arguments
|
||||
|
||||
- `--model`: HF id or local bundle path (default: `zhangj1an/AudioX`).
|
||||
- `--tasks`: any subset of `t2a t2m v2a v2m tv2a tv2m` (default: all).
|
||||
- `--video`: video file/URL — required for `v2*` and `tv2*`.
|
||||
- `--reference-audio`: optional audio prompt (audio-conditioned generation).
|
||||
- `--num-inference-steps`, `--guidance-scale`, `--seed`, `--seconds-total`,
|
||||
`--sample-rate`, `--output-dir`: generation knobs.
|
||||
|
||||
Outputs land in `<output-dir>/<task>.wav` as 16-bit stereo WAV.
|
||||
@@ -0,0 +1,110 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""End-to-end AudioX offline example covering the 6 t2*/v2*/tv2* tasks.
|
||||
|
||||
Provide a directory with the **vLLM-Omni AudioX safetensors bundle** (e.g. from
|
||||
``zhangj1an/AudioX`` on Hugging Face)::
|
||||
|
||||
huggingface-cli download zhangj1an/AudioX --local-dir ./audiox_weights
|
||||
python end2end.py --model ./audiox_weights
|
||||
python end2end.py --model ./audiox_weights --tasks t2a tv2a
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import soundfile
|
||||
import torch
|
||||
import torchaudio.functional as TF
|
||||
|
||||
from vllm_omni.entrypoints.omni import Omni
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
ROOT = Path(__file__).resolve().parent
|
||||
|
||||
SAMPLE_PROMPTS: dict[str, str] = {
|
||||
"t2a": "Fireworks burst twice, followed by a period of silence before a clock begins ticking.",
|
||||
"t2m": "Uplifting ukulele tune for a travel vlog",
|
||||
"v2a": "",
|
||||
"v2m": "",
|
||||
"tv2a": "drum beating sound and human talking",
|
||||
"tv2m": "uplifting music matching the scene",
|
||||
}
|
||||
|
||||
ALL_TASKS = ("t2a", "t2m", "v2a", "v2m", "tv2a", "tv2m")
|
||||
VIDEO_TASKS = frozenset({"v2a", "v2m", "tv2a", "tv2m"})
|
||||
TEXT_TASKS = frozenset({"t2a", "t2m", "tv2a", "tv2m"})
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
p = argparse.ArgumentParser(description="AudioX offline end-to-end (6 t2*/v2*/tv2* tasks).")
|
||||
p.add_argument("--model", default="zhangj1an/AudioX", help="HF id or local AudioX bundle path.")
|
||||
p.add_argument("--tasks", nargs="+", default=list(ALL_TASKS), choices=ALL_TASKS)
|
||||
p.add_argument("--video", default="", help="Video path / URL (required for v2*/tv2*).")
|
||||
p.add_argument("--reference-audio", default="", help="Optional audio prompt for audio-conditioned generation.")
|
||||
p.add_argument("--output-dir", default=str(ROOT / "audiox_task_outputs"))
|
||||
p.add_argument("--num-inference-steps", type=int, default=250)
|
||||
p.add_argument("--seconds-total", type=float, default=10.0)
|
||||
p.add_argument("--guidance-scale", type=float, default=6.0)
|
||||
p.add_argument("--seed", type=int, default=42)
|
||||
p.add_argument("--sample-rate", type=int, default=48000, help="Output WAV rate (resampled if != model rate).")
|
||||
return p.parse_args()
|
||||
|
||||
|
||||
def save_wav(audio: torch.Tensor, path: Path, sample_rate: int) -> None:
|
||||
"""Write 16-bit PCM WAV. ``audio`` is ``[channels, samples]`` float in [-1, 1]."""
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
soundfile.write(str(path), audio.clamp(-1.0, 1.0).cpu().T.numpy(), sample_rate, subtype="PCM_16")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
args = parse_args()
|
||||
|
||||
omni = Omni(model=args.model, model_class_name="AudioXPipeline")
|
||||
|
||||
for task in args.tasks:
|
||||
if task in VIDEO_TASKS and not args.video:
|
||||
raise SystemExit(f"task={task!r} requires --video")
|
||||
prompt = SAMPLE_PROMPTS[task] if task in TEXT_TASKS else ""
|
||||
extra: dict = {"audiox_task": task, "seconds_start": 0.0, "seconds_total": float(args.seconds_total)}
|
||||
if task in VIDEO_TASKS:
|
||||
extra["video_path"] = args.video
|
||||
if args.reference_audio:
|
||||
extra["audio_path"] = args.reference_audio
|
||||
|
||||
generator = torch.Generator(device=current_omni_platform.device_type).manual_seed(args.seed)
|
||||
t0 = time.perf_counter()
|
||||
outputs = omni.generate(
|
||||
prompt,
|
||||
OmniDiffusionSamplingParams(
|
||||
generator=generator,
|
||||
guidance_scale=args.guidance_scale,
|
||||
num_inference_steps=args.num_inference_steps,
|
||||
seed=args.seed,
|
||||
extra_args=extra,
|
||||
),
|
||||
)
|
||||
audio = outputs[0].request_output.multimodal_output.get("audio")
|
||||
if audio is None:
|
||||
raise RuntimeError(f"No audio produced for task {task!r}")
|
||||
audio = torch.as_tensor(audio).detach().cpu().float()
|
||||
if audio.ndim == 3:
|
||||
audio = audio[0]
|
||||
|
||||
model_sr = int(outputs[0].request_output.multimodal_output.get("audio_sample_rate") or 44100)
|
||||
if model_sr != args.sample_rate:
|
||||
audio = TF.resample(audio, model_sr, args.sample_rate)
|
||||
|
||||
out_path = Path(args.output_dir) / f"{task}.wav"
|
||||
save_wav(audio, out_path, args.sample_rate)
|
||||
print(f"[{task}] saved {out_path} ({time.perf_counter() - t0:.2f}s)")
|
||||
|
||||
omni.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,14 @@
|
||||
{
|
||||
"prompt": "A robotic arm, primarily white with black joints and cables, is shown in a clean, modern indoor setting with a white tabletop. The arm, equipped with a gripper holding a small, light green pitcher, is positioned above a clear glass containing a reddish-brown liquid and a spoon. The robotic arm is in the process of pouring a transparent liquid into the glass. To the left of the pitcher, there is an opened jar with a similar reddish-brown substance visible through its transparent body. In the background, a vase with white flowers and a brown couch are partially visible, adding to the contemporary ambiance. The lighting is bright, casting soft shadows on the table. The robotic arm's movements are smooth and controlled, demonstrating precision in its task. As the video progresses, the robotic arm completes the pour, leaving the glass half-filled with the reddish-brown liquid. The jar remains untouched throughout the sequence, and the spoon inside the glass remains stationary. The other robotic arm on the right side also stays stationary throughout the video. The final frame captures the robotic arm with the pitcher finishing the pour, with the glass now filled to a higher level, while the pitcher is slightly tilted but still held securely by the gripper.",
|
||||
"negative_prompt": "",
|
||||
"vision_path": "https://github.com/nvidia-cosmos/cosmos-dependencies/raw/refs/heads/assets/cosmos3/inputs/vision/robot_pouring.mp4",
|
||||
"height": 720,
|
||||
"width": 1280,
|
||||
"num_frames": 189,
|
||||
"num_inference_steps": 35,
|
||||
"guidance_scale": 6.0,
|
||||
"flow_shift": 10.0,
|
||||
"fps": 24,
|
||||
"condition_frame_indexes_vision": [0, 1],
|
||||
"condition_video_keep": "first"
|
||||
}
|
||||
@@ -2,12 +2,14 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
import PIL.Image
|
||||
import torch
|
||||
|
||||
from vllm_omni.diffusion.data import OmniDiffusionConfig
|
||||
from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig
|
||||
from vllm_omni.diffusion.models.qwen_image.pipeline_qwen_image_edit import QwenImageEditPipeline
|
||||
from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch
|
||||
from vllm_omni.diffusion.request import OmniDiffusionRequest
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@@ -16,14 +18,58 @@ class CustomPipeline(QwenImageEditPipeline):
|
||||
def __init__(self, *, od_config: OmniDiffusionConfig, prefix: str = ""):
|
||||
super().__init__(od_config=od_config, prefix=prefix)
|
||||
|
||||
def forward(self, req: DiffusionRequestBatch):
|
||||
def forward(
|
||||
self,
|
||||
req: OmniDiffusionRequest,
|
||||
prompt: str | list[str] | None = None,
|
||||
negative_prompt: str | list[str] | None = None,
|
||||
image: PIL.Image.Image | torch.Tensor | None = None,
|
||||
true_cfg_scale: float = 4.0,
|
||||
height: int | None = None,
|
||||
width: int | None = None,
|
||||
num_inference_steps: int = 50,
|
||||
sigmas: list[float] | None = None,
|
||||
guidance_scale: float = 1.0,
|
||||
num_images_per_prompt: int = 1,
|
||||
generator: torch.Generator | list[torch.Generator] | None = None,
|
||||
latents: torch.Tensor | None = None,
|
||||
prompt_embeds: torch.Tensor | None = None,
|
||||
prompt_embeds_mask: torch.Tensor | None = None,
|
||||
negative_prompt_embeds: torch.Tensor | None = None,
|
||||
negative_prompt_embeds_mask: torch.Tensor | None = None,
|
||||
output_type: str | None = "pil",
|
||||
attention_kwargs: dict[str, Any] | None = None,
|
||||
callback_on_step_end_tensor_inputs: list[str] = ["latents"],
|
||||
max_sequence_length: int = 512,
|
||||
) -> DiffusionOutput:
|
||||
"""Forward pass for image editing with dummy trajectory data."""
|
||||
# Customize num_inference_steps that best suits your model
|
||||
actual_num_steps = req.sampling_params.num_inference_steps or 50
|
||||
req.sampling_params.num_inference_steps = actual_num_steps
|
||||
|
||||
# Call parent's forward to get the normal output
|
||||
output = super().forward(req=req)
|
||||
output = super().forward(
|
||||
req=req,
|
||||
prompt=prompt,
|
||||
negative_prompt=negative_prompt,
|
||||
image=image,
|
||||
true_cfg_scale=true_cfg_scale,
|
||||
height=height,
|
||||
width=width,
|
||||
num_inference_steps=num_inference_steps,
|
||||
sigmas=sigmas,
|
||||
guidance_scale=guidance_scale,
|
||||
num_images_per_prompt=num_images_per_prompt,
|
||||
generator=generator,
|
||||
latents=latents,
|
||||
prompt_embeds=prompt_embeds,
|
||||
prompt_embeds_mask=prompt_embeds_mask,
|
||||
negative_prompt_embeds=negative_prompt_embeds,
|
||||
negative_prompt_embeds_mask=negative_prompt_embeds_mask,
|
||||
output_type=output_type,
|
||||
attention_kwargs=attention_kwargs,
|
||||
callback_on_step_end_tensor_inputs=callback_on_step_end_tensor_inputs,
|
||||
max_sequence_length=max_sequence_length,
|
||||
)
|
||||
|
||||
# Get actual num_inference_steps used
|
||||
actual_num_steps = req.sampling_params.num_inference_steps or num_inference_steps
|
||||
|
||||
# Create dummy trajectory data
|
||||
dummy_trajectory_latents = torch.randn(actual_num_steps, 1, 16, 64, 64, dtype=torch.float32)
|
||||
|
||||
@@ -74,12 +74,6 @@ def parse_args():
|
||||
parser.add_argument("--sys-type", type=str, default=None, help="Override system prompt type.")
|
||||
parser.add_argument("--deploy-config", type=str, default=None, help="Custom deploy YAML path.")
|
||||
parser.add_argument("--stage-configs-path", type=str, default=None, help="Custom legacy stage config YAML path.")
|
||||
parser.add_argument(
|
||||
"--stream",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Enable streaming CoT display; print AR text token-by-token in real time.",
|
||||
)
|
||||
parser.add_argument("--log-stats", action="store_true", default=False)
|
||||
parser.add_argument("--init-timeout", type=int, default=300, help="Initialization timeout in seconds.")
|
||||
parser.add_argument("--enforce-eager", action="store_true", help="Disable torch.compile.")
|
||||
@@ -228,8 +222,6 @@ def main():
|
||||
prompt_dict["modalities"] = ["text"]
|
||||
formatted_prompts.append(prompt_dict)
|
||||
|
||||
from vllm.sampling_params import RequestOutputKind
|
||||
|
||||
params_list = list(omni.default_sampling_params_list)
|
||||
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
@@ -261,10 +253,6 @@ def main():
|
||||
sp.width = args.width
|
||||
elif hasattr(sp, "stop_token_ids"):
|
||||
sp.stop_token_ids = ar_stop_token_ids
|
||||
# When --stream is set, request DELTA output from the AR stage
|
||||
# so we can display CoT text token-by-token in real time.
|
||||
if args.stream and hasattr(sp, "output_kind"):
|
||||
sp.output_kind = RequestOutputKind.DELTA
|
||||
|
||||
print(f"\n{'=' * 60}")
|
||||
print("HunyuanImage-3.0 Generation Configuration:")
|
||||
@@ -293,32 +281,22 @@ def main():
|
||||
print(f" Prompts: {prompts}")
|
||||
print(f"{'=' * 60}\n")
|
||||
|
||||
# When --stream is set, print AR CoT text token-by-token in real time.
|
||||
# Otherwise, collect and print the full AR text once when stage 0 finishes.
|
||||
omni_outputs = omni.generate(
|
||||
prompts=formatted_prompts,
|
||||
sampling_params_list=params_list,
|
||||
py_generator=True,
|
||||
use_tqdm=False,
|
||||
)
|
||||
omni_outputs = list(omni.generate(prompts=formatted_prompts, sampling_params_list=params_list))
|
||||
img_idx = 0
|
||||
for req_output in omni_outputs:
|
||||
ro = getattr(req_output, "request_output", None)
|
||||
stage_id = getattr(req_output, "stage_id", None)
|
||||
txt = ""
|
||||
if ro and getattr(ro, "outputs", None):
|
||||
txt = "".join(getattr(o, "text", "") or "" for o in ro.outputs)
|
||||
if not txt:
|
||||
ar_text = getattr(req_output, "custom_output", {}).get("ar_generated_text")
|
||||
if isinstance(ar_text, list):
|
||||
txt = "\n".join(text for text in ar_text if text)
|
||||
else:
|
||||
txt = ar_text or ""
|
||||
if txt:
|
||||
print(f"[Output] Text:\n{txt}")
|
||||
|
||||
# AR stage text — each CompletionOutput.text is already a delta when
|
||||
# output_kind=DELTA, so we can print it directly (matching the pattern
|
||||
# in serving_chat.py).
|
||||
if stage_id == 0 and ro and getattr(ro, "outputs", None):
|
||||
for o in ro.outputs:
|
||||
text = getattr(o, "text", "") or ""
|
||||
if text:
|
||||
print(text, end="", flush=True)
|
||||
# Non-streaming: one shot with full text — emit a trailing newline.
|
||||
if not args.stream:
|
||||
print(flush=True)
|
||||
|
||||
# Collect images from diffusion stage
|
||||
images = getattr(req_output, "images", None)
|
||||
if not images and ro and hasattr(ro, "images"):
|
||||
images = ro.images
|
||||
@@ -326,7 +304,7 @@ def main():
|
||||
for j, img in enumerate(images):
|
||||
save_path = os.path.join(args.output, f"output_{img_idx}_{j}.png")
|
||||
img.save(save_path)
|
||||
print(f"\n[Output] Saved image to {save_path}")
|
||||
print(f"[Output] Saved image to {save_path}")
|
||||
img_idx += 1
|
||||
|
||||
|
||||
|
||||
@@ -138,15 +138,6 @@ def parse_args() -> argparse.Namespace:
|
||||
required=True,
|
||||
help="Path(s) to input image file(s) (PNG, JPG, etc.). Can specify multiple images.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--deploy-config",
|
||||
type=str,
|
||||
default=None,
|
||||
help=(
|
||||
"Path to a deploy YAML. Required for multi-stage image-edit pipelines "
|
||||
"whose deploy config is not auto-loaded."
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--prompt",
|
||||
type=str,
|
||||
@@ -358,12 +349,7 @@ def parse_args() -> argparse.Namespace:
|
||||
parser.add_argument(
|
||||
"--enforce-eager",
|
||||
action="store_true",
|
||||
default=None,
|
||||
help=(
|
||||
"Disable torch.compile and force eager execution. Left unset (None) "
|
||||
"so it is only forwarded when explicitly given; "
|
||||
"otherwise the per-stage deploy YAML value wins."
|
||||
),
|
||||
help="Disable torch.compile and force eager execution.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--vae-use-slicing",
|
||||
@@ -482,7 +468,7 @@ def main():
|
||||
}
|
||||
|
||||
# Initialize Omni with appropriate pipeline
|
||||
omni_kwargs: dict[str, Any] = dict(
|
||||
omni = Omni(
|
||||
model=args.model,
|
||||
enable_layerwise_offload=args.enable_layerwise_offload,
|
||||
vae_use_slicing=args.vae_use_slicing,
|
||||
@@ -490,15 +476,11 @@ def main():
|
||||
cache_backend=args.cache_backend,
|
||||
cache_config=cache_config,
|
||||
parallel_config=parallel_config,
|
||||
enforce_eager=args.enforce_eager,
|
||||
enable_cpu_offload=args.enable_cpu_offload,
|
||||
enable_diffusion_pipeline_profiler=args.enable_diffusion_pipeline_profiler,
|
||||
profiler_config=args.profiler_config,
|
||||
)
|
||||
if args.enforce_eager is not None:
|
||||
omni_kwargs["enforce_eager"] = args.enforce_eager
|
||||
if args.deploy_config:
|
||||
omni_kwargs["deploy_config"] = args.deploy_config
|
||||
omni = Omni(**omni_kwargs)
|
||||
model_class_name = get_model_class_name(omni)
|
||||
declared_extra_body_params = get_extra_body_params(model_class_name)
|
||||
print("Pipeline loaded")
|
||||
|
||||
@@ -1,142 +1,15 @@
|
||||
# Image-To-Video
|
||||
|
||||
This shared example generates videos from images with VACE, Wan2.2,
|
||||
LTX-2/LTX-2.3, HunyuanVideo-1.5, Cosmos3, and other compatible pipelines.
|
||||
This example demonstrates how to generate videos from images using Wan2.2 Image-to-Video models with vLLM-Omni's offline inference API.
|
||||
|
||||
- `image_to_video.py`: command-line script for single video generation with advanced options.
|
||||
## Local CLI Usage
|
||||
|
||||
## Table of Contents
|
||||
|
||||
- [Overview](#overview)
|
||||
- [Prerequisites](#prerequisites)
|
||||
- [Quick Start](#quick-start)
|
||||
- [Key Arguments](#key-arguments)
|
||||
- [More CLI Examples](#more-cli-examples)
|
||||
- [Advanced Features](#advanced-features)
|
||||
- [FAQ](#faq)
|
||||
|
||||
## Overview
|
||||
|
||||
This folder provides a unified CLI script for image-to-video generation using vLLM-Omni diffusion/video pipelines. The script selects practical defaults for supported model families while still exposing common sampling, memory, and parallelism options.
|
||||
|
||||
### Supported Models
|
||||
|
||||
| Model | Default Resolution | Default Frames | Default Steps | Guidance | VRAM Notes |
|
||||
| ----- | ------------------ | -------------- | ------------- | -------- | ---------- |
|
||||
| `Wan-AI/Wan2.2-I2V-A14B-Diffusers` | 480 x 832 | 81 | 50 | 5.0 | Around 60 GiB BF16 for basic single-card usage |
|
||||
| `Wan-AI/Wan2.2-TI2V-5B-Diffusers` | 480 x 832 | 81 | 50 | 4.0 | Around 20–25 GiB BF16, smallest I2V model |
|
||||
| `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_i2v` | 480 x 832 | 121 | 50 | 6.0 | Around 100 GiB at default settings; the example enables `--enable-cpu-offload` + VAE tiling/slicing to fit an 80 GiB card |
|
||||
| LTX2 (local path + `--model-class-name LTX2ImageToVideoPipeline`) | 512 x 768 | 121 | 40 | 4.0 | Memory use depends on frame count and tensor parallelism |
|
||||
|
||||
!!! info
|
||||
Peak VRAM: based on basic single-card usage, batch size = 1, without any acceleration/optimization features. Some model weights cannot fit into one card with 80 GiB VRAM, which may need to use CPU offloading.
|
||||
|
||||
Default model: `Wan-AI/Wan2.2-I2V-A14B-Diffusers`.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
Download the example image used in the snippets below:
|
||||
Download the example image:
|
||||
|
||||
```bash
|
||||
curl -L -o cherry_blossom.jpg https://vllm-public-assets.s3.us-west-2.amazonaws.com/vision_model_images/cherry_blossom.jpg
|
||||
wget https://vllm-public-assets.s3.us-west-2.amazonaws.com/vision_model_images/cherry_blossom.jpg
|
||||
```
|
||||
|
||||
## Quick Start
|
||||
|
||||
### Python API
|
||||
|
||||
Single-prompt generation using TI2V-5B (lightweight):
|
||||
|
||||
```python
|
||||
import PIL.Image
|
||||
import torch
|
||||
|
||||
from vllm_omni.entrypoints.omni import Omni
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
|
||||
if __name__ == "__main__":
|
||||
image = PIL.Image.open("cherry_blossom.jpg").convert("RGB")
|
||||
image = image.resize((576, 320))
|
||||
|
||||
omni = Omni(
|
||||
model="Wan-AI/Wan2.2-TI2V-5B-Diffusers",
|
||||
flow_shift=12.0,
|
||||
)
|
||||
outputs = omni.generate(
|
||||
{
|
||||
"prompt": "Cherry blossoms swaying gently in the breeze, petals falling",
|
||||
"multi_modal_data": {"image": image},
|
||||
},
|
||||
OmniDiffusionSamplingParams(
|
||||
height=320,
|
||||
width=576,
|
||||
num_frames=17,
|
||||
num_inference_steps=20,
|
||||
guidance_scale=4.0,
|
||||
generator=torch.Generator(device="cuda").manual_seed(42),
|
||||
),
|
||||
)
|
||||
from diffusers.utils import export_to_video
|
||||
frames = outputs[0].request_output.images
|
||||
export_to_video(frames, "quick_test_i2v.mp4", fps=16)
|
||||
```
|
||||
|
||||
### Local CLI Usage
|
||||
|
||||
Quick test using TI2V-5B with a small resolution and few frames:
|
||||
|
||||
```bash
|
||||
python image_to_video.py \
|
||||
--model Wan-AI/Wan2.2-TI2V-5B-Diffusers \
|
||||
--image cherry_blossom.jpg \
|
||||
--prompt "Cherry blossoms swaying gently in the breeze, petals falling, smooth motion" \
|
||||
--height 320 \
|
||||
--width 576 \
|
||||
--num-frames 17 \
|
||||
--guidance-scale 4.0 \
|
||||
--num-inference-steps 20 \
|
||||
--flow-shift 12.0 \
|
||||
--fps 16 \
|
||||
--output quick_test_i2v.mp4
|
||||
```
|
||||
|
||||
## Key Arguments
|
||||
|
||||
| Argument | Type | Default | Description |
|
||||
| -------- | ---- | ------- | ----------- |
|
||||
| `--model` | str | `Wan-AI/Wan2.2-I2V-A14B-Diffusers` | Diffusers I2V model ID or local path |
|
||||
| `--model-class-name` | str | `None` | Override model class name (e.g., `LTX2ImageToVideoPipeline`) |
|
||||
| `--image` | str | (required) | Path to input image |
|
||||
| `--prompt` | str | `""` | Text description of desired motion/animation |
|
||||
| `--negative-prompt` | str | `""` | Optional list of artifacts to suppress |
|
||||
| `--seed` | int | `42` | Random seed for deterministic sampling |
|
||||
| `--guidance-scale` | float | `5.0` | CFG scale |
|
||||
| `--guidance-scale-high` | float | `None` | Separate CFG for high-noise stage (MoE only) |
|
||||
| `--height` | int | auto | Video height (auto-calculated from image if not set). Multiples of 16 |
|
||||
| `--width` | int | auto | Video width (auto-calculated from image if not set). Multiples of 16 |
|
||||
| `--num-frames` | int | `81` | Number of frames |
|
||||
| `--num-inference-steps` | int | `50` | Number of denoising steps |
|
||||
| `--boundary-ratio` | float | `0.875` | Boundary split ratio for two-stage MoE models |
|
||||
| `--flow-shift` | float | `5.0` | Scheduler flow shift (5.0 for 720p, 12.0 for 480p) |
|
||||
| `--sample-solver` | str | `unipc` | Wan2.2 sampling solver (`unipc` or `euler` for Lightning/Distill) |
|
||||
| `--fps` | int | `None` | Frames per second for the saved MP4 |
|
||||
| `--frame-rate` | float | `None` | Generation frame rate for pipelines that require it (e.g., LTX2) |
|
||||
| `--output` | str | `i2v_output.mp4` | Path to save the generated video |
|
||||
| `--vae-use-slicing` | flag | off | Enable VAE slicing for memory optimization |
|
||||
| `--vae-use-tiling` | flag | off | Enable VAE tiling for memory optimization |
|
||||
| `--enable-cpu-offload` | flag | off | Enable CPU offloading for diffusion models |
|
||||
| `--enable-layerwise-offload` | flag | off | Enable layerwise offloading on DiT modules |
|
||||
| `--cfg-parallel-size` | int | `1` | Set to `2` to enable CFG Parallel |
|
||||
| `--tensor-parallel-size` | int | `1` | Tensor parallel size (effective for models that support TP, e.g. LTX2) |
|
||||
| `--ulysses-degree` | int | `1` | Ulysses sequence parallel degree |
|
||||
| `--ring-degree` | int | `1` | Ring sequence parallel degree |
|
||||
| `--cache-backend` | str | `None` | Cache backend: `cache_dit` or `tea_cache` |
|
||||
| `--use-hsdp` | flag | off | Enable Hybrid Sharded Data Parallel |
|
||||
| `--hsdp-shard-size` | int | `-1` | GPUs per shard group (-1 auto-calculates) |
|
||||
| `--hsdp-replicate-size` | int | `1` | Number of replica groups for HSDP |
|
||||
|
||||
## More CLI Examples
|
||||
|
||||
### Wan2.2-I2V-A14B-Diffusers (MoE)
|
||||
|
||||
```bash
|
||||
@@ -144,7 +17,7 @@ python image_to_video.py \
|
||||
--model Wan-AI/Wan2.2-I2V-A14B-Diffusers \
|
||||
--image cherry_blossom.jpg \
|
||||
--prompt "Cherry blossoms swaying gently in the breeze, petals falling, smooth motion" \
|
||||
--negative-prompt "low quality, blurry" \
|
||||
--negative-prompt "<optional quality filter>" \
|
||||
--height 480 \
|
||||
--width 832 \
|
||||
--num-frames 48 \
|
||||
@@ -154,7 +27,7 @@ python image_to_video.py \
|
||||
--boundary-ratio 0.875 \
|
||||
--flow-shift 12.0 \
|
||||
--fps 16 \
|
||||
--output i2v_wan_moe.mp4
|
||||
--output i2v_output.mp4
|
||||
```
|
||||
|
||||
### Wan2.2-TI2V-5B-Diffusers (Unified)
|
||||
@@ -164,7 +37,7 @@ python image_to_video.py \
|
||||
--model Wan-AI/Wan2.2-TI2V-5B-Diffusers \
|
||||
--image cherry_blossom.jpg \
|
||||
--prompt "Cherry blossoms swaying gently in the breeze, petals falling, smooth motion" \
|
||||
--negative-prompt "low quality, blurry" \
|
||||
--negative-prompt "<optional quality filter>" \
|
||||
--height 480 \
|
||||
--width 832 \
|
||||
--num-frames 48 \
|
||||
@@ -172,129 +45,23 @@ python image_to_video.py \
|
||||
--num-inference-steps 40 \
|
||||
--flow-shift 12.0 \
|
||||
--fps 16 \
|
||||
--output i2v_wan_ti2v.mp4
|
||||
```
|
||||
|
||||
### HunyuanVideo-1.5 I2V (480p)
|
||||
|
||||
```bash
|
||||
python image_to_video.py \
|
||||
--model hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_i2v \
|
||||
--image cherry_blossom.jpg \
|
||||
--prompt "Cherry blossoms swaying gently in the breeze, petals falling, smooth motion" \
|
||||
--height 480 \
|
||||
--width 832 \
|
||||
--num-frames 121 \
|
||||
--guidance-scale 6.0 \
|
||||
--flow-shift 5.0 \
|
||||
--num-inference-steps 50 \
|
||||
--fps 24 \
|
||||
--enable-cpu-offload \
|
||||
--vae-use-tiling \
|
||||
--vae-use-slicing \
|
||||
--output hunyuan_i2v.mp4
|
||||
```
|
||||
|
||||
### LTX2 Image-to-Video
|
||||
|
||||
```bash
|
||||
python image_to_video.py \
|
||||
--model /path/to/LTX-2 \
|
||||
--model-class-name LTX2ImageToVideoPipeline \
|
||||
--image cherry_blossom.jpg \
|
||||
--prompt "A cinematic dolly shot of cherry blossoms" \
|
||||
--height 512 \
|
||||
--width 768 \
|
||||
--num-frames 121 \
|
||||
--num-inference-steps 40 \
|
||||
--guidance-scale 4.0 \
|
||||
--frame-rate 24 \
|
||||
--fps 24 \
|
||||
--output ltx2_i2v.mp4
|
||||
```
|
||||
|
||||
## Advanced Features
|
||||
|
||||
### CFG Parallel
|
||||
|
||||
Set `--cfg-parallel-size 2` to enable CFG Parallel for faster inference on multi-GPU setups.
|
||||
See more examples in the [cfg_parallel user guide](../../../docs/user_guide/diffusion/parallelism/cfg_parallel.md).
|
||||
|
||||
### Cache Acceleration
|
||||
|
||||
Use `--cache-backend cache_dit` for Cache-DiT acceleration or `--cache-backend tea_cache` for Timestep Embedding Aware Cache:
|
||||
|
||||
```bash
|
||||
python image_to_video.py \
|
||||
--model Wan-AI/Wan2.2-I2V-A14B-Diffusers \
|
||||
--image cherry_blossom.jpg \
|
||||
--prompt "Cherry blossoms swaying gently in the breeze" \
|
||||
--cache-backend cache_dit \
|
||||
--output i2v_cached.mp4
|
||||
```
|
||||
|
||||
### LTX-2.3
|
||||
|
||||
```bash
|
||||
python image_to_video.py \
|
||||
--model dg845/LTX-2.3-Diffusers \
|
||||
--model-class-name LTX23ImageToVideoPipeline \
|
||||
--image cherry_blossom.jpg \
|
||||
--prompt "Cherry blossoms swaying gently in the breeze with synchronized ambient sound" \
|
||||
--negative-prompt "worst quality, inconsistent motion, blurry, jittery, distorted" \
|
||||
--height 384 \
|
||||
--width 512 \
|
||||
--num-frames 25 \
|
||||
--guidance-scale 4.0 \
|
||||
--num-inference-steps 20 \
|
||||
--frame-rate 24 \
|
||||
--fps 24 \
|
||||
--output ltx23_i2v_output.mp4
|
||||
```
|
||||
|
||||
Use a Diffusers-format checkpoint such as `dg845/LTX-2.3-Diffusers`; the
|
||||
upstream `Lightricks/LTX-2.3` raw safetensors repo is not directly loadable by
|
||||
this pipeline. Pass `--model-class-name LTX23ImageToVideoPipeline` to select
|
||||
the LTX-2.3 image-to-video pipeline.
|
||||
|
||||
### Cosmos3
|
||||
|
||||
```bash
|
||||
# Cosmos3 bundles example frames under assets/ (any RGB image works too):
|
||||
python image_to_video.py \
|
||||
--model nvidia/Cosmos3-Nano \
|
||||
--image /path/to/Cosmos3-Nano/assets/example_i2v_input.jpg \
|
||||
--prompt "The scene comes to life with smooth, natural motion." \
|
||||
--negative-prompt "blurry, distorted, low quality" \
|
||||
--height 720 --width 1280 --num-frames 189 --fps 24 \
|
||||
--num-inference-steps 35 --guidance-scale 6.0 \
|
||||
--extra-body '{"flow_shift": 10.0, "max_sequence_length": 4096, "guardrails": false}' \
|
||||
--output cosmos3_i2v.mp4
|
||||
--output i2v_output.mp4
|
||||
```
|
||||
|
||||
Key arguments:
|
||||
|
||||
- `--model`: Model ID (I2V-A14B for MoE, TI2V-5B for unified T2V+I2V,
|
||||
LTX-2/LTX-2.3, Cosmos3, or VACE).
|
||||
- `--image`: Path to the first-frame or source image.
|
||||
- `--last-image`: Optional last-frame condition for models such as VACE.
|
||||
- `--mask-image`: Optional inpainting mask. White pixels are regenerated and black pixels are preserved.
|
||||
- `--reference-image`: Optional reference image; repeat it to provide multiple references.
|
||||
- `--extra-body`: JSON object of model-specific generation params, filtered against the model's declared `extra_body_params` (see [`vllm_omni/model_extras`](../../../vllm_omni/model_extras)). Used by Cosmos3.
|
||||
- `--model`: Model ID (I2V-A14B for MoE, TI2V-5B for unified T2V+I2V).
|
||||
- `--image`: Path to input image (required).
|
||||
- `--prompt`: Text description of desired motion/animation.
|
||||
- `--height/--width`: Output resolution (auto-calculated from image if not set).
|
||||
Wan dimensions should be multiples of 16; LTX dimensions should be multiples
|
||||
of 32.
|
||||
- `--num-frames`: Number of frames (model-specific default; LTX-style models
|
||||
work best with `8k + 1`; Cosmos3 defaults to 189).
|
||||
- `--height/--width`: Output resolution (auto-calculated from image if not set). Dimensions should be multiples of 16.
|
||||
- `--num-frames`: Number of frames (default 81).
|
||||
- `--guidance-scale` and `--guidance-scale-high`: CFG scale (applied to low/high-noise stages for MoE).
|
||||
- `--negative-prompt`: Optional list of artifacts to suppress.
|
||||
- `--boundary-ratio`: Boundary split ratio for two-stage MoE models.
|
||||
- `--flow-shift`: Scheduler flow shift (default: model-specific — Wan/LTX2 5.0, Cosmos3 10.0).
|
||||
- `--flow-shift`: Scheduler flow shift (5.0 for 720p, 12.0 for 480p).
|
||||
- `--sample-solver`: Wan2.2 sampling solver. Use `unipc` for the default multistep solver, or `euler` for Lightning/Distill checkpoints.
|
||||
- `--num-inference-steps`: Number of denoising steps (default: model-specific — Wan 50, LTX2 40, Cosmos3 35).
|
||||
- `--num-inference-steps`: Number of denoising steps (default 50).
|
||||
- `--fps`: Frames per second for the saved MP4 (requires `diffusers` export_to_video).
|
||||
- `--audio-sample-rate`: fallback audio sample rate for embedded audio.
|
||||
- `--output`: Path to save the generated video.
|
||||
- `--vae-use-slicing`: Enable VAE slicing for memory optimization.
|
||||
- `--vae-use-tiling`: Enable VAE tiling for memory optimization.
|
||||
@@ -309,106 +76,5 @@ Key arguments:
|
||||
|
||||
> ℹ️ If you encounter OOM errors, try using `--vae-use-slicing` and `--vae-use-tiling` to reduce memory usage.
|
||||
|
||||
## Wan2.1 VACE Conditional Tasks
|
||||
|
||||
The shared script selects the VACE conditioning structure from the media inputs.
|
||||
No explicit mode parameter is required: the script constructs the source video,
|
||||
mask, or reference images consumed by the VACE pipeline.
|
||||
Download the same Hugging Face assets used by the original VACE example:
|
||||
|
||||
```bash
|
||||
wget -O astronaut.jpg https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/astronaut.jpg
|
||||
wget -O vace_first_frame.png https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/flf2v_input_first_frame.png
|
||||
wget -O vace_last_frame.png https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/flf2v_input_last_frame.png
|
||||
```
|
||||
|
||||
### Image-to-Video (I2V)
|
||||
|
||||
```bash
|
||||
python image_to_video.py \
|
||||
--model Wan-AI/Wan2.1-VACE-1.3B-diffusers \
|
||||
--image astronaut.jpg \
|
||||
--prompt "An astronaut emerging from a cracked, otherworldly egg on the surface of the moon" \
|
||||
--seed 42 --height 480 --width 832 --num-frames 81 \
|
||||
--num-inference-steps 30 --guidance-scale 5.0 --flow-shift 5.0 \
|
||||
--vae-use-tiling --output vace_i2v_output.mp4
|
||||
```
|
||||
|
||||
### Video-to-Last-Frame (V2LF)
|
||||
|
||||
```bash
|
||||
python image_to_video.py \
|
||||
--model Wan-AI/Wan2.1-VACE-1.3B-diffusers \
|
||||
--last-image astronaut.jpg \
|
||||
--prompt "An astronaut emerging from a cracked, otherworldly egg on the surface of the moon" \
|
||||
--seed 42 --height 480 --width 832 --num-frames 81 \
|
||||
--num-inference-steps 30 --guidance-scale 5.0 --flow-shift 5.0 \
|
||||
--vae-use-tiling --output vace_v2lf_output.mp4
|
||||
```
|
||||
|
||||
### First-Last-Frame-to-Video (FLF2V)
|
||||
|
||||
```bash
|
||||
python image_to_video.py \
|
||||
--model Wan-AI/Wan2.1-VACE-1.3B-diffusers \
|
||||
--image vace_first_frame.png \
|
||||
--last-image vace_last_frame.png \
|
||||
--prompt "CG animation style, a small blue bird takes off from a branch and lands on another branch" \
|
||||
--seed 42 --height 512 --width 512 --num-frames 81 \
|
||||
--num-inference-steps 30 --guidance-scale 5.0 --flow-shift 5.0 \
|
||||
--vae-use-tiling --output vace_flf2v_output.mp4
|
||||
```
|
||||
|
||||
### Inpainting
|
||||
|
||||
Create a mask matching the original VACE example: a 160-pixel-wide white
|
||||
vertical stripe marks the region to regenerate.
|
||||
|
||||
```bash
|
||||
python - <<'PY'
|
||||
from PIL import Image
|
||||
|
||||
mask = Image.new("L", (832, 480), 0)
|
||||
mask.paste(255, (336, 0, 496, 480))
|
||||
mask.save("vace_center_mask.png")
|
||||
PY
|
||||
```
|
||||
|
||||
```bash
|
||||
python image_to_video.py \
|
||||
--model Wan-AI/Wan2.1-VACE-1.3B-diffusers \
|
||||
--image astronaut.jpg \
|
||||
--mask-image vace_center_mask.png \
|
||||
--prompt "Shrek, the ogre, walks out of a building in a happy mood" \
|
||||
--seed 42 --height 480 --width 832 --num-frames 81 \
|
||||
--num-inference-steps 30 --guidance-scale 5.0 --flow-shift 5.0 \
|
||||
--vae-use-tiling --output vace_inpaint_output.mp4
|
||||
```
|
||||
|
||||
### Reference-to-Video (R2V)
|
||||
|
||||
Repeat `--reference-image` to provide more than one reference image.
|
||||
|
||||
```bash
|
||||
python image_to_video.py \
|
||||
--model Wan-AI/Wan2.1-VACE-1.3B-diffusers \
|
||||
--reference-image astronaut.jpg \
|
||||
--prompt "Camera slowly zooms out from the character walking in a garden" \
|
||||
--seed 42 --height 480 --width 832 --num-frames 81 \
|
||||
--num-inference-steps 30 --guidance-scale 5.0 --flow-shift 5.0 \
|
||||
--vae-use-tiling --output vace_r2v_output.mp4
|
||||
```
|
||||
|
||||
The VACE T2V command is documented in the shared
|
||||
[`text_to_video.py`](../text_to_video/text_to_video.md#wan21-vace-t2v) example.
|
||||
|
||||
For Wan2.2 LightX2V-converted local Diffusers directories and related LoRA
|
||||
assets, see the [LoRA guide](../../../docs/user_guide/diffusion/lora.md#wan22-lightx2v-offline-assembly).
|
||||
|
||||
## FAQ
|
||||
|
||||
**OOM errors**: Try using `--vae-use-slicing` and `--vae-use-tiling` to reduce memory usage. For very large models, add `--enable-cpu-offload` or `--enable-layerwise-offload`.
|
||||
|
||||
**Auto-calculated resolution**: If `--height` and `--width` are not provided, the script calculates output dimensions from the input image while maintaining aspect ratio and targeting 480 x 832 area (or 512 x 768 for LTX2).
|
||||
|
||||
**Wan2.2 MoE vs unified**: `I2V-A14B` is a larger Mixture-of-Experts model with separate low/high-noise DiT stages (use `--guidance-scale-high` and `--boundary-ratio`). `TI2V-5B` is a smaller unified T2V+I2V model that does not need these extra arguments.
|
||||
@@ -2,15 +2,13 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""
|
||||
Image-to-Video generation example using Wan2.2 I2V/TI2V models, LTX2/LTX-2.3,
|
||||
HunyuanVideo-1.5, Cosmos3, or Wan2.1 VACE.
|
||||
Image-to-Video generation example using Wan2.2 I2V/TI2V models, LTX2, or HunyuanVideo-1.5.
|
||||
|
||||
Supports:
|
||||
- Wan2.2-I2V-A14B-Diffusers: MoE model with CLIP image encoder
|
||||
- Wan2.2-TI2V-5B-Diffusers: Unified T2V+I2V model (dense 5B)
|
||||
- LTX2 image-to-video pipeline
|
||||
- HunyuanVideo-1.5 I2V: SigLIP + VAE dual image conditioning
|
||||
- Wan2.1 VACE: first/last-frame, inpainting, and reference conditioning
|
||||
|
||||
Usage:
|
||||
# Wan I2V-A14B (MoE)
|
||||
@@ -28,27 +26,13 @@ Usage:
|
||||
--num-frames 121 --num-inference-steps 40 --guidance-scale 4.0 \
|
||||
--frame-rate 24 --fps 24 --output ltx2_i2v.mp4
|
||||
|
||||
# LTX-2.3 image-to-video
|
||||
python image_to_video.py --model dg845/LTX-2.3-Diffusers \
|
||||
--model-class-name LTX23ImageToVideoPipeline \
|
||||
--image input.jpg --prompt "A cinematic dolly shot of a boat" \
|
||||
--height 384 --width 512 --num-frames 25 --num-inference-steps 20 \
|
||||
--frame-rate 24 --fps 24 --output ltx23_i2v.mp4
|
||||
|
||||
# HunyuanVideo-1.5 I2V (480p)
|
||||
python image_to_video.py --model hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_i2v \
|
||||
--image input.jpg --prompt "A cat playing with yarn" \
|
||||
--flow-shift 5.0 --guidance-scale 6.0
|
||||
|
||||
# Cosmos3 I2V (image conditioning)
|
||||
python image_to_video.py --model nvidia/Cosmos3-Nano \
|
||||
--image input.jpg --prompt "The scene comes to life with smooth, natural motion." \
|
||||
--num-frames 189 --num-inference-steps 35 --guidance-scale 6.0 --fps 24 \
|
||||
--extra-body '{"flow_shift": 10.0, "max_sequence_length": 4096, "guardrails": false}'
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import functools
|
||||
import json
|
||||
import time
|
||||
from pathlib import Path
|
||||
@@ -59,62 +43,39 @@ import PIL.Image
|
||||
import torch
|
||||
|
||||
from vllm_omni.diffusion.data import DiffusionParallelConfig
|
||||
from vllm_omni.diffusion.utils.param_utils import apply_declared_extra_args
|
||||
from vllm_omni.entrypoints.omni import Omni
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
from vllm_omni.model_extras import (
|
||||
build_image_to_video_prompt,
|
||||
get_extra_body_params,
|
||||
get_model_class_name,
|
||||
)
|
||||
from vllm_omni.outputs import OmniRequestOutput
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
|
||||
def parse_json_object(value: str, flag_name: str = "argument") -> dict[str, Any]:
|
||||
"""Parse a CLI value as a JSON object, attributing errors to ``flag_name``."""
|
||||
def parse_profiler_config(value: str) -> dict[str, Any]:
|
||||
try:
|
||||
config = json.loads(value)
|
||||
except json.JSONDecodeError as e:
|
||||
raise argparse.ArgumentTypeError(f"{flag_name} must be valid JSON: {e}") from e
|
||||
raise argparse.ArgumentTypeError(f"--profiler-config must be valid JSON: {e}") from e
|
||||
if not isinstance(config, dict):
|
||||
raise argparse.ArgumentTypeError(f"{flag_name} must be a JSON object")
|
||||
raise argparse.ArgumentTypeError("--profiler-config must be a JSON object")
|
||||
return config
|
||||
|
||||
|
||||
parse_profiler_config = functools.partial(parse_json_object, flag_name="--profiler-config")
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(
|
||||
description=(
|
||||
"Generate a video from one or more images "
|
||||
"(Wan2.2, LTX2/LTX-2.3, HunyuanVideo-1.5, Cosmos3, or Wan2.1 VACE)."
|
||||
)
|
||||
)
|
||||
parser = argparse.ArgumentParser(description="Generate a video from an image (Wan2.2, LTX2, HunyuanVideo-1.5).")
|
||||
parser.add_argument(
|
||||
"--model",
|
||||
default="Wan-AI/Wan2.2-I2V-A14B-Diffusers",
|
||||
help="Diffusers I2V model ID or local path (Wan2.2, LTX2/LTX-2.3, HunyuanVideo-1.5, Cosmos3, or Wan2.1 VACE).",
|
||||
help="Diffusers I2V model ID or local path (Wan2.2 or HunyuanVideo-1.5).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--model-class-name",
|
||||
default=None,
|
||||
help="Override model class name (e.g., LTX2ImageToVideoPipeline or LTX23ImageToVideoPipeline).",
|
||||
)
|
||||
parser.add_argument("--image", help="Path to the first-frame or source image.")
|
||||
parser.add_argument("--last-image", help="Path to a last-frame condition (used by models such as VACE).")
|
||||
parser.add_argument("--mask-image", help="Path to an inpainting mask (used by models such as VACE).")
|
||||
parser.add_argument(
|
||||
"--reference-image",
|
||||
action="append",
|
||||
default=None,
|
||||
help="Path to a reference image. Repeat to provide multiple references.",
|
||||
help="Override model class name (e.g., LTX2ImageToVideoPipeline).",
|
||||
)
|
||||
parser.add_argument("--image", required=True, help="Path to input image.")
|
||||
parser.add_argument("--prompt", default="", help="Text prompt describing the desired motion.")
|
||||
parser.add_argument("--negative-prompt", default="", help="Negative prompt.")
|
||||
parser.add_argument("--seed", type=int, default=42, help="Random seed.")
|
||||
parser.add_argument("--guidance-scale", type=float, default=None, help="CFG scale. Default: model-specific.")
|
||||
parser.add_argument("--guidance-scale", type=float, default=5.0, help="CFG scale.")
|
||||
parser.add_argument(
|
||||
"--guidance-scale-high", type=float, default=None, help="Optional separate CFG for high-noise (MoE only)."
|
||||
)
|
||||
@@ -122,13 +83,8 @@ def parse_args() -> argparse.Namespace:
|
||||
"--height", type=int, default=None, help="Video height (auto-calculated from image if not set)."
|
||||
)
|
||||
parser.add_argument("--width", type=int, default=None, help="Video width (auto-calculated from image if not set).")
|
||||
parser.add_argument("--num-frames", type=int, default=None, help="Number of frames. Default: model-specific.")
|
||||
parser.add_argument(
|
||||
"--num-inference-steps",
|
||||
type=int,
|
||||
default=None,
|
||||
help="Sampling steps. Default: model-specific.",
|
||||
)
|
||||
parser.add_argument("--num-frames", type=int, default=81, help="Number of frames.")
|
||||
parser.add_argument("--num-inference-steps", type=int, default=50, help="Sampling steps.")
|
||||
parser.add_argument("--boundary-ratio", type=float, default=0.875, help="Boundary split ratio for MoE models.")
|
||||
parser.add_argument(
|
||||
"--frame-rate",
|
||||
@@ -137,10 +93,7 @@ def parse_args() -> argparse.Namespace:
|
||||
help="Optional generation frame rate (used by models like LTX2). Defaults to --fps.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--flow-shift",
|
||||
type=float,
|
||||
default=None,
|
||||
help="Scheduler flow_shift. Default: model-specific (Wan 5.0, Cosmos3 10.0).",
|
||||
"--flow-shift", type=float, default=5.0, help="Scheduler flow_shift (5.0 for 720p, 12.0 for 480p)."
|
||||
)
|
||||
parser.add_argument(
|
||||
"--sample-solver",
|
||||
@@ -291,19 +244,6 @@ def parse_args() -> argparse.Namespace:
|
||||
default=None,
|
||||
help='JSON profiler config for torch/cuda profiling, e.g. \'{"profiler":"torch","torch_profiler_dir":"./perf"}\'.',
|
||||
)
|
||||
parser.add_argument(
|
||||
"--extra-body",
|
||||
type=functools.partial(parse_json_object, flag_name="--extra-body"),
|
||||
default=None,
|
||||
help=(
|
||||
"Model-specific generation params as a JSON object. Keys are filtered "
|
||||
"against the model's declared extra_body_params (see vllm_omni/model_extras), "
|
||||
"so unknown keys for the chosen model are silently dropped. "
|
||||
'Cosmos3 V2V example: \'{"condition_frame_indexes_vision": [0, 1], '
|
||||
'"condition_video_keep": "first", "flow_shift": 10.0, '
|
||||
'"max_sequence_length": 4096, "guardrails": false}\'.'
|
||||
),
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
@@ -326,70 +266,32 @@ def main():
|
||||
generator = torch.Generator(device=current_omni_platform.device_type).manual_seed(args.seed)
|
||||
model_name = str(args.model).lower() if args.model is not None else ""
|
||||
model_class_name = args.model_class_name
|
||||
is_ltx2 = model_class_name in {"LTX2ImageToVideoPipeline", "LTX23ImageToVideoPipeline"}
|
||||
is_cosmos = "cosmos" in model_name or (model_class_name is not None and "cosmos" in model_class_name.lower())
|
||||
is_ltx2 = "ltx2" in model_name or (model_class_name and "ltx2" in model_class_name.lower())
|
||||
if model_class_name is None and is_ltx2:
|
||||
model_class_name = "LTX2ImageToVideoPipeline"
|
||||
|
||||
image = PIL.Image.open(args.image).convert("RGB") if args.image else None
|
||||
last_image = PIL.Image.open(args.last_image).convert("RGB") if args.last_image else None
|
||||
mask_image = PIL.Image.open(args.mask_image).convert("L") if args.mask_image else None
|
||||
reference_images = (
|
||||
[PIL.Image.open(path).convert("RGB") for path in args.reference_image] if args.reference_image else None
|
||||
)
|
||||
dimension_image = image or last_image or (reference_images[0] if reference_images else None)
|
||||
if dimension_image is None:
|
||||
raise ValueError("Provide --image, --last-image, or at least one --reference-image.")
|
||||
# Load input image
|
||||
image = PIL.Image.open(args.image).convert("RGB")
|
||||
|
||||
# Per-model generation defaults, applied only when the matching flag is omitted.
|
||||
# Cosmos3 would otherwise silently inherit the Wan2.2 defaults (wrong size/steps/shift).
|
||||
if is_cosmos:
|
||||
d_fps, d_guidance, d_num_frames, d_steps, d_flow_shift, d_max_area, d_mod = (
|
||||
24,
|
||||
6.0,
|
||||
189,
|
||||
35,
|
||||
10.0,
|
||||
1280 * 720,
|
||||
16,
|
||||
)
|
||||
elif is_ltx2:
|
||||
d_fps, d_guidance, d_num_frames, d_steps, d_flow_shift, d_max_area, d_mod = (
|
||||
24,
|
||||
4.0,
|
||||
121,
|
||||
40,
|
||||
5.0,
|
||||
512 * 768,
|
||||
32,
|
||||
)
|
||||
else: # Wan2.2 / HunyuanVideo-1.5
|
||||
d_fps, d_guidance, d_num_frames, d_steps, d_flow_shift, d_max_area, d_mod = 16, 5.0, 81, 50, 5.0, 480 * 832, 16
|
||||
|
||||
fps = args.fps if args.fps is not None else d_fps
|
||||
fps = args.fps if args.fps is not None else (24 if is_ltx2 else 16)
|
||||
frame_rate = args.frame_rate if args.frame_rate is not None else float(fps)
|
||||
guidance_scale = args.guidance_scale if args.guidance_scale is not None else d_guidance
|
||||
num_frames = args.num_frames if args.num_frames is not None else d_num_frames
|
||||
num_inference_steps = args.num_inference_steps if args.num_inference_steps is not None else d_steps
|
||||
flow_shift = args.flow_shift if args.flow_shift is not None else d_flow_shift
|
||||
guidance_scale = args.guidance_scale if args.guidance_scale is not None else (4.0 if is_ltx2 else 5.0)
|
||||
num_frames = args.num_frames if args.num_frames is not None else (121 if is_ltx2 else 81)
|
||||
num_inference_steps = args.num_inference_steps if args.num_inference_steps is not None else (40 if is_ltx2 else 50)
|
||||
|
||||
# Calculate dimensions if not provided (model-aware max area).
|
||||
# Calculate dimensions if not provided
|
||||
height = args.height
|
||||
width = args.width
|
||||
if height is None or width is None:
|
||||
calc_height, calc_width = calculate_dimensions(dimension_image, max_area=d_max_area, mod_value=d_mod)
|
||||
# Default to 480P area for Wan2.2 I2V, 512x768 area for LTX2
|
||||
max_area = 512 * 768 if is_ltx2 else 480 * 832
|
||||
mod_value = 32 if is_ltx2 else 16
|
||||
calc_height, calc_width = calculate_dimensions(image, max_area=max_area, mod_value=mod_value)
|
||||
height = height or calc_height
|
||||
width = width or calc_width
|
||||
|
||||
media_inputs: dict[str, Any] = {}
|
||||
if image is not None:
|
||||
media_inputs["image"] = image.resize((width, height), PIL.Image.Resampling.LANCZOS)
|
||||
if last_image is not None:
|
||||
media_inputs["last_image"] = last_image.resize((width, height), PIL.Image.Resampling.LANCZOS)
|
||||
if mask_image is not None:
|
||||
media_inputs["mask"] = mask_image.resize((width, height), PIL.Image.Resampling.NEAREST)
|
||||
if reference_images is not None:
|
||||
media_inputs["reference_images"] = [
|
||||
reference.resize((width, height), PIL.Image.Resampling.LANCZOS) for reference in reference_images
|
||||
]
|
||||
# Resize image to target dimensions
|
||||
image = image.resize((width, height), PIL.Image.Resampling.LANCZOS)
|
||||
|
||||
# Configure cache based on backend type
|
||||
cache_config = None
|
||||
@@ -440,7 +342,7 @@ def main():
|
||||
vae_use_slicing=args.vae_use_slicing,
|
||||
vae_use_tiling=args.vae_use_tiling,
|
||||
boundary_ratio=args.boundary_ratio,
|
||||
flow_shift=flow_shift,
|
||||
flow_shift=args.flow_shift,
|
||||
diffusion_kv_cache_dtype=args.diffusion_kv_cache_dtype,
|
||||
diffusion_kv_cache_skip_steps=args.diffusion_kv_cache_skip_steps,
|
||||
diffusion_kv_cache_skip_layers=args.diffusion_kv_cache_skip_layers,
|
||||
@@ -455,13 +357,7 @@ def main():
|
||||
)
|
||||
if args.quantization is not None:
|
||||
omni_kwargs["quantization"] = args.quantization
|
||||
# Cosmos3 loads its (gated) guardrail models at build time, so the guardrails
|
||||
# gate is an engine-level config (offline analog of the server's --no-guardrails).
|
||||
if args.extra_body and "guardrails" in args.extra_body:
|
||||
omni_kwargs["model_config"] = {"guardrails": bool(args.extra_body["guardrails"])}
|
||||
omni = Omni(**omni_kwargs)
|
||||
model_class_name = get_model_class_name(omni) or model_class_name
|
||||
declared_extra_body_params = get_extra_body_params(model_class_name)
|
||||
|
||||
if profiler_enabled:
|
||||
print("[Profiler] Starting profiling...")
|
||||
@@ -471,8 +367,8 @@ def main():
|
||||
print(f"\n{'=' * 60}")
|
||||
print("Generation Configuration:")
|
||||
print(f" Model: {args.model}")
|
||||
print(f" Inference steps: {num_inference_steps}")
|
||||
print(f" Frames: {num_frames}")
|
||||
print(f" Inference steps: {args.num_inference_steps}")
|
||||
print(f" Frames: {args.num_frames}")
|
||||
print(f" Solver: {args.sample_solver}")
|
||||
print(f" diffusion_kv_cache_dtype(config): {args.diffusion_kv_cache_dtype}")
|
||||
print(f" diffusion_kv_cache_skip_steps(config): {args.diffusion_kv_cache_skip_steps}")
|
||||
@@ -482,46 +378,33 @@ def main():
|
||||
f" tensor_parallel_size={args.tensor_parallel_size}, vae_patch_parallel_size={args.vae_patch_parallel_size},"
|
||||
f" pipeline_parallel_size={args.pipeline_parallel_size}"
|
||||
)
|
||||
print(f" Video size: {width}x{height}")
|
||||
print(f" Video size: {args.width}x{args.height}")
|
||||
print(f"{'=' * 60}\n")
|
||||
|
||||
prompt_dict = build_image_to_video_prompt(
|
||||
model_class_name=model_class_name,
|
||||
prompt=args.prompt,
|
||||
negative_prompt=args.negative_prompt,
|
||||
media_inputs=media_inputs,
|
||||
height=height,
|
||||
width=width,
|
||||
num_frames=num_frames,
|
||||
)
|
||||
sampling_params = OmniDiffusionSamplingParams(
|
||||
height=height,
|
||||
width=width,
|
||||
generator=generator,
|
||||
guidance_scale=guidance_scale,
|
||||
guidance_scale_2=args.guidance_scale_high,
|
||||
boundary_ratio=args.boundary_ratio,
|
||||
num_inference_steps=num_inference_steps,
|
||||
num_frames=num_frames,
|
||||
frame_rate=frame_rate,
|
||||
extra_args={
|
||||
"sample_solver": args.sample_solver,
|
||||
"flow_shift": flow_shift,
|
||||
},
|
||||
)
|
||||
|
||||
# Route model-specific knobs through extra_body, filtered against the model's
|
||||
# declared extra_body_params. Models without a declaration only forward explicit
|
||||
# --extra-body JSON (e.g. Cosmos3 V2V's condition_frame_indexes_vision).
|
||||
extra_body = dict(args.extra_body or {})
|
||||
if declared_extra_body_params:
|
||||
apply_declared_extra_args(sampling_params, declared_extra_body_params, extra_body)
|
||||
elif extra_body:
|
||||
sampling_params.extra_args.update({k: v for k, v in extra_body.items() if v is not None})
|
||||
|
||||
generation_start = time.perf_counter()
|
||||
# omni.generate() returns Generator[OmniRequestOutput, None, None]
|
||||
frames = omni.generate(prompt_dict, sampling_params)
|
||||
frames = omni.generate(
|
||||
{
|
||||
"prompt": args.prompt,
|
||||
"negative_prompt": args.negative_prompt,
|
||||
"multi_modal_data": {"image": image},
|
||||
},
|
||||
OmniDiffusionSamplingParams(
|
||||
height=height,
|
||||
width=width,
|
||||
generator=generator,
|
||||
guidance_scale=guidance_scale,
|
||||
guidance_scale_2=args.guidance_scale_high,
|
||||
boundary_ratio=args.boundary_ratio,
|
||||
num_inference_steps=num_inference_steps,
|
||||
num_frames=num_frames,
|
||||
frame_rate=frame_rate,
|
||||
extra_args={
|
||||
"sample_solver": args.sample_solver,
|
||||
"flow_shift": args.flow_shift,
|
||||
},
|
||||
),
|
||||
)
|
||||
generation_end = time.perf_counter()
|
||||
generation_time = generation_end - generation_start
|
||||
|
||||
@@ -529,7 +412,6 @@ def main():
|
||||
print(f"Total generation time: {generation_time:.4f} seconds ({generation_time * 1000:.2f} ms)")
|
||||
|
||||
audio = None
|
||||
audio_sample_rate = args.audio_sample_rate
|
||||
if isinstance(frames, list):
|
||||
frames = frames[0] if frames else None
|
||||
|
||||
@@ -540,13 +422,11 @@ def main():
|
||||
)
|
||||
if frames.multimodal_output and "audio" in frames.multimodal_output:
|
||||
audio = frames.multimodal_output["audio"]
|
||||
audio_sample_rate = frames.multimodal_output.get("audio_sample_rate", audio_sample_rate)
|
||||
if frames.is_pipeline_output and frames.request_output is not None:
|
||||
inner_output = frames.request_output
|
||||
if isinstance(inner_output, OmniRequestOutput):
|
||||
if inner_output.multimodal_output and "audio" in inner_output.multimodal_output:
|
||||
audio = inner_output.multimodal_output["audio"]
|
||||
audio_sample_rate = inner_output.multimodal_output.get("audio_sample_rate", audio_sample_rate)
|
||||
frames = inner_output
|
||||
if isinstance(frames, OmniRequestOutput):
|
||||
if frames.images:
|
||||
@@ -554,7 +434,6 @@ def main():
|
||||
frames, audio = frames.images[0]
|
||||
elif len(frames.images) == 1 and isinstance(frames.images[0], dict):
|
||||
audio = frames.images[0].get("audio")
|
||||
audio_sample_rate = frames.images[0].get("audio_sample_rate", audio_sample_rate)
|
||||
frames = frames.images[0].get("frames") or frames.images[0].get("video")
|
||||
else:
|
||||
frames = frames.images
|
||||
@@ -567,7 +446,6 @@ def main():
|
||||
frames, audio = first_item
|
||||
elif isinstance(first_item, dict):
|
||||
audio = first_item.get("audio")
|
||||
audio_sample_rate = first_item.get("audio_sample_rate", audio_sample_rate)
|
||||
frames = first_item.get("frames") or first_item.get("video")
|
||||
elif isinstance(first_item, list):
|
||||
frames = first_item
|
||||
@@ -576,7 +454,6 @@ def main():
|
||||
frames, audio = frames
|
||||
elif isinstance(frames, dict):
|
||||
audio = frames.get("audio")
|
||||
audio_sample_rate = frames.get("audio_sample_rate", audio_sample_rate)
|
||||
frames = frames.get("frames") or frames.get("video")
|
||||
|
||||
if frames is None:
|
||||
@@ -695,7 +572,7 @@ def main():
|
||||
frames_u8,
|
||||
audio_np,
|
||||
fps=float(fps),
|
||||
audio_sample_rate=audio_sample_rate,
|
||||
audio_sample_rate=args.audio_sample_rate,
|
||||
)
|
||||
with open(str(output_path), "wb") as f:
|
||||
f.write(video_bytes)
|
||||
|
||||
@@ -1,17 +1,24 @@
|
||||
# MammothModa2-Preview
|
||||
|
||||
> Text-to-image (T2I) generation has moved to the shared offline image example.
|
||||
> See the recipe: [`recipes/MammothModa2/MammothModa2-Preview.md`](../../../recipes/MammothModa2/MammothModa2-Preview.md).
|
||||
>
|
||||
> This directory retains the image-understanding (image → text) example below,
|
||||
> which is outside the scope of the diffusion `model_extras` example migration
|
||||
> (#4548 / RFC #4539: AR / Omni understanding examples keep dedicated scripts).
|
||||
|
||||
## Run example (MammothModa2-Preview)
|
||||
## Run examples (MammothModa2-Preview)
|
||||
|
||||
Download model
|
||||
```bash
|
||||
hf download bytedance-research/MammothModa2-Preview --local-dir ./MammothModa2-Preview
|
||||
hf bytedance-research/MammothModa2-Preview --local-dir ./MammothModa2-Preview
|
||||
```
|
||||
|
||||
### Text-to-Image (T2I)
|
||||
|
||||
```bash
|
||||
python examples/offline_inference/mammothmodal2_preview/run_mammothmoda2_t2i.py \
|
||||
--model ./MammothModa2-Preview \
|
||||
--stage-config ./vllm_omni/model_executor/stage_configs/mammoth_moda2.yaml \
|
||||
--prompt "A stylish woman riding a motorcycle in NYC, movie poster style" \
|
||||
--height 1024 \
|
||||
--width 1024 \
|
||||
--num-inference-steps 50 \
|
||||
--text-guidance-scale 4.0 \
|
||||
--out output.png
|
||||
```
|
||||
|
||||
### Image Summary
|
||||
@@ -19,7 +26,7 @@ hf download bytedance-research/MammothModa2-Preview --local-dir ./MammothModa2-P
|
||||
```bash
|
||||
python examples/offline_inference/mammothmodal2_preview/run_mammothmoda2_image_summarize.py \
|
||||
--model ./MammothModa2-Preview \
|
||||
--deploy-config ./vllm_omni/deploy/mammoth_moda2_ar.yaml \
|
||||
--stage-config ./vllm_omni/model_executor/stage_configs/mammoth_moda2_ar.yaml \
|
||||
--question "Summarize this image." \
|
||||
--image ./image.png
|
||||
```
|
||||
+5
-3
@@ -4,7 +4,7 @@ Offline inference example: MammothModa2 image summarization (single AR stage).
|
||||
Example:
|
||||
uv run python examples/offline_inference/mammothmodal2_preview/run_mammothmoda2_image_summarize.py \
|
||||
--model path/to/MammothModa2-Preview \
|
||||
--deploy-config vllm_omni/deploy/mammoth_moda2_ar.yaml \
|
||||
--stage-config vllm_omni/model_executor/stage_configs/mammoth_moda2_ar.yaml \
|
||||
--image /path/to/input.jpg \
|
||||
--question "Please summarize the content of this image."
|
||||
"""
|
||||
@@ -27,7 +27,9 @@ DEFAULT_QUESTION = "Please summarize the content of this image."
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description="MammothModa2 image summarization (offline, AR only).")
|
||||
parser.add_argument("--model", type=str, required=True, help="Path to model directory or model id.")
|
||||
parser.add_argument("--deploy-config", type=str, required=True, help="Path to deploy config yaml.")
|
||||
parser.add_argument(
|
||||
"--stage-config", type=str, required=True, help="Path to stage config yaml (single-stage AR->text)."
|
||||
)
|
||||
parser.add_argument("--image", type=str, required=True, help="Path to input image.")
|
||||
parser.add_argument("--question", type=str, default=DEFAULT_QUESTION, help="Question/instruction for the model.")
|
||||
parser.add_argument("--system", type=str, default=DEFAULT_SYSTEM, help="System prompt.")
|
||||
@@ -71,7 +73,7 @@ def main() -> None:
|
||||
|
||||
omni = Omni(
|
||||
model=args.model,
|
||||
deploy_config=args.deploy_config,
|
||||
stage_configs_path=args.stage_config,
|
||||
trust_remote_code=args.trust_remote_code,
|
||||
enable_diffusion_pipeline_profiler=args.enable_diffusion_pipeline_profiler,
|
||||
)
|
||||
|
||||
@@ -0,0 +1,248 @@
|
||||
"""
|
||||
Offline inference example for MammothModa2 Text-to-Image (T2I) generation.
|
||||
This script uses the vllm_omni.Omni pipeline with a multi-stage configuration.
|
||||
|
||||
Workflow:
|
||||
1. Stage 0 (AR): Generates visual tokens and their corresponding hidden states.
|
||||
2. Stage 1 (DiT): Consumes the hidden states as conditions to perform diffusion
|
||||
and VAE decoding to produce the final image.
|
||||
|
||||
Example Usage:
|
||||
uv run python examples/offline_inference/run_mammothmoda2_t2i.py \
|
||||
--model path/to/MammothModa2-Preview \
|
||||
--stage-config vllm_omni/model_executor/stage_configs/mammoth_moda2.yaml \
|
||||
--prompt "A stylish woman riding a motorcycle in NYC, movie poster style" \
|
||||
--out output.png
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import NamedTuple
|
||||
|
||||
import torch
|
||||
from PIL import Image
|
||||
from vllm.sampling_params import SamplingParams
|
||||
|
||||
from vllm_omni import Omni
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s")
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Constants
|
||||
# ---------------------------------------------------------------------------
|
||||
_PATCH_SIZE = 16 # AR image grid patch size (pixels per token)
|
||||
|
||||
|
||||
class T2IGenConfig(NamedTuple):
|
||||
eol_token_id: int
|
||||
visual_token_start_id: int
|
||||
visual_token_end_id: int
|
||||
top_k: int # AR sampling top-k (covers the full visual generation vocabulary)
|
||||
# Qwen2.5-VL special vision tokens: <|image_pad|>, <|video_pad|>, <|vision_start|>, <|vision_end|>
|
||||
visual_ids: list[int]
|
||||
|
||||
|
||||
def load_t2i_generation_config(model_dir: str) -> T2IGenConfig:
|
||||
"""Load T2I token IDs from t2i_generation_config.json and config.json.
|
||||
|
||||
Supports both local directory paths and HuggingFace Hub model IDs.
|
||||
"""
|
||||
model_path = Path(model_dir)
|
||||
|
||||
def _read_json(filename: str) -> dict:
|
||||
local = model_path / filename
|
||||
if local.exists():
|
||||
with local.open(encoding="utf-8") as f:
|
||||
return json.load(f)
|
||||
# Fall back to HuggingFace Hub when model_dir is a repo ID.
|
||||
try:
|
||||
from huggingface_hub import hf_hub_download
|
||||
except ImportError as exc:
|
||||
raise ImportError(
|
||||
"huggingface_hub is required to load configs from HF Hub. Install it with: pip install huggingface_hub"
|
||||
) from exc
|
||||
cached = hf_hub_download(repo_id=model_dir, filename=filename)
|
||||
with open(cached, encoding="utf-8") as f:
|
||||
return json.load(f)
|
||||
|
||||
gen_cfg = _read_json("t2i_generation_config.json")
|
||||
llm_cfg = _read_json("config.json").get("llm_config", {})
|
||||
|
||||
return T2IGenConfig(
|
||||
eol_token_id=int(gen_cfg["eol_token_id"]),
|
||||
visual_token_start_id=int(gen_cfg["visual_token_start_id"]),
|
||||
visual_token_end_id=int(gen_cfg["visual_token_end_id"]),
|
||||
top_k=int(gen_cfg["top_k"]),
|
||||
visual_ids=[
|
||||
int(llm_cfg["image_token_id"]),
|
||||
int(llm_cfg["video_token_id"]),
|
||||
int(llm_cfg["vision_start_token_id"]),
|
||||
int(llm_cfg["vision_end_token_id"]),
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
p = argparse.ArgumentParser(description="Run MammothModa2 T2I (AR -> DiT) with vLLM-Omni.")
|
||||
p.add_argument("--model", type=str, required=True, help="Path to the model directory.")
|
||||
p.add_argument("--stage-config", type=str, required=True, help="Path to the multi-stage YAML configuration.")
|
||||
p.add_argument(
|
||||
"--prompt",
|
||||
type=str,
|
||||
action="append",
|
||||
default=None,
|
||||
help=(
|
||||
"Text prompt for image generation. Can be provided multiple times "
|
||||
"to generate multiple images with shared height/width/CFG settings."
|
||||
),
|
||||
)
|
||||
p.add_argument("--height", type=int, default=1024, help="Output image height (must be a multiple of 16).")
|
||||
p.add_argument("--width", type=int, default=1024, help="Output image width (must be a multiple of 16).")
|
||||
p.add_argument("--num-inference-steps", type=int, default=50, help="Number of diffusion steps for the DiT stage.")
|
||||
p.add_argument(
|
||||
"--text-guidance-scale", type=float, default=9.0, help="Classifier-Free Guidance (CFG) scale for DiT."
|
||||
)
|
||||
p.add_argument(
|
||||
"--cfg-range",
|
||||
type=float,
|
||||
nargs=2,
|
||||
default=(0.0, 1.0),
|
||||
help="Relative step range [start, end] where CFG is active.",
|
||||
)
|
||||
p.add_argument("--out", type=str, default="output.png", help="Path to save the generated image.")
|
||||
p.add_argument("--trust-remote-code", action="store_true", help="Trust remote code when loading the model.")
|
||||
args = p.parse_args()
|
||||
if not args.prompt:
|
||||
args.prompt = ["A stylish woman with sunglasses riding a motorcycle in NYC."]
|
||||
return args
|
||||
|
||||
|
||||
def tensor_to_pil(image: torch.Tensor) -> Image.Image:
|
||||
"""Convert a normalized torch tensor [-1, 1] to a PIL Image."""
|
||||
if image.ndim == 4:
|
||||
image = image[0]
|
||||
image = image.detach().to("cpu")
|
||||
image = (image / 2 + 0.5).clamp(0, 1)
|
||||
image = (image * 255).to(torch.uint8)
|
||||
image = image.permute(1, 2, 0).contiguous().numpy()
|
||||
return Image.fromarray(image)
|
||||
|
||||
|
||||
def _format_prompt(user_prompt: str, ar_width: int, ar_height: int) -> str:
|
||||
"""Build the AR-stage prompt string including the image grid header."""
|
||||
return (
|
||||
"<|im_start|>system\nYou are a helpful image generator.<|im_end|>\n"
|
||||
f"<|im_start|>user\n{user_prompt}<|im_end|>\n"
|
||||
"<|im_start|>assistant\n"
|
||||
f"<|image start|>{ar_width}*{ar_height}<|image token|>"
|
||||
)
|
||||
|
||||
|
||||
def _collect_images(outputs: list) -> list[torch.Tensor]:
|
||||
"""Extract all image tensors produced by the final (DiT) stage."""
|
||||
images: list[torch.Tensor] = []
|
||||
for out in outputs:
|
||||
ro_item = getattr(out, "request_output", out)
|
||||
for completion in getattr(ro_item, "outputs", None) or []:
|
||||
mm = getattr(completion, "multimodal_output", None)
|
||||
if not isinstance(mm, dict) or "image" not in mm:
|
||||
raise RuntimeError(f"Missing image in multimodal output: {mm}")
|
||||
payload = mm["image"]
|
||||
for tensor in payload if isinstance(payload, list) else [payload]:
|
||||
if not isinstance(tensor, torch.Tensor):
|
||||
raise TypeError(f"Expected image tensor, got {type(tensor)}")
|
||||
images.append(tensor)
|
||||
return images
|
||||
|
||||
|
||||
def _save_images(images: list[torch.Tensor], out_path: str) -> list[str]:
|
||||
"""Save image tensors to disk.
|
||||
|
||||
Single image: written to *out_path* exactly.
|
||||
Multiple images: suffixed as ``<base>_0<ext>``, ``<base>_1<ext>``, …
|
||||
"""
|
||||
if not images:
|
||||
raise RuntimeError("No images to save.")
|
||||
base, ext = os.path.splitext(out_path)
|
||||
ext = ext or ".png"
|
||||
paths = []
|
||||
for i, tensor in enumerate(images):
|
||||
path = out_path if len(images) == 1 else f"{base}_{i}{ext}"
|
||||
tensor_to_pil(tensor).save(path)
|
||||
paths.append(path)
|
||||
return paths
|
||||
|
||||
|
||||
def main() -> None:
|
||||
args = parse_args()
|
||||
os.makedirs(os.path.dirname(args.out) or ".", exist_ok=True)
|
||||
|
||||
if args.height <= 0 or args.width <= 0:
|
||||
raise ValueError(f"Height and width must be positive, got {args.height}x{args.width}")
|
||||
if args.height % _PATCH_SIZE != 0 or args.width % _PATCH_SIZE != 0:
|
||||
raise ValueError(f"Height and width must be multiples of {_PATCH_SIZE}, got {args.height}x{args.width}")
|
||||
|
||||
ar_height = args.height // _PATCH_SIZE
|
||||
ar_width = args.width // _PATCH_SIZE
|
||||
gen_cfg = load_t2i_generation_config(args.model)
|
||||
expected_grid_tokens = ar_height * (ar_width + 1)
|
||||
|
||||
logger.info("Initializing Omni pipeline...")
|
||||
omni = Omni(model=args.model, stage_configs_path=args.stage_config, trust_remote_code=args.trust_remote_code)
|
||||
try:
|
||||
ar_sampling = SamplingParams(
|
||||
temperature=1.0,
|
||||
top_p=1.0,
|
||||
top_k=gen_cfg.top_k,
|
||||
max_tokens=max(1, expected_grid_tokens + 1), # +1 for hidden state of eoi
|
||||
detokenize=False,
|
||||
)
|
||||
dit_sampling = SamplingParams(
|
||||
temperature=0.0,
|
||||
top_p=1.0,
|
||||
top_k=-1,
|
||||
max_tokens=1,
|
||||
detokenize=False,
|
||||
)
|
||||
|
||||
additional_information = {
|
||||
"omni_task": ["t2i"],
|
||||
"ar_width": [ar_width],
|
||||
"ar_height": [ar_height],
|
||||
"eol_token_id": [gen_cfg.eol_token_id],
|
||||
"visual_token_start_id": [gen_cfg.visual_token_start_id],
|
||||
"visual_token_end_id": [gen_cfg.visual_token_end_id],
|
||||
"image_height": [args.height],
|
||||
"image_width": [args.width],
|
||||
"num_inference_steps": [args.num_inference_steps],
|
||||
"text_guidance_scale": [args.text_guidance_scale],
|
||||
"cfg_range": [args.cfg_range[0], args.cfg_range[1]],
|
||||
"visual_ids": gen_cfg.visual_ids,
|
||||
}
|
||||
inputs = [
|
||||
{
|
||||
"prompt": _format_prompt(p, ar_width, ar_height),
|
||||
"additional_information": dict(additional_information),
|
||||
}
|
||||
for p in args.prompt
|
||||
]
|
||||
|
||||
logger.info("Starting generation...")
|
||||
# omni.generate() returns a Generator; consume it to run the full pipeline.
|
||||
outputs = list(omni.generate(inputs, [ar_sampling, dit_sampling]))
|
||||
|
||||
logger.info("Post-processing and saving image(s)...")
|
||||
for path in _save_images(_collect_images(outputs), args.out):
|
||||
logger.info(f"Saved: {path}")
|
||||
finally:
|
||||
omni.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -90,40 +90,14 @@ Pass `--deploy-config /path/to/your_deploy.yaml` to any of the commands above to
|
||||
|
||||
### Image generation (text-to-image / img2img)
|
||||
|
||||
Image generation is served through the standard task examples.
|
||||
The diffusion-stage knobs are declared centrally in `vllm_omni/model_extras/ming_flash_omni.py` and routed via `--extra-body`. With deploy yaml assigned properly, the model can run through the shared example scripts like any other diffusion model:
|
||||
|
||||
Text-to-image (offline):
|
||||
|
||||
```bash
|
||||
python examples/offline_inference/text_to_image/text_to_image.py \
|
||||
--model Jonathan1909/Ming-flash-omni-2.0 \
|
||||
--deploy-config vllm_omni/deploy/ming_flash_omni_image.yaml \
|
||||
--prompt "Please draw a cute cat." \
|
||||
--height 1024 \
|
||||
--width 1024 \
|
||||
--extra-body '{"steps": 30, "cfg": 2.0, "seed": 42}' \
|
||||
--output ming_flash_omni_t2i.png
|
||||
```
|
||||
|
||||
Image-to-image (offline):
|
||||
```bash
|
||||
# Reference image: figures/cases/person_gen_05.png from the upstream Ming repo
|
||||
# https://github.com/inclusionAI/Ming/blob/3954fcb880ff5e61ff128bcf7f1ec344d46a6fe3/examples/vllm_demo.py
|
||||
wget https://raw.githubusercontent.com/inclusionAI/Ming/3954fcb880ff5e61ff128bcf7f1ec344d46a6fe3/figures/cases/person_gen_05.png
|
||||
|
||||
python examples/offline_inference/image_to_image/image_edit.py \
|
||||
--model Jonathan1909/Ming-flash-omni-2.0 \
|
||||
--deploy-config vllm_omni/deploy/ming_flash_omni_image.yaml \
|
||||
--image person_gen_05.png \
|
||||
--prompt "Put a pair of sunglasses on the person." \
|
||||
--extra-args '{"steps": 30, "cfg": 2.0, "seed": 42}' \
|
||||
--output ming_flash_omni_i2i.png
|
||||
```
|
||||
|
||||
For the online path and the full knob list
|
||||
(`steps`/`cfg`/`seed`/`byte5_text`/`negative_prompt`/`height`/`width`), see the
|
||||
[image-generation section in the recipe](../../../recipes/inclusionAI/Ming-flash-omni-2.0.md#image-generation-text-to-image--img2img).
|
||||
The diffusion-stage image-generation path is currently only wired through
|
||||
the online OpenAI-compatible chat endpoint (`/v1/chat/completions` with
|
||||
`"modalities": ["image"]`). For payloads, optional knobs
|
||||
(`steps`/`cfg`/`seed`/`byte5_text`/`negative_prompt`), and the img2img
|
||||
reference-image flow, see the
|
||||
[image-gen section in the online-serving README](../../online_serving/ming_flash_omni/README.md#image-generation-text-to-image).
|
||||
`end2end.py` does not yet exercise the imagegen stage; that is tracked as
|
||||
follow-up work.
|
||||
|
||||
## Online serving
|
||||
|
||||
|
||||
@@ -60,24 +60,3 @@ If media file paths are not provided, the script will use default assets. Suppor
|
||||
- `mixed_modalities`: Audio + image + video
|
||||
- `use_audio_in_video`: Extract audio from video
|
||||
- `text`: Text-only query
|
||||
|
||||
### Composable parallelism (strategy configs)
|
||||
|
||||
You can shard or replicate a stage with a small composable-parallel
|
||||
`strategy.yaml` overlaid onto the **bundled default deploy config** -- no bespoke
|
||||
deploy YAML required. Supply the strategy with `--strategy-config` and any matching
|
||||
device layout via `--stage-overrides` (explicit kwargs forwarded to `Omni`).
|
||||
|
||||
Both examples below require >= 3 GPUs (the thinker uses 2, talker + code2wav share a third):
|
||||
|
||||
```bash
|
||||
# Tensor-parallel the thinker (TP=2)
|
||||
python end2end.py --query-type text --num-prompts 6 \
|
||||
--strategy-config strategy_tp2.yaml \
|
||||
--stage-overrides '{"0": {"devices": "0,1"}, "1": {"devices": "2"}, "2": {"devices": "2"}}'
|
||||
|
||||
# Replicate the thinker across 2 engines (round-robin load balancing)
|
||||
python end2end.py --query-type text --num-prompts 6 \
|
||||
--strategy-config strategy_stage_replica.yaml \
|
||||
--stage-overrides '{"0": {"devices": "0,1"}, "1": {"devices": "2"}, "2": {"devices": "2"}}'
|
||||
```
|
||||
@@ -553,39 +553,6 @@ def parse_args():
|
||||
default=False,
|
||||
help="Use py_generator mode. The returned type of Omni.generate() is a Python Generator object.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--deploy-config",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Optional explicit deploy YAML (otherwise the bundled default is used).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--strategy-config",
|
||||
type=str,
|
||||
default=None,
|
||||
help=(
|
||||
"Optional composable-parallel strategy.yaml mapping role -> parallel axes "
|
||||
"(e.g. tp / stage_replica). Overlaid onto the merged stage configs."
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--stage-overrides",
|
||||
type=str,
|
||||
default=None,
|
||||
help=(
|
||||
"Optional JSON of per-stage overrides applied on top of the default "
|
||||
'deploy config, e.g. \'{"0": {"devices": "0,1"}}\' to give the '
|
||||
"thinker a 2-GPU pool. Lets you run a strategy on the bundled default "
|
||||
"deploy config without writing a bespoke deploy YAML."
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--omni-lb-policy",
|
||||
type=str,
|
||||
default=None,
|
||||
choices=["random", "round-robin", "least-queue-length"],
|
||||
help="StagePool load-balancer policy for replicated stages (orchestrator-level knob).",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
|
||||
@@ -1,22 +0,0 @@
|
||||
# Composable-parallel strategy for Qwen2.5-Omni.
|
||||
#
|
||||
# Replicate the thinker stage across 2 independent engines and let the omni
|
||||
# StagePool balance requests round-robin across them. The thinker's `devices`
|
||||
# entry acts as a per-replica template, so omni expands it to one GPU per replica.
|
||||
# The `routing: round_robin` below auto-derives the StagePool load-balancer policy,
|
||||
# so --omni-lb-policy is not needed.
|
||||
#
|
||||
# Rather than hand-write a bespoke deploy YAML, run on the bundled default deploy
|
||||
# config and pass the matching 2-GPU device template via --stage-overrides (explicit
|
||||
# kwargs): the thinker gets a two-GPU pool while talker + code2wav share a third
|
||||
# card. Requires >= 3 GPUs.
|
||||
#
|
||||
# Usage:
|
||||
# python3 end2end.py --query-type text --num-prompts 6 \
|
||||
# --strategy-config strategy_stage_replica.yaml \
|
||||
# --stage-overrides '{"0": {"devices": "0,1"}, "1": {"devices": "2"}, "2": {"devices": "2"}}'
|
||||
strategies:
|
||||
thinker:
|
||||
- axis: stage_replica
|
||||
size: 2
|
||||
routing: round_robin
|
||||
@@ -1,19 +0,0 @@
|
||||
# Composable-parallel strategy for Qwen2.5-Omni: tensor-parallel the thinker.
|
||||
#
|
||||
# Shard the thinker stage across 2 GPUs with tensor parallelism (TP=2). TP routing
|
||||
# is fixed (broadcast), so no `routing` / load-balancer policy is involved -- this
|
||||
# is the simplest axis and a good starting point for the mental model.
|
||||
#
|
||||
# The strategy sets tensor_parallel_size=2 on the thinker. Rather than hand-write a
|
||||
# bespoke deploy YAML, run on the bundled default deploy config and pass a matching
|
||||
# 2-GPU device layout via --stage-overrides (explicit kwargs), so the thinker gets a
|
||||
# two-GPU pool while talker + code2wav share a third card. Requires >= 3 GPUs.
|
||||
#
|
||||
# Usage:
|
||||
# python3 end2end.py --query-type text --num-prompts 6 \
|
||||
# --strategy-config strategy_tp2.yaml \
|
||||
# --stage-overrides '{"0": {"devices": "0,1"}, "1": {"devices": "2"}, "2": {"devices": "2"}}'
|
||||
strategies:
|
||||
thinker:
|
||||
- axis: tp
|
||||
size: 2
|
||||
@@ -38,9 +38,9 @@ python end2end.py --modality text2text \
|
||||
python end2end.py --prompt "A futuristic cityscape at sunset" \
|
||||
--width 2048 --height 1024 --think
|
||||
|
||||
# Text-to-image with cache acceleration (cache_dit or tea_cache)
|
||||
# Text-to-image with Cache-DiT acceleration
|
||||
python end2end.py --prompt "A cute cat" \
|
||||
--cache-backend tea_cache
|
||||
--cache-backend cache_dit \
|
||||
```
|
||||
|
||||
> **Note**: Default configuration works on a single **NVIDIA A100 (80GB)** or **H100** GPU.
|
||||
@@ -139,7 +139,7 @@ python end2end.py \
|
||||
| `--tensor-parallel-size` | 1 | Number of GPUs for tensor parallelism |
|
||||
| `--enforce-eager` | False | Disable torch.compile |
|
||||
| `--enable-cpu-offload` | False | Enable module-wise (sequential) CPU offload to reduce peak VRAM |
|
||||
| `--cache-backend` | None | Set to `cache_dit` or `tea_cache` for image generation acceleration |
|
||||
| `--cache-backend` | None | Set to `cache_dit` for Cache-DiT acceleration |
|
||||
| `--enable-cache-dit-summary` | False | Print Cache-DiT cache statistics after generation |
|
||||
|
||||
## Reducing GPU Memory Usage
|
||||
@@ -211,4 +211,4 @@ The img2img counterpart lives at
|
||||
## Online Serving
|
||||
|
||||
For OpenAI-compatible API serving, see
|
||||
[`examples/online_serving/sensenova_u1/`](../../online_serving/sensenova_u1/README.md).
|
||||
[`examples/online_serving/sensenova_u1/`](../../examples/online_serving/sensenova_u1/).
|
||||
@@ -139,12 +139,6 @@ def parse_args():
|
||||
default=1,
|
||||
help="Number of GPUs for tensor parallelism.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--cfg-parallel-size",
|
||||
type=int,
|
||||
default=1,
|
||||
help="Number of GPUs for CFG parallelism.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--enforce-eager",
|
||||
action="store_true",
|
||||
@@ -164,7 +158,7 @@ def parse_args():
|
||||
"--cache-backend",
|
||||
type=str,
|
||||
default=None,
|
||||
choices=["cache_dit", "tea_cache"],
|
||||
choices=["cache_dit"],
|
||||
help="Cache backend to use for image generation acceleration.",
|
||||
)
|
||||
parser.add_argument(
|
||||
@@ -200,7 +194,6 @@ def main():
|
||||
enable_cpu_offload=args.enable_cpu_offload,
|
||||
cache_backend=args.cache_backend,
|
||||
enable_cache_dit_summary=args.enable_cache_dit_summary,
|
||||
cfg_parallel_size=args.cfg_parallel_size,
|
||||
)
|
||||
|
||||
extra_args = {
|
||||
@@ -244,7 +237,6 @@ def main():
|
||||
print(f" Seed : {args.seed}")
|
||||
print(f" Think mode : {args.think}")
|
||||
print(f" TP size : {args.tensor_parallel_size}")
|
||||
print(f" CFG size : {args.cfg_parallel_size}")
|
||||
print(f" Cache backend : {args.cache_backend or 'none'}")
|
||||
print(f"{'=' * 60}\n")
|
||||
|
||||
|
||||
@@ -172,29 +172,6 @@ def parse_args() -> argparse.Namespace:
|
||||
default=1,
|
||||
help="Number of GPUs used for ulysses sequence parallelism.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--use-hsdp",
|
||||
action="store_true",
|
||||
help="Enable Hybrid Sharded Data Parallel to shard model weights across GPUs.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--hsdp-shard-size",
|
||||
type=int,
|
||||
default=-1,
|
||||
help=(
|
||||
"Number of GPUs to shard model weights across within each replica group. "
|
||||
"-1 (default) auto-calculates as world_size / replicate_size."
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--hsdp-replicate-size",
|
||||
type=int,
|
||||
default=1,
|
||||
help=(
|
||||
"Number of replica groups for HSDP. Each replica holds a full sharded copy. "
|
||||
"Default 1 means pure sharding (no replication)."
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--enforce-eager",
|
||||
action="store_true",
|
||||
@@ -257,9 +234,6 @@ def main():
|
||||
vae_patch_parallel_size=args.vae_patch_parallel_size,
|
||||
ring_degree=args.ring_degree,
|
||||
ulysses_degree=args.ulysses_degree,
|
||||
use_hsdp=args.use_hsdp,
|
||||
hsdp_shard_size=args.hsdp_shard_size,
|
||||
hsdp_replicate_size=args.hsdp_replicate_size,
|
||||
)
|
||||
|
||||
profiler_enabled = args.profiler_config is not None
|
||||
|
||||
@@ -1,12 +1,5 @@
|
||||
# Text-To-Audio
|
||||
|
||||
A unified script for text/video-to-audio generation. Supported models:
|
||||
|
||||
| Model | Tasks | Notes |
|
||||
|-------|-------|-------|
|
||||
| `stabilityai/stable-audio-open-1.0` | text-to-audio | gated; uses `--audio-length` |
|
||||
| `zhangj1an/AudioX` | `t2a` / `t2m` / `v2a` / `v2m` / `tv2a` / `tv2m` | pass `--task`; video tasks need `--video` |
|
||||
|
||||
The `stabilityai/stable-audio-open-1.0` pipeline generates audio from text prompts.
|
||||
|
||||
## Prerequisites
|
||||
@@ -50,45 +43,13 @@ python text_to_audio.py \
|
||||
--output stable_audio_output.wav
|
||||
```
|
||||
|
||||
### AudioX
|
||||
|
||||
AudioX supports six tasks. Sampler and reference knobs (declared in
|
||||
`vllm_omni/model_extras/audiox.py`) are passed via the generic `--extra-body`
|
||||
JSON flag, routed into sampling `extra_args`.
|
||||
|
||||
Text tasks (`t2a` / `t2m`):
|
||||
|
||||
```bash
|
||||
python text_to_audio.py \
|
||||
--model zhangj1an/AudioX --task t2a \
|
||||
--prompt "Fireworks burst twice, followed by a clock ticking." \
|
||||
--num-inference-steps 250 --guidance-scale 6.0 --audio-length 10.0 --seed 42 \
|
||||
--extra-body '{"sigma_min": 0.03, "sigma_max": 1000.0}' \
|
||||
--output t2a.wav
|
||||
```
|
||||
|
||||
Video-conditioned tasks (`v2a` / `v2m` / `tv2a` / `tv2m`) require `--video`:
|
||||
|
||||
```bash
|
||||
python text_to_audio.py \
|
||||
--model zhangj1an/AudioX --task tv2a \
|
||||
--prompt "drum beating sound and human talking" \
|
||||
--video https://zeyuet.github.io/AudioX/static/samples/V2M/1XeBotOFqHA.mp4 \
|
||||
--num-inference-steps 250 --guidance-scale 6.0 --audio-length 10.0 \
|
||||
--output tv2a.wav
|
||||
```
|
||||
|
||||
Key arguments:
|
||||
|
||||
- `--prompt`: text description (string).
|
||||
- `--task`: [AudioX] one of `t2a`/`t2m`/`v2a`/`v2m`/`tv2a`/`tv2m`.
|
||||
- `--video`: [AudioX `v2*`/`tv2*`] video file/URL for conditioning (→ `video_path`).
|
||||
- `--audio-start`: audio start offset in seconds (→ `audio_start_in_s` for Stable Audio, `seconds_start` for AudioX).
|
||||
- `--audio-length`: audio duration in seconds (audio length for Stable Audio, `seconds_total` for AudioX).
|
||||
- `--extra-body`: JSON dict of model-specific knobs (declared in `vllm_omni/model_extras/`), merged into sampling `extra_args`. For AudioX, sampler/reference knobs go here, e.g. `'{"sigma_min": 0.03, "sigma_max": 1000.0, "cfg_rescale": 0.0, "audio_path": "ref.wav"}'`.
|
||||
- `--negative-prompt`: negative prompt for classifier-free guidance.
|
||||
- `--seed`: integer seed for deterministic generation.
|
||||
- `--guidance-scale`: classifier-free guidance scale.
|
||||
- `--audio-length`: audio duration in seconds.
|
||||
- `--num-inference-steps`: diffusion sampling steps.(more steps = higher quality, slower).
|
||||
- `--use-hsdp`: enable HSDP weight sharding for the Stable Audio DiT.
|
||||
- `--hsdp-shard-size`: number of GPUs used for HSDP sharding.
|
||||
|
||||
@@ -2,70 +2,37 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""
|
||||
Example script for text/video-to-audio generation with diffusion audio models.
|
||||
Example script for text-to-audio generation using Stable Audio Open.
|
||||
|
||||
This script supports:
|
||||
* Stable Audio Open (`stabilityai/stable-audio-open-1.0`) — text-to-audio.
|
||||
* AudioX (`zhangj1an/AudioX`) — six tasks: t2a / t2m / v2a / v2m / tv2a / tv2m.
|
||||
|
||||
Model-specific generation knobs are declared in `vllm_omni/model_extras` and
|
||||
routed into `sampling_params.extra_args` via `apply_declared_extra_args`, so
|
||||
the same script serves both models without per-model branching in the engine.
|
||||
This script demonstrates how to generate audio from text prompts using
|
||||
the Stable Audio Open model with vLLM-Omni.
|
||||
|
||||
Usage:
|
||||
# Stable Audio Open
|
||||
python text_to_audio.py --prompt "The sound of a dog barking"
|
||||
python text_to_audio.py --prompt "A piano playing a gentle melody" --audio-length 10.0
|
||||
python text_to_audio.py --prompt "Thunder and rain sounds" --negative-prompt "Low quality"
|
||||
python text_to_audio.py --prompt "A soft synth pad" --cache-backend tea_cache
|
||||
|
||||
# AudioX (text-to-audio / text-to-music)
|
||||
python text_to_audio.py --model zhangj1an/AudioX --task t2a \
|
||||
--prompt "Fireworks burst twice, then a clock ticking." \
|
||||
--num-inference-steps 250 --guidance-scale 6.0 --audio-length 10.0
|
||||
# AudioX video-conditioned (v2*/tv2*) — pass a video file/URL
|
||||
python text_to_audio.py --model zhangj1an/AudioX --task tv2a \
|
||||
--prompt "drum beating sound and human talking" --video clip.mp4 \
|
||||
--extra-body '{"sigma_min": 0.03, "sigma_max": 1000.0}'
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
from vllm_omni.diffusion.data import DiffusionParallelConfig
|
||||
from vllm_omni.diffusion.utils.param_utils import apply_declared_extra_args
|
||||
from vllm_omni.entrypoints.omni import Omni
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
from vllm_omni.model_extras import get_extra_body_params, get_model_class_name
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
AUDIOX_TASKS = ("t2a", "t2m", "v2a", "v2m", "tv2a", "tv2m")
|
||||
AUDIOX_VIDEO_TASKS = frozenset({"v2a", "v2m", "tv2a", "tv2m"})
|
||||
|
||||
|
||||
def parse_extra_body(value: str) -> dict[str, Any]:
|
||||
"""Parse a JSON object string for --extra-body."""
|
||||
try:
|
||||
obj = json.loads(value)
|
||||
except json.JSONDecodeError as e:
|
||||
raise argparse.ArgumentTypeError(f"--extra-body must be valid JSON: {e}") from e
|
||||
if not isinstance(obj, dict):
|
||||
raise argparse.ArgumentTypeError("--extra-body must be a JSON object")
|
||||
return obj
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description="Generate audio with supported diffusion audio models.")
|
||||
parser = argparse.ArgumentParser(description="Generate audio with Stable Audio Open.")
|
||||
parser.add_argument(
|
||||
"--model",
|
||||
default="stabilityai/stable-audio-open-1.0",
|
||||
help="Audio diffusion model name or local path. Supported: stabilityai/stable-audio-open-1.0, zhangj1an/AudioX.",
|
||||
help="Stable Audio model name or local path.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--prompt",
|
||||
@@ -74,11 +41,8 @@ def parse_args() -> argparse.Namespace:
|
||||
)
|
||||
parser.add_argument(
|
||||
"--negative-prompt",
|
||||
default=None,
|
||||
help=(
|
||||
"Negative prompt for classifier-free guidance. Off by default; "
|
||||
'recommended for Stable Audio, e.g. --negative-prompt "Low quality".'
|
||||
),
|
||||
default="Low quality.",
|
||||
help="Negative prompt for classifier-free guidance.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--seed",
|
||||
@@ -96,19 +60,13 @@ def parse_args() -> argparse.Namespace:
|
||||
"--audio-start",
|
||||
type=float,
|
||||
default=0.0,
|
||||
help=(
|
||||
"Audio start offset in seconds. Applies to both models "
|
||||
"(audio_start_in_s for Stable Audio, seconds_start for AudioX)."
|
||||
),
|
||||
help="Audio start time in seconds.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--audio-length",
|
||||
type=float,
|
||||
default=10.0,
|
||||
help=(
|
||||
"Audio duration in seconds. Maps to audio length for Stable Audio "
|
||||
"(max ~47s for stable-audio-open-1.0) and seconds_total for AudioX."
|
||||
),
|
||||
help="Audio length in seconds (max ~47s for stable-audio-open-1.0).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--num-inference-steps",
|
||||
@@ -122,41 +80,11 @@ def parse_args() -> argparse.Namespace:
|
||||
default=1,
|
||||
help="Number of audio waveforms to generate for the given prompt.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--task",
|
||||
type=str,
|
||||
default=None,
|
||||
choices=list(AUDIOX_TASKS),
|
||||
help=(
|
||||
"[AudioX only] Generation task: t2a/t2m/v2a/v2m/tv2a/tv2m. "
|
||||
"v2*/tv2* require --video. Ignored for Stable Audio Open."
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--video",
|
||||
type=str,
|
||||
default=None,
|
||||
help="[AudioX v2*/tv2*] Video file path or URL used for conditioning.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--extra-body",
|
||||
type=parse_extra_body,
|
||||
default=None,
|
||||
help=(
|
||||
"Model-specific generation params as a JSON object, e.g. "
|
||||
'\'{"sigma_min": 0.03, "sigma_max": 1000.0}\'. Each key is filtered '
|
||||
"against the model's declared extra_body_params (see vllm_omni/model_extras), "
|
||||
"so unknown keys are silently dropped. Values here override the equivalent flags."
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output",
|
||||
type=str,
|
||||
default=None,
|
||||
help=(
|
||||
"Path to save the generated audio (WAV format). Defaults to "
|
||||
"audiox_output.wav for AudioX and stable_audio_output.wav otherwise."
|
||||
),
|
||||
default="stable_audio_output.wav",
|
||||
help="Path to save the generated audio (WAV format).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--sample-rate",
|
||||
@@ -275,8 +203,6 @@ def save_audio(audio_data: np.ndarray, output_path: str, sample_rate: int = 4410
|
||||
|
||||
def main():
|
||||
args = parse_args()
|
||||
model_name = str(args.model).lower() if args.model else ""
|
||||
is_audiox = "audiox" in model_name
|
||||
generator = torch.Generator(device=current_omni_platform.device_type).manual_seed(args.seed)
|
||||
cache_config = None
|
||||
if args.cache_backend == "tea_cache":
|
||||
@@ -285,14 +211,12 @@ def main():
|
||||
}
|
||||
|
||||
print(f"\n{'=' * 60}")
|
||||
print("Text/Video-to-Audio Generation")
|
||||
print("Stable Audio Open - Text-to-Audio Generation")
|
||||
print(f"{'=' * 60}")
|
||||
print(f" Model: {args.model}")
|
||||
if args.task is not None:
|
||||
print(f" Task: {args.task}")
|
||||
print(f" Prompt: {args.prompt}")
|
||||
print(f" Negative prompt: {args.negative_prompt}")
|
||||
print(f" Duration: {args.audio_length}s")
|
||||
print(f" Audio length: {args.audio_length}s")
|
||||
print(f" Inference steps: {args.num_inference_steps}")
|
||||
print(f" Guidance scale: {args.guidance_scale}")
|
||||
print(f" Cache backend: {args.cache_backend if args.cache_backend else 'None (no acceleration)'}")
|
||||
@@ -311,9 +235,8 @@ def main():
|
||||
hsdp_replicate_size=args.hsdp_replicate_size,
|
||||
)
|
||||
|
||||
# Initialize Omni. AudioX requires an explicit pipeline class; Stable Audio
|
||||
# is resolved from its config.
|
||||
omni_kwargs: dict[str, Any] = dict(
|
||||
# Initialize Omni with Stable Audio model
|
||||
omni = Omni(
|
||||
model=args.model,
|
||||
parallel_config=parallel_config,
|
||||
cache_backend=args.cache_backend,
|
||||
@@ -322,68 +245,43 @@ def main():
|
||||
enable_cpu_offload=args.enable_cpu_offload,
|
||||
enable_layerwise_offload=args.enable_layerwise_offload,
|
||||
)
|
||||
if is_audiox:
|
||||
omni_kwargs["model_class_name"] = "AudioXPipeline"
|
||||
omni = Omni(**omni_kwargs)
|
||||
model_class_name = get_model_class_name(omni)
|
||||
declared_extra_body_params = get_extra_body_params(model_class_name)
|
||||
|
||||
diffusion_params = OmniDiffusionSamplingParams(
|
||||
generator=generator,
|
||||
guidance_scale=args.guidance_scale,
|
||||
num_inference_steps=args.num_inference_steps,
|
||||
num_outputs_per_prompt=args.num_waveforms,
|
||||
seed=args.seed,
|
||||
)
|
||||
|
||||
# Negative prompt is opt-in for both models (off by default). Including a
|
||||
# non-empty negative prompt enables the separate CFG conditioning branch.
|
||||
prompt: dict[str, Any] = {"prompt": args.prompt}
|
||||
if args.negative_prompt:
|
||||
prompt["negative_prompt"] = args.negative_prompt
|
||||
|
||||
if model_class_name == "AudioXPipeline":
|
||||
if args.task is None:
|
||||
raise ValueError("AudioX requires --task (one of t2a/t2m/v2a/v2m/tv2a/tv2m).")
|
||||
if args.task in AUDIOX_VIDEO_TASKS and not args.video:
|
||||
raise ValueError(f"AudioX task {args.task!r} requires --video.")
|
||||
# AudioX core knobs are surfaced as shared flags; sampler/reference knobs
|
||||
# (sigma_min/sigma_max/cfg_rescale/audio_path) arrive via --extra-body and
|
||||
# are filtered against the declared extra_body contract.
|
||||
user_extra: dict[str, Any] = {
|
||||
"audiox_task": args.task,
|
||||
"seconds_start": args.audio_start,
|
||||
"seconds_total": args.audio_length,
|
||||
"video_path": args.video,
|
||||
}
|
||||
if args.extra_body:
|
||||
user_extra.update(args.extra_body)
|
||||
apply_declared_extra_args(diffusion_params, declared_extra_body_params, user_extra)
|
||||
else:
|
||||
# Stable Audio Open path (backward-compatible).
|
||||
audio_end_in_s = args.audio_start + args.audio_length
|
||||
diffusion_params.extra_args = {
|
||||
"audio_start_in_s": args.audio_start,
|
||||
"audio_end_in_s": audio_end_in_s,
|
||||
}
|
||||
if args.extra_body:
|
||||
diffusion_params.extra_args.update(args.extra_body)
|
||||
# Calculate audio end time
|
||||
audio_end_in_s = args.audio_start + args.audio_length
|
||||
|
||||
# Time profiling for generation
|
||||
generation_start = time.perf_counter()
|
||||
|
||||
outputs = omni.generate(prompt, diffusion_params)
|
||||
# Generate audio
|
||||
outputs = omni.generate(
|
||||
{
|
||||
"prompt": args.prompt,
|
||||
"negative_prompt": args.negative_prompt,
|
||||
},
|
||||
OmniDiffusionSamplingParams(
|
||||
generator=generator,
|
||||
guidance_scale=args.guidance_scale,
|
||||
num_inference_steps=args.num_inference_steps,
|
||||
num_outputs_per_prompt=args.num_waveforms,
|
||||
extra_args={
|
||||
"audio_start_in_s": args.audio_start,
|
||||
"audio_end_in_s": audio_end_in_s,
|
||||
},
|
||||
),
|
||||
)
|
||||
|
||||
generation_end = time.perf_counter()
|
||||
generation_time = generation_end - generation_start
|
||||
|
||||
print(f"Total generation time: {generation_time:.2f} seconds")
|
||||
|
||||
output = args.output or ("audiox_output.wav" if model_class_name == "AudioXPipeline" else "stable_audio_output.wav")
|
||||
output_path = Path(output)
|
||||
# Process and save audio
|
||||
output_path = Path(args.output)
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
suffix = output_path.suffix or ".wav"
|
||||
stem = output_path.stem or "stable_audio_output"
|
||||
|
||||
# Extract audio from omni.generate() outputs
|
||||
if not outputs:
|
||||
raise ValueError("No output generated from omni.generate()")
|
||||
|
||||
@@ -398,6 +296,7 @@ def main():
|
||||
if audio is None:
|
||||
raise ValueError("No audio output found in request_output")
|
||||
|
||||
# Handle different output formats
|
||||
if isinstance(audio, torch.Tensor):
|
||||
audio = audio.cpu().float().numpy()
|
||||
|
||||
@@ -424,7 +323,7 @@ def main():
|
||||
save_audio(audio, str(output_path), args.sample_rate)
|
||||
print(f"Saved generated audio to {output_path}")
|
||||
|
||||
print(f"\nGenerated ~{args.audio_length}s of audio at {args.sample_rate} Hz")
|
||||
print(f"\nGenerated {args.audio_length}s of audio at {args.sample_rate} Hz")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -179,6 +179,7 @@ python examples/offline_inference/text_to_image/text_to_image.py \
|
||||
--output flux2-dev.png
|
||||
```
|
||||
|
||||
|
||||
### HiDream-I1-Full Models
|
||||
|
||||
The `--auxiliary-text-encoder` parameter is required when running HiDream‑I1‑Full:
|
||||
@@ -198,9 +199,7 @@ python examples/offline_inference/text_to_image/text_to_image.py \
|
||||
|
||||
### Batch Requests (Multiple Prompts)
|
||||
|
||||
You can pass multiple prompts in a single `Omni.generate` call. `Omni`
|
||||
submits each prompt as an independent request and returns one output per
|
||||
prompt.
|
||||
You can pass multiple prompts in a single `generate` call.
|
||||
|
||||
```python
|
||||
from vllm_omni.entrypoints.omni import Omni
|
||||
@@ -225,10 +224,11 @@ if __name__ == "__main__":
|
||||
|
||||
!!! info
|
||||
|
||||
For diffusion pipelines, the input list is sliced into single-item requests
|
||||
before feeding into the diffusion pipeline. For request-level batching
|
||||
controls such as `max_num_seqs`, see
|
||||
[Request-Level Batching](../../../docs/user_guide/diffusion/request_batching.md).
|
||||
For diffusion pipelines, the stage config field `stage_args.[].runtime.max_batch_size` is 1 by
|
||||
default, and the input list is sliced into single-item requests before feeding into the diffusion
|
||||
pipeline. For models that do internally support batched inputs, you can
|
||||
[modify this configuration](../../../configuration/stage_configs.md) to let the model accept a
|
||||
longer batch of prompts.
|
||||
|
||||
### Negative Prompts
|
||||
|
||||
|
||||
@@ -2,7 +2,6 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import argparse
|
||||
import functools
|
||||
import json
|
||||
import time
|
||||
from pathlib import Path
|
||||
@@ -39,20 +38,16 @@ def is_nextstep_model(model_name: str) -> bool:
|
||||
return False
|
||||
|
||||
|
||||
def parse_json_object(value: str, flag_name: str = "argument") -> dict[str, Any]:
|
||||
"""Parse a CLI value as a JSON object, attributing errors to ``flag_name``."""
|
||||
def parse_profiler_config(value: str) -> dict[str, Any]:
|
||||
try:
|
||||
config = json.loads(value)
|
||||
except json.JSONDecodeError as e:
|
||||
raise argparse.ArgumentTypeError(f"{flag_name} must be valid JSON: {e}") from e
|
||||
raise argparse.ArgumentTypeError(f"--profiler-config must be valid JSON: {e}") from e
|
||||
if not isinstance(config, dict):
|
||||
raise argparse.ArgumentTypeError(f"{flag_name} must be a JSON object")
|
||||
raise argparse.ArgumentTypeError("--profiler-config must be a JSON object")
|
||||
return config
|
||||
|
||||
|
||||
parse_profiler_config = functools.partial(parse_json_object, flag_name="--profiler-config")
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description="Generate an image with supported diffusion models.")
|
||||
parser.add_argument(
|
||||
@@ -69,16 +64,7 @@ def parse_args() -> argparse.Namespace:
|
||||
"--stage-configs-path",
|
||||
type=str,
|
||||
default=None,
|
||||
help="[Deprecated] Path to a legacy stage_args-format YAML. Prefer --deploy-config.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--deploy-config",
|
||||
type=str,
|
||||
default=None,
|
||||
help=(
|
||||
"Path to a deploy YAML (new pipeline/stages format). Required for multi-stage "
|
||||
"text-to-image pipelines whose deploy config is not auto-loaded."
|
||||
),
|
||||
help="Path to a YAML file containing stage configurations for Omni.",
|
||||
)
|
||||
parser.add_argument("--prompt", default="a cup of coffee on the table", help="Text prompt for image generation.")
|
||||
parser.add_argument(
|
||||
@@ -164,12 +150,7 @@ def parse_args() -> argparse.Namespace:
|
||||
parser.add_argument(
|
||||
"--enforce-eager",
|
||||
action="store_true",
|
||||
default=None,
|
||||
help=(
|
||||
"Disable torch.compile and force eager execution. Left unset (None) "
|
||||
"so it is only forwarded when explicitly given; "
|
||||
"otherwise the per-stage deploy YAML value wins."
|
||||
),
|
||||
help="Disable torch.compile and force eager execution.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--enable-cpu-offload",
|
||||
@@ -235,13 +216,8 @@ def parse_args() -> argparse.Namespace:
|
||||
parser.add_argument(
|
||||
"--tensor-parallel-size",
|
||||
type=int,
|
||||
default=None,
|
||||
help=(
|
||||
"Number of GPUs used for tensor parallelism (TP) inside the DiT. "
|
||||
"Left unset so it is only forwarded when explicitly given; "
|
||||
"otherwise the per-stage deploy YAML (or engine default of 1) wins. "
|
||||
"Passing it always overrides the deploy YAML."
|
||||
),
|
||||
default=1,
|
||||
help="Number of GPUs used for tensor parallelism (TP) inside the DiT.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--enable-expert-parallel",
|
||||
@@ -293,7 +269,7 @@ def parse_args() -> argparse.Namespace:
|
||||
)
|
||||
parser.add_argument(
|
||||
"--extra-body",
|
||||
type=functools.partial(parse_json_object, flag_name="--extra-body"),
|
||||
type=parse_profiler_config,
|
||||
default=None,
|
||||
help=(
|
||||
"Model-specific generation params as a JSON object, e.g. "
|
||||
@@ -429,8 +405,10 @@ def main():
|
||||
"ring_degree": args.ring_degree,
|
||||
"ulysses_mode": args.ulysses_mode,
|
||||
"cfg_parallel_size": args.cfg_parallel_size,
|
||||
"tensor_parallel_size": args.tensor_parallel_size,
|
||||
"vae_patch_parallel_size": args.vae_patch_parallel_size,
|
||||
"enable_expert_parallel": args.enable_expert_parallel,
|
||||
"enforce_eager": args.enforce_eager,
|
||||
"enable_cpu_offload": args.enable_cpu_offload,
|
||||
"mode": "text-to-image",
|
||||
"log_stats": args.log_stats,
|
||||
@@ -442,21 +420,11 @@ def main():
|
||||
**lora_args,
|
||||
**quant_kwargs,
|
||||
}
|
||||
if args.tensor_parallel_size is not None:
|
||||
omni_kwargs["tensor_parallel_size"] = args.tensor_parallel_size
|
||||
if args.enforce_eager is not None:
|
||||
omni_kwargs["enforce_eager"] = args.enforce_eager
|
||||
if args.stage_configs_path:
|
||||
omni_kwargs["stage_configs_path"] = args.stage_configs_path
|
||||
if args.deploy_config:
|
||||
omni_kwargs["deploy_config"] = args.deploy_config
|
||||
if use_nextstep:
|
||||
# NextStep-1.1 requires explicit pipeline class
|
||||
omni_kwargs["model_class_name"] = "NextStep11Pipeline"
|
||||
# Cosmos3 loads its (gated) guardrail models at build time, so the guardrails
|
||||
# gate is an engine-level config (offline analog of the server's --no-guardrails).
|
||||
if args.extra_body and "guardrails" in args.extra_body:
|
||||
omni_kwargs["model_config"] = {"guardrails": bool(args.extra_body["guardrails"])}
|
||||
omni = Omni(**omni_kwargs)
|
||||
model_class_name = get_model_class_name(omni)
|
||||
declared_extra_body_params = get_extra_body_params(model_class_name)
|
||||
@@ -474,9 +442,8 @@ def main():
|
||||
print(f" Quantization: {args.quantization if args.quantization else 'None (BF16)'}")
|
||||
if ignored_layers:
|
||||
print(f" Ignored layers: {ignored_layers}")
|
||||
tp_display = args.tensor_parallel_size if args.tensor_parallel_size is not None else "deploy/default"
|
||||
print(
|
||||
f" Parallel configuration: tensor_parallel_size={tp_display}, "
|
||||
f" Parallel configuration: tensor_parallel_size={args.tensor_parallel_size}, "
|
||||
f"ulysses_degree={args.ulysses_degree}, ulysses_mode={args.ulysses_mode}, "
|
||||
f"ring_degree={args.ring_degree}, cfg_parallel_size={args.cfg_parallel_size}, "
|
||||
f"vae_patch_parallel_size={args.vae_patch_parallel_size}, "
|
||||
@@ -546,8 +513,8 @@ def main():
|
||||
diffusion_params.extra_args.update({k: v for k, v in user_extra.items() if v is not None})
|
||||
|
||||
if lora_request:
|
||||
diffusion_params.lora_request = lora_request
|
||||
diffusion_params.lora_scale = args.lora_scale
|
||||
diffusion_params.extra_args["lora_request"] = lora_request
|
||||
diffusion_params.extra_args["lora_scale"] = args.lora_scale
|
||||
|
||||
# Build per-stage sampling params for multi-stage models (e.g. BAGEL),
|
||||
# or wrap single diffusion params for single-stage models.
|
||||
@@ -613,12 +580,6 @@ def main():
|
||||
if images:
|
||||
break
|
||||
|
||||
# Fallback: generation-stage pipelines (e.g. MammothModa2's AR->DiT) return the
|
||||
# generated image as a tensor under multimodal_output instead of populating the
|
||||
# `images` field that diffusion-stage pipelines fill.
|
||||
if not images:
|
||||
images = _images_from_multimodal_output(outputs)
|
||||
|
||||
if not images:
|
||||
raise ValueError("No images found in request_output")
|
||||
|
||||
@@ -636,36 +597,5 @@ def main():
|
||||
print(f"Saved generated image to {save_path}")
|
||||
|
||||
|
||||
def _images_from_multimodal_output(outputs: list[Any]) -> list[Any]:
|
||||
"""Extract PIL images from multimodal_output tensors.
|
||||
|
||||
Generation-stage pipelines (e.g. MammothModa2's AR->DiT) return the generated
|
||||
image as a tensor (normalized to [-1, 1], CHW) under ``multimodal_output``
|
||||
rather than populating the ``images`` field. Convert any such tensors to PIL.
|
||||
"""
|
||||
from PIL import Image
|
||||
|
||||
pil_images: list[Any] = []
|
||||
for output in outputs:
|
||||
req_out = getattr(output, "request_output", output)
|
||||
for completion in getattr(req_out, "outputs", None) or []:
|
||||
# multimodal_output is a MultimodalPayload (a Mapping) keyed by modality,
|
||||
# matching how omni examples (ming_flash_omni / magi_human / dynin) read it.
|
||||
mm = getattr(completion, "multimodal_output", None) or {}
|
||||
if "image" not in mm:
|
||||
continue
|
||||
payload = mm["image"]
|
||||
for tensor in payload if isinstance(payload, list) else [payload]:
|
||||
if not isinstance(tensor, torch.Tensor):
|
||||
continue
|
||||
img = tensor.detach().to("cpu", dtype=torch.float32)
|
||||
if img.ndim == 4:
|
||||
img = img[0]
|
||||
img = (img / 2 + 0.5).clamp(0, 1).mul(255).to(torch.uint8)
|
||||
img = img.permute(1, 2, 0).contiguous().numpy()
|
||||
pil_images.append(Image.fromarray(img))
|
||||
return pil_images
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -23,7 +23,6 @@ list of supported architectures across all modalities, see
|
||||
| OmniVoice | `k2-fsa/OmniVoice` | 2 (gen + dec) | ✓ | — | voice design, language hint | 24 kHz |
|
||||
| Qwen3-TTS | `Qwen/Qwen3-TTS-12Hz-1.7B-{CustomVoice,VoiceDesign,Base}` | 2 (talker + code2wav) | ✓ (Base) | ✓ | 3 task variants | 24 kHz |
|
||||
| VoxCPM2 | `openbmb/VoxCPM2` | single (native AR) | ✓ | ✓ (online) | continuation | 48 kHz |
|
||||
| IndexTTS-2 | `IndexTeam/IndexTTS-2` | 2 (AR talker + S2Mel DiT + BigVGAN) | ✓ (required) | — | emotion control (`--emo-audio`, `--emo-text`, `--emo-vector`) | 22.05 kHz |
|
||||
| Voxtral TTS | `mistralai/Voxtral-4B-TTS-2603` | varies | ✓ | ✓ | voice presets | 24 kHz |
|
||||
|
||||
## Common Quick Start
|
||||
@@ -437,50 +436,6 @@ Streaming is exposed through the online OpenAI Speech API (`stream=true`). See [
|
||||
|
||||
---
|
||||
|
||||
## IndexTTS-2
|
||||
|
||||
2-stage TTS pipeline (GPT AR talker + S2Mel CFM DiT + BigVGAN vocoder) at 22.05 kHz. Every request requires reference audio for zero-shot voice cloning. Supports emotion conditioning via audio, text, or 8-dim vector.
|
||||
|
||||
### Quick start
|
||||
```bash
|
||||
python examples/offline_inference/text_to_speech/indextts2/end2end.py \
|
||||
--model IndexTeam/IndexTTS-2 \
|
||||
--text "你好，这是一个语音合成测试。" \
|
||||
--ref-audio /path/to/reference.wav
|
||||
```
|
||||
|
||||
### Emotion control
|
||||
```bash
|
||||
# Emotion from reference audio
|
||||
python examples/offline_inference/text_to_speech/indextts2/end2end.py \
|
||||
--model IndexTeam/IndexTTS-2 \
|
||||
--text "今天天气真好！" \
|
||||
--ref-audio /path/to/ref.wav \
|
||||
--emo-audio /path/to/happy.wav
|
||||
|
||||
# Emotion from 8-dim vector (happy angry sad afraid disgusted melancholy surprised calm)
|
||||
python examples/offline_inference/text_to_speech/indextts2/end2end.py \
|
||||
--model IndexTeam/IndexTTS-2 \
|
||||
--text "今天天气真好！" \
|
||||
--ref-audio /path/to/ref.wav \
|
||||
--emo-vector 1.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0
|
||||
|
||||
# Emotion from text description
|
||||
python examples/offline_inference/text_to_speech/indextts2/end2end.py \
|
||||
--model IndexTeam/IndexTTS-2 \
|
||||
--text "今天天气真好！" \
|
||||
--ref-audio /path/to/ref.wav \
|
||||
--emo-text "happy and excited"
|
||||
```
|
||||
|
||||
### Notes
|
||||
- `--ref-audio` is **required** — IndexTTS-2 does not support text-only synthesis.
|
||||
- Stage 0 (AR Talker): GPT-2 generates mel codes from text + reference audio.
|
||||
- Stage 1 (S2Mel + BigVGAN): CFM DiT converts mel codes to waveform at 22.05 kHz.
|
||||
- Deploy config: `vllm_omni/deploy/indextts2.yaml`. Stage 1 runs with `enforce_eager: true` (DiT has dynamic shapes).
|
||||
|
||||
---
|
||||
|
||||
## Voxtral TTS
|
||||
|
||||
Voxtral-4B-TTS (Mistral). Has its own argparse surface; uses voice presets and the `mistral_common` `SpeechRequest` protocol.
|
||||
@@ -521,100 +476,3 @@ Available voice presets are listed on the HF model card (`mistralai/Voxtral-4B-T
|
||||
- `--num-prompts N` replicates the prompt for performance measurement.
|
||||
- `--concurrency M` requires `--streaming` and must evenly divide `--num-prompts`.
|
||||
- Run `--help` for the full argument surface.
|
||||
|
||||
---
|
||||
|
||||
## SoulX-Singer
|
||||
|
||||
Singing voice synthesis (SVS) and conversion (SVC) at 24 kHz. Script: `soulxsinger/end2end.py`. Deploy: `vllm_omni/deploy/soulxsinger_svs.yaml` or `soulxsinger_svc.yaml`.
|
||||
|
||||
### Prerequisites
|
||||
|
||||
Download DiT and preprocess weights, then set up separate SVS / SVC view directories. Copy `soulxsinger/utils/phoneme/phone_set.json` from upstream [SoulX-Singer](https://github.com/Soul-AILab/SoulX-Singer) into the model weights dir as `phoneme/phone_set.json` — HuggingFace does not ship it.
|
||||
|
||||
```bash
|
||||
# 1. DiT weights
|
||||
export BASE=path/to/SoulX-Singer
|
||||
export PREPROCESS=path/to/SoulX-Singer-Preprocess
|
||||
export SVC_DIR=path/to/SoulX-Singer-svc
|
||||
|
||||
huggingface-cli download Soul-AILab/SoulX-Singer --local-dir "$BASE"
|
||||
|
||||
# 2. Preprocess weights (required)
|
||||
huggingface-cli download Soul-AILab/SoulX-Singer-Preprocess --local-dir "$PREPROCESS"
|
||||
export SOULX_PREPROCESS_WEIGHTS_DIR="$PREPROCESS"
|
||||
|
||||
# 3. SVS / SVC view directories
|
||||
mkdir -p "$SVC_DIR"
|
||||
cp $BASE/{config.yaml,README.md,assets} $SVC_DIR
|
||||
mv $BASE/model-svc.pt $SVC_DIR/model-svc.pt
|
||||
|
||||
cat > "$BASE/config.json" <<'EOF'
|
||||
{
|
||||
"model_type": "soulxsinger",
|
||||
"architectures": ["SoulXSingerPipeline"],
|
||||
"max_num_seqs": 1
|
||||
}
|
||||
EOF
|
||||
|
||||
cat > "$SVC_DIR/config.json" <<'EOF'
|
||||
{
|
||||
"model_type": "soulxsinger",
|
||||
"architectures": ["SoulXSingerSVCPipeline"],
|
||||
"max_num_seqs": 1
|
||||
}
|
||||
EOF
|
||||
```
|
||||
|
||||
`config.yaml` hyper-parameters live under `$BASE`; each view's `config.json` `architectures` field is the single source of truth for SVS vs SVC. Point `--model` at the matching directory (`$BASE` for SVS, `$SVC_DIR` for SVC). Deploy YAML is chosen automatically from `config.json`; optional `--svs` / `--svc` only assert the mode matches.
|
||||
|
||||
**Online preprocess** is the default: pass `--prompt-audio` and `--target-audio`, and the worker runs vocal separation, F0, and (for SVS) lyrics/MIDI before DiT. Install only what your run needs:
|
||||
|
||||
```bash
|
||||
pip install "BS-RoFormer" # vocal sep + F0 on GPU — SVS and SVC
|
||||
```
|
||||
|
||||
Mandarin SVS also needs FunASR and Chinese G2P; `ffmpeg` must be on `PATH`:
|
||||
|
||||
```bash
|
||||
# install optional dependencies:
|
||||
pip install -e ".[soulx-svs]"
|
||||
```
|
||||
|
||||
English SVS adds NeMo ASR and NLTK data; pass `--language English`:
|
||||
|
||||
```bash
|
||||
pip install "nemo_toolkit[asr]==2.6.1" lhotse==1.32.2
|
||||
python -c "import nltk; nltk.download('cmudict'); nltk.download('averaged_perceptron_tagger_eng')"
|
||||
```
|
||||
|
||||
**Precomputed metadata** is the alternative: pass both `--prompt-metadata-path` and `--target-metadata-path` and skip online ASR/ROSVOT — none of the packages above are required. JSON can be produced by integrated preprocess on a prior run, or by upstream [SoulX-Singer](https://github.com/Soul-AILab/SoulX-Singer) `preprocess/` scripts if you prefer to run that outside vLLM-Omni.
|
||||
|
||||
### Quick start
|
||||
|
||||
```bash
|
||||
# SVS — default demo audio: tests/assets/soulxsinger/zh_prompt.mp3 + music.mp3
|
||||
python examples/offline_inference/text_to_speech/soulxsinger/end2end.py \
|
||||
--model "$BASE" \
|
||||
--preprocess-weights-dir "$PREPROCESS" \
|
||||
--control score \
|
||||
--num-inference-steps 32 \
|
||||
-o output.wav
|
||||
|
||||
python examples/offline_inference/text_to_speech/soulxsinger/end2end.py \
|
||||
--model "$SVC_DIR" \
|
||||
--preprocess-weights-dir "$PREPROCESS" \
|
||||
--svc \
|
||||
--num-inference-steps 32 \
|
||||
-o output_svc.wav
|
||||
```
|
||||
|
||||
`SOULX_PREPROCESS_WEIGHTS_DIR` makes `--preprocess-weights-dir` optional. Long SVS targets are handled in one request. See `end2end.py --help` for `--pitch-shift`, `--vocal-sep`, `--auto-shift`, and language/control options.
|
||||
|
||||
### Notes
|
||||
|
||||
- Output: 24 kHz mono WAV; batch only.
|
||||
- Defaults match upstream: `--guidance-scale 3.0`, `--seed 42`, `--auto-shift` on.
|
||||
- SVS `--control`: `score` or `melody`. MIDI / lyric QC: upstream `midi_editor` only.
|
||||
|
||||
---
|
||||
@@ -101,7 +101,7 @@ def _extract_sample_rate(audio_mm: dict) -> int:
|
||||
def _modify_deploy_config(base_path: str, async_chunk: bool) -> str:
|
||||
"""Build deploy config with explicit sync/async mode and eager execution.
|
||||
|
||||
Mirrors the logic in ``tests/e2e/offline_inference/test_glm_tts_expansion.py``
|
||||
Mirrors the logic in ``tests/e2e/offline_inference/test_glm_tts.py``
|
||||
(``_get_deploy_config``) so that example runs match CI behavior.
|
||||
"""
|
||||
with open(base_path) as f:
|
||||
|
||||
@@ -1,187 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Offline inference example for IndexTTS2 via vLLM-Omni.
|
||||
|
||||
Two-stage pipeline: GPT AR (Stage 0) → S2Mel + BigVGAN (Stage 1).
|
||||
Output is 22050 Hz mono WAV.
|
||||
|
||||
Usage:
|
||||
python end2end.py \
|
||||
--model /path/to/IndexTeam/IndexTTS-2 \
|
||||
--text "你好，这是一个语音合成测试。" \
|
||||
--ref-audio /path/to/ref.wav
|
||||
|
||||
# With emotion audio:
|
||||
python end2end.py \
|
||||
--model /path/to/IndexTeam/IndexTTS-2 \
|
||||
--text "今天天气真好！" \
|
||||
--ref-audio /path/to/ref.wav \
|
||||
--emo-audio /path/to/happy.wav
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
import soundfile as sf
|
||||
import torch
|
||||
from vllm import SamplingParams
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
|
||||
os.environ.setdefault("VLLM_WORKER_MULTIPROC_METHOD", "spawn")
|
||||
|
||||
from vllm_omni import Omni # noqa: E402
|
||||
from vllm_omni.model_executor.models.indextts2.prompt_utils import ( # noqa: E402
|
||||
build_indextts2_prefill_prompt_ids,
|
||||
)
|
||||
|
||||
|
||||
def build_request(
|
||||
model: str,
|
||||
text: str,
|
||||
ref_audio_path: str | None = None,
|
||||
emo_audio_path: str | None = None,
|
||||
emo_text: str | None = None,
|
||||
emo_vector: list[float] | None = None,
|
||||
emo_alpha: float | None = None,
|
||||
use_emo_text: bool = False,
|
||||
use_random: bool = False,
|
||||
) -> dict:
|
||||
additional: dict = {"text": [text]}
|
||||
if ref_audio_path:
|
||||
additional["voice"] = [str(ref_audio_path)]
|
||||
if emo_audio_path:
|
||||
additional["emo_audio"] = [str(emo_audio_path)]
|
||||
if emo_text:
|
||||
additional["emo_text"] = [emo_text]
|
||||
if emo_vector is not None:
|
||||
additional["emo_vector"] = [emo_vector]
|
||||
if emo_alpha is not None:
|
||||
additional["emo_alpha"] = [emo_alpha]
|
||||
if use_emo_text:
|
||||
additional["use_emo_text"] = [True]
|
||||
if use_random:
|
||||
additional["use_random"] = [True]
|
||||
return {
|
||||
"prompt_token_ids": build_indextts2_prefill_prompt_ids(model, text),
|
||||
"additional_information": additional,
|
||||
}
|
||||
|
||||
|
||||
def save_audio(waveform: torch.Tensor, path: str, sample_rate: int = 22050) -> None:
|
||||
audio_np = waveform.float().numpy()
|
||||
sf.write(path, audio_np, sample_rate)
|
||||
print(f" Saved {path} ({audio_np.shape}, {sample_rate} Hz)")
|
||||
|
||||
|
||||
def extract_audio(mm: dict) -> tuple[torch.Tensor | None, int]:
|
||||
audio = mm.get("audio")
|
||||
if audio is None:
|
||||
audio = mm.get("model_outputs")
|
||||
if isinstance(audio, list):
|
||||
chunks = [chunk.reshape(-1) for chunk in audio if isinstance(chunk, torch.Tensor) and chunk.numel() > 0]
|
||||
audio = torch.cat(chunks, dim=0) if chunks else None
|
||||
|
||||
sr_val = mm.get("sr")
|
||||
if isinstance(sr_val, list):
|
||||
sr_val = sr_val[-1] if sr_val else None
|
||||
if hasattr(sr_val, "item"):
|
||||
sample_rate = int(sr_val.item())
|
||||
else:
|
||||
sample_rate = int(sr_val) if sr_val is not None else 22050
|
||||
|
||||
return audio if isinstance(audio, torch.Tensor) else None, sample_rate
|
||||
|
||||
|
||||
def main(args) -> None:
|
||||
omni = Omni(
|
||||
model=args.model,
|
||||
deploy_config=args.deploy_config,
|
||||
stage_init_timeout=args.stage_init_timeout,
|
||||
)
|
||||
|
||||
# Stage 0: GPT AR (text → mel codes)
|
||||
gpt_sampling = SamplingParams(
|
||||
temperature=0.8,
|
||||
top_p=0.8,
|
||||
top_k=30,
|
||||
max_tokens=1500,
|
||||
repetition_penalty=10.0,
|
||||
stop_token_ids=[8193],
|
||||
seed=args.seed if args.seed is not None else 42,
|
||||
detokenize=False,
|
||||
)
|
||||
# Stage 1: S2Mel + BigVGAN (non-AR, params mostly ignored)
|
||||
s2mel_sampling = SamplingParams(
|
||||
temperature=0.0,
|
||||
max_tokens=65536,
|
||||
detokenize=True,
|
||||
)
|
||||
sampling_params = [gpt_sampling, s2mel_sampling]
|
||||
|
||||
output_dir = Path(args.output_dir)
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
print(f"Synthesizing: {args.text!r}")
|
||||
if args.ref_audio:
|
||||
print(f" ref_audio: {args.ref_audio}")
|
||||
inputs = build_request(
|
||||
model=args.model,
|
||||
text=args.text,
|
||||
ref_audio_path=args.ref_audio,
|
||||
emo_audio_path=args.emo_audio,
|
||||
emo_text=args.emo_text,
|
||||
emo_vector=args.emo_vector,
|
||||
emo_alpha=args.emo_alpha,
|
||||
use_emo_text=args.use_emo_text,
|
||||
use_random=args.use_random,
|
||||
)
|
||||
|
||||
for i, omni_out in enumerate(omni.generate(inputs, sampling_params_list=sampling_params)):
|
||||
mm = omni_out.multimodal_output
|
||||
if not mm:
|
||||
print(f" [req {i}] No multimodal output.")
|
||||
continue
|
||||
audio, sr = extract_audio(mm)
|
||||
if audio is None:
|
||||
print(f" [req {i}] No waveform in multimodal_output.")
|
||||
continue
|
||||
out_path = str(output_dir / f"output_{i}.wav")
|
||||
save_audio(audio.cpu(), out_path, sr)
|
||||
|
||||
print("Done.")
|
||||
|
||||
|
||||
def parse_args():
|
||||
parser = FlexibleArgumentParser(description="IndexTTS2 offline inference")
|
||||
parser.add_argument("--model", required=True, help="HF model path for IndexTTS2.")
|
||||
parser.add_argument("--text", default="你好，这是IndexTTS2语音合成测试。")
|
||||
parser.add_argument("--ref-audio", required=True, help="Reference audio for voice cloning.")
|
||||
parser.add_argument("--emo-audio", default=None, help="Emotion reference audio.")
|
||||
parser.add_argument("--emo-text", default=None, help="Emotion description text.")
|
||||
parser.add_argument(
|
||||
"--emo-vector",
|
||||
type=float,
|
||||
nargs=8,
|
||||
default=None,
|
||||
help="8-dim emotion vector: happy angry sad afraid disgusted melancholic surprised calm.",
|
||||
)
|
||||
parser.add_argument("--emo-alpha", type=float, default=None, help="Emotion weight in [0, 1].")
|
||||
parser.add_argument("--use-emo-text", action="store_true", help="Infer emotion vector from emo-text or text.")
|
||||
parser.add_argument("--use-random", action="store_true", help="Use random emotion prototypes.")
|
||||
parser.add_argument("--seed", type=int, default=None)
|
||||
parser.add_argument(
|
||||
"--output-dir",
|
||||
default=os.path.join(
|
||||
os.environ.get("XDG_CACHE_HOME", os.path.join(os.path.expanduser("~"), ".cache")),
|
||||
"indextts2_output",
|
||||
),
|
||||
)
|
||||
parser.add_argument("--deploy-config", default=None)
|
||||
parser.add_argument("--stage-init-timeout", type=int, default=600)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main(parse_args())
|
||||
@@ -1,12 +1,11 @@
|
||||
"""Offline inference example for MOSS-TTS variants.
|
||||
|
||||
Covers six models:
|
||||
- OpenMOSS-Team/MOSS-TTS (8B, general TTS)
|
||||
- OpenMOSS-Team/MOSS-TTSD-v1.0 (8B, dialogue TTS)
|
||||
- OpenMOSS-Team/MOSS-SoundEffect (8B, sound effect synthesis)
|
||||
- OpenMOSS-Team/MOSS-TTS-Realtime (1.7B, low-latency streaming)
|
||||
- OpenMOSS-Team/MOSS-VoiceGenerator (1.7B, voice design)
|
||||
- OpenMOSS-Team/MOSS-TTS-Local-Transformer-v1.5 (n_vq=12, 48 kHz stereo)
|
||||
Covers all five models:
|
||||
- OpenMOSS-Team/MOSS-TTS (8B, general TTS)
|
||||
- OpenMOSS-Team/MOSS-TTSD-v1.0 (8B, dialogue TTS)
|
||||
- OpenMOSS-Team/MOSS-SoundEffect (8B, sound effect synthesis)
|
||||
- OpenMOSS-Team/MOSS-TTS-Realtime (1.7B, low-latency streaming)
|
||||
- OpenMOSS-Team/MOSS-VoiceGenerator(1.7B, voice design)
|
||||
|
||||
Usage
|
||||
-----
|
||||
@@ -211,18 +210,12 @@ def run_tts(args: argparse.Namespace) -> None:
|
||||
"OpenMOSS-Team/MOSS-SoundEffect": "moss_sound_effect",
|
||||
"OpenMOSS-Team/MOSS-TTS-Realtime": "moss_tts_realtime",
|
||||
"OpenMOSS-Team/MOSS-VoiceGenerator": "moss_voice_generator",
|
||||
"OpenMOSS-Team/MOSS-TTS-Local-Transformer-v1.5": "moss_tts_local",
|
||||
}
|
||||
deploy_config = deploy_map.get(args.model, "moss_tts")
|
||||
|
||||
is_sound_effect = "SoundEffect" in args.model
|
||||
is_voice_gen = "VoiceGenerator" in args.model
|
||||
is_realtime = "Realtime" in args.model
|
||||
# Local-v1.5 ships its own AutoProcessor (processor_config.json +
|
||||
# processing_moss_tts.py), so it reuses the same _build_unified_codes
|
||||
# path as the Delay variants below — just a different n_vq/codec.
|
||||
is_local = "Local" in args.model
|
||||
output_sr = 48000 if is_local else 24000
|
||||
|
||||
ref_audio = None
|
||||
builder_kwargs: dict = {}
|
||||
@@ -321,33 +314,27 @@ def run_tts(args: argparse.Namespace) -> None:
|
||||
wav = mm.get("model_outputs")
|
||||
if wav is None:
|
||||
continue
|
||||
# Local-v1.5's codec emits stereo chunks shaped (C, T); keep the
|
||||
# channel axis intact (don't flatten) so concatenation below stays
|
||||
# along the time axis. Mono variants stay (T,) as before.
|
||||
if isinstance(wav, list):
|
||||
non_empty = [w for w in wav if isinstance(w, torch.Tensor) and w.numel() > 0]
|
||||
non_empty = [w.reshape(-1) for w in wav if isinstance(w, torch.Tensor) and w.numel() > 0]
|
||||
if non_empty and t_first_audio is None:
|
||||
t_first_audio = _time.monotonic()
|
||||
chunks.extend(non_empty)
|
||||
elif isinstance(wav, torch.Tensor) and wav.numel() > 0:
|
||||
if t_first_audio is None:
|
||||
t_first_audio = _time.monotonic()
|
||||
chunks.append(wav)
|
||||
chunks.append(wav.reshape(-1))
|
||||
|
||||
t_gen_end = _time.monotonic()
|
||||
if not chunks:
|
||||
print("No audio in output — check model loading logs.")
|
||||
return
|
||||
|
||||
cat_dim = -1 if chunks[0].dim() == 2 else 0
|
||||
wav_t = torch.cat(chunks, dim=cat_dim).float().cpu()
|
||||
n_samples = wav_t.shape[-1]
|
||||
# soundfile wants (T,) mono or (T, C) multi-channel.
|
||||
sf.write(args.output, (wav_t.transpose(0, 1) if wav_t.dim() == 2 else wav_t).numpy(), output_sr)
|
||||
print(f"Saved {n_samples / output_sr:.2f}s of audio to {args.output}")
|
||||
wav_t = torch.cat(chunks).float().cpu()
|
||||
sf.write(args.output, wav_t.numpy(), 24000)
|
||||
print(f"Saved {len(wav_t) / 24000:.2f}s of audio to {args.output}")
|
||||
ttfa_ms = (t_first_audio - t_gen_start) * 1000.0 if t_first_audio is not None else float("nan")
|
||||
gen_total_ms = (t_gen_end - t_gen_start) * 1000.0
|
||||
print(f"TTFA_MS={ttfa_ms:.1f} GEN_TOTAL_MS={gen_total_ms:.1f} AUDIO_S={n_samples / output_sr:.2f}")
|
||||
print(f"TTFA_MS={ttfa_ms:.1f} GEN_TOTAL_MS={gen_total_ms:.1f} AUDIO_S={len(wav_t) / 24000:.2f}")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
|
||||
@@ -34,7 +34,7 @@ def run_e2e():
|
||||
help="Model name or path (HuggingFace or local)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--deploy-config",
|
||||
"--stage-config",
|
||||
type=str,
|
||||
default="vllm_omni/deploy/omnivoice.yaml",
|
||||
)
|
||||
@@ -87,14 +87,14 @@ def run_e2e():
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
if not os.path.exists(args.deploy_config):
|
||||
raise FileNotFoundError(f"Deploy config not found: {args.deploy_config}")
|
||||
if not os.path.exists(args.stage_config):
|
||||
raise FileNotFoundError(f"Stage config not found: {args.stage_config}")
|
||||
|
||||
print(f"Initializing OmniVoice with model={args.model}")
|
||||
|
||||
omni = Omni(
|
||||
model=args.model,
|
||||
deploy_config=args.deploy_config,
|
||||
stage_configs_path=args.stage_config,
|
||||
log_stats=True,
|
||||
)
|
||||
|
||||
|
||||
@@ -1,162 +0,0 @@
|
||||
"""Repeatable SoulX-Singer offline benchmark (preprocess + DiT, RTF + stage timings).
|
||||
|
||||
Usage:
|
||||
python benchmark.py --model /path/to/SoulX-Singer --svs \\
|
||||
--prompt-audio ../SoulX-Singer/example/audio/zh_prompt.mp3 \\
|
||||
--target-audio ../SoulX-Singer/example/audio/music.mp3 \\
|
||||
--preprocess-weights-dir /path/to/SoulX-Singer-Preprocess \\
|
||||
-o benchmark.wav
|
||||
|
||||
python benchmark.py ... --enable-diffusion-pipeline-profiler
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import statistics
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import soundfile as sf
|
||||
from end2end import (
|
||||
SVC_DEPLOY_CONFIG,
|
||||
SVS_DEPLOY_CONFIG,
|
||||
add_inference_args,
|
||||
build_sampling,
|
||||
extract_audio,
|
||||
resolve_preprocess_weights_dir,
|
||||
)
|
||||
|
||||
from vllm_omni.entrypoints.omni import Omni
|
||||
|
||||
|
||||
def _audio_duration_sec(outputs) -> float:
|
||||
audio_np, sr = extract_audio(outputs)
|
||||
return audio_np.size / sr
|
||||
|
||||
|
||||
def _stage_durations_from_output(outputs) -> dict[str, float]:
|
||||
omni_out = outputs[0]
|
||||
durations = getattr(omni_out, "stage_durations", None) or {}
|
||||
if durations:
|
||||
return dict(durations)
|
||||
ro = getattr(omni_out, "request_output", None)
|
||||
if ro is not None:
|
||||
return dict(getattr(ro, "stage_durations", None) or {})
|
||||
return {}
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description="SoulX-Singer offline benchmark")
|
||||
add_inference_args(parser)
|
||||
parser.add_argument("--warmup", type=int, default=1, help="Warmup runs (excluded from stats)")
|
||||
parser.add_argument("--runs", type=int, default=3, help="Measured runs")
|
||||
parser.add_argument(
|
||||
"--enable-diffusion-pipeline-profiler",
|
||||
action="store_true",
|
||||
help="Enable low-overhead stage timing in DiffusionOutput.stage_durations",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--enforce-eager",
|
||||
action="store_true",
|
||||
help="Disable torch.compile on diff_estimator (eager baseline for A/B)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--dtype",
|
||||
type=str,
|
||||
default=None,
|
||||
choices=["bfloat16", "bf16", "float16", "fp16", "half"],
|
||||
help="DiT trunk dtype (default from deploy YAML). Use float16 to match upstream --fp16.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output",
|
||||
"-o",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Save WAV from the last measured run (after warmup)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--save-each-run",
|
||||
action="store_true",
|
||||
help="With --output, also write measured runs as <stem>_run1.wav, _run2.wav, ...",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def main() -> None:
|
||||
args = parse_args()
|
||||
preprocess_dir = resolve_preprocess_weights_dir(args.preprocess_weights_dir)
|
||||
deploy_config = args.deploy_config or str(SVS_DEPLOY_CONFIG if args.svs else SVC_DEPLOY_CONFIG)
|
||||
mode = "SVS" if args.svs else "SVC"
|
||||
|
||||
omni_kwargs: dict = {
|
||||
"model": args.model,
|
||||
"deploy_config": deploy_config,
|
||||
"async_chunk": False, # SoulX-Singer currently supports only batch mode (pseudo-streaming was stashed)
|
||||
}
|
||||
if args.enable_diffusion_pipeline_profiler:
|
||||
omni_kwargs["enable_diffusion_pipeline_profiler"] = True
|
||||
if args.enforce_eager:
|
||||
omni_kwargs["enforce_eager"] = True
|
||||
if args.dtype is not None:
|
||||
omni_kwargs["dtype"] = args.dtype
|
||||
|
||||
compile_mode = "eager" if args.enforce_eager else "torch.compile"
|
||||
dtype_label = args.dtype or "from deploy YAML"
|
||||
print(f"Loading SoulX-Singer {mode} from {args.model} [{compile_mode}, dtype={dtype_label}]")
|
||||
|
||||
omni = Omni(**omni_kwargs)
|
||||
kind = "svs" if args.svs else "svc"
|
||||
sampling = build_sampling(args, preprocess_weights_dir=preprocess_dir, kind=kind)
|
||||
prompt = {"prompt_token_ids": [0]}
|
||||
|
||||
latencies_ms: list[float] = []
|
||||
rtfs: list[float] = []
|
||||
last_stages: dict[str, float] = {}
|
||||
last_measured_outputs = None
|
||||
|
||||
total_runs = args.warmup + args.runs
|
||||
for run_idx in range(total_runs):
|
||||
is_warmup = run_idx < args.warmup
|
||||
label = "warmup" if is_warmup else f"run {run_idx - args.warmup + 1}/{args.runs}"
|
||||
t0 = time.perf_counter()
|
||||
outputs = list(omni.generate([prompt], sampling))
|
||||
elapsed_ms = (time.perf_counter() - t0) * 1000.0
|
||||
audio_sec = _audio_duration_sec(outputs)
|
||||
rtf = (elapsed_ms / 1000.0) / audio_sec if audio_sec > 0 else float("inf")
|
||||
print(f"[{label}] client={elapsed_ms:.1f} ms, audio={audio_sec:.2f}s, RTF={rtf:.3f}")
|
||||
if not is_warmup:
|
||||
latencies_ms.append(elapsed_ms)
|
||||
rtfs.append(rtf)
|
||||
last_stages = _stage_durations_from_output(outputs)
|
||||
last_measured_outputs = outputs
|
||||
if args.output and args.save_each_run:
|
||||
measured_idx = run_idx - args.warmup + 1
|
||||
out_path = Path(args.output)
|
||||
run_path = out_path.with_name(f"{out_path.stem}_run{measured_idx}{out_path.suffix}")
|
||||
audio_np, sr = extract_audio(outputs)
|
||||
sf.write(str(run_path), audio_np, sr)
|
||||
print(f" saved {run_path} ({sr} Hz, {audio_np.size / sr:.2f}s)")
|
||||
|
||||
omni.close()
|
||||
|
||||
if args.output and last_measured_outputs is not None:
|
||||
audio_np, sr = extract_audio(last_measured_outputs)
|
||||
out_path = Path(args.output)
|
||||
out_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
sf.write(str(out_path), audio_np, sr)
|
||||
print(f"\nSaved last measured run → {out_path} ({sr} Hz, {audio_np.size / sr:.2f}s)")
|
||||
|
||||
if latencies_ms:
|
||||
print("\n=== Summary (measured runs) ===")
|
||||
print(f"client_ms: mean={statistics.mean(latencies_ms):.1f}, stdev={statistics.pstdev(latencies_ms):.1f}")
|
||||
print(f"RTF: mean={statistics.mean(rtfs):.3f}, stdev={statistics.pstdev(rtfs):.3f}")
|
||||
if last_stages:
|
||||
print("\nStage durations (last measured run):")
|
||||
for name, value in sorted(last_stages.items()):
|
||||
if name.endswith("_ms"):
|
||||
print(f" {name}: {value:.1f} ms")
|
||||
else:
|
||||
print(f" {name}: {value * 1000.0:.1f} ms")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,285 +0,0 @@
|
||||
"""Offline SoulX-Singer SVS / SVC: single-stage DiT (preprocess inline).
|
||||
|
||||
Pass raw prompt/target audio; vLLM-Omni runs integrated preprocess then inference.
|
||||
|
||||
Usage:
|
||||
# SVS (default)
|
||||
python end2end.py --model /path/to/SoulX-Singer \\
|
||||
--prompt-audio path/to/prompt/audio.mp3 \\
|
||||
--target-audio path/to/target/audio.mp3 \\
|
||||
--preprocess-weights-dir /path/to/SoulX-Singer-Preprocess \\
|
||||
-o output.wav
|
||||
|
||||
# SVC (config.json must declare SoulXSingerSVCPipeline; --model points at SVC view)
|
||||
python end2end.py --model /path/to/SoulX-Singer-svc \\
|
||||
--prompt-audio path/to/prompt/audio.mp3 \\
|
||||
--target-audio path/to/target/audio.mp3 \\
|
||||
--preprocess-weights-dir /path/to/SoulX-Singer-Preprocess \\
|
||||
-o output_svc.wav
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import soundfile as sf
|
||||
|
||||
from vllm_omni.diffusion.models.soulx_singer.utils import resolve_soulx_kind, validate_soulx_extra_args
|
||||
from vllm_omni.entrypoints.omni import Omni
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[4]
|
||||
SVS_DEPLOY_CONFIG = REPO_ROOT / "vllm_omni" / "deploy" / "soulxsinger_svs.yaml"
|
||||
SVC_DEPLOY_CONFIG = REPO_ROOT / "vllm_omni" / "deploy" / "soulxsinger_svc.yaml"
|
||||
DEFAULT_PROMPT_AUDIO = REPO_ROOT / "tests" / "assets" / "soulxsinger" / "zh_prompt.mp3"
|
||||
DEFAULT_TARGET_AUDIO = REPO_ROOT / "tests" / "assets" / "soulxsinger" / "music.mp3"
|
||||
_SAMPLE_RATE = 24000
|
||||
|
||||
|
||||
def _require_paths(paths: dict[str, str | None]) -> None:
|
||||
missing = [name for name, p in paths.items() if not p or not os.path.isfile(p)]
|
||||
if missing:
|
||||
raise FileNotFoundError("Missing file(s): " + ", ".join(f"{k}={paths[k]!r}" for k in missing))
|
||||
|
||||
|
||||
def resolve_preprocess_weights_dir(explicit: str | None) -> str:
|
||||
if explicit:
|
||||
path = Path(explicit).expanduser().resolve()
|
||||
if not (path / "rmvpe" / "rmvpe.pt").is_file():
|
||||
raise FileNotFoundError(f"--preprocess-weights-dir must contain rmvpe/rmvpe.pt: {path}")
|
||||
return str(path)
|
||||
env = os.environ.get("SOULX_PREPROCESS_WEIGHTS_DIR")
|
||||
if env:
|
||||
path = Path(env).expanduser().resolve()
|
||||
if (path / "rmvpe" / "rmvpe.pt").is_file():
|
||||
return str(path)
|
||||
raise FileNotFoundError(
|
||||
"SoulX preprocess weights not found. Pass --preprocess-weights-dir or set "
|
||||
"SOULX_PREPROCESS_WEIGHTS_DIR to a directory containing rmvpe/rmvpe.pt "
|
||||
"(huggingface-cli download Soul-AILab/SoulX-Singer-Preprocess)."
|
||||
)
|
||||
|
||||
|
||||
def build_sampling(
|
||||
args: argparse.Namespace,
|
||||
*,
|
||||
preprocess_weights_dir: str,
|
||||
kind: str,
|
||||
) -> OmniDiffusionSamplingParams:
|
||||
extra: dict = {
|
||||
"prompt_audio": os.path.abspath(args.prompt_audio),
|
||||
"target_audio": os.path.abspath(args.target_audio),
|
||||
"preprocess_weights_dir": preprocess_weights_dir,
|
||||
"vocal_sep": args.vocal_sep,
|
||||
"auto_shift": args.auto_shift,
|
||||
"pitch_shift": args.pitch_shift,
|
||||
}
|
||||
if kind == "svs":
|
||||
extra["language"] = args.language
|
||||
extra["control"] = args.control
|
||||
if args.prompt_metadata_path:
|
||||
extra["prompt_metadata_path"] = os.path.abspath(args.prompt_metadata_path)
|
||||
if args.target_metadata_path:
|
||||
extra["target_metadata_path"] = os.path.abspath(args.target_metadata_path)
|
||||
if args.prompt_metadata_path or args.target_metadata_path:
|
||||
extra["audio_path"] = os.path.abspath(args.prompt_wav_path or args.prompt_audio)
|
||||
|
||||
extra = validate_soulx_extra_args(kind, extra)
|
||||
|
||||
return OmniDiffusionSamplingParams(
|
||||
num_inference_steps=args.num_inference_steps,
|
||||
guidance_scale=args.guidance_scale,
|
||||
seed=args.seed,
|
||||
extra_args=extra,
|
||||
)
|
||||
|
||||
|
||||
def extract_audio(outputs) -> tuple[np.ndarray, int]:
|
||||
out = outputs[0]
|
||||
mm = getattr(out, "multimodal_output", None)
|
||||
if mm is None:
|
||||
ro = getattr(out, "request_output", None)
|
||||
if ro is not None:
|
||||
mm = getattr(ro, "multimodal_output", None)
|
||||
if mm is None and ro.outputs:
|
||||
mm = getattr(ro.outputs[0], "multimodal_output", None)
|
||||
if not mm or "audio" not in mm:
|
||||
raise RuntimeError("No audio in multimodal_output")
|
||||
|
||||
audio_val = mm["audio"]
|
||||
if isinstance(audio_val, list):
|
||||
chunks = [
|
||||
np.asarray(chunk.detach().cpu().float().numpy() if hasattr(chunk, "detach") else chunk).reshape(-1)
|
||||
for chunk in audio_val
|
||||
if chunk is not None
|
||||
]
|
||||
audio_np = np.concatenate(chunks, axis=0) if chunks else np.array([], dtype=np.float32)
|
||||
elif hasattr(audio_val, "cpu"):
|
||||
audio_np = audio_val.detach().cpu().numpy().squeeze()
|
||||
else:
|
||||
audio_np = np.asarray(audio_val, dtype=np.float32).squeeze()
|
||||
|
||||
sr_val = mm.get("sr", mm.get("audio_sample_rate", _SAMPLE_RATE))
|
||||
if isinstance(sr_val, list) and sr_val:
|
||||
sr_val = sr_val[-1]
|
||||
if hasattr(sr_val, "item"):
|
||||
sr_val = sr_val.item()
|
||||
return audio_np.astype(np.float32, copy=False), int(sr_val)
|
||||
|
||||
|
||||
def add_inference_args(parser: argparse.ArgumentParser) -> None:
|
||||
mode = parser.add_mutually_exclusive_group()
|
||||
mode.add_argument(
|
||||
"--svc",
|
||||
action="store_true",
|
||||
help="Assert SVC mode (must match config.json architectures)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--model",
|
||||
type=str,
|
||||
required=True,
|
||||
help="SoulX-Singer weight directory ($BASE for SVS, $SVC_DIR for SVC)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--deploy-config",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Override deploy YAML (default: soulxsinger_svs.yaml or soulxsinger_svc.yaml)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--prompt-audio",
|
||||
type=str,
|
||||
default=str(DEFAULT_PROMPT_AUDIO),
|
||||
help="Prompt / reference audio (mp3 or wav)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--target-audio",
|
||||
type=str,
|
||||
default=str(DEFAULT_TARGET_AUDIO),
|
||||
help="Target content audio (mp3 or wav)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--prompt-metadata-path",
|
||||
type=str,
|
||||
default=None,
|
||||
help="SVS precomputed prompt metadata.json (skips online ASR/ROSVOT for prompt)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--target-metadata-path",
|
||||
type=str,
|
||||
default=None,
|
||||
help="SVS precomputed target metadata.json (skips online ASR/ROSVOT for target)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--prompt-wav-path",
|
||||
type=str,
|
||||
default=None,
|
||||
help="SVS prompt vocal wav for metadata processor when using --prompt-metadata-path (default: --prompt-audio)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--preprocess-weights-dir",
|
||||
type=str,
|
||||
default=None,
|
||||
help="SoulX-Singer-Preprocess weights (or set SOULX_PREPROCESS_WEIGHTS_DIR)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--language",
|
||||
type=str,
|
||||
default="Mandarin",
|
||||
help="SVS lyrics language (Mandarin, Cantonese, English)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--control",
|
||||
type=str,
|
||||
default="melody",
|
||||
choices=["score", "melody"],
|
||||
help="SVS control mode",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--vocal-sep",
|
||||
action=argparse.BooleanOptionalAction,
|
||||
default=False,
|
||||
help="Run vocal separation during preprocess",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--auto-shift",
|
||||
action=argparse.BooleanOptionalAction,
|
||||
default=True,
|
||||
help="Auto pitch shift to align target pitch with prompt",
|
||||
)
|
||||
parser.add_argument("--pitch-shift", type=int, default=0)
|
||||
parser.add_argument("--num-inference-steps", type=int, default=32)
|
||||
parser.add_argument("--guidance-scale", type=float, default=3.0)
|
||||
parser.add_argument(
|
||||
"--seed",
|
||||
type=int,
|
||||
default=42,
|
||||
help="Optional CFM RNG seed. Omit for non-deterministic sampling.",
|
||||
)
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description="SoulX-Singer offline SVS / SVC (preprocess + DiT)")
|
||||
add_inference_args(parser)
|
||||
parser.add_argument("--output", "-o", type=str, default="output.wav")
|
||||
args = parser.parse_args()
|
||||
return args
|
||||
|
||||
|
||||
def main() -> None:
|
||||
args = parse_args()
|
||||
kind = resolve_soulx_kind(args.model)
|
||||
|
||||
if not (args.svc ^ (kind == "svs")):
|
||||
raise ValueError(f"mode mismatch: {args.model}/config.json (declares {kind!r}), --svc={args.svc}")
|
||||
|
||||
required = {
|
||||
"prompt_audio": args.prompt_audio,
|
||||
"target_audio": args.target_audio,
|
||||
}
|
||||
if kind == "svs" and bool(args.prompt_metadata_path) ^ bool(args.target_metadata_path):
|
||||
raise ValueError(
|
||||
"SVS precomputed metadata requires both --prompt-metadata-path and "
|
||||
"--target-metadata-path (or neither, for online preprocess)."
|
||||
)
|
||||
if args.prompt_metadata_path:
|
||||
required["prompt_metadata_path"] = args.prompt_metadata_path
|
||||
required["target_metadata_path"] = args.target_metadata_path
|
||||
if args.prompt_wav_path:
|
||||
required["prompt_wav_path"] = args.prompt_wav_path
|
||||
_require_paths(required)
|
||||
preprocess_dir = resolve_preprocess_weights_dir(args.preprocess_weights_dir)
|
||||
deploy_config = args.deploy_config or str(SVC_DEPLOY_CONFIG if kind == "svc" else SVS_DEPLOY_CONFIG)
|
||||
mode = kind.upper()
|
||||
|
||||
print(f"Loading SoulX-Singer {mode} from {args.model}")
|
||||
print(f" deploy: {deploy_config}")
|
||||
print(f" prompt: {args.prompt_audio}")
|
||||
print(f" target: {args.target_audio}")
|
||||
|
||||
omni = Omni(model=args.model, deploy_config=deploy_config, async_chunk=False) # SoulX-Singer is batch-only
|
||||
sampling = build_sampling(
|
||||
args,
|
||||
preprocess_weights_dir=preprocess_dir,
|
||||
kind=kind,
|
||||
)
|
||||
prompt = {"prompt_token_ids": [0], "multi_modal_data": {}}
|
||||
|
||||
print("Running generate (inline preprocess + DiT)...")
|
||||
outputs = list(omni.generate([prompt], sampling))
|
||||
if not outputs:
|
||||
raise RuntimeError("No output from omni.generate")
|
||||
|
||||
err = getattr(outputs[0], "error", None)
|
||||
if err is not None:
|
||||
raise RuntimeError(f"Pipeline error: {err}")
|
||||
|
||||
audio_np, sr = extract_audio(outputs)
|
||||
sf.write(args.output, audio_np, sr)
|
||||
print(f"Saved {args.output} ({sr} Hz, {len(audio_np) / sr:.2f}s)")
|
||||
omni.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Loaded 100 of 834 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user