Compare commits
13
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Hongsheng Liu
|
454b8a60be | ||
|
herotai214
|
c96f0bc773 | ||
|
Ting FU
|
f4640d9a2e | ||
|
Mu GuanLin
|
12d75cf63a | ||
|
Zhou TaichangandClaude Opus 5
|
66a9b0c84f | ||
|
wangyu
|
cee2e40c33 | ||
|
|
5014fea4a0 | ||
|
Weiming Liaoandwangyu
|
f2c5152df0 | ||
|
WeiandSamit
|
d3e0171cd7 | ||
|
Joshna-Medisetty
|
807db6efd7 | ||
|
Yueqian LinandSy03
|
d4a5eaac90 | ||
|
Yueqian Lin
|
8f115164c2 | ||
|
|
cbab1d4d8f |
No files matched your search
@@ -6,10 +6,6 @@
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# minicpm-challenge branch: skip entire AMD CI (no pipeline upload => no image build / tests).
|
||||
echo "--- Skipping AMD CI on this branch (including image build)"
|
||||
exit 0
|
||||
|
||||
source .buildkite/common/scripts/resolve_skip_ci.sh
|
||||
|
||||
if [[ -z "${RUN_ALL:-}" ]]; then
|
||||
|
||||
@@ -282,7 +282,7 @@ def _git(*args: str) -> subprocess.CompletedProcess[str]:
|
||||
|
||||
|
||||
def _resolve_diff_range() -> str | None:
|
||||
"""Return a git diff range for PR or push-branch builds, or None when unavailable."""
|
||||
"""Return a git diff range for PR or main builds, or None when unavailable."""
|
||||
is_pr = os.environ.get("BUILDKITE_PULL_REQUEST", "false") != "false" and os.environ.get(
|
||||
"BUILDKITE_PULL_REQUEST", ""
|
||||
)
|
||||
@@ -302,15 +302,13 @@ def _resolve_diff_range() -> str | None:
|
||||
return None
|
||||
return f"{base_ref}...{commit}"
|
||||
|
||||
# main: ready/nightly/weekly push CI; minicpm-challenge: merge (L3) push CI
|
||||
branch = os.environ.get("BUILDKITE_BRANCH", "")
|
||||
if branch in ("main", "minicpm-challenge"):
|
||||
if os.environ.get("BUILDKITE_BRANCH", "") == "main":
|
||||
if _git("rev-parse", "--verify", f"{commit}^").returncode != 0:
|
||||
_log(f"{branch} commit has no parent; using safe defaults")
|
||||
_log("main commit has no parent; using safe defaults")
|
||||
return None
|
||||
return f"{commit}^..{commit}"
|
||||
|
||||
_log("not PR/main/minicpm-challenge build; using safe defaults")
|
||||
_log("not PR/main build; using safe defaults")
|
||||
return None
|
||||
|
||||
|
||||
|
||||
@@ -59,12 +59,11 @@ E2E_GROUP_MARKER = "E2E Test"
|
||||
CI_MIRROR_HARDWARES_PATH = ROOT / ".buildkite/common/ci_mirror_hardwares.yml"
|
||||
|
||||
CUDA_NIGHTLY_ONLY = (
|
||||
'(build.pull_request.labels includes "nightly-test") || '
|
||||
'(build.branch == "minicpm-challenge" && build.env("NIGHTLY") == "1")'
|
||||
'(build.pull_request.labels includes "nightly-test") || (build.branch == "main" && build.env("NIGHTLY") == "1")'
|
||||
)
|
||||
NPU_NIGHTLY_ONLY = (
|
||||
'(build.branch == "minicpm-challenge" && build.env("NIGHTLY") == "1") || '
|
||||
'(build.branch != "minicpm-challenge" && ('
|
||||
'(build.branch == "main" && build.env("NIGHTLY") == "1") || '
|
||||
'(build.branch != "main" && ('
|
||||
'build.pull_request.labels includes "nightly-test" || '
|
||||
'build.pull_request.labels includes "omni-test" || '
|
||||
'build.pull_request.labels includes "tts-test" || '
|
||||
@@ -104,8 +103,7 @@ def _format_bootstrap_if(expr: str) -> str:
|
||||
|
||||
def _compute_bootstrap_if_exprs(*, decision, platform: str) -> dict[str, str]:
|
||||
disabled = "false"
|
||||
# L4 nightly scheduled push CI runs on minicpm-challenge (not main).
|
||||
nightly_main = 'build.branch == "minicpm-challenge" && build.env("NIGHTLY") == "1"'
|
||||
nightly_main = 'build.branch == "main" && build.env("NIGHTLY") == "1"'
|
||||
nightly_only = NPU_NIGHTLY_ONLY if platform == "npu" else CUDA_NIGHTLY_ONLY
|
||||
|
||||
if platform == "npu":
|
||||
@@ -120,12 +118,11 @@ def _compute_bootstrap_if_exprs(*, decision, platform: str) -> dict[str, str]:
|
||||
merge_base = disabled
|
||||
else:
|
||||
ready_pr = 'build.branch != "main" && build.pull_request.labels includes "ready"'
|
||||
# L3 merge push CI runs on minicpm-challenge (not main).
|
||||
merge_main = 'build.branch == "minicpm-challenge" && build.env("NIGHTLY") != "1" && build.env("WEEKLY") != "1"'
|
||||
merge_pr = 'build.branch != "minicpm-challenge" && build.pull_request.labels includes "merge-test"'
|
||||
merge_main = 'build.branch == "main" && build.env("NIGHTLY") != "1" && build.env("WEEKLY") != "1"'
|
||||
merge_pr = 'build.branch != "main" && build.pull_request.labels includes "merge-test"'
|
||||
nightly_label_if = (
|
||||
'(build.branch == "minicpm-challenge" && build.env("NIGHTLY") == "1") || '
|
||||
'(build.branch != "minicpm-challenge" && ('
|
||||
'(build.branch == "main" && build.env("NIGHTLY") == "1") || '
|
||||
'(build.branch != "main" && ('
|
||||
'build.pull_request.labels includes "nightly-test" || '
|
||||
'build.pull_request.labels includes "omni-test" || '
|
||||
'build.pull_request.labels includes "tts-test" || '
|
||||
@@ -143,8 +140,8 @@ def _compute_bootstrap_if_exprs(*, decision, platform: str) -> dict[str, str]:
|
||||
weekly_main = 'build.branch == "main" && build.env("WEEKLY") == "1"'
|
||||
|
||||
if decision.skip_all:
|
||||
# Docs / skip-mark only: no PR-label escape hatch. Scheduled
|
||||
# NIGHTLY=1 (minicpm-challenge) still runs L4 + L2/L3 (--e2e); WEEKLY=1 (main) still runs L5.
|
||||
# Docs / skip-mark only: no PR-label escape hatch. Main scheduled
|
||||
# NIGHTLY=1 still runs L4 + L2/L3 (--e2e); WEEKLY=1 still runs L5.
|
||||
image_expr = f"({nightly_main}) || ({weekly_main})" if platform == "cuda" else nightly_main
|
||||
ready_expr = nightly_main
|
||||
merge_expr = nightly_main if platform == "cuda" else disabled
|
||||
|
||||
@@ -8,35 +8,35 @@ steps:
|
||||
agents:
|
||||
queue: "cpu_queue_premerge"
|
||||
|
||||
# L2 Test — PR ready label (diff-aware) or minicpm-challenge+NIGHTLY=1 (--e2e, full E2E group)
|
||||
# L2 Test — PR ready label (diff-aware) or main+NIGHTLY=1 (--e2e, full E2E group)
|
||||
- label: "Upload Ready Pipeline"
|
||||
depends_on: image-build
|
||||
key: upload-ready-pipeline
|
||||
commands:
|
||||
- |
|
||||
EXTRA=""
|
||||
if [ "${NIGHTLY:-}" = "1" ] && [ "${BUILDKITE_BRANCH}" = "minicpm-challenge" ]; then
|
||||
if [ "${NIGHTLY:-}" = "1" ] && [ "${BUILDKITE_BRANCH}" = "main" ]; then
|
||||
EXTRA="--e2e"
|
||||
fi
|
||||
python3 .buildkite/common/scripts/upload_pipeline.py --upload $$EXTRA .buildkite/cuda/test-ready.yml
|
||||
agents:
|
||||
queue: "cpu_queue_premerge"
|
||||
|
||||
# L3 Test — PR merge-test label (diff-aware) or minicpm-challenge+NIGHTLY=1 (--e2e, full E2E group)
|
||||
# L3 Test — PR merge-test label (diff-aware) or main+NIGHTLY=1 (--e2e, full E2E group)
|
||||
- label: "Upload Merge Pipeline"
|
||||
depends_on: image-build
|
||||
key: upload-merge-pipeline
|
||||
commands:
|
||||
- |
|
||||
EXTRA=""
|
||||
if [ "${NIGHTLY:-}" = "1" ] && [ "${BUILDKITE_BRANCH}" = "minicpm-challenge" ]; then
|
||||
if [ "${NIGHTLY:-}" = "1" ] && [ "${BUILDKITE_BRANCH}" = "main" ]; then
|
||||
EXTRA="--e2e"
|
||||
fi
|
||||
python3 .buildkite/common/scripts/upload_pipeline.py --upload $$EXTRA .buildkite/cuda/test-merge.yml
|
||||
agents:
|
||||
queue: "cpu_queue_premerge"
|
||||
|
||||
# L4 Test — minicpm-challenge+NIGHTLY=1 (scheduled), or PR with specific label (e.g. add label then Rebuild)
|
||||
# L4 Test — main+NIGHTLY=1 (scheduled), or PR with specific label (e.g. add label then Rebuild)
|
||||
- label: "Upload Nightly Pipeline"
|
||||
depends_on: image-build
|
||||
key: upload-nightly-pipeline
|
||||
|
||||
+238
-204
@@ -18,227 +18,261 @@ steps:
|
||||
- "timeout 20m pytest -sv -m 'core_model and cpu' --ignore=tests/diffusion --ignore=tests/model_executor --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"
|
||||
mirror_hardwares: l4_1
|
||||
|
||||
# - group: ":card_index_dividers: Diffusion Test"
|
||||
# depends_on: upload-merge-pipeline
|
||||
# steps:
|
||||
# - label: "Diffusion · LoRA Test"
|
||||
# timeout_in_minutes: 30
|
||||
# commands:
|
||||
# - pytest -sv tests/diffusion/lora/ -m 'advanced_model and cuda' --run-level "advanced_model"
|
||||
# mirror_hardwares: l4_1
|
||||
- group: ":card_index_dividers: Diffusion Test"
|
||||
depends_on: upload-merge-pipeline
|
||||
steps:
|
||||
- label: "Diffusion · LoRA Test"
|
||||
timeout_in_minutes: 30
|
||||
commands:
|
||||
- pytest -sv tests/diffusion/lora/ -m 'advanced_model and cuda' --run-level "advanced_model"
|
||||
mirror_hardwares: l4_1
|
||||
|
||||
# - label: "Diffusion · Offloader Test"
|
||||
# timeout_in_minutes: 40
|
||||
# commands:
|
||||
# - timeout 40m pytest -s -v tests/diffusion/offloader -m "advanced_model and cuda"
|
||||
# mirror_hardwares: l4_1
|
||||
- label: "Diffusion · Offloader Test"
|
||||
timeout_in_minutes: 40
|
||||
commands:
|
||||
- timeout 40m pytest -s -v tests/diffusion/offloader -m "advanced_model and cuda"
|
||||
mirror_hardwares: l4_1
|
||||
|
||||
# - label: "Distributed Test"
|
||||
# depends_on: upload-merge-pipeline
|
||||
# timeout_in_minutes: 60
|
||||
# commands:
|
||||
# - |
|
||||
# timeout 55m bash -c "
|
||||
# set -e
|
||||
# export VLLM_IMAGE_FETCH_TIMEOUT=60
|
||||
# pytest -s -v tests/distributed -m 'advanced_model and cuda and H100' --run-level 'advanced_model'
|
||||
# "
|
||||
# mirror_hardwares: h100_1
|
||||
- label: "Distributed Test"
|
||||
depends_on: upload-merge-pipeline
|
||||
timeout_in_minutes: 60
|
||||
commands:
|
||||
- |
|
||||
timeout 55m bash -c "
|
||||
set -e
|
||||
export VLLM_IMAGE_FETCH_TIMEOUT=60
|
||||
pytest -s -v tests/distributed -m 'advanced_model and cuda and H100' --run-level 'advanced_model'
|
||||
"
|
||||
mirror_hardwares: h100_1
|
||||
|
||||
# - group: ":card_index_dividers: Entrypoints Test"
|
||||
# depends_on: upload-merge-pipeline
|
||||
# steps:
|
||||
# - label: "Entrypoint Test with H100"
|
||||
# timeout_in_minutes: 60
|
||||
# commands:
|
||||
# - pytest -s -v tests/entrypoints/ -m "advanced_model and cuda and H100" --run-level "advanced_model"
|
||||
# mirror_hardwares: h100_2
|
||||
- group: ":card_index_dividers: Entrypoints Test"
|
||||
depends_on: upload-merge-pipeline
|
||||
steps:
|
||||
- label: "Entrypoint Test with H100"
|
||||
# TEMPORARY 60 -> 90: engine startup/sleep-wake is ~13-15 min slower per
|
||||
# run on the vLLM 0.26 image (model loading ~2x, init ~2x; see builds
|
||||
# 2915 vs 2916), which pushed this suite past 60 min. Revert once the
|
||||
# 0.26 startup regression is understood and fixed.
|
||||
timeout_in_minutes: 90
|
||||
commands:
|
||||
- pytest -s -v tests/entrypoints/ -m "advanced_model and cuda and H100" --run-level "advanced_model"
|
||||
mirror_hardwares: h100_2
|
||||
|
||||
# - label: "Entrypoint Test with L4"
|
||||
# timeout_in_minutes: 30
|
||||
# commands:
|
||||
# - pytest -s -v tests/entrypoints/ -m "advanced_model and cuda and L4" --run-level "advanced_model"
|
||||
# mirror_hardwares: l4_1
|
||||
- label: "Entrypoint Test with L4"
|
||||
timeout_in_minutes: 60
|
||||
commands:
|
||||
- pytest -s -v tests/entrypoints/ -m "advanced_model and cuda and L4" --run-level "advanced_model"
|
||||
mirror_hardwares: l4_1
|
||||
|
||||
# - group: ":card_index_dividers: E2E Test"
|
||||
# depends_on: upload-merge-pipeline
|
||||
# steps:
|
||||
# - label: "TTS · Qwen3-TTS CustomVoice Test"
|
||||
# source_file_dependencies:
|
||||
# - tests/e2e/online_serving/test_qwen3_tts_customvoice.py
|
||||
# - tests/e2e/offline_inference/test_qwen3_tts_customvoice.py
|
||||
# - vllm_omni/model_executor/models/qwen3_tts/
|
||||
# - vllm_omni/model_executor/stage_input_processors/qwen3_tts.py
|
||||
# - vllm_omni/deploy/qwen3_tts.yaml
|
||||
# commands:
|
||||
# - |
|
||||
# timeout 40m bash -c "
|
||||
# export VLLM_LOGGING_LEVEL=DEBUG
|
||||
# export VLLM_ALLOW_LONG_MAX_MODEL_LEN=1
|
||||
# pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py tests/e2e/offline_inference/test_qwen3_tts_customvoice.py -m 'advanced_model and cuda' --run-level 'advanced_model'
|
||||
# "
|
||||
# mirror_hardwares: l4_1
|
||||
- group: ":card_index_dividers: E2E Test"
|
||||
depends_on: upload-merge-pipeline
|
||||
steps:
|
||||
- label: "TTS · Qwen3-TTS CustomVoice Test"
|
||||
source_file_dependencies:
|
||||
- tests/e2e/online_serving/test_qwen3_tts_customvoice.py
|
||||
- tests/e2e/offline_inference/test_qwen3_tts_customvoice.py
|
||||
- vllm_omni/model_executor/models/qwen3_tts/
|
||||
- vllm_omni/model_executor/stage_input_processors/qwen3_tts.py
|
||||
- vllm_omni/deploy/qwen3_tts.yaml
|
||||
commands:
|
||||
- |
|
||||
timeout 40m bash -c "
|
||||
export VLLM_LOGGING_LEVEL=DEBUG
|
||||
export VLLM_ALLOW_LONG_MAX_MODEL_LEN=1
|
||||
pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py tests/e2e/offline_inference/test_qwen3_tts_customvoice.py -m 'advanced_model and cuda' --run-level 'advanced_model'
|
||||
"
|
||||
mirror_hardwares: l4_1
|
||||
|
||||
# - label: "TTS · Qwen3-TTS Base Test"
|
||||
# source_file_dependencies:
|
||||
# - tests/e2e/online_serving/test_qwen3_tts_base.py
|
||||
# - tests/e2e/offline_inference/test_qwen3_tts_base.py
|
||||
# - vllm_omni/model_executor/models/qwen3_tts/
|
||||
# - vllm_omni/model_executor/stage_input_processors/qwen3_tts.py
|
||||
# - vllm_omni/deploy/qwen3_tts.yaml
|
||||
# commands:
|
||||
# - |
|
||||
# timeout 40m bash -c "
|
||||
# export VLLM_LOGGING_LEVEL=DEBUG
|
||||
# export VLLM_ALLOW_LONG_MAX_MODEL_LEN=1
|
||||
# pytest -s -v tests/e2e/online_serving/test_qwen3_tts_base.py tests/e2e/offline_inference/test_qwen3_tts_base.py -m 'advanced_model and cuda' --run-level 'advanced_model'
|
||||
# "
|
||||
# mirror_hardwares: l4_1
|
||||
- label: "TTS · Qwen3-TTS Base Test"
|
||||
source_file_dependencies:
|
||||
- tests/e2e/online_serving/test_qwen3_tts_base.py
|
||||
- tests/e2e/offline_inference/test_qwen3_tts_base.py
|
||||
- vllm_omni/model_executor/models/qwen3_tts/
|
||||
- vllm_omni/model_executor/stage_input_processors/qwen3_tts.py
|
||||
- vllm_omni/deploy/qwen3_tts.yaml
|
||||
commands:
|
||||
- |
|
||||
timeout 40m bash -c "
|
||||
export VLLM_LOGGING_LEVEL=DEBUG
|
||||
export VLLM_ALLOW_LONG_MAX_MODEL_LEN=1
|
||||
pytest -s -v tests/e2e/online_serving/test_qwen3_tts_base.py tests/e2e/offline_inference/test_qwen3_tts_base.py -m 'advanced_model and cuda' --run-level 'advanced_model'
|
||||
"
|
||||
mirror_hardwares: l4_1
|
||||
|
||||
# - label: "TTS · Higgs-Audio-V3 Test"
|
||||
# source_file_dependencies:
|
||||
# - tests/e2e/online_serving/test_higgs_audio_v3.py
|
||||
# - vllm_omni/model_executor/models/higgs_audio_v3/
|
||||
# - vllm_omni/model_executor/stage_input_processors/higgs_audio_v3.py
|
||||
# - vllm_omni/deploy/higgs_multimodal_qwen3.yaml
|
||||
# commands:
|
||||
# - |
|
||||
# timeout 20m bash -c "
|
||||
# export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
# export VLLM_USE_DEEP_GEMM=0
|
||||
# export VLLM_MOE_USE_DEEP_GEMM=0
|
||||
# pytest -s -v tests/e2e/online_serving/test_higgs_audio_v3.py -m 'advanced_model' --run-level 'advanced_model'
|
||||
# "
|
||||
# mirror_hardwares: l4_1
|
||||
- label: "TTS · Higgs-Audio-V3 Test"
|
||||
source_file_dependencies:
|
||||
- tests/e2e/online_serving/test_higgs_audio_v3.py
|
||||
- vllm_omni/model_executor/models/higgs_audio_v3/
|
||||
- vllm_omni/model_executor/stage_input_processors/higgs_audio_v3.py
|
||||
- vllm_omni/deploy/higgs_multimodal_qwen3.yaml
|
||||
commands:
|
||||
- |
|
||||
timeout 20m bash -c "
|
||||
export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
export VLLM_USE_DEEP_GEMM=0
|
||||
export VLLM_MOE_USE_DEEP_GEMM=0
|
||||
pytest -s -v tests/e2e/online_serving/test_higgs_audio_v3.py -m 'advanced_model' --run-level 'advanced_model'
|
||||
"
|
||||
mirror_hardwares: l4_1
|
||||
|
||||
# - label: "Omni · Qwen3-Omni Test"
|
||||
# source_file_dependencies:
|
||||
# - tests/e2e/offline_inference/test_qwen3_omni.py
|
||||
# - tests/e2e/online_serving/test_qwen3_omni.py
|
||||
# - vllm_omni/model_executor/models/qwen3_omni/
|
||||
# - vllm_omni/model_executor/stage_input_processors/qwen3_omni.py
|
||||
# - vllm_omni/deploy/qwen3_omni_moe.yaml
|
||||
# timeout_in_minutes: 50
|
||||
# commands:
|
||||
# - pytest -s -v tests/e2e/offline_inference/test_qwen3_omni.py tests/e2e/online_serving/test_qwen3_omni.py -m 'advanced_model and cuda' --run-level 'advanced_model'
|
||||
# mirror_hardwares: h100_2
|
||||
- label: "Omni · Qwen3-Omni Test"
|
||||
source_file_dependencies:
|
||||
- tests/e2e/offline_inference/test_qwen3_omni.py
|
||||
- tests/e2e/online_serving/test_qwen3_omni.py
|
||||
- vllm_omni/model_executor/models/qwen3_omni/
|
||||
- vllm_omni/model_executor/stage_input_processors/qwen3_omni.py
|
||||
- vllm_omni/deploy/qwen3_omni_moe.yaml
|
||||
timeout_in_minutes: 50
|
||||
commands:
|
||||
- pytest -s -v tests/e2e/offline_inference/test_qwen3_omni.py tests/e2e/online_serving/test_qwen3_omni.py -m 'advanced_model and cuda' --run-level 'advanced_model'
|
||||
mirror_hardwares: h100_2
|
||||
|
||||
- label: "Omni · MiniCPM-o 4.5 Test"
|
||||
source_file_dependencies:
|
||||
- tests/e2e/offline_inference/test_minicpmo_4_5.py
|
||||
- tests/e2e/online_serving/test_minicpmo_4_5.py
|
||||
- vllm_omni/deploy/minicpmo_4_5.yaml
|
||||
timeout_in_minutes: 50
|
||||
commands:
|
||||
- pytest -s -v tests/e2e/offline_inference/test_minicpmo_4_5.py tests/e2e/online_serving/test_minicpmo_4_5.py -m 'advanced_model and cuda' --run-level 'advanced_model'
|
||||
mirror_hardwares: h100_1
|
||||
agents:
|
||||
queue: "mithril-h100-pool"
|
||||
plugins:
|
||||
- kubernetes:
|
||||
podSpec:
|
||||
containers:
|
||||
- image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
|
||||
resources:
|
||||
limits:
|
||||
nvidia.com/gpu: 2
|
||||
volumeMounts:
|
||||
- name: devshm
|
||||
mountPath: /dev/shm
|
||||
- name: hf-cache
|
||||
mountPath: /root/.cache/huggingface
|
||||
env:
|
||||
- name: HF_HOME
|
||||
value: /root/.cache/huggingface
|
||||
- name: HF_TOKEN
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: hf-token-secret
|
||||
key: token
|
||||
nodeSelector:
|
||||
node.kubernetes.io/instance-type: gpu-h100-sxm
|
||||
volumes:
|
||||
- name: devshm
|
||||
emptyDir:
|
||||
medium: Memory
|
||||
- name: hf-cache
|
||||
hostPath:
|
||||
path: /mnt/hf-cache
|
||||
type: DirectoryOrCreate
|
||||
|
||||
- label: "Omni · MiniCPM-o 4.5 Duplex Test"
|
||||
timeout_in_minutes: 50
|
||||
- label: "TTS · VoxCPM2 Test"
|
||||
source_file_dependencies:
|
||||
- tests/e2e/offline_inference/test_voxcpm2_tts.py
|
||||
- tests/e2e/online_serving/test_voxcpm2_tts.py
|
||||
- vllm_omni/model_executor/models/voxcpm2/
|
||||
- vllm_omni/deploy/voxcpm2.yaml
|
||||
timeout_in_minutes: 30
|
||||
commands:
|
||||
- pytest -s -v tests/e2e/online_serving/test_minicpmo_4_5_duplex.py -m 'advanced_model and cuda' --run-level 'advanced_model'
|
||||
- |
|
||||
timeout 30m bash -c "
|
||||
export VLLM_LOGGING_LEVEL=DEBUG
|
||||
export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
pytest -s -v tests/e2e/offline_inference/test_voxcpm2_tts.py tests/e2e/online_serving/test_voxcpm2_tts.py -m 'advanced_model and cuda' --run-level 'advanced_model'
|
||||
"
|
||||
mirror_hardwares: l4_1
|
||||
|
||||
- label: "TTS · SoulX-Singer Test"
|
||||
source_file_dependencies:
|
||||
- tests/e2e/offline_inference/test_soulxsinger.py
|
||||
- vllm_omni/diffusion/models/soulx_singer/
|
||||
- vllm_omni/deploy/soulxsinger_svc.yaml
|
||||
- vllm_omni/deploy/soulxsinger_svs.yaml
|
||||
timeout_in_minutes: 30
|
||||
commands:
|
||||
- |
|
||||
timeout 30m bash -c "
|
||||
export VLLM_LOGGING_LEVEL=DEBUG
|
||||
export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
pytest -s -v tests/e2e/offline_inference/test_soulxsinger.py -k 'svc or precomputed' -m 'advanced_model and cuda' --run-level 'advanced_model'
|
||||
"
|
||||
mirror_hardwares: l4_1
|
||||
|
||||
- label: "Diffusion · Qwen Image Edit Test"
|
||||
source_file_dependencies:
|
||||
- tests/e2e/online_serving/test_qwen_image_edit.py
|
||||
- vllm_omni/diffusion/models/qwen_image/
|
||||
timeout_in_minutes: 20
|
||||
commands:
|
||||
- pytest -s -v tests/e2e/online_serving/test_qwen_image_edit.py -m 'advanced_model and cuda' --run-level 'advanced_model'
|
||||
mirror_hardwares: h100_1
|
||||
- label: "Diffusion · Bagel Test"
|
||||
source_file_dependencies:
|
||||
- tests/e2e/offline_inference/test_bagel.py
|
||||
- tests/e2e/online_serving/test_bagel.py
|
||||
- vllm_omni/model_executor/models/bagel/
|
||||
- vllm_omni/diffusion/models/bagel/
|
||||
- vllm_omni/model_executor/stage_input_processors/bagel.py
|
||||
- vllm_omni/deploy/bagel.yaml
|
||||
timeout_in_minutes: 20
|
||||
commands:
|
||||
- pytest -s -v tests/e2e/offline_inference/test_bagel.py tests/e2e/online_serving/test_bagel.py -m 'advanced_model and cuda' --run-level 'advanced_model'
|
||||
mirror_hardwares: h100_1
|
||||
|
||||
# - label: "TTS · VoxCPM2 Test"
|
||||
# source_file_dependencies:
|
||||
# - tests/e2e/offline_inference/test_voxcpm2_tts.py
|
||||
# - tests/e2e/online_serving/test_voxcpm2_tts.py
|
||||
# - vllm_omni/model_executor/models/voxcpm2/
|
||||
# - vllm_omni/deploy/voxcpm2.yaml
|
||||
# timeout_in_minutes: 30
|
||||
# commands:
|
||||
# - |
|
||||
# timeout 30m bash -c "
|
||||
# export VLLM_LOGGING_LEVEL=DEBUG
|
||||
# export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
# pytest -s -v tests/e2e/offline_inference/test_voxcpm2_tts.py tests/e2e/online_serving/test_voxcpm2_tts.py -m 'advanced_model and cuda' --run-level 'advanced_model'
|
||||
# "
|
||||
# mirror_hardwares: l4_1
|
||||
- label: "Diffusion · Qwen Image Test"
|
||||
source_file_dependencies:
|
||||
- tests/e2e/online_serving/test_qwen_image.py
|
||||
- vllm_omni/diffusion/models/qwen_image/
|
||||
commands:
|
||||
- |
|
||||
timeout 40m bash -c '
|
||||
export VLLM_IMAGE_FETCH_TIMEOUT=60
|
||||
pytest -s -v tests/e2e/online_serving/test_qwen_image.py -m 'advanced_model and cuda' --run-level 'advanced_model'
|
||||
'
|
||||
mirror_hardwares: h100_1
|
||||
|
||||
# - label: "TTS · SoulX-Singer Test"
|
||||
# source_file_dependencies:
|
||||
# - tests/e2e/offline_inference/test_soulxsinger.py
|
||||
# - vllm_omni/diffusion/models/soulx_singer/
|
||||
# - vllm_omni/deploy/soulxsinger_svc.yaml
|
||||
# - vllm_omni/deploy/soulxsinger_svs.yaml
|
||||
# timeout_in_minutes: 30
|
||||
# commands:
|
||||
# - |
|
||||
# timeout 30m bash -c "
|
||||
# export VLLM_LOGGING_LEVEL=DEBUG
|
||||
# export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
# pytest -s -v tests/e2e/offline_inference/test_soulxsinger.py -k 'svc or precomputed' -m 'advanced_model and cuda' --run-level 'advanced_model'
|
||||
# "
|
||||
# mirror_hardwares: l4_1
|
||||
- label: "Diffusion · Qwen Image Layered Test"
|
||||
source_file_dependencies:
|
||||
- tests/e2e/online_serving/test_qwen_image_layered.py
|
||||
- vllm_omni/diffusion/models/qwen_image/
|
||||
commands:
|
||||
- |
|
||||
timeout 40m bash -c "
|
||||
export VLLM_IMAGE_FETCH_TIMEOUT=60
|
||||
pytest -s -v tests/e2e/online_serving/test_qwen_image_layered.py -m 'advanced_model and cuda' --run-level 'advanced_model'
|
||||
"
|
||||
mirror_hardwares: h100_1
|
||||
|
||||
# - label: "Diffusion · Qwen Image Edit Test"
|
||||
# source_file_dependencies:
|
||||
# - tests/e2e/online_serving/test_qwen_image_edit.py
|
||||
# - vllm_omni/diffusion/models/qwen_image/
|
||||
# timeout_in_minutes: 20
|
||||
# commands:
|
||||
# - pytest -s -v tests/e2e/online_serving/test_qwen_image_edit.py -m 'advanced_model and cuda' --run-level 'advanced_model'
|
||||
# mirror_hardwares: h100_1
|
||||
# - label: "Diffusion · Bagel Test"
|
||||
# source_file_dependencies:
|
||||
# - tests/e2e/offline_inference/test_bagel.py
|
||||
# - tests/e2e/online_serving/test_bagel.py
|
||||
# - vllm_omni/model_executor/models/bagel/
|
||||
# - vllm_omni/diffusion/models/bagel/
|
||||
# - vllm_omni/model_executor/stage_input_processors/bagel.py
|
||||
# - vllm_omni/deploy/bagel.yaml
|
||||
# timeout_in_minutes: 20
|
||||
# commands:
|
||||
# - pytest -s -v tests/e2e/offline_inference/test_bagel.py tests/e2e/online_serving/test_bagel.py -m 'advanced_model and cuda' --run-level 'advanced_model'
|
||||
# mirror_hardwares: h100_1
|
||||
- label: "Diffusion · Wan22 Test"
|
||||
source_file_dependencies:
|
||||
- tests/e2e/offline_inference/test_wan22_t2v.py
|
||||
- tests/e2e/online_serving/test_wan22_t2v.py
|
||||
- vllm_omni/diffusion/models/wan2_2/
|
||||
commands:
|
||||
- |
|
||||
timeout 40m bash -c "
|
||||
export VLLM_IMAGE_FETCH_TIMEOUT=60
|
||||
pytest -s -v tests/e2e/offline_inference/test_wan22_t2v.py tests/e2e/online_serving/test_wan22_t2v.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model'
|
||||
"
|
||||
mirror_hardwares: h100_1
|
||||
|
||||
# - label: "Diffusion · Qwen Image Test"
|
||||
# source_file_dependencies:
|
||||
# - tests/e2e/online_serving/test_qwen_image.py
|
||||
# - vllm_omni/diffusion/models/qwen_image/
|
||||
# commands:
|
||||
# - |
|
||||
# timeout 40m bash -c '
|
||||
# export VLLM_IMAGE_FETCH_TIMEOUT=60
|
||||
# pytest -s -v tests/e2e/online_serving/test_qwen_image.py -m 'advanced_model and cuda' --run-level 'advanced_model'
|
||||
# '
|
||||
# mirror_hardwares: h100_1
|
||||
|
||||
# - label: "Diffusion · Qwen Image Layered Test"
|
||||
# source_file_dependencies:
|
||||
# - tests/e2e/online_serving/test_qwen_image_layered.py
|
||||
# - vllm_omni/diffusion/models/qwen_image/
|
||||
# commands:
|
||||
# - |
|
||||
# timeout 40m bash -c "
|
||||
# export VLLM_IMAGE_FETCH_TIMEOUT=60
|
||||
# pytest -s -v tests/e2e/online_serving/test_qwen_image_layered.py -m 'advanced_model and cuda' --run-level 'advanced_model'
|
||||
# "
|
||||
# mirror_hardwares: h100_1
|
||||
|
||||
# - label: "Diffusion · Wan22 Test"
|
||||
# source_file_dependencies:
|
||||
# - tests/e2e/offline_inference/test_wan22_t2v.py
|
||||
# - tests/e2e/online_serving/test_wan22_t2v.py
|
||||
# - vllm_omni/diffusion/models/wan2_2/
|
||||
# commands:
|
||||
# - |
|
||||
# timeout 40m bash -c "
|
||||
# export VLLM_IMAGE_FETCH_TIMEOUT=60
|
||||
# pytest -s -v tests/e2e/offline_inference/test_wan22_t2v.py tests/e2e/online_serving/test_wan22_t2v.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model'
|
||||
# "
|
||||
# mirror_hardwares: h100_1
|
||||
|
||||
# - label: "Diffusion · Z Image Test"
|
||||
# source_file_dependencies:
|
||||
# - tests/e2e/offline_inference/test_z_image.py
|
||||
# - tests/e2e/online_serving/test_z_image.py
|
||||
# - vllm_omni/diffusion/models/z_image/
|
||||
# timeout_in_minutes: 30
|
||||
# commands:
|
||||
# - pytest -s -v tests/e2e/offline_inference/test_z_image.py tests/e2e/online_serving/test_z_image.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model'
|
||||
# mirror_hardwares: l4_1
|
||||
# - label: "Diffusion · FLUX.2 Klein Test"
|
||||
# source_file_dependencies:
|
||||
# - tests/e2e/offline_inference/test_flux2_klein.py
|
||||
# - vllm_omni/diffusion/models/flux2_klein/
|
||||
# timeout_in_minutes: 30
|
||||
# commands:
|
||||
# - pytest -s -v tests/e2e/offline_inference/test_flux2_klein.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model'
|
||||
# mirror_hardwares: l4_1
|
||||
- label: "Diffusion · Z Image Test"
|
||||
source_file_dependencies:
|
||||
- tests/e2e/offline_inference/test_z_image.py
|
||||
- tests/e2e/online_serving/test_z_image.py
|
||||
- vllm_omni/diffusion/models/z_image/
|
||||
timeout_in_minutes: 30
|
||||
commands:
|
||||
- pytest -s -v tests/e2e/offline_inference/test_z_image.py tests/e2e/online_serving/test_z_image.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model'
|
||||
mirror_hardwares: l4_1
|
||||
- label: "Diffusion · FLUX.2 Klein Test"
|
||||
source_file_dependencies:
|
||||
- tests/e2e/offline_inference/test_flux2_klein.py
|
||||
- vllm_omni/diffusion/models/flux2_klein/
|
||||
timeout_in_minutes: 30
|
||||
commands:
|
||||
- pytest -s -v tests/e2e/offline_inference/test_flux2_klein.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model'
|
||||
mirror_hardwares: l4_1
|
||||
+449
-496
File diff suppressed because it is too large.
Load diff
+289
-252
@@ -18,302 +18,339 @@ steps:
|
||||
- "timeout 20m pytest -sv -m 'core_model and cpu' --ignore=tests/diffusion --ignore=tests/model_executor --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"
|
||||
mirror_hardwares: l4_1
|
||||
|
||||
# - label: "Custom Pipeline Test"
|
||||
# depends_on: upload-ready-pipeline
|
||||
# commands:
|
||||
# - timeout 20m pytest -s -v tests/e2e/offline_inference/custom_pipeline/ -m "core_model"
|
||||
# mirror_hardwares: l4_1
|
||||
- label: "Custom Pipeline Test"
|
||||
depends_on: upload-ready-pipeline
|
||||
commands:
|
||||
- timeout 20m pytest -s -v tests/e2e/offline_inference/custom_pipeline/ -m "core_model"
|
||||
mirror_hardwares: l4_1
|
||||
|
||||
# - group: ":card_index_dividers: Diffusion Test"
|
||||
# depends_on: upload-ready-pipeline
|
||||
# steps:
|
||||
- group: ":card_index_dividers: Diffusion Test"
|
||||
depends_on: upload-ready-pipeline
|
||||
steps:
|
||||
|
||||
# - label: "Diffusion · Offloader Test"
|
||||
# commands:
|
||||
# - timeout 40m pytest -s -v tests/diffusion/offloader -m "core_model and cuda" --run-level "core_model"
|
||||
# mirror_hardwares: l4_1
|
||||
- label: "Diffusion · Offloader Test"
|
||||
commands:
|
||||
- timeout 40m pytest -s -v tests/diffusion/offloader -m "core_model and cuda" --run-level "core_model"
|
||||
mirror_hardwares: l4_1
|
||||
|
||||
# - label: "Diffusion · Cache Test"
|
||||
# commands:
|
||||
# - timeout 15m pytest -sv tests/diffusion/cache -m 'core_model and cuda' --run-level "core_model"
|
||||
# mirror_hardwares: l4_1
|
||||
- label: "Diffusion · Cache Test"
|
||||
commands:
|
||||
- timeout 15m pytest -sv tests/diffusion/cache -m 'core_model and cuda' --run-level "core_model"
|
||||
mirror_hardwares: l4_1
|
||||
|
||||
# - label: "Diffusion · Batch Test"
|
||||
# commands:
|
||||
# - timeout 15m pytest -sv tests/diffusion/batching -m 'core_model and cuda' --run-level "core_model"
|
||||
# mirror_hardwares: l4_1
|
||||
- label: "Diffusion · Batch Test"
|
||||
commands:
|
||||
- timeout 15m pytest -sv tests/diffusion/batching -m 'core_model and cuda' --run-level "core_model"
|
||||
mirror_hardwares: l4_1
|
||||
|
||||
# - label: "Diffusion · Distributed Test"
|
||||
# source_file_dependencies:
|
||||
# - tests/diffusion/distributed/
|
||||
# - vllm_omni/diffusion/distributed/
|
||||
# - vllm_omni/diffusion/attention/parallel/
|
||||
# - vllm_omni/diffusion/data.py
|
||||
# - vllm_omni/diffusion/models/
|
||||
# commands:
|
||||
# - timeout 20m pytest -sv tests/diffusion/distributed -m 'core_model and cuda' --run-level "core_model"
|
||||
# mirror_hardwares: l4_4
|
||||
- label: "Diffusion · Distributed Test"
|
||||
source_file_dependencies:
|
||||
- tests/diffusion/distributed/
|
||||
- vllm_omni/diffusion/distributed/
|
||||
- vllm_omni/diffusion/attention/parallel/
|
||||
- vllm_omni/diffusion/data.py
|
||||
- vllm_omni/diffusion/models/
|
||||
commands:
|
||||
- timeout 20m pytest -sv tests/diffusion/distributed -m 'core_model and cuda' --run-level "core_model"
|
||||
mirror_hardwares: l4_4
|
||||
|
||||
# - label: "Diffusion · Model Test"
|
||||
# commands:
|
||||
# - timeout 15m pytest -sv tests/diffusion/models/ -m 'core_model and cuda' --run-level "core_model"
|
||||
# mirror_hardwares: l4_1
|
||||
- label: "Diffusion · Model Test"
|
||||
commands:
|
||||
- timeout 15m pytest -sv tests/diffusion/models/ -m 'core_model and cuda' --run-level "core_model"
|
||||
mirror_hardwares: l4_1
|
||||
|
||||
# - label: "Diffusion · Other Test"
|
||||
# commands:
|
||||
# - timeout 15m pytest -sv tests/diffusion/*.py -m 'core_model and cuda' --run-level "core_model"
|
||||
# mirror_hardwares: l4_1
|
||||
- label: "Diffusion · Other Test"
|
||||
commands:
|
||||
- timeout 15m pytest -sv tests/diffusion/*.py -m 'core_model and cuda' --run-level "core_model"
|
||||
mirror_hardwares: l4_1
|
||||
|
||||
# - group: ":card_index_dividers: Tiny Model Tests [Single GPU]"
|
||||
# depends_on: upload-ready-pipeline
|
||||
# steps:
|
||||
- group: ":card_index_dividers: Tiny Model Tests [Single GPU]"
|
||||
depends_on: upload-ready-pipeline
|
||||
steps:
|
||||
|
||||
# - label: "Tiny Model Tests · Diffusion (base)"
|
||||
# commands:
|
||||
# - timeout 15m pytest -sv tests/model_tests/diffusion -m 'core_model and cuda' --run-level "core_model"
|
||||
# mirror_hardwares: l4_1
|
||||
- label: "Tiny Model Tests · Diffusion (base)"
|
||||
commands:
|
||||
- timeout 15m pytest -sv tests/model_tests/diffusion -m 'core_model and cuda' --run-level "core_model"
|
||||
mirror_hardwares: l4_1
|
||||
|
||||
# - label: "Engine&Model Executor Test"
|
||||
# depends_on: upload-ready-pipeline
|
||||
# commands:
|
||||
# - |
|
||||
# timeout 15m bash -c "
|
||||
# pytest -sv tests/engine/ tests/model_executor/ -m 'core_model and cuda' --run-level 'core_model'
|
||||
# "
|
||||
# mirror_hardwares: l4_1
|
||||
- label: "Engine&Model Executor Test"
|
||||
depends_on: upload-ready-pipeline
|
||||
commands:
|
||||
- |
|
||||
timeout 15m bash -c "
|
||||
pytest -sv tests/engine/ tests/model_executor/ -m 'core_model and cuda' --run-level 'core_model'
|
||||
"
|
||||
mirror_hardwares: l4_1
|
||||
|
||||
# - label: "Distributed Test"
|
||||
# depends_on: upload-ready-pipeline
|
||||
# commands:
|
||||
# - |
|
||||
# timeout 20m bash -c "
|
||||
# pytest -sv tests/distributed/ -m 'core_model and cuda and L4' --run-level 'core_model'
|
||||
# "
|
||||
# mirror_hardwares: l4_1
|
||||
- label: "Distributed Test"
|
||||
depends_on: upload-ready-pipeline
|
||||
commands:
|
||||
- |
|
||||
timeout 20m bash -c "
|
||||
pytest -sv tests/distributed/ -m 'core_model and cuda and L4' --run-level 'core_model'
|
||||
"
|
||||
mirror_hardwares: l4_1
|
||||
|
||||
# - label: "Entrypoints Test"
|
||||
# depends_on: upload-ready-pipeline
|
||||
# commands:
|
||||
# - |
|
||||
# timeout 30m bash -c "
|
||||
# export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
# pytest -sv tests/entrypoints/ -m 'core_model and cuda' --run-level 'core_model'
|
||||
# "
|
||||
# mirror_hardwares: l4_1
|
||||
- label: "Entrypoints Test"
|
||||
depends_on: upload-ready-pipeline
|
||||
commands:
|
||||
- |
|
||||
timeout 30m bash -c "
|
||||
export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
pytest -sv tests/entrypoints/ -m 'core_model and cuda' --run-level 'core_model'
|
||||
"
|
||||
mirror_hardwares: l4_1
|
||||
|
||||
- group: ":card_index_dividers: E2E Test"
|
||||
depends_on: upload-ready-pipeline
|
||||
steps:
|
||||
|
||||
# - label: "RLHF · VeRL-Omni E2E Test"
|
||||
# source_file_dependencies:
|
||||
# - tests/e2e/offline_inference/rlhf_test/test_verl_omni_e2e.py
|
||||
# - tests/e2e/offline_inference/custom_pipeline/
|
||||
# - vllm_omni/diffusion/
|
||||
# - vllm_omni/engine/
|
||||
# - vllm_omni/entrypoints/
|
||||
# - vllm_omni/inputs/
|
||||
# - vllm_omni/outputs/
|
||||
# commands:
|
||||
# - |
|
||||
# timeout 20m bash -c "
|
||||
# pytest -s -v tests/e2e/offline_inference/rlhf_test/test_verl_omni_e2e.py
|
||||
# "
|
||||
# mirror_hardwares: l4_1
|
||||
- label: "RLHF · VeRL-Omni E2E Test"
|
||||
source_file_dependencies:
|
||||
- tests/e2e/offline_inference/rlhf_test/test_verl_omni_e2e.py
|
||||
- tests/e2e/offline_inference/custom_pipeline/
|
||||
- vllm_omni/diffusion/
|
||||
- vllm_omni/engine/
|
||||
- vllm_omni/entrypoints/
|
||||
- vllm_omni/inputs/
|
||||
- vllm_omni/outputs/
|
||||
commands:
|
||||
- |
|
||||
timeout 20m bash -c "
|
||||
pytest -s -v tests/e2e/offline_inference/rlhf_test/test_verl_omni_e2e.py
|
||||
"
|
||||
mirror_hardwares: l4_1
|
||||
|
||||
# - label: "Omni · Qwen3-Omni Test"
|
||||
# source_file_dependencies:
|
||||
# - tests/e2e/online_serving/test_qwen3_omni.py
|
||||
# - vllm_omni/model_executor/models/qwen3_omni/
|
||||
# - vllm_omni/model_executor/stage_input_processors/qwen3_omni.py
|
||||
# - vllm_omni/deploy/qwen3_omni_moe.yaml
|
||||
# commands:
|
||||
# - |
|
||||
# timeout 20m bash -c "
|
||||
# pytest -s -v tests/e2e/online_serving/test_qwen3_omni.py -m 'core_model' --run-level 'core_model'
|
||||
# "
|
||||
# mirror_hardwares: h100_2
|
||||
- label: "Omni · Qwen3-Omni Test"
|
||||
source_file_dependencies:
|
||||
- tests/e2e/online_serving/test_qwen3_omni.py
|
||||
- vllm_omni/model_executor/models/qwen3_omni/
|
||||
- vllm_omni/model_executor/stage_input_processors/qwen3_omni.py
|
||||
- vllm_omni/deploy/qwen3_omni_moe.yaml
|
||||
commands:
|
||||
- |
|
||||
timeout 20m bash -c "
|
||||
pytest -s -v tests/e2e/online_serving/test_qwen3_omni.py -m 'core_model' --run-level 'core_model'
|
||||
"
|
||||
mirror_hardwares: h100_2
|
||||
|
||||
- label: "Omni · MiniCPM-o 4.5 Offline Test"
|
||||
source_file_dependencies:
|
||||
- tests/e2e/offline_inference/test_minicpmo_4_5.py
|
||||
- vllm_omni/deploy/minicpmo_4_5.yaml
|
||||
commands:
|
||||
- |
|
||||
timeout 20m bash -c "
|
||||
pytest -s -v tests/e2e/offline_inference/test_minicpmo_4_5.py -m 'core_model' --run-level 'core_model'
|
||||
"
|
||||
mirror_hardwares: h100_1
|
||||
agents:
|
||||
queue: "mithril-h100-pool"
|
||||
plugins:
|
||||
- kubernetes:
|
||||
podSpec:
|
||||
containers:
|
||||
- image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
|
||||
resources:
|
||||
limits:
|
||||
nvidia.com/gpu: 2
|
||||
volumeMounts:
|
||||
- name: devshm
|
||||
mountPath: /dev/shm
|
||||
- name: hf-cache
|
||||
mountPath: /root/.cache/huggingface
|
||||
env:
|
||||
- name: HF_HOME
|
||||
value: /root/.cache/huggingface
|
||||
nodeSelector:
|
||||
node.kubernetes.io/instance-type: gpu-h100-sxm
|
||||
volumes:
|
||||
- name: devshm
|
||||
emptyDir:
|
||||
medium: Memory
|
||||
- name: hf-cache
|
||||
hostPath:
|
||||
path: /mnt/hf-cache
|
||||
type: DirectoryOrCreate
|
||||
|
||||
- label: "Omni · MiniCPM-o 4.5 Online Test"
|
||||
source_file_dependencies:
|
||||
- tests/e2e/online_serving/test_minicpmo_4_5.py
|
||||
- vllm_omni/deploy/minicpmo_4_5.yaml
|
||||
commands:
|
||||
- |
|
||||
timeout 20m bash -c "
|
||||
pytest -s -v tests/e2e/online_serving/test_minicpmo_4_5.py -m 'core_model' --run-level 'core_model'
|
||||
"
|
||||
mirror_hardwares: h100_1
|
||||
agents:
|
||||
queue: "mithril-h100-pool"
|
||||
plugins:
|
||||
- kubernetes:
|
||||
podSpec:
|
||||
containers:
|
||||
- image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
|
||||
resources:
|
||||
limits:
|
||||
nvidia.com/gpu: 2
|
||||
volumeMounts:
|
||||
- name: devshm
|
||||
mountPath: /dev/shm
|
||||
- name: hf-cache
|
||||
mountPath: /root/.cache/huggingface
|
||||
env:
|
||||
- name: HF_HOME
|
||||
value: /root/.cache/huggingface
|
||||
nodeSelector:
|
||||
node.kubernetes.io/instance-type: gpu-h100-sxm
|
||||
volumes:
|
||||
- name: devshm
|
||||
emptyDir:
|
||||
medium: Memory
|
||||
- name: hf-cache
|
||||
hostPath:
|
||||
path: /mnt/hf-cache
|
||||
type: DirectoryOrCreate
|
||||
|
||||
- label: "Omni · MiniCPM-o 4.5 Duplex Test"
|
||||
- label: "TTS · VoxCPM2 Test"
|
||||
source_file_dependencies:
|
||||
- tests/e2e/online_serving/test_voxcpm2_tts.py
|
||||
- vllm_omni/model_executor/models/voxcpm2/
|
||||
- vllm_omni/deploy/voxcpm2.yaml
|
||||
timeout_in_minutes: 30
|
||||
commands:
|
||||
- |
|
||||
timeout 30m bash -c "
|
||||
export VLLM_LOGGING_LEVEL=DEBUG
|
||||
export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
pytest -s -v tests/e2e/online_serving/test_voxcpm2_tts.py -m 'core_model' --run-level 'core_model'
|
||||
"
|
||||
mirror_hardwares: l4_1
|
||||
|
||||
- label: "TTS · Qwen3-TTS CustomVoice Test"
|
||||
source_file_dependencies:
|
||||
- tests/e2e/online_serving/test_qwen3_tts_customvoice.py
|
||||
- vllm_omni/model_executor/models/qwen3_tts/
|
||||
- vllm_omni/model_executor/stage_input_processors/qwen3_tts.py
|
||||
- vllm_omni/deploy/qwen3_tts.yaml
|
||||
commands:
|
||||
- |
|
||||
timeout 20m bash -c "
|
||||
pytest -s -v tests/e2e/online_serving/test_minicpmo_4_5_duplex.py -m 'core_model and cuda' --run-level 'core_model'
|
||||
export VLLM_ALLOW_LONG_MAX_MODEL_LEN=1
|
||||
pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py -m 'core_model' --run-level 'core_model'
|
||||
"
|
||||
mirror_hardwares: l4_1
|
||||
|
||||
- label: "TTS · Qwen3-TTS Base Test"
|
||||
source_file_dependencies:
|
||||
- tests/e2e/online_serving/test_qwen3_tts_base.py
|
||||
- vllm_omni/model_executor/models/qwen3_tts/
|
||||
- vllm_omni/model_executor/stage_input_processors/qwen3_tts.py
|
||||
- vllm_omni/deploy/qwen3_tts.yaml
|
||||
commands:
|
||||
- |
|
||||
timeout 20m bash -c "
|
||||
export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
export VLLM_ALLOW_LONG_MAX_MODEL_LEN=1
|
||||
export VLLM_OMNI_USE_V2_RUNNER=1
|
||||
pytest -s -v tests/e2e/online_serving/test_qwen3_tts_base.py -m 'core_model' --run-level 'core_model'
|
||||
"
|
||||
mirror_hardwares: l4_1
|
||||
|
||||
- label: "TTS · Higgs-Audio-V3 Test"
|
||||
source_file_dependencies:
|
||||
- tests/e2e/online_serving/test_higgs_audio_v3.py
|
||||
- vllm_omni/model_executor/models/higgs_audio_v3/
|
||||
- vllm_omni/model_executor/stage_input_processors/higgs_audio_v3.py
|
||||
- vllm_omni/deploy/higgs_multimodal_qwen3.yaml
|
||||
commands:
|
||||
- |
|
||||
timeout 20m bash -c "
|
||||
export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
export VLLM_USE_DEEP_GEMM=0
|
||||
export VLLM_MOE_USE_DEEP_GEMM=0
|
||||
pytest -s -v tests/e2e/online_serving/test_higgs_audio_v3.py -m 'core_model' --run-level 'core_model'
|
||||
"
|
||||
mirror_hardwares: l4_1
|
||||
|
||||
- label: "Diffusion · Qwen Image Edit Test"
|
||||
source_file_dependencies:
|
||||
- tests/e2e/online_serving/test_qwen_image_edit.py
|
||||
- vllm_omni/diffusion/models/qwen_image/
|
||||
commands:
|
||||
- |
|
||||
timeout 20m bash -c "
|
||||
pytest -s -v tests/e2e/online_serving/test_qwen_image_edit.py -m 'core_model' --run-level 'core_model'
|
||||
"
|
||||
mirror_hardwares: h100_1
|
||||
|
||||
- label: "Omni · MiniCPM-o 4.5 Perf Test"
|
||||
timeout_in_minutes: 60
|
||||
- label: "Diffusion · Bagel Test"
|
||||
source_file_dependencies:
|
||||
- tests/dfx/perf/tests/test_minicpmo_4_5_ready.json
|
||||
- vllm_omni/deploy/minicpmo_4_5.yaml
|
||||
- vllm_omni/model_executor/models/minicpmo_4_5/
|
||||
- vllm_omni/model_executor/models/step_audio2/
|
||||
- vllm_omni/model_executor/models/cosyvoice3/code2wav_core/
|
||||
- vllm_omni/model_executor/models/cosyvoice3/utils.py
|
||||
- vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py
|
||||
- tests/e2e/online_serving/test_bagel.py
|
||||
- vllm_omni/model_executor/models/bagel/
|
||||
- vllm_omni/diffusion/models/bagel/
|
||||
- vllm_omni/model_executor/stage_input_processors/bagel.py
|
||||
- vllm_omni/deploy/bagel.yaml
|
||||
commands:
|
||||
- export BENCHMARK_DIR=tests/dfx/perf/results
|
||||
- pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_minicpmo_4_5_ready.json -m 'core_model and cuda'
|
||||
- |
|
||||
timeout 40m bash -c "
|
||||
export VLLM_IMAGE_FETCH_TIMEOUT=60
|
||||
pytest -s -v tests/e2e/online_serving/test_bagel.py -m 'core_model' --run-level 'core_model'
|
||||
"
|
||||
mirror_hardwares: h100_1
|
||||
|
||||
# - label: "TTS · VoxCPM2 Test"
|
||||
# source_file_dependencies:
|
||||
# - tests/e2e/online_serving/test_voxcpm2_tts.py
|
||||
# - vllm_omni/model_executor/models/voxcpm2/
|
||||
# - vllm_omni/deploy/voxcpm2.yaml
|
||||
# timeout_in_minutes: 30
|
||||
# commands:
|
||||
# - |
|
||||
# timeout 30m bash -c "
|
||||
# export VLLM_LOGGING_LEVEL=DEBUG
|
||||
# export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
# pytest -s -v tests/e2e/online_serving/test_voxcpm2_tts.py -m 'core_model' --run-level 'core_model'
|
||||
# "
|
||||
# mirror_hardwares: l4_1
|
||||
- label: "Diffusion · Wan22 Test"
|
||||
source_file_dependencies:
|
||||
- tests/e2e/online_serving/test_wan22_t2v.py
|
||||
- vllm_omni/diffusion/models/wan2_2/
|
||||
commands:
|
||||
- |
|
||||
timeout 40m bash -c "
|
||||
export VLLM_IMAGE_FETCH_TIMEOUT=60
|
||||
pytest -s -v tests/e2e/online_serving/test_wan22_t2v.py -m 'core_model' --run-level 'core_model'
|
||||
"
|
||||
mirror_hardwares: h100_1
|
||||
|
||||
# - label: "TTS · Qwen3-TTS CustomVoice Test"
|
||||
# source_file_dependencies:
|
||||
# - tests/e2e/online_serving/test_qwen3_tts_customvoice.py
|
||||
# - vllm_omni/model_executor/models/qwen3_tts/
|
||||
# - vllm_omni/model_executor/stage_input_processors/qwen3_tts.py
|
||||
# - vllm_omni/deploy/qwen3_tts.yaml
|
||||
# commands:
|
||||
# - |
|
||||
# timeout 20m bash -c "
|
||||
# export VLLM_ALLOW_LONG_MAX_MODEL_LEN=1
|
||||
# pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py -m 'core_model' --run-level 'core_model'
|
||||
# "
|
||||
# mirror_hardwares: l4_1
|
||||
- label: "Diffusion · Cosmos3 Test"
|
||||
source_file_dependencies:
|
||||
- tests/e2e/online_serving/test_cosmos3.py
|
||||
- vllm_omni/diffusion/models/cosmos3/
|
||||
commands:
|
||||
- |
|
||||
timeout 40m bash -c "
|
||||
export VLLM_IMAGE_FETCH_TIMEOUT=60
|
||||
pytest -s -v tests/e2e/online_serving/test_cosmos3.py -m 'core_model' --run-level 'core_model'
|
||||
"
|
||||
mirror_hardwares: h100_1
|
||||
|
||||
# - label: "TTS · Qwen3-TTS Base Test"
|
||||
# source_file_dependencies:
|
||||
# - tests/e2e/online_serving/test_qwen3_tts_base.py
|
||||
# - vllm_omni/model_executor/models/qwen3_tts/
|
||||
# - vllm_omni/model_executor/stage_input_processors/qwen3_tts.py
|
||||
# - vllm_omni/deploy/qwen3_tts.yaml
|
||||
# commands:
|
||||
# - |
|
||||
# timeout 20m bash -c "
|
||||
# export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
# export VLLM_ALLOW_LONG_MAX_MODEL_LEN=1
|
||||
# export VLLM_OMNI_USE_V2_RUNNER=1
|
||||
# pytest -s -v tests/e2e/online_serving/test_qwen3_tts_base.py -m 'core_model' --run-level 'core_model'
|
||||
# "
|
||||
# mirror_hardwares: l4_1
|
||||
- label: "Diffusion · Qwen Image Test"
|
||||
source_file_dependencies:
|
||||
- tests/e2e/online_serving/test_qwen_image.py
|
||||
- vllm_omni/diffusion/models/qwen_image/
|
||||
commands:
|
||||
- |
|
||||
timeout 40m bash -c "
|
||||
export VLLM_IMAGE_FETCH_TIMEOUT=60
|
||||
pytest -s -v tests/e2e/online_serving/test_qwen_image.py -m 'core_model' --run-level 'core_model'
|
||||
"
|
||||
mirror_hardwares: h100_1
|
||||
|
||||
# - label: "TTS · Higgs-Audio-V3 Test"
|
||||
# source_file_dependencies:
|
||||
# - tests/e2e/online_serving/test_higgs_audio_v3.py
|
||||
# - vllm_omni/model_executor/models/higgs_audio_v3/
|
||||
# - vllm_omni/model_executor/stage_input_processors/higgs_audio_v3.py
|
||||
# - vllm_omni/deploy/higgs_multimodal_qwen3.yaml
|
||||
# commands:
|
||||
# - |
|
||||
# timeout 20m bash -c "
|
||||
# export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
# export VLLM_USE_DEEP_GEMM=0
|
||||
# export VLLM_MOE_USE_DEEP_GEMM=0
|
||||
# pytest -s -v tests/e2e/online_serving/test_higgs_audio_v3.py -m 'core_model' --run-level 'core_model'
|
||||
# "
|
||||
# mirror_hardwares: l4_1
|
||||
- label: "Diffusion · Qwen Image Layered Test"
|
||||
source_file_dependencies:
|
||||
- tests/e2e/online_serving/test_qwen_image_layered.py
|
||||
- vllm_omni/diffusion/models/qwen_image/
|
||||
commands:
|
||||
- |
|
||||
timeout 40m bash -c "
|
||||
export VLLM_IMAGE_FETCH_TIMEOUT=60
|
||||
pytest -s -v tests/e2e/online_serving/test_qwen_image_layered.py -m 'core_model' --run-level 'core_model'
|
||||
"
|
||||
mirror_hardwares: h100_1
|
||||
|
||||
# - label: "Diffusion · Qwen Image Edit Test"
|
||||
# source_file_dependencies:
|
||||
# - tests/e2e/online_serving/test_qwen_image_edit.py
|
||||
# - vllm_omni/diffusion/models/qwen_image/
|
||||
# commands:
|
||||
# - |
|
||||
# timeout 20m bash -c "
|
||||
# pytest -s -v tests/e2e/online_serving/test_qwen_image_edit.py -m 'core_model' --run-level 'core_model'
|
||||
# "
|
||||
# mirror_hardwares: h100_1
|
||||
|
||||
# - label: "Diffusion · Bagel Test"
|
||||
# source_file_dependencies:
|
||||
# - tests/e2e/online_serving/test_bagel.py
|
||||
# - vllm_omni/model_executor/models/bagel/
|
||||
# - vllm_omni/diffusion/models/bagel/
|
||||
# - vllm_omni/model_executor/stage_input_processors/bagel.py
|
||||
# - vllm_omni/deploy/bagel.yaml
|
||||
# commands:
|
||||
# - |
|
||||
# timeout 40m bash -c "
|
||||
# export VLLM_IMAGE_FETCH_TIMEOUT=60
|
||||
# pytest -s -v tests/e2e/online_serving/test_bagel.py -m 'core_model' --run-level 'core_model'
|
||||
# "
|
||||
# mirror_hardwares: h100_1
|
||||
|
||||
# - label: "Diffusion · Wan22 Test"
|
||||
# source_file_dependencies:
|
||||
# - tests/e2e/online_serving/test_wan22_t2v.py
|
||||
# - vllm_omni/diffusion/models/wan2_2/
|
||||
# commands:
|
||||
# - |
|
||||
# timeout 40m bash -c "
|
||||
# export VLLM_IMAGE_FETCH_TIMEOUT=60
|
||||
# pytest -s -v tests/e2e/online_serving/test_wan22_t2v.py -m 'core_model' --run-level 'core_model'
|
||||
# "
|
||||
# mirror_hardwares: h100_1
|
||||
|
||||
# - label: "Diffusion · Cosmos3 Test"
|
||||
# source_file_dependencies:
|
||||
# - tests/e2e/online_serving/test_cosmos3.py
|
||||
# - vllm_omni/diffusion/models/cosmos3/
|
||||
# commands:
|
||||
# - |
|
||||
# timeout 40m bash -c "
|
||||
# export VLLM_IMAGE_FETCH_TIMEOUT=60
|
||||
# pytest -s -v tests/e2e/online_serving/test_cosmos3.py -m 'core_model' --run-level 'core_model'
|
||||
# "
|
||||
# mirror_hardwares: h100_1
|
||||
|
||||
# - label: "Diffusion · Qwen Image Test"
|
||||
# source_file_dependencies:
|
||||
# - tests/e2e/online_serving/test_qwen_image.py
|
||||
# - vllm_omni/diffusion/models/qwen_image/
|
||||
# commands:
|
||||
# - |
|
||||
# timeout 40m bash -c "
|
||||
# export VLLM_IMAGE_FETCH_TIMEOUT=60
|
||||
# pytest -s -v tests/e2e/online_serving/test_qwen_image.py -m 'core_model' --run-level 'core_model'
|
||||
# "
|
||||
# mirror_hardwares: h100_1
|
||||
|
||||
# - label: "Diffusion · Qwen Image Layered Test"
|
||||
# source_file_dependencies:
|
||||
# - tests/e2e/online_serving/test_qwen_image_layered.py
|
||||
# - vllm_omni/diffusion/models/qwen_image/
|
||||
# commands:
|
||||
# - |
|
||||
# timeout 40m bash -c "
|
||||
# export VLLM_IMAGE_FETCH_TIMEOUT=60
|
||||
# pytest -s -v tests/e2e/online_serving/test_qwen_image_layered.py -m 'core_model' --run-level 'core_model'
|
||||
# "
|
||||
# mirror_hardwares: h100_1
|
||||
|
||||
# - label: "Diffusion · GGUF Plugin Test"
|
||||
# soft_fail: true
|
||||
# source_file_dependencies:
|
||||
# - tests/plugin/test_gguf_diffusion.py
|
||||
# - vllm_omni/diffusion/model_loader
|
||||
# commands:
|
||||
# - pip install "vllm-gguf-plugin>=0.0.3"
|
||||
# - timeout 30m pytest -sv tests/plugin/test_gguf_diffusion.py
|
||||
# mirror_hardwares: l4_1
|
||||
- label: "Diffusion · GGUF Plugin Test"
|
||||
soft_fail: true
|
||||
source_file_dependencies:
|
||||
- tests/plugin/test_gguf_diffusion.py
|
||||
- vllm_omni/diffusion/model_loader
|
||||
commands:
|
||||
- pip install "vllm-gguf-plugin>=0.0.3"
|
||||
- timeout 30m pytest -sv tests/plugin/test_gguf_diffusion.py
|
||||
mirror_hardwares: l4_1
|
||||
@@ -4,10 +4,6 @@
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# minicpm-challenge branch: skip entire Intel/XPU CI (no pipeline upload => no image build / tests).
|
||||
echo "--- Skipping Intel/XPU CI on this branch (including image build)"
|
||||
exit 0
|
||||
|
||||
source .buildkite/common/scripts/resolve_skip_ci.sh
|
||||
|
||||
upload_pipeline() {
|
||||
|
||||
@@ -1,119 +1,16 @@
|
||||
env:
|
||||
VLLM_WORKER_MULTIPROC_METHOD: spawn
|
||||
VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
|
||||
HF_HUB_DOWNLOAD_TIMEOUT: 300
|
||||
HF_HUB_ETAG_TIMEOUT: 60
|
||||
|
||||
steps:
|
||||
# - group: ":card_index_dividers: Diffusion X2V Model Test"
|
||||
# key: nightly-diffusion-x2v-group
|
||||
# depends_on: upload-nightly-pipeline
|
||||
# if: >-
|
||||
# build.env("NIGHTLY") == "1" ||
|
||||
# build.pull_request.labels includes "nightly-test" ||
|
||||
# build.pull_request.labels includes "diffusion-x2v-test"
|
||||
# steps:
|
||||
# - label: ":full_moon: Diffusion X2V · Function Test"
|
||||
# mirror_hardwares: a2b3_npu_4
|
||||
# env:
|
||||
# VLLM_WORKER_MULTIPROC_METHOD: spawn
|
||||
# HF_TOKEN: "${HF_TOKEN}"
|
||||
# commands:
|
||||
# - pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py -k npu_i2v --run-level "full_model"
|
||||
# - group: ":card_index_dividers: TTS Model Test"
|
||||
# key: nightly-tts-test-group
|
||||
# depends_on: upload-nightly-pipeline
|
||||
# if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test" || build.pull_request.labels includes "tts-test"
|
||||
# steps:
|
||||
# - label: ":full_moon: TTS · Perf Test"
|
||||
# mirror_hardwares: a3_npu_2
|
||||
# env:
|
||||
# VLLM_WORKER_MULTIPROC_METHOD: spawn
|
||||
# VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
|
||||
# HF_TOKEN: "${HF_TOKEN}"
|
||||
# commands:
|
||||
# - export BENCHMARK_DIR=tests/dfx/perf/results
|
||||
# - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
|
||||
# - |
|
||||
# set +e
|
||||
# pytest -s -v tests/dfx/perf/scripts/run_benchmark.py -m npu --test-config-file tests/dfx/perf/tests/test_tts.json
|
||||
# EXIT=$$?
|
||||
# buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
|
||||
# exit $$EXIT
|
||||
|
||||
- group: ":card_index_dividers: Omni Model Test"
|
||||
key: nightly-omni-group
|
||||
key: nightly-omni-test-group
|
||||
depends_on: upload-nightly-pipeline
|
||||
if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test" || build.pull_request.labels includes "omni-test"
|
||||
if: >-
|
||||
build.env("NIGHTLY") == "1" ||
|
||||
build.pull_request.labels includes "nightly-test" ||
|
||||
build.pull_request.labels includes "omni-test"
|
||||
steps:
|
||||
- label: ":full_moon: Omni · MiniCPM-o 4.5 Test"
|
||||
- label: ":full_moon: Omni · Perf Test · No Async Chunk"
|
||||
key: nightly-omni-performance-no-async-chunk
|
||||
timeout_in_minutes: 180
|
||||
mirror_hardwares: a3_npu_2
|
||||
env:
|
||||
VLLM_WORKER_MULTIPROC_METHOD: spawn
|
||||
VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
|
||||
HF_TOKEN: "${HF_TOKEN}"
|
||||
commands:
|
||||
- pytest -s -v tests/e2e/offline_inference/test_minicpmo_4_5.py tests/e2e/online_serving/test_minicpmo_4_5.py tests/e2e/online_serving/test_minicpmo_4_5_expansion.py -m 'full_model and npu and A3 and omni' --run-level 'full_model'
|
||||
- label: ":full_moon: Omni · MiniCPM-o 4.5 · Duplex Test"
|
||||
timeout_in_minutes: 180
|
||||
mirror_hardwares: a3_npu_2
|
||||
env:
|
||||
VLLM_WORKER_MULTIPROC_METHOD: spawn
|
||||
VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
|
||||
HF_TOKEN: "${HF_TOKEN}"
|
||||
commands:
|
||||
# NPU has no merge-level pipeline, so advanced_model duplex coverage
|
||||
# rides on nightly alongside full_model.
|
||||
#
|
||||
# test_duplex_soft_interrupt is CUDA-only and carries no npu mark, so
|
||||
# the marker filter below drops it: it needs real-time duplex
|
||||
# throughput that the current NPU stack does not reach.
|
||||
- >-
|
||||
pytest -s -v
|
||||
tests/e2e/online_serving/test_minicpmo_4_5_duplex.py
|
||||
tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py
|
||||
-m "(full_model or advanced_model) and npu and A3 and omni"
|
||||
--run-level full_model
|
||||
- label: ":full_moon: Omni · MiniCPM-o 4.5 · Accuracy Test"
|
||||
timeout_in_minutes: 180
|
||||
mirror_hardwares: a3_npu_2
|
||||
env:
|
||||
VLLM_WORKER_MULTIPROC_METHOD: spawn
|
||||
VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
|
||||
HF_TOKEN: "${HF_TOKEN}"
|
||||
commands:
|
||||
- export SEED_TTS_WER_EVAL=1
|
||||
- export SEED_TTS_SIM_EVAL=1
|
||||
- export SEED_TTS_EVAL_DEVICE=npu:1
|
||||
- |
|
||||
set +e
|
||||
pytest -s -v tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py -k "not videomme" -m "full_model and npu and A3" --run-level full_model
|
||||
EXIT=$$?
|
||||
buildkite-agent artifact upload "tests/e2e/accuracy/minicpmo_4_5/results/*.json"
|
||||
exit $$EXIT
|
||||
- label: ":full_moon: Omni · MiniCPM-o 4.5 · Video-MME Accuracy Test"
|
||||
key: nightly-omni-accuracy-minicpmo-4-5-videomme-npu
|
||||
timeout_in_minutes: 180
|
||||
mirror_hardwares: a3_npu_2
|
||||
env:
|
||||
VLLM_WORKER_MULTIPROC_METHOD: spawn
|
||||
VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
|
||||
HF_TOKEN: "${HF_TOKEN}"
|
||||
commands:
|
||||
# Video-MME needs no repo override here: the NPU runners pre-stage the dataset under
|
||||
# the lmms-eval mirror, which is now VIDEOMME_DEFAULT_HF_REPO, so snapshot_download
|
||||
# resolves it straight out of the hub cache.
|
||||
- |
|
||||
set +e
|
||||
pytest -s -v tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py -k videomme -m "full_model and npu and A3" --run-level full_model
|
||||
EXIT=$$?
|
||||
buildkite-agent artifact upload "tests/e2e/accuracy/minicpmo_4_5/results/*.json"
|
||||
exit $$EXIT
|
||||
- label: ":full_moon: Omni · MiniCPM-o 4.5 · Perf Test"
|
||||
key: nightly-omni-performance-minicpmo-4-5-npu
|
||||
timeout_in_minutes: 180
|
||||
mirror_hardwares: a3_npu_2
|
||||
mirror_hardwares: a3_npu_4
|
||||
env:
|
||||
VLLM_WORKER_MULTIPROC_METHOD: spawn
|
||||
VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
|
||||
@@ -122,29 +19,40 @@ steps:
|
||||
- export BENCHMARK_DIR=tests/dfx/perf/results
|
||||
- |
|
||||
set +e
|
||||
pytest -s -v tests/dfx/perf/scripts/run_benchmark.py -m "npu and A3" --test-config-file tests/dfx/perf/tests/test_minicpmo_4_5.json
|
||||
pytest -s -v tests/dfx/perf/scripts/run_benchmark.py -m npu --test-config-file tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json
|
||||
EXIT=$$?
|
||||
buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
|
||||
exit $$EXIT
|
||||
- label: ":full_moon: Omni · MiniCPM-o 4.5 · Perf Test (A2)"
|
||||
key: nightly-omni-performance-minicpmo-4-5-npu-a2
|
||||
- label: ":full_moon: Omni · Perf Test · Async Chunk"
|
||||
key: nightly-omni-performance-async-chunk
|
||||
timeout_in_minutes: 180
|
||||
mirror_hardwares: a2b3_npu_1
|
||||
mirror_hardwares: a3_npu_4
|
||||
env:
|
||||
VLLM_WORKER_MULTIPROC_METHOD: spawn
|
||||
VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
|
||||
HF_TOKEN: "${HF_TOKEN}"
|
||||
HF_HUB_OFFLINE: "1"
|
||||
commands:
|
||||
- export BENCHMARK_DIR=tests/dfx/perf/results
|
||||
- |
|
||||
set +e
|
||||
pytest -s -v tests/dfx/perf/scripts/run_benchmark.py -m "npu and A2" --test-config-file tests/dfx/perf/tests/test_minicpmo_4_5.json
|
||||
pytest -s -v tests/dfx/perf/scripts/run_benchmark.py -m npu --test-config-file tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json
|
||||
EXIT=$$?
|
||||
buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
|
||||
exit $$EXIT
|
||||
- label: ":full_moon: Omni · MiniCPM-o 4.5 · Duplex Seed-TTS Perf Test"
|
||||
key: nightly-omni-performance-minicpmo-4-5-duplex-seed-tts-npu
|
||||
- label: ":robot: Omni · MiniCPM-o 4.5 Test"
|
||||
mirror_hardwares: a3_npu_2
|
||||
env:
|
||||
VLLM_WORKER_MULTIPROC_METHOD: spawn
|
||||
VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
|
||||
HF_TOKEN: "${HF_TOKEN}"
|
||||
commands:
|
||||
- pytest -s -v tests/e2e/offline_inference/test_minicpmo_4_5.py tests/e2e/online_serving/test_minicpmo_4_5.py tests/e2e/online_serving/test_minicpmo_4_5_expansion.py -m 'full_model' --run-level 'full_model'
|
||||
- group: ":card_index_dividers: TTS Model Test"
|
||||
key: nightly-tts-test-group
|
||||
depends_on: upload-nightly-pipeline
|
||||
if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test" || build.pull_request.labels includes "tts-test"
|
||||
steps:
|
||||
- label: ":full_moon: TTS · Perf Test"
|
||||
timeout_in_minutes: 180
|
||||
mirror_hardwares: a3_npu_2
|
||||
env:
|
||||
@@ -153,9 +61,63 @@ steps:
|
||||
HF_TOKEN: "${HF_TOKEN}"
|
||||
commands:
|
||||
- export BENCHMARK_DIR=tests/dfx/perf/results
|
||||
- export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
|
||||
- |
|
||||
set +e
|
||||
pytest -s -v tests/dfx/perf/scripts/run_benchmark.py -m "npu and A3" --test-config-file tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json
|
||||
pytest -s -v tests/dfx/perf/scripts/run_benchmark.py -m npu --test-config-file tests/dfx/perf/tests/test_tts.json
|
||||
EXIT=$$?
|
||||
buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
|
||||
exit $$EXIT
|
||||
- group: ":card_index_dividers: Diffusion X2I(&A&T) Model Test"
|
||||
key: nightly-diffusion-x2iat-group
|
||||
depends_on: upload-nightly-pipeline
|
||||
if: >-
|
||||
build.env("NIGHTLY") == "1" ||
|
||||
build.pull_request.labels includes "nightly-test" ||
|
||||
build.pull_request.labels includes "diffusion-x2iat-test"
|
||||
steps:
|
||||
- label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · Qwen-Image-Edit-2511"
|
||||
key: nightly-diffusion-x2iat-performance-qwen-image-edit-2511
|
||||
timeout_in_minutes: 180
|
||||
mirror_hardwares: a3_npu_8
|
||||
env:
|
||||
VLLM_WORKER_MULTIPROC_METHOD: spawn
|
||||
VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
|
||||
HF_TOKEN: "${HF_TOKEN}"
|
||||
DIFFUSION_ATTENTION_BACKEND: TORCH_SDPA
|
||||
commands:
|
||||
- export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
|
||||
- export CACHE_DIT_VERSION=1.3.0
|
||||
- |
|
||||
set +e
|
||||
pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py -m npu --test-config-file tests/dfx/perf/tests/test_qwen_image_edit_2511_vllm_omni.json
|
||||
EXIT=$$?
|
||||
buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
|
||||
buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
|
||||
exit $$EXIT
|
||||
- group: ":card_index_dividers: Diffusion X2V Model Test"
|
||||
key: nightly-diffusion-x2v-group
|
||||
depends_on: upload-nightly-pipeline
|
||||
if: >-
|
||||
build.env("NIGHTLY") == "1" ||
|
||||
build.pull_request.labels includes "nightly-test" ||
|
||||
build.pull_request.labels includes "diffusion-x2v-test"
|
||||
steps:
|
||||
- label: ":full_moon: Diffusion X2V · Perf Test"
|
||||
key: nightly-diffusion-x2v-performance
|
||||
timeout_in_minutes: 180
|
||||
mirror_hardwares: a3_npu_4
|
||||
env:
|
||||
VLLM_WORKER_MULTIPROC_METHOD: spawn
|
||||
VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
|
||||
HF_TOKEN: "${HF_TOKEN}"
|
||||
DIFFUSION_ATTENTION_BACKEND: TORCH_SDPA
|
||||
commands:
|
||||
- export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
|
||||
- |
|
||||
set +e
|
||||
pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py -m npu --test-config-file tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json
|
||||
EXIT=$$?
|
||||
buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
|
||||
buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
|
||||
exit $$EXIT
|
||||
@@ -3,41 +3,32 @@ steps:
|
||||
key: ready-omni-group
|
||||
depends_on: upload-ready-pipeline
|
||||
steps:
|
||||
- label: "Omni · MiniCPM-o 4.5 Test"
|
||||
mirror_hardwares: a3_npu_2
|
||||
- label: ":robot: Omni · MiniCPM-o 4.5 Test"
|
||||
agents:
|
||||
queue: "ascend-a3"
|
||||
resource_class: "npu-2"
|
||||
image: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/vllm-omni-ci-npu:${BUILDKITE_COMMIT}-a3"
|
||||
plugins:
|
||||
- kubernetes:
|
||||
podSpecPatch:
|
||||
imagePullSecrets:
|
||||
- name: swr-secret
|
||||
env:
|
||||
VLLM_WORKER_MULTIPROC_METHOD: spawn
|
||||
VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
|
||||
HF_TOKEN: "${HF_TOKEN}"
|
||||
commands:
|
||||
- pytest -s -v tests/e2e/offline_inference/test_minicpmo_4_5.py tests/e2e/online_serving/test_minicpmo_4_5.py -m 'core_model and npu and A3 and omni' --run-level 'core_model'
|
||||
- label: "Omni · MiniCPM-o 4.5 Duplex Test"
|
||||
mirror_hardwares: a3_npu_2
|
||||
env:
|
||||
VLLM_WORKER_MULTIPROC_METHOD: spawn
|
||||
VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
|
||||
HF_TOKEN: "${HF_TOKEN}"
|
||||
commands:
|
||||
- pytest -s -v tests/e2e/online_serving/test_minicpmo_4_5_duplex.py -m 'core_model and npu and A3 and omni' --run-level 'core_model'
|
||||
- label: "Omni · MiniCPM-o 4.5 Perf Test"
|
||||
timeout_in_minutes: 60
|
||||
mirror_hardwares: a3_npu_2
|
||||
env:
|
||||
VLLM_WORKER_MULTIPROC_METHOD: spawn
|
||||
VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
|
||||
commands:
|
||||
- export BENCHMARK_DIR=tests/dfx/perf/results
|
||||
- pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_minicpmo_4_5_ready.json -m 'core_model and npu and A3 and omni'
|
||||
- pytest -s -v tests/e2e/offline_inference/test_minicpmo_4_5.py tests/e2e/online_serving/test_minicpmo_4_5.py -m 'core_model' --run-level 'core_model'
|
||||
|
||||
# - group: ":card_index_dividers: TTS Model Test"
|
||||
# key: ready-tts-group
|
||||
# depends_on: upload-ready-pipeline
|
||||
# steps:
|
||||
# - label: ":sound: TTS · Qwen3-TTS CustomVoice Test"
|
||||
# mirror_hardwares: a2b3_npu_1
|
||||
# env:
|
||||
# VLLM_WORKER_MULTIPROC_METHOD: spawn
|
||||
# VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
|
||||
# HF_TOKEN: "${HF_TOKEN}"
|
||||
# commands:
|
||||
# - pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py -m 'core_model' --run-level 'core_model'
|
||||
- group: ":card_index_dividers: TTS Model Test"
|
||||
key: ready-tts-group
|
||||
depends_on: upload-ready-pipeline
|
||||
steps:
|
||||
- label: ":sound: TTS · Qwen3-TTS CustomVoice Test"
|
||||
mirror_hardwares: a2b3_npu_1
|
||||
env:
|
||||
VLLM_WORKER_MULTIPROC_METHOD: spawn
|
||||
VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
|
||||
HF_TOKEN: "${HF_TOKEN}"
|
||||
commands:
|
||||
- pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py -m 'core_model and npu' --run-level 'core_model'
|
||||
@@ -1,7 +1,7 @@
|
||||
# Target vllm version: v0.25.0 (releases/v0.25.0 branch).
|
||||
# Target vllm version: v0.26.0 (releases/v0.26.0 branch).
|
||||
# Keep VLLM_BASE_TAG aligned with the target vLLM release on every rebase.
|
||||
ARG VLLM_BASE_IMAGE=vllm/vllm-openai
|
||||
ARG VLLM_BASE_TAG=v0.25.0
|
||||
ARG VLLM_BASE_TAG=v0.26.0
|
||||
|
||||
# We use registry-based layer caching for the dependencies,
|
||||
# where the tag is determined based on the hash of the contents
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
ARG BASE_IMAGE=vllm/vllm-openai:v0.25.0
|
||||
ARG BASE_IMAGE=vllm/vllm-openai:v0.26.0
|
||||
FROM ${BASE_IMAGE}
|
||||
|
||||
ARG COMMON_WORKDIR=/app
|
||||
|
||||
@@ -32,20 +32,18 @@ RUN source /usr/local/Ascend/ascend-toolkit/set_env.sh && \
|
||||
"pytest-cov>=4.0.0" \
|
||||
"pytest-mock>=3.10.0" \
|
||||
"openpyxl>=3.0.0" \
|
||||
"opencc==1.4.1" \
|
||||
"psutil>=7.2.0" \
|
||||
"soundfile>=0.13.1" \
|
||||
"pyttsx3>=2.99" \
|
||||
"opencv-python>=4.12.0.88" \
|
||||
"imageio[ffmpeg]>=0.6.0" \
|
||||
"jiwer>=3.0.0" \
|
||||
"zhon>=2.0.0" \
|
||||
"zhconv>=1.4.2" \
|
||||
"funasr>=1.0.0" \
|
||||
"hyperpyyaml>=1.2.0" \
|
||||
"onnx>=1.16.0" \
|
||||
"librosa>=0.10.0" \
|
||||
"jiwer>=4.0.0" \
|
||||
"zhon>=2.1.1" \
|
||||
"zhconv>=1.4.3" \
|
||||
"funasr>=1.3.0" \
|
||||
"datasets>=3.0.0" \
|
||||
--index-url http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple \
|
||||
--trusted-host cache-service.nginx-pypi-cache.svc.cluster.local \
|
||||
--default-timeout=60 && \
|
||||
|
||||
@@ -32,20 +32,18 @@ RUN source /usr/local/Ascend/ascend-toolkit/set_env.sh && \
|
||||
"pytest-cov>=4.0.0" \
|
||||
"pytest-mock>=3.10.0" \
|
||||
"openpyxl>=3.0.0" \
|
||||
"opencc==1.4.1" \
|
||||
"psutil>=7.2.0" \
|
||||
"soundfile>=0.13.1" \
|
||||
"pyttsx3>=2.99" \
|
||||
"opencv-python>=4.12.0.88" \
|
||||
"imageio[ffmpeg]>=0.6.0" \
|
||||
"jiwer>=3.0.0" \
|
||||
"zhon>=2.0.0" \
|
||||
"zhconv>=1.4.2" \
|
||||
"funasr>=1.0.0" \
|
||||
"hyperpyyaml>=1.2.0" \
|
||||
"onnx>=1.16.0" \
|
||||
"librosa>=0.10.0" \
|
||||
"jiwer>=4.0.0" \
|
||||
"zhon>=2.1.1" \
|
||||
"zhconv>=1.4.3" \
|
||||
"funasr>=1.3.0" \
|
||||
"datasets>=3.0.0" \
|
||||
--index-url http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple \
|
||||
--trusted-host cache-service.nginx-pypi-cache.svc.cluster.local \
|
||||
--default-timeout=60 && \
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
ARG BASE_IMAGE=vllm/vllm-openai-rocm:v0.25.0
|
||||
ARG BASE_IMAGE=vllm/vllm-openai-rocm:v0.26.0
|
||||
FROM ${BASE_IMAGE} AS base
|
||||
|
||||
# Declare a variable to know if we want to use the nightly build or the stable build.
|
||||
@@ -10,7 +10,7 @@ FROM ${BASE_IMAGE} AS base
|
||||
# we should swap over to use stable release ASAP.
|
||||
# We should avoid relying on custom commits.
|
||||
ARG USE_NIGHTLY_BUILD=0
|
||||
ARG VLLM_VERSION_OR_COMMIT_HASH=89138b21cc246ae944c741d5c399c148e2b770ab
|
||||
ARG VLLM_VERSION_OR_COMMIT_HASH=v0.26.0
|
||||
ARG ARG_PYTORCH_ROCM_ARCH
|
||||
ENV PYTORCH_ROCM_ARCH=${ARG_PYTORCH_ROCM_ARCH:-${PYTORCH_ROCM_ARCH}}
|
||||
|
||||
|
||||
@@ -88,7 +88,7 @@ ENV UV_EXTRA_INDEX_URL=${PIP_EXTRA_INDEX_URL}
|
||||
ENV UV_INDEX_STRATEGY="unsafe-best-match"
|
||||
ENV UV_LINK_MODE="copy"
|
||||
|
||||
ARG VLLM_VERSION=v0.25.0
|
||||
ARG VLLM_VERSION=v0.26.0
|
||||
RUN git clone -b ${VLLM_VERSION} https://github.com/vllm-project/vllm
|
||||
WORKDIR /workspace/vllm
|
||||
|
||||
|
||||
@@ -44,11 +44,9 @@ Pass **`--test-config-file`** to run one JSON file as-is, or omit it for the bul
|
||||
"ignore_eos": true,
|
||||
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"mean_ttft_ms": [500, 800],
|
||||
"mean_audio_ttfp_ms": [2000, 3500],
|
||||
"mean_audio_rtf": [0.25, 0.35]
|
||||
}
|
||||
"mean_ttft_ms": [500, 800],
|
||||
"mean_audio_ttfp_ms": [2000, 3500],
|
||||
"mean_audio_rtf": [0.25, 0.35]
|
||||
}
|
||||
}
|
||||
]
|
||||
@@ -144,6 +142,10 @@ pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py \
|
||||
--test-config-file tests/dfx/perf/tests/test_bagel_vllm_omni.json
|
||||
pytest -s -v tests/dfx/perf/scripts/run_benchmark.py \
|
||||
--test-config-file tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json
|
||||
|
||||
# Optional baseline assertion (default off)
|
||||
pytest -sv tests/dfx/perf/scripts/run_diffusion_benchmark.py --assert-baseline \
|
||||
--test-config-file tests/dfx/perf/tests/test_qwen_image_vllm_omni.json
|
||||
```
|
||||
|
||||
See also [Markers for Tests](#markers-for-tests) for registered hardware markers (`H100`, `L4`, `cuda`, `distributed_cuda`, …).
|
||||
@@ -200,4 +202,4 @@ You can add any benchmark running parameters you need here. For all optional par
|
||||
| num_prompts | int / array | Yes | 10,[10, 20, 30] | Number of requests. Supports single values or arrays. If a single value is used, it will be automatically expanded to match the number of qps or max_concurrency, e.g., [10,10,10]. If an array is used, its length must match the number of qps or max_concurrency. |
|
||||
| request_rate | float / array | No | 0.5, [0.5, 1, inf] | Queries per second. Supports single values or arrays. If a single value is used, it will be automatically expanded to match the number of num_prompts, e.g., [1,1,1]. If an array is used, its length must match the number of num_prompts. |
|
||||
| max_concurrency | int / array | No | 1, [1, 2, 3] | Maximum concurrent in-flight requests. Same array / expansion rules as `request_rate` (mutually exclusive with QPS mode). |
|
||||
| baseline | object | No | see below | Optional thresholds. **Hardware-nested only**: top-level keys must be `[hardware-resource]` pytest markers from `pyproject.toml` (`get_hardware_mark_list()`). Under each hardware key, metric names are free-form; values are scalars or sweep-aligned lists. Example: `{"H100": {"throughput_qps": […], "custom_stage_ms": 1.0}, "A3": {…}}`. Runtime filename aliases (`_RUNTIME_DEVICE_ALIASES`) are separate and do not expand this allowlist. |
|
||||
| baseline | object | No | see above | Optional per-metric thresholds; keys must match benchmark output fields. Scalar, list (per sweep step), or object (keyed by concurrency or QPS string).
|
||||
@@ -34,7 +34,7 @@ vLLM-Omni is a Python library that supports the following GPU variants. The libr
|
||||
|
||||
### Pre-built wheels
|
||||
|
||||
Note: Pre-built wheels are currently available for vLLM-Omni 0.11.0rc1, 0.12.0rc1, 0.14.0rc1, 0.14.0, 0.16.0, 0.18.0, 0.20.0, 0.21.0, 0.22.0, 0.23.0, 0.24.0, and 0.25.0. If you need a newer unreleased revision, please [build from source](https://docs.vllm.ai/projects/vllm-omni/en/latest/getting_started/installation/gpu/#build-wheel-from-source).
|
||||
Note: Pre-built wheels are currently available for vLLM-Omni 0.11.0rc1, 0.12.0rc1, 0.14.0rc1, 0.14.0, 0.16.0, 0.18.0, 0.20.0, 0.21.0, 0.22.0, 0.23.0, 0.24.0, 0.25.0, and 0.26.0. If you need a newer unreleased revision, please [build from source](https://docs.vllm.ai/projects/vllm-omni/en/latest/getting_started/installation/gpu/#build-wheel-from-source).
|
||||
|
||||
=== "NVIDIA CUDA"
|
||||
|
||||
|
||||
@@ -20,7 +20,7 @@ Therefore, it is recommended to install vLLM and vLLM-Omni with a **fresh new**
|
||||
|
||||
vLLM-Omni is built based on vLLM. Please install it with command below.
|
||||
```bash
|
||||
uv pip install vllm==0.25.0 --torch-backend=auto
|
||||
uv pip install vllm==0.26.0 --torch-backend=auto
|
||||
```
|
||||
|
||||
#### Installation of vLLM-Omni
|
||||
@@ -39,13 +39,13 @@ uv pip install 'vllm-omni[demo]'
|
||||
# --8<-- [start:build-wheel-from-source]
|
||||
|
||||
#### Installation of vLLM
|
||||
If you do not need to modify source code of vLLM, you can directly install the stable 0.25.0 release version of the library
|
||||
If you do not need to modify source code of vLLM, you can directly install the stable 0.26.0 release version of the library
|
||||
|
||||
```bash
|
||||
uv pip install vllm==0.25.0 --torch-backend=auto
|
||||
uv pip install vllm==0.26.0 --torch-backend=auto
|
||||
```
|
||||
|
||||
The 0.25.0 release of vLLM ships CUDA 13.0-compatible binaries by default. If you need a different CUDA variant or want to reuse an existing PyTorch installation, build vLLM from source instead.
|
||||
The 0.26.0 release of vLLM ships CUDA 13.0-compatible binaries by default. If you need a different CUDA variant or want to reuse an existing PyTorch installation, build vLLM from source instead.
|
||||
|
||||
#### Installation of vLLM-Omni
|
||||
Since vllm-omni is rapidly evolving, it's recommended to install it from source
|
||||
@@ -66,12 +66,12 @@ If you want to check, modify or debug with source code of vLLM, install the libr
|
||||
```bash
|
||||
git clone https://github.com/vllm-project/vllm.git
|
||||
cd vllm
|
||||
git checkout v0.25.0
|
||||
git checkout v0.26.0
|
||||
```
|
||||
Set up environment variables to get pre-built wheels. If there are internet problems, just download the whl file manually. And set `VLLM_PRECOMPILED_WHEEL_LOCATION` as your local absolute path of whl file.
|
||||
```bash
|
||||
#For CUDA 13.0 (the default for v0.25.0; the wheel filename has no `+cu130` suffix)
|
||||
export VLLM_PRECOMPILED_WHEEL_LOCATION=https://github.com/vllm-project/vllm/releases/download/v0.25.0/vllm-0.25.0-cp38-abi3-manylinux_2_28_x86_64.whl
|
||||
#For CUDA 13.0 (the default for v0.26.0; the wheel filename has no `+cu130` suffix)
|
||||
export VLLM_PRECOMPILED_WHEEL_LOCATION=https://github.com/vllm-project/vllm/releases/download/v0.26.0/vllm-0.26.0-cp38-abi3-manylinux_2_28_x86_64.whl
|
||||
```
|
||||
Install vllm with command below (If you have no existing PyTorch).
|
||||
```bash
|
||||
@@ -102,7 +102,7 @@ docker run --runtime nvidia --gpus 2 \
|
||||
--env "HF_TOKEN=$HF_TOKEN" \
|
||||
-p 8091:8091 \
|
||||
--ipc=host \
|
||||
vllm/vllm-omni:v0.25.0 \
|
||||
vllm/vllm-omni:v0.26.0 \
|
||||
vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091
|
||||
```
|
||||
|
||||
|
||||
@@ -13,7 +13,7 @@ vLLM-Omni current recommends the steps in under setup through Docker Images.
|
||||
|
||||
vLLM-Omni is built based on vLLM. Please install it with command below.
|
||||
```bash
|
||||
uv pip install vllm==0.25.0+rocm723 --extra-index-url https://wheels.vllm.ai/rocm/0.25.0/rocm723
|
||||
uv pip install vllm==0.26.0+rocm723 --extra-index-url https://wheels.vllm.ai/rocm/0.26.0/rocm723
|
||||
```
|
||||
|
||||
#### Installation of vLLM-Omni
|
||||
@@ -34,13 +34,13 @@ uv pip install onnxruntime-rocm
|
||||
# --8<-- [start:build-wheel-from-source]
|
||||
|
||||
#### Installation of vLLM
|
||||
If you do not need to modify source code of vLLM, you can directly install the stable 0.25.0 release version of the library
|
||||
If you do not need to modify source code of vLLM, you can directly install the stable 0.26.0 release version of the library
|
||||
|
||||
```bash
|
||||
uv pip install vllm==0.25.0+rocm723 --extra-index-url https://wheels.vllm.ai/rocm/0.25.0/rocm723
|
||||
uv pip install vllm==0.26.0+rocm723 --extra-index-url https://wheels.vllm.ai/rocm/0.26.0/rocm723
|
||||
```
|
||||
|
||||
The pre-built 0.25.0 vLLM wheel targets ROCm 7.2.3. If you need a different ROCm stack or want to reuse an existing PyTorch installation, build vLLM from source instead.
|
||||
The pre-built 0.26.0 vLLM wheel targets ROCm 7.2.3. If you need a different ROCm stack or want to reuse an existing PyTorch installation, build vLLM from source instead.
|
||||
|
||||
#### Installation of vLLM-Omni
|
||||
Since vllm-omni is rapidly evolving, it's recommended to install it from source
|
||||
@@ -58,7 +58,7 @@ If you want to check, modify or debug with source code of vLLM, install the libr
|
||||
```bash
|
||||
git clone https://github.com/vllm-project/vllm.git
|
||||
cd vllm
|
||||
git checkout v0.25.0
|
||||
git checkout v0.26.0
|
||||
python3 -m pip install -r requirements/rocm.txt
|
||||
python3 setup.py develop
|
||||
```
|
||||
@@ -130,7 +130,7 @@ docker run --rm \
|
||||
-v ~/.cache/huggingface:/root/.cache/huggingface \
|
||||
--env "HF_TOKEN=$HF_TOKEN" \
|
||||
-p 8091:8091 \
|
||||
vllm/vllm-omni-rocm:v0.25.0 \
|
||||
vllm/vllm-omni-rocm:v0.26.0 \
|
||||
--model Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091
|
||||
```
|
||||
|
||||
@@ -149,7 +149,7 @@ docker run --rm -it \
|
||||
-v ~/.cache/huggingface:/root/.cache/huggingface \
|
||||
--env "HF_TOKEN=$HF_TOKEN" \
|
||||
--entrypoint bash \
|
||||
vllm/vllm-omni-rocm:v0.25.0
|
||||
vllm/vllm-omni-rocm:v0.26.0
|
||||
```
|
||||
|
||||
# --8<-- [end:pre-built-images]
|
||||
@@ -19,10 +19,10 @@ uv venv --python 3.12 --seed
|
||||
source .venv/bin/activate
|
||||
|
||||
# On CUDA
|
||||
uv pip install vllm==0.25.0 --torch-backend=auto
|
||||
uv pip install vllm==0.26.0 --torch-backend=auto
|
||||
|
||||
# On ROCm
|
||||
uv pip install vllm==0.25.0+rocm723 --extra-index-url https://wheels.vllm.ai/rocm/0.25.0/rocm723
|
||||
uv pip install vllm==0.26.0+rocm723 --extra-index-url https://wheels.vllm.ai/rocm/0.26.0/rocm723
|
||||
|
||||
git clone https://github.com/vllm-project/vllm-omni.git
|
||||
cd vllm-omni
|
||||
@@ -34,7 +34,7 @@ For additional installation methods — please see the [installation guide](inst
|
||||
!!! note
|
||||
It is important to install the same major & minor version of vLLM and vLLM Omni, otherwise things may not work as expected. If the versions are misaligned, you will see a warning when you import vLLM Omni.
|
||||
|
||||
If you are seeing strange behavior with the `vllm` command not handling the `--omni` flag correctly, you most likely have a version mismatch with vLLM < `0.25.0` and vLLM Omni `0.25.0`, as vLLM Omni no longer hijacks the vLLM entrypoint. Updating vLLM should resolve this issue.
|
||||
If you are seeing strange behavior with the `vllm` command not handling the `--omni` flag correctly, you most likely have a version mismatch with vLLM < `0.26.0` and vLLM Omni `0.26.0`, as vLLM Omni no longer hijacks the vLLM entrypoint. Updating vLLM should resolve this issue.
|
||||
|
||||
## Offline Inference
|
||||
|
||||
|
||||
@@ -41,12 +41,8 @@ th {
|
||||
| `Wan22S2VPipeline` | Wan2.2-S2V | `Wan-AI/Wan2.2-S2V-14B` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
|
||||
| `Wan22VACEPipeline` | Wan2.1-VACE | `Wan-AI/Wan2.1-VACE-1.3B-diffusers`, `Wan-AI/Wan2.1-VACE-14B-diffusers` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
|
||||
| `Wan22VACEPipeline` | Wan2.2-VACE | `Pyros13/Wan2.2-VACE-Fun-A14B-Diffusers` | ✅︎ | | | |
|
||||
| `LTX2Pipeline` | LTX-2-T2V | `Lightricks/LTX-2` | ✅︎ | ✅︎ | | |
|
||||
| `LTX2ImageToVideoPipeline` | LTX-2-I2V | `Lightricks/LTX-2` | ✅︎ | ✅︎ | | |
|
||||
| `LTX2TwoStagesPipeline` | LTX-2-T2V | `rootonchair/LTX-2-19b-distilled` | ✅︎ | ✅︎ | | |
|
||||
| `LTX2ImageToVideoTwoStagesPipeline` | LTX-2-I2V | `rootonchair/LTX-2-19b-distilled` | ✅︎ | ✅︎ | | |
|
||||
| `LTX23Pipeline` | LTX-2.3-T2V | `diffusers/LTX-2.3-Diffusers` | ✅︎ | ✅︎ | | |
|
||||
| `LTX23ImageToVideoPipeline` | LTX-2.3-I2V | `diffusers/LTX-2.3-Diffusers` | ✅︎ | ✅︎ | | |
|
||||
| `LTX2Pipeline` | LTX-2 / LTX-2.3 one-stage T2V and I2V | `Lightricks/LTX-2`, `diffusers/LTX-2.3-Diffusers` | ✅︎ | ✅︎ | | |
|
||||
| `LTX2DistilledPipeline` | LTX-2 distilled two-stage T2V and I2V | `rootonchair/LTX-2-19b-distilled` | ✅︎ | ✅︎ | | |
|
||||
| `LingBotVideoPipeline` | LingBot-Video dense and MoE T2V | `robbyant/lingbot-video-dense-1.3b`, `robbyant/lingbot-video-moe-30b-a3b` | ✅︎ | | | |
|
||||
| `DreamZeroPipeline` | DreamZero-DROID | `GEAR-Dreams/DreamZero-DROID` | ✅︎ | ✅︎ | ✅︎ | ✅︎ |
|
||||
| `HeliosPipeline`, `HeliosPyramidPipeline` | Helios | `BestWishYsh/Helios-Base`, `BestWishYsh/Helios-Mid`, `BestWishYsh/Helios-Distilled` | ✅︎ | ✅︎ | ✅︎ | |
|
||||
|
||||
@@ -73,7 +73,7 @@ curl -L "http://localhost:8091/v1/videos/${video_id}/content" -o output.mp4
|
||||
| `audio_reference` | string | null | JSON-encoded audio reference for speech-to-video: `{"audio_url": "..."}` — supports HTTP(s) URLs or base64 data URLs |
|
||||
| `width` | integer | model default | Output video width |
|
||||
| `height` | integer | model default | Output video height |
|
||||
| `num_frames` | integer | model default | Number of generated frames |
|
||||
| `num_frames` | integer | 1 | Number of generated frames |
|
||||
| `fps` | integer | model default | Output frames per second |
|
||||
| `num_inference_steps` | integer | model default | Number of diffusion steps |
|
||||
| `guidance_scale` | number | null | CFG guidance scale for the low-noise stage |
|
||||
|
||||
@@ -4,7 +4,7 @@ Source <https://github.com/vllm-project/vllm-omni/tree/main/examples/offline_inf
|
||||
|
||||
|
||||
This example demonstrates how to generate videos from images using Wan2.2,
|
||||
LTX-2/LTX-2.3, and HunyuanVideo-1.5 Image-to-Video models with vLLM-Omni's
|
||||
LTX-2, and HunyuanVideo-1.5 Image-to-Video models with vLLM-Omni's
|
||||
offline inference API.
|
||||
|
||||
## Local CLI Usage
|
||||
@@ -53,34 +53,22 @@ python image_to_video.py \
|
||||
--output i2v_output.mp4
|
||||
```
|
||||
|
||||
### LTX-2.3
|
||||
### LTX-2
|
||||
|
||||
```bash
|
||||
python image_to_video.py \
|
||||
--model diffusers/LTX-2.3-Diffusers \
|
||||
--model-class-name LTX23ImageToVideoPipeline \
|
||||
--model Lightricks/LTX-2 \
|
||||
--image cherry_blossom.jpg \
|
||||
--prompt "Cherry blossoms swaying gently in the breeze with synchronized ambient sound" \
|
||||
--negative-prompt "worst quality, inconsistent motion, blurry, jittery, distorted" \
|
||||
--height 384 \
|
||||
--width 512 \
|
||||
--num-frames 25 \
|
||||
--guidance-scale 4.0 \
|
||||
--num-inference-steps 20 \
|
||||
--frame-rate 24 \
|
||||
--fps 24 \
|
||||
--output ltx23_i2v_output.mp4
|
||||
--output ltx2_i2v_output.mp4
|
||||
```
|
||||
|
||||
Use the Diffusers-format checkpoint `diffusers/LTX-2.3-Diffusers`; the
|
||||
upstream `Lightricks/LTX-2.3` raw safetensors repo is not directly loadable by
|
||||
this pipeline. Pass `--model-class-name LTX23ImageToVideoPipeline` to select
|
||||
the LTX-2.3 image-to-video pipeline.
|
||||
See the [LTX-2 recipe](../../../../recipes/LTX/LTX-2.md) for all checkpoints,
|
||||
pipeline selection, T2V, defaults, and advanced options.
|
||||
|
||||
Key arguments:
|
||||
|
||||
- `--model`: Model ID (I2V-A14B for MoE, TI2V-5B for unified T2V+I2V, or
|
||||
LTX-2/LTX-2.3).
|
||||
- `--model`: Model ID (I2V-A14B for MoE, TI2V-5B for unified T2V+I2V, or LTX-2).
|
||||
- `--image`: Path to input image (required).
|
||||
- `--prompt`: Text description of desired motion/animation.
|
||||
- `--height/--width`: Output resolution (auto-calculated from image if not set).
|
||||
|
||||
@@ -12,7 +12,7 @@ For backend selection and SageAttention usage, see the [Diffusion Attention Back
|
||||
| Model | Default Resolution | Default Frames | Default Steps | Guidance | VRAM (BF16) |
|
||||
|---|---|---|---|---|---|
|
||||
| `Wan-AI/Wan2.2-T2V-A14B-Diffusers` | 720x1280 | 81 | 40 | 4.0 | ~60 GiB |
|
||||
| `diffusers/LTX-2.3-Diffusers` | 384x512 | 25 | 20 | 4.0 | 96GB-class GPU |
|
||||
| `Lightricks/LTX-2` | 512x768 | 121 | 40 | video 3.0 / audio 7.0 | Model-dependent |
|
||||
| `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v` | 480x832 | 121 | 50 | 6.0 | 1×A100 80GB |
|
||||
| `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-720p_t2v` | 720x1280 | 121 | 50 | 6.0 | FP8 + VAE tiling required |
|
||||
|
||||
@@ -35,45 +35,17 @@ python text_to_video.py \
|
||||
--output t2v_out.mp4
|
||||
```
|
||||
|
||||
LTX2 example:
|
||||
### LTX-2
|
||||
|
||||
```bash
|
||||
python text_to_video.py \
|
||||
--model "Lightricks/LTX-2" \
|
||||
--prompt "A cinematic close-up of ocean waves at golden hour." \
|
||||
--negative-prompt "worst quality, inconsistent motion, blurry, jittery, distorted" \
|
||||
--height 512 \
|
||||
--width 768 \
|
||||
--num-frames 121 \
|
||||
--num-inference-steps 40 \
|
||||
--guidance-scale 4.0 \
|
||||
--frame-rate 24 \
|
||||
--output ltx2_out.mp4
|
||||
```
|
||||
|
||||
### LTX-2.3
|
||||
|
||||
```bash
|
||||
python text_to_video.py \
|
||||
--model diffusers/LTX-2.3-Diffusers \
|
||||
--model-class-name LTX23Pipeline \
|
||||
--model Lightricks/LTX-2 \
|
||||
--prompt "Cherry blossoms swaying gently in the breeze with synchronized ambient sound" \
|
||||
--negative-prompt "worst quality, inconsistent motion, blurry, jittery, distorted" \
|
||||
--height 384 \
|
||||
--width 512 \
|
||||
--num-frames 25 \
|
||||
--num-inference-steps 20 \
|
||||
--guidance-scale 4.0 \
|
||||
--frame-rate 24 \
|
||||
--fps 24 \
|
||||
--audio-sample-rate 48000 \
|
||||
--output ltx23_t2v_output.mp4
|
||||
--output ltx2_output.mp4
|
||||
```
|
||||
|
||||
Use the Diffusers-format checkpoint `diffusers/LTX-2.3-Diffusers`; the
|
||||
upstream `Lightricks/LTX-2.3` raw safetensors repo is not directly loadable by
|
||||
this pipeline. Pass `--model-class-name LTX23Pipeline` to select the LTX-2.3
|
||||
text-to-video pipeline explicitly.
|
||||
See the [LTX-2 recipe](../../../../recipes/LTX/LTX-2.md) for all checkpoints,
|
||||
pipeline selection, I2V, defaults, and advanced options.
|
||||
|
||||
### HunyuanVideo-1.5 (480p)
|
||||
|
||||
@@ -135,8 +107,7 @@ python text_to_video.py \
|
||||
### Common
|
||||
|
||||
- `--model`: Diffusers model ID or local path.
|
||||
- `--model-class-name`: Optional explicit pipeline class. Use `LTX23Pipeline`
|
||||
for LTX-2.3 text-to-video.
|
||||
- `--model-class-name`: Optional explicit pipeline override.
|
||||
- `--prompt`: text description (string).
|
||||
- `--height/--width`: output resolution. Default depends on model.
|
||||
- `--num-frames`: number of frames. Default depends on model.
|
||||
@@ -151,8 +122,7 @@ python text_to_video.py \
|
||||
- `--enable-cpu-offload`: enable CPU offloading for diffusion models.
|
||||
- `--enable-layerwise-offload`: enable layerwise offloading on DiT modules.
|
||||
- `--frame-rate`: generation FPS for pipelines that require it (e.g., LTX2).
|
||||
- `--audio-sample-rate`: audio sample rate for embedded audio (when the
|
||||
pipeline returns audio; LTX-2.3 outputs 48kHz audio).
|
||||
- `--audio-sample-rate`: fallback audio sample rate when the pipeline returns audio.
|
||||
- `--quantization`: quantization method (`fp8` for FP8, `gguf` for GGUF).
|
||||
- `--flow-shift`: scheduler flow_shift parameter.
|
||||
|
||||
|
||||
@@ -13,7 +13,6 @@ This example demonstrates how to deploy text-to-video models for online video ge
|
||||
| Wan2.1 T2V (14B) | `Wan-AI/Wan2.1-T2V-14B-Diffusers` |
|
||||
| Wan2.2 T2V | `Wan-AI/Wan2.2-T2V-A14B-Diffusers` |
|
||||
| LTX-2 | `Lightricks/LTX-2` |
|
||||
| LTX-2.3 | `diffusers/LTX-2.3-Diffusers` |
|
||||
|
||||
## Wan2.2 T2V
|
||||
|
||||
@@ -192,7 +191,7 @@ curl -X POST http://localhost:8091/v1/videos \
|
||||
| `negative_prompt` | str | None | Negative prompt |
|
||||
| `width` | int | None | Video width in pixels |
|
||||
| `height` | int | None | Video height in pixels |
|
||||
| `num_frames` | int | None | Number of frames to generate |
|
||||
| `num_frames` | int | 1 | Number of frames to generate; set explicitly for video generation |
|
||||
| `fps` | int | None | Frames per second for output video |
|
||||
| `num_inference_steps` | int | None | Number of denoising steps |
|
||||
| `guidance_scale` | float | None | CFG guidance scale (low-noise stage) |
|
||||
@@ -290,101 +289,12 @@ done
|
||||
|
||||
## LTX-2
|
||||
|
||||
### Start Server
|
||||
|
||||
#### Basic Start
|
||||
|
||||
```bash
|
||||
vllm serve Lightricks/LTX-2 --omni --port 8098 \
|
||||
--enforce-eager --boundary-ratio 1.0
|
||||
vllm serve Lightricks/LTX-2 --omni --port 8098
|
||||
```
|
||||
|
||||
For multi-GPU memory reduction, you can enable HSDP:
|
||||
|
||||
```bash
|
||||
vllm serve Lightricks/LTX-2 --omni --port 8098 \
|
||||
--enforce-eager --boundary-ratio 1.0 \
|
||||
--use-hsdp --hsdp-shard-size 2
|
||||
```
|
||||
|
||||
#### Start with Optimization Presets
|
||||
|
||||
Use the LTX-2 startup script with built-in optimization presets:
|
||||
|
||||
```bash
|
||||
# Baseline (1 GPU, eager)
|
||||
bash run_server_ltx2.sh baseline
|
||||
|
||||
# 4-GPU Ulysses sequence parallelism (lossless)
|
||||
bash run_server_ltx2.sh ulysses4
|
||||
|
||||
# Cache-DiT lossy acceleration (1 GPU, ~1.4× speedup)
|
||||
bash run_server_ltx2.sh cache-dit
|
||||
|
||||
# Best combo: 4-GPU Ulysses SP + Cache-DiT (~2.2× speedup)
|
||||
bash run_server_ltx2.sh best-combo
|
||||
```
|
||||
|
||||
#### Optimization Benchmarks
|
||||
|
||||
Benchmarked on H800, online serving (480×768, 41 frames, 20 steps, `seed=42`).
|
||||
"Inference" is the server-reported inference time; excludes HTTP/poll overhead.
|
||||
|
||||
| Preset | Server Command | Inference (s) | Speedup | Type |
|
||||
|--------|---------------|---------------|---------|------|
|
||||
| `baseline` | `--enforce-eager` | 10.3 | 1.00× | — |
|
||||
| `compile` | *(default, no --enforce-eager)* | ~10.3 (warm) | ~1.00× | Lossless |
|
||||
| `ulysses4` | `--enforce-eager --usp 4` | ~10.3 | ~1.00× | Lossless |
|
||||
| `cache-dit` | `--enforce-eager --cache-backend cache_dit` | 7.4 avg | ~1.4× | Lossy |
|
||||
| `best-combo` | `--enforce-eager --usp 4 --cache-backend cache_dit` | 4.7 avg | **~2.2×** | Lossless + Lossy |
|
||||
|
||||
**Observations**:
|
||||
- **torch.compile**: On H800, warm-request inference time matches the eager baseline (~10.3s).
|
||||
The first request pays ~6s compilation overhead. Benefit depends on model architecture and GPU.
|
||||
- **Ulysses SP (4 GPU)**: No measurable speedup alone for 41-frame generation at this resolution.
|
||||
Communication overhead outweighs gains at this sequence length.
|
||||
- **Cache-DiT**: Inference varies per request (6–10s) due to dynamic caching decisions.
|
||||
Average is ~7.4s (~1.4× speedup) with slight quality tradeoff.
|
||||
- **Best combo**: 4-GPU Ulysses SP + Cache-DiT synergize well — Cache-DiT reduces per-step
|
||||
computation, making the communication overhead of Ulysses SP worthwhile. Average ~4.7s
|
||||
(~2.2× speedup).
|
||||
- **FP8 quantization**: Reduces VRAM but does not speed up LTX-2 on H800 (compute-bound).
|
||||
|
||||
**Deployment Recommendations**:
|
||||
- For **production with quality priority**: use `baseline` with `--enforce-eager`
|
||||
- For **maximum throughput** (4 GPUs, quality tradeoff): use `best-combo` (~2.2× speedup)
|
||||
- For **single-GPU throughput**: use `cache-dit` (~1.4× speedup)
|
||||
- `--enforce-eager` is recommended to avoid torch.compile warmup latency on first request
|
||||
|
||||
### Send Requests (curl)
|
||||
|
||||
```bash
|
||||
# Using the provided script
|
||||
bash run_curl_ltx2.sh
|
||||
|
||||
# Or directly
|
||||
curl -sS -X POST http://localhost:8098/v1/videos \
|
||||
-H "Accept: application/json" \
|
||||
-F "prompt=A serene lakeside sunrise with mist over the water." \
|
||||
-F "width=768" \
|
||||
-F "height=480" \
|
||||
-F "num_frames=41" \
|
||||
-F "fps=24" \
|
||||
-F "num_inference_steps=20" \
|
||||
-F "guidance_scale=3.0" \
|
||||
-F "seed=42"
|
||||
```
|
||||
|
||||
## LTX-2.3
|
||||
|
||||
LTX-2.3 uses a Diffusers-format checkpoint and a dedicated pipeline class:
|
||||
|
||||
```bash
|
||||
vllm serve diffusers/LTX-2.3-Diffusers --omni --port 8098 \
|
||||
--model-class-name LTX23Pipeline
|
||||
```
|
||||
|
||||
Send requests through the same `/v1/videos` API shown in the LTX-2 example.
|
||||
See the [LTX-2 recipe](../../../../recipes/LTX/LTX-2.md) for all checkpoints,
|
||||
pipeline selection, requests, defaults, and advanced options.
|
||||
|
||||
## Example materials
|
||||
|
||||
|
||||
@@ -0,0 +1,114 @@
|
||||
# Audex (Nemotron-Labs-Audex-2B) offline inference
|
||||
|
||||
One checkpoint, four tasks — each script builds its own `Omni` engine with
|
||||
the right deploy yaml, so they run as plain `python <script>` with no extra
|
||||
flags. Pass the HF repo ROOT as `--model` (default
|
||||
`nvidia/Nemotron-Labs-Audex-2B`); per-stage subfolders and download subsets
|
||||
resolve automatically.
|
||||
|
||||
| script | pipeline (`vllm_omni/deploy/<name>.yaml`) | audio in | text out | speech out | general audio out |
|
||||
|---|---|---|---|---|---|
|
||||
| `text_to_speech.py` | `audex_tts` | ❌ | ❌ | ✅ | ❌ |
|
||||
| `text_to_audio.py` | `audex_tta` | ❌ | ❌ | ❌ | ✅ |
|
||||
| `audio_qa.py` | `audex_thinker_only` | ✅ | ✅ | ❌ | ❌ |
|
||||
| `speech_to_speech.py` | `audex_s2s` | ✅ | ✅ | ✅ | ❌ |
|
||||
|
||||
Shared plumbing (corpus/WAV IO, tokenizer loading, the CFG cond/uncond
|
||||
pair contract) lives in `common.py`; each task script keeps only its own
|
||||
prompt recipe and flow. Note `text_to_audio.py` here drives the Audex
|
||||
two-stage AR pipeline — for diffusion text-to-audio models (Stable Audio
|
||||
Open, AudioX) use the generic
|
||||
`examples/offline_inference/text_to_audio/text_to_audio.py` instead.
|
||||
|
||||
## text_to_speech.py — text → speech
|
||||
|
||||
Thinker generates `<speechcodec_N>` tokens; the streaming causal decoder
|
||||
turns them into 16 kHz WAVs (one file per prompt, slugified filename).
|
||||
|
||||
```bash
|
||||
# three built-in demo sentences -> results/audex_wavs/
|
||||
python examples/offline_inference/audex/text_to_speech.py
|
||||
|
||||
# your own texts, with classifier-free guidance (official quality setting)
|
||||
python examples/offline_inference/audex/text_to_speech.py \
|
||||
--texts "Hello there." "Nice to meet you." --cfg-scale 1.5 \
|
||||
--output-dir results/my_tts
|
||||
```
|
||||
|
||||
Key flags: `--texts` / `--texts-file` (TSV `utt_id<TAB>text`),
|
||||
`--cfg-scale` (default 1.0 = off; 1.5 recommended — guided requests run
|
||||
one at a time, each paired with a length-matched null prompt),
|
||||
`--temperature` (overrides the deploy yaml's stage-0 default).
|
||||
|
||||
## text_to_audio.py — caption → general audio (sound effects)
|
||||
|
||||
Same thinker over the interleaved 4-codebook `<audiocodec_N>` RVQ block,
|
||||
decoded by the external XCodec1 checkpoint. CFG is effectively mandatory
|
||||
(default scale 3.0). Clips are capped at `--codec-cap 4000` codec tokens
|
||||
(10 s).
|
||||
|
||||
```bash
|
||||
# three built-in demo captions -> results/audex_tta_wavs/
|
||||
python examples/offline_inference/audex/text_to_audio.py
|
||||
|
||||
python examples/offline_inference/audex/text_to_audio.py \
|
||||
--captions "Thunder rolling across a valley." --output-dir results/my_tta
|
||||
```
|
||||
|
||||
XCodec1 resolves from `--xcodec1-path`, the `XCODEC1_PATH` env var, or the
|
||||
default `hf-audio/xcodec-hubert-general-balanced` repo (downloaded on
|
||||
first use).
|
||||
|
||||
## audio_qa.py — audio (+ instruction) → text
|
||||
|
||||
Single-stage audio understanding on the full checkpoint (NV-Whisper
|
||||
encoder + projector + LM). The default question is ASR
|
||||
("Transcribe the input speech."); without `--audio-files`, vLLM's public
|
||||
`mary_had_lamb` asset is transcribed.
|
||||
|
||||
```bash
|
||||
# transcribe the built-in demo asset
|
||||
python examples/offline_inference/audex/audio_qa.py
|
||||
|
||||
# your own clips / free-form QA
|
||||
python examples/offline_inference/audex/audio_qa.py \
|
||||
--audio-files a.wav b.wav --question "What language is being spoken?"
|
||||
```
|
||||
|
||||
## speech_to_speech.py — spoken question → spoken answer
|
||||
|
||||
The official three-pass cascade over ONE `audex_s2s` deployment: ASR
|
||||
(audio → transcript, text-final) → chat (transcript → answer, text-final)
|
||||
→ TTS (answer → speech through the streaming decoder). Only the TTS pass
|
||||
touches the codec path. Without `--audio-file`, the `mary_had_lamb` asset
|
||||
is used as the spoken input.
|
||||
|
||||
```bash
|
||||
# full cascade on the built-in demo asset -> results/audex_s2s_answer.wav
|
||||
python examples/offline_inference/audex/speech_to_speech.py
|
||||
|
||||
python examples/offline_inference/audex/speech_to_speech.py \
|
||||
--audio-file question.wav --output results/answer.wav --cfg-scale 1.5
|
||||
```
|
||||
|
||||
`--cfg-scale` (default 1.5) applies to the TTS pass only; 1.0 disables it.
|
||||
|
||||
Online-serving counterparts (server + HTTP clients) live in
|
||||
`examples/online_serving/audex/`.
|
||||
|
||||
## 30B-A3B (nvidia/Nemotron-Labs-Audex-30B-A3B)
|
||||
|
||||
The same four scripts serve the 30B MoE checkpoint (hybrid Mamba + MoE
|
||||
NemotronH thinker; identical token space and decoder). The 30B REQUIRES an
|
||||
explicit 30B deploy yaml — the model root's default resolution lands on the
|
||||
2B-tuned configs:
|
||||
|
||||
python examples/offline_inference/audex/text_to_speech.py \
|
||||
--model nvidia/Nemotron-Labs-Audex-30B-A3B \
|
||||
--deploy-config vllm_omni/deploy/audex_tts_30b.yaml
|
||||
|
||||
Per-mode yamls: `audex_{tts,tta,thinker_only,s2s}_30b.yaml`. Defaults fit a
|
||||
single H100 80 GB (thinker + decoder share the card; prefix caching is off —
|
||||
hybrid Mamba); if long sequences OOM, set `tensor_parallel_size: 2` on
|
||||
stage 0. First run downloads ~60 GB. Verified on a single H100 80 GB: ~61 GiB
|
||||
weights + healthy KV headroom; TP2 was not needed.
|
||||
@@ -0,0 +1,93 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Offline Audex (Nemotron-Labs-Audex-2B) audio understanding example.
|
||||
|
||||
Speech (or general audio) + text instruction → text, through the
|
||||
single-stage ``audex_thinker_only`` pipeline: NV-Whisper
|
||||
encoder + relu2 projector + 2B dense LM on ``checkpoint_folder_full``.
|
||||
|
||||
The prompt format mirrors the official audioqa script: a ChatML user turn
|
||||
holding the ``<so_embedding>`` placeholder (expanded by the processor to
|
||||
750 embedding positions per 30 s clip) plus the instruction, with a closed
|
||||
``<think></think>`` priming.
|
||||
|
||||
Examples (without --audio-files, vLLM's public ``mary_had_lamb`` asset is
|
||||
transcribed):
|
||||
|
||||
# Transcribe WAVs (ASR):
|
||||
python examples/offline_inference/audex/audio_qa.py \\
|
||||
--audio-files a.wav b.wav
|
||||
|
||||
# Free-form audio QA:
|
||||
python examples/offline_inference/audex/audio_qa.py \\
|
||||
--audio-files a.wav --question "What language is being spoken?"
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
|
||||
from common import default_deploy_config, load_audio_file, request_index
|
||||
from vllm.assets.audio import AudioAsset
|
||||
|
||||
from vllm_omni import Omni
|
||||
|
||||
ASR_QUESTION = "Transcribe the input speech."
|
||||
|
||||
|
||||
def build_prompt(question: str) -> str:
|
||||
return f"<|im_start|>user\n<so_embedding>\n{question}<|im_end|>\n<|im_start|>assistant\n<think></think>"
|
||||
|
||||
|
||||
def parse_args():
|
||||
parser = argparse.ArgumentParser(description="Offline Audex audio understanding")
|
||||
parser.add_argument("--model", type=str, default="nvidia/Nemotron-Labs-Audex-2B")
|
||||
parser.add_argument(
|
||||
"--audio-files",
|
||||
type=str,
|
||||
nargs="+",
|
||||
default=None,
|
||||
help="Input clips. Defaults to vLLM's mary_had_lamb audio asset.",
|
||||
)
|
||||
parser.add_argument("--question", type=str, default=ASR_QUESTION)
|
||||
parser.add_argument(
|
||||
"--deploy-config",
|
||||
type=str,
|
||||
# The model root's default deploy yaml is the TTS pipeline; audio
|
||||
# understanding needs the single-stage thinker-only pipeline.
|
||||
default=default_deploy_config("audex_thinker_only.yaml"),
|
||||
help="Deploy yaml (defaults to the audex_thinker_only pipeline).",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def main():
|
||||
args = parse_args()
|
||||
engine = Omni(model=args.model, deploy_config=args.deploy_config, trust_remote_code=True)
|
||||
|
||||
prompt_text = build_prompt(args.question)
|
||||
if args.audio_files:
|
||||
clips = [load_audio_file(path) for path in args.audio_files]
|
||||
else:
|
||||
clips = [AudioAsset("mary_had_lamb").audio_and_sample_rate]
|
||||
prompts = []
|
||||
for audio, sr in clips:
|
||||
prompts.append(
|
||||
{
|
||||
"prompt": prompt_text,
|
||||
"multi_modal_data": {"audio": (audio, sr)},
|
||||
}
|
||||
)
|
||||
|
||||
outputs = engine.generate(prompts)
|
||||
|
||||
ordered = sorted(outputs, key=request_index)
|
||||
names = [Path(p).name for p in args.audio_files] if args.audio_files else ["mary_had_lamb"]
|
||||
for name, req_output in zip(names, ordered):
|
||||
text = req_output.outputs[0].text if req_output.outputs else ""
|
||||
print(f"{name}: {text.strip()}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,149 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Shared helpers for the offline Audex examples in this directory.
|
||||
|
||||
Import style follows the repo's example convention (same-directory module,
|
||||
see ``text_to_speech/qwen3_tts/tts_common.py``): scripts run as files, so
|
||||
``from common import ...`` resolves via the script's own directory.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import re
|
||||
from collections.abc import Callable
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
import soundfile as sf
|
||||
import torch
|
||||
|
||||
from vllm_omni import Omni
|
||||
from vllm_omni.model_executor.models.audex.prompt import build_null_prompt
|
||||
|
||||
SAMPLE_RATE = 16_000
|
||||
_DEPLOY_DIR = Path(__file__).resolve().parents[3] / "vllm_omni" / "deploy"
|
||||
|
||||
# Guided decoding sharpens the distribution, so the unguided temperature
|
||||
# (0.1) adds excess sampling noise under CFG. Measured on the en-24 gate
|
||||
# corpus at cfg 1.5: temp 0.1 -> CER 7.31%, temp 0.05 -> CER 6.87% (vs the
|
||||
# unguided baseline 7.24%).
|
||||
GUIDED_TEMPERATURE = 0.05
|
||||
|
||||
|
||||
def default_deploy_config(name: str) -> str:
|
||||
"""Absolute path of a deploy yaml bundled with vllm_omni."""
|
||||
return str(_DEPLOY_DIR / name)
|
||||
|
||||
|
||||
def slugify(text: str, fallback: str = "prompt") -> str:
|
||||
slug = re.sub(r"\s+", "_", text.strip().lower())
|
||||
slug = re.sub(r"[^a-z0-9_]+", "", slug)
|
||||
return slug[:48] or fallback
|
||||
|
||||
|
||||
def load_corpus(
|
||||
corpus_file: str | None,
|
||||
texts: list[str] | None,
|
||||
default_texts: tuple[str, ...],
|
||||
) -> list[tuple[str, str]]:
|
||||
"""(utt_id, text) pairs from a TSV corpus file, CLI texts, or defaults."""
|
||||
if corpus_file:
|
||||
corpus = []
|
||||
for line in Path(corpus_file).read_text().splitlines():
|
||||
if not line.strip():
|
||||
continue
|
||||
utt, text = line.split("\t", 1)
|
||||
corpus.append((utt, text.strip()))
|
||||
return corpus
|
||||
resolved = texts if texts else list(default_texts)
|
||||
return [(slugify(t), t) for t in resolved]
|
||||
|
||||
|
||||
def load_audio_file(path: str) -> tuple[np.ndarray, int]:
|
||||
"""Mono float32 samples + native sample rate (processors resample)."""
|
||||
audio, sr = sf.read(path, dtype="float32")
|
||||
if audio.ndim == 2:
|
||||
audio = audio.mean(axis=1)
|
||||
return audio, sr
|
||||
|
||||
|
||||
def extract_pcm(multimodal_output: dict) -> torch.Tensor:
|
||||
"""Flatten one request's audio payload to a 1-D float32 CPU tensor."""
|
||||
audio = multimodal_output.get("model_outputs")
|
||||
if audio is None:
|
||||
audio = multimodal_output.get("audio")
|
||||
if audio is None:
|
||||
raise ValueError(f"no audio key in multimodal_output: {list(multimodal_output.keys())}")
|
||||
if isinstance(audio, list):
|
||||
valid = [torch.as_tensor(a).float().cpu().reshape(-1) for a in audio if a is not None]
|
||||
if not valid:
|
||||
raise ValueError("audio list is empty")
|
||||
return torch.cat(valid, dim=0) if len(valid) > 1 else valid[0]
|
||||
return torch.as_tensor(audio).float().cpu().reshape(-1)
|
||||
|
||||
|
||||
def write_wav(path: Path, pcm: torch.Tensor, sample_rate: int = SAMPLE_RATE) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
arr = (np.clip(pcm.numpy(), -1.0, 1.0) * 32767.0).astype(np.int16)
|
||||
sf.write(str(path), arr, sample_rate, format="WAV", subtype="PCM_16")
|
||||
|
||||
|
||||
def load_audex_tokenizer(model: str, profile: str = "tts", folder: str = "checkpoint_folder_audiogen"):
|
||||
"""Thinker tokenizer for CFG/TTA prompt building (resolves the snapshot)."""
|
||||
from transformers import AutoTokenizer
|
||||
|
||||
from vllm_omni.model_executor.models.audex.checkpoint import ensure_audex_snapshot
|
||||
|
||||
root = ensure_audex_snapshot(model, profile=profile)
|
||||
return AutoTokenizer.from_pretrained(str(Path(root) / folder), trust_remote_code=True)
|
||||
|
||||
|
||||
def request_index(req_output) -> int:
|
||||
"""Submission index from the numeric request-id prefix.
|
||||
|
||||
``Omni.generate`` may return outputs out of submission order, and a
|
||||
lexicographic sort would misplace "10_..." before "2_...".
|
||||
"""
|
||||
match = re.search(r"(\d+)", str(req_output.request_id))
|
||||
return int(match.group(1)) if match else 0
|
||||
|
||||
|
||||
def guided_stage_params(
|
||||
engine: Omni,
|
||||
cond_prompt: str,
|
||||
tokenizer,
|
||||
cfg_scale: float,
|
||||
pair_id: str,
|
||||
*,
|
||||
temperature: float | None = None,
|
||||
extra_args: dict[str, Any] | None = None,
|
||||
null_prompt_builder: Callable[[str, Any], str] = build_null_prompt,
|
||||
) -> list:
|
||||
"""Stage sampling params carrying the CFG pair contract for ONE request.
|
||||
|
||||
Clones the engine's resolved defaults (never mutates the shared list),
|
||||
applies ``extra_args`` unconditionally (e.g. the TTA RVQ contract), and
|
||||
attaches the cond/uncond pair contract when ``cfg_scale > 1.0``. Guided
|
||||
requests must go one at a time: each needs its own pair id and a
|
||||
length-matched null prompt built from its own text.
|
||||
"""
|
||||
params = copy.deepcopy(engine.resolve_sampling_params_list(None))
|
||||
stage0 = params[0]
|
||||
if temperature is not None:
|
||||
stage0.temperature = temperature
|
||||
if stage0.extra_args is None:
|
||||
stage0.extra_args = {}
|
||||
if extra_args:
|
||||
stage0.extra_args.update(extra_args)
|
||||
if cfg_scale > 1.0:
|
||||
stage0.extra_args.update(
|
||||
{
|
||||
"cfg_scale": float(cfg_scale),
|
||||
"cfg_role": "cond",
|
||||
"cfg_pair_id": pair_id,
|
||||
"cfg_null_prompt": null_prompt_builder(cond_prompt, tokenizer),
|
||||
}
|
||||
)
|
||||
return params
|
||||
@@ -0,0 +1,277 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Offline Audex (Nemotron-Labs-Audex-2B) cascaded speech-to-speech example.
|
||||
|
||||
The official three-pass cascade over ONE ``audex_s2s``
|
||||
deployment:
|
||||
|
||||
1. ASR pass (audio + "Transcribe the input speech." → transcript, text)
|
||||
2. chat pass (transcript → answer, text)
|
||||
3. TTS pass (answer → speech, audio via the streaming causal decoder)
|
||||
|
||||
Text passes carry ``modalities: ["text"]`` and finish at stage 0; only the
|
||||
TTS pass (``modalities: ["audio"]``) streams codec frames into code2wav.
|
||||
|
||||
Example (without --audio-file, vLLM's public ``mary_had_lamb`` asset is
|
||||
used as the spoken input):
|
||||
|
||||
python examples/offline_inference/audex/speech_to_speech.py \\
|
||||
--audio-file question.wav --output results/answer.wav
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import copy
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
import torch
|
||||
from common import (
|
||||
SAMPLE_RATE,
|
||||
default_deploy_config,
|
||||
guided_stage_params,
|
||||
load_audex_tokenizer,
|
||||
load_audio_file,
|
||||
write_wav,
|
||||
)
|
||||
from vllm.assets.audio import AudioAsset
|
||||
|
||||
from vllm_omni import Omni
|
||||
from vllm_omni.model_executor.models.audex.prompt import build_cond_prompt
|
||||
|
||||
ASR_QUESTION = "Transcribe the input speech."
|
||||
SYSTEM_PROMPT = "You are a helpful and harmless assistant.\n\nYou are not allowed to use any tools."
|
||||
# The official chat-pass recipe below (SYSTEM_PROMPT + TEXT_INSTRUCTION
|
||||
# prefix, open <think> priming, temp 1.0/top_p 0.95, seed retry) applies to
|
||||
# the 30B-A3B ONLY (see is_30b()/_chat_prompt below and
|
||||
# tests/e2e/offline_inference/test_audex_s2s.py, which mirrors this same
|
||||
# split). It is load-bearing on the 30B: without the formatting-instruction
|
||||
# prefix a bare transcript turn is treated as a TTS request and the model
|
||||
# emits literal <speechcodec_N> tokens instead of a text answer (the system
|
||||
# prompt alone does not prevent this). The 2B must NOT use this recipe: it
|
||||
# is long-verified stable on a bare prompt + engine-default sampling, while
|
||||
# the official temp-1.0 seed-0 draw hallucinates off-language text (yet
|
||||
# closes </think> cleanly, so the seed retry never fires) and fails the
|
||||
# round-trip WER gate — measured, reproduced byte-identically.
|
||||
TEXT_INSTRUCTION = (
|
||||
"SYSTEM & FORMATTING INSTRUCTION: You are Audex, created by NVIDIA based on the Nemotron-Cascade-2 architecture. "
|
||||
"You may output your reasoning, followed by your final response. "
|
||||
"You may format your reasoning block however you like. "
|
||||
"However, your final response must strictly follow these rules: "
|
||||
"* Write in plain, unformatted prose like a book or newspaper article. "
|
||||
"* Do not use markdown, bullet points, lists, or headers. "
|
||||
"* Standard numbers, abbreviations, and symbols are acceptable. "
|
||||
"* [CRITICAL] You must press enter after every single sentence, placing each sentence on its own separate line."
|
||||
)
|
||||
|
||||
|
||||
def is_30b(model: str) -> bool:
|
||||
# Users pass arbitrary repo IDs or local snapshot paths, so this is a
|
||||
# heuristic (not an exact match against the official 30B-A3B repo ID)
|
||||
# — good enough for an example script; keep in sync with the same
|
||||
# branch in examples/online_serving/audex/client.py and
|
||||
# tests/e2e/offline_inference/test_audex_s2s.py.
|
||||
return "30b" in model.lower()
|
||||
|
||||
|
||||
def parse_args():
|
||||
parser = argparse.ArgumentParser(description="Offline Audex cascaded S2S")
|
||||
parser.add_argument("--model", type=str, default="nvidia/Nemotron-Labs-Audex-2B")
|
||||
parser.add_argument(
|
||||
"--audio-file",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Spoken question (WAV). Defaults to vLLM's mary_had_lamb audio asset.",
|
||||
)
|
||||
parser.add_argument("--output", type=str, default="results/audex_s2s_answer.wav")
|
||||
parser.add_argument(
|
||||
"--deploy-config",
|
||||
type=str,
|
||||
# The model root's default deploy yaml is the TTS-only pipeline; the
|
||||
# cascade needs the audio-capable 2-stage full pipeline.
|
||||
default=default_deploy_config("audex_s2s.yaml"),
|
||||
help="Deploy yaml (defaults to the audex_s2s pipeline).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--cfg-scale",
|
||||
type=float,
|
||||
default=1.5,
|
||||
help="CFG strength for the TTS pass (1.0 disables; official setting 1.5).",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def _chat_prompt(user_text: str) -> str:
|
||||
# Official chat-pass recipe: system prompt + formatting instruction
|
||||
# prefixed to the user text, thinking ENABLED via an open <think> block
|
||||
# (the official demo's web UI ships "Enable reasoning" checked, i.e.
|
||||
# build_chat_prompt(..., enable_thinking=True); chat_template.jinja then
|
||||
# renders "<|im_start|>assistant\n<think>\n"). The instruction invites
|
||||
# the model to reason before answering — priming the block open keeps
|
||||
# that reasoning inside <think>...</think> so _extract_final_response
|
||||
# below can drop it; with a closed block the 30B leaks the reasoning as
|
||||
# plain text in front of the answer.
|
||||
return (
|
||||
f"<|im_start|>system\n{SYSTEM_PROMPT}<|im_end|>\n"
|
||||
f"<|im_start|>user\n{TEXT_INSTRUCTION}\n\n{user_text}<|im_end|>\n"
|
||||
"<|im_start|>assistant\n<think>\n"
|
||||
)
|
||||
|
||||
|
||||
def _asr_prompt() -> str:
|
||||
return f"<|im_start|>user\n<so_embedding>\n{ASR_QUESTION}<|im_end|>\n<|im_start|>assistant\n<think></think>"
|
||||
|
||||
|
||||
def _extract_final_response(text: str) -> str:
|
||||
# Mirror the official demo's extract_spoken_answer
|
||||
# (cascaded_s2s_web_server.py): keep only what follows the LAST closing
|
||||
# think tag, falling back to the full text when the model skipped
|
||||
# reasoning or never closed the block. Benign no-op for the 2B, whose
|
||||
# answers carry no think block.
|
||||
if "</think>" in text:
|
||||
answer = text.rsplit("</think>", 1)[-1].strip()
|
||||
if answer:
|
||||
return answer
|
||||
return text.strip()
|
||||
|
||||
|
||||
def _clean(text: str) -> str:
|
||||
# The full checkpoint prefixes transcripts with a language tag sentence
|
||||
# and quotes the content; extract the quoted payload when present. Span
|
||||
# first-to-last quote (not split on every apostrophe) so contractions
|
||||
# inside the transcript (e.g. "don't") survive intact.
|
||||
text = text.strip()
|
||||
match = re.search(r"'(.*)'", text, flags=re.S)
|
||||
if match and match.group(1).strip():
|
||||
return match.group(1).strip()
|
||||
return text
|
||||
|
||||
|
||||
def _text_of(outputs) -> str:
|
||||
(req_output,) = outputs
|
||||
return (req_output.outputs[0].text or "").strip()
|
||||
|
||||
|
||||
CHAT_MAX_TOKENS = 2048 # official --text-max-tokens default
|
||||
# The official demo samples this pass unseeded; some trajectories deliberate
|
||||
# past the token budget without ever closing the think block, so retry a few
|
||||
# deterministic seeds until one closes (see the chat pass below).
|
||||
CHAT_SEEDS = (0, 1, 2)
|
||||
|
||||
|
||||
def _chat_params(engine: Omni, seed: int):
|
||||
# Official demo text-pass sampling (cascaded_s2s_web_server.py defaults:
|
||||
# --text-temperature 1.0, --text-top-p 0.95, --text-max-tokens 2048).
|
||||
# The yaml stage-0 defaults (temperature 0.1, top_k 80) are TTS-thinker
|
||||
# settings; under near-greedy decoding the 30B loops inside its
|
||||
# reasoning block until the token budget runs out and never emits
|
||||
# </think>, so the extracted "answer" would be truncated reasoning.
|
||||
params = copy.deepcopy(engine.resolve_sampling_params_list(None))
|
||||
stage0 = params[0]
|
||||
stage0.temperature = 1.0
|
||||
stage0.top_p = 0.95
|
||||
stage0.top_k = -1
|
||||
stage0.max_tokens = CHAT_MAX_TOKENS
|
||||
stage0.seed = seed
|
||||
return params
|
||||
|
||||
|
||||
def _chat_once(engine: Omni, prompt: str, seed: int) -> tuple[str, bool]:
|
||||
(req_output,) = engine.generate([{"prompt": prompt, "modalities": ["text"]}], _chat_params(engine, seed))
|
||||
out = req_output.outputs[0]
|
||||
text = (out.text or "").strip()
|
||||
truncated = getattr(out, "finish_reason", None) == "length" or len(out.token_ids or []) >= CHAT_MAX_TOKENS
|
||||
return text, truncated
|
||||
|
||||
|
||||
def main():
|
||||
args = parse_args()
|
||||
engine = Omni(model=args.model, deploy_config=args.deploy_config, trust_remote_code=True)
|
||||
|
||||
tokenizer = None
|
||||
if args.cfg_scale > 1.0:
|
||||
tokenizer = load_audex_tokenizer(args.model, profile="full", folder="checkpoint_folder_full")
|
||||
|
||||
if args.audio_file:
|
||||
audio, sr = load_audio_file(args.audio_file)
|
||||
else:
|
||||
audio, sr = AudioAsset("mary_had_lamb").audio_and_sample_rate
|
||||
|
||||
# Pass 1 — ASR (text-final; never touches the speech decoder).
|
||||
transcript = _clean(
|
||||
_text_of(
|
||||
engine.generate(
|
||||
[
|
||||
{
|
||||
"prompt": _asr_prompt(),
|
||||
"multi_modal_data": {"audio": (audio, sr)},
|
||||
"modalities": ["text"],
|
||||
}
|
||||
]
|
||||
)
|
||||
)
|
||||
)
|
||||
print(f"[1/3] transcript : {transcript}")
|
||||
if not transcript:
|
||||
raise SystemExit("ASR pass produced an empty transcript; aborting cascade")
|
||||
|
||||
# Pass 2 — chat answer (text-final). The recipe is size-conditional (see
|
||||
# the TEXT_INSTRUCTION comment above): the 30B requires the official
|
||||
# recipe, the 2B regresses under it.
|
||||
if is_30b(args.model):
|
||||
# Official recipe. The raw completion is
|
||||
# <reasoning...></think><answer>; speak only the final response.
|
||||
# Retry with the next seed only when the budget ran out before the
|
||||
# reasoning block closed (a natural stop without </think> is a
|
||||
# complete answer); if every seed spins, fall back to the official
|
||||
# demo's behavior of keeping the full text.
|
||||
raw = ""
|
||||
for seed in CHAT_SEEDS:
|
||||
raw, truncated = _chat_once(engine, _chat_prompt(transcript), seed)
|
||||
if "</think>" in raw or not truncated:
|
||||
break
|
||||
print(f"[2/3] chat seed {seed} hit the token budget mid-reasoning; retrying")
|
||||
answer = _extract_final_response(raw)
|
||||
else:
|
||||
# 2B: bare prompt with a closed think block, engine-default sampling
|
||||
# (yaml near-greedy), no retry; the answer is used as-is.
|
||||
chat_prompt = f"<|im_start|>user\n{transcript}<|im_end|>\n<|im_start|>assistant\n<think></think>"
|
||||
answer = _text_of(engine.generate([{"prompt": chat_prompt, "modalities": ["text"]}]))
|
||||
print(f"[2/3] answer : {answer}")
|
||||
if not answer:
|
||||
raise SystemExit("Chat pass produced an empty answer; aborting cascade")
|
||||
|
||||
# Pass 3 — TTS (audio-final; streams through code2wav). Both final
|
||||
# stages of the pipeline may emit an output record for the request;
|
||||
# collect the audio-bearing one.
|
||||
# The formatting instruction puts every sentence on its own line;
|
||||
# flatten to plain prose for the TTS thinker.
|
||||
tts_prompt = build_cond_prompt(re.sub(r"\s+", " ", answer).strip())
|
||||
outputs = engine.generate(
|
||||
[{"prompt": tts_prompt, "modalities": ["audio"]}],
|
||||
guided_stage_params(engine, tts_prompt, tokenizer, args.cfg_scale, "s2s-tts-pass"),
|
||||
)
|
||||
chunks: list[torch.Tensor] = []
|
||||
for req_output in outputs:
|
||||
mm = getattr(req_output, "multimodal_output", None) or {}
|
||||
if not mm and req_output.outputs:
|
||||
mm = getattr(req_output.outputs[0], "multimodal_output", None) or {}
|
||||
audio_val = mm.get("model_outputs", mm.get("audio"))
|
||||
if audio_val is None:
|
||||
continue
|
||||
if isinstance(audio_val, list):
|
||||
chunks.extend(torch.as_tensor(a).float().cpu().reshape(-1) for a in audio_val if a is not None)
|
||||
else:
|
||||
chunks.append(torch.as_tensor(audio_val).float().cpu().reshape(-1))
|
||||
pcm = torch.cat(chunks) if chunks else torch.empty(0)
|
||||
if pcm.numel() == 0:
|
||||
raise SystemExit("TTS pass produced empty audio")
|
||||
|
||||
out_path = Path(args.output)
|
||||
write_wav(out_path, pcm)
|
||||
print(f"[3/3] speech : {pcm.numel() / SAMPLE_RATE:.2f}s -> {out_path}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,157 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Offline Audex (Nemotron-Labs-Audex-2B) text-to-audio inference example.
|
||||
|
||||
Caption → general audio through the vLLM-Omni TTA pipeline: the audiogen
|
||||
thinker generates interleaved 4-codebook <audiocodec_N> RVQ tokens under an
|
||||
RVQ phase mask, and the external XCodec1 checkpoint decodes them to a 16 kHz
|
||||
mono WAV.
|
||||
|
||||
(For DIFFUSION text-to-audio models — Stable Audio Open, AudioX — use
|
||||
``examples/offline_inference/text_to_audio/text_to_audio.py`` instead; Audex
|
||||
TTA is a two-stage autoregressive pipeline with a different interface.)
|
||||
|
||||
Classifier-free guidance is effectively mandatory for TTA quality (official
|
||||
default scale 3.0), so every request submits a cond/uncond pair; requests
|
||||
run one at a time.
|
||||
|
||||
Pass the HF repo ROOT as --model. XCodec1 resolves from --xcodec1-path /
|
||||
XCODEC1_PATH / the default hf-audio repo.
|
||||
|
||||
Example:
|
||||
|
||||
python examples/offline_inference/audex/text_to_audio.py \\
|
||||
--captions "Heavy rain falling on a tin roof." \\
|
||||
--output-dir results/audex_tta_wavs
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import os
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
from common import (
|
||||
SAMPLE_RATE,
|
||||
default_deploy_config,
|
||||
extract_pcm,
|
||||
guided_stage_params,
|
||||
load_audex_tokenizer,
|
||||
load_corpus,
|
||||
write_wav,
|
||||
)
|
||||
|
||||
from vllm_omni import Omni
|
||||
from vllm_omni.model_executor.models.audex.prompt import build_tta_cond_prompt, build_tta_null_prompt
|
||||
from vllm_omni.model_executor.models.audex.tta import build_tta_phase_token_ids
|
||||
|
||||
DEFAULT_CAPTIONS = (
|
||||
"Heavy rain falling on a tin roof.",
|
||||
"A dog barking in the distance while birds chirp.",
|
||||
"Ocean waves crashing on a rocky shore.",
|
||||
)
|
||||
DEFAULT_CODEC_CAP = 4000
|
||||
|
||||
|
||||
def parse_args():
|
||||
parser = argparse.ArgumentParser(description="Offline Audex TTA inference")
|
||||
parser.add_argument("--model", type=str, default="nvidia/Nemotron-Labs-Audex-2B")
|
||||
parser.add_argument("--captions", type=str, nargs="+", default=None, help="Audio captions.")
|
||||
parser.add_argument(
|
||||
"--captions-file",
|
||||
type=str,
|
||||
default=None,
|
||||
help="TSV corpus: one 'utt_id<TAB>caption' per line (overrides --captions).",
|
||||
)
|
||||
parser.add_argument("--output-dir", type=str, default="results/audex_tta_wavs")
|
||||
parser.add_argument(
|
||||
"--deploy-config",
|
||||
type=str,
|
||||
# The model root's default deploy yaml is the TTS pipeline; TTA
|
||||
# prompts and RVQ sampling params require the dedicated TTA pipeline.
|
||||
default=default_deploy_config("audex_tta.yaml"),
|
||||
help="Deploy yaml (defaults to the audex_tta pipeline).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--xcodec1-path",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Local XCodec1 checkpoint (defaults to $XCODEC1_PATH or the hf-audio repo).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--cfg-scale",
|
||||
type=float,
|
||||
default=3.0,
|
||||
help="Classifier-free guidance strength (official TTA default 3.0; 1.0 disables).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--codec-cap",
|
||||
type=int,
|
||||
default=DEFAULT_CODEC_CAP,
|
||||
help="Max generated codec tokens before the phase mask forces <audiogen_end>.",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def main():
|
||||
args = parse_args()
|
||||
if args.xcodec1_path:
|
||||
os.environ["XCODEC1_PATH"] = args.xcodec1_path
|
||||
output_dir = Path(args.output_dir)
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
corpus = load_corpus(args.captions_file, args.captions, DEFAULT_CAPTIONS)
|
||||
|
||||
tokenizer = load_audex_tokenizer(args.model, profile="tta")
|
||||
phase_token_ids, start_tid, end_tid = build_tta_phase_token_ids(tokenizer)
|
||||
tta_rvq = {
|
||||
"phase_token_ids": phase_token_ids,
|
||||
"start_tid": start_tid,
|
||||
"end_tid": end_tid,
|
||||
"codec_cap": args.codec_cap,
|
||||
# The prompt ends with <audiogen_start>, so generation begins at
|
||||
# RVQ phase 0 immediately.
|
||||
"start_in_prompt": True,
|
||||
}
|
||||
|
||||
engine = Omni(model=args.model, deploy_config=args.deploy_config, trust_remote_code=True)
|
||||
|
||||
print(f"Model : {args.model}")
|
||||
print(f"Captions : {len(corpus)}")
|
||||
print(f"CFG scale : {args.cfg_scale}")
|
||||
print(f"Codec cap : {args.codec_cap}")
|
||||
print(f"Output dir : {output_dir}")
|
||||
|
||||
total_elapsed = 0.0
|
||||
total_dur = 0.0
|
||||
for utt, caption in corpus:
|
||||
cond_prompt = build_tta_cond_prompt(caption)
|
||||
sampling = guided_stage_params(
|
||||
engine,
|
||||
cond_prompt,
|
||||
tokenizer,
|
||||
args.cfg_scale,
|
||||
f"tta-{utt}",
|
||||
extra_args={"tta_rvq": tta_rvq},
|
||||
null_prompt_builder=build_tta_null_prompt,
|
||||
)
|
||||
t_start = time.perf_counter()
|
||||
outputs = engine.generate([cond_prompt], sampling)
|
||||
total_elapsed += time.perf_counter() - t_start
|
||||
|
||||
(req_output,) = outputs
|
||||
pcm = extract_pcm(req_output.outputs[0].multimodal_output)
|
||||
dur = pcm.numel() / SAMPLE_RATE
|
||||
if dur <= 0:
|
||||
raise RuntimeError(f"empty audio for {utt}")
|
||||
out_path = output_dir / f"{utt}.wav"
|
||||
write_wav(out_path, pcm)
|
||||
total_dur += dur
|
||||
print(f" {utt:<48} dur={dur:6.2f}s -> {out_path}")
|
||||
|
||||
rtf = total_elapsed / total_dur if total_dur > 0 else float("inf")
|
||||
print(f"Total infer : {total_elapsed:.2f}s total audio: {total_dur:.2f}s RTF: {rtf:.3f}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,162 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Offline Audex (Nemotron-Labs-Audex-2B) TTS inference example.
|
||||
|
||||
Runs Stage 0 (thinker) + Stage 1 (streaming causal speech decoder) end-to-end
|
||||
through the vLLM-Omni engine and saves a 16 kHz mono WAV per prompt.
|
||||
|
||||
Pass the HF repo ROOT as --model; per-stage subfolders resolve automatically.
|
||||
|
||||
Examples:
|
||||
|
||||
python examples/offline_inference/audex/text_to_speech.py \\
|
||||
--texts "The weather is so good today." \\
|
||||
--output-dir results/audex_wavs
|
||||
|
||||
# TSV corpus (utt_id<TAB>text per line), all requests in flight at once:
|
||||
python examples/offline_inference/audex/text_to_speech.py \\
|
||||
--texts-file corpus.tsv --output-dir results/audex_wavs
|
||||
|
||||
# One request at a time:
|
||||
python examples/offline_inference/audex/text_to_speech.py \\
|
||||
--texts-file corpus.tsv --sequential --output-dir results/audex_wavs
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
from common import (
|
||||
GUIDED_TEMPERATURE,
|
||||
SAMPLE_RATE,
|
||||
extract_pcm,
|
||||
guided_stage_params,
|
||||
load_audex_tokenizer,
|
||||
load_corpus,
|
||||
request_index,
|
||||
write_wav,
|
||||
)
|
||||
|
||||
from vllm_omni import Omni
|
||||
from vllm_omni.model_executor.models.audex.prompt import build_cond_prompt
|
||||
|
||||
DEFAULT_TEXTS = (
|
||||
"Hello world.",
|
||||
"The quick brown fox jumps over the lazy dog.",
|
||||
"Today is a beautiful day for a walk in the park.",
|
||||
)
|
||||
|
||||
|
||||
def parse_args():
|
||||
parser = argparse.ArgumentParser(description="Offline Audex TTS inference")
|
||||
parser.add_argument(
|
||||
"--model",
|
||||
type=str,
|
||||
default="nvidia/Nemotron-Labs-Audex-2B",
|
||||
help="HF repo root (or local snapshot root).",
|
||||
)
|
||||
parser.add_argument("--texts", type=str, nargs="+", default=None, help="Plain-text prompts.")
|
||||
parser.add_argument(
|
||||
"--texts-file",
|
||||
type=str,
|
||||
default=None,
|
||||
help="TSV corpus: one 'utt_id<TAB>text' per line (overrides --texts).",
|
||||
)
|
||||
parser.add_argument("--output-dir", type=str, default="results/audex_wavs")
|
||||
parser.add_argument("--deploy-config", type=str, default=None, help="Override the deploy config path.")
|
||||
parser.add_argument(
|
||||
"--sequential",
|
||||
action="store_true",
|
||||
help="Submit one request at a time instead of all at once.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--cfg-scale",
|
||||
type=float,
|
||||
default=1.0,
|
||||
help=(
|
||||
"Classifier-free guidance strength. 1.0 disables guidance (default, "
|
||||
"matches the v1 baseline); 1.5 is the official quality setting. "
|
||||
"Guided runs submit one request at a time (each carries its own "
|
||||
"length-matched null prompt and pair id)."
|
||||
),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--temperature",
|
||||
type=float,
|
||||
default=None,
|
||||
help="Override the deploy yaml's stage-0 sampling temperature.",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def _write_outputs(outputs, batch: list[tuple[str, str]], output_dir: Path) -> float:
|
||||
ordered = sorted(outputs, key=request_index)
|
||||
if len(ordered) != len(batch):
|
||||
raise RuntimeError(f"expected {len(batch)} outputs, got {len(ordered)}")
|
||||
|
||||
total_dur = 0.0
|
||||
for (utt, _text), req_output in zip(batch, ordered):
|
||||
pcm = extract_pcm(req_output.outputs[0].multimodal_output)
|
||||
dur = pcm.numel() / SAMPLE_RATE
|
||||
if dur <= 0:
|
||||
raise RuntimeError(f"empty audio for {utt}")
|
||||
out_path = output_dir / f"{utt}.wav"
|
||||
write_wav(out_path, pcm)
|
||||
total_dur += dur
|
||||
print(f" {utt:<40} dur={dur:6.2f}s -> {out_path}")
|
||||
return total_dur
|
||||
|
||||
|
||||
def main():
|
||||
args = parse_args()
|
||||
output_dir = Path(args.output_dir)
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
corpus = load_corpus(args.texts_file, args.texts, DEFAULT_TEXTS)
|
||||
|
||||
engine = Omni(model=args.model, deploy_config=args.deploy_config, trust_remote_code=True)
|
||||
|
||||
cfg_enabled = args.cfg_scale > 1.0
|
||||
tokenizer = load_audex_tokenizer(args.model) if cfg_enabled else None
|
||||
|
||||
print(f"Model : {args.model}")
|
||||
print(f"Prompts : {len(corpus)}")
|
||||
print(f"Mode : {'sequential' if args.sequential or cfg_enabled else 'batched'}")
|
||||
print(f"CFG scale : {args.cfg_scale}" + (" (guidance off)" if not cfg_enabled else ""))
|
||||
print(f"Output dir : {output_dir}")
|
||||
|
||||
total_elapsed = 0.0
|
||||
total_dur = 0.0
|
||||
# CFG runs one request at a time: each request needs its own pair id and
|
||||
# text-specific null prompt in the stage-0 sampling params.
|
||||
batches = [[item] for item in corpus] if (args.sequential or cfg_enabled) else [corpus]
|
||||
for batch in batches:
|
||||
prompts = [build_cond_prompt(text) for _utt, text in batch]
|
||||
sampling_params_list = None
|
||||
if cfg_enabled:
|
||||
sampling_params_list = guided_stage_params(
|
||||
engine,
|
||||
prompts[0],
|
||||
tokenizer,
|
||||
args.cfg_scale,
|
||||
f"cfg-{batch[0][0]}",
|
||||
temperature=GUIDED_TEMPERATURE,
|
||||
)
|
||||
if args.temperature is not None:
|
||||
import copy
|
||||
|
||||
if sampling_params_list is None:
|
||||
sampling_params_list = copy.deepcopy(engine.resolve_sampling_params_list(None))
|
||||
sampling_params_list[0].temperature = args.temperature
|
||||
t_start = time.perf_counter()
|
||||
outputs = engine.generate(prompts, sampling_params_list)
|
||||
total_elapsed += time.perf_counter() - t_start
|
||||
total_dur += _write_outputs(outputs, batch, output_dir)
|
||||
|
||||
rtf = total_elapsed / total_dur if total_dur > 0 else float("inf")
|
||||
print(f"Total infer : {total_elapsed:.2f}s total audio: {total_dur:.2f}s RTF: {rtf:.3f}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -1,7 +1,7 @@
|
||||
# Image-To-Video
|
||||
|
||||
This shared example generates videos from images with VACE, Wan2.2,
|
||||
LTX-2/LTX-2.3, HunyuanVideo-1.5, Cosmos3, and other compatible pipelines.
|
||||
This shared example generates videos from images with VACE, Wan2.2, LTX-2,
|
||||
HunyuanVideo-1.5, Cosmos3, and other compatible pipelines.
|
||||
|
||||
- `image_to_video.py`: command-line script for single video generation with advanced options.
|
||||
|
||||
@@ -26,7 +26,7 @@ This folder provides a unified CLI script for image-to-video generation using vL
|
||||
| `Wan-AI/Wan2.2-I2V-A14B-Diffusers` | 480 x 832 | 81 | 50 | 5.0 | Around 60 GiB BF16 for basic single-card usage |
|
||||
| `Wan-AI/Wan2.2-TI2V-5B-Diffusers` | 480 x 832 | 81 | 50 | 4.0 | Around 20–25 GiB BF16, smallest I2V model |
|
||||
| `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_i2v` | 480 x 832 | 121 | 50 | 6.0 | Around 100 GiB at default settings; the example enables `--enable-cpu-offload` + VAE tiling/slicing to fit an 80 GiB card |
|
||||
| LTX2 (local path + `--model-class-name LTX2ImageToVideoPipeline`) | 512 x 768 | 121 | 40 | 4.0 | Memory use depends on frame count and tensor parallelism |
|
||||
| `Lightricks/LTX-2` | 512 x 768 | 121 | 40 | video 3.0 / audio 7.0 | Memory use depends on frame count and tensor parallelism |
|
||||
|
||||
!!! info
|
||||
Peak VRAM: based on basic single-card usage, batch size = 1, without any acceleration/optimization features. Some model weights cannot fit into one card with 80 GiB VRAM, which may need to use CPU offloading.
|
||||
@@ -105,7 +105,7 @@ python image_to_video.py \
|
||||
| Argument | Type | Default | Description |
|
||||
| -------- | ---- | ------- | ----------- |
|
||||
| `--model` | str | `Wan-AI/Wan2.2-I2V-A14B-Diffusers` | Diffusers I2V model ID or local path |
|
||||
| `--model-class-name` | str | `None` | Override model class name (e.g., `LTX2ImageToVideoPipeline`) |
|
||||
| `--model-class-name` | str | `None` | Optional pipeline override |
|
||||
| `--image` | str | (required) | Path to input image |
|
||||
| `--prompt` | str | `""` | Text description of desired motion/animation |
|
||||
| `--negative-prompt` | str | `""` | Optional list of artifacts to suppress |
|
||||
@@ -195,24 +195,19 @@ python image_to_video.py \
|
||||
--output hunyuan_i2v.mp4
|
||||
```
|
||||
|
||||
### LTX2 Image-to-Video
|
||||
### LTX-2
|
||||
|
||||
```bash
|
||||
python image_to_video.py \
|
||||
--model /path/to/LTX-2 \
|
||||
--model-class-name LTX2ImageToVideoPipeline \
|
||||
--model Lightricks/LTX-2 \
|
||||
--image cherry_blossom.jpg \
|
||||
--prompt "A cinematic dolly shot of cherry blossoms" \
|
||||
--height 512 \
|
||||
--width 768 \
|
||||
--num-frames 121 \
|
||||
--num-inference-steps 40 \
|
||||
--guidance-scale 4.0 \
|
||||
--frame-rate 24 \
|
||||
--fps 24 \
|
||||
--output ltx2_i2v.mp4
|
||||
```
|
||||
|
||||
See the [LTX-2 recipe](../../../recipes/LTX/LTX-2.md) for all checkpoints,
|
||||
pipeline selection, T2V, defaults, and advanced options.
|
||||
|
||||
## Advanced Features
|
||||
|
||||
### CFG Parallel
|
||||
@@ -233,30 +228,6 @@ python image_to_video.py \
|
||||
--output i2v_cached.mp4
|
||||
```
|
||||
|
||||
### LTX-2.3
|
||||
|
||||
```bash
|
||||
python image_to_video.py \
|
||||
--model diffusers/LTX-2.3-Diffusers \
|
||||
--model-class-name LTX23ImageToVideoPipeline \
|
||||
--image cherry_blossom.jpg \
|
||||
--prompt "Cherry blossoms swaying gently in the breeze with synchronized ambient sound" \
|
||||
--negative-prompt "worst quality, inconsistent motion, blurry, jittery, distorted" \
|
||||
--height 384 \
|
||||
--width 512 \
|
||||
--num-frames 25 \
|
||||
--guidance-scale 4.0 \
|
||||
--num-inference-steps 20 \
|
||||
--frame-rate 24 \
|
||||
--fps 24 \
|
||||
--output ltx23_i2v_output.mp4
|
||||
```
|
||||
|
||||
Use the Diffusers-format checkpoint `diffusers/LTX-2.3-Diffusers`; the
|
||||
upstream `Lightricks/LTX-2.3` raw safetensors repo is not directly loadable by
|
||||
this pipeline. Pass `--model-class-name LTX23ImageToVideoPipeline` to select
|
||||
the LTX-2.3 image-to-video pipeline.
|
||||
|
||||
### Cosmos3
|
||||
|
||||
```bash
|
||||
@@ -274,8 +245,8 @@ python image_to_video.py \
|
||||
|
||||
Key arguments:
|
||||
|
||||
- `--model`: Model ID (I2V-A14B for MoE, TI2V-5B for unified T2V+I2V,
|
||||
LTX-2/LTX-2.3, Cosmos3, or VACE).
|
||||
- `--model`: Model ID (I2V-A14B for MoE, TI2V-5B for unified T2V+I2V, LTX-2,
|
||||
Cosmos3, or VACE).
|
||||
- `--image`: Path to the first-frame or source image.
|
||||
- `--last-image`: Optional last-frame condition for models such as VACE.
|
||||
- `--mask-image`: Optional inpainting mask. White pixels are regenerated and black pixels are preserved.
|
||||
@@ -290,7 +261,7 @@ Key arguments:
|
||||
- `--guidance-scale` and `--guidance-scale-high`: CFG scale (applied to low/high-noise stages for MoE).
|
||||
- `--negative-prompt`: Optional list of artifacts to suppress.
|
||||
- `--boundary-ratio`: Boundary split ratio for two-stage MoE models.
|
||||
- `--flow-shift`: Scheduler flow shift (default: model-specific — Wan/LTX2 5.0, Cosmos3 10.0).
|
||||
- `--flow-shift`: Scheduler flow shift (default: Wan 5.0, Cosmos3 10.0). LTX does not consume this option.
|
||||
- `--sample-solver`: Wan2.2 sampling solver. Use `unipc` for the default multistep solver, or `euler` for Lightning/Distill checkpoints.
|
||||
- `--num-inference-steps`: Number of denoising steps (default: model-specific — Wan 50, LTX2 40, Cosmos3 35).
|
||||
- `--fps`: Frames per second for the saved MP4 (requires `diffusers` export_to_video).
|
||||
|
||||
@@ -23,16 +23,14 @@ Usage:
|
||||
|
||||
# LTX2 image-to-video
|
||||
python image_to_video.py --model /path/to/LTX-2 \
|
||||
--model-class-name LTX2ImageToVideoPipeline \
|
||||
--image input.jpg --prompt "A cinematic dolly shot of a boat" \
|
||||
--num-frames 121 --num-inference-steps 40 --guidance-scale 4.0 \
|
||||
--num-frames 121 --num-inference-steps 40 \
|
||||
--frame-rate 24 --fps 24 --output ltx2_i2v.mp4
|
||||
|
||||
# LTX-2.3 image-to-video
|
||||
python image_to_video.py --model diffusers/LTX-2.3-Diffusers \
|
||||
--model-class-name LTX23ImageToVideoPipeline \
|
||||
--image input.jpg --prompt "A cinematic dolly shot of a boat" \
|
||||
--height 384 --width 512 --num-frames 25 --num-inference-steps 20 \
|
||||
--height 512 --width 768 --num-frames 121 --num-inference-steps 30 \
|
||||
--frame-rate 24 --fps 24 --output ltx23_i2v.mp4
|
||||
|
||||
# HunyuanVideo-1.5 I2V (480p)
|
||||
@@ -100,7 +98,7 @@ def parse_args() -> argparse.Namespace:
|
||||
parser.add_argument(
|
||||
"--model-class-name",
|
||||
default=None,
|
||||
help="Override model class name (e.g., LTX2ImageToVideoPipeline or LTX23ImageToVideoPipeline).",
|
||||
help="Override model class name (LTX checkpoints default to LTX2Pipeline).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--deploy-config",
|
||||
@@ -117,7 +115,7 @@ def parse_args() -> argparse.Namespace:
|
||||
help="Path to a reference image. Repeat to provide multiple references.",
|
||||
)
|
||||
parser.add_argument("--prompt", default="", help="Text prompt describing the desired motion.")
|
||||
parser.add_argument("--negative-prompt", default="", help="Negative prompt.")
|
||||
parser.add_argument("--negative-prompt", default=None, help="Negative prompt. Default: model-specific.")
|
||||
parser.add_argument("--seed", type=int, default=42, help="Random seed.")
|
||||
parser.add_argument("--guidance-scale", type=float, default=None, help="CFG scale. Default: model-specific.")
|
||||
parser.add_argument(
|
||||
@@ -331,7 +329,10 @@ def main():
|
||||
generator = torch.Generator(device=current_omni_platform.device_type).manual_seed(args.seed)
|
||||
model_name = str(args.model).lower() if args.model is not None else ""
|
||||
model_class_name = args.model_class_name
|
||||
is_ltx2 = model_class_name in {"LTX2ImageToVideoPipeline", "LTX23ImageToVideoPipeline"}
|
||||
model_class_name_lower = (model_class_name or "").lower()
|
||||
is_ltx2_distilled = "distilled" in model_class_name_lower or "distilled" in model_name
|
||||
is_ltx23 = "ltx23" in model_class_name_lower or "ltx-2.3" in model_name
|
||||
is_ltx2 = is_ltx2_distilled or is_ltx23 or "ltx2" in model_class_name_lower or "ltx-2" in model_name
|
||||
is_cosmos = "cosmos" in model_name or (model_class_name is not None and "cosmos" in model_class_name.lower())
|
||||
|
||||
image = PIL.Image.open(args.image).convert("RGB") if args.image else None
|
||||
@@ -356,13 +357,23 @@ def main():
|
||||
1280 * 720,
|
||||
16,
|
||||
)
|
||||
elif is_ltx2_distilled:
|
||||
d_fps, d_guidance, d_num_frames, d_steps, d_flow_shift, d_max_area, d_mod = (
|
||||
24,
|
||||
None,
|
||||
121,
|
||||
8,
|
||||
None,
|
||||
1024 * 1536,
|
||||
64,
|
||||
)
|
||||
elif is_ltx2:
|
||||
d_fps, d_guidance, d_num_frames, d_steps, d_flow_shift, d_max_area, d_mod = (
|
||||
24,
|
||||
4.0,
|
||||
None,
|
||||
121,
|
||||
40,
|
||||
5.0,
|
||||
30 if is_ltx23 else 40,
|
||||
None,
|
||||
512 * 768,
|
||||
32,
|
||||
)
|
||||
@@ -445,7 +456,6 @@ def main():
|
||||
vae_use_slicing=args.vae_use_slicing,
|
||||
vae_use_tiling=args.vae_use_tiling,
|
||||
boundary_ratio=args.boundary_ratio,
|
||||
flow_shift=flow_shift,
|
||||
diffusion_kv_cache_dtype=args.diffusion_kv_cache_dtype,
|
||||
diffusion_kv_cache_skip_steps=args.diffusion_kv_cache_skip_steps,
|
||||
diffusion_kv_cache_skip_layers=args.diffusion_kv_cache_skip_layers,
|
||||
@@ -460,6 +470,8 @@ def main():
|
||||
)
|
||||
if args.deploy_config:
|
||||
omni_kwargs["deploy_config"] = args.deploy_config
|
||||
if flow_shift is not None:
|
||||
omni_kwargs["flow_shift"] = flow_shift
|
||||
if args.quantization is not None:
|
||||
omni_kwargs["quantization"] = args.quantization
|
||||
# Cosmos3 loads its (gated) guardrail models at build time, so the guardrails
|
||||
@@ -492,10 +504,14 @@ def main():
|
||||
print(f" Video size: {width}x{height}")
|
||||
print(f"{'=' * 60}\n")
|
||||
|
||||
negative_prompt = args.negative_prompt
|
||||
if negative_prompt is None and not is_ltx2:
|
||||
# Preserve the historical empty-prompt behavior for non-LTX examples.
|
||||
negative_prompt = ""
|
||||
prompt_dict = build_image_to_video_prompt(
|
||||
model_class_name=model_class_name,
|
||||
prompt=args.prompt,
|
||||
negative_prompt=args.negative_prompt,
|
||||
negative_prompt=negative_prompt,
|
||||
media_inputs=media_inputs,
|
||||
height=height,
|
||||
width=width,
|
||||
@@ -511,11 +527,10 @@ def main():
|
||||
num_inference_steps=num_inference_steps,
|
||||
num_frames=num_frames,
|
||||
frame_rate=frame_rate,
|
||||
extra_args={
|
||||
"sample_solver": args.sample_solver,
|
||||
"flow_shift": flow_shift,
|
||||
},
|
||||
extra_args={"sample_solver": args.sample_solver},
|
||||
)
|
||||
if flow_shift is not None:
|
||||
sampling_params.extra_args["flow_shift"] = flow_shift
|
||||
|
||||
# Route model-specific knobs through extra_body, filtered against the model's
|
||||
# declared extra_body_params. Models without a declaration only forward explicit
|
||||
|
||||
@@ -1,9 +1,8 @@
|
||||
# MiniCPM-o 4.5: Offline inference
|
||||
|
||||
Three-stage pipeline: **Thinker** → **Talker** (codec) → **Code2Wav**
|
||||
Two-stage pipeline: **thinker** (multimodal understanding) → **talker + Token2Wav**
|
||||
(24 kHz speech). Deploy config auto-loads from
|
||||
`vllm_omni/deploy/minicpmo_4_5.yaml` (single-GPU default). For higher
|
||||
throughput, pass `--deploy-config vllm_omni/deploy/minicpmo_4_5_2gpu.yaml`.
|
||||
`vllm_omni/deploy/minicpmo_4_5.yaml` (2-GPU default).
|
||||
|
||||
## Setup
|
||||
|
||||
@@ -73,7 +72,7 @@ Supported `--query-type` values:
|
||||
|
||||
```bash
|
||||
python end2end.py --query-type text \
|
||||
--deploy-config /path/to/vllm_omni/deploy/minicpmo_4_5_2gpu.yaml
|
||||
--deploy-config /path/to/vllm_omni/deploy/minicpmo_4_5_8x4090.yaml
|
||||
```
|
||||
|
||||
## Notes
|
||||
@@ -84,8 +83,7 @@ python end2end.py --query-type text \
|
||||
- Output WAV is **24 kHz mono**.
|
||||
- Placeholders in the prompt are MiniCPM-style:
|
||||
`(<image>./</image>)`, `(<audio>./</audio>)`, `(<video>./</video>)`.
|
||||
- Default layout needs **1 GPU**. Use `minicpmo_4_5_2gpu.yaml` for the
|
||||
recommended two-GPU continuous-batching layout.
|
||||
- Default layout needs **2 GPUs**. Async chunking is off in the bundled YAMLs.
|
||||
|
||||
## Online serving
|
||||
|
||||
|
||||
@@ -16,6 +16,7 @@ import numpy as np
|
||||
import soundfile as sf
|
||||
import vllm
|
||||
from PIL import Image
|
||||
from vllm import SamplingParams
|
||||
from vllm.assets.audio import AudioAsset
|
||||
from vllm.assets.image import ImageAsset
|
||||
from vllm.assets.video import VideoAsset, video_to_ndarrays
|
||||
@@ -25,6 +26,8 @@ from vllm.multimodal.media.audio import load_audio
|
||||
from vllm_omni.entrypoints.omni import Omni
|
||||
from vllm_omni.utils.tracking_parser import TrackingArgumentParser
|
||||
|
||||
SEED = 42
|
||||
|
||||
default_system = (
|
||||
"You are MiniCPM-o, a helpful multimodal assistant that can "
|
||||
"understand images, audio and video, and respond in text and speech."
|
||||
@@ -282,17 +285,27 @@ def main(args):
|
||||
omni_kwargs["trust_remote_code"] = True
|
||||
omni = Omni(**omni_kwargs)
|
||||
|
||||
# Per-stage sampling defaults live in the deploy YAML
|
||||
# (``vllm_omni/deploy/minicpmo_4_5*.yaml``, ``default_sampling_params``),
|
||||
# which is the single source of truth that tracks the pipeline topology.
|
||||
# Passing None makes ``resolve_sampling_params_list`` load exactly those,
|
||||
# so adding or removing a stage cannot desynchronize this example.
|
||||
#
|
||||
# This used to hardcode a two-element list built for the old two-stage
|
||||
# topology (thinker → talker+Token2Wav). Since Code2Wav became its own
|
||||
# Stage 2, that list was both the wrong length and wrong in content
|
||||
# (talker ``max_tokens=1`` truncated the codec stream to one frame).
|
||||
sampling_params_list = None
|
||||
# Stage 0 (thinker): multimodal understanding → text (+ TTS span when enabled).
|
||||
thinker_sampling_params = SamplingParams(
|
||||
temperature=0.0,
|
||||
top_p=1.0,
|
||||
top_k=-1,
|
||||
max_tokens=2048,
|
||||
seed=SEED,
|
||||
detokenize=True,
|
||||
repetition_penalty=1.1,
|
||||
)
|
||||
# Stage 1 (talker + Token2Wav): max_tokens=1 satisfies the scheduler;
|
||||
# waveform is produced in-process by Token2Wav.
|
||||
talker_sampling_params = SamplingParams(
|
||||
temperature=0.0,
|
||||
top_p=1.0,
|
||||
top_k=-1,
|
||||
max_tokens=1,
|
||||
seed=SEED,
|
||||
detokenize=False,
|
||||
)
|
||||
sampling_params_list = [thinker_sampling_params, talker_sampling_params][: omni.num_stages]
|
||||
|
||||
if args.txt_prompts is None:
|
||||
prompts = [query_result.inputs for _ in range(args.num_prompts)]
|
||||
|
||||
@@ -8,7 +8,7 @@ A unified script for text-to-video generation. Supports multiple models with mod
|
||||
|---|---|---|---|---|---|
|
||||
| `Wan-AI/Wan2.1-VACE-1.3B-diffusers` | 480x832 | 81 | 30 | 5.0 | ~20 GiB (RTX 5090, VAE tiling) |
|
||||
| `Wan-AI/Wan2.2-T2V-A14B-Diffusers` | 720x1280 | 81 | 40 | 4.0 | ~60 GiB |
|
||||
| `diffusers/LTX-2.3-Diffusers` | 384x512 | 25 | 20 | 4.0 | 96GB-class GPU |
|
||||
| `Lightricks/LTX-2` | 512x768 | 121 | 40 | video 3.0 / audio 7.0 | Model-dependent |
|
||||
| `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-480p_t2v` | 480x832 | 121 | 50 | 6.0 | 1×A100 80GB |
|
||||
| `hunyuanvideo-community/HunyuanVideo-1.5-Diffusers-720p_t2v` | 720x1280 | 121 | 50 | 6.0 | FP8 + VAE tiling required |
|
||||
| `nvidia/Cosmos3-Nano` | 720x1280 | 189 | 35 | 6.0 | ~46 GiB (peak, 720p) |
|
||||
@@ -55,45 +55,17 @@ python text_to_video.py \
|
||||
--output vace_t2v_output.mp4
|
||||
```
|
||||
|
||||
LTX2 example:
|
||||
### LTX-2
|
||||
|
||||
```bash
|
||||
python text_to_video.py \
|
||||
--model "Lightricks/LTX-2" \
|
||||
--prompt "A cinematic close-up of ocean waves at golden hour." \
|
||||
--negative-prompt "worst quality, inconsistent motion, blurry, jittery, distorted" \
|
||||
--height 512 \
|
||||
--width 768 \
|
||||
--num-frames 121 \
|
||||
--num-inference-steps 40 \
|
||||
--guidance-scale 4.0 \
|
||||
--frame-rate 24 \
|
||||
--output ltx2_out.mp4
|
||||
```
|
||||
|
||||
### LTX-2.3
|
||||
|
||||
```bash
|
||||
python text_to_video.py \
|
||||
--model diffusers/LTX-2.3-Diffusers \
|
||||
--model-class-name LTX23Pipeline \
|
||||
--model Lightricks/LTX-2 \
|
||||
--prompt "Cherry blossoms swaying gently in the breeze with synchronized ambient sound" \
|
||||
--negative-prompt "worst quality, inconsistent motion, blurry, jittery, distorted" \
|
||||
--height 384 \
|
||||
--width 512 \
|
||||
--num-frames 25 \
|
||||
--num-inference-steps 20 \
|
||||
--guidance-scale 4.0 \
|
||||
--frame-rate 24 \
|
||||
--fps 24 \
|
||||
--audio-sample-rate 48000 \
|
||||
--output ltx23_t2v_output.mp4
|
||||
--output ltx2_output.mp4
|
||||
```
|
||||
|
||||
Use the Diffusers-format checkpoint `diffusers/LTX-2.3-Diffusers`; the
|
||||
upstream `Lightricks/LTX-2.3` raw safetensors repo is not directly loadable by
|
||||
this pipeline. Pass `--model-class-name LTX23Pipeline` to select the LTX-2.3
|
||||
text-to-video pipeline explicitly.
|
||||
See the [LTX-2 recipe](../../../recipes/LTX/LTX-2.md) for all checkpoints,
|
||||
pipeline selection, I2V, defaults, and advanced options.
|
||||
|
||||
### HunyuanVideo-1.5 (480p)
|
||||
|
||||
@@ -212,8 +184,7 @@ python text_to_video.py \
|
||||
### Common
|
||||
|
||||
- `--model`: Diffusers model ID or local path.
|
||||
- `--model-class-name`: Optional explicit pipeline class. Use `LTX23Pipeline`
|
||||
for LTX-2.3 text-to-video.
|
||||
- `--model-class-name`: Optional explicit pipeline override.
|
||||
- `--prompt`: text description (string).
|
||||
- `--height/--width`: output resolution. Default depends on model.
|
||||
- `--num-frames`: number of frames. Default depends on model.
|
||||
@@ -229,8 +200,7 @@ python text_to_video.py \
|
||||
- `--enable-cpu-offload`: enable CPU offloading for diffusion models.
|
||||
- `--enable-layerwise-offload`: enable layerwise offloading on DiT modules.
|
||||
- `--frame-rate`: generation FPS for pipelines that require it (e.g., LTX2).
|
||||
- `--audio-sample-rate`: audio sample rate for embedded audio (when the
|
||||
pipeline returns audio; LTX-2.3 outputs 48kHz audio).
|
||||
- `--audio-sample-rate`: fallback audio sample rate when the pipeline returns audio.
|
||||
- `--quantization`: quantization method (such as `fp8` for FP8).
|
||||
- `--flow-shift`: scheduler flow_shift parameter.
|
||||
- `--extra-body`: JSON object of model-specific generation params, filtered against the model's declared `extra_body_params` (see [`vllm_omni/model_extras`](../../../vllm_omni/model_extras)). Used by Cosmos3 (see above).
|
||||
|
||||
@@ -65,11 +65,42 @@ _MODEL_PRESETS = {
|
||||
"fps": 16,
|
||||
"output": "helios_output.mp4",
|
||||
},
|
||||
"ltx2": {
|
||||
"height": 512,
|
||||
"width": 768,
|
||||
"num_frames": 121,
|
||||
"num_inference_steps": 40,
|
||||
"fps": 24,
|
||||
"output": "ltx2_output.mp4",
|
||||
},
|
||||
"ltx2_distilled": {
|
||||
"height": 1024,
|
||||
"width": 1536,
|
||||
"num_frames": 121,
|
||||
"num_inference_steps": 8,
|
||||
"fps": 24,
|
||||
"output": "ltx2_distilled_output.mp4",
|
||||
},
|
||||
"ltx23": {
|
||||
"height": 512,
|
||||
"width": 768,
|
||||
"num_frames": 121,
|
||||
"num_inference_steps": 30,
|
||||
"fps": 24,
|
||||
"output": "ltx23_output.mp4",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _detect_preset(model: str) -> dict:
|
||||
def _detect_preset(model: str, model_class_name: str | None = None) -> dict:
|
||||
model_lower = model.lower()
|
||||
class_lower = (model_class_name or "").lower()
|
||||
if "distilled" in class_lower or "distilled" in model_lower:
|
||||
return _MODEL_PRESETS["ltx2_distilled"]
|
||||
if "ltx23" in class_lower or "ltx-2.3" in model_lower or "ltx_2.3" in model_lower:
|
||||
return _MODEL_PRESETS["ltx23"]
|
||||
if "ltx2" in class_lower or "ltx-2" in model_lower or "ltx_2" in model_lower:
|
||||
return _MODEL_PRESETS["ltx2"]
|
||||
if "vace" in model_lower:
|
||||
return _MODEL_PRESETS["vace"]
|
||||
if "cosmos" in model_lower:
|
||||
@@ -122,7 +153,7 @@ def parse_args() -> argparse.Namespace:
|
||||
parser.add_argument(
|
||||
"--model-class-name",
|
||||
default=None,
|
||||
help="Override model class name (e.g., LTX2TwoStagesVideoPipeline).",
|
||||
help="Override model class name (e.g., LTX2Pipeline).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--deploy-config",
|
||||
@@ -130,7 +161,7 @@ def parse_args() -> argparse.Namespace:
|
||||
help="Optional deploy config YAML to use for pipeline-backed runs.",
|
||||
)
|
||||
parser.add_argument("--prompt", default="A serene lakeside sunrise with mist over the water.", help="Text prompt.")
|
||||
parser.add_argument("--negative-prompt", default="", help="Negative prompt.")
|
||||
parser.add_argument("--negative-prompt", default=None, help="Negative prompt. Default: model-specific.")
|
||||
parser.add_argument(
|
||||
"--extra-body",
|
||||
type=parse_extra_body,
|
||||
@@ -331,7 +362,7 @@ def main():
|
||||
args = parse_args()
|
||||
model_class_name = args.model_class_name
|
||||
|
||||
preset = _detect_preset(args.model)
|
||||
preset = _detect_preset(args.model, model_class_name)
|
||||
for key, default_val in preset.items():
|
||||
if getattr(args, key.replace("-", "_"), None) is None:
|
||||
setattr(args, key.replace("-", "_"), default_val)
|
||||
@@ -422,8 +453,11 @@ def main():
|
||||
print(f"{'=' * 60}\n")
|
||||
|
||||
prompt_dict = {"prompt": args.prompt}
|
||||
if args.negative_prompt:
|
||||
if args.negative_prompt is not None:
|
||||
prompt_dict["negative_prompt"] = args.negative_prompt
|
||||
elif preset not in (_MODEL_PRESETS["ltx2"], _MODEL_PRESETS["ltx23"]):
|
||||
# Preserve the historical empty-prompt behavior for non-LTX examples.
|
||||
prompt_dict["negative_prompt"] = ""
|
||||
|
||||
sampling_kwargs = dict(
|
||||
height=args.height,
|
||||
|
||||
@@ -0,0 +1,89 @@
|
||||
# Audex (Nemotron-Labs-Audex-2B) online serving
|
||||
|
||||
One checkpoint, four deployment modes. `run_server.sh` starts the server
|
||||
for a given `MODE`; `client.py --mode <mode>` tests it. The capability
|
||||
matrix:
|
||||
|
||||
| mode (`vllm_omni/deploy/audex_<mode>.yaml`) | audio in | text out | speech out | general audio out | endpoint |
|
||||
|---|---|---|---|---|---|
|
||||
| `tts` | ❌ | ❌ | ✅ | ❌ | `/v1/audio/speech` |
|
||||
| `tta` | ❌ | ❌ | ❌ | ✅ | `/v1/audio/speech` |
|
||||
| `thinker_only` | ✅ | ✅ | ❌ | ❌ | `/v1/chat/completions` |
|
||||
| `s2s` | ✅ | ✅ | ✅ | ❌ | both |
|
||||
|
||||
Audio-input modes of `client.py` fall back to vLLM's public
|
||||
`mary_had_lamb` asset when `--audio-file` is omitted; CFG defaults follow
|
||||
the official settings per mode (tts/s2s 1.5, tta 3.0; `--cfg-scale 1.0`
|
||||
disables).
|
||||
|
||||
## tts — text → speech
|
||||
|
||||
Text-only thinker (`checkpoint_folder_audiogen`) emits `<speechcodec_N>`
|
||||
tokens; the streaming causal decoder produces 16 kHz WAVs. This is also
|
||||
the DEFAULT pipeline when the repo root is served without a deploy config.
|
||||
|
||||
```bash
|
||||
./run_server.sh # MODE=tts, port 8097
|
||||
|
||||
python client.py --mode tts --text "Hello there." --output hello.wav
|
||||
# or raw curl:
|
||||
curl -s http://localhost:8097/v1/audio/speech \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d '{"model": "nvidia/Nemotron-Labs-Audex-2B", "input": "Hello there.", "response_format": "wav", "extra_params": {"cfg_scale": 1.5}}' \
|
||||
-o hello.wav
|
||||
```
|
||||
|
||||
## tta — caption → general audio
|
||||
|
||||
Same thinker over the 4-codebook `<audiocodec_N>` RVQ block, decoded by
|
||||
the external XCodec1 checkpoint (auto-downloaded; override with
|
||||
`XCODEC1_PATH`). Clips are capped at 10 s.
|
||||
|
||||
```bash
|
||||
MODE=tta ./run_server.sh
|
||||
|
||||
python client.py --mode tta --caption "Heavy rain falling on a tin roof." \
|
||||
--output rain.wav
|
||||
```
|
||||
|
||||
## thinker_only — audio (+ instruction) → text
|
||||
|
||||
Single-stage audio understanding on the full checkpoint (NV-Whisper
|
||||
encoder + projector + LM). Send the clip as `input_audio` chat content.
|
||||
|
||||
```bash
|
||||
MODE=thinker_only ./run_server.sh
|
||||
|
||||
python client.py --mode thinker_only # transcribe the demo asset
|
||||
python client.py --mode thinker_only --audio-file a.wav \
|
||||
--question "What language is being spoken?"
|
||||
```
|
||||
|
||||
## s2s — spoken question → spoken answer
|
||||
|
||||
The audio-capable thinker plus the speech decoder in one deployment.
|
||||
`client.py --mode s2s` runs the official three passes: ASR (chat with
|
||||
`input_audio`) → chat on the transcript → TTS on the answer. Text-final
|
||||
passes carry `"modalities": ["text"]`; only the TTS pass streams through
|
||||
the decoder.
|
||||
|
||||
```bash
|
||||
MODE=s2s PORT=8098 ./run_server.sh
|
||||
|
||||
python client.py --mode s2s --port 8098 \
|
||||
--audio-file question.wav --output answer.wav
|
||||
```
|
||||
|
||||
Offline counterparts (no HTTP, one `Omni` engine per script) live in
|
||||
`examples/offline_inference/audex/`.
|
||||
|
||||
## 30B-A3B (nvidia/Nemotron-Labs-Audex-30B-A3B)
|
||||
|
||||
`SIZE=30b` switches run_server.sh to the 30B yamls and model id:
|
||||
|
||||
SIZE=30b ./run_server.sh # 30B tts on port 8097
|
||||
SIZE=30b MODE=s2s PORT=8098 ./run_server.sh
|
||||
|
||||
Pass `--model nvidia/Nemotron-Labs-Audex-30B-A3B` to the client — the server
|
||||
validates the model id and 404s on a mismatch. Single-H100 defaults; see the
|
||||
yaml comments for the TP2 fallback. First launch downloads ~60 GB.
|
||||
@@ -0,0 +1,319 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Audex online client for all four deployment modes.
|
||||
|
||||
Pick ``--mode`` to match the server's deploy yaml (see run_server.sh):
|
||||
|
||||
tts /v1/audio/speech: text -> speech WAV
|
||||
tta /v1/audio/speech: caption -> general-audio WAV
|
||||
thinker_only /v1/chat/completions: audio (+ question) -> text
|
||||
s2s the official three-pass cascade against one audex_s2s
|
||||
server: ASR (chat, input_audio) -> chat -> TTS
|
||||
|
||||
Audio-input modes fall back to vLLM's public ``mary_had_lamb`` asset when
|
||||
``--audio-file`` is omitted.
|
||||
|
||||
Examples:
|
||||
|
||||
python examples/online_serving/audex/client.py --mode tts \\
|
||||
--text "Hello there." --output hello.wav
|
||||
python examples/online_serving/audex/client.py --mode tta \\
|
||||
--caption "Heavy rain falling on a tin roof." --output rain.wav
|
||||
python examples/online_serving/audex/client.py --mode thinker_only
|
||||
python examples/online_serving/audex/client.py --mode s2s \\
|
||||
--audio-file question.wav --output answer.wav
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import base64
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
import requests
|
||||
|
||||
ASR_QUESTION = "Transcribe the input speech."
|
||||
SYSTEM_PROMPT = "You are a helpful and harmless assistant.\n\nYou are not allowed to use any tools."
|
||||
# The official chat-pass recipe below (SYSTEM_PROMPT + TEXT_INSTRUCTION
|
||||
# prefix, reasoning enabled, temp 1.0/top_p 0.95, seed retry) applies to the
|
||||
# 30B-A3B ONLY (see is_30b() below and
|
||||
# tests/e2e/offline_inference/test_audex_s2s.py, which mirrors this same
|
||||
# split). It is load-bearing on the 30B: without the formatting-instruction
|
||||
# prefix a bare transcript turn is treated as a TTS request and the model
|
||||
# emits literal <speechcodec_N> tokens instead of a text answer (the system
|
||||
# prompt alone does not prevent this). The 2B must NOT use this recipe: the
|
||||
# official temp-1.0 seed-0 draw measurably hallucinates off-language text on
|
||||
# the 2B (yet closes </think> cleanly, so the seed retry never fires) and
|
||||
# fails the round-trip WER gate — measured, reproduced byte-identically. The
|
||||
# 2B instead keeps its original bare-message / enable_thinking=False /
|
||||
# temperature=0.0 recipe.
|
||||
TEXT_INSTRUCTION = (
|
||||
"SYSTEM & FORMATTING INSTRUCTION: You are Audex, created by NVIDIA based on the Nemotron-Cascade-2 architecture. "
|
||||
"You may output your reasoning, followed by your final response. "
|
||||
"You may format your reasoning block however you like. "
|
||||
"However, your final response must strictly follow these rules: "
|
||||
"* Write in plain, unformatted prose like a book or newspaper article. "
|
||||
"* Do not use markdown, bullet points, lists, or headers. "
|
||||
"* Standard numbers, abbreviations, and symbols are acceptable. "
|
||||
"* [CRITICAL] You must press enter after every single sentence, placing each sentence on its own separate line."
|
||||
)
|
||||
DEFAULT_TEXT = "Hello there! This is the Audex text to speech pipeline."
|
||||
DEFAULT_CAPTION = "Heavy rain falling on a tin roof."
|
||||
# Official quality settings; 1.0 disables guidance.
|
||||
DEFAULT_CFG = {"tts": 1.5, "tta": 3.0, "s2s": 1.5}
|
||||
|
||||
|
||||
def is_30b(model: str) -> bool:
|
||||
# Users pass arbitrary repo IDs or local snapshot paths, so this is a
|
||||
# heuristic (not an exact match against the official 30B-A3B repo ID)
|
||||
# — good enough for an example script; keep in sync with the same
|
||||
# branch in examples/offline_inference/audex/speech_to_speech.py and
|
||||
# tests/e2e/offline_inference/test_audex_s2s.py.
|
||||
return "30b" in model.lower()
|
||||
|
||||
|
||||
def parse_args():
|
||||
parser = argparse.ArgumentParser(description="Audex online client (tts / tta / thinker_only / s2s)")
|
||||
parser.add_argument("--mode", choices=("tts", "tta", "thinker_only", "s2s"), default="s2s")
|
||||
parser.add_argument("--host", type=str, default="127.0.0.1")
|
||||
parser.add_argument("--port", type=int, default=8097)
|
||||
parser.add_argument("--model", type=str, default="nvidia/Nemotron-Labs-Audex-2B")
|
||||
parser.add_argument("--text", type=str, default=DEFAULT_TEXT, help="TTS input text (mode=tts).")
|
||||
parser.add_argument("--caption", type=str, default=DEFAULT_CAPTION, help="TTA caption (mode=tta).")
|
||||
parser.add_argument(
|
||||
"--audio-file",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Input WAV for thinker_only/s2s. Defaults to vLLM's mary_had_lamb asset.",
|
||||
)
|
||||
parser.add_argument("--question", type=str, default=ASR_QUESTION, help="Instruction (mode=thinker_only).")
|
||||
parser.add_argument("--output", type=str, default="results/audex_online.wav")
|
||||
parser.add_argument(
|
||||
"--cfg-scale",
|
||||
type=float,
|
||||
default=None,
|
||||
help="CFG strength; defaults per mode (tts/s2s 1.5, tta 3.0). 1.0 disables.",
|
||||
)
|
||||
parser.add_argument("--max-tokens", type=int, default=512)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def _audio_b64(audio_file: str | None) -> str:
|
||||
if audio_file:
|
||||
return base64.b64encode(Path(audio_file).read_bytes()).decode()
|
||||
# Fall back to vLLM's public asset, re-encoded as WAV for the chat API.
|
||||
import io
|
||||
|
||||
import numpy as np
|
||||
import soundfile as sf
|
||||
from vllm.assets.audio import AudioAsset
|
||||
|
||||
audio, sr = AudioAsset("mary_had_lamb").audio_and_sample_rate
|
||||
buf = io.BytesIO()
|
||||
sf.write(buf, np.asarray(audio, dtype="float32"), int(sr), format="WAV")
|
||||
return base64.b64encode(buf.getvalue()).decode()
|
||||
|
||||
|
||||
def _chat(
|
||||
base_url: str,
|
||||
model: str,
|
||||
messages: list[dict],
|
||||
max_tokens: int,
|
||||
enable_thinking: bool = False,
|
||||
temperature: float = 0.0,
|
||||
top_p: float = 1.0,
|
||||
seed: int = 0,
|
||||
) -> tuple[str, str]:
|
||||
resp = requests.post(
|
||||
f"{base_url}/v1/chat/completions",
|
||||
json={
|
||||
"model": model,
|
||||
"messages": messages,
|
||||
"max_tokens": max_tokens,
|
||||
"temperature": temperature,
|
||||
"top_p": top_p,
|
||||
"seed": seed,
|
||||
# Text-final passes must say so explicitly: without "modalities"
|
||||
# the server falls back to the deployment's configured output
|
||||
# modalities, which for the s2s pipeline include the audio final
|
||||
# stage — routing ASR/chat through code2wav instead of stopping
|
||||
# at stage 0.
|
||||
"modalities": ["text"],
|
||||
# The checkpoint's chat template thinking toggle. The official
|
||||
# cascaded demo transcribes with enable_thinking=False but runs
|
||||
# the chat pass with reasoning ENABLED (its web UI ships the
|
||||
# "Enable reasoning" checkbox checked), priming the generation
|
||||
# prompt with an open "<think>\n" so the model's reasoning stays
|
||||
# inside the think block; _extract_final_response then drops it.
|
||||
# With thinking disabled the 30B leaks its reasoning as plain
|
||||
# text in front of the final response.
|
||||
"chat_template_kwargs": {"enable_thinking": enable_thinking},
|
||||
},
|
||||
timeout=300,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
choice = resp.json()["choices"][0]
|
||||
return choice["message"]["content"].strip(), choice.get("finish_reason") or ""
|
||||
|
||||
|
||||
def _speech(base_url: str, model: str, text: str, cfg_scale: float, out_path: Path) -> None:
|
||||
payload: dict = {"model": model, "input": text, "response_format": "wav"}
|
||||
if cfg_scale > 1.0:
|
||||
payload["extra_params"] = {"cfg_scale": cfg_scale}
|
||||
resp = requests.post(f"{base_url}/v1/audio/speech", json=payload, timeout=600)
|
||||
resp.raise_for_status()
|
||||
out_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
out_path.write_bytes(resp.content)
|
||||
|
||||
|
||||
def _audio_question(audio_b64: str, question: str) -> list[dict]:
|
||||
return [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "input_audio", "input_audio": {"data": audio_b64, "format": "wav"}},
|
||||
{"type": "text", "text": question},
|
||||
],
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
def _tts_text(answer: str) -> str:
|
||||
# The formatting instruction's CRITICAL rule puts each final-response
|
||||
# sentence on its own line, and the official demo TTS-es those sentence
|
||||
# segments (split_sentence_segments). Mirror that per-line segmentation
|
||||
# but keep a single /v1/audio/speech request: speak the leading whole
|
||||
# sentences of the CLEANED final response up to ~200 chars, never
|
||||
# cutting mid-sentence. Markdown markup is stripped as before (it makes
|
||||
# the TTS thinker stop almost immediately).
|
||||
sentences = []
|
||||
for line in answer.splitlines():
|
||||
line = re.sub(r"[*#_`]+", "", line)
|
||||
line = re.sub(r"\s+", " ", line).strip()
|
||||
if line:
|
||||
sentences.append(line)
|
||||
picked: list[str] = []
|
||||
total = 0
|
||||
for sentence in sentences:
|
||||
if picked and total + len(sentence) > 200:
|
||||
break
|
||||
picked.append(sentence)
|
||||
total += len(sentence) + 1
|
||||
return " ".join(picked)
|
||||
|
||||
|
||||
def _extract_final_response(text: str) -> str:
|
||||
# Mirror the official demo's extract_spoken_answer
|
||||
# (cascaded_s2s_web_server.py): the chat pass reasons inside the primed
|
||||
# <think> block, so keep only what follows the LAST closing tag; fall
|
||||
# back to the full text when the model skipped reasoning or never
|
||||
# closed the block. Benign no-op for the 2B's clean answers.
|
||||
if "</think>" in text:
|
||||
answer = text.rsplit("</think>", 1)[-1].strip()
|
||||
if answer:
|
||||
return answer
|
||||
return text.strip()
|
||||
|
||||
|
||||
def _clean_transcript(text: str) -> str:
|
||||
# The full checkpoint prefixes transcripts with a language-tag sentence
|
||||
# and quotes the content; extract the quoted payload when present. Span
|
||||
# first-to-last quote (not split on every apostrophe) so contractions
|
||||
# inside the transcript (e.g. "don't") survive intact.
|
||||
text = text.strip()
|
||||
match = re.search(r"'(.*)'", text, flags=re.S)
|
||||
if match and match.group(1).strip():
|
||||
return match.group(1).strip()
|
||||
return text
|
||||
|
||||
|
||||
def main():
|
||||
args = parse_args()
|
||||
base_url = f"http://{args.host}:{args.port}"
|
||||
cfg_scale = args.cfg_scale if args.cfg_scale is not None else DEFAULT_CFG.get(args.mode, 1.0)
|
||||
out_path = Path(args.output)
|
||||
|
||||
if args.mode == "tts":
|
||||
_speech(base_url, args.model, args.text, cfg_scale, out_path)
|
||||
print(f"speech: {out_path.stat().st_size} bytes -> {out_path}")
|
||||
return
|
||||
|
||||
if args.mode == "tta":
|
||||
_speech(base_url, args.model, args.caption, cfg_scale, out_path)
|
||||
print(f"audio : {out_path.stat().st_size} bytes -> {out_path}")
|
||||
return
|
||||
|
||||
audio_b64 = _audio_b64(args.audio_file)
|
||||
|
||||
if args.mode == "thinker_only":
|
||||
content, _ = _chat(base_url, args.model, _audio_question(audio_b64, args.question), args.max_tokens)
|
||||
print(f"answer: {_extract_final_response(content)}")
|
||||
return
|
||||
|
||||
# s2s — the official three-pass cascade.
|
||||
asr_content, _ = _chat(base_url, args.model, _audio_question(audio_b64, ASR_QUESTION), args.max_tokens)
|
||||
transcript = _clean_transcript(asr_content)
|
||||
print(f"[1/3] transcript : {transcript}")
|
||||
if not transcript:
|
||||
raise SystemExit("ASR pass produced an empty transcript; aborting cascade")
|
||||
|
||||
# The chat-pass recipe is size-conditional (see the TEXT_INSTRUCTION
|
||||
# comment above): the 30B requires the official recipe, the 2B keeps its
|
||||
# original bare-message recipe.
|
||||
if is_30b(args.model):
|
||||
# Official recipe: system prompt + formatting instruction prefixed
|
||||
# to the transcript, reasoning enabled, and the official demo's
|
||||
# text-pass sampling (its defaults: --text-temperature 1.0,
|
||||
# --text-top-p 0.95, --text-max-tokens 2048). Near-greedy decoding
|
||||
# makes the 30B loop inside its reasoning block until the token
|
||||
# budget runs out without ever closing </think>. Even at the
|
||||
# official temperature some trajectories spin past the budget, so
|
||||
# retry a few deterministic seeds; a natural stop without </think>
|
||||
# is a complete answer, and if every seed spins we keep the full
|
||||
# text like the official demo does.
|
||||
chat_messages = [
|
||||
{"role": "system", "content": SYSTEM_PROMPT},
|
||||
{"role": "user", "content": f"{TEXT_INSTRUCTION}\n\n{transcript}"},
|
||||
]
|
||||
raw = ""
|
||||
for seed in (0, 1, 2):
|
||||
raw, finish_reason = _chat(
|
||||
base_url,
|
||||
args.model,
|
||||
chat_messages,
|
||||
max(args.max_tokens, 2048),
|
||||
enable_thinking=True,
|
||||
temperature=1.0,
|
||||
top_p=0.95,
|
||||
seed=seed,
|
||||
)
|
||||
if "</think>" in raw or finish_reason != "length":
|
||||
break
|
||||
print(f"[2/3] chat seed {seed} hit the token budget mid-reasoning; retrying")
|
||||
answer = _extract_final_response(raw)
|
||||
else:
|
||||
# 2B: bare user message (no system/instruction prefix), reasoning
|
||||
# disabled, temperature 0.0, no retry — its original recipe. The
|
||||
# official temp-1.0 seed-0 recipe measurably makes the 2B
|
||||
# hallucinate off-language answers (see the module-level comment).
|
||||
chat_messages = [{"role": "user", "content": transcript}]
|
||||
raw, _ = _chat(
|
||||
base_url,
|
||||
args.model,
|
||||
chat_messages,
|
||||
args.max_tokens,
|
||||
enable_thinking=False,
|
||||
temperature=0.0,
|
||||
)
|
||||
answer = _extract_final_response(raw)
|
||||
print(f"[2/3] answer : {answer}")
|
||||
if not answer:
|
||||
raise SystemExit("Chat pass produced an empty answer; aborting cascade")
|
||||
|
||||
_speech(base_url, args.model, _tts_text(answer), cfg_scale, out_path)
|
||||
print(f"[3/3] speech : {out_path.stat().st_size} bytes -> {out_path}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Executable
+57
@@ -0,0 +1,57 @@
|
||||
#!/bin/bash
|
||||
# Launch the vLLM-Omni server for Audex (Nemotron-Labs-Audex-2B).
|
||||
#
|
||||
# MODE picks the deployment (see README.md for the capability matrix):
|
||||
# tts text -> speech /v1/audio/speech (default)
|
||||
# tta caption -> general audio /v1/audio/speech
|
||||
# thinker_only audio -> text /v1/chat/completions
|
||||
# s2s cascaded speech-to-speech (both endpoints)
|
||||
#
|
||||
# Pass the HF repo ROOT as MODEL — per-stage subfolders resolve
|
||||
# automatically.
|
||||
#
|
||||
# Usage:
|
||||
# ./run_server.sh # 2B tts on port 8097, GPU 0
|
||||
# SIZE=30b ./run_server.sh # 30B-A3B tts (explicit 30B yaml)
|
||||
# SIZE=30b MODE=s2s PORT=8098 ./run_server.sh
|
||||
# MODE=tta ./run_server.sh # needs XCodec1 (auto-downloaded)
|
||||
# MODEL=/path/to/local/snapshot ./run_server.sh
|
||||
|
||||
set -e
|
||||
|
||||
MODE="${MODE:-${1:-tts}}"
|
||||
SIZE="${SIZE:-2b}"
|
||||
if [ "$SIZE" = "30b" ]; then
|
||||
MODEL="${MODEL:-nvidia/Nemotron-Labs-Audex-30B-A3B}"
|
||||
YAML_SUFFIX="_30b"
|
||||
elif [ "$SIZE" = "2b" ]; then
|
||||
MODEL="${MODEL:-nvidia/Nemotron-Labs-Audex-2B}"
|
||||
YAML_SUFFIX=""
|
||||
else
|
||||
echo "Unknown SIZE '$SIZE'; expected 2b|30b" >&2
|
||||
exit 1
|
||||
fi
|
||||
PORT="${PORT:-8097}"
|
||||
GPUS="${GPUS:-0}"
|
||||
|
||||
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)"
|
||||
DEPLOY_YAML="$REPO_ROOT/vllm_omni/deploy/audex_${MODE}${YAML_SUFFIX}.yaml"
|
||||
if [ ! -f "$DEPLOY_YAML" ]; then
|
||||
echo "Unknown MODE '$MODE' / SIZE '$SIZE' (no $DEPLOY_YAML); expected MODE=tts|tta|thinker_only|s2s, SIZE=2b|30b" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Starting Audex server"
|
||||
echo " MODE=$MODE ($DEPLOY_YAML)"
|
||||
echo " SIZE=$SIZE"
|
||||
echo " MODEL=$MODEL"
|
||||
echo " PORT=$PORT"
|
||||
echo " CUDA_VISIBLE_DEVICES=$GPUS"
|
||||
|
||||
CUDA_VISIBLE_DEVICES="$GPUS" \
|
||||
vllm-omni serve "$MODEL" \
|
||||
--host 0.0.0.0 \
|
||||
--port "$PORT" \
|
||||
--trust-remote-code \
|
||||
--stage-configs-path "$DEPLOY_YAML" \
|
||||
--omni
|
||||
@@ -23,17 +23,17 @@ including `librosa`.
|
||||
|
||||
The deploy config auto-loads via `--omni`.
|
||||
The default `vllm_omni/deploy/minicpmo_4_5.yaml` keeps all three stages on
|
||||
logical device 0 with memory budgets of 55%, 15%, and 18%, leaving headroom
|
||||
for runtime kernels such as the HiFi-GAN vocoder's cuDNN workspace. The
|
||||
profile admits at most four sequences per stage. For throughput,
|
||||
`minicpmo_4_5_2gpu.yaml` gives the Thinker GPU 0 (90%) and colocates the
|
||||
Talker (55%) and Code2Wav (35%) on GPU 1. That profile admits at most four
|
||||
concurrent sequences per stage.
|
||||
logical device 0 with memory budgets of 55%, 22%, and 22%. The profile admits
|
||||
at most four sequences per stage and bounds startup video profiling to 32
|
||||
frames per video. For throughput,`minicpmo_4_5_batching.yaml` gives the Thinker
|
||||
GPU 0 (90%) and colocates the Talker (55%) and Code2Wav (35%) on GPU 1. That
|
||||
profile admits at most four concurrent sequences per stage.
|
||||
|
||||
| deploy config | GPUs | Notes |
|
||||
|---|---|---|
|
||||
| `minicpmo_4_5.yaml` (default) | 1 | Memory-constrained compatibility layout. |
|
||||
| `minicpmo_4_5_2gpu.yaml` | 2 | Recommended continuous-batching layout; Talker and Code2Wav share GPU 1. |
|
||||
| `minicpmo_4_5_batching.yaml` | 2 | Recommended continuous-batching layout; Talker and Code2Wav share GPU 1. |
|
||||
| `minicpmo_4_5_2gpu.yaml` | 2 | Backward-compatible alias for `minicpmo_4_5_batching.yaml`. |
|
||||
| `minicpmo_4_5_3gpu.yaml` | 3 | One GPU per stage. |
|
||||
| `minicpmo_4_5_8x4090.yaml` | 8 | Full 8x4090 layout. |
|
||||
| `minicpmo_4_5_duplex.yaml` | 1 | Experimental native full-duplex overlay. |
|
||||
@@ -89,18 +89,17 @@ python openai_chat_completion_client_for_multimodal_generation.py \
|
||||
--prompt "Briefly introduce yourself."
|
||||
```
|
||||
|
||||
Streaming text + audio (use `--stream`):
|
||||
Streaming text + audio:
|
||||
|
||||
```bash
|
||||
python openai_chat_completion_client_for_multimodal_generation.py \
|
||||
--query-type text \
|
||||
--prompt "Briefly introduce yourself." \
|
||||
--port 8099 \
|
||||
--stream
|
||||
python streaming_chat_completion.py \
|
||||
--base-url http://localhost:8099/v1 \
|
||||
--output minicpmo_stream.wav
|
||||
```
|
||||
|
||||
The client prints text deltas as they arrive and saves streamed audio chunks
|
||||
to WAV files.
|
||||
The example prints text deltas immediately and joins the independently encoded
|
||||
24 kHz WAV audio deltas into one valid WAV file. Add `--text-only` to skip
|
||||
Talker and Code2Wav.
|
||||
|
||||
Shared helpers also work if you pass MiniCPM defaults yourself:
|
||||
|
||||
|
||||
@@ -6,7 +6,6 @@ class FullDuplexPcmPlayback extends AudioWorkletProcessor {
|
||||
this.playedFrames = 0;
|
||||
this.underrunFrames = 0;
|
||||
this.drain = null;
|
||||
this.terminalFadeFrames = 0;
|
||||
this.started = false;
|
||||
this.activeResponseId = null;
|
||||
this.initialBufferFrames = Math.round(sampleRate * 0.2);
|
||||
@@ -32,7 +31,6 @@ class FullDuplexPcmPlayback extends AudioWorkletProcessor {
|
||||
}
|
||||
} else if (message.type === 'drain') {
|
||||
this.drain = { responseId: message.responseId || null };
|
||||
this.terminalFadeFrames = Math.min(this.fadeFrames, this.bufferedFrames());
|
||||
if (!this.started && this.bufferedFrames() > 0) {
|
||||
this.bufferWaitFrames = 0;
|
||||
this.startPlayback();
|
||||
@@ -44,7 +42,6 @@ class FullDuplexPcmPlayback extends AudioWorkletProcessor {
|
||||
this.playedFrames = 0;
|
||||
this.underrunFrames = 0;
|
||||
this.drain = null;
|
||||
this.terminalFadeFrames = 0;
|
||||
this.started = false;
|
||||
this.activeResponseId = null;
|
||||
this.bufferWaitFrames = this.initialBufferFrames;
|
||||
@@ -79,7 +76,6 @@ class FullDuplexPcmPlayback extends AudioWorkletProcessor {
|
||||
this.playedFrames = 0;
|
||||
this.underrunFrames = 0;
|
||||
this.drain = null;
|
||||
this.terminalFadeFrames = 0;
|
||||
this.started = false;
|
||||
this.activeResponseId = null;
|
||||
this.bufferWaitFrames = this.initialBufferFrames;
|
||||
@@ -120,7 +116,6 @@ class FullDuplexPcmPlayback extends AudioWorkletProcessor {
|
||||
while (target < output.length && this.queue.length > 0) {
|
||||
const pcm = this.queue[0];
|
||||
const count = Math.min(output.length - target, pcm.length - this.offset);
|
||||
const remainingBeforeChunk = this.bufferedFrames();
|
||||
for (let index = 0; index < count; index += 1) {
|
||||
let sample = pcm[this.offset + index] / 32768;
|
||||
if (this.fadeInFrames > 0) {
|
||||
@@ -128,16 +123,6 @@ class FullDuplexPcmPlayback extends AudioWorkletProcessor {
|
||||
sample *= elapsed / this.fadeFrames;
|
||||
this.fadeInFrames -= 1;
|
||||
}
|
||||
const remainingFrames = remainingBeforeChunk - index;
|
||||
if (
|
||||
this.drain
|
||||
&& this.terminalFadeFrames > 0
|
||||
&& remainingFrames <= this.terminalFadeFrames
|
||||
) {
|
||||
sample *= this.terminalFadeFrames === 1
|
||||
? 0
|
||||
: (remainingFrames - 1) / (this.terminalFadeFrames - 1);
|
||||
}
|
||||
output[target + index] = sample;
|
||||
}
|
||||
target += count;
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Stream MiniCPM-o 4.5 text and audio from /v1/chat/completions."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import base64
|
||||
import io
|
||||
import wave
|
||||
from pathlib import Path
|
||||
|
||||
from openai import OpenAI
|
||||
|
||||
|
||||
def append_wav_chunk(encoded: str, pcm_parts: list[bytes]) -> tuple[int, int, int]:
|
||||
"""Decode one base64 WAV delta and append its PCM frames."""
|
||||
with wave.open(io.BytesIO(base64.b64decode(encoded)), "rb") as wav:
|
||||
channels = wav.getnchannels()
|
||||
sample_width = wav.getsampwidth()
|
||||
sample_rate = wav.getframerate()
|
||||
pcm_parts.append(wav.readframes(wav.getnframes()))
|
||||
return channels, sample_width, sample_rate
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--base-url", default="http://localhost:8099/v1")
|
||||
parser.add_argument("--model", default="openbmb/MiniCPM-o-4_5")
|
||||
parser.add_argument("--prompt", default="Say hello, then introduce vLLM in one sentence.")
|
||||
parser.add_argument("--output", type=Path, default=Path("minicpmo_stream.wav"))
|
||||
parser.add_argument("--text-only", action="store_true")
|
||||
args = parser.parse_args()
|
||||
|
||||
modalities = ["text"] if args.text_only else ["text", "audio"]
|
||||
stream = OpenAI(base_url=args.base_url, api_key="EMPTY").chat.completions.create(
|
||||
model=args.model,
|
||||
messages=[{"role": "user", "content": args.prompt}],
|
||||
modalities=modalities,
|
||||
stream=True,
|
||||
extra_body={"chat_template_kwargs": {"use_tts_template": not args.text_only}},
|
||||
)
|
||||
|
||||
pcm_parts: list[bytes] = []
|
||||
audio_format: tuple[int, int, int] | None = None
|
||||
for chunk in stream:
|
||||
for choice in chunk.choices:
|
||||
content = getattr(choice.delta, "content", None)
|
||||
if not content:
|
||||
continue
|
||||
if getattr(chunk, "modality", None) == "audio":
|
||||
current_format = append_wav_chunk(content, pcm_parts)
|
||||
if audio_format is not None and current_format != audio_format:
|
||||
raise ValueError(f"Audio format changed: {audio_format} -> {current_format}")
|
||||
audio_format = current_format
|
||||
elif getattr(chunk, "modality", None) == "text":
|
||||
print(content, end="", flush=True)
|
||||
|
||||
print()
|
||||
if pcm_parts and audio_format is not None:
|
||||
channels, sample_width, sample_rate = audio_format
|
||||
with wave.open(str(args.output), "wb") as wav:
|
||||
wav.setnchannels(channels)
|
||||
wav.setsampwidth(sample_width)
|
||||
wav.setframerate(sample_rate)
|
||||
wav.writeframes(b"".join(pcm_parts))
|
||||
print(f"Audio saved to {args.output} ({sample_rate} Hz)")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -248,82 +248,12 @@ done
|
||||
|
||||
## LTX-2
|
||||
|
||||
### Start Server
|
||||
|
||||
#### Basic Start
|
||||
|
||||
```bash
|
||||
vllm serve Lightricks/LTX-2 --omni --port 8098 \
|
||||
--enforce-eager --boundary-ratio 1.0
|
||||
vllm serve Lightricks/LTX-2 --omni --port 8098
|
||||
```
|
||||
|
||||
#### Start with Optimization Presets
|
||||
|
||||
Use the LTX-2 startup script with built-in optimization presets:
|
||||
|
||||
```bash
|
||||
# Baseline (1 GPU, eager)
|
||||
bash run_server_ltx2.sh baseline
|
||||
|
||||
# 4-GPU Ulysses sequence parallelism (lossless)
|
||||
bash run_server_ltx2.sh ulysses4
|
||||
|
||||
# Cache-DiT lossy acceleration (1 GPU, ~1.4× speedup)
|
||||
bash run_server_ltx2.sh cache-dit
|
||||
|
||||
# Best combo: 4-GPU Ulysses SP + Cache-DiT (~2.2× speedup)
|
||||
bash run_server_ltx2.sh best-combo
|
||||
```
|
||||
|
||||
#### Optimization Benchmarks
|
||||
|
||||
Benchmarked on H800, online serving (480×768, 41 frames, 20 steps, `seed=42`).
|
||||
"Inference" is the server-reported inference time; excludes HTTP/poll overhead.
|
||||
|
||||
| Preset | Server Command | Inference (s) | Speedup | Type |
|
||||
|--------|---------------|---------------|---------|------|
|
||||
| `baseline` | `--enforce-eager` | 10.3 | 1.00× | — |
|
||||
| `compile` | *(default, no --enforce-eager)* | ~10.3 (warm) | ~1.00× | Lossless |
|
||||
| `ulysses4` | `--enforce-eager --usp 4` | ~10.3 | ~1.00× | Lossless |
|
||||
| `cache-dit` | `--enforce-eager --cache-backend cache_dit` | 7.4 avg | ~1.4× | Lossy |
|
||||
| `best-combo` | `--enforce-eager --usp 4 --cache-backend cache_dit` | 4.7 avg | **~2.2×** | Lossless + Lossy |
|
||||
|
||||
**Observations**:
|
||||
- **torch.compile**: On H800, warm-request inference time matches the eager baseline (~10.3s).
|
||||
The first request pays ~6s compilation overhead. Benefit depends on model architecture and GPU.
|
||||
- **Ulysses SP (4 GPU)**: No measurable speedup alone for 41-frame generation at this resolution.
|
||||
Communication overhead outweighs gains at this sequence length.
|
||||
- **Cache-DiT**: Inference varies per request (6–10s) due to dynamic caching decisions.
|
||||
Average is ~7.4s (~1.4× speedup) with slight quality tradeoff.
|
||||
- **Best combo**: 4-GPU Ulysses SP + Cache-DiT synergize well — Cache-DiT reduces per-step
|
||||
computation, making the communication overhead of Ulysses SP worthwhile. Average ~4.7s
|
||||
(~2.2× speedup).
|
||||
- **FP8 quantization**: Reduces VRAM but does not speed up LTX-2 on H800 (compute-bound).
|
||||
|
||||
**Deployment Recommendations**:
|
||||
- For **production with quality priority**: use `baseline` with `--enforce-eager`
|
||||
- For **maximum throughput** (4 GPUs, quality tradeoff): use `best-combo` (~2.2× speedup)
|
||||
- For **single-GPU throughput**: use `cache-dit` (~1.4× speedup)
|
||||
- `--enforce-eager` is recommended to avoid torch.compile warmup latency on first request
|
||||
|
||||
### Send Requests (curl)
|
||||
|
||||
```bash
|
||||
# Using the provided script
|
||||
bash run_curl_ltx2.sh
|
||||
|
||||
# Or directly
|
||||
curl -sS -X POST http://localhost:8098/v1/videos \
|
||||
-H "Accept: application/json" \
|
||||
-F "prompt=A serene lakeside sunrise with mist over the water." \
|
||||
-F "width=768" \
|
||||
-F "height=480" \
|
||||
-F "num_frames=41" \
|
||||
-F "fps=24" \
|
||||
-F "num_inference_steps=20" \
|
||||
-F "guidance_scale=3.0" \
|
||||
-F "seed=42"
|
||||
```
|
||||
See the [LTX-2 recipe](../../../recipes/LTX/LTX-2.md) for all checkpoints,
|
||||
pipeline selection, requests, defaults, and advanced options.
|
||||
|
||||
## Helios
|
||||
|
||||
|
||||
@@ -22,7 +22,6 @@ create_response=$(
|
||||
-F "num_frames=41" \
|
||||
-F "fps=24" \
|
||||
-F "num_inference_steps=20" \
|
||||
-F "guidance_scale=3.0" \
|
||||
-F "seed=42"
|
||||
)
|
||||
|
||||
|
||||
@@ -21,8 +21,6 @@ set -euo pipefail
|
||||
|
||||
MODEL="${MODEL:-Lightricks/LTX-2}"
|
||||
PORT="${PORT:-8098}"
|
||||
FLOW_SHIFT="${FLOW_SHIFT:-1.0}"
|
||||
BOUNDARY_RATIO="${BOUNDARY_RATIO:-1.0}"
|
||||
|
||||
PRESET="${1:-baseline}"
|
||||
|
||||
@@ -66,19 +64,13 @@ case "$PRESET" in
|
||||
echo "Environment variables:"
|
||||
echo " MODEL - Model path (default: Lightricks/LTX-2)"
|
||||
echo " PORT - Server port (default: 8098)"
|
||||
echo " FLOW_SHIFT - Scheduler flow shift (default: 1.0)"
|
||||
echo " BOUNDARY_RATIO - Boundary ratio (default: 1.0)"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
echo "Model: $MODEL"
|
||||
echo "Port: $PORT"
|
||||
echo "Flow shift: $FLOW_SHIFT"
|
||||
echo "Boundary ratio: $BOUNDARY_RATIO"
|
||||
|
||||
vllm serve "$MODEL" --omni \
|
||||
--port "$PORT" \
|
||||
--flow-shift "$FLOW_SHIFT" \
|
||||
--boundary-ratio "$BOUNDARY_RATIO" \
|
||||
"${EXTRA_ARGS[@]}"
|
||||
+7
-10
@@ -243,16 +243,13 @@ markers = [
|
||||
"npu: Tests that run on NPU/Ascend (auto-added)",
|
||||
"musa: Tests that run on MUSA/Moore Threads (auto-added)",
|
||||
# specified computation resources marks (auto-added)
|
||||
# Tag with [hardware-resource] for tests.helpers.mark.get_hardware_mark_list().
|
||||
"H100: [hardware-resource] Tests that require H100 GPU",
|
||||
"H200: [hardware-resource] Tests that require H200 GPU",
|
||||
"L4: [hardware-resource] Tests that require L4 GPU",
|
||||
"B200: [hardware-resource] Tests that require B200 GPU",
|
||||
"MI325: [hardware-resource] Tests that require MI325 GPU (AMD/ROCm)",
|
||||
"B60: [hardware-resource] Tests that require Intel Arc Pro B60 XPU",
|
||||
"S5000: [hardware-resource] Tests that require S5000 GPU (Moore Threads/MUSA)",
|
||||
"A2: [hardware-resource] Tests that require A2 NPU",
|
||||
"A3: [hardware-resource] Tests that require A3 NPU",
|
||||
"H100: Tests that require H100 GPU",
|
||||
"L4: Tests that require L4 GPU",
|
||||
"MI325: Tests that require MI325 GPU (AMD/ROCm)",
|
||||
"B60: Tests that require Intel Arc Pro B60 XPU",
|
||||
"S5000: Tests that require S5000 GPU (Moore Threads/MUSA)",
|
||||
"A2: Tests that require A2 NPU",
|
||||
"A3: Tests that require A3 NPU",
|
||||
"distributed_cuda: Tests that require multi cards on CUDA platform",
|
||||
"distributed_rocm: Tests that require multi cards on ROCm platform",
|
||||
"distributed_xpu: Tests that require multi cards on XPU platform",
|
||||
|
||||
@@ -1,154 +0,0 @@
|
||||
# LTX-2.3
|
||||
|
||||
> 22B parameter text-to-video + audio generation model served via vLLM-Omni
|
||||
|
||||
## Summary
|
||||
|
||||
- Vendor: Lightricks
|
||||
- Model: `diffusers/LTX-2.3-Diffusers`
|
||||
- Task: Text-to-video and image-to-video with synchronized audio generation
|
||||
- Mode: Online serving (pure diffusion)
|
||||
- Maintainer: @oglok
|
||||
|
||||
## When to use this recipe
|
||||
|
||||
Use this recipe when you want to serve LTX-2.3 for text-to-video or
|
||||
image-to-video generation with audio. The model generates videos up to 20+
|
||||
seconds at 768x512 resolution with 48kHz audio from text prompts, and can
|
||||
condition video generation on an initial image. Start with a 96GB-class GPU for
|
||||
validation because the model combines a 22B parameter transformer with text
|
||||
encoder, VAE, and vocoder components.
|
||||
|
||||
## References
|
||||
|
||||
- Upstream raw checkpoints: <https://huggingface.co/Lightricks/LTX-2.3>
|
||||
- Diffusers-format checkpoint: <https://huggingface.co/diffusers/LTX-2.3-Diffusers>
|
||||
- Requires `diffusers >= 0.38.0` (install from git: `pip install git+https://github.com/huggingface/diffusers.git`)
|
||||
|
||||
## Serving
|
||||
|
||||
### Command
|
||||
|
||||
```bash
|
||||
vllm serve diffusers/LTX-2.3-Diffusers \
|
||||
--omni \
|
||||
--model-class-name LTX23Pipeline \
|
||||
--stage-init-timeout 600
|
||||
```
|
||||
|
||||
For image-to-video:
|
||||
|
||||
```bash
|
||||
vllm serve diffusers/LTX-2.3-Diffusers \
|
||||
--omni \
|
||||
--model-class-name LTX23ImageToVideoPipeline \
|
||||
--stage-init-timeout 600
|
||||
```
|
||||
|
||||
### T2V Verification
|
||||
|
||||
```bash
|
||||
# Health check
|
||||
curl http://localhost:8000/health
|
||||
|
||||
# Generate a 3-second video (81 frames at 24fps)
|
||||
curl -X POST http://localhost:8000/v1/videos \
|
||||
-F "prompt=A majestic bald eagle soaring over a misty mountain valley at dawn, golden sunlight breaking through clouds" \
|
||||
-F "negative_prompt=blurry, low quality, distorted, watermark" \
|
||||
-F "model=diffusers/LTX-2.3-Diffusers" \
|
||||
-F "num_frames=81" \
|
||||
-F "fps=24" \
|
||||
-F "size=768x512" \
|
||||
-F "num_inference_steps=30" \
|
||||
-F "guidance_scale=4.0" \
|
||||
-F "seed=42"
|
||||
|
||||
# Generate a 10-second video (241 frames)
|
||||
curl -X POST http://localhost:8000/v1/videos \
|
||||
-F "prompt=A cozy Japanese ramen shop at night in the rain, steam rising from bowls, neon signs reflecting on wet cobblestone streets" \
|
||||
-F "model=diffusers/LTX-2.3-Diffusers" \
|
||||
-F "num_frames=241" \
|
||||
-F "fps=24" \
|
||||
-F "size=768x512" \
|
||||
-F "num_inference_steps=30" \
|
||||
-F "guidance_scale=4.0"
|
||||
|
||||
# Generate a 20-second video (481 frames)
|
||||
curl -X POST http://localhost:8000/v1/videos \
|
||||
-F "prompt=An underwater coral reef teeming with tropical fish, sea turtles gliding gracefully, National Geographic documentary style" \
|
||||
-F "model=diffusers/LTX-2.3-Diffusers" \
|
||||
-F "num_frames=481" \
|
||||
-F "fps=24" \
|
||||
-F "size=768x512" \
|
||||
-F "num_inference_steps=30" \
|
||||
-F "guidance_scale=4.0"
|
||||
```
|
||||
|
||||
### I2V Verification
|
||||
|
||||
Start the server with `--model-class-name LTX23ImageToVideoPipeline`, then pass
|
||||
an image reference. The I2V pipeline requires `input_reference` or
|
||||
`image_reference` because the first frame is the conditioning input.
|
||||
|
||||
```bash
|
||||
curl http://localhost:8000/health
|
||||
|
||||
curl -X POST http://localhost:8000/v1/videos/sync \
|
||||
-F "prompt=A plush toy astronaut gently waving while the camera slowly pushes in." \
|
||||
-F "input_reference=@/absolute/path/to/reference.png" \
|
||||
-F "model=diffusers/LTX-2.3-Diffusers" \
|
||||
-F "num_frames=81" \
|
||||
-F "fps=24" \
|
||||
-F "size=768x512" \
|
||||
-F "num_inference_steps=30" \
|
||||
-F "guidance_scale=4.0" \
|
||||
-F "seed=42" \
|
||||
-o ltx23_i2v.mp4
|
||||
```
|
||||
|
||||
### Notes
|
||||
|
||||
- Memory usage: run a latest-head serving smoke or DFX benchmark on the target
|
||||
GPU before publishing load/peak VRAM numbers.
|
||||
- Checkpoint format: use a Diffusers-format checkpoint such as
|
||||
`diffusers/LTX-2.3-Diffusers`; the upstream `Lightricks/LTX-2.3` repository ships
|
||||
raw safetensors and does not contain the subfolder configs required by this
|
||||
pipeline loader.
|
||||
- Key flags:
|
||||
- `--stage-init-timeout 600`: Gives the initial `torch.compile` warmup enough time on large checkpoints
|
||||
- `--model-class-name LTX23Pipeline`: Selects the LTX-2.3 text-to-video pipeline (not LTX-2)
|
||||
- `--model-class-name LTX23ImageToVideoPipeline`: Selects the LTX-2.3 image-to-video pipeline
|
||||
- `input_reference=@...`: Uploads the I2V reference image directly
|
||||
- Audio: 48kHz AAC via BWE vocoder, automatically synced with video
|
||||
- CPU offloading: Text encoder (Gemma-3-12B), connectors, VAE, audio VAE, and vocoder stay on CPU and are moved to GPU only when needed
|
||||
- Supported resolutions: 768x512, 512x384 (must be divisible by 32)
|
||||
- Frame rate: 24 fps
|
||||
- Duration: Controlled by `num_frames` (frames = duration_seconds * 24 + 1)
|
||||
- Known limitations:
|
||||
- LTX-2.3 I2V currently supports first-frame image conditioning, matching the LTX image-conditioning contract.
|
||||
- Requires `diffusers >= 0.38.0` (not yet on PyPI, install from git)
|
||||
|
||||
## Hardware Support
|
||||
|
||||
## GPU
|
||||
|
||||
### 1x NVIDIA RTX PRO 6000 Blackwell (96GB)
|
||||
|
||||
#### Environment
|
||||
|
||||
- OS: Ubuntu 22.04
|
||||
- Python: 3.10+
|
||||
- Driver / runtime: CUDA 13.0, Driver 580.126.09
|
||||
- vLLM version: Match the PR or release checkout being validated
|
||||
- vLLM-Omni version or commit: Use the commit you are deploying from
|
||||
|
||||
### Benchmarking
|
||||
|
||||
Do not copy latency or VRAM numbers between commits or machines. Record them
|
||||
only after running a latest-head sweep on the target hardware.
|
||||
|
||||
For formal PR benchmarking, reuse
|
||||
`tests/dfx/perf/tests/test_ltx2_vllm_omni.json` with
|
||||
`tests/dfx/perf/scripts/run_diffusion_benchmark.py`. Each case in that file includes a JSON **`mark`** (`hardware_marks` + `full_model` + `diffusion`) for local `-m` filtering. That config captures the
|
||||
single-device eager baseline and CFG-parallel=2 case for a small 384x512,
|
||||
25-frame, 20-step workload.
|
||||
+295
-113
@@ -1,121 +1,303 @@
|
||||
# LTX-2
|
||||
# LTX-2 Family
|
||||
|
||||
> Text-to-video and image-to-video serving
|
||||
> LTX-2 and LTX-2.3 text-to-video and image-to-video generation with synchronized audio
|
||||
|
||||
## Summary
|
||||
## Pipelines
|
||||
|
||||
- Vendor: Lightricks
|
||||
- Model: `Lightricks/LTX-2`
|
||||
- Task: Text-to-video generation and image-to-video generation
|
||||
- Mode: Online serving with the OpenAI-compatible video API
|
||||
- Maintainer: Community
|
||||
| Model | Checkpoint | Task | `--model-class-name` | Batching |
|
||||
|---|---|---|---|---|
|
||||
| LTX-2 | `Lightricks/LTX-2` | One-stage T2V/I2V | `LTX2Pipeline` | Yes |
|
||||
| LTX-2 distilled | `rootonchair/LTX-2-19b-distilled` | Two-stage T2V/I2V | `LTX2DistilledPipeline` | No |
|
||||
| LTX-2.3 | `diffusers/LTX-2.3-Diffusers` | One-stage T2V/I2V | `LTX2Pipeline` | Yes |
|
||||
|
||||
## When to use this recipe
|
||||
`LTX2Pipeline` is the unified one-stage entry. Checkpoint metadata selects the
|
||||
LTX-2 or LTX-2.3 profile; omitting an image selects T2V, while one initial
|
||||
image selects I2V. Both one-stage repositories declare this class, so
|
||||
`--model-class-name` is optional. LTX-2.3 requires the Diffusers checkpoint;
|
||||
the raw `Lightricks/LTX-2.3` safetensors repository is not directly loadable.
|
||||
|
||||
Use this recipe when you want to deploy the base `Lightricks/LTX-2` model
|
||||
for both text-to-video and image-to-video generation. The same model ID is used for
|
||||
both tasks; switch the pipeline class with `--model-class-name` depending on
|
||||
whether you want T2V or I2V.
|
||||
`LTX2DistilledPipeline` is the unified distilled entry: omit `image` for T2V
|
||||
or provide one initial image for I2V. As in the official LTX pipelines,
|
||||
distilled mode is selected explicitly rather than inferred from the checkpoint
|
||||
name or metadata; always pass `--model-class-name LTX2DistilledPipeline`.
|
||||
|
||||
## API Migration
|
||||
|
||||
Only `req` may be passed positionally to `LTX2Pipeline`; every optional
|
||||
`forward` argument is keyword-only:
|
||||
|
||||
```python
|
||||
# No longer supported
|
||||
pipe(req, prompt)
|
||||
pipe(req, image, prompt)
|
||||
|
||||
# Supported
|
||||
pipe(req, prompt=prompt)
|
||||
pipe(req, image=image, prompt=prompt)
|
||||
```
|
||||
|
||||
The consolidation also removes these registry names without aliases:
|
||||
|
||||
| Removed name | Replacement |
|
||||
|---|---|
|
||||
| `LTX23Pipeline` | `LTX2Pipeline`; checkpoint metadata selects LTX-2.3 |
|
||||
| `LTX2ImageToVideoPipeline` | `LTX2Pipeline` with `image=` |
|
||||
| `LTX23ImageToVideoPipeline` | `LTX2Pipeline` with `image=`; checkpoint metadata selects LTX-2.3 |
|
||||
| `LTX2TwoStagesPipeline` | `LTX2DistilledPipeline` |
|
||||
| `LTX2ImageToVideoTwoStagesPipeline` | `LTX2DistilledPipeline` with `image=` |
|
||||
|
||||
Passing any second positional argument now raises `TypeError`. These changes
|
||||
affect direct Python callers and explicit `--model-class-name` overrides;
|
||||
offline and serving entrypoints already use named fields and are unaffected.
|
||||
|
||||
## One-Stage Defaults
|
||||
|
||||
| Parameter | LTX-2 | LTX-2.3 |
|
||||
|---|---:|---:|
|
||||
| Width × height | 768 × 512 | 768 × 512 |
|
||||
| Frames / frame rate | 121 / 24 | 121 / 24 |
|
||||
| Denoise steps | 40 | 30 |
|
||||
| Video/audio CFG | 3.0 / 7.0 | 3.0 / 7.0 |
|
||||
| Video/audio STG | 1.0 / 1.0 | 1.0 / 1.0 |
|
||||
| Video/audio modality guidance | 3.0 / 3.0 | 3.0 / 3.0 |
|
||||
| Video/audio rescale | 0.7 / 0.7 | 0.7 / 0.7 |
|
||||
| Video/audio STG blocks | `[29]` / `[29]` | `[28]` / `[28]` |
|
||||
|
||||
The model recipe also supplies the default negative prompt. Top-level
|
||||
`guidance_scale` overrides both video and audio CFG values. The online video
|
||||
API defaults `num_frames` to `1`, so set it explicitly; the offline LTX scripts
|
||||
default to `121`.
|
||||
|
||||
## Distilled Two-Stage Defaults
|
||||
|
||||
| Parameter | Value |
|
||||
|---|---:|
|
||||
| Final width × height | 1536 × 1024 |
|
||||
| Stage 1 width × height | 768 × 512 |
|
||||
| Frames / frame rate | 121 / 24 |
|
||||
| Stage 1 / Stage 2 denoise steps | 8 / 3 |
|
||||
| Guidance | Fixed positive-only |
|
||||
|
||||
The dimensions passed to the API are the final output dimensions and must be
|
||||
divisible by 64. Both one- and two-stage requests require `num_frames = 8k+1`.
|
||||
The distilled recipe fixes both sigma schedules; `num_inference_steps` must be
|
||||
`8` when supplied.
|
||||
|
||||
## Serving
|
||||
|
||||
Start either checkpoint:
|
||||
|
||||
```bash
|
||||
vllm serve Lightricks/LTX-2 --omni --stage-init-timeout 600
|
||||
```
|
||||
|
||||
```bash
|
||||
vllm serve diffusers/LTX-2.3-Diffusers --omni --stage-init-timeout 600
|
||||
```
|
||||
|
||||
Start the distilled checkpoint with its two-stage entry:
|
||||
|
||||
```bash
|
||||
vllm serve rootonchair/LTX-2-19b-distilled --omni \
|
||||
--model-class-name LTX2DistilledPipeline --stage-init-timeout 600
|
||||
```
|
||||
|
||||
The same server handles T2V and I2V. A T2V request using the selected recipe's
|
||||
default guidance is:
|
||||
|
||||
```bash
|
||||
curl -X POST http://localhost:8000/v1/videos/sync \
|
||||
-F "prompt=A cinematic close-up of ocean waves at golden hour." \
|
||||
-F "negative_prompt=worst quality, inconsistent motion, blurry, jittery, distorted" \
|
||||
-F "size=768x512" \
|
||||
-F "num_frames=121" \
|
||||
-F "fps=24" \
|
||||
-F "seed=42" \
|
||||
-o ltx_t2v.mp4
|
||||
```
|
||||
|
||||
For I2V, provide exactly one initial image:
|
||||
|
||||
```bash
|
||||
curl -X POST http://localhost:8000/v1/videos/sync \
|
||||
-F "prompt=A plush toy astronaut gently waving while the camera slowly pushes in." \
|
||||
-F "negative_prompt=worst quality, inconsistent motion, blurry, jittery, distorted" \
|
||||
-F "input_reference=@/absolute/path/to/reference.png" \
|
||||
-F "size=768x512" \
|
||||
-F "num_frames=121" \
|
||||
-F "fps=24" \
|
||||
-F "seed=42" \
|
||||
-o ltx_i2v.mp4
|
||||
```
|
||||
|
||||
Use `image_reference` for a URL or JSON-safe image reference. Do not provide it
|
||||
together with `input_reference`.
|
||||
|
||||
## Guidance
|
||||
|
||||
LTX supports independent video/audio CFG, spatio-temporal guidance (STG),
|
||||
cross-modality guidance, and rescaling:
|
||||
|
||||
| Parameter | Default | Effect | Alias |
|
||||
|---|---:|---|---|
|
||||
| `video_cfg_scale` | 3.0 | Video text CFG; `1.0` disables it | `video_cfg_guidance_scale` |
|
||||
| `audio_cfg_scale` | 7.0 | Audio text CFG; `1.0` disables it | `audio_cfg_guidance_scale` |
|
||||
| `video_stg_scale` | 1.0 | Video STG; `0.0` disables it | `video_stg_guidance_scale` |
|
||||
| `audio_stg_scale` | 1.0 | Audio STG; `0.0` disables it | `audio_stg_guidance_scale` |
|
||||
| `video_modality_scale` | 3.0 | Audio-to-video guidance; `1.0` disables it | `a2v_guidance_scale` |
|
||||
| `audio_modality_scale` | 3.0 | Video-to-audio guidance; `1.0` disables it | `v2a_guidance_scale` |
|
||||
| `video_rescale_scale` | 0.7 | Video guidance rescale; `0.0` disables it | — |
|
||||
| `audio_rescale_scale` | 0.7 | Audio guidance rescale; `0.0` disables it | — |
|
||||
| `video_stg_blocks` | `[29]` / `[28]` | Perturbed video transformer blocks | — |
|
||||
| `audio_stg_blocks` | `[29]` / `[28]` | Perturbed audio transformer blocks | — |
|
||||
|
||||
The STG block defaults are shown as LTX-2 / LTX-2.3. A canonical name wins
|
||||
over its alias. Explicit top-level `guidance_scale` wins over both CFG fields;
|
||||
omit it for independent video/audio CFG.
|
||||
|
||||
Pass these fields in online `extra_params` or offline `--extra-body`:
|
||||
|
||||
```bash
|
||||
# Add to the curl request above
|
||||
-F 'extra_params={"video_cfg_scale":3.0,"audio_cfg_scale":7.0}'
|
||||
|
||||
# Add to text_to_video.py or image_to_video.py
|
||||
--extra-body '{"video_cfg_scale":3.0,"audio_cfg_scale":7.0}'
|
||||
```
|
||||
|
||||
### Python API
|
||||
|
||||
Normal per-request values belong in `OmniDiffusionSamplingParams`. Pass the
|
||||
resulting `DiffusionRequestBatch` as the only positional argument:
|
||||
|
||||
```python
|
||||
from vllm_omni.diffusion.request import OmniDiffusionRequest
|
||||
from vllm_omni.diffusion.worker.request_batch import DiffusionRequestBatch
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
|
||||
params = OmniDiffusionSamplingParams(
|
||||
width=768,
|
||||
height=512,
|
||||
num_frames=121,
|
||||
frame_rate=24.0,
|
||||
num_inference_steps=30,
|
||||
seed=42,
|
||||
extra_args={"video_cfg_scale": 3.0, "audio_cfg_scale": 7.0},
|
||||
)
|
||||
req = DiffusionRequestBatch([
|
||||
OmniDiffusionRequest(
|
||||
prompt={"prompt": "Cherry blossoms moving in a light breeze"},
|
||||
sampling_params=params,
|
||||
request_id="ltx-example",
|
||||
)
|
||||
])
|
||||
|
||||
t2v_output = pipe(req)
|
||||
i2v_output = pipe(req, image=image)
|
||||
```
|
||||
|
||||
Direct keyword arguments are low-level fallbacks, for example
|
||||
`pipe(req, prompt=prompt, width=768)`. Request fields take precedence unless
|
||||
noted below.
|
||||
|
||||
### Complete `forward` Surface
|
||||
|
||||
| Argument | Type/default | Meaning and constraints |
|
||||
|---|---|---|
|
||||
| `req` | `DiffusionRequestBatch`, required | Only positional argument; contains prompts and per-request sampling parameters. |
|
||||
| `image` | image or batch, `None` | Direct value wins over request images; no image selects T2V. I2V accepts one image per prompt, and a batch cannot mix T2V/I2V. |
|
||||
| `prompt` | string or list, `None` | Positive-text fallback; request prompts win. Mutually exclusive with `prompt_embeds`. |
|
||||
| `negative_prompt` | string or list, `None` | Fallback after request values and before the recipe default; mutually exclusive with negative embeddings. |
|
||||
| `height` | `int`, `None` | Request → direct value → recipe default; divisible by 32 one-stage or 64 distilled. |
|
||||
| `width` | `int`, `None` | Same precedence and alignment as `height`. |
|
||||
| `num_frames` | `int`, `None` | Request → direct value → recipe default; must be `8k+1` and also determines audio duration with `frame_rate`. |
|
||||
| `frame_rate` | `float`, `None` | Request `frame_rate` → request `fps` → direct value → recipe default. |
|
||||
| `num_inference_steps` | `int`, `None` | Request → direct value → recipe default; minimum 2 one-stage, fixed at 8 for distilled Stage 1. Custom one-stage `sigmas` determine actual steps. |
|
||||
| `sigmas` | list of float, `None` | One-stage only. Request values win; every request in a fused batch must use the same schedule. |
|
||||
| `timesteps` | list of int, `None` | Compatibility slot; LTX accepts only `None`. Use `sigmas`. |
|
||||
| `guidance_scale` | `float`, `None` | One-stage common video/audio CFG fallback; an explicit request value wins. Distilled two-stage uses fixed positive-only guidance. |
|
||||
| `guidance_rescale` | `float`, `None` | Accepts only `None` or `0.0`; use the modality rescale fields. |
|
||||
| `noise_scale` | `float`, `0.0` | Compatibility slot; LTX accepts only `0.0`. |
|
||||
| `num_videos_per_prompt` | `int`, `1` | Output-count fallback; positive request `num_outputs_per_prompt` wins. |
|
||||
| `generator` | generator or list, `None` | Explicit RNG; otherwise request generators/seeds are collated. Lists must match the effective output batch. |
|
||||
| `latents` | tensor, `None` | Request tensors win. One-stage accepts packed `[B, S, C]` and validated unpacked video layouts. |
|
||||
| `audio_latents` | tensor, `None` | One-stage initial audio latents; request tensors win and are collated. |
|
||||
| `prompt_embeds` | tensor, `None` | Precomputed positive conditioning; requires `prompt_attention_mask` and cannot accompany `prompt`. |
|
||||
| `negative_prompt_embeds` | tensor, `None` | One-stage precomputed negative conditioning; requires its mask and cannot accompany `negative_prompt`. |
|
||||
| `prompt_attention_mask` | tensor, `None` | Mask for positive embeddings; request `prompt_attention_mask` or `attention_mask` wins. |
|
||||
| `negative_prompt_attention_mask` | tensor, `None` | Mask for negative embeddings; request negative mask fields win. |
|
||||
| `decode_timestep` | float or list, `0.0` | Video-VAE decode timestep; request value wins. Lists may match 1, prompt batch, or output batch. |
|
||||
| `decode_noise_scale` | float or list, `None` | Same list rules; request value wins. Defaults to `decode_timestep`. |
|
||||
| `output_type` | string, `"np"` | Request value wins. `"np"` decodes output; `"latent"` skips VAE/vocoder decode. |
|
||||
| `return_dict` | `bool`, `True` | Compatibility slot; only `True` is accepted. |
|
||||
| `attention_kwargs` | dict, `None` | Per-call values are unsupported; configure attention at engine startup. |
|
||||
| `max_sequence_length` | `int`, `None` | Request → direct value → tokenizer limit. |
|
||||
|
||||
Request-object naming differs only in a few places: `num_videos_per_prompt`
|
||||
maps to `num_outputs_per_prompt`; images and prompt text/embeddings live in the
|
||||
request prompt payload; LTX guidance fields live in sampling `extra_args`.
|
||||
|
||||
### Recipe-Specific Request Capabilities
|
||||
|
||||
| Override | One-stage | Distilled two-stage |
|
||||
|---|---|---|
|
||||
| Guidance | Supported | Fixed positive-only |
|
||||
| Negative prompt/embeddings | Supported | Rejected |
|
||||
| `num_inference_steps` | Supported | Fixed at 8 for Stage 1; Stage 2 uses 3 |
|
||||
| Custom `sigmas` | Supported | Rejected; both phases use fixed schedules |
|
||||
| Video/audio latents | Supported | Rejected |
|
||||
|
||||
These capability checks apply equally to direct `forward` keywords and
|
||||
values in `OmniDiffusionSamplingParams`; unsupported values fail instead of
|
||||
being silently ignored.
|
||||
|
||||
### Custom Sigma Schedules
|
||||
|
||||
One-stage Python requests may set final scheduler boundaries directly:
|
||||
|
||||
```python
|
||||
params = OmniDiffusionSamplingParams(sigmas=[1.0, 0.75, 0.5, 0.25])
|
||||
```
|
||||
|
||||
Each nonterminal value produces one denoise step, and a terminal `0.0` is
|
||||
appended when omitted. This schedule overrides `num_inference_steps`. All
|
||||
requests in a fused batch must use the same list. The video form API and
|
||||
bundled offline CLI do not currently expose `sigmas`.
|
||||
|
||||
### Constraints
|
||||
|
||||
- LTX rejects non-default `timesteps`, `flow_shift`, `guidance_rescale`,
|
||||
`noise_scale`, `attention_kwargs`, and `return_dict=False`. Use final
|
||||
`sigmas`, modality rescale fields, startup attention configuration, and the
|
||||
standard `DiffusionOutput` respectively.
|
||||
- Fused one-stage requests must resolve to identical LTX guidance and sigma
|
||||
schedules. Keep concurrent requests guidance-homogeneous or use
|
||||
`--max-num-seqs 1`.
|
||||
- `--cfg-parallel-size 2` supports only the CFG-only plan: disable STG and
|
||||
modality guidance and set both rescale fields to `0.0`.
|
||||
|
||||
## Operational Notes
|
||||
|
||||
- LTX-2 one-stage previously loaded and peaked at about 73.5 GiB on one H200
|
||||
141GB; remeasure on the deployed commit and hardware.
|
||||
- LTX-2.3 includes a 22B transformer, Gemma encoder, two VAEs, and a vocoder.
|
||||
Start on a 96GB-class GPU or use CPU/layerwise offload on smaller devices.
|
||||
- The distilled upsampler uses the configured pipeline dtype and participates
|
||||
in component discovery/device placement; it remains resident during denoise
|
||||
offload because it is used only at the phase boundary.
|
||||
- The output audio sample rate comes from the loaded components and is not a
|
||||
request parameter.
|
||||
- For benchmarks, use `tests/dfx/perf/tests/test_ltx2_vllm_omni.json` with
|
||||
`tests/dfx/perf/scripts/run_diffusion_benchmark.py`.
|
||||
- Current two-stage support is limited to the distilled checkpoint; official
|
||||
LTX two-stage/HQ execution remains out of scope.
|
||||
|
||||
## References
|
||||
|
||||
- Model: <https://huggingface.co/Lightricks/LTX-2>
|
||||
- User guide:
|
||||
[`docs/user_guide/examples/online_serving/text_to_video.md`](../../docs/user_guide/examples/online_serving/text_to_video.md)
|
||||
- Example guide:
|
||||
[`examples/online_serving/text_to_video/README.md`](../../examples/online_serving/text_to_video/README.md)
|
||||
[`examples/online_serving/image_to_video/README.md`](../../examples/online_serving/image_to_video/README.md)
|
||||
|
||||
## Hardware Support
|
||||
|
||||
## GPU
|
||||
|
||||
### 1x H200 141GB (Text-to-Video)
|
||||
|
||||
#### Environment
|
||||
|
||||
- OS: Ubuntu 22.04.5 LTS
|
||||
- Python: 3.12+
|
||||
- Driver / runtime: NVIDIA CUDA environment with H200 141GB GPU
|
||||
- vLLM version: Match the repository requirements from your current checkout
|
||||
- vLLM-Omni version or commit: Use the commit you are deploying from
|
||||
|
||||
#### Command
|
||||
|
||||
```bash
|
||||
vllm serve Lightricks/LTX-2 \
|
||||
--omni \
|
||||
--model-class-name LTX2Pipeline
|
||||
```
|
||||
|
||||
#### Verification
|
||||
|
||||
```bash
|
||||
curl http://localhost:8000/health
|
||||
|
||||
curl -X POST http://localhost:8000/v1/videos \
|
||||
-F "prompt=A cinematic close-up of ocean waves at golden hour." \
|
||||
-F "negative_prompt=worst quality, inconsistent motion, blurry, jittery, distorted" \
|
||||
-F "width=512" \
|
||||
-F "height=768" \
|
||||
-F "num_frames=121" \
|
||||
-F "fps=24" \
|
||||
-F "num_inference_steps=40" \
|
||||
-F "guidance_scale=4.0" \
|
||||
-o ltx2_t2v_h200.mp4
|
||||
```
|
||||
|
||||
#### Notes
|
||||
|
||||
- Memory usage: Model loads at ~73.5GB, peaks at ~73.5 GiB during inference.
|
||||
- Key flags:
|
||||
- `--model-class-name LTX2Pipeline` selects the text-to-video pipeline explicitly.
|
||||
- Known limitations:
|
||||
- This recipe is tested on 1x H200 validation; add measured latency and VRAM after running it on your target machine.
|
||||
|
||||
### 1x H200 141GB (Image-to-Video)
|
||||
|
||||
#### Environment
|
||||
|
||||
- OS: Ubuntu 22.04.5 LTS
|
||||
- Python: 3.12+
|
||||
- Driver / runtime: NVIDIA CUDA environment with H200 141GB GPU
|
||||
- vLLM version: Match the repository requirements from your current checkout
|
||||
- vLLM-Omni version or commit: Use the commit you are deploying from
|
||||
|
||||
#### Command
|
||||
|
||||
```bash
|
||||
vllm serve Lightricks/LTX-2 \
|
||||
--omni \
|
||||
--model-class-name LTX2ImageToVideoPipeline
|
||||
```
|
||||
|
||||
#### Verification
|
||||
|
||||
```bash
|
||||
curl http://localhost:8000/health
|
||||
|
||||
curl -X POST http://localhost:8000/v1/videos/sync \
|
||||
-F "prompt=A plush toy astronaut gently waving while the camera slowly pushes in." \
|
||||
-F "input_reference=@/absolute/path/to/reference.png" \
|
||||
-F "width=512" \
|
||||
-F "height=768" \
|
||||
-F "num_frames=121" \
|
||||
-F "fps=24" \
|
||||
-F "num_inference_steps=40" \
|
||||
-F "guidance_scale=4.0" \
|
||||
-o ltx2_i2v_h200.mp4
|
||||
```
|
||||
|
||||
#### Notes
|
||||
|
||||
- Memory usage: Model loads at ~73.5GB, peaks at ~73.5 GiB during inference.
|
||||
- Key flags:
|
||||
- `--model-class-name LTX2ImageToVideoPipeline` switches the same `Lightricks/LTX-2` weights into image-to-video mode.
|
||||
- `input_reference=@...` uploads the reference image directly
|
||||
- Known limitations:
|
||||
- Use `image_reference` when you want to pass a URL or JSON-safe image reference
|
||||
instead of uploading a file.
|
||||
- Do not send `input_reference` and `image_reference` in the same request.
|
||||
- This recipe is tested on 1x H200 validation; add measured latency and VRAM after running it on your target machine.
|
||||
- <https://huggingface.co/Lightricks/LTX-2>
|
||||
- <https://huggingface.co/Lightricks/LTX-2.3>
|
||||
- <https://huggingface.co/diffusers/LTX-2.3-Diffusers>
|
||||
- [Online video generation](../../docs/user_guide/examples/online_serving/text_to_video.md)
|
||||
- [Request batching](../../docs/user_guide/diffusion/request_batching.md)
|
||||
- [T2V offline example](../../examples/offline_inference/text_to_video/text_to_video.md)
|
||||
- [I2V offline example](../../examples/offline_inference/image_to_video/README.md)
|
||||
@@ -0,0 +1,142 @@
|
||||
# Nemotron-Labs Audex (2B / 30B-A3B)
|
||||
|
||||
> Speech suite on 1×H100: TTS, text-to-audio, audio understanding, and cascaded speech-to-speech
|
||||
|
||||
## Summary
|
||||
|
||||
- Vendor: NVIDIA
|
||||
- Model: `nvidia/Nemotron-Labs-Audex-2B`, `nvidia/Nemotron-Labs-Audex-30B-A3B`
|
||||
- Task: Text-to-speech (English), text-to-audio (sound effects), audio understanding (ASR / audio QA), cascaded speech-to-speech
|
||||
- Mode: Online serving (`/v1/audio/speech`, `/v1/chat/completions`) and offline batch inference
|
||||
- Maintainer: Community
|
||||
|
||||
## When to use this recipe
|
||||
|
||||
Use this recipe to serve either Audex checkpoint through one of its four
|
||||
deployment pipelines. Both sizes share the same repo layout, codec token
|
||||
space, and streaming causal speech decoder (16 kHz output); they differ in
|
||||
the thinker: the 2B is a dense LM, the 30B-A3B is a hybrid Mamba + MoE
|
||||
NemotronH (~3B active parameters). Pick the pipeline by task:
|
||||
|
||||
| pipeline (`vllm_omni/deploy/<name>[_30b].yaml`) | audio in | text out | speech out | general audio out | endpoint |
|
||||
|---|---|---|---|---|---|
|
||||
| `audex_tts` | ❌ | ❌ | ✅ | ❌ | `/v1/audio/speech` |
|
||||
| `audex_tta` | ❌ | ❌ | ❌ | ✅ | `/v1/audio/speech` |
|
||||
| `audex_thinker_only` | ✅ | ✅ | ❌ | ❌ | `/v1/chat/completions` |
|
||||
| `audex_s2s` | ✅ | ✅ | ✅ | ❌ | both |
|
||||
|
||||
## References
|
||||
|
||||
- Model cards: [Nemotron-Labs-Audex-2B](https://huggingface.co/nvidia/Nemotron-Labs-Audex-2B), [Nemotron-Labs-Audex-30B-A3B](https://huggingface.co/nvidia/Nemotron-Labs-Audex-30B-A3B)
|
||||
- Offline examples: [`examples/offline_inference/audex/`](../../examples/offline_inference/audex/)
|
||||
- Online examples: [`examples/online_serving/audex/`](../../examples/online_serving/audex/)
|
||||
|
||||
## Hardware Support
|
||||
|
||||
## GPU
|
||||
|
||||
### 1×H100 80GB — Audex-2B
|
||||
|
||||
#### Environment
|
||||
|
||||
- OS: Linux
|
||||
- Python: 3.12+
|
||||
- CUDA: 12.x
|
||||
- vLLM version: 0.24.0
|
||||
- vLLM-Omni version or commit: PR #4976 (branch `audex`)
|
||||
|
||||
#### Command
|
||||
|
||||
**Online serving** (TTS is the default pipeline; the repo root's
|
||||
`model_type` auto-resolves `audex_tts.yaml`):
|
||||
|
||||
```bash
|
||||
vllm-omni serve nvidia/Nemotron-Labs-Audex-2B \
|
||||
--host 0.0.0.0 --port 8097 \
|
||||
--trust-remote-code --omni
|
||||
# other modes: add --stage-configs-path vllm_omni/deploy/audex_{tta,thinker_only,s2s}.yaml
|
||||
# or use the launcher: MODE=s2s PORT=8098 examples/online_serving/audex/run_server.sh
|
||||
```
|
||||
|
||||
**Offline batch inference** (one script per task, each defaults to its
|
||||
correct deploy yaml):
|
||||
|
||||
```bash
|
||||
python examples/offline_inference/audex/text_to_speech.py \
|
||||
--texts "Hello world." --cfg-scale 1.5 --output-dir results/audex_wavs
|
||||
python examples/offline_inference/audex/text_to_audio.py # sound effects
|
||||
python examples/offline_inference/audex/audio_qa.py # ASR / audio QA
|
||||
python examples/offline_inference/audex/speech_to_speech.py # 3-pass cascade
|
||||
```
|
||||
|
||||
#### Verification
|
||||
|
||||
```bash
|
||||
curl -X POST http://localhost:8097/v1/audio/speech \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"model": "nvidia/Nemotron-Labs-Audex-2B",
|
||||
"input": "Hello, how are you?",
|
||||
"response_format": "wav",
|
||||
"extra_params": {"cfg_scale": 1.5}
|
||||
}' --output hello.wav
|
||||
# or the universal client (all four modes):
|
||||
python examples/online_serving/audex/client.py --mode tts --output hello.wav
|
||||
```
|
||||
|
||||
#### Notes
|
||||
|
||||
- Memory usage: TTS deploy — stage 0 (thinker) 0.4, stage 1 (decoder) 0.25
|
||||
GPU-memory fraction; both stages share GPU 0.
|
||||
- Key flags: `--trust-remote-code` and `--omni` are required.
|
||||
- Classifier-free guidance: `extra_params.cfg_scale` (1.0 = off; 1.5 is the
|
||||
official quality setting; TTA effectively requires its default 3.0).
|
||||
Measured en-24 self-transcribed CER: 6.87% guided vs 7.24% unguided.
|
||||
- Output: 16 kHz mono WAV. English, single built-in voice (no cloning).
|
||||
- TTA decodes through the external XCodec1 checkpoint
|
||||
(`hf-audio/xcodec-hubert-general-balanced`, auto-downloaded; override
|
||||
with `XCODEC1_PATH`).
|
||||
|
||||
### 1×H100 80GB — Audex-30B-A3B
|
||||
|
||||
#### Environment
|
||||
|
||||
Same as the 2B row above.
|
||||
|
||||
#### Command
|
||||
|
||||
The 30B REQUIRES its explicit deploy yaml (the shared HF `model_type`
|
||||
auto-resolves to the 2B-tuned config otherwise):
|
||||
|
||||
```bash
|
||||
vllm-omni serve nvidia/Nemotron-Labs-Audex-30B-A3B \
|
||||
--host 0.0.0.0 --port 8097 \
|
||||
--trust-remote-code --omni \
|
||||
--stage-configs-path vllm_omni/deploy/audex_tts_30b.yaml
|
||||
# or: SIZE=30b MODE=tts examples/online_serving/audex/run_server.sh
|
||||
```
|
||||
|
||||
Offline: same four scripts with
|
||||
`--model nvidia/Nemotron-Labs-Audex-30B-A3B --deploy-config vllm_omni/deploy/audex_<mode>_30b.yaml`.
|
||||
|
||||
#### Verification
|
||||
|
||||
Same curl / `client.py` as the 2B with
|
||||
`"model": "nvidia/Nemotron-Labs-Audex-30B-A3B"` (the server validates the
|
||||
model id). First launch downloads ~60 GB.
|
||||
|
||||
#### Notes
|
||||
|
||||
- Memory usage: ~61 GiB weights on one H100 80 GB (stage 0 at 0.85 GPU
|
||||
fraction + decoder at 0.08 on the same card); verified with healthy KV
|
||||
headroom. If long sequences OOM, set `tensor_parallel_size: 2` on
|
||||
stage 0 (documented in the yaml headers).
|
||||
- `enable_prefix_caching: false` on stage 0 — hybrid Mamba constraint.
|
||||
- CUDA graphs stay ON for the thinker: the NemotronH stage captures
|
||||
cleanly and eager mode costs ~7× single-stream TTS RTF (review-measured
|
||||
0.30 vs 2.08).
|
||||
- Quality (en-24 self-transcribed CER): 0.72% unguided baseline; 0.72% at
|
||||
cfg 1.5 with guided temperature 0.1.
|
||||
- Known limitation: the S2S chat pass needs the official
|
||||
formatting-instruction prompt (the bundled examples/client send it);
|
||||
a bare transcript turn makes the 30B answer in speech-codec tokens.
|
||||
@@ -31,8 +31,9 @@ also provided.
|
||||
- Default single-GPU compatibility layout (auto-loaded):
|
||||
[`vllm_omni/deploy/minicpmo_4_5.yaml`](../../vllm_omni/deploy/minicpmo_4_5.yaml)
|
||||
- Recommended 2-GPU continuous-batching layout:
|
||||
[`vllm_omni/deploy/minicpmo_4_5_2gpu.yaml`](../../vllm_omni/deploy/minicpmo_4_5_2gpu.yaml)
|
||||
- 3-GPU layout:
|
||||
[`vllm_omni/deploy/minicpmo_4_5_batching.yaml`](../../vllm_omni/deploy/minicpmo_4_5_batching.yaml)
|
||||
- 2-GPU and 3-GPU layouts:
|
||||
[`vllm_omni/deploy/minicpmo_4_5_2gpu.yaml`](../../vllm_omni/deploy/minicpmo_4_5_2gpu.yaml),
|
||||
[`vllm_omni/deploy/minicpmo_4_5_3gpu.yaml`](../../vllm_omni/deploy/minicpmo_4_5_3gpu.yaml)
|
||||
- 8x RTX 4090 layout:
|
||||
[`vllm_omni/deploy/minicpmo_4_5_8x4090.yaml`](../../vllm_omni/deploy/minicpmo_4_5_8x4090.yaml)
|
||||
@@ -67,8 +68,8 @@ Code2Wav consumes them through a shared-memory async connector.
|
||||
MiniCPM-o 4.5 now requires the three-stage topology: the Talker owns
|
||||
request-local codec generation and Code2Wav owns waveform state and
|
||||
reference-voice prompt features. `minicpmo_4_5.yaml` remains the stable
|
||||
single-GPU entry point; `minicpmo_4_5_2gpu.yaml` is the recommended
|
||||
two-GPU continuous-batching profile. The removed fused two-stage implementation is not retained as
|
||||
single-GPU entry point; `minicpmo_4_5_batching.yaml` is the recommended
|
||||
two-GPU profile. The removed fused two-stage implementation is not retained as
|
||||
a fallback because it would duplicate state machines and correctness paths.
|
||||
|
||||
## GPU
|
||||
@@ -78,9 +79,9 @@ a fallback because it would duplicate state machines and correctness paths.
|
||||
The default
|
||||
[`vllm_omni/deploy/minicpmo_4_5.yaml`](../../vllm_omni/deploy/minicpmo_4_5.yaml)
|
||||
co-locates Thinker, codec-only Talker, and Code2Wav on GPU 0. Their
|
||||
`gpu_memory_utilization` budgets are 0.55, 0.15, and 0.18 so concurrent
|
||||
Code2Wav activations keep residual headroom. All stages admit up to four
|
||||
sequences. Use the two-GPU profile for production throughput.
|
||||
`gpu_memory_utilization` budgets are 0.55, 0.22, and 0.22. All stages admit
|
||||
up to four sequences. Startup video profiling is bounded to 32 frames per
|
||||
video. Use the two-GPU profile for production throughput.
|
||||
|
||||
#### Environment
|
||||
|
||||
@@ -104,7 +105,7 @@ The deploy config is auto-loaded by the model registry — no
|
||||
For the recommended two-GPU layout, add:
|
||||
|
||||
```bash
|
||||
--deploy-config vllm_omni/deploy/minicpmo_4_5_2gpu.yaml
|
||||
--deploy-config vllm_omni/deploy/minicpmo_4_5_batching.yaml
|
||||
```
|
||||
|
||||
#### Performance comparison
|
||||
@@ -174,18 +175,16 @@ in another choice's `message.audio.data` (24 kHz mono, see Notes). With
|
||||
`modalities: ["text", "audio"]` you typically get two `choices` entries
|
||||
(one text, one audio).
|
||||
|
||||
**Streaming text + speech** (use `--stream`):
|
||||
**Streaming text + speech**:
|
||||
|
||||
```bash
|
||||
python examples/online_serving/minicpmo/openai_chat_completion_client_for_multimodal_generation.py \
|
||||
--query-type text \
|
||||
--prompt "Say hello, then introduce vLLM in one sentence." \
|
||||
--port 8099 \
|
||||
--stream
|
||||
python examples/online_serving/minicpmo/streaming_chat_completion.py \
|
||||
--base-url http://localhost:8099/v1 \
|
||||
--output minicpmo_stream.wav
|
||||
```
|
||||
|
||||
The client prints text deltas as they arrive and saves streamed audio chunks
|
||||
to WAV files.
|
||||
The client prints text deltas as they arrive and reconstructs one valid WAV
|
||||
from the independently encoded audio deltas.
|
||||
|
||||
**Gradio demo (text + image + audio + video UI)**:
|
||||
|
||||
@@ -203,8 +202,8 @@ speech output (TTS)"** checkbox on / off.
|
||||
|
||||
#### Notes
|
||||
|
||||
- Memory budget: Thinker, Talker, and Code2Wav reserve 0.55, 0.15, and
|
||||
0.18 of GPU 0 so concurrent Code2Wav activations keep residual headroom;
|
||||
- Memory budget: Thinker, Talker, and Code2Wav reserve 0.55, 0.22, and
|
||||
0.22 of GPU 0. The larger Thinker share protects its multimodal KV cache;
|
||||
all three model processes still share one CUDA device.
|
||||
- `--trust-remote-code` is required — the HF repo ships a custom
|
||||
`MiniCPMO` config / model class.
|
||||
@@ -215,13 +214,14 @@ speech output (TTS)"** checkbox on / off.
|
||||
contention. Talker AR
|
||||
state and Code2Wav caches are request-owned; Code2Wav batches only
|
||||
exact-shape-compatible chunks and does not fall back to serial decode.
|
||||
- Stage 0 pins `interleave_mm_strings=true` and
|
||||
`media_io_kwargs.video={fps:1,num_frames:128}` for Daily-Omni MiniCPM packs.
|
||||
- `limit_mm_per_prompt.video.num_frames: 32` bounds startup dummy profiling,
|
||||
not runtime media decoding. Use `media_io_kwargs.video.num_frames` when a
|
||||
matching URL/file video sampling limit is required.
|
||||
- `StageRequestStats.batch_size` is a request-scoped placeholder, not the
|
||||
scheduler's execution batch.
|
||||
- Single-GPU co-location trades throughput for hardware density: Stage 0/1
|
||||
CUDA Graph replay and eager Stage 2 vocoder kernels compete across three
|
||||
CUDA contexts. Use `minicpmo_4_5_2gpu.yaml` or a custom multi-GPU mapping for
|
||||
CUDA contexts. Use the 8x4090 config or a custom multi-GPU mapping for
|
||||
throughput-sensitive serving.
|
||||
|
||||
### 8 x RTX 4090 24GB (consumer-GPU layout)
|
||||
|
||||
+2
-2
@@ -46,8 +46,7 @@ recipes/
|
||||
| [`inclusionAI/Ming-omni-tts.md`](./inclusionAI/Ming-omni-tts.md) | Offline + online dense Ming TTS/audio generation | 1x H100 80GB / 1x AMD MI300X (ROCm 7.2) |
|
||||
| [`IndexTeam/IndexTTS-2.md`](./IndexTeam/IndexTTS-2.md) | Online serving for voice-cloned TTS with optional emotion control | 1x L4 24GB or larger CUDA GPU |
|
||||
| [`krea/Krea-2.md`](./krea/Krea-2.md) | Text-to-image (Turbo + Raw), offline + online, with LoRA | 1x H100 80GB |
|
||||
| [`LTX/LTX-2.md`](./LTX/LTX-2.md) | Text-to-video and image-to-video serving | 1x H200 141GB |
|
||||
| [`LTX/LTX-2.3.md`](./LTX/LTX-2.3.md) | Text-to-video with audio generation (22B) | 1x GPU (96GB VRAM) |
|
||||
| [`LTX/LTX-2.md`](./LTX/LTX-2.md) | LTX-2/LTX-2.3 text-to-video and image-to-video with synchronized audio | H200 141GB / 96GB-class GPU |
|
||||
| [`MammothModa2/MammothModa2.md`](./MammothModa2/MammothModa2.md) | Preview and Dev text-to-image (AR → DiT); Dev text/image understanding | Preview: 1x L40S 48GB / 1x ≥40GB GPU; Dev: 1x NVIDIA GPU with sufficient cache headroom |
|
||||
| [`mistralai/Voxtral-TTS.md`](./mistralai/Voxtral-TTS.md) | Online serving for TTS | 1x RTX 4090 24GB |
|
||||
| [`Robbyant/LingBot-Video.md`](./Robbyant/LingBot-Video.md) | Native dense and MoE text-to-video serving | Dense: 1x L20X; MoE: 1x L20X (~67.7 GiB peak) |
|
||||
@@ -57,6 +56,7 @@ recipes/
|
||||
| [`OpenBMB/MiniCPM-o-4_5.md`](./OpenBMB/MiniCPM-o-4_5.md) | Online serving for omni multimodal chat (text / image / audio / video → text + 24 kHz speech) | 2x A100/H100 80GB / 3x mid-tier GPU / 8x RTX 4090 24GB |
|
||||
| [`OpenBMB/VoxCPM2.md`](./OpenBMB/VoxCPM2.md) | Online + offline TTS with native AR pipeline (48 kHz, 30+ languages) | 1x RTX 4090 24GB |
|
||||
| [`OpenMOSS/MOSS-TTS.md`](./OpenMOSS/MOSS-TTS.md) | Online + offline multilingual TTS (MOSS-TTS family, 8B) | 1x H100 80GB |
|
||||
| [`NVIDIA/Nemotron-Labs-Audex.md`](./NVIDIA/Nemotron-Labs-Audex.md) | TTS / text-to-audio / audio understanding / cascaded S2S (Audex 2B + 30B-A3B) | 1x H100 80GB |
|
||||
| [`Qwen/Qwen-Image.md`](./Qwen/Qwen-Image.md) | Text-to-image serving with step-wise continuous batching replay and ModelOpt mixed FP8/NVFP4 | 1x A100 80GB / 2x B200 |
|
||||
| [`Qwen/Qwen-Image-2512.md`](./Qwen/Qwen-Image-2512.md) | Text-to-image serving with step-wise continuous batching replay and ModelOpt FP8 / mixed FP8/NVFP4 | 1x A800 80GB / 2x B200 |
|
||||
| [`Qwen/Qwen-Image-Edit.md`](./Qwen/Qwen-Image-Edit.md) | Text-guided single-image editing | 1x or 2x H200 141GB |
|
||||
|
||||
@@ -164,11 +164,10 @@ curl -s http://localhost:8091/v1/chat/completions \
|
||||
PR [#2495](https://github.com/vllm-project/vllm-omni/pull/2495) adds
|
||||
performance CI configs for the same DiT-only settings. The CI step is
|
||||
currently opt-in (gated by `RUN_HUNYUAN_IMAGE3_PERF=1`) with `soft_fail`
|
||||
enabled, intended for initial data collection. Perf runners no longer
|
||||
compare metrics against JSON baselines at assert time; the `baseline`
|
||||
blocks in the configs are reference-only and are copied into result JSON
|
||||
for reporting. Regression gates can be reintroduced once enough nightly
|
||||
data has been collected.
|
||||
enabled, intended for initial data collection. Performance assertions are
|
||||
skipped (`skip-performance-assertion: true`); the baseline values in the
|
||||
JSON configs are reference-only and will be promoted to regression gates
|
||||
once enough nightly data has been collected.
|
||||
|
||||
The user-facing equivalent is to launch one of the CLI commands above and
|
||||
generate 1024x1024 images with 50 denoising steps.
|
||||
|
||||
Binary file not shown.
Binary file not shown.
@@ -100,8 +100,9 @@ def _decode_kv_cache(
|
||||
head_size: int = 128,
|
||||
dtype: torch.dtype = torch.float16,
|
||||
) -> torch.Tensor:
|
||||
# vLLM >=0.23.0 KV cache layout: key/value live on dim 1.
|
||||
return torch.zeros((num_blocks, 2, block_size, num_kv_heads, head_size), dtype=dtype)
|
||||
# vLLM >=0.23.0 KV cache layout: (num_blocks, num_kv_heads, block_size,
|
||||
# 2*head_size) with K and V packed in the last dimension.
|
||||
return torch.zeros((num_blocks, num_kv_heads, block_size, 2 * head_size), dtype=dtype)
|
||||
|
||||
|
||||
def test_fish_kvcache_enabled_by_default(monkeypatch):
|
||||
@@ -221,10 +222,11 @@ def _vllm_kv_cache_shape(num_blocks, block_size, num_kv_heads, head_size):
|
||||
|
||||
|
||||
def test_fish_kvcache_backend_unbinds_kv_on_vllm_cache_layout(monkeypatch):
|
||||
# vLLM >=0.23.0 lays the KV cache out as (num_blocks, 2, block_size,
|
||||
# num_kv_heads, head_size). The backend must unbind on dim 1 so each of
|
||||
# key/value comes back as the 4-D (num_blocks, block_size, num_kv_heads,
|
||||
# head_size) tensor the decode kernel expects.
|
||||
# vLLM >=0.23.0 lays out the KV cache as (num_blocks, num_kv_heads,
|
||||
# block_size, 2*head_size) with K and V packed in the last dimension.
|
||||
# The backend must split on the content dim so each of key/value comes
|
||||
# back as the 4-D (num_blocks, block_size, num_kv_heads, head_size)
|
||||
# tensor the decode kernel expects.
|
||||
monkeypatch.setenv("VLLM_OMNI_FISH_KVCACHE_ATTN", "1")
|
||||
monkeypatch.setattr(fish_kvcache_backend, "is_available", lambda: True)
|
||||
monkeypatch.setattr(fish_kvcache_backend, "load_error", lambda: None)
|
||||
@@ -232,7 +234,8 @@ def test_fish_kvcache_backend_unbinds_kv_on_vllm_cache_layout(monkeypatch):
|
||||
|
||||
num_blocks, block_size, num_kv_heads, head_size = 8, 16, 8, 128
|
||||
cache_shape = _vllm_kv_cache_shape(num_blocks, block_size, num_kv_heads, head_size)
|
||||
assert cache_shape == (num_blocks, 2, block_size, num_kv_heads, head_size)
|
||||
# Upstream now returns a 4-D packed shape: (B, H, N, 2*D).
|
||||
assert cache_shape == (num_blocks, num_kv_heads, block_size, 2 * head_size)
|
||||
|
||||
captured = {}
|
||||
|
||||
@@ -262,20 +265,21 @@ def test_fish_kvcache_backend_unbinds_kv_on_vllm_cache_layout(monkeypatch):
|
||||
def test_fish_kvcache_vllm_cache_unbinds_then_falls_back_until_contiguous(monkeypatch):
|
||||
# End-to-end with the real vLLM cache layout and the real (unmocked) guard.
|
||||
#
|
||||
# The point of the unbind(1) fix: unbind(0) raised "too many values to
|
||||
# unpack" on the 5-D vLLM 0.23.0 layout, so the request crashed outright.
|
||||
# unbind(1) makes the request *work* again -- it unpacks cleanly and the
|
||||
# backend gracefully falls back to the original attention with correct
|
||||
# output. This test guards that fix and must NOT depend on the fast path
|
||||
# firing.
|
||||
# The point of the transpose+split fix: unbind(1) on the old 5-D layout
|
||||
# raised "too many values to unpack" on the vLLM 0.23.0 layout, and the
|
||||
# upstream now packs K/V into the last dimension (4-D shape). The
|
||||
# transpose(1, 2) + split correctly unpacks into separate key/value tensors
|
||||
# and the backend gracefully falls back to the original attention with
|
||||
# correct output. This test guards that fix and must NOT depend on the fast
|
||||
# path firing.
|
||||
#
|
||||
# It also documents a known limitation (tracked separately): unbind(1) on
|
||||
# the interleaved (num_blocks, 2, ...) layout yields *non-contiguous*
|
||||
# key/value views, which the guard rejects on the is_contiguous() check.
|
||||
# So the dimensions are compatible (4-D, block_size=16, head_size=128) --
|
||||
# the only thing keeping the fast path from engaging is contiguity, not
|
||||
# shape. Making the fast path actually fire would require a .contiguous()
|
||||
# copy and is out of scope here.
|
||||
# It also documents a known limitation (tracked separately):
|
||||
# transpose(1, 2) on the interleaved (B, H, N, 2*D) layout yields
|
||||
# *non-contiguous* key/value views, which the guard rejects on the
|
||||
# is_contiguous() check. So the dimensions are compatible (4-D,
|
||||
# block_size=16, head_size=128) -- the only thing keeping the fast path
|
||||
# from engaging is contiguity, not shape. Making the fast path actually
|
||||
# fire would require a .contiguous() copy and is out of scope here.
|
||||
fish_kvcache_backend.reset_fish_kvcache_attn_stats()
|
||||
monkeypatch.setenv("VLLM_OMNI_FISH_KVCACHE_ATTN", "1")
|
||||
monkeypatch.setattr(fish_kvcache_backend, "is_available", lambda: True)
|
||||
@@ -300,13 +304,14 @@ def test_fish_kvcache_vllm_cache_unbinds_then_falls_back_until_contiguous(monkey
|
||||
output = torch.zeros_like(query)
|
||||
kv_cache = torch.zeros(cache_shape, dtype=torch.float16)
|
||||
|
||||
# Root cause record: unbind(1) gives the right 4-D shape but a non-contiguous view.
|
||||
key_cache, value_cache = kv_cache.unbind(1)
|
||||
# Root cause record: transpose(1,2) + split gives the right 4-D shape but
|
||||
# a non-contiguous view.
|
||||
key_cache, value_cache = kv_cache.transpose(1, 2).split(head_size, dim=-1)
|
||||
assert tuple(key_cache.shape) == (num_blocks, block_size, num_kv_heads, head_size)
|
||||
assert not key_cache.is_contiguous()
|
||||
|
||||
# unbind(1) no longer crashes (unbind(0) would have raised here) and the
|
||||
# request completes via the correct fallback path.
|
||||
# transpose+split no longer crashes (unbind(1) would have raised here) and
|
||||
# the request completes via the correct fallback path.
|
||||
result = model.layers[0].self_attn.attn.impl.forward(None, query, None, None, kv_cache, _metadata(), output)
|
||||
|
||||
assert result is output
|
||||
|
||||
@@ -6,21 +6,13 @@ Unit tests for patch.py
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import base64
|
||||
import json
|
||||
import time
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
from pytest_mock import MockerFixture
|
||||
from vllm.benchmarks.lib.endpoint_request_func import RequestFuncInput
|
||||
|
||||
from vllm_omni.benchmarks.patch.patch import (
|
||||
MixRequestFuncOutput,
|
||||
async_request_openai_chat_omni_completions,
|
||||
async_request_openai_realtime_duplex,
|
||||
)
|
||||
from vllm_omni.experimental.fullduplex.client import RealtimeEventCollector
|
||||
from vllm_omni.benchmarks.patch.patch import MixRequestFuncOutput, async_request_openai_chat_omni_completions
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.benchmark, pytest.mark.cpu]
|
||||
|
||||
@@ -48,156 +40,6 @@ class MockResponse:
|
||||
pass
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_seed_tts_realtime_duplex_exports_per_request_metrics(monkeypatch):
|
||||
class FakeRealtimeClient:
|
||||
last_instance = None
|
||||
|
||||
def __init__(self, url):
|
||||
assert url == "ws://localhost:8000/v1/realtime?duplex=1"
|
||||
self.events = RealtimeEventCollector()
|
||||
self.configure_kwargs = None
|
||||
self.sent = []
|
||||
self.response_count = 0
|
||||
self.ack_count = 0
|
||||
type(self).last_instance = self
|
||||
|
||||
async def __aenter__(self):
|
||||
return self
|
||||
|
||||
async def __aexit__(self, *_args):
|
||||
return None
|
||||
|
||||
async def configure(self, model, **kwargs):
|
||||
assert model == "openbmb/MiniCPM-o-4_5"
|
||||
self.configure_kwargs = kwargs
|
||||
|
||||
async def send(self, event):
|
||||
self.sent.append(event)
|
||||
if event["type"] != "response.create":
|
||||
return
|
||||
self.response_count += 1
|
||||
response_id = f"resp-{self.response_count}"
|
||||
now = time.monotonic()
|
||||
self.events.add(
|
||||
{"type": "response.created", "response": {"id": response_id}},
|
||||
received_at_s=now + 0.01,
|
||||
)
|
||||
self.events.add(
|
||||
{
|
||||
"type": "response.audio_transcript.delta",
|
||||
"response_id": response_id,
|
||||
"delta": f"turn {self.response_count}",
|
||||
},
|
||||
received_at_s=now + 0.02,
|
||||
)
|
||||
self.events.add(
|
||||
{
|
||||
"type": "response.audio.delta",
|
||||
"response_id": response_id,
|
||||
"delta": base64.b64encode(b"\x00\x00" * 2400).decode(),
|
||||
"sample_rate_hz": 24_000,
|
||||
"metadata": {"audio_duration_ms": 100},
|
||||
},
|
||||
received_at_s=now + 0.03,
|
||||
)
|
||||
self.events.add(
|
||||
{"type": "response.done", "response": {"id": response_id}},
|
||||
received_at_s=now + 0.04,
|
||||
)
|
||||
|
||||
async def acknowledge_playback(self):
|
||||
self.ack_count += 1
|
||||
|
||||
async def close_session(self, **_kwargs):
|
||||
return None
|
||||
|
||||
monkeypatch.setattr(
|
||||
"vllm_omni.benchmarks.patch.patch.RealtimeDuplexClient",
|
||||
FakeRealtimeClient,
|
||||
)
|
||||
request_input = RequestFuncInput(
|
||||
model="openbmb/MiniCPM-o-4_5",
|
||||
model_name="openbmb/MiniCPM-o-4_5",
|
||||
prompt="hello",
|
||||
api_url="http://localhost:8000/v1/realtime",
|
||||
prompt_len=1,
|
||||
output_len=20,
|
||||
logprobs=None,
|
||||
multi_modal_content=None,
|
||||
ignore_eos=False,
|
||||
extra_body={"save_duplex_request_metrics": True},
|
||||
)
|
||||
request_input.seed_tts_speech_extra = {"ref_audio": "data:audio/wav;base64,AAAA"}
|
||||
request_input.seed_tts_system_prompt = "Speak exactly."
|
||||
request_input.seed_tts_turns = tuple(
|
||||
SimpleNamespace(utterance_id=f"utt-{index}", target_text=f"text {index}") for index in range(4)
|
||||
)
|
||||
|
||||
output = await async_request_openai_realtime_duplex(
|
||||
request_input,
|
||||
session=None,
|
||||
)
|
||||
|
||||
client = FakeRealtimeClient.last_instance
|
||||
assert client.configure_kwargs["native_duplex"] is False
|
||||
assert client.configure_kwargs["extra_body"] == {
|
||||
"ref_audio": "data:audio/wav;base64,AAAA",
|
||||
}
|
||||
assert client.sent == [
|
||||
event
|
||||
for index in range(4)
|
||||
for event in (
|
||||
{
|
||||
"type": "conversation.item.create",
|
||||
"item": {
|
||||
"type": "message",
|
||||
"role": "user",
|
||||
"content": [{"type": "input_text", "text": f"text {index}"}],
|
||||
},
|
||||
},
|
||||
{"type": "response.create"},
|
||||
)
|
||||
]
|
||||
assert client.ack_count == 4
|
||||
assert output.success is True
|
||||
assert output.generated_text == "turn 1 turn 2 turn 3 turn 4"
|
||||
assert output.audio_duration == pytest.approx(0.4)
|
||||
assert output.ttft > 0
|
||||
assert output.audio_ttfp > output.ttft
|
||||
assert output.audio_rtf > 0
|
||||
assert output.latency > 0
|
||||
assert output.tts_turn_pcm_bytes == [b"\x00\x00" * 2400] * 4
|
||||
assert output.tts_output_pcm_bytes == b"\x00\x00" * 9600
|
||||
session_id = output.duplex_request_metrics[0]["session_id"]
|
||||
assert len(output.duplex_request_metrics) == 4
|
||||
for index, metrics in enumerate(output.duplex_request_metrics):
|
||||
assert metrics == {
|
||||
"session_id": session_id,
|
||||
"request_index": index,
|
||||
"utterance_id": f"utt-{index}",
|
||||
"response_id": f"resp-{index + 1}",
|
||||
"source": "client_monotonic_receive",
|
||||
"measurement_origin": {
|
||||
"ttft": "conversation.item.create client send to first non-empty text delta",
|
||||
"ttfp": "conversation.item.create client send to first audio packet",
|
||||
"rtf": "request-start-to-last-audio receive time divided by emitted audio duration",
|
||||
},
|
||||
"ttft_ms": pytest.approx(20.0, abs=2.0),
|
||||
"ttfp_ms": pytest.approx(30.0, abs=2.0),
|
||||
"rtf": pytest.approx(0.3, abs=0.03),
|
||||
"audio_generation_ms": pytest.approx(30.0, abs=2.0),
|
||||
"audio_duration_ms": 100.0,
|
||||
}
|
||||
assert output.duplex_session_metrics == {
|
||||
"session_id": session_id,
|
||||
"audio_turn_count": 4,
|
||||
"mean_ttft_ms": pytest.approx(20.0, abs=2.0),
|
||||
"mean_ttfp_ms": pytest.approx(30.0, abs=2.0),
|
||||
"mean_rtf": pytest.approx(0.3, abs=0.03),
|
||||
}
|
||||
|
||||
|
||||
def create_sse_chunk(data_dict):
|
||||
"""Helper to create SSE formatted chunk"""
|
||||
return f"data: {json.dumps(data_dict)}\n\n".encode()
|
||||
|
||||
@@ -42,10 +42,7 @@ from benchmarks.accuracy.text_to_image.gbench import (
|
||||
summarize_generated_records as summarize_gebench_generated_records,
|
||||
summarize_gebench_results,
|
||||
)
|
||||
from tests.e2e.accuracy.qwen3_omni.qwen3_omni_acc_bench_core import (
|
||||
build_serve_common_argv,
|
||||
seed_tts_bench_argv,
|
||||
)
|
||||
from tests.e2e.accuracy.qwen3_omni.qwen3_omni_acc_bench_core import seed_tts_bench_argv
|
||||
from tests.e2e.accuracy.qwen3_omni.run_qwen_omni_acc_benchmark import sync_dataset_env_from_ns
|
||||
from vllm_omni.benchmarks.data_modules.seed_tts_dataset import resolve_seed_tts_root
|
||||
|
||||
@@ -60,25 +57,6 @@ def test_seed_tts_bench_argv_preserves_hf_repo_id_from_env(monkeypatch):
|
||||
assert argv[dataset_idx + 1] == "zhaochenyang20/seed-tts-eval"
|
||||
|
||||
|
||||
def test_build_serve_common_argv_accepts_realtime_seed_tts_route(tmp_path):
|
||||
argv = build_serve_common_argv(
|
||||
host="127.0.0.1",
|
||||
port=8000,
|
||||
model="openbmb/MiniCPM-o-4_5",
|
||||
num_prompts=4,
|
||||
max_concurrency=1,
|
||||
num_warmups=0,
|
||||
percentile_metrics="ttft,audio_ttfp,audio_rtf",
|
||||
result_dir=tmp_path,
|
||||
result_filename="result.json",
|
||||
backend="openai-realtime-tts",
|
||||
endpoint="/v1/realtime",
|
||||
)
|
||||
|
||||
assert argv[argv.index("--backend") + 1] == "openai-realtime-tts"
|
||||
assert argv[argv.index("--endpoint") + 1] == "/v1/realtime"
|
||||
|
||||
|
||||
def test_sync_dataset_env_preserves_seed_tts_hf_repo_id(monkeypatch):
|
||||
ns = argparse.Namespace(
|
||||
daily_omni_repo=None,
|
||||
|
||||
@@ -1,152 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Offline Daily-Omni sources: local mirrors must not fall back to the Hub id."""
|
||||
|
||||
import json
|
||||
import os
|
||||
import tarfile
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm_omni.benchmarks.data_modules.daily_omni_dataset import (
|
||||
DailyOmniDataset,
|
||||
daily_omni_local_qa_json,
|
||||
daily_omni_local_videos_dir,
|
||||
ensure_daily_omni_hub_root,
|
||||
resolve_daily_omni_local_root,
|
||||
)
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.benchmark]
|
||||
|
||||
_HUB_ID = "liarliar/Daily-Omni"
|
||||
|
||||
|
||||
def _write_qa_json(root: Path) -> Path:
|
||||
qa = root / "qa.json"
|
||||
qa.write_text(json.dumps([{"Question": "q", "Answer": "A", "video_id": "vid0"}]), encoding="utf-8")
|
||||
return qa
|
||||
|
||||
|
||||
def _stage_offline_hub_cache(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, *, with_qa: bool = True) -> Path:
|
||||
"""Build a hub cache holding ``liarliar/Daily-Omni`` and cut off all network access.
|
||||
|
||||
Reproduces the air-gapped NPU runner: the dataset is pre-staged with ``hf download``, so it
|
||||
only exists in the *hub* cache -- never in the ``datasets`` cache that ``load_dataset`` reads.
|
||||
"""
|
||||
constants = pytest.importorskip("huggingface_hub").constants
|
||||
|
||||
cache = tmp_path / "hub"
|
||||
repo_dir = cache / "datasets--liarliar--Daily-Omni"
|
||||
snapshot = repo_dir / "snapshots" / "deadbeef"
|
||||
snapshot.mkdir(parents=True)
|
||||
if with_qa:
|
||||
_write_qa_json(snapshot)
|
||||
(repo_dir / "refs").mkdir()
|
||||
(repo_dir / "refs" / "main").write_text("deadbeef", encoding="utf-8")
|
||||
|
||||
monkeypatch.setattr(constants, "HF_HUB_CACHE", str(cache))
|
||||
monkeypatch.setattr(constants, "HF_HUB_OFFLINE", True)
|
||||
return snapshot
|
||||
|
||||
|
||||
def test_resolve_local_root_returns_none_for_hub_id() -> None:
|
||||
assert resolve_daily_omni_local_root("liarliar/Daily-Omni") is None
|
||||
assert resolve_daily_omni_local_root("") is None
|
||||
assert resolve_daily_omni_local_root(None) is None
|
||||
|
||||
|
||||
def test_plain_local_mirror(tmp_path: Path) -> None:
|
||||
_write_qa_json(tmp_path)
|
||||
(tmp_path / "Videos" / "vid0").mkdir(parents=True)
|
||||
|
||||
root = resolve_daily_omni_local_root(str(tmp_path))
|
||||
assert root == tmp_path.resolve()
|
||||
assert daily_omni_local_qa_json(root) == (tmp_path / "qa.json").resolve()
|
||||
assert daily_omni_local_videos_dir(root) == root / "Videos"
|
||||
|
||||
|
||||
def test_hf_cache_dir_resolves_to_snapshot(tmp_path: Path) -> None:
|
||||
cache_dir = tmp_path / "datasets--liarliar--Daily-Omni"
|
||||
snapshot = cache_dir / "snapshots" / "deadbeef"
|
||||
snapshot.mkdir(parents=True)
|
||||
_write_qa_json(snapshot)
|
||||
(cache_dir / "refs").mkdir()
|
||||
(cache_dir / "refs" / "main").write_text("deadbeef", encoding="utf-8")
|
||||
|
||||
root = resolve_daily_omni_local_root(str(cache_dir))
|
||||
assert root == snapshot.resolve()
|
||||
assert daily_omni_local_qa_json(root) == (snapshot / "qa.json").resolve()
|
||||
|
||||
|
||||
def test_hf_cache_dir_without_refs_picks_newest_snapshot(tmp_path: Path) -> None:
|
||||
cache_dir = tmp_path / "datasets--liarliar--Daily-Omni"
|
||||
older = cache_dir / "snapshots" / "aaa"
|
||||
newer = cache_dir / "snapshots" / "bbb"
|
||||
older.mkdir(parents=True)
|
||||
newer.mkdir(parents=True)
|
||||
os.utime(older, (1, 1))
|
||||
os.utime(newer, (10, 10))
|
||||
|
||||
assert resolve_daily_omni_local_root(str(cache_dir)) == newer.resolve()
|
||||
|
||||
|
||||
def test_videos_tar_is_extracted(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setenv("HF_HOME", str(tmp_path / "hf_home"))
|
||||
root = tmp_path / "mirror"
|
||||
(root / "Videos" / "vid0").mkdir(parents=True)
|
||||
(root / "Videos" / "vid0" / "vid0_video.mp4").write_bytes(b"fake")
|
||||
with tarfile.open(root / "Videos.tar", "w") as tf:
|
||||
tf.add(root / "Videos", arcname="Videos")
|
||||
# Only the tarball is left in the mirror.
|
||||
for child in (root / "Videos").rglob("*"):
|
||||
if child.is_file():
|
||||
child.unlink()
|
||||
(root / "Videos" / "vid0").rmdir()
|
||||
(root / "Videos").rmdir()
|
||||
|
||||
videos_dir = daily_omni_local_videos_dir(root)
|
||||
assert videos_dir is not None
|
||||
assert (videos_dir / "vid0" / "vid0_video.mp4").is_file()
|
||||
|
||||
# Second call reuses the extraction instead of redoing it.
|
||||
assert daily_omni_local_videos_dir(root) == videos_dir
|
||||
|
||||
|
||||
def test_missing_qa_json_reports_none(tmp_path: Path) -> None:
|
||||
root = resolve_daily_omni_local_root(str(tmp_path))
|
||||
assert daily_omni_local_qa_json(root) is None
|
||||
assert daily_omni_local_videos_dir(root) is None
|
||||
|
||||
|
||||
def test_hub_id_resolves_to_offline_cache(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
snapshot = _stage_offline_hub_cache(tmp_path, monkeypatch)
|
||||
|
||||
root = ensure_daily_omni_hub_root(_HUB_ID)
|
||||
|
||||
assert root == snapshot.resolve()
|
||||
assert daily_omni_local_qa_json(root) == (snapshot / "qa.json").resolve()
|
||||
|
||||
|
||||
def test_dataset_loads_hub_id_without_network(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""The regression this file exists for: a Hub id must not require ``load_dataset``."""
|
||||
snapshot = _stage_offline_hub_cache(tmp_path, monkeypatch)
|
||||
# Any use of `datasets` here means we went back to the path that cannot read the hub cache.
|
||||
monkeypatch.setattr("vllm_omni.benchmarks.data_modules.daily_omni_dataset.load_dataset", None)
|
||||
|
||||
dataset = DailyOmniDataset(dataset_path=_HUB_ID, random_seed=0)
|
||||
|
||||
assert dataset.qa_json_path == (snapshot / "qa.json").resolve()
|
||||
assert [row["video_id"] for row in dataset.data] == ["vid0"]
|
||||
|
||||
|
||||
def test_uncached_hub_id_reports_both_attempts(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
_stage_offline_hub_cache(tmp_path, monkeypatch, with_qa=False)
|
||||
monkeypatch.setattr("vllm_omni.benchmarks.data_modules.daily_omni_dataset.load_dataset", None)
|
||||
|
||||
with pytest.raises(RuntimeError) as excinfo:
|
||||
DailyOmniDataset(dataset_path="someone-else/Daily-Omni", random_seed=0)
|
||||
|
||||
message = str(excinfo.value)
|
||||
assert "someone-else/Daily-Omni" in message
|
||||
assert "snapshot_download failed" in message
|
||||
assert "--daily-omni-qa-json" in message
|
||||
@@ -1,237 +0,0 @@
|
||||
"""Tests for Daily-Omni ``--daily-omni-pack-mode`` packing recipes.
|
||||
|
||||
The ``minicpm-interleave`` mode is an accuracy-critical protocol: MiniCPM-o only reaches
|
||||
OpenBMB's reported Daily-Omni score when frames and 1s audio segments arrive strictly
|
||||
interleaved, with ``max_slice_nums=1`` / ``use_image_id=False`` and no Qwen system prompt.
|
||||
These tests pin that contract so a refactor cannot silently regress the score.
|
||||
|
||||
vllm stubs are installed by tests/benchmarks/conftest.py before collection.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
# Load the data module directly (bypasses vllm_omni.__init__ heavy imports).
|
||||
_REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
_MODULE_PATH = _REPO_ROOT / "vllm_omni" / "benchmarks" / "data_modules" / "daily_omni_dataset.py"
|
||||
_MODULE_NAME = "vllm_omni.benchmarks.data_modules.daily_omni_dataset"
|
||||
|
||||
if _MODULE_NAME not in sys.modules:
|
||||
_spec = importlib.util.spec_from_file_location(_MODULE_NAME, _MODULE_PATH)
|
||||
_mod = importlib.util.module_from_spec(_spec)
|
||||
sys.modules[_MODULE_NAME] = _mod
|
||||
_spec.loader.exec_module(_mod)
|
||||
|
||||
from vllm_omni.benchmarks.data_modules.daily_omni_dataset import ( # noqa: E402
|
||||
DailyOmniDataset,
|
||||
_uniform_sample_indices,
|
||||
)
|
||||
|
||||
_VIDEO_ID = "vid001"
|
||||
_NUM_FRAMES = 5
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def video_dir(tmp_path: Path) -> Path:
|
||||
"""Official Daily-Omni ``Videos/`` layout with placeholder media files."""
|
||||
per_video = tmp_path / "Videos" / _VIDEO_ID
|
||||
per_video.mkdir(parents=True)
|
||||
(per_video / f"{_VIDEO_ID}_video.mp4").write_bytes(b"fake-mp4")
|
||||
(per_video / f"{_VIDEO_ID}_audio.wav").write_bytes(b"fake-wav")
|
||||
return tmp_path / "Videos"
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def qa_json(tmp_path: Path) -> Path:
|
||||
path = tmp_path / "qa.json"
|
||||
path.write_text(
|
||||
json.dumps(
|
||||
[
|
||||
{
|
||||
"video_id": _VIDEO_ID,
|
||||
"question": "What happens first?",
|
||||
"choice": {"A": "a", "B": "b", "C": "c", "D": "d"},
|
||||
"answer": "A",
|
||||
"task_type": "AV Event Alignment",
|
||||
"video_duration": "30s",
|
||||
}
|
||||
]
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
return path
|
||||
|
||||
|
||||
def _make_dataset(qa_json: Path, video_dir: Path, **kwargs) -> DailyOmniDataset:
|
||||
return DailyOmniDataset(
|
||||
qa_json_path=str(qa_json),
|
||||
video_dir=str(video_dir),
|
||||
disable_shuffle=True,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
|
||||
def _stub_extract(monkeypatch, dataset: DailyOmniDataset, counter: list[int]) -> None:
|
||||
"""Replace decord/ffmpeg extraction with deterministic in-memory frames + segments."""
|
||||
from PIL import Image
|
||||
|
||||
def fake_extract(video_path, *, audio_path, include_audio, **_kw):
|
||||
counter.append(1)
|
||||
frames = [Image.new("RGB", (16, 16), color=(i, i, i)) for i in range(_NUM_FRAMES)]
|
||||
segments = [np.zeros(1600, dtype=np.float32) for _ in range(_NUM_FRAMES)] if include_audio else []
|
||||
return frames, segments
|
||||
|
||||
monkeypatch.setattr(dataset, "_extract_minicpm_frame_audio_segments", fake_extract)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _uniform_sample_indices
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("n", "k"), [(10, 4), (100, 64), (7, 7), (3, 5), (1, 1)])
|
||||
def test_uniform_sample_indices_matches_numpy_linspace(n: int, k: int) -> None:
|
||||
got = _uniform_sample_indices(n, k)
|
||||
if n <= k:
|
||||
assert got == list(range(n))
|
||||
else:
|
||||
assert got == [int(i) for i in np.linspace(0, n - 1, k, dtype=int)]
|
||||
|
||||
|
||||
def test_uniform_sample_indices_non_positive_k() -> None:
|
||||
assert _uniform_sample_indices(10, 0) == []
|
||||
assert _uniform_sample_indices(10, -1) == []
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# minicpm-interleave packing
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_minicpm_interleave_alternates_image_and_audio(monkeypatch, qa_json, video_dir) -> None:
|
||||
ds = _make_dataset(qa_json, video_dir, pack_mode="minicpm-interleave", input_mode="all")
|
||||
_stub_extract(monkeypatch, ds, [])
|
||||
|
||||
parts, extra, position = ds._compose_daily_omni_multimodal(_VIDEO_ID, None)
|
||||
|
||||
assert position == "first"
|
||||
assert [p["type"] for p in parts] == ["image_url", "audio_url"] * _NUM_FRAMES
|
||||
assert extra["mm_processor_kwargs"] == {
|
||||
"use_audio_in_video": False,
|
||||
"max_slice_nums": 1,
|
||||
"use_image_id": False,
|
||||
}
|
||||
|
||||
|
||||
def test_minicpm_interleave_visual_mode_emits_frames_only(monkeypatch, qa_json, video_dir) -> None:
|
||||
ds = _make_dataset(qa_json, video_dir, pack_mode="minicpm-interleave", input_mode="visual")
|
||||
_stub_extract(monkeypatch, ds, [])
|
||||
|
||||
parts, _extra, _position = ds._compose_daily_omni_multimodal(_VIDEO_ID, None)
|
||||
|
||||
assert [p["type"] for p in parts] == ["image_url"] * _NUM_FRAMES
|
||||
|
||||
|
||||
def test_minicpm_interleave_writes_file_urls_under_video_dir(monkeypatch, qa_json, video_dir) -> None:
|
||||
ds = _make_dataset(qa_json, video_dir, pack_mode="minicpm-interleave", input_mode="all")
|
||||
_stub_extract(monkeypatch, ds, [])
|
||||
|
||||
parts, _extra, _position = ds._compose_daily_omni_multimodal(_VIDEO_ID, None)
|
||||
|
||||
segment_dir = video_dir / ".minicpm_daily_omni_interleave" / _VIDEO_ID
|
||||
for part in parts:
|
||||
url = part[part["type"]]["url"]
|
||||
assert url.startswith("file://")
|
||||
assert Path(url[len("file://") :]).parent == segment_dir.resolve()
|
||||
assert sorted(p.name for p in segment_dir.glob("frame_*.jpg")) == [f"frame_{i:04d}.jpg" for i in range(_NUM_FRAMES)]
|
||||
|
||||
|
||||
def test_minicpm_interleave_inline_mode_emits_data_urls(monkeypatch, qa_json, video_dir) -> None:
|
||||
ds = _make_dataset(
|
||||
qa_json,
|
||||
video_dir,
|
||||
pack_mode="minicpm-interleave",
|
||||
input_mode="all",
|
||||
inline_local_video=True,
|
||||
)
|
||||
_stub_extract(monkeypatch, ds, [])
|
||||
|
||||
parts, _extra, _position = ds._compose_daily_omni_multimodal(_VIDEO_ID, None)
|
||||
|
||||
assert parts[0]["image_url"]["url"].startswith("data:image/jpeg;base64,")
|
||||
assert parts[1]["audio_url"]["url"].startswith("data:audio/wav;base64,")
|
||||
assert not (video_dir / ".minicpm_daily_omni_interleave").exists()
|
||||
|
||||
|
||||
def test_minicpm_interleave_caches_file_parts_but_not_inline_payloads(monkeypatch, qa_json, video_dir) -> None:
|
||||
"""Inline base64 must not be retained: ~700 Daily-Omni videos would pin GBs of RSS."""
|
||||
file_calls: list[int] = []
|
||||
ds_file = _make_dataset(qa_json, video_dir, pack_mode="minicpm-interleave", input_mode="all")
|
||||
_stub_extract(monkeypatch, ds_file, file_calls)
|
||||
ds_file._compose_daily_omni_multimodal(_VIDEO_ID, None)
|
||||
ds_file._compose_daily_omni_multimodal(_VIDEO_ID, None)
|
||||
assert len(file_calls) == 1
|
||||
assert ds_file._minicpm_interleave_cache
|
||||
|
||||
inline_calls: list[int] = []
|
||||
ds_inline = _make_dataset(
|
||||
qa_json,
|
||||
video_dir,
|
||||
pack_mode="minicpm-interleave",
|
||||
input_mode="all",
|
||||
inline_local_video=True,
|
||||
)
|
||||
_stub_extract(monkeypatch, ds_inline, inline_calls)
|
||||
ds_inline._compose_daily_omni_multimodal(_VIDEO_ID, None)
|
||||
ds_inline._compose_daily_omni_multimodal(_VIDEO_ID, None)
|
||||
assert len(inline_calls) == 2
|
||||
assert not ds_inline._minicpm_interleave_cache
|
||||
|
||||
|
||||
def test_minicpm_interleave_missing_local_video_returns_none(qa_json, tmp_path) -> None:
|
||||
empty = tmp_path / "empty_videos"
|
||||
empty.mkdir()
|
||||
ds = _make_dataset(qa_json, empty, pack_mode="minicpm-interleave", input_mode="all")
|
||||
|
||||
parts, extra, _position = ds._compose_daily_omni_multimodal(_VIDEO_ID, None)
|
||||
|
||||
assert parts is None
|
||||
assert extra is None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# System prompt selection
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_minicpm_interleave_omits_qwen_system_message(monkeypatch, qa_json, video_dir) -> None:
|
||||
ds = _make_dataset(qa_json, video_dir, pack_mode="minicpm-interleave", input_mode="all")
|
||||
_stub_extract(monkeypatch, ds, [])
|
||||
parts, _extra, _position = ds._compose_daily_omni_multimodal(_VIDEO_ID, None)
|
||||
|
||||
messages = ds._build_daily_omni_openai_messages(parts, "q?", {"A": "a"})
|
||||
|
||||
assert [m["role"] for m in messages] == ["user"]
|
||||
|
||||
|
||||
def test_qwen_pack_mode_keeps_system_message(qa_json, video_dir) -> None:
|
||||
ds = _make_dataset(qa_json, video_dir, pack_mode="qwen", input_mode="all")
|
||||
|
||||
messages = ds._build_daily_omni_openai_messages([], "q?", {"A": "a"})
|
||||
|
||||
assert [m["role"] for m in messages] == ["system", "user"]
|
||||
assert "Qwen" in messages[0]["content"][0]["text"]
|
||||
|
||||
|
||||
def test_invalid_pack_mode_rejected(qa_json, video_dir) -> None:
|
||||
with pytest.raises(ValueError, match="pack_mode must be"):
|
||||
_make_dataset(qa_json, video_dir, pack_mode="bogus")
|
||||
@@ -28,13 +28,10 @@ if _MODULE_NAME not in sys.modules:
|
||||
_spec.loader.exec_module(_mod)
|
||||
|
||||
from vllm_omni.benchmarks.data_modules.seed_tts_dataset import ( # noqa: E402
|
||||
SeedTTSDataset,
|
||||
SeedTTSDesignDataset,
|
||||
SeedTTSDesignSampleRequest,
|
||||
SeedTTSSampleRequest,
|
||||
SeedTTSTextDataset,
|
||||
SeedTTSTextSampleRequest,
|
||||
SeedTTSTurn,
|
||||
)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -73,94 +70,6 @@ def mock_tokenizer(mocker):
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_seed_tts_dataset_groups_four_turns_with_one_reference(seed_tts_root, mock_tokenizer):
|
||||
ds = SeedTTSDataset(
|
||||
dataset_path=str(seed_tts_root),
|
||||
random_seed=0,
|
||||
locale="en",
|
||||
disable_shuffle=True,
|
||||
)
|
||||
|
||||
requests = ds.sample(
|
||||
mock_tokenizer,
|
||||
num_requests=1,
|
||||
turns_per_session=4,
|
||||
no_oversample=True,
|
||||
)
|
||||
|
||||
assert len(requests) == 1
|
||||
request = requests[0]
|
||||
assert [turn.utterance_id for turn in request.seed_tts_turns] == [
|
||||
"utt000",
|
||||
"utt001",
|
||||
"utt002",
|
||||
"utt003",
|
||||
]
|
||||
assert [turn.target_text for turn in request.seed_tts_turns] == [
|
||||
"target text 0",
|
||||
"target text 1",
|
||||
"target text 2",
|
||||
"target text 3",
|
||||
]
|
||||
assert request.seed_tts_speech_extra["ref_text"] == "ref text 0"
|
||||
assert request.seed_tts_ref_wav_path.endswith("utt000.wav")
|
||||
|
||||
|
||||
def test_seed_tts_eval_expands_grouped_pcm_by_turn():
|
||||
from vllm_omni.benchmarks.data_modules.seed_tts_eval import (
|
||||
_expand_seed_tts_turn_outputs,
|
||||
)
|
||||
|
||||
request = SeedTTSSampleRequest(
|
||||
prompt="target 0",
|
||||
prompt_len=4,
|
||||
expected_output_len=100,
|
||||
multi_modal_data=None,
|
||||
request_id="session-0",
|
||||
seed_tts_turns=tuple(
|
||||
SeedTTSTurn(utterance_id=f"utt-{index}", target_text=f"target {index}") for index in range(4)
|
||||
),
|
||||
)
|
||||
output = types.SimpleNamespace(
|
||||
success=True,
|
||||
tts_turn_pcm_bytes=[bytes([index]) for index in range(4)],
|
||||
)
|
||||
|
||||
requests, outputs = _expand_seed_tts_turn_outputs([request], [output])
|
||||
|
||||
assert [item.prompt for item in requests] == [f"target {index}" for index in range(4)]
|
||||
assert [item.seed_tts_utterance_id for item in requests] == [f"utt-{index}" for index in range(4)]
|
||||
assert [item.tts_output_pcm_bytes for item in outputs] == [bytes([index]) for index in range(4)]
|
||||
|
||||
|
||||
def test_seed_tts_eval_keeps_session_pcm_for_single_turn_chat():
|
||||
"""Chat-omni Seed-TTS sets session PCM only; expand must not wipe it."""
|
||||
from vllm_omni.benchmarks.data_modules.seed_tts_eval import (
|
||||
_expand_seed_tts_turn_outputs,
|
||||
)
|
||||
|
||||
request = SeedTTSSampleRequest(
|
||||
prompt="target 0",
|
||||
prompt_len=4,
|
||||
expected_output_len=100,
|
||||
multi_modal_data=None,
|
||||
request_id="session-0",
|
||||
seed_tts_turns=(SeedTTSTurn(utterance_id="utt-0", target_text="target 0"),),
|
||||
)
|
||||
pcm = b"\x01\x02\x03\x04"
|
||||
output = types.SimpleNamespace(
|
||||
success=True,
|
||||
tts_output_pcm_bytes=pcm,
|
||||
tts_turn_pcm_bytes=None,
|
||||
)
|
||||
|
||||
requests, outputs = _expand_seed_tts_turn_outputs([request], [output])
|
||||
|
||||
assert len(requests) == 1
|
||||
assert requests[0].prompt == "target 0"
|
||||
assert outputs[0].tts_output_pcm_bytes == pcm
|
||||
|
||||
|
||||
def test_seed_tts_text_dataset_omits_ref_audio(seed_tts_root, mock_tokenizer):
|
||||
ds = SeedTTSTextDataset(
|
||||
dataset_path=str(seed_tts_root),
|
||||
|
||||
@@ -392,6 +392,11 @@ def test_sub_config_fields_match_rfc_scopes():
|
||||
"enable_multithread_weight_load",
|
||||
"num_weight_load_threads",
|
||||
"disable_autocast",
|
||||
# Per-stage checkpoint resolution for repos whose stages live in
|
||||
# subfolders (e.g. Audex): mirrors StagePipelineConfig on the
|
||||
# legacy engine-args path.
|
||||
"model_subdir",
|
||||
"tokenizer_subdir",
|
||||
}
|
||||
assert {f.name for f in fields(OmniStageLoadConfig)} == {
|
||||
"load_format",
|
||||
|
||||
@@ -0,0 +1,182 @@
|
||||
"""Tests for the AR sampled-token logprob contract."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections import defaultdict
|
||||
from types import SimpleNamespace
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
from vllm.v1.engine import FinishReason
|
||||
from vllm.v1.outputs import LogprobsLists
|
||||
from vllm.v1.request import RequestStatus
|
||||
|
||||
from vllm_omni.core.sched.omni_ar_scheduler import OmniARScheduler, _slice_sampled_logprobs
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
def _logprobs(
|
||||
token_ids: list[list[int]],
|
||||
values: list[list[float]],
|
||||
*,
|
||||
cumulative_rows: list[int] | None = None,
|
||||
) -> LogprobsLists:
|
||||
return LogprobsLists(
|
||||
logprob_token_ids=np.asarray(token_ids, dtype=np.int32),
|
||||
logprobs=np.asarray(values, dtype=np.float32),
|
||||
sampled_token_ranks=np.zeros(len(token_ids), dtype=np.int32),
|
||||
cu_num_generated_tokens=cumulative_rows,
|
||||
)
|
||||
|
||||
|
||||
def test_slices_requested_rows_and_preserves_sampled_token_values() -> None:
|
||||
all_logprobs = _logprobs(
|
||||
[[3, 8], [4, 9], [11, 2], [12, 1]],
|
||||
[[-0.3, -1.0], [-0.4, -1.1], [-0.11, -2.0], [-0.12, -2.1]],
|
||||
cumulative_rows=[0, 2],
|
||||
)
|
||||
|
||||
sliced = _slice_sampled_logprobs(all_logprobs, req_index=1, sampled_token_ids=[11, 12])
|
||||
|
||||
np.testing.assert_array_equal(sliced.logprob_token_ids[:, 0], [11, 12])
|
||||
np.testing.assert_allclose(sliced.logprobs[:, 0], [-0.11, -0.12])
|
||||
|
||||
|
||||
def test_rejects_missing_logprobs() -> None:
|
||||
with pytest.raises(RuntimeError, match="model runner returned none"):
|
||||
_slice_sampled_logprobs(None, req_index=0, sampled_token_ids=[7])
|
||||
|
||||
|
||||
def test_rejects_row_count_mismatch() -> None:
|
||||
logprobs = _logprobs([[7, 1]], [[-0.7, -1.0]])
|
||||
|
||||
with pytest.raises(RuntimeError, match="row count does not match"):
|
||||
_slice_sampled_logprobs(logprobs, req_index=0, sampled_token_ids=[7, 8])
|
||||
|
||||
|
||||
def test_rejects_sampled_token_mismatch() -> None:
|
||||
logprobs = _logprobs([[9, 7]], [[-0.9, -1.0]])
|
||||
|
||||
with pytest.raises(RuntimeError, match="generated_token=7 logprob_token=9"):
|
||||
_slice_sampled_logprobs(logprobs, req_index=0, sampled_token_ids=[7])
|
||||
|
||||
|
||||
def test_rejects_non_finite_sampled_logprob() -> None:
|
||||
logprobs = _logprobs([[7, 1]], [[float("nan"), -1.0]])
|
||||
|
||||
with pytest.raises(RuntimeError, match="non-finite values at rows \\[0\\]"):
|
||||
_slice_sampled_logprobs(logprobs, req_index=0, sampled_token_ids=[7])
|
||||
|
||||
|
||||
class _Request:
|
||||
def __init__(self, request_id: str) -> None:
|
||||
self.request_id = request_id
|
||||
self.client_index = 0
|
||||
self.status = RequestStatus.RUNNING
|
||||
self.sampling_params = SimpleNamespace(num_logprobs=1)
|
||||
self.num_computed_tokens = 0
|
||||
self.num_output_placeholders = 0
|
||||
self.has_encoder_inputs = False
|
||||
self.pooling_params = None
|
||||
self.resumable = True
|
||||
self.stop_reason = None
|
||||
self.trace_headers = None
|
||||
self.num_nans_in_logits = 0
|
||||
|
||||
def is_finished(self) -> bool:
|
||||
return RequestStatus.is_finished(self.status)
|
||||
|
||||
def get_finished_reason(self):
|
||||
return RequestStatus.get_finished_reason(self.status)
|
||||
|
||||
def take_events(self):
|
||||
return None
|
||||
|
||||
def take_prefill_stats(self):
|
||||
return None
|
||||
|
||||
|
||||
class _RequestQueue(list):
|
||||
def remove_requests(self, requests) -> None:
|
||||
for request in requests:
|
||||
if request in self:
|
||||
self.remove(request)
|
||||
|
||||
|
||||
@pytest.mark.skip(reason="#issue 5484")
|
||||
def test_invalid_logprobs_finish_only_the_affected_scheduler_request() -> None:
|
||||
"""A bad runner response must not bring down unrelated batch requests."""
|
||||
bad = _Request("bad")
|
||||
good = _Request("good")
|
||||
scheduler = SimpleNamespace(
|
||||
perf_metrics=None,
|
||||
connector=None,
|
||||
chunk_transfer_adapter=None,
|
||||
requests={bad.request_id: bad, good.request_id: good},
|
||||
running=[bad, good],
|
||||
waiting=_RequestQueue(),
|
||||
skipped_waiting=_RequestQueue(),
|
||||
structured_output_manager=SimpleNamespace(should_advance=lambda _request: False),
|
||||
transfer_triggered_requests=set(),
|
||||
active_kv_transfers=set(),
|
||||
pending_stop_after_extraction=set(),
|
||||
waiting_for_transfer_free=set(),
|
||||
_new_prompt_len_snapshot={},
|
||||
finished_req_ids=set(),
|
||||
finished_req_ids_dict=defaultdict(set),
|
||||
kv_cache_manager=SimpleNamespace(take_events=lambda: None),
|
||||
kv_event_publisher=SimpleNamespace(publish=lambda _events: None),
|
||||
)
|
||||
update_calls: list[str] = []
|
||||
|
||||
def update_request(request, token_ids):
|
||||
update_calls.append(request.request_id)
|
||||
return token_ids, False
|
||||
|
||||
def free_request(request):
|
||||
scheduler.requests.pop(request.request_id, None)
|
||||
scheduler.finished_req_ids.add(request.request_id)
|
||||
scheduler.finished_req_ids_dict[request.client_index].add(request.request_id)
|
||||
return None
|
||||
|
||||
scheduler._update_request_with_output = update_request
|
||||
scheduler._process_kv_transfer_trigger = lambda _request, _tokens: False
|
||||
scheduler._handle_stopped_request = lambda _request: True
|
||||
scheduler._free_request = free_request
|
||||
scheduler.make_spec_decoding_stats = lambda *args, **kwargs: None
|
||||
scheduler.make_stats = lambda *args, **kwargs: None
|
||||
scheduler._capture_omni_connector_output = lambda _output: None
|
||||
|
||||
scheduler_output = SimpleNamespace(
|
||||
num_scheduled_tokens={"bad": 1, "good": 1},
|
||||
scheduled_spec_decode_tokens={},
|
||||
num_invalid_spec_tokens=0,
|
||||
)
|
||||
model_runner_output = SimpleNamespace(
|
||||
sampled_token_ids=[[7], [8]],
|
||||
logprobs=_logprobs(
|
||||
[[9, 1], [8, 2]],
|
||||
[[-0.9, -1.0], [-0.8, -1.0]],
|
||||
cumulative_rows=[0, 1],
|
||||
),
|
||||
prompt_logprobs_dict={},
|
||||
pooler_output=None,
|
||||
num_nans_in_logits=None,
|
||||
kv_connector_output=None,
|
||||
cudagraph_stats=None,
|
||||
req_id_to_index={"bad": 0, "good": 1},
|
||||
routed_experts=None,
|
||||
)
|
||||
|
||||
outputs = OmniARScheduler.update_from_output(scheduler, scheduler_output, model_runner_output)
|
||||
output_by_id = {output.request_id: output for output in outputs[0].outputs}
|
||||
|
||||
assert update_calls == ["good"]
|
||||
assert bad.status is RequestStatus.FINISHED_ERROR
|
||||
assert bad.stop_reason is not None and "misaligned" in bad.stop_reason
|
||||
assert "bad" not in scheduler.requests
|
||||
assert output_by_id["bad"].finish_reason is FinishReason.ERROR
|
||||
assert output_by_id["bad"].new_token_ids == []
|
||||
assert output_by_id["good"].new_token_ids == [8]
|
||||
np.testing.assert_array_equal(output_by_id["good"].new_logprobs.logprob_token_ids[:, 0], [8])
|
||||
@@ -73,6 +73,9 @@ def _run_resumable_segment_stop(
|
||||
|
||||
sched._update_request_with_output.side_effect = stop_request
|
||||
sched._handle_stopped_request.return_value = session_finished
|
||||
# vLLM 0.26 returns (kv_xfer_params, ec_xfer_params); an unconfigured
|
||||
# MagicMock iterates empty and fails to unpack at the call site.
|
||||
sched._free_request.return_value = (None, None)
|
||||
sched.chunk_transfer_adapter = None
|
||||
sched.running = [session]
|
||||
sched.waiting_for_transfer_free = set()
|
||||
@@ -137,6 +140,23 @@ def test_resumable_session_terminal_is_not_marked_as_segment_boundary() -> None:
|
||||
assert output.is_segment_finished is False
|
||||
|
||||
|
||||
def test_update_from_output_settles_in_flight_tokens() -> None:
|
||||
"""vLLM 0.26: schedule() increments num_in_flight_tokens per scheduled
|
||||
token; update_from_output must decrement it symmetrically. If the
|
||||
decrement is dropped the counter grows monotonically and both readers
|
||||
(allocate_slots, _connector_finished) clamp
|
||||
max(0, num_computed_tokens - num_in_flight_tokens) to zero forever,
|
||||
silently freezing sliding-window block freeing.
|
||||
"""
|
||||
session = _make_request()
|
||||
session.status = RequestStatus.RUNNING
|
||||
session.num_in_flight_tokens = 1 # as left by schedule() for this step
|
||||
|
||||
_run_resumable_segment_stop(session)
|
||||
|
||||
assert session.num_in_flight_tokens == 0
|
||||
|
||||
|
||||
def test_running_decode_step_without_inter_stage_payload_does_not_raise() -> None:
|
||||
"""A decode step that neither stops nor carries an inter-stage payload.
|
||||
|
||||
@@ -242,52 +262,9 @@ def test_explicit_streaming_payload_replaces_placeholder_prompt() -> None:
|
||||
"tts_token_ids": [10, 20],
|
||||
"meta": {"replace_streaming_prompt": True},
|
||||
}
|
||||
update.model_intermediate_buffer = {
|
||||
"ids": {"tts": [41, 42, 99]},
|
||||
"meta": {"turn_eos_token_id": 99},
|
||||
}
|
||||
|
||||
sched._update_request_as_session(session, update)
|
||||
|
||||
assert session.prompt_token_ids == [10, 20]
|
||||
assert session.additional_information == update.additional_information
|
||||
assert session.model_intermediate_buffer == {
|
||||
"ids": {"tts": [41, 42, 99]},
|
||||
"meta": {"turn_eos_token_id": 99},
|
||||
}
|
||||
assert session.status == RequestStatus.WAITING
|
||||
|
||||
|
||||
def test_model_intermediate_streaming_payload_replaces_computed_prompt() -> None:
|
||||
sched = _make_scheduler(stage_id=1)
|
||||
sched.chunk_transfer_adapter = SimpleNamespace(
|
||||
receives_chunks=False,
|
||||
segment_finished_requests=set(),
|
||||
)
|
||||
session = _make_request()
|
||||
session.status = RequestStatus.WAITING_FOR_STREAMING_REQ
|
||||
session.prompt_token_ids = [0] * 59
|
||||
session._all_token_ids.clear()
|
||||
session._all_token_ids.extend(session.prompt_token_ids)
|
||||
session.num_prompt_tokens = 59
|
||||
session.num_computed_tokens = 59
|
||||
update = _make_update([0] * 10)
|
||||
update.additional_information = None
|
||||
update.model_intermediate_buffer = {
|
||||
"ids": {"tts": list(range(8))},
|
||||
"hidden_states": {"tts": [[0.0]] * 8},
|
||||
"meta": {
|
||||
"replace_streaming_prompt": True,
|
||||
"next_stage_prompt_len": 10,
|
||||
},
|
||||
}
|
||||
|
||||
sched._update_request_as_session(session, update)
|
||||
|
||||
assert session.prompt_token_ids == [0] * 10
|
||||
assert list(session._all_token_ids) == [0] * 10
|
||||
assert session.num_prompt_tokens == 10
|
||||
assert session.num_computed_tokens == 0
|
||||
assert session.additional_information is None
|
||||
assert session.model_intermediate_buffer == update.model_intermediate_buffer
|
||||
assert session.status == RequestStatus.WAITING
|
||||
@@ -54,18 +54,6 @@ class _SchedulerStub(OmniGenerationScheduler):
|
||||
raise AssertionError("unexpected enqueue for skipped_waiting miss")
|
||||
|
||||
|
||||
class _AsyncChunkStopSchedulerStub(OmniGenerationScheduler):
|
||||
def __init__(self) -> None:
|
||||
self.num_waiting_for_streaming_input = 0
|
||||
self.chunk_transfer_adapter = SimpleNamespace(receives_chunks=True)
|
||||
self.enqueued: list[Request] = []
|
||||
self.enqueued_statuses: list[RequestStatus] = []
|
||||
|
||||
def _enqueue_waiting_request(self, session: Request) -> None:
|
||||
self.enqueued.append(session)
|
||||
self.enqueued_statuses.append(session.status)
|
||||
|
||||
|
||||
def _make_request(**kwargs) -> Request:
|
||||
sp = SamplingParams(max_tokens=8)
|
||||
defaults = dict(
|
||||
@@ -145,21 +133,6 @@ def test_resumable_generation_stop_marks_segment_boundary() -> None:
|
||||
assert output.is_segment_finished is True
|
||||
|
||||
|
||||
def test_async_chunk_resumable_stop_rearms_connector_polling() -> None:
|
||||
sched = _AsyncChunkStopSchedulerStub()
|
||||
session = _make_request(request_id="req-async-chunk-next-segment")
|
||||
session.status = RequestStatus.FINISHED_STOPPED
|
||||
session.resumable = True
|
||||
|
||||
finished = sched._handle_stopped_request(session)
|
||||
|
||||
assert finished is False
|
||||
assert sched.enqueued == [session]
|
||||
assert sched.enqueued_statuses == [RequestStatus.WAITING]
|
||||
assert session.status == RequestStatus.WAITING
|
||||
assert sched.num_waiting_for_streaming_input == 0
|
||||
|
||||
|
||||
class TestReplaceSessionWithStreamingUpdate:
|
||||
def test_resets_tokens_and_prompt_from_update(self) -> None:
|
||||
sched = _SchedulerStub()
|
||||
|
||||
@@ -76,8 +76,9 @@ def test_finish_requests_purges_finished_request_from_running(
|
||||
)
|
||||
|
||||
def fake_finish_requests(self, request_ids, finished_status):
|
||||
# super() returning the abort tuple but leaving self.running untouched
|
||||
return [(finished.request_id, 0)]
|
||||
# vLLM 0.26 super() returns the finished Request objects but here
|
||||
# leaves self.running untouched to simulate the missed removal.
|
||||
return [finished]
|
||||
|
||||
monkeypatch.setattr(scheduler_mod.VLLMScheduler, "finish_requests", fake_finish_requests)
|
||||
|
||||
@@ -88,7 +89,7 @@ def test_finish_requests_purges_finished_request_from_running(
|
||||
)
|
||||
|
||||
assert scheduler.running == []
|
||||
assert result == [(finished.request_id, 0)]
|
||||
assert result == [finished]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("scheduler_cls", "scheduler_mod"), _SCHEDULER_PARAMS)
|
||||
@@ -111,7 +112,7 @@ def test_finish_requests_purges_untracked_request_from_running(
|
||||
)
|
||||
|
||||
def fake_finish_requests(self, request_ids, finished_status):
|
||||
return [(untracked.request_id, 0)]
|
||||
return [untracked]
|
||||
|
||||
monkeypatch.setattr(scheduler_mod.VLLMScheduler, "finish_requests", fake_finish_requests)
|
||||
|
||||
@@ -122,7 +123,7 @@ def test_finish_requests_purges_untracked_request_from_running(
|
||||
)
|
||||
|
||||
assert scheduler.running == []
|
||||
assert result == [(untracked.request_id, 0)]
|
||||
assert result == [untracked]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("scheduler_cls", "scheduler_mod"), _SCHEDULER_PARAMS)
|
||||
@@ -136,7 +137,7 @@ def test_finish_requests_leaves_healthy_running_intact(
|
||||
is conservative and won't aggressively drop healthy entries.
|
||||
"""
|
||||
alive = _StubRequest("req-alive", RequestStatus.RUNNING)
|
||||
leaving_id = "req-leaving"
|
||||
leaving = _StubRequest("req-leaving", RequestStatus.FINISHED_STOPPED)
|
||||
scheduler = _make_scheduler(
|
||||
scheduler_cls,
|
||||
requests={alive.request_id: alive},
|
||||
@@ -145,13 +146,13 @@ def test_finish_requests_leaves_healthy_running_intact(
|
||||
)
|
||||
|
||||
def fake_finish_requests(self, request_ids, finished_status):
|
||||
return [(leaving_id, 0)]
|
||||
return [leaving]
|
||||
|
||||
monkeypatch.setattr(scheduler_mod.VLLMScheduler, "finish_requests", fake_finish_requests)
|
||||
|
||||
scheduler_cls.finish_requests(
|
||||
scheduler,
|
||||
[leaving_id],
|
||||
[leaving.request_id],
|
||||
RequestStatus.FINISHED_STOPPED,
|
||||
)
|
||||
|
||||
|
||||
@@ -53,7 +53,11 @@ def test_finish_requests_cleans_input_coordinator_for_finished_ids(
|
||||
def fake_finish_requests(self, request_ids, finished_status):
|
||||
assert request_ids == ["req-a", "req-b"]
|
||||
assert finished_status == RequestStatus.FINISHED_STOPPED
|
||||
return [("req-a", 0), ("req-b", 1)]
|
||||
# vLLM 0.26 returns the finished Request objects, not (req_id, client_index) tuples.
|
||||
return [
|
||||
SimpleNamespace(request_id="req-a", client_index=0),
|
||||
SimpleNamespace(request_id="req-b", client_index=1),
|
||||
]
|
||||
|
||||
monkeypatch.setattr(scheduler_mod.VLLMScheduler, "finish_requests", fake_finish_requests)
|
||||
|
||||
@@ -63,7 +67,7 @@ def test_finish_requests_cleans_input_coordinator_for_finished_ids(
|
||||
RequestStatus.FINISHED_STOPPED,
|
||||
)
|
||||
|
||||
assert result == [("req-a", 0), ("req-b", 1)]
|
||||
assert [(r.request_id, r.client_index) for r in result] == [("req-a", 0), ("req-b", 1)]
|
||||
assert coordinator.freed == ["req-a", "req-b"]
|
||||
|
||||
|
||||
@@ -82,7 +86,7 @@ def test_ar_free_request_cleans_input_coordinator_on_normal_free() -> None:
|
||||
|
||||
result = OmniARScheduler._free_request(scheduler, DummyRequest())
|
||||
|
||||
assert result is None
|
||||
assert result == (None, None)
|
||||
assert coordinator.freed == ["req-free"]
|
||||
assert scheduler._omits_kv_transfer_cache == {}
|
||||
assert scheduler._new_prompt_len_snapshot == {}
|
||||
|
||||
@@ -121,6 +121,10 @@ class TestFullPayloadCoordinatorSelection(unittest.TestCase):
|
||||
("Qwen3TTSCode2Wav", "code2wav"),
|
||||
("MiniCPMO45Code2Wav", "code2wav"),
|
||||
("CosyVoice3Model", "cosyvoice3_code2wav"),
|
||||
# audex: sync-path TTS decoder (async_chunk: false deploys) and
|
||||
# the TTA XCodec1 stage (always sync full-payload).
|
||||
("AudexCode2Wav", "audex_code2wav"),
|
||||
("AudexXCodec1", "audex_xcodec"),
|
||||
("IndexTTS2S2MelDecoder", "indextts2_s2mel_decoder"),
|
||||
("DyninOmniForConditionalGeneration", "token2image"),
|
||||
("DyninOmniForConditionalGeneration", "token2audio"),
|
||||
|
||||
+150
-93
@@ -3,13 +3,15 @@ import os
|
||||
import re
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.helpers.mark import get_hardware_mark_list, hardware_marks
|
||||
from tests.dfx.reliability.helpers import list_remote_process_pids_by_pattern, post_chat_completions_raw
|
||||
from tests.helpers.mark import hardware_marks
|
||||
from tests.helpers.runtime import OmniServerParams
|
||||
from tests.helpers.stage_config import modify_stage_config
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
@@ -117,6 +119,32 @@ def create_unique_server_pytest_params(
|
||||
]
|
||||
|
||||
|
||||
def create_benchmark_pytest_params(
|
||||
benchmark_configs: list[dict[str, Any]],
|
||||
server_to_benchmark_mapping: dict[str, dict],
|
||||
) -> list[Any]:
|
||||
"""Like :func:`create_benchmark_indices`, but wrap each index in ``pytest.param`` with JSON marks."""
|
||||
marks_by_name = _marks_by_test_name(benchmark_configs)
|
||||
params: list[Any] = []
|
||||
seen: set[str] = set()
|
||||
for config in benchmark_configs:
|
||||
test_name = config["test_name"]
|
||||
if test_name in seen:
|
||||
continue
|
||||
seen.add(test_name)
|
||||
params_list = get_benchmark_params_for_server(test_name, server_to_benchmark_mapping)
|
||||
id_suffixes = _unique_benchmark_param_id_suffixes(params_list)
|
||||
for idx, id_suffix in enumerate(id_suffixes):
|
||||
params.append(
|
||||
pytest.param(
|
||||
(test_name, idx),
|
||||
marks=marks_by_name.get(test_name, []),
|
||||
id=f"{test_name}-{id_suffix}",
|
||||
)
|
||||
)
|
||||
return params
|
||||
|
||||
|
||||
def create_paired_benchmark_pytest_params(
|
||||
server_entries: list[tuple[Any, str]],
|
||||
params_by_test_name: dict[str, list[Any]],
|
||||
@@ -329,6 +357,10 @@ def supports_video_generation(model_name: str) -> bool:
|
||||
return any(key in lower for key in ("wan", "video", "i2v", "t2v"))
|
||||
|
||||
|
||||
def supports_chat_generation(model_name: str) -> bool:
|
||||
return not supports_video_generation(model_name)
|
||||
|
||||
|
||||
def parse_stage_devices(stage_config_path: str) -> str:
|
||||
text = Path(stage_config_path).read_text(encoding="utf-8")
|
||||
raw_devices: list[str] = re.findall(r"^\s*devices:\s*\"?([0-9,\s]+)\"?\s*$", text, flags=re.MULTILINE)
|
||||
@@ -357,6 +389,48 @@ def assert_fault_exception(exc: Exception, error_keywords: tuple[str, ...]) -> N
|
||||
assert any(key in text for key in error_keywords), f"unexpected error under fault injection: {exc}"
|
||||
|
||||
|
||||
def wait_chat_request_ready(host: str, port: int, model: str, timeout_sec: int = 180) -> None:
|
||||
"""Poll a minimal chat request until success."""
|
||||
deadline = time.time() + timeout_sec
|
||||
payload = json.dumps(
|
||||
{
|
||||
"model": model,
|
||||
"messages": [{"role": "user", "content": "Say hello in one short sentence."}],
|
||||
"stream": False,
|
||||
"modalities": ["text"],
|
||||
}
|
||||
)
|
||||
last_error: str | None = None
|
||||
while time.time() < deadline:
|
||||
try:
|
||||
status, body = post_chat_completions_raw(host, port, payload)
|
||||
if status == 200:
|
||||
return
|
||||
last_error = f"http={status} body={body[:200]!r}"
|
||||
except Exception as exc: # noqa: BLE001
|
||||
last_error = str(exc)
|
||||
time.sleep(2)
|
||||
raise TimeoutError(f"runtime-teardown warmup request did not succeed within {timeout_sec}s: {last_error}")
|
||||
|
||||
|
||||
def assert_no_extra_worker_processes(
|
||||
baseline_pids: set[int],
|
||||
worker_pattern: str,
|
||||
timeout_sec: int = 60,
|
||||
) -> None:
|
||||
"""Ensure no extra worker PIDs remain after teardown."""
|
||||
deadline = time.time() + timeout_sec
|
||||
while time.time() < deadline:
|
||||
current = set(list_remote_process_pids_by_pattern(worker_pattern))
|
||||
extra = current - baseline_pids
|
||||
if not extra:
|
||||
return
|
||||
time.sleep(2)
|
||||
current = set(list_remote_process_pids_by_pattern(worker_pattern))
|
||||
extra = sorted(current - baseline_pids)
|
||||
assert not extra, f"orphan worker processes remain after container teardown: {extra}"
|
||||
|
||||
|
||||
def create_test_parameter_mapping(configs: list[dict[str, Any]]) -> dict[str, dict]:
|
||||
mapping = {}
|
||||
for config in configs:
|
||||
@@ -446,16 +520,36 @@ def _unique_benchmark_param_id_suffixes(params_list: list[dict[str, Any]]) -> li
|
||||
return unique
|
||||
|
||||
|
||||
# Device marketing-name tokens for ``get_runtime_resource_label`` only.
|
||||
# Longer / more specific tokens must appear before shorter substrings
|
||||
# (e.g. ``H200`` before ``H20``, ``910B4`` before ``910``).
|
||||
_RUNTIME_DEVICE_ALIASES: tuple[str, ...] = (
|
||||
def extract_mark_resource_label(mark_field: Any) -> str:
|
||||
"""Return a filename-safe hardware label from ``mark.hardware_marks.res`` values.
|
||||
|
||||
Example: ``{"cuda": "H100"}`` -> ``"H100"``; multiple platforms join with ``-``.
|
||||
|
||||
Prefer :func:`get_runtime_resource_label` for perf result filenames so labels
|
||||
reflect the machine that actually ran the benchmark.
|
||||
"""
|
||||
if isinstance(mark_field, list):
|
||||
for item in mark_field:
|
||||
if isinstance(item, dict) and "hardware_marks" in item:
|
||||
return extract_mark_resource_label(item)
|
||||
return "na"
|
||||
if not isinstance(mark_field, dict):
|
||||
return "na"
|
||||
hw = mark_field.get("hardware_marks")
|
||||
if not isinstance(hw, dict):
|
||||
return "na"
|
||||
res = hw.get("res")
|
||||
if not isinstance(res, dict) or not res:
|
||||
return "na"
|
||||
labels = [_safe_filename_token(value) for value in res.values()]
|
||||
return "-".join(labels) if labels else "na"
|
||||
|
||||
|
||||
_KNOWN_RUNTIME_RESOURCE_TOKENS: tuple[str, ...] = (
|
||||
"H100",
|
||||
"H800",
|
||||
"H200",
|
||||
"H20",
|
||||
"B200",
|
||||
"GB200",
|
||||
"L40S",
|
||||
"L40",
|
||||
"L4",
|
||||
@@ -484,7 +578,7 @@ def _normalize_runtime_device_label(raw: str) -> str:
|
||||
if not raw or not str(raw).strip():
|
||||
return "na"
|
||||
upper = str(raw).upper()
|
||||
for token in _RUNTIME_DEVICE_ALIASES:
|
||||
for token in _KNOWN_RUNTIME_RESOURCE_TOKENS:
|
||||
if token.upper() in upper:
|
||||
return _safe_filename_token(token)
|
||||
compact = re.sub(r"[^a-zA-Z0-9]+", "", str(raw))
|
||||
@@ -537,41 +631,34 @@ def resource_label_for_filename(resource_label: str | None) -> str:
|
||||
return token
|
||||
|
||||
|
||||
def is_hardware_nested_baseline(baseline: dict[str, Any]) -> bool:
|
||||
"""True for ``{"H100": {"metric": ...}, "A3": {...}}`` baselines.
|
||||
|
||||
Detection is structural + hardware-label allowlist (not metric-name hints):
|
||||
|
||||
- every top-level value is a non-empty dict (per-hardware metric map);
|
||||
- every top-level key is in :func:`tests.helpers.mark.get_hardware_mark_list`
|
||||
(``[hardware-resource]`` markers from ``pyproject.toml``).
|
||||
|
||||
Metric names under each hardware bucket are unconstrained.
|
||||
Runtime device aliases (:data:`_RUNTIME_DEVICE_ALIASES`) are unrelated.
|
||||
"""
|
||||
if not baseline:
|
||||
return False
|
||||
if not all(isinstance(value, dict) and bool(value) for value in baseline.values()):
|
||||
return False
|
||||
allowed = {label.upper() for label in get_hardware_mark_list()}
|
||||
return all((token := str(key).strip()) and token.upper() in allowed for key in baseline.keys())
|
||||
def extract_configs_resource_label(configs: list[dict[str, Any]]) -> str:
|
||||
"""Return runtime hardware label for perf result filenames."""
|
||||
del configs
|
||||
return get_runtime_resource_label()
|
||||
|
||||
|
||||
def resolve_baseline_value(
|
||||
baseline_raw: Any,
|
||||
*,
|
||||
sweep_index: int | None,
|
||||
max_concurrency: Any = None,
|
||||
request_rate: Any = None,
|
||||
) -> Any:
|
||||
"""Pick the baseline threshold for one metric at this sweep step.
|
||||
|
||||
Accepts a sweep-aligned list/tuple (indexed by ``sweep_index``) or a scalar.
|
||||
"""
|
||||
"""Pick the baseline threshold for this sweep step."""
|
||||
if baseline_raw is None:
|
||||
return None
|
||||
return 100000
|
||||
if isinstance(baseline_raw, dict):
|
||||
raise TypeError(
|
||||
"per-metric baseline dict keyed by concurrency/request-rate is not supported; "
|
||||
"use a sweep-aligned list or a scalar under each hardware bucket"
|
||||
if max_concurrency is not None:
|
||||
for key in (max_concurrency, str(max_concurrency)):
|
||||
if key in baseline_raw:
|
||||
return baseline_raw[key]
|
||||
if request_rate is not None:
|
||||
for key in (request_rate, str(request_rate)):
|
||||
if key in baseline_raw:
|
||||
return baseline_raw[key]
|
||||
raise KeyError(
|
||||
f"baseline dict has no key for max_concurrency={max_concurrency!r} "
|
||||
f"or request_rate={request_rate!r}; keys={list(baseline_raw.keys())!r}"
|
||||
)
|
||||
if isinstance(baseline_raw, (list, tuple)):
|
||||
if sweep_index is None:
|
||||
@@ -582,62 +669,22 @@ def resolve_baseline_value(
|
||||
return baseline_raw
|
||||
|
||||
|
||||
def resolve_baseline_for_sweep(
|
||||
baseline_config: dict[str, Any] | None,
|
||||
def _baseline_thresholds_for_step(
|
||||
baseline_data: dict[str, Any],
|
||||
*,
|
||||
sweep_index: int | None = None,
|
||||
max_concurrency: Any = None,
|
||||
request_rate: Any = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Resolve sweep-dependent baselines while keeping every hardware bucket.
|
||||
|
||||
Expects hardware-nested input only::
|
||||
|
||||
{"H100": {"throughput_qps": [a, b, c]}, "A3": {"throughput_qps": [x, y, z]}}
|
||||
|
||||
with ``sweep_index=2`` (e.g. concurrency 32) becomes::
|
||||
|
||||
{"H100": {"throughput_qps": c}, "A3": {"throughput_qps": z}}
|
||||
|
||||
Per-metric values may be a sweep-aligned list or a scalar.
|
||||
"""
|
||||
if not baseline_config:
|
||||
return {}
|
||||
if not is_hardware_nested_baseline(baseline_config):
|
||||
allowed = sorted(get_hardware_mark_list())
|
||||
unknown = [
|
||||
str(key).strip()
|
||||
for key in baseline_config.keys()
|
||||
if str(key).strip().upper() not in {label.upper() for label in allowed}
|
||||
]
|
||||
nested_shape = all(isinstance(value, dict) and bool(value) for value in baseline_config.values())
|
||||
if nested_shape and unknown:
|
||||
raise ValueError(
|
||||
"baseline top-level keys must be [hardware-resource] pytest markers from "
|
||||
"pyproject.toml (tool.pytest.ini_options.markers). "
|
||||
f"Unknown hardware label(s): {unknown!r}. "
|
||||
f"Known markers: {allowed!r}. "
|
||||
"Add a marker such as '\"A100: [hardware-resource] Tests that require A100 GPU\"' "
|
||||
"to pyproject.toml, then retry."
|
||||
)
|
||||
raise ValueError(
|
||||
"baseline must be hardware-nested with [hardware-resource] marker names as "
|
||||
"top-level keys, e.g. "
|
||||
'{"H100": {"throughput_qps": [...]}, "A3": {"custom_metric": 1.0}}. '
|
||||
f"got top-level keys={list(baseline_config.keys())!r}; "
|
||||
f"known markers={allowed!r}. "
|
||||
"If you need a new hardware bucket, add it to pyproject.toml markers with "
|
||||
"the [hardware-resource] tag."
|
||||
)
|
||||
|
||||
"""Resolve baseline config to one threshold per metric for this iteration."""
|
||||
return {
|
||||
hardware: {
|
||||
metric_name: resolve_baseline_value(
|
||||
baseline_raw,
|
||||
sweep_index=sweep_index,
|
||||
)
|
||||
for metric_name, baseline_raw in metrics.items()
|
||||
}
|
||||
for hardware, metrics in baseline_config.items()
|
||||
if isinstance(metrics, dict)
|
||||
metric_name: resolve_baseline_value(
|
||||
baseline_raw,
|
||||
sweep_index=sweep_index,
|
||||
max_concurrency=max_concurrency,
|
||||
request_rate=request_rate,
|
||||
)
|
||||
for metric_name, baseline_raw in baseline_data.items()
|
||||
}
|
||||
|
||||
|
||||
@@ -650,17 +697,17 @@ def run_benchmark(
|
||||
*,
|
||||
baseline_config: dict[str, Any] | None = None,
|
||||
sweep_index: int | None = None,
|
||||
request_rate: Any | None = None,
|
||||
max_concurrency: Any | None = None,
|
||||
random_input_len: Any | None = None,
|
||||
random_output_len: Any | None = None,
|
||||
resource_label: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Run one ``vllm bench serve --omni`` iteration and return parsed metrics.
|
||||
|
||||
After ``vllm bench`` writes the JSON, ``result["baseline"]`` stores the
|
||||
sweep-resolved baseline (when ``baseline_config`` is provided). Hardware-nested
|
||||
maps keep every hardware bucket, but each metric is reduced to the value for
|
||||
this concurrency / request-rate step. If the benchmark exits without writing a
|
||||
result file, ``result_omni_template.json`` is used as a fallback.
|
||||
After ``vllm bench`` writes the JSON, ``result["baseline"]`` holds the resolved per-metric thresholds
|
||||
(when ``baseline_config`` is provided). If the benchmark exits without writing a result file,
|
||||
``result_omni_template.json`` is used as a fallback.
|
||||
"""
|
||||
current_dt = datetime.now().strftime("%Y%m%d-%H%M%S")
|
||||
ri = _safe_filename_token(random_input_len)
|
||||
@@ -727,10 +774,11 @@ def run_benchmark(
|
||||
result = json.load(f)
|
||||
|
||||
if baseline_config:
|
||||
# Keep every hardware bucket; resolve list metrics to this sweep step.
|
||||
result["baseline"] = resolve_baseline_for_sweep(
|
||||
result["baseline"] = _baseline_thresholds_for_step(
|
||||
baseline_config,
|
||||
sweep_index=sweep_index,
|
||||
request_rate=request_rate,
|
||||
max_concurrency=max_concurrency,
|
||||
)
|
||||
else:
|
||||
result["baseline"] = {}
|
||||
@@ -752,3 +800,12 @@ def pytest_addoption(parser: pytest.Parser) -> None:
|
||||
default=None,
|
||||
help=("Path to benchmark config JSON. Example: --test-config-file tests/dfx/perf/tests/test_tts.json"),
|
||||
)
|
||||
parser.addoption(
|
||||
"--assert-baseline",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help=(
|
||||
"When set, omni/diffusion perf runners compare metrics against the baseline block in the JSON config "
|
||||
"(default: off)."
|
||||
),
|
||||
)
|
||||
@@ -21,12 +21,10 @@
|
||||
"num-input-images": 0,
|
||||
"enable-negative-prompt": false,
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"throughput_qps": 0,
|
||||
"latency_mean": 0,
|
||||
"peak_memory_mb_max": 0,
|
||||
"peak_memory_mb_mean": 0
|
||||
}
|
||||
"throughput_qps": 0,
|
||||
"latency_mean": 0,
|
||||
"peak_memory_mb_max": 0,
|
||||
"peak_memory_mb_mean": 0
|
||||
}
|
||||
},
|
||||
"result": {
|
||||
|
||||
@@ -46,11 +46,9 @@
|
||||
"median_audio_duration_s": 0,
|
||||
"p99_audio_duration_s": 0,
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"mean_ttft_ms": 0,
|
||||
"mean_audio_ttfp_ms": 0,
|
||||
"mean_audio_rtf": 0
|
||||
}
|
||||
"mean_ttft_ms": 0,
|
||||
"mean_audio_ttfp_ms": 0,
|
||||
"mean_audio_rtf": 0
|
||||
},
|
||||
"random_input_len": 0,
|
||||
"random_output_len": 0
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
import json
|
||||
import os
|
||||
import threading
|
||||
from collections.abc import Callable
|
||||
from contextlib import AbstractContextManager, ExitStack, contextmanager
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
@@ -15,10 +13,15 @@ from tests.dfx.conftest import (
|
||||
get_runtime_resource_label,
|
||||
is_diffusion_perf_config,
|
||||
load_benchmark_configs,
|
||||
resolve_baseline_value,
|
||||
run_benchmark,
|
||||
)
|
||||
from tests.helpers.runtime import OmniServer
|
||||
|
||||
# Compare metrics to each test JSON ``baseline`` block only when pytest is run with ``--assert-baseline``
|
||||
# (registered in ``tests/dfx/conftest.py``; default: off). ``run_benchmark`` and ``_resolve_baseline_value`` are
|
||||
# defined in the same module.
|
||||
#
|
||||
# Optional JSON field ``mark`` is applied as pytest marks via
|
||||
# ``create_paired_omni_benchmark_pytest_params`` (e.g. ``"mark": [{"hardware_marks":
|
||||
# {"res": {"cuda": "H100"}, "num_cards": 2}}, "full_model", "omni"]``).
|
||||
@@ -60,78 +63,35 @@ paired_benchmark_params = create_paired_omni_benchmark_pytest_params(BENCHMARK_C
|
||||
_omni_server_lock = threading.Lock()
|
||||
|
||||
|
||||
class _SingleActiveContext:
|
||||
"""Reuse one active context while its configuration key is unchanged."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._key: Any = None
|
||||
self._stack: ExitStack | None = None
|
||||
self._value: Any = None
|
||||
|
||||
def acquire(self, key: Any, factory: Callable[[], AbstractContextManager[Any]]) -> Any:
|
||||
if self._stack is not None and key == self._key:
|
||||
return self._value
|
||||
|
||||
self.close()
|
||||
stack = ExitStack()
|
||||
value = stack.enter_context(factory())
|
||||
self._key = key
|
||||
self._stack = stack
|
||||
self._value = value
|
||||
return value
|
||||
|
||||
def close(self) -> None:
|
||||
stack = self._stack
|
||||
self._key = None
|
||||
self._stack = None
|
||||
self._value = None
|
||||
if stack is not None:
|
||||
stack.close()
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _start_omni_server(server_param):
|
||||
test_name, model, stage_config_path, stage_overrides, extra_cli_args, use_omni = server_param
|
||||
|
||||
print(f"Starting OmniServer with test: {test_name}, model: {model}")
|
||||
|
||||
server_args: list[str] = []
|
||||
if use_omni:
|
||||
server_args += ["--stage-init-timeout", "600", "--init-timeout", "900"]
|
||||
# --deploy-config and --stage-overrides compose at the CLI (see vllm_omni/entrypoints/utils.py):
|
||||
# deploy-config sets the base; stage-overrides are applied on top. Both can be set.
|
||||
if stage_config_path:
|
||||
server_args = ["--deploy-config", stage_config_path] + server_args
|
||||
if stage_overrides:
|
||||
server_args = ["--stage-overrides", stage_overrides] + server_args
|
||||
if extra_cli_args:
|
||||
server_args = list(extra_cli_args) + server_args
|
||||
with OmniServer(model, server_args, use_omni=use_omni) as server:
|
||||
server.test_name = test_name
|
||||
print("OmniServer started successfully")
|
||||
yield server
|
||||
print("OmniServer stopping...")
|
||||
|
||||
print("OmniServer stopped")
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def omni_server_context():
|
||||
def omni_server(request):
|
||||
"""Start vLLM-Omni server as a subprocess with actual model weights.
|
||||
Reuse it for adjacent benchmark cases with the same server configuration.
|
||||
Uses session scope so the server starts only once for the entire test session.
|
||||
Multi-stage initialization can take 10-20+ minutes.
|
||||
"""
|
||||
with _omni_server_lock:
|
||||
active_context = _SingleActiveContext()
|
||||
try:
|
||||
yield active_context
|
||||
finally:
|
||||
active_context.close()
|
||||
test_name, model, stage_config_path, stage_overrides, extra_cli_args, use_omni = request.param
|
||||
|
||||
print(f"Starting OmniServer with test: {test_name}, model: {model}")
|
||||
|
||||
@pytest.fixture
|
||||
def omni_server(request, omni_server_context):
|
||||
return omni_server_context.acquire(request.param, lambda: _start_omni_server(request.param))
|
||||
server_args: list[str] = []
|
||||
if use_omni:
|
||||
server_args += ["--stage-init-timeout", "600", "--init-timeout", "900"]
|
||||
# --deploy-config and --stage-overrides compose at the CLI (see vllm_omni/entrypoints/utils.py):
|
||||
# deploy-config sets the base; stage-overrides are applied on top. Both can be set.
|
||||
if stage_config_path:
|
||||
server_args = ["--deploy-config", stage_config_path] + server_args
|
||||
if stage_overrides:
|
||||
server_args = ["--stage-overrides", stage_overrides] + server_args
|
||||
if extra_cli_args:
|
||||
server_args = list(extra_cli_args) + server_args
|
||||
with OmniServer(model, server_args, use_omni=use_omni) as server:
|
||||
server.test_name = test_name
|
||||
print("OmniServer started successfully")
|
||||
yield server
|
||||
print("OmniServer stopping...")
|
||||
|
||||
print("OmniServer stopped")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
@@ -157,19 +117,36 @@ def benchmark_params(request):
|
||||
}
|
||||
|
||||
|
||||
def assert_result(result, params, num_prompt) -> None:
|
||||
def assert_result(
|
||||
result,
|
||||
params,
|
||||
num_prompt,
|
||||
*,
|
||||
assert_baseline: bool,
|
||||
sweep_index: int | None = None,
|
||||
max_concurrency: Any | None = None,
|
||||
request_rate: Any | None = None,
|
||||
) -> None:
|
||||
assert result["completed"] == num_prompt, "Request failures exist"
|
||||
expected_audio_turns = params.get("expected_duplex_audio_turns_per_session")
|
||||
if expected_audio_turns is not None:
|
||||
session_metrics = result.get("duplex_session_metrics")
|
||||
assert isinstance(session_metrics, list), "Duplex session metrics are missing"
|
||||
assert len(session_metrics) == num_prompt, (
|
||||
f"Expected {num_prompt} duplex session metric rows, got {len(session_metrics)}"
|
||||
if not assert_baseline:
|
||||
return
|
||||
baseline_data = params.get("baseline", {})
|
||||
for metric_name, baseline_raw in baseline_data.items():
|
||||
current_value = result[metric_name]
|
||||
baseline_value = resolve_baseline_value(
|
||||
baseline_raw,
|
||||
sweep_index=sweep_index,
|
||||
max_concurrency=max_concurrency,
|
||||
request_rate=request_rate,
|
||||
)
|
||||
assert all(
|
||||
isinstance(metric, dict) and metric.get("audio_turn_count") == expected_audio_turns
|
||||
for metric in session_metrics
|
||||
), f"Not every duplex session emitted {expected_audio_turns} audio turns"
|
||||
if "throughput" in metric_name:
|
||||
if current_value <= baseline_value:
|
||||
print(
|
||||
f"ERROR: Throughput test results were below baseline: {metric_name}: {current_value} > {baseline_value}"
|
||||
)
|
||||
else:
|
||||
if current_value >= baseline_value:
|
||||
print(f"ERROR: Test results exceeded baseline: {metric_name}: {current_value} < {baseline_value}")
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
@@ -178,7 +155,7 @@ def assert_result(result, params, num_prompt) -> None:
|
||||
paired_benchmark_params,
|
||||
indirect=["omni_server", "benchmark_params"],
|
||||
)
|
||||
def test_performance_benchmark(omni_server, benchmark_params):
|
||||
def test_performance_benchmark(omni_server, benchmark_params, request):
|
||||
test_name = benchmark_params["test_name"]
|
||||
params = benchmark_params["params"]
|
||||
dataset_name = params.get("dataset_name", "")
|
||||
@@ -190,6 +167,7 @@ def test_performance_benchmark(omni_server, benchmark_params):
|
||||
print(f"Running benchmark for model: {model}")
|
||||
print(f"Benchmark parameters: {benchmark_params}")
|
||||
|
||||
assert_baseline = request.config.getoption("--assert-baseline", default=False)
|
||||
resource_label = get_runtime_resource_label()
|
||||
|
||||
def to_list(value, default=None):
|
||||
@@ -223,7 +201,6 @@ def test_performance_benchmark(omni_server, benchmark_params):
|
||||
"enabled",
|
||||
"eval_phase",
|
||||
"trust_remote_code",
|
||||
"expected_duplex_audio_turns_per_session",
|
||||
}
|
||||
|
||||
for key, value in params.items():
|
||||
@@ -249,7 +226,7 @@ def test_performance_benchmark(omni_server, benchmark_params):
|
||||
break
|
||||
|
||||
# QPS / request-rate sweep
|
||||
for sweep_index, (qps, num_prompt) in enumerate(zip(qps_list, num_prompt_list)):
|
||||
for i, (qps, num_prompt) in enumerate(zip(qps_list, num_prompt_list)):
|
||||
args = args + ["--request-rate", str(qps), "--num-prompts", str(num_prompt)]
|
||||
result = run_benchmark(
|
||||
args=args,
|
||||
@@ -258,15 +235,24 @@ def test_performance_benchmark(omni_server, benchmark_params):
|
||||
dataset_name=dataset_name,
|
||||
num_prompt=num_prompt,
|
||||
baseline_config=params.get("baseline"),
|
||||
sweep_index=sweep_index,
|
||||
sweep_index=i,
|
||||
request_rate=qps,
|
||||
max_concurrency=None,
|
||||
random_input_len=params.get("random_input_len"),
|
||||
random_output_len=params.get("random_output_len"),
|
||||
resource_label=resource_label,
|
||||
)
|
||||
assert_result(result, params, num_prompt)
|
||||
assert_result(
|
||||
result,
|
||||
params,
|
||||
num_prompt,
|
||||
assert_baseline=assert_baseline,
|
||||
sweep_index=i,
|
||||
request_rate=qps,
|
||||
)
|
||||
|
||||
# concurrency test
|
||||
for sweep_index, (concurrency, num_prompt) in enumerate(zip(max_concurrency_list, num_prompt_list)):
|
||||
for i, (concurrency, num_prompt) in enumerate(zip(max_concurrency_list, num_prompt_list)):
|
||||
args = args + ["--max-concurrency", str(concurrency), "--num-prompts", str(num_prompt), "--request-rate", "inf"]
|
||||
result = run_benchmark(
|
||||
args=args,
|
||||
@@ -275,9 +261,18 @@ def test_performance_benchmark(omni_server, benchmark_params):
|
||||
dataset_name=dataset_name,
|
||||
num_prompt=num_prompt,
|
||||
baseline_config=params.get("baseline"),
|
||||
sweep_index=sweep_index,
|
||||
sweep_index=i,
|
||||
request_rate=None,
|
||||
max_concurrency=concurrency,
|
||||
random_input_len=params.get("random_input_len"),
|
||||
random_output_len=params.get("random_output_len"),
|
||||
resource_label=resource_label,
|
||||
)
|
||||
assert_result(result, params, num_prompt)
|
||||
assert_result(
|
||||
result,
|
||||
params,
|
||||
num_prompt,
|
||||
assert_baseline=assert_baseline,
|
||||
sweep_index=i,
|
||||
max_concurrency=concurrency,
|
||||
)
|
||||
@@ -13,6 +13,8 @@ A config JSON file may be passed via --test-config-file. If omitted, every ``*.j
|
||||
pytest run_diffusion_benchmark.py -m "diffusion"
|
||||
pytest run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_vllm_omni.json
|
||||
|
||||
Optional: ``--assert-baseline`` compares metrics to the ``baseline`` block in each benchmark entry (default: off).
|
||||
|
||||
Optional JSON field ``mark`` is applied as pytest marks on that case via
|
||||
``pytest.param`` (e.g. ``"mark": [{"hardware_marks": {"res": {"cuda": "H100"}, "num_cards": 1}}, "full_model", "diffusion"]``).
|
||||
|
||||
@@ -44,7 +46,6 @@ from tests.dfx.conftest import (
|
||||
get_runtime_resource_label,
|
||||
hardware_json_value,
|
||||
is_diffusion_perf_config,
|
||||
resolve_baseline_for_sweep,
|
||||
resolve_pytest_marks,
|
||||
resource_label_for_filename,
|
||||
)
|
||||
@@ -273,6 +274,11 @@ def _write_result_record(record: dict[str, Any]) -> Path:
|
||||
return target
|
||||
|
||||
|
||||
def _append_to_aggregated_file(record: dict[str, Any]) -> None:
|
||||
"""Backward-compatible wrapper; prefer :func:`_write_result_record`."""
|
||||
_write_result_record(record)
|
||||
|
||||
|
||||
_server_lock = threading.Lock()
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -672,7 +678,7 @@ def run_benchmark(
|
||||
with tempfile.NamedTemporaryFile(mode="w", suffix=".json", prefix="diffusion_bench_tmp_", delete=False) as tmp:
|
||||
tmp_result_file = Path(tmp.name)
|
||||
|
||||
exclude_keys = {"baseline", "dataset", "task", "name"}
|
||||
exclude_keys = {"baseline", "dataset", "task", "name", "skip-performance-assertion"}
|
||||
|
||||
cmd = [
|
||||
sys.executable,
|
||||
@@ -766,17 +772,12 @@ def run_benchmark(
|
||||
completed = metrics.get("completed_requests", metrics.get("completed", 0))
|
||||
failed = metrics.get("failed_requests", metrics.get("failed", 0))
|
||||
|
||||
# Persist sweep-resolved baseline from params (already narrowed in _build_run_params).
|
||||
baseline = params.get("baseline") or {}
|
||||
metrics["baseline"] = baseline
|
||||
|
||||
record: dict[str, Any] = {
|
||||
"test_name": test_name,
|
||||
"endpoint": endpoint,
|
||||
"timestamp": timestamp,
|
||||
"server_params": server_cfg.get("server_params"),
|
||||
"benchmark_params": params,
|
||||
"baseline": baseline,
|
||||
"result": metrics,
|
||||
"log_file": str(log_file),
|
||||
"Model": model,
|
||||
@@ -826,14 +827,70 @@ def run_benchmark(
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _resolve_baseline_value(
|
||||
baseline_raw: Any,
|
||||
*,
|
||||
sweep_index: int | None,
|
||||
max_concurrency: Any = None,
|
||||
request_rate: Any = None,
|
||||
) -> Any:
|
||||
"""Pick the baseline threshold for this sweep step."""
|
||||
if baseline_raw is None:
|
||||
return 100000
|
||||
if isinstance(baseline_raw, dict):
|
||||
if max_concurrency is not None:
|
||||
for key in (max_concurrency, str(max_concurrency)):
|
||||
if key in baseline_raw:
|
||||
return baseline_raw[key]
|
||||
if request_rate is not None:
|
||||
for key in (request_rate, str(request_rate)):
|
||||
if key in baseline_raw:
|
||||
return baseline_raw[key]
|
||||
raise KeyError(
|
||||
f"baseline dict has no key for max_concurrency={max_concurrency!r} "
|
||||
f"or request_rate={request_rate!r}; keys={list(baseline_raw.keys())!r}"
|
||||
)
|
||||
if isinstance(baseline_raw, (list, tuple)):
|
||||
if sweep_index is None:
|
||||
raise ValueError("sweep_index is required when baseline is a list or tuple")
|
||||
return baseline_raw[sweep_index]
|
||||
return baseline_raw
|
||||
|
||||
|
||||
def assert_result(
|
||||
result: dict[str, Any],
|
||||
params: dict[str, Any],
|
||||
num_prompts: int,
|
||||
*,
|
||||
sweep_index: int | None = None,
|
||||
max_concurrency: Any = None,
|
||||
request_rate: Any = None,
|
||||
assert_baseline: bool = True,
|
||||
) -> None:
|
||||
"""Assert that the expected number of requests completed."""
|
||||
"""Assert that benchmark metrics satisfy the configured baselines."""
|
||||
completed = result.get("completed_requests", result.get("completed", 0))
|
||||
assert completed == num_prompts, f"Expected {num_prompts} completed requests, got {completed}"
|
||||
|
||||
if not assert_baseline:
|
||||
return
|
||||
if params.get("skip-performance-assertion", False):
|
||||
print("Skipping performance assertions.")
|
||||
return
|
||||
|
||||
for metric, baseline_raw in params.get("baseline", {}).items():
|
||||
current = result.get(metric)
|
||||
assert current is not None, f"Metric '{metric}' not found in result: {list(result.keys())}"
|
||||
threshold = _resolve_baseline_value(
|
||||
baseline_raw,
|
||||
sweep_index=sweep_index,
|
||||
max_concurrency=max_concurrency,
|
||||
request_rate=request_rate,
|
||||
)
|
||||
if "throughput" in metric:
|
||||
assert current >= threshold, f"{metric}: {current:.4f} < baseline {threshold}"
|
||||
else:
|
||||
assert current <= threshold, f"{metric}: {current:.4f} > baseline {threshold}"
|
||||
|
||||
|
||||
def _default_benchmark_endpoint_for_task(task: str) -> str:
|
||||
"""Return the default client-side benchmark endpoint for a diffusion task."""
|
||||
@@ -875,11 +932,15 @@ def _build_run_params(
|
||||
if max_concurrency is not None:
|
||||
run_params["max-concurrency"] = max_concurrency
|
||||
if "baseline" in params:
|
||||
# Keep all hardware buckets; pick the metric value for this sweep step only.
|
||||
run_params["baseline"] = resolve_baseline_for_sweep(
|
||||
params.get("baseline"),
|
||||
sweep_index=sweep_index,
|
||||
)
|
||||
run_params["baseline"] = {
|
||||
metric: _resolve_baseline_value(
|
||||
baseline_raw,
|
||||
sweep_index=sweep_index,
|
||||
max_concurrency=max_concurrency,
|
||||
request_rate=request_rate,
|
||||
)
|
||||
for metric, baseline_raw in params["baseline"].items()
|
||||
}
|
||||
return run_params
|
||||
|
||||
|
||||
@@ -902,30 +963,36 @@ def _iter_sweep_runs(params: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
|
||||
sweep_runs: list[dict[str, Any]] = []
|
||||
|
||||
for sweep_index, (request_rate, num_prompts) in enumerate(zip(request_rate_list, num_prompt_list)):
|
||||
for i, (request_rate, num_prompts) in enumerate(zip(request_rate_list, num_prompt_list)):
|
||||
sweep_runs.append(
|
||||
{
|
||||
"params": _build_run_params(
|
||||
params,
|
||||
request_rate=request_rate,
|
||||
num_prompts=num_prompts,
|
||||
sweep_index=sweep_index,
|
||||
sweep_index=i,
|
||||
),
|
||||
"num_prompts": num_prompts,
|
||||
"sweep_index": i,
|
||||
"request_rate": request_rate,
|
||||
"max_concurrency": None,
|
||||
}
|
||||
)
|
||||
|
||||
for sweep_index, (max_concurrency, num_prompts) in enumerate(zip(max_concurrency_list, num_prompt_list)):
|
||||
for i, (max_concurrency, num_prompts) in enumerate(zip(max_concurrency_list, num_prompt_list)):
|
||||
sweep_runs.append(
|
||||
{
|
||||
"params": _build_run_params(
|
||||
params,
|
||||
max_concurrency=max_concurrency,
|
||||
num_prompts=num_prompts,
|
||||
sweep_index=i,
|
||||
request_rate="inf",
|
||||
sweep_index=sweep_index,
|
||||
),
|
||||
"num_prompts": num_prompts,
|
||||
"sweep_index": i,
|
||||
"request_rate": None,
|
||||
"max_concurrency": max_concurrency,
|
||||
}
|
||||
)
|
||||
|
||||
@@ -936,9 +1003,11 @@ def _iter_sweep_runs(params: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
"params": _build_run_params(
|
||||
params,
|
||||
num_prompts=default_num_prompts,
|
||||
sweep_index=0,
|
||||
),
|
||||
"num_prompts": default_num_prompts,
|
||||
"sweep_index": None,
|
||||
"request_rate": None,
|
||||
"max_concurrency": None,
|
||||
}
|
||||
)
|
||||
|
||||
@@ -954,12 +1023,14 @@ def _iter_sweep_runs(params: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
paired_benchmark_params,
|
||||
indirect=["diffusion_server", "benchmark_params"],
|
||||
)
|
||||
def test_diffusion_performance_benchmark(diffusion_server, benchmark_params):
|
||||
def test_diffusion_performance_benchmark(diffusion_server, benchmark_params, request):
|
||||
"""Run the diffusion performance benchmark and verify request completion.
|
||||
|
||||
One server is started per unique parallel configuration (module scope).
|
||||
For each server, all benchmark parameter sets defined in the config JSON
|
||||
are executed sequentially; metrics are recorded to the aggregated result file.
|
||||
|
||||
Pass ``--assert-baseline`` to compare ``throughput_qps``, ``latency_mean``, etc. to the JSON ``baseline`` block.
|
||||
"""
|
||||
test_name = benchmark_params["test_name"]
|
||||
params = benchmark_params["params"]
|
||||
@@ -1002,4 +1073,14 @@ def test_diffusion_performance_benchmark(diffusion_server, benchmark_params):
|
||||
print(f"\n Aggregated results: {_aggregated_result_file_for_source(str(source))}")
|
||||
print("=" * 60)
|
||||
|
||||
assert_result(result, sweep_run["num_prompts"])
|
||||
assert_baseline = request.config.getoption("--assert-baseline", default=False)
|
||||
|
||||
assert_result(
|
||||
result,
|
||||
params,
|
||||
sweep_run["num_prompts"],
|
||||
sweep_index=sweep_run["sweep_index"],
|
||||
max_concurrency=sweep_run["max_concurrency"],
|
||||
request_rate=sweep_run["request_rate"],
|
||||
assert_baseline=assert_baseline,
|
||||
)
|
||||
@@ -40,12 +40,10 @@
|
||||
"max-concurrency": 1,
|
||||
"enable-negative-prompt": true,
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"throughput_qps": 0.495,
|
||||
"latency_mean": 2.0238,
|
||||
"peak_memory_mb_max": 29393.4286,
|
||||
"peak_memory_mb_mean": 29392.7143
|
||||
}
|
||||
"throughput_qps": 0.495,
|
||||
"latency_mean": 2.0238,
|
||||
"peak_memory_mb_max": 29393.4286,
|
||||
"peak_memory_mb_mean": 29392.7143
|
||||
}
|
||||
}
|
||||
]
|
||||
@@ -92,12 +90,10 @@
|
||||
"enable-negative-prompt": true,
|
||||
"num-input-images": 1,
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"throughput_qps": 0.3451,
|
||||
"latency_mean": 2.9031,
|
||||
"peak_memory_mb_max": 30480.2857,
|
||||
"peak_memory_mb_mean": 30479.2857
|
||||
}
|
||||
"throughput_qps": 0.3451,
|
||||
"latency_mean": 2.9031,
|
||||
"peak_memory_mb_max": 30480.2857,
|
||||
"peak_memory_mb_mean": 30479.2857
|
||||
}
|
||||
}
|
||||
]
|
||||
@@ -143,12 +139,10 @@
|
||||
"max-concurrency": 1,
|
||||
"enable-negative-prompt": true,
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"throughput_qps": 0.5048,
|
||||
"latency_mean": 1.987,
|
||||
"peak_memory_mb_max": 29380.0,
|
||||
"peak_memory_mb_mean": 29380.0
|
||||
}
|
||||
"throughput_qps": 0.5048,
|
||||
"latency_mean": 1.987,
|
||||
"peak_memory_mb_max": 29380.0,
|
||||
"peak_memory_mb_mean": 29380.0
|
||||
}
|
||||
}
|
||||
]
|
||||
@@ -195,12 +189,10 @@
|
||||
"enable-negative-prompt": true,
|
||||
"num-input-images": 1,
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"throughput_qps": 0.149,
|
||||
"latency_mean": 6.7157,
|
||||
"peak_memory_mb_max": 30574.0,
|
||||
"peak_memory_mb_mean": 30574.0
|
||||
}
|
||||
"throughput_qps": 0.149,
|
||||
"latency_mean": 6.7157,
|
||||
"peak_memory_mb_max": 30574.0,
|
||||
"peak_memory_mb_mean": 30574.0
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
@@ -24,12 +24,10 @@
|
||||
"warmup-num-inference-steps": 28,
|
||||
"enable-negative-prompt": true,
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"throughput_qps": [0.0303, 0.0302, 0.0302],
|
||||
"latency_mean": [32.6, 62.2, 111.7],
|
||||
"peak_memory_mb_max": 40500,
|
||||
"peak_memory_mb_mean": 40500
|
||||
}
|
||||
"throughput_qps": {"1": 0.0303, "2": 0.0302, "4": 0.0302},
|
||||
"latency_mean": {"1": 32.6, "2": 62.2, "4": 111.7},
|
||||
"peak_memory_mb_max": 40500,
|
||||
"peak_memory_mb_mean": 40500
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
@@ -24,12 +24,10 @@
|
||||
"warmup-num-inference-steps": 28,
|
||||
"enable-negative-prompt": true,
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"throughput_qps": [0.0549, 0.0547, 0.0549],
|
||||
"latency_mean": [18.0, 34.3, 61.4],
|
||||
"peak_memory_mb_max": 40500,
|
||||
"peak_memory_mb_mean": 40500
|
||||
}
|
||||
"throughput_qps": {"1": 0.0549, "2": 0.0547, "4": 0.0549},
|
||||
"latency_mean": {"1": 18.0, "2": 34.3, "4": 61.4},
|
||||
"peak_memory_mb_max": 40500,
|
||||
"peak_memory_mb_mean": 40500
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
@@ -48,11 +48,9 @@
|
||||
"guardrails": false
|
||||
},
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"throughput_qps": 1.107,
|
||||
"latency_mean": 0.8929,
|
||||
"peak_memory_mb_mean": 80000.0
|
||||
}
|
||||
"throughput_qps": 1.107,
|
||||
"latency_mean": 0.8929,
|
||||
"peak_memory_mb_mean": 80000.0
|
||||
}
|
||||
}
|
||||
]
|
||||
@@ -117,11 +115,9 @@
|
||||
"use_duration_template": false
|
||||
},
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"throughput_qps": 0.027,
|
||||
"latency_mean": 36.6824,
|
||||
"peak_memory_mb_mean": 30155.4
|
||||
}
|
||||
"throughput_qps": 0.027,
|
||||
"latency_mean": 36.6824,
|
||||
"peak_memory_mb_mean": 30155.4
|
||||
}
|
||||
}
|
||||
]
|
||||
@@ -185,11 +181,9 @@
|
||||
"guardrails": false
|
||||
},
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"throughput_qps": 0.027,
|
||||
"latency_mean": 36.968,
|
||||
"peak_memory_mb_mean": 30443.6
|
||||
}
|
||||
"throughput_qps": 0.027,
|
||||
"latency_mean": 36.968,
|
||||
"peak_memory_mb_mean": 30443.6
|
||||
}
|
||||
}
|
||||
]
|
||||
@@ -257,11 +251,9 @@
|
||||
"condition_video_keep": "first"
|
||||
},
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"throughput_qps": 0.027,
|
||||
"latency_mean": 37.2117,
|
||||
"peak_memory_mb_mean": 30135.6
|
||||
}
|
||||
"throughput_qps": 0.027,
|
||||
"latency_mean": 37.2117,
|
||||
"peak_memory_mb_mean": 30135.6
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
@@ -46,14 +46,12 @@
|
||||
"seed_tts_locale": "en",
|
||||
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"median_audio_ttfp_ms": [
|
||||
223.3584
|
||||
],
|
||||
"median_audio_rtf": [
|
||||
0.4393
|
||||
]
|
||||
}
|
||||
"median_audio_ttfp_ms": [
|
||||
223.3584
|
||||
],
|
||||
"median_audio_rtf": [
|
||||
0.4393
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -72,17 +70,15 @@
|
||||
"seed_tts_locale": "en",
|
||||
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"median_audio_ttfp_ms": [
|
||||
276.575
|
||||
],
|
||||
"median_audio_rtf": [
|
||||
0.5073
|
||||
],
|
||||
"audio_throughput": [
|
||||
27.3728
|
||||
]
|
||||
}
|
||||
"median_audio_ttfp_ms": [
|
||||
276.575
|
||||
],
|
||||
"median_audio_rtf": [
|
||||
0.5073
|
||||
],
|
||||
"audio_throughput": [
|
||||
27.3728
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
@@ -140,13 +140,11 @@
|
||||
"num-prompts": 96,
|
||||
"skip-performance-assertion": true,
|
||||
"baseline": {
|
||||
"H200": {
|
||||
"throughput_qps": 0.1874,
|
||||
"latency_p99": 101.852,
|
||||
"latency_mean": 79.0385,
|
||||
"peak_memory_mb_max": 70000,
|
||||
"peak_memory_mb_mean": 70000
|
||||
}
|
||||
"throughput_qps": 0.1874,
|
||||
"latency_p99": 101.852,
|
||||
"latency_mean": 79.0385,
|
||||
"peak_memory_mb_max": 70000,
|
||||
"peak_memory_mb_mean": 70000
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
@@ -36,12 +36,10 @@
|
||||
"max-concurrency": 1,
|
||||
"skip-performance-assertion": true,
|
||||
"baseline": {
|
||||
"H200": {
|
||||
"throughput_qps": 0.21,
|
||||
"latency_p99": 4.7469,
|
||||
"peak_memory_mb_max": 101100,
|
||||
"peak_memory_mb_mean": 101100
|
||||
}
|
||||
"throughput_qps": 0.21,
|
||||
"latency_p99": 4.7469,
|
||||
"peak_memory_mb_max": 101100,
|
||||
"peak_memory_mb_mean": 101100
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
@@ -36,12 +36,10 @@
|
||||
"max-concurrency": 1,
|
||||
"skip-performance-assertion": true,
|
||||
"baseline": {
|
||||
"H200": {
|
||||
"throughput_qps": 0.2,
|
||||
"latency_p99": 5.1025,
|
||||
"peak_memory_mb_max": 97402,
|
||||
"peak_memory_mb_mean": 97402
|
||||
}
|
||||
"throughput_qps": 0.20,
|
||||
"latency_p99": 5.1025,
|
||||
"peak_memory_mb_max": 97402,
|
||||
"peak_memory_mb_mean": 97402
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
@@ -35,13 +35,11 @@
|
||||
"max-concurrency": 1,
|
||||
"skip-performance-assertion": true,
|
||||
"baseline": {
|
||||
"H200": {
|
||||
"throughput_qps": 0.213,
|
||||
"latency_mean": 4.6953,
|
||||
"latency_p99": 4.8601,
|
||||
"peak_memory_mb_max": 56912.0,
|
||||
"peak_memory_mb_mean": 56912.0
|
||||
}
|
||||
"throughput_qps": 0.213,
|
||||
"latency_mean": 4.6953,
|
||||
"latency_p99": 4.8601,
|
||||
"peak_memory_mb_max": 56912.0,
|
||||
"peak_memory_mb_mean": 56912.0
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
@@ -26,7 +26,8 @@
|
||||
"warmup-requests": 1,
|
||||
"warmup-concurrency": 1,
|
||||
"warmup-num-inference-steps": 2,
|
||||
"enable-negative-prompt": true
|
||||
"enable-negative-prompt": true,
|
||||
"skip-performance-assertion": true
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@@ -1,144 +0,0 @@
|
||||
[
|
||||
{
|
||||
"test_name": "test_minicpmo_4_5_challenge",
|
||||
"mark": [
|
||||
{
|
||||
"hardware_marks": {
|
||||
"res": {
|
||||
"cuda": "H100",
|
||||
"npu": "A3"
|
||||
},
|
||||
"num_cards": 1
|
||||
}
|
||||
},
|
||||
"full_model",
|
||||
"omni"
|
||||
],
|
||||
"server_params": {
|
||||
"model": "openbmb/MiniCPM-o-4_5",
|
||||
"extra_cli_args": [
|
||||
"--deploy-config",
|
||||
"vllm_omni/deploy/minicpmo_4_5.yaml",
|
||||
"--trust-remote-code"
|
||||
]
|
||||
},
|
||||
"benchmark_params": [
|
||||
{
|
||||
"dataset_name": "seed-tts",
|
||||
"dataset_path": "zhaochenyang20/seed-tts-eval",
|
||||
"backend": "openai-chat-omni",
|
||||
"endpoint": "/v1/chat/completions",
|
||||
"num_prompts": [
|
||||
32,
|
||||
64,
|
||||
128
|
||||
],
|
||||
"max_concurrency": [
|
||||
1,
|
||||
4,
|
||||
8
|
||||
],
|
||||
"no_oversample": true,
|
||||
"disable_shuffle": true,
|
||||
"trust_remote_code": true,
|
||||
"seed_tts_locale": "zh",
|
||||
"extra_body": {
|
||||
"modalities": [
|
||||
"text",
|
||||
"audio"
|
||||
],
|
||||
"chat_template_kwargs": {
|
||||
"enable_thinking": false,
|
||||
"use_tts_template": true
|
||||
}
|
||||
},
|
||||
"percentile-metrics": "ttft,e2el,audio_ttfp,audio_rtf,audio_duration",
|
||||
"baseline": {
|
||||
"A3": {
|
||||
"request_throughput": [
|
||||
0.5383,
|
||||
0.6042,
|
||||
0.7547
|
||||
],
|
||||
"mean_ttft_ms": [
|
||||
333.2633,
|
||||
533.8656,
|
||||
547.3388
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
1857.2154,
|
||||
6600.5101,
|
||||
10450.6811
|
||||
],
|
||||
"mean_audio_ttfp_ms": [
|
||||
986.4666,
|
||||
3411.0754,
|
||||
3352.7515
|
||||
],
|
||||
"mean_audio_rtf": [
|
||||
0.4423,
|
||||
1.5734,
|
||||
2.3024
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"test_name": "test_minicpmo_4_5_a2_challenge",
|
||||
"mark": [
|
||||
{
|
||||
"hardware_marks": {
|
||||
"res": {
|
||||
"npu": "A2"
|
||||
},
|
||||
"num_cards": 1
|
||||
}
|
||||
},
|
||||
"full_model",
|
||||
"omni"
|
||||
],
|
||||
"server_params": {
|
||||
"model": "openbmb/MiniCPM-o-4_5",
|
||||
"extra_cli_args": [
|
||||
"--deploy-config",
|
||||
"vllm_omni/deploy/minicpmo_4_5.yaml",
|
||||
"--trust-remote-code"
|
||||
]
|
||||
},
|
||||
"benchmark_params": [
|
||||
{
|
||||
"dataset_name": "seed-tts",
|
||||
"dataset_path": "zhaochenyang20/seed-tts-eval",
|
||||
"backend": "openai-chat-omni",
|
||||
"endpoint": "/v1/chat/completions",
|
||||
"num_prompts": [
|
||||
32,
|
||||
64,
|
||||
128
|
||||
],
|
||||
"max_concurrency": [
|
||||
1,
|
||||
4,
|
||||
8
|
||||
],
|
||||
"no_oversample": true,
|
||||
"disable_shuffle": true,
|
||||
"trust_remote_code": true,
|
||||
"seed_tts_locale": "zh",
|
||||
"extra_body": {
|
||||
"modalities": [
|
||||
"text",
|
||||
"audio"
|
||||
],
|
||||
"chat_template_kwargs": {
|
||||
"enable_thinking": false,
|
||||
"use_tts_template": true
|
||||
}
|
||||
},
|
||||
"percentile-metrics": "ttft,e2el,audio_ttfp,audio_rtf,audio_duration"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
@@ -1,69 +0,0 @@
|
||||
[
|
||||
{
|
||||
"test_name": "test_minicpmo_4_5_duplex_seed_tts_challenge",
|
||||
"mark": [
|
||||
{
|
||||
"hardware_marks": {
|
||||
"res": {
|
||||
"cuda": "H100",
|
||||
"npu": "A3"
|
||||
},
|
||||
"num_cards": 1
|
||||
}
|
||||
},
|
||||
"full_model",
|
||||
"omni"
|
||||
],
|
||||
"server_params": {
|
||||
"model": "openbmb/MiniCPM-o-4_5",
|
||||
"extra_cli_args": [
|
||||
"--deploy-config",
|
||||
"vllm_omni/deploy/minicpmo_4_5_duplex.yaml",
|
||||
"--trust-remote-code"
|
||||
]
|
||||
},
|
||||
"benchmark_params": [
|
||||
{
|
||||
"dataset_name": "seed-tts",
|
||||
"dataset_path": "zhaochenyang20/seed-tts-eval",
|
||||
"backend": "openai-realtime-tts",
|
||||
"endpoint": "/v1/realtime",
|
||||
"num_prompts": [
|
||||
32
|
||||
],
|
||||
"max_concurrency": [
|
||||
1
|
||||
],
|
||||
"no_oversample": true,
|
||||
"disable_shuffle": true,
|
||||
"trust_remote_code": true,
|
||||
"seed_tts_locale": "zh",
|
||||
"seed_tts_turns_per_session": 4,
|
||||
"expected_duplex_audio_turns_per_session": 4,
|
||||
"extra_body": {
|
||||
"save_duplex_request_metrics": true
|
||||
},
|
||||
"percentile-metrics": "ttft,e2el,audio_ttfp,audio_rtf,audio_duration",
|
||||
"baseline": {
|
||||
"A3": {
|
||||
"request_throughput": [
|
||||
0.1594
|
||||
],
|
||||
"mean_ttft_ms": [
|
||||
274.164
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
6250.7696
|
||||
],
|
||||
"mean_audio_ttfp_ms": [
|
||||
994.422
|
||||
],
|
||||
"mean_audio_rtf": [
|
||||
0.46163
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
@@ -1,81 +0,0 @@
|
||||
[
|
||||
{
|
||||
"test_name": "test_minicpmo_4_5_ready",
|
||||
"mark": [
|
||||
{
|
||||
"hardware_marks": {
|
||||
"res": {
|
||||
"cuda": "H100",
|
||||
"npu": "A3"
|
||||
},
|
||||
"num_cards": 1
|
||||
}
|
||||
},
|
||||
"core_model",
|
||||
"omni"
|
||||
],
|
||||
"server_params": {
|
||||
"model": "openbmb/MiniCPM-o-4_5",
|
||||
"extra_cli_args": [
|
||||
"--deploy-config",
|
||||
"vllm_omni/deploy/minicpmo_4_5.yaml",
|
||||
"--trust-remote-code"
|
||||
]
|
||||
},
|
||||
"benchmark_params": [
|
||||
{
|
||||
"dataset_name": "seed-tts",
|
||||
"dataset_path": "zhaochenyang20/seed-tts-eval",
|
||||
"backend": "openai-chat-omni",
|
||||
"endpoint": "/v1/chat/completions",
|
||||
"num_prompts": [
|
||||
10,
|
||||
40
|
||||
],
|
||||
"max_concurrency": [
|
||||
1,
|
||||
4
|
||||
],
|
||||
"no_oversample": true,
|
||||
"disable_shuffle": true,
|
||||
"trust_remote_code": true,
|
||||
"seed_tts_locale": "zh",
|
||||
"extra_body": {
|
||||
"modalities": [
|
||||
"text",
|
||||
"audio"
|
||||
],
|
||||
"chat_template_kwargs": {
|
||||
"enable_thinking": false,
|
||||
"use_tts_template": true
|
||||
}
|
||||
},
|
||||
"percentile-metrics": "ttft,e2el,audio_ttfp,audio_rtf,audio_duration",
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"request_throughput": [
|
||||
0.7803,
|
||||
1.1008
|
||||
],
|
||||
"mean_ttft_ms": [
|
||||
166.9764,
|
||||
307.7133
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
1361.1682,
|
||||
3800.6332
|
||||
],
|
||||
"mean_audio_ttfp_ms": [
|
||||
635.8099,
|
||||
1615.6591
|
||||
],
|
||||
"mean_audio_rtf": [
|
||||
0.3151,
|
||||
0.8568
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
@@ -44,29 +44,27 @@
|
||||
"ignore_eos": true,
|
||||
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"mean_ttft_ms": [
|
||||
96.4061,
|
||||
140.8147,
|
||||
271.9025,
|
||||
362.2916,
|
||||
507.776
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
18507.4582,
|
||||
28365.4956,
|
||||
31907.9613,
|
||||
48161.5829,
|
||||
72630.8601
|
||||
],
|
||||
"mean_audio_rtf": [
|
||||
0.1175,
|
||||
0.18,
|
||||
0.2359,
|
||||
0.3278,
|
||||
0.6296
|
||||
]
|
||||
}
|
||||
"mean_ttft_ms": [
|
||||
96.4061,
|
||||
140.8147,
|
||||
271.9025,
|
||||
362.2916,
|
||||
507.776
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
18507.4582,
|
||||
28365.4956,
|
||||
31907.9613,
|
||||
48161.5829,
|
||||
72630.8601
|
||||
],
|
||||
"mean_audio_rtf": [
|
||||
0.1175,
|
||||
0.18,
|
||||
0.2359,
|
||||
0.3278,
|
||||
0.6296
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -93,17 +91,15 @@
|
||||
},
|
||||
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"mean_ttft_ms": [
|
||||
354.0951
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
3952.5615
|
||||
],
|
||||
"mean_audio_rtf": [
|
||||
0.117
|
||||
]
|
||||
}
|
||||
"mean_ttft_ms": [
|
||||
354.0951
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
3952.5615
|
||||
],
|
||||
"mean_audio_rtf": [
|
||||
0.117
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -132,17 +128,15 @@
|
||||
},
|
||||
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"mean_ttft_ms": [
|
||||
259.6317
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
4253.3009
|
||||
],
|
||||
"mean_audio_rtf": [
|
||||
0.1448
|
||||
]
|
||||
}
|
||||
"mean_ttft_ms": [
|
||||
259.6317
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
4253.3009
|
||||
],
|
||||
"mean_audio_rtf": [
|
||||
0.1448
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -173,17 +167,15 @@
|
||||
},
|
||||
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"mean_ttft_ms": [
|
||||
457.4453
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
5912.5623
|
||||
],
|
||||
"mean_audio_rtf": [
|
||||
0.1967
|
||||
]
|
||||
}
|
||||
"mean_ttft_ms": [
|
||||
457.4453
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
5912.5623
|
||||
],
|
||||
"mean_audio_rtf": [
|
||||
0.1967
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -214,22 +206,20 @@
|
||||
},
|
||||
"percentile-metrics": "ttft,tpot,itl,e2el",
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"mean_ttft_ms": [
|
||||
90.8033,
|
||||
202.1779,
|
||||
326.5864,
|
||||
544.8914,
|
||||
961.3929
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
4517.3411,
|
||||
7857.9711,
|
||||
9861.1687,
|
||||
13283.3363,
|
||||
19541.0736
|
||||
]
|
||||
}
|
||||
"mean_ttft_ms": [
|
||||
90.8033,
|
||||
202.1779,
|
||||
326.5864,
|
||||
544.8914,
|
||||
961.3929
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
4517.3411,
|
||||
7857.9711,
|
||||
9861.1687,
|
||||
13283.3363,
|
||||
19541.0736
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -261,14 +251,12 @@
|
||||
},
|
||||
"percentile-metrics": "ttft,tpot,itl,e2el",
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"mean_ttft_ms": [
|
||||
318.048
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
807.6114
|
||||
]
|
||||
}
|
||||
"mean_ttft_ms": [
|
||||
318.048
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
807.6114
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -302,14 +290,12 @@
|
||||
},
|
||||
"percentile-metrics": "ttft,tpot,itl,e2el",
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"mean_ttft_ms": [
|
||||
186.168
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
726.1137
|
||||
]
|
||||
}
|
||||
"mean_ttft_ms": [
|
||||
186.168
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
726.1137
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -345,14 +331,12 @@
|
||||
},
|
||||
"percentile-metrics": "ttft,tpot,itl,e2el",
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"mean_ttft_ms": [
|
||||
375.5282
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
1006.1663
|
||||
]
|
||||
}
|
||||
"mean_ttft_ms": [
|
||||
375.5282
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
1006.1663
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
@@ -35,12 +35,10 @@
|
||||
"ignore_eos": true,
|
||||
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"mean_ttft_ms": [130.5751, 278.6469, 324.7249, 911.6085],
|
||||
"mean_tpot_ms": [7.9854, 11.1209, 16.1507, 20.7485],
|
||||
"mean_audio_ttfp_ms": [486.486, 954.1791, 1289.6514, 2196.8275],
|
||||
"mean_audio_rtf": [0.1896, 0.3962, 0.4098, 0.7322]
|
||||
}
|
||||
"mean_ttft_ms": [130.5751, 278.6469, 324.7249, 911.6085],
|
||||
"mean_tpot_ms": [7.9854, 11.1209, 16.1507, 20.7485],
|
||||
"mean_audio_ttfp_ms": [486.486, 954.1791, 1289.6514, 2196.8275],
|
||||
"mean_audio_rtf": [0.1896, 0.3962, 0.4098, 0.7322]
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -63,11 +61,9 @@
|
||||
},
|
||||
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"mean_ttft_ms": [340.5122],
|
||||
"mean_audio_ttfp_ms": [463.0752],
|
||||
"mean_audio_rtf": [0.1153]
|
||||
}
|
||||
"mean_ttft_ms": [340.5122],
|
||||
"mean_audio_ttfp_ms": [463.0752],
|
||||
"mean_audio_rtf": [0.1153]
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -92,11 +88,9 @@
|
||||
},
|
||||
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"mean_ttft_ms": [236.926],
|
||||
"mean_audio_ttfp_ms": [395.2182],
|
||||
"mean_audio_rtf": [0.1259]
|
||||
}
|
||||
"mean_ttft_ms": [236.926],
|
||||
"mean_audio_ttfp_ms": [395.2182],
|
||||
"mean_audio_rtf": [0.1259]
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -123,11 +117,9 @@
|
||||
},
|
||||
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"mean_ttft_ms": [470.6126],
|
||||
"mean_audio_ttfp_ms": [665.5474],
|
||||
"mean_audio_rtf": [0.166]
|
||||
}
|
||||
"mean_ttft_ms": [470.6126],
|
||||
"mean_audio_ttfp_ms": [665.5474],
|
||||
"mean_audio_rtf": [0.166]
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
@@ -44,29 +44,27 @@
|
||||
"ignore_eos": true,
|
||||
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"mean_ttft_ms": [
|
||||
115.8348,
|
||||
173.1823,
|
||||
446.672,
|
||||
464.5985,
|
||||
3420.0634
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
25070.9523,
|
||||
31992.0646,
|
||||
37619.7669,
|
||||
50393.8257,
|
||||
136619.2743
|
||||
],
|
||||
"mean_audio_rtf": [
|
||||
0.169,
|
||||
0.2111,
|
||||
0.2475,
|
||||
0.3551,
|
||||
0.9266
|
||||
]
|
||||
}
|
||||
"mean_ttft_ms": [
|
||||
115.8348,
|
||||
173.1823,
|
||||
446.672,
|
||||
464.5985,
|
||||
3420.0634
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
25070.9523,
|
||||
31992.0646,
|
||||
37619.7669,
|
||||
50393.8257,
|
||||
136619.2743
|
||||
],
|
||||
"mean_audio_rtf": [
|
||||
0.169,
|
||||
0.2111,
|
||||
0.2475,
|
||||
0.3551,
|
||||
0.9266
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -93,17 +91,15 @@
|
||||
},
|
||||
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"mean_ttft_ms": [
|
||||
343.1377
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
4811.1123
|
||||
],
|
||||
"mean_audio_rtf": [
|
||||
0.1463
|
||||
]
|
||||
}
|
||||
"mean_ttft_ms": [
|
||||
343.1377
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
4811.1123
|
||||
],
|
||||
"mean_audio_rtf": [
|
||||
0.1463
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -132,17 +128,15 @@
|
||||
},
|
||||
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"mean_ttft_ms": [
|
||||
245.5446
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
5019.1256
|
||||
],
|
||||
"mean_audio_rtf": [
|
||||
0.1713
|
||||
]
|
||||
}
|
||||
"mean_ttft_ms": [
|
||||
245.5446
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
5019.1256
|
||||
],
|
||||
"mean_audio_rtf": [
|
||||
0.1713
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -173,17 +167,15 @@
|
||||
},
|
||||
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"mean_ttft_ms": [
|
||||
506.1407
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
6814.9656
|
||||
],
|
||||
"mean_audio_rtf": [
|
||||
0.2213
|
||||
]
|
||||
}
|
||||
"mean_ttft_ms": [
|
||||
506.1407
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
6814.9656
|
||||
],
|
||||
"mean_audio_rtf": [
|
||||
0.2213
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
@@ -51,22 +51,20 @@
|
||||
},
|
||||
"percentile-metrics": "ttft,tpot,itl,e2el",
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"mean_ttft_ms": [
|
||||
69.8081,
|
||||
173.0651,
|
||||
271.8108,
|
||||
413.4079,
|
||||
497.2546
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
4720.8225,
|
||||
7937.2532,
|
||||
9962.3566,
|
||||
13724.5479,
|
||||
18324.091
|
||||
]
|
||||
}
|
||||
"mean_ttft_ms": [
|
||||
69.8081,
|
||||
173.0651,
|
||||
271.8108,
|
||||
413.4079,
|
||||
497.2546
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
4720.8225,
|
||||
7937.2532,
|
||||
9962.3566,
|
||||
13724.5479,
|
||||
18324.091
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -99,14 +97,12 @@
|
||||
},
|
||||
"percentile-metrics": "ttft,tpot,itl,e2el",
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"mean_ttft_ms": [
|
||||
25.9477
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
517.3155
|
||||
]
|
||||
}
|
||||
"mean_ttft_ms": [
|
||||
25.9477
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
517.3155
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -141,14 +137,12 @@
|
||||
},
|
||||
"percentile-metrics": "ttft,tpot,itl,e2el",
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"mean_ttft_ms": [
|
||||
26.7056
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
551.308
|
||||
]
|
||||
}
|
||||
"mean_ttft_ms": [
|
||||
26.7056
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
551.308
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -185,14 +179,12 @@
|
||||
},
|
||||
"percentile-metrics": "ttft,tpot,itl,e2el",
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"mean_ttft_ms": [
|
||||
27.6906
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
596.5284
|
||||
]
|
||||
}
|
||||
"mean_ttft_ms": [
|
||||
27.6906
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
596.5284
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
@@ -34,12 +34,10 @@
|
||||
"num-input-images": 2,
|
||||
"enable-negative-prompt": true,
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"throughput_qps": 0.0721,
|
||||
"latency_mean": 13.8398,
|
||||
"peak_memory_mb_max": 57610.0,
|
||||
"peak_memory_mb_mean": 57610.0
|
||||
}
|
||||
"throughput_qps": 0.0721,
|
||||
"latency_mean": 13.8398,
|
||||
"peak_memory_mb_max": 57610.0,
|
||||
"peak_memory_mb_mean": 57610.0
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -54,12 +52,10 @@
|
||||
"num-input-images": 2,
|
||||
"enable-negative-prompt": true,
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"throughput_qps": 0.0186,
|
||||
"latency_mean": 53.6942,
|
||||
"peak_memory_mb_max": 67790.0,
|
||||
"peak_memory_mb_mean": 67790.0
|
||||
}
|
||||
"throughput_qps": 0.0186,
|
||||
"latency_mean": 53.6942,
|
||||
"peak_memory_mb_max": 67790.0,
|
||||
"peak_memory_mb_mean": 67790.0
|
||||
}
|
||||
}
|
||||
]
|
||||
@@ -104,12 +100,10 @@
|
||||
"num-input-images": 2,
|
||||
"enable-negative-prompt": true,
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"throughput_qps": 0.0538,
|
||||
"latency_mean": 18.5591,
|
||||
"peak_memory_mb_max": 56449.4286,
|
||||
"peak_memory_mb_mean": 56449.4286
|
||||
}
|
||||
"throughput_qps": 0.0538,
|
||||
"latency_mean": 18.5591,
|
||||
"peak_memory_mb_max": 56449.4286,
|
||||
"peak_memory_mb_mean": 56449.4286
|
||||
}
|
||||
}
|
||||
]
|
||||
@@ -163,12 +157,10 @@
|
||||
"num-input-images": 2,
|
||||
"enable-negative-prompt": true,
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"throughput_qps": 0.2776,
|
||||
"latency_mean": 3.5938,
|
||||
"peak_memory_mb_max": 58113.7143,
|
||||
"peak_memory_mb_mean": 58075.1429
|
||||
}
|
||||
"throughput_qps": 0.2776,
|
||||
"latency_mean": 3.5938,
|
||||
"peak_memory_mb_max": 58113.7143,
|
||||
"peak_memory_mb_mean": 58075.1429
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -183,12 +175,10 @@
|
||||
"num-input-images": 2,
|
||||
"enable-negative-prompt": true,
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"throughput_qps": 0.1051,
|
||||
"latency_mean": 9.4918,
|
||||
"peak_memory_mb_max": 68107.4286,
|
||||
"peak_memory_mb_mean": 68107.4286
|
||||
}
|
||||
"throughput_qps": 0.1051,
|
||||
"latency_mean": 9.4918,
|
||||
"peak_memory_mb_max": 68107.4286,
|
||||
"peak_memory_mb_mean": 68107.4286
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
@@ -33,12 +33,10 @@
|
||||
"max-concurrency": 1,
|
||||
"enable-negative-prompt": true,
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"throughput_qps": 0.1129,
|
||||
"latency_mean": 8.8352,
|
||||
"peak_memory_mb_max": 58572.0,
|
||||
"peak_memory_mb_mean": 58572.0
|
||||
}
|
||||
"throughput_qps": 0.1129,
|
||||
"latency_mean": 8.8352,
|
||||
"peak_memory_mb_max": 58572.0,
|
||||
"peak_memory_mb_mean": 58572.0
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -52,12 +50,10 @@
|
||||
"max-concurrency": 1,
|
||||
"enable-negative-prompt": true,
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"throughput_qps": 0.024,
|
||||
"latency_mean": 41.5686,
|
||||
"peak_memory_mb_max": 66368.0,
|
||||
"peak_memory_mb_mean": 66368.0
|
||||
}
|
||||
"throughput_qps": 0.024,
|
||||
"latency_mean": 41.5686,
|
||||
"peak_memory_mb_max": 66368.0,
|
||||
"peak_memory_mb_mean": 66368.0
|
||||
}
|
||||
}
|
||||
]
|
||||
@@ -100,12 +96,10 @@
|
||||
"max-concurrency": 1,
|
||||
"enable-negative-prompt": true,
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"throughput_qps": 0.0672,
|
||||
"latency_mean": 14.8517,
|
||||
"peak_memory_mb_max": 57076.0,
|
||||
"peak_memory_mb_mean": 57076.0
|
||||
}
|
||||
"throughput_qps": 0.0672,
|
||||
"latency_mean": 14.8517,
|
||||
"peak_memory_mb_max": 57076.0,
|
||||
"peak_memory_mb_mean": 57076.0
|
||||
}
|
||||
}
|
||||
]
|
||||
@@ -158,12 +152,10 @@
|
||||
"max-concurrency": 1,
|
||||
"enable-negative-prompt": true,
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"throughput_qps": 0.2879,
|
||||
"latency_mean": 3.4701,
|
||||
"peak_memory_mb_max": 58758.0,
|
||||
"peak_memory_mb_mean": 58758.0
|
||||
}
|
||||
"throughput_qps": 0.2879,
|
||||
"latency_mean": 3.4701,
|
||||
"peak_memory_mb_max": 58758.0,
|
||||
"peak_memory_mb_mean": 58758.0
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -177,12 +169,10 @@
|
||||
"max-concurrency": 1,
|
||||
"enable-negative-prompt": true,
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"throughput_qps": 0.1113,
|
||||
"latency_mean": 8.9656,
|
||||
"peak_memory_mb_max": 67424.0,
|
||||
"peak_memory_mb_mean": 67424.0
|
||||
}
|
||||
"throughput_qps": 0.1113,
|
||||
"latency_mean": 8.9656,
|
||||
"peak_memory_mb_max": 67424.0,
|
||||
"peak_memory_mb_mean": 67424.0
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
@@ -33,12 +33,10 @@
|
||||
"max-concurrency": 1,
|
||||
"enable-negative-prompt": true,
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"throughput_qps": 0.0703,
|
||||
"latency_mean": 14.208,
|
||||
"peak_memory_mb_max": 63225.1429,
|
||||
"peak_memory_mb_mean": 63225.1429
|
||||
}
|
||||
"throughput_qps": 0.0703,
|
||||
"latency_mean": 14.208,
|
||||
"peak_memory_mb_max": 63225.1429,
|
||||
"peak_memory_mb_mean": 63225.1429
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -52,12 +50,10 @@
|
||||
"max-concurrency": 1,
|
||||
"enable-negative-prompt": true,
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"throughput_qps": 0.0417,
|
||||
"latency_mean": 23.914,
|
||||
"peak_memory_mb_max": 63225.1429,
|
||||
"peak_memory_mb_mean": 63225.1429
|
||||
}
|
||||
"throughput_qps": 0.0417,
|
||||
"latency_mean": 23.914,
|
||||
"peak_memory_mb_max": 63225.1429,
|
||||
"peak_memory_mb_mean": 63225.1429
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
@@ -33,11 +33,9 @@
|
||||
"max-concurrency": 1,
|
||||
"enable-negative-prompt": true,
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"throughput_qps": 0.4153,
|
||||
"latency_mean": 2.4065,
|
||||
"peak_memory_mb_mean": 55121.4286
|
||||
}
|
||||
"throughput_qps": 0.4153,
|
||||
"latency_mean": 2.4065,
|
||||
"peak_memory_mb_mean": 55121.4286
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -51,11 +49,9 @@
|
||||
"max-concurrency": 1,
|
||||
"enable-negative-prompt": true,
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"throughput_qps": 0.0423,
|
||||
"latency_mean": 23.5991,
|
||||
"peak_memory_mb_mean": 65118.0
|
||||
}
|
||||
"throughput_qps": 0.0423,
|
||||
"latency_mean": 23.5991,
|
||||
"peak_memory_mb_mean": 65118.0
|
||||
}
|
||||
}
|
||||
]
|
||||
@@ -95,11 +91,9 @@
|
||||
"max-concurrency": 1,
|
||||
"enable-negative-prompt": true,
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"throughput_qps": 0.4175,
|
||||
"latency_mean": 2.3953,
|
||||
"peak_memory_mb_mean": 55088.5714
|
||||
}
|
||||
"throughput_qps": 0.4175,
|
||||
"latency_mean": 2.3953,
|
||||
"peak_memory_mb_mean": 55088.5714
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -113,11 +107,9 @@
|
||||
"max-concurrency": 1,
|
||||
"enable-negative-prompt": true,
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"throughput_qps": 0.0423,
|
||||
"latency_mean": 23.5768,
|
||||
"peak_memory_mb_mean": 66012.2857
|
||||
}
|
||||
"throughput_qps": 0.0423,
|
||||
"latency_mean": 23.5768,
|
||||
"peak_memory_mb_mean": 66012.2857
|
||||
}
|
||||
}
|
||||
]
|
||||
@@ -160,11 +152,9 @@
|
||||
"max-concurrency": 1,
|
||||
"enable-negative-prompt": true,
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"throughput_qps": 0.1214,
|
||||
"latency_mean": 8.2211,
|
||||
"peak_memory_mb_mean": 54501.1429
|
||||
}
|
||||
"throughput_qps": 0.1214,
|
||||
"latency_mean": 8.2211,
|
||||
"peak_memory_mb_mean": 54501.1429
|
||||
}
|
||||
}
|
||||
]
|
||||
@@ -217,11 +207,9 @@
|
||||
"max-concurrency": 1,
|
||||
"enable-negative-prompt": true,
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"throughput_qps": 0.559,
|
||||
"latency_mean": 1.794,
|
||||
"peak_memory_mb_mean": 55290.5714
|
||||
}
|
||||
"throughput_qps": 0.559,
|
||||
"latency_mean": 1.794,
|
||||
"peak_memory_mb_mean": 55290.5714
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -235,11 +223,9 @@
|
||||
"max-concurrency": 1,
|
||||
"enable-negative-prompt": true,
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"throughput_qps": 0.1918,
|
||||
"latency_mean": 5.2022,
|
||||
"peak_memory_mb_mean": 64838.8571
|
||||
}
|
||||
"throughput_qps": 0.1918,
|
||||
"latency_mean": 5.2022,
|
||||
"peak_memory_mb_mean": 64838.8571
|
||||
}
|
||||
}
|
||||
]
|
||||
@@ -283,11 +269,9 @@
|
||||
"warmup-num-inference-steps": 20,
|
||||
"enable-negative-prompt": true,
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"throughput_qps": [0.4125, 0.6714, 0.6674, 0.8159],
|
||||
"latency_mean": [2.4215, 2.992, 5.9293, 9.656],
|
||||
"peak_memory_mb_mean": [55147.4286, 55148.0, 55150.4643, 55152.3821]
|
||||
}
|
||||
"throughput_qps": [0.4125, 0.6714, 0.6674, 0.8159],
|
||||
"latency_mean": [2.4215, 2.992, 5.9293, 9.656],
|
||||
"peak_memory_mb_mean": [55147.4286, 55148.0, 55150.4643, 55152.3821]
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
"""Tests for DFX runner metadata field exclusion."""
|
||||
|
||||
import json
|
||||
from contextlib import contextmanager
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
@@ -17,7 +15,7 @@ def test_task_excluded_from_cli_args():
|
||||
"backend": "openai-audio-speech",
|
||||
"endpoint": "/v1/audio/speech",
|
||||
"percentile-metrics": "audio_rtf,audio_ttfp",
|
||||
"baseline": {"H100": {"mean_audio_rtf": [0.5]}},
|
||||
"baseline": {"mean_audio_rtf": [0.5]},
|
||||
}
|
||||
exclude_keys = {
|
||||
"request_rate",
|
||||
@@ -123,6 +121,35 @@ def test_resolve_pytest_marks_rejects_legacy_object_format():
|
||||
)
|
||||
|
||||
|
||||
def test_extract_mark_resource_label():
|
||||
from tests.dfx.conftest import extract_mark_resource_label
|
||||
|
||||
assert extract_mark_resource_label(None) == "na"
|
||||
assert (
|
||||
extract_mark_resource_label(
|
||||
[
|
||||
{"hardware_marks": {"res": {"cuda": "H100"}, "num_cards": 1}},
|
||||
"full_model",
|
||||
]
|
||||
)
|
||||
== "H100"
|
||||
)
|
||||
assert (
|
||||
extract_mark_resource_label(
|
||||
[
|
||||
{
|
||||
"hardware_marks": {
|
||||
"res": {"cuda": "H100", "rocm": "MI325"},
|
||||
"num_cards": 2,
|
||||
}
|
||||
},
|
||||
"diffusion",
|
||||
]
|
||||
)
|
||||
== "H100-MI325"
|
||||
)
|
||||
|
||||
|
||||
def test_resource_label_for_filename():
|
||||
from tests.dfx.conftest import resource_label_for_filename
|
||||
|
||||
@@ -140,14 +167,16 @@ def test_hardware_json_value():
|
||||
assert hardware_json_value(None) == ""
|
||||
|
||||
|
||||
def test_get_runtime_resource_label(monkeypatch):
|
||||
from tests.dfx.conftest import get_runtime_resource_label
|
||||
def test_extract_configs_resource_label(monkeypatch):
|
||||
from tests.dfx.conftest import extract_configs_resource_label, get_runtime_resource_label
|
||||
|
||||
monkeypatch.setattr(
|
||||
"tests.dfx.conftest._read_runtime_device_name",
|
||||
lambda *, device_id=0: "NVIDIA H100 80GB HBM3",
|
||||
)
|
||||
assert get_runtime_resource_label(refresh=True) == "H100"
|
||||
get_runtime_resource_label(refresh=True)
|
||||
assert extract_configs_resource_label([]) == "H100"
|
||||
get_runtime_resource_label(refresh=True)
|
||||
monkeypatch.setattr(
|
||||
"tests.dfx.conftest._read_runtime_device_name",
|
||||
lambda *, device_id=0: "Ascend910B2",
|
||||
@@ -225,9 +254,8 @@ def test_benchmark_param_id_suffix_from_task_eval_phase():
|
||||
]
|
||||
|
||||
|
||||
def test_paired_omni_benchmark_reuses_server_and_preserves_case_metadata(tmp_path, monkeypatch):
|
||||
def test_create_paired_omni_benchmark_pytest_params(tmp_path):
|
||||
from tests.dfx.conftest import create_paired_omni_benchmark_pytest_params
|
||||
from tests.dfx.perf.scripts import run_benchmark
|
||||
|
||||
configs = [
|
||||
{
|
||||
@@ -257,231 +285,3 @@ def test_paired_omni_benchmark_reuses_server_and_preserves_case_metadata(tmp_pat
|
||||
assert bench_row == ("test_tts", 0)
|
||||
assert any(m.name == "tts" for m in by_id["test_tts-p0"].marks)
|
||||
assert not any(m.name == "tts" for m in by_id["test_omni-p0"].marks)
|
||||
|
||||
omni_p0_server = by_id["test_omni-p0"].values[0]
|
||||
omni_p1_server = by_id["test_omni-p1"].values[0]
|
||||
tts_server = by_id["test_tts-p0"].values[0]
|
||||
assert omni_p0_server == omni_p1_server
|
||||
assert omni_p0_server != tts_server
|
||||
|
||||
events = []
|
||||
|
||||
@contextmanager
|
||||
def fake_start(server_param):
|
||||
events.append(("start", server_param))
|
||||
yield object()
|
||||
events.append(("stop", server_param))
|
||||
|
||||
monkeypatch.setattr(run_benchmark, "_start_omni_server", fake_start)
|
||||
active_context = run_benchmark._SingleActiveContext()
|
||||
try:
|
||||
first = run_benchmark.omni_server.__wrapped__(
|
||||
SimpleNamespace(param=omni_p0_server),
|
||||
active_context,
|
||||
)
|
||||
second = run_benchmark.omni_server.__wrapped__(
|
||||
SimpleNamespace(param=omni_p1_server),
|
||||
active_context,
|
||||
)
|
||||
assert first is second
|
||||
assert events == [("start", omni_p0_server)]
|
||||
|
||||
third = run_benchmark.omni_server.__wrapped__(
|
||||
SimpleNamespace(param=tts_server),
|
||||
active_context,
|
||||
)
|
||||
assert third is not first
|
||||
assert events == [
|
||||
("start", omni_p0_server),
|
||||
("stop", omni_p0_server),
|
||||
("start", tts_server),
|
||||
]
|
||||
finally:
|
||||
active_context.close()
|
||||
|
||||
assert events == [
|
||||
("start", omni_p0_server),
|
||||
("stop", omni_p0_server),
|
||||
("start", tts_server),
|
||||
("stop", tts_server),
|
||||
]
|
||||
|
||||
|
||||
def test_is_hardware_nested_baseline():
|
||||
from tests.dfx.conftest import (
|
||||
_RUNTIME_DEVICE_ALIASES,
|
||||
is_hardware_nested_baseline,
|
||||
)
|
||||
from tests.helpers.mark import get_hardware_mark_list
|
||||
|
||||
hardware_marks = get_hardware_mark_list()
|
||||
assert hardware_marks
|
||||
assert {"H100", "L4", "A3", "MI325", "B200"} <= hardware_marks
|
||||
assert "cuda" not in hardware_marks
|
||||
assert "full_model" not in hardware_marks
|
||||
|
||||
# Runtime aliases are a full static list (independent of baseline allowlist).
|
||||
assert "H100" in _RUNTIME_DEVICE_ALIASES
|
||||
assert "A100" in _RUNTIME_DEVICE_ALIASES
|
||||
|
||||
assert is_hardware_nested_baseline(
|
||||
{
|
||||
"H100": {"mean_ttft_ms": [1.0, 2.0], "mean_e2el_ms": [10.0, 20.0]},
|
||||
"A3": {"mean_ttft_ms": [0.9, 1.8], "mean_e2el_ms": [9.0, 18.0]},
|
||||
}
|
||||
)
|
||||
# Custom metric names under a known hardware label are allowed.
|
||||
assert is_hardware_nested_baseline({"H100": {"custom_stage_ms": 12.0, "foo_bar": [1.0, 2.0]}})
|
||||
assert is_hardware_nested_baseline({"A3": {"request_throughput": 0.5}})
|
||||
# Flat metric maps / concurrency-keyed maps are not hardware-nested.
|
||||
assert not is_hardware_nested_baseline({"mean_ttft_ms": [1.0, 2.0], "mean_e2el_ms": [10.0, 20.0]})
|
||||
assert not is_hardware_nested_baseline({"mean_ttft_ms": {"1": 1.0, "32": 2.0}})
|
||||
assert not is_hardware_nested_baseline({})
|
||||
# Alias-only labels (not [hardware-resource] markers) are rejected for baselines.
|
||||
assert not is_hardware_nested_baseline({"A100": {"throughput_qps": 1.0}})
|
||||
# Unknown top-level hardware label is rejected.
|
||||
assert not is_hardware_nested_baseline({"UnknownGPU": {"throughput_qps": 1.0}})
|
||||
# Empty per-hardware metric map is rejected.
|
||||
assert not is_hardware_nested_baseline({"H100": {}})
|
||||
|
||||
|
||||
def test_resolve_baseline_for_sweep_keeps_all_hardware_for_one_concurrency():
|
||||
from tests.dfx.conftest import resolve_baseline_for_sweep
|
||||
|
||||
baseline = {
|
||||
"H100": {
|
||||
"mean_ttft_ms": [96.4, 140.8, 271.9, 362.3, 507.8],
|
||||
"mean_e2el_ms": [18507.0, 28365.0, 31907.0, 48161.0, 72630.0],
|
||||
},
|
||||
"A3": {
|
||||
"mean_ttft_ms": [90.0, 130.0, 250.0, 340.0, 480.0],
|
||||
"mean_e2el_ms": [17000.0, 26000.0, 30000.0, 45000.0, 70000.0],
|
||||
},
|
||||
}
|
||||
# max_concurrency=[1,4,8,16,32] -> index 4 is concurrency 32
|
||||
got = resolve_baseline_for_sweep(baseline, sweep_index=4)
|
||||
assert got == {
|
||||
"H100": {"mean_ttft_ms": 507.8, "mean_e2el_ms": 72630.0},
|
||||
"A3": {"mean_ttft_ms": 480.0, "mean_e2el_ms": 70000.0},
|
||||
}
|
||||
# First sweep step keeps both hardware buckets too.
|
||||
got0 = resolve_baseline_for_sweep(baseline, sweep_index=0)
|
||||
assert set(got0) == {"H100", "A3"}
|
||||
assert got0["H100"]["mean_ttft_ms"] == 96.4
|
||||
assert got0["A3"]["mean_ttft_ms"] == 90.0
|
||||
|
||||
|
||||
def test_resolve_baseline_for_sweep_rejects_flat_baseline():
|
||||
from tests.dfx.conftest import resolve_baseline_for_sweep
|
||||
|
||||
with pytest.raises(ValueError, match=r"hardware-nested.*pyproject\.toml"):
|
||||
resolve_baseline_for_sweep(
|
||||
{"throughput_qps": [0.4, 0.6], "latency_mean": [1.0, 2.0]},
|
||||
sweep_index=1,
|
||||
)
|
||||
|
||||
|
||||
def test_resolve_baseline_for_sweep_rejects_unknown_hardware_label():
|
||||
from tests.dfx.conftest import resolve_baseline_for_sweep
|
||||
|
||||
with pytest.raises(ValueError, match=r"Unknown hardware label\(s\): \['A100'\].*pyproject\.toml"):
|
||||
resolve_baseline_for_sweep(
|
||||
{"A100": {"throughput_qps": 1.0}},
|
||||
sweep_index=0,
|
||||
)
|
||||
|
||||
|
||||
def test_resolve_baseline_for_sweep_supports_list_and_scalar_under_hardware():
|
||||
from tests.dfx.conftest import resolve_baseline_for_sweep
|
||||
|
||||
# Sweep-aligned lists under each hardware bucket (canonical form).
|
||||
listed = {
|
||||
"H100": {"throughput_qps": [0.4, 0.6, 0.8], "latency_mean": [1.0, 2.0, 3.0]},
|
||||
"A3": {"throughput_qps": [0.5, 0.7, 0.9], "latency_mean": [0.9, 1.8, 2.7]},
|
||||
}
|
||||
assert resolve_baseline_for_sweep(listed, sweep_index=1) == {
|
||||
"H100": {"throughput_qps": 0.6, "latency_mean": 2.0},
|
||||
"A3": {"throughput_qps": 0.7, "latency_mean": 1.8},
|
||||
}
|
||||
|
||||
# Scalars under hardware stay as-is (single-concurrency cases).
|
||||
scalar = {"H100": {"throughput_qps": 0.5}}
|
||||
assert resolve_baseline_for_sweep(scalar, sweep_index=0) == {"H100": {"throughput_qps": 0.5}}
|
||||
# Custom metric names are preserved.
|
||||
custom = {"H100": {"custom_stage_ms": [10.0, 20.0]}}
|
||||
assert resolve_baseline_for_sweep(custom, sweep_index=1) == {"H100": {"custom_stage_ms": 20.0}}
|
||||
assert resolve_baseline_for_sweep(None) == {}
|
||||
assert resolve_baseline_for_sweep({}) == {}
|
||||
|
||||
|
||||
def test_resolve_baseline_value_errors():
|
||||
from tests.dfx.conftest import resolve_baseline_value
|
||||
|
||||
with pytest.raises(ValueError, match="sweep_index"):
|
||||
resolve_baseline_value([1.0, 2.0], sweep_index=None)
|
||||
with pytest.raises(IndexError):
|
||||
resolve_baseline_value([1.0], sweep_index=1)
|
||||
with pytest.raises(TypeError, match="not supported"):
|
||||
resolve_baseline_value({"1": 0.4}, sweep_index=0)
|
||||
|
||||
|
||||
def test_diffusion_build_run_params_resolves_baseline_per_sweep(tmp_path, monkeypatch):
|
||||
"""``_build_run_params`` / ``_iter_sweep_runs`` narrow list baselines per concurrency."""
|
||||
import importlib
|
||||
import sys
|
||||
|
||||
cfg = tmp_path / "mini_diffusion_perf.json"
|
||||
cfg.write_text(
|
||||
json.dumps(
|
||||
[
|
||||
{
|
||||
"test_name": "test_mini",
|
||||
"server_type": "vllm-omni",
|
||||
"mark": [
|
||||
{"hardware_marks": {"res": {"cuda": "H100"}, "num_cards": 1}},
|
||||
"diffusion",
|
||||
"full_model",
|
||||
],
|
||||
"server_params": {"model": "m/mini"},
|
||||
"benchmark_params": [{"name": "p0", "num-prompts": 1, "max-concurrency": 1}],
|
||||
}
|
||||
]
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
monkeypatch.setattr(sys, "argv", ["pytest", "--test-config-file", str(cfg)])
|
||||
sys.modules.pop("tests.dfx.perf.scripts.run_diffusion_benchmark", None)
|
||||
from tests.dfx.perf.scripts import run_diffusion_benchmark as rdb
|
||||
|
||||
importlib.reload(rdb)
|
||||
|
||||
params = {
|
||||
"name": "c_sweep",
|
||||
"dataset": "random",
|
||||
"task": "t2i",
|
||||
"num-prompts": [8, 16, 32],
|
||||
"max-concurrency": [1, 8, 32],
|
||||
"baseline": {
|
||||
"H100": {"throughput_qps": [0.1, 0.2, 0.3], "latency_mean": [10.0, 20.0, 30.0]},
|
||||
"A3": {"throughput_qps": [0.11, 0.22, 0.33], "latency_mean": [9.0, 19.0, 29.0]},
|
||||
},
|
||||
}
|
||||
run32 = rdb._build_run_params(
|
||||
params,
|
||||
num_prompts=32,
|
||||
max_concurrency=32,
|
||||
request_rate="inf",
|
||||
sweep_index=2,
|
||||
)
|
||||
assert run32["max-concurrency"] == 32
|
||||
assert run32["baseline"] == {
|
||||
"H100": {"throughput_qps": 0.3, "latency_mean": 30.0},
|
||||
"A3": {"throughput_qps": 0.33, "latency_mean": 29.0},
|
||||
}
|
||||
|
||||
sweeps = rdb._iter_sweep_runs(params)
|
||||
assert len(sweeps) == 3
|
||||
assert sweeps[2]["params"]["max-concurrency"] == 32
|
||||
assert sweeps[2]["params"]["baseline"]["H100"]["throughput_qps"] == 0.3
|
||||
assert "A3" in sweeps[2]["params"]["baseline"]
|
||||
assert not isinstance(sweeps[2]["params"]["baseline"]["H100"]["throughput_qps"], list)
|
||||
+111
-137
@@ -39,14 +39,12 @@
|
||||
"seed_tts_locale": "en",
|
||||
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"median_audio_ttfp_ms": [
|
||||
141.2791
|
||||
],
|
||||
"median_audio_rtf": [
|
||||
0.1591
|
||||
]
|
||||
}
|
||||
"median_audio_ttfp_ms": [
|
||||
141.2791
|
||||
],
|
||||
"median_audio_rtf": [
|
||||
0.1591
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -69,23 +67,21 @@
|
||||
"seed_tts_locale": "en",
|
||||
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"median_audio_ttfp_ms": [
|
||||
226.747,
|
||||
647.1272,
|
||||
4777.2714
|
||||
],
|
||||
"median_audio_rtf": [
|
||||
0.3313,
|
||||
0.3642,
|
||||
1.2282
|
||||
],
|
||||
"audio_throughput": [
|
||||
24.2296,
|
||||
40.3552,
|
||||
43.2572
|
||||
]
|
||||
}
|
||||
"median_audio_ttfp_ms": [
|
||||
226.747,
|
||||
647.1272,
|
||||
4777.2714
|
||||
],
|
||||
"median_audio_rtf": [
|
||||
0.3313,
|
||||
0.3642,
|
||||
1.2282
|
||||
],
|
||||
"audio_throughput": [
|
||||
24.2296,
|
||||
40.3552,
|
||||
43.2572
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -106,11 +102,9 @@
|
||||
"seed_tts_wer_eval": true,
|
||||
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"mean_audio_rtf": [
|
||||
0.45
|
||||
]
|
||||
}
|
||||
"mean_audio_rtf": [
|
||||
0.45
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
@@ -159,14 +153,12 @@
|
||||
},
|
||||
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"median_audio_ttfp_ms": [
|
||||
46.8434
|
||||
],
|
||||
"median_audio_rtf": [
|
||||
0.1367
|
||||
]
|
||||
}
|
||||
"median_audio_ttfp_ms": [
|
||||
46.8434
|
||||
],
|
||||
"median_audio_rtf": [
|
||||
0.1367
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -194,23 +186,21 @@
|
||||
},
|
||||
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"median_audio_ttfp_ms": [
|
||||
75.3341,
|
||||
713.7727,
|
||||
5941.7179
|
||||
],
|
||||
"median_audio_rtf": [
|
||||
0.1859,
|
||||
0.3148,
|
||||
1.1662
|
||||
],
|
||||
"audio_throughput": [
|
||||
38.3858,
|
||||
55.3523,
|
||||
68.7032
|
||||
]
|
||||
}
|
||||
"median_audio_ttfp_ms": [
|
||||
75.3341,
|
||||
713.7727,
|
||||
5941.7179
|
||||
],
|
||||
"median_audio_rtf": [
|
||||
0.1859,
|
||||
0.3148,
|
||||
1.1662
|
||||
],
|
||||
"audio_throughput": [
|
||||
38.3858,
|
||||
55.3523,
|
||||
68.7032
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -236,11 +226,9 @@
|
||||
"seed_tts_wer_eval": true,
|
||||
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"mean_audio_rtf": [
|
||||
0.35
|
||||
]
|
||||
}
|
||||
"mean_audio_rtf": [
|
||||
0.35
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -264,17 +252,15 @@
|
||||
},
|
||||
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"median_audio_ttfp_ms": [
|
||||
56.746
|
||||
],
|
||||
"median_audio_rtf": [
|
||||
0.1571
|
||||
],
|
||||
"audio_throughput": [
|
||||
11.1226
|
||||
]
|
||||
}
|
||||
"median_audio_ttfp_ms": [
|
||||
56.746
|
||||
],
|
||||
"median_audio_rtf": [
|
||||
0.1571
|
||||
],
|
||||
"audio_throughput": [
|
||||
11.1226
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -297,14 +283,12 @@
|
||||
},
|
||||
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"median_audio_ttfp_ms": [
|
||||
46.3479
|
||||
],
|
||||
"median_audio_rtf": [
|
||||
0.1352
|
||||
]
|
||||
}
|
||||
"median_audio_ttfp_ms": [
|
||||
46.3479
|
||||
],
|
||||
"median_audio_rtf": [
|
||||
0.1352
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -331,23 +315,21 @@
|
||||
},
|
||||
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"median_audio_ttfp_ms": [
|
||||
75.5967,
|
||||
587.5877,
|
||||
4960.3726
|
||||
],
|
||||
"median_audio_rtf": [
|
||||
0.1871,
|
||||
0.3143,
|
||||
1.1703
|
||||
],
|
||||
"audio_throughput": [
|
||||
38.9378,
|
||||
55.1846,
|
||||
71.5708
|
||||
]
|
||||
}
|
||||
"median_audio_ttfp_ms": [
|
||||
75.5967,
|
||||
587.5877,
|
||||
4960.3726
|
||||
],
|
||||
"median_audio_rtf": [
|
||||
0.1871,
|
||||
0.3143,
|
||||
1.1703
|
||||
],
|
||||
"audio_throughput": [
|
||||
38.9378,
|
||||
55.1846,
|
||||
71.5708
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -370,17 +352,15 @@
|
||||
},
|
||||
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"median_audio_ttfp_ms": [
|
||||
55.5029
|
||||
],
|
||||
"median_audio_rtf": [
|
||||
0.1508
|
||||
],
|
||||
"audio_throughput": [
|
||||
9.4885
|
||||
]
|
||||
}
|
||||
"median_audio_ttfp_ms": [
|
||||
55.5029
|
||||
],
|
||||
"median_audio_rtf": [
|
||||
0.1508
|
||||
],
|
||||
"audio_throughput": [
|
||||
9.4885
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
@@ -422,14 +402,12 @@
|
||||
},
|
||||
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"median_audio_ttfp_ms": [
|
||||
82.1886
|
||||
],
|
||||
"median_audio_rtf": [
|
||||
0.1241
|
||||
]
|
||||
}
|
||||
"median_audio_ttfp_ms": [
|
||||
82.1886
|
||||
],
|
||||
"median_audio_rtf": [
|
||||
0.1241
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -451,17 +429,15 @@
|
||||
},
|
||||
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"median_audio_ttfp_ms": [
|
||||
160.9168
|
||||
],
|
||||
"median_audio_rtf": [
|
||||
0.1907
|
||||
],
|
||||
"audio_throughput": [
|
||||
39.2332
|
||||
]
|
||||
}
|
||||
"median_audio_ttfp_ms": [
|
||||
160.9168
|
||||
],
|
||||
"median_audio_rtf": [
|
||||
0.1907
|
||||
],
|
||||
"audio_throughput": [
|
||||
39.2332
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -483,17 +459,15 @@
|
||||
},
|
||||
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"median_audio_ttfp_ms": [
|
||||
100.1265
|
||||
],
|
||||
"median_audio_rtf": [
|
||||
0.1437
|
||||
],
|
||||
"audio_throughput": [
|
||||
11.7605
|
||||
]
|
||||
}
|
||||
"median_audio_ttfp_ms": [
|
||||
100.1265
|
||||
],
|
||||
"median_audio_rtf": [
|
||||
0.1437
|
||||
],
|
||||
"audio_throughput": [
|
||||
11.7605
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
@@ -37,14 +37,12 @@
|
||||
"seed_tts_locale": "en",
|
||||
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"median_audio_ttfp_ms": [
|
||||
137.645
|
||||
],
|
||||
"median_audio_rtf": [
|
||||
0.0497
|
||||
]
|
||||
}
|
||||
"median_audio_ttfp_ms": [
|
||||
137.645
|
||||
],
|
||||
"median_audio_rtf": [
|
||||
0.0497
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -63,17 +61,15 @@
|
||||
"seed_tts_locale": "en",
|
||||
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"median_audio_ttfp_ms": [
|
||||
475.3357
|
||||
],
|
||||
"median_audio_rtf": [
|
||||
0.1243
|
||||
],
|
||||
"audio_throughput": [
|
||||
36.6358
|
||||
]
|
||||
}
|
||||
"median_audio_ttfp_ms": [
|
||||
475.3357
|
||||
],
|
||||
"median_audio_rtf": [
|
||||
0.1243
|
||||
],
|
||||
"audio_throughput": [
|
||||
36.6358
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -94,11 +90,9 @@
|
||||
"seed_tts_wer_eval": true,
|
||||
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"mean_audio_rtf": [
|
||||
0.45
|
||||
]
|
||||
}
|
||||
"mean_audio_rtf": [
|
||||
0.45
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -120,14 +114,12 @@
|
||||
},
|
||||
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"median_audio_ttfp_ms": [
|
||||
79.6927
|
||||
],
|
||||
"median_audio_rtf": [
|
||||
0.0423
|
||||
]
|
||||
}
|
||||
"median_audio_ttfp_ms": [
|
||||
79.6927
|
||||
],
|
||||
"median_audio_rtf": [
|
||||
0.0423
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -149,17 +141,15 @@
|
||||
},
|
||||
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"median_audio_ttfp_ms": [
|
||||
262.6212
|
||||
],
|
||||
"median_audio_rtf": [
|
||||
0.0753
|
||||
],
|
||||
"audio_throughput": [
|
||||
103.5843
|
||||
]
|
||||
}
|
||||
"median_audio_ttfp_ms": [
|
||||
262.6212
|
||||
],
|
||||
"median_audio_rtf": [
|
||||
0.0753
|
||||
],
|
||||
"audio_throughput": [
|
||||
103.5843
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -181,17 +171,15 @@
|
||||
},
|
||||
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"median_audio_ttfp_ms": [
|
||||
80.1095
|
||||
],
|
||||
"median_audio_rtf": [
|
||||
0.0479
|
||||
],
|
||||
"audio_throughput": [
|
||||
17.2918
|
||||
]
|
||||
}
|
||||
"median_audio_ttfp_ms": [
|
||||
80.1095
|
||||
],
|
||||
"median_audio_rtf": [
|
||||
0.0479
|
||||
],
|
||||
"audio_throughput": [
|
||||
17.2918
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -215,11 +203,9 @@
|
||||
"seed_tts_wer_eval": true,
|
||||
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"mean_audio_rtf": [
|
||||
0.35
|
||||
]
|
||||
}
|
||||
"mean_audio_rtf": [
|
||||
0.35
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
@@ -42,11 +42,9 @@
|
||||
}
|
||||
],
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"throughput_qps": 0.0434,
|
||||
"latency_mean": 23.0654,
|
||||
"peak_memory_mb_mean": 76360.0
|
||||
}
|
||||
"throughput_qps": 0.0434,
|
||||
"latency_mean": 23.0654,
|
||||
"peak_memory_mb_mean": 76360.0
|
||||
}
|
||||
}
|
||||
]
|
||||
@@ -99,11 +97,9 @@
|
||||
}
|
||||
],
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"throughput_qps": 0.0606,
|
||||
"latency_mean": 16.4928,
|
||||
"peak_memory_mb_mean": 47164.8
|
||||
}
|
||||
"throughput_qps": 0.0606,
|
||||
"latency_mean": 16.4928,
|
||||
"peak_memory_mb_mean": 47164.8
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -126,11 +122,9 @@
|
||||
}
|
||||
],
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"throughput_qps": 0.0124,
|
||||
"latency_mean": 80.7784,
|
||||
"peak_memory_mb_mean": 55374.3429
|
||||
}
|
||||
"throughput_qps": 0.0124,
|
||||
"latency_mean": 80.7784,
|
||||
"peak_memory_mb_mean": 55374.3429
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""HTTP + WebSocket validation for Qwen3-TTS Base and Higgs Audio V3 ``/v1/audio/speech`` (and related routes)."""
|
||||
"""HTTP + WebSocket validation for Qwen3-TTS Base, Higgs Audio V3 and Audex TTA ``/v1/audio/speech`` (and related routes)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -70,6 +70,19 @@ _HIGGS_AUDIO_V3_SPEECH = [
|
||||
),
|
||||
]
|
||||
|
||||
_AUDEX_TTA_CAPTION = "Heavy rain falling on a tin roof."
|
||||
_AUDEX_TTA_SPEECH = [
|
||||
pytest.param(
|
||||
OmniServerParams(
|
||||
model="nvidia/Nemotron-Labs-Audex-2B",
|
||||
stage_config_path=get_deploy_config_path("audex_tta.yaml"),
|
||||
server_args=_SPEECH_SERVER_ARGS,
|
||||
),
|
||||
id="audex_tta",
|
||||
marks=_L4_SPEECH_HW,
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def _apply_batch_overrides(body: dict[str, object], *, loc: str, overrides: dict[str, object]) -> None:
|
||||
if loc == "batch":
|
||||
@@ -325,6 +338,37 @@ def test_speech_higgs_invalid_field_values(
|
||||
)
|
||||
|
||||
|
||||
# Audex TTA (``audex_tta`` deployment): the serving layer builds the caption
|
||||
# prompt and injects the RVQ phase contract + default guidance server-side, so
|
||||
# guidance knobs and speech-only fields must be rejected before generation.
|
||||
@pytest.mark.parametrize(
|
||||
"overrides, err_message",
|
||||
[
|
||||
pytest.param({"extra_params": {"cfg_scale": 99}}, "cfg_scale", id="cfg_scale_out_of_range"),
|
||||
pytest.param({"extra_params": {"cfg_scale": "big"}}, "cfg_scale", id="cfg_scale_wrong_type"),
|
||||
pytest.param({"extra_params": {"tta_rvq": {}}}, "tta_rvq", id="tta_rvq_not_overridable"),
|
||||
pytest.param({"input": " "}, "caption", id="caption_whitespace_only"),
|
||||
pytest.param({"voice": "alloy"}, "voice", id="voice_not_supported"),
|
||||
pytest.param(
|
||||
{"ref_audio": "https://example.com/ref.wav"},
|
||||
"reference audio",
|
||||
id="ref_audio_not_supported",
|
||||
),
|
||||
],
|
||||
)
|
||||
@pytest.mark.parametrize("omni_server", _AUDEX_TTA_SPEECH, indirect=True)
|
||||
def test_speech_audex_tta_invalid_field_values(
|
||||
omni_server: OmniServer,
|
||||
openai_client: OpenAIClientHandler,
|
||||
overrides: dict[str, object],
|
||||
err_message: str | tuple[str, ...],
|
||||
) -> None:
|
||||
body = {"model": omni_server.model, "input": _AUDEX_TTA_CAPTION, **overrides}
|
||||
openai_client.send_audio_speech_http_request(
|
||||
{"json": body, "timeout": 120, "err_code": 400, "err_message": err_message}
|
||||
)
|
||||
|
||||
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
# POST /v1/audio/speech/batch (JSON)
|
||||
# ─────────────────────────────────────────────────────────────────────────────
|
||||
|
||||
Loaded 100 of 297 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user