Compare commits
4
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Joshna-Medisetty
|
807db6efd7 | ||
|
Yueqian LinandSy03
|
d4a5eaac90 | ||
|
Yueqian Lin
|
8f115164c2 | ||
|
|
cbab1d4d8f |
No files matched your search
@@ -6,10 +6,6 @@
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# minicpm-challenge branch: skip entire AMD CI (no pipeline upload => no image build / tests).
|
||||
echo "--- Skipping AMD CI on this branch (including image build)"
|
||||
exit 0
|
||||
|
||||
source .buildkite/common/scripts/resolve_skip_ci.sh
|
||||
|
||||
if [[ -z "${RUN_ALL:-}" ]]; then
|
||||
|
||||
@@ -282,7 +282,7 @@ def _git(*args: str) -> subprocess.CompletedProcess[str]:
|
||||
|
||||
|
||||
def _resolve_diff_range() -> str | None:
|
||||
"""Return a git diff range for PR or push-branch builds, or None when unavailable."""
|
||||
"""Return a git diff range for PR or main builds, or None when unavailable."""
|
||||
is_pr = os.environ.get("BUILDKITE_PULL_REQUEST", "false") != "false" and os.environ.get(
|
||||
"BUILDKITE_PULL_REQUEST", ""
|
||||
)
|
||||
@@ -302,15 +302,13 @@ def _resolve_diff_range() -> str | None:
|
||||
return None
|
||||
return f"{base_ref}...{commit}"
|
||||
|
||||
# main: ready/nightly/weekly push CI; minicpm-challenge: merge (L3) push CI
|
||||
branch = os.environ.get("BUILDKITE_BRANCH", "")
|
||||
if branch in ("main", "minicpm-challenge"):
|
||||
if os.environ.get("BUILDKITE_BRANCH", "") == "main":
|
||||
if _git("rev-parse", "--verify", f"{commit}^").returncode != 0:
|
||||
_log(f"{branch} commit has no parent; using safe defaults")
|
||||
_log("main commit has no parent; using safe defaults")
|
||||
return None
|
||||
return f"{commit}^..{commit}"
|
||||
|
||||
_log("not PR/main/minicpm-challenge build; using safe defaults")
|
||||
_log("not PR/main build; using safe defaults")
|
||||
return None
|
||||
|
||||
|
||||
|
||||
@@ -59,12 +59,11 @@ E2E_GROUP_MARKER = "E2E Test"
|
||||
CI_MIRROR_HARDWARES_PATH = ROOT / ".buildkite/common/ci_mirror_hardwares.yml"
|
||||
|
||||
CUDA_NIGHTLY_ONLY = (
|
||||
'(build.pull_request.labels includes "nightly-test") || '
|
||||
'(build.branch == "minicpm-challenge" && build.env("NIGHTLY") == "1")'
|
||||
'(build.pull_request.labels includes "nightly-test") || (build.branch == "main" && build.env("NIGHTLY") == "1")'
|
||||
)
|
||||
NPU_NIGHTLY_ONLY = (
|
||||
'(build.branch == "minicpm-challenge" && build.env("NIGHTLY") == "1") || '
|
||||
'(build.branch != "minicpm-challenge" && ('
|
||||
'(build.branch == "main" && build.env("NIGHTLY") == "1") || '
|
||||
'(build.branch != "main" && ('
|
||||
'build.pull_request.labels includes "nightly-test" || '
|
||||
'build.pull_request.labels includes "omni-test" || '
|
||||
'build.pull_request.labels includes "tts-test" || '
|
||||
@@ -104,8 +103,7 @@ def _format_bootstrap_if(expr: str) -> str:
|
||||
|
||||
def _compute_bootstrap_if_exprs(*, decision, platform: str) -> dict[str, str]:
|
||||
disabled = "false"
|
||||
# L4 nightly scheduled push CI runs on minicpm-challenge (not main).
|
||||
nightly_main = 'build.branch == "minicpm-challenge" && build.env("NIGHTLY") == "1"'
|
||||
nightly_main = 'build.branch == "main" && build.env("NIGHTLY") == "1"'
|
||||
nightly_only = NPU_NIGHTLY_ONLY if platform == "npu" else CUDA_NIGHTLY_ONLY
|
||||
|
||||
if platform == "npu":
|
||||
@@ -120,12 +118,11 @@ def _compute_bootstrap_if_exprs(*, decision, platform: str) -> dict[str, str]:
|
||||
merge_base = disabled
|
||||
else:
|
||||
ready_pr = 'build.branch != "main" && build.pull_request.labels includes "ready"'
|
||||
# L3 merge push CI runs on minicpm-challenge (not main).
|
||||
merge_main = 'build.branch == "minicpm-challenge" && build.env("NIGHTLY") != "1" && build.env("WEEKLY") != "1"'
|
||||
merge_pr = 'build.branch != "minicpm-challenge" && build.pull_request.labels includes "merge-test"'
|
||||
merge_main = 'build.branch == "main" && build.env("NIGHTLY") != "1" && build.env("WEEKLY") != "1"'
|
||||
merge_pr = 'build.branch != "main" && build.pull_request.labels includes "merge-test"'
|
||||
nightly_label_if = (
|
||||
'(build.branch == "minicpm-challenge" && build.env("NIGHTLY") == "1") || '
|
||||
'(build.branch != "minicpm-challenge" && ('
|
||||
'(build.branch == "main" && build.env("NIGHTLY") == "1") || '
|
||||
'(build.branch != "main" && ('
|
||||
'build.pull_request.labels includes "nightly-test" || '
|
||||
'build.pull_request.labels includes "omni-test" || '
|
||||
'build.pull_request.labels includes "tts-test" || '
|
||||
@@ -143,8 +140,8 @@ def _compute_bootstrap_if_exprs(*, decision, platform: str) -> dict[str, str]:
|
||||
weekly_main = 'build.branch == "main" && build.env("WEEKLY") == "1"'
|
||||
|
||||
if decision.skip_all:
|
||||
# Docs / skip-mark only: no PR-label escape hatch. Scheduled
|
||||
# NIGHTLY=1 (minicpm-challenge) still runs L4 + L2/L3 (--e2e); WEEKLY=1 (main) still runs L5.
|
||||
# Docs / skip-mark only: no PR-label escape hatch. Main scheduled
|
||||
# NIGHTLY=1 still runs L4 + L2/L3 (--e2e); WEEKLY=1 still runs L5.
|
||||
image_expr = f"({nightly_main}) || ({weekly_main})" if platform == "cuda" else nightly_main
|
||||
ready_expr = nightly_main
|
||||
merge_expr = nightly_main if platform == "cuda" else disabled
|
||||
|
||||
@@ -8,35 +8,35 @@ steps:
|
||||
agents:
|
||||
queue: "cpu_queue_premerge"
|
||||
|
||||
# L2 Test — PR ready label (diff-aware) or minicpm-challenge+NIGHTLY=1 (--e2e, full E2E group)
|
||||
# L2 Test — PR ready label (diff-aware) or main+NIGHTLY=1 (--e2e, full E2E group)
|
||||
- label: "Upload Ready Pipeline"
|
||||
depends_on: image-build
|
||||
key: upload-ready-pipeline
|
||||
commands:
|
||||
- |
|
||||
EXTRA=""
|
||||
if [ "${NIGHTLY:-}" = "1" ] && [ "${BUILDKITE_BRANCH}" = "minicpm-challenge" ]; then
|
||||
if [ "${NIGHTLY:-}" = "1" ] && [ "${BUILDKITE_BRANCH}" = "main" ]; then
|
||||
EXTRA="--e2e"
|
||||
fi
|
||||
python3 .buildkite/common/scripts/upload_pipeline.py --upload $$EXTRA .buildkite/cuda/test-ready.yml
|
||||
agents:
|
||||
queue: "cpu_queue_premerge"
|
||||
|
||||
# L3 Test — PR merge-test label (diff-aware) or minicpm-challenge+NIGHTLY=1 (--e2e, full E2E group)
|
||||
# L3 Test — PR merge-test label (diff-aware) or main+NIGHTLY=1 (--e2e, full E2E group)
|
||||
- label: "Upload Merge Pipeline"
|
||||
depends_on: image-build
|
||||
key: upload-merge-pipeline
|
||||
commands:
|
||||
- |
|
||||
EXTRA=""
|
||||
if [ "${NIGHTLY:-}" = "1" ] && [ "${BUILDKITE_BRANCH}" = "minicpm-challenge" ]; then
|
||||
if [ "${NIGHTLY:-}" = "1" ] && [ "${BUILDKITE_BRANCH}" = "main" ]; then
|
||||
EXTRA="--e2e"
|
||||
fi
|
||||
python3 .buildkite/common/scripts/upload_pipeline.py --upload $$EXTRA .buildkite/cuda/test-merge.yml
|
||||
agents:
|
||||
queue: "cpu_queue_premerge"
|
||||
|
||||
# L4 Test — minicpm-challenge+NIGHTLY=1 (scheduled), or PR with specific label (e.g. add label then Rebuild)
|
||||
# L4 Test — main+NIGHTLY=1 (scheduled), or PR with specific label (e.g. add label then Rebuild)
|
||||
- label: "Upload Nightly Pipeline"
|
||||
depends_on: image-build
|
||||
key: upload-nightly-pipeline
|
||||
|
||||
+238
-204
@@ -18,227 +18,261 @@ steps:
|
||||
- "timeout 20m pytest -sv -m 'core_model and cpu' --ignore=tests/diffusion --ignore=tests/model_executor --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"
|
||||
mirror_hardwares: l4_1
|
||||
|
||||
# - group: ":card_index_dividers: Diffusion Test"
|
||||
# depends_on: upload-merge-pipeline
|
||||
# steps:
|
||||
# - label: "Diffusion · LoRA Test"
|
||||
# timeout_in_minutes: 30
|
||||
# commands:
|
||||
# - pytest -sv tests/diffusion/lora/ -m 'advanced_model and cuda' --run-level "advanced_model"
|
||||
# mirror_hardwares: l4_1
|
||||
- group: ":card_index_dividers: Diffusion Test"
|
||||
depends_on: upload-merge-pipeline
|
||||
steps:
|
||||
- label: "Diffusion · LoRA Test"
|
||||
timeout_in_minutes: 30
|
||||
commands:
|
||||
- pytest -sv tests/diffusion/lora/ -m 'advanced_model and cuda' --run-level "advanced_model"
|
||||
mirror_hardwares: l4_1
|
||||
|
||||
# - label: "Diffusion · Offloader Test"
|
||||
# timeout_in_minutes: 40
|
||||
# commands:
|
||||
# - timeout 40m pytest -s -v tests/diffusion/offloader -m "advanced_model and cuda"
|
||||
# mirror_hardwares: l4_1
|
||||
- label: "Diffusion · Offloader Test"
|
||||
timeout_in_minutes: 40
|
||||
commands:
|
||||
- timeout 40m pytest -s -v tests/diffusion/offloader -m "advanced_model and cuda"
|
||||
mirror_hardwares: l4_1
|
||||
|
||||
# - label: "Distributed Test"
|
||||
# depends_on: upload-merge-pipeline
|
||||
# timeout_in_minutes: 60
|
||||
# commands:
|
||||
# - |
|
||||
# timeout 55m bash -c "
|
||||
# set -e
|
||||
# export VLLM_IMAGE_FETCH_TIMEOUT=60
|
||||
# pytest -s -v tests/distributed -m 'advanced_model and cuda and H100' --run-level 'advanced_model'
|
||||
# "
|
||||
# mirror_hardwares: h100_1
|
||||
- label: "Distributed Test"
|
||||
depends_on: upload-merge-pipeline
|
||||
timeout_in_minutes: 60
|
||||
commands:
|
||||
- |
|
||||
timeout 55m bash -c "
|
||||
set -e
|
||||
export VLLM_IMAGE_FETCH_TIMEOUT=60
|
||||
pytest -s -v tests/distributed -m 'advanced_model and cuda and H100' --run-level 'advanced_model'
|
||||
"
|
||||
mirror_hardwares: h100_1
|
||||
|
||||
# - group: ":card_index_dividers: Entrypoints Test"
|
||||
# depends_on: upload-merge-pipeline
|
||||
# steps:
|
||||
# - label: "Entrypoint Test with H100"
|
||||
# timeout_in_minutes: 60
|
||||
# commands:
|
||||
# - pytest -s -v tests/entrypoints/ -m "advanced_model and cuda and H100" --run-level "advanced_model"
|
||||
# mirror_hardwares: h100_2
|
||||
- group: ":card_index_dividers: Entrypoints Test"
|
||||
depends_on: upload-merge-pipeline
|
||||
steps:
|
||||
- label: "Entrypoint Test with H100"
|
||||
# TEMPORARY 60 -> 90: engine startup/sleep-wake is ~13-15 min slower per
|
||||
# run on the vLLM 0.26 image (model loading ~2x, init ~2x; see builds
|
||||
# 2915 vs 2916), which pushed this suite past 60 min. Revert once the
|
||||
# 0.26 startup regression is understood and fixed.
|
||||
timeout_in_minutes: 90
|
||||
commands:
|
||||
- pytest -s -v tests/entrypoints/ -m "advanced_model and cuda and H100" --run-level "advanced_model"
|
||||
mirror_hardwares: h100_2
|
||||
|
||||
# - label: "Entrypoint Test with L4"
|
||||
# timeout_in_minutes: 30
|
||||
# commands:
|
||||
# - pytest -s -v tests/entrypoints/ -m "advanced_model and cuda and L4" --run-level "advanced_model"
|
||||
# mirror_hardwares: l4_1
|
||||
- label: "Entrypoint Test with L4"
|
||||
timeout_in_minutes: 60
|
||||
commands:
|
||||
- pytest -s -v tests/entrypoints/ -m "advanced_model and cuda and L4" --run-level "advanced_model"
|
||||
mirror_hardwares: l4_1
|
||||
|
||||
# - group: ":card_index_dividers: E2E Test"
|
||||
# depends_on: upload-merge-pipeline
|
||||
# steps:
|
||||
# - label: "TTS · Qwen3-TTS CustomVoice Test"
|
||||
# source_file_dependencies:
|
||||
# - tests/e2e/online_serving/test_qwen3_tts_customvoice.py
|
||||
# - tests/e2e/offline_inference/test_qwen3_tts_customvoice.py
|
||||
# - vllm_omni/model_executor/models/qwen3_tts/
|
||||
# - vllm_omni/model_executor/stage_input_processors/qwen3_tts.py
|
||||
# - vllm_omni/deploy/qwen3_tts.yaml
|
||||
# commands:
|
||||
# - |
|
||||
# timeout 40m bash -c "
|
||||
# export VLLM_LOGGING_LEVEL=DEBUG
|
||||
# export VLLM_ALLOW_LONG_MAX_MODEL_LEN=1
|
||||
# pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py tests/e2e/offline_inference/test_qwen3_tts_customvoice.py -m 'advanced_model and cuda' --run-level 'advanced_model'
|
||||
# "
|
||||
# mirror_hardwares: l4_1
|
||||
- group: ":card_index_dividers: E2E Test"
|
||||
depends_on: upload-merge-pipeline
|
||||
steps:
|
||||
- label: "TTS · Qwen3-TTS CustomVoice Test"
|
||||
source_file_dependencies:
|
||||
- tests/e2e/online_serving/test_qwen3_tts_customvoice.py
|
||||
- tests/e2e/offline_inference/test_qwen3_tts_customvoice.py
|
||||
- vllm_omni/model_executor/models/qwen3_tts/
|
||||
- vllm_omni/model_executor/stage_input_processors/qwen3_tts.py
|
||||
- vllm_omni/deploy/qwen3_tts.yaml
|
||||
commands:
|
||||
- |
|
||||
timeout 40m bash -c "
|
||||
export VLLM_LOGGING_LEVEL=DEBUG
|
||||
export VLLM_ALLOW_LONG_MAX_MODEL_LEN=1
|
||||
pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py tests/e2e/offline_inference/test_qwen3_tts_customvoice.py -m 'advanced_model and cuda' --run-level 'advanced_model'
|
||||
"
|
||||
mirror_hardwares: l4_1
|
||||
|
||||
# - label: "TTS · Qwen3-TTS Base Test"
|
||||
# source_file_dependencies:
|
||||
# - tests/e2e/online_serving/test_qwen3_tts_base.py
|
||||
# - tests/e2e/offline_inference/test_qwen3_tts_base.py
|
||||
# - vllm_omni/model_executor/models/qwen3_tts/
|
||||
# - vllm_omni/model_executor/stage_input_processors/qwen3_tts.py
|
||||
# - vllm_omni/deploy/qwen3_tts.yaml
|
||||
# commands:
|
||||
# - |
|
||||
# timeout 40m bash -c "
|
||||
# export VLLM_LOGGING_LEVEL=DEBUG
|
||||
# export VLLM_ALLOW_LONG_MAX_MODEL_LEN=1
|
||||
# pytest -s -v tests/e2e/online_serving/test_qwen3_tts_base.py tests/e2e/offline_inference/test_qwen3_tts_base.py -m 'advanced_model and cuda' --run-level 'advanced_model'
|
||||
# "
|
||||
# mirror_hardwares: l4_1
|
||||
- label: "TTS · Qwen3-TTS Base Test"
|
||||
source_file_dependencies:
|
||||
- tests/e2e/online_serving/test_qwen3_tts_base.py
|
||||
- tests/e2e/offline_inference/test_qwen3_tts_base.py
|
||||
- vllm_omni/model_executor/models/qwen3_tts/
|
||||
- vllm_omni/model_executor/stage_input_processors/qwen3_tts.py
|
||||
- vllm_omni/deploy/qwen3_tts.yaml
|
||||
commands:
|
||||
- |
|
||||
timeout 40m bash -c "
|
||||
export VLLM_LOGGING_LEVEL=DEBUG
|
||||
export VLLM_ALLOW_LONG_MAX_MODEL_LEN=1
|
||||
pytest -s -v tests/e2e/online_serving/test_qwen3_tts_base.py tests/e2e/offline_inference/test_qwen3_tts_base.py -m 'advanced_model and cuda' --run-level 'advanced_model'
|
||||
"
|
||||
mirror_hardwares: l4_1
|
||||
|
||||
# - label: "TTS · Higgs-Audio-V3 Test"
|
||||
# source_file_dependencies:
|
||||
# - tests/e2e/online_serving/test_higgs_audio_v3.py
|
||||
# - vllm_omni/model_executor/models/higgs_audio_v3/
|
||||
# - vllm_omni/model_executor/stage_input_processors/higgs_audio_v3.py
|
||||
# - vllm_omni/deploy/higgs_multimodal_qwen3.yaml
|
||||
# commands:
|
||||
# - |
|
||||
# timeout 20m bash -c "
|
||||
# export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
# export VLLM_USE_DEEP_GEMM=0
|
||||
# export VLLM_MOE_USE_DEEP_GEMM=0
|
||||
# pytest -s -v tests/e2e/online_serving/test_higgs_audio_v3.py -m 'advanced_model' --run-level 'advanced_model'
|
||||
# "
|
||||
# mirror_hardwares: l4_1
|
||||
- label: "TTS · Higgs-Audio-V3 Test"
|
||||
source_file_dependencies:
|
||||
- tests/e2e/online_serving/test_higgs_audio_v3.py
|
||||
- vllm_omni/model_executor/models/higgs_audio_v3/
|
||||
- vllm_omni/model_executor/stage_input_processors/higgs_audio_v3.py
|
||||
- vllm_omni/deploy/higgs_multimodal_qwen3.yaml
|
||||
commands:
|
||||
- |
|
||||
timeout 20m bash -c "
|
||||
export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
export VLLM_USE_DEEP_GEMM=0
|
||||
export VLLM_MOE_USE_DEEP_GEMM=0
|
||||
pytest -s -v tests/e2e/online_serving/test_higgs_audio_v3.py -m 'advanced_model' --run-level 'advanced_model'
|
||||
"
|
||||
mirror_hardwares: l4_1
|
||||
|
||||
# - label: "Omni · Qwen3-Omni Test"
|
||||
# source_file_dependencies:
|
||||
# - tests/e2e/offline_inference/test_qwen3_omni.py
|
||||
# - tests/e2e/online_serving/test_qwen3_omni.py
|
||||
# - vllm_omni/model_executor/models/qwen3_omni/
|
||||
# - vllm_omni/model_executor/stage_input_processors/qwen3_omni.py
|
||||
# - vllm_omni/deploy/qwen3_omni_moe.yaml
|
||||
# timeout_in_minutes: 50
|
||||
# commands:
|
||||
# - pytest -s -v tests/e2e/offline_inference/test_qwen3_omni.py tests/e2e/online_serving/test_qwen3_omni.py -m 'advanced_model and cuda' --run-level 'advanced_model'
|
||||
# mirror_hardwares: h100_2
|
||||
- label: "Omni · Qwen3-Omni Test"
|
||||
source_file_dependencies:
|
||||
- tests/e2e/offline_inference/test_qwen3_omni.py
|
||||
- tests/e2e/online_serving/test_qwen3_omni.py
|
||||
- vllm_omni/model_executor/models/qwen3_omni/
|
||||
- vllm_omni/model_executor/stage_input_processors/qwen3_omni.py
|
||||
- vllm_omni/deploy/qwen3_omni_moe.yaml
|
||||
timeout_in_minutes: 50
|
||||
commands:
|
||||
- pytest -s -v tests/e2e/offline_inference/test_qwen3_omni.py tests/e2e/online_serving/test_qwen3_omni.py -m 'advanced_model and cuda' --run-level 'advanced_model'
|
||||
mirror_hardwares: h100_2
|
||||
|
||||
- label: "Omni · MiniCPM-o 4.5 Test"
|
||||
source_file_dependencies:
|
||||
- tests/e2e/offline_inference/test_minicpmo_4_5.py
|
||||
- tests/e2e/online_serving/test_minicpmo_4_5.py
|
||||
- vllm_omni/deploy/minicpmo_4_5.yaml
|
||||
timeout_in_minutes: 50
|
||||
commands:
|
||||
- pytest -s -v tests/e2e/offline_inference/test_minicpmo_4_5.py tests/e2e/online_serving/test_minicpmo_4_5.py -m 'advanced_model and cuda' --run-level 'advanced_model'
|
||||
mirror_hardwares: h100_1
|
||||
agents:
|
||||
queue: "mithril-h100-pool"
|
||||
plugins:
|
||||
- kubernetes:
|
||||
podSpec:
|
||||
containers:
|
||||
- image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
|
||||
resources:
|
||||
limits:
|
||||
nvidia.com/gpu: 2
|
||||
volumeMounts:
|
||||
- name: devshm
|
||||
mountPath: /dev/shm
|
||||
- name: hf-cache
|
||||
mountPath: /root/.cache/huggingface
|
||||
env:
|
||||
- name: HF_HOME
|
||||
value: /root/.cache/huggingface
|
||||
- name: HF_TOKEN
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: hf-token-secret
|
||||
key: token
|
||||
nodeSelector:
|
||||
node.kubernetes.io/instance-type: gpu-h100-sxm
|
||||
volumes:
|
||||
- name: devshm
|
||||
emptyDir:
|
||||
medium: Memory
|
||||
- name: hf-cache
|
||||
hostPath:
|
||||
path: /mnt/hf-cache
|
||||
type: DirectoryOrCreate
|
||||
|
||||
- label: "Omni · MiniCPM-o 4.5 Duplex Test"
|
||||
timeout_in_minutes: 50
|
||||
- label: "TTS · VoxCPM2 Test"
|
||||
source_file_dependencies:
|
||||
- tests/e2e/offline_inference/test_voxcpm2_tts.py
|
||||
- tests/e2e/online_serving/test_voxcpm2_tts.py
|
||||
- vllm_omni/model_executor/models/voxcpm2/
|
||||
- vllm_omni/deploy/voxcpm2.yaml
|
||||
timeout_in_minutes: 30
|
||||
commands:
|
||||
- pytest -s -v tests/e2e/online_serving/test_minicpmo_4_5_duplex.py -m 'advanced_model and cuda' --run-level 'advanced_model'
|
||||
- |
|
||||
timeout 30m bash -c "
|
||||
export VLLM_LOGGING_LEVEL=DEBUG
|
||||
export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
pytest -s -v tests/e2e/offline_inference/test_voxcpm2_tts.py tests/e2e/online_serving/test_voxcpm2_tts.py -m 'advanced_model and cuda' --run-level 'advanced_model'
|
||||
"
|
||||
mirror_hardwares: l4_1
|
||||
|
||||
- label: "TTS · SoulX-Singer Test"
|
||||
source_file_dependencies:
|
||||
- tests/e2e/offline_inference/test_soulxsinger.py
|
||||
- vllm_omni/diffusion/models/soulx_singer/
|
||||
- vllm_omni/deploy/soulxsinger_svc.yaml
|
||||
- vllm_omni/deploy/soulxsinger_svs.yaml
|
||||
timeout_in_minutes: 30
|
||||
commands:
|
||||
- |
|
||||
timeout 30m bash -c "
|
||||
export VLLM_LOGGING_LEVEL=DEBUG
|
||||
export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
pytest -s -v tests/e2e/offline_inference/test_soulxsinger.py -k 'svc or precomputed' -m 'advanced_model and cuda' --run-level 'advanced_model'
|
||||
"
|
||||
mirror_hardwares: l4_1
|
||||
|
||||
- label: "Diffusion · Qwen Image Edit Test"
|
||||
source_file_dependencies:
|
||||
- tests/e2e/online_serving/test_qwen_image_edit.py
|
||||
- vllm_omni/diffusion/models/qwen_image/
|
||||
timeout_in_minutes: 20
|
||||
commands:
|
||||
- pytest -s -v tests/e2e/online_serving/test_qwen_image_edit.py -m 'advanced_model and cuda' --run-level 'advanced_model'
|
||||
mirror_hardwares: h100_1
|
||||
- label: "Diffusion · Bagel Test"
|
||||
source_file_dependencies:
|
||||
- tests/e2e/offline_inference/test_bagel.py
|
||||
- tests/e2e/online_serving/test_bagel.py
|
||||
- vllm_omni/model_executor/models/bagel/
|
||||
- vllm_omni/diffusion/models/bagel/
|
||||
- vllm_omni/model_executor/stage_input_processors/bagel.py
|
||||
- vllm_omni/deploy/bagel.yaml
|
||||
timeout_in_minutes: 20
|
||||
commands:
|
||||
- pytest -s -v tests/e2e/offline_inference/test_bagel.py tests/e2e/online_serving/test_bagel.py -m 'advanced_model and cuda' --run-level 'advanced_model'
|
||||
mirror_hardwares: h100_1
|
||||
|
||||
# - label: "TTS · VoxCPM2 Test"
|
||||
# source_file_dependencies:
|
||||
# - tests/e2e/offline_inference/test_voxcpm2_tts.py
|
||||
# - tests/e2e/online_serving/test_voxcpm2_tts.py
|
||||
# - vllm_omni/model_executor/models/voxcpm2/
|
||||
# - vllm_omni/deploy/voxcpm2.yaml
|
||||
# timeout_in_minutes: 30
|
||||
# commands:
|
||||
# - |
|
||||
# timeout 30m bash -c "
|
||||
# export VLLM_LOGGING_LEVEL=DEBUG
|
||||
# export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
# pytest -s -v tests/e2e/offline_inference/test_voxcpm2_tts.py tests/e2e/online_serving/test_voxcpm2_tts.py -m 'advanced_model and cuda' --run-level 'advanced_model'
|
||||
# "
|
||||
# mirror_hardwares: l4_1
|
||||
- label: "Diffusion · Qwen Image Test"
|
||||
source_file_dependencies:
|
||||
- tests/e2e/online_serving/test_qwen_image.py
|
||||
- vllm_omni/diffusion/models/qwen_image/
|
||||
commands:
|
||||
- |
|
||||
timeout 40m bash -c '
|
||||
export VLLM_IMAGE_FETCH_TIMEOUT=60
|
||||
pytest -s -v tests/e2e/online_serving/test_qwen_image.py -m 'advanced_model and cuda' --run-level 'advanced_model'
|
||||
'
|
||||
mirror_hardwares: h100_1
|
||||
|
||||
# - label: "TTS · SoulX-Singer Test"
|
||||
# source_file_dependencies:
|
||||
# - tests/e2e/offline_inference/test_soulxsinger.py
|
||||
# - vllm_omni/diffusion/models/soulx_singer/
|
||||
# - vllm_omni/deploy/soulxsinger_svc.yaml
|
||||
# - vllm_omni/deploy/soulxsinger_svs.yaml
|
||||
# timeout_in_minutes: 30
|
||||
# commands:
|
||||
# - |
|
||||
# timeout 30m bash -c "
|
||||
# export VLLM_LOGGING_LEVEL=DEBUG
|
||||
# export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
# pytest -s -v tests/e2e/offline_inference/test_soulxsinger.py -k 'svc or precomputed' -m 'advanced_model and cuda' --run-level 'advanced_model'
|
||||
# "
|
||||
# mirror_hardwares: l4_1
|
||||
- label: "Diffusion · Qwen Image Layered Test"
|
||||
source_file_dependencies:
|
||||
- tests/e2e/online_serving/test_qwen_image_layered.py
|
||||
- vllm_omni/diffusion/models/qwen_image/
|
||||
commands:
|
||||
- |
|
||||
timeout 40m bash -c "
|
||||
export VLLM_IMAGE_FETCH_TIMEOUT=60
|
||||
pytest -s -v tests/e2e/online_serving/test_qwen_image_layered.py -m 'advanced_model and cuda' --run-level 'advanced_model'
|
||||
"
|
||||
mirror_hardwares: h100_1
|
||||
|
||||
# - label: "Diffusion · Qwen Image Edit Test"
|
||||
# source_file_dependencies:
|
||||
# - tests/e2e/online_serving/test_qwen_image_edit.py
|
||||
# - vllm_omni/diffusion/models/qwen_image/
|
||||
# timeout_in_minutes: 20
|
||||
# commands:
|
||||
# - pytest -s -v tests/e2e/online_serving/test_qwen_image_edit.py -m 'advanced_model and cuda' --run-level 'advanced_model'
|
||||
# mirror_hardwares: h100_1
|
||||
# - label: "Diffusion · Bagel Test"
|
||||
# source_file_dependencies:
|
||||
# - tests/e2e/offline_inference/test_bagel.py
|
||||
# - tests/e2e/online_serving/test_bagel.py
|
||||
# - vllm_omni/model_executor/models/bagel/
|
||||
# - vllm_omni/diffusion/models/bagel/
|
||||
# - vllm_omni/model_executor/stage_input_processors/bagel.py
|
||||
# - vllm_omni/deploy/bagel.yaml
|
||||
# timeout_in_minutes: 20
|
||||
# commands:
|
||||
# - pytest -s -v tests/e2e/offline_inference/test_bagel.py tests/e2e/online_serving/test_bagel.py -m 'advanced_model and cuda' --run-level 'advanced_model'
|
||||
# mirror_hardwares: h100_1
|
||||
- label: "Diffusion · Wan22 Test"
|
||||
source_file_dependencies:
|
||||
- tests/e2e/offline_inference/test_wan22_t2v.py
|
||||
- tests/e2e/online_serving/test_wan22_t2v.py
|
||||
- vllm_omni/diffusion/models/wan2_2/
|
||||
commands:
|
||||
- |
|
||||
timeout 40m bash -c "
|
||||
export VLLM_IMAGE_FETCH_TIMEOUT=60
|
||||
pytest -s -v tests/e2e/offline_inference/test_wan22_t2v.py tests/e2e/online_serving/test_wan22_t2v.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model'
|
||||
"
|
||||
mirror_hardwares: h100_1
|
||||
|
||||
# - label: "Diffusion · Qwen Image Test"
|
||||
# source_file_dependencies:
|
||||
# - tests/e2e/online_serving/test_qwen_image.py
|
||||
# - vllm_omni/diffusion/models/qwen_image/
|
||||
# commands:
|
||||
# - |
|
||||
# timeout 40m bash -c '
|
||||
# export VLLM_IMAGE_FETCH_TIMEOUT=60
|
||||
# pytest -s -v tests/e2e/online_serving/test_qwen_image.py -m 'advanced_model and cuda' --run-level 'advanced_model'
|
||||
# '
|
||||
# mirror_hardwares: h100_1
|
||||
|
||||
# - label: "Diffusion · Qwen Image Layered Test"
|
||||
# source_file_dependencies:
|
||||
# - tests/e2e/online_serving/test_qwen_image_layered.py
|
||||
# - vllm_omni/diffusion/models/qwen_image/
|
||||
# commands:
|
||||
# - |
|
||||
# timeout 40m bash -c "
|
||||
# export VLLM_IMAGE_FETCH_TIMEOUT=60
|
||||
# pytest -s -v tests/e2e/online_serving/test_qwen_image_layered.py -m 'advanced_model and cuda' --run-level 'advanced_model'
|
||||
# "
|
||||
# mirror_hardwares: h100_1
|
||||
|
||||
# - label: "Diffusion · Wan22 Test"
|
||||
# source_file_dependencies:
|
||||
# - tests/e2e/offline_inference/test_wan22_t2v.py
|
||||
# - tests/e2e/online_serving/test_wan22_t2v.py
|
||||
# - vllm_omni/diffusion/models/wan2_2/
|
||||
# commands:
|
||||
# - |
|
||||
# timeout 40m bash -c "
|
||||
# export VLLM_IMAGE_FETCH_TIMEOUT=60
|
||||
# pytest -s -v tests/e2e/offline_inference/test_wan22_t2v.py tests/e2e/online_serving/test_wan22_t2v.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model'
|
||||
# "
|
||||
# mirror_hardwares: h100_1
|
||||
|
||||
# - label: "Diffusion · Z Image Test"
|
||||
# source_file_dependencies:
|
||||
# - tests/e2e/offline_inference/test_z_image.py
|
||||
# - tests/e2e/online_serving/test_z_image.py
|
||||
# - vllm_omni/diffusion/models/z_image/
|
||||
# timeout_in_minutes: 30
|
||||
# commands:
|
||||
# - pytest -s -v tests/e2e/offline_inference/test_z_image.py tests/e2e/online_serving/test_z_image.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model'
|
||||
# mirror_hardwares: l4_1
|
||||
# - label: "Diffusion · FLUX.2 Klein Test"
|
||||
# source_file_dependencies:
|
||||
# - tests/e2e/offline_inference/test_flux2_klein.py
|
||||
# - vllm_omni/diffusion/models/flux2_klein/
|
||||
# timeout_in_minutes: 30
|
||||
# commands:
|
||||
# - pytest -s -v tests/e2e/offline_inference/test_flux2_klein.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model'
|
||||
# mirror_hardwares: l4_1
|
||||
- label: "Diffusion · Z Image Test"
|
||||
source_file_dependencies:
|
||||
- tests/e2e/offline_inference/test_z_image.py
|
||||
- tests/e2e/online_serving/test_z_image.py
|
||||
- vllm_omni/diffusion/models/z_image/
|
||||
timeout_in_minutes: 30
|
||||
commands:
|
||||
- pytest -s -v tests/e2e/offline_inference/test_z_image.py tests/e2e/online_serving/test_z_image.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model'
|
||||
mirror_hardwares: l4_1
|
||||
- label: "Diffusion · FLUX.2 Klein Test"
|
||||
source_file_dependencies:
|
||||
- tests/e2e/offline_inference/test_flux2_klein.py
|
||||
- vllm_omni/diffusion/models/flux2_klein/
|
||||
timeout_in_minutes: 30
|
||||
commands:
|
||||
- pytest -s -v tests/e2e/offline_inference/test_flux2_klein.py -m 'advanced_model and cuda and diffusion' --run-level 'advanced_model'
|
||||
mirror_hardwares: l4_1
|
||||
+449
-496
File diff suppressed because it is too large.
Load diff
+289
-252
@@ -18,302 +18,339 @@ steps:
|
||||
- "timeout 20m pytest -sv -m 'core_model and cpu' --ignore=tests/diffusion --ignore=tests/model_executor --cov=vllm_omni --cov-report=term-missing:skip-covered --cov-report=xml"
|
||||
mirror_hardwares: l4_1
|
||||
|
||||
# - label: "Custom Pipeline Test"
|
||||
# depends_on: upload-ready-pipeline
|
||||
# commands:
|
||||
# - timeout 20m pytest -s -v tests/e2e/offline_inference/custom_pipeline/ -m "core_model"
|
||||
# mirror_hardwares: l4_1
|
||||
- label: "Custom Pipeline Test"
|
||||
depends_on: upload-ready-pipeline
|
||||
commands:
|
||||
- timeout 20m pytest -s -v tests/e2e/offline_inference/custom_pipeline/ -m "core_model"
|
||||
mirror_hardwares: l4_1
|
||||
|
||||
# - group: ":card_index_dividers: Diffusion Test"
|
||||
# depends_on: upload-ready-pipeline
|
||||
# steps:
|
||||
- group: ":card_index_dividers: Diffusion Test"
|
||||
depends_on: upload-ready-pipeline
|
||||
steps:
|
||||
|
||||
# - label: "Diffusion · Offloader Test"
|
||||
# commands:
|
||||
# - timeout 40m pytest -s -v tests/diffusion/offloader -m "core_model and cuda" --run-level "core_model"
|
||||
# mirror_hardwares: l4_1
|
||||
- label: "Diffusion · Offloader Test"
|
||||
commands:
|
||||
- timeout 40m pytest -s -v tests/diffusion/offloader -m "core_model and cuda" --run-level "core_model"
|
||||
mirror_hardwares: l4_1
|
||||
|
||||
# - label: "Diffusion · Cache Test"
|
||||
# commands:
|
||||
# - timeout 15m pytest -sv tests/diffusion/cache -m 'core_model and cuda' --run-level "core_model"
|
||||
# mirror_hardwares: l4_1
|
||||
- label: "Diffusion · Cache Test"
|
||||
commands:
|
||||
- timeout 15m pytest -sv tests/diffusion/cache -m 'core_model and cuda' --run-level "core_model"
|
||||
mirror_hardwares: l4_1
|
||||
|
||||
# - label: "Diffusion · Batch Test"
|
||||
# commands:
|
||||
# - timeout 15m pytest -sv tests/diffusion/batching -m 'core_model and cuda' --run-level "core_model"
|
||||
# mirror_hardwares: l4_1
|
||||
- label: "Diffusion · Batch Test"
|
||||
commands:
|
||||
- timeout 15m pytest -sv tests/diffusion/batching -m 'core_model and cuda' --run-level "core_model"
|
||||
mirror_hardwares: l4_1
|
||||
|
||||
# - label: "Diffusion · Distributed Test"
|
||||
# source_file_dependencies:
|
||||
# - tests/diffusion/distributed/
|
||||
# - vllm_omni/diffusion/distributed/
|
||||
# - vllm_omni/diffusion/attention/parallel/
|
||||
# - vllm_omni/diffusion/data.py
|
||||
# - vllm_omni/diffusion/models/
|
||||
# commands:
|
||||
# - timeout 20m pytest -sv tests/diffusion/distributed -m 'core_model and cuda' --run-level "core_model"
|
||||
# mirror_hardwares: l4_4
|
||||
- label: "Diffusion · Distributed Test"
|
||||
source_file_dependencies:
|
||||
- tests/diffusion/distributed/
|
||||
- vllm_omni/diffusion/distributed/
|
||||
- vllm_omni/diffusion/attention/parallel/
|
||||
- vllm_omni/diffusion/data.py
|
||||
- vllm_omni/diffusion/models/
|
||||
commands:
|
||||
- timeout 20m pytest -sv tests/diffusion/distributed -m 'core_model and cuda' --run-level "core_model"
|
||||
mirror_hardwares: l4_4
|
||||
|
||||
# - label: "Diffusion · Model Test"
|
||||
# commands:
|
||||
# - timeout 15m pytest -sv tests/diffusion/models/ -m 'core_model and cuda' --run-level "core_model"
|
||||
# mirror_hardwares: l4_1
|
||||
- label: "Diffusion · Model Test"
|
||||
commands:
|
||||
- timeout 15m pytest -sv tests/diffusion/models/ -m 'core_model and cuda' --run-level "core_model"
|
||||
mirror_hardwares: l4_1
|
||||
|
||||
# - label: "Diffusion · Other Test"
|
||||
# commands:
|
||||
# - timeout 15m pytest -sv tests/diffusion/*.py -m 'core_model and cuda' --run-level "core_model"
|
||||
# mirror_hardwares: l4_1
|
||||
- label: "Diffusion · Other Test"
|
||||
commands:
|
||||
- timeout 15m pytest -sv tests/diffusion/*.py -m 'core_model and cuda' --run-level "core_model"
|
||||
mirror_hardwares: l4_1
|
||||
|
||||
# - group: ":card_index_dividers: Tiny Model Tests [Single GPU]"
|
||||
# depends_on: upload-ready-pipeline
|
||||
# steps:
|
||||
- group: ":card_index_dividers: Tiny Model Tests [Single GPU]"
|
||||
depends_on: upload-ready-pipeline
|
||||
steps:
|
||||
|
||||
# - label: "Tiny Model Tests · Diffusion (base)"
|
||||
# commands:
|
||||
# - timeout 15m pytest -sv tests/model_tests/diffusion -m 'core_model and cuda' --run-level "core_model"
|
||||
# mirror_hardwares: l4_1
|
||||
- label: "Tiny Model Tests · Diffusion (base)"
|
||||
commands:
|
||||
- timeout 15m pytest -sv tests/model_tests/diffusion -m 'core_model and cuda' --run-level "core_model"
|
||||
mirror_hardwares: l4_1
|
||||
|
||||
# - label: "Engine&Model Executor Test"
|
||||
# depends_on: upload-ready-pipeline
|
||||
# commands:
|
||||
# - |
|
||||
# timeout 15m bash -c "
|
||||
# pytest -sv tests/engine/ tests/model_executor/ -m 'core_model and cuda' --run-level 'core_model'
|
||||
# "
|
||||
# mirror_hardwares: l4_1
|
||||
- label: "Engine&Model Executor Test"
|
||||
depends_on: upload-ready-pipeline
|
||||
commands:
|
||||
- |
|
||||
timeout 15m bash -c "
|
||||
pytest -sv tests/engine/ tests/model_executor/ -m 'core_model and cuda' --run-level 'core_model'
|
||||
"
|
||||
mirror_hardwares: l4_1
|
||||
|
||||
# - label: "Distributed Test"
|
||||
# depends_on: upload-ready-pipeline
|
||||
# commands:
|
||||
# - |
|
||||
# timeout 20m bash -c "
|
||||
# pytest -sv tests/distributed/ -m 'core_model and cuda and L4' --run-level 'core_model'
|
||||
# "
|
||||
# mirror_hardwares: l4_1
|
||||
- label: "Distributed Test"
|
||||
depends_on: upload-ready-pipeline
|
||||
commands:
|
||||
- |
|
||||
timeout 20m bash -c "
|
||||
pytest -sv tests/distributed/ -m 'core_model and cuda and L4' --run-level 'core_model'
|
||||
"
|
||||
mirror_hardwares: l4_1
|
||||
|
||||
# - label: "Entrypoints Test"
|
||||
# depends_on: upload-ready-pipeline
|
||||
# commands:
|
||||
# - |
|
||||
# timeout 30m bash -c "
|
||||
# export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
# pytest -sv tests/entrypoints/ -m 'core_model and cuda' --run-level 'core_model'
|
||||
# "
|
||||
# mirror_hardwares: l4_1
|
||||
- label: "Entrypoints Test"
|
||||
depends_on: upload-ready-pipeline
|
||||
commands:
|
||||
- |
|
||||
timeout 30m bash -c "
|
||||
export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
pytest -sv tests/entrypoints/ -m 'core_model and cuda' --run-level 'core_model'
|
||||
"
|
||||
mirror_hardwares: l4_1
|
||||
|
||||
- group: ":card_index_dividers: E2E Test"
|
||||
depends_on: upload-ready-pipeline
|
||||
steps:
|
||||
|
||||
# - label: "RLHF · VeRL-Omni E2E Test"
|
||||
# source_file_dependencies:
|
||||
# - tests/e2e/offline_inference/rlhf_test/test_verl_omni_e2e.py
|
||||
# - tests/e2e/offline_inference/custom_pipeline/
|
||||
# - vllm_omni/diffusion/
|
||||
# - vllm_omni/engine/
|
||||
# - vllm_omni/entrypoints/
|
||||
# - vllm_omni/inputs/
|
||||
# - vllm_omni/outputs/
|
||||
# commands:
|
||||
# - |
|
||||
# timeout 20m bash -c "
|
||||
# pytest -s -v tests/e2e/offline_inference/rlhf_test/test_verl_omni_e2e.py
|
||||
# "
|
||||
# mirror_hardwares: l4_1
|
||||
- label: "RLHF · VeRL-Omni E2E Test"
|
||||
source_file_dependencies:
|
||||
- tests/e2e/offline_inference/rlhf_test/test_verl_omni_e2e.py
|
||||
- tests/e2e/offline_inference/custom_pipeline/
|
||||
- vllm_omni/diffusion/
|
||||
- vllm_omni/engine/
|
||||
- vllm_omni/entrypoints/
|
||||
- vllm_omni/inputs/
|
||||
- vllm_omni/outputs/
|
||||
commands:
|
||||
- |
|
||||
timeout 20m bash -c "
|
||||
pytest -s -v tests/e2e/offline_inference/rlhf_test/test_verl_omni_e2e.py
|
||||
"
|
||||
mirror_hardwares: l4_1
|
||||
|
||||
# - label: "Omni · Qwen3-Omni Test"
|
||||
# source_file_dependencies:
|
||||
# - tests/e2e/online_serving/test_qwen3_omni.py
|
||||
# - vllm_omni/model_executor/models/qwen3_omni/
|
||||
# - vllm_omni/model_executor/stage_input_processors/qwen3_omni.py
|
||||
# - vllm_omni/deploy/qwen3_omni_moe.yaml
|
||||
# commands:
|
||||
# - |
|
||||
# timeout 20m bash -c "
|
||||
# pytest -s -v tests/e2e/online_serving/test_qwen3_omni.py -m 'core_model' --run-level 'core_model'
|
||||
# "
|
||||
# mirror_hardwares: h100_2
|
||||
- label: "Omni · Qwen3-Omni Test"
|
||||
source_file_dependencies:
|
||||
- tests/e2e/online_serving/test_qwen3_omni.py
|
||||
- vllm_omni/model_executor/models/qwen3_omni/
|
||||
- vllm_omni/model_executor/stage_input_processors/qwen3_omni.py
|
||||
- vllm_omni/deploy/qwen3_omni_moe.yaml
|
||||
commands:
|
||||
- |
|
||||
timeout 20m bash -c "
|
||||
pytest -s -v tests/e2e/online_serving/test_qwen3_omni.py -m 'core_model' --run-level 'core_model'
|
||||
"
|
||||
mirror_hardwares: h100_2
|
||||
|
||||
- label: "Omni · MiniCPM-o 4.5 Offline Test"
|
||||
source_file_dependencies:
|
||||
- tests/e2e/offline_inference/test_minicpmo_4_5.py
|
||||
- vllm_omni/deploy/minicpmo_4_5.yaml
|
||||
commands:
|
||||
- |
|
||||
timeout 20m bash -c "
|
||||
pytest -s -v tests/e2e/offline_inference/test_minicpmo_4_5.py -m 'core_model' --run-level 'core_model'
|
||||
"
|
||||
mirror_hardwares: h100_1
|
||||
agents:
|
||||
queue: "mithril-h100-pool"
|
||||
plugins:
|
||||
- kubernetes:
|
||||
podSpec:
|
||||
containers:
|
||||
- image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
|
||||
resources:
|
||||
limits:
|
||||
nvidia.com/gpu: 2
|
||||
volumeMounts:
|
||||
- name: devshm
|
||||
mountPath: /dev/shm
|
||||
- name: hf-cache
|
||||
mountPath: /root/.cache/huggingface
|
||||
env:
|
||||
- name: HF_HOME
|
||||
value: /root/.cache/huggingface
|
||||
nodeSelector:
|
||||
node.kubernetes.io/instance-type: gpu-h100-sxm
|
||||
volumes:
|
||||
- name: devshm
|
||||
emptyDir:
|
||||
medium: Memory
|
||||
- name: hf-cache
|
||||
hostPath:
|
||||
path: /mnt/hf-cache
|
||||
type: DirectoryOrCreate
|
||||
|
||||
- label: "Omni · MiniCPM-o 4.5 Online Test"
|
||||
source_file_dependencies:
|
||||
- tests/e2e/online_serving/test_minicpmo_4_5.py
|
||||
- vllm_omni/deploy/minicpmo_4_5.yaml
|
||||
commands:
|
||||
- |
|
||||
timeout 20m bash -c "
|
||||
pytest -s -v tests/e2e/online_serving/test_minicpmo_4_5.py -m 'core_model' --run-level 'core_model'
|
||||
"
|
||||
mirror_hardwares: h100_1
|
||||
agents:
|
||||
queue: "mithril-h100-pool"
|
||||
plugins:
|
||||
- kubernetes:
|
||||
podSpec:
|
||||
containers:
|
||||
- image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
|
||||
resources:
|
||||
limits:
|
||||
nvidia.com/gpu: 2
|
||||
volumeMounts:
|
||||
- name: devshm
|
||||
mountPath: /dev/shm
|
||||
- name: hf-cache
|
||||
mountPath: /root/.cache/huggingface
|
||||
env:
|
||||
- name: HF_HOME
|
||||
value: /root/.cache/huggingface
|
||||
nodeSelector:
|
||||
node.kubernetes.io/instance-type: gpu-h100-sxm
|
||||
volumes:
|
||||
- name: devshm
|
||||
emptyDir:
|
||||
medium: Memory
|
||||
- name: hf-cache
|
||||
hostPath:
|
||||
path: /mnt/hf-cache
|
||||
type: DirectoryOrCreate
|
||||
|
||||
- label: "Omni · MiniCPM-o 4.5 Duplex Test"
|
||||
- label: "TTS · VoxCPM2 Test"
|
||||
source_file_dependencies:
|
||||
- tests/e2e/online_serving/test_voxcpm2_tts.py
|
||||
- vllm_omni/model_executor/models/voxcpm2/
|
||||
- vllm_omni/deploy/voxcpm2.yaml
|
||||
timeout_in_minutes: 30
|
||||
commands:
|
||||
- |
|
||||
timeout 30m bash -c "
|
||||
export VLLM_LOGGING_LEVEL=DEBUG
|
||||
export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
pytest -s -v tests/e2e/online_serving/test_voxcpm2_tts.py -m 'core_model' --run-level 'core_model'
|
||||
"
|
||||
mirror_hardwares: l4_1
|
||||
|
||||
- label: "TTS · Qwen3-TTS CustomVoice Test"
|
||||
source_file_dependencies:
|
||||
- tests/e2e/online_serving/test_qwen3_tts_customvoice.py
|
||||
- vllm_omni/model_executor/models/qwen3_tts/
|
||||
- vllm_omni/model_executor/stage_input_processors/qwen3_tts.py
|
||||
- vllm_omni/deploy/qwen3_tts.yaml
|
||||
commands:
|
||||
- |
|
||||
timeout 20m bash -c "
|
||||
pytest -s -v tests/e2e/online_serving/test_minicpmo_4_5_duplex.py -m 'core_model and cuda' --run-level 'core_model'
|
||||
export VLLM_ALLOW_LONG_MAX_MODEL_LEN=1
|
||||
pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py -m 'core_model' --run-level 'core_model'
|
||||
"
|
||||
mirror_hardwares: l4_1
|
||||
|
||||
- label: "TTS · Qwen3-TTS Base Test"
|
||||
source_file_dependencies:
|
||||
- tests/e2e/online_serving/test_qwen3_tts_base.py
|
||||
- vllm_omni/model_executor/models/qwen3_tts/
|
||||
- vllm_omni/model_executor/stage_input_processors/qwen3_tts.py
|
||||
- vllm_omni/deploy/qwen3_tts.yaml
|
||||
commands:
|
||||
- |
|
||||
timeout 20m bash -c "
|
||||
export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
export VLLM_ALLOW_LONG_MAX_MODEL_LEN=1
|
||||
export VLLM_OMNI_USE_V2_RUNNER=1
|
||||
pytest -s -v tests/e2e/online_serving/test_qwen3_tts_base.py -m 'core_model' --run-level 'core_model'
|
||||
"
|
||||
mirror_hardwares: l4_1
|
||||
|
||||
- label: "TTS · Higgs-Audio-V3 Test"
|
||||
source_file_dependencies:
|
||||
- tests/e2e/online_serving/test_higgs_audio_v3.py
|
||||
- vllm_omni/model_executor/models/higgs_audio_v3/
|
||||
- vllm_omni/model_executor/stage_input_processors/higgs_audio_v3.py
|
||||
- vllm_omni/deploy/higgs_multimodal_qwen3.yaml
|
||||
commands:
|
||||
- |
|
||||
timeout 20m bash -c "
|
||||
export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
export VLLM_USE_DEEP_GEMM=0
|
||||
export VLLM_MOE_USE_DEEP_GEMM=0
|
||||
pytest -s -v tests/e2e/online_serving/test_higgs_audio_v3.py -m 'core_model' --run-level 'core_model'
|
||||
"
|
||||
mirror_hardwares: l4_1
|
||||
|
||||
- label: "Diffusion · Qwen Image Edit Test"
|
||||
source_file_dependencies:
|
||||
- tests/e2e/online_serving/test_qwen_image_edit.py
|
||||
- vllm_omni/diffusion/models/qwen_image/
|
||||
commands:
|
||||
- |
|
||||
timeout 20m bash -c "
|
||||
pytest -s -v tests/e2e/online_serving/test_qwen_image_edit.py -m 'core_model' --run-level 'core_model'
|
||||
"
|
||||
mirror_hardwares: h100_1
|
||||
|
||||
- label: "Omni · MiniCPM-o 4.5 Perf Test"
|
||||
timeout_in_minutes: 60
|
||||
- label: "Diffusion · Bagel Test"
|
||||
source_file_dependencies:
|
||||
- tests/dfx/perf/tests/test_minicpmo_4_5_ready.json
|
||||
- vllm_omni/deploy/minicpmo_4_5.yaml
|
||||
- vllm_omni/model_executor/models/minicpmo_4_5/
|
||||
- vllm_omni/model_executor/models/step_audio2/
|
||||
- vllm_omni/model_executor/models/cosyvoice3/code2wav_core/
|
||||
- vllm_omni/model_executor/models/cosyvoice3/utils.py
|
||||
- vllm_omni/model_executor/stage_input_processors/minicpmo_4_5_omni.py
|
||||
- tests/e2e/online_serving/test_bagel.py
|
||||
- vllm_omni/model_executor/models/bagel/
|
||||
- vllm_omni/diffusion/models/bagel/
|
||||
- vllm_omni/model_executor/stage_input_processors/bagel.py
|
||||
- vllm_omni/deploy/bagel.yaml
|
||||
commands:
|
||||
- export BENCHMARK_DIR=tests/dfx/perf/results
|
||||
- pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_minicpmo_4_5_ready.json -m 'core_model and cuda'
|
||||
- |
|
||||
timeout 40m bash -c "
|
||||
export VLLM_IMAGE_FETCH_TIMEOUT=60
|
||||
pytest -s -v tests/e2e/online_serving/test_bagel.py -m 'core_model' --run-level 'core_model'
|
||||
"
|
||||
mirror_hardwares: h100_1
|
||||
|
||||
# - label: "TTS · VoxCPM2 Test"
|
||||
# source_file_dependencies:
|
||||
# - tests/e2e/online_serving/test_voxcpm2_tts.py
|
||||
# - vllm_omni/model_executor/models/voxcpm2/
|
||||
# - vllm_omni/deploy/voxcpm2.yaml
|
||||
# timeout_in_minutes: 30
|
||||
# commands:
|
||||
# - |
|
||||
# timeout 30m bash -c "
|
||||
# export VLLM_LOGGING_LEVEL=DEBUG
|
||||
# export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
# pytest -s -v tests/e2e/online_serving/test_voxcpm2_tts.py -m 'core_model' --run-level 'core_model'
|
||||
# "
|
||||
# mirror_hardwares: l4_1
|
||||
- label: "Diffusion · Wan22 Test"
|
||||
source_file_dependencies:
|
||||
- tests/e2e/online_serving/test_wan22_t2v.py
|
||||
- vllm_omni/diffusion/models/wan2_2/
|
||||
commands:
|
||||
- |
|
||||
timeout 40m bash -c "
|
||||
export VLLM_IMAGE_FETCH_TIMEOUT=60
|
||||
pytest -s -v tests/e2e/online_serving/test_wan22_t2v.py -m 'core_model' --run-level 'core_model'
|
||||
"
|
||||
mirror_hardwares: h100_1
|
||||
|
||||
# - label: "TTS · Qwen3-TTS CustomVoice Test"
|
||||
# source_file_dependencies:
|
||||
# - tests/e2e/online_serving/test_qwen3_tts_customvoice.py
|
||||
# - vllm_omni/model_executor/models/qwen3_tts/
|
||||
# - vllm_omni/model_executor/stage_input_processors/qwen3_tts.py
|
||||
# - vllm_omni/deploy/qwen3_tts.yaml
|
||||
# commands:
|
||||
# - |
|
||||
# timeout 20m bash -c "
|
||||
# export VLLM_ALLOW_LONG_MAX_MODEL_LEN=1
|
||||
# pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py -m 'core_model' --run-level 'core_model'
|
||||
# "
|
||||
# mirror_hardwares: l4_1
|
||||
- label: "Diffusion · Cosmos3 Test"
|
||||
source_file_dependencies:
|
||||
- tests/e2e/online_serving/test_cosmos3.py
|
||||
- vllm_omni/diffusion/models/cosmos3/
|
||||
commands:
|
||||
- |
|
||||
timeout 40m bash -c "
|
||||
export VLLM_IMAGE_FETCH_TIMEOUT=60
|
||||
pytest -s -v tests/e2e/online_serving/test_cosmos3.py -m 'core_model' --run-level 'core_model'
|
||||
"
|
||||
mirror_hardwares: h100_1
|
||||
|
||||
# - label: "TTS · Qwen3-TTS Base Test"
|
||||
# source_file_dependencies:
|
||||
# - tests/e2e/online_serving/test_qwen3_tts_base.py
|
||||
# - vllm_omni/model_executor/models/qwen3_tts/
|
||||
# - vllm_omni/model_executor/stage_input_processors/qwen3_tts.py
|
||||
# - vllm_omni/deploy/qwen3_tts.yaml
|
||||
# commands:
|
||||
# - |
|
||||
# timeout 20m bash -c "
|
||||
# export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
# export VLLM_ALLOW_LONG_MAX_MODEL_LEN=1
|
||||
# export VLLM_OMNI_USE_V2_RUNNER=1
|
||||
# pytest -s -v tests/e2e/online_serving/test_qwen3_tts_base.py -m 'core_model' --run-level 'core_model'
|
||||
# "
|
||||
# mirror_hardwares: l4_1
|
||||
- label: "Diffusion · Qwen Image Test"
|
||||
source_file_dependencies:
|
||||
- tests/e2e/online_serving/test_qwen_image.py
|
||||
- vllm_omni/diffusion/models/qwen_image/
|
||||
commands:
|
||||
- |
|
||||
timeout 40m bash -c "
|
||||
export VLLM_IMAGE_FETCH_TIMEOUT=60
|
||||
pytest -s -v tests/e2e/online_serving/test_qwen_image.py -m 'core_model' --run-level 'core_model'
|
||||
"
|
||||
mirror_hardwares: h100_1
|
||||
|
||||
# - label: "TTS · Higgs-Audio-V3 Test"
|
||||
# source_file_dependencies:
|
||||
# - tests/e2e/online_serving/test_higgs_audio_v3.py
|
||||
# - vllm_omni/model_executor/models/higgs_audio_v3/
|
||||
# - vllm_omni/model_executor/stage_input_processors/higgs_audio_v3.py
|
||||
# - vllm_omni/deploy/higgs_multimodal_qwen3.yaml
|
||||
# commands:
|
||||
# - |
|
||||
# timeout 20m bash -c "
|
||||
# export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
# export VLLM_USE_DEEP_GEMM=0
|
||||
# export VLLM_MOE_USE_DEEP_GEMM=0
|
||||
# pytest -s -v tests/e2e/online_serving/test_higgs_audio_v3.py -m 'core_model' --run-level 'core_model'
|
||||
# "
|
||||
# mirror_hardwares: l4_1
|
||||
- label: "Diffusion · Qwen Image Layered Test"
|
||||
source_file_dependencies:
|
||||
- tests/e2e/online_serving/test_qwen_image_layered.py
|
||||
- vllm_omni/diffusion/models/qwen_image/
|
||||
commands:
|
||||
- |
|
||||
timeout 40m bash -c "
|
||||
export VLLM_IMAGE_FETCH_TIMEOUT=60
|
||||
pytest -s -v tests/e2e/online_serving/test_qwen_image_layered.py -m 'core_model' --run-level 'core_model'
|
||||
"
|
||||
mirror_hardwares: h100_1
|
||||
|
||||
# - label: "Diffusion · Qwen Image Edit Test"
|
||||
# source_file_dependencies:
|
||||
# - tests/e2e/online_serving/test_qwen_image_edit.py
|
||||
# - vllm_omni/diffusion/models/qwen_image/
|
||||
# commands:
|
||||
# - |
|
||||
# timeout 20m bash -c "
|
||||
# pytest -s -v tests/e2e/online_serving/test_qwen_image_edit.py -m 'core_model' --run-level 'core_model'
|
||||
# "
|
||||
# mirror_hardwares: h100_1
|
||||
|
||||
# - label: "Diffusion · Bagel Test"
|
||||
# source_file_dependencies:
|
||||
# - tests/e2e/online_serving/test_bagel.py
|
||||
# - vllm_omni/model_executor/models/bagel/
|
||||
# - vllm_omni/diffusion/models/bagel/
|
||||
# - vllm_omni/model_executor/stage_input_processors/bagel.py
|
||||
# - vllm_omni/deploy/bagel.yaml
|
||||
# commands:
|
||||
# - |
|
||||
# timeout 40m bash -c "
|
||||
# export VLLM_IMAGE_FETCH_TIMEOUT=60
|
||||
# pytest -s -v tests/e2e/online_serving/test_bagel.py -m 'core_model' --run-level 'core_model'
|
||||
# "
|
||||
# mirror_hardwares: h100_1
|
||||
|
||||
# - label: "Diffusion · Wan22 Test"
|
||||
# source_file_dependencies:
|
||||
# - tests/e2e/online_serving/test_wan22_t2v.py
|
||||
# - vllm_omni/diffusion/models/wan2_2/
|
||||
# commands:
|
||||
# - |
|
||||
# timeout 40m bash -c "
|
||||
# export VLLM_IMAGE_FETCH_TIMEOUT=60
|
||||
# pytest -s -v tests/e2e/online_serving/test_wan22_t2v.py -m 'core_model' --run-level 'core_model'
|
||||
# "
|
||||
# mirror_hardwares: h100_1
|
||||
|
||||
# - label: "Diffusion · Cosmos3 Test"
|
||||
# source_file_dependencies:
|
||||
# - tests/e2e/online_serving/test_cosmos3.py
|
||||
# - vllm_omni/diffusion/models/cosmos3/
|
||||
# commands:
|
||||
# - |
|
||||
# timeout 40m bash -c "
|
||||
# export VLLM_IMAGE_FETCH_TIMEOUT=60
|
||||
# pytest -s -v tests/e2e/online_serving/test_cosmos3.py -m 'core_model' --run-level 'core_model'
|
||||
# "
|
||||
# mirror_hardwares: h100_1
|
||||
|
||||
# - label: "Diffusion · Qwen Image Test"
|
||||
# source_file_dependencies:
|
||||
# - tests/e2e/online_serving/test_qwen_image.py
|
||||
# - vllm_omni/diffusion/models/qwen_image/
|
||||
# commands:
|
||||
# - |
|
||||
# timeout 40m bash -c "
|
||||
# export VLLM_IMAGE_FETCH_TIMEOUT=60
|
||||
# pytest -s -v tests/e2e/online_serving/test_qwen_image.py -m 'core_model' --run-level 'core_model'
|
||||
# "
|
||||
# mirror_hardwares: h100_1
|
||||
|
||||
# - label: "Diffusion · Qwen Image Layered Test"
|
||||
# source_file_dependencies:
|
||||
# - tests/e2e/online_serving/test_qwen_image_layered.py
|
||||
# - vllm_omni/diffusion/models/qwen_image/
|
||||
# commands:
|
||||
# - |
|
||||
# timeout 40m bash -c "
|
||||
# export VLLM_IMAGE_FETCH_TIMEOUT=60
|
||||
# pytest -s -v tests/e2e/online_serving/test_qwen_image_layered.py -m 'core_model' --run-level 'core_model'
|
||||
# "
|
||||
# mirror_hardwares: h100_1
|
||||
|
||||
# - label: "Diffusion · GGUF Plugin Test"
|
||||
# soft_fail: true
|
||||
# source_file_dependencies:
|
||||
# - tests/plugin/test_gguf_diffusion.py
|
||||
# - vllm_omni/diffusion/model_loader
|
||||
# commands:
|
||||
# - pip install "vllm-gguf-plugin>=0.0.3"
|
||||
# - timeout 30m pytest -sv tests/plugin/test_gguf_diffusion.py
|
||||
# mirror_hardwares: l4_1
|
||||
- label: "Diffusion · GGUF Plugin Test"
|
||||
soft_fail: true
|
||||
source_file_dependencies:
|
||||
- tests/plugin/test_gguf_diffusion.py
|
||||
- vllm_omni/diffusion/model_loader
|
||||
commands:
|
||||
- pip install "vllm-gguf-plugin>=0.0.3"
|
||||
- timeout 30m pytest -sv tests/plugin/test_gguf_diffusion.py
|
||||
mirror_hardwares: l4_1
|
||||
@@ -4,10 +4,6 @@
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# minicpm-challenge branch: skip entire Intel/XPU CI (no pipeline upload => no image build / tests).
|
||||
echo "--- Skipping Intel/XPU CI on this branch (including image build)"
|
||||
exit 0
|
||||
|
||||
source .buildkite/common/scripts/resolve_skip_ci.sh
|
||||
|
||||
upload_pipeline() {
|
||||
|
||||
@@ -1,161 +1,58 @@
|
||||
env:
|
||||
VLLM_WORKER_MULTIPROC_METHOD: spawn
|
||||
VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
|
||||
HF_HUB_DOWNLOAD_TIMEOUT: 300
|
||||
HF_HUB_ETAG_TIMEOUT: 60
|
||||
|
||||
steps:
|
||||
# - group: ":card_index_dividers: Diffusion X2V Model Test"
|
||||
# key: nightly-diffusion-x2v-group
|
||||
# depends_on: upload-nightly-pipeline
|
||||
# if: >-
|
||||
# build.env("NIGHTLY") == "1" ||
|
||||
# build.pull_request.labels includes "nightly-test" ||
|
||||
# build.pull_request.labels includes "diffusion-x2v-test"
|
||||
# steps:
|
||||
# - label: ":full_moon: Diffusion X2V · Function Test"
|
||||
# mirror_hardwares: a2b3_npu_4
|
||||
# env:
|
||||
# VLLM_WORKER_MULTIPROC_METHOD: spawn
|
||||
# HF_TOKEN: "${HF_TOKEN}"
|
||||
# commands:
|
||||
# - pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py -k npu_i2v --run-level "full_model"
|
||||
# - group: ":card_index_dividers: TTS Model Test"
|
||||
# key: nightly-tts-test-group
|
||||
# depends_on: upload-nightly-pipeline
|
||||
# if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test" || build.pull_request.labels includes "tts-test"
|
||||
# steps:
|
||||
# - label: ":full_moon: TTS · Perf Test"
|
||||
# mirror_hardwares: a3_npu_2
|
||||
# env:
|
||||
# VLLM_WORKER_MULTIPROC_METHOD: spawn
|
||||
# VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
|
||||
# HF_TOKEN: "${HF_TOKEN}"
|
||||
# commands:
|
||||
# - export BENCHMARK_DIR=tests/dfx/perf/results
|
||||
# - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
|
||||
# - |
|
||||
# set +e
|
||||
# pytest -s -v tests/dfx/perf/scripts/run_benchmark.py -m npu --test-config-file tests/dfx/perf/tests/test_tts.json
|
||||
# EXIT=$$?
|
||||
# buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
|
||||
# exit $$EXIT
|
||||
- group: ":card_index_dividers: Diffusion X2V Model Test"
|
||||
key: nightly-diffusion-x2v-group
|
||||
depends_on: upload-nightly-pipeline
|
||||
if: >-
|
||||
build.env("NIGHTLY") == "1" ||
|
||||
build.pull_request.labels includes "nightly-test" ||
|
||||
build.pull_request.labels includes "diffusion-x2v-test"
|
||||
steps:
|
||||
- label: ":full_moon: Diffusion X2V · Function Test"
|
||||
mirror_hardwares: a2b3_npu_4
|
||||
env:
|
||||
VLLM_WORKER_MULTIPROC_METHOD: spawn
|
||||
HF_TOKEN: "${HF_TOKEN}"
|
||||
commands:
|
||||
- pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py -k npu_i2v --run-level "full_model"
|
||||
- group: ":card_index_dividers: TTS Model Test"
|
||||
key: nightly-tts-test-group
|
||||
depends_on: upload-nightly-pipeline
|
||||
if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test" || build.pull_request.labels includes "tts-test"
|
||||
steps:
|
||||
- label: ":full_moon: TTS · Perf Test"
|
||||
mirror_hardwares: a3_npu_2
|
||||
env:
|
||||
VLLM_WORKER_MULTIPROC_METHOD: spawn
|
||||
VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
|
||||
HF_TOKEN: "${HF_TOKEN}"
|
||||
commands:
|
||||
- export BENCHMARK_DIR=tests/dfx/perf/results
|
||||
- export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
|
||||
- |
|
||||
set +e
|
||||
pytest -s -v tests/dfx/perf/scripts/run_benchmark.py -m npu --test-config-file tests/dfx/perf/tests/test_tts.json
|
||||
EXIT=$$?
|
||||
buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
|
||||
exit $$EXIT
|
||||
|
||||
- group: ":card_index_dividers: Omni Model Test"
|
||||
key: nightly-omni-group
|
||||
depends_on: upload-nightly-pipeline
|
||||
if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test" || build.pull_request.labels includes "omni-test"
|
||||
steps:
|
||||
- label: ":full_moon: Omni · MiniCPM-o 4.5 Test"
|
||||
timeout_in_minutes: 180
|
||||
mirror_hardwares: a3_npu_2
|
||||
- label: ":robot: Omni · MiniCPM-o 4.5 Test"
|
||||
agents:
|
||||
queue: "ascend-a3"
|
||||
resource_class: "npu-2"
|
||||
image: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/vllm-omni-ci-npu:${BUILDKITE_COMMIT}-a3"
|
||||
plugins:
|
||||
- kubernetes:
|
||||
podSpecPatch:
|
||||
imagePullSecrets:
|
||||
- name: swr-secret
|
||||
env:
|
||||
VLLM_WORKER_MULTIPROC_METHOD: spawn
|
||||
VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
|
||||
HF_TOKEN: "${HF_TOKEN}"
|
||||
commands:
|
||||
- pytest -s -v tests/e2e/offline_inference/test_minicpmo_4_5.py tests/e2e/online_serving/test_minicpmo_4_5.py tests/e2e/online_serving/test_minicpmo_4_5_expansion.py -m 'full_model and npu and A3 and omni' --run-level 'full_model'
|
||||
- label: ":full_moon: Omni · MiniCPM-o 4.5 · Duplex Test"
|
||||
timeout_in_minutes: 180
|
||||
mirror_hardwares: a3_npu_2
|
||||
env:
|
||||
VLLM_WORKER_MULTIPROC_METHOD: spawn
|
||||
VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
|
||||
HF_TOKEN: "${HF_TOKEN}"
|
||||
commands:
|
||||
# NPU has no merge-level pipeline, so advanced_model duplex coverage
|
||||
# rides on nightly alongside full_model.
|
||||
#
|
||||
# test_duplex_soft_interrupt is CUDA-only and carries no npu mark, so
|
||||
# the marker filter below drops it: it needs real-time duplex
|
||||
# throughput that the current NPU stack does not reach.
|
||||
- >-
|
||||
pytest -s -v
|
||||
tests/e2e/online_serving/test_minicpmo_4_5_duplex.py
|
||||
tests/e2e/online_serving/test_minicpmo_4_5_duplex_expansion.py
|
||||
-m "(full_model or advanced_model) and npu and A3 and omni"
|
||||
--run-level full_model
|
||||
- label: ":full_moon: Omni · MiniCPM-o 4.5 · Accuracy Test"
|
||||
timeout_in_minutes: 180
|
||||
mirror_hardwares: a3_npu_2
|
||||
env:
|
||||
VLLM_WORKER_MULTIPROC_METHOD: spawn
|
||||
VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
|
||||
HF_TOKEN: "${HF_TOKEN}"
|
||||
commands:
|
||||
- export SEED_TTS_WER_EVAL=1
|
||||
- export SEED_TTS_SIM_EVAL=1
|
||||
- export SEED_TTS_EVAL_DEVICE=npu:1
|
||||
- |
|
||||
set +e
|
||||
pytest -s -v tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py -k "not videomme" -m "full_model and npu and A3" --run-level full_model
|
||||
EXIT=$$?
|
||||
buildkite-agent artifact upload "tests/e2e/accuracy/minicpmo_4_5/results/*.json"
|
||||
exit $$EXIT
|
||||
- label: ":full_moon: Omni · MiniCPM-o 4.5 · Video-MME Accuracy Test"
|
||||
key: nightly-omni-accuracy-minicpmo-4-5-videomme-npu
|
||||
timeout_in_minutes: 180
|
||||
mirror_hardwares: a3_npu_2
|
||||
env:
|
||||
VLLM_WORKER_MULTIPROC_METHOD: spawn
|
||||
VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
|
||||
HF_TOKEN: "${HF_TOKEN}"
|
||||
commands:
|
||||
# Video-MME needs no repo override here: the NPU runners pre-stage the dataset under
|
||||
# the lmms-eval mirror, which is now VIDEOMME_DEFAULT_HF_REPO, so snapshot_download
|
||||
# resolves it straight out of the hub cache.
|
||||
- |
|
||||
set +e
|
||||
pytest -s -v tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py -k videomme -m "full_model and npu and A3" --run-level full_model
|
||||
EXIT=$$?
|
||||
buildkite-agent artifact upload "tests/e2e/accuracy/minicpmo_4_5/results/*.json"
|
||||
exit $$EXIT
|
||||
- label: ":full_moon: Omni · MiniCPM-o 4.5 · Perf Test"
|
||||
key: nightly-omni-performance-minicpmo-4-5-npu
|
||||
timeout_in_minutes: 180
|
||||
mirror_hardwares: a3_npu_2
|
||||
env:
|
||||
VLLM_WORKER_MULTIPROC_METHOD: spawn
|
||||
VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
|
||||
HF_TOKEN: "${HF_TOKEN}"
|
||||
commands:
|
||||
- export BENCHMARK_DIR=tests/dfx/perf/results
|
||||
- |
|
||||
set +e
|
||||
pytest -s -v tests/dfx/perf/scripts/run_benchmark.py -m "npu and A3" --test-config-file tests/dfx/perf/tests/test_minicpmo_4_5.json
|
||||
EXIT=$$?
|
||||
buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
|
||||
exit $$EXIT
|
||||
- label: ":full_moon: Omni · MiniCPM-o 4.5 · Perf Test (A2)"
|
||||
key: nightly-omni-performance-minicpmo-4-5-npu-a2
|
||||
timeout_in_minutes: 180
|
||||
mirror_hardwares: a2b3_npu_1
|
||||
env:
|
||||
VLLM_WORKER_MULTIPROC_METHOD: spawn
|
||||
VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
|
||||
HF_TOKEN: "${HF_TOKEN}"
|
||||
HF_HUB_OFFLINE: "1"
|
||||
commands:
|
||||
- export BENCHMARK_DIR=tests/dfx/perf/results
|
||||
- |
|
||||
set +e
|
||||
pytest -s -v tests/dfx/perf/scripts/run_benchmark.py -m "npu and A2" --test-config-file tests/dfx/perf/tests/test_minicpmo_4_5.json
|
||||
EXIT=$$?
|
||||
buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
|
||||
exit $$EXIT
|
||||
- label: ":full_moon: Omni · MiniCPM-o 4.5 · Duplex Seed-TTS Perf Test"
|
||||
key: nightly-omni-performance-minicpmo-4-5-duplex-seed-tts-npu
|
||||
timeout_in_minutes: 180
|
||||
mirror_hardwares: a3_npu_2
|
||||
env:
|
||||
VLLM_WORKER_MULTIPROC_METHOD: spawn
|
||||
VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
|
||||
HF_TOKEN: "${HF_TOKEN}"
|
||||
commands:
|
||||
- export BENCHMARK_DIR=tests/dfx/perf/results
|
||||
- |
|
||||
set +e
|
||||
pytest -s -v tests/dfx/perf/scripts/run_benchmark.py -m "npu and A3" --test-config-file tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json
|
||||
EXIT=$$?
|
||||
buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
|
||||
exit $$EXIT
|
||||
- pytest -s -v tests/e2e/offline_inference/test_minicpmo_4_5.py tests/e2e/online_serving/test_minicpmo_4_5.py tests/e2e/online_serving/test_minicpmo_4_5_expansion.py -m 'full_model' --run-level 'full_model'
|
||||
@@ -3,41 +3,32 @@ steps:
|
||||
key: ready-omni-group
|
||||
depends_on: upload-ready-pipeline
|
||||
steps:
|
||||
- label: "Omni · MiniCPM-o 4.5 Test"
|
||||
mirror_hardwares: a3_npu_2
|
||||
- label: ":robot: Omni · MiniCPM-o 4.5 Test"
|
||||
agents:
|
||||
queue: "ascend-a3"
|
||||
resource_class: "npu-2"
|
||||
image: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/vllm-omni-ci-npu:${BUILDKITE_COMMIT}-a3"
|
||||
plugins:
|
||||
- kubernetes:
|
||||
podSpecPatch:
|
||||
imagePullSecrets:
|
||||
- name: swr-secret
|
||||
env:
|
||||
VLLM_WORKER_MULTIPROC_METHOD: spawn
|
||||
VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
|
||||
HF_TOKEN: "${HF_TOKEN}"
|
||||
commands:
|
||||
- pytest -s -v tests/e2e/offline_inference/test_minicpmo_4_5.py tests/e2e/online_serving/test_minicpmo_4_5.py -m 'core_model and npu and A3 and omni' --run-level 'core_model'
|
||||
- label: "Omni · MiniCPM-o 4.5 Duplex Test"
|
||||
mirror_hardwares: a3_npu_2
|
||||
env:
|
||||
VLLM_WORKER_MULTIPROC_METHOD: spawn
|
||||
VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
|
||||
HF_TOKEN: "${HF_TOKEN}"
|
||||
commands:
|
||||
- pytest -s -v tests/e2e/online_serving/test_minicpmo_4_5_duplex.py -m 'core_model and npu and A3 and omni' --run-level 'core_model'
|
||||
- label: "Omni · MiniCPM-o 4.5 Perf Test"
|
||||
timeout_in_minutes: 60
|
||||
mirror_hardwares: a3_npu_2
|
||||
env:
|
||||
VLLM_WORKER_MULTIPROC_METHOD: spawn
|
||||
VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
|
||||
commands:
|
||||
- export BENCHMARK_DIR=tests/dfx/perf/results
|
||||
- pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_minicpmo_4_5_ready.json -m 'core_model and npu and A3 and omni'
|
||||
- pytest -s -v tests/e2e/offline_inference/test_minicpmo_4_5.py tests/e2e/online_serving/test_minicpmo_4_5.py -m 'core_model' --run-level 'core_model'
|
||||
|
||||
# - group: ":card_index_dividers: TTS Model Test"
|
||||
# key: ready-tts-group
|
||||
# depends_on: upload-ready-pipeline
|
||||
# steps:
|
||||
# - label: ":sound: TTS · Qwen3-TTS CustomVoice Test"
|
||||
# mirror_hardwares: a2b3_npu_1
|
||||
# env:
|
||||
# VLLM_WORKER_MULTIPROC_METHOD: spawn
|
||||
# VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
|
||||
# HF_TOKEN: "${HF_TOKEN}"
|
||||
# commands:
|
||||
# - pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py -m 'core_model' --run-level 'core_model'
|
||||
- group: ":card_index_dividers: TTS Model Test"
|
||||
key: ready-tts-group
|
||||
depends_on: upload-ready-pipeline
|
||||
steps:
|
||||
- label: ":sound: TTS · Qwen3-TTS CustomVoice Test"
|
||||
mirror_hardwares: a2b3_npu_1
|
||||
env:
|
||||
VLLM_WORKER_MULTIPROC_METHOD: spawn
|
||||
VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
|
||||
HF_TOKEN: "${HF_TOKEN}"
|
||||
commands:
|
||||
- pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py -m 'core_model' --run-level 'core_model'
|
||||
@@ -1,7 +1,7 @@
|
||||
# Target vllm version: v0.25.0 (releases/v0.25.0 branch).
|
||||
# Target vllm version: v0.26.0 (releases/v0.26.0 branch).
|
||||
# Keep VLLM_BASE_TAG aligned with the target vLLM release on every rebase.
|
||||
ARG VLLM_BASE_IMAGE=vllm/vllm-openai
|
||||
ARG VLLM_BASE_TAG=v0.25.0
|
||||
ARG VLLM_BASE_TAG=v0.26.0
|
||||
|
||||
# We use registry-based layer caching for the dependencies,
|
||||
# where the tag is determined based on the hash of the contents
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
ARG BASE_IMAGE=vllm/vllm-openai:v0.25.0
|
||||
ARG BASE_IMAGE=vllm/vllm-openai:v0.26.0
|
||||
FROM ${BASE_IMAGE}
|
||||
|
||||
ARG COMMON_WORKDIR=/app
|
||||
|
||||
@@ -32,7 +32,6 @@ RUN source /usr/local/Ascend/ascend-toolkit/set_env.sh && \
|
||||
"pytest-cov>=4.0.0" \
|
||||
"pytest-mock>=3.10.0" \
|
||||
"openpyxl>=3.0.0" \
|
||||
"opencc==1.4.1" \
|
||||
"psutil>=7.2.0" \
|
||||
"soundfile>=0.13.1" \
|
||||
"pyttsx3>=2.99" \
|
||||
@@ -41,11 +40,6 @@ RUN source /usr/local/Ascend/ascend-toolkit/set_env.sh && \
|
||||
"hyperpyyaml>=1.2.0" \
|
||||
"onnx>=1.16.0" \
|
||||
"librosa>=0.10.0" \
|
||||
"jiwer>=4.0.0" \
|
||||
"zhon>=2.1.1" \
|
||||
"zhconv>=1.4.3" \
|
||||
"funasr>=1.3.0" \
|
||||
"datasets>=3.0.0" \
|
||||
--index-url http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple \
|
||||
--trusted-host cache-service.nginx-pypi-cache.svc.cluster.local \
|
||||
--default-timeout=60 && \
|
||||
|
||||
@@ -32,7 +32,6 @@ RUN source /usr/local/Ascend/ascend-toolkit/set_env.sh && \
|
||||
"pytest-cov>=4.0.0" \
|
||||
"pytest-mock>=3.10.0" \
|
||||
"openpyxl>=3.0.0" \
|
||||
"opencc==1.4.1" \
|
||||
"psutil>=7.2.0" \
|
||||
"soundfile>=0.13.1" \
|
||||
"pyttsx3>=2.99" \
|
||||
@@ -41,11 +40,6 @@ RUN source /usr/local/Ascend/ascend-toolkit/set_env.sh && \
|
||||
"hyperpyyaml>=1.2.0" \
|
||||
"onnx>=1.16.0" \
|
||||
"librosa>=0.10.0" \
|
||||
"jiwer>=4.0.0" \
|
||||
"zhon>=2.1.1" \
|
||||
"zhconv>=1.4.3" \
|
||||
"funasr>=1.3.0" \
|
||||
"datasets>=3.0.0" \
|
||||
--index-url http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple \
|
||||
--trusted-host cache-service.nginx-pypi-cache.svc.cluster.local \
|
||||
--default-timeout=60 && \
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
ARG BASE_IMAGE=vllm/vllm-openai-rocm:v0.25.0
|
||||
ARG BASE_IMAGE=vllm/vllm-openai-rocm:v0.26.0
|
||||
FROM ${BASE_IMAGE} AS base
|
||||
|
||||
# Declare a variable to know if we want to use the nightly build or the stable build.
|
||||
@@ -10,7 +10,7 @@ FROM ${BASE_IMAGE} AS base
|
||||
# we should swap over to use stable release ASAP.
|
||||
# We should avoid relying on custom commits.
|
||||
ARG USE_NIGHTLY_BUILD=0
|
||||
ARG VLLM_VERSION_OR_COMMIT_HASH=89138b21cc246ae944c741d5c399c148e2b770ab
|
||||
ARG VLLM_VERSION_OR_COMMIT_HASH=v0.26.0
|
||||
ARG ARG_PYTORCH_ROCM_ARCH
|
||||
ENV PYTORCH_ROCM_ARCH=${ARG_PYTORCH_ROCM_ARCH:-${PYTORCH_ROCM_ARCH}}
|
||||
|
||||
|
||||
@@ -88,7 +88,7 @@ ENV UV_EXTRA_INDEX_URL=${PIP_EXTRA_INDEX_URL}
|
||||
ENV UV_INDEX_STRATEGY="unsafe-best-match"
|
||||
ENV UV_LINK_MODE="copy"
|
||||
|
||||
ARG VLLM_VERSION=v0.25.0
|
||||
ARG VLLM_VERSION=v0.26.0
|
||||
RUN git clone -b ${VLLM_VERSION} https://github.com/vllm-project/vllm
|
||||
WORKDIR /workspace/vllm
|
||||
|
||||
|
||||
@@ -44,11 +44,9 @@ Pass **`--test-config-file`** to run one JSON file as-is, or omit it for the bul
|
||||
"ignore_eos": true,
|
||||
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"mean_ttft_ms": [500, 800],
|
||||
"mean_audio_ttfp_ms": [2000, 3500],
|
||||
"mean_audio_rtf": [0.25, 0.35]
|
||||
}
|
||||
"mean_ttft_ms": [500, 800],
|
||||
"mean_audio_ttfp_ms": [2000, 3500],
|
||||
"mean_audio_rtf": [0.25, 0.35]
|
||||
}
|
||||
}
|
||||
]
|
||||
@@ -144,6 +142,10 @@ pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py \
|
||||
--test-config-file tests/dfx/perf/tests/test_bagel_vllm_omni.json
|
||||
pytest -s -v tests/dfx/perf/scripts/run_benchmark.py \
|
||||
--test-config-file tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json
|
||||
|
||||
# Optional baseline assertion (default off)
|
||||
pytest -sv tests/dfx/perf/scripts/run_diffusion_benchmark.py --assert-baseline \
|
||||
--test-config-file tests/dfx/perf/tests/test_qwen_image_vllm_omni.json
|
||||
```
|
||||
|
||||
See also [Markers for Tests](#markers-for-tests) for registered hardware markers (`H100`, `L4`, `cuda`, `distributed_cuda`, …).
|
||||
@@ -200,4 +202,4 @@ You can add any benchmark running parameters you need here. For all optional par
|
||||
| num_prompts | int / array | Yes | 10,[10, 20, 30] | Number of requests. Supports single values or arrays. If a single value is used, it will be automatically expanded to match the number of qps or max_concurrency, e.g., [10,10,10]. If an array is used, its length must match the number of qps or max_concurrency. |
|
||||
| request_rate | float / array | No | 0.5, [0.5, 1, inf] | Queries per second. Supports single values or arrays. If a single value is used, it will be automatically expanded to match the number of num_prompts, e.g., [1,1,1]. If an array is used, its length must match the number of num_prompts. |
|
||||
| max_concurrency | int / array | No | 1, [1, 2, 3] | Maximum concurrent in-flight requests. Same array / expansion rules as `request_rate` (mutually exclusive with QPS mode). |
|
||||
| baseline | object | No | see below | Optional thresholds. **Hardware-nested only**: top-level keys must be `[hardware-resource]` pytest markers from `pyproject.toml` (`get_hardware_mark_list()`). Under each hardware key, metric names are free-form; values are scalars or sweep-aligned lists. Example: `{"H100": {"throughput_qps": […], "custom_stage_ms": 1.0}, "A3": {…}}`. Runtime filename aliases (`_RUNTIME_DEVICE_ALIASES`) are separate and do not expand this allowlist. |
|
||||
| baseline | object | No | see above | Optional per-metric thresholds; keys must match benchmark output fields. Scalar, list (per sweep step), or object (keyed by concurrency or QPS string).
|
||||
@@ -34,7 +34,7 @@ vLLM-Omni is a Python library that supports the following GPU variants. The libr
|
||||
|
||||
### Pre-built wheels
|
||||
|
||||
Note: Pre-built wheels are currently available for vLLM-Omni 0.11.0rc1, 0.12.0rc1, 0.14.0rc1, 0.14.0, 0.16.0, 0.18.0, 0.20.0, 0.21.0, 0.22.0, 0.23.0, 0.24.0, and 0.25.0. If you need a newer unreleased revision, please [build from source](https://docs.vllm.ai/projects/vllm-omni/en/latest/getting_started/installation/gpu/#build-wheel-from-source).
|
||||
Note: Pre-built wheels are currently available for vLLM-Omni 0.11.0rc1, 0.12.0rc1, 0.14.0rc1, 0.14.0, 0.16.0, 0.18.0, 0.20.0, 0.21.0, 0.22.0, 0.23.0, 0.24.0, 0.25.0, and 0.26.0. If you need a newer unreleased revision, please [build from source](https://docs.vllm.ai/projects/vllm-omni/en/latest/getting_started/installation/gpu/#build-wheel-from-source).
|
||||
|
||||
=== "NVIDIA CUDA"
|
||||
|
||||
|
||||
@@ -20,7 +20,7 @@ Therefore, it is recommended to install vLLM and vLLM-Omni with a **fresh new**
|
||||
|
||||
vLLM-Omni is built based on vLLM. Please install it with command below.
|
||||
```bash
|
||||
uv pip install vllm==0.25.0 --torch-backend=auto
|
||||
uv pip install vllm==0.26.0 --torch-backend=auto
|
||||
```
|
||||
|
||||
#### Installation of vLLM-Omni
|
||||
@@ -39,13 +39,13 @@ uv pip install 'vllm-omni[demo]'
|
||||
# --8<-- [start:build-wheel-from-source]
|
||||
|
||||
#### Installation of vLLM
|
||||
If you do not need to modify source code of vLLM, you can directly install the stable 0.25.0 release version of the library
|
||||
If you do not need to modify source code of vLLM, you can directly install the stable 0.26.0 release version of the library
|
||||
|
||||
```bash
|
||||
uv pip install vllm==0.25.0 --torch-backend=auto
|
||||
uv pip install vllm==0.26.0 --torch-backend=auto
|
||||
```
|
||||
|
||||
The 0.25.0 release of vLLM ships CUDA 13.0-compatible binaries by default. If you need a different CUDA variant or want to reuse an existing PyTorch installation, build vLLM from source instead.
|
||||
The 0.26.0 release of vLLM ships CUDA 13.0-compatible binaries by default. If you need a different CUDA variant or want to reuse an existing PyTorch installation, build vLLM from source instead.
|
||||
|
||||
#### Installation of vLLM-Omni
|
||||
Since vllm-omni is rapidly evolving, it's recommended to install it from source
|
||||
@@ -66,12 +66,12 @@ If you want to check, modify or debug with source code of vLLM, install the libr
|
||||
```bash
|
||||
git clone https://github.com/vllm-project/vllm.git
|
||||
cd vllm
|
||||
git checkout v0.25.0
|
||||
git checkout v0.26.0
|
||||
```
|
||||
Set up environment variables to get pre-built wheels. If there are internet problems, just download the whl file manually. And set `VLLM_PRECOMPILED_WHEEL_LOCATION` as your local absolute path of whl file.
|
||||
```bash
|
||||
#For CUDA 13.0 (the default for v0.25.0; the wheel filename has no `+cu130` suffix)
|
||||
export VLLM_PRECOMPILED_WHEEL_LOCATION=https://github.com/vllm-project/vllm/releases/download/v0.25.0/vllm-0.25.0-cp38-abi3-manylinux_2_28_x86_64.whl
|
||||
#For CUDA 13.0 (the default for v0.26.0; the wheel filename has no `+cu130` suffix)
|
||||
export VLLM_PRECOMPILED_WHEEL_LOCATION=https://github.com/vllm-project/vllm/releases/download/v0.26.0/vllm-0.26.0-cp38-abi3-manylinux_2_28_x86_64.whl
|
||||
```
|
||||
Install vllm with command below (If you have no existing PyTorch).
|
||||
```bash
|
||||
@@ -102,7 +102,7 @@ docker run --runtime nvidia --gpus 2 \
|
||||
--env "HF_TOKEN=$HF_TOKEN" \
|
||||
-p 8091:8091 \
|
||||
--ipc=host \
|
||||
vllm/vllm-omni:v0.25.0 \
|
||||
vllm/vllm-omni:v0.26.0 \
|
||||
vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091
|
||||
```
|
||||
|
||||
|
||||
@@ -13,7 +13,7 @@ vLLM-Omni current recommends the steps in under setup through Docker Images.
|
||||
|
||||
vLLM-Omni is built based on vLLM. Please install it with command below.
|
||||
```bash
|
||||
uv pip install vllm==0.25.0+rocm723 --extra-index-url https://wheels.vllm.ai/rocm/0.25.0/rocm723
|
||||
uv pip install vllm==0.26.0+rocm723 --extra-index-url https://wheels.vllm.ai/rocm/0.26.0/rocm723
|
||||
```
|
||||
|
||||
#### Installation of vLLM-Omni
|
||||
@@ -34,13 +34,13 @@ uv pip install onnxruntime-rocm
|
||||
# --8<-- [start:build-wheel-from-source]
|
||||
|
||||
#### Installation of vLLM
|
||||
If you do not need to modify source code of vLLM, you can directly install the stable 0.25.0 release version of the library
|
||||
If you do not need to modify source code of vLLM, you can directly install the stable 0.26.0 release version of the library
|
||||
|
||||
```bash
|
||||
uv pip install vllm==0.25.0+rocm723 --extra-index-url https://wheels.vllm.ai/rocm/0.25.0/rocm723
|
||||
uv pip install vllm==0.26.0+rocm723 --extra-index-url https://wheels.vllm.ai/rocm/0.26.0/rocm723
|
||||
```
|
||||
|
||||
The pre-built 0.25.0 vLLM wheel targets ROCm 7.2.3. If you need a different ROCm stack or want to reuse an existing PyTorch installation, build vLLM from source instead.
|
||||
The pre-built 0.26.0 vLLM wheel targets ROCm 7.2.3. If you need a different ROCm stack or want to reuse an existing PyTorch installation, build vLLM from source instead.
|
||||
|
||||
#### Installation of vLLM-Omni
|
||||
Since vllm-omni is rapidly evolving, it's recommended to install it from source
|
||||
@@ -58,7 +58,7 @@ If you want to check, modify or debug with source code of vLLM, install the libr
|
||||
```bash
|
||||
git clone https://github.com/vllm-project/vllm.git
|
||||
cd vllm
|
||||
git checkout v0.25.0
|
||||
git checkout v0.26.0
|
||||
python3 -m pip install -r requirements/rocm.txt
|
||||
python3 setup.py develop
|
||||
```
|
||||
@@ -130,7 +130,7 @@ docker run --rm \
|
||||
-v ~/.cache/huggingface:/root/.cache/huggingface \
|
||||
--env "HF_TOKEN=$HF_TOKEN" \
|
||||
-p 8091:8091 \
|
||||
vllm/vllm-omni-rocm:v0.25.0 \
|
||||
vllm/vllm-omni-rocm:v0.26.0 \
|
||||
--model Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091
|
||||
```
|
||||
|
||||
@@ -149,7 +149,7 @@ docker run --rm -it \
|
||||
-v ~/.cache/huggingface:/root/.cache/huggingface \
|
||||
--env "HF_TOKEN=$HF_TOKEN" \
|
||||
--entrypoint bash \
|
||||
vllm/vllm-omni-rocm:v0.25.0
|
||||
vllm/vllm-omni-rocm:v0.26.0
|
||||
```
|
||||
|
||||
# --8<-- [end:pre-built-images]
|
||||
@@ -19,10 +19,10 @@ uv venv --python 3.12 --seed
|
||||
source .venv/bin/activate
|
||||
|
||||
# On CUDA
|
||||
uv pip install vllm==0.25.0 --torch-backend=auto
|
||||
uv pip install vllm==0.26.0 --torch-backend=auto
|
||||
|
||||
# On ROCm
|
||||
uv pip install vllm==0.25.0+rocm723 --extra-index-url https://wheels.vllm.ai/rocm/0.25.0/rocm723
|
||||
uv pip install vllm==0.26.0+rocm723 --extra-index-url https://wheels.vllm.ai/rocm/0.26.0/rocm723
|
||||
|
||||
git clone https://github.com/vllm-project/vllm-omni.git
|
||||
cd vllm-omni
|
||||
@@ -34,7 +34,7 @@ For additional installation methods — please see the [installation guide](inst
|
||||
!!! note
|
||||
It is important to install the same major & minor version of vLLM and vLLM Omni, otherwise things may not work as expected. If the versions are misaligned, you will see a warning when you import vLLM Omni.
|
||||
|
||||
If you are seeing strange behavior with the `vllm` command not handling the `--omni` flag correctly, you most likely have a version mismatch with vLLM < `0.25.0` and vLLM Omni `0.25.0`, as vLLM Omni no longer hijacks the vLLM entrypoint. Updating vLLM should resolve this issue.
|
||||
If you are seeing strange behavior with the `vllm` command not handling the `--omni` flag correctly, you most likely have a version mismatch with vLLM < `0.26.0` and vLLM Omni `0.26.0`, as vLLM Omni no longer hijacks the vLLM entrypoint. Updating vLLM should resolve this issue.
|
||||
|
||||
## Offline Inference
|
||||
|
||||
|
||||
@@ -1,9 +1,8 @@
|
||||
# MiniCPM-o 4.5: Offline inference
|
||||
|
||||
Three-stage pipeline: **Thinker** → **Talker** (codec) → **Code2Wav**
|
||||
Two-stage pipeline: **thinker** (multimodal understanding) → **talker + Token2Wav**
|
||||
(24 kHz speech). Deploy config auto-loads from
|
||||
`vllm_omni/deploy/minicpmo_4_5.yaml` (single-GPU default). For higher
|
||||
throughput, pass `--deploy-config vllm_omni/deploy/minicpmo_4_5_2gpu.yaml`.
|
||||
`vllm_omni/deploy/minicpmo_4_5.yaml` (2-GPU default).
|
||||
|
||||
## Setup
|
||||
|
||||
@@ -73,7 +72,7 @@ Supported `--query-type` values:
|
||||
|
||||
```bash
|
||||
python end2end.py --query-type text \
|
||||
--deploy-config /path/to/vllm_omni/deploy/minicpmo_4_5_2gpu.yaml
|
||||
--deploy-config /path/to/vllm_omni/deploy/minicpmo_4_5_8x4090.yaml
|
||||
```
|
||||
|
||||
## Notes
|
||||
@@ -84,8 +83,7 @@ python end2end.py --query-type text \
|
||||
- Output WAV is **24 kHz mono**.
|
||||
- Placeholders in the prompt are MiniCPM-style:
|
||||
`(<image>./</image>)`, `(<audio>./</audio>)`, `(<video>./</video>)`.
|
||||
- Default layout needs **1 GPU**. Use `minicpmo_4_5_2gpu.yaml` for the
|
||||
recommended two-GPU continuous-batching layout.
|
||||
- Default layout needs **2 GPUs**. Async chunking is off in the bundled YAMLs.
|
||||
|
||||
## Online serving
|
||||
|
||||
|
||||
@@ -16,6 +16,7 @@ import numpy as np
|
||||
import soundfile as sf
|
||||
import vllm
|
||||
from PIL import Image
|
||||
from vllm import SamplingParams
|
||||
from vllm.assets.audio import AudioAsset
|
||||
from vllm.assets.image import ImageAsset
|
||||
from vllm.assets.video import VideoAsset, video_to_ndarrays
|
||||
@@ -25,6 +26,8 @@ from vllm.multimodal.media.audio import load_audio
|
||||
from vllm_omni.entrypoints.omni import Omni
|
||||
from vllm_omni.utils.tracking_parser import TrackingArgumentParser
|
||||
|
||||
SEED = 42
|
||||
|
||||
default_system = (
|
||||
"You are MiniCPM-o, a helpful multimodal assistant that can "
|
||||
"understand images, audio and video, and respond in text and speech."
|
||||
@@ -282,17 +285,27 @@ def main(args):
|
||||
omni_kwargs["trust_remote_code"] = True
|
||||
omni = Omni(**omni_kwargs)
|
||||
|
||||
# Per-stage sampling defaults live in the deploy YAML
|
||||
# (``vllm_omni/deploy/minicpmo_4_5*.yaml``, ``default_sampling_params``),
|
||||
# which is the single source of truth that tracks the pipeline topology.
|
||||
# Passing None makes ``resolve_sampling_params_list`` load exactly those,
|
||||
# so adding or removing a stage cannot desynchronize this example.
|
||||
#
|
||||
# This used to hardcode a two-element list built for the old two-stage
|
||||
# topology (thinker → talker+Token2Wav). Since Code2Wav became its own
|
||||
# Stage 2, that list was both the wrong length and wrong in content
|
||||
# (talker ``max_tokens=1`` truncated the codec stream to one frame).
|
||||
sampling_params_list = None
|
||||
# Stage 0 (thinker): multimodal understanding → text (+ TTS span when enabled).
|
||||
thinker_sampling_params = SamplingParams(
|
||||
temperature=0.0,
|
||||
top_p=1.0,
|
||||
top_k=-1,
|
||||
max_tokens=2048,
|
||||
seed=SEED,
|
||||
detokenize=True,
|
||||
repetition_penalty=1.1,
|
||||
)
|
||||
# Stage 1 (talker + Token2Wav): max_tokens=1 satisfies the scheduler;
|
||||
# waveform is produced in-process by Token2Wav.
|
||||
talker_sampling_params = SamplingParams(
|
||||
temperature=0.0,
|
||||
top_p=1.0,
|
||||
top_k=-1,
|
||||
max_tokens=1,
|
||||
seed=SEED,
|
||||
detokenize=False,
|
||||
)
|
||||
sampling_params_list = [thinker_sampling_params, talker_sampling_params][: omni.num_stages]
|
||||
|
||||
if args.txt_prompts is None:
|
||||
prompts = [query_result.inputs for _ in range(args.num_prompts)]
|
||||
|
||||
@@ -23,17 +23,17 @@ including `librosa`.
|
||||
|
||||
The deploy config auto-loads via `--omni`.
|
||||
The default `vllm_omni/deploy/minicpmo_4_5.yaml` keeps all three stages on
|
||||
logical device 0 with memory budgets of 55%, 15%, and 18%, leaving headroom
|
||||
for runtime kernels such as the HiFi-GAN vocoder's cuDNN workspace. The
|
||||
profile admits at most four sequences per stage. For throughput,
|
||||
`minicpmo_4_5_2gpu.yaml` gives the Thinker GPU 0 (90%) and colocates the
|
||||
Talker (55%) and Code2Wav (35%) on GPU 1. That profile admits at most four
|
||||
concurrent sequences per stage.
|
||||
logical device 0 with memory budgets of 55%, 22%, and 22%. The profile admits
|
||||
at most four sequences per stage and bounds startup video profiling to 32
|
||||
frames per video. For throughput,`minicpmo_4_5_batching.yaml` gives the Thinker
|
||||
GPU 0 (90%) and colocates the Talker (55%) and Code2Wav (35%) on GPU 1. That
|
||||
profile admits at most four concurrent sequences per stage.
|
||||
|
||||
| deploy config | GPUs | Notes |
|
||||
|---|---|---|
|
||||
| `minicpmo_4_5.yaml` (default) | 1 | Memory-constrained compatibility layout. |
|
||||
| `minicpmo_4_5_2gpu.yaml` | 2 | Recommended continuous-batching layout; Talker and Code2Wav share GPU 1. |
|
||||
| `minicpmo_4_5_batching.yaml` | 2 | Recommended continuous-batching layout; Talker and Code2Wav share GPU 1. |
|
||||
| `minicpmo_4_5_2gpu.yaml` | 2 | Backward-compatible alias for `minicpmo_4_5_batching.yaml`. |
|
||||
| `minicpmo_4_5_3gpu.yaml` | 3 | One GPU per stage. |
|
||||
| `minicpmo_4_5_8x4090.yaml` | 8 | Full 8x4090 layout. |
|
||||
| `minicpmo_4_5_duplex.yaml` | 1 | Experimental native full-duplex overlay. |
|
||||
@@ -89,18 +89,17 @@ python openai_chat_completion_client_for_multimodal_generation.py \
|
||||
--prompt "Briefly introduce yourself."
|
||||
```
|
||||
|
||||
Streaming text + audio (use `--stream`):
|
||||
Streaming text + audio:
|
||||
|
||||
```bash
|
||||
python openai_chat_completion_client_for_multimodal_generation.py \
|
||||
--query-type text \
|
||||
--prompt "Briefly introduce yourself." \
|
||||
--port 8099 \
|
||||
--stream
|
||||
python streaming_chat_completion.py \
|
||||
--base-url http://localhost:8099/v1 \
|
||||
--output minicpmo_stream.wav
|
||||
```
|
||||
|
||||
The client prints text deltas as they arrive and saves streamed audio chunks
|
||||
to WAV files.
|
||||
The example prints text deltas immediately and joins the independently encoded
|
||||
24 kHz WAV audio deltas into one valid WAV file. Add `--text-only` to skip
|
||||
Talker and Code2Wav.
|
||||
|
||||
Shared helpers also work if you pass MiniCPM defaults yourself:
|
||||
|
||||
|
||||
@@ -6,7 +6,6 @@ class FullDuplexPcmPlayback extends AudioWorkletProcessor {
|
||||
this.playedFrames = 0;
|
||||
this.underrunFrames = 0;
|
||||
this.drain = null;
|
||||
this.terminalFadeFrames = 0;
|
||||
this.started = false;
|
||||
this.activeResponseId = null;
|
||||
this.initialBufferFrames = Math.round(sampleRate * 0.2);
|
||||
@@ -32,7 +31,6 @@ class FullDuplexPcmPlayback extends AudioWorkletProcessor {
|
||||
}
|
||||
} else if (message.type === 'drain') {
|
||||
this.drain = { responseId: message.responseId || null };
|
||||
this.terminalFadeFrames = Math.min(this.fadeFrames, this.bufferedFrames());
|
||||
if (!this.started && this.bufferedFrames() > 0) {
|
||||
this.bufferWaitFrames = 0;
|
||||
this.startPlayback();
|
||||
@@ -44,7 +42,6 @@ class FullDuplexPcmPlayback extends AudioWorkletProcessor {
|
||||
this.playedFrames = 0;
|
||||
this.underrunFrames = 0;
|
||||
this.drain = null;
|
||||
this.terminalFadeFrames = 0;
|
||||
this.started = false;
|
||||
this.activeResponseId = null;
|
||||
this.bufferWaitFrames = this.initialBufferFrames;
|
||||
@@ -79,7 +76,6 @@ class FullDuplexPcmPlayback extends AudioWorkletProcessor {
|
||||
this.playedFrames = 0;
|
||||
this.underrunFrames = 0;
|
||||
this.drain = null;
|
||||
this.terminalFadeFrames = 0;
|
||||
this.started = false;
|
||||
this.activeResponseId = null;
|
||||
this.bufferWaitFrames = this.initialBufferFrames;
|
||||
@@ -120,7 +116,6 @@ class FullDuplexPcmPlayback extends AudioWorkletProcessor {
|
||||
while (target < output.length && this.queue.length > 0) {
|
||||
const pcm = this.queue[0];
|
||||
const count = Math.min(output.length - target, pcm.length - this.offset);
|
||||
const remainingBeforeChunk = this.bufferedFrames();
|
||||
for (let index = 0; index < count; index += 1) {
|
||||
let sample = pcm[this.offset + index] / 32768;
|
||||
if (this.fadeInFrames > 0) {
|
||||
@@ -128,16 +123,6 @@ class FullDuplexPcmPlayback extends AudioWorkletProcessor {
|
||||
sample *= elapsed / this.fadeFrames;
|
||||
this.fadeInFrames -= 1;
|
||||
}
|
||||
const remainingFrames = remainingBeforeChunk - index;
|
||||
if (
|
||||
this.drain
|
||||
&& this.terminalFadeFrames > 0
|
||||
&& remainingFrames <= this.terminalFadeFrames
|
||||
) {
|
||||
sample *= this.terminalFadeFrames === 1
|
||||
? 0
|
||||
: (remainingFrames - 1) / (this.terminalFadeFrames - 1);
|
||||
}
|
||||
output[target + index] = sample;
|
||||
}
|
||||
target += count;
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Stream MiniCPM-o 4.5 text and audio from /v1/chat/completions."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import base64
|
||||
import io
|
||||
import wave
|
||||
from pathlib import Path
|
||||
|
||||
from openai import OpenAI
|
||||
|
||||
|
||||
def append_wav_chunk(encoded: str, pcm_parts: list[bytes]) -> tuple[int, int, int]:
|
||||
"""Decode one base64 WAV delta and append its PCM frames."""
|
||||
with wave.open(io.BytesIO(base64.b64decode(encoded)), "rb") as wav:
|
||||
channels = wav.getnchannels()
|
||||
sample_width = wav.getsampwidth()
|
||||
sample_rate = wav.getframerate()
|
||||
pcm_parts.append(wav.readframes(wav.getnframes()))
|
||||
return channels, sample_width, sample_rate
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--base-url", default="http://localhost:8099/v1")
|
||||
parser.add_argument("--model", default="openbmb/MiniCPM-o-4_5")
|
||||
parser.add_argument("--prompt", default="Say hello, then introduce vLLM in one sentence.")
|
||||
parser.add_argument("--output", type=Path, default=Path("minicpmo_stream.wav"))
|
||||
parser.add_argument("--text-only", action="store_true")
|
||||
args = parser.parse_args()
|
||||
|
||||
modalities = ["text"] if args.text_only else ["text", "audio"]
|
||||
stream = OpenAI(base_url=args.base_url, api_key="EMPTY").chat.completions.create(
|
||||
model=args.model,
|
||||
messages=[{"role": "user", "content": args.prompt}],
|
||||
modalities=modalities,
|
||||
stream=True,
|
||||
extra_body={"chat_template_kwargs": {"use_tts_template": not args.text_only}},
|
||||
)
|
||||
|
||||
pcm_parts: list[bytes] = []
|
||||
audio_format: tuple[int, int, int] | None = None
|
||||
for chunk in stream:
|
||||
for choice in chunk.choices:
|
||||
content = getattr(choice.delta, "content", None)
|
||||
if not content:
|
||||
continue
|
||||
if getattr(chunk, "modality", None) == "audio":
|
||||
current_format = append_wav_chunk(content, pcm_parts)
|
||||
if audio_format is not None and current_format != audio_format:
|
||||
raise ValueError(f"Audio format changed: {audio_format} -> {current_format}")
|
||||
audio_format = current_format
|
||||
elif getattr(chunk, "modality", None) == "text":
|
||||
print(content, end="", flush=True)
|
||||
|
||||
print()
|
||||
if pcm_parts and audio_format is not None:
|
||||
channels, sample_width, sample_rate = audio_format
|
||||
with wave.open(str(args.output), "wb") as wav:
|
||||
wav.setnchannels(channels)
|
||||
wav.setsampwidth(sample_width)
|
||||
wav.setframerate(sample_rate)
|
||||
wav.writeframes(b"".join(pcm_parts))
|
||||
print(f"Audio saved to {args.output} ({sample_rate} Hz)")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+7
-10
@@ -243,16 +243,13 @@ markers = [
|
||||
"npu: Tests that run on NPU/Ascend (auto-added)",
|
||||
"musa: Tests that run on MUSA/Moore Threads (auto-added)",
|
||||
# specified computation resources marks (auto-added)
|
||||
# Tag with [hardware-resource] for tests.helpers.mark.get_hardware_mark_list().
|
||||
"H100: [hardware-resource] Tests that require H100 GPU",
|
||||
"H200: [hardware-resource] Tests that require H200 GPU",
|
||||
"L4: [hardware-resource] Tests that require L4 GPU",
|
||||
"B200: [hardware-resource] Tests that require B200 GPU",
|
||||
"MI325: [hardware-resource] Tests that require MI325 GPU (AMD/ROCm)",
|
||||
"B60: [hardware-resource] Tests that require Intel Arc Pro B60 XPU",
|
||||
"S5000: [hardware-resource] Tests that require S5000 GPU (Moore Threads/MUSA)",
|
||||
"A2: [hardware-resource] Tests that require A2 NPU",
|
||||
"A3: [hardware-resource] Tests that require A3 NPU",
|
||||
"H100: Tests that require H100 GPU",
|
||||
"L4: Tests that require L4 GPU",
|
||||
"MI325: Tests that require MI325 GPU (AMD/ROCm)",
|
||||
"B60: Tests that require Intel Arc Pro B60 XPU",
|
||||
"S5000: Tests that require S5000 GPU (Moore Threads/MUSA)",
|
||||
"A2: Tests that require A2 NPU",
|
||||
"A3: Tests that require A3 NPU",
|
||||
"distributed_cuda: Tests that require multi cards on CUDA platform",
|
||||
"distributed_rocm: Tests that require multi cards on ROCm platform",
|
||||
"distributed_xpu: Tests that require multi cards on XPU platform",
|
||||
|
||||
@@ -31,8 +31,9 @@ also provided.
|
||||
- Default single-GPU compatibility layout (auto-loaded):
|
||||
[`vllm_omni/deploy/minicpmo_4_5.yaml`](../../vllm_omni/deploy/minicpmo_4_5.yaml)
|
||||
- Recommended 2-GPU continuous-batching layout:
|
||||
[`vllm_omni/deploy/minicpmo_4_5_2gpu.yaml`](../../vllm_omni/deploy/minicpmo_4_5_2gpu.yaml)
|
||||
- 3-GPU layout:
|
||||
[`vllm_omni/deploy/minicpmo_4_5_batching.yaml`](../../vllm_omni/deploy/minicpmo_4_5_batching.yaml)
|
||||
- 2-GPU and 3-GPU layouts:
|
||||
[`vllm_omni/deploy/minicpmo_4_5_2gpu.yaml`](../../vllm_omni/deploy/minicpmo_4_5_2gpu.yaml),
|
||||
[`vllm_omni/deploy/minicpmo_4_5_3gpu.yaml`](../../vllm_omni/deploy/minicpmo_4_5_3gpu.yaml)
|
||||
- 8x RTX 4090 layout:
|
||||
[`vllm_omni/deploy/minicpmo_4_5_8x4090.yaml`](../../vllm_omni/deploy/minicpmo_4_5_8x4090.yaml)
|
||||
@@ -67,8 +68,8 @@ Code2Wav consumes them through a shared-memory async connector.
|
||||
MiniCPM-o 4.5 now requires the three-stage topology: the Talker owns
|
||||
request-local codec generation and Code2Wav owns waveform state and
|
||||
reference-voice prompt features. `minicpmo_4_5.yaml` remains the stable
|
||||
single-GPU entry point; `minicpmo_4_5_2gpu.yaml` is the recommended
|
||||
two-GPU continuous-batching profile. The removed fused two-stage implementation is not retained as
|
||||
single-GPU entry point; `minicpmo_4_5_batching.yaml` is the recommended
|
||||
two-GPU profile. The removed fused two-stage implementation is not retained as
|
||||
a fallback because it would duplicate state machines and correctness paths.
|
||||
|
||||
## GPU
|
||||
@@ -78,9 +79,9 @@ a fallback because it would duplicate state machines and correctness paths.
|
||||
The default
|
||||
[`vllm_omni/deploy/minicpmo_4_5.yaml`](../../vllm_omni/deploy/minicpmo_4_5.yaml)
|
||||
co-locates Thinker, codec-only Talker, and Code2Wav on GPU 0. Their
|
||||
`gpu_memory_utilization` budgets are 0.55, 0.15, and 0.18 so concurrent
|
||||
Code2Wav activations keep residual headroom. All stages admit up to four
|
||||
sequences. Use the two-GPU profile for production throughput.
|
||||
`gpu_memory_utilization` budgets are 0.55, 0.22, and 0.22. All stages admit
|
||||
up to four sequences. Startup video profiling is bounded to 32 frames per
|
||||
video. Use the two-GPU profile for production throughput.
|
||||
|
||||
#### Environment
|
||||
|
||||
@@ -104,7 +105,7 @@ The deploy config is auto-loaded by the model registry — no
|
||||
For the recommended two-GPU layout, add:
|
||||
|
||||
```bash
|
||||
--deploy-config vllm_omni/deploy/minicpmo_4_5_2gpu.yaml
|
||||
--deploy-config vllm_omni/deploy/minicpmo_4_5_batching.yaml
|
||||
```
|
||||
|
||||
#### Performance comparison
|
||||
@@ -174,18 +175,16 @@ in another choice's `message.audio.data` (24 kHz mono, see Notes). With
|
||||
`modalities: ["text", "audio"]` you typically get two `choices` entries
|
||||
(one text, one audio).
|
||||
|
||||
**Streaming text + speech** (use `--stream`):
|
||||
**Streaming text + speech**:
|
||||
|
||||
```bash
|
||||
python examples/online_serving/minicpmo/openai_chat_completion_client_for_multimodal_generation.py \
|
||||
--query-type text \
|
||||
--prompt "Say hello, then introduce vLLM in one sentence." \
|
||||
--port 8099 \
|
||||
--stream
|
||||
python examples/online_serving/minicpmo/streaming_chat_completion.py \
|
||||
--base-url http://localhost:8099/v1 \
|
||||
--output minicpmo_stream.wav
|
||||
```
|
||||
|
||||
The client prints text deltas as they arrive and saves streamed audio chunks
|
||||
to WAV files.
|
||||
The client prints text deltas as they arrive and reconstructs one valid WAV
|
||||
from the independently encoded audio deltas.
|
||||
|
||||
**Gradio demo (text + image + audio + video UI)**:
|
||||
|
||||
@@ -203,8 +202,8 @@ speech output (TTS)"** checkbox on / off.
|
||||
|
||||
#### Notes
|
||||
|
||||
- Memory budget: Thinker, Talker, and Code2Wav reserve 0.55, 0.15, and
|
||||
0.18 of GPU 0 so concurrent Code2Wav activations keep residual headroom;
|
||||
- Memory budget: Thinker, Talker, and Code2Wav reserve 0.55, 0.22, and
|
||||
0.22 of GPU 0. The larger Thinker share protects its multimodal KV cache;
|
||||
all three model processes still share one CUDA device.
|
||||
- `--trust-remote-code` is required — the HF repo ships a custom
|
||||
`MiniCPMO` config / model class.
|
||||
@@ -215,13 +214,14 @@ speech output (TTS)"** checkbox on / off.
|
||||
contention. Talker AR
|
||||
state and Code2Wav caches are request-owned; Code2Wav batches only
|
||||
exact-shape-compatible chunks and does not fall back to serial decode.
|
||||
- Stage 0 pins `interleave_mm_strings=true` and
|
||||
`media_io_kwargs.video={fps:1,num_frames:128}` for Daily-Omni MiniCPM packs.
|
||||
- `limit_mm_per_prompt.video.num_frames: 32` bounds startup dummy profiling,
|
||||
not runtime media decoding. Use `media_io_kwargs.video.num_frames` when a
|
||||
matching URL/file video sampling limit is required.
|
||||
- `StageRequestStats.batch_size` is a request-scoped placeholder, not the
|
||||
scheduler's execution batch.
|
||||
- Single-GPU co-location trades throughput for hardware density: Stage 0/1
|
||||
CUDA Graph replay and eager Stage 2 vocoder kernels compete across three
|
||||
CUDA contexts. Use `minicpmo_4_5_2gpu.yaml` or a custom multi-GPU mapping for
|
||||
CUDA contexts. Use the 8x4090 config or a custom multi-GPU mapping for
|
||||
throughput-sensitive serving.
|
||||
|
||||
### 8 x RTX 4090 24GB (consumer-GPU layout)
|
||||
|
||||
@@ -164,11 +164,10 @@ curl -s http://localhost:8091/v1/chat/completions \
|
||||
PR [#2495](https://github.com/vllm-project/vllm-omni/pull/2495) adds
|
||||
performance CI configs for the same DiT-only settings. The CI step is
|
||||
currently opt-in (gated by `RUN_HUNYUAN_IMAGE3_PERF=1`) with `soft_fail`
|
||||
enabled, intended for initial data collection. Perf runners no longer
|
||||
compare metrics against JSON baselines at assert time; the `baseline`
|
||||
blocks in the configs are reference-only and are copied into result JSON
|
||||
for reporting. Regression gates can be reintroduced once enough nightly
|
||||
data has been collected.
|
||||
enabled, intended for initial data collection. Performance assertions are
|
||||
skipped (`skip-performance-assertion: true`); the baseline values in the
|
||||
JSON configs are reference-only and will be promoted to regression gates
|
||||
once enough nightly data has been collected.
|
||||
|
||||
The user-facing equivalent is to launch one of the CLI commands above and
|
||||
generate 1024x1024 images with 50 denoising steps.
|
||||
|
||||
Binary file not shown.
Binary file not shown.
@@ -100,8 +100,9 @@ def _decode_kv_cache(
|
||||
head_size: int = 128,
|
||||
dtype: torch.dtype = torch.float16,
|
||||
) -> torch.Tensor:
|
||||
# vLLM >=0.23.0 KV cache layout: key/value live on dim 1.
|
||||
return torch.zeros((num_blocks, 2, block_size, num_kv_heads, head_size), dtype=dtype)
|
||||
# vLLM >=0.23.0 KV cache layout: (num_blocks, num_kv_heads, block_size,
|
||||
# 2*head_size) with K and V packed in the last dimension.
|
||||
return torch.zeros((num_blocks, num_kv_heads, block_size, 2 * head_size), dtype=dtype)
|
||||
|
||||
|
||||
def test_fish_kvcache_enabled_by_default(monkeypatch):
|
||||
@@ -221,10 +222,11 @@ def _vllm_kv_cache_shape(num_blocks, block_size, num_kv_heads, head_size):
|
||||
|
||||
|
||||
def test_fish_kvcache_backend_unbinds_kv_on_vllm_cache_layout(monkeypatch):
|
||||
# vLLM >=0.23.0 lays the KV cache out as (num_blocks, 2, block_size,
|
||||
# num_kv_heads, head_size). The backend must unbind on dim 1 so each of
|
||||
# key/value comes back as the 4-D (num_blocks, block_size, num_kv_heads,
|
||||
# head_size) tensor the decode kernel expects.
|
||||
# vLLM >=0.23.0 lays out the KV cache as (num_blocks, num_kv_heads,
|
||||
# block_size, 2*head_size) with K and V packed in the last dimension.
|
||||
# The backend must split on the content dim so each of key/value comes
|
||||
# back as the 4-D (num_blocks, block_size, num_kv_heads, head_size)
|
||||
# tensor the decode kernel expects.
|
||||
monkeypatch.setenv("VLLM_OMNI_FISH_KVCACHE_ATTN", "1")
|
||||
monkeypatch.setattr(fish_kvcache_backend, "is_available", lambda: True)
|
||||
monkeypatch.setattr(fish_kvcache_backend, "load_error", lambda: None)
|
||||
@@ -232,7 +234,8 @@ def test_fish_kvcache_backend_unbinds_kv_on_vllm_cache_layout(monkeypatch):
|
||||
|
||||
num_blocks, block_size, num_kv_heads, head_size = 8, 16, 8, 128
|
||||
cache_shape = _vllm_kv_cache_shape(num_blocks, block_size, num_kv_heads, head_size)
|
||||
assert cache_shape == (num_blocks, 2, block_size, num_kv_heads, head_size)
|
||||
# Upstream now returns a 4-D packed shape: (B, H, N, 2*D).
|
||||
assert cache_shape == (num_blocks, num_kv_heads, block_size, 2 * head_size)
|
||||
|
||||
captured = {}
|
||||
|
||||
@@ -262,20 +265,21 @@ def test_fish_kvcache_backend_unbinds_kv_on_vllm_cache_layout(monkeypatch):
|
||||
def test_fish_kvcache_vllm_cache_unbinds_then_falls_back_until_contiguous(monkeypatch):
|
||||
# End-to-end with the real vLLM cache layout and the real (unmocked) guard.
|
||||
#
|
||||
# The point of the unbind(1) fix: unbind(0) raised "too many values to
|
||||
# unpack" on the 5-D vLLM 0.23.0 layout, so the request crashed outright.
|
||||
# unbind(1) makes the request *work* again -- it unpacks cleanly and the
|
||||
# backend gracefully falls back to the original attention with correct
|
||||
# output. This test guards that fix and must NOT depend on the fast path
|
||||
# firing.
|
||||
# The point of the transpose+split fix: unbind(1) on the old 5-D layout
|
||||
# raised "too many values to unpack" on the vLLM 0.23.0 layout, and the
|
||||
# upstream now packs K/V into the last dimension (4-D shape). The
|
||||
# transpose(1, 2) + split correctly unpacks into separate key/value tensors
|
||||
# and the backend gracefully falls back to the original attention with
|
||||
# correct output. This test guards that fix and must NOT depend on the fast
|
||||
# path firing.
|
||||
#
|
||||
# It also documents a known limitation (tracked separately): unbind(1) on
|
||||
# the interleaved (num_blocks, 2, ...) layout yields *non-contiguous*
|
||||
# key/value views, which the guard rejects on the is_contiguous() check.
|
||||
# So the dimensions are compatible (4-D, block_size=16, head_size=128) --
|
||||
# the only thing keeping the fast path from engaging is contiguity, not
|
||||
# shape. Making the fast path actually fire would require a .contiguous()
|
||||
# copy and is out of scope here.
|
||||
# It also documents a known limitation (tracked separately):
|
||||
# transpose(1, 2) on the interleaved (B, H, N, 2*D) layout yields
|
||||
# *non-contiguous* key/value views, which the guard rejects on the
|
||||
# is_contiguous() check. So the dimensions are compatible (4-D,
|
||||
# block_size=16, head_size=128) -- the only thing keeping the fast path
|
||||
# from engaging is contiguity, not shape. Making the fast path actually
|
||||
# fire would require a .contiguous() copy and is out of scope here.
|
||||
fish_kvcache_backend.reset_fish_kvcache_attn_stats()
|
||||
monkeypatch.setenv("VLLM_OMNI_FISH_KVCACHE_ATTN", "1")
|
||||
monkeypatch.setattr(fish_kvcache_backend, "is_available", lambda: True)
|
||||
@@ -300,13 +304,14 @@ def test_fish_kvcache_vllm_cache_unbinds_then_falls_back_until_contiguous(monkey
|
||||
output = torch.zeros_like(query)
|
||||
kv_cache = torch.zeros(cache_shape, dtype=torch.float16)
|
||||
|
||||
# Root cause record: unbind(1) gives the right 4-D shape but a non-contiguous view.
|
||||
key_cache, value_cache = kv_cache.unbind(1)
|
||||
# Root cause record: transpose(1,2) + split gives the right 4-D shape but
|
||||
# a non-contiguous view.
|
||||
key_cache, value_cache = kv_cache.transpose(1, 2).split(head_size, dim=-1)
|
||||
assert tuple(key_cache.shape) == (num_blocks, block_size, num_kv_heads, head_size)
|
||||
assert not key_cache.is_contiguous()
|
||||
|
||||
# unbind(1) no longer crashes (unbind(0) would have raised here) and the
|
||||
# request completes via the correct fallback path.
|
||||
# transpose+split no longer crashes (unbind(1) would have raised here) and
|
||||
# the request completes via the correct fallback path.
|
||||
result = model.layers[0].self_attn.attn.impl.forward(None, query, None, None, kv_cache, _metadata(), output)
|
||||
|
||||
assert result is output
|
||||
|
||||
@@ -6,21 +6,13 @@ Unit tests for patch.py
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import base64
|
||||
import json
|
||||
import time
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
from pytest_mock import MockerFixture
|
||||
from vllm.benchmarks.lib.endpoint_request_func import RequestFuncInput
|
||||
|
||||
from vllm_omni.benchmarks.patch.patch import (
|
||||
MixRequestFuncOutput,
|
||||
async_request_openai_chat_omni_completions,
|
||||
async_request_openai_realtime_duplex,
|
||||
)
|
||||
from vllm_omni.experimental.fullduplex.client import RealtimeEventCollector
|
||||
from vllm_omni.benchmarks.patch.patch import MixRequestFuncOutput, async_request_openai_chat_omni_completions
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.benchmark, pytest.mark.cpu]
|
||||
|
||||
@@ -48,156 +40,6 @@ class MockResponse:
|
||||
pass
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_seed_tts_realtime_duplex_exports_per_request_metrics(monkeypatch):
|
||||
class FakeRealtimeClient:
|
||||
last_instance = None
|
||||
|
||||
def __init__(self, url):
|
||||
assert url == "ws://localhost:8000/v1/realtime?duplex=1"
|
||||
self.events = RealtimeEventCollector()
|
||||
self.configure_kwargs = None
|
||||
self.sent = []
|
||||
self.response_count = 0
|
||||
self.ack_count = 0
|
||||
type(self).last_instance = self
|
||||
|
||||
async def __aenter__(self):
|
||||
return self
|
||||
|
||||
async def __aexit__(self, *_args):
|
||||
return None
|
||||
|
||||
async def configure(self, model, **kwargs):
|
||||
assert model == "openbmb/MiniCPM-o-4_5"
|
||||
self.configure_kwargs = kwargs
|
||||
|
||||
async def send(self, event):
|
||||
self.sent.append(event)
|
||||
if event["type"] != "response.create":
|
||||
return
|
||||
self.response_count += 1
|
||||
response_id = f"resp-{self.response_count}"
|
||||
now = time.monotonic()
|
||||
self.events.add(
|
||||
{"type": "response.created", "response": {"id": response_id}},
|
||||
received_at_s=now + 0.01,
|
||||
)
|
||||
self.events.add(
|
||||
{
|
||||
"type": "response.audio_transcript.delta",
|
||||
"response_id": response_id,
|
||||
"delta": f"turn {self.response_count}",
|
||||
},
|
||||
received_at_s=now + 0.02,
|
||||
)
|
||||
self.events.add(
|
||||
{
|
||||
"type": "response.audio.delta",
|
||||
"response_id": response_id,
|
||||
"delta": base64.b64encode(b"\x00\x00" * 2400).decode(),
|
||||
"sample_rate_hz": 24_000,
|
||||
"metadata": {"audio_duration_ms": 100},
|
||||
},
|
||||
received_at_s=now + 0.03,
|
||||
)
|
||||
self.events.add(
|
||||
{"type": "response.done", "response": {"id": response_id}},
|
||||
received_at_s=now + 0.04,
|
||||
)
|
||||
|
||||
async def acknowledge_playback(self):
|
||||
self.ack_count += 1
|
||||
|
||||
async def close_session(self, **_kwargs):
|
||||
return None
|
||||
|
||||
monkeypatch.setattr(
|
||||
"vllm_omni.benchmarks.patch.patch.RealtimeDuplexClient",
|
||||
FakeRealtimeClient,
|
||||
)
|
||||
request_input = RequestFuncInput(
|
||||
model="openbmb/MiniCPM-o-4_5",
|
||||
model_name="openbmb/MiniCPM-o-4_5",
|
||||
prompt="hello",
|
||||
api_url="http://localhost:8000/v1/realtime",
|
||||
prompt_len=1,
|
||||
output_len=20,
|
||||
logprobs=None,
|
||||
multi_modal_content=None,
|
||||
ignore_eos=False,
|
||||
extra_body={"save_duplex_request_metrics": True},
|
||||
)
|
||||
request_input.seed_tts_speech_extra = {"ref_audio": "data:audio/wav;base64,AAAA"}
|
||||
request_input.seed_tts_system_prompt = "Speak exactly."
|
||||
request_input.seed_tts_turns = tuple(
|
||||
SimpleNamespace(utterance_id=f"utt-{index}", target_text=f"text {index}") for index in range(4)
|
||||
)
|
||||
|
||||
output = await async_request_openai_realtime_duplex(
|
||||
request_input,
|
||||
session=None,
|
||||
)
|
||||
|
||||
client = FakeRealtimeClient.last_instance
|
||||
assert client.configure_kwargs["native_duplex"] is False
|
||||
assert client.configure_kwargs["extra_body"] == {
|
||||
"ref_audio": "data:audio/wav;base64,AAAA",
|
||||
}
|
||||
assert client.sent == [
|
||||
event
|
||||
for index in range(4)
|
||||
for event in (
|
||||
{
|
||||
"type": "conversation.item.create",
|
||||
"item": {
|
||||
"type": "message",
|
||||
"role": "user",
|
||||
"content": [{"type": "input_text", "text": f"text {index}"}],
|
||||
},
|
||||
},
|
||||
{"type": "response.create"},
|
||||
)
|
||||
]
|
||||
assert client.ack_count == 4
|
||||
assert output.success is True
|
||||
assert output.generated_text == "turn 1 turn 2 turn 3 turn 4"
|
||||
assert output.audio_duration == pytest.approx(0.4)
|
||||
assert output.ttft > 0
|
||||
assert output.audio_ttfp > output.ttft
|
||||
assert output.audio_rtf > 0
|
||||
assert output.latency > 0
|
||||
assert output.tts_turn_pcm_bytes == [b"\x00\x00" * 2400] * 4
|
||||
assert output.tts_output_pcm_bytes == b"\x00\x00" * 9600
|
||||
session_id = output.duplex_request_metrics[0]["session_id"]
|
||||
assert len(output.duplex_request_metrics) == 4
|
||||
for index, metrics in enumerate(output.duplex_request_metrics):
|
||||
assert metrics == {
|
||||
"session_id": session_id,
|
||||
"request_index": index,
|
||||
"utterance_id": f"utt-{index}",
|
||||
"response_id": f"resp-{index + 1}",
|
||||
"source": "client_monotonic_receive",
|
||||
"measurement_origin": {
|
||||
"ttft": "conversation.item.create client send to first non-empty text delta",
|
||||
"ttfp": "conversation.item.create client send to first audio packet",
|
||||
"rtf": "request-start-to-last-audio receive time divided by emitted audio duration",
|
||||
},
|
||||
"ttft_ms": pytest.approx(20.0, abs=2.0),
|
||||
"ttfp_ms": pytest.approx(30.0, abs=2.0),
|
||||
"rtf": pytest.approx(0.3, abs=0.03),
|
||||
"audio_generation_ms": pytest.approx(30.0, abs=2.0),
|
||||
"audio_duration_ms": 100.0,
|
||||
}
|
||||
assert output.duplex_session_metrics == {
|
||||
"session_id": session_id,
|
||||
"audio_turn_count": 4,
|
||||
"mean_ttft_ms": pytest.approx(20.0, abs=2.0),
|
||||
"mean_ttfp_ms": pytest.approx(30.0, abs=2.0),
|
||||
"mean_rtf": pytest.approx(0.3, abs=0.03),
|
||||
}
|
||||
|
||||
|
||||
def create_sse_chunk(data_dict):
|
||||
"""Helper to create SSE formatted chunk"""
|
||||
return f"data: {json.dumps(data_dict)}\n\n".encode()
|
||||
|
||||
@@ -42,10 +42,7 @@ from benchmarks.accuracy.text_to_image.gbench import (
|
||||
summarize_generated_records as summarize_gebench_generated_records,
|
||||
summarize_gebench_results,
|
||||
)
|
||||
from tests.e2e.accuracy.qwen3_omni.qwen3_omni_acc_bench_core import (
|
||||
build_serve_common_argv,
|
||||
seed_tts_bench_argv,
|
||||
)
|
||||
from tests.e2e.accuracy.qwen3_omni.qwen3_omni_acc_bench_core import seed_tts_bench_argv
|
||||
from tests.e2e.accuracy.qwen3_omni.run_qwen_omni_acc_benchmark import sync_dataset_env_from_ns
|
||||
from vllm_omni.benchmarks.data_modules.seed_tts_dataset import resolve_seed_tts_root
|
||||
|
||||
@@ -60,25 +57,6 @@ def test_seed_tts_bench_argv_preserves_hf_repo_id_from_env(monkeypatch):
|
||||
assert argv[dataset_idx + 1] == "zhaochenyang20/seed-tts-eval"
|
||||
|
||||
|
||||
def test_build_serve_common_argv_accepts_realtime_seed_tts_route(tmp_path):
|
||||
argv = build_serve_common_argv(
|
||||
host="127.0.0.1",
|
||||
port=8000,
|
||||
model="openbmb/MiniCPM-o-4_5",
|
||||
num_prompts=4,
|
||||
max_concurrency=1,
|
||||
num_warmups=0,
|
||||
percentile_metrics="ttft,audio_ttfp,audio_rtf",
|
||||
result_dir=tmp_path,
|
||||
result_filename="result.json",
|
||||
backend="openai-realtime-tts",
|
||||
endpoint="/v1/realtime",
|
||||
)
|
||||
|
||||
assert argv[argv.index("--backend") + 1] == "openai-realtime-tts"
|
||||
assert argv[argv.index("--endpoint") + 1] == "/v1/realtime"
|
||||
|
||||
|
||||
def test_sync_dataset_env_preserves_seed_tts_hf_repo_id(monkeypatch):
|
||||
ns = argparse.Namespace(
|
||||
daily_omni_repo=None,
|
||||
|
||||
@@ -1,152 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Offline Daily-Omni sources: local mirrors must not fall back to the Hub id."""
|
||||
|
||||
import json
|
||||
import os
|
||||
import tarfile
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm_omni.benchmarks.data_modules.daily_omni_dataset import (
|
||||
DailyOmniDataset,
|
||||
daily_omni_local_qa_json,
|
||||
daily_omni_local_videos_dir,
|
||||
ensure_daily_omni_hub_root,
|
||||
resolve_daily_omni_local_root,
|
||||
)
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.benchmark]
|
||||
|
||||
_HUB_ID = "liarliar/Daily-Omni"
|
||||
|
||||
|
||||
def _write_qa_json(root: Path) -> Path:
|
||||
qa = root / "qa.json"
|
||||
qa.write_text(json.dumps([{"Question": "q", "Answer": "A", "video_id": "vid0"}]), encoding="utf-8")
|
||||
return qa
|
||||
|
||||
|
||||
def _stage_offline_hub_cache(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, *, with_qa: bool = True) -> Path:
|
||||
"""Build a hub cache holding ``liarliar/Daily-Omni`` and cut off all network access.
|
||||
|
||||
Reproduces the air-gapped NPU runner: the dataset is pre-staged with ``hf download``, so it
|
||||
only exists in the *hub* cache -- never in the ``datasets`` cache that ``load_dataset`` reads.
|
||||
"""
|
||||
constants = pytest.importorskip("huggingface_hub").constants
|
||||
|
||||
cache = tmp_path / "hub"
|
||||
repo_dir = cache / "datasets--liarliar--Daily-Omni"
|
||||
snapshot = repo_dir / "snapshots" / "deadbeef"
|
||||
snapshot.mkdir(parents=True)
|
||||
if with_qa:
|
||||
_write_qa_json(snapshot)
|
||||
(repo_dir / "refs").mkdir()
|
||||
(repo_dir / "refs" / "main").write_text("deadbeef", encoding="utf-8")
|
||||
|
||||
monkeypatch.setattr(constants, "HF_HUB_CACHE", str(cache))
|
||||
monkeypatch.setattr(constants, "HF_HUB_OFFLINE", True)
|
||||
return snapshot
|
||||
|
||||
|
||||
def test_resolve_local_root_returns_none_for_hub_id() -> None:
|
||||
assert resolve_daily_omni_local_root("liarliar/Daily-Omni") is None
|
||||
assert resolve_daily_omni_local_root("") is None
|
||||
assert resolve_daily_omni_local_root(None) is None
|
||||
|
||||
|
||||
def test_plain_local_mirror(tmp_path: Path) -> None:
|
||||
_write_qa_json(tmp_path)
|
||||
(tmp_path / "Videos" / "vid0").mkdir(parents=True)
|
||||
|
||||
root = resolve_daily_omni_local_root(str(tmp_path))
|
||||
assert root == tmp_path.resolve()
|
||||
assert daily_omni_local_qa_json(root) == (tmp_path / "qa.json").resolve()
|
||||
assert daily_omni_local_videos_dir(root) == root / "Videos"
|
||||
|
||||
|
||||
def test_hf_cache_dir_resolves_to_snapshot(tmp_path: Path) -> None:
|
||||
cache_dir = tmp_path / "datasets--liarliar--Daily-Omni"
|
||||
snapshot = cache_dir / "snapshots" / "deadbeef"
|
||||
snapshot.mkdir(parents=True)
|
||||
_write_qa_json(snapshot)
|
||||
(cache_dir / "refs").mkdir()
|
||||
(cache_dir / "refs" / "main").write_text("deadbeef", encoding="utf-8")
|
||||
|
||||
root = resolve_daily_omni_local_root(str(cache_dir))
|
||||
assert root == snapshot.resolve()
|
||||
assert daily_omni_local_qa_json(root) == (snapshot / "qa.json").resolve()
|
||||
|
||||
|
||||
def test_hf_cache_dir_without_refs_picks_newest_snapshot(tmp_path: Path) -> None:
|
||||
cache_dir = tmp_path / "datasets--liarliar--Daily-Omni"
|
||||
older = cache_dir / "snapshots" / "aaa"
|
||||
newer = cache_dir / "snapshots" / "bbb"
|
||||
older.mkdir(parents=True)
|
||||
newer.mkdir(parents=True)
|
||||
os.utime(older, (1, 1))
|
||||
os.utime(newer, (10, 10))
|
||||
|
||||
assert resolve_daily_omni_local_root(str(cache_dir)) == newer.resolve()
|
||||
|
||||
|
||||
def test_videos_tar_is_extracted(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
monkeypatch.setenv("HF_HOME", str(tmp_path / "hf_home"))
|
||||
root = tmp_path / "mirror"
|
||||
(root / "Videos" / "vid0").mkdir(parents=True)
|
||||
(root / "Videos" / "vid0" / "vid0_video.mp4").write_bytes(b"fake")
|
||||
with tarfile.open(root / "Videos.tar", "w") as tf:
|
||||
tf.add(root / "Videos", arcname="Videos")
|
||||
# Only the tarball is left in the mirror.
|
||||
for child in (root / "Videos").rglob("*"):
|
||||
if child.is_file():
|
||||
child.unlink()
|
||||
(root / "Videos" / "vid0").rmdir()
|
||||
(root / "Videos").rmdir()
|
||||
|
||||
videos_dir = daily_omni_local_videos_dir(root)
|
||||
assert videos_dir is not None
|
||||
assert (videos_dir / "vid0" / "vid0_video.mp4").is_file()
|
||||
|
||||
# Second call reuses the extraction instead of redoing it.
|
||||
assert daily_omni_local_videos_dir(root) == videos_dir
|
||||
|
||||
|
||||
def test_missing_qa_json_reports_none(tmp_path: Path) -> None:
|
||||
root = resolve_daily_omni_local_root(str(tmp_path))
|
||||
assert daily_omni_local_qa_json(root) is None
|
||||
assert daily_omni_local_videos_dir(root) is None
|
||||
|
||||
|
||||
def test_hub_id_resolves_to_offline_cache(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
snapshot = _stage_offline_hub_cache(tmp_path, monkeypatch)
|
||||
|
||||
root = ensure_daily_omni_hub_root(_HUB_ID)
|
||||
|
||||
assert root == snapshot.resolve()
|
||||
assert daily_omni_local_qa_json(root) == (snapshot / "qa.json").resolve()
|
||||
|
||||
|
||||
def test_dataset_loads_hub_id_without_network(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""The regression this file exists for: a Hub id must not require ``load_dataset``."""
|
||||
snapshot = _stage_offline_hub_cache(tmp_path, monkeypatch)
|
||||
# Any use of `datasets` here means we went back to the path that cannot read the hub cache.
|
||||
monkeypatch.setattr("vllm_omni.benchmarks.data_modules.daily_omni_dataset.load_dataset", None)
|
||||
|
||||
dataset = DailyOmniDataset(dataset_path=_HUB_ID, random_seed=0)
|
||||
|
||||
assert dataset.qa_json_path == (snapshot / "qa.json").resolve()
|
||||
assert [row["video_id"] for row in dataset.data] == ["vid0"]
|
||||
|
||||
|
||||
def test_uncached_hub_id_reports_both_attempts(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
_stage_offline_hub_cache(tmp_path, monkeypatch, with_qa=False)
|
||||
monkeypatch.setattr("vllm_omni.benchmarks.data_modules.daily_omni_dataset.load_dataset", None)
|
||||
|
||||
with pytest.raises(RuntimeError) as excinfo:
|
||||
DailyOmniDataset(dataset_path="someone-else/Daily-Omni", random_seed=0)
|
||||
|
||||
message = str(excinfo.value)
|
||||
assert "someone-else/Daily-Omni" in message
|
||||
assert "snapshot_download failed" in message
|
||||
assert "--daily-omni-qa-json" in message
|
||||
@@ -1,237 +0,0 @@
|
||||
"""Tests for Daily-Omni ``--daily-omni-pack-mode`` packing recipes.
|
||||
|
||||
The ``minicpm-interleave`` mode is an accuracy-critical protocol: MiniCPM-o only reaches
|
||||
OpenBMB's reported Daily-Omni score when frames and 1s audio segments arrive strictly
|
||||
interleaved, with ``max_slice_nums=1`` / ``use_image_id=False`` and no Qwen system prompt.
|
||||
These tests pin that contract so a refactor cannot silently regress the score.
|
||||
|
||||
vllm stubs are installed by tests/benchmarks/conftest.py before collection.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
# Load the data module directly (bypasses vllm_omni.__init__ heavy imports).
|
||||
_REPO_ROOT = Path(__file__).resolve().parents[2]
|
||||
_MODULE_PATH = _REPO_ROOT / "vllm_omni" / "benchmarks" / "data_modules" / "daily_omni_dataset.py"
|
||||
_MODULE_NAME = "vllm_omni.benchmarks.data_modules.daily_omni_dataset"
|
||||
|
||||
if _MODULE_NAME not in sys.modules:
|
||||
_spec = importlib.util.spec_from_file_location(_MODULE_NAME, _MODULE_PATH)
|
||||
_mod = importlib.util.module_from_spec(_spec)
|
||||
sys.modules[_MODULE_NAME] = _mod
|
||||
_spec.loader.exec_module(_mod)
|
||||
|
||||
from vllm_omni.benchmarks.data_modules.daily_omni_dataset import ( # noqa: E402
|
||||
DailyOmniDataset,
|
||||
_uniform_sample_indices,
|
||||
)
|
||||
|
||||
_VIDEO_ID = "vid001"
|
||||
_NUM_FRAMES = 5
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def video_dir(tmp_path: Path) -> Path:
|
||||
"""Official Daily-Omni ``Videos/`` layout with placeholder media files."""
|
||||
per_video = tmp_path / "Videos" / _VIDEO_ID
|
||||
per_video.mkdir(parents=True)
|
||||
(per_video / f"{_VIDEO_ID}_video.mp4").write_bytes(b"fake-mp4")
|
||||
(per_video / f"{_VIDEO_ID}_audio.wav").write_bytes(b"fake-wav")
|
||||
return tmp_path / "Videos"
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def qa_json(tmp_path: Path) -> Path:
|
||||
path = tmp_path / "qa.json"
|
||||
path.write_text(
|
||||
json.dumps(
|
||||
[
|
||||
{
|
||||
"video_id": _VIDEO_ID,
|
||||
"question": "What happens first?",
|
||||
"choice": {"A": "a", "B": "b", "C": "c", "D": "d"},
|
||||
"answer": "A",
|
||||
"task_type": "AV Event Alignment",
|
||||
"video_duration": "30s",
|
||||
}
|
||||
]
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
return path
|
||||
|
||||
|
||||
def _make_dataset(qa_json: Path, video_dir: Path, **kwargs) -> DailyOmniDataset:
|
||||
return DailyOmniDataset(
|
||||
qa_json_path=str(qa_json),
|
||||
video_dir=str(video_dir),
|
||||
disable_shuffle=True,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
|
||||
def _stub_extract(monkeypatch, dataset: DailyOmniDataset, counter: list[int]) -> None:
|
||||
"""Replace decord/ffmpeg extraction with deterministic in-memory frames + segments."""
|
||||
from PIL import Image
|
||||
|
||||
def fake_extract(video_path, *, audio_path, include_audio, **_kw):
|
||||
counter.append(1)
|
||||
frames = [Image.new("RGB", (16, 16), color=(i, i, i)) for i in range(_NUM_FRAMES)]
|
||||
segments = [np.zeros(1600, dtype=np.float32) for _ in range(_NUM_FRAMES)] if include_audio else []
|
||||
return frames, segments
|
||||
|
||||
monkeypatch.setattr(dataset, "_extract_minicpm_frame_audio_segments", fake_extract)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# _uniform_sample_indices
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("n", "k"), [(10, 4), (100, 64), (7, 7), (3, 5), (1, 1)])
|
||||
def test_uniform_sample_indices_matches_numpy_linspace(n: int, k: int) -> None:
|
||||
got = _uniform_sample_indices(n, k)
|
||||
if n <= k:
|
||||
assert got == list(range(n))
|
||||
else:
|
||||
assert got == [int(i) for i in np.linspace(0, n - 1, k, dtype=int)]
|
||||
|
||||
|
||||
def test_uniform_sample_indices_non_positive_k() -> None:
|
||||
assert _uniform_sample_indices(10, 0) == []
|
||||
assert _uniform_sample_indices(10, -1) == []
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# minicpm-interleave packing
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_minicpm_interleave_alternates_image_and_audio(monkeypatch, qa_json, video_dir) -> None:
|
||||
ds = _make_dataset(qa_json, video_dir, pack_mode="minicpm-interleave", input_mode="all")
|
||||
_stub_extract(monkeypatch, ds, [])
|
||||
|
||||
parts, extra, position = ds._compose_daily_omni_multimodal(_VIDEO_ID, None)
|
||||
|
||||
assert position == "first"
|
||||
assert [p["type"] for p in parts] == ["image_url", "audio_url"] * _NUM_FRAMES
|
||||
assert extra["mm_processor_kwargs"] == {
|
||||
"use_audio_in_video": False,
|
||||
"max_slice_nums": 1,
|
||||
"use_image_id": False,
|
||||
}
|
||||
|
||||
|
||||
def test_minicpm_interleave_visual_mode_emits_frames_only(monkeypatch, qa_json, video_dir) -> None:
|
||||
ds = _make_dataset(qa_json, video_dir, pack_mode="minicpm-interleave", input_mode="visual")
|
||||
_stub_extract(monkeypatch, ds, [])
|
||||
|
||||
parts, _extra, _position = ds._compose_daily_omni_multimodal(_VIDEO_ID, None)
|
||||
|
||||
assert [p["type"] for p in parts] == ["image_url"] * _NUM_FRAMES
|
||||
|
||||
|
||||
def test_minicpm_interleave_writes_file_urls_under_video_dir(monkeypatch, qa_json, video_dir) -> None:
|
||||
ds = _make_dataset(qa_json, video_dir, pack_mode="minicpm-interleave", input_mode="all")
|
||||
_stub_extract(monkeypatch, ds, [])
|
||||
|
||||
parts, _extra, _position = ds._compose_daily_omni_multimodal(_VIDEO_ID, None)
|
||||
|
||||
segment_dir = video_dir / ".minicpm_daily_omni_interleave" / _VIDEO_ID
|
||||
for part in parts:
|
||||
url = part[part["type"]]["url"]
|
||||
assert url.startswith("file://")
|
||||
assert Path(url[len("file://") :]).parent == segment_dir.resolve()
|
||||
assert sorted(p.name for p in segment_dir.glob("frame_*.jpg")) == [f"frame_{i:04d}.jpg" for i in range(_NUM_FRAMES)]
|
||||
|
||||
|
||||
def test_minicpm_interleave_inline_mode_emits_data_urls(monkeypatch, qa_json, video_dir) -> None:
|
||||
ds = _make_dataset(
|
||||
qa_json,
|
||||
video_dir,
|
||||
pack_mode="minicpm-interleave",
|
||||
input_mode="all",
|
||||
inline_local_video=True,
|
||||
)
|
||||
_stub_extract(monkeypatch, ds, [])
|
||||
|
||||
parts, _extra, _position = ds._compose_daily_omni_multimodal(_VIDEO_ID, None)
|
||||
|
||||
assert parts[0]["image_url"]["url"].startswith("data:image/jpeg;base64,")
|
||||
assert parts[1]["audio_url"]["url"].startswith("data:audio/wav;base64,")
|
||||
assert not (video_dir / ".minicpm_daily_omni_interleave").exists()
|
||||
|
||||
|
||||
def test_minicpm_interleave_caches_file_parts_but_not_inline_payloads(monkeypatch, qa_json, video_dir) -> None:
|
||||
"""Inline base64 must not be retained: ~700 Daily-Omni videos would pin GBs of RSS."""
|
||||
file_calls: list[int] = []
|
||||
ds_file = _make_dataset(qa_json, video_dir, pack_mode="minicpm-interleave", input_mode="all")
|
||||
_stub_extract(monkeypatch, ds_file, file_calls)
|
||||
ds_file._compose_daily_omni_multimodal(_VIDEO_ID, None)
|
||||
ds_file._compose_daily_omni_multimodal(_VIDEO_ID, None)
|
||||
assert len(file_calls) == 1
|
||||
assert ds_file._minicpm_interleave_cache
|
||||
|
||||
inline_calls: list[int] = []
|
||||
ds_inline = _make_dataset(
|
||||
qa_json,
|
||||
video_dir,
|
||||
pack_mode="minicpm-interleave",
|
||||
input_mode="all",
|
||||
inline_local_video=True,
|
||||
)
|
||||
_stub_extract(monkeypatch, ds_inline, inline_calls)
|
||||
ds_inline._compose_daily_omni_multimodal(_VIDEO_ID, None)
|
||||
ds_inline._compose_daily_omni_multimodal(_VIDEO_ID, None)
|
||||
assert len(inline_calls) == 2
|
||||
assert not ds_inline._minicpm_interleave_cache
|
||||
|
||||
|
||||
def test_minicpm_interleave_missing_local_video_returns_none(qa_json, tmp_path) -> None:
|
||||
empty = tmp_path / "empty_videos"
|
||||
empty.mkdir()
|
||||
ds = _make_dataset(qa_json, empty, pack_mode="minicpm-interleave", input_mode="all")
|
||||
|
||||
parts, extra, _position = ds._compose_daily_omni_multimodal(_VIDEO_ID, None)
|
||||
|
||||
assert parts is None
|
||||
assert extra is None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# System prompt selection
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_minicpm_interleave_omits_qwen_system_message(monkeypatch, qa_json, video_dir) -> None:
|
||||
ds = _make_dataset(qa_json, video_dir, pack_mode="minicpm-interleave", input_mode="all")
|
||||
_stub_extract(monkeypatch, ds, [])
|
||||
parts, _extra, _position = ds._compose_daily_omni_multimodal(_VIDEO_ID, None)
|
||||
|
||||
messages = ds._build_daily_omni_openai_messages(parts, "q?", {"A": "a"})
|
||||
|
||||
assert [m["role"] for m in messages] == ["user"]
|
||||
|
||||
|
||||
def test_qwen_pack_mode_keeps_system_message(qa_json, video_dir) -> None:
|
||||
ds = _make_dataset(qa_json, video_dir, pack_mode="qwen", input_mode="all")
|
||||
|
||||
messages = ds._build_daily_omni_openai_messages([], "q?", {"A": "a"})
|
||||
|
||||
assert [m["role"] for m in messages] == ["system", "user"]
|
||||
assert "Qwen" in messages[0]["content"][0]["text"]
|
||||
|
||||
|
||||
def test_invalid_pack_mode_rejected(qa_json, video_dir) -> None:
|
||||
with pytest.raises(ValueError, match="pack_mode must be"):
|
||||
_make_dataset(qa_json, video_dir, pack_mode="bogus")
|
||||
@@ -28,13 +28,10 @@ if _MODULE_NAME not in sys.modules:
|
||||
_spec.loader.exec_module(_mod)
|
||||
|
||||
from vllm_omni.benchmarks.data_modules.seed_tts_dataset import ( # noqa: E402
|
||||
SeedTTSDataset,
|
||||
SeedTTSDesignDataset,
|
||||
SeedTTSDesignSampleRequest,
|
||||
SeedTTSSampleRequest,
|
||||
SeedTTSTextDataset,
|
||||
SeedTTSTextSampleRequest,
|
||||
SeedTTSTurn,
|
||||
)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -73,94 +70,6 @@ def mock_tokenizer(mocker):
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_seed_tts_dataset_groups_four_turns_with_one_reference(seed_tts_root, mock_tokenizer):
|
||||
ds = SeedTTSDataset(
|
||||
dataset_path=str(seed_tts_root),
|
||||
random_seed=0,
|
||||
locale="en",
|
||||
disable_shuffle=True,
|
||||
)
|
||||
|
||||
requests = ds.sample(
|
||||
mock_tokenizer,
|
||||
num_requests=1,
|
||||
turns_per_session=4,
|
||||
no_oversample=True,
|
||||
)
|
||||
|
||||
assert len(requests) == 1
|
||||
request = requests[0]
|
||||
assert [turn.utterance_id for turn in request.seed_tts_turns] == [
|
||||
"utt000",
|
||||
"utt001",
|
||||
"utt002",
|
||||
"utt003",
|
||||
]
|
||||
assert [turn.target_text for turn in request.seed_tts_turns] == [
|
||||
"target text 0",
|
||||
"target text 1",
|
||||
"target text 2",
|
||||
"target text 3",
|
||||
]
|
||||
assert request.seed_tts_speech_extra["ref_text"] == "ref text 0"
|
||||
assert request.seed_tts_ref_wav_path.endswith("utt000.wav")
|
||||
|
||||
|
||||
def test_seed_tts_eval_expands_grouped_pcm_by_turn():
|
||||
from vllm_omni.benchmarks.data_modules.seed_tts_eval import (
|
||||
_expand_seed_tts_turn_outputs,
|
||||
)
|
||||
|
||||
request = SeedTTSSampleRequest(
|
||||
prompt="target 0",
|
||||
prompt_len=4,
|
||||
expected_output_len=100,
|
||||
multi_modal_data=None,
|
||||
request_id="session-0",
|
||||
seed_tts_turns=tuple(
|
||||
SeedTTSTurn(utterance_id=f"utt-{index}", target_text=f"target {index}") for index in range(4)
|
||||
),
|
||||
)
|
||||
output = types.SimpleNamespace(
|
||||
success=True,
|
||||
tts_turn_pcm_bytes=[bytes([index]) for index in range(4)],
|
||||
)
|
||||
|
||||
requests, outputs = _expand_seed_tts_turn_outputs([request], [output])
|
||||
|
||||
assert [item.prompt for item in requests] == [f"target {index}" for index in range(4)]
|
||||
assert [item.seed_tts_utterance_id for item in requests] == [f"utt-{index}" for index in range(4)]
|
||||
assert [item.tts_output_pcm_bytes for item in outputs] == [bytes([index]) for index in range(4)]
|
||||
|
||||
|
||||
def test_seed_tts_eval_keeps_session_pcm_for_single_turn_chat():
|
||||
"""Chat-omni Seed-TTS sets session PCM only; expand must not wipe it."""
|
||||
from vllm_omni.benchmarks.data_modules.seed_tts_eval import (
|
||||
_expand_seed_tts_turn_outputs,
|
||||
)
|
||||
|
||||
request = SeedTTSSampleRequest(
|
||||
prompt="target 0",
|
||||
prompt_len=4,
|
||||
expected_output_len=100,
|
||||
multi_modal_data=None,
|
||||
request_id="session-0",
|
||||
seed_tts_turns=(SeedTTSTurn(utterance_id="utt-0", target_text="target 0"),),
|
||||
)
|
||||
pcm = b"\x01\x02\x03\x04"
|
||||
output = types.SimpleNamespace(
|
||||
success=True,
|
||||
tts_output_pcm_bytes=pcm,
|
||||
tts_turn_pcm_bytes=None,
|
||||
)
|
||||
|
||||
requests, outputs = _expand_seed_tts_turn_outputs([request], [output])
|
||||
|
||||
assert len(requests) == 1
|
||||
assert requests[0].prompt == "target 0"
|
||||
assert outputs[0].tts_output_pcm_bytes == pcm
|
||||
|
||||
|
||||
def test_seed_tts_text_dataset_omits_ref_audio(seed_tts_root, mock_tokenizer):
|
||||
ds = SeedTTSTextDataset(
|
||||
dataset_path=str(seed_tts_root),
|
||||
|
||||
@@ -73,6 +73,9 @@ def _run_resumable_segment_stop(
|
||||
|
||||
sched._update_request_with_output.side_effect = stop_request
|
||||
sched._handle_stopped_request.return_value = session_finished
|
||||
# vLLM 0.26 returns (kv_xfer_params, ec_xfer_params); an unconfigured
|
||||
# MagicMock iterates empty and fails to unpack at the call site.
|
||||
sched._free_request.return_value = (None, None)
|
||||
sched.chunk_transfer_adapter = None
|
||||
sched.running = [session]
|
||||
sched.waiting_for_transfer_free = set()
|
||||
@@ -137,6 +140,23 @@ def test_resumable_session_terminal_is_not_marked_as_segment_boundary() -> None:
|
||||
assert output.is_segment_finished is False
|
||||
|
||||
|
||||
def test_update_from_output_settles_in_flight_tokens() -> None:
|
||||
"""vLLM 0.26: schedule() increments num_in_flight_tokens per scheduled
|
||||
token; update_from_output must decrement it symmetrically. If the
|
||||
decrement is dropped the counter grows monotonically and both readers
|
||||
(allocate_slots, _connector_finished) clamp
|
||||
max(0, num_computed_tokens - num_in_flight_tokens) to zero forever,
|
||||
silently freezing sliding-window block freeing.
|
||||
"""
|
||||
session = _make_request()
|
||||
session.status = RequestStatus.RUNNING
|
||||
session.num_in_flight_tokens = 1 # as left by schedule() for this step
|
||||
|
||||
_run_resumable_segment_stop(session)
|
||||
|
||||
assert session.num_in_flight_tokens == 0
|
||||
|
||||
|
||||
def test_running_decode_step_without_inter_stage_payload_does_not_raise() -> None:
|
||||
"""A decode step that neither stops nor carries an inter-stage payload.
|
||||
|
||||
@@ -242,52 +262,9 @@ def test_explicit_streaming_payload_replaces_placeholder_prompt() -> None:
|
||||
"tts_token_ids": [10, 20],
|
||||
"meta": {"replace_streaming_prompt": True},
|
||||
}
|
||||
update.model_intermediate_buffer = {
|
||||
"ids": {"tts": [41, 42, 99]},
|
||||
"meta": {"turn_eos_token_id": 99},
|
||||
}
|
||||
|
||||
sched._update_request_as_session(session, update)
|
||||
|
||||
assert session.prompt_token_ids == [10, 20]
|
||||
assert session.additional_information == update.additional_information
|
||||
assert session.model_intermediate_buffer == {
|
||||
"ids": {"tts": [41, 42, 99]},
|
||||
"meta": {"turn_eos_token_id": 99},
|
||||
}
|
||||
assert session.status == RequestStatus.WAITING
|
||||
|
||||
|
||||
def test_model_intermediate_streaming_payload_replaces_computed_prompt() -> None:
|
||||
sched = _make_scheduler(stage_id=1)
|
||||
sched.chunk_transfer_adapter = SimpleNamespace(
|
||||
receives_chunks=False,
|
||||
segment_finished_requests=set(),
|
||||
)
|
||||
session = _make_request()
|
||||
session.status = RequestStatus.WAITING_FOR_STREAMING_REQ
|
||||
session.prompt_token_ids = [0] * 59
|
||||
session._all_token_ids.clear()
|
||||
session._all_token_ids.extend(session.prompt_token_ids)
|
||||
session.num_prompt_tokens = 59
|
||||
session.num_computed_tokens = 59
|
||||
update = _make_update([0] * 10)
|
||||
update.additional_information = None
|
||||
update.model_intermediate_buffer = {
|
||||
"ids": {"tts": list(range(8))},
|
||||
"hidden_states": {"tts": [[0.0]] * 8},
|
||||
"meta": {
|
||||
"replace_streaming_prompt": True,
|
||||
"next_stage_prompt_len": 10,
|
||||
},
|
||||
}
|
||||
|
||||
sched._update_request_as_session(session, update)
|
||||
|
||||
assert session.prompt_token_ids == [0] * 10
|
||||
assert list(session._all_token_ids) == [0] * 10
|
||||
assert session.num_prompt_tokens == 10
|
||||
assert session.num_computed_tokens == 0
|
||||
assert session.additional_information is None
|
||||
assert session.model_intermediate_buffer == update.model_intermediate_buffer
|
||||
assert session.status == RequestStatus.WAITING
|
||||
@@ -54,18 +54,6 @@ class _SchedulerStub(OmniGenerationScheduler):
|
||||
raise AssertionError("unexpected enqueue for skipped_waiting miss")
|
||||
|
||||
|
||||
class _AsyncChunkStopSchedulerStub(OmniGenerationScheduler):
|
||||
def __init__(self) -> None:
|
||||
self.num_waiting_for_streaming_input = 0
|
||||
self.chunk_transfer_adapter = SimpleNamespace(receives_chunks=True)
|
||||
self.enqueued: list[Request] = []
|
||||
self.enqueued_statuses: list[RequestStatus] = []
|
||||
|
||||
def _enqueue_waiting_request(self, session: Request) -> None:
|
||||
self.enqueued.append(session)
|
||||
self.enqueued_statuses.append(session.status)
|
||||
|
||||
|
||||
def _make_request(**kwargs) -> Request:
|
||||
sp = SamplingParams(max_tokens=8)
|
||||
defaults = dict(
|
||||
@@ -145,21 +133,6 @@ def test_resumable_generation_stop_marks_segment_boundary() -> None:
|
||||
assert output.is_segment_finished is True
|
||||
|
||||
|
||||
def test_async_chunk_resumable_stop_rearms_connector_polling() -> None:
|
||||
sched = _AsyncChunkStopSchedulerStub()
|
||||
session = _make_request(request_id="req-async-chunk-next-segment")
|
||||
session.status = RequestStatus.FINISHED_STOPPED
|
||||
session.resumable = True
|
||||
|
||||
finished = sched._handle_stopped_request(session)
|
||||
|
||||
assert finished is False
|
||||
assert sched.enqueued == [session]
|
||||
assert sched.enqueued_statuses == [RequestStatus.WAITING]
|
||||
assert session.status == RequestStatus.WAITING
|
||||
assert sched.num_waiting_for_streaming_input == 0
|
||||
|
||||
|
||||
class TestReplaceSessionWithStreamingUpdate:
|
||||
def test_resets_tokens_and_prompt_from_update(self) -> None:
|
||||
sched = _SchedulerStub()
|
||||
|
||||
@@ -76,8 +76,9 @@ def test_finish_requests_purges_finished_request_from_running(
|
||||
)
|
||||
|
||||
def fake_finish_requests(self, request_ids, finished_status):
|
||||
# super() returning the abort tuple but leaving self.running untouched
|
||||
return [(finished.request_id, 0)]
|
||||
# vLLM 0.26 super() returns the finished Request objects but here
|
||||
# leaves self.running untouched to simulate the missed removal.
|
||||
return [finished]
|
||||
|
||||
monkeypatch.setattr(scheduler_mod.VLLMScheduler, "finish_requests", fake_finish_requests)
|
||||
|
||||
@@ -88,7 +89,7 @@ def test_finish_requests_purges_finished_request_from_running(
|
||||
)
|
||||
|
||||
assert scheduler.running == []
|
||||
assert result == [(finished.request_id, 0)]
|
||||
assert result == [finished]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("scheduler_cls", "scheduler_mod"), _SCHEDULER_PARAMS)
|
||||
@@ -111,7 +112,7 @@ def test_finish_requests_purges_untracked_request_from_running(
|
||||
)
|
||||
|
||||
def fake_finish_requests(self, request_ids, finished_status):
|
||||
return [(untracked.request_id, 0)]
|
||||
return [untracked]
|
||||
|
||||
monkeypatch.setattr(scheduler_mod.VLLMScheduler, "finish_requests", fake_finish_requests)
|
||||
|
||||
@@ -122,7 +123,7 @@ def test_finish_requests_purges_untracked_request_from_running(
|
||||
)
|
||||
|
||||
assert scheduler.running == []
|
||||
assert result == [(untracked.request_id, 0)]
|
||||
assert result == [untracked]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("scheduler_cls", "scheduler_mod"), _SCHEDULER_PARAMS)
|
||||
@@ -136,7 +137,7 @@ def test_finish_requests_leaves_healthy_running_intact(
|
||||
is conservative and won't aggressively drop healthy entries.
|
||||
"""
|
||||
alive = _StubRequest("req-alive", RequestStatus.RUNNING)
|
||||
leaving_id = "req-leaving"
|
||||
leaving = _StubRequest("req-leaving", RequestStatus.FINISHED_STOPPED)
|
||||
scheduler = _make_scheduler(
|
||||
scheduler_cls,
|
||||
requests={alive.request_id: alive},
|
||||
@@ -145,13 +146,13 @@ def test_finish_requests_leaves_healthy_running_intact(
|
||||
)
|
||||
|
||||
def fake_finish_requests(self, request_ids, finished_status):
|
||||
return [(leaving_id, 0)]
|
||||
return [leaving]
|
||||
|
||||
monkeypatch.setattr(scheduler_mod.VLLMScheduler, "finish_requests", fake_finish_requests)
|
||||
|
||||
scheduler_cls.finish_requests(
|
||||
scheduler,
|
||||
[leaving_id],
|
||||
[leaving.request_id],
|
||||
RequestStatus.FINISHED_STOPPED,
|
||||
)
|
||||
|
||||
|
||||
@@ -53,7 +53,11 @@ def test_finish_requests_cleans_input_coordinator_for_finished_ids(
|
||||
def fake_finish_requests(self, request_ids, finished_status):
|
||||
assert request_ids == ["req-a", "req-b"]
|
||||
assert finished_status == RequestStatus.FINISHED_STOPPED
|
||||
return [("req-a", 0), ("req-b", 1)]
|
||||
# vLLM 0.26 returns the finished Request objects, not (req_id, client_index) tuples.
|
||||
return [
|
||||
SimpleNamespace(request_id="req-a", client_index=0),
|
||||
SimpleNamespace(request_id="req-b", client_index=1),
|
||||
]
|
||||
|
||||
monkeypatch.setattr(scheduler_mod.VLLMScheduler, "finish_requests", fake_finish_requests)
|
||||
|
||||
@@ -63,7 +67,7 @@ def test_finish_requests_cleans_input_coordinator_for_finished_ids(
|
||||
RequestStatus.FINISHED_STOPPED,
|
||||
)
|
||||
|
||||
assert result == [("req-a", 0), ("req-b", 1)]
|
||||
assert [(r.request_id, r.client_index) for r in result] == [("req-a", 0), ("req-b", 1)]
|
||||
assert coordinator.freed == ["req-a", "req-b"]
|
||||
|
||||
|
||||
@@ -82,7 +86,7 @@ def test_ar_free_request_cleans_input_coordinator_on_normal_free() -> None:
|
||||
|
||||
result = OmniARScheduler._free_request(scheduler, DummyRequest())
|
||||
|
||||
assert result is None
|
||||
assert result == (None, None)
|
||||
assert coordinator.freed == ["req-free"]
|
||||
assert scheduler._omits_kv_transfer_cache == {}
|
||||
assert scheduler._new_prompt_len_snapshot == {}
|
||||
|
||||
+150
-93
@@ -3,13 +3,15 @@ import os
|
||||
import re
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.helpers.mark import get_hardware_mark_list, hardware_marks
|
||||
from tests.dfx.reliability.helpers import list_remote_process_pids_by_pattern, post_chat_completions_raw
|
||||
from tests.helpers.mark import hardware_marks
|
||||
from tests.helpers.runtime import OmniServerParams
|
||||
from tests.helpers.stage_config import modify_stage_config
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
@@ -117,6 +119,32 @@ def create_unique_server_pytest_params(
|
||||
]
|
||||
|
||||
|
||||
def create_benchmark_pytest_params(
|
||||
benchmark_configs: list[dict[str, Any]],
|
||||
server_to_benchmark_mapping: dict[str, dict],
|
||||
) -> list[Any]:
|
||||
"""Like :func:`create_benchmark_indices`, but wrap each index in ``pytest.param`` with JSON marks."""
|
||||
marks_by_name = _marks_by_test_name(benchmark_configs)
|
||||
params: list[Any] = []
|
||||
seen: set[str] = set()
|
||||
for config in benchmark_configs:
|
||||
test_name = config["test_name"]
|
||||
if test_name in seen:
|
||||
continue
|
||||
seen.add(test_name)
|
||||
params_list = get_benchmark_params_for_server(test_name, server_to_benchmark_mapping)
|
||||
id_suffixes = _unique_benchmark_param_id_suffixes(params_list)
|
||||
for idx, id_suffix in enumerate(id_suffixes):
|
||||
params.append(
|
||||
pytest.param(
|
||||
(test_name, idx),
|
||||
marks=marks_by_name.get(test_name, []),
|
||||
id=f"{test_name}-{id_suffix}",
|
||||
)
|
||||
)
|
||||
return params
|
||||
|
||||
|
||||
def create_paired_benchmark_pytest_params(
|
||||
server_entries: list[tuple[Any, str]],
|
||||
params_by_test_name: dict[str, list[Any]],
|
||||
@@ -329,6 +357,10 @@ def supports_video_generation(model_name: str) -> bool:
|
||||
return any(key in lower for key in ("wan", "video", "i2v", "t2v"))
|
||||
|
||||
|
||||
def supports_chat_generation(model_name: str) -> bool:
|
||||
return not supports_video_generation(model_name)
|
||||
|
||||
|
||||
def parse_stage_devices(stage_config_path: str) -> str:
|
||||
text = Path(stage_config_path).read_text(encoding="utf-8")
|
||||
raw_devices: list[str] = re.findall(r"^\s*devices:\s*\"?([0-9,\s]+)\"?\s*$", text, flags=re.MULTILINE)
|
||||
@@ -357,6 +389,48 @@ def assert_fault_exception(exc: Exception, error_keywords: tuple[str, ...]) -> N
|
||||
assert any(key in text for key in error_keywords), f"unexpected error under fault injection: {exc}"
|
||||
|
||||
|
||||
def wait_chat_request_ready(host: str, port: int, model: str, timeout_sec: int = 180) -> None:
|
||||
"""Poll a minimal chat request until success."""
|
||||
deadline = time.time() + timeout_sec
|
||||
payload = json.dumps(
|
||||
{
|
||||
"model": model,
|
||||
"messages": [{"role": "user", "content": "Say hello in one short sentence."}],
|
||||
"stream": False,
|
||||
"modalities": ["text"],
|
||||
}
|
||||
)
|
||||
last_error: str | None = None
|
||||
while time.time() < deadline:
|
||||
try:
|
||||
status, body = post_chat_completions_raw(host, port, payload)
|
||||
if status == 200:
|
||||
return
|
||||
last_error = f"http={status} body={body[:200]!r}"
|
||||
except Exception as exc: # noqa: BLE001
|
||||
last_error = str(exc)
|
||||
time.sleep(2)
|
||||
raise TimeoutError(f"runtime-teardown warmup request did not succeed within {timeout_sec}s: {last_error}")
|
||||
|
||||
|
||||
def assert_no_extra_worker_processes(
|
||||
baseline_pids: set[int],
|
||||
worker_pattern: str,
|
||||
timeout_sec: int = 60,
|
||||
) -> None:
|
||||
"""Ensure no extra worker PIDs remain after teardown."""
|
||||
deadline = time.time() + timeout_sec
|
||||
while time.time() < deadline:
|
||||
current = set(list_remote_process_pids_by_pattern(worker_pattern))
|
||||
extra = current - baseline_pids
|
||||
if not extra:
|
||||
return
|
||||
time.sleep(2)
|
||||
current = set(list_remote_process_pids_by_pattern(worker_pattern))
|
||||
extra = sorted(current - baseline_pids)
|
||||
assert not extra, f"orphan worker processes remain after container teardown: {extra}"
|
||||
|
||||
|
||||
def create_test_parameter_mapping(configs: list[dict[str, Any]]) -> dict[str, dict]:
|
||||
mapping = {}
|
||||
for config in configs:
|
||||
@@ -446,16 +520,36 @@ def _unique_benchmark_param_id_suffixes(params_list: list[dict[str, Any]]) -> li
|
||||
return unique
|
||||
|
||||
|
||||
# Device marketing-name tokens for ``get_runtime_resource_label`` only.
|
||||
# Longer / more specific tokens must appear before shorter substrings
|
||||
# (e.g. ``H200`` before ``H20``, ``910B4`` before ``910``).
|
||||
_RUNTIME_DEVICE_ALIASES: tuple[str, ...] = (
|
||||
def extract_mark_resource_label(mark_field: Any) -> str:
|
||||
"""Return a filename-safe hardware label from ``mark.hardware_marks.res`` values.
|
||||
|
||||
Example: ``{"cuda": "H100"}`` -> ``"H100"``; multiple platforms join with ``-``.
|
||||
|
||||
Prefer :func:`get_runtime_resource_label` for perf result filenames so labels
|
||||
reflect the machine that actually ran the benchmark.
|
||||
"""
|
||||
if isinstance(mark_field, list):
|
||||
for item in mark_field:
|
||||
if isinstance(item, dict) and "hardware_marks" in item:
|
||||
return extract_mark_resource_label(item)
|
||||
return "na"
|
||||
if not isinstance(mark_field, dict):
|
||||
return "na"
|
||||
hw = mark_field.get("hardware_marks")
|
||||
if not isinstance(hw, dict):
|
||||
return "na"
|
||||
res = hw.get("res")
|
||||
if not isinstance(res, dict) or not res:
|
||||
return "na"
|
||||
labels = [_safe_filename_token(value) for value in res.values()]
|
||||
return "-".join(labels) if labels else "na"
|
||||
|
||||
|
||||
_KNOWN_RUNTIME_RESOURCE_TOKENS: tuple[str, ...] = (
|
||||
"H100",
|
||||
"H800",
|
||||
"H200",
|
||||
"H20",
|
||||
"B200",
|
||||
"GB200",
|
||||
"L40S",
|
||||
"L40",
|
||||
"L4",
|
||||
@@ -484,7 +578,7 @@ def _normalize_runtime_device_label(raw: str) -> str:
|
||||
if not raw or not str(raw).strip():
|
||||
return "na"
|
||||
upper = str(raw).upper()
|
||||
for token in _RUNTIME_DEVICE_ALIASES:
|
||||
for token in _KNOWN_RUNTIME_RESOURCE_TOKENS:
|
||||
if token.upper() in upper:
|
||||
return _safe_filename_token(token)
|
||||
compact = re.sub(r"[^a-zA-Z0-9]+", "", str(raw))
|
||||
@@ -537,41 +631,34 @@ def resource_label_for_filename(resource_label: str | None) -> str:
|
||||
return token
|
||||
|
||||
|
||||
def is_hardware_nested_baseline(baseline: dict[str, Any]) -> bool:
|
||||
"""True for ``{"H100": {"metric": ...}, "A3": {...}}`` baselines.
|
||||
|
||||
Detection is structural + hardware-label allowlist (not metric-name hints):
|
||||
|
||||
- every top-level value is a non-empty dict (per-hardware metric map);
|
||||
- every top-level key is in :func:`tests.helpers.mark.get_hardware_mark_list`
|
||||
(``[hardware-resource]`` markers from ``pyproject.toml``).
|
||||
|
||||
Metric names under each hardware bucket are unconstrained.
|
||||
Runtime device aliases (:data:`_RUNTIME_DEVICE_ALIASES`) are unrelated.
|
||||
"""
|
||||
if not baseline:
|
||||
return False
|
||||
if not all(isinstance(value, dict) and bool(value) for value in baseline.values()):
|
||||
return False
|
||||
allowed = {label.upper() for label in get_hardware_mark_list()}
|
||||
return all((token := str(key).strip()) and token.upper() in allowed for key in baseline.keys())
|
||||
def extract_configs_resource_label(configs: list[dict[str, Any]]) -> str:
|
||||
"""Return runtime hardware label for perf result filenames."""
|
||||
del configs
|
||||
return get_runtime_resource_label()
|
||||
|
||||
|
||||
def resolve_baseline_value(
|
||||
baseline_raw: Any,
|
||||
*,
|
||||
sweep_index: int | None,
|
||||
max_concurrency: Any = None,
|
||||
request_rate: Any = None,
|
||||
) -> Any:
|
||||
"""Pick the baseline threshold for one metric at this sweep step.
|
||||
|
||||
Accepts a sweep-aligned list/tuple (indexed by ``sweep_index``) or a scalar.
|
||||
"""
|
||||
"""Pick the baseline threshold for this sweep step."""
|
||||
if baseline_raw is None:
|
||||
return None
|
||||
return 100000
|
||||
if isinstance(baseline_raw, dict):
|
||||
raise TypeError(
|
||||
"per-metric baseline dict keyed by concurrency/request-rate is not supported; "
|
||||
"use a sweep-aligned list or a scalar under each hardware bucket"
|
||||
if max_concurrency is not None:
|
||||
for key in (max_concurrency, str(max_concurrency)):
|
||||
if key in baseline_raw:
|
||||
return baseline_raw[key]
|
||||
if request_rate is not None:
|
||||
for key in (request_rate, str(request_rate)):
|
||||
if key in baseline_raw:
|
||||
return baseline_raw[key]
|
||||
raise KeyError(
|
||||
f"baseline dict has no key for max_concurrency={max_concurrency!r} "
|
||||
f"or request_rate={request_rate!r}; keys={list(baseline_raw.keys())!r}"
|
||||
)
|
||||
if isinstance(baseline_raw, (list, tuple)):
|
||||
if sweep_index is None:
|
||||
@@ -582,62 +669,22 @@ def resolve_baseline_value(
|
||||
return baseline_raw
|
||||
|
||||
|
||||
def resolve_baseline_for_sweep(
|
||||
baseline_config: dict[str, Any] | None,
|
||||
def _baseline_thresholds_for_step(
|
||||
baseline_data: dict[str, Any],
|
||||
*,
|
||||
sweep_index: int | None = None,
|
||||
max_concurrency: Any = None,
|
||||
request_rate: Any = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Resolve sweep-dependent baselines while keeping every hardware bucket.
|
||||
|
||||
Expects hardware-nested input only::
|
||||
|
||||
{"H100": {"throughput_qps": [a, b, c]}, "A3": {"throughput_qps": [x, y, z]}}
|
||||
|
||||
with ``sweep_index=2`` (e.g. concurrency 32) becomes::
|
||||
|
||||
{"H100": {"throughput_qps": c}, "A3": {"throughput_qps": z}}
|
||||
|
||||
Per-metric values may be a sweep-aligned list or a scalar.
|
||||
"""
|
||||
if not baseline_config:
|
||||
return {}
|
||||
if not is_hardware_nested_baseline(baseline_config):
|
||||
allowed = sorted(get_hardware_mark_list())
|
||||
unknown = [
|
||||
str(key).strip()
|
||||
for key in baseline_config.keys()
|
||||
if str(key).strip().upper() not in {label.upper() for label in allowed}
|
||||
]
|
||||
nested_shape = all(isinstance(value, dict) and bool(value) for value in baseline_config.values())
|
||||
if nested_shape and unknown:
|
||||
raise ValueError(
|
||||
"baseline top-level keys must be [hardware-resource] pytest markers from "
|
||||
"pyproject.toml (tool.pytest.ini_options.markers). "
|
||||
f"Unknown hardware label(s): {unknown!r}. "
|
||||
f"Known markers: {allowed!r}. "
|
||||
"Add a marker such as '\"A100: [hardware-resource] Tests that require A100 GPU\"' "
|
||||
"to pyproject.toml, then retry."
|
||||
)
|
||||
raise ValueError(
|
||||
"baseline must be hardware-nested with [hardware-resource] marker names as "
|
||||
"top-level keys, e.g. "
|
||||
'{"H100": {"throughput_qps": [...]}, "A3": {"custom_metric": 1.0}}. '
|
||||
f"got top-level keys={list(baseline_config.keys())!r}; "
|
||||
f"known markers={allowed!r}. "
|
||||
"If you need a new hardware bucket, add it to pyproject.toml markers with "
|
||||
"the [hardware-resource] tag."
|
||||
)
|
||||
|
||||
"""Resolve baseline config to one threshold per metric for this iteration."""
|
||||
return {
|
||||
hardware: {
|
||||
metric_name: resolve_baseline_value(
|
||||
baseline_raw,
|
||||
sweep_index=sweep_index,
|
||||
)
|
||||
for metric_name, baseline_raw in metrics.items()
|
||||
}
|
||||
for hardware, metrics in baseline_config.items()
|
||||
if isinstance(metrics, dict)
|
||||
metric_name: resolve_baseline_value(
|
||||
baseline_raw,
|
||||
sweep_index=sweep_index,
|
||||
max_concurrency=max_concurrency,
|
||||
request_rate=request_rate,
|
||||
)
|
||||
for metric_name, baseline_raw in baseline_data.items()
|
||||
}
|
||||
|
||||
|
||||
@@ -650,17 +697,17 @@ def run_benchmark(
|
||||
*,
|
||||
baseline_config: dict[str, Any] | None = None,
|
||||
sweep_index: int | None = None,
|
||||
request_rate: Any | None = None,
|
||||
max_concurrency: Any | None = None,
|
||||
random_input_len: Any | None = None,
|
||||
random_output_len: Any | None = None,
|
||||
resource_label: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Run one ``vllm bench serve --omni`` iteration and return parsed metrics.
|
||||
|
||||
After ``vllm bench`` writes the JSON, ``result["baseline"]`` stores the
|
||||
sweep-resolved baseline (when ``baseline_config`` is provided). Hardware-nested
|
||||
maps keep every hardware bucket, but each metric is reduced to the value for
|
||||
this concurrency / request-rate step. If the benchmark exits without writing a
|
||||
result file, ``result_omni_template.json`` is used as a fallback.
|
||||
After ``vllm bench`` writes the JSON, ``result["baseline"]`` holds the resolved per-metric thresholds
|
||||
(when ``baseline_config`` is provided). If the benchmark exits without writing a result file,
|
||||
``result_omni_template.json`` is used as a fallback.
|
||||
"""
|
||||
current_dt = datetime.now().strftime("%Y%m%d-%H%M%S")
|
||||
ri = _safe_filename_token(random_input_len)
|
||||
@@ -727,10 +774,11 @@ def run_benchmark(
|
||||
result = json.load(f)
|
||||
|
||||
if baseline_config:
|
||||
# Keep every hardware bucket; resolve list metrics to this sweep step.
|
||||
result["baseline"] = resolve_baseline_for_sweep(
|
||||
result["baseline"] = _baseline_thresholds_for_step(
|
||||
baseline_config,
|
||||
sweep_index=sweep_index,
|
||||
request_rate=request_rate,
|
||||
max_concurrency=max_concurrency,
|
||||
)
|
||||
else:
|
||||
result["baseline"] = {}
|
||||
@@ -752,3 +800,12 @@ def pytest_addoption(parser: pytest.Parser) -> None:
|
||||
default=None,
|
||||
help=("Path to benchmark config JSON. Example: --test-config-file tests/dfx/perf/tests/test_tts.json"),
|
||||
)
|
||||
parser.addoption(
|
||||
"--assert-baseline",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help=(
|
||||
"When set, omni/diffusion perf runners compare metrics against the baseline block in the JSON config "
|
||||
"(default: off)."
|
||||
),
|
||||
)
|
||||
@@ -21,12 +21,10 @@
|
||||
"num-input-images": 0,
|
||||
"enable-negative-prompt": false,
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"throughput_qps": 0,
|
||||
"latency_mean": 0,
|
||||
"peak_memory_mb_max": 0,
|
||||
"peak_memory_mb_mean": 0
|
||||
}
|
||||
"throughput_qps": 0,
|
||||
"latency_mean": 0,
|
||||
"peak_memory_mb_max": 0,
|
||||
"peak_memory_mb_mean": 0
|
||||
}
|
||||
},
|
||||
"result": {
|
||||
|
||||
@@ -46,11 +46,9 @@
|
||||
"median_audio_duration_s": 0,
|
||||
"p99_audio_duration_s": 0,
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"mean_ttft_ms": 0,
|
||||
"mean_audio_ttfp_ms": 0,
|
||||
"mean_audio_rtf": 0
|
||||
}
|
||||
"mean_ttft_ms": 0,
|
||||
"mean_audio_ttfp_ms": 0,
|
||||
"mean_audio_rtf": 0
|
||||
},
|
||||
"random_input_len": 0,
|
||||
"random_output_len": 0
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
import json
|
||||
import os
|
||||
import threading
|
||||
from collections.abc import Callable
|
||||
from contextlib import AbstractContextManager, ExitStack, contextmanager
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
@@ -15,10 +13,15 @@ from tests.dfx.conftest import (
|
||||
get_runtime_resource_label,
|
||||
is_diffusion_perf_config,
|
||||
load_benchmark_configs,
|
||||
resolve_baseline_value,
|
||||
run_benchmark,
|
||||
)
|
||||
from tests.helpers.runtime import OmniServer
|
||||
|
||||
# Compare metrics to each test JSON ``baseline`` block only when pytest is run with ``--assert-baseline``
|
||||
# (registered in ``tests/dfx/conftest.py``; default: off). ``run_benchmark`` and ``_resolve_baseline_value`` are
|
||||
# defined in the same module.
|
||||
#
|
||||
# Optional JSON field ``mark`` is applied as pytest marks via
|
||||
# ``create_paired_omni_benchmark_pytest_params`` (e.g. ``"mark": [{"hardware_marks":
|
||||
# {"res": {"cuda": "H100"}, "num_cards": 2}}, "full_model", "omni"]``).
|
||||
@@ -60,78 +63,35 @@ paired_benchmark_params = create_paired_omni_benchmark_pytest_params(BENCHMARK_C
|
||||
_omni_server_lock = threading.Lock()
|
||||
|
||||
|
||||
class _SingleActiveContext:
|
||||
"""Reuse one active context while its configuration key is unchanged."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self._key: Any = None
|
||||
self._stack: ExitStack | None = None
|
||||
self._value: Any = None
|
||||
|
||||
def acquire(self, key: Any, factory: Callable[[], AbstractContextManager[Any]]) -> Any:
|
||||
if self._stack is not None and key == self._key:
|
||||
return self._value
|
||||
|
||||
self.close()
|
||||
stack = ExitStack()
|
||||
value = stack.enter_context(factory())
|
||||
self._key = key
|
||||
self._stack = stack
|
||||
self._value = value
|
||||
return value
|
||||
|
||||
def close(self) -> None:
|
||||
stack = self._stack
|
||||
self._key = None
|
||||
self._stack = None
|
||||
self._value = None
|
||||
if stack is not None:
|
||||
stack.close()
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _start_omni_server(server_param):
|
||||
test_name, model, stage_config_path, stage_overrides, extra_cli_args, use_omni = server_param
|
||||
|
||||
print(f"Starting OmniServer with test: {test_name}, model: {model}")
|
||||
|
||||
server_args: list[str] = []
|
||||
if use_omni:
|
||||
server_args += ["--stage-init-timeout", "600", "--init-timeout", "900"]
|
||||
# --deploy-config and --stage-overrides compose at the CLI (see vllm_omni/entrypoints/utils.py):
|
||||
# deploy-config sets the base; stage-overrides are applied on top. Both can be set.
|
||||
if stage_config_path:
|
||||
server_args = ["--deploy-config", stage_config_path] + server_args
|
||||
if stage_overrides:
|
||||
server_args = ["--stage-overrides", stage_overrides] + server_args
|
||||
if extra_cli_args:
|
||||
server_args = list(extra_cli_args) + server_args
|
||||
with OmniServer(model, server_args, use_omni=use_omni) as server:
|
||||
server.test_name = test_name
|
||||
print("OmniServer started successfully")
|
||||
yield server
|
||||
print("OmniServer stopping...")
|
||||
|
||||
print("OmniServer stopped")
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def omni_server_context():
|
||||
def omni_server(request):
|
||||
"""Start vLLM-Omni server as a subprocess with actual model weights.
|
||||
Reuse it for adjacent benchmark cases with the same server configuration.
|
||||
Uses session scope so the server starts only once for the entire test session.
|
||||
Multi-stage initialization can take 10-20+ minutes.
|
||||
"""
|
||||
with _omni_server_lock:
|
||||
active_context = _SingleActiveContext()
|
||||
try:
|
||||
yield active_context
|
||||
finally:
|
||||
active_context.close()
|
||||
test_name, model, stage_config_path, stage_overrides, extra_cli_args, use_omni = request.param
|
||||
|
||||
print(f"Starting OmniServer with test: {test_name}, model: {model}")
|
||||
|
||||
@pytest.fixture
|
||||
def omni_server(request, omni_server_context):
|
||||
return omni_server_context.acquire(request.param, lambda: _start_omni_server(request.param))
|
||||
server_args: list[str] = []
|
||||
if use_omni:
|
||||
server_args += ["--stage-init-timeout", "600", "--init-timeout", "900"]
|
||||
# --deploy-config and --stage-overrides compose at the CLI (see vllm_omni/entrypoints/utils.py):
|
||||
# deploy-config sets the base; stage-overrides are applied on top. Both can be set.
|
||||
if stage_config_path:
|
||||
server_args = ["--deploy-config", stage_config_path] + server_args
|
||||
if stage_overrides:
|
||||
server_args = ["--stage-overrides", stage_overrides] + server_args
|
||||
if extra_cli_args:
|
||||
server_args = list(extra_cli_args) + server_args
|
||||
with OmniServer(model, server_args, use_omni=use_omni) as server:
|
||||
server.test_name = test_name
|
||||
print("OmniServer started successfully")
|
||||
yield server
|
||||
print("OmniServer stopping...")
|
||||
|
||||
print("OmniServer stopped")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
@@ -157,19 +117,36 @@ def benchmark_params(request):
|
||||
}
|
||||
|
||||
|
||||
def assert_result(result, params, num_prompt) -> None:
|
||||
def assert_result(
|
||||
result,
|
||||
params,
|
||||
num_prompt,
|
||||
*,
|
||||
assert_baseline: bool,
|
||||
sweep_index: int | None = None,
|
||||
max_concurrency: Any | None = None,
|
||||
request_rate: Any | None = None,
|
||||
) -> None:
|
||||
assert result["completed"] == num_prompt, "Request failures exist"
|
||||
expected_audio_turns = params.get("expected_duplex_audio_turns_per_session")
|
||||
if expected_audio_turns is not None:
|
||||
session_metrics = result.get("duplex_session_metrics")
|
||||
assert isinstance(session_metrics, list), "Duplex session metrics are missing"
|
||||
assert len(session_metrics) == num_prompt, (
|
||||
f"Expected {num_prompt} duplex session metric rows, got {len(session_metrics)}"
|
||||
if not assert_baseline:
|
||||
return
|
||||
baseline_data = params.get("baseline", {})
|
||||
for metric_name, baseline_raw in baseline_data.items():
|
||||
current_value = result[metric_name]
|
||||
baseline_value = resolve_baseline_value(
|
||||
baseline_raw,
|
||||
sweep_index=sweep_index,
|
||||
max_concurrency=max_concurrency,
|
||||
request_rate=request_rate,
|
||||
)
|
||||
assert all(
|
||||
isinstance(metric, dict) and metric.get("audio_turn_count") == expected_audio_turns
|
||||
for metric in session_metrics
|
||||
), f"Not every duplex session emitted {expected_audio_turns} audio turns"
|
||||
if "throughput" in metric_name:
|
||||
if current_value <= baseline_value:
|
||||
print(
|
||||
f"ERROR: Throughput test results were below baseline: {metric_name}: {current_value} > {baseline_value}"
|
||||
)
|
||||
else:
|
||||
if current_value >= baseline_value:
|
||||
print(f"ERROR: Test results exceeded baseline: {metric_name}: {current_value} < {baseline_value}")
|
||||
|
||||
|
||||
@pytest.mark.benchmark
|
||||
@@ -178,7 +155,7 @@ def assert_result(result, params, num_prompt) -> None:
|
||||
paired_benchmark_params,
|
||||
indirect=["omni_server", "benchmark_params"],
|
||||
)
|
||||
def test_performance_benchmark(omni_server, benchmark_params):
|
||||
def test_performance_benchmark(omni_server, benchmark_params, request):
|
||||
test_name = benchmark_params["test_name"]
|
||||
params = benchmark_params["params"]
|
||||
dataset_name = params.get("dataset_name", "")
|
||||
@@ -190,6 +167,7 @@ def test_performance_benchmark(omni_server, benchmark_params):
|
||||
print(f"Running benchmark for model: {model}")
|
||||
print(f"Benchmark parameters: {benchmark_params}")
|
||||
|
||||
assert_baseline = request.config.getoption("--assert-baseline", default=False)
|
||||
resource_label = get_runtime_resource_label()
|
||||
|
||||
def to_list(value, default=None):
|
||||
@@ -223,7 +201,6 @@ def test_performance_benchmark(omni_server, benchmark_params):
|
||||
"enabled",
|
||||
"eval_phase",
|
||||
"trust_remote_code",
|
||||
"expected_duplex_audio_turns_per_session",
|
||||
}
|
||||
|
||||
for key, value in params.items():
|
||||
@@ -249,7 +226,7 @@ def test_performance_benchmark(omni_server, benchmark_params):
|
||||
break
|
||||
|
||||
# QPS / request-rate sweep
|
||||
for sweep_index, (qps, num_prompt) in enumerate(zip(qps_list, num_prompt_list)):
|
||||
for i, (qps, num_prompt) in enumerate(zip(qps_list, num_prompt_list)):
|
||||
args = args + ["--request-rate", str(qps), "--num-prompts", str(num_prompt)]
|
||||
result = run_benchmark(
|
||||
args=args,
|
||||
@@ -258,15 +235,24 @@ def test_performance_benchmark(omni_server, benchmark_params):
|
||||
dataset_name=dataset_name,
|
||||
num_prompt=num_prompt,
|
||||
baseline_config=params.get("baseline"),
|
||||
sweep_index=sweep_index,
|
||||
sweep_index=i,
|
||||
request_rate=qps,
|
||||
max_concurrency=None,
|
||||
random_input_len=params.get("random_input_len"),
|
||||
random_output_len=params.get("random_output_len"),
|
||||
resource_label=resource_label,
|
||||
)
|
||||
assert_result(result, params, num_prompt)
|
||||
assert_result(
|
||||
result,
|
||||
params,
|
||||
num_prompt,
|
||||
assert_baseline=assert_baseline,
|
||||
sweep_index=i,
|
||||
request_rate=qps,
|
||||
)
|
||||
|
||||
# concurrency test
|
||||
for sweep_index, (concurrency, num_prompt) in enumerate(zip(max_concurrency_list, num_prompt_list)):
|
||||
for i, (concurrency, num_prompt) in enumerate(zip(max_concurrency_list, num_prompt_list)):
|
||||
args = args + ["--max-concurrency", str(concurrency), "--num-prompts", str(num_prompt), "--request-rate", "inf"]
|
||||
result = run_benchmark(
|
||||
args=args,
|
||||
@@ -275,9 +261,18 @@ def test_performance_benchmark(omni_server, benchmark_params):
|
||||
dataset_name=dataset_name,
|
||||
num_prompt=num_prompt,
|
||||
baseline_config=params.get("baseline"),
|
||||
sweep_index=sweep_index,
|
||||
sweep_index=i,
|
||||
request_rate=None,
|
||||
max_concurrency=concurrency,
|
||||
random_input_len=params.get("random_input_len"),
|
||||
random_output_len=params.get("random_output_len"),
|
||||
resource_label=resource_label,
|
||||
)
|
||||
assert_result(result, params, num_prompt)
|
||||
assert_result(
|
||||
result,
|
||||
params,
|
||||
num_prompt,
|
||||
assert_baseline=assert_baseline,
|
||||
sweep_index=i,
|
||||
max_concurrency=concurrency,
|
||||
)
|
||||
@@ -13,6 +13,8 @@ A config JSON file may be passed via --test-config-file. If omitted, every ``*.j
|
||||
pytest run_diffusion_benchmark.py -m "diffusion"
|
||||
pytest run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_vllm_omni.json
|
||||
|
||||
Optional: ``--assert-baseline`` compares metrics to the ``baseline`` block in each benchmark entry (default: off).
|
||||
|
||||
Optional JSON field ``mark`` is applied as pytest marks on that case via
|
||||
``pytest.param`` (e.g. ``"mark": [{"hardware_marks": {"res": {"cuda": "H100"}, "num_cards": 1}}, "full_model", "diffusion"]``).
|
||||
|
||||
@@ -44,7 +46,6 @@ from tests.dfx.conftest import (
|
||||
get_runtime_resource_label,
|
||||
hardware_json_value,
|
||||
is_diffusion_perf_config,
|
||||
resolve_baseline_for_sweep,
|
||||
resolve_pytest_marks,
|
||||
resource_label_for_filename,
|
||||
)
|
||||
@@ -273,6 +274,11 @@ def _write_result_record(record: dict[str, Any]) -> Path:
|
||||
return target
|
||||
|
||||
|
||||
def _append_to_aggregated_file(record: dict[str, Any]) -> None:
|
||||
"""Backward-compatible wrapper; prefer :func:`_write_result_record`."""
|
||||
_write_result_record(record)
|
||||
|
||||
|
||||
_server_lock = threading.Lock()
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -349,17 +355,6 @@ def _kill_process_tree(pid: int) -> None:
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _resolve_offline_model(model: str) -> str:
|
||||
"""Under HF_HUB_OFFLINE, resolve a HF repo id to its local snapshot dir."""
|
||||
import huggingface_hub
|
||||
|
||||
if not model or os.path.isdir(model) or not huggingface_hub.constants.HF_HUB_OFFLINE:
|
||||
return model
|
||||
from huggingface_hub import snapshot_download
|
||||
|
||||
return snapshot_download(model, local_files_only=True)
|
||||
|
||||
|
||||
class DiffusionServer:
|
||||
"""Start a vLLM-Omni diffusion model server as a subprocess.
|
||||
|
||||
@@ -377,7 +372,7 @@ class DiffusionServer:
|
||||
port: int | None = None,
|
||||
) -> None:
|
||||
self.server_cfg: dict[str, Any] = server_cfg
|
||||
self.model = _resolve_offline_model(server_cfg["model"])
|
||||
self.model = server_cfg["model"]
|
||||
self.serve_args = server_cfg["serve_args"]
|
||||
self.host = "127.0.0.1"
|
||||
self.port = port if port is not None else get_open_port(self.host)
|
||||
@@ -672,7 +667,7 @@ def run_benchmark(
|
||||
with tempfile.NamedTemporaryFile(mode="w", suffix=".json", prefix="diffusion_bench_tmp_", delete=False) as tmp:
|
||||
tmp_result_file = Path(tmp.name)
|
||||
|
||||
exclude_keys = {"baseline", "dataset", "task", "name"}
|
||||
exclude_keys = {"baseline", "dataset", "task", "name", "skip-performance-assertion"}
|
||||
|
||||
cmd = [
|
||||
sys.executable,
|
||||
@@ -766,17 +761,12 @@ def run_benchmark(
|
||||
completed = metrics.get("completed_requests", metrics.get("completed", 0))
|
||||
failed = metrics.get("failed_requests", metrics.get("failed", 0))
|
||||
|
||||
# Persist sweep-resolved baseline from params (already narrowed in _build_run_params).
|
||||
baseline = params.get("baseline") or {}
|
||||
metrics["baseline"] = baseline
|
||||
|
||||
record: dict[str, Any] = {
|
||||
"test_name": test_name,
|
||||
"endpoint": endpoint,
|
||||
"timestamp": timestamp,
|
||||
"server_params": server_cfg.get("server_params"),
|
||||
"benchmark_params": params,
|
||||
"baseline": baseline,
|
||||
"result": metrics,
|
||||
"log_file": str(log_file),
|
||||
"Model": model,
|
||||
@@ -826,14 +816,70 @@ def run_benchmark(
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _resolve_baseline_value(
|
||||
baseline_raw: Any,
|
||||
*,
|
||||
sweep_index: int | None,
|
||||
max_concurrency: Any = None,
|
||||
request_rate: Any = None,
|
||||
) -> Any:
|
||||
"""Pick the baseline threshold for this sweep step."""
|
||||
if baseline_raw is None:
|
||||
return 100000
|
||||
if isinstance(baseline_raw, dict):
|
||||
if max_concurrency is not None:
|
||||
for key in (max_concurrency, str(max_concurrency)):
|
||||
if key in baseline_raw:
|
||||
return baseline_raw[key]
|
||||
if request_rate is not None:
|
||||
for key in (request_rate, str(request_rate)):
|
||||
if key in baseline_raw:
|
||||
return baseline_raw[key]
|
||||
raise KeyError(
|
||||
f"baseline dict has no key for max_concurrency={max_concurrency!r} "
|
||||
f"or request_rate={request_rate!r}; keys={list(baseline_raw.keys())!r}"
|
||||
)
|
||||
if isinstance(baseline_raw, (list, tuple)):
|
||||
if sweep_index is None:
|
||||
raise ValueError("sweep_index is required when baseline is a list or tuple")
|
||||
return baseline_raw[sweep_index]
|
||||
return baseline_raw
|
||||
|
||||
|
||||
def assert_result(
|
||||
result: dict[str, Any],
|
||||
params: dict[str, Any],
|
||||
num_prompts: int,
|
||||
*,
|
||||
sweep_index: int | None = None,
|
||||
max_concurrency: Any = None,
|
||||
request_rate: Any = None,
|
||||
assert_baseline: bool = True,
|
||||
) -> None:
|
||||
"""Assert that the expected number of requests completed."""
|
||||
"""Assert that benchmark metrics satisfy the configured baselines."""
|
||||
completed = result.get("completed_requests", result.get("completed", 0))
|
||||
assert completed == num_prompts, f"Expected {num_prompts} completed requests, got {completed}"
|
||||
|
||||
if not assert_baseline:
|
||||
return
|
||||
if params.get("skip-performance-assertion", False):
|
||||
print("Skipping performance assertions.")
|
||||
return
|
||||
|
||||
for metric, baseline_raw in params.get("baseline", {}).items():
|
||||
current = result.get(metric)
|
||||
assert current is not None, f"Metric '{metric}' not found in result: {list(result.keys())}"
|
||||
threshold = _resolve_baseline_value(
|
||||
baseline_raw,
|
||||
sweep_index=sweep_index,
|
||||
max_concurrency=max_concurrency,
|
||||
request_rate=request_rate,
|
||||
)
|
||||
if "throughput" in metric:
|
||||
assert current >= threshold, f"{metric}: {current:.4f} < baseline {threshold}"
|
||||
else:
|
||||
assert current <= threshold, f"{metric}: {current:.4f} > baseline {threshold}"
|
||||
|
||||
|
||||
def _default_benchmark_endpoint_for_task(task: str) -> str:
|
||||
"""Return the default client-side benchmark endpoint for a diffusion task."""
|
||||
@@ -875,11 +921,15 @@ def _build_run_params(
|
||||
if max_concurrency is not None:
|
||||
run_params["max-concurrency"] = max_concurrency
|
||||
if "baseline" in params:
|
||||
# Keep all hardware buckets; pick the metric value for this sweep step only.
|
||||
run_params["baseline"] = resolve_baseline_for_sweep(
|
||||
params.get("baseline"),
|
||||
sweep_index=sweep_index,
|
||||
)
|
||||
run_params["baseline"] = {
|
||||
metric: _resolve_baseline_value(
|
||||
baseline_raw,
|
||||
sweep_index=sweep_index,
|
||||
max_concurrency=max_concurrency,
|
||||
request_rate=request_rate,
|
||||
)
|
||||
for metric, baseline_raw in params["baseline"].items()
|
||||
}
|
||||
return run_params
|
||||
|
||||
|
||||
@@ -902,30 +952,36 @@ def _iter_sweep_runs(params: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
|
||||
sweep_runs: list[dict[str, Any]] = []
|
||||
|
||||
for sweep_index, (request_rate, num_prompts) in enumerate(zip(request_rate_list, num_prompt_list)):
|
||||
for i, (request_rate, num_prompts) in enumerate(zip(request_rate_list, num_prompt_list)):
|
||||
sweep_runs.append(
|
||||
{
|
||||
"params": _build_run_params(
|
||||
params,
|
||||
request_rate=request_rate,
|
||||
num_prompts=num_prompts,
|
||||
sweep_index=sweep_index,
|
||||
sweep_index=i,
|
||||
),
|
||||
"num_prompts": num_prompts,
|
||||
"sweep_index": i,
|
||||
"request_rate": request_rate,
|
||||
"max_concurrency": None,
|
||||
}
|
||||
)
|
||||
|
||||
for sweep_index, (max_concurrency, num_prompts) in enumerate(zip(max_concurrency_list, num_prompt_list)):
|
||||
for i, (max_concurrency, num_prompts) in enumerate(zip(max_concurrency_list, num_prompt_list)):
|
||||
sweep_runs.append(
|
||||
{
|
||||
"params": _build_run_params(
|
||||
params,
|
||||
max_concurrency=max_concurrency,
|
||||
num_prompts=num_prompts,
|
||||
sweep_index=i,
|
||||
request_rate="inf",
|
||||
sweep_index=sweep_index,
|
||||
),
|
||||
"num_prompts": num_prompts,
|
||||
"sweep_index": i,
|
||||
"request_rate": None,
|
||||
"max_concurrency": max_concurrency,
|
||||
}
|
||||
)
|
||||
|
||||
@@ -936,9 +992,11 @@ def _iter_sweep_runs(params: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
"params": _build_run_params(
|
||||
params,
|
||||
num_prompts=default_num_prompts,
|
||||
sweep_index=0,
|
||||
),
|
||||
"num_prompts": default_num_prompts,
|
||||
"sweep_index": None,
|
||||
"request_rate": None,
|
||||
"max_concurrency": None,
|
||||
}
|
||||
)
|
||||
|
||||
@@ -954,12 +1012,14 @@ def _iter_sweep_runs(params: dict[str, Any]) -> list[dict[str, Any]]:
|
||||
paired_benchmark_params,
|
||||
indirect=["diffusion_server", "benchmark_params"],
|
||||
)
|
||||
def test_diffusion_performance_benchmark(diffusion_server, benchmark_params):
|
||||
def test_diffusion_performance_benchmark(diffusion_server, benchmark_params, request):
|
||||
"""Run the diffusion performance benchmark and verify request completion.
|
||||
|
||||
One server is started per unique parallel configuration (module scope).
|
||||
For each server, all benchmark parameter sets defined in the config JSON
|
||||
are executed sequentially; metrics are recorded to the aggregated result file.
|
||||
|
||||
Pass ``--assert-baseline`` to compare ``throughput_qps``, ``latency_mean``, etc. to the JSON ``baseline`` block.
|
||||
"""
|
||||
test_name = benchmark_params["test_name"]
|
||||
params = benchmark_params["params"]
|
||||
@@ -1002,4 +1062,14 @@ def test_diffusion_performance_benchmark(diffusion_server, benchmark_params):
|
||||
print(f"\n Aggregated results: {_aggregated_result_file_for_source(str(source))}")
|
||||
print("=" * 60)
|
||||
|
||||
assert_result(result, sweep_run["num_prompts"])
|
||||
assert_baseline = request.config.getoption("--assert-baseline", default=False)
|
||||
|
||||
assert_result(
|
||||
result,
|
||||
params,
|
||||
sweep_run["num_prompts"],
|
||||
sweep_index=sweep_run["sweep_index"],
|
||||
max_concurrency=sweep_run["max_concurrency"],
|
||||
request_rate=sweep_run["request_rate"],
|
||||
assert_baseline=assert_baseline,
|
||||
)
|
||||
@@ -40,12 +40,10 @@
|
||||
"max-concurrency": 1,
|
||||
"enable-negative-prompt": true,
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"throughput_qps": 0.495,
|
||||
"latency_mean": 2.0238,
|
||||
"peak_memory_mb_max": 29393.4286,
|
||||
"peak_memory_mb_mean": 29392.7143
|
||||
}
|
||||
"throughput_qps": 0.495,
|
||||
"latency_mean": 2.0238,
|
||||
"peak_memory_mb_max": 29393.4286,
|
||||
"peak_memory_mb_mean": 29392.7143
|
||||
}
|
||||
}
|
||||
]
|
||||
@@ -92,12 +90,10 @@
|
||||
"enable-negative-prompt": true,
|
||||
"num-input-images": 1,
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"throughput_qps": 0.3451,
|
||||
"latency_mean": 2.9031,
|
||||
"peak_memory_mb_max": 30480.2857,
|
||||
"peak_memory_mb_mean": 30479.2857
|
||||
}
|
||||
"throughput_qps": 0.3451,
|
||||
"latency_mean": 2.9031,
|
||||
"peak_memory_mb_max": 30480.2857,
|
||||
"peak_memory_mb_mean": 30479.2857
|
||||
}
|
||||
}
|
||||
]
|
||||
@@ -143,12 +139,10 @@
|
||||
"max-concurrency": 1,
|
||||
"enable-negative-prompt": true,
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"throughput_qps": 0.5048,
|
||||
"latency_mean": 1.987,
|
||||
"peak_memory_mb_max": 29380.0,
|
||||
"peak_memory_mb_mean": 29380.0
|
||||
}
|
||||
"throughput_qps": 0.5048,
|
||||
"latency_mean": 1.987,
|
||||
"peak_memory_mb_max": 29380.0,
|
||||
"peak_memory_mb_mean": 29380.0
|
||||
}
|
||||
}
|
||||
]
|
||||
@@ -195,12 +189,10 @@
|
||||
"enable-negative-prompt": true,
|
||||
"num-input-images": 1,
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"throughput_qps": 0.149,
|
||||
"latency_mean": 6.7157,
|
||||
"peak_memory_mb_max": 30574.0,
|
||||
"peak_memory_mb_mean": 30574.0
|
||||
}
|
||||
"throughput_qps": 0.149,
|
||||
"latency_mean": 6.7157,
|
||||
"peak_memory_mb_max": 30574.0,
|
||||
"peak_memory_mb_mean": 30574.0
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
@@ -24,12 +24,10 @@
|
||||
"warmup-num-inference-steps": 28,
|
||||
"enable-negative-prompt": true,
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"throughput_qps": [0.0303, 0.0302, 0.0302],
|
||||
"latency_mean": [32.6, 62.2, 111.7],
|
||||
"peak_memory_mb_max": 40500,
|
||||
"peak_memory_mb_mean": 40500
|
||||
}
|
||||
"throughput_qps": {"1": 0.0303, "2": 0.0302, "4": 0.0302},
|
||||
"latency_mean": {"1": 32.6, "2": 62.2, "4": 111.7},
|
||||
"peak_memory_mb_max": 40500,
|
||||
"peak_memory_mb_mean": 40500
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
@@ -24,12 +24,10 @@
|
||||
"warmup-num-inference-steps": 28,
|
||||
"enable-negative-prompt": true,
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"throughput_qps": [0.0549, 0.0547, 0.0549],
|
||||
"latency_mean": [18.0, 34.3, 61.4],
|
||||
"peak_memory_mb_max": 40500,
|
||||
"peak_memory_mb_mean": 40500
|
||||
}
|
||||
"throughput_qps": {"1": 0.0549, "2": 0.0547, "4": 0.0549},
|
||||
"latency_mean": {"1": 18.0, "2": 34.3, "4": 61.4},
|
||||
"peak_memory_mb_max": 40500,
|
||||
"peak_memory_mb_mean": 40500
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
@@ -48,11 +48,9 @@
|
||||
"guardrails": false
|
||||
},
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"throughput_qps": 1.107,
|
||||
"latency_mean": 0.8929,
|
||||
"peak_memory_mb_mean": 80000.0
|
||||
}
|
||||
"throughput_qps": 1.107,
|
||||
"latency_mean": 0.8929,
|
||||
"peak_memory_mb_mean": 80000.0
|
||||
}
|
||||
}
|
||||
]
|
||||
@@ -117,11 +115,9 @@
|
||||
"use_duration_template": false
|
||||
},
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"throughput_qps": 0.027,
|
||||
"latency_mean": 36.6824,
|
||||
"peak_memory_mb_mean": 30155.4
|
||||
}
|
||||
"throughput_qps": 0.027,
|
||||
"latency_mean": 36.6824,
|
||||
"peak_memory_mb_mean": 30155.4
|
||||
}
|
||||
}
|
||||
]
|
||||
@@ -185,11 +181,9 @@
|
||||
"guardrails": false
|
||||
},
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"throughput_qps": 0.027,
|
||||
"latency_mean": 36.968,
|
||||
"peak_memory_mb_mean": 30443.6
|
||||
}
|
||||
"throughput_qps": 0.027,
|
||||
"latency_mean": 36.968,
|
||||
"peak_memory_mb_mean": 30443.6
|
||||
}
|
||||
}
|
||||
]
|
||||
@@ -257,11 +251,9 @@
|
||||
"condition_video_keep": "first"
|
||||
},
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"throughput_qps": 0.027,
|
||||
"latency_mean": 37.2117,
|
||||
"peak_memory_mb_mean": 30135.6
|
||||
}
|
||||
"throughput_qps": 0.027,
|
||||
"latency_mean": 37.2117,
|
||||
"peak_memory_mb_mean": 30135.6
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
@@ -46,14 +46,12 @@
|
||||
"seed_tts_locale": "en",
|
||||
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"median_audio_ttfp_ms": [
|
||||
223.3584
|
||||
],
|
||||
"median_audio_rtf": [
|
||||
0.4393
|
||||
]
|
||||
}
|
||||
"median_audio_ttfp_ms": [
|
||||
223.3584
|
||||
],
|
||||
"median_audio_rtf": [
|
||||
0.4393
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -72,17 +70,15 @@
|
||||
"seed_tts_locale": "en",
|
||||
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"median_audio_ttfp_ms": [
|
||||
276.575
|
||||
],
|
||||
"median_audio_rtf": [
|
||||
0.5073
|
||||
],
|
||||
"audio_throughput": [
|
||||
27.3728
|
||||
]
|
||||
}
|
||||
"median_audio_ttfp_ms": [
|
||||
276.575
|
||||
],
|
||||
"median_audio_rtf": [
|
||||
0.5073
|
||||
],
|
||||
"audio_throughput": [
|
||||
27.3728
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
@@ -140,13 +140,11 @@
|
||||
"num-prompts": 96,
|
||||
"skip-performance-assertion": true,
|
||||
"baseline": {
|
||||
"H200": {
|
||||
"throughput_qps": 0.1874,
|
||||
"latency_p99": 101.852,
|
||||
"latency_mean": 79.0385,
|
||||
"peak_memory_mb_max": 70000,
|
||||
"peak_memory_mb_mean": 70000
|
||||
}
|
||||
"throughput_qps": 0.1874,
|
||||
"latency_p99": 101.852,
|
||||
"latency_mean": 79.0385,
|
||||
"peak_memory_mb_max": 70000,
|
||||
"peak_memory_mb_mean": 70000
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
@@ -36,12 +36,10 @@
|
||||
"max-concurrency": 1,
|
||||
"skip-performance-assertion": true,
|
||||
"baseline": {
|
||||
"H200": {
|
||||
"throughput_qps": 0.21,
|
||||
"latency_p99": 4.7469,
|
||||
"peak_memory_mb_max": 101100,
|
||||
"peak_memory_mb_mean": 101100
|
||||
}
|
||||
"throughput_qps": 0.21,
|
||||
"latency_p99": 4.7469,
|
||||
"peak_memory_mb_max": 101100,
|
||||
"peak_memory_mb_mean": 101100
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
@@ -36,12 +36,10 @@
|
||||
"max-concurrency": 1,
|
||||
"skip-performance-assertion": true,
|
||||
"baseline": {
|
||||
"H200": {
|
||||
"throughput_qps": 0.2,
|
||||
"latency_p99": 5.1025,
|
||||
"peak_memory_mb_max": 97402,
|
||||
"peak_memory_mb_mean": 97402
|
||||
}
|
||||
"throughput_qps": 0.20,
|
||||
"latency_p99": 5.1025,
|
||||
"peak_memory_mb_max": 97402,
|
||||
"peak_memory_mb_mean": 97402
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
@@ -35,13 +35,11 @@
|
||||
"max-concurrency": 1,
|
||||
"skip-performance-assertion": true,
|
||||
"baseline": {
|
||||
"H200": {
|
||||
"throughput_qps": 0.213,
|
||||
"latency_mean": 4.6953,
|
||||
"latency_p99": 4.8601,
|
||||
"peak_memory_mb_max": 56912.0,
|
||||
"peak_memory_mb_mean": 56912.0
|
||||
}
|
||||
"throughput_qps": 0.213,
|
||||
"latency_mean": 4.6953,
|
||||
"latency_p99": 4.8601,
|
||||
"peak_memory_mb_max": 56912.0,
|
||||
"peak_memory_mb_mean": 56912.0
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
@@ -26,7 +26,8 @@
|
||||
"warmup-requests": 1,
|
||||
"warmup-concurrency": 1,
|
||||
"warmup-num-inference-steps": 2,
|
||||
"enable-negative-prompt": true
|
||||
"enable-negative-prompt": true,
|
||||
"skip-performance-assertion": true
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@@ -1,144 +0,0 @@
|
||||
[
|
||||
{
|
||||
"test_name": "test_minicpmo_4_5_challenge",
|
||||
"mark": [
|
||||
{
|
||||
"hardware_marks": {
|
||||
"res": {
|
||||
"cuda": "H100",
|
||||
"npu": "A3"
|
||||
},
|
||||
"num_cards": 1
|
||||
}
|
||||
},
|
||||
"full_model",
|
||||
"omni"
|
||||
],
|
||||
"server_params": {
|
||||
"model": "openbmb/MiniCPM-o-4_5",
|
||||
"extra_cli_args": [
|
||||
"--deploy-config",
|
||||
"vllm_omni/deploy/minicpmo_4_5.yaml",
|
||||
"--trust-remote-code"
|
||||
]
|
||||
},
|
||||
"benchmark_params": [
|
||||
{
|
||||
"dataset_name": "seed-tts",
|
||||
"dataset_path": "zhaochenyang20/seed-tts-eval",
|
||||
"backend": "openai-chat-omni",
|
||||
"endpoint": "/v1/chat/completions",
|
||||
"num_prompts": [
|
||||
32,
|
||||
64,
|
||||
128
|
||||
],
|
||||
"max_concurrency": [
|
||||
1,
|
||||
4,
|
||||
8
|
||||
],
|
||||
"no_oversample": true,
|
||||
"disable_shuffle": true,
|
||||
"trust_remote_code": true,
|
||||
"seed_tts_locale": "zh",
|
||||
"extra_body": {
|
||||
"modalities": [
|
||||
"text",
|
||||
"audio"
|
||||
],
|
||||
"chat_template_kwargs": {
|
||||
"enable_thinking": false,
|
||||
"use_tts_template": true
|
||||
}
|
||||
},
|
||||
"percentile-metrics": "ttft,e2el,audio_ttfp,audio_rtf,audio_duration",
|
||||
"baseline": {
|
||||
"A3": {
|
||||
"request_throughput": [
|
||||
0.5383,
|
||||
0.6042,
|
||||
0.7547
|
||||
],
|
||||
"mean_ttft_ms": [
|
||||
333.2633,
|
||||
533.8656,
|
||||
547.3388
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
1857.2154,
|
||||
6600.5101,
|
||||
10450.6811
|
||||
],
|
||||
"mean_audio_ttfp_ms": [
|
||||
986.4666,
|
||||
3411.0754,
|
||||
3352.7515
|
||||
],
|
||||
"mean_audio_rtf": [
|
||||
0.4423,
|
||||
1.5734,
|
||||
2.3024
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"test_name": "test_minicpmo_4_5_a2_challenge",
|
||||
"mark": [
|
||||
{
|
||||
"hardware_marks": {
|
||||
"res": {
|
||||
"npu": "A2"
|
||||
},
|
||||
"num_cards": 1
|
||||
}
|
||||
},
|
||||
"full_model",
|
||||
"omni"
|
||||
],
|
||||
"server_params": {
|
||||
"model": "openbmb/MiniCPM-o-4_5",
|
||||
"extra_cli_args": [
|
||||
"--deploy-config",
|
||||
"vllm_omni/deploy/minicpmo_4_5.yaml",
|
||||
"--trust-remote-code"
|
||||
]
|
||||
},
|
||||
"benchmark_params": [
|
||||
{
|
||||
"dataset_name": "seed-tts",
|
||||
"dataset_path": "zhaochenyang20/seed-tts-eval",
|
||||
"backend": "openai-chat-omni",
|
||||
"endpoint": "/v1/chat/completions",
|
||||
"num_prompts": [
|
||||
32,
|
||||
64,
|
||||
128
|
||||
],
|
||||
"max_concurrency": [
|
||||
1,
|
||||
4,
|
||||
8
|
||||
],
|
||||
"no_oversample": true,
|
||||
"disable_shuffle": true,
|
||||
"trust_remote_code": true,
|
||||
"seed_tts_locale": "zh",
|
||||
"extra_body": {
|
||||
"modalities": [
|
||||
"text",
|
||||
"audio"
|
||||
],
|
||||
"chat_template_kwargs": {
|
||||
"enable_thinking": false,
|
||||
"use_tts_template": true
|
||||
}
|
||||
},
|
||||
"percentile-metrics": "ttft,e2el,audio_ttfp,audio_rtf,audio_duration"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
@@ -1,69 +0,0 @@
|
||||
[
|
||||
{
|
||||
"test_name": "test_minicpmo_4_5_duplex_seed_tts_challenge",
|
||||
"mark": [
|
||||
{
|
||||
"hardware_marks": {
|
||||
"res": {
|
||||
"cuda": "H100",
|
||||
"npu": "A3"
|
||||
},
|
||||
"num_cards": 1
|
||||
}
|
||||
},
|
||||
"full_model",
|
||||
"omni"
|
||||
],
|
||||
"server_params": {
|
||||
"model": "openbmb/MiniCPM-o-4_5",
|
||||
"extra_cli_args": [
|
||||
"--deploy-config",
|
||||
"vllm_omni/deploy/minicpmo_4_5_duplex.yaml",
|
||||
"--trust-remote-code"
|
||||
]
|
||||
},
|
||||
"benchmark_params": [
|
||||
{
|
||||
"dataset_name": "seed-tts",
|
||||
"dataset_path": "zhaochenyang20/seed-tts-eval",
|
||||
"backend": "openai-realtime-tts",
|
||||
"endpoint": "/v1/realtime",
|
||||
"num_prompts": [
|
||||
32
|
||||
],
|
||||
"max_concurrency": [
|
||||
1
|
||||
],
|
||||
"no_oversample": true,
|
||||
"disable_shuffle": true,
|
||||
"trust_remote_code": true,
|
||||
"seed_tts_locale": "zh",
|
||||
"seed_tts_turns_per_session": 4,
|
||||
"expected_duplex_audio_turns_per_session": 4,
|
||||
"extra_body": {
|
||||
"save_duplex_request_metrics": true
|
||||
},
|
||||
"percentile-metrics": "ttft,e2el,audio_ttfp,audio_rtf,audio_duration",
|
||||
"baseline": {
|
||||
"A3": {
|
||||
"request_throughput": [
|
||||
0.1594
|
||||
],
|
||||
"mean_ttft_ms": [
|
||||
274.164
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
6250.7696
|
||||
],
|
||||
"mean_audio_ttfp_ms": [
|
||||
994.422
|
||||
],
|
||||
"mean_audio_rtf": [
|
||||
0.46163
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
@@ -1,81 +0,0 @@
|
||||
[
|
||||
{
|
||||
"test_name": "test_minicpmo_4_5_ready",
|
||||
"mark": [
|
||||
{
|
||||
"hardware_marks": {
|
||||
"res": {
|
||||
"cuda": "H100",
|
||||
"npu": "A3"
|
||||
},
|
||||
"num_cards": 1
|
||||
}
|
||||
},
|
||||
"core_model",
|
||||
"omni"
|
||||
],
|
||||
"server_params": {
|
||||
"model": "openbmb/MiniCPM-o-4_5",
|
||||
"extra_cli_args": [
|
||||
"--deploy-config",
|
||||
"vllm_omni/deploy/minicpmo_4_5.yaml",
|
||||
"--trust-remote-code"
|
||||
]
|
||||
},
|
||||
"benchmark_params": [
|
||||
{
|
||||
"dataset_name": "seed-tts",
|
||||
"dataset_path": "zhaochenyang20/seed-tts-eval",
|
||||
"backend": "openai-chat-omni",
|
||||
"endpoint": "/v1/chat/completions",
|
||||
"num_prompts": [
|
||||
10,
|
||||
40
|
||||
],
|
||||
"max_concurrency": [
|
||||
1,
|
||||
4
|
||||
],
|
||||
"no_oversample": true,
|
||||
"disable_shuffle": true,
|
||||
"trust_remote_code": true,
|
||||
"seed_tts_locale": "zh",
|
||||
"extra_body": {
|
||||
"modalities": [
|
||||
"text",
|
||||
"audio"
|
||||
],
|
||||
"chat_template_kwargs": {
|
||||
"enable_thinking": false,
|
||||
"use_tts_template": true
|
||||
}
|
||||
},
|
||||
"percentile-metrics": "ttft,e2el,audio_ttfp,audio_rtf,audio_duration",
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"request_throughput": [
|
||||
0.7803,
|
||||
1.1008
|
||||
],
|
||||
"mean_ttft_ms": [
|
||||
166.9764,
|
||||
307.7133
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
1361.1682,
|
||||
3800.6332
|
||||
],
|
||||
"mean_audio_ttfp_ms": [
|
||||
635.8099,
|
||||
1615.6591
|
||||
],
|
||||
"mean_audio_rtf": [
|
||||
0.3151,
|
||||
0.8568
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
@@ -5,8 +5,7 @@
|
||||
{
|
||||
"hardware_marks": {
|
||||
"res": {
|
||||
"cuda": "H100",
|
||||
"npu": "A3"
|
||||
"cuda": "H100"
|
||||
},
|
||||
"num_cards": 2
|
||||
}
|
||||
@@ -44,29 +43,27 @@
|
||||
"ignore_eos": true,
|
||||
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"mean_ttft_ms": [
|
||||
96.4061,
|
||||
140.8147,
|
||||
271.9025,
|
||||
362.2916,
|
||||
507.776
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
18507.4582,
|
||||
28365.4956,
|
||||
31907.9613,
|
||||
48161.5829,
|
||||
72630.8601
|
||||
],
|
||||
"mean_audio_rtf": [
|
||||
0.1175,
|
||||
0.18,
|
||||
0.2359,
|
||||
0.3278,
|
||||
0.6296
|
||||
]
|
||||
}
|
||||
"mean_ttft_ms": [
|
||||
96.4061,
|
||||
140.8147,
|
||||
271.9025,
|
||||
362.2916,
|
||||
507.776
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
18507.4582,
|
||||
28365.4956,
|
||||
31907.9613,
|
||||
48161.5829,
|
||||
72630.8601
|
||||
],
|
||||
"mean_audio_rtf": [
|
||||
0.1175,
|
||||
0.18,
|
||||
0.2359,
|
||||
0.3278,
|
||||
0.6296
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -93,17 +90,15 @@
|
||||
},
|
||||
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"mean_ttft_ms": [
|
||||
354.0951
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
3952.5615
|
||||
],
|
||||
"mean_audio_rtf": [
|
||||
0.117
|
||||
]
|
||||
}
|
||||
"mean_ttft_ms": [
|
||||
354.0951
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
3952.5615
|
||||
],
|
||||
"mean_audio_rtf": [
|
||||
0.117
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -132,17 +127,15 @@
|
||||
},
|
||||
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"mean_ttft_ms": [
|
||||
259.6317
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
4253.3009
|
||||
],
|
||||
"mean_audio_rtf": [
|
||||
0.1448
|
||||
]
|
||||
}
|
||||
"mean_ttft_ms": [
|
||||
259.6317
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
4253.3009
|
||||
],
|
||||
"mean_audio_rtf": [
|
||||
0.1448
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -173,17 +166,15 @@
|
||||
},
|
||||
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"mean_ttft_ms": [
|
||||
457.4453
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
5912.5623
|
||||
],
|
||||
"mean_audio_rtf": [
|
||||
0.1967
|
||||
]
|
||||
}
|
||||
"mean_ttft_ms": [
|
||||
457.4453
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
5912.5623
|
||||
],
|
||||
"mean_audio_rtf": [
|
||||
0.1967
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -214,22 +205,20 @@
|
||||
},
|
||||
"percentile-metrics": "ttft,tpot,itl,e2el",
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"mean_ttft_ms": [
|
||||
90.8033,
|
||||
202.1779,
|
||||
326.5864,
|
||||
544.8914,
|
||||
961.3929
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
4517.3411,
|
||||
7857.9711,
|
||||
9861.1687,
|
||||
13283.3363,
|
||||
19541.0736
|
||||
]
|
||||
}
|
||||
"mean_ttft_ms": [
|
||||
90.8033,
|
||||
202.1779,
|
||||
326.5864,
|
||||
544.8914,
|
||||
961.3929
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
4517.3411,
|
||||
7857.9711,
|
||||
9861.1687,
|
||||
13283.3363,
|
||||
19541.0736
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -261,14 +250,12 @@
|
||||
},
|
||||
"percentile-metrics": "ttft,tpot,itl,e2el",
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"mean_ttft_ms": [
|
||||
318.048
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
807.6114
|
||||
]
|
||||
}
|
||||
"mean_ttft_ms": [
|
||||
318.048
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
807.6114
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -302,14 +289,12 @@
|
||||
},
|
||||
"percentile-metrics": "ttft,tpot,itl,e2el",
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"mean_ttft_ms": [
|
||||
186.168
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
726.1137
|
||||
]
|
||||
}
|
||||
"mean_ttft_ms": [
|
||||
186.168
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
726.1137
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -345,14 +330,12 @@
|
||||
},
|
||||
"percentile-metrics": "ttft,tpot,itl,e2el",
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"mean_ttft_ms": [
|
||||
375.5282
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
1006.1663
|
||||
]
|
||||
}
|
||||
"mean_ttft_ms": [
|
||||
375.5282
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
1006.1663
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
@@ -35,12 +35,10 @@
|
||||
"ignore_eos": true,
|
||||
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"mean_ttft_ms": [130.5751, 278.6469, 324.7249, 911.6085],
|
||||
"mean_tpot_ms": [7.9854, 11.1209, 16.1507, 20.7485],
|
||||
"mean_audio_ttfp_ms": [486.486, 954.1791, 1289.6514, 2196.8275],
|
||||
"mean_audio_rtf": [0.1896, 0.3962, 0.4098, 0.7322]
|
||||
}
|
||||
"mean_ttft_ms": [130.5751, 278.6469, 324.7249, 911.6085],
|
||||
"mean_tpot_ms": [7.9854, 11.1209, 16.1507, 20.7485],
|
||||
"mean_audio_ttfp_ms": [486.486, 954.1791, 1289.6514, 2196.8275],
|
||||
"mean_audio_rtf": [0.1896, 0.3962, 0.4098, 0.7322]
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -63,11 +61,9 @@
|
||||
},
|
||||
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"mean_ttft_ms": [340.5122],
|
||||
"mean_audio_ttfp_ms": [463.0752],
|
||||
"mean_audio_rtf": [0.1153]
|
||||
}
|
||||
"mean_ttft_ms": [340.5122],
|
||||
"mean_audio_ttfp_ms": [463.0752],
|
||||
"mean_audio_rtf": [0.1153]
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -92,11 +88,9 @@
|
||||
},
|
||||
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"mean_ttft_ms": [236.926],
|
||||
"mean_audio_ttfp_ms": [395.2182],
|
||||
"mean_audio_rtf": [0.1259]
|
||||
}
|
||||
"mean_ttft_ms": [236.926],
|
||||
"mean_audio_ttfp_ms": [395.2182],
|
||||
"mean_audio_rtf": [0.1259]
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -123,11 +117,9 @@
|
||||
},
|
||||
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"mean_ttft_ms": [470.6126],
|
||||
"mean_audio_ttfp_ms": [665.5474],
|
||||
"mean_audio_rtf": [0.166]
|
||||
}
|
||||
"mean_ttft_ms": [470.6126],
|
||||
"mean_audio_ttfp_ms": [665.5474],
|
||||
"mean_audio_rtf": [0.166]
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
@@ -5,8 +5,7 @@
|
||||
{
|
||||
"hardware_marks": {
|
||||
"res": {
|
||||
"cuda": "H100",
|
||||
"npu": "A3"
|
||||
"cuda": "H100"
|
||||
},
|
||||
"num_cards": 2
|
||||
}
|
||||
@@ -44,29 +43,27 @@
|
||||
"ignore_eos": true,
|
||||
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"mean_ttft_ms": [
|
||||
115.8348,
|
||||
173.1823,
|
||||
446.672,
|
||||
464.5985,
|
||||
3420.0634
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
25070.9523,
|
||||
31992.0646,
|
||||
37619.7669,
|
||||
50393.8257,
|
||||
136619.2743
|
||||
],
|
||||
"mean_audio_rtf": [
|
||||
0.169,
|
||||
0.2111,
|
||||
0.2475,
|
||||
0.3551,
|
||||
0.9266
|
||||
]
|
||||
}
|
||||
"mean_ttft_ms": [
|
||||
115.8348,
|
||||
173.1823,
|
||||
446.672,
|
||||
464.5985,
|
||||
3420.0634
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
25070.9523,
|
||||
31992.0646,
|
||||
37619.7669,
|
||||
50393.8257,
|
||||
136619.2743
|
||||
],
|
||||
"mean_audio_rtf": [
|
||||
0.169,
|
||||
0.2111,
|
||||
0.2475,
|
||||
0.3551,
|
||||
0.9266
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -93,17 +90,15 @@
|
||||
},
|
||||
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"mean_ttft_ms": [
|
||||
343.1377
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
4811.1123
|
||||
],
|
||||
"mean_audio_rtf": [
|
||||
0.1463
|
||||
]
|
||||
}
|
||||
"mean_ttft_ms": [
|
||||
343.1377
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
4811.1123
|
||||
],
|
||||
"mean_audio_rtf": [
|
||||
0.1463
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -132,17 +127,15 @@
|
||||
},
|
||||
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"mean_ttft_ms": [
|
||||
245.5446
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
5019.1256
|
||||
],
|
||||
"mean_audio_rtf": [
|
||||
0.1713
|
||||
]
|
||||
}
|
||||
"mean_ttft_ms": [
|
||||
245.5446
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
5019.1256
|
||||
],
|
||||
"mean_audio_rtf": [
|
||||
0.1713
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -173,17 +166,15 @@
|
||||
},
|
||||
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"mean_ttft_ms": [
|
||||
506.1407
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
6814.9656
|
||||
],
|
||||
"mean_audio_rtf": [
|
||||
0.2213
|
||||
]
|
||||
}
|
||||
"mean_ttft_ms": [
|
||||
506.1407
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
6814.9656
|
||||
],
|
||||
"mean_audio_rtf": [
|
||||
0.2213
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
@@ -51,22 +51,20 @@
|
||||
},
|
||||
"percentile-metrics": "ttft,tpot,itl,e2el",
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"mean_ttft_ms": [
|
||||
69.8081,
|
||||
173.0651,
|
||||
271.8108,
|
||||
413.4079,
|
||||
497.2546
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
4720.8225,
|
||||
7937.2532,
|
||||
9962.3566,
|
||||
13724.5479,
|
||||
18324.091
|
||||
]
|
||||
}
|
||||
"mean_ttft_ms": [
|
||||
69.8081,
|
||||
173.0651,
|
||||
271.8108,
|
||||
413.4079,
|
||||
497.2546
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
4720.8225,
|
||||
7937.2532,
|
||||
9962.3566,
|
||||
13724.5479,
|
||||
18324.091
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -99,14 +97,12 @@
|
||||
},
|
||||
"percentile-metrics": "ttft,tpot,itl,e2el",
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"mean_ttft_ms": [
|
||||
25.9477
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
517.3155
|
||||
]
|
||||
}
|
||||
"mean_ttft_ms": [
|
||||
25.9477
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
517.3155
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -141,14 +137,12 @@
|
||||
},
|
||||
"percentile-metrics": "ttft,tpot,itl,e2el",
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"mean_ttft_ms": [
|
||||
26.7056
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
551.308
|
||||
]
|
||||
}
|
||||
"mean_ttft_ms": [
|
||||
26.7056
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
551.308
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -185,14 +179,12 @@
|
||||
},
|
||||
"percentile-metrics": "ttft,tpot,itl,e2el",
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"mean_ttft_ms": [
|
||||
27.6906
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
596.5284
|
||||
]
|
||||
}
|
||||
"mean_ttft_ms": [
|
||||
27.6906
|
||||
],
|
||||
"mean_e2el_ms": [
|
||||
596.5284
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
@@ -34,12 +34,10 @@
|
||||
"num-input-images": 2,
|
||||
"enable-negative-prompt": true,
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"throughput_qps": 0.0721,
|
||||
"latency_mean": 13.8398,
|
||||
"peak_memory_mb_max": 57610.0,
|
||||
"peak_memory_mb_mean": 57610.0
|
||||
}
|
||||
"throughput_qps": 0.0721,
|
||||
"latency_mean": 13.8398,
|
||||
"peak_memory_mb_max": 57610.0,
|
||||
"peak_memory_mb_mean": 57610.0
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -54,12 +52,10 @@
|
||||
"num-input-images": 2,
|
||||
"enable-negative-prompt": true,
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"throughput_qps": 0.0186,
|
||||
"latency_mean": 53.6942,
|
||||
"peak_memory_mb_max": 67790.0,
|
||||
"peak_memory_mb_mean": 67790.0
|
||||
}
|
||||
"throughput_qps": 0.0186,
|
||||
"latency_mean": 53.6942,
|
||||
"peak_memory_mb_max": 67790.0,
|
||||
"peak_memory_mb_mean": 67790.0
|
||||
}
|
||||
}
|
||||
]
|
||||
@@ -70,8 +66,7 @@
|
||||
{
|
||||
"hardware_marks": {
|
||||
"res": {
|
||||
"cuda": "H100",
|
||||
"npu": "A3"
|
||||
"cuda": "H100"
|
||||
},
|
||||
"num_cards": 4
|
||||
}
|
||||
@@ -104,12 +99,10 @@
|
||||
"num-input-images": 2,
|
||||
"enable-negative-prompt": true,
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"throughput_qps": 0.0538,
|
||||
"latency_mean": 18.5591,
|
||||
"peak_memory_mb_max": 56449.4286,
|
||||
"peak_memory_mb_mean": 56449.4286
|
||||
}
|
||||
"throughput_qps": 0.0538,
|
||||
"latency_mean": 18.5591,
|
||||
"peak_memory_mb_max": 56449.4286,
|
||||
"peak_memory_mb_mean": 56449.4286
|
||||
}
|
||||
}
|
||||
]
|
||||
@@ -163,12 +156,10 @@
|
||||
"num-input-images": 2,
|
||||
"enable-negative-prompt": true,
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"throughput_qps": 0.2776,
|
||||
"latency_mean": 3.5938,
|
||||
"peak_memory_mb_max": 58113.7143,
|
||||
"peak_memory_mb_mean": 58075.1429
|
||||
}
|
||||
"throughput_qps": 0.2776,
|
||||
"latency_mean": 3.5938,
|
||||
"peak_memory_mb_max": 58113.7143,
|
||||
"peak_memory_mb_mean": 58075.1429
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -183,12 +174,10 @@
|
||||
"num-input-images": 2,
|
||||
"enable-negative-prompt": true,
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"throughput_qps": 0.1051,
|
||||
"latency_mean": 9.4918,
|
||||
"peak_memory_mb_max": 68107.4286,
|
||||
"peak_memory_mb_mean": 68107.4286
|
||||
}
|
||||
"throughput_qps": 0.1051,
|
||||
"latency_mean": 9.4918,
|
||||
"peak_memory_mb_max": 68107.4286,
|
||||
"peak_memory_mb_mean": 68107.4286
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
@@ -33,12 +33,10 @@
|
||||
"max-concurrency": 1,
|
||||
"enable-negative-prompt": true,
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"throughput_qps": 0.1129,
|
||||
"latency_mean": 8.8352,
|
||||
"peak_memory_mb_max": 58572.0,
|
||||
"peak_memory_mb_mean": 58572.0
|
||||
}
|
||||
"throughput_qps": 0.1129,
|
||||
"latency_mean": 8.8352,
|
||||
"peak_memory_mb_max": 58572.0,
|
||||
"peak_memory_mb_mean": 58572.0
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -52,12 +50,10 @@
|
||||
"max-concurrency": 1,
|
||||
"enable-negative-prompt": true,
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"throughput_qps": 0.024,
|
||||
"latency_mean": 41.5686,
|
||||
"peak_memory_mb_max": 66368.0,
|
||||
"peak_memory_mb_mean": 66368.0
|
||||
}
|
||||
"throughput_qps": 0.024,
|
||||
"latency_mean": 41.5686,
|
||||
"peak_memory_mb_max": 66368.0,
|
||||
"peak_memory_mb_mean": 66368.0
|
||||
}
|
||||
}
|
||||
]
|
||||
@@ -100,12 +96,10 @@
|
||||
"max-concurrency": 1,
|
||||
"enable-negative-prompt": true,
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"throughput_qps": 0.0672,
|
||||
"latency_mean": 14.8517,
|
||||
"peak_memory_mb_max": 57076.0,
|
||||
"peak_memory_mb_mean": 57076.0
|
||||
}
|
||||
"throughput_qps": 0.0672,
|
||||
"latency_mean": 14.8517,
|
||||
"peak_memory_mb_max": 57076.0,
|
||||
"peak_memory_mb_mean": 57076.0
|
||||
}
|
||||
}
|
||||
]
|
||||
@@ -158,12 +152,10 @@
|
||||
"max-concurrency": 1,
|
||||
"enable-negative-prompt": true,
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"throughput_qps": 0.2879,
|
||||
"latency_mean": 3.4701,
|
||||
"peak_memory_mb_max": 58758.0,
|
||||
"peak_memory_mb_mean": 58758.0
|
||||
}
|
||||
"throughput_qps": 0.2879,
|
||||
"latency_mean": 3.4701,
|
||||
"peak_memory_mb_max": 58758.0,
|
||||
"peak_memory_mb_mean": 58758.0
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -177,12 +169,10 @@
|
||||
"max-concurrency": 1,
|
||||
"enable-negative-prompt": true,
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"throughput_qps": 0.1113,
|
||||
"latency_mean": 8.9656,
|
||||
"peak_memory_mb_max": 67424.0,
|
||||
"peak_memory_mb_mean": 67424.0
|
||||
}
|
||||
"throughput_qps": 0.1113,
|
||||
"latency_mean": 8.9656,
|
||||
"peak_memory_mb_max": 67424.0,
|
||||
"peak_memory_mb_mean": 67424.0
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
@@ -33,12 +33,10 @@
|
||||
"max-concurrency": 1,
|
||||
"enable-negative-prompt": true,
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"throughput_qps": 0.0703,
|
||||
"latency_mean": 14.208,
|
||||
"peak_memory_mb_max": 63225.1429,
|
||||
"peak_memory_mb_mean": 63225.1429
|
||||
}
|
||||
"throughput_qps": 0.0703,
|
||||
"latency_mean": 14.208,
|
||||
"peak_memory_mb_max": 63225.1429,
|
||||
"peak_memory_mb_mean": 63225.1429
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -52,12 +50,10 @@
|
||||
"max-concurrency": 1,
|
||||
"enable-negative-prompt": true,
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"throughput_qps": 0.0417,
|
||||
"latency_mean": 23.914,
|
||||
"peak_memory_mb_max": 63225.1429,
|
||||
"peak_memory_mb_mean": 63225.1429
|
||||
}
|
||||
"throughput_qps": 0.0417,
|
||||
"latency_mean": 23.914,
|
||||
"peak_memory_mb_max": 63225.1429,
|
||||
"peak_memory_mb_mean": 63225.1429
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
@@ -33,11 +33,9 @@
|
||||
"max-concurrency": 1,
|
||||
"enable-negative-prompt": true,
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"throughput_qps": 0.4153,
|
||||
"latency_mean": 2.4065,
|
||||
"peak_memory_mb_mean": 55121.4286
|
||||
}
|
||||
"throughput_qps": 0.4153,
|
||||
"latency_mean": 2.4065,
|
||||
"peak_memory_mb_mean": 55121.4286
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -51,11 +49,9 @@
|
||||
"max-concurrency": 1,
|
||||
"enable-negative-prompt": true,
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"throughput_qps": 0.0423,
|
||||
"latency_mean": 23.5991,
|
||||
"peak_memory_mb_mean": 65118.0
|
||||
}
|
||||
"throughput_qps": 0.0423,
|
||||
"latency_mean": 23.5991,
|
||||
"peak_memory_mb_mean": 65118.0
|
||||
}
|
||||
}
|
||||
]
|
||||
@@ -95,11 +91,9 @@
|
||||
"max-concurrency": 1,
|
||||
"enable-negative-prompt": true,
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"throughput_qps": 0.4175,
|
||||
"latency_mean": 2.3953,
|
||||
"peak_memory_mb_mean": 55088.5714
|
||||
}
|
||||
"throughput_qps": 0.4175,
|
||||
"latency_mean": 2.3953,
|
||||
"peak_memory_mb_mean": 55088.5714
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -113,11 +107,9 @@
|
||||
"max-concurrency": 1,
|
||||
"enable-negative-prompt": true,
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"throughput_qps": 0.0423,
|
||||
"latency_mean": 23.5768,
|
||||
"peak_memory_mb_mean": 66012.2857
|
||||
}
|
||||
"throughput_qps": 0.0423,
|
||||
"latency_mean": 23.5768,
|
||||
"peak_memory_mb_mean": 66012.2857
|
||||
}
|
||||
}
|
||||
]
|
||||
@@ -160,11 +152,9 @@
|
||||
"max-concurrency": 1,
|
||||
"enable-negative-prompt": true,
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"throughput_qps": 0.1214,
|
||||
"latency_mean": 8.2211,
|
||||
"peak_memory_mb_mean": 54501.1429
|
||||
}
|
||||
"throughput_qps": 0.1214,
|
||||
"latency_mean": 8.2211,
|
||||
"peak_memory_mb_mean": 54501.1429
|
||||
}
|
||||
}
|
||||
]
|
||||
@@ -217,11 +207,9 @@
|
||||
"max-concurrency": 1,
|
||||
"enable-negative-prompt": true,
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"throughput_qps": 0.559,
|
||||
"latency_mean": 1.794,
|
||||
"peak_memory_mb_mean": 55290.5714
|
||||
}
|
||||
"throughput_qps": 0.559,
|
||||
"latency_mean": 1.794,
|
||||
"peak_memory_mb_mean": 55290.5714
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -235,11 +223,9 @@
|
||||
"max-concurrency": 1,
|
||||
"enable-negative-prompt": true,
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"throughput_qps": 0.1918,
|
||||
"latency_mean": 5.2022,
|
||||
"peak_memory_mb_mean": 64838.8571
|
||||
}
|
||||
"throughput_qps": 0.1918,
|
||||
"latency_mean": 5.2022,
|
||||
"peak_memory_mb_mean": 64838.8571
|
||||
}
|
||||
}
|
||||
]
|
||||
@@ -283,11 +269,9 @@
|
||||
"warmup-num-inference-steps": 20,
|
||||
"enable-negative-prompt": true,
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"throughput_qps": [0.4125, 0.6714, 0.6674, 0.8159],
|
||||
"latency_mean": [2.4215, 2.992, 5.9293, 9.656],
|
||||
"peak_memory_mb_mean": [55147.4286, 55148.0, 55150.4643, 55152.3821]
|
||||
}
|
||||
"throughput_qps": [0.4125, 0.6714, 0.6674, 0.8159],
|
||||
"latency_mean": [2.4215, 2.992, 5.9293, 9.656],
|
||||
"peak_memory_mb_mean": [55147.4286, 55148.0, 55150.4643, 55152.3821]
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
"""Tests for DFX runner metadata field exclusion."""
|
||||
|
||||
import json
|
||||
from contextlib import contextmanager
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
@@ -17,7 +15,7 @@ def test_task_excluded_from_cli_args():
|
||||
"backend": "openai-audio-speech",
|
||||
"endpoint": "/v1/audio/speech",
|
||||
"percentile-metrics": "audio_rtf,audio_ttfp",
|
||||
"baseline": {"H100": {"mean_audio_rtf": [0.5]}},
|
||||
"baseline": {"mean_audio_rtf": [0.5]},
|
||||
}
|
||||
exclude_keys = {
|
||||
"request_rate",
|
||||
@@ -123,6 +121,35 @@ def test_resolve_pytest_marks_rejects_legacy_object_format():
|
||||
)
|
||||
|
||||
|
||||
def test_extract_mark_resource_label():
|
||||
from tests.dfx.conftest import extract_mark_resource_label
|
||||
|
||||
assert extract_mark_resource_label(None) == "na"
|
||||
assert (
|
||||
extract_mark_resource_label(
|
||||
[
|
||||
{"hardware_marks": {"res": {"cuda": "H100"}, "num_cards": 1}},
|
||||
"full_model",
|
||||
]
|
||||
)
|
||||
== "H100"
|
||||
)
|
||||
assert (
|
||||
extract_mark_resource_label(
|
||||
[
|
||||
{
|
||||
"hardware_marks": {
|
||||
"res": {"cuda": "H100", "rocm": "MI325"},
|
||||
"num_cards": 2,
|
||||
}
|
||||
},
|
||||
"diffusion",
|
||||
]
|
||||
)
|
||||
== "H100-MI325"
|
||||
)
|
||||
|
||||
|
||||
def test_resource_label_for_filename():
|
||||
from tests.dfx.conftest import resource_label_for_filename
|
||||
|
||||
@@ -140,14 +167,16 @@ def test_hardware_json_value():
|
||||
assert hardware_json_value(None) == ""
|
||||
|
||||
|
||||
def test_get_runtime_resource_label(monkeypatch):
|
||||
from tests.dfx.conftest import get_runtime_resource_label
|
||||
def test_extract_configs_resource_label(monkeypatch):
|
||||
from tests.dfx.conftest import extract_configs_resource_label, get_runtime_resource_label
|
||||
|
||||
monkeypatch.setattr(
|
||||
"tests.dfx.conftest._read_runtime_device_name",
|
||||
lambda *, device_id=0: "NVIDIA H100 80GB HBM3",
|
||||
)
|
||||
assert get_runtime_resource_label(refresh=True) == "H100"
|
||||
get_runtime_resource_label(refresh=True)
|
||||
assert extract_configs_resource_label([]) == "H100"
|
||||
get_runtime_resource_label(refresh=True)
|
||||
monkeypatch.setattr(
|
||||
"tests.dfx.conftest._read_runtime_device_name",
|
||||
lambda *, device_id=0: "Ascend910B2",
|
||||
@@ -225,9 +254,8 @@ def test_benchmark_param_id_suffix_from_task_eval_phase():
|
||||
]
|
||||
|
||||
|
||||
def test_paired_omni_benchmark_reuses_server_and_preserves_case_metadata(tmp_path, monkeypatch):
|
||||
def test_create_paired_omni_benchmark_pytest_params(tmp_path):
|
||||
from tests.dfx.conftest import create_paired_omni_benchmark_pytest_params
|
||||
from tests.dfx.perf.scripts import run_benchmark
|
||||
|
||||
configs = [
|
||||
{
|
||||
@@ -257,231 +285,3 @@ def test_paired_omni_benchmark_reuses_server_and_preserves_case_metadata(tmp_pat
|
||||
assert bench_row == ("test_tts", 0)
|
||||
assert any(m.name == "tts" for m in by_id["test_tts-p0"].marks)
|
||||
assert not any(m.name == "tts" for m in by_id["test_omni-p0"].marks)
|
||||
|
||||
omni_p0_server = by_id["test_omni-p0"].values[0]
|
||||
omni_p1_server = by_id["test_omni-p1"].values[0]
|
||||
tts_server = by_id["test_tts-p0"].values[0]
|
||||
assert omni_p0_server == omni_p1_server
|
||||
assert omni_p0_server != tts_server
|
||||
|
||||
events = []
|
||||
|
||||
@contextmanager
|
||||
def fake_start(server_param):
|
||||
events.append(("start", server_param))
|
||||
yield object()
|
||||
events.append(("stop", server_param))
|
||||
|
||||
monkeypatch.setattr(run_benchmark, "_start_omni_server", fake_start)
|
||||
active_context = run_benchmark._SingleActiveContext()
|
||||
try:
|
||||
first = run_benchmark.omni_server.__wrapped__(
|
||||
SimpleNamespace(param=omni_p0_server),
|
||||
active_context,
|
||||
)
|
||||
second = run_benchmark.omni_server.__wrapped__(
|
||||
SimpleNamespace(param=omni_p1_server),
|
||||
active_context,
|
||||
)
|
||||
assert first is second
|
||||
assert events == [("start", omni_p0_server)]
|
||||
|
||||
third = run_benchmark.omni_server.__wrapped__(
|
||||
SimpleNamespace(param=tts_server),
|
||||
active_context,
|
||||
)
|
||||
assert third is not first
|
||||
assert events == [
|
||||
("start", omni_p0_server),
|
||||
("stop", omni_p0_server),
|
||||
("start", tts_server),
|
||||
]
|
||||
finally:
|
||||
active_context.close()
|
||||
|
||||
assert events == [
|
||||
("start", omni_p0_server),
|
||||
("stop", omni_p0_server),
|
||||
("start", tts_server),
|
||||
("stop", tts_server),
|
||||
]
|
||||
|
||||
|
||||
def test_is_hardware_nested_baseline():
|
||||
from tests.dfx.conftest import (
|
||||
_RUNTIME_DEVICE_ALIASES,
|
||||
is_hardware_nested_baseline,
|
||||
)
|
||||
from tests.helpers.mark import get_hardware_mark_list
|
||||
|
||||
hardware_marks = get_hardware_mark_list()
|
||||
assert hardware_marks
|
||||
assert {"H100", "L4", "A3", "MI325", "B200"} <= hardware_marks
|
||||
assert "cuda" not in hardware_marks
|
||||
assert "full_model" not in hardware_marks
|
||||
|
||||
# Runtime aliases are a full static list (independent of baseline allowlist).
|
||||
assert "H100" in _RUNTIME_DEVICE_ALIASES
|
||||
assert "A100" in _RUNTIME_DEVICE_ALIASES
|
||||
|
||||
assert is_hardware_nested_baseline(
|
||||
{
|
||||
"H100": {"mean_ttft_ms": [1.0, 2.0], "mean_e2el_ms": [10.0, 20.0]},
|
||||
"A3": {"mean_ttft_ms": [0.9, 1.8], "mean_e2el_ms": [9.0, 18.0]},
|
||||
}
|
||||
)
|
||||
# Custom metric names under a known hardware label are allowed.
|
||||
assert is_hardware_nested_baseline({"H100": {"custom_stage_ms": 12.0, "foo_bar": [1.0, 2.0]}})
|
||||
assert is_hardware_nested_baseline({"A3": {"request_throughput": 0.5}})
|
||||
# Flat metric maps / concurrency-keyed maps are not hardware-nested.
|
||||
assert not is_hardware_nested_baseline({"mean_ttft_ms": [1.0, 2.0], "mean_e2el_ms": [10.0, 20.0]})
|
||||
assert not is_hardware_nested_baseline({"mean_ttft_ms": {"1": 1.0, "32": 2.0}})
|
||||
assert not is_hardware_nested_baseline({})
|
||||
# Alias-only labels (not [hardware-resource] markers) are rejected for baselines.
|
||||
assert not is_hardware_nested_baseline({"A100": {"throughput_qps": 1.0}})
|
||||
# Unknown top-level hardware label is rejected.
|
||||
assert not is_hardware_nested_baseline({"UnknownGPU": {"throughput_qps": 1.0}})
|
||||
# Empty per-hardware metric map is rejected.
|
||||
assert not is_hardware_nested_baseline({"H100": {}})
|
||||
|
||||
|
||||
def test_resolve_baseline_for_sweep_keeps_all_hardware_for_one_concurrency():
|
||||
from tests.dfx.conftest import resolve_baseline_for_sweep
|
||||
|
||||
baseline = {
|
||||
"H100": {
|
||||
"mean_ttft_ms": [96.4, 140.8, 271.9, 362.3, 507.8],
|
||||
"mean_e2el_ms": [18507.0, 28365.0, 31907.0, 48161.0, 72630.0],
|
||||
},
|
||||
"A3": {
|
||||
"mean_ttft_ms": [90.0, 130.0, 250.0, 340.0, 480.0],
|
||||
"mean_e2el_ms": [17000.0, 26000.0, 30000.0, 45000.0, 70000.0],
|
||||
},
|
||||
}
|
||||
# max_concurrency=[1,4,8,16,32] -> index 4 is concurrency 32
|
||||
got = resolve_baseline_for_sweep(baseline, sweep_index=4)
|
||||
assert got == {
|
||||
"H100": {"mean_ttft_ms": 507.8, "mean_e2el_ms": 72630.0},
|
||||
"A3": {"mean_ttft_ms": 480.0, "mean_e2el_ms": 70000.0},
|
||||
}
|
||||
# First sweep step keeps both hardware buckets too.
|
||||
got0 = resolve_baseline_for_sweep(baseline, sweep_index=0)
|
||||
assert set(got0) == {"H100", "A3"}
|
||||
assert got0["H100"]["mean_ttft_ms"] == 96.4
|
||||
assert got0["A3"]["mean_ttft_ms"] == 90.0
|
||||
|
||||
|
||||
def test_resolve_baseline_for_sweep_rejects_flat_baseline():
|
||||
from tests.dfx.conftest import resolve_baseline_for_sweep
|
||||
|
||||
with pytest.raises(ValueError, match=r"hardware-nested.*pyproject\.toml"):
|
||||
resolve_baseline_for_sweep(
|
||||
{"throughput_qps": [0.4, 0.6], "latency_mean": [1.0, 2.0]},
|
||||
sweep_index=1,
|
||||
)
|
||||
|
||||
|
||||
def test_resolve_baseline_for_sweep_rejects_unknown_hardware_label():
|
||||
from tests.dfx.conftest import resolve_baseline_for_sweep
|
||||
|
||||
with pytest.raises(ValueError, match=r"Unknown hardware label\(s\): \['A100'\].*pyproject\.toml"):
|
||||
resolve_baseline_for_sweep(
|
||||
{"A100": {"throughput_qps": 1.0}},
|
||||
sweep_index=0,
|
||||
)
|
||||
|
||||
|
||||
def test_resolve_baseline_for_sweep_supports_list_and_scalar_under_hardware():
|
||||
from tests.dfx.conftest import resolve_baseline_for_sweep
|
||||
|
||||
# Sweep-aligned lists under each hardware bucket (canonical form).
|
||||
listed = {
|
||||
"H100": {"throughput_qps": [0.4, 0.6, 0.8], "latency_mean": [1.0, 2.0, 3.0]},
|
||||
"A3": {"throughput_qps": [0.5, 0.7, 0.9], "latency_mean": [0.9, 1.8, 2.7]},
|
||||
}
|
||||
assert resolve_baseline_for_sweep(listed, sweep_index=1) == {
|
||||
"H100": {"throughput_qps": 0.6, "latency_mean": 2.0},
|
||||
"A3": {"throughput_qps": 0.7, "latency_mean": 1.8},
|
||||
}
|
||||
|
||||
# Scalars under hardware stay as-is (single-concurrency cases).
|
||||
scalar = {"H100": {"throughput_qps": 0.5}}
|
||||
assert resolve_baseline_for_sweep(scalar, sweep_index=0) == {"H100": {"throughput_qps": 0.5}}
|
||||
# Custom metric names are preserved.
|
||||
custom = {"H100": {"custom_stage_ms": [10.0, 20.0]}}
|
||||
assert resolve_baseline_for_sweep(custom, sweep_index=1) == {"H100": {"custom_stage_ms": 20.0}}
|
||||
assert resolve_baseline_for_sweep(None) == {}
|
||||
assert resolve_baseline_for_sweep({}) == {}
|
||||
|
||||
|
||||
def test_resolve_baseline_value_errors():
|
||||
from tests.dfx.conftest import resolve_baseline_value
|
||||
|
||||
with pytest.raises(ValueError, match="sweep_index"):
|
||||
resolve_baseline_value([1.0, 2.0], sweep_index=None)
|
||||
with pytest.raises(IndexError):
|
||||
resolve_baseline_value([1.0], sweep_index=1)
|
||||
with pytest.raises(TypeError, match="not supported"):
|
||||
resolve_baseline_value({"1": 0.4}, sweep_index=0)
|
||||
|
||||
|
||||
def test_diffusion_build_run_params_resolves_baseline_per_sweep(tmp_path, monkeypatch):
|
||||
"""``_build_run_params`` / ``_iter_sweep_runs`` narrow list baselines per concurrency."""
|
||||
import importlib
|
||||
import sys
|
||||
|
||||
cfg = tmp_path / "mini_diffusion_perf.json"
|
||||
cfg.write_text(
|
||||
json.dumps(
|
||||
[
|
||||
{
|
||||
"test_name": "test_mini",
|
||||
"server_type": "vllm-omni",
|
||||
"mark": [
|
||||
{"hardware_marks": {"res": {"cuda": "H100"}, "num_cards": 1}},
|
||||
"diffusion",
|
||||
"full_model",
|
||||
],
|
||||
"server_params": {"model": "m/mini"},
|
||||
"benchmark_params": [{"name": "p0", "num-prompts": 1, "max-concurrency": 1}],
|
||||
}
|
||||
]
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
monkeypatch.setattr(sys, "argv", ["pytest", "--test-config-file", str(cfg)])
|
||||
sys.modules.pop("tests.dfx.perf.scripts.run_diffusion_benchmark", None)
|
||||
from tests.dfx.perf.scripts import run_diffusion_benchmark as rdb
|
||||
|
||||
importlib.reload(rdb)
|
||||
|
||||
params = {
|
||||
"name": "c_sweep",
|
||||
"dataset": "random",
|
||||
"task": "t2i",
|
||||
"num-prompts": [8, 16, 32],
|
||||
"max-concurrency": [1, 8, 32],
|
||||
"baseline": {
|
||||
"H100": {"throughput_qps": [0.1, 0.2, 0.3], "latency_mean": [10.0, 20.0, 30.0]},
|
||||
"A3": {"throughput_qps": [0.11, 0.22, 0.33], "latency_mean": [9.0, 19.0, 29.0]},
|
||||
},
|
||||
}
|
||||
run32 = rdb._build_run_params(
|
||||
params,
|
||||
num_prompts=32,
|
||||
max_concurrency=32,
|
||||
request_rate="inf",
|
||||
sweep_index=2,
|
||||
)
|
||||
assert run32["max-concurrency"] == 32
|
||||
assert run32["baseline"] == {
|
||||
"H100": {"throughput_qps": 0.3, "latency_mean": 30.0},
|
||||
"A3": {"throughput_qps": 0.33, "latency_mean": 29.0},
|
||||
}
|
||||
|
||||
sweeps = rdb._iter_sweep_runs(params)
|
||||
assert len(sweeps) == 3
|
||||
assert sweeps[2]["params"]["max-concurrency"] == 32
|
||||
assert sweeps[2]["params"]["baseline"]["H100"]["throughput_qps"] == 0.3
|
||||
assert "A3" in sweeps[2]["params"]["baseline"]
|
||||
assert not isinstance(sweeps[2]["params"]["baseline"]["H100"]["throughput_qps"], list)
|
||||
+111
-137
@@ -39,14 +39,12 @@
|
||||
"seed_tts_locale": "en",
|
||||
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"median_audio_ttfp_ms": [
|
||||
141.2791
|
||||
],
|
||||
"median_audio_rtf": [
|
||||
0.1591
|
||||
]
|
||||
}
|
||||
"median_audio_ttfp_ms": [
|
||||
141.2791
|
||||
],
|
||||
"median_audio_rtf": [
|
||||
0.1591
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -69,23 +67,21 @@
|
||||
"seed_tts_locale": "en",
|
||||
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"median_audio_ttfp_ms": [
|
||||
226.747,
|
||||
647.1272,
|
||||
4777.2714
|
||||
],
|
||||
"median_audio_rtf": [
|
||||
0.3313,
|
||||
0.3642,
|
||||
1.2282
|
||||
],
|
||||
"audio_throughput": [
|
||||
24.2296,
|
||||
40.3552,
|
||||
43.2572
|
||||
]
|
||||
}
|
||||
"median_audio_ttfp_ms": [
|
||||
226.747,
|
||||
647.1272,
|
||||
4777.2714
|
||||
],
|
||||
"median_audio_rtf": [
|
||||
0.3313,
|
||||
0.3642,
|
||||
1.2282
|
||||
],
|
||||
"audio_throughput": [
|
||||
24.2296,
|
||||
40.3552,
|
||||
43.2572
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -106,11 +102,9 @@
|
||||
"seed_tts_wer_eval": true,
|
||||
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"mean_audio_rtf": [
|
||||
0.45
|
||||
]
|
||||
}
|
||||
"mean_audio_rtf": [
|
||||
0.45
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
@@ -159,14 +153,12 @@
|
||||
},
|
||||
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"median_audio_ttfp_ms": [
|
||||
46.8434
|
||||
],
|
||||
"median_audio_rtf": [
|
||||
0.1367
|
||||
]
|
||||
}
|
||||
"median_audio_ttfp_ms": [
|
||||
46.8434
|
||||
],
|
||||
"median_audio_rtf": [
|
||||
0.1367
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -194,23 +186,21 @@
|
||||
},
|
||||
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"median_audio_ttfp_ms": [
|
||||
75.3341,
|
||||
713.7727,
|
||||
5941.7179
|
||||
],
|
||||
"median_audio_rtf": [
|
||||
0.1859,
|
||||
0.3148,
|
||||
1.1662
|
||||
],
|
||||
"audio_throughput": [
|
||||
38.3858,
|
||||
55.3523,
|
||||
68.7032
|
||||
]
|
||||
}
|
||||
"median_audio_ttfp_ms": [
|
||||
75.3341,
|
||||
713.7727,
|
||||
5941.7179
|
||||
],
|
||||
"median_audio_rtf": [
|
||||
0.1859,
|
||||
0.3148,
|
||||
1.1662
|
||||
],
|
||||
"audio_throughput": [
|
||||
38.3858,
|
||||
55.3523,
|
||||
68.7032
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -236,11 +226,9 @@
|
||||
"seed_tts_wer_eval": true,
|
||||
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"mean_audio_rtf": [
|
||||
0.35
|
||||
]
|
||||
}
|
||||
"mean_audio_rtf": [
|
||||
0.35
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -264,17 +252,15 @@
|
||||
},
|
||||
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"median_audio_ttfp_ms": [
|
||||
56.746
|
||||
],
|
||||
"median_audio_rtf": [
|
||||
0.1571
|
||||
],
|
||||
"audio_throughput": [
|
||||
11.1226
|
||||
]
|
||||
}
|
||||
"median_audio_ttfp_ms": [
|
||||
56.746
|
||||
],
|
||||
"median_audio_rtf": [
|
||||
0.1571
|
||||
],
|
||||
"audio_throughput": [
|
||||
11.1226
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -297,14 +283,12 @@
|
||||
},
|
||||
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"median_audio_ttfp_ms": [
|
||||
46.3479
|
||||
],
|
||||
"median_audio_rtf": [
|
||||
0.1352
|
||||
]
|
||||
}
|
||||
"median_audio_ttfp_ms": [
|
||||
46.3479
|
||||
],
|
||||
"median_audio_rtf": [
|
||||
0.1352
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -331,23 +315,21 @@
|
||||
},
|
||||
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"median_audio_ttfp_ms": [
|
||||
75.5967,
|
||||
587.5877,
|
||||
4960.3726
|
||||
],
|
||||
"median_audio_rtf": [
|
||||
0.1871,
|
||||
0.3143,
|
||||
1.1703
|
||||
],
|
||||
"audio_throughput": [
|
||||
38.9378,
|
||||
55.1846,
|
||||
71.5708
|
||||
]
|
||||
}
|
||||
"median_audio_ttfp_ms": [
|
||||
75.5967,
|
||||
587.5877,
|
||||
4960.3726
|
||||
],
|
||||
"median_audio_rtf": [
|
||||
0.1871,
|
||||
0.3143,
|
||||
1.1703
|
||||
],
|
||||
"audio_throughput": [
|
||||
38.9378,
|
||||
55.1846,
|
||||
71.5708
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -370,17 +352,15 @@
|
||||
},
|
||||
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"median_audio_ttfp_ms": [
|
||||
55.5029
|
||||
],
|
||||
"median_audio_rtf": [
|
||||
0.1508
|
||||
],
|
||||
"audio_throughput": [
|
||||
9.4885
|
||||
]
|
||||
}
|
||||
"median_audio_ttfp_ms": [
|
||||
55.5029
|
||||
],
|
||||
"median_audio_rtf": [
|
||||
0.1508
|
||||
],
|
||||
"audio_throughput": [
|
||||
9.4885
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
@@ -422,14 +402,12 @@
|
||||
},
|
||||
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"median_audio_ttfp_ms": [
|
||||
82.1886
|
||||
],
|
||||
"median_audio_rtf": [
|
||||
0.1241
|
||||
]
|
||||
}
|
||||
"median_audio_ttfp_ms": [
|
||||
82.1886
|
||||
],
|
||||
"median_audio_rtf": [
|
||||
0.1241
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -451,17 +429,15 @@
|
||||
},
|
||||
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"median_audio_ttfp_ms": [
|
||||
160.9168
|
||||
],
|
||||
"median_audio_rtf": [
|
||||
0.1907
|
||||
],
|
||||
"audio_throughput": [
|
||||
39.2332
|
||||
]
|
||||
}
|
||||
"median_audio_ttfp_ms": [
|
||||
160.9168
|
||||
],
|
||||
"median_audio_rtf": [
|
||||
0.1907
|
||||
],
|
||||
"audio_throughput": [
|
||||
39.2332
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -483,17 +459,15 @@
|
||||
},
|
||||
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"median_audio_ttfp_ms": [
|
||||
100.1265
|
||||
],
|
||||
"median_audio_rtf": [
|
||||
0.1437
|
||||
],
|
||||
"audio_throughput": [
|
||||
11.7605
|
||||
]
|
||||
}
|
||||
"median_audio_ttfp_ms": [
|
||||
100.1265
|
||||
],
|
||||
"median_audio_rtf": [
|
||||
0.1437
|
||||
],
|
||||
"audio_throughput": [
|
||||
11.7605
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
@@ -37,14 +37,12 @@
|
||||
"seed_tts_locale": "en",
|
||||
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"median_audio_ttfp_ms": [
|
||||
137.645
|
||||
],
|
||||
"median_audio_rtf": [
|
||||
0.0497
|
||||
]
|
||||
}
|
||||
"median_audio_ttfp_ms": [
|
||||
137.645
|
||||
],
|
||||
"median_audio_rtf": [
|
||||
0.0497
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -63,17 +61,15 @@
|
||||
"seed_tts_locale": "en",
|
||||
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"median_audio_ttfp_ms": [
|
||||
475.3357
|
||||
],
|
||||
"median_audio_rtf": [
|
||||
0.1243
|
||||
],
|
||||
"audio_throughput": [
|
||||
36.6358
|
||||
]
|
||||
}
|
||||
"median_audio_ttfp_ms": [
|
||||
475.3357
|
||||
],
|
||||
"median_audio_rtf": [
|
||||
0.1243
|
||||
],
|
||||
"audio_throughput": [
|
||||
36.6358
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -94,11 +90,9 @@
|
||||
"seed_tts_wer_eval": true,
|
||||
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"mean_audio_rtf": [
|
||||
0.45
|
||||
]
|
||||
}
|
||||
"mean_audio_rtf": [
|
||||
0.45
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -120,14 +114,12 @@
|
||||
},
|
||||
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"median_audio_ttfp_ms": [
|
||||
79.6927
|
||||
],
|
||||
"median_audio_rtf": [
|
||||
0.0423
|
||||
]
|
||||
}
|
||||
"median_audio_ttfp_ms": [
|
||||
79.6927
|
||||
],
|
||||
"median_audio_rtf": [
|
||||
0.0423
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -149,17 +141,15 @@
|
||||
},
|
||||
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"median_audio_ttfp_ms": [
|
||||
262.6212
|
||||
],
|
||||
"median_audio_rtf": [
|
||||
0.0753
|
||||
],
|
||||
"audio_throughput": [
|
||||
103.5843
|
||||
]
|
||||
}
|
||||
"median_audio_ttfp_ms": [
|
||||
262.6212
|
||||
],
|
||||
"median_audio_rtf": [
|
||||
0.0753
|
||||
],
|
||||
"audio_throughput": [
|
||||
103.5843
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -181,17 +171,15 @@
|
||||
},
|
||||
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"median_audio_ttfp_ms": [
|
||||
80.1095
|
||||
],
|
||||
"median_audio_rtf": [
|
||||
0.0479
|
||||
],
|
||||
"audio_throughput": [
|
||||
17.2918
|
||||
]
|
||||
}
|
||||
"median_audio_ttfp_ms": [
|
||||
80.1095
|
||||
],
|
||||
"median_audio_rtf": [
|
||||
0.0479
|
||||
],
|
||||
"audio_throughput": [
|
||||
17.2918
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -215,11 +203,9 @@
|
||||
"seed_tts_wer_eval": true,
|
||||
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"mean_audio_rtf": [
|
||||
0.35
|
||||
]
|
||||
}
|
||||
"mean_audio_rtf": [
|
||||
0.35
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
@@ -42,11 +42,9 @@
|
||||
}
|
||||
],
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"throughput_qps": 0.0434,
|
||||
"latency_mean": 23.0654,
|
||||
"peak_memory_mb_mean": 76360.0
|
||||
}
|
||||
"throughput_qps": 0.0434,
|
||||
"latency_mean": 23.0654,
|
||||
"peak_memory_mb_mean": 76360.0
|
||||
}
|
||||
}
|
||||
]
|
||||
@@ -57,8 +55,7 @@
|
||||
{
|
||||
"hardware_marks": {
|
||||
"res": {
|
||||
"cuda": "H100",
|
||||
"npu": "A3"
|
||||
"cuda": "H100"
|
||||
},
|
||||
"num_cards": 2
|
||||
}
|
||||
@@ -99,11 +96,9 @@
|
||||
}
|
||||
],
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"throughput_qps": 0.0606,
|
||||
"latency_mean": 16.4928,
|
||||
"peak_memory_mb_mean": 47164.8
|
||||
}
|
||||
"throughput_qps": 0.0606,
|
||||
"latency_mean": 16.4928,
|
||||
"peak_memory_mb_mean": 47164.8
|
||||
}
|
||||
},
|
||||
{
|
||||
@@ -126,11 +121,9 @@
|
||||
}
|
||||
],
|
||||
"baseline": {
|
||||
"H100": {
|
||||
"throughput_qps": 0.0124,
|
||||
"latency_mean": 80.7784,
|
||||
"peak_memory_mb_mean": 55374.3429
|
||||
}
|
||||
"throughput_qps": 0.0124,
|
||||
"latency_mean": 80.7784,
|
||||
"peak_memory_mb_mean": 55374.3429
|
||||
}
|
||||
}
|
||||
]
|
||||
|
||||
@@ -14,15 +14,18 @@ from tests.helpers.stage_config import get_deploy_config_path
|
||||
|
||||
pytestmark = [pytest.mark.slow, pytest.mark.omni, pytest.mark.full_model]
|
||||
|
||||
_MINICPMO_DEPLOY = get_deploy_config_path("minicpmo_4_5_2gpu.yaml")
|
||||
_MINICPMO_DEPLOY = get_deploy_config_path("minicpmo_4_5_batching.yaml")
|
||||
|
||||
_MINICPMO_SERVER = [
|
||||
pytest.param(
|
||||
OmniServerParams(
|
||||
model="openbmb/MiniCPM-o-4_5",
|
||||
stage_config_path=_MINICPMO_DEPLOY,
|
||||
use_stage_cli=False,
|
||||
server_args=["--trust-remote-code"],
|
||||
use_stage_cli=True,
|
||||
server_args=[
|
||||
"--trust-remote-code",
|
||||
"--no-async-chunk",
|
||||
],
|
||||
),
|
||||
id="minicpmo_4_5",
|
||||
)
|
||||
@@ -48,12 +51,12 @@ def _prompt() -> str:
|
||||
return "What is the capital of China? Answer in 20 words."
|
||||
|
||||
|
||||
@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1)
|
||||
@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=2)
|
||||
@pytest.mark.parametrize("omni_server", _MINICPMO_SERVER, indirect=True)
|
||||
def test_invalid_audio_format_rejected(omni_server: OmniServerParams, openai_client: OpenAIClientHandler) -> None:
|
||||
"""
|
||||
Test that invalid audio format is properly rejected.
|
||||
Deploy Setting: default single GPU
|
||||
Deploy Setting: default 2GPU
|
||||
Input Modal: text + invalid audio format
|
||||
Output Modal: error
|
||||
"""
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import logging
|
||||
import os
|
||||
from types import SimpleNamespace
|
||||
|
||||
@@ -338,27 +339,43 @@ def test_transformer_load_weights_round_trip():
|
||||
assert torch.allclose(reloaded[native_name], expected)
|
||||
|
||||
|
||||
def test_transformer_load_weights_warns_for_unexpected_and_unloaded(caplog):
|
||||
def test_transformer_load_weights_warns_for_unexpected_and_unloaded():
|
||||
from vllm_omni.diffusion.models.boogu_image.boogu_image_transformer import (
|
||||
BooguImageTransformer2DModel,
|
||||
)
|
||||
|
||||
model = BooguImageTransformer2DModel(od_config=_tiny_od_config())
|
||||
native_params = dict(model.named_parameters())
|
||||
loaded_name = next(iter(native_params))
|
||||
checkpoint_name = _native_to_checkpoint_name(loaded_name)
|
||||
# vllm_omni's logger hierarchy hangs off vLLM's `vllm` logger, which sets
|
||||
# propagate=False, so records never reach the root logger that pytest's
|
||||
# caplog listens on. Capture at the emitting logger instead.
|
||||
messages: list[str] = []
|
||||
|
||||
loaded = model.load_weights(
|
||||
[
|
||||
(checkpoint_name, torch.randn_like(native_params[loaded_name])),
|
||||
("unexpected.weight", torch.ones(1)),
|
||||
]
|
||||
)
|
||||
class _Capture(logging.Handler):
|
||||
def emit(self, record: logging.LogRecord) -> None:
|
||||
messages.append(record.getMessage())
|
||||
|
||||
log = logging.getLogger("vllm_omni.diffusion.models.boogu_image.boogu_image_transformer")
|
||||
handler = _Capture(level=logging.WARNING)
|
||||
log.addHandler(handler)
|
||||
try:
|
||||
model = BooguImageTransformer2DModel(od_config=_tiny_od_config())
|
||||
native_params = dict(model.named_parameters())
|
||||
loaded_name = next(iter(native_params))
|
||||
checkpoint_name = _native_to_checkpoint_name(loaded_name)
|
||||
|
||||
loaded = model.load_weights(
|
||||
[
|
||||
(checkpoint_name, torch.randn_like(native_params[loaded_name])),
|
||||
("unexpected.weight", torch.ones(1)),
|
||||
]
|
||||
)
|
||||
finally:
|
||||
log.removeHandler(handler)
|
||||
|
||||
text = "\n".join(messages)
|
||||
assert loaded == {loaded_name}
|
||||
assert "Skipping unexpected checkpoint weight unexpected.weight" in caplog.text
|
||||
assert "Model parameters not loaded from checkpoint" in caplog.text
|
||||
assert next(name for name in native_params if name != loaded_name) in caplog.text
|
||||
assert "Skipping unexpected checkpoint weight unexpected.weight" in text
|
||||
assert "Model parameters not loaded from checkpoint" in text
|
||||
assert next(name for name in native_params if name != loaded_name) in text
|
||||
|
||||
|
||||
def test_transformer_forward_t2i_shape():
|
||||
|
||||
@@ -71,30 +71,6 @@ def test_streaming_payload_can_replace_placeholder_prompt(mocker: MockerFixture)
|
||||
request.update_block_hashes.assert_called_once_with()
|
||||
|
||||
|
||||
def test_streaming_payload_can_append_exact_prompt_length(mocker: MockerFixture) -> None:
|
||||
request = SimpleNamespace(
|
||||
_all_token_ids=[0, 0, 7, 8],
|
||||
_output_token_ids=[7, 8],
|
||||
prompt_token_ids=[0, 0],
|
||||
num_computed_tokens=4,
|
||||
num_prompt_tokens=2,
|
||||
update_block_hashes=mocker.Mock(),
|
||||
)
|
||||
payload = {
|
||||
"ids": {"prompt": [1, 2, 3]},
|
||||
"meta": {"next_stage_prompt_len": 3},
|
||||
}
|
||||
|
||||
construct_next_stage_streaming_input_prompt(payload, request)
|
||||
|
||||
assert request.prompt_token_ids == [0, 0, 7, 8, 0, 0, 0]
|
||||
assert request._all_token_ids == [0, 0, 7, 8, 0, 0, 0]
|
||||
assert request._output_token_ids == []
|
||||
assert request.num_computed_tokens == 4
|
||||
assert request.num_prompt_tokens == 7
|
||||
request.update_block_hashes.assert_called_once_with()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def build_adapter(monkeypatch, mocker: MockerFixture):
|
||||
def _build(
|
||||
@@ -369,23 +345,6 @@ def test_send_single_request_struct_preserves_segment_finished(build_adapter, mo
|
||||
assert sent_payload.meta.is_segment_finished.item() is True
|
||||
|
||||
|
||||
def test_send_single_request_respects_processor_receiver_boundary(build_adapter, monkeypatch):
|
||||
adapter, connector = build_adapter(stage_id=1)
|
||||
request = _req("req-stream", RequestStatus.WAITING, external_req_id="ext-stream")
|
||||
|
||||
adapter.custom_process_next_stage_input_func = lambda **kwargs: OmniPayloadStruct(
|
||||
meta=MetaStruct(is_segment_finished=torch.tensor(False, dtype=torch.bool))
|
||||
)
|
||||
monkeypatch.setattr(adapter, "cleanup", lambda *a, **kw: None)
|
||||
|
||||
adapter._send_single_request(
|
||||
{"multimodal_output": None, "request": request, "is_finished": False, "is_segment_finished": True}
|
||||
)
|
||||
|
||||
sent_payload = connector.put.call_args.kwargs["data"]
|
||||
assert sent_payload.meta.is_segment_finished.item() is False
|
||||
|
||||
|
||||
def test_save_async_skips_stale_resumable_chunk_until_dedup_is_reset(build_adapter):
|
||||
adapter, _ = build_adapter(stage_id=1)
|
||||
request = _req("req-stream", RequestStatus.WAITING, external_req_id="ext-stream")
|
||||
@@ -535,7 +494,7 @@ def test_fifo_promotion(build_adapter):
|
||||
adapter.process_pending_chunks(waiting_queue, running_queue)
|
||||
|
||||
assert list(adapter._active_streams) == ["req-1", "req-2"]
|
||||
assert waiting_queue == []
|
||||
assert waiting_queue == reqs[2:]
|
||||
assert reqs[0].status == RequestStatus.WAITING_FOR_CHUNK
|
||||
assert reqs[1].status == RequestStatus.WAITING_FOR_CHUNK
|
||||
assert reqs[2].status == RequestStatus.WAITING
|
||||
@@ -545,7 +504,6 @@ def test_fifo_promotion(build_adapter):
|
||||
# (commit c4d95fd9 — otherwise the terminal chunk deadlocks at c=8 K=2).
|
||||
# Simulate it here so promotion can pick up the freed slot.
|
||||
adapter._evict_finished_active_streams({"req-1"})
|
||||
adapter.restore_queues(waiting_queue, running_queue)
|
||||
adapter.process_pending_chunks(waiting_queue, running_queue)
|
||||
|
||||
assert list(adapter._active_streams) == ["req-2", "req-3"]
|
||||
@@ -554,22 +512,6 @@ def test_fifo_promotion(build_adapter):
|
||||
assert reqs[2].status == RequestStatus.WAITING_FOR_CHUNK
|
||||
|
||||
|
||||
def test_non_active_waiting_request_is_held_off_scheduler(build_adapter):
|
||||
adapter, _ = build_adapter(stage_id=2, max_num_seqs=2, active_stream_window=1)
|
||||
active = _req("req-active", RequestStatus.WAITING)
|
||||
non_active = _req("req-non-active", RequestStatus.WAITING)
|
||||
waiting_queue = DummyWaitingQueue([active, non_active])
|
||||
running_queue = []
|
||||
|
||||
adapter.process_pending_chunks(waiting_queue, running_queue)
|
||||
|
||||
assert waiting_queue == []
|
||||
assert active.status == RequestStatus.WAITING_FOR_CHUNK
|
||||
assert non_active.status == RequestStatus.WAITING
|
||||
assert list(adapter._pending_load_reqs) == [active]
|
||||
assert list(adapter.waiting_for_chunk_waiting_requests) == [active, non_active]
|
||||
|
||||
|
||||
def test_legacy_k0(build_adapter):
|
||||
adapter, _ = build_adapter(stage_id=1, max_num_seqs=1, active_stream_window=0)
|
||||
waiting_req = _req("waiting", RequestStatus.WAITING)
|
||||
@@ -598,13 +540,12 @@ def test_finished_releases_slot(build_adapter):
|
||||
|
||||
adapter.process_pending_chunks(waiting_queue, running_queue)
|
||||
assert list(adapter._active_streams) == ["req-1"]
|
||||
assert waiting_queue == []
|
||||
assert waiting_queue == [req_2]
|
||||
|
||||
adapter.finished_requests.add("req-1")
|
||||
# Eviction is deferred to postprocess_scheduler_output in the runtime path
|
||||
# (commit c4d95fd9). Simulate it so promotion can pick up the freed slot.
|
||||
adapter._evict_finished_active_streams({"req-1"})
|
||||
adapter.restore_queues(waiting_queue, running_queue)
|
||||
adapter.process_pending_chunks(waiting_queue, running_queue)
|
||||
|
||||
assert list(adapter._active_streams) == ["req-2"]
|
||||
@@ -870,6 +811,10 @@ def test_restore_queues_skips_requests_missing_from_scheduler_requests(build_ada
|
||||
class _HashableRequest(SimpleNamespace):
|
||||
"""SimpleNamespace that can be added to a set (needed by scheduler internals)."""
|
||||
|
||||
# vLLM 0.26: update_from_output settles this counter for every scheduled
|
||||
# request; real Requests initialise it to 0 (vllm/v1/request.py).
|
||||
num_in_flight_tokens = 0
|
||||
|
||||
def __hash__(self):
|
||||
return hash(self.request_id)
|
||||
|
||||
@@ -923,7 +868,7 @@ def test_generation_scheduler_calls_cleanup_on_finished(monkeypatch, mocker: Moc
|
||||
scheduler.requests = {"req-s1": request}
|
||||
|
||||
scheduler._handle_stopped_request = mocker.MagicMock(return_value=True)
|
||||
scheduler._free_request = mocker.MagicMock(return_value=None)
|
||||
scheduler._free_request = mocker.MagicMock(return_value=(None, None))
|
||||
scheduler._get_routed_experts = mocker.MagicMock(return_value=None)
|
||||
scheduler.running = [request]
|
||||
scheduler.waiting = mocker.MagicMock()
|
||||
@@ -1008,7 +953,7 @@ def test_ar_scheduler_defers_cleanup_and_queues_save_on_finished(mocker: MockerF
|
||||
scheduler._update_request_with_output = mocker.MagicMock(return_value=([], True))
|
||||
scheduler._process_kv_transfer_trigger = mocker.MagicMock(return_value=False)
|
||||
scheduler._handle_stopped_request = mocker.MagicMock(return_value=True)
|
||||
scheduler._free_request = mocker.MagicMock(return_value=None)
|
||||
scheduler._free_request = mocker.MagicMock(return_value=(None, None))
|
||||
scheduler._get_routed_experts = mocker.MagicMock(return_value=None)
|
||||
scheduler.running = [request]
|
||||
scheduler.waiting = mocker.MagicMock()
|
||||
@@ -1137,7 +1082,7 @@ def _build_deferred_finish_scheduler(mocker, *, running, pending_finish_reqs):
|
||||
scheduler._pending_finish_reqs = list(pending_finish_reqs)
|
||||
|
||||
scheduler._handle_stopped_request = mocker.MagicMock(return_value=True)
|
||||
scheduler._free_request = mocker.MagicMock(return_value=None)
|
||||
scheduler._free_request = mocker.MagicMock(return_value=(None, None))
|
||||
scheduler._get_routed_experts = mocker.MagicMock(return_value=None)
|
||||
scheduler.running = list(running)
|
||||
scheduler.waiting = mocker.MagicMock()
|
||||
@@ -1193,7 +1138,7 @@ def test_deferred_finish_emits_finished_output(mocker: MockerFixture):
|
||||
running=[request],
|
||||
pending_finish_reqs=[request],
|
||||
)
|
||||
scheduler._free_request.return_value = {"mock": "kv_params"}
|
||||
scheduler._free_request.return_value = ({"mock": "kv_params"}, None)
|
||||
|
||||
result = OmniGenerationScheduler.update_from_output(scheduler, sched_out, model_out)
|
||||
|
||||
|
||||
@@ -330,6 +330,31 @@ def test_normalize_layer_kv_rejects_invalid_inputs(kv_config, common_constants,
|
||||
assert normalized is None
|
||||
|
||||
|
||||
def test_normalize_layer_kv_unpacks_packed_layout(common_constants):
|
||||
"""The vLLM >= 0.23 packed layout (num_blocks, n_heads, block_size,
|
||||
2*head_dim) is unpacked into (num_blocks, block_size, n_heads, head_dim)
|
||||
key/value blocks when block_size is provided."""
|
||||
block_size = common_constants["block_size"]
|
||||
num_heads = common_constants["num_heads"]
|
||||
head_dim = common_constants["head_dim"]
|
||||
req_id = common_constants["req_id"]
|
||||
num_blocks = 10
|
||||
|
||||
layer_kv = torch.randn(num_blocks, num_heads, block_size, 2 * head_dim)
|
||||
|
||||
# Without block_size the 4-D packed layout is not recognized.
|
||||
assert normalize_layer_kv(layer_kv, req_id=req_id, layer_idx=0) is None
|
||||
|
||||
normalized = normalize_layer_kv(layer_kv, req_id=req_id, layer_idx=0, block_size=block_size)
|
||||
assert normalized is not None
|
||||
key_blocks, value_blocks = normalized
|
||||
assert key_blocks.shape == (num_blocks, block_size, num_heads, head_dim)
|
||||
assert value_blocks.shape == (num_blocks, block_size, num_heads, head_dim)
|
||||
reference = layer_kv.transpose(1, 2)
|
||||
assert torch.equal(key_blocks, reference[..., :head_dim])
|
||||
assert torch.equal(value_blocks, reference[..., head_dim:])
|
||||
|
||||
|
||||
def test_manager_reception(kv_config, mock_connector, common_constants):
|
||||
"""Test reception and injection logic in OmniKVTransferManager."""
|
||||
num_layers = common_constants["num_layers"]
|
||||
|
||||
@@ -1 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
@@ -1,296 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""MiniCPM-o 4.5 Daily-Omni + Seed-TTS + Video-MME accuracy regression coverage.
|
||||
|
||||
Daily-Omni settings follow the MiniCPM interleaved AV recipe (``minicpm-interleave``,
|
||||
``temperature=0``, text modalities with ``use_tts_template``, and server
|
||||
``--interleave-mm-strings`` + 1fps / 128-frame media-io kwargs, also pinned in
|
||||
``minicpmo_4_5.yaml``). Baseline **79.5**; gate **77.5** (drop ≤ 2pp).
|
||||
|
||||
Video-MME settings follow OpenBMB OmniEvalKit MiniCPM ``videomme`` (w/o subs):
|
||||
``minicpm-frames``, ``max_frames=96``, ``temperature=0``, ``output_len=128``.
|
||||
Baseline **69.0**; gate **67.0** (drop ≤ 2pp).
|
||||
|
||||
Seed-TTS (zh, seed-tts-eval protocol): WER baseline **1.414%** / gate **≤ 1.56%**
|
||||
(+10%).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.e2e.accuracy.qwen3_omni import run_qwen_omni_acc_benchmark as _acc_bench
|
||||
from tests.e2e.accuracy.qwen3_omni.qwen3_omni_acc_bench_core import (
|
||||
build_acc_benchmark_cli_argv,
|
||||
find_vllm_cli,
|
||||
)
|
||||
from tests.helpers.mark import hardware_test
|
||||
from tests.helpers.runtime import OmniServerParams
|
||||
from tests.helpers.stage_config import get_deploy_config_path
|
||||
|
||||
_MODEL = os.environ.get("VLLM_TEST_MINICPMO_4_5_MODEL", "openbmb/MiniCPM-o-4_5")
|
||||
_DEPLOY_CONFIG = get_deploy_config_path("minicpmo_4_5.yaml")
|
||||
_RESULT_DIR = Path(
|
||||
os.environ.get(
|
||||
"ACC_BENCH_RESULT_DIR",
|
||||
str(Path(__file__).resolve().parent / "results"),
|
||||
)
|
||||
)
|
||||
|
||||
# Accuracy gates: baseline minus allowed drop (≤2pp / ≤10% WER).
|
||||
_MIN_DAILY_OMNI_ACCURACY = 0.775 # 79.5 - 2pp
|
||||
_MIN_VIDEOMME_ACCURACY = float(os.environ.get("ACC_BENCH_MIN_VIDEOMME_ACCURACY", "0.67"))
|
||||
# jiwer reports a 0-1 ratio; table 1.414 / 1.56 is percent → 0.0156.
|
||||
_MAX_SEED_TTS_MEAN_WER = 0.0156
|
||||
# Full Seed-TTS zh split (seed-tts-eval/zh/meta.lst) is 2020 rows.
|
||||
_SEED_TTS_LOCALE = "zh"
|
||||
_SEED_TTS_NUM_PROMPTS = 2020
|
||||
# Match the validated Daily-Omni / Video-MME client body from the MiniCPM run scripts.
|
||||
_DAILY_EXTRA_BODY = {
|
||||
"modalities": ["text"],
|
||||
"chat_template_kwargs": {"enable_thinking": False},
|
||||
}
|
||||
_VIDEOMME_EXTRA_BODY = {
|
||||
"modalities": ["text"],
|
||||
"chat_template_kwargs": {"enable_thinking": False},
|
||||
}
|
||||
_SEED_EXTRA_BODY = {
|
||||
"modalities": ["text", "audio"],
|
||||
"chat_template_kwargs": {
|
||||
"enable_thinking": False,
|
||||
"use_tts_template": True,
|
||||
},
|
||||
}
|
||||
# Server flags required for MiniCPM interleaved image/audio packs (Daily-Omni only).
|
||||
# Do not reuse these for Seed-TTS: ``--interleave-mm-strings`` + TTS ref_audio can trip
|
||||
# msgspec ValidationError and kill the orchestrator mid-suite.
|
||||
# Also pinned in ``minicpmo_4_5.yaml`` stage 0 so they survive config-path routing.
|
||||
_DAILY_OMNI_SERVER_ARGS = [
|
||||
"--trust-remote-code",
|
||||
"--interleave-mm-strings",
|
||||
"--media-io-kwargs",
|
||||
'{"video":{"fps":1,"num_frames":128}}',
|
||||
]
|
||||
# Video-MME ``minicpm-frames`` sends sampled frames as image_url (no AV interleave).
|
||||
# Allow local ``file://`` frame-cache URLs so CI does not need megabyte-scale base64
|
||||
# payloads; the autouse fixture still appends ``--videomme-inline-local-video`` when
|
||||
# the allowlist is unavailable.
|
||||
_VIDEOMME_SERVER_ARGS = [
|
||||
"--trust-remote-code",
|
||||
"--allowed-local-media-path",
|
||||
os.environ.get("VLLM_TEST_ALLOWED_LOCAL_MEDIA_PATH", "/"),
|
||||
]
|
||||
_SEED_TTS_SERVER_ARGS = [
|
||||
"--trust-remote-code",
|
||||
]
|
||||
|
||||
pytestmark = [pytest.mark.full_model, pytest.mark.omni]
|
||||
|
||||
# Evaluated at setup time, before the ``omni_server`` fixture boots a server. An
|
||||
# ``importorskip`` inside the test body would skip only after paying several minutes
|
||||
# of startup, and the resulting skip is easy to miss in a nightly log.
|
||||
_missing_daily_omni_deps = [name for name in ("datasets", "huggingface_hub") if importlib.util.find_spec(name) is None]
|
||||
requires_daily_omni_deps = pytest.mark.skipif(
|
||||
bool(_missing_daily_omni_deps),
|
||||
reason=f"Daily-Omni accuracy bench needs {', '.join(_missing_daily_omni_deps)}",
|
||||
)
|
||||
|
||||
daily_test_params = [
|
||||
OmniServerParams(
|
||||
model=_MODEL,
|
||||
stage_config_path=_DEPLOY_CONFIG,
|
||||
server_args=list(_DAILY_OMNI_SERVER_ARGS),
|
||||
)
|
||||
]
|
||||
|
||||
videomme_test_params = [
|
||||
OmniServerParams(
|
||||
model=_MODEL,
|
||||
stage_config_path=_DEPLOY_CONFIG,
|
||||
server_args=list(_VIDEOMME_SERVER_ARGS),
|
||||
)
|
||||
]
|
||||
|
||||
seed_test_params = [
|
||||
OmniServerParams(
|
||||
model=_MODEL,
|
||||
stage_config_path=_DEPLOY_CONFIG,
|
||||
server_args=list(_SEED_TTS_SERVER_ARGS),
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
def _require_vllm_cli() -> None:
|
||||
try:
|
||||
find_vllm_cli()
|
||||
except FileNotFoundError as exc:
|
||||
pytest.skip(str(exc))
|
||||
|
||||
|
||||
def _optional_local_videomme_dataset_path() -> str | None:
|
||||
"""Return an explicit local Video-MME root, or ``None`` to use the Hub default.
|
||||
|
||||
Same rule as Daily-Omni / Seed-TTS: only an env-provided existing directory counts as
|
||||
local. Hard-coded workspace paths are intentionally not probed so CI defaults to the
|
||||
Hub id in ``VIDEOMME_DEFAULT_HF_REPO`` (``lmms-eval/Video-MME``).
|
||||
"""
|
||||
for key in ("VLLM_VIDEOMME_DATASET_PATH", "VIDEOMME_ROOT"):
|
||||
raw = os.environ.get(key, "").strip()
|
||||
if not raw:
|
||||
continue
|
||||
path = Path(raw).expanduser()
|
||||
if path.is_absolute() and path.is_dir():
|
||||
return str(path.resolve())
|
||||
return None
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _inline_local_media_when_needed(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""Inline Hub/local media when the server has no usable local-media allowlist.
|
||||
|
||||
Daily-Omni always inlines (matches the historical Qwen/MiniCPM accuracy fixture).
|
||||
Video-MME prefers ``file://`` via ``--allowed-local-media-path`` (see
|
||||
``_VIDEOMME_SERVER_ARGS``); force inline only when ``VLLM_VIDEOMME_FORCE_INLINE=1``.
|
||||
"""
|
||||
original_daily = _acc_bench.daily_omni_bench_argv
|
||||
original_videomme = _acc_bench.videomme_bench_argv
|
||||
|
||||
def _wrap_daily() -> list[str]:
|
||||
argv = list(original_daily())
|
||||
if "--daily-omni-inline-local-video" not in argv:
|
||||
argv.append("--daily-omni-inline-local-video")
|
||||
return argv
|
||||
|
||||
def _wrap_videomme() -> list[str]:
|
||||
argv = list(original_videomme())
|
||||
force_inline = os.environ.get("VLLM_VIDEOMME_FORCE_INLINE", "").strip().lower() in {
|
||||
"1",
|
||||
"true",
|
||||
"yes",
|
||||
}
|
||||
if force_inline and "--videomme-inline-local-video" not in argv:
|
||||
argv.append("--videomme-inline-local-video")
|
||||
return argv
|
||||
|
||||
monkeypatch.setattr(_acc_bench, "daily_omni_bench_argv", _wrap_daily)
|
||||
monkeypatch.setattr(_acc_bench, "videomme_bench_argv", _wrap_videomme)
|
||||
monkeypatch.setenv("VLLM_WORKER_MULTIPROC_METHOD", "spawn")
|
||||
|
||||
|
||||
@requires_daily_omni_deps
|
||||
@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1)
|
||||
@pytest.mark.parametrize("omni_server", daily_test_params, indirect=True)
|
||||
def test_minicpmo_4_5_daily_omni_accuracy_bench(omni_server) -> None:
|
||||
_require_vllm_cli()
|
||||
|
||||
argv = build_acc_benchmark_cli_argv(
|
||||
omni_server,
|
||||
skip_seed=True,
|
||||
skip_daily=False,
|
||||
skip_videomme=True,
|
||||
)
|
||||
argv.extend(
|
||||
[
|
||||
"--result-dir",
|
||||
str(_RESULT_DIR),
|
||||
"--min-daily-omni-accuracy",
|
||||
str(_MIN_DAILY_OMNI_ACCURACY),
|
||||
"--temperature",
|
||||
"0",
|
||||
"--output-len",
|
||||
"512",
|
||||
"--daily-omni-input-mode",
|
||||
"all",
|
||||
"--daily-omni-pack-mode",
|
||||
"minicpm-interleave",
|
||||
"--daily-extra-body-json",
|
||||
json.dumps(_DAILY_EXTRA_BODY, separators=(",", ":")),
|
||||
"--trust-remote-code",
|
||||
]
|
||||
)
|
||||
|
||||
assert _acc_bench.run_acc_benchmark(_acc_bench.parse_acc_benchmark_args(argv)) == 0
|
||||
|
||||
|
||||
@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1)
|
||||
@pytest.mark.parametrize("omni_server", videomme_test_params, indirect=True)
|
||||
def test_minicpmo_4_5_videomme_accuracy_bench(omni_server) -> None:
|
||||
"""Gate MiniCPM-o 4.5 Video-MME (w/o subs) overall accuracy at the OmniEvalKit recipe."""
|
||||
_require_vllm_cli()
|
||||
pytest.importorskip("datasets")
|
||||
pytest.importorskip("huggingface_hub")
|
||||
pytest.importorskip("av")
|
||||
|
||||
dataset_path = _optional_local_videomme_dataset_path()
|
||||
num_prompts = int(os.environ.get("ACC_BENCH_VIDEOMME_NUM_PROMPTS", "2700"))
|
||||
max_concurrency = int(os.environ.get("ACC_BENCH_VIDEOMME_MAX_CONCURRENCY", "4"))
|
||||
|
||||
argv = build_acc_benchmark_cli_argv(
|
||||
omni_server,
|
||||
skip_seed=True,
|
||||
skip_daily=True,
|
||||
skip_videomme=False,
|
||||
num_prompts=num_prompts,
|
||||
max_concurrency=max_concurrency,
|
||||
)
|
||||
argv.extend(
|
||||
[
|
||||
"--result-dir",
|
||||
str(_RESULT_DIR),
|
||||
"--min-videomme-accuracy",
|
||||
str(_MIN_VIDEOMME_ACCURACY),
|
||||
"--temperature",
|
||||
"0",
|
||||
"--output-len",
|
||||
"128",
|
||||
"--videomme-pack-mode",
|
||||
"minicpm-frames",
|
||||
"--videomme-max-frames",
|
||||
"96",
|
||||
"--videomme-duration",
|
||||
os.environ.get("ACC_BENCH_VIDEOMME_DURATION", "all"),
|
||||
"--videomme-extra-body-json",
|
||||
json.dumps(_VIDEOMME_EXTRA_BODY, separators=(",", ":")),
|
||||
"--trust-remote-code",
|
||||
]
|
||||
)
|
||||
# Absolute local root only; otherwise --videomme-repo (Hub) from build_acc_benchmark_cli_argv.
|
||||
if dataset_path is not None:
|
||||
argv.extend(["--videomme-dataset-path", dataset_path])
|
||||
|
||||
assert _acc_bench.run_acc_benchmark(_acc_bench.parse_acc_benchmark_args(argv)) == 0
|
||||
|
||||
|
||||
@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1)
|
||||
@pytest.mark.parametrize("omni_server", seed_test_params, indirect=True)
|
||||
def test_minicpmo_4_5_seed_tts_wer_bench(omni_server) -> None:
|
||||
_require_vllm_cli()
|
||||
pytest.importorskip("huggingface_hub")
|
||||
|
||||
argv = build_acc_benchmark_cli_argv(
|
||||
omni_server,
|
||||
skip_seed=False,
|
||||
skip_daily=True,
|
||||
skip_videomme=True,
|
||||
num_prompts=_SEED_TTS_NUM_PROMPTS,
|
||||
max_concurrency=4,
|
||||
)
|
||||
argv.extend(
|
||||
[
|
||||
"--result-dir",
|
||||
str(_RESULT_DIR),
|
||||
"--max-seed-tts-mean-wer",
|
||||
str(_MAX_SEED_TTS_MEAN_WER),
|
||||
"--seed-tts-locale",
|
||||
_SEED_TTS_LOCALE,
|
||||
"--seed-extra-body-json",
|
||||
json.dumps(_SEED_EXTRA_BODY, separators=(",", ":")),
|
||||
"--trust-remote-code",
|
||||
]
|
||||
)
|
||||
|
||||
assert _acc_bench.run_acc_benchmark(_acc_bench.parse_acc_benchmark_args(argv)) == 0
|
||||
@@ -1,5 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Shared helpers for Omni Daily-Omni / Seed-TTS / Video-MME ``vllm bench serve --omni`` accuracy runs.
|
||||
"""Shared helpers for Qwen3-Omni Daily-Omni / Seed-TTS ``vllm bench serve --omni`` accuracy runs.
|
||||
|
||||
Local dataset paths are **optional**. When ``VLLM_DAILY_OMNI_QA_JSON`` + ``VLLM_DAILY_OMNI_VIDEO_DIR``
|
||||
point to existing files, those are used with inline video. Otherwise the benchmark falls back to
|
||||
@@ -9,10 +9,6 @@ bench request that needs media downloads ``Videos.tar`` from the Hub when no vid
|
||||
Similarly for Seed-TTS: a local directory wins; otherwise ``--dataset-path`` uses the Hub id
|
||||
and ``huggingface_hub.snapshot_download`` inside ``resolve_seed_tts_root`` pulls files on demand.
|
||||
|
||||
Video-MME follows the same pattern: an existing local directory via ``VLLM_VIDEOMME_DATASET_PATH`` /
|
||||
``VIDEOMME_ROOT`` (or ``--videomme-dataset-path`` / ``--dataset-path``) wins; otherwise the Hub id
|
||||
``lmms-eval/Video-MME`` is used and ``resolve_videomme_root`` downloads parquet + video archives.
|
||||
|
||||
Use :func:`build_acc_benchmark_cli_argv` to assemble ``argv`` for a live Omni server (host/port/model
|
||||
and small bench defaults) before ``parse_args`` / ``run_acc_benchmark`` in the accuracy driver.
|
||||
"""
|
||||
@@ -23,13 +19,11 @@ import json
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any, Protocol
|
||||
|
||||
DEFAULT_DAILY_OMNI_HF_REPO = "liarliar/Daily-Omni"
|
||||
DEFAULT_SEED_TTS_HF_REPO = "zhaochenyang20/seed-tts-eval"
|
||||
DEFAULT_VIDEOMME_HF_REPO = "lmms-eval/Video-MME"
|
||||
|
||||
|
||||
class OmniBenchServerEndpoint(Protocol):
|
||||
@@ -45,7 +39,6 @@ def build_acc_benchmark_cli_argv(
|
||||
*,
|
||||
skip_seed: bool,
|
||||
skip_daily: bool,
|
||||
skip_videomme: bool = True,
|
||||
num_prompts: int | None = None,
|
||||
max_concurrency: int | None = None,
|
||||
) -> list[str]:
|
||||
@@ -55,9 +48,6 @@ def build_acc_benchmark_cli_argv(
|
||||
``--num-prompts`` / ``--max-concurrency`` defaults (overridable via ``ACC_BENCH_NUM_PROMPTS`` /
|
||||
``ACC_BENCH_MAX_CONCURRENCY``), and when Daily-Omni runs adds ``--daily-omni-repo`` so Hub QA
|
||||
matches :func:`daily_omni_bench_argv` once ``run_acc_benchmark`` mirrors ``--daily-omni-repo`` into env.
|
||||
|
||||
Video-MME is **opt-in** (``skip_videomme=True`` by default) so existing Daily-Omni / Seed-TTS
|
||||
callers stay unchanged; pass ``skip_videomme=False`` (emits ``--run-videomme``) to enable it.
|
||||
"""
|
||||
n_prompts = int(os.environ.get("ACC_BENCH_NUM_PROMPTS", "2000")) if num_prompts is None else int(num_prompts)
|
||||
n_conc = int(os.environ.get("ACC_BENCH_MAX_CONCURRENCY", "10")) if max_concurrency is None else int(max_concurrency)
|
||||
@@ -80,12 +70,6 @@ def build_acc_benchmark_cli_argv(
|
||||
argv.append("--skip-seed-tts")
|
||||
if skip_daily:
|
||||
argv.append("--skip-daily-omni")
|
||||
if skip_videomme:
|
||||
argv.append("--skip-videomme")
|
||||
else:
|
||||
argv.append("--run-videomme")
|
||||
repo = os.environ.get("VLLM_VIDEOMME_REPO", DEFAULT_VIDEOMME_HF_REPO).strip() or DEFAULT_VIDEOMME_HF_REPO
|
||||
argv.extend(["--videomme-repo", repo])
|
||||
return argv
|
||||
|
||||
|
||||
@@ -115,43 +99,6 @@ def daily_omni_bench_argv() -> list[str]:
|
||||
]
|
||||
|
||||
|
||||
def videomme_bench_argv() -> list[str]:
|
||||
"""CLI args for Video-MME (after ``vllm bench serve --omni``).
|
||||
|
||||
Same local-vs-Hub rule as :func:`seed_tts_bench_argv`:
|
||||
|
||||
* If ``VLLM_VIDEOMME_DATASET_PATH`` / ``VIDEOMME_ROOT`` names an existing directory,
|
||||
use that local mirror as ``--dataset-path``.
|
||||
* Otherwise pass the Hub id (``VLLM_VIDEOMME_REPO`` / ``lmms-eval/Video-MME``); the child
|
||||
bench downloads via ``huggingface_hub.snapshot_download``.
|
||||
"""
|
||||
root = os.environ.get("VLLM_VIDEOMME_DATASET_PATH", "").strip() or os.environ.get("VIDEOMME_ROOT", "").strip()
|
||||
if root:
|
||||
p = Path(root).expanduser()
|
||||
# Preserve Hub ids verbatim. Only canonicalize when the value is a real directory.
|
||||
if p.exists() and p.is_dir():
|
||||
return [
|
||||
"--dataset-name",
|
||||
"videomme",
|
||||
"--dataset-path",
|
||||
str(p.resolve()),
|
||||
]
|
||||
# Non-directory values (e.g. accidental Hub id in DATASET_PATH) fall through as path.
|
||||
return [
|
||||
"--dataset-name",
|
||||
"videomme",
|
||||
"--dataset-path",
|
||||
root,
|
||||
]
|
||||
repo = os.environ.get("VLLM_VIDEOMME_REPO", DEFAULT_VIDEOMME_HF_REPO).strip() or DEFAULT_VIDEOMME_HF_REPO
|
||||
return [
|
||||
"--dataset-name",
|
||||
"videomme",
|
||||
"--dataset-path",
|
||||
repo,
|
||||
]
|
||||
|
||||
|
||||
def seed_tts_bench_argv(*, locale: str = "en") -> list[str]:
|
||||
"""CLI args for Seed-TTS (after ``vllm bench serve --omni``)."""
|
||||
dp = os.environ.get("VLLM_SEED_TTS_DATASET_PATH", "").strip()
|
||||
@@ -174,13 +121,9 @@ def seed_tts_bench_argv(*, locale: str = "en") -> list[str]:
|
||||
|
||||
def find_vllm_cli() -> str:
|
||||
exe = shutil.which("vllm")
|
||||
if exe:
|
||||
return exe
|
||||
# Fallback when pytest is invoked via absolute venv path without activating PATH.
|
||||
sibling = Path(sys.executable).resolve().parent / "vllm"
|
||||
if sibling.is_file() and os.access(sibling, os.X_OK):
|
||||
return str(sibling)
|
||||
raise FileNotFoundError("Could not find `vllm` on PATH (install vLLM-Omni with CLI entrypoints).")
|
||||
if not exe:
|
||||
raise FileNotFoundError("Could not find `vllm` on PATH (install vLLM-Omni with CLI entrypoints).")
|
||||
return exe
|
||||
|
||||
|
||||
def run_vllm_bench_subprocess(vllm: str, argv: list[str], *, extra_env: dict[str, str] | None = None) -> None:
|
||||
@@ -207,11 +150,6 @@ def build_serve_common_argv(
|
||||
result_dir: Path,
|
||||
result_filename: str,
|
||||
ready_check_timeout_sec: int | None = None,
|
||||
trust_remote_code: bool = False,
|
||||
backend: str = "openai-chat-omni",
|
||||
endpoint: str = "/v1/chat/completions",
|
||||
temperature: float | None = None,
|
||||
output_len: int | None = None,
|
||||
) -> list[str]:
|
||||
out = [
|
||||
"bench",
|
||||
@@ -224,9 +162,9 @@ def build_serve_common_argv(
|
||||
"--model",
|
||||
model,
|
||||
"--endpoint",
|
||||
endpoint,
|
||||
"/v1/chat/completions",
|
||||
"--backend",
|
||||
backend,
|
||||
"openai-chat-omni",
|
||||
"--request-rate",
|
||||
"inf",
|
||||
"--num-prompts",
|
||||
@@ -246,12 +184,6 @@ def build_serve_common_argv(
|
||||
]
|
||||
if ready_check_timeout_sec is not None:
|
||||
out.extend(["--ready-check-timeout-sec", str(int(ready_check_timeout_sec))])
|
||||
if trust_remote_code:
|
||||
out.append("--trust-remote-code")
|
||||
if temperature is not None:
|
||||
out.extend(["--temperature", str(temperature)])
|
||||
if output_len is not None:
|
||||
out.extend(["--output-len", str(int(output_len))])
|
||||
return out
|
||||
|
||||
|
||||
@@ -261,12 +193,6 @@ def assert_daily_omni_scored(result: dict[str, Any]) -> None:
|
||||
assert int(result.get("daily_omni_evaluated_ok", 0) or 0) > 0, "no successful MCQ rows (daily_omni_evaluated_ok==0)"
|
||||
|
||||
|
||||
def assert_videomme_scored(result: dict[str, Any]) -> None:
|
||||
acc = result.get("videomme_accuracy")
|
||||
assert acc is not None, "videomme_accuracy missing — wrong dataset or benchmark wiring"
|
||||
assert int(result.get("videomme_evaluated_ok", 0) or 0) > 0, "no successful MCQ rows (videomme_evaluated_ok==0)"
|
||||
|
||||
|
||||
def assert_seed_tts_scored(result: dict[str, Any]) -> None:
|
||||
err = result.get("seed_tts_eval_setup_error")
|
||||
assert not err, f"Seed-TTS eval deps/setup failed: {err}"
|
||||
|
||||
@@ -11,9 +11,6 @@ H100 / MI325) because they launch the live Omni server inside the test.
|
||||
run **fails** if accuracy is strictly below **0.69** (``--min-daily-omni-accuracy`` / ``ACC_BENCH_MIN_DAILY_OMNI_ACCURACY``).
|
||||
2. **Seed-TTS** — ``seed-tts-eval``-style metrics when ``--seed-tts-wer-eval`` is used
|
||||
(WER / SIM / UTMOS keys from :func:`compute_seed_tts_wer_metrics`).
|
||||
3. **Video-MME** — opt-in via ``--run-videomme`` (skipped by default so Daily-Omni / Seed-TTS callers
|
||||
stay unchanged). MiniCPM-o 4.5 reports **70.4** on Video-MME (w/o subs); the default floor is
|
||||
**0.68** (``--min-videomme-accuracy`` / ``ACC_BENCH_MIN_VIDEOMME_ACCURACY``).
|
||||
|
||||
Prerequisites
|
||||
-------------
|
||||
@@ -24,8 +21,7 @@ Prerequisites
|
||||
On L4 you may need a smaller checkpoint, quantization, or tighter engine flags; this script
|
||||
only drives the **client** benchmark.
|
||||
|
||||
* ``vllm`` CLI from **vLLM-Omni** (so ``bench serve`` registers ``daily-omni`` / ``seed-tts`` /
|
||||
``videomme``).
|
||||
* ``vllm`` CLI from **vLLM-Omni** (so ``bench serve`` registers ``daily-omni`` / ``seed-tts``).
|
||||
|
||||
* **Daily-Omni** — if local ``qa.json`` + ``Videos/`` are not both provided (CLI or matching env),
|
||||
the client passes ``--dataset-path`` with a Hub id (default ``liarliar/Daily-Omni``). The **child**
|
||||
@@ -35,13 +31,6 @@ Prerequisites
|
||||
the dataset repo with ``--daily-omni-repo`` or ``VLLM_DAILY_OMNI_REPO``; override the tar repo
|
||||
with ``VLLM_DAILY_OMNI_MEDIA_REPO`` if needed.
|
||||
|
||||
* **Video-MME** — by default ``--dataset-path`` is the Hub id ``lmms-eval/Video-MME`` (override with
|
||||
``--videomme-repo`` / ``VLLM_VIDEOMME_REPO``). The child bench downloads parquet + video archives via
|
||||
``huggingface_hub.snapshot_download`` (needs ``huggingface_hub``, network or HF cache). Pass an
|
||||
existing local root with ``--videomme-dataset-path`` / ``VLLM_VIDEOMME_DATASET_PATH`` /
|
||||
``VIDEOMME_ROOT`` to skip the Hub download. For CI servers without ``--allowed-local-media-path``,
|
||||
pass ``--videomme-inline-local-video`` (or rely on the pytest fixture that appends it).
|
||||
|
||||
* **Seed-TTS** optional extras for WER/SIM/UTMOS::
|
||||
|
||||
pip install 'vllm-omni[dev]'
|
||||
@@ -66,18 +55,6 @@ Skip one suite, tighten gates::
|
||||
--skip-daily-omni \\
|
||||
--max-seed-tts-mean-wer 0.35 \\
|
||||
--min-seed-tts-mean-sim 0.75
|
||||
|
||||
MiniCPM-o Video-MME (w/o subs)::
|
||||
|
||||
python tests/e2e/accuracy/qwen3_omni/run_qwen_omni_acc_benchmark.py \\
|
||||
--skip-daily-omni --skip-seed-tts --run-videomme \\
|
||||
--videomme-pack-mode minicpm-frames \\
|
||||
--videomme-max-frames 96 \\
|
||||
--temperature 0 --output-len 128 \\
|
||||
--min-videomme-accuracy 0.68
|
||||
|
||||
# Optional local mirror (absolute path) instead of Hub download:
|
||||
# --videomme-dataset-path /path/to/Video-MME
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -98,7 +75,6 @@ from tests.e2e.accuracy.qwen3_omni.qwen3_omni_acc_bench_core import (
|
||||
load_benchmark_result,
|
||||
run_vllm_bench_subprocess,
|
||||
seed_tts_bench_argv,
|
||||
videomme_bench_argv,
|
||||
)
|
||||
|
||||
_REPO_ROOT = Path(__file__).resolve().parents[4]
|
||||
@@ -126,20 +102,6 @@ def _validate_daily_omni(result: dict[str, Any], *, min_accuracy: float | None)
|
||||
return errs
|
||||
|
||||
|
||||
def _validate_videomme(result: dict[str, Any], *, min_accuracy: float | None) -> list[str]:
|
||||
errs: list[str] = []
|
||||
acc = result.get("videomme_accuracy")
|
||||
if acc is None:
|
||||
errs.append("Missing videomme_accuracy (wrong dataset or no gold-evaluated rows).")
|
||||
return errs
|
||||
ev = int(result.get("videomme_evaluated_ok", 0) or 0)
|
||||
if ev <= 0:
|
||||
errs.append("videomme_evaluated_ok is 0; no successful MCQ rows to score.")
|
||||
if min_accuracy is not None and float(acc) + 1e-12 < float(min_accuracy):
|
||||
errs.append(f"videomme_accuracy={acc:.6f} < --min-videomme-accuracy={min_accuracy}")
|
||||
return errs
|
||||
|
||||
|
||||
def _validate_seed_tts(
|
||||
result: dict[str, Any],
|
||||
*,
|
||||
@@ -168,7 +130,7 @@ def _validate_seed_tts(
|
||||
|
||||
|
||||
def sync_dataset_env_from_ns(ns: argparse.Namespace) -> None:
|
||||
"""Mirror CLI path flags into env vars read by dataset ``*_bench_argv`` helpers."""
|
||||
"""Mirror CLI path flags into env vars read by ``daily_omni_bench_argv`` / ``seed_tts_bench_argv``."""
|
||||
repo = getattr(ns, "daily_omni_repo", None)
|
||||
if repo is not None and str(repo).strip():
|
||||
os.environ["VLLM_DAILY_OMNI_REPO"] = str(repo).strip()
|
||||
@@ -176,14 +138,6 @@ def sync_dataset_env_from_ns(ns: argparse.Namespace) -> None:
|
||||
os.environ["VLLM_DAILY_OMNI_QA_JSON"] = str(Path(ns.daily_omni_qa_json).expanduser().resolve())
|
||||
if ns.daily_omni_video_dir is not None:
|
||||
os.environ["VLLM_DAILY_OMNI_VIDEO_DIR"] = str(Path(ns.daily_omni_video_dir).expanduser().resolve())
|
||||
vm_repo = getattr(ns, "videomme_repo", None)
|
||||
if vm_repo is not None and str(vm_repo).strip():
|
||||
os.environ["VLLM_VIDEOMME_REPO"] = str(vm_repo).strip()
|
||||
vm_path = getattr(ns, "videomme_dataset_path", None)
|
||||
if vm_path is not None and str(vm_path).strip():
|
||||
raw = str(vm_path).strip()
|
||||
p = Path(raw).expanduser()
|
||||
os.environ["VLLM_VIDEOMME_DATASET_PATH"] = str(p.resolve()) if p.exists() and p.is_dir() else raw
|
||||
if ns.seed_tts_dataset_path is not None:
|
||||
# ``--seed-tts-dataset-path`` accepts either a local directory or a
|
||||
# Hugging Face repo id. Only resolve to an absolute filesystem path
|
||||
@@ -203,9 +157,6 @@ def _preserve_benchmark_dataset_env() -> Any:
|
||||
"VLLM_DAILY_OMNI_REPO",
|
||||
"VLLM_DAILY_OMNI_QA_JSON",
|
||||
"VLLM_DAILY_OMNI_VIDEO_DIR",
|
||||
"VLLM_VIDEOMME_REPO",
|
||||
"VLLM_VIDEOMME_DATASET_PATH",
|
||||
"VIDEOMME_ROOT",
|
||||
"VLLM_SEED_TTS_DATASET_PATH",
|
||||
"SEED_TTS_ROOT",
|
||||
)
|
||||
@@ -220,13 +171,7 @@ def _preserve_benchmark_dataset_env() -> Any:
|
||||
os.environ[key] = value
|
||||
|
||||
|
||||
def _build_common_args(
|
||||
ns: argparse.Namespace,
|
||||
*,
|
||||
result_filename: str,
|
||||
backend: str = "openai-chat-omni",
|
||||
endpoint: str = "/v1/chat/completions",
|
||||
) -> list[str]:
|
||||
def _build_common_args(ns: argparse.Namespace, *, result_filename: str) -> list[str]:
|
||||
return build_serve_common_argv(
|
||||
host=ns.host,
|
||||
port=ns.port,
|
||||
@@ -238,11 +183,6 @@ def _build_common_args(
|
||||
result_dir=ns.result_dir,
|
||||
result_filename=result_filename,
|
||||
ready_check_timeout_sec=ns.ready_check_timeout_sec,
|
||||
trust_remote_code=ns.trust_remote_code,
|
||||
backend=backend,
|
||||
endpoint=endpoint,
|
||||
temperature=getattr(ns, "temperature", None),
|
||||
output_len=getattr(ns, "output_len", None),
|
||||
)
|
||||
|
||||
|
||||
@@ -261,9 +201,6 @@ def run_daily_omni(ns: argparse.Namespace, vllm: str) -> Path:
|
||||
json.dumps(extra, ensure_ascii=False, separators=(",", ":")),
|
||||
]
|
||||
)
|
||||
pack_mode = getattr(ns, "daily_omni_pack_mode", None)
|
||||
if pack_mode:
|
||||
argv.extend(["--daily-omni-pack-mode", pack_mode])
|
||||
if ns.daily_omni_save_eval_items:
|
||||
argv.append("--daily-omni-save-eval-items")
|
||||
print("\n$", vllm, *argv, "\n", flush=True)
|
||||
@@ -274,58 +211,13 @@ def run_daily_omni(ns: argparse.Namespace, vllm: str) -> Path:
|
||||
return out
|
||||
|
||||
|
||||
def run_videomme(ns: argparse.Namespace, vllm: str) -> Path:
|
||||
ns.result_dir.mkdir(parents=True, exist_ok=True)
|
||||
tag = datetime.now().strftime("%Y%m%d-%H%M%S")
|
||||
result_filename = f"omni_acc_videomme_{tag}.json"
|
||||
extra = json.loads(ns.videomme_extra_body_json)
|
||||
argv = (
|
||||
_build_common_args(ns, result_filename=result_filename)
|
||||
+ videomme_bench_argv()
|
||||
+ [
|
||||
"--disable-shuffle",
|
||||
"--videomme-pack-mode",
|
||||
ns.videomme_pack_mode,
|
||||
"--videomme-duration",
|
||||
ns.videomme_duration,
|
||||
"--extra-body",
|
||||
json.dumps(extra, ensure_ascii=False, separators=(",", ":")),
|
||||
]
|
||||
)
|
||||
if ns.videomme_max_frames is not None:
|
||||
argv.extend(["--videomme-max-frames", str(int(ns.videomme_max_frames))])
|
||||
if ns.videomme_use_subtitle:
|
||||
argv.append("--videomme-use-subtitle")
|
||||
if ns.videomme_inline_local_video:
|
||||
argv.append("--videomme-inline-local-video")
|
||||
if ns.videomme_save_eval_items:
|
||||
argv.append("--videomme-save-eval-items")
|
||||
if ns.videomme_video_dir is not None:
|
||||
argv.extend(["--videomme-video-dir", str(Path(ns.videomme_video_dir).expanduser().resolve())])
|
||||
if ns.videomme_parquet is not None:
|
||||
argv.extend(["--videomme-parquet", str(Path(ns.videomme_parquet).expanduser().resolve())])
|
||||
if ns.videomme_subtitle_dir is not None:
|
||||
argv.extend(["--videomme-subtitle-dir", str(Path(ns.videomme_subtitle_dir).expanduser().resolve())])
|
||||
print("\n$", vllm, *argv, "\n", flush=True)
|
||||
run_vllm_bench_subprocess(vllm, argv)
|
||||
out = Path(ns.result_dir) / result_filename
|
||||
if not out.is_file():
|
||||
raise FileNotFoundError(f"Expected result JSON at {out}")
|
||||
return out
|
||||
|
||||
|
||||
def run_seed_tts(ns: argparse.Namespace, vllm: str) -> Path:
|
||||
ns.result_dir.mkdir(parents=True, exist_ok=True)
|
||||
tag = datetime.now().strftime("%Y%m%d-%H%M%S")
|
||||
result_filename = f"qwen_omni_acc_seed_tts_{tag}.json"
|
||||
extra = json.loads(ns.seed_extra_body_json)
|
||||
argv = (
|
||||
_build_common_args(
|
||||
ns,
|
||||
result_filename=result_filename,
|
||||
backend=ns.seed_backend,
|
||||
endpoint=ns.seed_endpoint,
|
||||
)
|
||||
_build_common_args(ns, result_filename=result_filename)
|
||||
+ seed_tts_bench_argv(locale=ns.seed_tts_locale)
|
||||
+ [
|
||||
"--seed-tts-wer-eval",
|
||||
@@ -333,8 +225,6 @@ def run_seed_tts(ns: argparse.Namespace, vllm: str) -> Path:
|
||||
json.dumps(extra, ensure_ascii=False, separators=(",", ":")),
|
||||
]
|
||||
)
|
||||
if ns.seed_tts_turns_per_session > 1:
|
||||
argv.extend(["--seed-tts-turns-per-session", str(ns.seed_tts_turns_per_session)])
|
||||
if ns.seed_tts_wer_save_items:
|
||||
argv.append("--seed-tts-wer-save-items")
|
||||
if ns.seed_tts_file_ref_audio:
|
||||
@@ -376,23 +266,6 @@ def build_arg_parser() -> argparse.ArgumentParser:
|
||||
help="If set, forwarded to ``vllm bench serve`` (probe first request until success). "
|
||||
"Omit to use upstream default (typically skip).",
|
||||
)
|
||||
p.add_argument(
|
||||
"--trust-remote-code",
|
||||
action="store_true",
|
||||
help="Forward trust_remote_code to ``vllm bench serve`` for models with custom code.",
|
||||
)
|
||||
p.add_argument(
|
||||
"--temperature",
|
||||
type=float,
|
||||
default=None,
|
||||
help="Optional sampling temperature forwarded to ``vllm bench serve`` (e.g. 0 for MiniCPM-o Daily-Omni MCQ).",
|
||||
)
|
||||
p.add_argument(
|
||||
"--output-len",
|
||||
type=int,
|
||||
default=None,
|
||||
help="Optional max output tokens forwarded to ``vllm bench serve`` (e.g. 512 for MiniCPM-o Daily-Omni).",
|
||||
)
|
||||
p.add_argument(
|
||||
"--result-dir",
|
||||
type=Path,
|
||||
@@ -401,20 +274,6 @@ def build_arg_parser() -> argparse.ArgumentParser:
|
||||
|
||||
p.add_argument("--skip-daily-omni", action="store_true")
|
||||
p.add_argument("--skip-seed-tts", action="store_true")
|
||||
# Video-MME is opt-in: default skip=True so existing Daily-Omni / Seed-TTS callers stay unchanged.
|
||||
p.set_defaults(skip_videomme=True)
|
||||
p.add_argument(
|
||||
"--skip-videomme",
|
||||
action="store_true",
|
||||
dest="skip_videomme",
|
||||
help="Skip Video-MME (default).",
|
||||
)
|
||||
p.add_argument(
|
||||
"--run-videomme",
|
||||
action="store_false",
|
||||
dest="skip_videomme",
|
||||
help="Enable the Video-MME accuracy suite.",
|
||||
)
|
||||
|
||||
p.add_argument(
|
||||
"--daily-omni-repo",
|
||||
@@ -436,13 +295,6 @@ def build_arg_parser() -> argparse.ArgumentParser:
|
||||
help="Optional local Videos root; if omitted, media is fetched lazily from Hub Videos.tar.",
|
||||
)
|
||||
p.add_argument("--daily-omni-input-mode", choices=("all", "visual", "audio"), default="all")
|
||||
p.add_argument(
|
||||
"--daily-omni-pack-mode",
|
||||
choices=("qwen", "minicpm-interleave"),
|
||||
default=None,
|
||||
help="Daily-Omni multimodal pack recipe. Use ``minicpm-interleave`` for MiniCPM-o "
|
||||
"(1fps interleaved image/audio; see PR #5606).",
|
||||
)
|
||||
p.add_argument(
|
||||
"--daily-extra-body-json",
|
||||
default='{"modalities":["text"]}',
|
||||
@@ -461,87 +313,6 @@ def build_arg_parser() -> argparse.ArgumentParser:
|
||||
"Default baseline 0.69; override with env ACC_BENCH_MIN_DAILY_OMNI_ACCURACY or pass 0 to disable the floor.",
|
||||
)
|
||||
|
||||
p.add_argument(
|
||||
"--videomme-repo",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Hugging Face dataset id for Video-MME Hub mode (sets VLLM_VIDEOMME_REPO). "
|
||||
"Default when no local path is set: lmms-eval/Video-MME. "
|
||||
"Ignored when a local --videomme-dataset-path / VLLM_VIDEOMME_DATASET_PATH is used.",
|
||||
)
|
||||
p.add_argument(
|
||||
"--videomme-dataset-path",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Optional local Video-MME root (parquet + videos). Existing directories skip the Hub "
|
||||
"download; Hub ids are also accepted and passed through as --dataset-path. "
|
||||
"Sets VLLM_VIDEOMME_DATASET_PATH.",
|
||||
)
|
||||
p.add_argument(
|
||||
"--videomme-parquet",
|
||||
type=Path,
|
||||
default=None,
|
||||
help="Optional explicit path to videomme/test-*.parquet.",
|
||||
)
|
||||
p.add_argument(
|
||||
"--videomme-video-dir",
|
||||
type=Path,
|
||||
default=None,
|
||||
help="Optional explicit directory of extracted Video-MME videos.",
|
||||
)
|
||||
p.add_argument(
|
||||
"--videomme-subtitle-dir",
|
||||
type=Path,
|
||||
default=None,
|
||||
help="Optional subtitle directory (used with --videomme-use-subtitle).",
|
||||
)
|
||||
p.add_argument(
|
||||
"--videomme-pack-mode",
|
||||
choices=("minicpm-frames", "minicpm-interleave", "video_url"),
|
||||
default="minicpm-frames",
|
||||
help="Video-MME multimodal pack recipe (OmniEvalKit MiniCPM default: minicpm-frames).",
|
||||
)
|
||||
p.add_argument(
|
||||
"--videomme-max-frames",
|
||||
type=int,
|
||||
default=None,
|
||||
help="Override max sampled frames (OmniEvalKit: 96 for minicpm-frames, 64 for interleave).",
|
||||
)
|
||||
p.add_argument(
|
||||
"--videomme-duration",
|
||||
choices=("all", "short", "medium", "long"),
|
||||
default="all",
|
||||
help="Filter Video-MME by duration bucket (default: all).",
|
||||
)
|
||||
p.add_argument(
|
||||
"--videomme-use-subtitle",
|
||||
action="store_true",
|
||||
help="Prepend subtitle text to the user prompt (Video-MME w/ subs).",
|
||||
)
|
||||
p.add_argument(
|
||||
"--videomme-inline-local-video",
|
||||
action="store_true",
|
||||
help="Embed frames/audio as base64 data URLs (no --allowed-local-media-path on the server).",
|
||||
)
|
||||
p.add_argument(
|
||||
"--videomme-extra-body-json",
|
||||
default='{"modalities":["text"],"chat_template_kwargs":{"enable_thinking":false}}',
|
||||
help="JSON merged into each chat request for Video-MME.",
|
||||
)
|
||||
p.add_argument(
|
||||
"--videomme-save-eval-items",
|
||||
action="store_true",
|
||||
help="Include per-request Video-MME accuracy rows in the saved JSON.",
|
||||
)
|
||||
p.add_argument(
|
||||
"--min-videomme-accuracy",
|
||||
type=float,
|
||||
default=float((os.environ.get("ACC_BENCH_MIN_VIDEOMME_ACCURACY") or "0.68").strip() or "0.68"),
|
||||
help="Fail when videomme_accuracy is strictly below this threshold (0–1). "
|
||||
"Default 0.68 (~2pp under MiniCPM-o 4.5 reported 70.4 w/o subs); "
|
||||
"override with ACC_BENCH_MIN_VIDEOMME_ACCURACY or pass 0 to disable the floor.",
|
||||
)
|
||||
|
||||
p.add_argument(
|
||||
"--seed-tts-dataset-path",
|
||||
type=str,
|
||||
@@ -555,22 +326,6 @@ def build_arg_parser() -> argparse.ArgumentParser:
|
||||
default='{"modalities":["text","audio"]}',
|
||||
help="JSON for Seed-TTS chat requests (must include audio for synthesis + PCM capture).",
|
||||
)
|
||||
p.add_argument(
|
||||
"--seed-backend",
|
||||
default="openai-chat-omni",
|
||||
help="Seed-TTS benchmark backend (for example openai-realtime-tts).",
|
||||
)
|
||||
p.add_argument(
|
||||
"--seed-endpoint",
|
||||
default="/v1/chat/completions",
|
||||
help="Seed-TTS serving endpoint corresponding to --seed-backend.",
|
||||
)
|
||||
p.add_argument(
|
||||
"--seed-tts-turns-per-session",
|
||||
type=int,
|
||||
default=1,
|
||||
help="Group this many Seed-TTS target texts into each Realtime session.",
|
||||
)
|
||||
p.add_argument("--seed-tts-wer-save-items", action="store_true")
|
||||
p.add_argument(
|
||||
"--seed-tts-file-ref-audio",
|
||||
@@ -611,7 +366,7 @@ def parse_acc_benchmark_args(argv: list[str] | None = None) -> argparse.Namespac
|
||||
|
||||
|
||||
def run_acc_benchmark(ns: argparse.Namespace) -> int:
|
||||
"""Run configured client benches against a running server; return 0 on success."""
|
||||
"""Run Daily-Omni and/or Seed-TTS client benches against a running server; return 0 on success."""
|
||||
failed: list[str] = []
|
||||
|
||||
with _preserve_benchmark_dataset_env():
|
||||
@@ -635,21 +390,6 @@ def run_acc_benchmark(ns: argparse.Namespace) -> int:
|
||||
flush=True,
|
||||
)
|
||||
|
||||
if not ns.skip_videomme:
|
||||
path = run_videomme(ns, vllm)
|
||||
print(f"\n[Video-MME] result JSON: {path}", flush=True)
|
||||
data = load_benchmark_result(path)
|
||||
errs = _validate_videomme(data, min_accuracy=ns.min_videomme_accuracy)
|
||||
if errs:
|
||||
failed.extend([f"[Video-MME] {e}" for e in errs])
|
||||
else:
|
||||
print(
|
||||
f"[Video-MME] videomme_accuracy={data.get('videomme_accuracy')} "
|
||||
f"evaluated_ok={data.get('videomme_evaluated_ok')} "
|
||||
f"per_duration={data.get('videomme_per_duration_accuracy')}",
|
||||
flush=True,
|
||||
)
|
||||
|
||||
if not ns.skip_seed_tts:
|
||||
path = run_seed_tts(ns, vllm)
|
||||
print(f"\n[Seed-TTS] result JSON: {path}", flush=True)
|
||||
@@ -660,11 +400,6 @@ def run_acc_benchmark(ns: argparse.Namespace) -> int:
|
||||
min_mean_sim=ns.min_seed_tts_mean_sim,
|
||||
min_mean_utmos=ns.min_seed_tts_mean_utmos,
|
||||
)
|
||||
if ns.seed_tts_turns_per_session > 1:
|
||||
expected_turns = ns.num_prompts * ns.seed_tts_turns_per_session
|
||||
evaluated_turns = int(data.get("seed_tts_content_evaluated") or 0)
|
||||
if evaluated_turns != expected_turns:
|
||||
errs.append(f"seed_tts_content_evaluated={evaluated_turns} != expected {expected_turns}")
|
||||
if errs:
|
||||
failed.extend([f"[Seed-TTS] {e}" for e in errs])
|
||||
else:
|
||||
|
||||
@@ -14,7 +14,7 @@ from tests.helpers.stage_config import get_deploy_config_path
|
||||
|
||||
models = ["openbmb/MiniCPM-o-4_5"]
|
||||
|
||||
_CI_DEPLOY = get_deploy_config_path("minicpmo_4_5.yaml")
|
||||
_CI_DEPLOY = get_deploy_config_path("minicpmo_4_5_batching.yaml")
|
||||
|
||||
|
||||
test_params = [(model, None, {"deploy_config": _CI_DEPLOY, "trust_remote_code": True}) for model in models]
|
||||
@@ -31,10 +31,11 @@ def get_question(prompt_type: str = "text") -> str:
|
||||
return prompts.get(prompt_type, prompts["text"])
|
||||
|
||||
|
||||
@pytest.mark.skip(reason="https://github.com/vllm-project/vllm-omni/issues/5437")
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1)
|
||||
@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=2)
|
||||
@pytest.mark.parametrize("omni_runner", test_params, indirect=True)
|
||||
def test_text_to_text(omni_runner, omni_runner_handler) -> None:
|
||||
"""Test processing text, generating text output."""
|
||||
@@ -44,7 +45,7 @@ def test_text_to_text(omni_runner, omni_runner_handler) -> None:
|
||||
|
||||
@pytest.mark.full_model
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1)
|
||||
@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=2)
|
||||
@pytest.mark.parametrize("omni_runner", test_params, indirect=True)
|
||||
def test_audio_to_text(omni_runner, omni_runner_handler) -> None:
|
||||
"""Test processing audio, generating text output."""
|
||||
@@ -57,7 +58,7 @@ def test_audio_to_text(omni_runner, omni_runner_handler) -> None:
|
||||
|
||||
@pytest.mark.full_model
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1)
|
||||
@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=2)
|
||||
@pytest.mark.parametrize("omni_runner", test_params, indirect=True)
|
||||
def test_image_to_text(omni_runner, omni_runner_handler) -> None:
|
||||
"""Test processing image, generating text output."""
|
||||
@@ -68,7 +69,7 @@ def test_image_to_text(omni_runner, omni_runner_handler) -> None:
|
||||
|
||||
@pytest.mark.full_model
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1)
|
||||
@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=2)
|
||||
@pytest.mark.parametrize("omni_runner", test_params, indirect=True)
|
||||
def test_video_to_text(omni_runner, omni_runner_handler) -> None:
|
||||
"""Test processing video, generating text output."""
|
||||
@@ -79,18 +80,19 @@ def test_video_to_text(omni_runner, omni_runner_handler) -> None:
|
||||
|
||||
@pytest.mark.full_model
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1)
|
||||
@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=2)
|
||||
@pytest.mark.parametrize("omni_runner", test_params, indirect=True)
|
||||
def test_text_to_audio(omni_runner, omni_runner_handler) -> None:
|
||||
"""Test processing text and generating audio through Talker and Code2Wav."""
|
||||
"""Test processing text, generating audio output through the talker token2wav path."""
|
||||
request_config = {"prompts": get_question("text"), "modalities": ["audio"]}
|
||||
omni_runner_handler.send_omni_request(request_config)
|
||||
|
||||
|
||||
@pytest.mark.skip(reason="https://github.com/vllm-project/vllm-omni/issues/5437")
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1)
|
||||
@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=2)
|
||||
@pytest.mark.parametrize("omni_runner", test_params, indirect=True)
|
||||
def test_mix_to_audio(omni_runner, omni_runner_handler) -> None:
|
||||
"""Test processing mixed modalities (image + audio), generating audio output."""
|
||||
@@ -109,7 +111,7 @@ def test_mix_to_audio(omni_runner, omni_runner_handler) -> None:
|
||||
|
||||
@pytest.mark.full_model
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1)
|
||||
@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=2)
|
||||
@pytest.mark.parametrize("omni_runner", test_params, indirect=True)
|
||||
def test_video_to_audio(omni_runner, omni_runner_handler) -> None:
|
||||
"""Test processing video, generating audio output."""
|
||||
|
||||
@@ -58,7 +58,7 @@ def _compose_request(model_name: str, text: str, voice: str) -> dict:
|
||||
|
||||
|
||||
@pytest.mark.parametrize("omni_runner_function", [_OMNI_RUNNER_PARAM], indirect=True)
|
||||
@hardware_test(res={"cuda": "L4"}, num_cards=1)
|
||||
@hardware_test(res={"cuda": "L4", "xpu": "B60"}, num_cards=1)
|
||||
def test_voxtral_tts_offline_basic(omni_runner_function: OmniRunner) -> None:
|
||||
"""Offline sync path; function-scoped runner so AsyncOmni test does not overlap a live Omni."""
|
||||
omni = omni_runner_function.omni
|
||||
|
||||
@@ -53,7 +53,6 @@ from vllm_omni.experimental.fullduplex.client import ( # noqa: E402
|
||||
RealtimeEventCollector,
|
||||
build_realtime_url,
|
||||
read_pcm16_wav,
|
||||
summarize_session_request_metrics,
|
||||
)
|
||||
|
||||
_url_with_model = build_realtime_url
|
||||
@@ -195,38 +194,6 @@ class DemoState:
|
||||
summaries[response_id] = timing
|
||||
return summaries
|
||||
|
||||
def session_request_metrics(
|
||||
self,
|
||||
*,
|
||||
session_id: str | None = None,
|
||||
) -> list[dict[str, object]]:
|
||||
"""Return benchmark-style metrics for every response in this session."""
|
||||
requests: list[dict[str, object]] = []
|
||||
for request_index, (response_id, timing) in enumerate(self.response_timing_summaries().items()):
|
||||
metrics = timing.get("request_metrics")
|
||||
if not isinstance(metrics, dict):
|
||||
continue
|
||||
requests.append(
|
||||
{
|
||||
"session_id": session_id,
|
||||
"request_index": request_index,
|
||||
"response_id": response_id,
|
||||
**metrics,
|
||||
}
|
||||
)
|
||||
return requests
|
||||
|
||||
def session_metric_summary(
|
||||
self,
|
||||
*,
|
||||
session_id: str | None = None,
|
||||
) -> dict[str, object]:
|
||||
"""Average TTFT, TTFP, and RTF across responses that emitted audio."""
|
||||
return summarize_session_request_metrics(
|
||||
self.session_request_metrics(session_id=session_id),
|
||||
session_id=session_id,
|
||||
)
|
||||
|
||||
def first_index(self, event_type: str, predicate=None) -> int | None:
|
||||
for index, event in enumerate(self.events):
|
||||
if event.get("type") != event_type:
|
||||
@@ -963,23 +930,12 @@ async def _send_clean_turn(
|
||||
if outcome == "listen":
|
||||
return None, "listen"
|
||||
else:
|
||||
try:
|
||||
await _wait_for(
|
||||
state,
|
||||
lambda: state.count("response.created") > before_created,
|
||||
timeout_s=timeout_s,
|
||||
label=f"{transcript} response.created",
|
||||
)
|
||||
except TimeoutError:
|
||||
# ``auto_response`` sessions only get ``response.created`` once the model
|
||||
# decides to speak, so report the listen decisions that took its place.
|
||||
raise TimeoutError(
|
||||
f"Timed out waiting for {transcript} response.created after "
|
||||
f"{state.count('input_audio_buffer.committed')} committed input turns; "
|
||||
f"model_listen={state.model_listen_count - before_model_listen} "
|
||||
f"buffering_listen={state.buffering_listen_count} "
|
||||
f"errors={state.count('error')}"
|
||||
) from None
|
||||
await _wait_for(
|
||||
state,
|
||||
lambda: state.count("response.created") > before_created,
|
||||
timeout_s=timeout_s,
|
||||
label=f"{transcript} response.created",
|
||||
)
|
||||
await _wait_for(
|
||||
state,
|
||||
lambda: eligible_response_id() is not None,
|
||||
@@ -1218,13 +1174,10 @@ async def _send_listen_only_overlap_pair(
|
||||
before_second_listen = state.model_listen_count
|
||||
overlap_started_while_active = not state.response_done(first_response_id)
|
||||
model_unit_ready_while_active = False
|
||||
second_unit_event_index: int | None = None
|
||||
|
||||
def record_model_unit_ready() -> None:
|
||||
nonlocal model_unit_ready_while_active, second_unit_event_index
|
||||
nonlocal model_unit_ready_while_active
|
||||
model_unit_ready_while_active = not state.response_done(first_response_id)
|
||||
if model_unit_ready_while_active:
|
||||
second_unit_event_index = len(state.events)
|
||||
|
||||
await _send_pcm16(
|
||||
ws,
|
||||
@@ -1286,13 +1239,14 @@ async def _send_listen_only_overlap_pair(
|
||||
label=transcripts[1],
|
||||
)
|
||||
|
||||
second_speech_index = _nth_event_index(state, "input_audio_buffer.speech_started", 2)
|
||||
first_done_index = _event_index_for_response(state, "response.done", first_response_id)
|
||||
overlap_ok = (
|
||||
overlap_started_while_active
|
||||
and model_unit_ready_while_active
|
||||
and second_unit_event_index is not None
|
||||
and second_speech_index is not None
|
||||
and first_done_index is not None
|
||||
and second_unit_event_index < first_done_index
|
||||
and second_speech_index < first_done_index
|
||||
)
|
||||
return [first_response_id, second_response_id], ["speak", second_outcome], overlap_ok
|
||||
|
||||
@@ -1426,17 +1380,6 @@ async def run_demo(args: argparse.Namespace) -> dict[str, object]:
|
||||
await ws.send(json.dumps({"type": "session.close"}))
|
||||
await _wait_for(state, lambda: state.count("session.closed") > 0, timeout_s=20, label="session.closed")
|
||||
finally:
|
||||
if state.count("session.created") > 0 and state.count("session.closed") == 0:
|
||||
try:
|
||||
await ws.send(json.dumps({"type": "session.close"}))
|
||||
await _wait_for(
|
||||
state,
|
||||
lambda: state.count("session.closed") > 0,
|
||||
timeout_s=min(args.timeout_s, 5),
|
||||
label="session.closed during cleanup",
|
||||
)
|
||||
except (ConnectionClosed, TimeoutError):
|
||||
pass
|
||||
stop.set()
|
||||
reader.cancel()
|
||||
try:
|
||||
@@ -1567,8 +1510,6 @@ async def run_demo(args: argparse.Namespace) -> dict[str, object]:
|
||||
"input_transcription_ok": input_transcription_ok,
|
||||
"completed_response_ids": completed_response_ids,
|
||||
"response_timings": state.response_timing_summaries(),
|
||||
"request_metrics": state.session_request_metrics(session_id=args.session_id),
|
||||
"session_metrics": state.session_metric_summary(session_id=args.session_id),
|
||||
"lifecycle_counts_ok": lifecycle_counts_ok,
|
||||
"validation_mode": validation_mode,
|
||||
"scenario": scenario,
|
||||
|
||||
@@ -8,7 +8,6 @@ import hashlib
|
||||
import json
|
||||
import sys
|
||||
import uuid
|
||||
from functools import lru_cache
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
|
||||
@@ -20,25 +19,7 @@ SCRIPT_DIR = Path(__file__).resolve().parent
|
||||
if str(SCRIPT_DIR) not in sys.path:
|
||||
sys.path.insert(0, str(SCRIPT_DIR))
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def _scenario_module():
|
||||
"""Delay heavyweight vLLM imports so ``--help`` remains directly executable."""
|
||||
import minicpmo_realtime_duplex_scenarios
|
||||
|
||||
return minicpmo_realtime_duplex_scenarios
|
||||
|
||||
|
||||
def _ref_audio_data_url(path: str) -> str:
|
||||
return _scenario_module()._ref_audio_data_url(path)
|
||||
|
||||
|
||||
def _url_with_model(*args, **kwargs) -> str:
|
||||
return _scenario_module()._url_with_model(*args, **kwargs)
|
||||
|
||||
|
||||
async def run_demo(args):
|
||||
return await _scenario_module().run_demo(args)
|
||||
from minicpmo_realtime_duplex_scenarios import _url_with_model, run_demo # noqa: E402
|
||||
|
||||
|
||||
def _with_resume_mode(url: str) -> str:
|
||||
@@ -133,12 +114,7 @@ async def _open_admission_session(
|
||||
args: argparse.Namespace,
|
||||
session_id: str,
|
||||
) -> tuple[object, dict[str, object]]:
|
||||
url = _url_with_model(
|
||||
args.url,
|
||||
args.model,
|
||||
autostart=False if getattr(args, "ref_audio", None) else None,
|
||||
session_id=session_id,
|
||||
)
|
||||
url = _url_with_model(args.url, args.model, session_id=session_id)
|
||||
ws = await websockets.connect(url, max_size=64 * 1024 * 1024)
|
||||
await ws.send(
|
||||
json.dumps(
|
||||
@@ -149,7 +125,6 @@ async def _open_admission_session(
|
||||
"model": args.model,
|
||||
"modalities": ["audio", "text"],
|
||||
"extra_body": {"minicpmo45_native_duplex": True},
|
||||
**({"ref_audio": _ref_audio_data_url(args.ref_audio)} if getattr(args, "ref_audio", None) else {}),
|
||||
},
|
||||
}
|
||||
)
|
||||
@@ -176,12 +151,7 @@ async def _admission_probe(args: argparse.Namespace, *, limit: int) -> dict[str,
|
||||
accepted.append(await _open_admission_session(args, f"{prefix}-accepted-{index}"))
|
||||
|
||||
overflow_id = f"{prefix}-overflow"
|
||||
overflow_url = _url_with_model(
|
||||
args.url,
|
||||
args.model,
|
||||
autostart=False if getattr(args, "ref_audio", None) else None,
|
||||
session_id=overflow_id,
|
||||
)
|
||||
overflow_url = _url_with_model(args.url, args.model, session_id=overflow_id)
|
||||
async with websockets.connect(overflow_url, max_size=64 * 1024 * 1024) as overflow:
|
||||
await overflow.send(
|
||||
json.dumps(
|
||||
@@ -192,11 +162,6 @@ async def _admission_probe(args: argparse.Namespace, *, limit: int) -> dict[str,
|
||||
"model": args.model,
|
||||
"modalities": ["audio", "text"],
|
||||
"extra_body": {"minicpmo45_native_duplex": True},
|
||||
**(
|
||||
{"ref_audio": _ref_audio_data_url(args.ref_audio)}
|
||||
if getattr(args, "ref_audio", None)
|
||||
else {}
|
||||
),
|
||||
},
|
||||
}
|
||||
)
|
||||
@@ -242,27 +207,16 @@ async def _resume_probe(
|
||||
session_id: str,
|
||||
expect_expired: bool = False,
|
||||
) -> dict[str, object]:
|
||||
url = _url_with_model(
|
||||
args.url,
|
||||
args.model,
|
||||
autostart=False if getattr(args, "ref_audio", None) else None,
|
||||
session_id=session_id,
|
||||
)
|
||||
url = _url_with_model(args.url, args.model, session_id=session_id)
|
||||
async with websockets.connect(url, max_size=64 * 1024 * 1024) as first:
|
||||
await first.send(
|
||||
json.dumps(
|
||||
{
|
||||
"type": "session.update",
|
||||
"session": {
|
||||
"session_id": session_id,
|
||||
"model": args.model,
|
||||
"modalities": ["audio", "text"],
|
||||
"extra_body": {"minicpmo45_native_duplex": True},
|
||||
**(
|
||||
{"ref_audio": _ref_audio_data_url(args.ref_audio)}
|
||||
if getattr(args, "ref_audio", None)
|
||||
else {}
|
||||
),
|
||||
},
|
||||
}
|
||||
)
|
||||
@@ -352,12 +306,7 @@ async def _takeover_probe(
|
||||
*,
|
||||
session_id: str,
|
||||
) -> dict[str, object]:
|
||||
url = _url_with_model(
|
||||
args.url,
|
||||
args.model,
|
||||
autostart=False if getattr(args, "ref_audio", None) else None,
|
||||
session_id=session_id,
|
||||
)
|
||||
url = _url_with_model(args.url, args.model, session_id=session_id)
|
||||
first = await websockets.connect(url, max_size=64 * 1024 * 1024)
|
||||
second = None
|
||||
try:
|
||||
@@ -366,15 +315,9 @@ async def _takeover_probe(
|
||||
{
|
||||
"type": "session.update",
|
||||
"session": {
|
||||
"session_id": session_id,
|
||||
"model": args.model,
|
||||
"modalities": ["audio", "text"],
|
||||
"extra_body": {"minicpmo45_native_duplex": True},
|
||||
**(
|
||||
{"ref_audio": _ref_audio_data_url(args.ref_audio)}
|
||||
if getattr(args, "ref_audio", None)
|
||||
else {}
|
||||
),
|
||||
},
|
||||
}
|
||||
)
|
||||
@@ -508,7 +451,20 @@ async def run_multi_session(args: argparse.Namespace) -> dict[str, object]:
|
||||
args,
|
||||
session_id=f"takeover-{args.takeover_session_index}-{uuid.uuid4().hex}",
|
||||
)
|
||||
lifecycle_result = await run_lifecycle_probes(args)
|
||||
expiry_result = None
|
||||
if args.expire_session_index is not None:
|
||||
if not 0 <= args.expire_session_index < args.sessions:
|
||||
raise ValueError("--expire-session-index is outside the session range")
|
||||
expiry_result = await _resume_probe(
|
||||
args,
|
||||
session_id=f"expire-{args.expire_session_index}-{uuid.uuid4().hex}",
|
||||
expect_expired=True,
|
||||
)
|
||||
admission_result = (
|
||||
await _admission_probe(args, limit=args.verify_admission_limit)
|
||||
if args.verify_admission_limit is not None
|
||||
else None
|
||||
)
|
||||
|
||||
session_results = await asyncio.gather(
|
||||
*(run_demo(_demo_args(args, index)) for index in range(args.sessions)),
|
||||
@@ -531,15 +487,16 @@ async def run_multi_session(args: argparse.Namespace) -> dict[str, object]:
|
||||
and semantic_isolation_ok
|
||||
and (resume_result is None or resume_result.get("ok") is True)
|
||||
and (takeover_result is None or takeover_result.get("ok") is True)
|
||||
and lifecycle_result["ok"] is True
|
||||
and (expiry_result is None or expiry_result.get("ok") is True)
|
||||
and (admission_result is None or admission_result.get("ok") is True)
|
||||
),
|
||||
"session_count": args.sessions,
|
||||
"identity_isolation_ok": identity_isolation_ok,
|
||||
"semantic_isolation_ok": semantic_isolation_ok,
|
||||
"resume": resume_result,
|
||||
"takeover": takeover_result,
|
||||
"expiry": lifecycle_result["expiry"],
|
||||
"admission": lifecycle_result["admission"],
|
||||
"expiry": expiry_result,
|
||||
"admission": admission_result,
|
||||
"failures": failures,
|
||||
"sessions": completed,
|
||||
}
|
||||
@@ -550,34 +507,6 @@ async def run_multi_session(args: argparse.Namespace) -> dict[str, object]:
|
||||
return result
|
||||
|
||||
|
||||
async def run_lifecycle_probes(args: argparse.Namespace) -> dict[str, object]:
|
||||
"""Run expiry and admission probes without requiring model output."""
|
||||
if args.sessions < 1:
|
||||
raise ValueError("--sessions must be positive")
|
||||
expiry_result = None
|
||||
if args.expire_session_index is not None:
|
||||
if not 0 <= args.expire_session_index < args.sessions:
|
||||
raise ValueError("--expire-session-index is outside the session range")
|
||||
expiry_result = await _resume_probe(
|
||||
args,
|
||||
session_id=f"expire-{args.expire_session_index}-{uuid.uuid4().hex}",
|
||||
expect_expired=True,
|
||||
)
|
||||
admission_result = (
|
||||
await _admission_probe(args, limit=args.verify_admission_limit)
|
||||
if args.verify_admission_limit is not None
|
||||
else None
|
||||
)
|
||||
return {
|
||||
"ok": (
|
||||
(expiry_result is None or expiry_result.get("ok") is True)
|
||||
and (admission_result is None or admission_result.get("ok") is True)
|
||||
),
|
||||
"expiry": expiry_result,
|
||||
"admission": admission_result,
|
||||
}
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--url", default="ws://127.0.0.1:8113/v1/realtime?duplex=1")
|
||||
|
||||
@@ -2,8 +2,8 @@
|
||||
|
||||
This E2E driver runs the public ``realtime_duplex_demo.py`` against a live
|
||||
duplex backend. Arbitrary audio defaults to the model-policy lifecycle contract.
|
||||
The stronger response-required mode binds the multi-response contract to a
|
||||
known input checksum and expected follow-up response.
|
||||
The stronger response-required mode binds the two-response contract to a known
|
||||
input checksum and expected second response.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -39,17 +39,6 @@ def _validate_input_sha256(path: Path, expected: str) -> str:
|
||||
return actual
|
||||
|
||||
|
||||
def _input_duration_s(path: Path) -> float:
|
||||
with wave.open(str(path), "rb") as wav_file:
|
||||
return wav_file.getnframes() / wav_file.getframerate()
|
||||
|
||||
|
||||
def _client_process_timeout_s(input_wav: Path, protocol_timeout_s: float) -> float:
|
||||
# The child first streams the WAV in real time, then can independently
|
||||
# exhaust its post-commit and session-close protocol waits.
|
||||
return _input_duration_s(input_wav) + 2 * protocol_timeout_s + 30.0
|
||||
|
||||
|
||||
def _read_json(path: Path) -> dict[str, object]:
|
||||
if not path.is_file():
|
||||
return {}
|
||||
@@ -222,7 +211,7 @@ def summarize_artifacts(
|
||||
validation_mode: str,
|
||||
min_responses: int,
|
||||
min_audio_deltas_per_response: int,
|
||||
expect_followup_response_substring: str | None,
|
||||
expect_second_response_substring: str | None,
|
||||
) -> dict[str, object]:
|
||||
if validation_mode not in {"model-policy", "response-required"}:
|
||||
raise ValueError(f"unsupported validation mode: {validation_mode}")
|
||||
@@ -285,20 +274,12 @@ def summarize_artifacts(
|
||||
)
|
||||
final_listen_after_commit = commit_index is not None and any(index > commit_index for index in listen_indices)
|
||||
transcript = "".join(str(summary.get("transcript") or "") for summary in response_summaries)
|
||||
followup_response_transcripts = [
|
||||
str(summary.get("transcript") or "")
|
||||
for summary in response_summaries[1:]
|
||||
if commit_index is not None
|
||||
and isinstance(summary.get("created_index"), int)
|
||||
and summary["created_index"] < commit_index
|
||||
]
|
||||
followup_response_transcript_ok = any(
|
||||
bool(_normalize_text(transcript)) for transcript in followup_response_transcripts
|
||||
)
|
||||
followup_response_transcript_expectation_ok = not expect_followup_response_substring or any(
|
||||
_normalize_text(expect_followup_response_substring) in _normalize_text(transcript)
|
||||
for transcript in followup_response_transcripts
|
||||
second_response_transcript = (
|
||||
str(response_summaries[1].get("transcript") or "") if len(response_summaries) >= 2 else ""
|
||||
)
|
||||
second_response_transcript_expectation_ok = not expect_second_response_substring or _normalize_text(
|
||||
expect_second_response_substring
|
||||
) in _normalize_text(second_response_transcript)
|
||||
error_events = [
|
||||
event
|
||||
for event in events
|
||||
@@ -329,8 +310,9 @@ def summarize_artifacts(
|
||||
mode_contract_ok = validation_mode == "model-policy" or (
|
||||
second_response_before_final_commit
|
||||
and listen_before_first_response
|
||||
and listen_between_responses
|
||||
and listen_after_last_done
|
||||
and followup_response_transcript_ok
|
||||
and second_response_transcript_expectation_ok
|
||||
)
|
||||
ok = common_contract_ok and mode_contract_ok
|
||||
return {
|
||||
@@ -355,10 +337,9 @@ def summarize_artifacts(
|
||||
"listen_after_response_before_commit": listen_after_response_before_commit,
|
||||
"final_listen_after_commit": final_listen_after_commit,
|
||||
"transcript": transcript,
|
||||
"followup_response_transcripts": followup_response_transcripts,
|
||||
"followup_response_transcript_ok": followup_response_transcript_ok,
|
||||
"expect_followup_response_substring": expect_followup_response_substring,
|
||||
"followup_response_transcript_expectation_ok": followup_response_transcript_expectation_ok,
|
||||
"second_response_transcript": second_response_transcript,
|
||||
"expect_second_response_substring": expect_second_response_substring,
|
||||
"second_response_transcript_expectation_ok": second_response_transcript_expectation_ok,
|
||||
"error_count": len(error_events),
|
||||
"cancelled_count": cancelled_count,
|
||||
"compact_sequence": _compact_sequence(events),
|
||||
@@ -407,7 +388,7 @@ async def run_soft_interrupt(args: argparse.Namespace) -> dict[str, object]:
|
||||
try:
|
||||
stdout_bytes, stderr_bytes = await asyncio.wait_for(
|
||||
process.communicate(),
|
||||
timeout=_client_process_timeout_s(input_wav, args.timeout_s),
|
||||
timeout=args.timeout_s + 30.0,
|
||||
)
|
||||
except TimeoutError:
|
||||
process.kill()
|
||||
@@ -425,7 +406,7 @@ async def run_soft_interrupt(args: argparse.Namespace) -> dict[str, object]:
|
||||
validation_mode=args.validation_mode,
|
||||
min_responses=args.min_responses,
|
||||
min_audio_deltas_per_response=args.min_audio_deltas_per_response,
|
||||
expect_followup_response_substring=args.expect_followup_response_substring,
|
||||
expect_second_response_substring=args.expect_second_response_substring,
|
||||
)
|
||||
summary.update(
|
||||
{
|
||||
@@ -463,12 +444,14 @@ def parse_args() -> argparse.Namespace:
|
||||
parser.add_argument("--min-responses", type=int, default=2)
|
||||
parser.add_argument("--min-audio-deltas-per-response", type=int, default=2)
|
||||
parser.add_argument("--input-sha256")
|
||||
parser.add_argument("--expect-followup-response-substring")
|
||||
parser.add_argument("--expect-second-response-substring")
|
||||
args = parser.parse_args()
|
||||
if args.validation_mode == "response-required" and args.min_responses < 2:
|
||||
parser.error("--min-responses must be at least 2")
|
||||
if args.validation_mode == "response-required" and not args.input_sha256:
|
||||
parser.error("--input-sha256 is required in response-required mode")
|
||||
if args.validation_mode == "response-required" and not args.expect_second_response_substring:
|
||||
parser.error("--expect-second-response-substring is required in response-required mode")
|
||||
if args.min_audio_deltas_per_response < 1:
|
||||
parser.error("--min-audio-deltas-per-response must be positive")
|
||||
return args
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
"""E2E online tests for MiniCPM-o 4.5 multimodal input and audio/text output.
|
||||
"""
|
||||
E2E Online tests for MiniCPM-o 4.5 model with multimodal input and audio / text output.
|
||||
|
||||
Exercises async chunk streaming (``--async-chunk``) across separate Thinker,
|
||||
Talker, and Code2Wav stages.
|
||||
MiniCPM-o 4.5 has ``async_chunk: false``, ``max_num_seqs: 1`` on both stages,
|
||||
and the vocoder runs in-process inside the talker stage rather than as a separate
|
||||
Code2Wav stage.
|
||||
"""
|
||||
|
||||
import os
|
||||
@@ -11,28 +13,26 @@ import pytest
|
||||
from tests.helpers.mark import hardware_test
|
||||
from tests.helpers.media import generate_synthetic_audio, generate_synthetic_image, generate_synthetic_video
|
||||
from tests.helpers.runtime import OmniServerParams, dummy_messages_from_mix_data
|
||||
from tests.helpers.stage_config import get_deploy_config_path, modify_stage_config
|
||||
from tests.helpers.stage_config import get_deploy_config_path
|
||||
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
|
||||
_MODEL = "openbmb/MiniCPM-o-4_5"
|
||||
_CI_DEPLOY = modify_stage_config(
|
||||
get_deploy_config_path("minicpmo_4_5.yaml"),
|
||||
updates={
|
||||
"stages": {0: {"default_sampling_params.max_tokens": 64}, 1: {"default_sampling_params.max_tokens": 1024}}
|
||||
},
|
||||
)
|
||||
_CI_DEPLOY = get_deploy_config_path("minicpmo_4_5_batching.yaml")
|
||||
|
||||
test_params = [
|
||||
pytest.param(
|
||||
OmniServerParams(
|
||||
model=_MODEL,
|
||||
stage_config_path=_CI_DEPLOY,
|
||||
use_stage_cli=False,
|
||||
server_args=["--trust-remote-code", "--async-chunk"],
|
||||
use_stage_cli=True,
|
||||
server_args=[
|
||||
"--trust-remote-code",
|
||||
"--no-async-chunk",
|
||||
],
|
||||
),
|
||||
id="async_chunk",
|
||||
),
|
||||
id="default",
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
@@ -65,15 +65,16 @@ def get_max_batch_size(size_type="few"):
|
||||
return batch_sizes.get(size_type, 5)
|
||||
|
||||
|
||||
@pytest.mark.skip(reason="https://github.com/vllm-project/vllm-omni/issues/5437")
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1)
|
||||
@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=2)
|
||||
@pytest.mark.parametrize("omni_server", test_params, indirect=True)
|
||||
def test_text_to_text_001(omni_server, openai_client) -> None:
|
||||
"""
|
||||
Test text-only input generating text output via OpenAI API.
|
||||
Deploy Setting: default single GPU
|
||||
Deploy Setting: default 2GPU
|
||||
Input Modal: text
|
||||
Output Modal: text
|
||||
Input Setting: stream=False
|
||||
@@ -93,13 +94,13 @@ def test_text_to_text_001(omni_server, openai_client) -> None:
|
||||
|
||||
@pytest.mark.full_model
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1)
|
||||
@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=2)
|
||||
@pytest.mark.parametrize("omni_server", test_params, indirect=True)
|
||||
def test_text_to_audio_001(omni_server, openai_client) -> None:
|
||||
"""
|
||||
Test text-only input generating text + audio output via OpenAI API.
|
||||
This exercises Talker TTS region detection and the Code2Wav stage.
|
||||
Deploy Setting: default single GPU
|
||||
This exercises the talker TTS region detection and in-process token2wav vocoder.
|
||||
Deploy Setting: default 2GPU
|
||||
Input Modal: text
|
||||
Output Modal: text + audio
|
||||
Input Setting: stream=True
|
||||
@@ -118,12 +119,12 @@ def test_text_to_audio_001(omni_server, openai_client) -> None:
|
||||
|
||||
@pytest.mark.full_model
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1)
|
||||
@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=2)
|
||||
@pytest.mark.parametrize("omni_server", test_params, indirect=True)
|
||||
def test_audio_to_text_audio_001(omni_server, openai_client) -> None:
|
||||
"""
|
||||
Test audio input generating text + audio output via OpenAI API.
|
||||
Deploy Setting: default single GPU
|
||||
Deploy Setting: default 2GPU
|
||||
Input Modal: text + audio
|
||||
Output Modal: text + audio
|
||||
Input Setting: stream=True
|
||||
@@ -132,21 +133,13 @@ def test_audio_to_text_audio_001(omni_server, openai_client) -> None:
|
||||
messages = dummy_messages_from_mix_data(
|
||||
system_prompt=get_system_prompt(),
|
||||
audio_data_url=audio_data_url,
|
||||
content_text=get_prompt(),
|
||||
content_text=get_prompt("mix"),
|
||||
)
|
||||
|
||||
request_config = {
|
||||
"model": omni_server.model,
|
||||
"messages": messages,
|
||||
"stream": True,
|
||||
"key_words": {"text": ["Beijing"]},
|
||||
"modalities": ["text", "audio"],
|
||||
"extra_body": {
|
||||
"chat_template_kwargs": {
|
||||
"use_tts_template": True,
|
||||
"enable_thinking": False,
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
openai_client.send_omni_request(request_config, request_num=get_max_batch_size())
|
||||
@@ -154,12 +147,12 @@ def test_audio_to_text_audio_001(omni_server, openai_client) -> None:
|
||||
|
||||
@pytest.mark.full_model
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1)
|
||||
@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=2)
|
||||
@pytest.mark.parametrize("omni_server", test_params, indirect=True)
|
||||
def test_image_to_text_audio_001(omni_server, openai_client) -> None:
|
||||
"""
|
||||
Test image input generating text + audio output via OpenAI API.
|
||||
Deploy Setting: default single GPU
|
||||
Deploy Setting: default 2GPU
|
||||
Input Modal: text + image
|
||||
Output Modal: text + audio
|
||||
Input Setting: stream=True
|
||||
@@ -182,12 +175,12 @@ def test_image_to_text_audio_001(omni_server, openai_client) -> None:
|
||||
|
||||
@pytest.mark.full_model
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1)
|
||||
@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=2)
|
||||
@pytest.mark.parametrize("omni_server", test_params, indirect=True)
|
||||
def test_video_to_text_audio_001(omni_server, openai_client) -> None:
|
||||
"""
|
||||
Test video input generating text + audio output via OpenAI API.
|
||||
Deploy Setting: default single GPU
|
||||
Deploy Setting: default 2GPU
|
||||
Input Modal: text + video
|
||||
Output Modal: text + audio
|
||||
Input Setting: stream=True
|
||||
@@ -208,15 +201,16 @@ def test_video_to_text_audio_001(omni_server, openai_client) -> None:
|
||||
openai_client.send_omni_request(request_config, request_num=get_max_batch_size())
|
||||
|
||||
|
||||
@pytest.mark.skip(reason="https://github.com/vllm-project/vllm-omni/issues/5437")
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1)
|
||||
@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=2)
|
||||
@pytest.mark.parametrize("omni_server", test_params, indirect=True)
|
||||
def test_mix_to_text_audio_001(omni_server, openai_client) -> None:
|
||||
"""
|
||||
Test multi-modal input (text + audio + video + image) generating text + audio output.
|
||||
Deploy Setting: default single GPU
|
||||
Deploy Setting: default 2GPU
|
||||
Input Modal: text + audio + video + image
|
||||
Output Modal: text + audio
|
||||
Input Setting: stream=True
|
||||
|
||||
@@ -1,176 +0,0 @@
|
||||
"""CI coverage for the MiniCPM-o 4.5 native-duplex Realtime API."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
import websockets
|
||||
|
||||
from tests.e2e.online_serving.minicpmo_realtime_duplex_scenarios import (
|
||||
_ref_audio_data_url,
|
||||
run_demo,
|
||||
)
|
||||
from tests.e2e.online_serving.run_minicpmo_realtime_duplex_multi_session import (
|
||||
run_multi_session,
|
||||
)
|
||||
from tests.helpers.mark import hardware_test
|
||||
from tests.helpers.minicpmo_4_5_duplex import (
|
||||
SERVER_PARAMS,
|
||||
demo_args,
|
||||
multi_session_args,
|
||||
realtime_url,
|
||||
resolve_ref_audio,
|
||||
validated_input_wav,
|
||||
)
|
||||
from vllm_omni.experimental.fullduplex.client import build_realtime_url
|
||||
|
||||
pytestmark = pytest.mark.omni
|
||||
|
||||
|
||||
def _assert_request_metrics(metrics: object, *, expected_count: int) -> None:
|
||||
assert isinstance(metrics, list)
|
||||
assert len(metrics) == expected_count
|
||||
for request_index, request in enumerate(metrics):
|
||||
assert isinstance(request["session_id"], str)
|
||||
assert request["request_index"] == request_index
|
||||
assert isinstance(request["response_id"], str)
|
||||
assert request["ttft_ms"] is not None and request["ttft_ms"] >= 0
|
||||
assert request["ttfp_ms"] >= 0
|
||||
assert request["rtf"] is not None and request["rtf"] >= 0
|
||||
assert request["audio_generation_ms"] >= 0
|
||||
assert request["audio_duration_ms"] > 0
|
||||
|
||||
|
||||
def _assert_session_metrics(metrics: object, *, expected_count: int) -> None:
|
||||
assert isinstance(metrics, dict)
|
||||
assert isinstance(metrics["session_id"], str)
|
||||
assert metrics["audio_turn_count"] == expected_count
|
||||
assert metrics["mean_ttft_ms"] is not None and metrics["mean_ttft_ms"] >= 0
|
||||
assert metrics["mean_ttfp_ms"] is not None and metrics["mean_ttfp_ms"] >= 0
|
||||
assert metrics["mean_rtf"] is not None and metrics["mean_rtf"] >= 0
|
||||
|
||||
|
||||
async def _receive_protocol_events(ws, required_types: set[str], *, timeout_s: float) -> list[dict[str, object]]:
|
||||
async def receive() -> list[dict[str, object]]:
|
||||
events: list[dict[str, object]] = []
|
||||
seen: set[str] = set()
|
||||
while not required_types.issubset(seen):
|
||||
raw = await ws.recv()
|
||||
if not isinstance(raw, str):
|
||||
continue
|
||||
event = json.loads(raw)
|
||||
if not isinstance(event, dict):
|
||||
continue
|
||||
events.append(event)
|
||||
event_type = event.get("type")
|
||||
if event_type == "error":
|
||||
raise AssertionError(f"WebSocket protocol smoke received an error: {event}")
|
||||
if isinstance(event_type, str):
|
||||
seen.add(event_type)
|
||||
return events
|
||||
|
||||
return await asyncio.wait_for(receive(), timeout=timeout_s)
|
||||
|
||||
|
||||
async def _run_protocol_smoke(*, url: str, model: str, ref_audio: Path) -> list[dict[str, object]]:
|
||||
session_id = f"duplex-ci-protocol-{uuid.uuid4().hex}"
|
||||
websocket_url = build_realtime_url(url, model, autostart=False, session_id=session_id)
|
||||
async with websockets.connect(websocket_url, max_size=64 * 1024 * 1024) as ws:
|
||||
await ws.send(
|
||||
json.dumps(
|
||||
{
|
||||
"type": "session.update",
|
||||
"session": {
|
||||
"session_id": session_id,
|
||||
"model": model,
|
||||
"modalities": ["audio", "text"],
|
||||
"ref_audio": _ref_audio_data_url(str(ref_audio)),
|
||||
"extra_body": {"minicpmo45_native_duplex": True},
|
||||
},
|
||||
}
|
||||
)
|
||||
)
|
||||
events = await _receive_protocol_events(
|
||||
ws,
|
||||
{"session.created", "session.updated"},
|
||||
timeout_s=60,
|
||||
)
|
||||
await ws.send(json.dumps({"type": "session.close"}))
|
||||
events.extend(await _receive_protocol_events(ws, {"session.closed"}, timeout_s=60))
|
||||
return events
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1)
|
||||
@pytest.mark.parametrize("omni_server", SERVER_PARAMS, indirect=True)
|
||||
def test_duplex_websocket_protocol_smoke(omni_server, model_prefix: str) -> None:
|
||||
ref_audio = resolve_ref_audio(model_prefix)
|
||||
events = asyncio.run(
|
||||
_run_protocol_smoke(
|
||||
url=realtime_url(omni_server),
|
||||
model=omni_server.model,
|
||||
ref_audio=ref_audio,
|
||||
)
|
||||
)
|
||||
event_types = [event.get("type") for event in events]
|
||||
assert "session.created" in event_types
|
||||
assert "session.updated" in event_types
|
||||
assert event_types[-1] == "session.closed"
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1)
|
||||
@pytest.mark.parametrize("omni_server", SERVER_PARAMS, indirect=True)
|
||||
def test_duplex_single_session_response_required(omni_server, model_prefix: str, tmp_path: Path) -> None:
|
||||
args = demo_args(
|
||||
omni_server=omni_server,
|
||||
input_wav=validated_input_wav(),
|
||||
ref_audio=resolve_ref_audio(model_prefix),
|
||||
output_dir=tmp_path / "single_session",
|
||||
)
|
||||
args.turns = 2
|
||||
# Every turn replays the same active-speech window as the first one. The default
|
||||
# shorter follow-up window is a different mid-utterance slice, which the native
|
||||
# duplex model may legitimately answer with "listen" instead of a response.
|
||||
args.turn_duration_ms = [args.first_turn_ms] * args.turns
|
||||
result = asyncio.run(run_demo(args))
|
||||
assert result["ok"] is True
|
||||
assert result["audio_delta_count"] > 0
|
||||
assert result["done_count"] == 2
|
||||
assert result["error_count"] == 0
|
||||
assert result["all_audio_responses_have_transcript"] is True
|
||||
assert result["transcript_delta_done_ok"] is True
|
||||
_assert_request_metrics(result["request_metrics"], expected_count=2)
|
||||
_assert_session_metrics(result["session_metrics"], expected_count=2)
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1)
|
||||
@pytest.mark.parametrize("omni_server", SERVER_PARAMS, indirect=True)
|
||||
def test_duplex_two_sessions_resume_and_takeover(omni_server, model_prefix: str, tmp_path: Path) -> None:
|
||||
result = asyncio.run(
|
||||
run_multi_session(
|
||||
multi_session_args(
|
||||
omni_server=omni_server,
|
||||
input_wav=validated_input_wav(),
|
||||
ref_audio=resolve_ref_audio(model_prefix),
|
||||
output_dir=tmp_path / "multi_session",
|
||||
response_required=True,
|
||||
)
|
||||
)
|
||||
)
|
||||
assert result["ok"] is True
|
||||
assert result["session_count"] == 2
|
||||
assert result["resume"]["ok"] is True
|
||||
assert result["takeover"]["ok"] is True
|
||||
assert not result["failures"]
|
||||
assert all(session["audio_delta_count"] > 0 for session in result["sessions"])
|
||||
assert all(session["done_count"] == 1 for session in result["sessions"])
|
||||
assert all(session["error_count"] == 0 for session in result["sessions"])
|
||||
for session in result["sessions"]:
|
||||
_assert_request_metrics(session["request_metrics"], expected_count=1)
|
||||
_assert_session_metrics(session["session_metrics"], expected_count=1)
|
||||
@@ -1,90 +0,0 @@
|
||||
"""Nightly lifecycle coverage for the MiniCPM-o 4.5 native-duplex API."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.e2e.online_serving.run_minicpmo_realtime_duplex_multi_session import (
|
||||
run_lifecycle_probes,
|
||||
)
|
||||
from tests.e2e.online_serving.run_minicpmo_realtime_duplex_soft_interrupt import (
|
||||
run_soft_interrupt,
|
||||
)
|
||||
from tests.helpers.mark import hardware_test
|
||||
from tests.helpers.minicpmo_4_5_duplex import (
|
||||
SERVER_PARAMS,
|
||||
SOFT_INTERRUPT_SHA256,
|
||||
multi_session_args,
|
||||
realtime_url,
|
||||
resolve_ref_audio,
|
||||
validated_input_wav,
|
||||
validated_soft_interrupt_wav,
|
||||
)
|
||||
|
||||
pytestmark = [pytest.mark.full_model, pytest.mark.omni]
|
||||
|
||||
|
||||
@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1)
|
||||
@pytest.mark.parametrize("omni_server", SERVER_PARAMS, indirect=True)
|
||||
def test_duplex_admission_and_expiry_reaper(omni_server, model_prefix: str, tmp_path: Path) -> None:
|
||||
args = multi_session_args(
|
||||
omni_server=omni_server,
|
||||
input_wav=validated_input_wav(),
|
||||
ref_audio=resolve_ref_audio(model_prefix),
|
||||
output_dir=tmp_path / "admission_expiry",
|
||||
response_required=False,
|
||||
)
|
||||
args.sessions = 1
|
||||
args.disconnect_session_index = None
|
||||
args.takeover_session_index = None
|
||||
args.expire_session_index = 0
|
||||
args.verify_admission_limit = 2
|
||||
result = asyncio.run(run_lifecycle_probes(args))
|
||||
|
||||
assert result["ok"] is True, json.dumps(result, ensure_ascii=False, indent=2)
|
||||
assert result["expiry"]["ok"] is True
|
||||
assert result["expiry"]["error_code"] == "session_resume_expired"
|
||||
assert result["admission"]["ok"] is True
|
||||
assert result["admission"]["overflow_error_code"] == "resource_exhausted"
|
||||
|
||||
|
||||
# CUDA-only for now: this contract asserts the model speaks while the user is
|
||||
# still talking, which only holds when the duplex pipeline sustains real-time
|
||||
# throughput. The current NPU stack runs several times slower than real time,
|
||||
# so it never reaches a mid-stream decision point.
|
||||
@hardware_test(res={"cuda": "H100"}, num_cards=1)
|
||||
@pytest.mark.parametrize("omni_server", SERVER_PARAMS, indirect=True)
|
||||
def test_duplex_soft_interrupt(omni_server, model_prefix: str, tmp_path: Path) -> None:
|
||||
input_wav = validated_soft_interrupt_wav()
|
||||
result = asyncio.run(
|
||||
run_soft_interrupt(
|
||||
SimpleNamespace(
|
||||
url=realtime_url(omni_server),
|
||||
model=omni_server.model,
|
||||
input_wav=str(input_wav),
|
||||
ref_audio=str(resolve_ref_audio(model_prefix)),
|
||||
output_dir=str(tmp_path / "soft_interrupt"),
|
||||
summary_output=None,
|
||||
chunk_ms=200,
|
||||
timeout_s=180.0,
|
||||
require_audio=True,
|
||||
no_realtime_pacing=False,
|
||||
validation_mode="response-required",
|
||||
min_responses=2,
|
||||
min_audio_deltas_per_response=2,
|
||||
input_sha256=SOFT_INTERRUPT_SHA256,
|
||||
expect_followup_response_substring=None,
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
assert result["ok"] is True, json.dumps(result, ensure_ascii=False, indent=2)
|
||||
assert result["error_count"] == 0
|
||||
assert result["response_lifecycle_ok"] is True
|
||||
assert result["response_audio_contract_ok"] is True
|
||||
assert result["followup_response_transcript_ok"] is True
|
||||
@@ -1,10 +1,7 @@
|
||||
"""E2E online expansion tests for MiniCPM-o 4.5.
|
||||
|
||||
These cover modality combinations, separate Code2Wav behavior, sequential
|
||||
request isolation, longer audio output, and Chinese speech.
|
||||
|
||||
Like Qwen3-Omni expansion, ``default`` exercises ``--no-async-chunk`` and
|
||||
``async_chunk`` exercises ``--async-chunk``.
|
||||
"""
|
||||
E2E Online expansion tests for MiniCPM-o 4.5 covering modality combinations,
|
||||
in-process token2wav vocoder behavior, max_num_seqs=1 serial execution,
|
||||
and edge-case validation.
|
||||
"""
|
||||
|
||||
import os
|
||||
@@ -21,29 +18,20 @@ pytestmark = [pytest.mark.full_model, pytest.mark.omni]
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
|
||||
_MODEL = "openbmb/MiniCPM-o-4_5"
|
||||
_CI_DEPLOY = get_deploy_config_path("minicpmo_4_5.yaml")
|
||||
_CI_DEPLOY = get_deploy_config_path("minicpmo_4_5_batching.yaml")
|
||||
|
||||
# Deploy YAML defaults to ``async_chunk: true``; keep sync and async as
|
||||
# separate parametrizations so both handoff paths are exercised.
|
||||
test_params = [
|
||||
pytest.param(
|
||||
OmniServerParams(
|
||||
model=_MODEL,
|
||||
stage_config_path=_CI_DEPLOY,
|
||||
use_stage_cli=False,
|
||||
server_args=["--trust-remote-code", "--no-async-chunk"],
|
||||
use_stage_cli=True,
|
||||
server_args=[
|
||||
"--trust-remote-code",
|
||||
],
|
||||
),
|
||||
id="default",
|
||||
),
|
||||
pytest.param(
|
||||
OmniServerParams(
|
||||
model=_MODEL,
|
||||
stage_config_path=_CI_DEPLOY,
|
||||
use_stage_cli=False,
|
||||
server_args=["--trust-remote-code", "--async-chunk"],
|
||||
),
|
||||
id="async_chunk",
|
||||
),
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
@@ -75,12 +63,12 @@ def get_max_batch_size(size_type="few"):
|
||||
return batch_sizes.get(size_type, 5)
|
||||
|
||||
|
||||
@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1)
|
||||
@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=2)
|
||||
@pytest.mark.parametrize("omni_server", test_params, indirect=True)
|
||||
def test_text_video_to_text_001(omni_server, openai_client) -> None:
|
||||
"""
|
||||
Test text + video input generating text output.
|
||||
Deploy Setting: default single GPU
|
||||
Deploy Setting: default 2GPU
|
||||
Input Modal: text + video
|
||||
Output Modal: text
|
||||
Input Setting: stream=False
|
||||
@@ -102,13 +90,15 @@ def test_text_video_to_text_001(omni_server, openai_client) -> None:
|
||||
openai_client.send_omni_request(request_config, request_num=get_max_batch_size())
|
||||
|
||||
|
||||
@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1)
|
||||
@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=2)
|
||||
@pytest.mark.parametrize("omni_server", test_params, indirect=True)
|
||||
def test_sequential_requests_independent(omni_server, openai_client) -> None:
|
||||
"""
|
||||
Verify that sequential requests produce independent results and that the
|
||||
second request does not receive the first request's audio or token state.
|
||||
Deploy Setting: default single GPU
|
||||
Verify that sequential requests produce independent results and state
|
||||
does not leak between requests. MiniCPM-o 4.5 has max_num_seqs=1 so
|
||||
requests are processed one-at-a-time; the second request must not
|
||||
receive the first request's audio or token state.
|
||||
Deploy Setting: default 2GPU
|
||||
Input Modal: text (two different prompts)
|
||||
Output Modal: text + audio (both)
|
||||
"""
|
||||
@@ -143,42 +133,33 @@ def test_sequential_requests_independent(omni_server, openai_client) -> None:
|
||||
)
|
||||
|
||||
|
||||
@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1)
|
||||
@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=2)
|
||||
@pytest.mark.parametrize("omni_server", test_params, indirect=True)
|
||||
def test_text_to_audio_long_output_001(omni_server, openai_client) -> None:
|
||||
"""
|
||||
Test text input generating a longer audio output to exercise the
|
||||
Code2Wav stage across multiple frames.
|
||||
Deploy Setting: default single GPU
|
||||
token2wav decoder across multiple frames.
|
||||
Deploy Setting: default 2GPU
|
||||
Input Modal: text (longer prompt)
|
||||
Output Modal: text + audio
|
||||
Input Setting: stream=True
|
||||
"""
|
||||
messages = dummy_messages_from_mix_data(
|
||||
system_prompt=get_system_prompt(),
|
||||
content_text="What is the capital of China? Answer in 40 words.",
|
||||
content_text="Tell me a short story about a cat in about 50 words.",
|
||||
)
|
||||
|
||||
request_config = {
|
||||
"model": omni_server.model,
|
||||
"messages": messages,
|
||||
"modalities": ["text", "audio"],
|
||||
"stream": True,
|
||||
# Delimit the assistant answer with the TTS template so the Talker
|
||||
# does not spend its codec-token budget speaking hidden reasoning.
|
||||
"extra_body": {
|
||||
"chat_template_kwargs": {
|
||||
"use_tts_template": True,
|
||||
"enable_thinking": False,
|
||||
}
|
||||
},
|
||||
"key_words": {"audio": ["Beijing"]},
|
||||
"key_words": {"audio": ["cat"]},
|
||||
}
|
||||
|
||||
openai_client.send_omni_request(request_config, request_num=get_max_batch_size())
|
||||
|
||||
|
||||
@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1)
|
||||
@hardware_test(res={"cuda": "H100", "npu": "A2"}, num_cards=2)
|
||||
@pytest.mark.parametrize("omni_server", test_params, indirect=True)
|
||||
def test_chinese_text_to_audio(omni_server, openai_client) -> None:
|
||||
"""
|
||||
|
||||
@@ -30,7 +30,7 @@ pytestmark = [
|
||||
]
|
||||
|
||||
|
||||
@hardware_test(res={"cuda": "L4"}, num_cards=1)
|
||||
@hardware_test(res={"cuda": "L4", "xpu": "B60"}, num_cards=1)
|
||||
def test_speech_english_basic(omni_server, openai_client) -> None:
|
||||
"""Test basic English TTS generation."""
|
||||
openai_client.send_audio_speech_request(
|
||||
@@ -46,7 +46,7 @@ def test_speech_english_basic(omni_server, openai_client) -> None:
|
||||
)
|
||||
|
||||
|
||||
@hardware_test(res={"cuda": "L4"}, num_cards=1)
|
||||
@hardware_test(res={"cuda": "L4", "xpu": "B60"}, num_cards=1)
|
||||
def test_speech_english_streaming(omni_server, openai_client) -> None:
|
||||
"""Test basic streaming English TTS generation (PCM via streaming API)."""
|
||||
openai_client.send_audio_speech_request(
|
||||
|
||||
@@ -300,56 +300,6 @@ def test_strip_single_engine_args_model_does_not_trigger_warning(mocker):
|
||||
assert "model" not in warned_args
|
||||
|
||||
|
||||
def test_new_format_deploy_yaml_does_not_strip_interleave_cli_overrides(tmp_path):
|
||||
"""New-format deploy YAMLs must not strip interleave_mm_strings / media_io_kwargs.
|
||||
|
||||
Pytest historically passed ``--stage-configs-path`` for deploy YAMLs; stripping
|
||||
those CLI flags dropped MiniCPM interleaved AV packing and cut Daily-Omni
|
||||
accuracy from ~78% to ~70%.
|
||||
"""
|
||||
from vllm_omni.entrypoints.utils import is_new_format_deploy_config
|
||||
|
||||
yaml_path = tmp_path / "minicpm_deploy.yaml"
|
||||
yaml_path.write_text(
|
||||
"""
|
||||
pipeline: minicpmo_4_5
|
||||
async_chunk: true
|
||||
stages:
|
||||
- stage_id: 0
|
||||
devices: "0"
|
||||
""".strip(),
|
||||
encoding="utf-8",
|
||||
)
|
||||
path = str(yaml_path)
|
||||
assert is_new_format_deploy_config(path)
|
||||
|
||||
kwargs = {
|
||||
"stage_configs_path": path,
|
||||
"trust_remote_code": True,
|
||||
"interleave_mm_strings": True,
|
||||
"media_io_kwargs": {"video": {"fps": 1, "num_frames": 128}},
|
||||
"gpu_memory_utilization": 0.55,
|
||||
}
|
||||
# Mirror AsyncOmniEngine._resolve_stage_configs strip decision.
|
||||
if kwargs.get("stage_configs_path") is not None and not is_new_format_deploy_config(kwargs["stage_configs_path"]):
|
||||
base = AsyncOmniEngine._strip_single_engine_args(dict(kwargs))
|
||||
else:
|
||||
base = dict(kwargs)
|
||||
assert base.get("interleave_mm_strings") is True
|
||||
assert base.get("media_io_kwargs") == {"video": {"fps": 1, "num_frames": 128}}
|
||||
# Parent EngineArgs fields also remain available for deploy merge.
|
||||
assert base.get("gpu_memory_utilization") == 0.55
|
||||
|
||||
legacy = tmp_path / "legacy.yaml"
|
||||
legacy.write_text("stage_args:\n - stage_id: 0\n", encoding="utf-8")
|
||||
assert not is_new_format_deploy_config(str(legacy))
|
||||
legacy_kwargs = dict(kwargs, stage_configs_path=str(legacy))
|
||||
stripped = AsyncOmniEngine._strip_single_engine_args(legacy_kwargs)
|
||||
assert "interleave_mm_strings" not in stripped
|
||||
assert "media_io_kwargs" not in stripped
|
||||
assert "gpu_memory_utilization" not in stripped
|
||||
|
||||
|
||||
# For https://github.com/vllm-project/vllm-omni/issues/3293
|
||||
def test_tensor_parallel_size_none_is_handled():
|
||||
"""Ensure the tensor parallel size of None isn't forwarded."""
|
||||
|
||||
@@ -18,7 +18,6 @@ def test_chunk_accumulation_policy_replaces_snapshots_and_drains_delta_state():
|
||||
accumulated = MultimodalPayload.from_dict(
|
||||
{
|
||||
"audio": torch.tensor([1.0]),
|
||||
"meta.segment_end": torch.tensor([0]),
|
||||
"meta.tts_is_last_chunk": torch.tensor([0]),
|
||||
"meta.turn_end": torch.tensor([0]),
|
||||
"meta.stable_request_value": "keep",
|
||||
@@ -27,7 +26,6 @@ def test_chunk_accumulation_policy_replaces_snapshots_and_drains_delta_state():
|
||||
incoming = MultimodalPayload.from_dict(
|
||||
{
|
||||
"audio": torch.tensor([2.0]),
|
||||
"meta.segment_end": torch.tensor([1]),
|
||||
"meta.tts_is_last_chunk": torch.tensor([1]),
|
||||
"meta.turn_end": torch.tensor([1]),
|
||||
}
|
||||
@@ -43,7 +41,6 @@ def test_chunk_accumulation_policy_replaces_snapshots_and_drains_delta_state():
|
||||
drain_delta_payload(merged)
|
||||
|
||||
assert "audio" not in merged
|
||||
assert "meta.segment_end" not in merged
|
||||
assert "meta.tts_is_last_chunk" not in merged
|
||||
assert "meta.turn_end" not in merged
|
||||
assert merged.metadata["meta.stable_request_value"] == "keep"
|
||||
@@ -7,24 +7,15 @@ import asyncio
|
||||
import queue
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
from unittest.mock import ANY, AsyncMock, MagicMock
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
import janus
|
||||
import pytest
|
||||
from vllm.outputs import CompletionOutput, RequestOutput
|
||||
from vllm.sampling_params import SamplingParams
|
||||
|
||||
from vllm_omni.engine.orchestrator import (
|
||||
Orchestrator,
|
||||
OrchestratorRequestState,
|
||||
_OrchestratorDuplexStagePort,
|
||||
)
|
||||
from vllm_omni.engine.orchestrator import Orchestrator, OrchestratorRequestState
|
||||
from vllm_omni.engine.stage_pool import StagePool
|
||||
from vllm_omni.experimental.fullduplex.engine.contracts import (
|
||||
DuplexStageRequestContext,
|
||||
DuplexStageSubmission,
|
||||
)
|
||||
from vllm_omni.experimental.fullduplex.engine.messages import DuplexFence
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
@@ -116,50 +107,11 @@ class FakePrewarmPool:
|
||||
|
||||
async def submit_initial(self, _request_id, _req_state, request, prompt_text=None):
|
||||
self.submitted.append(request)
|
||||
return 0
|
||||
|
||||
def get_bound_replica_id(self, _request_id):
|
||||
return 0
|
||||
|
||||
|
||||
def _duplex_stage_port_submission():
|
||||
stage_pools = []
|
||||
for stage_id in range(3):
|
||||
pool = SimpleNamespace(
|
||||
stage_client=SimpleNamespace(default_sampling_params=SamplingParams(max_tokens=1)),
|
||||
stage_vllm_config=SimpleNamespace(model_config=SimpleNamespace(max_model_len=64)),
|
||||
submit_initial=AsyncMock(return_value=stage_id + 10),
|
||||
submit_update=AsyncMock(return_value=stage_id + 20),
|
||||
)
|
||||
stage_pools.append(pool)
|
||||
request_states: dict[str, OrchestratorRequestState] = {}
|
||||
prewarm = AsyncMock()
|
||||
port = _OrchestratorDuplexStagePort(
|
||||
stage_pools=stage_pools,
|
||||
request_states=request_states,
|
||||
running_counter=None,
|
||||
cleanup_request_ids=AsyncMock(),
|
||||
async_chunk=True,
|
||||
prewarm_async_chunk_stages=prewarm,
|
||||
)
|
||||
context = DuplexStageRequestContext(
|
||||
request_id="req-duplex",
|
||||
session_id="session-duplex",
|
||||
fence=DuplexFence("session-duplex"),
|
||||
stage_id=0,
|
||||
final_stage_id=2,
|
||||
config_generation=0,
|
||||
sampling_params=tuple(SamplingParams(max_tokens=1) for _ in range(3)),
|
||||
)
|
||||
port.ensure_request(context)
|
||||
submission = DuplexStageSubmission(
|
||||
context=context,
|
||||
prompt={"prompt_token_ids": [1, 2]},
|
||||
already_submitted=False,
|
||||
)
|
||||
return port, stage_pools, request_states, prewarm, submission
|
||||
|
||||
|
||||
def _request_output(request_id: str) -> RequestOutput:
|
||||
completion = CompletionOutput(
|
||||
index=0,
|
||||
@@ -247,13 +199,11 @@ async def test_async_prewarm_skips_outgoing_only_stage() -> None:
|
||||
stage2 = FakePrewarmPool("receiver")
|
||||
orchestrator.stage_pools = [stage0, stage1, stage2]
|
||||
orchestrator._emit_tx_edge = lambda **_kwargs: None
|
||||
orchestrator._record_duplex_stage_submission = MagicMock()
|
||||
req_state = OrchestratorRequestState(
|
||||
request_id="req-prewarm",
|
||||
prompt={"prompt_token_ids": [1, 2]},
|
||||
sampling_params_list=[SamplingParams(max_tokens=1) for _ in range(3)],
|
||||
final_stage_id=2,
|
||||
duplex_identity=SimpleNamespace(),
|
||||
)
|
||||
|
||||
await orchestrator._prewarm_async_chunk_stages(
|
||||
@@ -266,23 +216,6 @@ async def test_async_prewarm_skips_outgoing_only_stage() -> None:
|
||||
assert len(stage2.submitted) == 1
|
||||
assert 1 not in req_state.stage_submit_ts
|
||||
assert 2 in req_state.stage_submit_ts
|
||||
orchestrator._record_duplex_stage_submission.assert_called_once_with(
|
||||
2,
|
||||
"req-prewarm",
|
||||
0,
|
||||
req_state,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_duplex_prewarm_runs_after_first_stage0_submission() -> None:
|
||||
port, stage_pools, request_states, prewarm, submission = _duplex_stage_port_submission()
|
||||
|
||||
result = await port.submit(submission)
|
||||
|
||||
assert result.stage_id == 0
|
||||
stage_pools[0].submit_initial.assert_awaited_once()
|
||||
prewarm.assert_awaited_once_with("req-duplex", ANY, request_states["req-duplex"])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
||||
@@ -14,7 +14,7 @@ from vllm.v1.engine import FinishReason
|
||||
from vllm.v1.engine.output_processor import OutputProcessor as VLLMOutputProcessor
|
||||
|
||||
from vllm_omni.outputs import output_processor
|
||||
from vllm_omni.outputs.output_modality import OutputModality, OutputModalityNames
|
||||
from vllm_omni.outputs.output_modality import OutputModalityNames
|
||||
from vllm_omni.outputs.output_processor import MultimodalOutputProcessor, OmniRequestState
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
@@ -452,127 +452,6 @@ def test_no_detokenizer_make_request_output():
|
||||
assert AUDIO in result.outputs[0].multimodal_output
|
||||
|
||||
|
||||
def test_no_detokenizer_process_outputs_returns_nonterminal_audio_chunk(monkeypatch):
|
||||
"""Generation-stage MM-only chunks must reach the orchestrator.
|
||||
|
||||
StagePool registers generation requests without a per-request collector,
|
||||
so the processor return value is the only path from Stage2 to
|
||||
Orchestrator._handle_processed_outputs().
|
||||
"""
|
||||
state = _make_no_detok_state(RequestOutputKind.DELTA)
|
||||
processor = object.__new__(MultimodalOutputProcessor)
|
||||
processor.output_modality = OutputModality.AUDIO
|
||||
processor.request_states = {"r": state}
|
||||
upstream_result = SimpleNamespace(request_outputs=[], reqs_to_abort=[])
|
||||
monkeypatch.setattr(
|
||||
VLLMOutputProcessor,
|
||||
"process_outputs",
|
||||
lambda *_args, **_kwargs: upstream_result,
|
||||
)
|
||||
engine_output = SimpleNamespace(
|
||||
request_id="r",
|
||||
multimodal_output={
|
||||
"model_outputs": torch.arange(8, dtype=torch.float32),
|
||||
"sr": torch.tensor(24000, dtype=torch.int32),
|
||||
},
|
||||
output_type="audio",
|
||||
pooling_output=None,
|
||||
new_token_ids=[],
|
||||
finish_reason=None,
|
||||
stop_reason=None,
|
||||
kv_transfer_params=None,
|
||||
routed_experts=None,
|
||||
num_cached_tokens=0,
|
||||
)
|
||||
|
||||
result = processor.process_outputs([engine_output])
|
||||
|
||||
assert len(result.request_outputs) == 1
|
||||
assert result.request_outputs[0].request_id == "r"
|
||||
assert AUDIO in result.request_outputs[0].outputs[0].multimodal_output
|
||||
|
||||
|
||||
def _make_mm_only_output_processor(monkeypatch):
|
||||
processor = object.__new__(MultimodalOutputProcessor)
|
||||
processor.output_modality = OutputModality.AUDIO
|
||||
processor.request_states = {"r": _make_no_detok_state(RequestOutputKind.DELTA)}
|
||||
monkeypatch.setattr(
|
||||
VLLMOutputProcessor,
|
||||
"process_outputs",
|
||||
lambda *_args, **_kwargs: SimpleNamespace(
|
||||
request_outputs=[],
|
||||
reqs_to_abort=[],
|
||||
),
|
||||
)
|
||||
|
||||
def finish_request(req_state):
|
||||
processor.request_states.pop(req_state.request_id, None)
|
||||
|
||||
processor._finish_request = finish_request
|
||||
return processor
|
||||
|
||||
|
||||
def _audio_engine_output(*, is_segment_finished: bool, is_last_chunk: bool):
|
||||
return SimpleNamespace(
|
||||
request_id="r",
|
||||
multimodal_output={
|
||||
"model_outputs": torch.arange(8, dtype=torch.float32),
|
||||
"meta.tts_is_last_chunk": torch.tensor([int(is_last_chunk)]),
|
||||
},
|
||||
output_type="audio",
|
||||
pooling_output=None,
|
||||
new_token_ids=[],
|
||||
finish_reason=FinishReason.STOP,
|
||||
stop_reason=None,
|
||||
kv_transfer_params=None,
|
||||
routed_experts=None,
|
||||
num_cached_tokens=0,
|
||||
is_segment_finished=is_segment_finished,
|
||||
)
|
||||
|
||||
|
||||
def test_mm_only_segment_finish_retains_request_state_for_next_audio_chunk(monkeypatch):
|
||||
processor = _make_mm_only_output_processor(monkeypatch)
|
||||
|
||||
first = processor.process_outputs([_audio_engine_output(is_segment_finished=True, is_last_chunk=False)])
|
||||
|
||||
assert "r" in processor.request_states
|
||||
assert len(first.request_outputs) == 1
|
||||
assert first.request_outputs[0].finished is False
|
||||
|
||||
second = processor.process_outputs([_audio_engine_output(is_segment_finished=False, is_last_chunk=True)])
|
||||
|
||||
assert len(second.request_outputs) == 1
|
||||
assert second.request_outputs[0].finished is True
|
||||
assert "r" not in processor.request_states
|
||||
|
||||
|
||||
def test_mm_only_nonfinal_audio_retains_state_without_segment_marker(monkeypatch):
|
||||
processor = _make_mm_only_output_processor(monkeypatch)
|
||||
|
||||
first = processor.process_outputs([_audio_engine_output(is_segment_finished=False, is_last_chunk=False)])
|
||||
|
||||
assert "r" in processor.request_states
|
||||
assert len(first.request_outputs) == 1
|
||||
assert first.request_outputs[0].finished is False
|
||||
|
||||
second = processor.process_outputs([_audio_engine_output(is_segment_finished=False, is_last_chunk=True)])
|
||||
|
||||
assert len(second.request_outputs) == 1
|
||||
assert second.request_outputs[0].finished is True
|
||||
assert "r" not in processor.request_states
|
||||
|
||||
|
||||
def test_mm_only_terminal_finish_removes_request_state(monkeypatch):
|
||||
processor = _make_mm_only_output_processor(monkeypatch)
|
||||
|
||||
result = processor.process_outputs([_audio_engine_output(is_segment_finished=False, is_last_chunk=True)])
|
||||
|
||||
assert len(result.request_outputs) == 1
|
||||
assert result.request_outputs[0].finished is True
|
||||
assert "r" not in processor.request_states
|
||||
|
||||
|
||||
def test_no_detokenizer_make_request_output_with_routed_experts():
|
||||
"""make_request_output accepts the routed_experts arg that the multimodal
|
||||
output channel (_process_mm_only_outputs) passes positionally, and attaches
|
||||
|
||||
@@ -3,7 +3,6 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import socket
|
||||
from contextlib import contextmanager
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
@@ -12,7 +11,6 @@ import pytest
|
||||
from pytest_mock import MockerFixture
|
||||
from vllm.v1.engine.utils import EngineZmqAddresses
|
||||
|
||||
from vllm_omni.config.stage_config import DuplexSessionRuntimeConfig
|
||||
from vllm_omni.engine.async_omni_engine import AsyncOmniEngine
|
||||
from vllm_omni.engine.stage_engine_core_client import StageEngineCoreClientBase
|
||||
from vllm_omni.engine.stage_engine_startup import (
|
||||
@@ -191,34 +189,6 @@ class TestOmniMasterServerAllocation:
|
||||
output_port = int(alloc.output_bind_address.split(":")[-1])
|
||||
assert len({handshake_port, input_port, output_port}) == 3
|
||||
|
||||
def test_route_ports_remain_os_reserved_until_master_releases_them(self):
|
||||
server = OmniMasterServer(master_address="127.0.0.1", master_port=15005, stage_ids=[0])
|
||||
alloc = server.get_allocation(0)
|
||||
ports = [
|
||||
int(alloc.handshake_bind_address.rsplit(":", 1)[-1]),
|
||||
int(alloc.input_bind_address.rsplit(":", 1)[-1]),
|
||||
int(alloc.output_bind_address.rsplit(":", 1)[-1]),
|
||||
]
|
||||
|
||||
for port in ports:
|
||||
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as probe:
|
||||
with pytest.raises(OSError):
|
||||
probe.bind(("127.0.0.1", port))
|
||||
|
||||
server.release_route_port_reservations(0, handshake=True, data=False)
|
||||
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as probe:
|
||||
probe.bind(("127.0.0.1", ports[0]))
|
||||
for port in ports[1:]:
|
||||
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as probe:
|
||||
with pytest.raises(OSError):
|
||||
probe.bind(("127.0.0.1", port))
|
||||
|
||||
server.stop()
|
||||
|
||||
for port in ports[1:]:
|
||||
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as probe:
|
||||
probe.bind(("127.0.0.1", port))
|
||||
|
||||
def test_get_zmq_addresses_returns_bind_addresses(self):
|
||||
server = OmniMasterServer(master_address="127.0.0.1", master_port=15002, stage_ids=[0])
|
||||
alloc = server.get_allocation(0)
|
||||
@@ -427,25 +397,6 @@ class TestSingleStageModeDetection:
|
||||
assert engine.single_stage_mode is False
|
||||
assert engine._single_stage_id_filter is None
|
||||
|
||||
def test_stage_configs_path_loads_duplex_runtime_config(self, mocker: MockerFixture):
|
||||
duplex_session = DuplexSessionRuntimeConfig(max_sessions=2)
|
||||
mocker.patch(
|
||||
"vllm_omni.engine.async_omni_engine.StageConfigFactory.get_pipeline_config",
|
||||
return_value=None,
|
||||
)
|
||||
load_deploy_config = mocker.patch(
|
||||
"vllm_omni.engine.async_omni_engine.load_deploy_config",
|
||||
return_value=SimpleNamespace(duplex_session=duplex_session),
|
||||
)
|
||||
|
||||
engine = self._make_engine_no_thread(
|
||||
mocker,
|
||||
stage_configs_path="/fake/duplex.yaml",
|
||||
)
|
||||
|
||||
load_deploy_config.assert_called_once_with("/fake/duplex.yaml")
|
||||
assert engine.duplex_session_config is duplex_session
|
||||
|
||||
def test_single_stage_mode_without_stage_id_has_no_filter(self, mocker: MockerFixture):
|
||||
engine = self._make_engine_no_thread(
|
||||
mocker,
|
||||
|
||||
@@ -51,19 +51,6 @@ from vllm_omni.outputs import OmniRequestOutput
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
def test_native_input_append_supports_explicit_session_opt_out():
|
||||
session = DuplexSession(
|
||||
"sid-explicit-tts",
|
||||
DuplexSessionConfig(model="test-model"),
|
||||
capabilities=DuplexCapabilities.minicpmo45_native(),
|
||||
)
|
||||
|
||||
assert OmniDuplexSessionHandler._uses_native_input_append(session) is True
|
||||
|
||||
session.config.extra_body["minicpmo45_native_duplex"] = False
|
||||
assert OmniDuplexSessionHandler._uses_native_input_append(session) is False
|
||||
|
||||
|
||||
class _ModelConfig:
|
||||
model = "test-model"
|
||||
|
||||
@@ -2533,52 +2520,6 @@ def test_duplex_auto_response_tts_scheduler_eos_fallback():
|
||||
assert results[0]["abort_data_plane_request"] is True
|
||||
|
||||
|
||||
def test_duplex_auto_response_rearms_tts_boundary_for_next_segment_in_same_turn():
|
||||
data_plane = _test_data_plane()
|
||||
request_id = "duplex-sid-next-segment-same-turn-e0-stage0"
|
||||
session = DuplexSession(
|
||||
session_id="sid-next-segment-same-turn",
|
||||
config=DuplexSessionConfig(extra_body={"auto_response": True}),
|
||||
)
|
||||
session.bind_response_turn(0)
|
||||
data_plane.begin_request(request_id)
|
||||
|
||||
first_segment = _duplex_tts_output(
|
||||
request_id=request_id,
|
||||
samples=24000,
|
||||
finished=False,
|
||||
tts_is_last_chunk=True,
|
||||
turn_id=0,
|
||||
)
|
||||
first_results = _project_data_plane(
|
||||
data_plane,
|
||||
{"data_plane_outputs": [first_segment]},
|
||||
session=session,
|
||||
)
|
||||
|
||||
assert len(first_results) == 1
|
||||
assert first_results[0]["abort_data_plane_request"] is True
|
||||
|
||||
data_plane.begin_request(request_id)
|
||||
second_segment = _duplex_tts_output(
|
||||
request_id=request_id,
|
||||
samples=48000,
|
||||
finished=False,
|
||||
text="hello again",
|
||||
tts_is_last_chunk=True,
|
||||
turn_id=0,
|
||||
)
|
||||
second_results = _project_data_plane(
|
||||
data_plane,
|
||||
{"data_plane_outputs": [second_segment]},
|
||||
session=session,
|
||||
)
|
||||
|
||||
assert len(second_results) == 1
|
||||
assert second_results[0]["audio_data"] == "wav-24000"
|
||||
assert second_results[0]["abort_data_plane_request"] is True
|
||||
|
||||
|
||||
def test_duplex_turn_end_is_not_swallowed_by_finished_segment_fallback():
|
||||
data_plane = _test_data_plane()
|
||||
request_id = "duplex-sid-turn-end-after-segment-e0-stage0"
|
||||
@@ -6261,27 +6202,19 @@ async def test_minicpmo_native_auto_response_keeps_request_bound_for_segment_con
|
||||
config_timeout_s=0.1,
|
||||
idle_timeout_s=1,
|
||||
)
|
||||
|
||||
def on_send(ws: TimedWebSocket, data: dict[str, Any]) -> None:
|
||||
if data.get("type") != "session.created":
|
||||
return
|
||||
session = handler._registry.get("sid-native-auto-continuation")
|
||||
assert session is not None
|
||||
session.capabilities.chunk_period_ms = 50
|
||||
ws.put(
|
||||
{
|
||||
"type": "input_audio_buffer.append",
|
||||
"audio": _pcm_f32_b64(800),
|
||||
"format": "pcm_f32le",
|
||||
"sample_rate_hz": 16000,
|
||||
}
|
||||
)
|
||||
ws.put({"type": "input_audio_buffer.commit", "final": True})
|
||||
|
||||
ws = TimedWebSocket(on_send=on_send)
|
||||
ws = TimedWebSocket()
|
||||
event = _native_session_create("sid-native-auto-continuation")
|
||||
event["session"]["extra_body"]["auto_response"] = True
|
||||
ws.put(event)
|
||||
ws.put(
|
||||
{
|
||||
"type": "input_audio_buffer.append",
|
||||
"audio": _pcm_f32_b64(16000),
|
||||
"format": "pcm_f32le",
|
||||
"sample_rate_hz": 16000,
|
||||
}
|
||||
)
|
||||
ws.put({"type": "input_audio_buffer.commit", "final": True})
|
||||
|
||||
await handler.handle_session(ws)
|
||||
|
||||
@@ -6292,112 +6225,6 @@ async def test_minicpmo_native_auto_response_keeps_request_bound_for_segment_con
|
||||
assert final is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_minicpmo_native_auto_response_real_input_preempts_unsent_silence():
|
||||
request_id = "duplex-sid-native-real-before-silence-e0-stage0"
|
||||
|
||||
class SequencedAppendEngine(FakeEngineClient):
|
||||
def __init__(self) -> None:
|
||||
control_result = {
|
||||
"operation": "append",
|
||||
"session_id": "sid-native-real-before-silence",
|
||||
"ok": True,
|
||||
"unsupported_count": 0,
|
||||
"error_count": 0,
|
||||
"stage_results": [
|
||||
{
|
||||
"result": {
|
||||
"data_plane_append": True,
|
||||
"request_id": request_id,
|
||||
"response_stage_id": 1,
|
||||
}
|
||||
}
|
||||
],
|
||||
}
|
||||
terminal_segment = _duplex_tts_output(
|
||||
request_id=request_id,
|
||||
samples=0,
|
||||
finished=True,
|
||||
tts_is_last_chunk=True,
|
||||
token_ids=[151645],
|
||||
)
|
||||
super().__init__(
|
||||
control_result=control_result,
|
||||
collect_outputs=[[terminal_segment], [], []],
|
||||
collect_delay_s=0.01,
|
||||
)
|
||||
self.append_sequence: list[str] = []
|
||||
self.first_real_started = asyncio.Event()
|
||||
self.second_real_started = asyncio.Event()
|
||||
self.real_count = 0
|
||||
|
||||
async def append_duplex_input_async(self, session_id: str, **kwargs):
|
||||
kwargs.pop("expected_epoch", None)
|
||||
result = await super().append_duplex_input_async(session_id, **kwargs)
|
||||
payload = kwargs.get("payload")
|
||||
is_silence = (
|
||||
isinstance(payload, dict)
|
||||
and payload.get("audio") == OmniDuplexSessionHandler._NATIVE_SILENCE_UNIT_PAYLOAD_AUDIO
|
||||
)
|
||||
if is_silence:
|
||||
self.append_sequence.append("silence")
|
||||
else:
|
||||
self.real_count += 1
|
||||
self.append_sequence.append(f"real-{self.real_count}")
|
||||
if self.real_count == 1:
|
||||
self.first_real_started.set()
|
||||
elif self.real_count == 2:
|
||||
self.second_real_started.set()
|
||||
return result
|
||||
|
||||
session_created = asyncio.Event()
|
||||
engine = SequencedAppendEngine()
|
||||
handler = OmniDuplexSessionHandler(
|
||||
chat_service=FakeChatService(engine),
|
||||
config_timeout_s=0.1,
|
||||
idle_timeout_s=1,
|
||||
)
|
||||
|
||||
def on_send(ws: TimedWebSocket, data: dict[str, Any]) -> None:
|
||||
if data.get("type") != "session.created":
|
||||
return
|
||||
session = handler._registry.get("sid-native-real-before-silence")
|
||||
assert session is not None
|
||||
session.capabilities.chunk_period_ms = 50
|
||||
ws.put(
|
||||
{
|
||||
"type": "input_audio_buffer.append",
|
||||
"audio": _pcm_f32_b64(800),
|
||||
"format": "pcm_f32le",
|
||||
"sample_rate_hz": 16000,
|
||||
}
|
||||
)
|
||||
session_created.set()
|
||||
|
||||
ws = TimedWebSocket(on_send=on_send, receive_timeout_s=2.0)
|
||||
event = _native_session_create("sid-native-real-before-silence")
|
||||
event["session"]["extra_body"]["auto_response"] = True
|
||||
ws.put(event)
|
||||
handler_task = asyncio.create_task(handler.handle_session(ws))
|
||||
|
||||
await asyncio.wait_for(session_created.wait(), timeout=1)
|
||||
await asyncio.wait_for(engine.first_real_started.wait(), timeout=1)
|
||||
await asyncio.sleep(0.02)
|
||||
ws.put(
|
||||
{
|
||||
"type": "input_audio_buffer.append",
|
||||
"audio": _pcm_f32_b64(800, value=0.07),
|
||||
"format": "pcm_f32le",
|
||||
"sample_rate_hz": 16000,
|
||||
}
|
||||
)
|
||||
await asyncio.wait_for(engine.second_real_started.wait(), timeout=1)
|
||||
ws.put({"type": "session.close"})
|
||||
await asyncio.wait_for(handler_task, timeout=2)
|
||||
|
||||
assert engine.append_sequence[:2] == ["real-1", "real-2"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_minicpmo_native_auto_response_real_input_waits_for_submitted_silence_tail():
|
||||
request_id = "duplex-sid-native-silence-tail-e0-stage0"
|
||||
@@ -6467,26 +6294,18 @@ async def test_minicpmo_native_auto_response_real_input_waits_for_submitted_sile
|
||||
config_timeout_s=0.1,
|
||||
idle_timeout_s=1,
|
||||
)
|
||||
|
||||
def on_send(ws: TimedWebSocket, data: dict[str, Any]) -> None:
|
||||
if data.get("type") != "session.created":
|
||||
return
|
||||
session = handler._registry.get("sid-native-silence-tail")
|
||||
assert session is not None
|
||||
session.capabilities.chunk_period_ms = 50
|
||||
ws.put(
|
||||
{
|
||||
"type": "input_audio_buffer.append",
|
||||
"audio": _pcm_f32_b64(800, value=0.05),
|
||||
"format": "pcm_f32le",
|
||||
"sample_rate_hz": 16000,
|
||||
}
|
||||
)
|
||||
|
||||
ws = TimedWebSocket(on_send=on_send, receive_timeout_s=2.0)
|
||||
ws = TimedWebSocket(receive_timeout_s=2.0)
|
||||
event = _native_session_create("sid-native-silence-tail")
|
||||
event["session"]["extra_body"]["auto_response"] = True
|
||||
ws.put(event)
|
||||
ws.put(
|
||||
{
|
||||
"type": "input_audio_buffer.append",
|
||||
"audio": _pcm_f32_b64(16000, value=0.05),
|
||||
"format": "pcm_f32le",
|
||||
"sample_rate_hz": 16000,
|
||||
}
|
||||
)
|
||||
handler_task = asyncio.create_task(handler.handle_session(ws))
|
||||
|
||||
await asyncio.wait_for(engine.silence_append_started.wait(), timeout=1)
|
||||
@@ -6511,135 +6330,6 @@ async def test_minicpmo_native_auto_response_real_input_waits_for_submitted_sile
|
||||
assert engine.append_sequence == ["real-1", "silence-started", "silence-returned", "real-2"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_minicpmo_native_auto_response_preserves_silence_continuations_across_append_tails():
|
||||
request_id = "duplex-sid-native-continuation-tails-e0-stage0"
|
||||
|
||||
class SequencedAppendEngine(FakeEngineClient):
|
||||
def __init__(self) -> None:
|
||||
control_result = {
|
||||
"operation": "append",
|
||||
"session_id": "sid-native-continuation-tails",
|
||||
"ok": True,
|
||||
"unsupported_count": 0,
|
||||
"error_count": 0,
|
||||
"stage_results": [
|
||||
{
|
||||
"result": {
|
||||
"data_plane_append": True,
|
||||
"request_id": request_id,
|
||||
"response_stage_id": 1,
|
||||
}
|
||||
}
|
||||
],
|
||||
}
|
||||
super().__init__(control_result=control_result)
|
||||
self.real_append_started = asyncio.Event()
|
||||
self.release_real_append = asyncio.Event()
|
||||
self.first_silence_started = asyncio.Event()
|
||||
self.release_first_silence = asyncio.Event()
|
||||
self.second_silence_started = asyncio.Event()
|
||||
self.release_collect = asyncio.Event()
|
||||
self.silence_count = 0
|
||||
|
||||
async def append_duplex_input_async(self, session_id: str, **kwargs):
|
||||
kwargs.pop("expected_epoch", None)
|
||||
payload = kwargs.get("payload")
|
||||
assert isinstance(payload, dict)
|
||||
result = await super().append_duplex_input_async(session_id, **kwargs)
|
||||
if payload.get("audio") != OmniDuplexSessionHandler._NATIVE_SILENCE_UNIT_PAYLOAD_AUDIO:
|
||||
self.real_append_started.set()
|
||||
await self.release_real_append.wait()
|
||||
return result
|
||||
|
||||
self.silence_count += 1
|
||||
if self.silence_count == 1:
|
||||
self.first_silence_started.set()
|
||||
await self.release_first_silence.wait()
|
||||
else:
|
||||
self.second_silence_started.set()
|
||||
return result
|
||||
|
||||
async def collect_duplex_data_plane_outputs_async(
|
||||
self,
|
||||
request_id: str,
|
||||
*,
|
||||
response_stage_id: int | None = None,
|
||||
timeout: float | None = None,
|
||||
) -> list[object]:
|
||||
del timeout
|
||||
self.collected.append((request_id, response_stage_id))
|
||||
await self.release_collect.wait()
|
||||
return []
|
||||
|
||||
session_created = asyncio.Event()
|
||||
|
||||
def on_send(_ws: TimedWebSocket, data: dict[str, Any]) -> None:
|
||||
if data.get("type") == "session.created":
|
||||
session_created.set()
|
||||
|
||||
engine = SequencedAppendEngine()
|
||||
handler = OmniDuplexSessionHandler(
|
||||
chat_service=FakeChatService(engine),
|
||||
config_timeout_s=0.1,
|
||||
idle_timeout_s=1,
|
||||
)
|
||||
ws = TimedWebSocket(on_send=on_send, receive_timeout_s=2.0)
|
||||
event = _native_session_create("sid-native-continuation-tails")
|
||||
event["session"]["extra_body"]["auto_response"] = True
|
||||
ws.put(event)
|
||||
ws.put(
|
||||
{
|
||||
"type": "input_audio_buffer.append",
|
||||
"audio": _pcm_f32_b64(16000, value=0.05),
|
||||
"format": "pcm_f32le",
|
||||
"sample_rate_hz": 16000,
|
||||
}
|
||||
)
|
||||
handler_task = asyncio.create_task(handler.handle_session(ws))
|
||||
|
||||
await asyncio.wait_for(session_created.wait(), timeout=1)
|
||||
await asyncio.wait_for(engine.real_append_started.wait(), timeout=1)
|
||||
session = handler._registry.get("sid-native-continuation-tails")
|
||||
assert session is not None
|
||||
session.capabilities.chunk_period_ms = 50
|
||||
session.bind_request(request_id)
|
||||
response_id = session.begin_response(turn_id=session.turn_id)
|
||||
native = handler._minicpmo_session_state(session)
|
||||
scheduler = native.silence_continuation_scheduler
|
||||
assert scheduler is not None
|
||||
|
||||
payload = handler._native_silence_unit_payload()
|
||||
payload["duplex_turn_id"] = session.turn_id
|
||||
kwargs = {
|
||||
"request_id": request_id,
|
||||
"owner_id": f"response:{response_id}",
|
||||
"response_id": response_id,
|
||||
"response_owned": True,
|
||||
"expected_epoch": session.epoch,
|
||||
"expected_incarnation": session.incarnation,
|
||||
"expected_model_turn_id": None,
|
||||
"send_json": ws.send_json,
|
||||
}
|
||||
assert await scheduler(payload, **kwargs) is True
|
||||
|
||||
next_schedule = asyncio.create_task(scheduler(payload, **kwargs))
|
||||
await asyncio.sleep(0.05)
|
||||
assert not next_schedule.done()
|
||||
|
||||
engine.release_real_append.set()
|
||||
await asyncio.wait_for(engine.first_silence_started.wait(), timeout=1)
|
||||
assert not next_schedule.done()
|
||||
|
||||
engine.release_first_silence.set()
|
||||
assert await asyncio.wait_for(next_schedule, timeout=1) is True
|
||||
await asyncio.wait_for(engine.second_silence_started.wait(), timeout=1)
|
||||
ws.put({"type": "session.close"})
|
||||
await asyncio.wait_for(handler_task, timeout=2)
|
||||
|
||||
assert engine.silence_count == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_minicpmo_native_duplex_uses_segment_text_metadata_for_transcript_cursor():
|
||||
request_id = "duplex-sid-native-repeat-text-e0-stage0"
|
||||
|
||||
@@ -201,6 +201,7 @@ def _assert_realtime_smoke(result: dict) -> None:
|
||||
def _assert_realtime_accuracy(
|
||||
result: dict,
|
||||
whisper_model_size: str = "large-v3",
|
||||
threshold: float = 0.8,
|
||||
) -> None:
|
||||
"""Assert that whisper transcription of audio output matches model text.
|
||||
|
||||
@@ -214,6 +215,12 @@ def _assert_realtime_accuracy(
|
||||
variability even though audio generation was correct. large-v3
|
||||
transcribes these clips reliably, so a failure here now points
|
||||
at the model, not the ASR grader.
|
||||
threshold: Minimum cosine similarity (with length penalty) required to
|
||||
pass. Default 0.8. Do not lower per-callsite without data:
|
||||
at 0.35 the assertion no longer detects real audio
|
||||
regressions. If a variant genuinely needs a different gate
|
||||
(e.g. whisper partial transcripts under async_chunk), propose
|
||||
it in its own PR with measurements.
|
||||
"""
|
||||
final_text = (result["transcription_text"] or "").strip()
|
||||
assert final_text, "Expected non-empty transcription (model text stream)"
|
||||
@@ -223,8 +230,9 @@ def _assert_realtime_accuracy(
|
||||
assert whisper_text, "Whisper returned empty string for synthesized output audio"
|
||||
|
||||
sim = cosine_similarity_text(whisper_text.lower(), final_text.lower())
|
||||
assert sim > 0.8, (
|
||||
f"Output audio transcript should match model text (sim={sim:.3f}): "
|
||||
assert sim > threshold, (
|
||||
f"Output audio transcript should match model text (sim={sim:.3f}, "
|
||||
f"threshold={threshold}): "
|
||||
f"whisper={whisper_text!r}, model_text={final_text!r}"
|
||||
)
|
||||
|
||||
|
||||
@@ -305,21 +305,11 @@ def mock_get_config(monkeypatch):
|
||||
"""Auto-mock get_config and related model loading functions."""
|
||||
import sys
|
||||
|
||||
fake_tokenizer = _ns()
|
||||
fake_tokenizer.encode = lambda *args, **kwargs: [1, 2, 3]
|
||||
fake_tokenizer.decode = lambda *args, **kwargs: "test"
|
||||
|
||||
def _mock_init_tokenizer_from_configs(model_config=None, **kwargs):
|
||||
return fake_tokenizer
|
||||
|
||||
monkeypatch.setattr(
|
||||
"vllm.transformers_utils.tokenizer.init_tokenizer_from_configs",
|
||||
_mock_init_tokenizer_from_configs,
|
||||
raising=False,
|
||||
)
|
||||
tokenizer_module_path = "vllm.transformers_utils.tokenizer"
|
||||
if tokenizer_module_path in sys.modules:
|
||||
setattr(sys.modules[tokenizer_module_path], "init_tokenizer_from_configs", _mock_init_tokenizer_from_configs)
|
||||
# TODO(pd-unskip): the tokenizer mock that lived here targeted
|
||||
# vllm.transformers_utils.tokenizer.init_tokenizer_from_configs, which was
|
||||
# deleted in vLLM 0.26 (the raising=False setattr made it a silent no-op).
|
||||
# When the module-level skip is lifted, mock whatever tokenizer seam the
|
||||
# revived PD entrypoint code actually uses.
|
||||
|
||||
def _mock_length_from_prompt_token_ids_or_embeds(prompt_token_ids=None, prompt_embeds=None):
|
||||
if prompt_token_ids is not None:
|
||||
@@ -344,13 +334,6 @@ def mock_get_config(monkeypatch):
|
||||
_mock_length_from_prompt_token_ids_or_embeds,
|
||||
)
|
||||
|
||||
monkeypatch.setattr(
|
||||
"vllm_omni.entrypoints.async_omni.init_tokenizer_from_configs", _mock_init_tokenizer_from_configs, raising=False
|
||||
)
|
||||
async_omni_path = "vllm_omni.entrypoints.async_omni"
|
||||
if async_omni_path in sys.modules:
|
||||
setattr(sys.modules[async_omni_path], "init_tokenizer_from_configs", _mock_init_tokenizer_from_configs)
|
||||
|
||||
fake_hf_config = _ns()
|
||||
fake_hf_config.model_type = "qwen2_5_omni"
|
||||
|
||||
|
||||
@@ -222,7 +222,6 @@ def test_run_headless_llm_registers_with_auto_assigned_replica_id(mocker: Mocker
|
||||
from vllm_omni.engine.stage_engine_startup import StageRegistrationResponse
|
||||
|
||||
stage_cfg = _make_stage_cfg(0, stage_type="llm")
|
||||
stage_cfg.engine_args["async_chunk"] = True
|
||||
parallel_config = SimpleNamespace(
|
||||
data_parallel_size_local=1,
|
||||
data_parallel_rank=0,
|
||||
@@ -242,10 +241,7 @@ def test_run_headless_llm_registers_with_auto_assigned_replica_id(mocker: Mocker
|
||||
"vllm_omni.distributed.omni_connectors.utils.initialization.resolve_omni_kv_config_for_stage",
|
||||
return_value=(None, None, None),
|
||||
)
|
||||
mock_connector_spec = mocker.patch(
|
||||
"vllm_omni.engine.stage_init_utils.get_stage_connector_spec",
|
||||
return_value={},
|
||||
)
|
||||
mocker.patch("vllm_omni.engine.stage_init_utils.get_stage_connector_spec", return_value={})
|
||||
mocker.patch("vllm_omni.engine.stage_init_utils.build_engine_args_dict", return_value={})
|
||||
mocker.patch(
|
||||
"vllm_omni.engine.stage_init_utils.build_vllm_config",
|
||||
@@ -281,7 +277,6 @@ def test_run_headless_llm_registers_with_auto_assigned_replica_id(mocker: Mocker
|
||||
assert kwargs["omni_stage_config"] is stage_cfg
|
||||
assert kwargs["replica_id"] is None
|
||||
assert "socket_ownership" not in kwargs
|
||||
assert mock_connector_spec.call_args.kwargs["async_chunk"] is True
|
||||
|
||||
assert mock_manager_cls.call_count == 1
|
||||
mgr_kwargs = mock_manager_cls.call_args.kwargs
|
||||
|
||||
@@ -3,8 +3,6 @@
|
||||
import os
|
||||
from collections import Counter
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
@@ -15,10 +13,6 @@ from vllm_omni.config.yaml_util import create_config
|
||||
from vllm_omni.diffusion.data import OmniDiffusionConfig
|
||||
from vllm_omni.engine.arg_utils import OmniEngineArgs
|
||||
from vllm_omni.engine.async_omni_engine import AsyncOmniEngine
|
||||
from vllm_omni.engine.stage_init_utils import (
|
||||
get_stage_connector_spec,
|
||||
load_omni_transfer_config_for_model,
|
||||
)
|
||||
from vllm_omni.entrypoints.utils import (
|
||||
_convert_dataclasses_to_dict,
|
||||
_filter_dict_like_object,
|
||||
@@ -319,58 +313,6 @@ class TestResolveModelConfigPath:
|
||||
assert result is not None
|
||||
assert "glm_image.yaml" in result
|
||||
|
||||
def test_uses_registry_default_deploy_when_hf_model_type_differs(
|
||||
self,
|
||||
mocker: MockerFixture,
|
||||
tmp_path,
|
||||
):
|
||||
from vllm_omni.config.stage_config import _DEPLOY_DIR as real_deploy_dir
|
||||
from vllm_omni.entrypoints import utils as utils_mod
|
||||
|
||||
hf_config = SimpleNamespace(
|
||||
model_type="minicpmo",
|
||||
architectures=["MiniCPMO"],
|
||||
version="4.5",
|
||||
)
|
||||
model = "local/minicpmo-registry-default-test"
|
||||
mocker.patch(
|
||||
"vllm_omni.entrypoints.utils.get_config",
|
||||
return_value=hf_config,
|
||||
)
|
||||
mocker.patch(
|
||||
"vllm_omni.config.config_factory.get_config",
|
||||
return_value=hf_config,
|
||||
)
|
||||
# A bare deploy/<hf_model_type>.yaml must not win over the registered
|
||||
# pipeline default when HF model_type is not an OMNI_PIPELINES key.
|
||||
(tmp_path / "minicpmo.yaml").write_text("stages: []\n", encoding="utf-8")
|
||||
(tmp_path / "minicpmo_4_5.yaml").write_bytes((real_deploy_dir / "minicpmo_4_5.yaml").read_bytes())
|
||||
mocker.patch.object(utils_mod, "_DEPLOY_DIR", tmp_path)
|
||||
|
||||
result = resolve_model_config_path(model)
|
||||
|
||||
assert result is not None
|
||||
assert Path(result).as_posix().endswith("minicpmo_4_5.yaml")
|
||||
assert Path(result).name == "minicpmo_4_5.yaml"
|
||||
transfer_config = load_omni_transfer_config_for_model(model, result)
|
||||
assert transfer_config is not None
|
||||
|
||||
sender = get_stage_connector_spec(
|
||||
omni_transfer_config=transfer_config,
|
||||
stage_id=1,
|
||||
async_chunk=True,
|
||||
)
|
||||
receiver = get_stage_connector_spec(
|
||||
omni_transfer_config=transfer_config,
|
||||
stage_id=2,
|
||||
async_chunk=True,
|
||||
)
|
||||
assert sender["extra"]["role"] == "sender"
|
||||
assert receiver["extra"]["role"] == "receiver"
|
||||
assert sender["extra"]["connector_get_sleep_s"] == 0.01
|
||||
assert sender["extra"]["connector_get_max_wait_first_chunk"] == 3000
|
||||
assert sender["extra"]["connector_get_max_wait"] == 300
|
||||
|
||||
|
||||
class TestLoadAndResolveStageConfigs:
|
||||
def test_load_and_resolve_with_kwargs(self, mocker: MockerFixture):
|
||||
|
||||
@@ -0,0 +1,29 @@
|
||||
import base64
|
||||
import io
|
||||
import runpy
|
||||
import wave
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
def test_append_wav_chunk_extracts_pcm_frames() -> None:
|
||||
example = Path(__file__).parents[3] / "examples" / "online_serving" / "minicpmo" / "streaming_chat_completion.py"
|
||||
append_wav_chunk = runpy.run_path(example)["append_wav_chunk"]
|
||||
encoded_buffer = io.BytesIO()
|
||||
with wave.open(encoded_buffer, "wb") as wav:
|
||||
wav.setnchannels(1)
|
||||
wav.setsampwidth(2)
|
||||
wav.setframerate(24000)
|
||||
wav.writeframes(b"\x01\x02\x03\x04")
|
||||
|
||||
pcm_parts: list[bytes] = []
|
||||
audio_format = append_wav_chunk(
|
||||
base64.b64encode(encoded_buffer.getvalue()).decode(),
|
||||
pcm_parts,
|
||||
)
|
||||
|
||||
assert audio_format == (1, 2, 24000)
|
||||
assert pcm_parts == [b"\x01\x02\x03\x04"]
|
||||
Loaded 100 of 183 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user