diff --git a/.buildkite/npu/test-npu-nightly.yml b/.buildkite/npu/test-npu-nightly.yml index b06201146..34daa08a9 100644 --- a/.buildkite/npu/test-npu-nightly.yml +++ b/.buildkite/npu/test-npu-nightly.yml @@ -1,3 +1,9 @@ +env: + VLLM_WORKER_MULTIPROC_METHOD: spawn + VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1" + HF_HUB_DOWNLOAD_TIMEOUT: 300 + HF_HUB_ETAG_TIMEOUT: 60 + steps: # - group: ":card_index_dividers: Diffusion X2V Model Test" # key: nightly-diffusion-x2v-group @@ -41,13 +47,14 @@ steps: if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test" || build.pull_request.labels includes "omni-test" steps: - label: ":full_moon: Omni · MiniCPM-o 4.5 Test" + timeout_in_minutes: 180 mirror_hardwares: a3_npu_2 env: VLLM_WORKER_MULTIPROC_METHOD: spawn VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1" HF_TOKEN: "${HF_TOKEN}" commands: - - pytest -s -v tests/e2e/offline_inference/test_minicpmo_4_5.py tests/e2e/online_serving/test_minicpmo_4_5.py tests/e2e/online_serving/test_minicpmo_4_5_expansion.py -m 'full_model' --run-level 'full_model' + - pytest -s -v tests/e2e/offline_inference/test_minicpmo_4_5.py tests/e2e/online_serving/test_minicpmo_4_5.py tests/e2e/online_serving/test_minicpmo_4_5_expansion.py -m 'full_model and npu and A3 and omni' --run-level 'full_model' - label: ":full_moon: Omni · MiniCPM-o 4.5 · Duplex Test" timeout_in_minutes: 180 mirror_hardwares: a3_npu_2 @@ -76,6 +83,8 @@ steps: VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1" HF_TOKEN: "${HF_TOKEN}" commands: + - export SEED_TTS_WER_EVAL=1 + - export SEED_TTS_EVAL_DEVICE=npu:1 - | set +e pytest -s -v tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py -m "full_model and npu and A3" --run-level full_model diff --git a/docker/Dockerfile.npu.ci b/docker/Dockerfile.npu.ci index b5aff4174..40cac6ee3 100644 --- a/docker/Dockerfile.npu.ci +++ b/docker/Dockerfile.npu.ci @@ -44,6 +44,7 @@ RUN source /usr/local/Ascend/ascend-toolkit/set_env.sh && \ "jiwer>=4.0.0" \ "zhon>=2.1.1" \ "zhconv>=1.4.3" \ + "datasets>=3.0.0" \ --index-url http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple \ --trusted-host cache-service.nginx-pypi-cache.svc.cluster.local \ --default-timeout=60 && \ diff --git a/docker/Dockerfile.npu.ci.a3 b/docker/Dockerfile.npu.ci.a3 index d50e1d163..4682643c1 100644 --- a/docker/Dockerfile.npu.ci.a3 +++ b/docker/Dockerfile.npu.ci.a3 @@ -44,6 +44,7 @@ RUN source /usr/local/Ascend/ascend-toolkit/set_env.sh && \ "jiwer>=4.0.0" \ "zhon>=2.1.1" \ "zhconv>=1.4.3" \ + "datasets>=3.0.0" \ --index-url http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple \ --trusted-host cache-service.nginx-pypi-cache.svc.cluster.local \ --default-timeout=60 && \ diff --git a/tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json b/tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json index ceaa40159..ec38cd545 100644 --- a/tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json +++ b/tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json @@ -29,7 +29,7 @@ "backend": "openai-realtime-tts", "endpoint": "/v1/realtime", "num_prompts": [ - 1 + 32 ], "max_concurrency": [ 1 diff --git a/tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py b/tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py index ef939b9be..a1416afce 100644 --- a/tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py +++ b/tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py @@ -13,6 +13,7 @@ Official MiniCPM-o 4.5 reports **70.4**; the gate is **0.68** (~2pp margin). from __future__ import annotations +import importlib.util import json import os from pathlib import Path @@ -83,6 +84,15 @@ _SEED_TTS_SERVER_ARGS = [ pytestmark = [pytest.mark.full_model, pytest.mark.omni] +# Evaluated at setup time, before the ``omni_server`` fixture boots a server. An +# ``importorskip`` inside the test body would skip only after paying several minutes +# of startup, and the resulting skip is easy to miss in a nightly log. +_missing_daily_omni_deps = [name for name in ("datasets", "huggingface_hub") if importlib.util.find_spec(name) is None] +requires_daily_omni_deps = pytest.mark.skipif( + bool(_missing_daily_omni_deps), + reason=f"Daily-Omni accuracy bench needs {', '.join(_missing_daily_omni_deps)}", +) + daily_test_params = [ OmniServerParams( model=_MODEL, @@ -165,12 +175,11 @@ def _inline_local_media_when_needed(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setenv("VLLM_WORKER_MULTIPROC_METHOD", "spawn") +@requires_daily_omni_deps @hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1) @pytest.mark.parametrize("omni_server", daily_test_params, indirect=True) def test_minicpmo_4_5_daily_omni_accuracy_bench(omni_server) -> None: _require_vllm_cli() - pytest.importorskip("datasets") - pytest.importorskip("huggingface_hub") argv = build_acc_benchmark_cli_argv( omni_server, diff --git a/vllm_omni/benchmarks/data_modules/seed_tts_eval.py b/vllm_omni/benchmarks/data_modules/seed_tts_eval.py index b634a78b6..ddfbbd630 100644 --- a/vllm_omni/benchmarks/data_modules/seed_tts_eval.py +++ b/vllm_omni/benchmarks/data_modules/seed_tts_eval.py @@ -125,11 +125,25 @@ def _get_eval_device() -> str: return explicit try: import torch - - return "cuda:0" if torch.cuda.is_available() else "cpu" except ImportError: return "cpu" + if torch.cuda.is_available(): + return "cuda:0" + + # Ascend: ``torch.npu`` only exists once ``torch_npu`` has been imported, which does + # not necessarily happen in the benchmark client process. Without this branch the + # eval silently lands on CPU, where Whisper-large-v3 needs hours for a full Seed-TTS + # run and the CI job is killed by its timeout instead of reporting a WER. + try: + import torch_npu # noqa: F401 + except ImportError: + pass + if hasattr(torch, "npu") and torch.npu.is_available(): + return "npu:0" + + return "cpu" + def _punctuation_all() -> str: from zhon.hanzi import punctuation