[CI/Build][MiniCPM-o] Fix NPU accuracy job timeout and silent Daily-O… (#5847)

Signed-off-by: psv666 <2693925048@qq.com>
Co-authored-by: amy-why-3459 <wuhaiyan17@huawei.com>

.buildkite/npu/test-npu-nightly.yml

@@ -1,3 +1,9 @@
+env:
+  VLLM_WORKER_MULTIPROC_METHOD: spawn
+  VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
+  HF_HUB_DOWNLOAD_TIMEOUT: 300
+  HF_HUB_ETAG_TIMEOUT: 60
+
 steps:
   # - group: ":card_index_dividers: Diffusion X2V Model Test"
   #   key: nightly-diffusion-x2v-group
@@ -41,13 +47,14 @@ steps:
     if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test" || build.pull_request.labels includes "omni-test"
     steps:
       - label: ":full_moon: Omni · MiniCPM-o 4.5 Test"
+        timeout_in_minutes: 180
         mirror_hardwares: a3_npu_2
         env:
           VLLM_WORKER_MULTIPROC_METHOD: spawn
           VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
           HF_TOKEN: "${HF_TOKEN}"
         commands:
-          - pytest -s -v tests/e2e/offline_inference/test_minicpmo_4_5.py tests/e2e/online_serving/test_minicpmo_4_5.py tests/e2e/online_serving/test_minicpmo_4_5_expansion.py -m 'full_model' --run-level 'full_model'
+          - pytest -s -v tests/e2e/offline_inference/test_minicpmo_4_5.py tests/e2e/online_serving/test_minicpmo_4_5.py tests/e2e/online_serving/test_minicpmo_4_5_expansion.py -m 'full_model and npu and A3 and omni' --run-level 'full_model'
       - label: ":full_moon: Omni · MiniCPM-o 4.5 · Duplex Test"
         timeout_in_minutes: 180
         mirror_hardwares: a3_npu_2
@@ -76,6 +83,8 @@ steps:
           VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
           HF_TOKEN: "${HF_TOKEN}"
         commands:
+          - export SEED_TTS_WER_EVAL=1
+          - export SEED_TTS_EVAL_DEVICE=npu:1
           - |
             set +e
             pytest -s -v tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py -m "full_model and npu and A3" --run-level full_model

docker/Dockerfile.npu.ci

@@ -44,6 +44,7 @@ RUN source /usr/local/Ascend/ascend-toolkit/set_env.sh && \
       "jiwer>=4.0.0" \
       "zhon>=2.1.1" \
       "zhconv>=1.4.3" \
+      "datasets>=3.0.0" \
       --index-url http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple \
       --trusted-host cache-service.nginx-pypi-cache.svc.cluster.local \
       --default-timeout=60 && \

docker/Dockerfile.npu.ci.a3

@@ -44,6 +44,7 @@ RUN source /usr/local/Ascend/ascend-toolkit/set_env.sh && \
       "jiwer>=4.0.0" \
       "zhon>=2.1.1" \
       "zhconv>=1.4.3" \
+      "datasets>=3.0.0" \
       --index-url http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple \
       --trusted-host cache-service.nginx-pypi-cache.svc.cluster.local \
       --default-timeout=60 && \

tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json

@@ -29,7 +29,7 @@
         "backend": "openai-realtime-tts",
         "endpoint": "/v1/realtime",
         "num_prompts": [
-          1
+          32
         ],
         "max_concurrency": [
           1

tests/e2e/accuracy/minicpmo_4_5/test_minicpmo_4_5.py

@@ -13,6 +13,7 @@ Official MiniCPM-o 4.5 reports **70.4**; the gate is **0.68** (~2pp margin).
 
 from __future__ import annotations
 
+import importlib.util
 import json
 import os
 from pathlib import Path
@@ -83,6 +84,15 @@ _SEED_TTS_SERVER_ARGS = [
 
 pytestmark = [pytest.mark.full_model, pytest.mark.omni]
 
+# Evaluated at setup time, before the ``omni_server`` fixture boots a server. An
+# ``importorskip`` inside the test body would skip only after paying several minutes
+# of startup, and the resulting skip is easy to miss in a nightly log.
+_missing_daily_omni_deps = [name for name in ("datasets", "huggingface_hub") if importlib.util.find_spec(name) is None]
+requires_daily_omni_deps = pytest.mark.skipif(
+    bool(_missing_daily_omni_deps),
+    reason=f"Daily-Omni accuracy bench needs {', '.join(_missing_daily_omni_deps)}",
+)
+
 daily_test_params = [
     OmniServerParams(
         model=_MODEL,
@@ -165,12 +175,11 @@ def _inline_local_media_when_needed(monkeypatch: pytest.MonkeyPatch) -> None:
     monkeypatch.setenv("VLLM_WORKER_MULTIPROC_METHOD", "spawn")
 
 
+@requires_daily_omni_deps
 @hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=1)
 @pytest.mark.parametrize("omni_server", daily_test_params, indirect=True)
 def test_minicpmo_4_5_daily_omni_accuracy_bench(omni_server) -> None:
     _require_vllm_cli()
-    pytest.importorskip("datasets")
-    pytest.importorskip("huggingface_hub")
 
     argv = build_acc_benchmark_cli_argv(
         omni_server,

vllm_omni/benchmarks/data_modules/seed_tts_eval.py

@@ -125,11 +125,25 @@ def _get_eval_device() -> str:
         return explicit
     try:
         import torch
-
-        return "cuda:0" if torch.cuda.is_available() else "cpu"
     except ImportError:
         return "cpu"
 
+    if torch.cuda.is_available():
+        return "cuda:0"
+
+    # Ascend: ``torch.npu`` only exists once ``torch_npu`` has been imported, which does
+    # not necessarily happen in the benchmark client process. Without this branch the
+    # eval silently lands on CPU, where Whisper-large-v3 needs hours for a full Seed-TTS
+    # run and the CI job is killed by its timeout instead of reporting a WER.
+    try:
+        import torch_npu  # noqa: F401
+    except ImportError:
+        pass
+    if hasattr(torch, "npu") and torch.npu.is_available():
+        return "npu:0"
+
+    return "cpu"
+
 
 def _punctuation_all() -> str:
     from zhon.hanzi import punctuation