From f2c5152df07c0c71f9b08bf9c67dc46309935b5d Mon Sep 17 00:00:00 2001 From: Weiming Liao Date: Tue, 28 Jul 2026 11:44:30 +0800 Subject: [PATCH] [CI][NPU]: Add nightly performance test for NPU - qwen3-omni/wan2.2/qwen-image-edit-2511 (#5339) Signed-off-by: Weiming Liao Co-authored-by: wangyu <53896905+yenuo26@users.noreply.github.com> --- .buildkite/npu/test-npu-nightly.yml | 107 ++++++++++++++---- docker/Dockerfile.npu.ci | 4 + docker/Dockerfile.npu.ci.a3 | 4 + .../perf/scripts/run_diffusion_benchmark.py | 13 ++- .../tests/test_qwen3_omni_async_chunk.json | 3 +- .../tests/test_qwen3_omni_no_async_chunk.json | 3 +- .../test_qwen_image_edit_2511_vllm_omni.json | 3 +- .../perf/tests/test_wan22_i2v_vllm_omni.json | 3 +- 8 files changed, 114 insertions(+), 26 deletions(-) diff --git a/.buildkite/npu/test-npu-nightly.yml b/.buildkite/npu/test-npu-nightly.yml index e05178c52..5aa003241 100644 --- a/.buildkite/npu/test-npu-nightly.yml +++ b/.buildkite/npu/test-npu-nightly.yml @@ -1,25 +1,59 @@ steps: - - group: ":card_index_dividers: Diffusion X2V Model Test" - key: nightly-diffusion-x2v-group + - group: ":card_index_dividers: Omni Model Test" + key: nightly-omni-test-group depends_on: upload-nightly-pipeline if: >- build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test" || - build.pull_request.labels includes "diffusion-x2v-test" + build.pull_request.labels includes "omni-test" steps: - - label: ":full_moon: Diffusion X2V · Function Test" - mirror_hardwares: a2b3_npu_4 + - label: ":full_moon: Omni · Perf Test · No Async Chunk" + key: nightly-omni-performance-no-async-chunk + timeout_in_minutes: 180 + mirror_hardwares: a3_npu_4 env: VLLM_WORKER_MULTIPROC_METHOD: spawn + VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1" HF_TOKEN: "${HF_TOKEN}" commands: - - pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py -k npu_i2v --run-level "full_model" + - export BENCHMARK_DIR=tests/dfx/perf/results + - | + set +e + pytest -s -v tests/dfx/perf/scripts/run_benchmark.py -m npu --test-config-file tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json + EXIT=$$? + buildkite-agent artifact upload "tests/dfx/perf/results/*.json" + exit $$EXIT + - label: ":full_moon: Omni · Perf Test · Async Chunk" + key: nightly-omni-performance-async-chunk + timeout_in_minutes: 180 + mirror_hardwares: a3_npu_4 + env: + VLLM_WORKER_MULTIPROC_METHOD: spawn + VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1" + HF_TOKEN: "${HF_TOKEN}" + commands: + - export BENCHMARK_DIR=tests/dfx/perf/results + - | + set +e + pytest -s -v tests/dfx/perf/scripts/run_benchmark.py -m npu --test-config-file tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json + EXIT=$$? + buildkite-agent artifact upload "tests/dfx/perf/results/*.json" + exit $$EXIT + - label: ":robot: Omni · MiniCPM-o 4.5 Test" + mirror_hardwares: a3_npu_2 + env: + VLLM_WORKER_MULTIPROC_METHOD: spawn + VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1" + HF_TOKEN: "${HF_TOKEN}" + commands: + - pytest -s -v tests/e2e/offline_inference/test_minicpmo_4_5.py tests/e2e/online_serving/test_minicpmo_4_5.py tests/e2e/online_serving/test_minicpmo_4_5_expansion.py -m 'full_model' --run-level 'full_model' - group: ":card_index_dividers: TTS Model Test" key: nightly-tts-test-group depends_on: upload-nightly-pipeline if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test" || build.pull_request.labels includes "tts-test" steps: - label: ":full_moon: TTS · Perf Test" + timeout_in_minutes: 180 mirror_hardwares: a3_npu_2 env: VLLM_WORKER_MULTIPROC_METHOD: spawn @@ -34,25 +68,56 @@ steps: EXIT=$$? buildkite-agent artifact upload "tests/dfx/perf/results/*.json" exit $$EXIT - - - group: ":card_index_dividers: Omni Model Test" - key: nightly-omni-group + - group: ":card_index_dividers: Diffusion X2I(&A&T) Model Test" + key: nightly-diffusion-x2iat-group depends_on: upload-nightly-pipeline - if: build.env("NIGHTLY") == "1" || build.pull_request.labels includes "nightly-test" || build.pull_request.labels includes "omni-test" + if: >- + build.env("NIGHTLY") == "1" || + build.pull_request.labels includes "nightly-test" || + build.pull_request.labels includes "diffusion-x2iat-test" steps: - - label: ":robot: Omni · MiniCPM-o 4.5 Test" - agents: - queue: "ascend-a3" - resource_class: "npu-2" - image: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry/vllm-omni-ci-npu:${BUILDKITE_COMMIT}-a3" - plugins: - - kubernetes: - podSpecPatch: - imagePullSecrets: - - name: swr-secret + - label: ":full_moon: Diffusion X2I(&A&T) · Perf Test · Qwen-Image-Edit-2511" + key: nightly-diffusion-x2iat-performance-qwen-image-edit-2511 + timeout_in_minutes: 180 + mirror_hardwares: a3_npu_8 env: VLLM_WORKER_MULTIPROC_METHOD: spawn VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1" HF_TOKEN: "${HF_TOKEN}" + DIFFUSION_ATTENTION_BACKEND: TORCH_SDPA commands: - - pytest -s -v tests/e2e/offline_inference/test_minicpmo_4_5.py tests/e2e/online_serving/test_minicpmo_4_5.py tests/e2e/online_serving/test_minicpmo_4_5_expansion.py -m 'full_model' --run-level 'full_model' + - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results + - export CACHE_DIT_VERSION=1.3.0 + - | + set +e + pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py -m npu --test-config-file tests/dfx/perf/tests/test_qwen_image_edit_2511_vllm_omni.json + EXIT=$$? + buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json" + buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log" + exit $$EXIT + - group: ":card_index_dividers: Diffusion X2V Model Test" + key: nightly-diffusion-x2v-group + depends_on: upload-nightly-pipeline + if: >- + build.env("NIGHTLY") == "1" || + build.pull_request.labels includes "nightly-test" || + build.pull_request.labels includes "diffusion-x2v-test" + steps: + - label: ":full_moon: Diffusion X2V · Perf Test" + key: nightly-diffusion-x2v-performance + timeout_in_minutes: 180 + mirror_hardwares: a3_npu_4 + env: + VLLM_WORKER_MULTIPROC_METHOD: spawn + VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1" + HF_TOKEN: "${HF_TOKEN}" + DIFFUSION_ATTENTION_BACKEND: TORCH_SDPA + commands: + - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results + - | + set +e + pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py -m npu --test-config-file tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json + EXIT=$$? + buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json" + buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log" + exit $$EXIT diff --git a/docker/Dockerfile.npu.ci b/docker/Dockerfile.npu.ci index c0b4af9b6..10a62fcac 100644 --- a/docker/Dockerfile.npu.ci +++ b/docker/Dockerfile.npu.ci @@ -37,6 +37,10 @@ RUN source /usr/local/Ascend/ascend-toolkit/set_env.sh && \ "pyttsx3>=2.99" \ "opencv-python>=4.12.0.88" \ "imageio[ffmpeg]>=0.6.0" \ + "jiwer>=3.0.0" \ + "zhon>=2.0.0" \ + "zhconv>=1.4.2" \ + "funasr>=1.0.0" \ "hyperpyyaml>=1.2.0" \ "onnx>=1.16.0" \ "librosa>=0.10.0" \ diff --git a/docker/Dockerfile.npu.ci.a3 b/docker/Dockerfile.npu.ci.a3 index b86d7c0f8..5ce526c96 100644 --- a/docker/Dockerfile.npu.ci.a3 +++ b/docker/Dockerfile.npu.ci.a3 @@ -37,6 +37,10 @@ RUN source /usr/local/Ascend/ascend-toolkit/set_env.sh && \ "pyttsx3>=2.99" \ "opencv-python>=4.12.0.88" \ "imageio[ffmpeg]>=0.6.0" \ + "jiwer>=3.0.0" \ + "zhon>=2.0.0" \ + "zhconv>=1.4.2" \ + "funasr>=1.0.0" \ "hyperpyyaml>=1.2.0" \ "onnx>=1.16.0" \ "librosa>=0.10.0" \ diff --git a/tests/dfx/perf/scripts/run_diffusion_benchmark.py b/tests/dfx/perf/scripts/run_diffusion_benchmark.py index d19c0900e..375d2a440 100644 --- a/tests/dfx/perf/scripts/run_diffusion_benchmark.py +++ b/tests/dfx/perf/scripts/run_diffusion_benchmark.py @@ -355,6 +355,17 @@ def _kill_process_tree(pid: int) -> None: # --------------------------------------------------------------------------- +def _resolve_offline_model(model: str) -> str: + """Under HF_HUB_OFFLINE, resolve a HF repo id to its local snapshot dir.""" + import huggingface_hub + + if not model or os.path.isdir(model) or not huggingface_hub.constants.HF_HUB_OFFLINE: + return model + from huggingface_hub import snapshot_download + + return snapshot_download(model, local_files_only=True) + + class DiffusionServer: """Start a vLLM-Omni diffusion model server as a subprocess. @@ -372,7 +383,7 @@ class DiffusionServer: port: int | None = None, ) -> None: self.server_cfg: dict[str, Any] = server_cfg - self.model = server_cfg["model"] + self.model = _resolve_offline_model(server_cfg["model"]) self.serve_args = server_cfg["serve_args"] self.host = "127.0.0.1" self.port = port if port is not None else get_open_port(self.host) diff --git a/tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json b/tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json index 013436d80..be108cbd6 100644 --- a/tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json +++ b/tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json @@ -5,7 +5,8 @@ { "hardware_marks": { "res": { - "cuda": "H100" + "cuda": "H100", + "npu": "A3" }, "num_cards": 2 } diff --git a/tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json b/tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json index 71c90f0e7..3fbeafb9d 100644 --- a/tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json +++ b/tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json @@ -5,7 +5,8 @@ { "hardware_marks": { "res": { - "cuda": "H100" + "cuda": "H100", + "npu": "A3" }, "num_cards": 2 } diff --git a/tests/dfx/perf/tests/test_qwen_image_edit_2511_vllm_omni.json b/tests/dfx/perf/tests/test_qwen_image_edit_2511_vllm_omni.json index 350cfb6bb..75a3d7744 100644 --- a/tests/dfx/perf/tests/test_qwen_image_edit_2511_vllm_omni.json +++ b/tests/dfx/perf/tests/test_qwen_image_edit_2511_vllm_omni.json @@ -66,7 +66,8 @@ { "hardware_marks": { "res": { - "cuda": "H100" + "cuda": "H100", + "npu": "A3" }, "num_cards": 4 } diff --git a/tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json b/tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json index 53c7da9b7..52441dae4 100644 --- a/tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json +++ b/tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json @@ -55,7 +55,8 @@ { "hardware_marks": { "res": { - "cuda": "H100" + "cuda": "H100", + "npu": "A3" }, "num_cards": 2 }