From 98e123f3e0531c5c2d1a74ad1791831ad3ca6e02 Mon Sep 17 00:00:00 2001 From: DarkMatter <1376576122@qq.com> Date: Thu, 24 Sep 2026 14:10:04 +0800 Subject: [PATCH] [CI/Build][NPU] Add E2E test on Ascend A3 (#7775) Signed-off-by: DarkMatter9527 <1376576122@qq.com> Signed-off-by: DarkMatter <1376576122@qq.com> Co-authored-by: Weiming Liao Co-authored-by: wangyu <53896905+yenuo26@users.noreply.github.com> --- .buildkite/cuda/test-ready.yml | 2 +- .buildkite/npu/test-npu-ready.yml | 71 ++++++++++++++-- .../e2e/online_serving/test_hunyuan_image.py | 80 +++++++++++++++++++ tests/e2e/online_serving/test_qwen3_omni.py | 38 +++++++-- tests/e2e/online_serving/test_qwen_image.py | 4 +- tests/e2e/online_serving/test_wan22_t2v.py | 35 +++++++- 6 files changed, 212 insertions(+), 18 deletions(-) create mode 100644 tests/e2e/online_serving/test_hunyuan_image.py diff --git a/.buildkite/cuda/test-ready.yml b/.buildkite/cuda/test-ready.yml index c474e64a2..633abb9d3 100644 --- a/.buildkite/cuda/test-ready.yml +++ b/.buildkite/cuda/test-ready.yml @@ -211,7 +211,7 @@ steps: - | timeout 40m bash -c " export VLLM_IMAGE_FETCH_TIMEOUT=60 - pytest -s -v tests/e2e/online_serving/test_wan22_t2v.py -m 'core_model' --run-level 'core_model' + pytest -s -v tests/e2e/online_serving/test_wan22_t2v.py -m 'core_model and cuda' --run-level 'core_model' " mirror_hardwares: h100_1 diff --git a/.buildkite/npu/test-npu-ready.yml b/.buildkite/npu/test-npu-ready.yml index d475aad5b..d0875682d 100644 --- a/.buildkite/npu/test-npu-ready.yml +++ b/.buildkite/npu/test-npu-ready.yml @@ -55,8 +55,8 @@ steps: -m 'core_model and npu and 310P' --run-level 'core_model' - - group: ":card_index_dividers: Omni Model Test" - key: ready-omni-group + - group: ":card_index_dividers: E2E Test" + key: ready-e2e-group depends_on: upload-ready-pipeline steps: - label: "Omni · MiniCPM-o 4.5 Test" @@ -66,6 +66,7 @@ steps: VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1" commands: - pytest -s -v tests/e2e/offline_inference/test_minicpmo_4_5.py tests/e2e/online_serving/test_minicpmo_4_5.py -m 'core_model and npu and A3 and omni' --run-level 'core_model' + - label: "Omni · MiniCPM-o 4.5 Duplex Test" mirror_hardwares: a3_npu_2 env: @@ -73,6 +74,7 @@ steps: VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1" commands: - pytest -s -v tests/e2e/online_serving/test_minicpmo_4_5_duplex.py -m 'core_model and npu and A3 and omni' --run-level 'core_model' + - label: "Omni · MiniCPM-o 4.5 Perf Test" timeout_in_minutes: 60 mirror_hardwares: a3_npu_2 @@ -83,10 +85,15 @@ steps: - export BENCHMARK_DIR=tests/dfx/perf/results - pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_minicpmo_4_5_ready.json -m 'core_model and npu and A3 and omni' - - group: ":card_index_dividers: TTS Model Test" - key: ready-tts-group - depends_on: upload-ready-pipeline - steps: + - label: "Omni · Qwen3-Omni Test" + mirror_hardwares: a3_npu_4 + env: + VLLM_WORKER_MULTIPROC_METHOD: spawn + VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1" + commands: + - pytest -s -v tests/e2e/online_serving/test_qwen3_omni.py -m 'core_model and npu and A3 and omni' --run-level 'core_model' + timeout_in_minutes: 60 + - label: ":sound: TTS · Qwen3-TTS CustomVoice Test" mirror_hardwares: a3_npu_2 env: @@ -95,3 +102,55 @@ steps: HF_TOKEN: "${HF_TOKEN}" commands: - pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py -m 'core_model and npu' --run-level 'core_model' + + - label: "Diffusion · HunyuanVideo-1.5 Test" + mirror_hardwares: a3_npu_2 + env: + VLLM_WORKER_MULTIPROC_METHOD: spawn + VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1" + commands: + - | + timeout 40m bash -c " + export VLLM_IMAGE_FETCH_TIMEOUT=60 + pytest -s -v tests/e2e/online_serving/test_hunyuan_video_15_expansion.py -m 'core_model and diffusion and npu' --run-level 'core_model' + " + timeout_in_minutes: 60 + + - label: "Diffusion · Qwen Image Test" + mirror_hardwares: a3_npu_2 + env: + VLLM_WORKER_MULTIPROC_METHOD: spawn + VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1" + commands: + - | + timeout 40m bash -c " + export VLLM_IMAGE_FETCH_TIMEOUT=60 + pytest -s -v tests/e2e/online_serving/test_qwen_image.py -m 'core_model and npu' --run-level 'core_model' + " + timeout_in_minutes: 60 + + - label: "Diffusion · Wan22 Test" + mirror_hardwares: a3_npu_2 + env: + VLLM_WORKER_MULTIPROC_METHOD: spawn + VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1" + commands: + - | + timeout 40m bash -c " + export VLLM_IMAGE_FETCH_TIMEOUT=60 + pytest -s -v tests/e2e/online_serving/test_wan22_t2v.py -m 'core_model and npu' --run-level 'core_model' + " + timeout_in_minutes: 60 + + - label: "Diffusion · Hunyuan Image Test" + mirror_hardwares: a3_npu_4 + env: + VLLM_WORKER_MULTIPROC_METHOD: spawn + VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1" + commands: + - | + timeout 40m bash -c " + export VLLM_IMAGE_FETCH_TIMEOUT=60 + pytest -s -v tests/e2e/online_serving/test_hunyuan_image.py -m 'core_model and diffusion and npu' --run-level 'core_model' + " + timeout_in_minutes: 60 diff --git a/tests/e2e/online_serving/test_hunyuan_image.py b/tests/e2e/online_serving/test_hunyuan_image.py new file mode 100644 index 000000000..91cd90f33 --- /dev/null +++ b/tests/e2e/online_serving/test_hunyuan_image.py @@ -0,0 +1,80 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + +""" +Online serving smoke for ``tencent/HunyuanImage-3.0-Instruct`` (text-to-image via +``/v1/images/generations``, DiT-only deploy). + +Uses ``hunyuan_image3_dit.yaml`` (single diffusion stage, TP=4 across 4 devices, +no AR / Mooncake connector) so this stays a cheap ready-path smoke. The DiT is +sharded across 4 cards on both CUDA and NPU (see the ``platforms.npu`` block in the +deploy yaml); heavier multi-config / parallel coverage lives in the perf +(``tests/dfx/perf/tests/test_hunyuan_image_tp*.json``) and reliability +(``tests/dfx/reliability/test_reliability_hunyuan_image.py``) suites. + +From ``tests/``:: + + pytest -s -v e2e/online_serving/test_hunyuan_image.py -m "core_model and diffusion" --run-level=core_model +""" + +import os + +import pytest + +from tests.helpers.mark import hardware_marks +from tests.helpers.runtime import OmniServer, OmniServerParams, OnlineOmniClient +from tests.helpers.stage_config import get_deploy_config_path + +os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" + +MODEL = "tencent/HunyuanImage-3.0-Instruct" +PROMPT = "A simple red apple on a white background." + +# HunyuanImage-3 DiT-only deploy shards the DiT across 4 devices (TP=4) on both +# CUDA and NPU (see hunyuan_image3_dit.yaml). This mirrors the hardware marks used +# by the perf suite (test_hunyuan_image_tp4.json). +FEATURE_MARKS = hardware_marks(res={"cuda": "H100", "npu": "A3"}, num_cards=4) + + +def _get_diffusion_feature_cases(model: str): + """Return a single default server row using the DiT-only deploy config.""" + return [ + pytest.param( + OmniServerParams( + model=model, + stage_config_path=get_deploy_config_path("hunyuan_image3_dit.yaml"), + ), + id="default", + marks=FEATURE_MARKS, + ), + ] + + +@pytest.mark.core_model +@pytest.mark.diffusion +@pytest.mark.parametrize("omni_server", _get_diffusion_feature_cases(MODEL), indirect=True) +def test_text_to_image_001(omni_server: OmniServer, online_client: OnlineOmniClient) -> None: + """Default HunyuanImage-3.0-Instruct T2I smoke through ``/v1/images/generations``.""" + responses = online_client.send_images_generations_http_request( + { + "json": { + "model": omni_server.model, + "prompt": PROMPT, + "size": "512x512", + "n": 1, + "response_format": "b64_json", + "num_inference_steps": 2, + "guidance_scale": 7.5, + "seed": 42, + # HunyuanImage-3 prompt-routing knobs (see reliability suite). + "bot_task": "none", + "use_system_prompt": "en_unified", + } + } + ) + response = responses[0] + assert response.success, response.error_message + payload = response.json_body + assert isinstance(payload, dict) + assert len(payload["data"]) == 1 + assert payload["data"][0]["b64_json"] diff --git a/tests/e2e/online_serving/test_qwen3_omni.py b/tests/e2e/online_serving/test_qwen3_omni.py index 5b4bbc51e..34aa68717 100644 --- a/tests/e2e/online_serving/test_qwen3_omni.py +++ b/tests/e2e/online_serving/test_qwen3_omni.py @@ -94,7 +94,14 @@ def get_max_batch_size(size_type="few"): @pytest.mark.core_model @pytest.mark.omni @pytest.mark.skipif(_USE_PD, reason="Temporarily skip PD mode in this test module.") -@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=3 if _USE_PD else 2) +@hardware_test( + res={"cuda": "H100", "rocm": "MI325", "npu": "A3"}, + num_cards={ + "cuda": 3 if _USE_PD else 2, + "rocm": 3 if _USE_PD else 2, + "npu": 3, + }, +) @pytest.mark.parametrize("omni_server", test_params, indirect=True) def test_mix_to_text_audio_001(omni_server, online_client) -> None: """ @@ -134,7 +141,14 @@ def test_mix_to_text_audio_001(omni_server, online_client) -> None: @pytest.mark.core_model @pytest.mark.omni @pytest.mark.skipif(_USE_PD, reason="Temporarily skip PD mode in this test module.") -@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=3 if _USE_PD else 2) +@hardware_test( + res={"cuda": "H100", "rocm": "MI325", "npu": "A3"}, + num_cards={ + "cuda": 3 if _USE_PD else 2, + "rocm": 3 if _USE_PD else 2, + "npu": 3, + }, +) @pytest.mark.parametrize("omni_server", test_params, indirect=True) def test_text_to_text_001(omni_server, online_client) -> None: """ @@ -336,7 +350,10 @@ def test_thinker_prefix_caching_async_chunk_shared_image_prefix(omni_server, onl @pytest.mark.advanced_model @pytest.mark.core_model @pytest.mark.omni -@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2) +@hardware_test( + res={"cuda": "H100", "rocm": "MI325", "npu": "A3"}, + num_cards={"cuda": 2, "rocm": 2, "npu": 3}, +) @pytest.mark.parametrize("omni_server", test_params, indirect=True) def test_completions_rejected_for_thinker_talker(omni_server, online_client) -> None: """Ensure Thinker-talker models reject /v1/completions; we do this because the @@ -360,7 +377,10 @@ def test_completions_rejected_for_thinker_talker(omni_server, online_client) -> @pytest.mark.advanced_model @pytest.mark.core_model @pytest.mark.omni -@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2) +@hardware_test( + res={"cuda": "H100", "rocm": "MI325", "npu": "A3"}, + num_cards={"cuda": 2, "rocm": 2, "npu": 3}, +) @pytest.mark.parametrize("omni_server", test_params, indirect=True) def test_batched_completions_text(omni_server, openai_client) -> None: """Ensure that we can make a batch chat completions request (text only).""" @@ -390,7 +410,10 @@ def test_batched_completions_text(omni_server, openai_client) -> None: @pytest.mark.advanced_model @pytest.mark.core_model @pytest.mark.omni -@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2) +@hardware_test( + res={"cuda": "H100", "rocm": "MI325", "npu": "A3"}, + num_cards={"cuda": 2, "rocm": 2, "npu": 3}, +) @pytest.mark.parametrize("omni_server", test_params, indirect=True) def test_batched_completions_audio_out(omni_server, openai_client) -> None: """Ensure that we can make a batch chat completions request (audio + text).""" @@ -425,7 +448,10 @@ def test_batched_completions_audio_out(omni_server, openai_client) -> None: @pytest.mark.advanced_model @pytest.mark.core_model @pytest.mark.omni -@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2) +@hardware_test( + res={"cuda": "H100", "rocm": "MI325", "npu": "A3"}, + num_cards={"cuda": 2, "rocm": 2, "npu": 3}, +) @pytest.mark.parametrize("omni_server", test_params, indirect=True) @pytest.mark.parametrize( "sad_opts", diff --git a/tests/e2e/online_serving/test_qwen_image.py b/tests/e2e/online_serving/test_qwen_image.py index aa037d9b8..37ecc780d 100644 --- a/tests/e2e/online_serving/test_qwen_image.py +++ b/tests/e2e/online_serving/test_qwen_image.py @@ -32,8 +32,8 @@ MODEL = "Qwen/Qwen-Image" T2I_PROMPT = "A photo of a cat sitting on a laptop keyboard, digital art style." NEGATIVE_PROMPT = "blurry, low quality" -SINGLE_CARD_FEATURE_MARKS = hardware_marks(res={"cuda": "H100"}) -PARALLEL_FEATURE_MARKS = hardware_marks(res={"cuda": "H100"}, num_cards=2) +SINGLE_CARD_FEATURE_MARKS = hardware_marks(res={"cuda": "H100", "npu": "A3"}) +PARALLEL_FEATURE_MARKS = hardware_marks(res={"cuda": "H100", "npu": "A3"}, num_cards=2) TEST_PROMPTS: list[dict[str, str]] = [ {"prompt": "a cup of coffee on a table", "negative_prompt": "low resolution"}, diff --git a/tests/e2e/online_serving/test_wan22_t2v.py b/tests/e2e/online_serving/test_wan22_t2v.py index d254a9de7..ddf893e06 100644 --- a/tests/e2e/online_serving/test_wan22_t2v.py +++ b/tests/e2e/online_serving/test_wan22_t2v.py @@ -20,6 +20,7 @@ import pytest from tests.helpers.mark import hardware_marks from tests.helpers.runtime import OmniServer, OmniServerParams, OnlineOmniClient +from vllm_omni.platforms import current_omni_platform os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" @@ -27,16 +28,44 @@ MODEL = "Wan-AI/Wan2.2-T2V-A14B-Diffusers" PROMPT = "Two anthropomorphic cats in boxing gear on a spotlighted stage." NEGATIVE_PROMPT = "low quality, blurry, watermark, text" -SINGLE_CARD_FEATURE_MARKS = hardware_marks(res={"cuda": "H100"}) +# CUDA / ROCm: single card, no extra server_args — behavior unchanged. +# Skip on NPU, where a single A3 (64 GB HBM) cannot hold Wan2.2-T2V-A14B. +CUDA_SINGLE_CARD_FEATURE_MARKS = [ + *hardware_marks(res={"cuda": "H100"}, num_cards=1), + pytest.mark.skipif( + current_omni_platform.is_npu(), + reason="CUDA/ROCm single-card path; skip on NPU", + ), +] + +# NPU: TP=2 across two A3 cards. +# Skip on any non-NPU platform. +NPU_TP2_FEATURE_MARKS = [ + *hardware_marks(res={"npu": "A3"}, num_cards=2), + pytest.mark.skipif( + not current_omni_platform.is_npu(), + reason="Requires Ascend NPU platform", + ), +] def _get_diffusion_feature_cases(model: str): - """Return a single default ``OmniServerParams`` row (no extra ``server_args``).""" + """Return one param per platform with explicit platform skip conditions.""" return [ + # CUDA: single card, no extra server_args pytest.param( OmniServerParams(model=model, init_timeout=1800, stage_init_timeout=1800, startup_timeout=2100), id="default", - marks=SINGLE_CARD_FEATURE_MARKS, + marks=CUDA_SINGLE_CARD_FEATURE_MARKS, + ), + # NPU: TP=2 across two cards + pytest.param( + OmniServerParams( + model=model, + server_args=["--tensor-parallel-size", "2"], + ), + id="default", + marks=NPU_TP2_FEATURE_MARKS, ), ]