[CI/Build][NPU] Add E2E test on Ascend A3 (#7775)

Signed-off-by: DarkMatter9527 <1376576122@qq.com>
Signed-off-by: DarkMatter <1376576122@qq.com>
Co-authored-by: Weiming Liao <fayespica@outlook.com>
Co-authored-by: wangyu <53896905+yenuo26@users.noreply.github.com>

.buildkite/cuda/test-ready.yml

@@ -211,7 +211,7 @@ steps:
           - |
             timeout 40m bash -c "
               export VLLM_IMAGE_FETCH_TIMEOUT=60
-              pytest -s -v tests/e2e/online_serving/test_wan22_t2v.py -m 'core_model' --run-level 'core_model'
+              pytest -s -v tests/e2e/online_serving/test_wan22_t2v.py -m 'core_model and cuda' --run-level 'core_model'
             "
         mirror_hardwares: h100_1
 

.buildkite/npu/test-npu-ready.yml

@@ -55,8 +55,8 @@ steps:
             -m 'core_model and npu and 310P'
             --run-level 'core_model'
 
-  - group: ":card_index_dividers: Omni Model Test"
-    key: ready-omni-group
+  - group: ":card_index_dividers: E2E Test"
+    key: ready-e2e-group
     depends_on: upload-ready-pipeline
     steps:
       - label: "Omni · MiniCPM-o 4.5 Test"
@@ -66,6 +66,7 @@ steps:
           VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
         commands:
           - pytest -s -v tests/e2e/offline_inference/test_minicpmo_4_5.py tests/e2e/online_serving/test_minicpmo_4_5.py -m 'core_model and npu and A3 and omni' --run-level 'core_model'
+
       - label: "Omni · MiniCPM-o 4.5 Duplex Test"
         mirror_hardwares: a3_npu_2
         env:
@@ -73,6 +74,7 @@ steps:
           VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
         commands:
           - pytest -s -v tests/e2e/online_serving/test_minicpmo_4_5_duplex.py -m 'core_model and npu and A3 and omni' --run-level 'core_model'
+
       - label: "Omni · MiniCPM-o 4.5 Perf Test"
         timeout_in_minutes: 60
         mirror_hardwares: a3_npu_2
@@ -83,10 +85,15 @@ steps:
           - export BENCHMARK_DIR=tests/dfx/perf/results
           - pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_minicpmo_4_5_ready.json -m 'core_model and npu and A3 and omni'
 
-  - group: ":card_index_dividers: TTS Model Test"
-    key: ready-tts-group
-    depends_on: upload-ready-pipeline
-    steps:
+      - label: "Omni · Qwen3-Omni Test"
+        mirror_hardwares: a3_npu_4
+        env:
+          VLLM_WORKER_MULTIPROC_METHOD: spawn
+          VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
+        commands:
+          - pytest -s -v tests/e2e/online_serving/test_qwen3_omni.py -m 'core_model and npu and A3 and omni' --run-level 'core_model'
+        timeout_in_minutes: 60
+
       - label: ":sound: TTS · Qwen3-TTS CustomVoice Test"
         mirror_hardwares: a3_npu_2
         env:
@@ -95,3 +102,55 @@ steps:
           HF_TOKEN: "${HF_TOKEN}"
         commands:
           - pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py -m 'core_model and npu' --run-level 'core_model'
+
+      - label: "Diffusion · HunyuanVideo-1.5 Test"
+        mirror_hardwares: a3_npu_2
+        env:
+            VLLM_WORKER_MULTIPROC_METHOD: spawn
+            VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
+        commands:
+            - |
+              timeout 40m bash -c "
+                export VLLM_IMAGE_FETCH_TIMEOUT=60
+                pytest -s -v tests/e2e/online_serving/test_hunyuan_video_15_expansion.py -m 'core_model and diffusion and npu' --run-level 'core_model'
+                "
+        timeout_in_minutes: 60
+
+      - label: "Diffusion · Qwen Image Test"
+        mirror_hardwares: a3_npu_2
+        env:
+            VLLM_WORKER_MULTIPROC_METHOD: spawn
+            VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
+        commands:
+            - |
+                timeout 40m bash -c "
+                export VLLM_IMAGE_FETCH_TIMEOUT=60
+                pytest -s -v tests/e2e/online_serving/test_qwen_image.py -m 'core_model and npu' --run-level 'core_model'
+                "
+        timeout_in_minutes: 60
+
+      - label: "Diffusion · Wan22 Test"
+        mirror_hardwares: a3_npu_2
+        env:
+            VLLM_WORKER_MULTIPROC_METHOD: spawn
+            VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
+        commands:
+            - |
+                timeout 40m bash -c "
+                export VLLM_IMAGE_FETCH_TIMEOUT=60
+                pytest -s -v tests/e2e/online_serving/test_wan22_t2v.py -m 'core_model and npu' --run-level 'core_model'
+                "
+        timeout_in_minutes: 60
+
+      - label: "Diffusion · Hunyuan Image Test"
+        mirror_hardwares: a3_npu_4
+        env:
+            VLLM_WORKER_MULTIPROC_METHOD: spawn
+            VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
+        commands:
+            - |
+                timeout 40m bash -c "
+                export VLLM_IMAGE_FETCH_TIMEOUT=60
+                pytest -s -v tests/e2e/online_serving/test_hunyuan_image.py -m 'core_model and diffusion and npu' --run-level 'core_model'
+                "
+        timeout_in_minutes: 60

tests/e2e/online_serving/test_hunyuan_image.py

@@ -0,0 +1,80 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+
+"""
+Online serving smoke for ``tencent/HunyuanImage-3.0-Instruct`` (text-to-image via
+``/v1/images/generations``, DiT-only deploy).
+
+Uses ``hunyuan_image3_dit.yaml`` (single diffusion stage, TP=4 across 4 devices,
+no AR / Mooncake connector) so this stays a cheap ready-path smoke. The DiT is
+sharded across 4 cards on both CUDA and NPU (see the ``platforms.npu`` block in the
+deploy yaml); heavier multi-config / parallel coverage lives in the perf
+(``tests/dfx/perf/tests/test_hunyuan_image_tp*.json``) and reliability
+(``tests/dfx/reliability/test_reliability_hunyuan_image.py``) suites.
+
+From ``tests/``::
+
+    pytest -s -v e2e/online_serving/test_hunyuan_image.py -m "core_model and diffusion" --run-level=core_model
+"""
+
+import os
+
+import pytest
+
+from tests.helpers.mark import hardware_marks
+from tests.helpers.runtime import OmniServer, OmniServerParams, OnlineOmniClient
+from tests.helpers.stage_config import get_deploy_config_path
+
+os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
+
+MODEL = "tencent/HunyuanImage-3.0-Instruct"
+PROMPT = "A simple red apple on a white background."
+
+# HunyuanImage-3 DiT-only deploy shards the DiT across 4 devices (TP=4) on both
+# CUDA and NPU (see hunyuan_image3_dit.yaml). This mirrors the hardware marks used
+# by the perf suite (test_hunyuan_image_tp4.json).
+FEATURE_MARKS = hardware_marks(res={"cuda": "H100", "npu": "A3"}, num_cards=4)
+
+
+def _get_diffusion_feature_cases(model: str):
+    """Return a single default server row using the DiT-only deploy config."""
+    return [
+        pytest.param(
+            OmniServerParams(
+                model=model,
+                stage_config_path=get_deploy_config_path("hunyuan_image3_dit.yaml"),
+            ),
+            id="default",
+            marks=FEATURE_MARKS,
+        ),
+    ]
+
+
+@pytest.mark.core_model
+@pytest.mark.diffusion
+@pytest.mark.parametrize("omni_server", _get_diffusion_feature_cases(MODEL), indirect=True)
+def test_text_to_image_001(omni_server: OmniServer, online_client: OnlineOmniClient) -> None:
+    """Default HunyuanImage-3.0-Instruct T2I smoke through ``/v1/images/generations``."""
+    responses = online_client.send_images_generations_http_request(
+        {
+            "json": {
+                "model": omni_server.model,
+                "prompt": PROMPT,
+                "size": "512x512",
+                "n": 1,
+                "response_format": "b64_json",
+                "num_inference_steps": 2,
+                "guidance_scale": 7.5,
+                "seed": 42,
+                # HunyuanImage-3 prompt-routing knobs (see reliability suite).
+                "bot_task": "none",
+                "use_system_prompt": "en_unified",
+            }
+        }
+    )
+    response = responses[0]
+    assert response.success, response.error_message
+    payload = response.json_body
+    assert isinstance(payload, dict)
+    assert len(payload["data"]) == 1
+    assert payload["data"][0]["b64_json"]

tests/e2e/online_serving/test_qwen3_omni.py

@@ -94,7 +94,14 @@ def get_max_batch_size(size_type="few"):
 @pytest.mark.core_model
 @pytest.mark.omni
 @pytest.mark.skipif(_USE_PD, reason="Temporarily skip PD mode in this test module.")
-@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=3 if _USE_PD else 2)
+@hardware_test(
+    res={"cuda": "H100", "rocm": "MI325", "npu": "A3"},
+    num_cards={
+        "cuda": 3 if _USE_PD else 2,
+        "rocm": 3 if _USE_PD else 2,
+        "npu": 3,
+    },
+)
 @pytest.mark.parametrize("omni_server", test_params, indirect=True)
 def test_mix_to_text_audio_001(omni_server, online_client) -> None:
     """
@@ -134,7 +141,14 @@ def test_mix_to_text_audio_001(omni_server, online_client) -> None:
 @pytest.mark.core_model
 @pytest.mark.omni
 @pytest.mark.skipif(_USE_PD, reason="Temporarily skip PD mode in this test module.")
-@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=3 if _USE_PD else 2)
+@hardware_test(
+    res={"cuda": "H100", "rocm": "MI325", "npu": "A3"},
+    num_cards={
+        "cuda": 3 if _USE_PD else 2,
+        "rocm": 3 if _USE_PD else 2,
+        "npu": 3,
+    },
+)
 @pytest.mark.parametrize("omni_server", test_params, indirect=True)
 def test_text_to_text_001(omni_server, online_client) -> None:
     """
@@ -336,7 +350,10 @@ def test_thinker_prefix_caching_async_chunk_shared_image_prefix(omni_server, onl
 @pytest.mark.advanced_model
 @pytest.mark.core_model
 @pytest.mark.omni
-@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
+@hardware_test(
+    res={"cuda": "H100", "rocm": "MI325", "npu": "A3"},
+    num_cards={"cuda": 2, "rocm": 2, "npu": 3},
+)
 @pytest.mark.parametrize("omni_server", test_params, indirect=True)
 def test_completions_rejected_for_thinker_talker(omni_server, online_client) -> None:
     """Ensure Thinker-talker models reject /v1/completions; we do this because the
@@ -360,7 +377,10 @@ def test_completions_rejected_for_thinker_talker(omni_server, online_client) ->
 @pytest.mark.advanced_model
 @pytest.mark.core_model
 @pytest.mark.omni
-@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
+@hardware_test(
+    res={"cuda": "H100", "rocm": "MI325", "npu": "A3"},
+    num_cards={"cuda": 2, "rocm": 2, "npu": 3},
+)
 @pytest.mark.parametrize("omni_server", test_params, indirect=True)
 def test_batched_completions_text(omni_server, openai_client) -> None:
     """Ensure that we can make a batch chat completions request (text only)."""
@@ -390,7 +410,10 @@ def test_batched_completions_text(omni_server, openai_client) -> None:
 @pytest.mark.advanced_model
 @pytest.mark.core_model
 @pytest.mark.omni
-@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
+@hardware_test(
+    res={"cuda": "H100", "rocm": "MI325", "npu": "A3"},
+    num_cards={"cuda": 2, "rocm": 2, "npu": 3},
+)
 @pytest.mark.parametrize("omni_server", test_params, indirect=True)
 def test_batched_completions_audio_out(omni_server, openai_client) -> None:
     """Ensure that we can make a batch chat completions request (audio + text)."""
@@ -425,7 +448,10 @@ def test_batched_completions_audio_out(omni_server, openai_client) -> None:
 @pytest.mark.advanced_model
 @pytest.mark.core_model
 @pytest.mark.omni
-@hardware_test(res={"cuda": "H100", "rocm": "MI325"}, num_cards=2)
+@hardware_test(
+    res={"cuda": "H100", "rocm": "MI325", "npu": "A3"},
+    num_cards={"cuda": 2, "rocm": 2, "npu": 3},
+)
 @pytest.mark.parametrize("omni_server", test_params, indirect=True)
 @pytest.mark.parametrize(
     "sad_opts",

tests/e2e/online_serving/test_qwen_image.py

@@ -32,8 +32,8 @@ MODEL = "Qwen/Qwen-Image"
 T2I_PROMPT = "A photo of a cat sitting on a laptop keyboard, digital art style."
 NEGATIVE_PROMPT = "blurry, low quality"
 
-SINGLE_CARD_FEATURE_MARKS = hardware_marks(res={"cuda": "H100"})
-PARALLEL_FEATURE_MARKS = hardware_marks(res={"cuda": "H100"}, num_cards=2)
+SINGLE_CARD_FEATURE_MARKS = hardware_marks(res={"cuda": "H100", "npu": "A3"})
+PARALLEL_FEATURE_MARKS = hardware_marks(res={"cuda": "H100", "npu": "A3"}, num_cards=2)
 
 TEST_PROMPTS: list[dict[str, str]] = [
     {"prompt": "a cup of coffee on a table", "negative_prompt": "low resolution"},

tests/e2e/online_serving/test_wan22_t2v.py

@@ -20,6 +20,7 @@ import pytest
 
 from tests.helpers.mark import hardware_marks
 from tests.helpers.runtime import OmniServer, OmniServerParams, OnlineOmniClient
+from vllm_omni.platforms import current_omni_platform
 
 os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
 
@@ -27,16 +28,44 @@ MODEL = "Wan-AI/Wan2.2-T2V-A14B-Diffusers"
 PROMPT = "Two anthropomorphic cats in boxing gear on a spotlighted stage."
 NEGATIVE_PROMPT = "low quality, blurry, watermark, text"
 
-SINGLE_CARD_FEATURE_MARKS = hardware_marks(res={"cuda": "H100"})
+# CUDA / ROCm: single card, no extra server_args — behavior unchanged.
+# Skip on NPU, where a single A3 (64 GB HBM) cannot hold Wan2.2-T2V-A14B.
+CUDA_SINGLE_CARD_FEATURE_MARKS = [
+    *hardware_marks(res={"cuda": "H100"}, num_cards=1),
+    pytest.mark.skipif(
+        current_omni_platform.is_npu(),
+        reason="CUDA/ROCm single-card path; skip on NPU",
+    ),
+]
+
+# NPU: TP=2 across two A3 cards.
+# Skip on any non-NPU platform.
+NPU_TP2_FEATURE_MARKS = [
+    *hardware_marks(res={"npu": "A3"}, num_cards=2),
+    pytest.mark.skipif(
+        not current_omni_platform.is_npu(),
+        reason="Requires Ascend NPU platform",
+    ),
+]
 
 
 def _get_diffusion_feature_cases(model: str):
-    """Return a single default ``OmniServerParams`` row (no extra ``server_args``)."""
+    """Return one param per platform with explicit platform skip conditions."""
     return [
+        # CUDA: single card, no extra server_args
         pytest.param(
             OmniServerParams(model=model, init_timeout=1800, stage_init_timeout=1800, startup_timeout=2100),
             id="default",
-            marks=SINGLE_CARD_FEATURE_MARKS,
+            marks=CUDA_SINGLE_CARD_FEATURE_MARKS,
+        ),
+        # NPU: TP=2 across two cards
+        pytest.param(
+            OmniServerParams(
+                model=model,
+                server_args=["--tensor-parallel-size", "2"],
+            ),
+            id="default",
+            marks=NPU_TP2_FEATURE_MARKS,
         ),
     ]