Adapt HunyuanImage3.0 test for npu (#5436)

Signed-off-by: zzh <943967662@qq.com>
2026-08-03 06:31:50 +00:00
1 parent 24741961eb
commit e0c2b9d73f
@@ -95,6 +95,80 @@ steps:
            buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
            buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
            exit $$EXIT
+  - group: ":card_index_dividers: Diffusion I2I Model Test"
+    key: nightly-HunyuanImage3-x2i-test-group
+    depends_on: upload-nightly-pipeline
+    if: >-
+      build.env("NIGHTLY") == "1" ||
+      build.pull_request.labels includes "nightly-test" ||
+      build.pull_request.labels includes "diffusion-x2iat-test"
+    steps:
+      - label: ":full_moon: HunyuanImage3-DIT · Accuracy Test"
+        timeout_in_minutes: 180
+        mirror_hardwares: a3_npu_4
+        env:
+          VLLM_WORKER_MULTIPROC_METHOD: spawn
+          HUNYUAN_IMAGE3_MODEL: "tencent/HunyuanImage-3.0-Instruct"
+          HUNYUAN_IMAGE3_DEVICES: "0,1,2,3"
+          DIFFUSION_ATTENTION_BACKEND: TORCH_SDPA
+        commands:
+          - pytest -s -v tests/e2e/accuracy/test_hunyuan_image3_pixel_accuracy.py -m "full_model and A3" --run-level full_model
+  - group: ":card_index_dividers: Diffusion Benchmark Perf Test"
+    key: nightly-diffusion-benchmark-group
+    depends_on: upload-nightly-pipeline
+    if: >-
+      build.env("NIGHTLY") == "1" ||
+      build.pull_request.labels includes "nightly-test" ||
+      build.pull_request.labels includes "diffusion-x2iat-test"
+    steps:
+      - label: ":full_moon: Diffusion Perf · HunyuanImage3 TP=2 CFGP=2"
+        timeout_in_minutes: 180
+        mirror_hardwares: a3_npu_4
+        env:
+          VLLM_WORKER_MULTIPROC_METHOD: spawn
+          HF_TOKEN: "${HF_TOKEN}"
+          DIFFUSION_ATTENTION_BACKEND: TORCH_SDPA
+        commands:
+          - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
+          - |
+            set +e
+            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py -m A3 --test-config-file tests/dfx/perf/tests/test_hunyuan_image_tp2_cfgp2.json
+            EXIT=$$?
+            buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
+            buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
+            exit $$EXIT
+      - label: ":full_moon: Diffusion Perf · HunyuanImage3 TP=2 SP=2"
+        timeout_in_minutes: 180
+        mirror_hardwares: a3_npu_4
+        env:
+          VLLM_WORKER_MULTIPROC_METHOD: spawn
+          HF_TOKEN: "${HF_TOKEN}"
+          DIFFUSION_ATTENTION_BACKEND: TORCH_SDPA
+        commands:
+          - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
+          - |
+            set +e
+            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py -m A3 --test-config-file tests/dfx/perf/tests/test_hunyuan_image_tp2_sp2.json
+            EXIT=$$?
+            buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
+            buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
+            exit $$EXIT
+      - label: ":full_moon: Diffusion Perf · HunyuanImage3 TP=4"
+        timeout_in_minutes: 180
+        mirror_hardwares: a3_npu_4
+        env:
+          VLLM_WORKER_MULTIPROC_METHOD: spawn
+          HF_TOKEN: "${HF_TOKEN}"
+          DIFFUSION_ATTENTION_BACKEND: TORCH_SDPA
+        commands:
+          - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results
+          - |
+            set +e
+            pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py -m A3 --test-config-file tests/dfx/perf/tests/test_hunyuan_image_tp4.json
+            EXIT=$$?
+            buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json"
+            buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log"
+            exit $$EXIT
  - group: ":card_index_dividers: Diffusion X2V Model Test"
    key: nightly-diffusion-x2v-group
    depends_on: upload-nightly-pipeline
@@ -103,6 +177,14 @@ steps:
      build.pull_request.labels includes "nightly-test" ||
      build.pull_request.labels includes "diffusion-x2v-test"
    steps:
+      - label: ":full_moon: Diffusion X2V · Function Test"
+        mirror_hardwares: a2b3_npu_4
+        env:
+          VLLM_WORKER_MULTIPROC_METHOD: spawn
+          VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1"
+          HF_TOKEN: "${HF_TOKEN}"
+        commands:
+          - pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py -k npu_i2v --run-level "full_model"
      - label: ":full_moon: Diffusion X2V · Perf Test"
        key: nightly-diffusion-x2v-performance
        timeout_in_minutes: 180
@@ -45,6 +45,7 @@ RUN source /usr/local/Ascend/ascend-toolkit/set_env.sh && \
      "hyperpyyaml>=1.2.0" \
      "onnx>=1.16.0" \
      "librosa>=0.10.0" \
+      "torchmetrics>=1.9.0" \
      --index-url http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple \
      --trusted-host cache-service.nginx-pypi-cache.svc.cluster.local \
      --default-timeout=60 && \
@@ -5,7 +5,8 @@
      {
        "hardware_marks": {
          "res": {
-            "cuda": "H100"
+            "cuda": "H100",
+            "npu": "A3"
          },
          "num_cards": 4
        }
@@ -27,16 +28,9 @@
            "async_chunk": false,
            "trust_remote_code": true,
            "connectors": {
-              "rdma_connector": {
-                "name": "MooncakeTransferEngineConnector",
-                "extra": {
-                  "host": "auto",
-                  "zmq_port": 50051,
-                  "protocol": "rdma",
-                  "device_name": "",
-                  "memory_pool_size": 17179869184,
-                  "memory_pool_device": "cpu"
-                }
+              "shared_mem_connector": {
+                "name": "SharedMemoryConnector",
+                "extra": {}
              }
            },
            "stages": [
@@ -66,7 +60,7 @@
                  "need_send_cache": true
                },
                "output_connectors": {
-                  "to_stage_1": "rdma_connector"
+                  "to_stage_1": "shared_mem_connector"
                },
                "default_sampling_params": {
                  "temperature": 0.0,
@@ -96,7 +90,7 @@
                },
                "vae_use_tiling": true,
                "input_connectors": {
-                  "from_stage_0": "rdma_connector"
+                  "from_stage_0": "shared_mem_connector"
                },
                "default_sampling_params": {
                  "num_inference_steps": 50,
@@ -5,7 +5,8 @@
      {
        "hardware_marks": {
          "res": {
-            "cuda": "H100"
+            "cuda": "H100",
+            "npu": "A3"
          },
          "num_cards": 4
        }
@@ -5,7 +5,8 @@
      {
        "hardware_marks": {
          "res": {
-            "cuda": "H100"
+            "cuda": "H100",
+            "npu": "A3"
          },
          "num_cards": 4
        }
@@ -5,7 +5,8 @@
      {
        "hardware_marks": {
          "res": {
-            "cuda": "H100"
+            "cuda": "H100",
+            "npu": "A3"
          },
          "num_cards": 4
        }
@@ -32,10 +32,28 @@ PROMPT = "A brown and white dog is running on the grass."
 MEAN_THRESHOLD = 3e-2
 P99_THRESHOLD = 3e-1
 SSIM_THRESHOLD = 0.97
-PSNR_THRESHOLD = 30.0
+PSNR_THRESHOLD_CUDA = 30.0
+PSNR_THRESHOLD_NPU = 26.0
+
+
+def _psnr_threshold() -> float:
+    from vllm_omni.platforms import current_omni_platform
+
+    return PSNR_THRESHOLD_NPU if current_omni_platform.is_npu() else PSNR_THRESHOLD_CUDA
+

-BASELINE_PATH = get_asset_path("hunyuan/hunyuan_baseline.png")
 _REPO_ROOT = Path(__file__).resolve().parent.parent.parent.parent
+
+
+def _baseline_path() -> Path:
+    from vllm_omni.platforms import current_omni_platform
+
+    suffix = "_npu" if current_omni_platform.is_npu() else ""
+    path = get_asset_path(f"hunyuan/hunyuan_baseline{suffix}.png")
+    assert path.exists(), f"Baseline image not found at {path}"
+    return path
+
+
 _OFFLINE_SCRIPT = _REPO_ROOT / "examples" / "offline_inference" / "hunyuan_image3" / "end2end.py"

 # DiT-only deploy config with trust_remote_code (based on hunyuan_image3_dit.yaml).
@@ -74,6 +92,23 @@ _DEPLOY_CONFIG = {
            },
        },
    ],
+    "platforms": {
+        "npu": {
+            "stages": [
+                {
+                    "stage_id": 0,
+                    "gpu_memory_utilization": 0.65,
+                    "devices": "0,1,2,3",
+                    "moe_backend": "auto",
+                    "max_num_batched_tokens": 32768,
+                    "parallel_config": {
+                        "tensor_parallel_size": 4,
+                        "enable_expert_parallel": True,
+                    },
+                },
+            ],
+        },
+    },
 }


@@ -216,8 +251,8 @@ def _run_vllm_omni_hunyuan_image3_offline(*, model: str, deploy_config: str, out


 def _assert_against_baseline(image: Image.Image, label: str) -> None:
-    assert BASELINE_PATH.exists(), f"Baseline image not found at {BASELINE_PATH}"
-    baseline_image = Image.open(BASELINE_PATH).convert("RGB")
+    baseline_path = _baseline_path()
+    baseline_image = Image.open(baseline_path).convert("RGB")

    assert_images_pixel_close(
        model_name=f"{MODEL_NAME} ({label} vs baseline)",
@@ -231,11 +266,11 @@ def _assert_against_baseline(image: Image.Image, label: str) -> None:
        vllm_image=image,
        diffusers_image=baseline_image,
        ssim_threshold=SSIM_THRESHOLD,
-        psnr_threshold=PSNR_THRESHOLD,
+        psnr_threshold=_psnr_threshold(),
    )


-@hardware_test(res={"cuda": "H100"}, num_cards=4)
+@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=4)
 def test_hunyuan_image3_pixel_accuracy_online(accuracy_artifact_root: Path) -> None:
    model = _model_name()
    output_dir = model_output_dir(accuracy_artifact_root, MODEL_NAME)
@@ -249,7 +284,7 @@ def test_hunyuan_image3_pixel_accuracy_online(accuracy_artifact_root: Path) -> N
    _assert_against_baseline(image, "online")


-@hardware_test(res={"cuda": "H100"}, num_cards=4)
+@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=4)
 def test_hunyuan_image3_pixel_accuracy_offline(accuracy_artifact_root: Path) -> None:
    model = _model_name()
    output_dir = model_output_dir(accuracy_artifact_root, MODEL_NAME)