diff --git a/.buildkite/npu/test-npu-nightly.yml b/.buildkite/npu/test-npu-nightly.yml index 5aa003241..10626600c 100644 --- a/.buildkite/npu/test-npu-nightly.yml +++ b/.buildkite/npu/test-npu-nightly.yml @@ -95,6 +95,80 @@ steps: buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json" buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log" exit $$EXIT + - group: ":card_index_dividers: Diffusion I2I Model Test" + key: nightly-HunyuanImage3-x2i-test-group + depends_on: upload-nightly-pipeline + if: >- + build.env("NIGHTLY") == "1" || + build.pull_request.labels includes "nightly-test" || + build.pull_request.labels includes "diffusion-x2iat-test" + steps: + - label: ":full_moon: HunyuanImage3-DIT · Accuracy Test" + timeout_in_minutes: 180 + mirror_hardwares: a3_npu_4 + env: + VLLM_WORKER_MULTIPROC_METHOD: spawn + HUNYUAN_IMAGE3_MODEL: "tencent/HunyuanImage-3.0-Instruct" + HUNYUAN_IMAGE3_DEVICES: "0,1,2,3" + DIFFUSION_ATTENTION_BACKEND: TORCH_SDPA + commands: + - pytest -s -v tests/e2e/accuracy/test_hunyuan_image3_pixel_accuracy.py -m "full_model and A3" --run-level full_model + - group: ":card_index_dividers: Diffusion Benchmark Perf Test" + key: nightly-diffusion-benchmark-group + depends_on: upload-nightly-pipeline + if: >- + build.env("NIGHTLY") == "1" || + build.pull_request.labels includes "nightly-test" || + build.pull_request.labels includes "diffusion-x2iat-test" + steps: + - label: ":full_moon: Diffusion Perf · HunyuanImage3 TP=2 CFGP=2" + timeout_in_minutes: 180 + mirror_hardwares: a3_npu_4 + env: + VLLM_WORKER_MULTIPROC_METHOD: spawn + HF_TOKEN: "${HF_TOKEN}" + DIFFUSION_ATTENTION_BACKEND: TORCH_SDPA + commands: + - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results + - | + set +e + pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py -m A3 --test-config-file tests/dfx/perf/tests/test_hunyuan_image_tp2_cfgp2.json + EXIT=$$? + buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json" + buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log" + exit $$EXIT + - label: ":full_moon: Diffusion Perf · HunyuanImage3 TP=2 SP=2" + timeout_in_minutes: 180 + mirror_hardwares: a3_npu_4 + env: + VLLM_WORKER_MULTIPROC_METHOD: spawn + HF_TOKEN: "${HF_TOKEN}" + DIFFUSION_ATTENTION_BACKEND: TORCH_SDPA + commands: + - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results + - | + set +e + pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py -m A3 --test-config-file tests/dfx/perf/tests/test_hunyuan_image_tp2_sp2.json + EXIT=$$? + buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json" + buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log" + exit $$EXIT + - label: ":full_moon: Diffusion Perf · HunyuanImage3 TP=4" + timeout_in_minutes: 180 + mirror_hardwares: a3_npu_4 + env: + VLLM_WORKER_MULTIPROC_METHOD: spawn + HF_TOKEN: "${HF_TOKEN}" + DIFFUSION_ATTENTION_BACKEND: TORCH_SDPA + commands: + - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results + - | + set +e + pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py -m A3 --test-config-file tests/dfx/perf/tests/test_hunyuan_image_tp4.json + EXIT=$$? + buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json" + buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log" + exit $$EXIT - group: ":card_index_dividers: Diffusion X2V Model Test" key: nightly-diffusion-x2v-group depends_on: upload-nightly-pipeline @@ -103,6 +177,14 @@ steps: build.pull_request.labels includes "nightly-test" || build.pull_request.labels includes "diffusion-x2v-test" steps: + - label: ":full_moon: Diffusion X2V · Function Test" + mirror_hardwares: a2b3_npu_4 + env: + VLLM_WORKER_MULTIPROC_METHOD: spawn + VLLM_ALLOW_LONG_MAX_MODEL_LEN: "1" + HF_TOKEN: "${HF_TOKEN}" + commands: + - pytest -s -v tests/e2e/online_serving/test_wan22_expansion.py -k npu_i2v --run-level "full_model" - label: ":full_moon: Diffusion X2V · Perf Test" key: nightly-diffusion-x2v-performance timeout_in_minutes: 180 diff --git a/docker/Dockerfile.npu.ci.a3 b/docker/Dockerfile.npu.ci.a3 index b605567c5..383eea892 100644 --- a/docker/Dockerfile.npu.ci.a3 +++ b/docker/Dockerfile.npu.ci.a3 @@ -45,6 +45,7 @@ RUN source /usr/local/Ascend/ascend-toolkit/set_env.sh && \ "hyperpyyaml>=1.2.0" \ "onnx>=1.16.0" \ "librosa>=0.10.0" \ + "torchmetrics>=1.9.0" \ --index-url http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple \ --trusted-host cache-service.nginx-pypi-cache.svc.cluster.local \ --default-timeout=60 && \ diff --git a/tests/assets/hunyuan/hunyuan_baseline_npu.png b/tests/assets/hunyuan/hunyuan_baseline_npu.png new file mode 100644 index 000000000..1de1e8ff0 Binary files /dev/null and b/tests/assets/hunyuan/hunyuan_baseline_npu.png differ diff --git a/tests/dfx/perf/tests/test_hunyuan_image3_it2i.json b/tests/dfx/perf/tests/test_hunyuan_image3_it2i.json index ab8a042e1..d2a20af7a 100644 --- a/tests/dfx/perf/tests/test_hunyuan_image3_it2i.json +++ b/tests/dfx/perf/tests/test_hunyuan_image3_it2i.json @@ -5,7 +5,8 @@ { "hardware_marks": { "res": { - "cuda": "H100" + "cuda": "H100", + "npu": "A3" }, "num_cards": 4 } @@ -27,16 +28,9 @@ "async_chunk": false, "trust_remote_code": true, "connectors": { - "rdma_connector": { - "name": "MooncakeTransferEngineConnector", - "extra": { - "host": "auto", - "zmq_port": 50051, - "protocol": "rdma", - "device_name": "", - "memory_pool_size": 17179869184, - "memory_pool_device": "cpu" - } + "shared_mem_connector": { + "name": "SharedMemoryConnector", + "extra": {} } }, "stages": [ @@ -66,7 +60,7 @@ "need_send_cache": true }, "output_connectors": { - "to_stage_1": "rdma_connector" + "to_stage_1": "shared_mem_connector" }, "default_sampling_params": { "temperature": 0.0, @@ -96,7 +90,7 @@ }, "vae_use_tiling": true, "input_connectors": { - "from_stage_0": "rdma_connector" + "from_stage_0": "shared_mem_connector" }, "default_sampling_params": { "num_inference_steps": 50, diff --git a/tests/dfx/perf/tests/test_hunyuan_image_tp2_cfgp2.json b/tests/dfx/perf/tests/test_hunyuan_image_tp2_cfgp2.json index da761616a..733579e96 100644 --- a/tests/dfx/perf/tests/test_hunyuan_image_tp2_cfgp2.json +++ b/tests/dfx/perf/tests/test_hunyuan_image_tp2_cfgp2.json @@ -5,7 +5,8 @@ { "hardware_marks": { "res": { - "cuda": "H100" + "cuda": "H100", + "npu": "A3" }, "num_cards": 4 } diff --git a/tests/dfx/perf/tests/test_hunyuan_image_tp2_sp2.json b/tests/dfx/perf/tests/test_hunyuan_image_tp2_sp2.json index a719b3bbe..27b04a469 100644 --- a/tests/dfx/perf/tests/test_hunyuan_image_tp2_sp2.json +++ b/tests/dfx/perf/tests/test_hunyuan_image_tp2_sp2.json @@ -5,7 +5,8 @@ { "hardware_marks": { "res": { - "cuda": "H100" + "cuda": "H100", + "npu": "A3" }, "num_cards": 4 } diff --git a/tests/dfx/perf/tests/test_hunyuan_image_tp4.json b/tests/dfx/perf/tests/test_hunyuan_image_tp4.json index 5a71af5d7..28842164c 100644 --- a/tests/dfx/perf/tests/test_hunyuan_image_tp4.json +++ b/tests/dfx/perf/tests/test_hunyuan_image_tp4.json @@ -5,7 +5,8 @@ { "hardware_marks": { "res": { - "cuda": "H100" + "cuda": "H100", + "npu": "A3" }, "num_cards": 4 } diff --git a/tests/e2e/accuracy/test_hunyuan_image3_pixel_accuracy.py b/tests/e2e/accuracy/test_hunyuan_image3_pixel_accuracy.py index 34131dc5a..16cd9c13c 100644 --- a/tests/e2e/accuracy/test_hunyuan_image3_pixel_accuracy.py +++ b/tests/e2e/accuracy/test_hunyuan_image3_pixel_accuracy.py @@ -32,10 +32,28 @@ PROMPT = "A brown and white dog is running on the grass." MEAN_THRESHOLD = 3e-2 P99_THRESHOLD = 3e-1 SSIM_THRESHOLD = 0.97 -PSNR_THRESHOLD = 30.0 +PSNR_THRESHOLD_CUDA = 30.0 +PSNR_THRESHOLD_NPU = 26.0 + + +def _psnr_threshold() -> float: + from vllm_omni.platforms import current_omni_platform + + return PSNR_THRESHOLD_NPU if current_omni_platform.is_npu() else PSNR_THRESHOLD_CUDA + -BASELINE_PATH = get_asset_path("hunyuan/hunyuan_baseline.png") _REPO_ROOT = Path(__file__).resolve().parent.parent.parent.parent + + +def _baseline_path() -> Path: + from vllm_omni.platforms import current_omni_platform + + suffix = "_npu" if current_omni_platform.is_npu() else "" + path = get_asset_path(f"hunyuan/hunyuan_baseline{suffix}.png") + assert path.exists(), f"Baseline image not found at {path}" + return path + + _OFFLINE_SCRIPT = _REPO_ROOT / "examples" / "offline_inference" / "hunyuan_image3" / "end2end.py" # DiT-only deploy config with trust_remote_code (based on hunyuan_image3_dit.yaml). @@ -74,6 +92,23 @@ _DEPLOY_CONFIG = { }, }, ], + "platforms": { + "npu": { + "stages": [ + { + "stage_id": 0, + "gpu_memory_utilization": 0.65, + "devices": "0,1,2,3", + "moe_backend": "auto", + "max_num_batched_tokens": 32768, + "parallel_config": { + "tensor_parallel_size": 4, + "enable_expert_parallel": True, + }, + }, + ], + }, + }, } @@ -216,8 +251,8 @@ def _run_vllm_omni_hunyuan_image3_offline(*, model: str, deploy_config: str, out def _assert_against_baseline(image: Image.Image, label: str) -> None: - assert BASELINE_PATH.exists(), f"Baseline image not found at {BASELINE_PATH}" - baseline_image = Image.open(BASELINE_PATH).convert("RGB") + baseline_path = _baseline_path() + baseline_image = Image.open(baseline_path).convert("RGB") assert_images_pixel_close( model_name=f"{MODEL_NAME} ({label} vs baseline)", @@ -231,11 +266,11 @@ def _assert_against_baseline(image: Image.Image, label: str) -> None: vllm_image=image, diffusers_image=baseline_image, ssim_threshold=SSIM_THRESHOLD, - psnr_threshold=PSNR_THRESHOLD, + psnr_threshold=_psnr_threshold(), ) -@hardware_test(res={"cuda": "H100"}, num_cards=4) +@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=4) def test_hunyuan_image3_pixel_accuracy_online(accuracy_artifact_root: Path) -> None: model = _model_name() output_dir = model_output_dir(accuracy_artifact_root, MODEL_NAME) @@ -249,7 +284,7 @@ def test_hunyuan_image3_pixel_accuracy_online(accuracy_artifact_root: Path) -> N _assert_against_baseline(image, "online") -@hardware_test(res={"cuda": "H100"}, num_cards=4) +@hardware_test(res={"cuda": "H100", "npu": "A3"}, num_cards=4) def test_hunyuan_image3_pixel_accuracy_offline(accuracy_artifact_root: Path) -> None: model = _model_name() output_dir = model_output_dir(accuracy_artifact_root, MODEL_NAME)