From fb4204ebf658ccff4b7cffb19473adc77da72f62 Mon Sep 17 00:00:00 2001 From: bjf-frz Date: Fri, 24 Apr 2026 16:58:45 +0800 Subject: [PATCH] [CI][Perf] Add Wan22 i2v perf nightly ci (#3063) Signed-off-by: bjf-frz --- .buildkite/test-nightly.yml | 49 ++++++++ .../diffusion/diffusion_benchmark_serving.py | 8 +- .../perf/scripts/run_diffusion_benchmark.py | 61 ++++++---- .../perf/tests/test_wan22_i2v_vllm_omni.json | 107 ++++++++++++++++++ 4 files changed, 202 insertions(+), 23 deletions(-) create mode 100644 tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json diff --git a/.buildkite/test-nightly.yml b/.buildkite/test-nightly.yml index e35b20904..ce67b76d9 100644 --- a/.buildkite/test-nightly.yml +++ b/.buildkite/test-nightly.yml @@ -633,6 +633,53 @@ steps: path: /mnt/hf-cache type: DirectoryOrCreate + - label: ":full_moon: Diffusion X2V ยท Perf Test" + key: nightly-diffusion-x2v-performance + timeout_in_minutes: 180 + commands: + - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results + - export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN + - | + set +e + pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json + EXIT1=$$? + buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json" + buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log" + exit $$EXIT1 + agents: + queue: "mithril-h100-pool" + plugins: + - kubernetes: + podSpec: + containers: + - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT + resources: + limits: + nvidia.com/gpu: 2 + volumeMounts: + - name: devshm + mountPath: /dev/shm + - name: hf-cache + mountPath: /root/.cache/huggingface + env: + - name: HF_HOME + value: /root/.cache/huggingface + - name: HF_TOKEN + valueFrom: + secretKeyRef: + name: hf-token-secret + key: token + nodeSelector: + node.kubernetes.io/instance-type: gpu-h100-sxm + volumes: + - name: devshm + emptyDir: + medium: Memory + - name: hf-cache + hostPath: + path: /mnt/hf-cache + type: DirectoryOrCreate + - label: ":bar_chart: Testcase Statistics" key: nightly-testcase-statistics timeout_in_minutes: 120 @@ -682,6 +729,7 @@ steps: - nightly-omni-performance - nightly-tts-performance - nightly-diffusion-x2iat-performance + - nightly-diffusion-x2v-performance - nightly-testcase-statistics if: build.env("NIGHTLY") == "1" commands: @@ -691,6 +739,7 @@ steps: - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-tts-performance - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-omni-performance - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance + - buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2v-performance - buildkite-agent artifact download "tests/dfx/perf/results/*.html" . --step nightly-testcase-statistics - python tools/nightly/generate_nightly_perf_excel.py - python tools/nightly/generate_nightly_perf_html.py diff --git a/benchmarks/diffusion/diffusion_benchmark_serving.py b/benchmarks/diffusion/diffusion_benchmark_serving.py index 32ec48a69..77b36b3d9 100644 --- a/benchmarks/diffusion/diffusion_benchmark_serving.py +++ b/benchmarks/diffusion/diffusion_benchmark_serving.py @@ -12,15 +12,15 @@ Supports multiple backends: - v1/videos: Use /v1/videos endpoint Usage: - # Video (vllm-omni backend) + # Video (v1/videos backend) t2v: python3 benchmarks/diffusion/diffusion_benchmark_serving.py \ - --backend vllm-omni --dataset vbench --task t2v --num-prompts 10 \ + --backend v1/videos --dataset vbench --task t2v --num-prompts 10 \ --height 480 --width 640 --fps 16 --num-frames 80 i2v: python3 benchmarks/diffusion/diffusion_benchmark_serving.py \ - --backend vllm-omni --dataset vbench --task i2v --num-prompts 10 + --backend v1/videos --dataset vbench --task i2v --num-prompts 10 # Image (vllm-omni backend) @@ -49,7 +49,7 @@ Usage: --backend openai --dataset vbench --task t2i --num-prompts 10 \ --height 1024 --width 1024 --port 3000 - # Video (v1/vedeos) + # Video (v1/videos) t2v: python3 benchmarks/diffusion/diffusion_benchmark_serving.py \ --backend v1/videos --dataset random --task t2v --num-prompts 1 \ diff --git a/tests/dfx/perf/scripts/run_diffusion_benchmark.py b/tests/dfx/perf/scripts/run_diffusion_benchmark.py index d41f80dc7..7513c2d3f 100644 --- a/tests/dfx/perf/scripts/run_diffusion_benchmark.py +++ b/tests/dfx/perf/scripts/run_diffusion_benchmark.py @@ -1,16 +1,17 @@ """ Performance benchmark CI runner for diffusion models. -Supports vLLM-Omni server backend: - - vllm-omni (default): starts DiffusionServer via vllm_omni.entrypoints.cli.main, - benchmarks with diffusion_benchmark_serving.py --backend vllm-omni +This runner separates two concepts: + +1. ``server_type``: how the serving process is started. + Currently only ``vllm-omni`` is supported here. +2. ``benchmark_backend``: which serving API the benchmark client calls. + Examples: ``vllm-omni`` for ``/v1/chat/completions`` and ``v1/videos`` + for async video jobs. A config JSON file is REQUIRED via --test-config-file: pytest run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_vllm_omni.json -JSON config entries use a "server_type" field, and this runner executes -the vllm-omni path. - All benchmark results for a session are consolidated into a single JSON file under BENCHMARK_RESULT_DIR (override via the DIFFUSION_BENCHMARK_DIR environment variable). Each entry in the file contains the test metadata (test_name, backend, benchmark_params, @@ -389,17 +390,18 @@ def _unique_server_params(configs: list[dict[str, Any]]) -> list[dict[str, Any]] if test_name in seen: continue seen.add(test_name) - if cfg.get("server_type", "vllm-omni") != "vllm-omni": - raise ValueError(f"Unsupported server_type in config: {cfg.get('server_type')}") + server_type = cfg.get("server_type", "vllm-omni") + if server_type != "vllm-omni": + raise ValueError(f"Unsupported server_type in config: {server_type}") serve_args_dict = cfg["server_params"].get("serve_args", {}) result.append( { "test_name": test_name, - "server_type": "vllm-omni", + "server_type": server_type, "model": cfg["server_params"]["model"], "serve_args_dict": serve_args_dict, "serve_args": _build_serve_args(serve_args_dict), - "benchmark_backend": "vllm-omni", + "benchmark_backend": cfg.get("benchmark_backend"), "server_params": cfg["server_params"], } ) @@ -499,7 +501,8 @@ def run_benchmark( log_dir = BENCHMARK_RESULT_DIR / "logs" log_dir.mkdir(parents=True, exist_ok=True) - log_file = log_dir / f"{test_name}_{backend}_{timestamp}.log" + backend_label = backend.replace("/", "_") + log_file = log_dir / f"{test_name}_{backend_label}_{timestamp}.log" with tempfile.NamedTemporaryFile(mode="w", suffix=".json", prefix="diffusion_bench_tmp_", delete=False) as tmp: tmp_result_file = Path(tmp.name) @@ -585,6 +588,7 @@ def run_benchmark( tmp_result_file.unlink(missing_ok=True) server_cfg = server_cfg or {} + server_type = cast(str, server_cfg.get("server_type", "vllm-omni")) serve_args_dict = server_cfg.get("serve_args_dict", {}) if not isinstance(serve_args_dict, dict): serve_args_dict = {} @@ -601,18 +605,19 @@ def run_benchmark( "result": metrics, "log_file": str(log_file), "Model": model, - "Framework": backend, + "Framework": server_type, + "API Backend": backend, "Hardware": "", "Deployment": "", "Task": params.get("task", "t2i"), "Dataset": params.get("dataset", "random"), "resolution": _to_resolution_string(params), - "Parallelism": _to_parallelism_string(backend, serve_args_dict), + "Parallelism": _to_parallelism_string(server_type, serve_args_dict), "max_concurrency": params.get("max-concurrency", ""), - "Cache": _to_cache_string(backend, serve_args_dict), - "Quantization": _to_quantization_value(backend, serve_args_dict), - "offload": _to_offload_string(backend, serve_args_dict), - "compile": _to_compile_value(backend, serve_args_dict), + "Cache": _to_cache_string(server_type, serve_args_dict), + "Quantization": _to_quantization_value(server_type, serve_args_dict), + "offload": _to_offload_string(server_type, serve_args_dict), + "compile": _to_compile_value(server_type, serve_args_dict), "Attn_backend": os.environ.get("DIFFUSION_ATTENTION_BACKEND", ""), "num_inference_steps": params.get("num-inference-steps", ""), "completed": completed, @@ -661,6 +666,23 @@ def assert_result(result: dict[str, Any], params: dict[str, Any]) -> None: assert current <= threshold, f"{metric}: {current:.4f} > baseline {threshold}" +def _default_benchmark_backend_for_task(task: str) -> str: + """Return the default client-side benchmark backend for a diffusion task.""" + if task in {"t2v", "i2v", "ti2v"}: + return "v1/videos" + if task in {"t2i", "i2i", "ti2i"}: + return "vllm-omni" + raise ValueError(f"Unsupported task for benchmark backend resolution: {task}") + + +def _resolve_benchmark_backend(server_cfg: dict[str, Any], params: dict[str, Any]) -> str: + """Resolve which serving API the benchmark client should call.""" + configured = server_cfg.get("benchmark_backend") + if configured: + return cast(str, configured) + return _default_benchmark_backend_for_task(cast(str, params.get("task", "t2i"))) + + # --------------------------------------------------------------------------- # Test entry point # --------------------------------------------------------------------------- @@ -685,7 +707,8 @@ def test_diffusion_performance_benchmark(diffusion_server, benchmark_params): """ test_name = benchmark_params["test_name"] params = benchmark_params["params"] - backend = diffusion_server.server_type # "vllm-omni" + server_cfg = getattr(diffusion_server, "server_cfg", {}) + backend = _resolve_benchmark_backend(server_cfg, params) result = run_benchmark( host=diffusion_server.host, @@ -694,7 +717,7 @@ def test_diffusion_performance_benchmark(diffusion_server, benchmark_params): params=params, test_name=test_name, backend=backend, - server_cfg=getattr(diffusion_server, "server_cfg", {}), + server_cfg=server_cfg, source_file=cast(str, CONFIG_FILE_PATH), ) diff --git a/tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json b/tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json new file mode 100644 index 000000000..58a17c980 --- /dev/null +++ b/tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json @@ -0,0 +1,107 @@ +[ + { + "test_name": "test_wan22_i2v_single_device", + "description": "Single-device baseline", + "server_type": "vllm-omni", + "server_params": { + "model": "Wan-AI/Wan2.2-I2V-A14B-Diffusers", + "serve_args": { + "enable-diffusion-pipeline-profiler": true + } + }, + "benchmark_params": [ + { + "name": "832x480_frames81_steps4", + "dataset": "random", + "task": "i2v", + "num-prompts": 10, + "max-concurrency": 1, + "num-input-images": 1, + "seed": 42, + "enable-negative-prompt": true, + "random-request-config": [ + { + "width": 832, + "height": 480, + "num_inference_steps": 4, + "num_frames": 81, + "fps": 16, + "weight": 1 + } + ], + "baseline": { + "throughput_qps": 0.034, + "latency_mean": 26.0, + "peak_memory_mb_mean": 80000 + } + } + ] + }, + { + "test_name": "test_wan22_i2v_usp2_vae_patch2_hsdp_slicing", + "description": "USP=2 + VAE patch parallel=2 + HSDP + VAE slicing", + "server_type": "vllm-omni", + "server_params": { + "model": "Wan-AI/Wan2.2-I2V-A14B-Diffusers", + "serve_args": { + "usp": 2, + "vae-patch-parallel-size": 2, + "use-hsdp": true, + "vae-use-slicing": true, + "enable-diffusion-pipeline-profiler": true + } + }, + "benchmark_params": [ + { + "name": "832x480_frames81_steps4", + "dataset": "random", + "task": "i2v", + "num-prompts": 10, + "max-concurrency": 1, + "num-input-images": 1, + "seed": 42, + "enable-negative-prompt": true, + "random-request-config": [ + { + "width": 832, + "height": 480, + "num_inference_steps": 4, + "num_frames": 81, + "fps": 16, + "weight": 1 + } + ], + "baseline": { + "throughput_qps": 0.042, + "latency_mean": 21.6, + "peak_memory_mb_mean": 55300 + } + }, + { + "name": "1280x720_frames121_steps4", + "dataset": "random", + "task": "i2v", + "num-prompts": 10, + "max-concurrency": 1, + "num-input-images": 1, + "seed": 42, + "enable-negative-prompt": true, + "random-request-config": [ + { + "width": 1280, + "height": 720, + "num_inference_steps": 4, + "num_frames": 121, + "fps": 16, + "weight": 1 + } + ], + "baseline": { + "throughput_qps": 0.0085, + "latency_mean": 101.6, + "peak_memory_mb_mean": 65200 + } + } + ] + } +]