From d0b531520d67f6a496afd6c1495aa765e0c78e65 Mon Sep 17 00:00:00 2001 From: dengyunyang <584797741@qq.com> Date: Tue, 5 May 2026 20:05:46 +0800 Subject: [PATCH] [Performance CI ]Hunyuan Image 3.0 DIT bench test (#2495) Signed-off-by: zhou zhuoxin Signed-off-by: dengyunyang <584797741@qq.com> Signed-off-by: TaffyOfficial <2324465096@qq.com> Signed-off-by: ChenZhao Signed-off-by: zuiho <2324465096@qq.com> Co-authored-by: zhou zhuoxin Co-authored-by: Gao Han Co-authored-by: TaffyOfficial <2324465096@qq.com> Co-authored-by: Claude Opus 4.7 (1M context) Co-authored-by: TaffyOfficial <2587297563@qq.com> --- .buildkite/test-nightly.yml | 56 ++++- recipes/README.md | 3 + .../HunyuanImage-3.0-Instruct.md | 225 ++++++++++++++++++ .../perf/scripts/run_diffusion_benchmark.py | 6 +- .../test_hunyuan_image_tp2_fp8_cfgp2.json | 37 +++ .../tests/test_hunyuan_image_tp2_fp8_sp2.json | 37 +++ .../tests/test_hunyuan_image_tp4_fp8.json | 36 +++ .../hunyuan_image3/pipeline_hunyuan_image3.py | 2 + 8 files changed, 400 insertions(+), 2 deletions(-) create mode 100644 recipes/Tencent-Hunyuan/HunyuanImage-3.0-Instruct.md create mode 100644 tests/dfx/perf/tests/test_hunyuan_image_tp2_fp8_cfgp2.json create mode 100644 tests/dfx/perf/tests/test_hunyuan_image_tp2_fp8_sp2.json create mode 100644 tests/dfx/perf/tests/test_hunyuan_image_tp4_fp8.json diff --git a/.buildkite/test-nightly.yml b/.buildkite/test-nightly.yml index 6f206422a..675ce4e6b 100644 --- a/.buildkite/test-nightly.yml +++ b/.buildkite/test-nightly.yml @@ -550,7 +550,7 @@ steps: - label: ":full_moon: Diffusion X2I(&A&T) · Perf Test" key: nightly-diffusion-x2iat-performance - timeout_in_minutes: 180 + timeout_in_minutes: 360 commands: - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results - export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN @@ -604,6 +604,60 @@ steps: path: /mnt/hf-cache type: DirectoryOrCreate + - label: ":full_moon: Diffusion X2I(&A&T) · HunyuanImage3 · DiT Perf Test" + key: nightly-hunyuan-image3-performance + soft_fail: true + timeout_in_minutes: 120 + if: build.env("RUN_HUNYUAN_IMAGE3_PERF") == "1" + commands: + - export DIFFUSION_BENCHMARK_DIR=tests/dfx/perf/results + - export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN + - export CACHE_DIT_VERSION=1.3.0 + - | + set +e + pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_hunyuan_image_tp4_fp8.json + EXIT1=$$? + pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_hunyuan_image_tp2_fp8_sp2.json + EXIT2=$$? + pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_hunyuan_image_tp2_fp8_cfgp2.json + EXIT3=$$? + buildkite-agent artifact upload "tests/dfx/perf/results/diffusion_result_*.json" + buildkite-agent artifact upload "tests/dfx/perf/results/logs/*.log" + exit $$((EXIT1 | EXIT2 | EXIT3)) + agents: + queue: "mithril-h100-pool" + plugins: + - kubernetes: + podSpec: + containers: + - image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT + resources: + limits: + nvidia.com/gpu: 4 + volumeMounts: + - name: devshm + mountPath: /dev/shm + - name: hf-cache + mountPath: /root/.cache/huggingface + env: + - name: HF_HOME + value: /root/.cache/huggingface + - name: HF_TOKEN + valueFrom: + secretKeyRef: + name: hf-token-secret + key: token + nodeSelector: + node.kubernetes.io/instance-type: gpu-h100-sxm + volumes: + - name: devshm + emptyDir: + medium: Memory + - name: hf-cache + hostPath: + path: /mnt/hf-cache + type: DirectoryOrCreate + # Diffusion x2v only (Wan, HunyuanVideo, …). x2i/x2a/x2t live in the X2I group above, not here. - group: ":card_index_dividers: Diffusion X2V Model Test" key: nightly-diffusion-x2v-group diff --git a/recipes/README.md b/recipes/README.md index 0c490dace..ac493e75f 100644 --- a/recipes/README.md +++ b/recipes/README.md @@ -29,6 +29,9 @@ recipes/ multimodal chat on `1x A100 80GB` - [`Wan-AI/Wan2.2-I2V.md`](./Wan-AI/Wan2.2-I2V.md): image-to-video serving recipe for Wan2.2 14B on `8x Ascend NPU (A2/A3)` +- [`Tencent-Hunyuan/HunyuanImage-3.0-Instruct.md`](./Tencent-Hunyuan/HunyuanImage-3.0-Instruct.md): + DiT-only text-to-image serving and benchmark recipe for HunyuanImage-3.0-Instruct + on `4x H100/H800 80GB` - [`inclusionAI/Ming-flash-omni-2.0.md`](./inclusionAI/Ming-flash-omni-2.0.md): online serving recipe for multimodal chat (`4x H100 80GB`) and standalone TTS (`1x H100 80GB`) - [`Baidu/ERNIE-Image.md`](./Baidu/ERNIE-Image.md): text-to-image serving diff --git a/recipes/Tencent-Hunyuan/HunyuanImage-3.0-Instruct.md b/recipes/Tencent-Hunyuan/HunyuanImage-3.0-Instruct.md new file mode 100644 index 000000000..2ef99317e --- /dev/null +++ b/recipes/Tencent-Hunyuan/HunyuanImage-3.0-Instruct.md @@ -0,0 +1,225 @@ +# HunyuanImage-3.0-Instruct DiT Image Generation on 4x GPU + +> DiT-only text-to-image recipe for HunyuanImage-3.0-Instruct with FP8, +> tensor parallelism, sequence parallelism, and CFG parallelism. + +## Summary + +- Vendor: Tencent Hunyuan +- Model: `tencent/HunyuanImage-3.0-Instruct` +- Task: Text-to-image generation +- Mode: Online serving and performance benchmarking, DiT stage only +- Maintainer: Community + +## When to use this recipe + +Use this recipe when you want to run or benchmark the HunyuanImage-3.0 DiT +stage directly. This is the recommended first setup when validating DiT +throughput, memory, FP8 kernels, sequence parallelism, or CFG parallelism. + +The recipe covers three 4-GPU configurations: + +| Configuration | Parallelism | Notes | +| --- | --- | --- | +| `tp4_fp8` | TP=4 | Lowest per-GPU memory, higher communication overhead | +| `tp2_fp8_sp2` | TP=2, SP=2, Ulysses=2 | Splits sequence work across two GPUs per TP group | +| `tp2_fp8_cfgp2` | TP=2, CFG=2 | Runs CFG branches in parallel; fastest validated DiT setup | + +## References + +- Model: +- Offline example: + [`examples/offline_inference/hunyuan_image3`](../../examples/offline_inference/hunyuan_image3) +- Related PRs: + [#2495](https://github.com/vllm-project/vllm-omni/pull/2495) for DiT performance CI, + [#3055](https://github.com/vllm-project/vllm-omni/pull/3055) for GEBench accuracy CI, + and [#3104](https://github.com/vllm-project/vllm-omni/pull/3104) for the T2I L3 dummy guard. + +## Hardware Support + +## GPU + +### 4x H100/H800 80GB + +#### Environment + +- OS: Linux +- Python: 3.10+ +- Driver / runtime: CUDA-capable runtime matching the repository build +- vLLM minimum: 0.19.0, matching the current vLLM-Omni quickstart baseline. +- vLLM-Omni minimum: PR #2495 branch, or the first release that contains + HunyuanImage-3.0 DiT serving with the CLI flags below. +- Optional environment variables: + +```bash +export CACHE_DIT_VERSION=1.3.0 +``` + +HunyuanImage-3.0 sets the diffusion attention backend to `TORCH_SDPA` +internally because the model mixes causal and full attention. + +Graph mode is not part of this validated recipe. Keep `--enforce-eager` for +the FP8 DiT configurations below unless you separately validate graph mode for +the same checkpoint, parallelism, and image settings. + +#### Commands + +Start the DiT-only server with one of the following CLI-only configurations. +These commands use explicit CLI flags for all parallelism and runtime settings. + +**TP=4 + FP8** + +```bash +vllm serve tencent/HunyuanImage-3.0-Instruct \ + --omni \ + --port 8091 \ + --tensor-parallel-size 4 \ + --quantization fp8 \ + --distributed-executor-backend mp \ + --enforce-eager \ + --enable-diffusion-pipeline-profiler +``` + +**TP=2 + FP8 + SP=2** + +```bash +vllm serve tencent/HunyuanImage-3.0-Instruct \ + --omni \ + --port 8091 \ + --tensor-parallel-size 2 \ + --usp 2 \ + --quantization fp8 \ + --distributed-executor-backend mp \ + --enforce-eager \ + --enable-diffusion-pipeline-profiler +``` + +**TP=2 + FP8 + CFG=2** + +```bash +vllm serve tencent/HunyuanImage-3.0-Instruct \ + --omni \ + --port 8091 \ + --tensor-parallel-size 2 \ + --cfg-parallel-size 2 \ + --quantization fp8 \ + --distributed-executor-backend mp \ + --enforce-eager \ + --enable-diffusion-pipeline-profiler +``` + +Generate one 1024x1024 image: + +```bash +curl -s http://localhost:8091/v1/chat/completions \ + -H "Content-Type: application/json" \ + -d '{ + "messages": [ + {"role": "user", "content": "A cinematic photo of a glass observatory on Mars at sunrise"} + ], + "extra_body": { + "height": 1024, + "width": 1024, + "num_inference_steps": 50, + "guidance_scale": 5.0, + "seed": 42 + } + }' | jq -r '.choices[0].message.content[0].image_url.url' \ + | cut -d',' -f2- \ + | base64 -d > hunyuan_image3_output.png +``` + +#### Benchmark + +PR [#2495](https://github.com/vllm-project/vllm-omni/pull/2495) adds +performance CI configs for the same DiT-only settings. The CI step is +currently opt-in (gated by `RUN_HUNYUAN_IMAGE3_PERF=1`) with `soft_fail` +enabled, intended for initial data collection. Performance assertions are +skipped (`skip-performance-assertion: true`); the baseline values in the +JSON configs are reference-only and will be promoted to regression gates +once enough nightly data has been collected. + +The user-facing equivalent is to launch one of the CLI commands above and +generate 1024x1024 images with 50 denoising steps. + +#### Verification + +Check that: + +- The server responds on `http://localhost:8091/health`. +- The generation request writes a valid PNG file. +- Logs include `Selected CutlassFP8ScaledMMLinearKernel` for dense FP8 + linear layers and `Using TRITON Fp8 MoE` for MoE layers. +- With `--enable-diffusion-pipeline-profiler`, logs include per-stage timings + such as `model.forward`, `patch_embed.forward`, `final_layer.forward`, and + `vae.decode`. + +Validated benchmark characteristics for 1024x1024, 50 denoising steps, +batch size 1: + +| Configuration | Latency | Peak memory | +| --- | ---: | ---: | +| `tp4_fp8` | about 13.7s | about 47 GB | +| `tp2_fp8_sp2` | about 12.1s | about 66 GB | +| `tp2_fp8_cfgp2` | about 10.0s | about 66 GB | + +#### Related Accuracy Smoke Data + +PR [#3055](https://github.com/vllm-project/vllm-omni/pull/3055) adds a +DiT-only GEBench smoke setup for CI accuracy validation. Its validated +configuration was: + +- Hardware: 4x H100. +- Runtime: TP=4 with expert parallel enabled, `bfloat16`, + `distributed_executor_backend=mp`, `max_num_seqs=1`, `enforce_eager=True`. +- Task scope: T2I-only GEBench type3/type4, 4 samples per type, 28 denoising + steps. +- Judge: `QuantTrio/Qwen3-VL-30B-A3B-Instruct-AWQ`. + +Validated score summary: + +| Metric | Score | +| --- | ---: | +| overall_mean | 0.955 | +| type3 overall_mean | 0.91 | +| type4 overall_mean | 1.00 | + +The CI assertion threshold is `overall_mean >= 0.45`, so the smoke result is +comfortably above the gate. The generate server and judge server run +sequentially through the `OmniServer` fixture, and +`VLLM_TEST_CLEAN_GPU_MEMORY=1` is used to wait for memory cleanup between +server lifetimes. + +The lower-cost 2-GPU Instruct setup was tried for this smoke path but did not +fit in memory. A previous 2-GPU experiment used the base HunyuanImage-3.0 +checkpoint with FP8, but that base checkpoint is not available in the CI HF +cache. The validated CI-ready Instruct setup is therefore 4x H100 TP=4 with +expert parallel. + +#### Related Functional Guard + +PR [#3104](https://github.com/vllm-project/vllm-omni/pull/3104) adds an L3 +dummy guard for the T2I request path. The guard exercises +`HunyuanImage3Pipeline.forward()` without loading the full checkpoint by +stubbing `prepare_model_inputs()` and `_generate()`. It verifies propagation +of: + +- prompt and system prompt selection; +- output image size; +- inference steps and guidance scale; +- request generator; +- image `DiffusionOutput` and `stage_durations`. + +#### Notes + +- This recipe is DiT-only and does not cover end-to-end HunyuanImage serving. +- `tp2_fp8_cfgp2` is usually fastest because CFG branches run in parallel. + Individual layer timing can still look slower than `tp4_fp8` because each + CFG branch uses TP=2, so each GPU owns a larger shard than in TP=4. +- `tp4_fp8` has the lowest per-GPU memory because weights are sharded across + all four GPUs, but it pays more all-reduce communication overhead. +- `tp2_fp8_sp2` can improve model-forward latency by splitting sequence work, + while adding all-to-all communication overhead. +- If you see OOM on 80GB GPUs, reduce image size, request concurrency, or use + the TP=4 configuration before increasing batch size. GPU memory utilization + is not a useful primary tuning knob for this DiT-only recipe. diff --git a/tests/dfx/perf/scripts/run_diffusion_benchmark.py b/tests/dfx/perf/scripts/run_diffusion_benchmark.py index 7513c2d3f..f33cacc07 100644 --- a/tests/dfx/perf/scripts/run_diffusion_benchmark.py +++ b/tests/dfx/perf/scripts/run_diffusion_benchmark.py @@ -507,7 +507,7 @@ def run_benchmark( with tempfile.NamedTemporaryFile(mode="w", suffix=".json", prefix="diffusion_bench_tmp_", delete=False) as tmp: tmp_result_file = Path(tmp.name) - exclude_keys = {"baseline", "dataset", "task", "name"} + exclude_keys = {"baseline", "dataset", "task", "name", "skip-performance-assertion"} cmd = [ sys.executable, @@ -657,6 +657,10 @@ def assert_result(result: dict[str, Any], params: dict[str, Any]) -> None: completed = result.get("completed_requests", result.get("completed", 0)) assert completed == num_prompts, f"Expected {num_prompts} completed requests, got {completed}" + if params.get("skip-performance-assertion", False): + print("Skipping performance assertions.") + return + for metric, threshold in params.get("baseline", {}).items(): current = result.get(metric) assert current is not None, f"Metric '{metric}' not found in result: {list(result.keys())}" diff --git a/tests/dfx/perf/tests/test_hunyuan_image_tp2_fp8_cfgp2.json b/tests/dfx/perf/tests/test_hunyuan_image_tp2_fp8_cfgp2.json new file mode 100644 index 000000000..e6271545f --- /dev/null +++ b/tests/dfx/perf/tests/test_hunyuan_image_tp2_fp8_cfgp2.json @@ -0,0 +1,37 @@ +[ + { + "test_name": "test_hunyuan_image_tp2_fp8_cfgp2", + "description": "TP=2 Quantization=fp8 CfgP=2 baseline", + "server_type": "vllm-omni", + "server_params": { + "model": "tencent/HunyuanImage-3.0-Instruct", + "serve_args": { + "tensor-parallel-size": 2, + "cfg-parallel-size": 2, + "quantization": "fp8", + "distributed-executor-backend": "mp", + "enforce-eager": true, + "enable-diffusion-pipeline-profiler": true + } + }, + "benchmark_params": [ + { + "name": "1024x1024_steps50", + "dataset": "random", + "task": "t2i", + "width": 1024, + "height": 1024, + "num-inference-steps": 50, + "num-prompts": 10, + "max-concurrency": 1, + "skip-performance-assertion": true, + "baseline": { + "throughput_qps": 0.1035, + "latency_p99": 9.9057, + "peak_memory_mb_max": 66470, + "peak_memory_mb_mean": 66470 + } + } + ] + } +] diff --git a/tests/dfx/perf/tests/test_hunyuan_image_tp2_fp8_sp2.json b/tests/dfx/perf/tests/test_hunyuan_image_tp2_fp8_sp2.json new file mode 100644 index 000000000..0112fc51a --- /dev/null +++ b/tests/dfx/perf/tests/test_hunyuan_image_tp2_fp8_sp2.json @@ -0,0 +1,37 @@ +[ + { + "test_name": "test_hunyuan_image_tp2_fp8_sp2", + "description": "TP=2 Quantization=fp8 SP=2 baseline", + "server_type": "vllm-omni", + "server_params": { + "model": "tencent/HunyuanImage-3.0-Instruct", + "serve_args": { + "tensor-parallel-size": 2, + "usp": 2, + "quantization": "fp8", + "distributed-executor-backend": "mp", + "enforce-eager": true, + "enable-diffusion-pipeline-profiler": true + } + }, + "benchmark_params": [ + { + "name": "1024x1024_steps50", + "dataset": "random", + "task": "t2i", + "width": 1024, + "height": 1024, + "num-inference-steps": 50, + "num-prompts": 10, + "max-concurrency": 1, + "skip-performance-assertion": true, + "baseline": { + "throughput_qps": 0.08, + "latency_p99": 12.0731, + "peak_memory_mb_max": 66314, + "peak_memory_mb_mean": 66314 + } + } + ] + } +] diff --git a/tests/dfx/perf/tests/test_hunyuan_image_tp4_fp8.json b/tests/dfx/perf/tests/test_hunyuan_image_tp4_fp8.json new file mode 100644 index 000000000..8f6bad2f8 --- /dev/null +++ b/tests/dfx/perf/tests/test_hunyuan_image_tp4_fp8.json @@ -0,0 +1,36 @@ +[ + { + "test_name": "test_hunyuan_image_tp4_fp8", + "description": "TP=4 Quantization=fp8 baseline", + "server_type": "vllm-omni", + "server_params": { + "model": "tencent/HunyuanImage-3.0-Instruct", + "serve_args": { + "tensor-parallel-size": 4, + "quantization": "fp8", + "distributed-executor-backend": "mp", + "enforce-eager": true, + "enable-diffusion-pipeline-profiler": true + } + }, + "benchmark_params": [ + { + "name": "1024x1024_steps50", + "dataset": "random", + "task": "t2i", + "width": 1024, + "height": 1024, + "num-inference-steps": 50, + "num-prompts": 10, + "max-concurrency": 1, + "skip-performance-assertion": true, + "baseline": { + "throughput_qps": 0.08, + "latency_p99": 13.1227, + "peak_memory_mb_max": 46838, + "peak_memory_mb_mean": 46838 + } + } + ] + } +] diff --git a/vllm_omni/diffusion/models/hunyuan_image3/pipeline_hunyuan_image3.py b/vllm_omni/diffusion/models/hunyuan_image3/pipeline_hunyuan_image3.py index 84a7787ad..dbbad4e71 100644 --- a/vllm_omni/diffusion/models/hunyuan_image3/pipeline_hunyuan_image3.py +++ b/vllm_omni/diffusion/models/hunyuan_image3/pipeline_hunyuan_image3.py @@ -80,6 +80,8 @@ class HunyuanImage3Pipeline(HunyuanImage3PreTrainedModel, GenerationMixin, Diffu "model.layers[0].mlp.forward", "vae.encode", "vae.decode", + "patch_embed.forward", + "final_layer.forward", ] def __init__(self, od_config: OmniDiffusionConfig) -> None: