Files
vllm-omni/tests/dfx/perf/tests/test_qwen_image_vllm_omni.json
T
1761253005 [CI] Update perf json baselines (#4289)
Signed-off-by: Alicia <115451386+congw729@users.noreply.github.com>
Signed-off-by: amy-why-3459 <wuhaiyan17@huawei.com>
Signed-off-by: herotai214 <herotai214@gmail.com>
Signed-off-by: bjf-frz <frz123db@gmail.com>
Signed-off-by: dengyunyang <584797741@qq.com>
Signed-off-by: Huang, Zeyu <11222265+fhfuih@users.noreply.github.com>
Signed-off-by: CHEN <116010019@link.cuhk.edu.cn>
Signed-off-by: NumberWan <wantszkin2003@gmail.com>
Co-authored-by: amy-why-3459 <wuhaiyan17@huawei.com>
Co-authored-by: herotai214 <herotai214@gmail.com>
Co-authored-by: bjf-frz <frz123db@gmail.com>
Co-authored-by: dengyunyang <584797741@qq.com>
Co-authored-by: Huang, Zeyu <11222265+fhfuih@users.noreply.github.com>
Co-authored-by: CHEN <116010019@link.cuhk.edu.cn>
Co-authored-by: rein yang <73573651+R2-Y@users.noreply.github.com>
Co-authored-by: NumberWan <wantszkin2003@gmail.com>
2026-06-17 14:02:50 +08:00

220 lines
7.5 KiB
JSON

[
{
"test_name": "test_qwen_image_single_device",
"description": "Single-device baseline (no parallelism)",
"server_type": "vllm-omni",
"server_params": {
"model": "Qwen/Qwen-Image",
"serve_args": {
"enable-diffusion-pipeline-profiler": true
}
},
"benchmark_params": [
{
"name": "512x512_steps20",
"dataset": "random",
"task": "t2i",
"width": 512,
"height": 512,
"num-inference-steps": 20,
"num-prompts": 10,
"max-concurrency": 1,
"enable-negative-prompt": true,
"baseline": {
"throughput_qps": 0.4153,
"latency_mean": 2.4065,
"peak_memory_mb_mean": 52289.9
}
},
{
"name": "1536x1536_steps35",
"dataset": "random",
"task": "t2i",
"width": 1536,
"height": 1536,
"num-inference-steps": 35,
"num-prompts": 10,
"max-concurrency": 1,
"enable-negative-prompt": true,
"baseline": {
"throughput_qps": 0.0423,
"latency_mean": 23.5991,
"peak_memory_mb_mean": 61360.5
}
}
]
},
{
"test_name": "test_qwen_image_single_device_step_execution",
"description": "Single-device baseline (no parallelism) with step execution",
"server_type": "vllm-omni",
"server_params": {
"model": "Qwen/Qwen-Image",
"serve_args": {
"enable-diffusion-pipeline-profiler": true,
"step-execution": true
}
},
"benchmark_params": [
{
"name": "512x512_steps20",
"dataset": "random",
"task": "t2i",
"width": 512,
"height": 512,
"num-inference-steps": 20,
"num-prompts": 10,
"max-concurrency": 1,
"enable-negative-prompt": true,
"baseline": {
"throughput_qps": 0.4175,
"latency_mean": 2.3953,
"peak_memory_mb_mean": 56000
}
},
{
"name": "1536x1536_steps35",
"dataset": "random",
"task": "t2i",
"width": 1536,
"height": 1536,
"num-inference-steps": 35,
"num-prompts": 10,
"max-concurrency": 1,
"enable-negative-prompt": true,
"baseline": {
"throughput_qps": 0.0423,
"latency_mean": 23.5768,
"peak_memory_mb_mean": 65000
}
}
]
},
{
"test_name": "test_qwen_image_ulysses2_cfg2_vae_patch4",
"description": "Ulysses SP=2 + CFG-parallel=2 + VAE Patch Parallel=4",
"server_type": "vllm-omni",
"server_params": {
"model": "Qwen/Qwen-Image",
"serve_args": {
"ulysses-degree": 2,
"cfg-parallel-size": 2,
"vae-patch-parallel-size": 4,
"vae-use-tiling": true,
"enable-diffusion-pipeline-profiler": true
}
},
"benchmark_params": [
{
"name": "1536x1536_steps35",
"dataset": "random",
"task": "t2i",
"width": 1536,
"height": 1536,
"num-inference-steps": 35,
"num-prompts": 10,
"max-concurrency": 1,
"enable-negative-prompt": true,
"baseline": {
"throughput_qps": 0.1214,
"latency_mean": 8.2211,
"peak_memory_mb_mean": 51704.7
}
}
]
},
{
"test_name": "test_qwen_image_ulysses2_cfg2_cache_dit",
"description": "Ulysses SP=2 + CFG-parallel=2 + CacheDiT acceleration",
"server_type": "vllm-omni",
"server_params": {
"model": "Qwen/Qwen-Image",
"serve_args": {
"ulysses-degree": 2,
"cfg-parallel-size": 2,
"cache-backend": "cache_dit",
"cache-config": {
"Fn_compute_blocks": 1,
"Bn_compute_blocks": 0,
"max_warmup_steps": 4,
"residual_diff_threshold": 0.24,
"max_continuous_cached_steps": 3,
"enable_taylorseer": false,
"taylorseer_order": 1,
"scm_steps_mask_policy": null,
"scm_steps_policy": "dynamic"
},
"enable-diffusion-pipeline-profiler": true
}
},
"benchmark_params": [
{
"name": "512x512_steps20",
"dataset": "random",
"task": "t2i",
"width": 512,
"height": 512,
"num-inference-steps": 20,
"num-prompts": 10,
"max-concurrency": 1,
"enable-negative-prompt": true,
"baseline": {
"throughput_qps": 0.559,
"latency_mean": 1.794,
"peak_memory_mb_mean": 52397.6775
}
},
{
"name": "1536x1536_steps35",
"dataset": "random",
"task": "t2i",
"width": 1536,
"height": 1536,
"num-inference-steps": 35,
"num-prompts": 10,
"max-concurrency": 1,
"enable-negative-prompt": true,
"baseline": {
"throughput_qps": 0.1918,
"latency_mean": 5.2022,
"peak_memory_mb_mean": 61468.8
}
}
]
},
{
"test_name": "test_qwen_image_single_device_step_execution_high_concurrency",
"description": "Single-device Qwen-Image step-level execution benchmark at 512x512, 20 steps, concurrency sweep",
"server_type": "vllm-omni",
"server_params": {
"model": "Qwen/Qwen-Image",
"serve_args": {
"enable-diffusion-pipeline-profiler": true,
"step-execution": true,
"max-num-seqs": 8
}
},
"benchmark_params": [
{
"name": "512x512_steps20_high_concurrency",
"dataset": "random",
"task": "t2i",
"width": 512,
"height": 512,
"num-inference-steps": 20,
"num-prompts": [20, 40, 80, 160],
"max-concurrency": [1, 2, 4, 8],
"warmup-requests": 8,
"warmup-concurrency": 8,
"warmup-num-inference-steps": 20,
"enable-negative-prompt": true,
"baseline": {
"throughput_qps": [0.4125, 0.6714, 0.6674, 0.8159],
"latency_mean": [2.4215, 2.992, 5.9293, 9.656],
"peak_memory_mb_mean": [67000, 67000, 67000, 67000]
}
}
]
}
]