anodyine/vllm-omni
1
0 Fork 0
Code Issues 4 Pull Requests 2 Actions Packages Projects Releases Wiki Activity

[CI/Build] Update perf baselines base on 8/1-8/7 7 days avg (#6201)

Browse Source 
Signed-off-by: Alicia <115451386+congw729@users.noreply.github.com>
This commit is contained in:
Alicia authored and GitHub committed 2026-08-20 18:56:23 +00:00
1 parent b3270dec8d
commit b2011a3edd
14 files changed
+134 -394
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

tests/dfx/perf/tests/test_bagel_vllm_omni.json
+8 -8
View File
@@ -43,8 +43,8 @@
"H100": {
"throughput_qps": 0.495,
"latency_mean": 2.0238,
"peak_memory_mb_max": 29393.4286,
"peak_memory_mb_mean": 29392.7143
"peak_memory_mb_max": 29392.0,
"peak_memory_mb_mean": 29392.0
}
}
}
@@ -93,10 +93,10 @@
"num-input-images": 1,
"baseline": {
"H100": {
"throughput_qps": 0.3451,
"latency_mean": 2.9031,
"peak_memory_mb_max": 30480.2857,
"peak_memory_mb_mean": 30479.2857
"throughput_qps": 0.3499,
"latency_mean": 2.8701,
"peak_memory_mb_max": 30466.0,
"peak_memory_mb_mean": 30465.0
}
}
}
@@ -196,8 +196,8 @@
"num-input-images": 1,
"baseline": {
"H100": {
"throughput_qps": 0.149,
"latency_mean": 6.7157,
"throughput_qps": 0.1533,
"latency_mean": 6.5243,
"peak_memory_mb_max": 30574.0,
"peak_memory_mb_mean": 30574.0
}
tests/dfx/perf/tests/test_cosmos3_vllm_omni.json
+11 -11
View File
@@ -49,8 +49,8 @@
},
"baseline": {
"H100": {
"throughput_qps": 1.107,
"latency_mean": 0.8929,
"throughput_qps": 1.1958,
"latency_mean": 0.843,
"peak_memory_mb_mean": 80000.0
}
}
@@ -118,9 +118,9 @@
},
"baseline": {
"H100": {
"throughput_qps": 0.027,
"latency_mean": 36.6824,
"peak_memory_mb_mean": 30155.4
"throughput_qps": 0.0307,
"latency_mean": 32.7665,
"peak_memory_mb_mean": 28762.4762
}
}
}
@@ -186,9 +186,9 @@
},
"baseline": {
"H100": {
"throughput_qps": 0.027,
"latency_mean": 36.968,
"peak_memory_mb_mean": 30443.6
"throughput_qps": 0.0305,
"latency_mean": 33.1009,
"peak_memory_mb_mean": 29441.1429
}
}
}
@@ -258,9 +258,9 @@
},
"baseline": {
"H100": {
"throughput_qps": 0.027,
"latency_mean": 37.2117,
"peak_memory_mb_mean": 30135.6
"throughput_qps": 0.0299,
"latency_mean": 33.925,
"peak_memory_mb_mean": 29443.1429
}
}
}
tests/dfx/perf/tests/test_hunyuan_image3_it2i.json
+1 -3
View File
@@ -137,9 +137,7 @@
"H200": {
"throughput_qps": 0.1874,
"latency_p99": 101.852,
"latency_mean": 79.0385,
"peak_memory_mb_max": 70000,
"peak_memory_mb_mean": 70000
"latency_mean": 79.0385
}
}
}
tests/dfx/perf/tests/test_minicpmo_4_5.json
+15 -15
View File
@@ -56,29 +56,29 @@
"baseline": {
"H100": {
"request_throughput": [
0.7803,
1.1008,
1.2027
0.9731,
1.1986,
1.686
],
"mean_ttft_ms": [
166.9764,
307.7133,
323.584
152.5665,
270.2559,
301.455
],
"mean_e2el_ms": [
1361.1682,
3800.6332,
6961.8636
1027.2953,
3306.3735,
4686.5366
],
"mean_audio_ttfp_ms": [
635.8099,
1615.6591,
2888.9831
458.1553,
2030.2335,
2068.6233
],
"mean_audio_rtf": [
0.3151,
0.8568,
1.5425
0.2357,
0.791,
1.0324
]
},
"A3": {
tests/dfx/perf/tests/test_minicpmo_4_5_duplex_seed_tts.json
+5 -5
View File
@@ -47,19 +47,19 @@
"baseline": {
"H100": {
"request_throughput": [
0.1594
0.3038
],
"mean_ttft_ms": [
274.164
139.997
],
"mean_e2el_ms": [
6250.7696
3269.8807
],
"mean_audio_ttfp_ms": [
994.422
456.073
],
"mean_audio_rtf": [
0.46163
0.2254
]
},
"A3": {
tests/dfx/perf/tests/test_minicpmo_4_5_ready.json
+10 -10
View File
@@ -54,24 +54,24 @@
"baseline": {
"H100": {
"request_throughput": [
0.7803,
1.1008
0.9731,
1.1986
],
"mean_ttft_ms": [
166.9764,
307.7133
152.5665,
270.2559
],
"mean_e2el_ms": [
1361.1682,
3800.6332
1027.2953,
3306.3735
],
"mean_audio_ttfp_ms": [
635.8099,
1615.6591
458.1553,
2030.2335
],
"mean_audio_rtf": [
0.3151,
0.8568
0.2357,
0.791
]
}
}
tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json
+20 -52
View File
@@ -45,26 +45,19 @@
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
"baseline": {
"H100": {
"mean_ttft_ms": [
96.4061,
140.8147,
271.9025,
362.2916,
507.776
],
"mean_e2el_ms": [
18507.4582,
28365.4956,
11701.3818,
27510.3956,
31907.9613,
48161.5829,
72630.8601
42173.9832,
63970.0751
],
"mean_audio_rtf": [
0.1175,
0.18,
0.2359,
0.3278,
0.6296
0.1647,
0.2003,
0.2726,
0.4195
]
}
}
@@ -94,14 +87,11 @@
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
"baseline": {
"H100": {
"mean_ttft_ms": [
354.0951
],
"mean_e2el_ms": [
3952.5615
3646.797
],
"mean_audio_rtf": [
0.117
0.1098
]
}
}
@@ -133,14 +123,11 @@
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
"baseline": {
"H100": {
"mean_ttft_ms": [
259.6317
],
"mean_e2el_ms": [
4253.3009
4059.0878
],
"mean_audio_rtf": [
0.1448
0.1335
]
}
}
@@ -174,14 +161,11 @@
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
"baseline": {
"H100": {
"mean_ttft_ms": [
457.4453
],
"mean_e2el_ms": [
5912.5623
5639.5769
],
"mean_audio_rtf": [
0.1967
0.1798
]
}
}
@@ -215,19 +199,12 @@
"percentile-metrics": "ttft,tpot,itl,e2el",
"baseline": {
"H100": {
"mean_ttft_ms": [
90.8033,
202.1779,
326.5864,
544.8914,
961.3929
],
"mean_e2el_ms": [
4517.3411,
7857.9711,
4354.8945,
7672.4248,
9861.1687,
13283.3363,
19541.0736
13086.8865,
18716.709
]
}
}
@@ -262,11 +239,8 @@
"percentile-metrics": "ttft,tpot,itl,e2el",
"baseline": {
"H100": {
"mean_ttft_ms": [
318.048
],
"mean_e2el_ms": [
807.6114
756.8415
]
}
}
@@ -303,11 +277,8 @@
"percentile-metrics": "ttft,tpot,itl,e2el",
"baseline": {
"H100": {
"mean_ttft_ms": [
186.168
],
"mean_e2el_ms": [
726.1137
694.9777
]
}
}
@@ -346,9 +317,6 @@
"percentile-metrics": "ttft,tpot,itl,e2el",
"baseline": {
"H100": {
"mean_ttft_ms": [
375.5282
],
"mean_e2el_ms": [
1006.1663
]
tests/dfx/perf/tests/test_qwen3_omni_multi_replicas.json
+9 -13
View File
@@ -36,10 +36,9 @@
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
"baseline": {
"H100": {
"mean_ttft_ms": [130.5751, 278.6469, 324.7249, 911.6085],
"mean_tpot_ms": [7.9854, 11.1209, 16.1507, 20.7485],
"mean_audio_ttfp_ms": [486.486, 954.1791, 1289.6514, 2196.8275],
"mean_audio_rtf": [0.1896, 0.3962, 0.4098, 0.7322]
"mean_tpot_ms": [7.6955, 10.6457, 12.7363, 15.1608],
"mean_audio_ttfp_ms": [325.1624, 602.5192, 699.3891, 1024.088],
"mean_audio_rtf": [0.1689, 0.2088, 0.2397, 0.2791]
}
}
},
@@ -64,9 +63,8 @@
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
"baseline": {
"H100": {
"mean_ttft_ms": [340.5122],
"mean_audio_ttfp_ms": [463.0752],
"mean_audio_rtf": [0.1153]
"mean_audio_ttfp_ms": [436.4663],
"mean_audio_rtf": [0.1069]
}
}
},
@@ -93,9 +91,8 @@
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
"baseline": {
"H100": {
"mean_ttft_ms": [236.926],
"mean_audio_ttfp_ms": [395.2182],
"mean_audio_rtf": [0.1259]
"mean_audio_ttfp_ms": [376.4911],
"mean_audio_rtf": [0.1145]
}
}
},
@@ -124,9 +121,8 @@
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
"baseline": {
"H100": {
"mean_ttft_ms": [470.6126],
"mean_audio_ttfp_ms": [665.5474],
"mean_audio_rtf": [0.166]
"mean_audio_ttfp_ms": [638.2227],
"mean_audio_rtf": [0.1438]
}
}
}
tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json
+15 -31
View File
@@ -45,26 +45,19 @@
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
"baseline": {
"H100": {
"mean_ttft_ms": [
115.8348,
173.1823,
446.672,
464.5985,
3420.0634
],
"mean_e2el_ms": [
25070.9523,
31992.0646,
37619.7669,
50393.8257,
136619.2743
15793.1209,
30658.6632,
37463.4943,
47990.5839,
64980.2199
],
"mean_audio_rtf": [
0.169,
0.2111,
0.2475,
0.3551,
0.9266
0.2041,
0.2382,
0.3177,
0.4595
]
}
}
@@ -94,14 +87,11 @@
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
"baseline": {
"H100": {
"mean_ttft_ms": [
343.1377
],
"mean_e2el_ms": [
4811.1123
4556.4347
],
"mean_audio_rtf": [
0.1463
0.138
]
}
}
@@ -133,14 +123,11 @@
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
"baseline": {
"H100": {
"mean_ttft_ms": [
245.5446
],
"mean_e2el_ms": [
5019.1256
4768.0972
],
"mean_audio_rtf": [
0.1713
0.1609
]
}
}
@@ -174,14 +161,11 @@
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
"baseline": {
"H100": {
"mean_ttft_ms": [
506.1407
],
"mean_e2el_ms": [
6814.9656
6395.3517
],
"mean_audio_rtf": [
0.2213
0.2048
]
}
}
tests/dfx/perf/tests/test_qwen3_omni_vllm_text.json
+8 -24
View File
@@ -52,19 +52,12 @@
"percentile-metrics": "ttft,tpot,itl,e2el",
"baseline": {
"H100": {
"mean_ttft_ms": [
69.8081,
173.0651,
271.8108,
413.4079,
497.2546
],
"mean_e2el_ms": [
4720.8225,
7937.2532,
9962.3566,
13724.5479,
18324.091
4619.0068,
7868.3523,
9760.6175,
13422.3491,
18007.6918
]
}
}
@@ -100,11 +93,8 @@
"percentile-metrics": "ttft,tpot,itl,e2el",
"baseline": {
"H100": {
"mean_ttft_ms": [
25.9477
],
"mean_e2el_ms": [
517.3155
491.2534
]
}
}
@@ -142,11 +132,8 @@
"percentile-metrics": "ttft,tpot,itl,e2el",
"baseline": {
"H100": {
"mean_ttft_ms": [
26.7056
],
"mean_e2el_ms": [
551.308
524.5321
]
}
}
@@ -186,11 +173,8 @@
"percentile-metrics": "ttft,tpot,itl,e2el",
"baseline": {
"H100": {
"mean_ttft_ms": [
27.6906
],
"mean_e2el_ms": [
596.5284
559.6674
]
}
}
tests/dfx/perf/tests/test_qwen_image_edit_2511_vllm_omni.json
+20 -20
View File
@@ -35,10 +35,10 @@
"enable-negative-prompt": true,
"baseline": {
"H100": {
"throughput_qps": 0.0721,
"latency_mean": 13.8398,
"peak_memory_mb_max": 57610.0,
"peak_memory_mb_mean": 57610.0
"throughput_qps": 0.0684,
"latency_mean": 14.6249,
"peak_memory_mb_max": 57632.0,
"peak_memory_mb_mean": 57632.0
}
}
},
@@ -55,10 +55,10 @@
"enable-negative-prompt": true,
"baseline": {
"H100": {
"throughput_qps": 0.0186,
"latency_mean": 53.6942,
"peak_memory_mb_max": 67790.0,
"peak_memory_mb_mean": 67790.0
"throughput_qps": 0.0177,
"latency_mean": 56.5285,
"peak_memory_mb_max": 67840.0,
"peak_memory_mb_mean": 67840.0
}
}
}
@@ -105,10 +105,10 @@
"enable-negative-prompt": true,
"baseline": {
"H100": {
"throughput_qps": 0.0538,
"latency_mean": 18.5591,
"peak_memory_mb_max": 56449.4286,
"peak_memory_mb_mean": 56449.4286
"throughput_qps": 0.0515,
"latency_mean": 19.4345,
"peak_memory_mb_max": 56348.0,
"peak_memory_mb_mean": 56348.0
}
}
}
@@ -164,10 +164,10 @@
"enable-negative-prompt": true,
"baseline": {
"H100": {
"throughput_qps": 0.2776,
"latency_mean": 3.5938,
"peak_memory_mb_max": 58113.7143,
"peak_memory_mb_mean": 58075.1429
"throughput_qps": 0.2479,
"latency_mean": 4.0389,
"peak_memory_mb_max": 57750.0,
"peak_memory_mb_mean": 57750.0
}
}
},
@@ -184,10 +184,10 @@
"enable-negative-prompt": true,
"baseline": {
"H100": {
"throughput_qps": 0.1051,
"latency_mean": 9.4918,
"peak_memory_mb_max": 68107.4286,
"peak_memory_mb_mean": 68107.4286
"throughput_qps": 0.0948,
"latency_mean": 10.5462,
"peak_memory_mb_max": 67764.0,
"peak_memory_mb_mean": 67764.0
}
}
}
tests/dfx/perf/tests/test_qwen_image_edit_vllm_omni.json
-190
View File
@@ -1,190 +0,0 @@
[
{
"test_name": "test_qwen_image_edit_single_device",
"mark": [
{
"hardware_marks": {
"res": {
"cuda": "H100"
},
"num_cards": 1
}
},
"local_model",
"diffusion"
],
"description": "Single-device baseline",
"server_type": "vllm-omni",
"server_params": {
"model": "Qwen/Qwen-Image-Edit",
"serve_args": {
"enable-diffusion-pipeline-profiler": true
}
},
"benchmark_params": [
{
"name": "512x512_steps20_i2i",
"dataset": "random",
"task": "i2i",
"width": 512,
"height": 512,
"num-inference-steps": 20,
"num-prompts": 10,
"max-concurrency": 1,
"enable-negative-prompt": true,
"baseline": {
"H100": {
"throughput_qps": 0.1129,
"latency_mean": 8.8352,
"peak_memory_mb_max": 58572.0,
"peak_memory_mb_mean": 58572.0
}
}
},
{
"name": "1536x1536_steps35_i2i",
"dataset": "random",
"task": "i2i",
"width": 1536,
"height": 1536,
"num-inference-steps": 35,
"num-prompts": 10,
"max-concurrency": 1,
"enable-negative-prompt": true,
"baseline": {
"H100": {
"throughput_qps": 0.024,
"latency_mean": 41.5686,
"peak_memory_mb_max": 66368.0,
"peak_memory_mb_mean": 66368.0
}
}
}
]
},
{
"test_name": "test_qwen_image_edit_ulysses2_cfg2_vae_patch4",
"mark": [
{
"hardware_marks": {
"res": {
"cuda": "H100"
},
"num_cards": 4
}
},
"local_model",
"diffusion"
],
"description": "Ulysses SP=2 + CFG=2 + VAE patch parallel=4",
"server_type": "vllm-omni",
"server_params": {
"model": "Qwen/Qwen-Image-Edit",
"serve_args": {
"ulysses-degree": 2,
"cfg-parallel-size": 2,
"vae-patch-parallel-size": 4,
"vae-use-tiling": true,
"enable-diffusion-pipeline-profiler": true
}
},
"benchmark_params": [
{
"name": "1536x1536_steps35_i2i",
"dataset": "random",
"task": "i2i",
"width": 1536,
"height": 1536,
"num-inference-steps": 35,
"num-prompts": 10,
"max-concurrency": 1,
"enable-negative-prompt": true,
"baseline": {
"H100": {
"throughput_qps": 0.0672,
"latency_mean": 14.8517,
"peak_memory_mb_max": 57076.0,
"peak_memory_mb_mean": 57076.0
}
}
}
]
},
{
"test_name": "test_qwen_image_edit_ulysses2_cfg2_cache_dit",
"mark": [
{
"hardware_marks": {
"res": {
"cuda": "H100"
},
"num_cards": 4
}
},
"local_model",
"diffusion"
],
"description": "Ulysses SP=2 + CFG=2 + CacheDiT",
"server_type": "vllm-omni",
"server_params": {
"model": "Qwen/Qwen-Image-Edit",
"serve_args": {
"ulysses-degree": 2,
"cfg-parallel-size": 2,
"cache-backend": "cache_dit",
"cache-config": {
"Fn_compute_blocks": 1,
"Bn_compute_blocks": 0,
"max_warmup_steps": 4,
"residual_diff_threshold": 0.24,
"max_continuous_cached_steps": 3,
"enable_taylorseer": false,
"taylorseer_order": 1,
"scm_steps_mask_policy": null,
"scm_steps_policy": "dynamic"
},
"enable-diffusion-pipeline-profiler": true
}
},
"benchmark_params": [
{
"name": "512x512_steps20_i2i",
"dataset": "random",
"task": "i2i",
"width": 512,
"height": 512,
"num-inference-steps": 20,
"num-prompts": 10,
"max-concurrency": 1,
"enable-negative-prompt": true,
"baseline": {
"H100": {
"throughput_qps": 0.2879,
"latency_mean": 3.4701,
"peak_memory_mb_max": 58758.0,
"peak_memory_mb_mean": 58758.0
}
}
},
{
"name": "1536x1536_steps35_i2i",
"dataset": "random",
"task": "i2i",
"width": 1536,
"height": 1536,
"num-inference-steps": 35,
"num-prompts": 10,
"max-concurrency": 1,
"enable-negative-prompt": true,
"baseline": {
"H100": {
"throughput_qps": 0.1113,
"latency_mean": 8.9656,
"peak_memory_mb_max": 67424.0,
"peak_memory_mb_mean": 67424.0
}
}
}
]
}
]
tests/dfx/perf/tests/test_qwen_image_layered_vllm_omni.json
+2 -2
View File
@@ -55,8 +55,8 @@
"H100": {
"throughput_qps": 0.0417,
"latency_mean": 23.914,
"peak_memory_mb_max": 63225.1429,
"peak_memory_mb_mean": 63225.1429
"peak_memory_mb_max": 62912.0,
"peak_memory_mb_mean": 62912.0
}
}
}
tests/dfx/perf/tests/test_qwen_image_vllm_omni.json
+10 -10
View File
@@ -36,7 +36,7 @@
"H100": {
"throughput_qps": 0.4153,
"latency_mean": 2.4065,
"peak_memory_mb_mean": 55121.4286
"peak_memory_mb_mean": 55062.0
}
}
},
@@ -54,7 +54,7 @@
"H100": {
"throughput_qps": 0.0423,
"latency_mean": 23.5991,
"peak_memory_mb_mean": 65118.0
"peak_memory_mb_mean": 64546.0
}
}
}
@@ -98,7 +98,7 @@
"H100": {
"throughput_qps": 0.4175,
"latency_mean": 2.3953,
"peak_memory_mb_mean": 55088.5714
"peak_memory_mb_mean": 54952.0
}
}
},
@@ -116,7 +116,7 @@
"H100": {
"throughput_qps": 0.0423,
"latency_mean": 23.5768,
"peak_memory_mb_mean": 66012.2857
"peak_memory_mb_mean": 64500.0
}
}
}
@@ -163,7 +163,7 @@
"H100": {
"throughput_qps": 0.1214,
"latency_mean": 8.2211,
"peak_memory_mb_mean": 54501.1429
"peak_memory_mb_mean": 54474.0
}
}
}
@@ -220,7 +220,7 @@
"H100": {
"throughput_qps": 0.559,
"latency_mean": 1.794,
"peak_memory_mb_mean": 55290.5714
"peak_memory_mb_mean": 55174.0
}
}
},
@@ -238,7 +238,7 @@
"H100": {
"throughput_qps": 0.1918,
"latency_mean": 5.2022,
"peak_memory_mb_mean": 64838.8571
"peak_memory_mb_mean": 64750.0
}
}
}
@@ -284,9 +284,9 @@
"enable-negative-prompt": true,
"baseline": {
"H100": {
"throughput_qps": [0.4125, 0.6714, 0.6674, 0.8159],
"latency_mean": [2.4215, 2.992, 5.9293, 9.656],
"peak_memory_mb_mean": [55147.4286, 55148.0, 55150.4643, 55152.3821]
"throughput_qps": [0.3937, 0.6606, 0.6188, 0.7752],
"latency_mean": [2.5466, 3.0235, 6.3948, 10.1944],
"peak_memory_mb_mean": [55149.0, 55150.0, 55153.1458, 55156.325]
}
}
}