361 lines
8.4 KiB
JSON
361 lines
8.4 KiB
JSON
[
|
|
{
|
|
"test_name": "test_qwen3_omni_chunk",
|
|
"mark": [
|
|
{
|
|
"hardware_marks": {
|
|
"res": {
|
|
"cuda": "H100",
|
|
"npu": "A3"
|
|
},
|
|
"num_cards": 2
|
|
}
|
|
},
|
|
"full_model",
|
|
"omni"
|
|
],
|
|
"server_params": {
|
|
"model": "Qwen/Qwen3-Omni-30B-A3B-Instruct",
|
|
"extra_cli_args": [
|
|
"--async-chunk"
|
|
]
|
|
},
|
|
"benchmark_params": [
|
|
{
|
|
"dataset_name": "random",
|
|
"backend": "openai-chat-omni",
|
|
"endpoint": "/v1/chat/completions",
|
|
"num_prompts": [
|
|
4,
|
|
16,
|
|
32,
|
|
64,
|
|
128
|
|
],
|
|
"max_concurrency": [
|
|
1,
|
|
4,
|
|
8,
|
|
16,
|
|
32
|
|
],
|
|
"random_input_len": 2500,
|
|
"random_output_len": 900,
|
|
"ignore_eos": true,
|
|
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
|
|
"baseline": {
|
|
"H100": {
|
|
"mean_ttft_ms": [
|
|
96.4061,
|
|
140.8147,
|
|
271.9025,
|
|
362.2916,
|
|
507.776
|
|
],
|
|
"mean_e2el_ms": [
|
|
18507.4582,
|
|
28365.4956,
|
|
31907.9613,
|
|
48161.5829,
|
|
72630.8601
|
|
],
|
|
"mean_audio_rtf": [
|
|
0.1175,
|
|
0.18,
|
|
0.2359,
|
|
0.3278,
|
|
0.6296
|
|
]
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"dataset_name": "random-mm",
|
|
"backend": "openai-chat-omni",
|
|
"endpoint": "/v1/chat/completions",
|
|
"num_prompts": [
|
|
10
|
|
],
|
|
"request_rate": [
|
|
0.1
|
|
],
|
|
"random_input_len": 100,
|
|
"random_output_len": 100,
|
|
"random_range_ratio": 0.0,
|
|
"ignore_eos": true,
|
|
"random_mm_base_items_per_request": 1,
|
|
"random_mm_num_mm_items_range_ratio": 0.5,
|
|
"random_mm_limit_mm_per_prompt": {
|
|
"audio": 1
|
|
},
|
|
"random_mm_bucket_config": {
|
|
"(0, 60, 3)": 1.0
|
|
},
|
|
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
|
|
"baseline": {
|
|
"H100": {
|
|
"mean_ttft_ms": [
|
|
354.0951
|
|
],
|
|
"mean_e2el_ms": [
|
|
3952.5615
|
|
],
|
|
"mean_audio_rtf": [
|
|
0.117
|
|
]
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"dataset_name": "random-mm",
|
|
"backend": "openai-chat-omni",
|
|
"endpoint": "/v1/chat/completions",
|
|
"num_prompts": [
|
|
40
|
|
],
|
|
"request_rate": [
|
|
0.5
|
|
],
|
|
"random_input_len": 100,
|
|
"random_output_len": 100,
|
|
"random_range_ratio": 0.0,
|
|
"ignore_eos": true,
|
|
"random_mm_base_items_per_request": 2,
|
|
"random_mm_num_mm_items_range_ratio": 0.5,
|
|
"random_mm_limit_mm_per_prompt": {
|
|
"image": 1,
|
|
"video": 1
|
|
},
|
|
"random_mm_bucket_config": {
|
|
"(256, 256, 1)": 0.5,
|
|
"(720, 1280, 2)": 0.5
|
|
},
|
|
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
|
|
"baseline": {
|
|
"H100": {
|
|
"mean_ttft_ms": [
|
|
259.6317
|
|
],
|
|
"mean_e2el_ms": [
|
|
4253.3009
|
|
],
|
|
"mean_audio_rtf": [
|
|
0.1448
|
|
]
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"dataset_name": "random-mm",
|
|
"backend": "openai-chat-omni",
|
|
"endpoint": "/v1/chat/completions",
|
|
"num_prompts": [
|
|
100
|
|
],
|
|
"request_rate": [
|
|
1.0
|
|
],
|
|
"random_input_len": 100,
|
|
"random_output_len": 100,
|
|
"random_range_ratio": 0.0,
|
|
"ignore_eos": true,
|
|
"random_mm_base_items_per_request": 3,
|
|
"random_mm_num_mm_items_range_ratio": 0.5,
|
|
"random_mm_limit_mm_per_prompt": {
|
|
"image": 1,
|
|
"video": 1,
|
|
"audio": 1
|
|
},
|
|
"random_mm_bucket_config": {
|
|
"(256, 256, 1)": 0.34,
|
|
"(720, 1280, 2)": 0.33,
|
|
"(0, 60, 3)": 0.33
|
|
},
|
|
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
|
|
"baseline": {
|
|
"H100": {
|
|
"mean_ttft_ms": [
|
|
457.4453
|
|
],
|
|
"mean_e2el_ms": [
|
|
5912.5623
|
|
],
|
|
"mean_audio_rtf": [
|
|
0.1967
|
|
]
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"dataset_name": "random",
|
|
"backend": "openai-chat-omni",
|
|
"endpoint": "/v1/chat/completions",
|
|
"num_prompts": [
|
|
4,
|
|
16,
|
|
32,
|
|
64,
|
|
128
|
|
],
|
|
"max_concurrency": [
|
|
1,
|
|
4,
|
|
8,
|
|
16,
|
|
32
|
|
],
|
|
"random_input_len": 2500,
|
|
"random_output_len": 900,
|
|
"ignore_eos": true,
|
|
"extra_body": {
|
|
"modalities": [
|
|
"text"
|
|
]
|
|
},
|
|
"percentile-metrics": "ttft,tpot,itl,e2el",
|
|
"baseline": {
|
|
"H100": {
|
|
"mean_ttft_ms": [
|
|
90.8033,
|
|
202.1779,
|
|
326.5864,
|
|
544.8914,
|
|
961.3929
|
|
],
|
|
"mean_e2el_ms": [
|
|
4517.3411,
|
|
7857.9711,
|
|
9861.1687,
|
|
13283.3363,
|
|
19541.0736
|
|
]
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"dataset_name": "random-mm",
|
|
"backend": "openai-chat-omni",
|
|
"endpoint": "/v1/chat/completions",
|
|
"num_prompts": [
|
|
10
|
|
],
|
|
"request_rate": [
|
|
0.1
|
|
],
|
|
"random_input_len": 100,
|
|
"random_output_len": 100,
|
|
"random_range_ratio": 0.0,
|
|
"ignore_eos": true,
|
|
"extra_body": {
|
|
"modalities": [
|
|
"text"
|
|
]
|
|
},
|
|
"random_mm_base_items_per_request": 1,
|
|
"random_mm_num_mm_items_range_ratio": 0.5,
|
|
"random_mm_limit_mm_per_prompt": {
|
|
"audio": 1
|
|
},
|
|
"random_mm_bucket_config": {
|
|
"(0, 60, 3)": 1.0
|
|
},
|
|
"percentile-metrics": "ttft,tpot,itl,e2el",
|
|
"baseline": {
|
|
"H100": {
|
|
"mean_ttft_ms": [
|
|
318.048
|
|
],
|
|
"mean_e2el_ms": [
|
|
807.6114
|
|
]
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"dataset_name": "random-mm",
|
|
"backend": "openai-chat-omni",
|
|
"endpoint": "/v1/chat/completions",
|
|
"num_prompts": [
|
|
40
|
|
],
|
|
"request_rate": [
|
|
0.5
|
|
],
|
|
"random_input_len": 100,
|
|
"random_output_len": 100,
|
|
"random_range_ratio": 0.0,
|
|
"ignore_eos": true,
|
|
"extra_body": {
|
|
"modalities": [
|
|
"text"
|
|
]
|
|
},
|
|
"random_mm_base_items_per_request": 2,
|
|
"random_mm_num_mm_items_range_ratio": 0.5,
|
|
"random_mm_limit_mm_per_prompt": {
|
|
"image": 1,
|
|
"video": 1
|
|
},
|
|
"random_mm_bucket_config": {
|
|
"(256, 256, 1)": 0.5,
|
|
"(720, 1280, 2)": 0.5
|
|
},
|
|
"percentile-metrics": "ttft,tpot,itl,e2el",
|
|
"baseline": {
|
|
"H100": {
|
|
"mean_ttft_ms": [
|
|
186.168
|
|
],
|
|
"mean_e2el_ms": [
|
|
726.1137
|
|
]
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"dataset_name": "random-mm",
|
|
"backend": "openai-chat-omni",
|
|
"endpoint": "/v1/chat/completions",
|
|
"num_prompts": [
|
|
100
|
|
],
|
|
"request_rate": [
|
|
1.0
|
|
],
|
|
"random_input_len": 100,
|
|
"random_output_len": 100,
|
|
"random_range_ratio": 0.0,
|
|
"ignore_eos": true,
|
|
"extra_body": {
|
|
"modalities": [
|
|
"text"
|
|
]
|
|
},
|
|
"random_mm_base_items_per_request": 3,
|
|
"random_mm_num_mm_items_range_ratio": 0.5,
|
|
"random_mm_limit_mm_per_prompt": {
|
|
"image": 1,
|
|
"video": 1,
|
|
"audio": 1
|
|
},
|
|
"random_mm_bucket_config": {
|
|
"(256, 256, 1)": 0.34,
|
|
"(720, 1280, 2)": 0.33,
|
|
"(0, 60, 3)": 0.33
|
|
},
|
|
"percentile-metrics": "ttft,tpot,itl,e2el",
|
|
"baseline": {
|
|
"H100": {
|
|
"mean_ttft_ms": [
|
|
375.5282
|
|
],
|
|
"mean_e2el_ms": [
|
|
1006.1663
|
|
]
|
|
}
|
|
}
|
|
}
|
|
]
|
|
}
|
|
]
|