From ef3f72b9ae0bee0baf45258abde55bec3ae6752d Mon Sep 17 00:00:00 2001 From: amy-why-3459 Date: Mon, 13 Apr 2026 19:03:13 +0800 Subject: [PATCH] [Tests][Qwen3-Omni]Modify Qwen3-Omni performance test cases (#2600) Signed-off-by: amy-why-3459 --- tests/dfx/perf/scripts/run_benchmark.py | 2 + tests/dfx/perf/tests/test.json | 317 +++++++++++++++++------- 2 files changed, 225 insertions(+), 94 deletions(-) diff --git a/tests/dfx/perf/scripts/run_benchmark.py b/tests/dfx/perf/scripts/run_benchmark.py index c566c2e0a..b64cc0d95 100644 --- a/tests/dfx/perf/scripts/run_benchmark.py +++ b/tests/dfx/perf/scripts/run_benchmark.py @@ -72,6 +72,8 @@ def run_benchmark( ["vllm", "bench", "serve", "--omni"] + args + [ + "--num-warmups", + "2", "--save-result", "--result-dir", os.environ.get("BENCHMARK_DIR", "tests"), diff --git a/tests/dfx/perf/tests/test.json b/tests/dfx/perf/tests/test.json index fe7e38046..159e27a06 100644 --- a/tests/dfx/perf/tests/test.json +++ b/tests/dfx/perf/tests/test.json @@ -10,83 +10,97 @@ "dataset_name": "random", "backend": "openai-chat-omni", "endpoint": "/v1/chat/completions", - "num_prompts": [ - 10, - 40, - 100 - ], - "max_concurrency": [ - 1, - 4, - 10 - ], - "random_input_len": 100, - "random_output_len": 100, + "num_prompts": [4, 16, 40], + "max_concurrency": [1, 4, 10], + "random_input_len": 2500, + "random_output_len": 900, "ignore_eos": true, "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { "mean_ttft_ms": [1000, 3000, 5000], - "mean_audio_ttfp_ms": [8000, 10000, 13000], - "mean_audio_rtf": [0.2, 0.25, 0.45] + "mean_audio_ttfp_ms": [30000, 60000, 90000], + "mean_audio_rtf": [0.35, 0.45, 0.55] } }, { "dataset_name": "random-mm", "backend": "openai-chat-omni", "endpoint": "/v1/chat/completions", - "num_prompts": [ - 10, - 40, - 100 - ], - "request_rate": [ - 0.1, - 0.3, - 0.5 - ], + "num_prompts": [10], + "request_rate": [0.1], + "random_input_len": 100, + "random_output_len": 100, + "random_range_ratio": 0.0, + "ignore_eos": true, + "random_mm_base_items_per_request": 1, + "random_mm_num_mm_items_range_ratio": 0.5, + "random_mm_limit_mm_per_prompt": { + "audio": 1 + }, + "random_mm_bucket_config": { + "(0, 60, 3)": 1.0 + }, + "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", + "baseline": { + "mean_ttft_ms": [2000], + "mean_audio_ttfp_ms": [10000], + "mean_audio_rtf": [0.25] + } + }, + { + "dataset_name": "random-mm", + "backend": "openai-chat-omni", + "endpoint": "/v1/chat/completions", + "num_prompts": [40], + "request_rate": [0.3], + "random_input_len": 100, + "random_output_len": 100, + "random_range_ratio": 0.0, + "ignore_eos": true, + "random_mm_base_items_per_request": 2, + "random_mm_num_mm_items_range_ratio": 0.5, + "random_mm_limit_mm_per_prompt": { + "image": 1, + "video": 1 + }, + "random_mm_bucket_config": { + "(256, 256, 1)": 0.5, + "(720, 1280, 2)": 0.5 + }, + "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", + "baseline": { + "mean_ttft_ms": [4000], + "mean_audio_ttfp_ms": [13000], + "mean_audio_rtf": [0.35] + } + }, + { + "dataset_name": "random-mm", + "backend": "openai-chat-omni", + "endpoint": "/v1/chat/completions", + "num_prompts": [100], + "request_rate": [0.5], "random_input_len": 100, "random_output_len": 100, "random_range_ratio": 0.0, "ignore_eos": true, "random_mm_base_items_per_request": 3, - "random_mm_num_mm_items_range_ratio": 0, + "random_mm_num_mm_items_range_ratio": 0.5, "random_mm_limit_mm_per_prompt": { "image": 1, "video": 1, "audio": 1 }, "random_mm_bucket_config": { - "(32, 32, 1)": 0.5, - "(0, 1, 1)": 0.1, - "(32, 32, 2)": 0.4 + "(256, 256, 1)": 0.34, + "(720, 1280, 2)": 0.33, + "(0, 60, 3)": 0.33 }, "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { - "mean_ttft_ms": [2000, 4000, 6000], - "mean_audio_ttfp_ms": [10000, 13000, 15000], - "mean_audio_rtf": [0.25, 0.35, 0.45] - } - }, - { - "dataset_name": "random", - "backend": "openai-chat-omni", - "endpoint": "/v1/chat/completions", - "num_prompts": [ - 4, - 16 - ], - "max_concurrency": [ - 1, - 4 - ], - "random_input_len": 2500, - "random_output_len": 900, - "ignore_eos": true, - "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", - "baseline": { - "mean_ttft_ms": [1000, 3000], - "mean_audio_ttfp_ms": [30000, 60000], - "mean_audio_rtf": [0.35, 0.45] + "mean_ttft_ms": [6000], + "mean_audio_ttfp_ms": [15000], + "mean_audio_rtf": [0.45] } } ] @@ -120,18 +134,10 @@ "dataset_name": "random", "backend": "openai-chat-omni", "endpoint": "/v1/chat/completions", - "num_prompts": [ - 10, - 40, - 100 - ], - "max_concurrency": [ - 1, - 4, - 10 - ], - "random_input_len": 100, - "random_output_len": 100, + "num_prompts": [4, 16, 40], + "max_concurrency": [1, 4, 10], + "random_input_len": 2500, + "random_output_len": 900, "ignore_eos": true, "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { @@ -144,59 +150,182 @@ "dataset_name": "random-mm", "backend": "openai-chat-omni", "endpoint": "/v1/chat/completions", - "num_prompts": [ - 10, - 40, - 100 - ], - "request_rate": [ - 0.1, - 0.3, - 0.5 - ], + "num_prompts": [10], + "request_rate": [0.1], + "random_input_len": 100, + "random_output_len": 100, + "random_range_ratio": 0.0, + "ignore_eos": true, + "random_mm_base_items_per_request": 1, + "random_mm_num_mm_items_range_ratio": 0.5, + "random_mm_limit_mm_per_prompt": { + "audio": 1 + }, + "random_mm_bucket_config": { + "(0, 60, 3)": 1.0 + }, + "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", + "baseline": { + "mean_ttft_ms": [2000], + "mean_audio_ttfp_ms": [2000], + "mean_audio_rtf": [0.25] + } + }, + { + "dataset_name": "random-mm", + "backend": "openai-chat-omni", + "endpoint": "/v1/chat/completions", + "num_prompts": [40], + "request_rate": [0.3], + "random_input_len": 100, + "random_output_len": 100, + "random_range_ratio": 0.0, + "ignore_eos": true, + "random_mm_base_items_per_request": 2, + "random_mm_num_mm_items_range_ratio": 0.5, + "random_mm_limit_mm_per_prompt": { + "image": 1, + "video": 1 + }, + "random_mm_bucket_config": { + "(256, 256, 1)": 0.5, + "(720, 1280, 2)": 0.5 + }, + "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", + "baseline": { + "mean_ttft_ms": [4000], + "mean_audio_ttfp_ms": [4000], + "mean_audio_rtf": [0.4] + } + }, + { + "dataset_name": "random-mm", + "backend": "openai-chat-omni", + "endpoint": "/v1/chat/completions", + "num_prompts": [100], + "request_rate": [0.5], "random_input_len": 100, "random_output_len": 100, "random_range_ratio": 0.0, "ignore_eos": true, "random_mm_base_items_per_request": 3, - "random_mm_num_mm_items_range_ratio": 0, + "random_mm_num_mm_items_range_ratio": 0.5, "random_mm_limit_mm_per_prompt": { "image": 1, "video": 1, "audio": 1 }, "random_mm_bucket_config": { - "(32, 32, 1)": 0.5, - "(0, 1, 1)": 0.1, - "(32, 32, 2)": 0.4 + "(256, 256, 1)": 0.34, + "(720, 1280, 2)": 0.33, + "(0, 60, 3)": 0.33 }, "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { - "mean_ttft_ms": [2000, 4000, 6000], - "mean_audio_ttfp_ms": [2000, 4000, 6000], - "mean_audio_rtf": [0.25, 0.4, 0.7] + "mean_ttft_ms": [6000], + "mean_audio_ttfp_ms": [6000], + "mean_audio_rtf": [0.7] } }, { "dataset_name": "random", "backend": "openai-chat-omni", "endpoint": "/v1/chat/completions", - "num_prompts": [ - 4, - 16 - ], - "max_concurrency": [ - 1, - 4 - ], + "num_prompts": [4, 16, 40], + "max_concurrency": [1, 4, 10], "random_input_len": 2500, "random_output_len": 900, "ignore_eos": true, - "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", + "extra_body": { + "modalities": ["text"] + }, + "percentile-metrics": "ttft,tpot,itl,e2el", "baseline": { - "mean_ttft_ms": [1000, 3000], - "mean_audio_ttfp_ms": [1000, 3000], - "mean_audio_rtf": [0.35, 0.45] + "mean_ttft_ms": [1000, 3000, 5000] + } + }, + { + "dataset_name": "random-mm", + "backend": "openai-chat-omni", + "endpoint": "/v1/chat/completions", + "num_prompts": [10], + "request_rate": [0.1], + "random_input_len": 100, + "random_output_len": 100, + "random_range_ratio": 0.0, + "ignore_eos": true, + "extra_body": { + "modalities": ["text"] + }, + "random_mm_base_items_per_request": 1, + "random_mm_num_mm_items_range_ratio": 0.5, + "random_mm_limit_mm_per_prompt": { + "audio": 1 + }, + "random_mm_bucket_config": { + "(0, 60, 3)": 1.0 + }, + "percentile-metrics": "ttft,tpot,itl,e2el", + "baseline": { + "mean_ttft_ms": [2000] + } + }, + { + "dataset_name": "random-mm", + "backend": "openai-chat-omni", + "endpoint": "/v1/chat/completions", + "num_prompts": [40], + "request_rate": [0.3], + "random_input_len": 100, + "random_output_len": 100, + "random_range_ratio": 0.0, + "ignore_eos": true, + "extra_body": { + "modalities": ["text"] + }, + "random_mm_base_items_per_request": 2, + "random_mm_num_mm_items_range_ratio": 0.5, + "random_mm_limit_mm_per_prompt": { + "image": 1, + "video": 1 + }, + "random_mm_bucket_config": { + "(256, 256, 1)": 0.5, + "(720, 1280, 2)": 0.5 + }, + "percentile-metrics": "ttft,tpot,itl,e2el", + "baseline": { + "mean_ttft_ms": [4000] + } + }, + { + "dataset_name": "random-mm", + "backend": "openai-chat-omni", + "endpoint": "/v1/chat/completions", + "num_prompts": [100], + "request_rate": [0.5], + "random_input_len": 100, + "random_output_len": 100, + "random_range_ratio": 0.0, + "ignore_eos": true, + "extra_body": { + "modalities": ["text"] + }, + "random_mm_base_items_per_request": 3, + "random_mm_num_mm_items_range_ratio": 0.5, + "random_mm_limit_mm_per_prompt": { + "image": 1, + "video": 1, + "audio": 1 + }, + "random_mm_bucket_config": { + "(256, 256, 1)": 0.34, + "(720, 1280, 2)": 0.33, + "(0, 60, 3)": 0.33 + }, + "percentile-metrics": "ttft,tpot,itl,e2el", + "baseline": { + "mean_ttft_ms": [6000] } } ]