anodyine/vllm-omni
1
0 Fork 0
Code Issues 4 Pull Requests 2 Actions Packages Projects Releases Wiki Activity

[skip ci][Tests] Splitting Qwen3-omni's performance test cases (#3501)

Browse Source 
Signed-off-by: amy-why-3459 <wuhaiyan17@huawei.com>
This commit is contained in:
amy-why-3459 authored and GitHub committed 2026-05-13 17:54:21 +08:00
1 parent a715abd447
commit 2e8e3057bc
6 files changed
+554 -455
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

.buildkite/test-nightly.yml
+99 -5
View File
@@ -169,14 +169,104 @@ steps:
path: /mnt/hf-cache
type: DirectoryOrCreate
- label: ":full_moon: Omni · Perf Test"
key: nightly-omni-performance
- label: ":full_moon: Omni · Perf Test · No Async Chunk"
key: nightly-omni-performance-no-async-chunk
timeout_in_minutes: 180
commands:
- export BENCHMARK_DIR=tests/dfx/perf/results
- |
set +e
pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_omni.json
pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json
EXIT=$$?
buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
exit $$EXIT
agents:
queue: "mithril-h100-pool"
plugins:
- kubernetes:
podSpec:
containers:
- image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
resources:
limits:
nvidia.com/gpu: 2
volumeMounts:
- name: devshm
mountPath: /dev/shm
- name: hf-cache
mountPath: /root/.cache/huggingface
env:
- name: HF_HOME
value: /root/.cache/huggingface
- name: HF_TOKEN
valueFrom:
secretKeyRef:
name: hf-token-secret
key: token
nodeSelector:
node.kubernetes.io/instance-type: gpu-h100-sxm
volumes:
- name: devshm
emptyDir:
medium: Memory
- name: hf-cache
hostPath:
path: /mnt/hf-cache
type: DirectoryOrCreate
- label: ":full_moon: Omni · Perf Test · Async Chunk"
key: nightly-omni-performance-async-chunk
timeout_in_minutes: 180
commands:
- export BENCHMARK_DIR=tests/dfx/perf/results
- |
set +e
pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json
EXIT=$$?
buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
exit $$EXIT
agents:
queue: "mithril-h100-pool"
plugins:
- kubernetes:
podSpec:
containers:
- image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
resources:
limits:
nvidia.com/gpu: 2
volumeMounts:
- name: devshm
mountPath: /dev/shm
- name: hf-cache
mountPath: /root/.cache/huggingface
env:
- name: HF_HOME
value: /root/.cache/huggingface
- name: HF_TOKEN
valueFrom:
secretKeyRef:
name: hf-token-secret
key: token
nodeSelector:
node.kubernetes.io/instance-type: gpu-h100-sxm
volumes:
- name: devshm
emptyDir:
medium: Memory
- name: hf-cache
hostPath:
path: /mnt/hf-cache
type: DirectoryOrCreate
- label: ":full_moon: Omni · Perf Test · vLLM Text"
key: nightly-omni-performance-vllm-text
timeout_in_minutes: 180
commands:
- export BENCHMARK_DIR=tests/dfx/perf/results
- |
set +e
pytest -s -v tests/dfx/perf/scripts/run_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen3_omni_vllm_text.json
EXIT=$$?
buildkite-agent artifact upload "tests/dfx/perf/results/*.json"
exit $$EXIT
@@ -1176,7 +1266,9 @@ steps:
- label: ":email: Nightly Collection & Email"
key: nightly-perf-distribution
depends_on:
- nightly-omni-performance
- nightly-omni-performance-no-async-chunk
- nightly-omni-performance-async-chunk
- nightly-omni-performance-vllm-text
- nightly-tts-performance
- nightly-diffusion-x2iat-performance-qwen-image
- nightly-diffusion-x2iat-performance-qwen-image-edit
@@ -1190,7 +1282,9 @@ steps:
- export DEFAULT_INPUT_DIR=tests/dfx/perf/results
- export DEFAULT_OUTPUT_DIR=tests/dfx/perf/results
- buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-tts-performance
- buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-omni-performance
- buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-omni-performance-no-async-chunk
- buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-omni-performance-async-chunk
- buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-omni-performance-vllm-text
- buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-qwen-image
- buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-qwen-image-edit
- buildkite-agent artifact download "tests/dfx/perf/results/*.json" . --step nightly-diffusion-x2iat-performance-qwen-image-edit-2509
.gitignore
+1
View File
@@ -239,6 +239,7 @@ data/
datasets/
*.csv
*.json
!tests/dfx/perf/tests/*.json
!apps/ComfyUI-vLLM-Omni/example_workflows/*.json
*.jsonl
*.parquet
tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json
+209
View File
@@ -0,0 +1,209 @@
[
{
"test_name": "test_qwen3_omni_chunk",
"server_params": {
"model": "Qwen/Qwen3-Omni-30B-A3B-Instruct",
"extra_cli_args": ["--async-chunk"]
},
"benchmark_params": [
{
"dataset_name": "random",
"backend": "openai-chat-omni",
"endpoint": "/v1/chat/completions",
"num_prompts": [4, 16, 32, 64],
"max_concurrency": [1, 4, 8, 16],
"random_input_len": 2500,
"random_output_len": 900,
"ignore_eos": true,
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
"baseline": {
"mean_ttft_ms": [1000, 3000, 5000, 7000],
"mean_audio_ttfp_ms": [1000, 3000, 5000, 7000],
"mean_audio_rtf": [0.2, 0.35, 0.6, 0.85]
}
},
{
"dataset_name": "random-mm",
"backend": "openai-chat-omni",
"endpoint": "/v1/chat/completions",
"num_prompts": [10],
"request_rate": [0.1],
"random_input_len": 100,
"random_output_len": 100,
"random_range_ratio": 0.0,
"ignore_eos": true,
"random_mm_base_items_per_request": 1,
"random_mm_num_mm_items_range_ratio": 0.5,
"random_mm_limit_mm_per_prompt": {
"audio": 1
},
"random_mm_bucket_config": {
"(0, 60, 3)": 1.0
},
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
"baseline": {
"mean_ttft_ms": [2000],
"mean_audio_ttfp_ms": [2000],
"mean_audio_rtf": [0.25]
}
},
{
"dataset_name": "random-mm",
"backend": "openai-chat-omni",
"endpoint": "/v1/chat/completions",
"num_prompts": [40],
"request_rate": [0.5],
"random_input_len": 100,
"random_output_len": 100,
"random_range_ratio": 0.0,
"ignore_eos": true,
"random_mm_base_items_per_request": 2,
"random_mm_num_mm_items_range_ratio": 0.5,
"random_mm_limit_mm_per_prompt": {
"image": 1,
"video": 1
},
"random_mm_bucket_config": {
"(256, 256, 1)": 0.5,
"(720, 1280, 2)": 0.5
},
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
"baseline": {
"mean_ttft_ms": [6000],
"mean_audio_ttfp_ms": [6000],
"mean_audio_rtf": [0.7]
}
},
{
"dataset_name": "random-mm",
"backend": "openai-chat-omni",
"endpoint": "/v1/chat/completions",
"num_prompts": [100],
"request_rate": [1.0],
"random_input_len": 100,
"random_output_len": 100,
"random_range_ratio": 0.0,
"ignore_eos": true,
"random_mm_base_items_per_request": 3,
"random_mm_num_mm_items_range_ratio": 0.5,
"random_mm_limit_mm_per_prompt": {
"image": 1,
"video": 1,
"audio": 1
},
"random_mm_bucket_config": {
"(256, 256, 1)": 0.34,
"(720, 1280, 2)": 0.33,
"(0, 60, 3)": 0.33
},
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
"baseline": {
"mean_ttft_ms": [12000],
"mean_audio_ttfp_ms": [12000],
"mean_audio_rtf": [1.0]
}
},
{
"dataset_name": "random",
"backend": "openai-chat-omni",
"endpoint": "/v1/chat/completions",
"num_prompts": [4, 16, 32, 64, 128],
"max_concurrency": [1, 4, 8, 16, 32],
"random_input_len": 2500,
"random_output_len": 900,
"ignore_eos": true,
"extra_body": {
"modalities": ["text"]
},
"percentile-metrics": "ttft,tpot,itl,e2el",
"baseline": {
"mean_ttft_ms": [1000, 3000, 5000, 7000, 9000]
}
},
{
"dataset_name": "random-mm",
"backend": "openai-chat-omni",
"endpoint": "/v1/chat/completions",
"num_prompts": [10],
"request_rate": [0.1],
"random_input_len": 100,
"random_output_len": 100,
"random_range_ratio": 0.0,
"ignore_eos": true,
"extra_body": {
"modalities": ["text"]
},
"random_mm_base_items_per_request": 1,
"random_mm_num_mm_items_range_ratio": 0.5,
"random_mm_limit_mm_per_prompt": {
"audio": 1
},
"random_mm_bucket_config": {
"(0, 60, 3)": 1.0
},
"percentile-metrics": "ttft,tpot,itl,e2el",
"baseline": {
"mean_ttft_ms": [2000]
}
},
{
"dataset_name": "random-mm",
"backend": "openai-chat-omni",
"endpoint": "/v1/chat/completions",
"num_prompts": [40],
"request_rate": [0.5],
"random_input_len": 100,
"random_output_len": 100,
"random_range_ratio": 0.0,
"ignore_eos": true,
"extra_body": {
"modalities": ["text"]
},
"random_mm_base_items_per_request": 2,
"random_mm_num_mm_items_range_ratio": 0.5,
"random_mm_limit_mm_per_prompt": {
"image": 1,
"video": 1
},
"random_mm_bucket_config": {
"(256, 256, 1)": 0.5,
"(720, 1280, 2)": 0.5
},
"percentile-metrics": "ttft,tpot,itl,e2el",
"baseline": {
"mean_ttft_ms": [6000]
}
},
{
"dataset_name": "random-mm",
"backend": "openai-chat-omni",
"endpoint": "/v1/chat/completions",
"num_prompts": [100],
"request_rate": [1.0],
"random_input_len": 100,
"random_output_len": 100,
"random_range_ratio": 0.0,
"ignore_eos": true,
"extra_body": {
"modalities": ["text"]
},
"random_mm_base_items_per_request": 3,
"random_mm_num_mm_items_range_ratio": 0.5,
"random_mm_limit_mm_per_prompt": {
"image": 1,
"video": 1,
"audio": 1
},
"random_mm_bucket_config": {
"(256, 256, 1)": 0.34,
"(720, 1280, 2)": 0.33,
"(0, 60, 3)": 0.33
},
"percentile-metrics": "ttft,tpot,itl,e2el",
"baseline": {
"mean_ttft_ms": [6000]
}
}
]
}
]
tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json
+108
View File
@@ -0,0 +1,108 @@
[
{
"test_name": "test_qwen3_omni",
"server_params": {
"model": "Qwen/Qwen3-Omni-30B-A3B-Instruct",
"extra_cli_args": ["--no-async-chunk"]
},
"benchmark_params": [
{
"dataset_name": "random",
"backend": "openai-chat-omni",
"endpoint": "/v1/chat/completions",
"num_prompts": [4, 16, 32, 64, 128],
"max_concurrency": [1, 4, 8, 16, 32],
"random_input_len": 2500,
"random_output_len": 900,
"ignore_eos": true,
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
"baseline": {
"mean_ttft_ms": [1000, 3000, 5000, 7000, 9000],
"mean_audio_ttfp_ms": [30000, 60000, 90000, 120000, 150000],
"mean_audio_rtf": [0.35, 0.45, 0.55, 0.65, 0.75]
}
},
{
"dataset_name": "random-mm",
"backend": "openai-chat-omni",
"endpoint": "/v1/chat/completions",
"num_prompts": [10],
"request_rate": [0.1],
"random_input_len": 100,
"random_output_len": 100,
"random_range_ratio": 0.0,
"ignore_eos": true,
"random_mm_base_items_per_request": 1,
"random_mm_num_mm_items_range_ratio": 0.5,
"random_mm_limit_mm_per_prompt": {
"audio": 1
},
"random_mm_bucket_config": {
"(0, 60, 3)": 1.0
},
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
"baseline": {
"mean_ttft_ms": [2000],
"mean_audio_ttfp_ms": [10000],
"mean_audio_rtf": [0.25]
}
},
{
"dataset_name": "random-mm",
"backend": "openai-chat-omni",
"endpoint": "/v1/chat/completions",
"num_prompts": [40],
"request_rate": [0.5],
"random_input_len": 100,
"random_output_len": 100,
"random_range_ratio": 0.0,
"ignore_eos": true,
"random_mm_base_items_per_request": 2,
"random_mm_num_mm_items_range_ratio": 0.5,
"random_mm_limit_mm_per_prompt": {
"image": 1,
"video": 1
},
"random_mm_bucket_config": {
"(256, 256, 1)": 0.5,
"(720, 1280, 2)": 0.5
},
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
"baseline": {
"mean_ttft_ms": [6000],
"mean_audio_ttfp_ms": [15000],
"mean_audio_rtf": [0.45]
}
},
{
"dataset_name": "random-mm",
"backend": "openai-chat-omni",
"endpoint": "/v1/chat/completions",
"num_prompts": [100],
"request_rate": [1.0],
"random_input_len": 100,
"random_output_len": 100,
"random_range_ratio": 0.0,
"ignore_eos": true,
"random_mm_base_items_per_request": 3,
"random_mm_num_mm_items_range_ratio": 0.5,
"random_mm_limit_mm_per_prompt": {
"image": 1,
"video": 1,
"audio": 1
},
"random_mm_bucket_config": {
"(256, 256, 1)": 0.34,
"(720, 1280, 2)": 0.33,
"(0, 60, 3)": 0.33
},
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
"baseline": {
"mean_ttft_ms": [12000],
"mean_audio_ttfp_ms": [18000],
"mean_audio_rtf": [0.9]
}
}
]
}
]
tests/dfx/perf/tests/test_qwen3_omni_vllm_text.json
+137
View File
@@ -0,0 +1,137 @@
[
{
"test_name": "test_qwen3_omni_vllm_text",
"server_params": {
"model": "Qwen/Qwen3-Omni-30B-A3B-Instruct",
"use_omni": false,
"extra_cli_args": ["--no-enable-prefix-caching"]
},
"benchmark_params": [
{
"dataset_name": "random",
"backend": "vllm",
"model": "Qwen/Qwen3-Omni-30B-A3B-Instruct",
"endpoint": "/v1/completions",
"num_prompts": [4, 16, 32, 64, 128],
"max_concurrency": [1, 4, 8, 16, 32],
"random_input_len": 2500,
"random_output_len": 900,
"temperature": 0.4,
"top_p": 0.9,
"top_k": 1,
"seed": 42,
"repetition_penalty": 1.05,
"ignore_eos": true,
"extra_body": {
"modalities": ["text"]
},
"percentile-metrics": "ttft,tpot,itl,e2el",
"baseline": {
"mean_ttft_ms": [1000, 3000, 5000, 7000, 9000]
}
},
{
"dataset_name": "random-mm",
"backend": "vllm",
"model": "Qwen/Qwen3-Omni-30B-A3B-Instruct",
"endpoint": "/v1/completions",
"num_prompts": [10],
"request_rate": [0.1],
"random_input_len": 100,
"random_output_len": 100,
"random_range_ratio": 0.0,
"temperature": 0.4,
"top_p": 0.9,
"top_k": 1,
"seed": 42,
"repetition_penalty": 1.05,
"ignore_eos": true,
"extra_body": {
"modalities": ["text"]
},
"random_mm_base_items_per_request": 1,
"random_mm_num_mm_items_range_ratio": 0.5,
"random_mm_limit_mm_per_prompt": {
"audio": 1
},
"random_mm_bucket_config": {
"(0, 60, 3)": 1.0
},
"percentile-metrics": "ttft,tpot,itl,e2el",
"baseline": {
"mean_ttft_ms": [2000]
}
},
{
"dataset_name": "random-mm",
"backend": "vllm",
"model": "Qwen/Qwen3-Omni-30B-A3B-Instruct",
"endpoint": "/v1/completions",
"num_prompts": [40],
"request_rate": [0.5],
"random_input_len": 100,
"random_output_len": 100,
"random_range_ratio": 0.0,
"temperature": 0.4,
"top_p": 0.9,
"top_k": 1,
"seed": 42,
"repetition_penalty": 1.05,
"ignore_eos": true,
"extra_body": {
"modalities": ["text"]
},
"random_mm_base_items_per_request": 2,
"random_mm_num_mm_items_range_ratio": 0.5,
"random_mm_limit_mm_per_prompt": {
"image": 1,
"video": 1
},
"random_mm_bucket_config": {
"(256, 256, 1)": 0.5,
"(720, 1280, 2)": 0.5
},
"percentile-metrics": "ttft,tpot,itl,e2el",
"baseline": {
"mean_ttft_ms": [6000]
}
},
{
"dataset_name": "random-mm",
"backend": "vllm",
"model": "Qwen/Qwen3-Omni-30B-A3B-Instruct",
"endpoint": "/v1/completions",
"num_prompts": [100],
"request_rate": [1.0],
"random_input_len": 100,
"random_output_len": 100,
"random_range_ratio": 0.0,
"temperature": 0.4,
"top_p": 0.9,
"top_k": 1,
"seed": 42,
"repetition_penalty": 1.05,
"ignore_eos": true,
"extra_body": {
"modalities": ["text"]
},
"random_mm_base_items_per_request": 3,
"random_mm_num_mm_items_range_ratio": 0.5,
"random_mm_limit_mm_per_prompt": {
"image": 1,
"video": 1,
"audio": 1
},
"random_mm_bucket_config": {
"(256, 256, 1)": 0.34,
"(720, 1280, 2)": 0.33,
"(0, 60, 3)": 0.33
},
"percentile-metrics": "ttft,tpot,itl,e2el",
"baseline": {
"mean_ttft_ms": [6000]
}
}
]
}
]
tests/dfx/perf/tests/test_qwen_omni.json
-450
View File
@@ -1,450 +0,0 @@
[
{
"test_name": "test_qwen3_omni",
"server_params": {
"model": "Qwen/Qwen3-Omni-30B-A3B-Instruct",
"extra_cli_args": ["--no-async-chunk"]
},
"benchmark_params": [
{
"dataset_name": "random",
"backend": "openai-chat-omni",
"endpoint": "/v1/chat/completions",
"num_prompts": [4, 16, 32, 64, 128],
"max_concurrency": [1, 4, 8, 16, 32],
"random_input_len": 2500,
"random_output_len": 900,
"ignore_eos": true,
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
"baseline": {
"mean_ttft_ms": [1000, 3000, 5000, 7000, 9000],
"mean_audio_ttfp_ms": [30000, 60000, 90000, 120000, 150000],
"mean_audio_rtf": [0.35, 0.45, 0.55, 0.65, 0.75]
}
},
{
"dataset_name": "random-mm",
"backend": "openai-chat-omni",
"endpoint": "/v1/chat/completions",
"num_prompts": [10],
"request_rate": [0.1],
"random_input_len": 100,
"random_output_len": 100,
"random_range_ratio": 0.0,
"ignore_eos": true,
"random_mm_base_items_per_request": 1,
"random_mm_num_mm_items_range_ratio": 0.5,
"random_mm_limit_mm_per_prompt": {
"audio": 1
},
"random_mm_bucket_config": {
"(0, 60, 3)": 1.0
},
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
"baseline": {
"mean_ttft_ms": [2000],
"mean_audio_ttfp_ms": [10000],
"mean_audio_rtf": [0.25]
}
},
{
"dataset_name": "random-mm",
"backend": "openai-chat-omni",
"endpoint": "/v1/chat/completions",
"num_prompts": [40],
"request_rate": [0.5],
"random_input_len": 100,
"random_output_len": 100,
"random_range_ratio": 0.0,
"ignore_eos": true,
"random_mm_base_items_per_request": 2,
"random_mm_num_mm_items_range_ratio": 0.5,
"random_mm_limit_mm_per_prompt": {
"image": 1,
"video": 1
},
"random_mm_bucket_config": {
"(256, 256, 1)": 0.5,
"(720, 1280, 2)": 0.5
},
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
"baseline": {
"mean_ttft_ms": [6000],
"mean_audio_ttfp_ms": [15000],
"mean_audio_rtf": [0.45]
}
},
{
"dataset_name": "random-mm",
"backend": "openai-chat-omni",
"endpoint": "/v1/chat/completions",
"num_prompts": [100],
"request_rate": [1.0],
"random_input_len": 100,
"random_output_len": 100,
"random_range_ratio": 0.0,
"ignore_eos": true,
"random_mm_base_items_per_request": 3,
"random_mm_num_mm_items_range_ratio": 0.5,
"random_mm_limit_mm_per_prompt": {
"image": 1,
"video": 1,
"audio": 1
},
"random_mm_bucket_config": {
"(256, 256, 1)": 0.34,
"(720, 1280, 2)": 0.33,
"(0, 60, 3)": 0.33
},
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
"baseline": {
"mean_ttft_ms": [12000],
"mean_audio_ttfp_ms": [18000],
"mean_audio_rtf": [0.9]
}
}
]
},
{
"test_name": "test_qwen3_omni_chunk",
"server_params": {
"model": "Qwen/Qwen3-Omni-30B-A3B-Instruct",
"extra_cli_args": ["--async-chunk"]
},
"benchmark_params": [
{
"dataset_name": "random",
"backend": "openai-chat-omni",
"endpoint": "/v1/chat/completions",
"num_prompts": [4, 16, 32, 64],
"max_concurrency": [1, 4, 8, 16],
"random_input_len": 2500,
"random_output_len": 900,
"ignore_eos": true,
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
"baseline": {
"mean_ttft_ms": [1000, 3000, 5000, 7000],
"mean_audio_ttfp_ms": [1000, 3000, 5000, 7000],
"mean_audio_rtf": [0.2, 0.35, 0.6, 0.85]
}
},
{
"dataset_name": "random-mm",
"backend": "openai-chat-omni",
"endpoint": "/v1/chat/completions",
"num_prompts": [10],
"request_rate": [0.1],
"random_input_len": 100,
"random_output_len": 100,
"random_range_ratio": 0.0,
"ignore_eos": true,
"random_mm_base_items_per_request": 1,
"random_mm_num_mm_items_range_ratio": 0.5,
"random_mm_limit_mm_per_prompt": {
"audio": 1
},
"random_mm_bucket_config": {
"(0, 60, 3)": 1.0
},
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
"baseline": {
"mean_ttft_ms": [2000],
"mean_audio_ttfp_ms": [2000],
"mean_audio_rtf": [0.25]
}
},
{
"dataset_name": "random-mm",
"backend": "openai-chat-omni",
"endpoint": "/v1/chat/completions",
"num_prompts": [40],
"request_rate": [0.5],
"random_input_len": 100,
"random_output_len": 100,
"random_range_ratio": 0.0,
"ignore_eos": true,
"random_mm_base_items_per_request": 2,
"random_mm_num_mm_items_range_ratio": 0.5,
"random_mm_limit_mm_per_prompt": {
"image": 1,
"video": 1
},
"random_mm_bucket_config": {
"(256, 256, 1)": 0.5,
"(720, 1280, 2)": 0.5
},
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
"baseline": {
"mean_ttft_ms": [6000],
"mean_audio_ttfp_ms": [6000],
"mean_audio_rtf": [0.7]
}
},
{
"dataset_name": "random-mm",
"backend": "openai-chat-omni",
"endpoint": "/v1/chat/completions",
"num_prompts": [100],
"request_rate": [1.0],
"random_input_len": 100,
"random_output_len": 100,
"random_range_ratio": 0.0,
"ignore_eos": true,
"random_mm_base_items_per_request": 3,
"random_mm_num_mm_items_range_ratio": 0.5,
"random_mm_limit_mm_per_prompt": {
"image": 1,
"video": 1,
"audio": 1
},
"random_mm_bucket_config": {
"(256, 256, 1)": 0.34,
"(720, 1280, 2)": 0.33,
"(0, 60, 3)": 0.33
},
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
"baseline": {
"mean_ttft_ms": [12000],
"mean_audio_ttfp_ms": [12000],
"mean_audio_rtf": [1.0]
}
},
{
"dataset_name": "random",
"backend": "openai-chat-omni",
"endpoint": "/v1/chat/completions",
"num_prompts": [4, 16, 32, 64, 128],
"max_concurrency": [1, 4, 8, 16, 32],
"random_input_len": 2500,
"random_output_len": 900,
"ignore_eos": true,
"extra_body": {
"modalities": ["text"]
},
"percentile-metrics": "ttft,tpot,itl,e2el",
"baseline": {
"mean_ttft_ms": [1000, 3000, 5000, 7000, 9000]
}
},
{
"dataset_name": "random-mm",
"backend": "openai-chat-omni",
"endpoint": "/v1/chat/completions",
"num_prompts": [10],
"request_rate": [0.1],
"random_input_len": 100,
"random_output_len": 100,
"random_range_ratio": 0.0,
"ignore_eos": true,
"extra_body": {
"modalities": ["text"]
},
"random_mm_base_items_per_request": 1,
"random_mm_num_mm_items_range_ratio": 0.5,
"random_mm_limit_mm_per_prompt": {
"audio": 1
},
"random_mm_bucket_config": {
"(0, 60, 3)": 1.0
},
"percentile-metrics": "ttft,tpot,itl,e2el",
"baseline": {
"mean_ttft_ms": [2000]
}
},
{
"dataset_name": "random-mm",
"backend": "openai-chat-omni",
"endpoint": "/v1/chat/completions",
"num_prompts": [40],
"request_rate": [0.5],
"random_input_len": 100,
"random_output_len": 100,
"random_range_ratio": 0.0,
"ignore_eos": true,
"extra_body": {
"modalities": ["text"]
},
"random_mm_base_items_per_request": 2,
"random_mm_num_mm_items_range_ratio": 0.5,
"random_mm_limit_mm_per_prompt": {
"image": 1,
"video": 1
},
"random_mm_bucket_config": {
"(256, 256, 1)": 0.5,
"(720, 1280, 2)": 0.5
},
"percentile-metrics": "ttft,tpot,itl,e2el",
"baseline": {
"mean_ttft_ms": [6000]
}
},
{
"dataset_name": "random-mm",
"backend": "openai-chat-omni",
"endpoint": "/v1/chat/completions",
"num_prompts": [100],
"request_rate": [1.0],
"random_input_len": 100,
"random_output_len": 100,
"random_range_ratio": 0.0,
"ignore_eos": true,
"extra_body": {
"modalities": ["text"]
},
"random_mm_base_items_per_request": 3,
"random_mm_num_mm_items_range_ratio": 0.5,
"random_mm_limit_mm_per_prompt": {
"image": 1,
"video": 1,
"audio": 1
},
"random_mm_bucket_config": {
"(256, 256, 1)": 0.34,
"(720, 1280, 2)": 0.33,
"(0, 60, 3)": 0.33
},
"percentile-metrics": "ttft,tpot,itl,e2el",
"baseline": {
"mean_ttft_ms": [6000]
}
}
]
},
{
"test_name": "test_qwen3_omni_vllm_text",
"server_params": {
"model": "Qwen/Qwen3-Omni-30B-A3B-Instruct",
"use_omni": false,
"extra_cli_args": ["--no-enable-prefix-caching"]
},
"benchmark_params": [
{
"dataset_name": "random",
"backend": "vllm",
"model": "Qwen/Qwen3-Omni-30B-A3B-Instruct",
"endpoint": "/v1/completions",
"num_prompts": [4, 16, 32, 64, 128],
"max_concurrency": [1, 4, 8, 16, 32],
"random_input_len": 2500,
"random_output_len": 900,
"temperature": 0.4,
"top_p": 0.9,
"top_k": 1,
"seed": 42,
"repetition_penalty": 1.05,
"ignore_eos": true,
"extra_body": {
"modalities": ["text"]
},
"percentile-metrics": "ttft,tpot,itl,e2el",
"baseline": {
"mean_ttft_ms": [1000, 3000, 5000, 7000, 9000]
}
},
{
"dataset_name": "random-mm",
"backend": "vllm",
"model": "Qwen/Qwen3-Omni-30B-A3B-Instruct",
"endpoint": "/v1/completions",
"num_prompts": [10],
"request_rate": [0.1],
"random_input_len": 100,
"random_output_len": 100,
"random_range_ratio": 0.0,
"temperature": 0.4,
"top_p": 0.9,
"top_k": 1,
"seed": 42,
"repetition_penalty": 1.05,
"ignore_eos": true,
"extra_body": {
"modalities": ["text"]
},
"random_mm_base_items_per_request": 1,
"random_mm_num_mm_items_range_ratio": 0.5,
"random_mm_limit_mm_per_prompt": {
"audio": 1
},
"random_mm_bucket_config": {
"(0, 60, 3)": 1.0
},
"percentile-metrics": "ttft,tpot,itl,e2el",
"baseline": {
"mean_ttft_ms": [2000]
}
},
{
"dataset_name": "random-mm",
"backend": "vllm",
"model": "Qwen/Qwen3-Omni-30B-A3B-Instruct",
"endpoint": "/v1/completions",
"num_prompts": [40],
"request_rate": [0.5],
"random_input_len": 100,
"random_output_len": 100,
"random_range_ratio": 0.0,
"temperature": 0.4,
"top_p": 0.9,
"top_k": 1,
"seed": 42,
"repetition_penalty": 1.05,
"ignore_eos": true,
"extra_body": {
"modalities": ["text"]
},
"random_mm_base_items_per_request": 2,
"random_mm_num_mm_items_range_ratio": 0.5,
"random_mm_limit_mm_per_prompt": {
"image": 1,
"video": 1
},
"random_mm_bucket_config": {
"(256, 256, 1)": 0.5,
"(720, 1280, 2)": 0.5
},
"percentile-metrics": "ttft,tpot,itl,e2el",
"baseline": {
"mean_ttft_ms": [6000]
}
},
{
"dataset_name": "random-mm",
"backend": "vllm",
"model": "Qwen/Qwen3-Omni-30B-A3B-Instruct",
"endpoint": "/v1/completions",
"num_prompts": [100],
"request_rate": [1.0],
"random_input_len": 100,
"random_output_len": 100,
"random_range_ratio": 0.0,
"temperature": 0.4,
"top_p": 0.9,
"top_k": 1,
"seed": 42,
"repetition_penalty": 1.05,
"ignore_eos": true,
"extra_body": {
"modalities": ["text"]
},
"random_mm_base_items_per_request": 3,
"random_mm_num_mm_items_range_ratio": 0.5,
"random_mm_limit_mm_per_prompt": {
"image": 1,
"video": 1,
"audio": 1
},
"random_mm_bucket_config": {
"(256, 256, 1)": 0.34,
"(720, 1280, 2)": 0.33,
"(0, 60, 3)": 0.33
},
"percentile-metrics": "ttft,tpot,itl,e2el",
"baseline": {
"mean_ttft_ms": [6000]
}
}
]
}
]