anodyine/vllm-omni
1
0 Fork 0
Code Issues 4 Pull Requests 2 Actions Packages Projects Releases Wiki Activity

[CI] optional --assert-baseline and update perf JSON baselines (#3695)

Browse Source 
Signed-off-by: wangyu <410167048@qq.com>
Signed-off-by: wangyu <53896905+yenuo26@users.noreply.github.com>
Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
This commit is contained in:
wangyuandHongsheng Liu authored and GitHub committed 2026-05-20 09:15:53 +08:00
1 parent a3d4ed809d
commit 4e82192fd6
12 files changed
+666 -300
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

tests/dfx/conftest.py
+32 -5
View File
@@ -279,6 +279,10 @@ def create_benchmark_indices(
return indices
# Omni perf/stability: ``vllm bench serve --omni`` result template (used if bench exits without writing JSON).
OMNI_RESULT_TEMPLATE_PATH = Path(__file__).resolve().parent / "perf" / "scripts" / "result_omni_template.json"
def _safe_filename_token(value: Any | None, *, default: str = "na") -> str:
"""Make a single path segment safe for result filenames on common filesystems."""
if value is None:
@@ -289,7 +293,7 @@ def _safe_filename_token(value: Any | None, *, default: str = "na") -> str:
return s if s else default
def _resolve_baseline_value(
def resolve_baseline_value(
baseline_raw: Any,
*,
sweep_index: int | None,
@@ -330,7 +334,7 @@ def _baseline_thresholds_for_step(
) -> dict[str, Any]:
"""Resolve baseline config to one threshold per metric for this iteration."""
return {
metric_name: _resolve_baseline_value(
metric_name: resolve_baseline_value(
baseline_raw,
sweep_index=sweep_index,
max_concurrency=max_concurrency,
@@ -354,7 +358,12 @@ def run_benchmark(
random_input_len: Any | None = None,
random_output_len: Any | None = None,
) -> dict[str, Any]:
"""Run one ``vllm bench serve --omni`` iteration and return parsed metrics."""
"""Run one ``vllm bench serve --omni`` iteration and return parsed metrics.
After ``vllm bench`` writes the JSON, ``result["baseline"]`` holds the resolved per-metric thresholds
(when ``baseline_config`` is provided). If the benchmark exits without writing a result file,
``result_omni_template.json`` is used as a fallback.
"""
current_dt = datetime.now().strftime("%Y%m%d-%H%M%S")
ri = _safe_filename_token(random_input_len)
ro = _safe_filename_token(random_output_len)
@@ -403,8 +412,17 @@ def run_benchmark(
result_dir = "./"
result_path = os.path.join(result_dir, result_filename)
with open(result_path, encoding="utf-8") as f:
result = json.load(f)
if not os.path.exists(result_path):
with open(OMNI_RESULT_TEMPLATE_PATH, encoding="utf-8") as f:
template_result: dict[str, Any] = json.load(f)
Path(result_path).parent.mkdir(parents=True, exist_ok=True)
with open(result_path, "w", encoding="utf-8") as f:
json.dump(template_result, f, ensure_ascii=False, indent=2)
print(f"Benchmark result file not generated, fallback to template: {result_path}")
result = template_result
else:
with open(result_path, encoding="utf-8") as f:
result = json.load(f)
if baseline_config:
result["baseline"] = _baseline_thresholds_for_step(
@@ -432,3 +450,12 @@ def pytest_addoption(parser: pytest.Parser) -> None:
default=None,
help=("Path to benchmark config JSON. Example: --test-config-file tests/dfx/perf/tests/test_tts.json"),
)
parser.addoption(
"--assert-baseline",
action="store_true",
default=False,
help=(
"When set, omni/diffusion perf runners compare metrics against the baseline block in the JSON config "
"(default: off)."
),
)
tests/dfx/perf/scripts/run_benchmark.py
+20 -75
View File
@@ -12,12 +12,17 @@ from tests.dfx.conftest import (
create_unique_server_params,
get_benchmark_params_for_server,
load_configs,
resolve_baseline_value,
run_benchmark,
)
from tests.helpers.runtime import OmniServer
pytestmark = [pytest.mark.full_model]
# Compare metrics to each test JSON ``baseline`` block only when pytest is run with ``--assert-baseline``
# (registered in ``tests/dfx/conftest.py``; default: off). ``run_benchmark`` and ``_resolve_baseline_value`` are
# defined in the same module.
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
@@ -46,7 +51,6 @@ if CONFIG_FILE_PATH is None:
CONFIG_FILE_PATH = _DEFAULT_CONFIG_FILE
BENCHMARK_CONFIGS = load_configs(CONFIG_FILE_PATH)
OMNI_RESULT_TEMPLATE_PATH = Path(__file__).parent / "result_omni_template.json"
DEPLOY_CONFIGS_DIR = Path(__file__).parent.parent / "deploy"
@@ -87,16 +91,6 @@ def omni_server(request):
print("OmniServer stopped")
def _safe_filename_token(value: Any | None, *, default: str = "na") -> str:
"""Make a single path segment safe for result filenames on common filesystems."""
if value is None:
return default
s = str(value).strip()
for bad in ("/", "\\", ":", "*", "?", '"', "<", ">", "|"):
s = s.replace(bad, "_")
return s if s else default
benchmark_indices = create_benchmark_indices(BENCHMARK_CONFIGS, server_to_benchmark_mapping)
@@ -126,76 +120,23 @@ def benchmark_params(request, omni_server):
}
def _resolve_baseline_value(
baseline_raw: Any,
*,
sweep_index: int | None,
max_concurrency: Any = None,
request_rate: Any = None,
) -> Any:
"""Pick the baseline threshold for this sweep step.
Supported shapes per metric:
- **Scalar** — same threshold for every concurrency / QPS.
- **List** — aligned with ``max_concurrency`` / ``request_rate`` sweep order; use ``sweep_index``.
- **Dict** — keyed by concurrency or rate, e.g. ``{"1": 500, "4": 800}`` (keys are strings in JSON).
For dict lookup, ``max_concurrency`` is preferred when both are set (concurrency sweep).
"""
if baseline_raw is None:
# If no baseline is set, the maximum value will be used.
return 100000
if isinstance(baseline_raw, dict):
if max_concurrency is not None:
for key in (max_concurrency, str(max_concurrency)):
if key in baseline_raw:
return baseline_raw[key]
if request_rate is not None:
for key in (request_rate, str(request_rate)):
if key in baseline_raw:
return baseline_raw[key]
raise KeyError(
f"baseline dict has no key for max_concurrency={max_concurrency!r} "
f"or request_rate={request_rate!r}; keys={list(baseline_raw.keys())!r}"
)
if isinstance(baseline_raw, (list, tuple)):
return baseline_raw[sweep_index]
return baseline_raw
def _baseline_thresholds_for_step(
baseline_data: dict[str, Any],
*,
sweep_index: int | None = None,
max_concurrency: Any = None,
request_rate: Any = None,
) -> dict[str, Any]:
"""Resolve ``test.json`` ``baseline`` block to one threshold per metric (same as ``assert_result``)."""
return {
metric_name: _resolve_baseline_value(
baseline_raw,
sweep_index=sweep_index,
max_concurrency=max_concurrency,
request_rate=request_rate,
)
for metric_name, baseline_raw in baseline_data.items()
}
def assert_result(
result,
params,
num_prompt,
*,
assert_baseline: bool,
sweep_index: int | None = None,
max_concurrency: Any = None,
request_rate: Any = None,
max_concurrency: Any | None = None,
request_rate: Any | None = None,
) -> None:
assert result["completed"] == num_prompt, "Request failures exist"
if not assert_baseline:
return
baseline_data = params.get("baseline", {})
for metric_name, baseline_raw in baseline_data.items():
current_value = result[metric_name]
baseline_value = _resolve_baseline_value(
baseline_value = resolve_baseline_value(
baseline_raw,
sweep_index=sweep_index,
max_concurrency=max_concurrency,
@@ -214,7 +155,7 @@ def assert_result(
@pytest.mark.benchmark
@pytest.mark.parametrize("omni_server", test_params, indirect=True)
@pytest.mark.parametrize("benchmark_params", benchmark_indices, indirect=True)
def test_performance_benchmark(omni_server, benchmark_params):
def test_performance_benchmark(omni_server, benchmark_params, request):
test_name = benchmark_params["test_name"]
params = benchmark_params["params"]
dataset_name = params.get("dataset_name", "")
@@ -226,6 +167,8 @@ def test_performance_benchmark(omni_server, benchmark_params):
print(f"Running benchmark for model: {model}")
print(f"Benchmark parameters: {benchmark_params}")
assert_baseline = request.config.getoption("--assert-baseline", default=False)
def to_list(value, default=None):
if value is None:
return [] if default is None else [default]
@@ -264,7 +207,7 @@ def test_performance_benchmark(omni_server, benchmark_params):
elif not isinstance(value, bool):
args.extend([arg_name, str(value)])
# QPS test (sweep_index aligns with qps_list / num_prompt_list for this loop)
# QPS / request-rate sweep
for i, (qps, num_prompt) in enumerate(zip(qps_list, num_prompt_list)):
args = args + ["--request-rate", str(qps), "--num-prompts", str(num_prompt)]
result = run_benchmark(
@@ -283,12 +226,13 @@ def test_performance_benchmark(omni_server, benchmark_params):
assert_result(
result,
params,
num_prompt=num_prompt,
num_prompt,
assert_baseline=assert_baseline,
sweep_index=i,
request_rate=qps,
)
# concurrency test (sweep_index aligns with max_concurrency_list for separate thresholds per concurrency)
# concurrency test
for i, (concurrency, num_prompt) in enumerate(zip(max_concurrency_list, num_prompt_list)):
args = args + ["--max-concurrency", str(concurrency), "--num-prompts", str(num_prompt), "--request-rate", "inf"]
result = run_benchmark(
@@ -307,7 +251,8 @@ def test_performance_benchmark(omni_server, benchmark_params):
assert_result(
result,
params,
num_prompt=num_prompt,
num_prompt,
assert_baseline=assert_baseline,
sweep_index=i,
max_concurrency=concurrency,
)
tests/dfx/perf/scripts/run_diffusion_benchmark.py
+12 -9
View File
@@ -11,6 +11,8 @@ This runner separates two concepts:
A config JSON file is REQUIRED via --test-config-file:
pytest run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_vllm_omni.json
Optional: ``--assert-baseline`` compares metrics to the ``baseline`` block in each benchmark entry (default: off).
All benchmark results for a session are consolidated into a single JSON file under
BENCHMARK_RESULT_DIR (override via the DIFFUSION_BENCHMARK_DIR environment variable).
Each entry in the file contains the test metadata (test_name, endpoint, benchmark_params,
@@ -652,12 +654,14 @@ def run_benchmark(
# ---------------------------------------------------------------------------
def assert_result(result: dict[str, Any], params: dict[str, Any]) -> None:
"""Assert that benchmark metrics satisfy the configured baselines."""
def assert_result(result: dict[str, Any], params: dict[str, Any], *, assert_baseline: bool) -> None:
"""Assert request completion; optionally assert metrics against ``baseline`` in *params*."""
num_prompts = params.get("num-prompts", 10)
completed = result.get("completed_requests", result.get("completed", 0))
assert completed == num_prompts, f"Expected {num_prompts} completed requests, got {completed}"
if not assert_baseline:
return
if params.get("skip-performance-assertion", False):
print("Skipping performance assertions.")
return
@@ -699,16 +703,14 @@ def _resolve_benchmark_endpoint(server_cfg: dict[str, Any], params: dict[str, An
indirect=True,
)
@pytest.mark.parametrize("benchmark_params", benchmark_indices, indirect=True)
def test_diffusion_performance_benchmark(diffusion_server, benchmark_params):
"""Run the diffusion performance benchmark and assert against baselines.
def test_diffusion_performance_benchmark(diffusion_server, benchmark_params, request):
"""Run the diffusion performance benchmark and verify request completion.
One server is started per unique parallel configuration (module scope).
For each server, all benchmark parameter sets defined in the config JSON
are executed sequentially; results are asserted against the baselines.
are executed sequentially; metrics are recorded to the aggregated result file.
Tracked metrics:
- throughput_qps (higher is better)
- latency_p50, latency_p99 (lower is better)
Pass ``--assert-baseline`` to compare ``throughput_qps``, ``latency_mean``, etc. to the JSON ``baseline`` block.
"""
test_name = benchmark_params["test_name"]
params = benchmark_params["params"]
@@ -744,4 +746,5 @@ def test_diffusion_performance_benchmark(diffusion_server, benchmark_params):
print(f"\n Aggregated results: {AGGREGATED_RESULT_FILE}")
print("=" * 60)
assert_result(result, params)
assert_baseline = request.config.getoption("--assert-baseline", default=False)
assert_result(result, params, assert_baseline=assert_baseline)
tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json
+159 -37
View File
@@ -3,31 +3,67 @@
"test_name": "test_qwen3_omni_chunk",
"server_params": {
"model": "Qwen/Qwen3-Omni-30B-A3B-Instruct",
"extra_cli_args": ["--async-chunk"]
"extra_cli_args": [
"--async-chunk"
]
},
"benchmark_params": [
{
"dataset_name": "random",
"backend": "openai-chat-omni",
"endpoint": "/v1/chat/completions",
"num_prompts": [4, 16, 32, 64, 128],
"max_concurrency": [1, 4, 8, 16, 32],
"num_prompts": [
4,
16,
32,
64,
128
],
"max_concurrency": [
1,
4,
8,
16,
32
],
"random_input_len": 2500,
"random_output_len": 900,
"ignore_eos": true,
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
"baseline": {
"mean_ttft_ms": [1000, 3000, 5000, 7000, 9000],
"mean_audio_ttfp_ms": [1000, 3000, 5000, 7000, 9000],
"mean_audio_rtf": [0.2, 0.35, 0.6, 0.85, 0.9]
"mean_ttft_ms": [
400,
300,
400,
600,
900
],
"mean_e2el_ms": [
26000,
43000,
52000,
72000,
109000
],
"mean_audio_rtf": [
0.15,
0.30,
0.35,
0.50,
0.75
]
}
},
{
"dataset_name": "random-mm",
"backend": "openai-chat-omni",
"endpoint": "/v1/chat/completions",
"num_prompts": [10],
"request_rate": [0.1],
"num_prompts": [
10
],
"request_rate": [
0.1
],
"random_input_len": 100,
"random_output_len": 100,
"random_range_ratio": 0.0,
@@ -42,17 +78,27 @@
},
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
"baseline": {
"mean_ttft_ms": [2000],
"mean_audio_ttfp_ms": [2000],
"mean_audio_rtf": [0.25]
"mean_ttft_ms": [
600
],
"mean_e2el_ms": [
6000
],
"mean_audio_rtf": [
0.20
]
}
},
{
"dataset_name": "random-mm",
"backend": "openai-chat-omni",
"endpoint": "/v1/chat/completions",
"num_prompts": [40],
"request_rate": [0.5],
"num_prompts": [
40
],
"request_rate": [
0.5
],
"random_input_len": 100,
"random_output_len": 100,
"random_range_ratio": 0.0,
@@ -69,17 +115,27 @@
},
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
"baseline": {
"mean_ttft_ms": [6000],
"mean_audio_ttfp_ms": [6000],
"mean_audio_rtf": [0.7]
"mean_ttft_ms": [
300
],
"mean_e2el_ms": [
7000
],
"mean_audio_rtf": [
0.25
]
}
},
{
"dataset_name": "random-mm",
"backend": "openai-chat-omni",
"endpoint": "/v1/chat/completions",
"num_prompts": [100],
"request_rate": [1.0],
"num_prompts": [
100
],
"request_rate": [
1.0
],
"random_input_len": 100,
"random_output_len": 100,
"random_range_ratio": 0.0,
@@ -98,40 +154,79 @@
},
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
"baseline": {
"mean_ttft_ms": [12000],
"mean_audio_ttfp_ms": [12000],
"mean_audio_rtf": [1.0]
"mean_ttft_ms": [
700
],
"mean_e2el_ms": [
11000
],
"mean_audio_rtf": [
0.35
]
}
},
{
"dataset_name": "random",
"backend": "openai-chat-omni",
"endpoint": "/v1/chat/completions",
"num_prompts": [4, 16, 32, 64, 128],
"max_concurrency": [1, 4, 8, 16, 32],
"num_prompts": [
4,
16,
32,
64,
128
],
"max_concurrency": [
1,
4,
8,
16,
32
],
"random_input_len": 2500,
"random_output_len": 900,
"ignore_eos": true,
"extra_body": {
"modalities": ["text"]
"modalities": [
"text"
]
},
"percentile-metrics": "ttft,tpot,itl,e2el",
"baseline": {
"mean_ttft_ms": [1000, 3000, 5000, 7000, 9000]
"mean_ttft_ms": [
100,
300,
500,
800,
1500
],
"mean_e2el_ms": [
5000,
8000,
10000,
14000,
20000
]
}
},
{
"dataset_name": "random-mm",
"backend": "openai-chat-omni",
"endpoint": "/v1/chat/completions",
"num_prompts": [10],
"request_rate": [0.1],
"num_prompts": [
10
],
"request_rate": [
0.1
],
"random_input_len": 100,
"random_output_len": 100,
"random_range_ratio": 0.0,
"ignore_eos": true,
"extra_body": {
"modalities": ["text"]
"modalities": [
"text"
]
},
"random_mm_base_items_per_request": 1,
"random_mm_num_mm_items_range_ratio": 0.5,
@@ -143,21 +238,32 @@
},
"percentile-metrics": "ttft,tpot,itl,e2el",
"baseline": {
"mean_ttft_ms": [2000]
"mean_ttft_ms": [
500
],
"mean_e2el_ms": [
1000
]
}
},
{
"dataset_name": "random-mm",
"backend": "openai-chat-omni",
"endpoint": "/v1/chat/completions",
"num_prompts": [40],
"request_rate": [0.5],
"num_prompts": [
40
],
"request_rate": [
0.5
],
"random_input_len": 100,
"random_output_len": 100,
"random_range_ratio": 0.0,
"ignore_eos": true,
"extra_body": {
"modalities": ["text"]
"modalities": [
"text"
]
},
"random_mm_base_items_per_request": 2,
"random_mm_num_mm_items_range_ratio": 0.5,
@@ -171,21 +277,32 @@
},
"percentile-metrics": "ttft,tpot,itl,e2el",
"baseline": {
"mean_ttft_ms": [6000]
"mean_ttft_ms": [
200
],
"mean_e2el_ms": [
800
]
}
},
{
"dataset_name": "random-mm",
"backend": "openai-chat-omni",
"endpoint": "/v1/chat/completions",
"num_prompts": [100],
"request_rate": [1.0],
"num_prompts": [
100
],
"request_rate": [
1.0
],
"random_input_len": 100,
"random_output_len": 100,
"random_range_ratio": 0.0,
"ignore_eos": true,
"extra_body": {
"modalities": ["text"]
"modalities": [
"text"
]
},
"random_mm_base_items_per_request": 3,
"random_mm_num_mm_items_range_ratio": 0.5,
@@ -201,7 +318,12 @@
},
"percentile-metrics": "ttft,tpot,itl,e2el",
"baseline": {
"mean_ttft_ms": [6000]
"mean_ttft_ms": [
600
],
"mean_e2el_ms": [
1300
]
}
}
]
tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json
+83 -21
View File
@@ -3,31 +3,67 @@
"test_name": "test_qwen3_omni",
"server_params": {
"model": "Qwen/Qwen3-Omni-30B-A3B-Instruct",
"extra_cli_args": ["--no-async-chunk"]
"extra_cli_args": [
"--no-async-chunk"
]
},
"benchmark_params": [
{
"dataset_name": "random",
"backend": "openai-chat-omni",
"endpoint": "/v1/chat/completions",
"num_prompts": [4, 16, 32, 64, 128],
"max_concurrency": [1, 4, 8, 16, 32],
"num_prompts": [
4,
16,
32,
64,
128
],
"max_concurrency": [
1,
4,
8,
16,
32
],
"random_input_len": 2500,
"random_output_len": 900,
"ignore_eos": true,
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
"baseline": {
"mean_ttft_ms": [1000, 3000, 5000, 7000, 9000],
"mean_audio_ttfp_ms": [30000, 60000, 90000, 120000, 150000],
"mean_audio_rtf": [0.35, 0.45, 0.55, 0.65, 0.75]
"mean_ttft_ms": [
400,
300,
400,
600,
900
],
"mean_e2el_ms": [
32000,
34000,
45000,
61000,
90000
],
"mean_audio_rtf": [
0.20,
0.25,
0.30,
0.45,
0.60
]
}
},
{
"dataset_name": "random-mm",
"backend": "openai-chat-omni",
"endpoint": "/v1/chat/completions",
"num_prompts": [10],
"request_rate": [0.1],
"num_prompts": [
10
],
"request_rate": [
0.1
],
"random_input_len": 100,
"random_output_len": 100,
"random_range_ratio": 0.0,
@@ -42,17 +78,27 @@
},
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
"baseline": {
"mean_ttft_ms": [2000],
"mean_audio_ttfp_ms": [10000],
"mean_audio_rtf": [0.25]
"mean_ttft_ms": [
500
],
"mean_e2el_ms": [
6000
],
"mean_audio_rtf": [
0.20
]
}
},
{
"dataset_name": "random-mm",
"backend": "openai-chat-omni",
"endpoint": "/v1/chat/completions",
"num_prompts": [40],
"request_rate": [0.5],
"num_prompts": [
40
],
"request_rate": [
0.5
],
"random_input_len": 100,
"random_output_len": 100,
"random_range_ratio": 0.0,
@@ -69,17 +115,27 @@
},
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
"baseline": {
"mean_ttft_ms": [6000],
"mean_audio_ttfp_ms": [15000],
"mean_audio_rtf": [0.45]
"mean_ttft_ms": [
300
],
"mean_e2el_ms": [
7000
],
"mean_audio_rtf": [
0.20
]
}
},
{
"dataset_name": "random-mm",
"backend": "openai-chat-omni",
"endpoint": "/v1/chat/completions",
"num_prompts": [100],
"request_rate": [1.0],
"num_prompts": [
100
],
"request_rate": [
1.0
],
"random_input_len": 100,
"random_output_len": 100,
"random_range_ratio": 0.0,
@@ -98,9 +154,15 @@
},
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
"baseline": {
"mean_ttft_ms": [12000],
"mean_audio_ttfp_ms": [18000],
"mean_audio_rtf": [0.9]
"mean_ttft_ms": [
700
],
"mean_e2el_ms": [
9000
],
"mean_audio_rtf": [
0.30
]
}
}
]
tests/dfx/perf/tests/test_qwen3_omni_vllm_text.json
+79 -17
View File
@@ -4,7 +4,9 @@
"server_params": {
"model": "Qwen/Qwen3-Omni-30B-A3B-Instruct",
"use_omni": false,
"extra_cli_args": ["--no-enable-prefix-caching"]
"extra_cli_args": [
"--no-enable-prefix-caching"
]
},
"benchmark_params": [
{
@@ -12,8 +14,20 @@
"backend": "vllm",
"model": "Qwen/Qwen3-Omni-30B-A3B-Instruct",
"endpoint": "/v1/completions",
"num_prompts": [4, 16, 32, 64, 128],
"max_concurrency": [1, 4, 8, 16, 32],
"num_prompts": [
4,
16,
32,
64,
128
],
"max_concurrency": [
1,
4,
8,
16,
32
],
"random_input_len": 2500,
"random_output_len": 900,
"temperature": 0.4,
@@ -23,11 +37,26 @@
"repetition_penalty": 1.05,
"ignore_eos": true,
"extra_body": {
"modalities": ["text"]
"modalities": [
"text"
]
},
"percentile-metrics": "ttft,tpot,itl,e2el",
"baseline": {
"mean_ttft_ms": [1000, 3000, 5000, 7000, 9000]
"mean_ttft_ms": [
300,
300,
1000,
600,
800
],
"mean_e2el_ms": [
6000,
8000,
11000,
13000,
17000
]
}
},
{
@@ -35,8 +64,12 @@
"backend": "vllm",
"model": "Qwen/Qwen3-Omni-30B-A3B-Instruct",
"endpoint": "/v1/completions",
"num_prompts": [10],
"request_rate": [0.1],
"num_prompts": [
10
],
"request_rate": [
0.1
],
"random_input_len": 100,
"random_output_len": 100,
"random_range_ratio": 0.0,
@@ -47,7 +80,9 @@
"repetition_penalty": 1.05,
"ignore_eos": true,
"extra_body": {
"modalities": ["text"]
"modalities": [
"text"
]
},
"random_mm_base_items_per_request": 1,
"random_mm_num_mm_items_range_ratio": 0.5,
@@ -59,7 +94,12 @@
},
"percentile-metrics": "ttft,tpot,itl,e2el",
"baseline": {
"mean_ttft_ms": [2000]
"mean_ttft_ms": [
30
],
"mean_e2el_ms": [
540
]
}
},
{
@@ -67,8 +107,12 @@
"backend": "vllm",
"model": "Qwen/Qwen3-Omni-30B-A3B-Instruct",
"endpoint": "/v1/completions",
"num_prompts": [40],
"request_rate": [0.5],
"num_prompts": [
40
],
"request_rate": [
0.5
],
"random_input_len": 100,
"random_output_len": 100,
"random_range_ratio": 0.0,
@@ -79,7 +123,9 @@
"repetition_penalty": 1.05,
"ignore_eos": true,
"extra_body": {
"modalities": ["text"]
"modalities": [
"text"
]
},
"random_mm_base_items_per_request": 2,
"random_mm_num_mm_items_range_ratio": 0.5,
@@ -93,7 +139,12 @@
},
"percentile-metrics": "ttft,tpot,itl,e2el",
"baseline": {
"mean_ttft_ms": [6000]
"mean_ttft_ms": [
30
],
"mean_e2el_ms": [
600
]
}
},
{
@@ -101,8 +152,12 @@
"backend": "vllm",
"model": "Qwen/Qwen3-Omni-30B-A3B-Instruct",
"endpoint": "/v1/completions",
"num_prompts": [100],
"request_rate": [1.0],
"num_prompts": [
100
],
"request_rate": [
1.0
],
"random_input_len": 100,
"random_output_len": 100,
"random_range_ratio": 0.0,
@@ -113,7 +168,9 @@
"repetition_penalty": 1.05,
"ignore_eos": true,
"extra_body": {
"modalities": ["text"]
"modalities": [
"text"
]
},
"random_mm_base_items_per_request": 3,
"random_mm_num_mm_items_range_ratio": 0.5,
@@ -129,7 +186,12 @@
},
"percentile-metrics": "ttft,tpot,itl,e2el",
"baseline": {
"mean_ttft_ms": [6000]
"mean_ttft_ms": [
40
],
"mean_e2el_ms": [
700
]
}
}
]
tests/dfx/perf/tests/test_qwen_image_edit_2509_vllm_omni.json
+22 -22
View File
@@ -22,10 +22,10 @@
"num-input-images": 2,
"enable-negative-prompt": true,
"baseline": {
"throughput_qps": 0.05,
"latency_mean": 18,
"peak_memory_mb_max": 78500,
"peak_memory_mb_mean": 78500
"throughput_qps": 0.06,
"latency_mean": 20,
"peak_memory_mb_max": 57000,
"peak_memory_mb_mean": 57000
}
},
{
@@ -41,9 +41,9 @@
"enable-negative-prompt": true,
"baseline": {
"throughput_qps": 0.01,
"latency_mean": 70,
"peak_memory_mb_max": 81000,
"peak_memory_mb_mean": 81000
"latency_mean": 60,
"peak_memory_mb_max": 67000,
"peak_memory_mb_mean": 67000
}
}
]
@@ -75,10 +75,10 @@
"num-input-images": 2,
"enable-negative-prompt": true,
"baseline": {
"throughput_qps": 0.1,
"latency_mean": 12,
"peak_memory_mb_max": 69000,
"peak_memory_mb_mean": 69000
"throughput_qps": 0.15,
"latency_mean": 10,
"peak_memory_mb_max": 55000,
"peak_memory_mb_mean": 55000
}
},
{
@@ -93,10 +93,10 @@
"num-input-images": 2,
"enable-negative-prompt": true,
"baseline": {
"throughput_qps": 0.03,
"latency_mean": 28,
"peak_memory_mb_max": 69000,
"peak_memory_mb_mean": 69000
"throughput_qps": 0.05,
"latency_mean": 20,
"peak_memory_mb_max": 55000,
"peak_memory_mb_mean": 55000
}
}
]
@@ -138,10 +138,10 @@
"num-input-images": 2,
"enable-negative-prompt": true,
"baseline": {
"throughput_qps": 0.10,
"latency_mean": 12,
"peak_memory_mb_max": 73000,
"peak_memory_mb_mean": 73000
"throughput_qps": 0.21,
"latency_mean": 10,
"peak_memory_mb_max": 57000,
"peak_memory_mb_mean": 57000
}
},
{
@@ -156,10 +156,10 @@
"num-input-images": 2,
"enable-negative-prompt": true,
"baseline": {
"throughput_qps": 0.05,
"throughput_qps": 0.08,
"latency_mean": 20,
"peak_memory_mb_max": 81000,
"peak_memory_mb_mean": 81000
"peak_memory_mb_max": 66000,
"peak_memory_mb_mean": 66000
}
}
]
tests/dfx/perf/tests/test_qwen_image_edit_vllm_omni.json
+24 -24
View File
@@ -21,10 +21,10 @@
"max-concurrency": 1,
"enable-negative-prompt": true,
"baseline": {
"throughput_qps": 0.05,
"latency_mean": 15.0,
"peak_memory_mb_max": 72500,
"peak_memory_mb_mean": 72500
"throughput_qps": 0.1,
"latency_mean": 10,
"peak_memory_mb_max": 57000,
"peak_memory_mb_mean": 57000
}
},
{
@@ -38,10 +38,10 @@
"max-concurrency": 1,
"enable-negative-prompt": true,
"baseline": {
"throughput_qps": 0.01,
"latency_mean": 65.6,
"peak_memory_mb_max": 80777,
"peak_memory_mb_mean": 80777
"throughput_qps": 0.02,
"latency_mean": 50,
"peak_memory_mb_max": 65000,
"peak_memory_mb_mean": 65000
}
}
]
@@ -72,10 +72,10 @@
"max-concurrency": 1,
"enable-negative-prompt": true,
"baseline": {
"throughput_qps": 0.10,
"latency_mean": 7.2,
"peak_memory_mb_max": 68100,
"peak_memory_mb_mean": 68100
"throughput_qps": 0.21,
"latency_mean": 10,
"peak_memory_mb_max": 56000,
"peak_memory_mb_mean": 56000
}
},
{
@@ -89,10 +89,10 @@
"max-concurrency": 1,
"enable-negative-prompt": true,
"baseline": {
"throughput_qps": 0.03,
"latency_mean": 24.0,
"peak_memory_mb_max": 68100,
"peak_memory_mb_mean": 68100
"throughput_qps": 0.06,
"latency_mean": 20,
"peak_memory_mb_max": 56000,
"peak_memory_mb_mean": 56000
}
}
]
@@ -133,10 +133,10 @@
"max-concurrency": 1,
"enable-negative-prompt": true,
"baseline": {
"throughput_qps": 0.1,
"latency_mean": 6.5,
"peak_memory_mb_max": 72600,
"peak_memory_mb_mean": 72600
"throughput_qps": 0.26,
"latency_mean": 10,
"peak_memory_mb_max": 58000,
"peak_memory_mb_mean": 58000
}
},
{
@@ -150,10 +150,10 @@
"max-concurrency": 1,
"enable-negative-prompt": true,
"baseline": {
"throughput_qps": 0.05,
"latency_mean": 16.0,
"peak_memory_mb_max": 81000,
"peak_memory_mb_mean": 81000
"throughput_qps": 0.1,
"latency_mean": 10,
"peak_memory_mb_max": 67000,
"peak_memory_mb_mean": 67000
}
}
]
tests/dfx/perf/tests/test_qwen_image_layered_vllm_omni.json
+8 -8
View File
@@ -21,10 +21,10 @@
"max-concurrency": 1,
"enable-negative-prompt": true,
"baseline": {
"throughput_qps": 0.02,
"latency_mean": 40.0,
"peak_memory_mb_max": 70000,
"peak_memory_mb_mean": 70000
"throughput_qps": 0.05,
"latency_mean": 20,
"peak_memory_mb_max": 63000,
"peak_memory_mb_mean": 63000
}
},
{
@@ -38,10 +38,10 @@
"max-concurrency": 1,
"enable-negative-prompt": true,
"baseline": {
"throughput_qps": 0.005,
"latency_mean": 80.0,
"peak_memory_mb_max": 70000,
"peak_memory_mb_mean": 70000
"throughput_qps": 0.03,
"latency_mean": 30,
"peak_memory_mb_max": 63000,
"peak_memory_mb_mean": 63000
}
}
]
tests/dfx/perf/tests/test_qwen_image_vllm_omni.json
+21 -21
View File
@@ -21,9 +21,9 @@
"max-concurrency": 1,
"enable-negative-prompt": true,
"baseline": {
"throughput_qps": 0.30,
"latency_mean": 3.50,
"peak_memory_mb_mean": 67000
"throughput_qps": 0.35,
"latency_mean": 3,
"peak_memory_mb_mean": 56000
}
},
{
@@ -37,9 +37,9 @@
"max-concurrency": 1,
"enable-negative-prompt": true,
"baseline": {
"throughput_qps": 0.037,
"latency_mean": 27.0,
"peak_memory_mb_mean": 74000
"throughput_qps": 0.03,
"latency_mean": 26,
"peak_memory_mb_mean": 65000
}
}
]
@@ -67,9 +67,9 @@
"max-concurrency": 1,
"enable-negative-prompt": true,
"baseline": {
"throughput_qps": 0.30,
"latency_mean": 3.50,
"peak_memory_mb_mean": 67000
"throughput_qps": 0.35,
"latency_mean": 3,
"peak_memory_mb_mean": 56000
}
},
{
@@ -83,9 +83,9 @@
"max-concurrency": 1,
"enable-negative-prompt": true,
"baseline": {
"throughput_qps": 0.037,
"latency_mean": 27.0,
"peak_memory_mb_mean": 74000
"throughput_qps": 0.03,
"latency_mean": 26,
"peak_memory_mb_mean": 65000
}
}
]
@@ -116,9 +116,9 @@
"max-concurrency": 1,
"enable-negative-prompt": true,
"baseline": {
"throughput_qps": 0.1,
"latency_mean": 9.1,
"peak_memory_mb_mean": 61000
"throughput_qps": 0.11,
"latency_mean": 9,
"peak_memory_mb_mean": 55000
}
}
]
@@ -159,9 +159,9 @@
"max-concurrency": 1,
"enable-negative-prompt": true,
"baseline": {
"throughput_qps": 0.1,
"latency_mean": 3.50,
"peak_memory_mb_mean": 67000
"throughput_qps": 0.46,
"latency_mean": 3,
"peak_memory_mb_mean": 56000
}
},
{
@@ -175,9 +175,9 @@
"max-concurrency": 1,
"enable-negative-prompt": true,
"baseline": {
"throughput_qps": 0.1,
"latency_mean": 6.15,
"peak_memory_mb_mean": 74000
"throughput_qps": 0.18,
"latency_mean": 6,
"peak_memory_mb_mean": 67000
}
}
]
tests/dfx/perf/tests/test_tts.json
+197 -52
View File
@@ -4,7 +4,9 @@
"server_params": {
"model": "Qwen/Qwen3-TTS-12Hz-1.7B-Base",
"stage_overrides": {
"1": {"devices": "1"}
"1": {
"devices": "1"
}
}
},
"benchmark_params": [
@@ -15,13 +17,21 @@
"backend": "openai-audio-speech",
"endpoint": "/v1/audio/speech",
"dataset_path": "linyueqian/seed-tts-eval-subset",
"num_prompts": [20],
"max_concurrency": [1],
"num_prompts": [
20
],
"max_concurrency": [
1
],
"seed_tts_locale": "en",
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
"baseline": {
"median_audio_ttfp_ms": [350],
"median_audio_rtf": [0.25]
"median_audio_ttfp_ms": [
3000
],
"median_audio_rtf": [
0.6
]
}
},
{
@@ -31,14 +41,34 @@
"backend": "openai-audio-speech",
"endpoint": "/v1/audio/speech",
"dataset_path": "linyueqian/seed-tts-eval-subset",
"num_prompts": [80, 128, 128],
"max_concurrency": [8, 16, 64],
"num_prompts": [
80,
128,
128
],
"max_concurrency": [
8,
16,
64
],
"seed_tts_locale": "en",
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
"baseline": {
"median_audio_ttfp_ms": [3500, 30000, 7000],
"median_audio_rtf": [0.75, 10.0, 1.5],
"audio_throughput": [10.0, 1.0, 40.0]
"median_audio_ttfp_ms": [
1000,
3000,
14000
],
"median_audio_rtf": [
0.70,
1.15,
3.90
],
"audio_throughput": [
12,
14,
14
]
}
},
{
@@ -49,13 +79,19 @@
"backend": "openai-audio-speech",
"endpoint": "/v1/audio/speech",
"dataset_path": "linyueqian/seed-tts-eval-subset",
"num_prompts": [200],
"max_concurrency": [4],
"num_prompts": [
200
],
"max_concurrency": [
4
],
"seed_tts_locale": "en",
"seed_tts_wer_eval": true,
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
"baseline": {
"mean_audio_rtf": [0.45]
"mean_audio_rtf": [
0.45
]
}
}
]
@@ -65,7 +101,9 @@
"server_params": {
"model": "Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice",
"stage_overrides": {
"1": {"devices": "1"}
"1": {
"devices": "1"
}
}
},
"benchmark_params": [
@@ -76,14 +114,26 @@
"backend": "openai-audio-speech",
"endpoint": "/v1/audio/speech",
"dataset_path": "benchmarks/build_dataset/seed_tts_smoke",
"num_prompts": [20],
"max_concurrency": [1],
"num_prompts": [
20
],
"max_concurrency": [
1
],
"seed_tts_locale": "en",
"extra_body": {"voice": "Vivian", "language": "English", "task_type": "CustomVoice"},
"extra_body": {
"voice": "Vivian",
"language": "English",
"task_type": "CustomVoice"
},
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
"baseline": {
"median_audio_ttfp_ms": [150],
"median_audio_rtf": [0.15]
"median_audio_ttfp_ms": [
2000
],
"median_audio_rtf": [
0.40
]
}
},
{
@@ -93,15 +143,39 @@
"backend": "openai-audio-speech",
"endpoint": "/v1/audio/speech",
"dataset_path": "benchmarks/build_dataset/seed_tts_smoke",
"num_prompts": [80, 128, 128],
"max_concurrency": [8, 16, 64],
"num_prompts": [
80,
128,
128
],
"max_concurrency": [
8,
16,
64
],
"seed_tts_locale": "en",
"extra_body": {"voice": "Vivian", "language": "English", "task_type": "CustomVoice"},
"extra_body": {
"voice": "Vivian",
"language": "English",
"task_type": "CustomVoice"
},
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
"baseline": {
"median_audio_ttfp_ms": [1500, 30000, 3000],
"median_audio_rtf": [0.30, 10.0, 0.60],
"audio_throughput": [30.0, 1.0, 100.0]
"median_audio_ttfp_ms": [
300,
1000,
5000
],
"median_audio_rtf": [
0.25,
0.35,
1.10
],
"audio_throughput": [
30,
40,
40
]
}
},
{
@@ -112,14 +186,24 @@
"backend": "openai-audio-speech",
"endpoint": "/v1/audio/speech",
"dataset_path": "benchmarks/build_dataset/seed_tts_smoke",
"num_prompts": [200],
"max_concurrency": [4],
"num_prompts": [
200
],
"max_concurrency": [
4
],
"seed_tts_locale": "en",
"extra_body": {"voice": "Vivian", "language": "English", "task_type": "CustomVoice"},
"extra_body": {
"voice": "Vivian",
"language": "English",
"task_type": "CustomVoice"
},
"seed_tts_wer_eval": true,
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
"baseline": {
"mean_audio_rtf": [0.35]
"mean_audio_rtf": [
0.35
]
}
},
{
@@ -129,15 +213,29 @@
"backend": "openai-audio-speech",
"endpoint": "/v1/audio/speech",
"dataset_path": "benchmarks/build_dataset/seed_tts_smoke",
"num_prompts": [100],
"request_rate": [2.0],
"num_prompts": [
100
],
"request_rate": [
2.0
],
"seed_tts_locale": "en",
"extra_body": {"voice": "Vivian", "language": "English", "task_type": "CustomVoice"},
"extra_body": {
"voice": "Vivian",
"language": "English",
"task_type": "CustomVoice"
},
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
"baseline": {
"median_audio_ttfp_ms": [3000],
"median_audio_rtf": [0.25],
"audio_throughput": [4.0]
"median_audio_ttfp_ms": [
100
],
"median_audio_rtf": [
0.20
],
"audio_throughput": [
10
]
}
},
{
@@ -147,14 +245,25 @@
"backend": "openai-audio-speech",
"endpoint": "/v1/audio/speech",
"dataset_path": "benchmarks/build_dataset/seed_tts_design",
"num_prompts": [20],
"max_concurrency": [1],
"num_prompts": [
20
],
"max_concurrency": [
1
],
"seed_tts_locale": "en",
"extra_body": {"task_type": "VoiceDesign", "language": "English"},
"extra_body": {
"task_type": "VoiceDesign",
"language": "English"
},
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
"baseline": {
"median_audio_ttfp_ms": [150],
"median_audio_rtf": [0.15]
"median_audio_ttfp_ms": [
100
],
"median_audio_rtf": [
0.15
]
}
},
{
@@ -164,15 +273,38 @@
"backend": "openai-audio-speech",
"endpoint": "/v1/audio/speech",
"dataset_path": "benchmarks/build_dataset/seed_tts_design",
"num_prompts": [80, 128, 128],
"max_concurrency": [8, 16, 64],
"num_prompts": [
80,
128,
128
],
"max_concurrency": [
8,
16,
64
],
"seed_tts_locale": "en",
"extra_body": {"task_type": "VoiceDesign", "language": "English"},
"extra_body": {
"task_type": "VoiceDesign",
"language": "English"
},
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
"baseline": {
"median_audio_ttfp_ms": [1500, 30000, 3500],
"median_audio_rtf": [0.35, 10.0, 0.70],
"audio_throughput": [25.0, 1.0, 80.0]
"median_audio_ttfp_ms": [
100,
1000,
5000
],
"median_audio_rtf": [
0.25,
0.40,
1.20
],
"audio_throughput": [
30,
40,
40
]
}
},
{
@@ -182,15 +314,28 @@
"backend": "openai-audio-speech",
"endpoint": "/v1/audio/speech",
"dataset_path": "benchmarks/build_dataset/seed_tts_design",
"num_prompts": [100],
"request_rate": [2.0],
"num_prompts": [
100
],
"request_rate": [
2.0
],
"seed_tts_locale": "en",
"extra_body": {"task_type": "VoiceDesign", "language": "English"},
"extra_body": {
"task_type": "VoiceDesign",
"language": "English"
},
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
"baseline": {
"median_audio_ttfp_ms": [3500],
"median_audio_rtf": [0.30],
"audio_throughput": [4.0]
"median_audio_ttfp_ms": [
100
],
"median_audio_rtf": [
0.20
],
"audio_throughput": [
9
]
}
}
]
tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json
+9 -9
View File
@@ -30,9 +30,9 @@
}
],
"baseline": {
"throughput_qps": 0.034,
"latency_mean": 26.0,
"peak_memory_mb_mean": 80000
"throughput_qps": 0.04,
"latency_mean": 30,
"peak_memory_mb_mean": 75000
}
}
]
@@ -72,9 +72,9 @@
}
],
"baseline": {
"throughput_qps": 0.042,
"latency_mean": 21.6,
"peak_memory_mb_mean": 55300
"throughput_qps": 0.06,
"latency_mean": 20,
"peak_memory_mb_mean": 48000
}
},
{
@@ -97,9 +97,9 @@
}
],
"baseline": {
"throughput_qps": 0.0085,
"latency_mean": 101.6,
"peak_memory_mb_mean": 65200
"throughput_qps": 0.01,
"latency_mean": 90,
"peak_memory_mb_mean": 56000
}
}
]