diff --git a/tests/dfx/conftest.py b/tests/dfx/conftest.py index 9332a24f5..a22f760b2 100644 --- a/tests/dfx/conftest.py +++ b/tests/dfx/conftest.py @@ -279,6 +279,10 @@ def create_benchmark_indices( return indices +# Omni perf/stability: ``vllm bench serve --omni`` result template (used if bench exits without writing JSON). +OMNI_RESULT_TEMPLATE_PATH = Path(__file__).resolve().parent / "perf" / "scripts" / "result_omni_template.json" + + def _safe_filename_token(value: Any | None, *, default: str = "na") -> str: """Make a single path segment safe for result filenames on common filesystems.""" if value is None: @@ -289,7 +293,7 @@ def _safe_filename_token(value: Any | None, *, default: str = "na") -> str: return s if s else default -def _resolve_baseline_value( +def resolve_baseline_value( baseline_raw: Any, *, sweep_index: int | None, @@ -330,7 +334,7 @@ def _baseline_thresholds_for_step( ) -> dict[str, Any]: """Resolve baseline config to one threshold per metric for this iteration.""" return { - metric_name: _resolve_baseline_value( + metric_name: resolve_baseline_value( baseline_raw, sweep_index=sweep_index, max_concurrency=max_concurrency, @@ -354,7 +358,12 @@ def run_benchmark( random_input_len: Any | None = None, random_output_len: Any | None = None, ) -> dict[str, Any]: - """Run one ``vllm bench serve --omni`` iteration and return parsed metrics.""" + """Run one ``vllm bench serve --omni`` iteration and return parsed metrics. + + After ``vllm bench`` writes the JSON, ``result["baseline"]`` holds the resolved per-metric thresholds + (when ``baseline_config`` is provided). If the benchmark exits without writing a result file, + ``result_omni_template.json`` is used as a fallback. + """ current_dt = datetime.now().strftime("%Y%m%d-%H%M%S") ri = _safe_filename_token(random_input_len) ro = _safe_filename_token(random_output_len) @@ -403,8 +412,17 @@ def run_benchmark( result_dir = "./" result_path = os.path.join(result_dir, result_filename) - with open(result_path, encoding="utf-8") as f: - result = json.load(f) + if not os.path.exists(result_path): + with open(OMNI_RESULT_TEMPLATE_PATH, encoding="utf-8") as f: + template_result: dict[str, Any] = json.load(f) + Path(result_path).parent.mkdir(parents=True, exist_ok=True) + with open(result_path, "w", encoding="utf-8") as f: + json.dump(template_result, f, ensure_ascii=False, indent=2) + print(f"Benchmark result file not generated, fallback to template: {result_path}") + result = template_result + else: + with open(result_path, encoding="utf-8") as f: + result = json.load(f) if baseline_config: result["baseline"] = _baseline_thresholds_for_step( @@ -432,3 +450,12 @@ def pytest_addoption(parser: pytest.Parser) -> None: default=None, help=("Path to benchmark config JSON. Example: --test-config-file tests/dfx/perf/tests/test_tts.json"), ) + parser.addoption( + "--assert-baseline", + action="store_true", + default=False, + help=( + "When set, omni/diffusion perf runners compare metrics against the baseline block in the JSON config " + "(default: off)." + ), + ) diff --git a/tests/dfx/perf/scripts/run_benchmark.py b/tests/dfx/perf/scripts/run_benchmark.py index 74332889c..55ad2f8b2 100644 --- a/tests/dfx/perf/scripts/run_benchmark.py +++ b/tests/dfx/perf/scripts/run_benchmark.py @@ -12,12 +12,17 @@ from tests.dfx.conftest import ( create_unique_server_params, get_benchmark_params_for_server, load_configs, + resolve_baseline_value, run_benchmark, ) from tests.helpers.runtime import OmniServer pytestmark = [pytest.mark.full_model] +# Compare metrics to each test JSON ``baseline`` block only when pytest is run with ``--assert-baseline`` +# (registered in ``tests/dfx/conftest.py``; default: off). ``run_benchmark`` and ``_resolve_baseline_value`` are +# defined in the same module. + os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" @@ -46,7 +51,6 @@ if CONFIG_FILE_PATH is None: CONFIG_FILE_PATH = _DEFAULT_CONFIG_FILE BENCHMARK_CONFIGS = load_configs(CONFIG_FILE_PATH) -OMNI_RESULT_TEMPLATE_PATH = Path(__file__).parent / "result_omni_template.json" DEPLOY_CONFIGS_DIR = Path(__file__).parent.parent / "deploy" @@ -87,16 +91,6 @@ def omni_server(request): print("OmniServer stopped") -def _safe_filename_token(value: Any | None, *, default: str = "na") -> str: - """Make a single path segment safe for result filenames on common filesystems.""" - if value is None: - return default - s = str(value).strip() - for bad in ("/", "\\", ":", "*", "?", '"', "<", ">", "|"): - s = s.replace(bad, "_") - return s if s else default - - benchmark_indices = create_benchmark_indices(BENCHMARK_CONFIGS, server_to_benchmark_mapping) @@ -126,76 +120,23 @@ def benchmark_params(request, omni_server): } -def _resolve_baseline_value( - baseline_raw: Any, - *, - sweep_index: int | None, - max_concurrency: Any = None, - request_rate: Any = None, -) -> Any: - """Pick the baseline threshold for this sweep step. - - Supported shapes per metric: - - **Scalar** — same threshold for every concurrency / QPS. - - **List** — aligned with ``max_concurrency`` / ``request_rate`` sweep order; use ``sweep_index``. - - **Dict** — keyed by concurrency or rate, e.g. ``{"1": 500, "4": 800}`` (keys are strings in JSON). - - For dict lookup, ``max_concurrency`` is preferred when both are set (concurrency sweep). - """ - if baseline_raw is None: - # If no baseline is set, the maximum value will be used. - return 100000 - if isinstance(baseline_raw, dict): - if max_concurrency is not None: - for key in (max_concurrency, str(max_concurrency)): - if key in baseline_raw: - return baseline_raw[key] - if request_rate is not None: - for key in (request_rate, str(request_rate)): - if key in baseline_raw: - return baseline_raw[key] - raise KeyError( - f"baseline dict has no key for max_concurrency={max_concurrency!r} " - f"or request_rate={request_rate!r}; keys={list(baseline_raw.keys())!r}" - ) - if isinstance(baseline_raw, (list, tuple)): - return baseline_raw[sweep_index] - return baseline_raw - - -def _baseline_thresholds_for_step( - baseline_data: dict[str, Any], - *, - sweep_index: int | None = None, - max_concurrency: Any = None, - request_rate: Any = None, -) -> dict[str, Any]: - """Resolve ``test.json`` ``baseline`` block to one threshold per metric (same as ``assert_result``).""" - return { - metric_name: _resolve_baseline_value( - baseline_raw, - sweep_index=sweep_index, - max_concurrency=max_concurrency, - request_rate=request_rate, - ) - for metric_name, baseline_raw in baseline_data.items() - } - - def assert_result( result, params, num_prompt, *, + assert_baseline: bool, sweep_index: int | None = None, - max_concurrency: Any = None, - request_rate: Any = None, + max_concurrency: Any | None = None, + request_rate: Any | None = None, ) -> None: assert result["completed"] == num_prompt, "Request failures exist" + if not assert_baseline: + return baseline_data = params.get("baseline", {}) for metric_name, baseline_raw in baseline_data.items(): current_value = result[metric_name] - baseline_value = _resolve_baseline_value( + baseline_value = resolve_baseline_value( baseline_raw, sweep_index=sweep_index, max_concurrency=max_concurrency, @@ -214,7 +155,7 @@ def assert_result( @pytest.mark.benchmark @pytest.mark.parametrize("omni_server", test_params, indirect=True) @pytest.mark.parametrize("benchmark_params", benchmark_indices, indirect=True) -def test_performance_benchmark(omni_server, benchmark_params): +def test_performance_benchmark(omni_server, benchmark_params, request): test_name = benchmark_params["test_name"] params = benchmark_params["params"] dataset_name = params.get("dataset_name", "") @@ -226,6 +167,8 @@ def test_performance_benchmark(omni_server, benchmark_params): print(f"Running benchmark for model: {model}") print(f"Benchmark parameters: {benchmark_params}") + assert_baseline = request.config.getoption("--assert-baseline", default=False) + def to_list(value, default=None): if value is None: return [] if default is None else [default] @@ -264,7 +207,7 @@ def test_performance_benchmark(omni_server, benchmark_params): elif not isinstance(value, bool): args.extend([arg_name, str(value)]) - # QPS test (sweep_index aligns with qps_list / num_prompt_list for this loop) + # QPS / request-rate sweep for i, (qps, num_prompt) in enumerate(zip(qps_list, num_prompt_list)): args = args + ["--request-rate", str(qps), "--num-prompts", str(num_prompt)] result = run_benchmark( @@ -283,12 +226,13 @@ def test_performance_benchmark(omni_server, benchmark_params): assert_result( result, params, - num_prompt=num_prompt, + num_prompt, + assert_baseline=assert_baseline, sweep_index=i, request_rate=qps, ) - # concurrency test (sweep_index aligns with max_concurrency_list for separate thresholds per concurrency) + # concurrency test for i, (concurrency, num_prompt) in enumerate(zip(max_concurrency_list, num_prompt_list)): args = args + ["--max-concurrency", str(concurrency), "--num-prompts", str(num_prompt), "--request-rate", "inf"] result = run_benchmark( @@ -307,7 +251,8 @@ def test_performance_benchmark(omni_server, benchmark_params): assert_result( result, params, - num_prompt=num_prompt, + num_prompt, + assert_baseline=assert_baseline, sweep_index=i, max_concurrency=concurrency, ) diff --git a/tests/dfx/perf/scripts/run_diffusion_benchmark.py b/tests/dfx/perf/scripts/run_diffusion_benchmark.py index 9059904e4..22e844427 100644 --- a/tests/dfx/perf/scripts/run_diffusion_benchmark.py +++ b/tests/dfx/perf/scripts/run_diffusion_benchmark.py @@ -11,6 +11,8 @@ This runner separates two concepts: A config JSON file is REQUIRED via --test-config-file: pytest run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_vllm_omni.json +Optional: ``--assert-baseline`` compares metrics to the ``baseline`` block in each benchmark entry (default: off). + All benchmark results for a session are consolidated into a single JSON file under BENCHMARK_RESULT_DIR (override via the DIFFUSION_BENCHMARK_DIR environment variable). Each entry in the file contains the test metadata (test_name, endpoint, benchmark_params, @@ -652,12 +654,14 @@ def run_benchmark( # --------------------------------------------------------------------------- -def assert_result(result: dict[str, Any], params: dict[str, Any]) -> None: - """Assert that benchmark metrics satisfy the configured baselines.""" +def assert_result(result: dict[str, Any], params: dict[str, Any], *, assert_baseline: bool) -> None: + """Assert request completion; optionally assert metrics against ``baseline`` in *params*.""" num_prompts = params.get("num-prompts", 10) completed = result.get("completed_requests", result.get("completed", 0)) assert completed == num_prompts, f"Expected {num_prompts} completed requests, got {completed}" + if not assert_baseline: + return if params.get("skip-performance-assertion", False): print("Skipping performance assertions.") return @@ -699,16 +703,14 @@ def _resolve_benchmark_endpoint(server_cfg: dict[str, Any], params: dict[str, An indirect=True, ) @pytest.mark.parametrize("benchmark_params", benchmark_indices, indirect=True) -def test_diffusion_performance_benchmark(diffusion_server, benchmark_params): - """Run the diffusion performance benchmark and assert against baselines. +def test_diffusion_performance_benchmark(diffusion_server, benchmark_params, request): + """Run the diffusion performance benchmark and verify request completion. One server is started per unique parallel configuration (module scope). For each server, all benchmark parameter sets defined in the config JSON - are executed sequentially; results are asserted against the baselines. + are executed sequentially; metrics are recorded to the aggregated result file. - Tracked metrics: - - throughput_qps (higher is better) - - latency_p50, latency_p99 (lower is better) + Pass ``--assert-baseline`` to compare ``throughput_qps``, ``latency_mean``, etc. to the JSON ``baseline`` block. """ test_name = benchmark_params["test_name"] params = benchmark_params["params"] @@ -744,4 +746,5 @@ def test_diffusion_performance_benchmark(diffusion_server, benchmark_params): print(f"\n Aggregated results: {AGGREGATED_RESULT_FILE}") print("=" * 60) - assert_result(result, params) + assert_baseline = request.config.getoption("--assert-baseline", default=False) + assert_result(result, params, assert_baseline=assert_baseline) diff --git a/tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json b/tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json index 98e311748..bc954cee5 100644 --- a/tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json +++ b/tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json @@ -3,31 +3,67 @@ "test_name": "test_qwen3_omni_chunk", "server_params": { "model": "Qwen/Qwen3-Omni-30B-A3B-Instruct", - "extra_cli_args": ["--async-chunk"] + "extra_cli_args": [ + "--async-chunk" + ] }, "benchmark_params": [ { "dataset_name": "random", "backend": "openai-chat-omni", "endpoint": "/v1/chat/completions", - "num_prompts": [4, 16, 32, 64, 128], - "max_concurrency": [1, 4, 8, 16, 32], + "num_prompts": [ + 4, + 16, + 32, + 64, + 128 + ], + "max_concurrency": [ + 1, + 4, + 8, + 16, + 32 + ], "random_input_len": 2500, "random_output_len": 900, "ignore_eos": true, "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { - "mean_ttft_ms": [1000, 3000, 5000, 7000, 9000], - "mean_audio_ttfp_ms": [1000, 3000, 5000, 7000, 9000], - "mean_audio_rtf": [0.2, 0.35, 0.6, 0.85, 0.9] + "mean_ttft_ms": [ + 400, + 300, + 400, + 600, + 900 + ], + "mean_e2el_ms": [ + 26000, + 43000, + 52000, + 72000, + 109000 + ], + "mean_audio_rtf": [ + 0.15, + 0.30, + 0.35, + 0.50, + 0.75 + ] } }, { "dataset_name": "random-mm", "backend": "openai-chat-omni", "endpoint": "/v1/chat/completions", - "num_prompts": [10], - "request_rate": [0.1], + "num_prompts": [ + 10 + ], + "request_rate": [ + 0.1 + ], "random_input_len": 100, "random_output_len": 100, "random_range_ratio": 0.0, @@ -42,17 +78,27 @@ }, "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { - "mean_ttft_ms": [2000], - "mean_audio_ttfp_ms": [2000], - "mean_audio_rtf": [0.25] + "mean_ttft_ms": [ + 600 + ], + "mean_e2el_ms": [ + 6000 + ], + "mean_audio_rtf": [ + 0.20 + ] } }, { "dataset_name": "random-mm", "backend": "openai-chat-omni", "endpoint": "/v1/chat/completions", - "num_prompts": [40], - "request_rate": [0.5], + "num_prompts": [ + 40 + ], + "request_rate": [ + 0.5 + ], "random_input_len": 100, "random_output_len": 100, "random_range_ratio": 0.0, @@ -69,17 +115,27 @@ }, "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { - "mean_ttft_ms": [6000], - "mean_audio_ttfp_ms": [6000], - "mean_audio_rtf": [0.7] + "mean_ttft_ms": [ + 300 + ], + "mean_e2el_ms": [ + 7000 + ], + "mean_audio_rtf": [ + 0.25 + ] } }, { "dataset_name": "random-mm", "backend": "openai-chat-omni", "endpoint": "/v1/chat/completions", - "num_prompts": [100], - "request_rate": [1.0], + "num_prompts": [ + 100 + ], + "request_rate": [ + 1.0 + ], "random_input_len": 100, "random_output_len": 100, "random_range_ratio": 0.0, @@ -98,40 +154,79 @@ }, "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { - "mean_ttft_ms": [12000], - "mean_audio_ttfp_ms": [12000], - "mean_audio_rtf": [1.0] + "mean_ttft_ms": [ + 700 + ], + "mean_e2el_ms": [ + 11000 + ], + "mean_audio_rtf": [ + 0.35 + ] } }, { "dataset_name": "random", "backend": "openai-chat-omni", "endpoint": "/v1/chat/completions", - "num_prompts": [4, 16, 32, 64, 128], - "max_concurrency": [1, 4, 8, 16, 32], + "num_prompts": [ + 4, + 16, + 32, + 64, + 128 + ], + "max_concurrency": [ + 1, + 4, + 8, + 16, + 32 + ], "random_input_len": 2500, "random_output_len": 900, "ignore_eos": true, "extra_body": { - "modalities": ["text"] + "modalities": [ + "text" + ] }, "percentile-metrics": "ttft,tpot,itl,e2el", "baseline": { - "mean_ttft_ms": [1000, 3000, 5000, 7000, 9000] + "mean_ttft_ms": [ + 100, + 300, + 500, + 800, + 1500 + ], + "mean_e2el_ms": [ + 5000, + 8000, + 10000, + 14000, + 20000 + ] } }, { "dataset_name": "random-mm", "backend": "openai-chat-omni", "endpoint": "/v1/chat/completions", - "num_prompts": [10], - "request_rate": [0.1], + "num_prompts": [ + 10 + ], + "request_rate": [ + 0.1 + ], "random_input_len": 100, "random_output_len": 100, "random_range_ratio": 0.0, "ignore_eos": true, "extra_body": { - "modalities": ["text"] + "modalities": [ + "text" + ] }, "random_mm_base_items_per_request": 1, "random_mm_num_mm_items_range_ratio": 0.5, @@ -143,21 +238,32 @@ }, "percentile-metrics": "ttft,tpot,itl,e2el", "baseline": { - "mean_ttft_ms": [2000] + "mean_ttft_ms": [ + 500 + ], + "mean_e2el_ms": [ + 1000 + ] } }, { "dataset_name": "random-mm", "backend": "openai-chat-omni", "endpoint": "/v1/chat/completions", - "num_prompts": [40], - "request_rate": [0.5], + "num_prompts": [ + 40 + ], + "request_rate": [ + 0.5 + ], "random_input_len": 100, "random_output_len": 100, "random_range_ratio": 0.0, "ignore_eos": true, "extra_body": { - "modalities": ["text"] + "modalities": [ + "text" + ] }, "random_mm_base_items_per_request": 2, "random_mm_num_mm_items_range_ratio": 0.5, @@ -171,21 +277,32 @@ }, "percentile-metrics": "ttft,tpot,itl,e2el", "baseline": { - "mean_ttft_ms": [6000] + "mean_ttft_ms": [ + 200 + ], + "mean_e2el_ms": [ + 800 + ] } }, { "dataset_name": "random-mm", "backend": "openai-chat-omni", "endpoint": "/v1/chat/completions", - "num_prompts": [100], - "request_rate": [1.0], + "num_prompts": [ + 100 + ], + "request_rate": [ + 1.0 + ], "random_input_len": 100, "random_output_len": 100, "random_range_ratio": 0.0, "ignore_eos": true, "extra_body": { - "modalities": ["text"] + "modalities": [ + "text" + ] }, "random_mm_base_items_per_request": 3, "random_mm_num_mm_items_range_ratio": 0.5, @@ -201,7 +318,12 @@ }, "percentile-metrics": "ttft,tpot,itl,e2el", "baseline": { - "mean_ttft_ms": [6000] + "mean_ttft_ms": [ + 600 + ], + "mean_e2el_ms": [ + 1300 + ] } } ] diff --git a/tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json b/tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json index b91a8a89b..e57acecba 100644 --- a/tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json +++ b/tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json @@ -3,31 +3,67 @@ "test_name": "test_qwen3_omni", "server_params": { "model": "Qwen/Qwen3-Omni-30B-A3B-Instruct", - "extra_cli_args": ["--no-async-chunk"] + "extra_cli_args": [ + "--no-async-chunk" + ] }, "benchmark_params": [ { "dataset_name": "random", "backend": "openai-chat-omni", "endpoint": "/v1/chat/completions", - "num_prompts": [4, 16, 32, 64, 128], - "max_concurrency": [1, 4, 8, 16, 32], + "num_prompts": [ + 4, + 16, + 32, + 64, + 128 + ], + "max_concurrency": [ + 1, + 4, + 8, + 16, + 32 + ], "random_input_len": 2500, "random_output_len": 900, "ignore_eos": true, "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { - "mean_ttft_ms": [1000, 3000, 5000, 7000, 9000], - "mean_audio_ttfp_ms": [30000, 60000, 90000, 120000, 150000], - "mean_audio_rtf": [0.35, 0.45, 0.55, 0.65, 0.75] + "mean_ttft_ms": [ + 400, + 300, + 400, + 600, + 900 + ], + "mean_e2el_ms": [ + 32000, + 34000, + 45000, + 61000, + 90000 + ], + "mean_audio_rtf": [ + 0.20, + 0.25, + 0.30, + 0.45, + 0.60 + ] } }, { "dataset_name": "random-mm", "backend": "openai-chat-omni", "endpoint": "/v1/chat/completions", - "num_prompts": [10], - "request_rate": [0.1], + "num_prompts": [ + 10 + ], + "request_rate": [ + 0.1 + ], "random_input_len": 100, "random_output_len": 100, "random_range_ratio": 0.0, @@ -42,17 +78,27 @@ }, "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { - "mean_ttft_ms": [2000], - "mean_audio_ttfp_ms": [10000], - "mean_audio_rtf": [0.25] + "mean_ttft_ms": [ + 500 + ], + "mean_e2el_ms": [ + 6000 + ], + "mean_audio_rtf": [ + 0.20 + ] } }, { "dataset_name": "random-mm", "backend": "openai-chat-omni", "endpoint": "/v1/chat/completions", - "num_prompts": [40], - "request_rate": [0.5], + "num_prompts": [ + 40 + ], + "request_rate": [ + 0.5 + ], "random_input_len": 100, "random_output_len": 100, "random_range_ratio": 0.0, @@ -69,17 +115,27 @@ }, "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { - "mean_ttft_ms": [6000], - "mean_audio_ttfp_ms": [15000], - "mean_audio_rtf": [0.45] + "mean_ttft_ms": [ + 300 + ], + "mean_e2el_ms": [ + 7000 + ], + "mean_audio_rtf": [ + 0.20 + ] } }, { "dataset_name": "random-mm", "backend": "openai-chat-omni", "endpoint": "/v1/chat/completions", - "num_prompts": [100], - "request_rate": [1.0], + "num_prompts": [ + 100 + ], + "request_rate": [ + 1.0 + ], "random_input_len": 100, "random_output_len": 100, "random_range_ratio": 0.0, @@ -98,9 +154,15 @@ }, "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { - "mean_ttft_ms": [12000], - "mean_audio_ttfp_ms": [18000], - "mean_audio_rtf": [0.9] + "mean_ttft_ms": [ + 700 + ], + "mean_e2el_ms": [ + 9000 + ], + "mean_audio_rtf": [ + 0.30 + ] } } ] diff --git a/tests/dfx/perf/tests/test_qwen3_omni_vllm_text.json b/tests/dfx/perf/tests/test_qwen3_omni_vllm_text.json index aa2f79033..2f580aac7 100644 --- a/tests/dfx/perf/tests/test_qwen3_omni_vllm_text.json +++ b/tests/dfx/perf/tests/test_qwen3_omni_vllm_text.json @@ -4,7 +4,9 @@ "server_params": { "model": "Qwen/Qwen3-Omni-30B-A3B-Instruct", "use_omni": false, - "extra_cli_args": ["--no-enable-prefix-caching"] + "extra_cli_args": [ + "--no-enable-prefix-caching" + ] }, "benchmark_params": [ { @@ -12,8 +14,20 @@ "backend": "vllm", "model": "Qwen/Qwen3-Omni-30B-A3B-Instruct", "endpoint": "/v1/completions", - "num_prompts": [4, 16, 32, 64, 128], - "max_concurrency": [1, 4, 8, 16, 32], + "num_prompts": [ + 4, + 16, + 32, + 64, + 128 + ], + "max_concurrency": [ + 1, + 4, + 8, + 16, + 32 + ], "random_input_len": 2500, "random_output_len": 900, "temperature": 0.4, @@ -23,11 +37,26 @@ "repetition_penalty": 1.05, "ignore_eos": true, "extra_body": { - "modalities": ["text"] + "modalities": [ + "text" + ] }, "percentile-metrics": "ttft,tpot,itl,e2el", "baseline": { - "mean_ttft_ms": [1000, 3000, 5000, 7000, 9000] + "mean_ttft_ms": [ + 300, + 300, + 1000, + 600, + 800 + ], + "mean_e2el_ms": [ + 6000, + 8000, + 11000, + 13000, + 17000 + ] } }, { @@ -35,8 +64,12 @@ "backend": "vllm", "model": "Qwen/Qwen3-Omni-30B-A3B-Instruct", "endpoint": "/v1/completions", - "num_prompts": [10], - "request_rate": [0.1], + "num_prompts": [ + 10 + ], + "request_rate": [ + 0.1 + ], "random_input_len": 100, "random_output_len": 100, "random_range_ratio": 0.0, @@ -47,7 +80,9 @@ "repetition_penalty": 1.05, "ignore_eos": true, "extra_body": { - "modalities": ["text"] + "modalities": [ + "text" + ] }, "random_mm_base_items_per_request": 1, "random_mm_num_mm_items_range_ratio": 0.5, @@ -59,7 +94,12 @@ }, "percentile-metrics": "ttft,tpot,itl,e2el", "baseline": { - "mean_ttft_ms": [2000] + "mean_ttft_ms": [ + 30 + ], + "mean_e2el_ms": [ + 540 + ] } }, { @@ -67,8 +107,12 @@ "backend": "vllm", "model": "Qwen/Qwen3-Omni-30B-A3B-Instruct", "endpoint": "/v1/completions", - "num_prompts": [40], - "request_rate": [0.5], + "num_prompts": [ + 40 + ], + "request_rate": [ + 0.5 + ], "random_input_len": 100, "random_output_len": 100, "random_range_ratio": 0.0, @@ -79,7 +123,9 @@ "repetition_penalty": 1.05, "ignore_eos": true, "extra_body": { - "modalities": ["text"] + "modalities": [ + "text" + ] }, "random_mm_base_items_per_request": 2, "random_mm_num_mm_items_range_ratio": 0.5, @@ -93,7 +139,12 @@ }, "percentile-metrics": "ttft,tpot,itl,e2el", "baseline": { - "mean_ttft_ms": [6000] + "mean_ttft_ms": [ + 30 + ], + "mean_e2el_ms": [ + 600 + ] } }, { @@ -101,8 +152,12 @@ "backend": "vllm", "model": "Qwen/Qwen3-Omni-30B-A3B-Instruct", "endpoint": "/v1/completions", - "num_prompts": [100], - "request_rate": [1.0], + "num_prompts": [ + 100 + ], + "request_rate": [ + 1.0 + ], "random_input_len": 100, "random_output_len": 100, "random_range_ratio": 0.0, @@ -113,7 +168,9 @@ "repetition_penalty": 1.05, "ignore_eos": true, "extra_body": { - "modalities": ["text"] + "modalities": [ + "text" + ] }, "random_mm_base_items_per_request": 3, "random_mm_num_mm_items_range_ratio": 0.5, @@ -129,7 +186,12 @@ }, "percentile-metrics": "ttft,tpot,itl,e2el", "baseline": { - "mean_ttft_ms": [6000] + "mean_ttft_ms": [ + 40 + ], + "mean_e2el_ms": [ + 700 + ] } } ] diff --git a/tests/dfx/perf/tests/test_qwen_image_edit_2509_vllm_omni.json b/tests/dfx/perf/tests/test_qwen_image_edit_2509_vllm_omni.json index 7d1fbbfa7..d13f37053 100644 --- a/tests/dfx/perf/tests/test_qwen_image_edit_2509_vllm_omni.json +++ b/tests/dfx/perf/tests/test_qwen_image_edit_2509_vllm_omni.json @@ -22,10 +22,10 @@ "num-input-images": 2, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.05, - "latency_mean": 18, - "peak_memory_mb_max": 78500, - "peak_memory_mb_mean": 78500 + "throughput_qps": 0.06, + "latency_mean": 20, + "peak_memory_mb_max": 57000, + "peak_memory_mb_mean": 57000 } }, { @@ -41,9 +41,9 @@ "enable-negative-prompt": true, "baseline": { "throughput_qps": 0.01, - "latency_mean": 70, - "peak_memory_mb_max": 81000, - "peak_memory_mb_mean": 81000 + "latency_mean": 60, + "peak_memory_mb_max": 67000, + "peak_memory_mb_mean": 67000 } } ] @@ -75,10 +75,10 @@ "num-input-images": 2, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.1, - "latency_mean": 12, - "peak_memory_mb_max": 69000, - "peak_memory_mb_mean": 69000 + "throughput_qps": 0.15, + "latency_mean": 10, + "peak_memory_mb_max": 55000, + "peak_memory_mb_mean": 55000 } }, { @@ -93,10 +93,10 @@ "num-input-images": 2, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.03, - "latency_mean": 28, - "peak_memory_mb_max": 69000, - "peak_memory_mb_mean": 69000 + "throughput_qps": 0.05, + "latency_mean": 20, + "peak_memory_mb_max": 55000, + "peak_memory_mb_mean": 55000 } } ] @@ -138,10 +138,10 @@ "num-input-images": 2, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.10, - "latency_mean": 12, - "peak_memory_mb_max": 73000, - "peak_memory_mb_mean": 73000 + "throughput_qps": 0.21, + "latency_mean": 10, + "peak_memory_mb_max": 57000, + "peak_memory_mb_mean": 57000 } }, { @@ -156,10 +156,10 @@ "num-input-images": 2, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.05, + "throughput_qps": 0.08, "latency_mean": 20, - "peak_memory_mb_max": 81000, - "peak_memory_mb_mean": 81000 + "peak_memory_mb_max": 66000, + "peak_memory_mb_mean": 66000 } } ] diff --git a/tests/dfx/perf/tests/test_qwen_image_edit_vllm_omni.json b/tests/dfx/perf/tests/test_qwen_image_edit_vllm_omni.json index f68201db5..8ae370df6 100644 --- a/tests/dfx/perf/tests/test_qwen_image_edit_vllm_omni.json +++ b/tests/dfx/perf/tests/test_qwen_image_edit_vllm_omni.json @@ -21,10 +21,10 @@ "max-concurrency": 1, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.05, - "latency_mean": 15.0, - "peak_memory_mb_max": 72500, - "peak_memory_mb_mean": 72500 + "throughput_qps": 0.1, + "latency_mean": 10, + "peak_memory_mb_max": 57000, + "peak_memory_mb_mean": 57000 } }, { @@ -38,10 +38,10 @@ "max-concurrency": 1, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.01, - "latency_mean": 65.6, - "peak_memory_mb_max": 80777, - "peak_memory_mb_mean": 80777 + "throughput_qps": 0.02, + "latency_mean": 50, + "peak_memory_mb_max": 65000, + "peak_memory_mb_mean": 65000 } } ] @@ -72,10 +72,10 @@ "max-concurrency": 1, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.10, - "latency_mean": 7.2, - "peak_memory_mb_max": 68100, - "peak_memory_mb_mean": 68100 + "throughput_qps": 0.21, + "latency_mean": 10, + "peak_memory_mb_max": 56000, + "peak_memory_mb_mean": 56000 } }, { @@ -89,10 +89,10 @@ "max-concurrency": 1, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.03, - "latency_mean": 24.0, - "peak_memory_mb_max": 68100, - "peak_memory_mb_mean": 68100 + "throughput_qps": 0.06, + "latency_mean": 20, + "peak_memory_mb_max": 56000, + "peak_memory_mb_mean": 56000 } } ] @@ -133,10 +133,10 @@ "max-concurrency": 1, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.1, - "latency_mean": 6.5, - "peak_memory_mb_max": 72600, - "peak_memory_mb_mean": 72600 + "throughput_qps": 0.26, + "latency_mean": 10, + "peak_memory_mb_max": 58000, + "peak_memory_mb_mean": 58000 } }, { @@ -150,10 +150,10 @@ "max-concurrency": 1, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.05, - "latency_mean": 16.0, - "peak_memory_mb_max": 81000, - "peak_memory_mb_mean": 81000 + "throughput_qps": 0.1, + "latency_mean": 10, + "peak_memory_mb_max": 67000, + "peak_memory_mb_mean": 67000 } } ] diff --git a/tests/dfx/perf/tests/test_qwen_image_layered_vllm_omni.json b/tests/dfx/perf/tests/test_qwen_image_layered_vllm_omni.json index 3cf13509c..c0cd3b7c0 100644 --- a/tests/dfx/perf/tests/test_qwen_image_layered_vllm_omni.json +++ b/tests/dfx/perf/tests/test_qwen_image_layered_vllm_omni.json @@ -21,10 +21,10 @@ "max-concurrency": 1, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.02, - "latency_mean": 40.0, - "peak_memory_mb_max": 70000, - "peak_memory_mb_mean": 70000 + "throughput_qps": 0.05, + "latency_mean": 20, + "peak_memory_mb_max": 63000, + "peak_memory_mb_mean": 63000 } }, { @@ -38,10 +38,10 @@ "max-concurrency": 1, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.005, - "latency_mean": 80.0, - "peak_memory_mb_max": 70000, - "peak_memory_mb_mean": 70000 + "throughput_qps": 0.03, + "latency_mean": 30, + "peak_memory_mb_max": 63000, + "peak_memory_mb_mean": 63000 } } ] diff --git a/tests/dfx/perf/tests/test_qwen_image_vllm_omni.json b/tests/dfx/perf/tests/test_qwen_image_vllm_omni.json index cdd0cac2c..e05f1b945 100644 --- a/tests/dfx/perf/tests/test_qwen_image_vllm_omni.json +++ b/tests/dfx/perf/tests/test_qwen_image_vllm_omni.json @@ -21,9 +21,9 @@ "max-concurrency": 1, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.30, - "latency_mean": 3.50, - "peak_memory_mb_mean": 67000 + "throughput_qps": 0.35, + "latency_mean": 3, + "peak_memory_mb_mean": 56000 } }, { @@ -37,9 +37,9 @@ "max-concurrency": 1, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.037, - "latency_mean": 27.0, - "peak_memory_mb_mean": 74000 + "throughput_qps": 0.03, + "latency_mean": 26, + "peak_memory_mb_mean": 65000 } } ] @@ -67,9 +67,9 @@ "max-concurrency": 1, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.30, - "latency_mean": 3.50, - "peak_memory_mb_mean": 67000 + "throughput_qps": 0.35, + "latency_mean": 3, + "peak_memory_mb_mean": 56000 } }, { @@ -83,9 +83,9 @@ "max-concurrency": 1, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.037, - "latency_mean": 27.0, - "peak_memory_mb_mean": 74000 + "throughput_qps": 0.03, + "latency_mean": 26, + "peak_memory_mb_mean": 65000 } } ] @@ -116,9 +116,9 @@ "max-concurrency": 1, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.1, - "latency_mean": 9.1, - "peak_memory_mb_mean": 61000 + "throughput_qps": 0.11, + "latency_mean": 9, + "peak_memory_mb_mean": 55000 } } ] @@ -159,9 +159,9 @@ "max-concurrency": 1, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.1, - "latency_mean": 3.50, - "peak_memory_mb_mean": 67000 + "throughput_qps": 0.46, + "latency_mean": 3, + "peak_memory_mb_mean": 56000 } }, { @@ -175,9 +175,9 @@ "max-concurrency": 1, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.1, - "latency_mean": 6.15, - "peak_memory_mb_mean": 74000 + "throughput_qps": 0.18, + "latency_mean": 6, + "peak_memory_mb_mean": 67000 } } ] diff --git a/tests/dfx/perf/tests/test_tts.json b/tests/dfx/perf/tests/test_tts.json index db197c34b..733c40963 100644 --- a/tests/dfx/perf/tests/test_tts.json +++ b/tests/dfx/perf/tests/test_tts.json @@ -4,7 +4,9 @@ "server_params": { "model": "Qwen/Qwen3-TTS-12Hz-1.7B-Base", "stage_overrides": { - "1": {"devices": "1"} + "1": { + "devices": "1" + } } }, "benchmark_params": [ @@ -15,13 +17,21 @@ "backend": "openai-audio-speech", "endpoint": "/v1/audio/speech", "dataset_path": "linyueqian/seed-tts-eval-subset", - "num_prompts": [20], - "max_concurrency": [1], + "num_prompts": [ + 20 + ], + "max_concurrency": [ + 1 + ], "seed_tts_locale": "en", "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { - "median_audio_ttfp_ms": [350], - "median_audio_rtf": [0.25] + "median_audio_ttfp_ms": [ + 3000 + ], + "median_audio_rtf": [ + 0.6 + ] } }, { @@ -31,14 +41,34 @@ "backend": "openai-audio-speech", "endpoint": "/v1/audio/speech", "dataset_path": "linyueqian/seed-tts-eval-subset", - "num_prompts": [80, 128, 128], - "max_concurrency": [8, 16, 64], + "num_prompts": [ + 80, + 128, + 128 + ], + "max_concurrency": [ + 8, + 16, + 64 + ], "seed_tts_locale": "en", "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { - "median_audio_ttfp_ms": [3500, 30000, 7000], - "median_audio_rtf": [0.75, 10.0, 1.5], - "audio_throughput": [10.0, 1.0, 40.0] + "median_audio_ttfp_ms": [ + 1000, + 3000, + 14000 + ], + "median_audio_rtf": [ + 0.70, + 1.15, + 3.90 + ], + "audio_throughput": [ + 12, + 14, + 14 + ] } }, { @@ -49,13 +79,19 @@ "backend": "openai-audio-speech", "endpoint": "/v1/audio/speech", "dataset_path": "linyueqian/seed-tts-eval-subset", - "num_prompts": [200], - "max_concurrency": [4], + "num_prompts": [ + 200 + ], + "max_concurrency": [ + 4 + ], "seed_tts_locale": "en", "seed_tts_wer_eval": true, "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { - "mean_audio_rtf": [0.45] + "mean_audio_rtf": [ + 0.45 + ] } } ] @@ -65,7 +101,9 @@ "server_params": { "model": "Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice", "stage_overrides": { - "1": {"devices": "1"} + "1": { + "devices": "1" + } } }, "benchmark_params": [ @@ -76,14 +114,26 @@ "backend": "openai-audio-speech", "endpoint": "/v1/audio/speech", "dataset_path": "benchmarks/build_dataset/seed_tts_smoke", - "num_prompts": [20], - "max_concurrency": [1], + "num_prompts": [ + 20 + ], + "max_concurrency": [ + 1 + ], "seed_tts_locale": "en", - "extra_body": {"voice": "Vivian", "language": "English", "task_type": "CustomVoice"}, + "extra_body": { + "voice": "Vivian", + "language": "English", + "task_type": "CustomVoice" + }, "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { - "median_audio_ttfp_ms": [150], - "median_audio_rtf": [0.15] + "median_audio_ttfp_ms": [ + 2000 + ], + "median_audio_rtf": [ + 0.40 + ] } }, { @@ -93,15 +143,39 @@ "backend": "openai-audio-speech", "endpoint": "/v1/audio/speech", "dataset_path": "benchmarks/build_dataset/seed_tts_smoke", - "num_prompts": [80, 128, 128], - "max_concurrency": [8, 16, 64], + "num_prompts": [ + 80, + 128, + 128 + ], + "max_concurrency": [ + 8, + 16, + 64 + ], "seed_tts_locale": "en", - "extra_body": {"voice": "Vivian", "language": "English", "task_type": "CustomVoice"}, + "extra_body": { + "voice": "Vivian", + "language": "English", + "task_type": "CustomVoice" + }, "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { - "median_audio_ttfp_ms": [1500, 30000, 3000], - "median_audio_rtf": [0.30, 10.0, 0.60], - "audio_throughput": [30.0, 1.0, 100.0] + "median_audio_ttfp_ms": [ + 300, + 1000, + 5000 + ], + "median_audio_rtf": [ + 0.25, + 0.35, + 1.10 + ], + "audio_throughput": [ + 30, + 40, + 40 + ] } }, { @@ -112,14 +186,24 @@ "backend": "openai-audio-speech", "endpoint": "/v1/audio/speech", "dataset_path": "benchmarks/build_dataset/seed_tts_smoke", - "num_prompts": [200], - "max_concurrency": [4], + "num_prompts": [ + 200 + ], + "max_concurrency": [ + 4 + ], "seed_tts_locale": "en", - "extra_body": {"voice": "Vivian", "language": "English", "task_type": "CustomVoice"}, + "extra_body": { + "voice": "Vivian", + "language": "English", + "task_type": "CustomVoice" + }, "seed_tts_wer_eval": true, "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { - "mean_audio_rtf": [0.35] + "mean_audio_rtf": [ + 0.35 + ] } }, { @@ -129,15 +213,29 @@ "backend": "openai-audio-speech", "endpoint": "/v1/audio/speech", "dataset_path": "benchmarks/build_dataset/seed_tts_smoke", - "num_prompts": [100], - "request_rate": [2.0], + "num_prompts": [ + 100 + ], + "request_rate": [ + 2.0 + ], "seed_tts_locale": "en", - "extra_body": {"voice": "Vivian", "language": "English", "task_type": "CustomVoice"}, + "extra_body": { + "voice": "Vivian", + "language": "English", + "task_type": "CustomVoice" + }, "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { - "median_audio_ttfp_ms": [3000], - "median_audio_rtf": [0.25], - "audio_throughput": [4.0] + "median_audio_ttfp_ms": [ + 100 + ], + "median_audio_rtf": [ + 0.20 + ], + "audio_throughput": [ + 10 + ] } }, { @@ -147,14 +245,25 @@ "backend": "openai-audio-speech", "endpoint": "/v1/audio/speech", "dataset_path": "benchmarks/build_dataset/seed_tts_design", - "num_prompts": [20], - "max_concurrency": [1], + "num_prompts": [ + 20 + ], + "max_concurrency": [ + 1 + ], "seed_tts_locale": "en", - "extra_body": {"task_type": "VoiceDesign", "language": "English"}, + "extra_body": { + "task_type": "VoiceDesign", + "language": "English" + }, "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { - "median_audio_ttfp_ms": [150], - "median_audio_rtf": [0.15] + "median_audio_ttfp_ms": [ + 100 + ], + "median_audio_rtf": [ + 0.15 + ] } }, { @@ -164,15 +273,38 @@ "backend": "openai-audio-speech", "endpoint": "/v1/audio/speech", "dataset_path": "benchmarks/build_dataset/seed_tts_design", - "num_prompts": [80, 128, 128], - "max_concurrency": [8, 16, 64], + "num_prompts": [ + 80, + 128, + 128 + ], + "max_concurrency": [ + 8, + 16, + 64 + ], "seed_tts_locale": "en", - "extra_body": {"task_type": "VoiceDesign", "language": "English"}, + "extra_body": { + "task_type": "VoiceDesign", + "language": "English" + }, "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { - "median_audio_ttfp_ms": [1500, 30000, 3500], - "median_audio_rtf": [0.35, 10.0, 0.70], - "audio_throughput": [25.0, 1.0, 80.0] + "median_audio_ttfp_ms": [ + 100, + 1000, + 5000 + ], + "median_audio_rtf": [ + 0.25, + 0.40, + 1.20 + ], + "audio_throughput": [ + 30, + 40, + 40 + ] } }, { @@ -182,15 +314,28 @@ "backend": "openai-audio-speech", "endpoint": "/v1/audio/speech", "dataset_path": "benchmarks/build_dataset/seed_tts_design", - "num_prompts": [100], - "request_rate": [2.0], + "num_prompts": [ + 100 + ], + "request_rate": [ + 2.0 + ], "seed_tts_locale": "en", - "extra_body": {"task_type": "VoiceDesign", "language": "English"}, + "extra_body": { + "task_type": "VoiceDesign", + "language": "English" + }, "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { - "median_audio_ttfp_ms": [3500], - "median_audio_rtf": [0.30], - "audio_throughput": [4.0] + "median_audio_ttfp_ms": [ + 100 + ], + "median_audio_rtf": [ + 0.20 + ], + "audio_throughput": [ + 9 + ] } } ] diff --git a/tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json b/tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json index 58a17c980..d70a28efd 100644 --- a/tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json +++ b/tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json @@ -30,9 +30,9 @@ } ], "baseline": { - "throughput_qps": 0.034, - "latency_mean": 26.0, - "peak_memory_mb_mean": 80000 + "throughput_qps": 0.04, + "latency_mean": 30, + "peak_memory_mb_mean": 75000 } } ] @@ -72,9 +72,9 @@ } ], "baseline": { - "throughput_qps": 0.042, - "latency_mean": 21.6, - "peak_memory_mb_mean": 55300 + "throughput_qps": 0.06, + "latency_mean": 20, + "peak_memory_mb_mean": 48000 } }, { @@ -97,9 +97,9 @@ } ], "baseline": { - "throughput_qps": 0.0085, - "latency_mean": 101.6, - "peak_memory_mb_mean": 65200 + "throughput_qps": 0.01, + "latency_mean": 90, + "peak_memory_mb_mean": 56000 } } ]