diff --git a/tests/dfx/conftest.py b/tests/dfx/conftest.py index 9332a24f5..a22f760b2 100644 --- a/tests/dfx/conftest.py +++ b/tests/dfx/conftest.py @@ -279,6 +279,10 @@ def create_benchmark_indices( return indices +# Omni perf/stability: ``vllm bench serve --omni`` result template (used if bench exits without writing JSON). +OMNI_RESULT_TEMPLATE_PATH = Path(__file__).resolve().parent / "perf" / "scripts" / "result_omni_template.json" + + def _safe_filename_token(value: Any | None, *, default: str = "na") -> str: """Make a single path segment safe for result filenames on common filesystems.""" if value is None: @@ -289,7 +293,7 @@ def _safe_filename_token(value: Any | None, *, default: str = "na") -> str: return s if s else default -def _resolve_baseline_value( +def resolve_baseline_value( baseline_raw: Any, *, sweep_index: int | None, @@ -330,7 +334,7 @@ def _baseline_thresholds_for_step( ) -> dict[str, Any]: """Resolve baseline config to one threshold per metric for this iteration.""" return { - metric_name: _resolve_baseline_value( + metric_name: resolve_baseline_value( baseline_raw, sweep_index=sweep_index, max_concurrency=max_concurrency, @@ -354,7 +358,12 @@ def run_benchmark( random_input_len: Any | None = None, random_output_len: Any | None = None, ) -> dict[str, Any]: - """Run one ``vllm bench serve --omni`` iteration and return parsed metrics.""" + """Run one ``vllm bench serve --omni`` iteration and return parsed metrics. + + After ``vllm bench`` writes the JSON, ``result["baseline"]`` holds the resolved per-metric thresholds + (when ``baseline_config`` is provided). If the benchmark exits without writing a result file, + ``result_omni_template.json`` is used as a fallback. + """ current_dt = datetime.now().strftime("%Y%m%d-%H%M%S") ri = _safe_filename_token(random_input_len) ro = _safe_filename_token(random_output_len) @@ -403,8 +412,17 @@ def run_benchmark( result_dir = "./" result_path = os.path.join(result_dir, result_filename) - with open(result_path, encoding="utf-8") as f: - result = json.load(f) + if not os.path.exists(result_path): + with open(OMNI_RESULT_TEMPLATE_PATH, encoding="utf-8") as f: + template_result: dict[str, Any] = json.load(f) + Path(result_path).parent.mkdir(parents=True, exist_ok=True) + with open(result_path, "w", encoding="utf-8") as f: + json.dump(template_result, f, ensure_ascii=False, indent=2) + print(f"Benchmark result file not generated, fallback to template: {result_path}") + result = template_result + else: + with open(result_path, encoding="utf-8") as f: + result = json.load(f) if baseline_config: result["baseline"] = _baseline_thresholds_for_step( @@ -432,3 +450,12 @@ def pytest_addoption(parser: pytest.Parser) -> None: default=None, help=("Path to benchmark config JSON. Example: --test-config-file tests/dfx/perf/tests/test_tts.json"), ) + parser.addoption( + "--assert-baseline", + action="store_true", + default=False, + help=( + "When set, omni/diffusion perf runners compare metrics against the baseline block in the JSON config " + "(default: off)." + ), + ) diff --git a/tests/dfx/perf/scripts/run_benchmark.py b/tests/dfx/perf/scripts/run_benchmark.py index 74332889c..55ad2f8b2 100644 --- a/tests/dfx/perf/scripts/run_benchmark.py +++ b/tests/dfx/perf/scripts/run_benchmark.py @@ -12,12 +12,17 @@ from tests.dfx.conftest import ( create_unique_server_params, get_benchmark_params_for_server, load_configs, + resolve_baseline_value, run_benchmark, ) from tests.helpers.runtime import OmniServer pytestmark = [pytest.mark.full_model] +# Compare metrics to each test JSON ``baseline`` block only when pytest is run with ``--assert-baseline`` +# (registered in ``tests/dfx/conftest.py``; default: off). ``run_benchmark`` and ``_resolve_baseline_value`` are +# defined in the same module. + os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn" @@ -46,7 +51,6 @@ if CONFIG_FILE_PATH is None: CONFIG_FILE_PATH = _DEFAULT_CONFIG_FILE BENCHMARK_CONFIGS = load_configs(CONFIG_FILE_PATH) -OMNI_RESULT_TEMPLATE_PATH = Path(__file__).parent / "result_omni_template.json" DEPLOY_CONFIGS_DIR = Path(__file__).parent.parent / "deploy" @@ -87,16 +91,6 @@ def omni_server(request): print("OmniServer stopped") -def _safe_filename_token(value: Any | None, *, default: str = "na") -> str: - """Make a single path segment safe for result filenames on common filesystems.""" - if value is None: - return default - s = str(value).strip() - for bad in ("/", "\\", ":", "*", "?", '"', "<", ">", "|"): - s = s.replace(bad, "_") - return s if s else default - - benchmark_indices = create_benchmark_indices(BENCHMARK_CONFIGS, server_to_benchmark_mapping) @@ -126,76 +120,23 @@ def benchmark_params(request, omni_server): } -def _resolve_baseline_value( - baseline_raw: Any, - *, - sweep_index: int | None, - max_concurrency: Any = None, - request_rate: Any = None, -) -> Any: - """Pick the baseline threshold for this sweep step. - - Supported shapes per metric: - - **Scalar** — same threshold for every concurrency / QPS. - - **List** — aligned with ``max_concurrency`` / ``request_rate`` sweep order; use ``sweep_index``. - - **Dict** — keyed by concurrency or rate, e.g. ``{"1": 500, "4": 800}`` (keys are strings in JSON). - - For dict lookup, ``max_concurrency`` is preferred when both are set (concurrency sweep). - """ - if baseline_raw is None: - # If no baseline is set, the maximum value will be used. - return 100000 - if isinstance(baseline_raw, dict): - if max_concurrency is not None: - for key in (max_concurrency, str(max_concurrency)): - if key in baseline_raw: - return baseline_raw[key] - if request_rate is not None: - for key in (request_rate, str(request_rate)): - if key in baseline_raw: - return baseline_raw[key] - raise KeyError( - f"baseline dict has no key for max_concurrency={max_concurrency!r} " - f"or request_rate={request_rate!r}; keys={list(baseline_raw.keys())!r}" - ) - if isinstance(baseline_raw, (list, tuple)): - return baseline_raw[sweep_index] - return baseline_raw - - -def _baseline_thresholds_for_step( - baseline_data: dict[str, Any], - *, - sweep_index: int | None = None, - max_concurrency: Any = None, - request_rate: Any = None, -) -> dict[str, Any]: - """Resolve ``test.json`` ``baseline`` block to one threshold per metric (same as ``assert_result``).""" - return { - metric_name: _resolve_baseline_value( - baseline_raw, - sweep_index=sweep_index, - max_concurrency=max_concurrency, - request_rate=request_rate, - ) - for metric_name, baseline_raw in baseline_data.items() - } - - def assert_result( result, params, num_prompt, *, + assert_baseline: bool, sweep_index: int | None = None, - max_concurrency: Any = None, - request_rate: Any = None, + max_concurrency: Any | None = None, + request_rate: Any | None = None, ) -> None: assert result["completed"] == num_prompt, "Request failures exist" + if not assert_baseline: + return baseline_data = params.get("baseline", {}) for metric_name, baseline_raw in baseline_data.items(): current_value = result[metric_name] - baseline_value = _resolve_baseline_value( + baseline_value = resolve_baseline_value( baseline_raw, sweep_index=sweep_index, max_concurrency=max_concurrency, @@ -214,7 +155,7 @@ def assert_result( @pytest.mark.benchmark @pytest.mark.parametrize("omni_server", test_params, indirect=True) @pytest.mark.parametrize("benchmark_params", benchmark_indices, indirect=True) -def test_performance_benchmark(omni_server, benchmark_params): +def test_performance_benchmark(omni_server, benchmark_params, request): test_name = benchmark_params["test_name"] params = benchmark_params["params"] dataset_name = params.get("dataset_name", "") @@ -226,6 +167,8 @@ def test_performance_benchmark(omni_server, benchmark_params): print(f"Running benchmark for model: {model}") print(f"Benchmark parameters: {benchmark_params}") + assert_baseline = request.config.getoption("--assert-baseline", default=False) + def to_list(value, default=None): if value is None: return [] if default is None else [default] @@ -264,7 +207,7 @@ def test_performance_benchmark(omni_server, benchmark_params): elif not isinstance(value, bool): args.extend([arg_name, str(value)]) - # QPS test (sweep_index aligns with qps_list / num_prompt_list for this loop) + # QPS / request-rate sweep for i, (qps, num_prompt) in enumerate(zip(qps_list, num_prompt_list)): args = args + ["--request-rate", str(qps), "--num-prompts", str(num_prompt)] result = run_benchmark( @@ -283,12 +226,13 @@ def test_performance_benchmark(omni_server, benchmark_params): assert_result( result, params, - num_prompt=num_prompt, + num_prompt, + assert_baseline=assert_baseline, sweep_index=i, request_rate=qps, ) - # concurrency test (sweep_index aligns with max_concurrency_list for separate thresholds per concurrency) + # concurrency test for i, (concurrency, num_prompt) in enumerate(zip(max_concurrency_list, num_prompt_list)): args = args + ["--max-concurrency", str(concurrency), "--num-prompts", str(num_prompt), "--request-rate", "inf"] result = run_benchmark( @@ -307,7 +251,8 @@ def test_performance_benchmark(omni_server, benchmark_params): assert_result( result, params, - num_prompt=num_prompt, + num_prompt, + assert_baseline=assert_baseline, sweep_index=i, max_concurrency=concurrency, ) diff --git a/tests/dfx/perf/scripts/run_diffusion_benchmark.py b/tests/dfx/perf/scripts/run_diffusion_benchmark.py index 9059904e4..22e844427 100644 --- a/tests/dfx/perf/scripts/run_diffusion_benchmark.py +++ b/tests/dfx/perf/scripts/run_diffusion_benchmark.py @@ -11,6 +11,8 @@ This runner separates two concepts: A config JSON file is REQUIRED via --test-config-file: pytest run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_vllm_omni.json +Optional: ``--assert-baseline`` compares metrics to the ``baseline`` block in each benchmark entry (default: off). + All benchmark results for a session are consolidated into a single JSON file under BENCHMARK_RESULT_DIR (override via the DIFFUSION_BENCHMARK_DIR environment variable). Each entry in the file contains the test metadata (test_name, endpoint, benchmark_params, @@ -652,12 +654,14 @@ def run_benchmark( # --------------------------------------------------------------------------- -def assert_result(result: dict[str, Any], params: dict[str, Any]) -> None: - """Assert that benchmark metrics satisfy the configured baselines.""" +def assert_result(result: dict[str, Any], params: dict[str, Any], *, assert_baseline: bool) -> None: + """Assert request completion; optionally assert metrics against ``baseline`` in *params*.""" num_prompts = params.get("num-prompts", 10) completed = result.get("completed_requests", result.get("completed", 0)) assert completed == num_prompts, f"Expected {num_prompts} completed requests, got {completed}" + if not assert_baseline: + return if params.get("skip-performance-assertion", False): print("Skipping performance assertions.") return @@ -699,16 +703,14 @@ def _resolve_benchmark_endpoint(server_cfg: dict[str, Any], params: dict[str, An indirect=True, ) @pytest.mark.parametrize("benchmark_params", benchmark_indices, indirect=True) -def test_diffusion_performance_benchmark(diffusion_server, benchmark_params): - """Run the diffusion performance benchmark and assert against baselines. +def test_diffusion_performance_benchmark(diffusion_server, benchmark_params, request): + """Run the diffusion performance benchmark and verify request completion. One server is started per unique parallel configuration (module scope). For each server, all benchmark parameter sets defined in the config JSON - are executed sequentially; results are asserted against the baselines. + are executed sequentially; metrics are recorded to the aggregated result file. - Tracked metrics: - - throughput_qps (higher is better) - - latency_p50, latency_p99 (lower is better) + Pass ``--assert-baseline`` to compare ``throughput_qps``, ``latency_mean``, etc. to the JSON ``baseline`` block. """ test_name = benchmark_params["test_name"] params = benchmark_params["params"] @@ -744,4 +746,5 @@ def test_diffusion_performance_benchmark(diffusion_server, benchmark_params): print(f"\n Aggregated results: {AGGREGATED_RESULT_FILE}") print("=" * 60) - assert_result(result, params) + assert_baseline = request.config.getoption("--assert-baseline", default=False) + assert_result(result, params, assert_baseline=assert_baseline) diff --git a/tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json b/tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json index 98e311748..bc954cee5 100644 --- a/tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json +++ b/tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json @@ -3,31 +3,67 @@ "test_name": "test_qwen3_omni_chunk", "server_params": { "model": "Qwen/Qwen3-Omni-30B-A3B-Instruct", - "extra_cli_args": ["--async-chunk"] + "extra_cli_args": [ + "--async-chunk" + ] }, "benchmark_params": [ { "dataset_name": "random", "backend": "openai-chat-omni", "endpoint": "/v1/chat/completions", - "num_prompts": [4, 16, 32, 64, 128], - "max_concurrency": [1, 4, 8, 16, 32], + "num_prompts": [ + 4, + 16, + 32, + 64, + 128 + ], + "max_concurrency": [ + 1, + 4, + 8, + 16, + 32 + ], "random_input_len": 2500, "random_output_len": 900, "ignore_eos": true, "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { - "mean_ttft_ms": [1000, 3000, 5000, 7000, 9000], - "mean_audio_ttfp_ms": [1000, 3000, 5000, 7000, 9000], - "mean_audio_rtf": [0.2, 0.35, 0.6, 0.85, 0.9] + "mean_ttft_ms": [ + 400, + 300, + 400, + 600, + 900 + ], + "mean_e2el_ms": [ + 26000, + 43000, + 52000, + 72000, + 109000 + ], + "mean_audio_rtf": [ + 0.15, + 0.30, + 0.35, + 0.50, + 0.75 + ] } }, { "dataset_name": "random-mm", "backend": "openai-chat-omni", "endpoint": "/v1/chat/completions", - "num_prompts": [10], - "request_rate": [0.1], + "num_prompts": [ + 10 + ], + "request_rate": [ + 0.1 + ], "random_input_len": 100, "random_output_len": 100, "random_range_ratio": 0.0, @@ -42,17 +78,27 @@ }, "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { - "mean_ttft_ms": [2000], - "mean_audio_ttfp_ms": [2000], - "mean_audio_rtf": [0.25] + "mean_ttft_ms": [ + 600 + ], + "mean_e2el_ms": [ + 6000 + ], + "mean_audio_rtf": [ + 0.20 + ] } }, { "dataset_name": "random-mm", "backend": "openai-chat-omni", "endpoint": "/v1/chat/completions", - "num_prompts": [40], - "request_rate": [0.5], + "num_prompts": [ + 40 + ], + "request_rate": [ + 0.5 + ], "random_input_len": 100, "random_output_len": 100, "random_range_ratio": 0.0, @@ -69,17 +115,27 @@ }, "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { - "mean_ttft_ms": [6000], - "mean_audio_ttfp_ms": [6000], - "mean_audio_rtf": [0.7] + "mean_ttft_ms": [ + 300 + ], + "mean_e2el_ms": [ + 7000 + ], + "mean_audio_rtf": [ + 0.25 + ] } }, { "dataset_name": "random-mm", "backend": "openai-chat-omni", "endpoint": "/v1/chat/completions", - "num_prompts": [100], - "request_rate": [1.0], + "num_prompts": [ + 100 + ], + "request_rate": [ + 1.0 + ], "random_input_len": 100, "random_output_len": 100, "random_range_ratio": 0.0, @@ -98,40 +154,79 @@ }, "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { - "mean_ttft_ms": [12000], - "mean_audio_ttfp_ms": [12000], - "mean_audio_rtf": [1.0] + "mean_ttft_ms": [ + 700 + ], + "mean_e2el_ms": [ + 11000 + ], + "mean_audio_rtf": [ + 0.35 + ] } }, { "dataset_name": "random", "backend": "openai-chat-omni", "endpoint": "/v1/chat/completions", - "num_prompts": [4, 16, 32, 64, 128], - "max_concurrency": [1, 4, 8, 16, 32], + "num_prompts": [ + 4, + 16, + 32, + 64, + 128 + ], + "max_concurrency": [ + 1, + 4, + 8, + 16, + 32 + ], "random_input_len": 2500, "random_output_len": 900, "ignore_eos": true, "extra_body": { - "modalities": ["text"] + "modalities": [ + "text" + ] }, "percentile-metrics": "ttft,tpot,itl,e2el", "baseline": { - "mean_ttft_ms": [1000, 3000, 5000, 7000, 9000] + "mean_ttft_ms": [ + 100, + 300, + 500, + 800, + 1500 + ], + "mean_e2el_ms": [ + 5000, + 8000, + 10000, + 14000, + 20000 + ] } }, { "dataset_name": "random-mm", "backend": "openai-chat-omni", "endpoint": "/v1/chat/completions", - "num_prompts": [10], - "request_rate": [0.1], + "num_prompts": [ + 10 + ], + "request_rate": [ + 0.1 + ], "random_input_len": 100, "random_output_len": 100, "random_range_ratio": 0.0, "ignore_eos": true, "extra_body": { - "modalities": ["text"] + "modalities": [ + "text" + ] }, "random_mm_base_items_per_request": 1, "random_mm_num_mm_items_range_ratio": 0.5, @@ -143,21 +238,32 @@ }, "percentile-metrics": "ttft,tpot,itl,e2el", "baseline": { - "mean_ttft_ms": [2000] + "mean_ttft_ms": [ + 500 + ], + "mean_e2el_ms": [ + 1000 + ] } }, { "dataset_name": "random-mm", "backend": "openai-chat-omni", "endpoint": "/v1/chat/completions", - "num_prompts": [40], - "request_rate": [0.5], + "num_prompts": [ + 40 + ], + "request_rate": [ + 0.5 + ], "random_input_len": 100, "random_output_len": 100, "random_range_ratio": 0.0, "ignore_eos": true, "extra_body": { - "modalities": ["text"] + "modalities": [ + "text" + ] }, "random_mm_base_items_per_request": 2, "random_mm_num_mm_items_range_ratio": 0.5, @@ -171,21 +277,32 @@ }, "percentile-metrics": "ttft,tpot,itl,e2el", "baseline": { - "mean_ttft_ms": [6000] + "mean_ttft_ms": [ + 200 + ], + "mean_e2el_ms": [ + 800 + ] } }, { "dataset_name": "random-mm", "backend": "openai-chat-omni", "endpoint": "/v1/chat/completions", - "num_prompts": [100], - "request_rate": [1.0], + "num_prompts": [ + 100 + ], + "request_rate": [ + 1.0 + ], "random_input_len": 100, "random_output_len": 100, "random_range_ratio": 0.0, "ignore_eos": true, "extra_body": { - "modalities": ["text"] + "modalities": [ + "text" + ] }, "random_mm_base_items_per_request": 3, "random_mm_num_mm_items_range_ratio": 0.5, @@ -201,7 +318,12 @@ }, "percentile-metrics": "ttft,tpot,itl,e2el", "baseline": { - "mean_ttft_ms": [6000] + "mean_ttft_ms": [ + 600 + ], + "mean_e2el_ms": [ + 1300 + ] } } ] diff --git a/tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json b/tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json index b91a8a89b..e57acecba 100644 --- a/tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json +++ b/tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json @@ -3,31 +3,67 @@ "test_name": "test_qwen3_omni", "server_params": { "model": "Qwen/Qwen3-Omni-30B-A3B-Instruct", - "extra_cli_args": ["--no-async-chunk"] + "extra_cli_args": [ + "--no-async-chunk" + ] }, "benchmark_params": [ { "dataset_name": "random", "backend": "openai-chat-omni", "endpoint": "/v1/chat/completions", - "num_prompts": [4, 16, 32, 64, 128], - "max_concurrency": [1, 4, 8, 16, 32], + "num_prompts": [ + 4, + 16, + 32, + 64, + 128 + ], + "max_concurrency": [ + 1, + 4, + 8, + 16, + 32 + ], "random_input_len": 2500, "random_output_len": 900, "ignore_eos": true, "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { - "mean_ttft_ms": [1000, 3000, 5000, 7000, 9000], - "mean_audio_ttfp_ms": [30000, 60000, 90000, 120000, 150000], - "mean_audio_rtf": [0.35, 0.45, 0.55, 0.65, 0.75] + "mean_ttft_ms": [ + 400, + 300, + 400, + 600, + 900 + ], + "mean_e2el_ms": [ + 32000, + 34000, + 45000, + 61000, + 90000 + ], + "mean_audio_rtf": [ + 0.20, + 0.25, + 0.30, + 0.45, + 0.60 + ] } }, { "dataset_name": "random-mm", "backend": "openai-chat-omni", "endpoint": "/v1/chat/completions", - "num_prompts": [10], - "request_rate": [0.1], + "num_prompts": [ + 10 + ], + "request_rate": [ + 0.1 + ], "random_input_len": 100, "random_output_len": 100, "random_range_ratio": 0.0, @@ -42,17 +78,27 @@ }, "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { - "mean_ttft_ms": [2000], - "mean_audio_ttfp_ms": [10000], - "mean_audio_rtf": [0.25] + "mean_ttft_ms": [ + 500 + ], + "mean_e2el_ms": [ + 6000 + ], + "mean_audio_rtf": [ + 0.20 + ] } }, { "dataset_name": "random-mm", "backend": "openai-chat-omni", "endpoint": "/v1/chat/completions", - "num_prompts": [40], - "request_rate": [0.5], + "num_prompts": [ + 40 + ], + "request_rate": [ + 0.5 + ], "random_input_len": 100, "random_output_len": 100, "random_range_ratio": 0.0, @@ -69,17 +115,27 @@ }, "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { - "mean_ttft_ms": [6000], - "mean_audio_ttfp_ms": [15000], - "mean_audio_rtf": [0.45] + "mean_ttft_ms": [ + 300 + ], + "mean_e2el_ms": [ + 7000 + ], + "mean_audio_rtf": [ + 0.20 + ] } }, { "dataset_name": "random-mm", "backend": "openai-chat-omni", "endpoint": "/v1/chat/completions", - "num_prompts": [100], - "request_rate": [1.0], + "num_prompts": [ + 100 + ], + "request_rate": [ + 1.0 + ], "random_input_len": 100, "random_output_len": 100, "random_range_ratio": 0.0, @@ -98,9 +154,15 @@ }, "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { - "mean_ttft_ms": [12000], - "mean_audio_ttfp_ms": [18000], - "mean_audio_rtf": [0.9] + "mean_ttft_ms": [ + 700 + ], + "mean_e2el_ms": [ + 9000 + ], + "mean_audio_rtf": [ + 0.30 + ] } } ] diff --git a/tests/dfx/perf/tests/test_qwen3_omni_vllm_text.json b/tests/dfx/perf/tests/test_qwen3_omni_vllm_text.json index aa2f79033..2f580aac7 100644 --- a/tests/dfx/perf/tests/test_qwen3_omni_vllm_text.json +++ b/tests/dfx/perf/tests/test_qwen3_omni_vllm_text.json @@ -4,7 +4,9 @@ "server_params": { "model": "Qwen/Qwen3-Omni-30B-A3B-Instruct", "use_omni": false, - "extra_cli_args": ["--no-enable-prefix-caching"] + "extra_cli_args": [ + "--no-enable-prefix-caching" + ] }, "benchmark_params": [ { @@ -12,8 +14,20 @@ "backend": "vllm", "model": "Qwen/Qwen3-Omni-30B-A3B-Instruct", "endpoint": "/v1/completions", - "num_prompts": [4, 16, 32, 64, 128], - "max_concurrency": [1, 4, 8, 16, 32], + "num_prompts": [ + 4, + 16, + 32, + 64, + 128 + ], + "max_concurrency": [ + 1, + 4, + 8, + 16, + 32 + ], "random_input_len": 2500, "random_output_len": 900, "temperature": 0.4, @@ -23,11 +37,26 @@ "repetition_penalty": 1.05, "ignore_eos": true, "extra_body": { - "modalities": ["text"] + "modalities": [ + "text" + ] }, "percentile-metrics": "ttft,tpot,itl,e2el", "baseline": { - "mean_ttft_ms": [1000, 3000, 5000, 7000, 9000] + "mean_ttft_ms": [ + 300, + 300, + 1000, + 600, + 800 + ], + "mean_e2el_ms": [ + 6000, + 8000, + 11000, + 13000, + 17000 + ] } }, { @@ -35,8 +64,12 @@ "backend": "vllm", "model": "Qwen/Qwen3-Omni-30B-A3B-Instruct", "endpoint": "/v1/completions", - "num_prompts": [10], - "request_rate": [0.1], + "num_prompts": [ + 10 + ], + "request_rate": [ + 0.1 + ], "random_input_len": 100, "random_output_len": 100, "random_range_ratio": 0.0, @@ -47,7 +80,9 @@ "repetition_penalty": 1.05, "ignore_eos": true, "extra_body": { - "modalities": ["text"] + "modalities": [ + "text" + ] }, "random_mm_base_items_per_request": 1, "random_mm_num_mm_items_range_ratio": 0.5, @@ -59,7 +94,12 @@ }, "percentile-metrics": "ttft,tpot,itl,e2el", "baseline": { - "mean_ttft_ms": [2000] + "mean_ttft_ms": [ + 30 + ], + "mean_e2el_ms": [ + 540 + ] } }, { @@ -67,8 +107,12 @@ "backend": "vllm", "model": "Qwen/Qwen3-Omni-30B-A3B-Instruct", "endpoint": "/v1/completions", - "num_prompts": [40], - "request_rate": [0.5], + "num_prompts": [ + 40 + ], + "request_rate": [ + 0.5 + ], "random_input_len": 100, "random_output_len": 100, "random_range_ratio": 0.0, @@ -79,7 +123,9 @@ "repetition_penalty": 1.05, "ignore_eos": true, "extra_body": { - "modalities": ["text"] + "modalities": [ + "text" + ] }, "random_mm_base_items_per_request": 2, "random_mm_num_mm_items_range_ratio": 0.5, @@ -93,7 +139,12 @@ }, "percentile-metrics": "ttft,tpot,itl,e2el", "baseline": { - "mean_ttft_ms": [6000] + "mean_ttft_ms": [ + 30 + ], + "mean_e2el_ms": [ + 600 + ] } }, { @@ -101,8 +152,12 @@ "backend": "vllm", "model": "Qwen/Qwen3-Omni-30B-A3B-Instruct", "endpoint": "/v1/completions", - "num_prompts": [100], - "request_rate": [1.0], + "num_prompts": [ + 100 + ], + "request_rate": [ + 1.0 + ], "random_input_len": 100, "random_output_len": 100, "random_range_ratio": 0.0, @@ -113,7 +168,9 @@ "repetition_penalty": 1.05, "ignore_eos": true, "extra_body": { - "modalities": ["text"] + "modalities": [ + "text" + ] }, "random_mm_base_items_per_request": 3, "random_mm_num_mm_items_range_ratio": 0.5, @@ -129,7 +186,12 @@ }, "percentile-metrics": "ttft,tpot,itl,e2el", "baseline": { - "mean_ttft_ms": [6000] + "mean_ttft_ms": [ + 40 + ], + "mean_e2el_ms": [ + 700 + ] } } ] diff --git a/tests/dfx/perf/tests/test_qwen_image_edit_2509_vllm_omni.json b/tests/dfx/perf/tests/test_qwen_image_edit_2509_vllm_omni.json index 7d1fbbfa7..d13f37053 100644 --- a/tests/dfx/perf/tests/test_qwen_image_edit_2509_vllm_omni.json +++ b/tests/dfx/perf/tests/test_qwen_image_edit_2509_vllm_omni.json @@ -22,10 +22,10 @@ "num-input-images": 2, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.05, - "latency_mean": 18, - "peak_memory_mb_max": 78500, - "peak_memory_mb_mean": 78500 + "throughput_qps": 0.06, + "latency_mean": 20, + "peak_memory_mb_max": 57000, + "peak_memory_mb_mean": 57000 } }, { @@ -41,9 +41,9 @@ "enable-negative-prompt": true, "baseline": { "throughput_qps": 0.01, - "latency_mean": 70, - "peak_memory_mb_max": 81000, - "peak_memory_mb_mean": 81000 + "latency_mean": 60, + "peak_memory_mb_max": 67000, + "peak_memory_mb_mean": 67000 } } ] @@ -75,10 +75,10 @@ "num-input-images": 2, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.1, - "latency_mean": 12, - "peak_memory_mb_max": 69000, - "peak_memory_mb_mean": 69000 + "throughput_qps": 0.15, + "latency_mean": 10, + "peak_memory_mb_max": 55000, + "peak_memory_mb_mean": 55000 } }, { @@ -93,10 +93,10 @@ "num-input-images": 2, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.03, - "latency_mean": 28, - "peak_memory_mb_max": 69000, - "peak_memory_mb_mean": 69000 + "throughput_qps": 0.05, + "latency_mean": 20, + "peak_memory_mb_max": 55000, + "peak_memory_mb_mean": 55000 } } ] @@ -138,10 +138,10 @@ "num-input-images": 2, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.10, - "latency_mean": 12, - "peak_memory_mb_max": 73000, - "peak_memory_mb_mean": 73000 + "throughput_qps": 0.21, + "latency_mean": 10, + "peak_memory_mb_max": 57000, + "peak_memory_mb_mean": 57000 } }, { @@ -156,10 +156,10 @@ "num-input-images": 2, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.05, + "throughput_qps": 0.08, "latency_mean": 20, - "peak_memory_mb_max": 81000, - "peak_memory_mb_mean": 81000 + "peak_memory_mb_max": 66000, + "peak_memory_mb_mean": 66000 } } ] diff --git a/tests/dfx/perf/tests/test_qwen_image_edit_vllm_omni.json b/tests/dfx/perf/tests/test_qwen_image_edit_vllm_omni.json index f68201db5..8ae370df6 100644 --- a/tests/dfx/perf/tests/test_qwen_image_edit_vllm_omni.json +++ b/tests/dfx/perf/tests/test_qwen_image_edit_vllm_omni.json @@ -21,10 +21,10 @@ "max-concurrency": 1, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.05, - "latency_mean": 15.0, - "peak_memory_mb_max": 72500, - "peak_memory_mb_mean": 72500 + "throughput_qps": 0.1, + "latency_mean": 10, + "peak_memory_mb_max": 57000, + "peak_memory_mb_mean": 57000 } }, { @@ -38,10 +38,10 @@ "max-concurrency": 1, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.01, - "latency_mean": 65.6, - "peak_memory_mb_max": 80777, - "peak_memory_mb_mean": 80777 + "throughput_qps": 0.02, + "latency_mean": 50, + "peak_memory_mb_max": 65000, + "peak_memory_mb_mean": 65000 } } ] @@ -72,10 +72,10 @@ "max-concurrency": 1, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.10, - "latency_mean": 7.2, - "peak_memory_mb_max": 68100, - "peak_memory_mb_mean": 68100 + "throughput_qps": 0.21, + "latency_mean": 10, + "peak_memory_mb_max": 56000, + "peak_memory_mb_mean": 56000 } }, { @@ -89,10 +89,10 @@ "max-concurrency": 1, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.03, - "latency_mean": 24.0, - "peak_memory_mb_max": 68100, - "peak_memory_mb_mean": 68100 + "throughput_qps": 0.06, + "latency_mean": 20, + "peak_memory_mb_max": 56000, + "peak_memory_mb_mean": 56000 } } ] @@ -133,10 +133,10 @@ "max-concurrency": 1, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.1, - "latency_mean": 6.5, - "peak_memory_mb_max": 72600, - "peak_memory_mb_mean": 72600 + "throughput_qps": 0.26, + "latency_mean": 10, + "peak_memory_mb_max": 58000, + "peak_memory_mb_mean": 58000 } }, { @@ -150,10 +150,10 @@ "max-concurrency": 1, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.05, - "latency_mean": 16.0, - "peak_memory_mb_max": 81000, - "peak_memory_mb_mean": 81000 + "throughput_qps": 0.1, + "latency_mean": 10, + "peak_memory_mb_max": 67000, + "peak_memory_mb_mean": 67000 } } ] diff --git a/tests/dfx/perf/tests/test_qwen_image_layered_vllm_omni.json b/tests/dfx/perf/tests/test_qwen_image_layered_vllm_omni.json index 3cf13509c..c0cd3b7c0 100644 --- a/tests/dfx/perf/tests/test_qwen_image_layered_vllm_omni.json +++ b/tests/dfx/perf/tests/test_qwen_image_layered_vllm_omni.json @@ -21,10 +21,10 @@ "max-concurrency": 1, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.02, - "latency_mean": 40.0, - "peak_memory_mb_max": 70000, - "peak_memory_mb_mean": 70000 + "throughput_qps": 0.05, + "latency_mean": 20, + "peak_memory_mb_max": 63000, + "peak_memory_mb_mean": 63000 } }, { @@ -38,10 +38,10 @@ "max-concurrency": 1, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.005, - "latency_mean": 80.0, - "peak_memory_mb_max": 70000, - "peak_memory_mb_mean": 70000 + "throughput_qps": 0.03, + "latency_mean": 30, + "peak_memory_mb_max": 63000, + "peak_memory_mb_mean": 63000 } } ] diff --git a/tests/dfx/perf/tests/test_qwen_image_vllm_omni.json b/tests/dfx/perf/tests/test_qwen_image_vllm_omni.json index cdd0cac2c..e05f1b945 100644 --- a/tests/dfx/perf/tests/test_qwen_image_vllm_omni.json +++ b/tests/dfx/perf/tests/test_qwen_image_vllm_omni.json @@ -21,9 +21,9 @@ "max-concurrency": 1, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.30, - "latency_mean": 3.50, - "peak_memory_mb_mean": 67000 + "throughput_qps": 0.35, + "latency_mean": 3, + "peak_memory_mb_mean": 56000 } }, { @@ -37,9 +37,9 @@ "max-concurrency": 1, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.037, - "latency_mean": 27.0, - "peak_memory_mb_mean": 74000 + "throughput_qps": 0.03, + "latency_mean": 26, + "peak_memory_mb_mean": 65000 } } ] @@ -67,9 +67,9 @@ "max-concurrency": 1, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.30, - "latency_mean": 3.50, - "peak_memory_mb_mean": 67000 + "throughput_qps": 0.35, + "latency_mean": 3, + "peak_memory_mb_mean": 56000 } }, { @@ -83,9 +83,9 @@ "max-concurrency": 1, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.037, - "latency_mean": 27.0, - "peak_memory_mb_mean": 74000 + "throughput_qps": 0.03, + "latency_mean": 26, + "peak_memory_mb_mean": 65000 } } ] @@ -116,9 +116,9 @@ "max-concurrency": 1, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.1, - "latency_mean": 9.1, - "peak_memory_mb_mean": 61000 + "throughput_qps": 0.11, + "latency_mean": 9, + "peak_memory_mb_mean": 55000 } } ] @@ -159,9 +159,9 @@ "max-concurrency": 1, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.1, - "latency_mean": 3.50, - "peak_memory_mb_mean": 67000 + "throughput_qps": 0.46, + "latency_mean": 3, + "peak_memory_mb_mean": 56000 } }, { @@ -175,9 +175,9 @@ "max-concurrency": 1, "enable-negative-prompt": true, "baseline": { - "throughput_qps": 0.1, - "latency_mean": 6.15, - "peak_memory_mb_mean": 74000 + "throughput_qps": 0.18, + "latency_mean": 6, + "peak_memory_mb_mean": 67000 } } ] diff --git a/tests/dfx/perf/tests/test_tts.json b/tests/dfx/perf/tests/test_tts.json index db197c34b..733c40963 100644 --- a/tests/dfx/perf/tests/test_tts.json +++ b/tests/dfx/perf/tests/test_tts.json @@ -4,7 +4,9 @@ "server_params": { "model": "Qwen/Qwen3-TTS-12Hz-1.7B-Base", "stage_overrides": { - "1": {"devices": "1"} + "1": { + "devices": "1" + } } }, "benchmark_params": [ @@ -15,13 +17,21 @@ "backend": "openai-audio-speech", "endpoint": "/v1/audio/speech", "dataset_path": "linyueqian/seed-tts-eval-subset", - "num_prompts": [20], - "max_concurrency": [1], + "num_prompts": [ + 20 + ], + "max_concurrency": [ + 1 + ], "seed_tts_locale": "en", "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { - "median_audio_ttfp_ms": [350], - "median_audio_rtf": [0.25] + "median_audio_ttfp_ms": [ + 3000 + ], + "median_audio_rtf": [ + 0.6 + ] } }, { @@ -31,14 +41,34 @@ "backend": "openai-audio-speech", "endpoint": "/v1/audio/speech", "dataset_path": "linyueqian/seed-tts-eval-subset", - "num_prompts": [80, 128, 128], - "max_concurrency": [8, 16, 64], + "num_prompts": [ + 80, + 128, + 128 + ], + "max_concurrency": [ + 8, + 16, + 64 + ], "seed_tts_locale": "en", "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { - "median_audio_ttfp_ms": [3500, 30000, 7000], - "median_audio_rtf": [0.75, 10.0, 1.5], - "audio_throughput": [10.0, 1.0, 40.0] + "median_audio_ttfp_ms": [ + 1000, + 3000, + 14000 + ], + "median_audio_rtf": [ + 0.70, + 1.15, + 3.90 + ], + "audio_throughput": [ + 12, + 14, + 14 + ] } }, { @@ -49,13 +79,19 @@ "backend": "openai-audio-speech", "endpoint": "/v1/audio/speech", "dataset_path": "linyueqian/seed-tts-eval-subset", - "num_prompts": [200], - "max_concurrency": [4], + "num_prompts": [ + 200 + ], + "max_concurrency": [ + 4 + ], "seed_tts_locale": "en", "seed_tts_wer_eval": true, "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { - "mean_audio_rtf": [0.45] + "mean_audio_rtf": [ + 0.45 + ] } } ] @@ -65,7 +101,9 @@ "server_params": { "model": "Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice", "stage_overrides": { - "1": {"devices": "1"} + "1": { + "devices": "1" + } } }, "benchmark_params": [ @@ -76,14 +114,26 @@ "backend": "openai-audio-speech", "endpoint": "/v1/audio/speech", "dataset_path": "benchmarks/build_dataset/seed_tts_smoke", - "num_prompts": [20], - "max_concurrency": [1], + "num_prompts": [ + 20 + ], + "max_concurrency": [ + 1 + ], "seed_tts_locale": "en", - "extra_body": {"voice": "Vivian", "language": "English", "task_type": "CustomVoice"}, + "extra_body": { + "voice": "Vivian", + "language": "English", + "task_type": "CustomVoice" + }, "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { - "median_audio_ttfp_ms": [150], - "median_audio_rtf": [0.15] + "median_audio_ttfp_ms": [ + 2000 + ], + "median_audio_rtf": [ + 0.40 + ] } }, { @@ -93,15 +143,39 @@ "backend": "openai-audio-speech", "endpoint": "/v1/audio/speech", "dataset_path": "benchmarks/build_dataset/seed_tts_smoke", - "num_prompts": [80, 128, 128], - "max_concurrency": [8, 16, 64], + "num_prompts": [ + 80, + 128, + 128 + ], + "max_concurrency": [ + 8, + 16, + 64 + ], "seed_tts_locale": "en", - "extra_body": {"voice": "Vivian", "language": "English", "task_type": "CustomVoice"}, + "extra_body": { + "voice": "Vivian", + "language": "English", + "task_type": "CustomVoice" + }, "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { - "median_audio_ttfp_ms": [1500, 30000, 3000], - "median_audio_rtf": [0.30, 10.0, 0.60], - "audio_throughput": [30.0, 1.0, 100.0] + "median_audio_ttfp_ms": [ + 300, + 1000, + 5000 + ], + "median_audio_rtf": [ + 0.25, + 0.35, + 1.10 + ], + "audio_throughput": [ + 30, + 40, + 40 + ] } }, { @@ -112,14 +186,24 @@ "backend": "openai-audio-speech", "endpoint": "/v1/audio/speech", "dataset_path": "benchmarks/build_dataset/seed_tts_smoke", - "num_prompts": [200], - "max_concurrency": [4], + "num_prompts": [ + 200 + ], + "max_concurrency": [ + 4 + ], "seed_tts_locale": "en", - "extra_body": {"voice": "Vivian", "language": "English", "task_type": "CustomVoice"}, + "extra_body": { + "voice": "Vivian", + "language": "English", + "task_type": "CustomVoice" + }, "seed_tts_wer_eval": true, "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { - "mean_audio_rtf": [0.35] + "mean_audio_rtf": [ + 0.35 + ] } }, { @@ -129,15 +213,29 @@ "backend": "openai-audio-speech", "endpoint": "/v1/audio/speech", "dataset_path": "benchmarks/build_dataset/seed_tts_smoke", - "num_prompts": [100], - "request_rate": [2.0], + "num_prompts": [ + 100 + ], + "request_rate": [ + 2.0 + ], "seed_tts_locale": "en", - "extra_body": {"voice": "Vivian", "language": "English", "task_type": "CustomVoice"}, + "extra_body": { + "voice": "Vivian", + "language": "English", + "task_type": "CustomVoice" + }, "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { - "median_audio_ttfp_ms": [3000], - "median_audio_rtf": [0.25], - "audio_throughput": [4.0] + "median_audio_ttfp_ms": [ + 100 + ], + "median_audio_rtf": [ + 0.20 + ], + "audio_throughput": [ + 10 + ] } }, { @@ -147,14 +245,25 @@ "backend": "openai-audio-speech", "endpoint": "/v1/audio/speech", "dataset_path": "benchmarks/build_dataset/seed_tts_design", - "num_prompts": [20], - "max_concurrency": [1], + "num_prompts": [ + 20 + ], + "max_concurrency": [ + 1 + ], "seed_tts_locale": "en", - "extra_body": {"task_type": "VoiceDesign", "language": "English"}, + "extra_body": { + "task_type": "VoiceDesign", + "language": "English" + }, "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { - "median_audio_ttfp_ms": [150], - "median_audio_rtf": [0.15] + "median_audio_ttfp_ms": [ + 100 + ], + "median_audio_rtf": [ + 0.15 + ] } }, { @@ -164,15 +273,38 @@ "backend": "openai-audio-speech", "endpoint": "/v1/audio/speech", "dataset_path": "benchmarks/build_dataset/seed_tts_design", - "num_prompts": [80, 128, 128], - "max_concurrency": [8, 16, 64], + "num_prompts": [ + 80, + 128, + 128 + ], + "max_concurrency": [ + 8, + 16, + 64 + ], "seed_tts_locale": "en", - "extra_body": {"task_type": "VoiceDesign", "language": "English"}, + "extra_body": { + "task_type": "VoiceDesign", + "language": "English" + }, "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { - "median_audio_ttfp_ms": [1500, 30000, 3500], - "median_audio_rtf": [0.35, 10.0, 0.70], - "audio_throughput": [25.0, 1.0, 80.0] + "median_audio_ttfp_ms": [ + 100, + 1000, + 5000 + ], + "median_audio_rtf": [ + 0.25, + 0.40, + 1.20 + ], + "audio_throughput": [ + 30, + 40, + 40 + ] } }, { @@ -182,15 +314,28 @@ "backend": "openai-audio-speech", "endpoint": "/v1/audio/speech", "dataset_path": "benchmarks/build_dataset/seed_tts_design", - "num_prompts": [100], - "request_rate": [2.0], + "num_prompts": [ + 100 + ], + "request_rate": [ + 2.0 + ], "seed_tts_locale": "en", - "extra_body": {"task_type": "VoiceDesign", "language": "English"}, + "extra_body": { + "task_type": "VoiceDesign", + "language": "English" + }, "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration", "baseline": { - "median_audio_ttfp_ms": [3500], - "median_audio_rtf": [0.30], - "audio_throughput": [4.0] + "median_audio_ttfp_ms": [ + 100 + ], + "median_audio_rtf": [ + 0.20 + ], + "audio_throughput": [ + 9 + ] } } ] diff --git a/tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json b/tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json index 58a17c980..d70a28efd 100644 --- a/tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json +++ b/tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json @@ -30,9 +30,9 @@ } ], "baseline": { - "throughput_qps": 0.034, - "latency_mean": 26.0, - "peak_memory_mb_mean": 80000 + "throughput_qps": 0.04, + "latency_mean": 30, + "peak_memory_mb_mean": 75000 } } ] @@ -72,9 +72,9 @@ } ], "baseline": { - "throughput_qps": 0.042, - "latency_mean": 21.6, - "peak_memory_mb_mean": 55300 + "throughput_qps": 0.06, + "latency_mean": 20, + "peak_memory_mb_mean": 48000 } }, { @@ -97,9 +97,9 @@ } ], "baseline": { - "throughput_qps": 0.0085, - "latency_mean": 101.6, - "peak_memory_mb_mean": 65200 + "throughput_qps": 0.01, + "latency_mean": 90, + "peak_memory_mb_mean": 56000 } } ] diff --git a/tests/diffusion/distributed/test_autoencoder_kl_hunyuan.py b/tests/diffusion/distributed/test_autoencoder_kl_hunyuan.py new file mode 100644 index 000000000..58ce14546 --- /dev/null +++ b/tests/diffusion/distributed/test_autoencoder_kl_hunyuan.py @@ -0,0 +1,79 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +import pytest +import torch + +from vllm_omni.diffusion.distributed.autoencoders.autoencoder_kl_hunyuan import ( + DistributedAutoencoderKLHunyuan, +) + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu] + + +class _DummyDistributedAutoencoderKLHunyuan(DistributedAutoencoderKLHunyuan): + def __init__(self): + torch.nn.Module.__init__(self) + self.tile_latent_min_size = 8 + self.tile_sample_min_size = 8 + self.tile_overlap_factor = 0.25 + self.use_spatial_tiling = False + + @property + def dtype(self) -> torch.dtype: + return torch.float32 + + def decoder(self, x: torch.Tensor) -> torch.Tensor: + return x + 10 + + def encoder(self, x: torch.Tensor) -> torch.Tensor: + return x + 20 + + def blend_v(self, _a: torch.Tensor, b: torch.Tensor, _blend_extent: int) -> torch.Tensor: + return b + + def blend_h(self, _a: torch.Tensor, b: torch.Tensor, _blend_extent: int) -> torch.Tensor: + return b + + +def test_hunyuan_vae_use_tiling_aliases_spatial_tiling(): + # Verify use_tiling property maps to use_spatial_tiling. + vae = _DummyDistributedAutoencoderKLHunyuan() + + assert not vae.use_tiling + + vae.use_tiling = True + + assert vae.use_spatial_tiling + + +def test_hunyuan_vae_decode_tiles_round_trip(): + # Validate decode tile split/exec/merge returns expected reconstructed tensor. + vae = _DummyDistributedAutoencoderKLHunyuan() + z = torch.arange(144, dtype=torch.float32).reshape(1, 1, 1, 12, 12) + + tile_tasks, grid_spec = vae.tile_split(z) + decoded_tiles = {task.grid_coord: vae.tile_exec(task) for task in tile_tasks} + output = vae.tile_merge(decoded_tiles, grid_spec) + + assert grid_spec.split_dims == (3, 4) + assert grid_spec.grid_shape == (2, 2) + assert grid_spec.tile_spec == {"blend_extent": 2, "row_limit": 6} + assert len(tile_tasks) == 4 + assert torch.equal(output, z + 10) + + +def test_hunyuan_vae_encode_tiles_round_trip(): + # Validate encode tile split/exec/merge returns expected latent tensor. + vae = _DummyDistributedAutoencoderKLHunyuan() + x = torch.arange(144, dtype=torch.float32).reshape(1, 1, 1, 12, 12) + + tile_tasks, grid_spec = vae.encode_tile_split(x) + encoded_tiles = {task.grid_coord: vae.encode_tile_exec(task) for task in tile_tasks} + output = vae.encode_tile_merge(encoded_tiles, grid_spec) + + assert grid_spec.split_dims == (3, 4) + assert grid_spec.grid_shape == (2, 2) + assert grid_spec.tile_spec == {"blend_extent": 2, "row_limit": 6} + assert len(tile_tasks) == 4 + assert torch.equal(output, x + 20) diff --git a/vllm_omni/diffusion/distributed/autoencoders/autoencoder_kl_hunyuan.py b/vllm_omni/diffusion/distributed/autoencoders/autoencoder_kl_hunyuan.py new file mode 100644 index 000000000..c0e3b4d6c --- /dev/null +++ b/vllm_omni/diffusion/distributed/autoencoders/autoencoder_kl_hunyuan.py @@ -0,0 +1,145 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from typing import Any + +import torch +from vllm.logger import init_logger + +from vllm_omni.diffusion.distributed.autoencoders.distributed_vae_executor import ( + DistributedOperator, + DistributedVaeMixin, + GridSpec, + TileTask, +) +from vllm_omni.diffusion.models.hunyuan_image3.autoencoder import AutoencoderKLConv3D + +logger = init_logger(__name__) + + +class DistributedAutoencoderKLHunyuan(AutoencoderKLConv3D, DistributedVaeMixin): + @classmethod + def from_config(cls, config: Any, **kwargs: Any): + model = super().from_config(config, **kwargs) + model.init_distributed() + return model + + @classmethod + def from_pretrained(cls, *args: Any, **kwargs: Any): + model = super().from_pretrained(*args, **kwargs) + model.init_distributed() + return model + + @property + def use_tiling(self) -> bool: + return self.use_spatial_tiling + + @use_tiling.setter + def use_tiling(self, use_tiling: bool) -> None: + self.use_spatial_tiling = use_tiling + + def tile_split(self, z: torch.Tensor) -> tuple[list[TileTask], GridSpec]: + _, _, _, height, width = z.shape + overlap_size = int(self.tile_latent_min_size * (1 - self.tile_overlap_factor)) + blend_extent = int(self.tile_sample_min_size * self.tile_overlap_factor) + row_limit = int(self.tile_sample_min_size - blend_extent) + + tiletask_list = [] + for i in range(0, height, overlap_size): + for j in range(0, width, overlap_size): + tile = z[:, :, :, i : i + self.tile_latent_min_size, j : j + self.tile_latent_min_size] + tiletask_list.append( + TileTask( + len(tiletask_list), + (i // overlap_size, j // overlap_size), + tile, + workload=tile.shape[3] * tile.shape[4], + ) + ) + + grid_spec = GridSpec( + split_dims=(3, 4), + grid_shape=(tiletask_list[-1].grid_coord[0] + 1, tiletask_list[-1].grid_coord[1] + 1), + tile_spec={"blend_extent": blend_extent, "row_limit": row_limit}, + output_dtype=self.dtype, + ) + return tiletask_list, grid_spec + + def tile_exec(self, task: TileTask) -> torch.Tensor: + return self.decoder(task.tensor) + + def tile_merge(self, coord_tensor_map: dict[tuple[int, ...], torch.Tensor], grid_spec: GridSpec) -> torch.Tensor: + grid_h, grid_w = grid_spec.grid_shape + result_rows = [] + for i in range(grid_h): + result_row = [] + for j in range(grid_w): + tile = coord_tensor_map[(i, j)] + if i > 0: + tile = self.blend_v(coord_tensor_map[(i - 1, j)], tile, grid_spec.tile_spec["blend_extent"]) + if j > 0: + tile = self.blend_h(coord_tensor_map[(i, j - 1)], tile, grid_spec.tile_spec["blend_extent"]) + result_row.append(tile[:, :, :, : grid_spec.tile_spec["row_limit"], : grid_spec.tile_spec["row_limit"]]) + result_rows.append(torch.cat(result_row, dim=-1)) + return torch.cat(result_rows, dim=-2) + + def encode_tile_split(self, x: torch.Tensor) -> tuple[list[TileTask], GridSpec]: + _, _, _, height, width = x.shape + overlap_size = int(self.tile_sample_min_size * (1 - self.tile_overlap_factor)) + blend_extent = int(self.tile_latent_min_size * self.tile_overlap_factor) + row_limit = int(self.tile_latent_min_size - blend_extent) + + tiletask_list = [] + for i in range(0, height, overlap_size): + for j in range(0, width, overlap_size): + tile = x[:, :, :, i : i + self.tile_sample_min_size, j : j + self.tile_sample_min_size] + tiletask_list.append( + TileTask( + len(tiletask_list), + (i // overlap_size, j // overlap_size), + tile, + workload=tile.shape[3] * tile.shape[4], + ) + ) + + grid_spec = GridSpec( + split_dims=(3, 4), + grid_shape=(tiletask_list[-1].grid_coord[0] + 1, tiletask_list[-1].grid_coord[1] + 1), + tile_spec={"blend_extent": blend_extent, "row_limit": row_limit}, + output_dtype=self.dtype, + ) + return tiletask_list, grid_spec + + def encode_tile_exec(self, task: TileTask) -> torch.Tensor: + return self.encoder(task.tensor) + + def encode_tile_merge( + self, coord_tensor_map: dict[tuple[int, ...], torch.Tensor], grid_spec: GridSpec + ) -> torch.Tensor: + return self.tile_merge(coord_tensor_map, grid_spec) + + def spatial_tiled_encode(self, x: torch.Tensor): + if not self.is_distributed_enabled(): + return super().spatial_tiled_encode(x) + + logger.debug("Encode running with distributed executor") + return self.distributed_executor.execute( + x, + DistributedOperator( + split=self.encode_tile_split, + exec=self.encode_tile_exec, + merge=self.encode_tile_merge, + ), + broadcast_result=True, + ) + + def spatial_tiled_decode(self, z: torch.Tensor): + if not self.is_distributed_enabled(): + return super().spatial_tiled_decode(z) + + logger.debug("Decode running with distributed executor") + return self.distributed_executor.execute( + z, + DistributedOperator(split=self.tile_split, exec=self.tile_exec, merge=self.tile_merge), + broadcast_result=True, + ) diff --git a/vllm_omni/diffusion/models/hunyuan_image3/pipeline_hunyuan_image3.py b/vllm_omni/diffusion/models/hunyuan_image3/pipeline_hunyuan_image3.py index 73b89bb11..677d67796 100644 --- a/vllm_omni/diffusion/models/hunyuan_image3/pipeline_hunyuan_image3.py +++ b/vllm_omni/diffusion/models/hunyuan_image3/pipeline_hunyuan_image3.py @@ -30,7 +30,6 @@ from vllm_omni.diffusion.request import OmniDiffusionRequest from vllm_omni.inputs.data import OmniTextPrompt from vllm_omni.model_executor.models.hunyuan_image3.siglip2 import Siglip2VisionTransformer -from .autoencoder import AutoencoderKLConv3D from .hunyuan_image3_tokenizer import TokenizerWrapper from .hunyuan_image3_transformer import ( CausalMMOutputWithPast, @@ -343,7 +342,13 @@ class HunyuanImage3Pipeline( quant_config = od_config.quantization_config self.model = HunyuanImage3Model(self.hf_config, quant_config=quant_config) self.transformer = self.model - self.vae = AutoencoderKLConv3D.from_config(self.hf_config.vae) + # Lazy import to break circular dependency: + # autoencoder_kl_hunyuan -> hunyuan_image3/__init__ -> pipeline_hunyuan_image3 -> autoencoder_kl_hunyuan + from vllm_omni.diffusion.distributed.autoencoders.autoencoder_kl_hunyuan import ( # noqa: PLC0415 + DistributedAutoencoderKLHunyuan, + ) + + self.vae = DistributedAutoencoderKLHunyuan.from_config(self.hf_config.vae) self.vae.use_spatial_tiling = self.od_config.vae_use_tiling self._pipeline = None self._tkwrapper = TokenizerWrapper(od_config.model)