Merge branch 'main' into docs/add-new-committers

Correct yongxiang mapping in the diffusion cache ownership list.

Signed-off-by: hsliuustc0106 <liuhongsheng4@huawei.com>
Co-authored-by: Cursor <cursoragent@cursor.com>

docs/community/governance.md

@@ -39,11 +39,34 @@ Committers have write access and merge rights. They typically have deep expertis
 - [@linyueqian](https://github.com/linyueqian): TTS and Omni Support
 - [@lishunyang12](https://github.com/lishunyang12): Quantization and Configuration
 - [@princepride](https://github.com/princepride): Diffusion and Omni Support
-- [@SamitHuang](https://github.com/SamitHuang): RL and Diffusion
+- [@RuixiangMa](https://github.com/RuixiangMa): Diffusion models, parallel, cache, and docs
+- [@SamitHuang](https://github.com/SamitHuang): RL, Diffusion, and cache
 - [@tzhouam](https://github.com/tzhouam): Engine and New Model Support
-- [@wtomin](https://github.com/wtomin): Diffusion and Parallelism
+- [@wtomin](https://github.com/wtomin): Diffusion models, parallel, and docs
 - [@ZeldaHuang](https://github.com/ZeldaHuang): Omni Support
-- [@ZJY0516](https://github.com/ZJY0516): Diffusion and CustomOp
+- [@ZJY0516](https://github.com/ZJY0516): Diffusion attention backend, kernel fusion, and CustomOp
+- [@yuanheng-zhao](https://github.com/yuanheng-zhao): Diffusion cache, offload, and Omni Support
+
+## Diffusion Workload Division
+
+This section breaks down diffusion-related responsibilities. If you have PRs touching these areas, please ping the listed owners for review.
+
+### Runtime-related
+
+- Diffusion models: @RuixiangMa, @wtomin
+
+### Optimization
+
+- Parallel: @RuixiangMa, @wtomin
+- Attention backend: @ZJY0516
+- Cache: @yuanheng-zhao, @RuixiangMa, @SamitHuang
+- Offload: @yuanheng-zhao
+- Quantization: @lishunyang12, @david6666666
+- Kernel fusion / communication-computation: @ZJY0516
+
+### Docs & Test
+
+- Diffusion docs: @RuixiangMa, @wtomin
 
 ## Meetings
 

tests/dfx/conftest.py

@@ -279,6 +279,10 @@ def create_benchmark_indices(
     return indices
 
 
+# Omni perf/stability: ``vllm bench serve --omni`` result template (used if bench exits without writing JSON).
+OMNI_RESULT_TEMPLATE_PATH = Path(__file__).resolve().parent / "perf" / "scripts" / "result_omni_template.json"
+
+
 def _safe_filename_token(value: Any | None, *, default: str = "na") -> str:
     """Make a single path segment safe for result filenames on common filesystems."""
     if value is None:
@@ -289,7 +293,7 @@ def _safe_filename_token(value: Any | None, *, default: str = "na") -> str:
     return s if s else default
 
 
-def _resolve_baseline_value(
+def resolve_baseline_value(
     baseline_raw: Any,
     *,
     sweep_index: int | None,
@@ -330,7 +334,7 @@ def _baseline_thresholds_for_step(
 ) -> dict[str, Any]:
     """Resolve baseline config to one threshold per metric for this iteration."""
     return {
-        metric_name: _resolve_baseline_value(
+        metric_name: resolve_baseline_value(
             baseline_raw,
             sweep_index=sweep_index,
             max_concurrency=max_concurrency,
@@ -354,7 +358,12 @@ def run_benchmark(
     random_input_len: Any | None = None,
     random_output_len: Any | None = None,
 ) -> dict[str, Any]:
-    """Run one ``vllm bench serve --omni`` iteration and return parsed metrics."""
+    """Run one ``vllm bench serve --omni`` iteration and return parsed metrics.
+
+    After ``vllm bench`` writes the JSON, ``result["baseline"]`` holds the resolved per-metric thresholds
+    (when ``baseline_config`` is provided). If the benchmark exits without writing a result file,
+    ``result_omni_template.json`` is used as a fallback.
+    """
     current_dt = datetime.now().strftime("%Y%m%d-%H%M%S")
     ri = _safe_filename_token(random_input_len)
     ro = _safe_filename_token(random_output_len)
@@ -403,8 +412,17 @@ def run_benchmark(
         result_dir = "./"
 
     result_path = os.path.join(result_dir, result_filename)
-    with open(result_path, encoding="utf-8") as f:
+    if not os.path.exists(result_path):
-        result = json.load(f)
+        with open(OMNI_RESULT_TEMPLATE_PATH, encoding="utf-8") as f:
+            template_result: dict[str, Any] = json.load(f)
+        Path(result_path).parent.mkdir(parents=True, exist_ok=True)
+        with open(result_path, "w", encoding="utf-8") as f:
+            json.dump(template_result, f, ensure_ascii=False, indent=2)
+        print(f"Benchmark result file not generated, fallback to template: {result_path}")
+        result = template_result
+    else:
+        with open(result_path, encoding="utf-8") as f:
+            result = json.load(f)
 
     if baseline_config:
         result["baseline"] = _baseline_thresholds_for_step(
@@ -432,3 +450,12 @@ def pytest_addoption(parser: pytest.Parser) -> None:
         default=None,
         help=("Path to benchmark config JSON. Example: --test-config-file tests/dfx/perf/tests/test_tts.json"),
     )
+    parser.addoption(
+        "--assert-baseline",
+        action="store_true",
+        default=False,
+        help=(
+            "When set, omni/diffusion perf runners compare metrics against the baseline block in the JSON config "
+            "(default: off)."
+        ),
+    )

tests/dfx/perf/scripts/run_benchmark.py

@@ -12,12 +12,17 @@ from tests.dfx.conftest import (
     create_unique_server_params,
     get_benchmark_params_for_server,
     load_configs,
+    resolve_baseline_value,
     run_benchmark,
 )
 from tests.helpers.runtime import OmniServer
 
 pytestmark = [pytest.mark.full_model]
 
+# Compare metrics to each test JSON ``baseline`` block only when pytest is run with ``--assert-baseline``
+# (registered in ``tests/dfx/conftest.py``; default: off). ``run_benchmark`` and ``_resolve_baseline_value`` are
+# defined in the same module.
+
 
 os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
 
@@ -46,7 +51,6 @@ if CONFIG_FILE_PATH is None:
     CONFIG_FILE_PATH = _DEFAULT_CONFIG_FILE
 
 BENCHMARK_CONFIGS = load_configs(CONFIG_FILE_PATH)
-OMNI_RESULT_TEMPLATE_PATH = Path(__file__).parent / "result_omni_template.json"
 
 
 DEPLOY_CONFIGS_DIR = Path(__file__).parent.parent / "deploy"
@@ -87,16 +91,6 @@ def omni_server(request):
         print("OmniServer stopped")
 
 
-def _safe_filename_token(value: Any | None, *, default: str = "na") -> str:
-    """Make a single path segment safe for result filenames on common filesystems."""
-    if value is None:
-        return default
-    s = str(value).strip()
-    for bad in ("/", "\\", ":", "*", "?", '"', "<", ">", "|"):
-        s = s.replace(bad, "_")
-    return s if s else default
-
-
 benchmark_indices = create_benchmark_indices(BENCHMARK_CONFIGS, server_to_benchmark_mapping)
 
 
@@ -126,76 +120,23 @@ def benchmark_params(request, omni_server):
     }
 
 
-def _resolve_baseline_value(
-    baseline_raw: Any,
-    *,
-    sweep_index: int | None,
-    max_concurrency: Any = None,
-    request_rate: Any = None,
-) -> Any:
-    """Pick the baseline threshold for this sweep step.
-
-    Supported shapes per metric:
-    - **Scalar** — same threshold for every concurrency / QPS.
-    - **List** — aligned with ``max_concurrency`` / ``request_rate`` sweep order; use ``sweep_index``.
-    - **Dict** — keyed by concurrency or rate, e.g. ``{"1": 500, "4": 800}`` (keys are strings in JSON).
-
-    For dict lookup, ``max_concurrency`` is preferred when both are set (concurrency sweep).
-    """
-    if baseline_raw is None:
-        # If no baseline is set, the maximum value will be used.
-        return 100000
-    if isinstance(baseline_raw, dict):
-        if max_concurrency is not None:
-            for key in (max_concurrency, str(max_concurrency)):
-                if key in baseline_raw:
-                    return baseline_raw[key]
-        if request_rate is not None:
-            for key in (request_rate, str(request_rate)):
-                if key in baseline_raw:
-                    return baseline_raw[key]
-        raise KeyError(
-            f"baseline dict has no key for max_concurrency={max_concurrency!r} "
-            f"or request_rate={request_rate!r}; keys={list(baseline_raw.keys())!r}"
-        )
-    if isinstance(baseline_raw, (list, tuple)):
-        return baseline_raw[sweep_index]
-    return baseline_raw
-
-
-def _baseline_thresholds_for_step(
-    baseline_data: dict[str, Any],
-    *,
-    sweep_index: int | None = None,
-    max_concurrency: Any = None,
-    request_rate: Any = None,
-) -> dict[str, Any]:
-    """Resolve ``test.json`` ``baseline`` block to one threshold per metric (same as ``assert_result``)."""
-    return {
-        metric_name: _resolve_baseline_value(
-            baseline_raw,
-            sweep_index=sweep_index,
-            max_concurrency=max_concurrency,
-            request_rate=request_rate,
-        )
-        for metric_name, baseline_raw in baseline_data.items()
-    }
-
-
 def assert_result(
     result,
     params,
     num_prompt,
     *,
+    assert_baseline: bool,
     sweep_index: int | None = None,
-    max_concurrency: Any = None,
+    max_concurrency: Any | None = None,
-    request_rate: Any = None,
+    request_rate: Any | None = None,
 ) -> None:
     assert result["completed"] == num_prompt, "Request failures exist"
+    if not assert_baseline:
+        return
     baseline_data = params.get("baseline", {})
     for metric_name, baseline_raw in baseline_data.items():
         current_value = result[metric_name]
-        baseline_value = _resolve_baseline_value(
+        baseline_value = resolve_baseline_value(
             baseline_raw,
             sweep_index=sweep_index,
             max_concurrency=max_concurrency,
@@ -214,7 +155,7 @@ def assert_result(
 @pytest.mark.benchmark
 @pytest.mark.parametrize("omni_server", test_params, indirect=True)
 @pytest.mark.parametrize("benchmark_params", benchmark_indices, indirect=True)
-def test_performance_benchmark(omni_server, benchmark_params):
+def test_performance_benchmark(omni_server, benchmark_params, request):
     test_name = benchmark_params["test_name"]
     params = benchmark_params["params"]
     dataset_name = params.get("dataset_name", "")
@@ -226,6 +167,8 @@ def test_performance_benchmark(omni_server, benchmark_params):
     print(f"Running benchmark for model: {model}")
     print(f"Benchmark parameters: {benchmark_params}")
 
+    assert_baseline = request.config.getoption("--assert-baseline", default=False)
+
     def to_list(value, default=None):
         if value is None:
             return [] if default is None else [default]
@@ -264,7 +207,7 @@ def test_performance_benchmark(omni_server, benchmark_params):
         elif not isinstance(value, bool):
             args.extend([arg_name, str(value)])
 
-    # QPS test (sweep_index aligns with qps_list / num_prompt_list for this loop)
+    # QPS / request-rate sweep
     for i, (qps, num_prompt) in enumerate(zip(qps_list, num_prompt_list)):
         args = args + ["--request-rate", str(qps), "--num-prompts", str(num_prompt)]
         result = run_benchmark(
@@ -283,12 +226,13 @@ def test_performance_benchmark(omni_server, benchmark_params):
         assert_result(
             result,
             params,
-            num_prompt=num_prompt,
+            num_prompt,
+            assert_baseline=assert_baseline,
             sweep_index=i,
             request_rate=qps,
         )
 
-    # concurrency test (sweep_index aligns with max_concurrency_list for separate thresholds per concurrency)
+    # concurrency test
     for i, (concurrency, num_prompt) in enumerate(zip(max_concurrency_list, num_prompt_list)):
         args = args + ["--max-concurrency", str(concurrency), "--num-prompts", str(num_prompt), "--request-rate", "inf"]
         result = run_benchmark(
@@ -307,7 +251,8 @@ def test_performance_benchmark(omni_server, benchmark_params):
         assert_result(
             result,
             params,
-            num_prompt=num_prompt,
+            num_prompt,
+            assert_baseline=assert_baseline,
             sweep_index=i,
             max_concurrency=concurrency,
         )

tests/dfx/perf/scripts/run_diffusion_benchmark.py

@@ -11,6 +11,8 @@ This runner separates two concepts:
 A config JSON file is REQUIRED via --test-config-file:
   pytest run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_vllm_omni.json
 
+Optional: ``--assert-baseline`` compares metrics to the ``baseline`` block in each benchmark entry (default: off).
+
 All benchmark results for a session are consolidated into a single JSON file under
 BENCHMARK_RESULT_DIR (override via the DIFFUSION_BENCHMARK_DIR environment variable).
 Each entry in the file contains the test metadata (test_name, endpoint, benchmark_params,
@@ -652,12 +654,14 @@ def run_benchmark(
 # ---------------------------------------------------------------------------
 
 
-def assert_result(result: dict[str, Any], params: dict[str, Any]) -> None:
+def assert_result(result: dict[str, Any], params: dict[str, Any], *, assert_baseline: bool) -> None:
-    """Assert that benchmark metrics satisfy the configured baselines."""
+    """Assert request completion; optionally assert metrics against ``baseline`` in *params*."""
     num_prompts = params.get("num-prompts", 10)
     completed = result.get("completed_requests", result.get("completed", 0))
     assert completed == num_prompts, f"Expected {num_prompts} completed requests, got {completed}"
 
+    if not assert_baseline:
+        return
     if params.get("skip-performance-assertion", False):
         print("Skipping performance assertions.")
         return
@@ -699,16 +703,14 @@ def _resolve_benchmark_endpoint(server_cfg: dict[str, Any], params: dict[str, An
     indirect=True,
 )
 @pytest.mark.parametrize("benchmark_params", benchmark_indices, indirect=True)
-def test_diffusion_performance_benchmark(diffusion_server, benchmark_params):
+def test_diffusion_performance_benchmark(diffusion_server, benchmark_params, request):
-    """Run the diffusion performance benchmark and assert against baselines.
+    """Run the diffusion performance benchmark and verify request completion.
 
     One server is started per unique parallel configuration (module scope).
     For each server, all benchmark parameter sets defined in the config JSON
-    are executed sequentially; results are asserted against the baselines.
+    are executed sequentially; metrics are recorded to the aggregated result file.
 
-    Tracked metrics:
+    Pass ``--assert-baseline`` to compare ``throughput_qps``, ``latency_mean``, etc. to the JSON ``baseline`` block.
-        - throughput_qps          (higher is better)
-        - latency_p50, latency_p99 (lower is better)
     """
     test_name = benchmark_params["test_name"]
     params = benchmark_params["params"]
@@ -744,4 +746,5 @@ def test_diffusion_performance_benchmark(diffusion_server, benchmark_params):
     print(f"\n  Aggregated results: {AGGREGATED_RESULT_FILE}")
     print("=" * 60)
 
-    assert_result(result, params)
+    assert_baseline = request.config.getoption("--assert-baseline", default=False)
+    assert_result(result, params, assert_baseline=assert_baseline)

tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json

@@ -3,31 +3,67 @@
         "test_name": "test_qwen3_omni_chunk",
         "server_params": {
             "model": "Qwen/Qwen3-Omni-30B-A3B-Instruct",
-            "extra_cli_args": ["--async-chunk"]
+            "extra_cli_args": [
+                "--async-chunk"
+            ]
         },
         "benchmark_params": [
             {
                 "dataset_name": "random",
                 "backend": "openai-chat-omni",
                 "endpoint": "/v1/chat/completions",
-                "num_prompts": [4, 16, 32, 64, 128],
+                "num_prompts": [
-                "max_concurrency": [1, 4, 8, 16, 32],
+                    4,
+                    16,
+                    32,
+                    64,
+                    128
+                ],
+                "max_concurrency": [
+                    1,
+                    4,
+                    8,
+                    16,
+                    32
+                ],
                 "random_input_len": 2500,
                 "random_output_len": 900,
                 "ignore_eos": true,
                 "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
                 "baseline": {
-                    "mean_ttft_ms": [1000, 3000, 5000, 7000, 9000],
+                    "mean_ttft_ms": [
-                    "mean_audio_ttfp_ms": [1000, 3000, 5000, 7000, 9000],
+                        400,
-                    "mean_audio_rtf": [0.2, 0.35, 0.6, 0.85, 0.9]
+                        300,
+                        400,
+                        600,
+                        900
+                    ],
+                    "mean_e2el_ms": [
+                        26000,
+                        43000,
+                        52000,
+                        72000,
+                        109000
+                    ],
+                    "mean_audio_rtf": [
+                        0.15,
+                        0.30,
+                        0.35,
+                        0.50,
+                        0.75
+                    ]
                 }
             },
             {
                 "dataset_name": "random-mm",
                 "backend": "openai-chat-omni",
                 "endpoint": "/v1/chat/completions",
-                "num_prompts": [10],
+                "num_prompts": [
-                "request_rate": [0.1],
+                    10
+                ],
+                "request_rate": [
+                    0.1
+                ],
                 "random_input_len": 100,
                 "random_output_len": 100,
                 "random_range_ratio": 0.0,
@@ -42,17 +78,27 @@
                 },
                 "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
                 "baseline": {
-                    "mean_ttft_ms": [2000],
+                    "mean_ttft_ms": [
-                    "mean_audio_ttfp_ms": [2000],
+                        600
-                    "mean_audio_rtf": [0.25]
+                    ],
+                    "mean_e2el_ms": [
+                        6000
+                    ],
+                    "mean_audio_rtf": [
+                        0.20
+                    ]
                 }
             },
             {
                 "dataset_name": "random-mm",
                 "backend": "openai-chat-omni",
                 "endpoint": "/v1/chat/completions",
-                "num_prompts": [40],
+                "num_prompts": [
-                "request_rate": [0.5],
+                    40
+                ],
+                "request_rate": [
+                    0.5
+                ],
                 "random_input_len": 100,
                 "random_output_len": 100,
                 "random_range_ratio": 0.0,
@@ -69,17 +115,27 @@
                 },
                 "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
                 "baseline": {
-                    "mean_ttft_ms": [6000],
+                    "mean_ttft_ms": [
-                    "mean_audio_ttfp_ms": [6000],
+                        300
-                    "mean_audio_rtf": [0.7]
+                    ],
+                    "mean_e2el_ms": [
+                        7000
+                    ],
+                    "mean_audio_rtf": [
+                        0.25
+                    ]
                 }
             },
             {
                 "dataset_name": "random-mm",
                 "backend": "openai-chat-omni",
                 "endpoint": "/v1/chat/completions",
-                "num_prompts": [100],
+                "num_prompts": [
-                "request_rate": [1.0],
+                    100
+                ],
+                "request_rate": [
+                    1.0
+                ],
                 "random_input_len": 100,
                 "random_output_len": 100,
                 "random_range_ratio": 0.0,
@@ -98,40 +154,79 @@
                 },
                 "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
                 "baseline": {
-                    "mean_ttft_ms": [12000],
+                    "mean_ttft_ms": [
-                    "mean_audio_ttfp_ms": [12000],
+                        700
-                    "mean_audio_rtf": [1.0]
+                    ],
+                    "mean_e2el_ms": [
+                        11000
+                    ],
+                    "mean_audio_rtf": [
+                        0.35
+                    ]
                 }
             },
             {
                 "dataset_name": "random",
                 "backend": "openai-chat-omni",
                 "endpoint": "/v1/chat/completions",
-                "num_prompts": [4, 16, 32, 64, 128],
+                "num_prompts": [
-                "max_concurrency": [1, 4, 8, 16, 32],
+                    4,
+                    16,
+                    32,
+                    64,
+                    128
+                ],
+                "max_concurrency": [
+                    1,
+                    4,
+                    8,
+                    16,
+                    32
+                ],
                 "random_input_len": 2500,
                 "random_output_len": 900,
                 "ignore_eos": true,
                 "extra_body": {
-                    "modalities": ["text"]
+                    "modalities": [
+                        "text"
+                    ]
                 },
                 "percentile-metrics": "ttft,tpot,itl,e2el",
                 "baseline": {
-                    "mean_ttft_ms": [1000, 3000, 5000, 7000, 9000]
+                    "mean_ttft_ms": [
+                        100,
+                        300,
+                        500,
+                        800,
+                        1500
+                    ],
+                    "mean_e2el_ms": [
+                        5000,
+                        8000,
+                        10000,
+                        14000,
+                        20000
+                    ]
                 }
             },
             {
                 "dataset_name": "random-mm",
                 "backend": "openai-chat-omni",
                 "endpoint": "/v1/chat/completions",
-                "num_prompts": [10],
+                "num_prompts": [
-                "request_rate": [0.1],
+                    10
+                ],
+                "request_rate": [
+                    0.1
+                ],
                 "random_input_len": 100,
                 "random_output_len": 100,
                 "random_range_ratio": 0.0,
                 "ignore_eos": true,
                 "extra_body": {
-                    "modalities": ["text"]
+                    "modalities": [
+                        "text"
+                    ]
                 },
                 "random_mm_base_items_per_request": 1,
                 "random_mm_num_mm_items_range_ratio": 0.5,
@@ -143,21 +238,32 @@
                 },
                 "percentile-metrics": "ttft,tpot,itl,e2el",
                 "baseline": {
-                    "mean_ttft_ms": [2000]
+                    "mean_ttft_ms": [
+                        500
+                    ],
+                    "mean_e2el_ms": [
+                        1000
+                    ]
                 }
             },
             {
                 "dataset_name": "random-mm",
                 "backend": "openai-chat-omni",
                 "endpoint": "/v1/chat/completions",
-                "num_prompts": [40],
+                "num_prompts": [
-                "request_rate": [0.5],
+                    40
+                ],
+                "request_rate": [
+                    0.5
+                ],
                 "random_input_len": 100,
                 "random_output_len": 100,
                 "random_range_ratio": 0.0,
                 "ignore_eos": true,
                 "extra_body": {
-                    "modalities": ["text"]
+                    "modalities": [
+                        "text"
+                    ]
                 },
                 "random_mm_base_items_per_request": 2,
                 "random_mm_num_mm_items_range_ratio": 0.5,
@@ -171,21 +277,32 @@
                 },
                 "percentile-metrics": "ttft,tpot,itl,e2el",
                 "baseline": {
-                    "mean_ttft_ms": [6000]
+                    "mean_ttft_ms": [
+                        200
+                    ],
+                    "mean_e2el_ms": [
+                        800
+                    ]
                 }
             },
             {
                 "dataset_name": "random-mm",
                 "backend": "openai-chat-omni",
                 "endpoint": "/v1/chat/completions",
-                "num_prompts": [100],
+                "num_prompts": [
-                "request_rate": [1.0],
+                    100
+                ],
+                "request_rate": [
+                    1.0
+                ],
                 "random_input_len": 100,
                 "random_output_len": 100,
                 "random_range_ratio": 0.0,
                 "ignore_eos": true,
                 "extra_body": {
-                    "modalities": ["text"]
+                    "modalities": [
+                        "text"
+                    ]
                 },
                 "random_mm_base_items_per_request": 3,
                 "random_mm_num_mm_items_range_ratio": 0.5,
@@ -201,7 +318,12 @@
                 },
                 "percentile-metrics": "ttft,tpot,itl,e2el",
                 "baseline": {
-                    "mean_ttft_ms": [6000]
+                    "mean_ttft_ms": [
+                        600
+                    ],
+                    "mean_e2el_ms": [
+                        1300
+                    ]
                 }
             }
         ]

tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json

@@ -3,31 +3,67 @@
         "test_name": "test_qwen3_omni",
         "server_params": {
             "model": "Qwen/Qwen3-Omni-30B-A3B-Instruct",
-            "extra_cli_args": ["--no-async-chunk"]
+            "extra_cli_args": [
+                "--no-async-chunk"
+            ]
         },
         "benchmark_params": [
             {
                 "dataset_name": "random",
                 "backend": "openai-chat-omni",
                 "endpoint": "/v1/chat/completions",
-                "num_prompts": [4, 16, 32, 64, 128],
+                "num_prompts": [
-                "max_concurrency": [1, 4, 8, 16, 32],
+                    4,
+                    16,
+                    32,
+                    64,
+                    128
+                ],
+                "max_concurrency": [
+                    1,
+                    4,
+                    8,
+                    16,
+                    32
+                ],
                 "random_input_len": 2500,
                 "random_output_len": 900,
                 "ignore_eos": true,
                 "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
                 "baseline": {
-                    "mean_ttft_ms": [1000, 3000, 5000, 7000, 9000],
+                    "mean_ttft_ms": [
-                    "mean_audio_ttfp_ms": [30000, 60000, 90000, 120000, 150000],
+                        400,
-                    "mean_audio_rtf": [0.35, 0.45, 0.55, 0.65, 0.75]
+                        300,
+                        400,
+                        600,
+                        900
+                    ],
+                    "mean_e2el_ms": [
+                        32000,
+                        34000,
+                        45000,
+                        61000,
+                        90000
+                    ],
+                    "mean_audio_rtf": [
+                        0.20,
+                        0.25,
+                        0.30,
+                        0.45,
+                        0.60
+                    ]
                 }
             },
             {
                 "dataset_name": "random-mm",
                 "backend": "openai-chat-omni",
                 "endpoint": "/v1/chat/completions",
-                "num_prompts": [10],
+                "num_prompts": [
-                "request_rate": [0.1],
+                    10
+                ],
+                "request_rate": [
+                    0.1
+                ],
                 "random_input_len": 100,
                 "random_output_len": 100,
                 "random_range_ratio": 0.0,
@@ -42,17 +78,27 @@
                 },
                 "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
                 "baseline": {
-                    "mean_ttft_ms": [2000],
+                    "mean_ttft_ms": [
-                    "mean_audio_ttfp_ms": [10000],
+                        500
-                    "mean_audio_rtf": [0.25]
+                    ],
+                    "mean_e2el_ms": [
+                        6000
+                    ],
+                    "mean_audio_rtf": [
+                        0.20
+                    ]
                 }
             },
             {
                 "dataset_name": "random-mm",
                 "backend": "openai-chat-omni",
                 "endpoint": "/v1/chat/completions",
-                "num_prompts": [40],
+                "num_prompts": [
-                "request_rate": [0.5],
+                    40
+                ],
+                "request_rate": [
+                    0.5
+                ],
                 "random_input_len": 100,
                 "random_output_len": 100,
                 "random_range_ratio": 0.0,
@@ -69,17 +115,27 @@
                 },
                 "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
                 "baseline": {
-                    "mean_ttft_ms": [6000],
+                    "mean_ttft_ms": [
-                    "mean_audio_ttfp_ms": [15000],
+                        300
-                    "mean_audio_rtf": [0.45]
+                    ],
+                    "mean_e2el_ms": [
+                        7000
+                    ],
+                    "mean_audio_rtf": [
+                        0.20
+                    ]
                 }
             },
             {
                 "dataset_name": "random-mm",
                 "backend": "openai-chat-omni",
                 "endpoint": "/v1/chat/completions",
-                "num_prompts": [100],
+                "num_prompts": [
-                "request_rate": [1.0],
+                    100
+                ],
+                "request_rate": [
+                    1.0
+                ],
                 "random_input_len": 100,
                 "random_output_len": 100,
                 "random_range_ratio": 0.0,
@@ -98,9 +154,15 @@
                 },
                 "percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
                 "baseline": {
-                    "mean_ttft_ms": [12000],
+                    "mean_ttft_ms": [
-                    "mean_audio_ttfp_ms": [18000],
+                        700
-                    "mean_audio_rtf": [0.9]
+                    ],
+                    "mean_e2el_ms": [
+                        9000
+                    ],
+                    "mean_audio_rtf": [
+                        0.30
+                    ]
                 }
             }
         ]

tests/dfx/perf/tests/test_qwen3_omni_vllm_text.json

@@ -4,7 +4,9 @@
         "server_params": {
             "model": "Qwen/Qwen3-Omni-30B-A3B-Instruct",
             "use_omni": false,
-            "extra_cli_args": ["--no-enable-prefix-caching"]
+            "extra_cli_args": [
+                "--no-enable-prefix-caching"
+            ]
         },
         "benchmark_params": [
             {
@@ -12,8 +14,20 @@
                 "backend": "vllm",
                 "model": "Qwen/Qwen3-Omni-30B-A3B-Instruct",
                 "endpoint": "/v1/completions",
-                "num_prompts": [4, 16, 32, 64, 128],
+                "num_prompts": [
-                "max_concurrency": [1, 4, 8, 16, 32],
+                    4,
+                    16,
+                    32,
+                    64,
+                    128
+                ],
+                "max_concurrency": [
+                    1,
+                    4,
+                    8,
+                    16,
+                    32
+                ],
                 "random_input_len": 2500,
                 "random_output_len": 900,
                 "temperature": 0.4,
@@ -23,11 +37,26 @@
                 "repetition_penalty": 1.05,
                 "ignore_eos": true,
                 "extra_body": {
-                    "modalities": ["text"]
+                    "modalities": [
+                        "text"
+                    ]
                 },
                 "percentile-metrics": "ttft,tpot,itl,e2el",
                 "baseline": {
-                    "mean_ttft_ms": [1000, 3000, 5000, 7000, 9000]
+                    "mean_ttft_ms": [
+                        300,
+                        300,
+                        1000,
+                        600,
+                        800
+                    ],
+                    "mean_e2el_ms": [
+                        6000,
+                        8000,
+                        11000,
+                        13000,
+                        17000
+                    ]
                 }
             },
             {
@@ -35,8 +64,12 @@
                 "backend": "vllm",
                 "model": "Qwen/Qwen3-Omni-30B-A3B-Instruct",
                 "endpoint": "/v1/completions",
-                "num_prompts": [10],
+                "num_prompts": [
-                "request_rate": [0.1],
+                    10
+                ],
+                "request_rate": [
+                    0.1
+                ],
                 "random_input_len": 100,
                 "random_output_len": 100,
                 "random_range_ratio": 0.0,
@@ -47,7 +80,9 @@
                 "repetition_penalty": 1.05,
                 "ignore_eos": true,
                 "extra_body": {
-                    "modalities": ["text"]
+                    "modalities": [
+                        "text"
+                    ]
                 },
                 "random_mm_base_items_per_request": 1,
                 "random_mm_num_mm_items_range_ratio": 0.5,
@@ -59,7 +94,12 @@
                 },
                 "percentile-metrics": "ttft,tpot,itl,e2el",
                 "baseline": {
-                    "mean_ttft_ms": [2000]
+                    "mean_ttft_ms": [
+                        30
+                    ],
+                    "mean_e2el_ms": [
+                        540
+                    ]
                 }
             },
             {
@@ -67,8 +107,12 @@
                 "backend": "vllm",
                 "model": "Qwen/Qwen3-Omni-30B-A3B-Instruct",
                 "endpoint": "/v1/completions",
-                "num_prompts": [40],
+                "num_prompts": [
-                "request_rate": [0.5],
+                    40
+                ],
+                "request_rate": [
+                    0.5
+                ],
                 "random_input_len": 100,
                 "random_output_len": 100,
                 "random_range_ratio": 0.0,
@@ -79,7 +123,9 @@
                 "repetition_penalty": 1.05,
                 "ignore_eos": true,
                 "extra_body": {
-                    "modalities": ["text"]
+                    "modalities": [
+                        "text"
+                    ]
                 },
                 "random_mm_base_items_per_request": 2,
                 "random_mm_num_mm_items_range_ratio": 0.5,
@@ -93,7 +139,12 @@
                 },
                 "percentile-metrics": "ttft,tpot,itl,e2el",
                 "baseline": {
-                    "mean_ttft_ms": [6000]
+                    "mean_ttft_ms": [
+                        30
+                    ],
+                    "mean_e2el_ms": [
+                        600
+                    ]
                 }
             },
             {
@@ -101,8 +152,12 @@
                 "backend": "vllm",
                 "model": "Qwen/Qwen3-Omni-30B-A3B-Instruct",
                 "endpoint": "/v1/completions",
-                "num_prompts": [100],
+                "num_prompts": [
-                "request_rate": [1.0],
+                    100
+                ],
+                "request_rate": [
+                    1.0
+                ],
                 "random_input_len": 100,
                 "random_output_len": 100,
                 "random_range_ratio": 0.0,
@@ -113,7 +168,9 @@
                 "repetition_penalty": 1.05,
                 "ignore_eos": true,
                 "extra_body": {
-                    "modalities": ["text"]
+                    "modalities": [
+                        "text"
+                    ]
                 },
                 "random_mm_base_items_per_request": 3,
                 "random_mm_num_mm_items_range_ratio": 0.5,
@@ -129,7 +186,12 @@
                 },
                 "percentile-metrics": "ttft,tpot,itl,e2el",
                 "baseline": {
-                    "mean_ttft_ms": [6000]
+                    "mean_ttft_ms": [
+                        40
+                    ],
+                    "mean_e2el_ms": [
+                        700
+                    ]
                 }
             }
         ]

tests/dfx/perf/tests/test_qwen_image_edit_2509_vllm_omni.json

@@ -22,10 +22,10 @@
                 "num-input-images": 2,
                 "enable-negative-prompt": true,
                 "baseline": {
-                    "throughput_qps": 0.05,
+                    "throughput_qps": 0.06,
-                    "latency_mean": 18,
+                    "latency_mean": 20,
-                    "peak_memory_mb_max": 78500,
+                    "peak_memory_mb_max": 57000,
-                    "peak_memory_mb_mean": 78500
+                    "peak_memory_mb_mean": 57000
                 }
             },
             {
@@ -41,9 +41,9 @@
                 "enable-negative-prompt": true,
                 "baseline": {
                     "throughput_qps": 0.01,
-                    "latency_mean": 70,
+                    "latency_mean": 60,
-                    "peak_memory_mb_max": 81000,
+                    "peak_memory_mb_max": 67000,
-                    "peak_memory_mb_mean": 81000
+                    "peak_memory_mb_mean": 67000
                 }
             }
         ]
@@ -75,10 +75,10 @@
                 "num-input-images": 2,
                 "enable-negative-prompt": true,
                 "baseline": {
-                    "throughput_qps": 0.1,
+                    "throughput_qps": 0.15,
-                    "latency_mean": 12,
+                    "latency_mean": 10,
-                    "peak_memory_mb_max": 69000,
+                    "peak_memory_mb_max": 55000,
-                    "peak_memory_mb_mean": 69000
+                    "peak_memory_mb_mean": 55000
                 }
             },
             {
@@ -93,10 +93,10 @@
                 "num-input-images": 2,
                 "enable-negative-prompt": true,
                 "baseline": {
-                    "throughput_qps": 0.03,
+                    "throughput_qps": 0.05,
-                    "latency_mean": 28,
+                    "latency_mean": 20,
-                    "peak_memory_mb_max": 69000,
+                    "peak_memory_mb_max": 55000,
-                    "peak_memory_mb_mean": 69000
+                    "peak_memory_mb_mean": 55000
                 }
             }
         ]
@@ -138,10 +138,10 @@
                 "num-input-images": 2,
                 "enable-negative-prompt": true,
                 "baseline": {
-                    "throughput_qps": 0.10,
+                    "throughput_qps": 0.21,
-                    "latency_mean": 12,
+                    "latency_mean": 10,
-                    "peak_memory_mb_max": 73000,
+                    "peak_memory_mb_max": 57000,
-                    "peak_memory_mb_mean": 73000
+                    "peak_memory_mb_mean": 57000
                 }
             },
             {
@@ -156,10 +156,10 @@
                 "num-input-images": 2,
                 "enable-negative-prompt": true,
                 "baseline": {
-                    "throughput_qps": 0.05,
+                    "throughput_qps": 0.08,
                     "latency_mean": 20,
-                    "peak_memory_mb_max": 81000,
+                    "peak_memory_mb_max": 66000,
-                    "peak_memory_mb_mean": 81000
+                    "peak_memory_mb_mean": 66000
                 }
             }
         ]

tests/dfx/perf/tests/test_qwen_image_edit_vllm_omni.json

@@ -21,10 +21,10 @@
                 "max-concurrency": 1,
                 "enable-negative-prompt": true,
                 "baseline": {
-                    "throughput_qps": 0.05,
+                    "throughput_qps": 0.1,
-                    "latency_mean": 15.0,
+                    "latency_mean": 10,
-                    "peak_memory_mb_max": 72500,
+                    "peak_memory_mb_max": 57000,
-                    "peak_memory_mb_mean": 72500
+                    "peak_memory_mb_mean": 57000
                 }
             },
             {
@@ -38,10 +38,10 @@
                 "max-concurrency": 1,
                 "enable-negative-prompt": true,
                 "baseline": {
-                    "throughput_qps": 0.01,
+                    "throughput_qps": 0.02,
-                    "latency_mean": 65.6,
+                    "latency_mean": 50,
-                    "peak_memory_mb_max": 80777,
+                    "peak_memory_mb_max": 65000,
-                    "peak_memory_mb_mean": 80777
+                    "peak_memory_mb_mean": 65000
                 }
             }
         ]
@@ -72,10 +72,10 @@
                 "max-concurrency": 1,
                 "enable-negative-prompt": true,
                 "baseline": {
-                    "throughput_qps": 0.10,
+                    "throughput_qps": 0.21,
-                    "latency_mean": 7.2,
+                    "latency_mean": 10,
-                    "peak_memory_mb_max": 68100,
+                    "peak_memory_mb_max": 56000,
-                    "peak_memory_mb_mean": 68100
+                    "peak_memory_mb_mean": 56000
                 }
             },
             {
@@ -89,10 +89,10 @@
                 "max-concurrency": 1,
                 "enable-negative-prompt": true,
                 "baseline": {
-                    "throughput_qps": 0.03,
+                    "throughput_qps": 0.06,
-                    "latency_mean": 24.0,
+                    "latency_mean": 20,
-                    "peak_memory_mb_max": 68100,
+                    "peak_memory_mb_max": 56000,
-                    "peak_memory_mb_mean": 68100
+                    "peak_memory_mb_mean": 56000
                 }
             }
         ]
@@ -133,10 +133,10 @@
                 "max-concurrency": 1,
                 "enable-negative-prompt": true,
                 "baseline": {
-                    "throughput_qps": 0.1,
+                    "throughput_qps": 0.26,
-                    "latency_mean": 6.5,
+                    "latency_mean": 10,
-                    "peak_memory_mb_max": 72600,
+                    "peak_memory_mb_max": 58000,
-                    "peak_memory_mb_mean": 72600
+                    "peak_memory_mb_mean": 58000
                 }
             },
             {
@@ -150,10 +150,10 @@
                 "max-concurrency": 1,
                 "enable-negative-prompt": true,
                 "baseline": {
-                    "throughput_qps": 0.05,
+                    "throughput_qps": 0.1,
-                    "latency_mean": 16.0,
+                    "latency_mean": 10,
-                    "peak_memory_mb_max": 81000,
+                    "peak_memory_mb_max": 67000,
-                    "peak_memory_mb_mean": 81000
+                    "peak_memory_mb_mean": 67000
                 }
             }
         ]

tests/dfx/perf/tests/test_qwen_image_layered_vllm_omni.json

@@ -21,10 +21,10 @@
                 "max-concurrency": 1,
                 "enable-negative-prompt": true,
                 "baseline": {
-                    "throughput_qps": 0.02,
+                    "throughput_qps": 0.05,
-                    "latency_mean": 40.0,
+                    "latency_mean": 20,
-                    "peak_memory_mb_max": 70000,
+                    "peak_memory_mb_max": 63000,
-                    "peak_memory_mb_mean": 70000
+                    "peak_memory_mb_mean": 63000
                 }
             },
             {
@@ -38,10 +38,10 @@
                 "max-concurrency": 1,
                 "enable-negative-prompt": true,
                 "baseline": {
-                    "throughput_qps": 0.005,
+                    "throughput_qps": 0.03,
-                    "latency_mean": 80.0,
+                    "latency_mean": 30,
-                    "peak_memory_mb_max": 70000,
+                    "peak_memory_mb_max": 63000,
-                    "peak_memory_mb_mean": 70000
+                    "peak_memory_mb_mean": 63000
                 }
             }
         ]

tests/dfx/perf/tests/test_qwen_image_vllm_omni.json

@@ -21,9 +21,9 @@
                 "max-concurrency": 1,
                 "enable-negative-prompt": true,
                 "baseline": {
-                    "throughput_qps": 0.30,
+                    "throughput_qps": 0.35,
-                    "latency_mean": 3.50,
+                    "latency_mean": 3,
-                    "peak_memory_mb_mean": 67000
+                    "peak_memory_mb_mean": 56000
                 }
             },
             {
@@ -37,9 +37,9 @@
                 "max-concurrency": 1,
                 "enable-negative-prompt": true,
                 "baseline": {
-                    "throughput_qps": 0.037,
+                    "throughput_qps": 0.03,
-                    "latency_mean": 27.0,
+                    "latency_mean": 26,
-                    "peak_memory_mb_mean": 74000
+                    "peak_memory_mb_mean": 65000
                 }
             }
         ]
@@ -67,9 +67,9 @@
                 "max-concurrency": 1,
                 "enable-negative-prompt": true,
                 "baseline": {
-                    "throughput_qps": 0.30,
+                    "throughput_qps": 0.35,
-                    "latency_mean": 3.50,
+                    "latency_mean": 3,
-                    "peak_memory_mb_mean": 67000
+                    "peak_memory_mb_mean": 56000
                 }
             },
             {
@@ -83,9 +83,9 @@
                 "max-concurrency": 1,
                 "enable-negative-prompt": true,
                 "baseline": {
-                    "throughput_qps": 0.037,
+                    "throughput_qps": 0.03,
-                    "latency_mean": 27.0,
+                    "latency_mean": 26,
-                    "peak_memory_mb_mean": 74000
+                    "peak_memory_mb_mean": 65000
                 }
             }
         ]
@@ -116,9 +116,9 @@
                 "max-concurrency": 1,
                 "enable-negative-prompt": true,
                 "baseline": {
-                    "throughput_qps": 0.1,
+                    "throughput_qps": 0.11,
-                    "latency_mean": 9.1,
+                    "latency_mean": 9,
-                    "peak_memory_mb_mean": 61000
+                    "peak_memory_mb_mean": 55000
                 }
             }
         ]
@@ -159,9 +159,9 @@
                 "max-concurrency": 1,
                 "enable-negative-prompt": true,
                 "baseline": {
-                    "throughput_qps": 0.1,
+                    "throughput_qps": 0.46,
-                    "latency_mean": 3.50,
+                    "latency_mean": 3,
-                    "peak_memory_mb_mean": 67000
+                    "peak_memory_mb_mean": 56000
                 }
             },
             {
@@ -175,9 +175,9 @@
                 "max-concurrency": 1,
                 "enable-negative-prompt": true,
                 "baseline": {
-                    "throughput_qps": 0.1,
+                    "throughput_qps": 0.18,
-                    "latency_mean": 6.15,
+                    "latency_mean": 6,
-                    "peak_memory_mb_mean": 74000
+                    "peak_memory_mb_mean": 67000
                 }
             }
         ]

tests/dfx/perf/tests/test_tts.json

@@ -4,7 +4,9 @@
     "server_params": {
       "model": "Qwen/Qwen3-TTS-12Hz-1.7B-Base",
       "stage_overrides": {
-        "1": {"devices": "1"}
+        "1": {
+          "devices": "1"
+        }
       }
     },
     "benchmark_params": [
@@ -15,13 +17,21 @@
         "backend": "openai-audio-speech",
         "endpoint": "/v1/audio/speech",
         "dataset_path": "linyueqian/seed-tts-eval-subset",
-        "num_prompts": [20],
+        "num_prompts": [
-        "max_concurrency": [1],
+          20
+        ],
+        "max_concurrency": [
+          1
+        ],
         "seed_tts_locale": "en",
         "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
         "baseline": {
-          "median_audio_ttfp_ms": [350],
+          "median_audio_ttfp_ms": [
-          "median_audio_rtf": [0.25]
+            3000
+          ],
+          "median_audio_rtf": [
+            0.6
+          ]
         }
       },
       {
@@ -31,14 +41,34 @@
         "backend": "openai-audio-speech",
         "endpoint": "/v1/audio/speech",
         "dataset_path": "linyueqian/seed-tts-eval-subset",
-        "num_prompts": [80, 128, 128],
+        "num_prompts": [
-        "max_concurrency": [8, 16, 64],
+          80,
+          128,
+          128
+        ],
+        "max_concurrency": [
+          8,
+          16,
+          64
+        ],
         "seed_tts_locale": "en",
         "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
         "baseline": {
-          "median_audio_ttfp_ms": [3500, 30000, 7000],
+          "median_audio_ttfp_ms": [
-          "median_audio_rtf": [0.75, 10.0, 1.5],
+            1000,
-          "audio_throughput": [10.0, 1.0, 40.0]
+            3000,
+            14000
+          ],
+          "median_audio_rtf": [
+            0.70,
+            1.15,
+            3.90
+          ],
+          "audio_throughput": [
+            12,
+            14,
+            14
+          ]
         }
       },
       {
@@ -49,13 +79,19 @@
         "backend": "openai-audio-speech",
         "endpoint": "/v1/audio/speech",
         "dataset_path": "linyueqian/seed-tts-eval-subset",
-        "num_prompts": [200],
+        "num_prompts": [
-        "max_concurrency": [4],
+          200
+        ],
+        "max_concurrency": [
+          4
+        ],
         "seed_tts_locale": "en",
         "seed_tts_wer_eval": true,
         "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
         "baseline": {
-          "mean_audio_rtf": [0.45]
+          "mean_audio_rtf": [
+            0.45
+          ]
         }
       }
     ]
@@ -65,7 +101,9 @@
     "server_params": {
       "model": "Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice",
       "stage_overrides": {
-        "1": {"devices": "1"}
+        "1": {
+          "devices": "1"
+        }
       }
     },
     "benchmark_params": [
@@ -76,14 +114,26 @@
         "backend": "openai-audio-speech",
         "endpoint": "/v1/audio/speech",
         "dataset_path": "benchmarks/build_dataset/seed_tts_smoke",
-        "num_prompts": [20],
+        "num_prompts": [
-        "max_concurrency": [1],
+          20
+        ],
+        "max_concurrency": [
+          1
+        ],
         "seed_tts_locale": "en",
-        "extra_body": {"voice": "Vivian", "language": "English", "task_type": "CustomVoice"},
+        "extra_body": {
+          "voice": "Vivian",
+          "language": "English",
+          "task_type": "CustomVoice"
+        },
         "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
         "baseline": {
-          "median_audio_ttfp_ms": [150],
+          "median_audio_ttfp_ms": [
-          "median_audio_rtf": [0.15]
+            2000
+          ],
+          "median_audio_rtf": [
+            0.40
+          ]
         }
       },
       {
@@ -93,15 +143,39 @@
         "backend": "openai-audio-speech",
         "endpoint": "/v1/audio/speech",
         "dataset_path": "benchmarks/build_dataset/seed_tts_smoke",
-        "num_prompts": [80, 128, 128],
+        "num_prompts": [
-        "max_concurrency": [8, 16, 64],
+          80,
+          128,
+          128
+        ],
+        "max_concurrency": [
+          8,
+          16,
+          64
+        ],
         "seed_tts_locale": "en",
-        "extra_body": {"voice": "Vivian", "language": "English", "task_type": "CustomVoice"},
+        "extra_body": {
+          "voice": "Vivian",
+          "language": "English",
+          "task_type": "CustomVoice"
+        },
         "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
         "baseline": {
-          "median_audio_ttfp_ms": [1500, 30000, 3000],
+          "median_audio_ttfp_ms": [
-          "median_audio_rtf": [0.30, 10.0, 0.60],
+            300,
-          "audio_throughput": [30.0, 1.0, 100.0]
+            1000,
+            5000
+          ],
+          "median_audio_rtf": [
+            0.25,
+            0.35,
+            1.10
+          ],
+          "audio_throughput": [
+            30,
+            40,
+            40
+          ]
         }
       },
       {
@@ -112,14 +186,24 @@
         "backend": "openai-audio-speech",
         "endpoint": "/v1/audio/speech",
         "dataset_path": "benchmarks/build_dataset/seed_tts_smoke",
-        "num_prompts": [200],
+        "num_prompts": [
-        "max_concurrency": [4],
+          200
+        ],
+        "max_concurrency": [
+          4
+        ],
         "seed_tts_locale": "en",
-        "extra_body": {"voice": "Vivian", "language": "English", "task_type": "CustomVoice"},
+        "extra_body": {
+          "voice": "Vivian",
+          "language": "English",
+          "task_type": "CustomVoice"
+        },
         "seed_tts_wer_eval": true,
         "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
         "baseline": {
-          "mean_audio_rtf": [0.35]
+          "mean_audio_rtf": [
+            0.35
+          ]
         }
       },
       {
@@ -129,15 +213,29 @@
         "backend": "openai-audio-speech",
         "endpoint": "/v1/audio/speech",
         "dataset_path": "benchmarks/build_dataset/seed_tts_smoke",
-        "num_prompts": [100],
+        "num_prompts": [
-        "request_rate": [2.0],
+          100
+        ],
+        "request_rate": [
+          2.0
+        ],
         "seed_tts_locale": "en",
-        "extra_body": {"voice": "Vivian", "language": "English", "task_type": "CustomVoice"},
+        "extra_body": {
+          "voice": "Vivian",
+          "language": "English",
+          "task_type": "CustomVoice"
+        },
         "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
         "baseline": {
-          "median_audio_ttfp_ms": [3000],
+          "median_audio_ttfp_ms": [
-          "median_audio_rtf": [0.25],
+            100
-          "audio_throughput": [4.0]
+          ],
+          "median_audio_rtf": [
+            0.20
+          ],
+          "audio_throughput": [
+            10
+          ]
         }
       },
       {
@@ -147,14 +245,25 @@
         "backend": "openai-audio-speech",
         "endpoint": "/v1/audio/speech",
         "dataset_path": "benchmarks/build_dataset/seed_tts_design",
-        "num_prompts": [20],
+        "num_prompts": [
-        "max_concurrency": [1],
+          20
+        ],
+        "max_concurrency": [
+          1
+        ],
         "seed_tts_locale": "en",
-        "extra_body": {"task_type": "VoiceDesign", "language": "English"},
+        "extra_body": {
+          "task_type": "VoiceDesign",
+          "language": "English"
+        },
         "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
         "baseline": {
-          "median_audio_ttfp_ms": [150],
+          "median_audio_ttfp_ms": [
-          "median_audio_rtf": [0.15]
+            100
+          ],
+          "median_audio_rtf": [
+            0.15
+          ]
         }
       },
       {
@@ -164,15 +273,38 @@
         "backend": "openai-audio-speech",
         "endpoint": "/v1/audio/speech",
         "dataset_path": "benchmarks/build_dataset/seed_tts_design",
-        "num_prompts": [80, 128, 128],
+        "num_prompts": [
-        "max_concurrency": [8, 16, 64],
+          80,
+          128,
+          128
+        ],
+        "max_concurrency": [
+          8,
+          16,
+          64
+        ],
         "seed_tts_locale": "en",
-        "extra_body": {"task_type": "VoiceDesign", "language": "English"},
+        "extra_body": {
+          "task_type": "VoiceDesign",
+          "language": "English"
+        },
         "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
         "baseline": {
-          "median_audio_ttfp_ms": [1500, 30000, 3500],
+          "median_audio_ttfp_ms": [
-          "median_audio_rtf": [0.35, 10.0, 0.70],
+            100,
-          "audio_throughput": [25.0, 1.0, 80.0]
+            1000,
+            5000
+          ],
+          "median_audio_rtf": [
+            0.25,
+            0.40,
+            1.20
+          ],
+          "audio_throughput": [
+            30,
+            40,
+            40
+          ]
         }
       },
       {
@@ -182,15 +314,28 @@
         "backend": "openai-audio-speech",
         "endpoint": "/v1/audio/speech",
         "dataset_path": "benchmarks/build_dataset/seed_tts_design",
-        "num_prompts": [100],
+        "num_prompts": [
-        "request_rate": [2.0],
+          100
+        ],
+        "request_rate": [
+          2.0
+        ],
         "seed_tts_locale": "en",
-        "extra_body": {"task_type": "VoiceDesign", "language": "English"},
+        "extra_body": {
+          "task_type": "VoiceDesign",
+          "language": "English"
+        },
         "percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
         "baseline": {
-          "median_audio_ttfp_ms": [3500],
+          "median_audio_ttfp_ms": [
-          "median_audio_rtf": [0.30],
+            100
-          "audio_throughput": [4.0]
+          ],
+          "median_audio_rtf": [
+            0.20
+          ],
+          "audio_throughput": [
+            9
+          ]
         }
       }
     ]

tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json

@@ -30,9 +30,9 @@
                     }
                 ],
                 "baseline": {
-                    "throughput_qps": 0.034,
+                    "throughput_qps": 0.04,
-                    "latency_mean": 26.0,
+                    "latency_mean": 30,
-                    "peak_memory_mb_mean": 80000
+                    "peak_memory_mb_mean": 75000
                 }
             }
         ]
@@ -72,9 +72,9 @@
                     }
                 ],
                 "baseline": {
-                    "throughput_qps": 0.042,
+                    "throughput_qps": 0.06,
-                    "latency_mean": 21.6,
+                    "latency_mean": 20,
-                    "peak_memory_mb_mean": 55300
+                    "peak_memory_mb_mean": 48000
                 }
             },
             {
@@ -97,9 +97,9 @@
                     }
                 ],
                 "baseline": {
-                    "throughput_qps": 0.0085,
+                    "throughput_qps": 0.01,
-                    "latency_mean": 101.6,
+                    "latency_mean": 90,
-                    "peak_memory_mb_mean": 65200
+                    "peak_memory_mb_mean": 56000
                 }
             }
         ]

tests/diffusion/distributed/test_autoencoder_kl_hunyuan.py

@@ -0,0 +1,79 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+import pytest
+import torch
+
+from vllm_omni.diffusion.distributed.autoencoders.autoencoder_kl_hunyuan import (
+    DistributedAutoencoderKLHunyuan,
+)
+
+pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
+
+
+class _DummyDistributedAutoencoderKLHunyuan(DistributedAutoencoderKLHunyuan):
+    def __init__(self):
+        torch.nn.Module.__init__(self)
+        self.tile_latent_min_size = 8
+        self.tile_sample_min_size = 8
+        self.tile_overlap_factor = 0.25
+        self.use_spatial_tiling = False
+
+    @property
+    def dtype(self) -> torch.dtype:
+        return torch.float32
+
+    def decoder(self, x: torch.Tensor) -> torch.Tensor:
+        return x + 10
+
+    def encoder(self, x: torch.Tensor) -> torch.Tensor:
+        return x + 20
+
+    def blend_v(self, _a: torch.Tensor, b: torch.Tensor, _blend_extent: int) -> torch.Tensor:
+        return b
+
+    def blend_h(self, _a: torch.Tensor, b: torch.Tensor, _blend_extent: int) -> torch.Tensor:
+        return b
+
+
+def test_hunyuan_vae_use_tiling_aliases_spatial_tiling():
+    # Verify use_tiling property maps to use_spatial_tiling.
+    vae = _DummyDistributedAutoencoderKLHunyuan()
+
+    assert not vae.use_tiling
+
+    vae.use_tiling = True
+
+    assert vae.use_spatial_tiling
+
+
+def test_hunyuan_vae_decode_tiles_round_trip():
+    # Validate decode tile split/exec/merge returns expected reconstructed tensor.
+    vae = _DummyDistributedAutoencoderKLHunyuan()
+    z = torch.arange(144, dtype=torch.float32).reshape(1, 1, 1, 12, 12)
+
+    tile_tasks, grid_spec = vae.tile_split(z)
+    decoded_tiles = {task.grid_coord: vae.tile_exec(task) for task in tile_tasks}
+    output = vae.tile_merge(decoded_tiles, grid_spec)
+
+    assert grid_spec.split_dims == (3, 4)
+    assert grid_spec.grid_shape == (2, 2)
+    assert grid_spec.tile_spec == {"blend_extent": 2, "row_limit": 6}
+    assert len(tile_tasks) == 4
+    assert torch.equal(output, z + 10)
+
+
+def test_hunyuan_vae_encode_tiles_round_trip():
+    # Validate encode tile split/exec/merge returns expected latent tensor.
+    vae = _DummyDistributedAutoencoderKLHunyuan()
+    x = torch.arange(144, dtype=torch.float32).reshape(1, 1, 1, 12, 12)
+
+    tile_tasks, grid_spec = vae.encode_tile_split(x)
+    encoded_tiles = {task.grid_coord: vae.encode_tile_exec(task) for task in tile_tasks}
+    output = vae.encode_tile_merge(encoded_tiles, grid_spec)
+
+    assert grid_spec.split_dims == (3, 4)
+    assert grid_spec.grid_shape == (2, 2)
+    assert grid_spec.tile_spec == {"blend_extent": 2, "row_limit": 6}
+    assert len(tile_tasks) == 4
+    assert torch.equal(output, x + 20)

vllm_omni/diffusion/distributed/autoencoders/autoencoder_kl_hunyuan.py

@@ -0,0 +1,145 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+from typing import Any
+
+import torch
+from vllm.logger import init_logger
+
+from vllm_omni.diffusion.distributed.autoencoders.distributed_vae_executor import (
+    DistributedOperator,
+    DistributedVaeMixin,
+    GridSpec,
+    TileTask,
+)
+from vllm_omni.diffusion.models.hunyuan_image3.autoencoder import AutoencoderKLConv3D
+
+logger = init_logger(__name__)
+
+
+class DistributedAutoencoderKLHunyuan(AutoencoderKLConv3D, DistributedVaeMixin):
+    @classmethod
+    def from_config(cls, config: Any, **kwargs: Any):
+        model = super().from_config(config, **kwargs)
+        model.init_distributed()
+        return model
+
+    @classmethod
+    def from_pretrained(cls, *args: Any, **kwargs: Any):
+        model = super().from_pretrained(*args, **kwargs)
+        model.init_distributed()
+        return model
+
+    @property
+    def use_tiling(self) -> bool:
+        return self.use_spatial_tiling
+
+    @use_tiling.setter
+    def use_tiling(self, use_tiling: bool) -> None:
+        self.use_spatial_tiling = use_tiling
+
+    def tile_split(self, z: torch.Tensor) -> tuple[list[TileTask], GridSpec]:
+        _, _, _, height, width = z.shape
+        overlap_size = int(self.tile_latent_min_size * (1 - self.tile_overlap_factor))
+        blend_extent = int(self.tile_sample_min_size * self.tile_overlap_factor)
+        row_limit = int(self.tile_sample_min_size - blend_extent)
+
+        tiletask_list = []
+        for i in range(0, height, overlap_size):
+            for j in range(0, width, overlap_size):
+                tile = z[:, :, :, i : i + self.tile_latent_min_size, j : j + self.tile_latent_min_size]
+                tiletask_list.append(
+                    TileTask(
+                        len(tiletask_list),
+                        (i // overlap_size, j // overlap_size),
+                        tile,
+                        workload=tile.shape[3] * tile.shape[4],
+                    )
+                )
+
+        grid_spec = GridSpec(
+            split_dims=(3, 4),
+            grid_shape=(tiletask_list[-1].grid_coord[0] + 1, tiletask_list[-1].grid_coord[1] + 1),
+            tile_spec={"blend_extent": blend_extent, "row_limit": row_limit},
+            output_dtype=self.dtype,
+        )
+        return tiletask_list, grid_spec
+
+    def tile_exec(self, task: TileTask) -> torch.Tensor:
+        return self.decoder(task.tensor)
+
+    def tile_merge(self, coord_tensor_map: dict[tuple[int, ...], torch.Tensor], grid_spec: GridSpec) -> torch.Tensor:
+        grid_h, grid_w = grid_spec.grid_shape
+        result_rows = []
+        for i in range(grid_h):
+            result_row = []
+            for j in range(grid_w):
+                tile = coord_tensor_map[(i, j)]
+                if i > 0:
+                    tile = self.blend_v(coord_tensor_map[(i - 1, j)], tile, grid_spec.tile_spec["blend_extent"])
+                if j > 0:
+                    tile = self.blend_h(coord_tensor_map[(i, j - 1)], tile, grid_spec.tile_spec["blend_extent"])
+                result_row.append(tile[:, :, :, : grid_spec.tile_spec["row_limit"], : grid_spec.tile_spec["row_limit"]])
+            result_rows.append(torch.cat(result_row, dim=-1))
+        return torch.cat(result_rows, dim=-2)
+
+    def encode_tile_split(self, x: torch.Tensor) -> tuple[list[TileTask], GridSpec]:
+        _, _, _, height, width = x.shape
+        overlap_size = int(self.tile_sample_min_size * (1 - self.tile_overlap_factor))
+        blend_extent = int(self.tile_latent_min_size * self.tile_overlap_factor)
+        row_limit = int(self.tile_latent_min_size - blend_extent)
+
+        tiletask_list = []
+        for i in range(0, height, overlap_size):
+            for j in range(0, width, overlap_size):
+                tile = x[:, :, :, i : i + self.tile_sample_min_size, j : j + self.tile_sample_min_size]
+                tiletask_list.append(
+                    TileTask(
+                        len(tiletask_list),
+                        (i // overlap_size, j // overlap_size),
+                        tile,
+                        workload=tile.shape[3] * tile.shape[4],
+                    )
+                )
+
+        grid_spec = GridSpec(
+            split_dims=(3, 4),
+            grid_shape=(tiletask_list[-1].grid_coord[0] + 1, tiletask_list[-1].grid_coord[1] + 1),
+            tile_spec={"blend_extent": blend_extent, "row_limit": row_limit},
+            output_dtype=self.dtype,
+        )
+        return tiletask_list, grid_spec
+
+    def encode_tile_exec(self, task: TileTask) -> torch.Tensor:
+        return self.encoder(task.tensor)
+
+    def encode_tile_merge(
+        self, coord_tensor_map: dict[tuple[int, ...], torch.Tensor], grid_spec: GridSpec
+    ) -> torch.Tensor:
+        return self.tile_merge(coord_tensor_map, grid_spec)
+
+    def spatial_tiled_encode(self, x: torch.Tensor):
+        if not self.is_distributed_enabled():
+            return super().spatial_tiled_encode(x)
+
+        logger.debug("Encode running with distributed executor")
+        return self.distributed_executor.execute(
+            x,
+            DistributedOperator(
+                split=self.encode_tile_split,
+                exec=self.encode_tile_exec,
+                merge=self.encode_tile_merge,
+            ),
+            broadcast_result=True,
+        )
+
+    def spatial_tiled_decode(self, z: torch.Tensor):
+        if not self.is_distributed_enabled():
+            return super().spatial_tiled_decode(z)
+
+        logger.debug("Decode running with distributed executor")
+        return self.distributed_executor.execute(
+            z,
+            DistributedOperator(split=self.tile_split, exec=self.tile_exec, merge=self.tile_merge),
+            broadcast_result=True,
+        )

vllm_omni/diffusion/models/hunyuan_image3/pipeline_hunyuan_image3.py

@@ -30,7 +30,6 @@ from vllm_omni.diffusion.request import OmniDiffusionRequest
 from vllm_omni.inputs.data import OmniTextPrompt
 from vllm_omni.model_executor.models.hunyuan_image3.siglip2 import Siglip2VisionTransformer
 
-from .autoencoder import AutoencoderKLConv3D
 from .hunyuan_image3_tokenizer import TokenizerWrapper
 from .hunyuan_image3_transformer import (
     CausalMMOutputWithPast,
@@ -343,7 +342,13 @@ class HunyuanImage3Pipeline(
         quant_config = od_config.quantization_config
         self.model = HunyuanImage3Model(self.hf_config, quant_config=quant_config)
         self.transformer = self.model
-        self.vae = AutoencoderKLConv3D.from_config(self.hf_config.vae)
+        # Lazy import to break circular dependency:
+        # autoencoder_kl_hunyuan -> hunyuan_image3/__init__ -> pipeline_hunyuan_image3 -> autoencoder_kl_hunyuan
+        from vllm_omni.diffusion.distributed.autoencoders.autoencoder_kl_hunyuan import (  # noqa: PLC0415
+            DistributedAutoencoderKLHunyuan,
+        )
+
+        self.vae = DistributedAutoencoderKLHunyuan.from_config(self.hf_config.vae)
         self.vae.use_spatial_tiling = self.od_config.vae_use_tiling
         self._pipeline = None
         self._tkwrapper = TokenizerWrapper(od_config.model)