anodyine/vllm-omni
1
0 Fork 0
Code Issues 4 Pull Requests 2 Actions Packages Projects Releases Wiki Activity

[CI]For NPU CI, Nest perf baselines by hardware label (H100/A3) (#5402)

Browse Source 
Signed-off-by: wangyu <410167048@qq.com>
This commit is contained in:
wangyu authored and GitHub committed 2026-08-05 17:35:41 +08:00
1 parent 6510de344e
commit 7fb62db5c2
26 files changed
+675 -806
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

docs/contributing/ci/test_examples/l4_performance_tests.inc.md
+6 -8
View File
@@ -44,9 +44,11 @@ Pass **`--test-config-file`** to run one JSON file as-is, or omit it for the bul
"ignore_eos": true,
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
"baseline": {
"mean_ttft_ms": [500, 800],
"mean_audio_ttfp_ms": [2000, 3500],
"mean_audio_rtf": [0.25, 0.35]
"H100": {
"mean_ttft_ms": [500, 800],
"mean_audio_ttfp_ms": [2000, 3500],
"mean_audio_rtf": [0.25, 0.35]
}
}
}
]
@@ -142,10 +144,6 @@ pytest -s -v tests/dfx/perf/scripts/run_diffusion_benchmark.py \
--test-config-file tests/dfx/perf/tests/test_bagel_vllm_omni.json
pytest -s -v tests/dfx/perf/scripts/run_benchmark.py \
--test-config-file tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json
# Optional baseline assertion (default off)
pytest -sv tests/dfx/perf/scripts/run_diffusion_benchmark.py --assert-baseline \
--test-config-file tests/dfx/perf/tests/test_qwen_image_vllm_omni.json
```
See also [Markers for Tests](#markers-for-tests) for registered hardware markers (`H100`, `L4`, `cuda`, `distributed_cuda`, …).
@@ -202,4 +200,4 @@ You can add any benchmark running parameters you need here. For all optional par
| num_prompts | int / array | Yes | 10,[10, 20, 30] | Number of requests. Supports single values or arrays. If a single value is used, it will be automatically expanded to match the number of qps or max_concurrency, e.g., [10,10,10]. If an array is used, its length must match the number of qps or max_concurrency. |
| request_rate | float / array | No | 0.5, [0.5, 1, inf] | Queries per second. Supports single values or arrays. If a single value is used, it will be automatically expanded to match the number of num_prompts, e.g., [1,1,1]. If an array is used, its length must match the number of num_prompts. |
| max_concurrency | int / array | No | 1, [1, 2, 3] | Maximum concurrent in-flight requests. Same array / expansion rules as `request_rate` (mutually exclusive with QPS mode). |
| baseline | object | No | see above | Optional per-metric thresholds; keys must match benchmark output fields. Scalar, list (per sweep step), or object (keyed by concurrency or QPS string).
| baseline | object | No | see below | Optional thresholds. Prefer hardware-nested form: `{"H100": {"throughput_qps": …}, "A3": {…}}` (selected via runtime hardware label). Legacy flat `{metric: threshold}` still works. Per metric: scalar, list (sweep step), or object keyed by concurrency / QPS.
recipes/Tencent/HunyuanImage-3.0-Instruct.md
+5 -4
View File
@@ -164,10 +164,11 @@ curl -s http://localhost:8091/v1/chat/completions \
PR [#2495](https://github.com/vllm-project/vllm-omni/pull/2495) adds
performance CI configs for the same DiT-only settings. The CI step is
currently opt-in (gated by `RUN_HUNYUAN_IMAGE3_PERF=1`) with `soft_fail`
enabled, intended for initial data collection. Performance assertions are
skipped (`skip-performance-assertion: true`); the baseline values in the
JSON configs are reference-only and will be promoted to regression gates
once enough nightly data has been collected.
enabled, intended for initial data collection. Perf runners no longer
compare metrics against JSON baselines at assert time; the `baseline`
blocks in the configs are reference-only and are copied into result JSON
for reporting. Regression gates can be reintroduced once enough nightly
data has been collected.
The user-facing equivalent is to launch one of the CLI commands above and
generate 1024x1024 images with 50 denoising steps.
tests/dfx/conftest.py
+8 -146
View File
@@ -1,16 +1,15 @@
import copy
import json
import os
import re
import subprocess
import threading
import time
from datetime import datetime
from pathlib import Path
from typing import Any
import pytest
from tests.dfx.reliability.helpers import list_remote_process_pids_by_pattern, post_chat_completions_raw
from tests.helpers.mark import hardware_marks
from tests.helpers.runtime import OmniServerParams
from tests.helpers.stage_config import modify_stage_config
@@ -119,32 +118,6 @@ def create_unique_server_pytest_params(
]
def create_benchmark_pytest_params(
benchmark_configs: list[dict[str, Any]],
server_to_benchmark_mapping: dict[str, dict],
) -> list[Any]:
"""Like :func:`create_benchmark_indices`, but wrap each index in ``pytest.param`` with JSON marks."""
marks_by_name = _marks_by_test_name(benchmark_configs)
params: list[Any] = []
seen: set[str] = set()
for config in benchmark_configs:
test_name = config["test_name"]
if test_name in seen:
continue
seen.add(test_name)
params_list = get_benchmark_params_for_server(test_name, server_to_benchmark_mapping)
id_suffixes = _unique_benchmark_param_id_suffixes(params_list)
for idx, id_suffix in enumerate(id_suffixes):
params.append(
pytest.param(
(test_name, idx),
marks=marks_by_name.get(test_name, []),
id=f"{test_name}-{id_suffix}",
)
)
return params
def create_paired_benchmark_pytest_params(
server_entries: list[tuple[Any, str]],
params_by_test_name: dict[str, list[Any]],
@@ -357,10 +330,6 @@ def supports_video_generation(model_name: str) -> bool:
return any(key in lower for key in ("wan", "video", "i2v", "t2v"))
def supports_chat_generation(model_name: str) -> bool:
return not supports_video_generation(model_name)
def parse_stage_devices(stage_config_path: str) -> str:
text = Path(stage_config_path).read_text(encoding="utf-8")
raw_devices: list[str] = re.findall(r"^\s*devices:\s*\"?([0-9,\s]+)\"?\s*$", text, flags=re.MULTILINE)
@@ -389,48 +358,6 @@ def assert_fault_exception(exc: Exception, error_keywords: tuple[str, ...]) -> N
assert any(key in text for key in error_keywords), f"unexpected error under fault injection: {exc}"
def wait_chat_request_ready(host: str, port: int, model: str, timeout_sec: int = 180) -> None:
"""Poll a minimal chat request until success."""
deadline = time.time() + timeout_sec
payload = json.dumps(
{
"model": model,
"messages": [{"role": "user", "content": "Say hello in one short sentence."}],
"stream": False,
"modalities": ["text"],
}
)
last_error: str | None = None
while time.time() < deadline:
try:
status, body = post_chat_completions_raw(host, port, payload)
if status == 200:
return
last_error = f"http={status} body={body[:200]!r}"
except Exception as exc: # noqa: BLE001
last_error = str(exc)
time.sleep(2)
raise TimeoutError(f"runtime-teardown warmup request did not succeed within {timeout_sec}s: {last_error}")
def assert_no_extra_worker_processes(
baseline_pids: set[int],
worker_pattern: str,
timeout_sec: int = 60,
) -> None:
"""Ensure no extra worker PIDs remain after teardown."""
deadline = time.time() + timeout_sec
while time.time() < deadline:
current = set(list_remote_process_pids_by_pattern(worker_pattern))
extra = current - baseline_pids
if not extra:
return
time.sleep(2)
current = set(list_remote_process_pids_by_pattern(worker_pattern))
extra = sorted(current - baseline_pids)
assert not extra, f"orphan worker processes remain after container teardown: {extra}"
def create_test_parameter_mapping(configs: list[dict[str, Any]]) -> dict[str, dict]:
mapping = {}
for config in configs:
@@ -637,57 +564,6 @@ def extract_configs_resource_label(configs: list[dict[str, Any]]) -> str:
return get_runtime_resource_label()
def resolve_baseline_value(
baseline_raw: Any,
*,
sweep_index: int | None,
max_concurrency: Any = None,
request_rate: Any = None,
) -> Any:
"""Pick the baseline threshold for this sweep step."""
if baseline_raw is None:
return 100000
if isinstance(baseline_raw, dict):
if max_concurrency is not None:
for key in (max_concurrency, str(max_concurrency)):
if key in baseline_raw:
return baseline_raw[key]
if request_rate is not None:
for key in (request_rate, str(request_rate)):
if key in baseline_raw:
return baseline_raw[key]
raise KeyError(
f"baseline dict has no key for max_concurrency={max_concurrency!r} "
f"or request_rate={request_rate!r}; keys={list(baseline_raw.keys())!r}"
)
if isinstance(baseline_raw, (list, tuple)):
if sweep_index is None:
raise ValueError("list baseline requires sweep_index")
if not (0 <= sweep_index < len(baseline_raw)):
raise IndexError(f"baseline list len={len(baseline_raw)} has no index {sweep_index}")
return baseline_raw[sweep_index]
return baseline_raw
def _baseline_thresholds_for_step(
baseline_data: dict[str, Any],
*,
sweep_index: int | None = None,
max_concurrency: Any = None,
request_rate: Any = None,
) -> dict[str, Any]:
"""Resolve baseline config to one threshold per metric for this iteration."""
return {
metric_name: resolve_baseline_value(
baseline_raw,
sweep_index=sweep_index,
max_concurrency=max_concurrency,
request_rate=request_rate,
)
for metric_name, baseline_raw in baseline_data.items()
}
def run_benchmark(
args: list[str],
test_name: str,
@@ -696,18 +572,17 @@ def run_benchmark(
num_prompt: int,
*,
baseline_config: dict[str, Any] | None = None,
sweep_index: int | None = None,
request_rate: Any | None = None,
max_concurrency: Any | None = None,
random_input_len: Any | None = None,
random_output_len: Any | None = None,
resource_label: str | None = None,
) -> dict[str, Any]:
"""Run one ``vllm bench serve --omni`` iteration and return parsed metrics.
After ``vllm bench`` writes the JSON, ``result["baseline"]`` holds the resolved per-metric thresholds
(when ``baseline_config`` is provided). If the benchmark exits without writing a result file,
``result_omni_template.json`` is used as a fallback.
After ``vllm bench`` writes the JSON, ``result["baseline"]`` stores the raw
``baseline`` object from the test config (when ``baseline_config`` is provided),
including any hardware-nested map (e.g. ``{"H100": {...}}``). Downstream report
tooling can select a hardware bucket when needed. If the benchmark exits without
writing a result file, ``result_omni_template.json`` is used as a fallback.
"""
current_dt = datetime.now().strftime("%Y%m%d-%H%M%S")
ri = _safe_filename_token(random_input_len)
@@ -774,12 +649,8 @@ def run_benchmark(
result = json.load(f)
if baseline_config:
result["baseline"] = _baseline_thresholds_for_step(
baseline_config,
sweep_index=sweep_index,
request_rate=request_rate,
max_concurrency=max_concurrency,
)
# Persist the JSON baseline as-is (no hardware / sweep resolution).
result["baseline"] = copy.deepcopy(baseline_config)
else:
result["baseline"] = {}
if random_input_len is not None:
@@ -800,12 +671,3 @@ def pytest_addoption(parser: pytest.Parser) -> None:
default=None,
help=("Path to benchmark config JSON. Example: --test-config-file tests/dfx/perf/tests/test_tts.json"),
)
parser.addoption(
"--assert-baseline",
action="store_true",
default=False,
help=(
"When set, omni/diffusion perf runners compare metrics against the baseline block in the JSON config "
"(default: off)."
),
)
tests/dfx/perf/scripts/run_benchmark.py
+6 -60
View File
@@ -15,15 +15,10 @@ from tests.dfx.conftest import (
get_runtime_resource_label,
is_diffusion_perf_config,
load_benchmark_configs,
resolve_baseline_value,
run_benchmark,
)
from tests.helpers.runtime import OmniServer
# Compare metrics to each test JSON ``baseline`` block only when pytest is run with ``--assert-baseline``
# (registered in ``tests/dfx/conftest.py``; default: off). ``run_benchmark`` and ``_resolve_baseline_value`` are
# defined in the same module.
#
# Optional JSON field ``mark`` is applied as pytest marks via
# ``create_paired_omni_benchmark_pytest_params`` (e.g. ``"mark": [{"hardware_marks":
# {"res": {"cuda": "H100"}, "num_cards": 2}}, "full_model", "omni"]``).
@@ -162,36 +157,8 @@ def benchmark_params(request):
}
def assert_result(
result,
params,
num_prompt,
*,
assert_baseline: bool,
sweep_index: int | None = None,
max_concurrency: Any | None = None,
request_rate: Any | None = None,
) -> None:
def assert_result(result, num_prompt) -> None:
assert result["completed"] == num_prompt, "Request failures exist"
if not assert_baseline:
return
baseline_data = params.get("baseline", {})
for metric_name, baseline_raw in baseline_data.items():
current_value = result[metric_name]
baseline_value = resolve_baseline_value(
baseline_raw,
sweep_index=sweep_index,
max_concurrency=max_concurrency,
request_rate=request_rate,
)
if "throughput" in metric_name:
if current_value <= baseline_value:
print(
f"ERROR: Throughput test results were below baseline: {metric_name}: {current_value} > {baseline_value}"
)
else:
if current_value >= baseline_value:
print(f"ERROR: Test results exceeded baseline: {metric_name}: {current_value} < {baseline_value}")
@pytest.mark.benchmark
@@ -200,7 +167,7 @@ def assert_result(
paired_benchmark_params,
indirect=["omni_server", "benchmark_params"],
)
def test_performance_benchmark(omni_server, benchmark_params, request):
def test_performance_benchmark(omni_server, benchmark_params):
test_name = benchmark_params["test_name"]
params = benchmark_params["params"]
dataset_name = params.get("dataset_name", "")
@@ -212,7 +179,6 @@ def test_performance_benchmark(omni_server, benchmark_params, request):
print(f"Running benchmark for model: {model}")
print(f"Benchmark parameters: {benchmark_params}")
assert_baseline = request.config.getoption("--assert-baseline", default=False)
resource_label = get_runtime_resource_label()
def to_list(value, default=None):
@@ -271,7 +237,7 @@ def test_performance_benchmark(omni_server, benchmark_params, request):
break
# QPS / request-rate sweep
for i, (qps, num_prompt) in enumerate(zip(qps_list, num_prompt_list)):
for qps, num_prompt in zip(qps_list, num_prompt_list):
args = args + ["--request-rate", str(qps), "--num-prompts", str(num_prompt)]
result = run_benchmark(
args=args,
@@ -280,24 +246,14 @@ def test_performance_benchmark(omni_server, benchmark_params, request):
dataset_name=dataset_name,
num_prompt=num_prompt,
baseline_config=params.get("baseline"),
sweep_index=i,
request_rate=qps,
max_concurrency=None,
random_input_len=params.get("random_input_len"),
random_output_len=params.get("random_output_len"),
resource_label=resource_label,
)
assert_result(
result,
params,
num_prompt,
assert_baseline=assert_baseline,
sweep_index=i,
request_rate=qps,
)
assert_result(result, num_prompt)
# concurrency test
for i, (concurrency, num_prompt) in enumerate(zip(max_concurrency_list, num_prompt_list)):
for concurrency, num_prompt in zip(max_concurrency_list, num_prompt_list):
args = args + ["--max-concurrency", str(concurrency), "--num-prompts", str(num_prompt), "--request-rate", "inf"]
result = run_benchmark(
args=args,
@@ -306,18 +262,8 @@ def test_performance_benchmark(omni_server, benchmark_params, request):
dataset_name=dataset_name,
num_prompt=num_prompt,
baseline_config=params.get("baseline"),
sweep_index=i,
request_rate=None,
max_concurrency=concurrency,
random_input_len=params.get("random_input_len"),
random_output_len=params.get("random_output_len"),
resource_label=resource_label,
)
assert_result(
result,
params,
num_prompt,
assert_baseline=assert_baseline,
sweep_index=i,
max_concurrency=concurrency,
)
assert_result(result, num_prompt)
tests/dfx/perf/scripts/run_diffusion_benchmark.py
+12 -103
View File
@@ -13,8 +13,6 @@ A config JSON file may be passed via --test-config-file. If omitted, every ``*.j
pytest run_diffusion_benchmark.py -m "diffusion"
pytest run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_vllm_omni.json
Optional: ``--assert-baseline`` compares metrics to the ``baseline`` block in each benchmark entry (default: off).
Optional JSON field ``mark`` is applied as pytest marks on that case via
``pytest.param`` (e.g. ``"mark": [{"hardware_marks": {"res": {"cuda": "H100"}, "num_cards": 1}}, "full_model", "diffusion"]``).
@@ -25,6 +23,7 @@ of all runs from cases in that file). Bulk load without ``--test-config-file`` u
the same per-file aggregation; ``-m`` only selects which cases run.
"""
import copy
import json
import os
import socket
@@ -274,11 +273,6 @@ def _write_result_record(record: dict[str, Any]) -> Path:
return target
def _append_to_aggregated_file(record: dict[str, Any]) -> None:
"""Backward-compatible wrapper; prefer :func:`_write_result_record`."""
_write_result_record(record)
_server_lock = threading.Lock()
# ---------------------------------------------------------------------------
@@ -678,7 +672,7 @@ def run_benchmark(
with tempfile.NamedTemporaryFile(mode="w", suffix=".json", prefix="diffusion_bench_tmp_", delete=False) as tmp:
tmp_result_file = Path(tmp.name)
exclude_keys = {"baseline", "dataset", "task", "name", "skip-performance-assertion"}
exclude_keys = {"baseline", "dataset", "task", "name"}
cmd = [
sys.executable,
@@ -772,12 +766,17 @@ def run_benchmark(
completed = metrics.get("completed_requests", metrics.get("completed", 0))
failed = metrics.get("failed_requests", metrics.get("failed", 0))
# Persist the JSON baseline as-is (no hardware / sweep resolution).
baseline = copy.deepcopy(params.get("baseline") or {})
metrics["baseline"] = baseline
record: dict[str, Any] = {
"test_name": test_name,
"endpoint": endpoint,
"timestamp": timestamp,
"server_params": server_cfg.get("server_params"),
"benchmark_params": params,
"baseline": baseline,
"result": metrics,
"log_file": str(log_file),
"Model": model,
@@ -827,70 +826,14 @@ def run_benchmark(
# ---------------------------------------------------------------------------
def _resolve_baseline_value(
baseline_raw: Any,
*,
sweep_index: int | None,
max_concurrency: Any = None,
request_rate: Any = None,
) -> Any:
"""Pick the baseline threshold for this sweep step."""
if baseline_raw is None:
return 100000
if isinstance(baseline_raw, dict):
if max_concurrency is not None:
for key in (max_concurrency, str(max_concurrency)):
if key in baseline_raw:
return baseline_raw[key]
if request_rate is not None:
for key in (request_rate, str(request_rate)):
if key in baseline_raw:
return baseline_raw[key]
raise KeyError(
f"baseline dict has no key for max_concurrency={max_concurrency!r} "
f"or request_rate={request_rate!r}; keys={list(baseline_raw.keys())!r}"
)
if isinstance(baseline_raw, (list, tuple)):
if sweep_index is None:
raise ValueError("sweep_index is required when baseline is a list or tuple")
return baseline_raw[sweep_index]
return baseline_raw
def assert_result(
result: dict[str, Any],
params: dict[str, Any],
num_prompts: int,
*,
sweep_index: int | None = None,
max_concurrency: Any = None,
request_rate: Any = None,
assert_baseline: bool = True,
) -> None:
"""Assert that benchmark metrics satisfy the configured baselines."""
"""Assert that the expected number of requests completed."""
completed = result.get("completed_requests", result.get("completed", 0))
assert completed == num_prompts, f"Expected {num_prompts} completed requests, got {completed}"
if not assert_baseline:
return
if params.get("skip-performance-assertion", False):
print("Skipping performance assertions.")
return
for metric, baseline_raw in params.get("baseline", {}).items():
current = result.get(metric)
assert current is not None, f"Metric '{metric}' not found in result: {list(result.keys())}"
threshold = _resolve_baseline_value(
baseline_raw,
sweep_index=sweep_index,
max_concurrency=max_concurrency,
request_rate=request_rate,
)
if "throughput" in metric:
assert current >= threshold, f"{metric}: {current:.4f} < baseline {threshold}"
else:
assert current <= threshold, f"{metric}: {current:.4f} > baseline {threshold}"
def _default_benchmark_endpoint_for_task(task: str) -> str:
"""Return the default client-side benchmark endpoint for a diffusion task."""
@@ -919,7 +862,6 @@ def _build_run_params(
params: dict[str, Any],
*,
num_prompts: int,
sweep_index: int | None = None,
request_rate: Any | None = None,
max_concurrency: Any | None = None,
) -> dict[str, Any]:
@@ -931,16 +873,6 @@ def _build_run_params(
run_params["request-rate"] = request_rate
if max_concurrency is not None:
run_params["max-concurrency"] = max_concurrency
if "baseline" in params:
run_params["baseline"] = {
metric: _resolve_baseline_value(
baseline_raw,
sweep_index=sweep_index,
max_concurrency=max_concurrency,
request_rate=request_rate,
)
for metric, baseline_raw in params["baseline"].items()
}
return run_params
@@ -963,36 +895,28 @@ def _iter_sweep_runs(params: dict[str, Any]) -> list[dict[str, Any]]:
sweep_runs: list[dict[str, Any]] = []
for i, (request_rate, num_prompts) in enumerate(zip(request_rate_list, num_prompt_list)):
for request_rate, num_prompts in zip(request_rate_list, num_prompt_list):
sweep_runs.append(
{
"params": _build_run_params(
params,
request_rate=request_rate,
num_prompts=num_prompts,
sweep_index=i,
),
"num_prompts": num_prompts,
"sweep_index": i,
"request_rate": request_rate,
"max_concurrency": None,
}
)
for i, (max_concurrency, num_prompts) in enumerate(zip(max_concurrency_list, num_prompt_list)):
for max_concurrency, num_prompts in zip(max_concurrency_list, num_prompt_list):
sweep_runs.append(
{
"params": _build_run_params(
params,
max_concurrency=max_concurrency,
num_prompts=num_prompts,
sweep_index=i,
request_rate="inf",
),
"num_prompts": num_prompts,
"sweep_index": i,
"request_rate": None,
"max_concurrency": max_concurrency,
}
)
@@ -1005,9 +929,6 @@ def _iter_sweep_runs(params: dict[str, Any]) -> list[dict[str, Any]]:
num_prompts=default_num_prompts,
),
"num_prompts": default_num_prompts,
"sweep_index": None,
"request_rate": None,
"max_concurrency": None,
}
)
@@ -1023,14 +944,12 @@ def _iter_sweep_runs(params: dict[str, Any]) -> list[dict[str, Any]]:
paired_benchmark_params,
indirect=["diffusion_server", "benchmark_params"],
)
def test_diffusion_performance_benchmark(diffusion_server, benchmark_params, request):
def test_diffusion_performance_benchmark(diffusion_server, benchmark_params):
"""Run the diffusion performance benchmark and verify request completion.
One server is started per unique parallel configuration (module scope).
For each server, all benchmark parameter sets defined in the config JSON
are executed sequentially; metrics are recorded to the aggregated result file.
Pass ``--assert-baseline`` to compare ``throughput_qps``, ``latency_mean``, etc. to the JSON ``baseline`` block.
"""
test_name = benchmark_params["test_name"]
params = benchmark_params["params"]
@@ -1073,14 +992,4 @@ def test_diffusion_performance_benchmark(diffusion_server, benchmark_params, req
print(f"\n Aggregated results: {_aggregated_result_file_for_source(str(source))}")
print("=" * 60)
assert_baseline = request.config.getoption("--assert-baseline", default=False)
assert_result(
result,
params,
sweep_run["num_prompts"],
sweep_index=sweep_run["sweep_index"],
max_concurrency=sweep_run["max_concurrency"],
request_rate=sweep_run["request_rate"],
assert_baseline=assert_baseline,
)
assert_result(result, sweep_run["num_prompts"])
tests/dfx/perf/tests/test_bagel_vllm_omni.json
+24 -16
View File
@@ -40,10 +40,12 @@
"max-concurrency": 1,
"enable-negative-prompt": true,
"baseline": {
"throughput_qps": 0.495,
"latency_mean": 2.0238,
"peak_memory_mb_max": 29393.4286,
"peak_memory_mb_mean": 29392.7143
"H100": {
"throughput_qps": 0.495,
"latency_mean": 2.0238,
"peak_memory_mb_max": 29393.4286,
"peak_memory_mb_mean": 29392.7143
}
}
}
]
@@ -90,10 +92,12 @@
"enable-negative-prompt": true,
"num-input-images": 1,
"baseline": {
"throughput_qps": 0.3451,
"latency_mean": 2.9031,
"peak_memory_mb_max": 30480.2857,
"peak_memory_mb_mean": 30479.2857
"H100": {
"throughput_qps": 0.3451,
"latency_mean": 2.9031,
"peak_memory_mb_max": 30480.2857,
"peak_memory_mb_mean": 30479.2857
}
}
}
]
@@ -139,10 +143,12 @@
"max-concurrency": 1,
"enable-negative-prompt": true,
"baseline": {
"throughput_qps": 0.5048,
"latency_mean": 1.987,
"peak_memory_mb_max": 29380.0,
"peak_memory_mb_mean": 29380.0
"H100": {
"throughput_qps": 0.5048,
"latency_mean": 1.987,
"peak_memory_mb_max": 29380.0,
"peak_memory_mb_mean": 29380.0
}
}
}
]
@@ -189,10 +195,12 @@
"enable-negative-prompt": true,
"num-input-images": 1,
"baseline": {
"throughput_qps": 0.149,
"latency_mean": 6.7157,
"peak_memory_mb_max": 30574.0,
"peak_memory_mb_mean": 30574.0
"H100": {
"throughput_qps": 0.149,
"latency_mean": 6.7157,
"peak_memory_mb_max": 30574.0,
"peak_memory_mb_mean": 30574.0
}
}
}
]
tests/dfx/perf/tests/test_boogu_image_edit_vllm_omni.json
+6 -4
View File
@@ -24,10 +24,12 @@
"warmup-num-inference-steps": 28,
"enable-negative-prompt": true,
"baseline": {
"throughput_qps": {"1": 0.0303, "2": 0.0302, "4": 0.0302},
"latency_mean": {"1": 32.6, "2": 62.2, "4": 111.7},
"peak_memory_mb_max": 40500,
"peak_memory_mb_mean": 40500
"H100": {
"throughput_qps": {"1": 0.0303, "2": 0.0302, "4": 0.0302},
"latency_mean": {"1": 32.6, "2": 62.2, "4": 111.7},
"peak_memory_mb_max": 40500,
"peak_memory_mb_mean": 40500
}
}
}
]
tests/dfx/perf/tests/test_boogu_image_vllm_omni.json
+6 -4
View File
@@ -24,10 +24,12 @@
"warmup-num-inference-steps": 28,
"enable-negative-prompt": true,
"baseline": {
"throughput_qps": {"1": 0.0549, "2": 0.0547, "4": 0.0549},
"latency_mean": {"1": 18.0, "2": 34.3, "4": 61.4},
"peak_memory_mb_max": 40500,
"peak_memory_mb_mean": 40500
"H100": {
"throughput_qps": {"1": 0.0549, "2": 0.0547, "4": 0.0549},
"latency_mean": {"1": 18.0, "2": 34.3, "4": 61.4},
"peak_memory_mb_max": 40500,
"peak_memory_mb_mean": 40500
}
}
}
]
tests/dfx/perf/tests/test_cosmos3_vllm_omni.json
+20 -12
View File
@@ -48,9 +48,11 @@
"guardrails": false
},
"baseline": {
"throughput_qps": 1.107,
"latency_mean": 0.8929,
"peak_memory_mb_mean": 80000.0
"H100": {
"throughput_qps": 1.107,
"latency_mean": 0.8929,
"peak_memory_mb_mean": 80000.0
}
}
}
]
@@ -115,9 +117,11 @@
"use_duration_template": false
},
"baseline": {
"throughput_qps": 0.027,
"latency_mean": 36.6824,
"peak_memory_mb_mean": 30155.4
"H100": {
"throughput_qps": 0.027,
"latency_mean": 36.6824,
"peak_memory_mb_mean": 30155.4
}
}
}
]
@@ -181,9 +185,11 @@
"guardrails": false
},
"baseline": {
"throughput_qps": 0.027,
"latency_mean": 36.968,
"peak_memory_mb_mean": 30443.6
"H100": {
"throughput_qps": 0.027,
"latency_mean": 36.968,
"peak_memory_mb_mean": 30443.6
}
}
}
]
@@ -251,9 +257,11 @@
"condition_video_keep": "first"
},
"baseline": {
"throughput_qps": 0.027,
"latency_mean": 37.2117,
"peak_memory_mb_mean": 30135.6
"H100": {
"throughput_qps": 0.027,
"latency_mean": 37.2117,
"peak_memory_mb_mean": 30135.6
}
}
}
]
tests/dfx/perf/tests/test_higgs_audio_v3.json
+19 -15
View File
@@ -46,12 +46,14 @@
"seed_tts_locale": "en",
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
"baseline": {
"median_audio_ttfp_ms": [
223.3584
],
"median_audio_rtf": [
0.4393
]
"H100": {
"median_audio_ttfp_ms": [
223.3584
],
"median_audio_rtf": [
0.4393
]
}
}
},
{
@@ -70,15 +72,17 @@
"seed_tts_locale": "en",
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
"baseline": {
"median_audio_ttfp_ms": [
276.575
],
"median_audio_rtf": [
0.5073
],
"audio_throughput": [
27.3728
]
"H100": {
"median_audio_ttfp_ms": [
276.575
],
"median_audio_rtf": [
0.5073
],
"audio_throughput": [
27.3728
]
}
}
}
]
tests/dfx/perf/tests/test_hunyuan_image3_it2i.json
+7 -6
View File
@@ -133,13 +133,14 @@
"bot-task": "think_recaption",
"max-concurrency": 16,
"num-prompts": 96,
"skip-performance-assertion": true,
"baseline": {
"throughput_qps": 0.1874,
"latency_p99": 101.852,
"latency_mean": 79.0385,
"peak_memory_mb_max": 70000,
"peak_memory_mb_mean": 70000
"H100": {
"throughput_qps": 0.1874,
"latency_p99": 101.852,
"latency_mean": 79.0385,
"peak_memory_mb_max": 70000,
"peak_memory_mb_mean": 70000
}
}
}
]
tests/dfx/perf/tests/test_hunyuan_image_tp2_cfgp2.json
+6 -5
View File
@@ -37,12 +37,13 @@
"num-inference-steps": 8,
"num-prompts": 10,
"max-concurrency": 1,
"skip-performance-assertion": true,
"baseline": {
"throughput_qps": 0.21,
"latency_p99": 4.7469,
"peak_memory_mb_max": 101100,
"peak_memory_mb_mean": 101100
"H100": {
"throughput_qps": 0.21,
"latency_p99": 4.7469,
"peak_memory_mb_max": 101100,
"peak_memory_mb_mean": 101100
}
}
}
]
tests/dfx/perf/tests/test_hunyuan_image_tp2_sp2.json
+6 -5
View File
@@ -37,12 +37,13 @@
"num-inference-steps": 8,
"num-prompts": 10,
"max-concurrency": 1,
"skip-performance-assertion": true,
"baseline": {
"throughput_qps": 0.20,
"latency_p99": 5.1025,
"peak_memory_mb_max": 97402,
"peak_memory_mb_mean": 97402
"H100": {
"throughput_qps": 0.20,
"latency_p99": 5.1025,
"peak_memory_mb_max": 97402,
"peak_memory_mb_mean": 97402
}
}
}
]
tests/dfx/perf/tests/test_hunyuan_image_tp4.json
+7 -6
View File
@@ -36,13 +36,14 @@
"num-inference-steps": 8,
"num-prompts": 10,
"max-concurrency": 1,
"skip-performance-assertion": true,
"baseline": {
"throughput_qps": 0.213,
"latency_mean": 4.6953,
"latency_p99": 4.8601,
"peak_memory_mb_max": 56912.0,
"peak_memory_mb_mean": 56912.0
"H100": {
"throughput_qps": 0.213,
"latency_mean": 4.6953,
"latency_p99": 4.8601,
"peak_memory_mb_max": 56912.0,
"peak_memory_mb_mean": 56912.0
}
}
}
]
tests/dfx/perf/tests/test_lingbot_video_vllm_omni.json
+1 -2
View File
@@ -26,8 +26,7 @@
"warmup-requests": 1,
"warmup-concurrency": 1,
"warmup-num-inference-steps": 2,
"enable-negative-prompt": true,
"skip-performance-assertion": true
"enable-negative-prompt": true
}
]
}
tests/dfx/perf/tests/test_qwen3_omni_async_chunk.json
+96 -80
View File
@@ -44,27 +44,29 @@
"ignore_eos": true,
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
"baseline": {
"mean_ttft_ms": [
96.4061,
140.8147,
271.9025,
362.2916,
507.776
],
"mean_e2el_ms": [
18507.4582,
28365.4956,
31907.9613,
48161.5829,
72630.8601
],
"mean_audio_rtf": [
0.1175,
0.18,
0.2359,
0.3278,
0.6296
]
"H100": {
"mean_ttft_ms": [
96.4061,
140.8147,
271.9025,
362.2916,
507.776
],
"mean_e2el_ms": [
18507.4582,
28365.4956,
31907.9613,
48161.5829,
72630.8601
],
"mean_audio_rtf": [
0.1175,
0.18,
0.2359,
0.3278,
0.6296
]
}
}
},
{
@@ -91,15 +93,17 @@
},
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
"baseline": {
"mean_ttft_ms": [
354.0951
],
"mean_e2el_ms": [
3952.5615
],
"mean_audio_rtf": [
0.117
]
"H100": {
"mean_ttft_ms": [
354.0951
],
"mean_e2el_ms": [
3952.5615
],
"mean_audio_rtf": [
0.117
]
}
}
},
{
@@ -128,15 +132,17 @@
},
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
"baseline": {
"mean_ttft_ms": [
259.6317
],
"mean_e2el_ms": [
4253.3009
],
"mean_audio_rtf": [
0.1448
]
"H100": {
"mean_ttft_ms": [
259.6317
],
"mean_e2el_ms": [
4253.3009
],
"mean_audio_rtf": [
0.1448
]
}
}
},
{
@@ -167,15 +173,17 @@
},
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
"baseline": {
"mean_ttft_ms": [
457.4453
],
"mean_e2el_ms": [
5912.5623
],
"mean_audio_rtf": [
0.1967
]
"H100": {
"mean_ttft_ms": [
457.4453
],
"mean_e2el_ms": [
5912.5623
],
"mean_audio_rtf": [
0.1967
]
}
}
},
{
@@ -206,20 +214,22 @@
},
"percentile-metrics": "ttft,tpot,itl,e2el",
"baseline": {
"mean_ttft_ms": [
90.8033,
202.1779,
326.5864,
544.8914,
961.3929
],
"mean_e2el_ms": [
4517.3411,
7857.9711,
9861.1687,
13283.3363,
19541.0736
]
"H100": {
"mean_ttft_ms": [
90.8033,
202.1779,
326.5864,
544.8914,
961.3929
],
"mean_e2el_ms": [
4517.3411,
7857.9711,
9861.1687,
13283.3363,
19541.0736
]
}
}
},
{
@@ -251,12 +261,14 @@
},
"percentile-metrics": "ttft,tpot,itl,e2el",
"baseline": {
"mean_ttft_ms": [
318.048
],
"mean_e2el_ms": [
807.6114
]
"H100": {
"mean_ttft_ms": [
318.048
],
"mean_e2el_ms": [
807.6114
]
}
}
},
{
@@ -290,12 +302,14 @@
},
"percentile-metrics": "ttft,tpot,itl,e2el",
"baseline": {
"mean_ttft_ms": [
186.168
],
"mean_e2el_ms": [
726.1137
]
"H100": {
"mean_ttft_ms": [
186.168
],
"mean_e2el_ms": [
726.1137
]
}
}
},
{
@@ -331,12 +345,14 @@
},
"percentile-metrics": "ttft,tpot,itl,e2el",
"baseline": {
"mean_ttft_ms": [
375.5282
],
"mean_e2el_ms": [
1006.1663
]
"H100": {
"mean_ttft_ms": [
375.5282
],
"mean_e2el_ms": [
1006.1663
]
}
}
}
]
tests/dfx/perf/tests/test_qwen3_omni_multi_replicas.json
+21 -13
View File
@@ -35,10 +35,12 @@
"ignore_eos": true,
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
"baseline": {
"mean_ttft_ms": [130.5751, 278.6469, 324.7249, 911.6085],
"mean_tpot_ms": [7.9854, 11.1209, 16.1507, 20.7485],
"mean_audio_ttfp_ms": [486.486, 954.1791, 1289.6514, 2196.8275],
"mean_audio_rtf": [0.1896, 0.3962, 0.4098, 0.7322]
"H100": {
"mean_ttft_ms": [130.5751, 278.6469, 324.7249, 911.6085],
"mean_tpot_ms": [7.9854, 11.1209, 16.1507, 20.7485],
"mean_audio_ttfp_ms": [486.486, 954.1791, 1289.6514, 2196.8275],
"mean_audio_rtf": [0.1896, 0.3962, 0.4098, 0.7322]
}
}
},
{
@@ -61,9 +63,11 @@
},
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
"baseline": {
"mean_ttft_ms": [340.5122],
"mean_audio_ttfp_ms": [463.0752],
"mean_audio_rtf": [0.1153]
"H100": {
"mean_ttft_ms": [340.5122],
"mean_audio_ttfp_ms": [463.0752],
"mean_audio_rtf": [0.1153]
}
}
},
{
@@ -88,9 +92,11 @@
},
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
"baseline": {
"mean_ttft_ms": [236.926],
"mean_audio_ttfp_ms": [395.2182],
"mean_audio_rtf": [0.1259]
"H100": {
"mean_ttft_ms": [236.926],
"mean_audio_ttfp_ms": [395.2182],
"mean_audio_rtf": [0.1259]
}
}
},
{
@@ -117,9 +123,11 @@
},
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
"baseline": {
"mean_ttft_ms": [470.6126],
"mean_audio_ttfp_ms": [665.5474],
"mean_audio_rtf": [0.166]
"H100": {
"mean_ttft_ms": [470.6126],
"mean_audio_ttfp_ms": [665.5474],
"mean_audio_rtf": [0.166]
}
}
}
]
tests/dfx/perf/tests/test_qwen3_omni_no_async_chunk.json
+56 -48
View File
@@ -44,27 +44,29 @@
"ignore_eos": true,
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
"baseline": {
"mean_ttft_ms": [
115.8348,
173.1823,
446.672,
464.5985,
3420.0634
],
"mean_e2el_ms": [
25070.9523,
31992.0646,
37619.7669,
50393.8257,
136619.2743
],
"mean_audio_rtf": [
0.169,
0.2111,
0.2475,
0.3551,
0.9266
]
"H100": {
"mean_ttft_ms": [
115.8348,
173.1823,
446.672,
464.5985,
3420.0634
],
"mean_e2el_ms": [
25070.9523,
31992.0646,
37619.7669,
50393.8257,
136619.2743
],
"mean_audio_rtf": [
0.169,
0.2111,
0.2475,
0.3551,
0.9266
]
}
}
},
{
@@ -91,15 +93,17 @@
},
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
"baseline": {
"mean_ttft_ms": [
343.1377
],
"mean_e2el_ms": [
4811.1123
],
"mean_audio_rtf": [
0.1463
]
"H100": {
"mean_ttft_ms": [
343.1377
],
"mean_e2el_ms": [
4811.1123
],
"mean_audio_rtf": [
0.1463
]
}
}
},
{
@@ -128,15 +132,17 @@
},
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
"baseline": {
"mean_ttft_ms": [
245.5446
],
"mean_e2el_ms": [
5019.1256
],
"mean_audio_rtf": [
0.1713
]
"H100": {
"mean_ttft_ms": [
245.5446
],
"mean_e2el_ms": [
5019.1256
],
"mean_audio_rtf": [
0.1713
]
}
}
},
{
@@ -167,15 +173,17 @@
},
"percentile-metrics": "ttft,tpot,itl,e2el,audio_rtf,audio_ttfp,audio_duration",
"baseline": {
"mean_ttft_ms": [
506.1407
],
"mean_e2el_ms": [
6814.9656
],
"mean_audio_rtf": [
0.2213
]
"H100": {
"mean_ttft_ms": [
506.1407
],
"mean_e2el_ms": [
6814.9656
],
"mean_audio_rtf": [
0.2213
]
}
}
}
]
tests/dfx/perf/tests/test_qwen3_omni_vllm_text.json
+40 -32
View File
@@ -51,20 +51,22 @@
},
"percentile-metrics": "ttft,tpot,itl,e2el",
"baseline": {
"mean_ttft_ms": [
69.8081,
173.0651,
271.8108,
413.4079,
497.2546
],
"mean_e2el_ms": [
4720.8225,
7937.2532,
9962.3566,
13724.5479,
18324.091
]
"H100": {
"mean_ttft_ms": [
69.8081,
173.0651,
271.8108,
413.4079,
497.2546
],
"mean_e2el_ms": [
4720.8225,
7937.2532,
9962.3566,
13724.5479,
18324.091
]
}
}
},
{
@@ -97,12 +99,14 @@
},
"percentile-metrics": "ttft,tpot,itl,e2el",
"baseline": {
"mean_ttft_ms": [
25.9477
],
"mean_e2el_ms": [
517.3155
]
"H100": {
"mean_ttft_ms": [
25.9477
],
"mean_e2el_ms": [
517.3155
]
}
}
},
{
@@ -137,12 +141,14 @@
},
"percentile-metrics": "ttft,tpot,itl,e2el",
"baseline": {
"mean_ttft_ms": [
26.7056
],
"mean_e2el_ms": [
551.308
]
"H100": {
"mean_ttft_ms": [
26.7056
],
"mean_e2el_ms": [
551.308
]
}
}
},
{
@@ -179,12 +185,14 @@
},
"percentile-metrics": "ttft,tpot,itl,e2el",
"baseline": {
"mean_ttft_ms": [
27.6906
],
"mean_e2el_ms": [
596.5284
]
"H100": {
"mean_ttft_ms": [
27.6906
],
"mean_e2el_ms": [
596.5284
]
}
}
}
]
tests/dfx/perf/tests/test_qwen_image_edit_2511_vllm_omni.json
+30 -20
View File
@@ -34,10 +34,12 @@
"num-input-images": 2,
"enable-negative-prompt": true,
"baseline": {
"throughput_qps": 0.0721,
"latency_mean": 13.8398,
"peak_memory_mb_max": 57610.0,
"peak_memory_mb_mean": 57610.0
"H100": {
"throughput_qps": 0.0721,
"latency_mean": 13.8398,
"peak_memory_mb_max": 57610.0,
"peak_memory_mb_mean": 57610.0
}
}
},
{
@@ -52,10 +54,12 @@
"num-input-images": 2,
"enable-negative-prompt": true,
"baseline": {
"throughput_qps": 0.0186,
"latency_mean": 53.6942,
"peak_memory_mb_max": 67790.0,
"peak_memory_mb_mean": 67790.0
"H100": {
"throughput_qps": 0.0186,
"latency_mean": 53.6942,
"peak_memory_mb_max": 67790.0,
"peak_memory_mb_mean": 67790.0
}
}
}
]
@@ -100,10 +104,12 @@
"num-input-images": 2,
"enable-negative-prompt": true,
"baseline": {
"throughput_qps": 0.0538,
"latency_mean": 18.5591,
"peak_memory_mb_max": 56449.4286,
"peak_memory_mb_mean": 56449.4286
"H100": {
"throughput_qps": 0.0538,
"latency_mean": 18.5591,
"peak_memory_mb_max": 56449.4286,
"peak_memory_mb_mean": 56449.4286
}
}
}
]
@@ -157,10 +163,12 @@
"num-input-images": 2,
"enable-negative-prompt": true,
"baseline": {
"throughput_qps": 0.2776,
"latency_mean": 3.5938,
"peak_memory_mb_max": 58113.7143,
"peak_memory_mb_mean": 58075.1429
"H100": {
"throughput_qps": 0.2776,
"latency_mean": 3.5938,
"peak_memory_mb_max": 58113.7143,
"peak_memory_mb_mean": 58075.1429
}
}
},
{
@@ -175,10 +183,12 @@
"num-input-images": 2,
"enable-negative-prompt": true,
"baseline": {
"throughput_qps": 0.1051,
"latency_mean": 9.4918,
"peak_memory_mb_max": 68107.4286,
"peak_memory_mb_mean": 68107.4286
"H100": {
"throughput_qps": 0.1051,
"latency_mean": 9.4918,
"peak_memory_mb_max": 68107.4286,
"peak_memory_mb_mean": 68107.4286
}
}
}
]
tests/dfx/perf/tests/test_qwen_image_edit_vllm_omni.json
+30 -20
View File
@@ -33,10 +33,12 @@
"max-concurrency": 1,
"enable-negative-prompt": true,
"baseline": {
"throughput_qps": 0.1129,
"latency_mean": 8.8352,
"peak_memory_mb_max": 58572.0,
"peak_memory_mb_mean": 58572.0
"H100": {
"throughput_qps": 0.1129,
"latency_mean": 8.8352,
"peak_memory_mb_max": 58572.0,
"peak_memory_mb_mean": 58572.0
}
}
},
{
@@ -50,10 +52,12 @@
"max-concurrency": 1,
"enable-negative-prompt": true,
"baseline": {
"throughput_qps": 0.024,
"latency_mean": 41.5686,
"peak_memory_mb_max": 66368.0,
"peak_memory_mb_mean": 66368.0
"H100": {
"throughput_qps": 0.024,
"latency_mean": 41.5686,
"peak_memory_mb_max": 66368.0,
"peak_memory_mb_mean": 66368.0
}
}
}
]
@@ -96,10 +100,12 @@
"max-concurrency": 1,
"enable-negative-prompt": true,
"baseline": {
"throughput_qps": 0.0672,
"latency_mean": 14.8517,
"peak_memory_mb_max": 57076.0,
"peak_memory_mb_mean": 57076.0
"H100": {
"throughput_qps": 0.0672,
"latency_mean": 14.8517,
"peak_memory_mb_max": 57076.0,
"peak_memory_mb_mean": 57076.0
}
}
}
]
@@ -152,10 +158,12 @@
"max-concurrency": 1,
"enable-negative-prompt": true,
"baseline": {
"throughput_qps": 0.2879,
"latency_mean": 3.4701,
"peak_memory_mb_max": 58758.0,
"peak_memory_mb_mean": 58758.0
"H100": {
"throughput_qps": 0.2879,
"latency_mean": 3.4701,
"peak_memory_mb_max": 58758.0,
"peak_memory_mb_mean": 58758.0
}
}
},
{
@@ -169,10 +177,12 @@
"max-concurrency": 1,
"enable-negative-prompt": true,
"baseline": {
"throughput_qps": 0.1113,
"latency_mean": 8.9656,
"peak_memory_mb_max": 67424.0,
"peak_memory_mb_mean": 67424.0
"H100": {
"throughput_qps": 0.1113,
"latency_mean": 8.9656,
"peak_memory_mb_max": 67424.0,
"peak_memory_mb_mean": 67424.0
}
}
}
]
tests/dfx/perf/tests/test_qwen_image_layered_vllm_omni.json
+12 -8
View File
@@ -33,10 +33,12 @@
"max-concurrency": 1,
"enable-negative-prompt": true,
"baseline": {
"throughput_qps": 0.0703,
"latency_mean": 14.208,
"peak_memory_mb_max": 63225.1429,
"peak_memory_mb_mean": 63225.1429
"H100": {
"throughput_qps": 0.0703,
"latency_mean": 14.208,
"peak_memory_mb_max": 63225.1429,
"peak_memory_mb_mean": 63225.1429
}
}
},
{
@@ -50,10 +52,12 @@
"max-concurrency": 1,
"enable-negative-prompt": true,
"baseline": {
"throughput_qps": 0.0417,
"latency_mean": 23.914,
"peak_memory_mb_max": 63225.1429,
"peak_memory_mb_mean": 63225.1429
"H100": {
"throughput_qps": 0.0417,
"latency_mean": 23.914,
"peak_memory_mb_max": 63225.1429,
"peak_memory_mb_mean": 63225.1429
}
}
}
]
tests/dfx/perf/tests/test_qwen_image_vllm_omni.json
+40 -24
View File
@@ -33,9 +33,11 @@
"max-concurrency": 1,
"enable-negative-prompt": true,
"baseline": {
"throughput_qps": 0.4153,
"latency_mean": 2.4065,
"peak_memory_mb_mean": 55121.4286
"H100": {
"throughput_qps": 0.4153,
"latency_mean": 2.4065,
"peak_memory_mb_mean": 55121.4286
}
}
},
{
@@ -49,9 +51,11 @@
"max-concurrency": 1,
"enable-negative-prompt": true,
"baseline": {
"throughput_qps": 0.0423,
"latency_mean": 23.5991,
"peak_memory_mb_mean": 65118.0
"H100": {
"throughput_qps": 0.0423,
"latency_mean": 23.5991,
"peak_memory_mb_mean": 65118.0
}
}
}
]
@@ -91,9 +95,11 @@
"max-concurrency": 1,
"enable-negative-prompt": true,
"baseline": {
"throughput_qps": 0.4175,
"latency_mean": 2.3953,
"peak_memory_mb_mean": 55088.5714
"H100": {
"throughput_qps": 0.4175,
"latency_mean": 2.3953,
"peak_memory_mb_mean": 55088.5714
}
}
},
{
@@ -107,9 +113,11 @@
"max-concurrency": 1,
"enable-negative-prompt": true,
"baseline": {
"throughput_qps": 0.0423,
"latency_mean": 23.5768,
"peak_memory_mb_mean": 66012.2857
"H100": {
"throughput_qps": 0.0423,
"latency_mean": 23.5768,
"peak_memory_mb_mean": 66012.2857
}
}
}
]
@@ -152,9 +160,11 @@
"max-concurrency": 1,
"enable-negative-prompt": true,
"baseline": {
"throughput_qps": 0.1214,
"latency_mean": 8.2211,
"peak_memory_mb_mean": 54501.1429
"H100": {
"throughput_qps": 0.1214,
"latency_mean": 8.2211,
"peak_memory_mb_mean": 54501.1429
}
}
}
]
@@ -207,9 +217,11 @@
"max-concurrency": 1,
"enable-negative-prompt": true,
"baseline": {
"throughput_qps": 0.559,
"latency_mean": 1.794,
"peak_memory_mb_mean": 55290.5714
"H100": {
"throughput_qps": 0.559,
"latency_mean": 1.794,
"peak_memory_mb_mean": 55290.5714
}
}
},
{
@@ -223,9 +235,11 @@
"max-concurrency": 1,
"enable-negative-prompt": true,
"baseline": {
"throughput_qps": 0.1918,
"latency_mean": 5.2022,
"peak_memory_mb_mean": 64838.8571
"H100": {
"throughput_qps": 0.1918,
"latency_mean": 5.2022,
"peak_memory_mb_mean": 64838.8571
}
}
}
]
@@ -269,9 +283,11 @@
"warmup-num-inference-steps": 20,
"enable-negative-prompt": true,
"baseline": {
"throughput_qps": [0.4125, 0.6714, 0.6674, 0.8159],
"latency_mean": [2.4215, 2.992, 5.9293, 9.656],
"peak_memory_mb_mean": [55147.4286, 55148.0, 55150.4643, 55152.3821]
"H100": {
"throughput_qps": [0.4125, 0.6714, 0.6674, 0.8159],
"latency_mean": [2.4215, 2.992, 5.9293, 9.656],
"peak_memory_mb_mean": [55147.4286, 55148.0, 55150.4643, 55152.3821]
}
}
}
]
tests/dfx/perf/tests/test_tts.json
+137 -111
View File
@@ -39,12 +39,14 @@
"seed_tts_locale": "en",
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
"baseline": {
"median_audio_ttfp_ms": [
141.2791
],
"median_audio_rtf": [
0.1591
]
"H100": {
"median_audio_ttfp_ms": [
141.2791
],
"median_audio_rtf": [
0.1591
]
}
}
},
{
@@ -67,21 +69,23 @@
"seed_tts_locale": "en",
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
"baseline": {
"median_audio_ttfp_ms": [
226.747,
647.1272,
4777.2714
],
"median_audio_rtf": [
0.3313,
0.3642,
1.2282
],
"audio_throughput": [
24.2296,
40.3552,
43.2572
]
"H100": {
"median_audio_ttfp_ms": [
226.747,
647.1272,
4777.2714
],
"median_audio_rtf": [
0.3313,
0.3642,
1.2282
],
"audio_throughput": [
24.2296,
40.3552,
43.2572
]
}
}
},
{
@@ -102,9 +106,11 @@
"seed_tts_wer_eval": true,
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
"baseline": {
"mean_audio_rtf": [
0.45
]
"H100": {
"mean_audio_rtf": [
0.45
]
}
}
}
]
@@ -153,12 +159,14 @@
},
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
"baseline": {
"median_audio_ttfp_ms": [
46.8434
],
"median_audio_rtf": [
0.1367
]
"H100": {
"median_audio_ttfp_ms": [
46.8434
],
"median_audio_rtf": [
0.1367
]
}
}
},
{
@@ -186,21 +194,23 @@
},
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
"baseline": {
"median_audio_ttfp_ms": [
75.3341,
713.7727,
5941.7179
],
"median_audio_rtf": [
0.1859,
0.3148,
1.1662
],
"audio_throughput": [
38.3858,
55.3523,
68.7032
]
"H100": {
"median_audio_ttfp_ms": [
75.3341,
713.7727,
5941.7179
],
"median_audio_rtf": [
0.1859,
0.3148,
1.1662
],
"audio_throughput": [
38.3858,
55.3523,
68.7032
]
}
}
},
{
@@ -226,9 +236,11 @@
"seed_tts_wer_eval": true,
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
"baseline": {
"mean_audio_rtf": [
0.35
]
"H100": {
"mean_audio_rtf": [
0.35
]
}
}
},
{
@@ -252,15 +264,17 @@
},
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
"baseline": {
"median_audio_ttfp_ms": [
56.746
],
"median_audio_rtf": [
0.1571
],
"audio_throughput": [
11.1226
]
"H100": {
"median_audio_ttfp_ms": [
56.746
],
"median_audio_rtf": [
0.1571
],
"audio_throughput": [
11.1226
]
}
}
},
{
@@ -283,12 +297,14 @@
},
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
"baseline": {
"median_audio_ttfp_ms": [
46.3479
],
"median_audio_rtf": [
0.1352
]
"H100": {
"median_audio_ttfp_ms": [
46.3479
],
"median_audio_rtf": [
0.1352
]
}
}
},
{
@@ -315,21 +331,23 @@
},
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
"baseline": {
"median_audio_ttfp_ms": [
75.5967,
587.5877,
4960.3726
],
"median_audio_rtf": [
0.1871,
0.3143,
1.1703
],
"audio_throughput": [
38.9378,
55.1846,
71.5708
]
"H100": {
"median_audio_ttfp_ms": [
75.5967,
587.5877,
4960.3726
],
"median_audio_rtf": [
0.1871,
0.3143,
1.1703
],
"audio_throughput": [
38.9378,
55.1846,
71.5708
]
}
}
},
{
@@ -352,15 +370,17 @@
},
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
"baseline": {
"median_audio_ttfp_ms": [
55.5029
],
"median_audio_rtf": [
0.1508
],
"audio_throughput": [
9.4885
]
"H100": {
"median_audio_ttfp_ms": [
55.5029
],
"median_audio_rtf": [
0.1508
],
"audio_throughput": [
9.4885
]
}
}
}
]
@@ -402,12 +422,14 @@
},
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
"baseline": {
"median_audio_ttfp_ms": [
82.1886
],
"median_audio_rtf": [
0.1241
]
"H100": {
"median_audio_ttfp_ms": [
82.1886
],
"median_audio_rtf": [
0.1241
]
}
}
},
{
@@ -429,15 +451,17 @@
},
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
"baseline": {
"median_audio_ttfp_ms": [
160.9168
],
"median_audio_rtf": [
0.1907
],
"audio_throughput": [
39.2332
]
"H100": {
"median_audio_ttfp_ms": [
160.9168
],
"median_audio_rtf": [
0.1907
],
"audio_throughput": [
39.2332
]
}
}
},
{
@@ -459,15 +483,17 @@
},
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
"baseline": {
"median_audio_ttfp_ms": [
100.1265
],
"median_audio_rtf": [
0.1437
],
"audio_throughput": [
11.7605
]
"H100": {
"median_audio_ttfp_ms": [
100.1265
],
"median_audio_rtf": [
0.1437
],
"audio_throughput": [
11.7605
]
}
}
}
]
tests/dfx/perf/tests/test_voxcpm2.json
+59 -45
View File
@@ -37,12 +37,14 @@
"seed_tts_locale": "en",
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
"baseline": {
"median_audio_ttfp_ms": [
137.645
],
"median_audio_rtf": [
0.0497
]
"H100": {
"median_audio_ttfp_ms": [
137.645
],
"median_audio_rtf": [
0.0497
]
}
}
},
{
@@ -61,15 +63,17 @@
"seed_tts_locale": "en",
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
"baseline": {
"median_audio_ttfp_ms": [
475.3357
],
"median_audio_rtf": [
0.1243
],
"audio_throughput": [
36.6358
]
"H100": {
"median_audio_ttfp_ms": [
475.3357
],
"median_audio_rtf": [
0.1243
],
"audio_throughput": [
36.6358
]
}
}
},
{
@@ -90,9 +94,11 @@
"seed_tts_wer_eval": true,
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
"baseline": {
"mean_audio_rtf": [
0.45
]
"H100": {
"mean_audio_rtf": [
0.45
]
}
}
},
{
@@ -114,12 +120,14 @@
},
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
"baseline": {
"median_audio_ttfp_ms": [
79.6927
],
"median_audio_rtf": [
0.0423
]
"H100": {
"median_audio_ttfp_ms": [
79.6927
],
"median_audio_rtf": [
0.0423
]
}
}
},
{
@@ -141,15 +149,17 @@
},
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
"baseline": {
"median_audio_ttfp_ms": [
262.6212
],
"median_audio_rtf": [
0.0753
],
"audio_throughput": [
103.5843
]
"H100": {
"median_audio_ttfp_ms": [
262.6212
],
"median_audio_rtf": [
0.0753
],
"audio_throughput": [
103.5843
]
}
}
},
{
@@ -171,15 +181,17 @@
},
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
"baseline": {
"median_audio_ttfp_ms": [
80.1095
],
"median_audio_rtf": [
0.0479
],
"audio_throughput": [
17.2918
]
"H100": {
"median_audio_ttfp_ms": [
80.1095
],
"median_audio_rtf": [
0.0479
],
"audio_throughput": [
17.2918
]
}
}
},
{
@@ -203,9 +215,11 @@
"seed_tts_wer_eval": true,
"percentile-metrics": "ttft,e2el,audio_rtf,audio_ttfp,audio_duration",
"baseline": {
"mean_audio_rtf": [
0.35
]
"H100": {
"mean_audio_rtf": [
0.35
]
}
}
}
]
tests/dfx/perf/tests/test_wan22_i2v_vllm_omni.json
+15 -9
View File
@@ -42,9 +42,11 @@
}
],
"baseline": {
"throughput_qps": 0.0434,
"latency_mean": 23.0654,
"peak_memory_mb_mean": 76360.0
"H100": {
"throughput_qps": 0.0434,
"latency_mean": 23.0654,
"peak_memory_mb_mean": 76360.0
}
}
}
]
@@ -97,9 +99,11 @@
}
],
"baseline": {
"throughput_qps": 0.0606,
"latency_mean": 16.4928,
"peak_memory_mb_mean": 47164.8
"H100": {
"throughput_qps": 0.0606,
"latency_mean": 16.4928,
"peak_memory_mb_mean": 47164.8
}
}
},
{
@@ -122,9 +126,11 @@
}
],
"baseline": {
"throughput_qps": 0.0124,
"latency_mean": 80.7784,
"peak_memory_mb_mean": 55374.3429
"H100": {
"throughput_qps": 0.0124,
"latency_mean": 80.7784,
"peak_memory_mb_mean": 55374.3429
}
}
}
]