anodyine/vllm-omni
1
0 Fork 0
Code Issues 4 Pull Requests 2 Actions Packages Projects Releases Wiki Activity

[CI/Build] Resolve hardware-nested perf baselines per concurrency sweep (#5858)

Browse Source 
Signed-off-by: wangyu <410167048@qq.com>
Co-authored-by: amy-why-3459 <wuhaiyan17@huawei.com>
This commit is contained in:
wangyuandamy-why-3459 authored and GitHub committed 2026-08-07 16:54:38 +08:00
1 parent 1be47550d0
commit 0a12ac52ee
33 files changed
+1207 -978
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

tests/dfx/perf/scripts/diffusion_result_template.json
+6 -4
View File
@@ -21,10 +21,12 @@
"num-input-images": 0,
"enable-negative-prompt": false,
"baseline": {
"throughput_qps": 0,
"latency_mean": 0,
"peak_memory_mb_max": 0,
"peak_memory_mb_mean": 0
"H100": {
"throughput_qps": 0,
"latency_mean": 0,
"peak_memory_mb_max": 0,
"peak_memory_mb_mean": 0
}
}
},
"result": {
tests/dfx/perf/scripts/result_omni_template.json
+5 -3
View File
@@ -46,9 +46,11 @@
"median_audio_duration_s": 0,
"p99_audio_duration_s": 0,
"baseline": {
"mean_ttft_ms": 0,
"mean_audio_ttfp_ms": 0,
"mean_audio_rtf": 0
"H100": {
"mean_ttft_ms": 0,
"mean_audio_ttfp_ms": 0,
"mean_audio_rtf": 0
}
},
"random_input_len": 0,
"random_output_len": 0
tests/dfx/perf/scripts/run_benchmark.py
+75 -83
View File
@@ -1,6 +1,8 @@
import json
import os
import threading
from collections.abc import Callable
from contextlib import AbstractContextManager, ExitStack, contextmanager
from pathlib import Path
from typing import Any
@@ -13,15 +15,10 @@ from tests.dfx.conftest import (
get_runtime_resource_label,
is_diffusion_perf_config,
load_benchmark_configs,
resolve_baseline_value,
run_benchmark,
)
from tests.helpers.runtime import OmniServer
# Compare metrics to each test JSON ``baseline`` block only when pytest is run with ``--assert-baseline``
# (registered in ``tests/dfx/conftest.py``; default: off). ``run_benchmark`` and ``_resolve_baseline_value`` are
# defined in the same module.
#
# Optional JSON field ``mark`` is applied as pytest marks via
# ``create_paired_omni_benchmark_pytest_params`` (e.g. ``"mark": [{"hardware_marks":
# {"res": {"cuda": "H100"}, "num_cards": 2}}, "full_model", "omni"]``).
@@ -63,35 +60,78 @@ paired_benchmark_params = create_paired_omni_benchmark_pytest_params(BENCHMARK_C
_omni_server_lock = threading.Lock()
class _SingleActiveContext:
"""Reuse one active context while its configuration key is unchanged."""
def __init__(self) -> None:
self._key: Any = None
self._stack: ExitStack | None = None
self._value: Any = None
def acquire(self, key: Any, factory: Callable[[], AbstractContextManager[Any]]) -> Any:
if self._stack is not None and key == self._key:
return self._value
self.close()
stack = ExitStack()
value = stack.enter_context(factory())
self._key = key
self._stack = stack
self._value = value
return value
def close(self) -> None:
stack = self._stack
self._key = None
self._stack = None
self._value = None
if stack is not None:
stack.close()
@contextmanager
def _start_omni_server(server_param):
test_name, model, stage_config_path, stage_overrides, extra_cli_args, use_omni = server_param
print(f"Starting OmniServer with test: {test_name}, model: {model}")
server_args: list[str] = []
if use_omni:
server_args += ["--stage-init-timeout", "600", "--init-timeout", "900"]
# --deploy-config and --stage-overrides compose at the CLI (see vllm_omni/entrypoints/utils.py):
# deploy-config sets the base; stage-overrides are applied on top. Both can be set.
if stage_config_path:
server_args = ["--deploy-config", stage_config_path] + server_args
if stage_overrides:
server_args = ["--stage-overrides", stage_overrides] + server_args
if extra_cli_args:
server_args = list(extra_cli_args) + server_args
with OmniServer(model, server_args, use_omni=use_omni) as server:
server.test_name = test_name
print("OmniServer started successfully")
yield server
print("OmniServer stopping...")
print("OmniServer stopped")
@pytest.fixture(scope="module")
def omni_server(request):
def omni_server_context():
"""Start vLLM-Omni server as a subprocess with actual model weights.
Uses session scope so the server starts only once for the entire test session.
Reuse it for adjacent benchmark cases with the same server configuration.
Multi-stage initialization can take 10-20+ minutes.
"""
with _omni_server_lock:
test_name, model, stage_config_path, stage_overrides, extra_cli_args, use_omni = request.param
active_context = _SingleActiveContext()
try:
yield active_context
finally:
active_context.close()
print(f"Starting OmniServer with test: {test_name}, model: {model}")
server_args: list[str] = []
if use_omni:
server_args += ["--stage-init-timeout", "600", "--init-timeout", "900"]
# --deploy-config and --stage-overrides compose at the CLI (see vllm_omni/entrypoints/utils.py):
# deploy-config sets the base; stage-overrides are applied on top. Both can be set.
if stage_config_path:
server_args = ["--deploy-config", stage_config_path] + server_args
if stage_overrides:
server_args = ["--stage-overrides", stage_overrides] + server_args
if extra_cli_args:
server_args = list(extra_cli_args) + server_args
with OmniServer(model, server_args, use_omni=use_omni) as server:
server.test_name = test_name
print("OmniServer started successfully")
yield server
print("OmniServer stopping...")
print("OmniServer stopped")
@pytest.fixture
def omni_server(request, omni_server_context):
return omni_server_context.acquire(request.param, lambda: _start_omni_server(request.param))
@pytest.fixture
@@ -117,16 +157,7 @@ def benchmark_params(request):
}
def assert_result(
result,
params,
num_prompt,
*,
assert_baseline: bool,
sweep_index: int | None = None,
max_concurrency: Any | None = None,
request_rate: Any | None = None,
) -> None:
def assert_result(result, params, num_prompt) -> None:
assert result["completed"] == num_prompt, "Request failures exist"
expected_audio_turns = params.get("expected_duplex_audio_turns_per_session")
if expected_audio_turns is not None:
@@ -139,25 +170,6 @@ def assert_result(
isinstance(metric, dict) and metric.get("audio_turn_count") == expected_audio_turns
for metric in session_metrics
), f"Not every duplex session emitted {expected_audio_turns} audio turns"
if not assert_baseline:
return
baseline_data = params.get("baseline", {})
for metric_name, baseline_raw in baseline_data.items():
current_value = result[metric_name]
baseline_value = resolve_baseline_value(
baseline_raw,
sweep_index=sweep_index,
max_concurrency=max_concurrency,
request_rate=request_rate,
)
if "throughput" in metric_name:
assert current_value >= baseline_value, (
f"Throughput test result was below baseline: {metric_name}: {current_value} < {baseline_value}"
)
else:
assert current_value <= baseline_value, (
f"Test result exceeded baseline: {metric_name}: {current_value} > {baseline_value}"
)
@pytest.mark.benchmark
@@ -166,7 +178,7 @@ def assert_result(
paired_benchmark_params,
indirect=["omni_server", "benchmark_params"],
)
def test_performance_benchmark(omni_server, benchmark_params, request):
def test_performance_benchmark(omni_server, benchmark_params):
test_name = benchmark_params["test_name"]
params = benchmark_params["params"]
dataset_name = params.get("dataset_name", "")
@@ -178,7 +190,6 @@ def test_performance_benchmark(omni_server, benchmark_params, request):
print(f"Running benchmark for model: {model}")
print(f"Benchmark parameters: {benchmark_params}")
assert_baseline = request.config.getoption("--assert-baseline", default=False)
resource_label = get_runtime_resource_label()
def to_list(value, default=None):
@@ -213,7 +224,6 @@ def test_performance_benchmark(omni_server, benchmark_params, request):
"eval_phase",
"trust_remote_code",
"expected_duplex_audio_turns_per_session",
"baseline_hardware",
}
for key, value in params.items():
@@ -239,7 +249,7 @@ def test_performance_benchmark(omni_server, benchmark_params, request):
break
# QPS / request-rate sweep
for i, (qps, num_prompt) in enumerate(zip(qps_list, num_prompt_list)):
for sweep_index, (qps, num_prompt) in enumerate(zip(qps_list, num_prompt_list)):
args = args + ["--request-rate", str(qps), "--num-prompts", str(num_prompt)]
result = run_benchmark(
args=args,
@@ -248,24 +258,15 @@ def test_performance_benchmark(omni_server, benchmark_params, request):
dataset_name=dataset_name,
num_prompt=num_prompt,
baseline_config=params.get("baseline"),
sweep_index=i,
request_rate=qps,
max_concurrency=None,
sweep_index=sweep_index,
random_input_len=params.get("random_input_len"),
random_output_len=params.get("random_output_len"),
resource_label=resource_label,
)
assert_result(
result,
params,
num_prompt,
assert_baseline=assert_baseline,
sweep_index=i,
request_rate=qps,
)
assert_result(result, params, num_prompt)
# concurrency test
for i, (concurrency, num_prompt) in enumerate(zip(max_concurrency_list, num_prompt_list)):
for sweep_index, (concurrency, num_prompt) in enumerate(zip(max_concurrency_list, num_prompt_list)):
args = args + ["--max-concurrency", str(concurrency), "--num-prompts", str(num_prompt), "--request-rate", "inf"]
result = run_benchmark(
args=args,
@@ -274,18 +275,9 @@ def test_performance_benchmark(omni_server, benchmark_params, request):
dataset_name=dataset_name,
num_prompt=num_prompt,
baseline_config=params.get("baseline"),
sweep_index=i,
request_rate=None,
max_concurrency=concurrency,
sweep_index=sweep_index,
random_input_len=params.get("random_input_len"),
random_output_len=params.get("random_output_len"),
resource_label=resource_label,
)
assert_result(
result,
params,
num_prompt,
assert_baseline=assert_baseline,
sweep_index=i,
max_concurrency=concurrency,
)
assert_result(result, params, num_prompt)
tests/dfx/perf/scripts/run_diffusion_benchmark.py
+32 -102
View File
@@ -13,8 +13,6 @@ A config JSON file may be passed via --test-config-file. If omitted, every ``*.j
pytest run_diffusion_benchmark.py -m "diffusion"
pytest run_diffusion_benchmark.py --test-config-file tests/dfx/perf/tests/test_qwen_image_vllm_omni.json
Optional: ``--assert-baseline`` compares metrics to the ``baseline`` block in each benchmark entry (default: off).
Optional JSON field ``mark`` is applied as pytest marks on that case via
``pytest.param`` (e.g. ``"mark": [{"hardware_marks": {"res": {"cuda": "H100"}, "num_cards": 1}}, "full_model", "diffusion"]``).
@@ -46,6 +44,7 @@ from tests.dfx.conftest import (
get_runtime_resource_label,
hardware_json_value,
is_diffusion_perf_config,
resolve_baseline_for_sweep,
resolve_pytest_marks,
resource_label_for_filename,
)
@@ -274,11 +273,6 @@ def _write_result_record(record: dict[str, Any]) -> Path:
return target
def _append_to_aggregated_file(record: dict[str, Any]) -> None:
"""Backward-compatible wrapper; prefer :func:`_write_result_record`."""
_write_result_record(record)
_server_lock = threading.Lock()
# ---------------------------------------------------------------------------
@@ -355,6 +349,17 @@ def _kill_process_tree(pid: int) -> None:
# ---------------------------------------------------------------------------
def _resolve_offline_model(model: str) -> str:
"""Under HF_HUB_OFFLINE, resolve a HF repo id to its local snapshot dir."""
import huggingface_hub
if not model or os.path.isdir(model) or not huggingface_hub.constants.HF_HUB_OFFLINE:
return model
from huggingface_hub import snapshot_download
return snapshot_download(model, local_files_only=True)
class DiffusionServer:
"""Start a vLLM-Omni diffusion model server as a subprocess.
@@ -372,7 +377,7 @@ class DiffusionServer:
port: int | None = None,
) -> None:
self.server_cfg: dict[str, Any] = server_cfg
self.model = server_cfg["model"]
self.model = _resolve_offline_model(server_cfg["model"])
self.serve_args = server_cfg["serve_args"]
self.host = "127.0.0.1"
self.port = port if port is not None else get_open_port(self.host)
@@ -667,7 +672,7 @@ def run_benchmark(
with tempfile.NamedTemporaryFile(mode="w", suffix=".json", prefix="diffusion_bench_tmp_", delete=False) as tmp:
tmp_result_file = Path(tmp.name)
exclude_keys = {"baseline", "dataset", "task", "name", "skip-performance-assertion"}
exclude_keys = {"baseline", "dataset", "task", "name"}
cmd = [
sys.executable,
@@ -761,12 +766,17 @@ def run_benchmark(
completed = metrics.get("completed_requests", metrics.get("completed", 0))
failed = metrics.get("failed_requests", metrics.get("failed", 0))
# Persist sweep-resolved baseline from params (already narrowed in _build_run_params).
baseline = params.get("baseline") or {}
metrics["baseline"] = baseline
record: dict[str, Any] = {
"test_name": test_name,
"endpoint": endpoint,
"timestamp": timestamp,
"server_params": server_cfg.get("server_params"),
"benchmark_params": params,
"baseline": baseline,
"result": metrics,
"log_file": str(log_file),
"Model": model,
@@ -816,70 +826,14 @@ def run_benchmark(
# ---------------------------------------------------------------------------
def _resolve_baseline_value(
baseline_raw: Any,
*,
sweep_index: int | None,
max_concurrency: Any = None,
request_rate: Any = None,
) -> Any:
"""Pick the baseline threshold for this sweep step."""
if baseline_raw is None:
return 100000
if isinstance(baseline_raw, dict):
if max_concurrency is not None:
for key in (max_concurrency, str(max_concurrency)):
if key in baseline_raw:
return baseline_raw[key]
if request_rate is not None:
for key in (request_rate, str(request_rate)):
if key in baseline_raw:
return baseline_raw[key]
raise KeyError(
f"baseline dict has no key for max_concurrency={max_concurrency!r} "
f"or request_rate={request_rate!r}; keys={list(baseline_raw.keys())!r}"
)
if isinstance(baseline_raw, (list, tuple)):
if sweep_index is None:
raise ValueError("sweep_index is required when baseline is a list or tuple")
return baseline_raw[sweep_index]
return baseline_raw
def assert_result(
result: dict[str, Any],
params: dict[str, Any],
num_prompts: int,
*,
sweep_index: int | None = None,
max_concurrency: Any = None,
request_rate: Any = None,
assert_baseline: bool = True,
) -> None:
"""Assert that benchmark metrics satisfy the configured baselines."""
"""Assert that the expected number of requests completed."""
completed = result.get("completed_requests", result.get("completed", 0))
assert completed == num_prompts, f"Expected {num_prompts} completed requests, got {completed}"
if not assert_baseline:
return
if params.get("skip-performance-assertion", False):
print("Skipping performance assertions.")
return
for metric, baseline_raw in params.get("baseline", {}).items():
current = result.get(metric)
assert current is not None, f"Metric '{metric}' not found in result: {list(result.keys())}"
threshold = _resolve_baseline_value(
baseline_raw,
sweep_index=sweep_index,
max_concurrency=max_concurrency,
request_rate=request_rate,
)
if "throughput" in metric:
assert current >= threshold, f"{metric}: {current:.4f} < baseline {threshold}"
else:
assert current <= threshold, f"{metric}: {current:.4f} > baseline {threshold}"
def _default_benchmark_endpoint_for_task(task: str) -> str:
"""Return the default client-side benchmark endpoint for a diffusion task."""
@@ -921,15 +875,11 @@ def _build_run_params(
if max_concurrency is not None:
run_params["max-concurrency"] = max_concurrency
if "baseline" in params:
run_params["baseline"] = {
metric: _resolve_baseline_value(
baseline_raw,
sweep_index=sweep_index,
max_concurrency=max_concurrency,
request_rate=request_rate,
)
for metric, baseline_raw in params["baseline"].items()
}
# Keep all hardware buckets; pick the metric value for this sweep step only.
run_params["baseline"] = resolve_baseline_for_sweep(
params.get("baseline"),
sweep_index=sweep_index,
)
return run_params
@@ -952,36 +902,30 @@ def _iter_sweep_runs(params: dict[str, Any]) -> list[dict[str, Any]]:
sweep_runs: list[dict[str, Any]] = []
for i, (request_rate, num_prompts) in enumerate(zip(request_rate_list, num_prompt_list)):
for sweep_index, (request_rate, num_prompts) in enumerate(zip(request_rate_list, num_prompt_list)):
sweep_runs.append(
{
"params": _build_run_params(
params,
request_rate=request_rate,
num_prompts=num_prompts,
sweep_index=i,
sweep_index=sweep_index,
),
"num_prompts": num_prompts,
"sweep_index": i,
"request_rate": request_rate,
"max_concurrency": None,
}
)
for i, (max_concurrency, num_prompts) in enumerate(zip(max_concurrency_list, num_prompt_list)):
for sweep_index, (max_concurrency, num_prompts) in enumerate(zip(max_concurrency_list, num_prompt_list)):
sweep_runs.append(
{
"params": _build_run_params(
params,
max_concurrency=max_concurrency,
num_prompts=num_prompts,
sweep_index=i,
request_rate="inf",
sweep_index=sweep_index,
),
"num_prompts": num_prompts,
"sweep_index": i,
"request_rate": None,
"max_concurrency": max_concurrency,
}
)
@@ -992,11 +936,9 @@ def _iter_sweep_runs(params: dict[str, Any]) -> list[dict[str, Any]]:
"params": _build_run_params(
params,
num_prompts=default_num_prompts,
sweep_index=0,
),
"num_prompts": default_num_prompts,
"sweep_index": None,
"request_rate": None,
"max_concurrency": None,
}
)
@@ -1012,14 +954,12 @@ def _iter_sweep_runs(params: dict[str, Any]) -> list[dict[str, Any]]:
paired_benchmark_params,
indirect=["diffusion_server", "benchmark_params"],
)
def test_diffusion_performance_benchmark(diffusion_server, benchmark_params, request):
def test_diffusion_performance_benchmark(diffusion_server, benchmark_params):
"""Run the diffusion performance benchmark and verify request completion.
One server is started per unique parallel configuration (module scope).
For each server, all benchmark parameter sets defined in the config JSON
are executed sequentially; metrics are recorded to the aggregated result file.
Pass ``--assert-baseline`` to compare ``throughput_qps``, ``latency_mean``, etc. to the JSON ``baseline`` block.
"""
test_name = benchmark_params["test_name"]
params = benchmark_params["params"]
@@ -1062,14 +1002,4 @@ def test_diffusion_performance_benchmark(diffusion_server, benchmark_params, req
print(f"\n Aggregated results: {_aggregated_result_file_for_source(str(source))}")
print("=" * 60)
assert_baseline = request.config.getoption("--assert-baseline", default=False)
assert_result(
result,
params,
sweep_run["num_prompts"],
sweep_index=sweep_run["sweep_index"],
max_concurrency=sweep_run["max_concurrency"],
request_rate=sweep_run["request_rate"],
assert_baseline=assert_baseline,
)
assert_result(result, sweep_run["num_prompts"])