670 lines
28 KiB
Python
670 lines
28 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
|
|
|
import json
|
|
import math
|
|
import os
|
|
import threading
|
|
from collections.abc import Callable
|
|
from contextlib import AbstractContextManager, ExitStack, contextmanager
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import pytest
|
|
|
|
from tests.dfx.conftest import (
|
|
create_paired_omni_benchmark_pytest_params,
|
|
create_test_parameter_mapping,
|
|
get_benchmark_params_for_server,
|
|
get_runtime_resource_label,
|
|
is_diffusion_perf_config,
|
|
load_benchmark_configs,
|
|
run_benchmark,
|
|
)
|
|
from tests.helpers.runtime import OmniServer
|
|
|
|
# Optional JSON field ``mark`` is applied as pytest marks via
|
|
# ``create_paired_omni_benchmark_pytest_params`` (e.g. ``"mark": [{"hardware_marks":
|
|
# {"res": {"cuda": "H100"}, "num_cards": 2}}, "full_model", "omni"]``).
|
|
|
|
|
|
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
|
|
|
|
|
def _get_config_file_from_argv() -> str | None:
|
|
"""Read ``--test-config-file`` from ``sys.argv`` at import time so parametrization can use it."""
|
|
import sys
|
|
|
|
for i, arg in enumerate(sys.argv):
|
|
if arg == "--test-config-file" and i + 1 < len(sys.argv):
|
|
return sys.argv[i + 1]
|
|
if arg.startswith("--test-config-file="):
|
|
return arg.split("=", 1)[1]
|
|
return None
|
|
|
|
|
|
_PERF_TESTS_DIR = Path(__file__).resolve().parent.parent / "tests"
|
|
|
|
CONFIG_FILE_PATH = _get_config_file_from_argv()
|
|
if CONFIG_FILE_PATH is None:
|
|
_all_configs = load_benchmark_configs(config_dir=_PERF_TESTS_DIR)
|
|
BENCHMARK_CONFIGS = [cfg for cfg in _all_configs if not is_diffusion_perf_config(cfg)]
|
|
print(
|
|
f"No --test-config-file: loaded {len(BENCHMARK_CONFIGS)} omni/tts/generation case(s) from "
|
|
f"{_PERF_TESTS_DIR}/*.json (skipped {len(_all_configs) - len(BENCHMARK_CONFIGS)} diffusion; "
|
|
f"use -m to filter, e.g. -m tts)"
|
|
)
|
|
else:
|
|
_loaded = load_benchmark_configs(CONFIG_FILE_PATH)
|
|
BENCHMARK_CONFIGS = [cfg for cfg in _loaded if not is_diffusion_perf_config(cfg)]
|
|
skipped = len(_loaded) - len(BENCHMARK_CONFIGS)
|
|
if skipped:
|
|
print(
|
|
f"--test-config-file: loaded {len(BENCHMARK_CONFIGS)} omni/tts/generation case(s); "
|
|
f"skipped {skipped} remaining diffusion case(s) (chat completions / custom jsonl)"
|
|
)
|
|
|
|
DEPLOY_CONFIGS_DIR = Path(__file__).parent.parent / "deploy"
|
|
server_to_benchmark_mapping = create_test_parameter_mapping(BENCHMARK_CONFIGS)
|
|
paired_benchmark_params = create_paired_omni_benchmark_pytest_params(BENCHMARK_CONFIGS, DEPLOY_CONFIGS_DIR)
|
|
|
|
_omni_server_lock = threading.Lock()
|
|
|
|
|
|
class _SingleActiveContext:
|
|
"""Reuse one active context while its configuration key is unchanged."""
|
|
|
|
def __init__(self) -> None:
|
|
self._key: Any = None
|
|
self._stack: ExitStack | None = None
|
|
self._value: Any = None
|
|
|
|
def acquire(self, key: Any, factory: Callable[[], AbstractContextManager[Any]]) -> Any:
|
|
if self._stack is not None and key == self._key:
|
|
return self._value
|
|
|
|
self.close()
|
|
stack = ExitStack()
|
|
value = stack.enter_context(factory())
|
|
self._key = key
|
|
self._stack = stack
|
|
self._value = value
|
|
return value
|
|
|
|
def close(self) -> None:
|
|
stack = self._stack
|
|
self._key = None
|
|
self._stack = None
|
|
self._value = None
|
|
if stack is not None:
|
|
stack.close()
|
|
|
|
|
|
def _config_for_test(test_name: str) -> dict[str, object] | None:
|
|
for config in BENCHMARK_CONFIGS:
|
|
if isinstance(config, dict) and config.get("test_name") == test_name:
|
|
return config
|
|
return None
|
|
|
|
|
|
def _judge_server_params_for_test(test_name: str) -> dict[str, object] | None:
|
|
config = _config_for_test(test_name)
|
|
if config is None:
|
|
return None
|
|
raw = config.get("judge_server_params")
|
|
if raw is None:
|
|
return None
|
|
if not isinstance(raw, dict):
|
|
raise TypeError(f"judge_server_params for {test_name} must be an object")
|
|
return raw
|
|
|
|
|
|
def _resolve_judge_cuda_visible_devices(configured: object) -> str:
|
|
"""Map a judge card index onto the process-visible CUDA device list.
|
|
|
|
``judge_server_params.cuda_visible_devices`` is an index into
|
|
``CUDA_VISIBLE_DEVICES`` when that env is set (``1`` with
|
|
``CUDA_VISIBLE_DEVICES=2,3`` selects physical GPU 3). Without the env it
|
|
is used as a raw device list.
|
|
"""
|
|
if not isinstance(configured, str) or not configured:
|
|
raise ValueError("judge_server_params.cuda_visible_devices must be a non-empty string")
|
|
parent = os.environ.get("CUDA_VISIBLE_DEVICES")
|
|
if parent is None or not parent.strip():
|
|
return configured
|
|
visible = [part.strip() for part in parent.split(",") if part.strip()]
|
|
if configured.isdigit():
|
|
index = int(configured)
|
|
if index < 0 or index >= len(visible):
|
|
raise ValueError(
|
|
f"judge_server_params.cuda_visible_devices={configured!r} is outside CUDA_VISIBLE_DEVICES={parent!r}"
|
|
)
|
|
return visible[index]
|
|
return configured
|
|
|
|
|
|
@contextmanager
|
|
def _start_judge_server(judge_params: dict[str, object]):
|
|
model = judge_params.get("model")
|
|
if not isinstance(model, str) or not model:
|
|
raise ValueError("judge_server_params.model must be a non-empty string")
|
|
extra = judge_params.get("extra_cli_args") or ()
|
|
if not isinstance(extra, list | tuple):
|
|
raise TypeError("judge_server_params.extra_cli_args must be a list")
|
|
visible = _resolve_judge_cuda_visible_devices(judge_params.get("cuda_visible_devices", "1"))
|
|
print(f"Starting OmniInteract judge with model: {model} on CUDA_VISIBLE_DEVICES={visible}")
|
|
with OmniServer(
|
|
model,
|
|
[str(item) for item in extra],
|
|
use_omni=bool(judge_params.get("use_omni", False)),
|
|
env_dict={"CUDA_VISIBLE_DEVICES": visible},
|
|
) as judge:
|
|
print(f"OmniInteract judge started on {judge.host}:{judge.port}")
|
|
yield judge
|
|
print("OmniInteract judge stopping...")
|
|
print("OmniInteract judge stopped")
|
|
|
|
|
|
# OmniServer defaults for flags not already present in JSON ``serve_args`` /
|
|
# ``extra_cli_args``. Add new (flag, value) pairs here rather than special-casing.
|
|
_OMNI_DEFAULT_SERVER_ARGS: tuple[tuple[str, str], ...] = (
|
|
("--stage-init-timeout", "600"),
|
|
("--init-timeout", "900"),
|
|
)
|
|
|
|
|
|
def _cli_flag_names(cli_args: tuple[str, ...] | list[str]) -> set[str]:
|
|
"""Return long-option names present in a flat CLI argv list."""
|
|
names: set[str] = set()
|
|
for item in cli_args:
|
|
token = str(item)
|
|
if not token.startswith("--"):
|
|
continue
|
|
names.add(token.split("=", 1)[0])
|
|
return names
|
|
|
|
|
|
def _merge_omni_default_server_args(
|
|
extra_cli_args: tuple[str, ...] | list[str],
|
|
*,
|
|
use_omni: bool,
|
|
defaults: tuple[tuple[str, str], ...] = _OMNI_DEFAULT_SERVER_ARGS,
|
|
) -> list[str]:
|
|
"""Fill Omni defaults for flags not already set in JSON-derived CLI args.
|
|
|
|
JSON ``serve_args`` / ``extra_cli_args`` win; only missing flags are appended.
|
|
"""
|
|
if not use_omni:
|
|
return []
|
|
present = _cli_flag_names(extra_cli_args)
|
|
args: list[str] = []
|
|
for flag, value in defaults:
|
|
if flag not in present:
|
|
args += [flag, value]
|
|
return args
|
|
|
|
|
|
def _omni_server_env() -> dict[str, str]:
|
|
"""Writable video/image storage for ``/v1/videos`` and related generation APIs."""
|
|
result_dir = Path(os.environ.get("BENCHMARK_DIR", "tests/dfx/perf/results"))
|
|
storage_path = Path(os.environ.get("VLLM_OMNI_STORAGE_PATH", str(result_dir / "storage")))
|
|
storage_path.mkdir(parents=True, exist_ok=True)
|
|
return {"VLLM_OMNI_STORAGE_PATH": str(storage_path)}
|
|
|
|
|
|
def _resolve_offline_model(model: str) -> str:
|
|
"""Resolve HF ids / MiniMax env overrides the same way as the diffusion runner."""
|
|
import huggingface_hub
|
|
|
|
from vllm_omni.transformers_utils.repo_utils import hf_api
|
|
|
|
if not model or os.path.isdir(model):
|
|
return model
|
|
|
|
model_env_overrides = {
|
|
"MiniMaxAI/MiniMax-H3": "VLLM_TEST_MINIMAX_H3_MODEL",
|
|
"MiniMaxAI/MiniMax-H3/FL2VA": "VLLM_TEST_MINIMAX_H3_FL2VA_MODEL",
|
|
"MiniMaxAI/MiniMax-H3/Ref2VA": "VLLM_TEST_MINIMAX_H3_REF2VA_MODEL",
|
|
}
|
|
env_name = model_env_overrides.get(model)
|
|
if env_name:
|
|
env_model = os.environ.get(env_name)
|
|
if env_model:
|
|
return env_model
|
|
|
|
parts = model.split("/")
|
|
if len(parts) >= 3:
|
|
repo_id = "/".join(parts[:2])
|
|
subfolder = "/".join(parts[2:])
|
|
snapshot_root = hf_api().snapshot_download(
|
|
repo_id,
|
|
allow_patterns=[f"{subfolder}/**"],
|
|
local_files_only=huggingface_hub.constants.HF_HUB_OFFLINE,
|
|
)
|
|
return str(Path(snapshot_root) / subfolder)
|
|
|
|
if not huggingface_hub.constants.HF_HUB_OFFLINE:
|
|
return model
|
|
return hf_api().snapshot_download(model, local_files_only=True)
|
|
|
|
|
|
@contextmanager
|
|
def _start_omni_server(server_param):
|
|
test_name, model, stage_config_path, stage_overrides, extra_cli_args, use_omni = server_param
|
|
extra = tuple(extra_cli_args or ())
|
|
model = _resolve_offline_model(model)
|
|
|
|
print(f"Starting OmniServer with test: {test_name}, model: {model}")
|
|
|
|
server_args: list[str] = _merge_omni_default_server_args(extra, use_omni=use_omni)
|
|
# --deploy-config and --stage-overrides compose at the CLI (see vllm_omni/entrypoints/utils.py):
|
|
# deploy-config sets the base; stage-overrides are applied on top. Both can be set.
|
|
if stage_config_path:
|
|
server_args = ["--deploy-config", stage_config_path] + server_args
|
|
if stage_overrides:
|
|
server_args = ["--stage-overrides", stage_overrides] + server_args
|
|
if extra:
|
|
server_args = list(extra) + server_args
|
|
judge_params = _judge_server_params_for_test(test_name)
|
|
with OmniServer(model, server_args, use_omni=use_omni, env_dict=_omni_server_env()) as server:
|
|
server.test_name = test_name
|
|
server.judge_base_url = None
|
|
print("OmniServer started successfully")
|
|
if judge_params is None:
|
|
yield server
|
|
else:
|
|
with _start_judge_server(judge_params) as judge:
|
|
server.judge_base_url = f"http://{judge.host}:{judge.port}"
|
|
yield server
|
|
print("OmniServer stopping...")
|
|
|
|
print("OmniServer stopped")
|
|
|
|
|
|
@pytest.fixture(scope="module")
|
|
def omni_server_context():
|
|
"""Start vLLM-Omni server as a subprocess with actual model weights.
|
|
Reuse it for adjacent benchmark cases with the same server configuration.
|
|
Multi-stage initialization can take 10-20+ minutes.
|
|
"""
|
|
with _omni_server_lock:
|
|
active_context = _SingleActiveContext()
|
|
try:
|
|
yield active_context
|
|
finally:
|
|
active_context.close()
|
|
|
|
|
|
@pytest.fixture
|
|
def omni_server(request, omni_server_context):
|
|
return omni_server_context.acquire(request.param, lambda: _start_omni_server(request.param))
|
|
|
|
|
|
@pytest.fixture
|
|
def benchmark_params(request):
|
|
"""Benchmark parameters fixture; paired with ``omni_server`` via parametrization."""
|
|
test_name, param_index = request.param
|
|
|
|
all_params = get_benchmark_params_for_server(test_name, server_to_benchmark_mapping)
|
|
|
|
if not all_params:
|
|
raise ValueError(f"No benchmark parameters found for test: {test_name}")
|
|
|
|
if param_index >= len(all_params):
|
|
raise ValueError(f"No benchmark parameters found for index {param_index} in test: {test_name}")
|
|
|
|
current = param_index + 1
|
|
total = len(all_params)
|
|
print(f"\n Running benchmark {current}/{total} for {test_name}")
|
|
|
|
return {
|
|
"test_name": test_name,
|
|
"params": all_params[param_index],
|
|
}
|
|
|
|
|
|
def _resolve_num_warmups(params: dict[str, Any], *, default: int) -> int:
|
|
value = params.get("num_warmups")
|
|
if value is None:
|
|
return default
|
|
if not isinstance(value, int) or isinstance(value, bool) or value < 0:
|
|
raise ValueError("num_warmups must be a non-negative integer")
|
|
return value
|
|
|
|
|
|
def _assert_fixed_stage_workloads(result: dict[str, Any], params: dict[str, Any], num_prompts: int) -> None:
|
|
"""Reject successful responses that did less work than a fixed-length benchmark requested."""
|
|
extra_body = params.get("extra_body") or params.get("extra-body") or {}
|
|
fixed_lengths = {
|
|
str(stage): sampling["max_tokens"]
|
|
for stage, sampling in enumerate(extra_body.get("sampling_params_list", []))
|
|
if isinstance(sampling, dict)
|
|
and isinstance(sampling.get("max_tokens"), int)
|
|
and sampling["max_tokens"] > 0
|
|
and sampling.get("min_tokens") == sampling["max_tokens"]
|
|
}
|
|
if not fixed_lengths:
|
|
return
|
|
|
|
snapshots = result.get("request_stage_metrics")
|
|
assert isinstance(snapshots, list) and len(snapshots) == num_prompts, (
|
|
"Fixed stage workload: missing per-request stage metrics"
|
|
)
|
|
for request_index, snapshot in enumerate(snapshots):
|
|
for stage, expected in fixed_lengths.items():
|
|
metrics = snapshot.get(stage) if isinstance(snapshot, dict) else None
|
|
actual = metrics.get("num_tokens_out") if isinstance(metrics, dict) else None
|
|
assert actual == expected, (
|
|
f"Fixed stage workload: request {request_index}, stage {stage}: "
|
|
f"expected {expected} output tokens, got {actual!r}"
|
|
)
|
|
stages = ", ".join(f"stage {stage}={expected}" for stage, expected in fixed_lengths.items())
|
|
print(f"Fixed stage workload OK: {num_prompts} requests, num_tokens_out {stages}")
|
|
|
|
|
|
def _is_finite_number(value: object) -> bool:
|
|
return isinstance(value, int | float) and not isinstance(value, bool) and math.isfinite(value)
|
|
|
|
|
|
_OMNIINTERACT_AGGREGATE_MIN_IA_QTF1 = "omniinteract_aggregate_min_ia_qtf1"
|
|
_OMNIINTERACT_AGGREGATE_SUBSETS = "omniinteract_aggregate_subsets"
|
|
_OMNIINTERACT_AGGREGATE_GROUP = "omniinteract_aggregate_group"
|
|
_OMNIINTERACT_AGGREGATE_COUNTS: dict[tuple[str, str], tuple[float, float, float]] = {}
|
|
_OMNIINTERACT_MIN_EVALUATED_CASES = 1
|
|
|
|
|
|
def _reset_omniinteract_aggregate_counts() -> None:
|
|
_OMNIINTERACT_AGGREGATE_COUNTS.clear()
|
|
|
|
|
|
def _ia_qtf1_from_counts(tp: float, fp: float, fn: float) -> float:
|
|
precision = tp / (tp + fp) if tp + fp > 0 else 0.0
|
|
recall = tp / (tp + fn) if tp + fn > 0 else 0.0
|
|
return (2.0 * precision * recall / (precision + recall)) if precision + recall > 0 else 0.0
|
|
|
|
|
|
def _assert_subset_evaluated(accuracy: dict[str, object]) -> None:
|
|
"""Reject a subset whose clipped or cancelled cases left nothing to score.
|
|
|
|
Individual ineligible cases may be skipped. A subset with fewer than
|
|
``_OMNIINTERACT_MIN_EVALUATED_CASES`` evaluated cases must not pass, or its
|
|
zero TP/FP/FN would be pooled as a successful accuracy result.
|
|
"""
|
|
|
|
evaluated = accuracy.get("evaluated")
|
|
if not isinstance(evaluated, int) or isinstance(evaluated, bool):
|
|
raise AssertionError("OmniInteract accuracy evaluated count is missing")
|
|
if evaluated < _OMNIINTERACT_MIN_EVALUATED_CASES:
|
|
raise AssertionError(
|
|
f"OmniInteract subset evaluated {evaluated} cases (skipped={accuracy.get('skipped')}); "
|
|
f"at least {_OMNIINTERACT_MIN_EVALUATED_CASES} evaluated case is required"
|
|
)
|
|
|
|
|
|
def _finite_accuracy_count(value: object, name: str) -> float:
|
|
assert _is_finite_number(value), f"OmniInteract accuracy summary {name} is missing"
|
|
return float(value)
|
|
|
|
|
|
def _aggregate_accuracy_params_for_test(test_name: str) -> dict[str, object]:
|
|
config = _config_for_test(test_name)
|
|
if config is None:
|
|
return {}
|
|
min_ia_qtf1 = config.get(_OMNIINTERACT_AGGREGATE_MIN_IA_QTF1)
|
|
if min_ia_qtf1 is None:
|
|
return {}
|
|
if not _is_finite_number(min_ia_qtf1):
|
|
raise ValueError("omniinteract_aggregate_min_ia_qtf1 must be a finite number")
|
|
subsets: list[str] = []
|
|
raw_params = config.get("benchmark_params") or []
|
|
if not isinstance(raw_params, list):
|
|
raise TypeError(f"benchmark_params for {test_name} must be a list")
|
|
for item in raw_params:
|
|
if not isinstance(item, dict) or not item.get("omniinteract_evaluate"):
|
|
continue
|
|
subset = item.get("omniinteract_subsets")
|
|
if not isinstance(subset, str) or not subset:
|
|
raise ValueError("omniinteract_aggregate_min_ia_qtf1 requires omniinteract_subsets on each evaluated case")
|
|
subsets.append(subset)
|
|
if not subsets:
|
|
raise ValueError("omniinteract_aggregate_min_ia_qtf1 requires at least one evaluated OmniInteract subset")
|
|
if len(set(subsets)) != len(subsets):
|
|
raise ValueError("omniinteract_aggregate_subsets must not contain duplicates")
|
|
return {
|
|
_OMNIINTERACT_AGGREGATE_MIN_IA_QTF1: float(min_ia_qtf1),
|
|
_OMNIINTERACT_AGGREGATE_SUBSETS: subsets,
|
|
_OMNIINTERACT_AGGREGATE_GROUP: test_name,
|
|
}
|
|
|
|
|
|
def _maybe_assert_aggregate_ia_qtf1(params: dict[str, object], acc_summary: dict[str, object]) -> None:
|
|
min_ia_qtf1 = params.get(_OMNIINTERACT_AGGREGATE_MIN_IA_QTF1)
|
|
if min_ia_qtf1 is None:
|
|
return
|
|
assert _is_finite_number(min_ia_qtf1), "omniinteract_aggregate_min_ia_qtf1 must be a finite number"
|
|
subsets = params.get(_OMNIINTERACT_AGGREGATE_SUBSETS)
|
|
if not isinstance(subsets, list | tuple) or not subsets:
|
|
raise ValueError("omniinteract_aggregate_min_ia_qtf1 requires omniinteract_aggregate_subsets")
|
|
if len(set(subsets)) != len(subsets):
|
|
raise ValueError("omniinteract_aggregate_subsets must not contain duplicates")
|
|
if not all(isinstance(item, str) and item for item in subsets):
|
|
raise ValueError("omniinteract_aggregate_subsets must be non-empty strings")
|
|
subset = params.get("omniinteract_subsets")
|
|
if not isinstance(subset, str) or not subset:
|
|
raise ValueError("omniinteract_aggregate_min_ia_qtf1 requires omniinteract_subsets")
|
|
if subset not in subsets:
|
|
raise ValueError(f"omniinteract_subsets {subset!r} is not in omniinteract_aggregate_subsets")
|
|
group = params.get(_OMNIINTERACT_AGGREGATE_GROUP)
|
|
if not isinstance(group, str) or not group:
|
|
raise ValueError("omniinteract_aggregate_min_ia_qtf1 requires omniinteract_aggregate_group")
|
|
tp = _finite_accuracy_count(acc_summary.get("Global_TP"), "Global_TP")
|
|
fp = _finite_accuracy_count(acc_summary.get("Global_FP"), "Global_FP")
|
|
fn = _finite_accuracy_count(acc_summary.get("Global_FN"), "Global_FN")
|
|
_OMNIINTERACT_AGGREGATE_COUNTS[(group, subset)] = (tp, fp, fn)
|
|
recorded = [_OMNIINTERACT_AGGREGATE_COUNTS.get((group, name)) for name in subsets]
|
|
present = [counts for counts in recorded if counts is not None]
|
|
if len(present) != len(subsets):
|
|
return
|
|
total_tp = sum(counts[0] for counts in present)
|
|
total_fp = sum(counts[1] for counts in present)
|
|
total_fn = sum(counts[2] for counts in present)
|
|
ia_qtf1 = _ia_qtf1_from_counts(total_tp, total_fp, total_fn)
|
|
print(
|
|
f"OmniInteract aggregate All Global IA-QTF1: {ia_qtf1:.6f} "
|
|
f"(TP={total_tp:.6f} / FP={total_fp:.6f} / FN={total_fn:.6f})"
|
|
)
|
|
assert ia_qtf1 >= float(min_ia_qtf1), f"OmniInteract aggregate All Global IA-QTF1 {ia_qtf1} is below {min_ia_qtf1}"
|
|
|
|
|
|
def assert_result(result, params, num_prompt) -> None:
|
|
assert result["completed"] == num_prompt, "Request failures exist"
|
|
_assert_fixed_stage_workloads(result, params, num_prompt)
|
|
if params.get("dataset_name") == "omniinteract":
|
|
summary = result.get("omniinteract")
|
|
assert isinstance(summary, dict), "OmniInteract summary is missing"
|
|
assert (summary.get("total"), summary.get("success"), summary.get("failed")) == (
|
|
num_prompt,
|
|
num_prompt,
|
|
0,
|
|
), "OmniInteract requests did not all succeed"
|
|
assert summary.get("artifacts_complete") is True, "OmniInteract artifacts are incomplete"
|
|
if params.get("omniinteract_evaluate"):
|
|
accuracy = summary.get("accuracy")
|
|
assert isinstance(accuracy, dict), "OmniInteract accuracy is missing"
|
|
assert accuracy.get("status") == "ok", "OmniInteract accuracy did not complete"
|
|
assert accuracy.get("failed") == 0, "OmniInteract accuracy reported failed cases"
|
|
_assert_subset_evaluated(accuracy)
|
|
acc_summary = accuracy.get("summary")
|
|
assert isinstance(acc_summary, dict), "OmniInteract accuracy summary is missing"
|
|
ia_qtf1 = acc_summary.get("IA_QTF1")
|
|
assert _is_finite_number(ia_qtf1), "OmniInteract All Global IA-QTF1 is missing"
|
|
min_ia_qtf1 = params.get("omniinteract_min_ia_qtf1")
|
|
if min_ia_qtf1 is not None:
|
|
assert _is_finite_number(min_ia_qtf1), "omniinteract_min_ia_qtf1 must be a finite number"
|
|
assert float(ia_qtf1) >= float(min_ia_qtf1), (
|
|
f"OmniInteract All Global IA-QTF1 {ia_qtf1} is below {min_ia_qtf1}"
|
|
)
|
|
_maybe_assert_aggregate_ia_qtf1(params, acc_summary)
|
|
baseline = params.get("baseline")
|
|
hardware = result.get("Hardware")
|
|
hardware_baseline = baseline.get(hardware) if isinstance(baseline, dict) and isinstance(hardware, str) else None
|
|
if isinstance(hardware_baseline, dict) and "mean_tpot_ms" in hardware_baseline:
|
|
num_tpot_samples = result.get("num_tpot_samples")
|
|
mean_tpot_ms = result.get("mean_tpot_ms")
|
|
assert isinstance(num_tpot_samples, int) and not isinstance(num_tpot_samples, bool) and num_tpot_samples > 0, (
|
|
"TPOT baseline is configured, but no measurable TPOT samples were produced"
|
|
)
|
|
assert (
|
|
isinstance(mean_tpot_ms, int | float) and not isinstance(mean_tpot_ms, bool) and math.isfinite(mean_tpot_ms)
|
|
), "TPOT baseline is configured, but mean_tpot_ms is not finite"
|
|
expected_audio_turns = params.get("expected_duplex_audio_turns_per_session")
|
|
if expected_audio_turns is not None:
|
|
session_metrics = result.get("duplex_session_metrics")
|
|
assert isinstance(session_metrics, list), "Duplex session metrics are missing"
|
|
assert len(session_metrics) == num_prompt, (
|
|
f"Expected {num_prompt} duplex session metric rows, got {len(session_metrics)}"
|
|
)
|
|
assert all(
|
|
isinstance(metric, dict) and metric.get("audio_turn_count") == expected_audio_turns
|
|
for metric in session_metrics
|
|
), f"Not every duplex session emitted {expected_audio_turns} audio turns"
|
|
|
|
|
|
@pytest.mark.benchmark
|
|
@pytest.mark.parametrize(
|
|
"omni_server,benchmark_params",
|
|
paired_benchmark_params,
|
|
indirect=["omni_server", "benchmark_params"],
|
|
)
|
|
def test_performance_benchmark(omni_server, benchmark_params):
|
|
test_name = benchmark_params["test_name"]
|
|
params = dict(benchmark_params["params"])
|
|
dataset_name = params.get("dataset_name", "")
|
|
if params.get("omniinteract_evaluate"):
|
|
judge_base_url = getattr(omni_server, "judge_base_url", None)
|
|
if not isinstance(judge_base_url, str) or not judge_base_url:
|
|
raise ValueError("omniinteract_evaluate requires top-level judge_server_params to start a judge")
|
|
params["omniinteract_judge_base_url"] = judge_base_url
|
|
|
|
host = omni_server.host
|
|
port = omni_server.port
|
|
model = omni_server.model
|
|
|
|
print(f"Running benchmark for model: {model}")
|
|
print(f"Benchmark parameters: {benchmark_params}")
|
|
|
|
resource_label = get_runtime_resource_label()
|
|
|
|
def to_list(value, default=None):
|
|
if value is None:
|
|
return [] if default is None else [default]
|
|
return [value] if not isinstance(value, (list, tuple)) else list(value)
|
|
|
|
qps_list = to_list(params.get("request_rate"))
|
|
num_prompt_list = to_list(params.get("num_prompts"))
|
|
max_concurrency_list = to_list(params.get("max_concurrency"))
|
|
|
|
max_len = max(len(qps_list), len(max_concurrency_list))
|
|
if len(num_prompt_list) == 1 and max_len > 1:
|
|
num_prompt_list = num_prompt_list * max_len
|
|
elif max_len == 1 and len(num_prompt_list) > 1:
|
|
if len(qps_list) == 1:
|
|
qps_list = qps_list * len(num_prompt_list)
|
|
if len(max_concurrency_list) == 1:
|
|
max_concurrency_list = max_concurrency_list * len(num_prompt_list)
|
|
max_len = max(len(qps_list), len(max_concurrency_list))
|
|
elif len(num_prompt_list) != max_len and max_len > 0:
|
|
raise ValueError("The number of prompts does not match the QPS or max_concurrency")
|
|
|
|
args = ["--host", host, "--port", str(port)]
|
|
exclude_keys = {
|
|
"request_rate",
|
|
"baseline",
|
|
"num_prompts",
|
|
"max_concurrency",
|
|
"num_warmups",
|
|
"task",
|
|
"enabled",
|
|
"eval_phase",
|
|
"trust_remote_code",
|
|
"expected_duplex_audio_turns_per_session",
|
|
"omniinteract_min_ia_qtf1",
|
|
_OMNIINTERACT_AGGREGATE_MIN_IA_QTF1,
|
|
_OMNIINTERACT_AGGREGATE_SUBSETS,
|
|
_OMNIINTERACT_AGGREGATE_GROUP,
|
|
"judge_server_params",
|
|
"name",
|
|
"enable_negative_prompt",
|
|
"random_request_config",
|
|
"num_input_images",
|
|
"warmup_requests",
|
|
"warmup_concurrency",
|
|
"warmup_num_inference_steps",
|
|
}
|
|
|
|
param_keys = {str(key).replace("-", "_") for key in params}
|
|
if "model" not in param_keys:
|
|
args.extend(["--model", str(model)])
|
|
|
|
for key, value in params.items():
|
|
if key in exclude_keys or value is None:
|
|
continue
|
|
if key in {"extra_body", "extra-body"} and value == {}:
|
|
continue
|
|
|
|
arg_name = f"--{key.replace('_', '-')}"
|
|
|
|
if isinstance(value, bool) and value:
|
|
args.append(arg_name)
|
|
elif isinstance(value, dict):
|
|
json_str = json.dumps(value, ensure_ascii=False, separators=(",", ":"))
|
|
args.extend([arg_name, json_str])
|
|
elif not isinstance(value, bool):
|
|
args.extend([arg_name, str(value)])
|
|
|
|
for config in BENCHMARK_CONFIGS:
|
|
if config.get("test_name") != test_name:
|
|
continue
|
|
server_params = config.get("server_params") or {}
|
|
if server_params.get("trust_remote_code") or params.get("trust_remote_code"):
|
|
args.append("--trust-remote-code")
|
|
break
|
|
|
|
# QPS / request-rate sweep
|
|
for sweep_index, (qps, num_prompt) in enumerate(zip(qps_list, num_prompt_list)):
|
|
args = args + ["--request-rate", str(qps), "--num-prompts", str(num_prompt)]
|
|
result = run_benchmark(
|
|
args=args,
|
|
test_name=test_name,
|
|
flow=qps,
|
|
dataset_name=dataset_name,
|
|
num_prompt=num_prompt,
|
|
baseline_config=params.get("baseline"),
|
|
sweep_index=sweep_index,
|
|
random_input_len=params.get("random_input_len"),
|
|
random_output_len=params.get("random_output_len"),
|
|
resource_label=resource_label,
|
|
num_warmups=_resolve_num_warmups(params, default=2),
|
|
benchmark_params_name=params.get("name") if isinstance(params.get("name"), str) else None,
|
|
)
|
|
assert_result(result, {**params, **_aggregate_accuracy_params_for_test(test_name)}, num_prompt)
|
|
|
|
# concurrency test
|
|
for sweep_index, (concurrency, num_prompt) in enumerate(zip(max_concurrency_list, num_prompt_list)):
|
|
args = args + ["--max-concurrency", str(concurrency), "--num-prompts", str(num_prompt), "--request-rate", "inf"]
|
|
result = run_benchmark(
|
|
args=args,
|
|
test_name=test_name,
|
|
flow=concurrency,
|
|
dataset_name=dataset_name,
|
|
num_prompt=num_prompt,
|
|
baseline_config=params.get("baseline"),
|
|
sweep_index=sweep_index,
|
|
random_input_len=params.get("random_input_len"),
|
|
random_output_len=params.get("random_output_len"),
|
|
resource_label=resource_label,
|
|
num_warmups=_resolve_num_warmups(params, default=max(2, int(concurrency))),
|
|
benchmark_params_name=params.get("name") if isinstance(params.get("name"), str) else None,
|
|
)
|
|
assert_result(result, {**params, **_aggregate_accuracy_params_for_test(test_name)}, num_prompt)
|