[Frontend][Benchmark] Add judger and accuracy metrics for OmniInterac… (#7317)
Co-authored-by: Gao Han <hgaoaf@connect.ust.hk>
This commit is contained in:
1 parent
1d77e96d4d
commit
926ba72793
13 files changed
+4165
-30
No files matched your search
@@ -100,6 +100,71 @@ class _SingleActiveContext:
|
||||
stack.close()
|
||||
|
||||
|
||||
def _config_for_test(test_name: str) -> dict[str, object] | None:
|
||||
for config in BENCHMARK_CONFIGS:
|
||||
if isinstance(config, dict) and config.get("test_name") == test_name:
|
||||
return config
|
||||
return None
|
||||
|
||||
|
||||
def _judge_server_params_for_test(test_name: str) -> dict[str, object] | None:
|
||||
config = _config_for_test(test_name)
|
||||
if config is None:
|
||||
return None
|
||||
raw = config.get("judge_server_params")
|
||||
if raw is None:
|
||||
return None
|
||||
if not isinstance(raw, dict):
|
||||
raise TypeError(f"judge_server_params for {test_name} must be an object")
|
||||
return raw
|
||||
|
||||
|
||||
def _resolve_judge_cuda_visible_devices(configured: object) -> str:
|
||||
"""Map a judge card index onto the process-visible CUDA device list.
|
||||
|
||||
``judge_server_params.cuda_visible_devices`` is an index into
|
||||
``CUDA_VISIBLE_DEVICES`` when that env is set (``1`` with
|
||||
``CUDA_VISIBLE_DEVICES=2,3`` selects physical GPU 3). Without the env it
|
||||
is used as a raw device list.
|
||||
"""
|
||||
if not isinstance(configured, str) or not configured:
|
||||
raise ValueError("judge_server_params.cuda_visible_devices must be a non-empty string")
|
||||
parent = os.environ.get("CUDA_VISIBLE_DEVICES")
|
||||
if parent is None or not parent.strip():
|
||||
return configured
|
||||
visible = [part.strip() for part in parent.split(",") if part.strip()]
|
||||
if configured.isdigit():
|
||||
index = int(configured)
|
||||
if index < 0 or index >= len(visible):
|
||||
raise ValueError(
|
||||
f"judge_server_params.cuda_visible_devices={configured!r} is outside CUDA_VISIBLE_DEVICES={parent!r}"
|
||||
)
|
||||
return visible[index]
|
||||
return configured
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _start_judge_server(judge_params: dict[str, object]):
|
||||
model = judge_params.get("model")
|
||||
if not isinstance(model, str) or not model:
|
||||
raise ValueError("judge_server_params.model must be a non-empty string")
|
||||
extra = judge_params.get("extra_cli_args") or ()
|
||||
if not isinstance(extra, list | tuple):
|
||||
raise TypeError("judge_server_params.extra_cli_args must be a list")
|
||||
visible = _resolve_judge_cuda_visible_devices(judge_params.get("cuda_visible_devices", "1"))
|
||||
print(f"Starting OmniInteract judge with model: {model} on CUDA_VISIBLE_DEVICES={visible}")
|
||||
with OmniServer(
|
||||
model,
|
||||
[str(item) for item in extra],
|
||||
use_omni=bool(judge_params.get("use_omni", False)),
|
||||
env_dict={"CUDA_VISIBLE_DEVICES": visible},
|
||||
) as judge:
|
||||
print(f"OmniInteract judge started on {judge.host}:{judge.port}")
|
||||
yield judge
|
||||
print("OmniInteract judge stopping...")
|
||||
print("OmniInteract judge stopped")
|
||||
|
||||
|
||||
# OmniServer defaults for flags not already present in JSON ``serve_args`` /
|
||||
# ``extra_cli_args``. Add new (flag, value) pairs here rather than special-casing.
|
||||
_OMNI_DEFAULT_SERVER_ARGS: tuple[tuple[str, str], ...] = (
|
||||
@@ -200,10 +265,17 @@ def _start_omni_server(server_param):
|
||||
server_args = ["--stage-overrides", stage_overrides] + server_args
|
||||
if extra:
|
||||
server_args = list(extra) + server_args
|
||||
judge_params = _judge_server_params_for_test(test_name)
|
||||
with OmniServer(model, server_args, use_omni=use_omni, env_dict=_omni_server_env()) as server:
|
||||
server.test_name = test_name
|
||||
server.judge_base_url = None
|
||||
print("OmniServer started successfully")
|
||||
yield server
|
||||
if judge_params is None:
|
||||
yield server
|
||||
else:
|
||||
with _start_judge_server(judge_params) as judge:
|
||||
server.judge_base_url = f"http://{judge.host}:{judge.port}"
|
||||
yield server
|
||||
print("OmniServer stopping...")
|
||||
|
||||
print("OmniServer stopped")
|
||||
@@ -260,6 +332,120 @@ def _resolve_num_warmups(params: dict[str, Any], *, default: int) -> int:
|
||||
return value
|
||||
|
||||
|
||||
def _is_finite_number(value: object) -> bool:
|
||||
return isinstance(value, int | float) and not isinstance(value, bool) and math.isfinite(value)
|
||||
|
||||
|
||||
_OMNIINTERACT_AGGREGATE_MIN_IA_QTF1 = "omniinteract_aggregate_min_ia_qtf1"
|
||||
_OMNIINTERACT_AGGREGATE_SUBSETS = "omniinteract_aggregate_subsets"
|
||||
_OMNIINTERACT_AGGREGATE_GROUP = "omniinteract_aggregate_group"
|
||||
_OMNIINTERACT_AGGREGATE_COUNTS: dict[tuple[str, str], tuple[float, float, float]] = {}
|
||||
_OMNIINTERACT_MIN_EVALUATED_CASES = 1
|
||||
|
||||
|
||||
def _reset_omniinteract_aggregate_counts() -> None:
|
||||
_OMNIINTERACT_AGGREGATE_COUNTS.clear()
|
||||
|
||||
|
||||
def _ia_qtf1_from_counts(tp: float, fp: float, fn: float) -> float:
|
||||
precision = tp / (tp + fp) if tp + fp > 0 else 0.0
|
||||
recall = tp / (tp + fn) if tp + fn > 0 else 0.0
|
||||
return (2.0 * precision * recall / (precision + recall)) if precision + recall > 0 else 0.0
|
||||
|
||||
|
||||
def _assert_subset_evaluated(accuracy: dict[str, object]) -> None:
|
||||
"""Reject a subset whose clipped or cancelled cases left nothing to score.
|
||||
|
||||
Individual ineligible cases may be skipped. A subset with fewer than
|
||||
``_OMNIINTERACT_MIN_EVALUATED_CASES`` evaluated cases must not pass, or its
|
||||
zero TP/FP/FN would be pooled as a successful accuracy result.
|
||||
"""
|
||||
|
||||
evaluated = accuracy.get("evaluated")
|
||||
if not isinstance(evaluated, int) or isinstance(evaluated, bool):
|
||||
raise AssertionError("OmniInteract accuracy evaluated count is missing")
|
||||
if evaluated < _OMNIINTERACT_MIN_EVALUATED_CASES:
|
||||
raise AssertionError(
|
||||
f"OmniInteract subset evaluated {evaluated} cases (skipped={accuracy.get('skipped')}); "
|
||||
f"at least {_OMNIINTERACT_MIN_EVALUATED_CASES} evaluated case is required"
|
||||
)
|
||||
|
||||
|
||||
def _finite_accuracy_count(value: object, name: str) -> float:
|
||||
assert _is_finite_number(value), f"OmniInteract accuracy summary {name} is missing"
|
||||
return float(value)
|
||||
|
||||
|
||||
def _aggregate_accuracy_params_for_test(test_name: str) -> dict[str, object]:
|
||||
config = _config_for_test(test_name)
|
||||
if config is None:
|
||||
return {}
|
||||
min_ia_qtf1 = config.get(_OMNIINTERACT_AGGREGATE_MIN_IA_QTF1)
|
||||
if min_ia_qtf1 is None:
|
||||
return {}
|
||||
if not _is_finite_number(min_ia_qtf1):
|
||||
raise ValueError("omniinteract_aggregate_min_ia_qtf1 must be a finite number")
|
||||
subsets: list[str] = []
|
||||
raw_params = config.get("benchmark_params") or []
|
||||
if not isinstance(raw_params, list):
|
||||
raise TypeError(f"benchmark_params for {test_name} must be a list")
|
||||
for item in raw_params:
|
||||
if not isinstance(item, dict) or not item.get("omniinteract_evaluate"):
|
||||
continue
|
||||
subset = item.get("omniinteract_subsets")
|
||||
if not isinstance(subset, str) or not subset:
|
||||
raise ValueError("omniinteract_aggregate_min_ia_qtf1 requires omniinteract_subsets on each evaluated case")
|
||||
subsets.append(subset)
|
||||
if not subsets:
|
||||
raise ValueError("omniinteract_aggregate_min_ia_qtf1 requires at least one evaluated OmniInteract subset")
|
||||
if len(set(subsets)) != len(subsets):
|
||||
raise ValueError("omniinteract_aggregate_subsets must not contain duplicates")
|
||||
return {
|
||||
_OMNIINTERACT_AGGREGATE_MIN_IA_QTF1: float(min_ia_qtf1),
|
||||
_OMNIINTERACT_AGGREGATE_SUBSETS: subsets,
|
||||
_OMNIINTERACT_AGGREGATE_GROUP: test_name,
|
||||
}
|
||||
|
||||
|
||||
def _maybe_assert_aggregate_ia_qtf1(params: dict[str, object], acc_summary: dict[str, object]) -> None:
|
||||
min_ia_qtf1 = params.get(_OMNIINTERACT_AGGREGATE_MIN_IA_QTF1)
|
||||
if min_ia_qtf1 is None:
|
||||
return
|
||||
assert _is_finite_number(min_ia_qtf1), "omniinteract_aggregate_min_ia_qtf1 must be a finite number"
|
||||
subsets = params.get(_OMNIINTERACT_AGGREGATE_SUBSETS)
|
||||
if not isinstance(subsets, list | tuple) or not subsets:
|
||||
raise ValueError("omniinteract_aggregate_min_ia_qtf1 requires omniinteract_aggregate_subsets")
|
||||
if len(set(subsets)) != len(subsets):
|
||||
raise ValueError("omniinteract_aggregate_subsets must not contain duplicates")
|
||||
if not all(isinstance(item, str) and item for item in subsets):
|
||||
raise ValueError("omniinteract_aggregate_subsets must be non-empty strings")
|
||||
subset = params.get("omniinteract_subsets")
|
||||
if not isinstance(subset, str) or not subset:
|
||||
raise ValueError("omniinteract_aggregate_min_ia_qtf1 requires omniinteract_subsets")
|
||||
if subset not in subsets:
|
||||
raise ValueError(f"omniinteract_subsets {subset!r} is not in omniinteract_aggregate_subsets")
|
||||
group = params.get(_OMNIINTERACT_AGGREGATE_GROUP)
|
||||
if not isinstance(group, str) or not group:
|
||||
raise ValueError("omniinteract_aggregate_min_ia_qtf1 requires omniinteract_aggregate_group")
|
||||
tp = _finite_accuracy_count(acc_summary.get("Global_TP"), "Global_TP")
|
||||
fp = _finite_accuracy_count(acc_summary.get("Global_FP"), "Global_FP")
|
||||
fn = _finite_accuracy_count(acc_summary.get("Global_FN"), "Global_FN")
|
||||
_OMNIINTERACT_AGGREGATE_COUNTS[(group, subset)] = (tp, fp, fn)
|
||||
recorded = [_OMNIINTERACT_AGGREGATE_COUNTS.get((group, name)) for name in subsets]
|
||||
present = [counts for counts in recorded if counts is not None]
|
||||
if len(present) != len(subsets):
|
||||
return
|
||||
total_tp = sum(counts[0] for counts in present)
|
||||
total_fp = sum(counts[1] for counts in present)
|
||||
total_fn = sum(counts[2] for counts in present)
|
||||
ia_qtf1 = _ia_qtf1_from_counts(total_tp, total_fp, total_fn)
|
||||
print(
|
||||
f"OmniInteract aggregate All Global IA-QTF1: {ia_qtf1:.6f} "
|
||||
f"(TP={total_tp:.6f} / FP={total_fp:.6f} / FN={total_fn:.6f})"
|
||||
)
|
||||
assert ia_qtf1 >= float(min_ia_qtf1), f"OmniInteract aggregate All Global IA-QTF1 {ia_qtf1} is below {min_ia_qtf1}"
|
||||
|
||||
|
||||
def assert_result(result, params, num_prompt) -> None:
|
||||
assert result["completed"] == num_prompt, "Request failures exist"
|
||||
if params.get("dataset_name") == "omniinteract":
|
||||
@@ -271,6 +457,23 @@ def assert_result(result, params, num_prompt) -> None:
|
||||
0,
|
||||
), "OmniInteract requests did not all succeed"
|
||||
assert summary.get("artifacts_complete") is True, "OmniInteract artifacts are incomplete"
|
||||
if params.get("omniinteract_evaluate"):
|
||||
accuracy = summary.get("accuracy")
|
||||
assert isinstance(accuracy, dict), "OmniInteract accuracy is missing"
|
||||
assert accuracy.get("status") == "ok", "OmniInteract accuracy did not complete"
|
||||
assert accuracy.get("failed") == 0, "OmniInteract accuracy reported failed cases"
|
||||
_assert_subset_evaluated(accuracy)
|
||||
acc_summary = accuracy.get("summary")
|
||||
assert isinstance(acc_summary, dict), "OmniInteract accuracy summary is missing"
|
||||
ia_qtf1 = acc_summary.get("IA_QTF1")
|
||||
assert _is_finite_number(ia_qtf1), "OmniInteract All Global IA-QTF1 is missing"
|
||||
min_ia_qtf1 = params.get("omniinteract_min_ia_qtf1")
|
||||
if min_ia_qtf1 is not None:
|
||||
assert _is_finite_number(min_ia_qtf1), "omniinteract_min_ia_qtf1 must be a finite number"
|
||||
assert float(ia_qtf1) >= float(min_ia_qtf1), (
|
||||
f"OmniInteract All Global IA-QTF1 {ia_qtf1} is below {min_ia_qtf1}"
|
||||
)
|
||||
_maybe_assert_aggregate_ia_qtf1(params, acc_summary)
|
||||
baseline = params.get("baseline")
|
||||
hardware = result.get("Hardware")
|
||||
hardware_baseline = baseline.get(hardware) if isinstance(baseline, dict) and isinstance(hardware, str) else None
|
||||
@@ -304,8 +507,13 @@ def assert_result(result, params, num_prompt) -> None:
|
||||
)
|
||||
def test_performance_benchmark(omni_server, benchmark_params):
|
||||
test_name = benchmark_params["test_name"]
|
||||
params = benchmark_params["params"]
|
||||
params = dict(benchmark_params["params"])
|
||||
dataset_name = params.get("dataset_name", "")
|
||||
if params.get("omniinteract_evaluate"):
|
||||
judge_base_url = getattr(omni_server, "judge_base_url", None)
|
||||
if not isinstance(judge_base_url, str) or not judge_base_url:
|
||||
raise ValueError("omniinteract_evaluate requires top-level judge_server_params to start a judge")
|
||||
params["omniinteract_judge_base_url"] = judge_base_url
|
||||
|
||||
host = omni_server.host
|
||||
port = omni_server.port
|
||||
@@ -349,6 +557,11 @@ def test_performance_benchmark(omni_server, benchmark_params):
|
||||
"eval_phase",
|
||||
"trust_remote_code",
|
||||
"expected_duplex_audio_turns_per_session",
|
||||
"omniinteract_min_ia_qtf1",
|
||||
_OMNIINTERACT_AGGREGATE_MIN_IA_QTF1,
|
||||
_OMNIINTERACT_AGGREGATE_SUBSETS,
|
||||
_OMNIINTERACT_AGGREGATE_GROUP,
|
||||
"judge_server_params",
|
||||
"name",
|
||||
"enable_negative_prompt",
|
||||
"random_request_config",
|
||||
@@ -403,7 +616,7 @@ def test_performance_benchmark(omni_server, benchmark_params):
|
||||
num_warmups=_resolve_num_warmups(params, default=2),
|
||||
benchmark_params_name=params.get("name") if isinstance(params.get("name"), str) else None,
|
||||
)
|
||||
assert_result(result, params, num_prompt)
|
||||
assert_result(result, {**params, **_aggregate_accuracy_params_for_test(test_name)}, num_prompt)
|
||||
|
||||
# concurrency test
|
||||
for sweep_index, (concurrency, num_prompt) in enumerate(zip(max_concurrency_list, num_prompt_list)):
|
||||
@@ -422,4 +635,4 @@ def test_performance_benchmark(omni_server, benchmark_params):
|
||||
num_warmups=_resolve_num_warmups(params, default=max(2, int(concurrency))),
|
||||
benchmark_params_name=params.get("name") if isinstance(params.get("name"), str) else None,
|
||||
)
|
||||
assert_result(result, params, num_prompt)
|
||||
assert_result(result, {**params, **_aggregate_accuracy_params_for_test(test_name)}, num_prompt)
|
||||
Reference in new issue
Block a user