[Frontend][Benchmark] Add judger and accuracy metrics for OmniInterac… (#7317)
Co-authored-by: Gao Han <hgaoaf@connect.ust.hk>
This commit is contained in:
1 parent
1d77e96d4d
commit
926ba72793
13 files changed
+4165
-30
No files matched your search
@@ -54,6 +54,7 @@ from vllm_omni.benchmarks.data_modules.daily_omni_dataset import (
|
||||
from vllm_omni.benchmarks.data_modules.omniinteract_dataset import (
|
||||
DEFAULT_OMNIINTERACT_REPO,
|
||||
OmniInteractDataset,
|
||||
OmniInteractEvaluationOptions,
|
||||
OmniInteractPreparedInput,
|
||||
OmniInteractSampleRequest,
|
||||
OmniInteractSessionOptions,
|
||||
@@ -92,6 +93,7 @@ from vllm_omni.benchmarks.omniinteract import (
|
||||
from vllm_omni.benchmarks.omniinteract import (
|
||||
write_batch_artifacts as write_omniinteract_batch_artifacts,
|
||||
)
|
||||
from vllm_omni.benchmarks.omniinteract_eval import evaluate_batch as evaluate_omniinteract_batch
|
||||
from vllm_omni.metrics import definitions as defs
|
||||
from vllm_omni.metrics.utils import coerce_bool, coerce_positive_float_scalar, coerce_positive_int_scalar
|
||||
|
||||
@@ -429,6 +431,39 @@ def _finalize_omniinteract_batch(
|
||||
return compact_summary
|
||||
|
||||
|
||||
async def _evaluate_omniinteract_batch(
|
||||
input_requests: list[SampleRequest],
|
||||
outputs: list[RequestFuncOutput],
|
||||
) -> dict[str, object] | None:
|
||||
rows = [
|
||||
(sample, output)
|
||||
for sample, output in zip(input_requests, outputs, strict=True)
|
||||
if isinstance(sample, OmniInteractSampleRequest)
|
||||
]
|
||||
if not rows:
|
||||
return None
|
||||
options = rows[0][0].omniinteract_options
|
||||
if not isinstance(options, OmniInteractSessionOptions) or options.evaluation is None:
|
||||
return None
|
||||
cases, results = [], []
|
||||
for sample, output in rows:
|
||||
result = getattr(output, "omniinteract_case_result", None)
|
||||
if sample.omniinteract_case is None or not isinstance(result, OmniInteractCaseResult):
|
||||
raise RuntimeError("OmniInteract benchmark output lost its dataset identity")
|
||||
cases.append(sample.omniinteract_case)
|
||||
results.append(result)
|
||||
try:
|
||||
return await asyncio.to_thread(
|
||||
evaluate_omniinteract_batch,
|
||||
cases,
|
||||
results,
|
||||
options.evaluation,
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001 - post-hoc accuracy must not fail a finished benchmark
|
||||
logger.exception("OmniInteract evaluation failed")
|
||||
return {"status": "failed", "error": str(exc)}
|
||||
|
||||
|
||||
def _prepare_omniinteract_batch(input_requests: list[SampleRequest]) -> None:
|
||||
roots: set[Path] = set()
|
||||
for sample in input_requests:
|
||||
@@ -556,14 +591,32 @@ def get_samples(args, tokenizer, **kwargs):
|
||||
subsets=tuple(getattr(args, "omniinteract_subsets")),
|
||||
random_seed=args.seed,
|
||||
disable_shuffle=getattr(args, "disable_shuffle", False),
|
||||
scenario_tags=tuple(getattr(args, "omniinteract_scenario_tags", None) or ()),
|
||||
scenario_focus=bool(getattr(args, "omniinteract_scenario_focus", False)),
|
||||
video_list=getattr(args, "omniinteract_video_list", None),
|
||||
)
|
||||
output_root = Path(getattr(args, "omniinteract_output_dir"))
|
||||
evaluation = None
|
||||
if bool(getattr(args, "omniinteract_evaluate", False)):
|
||||
evaluation_output = getattr(args, "omniinteract_eval_output_dir", None)
|
||||
evaluation = OmniInteractEvaluationOptions(
|
||||
judge_base_url=str(getattr(args, "omniinteract_judge_base_url", "http://127.0.0.1:8000")),
|
||||
judge_model=str(getattr(args, "omniinteract_judge_model", "")),
|
||||
judge_api_key=str(getattr(args, "omniinteract_judge_api_key", "EMPTY")),
|
||||
judge_timeout_s=float(getattr(args, "omniinteract_judge_timeout_s", 60.0)),
|
||||
judge_max_tokens=int(getattr(args, "omniinteract_judge_max_tokens", 512)),
|
||||
workers=int(getattr(args, "omniinteract_eval_workers", 8)),
|
||||
output_dir=Path(evaluation_output) if evaluation_output else output_root / "evaluation",
|
||||
skip_existing=bool(getattr(args, "omniinteract_eval_skip_existing", False)),
|
||||
)
|
||||
options = OmniInteractSessionOptions(
|
||||
output_root=Path(getattr(args, "omniinteract_output_dir")),
|
||||
output_root=output_root,
|
||||
timeout_s=float(getattr(args, "omniinteract_timeout_s")),
|
||||
media_timeout_s=float(getattr(args, "omniinteract_media_timeout_s")),
|
||||
ref_audio=str(getattr(args, "omniinteract_ref_audio")),
|
||||
require_response=bool(getattr(args, "omniinteract_require_response")),
|
||||
max_video_duration_s=float(getattr(args, "omniinteract_max_video_duration_s")),
|
||||
evaluation=evaluation,
|
||||
)
|
||||
requests = dataset.sample(
|
||||
tokenizer,
|
||||
@@ -3478,6 +3531,9 @@ async def benchmark(
|
||||
benchmark_duration = time.perf_counter() - benchmark_start_time
|
||||
|
||||
omniinteract_summary = _finalize_omniinteract_batch(input_requests, outputs)
|
||||
omniinteract_evaluation = await _evaluate_omniinteract_batch(input_requests, outputs)
|
||||
if omniinteract_summary is not None and omniinteract_evaluation is not None:
|
||||
omniinteract_summary["accuracy"] = omniinteract_evaluation
|
||||
|
||||
metrics: Any
|
||||
actual_output_lens: list[int] | int
|
||||
|
||||
Reference in new issue
Block a user