[Frontend][Benchmark] Add judger and accuracy metrics for OmniInterac… (#7317)

Co-authored-by: Gao Han <hgaoaf@connect.ust.hk>
This commit is contained in:
ZacheryAUandGao Han authored and GitHub committed 2026-09-24 08:34:23 +00:00
1 parent 1d77e96d4d
commit 926ba72793
13 files changed
+4165 -30

No files matched your search

@@ -32,6 +32,13 @@ def _positive_finite_float(value: str) -> float:
return parsed
def _positive_int(value: str) -> int:
parsed = int(value)
if parsed <= 0:
raise argparse.ArgumentTypeError(f"must be a positive integer, got {value!r}")
return parsed
def _existing_file(value: str) -> str:
path = Path(value).expanduser()
if not path.is_file():
@@ -42,6 +49,7 @@ def _existing_file(value: str) -> str:
def add_omniinteract_cli_args(parser: argparse.ArgumentParser) -> None:
from vllm_omni.benchmarks.data_modules.omniinteract_dataset import (
DEFAULT_MAX_VIDEO_DURATION_S,
OMNIINTERACT_SCENARIO_TAGS,
OMNIINTERACT_SUBSETS,
)
@@ -49,6 +57,35 @@ def add_omniinteract_cli_args(parser: argparse.ArgumentParser) -> None:
group.add_argument(
"--omniinteract-subsets", nargs="+", choices=OMNIINTERACT_SUBSETS, default=list(OMNIINTERACT_SUBSETS)
)
group.add_argument(
"--omniinteract-scenario-tags",
nargs="+",
choices=OMNIINTERACT_SCENARIO_TAGS,
default=None,
help=(
"Scenario tags used when sampling cases: realtime, proactive, nested, interrupted, 1qna. "
"Default behavior covers each requested tag with at least one case (when available), "
"then fills remaining --num-prompts from the rest of the selected subsets."
),
)
group.add_argument(
"--omniinteract-scenario-focus",
action="store_true",
help=(
"Only run cases matching --omniinteract-scenario-tags. "
"Without this flag, tags are used for coverage-first sampling."
),
)
group.add_argument(
"--omniinteract-video-list",
type=_existing_file,
help=(
"JSONL of sampled OmniInteract cases (video_path, output_name, subset), "
"typically sampled_cases.jsonl from a prior run. Preserves list order. "
"Cannot be combined with --seed, --dataset-path, or --omniinteract-scenario-tags. "
"With this flag, --num-prompts caps the prefix of the list (0 or oversized = all)."
),
)
group.add_argument(
"--omniinteract-timeout-s", type=_positive_finite_float, default=900.0, help="Complete session timeout."
)
@@ -76,6 +113,50 @@ def add_omniinteract_cli_args(parser: argparse.ArgumentParser) -> None:
default=Path("omniinteract-output"),
help="Directory for case and evaluator artifacts.",
)
group.add_argument(
"--omniinteract-evaluate",
action="store_true",
help="Run the text-only OmniInteract judge after generation and print an accuracy report.",
)
group.add_argument(
"--omniinteract-judge-base-url",
default="http://127.0.0.1:8000",
help="Base URL of an already-running OpenAI-compatible judge server.",
)
group.add_argument("--omniinteract-judge-model", help="Model name exposed by the judge server.")
group.add_argument(
"--omniinteract-judge-api-key",
default="EMPTY",
help="Bearer token for the judge server.",
)
group.add_argument(
"--omniinteract-judge-timeout-s",
type=_positive_finite_float,
default=60.0,
help="Timeout for each judge request.",
)
group.add_argument(
"--omniinteract-judge-max-tokens",
type=_positive_int,
default=512,
help="Maximum completion tokens for each judge request.",
)
group.add_argument(
"--omniinteract-eval-workers",
type=_positive_int,
default=8,
help="Maximum number of concurrent judge requests.",
)
group.add_argument(
"--omniinteract-eval-output-dir",
type=Path,
help="Accuracy artifact directory; defaults to OUTPUT_DIR/evaluation.",
)
group.add_argument(
"--omniinteract-eval-skip-existing",
action="store_true",
help="Reuse successful per-case evaluation artifacts.",
)
def add_multi_stage_cli_args(parser: argparse.ArgumentParser) -> None:
@@ -421,6 +502,22 @@ def preprocess_serve_args(args: argparse.Namespace) -> None:
raise ValueError("OmniInteract requires --endpoint /v1/realtime")
if not getattr(args, "omniinteract_ref_audio", None):
raise ValueError("OmniInteract requires --omniinteract-ref-audio")
if getattr(args, "omniinteract_evaluate", False) and not getattr(args, "omniinteract_judge_model", None):
raise ValueError("OmniInteract evaluation requires --omniinteract-judge-model")
if getattr(args, "omniinteract_video_list", None):
explicit = getattr(args, "explicit_keys", ())
if "seed" in explicit:
raise ValueError("--omniinteract-video-list cannot be combined with --seed")
if "dataset_path" in explicit:
raise ValueError("--omniinteract-video-list cannot be combined with --dataset-path")
if getattr(args, "omniinteract_scenario_tags", None):
raise ValueError("--omniinteract-video-list cannot be combined with --omniinteract-scenario-tags")
if getattr(args, "omniinteract_scenario_focus", False):
raise ValueError("--omniinteract-video-list cannot be combined with --omniinteract-scenario-focus")
if getattr(args, "omniinteract_scenario_focus", False) and not getattr(
args, "omniinteract_scenario_tags", None
):
raise ValueError("--omniinteract-scenario-focus requires --omniinteract-scenario-tags")
if getattr(args, "ignore_eos", False):
raise ValueError("OmniInteract does not support --ignore-eos")
if getattr(args, "profile", False):