[Frontend][Benchmark] Add judger and accuracy metrics for OmniInterac… (#7317)

Co-authored-by: Gao Han <hgaoaf@connect.ust.hk>
This commit is contained in:
ZacheryAUandGao Han authored and GitHub committed 2026-09-24 08:34:23 +00:00
1 parent 1d77e96d4d
commit 926ba72793
13 files changed
+4165 -30

No files matched your search

+47
View File
@@ -224,6 +224,53 @@ def test_preprocess_serve_args_applies_safe_omniinteract_prompt_default(
assert args.num_prompts == expected
@pytest.mark.parametrize(
("extra", "match"),
[
(["--seed", "7"], "cannot be combined with --seed"),
(["--dataset-path", "/tmp/omniinteract-data"], "cannot be combined with --dataset-path"),
(["--omniinteract-scenario-tags", "realtime"], "cannot be combined with --omniinteract-scenario-tags"),
(["--omniinteract-scenario-focus"], "cannot be combined with --omniinteract-scenario-focus"),
],
)
def test_preprocess_serve_args_rejects_omniinteract_video_list_conflicts(
tmp_path: Path,
extra: list[str],
match: str,
) -> None:
ref = tmp_path / "ref.wav"
ref.touch()
video_list = tmp_path / "list.jsonl"
video_list.write_text("{}\n")
parser = TrackingArgumentParser()
parser.add_argument("--dataset-name", default="sharegpt")
parser.add_argument("--backend", default="vllm")
parser.add_argument("--endpoint", default="/v1/completions")
parser.add_argument("--model", default="dummy")
parser.add_argument("--seed", type=int, default=0)
parser.add_argument("--dataset-path", default=None)
parser.add_argument("--num-prompts", type=int, default=1000)
parser.add_argument("--max-concurrency", type=int, default=None)
add_omni_args(parser)
args = parser.parse_args(
[
"--dataset-name",
"omniinteract",
"--backend",
"openai-realtime-duplex",
"--endpoint",
"/v1/realtime",
"--omniinteract-ref-audio",
str(ref),
"--omniinteract-video-list",
str(video_list),
*extra,
]
)
with pytest.raises(ValueError, match=match):
preprocess_serve_args(args)
@pytest.mark.parametrize(
("argv", "expected_extra_body", "expected_explicit"),
[