# SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project """Video-MME multiple-choice accuracy scoring for vLLM-Omni bench serve. Answer extraction ports ``extract_characters_regex`` from the official Video-MME script (MME-Benchmarks ``eval_your_results.py``, also used by lmms-eval). **Denominator:** responses where no A–D can be extracted are counted as wrong, which matches lmms-eval and OmniEvalKit (the numbers this bench is compared against). The official standalone script instead drops them from ``answered``; ``videomme_parse_failed`` is reported so that view can be recovered. """ from __future__ import annotations from typing import Any import regex as re from vllm.benchmarks.lib.endpoint_request_func import RequestFuncOutput from vllm_omni.benchmarks.data_modules.videomme_dataset import VideoMMESampleRequest _VALID = frozenset("ABCD") VIDEOMME_DURATION_KEYS: tuple[str, ...] = ("short", "medium", "long") #: Report dimension -> ``VideoMMESampleRequest`` attribute (official Video-MME breakdowns). _DIMENSIONS: dict[str, str] = { "duration": "videomme_duration", "domain": "videomme_domain", "sub_category": "videomme_sub_category", "task": "videomme_task_type", } # Official prefix list, with the two accidental string concatenations in the upstream # tuple (missing commas after "The best option is" / "Best answer:") split apart. _ANSWER_PREFIXES = ( "The best answer is", "The correct answer is", "The answer is", "The answer", "The best option is", "The correct option is", "Best answer:", "Best option:", "Answer:", "Option:", "The correct answer", "The correct option", ) def extract_characters_regex(text: str | None) -> str | None: """Port of official Video-MME / lmms-eval ``extract_characters_regex``.""" if not text: return None s = str(text).strip() for prefix in _ANSWER_PREFIXES: s = s.replace(prefix, "") if len(s.split()) > 10 and not re.search(r"[ABCD]", s): return None match = re.search(r"[ABCD]", s) return match.group(0) if match else None def normalize_gold_answer(gold: str) -> str | None: g = (gold or "").strip().upper() if len(g) == 1 and g in _VALID: return g m = re.search(r"([ABCD])\b", g) return m.group(1).upper() if m else None def _bucket(req: VideoMMESampleRequest, dimension: str) -> str: value = (getattr(req, _DIMENSIONS[dimension]) or "").strip() if dimension == "duration": return value.lower() return value or "unknown" def compute_videomme_accuracy_metrics( input_requests: list[Any], outputs: list[RequestFuncOutput], *, include_per_item: bool = False, ) -> dict[str, Any] | None: """If all requests are :class:`VideoMMESampleRequest`, compute accuracy stats. Rows without a gold answer are skipped (``no_gold``). Failed HTTP requests are excluded from ``videomme_accuracy`` and counted in ``videomme_accuracy_incl_http_fail`` instead. The accuracy wrapper gate (``_validate_videomme``) additionally rejects HTTP failures, duplicate question IDs, skipped source rows and missing gold answers. Direct ``vllm bench serve`` still reports both metrics without that completeness check. ``videomme_submitted`` / ``videomme_unique_question_ids`` show when oversampling made the request count differ from unique question coverage. ``videomme_skipped_rows`` counts source rows skipped during sampling (for example, missing or undecodable media), before any oversampling. """ if not input_requests or len(input_requests) != len(outputs): return None if not all(isinstance(r, VideoMMESampleRequest) for r in input_requests): return None stats: dict[str, dict[str, dict[str, int]]] = { dim: ({k: {"correct": 0, "total": 0} for k in VIDEOMME_DURATION_KEYS} if dim == "duration" else {}) for dim in _DIMENSIONS } items: list[dict[str, Any]] = [] correct = evaluated = no_gold = request_failed = parse_failed = 0 for req, out in zip(input_requests, outputs, strict=True): assert isinstance(req, VideoMMESampleRequest) gold_raw = (req.videomme_gold_answer or "").strip() buckets = {dim: _bucket(req, dim) for dim in _DIMENSIONS} row: dict[str, Any] = { "request_id": req.request_id, "video_id": req.videomme_video_id, "question_id": req.videomme_question_id, **buckets, } if not gold_raw: no_gold += 1 if include_per_item: items.append({**row, "skipped": True, "reason": "no_gold"}) continue evaluated += 1 gold_norm = normalize_gold_answer(gold_raw) pred = extract_characters_regex(out.generated_text) if out.success else None is_correct = pred is not None and pred.upper() == (gold_norm or gold_raw).upper() if not out.success: request_failed += 1 else: if pred is None: parse_failed += 1 # Only successful responses enter the per-dimension denominators, mirroring # the overall videomme_accuracy definition. for dim, key in buckets.items(): bucket = stats[dim].setdefault(key, {"correct": 0, "total": 0}) bucket["total"] += 1 bucket["correct"] += int(is_correct) if is_correct: correct += 1 if include_per_item: items.append( { **row, "gold": gold_raw, "gold_normalized": gold_norm, "predicted": pred, "correct": is_correct, "parse_failed": out.success and pred is None, **({"error": (out.error or "")[:500]} if not out.success else {}), } ) evaluated_ok = evaluated - request_failed unique_question_ids = { (req.videomme_question_id or req.request_id) for req in input_requests if isinstance(req, VideoMMESampleRequest) } result: dict[str, Any] = { "videomme_accuracy": (correct / evaluated_ok) if evaluated_ok else None, "videomme_accuracy_incl_http_fail": (correct / evaluated) if evaluated else None, "videomme_correct": correct, "videomme_evaluated": evaluated, "videomme_evaluated_ok": evaluated_ok, "videomme_no_gold": no_gold, "videomme_request_failed": request_failed, "videomme_parse_failed": parse_failed, "videomme_submitted": len(input_requests), "videomme_unique_question_ids": len(unique_question_ids), "videomme_skipped_rows": max(req.videomme_skipped_rows for req in input_requests), } for dim, per_bucket in stats.items(): result[f"videomme_per_{dim}"] = {k: dict(v) for k, v in per_bucket.items()} result[f"videomme_per_{dim}_accuracy"] = { k: (v["correct"] / v["total"]) if v["total"] else None for k, v in per_bucket.items() } if include_per_item: result["videomme_eval_items"] = items return result def print_videomme_accuracy_summary(metrics: dict[str, Any]) -> None: """Pretty-print the Video-MME accuracy block (stdout).""" acc = metrics.get("videomme_accuracy") if acc is None and not metrics.get("videomme_evaluated", 0): return print("{s:{c}^{n}}".format(s=" Video-MME accuracy (MCQ) ", n=50, c="=")) ok = int(metrics.get("videomme_evaluated_ok", 0) or 0) if ok and acc is not None: print(f"Overall Accuracy: {metrics.get('videomme_correct', 0)}/{ok} = {acc:.2%}") else: print("Overall Accuracy: 0/0 = N/A (no successful HTTP responses)") for label, key in ( ("Submitted (gold present):", "videomme_evaluated"), ("Successful HTTP (denominator):", "videomme_evaluated_ok"), ("Correct:", "videomme_correct"), ("Skipped (no gold):", "videomme_no_gold"), ("HTTP failed:", "videomme_request_failed"), ("Parsed OK but no A-D found:", "videomme_parse_failed"), ("Submitted requests:", "videomme_submitted"), ("Unique questions:", "videomme_unique_question_ids"), ("Skipped source rows:", "videomme_skipped_rows"), ): print(f"{label:<40} {metrics.get(key, 0):<10}") for dim, title in ( ("duration", "Duration"), ("domain", "Domain"), ("sub_category", "Sub Category"), ("task", "Task Type"), ): per_acc = metrics.get(f"videomme_per_{dim}_accuracy") or {} if not per_acc: continue counts = metrics.get(f"videomme_per_{dim}") or {} names = VIDEOMME_DURATION_KEYS if dim == "duration" else sorted(per_acc) print(f"\n--- Accuracy by {title} ---") for name in names: st = counts.get(name) or {} total = int(st.get("total", 0) or 0) value = per_acc.get(name) if total and value is not None: print(f"{name}: {int(st.get('correct', 0))}/{total} = {value:.2%}") else: print(f"{name}: 0/0 = N/A") print("=" * 50)