237 lines
9.1 KiB
Python
237 lines
9.1 KiB
Python
|
# SPDX-License-Identifier: Apache-2.0
|
|||
|
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
|||
|
|
|
|||
|
|
"""Video-MME multiple-choice accuracy scoring for vLLM-Omni bench serve.
|
|||
|
|
|
|||
|
|
Answer extraction ports ``extract_characters_regex`` from the official Video-MME
|
|||
|
|
script (MME-Benchmarks ``eval_your_results.py``, also used by lmms-eval).
|
|||
|
|
|
|||
|
|
**Denominator:** responses where no A–D can be extracted are counted as wrong, which
|
|||
|
|
matches lmms-eval and OmniEvalKit (the numbers this bench is compared against). The
|
|||
|
|
official standalone script instead drops them from ``answered``; ``videomme_parse_failed``
|
|||
|
|
is reported so that view can be recovered.
|
|||
|
|
"""
|
|||
|
|
|
|||
|
|
from __future__ import annotations
|
|||
|
|
|
|||
|
|
from typing import Any
|
|||
|
|
|
|||
|
|
import regex as re
|
|||
|
|
from vllm.benchmarks.lib.endpoint_request_func import RequestFuncOutput
|
|||
|
|
|
|||
|
|
from vllm_omni.benchmarks.data_modules.videomme_dataset import VideoMMESampleRequest
|
|||
|
|
|
|||
|
|
_VALID = frozenset("ABCD")
|
|||
|
|
|
|||
|
|
VIDEOMME_DURATION_KEYS: tuple[str, ...] = ("short", "medium", "long")
|
|||
|
|
|
|||
|
|
#: Report dimension -> ``VideoMMESampleRequest`` attribute (official Video-MME breakdowns).
|
|||
|
|
_DIMENSIONS: dict[str, str] = {
|
|||
|
|
"duration": "videomme_duration",
|
|||
|
|
"domain": "videomme_domain",
|
|||
|
|
"sub_category": "videomme_sub_category",
|
|||
|
|
"task": "videomme_task_type",
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
# Official prefix list, with the two accidental string concatenations in the upstream
|
|||
|
|
# tuple (missing commas after "The best option is" / "Best answer:") split apart.
|
|||
|
|
_ANSWER_PREFIXES = (
|
|||
|
|
"The best answer is",
|
|||
|
|
"The correct answer is",
|
|||
|
|
"The answer is",
|
|||
|
|
"The answer",
|
|||
|
|
"The best option is",
|
|||
|
|
"The correct option is",
|
|||
|
|
"Best answer:",
|
|||
|
|
"Best option:",
|
|||
|
|
"Answer:",
|
|||
|
|
"Option:",
|
|||
|
|
"The correct answer",
|
|||
|
|
"The correct option",
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
|
|||
|
|
def extract_characters_regex(text: str | None) -> str | None:
|
|||
|
|
"""Port of official Video-MME / lmms-eval ``extract_characters_regex``."""
|
|||
|
|
if not text:
|
|||
|
|
return None
|
|||
|
|
s = str(text).strip()
|
|||
|
|
for prefix in _ANSWER_PREFIXES:
|
|||
|
|
s = s.replace(prefix, "")
|
|||
|
|
if len(s.split()) > 10 and not re.search(r"[ABCD]", s):
|
|||
|
|
return None
|
|||
|
|
match = re.search(r"[ABCD]", s)
|
|||
|
|
return match.group(0) if match else None
|
|||
|
|
|
|||
|
|
|
|||
|
|
def normalize_gold_answer(gold: str) -> str | None:
|
|||
|
|
g = (gold or "").strip().upper()
|
|||
|
|
if len(g) == 1 and g in _VALID:
|
|||
|
|
return g
|
|||
|
|
m = re.search(r"([ABCD])\b", g)
|
|||
|
|
return m.group(1).upper() if m else None
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _bucket(req: VideoMMESampleRequest, dimension: str) -> str:
|
|||
|
|
value = (getattr(req, _DIMENSIONS[dimension]) or "").strip()
|
|||
|
|
if dimension == "duration":
|
|||
|
|
return value.lower()
|
|||
|
|
return value or "unknown"
|
|||
|
|
|
|||
|
|
|
|||
|
|
def compute_videomme_accuracy_metrics(
|
|||
|
|
input_requests: list[Any],
|
|||
|
|
outputs: list[RequestFuncOutput],
|
|||
|
|
*,
|
|||
|
|
include_per_item: bool = False,
|
|||
|
|
) -> dict[str, Any] | None:
|
|||
|
|
"""If all requests are :class:`VideoMMESampleRequest`, compute accuracy stats.
|
|||
|
|
|
|||
|
|
Rows without a gold answer are skipped (``no_gold``). Failed HTTP requests are
|
|||
|
|
excluded from ``videomme_accuracy`` and counted in
|
|||
|
|
``videomme_accuracy_incl_http_fail`` instead. The accuracy wrapper gate
|
|||
|
|
(``_validate_videomme``) additionally rejects HTTP failures, duplicate question
|
|||
|
|
IDs, skipped source rows and missing gold answers. Direct
|
|||
|
|
``vllm bench serve`` still reports both metrics without that completeness
|
|||
|
|
check.
|
|||
|
|
|
|||
|
|
``videomme_submitted`` / ``videomme_unique_question_ids`` show when
|
|||
|
|
oversampling made the request count differ from unique question coverage.
|
|||
|
|
``videomme_skipped_rows`` counts source rows skipped during sampling (for
|
|||
|
|
example, missing or undecodable media), before any oversampling.
|
|||
|
|
"""
|
|||
|
|
if not input_requests or len(input_requests) != len(outputs):
|
|||
|
|
return None
|
|||
|
|
if not all(isinstance(r, VideoMMESampleRequest) for r in input_requests):
|
|||
|
|
return None
|
|||
|
|
|
|||
|
|
stats: dict[str, dict[str, dict[str, int]]] = {
|
|||
|
|
dim: ({k: {"correct": 0, "total": 0} for k in VIDEOMME_DURATION_KEYS} if dim == "duration" else {})
|
|||
|
|
for dim in _DIMENSIONS
|
|||
|
|
}
|
|||
|
|
items: list[dict[str, Any]] = []
|
|||
|
|
correct = evaluated = no_gold = request_failed = parse_failed = 0
|
|||
|
|
|
|||
|
|
for req, out in zip(input_requests, outputs, strict=True):
|
|||
|
|
assert isinstance(req, VideoMMESampleRequest)
|
|||
|
|
gold_raw = (req.videomme_gold_answer or "").strip()
|
|||
|
|
buckets = {dim: _bucket(req, dim) for dim in _DIMENSIONS}
|
|||
|
|
row: dict[str, Any] = {
|
|||
|
|
"request_id": req.request_id,
|
|||
|
|
"video_id": req.videomme_video_id,
|
|||
|
|
"question_id": req.videomme_question_id,
|
|||
|
|
**buckets,
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
if not gold_raw:
|
|||
|
|
no_gold += 1
|
|||
|
|
if include_per_item:
|
|||
|
|
items.append({**row, "skipped": True, "reason": "no_gold"})
|
|||
|
|
continue
|
|||
|
|
|
|||
|
|
evaluated += 1
|
|||
|
|
gold_norm = normalize_gold_answer(gold_raw)
|
|||
|
|
pred = extract_characters_regex(out.generated_text) if out.success else None
|
|||
|
|
is_correct = pred is not None and pred.upper() == (gold_norm or gold_raw).upper()
|
|||
|
|
|
|||
|
|
if not out.success:
|
|||
|
|
request_failed += 1
|
|||
|
|
else:
|
|||
|
|
if pred is None:
|
|||
|
|
parse_failed += 1
|
|||
|
|
# Only successful responses enter the per-dimension denominators, mirroring
|
|||
|
|
# the overall videomme_accuracy definition.
|
|||
|
|
for dim, key in buckets.items():
|
|||
|
|
bucket = stats[dim].setdefault(key, {"correct": 0, "total": 0})
|
|||
|
|
bucket["total"] += 1
|
|||
|
|
bucket["correct"] += int(is_correct)
|
|||
|
|
|
|||
|
|
if is_correct:
|
|||
|
|
correct += 1
|
|||
|
|
|
|||
|
|
if include_per_item:
|
|||
|
|
items.append(
|
|||
|
|
{
|
|||
|
|
**row,
|
|||
|
|
"gold": gold_raw,
|
|||
|
|
"gold_normalized": gold_norm,
|
|||
|
|
"predicted": pred,
|
|||
|
|
"correct": is_correct,
|
|||
|
|
"parse_failed": out.success and pred is None,
|
|||
|
|
**({"error": (out.error or "")[:500]} if not out.success else {}),
|
|||
|
|
}
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
evaluated_ok = evaluated - request_failed
|
|||
|
|
unique_question_ids = {
|
|||
|
|
(req.videomme_question_id or req.request_id) for req in input_requests if isinstance(req, VideoMMESampleRequest)
|
|||
|
|
}
|
|||
|
|
result: dict[str, Any] = {
|
|||
|
|
"videomme_accuracy": (correct / evaluated_ok) if evaluated_ok else None,
|
|||
|
|
"videomme_accuracy_incl_http_fail": (correct / evaluated) if evaluated else None,
|
|||
|
|
"videomme_correct": correct,
|
|||
|
|
"videomme_evaluated": evaluated,
|
|||
|
|
"videomme_evaluated_ok": evaluated_ok,
|
|||
|
|
"videomme_no_gold": no_gold,
|
|||
|
|
"videomme_request_failed": request_failed,
|
|||
|
|
"videomme_parse_failed": parse_failed,
|
|||
|
|
"videomme_submitted": len(input_requests),
|
|||
|
|
"videomme_unique_question_ids": len(unique_question_ids),
|
|||
|
|
"videomme_skipped_rows": max(req.videomme_skipped_rows for req in input_requests),
|
|||
|
|
}
|
|||
|
|
for dim, per_bucket in stats.items():
|
|||
|
|
result[f"videomme_per_{dim}"] = {k: dict(v) for k, v in per_bucket.items()}
|
|||
|
|
result[f"videomme_per_{dim}_accuracy"] = {
|
|||
|
|
k: (v["correct"] / v["total"]) if v["total"] else None for k, v in per_bucket.items()
|
|||
|
|
}
|
|||
|
|
if include_per_item:
|
|||
|
|
result["videomme_eval_items"] = items
|
|||
|
|
return result
|
|||
|
|
|
|||
|
|
|
|||
|
|
def print_videomme_accuracy_summary(metrics: dict[str, Any]) -> None:
|
|||
|
|
"""Pretty-print the Video-MME accuracy block (stdout)."""
|
|||
|
|
acc = metrics.get("videomme_accuracy")
|
|||
|
|
if acc is None and not metrics.get("videomme_evaluated", 0):
|
|||
|
|
return
|
|||
|
|
|
|||
|
|
print("{s:{c}^{n}}".format(s=" Video-MME accuracy (MCQ) ", n=50, c="="))
|
|||
|
|
ok = int(metrics.get("videomme_evaluated_ok", 0) or 0)
|
|||
|
|
if ok and acc is not None:
|
|||
|
|
print(f"Overall Accuracy: {metrics.get('videomme_correct', 0)}/{ok} = {acc:.2%}")
|
|||
|
|
else:
|
|||
|
|
print("Overall Accuracy: 0/0 = N/A (no successful HTTP responses)")
|
|||
|
|
for label, key in (
|
|||
|
|
("Submitted (gold present):", "videomme_evaluated"),
|
|||
|
|
("Successful HTTP (denominator):", "videomme_evaluated_ok"),
|
|||
|
|
("Correct:", "videomme_correct"),
|
|||
|
|
("Skipped (no gold):", "videomme_no_gold"),
|
|||
|
|
("HTTP failed:", "videomme_request_failed"),
|
|||
|
|
("Parsed OK but no A-D found:", "videomme_parse_failed"),
|
|||
|
|
("Submitted requests:", "videomme_submitted"),
|
|||
|
|
("Unique questions:", "videomme_unique_question_ids"),
|
|||
|
|
("Skipped source rows:", "videomme_skipped_rows"),
|
|||
|
|
):
|
|||
|
|
print(f"{label:<40} {metrics.get(key, 0):<10}")
|
|||
|
|
|
|||
|
|
for dim, title in (
|
|||
|
|
("duration", "Duration"),
|
|||
|
|
("domain", "Domain"),
|
|||
|
|
("sub_category", "Sub Category"),
|
|||
|
|
("task", "Task Type"),
|
|||
|
|
):
|
|||
|
|
per_acc = metrics.get(f"videomme_per_{dim}_accuracy") or {}
|
|||
|
|
if not per_acc:
|
|||
|
|
continue
|
|||
|
|
counts = metrics.get(f"videomme_per_{dim}") or {}
|
|||
|
|
names = VIDEOMME_DURATION_KEYS if dim == "duration" else sorted(per_acc)
|
|||
|
|
print(f"\n--- Accuracy by {title} ---")
|
|||
|
|
for name in names:
|
|||
|
|
st = counts.get(name) or {}
|
|||
|
|
total = int(st.get("total", 0) or 0)
|
|||
|
|
value = per_acc.get(name)
|
|||
|
|
if total and value is not None:
|
|||
|
|
print(f"{name}: {int(st.get('correct', 0))}/{total} = {value:.2%}")
|
|||
|
|
else:
|
|||
|
|
print(f"{name}: 0/0 = N/A")
|
|||
|
|
print("=" * 50)
|