[Benchmark][MiniCPM-o] Port Video-MME dataset support to Omni bench s… (#6987)

Signed-off-by: amy-why-3459 <wuhaiyan17@huawei.com>
This commit is contained in:
amy-why-3459 authored and GitHub committed 2026-09-20 17:15:53 +08:00
1 parent ea62f86718
commit 9f5521351e
15 files changed
+2432 -59

No files matched your search

+198 -49
View File
@@ -14,7 +14,7 @@ import time
import traceback
import uuid
import wave
from collections.abc import Callable, Iterable, Mapping
from collections.abc import Callable, Iterable, Iterator, Mapping
from dataclasses import dataclass, replace
from datetime import datetime
from pathlib import Path
@@ -67,6 +67,12 @@ from vllm_omni.benchmarks.data_modules.seed_tts_dataset import (
)
from vllm_omni.benchmarks.data_modules.sound_effect_dataset import SoundEffectDataset
from vllm_omni.benchmarks.data_modules.ttsd_dataset import TTSDDataset
from vllm_omni.benchmarks.data_modules.videomme_dataset import (
VIDEOMME_DEFAULT_HF_REPO,
VideoMMEDataset,
VideoMMESampleRequest,
resolve_videomme_local_root,
)
from vllm_omni.benchmarks.omniinteract import (
VIDEO_FPS,
OmniInteractBenchmarkConfig,
@@ -283,14 +289,14 @@ def _merge_extra_body_mm_kwargs(base: dict | None, overlay: dict | None) -> dict
return out
def _attach_daily_omni_to_request_func_input(sample: SampleRequest, rfi: RequestFuncInput) -> None:
"""Apply per-request OpenAI fields (``mm_processor_kwargs``, messages) for Daily-Omni."""
if not isinstance(sample, DailyOmniSampleRequest):
def _attach_omni_chat_to_request_func_input(sample: SampleRequest, rfi: RequestFuncInput) -> None:
"""Apply per-request OpenAI fields (``mm_processor_kwargs``, messages) for Daily-Omni / Video-MME."""
if not isinstance(sample, (DailyOmniSampleRequest, VideoMMESampleRequest)):
return
rfi.extra_body = _merge_extra_body_mm_kwargs(rfi.extra_body, sample.omni_extra_body)
if sample.omni_chat_messages is not None:
setattr(rfi, "omni_chat_messages", sample.omni_chat_messages)
else:
elif isinstance(sample, DailyOmniSampleRequest):
setattr(rfi, "mm_position", sample.omni_chat_mm_position)
@@ -334,6 +340,29 @@ def _attach_omniinteract_to_request_func_input(sample: SampleRequest, rfi: Reque
setattr(rfi, "omniinteract_prepared_input", sample.omniinteract_prepared_input)
def _async_limiter(max_concurrency: int | None) -> contextlib.AbstractAsyncContextManager[object]:
if max_concurrency:
return asyncio.Semaphore(max_concurrency)
return contextlib.nullcontext()
def _as_float(value: object, default: float = 0.0) -> float:
"""Coerce loosely-typed metric dict values to ``float`` for mypy.
Strings are parsed rather than dropped: JSON-sourced session metrics such as
``audio_duration_ms`` may arrive quoted, and silently reporting ``0.0`` would
corrupt the derived TTFT / RTF numbers.
"""
if isinstance(value, bool) or value is None:
return default
if isinstance(value, (int, float, str)):
try:
return float(value)
except ValueError:
return default
return default
def _append_error(existing: str, message: str) -> str:
return f"{existing}\n{message}" if existing else message
@@ -404,9 +433,13 @@ def _prepare_omniinteract_batch(input_requests: list[SampleRequest]) -> None:
for sample in input_requests:
if not isinstance(sample, OmniInteractSampleRequest):
continue
root = sample.omniinteract_options.output_root.resolve()
options = sample.omniinteract_options
case = sample.omniinteract_case
if options is None or case is None:
raise RuntimeError("OmniInteract benchmark input lost its dataset identity")
root = options.output_root.resolve()
roots.add(root)
clear_case_artifacts(sample.omniinteract_options.output_root, sample.omniinteract_case)
clear_case_artifacts(options.output_root, case)
for root in roots:
clear_batch_artifacts(root)
@@ -428,11 +461,55 @@ def _daily_omni_repo_from_args(args) -> str | None:
return None
def _looks_like_hf_dataset_id(value: str) -> bool:
"""True for Hub ids such as ``org/name``; false for local paths."""
raw = value.strip()
if not raw or raw.startswith((".", "~", "/")) or "\\" in raw:
return False
parts = raw.split("/")
return len(parts) == 2 and all(part.strip() and part.strip() not in (".", "..") for part in parts)
def _videomme_repo_from_args(args, *, explicit: bool = False) -> str | None:
"""Resolve a Hugging Face repo id for Video-MME from CLI args.
``--dataset-name hf`` auto-detect only recognizes the official
``lmms-eval/Video-MME`` id so a custom Hub dataset is not silently treated
as Video-MME. Explicit ``--dataset-name videomme`` accepts any ``org/name``
Hub id (for ``--videomme-repo`` / ``VLLM_VIDEOMME_REPO`` overrides) and
raises when ``--dataset-path`` is neither a local directory nor a Hub id.
"""
official = {p.lower() for p in VideoMMEDataset.SUPPORTED_DATASET_PATHS}
official.add(VIDEOMME_DEFAULT_HF_REPO.lower())
candidates: list[str] = []
for attr in ("dataset_path", "hf_name"):
val = getattr(args, attr, None)
if isinstance(val, str) and val.strip():
candidates.append(val.strip())
for raw in candidates:
if resolve_videomme_local_root(raw) is not None:
continue
if raw.lower() in official:
return raw
if explicit and _looks_like_hf_dataset_id(raw):
return raw
if explicit:
raise ValueError(
f"Unsupported Video-MME --dataset-path={raw!r}. Pass an existing local "
"directory, a Hugging Face dataset id (org/name), or omit --dataset-path "
f"to use {VIDEOMME_DEFAULT_HF_REPO}."
)
return None
def get_samples(args, tokenizer):
# Daily-Omni: explicit dataset name, or hf + matching path/hf-name
is_daily_omni = args.dataset_name == "daily-omni" or (
args.dataset_name == "hf" and _daily_omni_repo_from_args(args) is not None
)
is_videomme = args.dataset_name == "videomme" or (
args.dataset_name == "hf" and _videomme_repo_from_args(args) is not None
)
is_seed_tts = args.dataset_name in (
"seed-tts",
"seed-tts-text",
@@ -449,7 +526,7 @@ def get_samples(args, tokenizer):
"openai-realtime-duplex",
"daily-omni",
]
is_omni_dataset = is_daily_omni or is_seed_tts or is_omniinteract or args.dataset_name == "random-mm"
is_omni_dataset = is_daily_omni or is_videomme or is_seed_tts or is_omniinteract or args.dataset_name == "random-mm"
if not is_omni_backend and not is_omni_dataset:
# Not an omni-related request, delegate to original implementation
@@ -625,6 +702,54 @@ def get_samples(args, tokenizer):
)
return input_requests
if is_videomme:
if args.backend not in ["openai-chat-omni", "daily-omni"]:
raise ValueError(
f"Video-MME dataset requires a multimodal backend that supports video. "
f"Got backend='{args.backend}'. Please use '--backend openai-chat-omni'"
)
# Resolve the source identity here; the dataset owns loading and extraction.
local_root = resolve_videomme_local_root(getattr(args, "dataset_path", None)) or (
resolve_videomme_local_root(getattr(args, "hf_name", None))
)
source = (
str(local_root)
if local_root is not None
else (_videomme_repo_from_args(args, explicit=args.dataset_name == "videomme") or VIDEOMME_DEFAULT_HF_REPO)
)
dataset = VideoMMEDataset(
parquet_path=getattr(args, "videomme_parquet", None),
dataset_path=source,
dataset_split=getattr(args, "hf_split", None) or "test",
dataset_subset=getattr(args, "hf_subset", None),
random_seed=args.seed,
video_dir=getattr(args, "videomme_video_dir", None),
subtitle_dir=getattr(args, "videomme_subtitle_dir", None),
pack_mode=getattr(args, "videomme_pack_mode", "minicpm-frames"),
max_frames=getattr(args, "videomme_max_frames", None),
duration_filter=getattr(args, "videomme_duration", "all"),
use_subtitle=getattr(args, "videomme_use_subtitle", False),
inline_local_video=getattr(args, "videomme_inline_local_video", False),
trust_remote_code=getattr(args, "trust_remote_code", False),
no_stream=getattr(args, "no_stream", False),
disable_shuffle=getattr(args, "disable_shuffle", False),
)
out_len = getattr(args, "output_len", None)
if out_len is None:
out_len = getattr(args, "hf_output_len", None)
if out_len is None:
out_len = VideoMMEDataset.DEFAULT_OUTPUT_LEN
return dataset.sample(
tokenizer=tokenizer,
num_requests=args.num_prompts,
output_len=out_len,
request_id_prefix=args.request_id_prefix,
no_oversample=args.no_oversample,
)
if is_seed_tts:
if args.backend not in (
"openai-audio-speech",
@@ -711,7 +836,7 @@ datasets.get_samples = get_samples
_serve_mod = sys.modules.get("vllm.benchmarks.serve")
if _serve_mod is not None:
_serve_mod.get_samples = get_samples
setattr(_serve_mod, "get_samples", get_samples)
@dataclass
@@ -2637,7 +2762,9 @@ async def async_request_openai_realtime_duplex(
output.ttft = (metric_mean(session_metrics.get("ttft_ms")) or 0.0) / 1000.0
output.audio_ttfp = (metric_mean(session_metrics.get("ttfp_ms")) or 0.0) / 1000.0
output.audio_rtf = metric_mean(session_metrics.get("rtf")) or 0.0
output.audio_duration = sum(float(metric.get("audio_duration_ms") or 0.0) for metric in turn_metrics) / 1000.0
output.audio_duration = (
sum((_as_float(metric.get("audio_duration_ms")) for metric in turn_metrics), start=0.0) / 1000.0
)
output.audio_frames = int(output.audio_duration * _SEED_TTS_OUTPUT_SAMPLE_RATE_HZ)
output.latency = request_finished_at - output.start_time
output.tts_turn_pcm_bytes = turn_pcm_bytes
@@ -2811,7 +2938,7 @@ async def benchmark(
extra_body=test_extra_body,
chat_messages=test_chat_messages,
)
_attach_daily_omni_to_request_func_input(input_requests[0], test_input)
_attach_omni_chat_to_request_func_input(input_requests[0], test_input)
_attach_seed_tts_to_request_func_input(input_requests[0], test_input)
_attach_omniinteract_to_request_func_input(input_requests[0], test_input)
@@ -2836,7 +2963,7 @@ async def benchmark(
if num_warmups > 0:
print(f"Warming up with {num_warmups} requests...")
warmup_pbar = None if disable_tqdm else tqdm(total=num_warmups)
warmup_semaphore = asyncio.Semaphore(max_concurrency) if max_concurrency else contextlib.nullcontext()
warmup_semaphore = _async_limiter(max_concurrency)
warmup_tasks = []
async def warmup_limited_request_func():
@@ -2854,12 +2981,13 @@ async def benchmark(
print("Starting main benchmark run...")
lora_iter: Iterator[str] | None = None
if lora_modules:
lora_modules_list = list(lora_modules)
if lora_assignment == "round-robin":
lora_modules = iter([lora_modules_list[i % len(lora_modules_list)] for i in range(len(input_requests))])
lora_iter = iter([lora_modules_list[i % len(lora_modules_list)] for i in range(len(input_requests))])
else:
lora_modules = iter([random.choice(lora_modules_list) for _ in range(len(input_requests))])
lora_iter = iter([random.choice(lora_modules_list) for _ in range(len(input_requests))])
if profile:
print("Starting profiler...")
@@ -2877,7 +3005,7 @@ async def benchmark(
extra_body=test_extra_body,
chat_messages=test_chat_messages,
)
_attach_daily_omni_to_request_func_input(input_requests[0], profile_input)
_attach_omni_chat_to_request_func_input(input_requests[0], profile_input)
_attach_seed_tts_to_request_func_input(input_requests[0], profile_input)
profile_output = await request_func(request_func_input=profile_input, session=session)
if profile_output.success:
@@ -2898,7 +3026,7 @@ async def benchmark(
pbar = None if disable_tqdm else tqdm(total=len(input_requests))
semaphore = asyncio.Semaphore(max_concurrency) if max_concurrency else contextlib.nullcontext()
semaphore = _async_limiter(max_concurrency)
async def limited_request_func(request_func_input, session, pbar):
async with semaphore:
@@ -2969,8 +3097,8 @@ async def benchmark(
)
per_request_extra_body = _merge_overrides(extra_body, request.request_overrides)
req_model_id, req_model_name = model_id, model_name
if lora_modules:
req_lora_module = next(lora_modules)
if lora_iter is not None:
req_lora_module = next(lora_iter)
req_model_id, req_model_name = req_lora_module, req_lora_module
request_func_input = RequestFuncInput(
@@ -2988,7 +3116,7 @@ async def benchmark(
request_id=request_id,
chat_messages=request.chat_messages,
)
_attach_daily_omni_to_request_func_input(request, request_func_input)
_attach_omni_chat_to_request_func_input(request, request_func_input)
_attach_seed_tts_to_request_func_input(request, request_func_input)
_attach_omniinteract_to_request_func_input(request, request_func_input)
tasks.append(
@@ -3007,6 +3135,8 @@ async def benchmark(
omniinteract_summary = _finalize_omniinteract_batch(input_requests, outputs)
metrics: Any
actual_output_lens: list[int] | int
if task_type == TaskType.GENERATION:
metrics, actual_output_lens = calculate_metrics(
input_requests=input_requests,
@@ -3031,6 +3161,10 @@ async def benchmark(
actual_output_lens = 0
if isinstance(metrics, MultiModalsBenchmarkMetrics):
# ``make_dataclass`` types this class as ``type``, so isinstance()
# narrows to ``object`` and attribute access fails. Keep the runtime
# check and read fields through ``Any``.
mm_metrics: Any = metrics
def measured_ttft(output: RequestFuncOutput) -> float | None:
session_metrics = getattr(output, "duplex_session_metrics", None)
@@ -3040,33 +3174,33 @@ async def benchmark(
result = {
"duration": benchmark_duration,
"completed": metrics.completed,
"failed": metrics.failed,
"total_input_tokens": metrics.total_input,
"total_output_tokens": metrics.total_output,
"request_throughput": metrics.request_throughput,
"request_goodput": metrics.request_goodput if goodput_config_dict else None,
"output_throughput": metrics.output_throughput,
"total_token_throughput": metrics.total_token_throughput,
defs.TOTAL_AUDIO_DURATION_S: getattr(metrics, defs.TOTAL_AUDIO_DURATION_S),
defs.TOTAL_AUDIO_FRAMES: getattr(metrics, defs.TOTAL_AUDIO_FRAMES),
defs.AUDIO_THROUGHPUT: getattr(metrics, defs.AUDIO_THROUGHPUT),
defs.TOTAL_IMAGES: getattr(metrics, defs.TOTAL_IMAGES),
defs.IMAGE_THROUGHPUT: getattr(metrics, defs.IMAGE_THROUGHPUT),
defs.AVERAGE_PIXELS_PER_IMAGE: getattr(metrics, defs.AVERAGE_PIXELS_PER_IMAGE),
defs.MEAN_DENOISE_STEP_LATENCY_MS: getattr(metrics, defs.MEAN_DENOISE_STEP_LATENCY_MS),
defs.TOTAL_VIDEO_DURATION_S: getattr(metrics, defs.TOTAL_VIDEO_DURATION_S),
defs.TOTAL_VIDEO_FRAMES: getattr(metrics, defs.TOTAL_VIDEO_FRAMES),
defs.VIDEO_THROUGHPUT: getattr(metrics, defs.VIDEO_THROUGHPUT),
defs.MEAN_VIDEO_RTF: getattr(metrics, defs.MEAN_VIDEO_RTF),
defs.MEDIAN_VIDEO_RTF: getattr(metrics, defs.MEDIAN_VIDEO_RTF),
defs.PERCENTILES_VIDEO_RTF: getattr(metrics, defs.PERCENTILES_VIDEO_RTF),
defs.MEAN_VIDEO_GENERATION_MS: getattr(metrics, defs.MEAN_VIDEO_GENERATION_MS),
defs.MEDIAN_VIDEO_GENERATION_MS: getattr(metrics, defs.MEDIAN_VIDEO_GENERATION_MS),
defs.PERCENTILES_VIDEO_GENERATION_MS: getattr(metrics, defs.PERCENTILES_VIDEO_GENERATION_MS),
defs.MEAN_PEAK_MEMORY_MB: getattr(metrics, defs.MEAN_PEAK_MEMORY_MB),
defs.MEDIAN_PEAK_MEMORY_MB: getattr(metrics, defs.MEDIAN_PEAK_MEMORY_MB),
defs.PERCENTILES_PEAK_MEMORY_MB: getattr(metrics, defs.PERCENTILES_PEAK_MEMORY_MB),
"completed": mm_metrics.completed,
"failed": mm_metrics.failed,
"total_input_tokens": mm_metrics.total_input,
"total_output_tokens": mm_metrics.total_output,
"request_throughput": mm_metrics.request_throughput,
"request_goodput": mm_metrics.request_goodput if goodput_config_dict else None,
"output_throughput": mm_metrics.output_throughput,
"total_token_throughput": mm_metrics.total_token_throughput,
defs.TOTAL_AUDIO_DURATION_S: getattr(mm_metrics, defs.TOTAL_AUDIO_DURATION_S),
defs.TOTAL_AUDIO_FRAMES: getattr(mm_metrics, defs.TOTAL_AUDIO_FRAMES),
defs.AUDIO_THROUGHPUT: getattr(mm_metrics, defs.AUDIO_THROUGHPUT),
defs.TOTAL_IMAGES: getattr(mm_metrics, defs.TOTAL_IMAGES),
defs.IMAGE_THROUGHPUT: getattr(mm_metrics, defs.IMAGE_THROUGHPUT),
defs.AVERAGE_PIXELS_PER_IMAGE: getattr(mm_metrics, defs.AVERAGE_PIXELS_PER_IMAGE),
defs.MEAN_DENOISE_STEP_LATENCY_MS: getattr(mm_metrics, defs.MEAN_DENOISE_STEP_LATENCY_MS),
defs.TOTAL_VIDEO_DURATION_S: getattr(mm_metrics, defs.TOTAL_VIDEO_DURATION_S),
defs.TOTAL_VIDEO_FRAMES: getattr(mm_metrics, defs.TOTAL_VIDEO_FRAMES),
defs.VIDEO_THROUGHPUT: getattr(mm_metrics, defs.VIDEO_THROUGHPUT),
defs.MEAN_VIDEO_RTF: getattr(mm_metrics, defs.MEAN_VIDEO_RTF),
defs.MEDIAN_VIDEO_RTF: getattr(mm_metrics, defs.MEDIAN_VIDEO_RTF),
defs.PERCENTILES_VIDEO_RTF: getattr(mm_metrics, defs.PERCENTILES_VIDEO_RTF),
defs.MEAN_VIDEO_GENERATION_MS: getattr(mm_metrics, defs.MEAN_VIDEO_GENERATION_MS),
defs.MEDIAN_VIDEO_GENERATION_MS: getattr(mm_metrics, defs.MEDIAN_VIDEO_GENERATION_MS),
defs.PERCENTILES_VIDEO_GENERATION_MS: getattr(mm_metrics, defs.PERCENTILES_VIDEO_GENERATION_MS),
defs.MEAN_PEAK_MEMORY_MB: getattr(mm_metrics, defs.MEAN_PEAK_MEMORY_MB),
defs.MEDIAN_PEAK_MEMORY_MB: getattr(mm_metrics, defs.MEDIAN_PEAK_MEMORY_MB),
defs.PERCENTILES_PEAK_MEMORY_MB: getattr(mm_metrics, defs.PERCENTILES_PEAK_MEMORY_MB),
"input_lens": [output.prompt_len for output in outputs],
"start_times": [output.start_time for output in outputs],
"output_lens": actual_output_lens,
@@ -3074,9 +3208,9 @@ async def benchmark(
"itls": [output.itl for output in outputs],
"generated_texts": [output.generated_text for output in outputs],
"errors": [output.error for output in outputs],
"max_output_tokens_per_s": metrics.max_output_tokens_per_s,
"max_concurrent_requests": metrics.max_concurrent_requests,
"rtfx": metrics.rtfx,
"max_output_tokens_per_s": mm_metrics.max_output_tokens_per_s,
"max_concurrent_requests": mm_metrics.max_concurrent_requests,
"rtfx": mm_metrics.rtfx,
}
for sample_count in (
"num_ttft_samples",
@@ -3150,6 +3284,21 @@ async def benchmark(
result.update(_daily_acc)
print_daily_omni_accuracy_summary(_daily_acc)
from vllm_omni.benchmarks.data_modules.videomme_eval import (
compute_videomme_accuracy_metrics,
print_videomme_accuracy_summary,
)
_save_vm = os.environ.get("VIDEOMME_SAVE_EVAL_ITEMS", "").lower() in (
"1",
"true",
"yes",
)
_vm_acc = compute_videomme_accuracy_metrics(input_requests, outputs, include_per_item=_save_vm)
if _vm_acc is not None:
result.update(_vm_acc)
print_videomme_accuracy_summary(_vm_acc)
if _seed_tts_capture_pcm_for_wer():
from vllm_omni.benchmarks.data_modules.seed_tts_eval import (
compute_seed_tts_wer_metrics,