[Benchmark][MiniCPM-o] Port Video-MME dataset support to Omni bench s… (#6987)
Signed-off-by: amy-why-3459 <wuhaiyan17@huawei.com>
This commit is contained in:
1 parent
ea62f86718
commit
9f5521351e
15 files changed
+2432
-59
No files matched your search
@@ -14,7 +14,7 @@ import time
|
||||
import traceback
|
||||
import uuid
|
||||
import wave
|
||||
from collections.abc import Callable, Iterable, Mapping
|
||||
from collections.abc import Callable, Iterable, Iterator, Mapping
|
||||
from dataclasses import dataclass, replace
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
@@ -67,6 +67,12 @@ from vllm_omni.benchmarks.data_modules.seed_tts_dataset import (
|
||||
)
|
||||
from vllm_omni.benchmarks.data_modules.sound_effect_dataset import SoundEffectDataset
|
||||
from vllm_omni.benchmarks.data_modules.ttsd_dataset import TTSDDataset
|
||||
from vllm_omni.benchmarks.data_modules.videomme_dataset import (
|
||||
VIDEOMME_DEFAULT_HF_REPO,
|
||||
VideoMMEDataset,
|
||||
VideoMMESampleRequest,
|
||||
resolve_videomme_local_root,
|
||||
)
|
||||
from vllm_omni.benchmarks.omniinteract import (
|
||||
VIDEO_FPS,
|
||||
OmniInteractBenchmarkConfig,
|
||||
@@ -283,14 +289,14 @@ def _merge_extra_body_mm_kwargs(base: dict | None, overlay: dict | None) -> dict
|
||||
return out
|
||||
|
||||
|
||||
def _attach_daily_omni_to_request_func_input(sample: SampleRequest, rfi: RequestFuncInput) -> None:
|
||||
"""Apply per-request OpenAI fields (``mm_processor_kwargs``, messages) for Daily-Omni."""
|
||||
if not isinstance(sample, DailyOmniSampleRequest):
|
||||
def _attach_omni_chat_to_request_func_input(sample: SampleRequest, rfi: RequestFuncInput) -> None:
|
||||
"""Apply per-request OpenAI fields (``mm_processor_kwargs``, messages) for Daily-Omni / Video-MME."""
|
||||
if not isinstance(sample, (DailyOmniSampleRequest, VideoMMESampleRequest)):
|
||||
return
|
||||
rfi.extra_body = _merge_extra_body_mm_kwargs(rfi.extra_body, sample.omni_extra_body)
|
||||
if sample.omni_chat_messages is not None:
|
||||
setattr(rfi, "omni_chat_messages", sample.omni_chat_messages)
|
||||
else:
|
||||
elif isinstance(sample, DailyOmniSampleRequest):
|
||||
setattr(rfi, "mm_position", sample.omni_chat_mm_position)
|
||||
|
||||
|
||||
@@ -334,6 +340,29 @@ def _attach_omniinteract_to_request_func_input(sample: SampleRequest, rfi: Reque
|
||||
setattr(rfi, "omniinteract_prepared_input", sample.omniinteract_prepared_input)
|
||||
|
||||
|
||||
def _async_limiter(max_concurrency: int | None) -> contextlib.AbstractAsyncContextManager[object]:
|
||||
if max_concurrency:
|
||||
return asyncio.Semaphore(max_concurrency)
|
||||
return contextlib.nullcontext()
|
||||
|
||||
|
||||
def _as_float(value: object, default: float = 0.0) -> float:
|
||||
"""Coerce loosely-typed metric dict values to ``float`` for mypy.
|
||||
|
||||
Strings are parsed rather than dropped: JSON-sourced session metrics such as
|
||||
``audio_duration_ms`` may arrive quoted, and silently reporting ``0.0`` would
|
||||
corrupt the derived TTFT / RTF numbers.
|
||||
"""
|
||||
if isinstance(value, bool) or value is None:
|
||||
return default
|
||||
if isinstance(value, (int, float, str)):
|
||||
try:
|
||||
return float(value)
|
||||
except ValueError:
|
||||
return default
|
||||
return default
|
||||
|
||||
|
||||
def _append_error(existing: str, message: str) -> str:
|
||||
return f"{existing}\n{message}" if existing else message
|
||||
|
||||
@@ -404,9 +433,13 @@ def _prepare_omniinteract_batch(input_requests: list[SampleRequest]) -> None:
|
||||
for sample in input_requests:
|
||||
if not isinstance(sample, OmniInteractSampleRequest):
|
||||
continue
|
||||
root = sample.omniinteract_options.output_root.resolve()
|
||||
options = sample.omniinteract_options
|
||||
case = sample.omniinteract_case
|
||||
if options is None or case is None:
|
||||
raise RuntimeError("OmniInteract benchmark input lost its dataset identity")
|
||||
root = options.output_root.resolve()
|
||||
roots.add(root)
|
||||
clear_case_artifacts(sample.omniinteract_options.output_root, sample.omniinteract_case)
|
||||
clear_case_artifacts(options.output_root, case)
|
||||
for root in roots:
|
||||
clear_batch_artifacts(root)
|
||||
|
||||
@@ -428,11 +461,55 @@ def _daily_omni_repo_from_args(args) -> str | None:
|
||||
return None
|
||||
|
||||
|
||||
def _looks_like_hf_dataset_id(value: str) -> bool:
|
||||
"""True for Hub ids such as ``org/name``; false for local paths."""
|
||||
raw = value.strip()
|
||||
if not raw or raw.startswith((".", "~", "/")) or "\\" in raw:
|
||||
return False
|
||||
parts = raw.split("/")
|
||||
return len(parts) == 2 and all(part.strip() and part.strip() not in (".", "..") for part in parts)
|
||||
|
||||
|
||||
def _videomme_repo_from_args(args, *, explicit: bool = False) -> str | None:
|
||||
"""Resolve a Hugging Face repo id for Video-MME from CLI args.
|
||||
|
||||
``--dataset-name hf`` auto-detect only recognizes the official
|
||||
``lmms-eval/Video-MME`` id so a custom Hub dataset is not silently treated
|
||||
as Video-MME. Explicit ``--dataset-name videomme`` accepts any ``org/name``
|
||||
Hub id (for ``--videomme-repo`` / ``VLLM_VIDEOMME_REPO`` overrides) and
|
||||
raises when ``--dataset-path`` is neither a local directory nor a Hub id.
|
||||
"""
|
||||
official = {p.lower() for p in VideoMMEDataset.SUPPORTED_DATASET_PATHS}
|
||||
official.add(VIDEOMME_DEFAULT_HF_REPO.lower())
|
||||
candidates: list[str] = []
|
||||
for attr in ("dataset_path", "hf_name"):
|
||||
val = getattr(args, attr, None)
|
||||
if isinstance(val, str) and val.strip():
|
||||
candidates.append(val.strip())
|
||||
for raw in candidates:
|
||||
if resolve_videomme_local_root(raw) is not None:
|
||||
continue
|
||||
if raw.lower() in official:
|
||||
return raw
|
||||
if explicit and _looks_like_hf_dataset_id(raw):
|
||||
return raw
|
||||
if explicit:
|
||||
raise ValueError(
|
||||
f"Unsupported Video-MME --dataset-path={raw!r}. Pass an existing local "
|
||||
"directory, a Hugging Face dataset id (org/name), or omit --dataset-path "
|
||||
f"to use {VIDEOMME_DEFAULT_HF_REPO}."
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def get_samples(args, tokenizer):
|
||||
# Daily-Omni: explicit dataset name, or hf + matching path/hf-name
|
||||
is_daily_omni = args.dataset_name == "daily-omni" or (
|
||||
args.dataset_name == "hf" and _daily_omni_repo_from_args(args) is not None
|
||||
)
|
||||
is_videomme = args.dataset_name == "videomme" or (
|
||||
args.dataset_name == "hf" and _videomme_repo_from_args(args) is not None
|
||||
)
|
||||
is_seed_tts = args.dataset_name in (
|
||||
"seed-tts",
|
||||
"seed-tts-text",
|
||||
@@ -449,7 +526,7 @@ def get_samples(args, tokenizer):
|
||||
"openai-realtime-duplex",
|
||||
"daily-omni",
|
||||
]
|
||||
is_omni_dataset = is_daily_omni or is_seed_tts or is_omniinteract or args.dataset_name == "random-mm"
|
||||
is_omni_dataset = is_daily_omni or is_videomme or is_seed_tts or is_omniinteract or args.dataset_name == "random-mm"
|
||||
|
||||
if not is_omni_backend and not is_omni_dataset:
|
||||
# Not an omni-related request, delegate to original implementation
|
||||
@@ -625,6 +702,54 @@ def get_samples(args, tokenizer):
|
||||
)
|
||||
return input_requests
|
||||
|
||||
if is_videomme:
|
||||
if args.backend not in ["openai-chat-omni", "daily-omni"]:
|
||||
raise ValueError(
|
||||
f"Video-MME dataset requires a multimodal backend that supports video. "
|
||||
f"Got backend='{args.backend}'. Please use '--backend openai-chat-omni'"
|
||||
)
|
||||
|
||||
# Resolve the source identity here; the dataset owns loading and extraction.
|
||||
local_root = resolve_videomme_local_root(getattr(args, "dataset_path", None)) or (
|
||||
resolve_videomme_local_root(getattr(args, "hf_name", None))
|
||||
)
|
||||
source = (
|
||||
str(local_root)
|
||||
if local_root is not None
|
||||
else (_videomme_repo_from_args(args, explicit=args.dataset_name == "videomme") or VIDEOMME_DEFAULT_HF_REPO)
|
||||
)
|
||||
dataset = VideoMMEDataset(
|
||||
parquet_path=getattr(args, "videomme_parquet", None),
|
||||
dataset_path=source,
|
||||
dataset_split=getattr(args, "hf_split", None) or "test",
|
||||
dataset_subset=getattr(args, "hf_subset", None),
|
||||
random_seed=args.seed,
|
||||
video_dir=getattr(args, "videomme_video_dir", None),
|
||||
subtitle_dir=getattr(args, "videomme_subtitle_dir", None),
|
||||
pack_mode=getattr(args, "videomme_pack_mode", "minicpm-frames"),
|
||||
max_frames=getattr(args, "videomme_max_frames", None),
|
||||
duration_filter=getattr(args, "videomme_duration", "all"),
|
||||
use_subtitle=getattr(args, "videomme_use_subtitle", False),
|
||||
inline_local_video=getattr(args, "videomme_inline_local_video", False),
|
||||
trust_remote_code=getattr(args, "trust_remote_code", False),
|
||||
no_stream=getattr(args, "no_stream", False),
|
||||
disable_shuffle=getattr(args, "disable_shuffle", False),
|
||||
)
|
||||
|
||||
out_len = getattr(args, "output_len", None)
|
||||
if out_len is None:
|
||||
out_len = getattr(args, "hf_output_len", None)
|
||||
if out_len is None:
|
||||
out_len = VideoMMEDataset.DEFAULT_OUTPUT_LEN
|
||||
|
||||
return dataset.sample(
|
||||
tokenizer=tokenizer,
|
||||
num_requests=args.num_prompts,
|
||||
output_len=out_len,
|
||||
request_id_prefix=args.request_id_prefix,
|
||||
no_oversample=args.no_oversample,
|
||||
)
|
||||
|
||||
if is_seed_tts:
|
||||
if args.backend not in (
|
||||
"openai-audio-speech",
|
||||
@@ -711,7 +836,7 @@ datasets.get_samples = get_samples
|
||||
|
||||
_serve_mod = sys.modules.get("vllm.benchmarks.serve")
|
||||
if _serve_mod is not None:
|
||||
_serve_mod.get_samples = get_samples
|
||||
setattr(_serve_mod, "get_samples", get_samples)
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -2637,7 +2762,9 @@ async def async_request_openai_realtime_duplex(
|
||||
output.ttft = (metric_mean(session_metrics.get("ttft_ms")) or 0.0) / 1000.0
|
||||
output.audio_ttfp = (metric_mean(session_metrics.get("ttfp_ms")) or 0.0) / 1000.0
|
||||
output.audio_rtf = metric_mean(session_metrics.get("rtf")) or 0.0
|
||||
output.audio_duration = sum(float(metric.get("audio_duration_ms") or 0.0) for metric in turn_metrics) / 1000.0
|
||||
output.audio_duration = (
|
||||
sum((_as_float(metric.get("audio_duration_ms")) for metric in turn_metrics), start=0.0) / 1000.0
|
||||
)
|
||||
output.audio_frames = int(output.audio_duration * _SEED_TTS_OUTPUT_SAMPLE_RATE_HZ)
|
||||
output.latency = request_finished_at - output.start_time
|
||||
output.tts_turn_pcm_bytes = turn_pcm_bytes
|
||||
@@ -2811,7 +2938,7 @@ async def benchmark(
|
||||
extra_body=test_extra_body,
|
||||
chat_messages=test_chat_messages,
|
||||
)
|
||||
_attach_daily_omni_to_request_func_input(input_requests[0], test_input)
|
||||
_attach_omni_chat_to_request_func_input(input_requests[0], test_input)
|
||||
_attach_seed_tts_to_request_func_input(input_requests[0], test_input)
|
||||
_attach_omniinteract_to_request_func_input(input_requests[0], test_input)
|
||||
|
||||
@@ -2836,7 +2963,7 @@ async def benchmark(
|
||||
if num_warmups > 0:
|
||||
print(f"Warming up with {num_warmups} requests...")
|
||||
warmup_pbar = None if disable_tqdm else tqdm(total=num_warmups)
|
||||
warmup_semaphore = asyncio.Semaphore(max_concurrency) if max_concurrency else contextlib.nullcontext()
|
||||
warmup_semaphore = _async_limiter(max_concurrency)
|
||||
warmup_tasks = []
|
||||
|
||||
async def warmup_limited_request_func():
|
||||
@@ -2854,12 +2981,13 @@ async def benchmark(
|
||||
|
||||
print("Starting main benchmark run...")
|
||||
|
||||
lora_iter: Iterator[str] | None = None
|
||||
if lora_modules:
|
||||
lora_modules_list = list(lora_modules)
|
||||
if lora_assignment == "round-robin":
|
||||
lora_modules = iter([lora_modules_list[i % len(lora_modules_list)] for i in range(len(input_requests))])
|
||||
lora_iter = iter([lora_modules_list[i % len(lora_modules_list)] for i in range(len(input_requests))])
|
||||
else:
|
||||
lora_modules = iter([random.choice(lora_modules_list) for _ in range(len(input_requests))])
|
||||
lora_iter = iter([random.choice(lora_modules_list) for _ in range(len(input_requests))])
|
||||
|
||||
if profile:
|
||||
print("Starting profiler...")
|
||||
@@ -2877,7 +3005,7 @@ async def benchmark(
|
||||
extra_body=test_extra_body,
|
||||
chat_messages=test_chat_messages,
|
||||
)
|
||||
_attach_daily_omni_to_request_func_input(input_requests[0], profile_input)
|
||||
_attach_omni_chat_to_request_func_input(input_requests[0], profile_input)
|
||||
_attach_seed_tts_to_request_func_input(input_requests[0], profile_input)
|
||||
profile_output = await request_func(request_func_input=profile_input, session=session)
|
||||
if profile_output.success:
|
||||
@@ -2898,7 +3026,7 @@ async def benchmark(
|
||||
|
||||
pbar = None if disable_tqdm else tqdm(total=len(input_requests))
|
||||
|
||||
semaphore = asyncio.Semaphore(max_concurrency) if max_concurrency else contextlib.nullcontext()
|
||||
semaphore = _async_limiter(max_concurrency)
|
||||
|
||||
async def limited_request_func(request_func_input, session, pbar):
|
||||
async with semaphore:
|
||||
@@ -2969,8 +3097,8 @@ async def benchmark(
|
||||
)
|
||||
per_request_extra_body = _merge_overrides(extra_body, request.request_overrides)
|
||||
req_model_id, req_model_name = model_id, model_name
|
||||
if lora_modules:
|
||||
req_lora_module = next(lora_modules)
|
||||
if lora_iter is not None:
|
||||
req_lora_module = next(lora_iter)
|
||||
req_model_id, req_model_name = req_lora_module, req_lora_module
|
||||
|
||||
request_func_input = RequestFuncInput(
|
||||
@@ -2988,7 +3116,7 @@ async def benchmark(
|
||||
request_id=request_id,
|
||||
chat_messages=request.chat_messages,
|
||||
)
|
||||
_attach_daily_omni_to_request_func_input(request, request_func_input)
|
||||
_attach_omni_chat_to_request_func_input(request, request_func_input)
|
||||
_attach_seed_tts_to_request_func_input(request, request_func_input)
|
||||
_attach_omniinteract_to_request_func_input(request, request_func_input)
|
||||
tasks.append(
|
||||
@@ -3007,6 +3135,8 @@ async def benchmark(
|
||||
|
||||
omniinteract_summary = _finalize_omniinteract_batch(input_requests, outputs)
|
||||
|
||||
metrics: Any
|
||||
actual_output_lens: list[int] | int
|
||||
if task_type == TaskType.GENERATION:
|
||||
metrics, actual_output_lens = calculate_metrics(
|
||||
input_requests=input_requests,
|
||||
@@ -3031,6 +3161,10 @@ async def benchmark(
|
||||
actual_output_lens = 0
|
||||
|
||||
if isinstance(metrics, MultiModalsBenchmarkMetrics):
|
||||
# ``make_dataclass`` types this class as ``type``, so isinstance()
|
||||
# narrows to ``object`` and attribute access fails. Keep the runtime
|
||||
# check and read fields through ``Any``.
|
||||
mm_metrics: Any = metrics
|
||||
|
||||
def measured_ttft(output: RequestFuncOutput) -> float | None:
|
||||
session_metrics = getattr(output, "duplex_session_metrics", None)
|
||||
@@ -3040,33 +3174,33 @@ async def benchmark(
|
||||
|
||||
result = {
|
||||
"duration": benchmark_duration,
|
||||
"completed": metrics.completed,
|
||||
"failed": metrics.failed,
|
||||
"total_input_tokens": metrics.total_input,
|
||||
"total_output_tokens": metrics.total_output,
|
||||
"request_throughput": metrics.request_throughput,
|
||||
"request_goodput": metrics.request_goodput if goodput_config_dict else None,
|
||||
"output_throughput": metrics.output_throughput,
|
||||
"total_token_throughput": metrics.total_token_throughput,
|
||||
defs.TOTAL_AUDIO_DURATION_S: getattr(metrics, defs.TOTAL_AUDIO_DURATION_S),
|
||||
defs.TOTAL_AUDIO_FRAMES: getattr(metrics, defs.TOTAL_AUDIO_FRAMES),
|
||||
defs.AUDIO_THROUGHPUT: getattr(metrics, defs.AUDIO_THROUGHPUT),
|
||||
defs.TOTAL_IMAGES: getattr(metrics, defs.TOTAL_IMAGES),
|
||||
defs.IMAGE_THROUGHPUT: getattr(metrics, defs.IMAGE_THROUGHPUT),
|
||||
defs.AVERAGE_PIXELS_PER_IMAGE: getattr(metrics, defs.AVERAGE_PIXELS_PER_IMAGE),
|
||||
defs.MEAN_DENOISE_STEP_LATENCY_MS: getattr(metrics, defs.MEAN_DENOISE_STEP_LATENCY_MS),
|
||||
defs.TOTAL_VIDEO_DURATION_S: getattr(metrics, defs.TOTAL_VIDEO_DURATION_S),
|
||||
defs.TOTAL_VIDEO_FRAMES: getattr(metrics, defs.TOTAL_VIDEO_FRAMES),
|
||||
defs.VIDEO_THROUGHPUT: getattr(metrics, defs.VIDEO_THROUGHPUT),
|
||||
defs.MEAN_VIDEO_RTF: getattr(metrics, defs.MEAN_VIDEO_RTF),
|
||||
defs.MEDIAN_VIDEO_RTF: getattr(metrics, defs.MEDIAN_VIDEO_RTF),
|
||||
defs.PERCENTILES_VIDEO_RTF: getattr(metrics, defs.PERCENTILES_VIDEO_RTF),
|
||||
defs.MEAN_VIDEO_GENERATION_MS: getattr(metrics, defs.MEAN_VIDEO_GENERATION_MS),
|
||||
defs.MEDIAN_VIDEO_GENERATION_MS: getattr(metrics, defs.MEDIAN_VIDEO_GENERATION_MS),
|
||||
defs.PERCENTILES_VIDEO_GENERATION_MS: getattr(metrics, defs.PERCENTILES_VIDEO_GENERATION_MS),
|
||||
defs.MEAN_PEAK_MEMORY_MB: getattr(metrics, defs.MEAN_PEAK_MEMORY_MB),
|
||||
defs.MEDIAN_PEAK_MEMORY_MB: getattr(metrics, defs.MEDIAN_PEAK_MEMORY_MB),
|
||||
defs.PERCENTILES_PEAK_MEMORY_MB: getattr(metrics, defs.PERCENTILES_PEAK_MEMORY_MB),
|
||||
"completed": mm_metrics.completed,
|
||||
"failed": mm_metrics.failed,
|
||||
"total_input_tokens": mm_metrics.total_input,
|
||||
"total_output_tokens": mm_metrics.total_output,
|
||||
"request_throughput": mm_metrics.request_throughput,
|
||||
"request_goodput": mm_metrics.request_goodput if goodput_config_dict else None,
|
||||
"output_throughput": mm_metrics.output_throughput,
|
||||
"total_token_throughput": mm_metrics.total_token_throughput,
|
||||
defs.TOTAL_AUDIO_DURATION_S: getattr(mm_metrics, defs.TOTAL_AUDIO_DURATION_S),
|
||||
defs.TOTAL_AUDIO_FRAMES: getattr(mm_metrics, defs.TOTAL_AUDIO_FRAMES),
|
||||
defs.AUDIO_THROUGHPUT: getattr(mm_metrics, defs.AUDIO_THROUGHPUT),
|
||||
defs.TOTAL_IMAGES: getattr(mm_metrics, defs.TOTAL_IMAGES),
|
||||
defs.IMAGE_THROUGHPUT: getattr(mm_metrics, defs.IMAGE_THROUGHPUT),
|
||||
defs.AVERAGE_PIXELS_PER_IMAGE: getattr(mm_metrics, defs.AVERAGE_PIXELS_PER_IMAGE),
|
||||
defs.MEAN_DENOISE_STEP_LATENCY_MS: getattr(mm_metrics, defs.MEAN_DENOISE_STEP_LATENCY_MS),
|
||||
defs.TOTAL_VIDEO_DURATION_S: getattr(mm_metrics, defs.TOTAL_VIDEO_DURATION_S),
|
||||
defs.TOTAL_VIDEO_FRAMES: getattr(mm_metrics, defs.TOTAL_VIDEO_FRAMES),
|
||||
defs.VIDEO_THROUGHPUT: getattr(mm_metrics, defs.VIDEO_THROUGHPUT),
|
||||
defs.MEAN_VIDEO_RTF: getattr(mm_metrics, defs.MEAN_VIDEO_RTF),
|
||||
defs.MEDIAN_VIDEO_RTF: getattr(mm_metrics, defs.MEDIAN_VIDEO_RTF),
|
||||
defs.PERCENTILES_VIDEO_RTF: getattr(mm_metrics, defs.PERCENTILES_VIDEO_RTF),
|
||||
defs.MEAN_VIDEO_GENERATION_MS: getattr(mm_metrics, defs.MEAN_VIDEO_GENERATION_MS),
|
||||
defs.MEDIAN_VIDEO_GENERATION_MS: getattr(mm_metrics, defs.MEDIAN_VIDEO_GENERATION_MS),
|
||||
defs.PERCENTILES_VIDEO_GENERATION_MS: getattr(mm_metrics, defs.PERCENTILES_VIDEO_GENERATION_MS),
|
||||
defs.MEAN_PEAK_MEMORY_MB: getattr(mm_metrics, defs.MEAN_PEAK_MEMORY_MB),
|
||||
defs.MEDIAN_PEAK_MEMORY_MB: getattr(mm_metrics, defs.MEDIAN_PEAK_MEMORY_MB),
|
||||
defs.PERCENTILES_PEAK_MEMORY_MB: getattr(mm_metrics, defs.PERCENTILES_PEAK_MEMORY_MB),
|
||||
"input_lens": [output.prompt_len for output in outputs],
|
||||
"start_times": [output.start_time for output in outputs],
|
||||
"output_lens": actual_output_lens,
|
||||
@@ -3074,9 +3208,9 @@ async def benchmark(
|
||||
"itls": [output.itl for output in outputs],
|
||||
"generated_texts": [output.generated_text for output in outputs],
|
||||
"errors": [output.error for output in outputs],
|
||||
"max_output_tokens_per_s": metrics.max_output_tokens_per_s,
|
||||
"max_concurrent_requests": metrics.max_concurrent_requests,
|
||||
"rtfx": metrics.rtfx,
|
||||
"max_output_tokens_per_s": mm_metrics.max_output_tokens_per_s,
|
||||
"max_concurrent_requests": mm_metrics.max_concurrent_requests,
|
||||
"rtfx": mm_metrics.rtfx,
|
||||
}
|
||||
for sample_count in (
|
||||
"num_ttft_samples",
|
||||
@@ -3150,6 +3284,21 @@ async def benchmark(
|
||||
result.update(_daily_acc)
|
||||
print_daily_omni_accuracy_summary(_daily_acc)
|
||||
|
||||
from vllm_omni.benchmarks.data_modules.videomme_eval import (
|
||||
compute_videomme_accuracy_metrics,
|
||||
print_videomme_accuracy_summary,
|
||||
)
|
||||
|
||||
_save_vm = os.environ.get("VIDEOMME_SAVE_EVAL_ITEMS", "").lower() in (
|
||||
"1",
|
||||
"true",
|
||||
"yes",
|
||||
)
|
||||
_vm_acc = compute_videomme_accuracy_metrics(input_requests, outputs, include_per_item=_save_vm)
|
||||
if _vm_acc is not None:
|
||||
result.update(_vm_acc)
|
||||
print_videomme_accuracy_summary(_vm_acc)
|
||||
|
||||
if _seed_tts_capture_pcm_for_wer():
|
||||
from vllm_omni.benchmarks.data_modules.seed_tts_eval import (
|
||||
compute_seed_tts_wer_metrics,
|
||||
|
||||
Reference in new issue
Block a user