[Benchmark]Omni-modality model accuracy benchmark(Daily-Omni & seed-tts-eval) (#2558)

Signed-off-by: amy-why-3459 <wuhaiyan17@huawei.com>
This commit is contained in:
amy-why-3459 authored and GitHub committed 2026-04-15 11:07:35 +08:00
1 parent a782ae4780
commit 227bab3038
10 files changed
+3041 -9

No files matched your search

+324 -8
View File
@@ -6,6 +6,7 @@ import json
import os
import random
import ssl
import sys
import time
import traceback
from collections.abc import Iterable
@@ -33,15 +34,245 @@ from vllm.logger import init_logger
from vllm.tokenizers import TokenizerLike
logger = init_logger(__name__)
from vllm_omni.benchmarks.data_modules.daily_omni_dataset import DailyOmniDataset, DailyOmniSampleRequest
from vllm_omni.benchmarks.data_modules.random_multi_modal_dataset import OmniRandomMultiModalDataset
from vllm_omni.benchmarks.data_modules.seed_tts_dataset import (
SEED_TTS_DEFAULT_OMNI_SYSTEM_PROMPT,
SeedTTSDataset,
SeedTTSSampleRequest,
)
get_samples_old = datasets.get_samples
_DEFAULT_DAILY_OMNI_REPO = "liarliar/Daily-Omni"
def _seed_tts_capture_pcm_for_wer() -> bool:
return os.environ.get("SEED_TTS_WER_EVAL", "").lower() in (
"1",
"true",
"yes",
)
def _merge_extra_body_mm_kwargs(base: dict | None, overlay: dict | None) -> dict | None:
"""Shallow-merge ``extra_body`` dicts; deep-merge ``mm_processor_kwargs`` if both set."""
if not base and not overlay:
return None
out = dict(base or {})
if not overlay:
return out
for k, v in overlay.items():
if k == "mm_processor_kwargs" and isinstance(v, dict):
prev = out.get("mm_processor_kwargs")
merged_kw = {**(prev if isinstance(prev, dict) else {}), **v}
out["mm_processor_kwargs"] = merged_kw
else:
out[k] = v
return out
def _attach_daily_omni_to_request_func_input(sample: SampleRequest, rfi: RequestFuncInput) -> None:
"""Apply per-request OpenAI fields (``mm_processor_kwargs``, messages) for Daily-Omni."""
if not isinstance(sample, DailyOmniSampleRequest):
return
rfi.extra_body = _merge_extra_body_mm_kwargs(rfi.extra_body, sample.omni_extra_body)
if sample.omni_chat_messages is not None:
setattr(rfi, "omni_chat_messages", sample.omni_chat_messages)
else:
setattr(rfi, "mm_position", sample.omni_chat_mm_position)
def _attach_seed_tts_to_request_func_input(sample: SampleRequest, rfi: RequestFuncInput) -> None:
"""Merge Seed-TTS per-row TTS fields (ref_audio, ref_text, task_type, …) into ``extra_body``.
Used by both ``/v1/audio/speech`` and ``/v1/chat/completions`` (flattened into JSON body).
For ``openai-chat-omni``, also sets ``omni_chat_messages`` (system + user) so Qwen3-Omni
follows the same role layout as official TTS / multimodal demos. ``/v1/audio/speech`` ignores
``messages`` and only uses ``input`` + body fields.
Flags ``openai-chat-omni`` to request audio output and optionally export PCM for WER.
"""
if not isinstance(sample, SeedTTSSampleRequest):
return
ex = sample.seed_tts_speech_extra
if not ex:
return
base = dict(rfi.extra_body) if rfi.extra_body else {}
base.update(ex)
rfi.extra_body = base
# Used by request funcs to force streaming TTS behavior and to export PCM when WER is on.
setattr(rfi, "seed_tts_row", True)
sys_prompt = (sample.seed_tts_system_prompt or "").strip() or SEED_TTS_DEFAULT_OMNI_SYSTEM_PROMPT
setattr(
rfi,
"omni_chat_messages",
[
{"role": "system", "content": [{"type": "text", "text": sys_prompt}]},
{"role": "user", "content": [{"type": "text", "text": sample.prompt}]},
],
)
def _daily_omni_repo_from_args(args) -> str | None:
"""Resolve HuggingFace repo id for Daily-Omni from CLI args.
vLLM allows ``--dataset-path`` to be a local path while the real HF id is
passed via ``--hf-name``. Upstream ``get_samples`` for ``hf`` only matches
a fixed elif-chain and never discovers Omni's loader, so we must detect
Daily-Omni here using either field.
"""
dp = getattr(args, "dataset_path", None)
hn = getattr(args, "hf_name", None)
if dp in DailyOmniDataset.SUPPORTED_DATASET_PATHS:
return dp
if hn in DailyOmniDataset.SUPPORTED_DATASET_PATHS:
return hn
return None
def get_samples(args, tokenizer):
if args.backend not in ["openai-chat-omni", "openai-audio-speech"]:
# Daily-Omni: explicit dataset name, or hf + matching path/hf-name
is_daily_omni = args.dataset_name == "daily-omni" or (
args.dataset_name == "hf" and _daily_omni_repo_from_args(args) is not None
)
is_seed_tts = args.dataset_name == "seed-tts"
# Check if we need to handle omni-related backends/datasets
is_omni_backend = args.backend in ["openai-chat-omni", "openai-audio-speech", "daily-omni"]
is_omni_dataset = is_daily_omni or is_seed_tts or args.dataset_name == "random-mm"
if not is_omni_backend and not is_omni_dataset:
# Not an omni-related request, delegate to original implementation
return get_samples_old(args, tokenizer)
elif args.dataset_name == "random-mm":
# Handle Daily-Omni dataset
if is_daily_omni:
# Support:
# --dataset-name daily-omni [--dataset-path liarliar/Daily-Omni]
# --dataset-name daily-omni --daily-omni-qa-json /path/to/qa.json (offline QA)
# --dataset-name hf --dataset-path liarliar/Daily-Omni
# --dataset-name hf --hf-name liarliar/Daily-Omni (dataset-path may be local)
# Validate backend supports multimodal (video)
if args.backend not in ["openai-chat-omni", "daily-omni"]:
raise ValueError(
f"Daily-Omni dataset requires a multimodal backend that supports video. "
f"Got backend='{args.backend}'. Please use '--backend openai-chat-omni'"
)
# Determine video directory if specified (for local video files)
video_dir = getattr(args, "daily_omni_video_dir", None)
# Get HF split (default to "train"; unused when loading from local qa.json)
dataset_split = getattr(args, "hf_split", None) or "train"
qa_json = getattr(args, "daily_omni_qa_json", None)
if isinstance(qa_json, str):
qa_json = qa_json.strip() or None
if qa_json is not None:
logger.info(
"Loading Daily-Omni dataset: qa_json=%s, video_dir=%s (Hub not used for QA)",
qa_json,
video_dir,
)
dataset = DailyOmniDataset(
qa_json_path=qa_json,
dataset_path=None,
dataset_split=dataset_split,
random_seed=args.seed,
video_dir=video_dir,
input_mode=getattr(args, "daily_omni_input_mode", "all"),
inline_local_video=getattr(args, "daily_omni_inline_local_video", False),
trust_remote_code=getattr(args, "trust_remote_code", False),
disable_shuffle=getattr(args, "disable_shuffle", False),
)
else:
repo_id = _daily_omni_repo_from_args(args)
if args.dataset_name == "daily-omni":
if repo_id is None:
repo_id = _DEFAULT_DAILY_OMNI_REPO
elif repo_id is None:
raise ValueError(
"Daily-Omni with --dataset-name hf requires "
f"--dataset-path {_DEFAULT_DAILY_OMNI_REPO} or "
f"--hf-name {_DEFAULT_DAILY_OMNI_REPO}."
)
logger.info(
"Loading Daily-Omni dataset: hf_repo=%s, split=%s, video_dir=%s",
repo_id,
dataset_split,
video_dir,
)
dataset = DailyOmniDataset(
dataset_path=repo_id,
dataset_split=dataset_split,
dataset_subset=getattr(args, "hf_subset", None),
random_seed=args.seed,
video_dir=video_dir,
input_mode=getattr(args, "daily_omni_input_mode", "all"),
inline_local_video=getattr(args, "daily_omni_inline_local_video", False),
trust_remote_code=getattr(args, "trust_remote_code", False),
no_stream=getattr(args, "no_stream", False),
disable_shuffle=getattr(args, "disable_shuffle", False),
)
out_len = getattr(args, "output_len", None)
if out_len is None:
out_len = getattr(args, "hf_output_len", None)
if out_len is None:
out_len = DailyOmniDataset.DEFAULT_OUTPUT_LEN
input_requests = dataset.sample(
tokenizer=tokenizer,
num_requests=args.num_prompts,
output_len=out_len,
request_id_prefix=args.request_id_prefix,
no_oversample=args.no_oversample,
)
return input_requests
if is_seed_tts:
if args.backend not in ("openai-audio-speech", "openai-chat-omni"):
raise ValueError(
"Seed-TTS requires --backend openai-audio-speech (POST /v1/audio/speech) or "
"--backend openai-chat-omni (POST /v1/chat/completions with ref_audio/ref_text). "
f"Got backend={args.backend!r}."
)
repo_id = getattr(args, "dataset_path", None) or getattr(args, "hf_name", None)
if not repo_id:
raise ValueError(
"Seed-TTS requires --dataset-path (HF dataset repo id or local directory) or "
"--hf-name for the Hub dataset id."
)
dataset = SeedTTSDataset(
dataset_path=repo_id,
random_seed=args.seed,
locale=getattr(args, "seed_tts_locale", "en"),
inline_ref_audio=not getattr(args, "seed_tts_file_ref_audio", False),
seed_tts_root=getattr(args, "seed_tts_root", None),
system_prompt=getattr(args, "seed_tts_system_prompt", None),
disable_shuffle=getattr(args, "disable_shuffle", False),
)
out_len = getattr(args, "output_len", None)
if out_len is None:
out_len = getattr(args, "hf_output_len", None)
if out_len is None:
out_len = SeedTTSDataset.DEFAULT_OUTPUT_LEN
return dataset.sample(
tokenizer=tokenizer,
num_requests=args.num_prompts,
output_len=out_len,
request_id_prefix=args.request_id_prefix,
no_oversample=args.no_oversample,
)
# Handle random-mm dataset (Omni's synthetic multimodal dataset)
if args.dataset_name == "random-mm":
dataset = OmniRandomMultiModalDataset(random_seed=args.seed, dataset_path=args.dataset_path)
input_requests = dataset.sample(
tokenizer=tokenizer,
@@ -64,6 +295,10 @@ def get_samples(args, tokenizer):
datasets.get_samples = get_samples
_serve_mod = sys.modules.get("vllm.benchmarks.serve")
if _serve_mod is not None:
_serve_mod.get_samples = get_samples
@dataclass
class MixRequestFuncOutput(RequestFuncOutput):
@@ -72,6 +307,9 @@ class MixRequestFuncOutput(RequestFuncOutput):
audio_frames: int = 0
audio_rtf: float = 0.0
text_latency: float = 0.0
#: Raw PCM s16le mono at 24 kHz for Seed-TTS WER: from ``/v1/audio/speech`` stream or
#: resampled export after ``openai-chat-omni`` audio deltas.
tts_output_pcm_bytes: bytes | None = None
async def async_request_openai_chat_omni_completions(
@@ -83,13 +321,17 @@ async def async_request_openai_chat_omni_completions(
api_url = request_func_input.api_url
_validate_api_url(api_url, "OpenAI Chat Completions API", "chat/completions")
content = _get_chat_content(request_func_input, mm_position=mm_position)
omni_messages = getattr(request_func_input, "omni_chat_messages", None)
if omni_messages is not None:
messages_payload = omni_messages
else:
effective_mm_position = getattr(request_func_input, "mm_position", mm_position)
content = _get_chat_content(request_func_input, mm_position=effective_mm_position)
messages_payload = [{"role": "user", "content": content}]
payload = {
"model": request_func_input.model_name if request_func_input.model_name else request_func_input.model,
"messages": [
{"role": "user", "content": content},
],
"messages": messages_payload,
"temperature": 0.0,
"max_tokens": request_func_input.output_len,
"stream": True,
@@ -98,6 +340,10 @@ async def async_request_openai_chat_omni_completions(
},
}
_update_payload_common(payload, request_func_input)
# Seed-TTS via chat: voice-clone fields live on the body; ensure audio is streamed.
if getattr(request_func_input, "seed_tts_row", False):
if payload.get("modalities") is None:
payload["modalities"] = ["text", "audio"]
response_format = payload.get("response_format", "wav")
if response_format == "pcm":
@@ -167,7 +413,10 @@ async def async_request_openai_chat_omni_completions(
data = json.loads(chunk)
if choices := data.get("choices"):
modality = data.get("modality")
content = choices[0]["delta"].get("content")
delta = choices[0].get("delta") or {}
content = delta.get("content")
if not content and isinstance(delta.get("audio"), dict):
content = delta["audio"].get("data")
if modality == "text":
# First token
if ttft == 0.0:
@@ -182,7 +431,7 @@ async def async_request_openai_chat_omni_completions(
if output.audio_ttfp == 0.0:
output.audio_ttfp = timestamp - st
audio_generate_time = timestamp - st
if content != "":
if content:
audio_bytes = base64.b64decode(content)
seg = AudioSegment.from_file(io.BytesIO(audio_bytes))
if seg is not None:
@@ -214,6 +463,12 @@ async def async_request_openai_chat_omni_completions(
else:
output.audio_rtf = 0
logger.warning("Audio duration is zero")
if _seed_tts_capture_pcm_for_wer() and getattr(request_func_input, "seed_tts_row", False):
try:
seg = generated_audio.set_frame_rate(24000).set_channels(1).set_sample_width(2)
output.tts_output_pcm_bytes = bytes(seg.raw_data)
except Exception as ex:
logger.warning("seed_tts WER PCM export failed: %s", ex)
output.success = True
else:
output.error = response.reason or ""
@@ -268,6 +523,10 @@ async def async_request_openai_audio_speech(
"response_format": "pcm",
}
_update_payload_common(payload, request_func_input)
# Seed-TTS + WER: ``--extra-body`` may set stream=false / other formats; speech must stream PCM.
if getattr(request_func_input, "seed_tts_row", False) and _seed_tts_capture_pcm_for_wer():
payload["stream"] = True
payload["response_format"] = "pcm"
headers = {
"Content-Type": "application/json",
@@ -286,6 +545,8 @@ async def async_request_openai_audio_speech(
st = time.perf_counter()
output.start_time = st
total_pcm_bytes = 0
capture_wer_pcm = _seed_tts_capture_pcm_for_wer() and getattr(request_func_input, "seed_tts_row", False)
pcm_capture = bytearray() if capture_wer_pcm else None
try:
async with session.post(url=api_url, json=payload, headers=headers) as response:
if response.status == 200:
@@ -297,6 +558,8 @@ async def async_request_openai_audio_speech(
output.audio_ttfp = timestamp - st
output.ttft = output.audio_ttfp
total_pcm_bytes += len(chunk)
if pcm_capture is not None:
pcm_capture.extend(chunk)
end_time = time.perf_counter()
output.latency = end_time - st
@@ -309,6 +572,16 @@ async def async_request_openai_audio_speech(
else:
output.audio_rtf = 0
logger.warning("Audio duration is zero")
if pcm_capture is not None and pcm_capture:
output.tts_output_pcm_bytes = bytes(pcm_capture)
elif capture_wer_pcm:
ct = response.headers.get("Content-Type", "")
logger.warning(
"Seed-TTS WER: HTTP 200 but no PCM bytes (Content-Type=%r, url=%s). "
"Check stream=true and response_format=pcm on the server.",
ct,
api_url,
)
output.success = True
else:
output.error = response.reason or ""
@@ -331,6 +604,12 @@ ASYNC_REQUEST_FUNCS["openai-audio-speech"] = async_request_openai_audio_speech
if "openai-audio-speech" not in OPENAI_COMPATIBLE_BACKENDS:
OPENAI_COMPATIBLE_BACKENDS.append("openai-audio-speech")
# Daily-Omni backend for audio-visual reasoning benchmark
# Reuses openai-chat-omni completions for video+text understanding
ASYNC_REQUEST_FUNCS["daily-omni"] = async_request_openai_chat_omni_completions
if "daily-omni" not in OPENAI_COMPATIBLE_BACKENDS:
OPENAI_COMPATIBLE_BACKENDS.append("daily-omni")
# ruff: noqa: E402
# Prevent import order from causing patch failures
from vllm.benchmarks import serve
@@ -422,6 +701,8 @@ async def benchmark(
extra_headers=extra_headers,
extra_body=extra_body,
)
_attach_daily_omni_to_request_func_input(input_requests[0], test_input)
_attach_seed_tts_to_request_func_input(input_requests[0], test_input)
if ready_check_timeout_sec > 0:
test_output = await wait_for_endpoint(
@@ -484,6 +765,8 @@ async def benchmark(
extra_headers=extra_headers,
extra_body=extra_body,
)
_attach_daily_omni_to_request_func_input(input_requests[0], profile_input)
_attach_seed_tts_to_request_func_input(input_requests[0], profile_input)
profile_output = await request_func(request_func_input=profile_input, session=session)
if profile_output.success:
print("Profiler started")
@@ -564,6 +847,8 @@ async def benchmark(
extra_body=extra_body,
request_id=request_id,
)
_attach_daily_omni_to_request_func_input(request, request_func_input)
_attach_seed_tts_to_request_func_input(request, request_func_input)
tasks.append(
asyncio.create_task(limited_request_func(request_func_input=request_func_input, session=session, pbar=pbar))
)
@@ -631,6 +916,37 @@ async def benchmark(
"errors": [output.error for output in outputs],
}
from vllm_omni.benchmarks.data_modules.daily_omni_eval import (
compute_daily_omni_accuracy_metrics,
print_daily_omni_accuracy_summary,
)
_save_items = os.environ.get("DAILY_OMNI_SAVE_EVAL_ITEMS", "").lower() in (
"1",
"true",
"yes",
)
_daily_acc = compute_daily_omni_accuracy_metrics(input_requests, outputs, include_per_item=_save_items)
if _daily_acc is not None:
result.update(_daily_acc)
print_daily_omni_accuracy_summary(_daily_acc)
if _seed_tts_capture_pcm_for_wer():
from vllm_omni.benchmarks.data_modules.seed_tts_eval import (
compute_seed_tts_wer_metrics,
print_seed_tts_wer_summary,
)
_save_wer = os.environ.get("SEED_TTS_WER_SAVE_ITEMS", "").lower() in (
"1",
"true",
"yes",
)
_wer_m = compute_seed_tts_wer_metrics(input_requests, outputs, include_per_item=_save_wer)
if _wer_m is not None:
result.update(_wer_m)
print_seed_tts_wer_summary(_wer_m)
if rps_change_events:
result["rps_change_events"] = rps_change_events