[Benchmark]Omni-modality model accuracy benchmark(Daily-Omni & seed-tts-eval) (#2558)
Signed-off-by: amy-why-3459 <wuhaiyan17@huawei.com>
This commit is contained in:
1 parent
a782ae4780
commit
227bab3038
10 files changed
+3041
-9
No files matched your search
@@ -6,6 +6,7 @@ import json
|
||||
import os
|
||||
import random
|
||||
import ssl
|
||||
import sys
|
||||
import time
|
||||
import traceback
|
||||
from collections.abc import Iterable
|
||||
@@ -33,15 +34,245 @@ from vllm.logger import init_logger
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
from vllm_omni.benchmarks.data_modules.daily_omni_dataset import DailyOmniDataset, DailyOmniSampleRequest
|
||||
from vllm_omni.benchmarks.data_modules.random_multi_modal_dataset import OmniRandomMultiModalDataset
|
||||
from vllm_omni.benchmarks.data_modules.seed_tts_dataset import (
|
||||
SEED_TTS_DEFAULT_OMNI_SYSTEM_PROMPT,
|
||||
SeedTTSDataset,
|
||||
SeedTTSSampleRequest,
|
||||
)
|
||||
|
||||
get_samples_old = datasets.get_samples
|
||||
|
||||
_DEFAULT_DAILY_OMNI_REPO = "liarliar/Daily-Omni"
|
||||
|
||||
|
||||
def _seed_tts_capture_pcm_for_wer() -> bool:
|
||||
return os.environ.get("SEED_TTS_WER_EVAL", "").lower() in (
|
||||
"1",
|
||||
"true",
|
||||
"yes",
|
||||
)
|
||||
|
||||
|
||||
def _merge_extra_body_mm_kwargs(base: dict | None, overlay: dict | None) -> dict | None:
|
||||
"""Shallow-merge ``extra_body`` dicts; deep-merge ``mm_processor_kwargs`` if both set."""
|
||||
if not base and not overlay:
|
||||
return None
|
||||
out = dict(base or {})
|
||||
if not overlay:
|
||||
return out
|
||||
for k, v in overlay.items():
|
||||
if k == "mm_processor_kwargs" and isinstance(v, dict):
|
||||
prev = out.get("mm_processor_kwargs")
|
||||
merged_kw = {**(prev if isinstance(prev, dict) else {}), **v}
|
||||
out["mm_processor_kwargs"] = merged_kw
|
||||
else:
|
||||
out[k] = v
|
||||
return out
|
||||
|
||||
|
||||
def _attach_daily_omni_to_request_func_input(sample: SampleRequest, rfi: RequestFuncInput) -> None:
|
||||
"""Apply per-request OpenAI fields (``mm_processor_kwargs``, messages) for Daily-Omni."""
|
||||
if not isinstance(sample, DailyOmniSampleRequest):
|
||||
return
|
||||
rfi.extra_body = _merge_extra_body_mm_kwargs(rfi.extra_body, sample.omni_extra_body)
|
||||
if sample.omni_chat_messages is not None:
|
||||
setattr(rfi, "omni_chat_messages", sample.omni_chat_messages)
|
||||
else:
|
||||
setattr(rfi, "mm_position", sample.omni_chat_mm_position)
|
||||
|
||||
|
||||
def _attach_seed_tts_to_request_func_input(sample: SampleRequest, rfi: RequestFuncInput) -> None:
|
||||
"""Merge Seed-TTS per-row TTS fields (ref_audio, ref_text, task_type, …) into ``extra_body``.
|
||||
|
||||
Used by both ``/v1/audio/speech`` and ``/v1/chat/completions`` (flattened into JSON body).
|
||||
For ``openai-chat-omni``, also sets ``omni_chat_messages`` (system + user) so Qwen3-Omni
|
||||
follows the same role layout as official TTS / multimodal demos. ``/v1/audio/speech`` ignores
|
||||
``messages`` and only uses ``input`` + body fields.
|
||||
Flags ``openai-chat-omni`` to request audio output and optionally export PCM for WER.
|
||||
"""
|
||||
if not isinstance(sample, SeedTTSSampleRequest):
|
||||
return
|
||||
ex = sample.seed_tts_speech_extra
|
||||
if not ex:
|
||||
return
|
||||
base = dict(rfi.extra_body) if rfi.extra_body else {}
|
||||
base.update(ex)
|
||||
rfi.extra_body = base
|
||||
# Used by request funcs to force streaming TTS behavior and to export PCM when WER is on.
|
||||
setattr(rfi, "seed_tts_row", True)
|
||||
sys_prompt = (sample.seed_tts_system_prompt or "").strip() or SEED_TTS_DEFAULT_OMNI_SYSTEM_PROMPT
|
||||
setattr(
|
||||
rfi,
|
||||
"omni_chat_messages",
|
||||
[
|
||||
{"role": "system", "content": [{"type": "text", "text": sys_prompt}]},
|
||||
{"role": "user", "content": [{"type": "text", "text": sample.prompt}]},
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
def _daily_omni_repo_from_args(args) -> str | None:
|
||||
"""Resolve HuggingFace repo id for Daily-Omni from CLI args.
|
||||
|
||||
vLLM allows ``--dataset-path`` to be a local path while the real HF id is
|
||||
passed via ``--hf-name``. Upstream ``get_samples`` for ``hf`` only matches
|
||||
a fixed elif-chain and never discovers Omni's loader, so we must detect
|
||||
Daily-Omni here using either field.
|
||||
"""
|
||||
dp = getattr(args, "dataset_path", None)
|
||||
hn = getattr(args, "hf_name", None)
|
||||
if dp in DailyOmniDataset.SUPPORTED_DATASET_PATHS:
|
||||
return dp
|
||||
if hn in DailyOmniDataset.SUPPORTED_DATASET_PATHS:
|
||||
return hn
|
||||
return None
|
||||
|
||||
|
||||
def get_samples(args, tokenizer):
|
||||
if args.backend not in ["openai-chat-omni", "openai-audio-speech"]:
|
||||
# Daily-Omni: explicit dataset name, or hf + matching path/hf-name
|
||||
is_daily_omni = args.dataset_name == "daily-omni" or (
|
||||
args.dataset_name == "hf" and _daily_omni_repo_from_args(args) is not None
|
||||
)
|
||||
is_seed_tts = args.dataset_name == "seed-tts"
|
||||
|
||||
# Check if we need to handle omni-related backends/datasets
|
||||
is_omni_backend = args.backend in ["openai-chat-omni", "openai-audio-speech", "daily-omni"]
|
||||
is_omni_dataset = is_daily_omni or is_seed_tts or args.dataset_name == "random-mm"
|
||||
|
||||
if not is_omni_backend and not is_omni_dataset:
|
||||
# Not an omni-related request, delegate to original implementation
|
||||
return get_samples_old(args, tokenizer)
|
||||
elif args.dataset_name == "random-mm":
|
||||
|
||||
# Handle Daily-Omni dataset
|
||||
if is_daily_omni:
|
||||
# Support:
|
||||
# --dataset-name daily-omni [--dataset-path liarliar/Daily-Omni]
|
||||
# --dataset-name daily-omni --daily-omni-qa-json /path/to/qa.json (offline QA)
|
||||
# --dataset-name hf --dataset-path liarliar/Daily-Omni
|
||||
# --dataset-name hf --hf-name liarliar/Daily-Omni (dataset-path may be local)
|
||||
|
||||
# Validate backend supports multimodal (video)
|
||||
if args.backend not in ["openai-chat-omni", "daily-omni"]:
|
||||
raise ValueError(
|
||||
f"Daily-Omni dataset requires a multimodal backend that supports video. "
|
||||
f"Got backend='{args.backend}'. Please use '--backend openai-chat-omni'"
|
||||
)
|
||||
|
||||
# Determine video directory if specified (for local video files)
|
||||
video_dir = getattr(args, "daily_omni_video_dir", None)
|
||||
|
||||
# Get HF split (default to "train"; unused when loading from local qa.json)
|
||||
dataset_split = getattr(args, "hf_split", None) or "train"
|
||||
|
||||
qa_json = getattr(args, "daily_omni_qa_json", None)
|
||||
if isinstance(qa_json, str):
|
||||
qa_json = qa_json.strip() or None
|
||||
|
||||
if qa_json is not None:
|
||||
logger.info(
|
||||
"Loading Daily-Omni dataset: qa_json=%s, video_dir=%s (Hub not used for QA)",
|
||||
qa_json,
|
||||
video_dir,
|
||||
)
|
||||
dataset = DailyOmniDataset(
|
||||
qa_json_path=qa_json,
|
||||
dataset_path=None,
|
||||
dataset_split=dataset_split,
|
||||
random_seed=args.seed,
|
||||
video_dir=video_dir,
|
||||
input_mode=getattr(args, "daily_omni_input_mode", "all"),
|
||||
inline_local_video=getattr(args, "daily_omni_inline_local_video", False),
|
||||
trust_remote_code=getattr(args, "trust_remote_code", False),
|
||||
disable_shuffle=getattr(args, "disable_shuffle", False),
|
||||
)
|
||||
else:
|
||||
repo_id = _daily_omni_repo_from_args(args)
|
||||
if args.dataset_name == "daily-omni":
|
||||
if repo_id is None:
|
||||
repo_id = _DEFAULT_DAILY_OMNI_REPO
|
||||
elif repo_id is None:
|
||||
raise ValueError(
|
||||
"Daily-Omni with --dataset-name hf requires "
|
||||
f"--dataset-path {_DEFAULT_DAILY_OMNI_REPO} or "
|
||||
f"--hf-name {_DEFAULT_DAILY_OMNI_REPO}."
|
||||
)
|
||||
|
||||
logger.info(
|
||||
"Loading Daily-Omni dataset: hf_repo=%s, split=%s, video_dir=%s",
|
||||
repo_id,
|
||||
dataset_split,
|
||||
video_dir,
|
||||
)
|
||||
|
||||
dataset = DailyOmniDataset(
|
||||
dataset_path=repo_id,
|
||||
dataset_split=dataset_split,
|
||||
dataset_subset=getattr(args, "hf_subset", None),
|
||||
random_seed=args.seed,
|
||||
video_dir=video_dir,
|
||||
input_mode=getattr(args, "daily_omni_input_mode", "all"),
|
||||
inline_local_video=getattr(args, "daily_omni_inline_local_video", False),
|
||||
trust_remote_code=getattr(args, "trust_remote_code", False),
|
||||
no_stream=getattr(args, "no_stream", False),
|
||||
disable_shuffle=getattr(args, "disable_shuffle", False),
|
||||
)
|
||||
|
||||
out_len = getattr(args, "output_len", None)
|
||||
if out_len is None:
|
||||
out_len = getattr(args, "hf_output_len", None)
|
||||
if out_len is None:
|
||||
out_len = DailyOmniDataset.DEFAULT_OUTPUT_LEN
|
||||
|
||||
input_requests = dataset.sample(
|
||||
tokenizer=tokenizer,
|
||||
num_requests=args.num_prompts,
|
||||
output_len=out_len,
|
||||
request_id_prefix=args.request_id_prefix,
|
||||
no_oversample=args.no_oversample,
|
||||
)
|
||||
return input_requests
|
||||
|
||||
if is_seed_tts:
|
||||
if args.backend not in ("openai-audio-speech", "openai-chat-omni"):
|
||||
raise ValueError(
|
||||
"Seed-TTS requires --backend openai-audio-speech (POST /v1/audio/speech) or "
|
||||
"--backend openai-chat-omni (POST /v1/chat/completions with ref_audio/ref_text). "
|
||||
f"Got backend={args.backend!r}."
|
||||
)
|
||||
repo_id = getattr(args, "dataset_path", None) or getattr(args, "hf_name", None)
|
||||
if not repo_id:
|
||||
raise ValueError(
|
||||
"Seed-TTS requires --dataset-path (HF dataset repo id or local directory) or "
|
||||
"--hf-name for the Hub dataset id."
|
||||
)
|
||||
|
||||
dataset = SeedTTSDataset(
|
||||
dataset_path=repo_id,
|
||||
random_seed=args.seed,
|
||||
locale=getattr(args, "seed_tts_locale", "en"),
|
||||
inline_ref_audio=not getattr(args, "seed_tts_file_ref_audio", False),
|
||||
seed_tts_root=getattr(args, "seed_tts_root", None),
|
||||
system_prompt=getattr(args, "seed_tts_system_prompt", None),
|
||||
disable_shuffle=getattr(args, "disable_shuffle", False),
|
||||
)
|
||||
out_len = getattr(args, "output_len", None)
|
||||
if out_len is None:
|
||||
out_len = getattr(args, "hf_output_len", None)
|
||||
if out_len is None:
|
||||
out_len = SeedTTSDataset.DEFAULT_OUTPUT_LEN
|
||||
return dataset.sample(
|
||||
tokenizer=tokenizer,
|
||||
num_requests=args.num_prompts,
|
||||
output_len=out_len,
|
||||
request_id_prefix=args.request_id_prefix,
|
||||
no_oversample=args.no_oversample,
|
||||
)
|
||||
|
||||
# Handle random-mm dataset (Omni's synthetic multimodal dataset)
|
||||
if args.dataset_name == "random-mm":
|
||||
dataset = OmniRandomMultiModalDataset(random_seed=args.seed, dataset_path=args.dataset_path)
|
||||
input_requests = dataset.sample(
|
||||
tokenizer=tokenizer,
|
||||
@@ -64,6 +295,10 @@ def get_samples(args, tokenizer):
|
||||
|
||||
datasets.get_samples = get_samples
|
||||
|
||||
_serve_mod = sys.modules.get("vllm.benchmarks.serve")
|
||||
if _serve_mod is not None:
|
||||
_serve_mod.get_samples = get_samples
|
||||
|
||||
|
||||
@dataclass
|
||||
class MixRequestFuncOutput(RequestFuncOutput):
|
||||
@@ -72,6 +307,9 @@ class MixRequestFuncOutput(RequestFuncOutput):
|
||||
audio_frames: int = 0
|
||||
audio_rtf: float = 0.0
|
||||
text_latency: float = 0.0
|
||||
#: Raw PCM s16le mono at 24 kHz for Seed-TTS WER: from ``/v1/audio/speech`` stream or
|
||||
#: resampled export after ``openai-chat-omni`` audio deltas.
|
||||
tts_output_pcm_bytes: bytes | None = None
|
||||
|
||||
|
||||
async def async_request_openai_chat_omni_completions(
|
||||
@@ -83,13 +321,17 @@ async def async_request_openai_chat_omni_completions(
|
||||
api_url = request_func_input.api_url
|
||||
_validate_api_url(api_url, "OpenAI Chat Completions API", "chat/completions")
|
||||
|
||||
content = _get_chat_content(request_func_input, mm_position=mm_position)
|
||||
omni_messages = getattr(request_func_input, "omni_chat_messages", None)
|
||||
if omni_messages is not None:
|
||||
messages_payload = omni_messages
|
||||
else:
|
||||
effective_mm_position = getattr(request_func_input, "mm_position", mm_position)
|
||||
content = _get_chat_content(request_func_input, mm_position=effective_mm_position)
|
||||
messages_payload = [{"role": "user", "content": content}]
|
||||
|
||||
payload = {
|
||||
"model": request_func_input.model_name if request_func_input.model_name else request_func_input.model,
|
||||
"messages": [
|
||||
{"role": "user", "content": content},
|
||||
],
|
||||
"messages": messages_payload,
|
||||
"temperature": 0.0,
|
||||
"max_tokens": request_func_input.output_len,
|
||||
"stream": True,
|
||||
@@ -98,6 +340,10 @@ async def async_request_openai_chat_omni_completions(
|
||||
},
|
||||
}
|
||||
_update_payload_common(payload, request_func_input)
|
||||
# Seed-TTS via chat: voice-clone fields live on the body; ensure audio is streamed.
|
||||
if getattr(request_func_input, "seed_tts_row", False):
|
||||
if payload.get("modalities") is None:
|
||||
payload["modalities"] = ["text", "audio"]
|
||||
|
||||
response_format = payload.get("response_format", "wav")
|
||||
if response_format == "pcm":
|
||||
@@ -167,7 +413,10 @@ async def async_request_openai_chat_omni_completions(
|
||||
data = json.loads(chunk)
|
||||
if choices := data.get("choices"):
|
||||
modality = data.get("modality")
|
||||
content = choices[0]["delta"].get("content")
|
||||
delta = choices[0].get("delta") or {}
|
||||
content = delta.get("content")
|
||||
if not content and isinstance(delta.get("audio"), dict):
|
||||
content = delta["audio"].get("data")
|
||||
if modality == "text":
|
||||
# First token
|
||||
if ttft == 0.0:
|
||||
@@ -182,7 +431,7 @@ async def async_request_openai_chat_omni_completions(
|
||||
if output.audio_ttfp == 0.0:
|
||||
output.audio_ttfp = timestamp - st
|
||||
audio_generate_time = timestamp - st
|
||||
if content != "":
|
||||
if content:
|
||||
audio_bytes = base64.b64decode(content)
|
||||
seg = AudioSegment.from_file(io.BytesIO(audio_bytes))
|
||||
if seg is not None:
|
||||
@@ -214,6 +463,12 @@ async def async_request_openai_chat_omni_completions(
|
||||
else:
|
||||
output.audio_rtf = 0
|
||||
logger.warning("Audio duration is zero")
|
||||
if _seed_tts_capture_pcm_for_wer() and getattr(request_func_input, "seed_tts_row", False):
|
||||
try:
|
||||
seg = generated_audio.set_frame_rate(24000).set_channels(1).set_sample_width(2)
|
||||
output.tts_output_pcm_bytes = bytes(seg.raw_data)
|
||||
except Exception as ex:
|
||||
logger.warning("seed_tts WER PCM export failed: %s", ex)
|
||||
output.success = True
|
||||
else:
|
||||
output.error = response.reason or ""
|
||||
@@ -268,6 +523,10 @@ async def async_request_openai_audio_speech(
|
||||
"response_format": "pcm",
|
||||
}
|
||||
_update_payload_common(payload, request_func_input)
|
||||
# Seed-TTS + WER: ``--extra-body`` may set stream=false / other formats; speech must stream PCM.
|
||||
if getattr(request_func_input, "seed_tts_row", False) and _seed_tts_capture_pcm_for_wer():
|
||||
payload["stream"] = True
|
||||
payload["response_format"] = "pcm"
|
||||
|
||||
headers = {
|
||||
"Content-Type": "application/json",
|
||||
@@ -286,6 +545,8 @@ async def async_request_openai_audio_speech(
|
||||
st = time.perf_counter()
|
||||
output.start_time = st
|
||||
total_pcm_bytes = 0
|
||||
capture_wer_pcm = _seed_tts_capture_pcm_for_wer() and getattr(request_func_input, "seed_tts_row", False)
|
||||
pcm_capture = bytearray() if capture_wer_pcm else None
|
||||
try:
|
||||
async with session.post(url=api_url, json=payload, headers=headers) as response:
|
||||
if response.status == 200:
|
||||
@@ -297,6 +558,8 @@ async def async_request_openai_audio_speech(
|
||||
output.audio_ttfp = timestamp - st
|
||||
output.ttft = output.audio_ttfp
|
||||
total_pcm_bytes += len(chunk)
|
||||
if pcm_capture is not None:
|
||||
pcm_capture.extend(chunk)
|
||||
|
||||
end_time = time.perf_counter()
|
||||
output.latency = end_time - st
|
||||
@@ -309,6 +572,16 @@ async def async_request_openai_audio_speech(
|
||||
else:
|
||||
output.audio_rtf = 0
|
||||
logger.warning("Audio duration is zero")
|
||||
if pcm_capture is not None and pcm_capture:
|
||||
output.tts_output_pcm_bytes = bytes(pcm_capture)
|
||||
elif capture_wer_pcm:
|
||||
ct = response.headers.get("Content-Type", "")
|
||||
logger.warning(
|
||||
"Seed-TTS WER: HTTP 200 but no PCM bytes (Content-Type=%r, url=%s). "
|
||||
"Check stream=true and response_format=pcm on the server.",
|
||||
ct,
|
||||
api_url,
|
||||
)
|
||||
output.success = True
|
||||
else:
|
||||
output.error = response.reason or ""
|
||||
@@ -331,6 +604,12 @@ ASYNC_REQUEST_FUNCS["openai-audio-speech"] = async_request_openai_audio_speech
|
||||
if "openai-audio-speech" not in OPENAI_COMPATIBLE_BACKENDS:
|
||||
OPENAI_COMPATIBLE_BACKENDS.append("openai-audio-speech")
|
||||
|
||||
# Daily-Omni backend for audio-visual reasoning benchmark
|
||||
# Reuses openai-chat-omni completions for video+text understanding
|
||||
ASYNC_REQUEST_FUNCS["daily-omni"] = async_request_openai_chat_omni_completions
|
||||
if "daily-omni" not in OPENAI_COMPATIBLE_BACKENDS:
|
||||
OPENAI_COMPATIBLE_BACKENDS.append("daily-omni")
|
||||
|
||||
# ruff: noqa: E402
|
||||
# Prevent import order from causing patch failures
|
||||
from vllm.benchmarks import serve
|
||||
@@ -422,6 +701,8 @@ async def benchmark(
|
||||
extra_headers=extra_headers,
|
||||
extra_body=extra_body,
|
||||
)
|
||||
_attach_daily_omni_to_request_func_input(input_requests[0], test_input)
|
||||
_attach_seed_tts_to_request_func_input(input_requests[0], test_input)
|
||||
|
||||
if ready_check_timeout_sec > 0:
|
||||
test_output = await wait_for_endpoint(
|
||||
@@ -484,6 +765,8 @@ async def benchmark(
|
||||
extra_headers=extra_headers,
|
||||
extra_body=extra_body,
|
||||
)
|
||||
_attach_daily_omni_to_request_func_input(input_requests[0], profile_input)
|
||||
_attach_seed_tts_to_request_func_input(input_requests[0], profile_input)
|
||||
profile_output = await request_func(request_func_input=profile_input, session=session)
|
||||
if profile_output.success:
|
||||
print("Profiler started")
|
||||
@@ -564,6 +847,8 @@ async def benchmark(
|
||||
extra_body=extra_body,
|
||||
request_id=request_id,
|
||||
)
|
||||
_attach_daily_omni_to_request_func_input(request, request_func_input)
|
||||
_attach_seed_tts_to_request_func_input(request, request_func_input)
|
||||
tasks.append(
|
||||
asyncio.create_task(limited_request_func(request_func_input=request_func_input, session=session, pbar=pbar))
|
||||
)
|
||||
@@ -631,6 +916,37 @@ async def benchmark(
|
||||
"errors": [output.error for output in outputs],
|
||||
}
|
||||
|
||||
from vllm_omni.benchmarks.data_modules.daily_omni_eval import (
|
||||
compute_daily_omni_accuracy_metrics,
|
||||
print_daily_omni_accuracy_summary,
|
||||
)
|
||||
|
||||
_save_items = os.environ.get("DAILY_OMNI_SAVE_EVAL_ITEMS", "").lower() in (
|
||||
"1",
|
||||
"true",
|
||||
"yes",
|
||||
)
|
||||
_daily_acc = compute_daily_omni_accuracy_metrics(input_requests, outputs, include_per_item=_save_items)
|
||||
if _daily_acc is not None:
|
||||
result.update(_daily_acc)
|
||||
print_daily_omni_accuracy_summary(_daily_acc)
|
||||
|
||||
if _seed_tts_capture_pcm_for_wer():
|
||||
from vllm_omni.benchmarks.data_modules.seed_tts_eval import (
|
||||
compute_seed_tts_wer_metrics,
|
||||
print_seed_tts_wer_summary,
|
||||
)
|
||||
|
||||
_save_wer = os.environ.get("SEED_TTS_WER_SAVE_ITEMS", "").lower() in (
|
||||
"1",
|
||||
"true",
|
||||
"yes",
|
||||
)
|
||||
_wer_m = compute_seed_tts_wer_metrics(input_requests, outputs, include_per_item=_save_wer)
|
||||
if _wer_m is not None:
|
||||
result.update(_wer_m)
|
||||
print_seed_tts_wer_summary(_wer_m)
|
||||
|
||||
if rps_change_events:
|
||||
result["rps_change_events"] = rps_change_events
|
||||
|
||||
|
||||
Reference in new issue
Block a user