[CI][MiniCPM-o] Add MiniCPM-o 4.5 accuracy and performance coverage (#5524)

Signed-off-by: R2-Y <ruiruyang2@gmail.com>
Signed-off-by: psv666 <2693925048@qq.com>
Co-authored-by: psv666 <2693925048@qq.com>
Co-authored-by: Cursor <cursoragent@cursor.com>
Co-authored-by: amy-why-3459 <wuhaiyan17@huawei.com>

vllm_omni/benchmarks/data_modules/seed_tts_dataset.py

@@ -55,6 +55,16 @@ class SeedTTSSampleRequest(SampleRequest):
     seed_tts_system_prompt: str = ""
     #: Local path to reference prompt WAV (for SIM vs. synthesized PCM in ``seed_tts_eval``).
     seed_tts_ref_wav_path: str = ""
+    #: Ordered text targets evaluated as turns in one Realtime session.
+    seed_tts_turns: tuple[SeedTTSTurn, ...] = ()
+
+
+@dataclass(frozen=True)
+class SeedTTSTurn:
+    """One target utterance within a grouped Realtime Seed-TTS session."""
+
+    utterance_id: str
+    target_text: str
 
 
 @dataclass
@@ -216,24 +226,39 @@ class SeedTTSDataset(BenchmarkDataset):
     ) -> list[SampleRequest]:
         if output_len is None:
             output_len = self.DEFAULT_OUTPUT_LEN
+        turns_per_session = int(kwargs.pop("turns_per_session", 1))
+        if turns_per_session < 1:
+            raise ValueError("turns_per_session must be at least 1")
 
         tok = get_cached_tokenizer(tokenizer)
         out: list[SampleRequest] = []
-        for i, row in enumerate(self._rows):
-            if len(out) >= num_requests:
-                break
+        valid_rows: list[tuple[_SeedTTSRow, Path]] = []
+        for row in self._rows:
             wav_path = (self._root / self.locale / row.prompt_wav_rel).resolve()
             if not wav_path.is_file():
                 logger.warning("Missing prompt wav for %s: %s", row.utterance_id, wav_path)
                 continue
+            valid_rows.append((row, wav_path))
 
-            target = row.target_text
-            prompt_len = len(tok.encode(f"{self._system_prompt}\n{target}"))
+        for session_index in range(num_requests):
+            offset = session_index * turns_per_session
+            group = valid_rows[offset : offset + turns_per_session]
+            if len(group) < turns_per_session:
+                break
+            reference_row, reference_wav_path = group[0]
+            turns = tuple(
+                SeedTTSTurn(
+                    utterance_id=row.utterance_id,
+                    target_text=row.target_text,
+                )
+                for row, _ in group
+            )
+            prompt_len = sum(len(tok.encode(f"{self._system_prompt}\n{turn.target_text}")) for turn in turns)
             lang = "English" if self.locale == "en" else "Chinese"
-            ref_uri = _ref_audio_payload(wav_path, inline=self.inline_ref_audio)
+            ref_uri = _ref_audio_payload(reference_wav_path, inline=self.inline_ref_audio)
             speech_extra: dict[str, Any] = {
                 "ref_audio": ref_uri,
-                "ref_text": row.ref_text,
+                "ref_text": reference_row.ref_text,
                 "task_type": "Base",
                 "language": lang,
                 "max_new_tokens": output_len,
@@ -241,20 +266,26 @@ class SeedTTSDataset(BenchmarkDataset):
 
             out.append(
                 SeedTTSSampleRequest(
-                    prompt=target,
+                    prompt=turns[0].target_text,
                     prompt_len=prompt_len,
                     expected_output_len=output_len,
                     multi_modal_data=None,
-                    request_id=f"{request_id_prefix}{i}",
+                    request_id=f"{request_id_prefix}{session_index}",
                     seed_tts_speech_extra=speech_extra,
-                    seed_tts_utterance_id=row.utterance_id,
+                    seed_tts_utterance_id=turns[0].utterance_id,
                     seed_tts_locale=self.locale,
                     seed_tts_system_prompt=self._system_prompt,
-                    seed_tts_ref_wav_path=str(wav_path),
+                    seed_tts_ref_wav_path=str(reference_wav_path),
+                    seed_tts_turns=turns,
                 )
             )
 
-        logger.info("Seed-TTS: built %d requests (asked %d)", len(out), num_requests)
+        logger.info(
+            "Seed-TTS: built %d sessions (asked %d, turns_per_session=%d)",
+            len(out),
+            num_requests,
+            turns_per_session,
+        )
         self.maybe_oversample_requests(out, num_requests, request_id_prefix, no_oversample)
         return out
 

vllm_omni/benchmarks/data_modules/seed_tts_eval.py

@@ -54,6 +54,8 @@ import string
 import tempfile
 import threading
 import wave
+from copy import copy
+from dataclasses import replace
 from pathlib import Path
 from typing import Any
 
@@ -123,11 +125,25 @@ def _get_eval_device() -> str:
         return explicit
     try:
         import torch
-
-        return "cuda:0" if torch.cuda.is_available() else "cpu"
     except ImportError:
         return "cpu"
 
+    if torch.cuda.is_available():
+        return "cuda:0"
+
+    # Ascend: ``torch.npu`` only exists once ``torch_npu`` has been imported, which does
+    # not necessarily happen in the benchmark client process. Without this branch the
+    # eval silently lands on CPU, where Whisper-large-v3 needs hours for a full Seed-TTS
+    # run and the CI job is killed by its timeout instead of reporting a WER.
+    try:
+        import torch_npu  # noqa: F401
+    except ImportError:
+        pass
+    if hasattr(torch, "npu") and torch.npu.is_available():
+        return "npu:0"
+
+    return "cpu"
+
 
 def _punctuation_all() -> str:
     from zhon.hanzi import punctuation
@@ -512,6 +528,42 @@ def _missing_deps_message(lang: str) -> str | None:
     return None
 
 
+def _expand_seed_tts_turn_outputs(
+    input_requests: list[SeedTTSSampleRequest],
+    outputs: list[Any],
+) -> tuple[list[SeedTTSSampleRequest], list[Any]]:
+    """Expand grouped Realtime sessions into one request/output pair per turn."""
+    expanded_requests: list[SeedTTSSampleRequest] = []
+    expanded_outputs: list[Any] = []
+    for request, output in zip(input_requests, outputs, strict=True):
+        turns = request.seed_tts_turns
+        if not turns:
+            expanded_requests.append(request)
+            expanded_outputs.append(output)
+            continue
+        turn_pcm = getattr(output, "tts_turn_pcm_bytes", None)
+        session_pcm = getattr(output, "tts_output_pcm_bytes", None)
+        for turn_index, turn in enumerate(turns):
+            expanded_requests.append(
+                replace(
+                    request,
+                    prompt=turn.target_text,
+                    seed_tts_utterance_id=turn.utterance_id,
+                    seed_tts_turns=(),
+                )
+            )
+            turn_output = copy(output)
+            if isinstance(turn_pcm, list) and turn_index < len(turn_pcm):
+                turn_output.tts_output_pcm_bytes = turn_pcm[turn_index]
+            elif len(turns) == 1:
+                # openai-chat-omni Seed-TTS only fills the session-level PCM field.
+                turn_output.tts_output_pcm_bytes = session_pcm
+            else:
+                turn_output.tts_output_pcm_bytes = None
+            expanded_outputs.append(turn_output)
+    return expanded_requests, expanded_outputs
+
+
 def compute_seed_tts_wer_metrics(
     input_requests: list[SampleRequest],
     outputs: list[Any],
@@ -524,6 +576,8 @@ def compute_seed_tts_wer_metrics(
         return None
     if not all(isinstance(r, SeedTTSSampleRequest) for r in input_requests):
         return None
+    session_count = len(input_requests)
+    input_requests, outputs = _expand_seed_tts_turn_outputs(input_requests, outputs)
 
     first = input_requests[0]
     assert isinstance(first, SeedTTSSampleRequest)
@@ -759,6 +813,8 @@ def compute_seed_tts_wer_metrics(
 
     result: dict[str, Any] = {
         "seed_tts_eval_protocol": "seed-tts-eval",
+        "seed_tts_session_count": session_count,
+        "seed_tts_turn_count": len(input_requests),
         "seed_tts_content_evaluated": len(errs),
         "seed_tts_content_error_mean": statistics.fmean(errs) if errs else None,
         "seed_tts_content_error_median": statistics.median(errs) if errs else None,