[CI][MiniCPM-o] Add MiniCPM-o 4.5 accuracy and performance coverage (#5524)

Signed-off-by: R2-Y <ruiruyang2@gmail.com>
Signed-off-by: psv666 <2693925048@qq.com>
Co-authored-by: psv666 <2693925048@qq.com>
Co-authored-by: Cursor <cursoragent@cursor.com>
Co-authored-by: amy-why-3459 <wuhaiyan17@huawei.com>
This commit is contained in:
authored and GitHub committed 2026-08-11 14:53:49 +08:00
1 parent c4a14091c9
commit 322600bfb2
59 files changed
+1908 -155

No files matched your search

@@ -55,6 +55,16 @@ class SeedTTSSampleRequest(SampleRequest):
seed_tts_system_prompt: str = ""
#: Local path to reference prompt WAV (for SIM vs. synthesized PCM in ``seed_tts_eval``).
seed_tts_ref_wav_path: str = ""
#: Ordered text targets evaluated as turns in one Realtime session.
seed_tts_turns: tuple[SeedTTSTurn, ...] = ()
@dataclass(frozen=True)
class SeedTTSTurn:
"""One target utterance within a grouped Realtime Seed-TTS session."""
utterance_id: str
target_text: str
@dataclass
@@ -216,24 +226,39 @@ class SeedTTSDataset(BenchmarkDataset):
) -> list[SampleRequest]:
if output_len is None:
output_len = self.DEFAULT_OUTPUT_LEN
turns_per_session = int(kwargs.pop("turns_per_session", 1))
if turns_per_session < 1:
raise ValueError("turns_per_session must be at least 1")
tok = get_cached_tokenizer(tokenizer)
out: list[SampleRequest] = []
for i, row in enumerate(self._rows):
if len(out) >= num_requests:
break
valid_rows: list[tuple[_SeedTTSRow, Path]] = []
for row in self._rows:
wav_path = (self._root / self.locale / row.prompt_wav_rel).resolve()
if not wav_path.is_file():
logger.warning("Missing prompt wav for %s: %s", row.utterance_id, wav_path)
continue
valid_rows.append((row, wav_path))
target = row.target_text
prompt_len = len(tok.encode(f"{self._system_prompt}\n{target}"))
for session_index in range(num_requests):
offset = session_index * turns_per_session
group = valid_rows[offset : offset + turns_per_session]
if len(group) < turns_per_session:
break
reference_row, reference_wav_path = group[0]
turns = tuple(
SeedTTSTurn(
utterance_id=row.utterance_id,
target_text=row.target_text,
)
for row, _ in group
)
prompt_len = sum(len(tok.encode(f"{self._system_prompt}\n{turn.target_text}")) for turn in turns)
lang = "English" if self.locale == "en" else "Chinese"
ref_uri = _ref_audio_payload(wav_path, inline=self.inline_ref_audio)
ref_uri = _ref_audio_payload(reference_wav_path, inline=self.inline_ref_audio)
speech_extra: dict[str, Any] = {
"ref_audio": ref_uri,
"ref_text": row.ref_text,
"ref_text": reference_row.ref_text,
"task_type": "Base",
"language": lang,
"max_new_tokens": output_len,
@@ -241,20 +266,26 @@ class SeedTTSDataset(BenchmarkDataset):
out.append(
SeedTTSSampleRequest(
prompt=target,
prompt=turns[0].target_text,
prompt_len=prompt_len,
expected_output_len=output_len,
multi_modal_data=None,
request_id=f"{request_id_prefix}{i}",
request_id=f"{request_id_prefix}{session_index}",
seed_tts_speech_extra=speech_extra,
seed_tts_utterance_id=row.utterance_id,
seed_tts_utterance_id=turns[0].utterance_id,
seed_tts_locale=self.locale,
seed_tts_system_prompt=self._system_prompt,
seed_tts_ref_wav_path=str(wav_path),
seed_tts_ref_wav_path=str(reference_wav_path),
seed_tts_turns=turns,
)
)
logger.info("Seed-TTS: built %d requests (asked %d)", len(out), num_requests)
logger.info(
"Seed-TTS: built %d sessions (asked %d, turns_per_session=%d)",
len(out),
num_requests,
turns_per_session,
)
self.maybe_oversample_requests(out, num_requests, request_id_prefix, no_oversample)
return out