diff --git a/tests/e2e/online_serving/test_minicpmo_4_5.py b/tests/e2e/online_serving/test_minicpmo_4_5.py index a1d045b72..c4733f32c 100644 --- a/tests/e2e/online_serving/test_minicpmo_4_5.py +++ b/tests/e2e/online_serving/test_minicpmo_4_5.py @@ -361,13 +361,20 @@ def test_mix_to_text_audio_001(omni_server, openai_client) -> None: """ video_data_url = f"data:video/mp4;base64,{generate_synthetic_video(24, 24, 20)['base64']}" image_data_url = f"data:image/jpeg;base64,{generate_synthetic_image(24, 24)['base64']}" - audio_data_url = f"data:audio/wav;base64,{generate_synthetic_audio(5, 1)['base64']}" + # A loop of the default single word "test" can make the answer repeat + # to the token limit, leaving TTS/ASR to count a long run of identical words + # (#7630). Exercise speech perception with a meaningful phrase instead. + audio = generate_synthetic_audio(5, 1, phrase_text="The weather is sunny today.") + audio_data_url = f"data:audio/wav;base64,{audio['base64']}" messages = dummy_messages_from_mix_data( system_prompt=get_system_prompt(), video_data_url=video_data_url, image_data_url=image_data_url, audio_data_url=audio_data_url, - content_text=get_prompt("mix"), + content_text=( + "Briefly describe the image and video, and summarize what the voice says. " + "Answer in one short sentence without repeating the spoken phrase." + ), ) request_config = { diff --git a/tests/helpers/tests/test_assertions.py b/tests/helpers/tests/test_assertions.py index a64cbe3b9..e2060152f 100644 --- a/tests/helpers/tests/test_assertions.py +++ b/tests/helpers/tests/test_assertions.py @@ -328,6 +328,21 @@ def test_bounded_tail_after_complete_answer_passes(): ) +def test_minicpmo_mix_runaway_repetition_still_fails(): + # #7630, Buildkite 15299 lines 1472-1475: 49 copies in the answer, + # 112 in the transcript. This is not a bounded noise tail and must not + # be hidden by relaxing the shared audio/text similarity gate. + prefix = "A black background with some colorful patterns appears, accompanied by a voice saying " + expected = prefix + '"' + " ".join(["test"] * 49) + transcript = prefix + ", ".join(["test"] * 112) + response = OmniResponse(success=True, text_content=expected, audio_content=transcript) + + assert assertions.cosine_similarity_text(transcript, expected) == pytest.approx(0.675451892050252) + assert not assertions._transcript_has_bounded_tail(transcript, expected) + with pytest.raises(AssertionError, match=assertions.AUDIO_MISMATCH_MESSAGE): + assertions.assert_omni_response(response, {"modalities": ["text", "audio"]}, "advanced_model") + + def test_bounded_tail_exact_match_passes(): assert assertions._transcript_has_bounded_tail( "The squares in this image are black.",