[Bugfix] Tolerate a bounded TTS tail in the audio-text similarity gate (#7358)

Signed-off-by: tly <2200895168@qq.com>

tests/helpers/assertions.py

@@ -171,6 +171,38 @@ def _short_transcript_contains_expected(transcript: str, expected: str) -> bool:
     return short_text and small_word_delta and expected_clean in transcript_clean
 
 
+_MAX_TAIL_WORDS_FLOOR = 2
+_TAIL_WORD_RATIO = 0.2
+
+
+def _transcript_has_bounded_tail(transcript: str, expected: str) -> bool:
+    """Pass when the expected text is spoken verbatim, only trailed by noise.
+
+    A short, unrelated tail after the complete expected sentence is the exact
+    case #6828's rationale anticipated: a different valid TTS sample can append
+    an audible tail even when the spoken text is identical, and Whisper can
+    hallucinate brief words on a non-speech tail. The length-penalized n-gram
+    cosine stays below the gate for such tails, so accept the match when the
+    expected text is a word-aligned prefix of the transcript and the trailing
+    words stay within the floor/ratio bounds.
+    """
+    transcript_clean = preprocess_text(transcript)
+    expected_clean = preprocess_text(expected)
+    if not transcript_clean or not expected_clean:
+        return False
+
+    transcript_words = transcript_clean.split()
+    expected_words = expected_clean.split()
+    if not transcript_words or not expected_words:
+        return False
+
+    if transcript_words[: len(expected_words)] != expected_words:
+        return False
+    tail_words = len(transcript_words) - len(expected_words)
+    max_tail_words = max(_MAX_TAIL_WORDS_FLOOR, math.ceil(_TAIL_WORD_RATIO * len(expected_words)))
+    return tail_words <= max_tail_words
+
+
 def assert_image_diffusion_response(
     response: "DiffusionResponse",
     request_config: dict[str, Any],
@@ -834,7 +866,16 @@ def assert_omni_response(response: Any, request_config: dict[str, Any], run_leve
                         text_output.lower(),
                     )
                     print(f"similarity is: {similarity}")
-                    assert similarity > similarity_threshold, AUDIO_MISMATCH_MESSAGE
+                    if similarity <= similarity_threshold and _transcript_has_bounded_tail(transcript, text_output):
+                        # The full answer is spoken verbatim and only a short
+                        # noise tail follows it; see #6828's rationale.
+                        print(
+                            "bounded-tail containment check passed: "
+                            f"text={text_output!r} is a word-aligned prefix of "
+                            f"transcript={transcript!r}"
+                        )
+                    else:
+                        assert similarity > similarity_threshold, AUDIO_MISMATCH_MESSAGE
             if audio_ref_text:
                 assert transcript is not None, "No audio transcript for reference-text validation"
                 audio_similarity = cosine_similarity_text(

tests/helpers/tests/test_assertions.py

@@ -126,3 +126,50 @@ def test_escalated_transcript_keeps_declared_language(monkeypatch):
 
     assert captured["model_size"] == "large-v3"
     assert captured["language"] == "en"
+
+
+def test_bounded_tail_after_complete_answer_passes():
+    # The exact shape from the #6815 nightly failure: the full answer is spoken
+    # verbatim, then a two-word unrelated tail trips the cosine gate.
+    assert assertions._transcript_has_bounded_tail(
+        "The squares in this image are black. nack shit.",
+        "The squares in this image are black.",
+    )
+
+
+def test_bounded_tail_exact_match_passes():
+    assert assertions._transcript_has_bounded_tail(
+        "The squares in this image are black.",
+        "The squares in this image are black.",
+    )
+
+
+def test_bounded_tail_rejects_different_content():
+    assert not assertions._transcript_has_bounded_tail(
+        "The circles in this picture are black and white.",
+        "The squares in this image are black.",
+    )
+
+
+def test_bounded_tail_rejects_long_tail():
+    # Eight expected words allow at most max(2, ceil(0.2 * 8)) = 2 tail words.
+    assert not assertions._transcript_has_bounded_tail(
+        "The squares in this image are black and some extra words follow here",
+        "The squares in this image are black.",
+    )
+
+
+def test_bounded_tail_rejects_word_boundary_split():
+    assert not assertions._transcript_has_bounded_tail(
+        "The squares in this image are blacks.",
+        "The squares in this image are black.",
+    )
+
+
+def test_bounded_tail_allows_ratio_bound_for_long_answers():
+    expected = " ".join(["word"] * 20)
+    four_word_tail = expected + " one two three four"
+    five_word_tail = expected + " one two three four five"
+    # Twenty expected words allow up to max(2, ceil(0.2 * 20)) = 4 tail words.
+    assert assertions._transcript_has_bounded_tail(four_word_tail, expected)
+    assert not assertions._transcript_has_bounded_tail(five_word_tail, expected)