[CI/Build] Gate one-word pronunciation on a success rate, not per request (#5681)

Signed-off-by: Shenglei Fu <sfu@confluent.io>
Co-authored-by: amy-why-3459 <wuhaiyan17@huawei.com>
This commit is contained in:
Shenglei Fuandamy-why-3459 authored and GitHub committed 2026-08-17 14:02:50 +08:00
1 parent e96d3fe2a6
commit 5664e69dbf
4 files changed
+291 -35

No files matched your search

+109 -4
View File
@@ -2,12 +2,16 @@
import io
import json
import math
import tempfile
import threading
import wave
from collections.abc import Callable
from concurrent.futures import ThreadPoolExecutor
from dataclasses import dataclass
from io import BytesIO
from pathlib import Path
from typing import TYPE_CHECKING, Any, Literal
from typing import TYPE_CHECKING, Any, Literal, TypeVar
if TYPE_CHECKING:
from tests.helpers.runtime import DiffusionResponse
@@ -24,8 +28,109 @@ from tests.helpers.media import (
preprocess_text,
)
_ResponseT = TypeVar("_ResponseT")
_GENDER_PIPELINE = None
_GENDER_PIPELINE_LOCK = threading.Lock()
# Assertions that sample a random variable rather than testing whether serving works:
# whether TTS audio is intelligible enough to transcribe back, and whether a preset voice's
# timbre estimates as the expected gender. Both are decided by the model and the estimator,
# so a caller measuring a success rate has to tell them apart from a serving failure. The
# messages are constants because a caller matching the wording by hand would silently stop
# matching if it were reworded, reclassifying every quality failure as a hard failure.
AUDIO_MISMATCH_MESSAGE = "The audio content is not same as the text"
GENDER_MISMATCH_MESSAGE = "estimated gender is"
_QUALITY_FAILURE_MESSAGES = (AUDIO_MISMATCH_MESSAGE, GENDER_MISMATCH_MESSAGE)
@dataclass(frozen=True)
class SuccessRateGate:
"""Gate a sampled quality check on how many requests pass, not on each one passing.
Built by the send helpers from their own arguments, so a test states a policy and never
touches this class. See ``collect_at_success_rate``.
Attributes:
min_successes: Fail below this many successes out of ``request_num``. Predeclare it
from a measured baseline; tuning it until CI passes defeats the gate.
concurrency: In-flight requests, in batches until the sample count is reached.
Defaults to sending all of them at once.
"""
min_successes: int
concurrency: int | None = None
def is_quality_failure(exc: BaseException) -> bool:
"""True when ``exc`` is one of the sampled quality assertions above.
A caller absorbing a few of these into a success-rate budget must not absorb an HTTP
failure, missing audio or a text-keyword miss as well, or a broken server passes.
"""
if not isinstance(exc, AssertionError):
return False
return any(message in str(exc) for message in _QUALITY_FAILURE_MESSAGES)
def collect_at_success_rate(
sample: Callable[[], _ResponseT],
gate: SuccessRateGate,
*,
request_num: int,
report: Callable[[str], None] = print,
) -> list[_ResponseT]:
"""Send ``request_num`` samples and gate on how many passed, returning the successes.
``sample`` sends one request and asserts it. Only the sampled quality assertions are
counted; anything else propagates immediately. Does not retry and does not exit early,
since both would bias the rate being measured.
"""
if gate.min_successes > request_num:
raise ValueError(f"min_successes={gate.min_successes} exceeds request_num={request_num}, gate can never pass")
concurrency = min(gate.concurrency or request_num, request_num)
responses: list[_ResponseT] = []
failures: list[str] = []
lock = threading.Lock()
def _one_sample(_: int) -> None:
try:
response = sample()
except AssertionError as exc:
if not is_quality_failure(exc):
raise
with lock:
failures.append(str(exc))
return
with lock:
responses.append(response)
for start in range(0, request_num, concurrency):
size = min(concurrency, request_num - start)
with ThreadPoolExecutor(max_workers=size) as executor:
for _ in executor.map(_one_sample, range(size)):
pass
successes = request_num - len(failures)
report(f"success rate: {successes}/{request_num} succeeded")
assert successes >= gate.min_successes, (
f"success rate {successes}/{request_num} is below the "
f"{gate.min_successes}/{request_num} gate "
f"(95% CI {wilson_interval(successes, request_num)}). Failures:\n"
+ "\n".join(f" - {failure}" for failure in failures)
)
return responses
def wilson_interval(successes: int, total: int, z: float = 1.96) -> str:
"""Format a 95% Wilson score interval, so a rate failure shows how marginal it is."""
p = successes / total
denom = 1 + z * z / total
centre = (p + z * z / (2 * total)) / denom
half = z / denom * math.sqrt(p * (1 - p) / total + z * z / (4 * total * total))
return f"{max(0.0, centre - half):.1%}-{min(1.0, centre + half):.1%}"
# Transcript gates default to whisper ``small`` for speed. ``small`` mishears a
# short TTS clip ~0.5% of the time (e.g. "Hello"->"fellow", or hallucinating a
# leading SFX token), which flakes the deterministic similarity gate. Short
@@ -474,7 +579,7 @@ def _assert_preset_voice_gender_from_audio(
print(f"Preset voice gender check: preset={key!r}, estimated={estimated_gender!r}, expected={expected_gender!r}")
if estimated_gender != "unknown":
assert estimated_gender == expected_gender, (
f"{voice_name!r} is expected {expected_gender}, but estimated gender is {estimated_gender!r}"
f"{voice_name!r} is expected {expected_gender}, but {GENDER_MISMATCH_MESSAGE} {estimated_gender!r}"
)
@@ -655,7 +760,7 @@ def assert_omni_response(response: Any, request_config: dict[str, Any], run_leve
shorter_clean = _re.sub(r"[^\w\s]", "", shorter).strip()
longer_clean = _re.sub(r"[^\w\s]", "", longer).strip()
assert shorter_clean and (shorter_clean in longer_clean), (
f"The audio content is not same as the text "
f"{AUDIO_MISMATCH_MESSAGE} "
f"(short-text containment check failed: "
f"text={text_output!r}, transcript={transcript!r})"
)
@@ -666,7 +771,7 @@ def assert_omni_response(response: Any, request_config: dict[str, Any], run_leve
text_output.lower(),
)
print(f"similarity is: {similarity}")
assert similarity > similarity_threshold, "The audio content is not same as the text"
assert similarity > similarity_threshold, AUDIO_MISMATCH_MESSAGE
if audio_ref_text:
assert transcript is not None, "No audio transcript for reference-text validation"
audio_similarity = cosine_similarity_text(