1259 lines
52 KiB
Python
1259 lines
52 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
|
|
|
"""Assertion and response validation helpers for tests."""
|
|
|
|
import base64
|
|
import hashlib
|
|
import io
|
|
import json
|
|
import math
|
|
import os
|
|
import tempfile
|
|
import threading
|
|
import wave
|
|
from collections.abc import Callable
|
|
from concurrent.futures import ThreadPoolExecutor
|
|
from dataclasses import dataclass
|
|
from io import BytesIO
|
|
from pathlib import Path
|
|
from typing import TYPE_CHECKING, Any, Literal, TypeVar
|
|
|
|
if TYPE_CHECKING:
|
|
from tests.helpers.client import DiffusionResponse
|
|
|
|
import av
|
|
import numpy as np
|
|
import soundfile as sf
|
|
from PIL import Image
|
|
|
|
from tests.helpers.media import (
|
|
convert_audio_bytes_to_text,
|
|
cosine_similarity_text,
|
|
preprocess_text,
|
|
)
|
|
|
|
_ResponseT = TypeVar("_ResponseT")
|
|
|
|
_GENDER_PIPELINE = None
|
|
_GENDER_PIPELINE_LOCK = threading.Lock()
|
|
|
|
# Assertions that sample a random variable rather than testing whether serving works:
|
|
# whether TTS audio is intelligible enough to transcribe back, and whether a preset voice's
|
|
# timbre estimates as the expected gender. Both are decided by the model and the estimator,
|
|
# so a caller measuring a success rate has to tell them apart from a serving failure. The
|
|
# messages are constants because a caller matching the wording by hand would silently stop
|
|
# matching if it were reworded, reclassifying every quality failure as a hard failure.
|
|
AUDIO_MISMATCH_MESSAGE = "The audio content is not same as the text"
|
|
GENDER_MISMATCH_MESSAGE = "estimated gender is"
|
|
_QUALITY_FAILURE_MESSAGES = (AUDIO_MISMATCH_MESSAGE, GENDER_MISMATCH_MESSAGE)
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class SuccessRateGate:
|
|
"""Gate a sampled quality check on how many requests pass, not on each one passing.
|
|
|
|
Built by the send helpers from their own arguments, so a test states a policy and never
|
|
touches this class. See ``collect_at_success_rate``.
|
|
|
|
Attributes:
|
|
min_successes: Fail below this many successes out of ``request_num``. Predeclare it
|
|
from a measured baseline; tuning it until CI passes defeats the gate.
|
|
concurrency: In-flight requests, in batches until the sample count is reached.
|
|
Defaults to sending all of them at once.
|
|
"""
|
|
|
|
min_successes: int
|
|
concurrency: int | None = None
|
|
|
|
|
|
def is_quality_failure(exc: BaseException) -> bool:
|
|
"""True when ``exc`` is one of the sampled quality assertions above.
|
|
|
|
A caller absorbing a few of these into a success-rate budget must not absorb an HTTP
|
|
failure, missing audio or a text-keyword miss as well, or a broken server passes.
|
|
"""
|
|
if not isinstance(exc, AssertionError):
|
|
return False
|
|
return any(message in str(exc) for message in _QUALITY_FAILURE_MESSAGES)
|
|
|
|
|
|
def collect_at_success_rate(
|
|
sample: Callable[[], _ResponseT],
|
|
gate: SuccessRateGate,
|
|
*,
|
|
request_num: int,
|
|
report: Callable[[str], None] = print,
|
|
) -> list[_ResponseT]:
|
|
"""Send ``request_num`` samples and gate on how many passed, returning the successes.
|
|
|
|
``sample`` sends one request and asserts it. Only the sampled quality assertions are
|
|
counted; anything else propagates immediately. Does not retry and does not exit early,
|
|
since both would bias the rate being measured.
|
|
"""
|
|
if gate.min_successes > request_num:
|
|
raise ValueError(f"min_successes={gate.min_successes} exceeds request_num={request_num}, gate can never pass")
|
|
concurrency = min(gate.concurrency or request_num, request_num)
|
|
responses: list[_ResponseT] = []
|
|
failures: list[str] = []
|
|
lock = threading.Lock()
|
|
|
|
def _one_sample(_: int) -> None:
|
|
try:
|
|
response = sample()
|
|
except AssertionError as exc:
|
|
if not is_quality_failure(exc):
|
|
raise
|
|
with lock:
|
|
failures.append(str(exc))
|
|
return
|
|
with lock:
|
|
responses.append(response)
|
|
|
|
for start in range(0, request_num, concurrency):
|
|
size = min(concurrency, request_num - start)
|
|
with ThreadPoolExecutor(max_workers=size) as executor:
|
|
for _ in executor.map(_one_sample, range(size)):
|
|
pass
|
|
|
|
successes = request_num - len(failures)
|
|
report(f"success rate: {successes}/{request_num} succeeded")
|
|
assert successes >= gate.min_successes, (
|
|
f"success rate {successes}/{request_num} is below the "
|
|
f"{gate.min_successes}/{request_num} gate "
|
|
f"(95% CI {wilson_interval(successes, request_num)}). Failures:\n"
|
|
+ "\n".join(f" - {failure}" for failure in failures)
|
|
)
|
|
return responses
|
|
|
|
|
|
def wilson_interval(successes: int, total: int, z: float = 1.96) -> str:
|
|
"""Format a 95% Wilson score interval, so a rate failure shows how marginal it is."""
|
|
p = successes / total
|
|
denom = 1 + z * z / total
|
|
centre = (p + z * z / (2 * total)) / denom
|
|
half = z / denom * math.sqrt(p * (1 - p) / total + z * z / (4 * total * total))
|
|
return f"{max(0.0, centre - half):.1%}-{min(1.0, centre + half):.1%}"
|
|
|
|
|
|
# Transcript gates default to whisper ``small`` for speed. ``small`` mishears a
|
|
# short TTS clip ~0.5% of the time (e.g. "Hello"->"fellow", or hallucinating a
|
|
# leading SFX token), which flakes the deterministic similarity gate. Short
|
|
# clips first get a conservative containment fallback for minor ASR repeats/noise.
|
|
# A test can also opt in to ASR escalation by setting
|
|
# ``transcript_escalation_model`` to a whisper model name (e.g. ``"large-v3"``)
|
|
# in its request_config: on a failed fast pass the clip is re-transcribed with
|
|
# that stronger ASR before the test fails, so a weak-ASR mishear is rescued while
|
|
# a genuine model artifact still fails (the strong ASR mismatches too).
|
|
_PCM_SPEECH_SAMPLE_RATE_HZ = 24_000
|
|
_MIN_PCM_SPEECH_HNR_DB = 1.0
|
|
_PRESET_VOICE_GENDER_MAP: dict[str, str] = {
|
|
"serena": "female",
|
|
"uncle_fu": "male",
|
|
"chelsie": "female",
|
|
"clone": "female",
|
|
"ethan": "male",
|
|
}
|
|
|
|
|
|
def _short_transcript_contains_expected(transcript: str, expected: str) -> bool:
|
|
"""Allow minor ASR repeats/noise for very short speech clips."""
|
|
transcript_clean = preprocess_text(transcript)
|
|
expected_clean = preprocess_text(expected)
|
|
if not transcript_clean or not expected_clean:
|
|
return False
|
|
|
|
transcript_words = transcript_clean.split()
|
|
expected_words = expected_clean.split()
|
|
if not transcript_words or not expected_words:
|
|
return False
|
|
|
|
short_text = min(len(transcript_clean), len(expected_clean)) <= 15
|
|
small_word_delta = len(transcript_words) <= len(expected_words) + 2
|
|
return short_text and small_word_delta and expected_clean in transcript_clean
|
|
|
|
|
|
_MAX_TAIL_WORDS_FLOOR = 2
|
|
_TAIL_WORD_RATIO = 0.2
|
|
|
|
|
|
def _transcript_has_bounded_tail(transcript: str, expected: str) -> bool:
|
|
"""Pass when the expected text is spoken verbatim, only trailed by noise.
|
|
|
|
A short, unrelated tail after the complete expected sentence is the exact
|
|
case #6828's rationale anticipated: a different valid TTS sample can append
|
|
an audible tail even when the spoken text is identical, and Whisper can
|
|
hallucinate brief words on a non-speech tail. The length-penalized n-gram
|
|
cosine stays below the gate for such tails, so accept the match when the
|
|
expected text is a word-aligned prefix of the transcript and the trailing
|
|
words stay within the floor/ratio bounds.
|
|
"""
|
|
transcript_clean = preprocess_text(transcript)
|
|
expected_clean = preprocess_text(expected)
|
|
if not transcript_clean or not expected_clean:
|
|
return False
|
|
|
|
transcript_words = transcript_clean.split()
|
|
expected_words = expected_clean.split()
|
|
if not transcript_words or not expected_words:
|
|
return False
|
|
|
|
if transcript_words[: len(expected_words)] != expected_words:
|
|
return False
|
|
tail_words = len(transcript_words) - len(expected_words)
|
|
max_tail_words = max(_MAX_TAIL_WORDS_FLOOR, math.ceil(_TAIL_WORD_RATIO * len(expected_words)))
|
|
return tail_words <= max_tail_words
|
|
|
|
|
|
def assert_image_diffusion_response(
|
|
response: "DiffusionResponse",
|
|
request_config: dict[str, Any],
|
|
run_level: str | None = None,
|
|
) -> None:
|
|
"""
|
|
Validate image diffusion response.
|
|
|
|
Expected request_config schema:
|
|
{
|
|
"request_type": "image",
|
|
"extra_body": {
|
|
"num_outputs_per_prompt": 1,
|
|
"width": ...,
|
|
"height": ...,
|
|
...
|
|
}
|
|
}
|
|
"""
|
|
assert response.images is not None, "Image response is None"
|
|
assert len(response.images) > 0, "No images in response"
|
|
|
|
extra_body = request_config.get("extra_body") or {}
|
|
|
|
num_outputs_per_prompt = extra_body.get("num_outputs_per_prompt")
|
|
if num_outputs_per_prompt is not None:
|
|
assert len(response.images) == num_outputs_per_prompt, (
|
|
f"Expected {num_outputs_per_prompt} images, got {len(response.images)}"
|
|
)
|
|
|
|
if run_level in {"advanced_model", "full_model"}:
|
|
width = extra_body.get("width")
|
|
height = extra_body.get("height")
|
|
|
|
if width is not None or height is not None:
|
|
if isinstance(width, (list, tuple)) and isinstance(height, (list, tuple)):
|
|
assert len(response.images) == len(width) == len(height), (
|
|
f"Per-output size lists require one image per entry; got {len(response.images)} images, "
|
|
f"len(width)={len(width)}, len(height)={len(height)}"
|
|
)
|
|
for img, w, h in zip(response.images, width, height, strict=True):
|
|
assert_image_valid(img, width=int(w), height=int(h))
|
|
else:
|
|
for img in response.images:
|
|
assert_image_valid(
|
|
img,
|
|
width=_maybe_int(width) if width is not None else None,
|
|
height=_maybe_int(height) if height is not None else None,
|
|
)
|
|
|
|
|
|
def assert_images_generations_response(
|
|
response: dict[str, Any],
|
|
request_config: dict[str, Any],
|
|
run_level: str | None = None,
|
|
) -> None:
|
|
"""Validate a successful ``/v1/images/generations`` JSON response."""
|
|
del run_level
|
|
request_body = request_config.get("json") or {}
|
|
data = response.get("data")
|
|
assert isinstance(data, list), "Image generation response is missing data[]"
|
|
|
|
expected_count = int(request_body.get("n", 1))
|
|
assert len(data) == expected_count, f"Expected {expected_count} images, got {len(data)}"
|
|
|
|
width = height = None
|
|
size = request_body.get("size")
|
|
if isinstance(size, str) and "x" in size:
|
|
width_value, height_value = size.lower().split("x", 1)
|
|
width, height = int(width_value), int(height_value)
|
|
|
|
for item in data:
|
|
assert isinstance(item, dict), "Image generation data entries must be objects"
|
|
b64_json = item.get("b64_json")
|
|
assert isinstance(b64_json, str) and b64_json, "Image generation response is missing b64_json"
|
|
image = Image.open(io.BytesIO(base64.b64decode(b64_json)))
|
|
image.load()
|
|
assert_image_valid(image, width=width, height=height)
|
|
|
|
|
|
def assert_video_diffusion_response(
|
|
response: "DiffusionResponse",
|
|
request_config: dict[str, Any],
|
|
run_level: str | None = None,
|
|
) -> None:
|
|
"""
|
|
Validate video diffusion response.
|
|
|
|
Expected request_config schema:
|
|
{
|
|
"request_type": "video",
|
|
"form_data": {
|
|
"prompt": "...",
|
|
"num_frames": ...,
|
|
"width": ...,
|
|
"height": ...,
|
|
"fps": ...,
|
|
...
|
|
},
|
|
"expected_audio": {"sample_rate": 44100, "channels": 2},
|
|
"fps_tolerance": 0.01,
|
|
}
|
|
"""
|
|
form_data = request_config.get("form_data", {})
|
|
|
|
assert response.videos is not None, "Video response is None"
|
|
assert len(response.videos) > 0, "No videos in response"
|
|
|
|
expected_frames = _maybe_int(form_data.get("num_frames"))
|
|
expected_width = _maybe_int(form_data.get("width"))
|
|
expected_height = _maybe_int(form_data.get("height"))
|
|
expected_fps = _maybe_float(form_data.get("fps"))
|
|
expected_audio = request_config.get("expected_audio")
|
|
|
|
# Skip num_frames assertion for Helios models because they round up frames
|
|
model = request_config.get("model", "")
|
|
if "Helios" in model:
|
|
expected_frames = None
|
|
|
|
for vid_bytes in response.videos:
|
|
assert_video_valid(
|
|
vid_bytes,
|
|
num_frames=expected_frames,
|
|
width=expected_width,
|
|
height=expected_height,
|
|
fps=expected_fps,
|
|
fps_tolerance=request_config.get("fps_tolerance", 1.0),
|
|
)
|
|
if expected_audio is not None:
|
|
assert isinstance(expected_audio, dict), "expected_audio must be an object"
|
|
sample_rate = _maybe_int(expected_audio.get("sample_rate"))
|
|
channels = _maybe_int(expected_audio.get("channels"))
|
|
assert sample_rate is not None, "expected_audio.sample_rate is required"
|
|
assert channels is not None, "expected_audio.channels is required"
|
|
assert_video_audio_valid(
|
|
vid_bytes,
|
|
sample_rate=sample_rate,
|
|
channels=channels,
|
|
)
|
|
|
|
|
|
def assert_video_first_frame_matches(
|
|
video: Path | bytes | BytesIO,
|
|
expected: np.ndarray,
|
|
*,
|
|
max_mean_absolute_error: float,
|
|
) -> None:
|
|
"""Assert that an encoded video's first frame matches a reference image."""
|
|
if isinstance(video, Path):
|
|
source: str | BytesIO = str(video)
|
|
else:
|
|
video_bytes = video if isinstance(video, bytes) else video.getvalue()
|
|
source = BytesIO(video_bytes)
|
|
|
|
with av.open(source) as container:
|
|
first_frame = next(container.decode(video=0)).to_ndarray(format="rgb24")
|
|
|
|
assert first_frame.shape == expected.shape
|
|
mean_absolute_error = float(np.abs(first_frame.astype(np.float32) - expected).mean() / 255.0)
|
|
assert mean_absolute_error < max_mean_absolute_error, (
|
|
f"Expected first-frame MAE < {max_mean_absolute_error}, got {mean_absolute_error:.6f}."
|
|
)
|
|
|
|
|
|
def assert_video_first_frames_differ(
|
|
video: bytes,
|
|
reference: bytes,
|
|
*,
|
|
min_mean_absolute_error: float,
|
|
) -> None:
|
|
"""Compare decoded pixels, ignoring differences in MP4 encoding metadata."""
|
|
with av.open(BytesIO(video)) as container:
|
|
actual = next(container.decode(video=0)).to_ndarray(format="rgb24")
|
|
with av.open(BytesIO(reference)) as container:
|
|
expected = next(container.decode(video=0)).to_ndarray(format="rgb24")
|
|
|
|
assert actual.shape == expected.shape
|
|
mean_absolute_error = float(np.abs(actual.astype(np.float32) - expected).mean() / 255.0)
|
|
assert mean_absolute_error > min_mean_absolute_error, (
|
|
f"Expected first-frame MAE > {min_mean_absolute_error}, got {mean_absolute_error:.6f}."
|
|
)
|
|
|
|
|
|
def assert_audio_diffusion_response(
|
|
response: "DiffusionResponse",
|
|
request_config: dict[str, Any],
|
|
run_level: str | None = None,
|
|
) -> None:
|
|
"""
|
|
Validate audio diffusion response.
|
|
|
|
`response.audios` carries one entry per choice, each a `dict` with raw WAV
|
|
bytes (`wav_bytes`) and the OpenAI audio metadata (`id`, `expires_at`).
|
|
"""
|
|
assert response.audios, "Audio response is empty"
|
|
for audio in response.audios:
|
|
wav_bytes = audio.get("wav_bytes")
|
|
assert wav_bytes, "Audio entry missing decoded WAV bytes"
|
|
with wave.open(io.BytesIO(wav_bytes), "rb") as wav_file:
|
|
assert wav_file.getnframes() > 0, "Decoded WAV has zero frames"
|
|
assert wav_file.getframerate() > 0, "Decoded WAV has invalid sample rate"
|
|
|
|
|
|
def _maybe_int(value: Any) -> int | None:
|
|
if value is None:
|
|
return None
|
|
return int(value)
|
|
|
|
|
|
def _maybe_float(value: Any) -> float | None:
|
|
if value is None:
|
|
return None
|
|
return float(value)
|
|
|
|
|
|
def assert_image_valid(image: Path | Image.Image, *, width: int | None = None, height: int | None = None):
|
|
"""Assert the file is a loadable image with optional exact dimensions."""
|
|
if isinstance(image, Path):
|
|
assert image.exists(), f"Image not found: {image}"
|
|
image = Image.open(image)
|
|
image.load()
|
|
assert image.width > 0 and image.height > 0
|
|
if width is not None:
|
|
assert image.width == width, f"Expected width={width}, got {image.width}"
|
|
if height is not None:
|
|
assert image.height == height, f"Expected height={height}, got {image.height}"
|
|
return image
|
|
|
|
|
|
def assert_video_valid(
|
|
video: Path | bytes | BytesIO,
|
|
*,
|
|
num_frames: int | None = None,
|
|
width: int | None = None,
|
|
height: int | None = None,
|
|
fps: float | None = None,
|
|
fps_tolerance: float = 1.0,
|
|
) -> dict[str, int | float]:
|
|
"""Assert the MP4 has the expected resolution and frame count.
|
|
|
|
For several diffusion backends, encoded MP4 frame count follows a codec-aligned
|
|
convention (e.g. request `num_frames=8` can produce 9 encoded frames). Keep
|
|
this compatibility behavior to avoid false negatives in online-serving tests.
|
|
Fixed-rate models can request a stricter ``fps_tolerance`` in frames/second.
|
|
"""
|
|
temp_path = None
|
|
cap = None
|
|
try:
|
|
import cv2
|
|
|
|
if isinstance(video, Path):
|
|
if not video.exists():
|
|
raise AssertionError(f"Video file not found: {video}")
|
|
video_path = str(video)
|
|
else:
|
|
suffix = ".mp4"
|
|
with tempfile.NamedTemporaryFile(delete=False, suffix=suffix, mode="wb") as tmp:
|
|
if isinstance(video, bytes):
|
|
tmp.write(video)
|
|
elif isinstance(video, BytesIO):
|
|
tmp.write(video.getvalue())
|
|
else:
|
|
raise TypeError(f"Unsupported video type: {type(video)}")
|
|
temp_path = Path(tmp.name)
|
|
video_path = str(temp_path)
|
|
|
|
cap = cv2.VideoCapture(video_path)
|
|
if not cap.isOpened():
|
|
raise AssertionError("Failed to open video capture")
|
|
|
|
actual_width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
|
|
actual_height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
|
|
actual_fps = float(cap.get(cv2.CAP_PROP_FPS))
|
|
actual_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))
|
|
|
|
if width is not None:
|
|
assert actual_width == width, f"Expected width={width}, got {actual_width}"
|
|
if height is not None:
|
|
assert actual_height == height, f"Expected height={height}, got {actual_height}"
|
|
if fps is not None:
|
|
assert math.isfinite(actual_fps) and actual_fps > 0, f"Invalid video fps: {actual_fps}"
|
|
assert abs(actual_fps - float(fps)) < fps_tolerance, f"Expected fps~={fps}, got {actual_fps}"
|
|
if num_frames is not None:
|
|
expected_frames = (int(num_frames) // 4) * 4 + 1
|
|
assert actual_frames == expected_frames, f"Expected frames={expected_frames}, got {actual_frames}"
|
|
|
|
return {
|
|
"width": actual_width,
|
|
"height": actual_height,
|
|
"fps": actual_fps,
|
|
"num_frames": actual_frames,
|
|
}
|
|
except Exception as e:
|
|
print(f"ERROR: {type(e).__name__}: {e}", flush=True)
|
|
raise
|
|
finally:
|
|
if cap is not None:
|
|
cap.release()
|
|
if temp_path and temp_path.exists():
|
|
try:
|
|
temp_path.unlink()
|
|
except OSError:
|
|
pass
|
|
|
|
|
|
def assert_video_audio_valid(
|
|
video: Path | bytes | BytesIO,
|
|
*,
|
|
sample_rate: int,
|
|
channels: int,
|
|
) -> None:
|
|
"""Assert that an encoded video carries an audio stream with the requested format."""
|
|
if isinstance(video, Path):
|
|
source: str | BytesIO = str(video)
|
|
elif isinstance(video, bytes):
|
|
source = BytesIO(video)
|
|
elif isinstance(video, BytesIO):
|
|
video.seek(0)
|
|
source = video
|
|
else:
|
|
raise TypeError(f"Unsupported video type: {type(video)}")
|
|
|
|
with av.open(source) as container:
|
|
audio_streams = list(container.streams.audio)
|
|
assert audio_streams, "Video response does not contain an audio stream"
|
|
audio_stream = audio_streams[0]
|
|
actual_sample_rate = int(audio_stream.rate or 0)
|
|
actual_channels = int(audio_stream.channels or 0)
|
|
assert actual_sample_rate == sample_rate, f"Expected audio sample rate={sample_rate}, got {actual_sample_rate}"
|
|
assert actual_channels == channels, f"Expected audio channels={channels}, got {actual_channels}"
|
|
|
|
|
|
def assert_audio_valid(
|
|
audio_or_path: Path | np.ndarray,
|
|
*,
|
|
sample_rate: int,
|
|
channels: int,
|
|
duration_s: float,
|
|
) -> None:
|
|
"""Assert WAV file or (batch, channels, samples) ndarray matches expected audio format."""
|
|
expected_samples = int(duration_s * sample_rate)
|
|
if isinstance(audio_or_path, np.ndarray):
|
|
audio = audio_or_path
|
|
assert audio.ndim == 3, f"Expected audio ndim=3 (batch, channels, samples), got shape {audio.shape}"
|
|
assert audio.shape[0] == 1, f"Expected batch size 1, got {audio.shape[0]}"
|
|
assert audio.shape[1] == channels, f"Expected {channels} channels, got {audio.shape[1]}"
|
|
assert audio.shape[2] == expected_samples, (
|
|
f"Expected {expected_samples} samples ({duration_s}s @ {sample_rate} Hz), got {audio.shape[2]}"
|
|
)
|
|
return
|
|
|
|
path = audio_or_path
|
|
assert path.exists(), f"Audio not found: {path}"
|
|
info = sf.info(str(path))
|
|
assert info.samplerate == sample_rate, f"Expected sample_rate={sample_rate}, got {info.samplerate}"
|
|
assert info.channels == channels, f"Expected {channels} channel(s), got {info.channels}"
|
|
assert info.frames == expected_samples, (
|
|
f"Expected {expected_samples} frames ({duration_s}s @ {sample_rate} Hz), got {info.frames}"
|
|
)
|
|
|
|
|
|
def _load_gender_pipeline():
|
|
global _GENDER_PIPELINE
|
|
if _GENDER_PIPELINE is not None:
|
|
return _GENDER_PIPELINE
|
|
model_name = "7wolf/wav2vec2-base-gender-classification"
|
|
try:
|
|
from transformers import pipeline
|
|
|
|
_GENDER_PIPELINE = pipeline(task="audio-classification", model=model_name, device=-1)
|
|
return _GENDER_PIPELINE
|
|
except Exception as exc: # pragma: no cover
|
|
print(f"Warning: failed to create gender pipeline '{model_name}': {exc}")
|
|
_GENDER_PIPELINE = None
|
|
return None
|
|
|
|
|
|
def _median_pitch_hz_from_autocorr(mono: np.ndarray, sr: int) -> float | None:
|
|
x = np.asarray(mono, dtype=np.float64)
|
|
x = x - np.mean(x)
|
|
if x.size < int(0.15 * sr):
|
|
return None
|
|
frame_len = int(0.04 * sr)
|
|
hop = max(frame_len // 2, 1)
|
|
f0_min_hz, f0_max_hz = 70.0, 400.0
|
|
lag_min = max(1, int(sr / f0_max_hz))
|
|
lag_max = min(frame_len - 2, int(sr / f0_min_hz))
|
|
if lag_max <= lag_min:
|
|
return None
|
|
win = np.hamming(frame_len)
|
|
pitches: list[float] = []
|
|
for start in range(0, int(x.shape[0]) - frame_len, hop):
|
|
frame = x[start : start + frame_len] * win
|
|
frame = frame - np.mean(frame)
|
|
if float(np.sqrt(np.mean(frame**2))) < 1e-4:
|
|
continue
|
|
ac = np.correlate(frame, frame, mode="full")[frame_len - 1 :]
|
|
ac = ac / (float(ac[0]) + 1e-12)
|
|
region = ac[lag_min : lag_max + 1]
|
|
peak_rel = int(np.argmax(region))
|
|
peak_lag = peak_rel + lag_min
|
|
if peak_lag <= 0:
|
|
continue
|
|
f0 = float(sr) / float(peak_lag)
|
|
if f0_min_hz <= f0 <= f0_max_hz:
|
|
pitches.append(f0)
|
|
if len(pitches) < 4:
|
|
return None
|
|
return float(np.median(np.asarray(pitches, dtype=np.float64)))
|
|
|
|
|
|
def _estimate_voice_gender_from_audio(audio_bytes: bytes) -> str:
|
|
data, sr = sf.read(io.BytesIO(audio_bytes), dtype="float32", always_2d=True)
|
|
if data.size == 0:
|
|
raise ValueError("Empty audio")
|
|
mono = np.mean(data, axis=1)
|
|
try:
|
|
target_sr = 16000
|
|
if int(sr) != target_sr and mono.size > 1:
|
|
src_len = int(mono.shape[0])
|
|
dst_len = max(1, int(round(src_len * float(target_sr) / float(sr))))
|
|
src_idx = np.arange(src_len, dtype=np.float32)
|
|
dst_idx = np.linspace(0, src_len - 1, dst_len, dtype=np.float32)
|
|
mono = np.interp(dst_idx, src_idx, mono.astype(np.float32, copy=False)).astype(np.float32)
|
|
sr = target_sr
|
|
|
|
median_f0 = _median_pitch_hz_from_autocorr(mono, sr)
|
|
clf = _load_gender_pipeline()
|
|
if clf is None:
|
|
print("gender model not available, returning 'unknown'")
|
|
return "unknown"
|
|
with _GENDER_PIPELINE_LOCK:
|
|
outputs = clf(mono, sampling_rate=sr)
|
|
if not outputs:
|
|
return "unknown"
|
|
top = outputs[0]
|
|
label = str(top.get("label", "")).lower()
|
|
conf = float(top.get("score", 0.0))
|
|
if conf < 0.6:
|
|
gender = "unknown"
|
|
elif ("female" in label) or ("жен" in label):
|
|
gender = "female"
|
|
elif ("male" in label) or ("муж" in label):
|
|
gender = "male"
|
|
else:
|
|
gender = "unknown"
|
|
|
|
if gender == "female" and median_f0 is not None and median_f0 < 165.0 and conf < 0.88:
|
|
print(f"gender pitch assist: reclassifying female->male (median_f0={median_f0:.1f} Hz, conf={conf:.3f})")
|
|
gender = "male"
|
|
elif gender == "male" and median_f0 is not None and median_f0 > 230.0 and conf < 0.88:
|
|
print(f"gender pitch assist: reclassifying male->female (median_f0={median_f0:.1f} Hz, conf={conf:.3f})")
|
|
gender = "female"
|
|
print(
|
|
f"gender classifier: label={label}, conf={conf:.3f}, gender={gender}"
|
|
+ (f", median_f0={median_f0:.1f}Hz" if median_f0 is not None else "")
|
|
)
|
|
return gender
|
|
except Exception as exc: # pragma: no cover
|
|
print(f"Warning: gender classification failed, returning 'unknown': {exc}")
|
|
return "unknown"
|
|
|
|
|
|
def _assert_preset_voice_gender_from_audio(
|
|
audio_bytes: bytes | None,
|
|
voice_name: str | None,
|
|
*,
|
|
response_format: str | None = None,
|
|
) -> None:
|
|
"""If ``voice_name`` matches a known preset, assert classifier gender matches (skip when unknown)."""
|
|
if response_format == "pcm":
|
|
return
|
|
if not voice_name or not audio_bytes:
|
|
return
|
|
key = str(voice_name).lower()
|
|
expected_gender = _PRESET_VOICE_GENDER_MAP.get(key)
|
|
if expected_gender is None:
|
|
return
|
|
estimated_gender = _estimate_voice_gender_from_audio(audio_bytes)
|
|
print(f"Preset voice gender check: preset={key!r}, estimated={estimated_gender!r}, expected={expected_gender!r}")
|
|
if estimated_gender != "unknown":
|
|
assert estimated_gender == expected_gender, (
|
|
f"{voice_name!r} is expected {expected_gender}, but {GENDER_MISMATCH_MESSAGE} {estimated_gender!r}"
|
|
)
|
|
|
|
|
|
def _compute_pcm_hnr_db(pcm_samples: np.ndarray, sr: int = _PCM_SPEECH_SAMPLE_RATE_HZ) -> float:
|
|
frame_len = int(0.03 * sr)
|
|
hop = frame_len // 2
|
|
hnr_values: list[float] = []
|
|
for start in range(0, len(pcm_samples) - frame_len, hop):
|
|
frame = pcm_samples[start : start + frame_len].astype(np.float32, copy=False)
|
|
frame = frame - np.mean(frame)
|
|
if np.max(np.abs(frame)) < 0.01:
|
|
continue
|
|
ac = np.correlate(frame, frame, mode="full")[len(frame) - 1 :]
|
|
ac = ac / (ac[0] + 1e-10)
|
|
min_lag = int(sr / 400)
|
|
max_lag = min(int(sr / 80), len(ac))
|
|
if min_lag >= max_lag:
|
|
continue
|
|
peak = float(np.max(ac[min_lag:max_lag]))
|
|
if 0 < peak < 1:
|
|
hnr_values.append(10 * np.log10(peak / (1 - peak + 1e-10)))
|
|
# Silence and constant signals provide no usable harmonic estimate. They
|
|
# must not pass a model-specific floor of zero or below.
|
|
return float(np.mean(hnr_values)) if hnr_values else float("-inf")
|
|
|
|
|
|
def _assert_pcm_int16_speech_hnr(
|
|
audio_bytes: bytes,
|
|
min_hnr_db: float = _MIN_PCM_SPEECH_HNR_DB,
|
|
sr: int = _PCM_SPEECH_SAMPLE_RATE_HZ,
|
|
) -> None:
|
|
"""Validate harmonic-to-noise ratio on raw int16 PCM from /v1/audio/speech.
|
|
|
|
min_hnr_db defaults to the global _MIN_PCM_SPEECH_HNR_DB (1.0 dB),
|
|
which matches the cleaner TTS models the helper was originally calibrated
|
|
for. Quieter codecs (e.g. MOSS-TTS-Nano, whose voice_clone output is
|
|
intrinsically around -2 dB) can pass a lower per-test threshold via
|
|
request_config["min_hnr_db"] to keep the catastrophic-failure check
|
|
while not gating CI on a model-intrinsic property.
|
|
|
|
``sr`` must be the stream's real sample rate: the autocorrelation lag window
|
|
is derived from it, so a wrong rate detunes the 80-400 Hz pitch search and
|
|
understates HNR for models that do not decode at 24 kHz.
|
|
"""
|
|
assert audio_bytes is not None and len(audio_bytes) >= 2, "missing PCM bytes"
|
|
assert len(audio_bytes) % 2 == 0, "PCM byte length must be aligned to int16"
|
|
pcm_samples = np.frombuffer(audio_bytes, dtype=np.int16).astype(np.float32) / 32768.0
|
|
hnr = _compute_pcm_hnr_db(pcm_samples, sr=sr)
|
|
print(f"PCM speech HNR: {hnr:.2f} dB (threshold: {min_hnr_db} dB, sr={sr})")
|
|
assert hnr >= min_hnr_db, f"PCM speech HNR={hnr:.2f} dB is below the configured floor of {min_hnr_db} dB."
|
|
|
|
|
|
def _response_has_audio_output(response: Any) -> bool:
|
|
if response.audio_bytes:
|
|
return len(response.audio_bytes) > 0
|
|
if isinstance(getattr(response, "audio_content", None), str) and response.audio_content.strip():
|
|
return True
|
|
audio_data = getattr(response, "audio_data", None)
|
|
return bool(audio_data)
|
|
|
|
|
|
def _omni_assertion_needs_audio_transcript(request_config: dict[str, Any], run_level: str | None) -> bool:
|
|
if run_level not in {"advanced_model", "full_model"}:
|
|
return False
|
|
modalities = request_config.get("modalities", ["text", "audio"])
|
|
if "audio" not in modalities:
|
|
return False
|
|
keywords_dict = request_config.get("key_words", {}) or {}
|
|
# When text is not an output modality, the keyword loop validates keywords
|
|
# against the audio transcript -- for keywords under ANY word_type
|
|
# (text/image/audio/video), not just "audio". Mirror that here so the
|
|
# transcript is actually computed; otherwise the loop hits
|
|
# `assert transcript is not None` with transcript=None (e.g. an audio-only
|
|
# request carrying key_words={"text": [...]}).
|
|
if "text" not in modalities and any(
|
|
keywords_dict.get(word_type) for word_type in ("text", "image", "audio", "video")
|
|
):
|
|
return True
|
|
if request_config.get("audio_ref_text"):
|
|
return True
|
|
return "text" in modalities
|
|
|
|
|
|
def _speech_assertion_needs_audio_transcript(request_config: dict[str, Any], run_level: str | None) -> bool:
|
|
if run_level not in {"advanced_model", "full_model"}:
|
|
return False
|
|
if request_config.get("response_format") == "pcm" and "transcript_pcm_sample_rate" not in request_config:
|
|
return False
|
|
return bool(request_config.get("transcript_expected_text", request_config.get("input")))
|
|
|
|
|
|
def _speech_pcm_sample_rate(request_config: dict[str, Any]) -> int:
|
|
expected = request_config.get("expected_sample_rate")
|
|
if "transcript_pcm_sample_rate" in request_config:
|
|
sample_rate = request_config["transcript_pcm_sample_rate"]
|
|
assert type(sample_rate) is int and sample_rate > 0, "transcript_pcm_sample_rate must be a positive integer"
|
|
assert expected is None or int(expected) == sample_rate, (
|
|
"transcript_pcm_sample_rate must match expected_sample_rate"
|
|
)
|
|
return sample_rate
|
|
return int(expected or _PCM_SPEECH_SAMPLE_RATE_HZ)
|
|
|
|
|
|
def _speech_audio_for_transcription(audio_bytes: bytes | None, request_config: dict[str, Any]) -> bytes | None:
|
|
"""Give ASR a WAV container when a speech test opts into raw PCM transcription."""
|
|
if not audio_bytes or request_config.get("response_format") != "pcm":
|
|
return audio_bytes
|
|
sample_rate = _speech_pcm_sample_rate(request_config)
|
|
assert len(audio_bytes) % 2 == 0, "PCM byte length must be aligned to int16"
|
|
with io.BytesIO() as buffer:
|
|
with wave.open(buffer, "wb") as wav_file:
|
|
wav_file.setnchannels(1)
|
|
wav_file.setsampwidth(2)
|
|
wav_file.setframerate(sample_rate)
|
|
wav_file.writeframes(audio_bytes)
|
|
return buffer.getvalue()
|
|
|
|
|
|
def _resolve_audio_transcript(
|
|
response: Any,
|
|
request_config: dict[str, Any],
|
|
run_level: str | None,
|
|
*,
|
|
speech_api: bool,
|
|
) -> str | None:
|
|
"""Run Whisper only when this run_level / request_config needs a transcript for assertions."""
|
|
needs = (
|
|
_speech_assertion_needs_audio_transcript(request_config, run_level)
|
|
if speech_api
|
|
else _omni_assertion_needs_audio_transcript(request_config, run_level)
|
|
)
|
|
if not needs:
|
|
return None
|
|
existing = getattr(response, "audio_content", None)
|
|
if isinstance(existing, str) and existing.strip():
|
|
return existing
|
|
audio_bytes = (
|
|
_speech_audio_for_transcription(getattr(response, "audio_bytes", None), request_config)
|
|
if speech_api
|
|
else getattr(response, "audio_bytes", None)
|
|
)
|
|
if not audio_bytes:
|
|
return None
|
|
return convert_audio_bytes_to_text(
|
|
audio_bytes,
|
|
model_size=request_config.get("transcript_model", "small"),
|
|
language=request_config.get("transcript_language"),
|
|
)
|
|
|
|
|
|
def assert_omni_response(response: Any, request_config: dict[str, Any], run_level):
|
|
"""
|
|
Validate response results.
|
|
|
|
Args:
|
|
response: OmniResponse object
|
|
|
|
Raises:
|
|
AssertionError: When the response does not meet validation criteria
|
|
"""
|
|
assert response.success, "The request failed."
|
|
|
|
modalities = request_config.get("modalities", ["text", "audio"])
|
|
|
|
if run_level in {"advanced_model", "full_model"}:
|
|
transcript = _resolve_audio_transcript(response, request_config, run_level, speech_api=False)
|
|
# Verify output success
|
|
if "audio" in modalities:
|
|
assert _response_has_audio_output(response), "No audio output is generated"
|
|
if transcript is not None:
|
|
print(f"audio content is: {transcript}")
|
|
speaker = request_config.get("speaker")
|
|
if speaker:
|
|
_assert_preset_voice_gender_from_audio(
|
|
response.audio_bytes,
|
|
speaker,
|
|
response_format=request_config.get("response_format"),
|
|
)
|
|
if "text" in modalities:
|
|
assert response.text_content is not None, "No text output is generated"
|
|
print(f"text content is: {response.text_content}")
|
|
|
|
# Verify keywords in output
|
|
word_types = ["text", "image", "audio", "video"]
|
|
keywords_dict = request_config.get("key_words", {})
|
|
for word_type in word_types:
|
|
keywords = keywords_dict.get(word_type)
|
|
if "text" in modalities:
|
|
if keywords:
|
|
text_lower = response.text_content.lower()
|
|
assert any(str(kw).lower() in text_lower for kw in keywords), (
|
|
"The output does not contain any of the keywords."
|
|
)
|
|
else:
|
|
if keywords:
|
|
assert transcript is not None, "No audio transcript for keyword validation"
|
|
audio_lower = transcript.lower()
|
|
assert any(str(kw).lower() in audio_lower for kw in keywords), (
|
|
"The output does not contain any of the keywords."
|
|
)
|
|
|
|
# Verify similarity (Whisper transcript vs streamed/detokenized text)
|
|
if "audio" in modalities:
|
|
audio_ref_text = request_config.get("audio_ref_text")
|
|
similarity_threshold = request_config.get("similarity_threshold", 0.8)
|
|
if "text" in modalities:
|
|
assert transcript is not None, "No audio transcript for similarity validation"
|
|
text_output = (response.text_content or "").strip()
|
|
# For very short outputs (e.g. one-word answers), n-gram cosine
|
|
# similarity with length penalty is unreliable because Whisper
|
|
# may hallucinate extra context around the short utterance. Use
|
|
# a containment check instead: the shorter text must appear in
|
|
# the longer one (after preprocessing removes punctuation).
|
|
_SHORT_TEXT_THRESHOLD = 15
|
|
if len(text_output) <= _SHORT_TEXT_THRESHOLD or len(transcript) <= _SHORT_TEXT_THRESHOLD:
|
|
shorter = text_output.lower() if len(text_output) <= len(transcript) else transcript.lower()
|
|
longer = transcript.lower() if len(text_output) <= len(transcript) else text_output.lower()
|
|
import re as _re
|
|
|
|
shorter_clean = _re.sub(r"[^\w\s]", "", shorter).strip()
|
|
longer_clean = _re.sub(r"[^\w\s]", "", longer).strip()
|
|
assert shorter_clean and (shorter_clean in longer_clean), (
|
|
f"{AUDIO_MISMATCH_MESSAGE} "
|
|
f"(short-text containment check failed: "
|
|
f"text={text_output!r}, transcript={transcript!r})"
|
|
)
|
|
print(f"short-text containment check passed: {shorter_clean!r} in {longer_clean!r}")
|
|
else:
|
|
similarity = cosine_similarity_text(
|
|
transcript.lower(),
|
|
text_output.lower(),
|
|
)
|
|
print(f"similarity is: {similarity}")
|
|
if similarity <= similarity_threshold and _transcript_has_bounded_tail(transcript, text_output):
|
|
# The full answer is spoken verbatim and only a short
|
|
# noise tail follows it; see #6828's rationale.
|
|
print(
|
|
"bounded-tail containment check passed: "
|
|
f"text={text_output!r} is a word-aligned prefix of "
|
|
f"transcript={transcript!r}"
|
|
)
|
|
else:
|
|
assert similarity > similarity_threshold, AUDIO_MISMATCH_MESSAGE
|
|
if audio_ref_text:
|
|
assert transcript is not None, "No audio transcript for reference-text validation"
|
|
audio_similarity = cosine_similarity_text(
|
|
transcript.strip().lower(),
|
|
str(audio_ref_text).lower(),
|
|
)
|
|
assert audio_similarity > similarity_threshold, (
|
|
f"The audio content does not match reference text: similarity={audio_similarity:.3f}"
|
|
)
|
|
|
|
|
|
def _assert_transcript_matches(
|
|
transcript: str,
|
|
audio_bytes: bytes | None,
|
|
expected_text: Any,
|
|
*,
|
|
threshold: float,
|
|
escalation_model: str | None = None,
|
|
language: str | None = None,
|
|
) -> None:
|
|
"""Assert spoken audio matches ``expected_text``.
|
|
|
|
``transcript`` is the fast whisper-``small`` result. If it clears
|
|
``threshold`` the check passes immediately.
|
|
|
|
If the cosine check fails, very short clips get a conservative containment
|
|
fallback that accepts minor ASR repeats/noise only when the expected text is
|
|
still present and the transcript has few extra words.
|
|
|
|
When ``escalation_model`` is set (opt-in via the ``transcript_escalation_model``
|
|
request_config key) and the fast check plus containment fallback fail, the
|
|
clip is re-transcribed with that stronger ASR and the assertion is decided on
|
|
its verdict -- so a weak whisper-``small`` mishear on a short clip does not
|
|
flake the gate, while a genuine model artifact still fails (the strong ASR
|
|
mismatches too). ``language`` applies to that escalated pass too, so a request
|
|
that pins a language does not silently fall back to auto-detection on retry.
|
|
"""
|
|
expected = str(expected_text).strip().lower()
|
|
similarity = cosine_similarity_text(transcript.strip().lower(), expected)
|
|
print(f"Cosine similarity: {similarity:.3f}")
|
|
if similarity > threshold:
|
|
return
|
|
|
|
if _short_transcript_contains_expected(transcript, expected):
|
|
print("short speech containment check passed")
|
|
return
|
|
|
|
if escalation_model and audio_bytes:
|
|
print(
|
|
f"whisper-small below threshold ({similarity:.2f} <= {threshold}); "
|
|
f"escalating to whisper-{escalation_model} to rule out an ASR mishear"
|
|
)
|
|
strong_transcript = convert_audio_bytes_to_text(audio_bytes, model_size=escalation_model, language=language)
|
|
strong_similarity = cosine_similarity_text(strong_transcript.strip().lower(), expected)
|
|
print(
|
|
f"audio content (whisper-{escalation_model}): {strong_transcript}\n"
|
|
f"Cosine similarity (whisper-{escalation_model}): {strong_similarity:.3f}"
|
|
)
|
|
assert strong_similarity > threshold, (
|
|
f"Transcript doesn't match input after ASR escalation: "
|
|
f"input={expected_text!r}; whisper-small='{transcript}' (sim={similarity:.2f}); "
|
|
f"whisper-{escalation_model}='{strong_transcript}' (sim={strong_similarity:.2f})"
|
|
)
|
|
return
|
|
|
|
assert similarity > threshold, (
|
|
f"Transcript doesn't match input: similarity={similarity:.2f}, transcript='{transcript}'"
|
|
)
|
|
|
|
|
|
def _retain_failed_speech_audio(
|
|
audio_bytes: bytes | None, transcript: str, expected_text: Any, error: AssertionError
|
|
) -> None:
|
|
"""Retain the exact ASR input when a CI speech-content assertion fails."""
|
|
directory = os.environ.get("VLLM_OMNI_FAILED_SPEECH_AUDIO_DIR")
|
|
if not directory or not audio_bytes:
|
|
return
|
|
stem = f"failed_speech_{hashlib.sha256(audio_bytes).hexdigest()[:16]}"
|
|
root = Path(directory)
|
|
try:
|
|
root.mkdir(parents=True, exist_ok=True)
|
|
(root / f"{stem}.wav").write_bytes(audio_bytes)
|
|
(root / f"{stem}.json").write_text(
|
|
json.dumps(
|
|
{"transcript": transcript, "expected_text": str(expected_text), "assertion": str(error)},
|
|
ensure_ascii=False,
|
|
indent=2,
|
|
)
|
|
)
|
|
print(f"Retained failed speech ASR input: {root / f'{stem}.wav'}")
|
|
except OSError as exc:
|
|
# Diagnostic I/O must never replace or suppress the content failure.
|
|
print(f"Could not retain failed speech ASR input: {exc}")
|
|
|
|
|
|
def assert_audio_speech_response(response: Any, request_config: dict[str, Any], run_level: str | None = None) -> None:
|
|
"""Validate speech API results from :class:`~tests.helpers.client.OmniResponse`.
|
|
|
|
Success-path checks only. Negative / contract cases belong on
|
|
:meth:`~tests.helpers.client.OnlineOmniClient.send_audio_speech_http_request`
|
|
with :func:`assert_http_error`.
|
|
"""
|
|
assert response.success, "The request failed."
|
|
|
|
if request_config.get("word_timestamps"):
|
|
timestamps = response.word_timestamps
|
|
assert isinstance(timestamps, list) and timestamps, "Expected nonempty X-Word-Timestamps"
|
|
previous_start = 0
|
|
for timestamp in timestamps:
|
|
assert isinstance(timestamp["word"], str) and timestamp["word"]
|
|
start, end = timestamp["start_ms"], timestamp["end_ms"]
|
|
assert isinstance(start, int) and isinstance(end, int)
|
|
assert previous_start <= start <= end
|
|
previous_start = start
|
|
|
|
# Optional floor on decoded audio size (models with very short clips may use a lower value).
|
|
min_audio = request_config.get("min_audio_bytes")
|
|
if min_audio is not None:
|
|
n = int(min_audio)
|
|
if n > 0:
|
|
ab = response.audio_bytes
|
|
assert ab is not None, "Expected audio bytes when min_audio_bytes is set"
|
|
assert len(ab) > n, f"Audio payload too small: {len(ab)} bytes, expected more than {n} (min_audio_bytes)"
|
|
|
|
req_fmt = request_config.get("response_format")
|
|
if req_fmt == "pcm" and response.audio_bytes:
|
|
if response.audio_format:
|
|
assert "pcm" in response.audio_format.lower(), (
|
|
f"Expected audio/pcm content-type, got {response.audio_format!r}"
|
|
)
|
|
elif req_fmt == "wav" and response.audio_format:
|
|
assert req_fmt in response.audio_format
|
|
|
|
expected_sample_rate = request_config.get("sample_rate")
|
|
if expected_sample_rate is not None and req_fmt == "wav" and response.audio_bytes:
|
|
info = sf.info(io.BytesIO(response.audio_bytes))
|
|
assert info.samplerate == int(expected_sample_rate), (
|
|
f"Expected sample_rate={expected_sample_rate}, got {info.samplerate}"
|
|
)
|
|
|
|
if run_level in {"advanced_model", "full_model"}:
|
|
if req_fmt == "pcm" and response.audio_bytes:
|
|
min_hnr_db = float(request_config.get("min_hnr_db", _MIN_PCM_SPEECH_HNR_DB))
|
|
# Raw PCM carries no header, so the HNR pitch search has to be told
|
|
# the rate. Defaulting to 24 kHz for a 44.1 kHz model shifts the
|
|
# search window to 147-735 Hz and misses the fundamental entirely,
|
|
# scoring clean speech ~1.9 dB lower than it is.
|
|
_assert_pcm_int16_speech_hnr(
|
|
response.audio_bytes,
|
|
min_hnr_db=min_hnr_db,
|
|
sr=_speech_pcm_sample_rate(request_config),
|
|
)
|
|
|
|
transcript = _resolve_audio_transcript(response, request_config, run_level, speech_api=True)
|
|
if transcript is not None:
|
|
expected_text = request_config.get("transcript_expected_text", request_config.get("input"))
|
|
if expected_text:
|
|
print(f"audio content is: {transcript}")
|
|
print(f"input text is: {expected_text}")
|
|
transcription_audio = _speech_audio_for_transcription(
|
|
getattr(response, "audio_bytes", None), request_config
|
|
)
|
|
try:
|
|
_assert_transcript_matches(
|
|
transcript,
|
|
transcription_audio,
|
|
expected_text,
|
|
threshold=0.9,
|
|
escalation_model=request_config.get("transcript_escalation_model"),
|
|
language=request_config.get("transcript_language"),
|
|
)
|
|
except AssertionError as exc:
|
|
_retain_failed_speech_audio(transcription_audio, transcript, expected_text, exc)
|
|
raise
|
|
_assert_preset_voice_gender_from_audio(
|
|
response.audio_bytes,
|
|
request_config.get("voice"),
|
|
response_format=request_config.get("response_format"),
|
|
)
|
|
|
|
|
|
def assert_diffusion_response(
|
|
response: "DiffusionResponse", request_config: dict[str, Any], run_level: str | None = None
|
|
):
|
|
assert response.success, "The request failed."
|
|
has_any_content = any(content is not None for content in (response.images, response.videos, response.audios))
|
|
assert has_any_content, "Response contains no images, videos, or audios"
|
|
if response.images is not None:
|
|
assert_image_diffusion_response(response=response, request_config=request_config, run_level=run_level)
|
|
if response.videos is not None:
|
|
assert_video_diffusion_response(response=response, request_config=request_config, run_level=run_level)
|
|
if response.audios is not None:
|
|
assert_audio_diffusion_response(response=response, request_config=request_config, run_level=run_level)
|
|
|
|
|
|
def _http_response_body_materialize(resp: Any) -> tuple[bytes, dict[str, Any] | None]:
|
|
"""Serialize ``HttpResponse``-like body to UTF-8 bytes and parse a JSON object when possible."""
|
|
jb = getattr(resp, "json_body", None)
|
|
if jb is not None:
|
|
raw = json.dumps(jb, ensure_ascii=False).encode("utf-8")
|
|
if isinstance(jb, dict):
|
|
return raw, jb
|
|
try:
|
|
parsed = json.loads(raw.decode("utf-8", errors="replace"))
|
|
except json.JSONDecodeError:
|
|
return raw, None
|
|
return raw, parsed if isinstance(parsed, dict) else None
|
|
err = getattr(resp, "error_message", None)
|
|
raw = (err or "").encode("utf-8", errors="replace")
|
|
try:
|
|
parsed = json.loads(raw.decode("utf-8", errors="replace"))
|
|
except json.JSONDecodeError:
|
|
return raw, None
|
|
return raw, parsed if isinstance(parsed, dict) else None
|
|
|
|
|
|
def assert_err_message_in_text(
|
|
haystack: str,
|
|
err_message: str | tuple[str, ...] | list[str] | set[str] | frozenset[str],
|
|
*,
|
|
sequence_match: Literal["all", "any"] = "all",
|
|
) -> None:
|
|
"""Assert ``err_message`` appears in ``haystack`` (case-insensitive).
|
|
|
|
For non-string sequences: ``sequence_match='all'`` requires every substring (HTTP-style);
|
|
``sequence_match='any'`` requires at least one (WebSocket first-frame JSON helpers).
|
|
"""
|
|
hl = haystack.lower()
|
|
if isinstance(err_message, (list, tuple, set, frozenset)):
|
|
if sequence_match == "all":
|
|
missing = [s for s in err_message if str(s).lower() not in hl]
|
|
assert not missing, (
|
|
f"Expected error text to contain all of {err_message!r}; missing {missing!r}. haystack={haystack!r}"
|
|
)
|
|
else:
|
|
assert any(str(s).lower() in hl for s in err_message), (
|
|
f"Expected error text to contain one of {err_message!r}. haystack={haystack!r}"
|
|
)
|
|
else:
|
|
assert str(err_message).lower() in hl, f"Expected error text to contain {str(err_message)!r}, got: {haystack!r}"
|
|
|
|
|
|
def assert_http_error(
|
|
resp: Any,
|
|
*,
|
|
err_code: int | tuple[int, ...] | list[int] | None = None,
|
|
err_message: str | tuple[str, ...] | list[str] | None = None,
|
|
websocket_json_message: bool = False,
|
|
) -> dict[str, Any] | None:
|
|
"""Validate a raw-HTTP :class:`~tests.helpers.client.HttpResponse`-like object.
|
|
|
|
Used by :class:`~tests.helpers.client.OnlineOmniClient` ``send_*_http_request`` helpers when
|
|
``request_config`` contains optional ``err_code`` and/or ``err_message``.
|
|
|
|
When ``websocket_json_message=True``, only ``json_body`` is checked (first JSON WebSocket text frame).
|
|
Tuple/list ``err_message`` then uses **any** substring match; HTTP mode still requires **all** pieces.
|
|
|
|
- ``err_code``: exact HTTP ``int``, or membership if a non-string sequence (e.g. ``(400, 422)``).
|
|
When ``err_code`` is set and the actual status is a client error in ``400..499`` other than ``404``,
|
|
the JSON body must include FastAPI ``detail`` and/or OpenAI-style ``error`` (2xx skips this).
|
|
- ``err_message``: substring match (case-insensitive) against serialized ``json_body`` and ``error_message``.
|
|
If ``err_message`` is a non-string sequence (``list`` / ``tuple`` / ``set`` / ``frozenset``), **every**
|
|
element must appear as a substring; a plain ``str`` still requires that single substring.
|
|
"""
|
|
if websocket_json_message:
|
|
if err_code is not None:
|
|
raise ValueError("assert_http_error: err_code is incompatible with websocket_json_message=True")
|
|
jb_ws = getattr(resp, "json_body", None)
|
|
assert jb_ws is not None, resp
|
|
if err_message is None:
|
|
return jb_ws if isinstance(jb_ws, dict) else None
|
|
assert_err_message_in_text(
|
|
json.dumps(jb_ws, ensure_ascii=False),
|
|
err_message,
|
|
sequence_match="any",
|
|
)
|
|
return jb_ws if isinstance(jb_ws, dict) else None
|
|
|
|
if err_code is None and err_message is None:
|
|
return None
|
|
|
|
actual = getattr(resp, "status_code", None)
|
|
assert actual is not None, "response missing status_code"
|
|
|
|
if err_code is not None:
|
|
if isinstance(err_code, int):
|
|
assert actual == err_code, (resp, err_code)
|
|
else:
|
|
allowed = tuple(err_code)
|
|
assert actual in allowed, (resp, allowed)
|
|
|
|
body_bytes, payload = _http_response_body_materialize(resp)
|
|
|
|
if err_code is not None and actual is not None and 400 <= actual < 500 and actual != 404:
|
|
assert payload is not None, getattr(resp, "error_message", resp)
|
|
assert "detail" in payload or "error" in payload, payload
|
|
|
|
if err_message is not None:
|
|
pieces: list[str] = []
|
|
jb = getattr(resp, "json_body", None)
|
|
if jb is not None:
|
|
pieces.append(json.dumps(jb, ensure_ascii=False))
|
|
em = getattr(resp, "error_message", None)
|
|
if em:
|
|
pieces.append(str(em))
|
|
haystack = "\n".join(pieces) if pieces else body_bytes.decode("utf-8", errors="replace")
|
|
assert_err_message_in_text(haystack, err_message, sequence_match="all")
|
|
|
|
return payload
|
|
|
|
|
|
__all__ = [
|
|
"assert_audio_diffusion_response",
|
|
"assert_audio_speech_response",
|
|
"assert_diffusion_response",
|
|
"assert_err_message_in_text",
|
|
"assert_http_error",
|
|
"assert_image_diffusion_response",
|
|
"assert_image_valid",
|
|
"assert_omni_response",
|
|
"assert_video_diffusion_response",
|
|
"assert_video_valid",
|
|
"assert_video_audio_valid",
|
|
"assert_audio_valid",
|
|
]
|