diff --git a/tests/helpers/media.py b/tests/helpers/media.py index 623c218ad..3e6331cd6 100644 --- a/tests/helpers/media.py +++ b/tests/helpers/media.py @@ -600,7 +600,7 @@ def concat_audio(audio_val) -> np.ndarray: def preprocess_text(text): import opencc - word_to_num = { + word_normalizations = { "zero": "0", "one": "1", "two": "2", @@ -612,10 +612,11 @@ def preprocess_text(text): "eight": "8", "nine": "9", "ten": "10", + "kilohertz": "khz", } - for word, num in word_to_num.items(): + for word, replacement in word_normalizations.items(): pattern = r"\b" + re.escape(word) + r"\b" - text = re.sub(pattern, num, text, flags=re.IGNORECASE) + text = re.sub(pattern, replacement, text, flags=re.IGNORECASE) text = re.sub(r"[^\w\s]", "", text) text = re.sub(r"\s+", " ", text) diff --git a/tests/helpers/tests/test_media.py b/tests/helpers/tests/test_media.py index 8045f7a69..d41999b21 100644 --- a/tests/helpers/tests/test_media.py +++ b/tests/helpers/tests/test_media.py @@ -15,6 +15,14 @@ from tests.helpers import media pytestmark = [pytest.mark.core_model, pytest.mark.cpu] +def test_preprocess_text_normalizes_spoken_kilohertz(): + expected = "This response should be encoded as eight kilohertz audio." + transcript = "This response should be encoded as 8 kHz audio." + + assert media.preprocess_text(expected) == media.preprocess_text(transcript) + assert media.cosine_similarity_text(expected, transcript) == pytest.approx(1.0) + + def _stub_transcribe(monkeypatch) -> dict: """Capture the kwargs the helper hands to whisper, with no model and no device.""" captured: dict = {}