Preserve baseline ASR fixtures for matching-main failure classification

Keep documentation and narrowband speech evaluator settings identical to Omni main. Matching-main failures remain documented failures rather than reasons to change evaluation or retry for a green build. Retain failed-audio diagnostics and release compatibility fixes.

Signed-off-by: tzhouam <tzhouam@connect.ust.hk>
This commit is contained in:
tzhouam committed 2026-10-04 17:54:28 +08:00
1 parent 0e5e37bde6
commit 2f17684774
3 files changed
+1 -86

No files matched your search

@@ -148,9 +148,6 @@ def test_sample_rate_001(omni_server, online_client) -> None:
"stream": False,
"response_format": "wav",
"sample_rate": 8000,
# Whisper-small can mishear the narrowband "kilohertz" pronunciation.
# Recheck the same WAV with stronger ASR, preserving the 0.9 gate.
"transcript_escalation_model": "large-v3",
"task_type": "CustomVoice",
"voice": "vivian",
}
@@ -181,9 +181,7 @@ def test_modality_control_003(omni_server) -> None:
# Verify text output same as audio output
wav_path = extract_last_audio_saved_path(result)
# Whisper-small hallucinates a repeated-letter tail on this fixture's WAV.
# Use stronger ASR while retaining the keyword and similarity assertions.
audio_content = convert_audio_file_to_text(output_path=f"./{wav_path}", model_size="large-v3")
audio_content = convert_audio_file_to_text(output_path=f"./{wav_path}")
print(f"text content is: {text_content}")
assert "cherry blossom" in audio_content, "The output does not contain any of the keywords."
print(f"audio content is: {audio_content}")
-80
View File
@@ -21,36 +21,6 @@ from tests.helpers.client import OmniResponse
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
@pytest.mark.parametrize("result", ["matching", "wrong_content", "missing_keyword"])
def test_qwen3_omni_documentation_strong_asr_preserves_content_gates(monkeypatch, result):
from tests.examples.online_serving import test_qwen3_omni as documentation
expected = "cherry blossom trees fill the foreground while a tower stands against the blue sky."
transcripts = {
"matching": expected,
"wrong_content": "cherry blossom is unrelated to this recording about baking bread in a kitchen.",
"missing_keyword": expected.replace("cherry blossom", "plum flowers"),
}
monkeypatch.setattr(
documentation,
"run_cmd",
lambda _command: f"Chat completion output from text: {expected}\nAudio saved to retained.wav\n",
)
def transcribe(output_path, model_size="small", language=None):
assert output_path == "./retained.wav"
return transcripts[result] if model_size == "large-v3" else "f" * 700
monkeypatch.setattr(documentation, "convert_audio_file_to_text", transcribe)
error = {
"wrong_content": "The audio content is not same as the text",
"missing_keyword": "The output does not contain any of the keywords",
}.get(result)
expectation = pytest.raises(AssertionError, match=error) if error else nullcontext()
with expectation:
documentation.test_modality_control_003(SimpleNamespace(model="Qwen/Qwen3-Omni-30B-A3B-Instruct"))
def test_short_transcript_repeat_passes_containment_fallback():
_assert_transcript_matches(
" How... how are you?",
@@ -70,56 +40,6 @@ def test_short_transcript_unrelated_text_still_fails():
)
@pytest.mark.parametrize("result", ["small_matches", "strong_matches", "both_mismatch", "wrong_sample_rate"])
def test_qwen3_tts_narrowband_asr_preserves_content_and_sample_rate_gates(monkeypatch, result):
from tests.e2e.online_serving import test_qwen3_tts_customvoice_expansion as speech
expected = "This response should be encoded as eight kilohertz audio."
sample_rate = 24_000 if result == "wrong_sample_rate" else 8000
buffer = io.BytesIO()
with wave.open(buffer, "wb") as wav:
wav.setnchannels(1)
wav.setsampwidth(2)
wav.setframerate(sample_rate)
wav.writeframes(_pcm_sine(sample_rate=sample_rate))
audio_bytes = buffer.getvalue()
models = []
def transcribe(raw_bytes, model_size="small", language=None):
assert raw_bytes == audio_bytes
models.append(model_size)
if result == "small_matches":
return expected
if model_size == "large-v3" and result == "strong_matches":
return "This response should be encoded as 8kHz audio."
return "This response should be encoded as 8K red audio."
monkeypatch.setattr(assertions, "convert_audio_bytes_to_text", transcribe)
monkeypatch.setattr(assertions, "_assert_preset_voice_gender_from_audio", lambda *_args, **_kwargs: None)
def request(config):
assert config["input"] == expected
assert_audio_speech_response(
OmniResponse(success=True, audio_bytes=audio_bytes, audio_format="audio/wav"), config, "full_model"
)
client = SimpleNamespace(send_audio_speech_request=request)
error = {
"both_mismatch": "after ASR escalation",
"wrong_sample_rate": "Expected sample_rate=8000",
}.get(result)
expectation = pytest.raises(AssertionError, match=error) if error else nullcontext()
with expectation:
speech.test_sample_rate_001(SimpleNamespace(model=speech.MODEL), client)
expected_models = {
"small_matches": ["small"],
"strong_matches": ["small", "large-v3"],
"both_mismatch": ["small", "large-v3"],
"wrong_sample_rate": [],
}[result]
assert models == expected_models
def _capture_transcribe(monkeypatch) -> dict:
captured: dict = {}