Compare commits

..
4 changed files with 28 additions and 127 deletions

No files matched your search

-5
View File
@@ -156,11 +156,6 @@ steps:
- label: ":full_moon: TTS · Function Test with L4" - label: ":full_moon: TTS · Function Test with L4"
source_file_dependencies: tts_qwen3_tts_function source_file_dependencies: tts_qwen3_tts_function
timeout_in_minutes: 120 timeout_in_minutes: 120
env:
VLLM_OMNI_FAILED_SPEECH_AUDIO_DIR: .
artifact_paths:
- failed_speech_*.wav
- failed_speech_*.json
commands: commands:
- export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1" - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1"
- export VLLM_USE_DEEP_GEMM="0" - export VLLM_USE_DEEP_GEMM="0"
+7 -41
View File
@@ -4,11 +4,9 @@
"""Assertion and response validation helpers for tests.""" """Assertion and response validation helpers for tests."""
import base64 import base64
import hashlib
import io import io
import json import json
import math import math
import os
import tempfile import tempfile
import threading import threading
import wave import wave
@@ -1001,31 +999,6 @@ def _assert_transcript_matches(
) )
def _retain_failed_speech_audio(
audio_bytes: bytes | None, transcript: str, expected_text: Any, error: AssertionError
) -> None:
"""Retain the exact ASR input when a CI speech-content assertion fails."""
directory = os.environ.get("VLLM_OMNI_FAILED_SPEECH_AUDIO_DIR")
if not directory or not audio_bytes:
return
stem = f"failed_speech_{hashlib.sha256(audio_bytes).hexdigest()[:16]}"
root = Path(directory)
try:
root.mkdir(parents=True, exist_ok=True)
(root / f"{stem}.wav").write_bytes(audio_bytes)
(root / f"{stem}.json").write_text(
json.dumps(
{"transcript": transcript, "expected_text": str(expected_text), "assertion": str(error)},
ensure_ascii=False,
indent=2,
)
)
print(f"Retained failed speech ASR input: {root / f'{stem}.wav'}")
except OSError as exc:
# Diagnostic I/O must never replace or suppress the content failure.
print(f"Could not retain failed speech ASR input: {exc}")
def assert_audio_speech_response(response: Any, request_config: dict[str, Any], run_level: str | None = None) -> None: def assert_audio_speech_response(response: Any, request_config: dict[str, Any], run_level: str | None = None) -> None:
"""Validate speech API results from :class:`~tests.helpers.client.OmniResponse`. """Validate speech API results from :class:`~tests.helpers.client.OmniResponse`.
@@ -1090,21 +1063,14 @@ def assert_audio_speech_response(response: Any, request_config: dict[str, Any],
if expected_text: if expected_text:
print(f"audio content is: {transcript}") print(f"audio content is: {transcript}")
print(f"input text is: {expected_text}") print(f"input text is: {expected_text}")
transcription_audio = _speech_audio_for_transcription( _assert_transcript_matches(
getattr(response, "audio_bytes", None), request_config transcript,
_speech_audio_for_transcription(getattr(response, "audio_bytes", None), request_config),
expected_text,
threshold=0.9,
escalation_model=request_config.get("transcript_escalation_model"),
language=request_config.get("transcript_language"),
) )
try:
_assert_transcript_matches(
transcript,
transcription_audio,
expected_text,
threshold=0.9,
escalation_model=request_config.get("transcript_escalation_model"),
language=request_config.get("transcript_language"),
)
except AssertionError as exc:
_retain_failed_speech_audio(transcription_audio, transcript, expected_text, exc)
raise
_assert_preset_voice_gender_from_audio( _assert_preset_voice_gender_from_audio(
response.audio_bytes, response.audio_bytes,
request_config.get("voice"), request_config.get("voice"),
-81
View File
@@ -2,9 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project # SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
import io import io
import json
import wave import wave
from contextlib import nullcontext
from types import SimpleNamespace from types import SimpleNamespace
import numpy as np import numpy as np
@@ -251,85 +249,6 @@ def test_pcm_transcript_escalation_uses_same_valid_wav_and_rejects_two_wrong_tra
_assert_wav_wraps_pcm(wav_bytes, pcm_bytes, 24_000) _assert_wav_wraps_pcm(wav_bytes, pcm_bytes, 24_000)
@pytest.mark.parametrize("response_format", ["wav", "pcm"])
@pytest.mark.parametrize("escalation_model", [None, "large-v3"])
def test_failed_speech_capture_retains_exact_asr_input_and_original_failure(
monkeypatch, tmp_path, response_format, escalation_model
):
pcm_bytes = _pcm_sine()
wav = io.BytesIO()
with wave.open(wav, "wb") as writer:
writer.setnchannels(1)
writer.setsampwidth(2)
writer.setframerate(24_000)
writer.writeframes(pcm_bytes)
audio_bytes = wav.getvalue() if response_format == "wav" else pcm_bytes
transcribed = []
def transcribe(audio_bytes, **kwargs):
transcribed.append(audio_bytes)
return "this is unrelated speech"
directory = tmp_path / "failed_audio"
monkeypatch.setenv("VLLM_OMNI_FAILED_SPEECH_AUDIO_DIR", str(directory))
monkeypatch.setattr(assertions, "convert_audio_bytes_to_text", transcribe)
config = {
"input": "please read this requested sentence accurately",
"response_format": response_format,
"transcript_pcm_sample_rate": 24_000,
"transcript_escalation_model": escalation_model,
}
response = OmniResponse(success=True, audio_bytes=audio_bytes, audio_format=f"audio/{response_format}")
with pytest.raises(AssertionError, match="Transcript doesn't match input") as error:
assert_audio_speech_response(response, config, "full_model")
clips = list(directory.glob("*.wav"))
assert len(clips) == 1
assert all(clip == clips[0].read_bytes() for clip in transcribed)
_assert_wav_wraps_pcm(clips[0].read_bytes(), pcm_bytes, 24_000)
metadata = json.loads(clips[0].with_suffix(".json").read_text())
assert metadata["expected_text"] == config["input"]
assert metadata["transcript"] == "this is unrelated speech"
assert metadata["assertion"] == str(error.value)
@pytest.mark.parametrize("capture_enabled", [False, True])
@pytest.mark.parametrize("content_failure", [False, True])
def test_speech_capture_is_opt_in_and_does_not_write_successes(monkeypatch, tmp_path, capture_enabled, content_failure):
directory = tmp_path / "failed_audio"
if capture_enabled:
monkeypatch.setenv("VLLM_OMNI_FAILED_SPEECH_AUDIO_DIR", str(directory))
else:
monkeypatch.delenv("VLLM_OMNI_FAILED_SPEECH_AUDIO_DIR", raising=False)
expected = "please read this requested sentence accurately"
monkeypatch.setattr(
assertions,
"convert_audio_bytes_to_text",
lambda *_args, **_kwargs: "this is unrelated speech" if content_failure else expected,
)
with pytest.raises(AssertionError, match="Transcript doesn't match input") if content_failure else nullcontext():
assert_audio_speech_response(
OmniResponse(success=True, audio_bytes=b"audio", audio_format="audio/wav"),
{"input": expected, "response_format": "wav"},
"full_model",
)
assert directory.exists() is (capture_enabled and content_failure)
assert len(list(tmp_path.rglob("*.wav"))) == int(capture_enabled and content_failure)
def test_speech_capture_io_failure_does_not_mask_content_failure(monkeypatch, tmp_path):
directory = tmp_path / "blocked"
directory.write_text("a file cannot be an output directory")
monkeypatch.setenv("VLLM_OMNI_FAILED_SPEECH_AUDIO_DIR", str(directory))
monkeypatch.setattr(assertions, "convert_audio_bytes_to_text", lambda *_args, **_kwargs: "this is unrelated speech")
with pytest.raises(AssertionError, match="Transcript doesn't match input"):
assert_audio_speech_response(
OmniResponse(success=True, audio_bytes=b"audio", audio_format="audio/wav"),
{"input": "please read this requested sentence accurately", "response_format": "wav"},
"full_model",
)
@pytest.mark.parametrize( @pytest.mark.parametrize(
"pcm_bytes", "pcm_bytes",
[ [
@@ -981,6 +981,27 @@ def test_cosyvoice3_full_payload_replace_keys_present() -> None:
) )
def test_cosyvoice3_conditioning_replaces_snapshot_while_audio_accumulates() -> None:
from vllm_omni.model_executor.stage_input_processors.cosyvoice3 import _FULL_PAYLOAD_REPLACE_KEYS
from vllm_omni.outputs.mm_outputs import MultimodalPayload
from vllm_omni.outputs.output_modality import OutputModality
previous = {key: torch.ones(1, 174) for key in _FULL_PAYLOAD_REPLACE_KEYS}
padded = {key: torch.zeros(1, 1500) for key in _FULL_PAYLOAD_REPLACE_KEYS}
previous["embed.speech_token_len"] = torch.tensor([174])
padded["embed.speech_token_len"] = torch.tensor([174])
payload = MultimodalPayload.from_dict({**previous, "audio": torch.ones(2)})
incoming = MultimodalPayload.from_dict({**padded, "audio": torch.ones(3)})
assert payload is not None and incoming is not None
payload = payload.merged_with(incoming)
payload.consolidate_tensors(OutputModality.AUDIO)
for key in _FULL_PAYLOAD_REPLACE_KEYS:
assert torch.equal(payload[key], padded[key])
assert torch.equal(payload["audio"], torch.ones(5))
def test_ming_flash_omni_thinker2talker_token_only_smoke() -> None: def test_ming_flash_omni_thinker2talker_token_only_smoke() -> None:
"""Smoke: ming_flash_omni token-only carries voice metadata.""" """Smoke: ming_flash_omni token-only carries voice metadata."""
from vllm_omni.model_executor.stage_input_processors.ming_flash_omni import ( from vllm_omni.model_executor.stage_input_processors.ming_flash_omni import (