diff --git a/tests/benchmarks/test_seed_tts_dataset_variants.py b/tests/benchmarks/test_seed_tts_dataset_variants.py index 31f66bd7d..17ee6df4c 100644 --- a/tests/benchmarks/test_seed_tts_dataset_variants.py +++ b/tests/benchmarks/test_seed_tts_dataset_variants.py @@ -26,6 +26,7 @@ _MODULE_NAME = "vllm_omni.benchmarks.data_modules.seed_tts_dataset" if _MODULE_NAME not in sys.modules: _spec = importlib.util.spec_from_file_location(_MODULE_NAME, _MODULE_PATH) + assert _spec is not None and _spec.loader is not None _mod = importlib.util.module_from_spec(_spec) sys.modules[_MODULE_NAME] = _mod _spec.loader.exec_module(_mod) @@ -237,7 +238,8 @@ def test_seed_tts_design_dataset_rejects_missing_description(seed_tts_design_roo assert req.seed_tts_utterance_id == "ok" -def test_seed_tts_whisper_transcribe_passes_attention_mask(monkeypatch): +@pytest.mark.parametrize("audio_seconds", [0.1, 61]) +def test_seed_tts_whisper_transcribe_passes_attention_mask(monkeypatch, audio_seconds): from vllm_omni.benchmarks.data_modules import seed_tts_eval calls = {} @@ -252,8 +254,9 @@ def test_seed_tts_whisper_transcribe_passes_attention_mask(monkeypatch): return self class FakeProcessor: - def __call__(self, wav, *, sampling_rate, return_tensors, return_attention_mask=False): + def __call__(self, wav, *, sampling_rate, return_tensors, return_attention_mask=False, **kwargs): calls["return_attention_mask"] = return_attention_mask + calls["processor_kwargs"] = kwargs assert sampling_rate == 16000 assert return_tensors == "pt" assert len(wav) > 0 @@ -262,9 +265,10 @@ def test_seed_tts_whisper_transcribe_passes_attention_mask(monkeypatch): attention_mask=FakeTensor("mask") if return_attention_mask else None, ) - def get_decoder_prompt_ids(self, *, language, task): + def get_decoder_prompt_ids(self, *, language, task, no_timestamps=True): assert language == "english" assert task == "transcribe" + calls["no_timestamps"] = no_timestamps return [(1, 2)] def batch_decode(self, predicted_ids, *, skip_special_tokens): @@ -283,10 +287,18 @@ def test_seed_tts_whisper_transcribe_passes_attention_mask(monkeypatch): monkeypatch.setattr(seed_tts_eval, "_en_model", FakeModel()) monkeypatch.setattr(seed_tts_eval, "_device", "cuda:1") - text = seed_tts_eval._transcribe_en_f32_16k(np.ones(1600, dtype=np.float32)) + text = seed_tts_eval._transcribe_en_f32_16k(np.ones(int(audio_seconds * 16000), dtype=np.float32)) assert text == "hello" assert calls["return_attention_mask"] is True assert calls["input_features"].device == "cuda:1" assert calls["generate_kwargs"]["attention_mask"].device == "cuda:1" assert calls["generate_kwargs"]["forced_decoder_ids"] == [(1, 2)] + if audio_seconds > 30: + assert calls["processor_kwargs"] == {"truncation": False, "padding": "longest"} + assert calls["generate_kwargs"]["return_timestamps"] is True + assert calls["no_timestamps"] is False + else: + assert calls["processor_kwargs"] == {} + assert "return_timestamps" not in calls["generate_kwargs"] + assert calls["no_timestamps"] is True diff --git a/vllm_omni/benchmarks/data_modules/seed_tts_eval.py b/vllm_omni/benchmarks/data_modules/seed_tts_eval.py index ddfbbd630..18a72a6c1 100644 --- a/vllm_omni/benchmarks/data_modules/seed_tts_eval.py +++ b/vllm_omni/benchmarks/data_modules/seed_tts_eval.py @@ -1,3 +1,6 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + """Seed-TTS WER aligned with Bytedance ``seed-tts-eval`` / ``run_wer.py``. Matches the published protocol (see Hugging Face dataset card and @@ -460,25 +463,33 @@ def _transcribe_en_f32_16k(wav_f32: np.ndarray) -> str: return "" with _lock: assert _en_processor is not None and _en_model is not None and _device is not None + # Whisper's default feature extraction truncates at 30 seconds. Keep + # abnormal tails so WER evaluates the complete generated response. + long_audio = len(wav_f32) > 30 * 16000 + processor_kwargs = {"truncation": False, "padding": "longest"} if long_audio else {} try: inputs = _en_processor( wav_f32, sampling_rate=16000, return_tensors="pt", return_attention_mask=True, + **processor_kwargs, ) except TypeError: - inputs = _en_processor(wav_f32, sampling_rate=16000, return_tensors="pt") + inputs = _en_processor(wav_f32, sampling_rate=16000, return_tensors="pt", **processor_kwargs) input_features = inputs.input_features.to(_device) attention_mask = getattr(inputs, "attention_mask", None) if attention_mask is None and isinstance(inputs, dict): attention_mask = inputs.get("attention_mask") generate_kwargs: dict[str, Any] = {} + if long_audio: + generate_kwargs["return_timestamps"] = True if attention_mask is not None: generate_kwargs["attention_mask"] = attention_mask.to(_device) with torch.no_grad(): try: - forced = _en_processor.get_decoder_prompt_ids(language="english", task="transcribe") + prompt_kwargs = {"no_timestamps": False} if long_audio else {} + forced = _en_processor.get_decoder_prompt_ids(language="english", task="transcribe", **prompt_kwargs) predicted_ids = _en_model.generate(input_features, forced_decoder_ids=forced, **generate_kwargs) except Exception: predicted_ids = _en_model.generate( @@ -544,14 +555,13 @@ def _expand_seed_tts_turn_outputs( turn_pcm = getattr(output, "tts_turn_pcm_bytes", None) session_pcm = getattr(output, "tts_output_pcm_bytes", None) for turn_index, turn in enumerate(turns): - expanded_requests.append( - replace( - request, - prompt=turn.target_text, - seed_tts_utterance_id=turn.utterance_id, - seed_tts_turns=(), - ) + turn_request = replace( + request, + seed_tts_utterance_id=turn.utterance_id, + seed_tts_turns=(), ) + turn_request.prompt = turn.target_text + expanded_requests.append(turn_request) turn_output = copy(output) if isinstance(turn_pcm, list) and turn_index < len(turn_pcm): turn_output.tts_output_pcm_bytes = turn_pcm[turn_index]