From 5cce797c036dbe919763d40ee08c681ba89fa0f6 Mon Sep 17 00:00:00 2001 From: TJian Date: Tue, 16 Jun 2026 16:39:23 +0800 Subject: [PATCH] [Bugfix] [CI] [ROCm] [CUDA] Fix voxtral test (#4380) Signed-off-by: tjtanaa --- .buildkite/test-amd-merge.yml | 21 +++++++++++++++---- .buildkite/test-amd-ready.yaml | 14 ------------- .../omni_connectors/test_shm_connector.py | 13 ++++++++++++ .../connectors/shm_connector.py | 8 +++++++ .../entrypoints/openai/audio_utils_mixin.py | 9 ++++++-- 5 files changed, 45 insertions(+), 20 deletions(-) diff --git a/.buildkite/test-amd-merge.yml b/.buildkite/test-amd-merge.yml index 1a5217b84..b1dcb78d2 100644 --- a/.buildkite/test-amd-merge.yml +++ b/.buildkite/test-amd-merge.yml @@ -179,17 +179,30 @@ steps: pytest -s -v tests/e2e/online_serving/test_qwen3_tts_base.py tests/e2e/offline_inference/test_qwen3_tts_base.py -m "advanced_model" --run-level "advanced_model" ' - - label: "Voxtral-TTS E2E Test" + # It is the whisper ASR validation step that is slow, not the Voxtral TTS generation. + # whisper ASR runs on CPU. + - label: "Voxtral-TTS E2E Test (Online Serving)" agent_pool: mi325_1 depends_on: amd-build mirror_hardwares: [amdproduction] grade: Blocking commands: - | - timeout 20m bash -c ' - export VLLM_LOGGING_LEVEL=DEBUG + timeout 40m bash -c ' export VLLM_WORKER_MULTIPROC_METHOD=spawn - pytest -s -v tests/e2e/online_serving/test_voxtral_tts.py tests/e2e/offline_inference/test_voxtral_tts.py -m "advanced_model" --run-level "advanced_model" + pytest -s -v tests/e2e/online_serving/test_voxtral_tts.py -m "advanced_model" --run-level "advanced_model" + ' + + - label: "Voxtral-TTS E2E Test (Offline Inference)" + agent_pool: mi325_1 + depends_on: amd-build + mirror_hardwares: [amdproduction] + grade: Blocking + commands: + - | + timeout 30m bash -c ' + export VLLM_WORKER_MULTIPROC_METHOD=spawn + pytest -s -v tests/e2e/offline_inference/test_voxtral_tts.py -m "advanced_model" --run-level "advanced_model" ' # TODO: Bagel test on ROCm is very unstable. @tjtanaa diff --git a/.buildkite/test-amd-ready.yaml b/.buildkite/test-amd-ready.yaml index 9da9507c8..1da5edd19 100644 --- a/.buildkite/test-amd-ready.yaml +++ b/.buildkite/test-amd-ready.yaml @@ -185,20 +185,6 @@ steps: - export VLLM_ALLOW_LONG_MAX_MODEL_LEN="1" - timeout 30m pytest -s -v tests/e2e/online_serving/test_qwen3_tts_customvoice.py -m "core_model" --run-level "core_model" - - label: "Voxtral-TTS E2E Test" - agent_pool: mi325_1 - depends_on: amd-build - mirror_hardwares: [amdproduction] - grade: Blocking - commands: - - | - timeout 20m bash -c ' - export VLLM_LOGGING_LEVEL=DEBUG - export VLLM_WORKER_MULTIPROC_METHOD=spawn - pytest -s -v tests/e2e/online_serving/test_voxtral_tts.py -m "advanced_model" --run-level "advanced_model" - pytest -s -v tests/e2e/offline_inference/test_voxtral_tts.py -m "advanced_model" --run-level "advanced_model" - ' - - label: "CosyVoice3-TTS E2E Test" agent_pool: mi325_1 depends_on: amd-build diff --git a/tests/distributed/omni_connectors/test_shm_connector.py b/tests/distributed/omni_connectors/test_shm_connector.py index 73928aaf6..572376aef 100644 --- a/tests/distributed/omni_connectors/test_shm_connector.py +++ b/tests/distributed/omni_connectors/test_shm_connector.py @@ -41,6 +41,19 @@ class TestKeyBasedReadWrite: result = connector.get("s0", "s1", "no_such_key_xyz", metadata=None) assert result is None + def test_get_empty_shm_race_returns_none(self, connector, monkeypatch): + def raise_empty_file(*args, **kwargs): + raise ValueError("cannot mmap an empty file") + + monkeypatch.setattr( + "vllm_omni.distributed.omni_connectors.connectors.shm_connector.shm_pkg.SharedMemory", + raise_empty_file, + ) + + result = connector.get("s0", "s1", "not_ready_yet", metadata=None) + + assert result is None + def test_rank_aware_keys_independent(self, connector): """Each TP rank writes/reads its own key — simulates homogeneous TP.""" payloads = {} diff --git a/vllm_omni/distributed/omni_connectors/connectors/shm_connector.py b/vllm_omni/distributed/omni_connectors/connectors/shm_connector.py index 63466d116..5de0d54db 100644 --- a/vllm_omni/distributed/omni_connectors/connectors/shm_connector.py +++ b/vllm_omni/distributed/omni_connectors/connectors/shm_connector.py @@ -116,6 +116,14 @@ class SharedMemoryConnector(OmniConnectorBase): return result except FileNotFoundError: return None + except ValueError as e: + # A receiver can observe a newly-created POSIX SHM object before + # the writer has finished sizing it. Treat that as "not ready yet" + # so async polling can retry without a traceback. + if "empty file" in str(e): + return None + logger.debug("_get_by_key: unexpected error reading SHM segment %s", get_key, exc_info=True) + return None except Exception: logger.debug("_get_by_key: unexpected error reading SHM segment %s", get_key, exc_info=True) return None diff --git a/vllm_omni/entrypoints/openai/audio_utils_mixin.py b/vllm_omni/entrypoints/openai/audio_utils_mixin.py index 11d792df6..621ba9520 100644 --- a/vllm_omni/entrypoints/openai/audio_utils_mixin.py +++ b/vllm_omni/entrypoints/openai/audio_utils_mixin.py @@ -96,12 +96,16 @@ class AudioMixin: else: waveform = torch.from_numpy(audio_tensor).unsqueeze(0) - # Match librosa.stft defaults: n_fft=2048, hop_length=n_fft//4 - n_fft = 2048 + # Use a speech-sized analysis window. The previous 2048-sample + # window is tuned for music and can smear short consonants after + # aggressive compression, which makes ASR transcript checks flaky. + n_fft = 768 hop_length = n_fft // 4 + window = torch.hann_window(n_fft, device=waveform.device, dtype=waveform.dtype) to_spec = torchaudio.transforms.Spectrogram( n_fft=n_fft, hop_length=hop_length, + window_fn=lambda *_args, **_kwargs: window, power=None, ) stretch = torchaudio.transforms.TimeStretch( @@ -111,6 +115,7 @@ class AudioMixin: to_wave = torchaudio.transforms.InverseSpectrogram( n_fft=n_fft, hop_length=hop_length, + window_fn=lambda *_args, **_kwargs: window, ) spec = to_spec(waveform)