# SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project """Audio serving utility mixin. PUT HERE: - AudioMixin helpers that convert audio tensors/bytes/formats for speech and audio serving classes (shared by multiple serving paths). DO NOT PUT HERE: - FastAPI route / form / job orchestration peeled from ``api_server.py``. Put those under an audio package ``helpers.py`` (or audio serving modules) when extracted. LONGEVITY: - This root mixin is a **temporary shared home**. - TODO(#5227, P1.1): tidy up / move with the audio/speech family split in the Phase 1 audio PR; do not treat this file as the long-term owner. - Endpoint-family helpers under an audio package are the longer home for route-adjacent logic. See ``openai/README.md`` (utils vs helpers, no overlap). """ from io import BytesIO import numpy as np import torch import torchaudio from vllm.logger import init_logger from vllm_omni.entrypoints.openai.protocol.audio import ( DEFAULT_AUDIO_FORMAT, AudioChunkMetadata, AudioResponse, CreateAudio, ) # Re-exported: serving_speech and the audio tests import it from here. from vllm_omni.utils.audio_resample import StreamingAudioResampler # noqa: F401 try: import soundfile except ImportError: soundfile = None logger = init_logger(__name__) class AudioMixin: """Mixin class to add audio-related utilities.""" def create_audio(self, audio_obj: CreateAudio) -> AudioResponse: """Convert audio tensor to bytes in the specified format.""" audio_tensor = audio_obj.audio_tensor sample_rate = audio_obj.sample_rate response_format = audio_obj.response_format.lower() base64_encode = audio_obj.base64_encode speed = audio_obj.speed if soundfile is None: raise ImportError( "soundfile is required for audio generation. Please install it with: pip install soundfile" ) if audio_tensor.ndim > 2: raise ValueError( f"Unsupported audio tensor dimension: {audio_tensor.ndim}. " "Only mono (1D) and stereo (2D) are supported." ) if audio_tensor.ndim == 2 and audio_tensor.shape[0] == 2: # Convert from [channels, samples] to [samples, channels] audio_tensor = audio_tensor.T audio_tensor, sample_rate = self._apply_speed_adjustment(audio_tensor, speed, sample_rate) if audio_obj.output_sample_rate is not None and audio_obj.output_sample_rate != sample_rate: audio_tensor = self._resample_audio(audio_tensor, sample_rate, audio_obj.output_sample_rate) sample_rate = audio_obj.output_sample_rate supported_formats = { "wav": ("WAV", "audio/wav", {}), "pcm": ("RAW", "audio/pcm", {"subtype": "PCM_16"}), "flac": ("FLAC", "audio/flac", {}), "mp3": ("MP3", "audio/mpeg", {}), "opus": ("OGG", "audio/ogg", {"subtype": "OPUS"}), } if response_format not in supported_formats: logger.warning(f"Unsupported response format '{response_format}', defaulting to '{DEFAULT_AUDIO_FORMAT}'.") response_format = DEFAULT_AUDIO_FORMAT soundfile_format, media_type, kwargs = supported_formats[response_format] with BytesIO() as buffer: soundfile.write(buffer, audio_tensor, sample_rate, format=soundfile_format, **kwargs) audio_data = buffer.getvalue() if base64_encode: import base64 audio_data = base64.b64encode(audio_data).decode("utf-8") return AudioResponse( audio_data=audio_data, media_type=media_type, audio_metadata=AudioChunkMetadata( format=response_format, sample_rate_hz=int(sample_rate), frame_count=int(audio_tensor.shape[0]), channels=1 if audio_tensor.ndim == 1 else int(audio_tensor.shape[1]), ), ) @staticmethod def _resample_audio(audio_tensor: np.ndarray, source_rate: int, target_rate: int) -> np.ndarray: """Resample complete audio while preserving soundfile's channels-last layout.""" if source_rate == target_rate: return audio_tensor audio_array = np.asarray(audio_tensor) if not np.issubdtype(audio_array.dtype, np.floating): audio_array = audio_array.astype(np.float32) waveform = torch.from_numpy(audio_array.T.copy() if audio_array.ndim == 2 else audio_array.copy()) resampled = torchaudio.functional.resample(waveform, source_rate, target_rate).cpu().numpy() return resampled.T if audio_array.ndim == 2 else resampled def _apply_speed_adjustment(self, audio_tensor: np.ndarray, speed: float, sample_rate: int): """Apply speed adjustment to the audio tensor while preserving pitch. Uses torchaudio's phase vocoder (Spectrogram → TimeStretch → InverseSpectrogram) to stretch/compress audio in time without changing pitch. """ if speed == 1.0: return audio_tensor, sample_rate try: if not np.issubdtype(audio_tensor.dtype, np.floating): audio_tensor = audio_tensor.astype(np.float32) # Stereo numpy arrays use channels-last (T, C); # torch expects channels-first (C, T). channels_last = audio_tensor.ndim == 2 if channels_last: waveform = torch.from_numpy(audio_tensor.T) else: waveform = torch.from_numpy(audio_tensor).unsqueeze(0) # Use a speech-sized analysis window. The previous 2048-sample # window is tuned for music and can smear short consonants after # aggressive compression, which makes ASR transcript checks flaky. n_fft = 768 hop_length = n_fft // 4 window = torch.hann_window(n_fft, device=waveform.device, dtype=waveform.dtype) to_spec = torchaudio.transforms.Spectrogram( n_fft=n_fft, hop_length=hop_length, window_fn=lambda *_args, **_kwargs: window, power=None, ) stretch = torchaudio.transforms.TimeStretch( n_freq=n_fft // 2 + 1, hop_length=hop_length, ) to_wave = torchaudio.transforms.InverseSpectrogram( n_fft=n_fft, hop_length=hop_length, window_fn=lambda *_args, **_kwargs: window, ) spec = to_spec(waveform) stretched = stretch(spec, speed) expected_length = int(audio_tensor.shape[0] / speed) result = to_wave(stretched, length=expected_length) result = result.squeeze(0).numpy() if channels_last: result = result.T return result, sample_rate except Exception as e: logger.error(f"An error occurred during speed adjustment: {e}") raise ValueError("Failed to apply speed adjustment.") from e