diff --git a/vllm_omni/model_executor/models/step_audio2/step_audio2_thinker.py b/vllm_omni/model_executor/models/step_audio2/step_audio2_thinker.py index b56229cf6..b00e24870 100644 --- a/vllm_omni/model_executor/models/step_audio2/step_audio2_thinker.py +++ b/vllm_omni/model_executor/models/step_audio2/step_audio2_thinker.py @@ -13,7 +13,6 @@ from __future__ import annotations from collections.abc import Iterable, Mapping, Sequence from typing import TYPE_CHECKING, Any, TypedDict -import librosa # noqa: TID251 import numpy as np import torch import torch.nn as nn @@ -52,6 +51,7 @@ from vllm_omni.model_executor.models.step_audio2.step_audio2_constants import ( DEFAULT_TOKEN_CONFIG, STEP_AUDIO2_AUDIO_PATCH_TOKEN_ID, ) +from vllm_omni.utils.audio import mel_filter_bank if TYPE_CHECKING: pass @@ -65,7 +65,7 @@ if TYPE_CHECKING: def _mel_filters(n_mels: int) -> torch.Tensor: """Generate mel filter banks""" assert n_mels in {80, 128}, f"Unsupported n_mels: {n_mels}" - return torch.from_numpy(librosa.filters.mel(sr=16000, n_fft=400, n_mels=n_mels)) + return mel_filter_bank(sr=16000, n_fft=400, n_mels=n_mels) def _normalize_audio(audio: Any) -> torch.Tensor: diff --git a/vllm_omni/model_executor/models/step_audio2/step_audio2_token2wav.py b/vllm_omni/model_executor/models/step_audio2/step_audio2_token2wav.py index 9666a4f5a..6e9236766 100644 --- a/vllm_omni/model_executor/models/step_audio2/step_audio2_token2wav.py +++ b/vllm_omni/model_executor/models/step_audio2/step_audio2_token2wav.py @@ -1,14 +1,15 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project + import io import os from collections.abc import Iterable from dataclasses import dataclass, field from typing import Any -import librosa # noqa: TID251 import numpy as np import onnxruntime import s3tokenizer -import soundfile as sf import torch import torch.nn as nn import torchaudio @@ -23,6 +24,8 @@ from vllm.config import VllmConfig from vllm.logger import init_logger from vllm.model_executor.models.interfaces import SupportsPP from vllm.model_executor.models.utils import WeightsMapper +from vllm.multimodal.audio import AudioResampler +from vllm.multimodal.media.audio import load_audio from vllm.sequence import IntermediateTensors from vllm.v1.outputs import SamplerOutput from vllm.v1.sample.metadata import SamplingMetadata @@ -231,19 +234,11 @@ class StepAudio2Token2WavCore(nn.Module): def _prepare_prompt(self, prompt_wav: str): """Prepare prompt audio for conditioning""" - # Prefer soundfile/librosa path to avoid torchaudio->torchcodec runtime coupling. - try: - audio_np_raw, sample_rate = sf.read(prompt_wav, dtype="float32", always_2d=False) - except Exception: - audio_np_raw, sample_rate = librosa.load(prompt_wav, sr=None, mono=True) - if isinstance(audio_np_raw, np.ndarray) and audio_np_raw.ndim > 1: - audio_np_raw = np.mean(audio_np_raw, axis=-1) + audio_np_raw, sample_rate = load_audio(prompt_wav, sr=None, mono=True) sample_rate = int(sample_rate) # 16 kHz branch: s3tokenizer + speaker embedding - audio_16k = audio_np_raw.astype(np.float32) - if sample_rate != 16000: - audio_16k = librosa.resample(y=audio_16k, orig_sr=sample_rate, target_sr=16000) + audio_16k = AudioResampler(target_sr=16000).resample(audio_np_raw.astype(np.float32), orig_sr=sample_rate) audio = torch.from_numpy(audio_16k) mels = s3tokenizer.log_mel_spectrogram(audio) mels, mels_lens = s3tokenizer.padding([mels]) @@ -257,9 +252,7 @@ class StepAudio2Token2WavCore(nn.Module): # 24 kHz branch: mel spectrogram for flow model conditioning # Must resample from the ORIGINAL audio, not the 16 kHz version. - audio_24k = audio_np_raw.astype(np.float32) - if sample_rate != 24000: - audio_24k = librosa.resample(y=audio_24k, orig_sr=sample_rate, target_sr=24000) + audio_24k = AudioResampler(target_sr=24000).resample(audio_np_raw.astype(np.float32), orig_sr=sample_rate) audio_24k_t = torch.from_numpy(audio_24k).unsqueeze(0) # [1, T] prompt_mel = ( mel_spectrogram(