[Refactor] Remove dependency on librosa (#2273)

Signed-off-by: Nick Cao <ncao@redhat.com>
Co-authored-by: Claude <noreply@anthropic.com>

examples/offline_inference/bagel/README.md

@@ -247,13 +247,6 @@ For more details on the Mooncake connector and multi-node setup, see the [Moonca
 
 ## FAQ
 
-- If you encounter an error about the backend of librosa, try to install ffmpeg with the command below.
-
-```bash
-sudo apt update
-sudo apt install ffmpeg
-```
-
 - If you don’t know how much VRAM is needed for the model or encounter the OOM error, you can try to decrease the max_model_len.
 
 | Stage               | VRAM                         |

examples/offline_inference/cosyvoice3/README.md

@@ -7,7 +7,7 @@ Install dependencies:
 uv pip install -e .
 ```
 
-> **Note:** This includes required libraries such as `librosa`, `soundfile`,
+> **Note:** This includes required libraries such as `soundfile`,
 > `onnxruntime`, `x-transformers`, and `einops` via
 > `requirements/common.txt` and platform-specific requirements files.
 

examples/offline_inference/cosyvoice3/verify_e2e_cosyvoice.py

@@ -2,13 +2,12 @@
 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
 import argparse
 import os
-from pathlib import Path
 
-import librosa
 import numpy as np
 import soundfile as sf
 from vllm import SamplingParams
 from vllm.assets.audio import AudioAsset
+from vllm.multimodal.media.audio import load_audio
 
 from vllm_omni.entrypoints.omni import Omni
 from vllm_omni.model_executor.models.cosyvoice3.config import CosyVoice3Config
@@ -16,22 +15,6 @@ from vllm_omni.model_executor.models.cosyvoice3.tokenizer import get_qwen_tokeni
 from vllm_omni.model_executor.models.cosyvoice3.utils import extract_text_token
 
 
-def _ensure_mel_filters_asset() -> None:
-    repo_root = Path(__file__).resolve().parents[3]
-    filters_path = repo_root / "vllm_omni" / "model_executor" / "models" / "cosyvoice3" / "assets" / "mel_filters.npz"
-    if filters_path.exists():
-        return
-
-    source_url = "https://raw.githubusercontent.com/openai/whisper/main/whisper/assets/mel_filters.npz"
-    raise FileNotFoundError(
-        "Missing CosyVoice3 mel filter asset:\n"
-        f"  {filters_path}\n"
-        "Download it with:\n"
-        f"  mkdir -p {filters_path.parent} && "
-        f"curl -L {source_url} -o {filters_path}"
-    )
-
-
 def run_e2e():
     parser = argparse.ArgumentParser()
     # ""FunAudioLLM/Fun-CosyVoice3-0.5B-2512
@@ -56,7 +39,6 @@ def run_e2e():
         help="Path to tokenizer directory (e.g., <model_path>/CosyVoice-BlankEN).",
     )
     args = parser.parse_args()
-    _ensure_mel_filters_asset()
     # Ensure tokenizer directory exists
     if not os.path.exists(args.tokenizer):
         raise FileNotFoundError(f"{args.tokenizer} does not exist!")
@@ -85,7 +67,7 @@ def run_e2e():
         if not os.path.exists(args.audio_path):
             raise FileNotFoundError(f"Audio file not found: {args.audio_path}")
         # Load at native sample rate
-        audio_signal, sr = librosa.load(args.audio_path, sr=None)
+        audio_signal, sr = load_audio(args.audio_path, sr=None)
 
         # Validate sample rate before processing (similar to original CosyVoice)
         min_sr = 16000

examples/offline_inference/mimo_audio/README.md

@@ -190,29 +190,6 @@ Note: This task uses hardcoded message lists in the script.
 
 ## Troubleshooting
 
-### Audio dependencies (soundfile, librosa)
-
-This example depends on **soundfile** (read/write WAV) and **librosa** (load audio including MP3). Install the project requirements first:
-
-```bash
-pip install -r requirements/common.txt
-# or at least: pip install soundfile>=0.13.1 librosa>=0.11.0
-```
-
-- **`soundfile` / libsndfile not found**  
-  `soundfile` uses the C library **libsndfile**. On Linux, install the system package before pip:
-  - Debian/Ubuntu: `sudo apt-get install libsndfile1`
-  - For development builds: `sudo apt-get install libsndfile1-dev`
-  - Then: `pip install soundfile`
-
-- **`librosa` fails to load MP3 or reports "No backend available"**  
-  Loading MP3 (e.g. in `spoken_dialogue_sft_multiturn` with `.mp3` files) uses **ffmpeg** as the backend. Install ffmpeg:
-  - Debian/Ubuntu: `sudo apt-get install ffmpeg`
-  - macOS: `brew install ffmpeg`
-
-- **`ImportError: No module named 'soundfile'` or `ModuleNotFoundError: ... librosa`**  
-  Ensure you are in the same Python environment where vLLM Omni and the example dependencies are installed, and that `requirements/common.txt` (or the packages above) are installed.
-
 ### Tokenizer path
 
 - **`MIMO_AUDIO_TOKENIZER_PATH` not set or model fails to find tokenizer**  

examples/offline_inference/mimo_audio/message_convert.py

@@ -5,12 +5,12 @@ import random
 import re
 from collections.abc import Callable
 
-import librosa
 import numpy as np
 import torch
 import torchaudio
 from process_speechdata import InputSegment, StreamingInputSegment
 from torchaudio.transforms import MelSpectrogram
+from vllm.multimodal.media.audio import load_audio
 
 speech_zeroemb_idx = 151667
 empty_token = "<|empty|>"
@@ -685,7 +685,7 @@ def get_audio_data(audio_url):
         # File path
         audio_file = audio_url
 
-    audio_signal, sr = librosa.load(audio_file, sr=24000)
+    audio_signal, sr = load_audio(audio_file, sr=24000)
     audio_data = (audio_signal.astype(np.float32), sr)
     return audio_data
 

examples/offline_inference/omnivoice/end2end.py

@@ -103,9 +103,9 @@ def run_e2e():
         if not os.path.exists(args.ref_audio):
             raise FileNotFoundError(f"Reference audio not found: {args.ref_audio}")
 
-        import librosa
+        from vllm.multimodal.media.audio import load_audio
 
-        audio_signal, sr = librosa.load(args.ref_audio, sr=None)
+        audio_signal, sr = load_audio(args.ref_audio, sr=None)
         multi_modal_data["audio"] = (audio_signal.astype(np.float32), sr)
         mm_processor_kwargs["ref_text"] = args.ref_text or ""
         mm_processor_kwargs["sample_rate"] = sr

examples/offline_inference/qwen2_5_omni/README.md

@@ -60,11 +60,3 @@ If media file paths are not provided, the script will use default assets. Suppor
 - `mixed_modalities`: Audio + image + video
 - `use_audio_in_video`: Extract audio from video
 - `text`: Text-only query
-
-### FAQ
-
-If you encounter error about backend of librosa, try to install ffmpeg with command below.
-```
-sudo apt update
-sudo apt install ffmpeg
-```

examples/offline_inference/qwen2_5_omni/end2end.py

@@ -9,7 +9,6 @@ import os
 import time
 from typing import NamedTuple
 
-import librosa
 import numpy as np
 import soundfile as sf
 from PIL import Image
@@ -17,6 +16,7 @@ from vllm.assets.audio import AudioAsset
 from vllm.assets.image import ImageAsset
 from vllm.assets.video import VideoAsset, video_to_ndarrays
 from vllm.multimodal.image import convert_image_mode
+from vllm.multimodal.media.audio import load_audio
 from vllm.sampling_params import SamplingParams
 from vllm.utils.argparse_utils import FlexibleArgumentParser
 
@@ -96,7 +96,7 @@ def get_mixed_modalities_query(
     if audio_path:
         if not os.path.exists(audio_path):
             raise FileNotFoundError(f"Audio file not found: {audio_path}")
-        audio_signal, sr = librosa.load(audio_path, sr=sampling_rate)
+        audio_signal, sr = load_audio(audio_path, sr=sampling_rate)
         audio_data = (audio_signal.astype(np.float32), sr)
     else:
         audio_data = AudioAsset("mary_had_lamb").audio_and_sample_rate
@@ -130,7 +130,7 @@ def get_use_audio_in_video_query(
             raise FileNotFoundError(f"Video file not found: {video_path}")
         video_frames = video_to_ndarrays(video_path, num_frames=num_frames)
         # Extract audio from video file
-        audio_signal, sr = librosa.load(video_path, sr=sampling_rate)
+        audio_signal, sr = load_audio(video_path, sr=sampling_rate)
         audio = (audio_signal.astype(np.float32), sr)
     else:
         asset = VideoAsset(name="baby_reading", num_frames=num_frames)
@@ -165,7 +165,7 @@ def get_multi_audios_query(audio_path: str | None = None, sampling_rate: int = 1
     if audio_path:
         if not os.path.exists(audio_path):
             raise FileNotFoundError(f"Audio file not found: {audio_path}")
-        audio_signal, sr = librosa.load(audio_path, sr=sampling_rate)
+        audio_signal, sr = load_audio(audio_path, sr=sampling_rate)
         audio_data = (audio_signal.astype(np.float32), sr)
         # Use the provided audio as the first audio, default as second
         audio_list = [
@@ -261,7 +261,7 @@ def get_audio_query(question: str = None, audio_path: str | None = None, samplin
     if audio_path:
         if not os.path.exists(audio_path):
             raise FileNotFoundError(f"Audio file not found: {audio_path}")
-        audio_signal, sr = librosa.load(audio_path, sr=sampling_rate)
+        audio_signal, sr = load_audio(audio_path, sr=sampling_rate)
         audio_data = (audio_signal.astype(np.float32), sr)
     else:
         audio_data = AudioAsset("mary_had_lamb").audio_and_sample_rate

examples/offline_inference/qwen3_omni/README.md

@@ -108,11 +108,3 @@ python end2end_async_chunk.py \
 > recommended entry point for non-async-chunk workflows. Only use the
 > async_chunk example when you need the stage-level concurrency semantics
 > described in PR #962 / #1151.
-
-### FAQ
-
-If you encounter error about backend of librosa, try to install ffmpeg with command below.
-```
-sudo apt update
-sudo apt install ffmpeg
-```

examples/offline_inference/qwen3_omni/end2end.py

@@ -9,7 +9,6 @@ import os
 import time
 from typing import NamedTuple
 
-import librosa
 import numpy as np
 import soundfile as sf
 import vllm
@@ -19,6 +18,7 @@ from vllm.assets.audio import AudioAsset
 from vllm.assets.image import ImageAsset
 from vllm.assets.video import VideoAsset, video_to_ndarrays
 from vllm.multimodal.image import convert_image_mode
+from vllm.multimodal.media.audio import load_audio
 from vllm.utils.argparse_utils import FlexibleArgumentParser
 
 from vllm_omni.entrypoints.omni import Omni
@@ -129,7 +129,7 @@ def get_audio_query(question: str = None, audio_path: str | None = None, samplin
     if audio_path:
         if not os.path.exists(audio_path):
             raise FileNotFoundError(f"Audio file not found: {audio_path}")
-        audio_signal, sr = librosa.load(audio_path, sr=sampling_rate)
+        audio_signal, sr = load_audio(audio_path, sr=sampling_rate)
         audio_data = (audio_signal.astype(np.float32), sr)
     else:
         audio_data = AudioAsset("mary_had_lamb").audio_and_sample_rate
@@ -183,7 +183,7 @@ def get_mixed_modalities_query(
     if audio_path:
         if not os.path.exists(audio_path):
             raise FileNotFoundError(f"Audio file not found: {audio_path}")
-        audio_signal, sr = librosa.load(audio_path, sr=sampling_rate)
+        audio_signal, sr = load_audio(audio_path, sr=sampling_rate)
         audio_data = (audio_signal.astype(np.float32), sr)
     else:
         audio_data = AudioAsset("mary_had_lamb").audio_and_sample_rate

examples/offline_inference/qwen3_omni/end2end_async_chunk.py

@@ -32,13 +32,13 @@ import torch
 
 os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
 
-import librosa
 from PIL import Image
 from vllm import SamplingParams
 from vllm.assets.audio import AudioAsset
 from vllm.assets.image import ImageAsset
 from vllm.assets.video import VideoAsset, video_to_ndarrays
 from vllm.multimodal.image import convert_image_mode
+from vllm.multimodal.media.audio import load_audio
 from vllm.utils.argparse_utils import FlexibleArgumentParser
 
 from vllm_omni.entrypoints.async_omni import AsyncOmni
@@ -89,7 +89,7 @@ def get_audio_query(
     if audio_path:
         if not os.path.exists(audio_path):
             raise FileNotFoundError(f"Audio file not found: {audio_path}")
-        audio_signal, sr = librosa.load(audio_path, sr=sampling_rate)
+        audio_signal, sr = load_audio(audio_path, sr=sampling_rate)
         audio_data = (audio_signal.astype(np.float32), sr)
     else:
         audio_data = AudioAsset("mary_had_lamb").audio_and_sample_rate

examples/offline_inference/x_to_video_audio/x_to_video_audio.py

@@ -5,8 +5,8 @@ import argparse
 import re
 import time
 
-import librosa
 from PIL import Image
+from vllm.multimodal.media.audio import load_audio
 
 from vllm_omni.diffusion.data import DiffusionParallelConfig
 from vllm_omni.entrypoints.omni import Omni
@@ -69,7 +69,7 @@ def load_image_and_audio(image_paths, audio_paths):
             image.append(img)
 
     for path in audio_paths:
-        audio_array, sr = librosa.load(path, sr=16000)
+        audio_array, sr = load_audio(path, sr=16000)
         audio_array = audio_array[int(sr * 1) : int(sr * 3)]
         audio.append(audio_array)
     return image, audio

examples/online_serving/bagel/README.md

@@ -354,13 +354,6 @@ curl http://localhost:8091/v1/chat/completions \
 
 ## FAQ
 
-- If you encounter an error about the backend of librosa, try to install ffmpeg with the command below.
-
-```bash
-sudo apt update
-sudo apt install ffmpeg
-```
-
 - If you don’t know how much VRAM is needed for the model or encounter the OOM error, you can try to decrease the max_model_len.
 
 | Stage               | VRAM                         |

examples/online_serving/qwen2_5_omni/README.md

@@ -208,11 +208,3 @@ The gradio script supports the following arguments:
 - `--ip`: Host/IP for Gradio server (default: 127.0.0.1)
 - `--port`: Port for Gradio server (default: 7861)
 - `--share`: Share the Gradio demo publicly (creates a public link)
-
-### FAQ
-
-If you encounter error about backend of librosa, try to install ffmpeg with command below.
-```
-sudo apt update
-sudo apt install ffmpeg
-```

examples/online_serving/qwen3_omni/README.md

@@ -43,11 +43,9 @@ python examples/online_serving/openai_chat_completion_client_for_multimodal_gene
 **Dependencies:**
 
 ```bash
-pip install websockets librosa numpy
+pip install websockets numpy
 ```
 
-(ffmpeg may be required by `librosa` for some formats; see the FAQ below.)
-
 **From this directory** (`examples/online_serving/qwen3_omni`):
 
 ```bash
@@ -105,12 +103,6 @@ bash run_curl_multimodal_generation.sh use_image
 
 ### FAQ
 
-If you encounter error about backend of librosa, try to install ffmpeg with command below.
-```
-sudo apt update
-sudo apt install ffmpeg
-```
-
 ## Modality control
 You can control output modalities to specify which types of output the model should generate. This is useful when you only need text output and want to skip audio generation stages for better performance.
 

examples/online_serving/qwen3_omni/openai_realtime_client.py

@@ -10,7 +10,7 @@ model, for example:
 Requirements:
 - vllm with audio support
 - websockets
-- librosa
+- soundfile
 - numpy
 
 The script:
@@ -25,10 +25,10 @@ import asyncio
 import base64
 import json
 
-import librosa
 import numpy as np
 import websockets
 from vllm.assets.audio import AudioAsset
+from vllm.multimodal.media.audio import load_audio
 
 
 def audio_to_pcm16_base64(audio_path: str) -> str:
@@ -36,7 +36,7 @@ def audio_to_pcm16_base64(audio_path: str) -> str:
     Load an audio file and convert it to base64-encoded PCM16 @ 16kHz.
     """
     # Load audio and resample to 16kHz mono
-    audio, _ = librosa.load(audio_path, sr=16000, mono=True)
+    audio, _ = load_audio(audio_path, sr=16000, mono=True)
     # Convert to PCM16
     pcm16 = (audio * 32767).astype(np.int16)
     # Encode as base64

examples/online_serving/qwen3_tts/README.md

@@ -192,14 +192,6 @@ with open("output.wav", "wb") as f:
     f.write(response.content)
 ```
 
-### FAQ
-
-If you encounter error about backend of librosa, try to install ffmpeg with command below.
-```
-sudo apt update
-sudo apt install ffmpeg
-```
-
 ## API Reference
 
 ### Voices Endpoint

examples/online_serving/qwen3_tts/speaker_embedding_interpolation.py

@@ -5,7 +5,7 @@ speaker encoder from a Qwen3-TTS checkpoint, then interpolates between them
 using SLERP and sends the result to the /v1/audio/speech API.
 
 Requirements:
-    pip install torch librosa soundfile numpy httpx
+    pip install torch resampy soundfile numpy httpx
 
 Examples:
     # Extract and save an embedding
@@ -143,17 +143,17 @@ def _load_speaker_encoder_weights(encoder: torch.nn.Module, model_path: str) ->
 
 def compute_mel_spectrogram(audio: np.ndarray, sr: int = 24000) -> torch.Tensor:
     """Compute 128-bin mel spectrogram matching Qwen3-TTS's extraction pipeline."""
-    import librosa
+    from vllm.multimodal.audio import resample_audio_resampy
 
     # Resample to 24kHz if needed
     if sr != 24000:
-        audio = librosa.resample(audio.astype(np.float32), orig_sr=sr, target_sr=24000)
+        audio = resample_audio_resampy(audio.astype(np.float32), orig_sr=sr, target_sr=24000)
 
     y = torch.from_numpy(audio).unsqueeze(0).float()
 
-    from librosa.filters import mel as librosa_mel_fn
+    from vllm_omni.utils.audio import mel_filter_bank
 
-    mel_basis = torch.from_numpy(librosa_mel_fn(sr=24000, n_fft=1024, n_mels=128, fmin=0, fmax=12000)).float()
+    mel_basis = mel_filter_bank(sr=24000, n_fft=1024, n_mels=128, fmin=0, fmax=12000)
 
     n_fft = 1024
     hop_size = 256
@@ -180,9 +180,9 @@ def compute_mel_spectrogram(audio: np.ndarray, sr: int = 24000) -> torch.Tensor:
 @torch.inference_mode()
 def extract_embedding(encoder: torch.nn.Module, audio_path: str, device: str = "cpu") -> np.ndarray:
     """Extract a 1024-dim speaker embedding from an audio file."""
-    import librosa
+    from vllm.multimodal.media.audio import load_audio
 
-    audio, sr = librosa.load(audio_path, sr=None, mono=True)
+    audio, sr = load_audio(audio_path, sr=None, mono=True)
     mel = compute_mel_spectrogram(audio, sr).to(device)
     embedding = encoder(mel.to(next(encoder.parameters()).dtype))[0]
     return embedding.float().cpu().numpy()