[Refactor] Remove dependency on librosa (#2273)
Signed-off-by: Nick Cao <ncao@redhat.com> Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
1 parent
687405c5f2
commit
2bc183f6f0
46 files changed
+229
-323
No files matched your search
@@ -247,13 +247,6 @@ For more details on the Mooncake connector and multi-node setup, see the [Moonca
|
||||
|
||||
## FAQ
|
||||
|
||||
- If you encounter an error about the backend of librosa, try to install ffmpeg with the command below.
|
||||
|
||||
```bash
|
||||
sudo apt update
|
||||
sudo apt install ffmpeg
|
||||
```
|
||||
|
||||
- If you don’t know how much VRAM is needed for the model or encounter the OOM error, you can try to decrease the max_model_len.
|
||||
|
||||
| Stage | VRAM |
|
||||
|
||||
@@ -7,7 +7,7 @@ Install dependencies:
|
||||
uv pip install -e .
|
||||
```
|
||||
|
||||
> **Note:** This includes required libraries such as `librosa`, `soundfile`,
|
||||
> **Note:** This includes required libraries such as `soundfile`,
|
||||
> `onnxruntime`, `x-transformers`, and `einops` via
|
||||
> `requirements/common.txt` and platform-specific requirements files.
|
||||
|
||||
|
||||
@@ -2,13 +2,12 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import argparse
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
import librosa
|
||||
import numpy as np
|
||||
import soundfile as sf
|
||||
from vllm import SamplingParams
|
||||
from vllm.assets.audio import AudioAsset
|
||||
from vllm.multimodal.media.audio import load_audio
|
||||
|
||||
from vllm_omni.entrypoints.omni import Omni
|
||||
from vllm_omni.model_executor.models.cosyvoice3.config import CosyVoice3Config
|
||||
@@ -16,22 +15,6 @@ from vllm_omni.model_executor.models.cosyvoice3.tokenizer import get_qwen_tokeni
|
||||
from vllm_omni.model_executor.models.cosyvoice3.utils import extract_text_token
|
||||
|
||||
|
||||
def _ensure_mel_filters_asset() -> None:
|
||||
repo_root = Path(__file__).resolve().parents[3]
|
||||
filters_path = repo_root / "vllm_omni" / "model_executor" / "models" / "cosyvoice3" / "assets" / "mel_filters.npz"
|
||||
if filters_path.exists():
|
||||
return
|
||||
|
||||
source_url = "https://raw.githubusercontent.com/openai/whisper/main/whisper/assets/mel_filters.npz"
|
||||
raise FileNotFoundError(
|
||||
"Missing CosyVoice3 mel filter asset:\n"
|
||||
f" {filters_path}\n"
|
||||
"Download it with:\n"
|
||||
f" mkdir -p {filters_path.parent} && "
|
||||
f"curl -L {source_url} -o {filters_path}"
|
||||
)
|
||||
|
||||
|
||||
def run_e2e():
|
||||
parser = argparse.ArgumentParser()
|
||||
# ""FunAudioLLM/Fun-CosyVoice3-0.5B-2512
|
||||
@@ -56,7 +39,6 @@ def run_e2e():
|
||||
help="Path to tokenizer directory (e.g., <model_path>/CosyVoice-BlankEN).",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
_ensure_mel_filters_asset()
|
||||
# Ensure tokenizer directory exists
|
||||
if not os.path.exists(args.tokenizer):
|
||||
raise FileNotFoundError(f"{args.tokenizer} does not exist!")
|
||||
@@ -85,7 +67,7 @@ def run_e2e():
|
||||
if not os.path.exists(args.audio_path):
|
||||
raise FileNotFoundError(f"Audio file not found: {args.audio_path}")
|
||||
# Load at native sample rate
|
||||
audio_signal, sr = librosa.load(args.audio_path, sr=None)
|
||||
audio_signal, sr = load_audio(args.audio_path, sr=None)
|
||||
|
||||
# Validate sample rate before processing (similar to original CosyVoice)
|
||||
min_sr = 16000
|
||||
|
||||
@@ -190,29 +190,6 @@ Note: This task uses hardcoded message lists in the script.
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Audio dependencies (soundfile, librosa)
|
||||
|
||||
This example depends on **soundfile** (read/write WAV) and **librosa** (load audio including MP3). Install the project requirements first:
|
||||
|
||||
```bash
|
||||
pip install -r requirements/common.txt
|
||||
# or at least: pip install soundfile>=0.13.1 librosa>=0.11.0
|
||||
```
|
||||
|
||||
- **`soundfile` / libsndfile not found**
|
||||
`soundfile` uses the C library **libsndfile**. On Linux, install the system package before pip:
|
||||
- Debian/Ubuntu: `sudo apt-get install libsndfile1`
|
||||
- For development builds: `sudo apt-get install libsndfile1-dev`
|
||||
- Then: `pip install soundfile`
|
||||
|
||||
- **`librosa` fails to load MP3 or reports "No backend available"**
|
||||
Loading MP3 (e.g. in `spoken_dialogue_sft_multiturn` with `.mp3` files) uses **ffmpeg** as the backend. Install ffmpeg:
|
||||
- Debian/Ubuntu: `sudo apt-get install ffmpeg`
|
||||
- macOS: `brew install ffmpeg`
|
||||
|
||||
- **`ImportError: No module named 'soundfile'` or `ModuleNotFoundError: ... librosa`**
|
||||
Ensure you are in the same Python environment where vLLM Omni and the example dependencies are installed, and that `requirements/common.txt` (or the packages above) are installed.
|
||||
|
||||
### Tokenizer path
|
||||
|
||||
- **`MIMO_AUDIO_TOKENIZER_PATH` not set or model fails to find tokenizer**
|
||||
|
||||
@@ -5,12 +5,12 @@ import random
|
||||
import re
|
||||
from collections.abc import Callable
|
||||
|
||||
import librosa
|
||||
import numpy as np
|
||||
import torch
|
||||
import torchaudio
|
||||
from process_speechdata import InputSegment, StreamingInputSegment
|
||||
from torchaudio.transforms import MelSpectrogram
|
||||
from vllm.multimodal.media.audio import load_audio
|
||||
|
||||
speech_zeroemb_idx = 151667
|
||||
empty_token = "<|empty|>"
|
||||
@@ -685,7 +685,7 @@ def get_audio_data(audio_url):
|
||||
# File path
|
||||
audio_file = audio_url
|
||||
|
||||
audio_signal, sr = librosa.load(audio_file, sr=24000)
|
||||
audio_signal, sr = load_audio(audio_file, sr=24000)
|
||||
audio_data = (audio_signal.astype(np.float32), sr)
|
||||
return audio_data
|
||||
|
||||
|
||||
@@ -103,9 +103,9 @@ def run_e2e():
|
||||
if not os.path.exists(args.ref_audio):
|
||||
raise FileNotFoundError(f"Reference audio not found: {args.ref_audio}")
|
||||
|
||||
import librosa
|
||||
from vllm.multimodal.media.audio import load_audio
|
||||
|
||||
audio_signal, sr = librosa.load(args.ref_audio, sr=None)
|
||||
audio_signal, sr = load_audio(args.ref_audio, sr=None)
|
||||
multi_modal_data["audio"] = (audio_signal.astype(np.float32), sr)
|
||||
mm_processor_kwargs["ref_text"] = args.ref_text or ""
|
||||
mm_processor_kwargs["sample_rate"] = sr
|
||||
|
||||
@@ -60,11 +60,3 @@ If media file paths are not provided, the script will use default assets. Suppor
|
||||
- `mixed_modalities`: Audio + image + video
|
||||
- `use_audio_in_video`: Extract audio from video
|
||||
- `text`: Text-only query
|
||||
|
||||
### FAQ
|
||||
|
||||
If you encounter error about backend of librosa, try to install ffmpeg with command below.
|
||||
```
|
||||
sudo apt update
|
||||
sudo apt install ffmpeg
|
||||
```
|
||||
@@ -9,7 +9,6 @@ import os
|
||||
import time
|
||||
from typing import NamedTuple
|
||||
|
||||
import librosa
|
||||
import numpy as np
|
||||
import soundfile as sf
|
||||
from PIL import Image
|
||||
@@ -17,6 +16,7 @@ from vllm.assets.audio import AudioAsset
|
||||
from vllm.assets.image import ImageAsset
|
||||
from vllm.assets.video import VideoAsset, video_to_ndarrays
|
||||
from vllm.multimodal.image import convert_image_mode
|
||||
from vllm.multimodal.media.audio import load_audio
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
|
||||
@@ -96,7 +96,7 @@ def get_mixed_modalities_query(
|
||||
if audio_path:
|
||||
if not os.path.exists(audio_path):
|
||||
raise FileNotFoundError(f"Audio file not found: {audio_path}")
|
||||
audio_signal, sr = librosa.load(audio_path, sr=sampling_rate)
|
||||
audio_signal, sr = load_audio(audio_path, sr=sampling_rate)
|
||||
audio_data = (audio_signal.astype(np.float32), sr)
|
||||
else:
|
||||
audio_data = AudioAsset("mary_had_lamb").audio_and_sample_rate
|
||||
@@ -130,7 +130,7 @@ def get_use_audio_in_video_query(
|
||||
raise FileNotFoundError(f"Video file not found: {video_path}")
|
||||
video_frames = video_to_ndarrays(video_path, num_frames=num_frames)
|
||||
# Extract audio from video file
|
||||
audio_signal, sr = librosa.load(video_path, sr=sampling_rate)
|
||||
audio_signal, sr = load_audio(video_path, sr=sampling_rate)
|
||||
audio = (audio_signal.astype(np.float32), sr)
|
||||
else:
|
||||
asset = VideoAsset(name="baby_reading", num_frames=num_frames)
|
||||
@@ -165,7 +165,7 @@ def get_multi_audios_query(audio_path: str | None = None, sampling_rate: int = 1
|
||||
if audio_path:
|
||||
if not os.path.exists(audio_path):
|
||||
raise FileNotFoundError(f"Audio file not found: {audio_path}")
|
||||
audio_signal, sr = librosa.load(audio_path, sr=sampling_rate)
|
||||
audio_signal, sr = load_audio(audio_path, sr=sampling_rate)
|
||||
audio_data = (audio_signal.astype(np.float32), sr)
|
||||
# Use the provided audio as the first audio, default as second
|
||||
audio_list = [
|
||||
@@ -261,7 +261,7 @@ def get_audio_query(question: str = None, audio_path: str | None = None, samplin
|
||||
if audio_path:
|
||||
if not os.path.exists(audio_path):
|
||||
raise FileNotFoundError(f"Audio file not found: {audio_path}")
|
||||
audio_signal, sr = librosa.load(audio_path, sr=sampling_rate)
|
||||
audio_signal, sr = load_audio(audio_path, sr=sampling_rate)
|
||||
audio_data = (audio_signal.astype(np.float32), sr)
|
||||
else:
|
||||
audio_data = AudioAsset("mary_had_lamb").audio_and_sample_rate
|
||||
|
||||
@@ -108,11 +108,3 @@ python end2end_async_chunk.py \
|
||||
> recommended entry point for non-async-chunk workflows. Only use the
|
||||
> async_chunk example when you need the stage-level concurrency semantics
|
||||
> described in PR #962 / #1151.
|
||||
|
||||
### FAQ
|
||||
|
||||
If you encounter error about backend of librosa, try to install ffmpeg with command below.
|
||||
```
|
||||
sudo apt update
|
||||
sudo apt install ffmpeg
|
||||
```
|
||||
@@ -9,7 +9,6 @@ import os
|
||||
import time
|
||||
from typing import NamedTuple
|
||||
|
||||
import librosa
|
||||
import numpy as np
|
||||
import soundfile as sf
|
||||
import vllm
|
||||
@@ -19,6 +18,7 @@ from vllm.assets.audio import AudioAsset
|
||||
from vllm.assets.image import ImageAsset
|
||||
from vllm.assets.video import VideoAsset, video_to_ndarrays
|
||||
from vllm.multimodal.image import convert_image_mode
|
||||
from vllm.multimodal.media.audio import load_audio
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
|
||||
from vllm_omni.entrypoints.omni import Omni
|
||||
@@ -129,7 +129,7 @@ def get_audio_query(question: str = None, audio_path: str | None = None, samplin
|
||||
if audio_path:
|
||||
if not os.path.exists(audio_path):
|
||||
raise FileNotFoundError(f"Audio file not found: {audio_path}")
|
||||
audio_signal, sr = librosa.load(audio_path, sr=sampling_rate)
|
||||
audio_signal, sr = load_audio(audio_path, sr=sampling_rate)
|
||||
audio_data = (audio_signal.astype(np.float32), sr)
|
||||
else:
|
||||
audio_data = AudioAsset("mary_had_lamb").audio_and_sample_rate
|
||||
@@ -183,7 +183,7 @@ def get_mixed_modalities_query(
|
||||
if audio_path:
|
||||
if not os.path.exists(audio_path):
|
||||
raise FileNotFoundError(f"Audio file not found: {audio_path}")
|
||||
audio_signal, sr = librosa.load(audio_path, sr=sampling_rate)
|
||||
audio_signal, sr = load_audio(audio_path, sr=sampling_rate)
|
||||
audio_data = (audio_signal.astype(np.float32), sr)
|
||||
else:
|
||||
audio_data = AudioAsset("mary_had_lamb").audio_and_sample_rate
|
||||
|
||||
@@ -32,13 +32,13 @@ import torch
|
||||
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
|
||||
import librosa
|
||||
from PIL import Image
|
||||
from vllm import SamplingParams
|
||||
from vllm.assets.audio import AudioAsset
|
||||
from vllm.assets.image import ImageAsset
|
||||
from vllm.assets.video import VideoAsset, video_to_ndarrays
|
||||
from vllm.multimodal.image import convert_image_mode
|
||||
from vllm.multimodal.media.audio import load_audio
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
|
||||
from vllm_omni.entrypoints.async_omni import AsyncOmni
|
||||
@@ -89,7 +89,7 @@ def get_audio_query(
|
||||
if audio_path:
|
||||
if not os.path.exists(audio_path):
|
||||
raise FileNotFoundError(f"Audio file not found: {audio_path}")
|
||||
audio_signal, sr = librosa.load(audio_path, sr=sampling_rate)
|
||||
audio_signal, sr = load_audio(audio_path, sr=sampling_rate)
|
||||
audio_data = (audio_signal.astype(np.float32), sr)
|
||||
else:
|
||||
audio_data = AudioAsset("mary_had_lamb").audio_and_sample_rate
|
||||
|
||||
@@ -5,8 +5,8 @@ import argparse
|
||||
import re
|
||||
import time
|
||||
|
||||
import librosa
|
||||
from PIL import Image
|
||||
from vllm.multimodal.media.audio import load_audio
|
||||
|
||||
from vllm_omni.diffusion.data import DiffusionParallelConfig
|
||||
from vllm_omni.entrypoints.omni import Omni
|
||||
@@ -69,7 +69,7 @@ def load_image_and_audio(image_paths, audio_paths):
|
||||
image.append(img)
|
||||
|
||||
for path in audio_paths:
|
||||
audio_array, sr = librosa.load(path, sr=16000)
|
||||
audio_array, sr = load_audio(path, sr=16000)
|
||||
audio_array = audio_array[int(sr * 1) : int(sr * 3)]
|
||||
audio.append(audio_array)
|
||||
return image, audio
|
||||
|
||||
Reference in new issue
Block a user