[Refactor] Remove dependency on librosa (#2273)
Signed-off-by: Nick Cao <ncao@redhat.com> Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
1 parent
687405c5f2
commit
2bc183f6f0
46 files changed
+229
-323
No files matched your search
@@ -354,13 +354,6 @@ curl http://localhost:8091/v1/chat/completions \
|
||||
|
||||
## FAQ
|
||||
|
||||
- If you encounter an error about the backend of librosa, try to install ffmpeg with the command below.
|
||||
|
||||
```bash
|
||||
sudo apt update
|
||||
sudo apt install ffmpeg
|
||||
```
|
||||
|
||||
- If you don’t know how much VRAM is needed for the model or encounter the OOM error, you can try to decrease the max_model_len.
|
||||
|
||||
| Stage | VRAM |
|
||||
|
||||
@@ -208,11 +208,3 @@ The gradio script supports the following arguments:
|
||||
- `--ip`: Host/IP for Gradio server (default: 127.0.0.1)
|
||||
- `--port`: Port for Gradio server (default: 7861)
|
||||
- `--share`: Share the Gradio demo publicly (creates a public link)
|
||||
|
||||
### FAQ
|
||||
|
||||
If you encounter error about backend of librosa, try to install ffmpeg with command below.
|
||||
```
|
||||
sudo apt update
|
||||
sudo apt install ffmpeg
|
||||
```
|
||||
@@ -43,11 +43,9 @@ python examples/online_serving/openai_chat_completion_client_for_multimodal_gene
|
||||
**Dependencies:**
|
||||
|
||||
```bash
|
||||
pip install websockets librosa numpy
|
||||
pip install websockets numpy
|
||||
```
|
||||
|
||||
(ffmpeg may be required by `librosa` for some formats; see the FAQ below.)
|
||||
|
||||
**From this directory** (`examples/online_serving/qwen3_omni`):
|
||||
|
||||
```bash
|
||||
@@ -105,12 +103,6 @@ bash run_curl_multimodal_generation.sh use_image
|
||||
|
||||
### FAQ
|
||||
|
||||
If you encounter error about backend of librosa, try to install ffmpeg with command below.
|
||||
```
|
||||
sudo apt update
|
||||
sudo apt install ffmpeg
|
||||
```
|
||||
|
||||
## Modality control
|
||||
You can control output modalities to specify which types of output the model should generate. This is useful when you only need text output and want to skip audio generation stages for better performance.
|
||||
|
||||
|
||||
@@ -10,7 +10,7 @@ model, for example:
|
||||
Requirements:
|
||||
- vllm with audio support
|
||||
- websockets
|
||||
- librosa
|
||||
- soundfile
|
||||
- numpy
|
||||
|
||||
The script:
|
||||
@@ -25,10 +25,10 @@ import asyncio
|
||||
import base64
|
||||
import json
|
||||
|
||||
import librosa
|
||||
import numpy as np
|
||||
import websockets
|
||||
from vllm.assets.audio import AudioAsset
|
||||
from vllm.multimodal.media.audio import load_audio
|
||||
|
||||
|
||||
def audio_to_pcm16_base64(audio_path: str) -> str:
|
||||
@@ -36,7 +36,7 @@ def audio_to_pcm16_base64(audio_path: str) -> str:
|
||||
Load an audio file and convert it to base64-encoded PCM16 @ 16kHz.
|
||||
"""
|
||||
# Load audio and resample to 16kHz mono
|
||||
audio, _ = librosa.load(audio_path, sr=16000, mono=True)
|
||||
audio, _ = load_audio(audio_path, sr=16000, mono=True)
|
||||
# Convert to PCM16
|
||||
pcm16 = (audio * 32767).astype(np.int16)
|
||||
# Encode as base64
|
||||
|
||||
@@ -192,14 +192,6 @@ with open("output.wav", "wb") as f:
|
||||
f.write(response.content)
|
||||
```
|
||||
|
||||
### FAQ
|
||||
|
||||
If you encounter error about backend of librosa, try to install ffmpeg with command below.
|
||||
```
|
||||
sudo apt update
|
||||
sudo apt install ffmpeg
|
||||
```
|
||||
|
||||
## API Reference
|
||||
|
||||
### Voices Endpoint
|
||||
|
||||
@@ -5,7 +5,7 @@ speaker encoder from a Qwen3-TTS checkpoint, then interpolates between them
|
||||
using SLERP and sends the result to the /v1/audio/speech API.
|
||||
|
||||
Requirements:
|
||||
pip install torch librosa soundfile numpy httpx
|
||||
pip install torch resampy soundfile numpy httpx
|
||||
|
||||
Examples:
|
||||
# Extract and save an embedding
|
||||
@@ -143,17 +143,17 @@ def _load_speaker_encoder_weights(encoder: torch.nn.Module, model_path: str) ->
|
||||
|
||||
def compute_mel_spectrogram(audio: np.ndarray, sr: int = 24000) -> torch.Tensor:
|
||||
"""Compute 128-bin mel spectrogram matching Qwen3-TTS's extraction pipeline."""
|
||||
import librosa
|
||||
from vllm.multimodal.audio import resample_audio_resampy
|
||||
|
||||
# Resample to 24kHz if needed
|
||||
if sr != 24000:
|
||||
audio = librosa.resample(audio.astype(np.float32), orig_sr=sr, target_sr=24000)
|
||||
audio = resample_audio_resampy(audio.astype(np.float32), orig_sr=sr, target_sr=24000)
|
||||
|
||||
y = torch.from_numpy(audio).unsqueeze(0).float()
|
||||
|
||||
from librosa.filters import mel as librosa_mel_fn
|
||||
from vllm_omni.utils.audio import mel_filter_bank
|
||||
|
||||
mel_basis = torch.from_numpy(librosa_mel_fn(sr=24000, n_fft=1024, n_mels=128, fmin=0, fmax=12000)).float()
|
||||
mel_basis = mel_filter_bank(sr=24000, n_fft=1024, n_mels=128, fmin=0, fmax=12000)
|
||||
|
||||
n_fft = 1024
|
||||
hop_size = 256
|
||||
@@ -180,9 +180,9 @@ def compute_mel_spectrogram(audio: np.ndarray, sr: int = 24000) -> torch.Tensor:
|
||||
@torch.inference_mode()
|
||||
def extract_embedding(encoder: torch.nn.Module, audio_path: str, device: str = "cpu") -> np.ndarray:
|
||||
"""Extract a 1024-dim speaker embedding from an audio file."""
|
||||
import librosa
|
||||
from vllm.multimodal.media.audio import load_audio
|
||||
|
||||
audio, sr = librosa.load(audio_path, sr=None, mono=True)
|
||||
audio, sr = load_audio(audio_path, sr=None, mono=True)
|
||||
mel = compute_mel_spectrogram(audio, sr).to(device)
|
||||
embedding = encoder(mel.to(next(encoder.parameters()).dtype))[0]
|
||||
return embedding.float().cpu().numpy()
|
||||
|
||||
Reference in new issue
Block a user