anodyine/vllm-omni
1
0 Fork 0
Code Issues 4 Pull Requests 2 Actions Packages Projects Releases Wiki Activity

[Bugfix] Use upstream MediaConnector for ref_audio resolution (#1661)

Browse Source 
Signed-off-by: linyueqian <linyueqian@outlook.com>
This commit is contained in:
Yueqian Lin authored and GitHub committed 2026-03-09 01:38:26 +00:00
1 parent 0e432e524c
commit a9f694b08a
5 files changed
+41 -68
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

docs/serving/speech_api.md
+1 -1
View File
@@ -104,7 +104,7 @@ Content-Type: application/json
| Parameter | Type | Default | Description |
|-----------|------|---------|-------------|
| `ref_audio` | string | null | Reference audio (URL or base64 data URL) |
| `ref_audio` | string | null | Reference audio (HTTP URL, base64 data URL, or `file://` URI with `--allowed-local-media-path`) |
| `ref_text` | string | null | Transcript of reference audio |
| `x_vector_only_mode` | bool | null | Use speaker embedding only (no ICL) |
docs/user_guide/examples/offline_inference/qwen3_tts.md
+1 -1
View File
@@ -11,7 +11,7 @@ Qwen3 TTS provides multiple task variants for speech generation:
- **CustomVoice**: Generate speech with a known speaker identity (speaker ID) and optional instruction.
- **VoiceDesign**: Generate speech from text plus a descriptive instruction that designs a new voice.
- **Base**: Voice cloning using a reference audio + reference transcript, with optional mode selection.
- **Base**: Voice cloning using a reference audio + reference transcript, with optional mode selection. The `ref_audio` field accepts a local file path, HTTP URL, or base64 data URL.
## Setup
Please refer to the [stage configuration documentation](https://docs.vllm.ai/projects/vllm-omni/en/latest/configuration/stage_configs/) to configure memory allocation appropriately for your hardware setup.
docs/user_guide/examples/online_serving/qwen3_tts.md
+3 -3
View File
@@ -112,7 +112,7 @@ The Python client supports the following command-line arguments:
- `--voice`: Speaker/voice name (default: `vivian`). Options: `vivian`, `ryan`, `aiden`, etc.
- `--language`: Language. Options: `Auto`, `Chinese`, `English`, `Japanese`, `Korean`, `German`, `French`, `Russian`, `Portuguese`, `Spanish`, `Italian`
- `--instructions`: Voice style/emotion instructions
- `--ref-audio`: Reference audio file path or URL for voice cloning (Base task)
- `--ref-audio`: Reference audio file path or URL for voice cloning (Base task). Local paths are automatically base64-encoded by the client before sending to the server.
- `--ref-text`: Reference audio transcript for voice cloning (Base task)
- `--response-format`: Audio output format (default: `wav`). Options: `wav`, `mp3`, `flac`, `pcm`, `aac`, `opus`
- `--output` (or `-o`): Output audio file path (default: `tts_output.wav`)
@@ -220,7 +220,7 @@ Lists available voices for the loaded model:
"task_type": "CustomVoice",
"language": "Auto",
"instructions": "Optional style instructions",
"ref_audio": "URL or base64 for voice cloning",
"ref_audio": "HTTP URL, base64 data URL, or file:// URI for voice cloning",
"ref_text": "Reference audio transcript",
"x_vector_only_mode": false,
"max_new_tokens": 2048
@@ -261,7 +261,7 @@ Returns binary audio data with appropriate `Content-Type` header (e.g., `audio/w
| Parameter | Type | Required | Description |
| -------------------- | ------ | -------- | -------------------------------------------- |
| `ref_audio` | string | **Yes** | Reference audio (URL or base64 data URL) |
| `ref_audio` | string | **Yes** | Reference audio (HTTP URL, base64 data URL, or `file://` URI with `--allowed-local-media-path`) |
| `ref_text` | string | No | Transcript of reference audio (for ICL mode) |
| `x_vector_only_mode` | bool | No | Use speaker embedding only (no ICL) |
vllm_omni/entrypoints/openai/serving_speech.py
+23 -59
View File
@@ -1,21 +1,15 @@
import asyncio
import base64
import io
import ipaddress
import json
import math
import os
import socket
from typing import Any
from urllib.parse import urlparse
from urllib.request import urlopen
import numpy as np
import soundfile as sf
from fastapi import Request
from fastapi.responses import Response, StreamingResponse
from vllm.entrypoints.openai.engine.serving import OpenAIServing
from vllm.logger import init_logger
from vllm.multimodal.media import MediaConnector
from vllm.utils import random_uuid
from vllm_omni.entrypoints.openai.audio_utils_mixin import AudioMixin
@@ -27,19 +21,6 @@ from vllm_omni.outputs import OmniRequestOutput
logger = init_logger(__name__)
_REF_AUDIO_TIMEOUT_S = 15
_REF_AUDIO_MAX_BYTES = 50 * 1024 * 1024 # 50 MB
_REF_AUDIO_BLOCKED_NETWORKS = [
ipaddress.ip_network("127.0.0.0/8"),
ipaddress.ip_network("10.0.0.0/8"),
ipaddress.ip_network("172.16.0.0/12"),
ipaddress.ip_network("192.168.0.0/16"),
ipaddress.ip_network("169.254.0.0/16"),
ipaddress.ip_network("::1/128"),
ipaddress.ip_network("fc00::/7"),
ipaddress.ip_network("fe80::/10"),
]
# TTS Configuration (currently supports Qwen3-TTS)
_TTS_MODEL_STAGES: set[str] = {"qwen3_tts"}
_TTS_LANGUAGES: set[str] = {
@@ -268,8 +249,12 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
if request.ref_audio is None:
return "Base task requires 'ref_audio' for voice cloning"
# Validate ref_audio format
if not (request.ref_audio.startswith(("http://", "https://")) or request.ref_audio.startswith("data:")):
return "ref_audio must be a URL (http/https) or base64 data URL (data:...)"
if not (
request.ref_audio.startswith(("http://", "https://"))
or request.ref_audio.startswith("data:")
or request.ref_audio.startswith("file://")
):
return "ref_audio must be a URL (http/https), base64 data URL (data:...), or file URI (file://...)"
# In-context voice cloning (default) requires non-empty ref_text.
# x_vector_only_mode skips in-context and only uses speaker embedding.
if not request.x_vector_only_mode:
@@ -303,44 +288,23 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
return None
@staticmethod
async def _resolve_ref_audio(ref_audio_str: str) -> tuple[list[float], int]:
"""Resolve ref_audio URL/base64 to (wav_samples, sample_rate)."""
parsed = urlparse(ref_audio_str)
async def _resolve_ref_audio(self, ref_audio_str: str) -> tuple[list[float], int]:
"""Resolve ref_audio to (wav_samples, sample_rate).
def _check_ssrf(url: str) -> None:
host = urlparse(url).hostname
if not host:
raise ValueError("ref_audio URL must include a hostname")
for info in socket.getaddrinfo(host, None):
ip_str = str(info[4][0]).split("%", 1)[0]
addr = ipaddress.ip_address(ip_str)
if any(addr in net for net in _REF_AUDIO_BLOCKED_NETWORKS):
raise ValueError(f"ref_audio URL resolves to blocked address: {addr}")
def _fetch_sync() -> tuple[np.ndarray, int]:
if parsed.scheme in ("http", "https"):
_check_ssrf(ref_audio_str)
with urlopen(ref_audio_str, timeout=_REF_AUDIO_TIMEOUT_S) as resp:
data = resp.read(_REF_AUDIO_MAX_BYTES + 1)
if len(data) > _REF_AUDIO_MAX_BYTES:
raise ValueError(f"ref_audio URL exceeds {_REF_AUDIO_MAX_BYTES} bytes")
buf = io.BytesIO(data)
elif ref_audio_str.startswith("data:"):
b64 = ref_audio_str
if "," in b64:
b64 = b64.split(",", 1)[1]
buf = io.BytesIO(base64.b64decode(b64))
else:
raise ValueError("ref_audio must be an http(s) URL or data: base64 URI")
audio, sr = sf.read(buf, dtype="float32", always_2d=False)
if isinstance(audio, np.ndarray) and audio.ndim > 1:
audio = np.mean(audio, axis=-1)
return np.asarray(audio, dtype=np.float32), int(sr)
loop = asyncio.get_running_loop()
wav_np, sr = await loop.run_in_executor(None, _fetch_sync)
return wav_np.tolist(), sr
Delegates to upstream vLLM's MediaConnector which handles http(s)
URLs, ``data:`` base64 URIs, and ``file:`` local paths (the latter
gated by ``--allowed-local-media-path``).
"""
model_config = self.model_config
connector = MediaConnector(
allowed_local_media_path=model_config.allowed_local_media_path,
allowed_media_domains=model_config.allowed_media_domains,
)
wav_np, sr = await connector.fetch_audio_async(ref_audio_str)
wav_np = np.asarray(wav_np, dtype=np.float32)
if wav_np.ndim > 1:
wav_np = np.mean(wav_np, axis=-1)
return wav_np.tolist(), int(sr)
async def _generate_pcm_chunks(self, generator, request_id: str):
"""Generate PCM audio chunks for streaming response.
vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_talker.py
+13 -4
View File
@@ -831,7 +831,9 @@ class Qwen3TTSTalkerForConditionalGeneration(nn.Module):
def _is_url(self, s: str) -> bool:
try:
u = urlparse(s)
return u.scheme in ("http", "https") and bool(u.netloc)
if u.scheme in ("http", "https"):
return bool(u.netloc)
return u.scheme == "file"
except Exception:
return False
@@ -841,12 +843,19 @@ class Qwen3TTSTalkerForConditionalGeneration(nn.Module):
return base64.b64decode(b64)
def _load_audio_to_np(self, x: str) -> tuple[np.ndarray, int]:
"""Load audio from local path or base64 data URI (no network I/O)."""
"""Load audio from local path, URL, or base64 data URI.
Uses upstream vLLM's MediaConnector for http(s) URLs and ``file:``
URIs, with unrestricted local access (offline inference is trusted).
"""
import librosa
if self._is_url(x):
raise ValueError("ref_audio URLs must be resolved by the serving layer before reaching the model worker.")
if self._is_probably_base64(x):
from vllm.multimodal.media import MediaConnector
connector = MediaConnector(allowed_local_media_path="/")
audio, sr = connector.fetch_audio(x)
elif self._is_probably_base64(x):
wav_bytes = self._decode_base64_to_wav_bytes(x)
with io.BytesIO(wav_bytes) as f:
audio, sr = sf.read(f, dtype="float32", always_2d=False)