From a9f694b08ad72a945c450ebab1f74dd961c7fd22 Mon Sep 17 00:00:00 2001 From: Yueqian Lin <70319226+linyueqian@users.noreply.github.com> Date: Sun, 8 Mar 2026 21:38:26 -0400 Subject: [PATCH] [Bugfix] Use upstream MediaConnector for ref_audio resolution (#1661) Signed-off-by: linyueqian --- docs/serving/speech_api.md | 2 +- .../examples/offline_inference/qwen3_tts.md | 2 +- .../examples/online_serving/qwen3_tts.md | 6 +- .../entrypoints/openai/serving_speech.py | 82 ++++++------------- .../models/qwen3_tts/qwen3_tts_talker.py | 17 +++- 5 files changed, 41 insertions(+), 68 deletions(-) diff --git a/docs/serving/speech_api.md b/docs/serving/speech_api.md index b22aee84c..a305981f3 100644 --- a/docs/serving/speech_api.md +++ b/docs/serving/speech_api.md @@ -104,7 +104,7 @@ Content-Type: application/json | Parameter | Type | Default | Description | |-----------|------|---------|-------------| -| `ref_audio` | string | null | Reference audio (URL or base64 data URL) | +| `ref_audio` | string | null | Reference audio (HTTP URL, base64 data URL, or `file://` URI with `--allowed-local-media-path`) | | `ref_text` | string | null | Transcript of reference audio | | `x_vector_only_mode` | bool | null | Use speaker embedding only (no ICL) | diff --git a/docs/user_guide/examples/offline_inference/qwen3_tts.md b/docs/user_guide/examples/offline_inference/qwen3_tts.md index 5c3b044d7..afadd39f4 100644 --- a/docs/user_guide/examples/offline_inference/qwen3_tts.md +++ b/docs/user_guide/examples/offline_inference/qwen3_tts.md @@ -11,7 +11,7 @@ Qwen3 TTS provides multiple task variants for speech generation: - **CustomVoice**: Generate speech with a known speaker identity (speaker ID) and optional instruction. - **VoiceDesign**: Generate speech from text plus a descriptive instruction that designs a new voice. -- **Base**: Voice cloning using a reference audio + reference transcript, with optional mode selection. +- **Base**: Voice cloning using a reference audio + reference transcript, with optional mode selection. The `ref_audio` field accepts a local file path, HTTP URL, or base64 data URL. ## Setup Please refer to the [stage configuration documentation](https://docs.vllm.ai/projects/vllm-omni/en/latest/configuration/stage_configs/) to configure memory allocation appropriately for your hardware setup. diff --git a/docs/user_guide/examples/online_serving/qwen3_tts.md b/docs/user_guide/examples/online_serving/qwen3_tts.md index 77a3947b5..e3de00515 100644 --- a/docs/user_guide/examples/online_serving/qwen3_tts.md +++ b/docs/user_guide/examples/online_serving/qwen3_tts.md @@ -112,7 +112,7 @@ The Python client supports the following command-line arguments: - `--voice`: Speaker/voice name (default: `vivian`). Options: `vivian`, `ryan`, `aiden`, etc. - `--language`: Language. Options: `Auto`, `Chinese`, `English`, `Japanese`, `Korean`, `German`, `French`, `Russian`, `Portuguese`, `Spanish`, `Italian` - `--instructions`: Voice style/emotion instructions -- `--ref-audio`: Reference audio file path or URL for voice cloning (Base task) +- `--ref-audio`: Reference audio file path or URL for voice cloning (Base task). Local paths are automatically base64-encoded by the client before sending to the server. - `--ref-text`: Reference audio transcript for voice cloning (Base task) - `--response-format`: Audio output format (default: `wav`). Options: `wav`, `mp3`, `flac`, `pcm`, `aac`, `opus` - `--output` (or `-o`): Output audio file path (default: `tts_output.wav`) @@ -220,7 +220,7 @@ Lists available voices for the loaded model: "task_type": "CustomVoice", "language": "Auto", "instructions": "Optional style instructions", - "ref_audio": "URL or base64 for voice cloning", + "ref_audio": "HTTP URL, base64 data URL, or file:// URI for voice cloning", "ref_text": "Reference audio transcript", "x_vector_only_mode": false, "max_new_tokens": 2048 @@ -261,7 +261,7 @@ Returns binary audio data with appropriate `Content-Type` header (e.g., `audio/w | Parameter | Type | Required | Description | | -------------------- | ------ | -------- | -------------------------------------------- | -| `ref_audio` | string | **Yes** | Reference audio (URL or base64 data URL) | +| `ref_audio` | string | **Yes** | Reference audio (HTTP URL, base64 data URL, or `file://` URI with `--allowed-local-media-path`) | | `ref_text` | string | No | Transcript of reference audio (for ICL mode) | | `x_vector_only_mode` | bool | No | Use speaker embedding only (no ICL) | diff --git a/vllm_omni/entrypoints/openai/serving_speech.py b/vllm_omni/entrypoints/openai/serving_speech.py index 8c292cc7a..d53e4176c 100644 --- a/vllm_omni/entrypoints/openai/serving_speech.py +++ b/vllm_omni/entrypoints/openai/serving_speech.py @@ -1,21 +1,15 @@ import asyncio -import base64 -import io -import ipaddress import json import math import os -import socket from typing import Any -from urllib.parse import urlparse -from urllib.request import urlopen import numpy as np -import soundfile as sf from fastapi import Request from fastapi.responses import Response, StreamingResponse from vllm.entrypoints.openai.engine.serving import OpenAIServing from vllm.logger import init_logger +from vllm.multimodal.media import MediaConnector from vllm.utils import random_uuid from vllm_omni.entrypoints.openai.audio_utils_mixin import AudioMixin @@ -27,19 +21,6 @@ from vllm_omni.outputs import OmniRequestOutput logger = init_logger(__name__) -_REF_AUDIO_TIMEOUT_S = 15 -_REF_AUDIO_MAX_BYTES = 50 * 1024 * 1024 # 50 MB -_REF_AUDIO_BLOCKED_NETWORKS = [ - ipaddress.ip_network("127.0.0.0/8"), - ipaddress.ip_network("10.0.0.0/8"), - ipaddress.ip_network("172.16.0.0/12"), - ipaddress.ip_network("192.168.0.0/16"), - ipaddress.ip_network("169.254.0.0/16"), - ipaddress.ip_network("::1/128"), - ipaddress.ip_network("fc00::/7"), - ipaddress.ip_network("fe80::/10"), -] - # TTS Configuration (currently supports Qwen3-TTS) _TTS_MODEL_STAGES: set[str] = {"qwen3_tts"} _TTS_LANGUAGES: set[str] = { @@ -268,8 +249,12 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): if request.ref_audio is None: return "Base task requires 'ref_audio' for voice cloning" # Validate ref_audio format - if not (request.ref_audio.startswith(("http://", "https://")) or request.ref_audio.startswith("data:")): - return "ref_audio must be a URL (http/https) or base64 data URL (data:...)" + if not ( + request.ref_audio.startswith(("http://", "https://")) + or request.ref_audio.startswith("data:") + or request.ref_audio.startswith("file://") + ): + return "ref_audio must be a URL (http/https), base64 data URL (data:...), or file URI (file://...)" # In-context voice cloning (default) requires non-empty ref_text. # x_vector_only_mode skips in-context and only uses speaker embedding. if not request.x_vector_only_mode: @@ -303,44 +288,23 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): return None - @staticmethod - async def _resolve_ref_audio(ref_audio_str: str) -> tuple[list[float], int]: - """Resolve ref_audio URL/base64 to (wav_samples, sample_rate).""" - parsed = urlparse(ref_audio_str) + async def _resolve_ref_audio(self, ref_audio_str: str) -> tuple[list[float], int]: + """Resolve ref_audio to (wav_samples, sample_rate). - def _check_ssrf(url: str) -> None: - host = urlparse(url).hostname - if not host: - raise ValueError("ref_audio URL must include a hostname") - for info in socket.getaddrinfo(host, None): - ip_str = str(info[4][0]).split("%", 1)[0] - addr = ipaddress.ip_address(ip_str) - if any(addr in net for net in _REF_AUDIO_BLOCKED_NETWORKS): - raise ValueError(f"ref_audio URL resolves to blocked address: {addr}") - - def _fetch_sync() -> tuple[np.ndarray, int]: - if parsed.scheme in ("http", "https"): - _check_ssrf(ref_audio_str) - with urlopen(ref_audio_str, timeout=_REF_AUDIO_TIMEOUT_S) as resp: - data = resp.read(_REF_AUDIO_MAX_BYTES + 1) - if len(data) > _REF_AUDIO_MAX_BYTES: - raise ValueError(f"ref_audio URL exceeds {_REF_AUDIO_MAX_BYTES} bytes") - buf = io.BytesIO(data) - elif ref_audio_str.startswith("data:"): - b64 = ref_audio_str - if "," in b64: - b64 = b64.split(",", 1)[1] - buf = io.BytesIO(base64.b64decode(b64)) - else: - raise ValueError("ref_audio must be an http(s) URL or data: base64 URI") - audio, sr = sf.read(buf, dtype="float32", always_2d=False) - if isinstance(audio, np.ndarray) and audio.ndim > 1: - audio = np.mean(audio, axis=-1) - return np.asarray(audio, dtype=np.float32), int(sr) - - loop = asyncio.get_running_loop() - wav_np, sr = await loop.run_in_executor(None, _fetch_sync) - return wav_np.tolist(), sr + Delegates to upstream vLLM's MediaConnector which handles http(s) + URLs, ``data:`` base64 URIs, and ``file:`` local paths (the latter + gated by ``--allowed-local-media-path``). + """ + model_config = self.model_config + connector = MediaConnector( + allowed_local_media_path=model_config.allowed_local_media_path, + allowed_media_domains=model_config.allowed_media_domains, + ) + wav_np, sr = await connector.fetch_audio_async(ref_audio_str) + wav_np = np.asarray(wav_np, dtype=np.float32) + if wav_np.ndim > 1: + wav_np = np.mean(wav_np, axis=-1) + return wav_np.tolist(), int(sr) async def _generate_pcm_chunks(self, generator, request_id: str): """Generate PCM audio chunks for streaming response. diff --git a/vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_talker.py b/vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_talker.py index e80f1785f..a1f21a6a6 100644 --- a/vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_talker.py +++ b/vllm_omni/model_executor/models/qwen3_tts/qwen3_tts_talker.py @@ -831,7 +831,9 @@ class Qwen3TTSTalkerForConditionalGeneration(nn.Module): def _is_url(self, s: str) -> bool: try: u = urlparse(s) - return u.scheme in ("http", "https") and bool(u.netloc) + if u.scheme in ("http", "https"): + return bool(u.netloc) + return u.scheme == "file" except Exception: return False @@ -841,12 +843,19 @@ class Qwen3TTSTalkerForConditionalGeneration(nn.Module): return base64.b64decode(b64) def _load_audio_to_np(self, x: str) -> tuple[np.ndarray, int]: - """Load audio from local path or base64 data URI (no network I/O).""" + """Load audio from local path, URL, or base64 data URI. + + Uses upstream vLLM's MediaConnector for http(s) URLs and ``file:`` + URIs, with unrestricted local access (offline inference is trusted). + """ import librosa if self._is_url(x): - raise ValueError("ref_audio URLs must be resolved by the serving layer before reaching the model worker.") - if self._is_probably_base64(x): + from vllm.multimodal.media import MediaConnector + + connector = MediaConnector(allowed_local_media_path="/") + audio, sr = connector.fetch_audio(x) + elif self._is_probably_base64(x): wav_bytes = self._decode_base64_to_wav_bytes(x) with io.BytesIO(wav_bytes) as f: audio, sr = sf.read(f, dtype="float32", always_2d=False)