Add Fish Speech S2 Pro support with online serving and voice cloning (#1798)
Signed-off-by: linyueqian <linyueqian@outlook.com>
This commit is contained in:
1 parent
36c6cb9634
commit
366b3368ea
25 files changed
+3459
-7
No files matched your search
@@ -0,0 +1,75 @@
|
||||
# Fish Speech S2 Pro
|
||||
|
||||
This directory contains an offline demo for running Fish Speech S2 Pro (`fishaudio/s2-pro`) with vLLM Omni. It supports text-only synthesis and voice cloning with reference audio.
|
||||
|
||||
## Model Overview
|
||||
|
||||
[Fish Speech S2 Pro](https://huggingface.co/fishaudio/s2-pro) is a 4B dual-AR text-to-speech model by FishAudio. It uses:
|
||||
|
||||
- **Slow AR**: Generates semantic tokens autoregressively (Qwen3-based backbone)
|
||||
- **Fast AR**: Predicts residual codebook tokens from semantic tokens
|
||||
- **DAC Decoder**: Converts codec codes to 44.1 kHz audio waveform
|
||||
|
||||
The model produces high-quality speech with voice cloning capabilities.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
Install the `fish-speech` package for the DAC codec:
|
||||
|
||||
```bash
|
||||
pip install fish-speech
|
||||
```
|
||||
|
||||
## Setup
|
||||
|
||||
Please refer to the [stage configuration documentation](https://docs.vllm.ai/projects/vllm-omni/en/latest/configuration/stage_configs/) to configure memory allocation appropriately for your hardware setup.
|
||||
|
||||
## Quick Start
|
||||
|
||||
### Text-Only Synthesis
|
||||
|
||||
```bash
|
||||
python end2end.py --text "Hello, this is a test of the Fish Speech text to speech system."
|
||||
```
|
||||
|
||||
Generated audio files are saved to `output_audio/` by default.
|
||||
|
||||
### Voice Cloning
|
||||
|
||||
Provide a reference audio file and its transcript to clone a voice:
|
||||
|
||||
```bash
|
||||
python end2end.py \
|
||||
--text "Hello, this is a cloned voice." \
|
||||
--ref-audio /path/to/reference.wav \
|
||||
--ref-text "Transcript of the reference audio."
|
||||
```
|
||||
|
||||
## Streaming Mode
|
||||
|
||||
Add `--streaming` to stream audio chunks progressively via `AsyncOmni` (requires `async_chunk: true` in the stage config):
|
||||
|
||||
```bash
|
||||
python end2end.py --text "Hello, this is a test." --streaming
|
||||
```
|
||||
|
||||
Each DAC decoder chunk is logged as it arrives. The final WAV file is written once generation completes.
|
||||
|
||||
## Configuration
|
||||
|
||||
| Argument | Default | Description |
|
||||
|----------|---------|-------------|
|
||||
| `--text` | `"Hello, this is a test..."` | Text to synthesize |
|
||||
| `--ref-audio` | None | Path to reference audio for voice cloning |
|
||||
| `--ref-text` | None | Transcript of the reference audio |
|
||||
| `--model` | `fishaudio/s2-pro` | HuggingFace model path |
|
||||
| `--stage-configs-path` | `fish_speech_s2_pro.yaml` | Path to stage configs YAML |
|
||||
| `--output-dir` | `output_audio` | Output directory for WAV files |
|
||||
| `--streaming` | False | Enable streaming via AsyncOmni |
|
||||
| `--stage-init-timeout` | 600 | Stage initialization timeout (seconds) |
|
||||
|
||||
## Notes
|
||||
|
||||
- Output audio is 44.1 kHz mono WAV.
|
||||
- The DAC codec weights (`codec.pth`) are loaded lazily from the model directory.
|
||||
- Voice cloning encodes the reference audio using the DAC codec to extract semantic codes, then prepends them as a system message.
|
||||
@@ -0,0 +1,252 @@
|
||||
"""Offline inference demo for Fish Speech S2 Pro via vLLM Omni.
|
||||
|
||||
Generates speech from text using the fishaudio/s2-pro model.
|
||||
Supports both text-only synthesis and voice cloning with reference audio.
|
||||
|
||||
Usage:
|
||||
# Text-only synthesis
|
||||
python end2end.py --text "Hello, this is a test."
|
||||
|
||||
# Voice cloning with reference audio
|
||||
python end2end.py --text "Hello, this is a test." --ref-audio ref.wav --ref-text "Reference text."
|
||||
|
||||
# Streaming mode
|
||||
python end2end.py --text "Hello, this is a test." --streaming
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
|
||||
import soundfile as sf
|
||||
import torch
|
||||
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
|
||||
from vllm.utils.argparse_utils import FlexibleArgumentParser
|
||||
|
||||
from vllm_omni import AsyncOmni, Omni
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
DEFAULT_MODEL = "fishaudio/s2-pro"
|
||||
DEFAULT_STAGE_CONFIG = os.path.join(
|
||||
os.path.dirname(__file__),
|
||||
"..",
|
||||
"..",
|
||||
"..",
|
||||
"vllm_omni",
|
||||
"model_executor",
|
||||
"stage_configs",
|
||||
"fish_speech_s2_pro.yaml",
|
||||
)
|
||||
|
||||
|
||||
def build_prompt(
|
||||
text: str,
|
||||
ref_audio_path: str | None = None,
|
||||
ref_text: str | None = None,
|
||||
model_name: str = DEFAULT_MODEL,
|
||||
) -> dict:
|
||||
"""Build a prompt for Fish Speech S2 Pro.
|
||||
|
||||
Uses Qwen3 chat template format with <|voice|> modality marker:
|
||||
<|im_start|>user\n<|speaker:0|>text<|im_end|>\n<|im_start|>assistant\n<|voice|>
|
||||
|
||||
The model then generates semantic tokens autoregressively.
|
||||
"""
|
||||
from transformers import AutoTokenizer
|
||||
|
||||
tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
|
||||
|
||||
# Build user message with speaker tag prefix.
|
||||
user_text = f"<|speaker:0|>{text}"
|
||||
|
||||
# Use chat template for user turn + assistant generation prompt.
|
||||
messages = [{"role": "user", "content": user_text}]
|
||||
prompt_ids = tokenizer.apply_chat_template(
|
||||
messages,
|
||||
tokenize=True,
|
||||
add_generation_prompt=True,
|
||||
)
|
||||
|
||||
# Append <|voice|> token (151673) to signal voice generation mode.
|
||||
voice_token_id = tokenizer.encode("<|voice|>", add_special_tokens=False)
|
||||
prompt_ids = prompt_ids + voice_token_id
|
||||
|
||||
additional_information = {
|
||||
"text": [text],
|
||||
"max_new_tokens": [4096],
|
||||
}
|
||||
|
||||
if ref_audio_path:
|
||||
additional_information["ref_audio"] = [ref_audio_path]
|
||||
if ref_text:
|
||||
additional_information["ref_text"] = [ref_text]
|
||||
|
||||
return {
|
||||
"prompt_token_ids": prompt_ids,
|
||||
"additional_information": additional_information,
|
||||
}
|
||||
|
||||
|
||||
def _save_wav(output_dir: str, request_id: str, mm: dict) -> None:
|
||||
"""Concatenate audio chunks and write to a wav file."""
|
||||
audio_data = mm["audio"]
|
||||
sr_raw = mm["sr"]
|
||||
sr_val = sr_raw[-1] if isinstance(sr_raw, list) and sr_raw else sr_raw
|
||||
sr = sr_val.item() if hasattr(sr_val, "item") else int(sr_val)
|
||||
audio_tensor = torch.cat(audio_data, dim=-1) if isinstance(audio_data, list) else audio_data
|
||||
out_wav = os.path.join(output_dir, f"output_{request_id}.wav")
|
||||
sf.write(out_wav, audio_tensor.float().cpu().numpy().flatten(), samplerate=sr, format="WAV")
|
||||
logger.info("Request %s: saved audio to %s (sr=%d)", request_id, out_wav, sr)
|
||||
|
||||
|
||||
def main(args):
|
||||
"""Run offline inference with Omni."""
|
||||
output_dir = args.output_dir
|
||||
os.makedirs(output_dir, exist_ok=True)
|
||||
|
||||
stage_configs_path = args.stage_configs_path or DEFAULT_STAGE_CONFIG
|
||||
model_name = args.model or DEFAULT_MODEL
|
||||
|
||||
inputs = [
|
||||
build_prompt(
|
||||
text=args.text,
|
||||
ref_audio_path=args.ref_audio,
|
||||
ref_text=args.ref_text,
|
||||
model_name=model_name,
|
||||
)
|
||||
]
|
||||
|
||||
omni = Omni(
|
||||
model=model_name,
|
||||
stage_configs_path=stage_configs_path,
|
||||
log_stats=args.log_stats,
|
||||
stage_init_timeout=args.stage_init_timeout,
|
||||
)
|
||||
|
||||
t_start = time.perf_counter()
|
||||
for stage_outputs in omni.generate(inputs):
|
||||
for output in stage_outputs.request_output:
|
||||
_save_wav(output_dir, output.request_id, output.outputs[0].multimodal_output)
|
||||
t_end = time.perf_counter()
|
||||
logger.info("Total inference time: %.1f ms", (t_end - t_start) * 1000)
|
||||
|
||||
|
||||
async def main_streaming(args):
|
||||
"""Run offline inference with AsyncOmni for streaming audio chunks."""
|
||||
output_dir = args.output_dir
|
||||
os.makedirs(output_dir, exist_ok=True)
|
||||
|
||||
stage_configs_path = args.stage_configs_path or DEFAULT_STAGE_CONFIG
|
||||
model_name = args.model or DEFAULT_MODEL
|
||||
|
||||
prompt = build_prompt(
|
||||
text=args.text,
|
||||
ref_audio_path=args.ref_audio,
|
||||
ref_text=args.ref_text,
|
||||
model_name=model_name,
|
||||
)
|
||||
|
||||
omni = AsyncOmni(
|
||||
model=model_name,
|
||||
stage_configs_path=stage_configs_path,
|
||||
log_stats=args.log_stats,
|
||||
stage_init_timeout=args.stage_init_timeout,
|
||||
)
|
||||
|
||||
request_id = "0"
|
||||
t_start = time.perf_counter()
|
||||
t_prev = t_start
|
||||
chunk_idx = 0
|
||||
|
||||
async for stage_output in omni.generate(prompt, request_id=request_id):
|
||||
mm = stage_output.request_output.outputs[0].multimodal_output
|
||||
if not stage_output.finished:
|
||||
t_now = time.perf_counter()
|
||||
audio = mm.get("audio")
|
||||
n = len(audio) if isinstance(audio, list) else (0 if audio is None else 1)
|
||||
dt_ms = (t_now - t_prev) * 1000
|
||||
ttfa_ms = (t_now - t_start) * 1000
|
||||
if chunk_idx == 0:
|
||||
logger.info("Request %s: chunk %d samples=%d TTFA=%.1fms", request_id, chunk_idx, n, ttfa_ms)
|
||||
else:
|
||||
logger.info("Request %s: chunk %d samples=%d inter_chunk=%.1fms", request_id, chunk_idx, n, dt_ms)
|
||||
t_prev = t_now
|
||||
chunk_idx += 1
|
||||
else:
|
||||
t_end = time.perf_counter()
|
||||
total_ms = (t_end - t_start) * 1000
|
||||
logger.info("Request %s: done total=%.1fms chunks=%d", request_id, total_ms, chunk_idx)
|
||||
_save_wav(output_dir, request_id, mm)
|
||||
|
||||
|
||||
def parse_args():
|
||||
parser = FlexibleArgumentParser(
|
||||
description="Fish Speech S2 Pro offline inference with vLLM Omni",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--text",
|
||||
type=str,
|
||||
default="Hello, this is a test of the Fish Speech text to speech system.",
|
||||
help="Text to synthesize.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--ref-audio",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Path to reference audio for voice cloning.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--ref-text",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Reference text corresponding to the reference audio.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--model",
|
||||
type=str,
|
||||
default=None,
|
||||
help=f"HuggingFace model path (default: {DEFAULT_MODEL}).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--stage-configs-path",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Path to stage configs YAML.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output-dir",
|
||||
default="output_audio",
|
||||
help="Output directory for generated wav files.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--log-stats",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Enable writing detailed statistics.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--stage-init-timeout",
|
||||
type=int,
|
||||
default=600,
|
||||
help="Timeout for initializing stages in seconds.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--streaming",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Stream audio chunks as they arrive via AsyncOmni.",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
args = parse_args()
|
||||
if args.streaming:
|
||||
asyncio.run(main_streaming(args))
|
||||
else:
|
||||
main(args)
|
||||
@@ -0,0 +1,172 @@
|
||||
# Fish Speech S2 Pro
|
||||
|
||||
## Model
|
||||
|
||||
| Model | Description |
|
||||
|-------|-------------|
|
||||
| `fishaudio/s2-pro` | Fish Speech S2 Pro -- 4B dual-AR TTS model with DAC codec (44.1 kHz) |
|
||||
|
||||
## Gradio Demo
|
||||
|
||||
An interactive Gradio demo is available with text-to-speech synthesis, voice cloning, and streaming support.
|
||||
|
||||
```bash
|
||||
# Option 1: Launch server + Gradio together
|
||||
./run_gradio_demo.sh
|
||||
|
||||
# Option 2: If server is already running
|
||||
python gradio_demo.py --api-base http://localhost:8091
|
||||
```
|
||||
|
||||
Then open http://localhost:7860 in your browser.
|
||||
|
||||
Features:
|
||||
|
||||
- Text-to-speech synthesis
|
||||
- Voice cloning from uploaded audio or URL
|
||||
- Streaming mode (progressive PCM playback)
|
||||
|
||||
## Launch the Server
|
||||
|
||||
```bash
|
||||
vllm-omni serve fishaudio/s2-pro \
|
||||
--stage-configs-path vllm_omni/model_executor/stage_configs/fish_speech_s2_pro.yaml \
|
||||
--omni \
|
||||
--port 8091 \
|
||||
--trust-remote-code \
|
||||
--enforce-eager \
|
||||
--gpu-memory-utilization 0.9
|
||||
```
|
||||
|
||||
Or use the convenience script:
|
||||
|
||||
```bash
|
||||
./run_server.sh
|
||||
```
|
||||
|
||||
## Send TTS Request
|
||||
|
||||
### Using curl
|
||||
|
||||
```bash
|
||||
# Basic TTS
|
||||
curl -X POST http://localhost:8091/v1/audio/speech \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"input": "Hello, how are you?",
|
||||
"voice": "default",
|
||||
"response_format": "wav"
|
||||
}' --output output.wav
|
||||
```
|
||||
|
||||
### Voice Cloning
|
||||
|
||||
Provide a reference audio (URL or base64 data URL) and its transcript:
|
||||
|
||||
```bash
|
||||
curl -X POST http://localhost:8091/v1/audio/speech \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"input": "Hello, this is a cloned voice.",
|
||||
"voice": "default",
|
||||
"ref_audio": "https://example.com/reference.wav",
|
||||
"ref_text": "Transcript of the reference audio."
|
||||
}' --output cloned.wav
|
||||
```
|
||||
|
||||
### Using Python
|
||||
|
||||
```python
|
||||
import httpx
|
||||
|
||||
# Basic TTS
|
||||
response = httpx.post(
|
||||
"http://localhost:8091/v1/audio/speech",
|
||||
json={
|
||||
"input": "Hello, how are you?",
|
||||
"voice": "default",
|
||||
"response_format": "wav",
|
||||
},
|
||||
timeout=300.0,
|
||||
)
|
||||
|
||||
with open("output.wav", "wb") as f:
|
||||
f.write(response.content)
|
||||
```
|
||||
|
||||
### Using the CLI Client
|
||||
|
||||
```bash
|
||||
cd examples/online_serving/fish_speech
|
||||
|
||||
# Basic TTS
|
||||
python speech_client.py --text "Hello, how are you?"
|
||||
|
||||
# Voice cloning
|
||||
python speech_client.py \
|
||||
--text "Hello, this is a cloned voice." \
|
||||
--ref-audio /path/to/reference.wav \
|
||||
--ref-text "Transcript of the reference audio."
|
||||
|
||||
# Streaming PCM output
|
||||
python speech_client.py --text "Hello world" --stream --output output.pcm
|
||||
```
|
||||
|
||||
The CLI client supports:
|
||||
|
||||
- `--api-base`: API base URL (default: `http://localhost:8091`)
|
||||
- `--model` (or `-m`): Model name (default: `fishaudio/s2-pro`)
|
||||
- `--text`: Text to synthesize (required)
|
||||
- `--ref-audio`: Reference audio for voice cloning (local path or URL)
|
||||
- `--ref-text`: Transcript of the reference audio
|
||||
- `--stream`: Enable streaming (PCM output)
|
||||
- `--response-format`: Audio format: wav, mp3, flac, pcm, aac, opus (default: wav)
|
||||
- `--output` (or `-o`): Output file path
|
||||
|
||||
## Streaming
|
||||
|
||||
Set `stream=true` with `response_format="pcm"` to receive raw PCM audio chunks as they are decoded:
|
||||
|
||||
```bash
|
||||
curl -X POST http://localhost:8091/v1/audio/speech \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"input": "Hello, how are you?",
|
||||
"voice": "default",
|
||||
"stream": true,
|
||||
"response_format": "pcm"
|
||||
}' --no-buffer | play -t raw -r 44100 -e signed -b 16 -c 1 -
|
||||
```
|
||||
|
||||
**Note:** Fish Speech outputs at 44.1 kHz (unlike Qwen3-TTS which outputs at 24 kHz).
|
||||
|
||||
## API Parameters
|
||||
|
||||
Fish Speech uses the same `/v1/audio/speech` endpoint as Qwen3-TTS. See the [Speech API reference](https://docs.vllm.ai/projects/vllm-omni/en/latest/serving/speech_api/) for full parameter documentation.
|
||||
|
||||
Key parameters for Fish Speech:
|
||||
|
||||
| Parameter | Type | Default | Description |
|
||||
|-----------|------|---------|-------------|
|
||||
| `input` | string | **required** | Text to synthesize |
|
||||
| `voice` | string | "default" | Voice name (use "default" for Fish Speech) |
|
||||
| `response_format` | string | "wav" | Audio format: wav, mp3, flac, pcm, aac, opus |
|
||||
| `ref_audio` | string | null | Reference audio URL or base64 data URL for voice cloning |
|
||||
| `ref_text` | string | null | Transcript of reference audio (required for voice cloning) |
|
||||
| `max_new_tokens` | int | 4096 | Maximum tokens to generate |
|
||||
| `stream` | bool | false | Stream raw PCM chunks |
|
||||
|
||||
## Prerequisites
|
||||
|
||||
Install the `fish-speech` package for the DAC codec:
|
||||
|
||||
```bash
|
||||
pip install fish-speech
|
||||
```
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
1. **No audio output**: Make sure the `fish-speech` package is installed for the DAC decoder
|
||||
2. **Connection refused**: Ensure the server is running on the correct port
|
||||
3. **Flashinfer version mismatch**: Set `FLASHINFER_DISABLE_VERSION_CHECK=1` if you see version warnings
|
||||
4. **Out of memory**: Reduce `--gpu-memory-utilization` or use a GPU with more VRAM
|
||||
@@ -0,0 +1,272 @@
|
||||
"""Gradio demo for Fish Speech S2 Pro online serving via /v1/audio/speech API.
|
||||
|
||||
Supports:
|
||||
- Text-to-speech synthesis
|
||||
- Voice cloning from reference audio (upload or URL)
|
||||
- Streaming (progressive PCM chunks) and non-streaming modes
|
||||
|
||||
Usage:
|
||||
# Start the server first (see run_server.sh), then:
|
||||
python gradio_demo.py --api-base http://localhost:8091
|
||||
|
||||
# Or use run_gradio_demo.sh to start both server and demo together.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import base64
|
||||
import io
|
||||
|
||||
import gradio as gr
|
||||
import httpx
|
||||
import numpy as np
|
||||
import soundfile as sf
|
||||
|
||||
PCM_SAMPLE_RATE = 44100
|
||||
DEFAULT_API_BASE = "http://localhost:8091"
|
||||
|
||||
|
||||
def encode_audio_to_base64(audio_data: tuple) -> str:
|
||||
"""Encode Gradio audio input (sample_rate, numpy_array) to base64 data URL."""
|
||||
sample_rate, audio_np = audio_data
|
||||
if audio_np.dtype != np.int16:
|
||||
if audio_np.dtype in (np.float32, np.float64):
|
||||
audio_np = np.clip(audio_np, -1.0, 1.0)
|
||||
audio_np = (audio_np * 32767).astype(np.int16)
|
||||
else:
|
||||
audio_np = audio_np.astype(np.int16)
|
||||
buf = io.BytesIO()
|
||||
sf.write(buf, audio_np, sample_rate, format="WAV")
|
||||
wav_b64 = base64.b64encode(buf.getvalue()).decode("utf-8")
|
||||
return f"data:audio/wav;base64,{wav_b64}"
|
||||
|
||||
|
||||
def build_payload(
|
||||
text: str,
|
||||
ref_audio: tuple | None,
|
||||
ref_audio_url: str,
|
||||
ref_text: str,
|
||||
response_format: str = "wav",
|
||||
stream: bool = False,
|
||||
) -> dict:
|
||||
"""Build the /v1/audio/speech request payload."""
|
||||
if not text or not text.strip():
|
||||
raise gr.Error("Please enter text to synthesize.")
|
||||
|
||||
payload: dict = {
|
||||
"input": text.strip(),
|
||||
"voice": "default",
|
||||
"response_format": "pcm" if stream else response_format,
|
||||
"stream": stream,
|
||||
}
|
||||
|
||||
# Voice cloning: ref_audio takes priority over URL.
|
||||
ref_url_stripped = ref_audio_url.strip() if ref_audio_url else ""
|
||||
if ref_audio is not None:
|
||||
payload["ref_audio"] = encode_audio_to_base64(ref_audio)
|
||||
elif ref_url_stripped:
|
||||
payload["ref_audio"] = ref_url_stripped
|
||||
|
||||
if "ref_audio" in payload:
|
||||
if not ref_text or not ref_text.strip():
|
||||
raise gr.Error("Voice cloning requires a transcript of the reference audio.")
|
||||
payload["ref_text"] = ref_text.strip()
|
||||
|
||||
return payload
|
||||
|
||||
|
||||
def generate_speech(api_base: str, text: str, ref_audio, ref_audio_url, ref_text, response_format):
|
||||
"""Non-streaming: call /v1/audio/speech and return full audio."""
|
||||
payload = build_payload(text, ref_audio, ref_audio_url, ref_text, response_format, stream=False)
|
||||
|
||||
try:
|
||||
with httpx.Client(timeout=300.0) as client:
|
||||
resp = client.post(
|
||||
f"{api_base}/v1/audio/speech",
|
||||
json=payload,
|
||||
headers={"Content-Type": "application/json", "Authorization": "Bearer EMPTY"},
|
||||
)
|
||||
except httpx.TimeoutException:
|
||||
raise gr.Error("Request timed out. The server may be busy.")
|
||||
except httpx.ConnectError:
|
||||
raise gr.Error(f"Cannot connect to server at {api_base}. Is the server running?")
|
||||
|
||||
if resp.status_code != 200:
|
||||
raise gr.Error(f"Server error ({resp.status_code}): {resp.text}")
|
||||
|
||||
content_type = resp.headers.get("content-type", "")
|
||||
if "application/json" in content_type:
|
||||
try:
|
||||
raise gr.Error(f"Server error: {resp.json()}")
|
||||
except ValueError:
|
||||
pass
|
||||
|
||||
try:
|
||||
if response_format == "pcm":
|
||||
audio_np = np.frombuffer(resp.content, dtype=np.int16).astype(np.float32) / 32767.0
|
||||
return (PCM_SAMPLE_RATE, audio_np)
|
||||
audio_np, sample_rate = sf.read(io.BytesIO(resp.content))
|
||||
if audio_np.ndim > 1:
|
||||
audio_np = audio_np[:, 0]
|
||||
return (sample_rate, audio_np.astype(np.float32))
|
||||
except Exception as e:
|
||||
raise gr.Error(f"Failed to decode audio: {e}")
|
||||
|
||||
|
||||
def generate_speech_stream(api_base, text, ref_audio, ref_audio_url, ref_text, response_format, chunk_seconds=0.5):
|
||||
"""Streaming: yield PCM chunks for progressive playback."""
|
||||
payload = build_payload(text, ref_audio, ref_audio_url, ref_text, response_format, stream=True)
|
||||
|
||||
min_chunk_samples = int(PCM_SAMPLE_RATE * chunk_seconds)
|
||||
prebuffer_chunks = 2
|
||||
pending: list[np.ndarray] = []
|
||||
pending_len = 0
|
||||
chunks_yielded = 0
|
||||
prebuffer: list[np.ndarray] = []
|
||||
|
||||
try:
|
||||
leftover = b""
|
||||
with httpx.Client(timeout=300.0) as client:
|
||||
with client.stream(
|
||||
"POST",
|
||||
f"{api_base}/v1/audio/speech",
|
||||
json=payload,
|
||||
headers={"Content-Type": "application/json", "Authorization": "Bearer EMPTY"},
|
||||
) as resp:
|
||||
if resp.status_code != 200:
|
||||
resp.read()
|
||||
raise gr.Error(f"Server error ({resp.status_code}): {resp.text}")
|
||||
for chunk in resp.iter_bytes():
|
||||
if not chunk:
|
||||
continue
|
||||
raw = leftover + chunk
|
||||
usable = len(raw) - (len(raw) % 2)
|
||||
leftover = raw[usable:]
|
||||
if usable == 0:
|
||||
continue
|
||||
float_samples = np.frombuffer(raw[:usable], dtype=np.int16).copy().astype(np.float32) / 32767.0
|
||||
pending.append(float_samples)
|
||||
pending_len += len(float_samples)
|
||||
if pending_len >= min_chunk_samples:
|
||||
audio_chunk = np.concatenate(pending)
|
||||
pending.clear()
|
||||
pending_len = 0
|
||||
if chunks_yielded < prebuffer_chunks:
|
||||
prebuffer.append(audio_chunk)
|
||||
chunks_yielded += 1
|
||||
if chunks_yielded == prebuffer_chunks:
|
||||
yield (PCM_SAMPLE_RATE, np.concatenate(prebuffer))
|
||||
prebuffer.clear()
|
||||
else:
|
||||
yield (PCM_SAMPLE_RATE, audio_chunk)
|
||||
remaining = prebuffer + pending
|
||||
if remaining:
|
||||
yield (PCM_SAMPLE_RATE, np.concatenate(remaining))
|
||||
except httpx.TimeoutException:
|
||||
raise gr.Error("Request timed out. The server may be busy.")
|
||||
except httpx.ConnectError:
|
||||
raise gr.Error(f"Cannot connect to server at {api_base}. Is the server running?")
|
||||
|
||||
|
||||
def on_stream_change(stream: bool):
|
||||
"""Lock format to PCM when streaming."""
|
||||
if stream:
|
||||
return gr.update(value="pcm", interactive=False)
|
||||
return gr.update(value="wav", interactive=True)
|
||||
|
||||
|
||||
def build_interface(api_base: str, stream_chunk_seconds: float = 0.5):
|
||||
"""Build the Gradio interface."""
|
||||
with gr.Blocks(title="Fish Speech S2 Pro Demo") as demo:
|
||||
gr.Markdown("# Fish Speech S2 Pro - Text to Speech")
|
||||
gr.Markdown(f"**Server:** `{api_base}` | **Model:** fishaudio/s2-pro | **Output:** 44.1kHz")
|
||||
|
||||
with gr.Row():
|
||||
with gr.Column(scale=3):
|
||||
text_input = gr.Textbox(
|
||||
label="Text to Synthesize",
|
||||
placeholder="Enter text here...",
|
||||
lines=4,
|
||||
)
|
||||
|
||||
with gr.Accordion("Voice Cloning (optional)", open=False):
|
||||
gr.Markdown(
|
||||
"Upload or link a short reference audio (10-30s) and provide its transcript to clone the voice."
|
||||
)
|
||||
ref_audio = gr.Audio(
|
||||
label="Reference Audio",
|
||||
type="numpy",
|
||||
sources=["upload", "microphone"],
|
||||
)
|
||||
ref_audio_url = gr.Textbox(
|
||||
label="Reference Audio URL (alternative to upload)",
|
||||
placeholder="https://example.com/reference.wav",
|
||||
lines=1,
|
||||
)
|
||||
ref_text = gr.Textbox(
|
||||
label="Reference Audio Transcript (required for cloning)",
|
||||
placeholder="Exact transcript of the reference audio...",
|
||||
lines=2,
|
||||
)
|
||||
|
||||
with gr.Row():
|
||||
response_format = gr.Dropdown(
|
||||
choices=["wav", "mp3", "flac", "pcm"],
|
||||
value="wav",
|
||||
label="Audio Format",
|
||||
scale=1,
|
||||
)
|
||||
stream_checkbox = gr.Checkbox(
|
||||
label="Stream output",
|
||||
value=False,
|
||||
info="Progressive PCM streaming",
|
||||
scale=1,
|
||||
)
|
||||
|
||||
generate_btn = gr.Button("Generate Speech", variant="primary", size="lg")
|
||||
|
||||
with gr.Column(scale=2):
|
||||
audio_output = gr.Audio(
|
||||
label="Generated Audio",
|
||||
interactive=False,
|
||||
streaming=True,
|
||||
autoplay=True,
|
||||
)
|
||||
gr.Markdown(
|
||||
"### About\n"
|
||||
"- **Fish Speech S2 Pro** by FishAudio: 4B dual-AR model\n"
|
||||
"- **Voice cloning**: upload 10-30s reference + transcript\n"
|
||||
"- **Streaming**: real-time PCM output\n"
|
||||
"- **44.1kHz** output via DAC codec"
|
||||
)
|
||||
|
||||
stream_checkbox.change(fn=on_stream_change, inputs=[stream_checkbox], outputs=[response_format])
|
||||
|
||||
all_inputs = [text_input, ref_audio, ref_audio_url, ref_text, response_format]
|
||||
|
||||
def dispatch(stream_enabled, *args):
|
||||
if stream_enabled:
|
||||
yield from generate_speech_stream(api_base, *args, chunk_seconds=stream_chunk_seconds)
|
||||
else:
|
||||
yield generate_speech(api_base, *args)
|
||||
|
||||
generate_btn.click(fn=dispatch, inputs=[stream_checkbox] + all_inputs, outputs=[audio_output])
|
||||
demo.queue()
|
||||
return demo
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Gradio demo for Fish Speech S2 Pro")
|
||||
parser.add_argument("--api-base", default=DEFAULT_API_BASE, help=f"API base URL (default: {DEFAULT_API_BASE})")
|
||||
parser.add_argument("--host", default="0.0.0.0", help="Gradio host (default: 0.0.0.0)")
|
||||
parser.add_argument("--port", type=int, default=7860, help="Gradio port (default: 7860)")
|
||||
parser.add_argument("--share", action="store_true", help="Share publicly")
|
||||
parser.add_argument("--stream-chunk-seconds", type=float, default=0.5, help="Seconds per streaming chunk")
|
||||
args = parser.parse_args()
|
||||
|
||||
print(f"Connecting to vLLM server at: {args.api_base}")
|
||||
demo = build_interface(args.api_base, stream_chunk_seconds=args.stream_chunk_seconds)
|
||||
demo.launch(server_name=args.host, server_port=args.port, share=args.share)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+48
@@ -0,0 +1,48 @@
|
||||
#!/bin/bash
|
||||
# Launch Fish Speech S2 Pro server + Gradio demo together.
|
||||
#
|
||||
# Usage:
|
||||
# ./run_gradio_demo.sh
|
||||
# CUDA_VISIBLE_DEVICES=0 PORT=8091 GRADIO_PORT=7860 ./run_gradio_demo.sh
|
||||
|
||||
set -e
|
||||
|
||||
MODEL="${MODEL:-fishaudio/s2-pro}"
|
||||
PORT="${PORT:-8091}"
|
||||
GRADIO_PORT="${GRADIO_PORT:-7860}"
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
REPO_ROOT="$(cd "$SCRIPT_DIR/../../.." && pwd)"
|
||||
|
||||
echo "Starting Fish Speech S2 Pro server (port $PORT)..."
|
||||
FLASHINFER_DISABLE_VERSION_CHECK=1 \
|
||||
vllm-omni serve "$MODEL" \
|
||||
--stage-configs-path "$REPO_ROOT/vllm_omni/model_executor/stage_configs/fish_speech_s2_pro.yaml" \
|
||||
--host 0.0.0.0 \
|
||||
--port "$PORT" \
|
||||
--gpu-memory-utilization 0.9 \
|
||||
--trust-remote-code \
|
||||
--enforce-eager \
|
||||
--omni &
|
||||
SERVER_PID=$!
|
||||
|
||||
cleanup() {
|
||||
echo "Stopping server (PID $SERVER_PID)..."
|
||||
kill $SERVER_PID 2>/dev/null
|
||||
wait $SERVER_PID 2>/dev/null
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
# Wait for server to be ready.
|
||||
echo "Waiting for server to start..."
|
||||
for i in $(seq 1 120); do
|
||||
if curl -s "http://localhost:$PORT/health" > /dev/null 2>&1; then
|
||||
echo "Server ready."
|
||||
break
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
|
||||
echo "Starting Gradio demo (port $GRADIO_PORT)..."
|
||||
python "$SCRIPT_DIR/gradio_demo.py" \
|
||||
--api-base "http://localhost:$PORT" \
|
||||
--port "$GRADIO_PORT"
|
||||
+23
@@ -0,0 +1,23 @@
|
||||
#!/bin/bash
|
||||
# Launch vLLM-Omni server for Fish Speech S2 Pro
|
||||
#
|
||||
# Usage:
|
||||
# ./run_server.sh
|
||||
# CUDA_VISIBLE_DEVICES=0 ./run_server.sh
|
||||
|
||||
set -e
|
||||
|
||||
MODEL="${MODEL:-fishaudio/s2-pro}"
|
||||
PORT="${PORT:-8091}"
|
||||
|
||||
echo "Starting Fish Speech S2 Pro server with model: $MODEL"
|
||||
|
||||
FLASHINFER_DISABLE_VERSION_CHECK=1 \
|
||||
vllm-omni serve "$MODEL" \
|
||||
--stage-configs-path vllm_omni/model_executor/stage_configs/fish_speech_s2_pro.yaml \
|
||||
--host 0.0.0.0 \
|
||||
--port "$PORT" \
|
||||
--gpu-memory-utilization 0.9 \
|
||||
--trust-remote-code \
|
||||
--enforce-eager \
|
||||
--omni
|
||||
@@ -0,0 +1,129 @@
|
||||
"""Client for Fish Speech S2 Pro via /v1/audio/speech endpoint.
|
||||
|
||||
Examples:
|
||||
# Basic TTS
|
||||
python speech_client.py --text "Hello, how are you?"
|
||||
|
||||
# Voice cloning
|
||||
python speech_client.py --text "Hello, how are you?" \
|
||||
--ref-audio ref.wav --ref-text "This is the reference transcript."
|
||||
|
||||
# Streaming PCM output
|
||||
python speech_client.py --text "Hello world" --stream --output output.pcm
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import base64
|
||||
import os
|
||||
|
||||
import httpx
|
||||
|
||||
DEFAULT_API_BASE = "http://localhost:8091"
|
||||
DEFAULT_API_KEY = "EMPTY"
|
||||
|
||||
|
||||
def encode_audio_to_base64(audio_path: str) -> str:
|
||||
"""Encode a local audio file to base64 data URL."""
|
||||
if not os.path.exists(audio_path):
|
||||
raise FileNotFoundError(f"Audio file not found: {audio_path}")
|
||||
ext = audio_path.lower().rsplit(".", 1)[-1]
|
||||
mime_map = {"wav": "audio/wav", "mp3": "audio/mpeg", "flac": "audio/flac", "ogg": "audio/ogg"}
|
||||
mime_type = mime_map.get(ext, "audio/wav")
|
||||
with open(audio_path, "rb") as f:
|
||||
audio_b64 = base64.b64encode(f.read()).decode("utf-8")
|
||||
return f"data:{mime_type};base64,{audio_b64}"
|
||||
|
||||
|
||||
def run_tts(args) -> None:
|
||||
"""Generate speech via /v1/audio/speech API."""
|
||||
payload = {
|
||||
"model": args.model,
|
||||
"input": args.text,
|
||||
"voice": "default",
|
||||
"response_format": args.response_format,
|
||||
}
|
||||
|
||||
# Voice cloning parameters.
|
||||
if args.ref_audio:
|
||||
if args.ref_audio.startswith(("http://", "https://")):
|
||||
payload["ref_audio"] = args.ref_audio
|
||||
else:
|
||||
payload["ref_audio"] = encode_audio_to_base64(args.ref_audio)
|
||||
if args.ref_text:
|
||||
payload["ref_text"] = args.ref_text
|
||||
|
||||
if args.stream:
|
||||
payload["stream"] = True
|
||||
payload["response_format"] = "pcm"
|
||||
|
||||
print(f"Model: {args.model}")
|
||||
print(f"Text: {args.text}")
|
||||
if args.ref_audio:
|
||||
print(f"Voice cloning: ref_audio={args.ref_audio}, ref_text={args.ref_text}")
|
||||
print("Generating audio...")
|
||||
|
||||
api_url = f"{args.api_base}/v1/audio/speech"
|
||||
headers = {
|
||||
"Content-Type": "application/json",
|
||||
"Authorization": f"Bearer {args.api_key}",
|
||||
}
|
||||
|
||||
if args.stream:
|
||||
output_path = args.output or "output.pcm"
|
||||
with httpx.Client(timeout=300.0) as client:
|
||||
with client.stream("POST", api_url, json=payload, headers=headers) as resp:
|
||||
if resp.status_code != 200:
|
||||
print(f"Error: {resp.status_code}")
|
||||
print(resp.read().decode())
|
||||
return
|
||||
total_bytes = 0
|
||||
with open(output_path, "wb") as f:
|
||||
for chunk in resp.iter_bytes():
|
||||
f.write(chunk)
|
||||
total_bytes += len(chunk)
|
||||
print(f"Streamed {total_bytes} bytes to: {output_path}")
|
||||
else:
|
||||
with httpx.Client(timeout=300.0) as client:
|
||||
response = client.post(api_url, json=payload, headers=headers)
|
||||
|
||||
if response.status_code != 200:
|
||||
print(f"Error: {response.status_code}")
|
||||
print(response.text)
|
||||
return
|
||||
|
||||
try:
|
||||
text = response.content.decode("utf-8")
|
||||
if text.startswith('{"error"'):
|
||||
print(f"Error: {text}")
|
||||
return
|
||||
except UnicodeDecodeError:
|
||||
pass
|
||||
|
||||
output_path = args.output or "output.wav"
|
||||
with open(output_path, "wb") as f:
|
||||
f.write(response.content)
|
||||
print(f"Audio saved to: {output_path}")
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Fish Speech S2 Pro TTS client")
|
||||
parser.add_argument("--api-base", default=DEFAULT_API_BASE, help="API base URL")
|
||||
parser.add_argument("--api-key", default=DEFAULT_API_KEY, help="API key")
|
||||
parser.add_argument("--model", "-m", default="fishaudio/s2-pro", help="Model name")
|
||||
parser.add_argument("--text", required=True, help="Text to synthesize")
|
||||
parser.add_argument("--ref-audio", default=None, help="Reference audio for voice cloning (path or URL)")
|
||||
parser.add_argument("--ref-text", default=None, help="Transcript of reference audio")
|
||||
parser.add_argument("--stream", action="store_true", help="Enable streaming (PCM output)")
|
||||
parser.add_argument(
|
||||
"--response-format",
|
||||
default="wav",
|
||||
choices=["wav", "mp3", "flac", "pcm", "aac", "opus"],
|
||||
help="Audio format (default: wav)",
|
||||
)
|
||||
parser.add_argument("--output", "-o", default=None, help="Output file path")
|
||||
args = parser.parse_args()
|
||||
run_tts(args)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in new issue
Block a user