[Maintenance] Refresh MOSS-TTS optimization on main
Signed-off-by: Sy03 <1370724210@qq.com>
This commit is contained in:
commit
28dd49a541
334 files changed
+24909
-6187
No files matched your search
@@ -5,10 +5,12 @@ This example shows how to use vLLM-Omni for running offline inference
|
||||
with the correct prompt format on Qwen2.5-Omni
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import functools
|
||||
import json
|
||||
import os
|
||||
import time
|
||||
from typing import NamedTuple
|
||||
from typing import Any, NamedTuple
|
||||
|
||||
import numpy as np
|
||||
import soundfile as sf
|
||||
@@ -26,6 +28,20 @@ from vllm_omni.utils.tracking_parser import TrackingArgumentParser
|
||||
SEED = 42
|
||||
|
||||
|
||||
def parse_json_object(value: str, flag_name: str = "argument") -> dict[str, Any]:
|
||||
"""Parse a CLI value as a JSON object, attributing errors to ``flag_name``."""
|
||||
try:
|
||||
config = json.loads(value)
|
||||
except json.JSONDecodeError as e:
|
||||
raise argparse.ArgumentTypeError(f"{flag_name} must be valid JSON: {e}") from e
|
||||
if not isinstance(config, dict):
|
||||
raise argparse.ArgumentTypeError(f"{flag_name} must be a JSON object")
|
||||
return config
|
||||
|
||||
|
||||
parse_profiler_config = functools.partial(parse_json_object, flag_name="--profiler-config")
|
||||
|
||||
|
||||
class QueryResult(NamedTuple):
|
||||
inputs: dict
|
||||
limit_mm_per_prompt: dict[str, int]
|
||||
@@ -378,11 +394,10 @@ def main(args):
|
||||
for i, prompt in enumerate(prompts):
|
||||
prompt["modalities"] = output_modalities
|
||||
|
||||
profiler_enabled = bool(os.getenv("VLLM_TORCH_PROFILER_DIR"))
|
||||
if profiler_enabled and hasattr(omni, "start_profile"):
|
||||
profiler_enabled = args.profiler_config is not None
|
||||
if profiler_enabled:
|
||||
print("[Profiler] Starting profiling...")
|
||||
omni.start_profile(stages=[0])
|
||||
elif profiler_enabled:
|
||||
print("[Warn] VLLM_TORCH_PROFILER_DIR is set, but current engine does not support profiler controls.")
|
||||
omni_generator = omni.generate(prompts, sampling_params_list, py_generator=args.py_generator)
|
||||
|
||||
# Determine output directory: prefer --output-dir; fallback to --output-wav
|
||||
@@ -418,7 +433,7 @@ def main(args):
|
||||
print(f"Request ID: {request_id}, Saved audio to {output_wav}")
|
||||
|
||||
processed_count += 1
|
||||
if profiler_enabled and hasattr(omni, "stop_profile") and processed_count >= total_requests:
|
||||
if profiler_enabled and processed_count >= total_requests:
|
||||
print(f"[Info] Processed {processed_count}/{total_requests}. Stopping profiler inside active loop...")
|
||||
# Stop the profiler while workers are still alive
|
||||
omni.stop_profile()
|
||||
@@ -476,6 +491,12 @@ def parse_args():
|
||||
default=300,
|
||||
help="Timeout for initializing stages in seconds (default: 300)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--profiler-config",
|
||||
type=parse_profiler_config,
|
||||
default=None,
|
||||
help='JSON profiler config for torch/cuda profiling, e.g. \'{"profiler":"torch","torch_profiler_dir":"./perf"}\'.',
|
||||
)
|
||||
parser.add_argument(
|
||||
"--shm-threshold-bytes",
|
||||
type=int,
|
||||
|
||||
@@ -14,6 +14,7 @@ list of supported architectures across all modalities, see
|
||||
|
||||
| Model | HuggingFace repo | Stages | Voice cloning | Streaming | Special modes | Sample rate |
|
||||
|---|---|---|---|---|---|---|
|
||||
| Audio8 TTS Preview | `Audio8/Audio8-TTS-Preview-0.6b` | dual-AR | ✓ | ✓ | 11 languages | 44.1 kHz |
|
||||
| CosyVoice3 | `FunAudioLLM/Fun-CosyVoice3-0.5B-2512` | 2 (talker + code2wav) | ✓ | ✓ | — | 24 kHz |
|
||||
| Fish Speech S2 Pro | `fishaudio/s2-pro` | dual-AR | ✓ | ✓ | — | 44.1 kHz |
|
||||
| Gepard-1.0 | `nineninesix/gepard-1.0` | single (native AR + NanoCodec) | — (zero-shot; cloning WIP) | — (serving WIP) | zero-shot | 22.05 kHz |
|
||||
@@ -44,6 +45,57 @@ python examples/offline_inference/text_to_speech/<model>/end2end.py \
|
||||
|
||||
---
|
||||
|
||||
## Audio8 TTS Preview
|
||||
|
||||
0.6B DualAR text-to-speech model (Audio8) with its own 44.1 kHz neural audio
|
||||
codec: a slow AR transformer predicts one semantic token per codec frame, and a
|
||||
4-layer fast AR predicts that frame's remaining 9 codebooks.
|
||||
|
||||
### Prerequisites
|
||||
No extra packages: the codec architecture ships in tree and its weights
|
||||
(`codec.pth`) come with the checkpoint.
|
||||
|
||||
### Quick start
|
||||
```bash
|
||||
python examples/offline_inference/text_to_speech/audio8_tts/end2end.py \
|
||||
--text "Welcome to Audio8 TTS, a compact multilingual text to speech model."
|
||||
```
|
||||
|
||||
### Voice cloning
|
||||
```bash
|
||||
python examples/offline_inference/text_to_speech/audio8_tts/end2end.py \
|
||||
--text "Welcome to Audio8 TTS." \
|
||||
--ref-audio /path/to/reference.wav \
|
||||
--ref-text "The exact transcript of the reference recording."
|
||||
```
|
||||
The reference transcript must match what is actually spoken in the clip;
|
||||
a mismatch degrades both stability and speaker similarity.
|
||||
|
||||
### Streaming
|
||||
```bash
|
||||
python examples/offline_inference/text_to_speech/audio8_tts/end2end.py \
|
||||
--text "Welcome to Audio8 TTS." --streaming
|
||||
```
|
||||
|
||||
### Notes
|
||||
- Output: 44.1 kHz mono WAV; the codec emits ~21.5 frames/s (2048 samples per frame).
|
||||
- Context is 2048 packed text+audio positions, so `max_model_len` is 2048.
|
||||
- Greedy decoding (`temperature: 0`) is degenerate for this checkpoint — it
|
||||
ends the utterance almost immediately. Keep the deploy defaults
|
||||
(`temperature 0.7`, `top_k 50`, `top_p 0.9`), which also match
|
||||
`generation_config.json`.
|
||||
- Repetition-Aware Sampling is applied to the semantic token: a repeat inside
|
||||
the last 10 frames is re-drawn from a flatter distribution instead of being
|
||||
masked out.
|
||||
- `--num-prompts 4` smoke-tests concurrency (stage 0 runs `max_num_seqs: 4`).
|
||||
|
||||
### Measured behaviour (1x H20, shared GPU)
|
||||
`--streaming` on a 58-character English sentence: TTFA ~0.74 s (4-frame first
|
||||
chunk), 7 chunks, 1.55 s wall for 3.16 s of audio (RTF ~0.49). Treat these as
|
||||
smoke-test magnitudes, not a benchmark.
|
||||
|
||||
---
|
||||
|
||||
## CosyVoice3
|
||||
|
||||
2-stage TTS pipeline (`talker` + `code2wav`) at 24 kHz.
|
||||
@@ -130,11 +182,7 @@ Text → [Stage 0: AR] → Speech Tokens → [Stage 1: DiT + HiFT] → Audio (24
|
||||
## Fish Speech S2 Pro
|
||||
|
||||
4B dual-AR text-to-speech model from FishAudio with the DAC codec at 44.1 kHz.
|
||||
|
||||
### Prerequisites
|
||||
```bash
|
||||
pip install fish-speech
|
||||
```
|
||||
No extra packages are required; the DAC codec is vendored in vLLM-Omni.
|
||||
|
||||
### Quick start
|
||||
```bash
|
||||
|
||||
@@ -0,0 +1,237 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
"""Offline inference demo for Audio8 TTS Preview 0.6B via vLLM Omni.
|
||||
|
||||
Usage:
|
||||
# Text-only synthesis
|
||||
python end2end.py --text "Welcome to Audio8 TTS."
|
||||
|
||||
# Zero-shot voice cloning (the transcript must match the reference audio)
|
||||
python end2end.py --text "Welcome to Audio8 TTS." \
|
||||
--ref-audio reference.wav --ref-text "The exact transcript of the reference recording."
|
||||
|
||||
# Streaming (per-chunk TTFA / inter-chunk timings)
|
||||
python end2end.py --text "Welcome to Audio8 TTS." --streaming
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
import time
|
||||
|
||||
import numpy as np
|
||||
import soundfile as sf
|
||||
import torch
|
||||
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
|
||||
from vllm_omni import AsyncOmni, Omni
|
||||
from vllm_omni.model_executor.models.audio8_tts.codec_utils import estimate_reference_code_frames
|
||||
from vllm_omni.model_executor.models.audio8_tts.prompt_utils import (
|
||||
build_text_only_prompt_ids,
|
||||
estimate_voice_clone_prompt_len,
|
||||
normalize_text,
|
||||
)
|
||||
from vllm_omni.utils.tracking_parser import TrackingArgumentParser
|
||||
|
||||
DEFAULT_MODEL = "Audio8/Audio8-TTS-Preview-0.6b"
|
||||
|
||||
|
||||
def build_prompt(
|
||||
text: str,
|
||||
ref_audio_path: str | None = None,
|
||||
ref_text: str | None = None,
|
||||
model_name: str = DEFAULT_MODEL,
|
||||
) -> dict:
|
||||
"""Build an engine prompt, using the same protocol as online serving."""
|
||||
from transformers import AutoTokenizer
|
||||
|
||||
tokenizer = AutoTokenizer.from_pretrained(model_name)
|
||||
if ref_audio_path is None and ref_text is None:
|
||||
prompt_ids, normalized_text = build_text_only_prompt_ids(tokenizer, text)
|
||||
return {
|
||||
"prompt_token_ids": prompt_ids,
|
||||
"additional_information": {"text": [normalized_text]},
|
||||
}
|
||||
|
||||
if not ref_audio_path or not ref_text:
|
||||
raise ValueError("Audio8 TTS voice cloning requires both --ref-audio and --ref-text")
|
||||
|
||||
normalized_text = normalize_text(text)
|
||||
normalized_ref_text = normalize_text(ref_text, add_default_speaker=True)
|
||||
ref_audio_wav, ref_audio_sr = sf.read(ref_audio_path, dtype="float32", always_2d=False)
|
||||
if ref_audio_wav.ndim > 1:
|
||||
ref_audio_wav = ref_audio_wav.mean(axis=-1)
|
||||
ref_frames = estimate_reference_code_frames(int(ref_audio_wav.shape[0]), int(ref_audio_sr))
|
||||
placeholder_len = estimate_voice_clone_prompt_len(tokenizer, normalized_text, normalized_ref_text, ref_frames)
|
||||
|
||||
# The clone prompt embeds the reference audio's own codec codes, so the
|
||||
# model builds it in preprocess(); we only reserve the exact length here.
|
||||
return {
|
||||
"prompt_token_ids": [1] * placeholder_len,
|
||||
"additional_information": {
|
||||
"text": normalized_text,
|
||||
"ref_text": normalized_ref_text,
|
||||
"ref_audio_wav": torch.from_numpy(np.asarray(ref_audio_wav, dtype=np.float32)),
|
||||
"ref_audio_sr": int(ref_audio_sr),
|
||||
"audio8_structured_voice_clone": True,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def extract_audio(multimodal_output: dict) -> tuple[torch.Tensor, int]:
|
||||
"""Pull the waveform and sample rate out of a multimodal output.
|
||||
|
||||
The output processor concatenates the per-step delta tensors under
|
||||
``model_outputs``; ``audio`` is the older alias. Never use ``a or b`` on
|
||||
these values -- truthiness on a tensor raises.
|
||||
"""
|
||||
audio = multimodal_output.get("model_outputs")
|
||||
if audio is None:
|
||||
audio = multimodal_output.get("audio")
|
||||
if audio is None:
|
||||
raise ValueError(f"No audio in multimodal_output: {sorted(multimodal_output)}")
|
||||
if isinstance(audio, list):
|
||||
audio = torch.cat([torch.as_tensor(chunk).reshape(-1) for chunk in audio], dim=0)
|
||||
audio = torch.as_tensor(audio).reshape(-1)
|
||||
|
||||
sr_raw = multimodal_output.get("sr")
|
||||
if isinstance(sr_raw, list):
|
||||
sr_raw = sr_raw[-1] if sr_raw else None
|
||||
if sr_raw is None:
|
||||
raise ValueError("Missing sample rate in multimodal_output")
|
||||
sample_rate = int(sr_raw.item()) if hasattr(sr_raw, "item") else int(sr_raw)
|
||||
return audio, sample_rate
|
||||
|
||||
|
||||
def _save_wav(output_dir: str, request_id: str, audio: torch.Tensor, sample_rate: int) -> None:
|
||||
out_wav = os.path.join(output_dir, f"output_{request_id}.wav")
|
||||
sf.write(out_wav, audio.float().cpu().numpy(), samplerate=sample_rate, format="WAV")
|
||||
duration = audio.numel() / sample_rate
|
||||
# print(), not logging: vLLM reconfigures logging on engine start and
|
||||
# disables loggers created before that, which would swallow these lines.
|
||||
print(f"Request {request_id}: saved {out_wav} (sr={sample_rate}, {duration:.2f}s)")
|
||||
|
||||
|
||||
def _omni_kwargs(args, model_name: str) -> dict:
|
||||
kwargs = {
|
||||
"model": model_name,
|
||||
"log_stats": args.log_stats,
|
||||
"stage_init_timeout": args.stage_init_timeout,
|
||||
}
|
||||
if args.deploy_config:
|
||||
kwargs["deploy_config"] = args.deploy_config
|
||||
return kwargs
|
||||
|
||||
|
||||
def main(args) -> None:
|
||||
os.makedirs(args.output_dir, exist_ok=True)
|
||||
model_name = args.model or DEFAULT_MODEL
|
||||
prompts = [
|
||||
build_prompt(text, args.ref_audio, args.ref_text, model_name) for text in ([args.text] * args.num_prompts)
|
||||
]
|
||||
|
||||
omni = Omni(**_omni_kwargs(args, model_name))
|
||||
t_start = time.perf_counter()
|
||||
total_audio_seconds = 0.0
|
||||
for stage_outputs in omni.generate(prompts):
|
||||
request_output = stage_outputs
|
||||
if request_output is None or not request_output.outputs:
|
||||
continue
|
||||
audio, sample_rate = extract_audio(request_output.outputs[0].multimodal_output)
|
||||
assert audio.numel() > 0, "Audio8 TTS produced an empty waveform"
|
||||
total_audio_seconds += audio.numel() / sample_rate
|
||||
_save_wav(args.output_dir, request_output.request_id, audio, sample_rate)
|
||||
elapsed = time.perf_counter() - t_start
|
||||
rtf = elapsed / max(total_audio_seconds, 1e-6)
|
||||
print(f"Total {elapsed * 1000:.1f} ms for {total_audio_seconds:.2f} s of audio (RTF={rtf:.3f})")
|
||||
|
||||
|
||||
async def main_streaming(args) -> None:
|
||||
os.makedirs(args.output_dir, exist_ok=True)
|
||||
model_name = args.model or DEFAULT_MODEL
|
||||
prompt = build_prompt(args.text, args.ref_audio, args.ref_text, model_name)
|
||||
|
||||
omni = AsyncOmni(**_omni_kwargs(args, model_name))
|
||||
request_id = "0"
|
||||
chunks: list[torch.Tensor] = []
|
||||
sample_rate = None
|
||||
t_start = time.perf_counter()
|
||||
t_prev = t_start
|
||||
chunk_index = 0
|
||||
|
||||
async for stage_output in omni.generate(prompt, request_id=request_id):
|
||||
multimodal_output = stage_output.outputs[0].multimodal_output
|
||||
if not multimodal_output:
|
||||
continue
|
||||
audio = multimodal_output.get("model_outputs")
|
||||
if audio is None:
|
||||
audio = multimodal_output.get("audio")
|
||||
if audio is not None:
|
||||
if isinstance(audio, list):
|
||||
chunks.extend(torch.as_tensor(chunk).reshape(-1) for chunk in audio)
|
||||
else:
|
||||
chunks.append(torch.as_tensor(audio).reshape(-1))
|
||||
sr_raw = multimodal_output.get("sr")
|
||||
if isinstance(sr_raw, list):
|
||||
sr_raw = sr_raw[-1] if sr_raw else None
|
||||
if sr_raw is not None:
|
||||
sample_rate = int(sr_raw.item()) if hasattr(sr_raw, "item") else int(sr_raw)
|
||||
|
||||
if stage_output.finished:
|
||||
break
|
||||
now = time.perf_counter()
|
||||
if chunk_index == 0:
|
||||
print(f"chunk 0: TTFA={(now - t_start) * 1000:.1f} ms")
|
||||
else:
|
||||
print(f"chunk {chunk_index}: inter_chunk={(now - t_prev) * 1000:.1f} ms")
|
||||
t_prev = now
|
||||
chunk_index += 1
|
||||
|
||||
if not chunks or sample_rate is None:
|
||||
raise RuntimeError("Audio8 TTS streaming produced no audio")
|
||||
_save_wav(args.output_dir, request_id, torch.cat(chunks, dim=0), sample_rate)
|
||||
print(f"Streaming done: total={(time.perf_counter() - t_start) * 1000:.1f} ms chunks={chunk_index}")
|
||||
|
||||
|
||||
def parse_args():
|
||||
parser = TrackingArgumentParser(description="Audio8 TTS Preview offline inference with vLLM Omni")
|
||||
parser.add_argument("--model", type=str, default=DEFAULT_MODEL, help="Model path or HuggingFace repo ID.")
|
||||
parser.add_argument(
|
||||
"--text",
|
||||
type=str,
|
||||
default="Welcome to Audio8 TTS, a compact multilingual text to speech model.",
|
||||
help="Text to synthesize.",
|
||||
)
|
||||
parser.add_argument("--ref-audio", type=str, default=None, help="Reference audio path for voice cloning.")
|
||||
parser.add_argument(
|
||||
"--ref-text",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Transcript of --ref-audio; required for voice cloning and must match the recording.",
|
||||
)
|
||||
parser.add_argument("--output-dir", type=str, default="output_audio", help="Directory for output WAV files.")
|
||||
parser.add_argument(
|
||||
"--num-prompts",
|
||||
type=int,
|
||||
default=1,
|
||||
help="Duplicate the prompt N times to smoke-test concurrent requests.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--deploy-config",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Override the deploy config path; defaults to vllm_omni/deploy/audio8_tts.yaml via model_type.",
|
||||
)
|
||||
parser.add_argument("--streaming", action="store_true", help="Use AsyncOmni and report per-chunk latency.")
|
||||
parser.add_argument("--log-stats", action="store_true", help="Enable engine stats logging.")
|
||||
parser.add_argument("--stage-init-timeout", type=int, default=600, help="Per-stage init timeout in seconds.")
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parsed_args = parse_args()
|
||||
if parsed_args.streaming:
|
||||
asyncio.run(main_streaming(parsed_args))
|
||||
else:
|
||||
main(parsed_args)
|
||||
@@ -1,9 +1,12 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import argparse
|
||||
import functools
|
||||
import json
|
||||
import os
|
||||
import urllib.request
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
import soundfile as sf
|
||||
@@ -29,6 +32,20 @@ def _default_ref_audio() -> str:
|
||||
return str(dest)
|
||||
|
||||
|
||||
def parse_json_object(value: str, flag_name: str = "argument") -> dict[str, Any]:
|
||||
"""Parse a CLI value as a JSON object, attributing errors to ``flag_name``."""
|
||||
try:
|
||||
config = json.loads(value)
|
||||
except json.JSONDecodeError as e:
|
||||
raise argparse.ArgumentTypeError(f"{flag_name} must be valid JSON: {e}") from e
|
||||
if not isinstance(config, dict):
|
||||
raise argparse.ArgumentTypeError(f"{flag_name} must be a JSON object")
|
||||
return config
|
||||
|
||||
|
||||
parse_profiler_config = functools.partial(parse_json_object, flag_name="--profiler-config")
|
||||
|
||||
|
||||
def run_e2e():
|
||||
parser = argparse.ArgumentParser()
|
||||
# ""FunAudioLLM/Fun-CosyVoice3-0.5B-2512
|
||||
@@ -64,6 +81,12 @@ def run_e2e():
|
||||
required=True,
|
||||
help="Path to tokenizer directory (e.g., <model_path>/CosyVoice-BlankEN).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--profiler-config",
|
||||
type=parse_profiler_config,
|
||||
default=None,
|
||||
help='JSON profiler config for torch/cuda profiling, e.g. \'{"profiler":"torch","torch_profiler_dir":"./perf"}\'.',
|
||||
)
|
||||
args = parser.parse_args()
|
||||
# Ensure tokenizer directory exists
|
||||
if not os.path.exists(args.tokenizer):
|
||||
@@ -79,6 +102,7 @@ def run_e2e():
|
||||
deploy_config=args.deploy_config,
|
||||
tokenizer=args.tokenizer,
|
||||
log_stats=True,
|
||||
profiler_config=args.profiler_config,
|
||||
)
|
||||
|
||||
sampling_cfg = {"top_p": 0.8, "top_k": 25, "eos_token_id": 6561 + 1}
|
||||
@@ -149,8 +173,8 @@ def run_e2e():
|
||||
|
||||
sampling_params_list = [gpt_sampling, s2mel_sampling]
|
||||
|
||||
# Start profiling (requires VLLM_TORCH_PROFILER_DIR env var)
|
||||
if os.environ.get("VLLM_TORCH_PROFILER_DIR"):
|
||||
profiler_enabled = args.profiler_config is not None
|
||||
if profiler_enabled:
|
||||
print("Starting profiler...")
|
||||
omni.start_profile()
|
||||
|
||||
@@ -158,7 +182,7 @@ def run_e2e():
|
||||
outputs = list(omni.generate(prompts, sampling_params_list=sampling_params_list[:2]))
|
||||
|
||||
# Stop profiling and get results
|
||||
if os.environ.get("VLLM_TORCH_PROFILER_DIR"):
|
||||
if profiler_enabled:
|
||||
print("Stopping profiler...")
|
||||
profile_results = omni.stop_profile()
|
||||
print(f"Profile traces saved to: {profile_results}")
|
||||
|
||||
@@ -264,7 +264,7 @@ python examples/online_serving/openai_chat_completion_client_for_multimodal_gene
|
||||
|
||||
#### Realtime WebSocket client (`openai_realtime_client.py`)
|
||||
|
||||
[`openai_realtime_client.py`](./openai_realtime_client.py) connects to **`ws://<host>:<port>/v1/realtime`**, streams a local WAV as **PCM16 mono @ 16 kHz** in fixed-size chunks (OpenAI-style `input_audio_buffer.append` / `commit`), and receives **`response.audio.delta`** (incremental PCM for the reply) plus **`transcription.*`** events. By default it concatenates audio deltas and writes **`--output-wav`** (model output is typically **24 kHz**). Optional **`--delta-dump-dir`** saves each delta as `delta_000001.wav`, … for debugging.
|
||||
[`openai_realtime_client.py`](./openai_realtime_client.py) connects to **`ws://<host>:<port>/v1/realtime`**, streams a local WAV as **PCM16 mono @ 16 kHz** in fixed-size chunks, and receives **`response.audio.*`**, **`response.output_text.*`**, and transcript events. By default it uses explicit `input_audio_buffer.commit`; pass **`--server-vad`** to let the server detect speech endpoints and commit the turn automatically. The client concatenates audio deltas and writes **`--output-wav`** (model output is typically **24 kHz**). Optional **`--delta-dump-dir`** saves each delta as `delta_000001.wav`, … for debugging.
|
||||
|
||||
Streaming input works well for translation-style use cases; if the Thinker runs while input is still incomplete, consider limiting **`max_tokens`** in your session / server defaults to avoid over-generation.
|
||||
|
||||
@@ -282,6 +282,7 @@ python openai_realtime_client.py \
|
||||
--model Qwen/Qwen3-Omni-30B-A3B-Instruct \
|
||||
--input-wav /path/to/input_16k_mono.wav \
|
||||
--output-wav realtime_output.wav \
|
||||
--server-vad \
|
||||
--delta-dump-dir ./rt_delta_wavs
|
||||
```
|
||||
|
||||
@@ -297,15 +298,32 @@ python openai_realtime_client.py \
|
||||
| `--chunk-ms` | `200` | Size of each uploaded audio chunk (milliseconds of audio) |
|
||||
| `--send-delay-ms` | `0` | Delay between chunk sends (simulate realtime upload) |
|
||||
| `--delta-dump-dir` | *(optional)* | Directory to write per-`response.audio.delta` WAV files |
|
||||
| `--server-vad` | disabled | Enable server-side Silero VAD instead of sending explicit commits |
|
||||
| `--num-requests` | `1` | Number of sequential sessions (see `--concurrency`) |
|
||||
| `--concurrency` | `1` | Max concurrent WebSocket sessions when `--num-requests` > 1 |
|
||||
|
||||
Ensure the server is running, for example:
|
||||
Server VAD uses the pinned Silero v6.2 ONNX artifact from `istupakov/silero-vad-onnx` and never downloads model files while processing a session. Make the artifact available in the Hugging Face cache before startup, or configure a local artifact in the deployment YAML:
|
||||
|
||||
```yaml
|
||||
duplex_session:
|
||||
server_vad_model_path: /models/silero_vad.onnx
|
||||
```
|
||||
|
||||
Start from the bundled Qwen3-Omni deployment YAML, add the fields above at the top level, and serve that complete configuration:
|
||||
|
||||
```bash
|
||||
vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct --omni --port 8091
|
||||
cp vllm_omni/deploy/qwen3_omni_moe.yaml /path/to/qwen3_omni_server_vad.yaml
|
||||
# Edit /path/to/qwen3_omni_server_vad.yaml and add duplex_session.
|
||||
vllm serve Qwen/Qwen3-Omni-30B-A3B-Instruct \
|
||||
--omni \
|
||||
--port 8091 \
|
||||
--deploy-config /path/to/qwen3_omni_server_vad.yaml
|
||||
```
|
||||
|
||||
Keep Qwen's default `session_mode: turn`; `duplex_session` enables the Realtime handler without changing scheduler
|
||||
semantics. Bare `/v1/realtime` selects that handler. The client uses `?duplex=0` only for the existing non-Server-VAD
|
||||
wire flow; `?duplex=1` remains a supported compatibility alias.
|
||||
|
||||
The Python client supports the following command-line arguments:
|
||||
|
||||
- `--query-type` (or `-q`): Query type (default: `use_video`). Options: `text`, `use_audio`, `use_image`, `use_video`
|
||||
|
||||
@@ -1,9 +1,12 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
"""Realtime client for vLLM-Omni /v1/realtime (audio + text events).
|
||||
|
||||
This client:
|
||||
1) Reads a local WAV file (must be mono, 16-bit PCM, 16kHz),
|
||||
2) Streams PCM16 chunks to /v1/realtime with OpenAI-style events,
|
||||
3) Receives response.audio.* and transcription.* events,
|
||||
3) Receives response.audio.*, response.output_text.*, and transcript events,
|
||||
4) Saves synthesized audio to an output WAV file and optional text file.
|
||||
|
||||
By default each ``response.audio.delta`` is treated as an **incremental PCM**
|
||||
@@ -18,6 +21,7 @@ Usage:
|
||||
--model Qwen/Qwen3-Omni-30B-A3B-Instruct \
|
||||
--input-wav input_16k_mono.wav \
|
||||
--output-wav realtime_output.wav \
|
||||
--server-vad \
|
||||
--delta-dump-dir ./rt_delta_wavs
|
||||
|
||||
Dependencies:
|
||||
@@ -32,6 +36,7 @@ import base64
|
||||
import json
|
||||
import wave
|
||||
from pathlib import Path
|
||||
from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
|
||||
|
||||
try:
|
||||
import websockets
|
||||
@@ -70,6 +75,14 @@ def _write_wav_pcm16(path: Path, pcm16_bytes: bytes, sample_rate_hz: int) -> Non
|
||||
wf.writeframes(pcm16_bytes)
|
||||
|
||||
|
||||
def _with_realtime_route(url: str, *, server_vad: bool) -> str:
|
||||
parts = urlsplit(url)
|
||||
query = [(key, value) for key, value in parse_qsl(parts.query, keep_blank_values=True) if key != "duplex"]
|
||||
if not server_vad:
|
||||
query.append(("duplex", "0"))
|
||||
return urlunsplit(parts._replace(query=urlencode(query)))
|
||||
|
||||
|
||||
async def run_client(
|
||||
url: str,
|
||||
model: str,
|
||||
@@ -79,6 +92,7 @@ async def run_client(
|
||||
chunk_ms: int,
|
||||
send_delay_ms: int,
|
||||
delta_dump_dir: Path | None,
|
||||
server_vad: bool,
|
||||
request_idx: int = 1,
|
||||
total_requests: int = 1,
|
||||
) -> None:
|
||||
@@ -92,23 +106,44 @@ async def run_client(
|
||||
delta_index = 0
|
||||
text_chunks: list[str] = []
|
||||
final_text: str = ""
|
||||
final_audio_transcript: str = ""
|
||||
|
||||
if delta_dump_dir is not None:
|
||||
delta_dump_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Keep the CLI mode authoritative even if the input URL already contains
|
||||
# an explicit duplex query parameter.
|
||||
url = _with_realtime_route(url, server_vad=server_vad)
|
||||
async with websockets.connect(url, max_size=64 * 1024 * 1024) as ws:
|
||||
# 1) Validate model.
|
||||
await ws.send(
|
||||
json.dumps(
|
||||
{
|
||||
"type": "session.update",
|
||||
if server_vad:
|
||||
session_update = {
|
||||
"type": "session.update",
|
||||
"session": {
|
||||
"model": model,
|
||||
}
|
||||
)
|
||||
)
|
||||
"audio": {
|
||||
"input": {
|
||||
"format": {"type": "audio/pcm", "rate": 16_000},
|
||||
"turn_detection": {
|
||||
"type": "server_vad",
|
||||
"threshold": 0.5,
|
||||
"prefix_padding_ms": 300,
|
||||
"silence_duration_ms": 500,
|
||||
"create_response": True,
|
||||
"interrupt_response": False,
|
||||
},
|
||||
}
|
||||
},
|
||||
},
|
||||
}
|
||||
else:
|
||||
# Preserve the existing non-duplex wire format.
|
||||
session_update = {"type": "session.update", "model": model}
|
||||
await ws.send(json.dumps(session_update))
|
||||
|
||||
# 2) Start generation once (non-final commit).
|
||||
await ws.send(json.dumps({"type": "input_audio_buffer.commit", "final": False}))
|
||||
# 2) Preserve the legacy explicit-commit flow unless Server VAD owns endpointing.
|
||||
if not server_vad:
|
||||
await ws.send(json.dumps({"type": "input_audio_buffer.commit", "final": False}))
|
||||
|
||||
# 3) Stream audio chunks.
|
||||
for i in range(0, len(pcm16), chunk_bytes):
|
||||
@@ -124,8 +159,20 @@ async def run_client(
|
||||
if send_delay_ms > 0:
|
||||
await asyncio.sleep(send_delay_ms / 1000.0)
|
||||
|
||||
# 4) Final commit closes input stream.
|
||||
await ws.send(json.dumps({"type": "input_audio_buffer.commit", "final": True}))
|
||||
# 4) Finalize the turn explicitly, or append silence for Server VAD endpointing.
|
||||
if server_vad:
|
||||
trailing_silence = bytes(16_000 * 2)
|
||||
for i in range(0, len(trailing_silence), chunk_bytes):
|
||||
await ws.send(
|
||||
json.dumps(
|
||||
{
|
||||
"type": "input_audio_buffer.append",
|
||||
"audio": base64.b64encode(trailing_silence[i : i + chunk_bytes]).decode("utf-8"),
|
||||
}
|
||||
)
|
||||
)
|
||||
else:
|
||||
await ws.send(json.dumps({"type": "input_audio_buffer.commit", "final": True}))
|
||||
|
||||
# 5) Receive server events until audio done.
|
||||
while True:
|
||||
@@ -144,7 +191,7 @@ async def run_client(
|
||||
sr = event.get("sample_rate_hz")
|
||||
if isinstance(sr, int) and sr > 0:
|
||||
output_sample_rate = sr
|
||||
audio_b64 = event.get("audio", "")
|
||||
audio_b64 = event.get("delta" if server_vad else "audio", "")
|
||||
if audio_b64:
|
||||
pcm_delta = base64.b64decode(audio_b64)
|
||||
incremental_pcm_parts.append(pcm_delta)
|
||||
@@ -158,14 +205,14 @@ async def run_client(
|
||||
)
|
||||
continue
|
||||
|
||||
if event_type == "transcription.delta":
|
||||
if event_type in {"response.output_text.delta", "transcription.delta"}:
|
||||
delta = event.get("delta", "")
|
||||
if delta:
|
||||
text_chunks.append(delta)
|
||||
print(delta, end="", flush=True)
|
||||
continue
|
||||
|
||||
if event_type == "transcription.done":
|
||||
if event_type in {"response.output_text.done", "transcription.done"}:
|
||||
final_text = event.get("text", "") or "".join(text_chunks)
|
||||
usage = event.get("usage")
|
||||
final_text_with_tag = f"Final transcription: {final_text}"
|
||||
@@ -176,7 +223,16 @@ async def run_client(
|
||||
print(f"{log_prefix}text usage: {usage}")
|
||||
continue
|
||||
|
||||
if event_type == "response.audio_transcript.done":
|
||||
final_audio_transcript = event.get("transcript", "")
|
||||
continue
|
||||
|
||||
if event_type == "response.audio.done":
|
||||
if not server_vad:
|
||||
break
|
||||
continue
|
||||
|
||||
if event_type == "response.done" and server_vad:
|
||||
break
|
||||
|
||||
if event_type == "error":
|
||||
@@ -191,7 +247,7 @@ async def run_client(
|
||||
print(f"{log_prefix}Saved realtime audio to: {output_wav} (incremental chunks joined)")
|
||||
|
||||
if output_text is not None:
|
||||
text_to_save = final_text if final_text else "".join(text_chunks)
|
||||
text_to_save = final_text or "".join(text_chunks) or final_audio_transcript
|
||||
output_text.parent.mkdir(parents=True, exist_ok=True)
|
||||
output_text.write_text(text_to_save, encoding="utf-8")
|
||||
print(f"{log_prefix}Saved realtime text to: {output_text}")
|
||||
@@ -213,6 +269,7 @@ async def run_clients_concurrent(
|
||||
chunk_ms: int,
|
||||
send_delay_ms: int,
|
||||
delta_dump_dir: Path | None,
|
||||
server_vad: bool,
|
||||
num_requests: int,
|
||||
concurrency: int,
|
||||
) -> None:
|
||||
@@ -235,6 +292,7 @@ async def run_clients_concurrent(
|
||||
chunk_ms=chunk_ms,
|
||||
send_delay_ms=send_delay_ms,
|
||||
delta_dump_dir=per_delta_dir,
|
||||
server_vad=server_vad,
|
||||
request_idx=index,
|
||||
total_requests=num_requests,
|
||||
)
|
||||
@@ -283,6 +341,11 @@ def main() -> None:
|
||||
default=None,
|
||||
help="If set, each response.audio.delta is saved as delta_NNNNNN.wav under this directory",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--server-vad",
|
||||
action="store_true",
|
||||
help="Let the server detect speech endpoints instead of sending input_audio_buffer.commit",
|
||||
)
|
||||
parser.add_argument("--num-requests", type=int, default=1, help="Total number of requests to send")
|
||||
parser.add_argument(
|
||||
"--concurrency",
|
||||
@@ -309,6 +372,7 @@ def main() -> None:
|
||||
chunk_ms=args.chunk_ms,
|
||||
send_delay_ms=args.send_delay_ms,
|
||||
delta_dump_dir=args.delta_dump_dir,
|
||||
server_vad=args.server_vad,
|
||||
)
|
||||
)
|
||||
else:
|
||||
@@ -322,6 +386,7 @@ def main() -> None:
|
||||
chunk_ms=args.chunk_ms,
|
||||
send_delay_ms=args.send_delay_ms,
|
||||
delta_dump_dir=args.delta_dump_dir,
|
||||
server_vad=args.server_vad,
|
||||
num_requests=args.num_requests,
|
||||
concurrency=concurrency,
|
||||
)
|
||||
|
||||
@@ -14,6 +14,7 @@ For the full list of supported architectures across all modalities, see
|
||||
|
||||
| Model | HuggingFace repo | Voice cloning | Streaming | Voice presets / upload | Gradio demo |
|
||||
| --- | --- | --- | --- | --- | --- |
|
||||
| Audio8 TTS Preview | `Audio8/Audio8-TTS-Preview-0.6b` | ✓ (`ref_audio`+`ref_text`) | ✓ (PCM stream) | uploaded audio voice only; no presets | ✓ |
|
||||
| Fish Speech S2 Pro | `fishaudio/s2-pro` | ✓ (`ref_audio`+`ref_text`) | ✓ (PCM stream) | — | ✓ |
|
||||
| GLM-TTS | `zai-org/GLM-TTS` | ✓ (`ref_audio`+`ref_text`, required) | ✓ (PCM stream) | — | ✓ |
|
||||
| IndexTTS-2 | `IndexTeam/IndexTTS-2` | ✓ (`ref_audio` or uploaded `voice`) | `stream=true` response, non-chunk | uploaded audio voice only; no presets | — |
|
||||
@@ -253,16 +254,152 @@ python examples/online_serving/text_to_speech/indextts2/speech_client.py \
|
||||
|
||||
---
|
||||
|
||||
## Fish Speech S2 Pro
|
||||
## Audio8 TTS Preview
|
||||
|
||||
4B dual-AR TTS at 44.1 kHz. Server uses the DAC codec.
|
||||
0.6B DualAR TTS at 44.1 kHz, 11 languages, zero-shot voice cloning.
|
||||
|
||||
### Prerequisites
|
||||
|
||||
None beyond the base install: the neural audio codec is implemented in tree and
|
||||
its weights (`codec.pth`) ship with the checkpoint.
|
||||
|
||||
### Launch
|
||||
|
||||
```bash
|
||||
pip install fish-speech
|
||||
vllm serve Audio8/Audio8-TTS-Preview-0.6b --omni --port 8092
|
||||
# or:
|
||||
./audio8_tts/run_server.sh
|
||||
```
|
||||
|
||||
The deploy config auto-loads from `vllm_omni/deploy/audio8_tts.yaml` (HF
|
||||
`model_type` is `arktts`). Do **not** pass `--trust-remote-code`: vllm-omni
|
||||
registers its own `arktts` config, and transformers would otherwise prefer the
|
||||
checkpoint's remote code and bypass it.
|
||||
|
||||
### Text-only synthesis
|
||||
|
||||
```bash
|
||||
curl -X POST http://localhost:8092/v1/audio/speech \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"input": "Welcome to Audio8 TTS.",
|
||||
"response_format": "wav"
|
||||
}' --output output.wav
|
||||
```
|
||||
|
||||
### Voice cloning
|
||||
|
||||
```bash
|
||||
curl -X POST http://localhost:8092/v1/audio/speech \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{
|
||||
"input": "Welcome to Audio8 TTS.",
|
||||
"ref_audio": "https://example.com/reference.wav",
|
||||
"ref_text": "The exact transcript of the reference recording."
|
||||
}' --output cloned.wav
|
||||
```
|
||||
|
||||
`ref_audio` also accepts a `data:audio/wav;base64,...` URL. `ref_text` is
|
||||
mandatory whenever `ref_audio` is present and must match the recording.
|
||||
Uploading a voice via `POST /v1/audio/voices` lets the server reuse the encoded
|
||||
reference codes across requests (`voice: "<name>"`).
|
||||
|
||||
### Streaming
|
||||
|
||||
```bash
|
||||
python audio8_tts/speech_client.py --text "Welcome to Audio8 TTS." --stream --output out.pcm
|
||||
ffplay -f s16le -ar 44100 -ac 1 out.pcm
|
||||
```
|
||||
|
||||
### Gradio demo
|
||||
|
||||
```bash
|
||||
./audio8_tts/run_gradio_demo.sh # server + demo
|
||||
python audio8_tts/gradio_demo.py --api-base http://localhost:8092 # demo only
|
||||
```
|
||||
|
||||
### Benchmark
|
||||
|
||||
```bash
|
||||
export VLLM_OMNI_BENCH_AUDIO_SAMPLE_RATE=44100 # required: harness defaults to 24 kHz
|
||||
vllm bench serve --omni --host 127.0.0.1 --port 8092 \
|
||||
--model Audio8/Audio8-TTS-Preview-0.6b \
|
||||
--backend openai-audio-speech --endpoint /v1/audio/speech \
|
||||
--dataset-name seed-tts-text --dataset-path benchmarks/build_dataset/seed_tts_smoke \
|
||||
--seed-tts-locale en --num-prompts 20 --num-warmups 2 \
|
||||
--max-concurrency 1 --request-rate inf \
|
||||
--percentile-metrics ttft,e2el,audio_rtf,audio_ttfp,audio_duration,audio_underrun
|
||||
```
|
||||
|
||||
Measured on 1x H20 **shared with an unrelated job holding ~80% SM**, so treat
|
||||
these as a lower bound. 20 prompts from `seed_tts_smoke/en`, mean audio 4.5 s,
|
||||
deploy defaults:
|
||||
|
||||
| | HF reference (batch=1) | vllm-omni c=1 | c=4 | c=8 |
|
||||
| --- | --- | --- | --- | --- |
|
||||
| RTF (mean) | 1.008 | **0.19** | 0.30 | 0.54 |
|
||||
| Throughput (req/s) | 0.23 | 1.18 | 2.84 | **2.93** |
|
||||
| E2E latency (mean, ms) | 4375 | 850 | 1349 | 2361 |
|
||||
| Time to first packet (mean, ms) | n/a (no streaming) | 63 | 105 | 1124 |
|
||||
| Underrun p99 / continuity OK | n/a | 0 s / 100% | 0 s / 100% | 0 s / 100% |
|
||||
|
||||
~5.3x lower per-request RTF than the upstream reference implementation at
|
||||
concurrency 1, ~12.8x aggregate throughput at concurrency 8.
|
||||
|
||||
#### Stage-0 `max_num_seqs`: first-packet vs throughput
|
||||
|
||||
This subsection is operator tuning guidance, not a claim this PR optimizes
|
||||
anything: the shipped deploy default is unchanged (`max_num_seqs: 4`). The
|
||||
numbers below are a single run on the same contended H20 as above (one unrelated
|
||||
job holding ~80% SM, no repeats), so read them as directional, not as measured
|
||||
speedups. Same setup, varying stage 0's `max_num_seqs` (stage 1 stays at 1):
|
||||
|
||||
| stage-0 `max_num_seqs` | client concurrency | req/s | RTF | TTFP mean / p99 (ms) | underrun p99 (s) |
|
||||
| --- | --- | --- | --- | --- | --- |
|
||||
| 4 (default) | 1 | 1.18 | 0.19 | 63 / 67 | 0.00 |
|
||||
| 4 | 4 | 2.84 | 0.30 | 105 / 148 | 0.00 |
|
||||
| 4 | 8 | 2.93 | 0.54 | 1124 / 1642 | 0.00 |
|
||||
| 8 | 1 | 1.16 | 0.19 | 64 / 70 | 0.00 |
|
||||
| 8 | 4 | 2.88 | 0.30 | 107 / 148 | 0.00 |
|
||||
| 8 | 8 | **3.72** | 0.40 | **247** / 733 | 0.22 |
|
||||
| 8 | 16 | 3.91 | 0.65 | 1366 / 2530 | 0.29 |
|
||||
|
||||
Reading of this:
|
||||
|
||||
- Below the limit (`concurrency <= max_num_seqs`) the setting is irrelevant, as
|
||||
expected — c=1 and c=4 are identical for both.
|
||||
- At c=8, raising the limit to 8 admits every request instead of queueing half
|
||||
of them: in this single run, roughly +27% throughput and ~4.5x lower
|
||||
first-packet latency.
|
||||
- Throughput saturates around 3.7-3.9 req/s (~17 s of audio per second). Past
|
||||
c=8 you only buy queueing delay.
|
||||
- The cost of admitting more streams is a 0.22-0.29 s worst-case buffer deficit
|
||||
in 1 of 20 requests (`Streaming continuity OK rate` 95%), because stage 1 runs
|
||||
`max_num_seqs: 1` and the codec window round-robins across streams. Client-side
|
||||
prebuffering (the bundled Gradio demo holds 2 chunks) absorbs that; raising
|
||||
stage 1's `max_num_seqs` is the untested knob if you need both.
|
||||
|
||||
The shipped default stays at 4 because every measured point there had zero
|
||||
underrun. Raise it to 8 if first-packet latency under load matters more than a
|
||||
rare sub-300 ms gap.
|
||||
|
||||
### Notes
|
||||
|
||||
- Output: 44.1 kHz mono; ~21.5 codec frames per second.
|
||||
- No built-in speaker presets. Omit `voice` for a random timbre, or clone one.
|
||||
- `max_new_tokens` caps generated codec frames (1 frame ~= 46 ms).
|
||||
- Context is 2048 packed text+audio positions (`max_model_len: 2048`).
|
||||
- Do not send `temperature: 0`: greedy decoding is degenerate for this
|
||||
checkpoint and truncates the utterance. The deploy defaults
|
||||
(0.7 / top_k 50 / top_p 0.9) mirror `generation_config.json`.
|
||||
|
||||
---
|
||||
|
||||
## Fish Speech S2 Pro
|
||||
|
||||
4B dual-AR TTS at 44.1 kHz. Server uses the DAC codec, which is vendored in
|
||||
vLLM-Omni (no extra packages required).
|
||||
|
||||
### Kvcache attention fast path
|
||||
|
||||
Fish Speech S2 Pro uses a Triton decode-only kvcache attention fast path by
|
||||
@@ -708,13 +845,21 @@ Raw PCM streaming requires `stream_format="audio"`, `response_format="pcm"`, and
|
||||
|
||||
### Streaming WebSocket
|
||||
|
||||
The `/v1/audio/speech/stream` endpoint accepts text incrementally and synthesizes the buffered text as one continuous request on `input.done`:
|
||||
The `/v1/audio/speech/stream` endpoint accepts text incrementally and, by default, synthesizes the buffered text as one continuous request on `input.done`:
|
||||
|
||||
```bash
|
||||
python qwen3_tts/streaming_speech_client.py --text "Hello world. How are you? I am fine."
|
||||
python qwen3_tts/streaming_speech_client.py --text "..." --simulate-stt --stt-delay 0.1
|
||||
```
|
||||
|
||||
For per-sentence audio (including Indic danda `।`) before `input.done`:
|
||||
|
||||
```bash
|
||||
python qwen3_tts/streaming_speech_client.py \
|
||||
--text "नमस्ते। कैसे हो?" \
|
||||
--split-granularity sentence
|
||||
```
|
||||
|
||||
`input.done` flushes without closing, so repeating `--text` synthesizes several utterances over one connection:
|
||||
|
||||
```bash
|
||||
|
||||
@@ -0,0 +1,258 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
"""Gradio demo for Audio8 TTS Preview via the /v1/audio/speech API.
|
||||
|
||||
Supports:
|
||||
- text-to-speech synthesis in 11 languages
|
||||
- zero-shot voice cloning from reference audio (upload, microphone, or URL)
|
||||
- streaming (progressive PCM chunks) and non-streaming modes
|
||||
|
||||
Usage:
|
||||
# Start the server first (see run_server.sh), then:
|
||||
python gradio_demo.py --api-base http://localhost:8092
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import base64
|
||||
import io
|
||||
|
||||
try:
|
||||
import gradio as gr
|
||||
except ImportError:
|
||||
raise ImportError("gradio is required to run this demo. Install it with: pip install 'vllm-omni[demo]'") from None
|
||||
import httpx
|
||||
import numpy as np
|
||||
import soundfile as sf
|
||||
|
||||
#: Audio8 TTS decodes at 44.1 kHz; raw PCM must be interpreted at that rate.
|
||||
PCM_SAMPLE_RATE = 44100
|
||||
DEFAULT_API_BASE = "http://localhost:8092"
|
||||
|
||||
|
||||
def encode_audio_to_base64(audio_data: tuple) -> str:
|
||||
"""Encode a Gradio ``(sample_rate, numpy_array)`` input as a data URL."""
|
||||
sample_rate, audio_np = audio_data
|
||||
if audio_np.dtype != np.int16:
|
||||
if audio_np.dtype in (np.float32, np.float64):
|
||||
audio_np = (np.clip(audio_np, -1.0, 1.0) * 32767).astype(np.int16)
|
||||
else:
|
||||
audio_np = audio_np.astype(np.int16)
|
||||
buf = io.BytesIO()
|
||||
sf.write(buf, audio_np, sample_rate, format="WAV")
|
||||
return f"data:audio/wav;base64,{base64.b64encode(buf.getvalue()).decode('utf-8')}"
|
||||
|
||||
|
||||
def build_payload(
|
||||
text: str,
|
||||
ref_audio: tuple | None,
|
||||
ref_audio_url: str,
|
||||
ref_text: str,
|
||||
response_format: str = "wav",
|
||||
stream: bool = False,
|
||||
) -> dict:
|
||||
"""Build the ``/v1/audio/speech`` payload.
|
||||
|
||||
Audio8 TTS has no built-in speakers: omit ``voice`` for a random timbre, or
|
||||
supply ``ref_audio`` + ``ref_text`` to clone a voice.
|
||||
"""
|
||||
if not text or not text.strip():
|
||||
raise gr.Error("Please enter text to synthesize.")
|
||||
|
||||
payload: dict = {
|
||||
"input": text.strip(),
|
||||
"response_format": "pcm" if stream else response_format,
|
||||
"stream": stream,
|
||||
}
|
||||
if stream:
|
||||
payload["stream_format"] = "audio"
|
||||
|
||||
ref_url = ref_audio_url.strip() if ref_audio_url else ""
|
||||
if ref_audio is not None:
|
||||
payload["ref_audio"] = encode_audio_to_base64(ref_audio)
|
||||
elif ref_url:
|
||||
payload["ref_audio"] = ref_url
|
||||
|
||||
if "ref_audio" in payload:
|
||||
if not ref_text or not ref_text.strip():
|
||||
raise gr.Error("Voice cloning requires the transcript of the reference audio.")
|
||||
payload["ref_text"] = ref_text.strip()
|
||||
|
||||
return payload
|
||||
|
||||
|
||||
def generate_speech(api_base: str, text: str, ref_audio, ref_audio_url, ref_text, response_format):
|
||||
"""Non-streaming request; returns the whole waveform."""
|
||||
payload = build_payload(text, ref_audio, ref_audio_url, ref_text, response_format, stream=False)
|
||||
try:
|
||||
with httpx.Client(timeout=300.0) as client:
|
||||
resp = client.post(
|
||||
f"{api_base}/v1/audio/speech",
|
||||
json=payload,
|
||||
headers={"Content-Type": "application/json", "Authorization": "Bearer EMPTY"},
|
||||
)
|
||||
except httpx.TimeoutException:
|
||||
raise gr.Error("Request timed out. The server may be busy.") from None
|
||||
except httpx.ConnectError:
|
||||
raise gr.Error(f"Cannot connect to server at {api_base}. Is the server running?") from None
|
||||
|
||||
if resp.status_code != 200:
|
||||
raise gr.Error(f"Server error ({resp.status_code}): {resp.text}")
|
||||
if "application/json" in resp.headers.get("content-type", ""):
|
||||
raise gr.Error(f"Server error: {resp.text}")
|
||||
|
||||
try:
|
||||
if response_format == "pcm":
|
||||
audio_np = np.frombuffer(resp.content, dtype=np.int16).astype(np.float32) / 32767.0
|
||||
return (PCM_SAMPLE_RATE, audio_np)
|
||||
audio_np, sample_rate = sf.read(io.BytesIO(resp.content))
|
||||
if audio_np.ndim > 1:
|
||||
audio_np = audio_np[:, 0]
|
||||
return (sample_rate, audio_np.astype(np.float32))
|
||||
except Exception as e:
|
||||
raise gr.Error(f"Failed to decode audio: {e}") from None
|
||||
|
||||
|
||||
def generate_speech_stream(api_base, text, ref_audio, ref_audio_url, ref_text, response_format, chunk_seconds=0.5):
|
||||
"""Streaming request; yields PCM chunks for progressive playback."""
|
||||
payload = build_payload(text, ref_audio, ref_audio_url, ref_text, response_format, stream=True)
|
||||
min_chunk_samples = int(PCM_SAMPLE_RATE * chunk_seconds)
|
||||
prebuffer_chunks = 2
|
||||
pending: list[np.ndarray] = []
|
||||
pending_len = 0
|
||||
chunks_yielded = 0
|
||||
prebuffer: list[np.ndarray] = []
|
||||
leftover = b""
|
||||
|
||||
try:
|
||||
with (
|
||||
httpx.Client(timeout=300.0) as client,
|
||||
client.stream(
|
||||
"POST",
|
||||
f"{api_base}/v1/audio/speech",
|
||||
json=payload,
|
||||
headers={"Content-Type": "application/json", "Authorization": "Bearer EMPTY"},
|
||||
) as resp,
|
||||
):
|
||||
if resp.status_code != 200:
|
||||
resp.read()
|
||||
raise gr.Error(f"Server error ({resp.status_code}): {resp.text}")
|
||||
for chunk in resp.iter_bytes():
|
||||
if not chunk:
|
||||
continue
|
||||
raw = leftover + chunk
|
||||
# s16le: never split a sample across chunk boundaries.
|
||||
usable = len(raw) - (len(raw) % 2)
|
||||
leftover = raw[usable:]
|
||||
if usable == 0:
|
||||
continue
|
||||
samples = np.frombuffer(raw[:usable], dtype=np.int16).copy().astype(np.float32) / 32767.0
|
||||
pending.append(samples)
|
||||
pending_len += len(samples)
|
||||
if pending_len < min_chunk_samples:
|
||||
continue
|
||||
audio_chunk = np.concatenate(pending)
|
||||
pending.clear()
|
||||
pending_len = 0
|
||||
if chunks_yielded < prebuffer_chunks:
|
||||
prebuffer.append(audio_chunk)
|
||||
chunks_yielded += 1
|
||||
if chunks_yielded == prebuffer_chunks:
|
||||
yield (PCM_SAMPLE_RATE, np.concatenate(prebuffer))
|
||||
prebuffer.clear()
|
||||
else:
|
||||
yield (PCM_SAMPLE_RATE, audio_chunk)
|
||||
remaining = prebuffer + pending
|
||||
if remaining:
|
||||
yield (PCM_SAMPLE_RATE, np.concatenate(remaining))
|
||||
except httpx.TimeoutException:
|
||||
raise gr.Error("Request timed out. The server may be busy.") from None
|
||||
except httpx.ConnectError:
|
||||
raise gr.Error(f"Cannot connect to server at {api_base}. Is the server running?") from None
|
||||
|
||||
|
||||
def on_stream_change(stream: bool):
|
||||
"""Streaming only makes sense with raw PCM."""
|
||||
if stream:
|
||||
return gr.update(value="pcm", interactive=False)
|
||||
return gr.update(value="wav", interactive=True)
|
||||
|
||||
|
||||
def build_interface(api_base: str, stream_chunk_seconds: float = 0.5):
|
||||
with gr.Blocks(title="Audio8 TTS Preview Demo") as demo:
|
||||
gr.Markdown("# Audio8 TTS Preview 0.6B - Text to Speech")
|
||||
gr.Markdown(f"**Server:** `{api_base}` | **Model:** Audio8/Audio8-TTS-Preview-0.6b | **Output:** 44.1 kHz")
|
||||
|
||||
with gr.Row():
|
||||
with gr.Column(scale=3):
|
||||
text_input = gr.Textbox(label="Text to Synthesize", placeholder="Enter text here...", lines=4)
|
||||
|
||||
with gr.Accordion("Voice Cloning (optional)", open=False):
|
||||
gr.Markdown(
|
||||
"Upload or link a short reference recording (5-30 s) and paste its **exact** transcript. "
|
||||
"A transcript that does not match the audio degrades both stability and speaker similarity."
|
||||
)
|
||||
ref_audio = gr.Audio(label="Reference Audio", type="numpy", sources=["upload", "microphone"])
|
||||
ref_audio_url = gr.Textbox(
|
||||
label="Reference Audio URL (alternative to upload)",
|
||||
placeholder="https://example.com/reference.wav",
|
||||
lines=1,
|
||||
)
|
||||
ref_text = gr.Textbox(
|
||||
label="Reference Audio Transcript (required for cloning)",
|
||||
placeholder="Exact transcript of the reference audio...",
|
||||
lines=2,
|
||||
)
|
||||
|
||||
with gr.Row():
|
||||
response_format = gr.Dropdown(
|
||||
choices=["wav", "mp3", "flac", "pcm"], value="wav", label="Audio Format", scale=1
|
||||
)
|
||||
stream_checkbox = gr.Checkbox(
|
||||
label="Stream output", value=False, info="Progressive PCM streaming", scale=1
|
||||
)
|
||||
|
||||
generate_btn = gr.Button("Generate Speech", variant="primary", size="lg")
|
||||
|
||||
with gr.Column(scale=2):
|
||||
audio_output = gr.Audio(label="Generated Audio", interactive=False, streaming=True, autoplay=True)
|
||||
gr.Markdown(
|
||||
"### About\n"
|
||||
"- **Audio8 TTS Preview** — 0.6B DualAR model (slow AR + fast AR codebook decoder)\n"
|
||||
"- **Languages**: Cantonese, Chinese, Dutch, English, French, German, "
|
||||
"Italian, Japanese, Korean, Polish, Spanish\n"
|
||||
"- **Voice cloning**: zero-shot from a reference clip + transcript\n"
|
||||
"- **44.1 kHz** output, ~21.5 codec frames per second"
|
||||
)
|
||||
|
||||
stream_checkbox.change(fn=on_stream_change, inputs=[stream_checkbox], outputs=[response_format])
|
||||
all_inputs = [text_input, ref_audio, ref_audio_url, ref_text, response_format]
|
||||
|
||||
def dispatch(stream_enabled, *args):
|
||||
if stream_enabled:
|
||||
yield from generate_speech_stream(api_base, *args, chunk_seconds=stream_chunk_seconds)
|
||||
else:
|
||||
yield generate_speech(api_base, *args)
|
||||
|
||||
generate_btn.click(fn=dispatch, inputs=[stream_checkbox] + all_inputs, outputs=[audio_output])
|
||||
demo.queue()
|
||||
return demo
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Gradio demo for Audio8 TTS Preview")
|
||||
parser.add_argument("--api-base", default=DEFAULT_API_BASE, help=f"API base URL (default: {DEFAULT_API_BASE})")
|
||||
parser.add_argument("--host", default="0.0.0.0", help="Gradio host")
|
||||
parser.add_argument("--port", type=int, default=7861, help="Gradio port")
|
||||
parser.add_argument("--share", action="store_true", help="Create a public share link")
|
||||
parser.add_argument("--stream-chunk-seconds", type=float, default=0.5, help="Seconds of audio per yielded chunk")
|
||||
args = parser.parse_args()
|
||||
|
||||
print(f"Connecting to vLLM server at: {args.api_base}")
|
||||
build_interface(args.api_base, stream_chunk_seconds=args.stream_chunk_seconds).launch(
|
||||
server_name=args.host, server_port=args.port, share=args.share
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,44 @@
|
||||
#!/bin/bash
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
# Launch the Audio8 TTS Preview server + Gradio demo together.
|
||||
#
|
||||
# Usage:
|
||||
# ./run_gradio_demo.sh
|
||||
# CUDA_VISIBLE_DEVICES=0 PORT=8092 GRADIO_PORT=7861 ./run_gradio_demo.sh
|
||||
|
||||
set -e
|
||||
|
||||
MODEL="${MODEL:-Audio8/Audio8-TTS-Preview-0.6b}"
|
||||
PORT="${PORT:-8092}"
|
||||
GRADIO_PORT="${GRADIO_PORT:-7861}"
|
||||
SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
|
||||
echo "Starting Audio8 TTS Preview server (port $PORT)..."
|
||||
vllm serve "$MODEL" \
|
||||
--omni \
|
||||
--host 0.0.0.0 \
|
||||
--port "$PORT" &
|
||||
SERVER_PID=$!
|
||||
|
||||
cleanup() {
|
||||
echo "Stopping server (PID $SERVER_PID)..."
|
||||
kill $SERVER_PID 2>/dev/null
|
||||
wait $SERVER_PID 2>/dev/null
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
echo "Waiting for server to start..."
|
||||
for _ in $(seq 1 120); do
|
||||
if curl -s "http://localhost:$PORT/health" > /dev/null 2>&1; then
|
||||
echo "Server ready."
|
||||
break
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
|
||||
echo "Starting Gradio demo (port $GRADIO_PORT)..."
|
||||
python "$SCRIPT_DIR/gradio_demo.py" \
|
||||
--api-base "http://localhost:$PORT" \
|
||||
--port "$GRADIO_PORT"
|
||||
@@ -0,0 +1,24 @@
|
||||
#!/bin/bash
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
# Launch a vLLM-Omni server for Audio8 TTS Preview 0.6B.
|
||||
#
|
||||
# Usage:
|
||||
# ./run_server.sh
|
||||
# CUDA_VISIBLE_DEVICES=0 PORT=8092 ./run_server.sh
|
||||
|
||||
set -e
|
||||
|
||||
MODEL="${MODEL:-Audio8/Audio8-TTS-Preview-0.6b}"
|
||||
PORT="${PORT:-8092}"
|
||||
|
||||
echo "Starting Audio8 TTS Preview server with model: $MODEL"
|
||||
|
||||
# --trust-remote-code is deliberately NOT passed: vllm-omni registers its own
|
||||
# `arktts` config, and transformers would otherwise prefer the checkpoint's
|
||||
# remote code and bypass it.
|
||||
vllm serve "$MODEL" \
|
||||
--omni \
|
||||
--host 0.0.0.0 \
|
||||
--port "$PORT"
|
||||
@@ -0,0 +1,131 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
"""Client for Audio8 TTS Preview via the /v1/audio/speech endpoint.
|
||||
|
||||
Examples:
|
||||
# Basic TTS
|
||||
python speech_client.py --text "Welcome to Audio8 TTS."
|
||||
|
||||
# Zero-shot voice cloning (the transcript must match the reference audio)
|
||||
python speech_client.py --text "Welcome to Audio8 TTS." \
|
||||
--ref-audio ref.wav --ref-text "The exact transcript of the reference recording."
|
||||
|
||||
# Streaming PCM output
|
||||
python speech_client.py --text "Welcome to Audio8 TTS." --stream --output output.pcm
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import base64
|
||||
import os
|
||||
|
||||
import httpx
|
||||
|
||||
DEFAULT_API_BASE = "http://localhost:8092"
|
||||
DEFAULT_API_KEY = "EMPTY"
|
||||
DEFAULT_MODEL = "Audio8/Audio8-TTS-Preview-0.6b"
|
||||
#: The codec runs at 44.1 kHz, so raw PCM must be played back at that rate.
|
||||
PCM_SAMPLE_RATE = 44100
|
||||
|
||||
|
||||
def encode_audio_to_base64(audio_path: str) -> str:
|
||||
"""Encode a local audio file as a base64 data URL."""
|
||||
if not os.path.exists(audio_path):
|
||||
raise FileNotFoundError(f"Audio file not found: {audio_path}")
|
||||
ext = audio_path.lower().rsplit(".", 1)[-1]
|
||||
mime_type = {"wav": "audio/wav", "mp3": "audio/mpeg", "flac": "audio/flac", "ogg": "audio/ogg"}.get(
|
||||
ext, "audio/wav"
|
||||
)
|
||||
with open(audio_path, "rb") as handle:
|
||||
audio_b64 = base64.b64encode(handle.read()).decode("utf-8")
|
||||
return f"data:{mime_type};base64,{audio_b64}"
|
||||
|
||||
|
||||
def build_payload(args) -> dict:
|
||||
payload = {
|
||||
"model": args.model,
|
||||
"input": args.text,
|
||||
"response_format": args.response_format,
|
||||
}
|
||||
if args.ref_audio:
|
||||
if args.ref_audio.startswith(("http://", "https://")):
|
||||
payload["ref_audio"] = args.ref_audio
|
||||
else:
|
||||
payload["ref_audio"] = encode_audio_to_base64(args.ref_audio)
|
||||
if args.ref_text:
|
||||
payload["ref_text"] = args.ref_text
|
||||
if args.max_new_tokens is not None:
|
||||
payload["max_new_tokens"] = args.max_new_tokens
|
||||
if args.stream:
|
||||
payload["stream"] = True
|
||||
payload["stream_format"] = "audio"
|
||||
payload["response_format"] = "pcm"
|
||||
return payload
|
||||
|
||||
|
||||
def run_tts(args) -> None:
|
||||
payload = build_payload(args)
|
||||
api_url = f"{args.api_base}/v1/audio/speech"
|
||||
headers = {"Content-Type": "application/json", "Authorization": f"Bearer {args.api_key}"}
|
||||
|
||||
print(f"Model: {args.model}")
|
||||
print(f"Text: {args.text}")
|
||||
if args.ref_audio:
|
||||
print(f"Voice cloning: ref_audio={args.ref_audio!r} ref_text={args.ref_text!r}")
|
||||
|
||||
if args.stream:
|
||||
output_path = args.output or "output.pcm"
|
||||
with (
|
||||
httpx.Client(timeout=300.0) as client,
|
||||
client.stream("POST", api_url, json=payload, headers=headers) as resp,
|
||||
):
|
||||
if resp.status_code != 200:
|
||||
print(f"Error {resp.status_code}: {resp.read().decode()}")
|
||||
return
|
||||
total_bytes = 0
|
||||
with open(output_path, "wb") as handle:
|
||||
for chunk in resp.iter_bytes():
|
||||
handle.write(chunk)
|
||||
total_bytes += len(chunk)
|
||||
seconds = total_bytes / 2 / PCM_SAMPLE_RATE
|
||||
print(f"Streamed {total_bytes} bytes (~{seconds:.2f}s of s16le @ {PCM_SAMPLE_RATE} Hz) to {output_path}")
|
||||
print(f"Play with: ffplay -f s16le -ar {PCM_SAMPLE_RATE} -ac 1 {output_path}")
|
||||
return
|
||||
|
||||
with httpx.Client(timeout=300.0) as client:
|
||||
response = client.post(api_url, json=payload, headers=headers)
|
||||
if response.status_code != 200:
|
||||
print(f"Error {response.status_code}: {response.text}")
|
||||
return
|
||||
if response.headers.get("content-type", "").startswith("application/json"):
|
||||
print(f"Error: {response.text}")
|
||||
return
|
||||
|
||||
output_path = args.output or f"output.{args.response_format}"
|
||||
with open(output_path, "wb") as handle:
|
||||
handle.write(response.content)
|
||||
print(f"Audio saved to: {output_path} ({len(response.content)} bytes)")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Audio8 TTS Preview client")
|
||||
parser.add_argument("--api-base", default=DEFAULT_API_BASE, help="API base URL")
|
||||
parser.add_argument("--api-key", default=DEFAULT_API_KEY, help="API key")
|
||||
parser.add_argument("--model", "-m", default=DEFAULT_MODEL, help="Model name")
|
||||
parser.add_argument("--text", required=True, help="Text to synthesize")
|
||||
parser.add_argument("--ref-audio", default=None, help="Reference audio for voice cloning (path or URL)")
|
||||
parser.add_argument("--ref-text", default=None, help="Transcript of the reference audio")
|
||||
parser.add_argument("--max-new-tokens", type=int, default=None, help="Cap the number of generated codec frames")
|
||||
parser.add_argument("--stream", action="store_true", help="Stream PCM instead of returning a whole file")
|
||||
parser.add_argument(
|
||||
"--response-format",
|
||||
default="wav",
|
||||
choices=["wav", "mp3", "flac", "pcm", "aac", "opus"],
|
||||
help="Audio format (default: wav)",
|
||||
)
|
||||
parser.add_argument("--output", "-o", default=None, help="Output file path")
|
||||
run_tts(parser.parse_args())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -24,6 +24,11 @@ Usage:
|
||||
--text "Hello world. How are you? I am fine." \
|
||||
--simulate-stt --stt-delay 0.1
|
||||
|
||||
# Opt into per-sentence synthesis (Indic danda, CJK, Latin)
|
||||
python streaming_speech_client.py \
|
||||
--text "नमस्ते। कैसे हो?" \
|
||||
--split-granularity sentence
|
||||
|
||||
# Receive JSON sidecar chunks with word-level timestamps
|
||||
python streaming_speech_client.py \
|
||||
--text "Hello world. How are you?" \
|
||||
@@ -73,8 +78,9 @@ except ImportError:
|
||||
def frame_basename(msg: dict) -> str:
|
||||
"""Name a file after the utterance and sentence a frame belongs to.
|
||||
|
||||
Every utterance is sentence 0, so the utterance index is what keeps the
|
||||
files of one connection from overwriting each other.
|
||||
Every utterance uses `sentence_index` for units inside that flush.
|
||||
With default `split_granularity=none` that index stays 0; sentence mode
|
||||
uses 0, 1, ... so files from one connection do not overwrite each other.
|
||||
"""
|
||||
return f"utterance_{msg['utterance_index']:03d}_sentence_{msg['sentence_index']:03d}"
|
||||
|
||||
@@ -368,6 +374,13 @@ def main():
|
||||
)
|
||||
parser.add_argument("--speed", type=float, default=1.0, help="Playback speed (0.25-4.0)")
|
||||
parser.add_argument("--max-new-tokens", type=int, default=None, help="Max tokens")
|
||||
parser.add_argument("--seed", type=int, default=None, help="Sampling seed forwarded to the server")
|
||||
parser.add_argument(
|
||||
"--split-granularity",
|
||||
default=None,
|
||||
choices=["none", "sentence", "clause"],
|
||||
help="Text split mode (default: server none = one request per input.done)",
|
||||
)
|
||||
|
||||
# Base task options
|
||||
parser.add_argument("--ref-audio", default=None, help="Reference audio")
|
||||
@@ -409,6 +422,8 @@ def main():
|
||||
"max_new_tokens",
|
||||
"ref_audio",
|
||||
"ref_text",
|
||||
"seed",
|
||||
"split_granularity",
|
||||
]:
|
||||
val = getattr(args, key.replace("-", "_"), None)
|
||||
if val is not None:
|
||||
|
||||
Reference in new issue
Block a user