From d9e745ce2c562be06913cf27c3c9942a56154b93 Mon Sep 17 00:00:00 2001 From: Yueqian Lin <70319226+linyueqian@users.noreply.github.com> Date: Mon, 13 Apr 2026 02:30:56 -0400 Subject: [PATCH] [Fix] VoxCPM2: support raw audio for voice cloning via OpenAI API (#2720) Signed-off-by: Yueqian Lin --- examples/online_serving/voxcpm2/README.md | 42 ++++++ .../voxcpm2/openai_speech_client.py | 108 +++++++++++++++ .../models/voxcpm2/voxcpm2_talker.py | 130 +++++++++++++++++- 3 files changed, 277 insertions(+), 3 deletions(-) create mode 100644 examples/online_serving/voxcpm2/README.md create mode 100644 examples/online_serving/voxcpm2/openai_speech_client.py diff --git a/examples/online_serving/voxcpm2/README.md b/examples/online_serving/voxcpm2/README.md new file mode 100644 index 000000000..8735180f0 --- /dev/null +++ b/examples/online_serving/voxcpm2/README.md @@ -0,0 +1,42 @@ +# VoxCPM2 Online Serving + +Serve VoxCPM2 TTS via the OpenAI-compatible `/v1/audio/speech` endpoint. + +## Start the Server + +```bash +python -m vllm_omni.entrypoints.openai.api_server \ + --model openbmb/VoxCPM2 \ + --stage-configs-path vllm_omni/model_executor/stage_configs/voxcpm2.yaml \ + --host 0.0.0.0 --port 8000 +``` + +## Zero-shot Synthesis + +```bash +python openai_speech_client.py --text "Hello, this is VoxCPM2." +``` + +Or with curl: + +```bash +curl -X POST http://localhost:8000/v1/audio/speech \ + -H "Content-Type: application/json" \ + -d '{"model": "voxcpm2", "input": "Hello, this is VoxCPM2.", "voice": "default"}' \ + --output output.wav +``` + +## Voice Cloning + +Clone a speaker's voice using a reference audio file: + +```bash +python openai_speech_client.py \ + --text "This should sound like the reference speaker." \ + --ref-audio /path/to/reference.wav +``` + +The `--ref-audio` parameter accepts: +- Local file path (auto-encoded to base64) +- URL (`https://...`) +- Base64 data URI (`data:audio/wav;base64,...`) diff --git a/examples/online_serving/voxcpm2/openai_speech_client.py b/examples/online_serving/voxcpm2/openai_speech_client.py new file mode 100644 index 000000000..a117d24fd --- /dev/null +++ b/examples/online_serving/voxcpm2/openai_speech_client.py @@ -0,0 +1,108 @@ +"""OpenAI-compatible client for VoxCPM2 TTS via /v1/audio/speech endpoint. + +Examples: + # Zero-shot synthesis + python openai_speech_client.py --text "Hello, this is VoxCPM2." + + # Voice cloning with a local reference audio file + python openai_speech_client.py --text "Hello world" \ + --ref-audio /path/to/reference.wav + + # Voice cloning with a URL + python openai_speech_client.py --text "Hello world" \ + --ref-audio "https://example.com/reference.wav" + +Server setup: + python -m vllm_omni.entrypoints.openai.api_server \ + --model openbmb/VoxCPM2 \ + --stage-configs-path vllm_omni/model_executor/stage_configs/voxcpm2.yaml \ + --host 0.0.0.0 --port 8000 +""" + +from __future__ import annotations + +import argparse +import base64 +import os + +import httpx + +DEFAULT_API_BASE = "http://localhost:8000" +DEFAULT_API_KEY = "sk-empty" + + +def encode_audio_to_base64(audio_path: str) -> str: + """Encode a local audio file to a base64 data URL.""" + if not os.path.exists(audio_path): + raise FileNotFoundError(f"Audio file not found: {audio_path}") + + ext = audio_path.lower().rsplit(".", 1)[-1] + mime = { + "wav": "audio/wav", + "mp3": "audio/mpeg", + "flac": "audio/flac", + "ogg": "audio/ogg", + }.get(ext, "audio/wav") + + with open(audio_path, "rb") as f: + b64 = base64.b64encode(f.read()).decode("utf-8") + return f"data:{mime};base64,{b64}" + + +def main() -> None: + parser = argparse.ArgumentParser(description="VoxCPM2 OpenAI speech client") + parser.add_argument("--text", type=str, required=True, help="Text to synthesize") + parser.add_argument( + "--ref-audio", + type=str, + default=None, + help="Reference audio for voice cloning (local path, URL, or data: URI)", + ) + parser.add_argument("--model", type=str, default="voxcpm2") + parser.add_argument("--output", type=str, default="output.wav") + parser.add_argument("--api-base", type=str, default=DEFAULT_API_BASE) + parser.add_argument("--api-key", type=str, default=DEFAULT_API_KEY) + parser.add_argument("--response-format", type=str, default="wav") + args = parser.parse_args() + + # VoxCPM2 has no predefined voices. The "voice" field is required by + # the OpenAI API schema but ignored by VoxCPM2 — use any placeholder. + # For voice cloning, pass --ref-audio instead. + payload: dict = { + "model": args.model, + "input": args.text, + "voice": "default", + "response_format": args.response_format, + } + + if args.ref_audio: + ref = args.ref_audio + if ref.startswith(("http://", "https://", "data:")): + payload["ref_audio"] = ref + else: + payload["ref_audio"] = encode_audio_to_base64(ref) + + url = f"{args.api_base}/v1/audio/speech" + print(f"POST {url}") + print(f" text: {args.text}") + if args.ref_audio: + print(f" ref_audio: {args.ref_audio[:80]}...") + + with httpx.Client(timeout=300) as client: + resp = client.post( + url, + json=payload, + headers={"Authorization": f"Bearer {args.api_key}"}, + ) + + if resp.status_code != 200: + print(f"Error {resp.status_code}: {resp.text[:500]}") + return + + with open(args.output, "wb") as f: + f.write(resp.content) + print(f"Saved: {args.output} ({len(resp.content):,} bytes)") + + +if __name__ == "__main__": + main() diff --git a/vllm_omni/model_executor/models/voxcpm2/voxcpm2_talker.py b/vllm_omni/model_executor/models/voxcpm2/voxcpm2_talker.py index ade68b673..b9faf9fa3 100644 --- a/vllm_omni/model_executor/models/voxcpm2/voxcpm2_talker.py +++ b/vllm_omni/model_executor/models/voxcpm2/voxcpm2_talker.py @@ -22,6 +22,7 @@ from __future__ import annotations from collections.abc import Iterable from typing import Any +import librosa import torch import torch.nn as nn from vllm.config import VllmConfig @@ -41,6 +42,53 @@ from .voxcpm2_import_utils import import_voxcpm2_core logger = init_logger(__name__) +def _encode_raw_audio( + tts: nn.Module, + samples: list[float] | torch.Tensor, + sr: int, + padding_mode: str = "right", +) -> torch.Tensor: + """Encode raw audio samples using the native VoxCPM2 AudioVAE. + + Mirrors ``VoxCPM2Model._encode_wav`` but accepts in-memory samples + instead of a file path. This is needed for the OpenAI speech API + where ``_resolve_ref_audio`` returns decoded audio data. + + Args: + tts: Native VoxCPM2 tts_model instance. + samples: Audio samples (mono, float32). + sr: Sample rate of the input audio. + padding_mode: "right" (default) or "left" padding. + + Returns: + audio_feat: (T, P, D) tensor of latent patches. + """ + if isinstance(samples, list): + audio = torch.tensor(samples, dtype=torch.float32) + else: + audio = samples.float() + + if audio.ndim == 1: + audio = audio.unsqueeze(0) + + # Resample to the model's expected encoding sample rate + encode_sr = tts._encode_sample_rate + if sr != encode_sr: + audio_np = audio.squeeze(0).numpy() + audio_np = librosa.resample(audio_np, orig_sr=sr, target_sr=encode_sr) + audio = torch.from_numpy(audio_np).unsqueeze(0) + + # Pad to patch boundary + patch_len = tts.patch_size * tts.chunk_size + if audio.size(1) % patch_len != 0: + padding_size = patch_len - audio.size(1) % patch_len + pad = (padding_size, 0) if padding_mode == "left" else (0, padding_size) + audio = torch.nn.functional.pad(audio, pad) + + feat = tts.audio_vae.encode(audio.to(tts.device), encode_sr).cpu() + return feat.view(tts.audio_vae.latent_dim, -1, tts.patch_size).permute(1, 2, 0) + + class VoxCPM2TalkerForConditionalGeneration(nn.Module): """VoxCPM2 talker using native MiniCPM4 base_lm. @@ -83,6 +131,82 @@ class VoxCPM2TalkerForConditionalGeneration(nn.Module): assert self._tts is not None, "Model not loaded yet" return self._tts + def _build_prompt_cache( + self, + ref_audio: Any = None, + prompt_audio: Any = None, + prompt_text: str | None = None, + ) -> dict | None: + """Build prompt cache, handling both file paths and raw audio data. + + The OpenAI speech API sends decoded audio as [samples_list, sr] + via ``_resolve_ref_audio``, while offline usage sends file paths. + This method detects the format and routes accordingly. + """ + tts = self.tts + + def _is_raw_audio(v: Any) -> bool: + """Check if value is [samples, sr] from serving_speech.""" + return ( + isinstance(v, (list, tuple)) + and len(v) == 2 + and isinstance(v[1], int) + and isinstance(v[0], (list, torch.Tensor)) + ) + + # If all inputs are file paths (or None), use native build_prompt_cache + if not _is_raw_audio(ref_audio) and not _is_raw_audio(prompt_audio): + return tts.build_prompt_cache( + prompt_text=prompt_text, + prompt_wav_path=prompt_audio, + reference_wav_path=ref_audio, + ) + + # Raw audio path: encode directly + cache: dict[str, Any] = {} + + if ref_audio is not None: + if _is_raw_audio(ref_audio): + samples, sr = ref_audio + cache["ref_audio_feat"] = _encode_raw_audio( + tts, + samples, + sr, + padding_mode="right", + ) + else: + cache["ref_audio_feat"] = tts._encode_wav( + ref_audio, + padding_mode="right", + ) + + if prompt_audio is not None and prompt_text is not None: + cache["prompt_text"] = prompt_text + if _is_raw_audio(prompt_audio): + samples, sr = prompt_audio + cache["audio_feat"] = _encode_raw_audio( + tts, + samples, + sr, + padding_mode="left", + ) + else: + cache["audio_feat"] = tts._encode_wav( + prompt_audio, + padding_mode="left", + ) + + has_ref = "ref_audio_feat" in cache + has_prompt = "audio_feat" in cache + if has_ref and has_prompt: + cache["mode"] = "ref_continuation" + elif has_ref: + cache["mode"] = "reference" + else: + cache["mode"] = "continuation" + + return cache + # -------------------- vllm hooks -------------------- def embed_input_ids(self, input_ids: torch.Tensor, **_: Any) -> torch.Tensor: @@ -482,10 +606,10 @@ class VoxCPM2TalkerForConditionalGeneration(nn.Module): self._prompt_cache = None if ref_audio or (prompt_audio and prompt_text): try: - self._prompt_cache = self.tts.build_prompt_cache( + self._prompt_cache = self._build_prompt_cache( + ref_audio=ref_audio, + prompt_audio=prompt_audio, prompt_text=prompt_text, - prompt_wav_path=prompt_audio, - reference_wav_path=ref_audio, ) except Exception as e: logger.warning("build_prompt_cache failed: %s; falling back to zero-shot", e)