[Fix] VoxCPM2: support raw audio for voice cloning via OpenAI API (#2720)

Signed-off-by: Yueqian Lin <linyueqian@outlook.com>

examples/online_serving/voxcpm2/README.md

@@ -0,0 +1,42 @@
+# VoxCPM2 Online Serving
+
+Serve VoxCPM2 TTS via the OpenAI-compatible `/v1/audio/speech` endpoint.
+
+## Start the Server
+
+```bash
+python -m vllm_omni.entrypoints.openai.api_server \
+    --model openbmb/VoxCPM2 \
+    --stage-configs-path vllm_omni/model_executor/stage_configs/voxcpm2.yaml \
+    --host 0.0.0.0 --port 8000
+```
+
+## Zero-shot Synthesis
+
+```bash
+python openai_speech_client.py --text "Hello, this is VoxCPM2."
+```
+
+Or with curl:
+
+```bash
+curl -X POST http://localhost:8000/v1/audio/speech \
+  -H "Content-Type: application/json" \
+  -d '{"model": "voxcpm2", "input": "Hello, this is VoxCPM2.", "voice": "default"}' \
+  --output output.wav
+```
+
+## Voice Cloning
+
+Clone a speaker's voice using a reference audio file:
+
+```bash
+python openai_speech_client.py \
+    --text "This should sound like the reference speaker." \
+    --ref-audio /path/to/reference.wav
+```
+
+The `--ref-audio` parameter accepts:
+- Local file path (auto-encoded to base64)
+- URL (`https://...`)
+- Base64 data URI (`data:audio/wav;base64,...`)

examples/online_serving/voxcpm2/openai_speech_client.py

@@ -0,0 +1,108 @@
+"""OpenAI-compatible client for VoxCPM2 TTS via /v1/audio/speech endpoint.
+
+Examples:
+    # Zero-shot synthesis
+    python openai_speech_client.py --text "Hello, this is VoxCPM2."
+
+    # Voice cloning with a local reference audio file
+    python openai_speech_client.py --text "Hello world" \
+        --ref-audio /path/to/reference.wav
+
+    # Voice cloning with a URL
+    python openai_speech_client.py --text "Hello world" \
+        --ref-audio "https://example.com/reference.wav"
+
+Server setup:
+    python -m vllm_omni.entrypoints.openai.api_server \
+        --model openbmb/VoxCPM2 \
+        --stage-configs-path vllm_omni/model_executor/stage_configs/voxcpm2.yaml \
+        --host 0.0.0.0 --port 8000
+"""
+
+from __future__ import annotations
+
+import argparse
+import base64
+import os
+
+import httpx
+
+DEFAULT_API_BASE = "http://localhost:8000"
+DEFAULT_API_KEY = "sk-empty"
+
+
+def encode_audio_to_base64(audio_path: str) -> str:
+    """Encode a local audio file to a base64 data URL."""
+    if not os.path.exists(audio_path):
+        raise FileNotFoundError(f"Audio file not found: {audio_path}")
+
+    ext = audio_path.lower().rsplit(".", 1)[-1]
+    mime = {
+        "wav": "audio/wav",
+        "mp3": "audio/mpeg",
+        "flac": "audio/flac",
+        "ogg": "audio/ogg",
+    }.get(ext, "audio/wav")
+
+    with open(audio_path, "rb") as f:
+        b64 = base64.b64encode(f.read()).decode("utf-8")
+    return f"data:{mime};base64,{b64}"
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="VoxCPM2 OpenAI speech client")
+    parser.add_argument("--text", type=str, required=True, help="Text to synthesize")
+    parser.add_argument(
+        "--ref-audio",
+        type=str,
+        default=None,
+        help="Reference audio for voice cloning (local path, URL, or data: URI)",
+    )
+    parser.add_argument("--model", type=str, default="voxcpm2")
+    parser.add_argument("--output", type=str, default="output.wav")
+    parser.add_argument("--api-base", type=str, default=DEFAULT_API_BASE)
+    parser.add_argument("--api-key", type=str, default=DEFAULT_API_KEY)
+    parser.add_argument("--response-format", type=str, default="wav")
+    args = parser.parse_args()
+
+    # VoxCPM2 has no predefined voices. The "voice" field is required by
+    # the OpenAI API schema but ignored by VoxCPM2 — use any placeholder.
+    # For voice cloning, pass --ref-audio instead.
+    payload: dict = {
+        "model": args.model,
+        "input": args.text,
+        "voice": "default",
+        "response_format": args.response_format,
+    }
+
+    if args.ref_audio:
+        ref = args.ref_audio
+        if ref.startswith(("http://", "https://", "data:")):
+            payload["ref_audio"] = ref
+        else:
+            payload["ref_audio"] = encode_audio_to_base64(ref)
+
+    url = f"{args.api_base}/v1/audio/speech"
+    print(f"POST {url}")
+    print(f"  text: {args.text}")
+    if args.ref_audio:
+        print(f"  ref_audio: {args.ref_audio[:80]}...")
+
+    with httpx.Client(timeout=300) as client:
+        resp = client.post(
+            url,
+            json=payload,
+            headers={"Authorization": f"Bearer {args.api_key}"},
+        )
+
+    if resp.status_code != 200:
+        print(f"Error {resp.status_code}: {resp.text[:500]}")
+        return
+
+    with open(args.output, "wb") as f:
+        f.write(resp.content)
+    print(f"Saved: {args.output} ({len(resp.content):,} bytes)")
+
+
+if __name__ == "__main__":
+    main()

vllm_omni/model_executor/models/voxcpm2/voxcpm2_talker.py

@@ -22,6 +22,7 @@ from __future__ import annotations
 from collections.abc import Iterable
 from typing import Any
 
+import librosa
 import torch
 import torch.nn as nn
 from vllm.config import VllmConfig
@@ -41,6 +42,53 @@ from .voxcpm2_import_utils import import_voxcpm2_core
 logger = init_logger(__name__)
 
 
+def _encode_raw_audio(
+    tts: nn.Module,
+    samples: list[float] | torch.Tensor,
+    sr: int,
+    padding_mode: str = "right",
+) -> torch.Tensor:
+    """Encode raw audio samples using the native VoxCPM2 AudioVAE.
+
+    Mirrors ``VoxCPM2Model._encode_wav`` but accepts in-memory samples
+    instead of a file path.  This is needed for the OpenAI speech API
+    where ``_resolve_ref_audio`` returns decoded audio data.
+
+    Args:
+        tts: Native VoxCPM2 tts_model instance.
+        samples: Audio samples (mono, float32).
+        sr: Sample rate of the input audio.
+        padding_mode: "right" (default) or "left" padding.
+
+    Returns:
+        audio_feat: (T, P, D) tensor of latent patches.
+    """
+    if isinstance(samples, list):
+        audio = torch.tensor(samples, dtype=torch.float32)
+    else:
+        audio = samples.float()
+
+    if audio.ndim == 1:
+        audio = audio.unsqueeze(0)
+
+    # Resample to the model's expected encoding sample rate
+    encode_sr = tts._encode_sample_rate
+    if sr != encode_sr:
+        audio_np = audio.squeeze(0).numpy()
+        audio_np = librosa.resample(audio_np, orig_sr=sr, target_sr=encode_sr)
+        audio = torch.from_numpy(audio_np).unsqueeze(0)
+
+    # Pad to patch boundary
+    patch_len = tts.patch_size * tts.chunk_size
+    if audio.size(1) % patch_len != 0:
+        padding_size = patch_len - audio.size(1) % patch_len
+        pad = (padding_size, 0) if padding_mode == "left" else (0, padding_size)
+        audio = torch.nn.functional.pad(audio, pad)
+
+    feat = tts.audio_vae.encode(audio.to(tts.device), encode_sr).cpu()
+    return feat.view(tts.audio_vae.latent_dim, -1, tts.patch_size).permute(1, 2, 0)
+
+
 class VoxCPM2TalkerForConditionalGeneration(nn.Module):
     """VoxCPM2 talker using native MiniCPM4 base_lm.
 
@@ -83,6 +131,82 @@ class VoxCPM2TalkerForConditionalGeneration(nn.Module):
         assert self._tts is not None, "Model not loaded yet"
         return self._tts
 
+    def _build_prompt_cache(
+        self,
+        ref_audio: Any = None,
+        prompt_audio: Any = None,
+        prompt_text: str | None = None,
+    ) -> dict | None:
+        """Build prompt cache, handling both file paths and raw audio data.
+
+        The OpenAI speech API sends decoded audio as [samples_list, sr]
+        via ``_resolve_ref_audio``, while offline usage sends file paths.
+        This method detects the format and routes accordingly.
+        """
+        tts = self.tts
+
+        def _is_raw_audio(v: Any) -> bool:
+            """Check if value is [samples, sr] from serving_speech."""
+            return (
+                isinstance(v, (list, tuple))
+                and len(v) == 2
+                and isinstance(v[1], int)
+                and isinstance(v[0], (list, torch.Tensor))
+            )
+
+        # If all inputs are file paths (or None), use native build_prompt_cache
+        if not _is_raw_audio(ref_audio) and not _is_raw_audio(prompt_audio):
+            return tts.build_prompt_cache(
+                prompt_text=prompt_text,
+                prompt_wav_path=prompt_audio,
+                reference_wav_path=ref_audio,
+            )
+
+        # Raw audio path: encode directly
+        cache: dict[str, Any] = {}
+
+        if ref_audio is not None:
+            if _is_raw_audio(ref_audio):
+                samples, sr = ref_audio
+                cache["ref_audio_feat"] = _encode_raw_audio(
+                    tts,
+                    samples,
+                    sr,
+                    padding_mode="right",
+                )
+            else:
+                cache["ref_audio_feat"] = tts._encode_wav(
+                    ref_audio,
+                    padding_mode="right",
+                )
+
+        if prompt_audio is not None and prompt_text is not None:
+            cache["prompt_text"] = prompt_text
+            if _is_raw_audio(prompt_audio):
+                samples, sr = prompt_audio
+                cache["audio_feat"] = _encode_raw_audio(
+                    tts,
+                    samples,
+                    sr,
+                    padding_mode="left",
+                )
+            else:
+                cache["audio_feat"] = tts._encode_wav(
+                    prompt_audio,
+                    padding_mode="left",
+                )
+
+        has_ref = "ref_audio_feat" in cache
+        has_prompt = "audio_feat" in cache
+        if has_ref and has_prompt:
+            cache["mode"] = "ref_continuation"
+        elif has_ref:
+            cache["mode"] = "reference"
+        else:
+            cache["mode"] = "continuation"
+
+        return cache
+
     # -------------------- vllm hooks --------------------
 
     def embed_input_ids(self, input_ids: torch.Tensor, **_: Any) -> torch.Tensor:
@@ -482,10 +606,10 @@ class VoxCPM2TalkerForConditionalGeneration(nn.Module):
             self._prompt_cache = None
             if ref_audio or (prompt_audio and prompt_text):
                 try:
-                    self._prompt_cache = self.tts.build_prompt_cache(
+                    self._prompt_cache = self._build_prompt_cache(
+                        ref_audio=ref_audio,
+                        prompt_audio=prompt_audio,
                         prompt_text=prompt_text,
-                        prompt_wav_path=prompt_audio,
-                        reference_wav_path=ref_audio,
                     )
                 except Exception as e:
                     logger.warning("build_prompt_cache failed: %s; falling back to zero-shot", e)