From 4d9262a6eb689fe55d07e7a1c534fe5da23a6e17 Mon Sep 17 00:00:00 2001 From: boatman <109857087+sphinxkkkbc@users.noreply.github.com> Date: Fri, 29 May 2026 15:42:08 +0800 Subject: [PATCH] [Feat]Support voice clone for omnivoice in online serving & add seed parameter for reproducible (#3668) Signed-off-by: sphinxkkkbc Co-authored-by: sphinxkkkbc --- .../text_to_speech/README.md | 8 +++ .../text_to_speech/omnivoice/end2end.py | 8 ++- .../online_serving/text_to_speech/README.md | 25 +++++-- .../text_to_speech/omnivoice/speech_client.py | 65 ++++++++++++++++++- .../test_omnivoice_expansion.py | 35 ++++++++++ tests/helpers/runtime.py | 2 +- .../models/omnivoice/pipeline_omnivoice.py | 6 +- .../entrypoints/openai/serving_speech.py | 14 ++-- .../models/omnivoice/omnivoice_generator.py | 17 +++-- 9 files changed, 158 insertions(+), 22 deletions(-) diff --git a/examples/offline_inference/text_to_speech/README.md b/examples/offline_inference/text_to_speech/README.md index 0bf0efc4e..a0e9b529b 100644 --- a/examples/offline_inference/text_to_speech/README.md +++ b/examples/offline_inference/text_to_speech/README.md @@ -271,6 +271,14 @@ python examples/offline_inference/text_to_speech/omnivoice/end2end.py \ --lang zh ``` +### Seed for Reproducibility +```bash +python examples/offline_inference/text_to_speech/omnivoice/end2end.py \ + --model k2-fsa/OmniVoice \ + --text "Hello, this is a test." \ + --seed 42 +``` + ### Notes - Stage 0 (Generator): Qwen3-0.6B with 32-step iterative unmasking. - Stage 1 (Decoder): HiggsAudioV2 RVQ + DAC at 24 kHz. diff --git a/examples/offline_inference/text_to_speech/omnivoice/end2end.py b/examples/offline_inference/text_to_speech/omnivoice/end2end.py index 16d44fe7e..f3c23e60b 100644 --- a/examples/offline_inference/text_to_speech/omnivoice/end2end.py +++ b/examples/offline_inference/text_to_speech/omnivoice/end2end.py @@ -80,6 +80,12 @@ def run_e2e(): default=600, help="Stage initialization timeout in seconds", ) + parser.add_argument( + "--seed", + type=int, + default=None, + help="Random seed for generation", + ) nullify_stage_engine_defaults(parser) args = parser.parse_args() @@ -122,7 +128,7 @@ def run_e2e(): if mm_processor_kwargs: prompts["mm_processor_kwargs"] = mm_processor_kwargs - sampling_params_list = [OmniDiffusionSamplingParams()] + sampling_params_list = [OmniDiffusionSamplingParams(extra_args={"seed": args.seed})] print(f"Generating speech for: {args.text}") diff --git a/examples/online_serving/text_to_speech/README.md b/examples/online_serving/text_to_speech/README.md index cc4afbaeb..12dafde41 100644 --- a/examples/online_serving/text_to_speech/README.md +++ b/examples/online_serving/text_to_speech/README.md @@ -18,7 +18,7 @@ For the full list of supported architectures across all modalities, see | GLM-TTS | `zai-org/GLM-TTS` | ✓ (`ref_audio`+`ref_text`, required) | ✓ (PCM stream) | — | ✓ | | Ming-flash-omni-TTS | `Jonathan1909/Ming-flash-omni-2.0` | — (caption-controlled) | — | caption fields (`instructions`) | — | | MOSS-TTS-Nano | `OpenMOSS-Team/MOSS-TTS-Nano` | ✓ (`ref_audio` required) | ✓ (PCM stream) | — | ✓ | -| OmniVoice | `k2-fsa/OmniVoice` | (offline only) | — | — | — | +| OmniVoice | `k2-fsa/OmniVoice` | ✓ | — | — | — | | Qwen3-TTS | `Qwen/Qwen3-TTS-12Hz-1.7B-{CustomVoice,VoiceDesign,Base}` | ✓ (Base) | ✓ (PCM + WebSocket) | ✓ (presets + `/v1/audio/voices` upload) | ✓ (standard + FastRTC) | | VoxCPM2 | `openbmb/VoxCPM2` | ✓ | ✓ (AudioWorklet via gradio) | — | ✓ | | Voxtral TTS | `mistralai/Voxtral-4B-TTS-2603` | ✓ (gated upstream) | ✓ | ✓ (presets) | ✓ | @@ -321,16 +321,29 @@ vllm serve k2-fsa/OmniVoice --omni --port 8091 --trust-remote-code ### CLI client ```bash cd examples/online_serving/text_to_speech/omnivoice +# Text-only (auto voice) python speech_client.py --text "Hello, how are you?" + +# Language hint python speech_client.py --text "Bonjour, comment allez-vous?" --language French +# Voice cloning (reference audio + optional ref_text) +python speech_client.py \ +--text "Bonjour, comment allez-vous?" \ +--ref-audio /path/to/ref_audio.wav \ +--ref-text "Bonjour, comment allez-vous?" + +# Style instruction (voice design-style control) +python speech_client.py \ +--text "Bonjour, comment allez-vous?" \ +--language French \ +--instructions "loud voice" + +# Deterministic output with seed parameter +python speech_client.py --text "Hello, how are you?" --seed 42 ``` -The client supports `--api-base`, `--model`, `--text`, `--response-format`, `--language`, `--output`. +The client supports `--api-base`, `--model`, `--text`, `--response-format`, `--language`, `--voice`, `--ref-audio`, `--ref-text`, `--instructions`, `--seed`, and `--output`. -### Notes -- Voice cloning and voice design require offline inference; see the [offline OmniVoice section](../../offline_inference/text_to_speech/README.md#omnivoice). - ---- ## Qwen3-TTS diff --git a/examples/online_serving/text_to_speech/omnivoice/speech_client.py b/examples/online_serving/text_to_speech/omnivoice/speech_client.py index 4c3ec334b..9c9b8128e 100644 --- a/examples/online_serving/text_to_speech/omnivoice/speech_client.py +++ b/examples/online_serving/text_to_speech/omnivoice/speech_client.py @@ -12,6 +12,8 @@ Examples: """ import argparse +import base64 +import os import httpx @@ -19,6 +21,24 @@ DEFAULT_API_BASE = "http://localhost:8091" DEFAULT_API_KEY = "EMPTY" +def encode_audio_to_base64(audio_path: str) -> str: + """Encode a local audio file to a base64 data URL.""" + if not os.path.exists(audio_path): + raise FileNotFoundError(f"Audio file not found: {audio_path}") + + ext = audio_path.lower().rsplit(".", 1)[-1] + mime = { + "wav": "audio/wav", + "mp3": "audio/mpeg", + "flac": "audio/flac", + "ogg": "audio/ogg", + }.get(ext, "audio/wav") + + with open(audio_path, "rb") as f: + b64 = base64.b64encode(f.read()).decode("utf-8") + return f"data:{mime};base64,{b64}" + + def run_tts(args) -> None: """Generate speech via /v1/audio/speech API.""" payload = { @@ -26,16 +46,36 @@ def run_tts(args) -> None: "input": args.text, "response_format": args.response_format, } + if args.seed is not None: + payload["extra_params"] = {} + payload["extra_params"]["seed"] = args.seed if args.voice: payload["voice"] = args.voice if args.language: payload["language"] = args.language + if args.ref_audio: + ref = args.ref_audio + if ref.startswith(("http://", "https://", "data:")): + payload["ref_audio"] = ref + else: + payload["ref_audio"] = encode_audio_to_base64(ref) + + if args.ref_text: + payload["ref_text"] = args.ref_text + + if args.instructions: + payload["instructions"] = args.instructions + print(f"Model: {args.model}") print(f"Text: {args.text}") + if args.seed: + print(f"Seed: {args.seed}") + if args.voice: print(f"Voice: {args.voice}") + if args.language: print(f"Language: {args.language}") print("Generating audio...") @@ -45,7 +85,6 @@ def run_tts(args) -> None: "Content-Type": "application/json", "Authorization": f"Bearer {args.api_key}", } - with httpx.Client(timeout=300.0) as client: response = client.post(api_url, json=payload, headers=headers) @@ -86,6 +125,30 @@ def main(): choices=["wav", "mp3", "flac", "pcm", "aac", "opus"], help="Audio format (default: wav)", ) + parser.add_argument( + "--ref-audio", + type=str, + default=None, + help="Reference audio for voice cloning (local path, URL, or data: URI)", + ) + parser.add_argument( + "--ref-text", + type=str, + default=None, + help="Reference text for voice cloning", + ) + parser.add_argument( + "--instructions", + type=str, + default=None, + help="Voice style/emotion instructions", + ) + parser.add_argument( + "--seed", + type=int, + default=None, + help="Random seed for generation, default: None for stochastic output)", + ) parser.add_argument("--output", "-o", default=None, help="Output file path") args = parser.parse_args() run_tts(args) diff --git a/tests/e2e/online_serving/test_omnivoice_expansion.py b/tests/e2e/online_serving/test_omnivoice_expansion.py index 631c799cd..ff41ee413 100644 --- a/tests/e2e/online_serving/test_omnivoice_expansion.py +++ b/tests/e2e/online_serving/test_omnivoice_expansion.py @@ -74,6 +74,41 @@ class TestOmniVoiceTTS: openai_client.send_audio_speech_request(request_config) +@pytest.mark.parametrize("omni_server", TEST_PARAMS, indirect=True) +class TestOmniVoiceSeed: + """E2E tests for OmniVoice Seed Params.""" + + @hardware_test(res={"cuda": "L4"}, num_cards=1) + def test_speech_auto_voice_seed_deterministic(self, omni_server, openai_client) -> None: + cfg = { + "model": omni_server.model, + "input": get_prompt("text"), + "response_format": "wav", + "seed": 42, + "min_audio_bytes": _DEFAULT_MIN_AUDIO_BYTES, + } + + r1 = openai_client.send_audio_speech_request(cfg)[0] + r2 = openai_client.send_audio_speech_request(cfg)[0] + assert r1.audio_bytes == r2.audio_bytes + + @hardware_test(res={"cuda": "L4"}, num_cards=1) + def test_speech_auto_voice_seed_non_deterministic(self, omni_server, openai_client) -> None: + cfg = { + "model": omni_server.model, + "input": get_prompt("text"), + "response_format": "wav", + "min_audio_bytes": _DEFAULT_MIN_AUDIO_BYTES, + } + + cfg1 = {**cfg, "seed": 42} + cfg2 = {**cfg, "seed": 43} + + r1 = openai_client.send_audio_speech_request(cfg1)[0] + r2 = openai_client.send_audio_speech_request(cfg2)[0] + assert r1.audio_bytes != r2.audio_bytes + + @pytest.mark.skipif(not _HAS_VOICE_CLONE, reason="Voice cloning requires transformers>=5.3.0") @pytest.mark.parametrize("omni_server", TEST_PARAMS, indirect=True) class TestOmniVoiceVoiceCloning: diff --git a/tests/helpers/runtime.py b/tests/helpers/runtime.py index 8b007dc16..7fe7fc90a 100644 --- a/tests/helpers/runtime.py +++ b/tests/helpers/runtime.py @@ -1671,7 +1671,7 @@ class OpenAIClientHandler: # Qwen3-TTS custom fields, forwarded via extra_body. extra_body: dict[str, Any] = {} # Keep this list aligned with vllm_omni.entrypoints.openai.protocol.audio params. - for key in ("task_type", "ref_text", "ref_audio", "language", "max_new_tokens"): + for key in ("task_type", "ref_text", "ref_audio", "language", "max_new_tokens", "seed"): if key in request_config: extra_body[key] = request_config[key] diff --git a/vllm_omni/diffusion/models/omnivoice/pipeline_omnivoice.py b/vllm_omni/diffusion/models/omnivoice/pipeline_omnivoice.py index 51a93de2c..745baae13 100644 --- a/vllm_omni/diffusion/models/omnivoice/pipeline_omnivoice.py +++ b/vllm_omni/diffusion/models/omnivoice/pipeline_omnivoice.py @@ -147,6 +147,8 @@ class OmniVoicePipeline(nn.Module, SupportAudioOutput): ref_text = None lang = "None" instruct = "None" + extra = req.sampling_params.extra_args or {} + seed = extra.get("seed", None) voice_name = None if isinstance(prompt, dict): @@ -157,7 +159,6 @@ class OmniVoicePipeline(nn.Module, SupportAudioOutput): voice_name = prompt.get("voice_name") lang = prompt.get("lang") instruct = prompt.get("instruct") - # OmniTextPrompt format (used by offline Omni.generate path): # ref_audio comes via multi_modal_data["audio"] and the rest via # mm_processor_kwargs. Fall back to those when top-level keys are @@ -262,7 +263,6 @@ class OmniVoicePipeline(nn.Module, SupportAudioOutput): else: cond_ids = torch.cat([text_ids, target_ids], dim=1) cond_len = cond_ids.shape[1] - uncond_ids = target_ids.clone() uncond_len = target_len max_len = max(cond_len, uncond_len) @@ -297,8 +297,8 @@ class OmniVoicePipeline(nn.Module, SupportAudioOutput): layer_penalty_factor=self.layer_penalty_factor, position_temperature=self.position_temperature, class_temperature=self.class_temperature, + seed=seed, ) - # Decode tokens to audio audio = self.decoder(tokens) # [1, 1, samples] diff --git a/vllm_omni/entrypoints/openai/serving_speech.py b/vllm_omni/entrypoints/openai/serving_speech.py index 4497f0409..458e40653 100644 --- a/vllm_omni/entrypoints/openai/serving_speech.py +++ b/vllm_omni/entrypoints/openai/serving_speech.py @@ -2813,19 +2813,21 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): request.input[:50] + "..." if len(request.input) > 50 else request.input, "ref_audio" in prompt, ) - + if request.extra_params is not None and not isinstance(request.extra_params, dict): + raise ValueError("extra_params must be a JSON object/dict.") + extra = dict(request.extra_params or {}) + if request.seed is not None: + extra["seed"] = request.seed # Apply extra_params from the request to sampling params sampling_params_list = self._diffusion_engine.default_sampling_params_list - if request.extra_params is not None: - if not isinstance(request.extra_params, dict): - raise ValueError("extra_params must be a JSON object/dict.") + if extra: import copy sampling_params_list = copy.deepcopy(sampling_params_list) if sampling_params_list[0].extra_args is None: sampling_params_list[0].extra_args = {} - sampling_params_list[0].extra_args.update(request.extra_params) - logger.info("Applied extra_params to diffusion: %s", request.extra_params) + sampling_params_list[0].extra_args.update(extra) + logger.info("Applied extra_params to diffusion: %s", extra) generator = self._diffusion_engine.generate( prompt=prompt, diff --git a/vllm_omni/model_executor/models/omnivoice/omnivoice_generator.py b/vllm_omni/model_executor/models/omnivoice/omnivoice_generator.py index 0de2a1f60..fc5befeb9 100644 --- a/vllm_omni/model_executor/models/omnivoice/omnivoice_generator.py +++ b/vllm_omni/model_executor/models/omnivoice/omnivoice_generator.py @@ -13,6 +13,7 @@ via FlashAttention/SageAttention/SDPA backends. from __future__ import annotations import math +import random import threading from collections import OrderedDict @@ -225,9 +226,13 @@ def _get_time_steps( return shifted -def _gumbel_sample(logits: torch.Tensor, temperature: float) -> torch.Tensor: +def _gumbel_sample(logits: torch.Tensor, temperature: float, generator: torch.Generator) -> torch.Tensor: """Add Gumbel noise for stochastic position selection.""" - noise = -torch.log(-torch.log(torch.rand_like(logits).clamp(min=1e-8))) + noise = -torch.log( + -torch.log( + torch.rand(logits.shape, generator=generator, device=logits.device, dtype=logits.dtype).clamp(min=1e-8) + ) + ) return logits / max(temperature, 1e-8) + noise @@ -791,6 +796,7 @@ class OmniVoiceGenerator(nn.Module): audio_mask: torch.Tensor, attention_mask: torch.Tensor, target_lens: list[int], + seed: int | None = None, num_step: int = 32, guidance_scale: float = 2.0, t_shift: float = 0.1, @@ -820,6 +826,9 @@ class OmniVoiceGenerator(nn.Module): max_target_len = max(target_lens) mask_id = self.config.audio_mask_id num_codebooks = self.config.num_audio_codebook + if seed is None: + seed = random.randint(0, 2**63 - 1) + generator = torch.Generator(device=device).manual_seed(seed) # Initialize all target tokens as [MASK] tokens = torch.full( @@ -896,7 +905,7 @@ class OmniVoiceGenerator(nn.Module): # Token prediction if class_temperature > 0.0: - pred_tokens = _gumbel_sample(log_probs, class_temperature).argmax(dim=-1) + pred_tokens = _gumbel_sample(log_probs, class_temperature, generator).argmax(dim=-1) else: pred_tokens = log_probs.argmax(dim=-1) # [1, 8, T] @@ -908,7 +917,7 @@ class OmniVoiceGenerator(nn.Module): # Gumbel noise for position selection if position_temperature > 0.0: - scores = _gumbel_sample(scores, position_temperature) + scores = _gumbel_sample(scores, position_temperature, generator) # Mask out already unmasked positions sample_tokens = tokens[i : i + 1, :, :t_len]