From 3adae7ca6fe461fa9b4f0b9a704d082d83da3044 Mon Sep 17 00:00:00 2001 From: Hongsheng Liu Date: Tue, 28 Apr 2026 14:29:26 +0800 Subject: [PATCH] =?UTF-8?q?Revert=20"[Doc][Frontend][Model][VoxCPM2]=20Sup?= =?UTF-8?q?port=20instructions=20and=20per-request=20=E2=80=A6"?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit 1069a144a85536a0c8d944686889e18f80095cad. --- docs/serving/speech_api.md | 72 ---- .../openai_api/test_serving_speech_voxcpm2.py | 330 ------------------ .../entrypoints/openai/protocol/audio.py | 7 +- .../entrypoints/openai/serving_speech.py | 56 +-- .../models/voxcpm2/voxcpm2_talker.py | 16 +- 5 files changed, 9 insertions(+), 472 deletions(-) delete mode 100644 tests/entrypoints/openai_api/test_serving_speech_voxcpm2.py diff --git a/docs/serving/speech_api.md b/docs/serving/speech_api.md index 209d98cfe..8f78d6a20 100644 --- a/docs/serving/speech_api.md +++ b/docs/serving/speech_api.md @@ -5,7 +5,6 @@ vLLM-Omni provides an OpenAI-compatible API for text-to-speech (TTS) generation. - **Qwen3-TTS** (`Qwen/Qwen3-TTS-12Hz-*`) -- Qwen3-based TTS with CustomVoice, VoiceDesign, and Base (voice cloning) task types. Output: 24 kHz. - **Fish Speech S2 Pro** (`fishaudio/s2-pro`) -- Dual-AR TTS with DAC codec. Supports text-to-speech and voice cloning via reference audio. Output: 44.1 kHz. - **Voxtral TTS** (`mistralai/Voxtral-4B-TTS-2603`) -- AR + FlowMatching TTS with preset voices. Output: 24 kHz. -- **VoxCPM2** (`openbmb/VoxCPM2`) -- Single-stage AR + DiT/CFM TTS. Three synthesis modes: Voice Design (text-only voice description), Controllable Cloning (reference audio + style instructions), and Hi-Fi Cloning (reference audio + transcript for tightest identity match; aka "Ultimate Cloning" in the Gradio demo). Output: 48 kHz. Each server instance runs a single model (specified at startup via `vllm serve --omni`). @@ -27,9 +26,6 @@ vllm serve fishaudio/s2-pro --omni --port 8091 # Voxtral TTS vllm serve mistralai/Voxtral-4B-TTS-2603 --omni --port 8091 - -# VoxCPM2 -vllm serve openbmb/VoxCPM2 --omni --port 8091 ``` ### Generate Speech @@ -123,16 +119,6 @@ Content-Type: application/json | `ref_text` | string | null | Transcript of reference audio | | `x_vector_only_mode` | bool | null | Use speaker embedding only (no ICL) | -#### VoxCPM2-specific Parameters - -VoxCPM2 reuses the shared `extra_params: dict` field (introduced in #2338 for Voxtral TTS) for model-specific knobs rather than adding new top-level fields: - -| Key (inside `extra_params`) | Type | Default | Description | -|-----------------------------|------|---------|-------------| -| `cfg_value` | float | 2.0 | Classifier-free guidance scale for the CFM decoder. Range `0.1`–`10.0`; typical useful range is `1.5`–`3.0`. Higher values track the text more strictly at the cost of naturalness. | - -Send as e.g. `{"input": "...", "extra_params": {"cfg_value": 2.5}}`. VoxCPM2 also reuses the standard `instructions`, `ref_audio`, and `ref_text` fields, see the mode table below for how they combine. - ### Response Format Returns binary audio data with appropriate `Content-Type` header (e.g., `audio/wav`). @@ -557,64 +543,6 @@ Fish Speech uses `ref_audio` and `ref_text` for voice cloning (no `task_type` ne |-------|-------------| | `mistralai/Voxtral-4B-TTS-2603` | 3B AR + FlowMatching TTS. Supports text-to-speech with preset voices. | -### VoxCPM2 - -| Model | Description | -|-------|-------------| -| `openbmb/VoxCPM2` | Single-stage AR + DiT/CFM TTS. 48 kHz output, multilingual (primary EN/ZH, usable cross-lingual identity transfer via Ultimate Cloning). | - -VoxCPM2 exposes three synthesis modes through the same `/v1/audio/speech` endpoint — the mode is inferred from which fields are present in the request. - -| Mode | Required fields | Optional fields | Notes | -|------|-----------------|-----------------|-------| -| **Voice Design** | `input` | `instructions`, `extra_params.cfg_value` | Synthesize a voice from a text description alone. `instructions` is prepended to the target text as `(instructions)input` following `voxcpm.cli.build_final_text`. Both English and Chinese instructions are supported. | -| **Controllable Cloning** | `input`, `ref_audio` | `instructions`, `extra_params.cfg_value` | Clone the reference speaker's timbre; use `instructions` to steer emotion, pace, or other style attributes while preserving identity. | -| **Hi-Fi Cloning** | `input`, `ref_audio`, `ref_text` | `extra_params.cfg_value` | The model treats `ref_audio` + `ref_text` as a spoken prefix and continues from it. Reproduces every vocal nuance of the reference. Per the canonical VoxCPM2 docs, when Hi-Fi mode is enabled the control instruction is ignored; the server drops `instructions` in this mode automatically if present. | - -Reference audio guidelines: 16 kHz mono, at least 5 seconds recommended for a good clone (1 second hard minimum, 30 seconds hard maximum). Clean, non-reverberant speech; the same `ref_audio` can be reused across Controllable and Hi-Fi modes. - -Inline non-verbal tags in `input` are passed through as regular text and recognized by the model when present (for example `[laughing]`, `[sigh]`, `[Uhm]`, `[Question-ah]`). See the upstream [VoxCPM2 cookbook](https://voxcpm.readthedocs.io/en/latest/cookbook.html) for the full tag list and usage tips. - -**Examples** - -Voice Design (no reference, pure text description): - -```bash -curl -X POST http://localhost:8091/v1/audio/speech \ - -H "Content-Type: application/json" \ - -d '{ - "input": "Hello everyone. Today I want to talk about open source AI.", - "instructions": "A warm young woman with a gentle, friendly voice", - "extra_params": {"cfg_value": 2.5} - }' --output voice_design.wav -``` - -Controllable Cloning (reference audio + style instructions): - -```bash -curl -X POST http://localhost:8091/v1/audio/speech \ - -H "Content-Type: application/json" \ - -d '{ - "input": "Good morning, let me tell you a short story.", - "ref_audio": "https://example.com/reference.wav", - "instructions": "Warm storyteller, slow cadence, dramatic pauses", - "extra_params": {"cfg_value": 2.5} - }' --output controllable_cloning.wav -``` - -Hi-Fi Cloning (reference audio + transcript; tightest identity): - -```bash -curl -X POST http://localhost:8091/v1/audio/speech \ - -H "Content-Type: application/json" \ - -d '{ - "input": "Welcome everyone. I will give a brief update on the project today.", - "ref_audio": "https://example.com/reference.wav", - "ref_text": "Transcript of what the reference speaker is saying in the reference clip.", - "extra_params": {"cfg_value": 2.5} - }' --output ultimate_cloning.wav -``` - ## Error Responses ### 400 Bad Request diff --git a/tests/entrypoints/openai_api/test_serving_speech_voxcpm2.py b/tests/entrypoints/openai_api/test_serving_speech_voxcpm2.py deleted file mode 100644 index 273648365..000000000 --- a/tests/entrypoints/openai_api/test_serving_speech_voxcpm2.py +++ /dev/null @@ -1,330 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# SPDX-FileCopyrightText: Copyright contributors to the vLLM project - -"""UTs for VoxCPM2 OpenAI speech serving behavior (`instructions` + `cfg_value`).""" - -import asyncio -from types import SimpleNamespace -from unittest.mock import AsyncMock - -import pytest -from pytest_mock import MockerFixture - -from vllm_omni.entrypoints.openai.protocol.audio import OpenAICreateSpeechRequest -from vllm_omni.entrypoints.openai.serving_speech import OmniOpenAIServingSpeech - -pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.omni] - - -@pytest.fixture -def voxcpm2_server(mocker: MockerFixture): - """Mock an OmniOpenAIServingSpeech instance detected as `voxcpm2`. - - Single `latent_generator` stage with `model_arch=VoxCPM2...` so - `_detect_tts_model_type` returns ``voxcpm2``. - """ - mocker.patch.object(OmniOpenAIServingSpeech, "_load_supported_speakers", return_value=set()) - mocker.patch.object(OmniOpenAIServingSpeech, "_load_codec_frame_rate", return_value=None) - - mock_engine_client = mocker.MagicMock() - mock_engine_client.errored = False - mock_engine_client.model_config = mocker.MagicMock( - model="openbmb/VoxCPM2", - hf_config=SimpleNamespace( - audio_vae_config={"sample_rate": 16000, "encoder_rates": [2, 5, 8, 8]}, - patch_size=4, - ), - ) - mock_engine_client.default_sampling_params_list = [SimpleNamespace(max_tokens=4096)] - mock_engine_client.tts_batch_max_items = 32 - # Explicit None so `_compute_max_instructions_length` skips the CLI-override - # branch (MagicMock attributes are truthy by default) and falls through to - # the `_TTS_MAX_INSTRUCTIONS_LENGTH = 500` default. - mock_engine_client.tts_max_instructions_length = None - mock_engine_client.generate = mocker.MagicMock(return_value="generator") - mock_engine_client.stage_configs = [ - SimpleNamespace( - engine_args=SimpleNamespace( - model_stage="latent_generator", - model_arch="VoxCPM2TalkerForConditionalGeneration", - ), - tts_args={}, - ), - ] - - mock_models = mocker.MagicMock() - mock_models.is_base_model.return_value = True - - server = OmniOpenAIServingSpeech( - engine_client=mock_engine_client, - models=mock_models, - request_logger=mocker.MagicMock(), - ) - # Lazy tokenizer path is not exercised by these tests; provide a no-op encoder - # and sentinel tokenizer / split_map so `_build_voxcpm2_prompt` can run. - server._voxcpm2_tokenizer = mocker.MagicMock(name="voxcpm2_tokenizer") - server._voxcpm2_split_map = {} - mocker.patch.object(server, "_voxcpm2_encode", return_value=[]) - return server - - -@pytest.fixture -def mock_build_voxcpm2_prompt(mocker: MockerFixture): - """Patch `build_voxcpm2_prompt` so we can inspect its call args without - touching the real VoxCPM2 tokenizer / audio VAE. - - Returns the prompt dict shape the real helper would produce; the test - can then also assert `cfg_value` stashing into `additional_information`. - """ - return mocker.patch( - "vllm_omni.model_executor.models.voxcpm2.voxcpm2_talker.build_voxcpm2_prompt", - return_value={"prompt_token_ids": [1], "additional_information": {"text_token_ids": [[]]}}, - ) - - -class TestVoxCPM2Serving: - def test_voxcpm2_model_type_detection(self, voxcpm2_server): - assert voxcpm2_server._tts_model_type == "voxcpm2" - assert voxcpm2_server._is_tts is True - - def test_voxcpm2_accepts_any_text_input(self, voxcpm2_server): - """VoxCPM2 skips the strict voxcpm validator (see `_validate_tts_request`).""" - request = OpenAICreateSpeechRequest(input="مرحباً", instructions="warm tone") - assert voxcpm2_server._validate_tts_request(request) is None - - def test_build_prompt_text_only(self, voxcpm2_server, mock_build_voxcpm2_prompt): - """No instructions, no cfg_value: text flows through unchanged, no cfg stash.""" - request = OpenAICreateSpeechRequest(input="hello voxcpm2") - prompt = asyncio.run(voxcpm2_server._build_voxcpm2_prompt(request)) - - assert mock_build_voxcpm2_prompt.call_args.kwargs["text"] == "hello voxcpm2" - assert "cfg_value" not in prompt.get("additional_information", {}) - - def test_build_prompt_prepends_instructions(self, voxcpm2_server, mock_build_voxcpm2_prompt): - """`instructions` wraps in parens and prepends to text (native VoxCPM2 convention).""" - request = OpenAICreateSpeechRequest( - input="hello voxcpm2", - instructions="A warm young woman", - ) - asyncio.run(voxcpm2_server._build_voxcpm2_prompt(request)) - - assert mock_build_voxcpm2_prompt.call_args.kwargs["text"] == "(A warm young woman)hello voxcpm2" - - def test_build_prompt_strips_instructions_whitespace(self, voxcpm2_server, mock_build_voxcpm2_prompt): - """Leading/trailing whitespace in `instructions` is stripped before prepending.""" - request = OpenAICreateSpeechRequest(input="hello", instructions=" calm radio ") - asyncio.run(voxcpm2_server._build_voxcpm2_prompt(request)) - - assert mock_build_voxcpm2_prompt.call_args.kwargs["text"] == "(calm radio)hello" - - def test_build_prompt_stashes_cfg_value(self, voxcpm2_server, mock_build_voxcpm2_prompt): - """`extra_params['cfg_value']` lands in `additional_information` for the talker to lift.""" - request = OpenAICreateSpeechRequest(input="hello", extra_params={"cfg_value": 2.7}) - prompt = asyncio.run(voxcpm2_server._build_voxcpm2_prompt(request)) - - assert prompt["additional_information"]["cfg_value"] == 2.7 - - def test_build_prompt_omits_cfg_value_when_extra_params_missing(self, voxcpm2_server, mock_build_voxcpm2_prompt): - """Omitting `cfg_value` must not add the key (talker falls back to its default).""" - request = OpenAICreateSpeechRequest(input="hello") - prompt = asyncio.run(voxcpm2_server._build_voxcpm2_prompt(request)) - - assert "cfg_value" not in prompt["additional_information"] - - def test_build_prompt_omits_cfg_value_when_extra_params_has_other_keys( - self, voxcpm2_server, mock_build_voxcpm2_prompt - ): - """`extra_params` set but without `cfg_value` must not affect the prompt.""" - request = OpenAICreateSpeechRequest(input="hello", extra_params={"some_other_knob": 1}) - prompt = asyncio.run(voxcpm2_server._build_voxcpm2_prompt(request)) - - assert "cfg_value" not in prompt["additional_information"] - - def test_build_prompt_instructions_and_cfg_together(self, voxcpm2_server, mock_build_voxcpm2_prompt): - """Both features compose: text prefixed AND cfg_value stashed.""" - request = OpenAICreateSpeechRequest( - input="hello", - instructions="excited", - extra_params={"cfg_value": 2.5}, - ) - prompt = asyncio.run(voxcpm2_server._build_voxcpm2_prompt(request)) - - assert mock_build_voxcpm2_prompt.call_args.kwargs["text"] == "(excited)hello" - assert prompt["additional_information"]["cfg_value"] == 2.5 - - def test_build_prompt_hifi_cloning_ref_audio_ref_text_cfg( - self, voxcpm2_server, mock_build_voxcpm2_prompt, mocker: MockerFixture - ): - """Hi-Fi Cloning (`ref_audio` + `ref_text`) resolves audio and threads cfg_value.""" - voxcpm2_server._resolve_ref_audio = AsyncMock(return_value=([0.1, -0.1, 0.2], 16000)) - - request = OpenAICreateSpeechRequest( - input="clone me", - ref_audio="data:audio/wav;base64,QUJD", - ref_text="reference transcript", - extra_params={"cfg_value": 2.7}, - ) - prompt = asyncio.run(voxcpm2_server._build_voxcpm2_prompt(request)) - - kwargs = mock_build_voxcpm2_prompt.call_args.kwargs - assert kwargs["text"] == "clone me" - assert kwargs["ref_audio"] == [0.1, -0.1, 0.2] - assert kwargs["ref_sr"] == 16000 - assert kwargs["ref_text"] == "reference transcript" - assert prompt["additional_information"]["cfg_value"] == 2.7 - voxcpm2_server._resolve_ref_audio.assert_awaited_once_with("data:audio/wav;base64,QUJD") - - def test_validate_rejects_instructions_with_ref_text(self, voxcpm2_server): - """Hi-Fi Cloning (ref_audio + ref_text) + instructions returns a 400-level - error string from the validator instead of silently dropping `instructions`. - - Matches upstream voxcpm CLI's hard error on the same combo - (src/voxcpm/cli.py:128-131: --control + --prompt-text). - """ - request = OpenAICreateSpeechRequest( - input="hello", - ref_audio="data:audio/wav;base64,QUJD", - ref_text="reference transcript", - instructions="this is incompatible with Hi-Fi mode", - ) - error = voxcpm2_server._validate_tts_request(request) - assert error is not None - assert "Hi-Fi" in error - assert "ref_text" in error - - def test_validate_accepts_instructions_without_ref_text(self, voxcpm2_server): - """`instructions` is fine in Voice Design and Controllable Cloning modes - (i.e. whenever `ref_text` is not also set).""" - request = OpenAICreateSpeechRequest(input="hello", instructions="warm tone") - assert voxcpm2_server._validate_tts_request(request) is None - - def test_validate_accepts_ref_text_without_instructions(self, voxcpm2_server): - """Hi-Fi Cloning (ref_audio + ref_text, no instructions) is fine.""" - request = OpenAICreateSpeechRequest( - input="hello", - ref_audio="data:audio/wav;base64,QUJD", - ref_text="reference transcript", - ) - assert voxcpm2_server._validate_tts_request(request) is None - - def test_validate_rejects_overlong_instructions(self, voxcpm2_server): - """Instructions longer than `_max_instructions_length` (500 default) are rejected - with an error message that includes the actual length, the cap, and an - upstream-context hint so users coming from the voxcpm CLI understand why - the limit exists. - """ - oversize = "x" * (voxcpm2_server._max_instructions_length + 1) - request = OpenAICreateSpeechRequest(input="hello", instructions=oversize) - error = voxcpm2_server._validate_tts_request(request) - assert error is not None - assert str(len(oversize)) in error - assert str(voxcpm2_server._max_instructions_length) in error - assert "Upstream voxcpm has no cap" in error - - def test_validate_accepts_at_limit_instructions(self, voxcpm2_server): - """Instructions exactly at `_max_instructions_length` are accepted.""" - at_limit = "x" * voxcpm2_server._max_instructions_length - request = OpenAICreateSpeechRequest(input="hello", instructions=at_limit) - assert voxcpm2_server._validate_tts_request(request) is None - - def test_prepare_speech_generation_runs_validator_for_voxcpm2_length( - self, voxcpm2_server, mock_build_voxcpm2_prompt, mocker: MockerFixture - ): - """Single-request `/v1/audio/speech` path must invoke `_validate_tts_request` - before building the prompt, so the instructions-length cap actually fires - for normal (non-batch) requests too. Regression guard for #3118 P2. - """ - oversize = "x" * (voxcpm2_server._max_instructions_length + 1) - request = OpenAICreateSpeechRequest(input="hello", instructions=oversize) - with pytest.raises(ValueError, match=str(voxcpm2_server._max_instructions_length)): - asyncio.run(voxcpm2_server._prepare_speech_generation(request)) - # The prompt builder must NOT be reached when validation fails. - mock_build_voxcpm2_prompt.assert_not_called() - - def test_prepare_speech_generation_runs_validator_for_voxcpm2_hifi( - self, voxcpm2_server, mock_build_voxcpm2_prompt, mocker: MockerFixture - ): - """Single-request path must also surface the Hi-Fi mode rejection - (instructions + ref_text), not silently strip `instructions`. - """ - request = OpenAICreateSpeechRequest( - input="hello", - ref_audio="data:audio/wav;base64,QUJD", - ref_text="reference transcript", - instructions="incompatible with Hi-Fi mode", - ) - with pytest.raises(ValueError, match="Hi-Fi"): - asyncio.run(voxcpm2_server._prepare_speech_generation(request)) - mock_build_voxcpm2_prompt.assert_not_called() - - @pytest.mark.parametrize("cfg", [0.1, 0.5, 1.5, 2.0, 2.7, 5.0, 10.0]) - def test_cfg_value_accepts_range(self, voxcpm2_server, mock_build_voxcpm2_prompt, cfg): - """`_build_voxcpm2_prompt` accepts cfg_value within 0.1-10.0 inclusive.""" - request = OpenAICreateSpeechRequest(input="hello", extra_params={"cfg_value": cfg}) - prompt = asyncio.run(voxcpm2_server._build_voxcpm2_prompt(request)) - assert prompt["additional_information"]["cfg_value"] == cfg - - @pytest.mark.parametrize("cfg", [0.0, -1.0, 10.5, 100.0]) - def test_cfg_value_rejects_out_of_range(self, voxcpm2_server, mock_build_voxcpm2_prompt, cfg): - """Out-of-range `cfg_value` (validated in `_build_voxcpm2_prompt`) raises ValueError.""" - request = OpenAICreateSpeechRequest(input="hello", extra_params={"cfg_value": cfg}) - with pytest.raises(ValueError, match="out of range"): - asyncio.run(voxcpm2_server._build_voxcpm2_prompt(request)) - - @pytest.mark.parametrize("bad", ["abc", None, [1, 2], {"x": 1}]) - def test_cfg_value_rejects_non_numeric(self, voxcpm2_server, mock_build_voxcpm2_prompt, bad): - """Non-numeric `extra_params['cfg_value']` raises ValueError instead of crashing the talker.""" - request = OpenAICreateSpeechRequest(input="hello", extra_params={"cfg_value": bad}) - with pytest.raises(ValueError, match="must be a number|out of range"): - asyncio.run(voxcpm2_server._build_voxcpm2_prompt(request)) - - -class TestVoxCPM2DecodeStepCfgPropagation: - """Static AST regression guard for #3118 P1: ensure both `_finish_prefill` - and `_finish_decode` invoke `_run_cfm` with the per-request `cfg_value` - kwarg. Originally only `_finish_prefill` threaded the kwarg, so requests - longer than the prefill patch fell back to `self._cfg_value = 2.0` for - every decode step. This test parses the talker source and asserts the - kwarg is present at every `_run_cfm` call site inside the two methods, - so a future refactor that drops the kwarg from either method fails CI - even before the behavioral cfg-sweep tests run. - """ - - @pytest.fixture(scope="class") - def talker_class_ast(self): - import ast - import inspect - - from vllm_omni.model_executor.models.voxcpm2 import voxcpm2_talker - - src = inspect.getsource(voxcpm2_talker) - tree = ast.parse(src) - cls = next( - n - for n in ast.walk(tree) - if isinstance(n, ast.ClassDef) and n.name == "VoxCPM2TalkerForConditionalGeneration" - ) - return cls - - def _run_cfm_calls_in(self, cls_ast, method_name): - import ast - - method = next(n for n in cls_ast.body if isinstance(n, ast.FunctionDef) and n.name == method_name) - return [ - n - for n in ast.walk(method) - if isinstance(n, ast.Call) and isinstance(n.func, ast.Attribute) and n.func.attr == "_run_cfm" - ] - - @pytest.mark.parametrize("method_name", ["_finish_prefill", "_finish_decode"]) - def test_run_cfm_called_with_cfg_value_kwarg(self, talker_class_ast, method_name): - calls = self._run_cfm_calls_in(talker_class_ast, method_name) - assert calls, f"{method_name} must call self._run_cfm at least once" - for call in calls: - kwarg_names = {kw.arg for kw in call.keywords if kw.arg} - assert "cfg_value" in kwarg_names, ( - f"{method_name}: every self._run_cfm call must thread the " - f"per-request `cfg_value` kwarg (the bug fixed in #3118 P1 was " - f"that `_finish_decode` dropped this kwarg, so all decode steps " - f"silently fell back to the talker default `self._cfg_value`)." - ) diff --git a/vllm_omni/entrypoints/openai/protocol/audio.py b/vllm_omni/entrypoints/openai/protocol/audio.py index ec3b4516e..5144e0744 100644 --- a/vllm_omni/entrypoints/openai/protocol/audio.py +++ b/vllm_omni/entrypoints/openai/protocol/audio.py @@ -83,12 +83,7 @@ class OpenAICreateSpeechRequest(BaseModel): ) extra_params: dict[str, Any] | None = Field( default=None, - description=( - "Optional model-specific parameters passed directly to the model's extra_args. " - "VoxCPM2 reads `cfg_value` (float in 0.1-10.0; default 2.0) from this dict to set " - "the classifier-free guidance scale per request; Voxtral TTS reads `cfg_alpha` here " - "the same way." - ), + description=("Optional model-specific parameters passed directly to the model's extra_args."), ) @field_validator("stream_format") diff --git a/vllm_omni/entrypoints/openai/serving_speech.py b/vllm_omni/entrypoints/openai/serving_speech.py index d17b13516..e1e165637 100644 --- a/vllm_omni/entrypoints/openai/serving_speech.py +++ b/vllm_omni/entrypoints/openai/serving_speech.py @@ -492,39 +492,15 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): ref_sr = None if request.ref_audio is not None: ref_audio, ref_sr = await self._resolve_ref_audio(request.ref_audio) - # VoxCPM2 uses a text-prefix convention for style/voice instructions: - # the native CLI wraps the instruction in parens and prepends it. - # Mirror that here so `instructions` works through the OpenAI endpoint. - # The Hi-Fi Cloning mode (ref_audio + ref_text) + instructions combo - # is rejected upstream in `_validate_tts_request`, so by the time we - # reach this builder we can assume the combination is valid. - text = request.input - if request.instructions: - text = f"({request.instructions.strip()}){text}" - prompt = build_voxcpm2_prompt( + return build_voxcpm2_prompt( hf_config=self.engine_client.model_config.hf_config, tokenizer=self._voxcpm2_tokenizer, split_map=self._voxcpm2_split_map, - text=text, + text=request.input, ref_audio=ref_audio, ref_sr=ref_sr, ref_text=request.ref_text, ) - # Per-request cfg_value threads via additional_information; the talker - # lifts it onto _RequestState in preprocess() and uses it in _run_cfm. - # Following the convention from #2338 (Voxtral TTS / cfg_alpha), the - # value is provided via the shared `extra_params` dict rather than as - # a top-level protocol field, keeping OpenAICreateSpeechRequest free - # of model-specific knobs. - if request.extra_params and "cfg_value" in request.extra_params: - try: - cfg_value = float(request.extra_params["cfg_value"]) - except (TypeError, ValueError) as e: - raise ValueError(f"extra_params['cfg_value'] must be a number: {e}") from e - if not 0.1 <= cfg_value <= 10.0: - raise ValueError(f"extra_params['cfg_value']={cfg_value} out of range (0.1-10.0)") - prompt.setdefault("additional_information", {})["cfg_value"] = cfg_value - return prompt def _get_uploaded_audio_data(self, voice_name: str) -> str | None: """Get base64 encoded audio data for uploaded voice.""" @@ -880,27 +856,7 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): if self._tts_model_type == "voxcpm": return self._validate_voxcpm_request(request) if self._tts_model_type == "voxcpm2": - # Hi-Fi Cloning mode (`ref_audio` + `ref_text`) does not use the - # control instruction. Upstream voxcpm CLI errors out on the same - # combination (src/voxcpm/cli.py:128-131: --control cannot be used - # together with --prompt-text); we surface a 400 to match instead - # of silently dropping user input. - if request.instructions and request.ref_text: - return ( - "`instructions` is incompatible with `ref_text` in Hi-Fi Cloning " - "mode (the control instruction is ignored when ref_text is set)." - ) - # Cap `instructions` length for parity with qwen3_tts / voxtral / - # fish_tts (prevents blowing the prefill budget via an unbounded - # parenthetical prefix). Upstream voxcpm has no cap; this is a - # serving-side safety net, see PR #3118 discussion. - if request.instructions and len(request.instructions) > self._max_instructions_length: - return ( - f"Instructions too long ({len(request.instructions)}/" - f"{self._max_instructions_length} chars). Upstream voxcpm has no cap; " - f"this limit prevents exceeding the prefill budget." - ) - return None + return None # VoxCPM2 accepts any text input if self._tts_model_type == "ming_flash_omni_tts": return self._validate_ming_tts_request(request) if self._tts_model_type == "moss_tts_nano": @@ -1761,12 +1717,6 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): if request.instructions: prompt["instruct"] = request.instructions elif self._tts_model_type == "voxcpm2": - # Run the same validator the batch endpoint uses so single-request - # /v1/audio/speech enforces the instructions-length cap and any - # other voxcpm2-specific checks (e.g. Hi-Fi mode + instructions). - validation_error = self._validate_tts_request(request) - if validation_error: - raise ValueError(validation_error) prompt = await self._build_voxcpm2_prompt(request) tts_params = {} elif self._is_tts: diff --git a/vllm_omni/model_executor/models/voxcpm2/voxcpm2_talker.py b/vllm_omni/model_executor/models/voxcpm2/voxcpm2_talker.py index a3c98842a..0a9246251 100644 --- a/vllm_omni/model_executor/models/voxcpm2/voxcpm2_talker.py +++ b/vllm_omni/model_executor/models/voxcpm2/voxcpm2_talker.py @@ -185,8 +185,6 @@ class _RequestState: prefill_masks: tuple | None = None is_stopping: bool = False last_decoded_audio: torch.Tensor | None = None - # Per-request classifier-free guidance override (None -> talker default). - cfg_value: float | None = None @dataclasses.dataclass @@ -892,8 +890,7 @@ class VoxCPM2TalkerForConditionalGeneration(nn.Module): res_input = tts.fusion_concat_proj(torch.cat([lm_h, prev], dim=-1)) return res_input, {"new_lm_hidden": lm_h} - def _run_cfm(self, dit_h: torch.Tensor, cond: torch.Tensor, cfg_value: float | None = None) -> torch.Tensor: - cfg = self._cfg_value if cfg_value is None else float(cfg_value) + def _run_cfm(self, dit_h: torch.Tensor, cond: torch.Tensor) -> torch.Tensor: if self._cfm_buffers is not None: return _optimized_solve_euler( self.tts.feat_decoder, @@ -901,7 +898,7 @@ class VoxCPM2TalkerForConditionalGeneration(nn.Module): self._patch_size, cond, self._inference_timesteps, - cfg, + self._cfg_value, self._cfm_buffers, cfg_cutoff_ratio=self._cfg_cutoff_ratio, perf=self._perf, @@ -911,7 +908,7 @@ class VoxCPM2TalkerForConditionalGeneration(nn.Module): patch_size=self._patch_size, cond=cond, n_timesteps=self._inference_timesteps, - cfg_value=cfg, + cfg_value=self._cfg_value, ).transpose(1, 2) def _finish_prefill(self, state: _RequestState, meta: dict, res_out: torch.Tensor, dev: Any): @@ -927,7 +924,7 @@ class VoxCPM2TalkerForConditionalGeneration(nn.Module): if self._enable_torch_compile: self._setup_torch_compile() - pred_feat = self._run_cfm(dit_h, prefix_feat_cond.transpose(1, 2).contiguous(), cfg_value=state.cfg_value) + pred_feat = self._run_cfm(dit_h, prefix_feat_cond.transpose(1, 2).contiguous()) with torch.no_grad(): curr_embed = tts.enc_to_lm_proj(tts.feat_encoder(pred_feat.unsqueeze(1))).squeeze(1) @@ -960,7 +957,7 @@ class VoxCPM2TalkerForConditionalGeneration(nn.Module): if pfc.ndim == 2: pfc = pfc.unsqueeze(0) - pred_feat = self._run_cfm(dit_h, pfc.transpose(1, 2).contiguous(), cfg_value=state.cfg_value) + pred_feat = self._run_cfm(dit_h, pfc.transpose(1, 2).contiguous()) next_embed = tts.enc_to_lm_proj(tts.feat_encoder(pred_feat.unsqueeze(1))).squeeze(1) state.precomputed_stop_logits = stop_fn(lm_h).detach() @@ -1143,9 +1140,6 @@ class VoxCPM2TalkerForConditionalGeneration(nn.Module): state.is_stopping = False state.last_decoded_audio = None - # Per-request cfg_value override (threaded via additional_information). - state.cfg_value = info_dict.get("cfg_value") - # Voice clone / continuation ref_audio = info_dict.get("reference_audio") or info_dict.get("ref_audio") prompt_audio = info_dict.get("prompt_audio")