diff --git a/docs/serving/speech_api.md b/docs/serving/speech_api.md index 8f78d6a20..209d98cfe 100644 --- a/docs/serving/speech_api.md +++ b/docs/serving/speech_api.md @@ -5,6 +5,7 @@ vLLM-Omni provides an OpenAI-compatible API for text-to-speech (TTS) generation. - **Qwen3-TTS** (`Qwen/Qwen3-TTS-12Hz-*`) -- Qwen3-based TTS with CustomVoice, VoiceDesign, and Base (voice cloning) task types. Output: 24 kHz. - **Fish Speech S2 Pro** (`fishaudio/s2-pro`) -- Dual-AR TTS with DAC codec. Supports text-to-speech and voice cloning via reference audio. Output: 44.1 kHz. - **Voxtral TTS** (`mistralai/Voxtral-4B-TTS-2603`) -- AR + FlowMatching TTS with preset voices. Output: 24 kHz. +- **VoxCPM2** (`openbmb/VoxCPM2`) -- Single-stage AR + DiT/CFM TTS. Three synthesis modes: Voice Design (text-only voice description), Controllable Cloning (reference audio + style instructions), and Hi-Fi Cloning (reference audio + transcript for tightest identity match; aka "Ultimate Cloning" in the Gradio demo). Output: 48 kHz. Each server instance runs a single model (specified at startup via `vllm serve --omni`). @@ -26,6 +27,9 @@ vllm serve fishaudio/s2-pro --omni --port 8091 # Voxtral TTS vllm serve mistralai/Voxtral-4B-TTS-2603 --omni --port 8091 + +# VoxCPM2 +vllm serve openbmb/VoxCPM2 --omni --port 8091 ``` ### Generate Speech @@ -119,6 +123,16 @@ Content-Type: application/json | `ref_text` | string | null | Transcript of reference audio | | `x_vector_only_mode` | bool | null | Use speaker embedding only (no ICL) | +#### VoxCPM2-specific Parameters + +VoxCPM2 reuses the shared `extra_params: dict` field (introduced in #2338 for Voxtral TTS) for model-specific knobs rather than adding new top-level fields: + +| Key (inside `extra_params`) | Type | Default | Description | +|-----------------------------|------|---------|-------------| +| `cfg_value` | float | 2.0 | Classifier-free guidance scale for the CFM decoder. Range `0.1`–`10.0`; typical useful range is `1.5`–`3.0`. Higher values track the text more strictly at the cost of naturalness. | + +Send as e.g. `{"input": "...", "extra_params": {"cfg_value": 2.5}}`. VoxCPM2 also reuses the standard `instructions`, `ref_audio`, and `ref_text` fields, see the mode table below for how they combine. + ### Response Format Returns binary audio data with appropriate `Content-Type` header (e.g., `audio/wav`). @@ -543,6 +557,64 @@ Fish Speech uses `ref_audio` and `ref_text` for voice cloning (no `task_type` ne |-------|-------------| | `mistralai/Voxtral-4B-TTS-2603` | 3B AR + FlowMatching TTS. Supports text-to-speech with preset voices. | +### VoxCPM2 + +| Model | Description | +|-------|-------------| +| `openbmb/VoxCPM2` | Single-stage AR + DiT/CFM TTS. 48 kHz output, multilingual (primary EN/ZH, usable cross-lingual identity transfer via Ultimate Cloning). | + +VoxCPM2 exposes three synthesis modes through the same `/v1/audio/speech` endpoint — the mode is inferred from which fields are present in the request. + +| Mode | Required fields | Optional fields | Notes | +|------|-----------------|-----------------|-------| +| **Voice Design** | `input` | `instructions`, `extra_params.cfg_value` | Synthesize a voice from a text description alone. `instructions` is prepended to the target text as `(instructions)input` following `voxcpm.cli.build_final_text`. Both English and Chinese instructions are supported. | +| **Controllable Cloning** | `input`, `ref_audio` | `instructions`, `extra_params.cfg_value` | Clone the reference speaker's timbre; use `instructions` to steer emotion, pace, or other style attributes while preserving identity. | +| **Hi-Fi Cloning** | `input`, `ref_audio`, `ref_text` | `extra_params.cfg_value` | The model treats `ref_audio` + `ref_text` as a spoken prefix and continues from it. Reproduces every vocal nuance of the reference. Per the canonical VoxCPM2 docs, when Hi-Fi mode is enabled the control instruction is ignored; the server drops `instructions` in this mode automatically if present. | + +Reference audio guidelines: 16 kHz mono, at least 5 seconds recommended for a good clone (1 second hard minimum, 30 seconds hard maximum). Clean, non-reverberant speech; the same `ref_audio` can be reused across Controllable and Hi-Fi modes. + +Inline non-verbal tags in `input` are passed through as regular text and recognized by the model when present (for example `[laughing]`, `[sigh]`, `[Uhm]`, `[Question-ah]`). See the upstream [VoxCPM2 cookbook](https://voxcpm.readthedocs.io/en/latest/cookbook.html) for the full tag list and usage tips. + +**Examples** + +Voice Design (no reference, pure text description): + +```bash +curl -X POST http://localhost:8091/v1/audio/speech \ + -H "Content-Type: application/json" \ + -d '{ + "input": "Hello everyone. Today I want to talk about open source AI.", + "instructions": "A warm young woman with a gentle, friendly voice", + "extra_params": {"cfg_value": 2.5} + }' --output voice_design.wav +``` + +Controllable Cloning (reference audio + style instructions): + +```bash +curl -X POST http://localhost:8091/v1/audio/speech \ + -H "Content-Type: application/json" \ + -d '{ + "input": "Good morning, let me tell you a short story.", + "ref_audio": "https://example.com/reference.wav", + "instructions": "Warm storyteller, slow cadence, dramatic pauses", + "extra_params": {"cfg_value": 2.5} + }' --output controllable_cloning.wav +``` + +Hi-Fi Cloning (reference audio + transcript; tightest identity): + +```bash +curl -X POST http://localhost:8091/v1/audio/speech \ + -H "Content-Type: application/json" \ + -d '{ + "input": "Welcome everyone. I will give a brief update on the project today.", + "ref_audio": "https://example.com/reference.wav", + "ref_text": "Transcript of what the reference speaker is saying in the reference clip.", + "extra_params": {"cfg_value": 2.5} + }' --output ultimate_cloning.wav +``` + ## Error Responses ### 400 Bad Request diff --git a/tests/entrypoints/openai_api/test_serving_speech_voxcpm2.py b/tests/entrypoints/openai_api/test_serving_speech_voxcpm2.py new file mode 100644 index 000000000..273648365 --- /dev/null +++ b/tests/entrypoints/openai_api/test_serving_speech_voxcpm2.py @@ -0,0 +1,330 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +"""UTs for VoxCPM2 OpenAI speech serving behavior (`instructions` + `cfg_value`).""" + +import asyncio +from types import SimpleNamespace +from unittest.mock import AsyncMock + +import pytest +from pytest_mock import MockerFixture + +from vllm_omni.entrypoints.openai.protocol.audio import OpenAICreateSpeechRequest +from vllm_omni.entrypoints.openai.serving_speech import OmniOpenAIServingSpeech + +pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.omni] + + +@pytest.fixture +def voxcpm2_server(mocker: MockerFixture): + """Mock an OmniOpenAIServingSpeech instance detected as `voxcpm2`. + + Single `latent_generator` stage with `model_arch=VoxCPM2...` so + `_detect_tts_model_type` returns ``voxcpm2``. + """ + mocker.patch.object(OmniOpenAIServingSpeech, "_load_supported_speakers", return_value=set()) + mocker.patch.object(OmniOpenAIServingSpeech, "_load_codec_frame_rate", return_value=None) + + mock_engine_client = mocker.MagicMock() + mock_engine_client.errored = False + mock_engine_client.model_config = mocker.MagicMock( + model="openbmb/VoxCPM2", + hf_config=SimpleNamespace( + audio_vae_config={"sample_rate": 16000, "encoder_rates": [2, 5, 8, 8]}, + patch_size=4, + ), + ) + mock_engine_client.default_sampling_params_list = [SimpleNamespace(max_tokens=4096)] + mock_engine_client.tts_batch_max_items = 32 + # Explicit None so `_compute_max_instructions_length` skips the CLI-override + # branch (MagicMock attributes are truthy by default) and falls through to + # the `_TTS_MAX_INSTRUCTIONS_LENGTH = 500` default. + mock_engine_client.tts_max_instructions_length = None + mock_engine_client.generate = mocker.MagicMock(return_value="generator") + mock_engine_client.stage_configs = [ + SimpleNamespace( + engine_args=SimpleNamespace( + model_stage="latent_generator", + model_arch="VoxCPM2TalkerForConditionalGeneration", + ), + tts_args={}, + ), + ] + + mock_models = mocker.MagicMock() + mock_models.is_base_model.return_value = True + + server = OmniOpenAIServingSpeech( + engine_client=mock_engine_client, + models=mock_models, + request_logger=mocker.MagicMock(), + ) + # Lazy tokenizer path is not exercised by these tests; provide a no-op encoder + # and sentinel tokenizer / split_map so `_build_voxcpm2_prompt` can run. + server._voxcpm2_tokenizer = mocker.MagicMock(name="voxcpm2_tokenizer") + server._voxcpm2_split_map = {} + mocker.patch.object(server, "_voxcpm2_encode", return_value=[]) + return server + + +@pytest.fixture +def mock_build_voxcpm2_prompt(mocker: MockerFixture): + """Patch `build_voxcpm2_prompt` so we can inspect its call args without + touching the real VoxCPM2 tokenizer / audio VAE. + + Returns the prompt dict shape the real helper would produce; the test + can then also assert `cfg_value` stashing into `additional_information`. + """ + return mocker.patch( + "vllm_omni.model_executor.models.voxcpm2.voxcpm2_talker.build_voxcpm2_prompt", + return_value={"prompt_token_ids": [1], "additional_information": {"text_token_ids": [[]]}}, + ) + + +class TestVoxCPM2Serving: + def test_voxcpm2_model_type_detection(self, voxcpm2_server): + assert voxcpm2_server._tts_model_type == "voxcpm2" + assert voxcpm2_server._is_tts is True + + def test_voxcpm2_accepts_any_text_input(self, voxcpm2_server): + """VoxCPM2 skips the strict voxcpm validator (see `_validate_tts_request`).""" + request = OpenAICreateSpeechRequest(input="مرحباً", instructions="warm tone") + assert voxcpm2_server._validate_tts_request(request) is None + + def test_build_prompt_text_only(self, voxcpm2_server, mock_build_voxcpm2_prompt): + """No instructions, no cfg_value: text flows through unchanged, no cfg stash.""" + request = OpenAICreateSpeechRequest(input="hello voxcpm2") + prompt = asyncio.run(voxcpm2_server._build_voxcpm2_prompt(request)) + + assert mock_build_voxcpm2_prompt.call_args.kwargs["text"] == "hello voxcpm2" + assert "cfg_value" not in prompt.get("additional_information", {}) + + def test_build_prompt_prepends_instructions(self, voxcpm2_server, mock_build_voxcpm2_prompt): + """`instructions` wraps in parens and prepends to text (native VoxCPM2 convention).""" + request = OpenAICreateSpeechRequest( + input="hello voxcpm2", + instructions="A warm young woman", + ) + asyncio.run(voxcpm2_server._build_voxcpm2_prompt(request)) + + assert mock_build_voxcpm2_prompt.call_args.kwargs["text"] == "(A warm young woman)hello voxcpm2" + + def test_build_prompt_strips_instructions_whitespace(self, voxcpm2_server, mock_build_voxcpm2_prompt): + """Leading/trailing whitespace in `instructions` is stripped before prepending.""" + request = OpenAICreateSpeechRequest(input="hello", instructions=" calm radio ") + asyncio.run(voxcpm2_server._build_voxcpm2_prompt(request)) + + assert mock_build_voxcpm2_prompt.call_args.kwargs["text"] == "(calm radio)hello" + + def test_build_prompt_stashes_cfg_value(self, voxcpm2_server, mock_build_voxcpm2_prompt): + """`extra_params['cfg_value']` lands in `additional_information` for the talker to lift.""" + request = OpenAICreateSpeechRequest(input="hello", extra_params={"cfg_value": 2.7}) + prompt = asyncio.run(voxcpm2_server._build_voxcpm2_prompt(request)) + + assert prompt["additional_information"]["cfg_value"] == 2.7 + + def test_build_prompt_omits_cfg_value_when_extra_params_missing(self, voxcpm2_server, mock_build_voxcpm2_prompt): + """Omitting `cfg_value` must not add the key (talker falls back to its default).""" + request = OpenAICreateSpeechRequest(input="hello") + prompt = asyncio.run(voxcpm2_server._build_voxcpm2_prompt(request)) + + assert "cfg_value" not in prompt["additional_information"] + + def test_build_prompt_omits_cfg_value_when_extra_params_has_other_keys( + self, voxcpm2_server, mock_build_voxcpm2_prompt + ): + """`extra_params` set but without `cfg_value` must not affect the prompt.""" + request = OpenAICreateSpeechRequest(input="hello", extra_params={"some_other_knob": 1}) + prompt = asyncio.run(voxcpm2_server._build_voxcpm2_prompt(request)) + + assert "cfg_value" not in prompt["additional_information"] + + def test_build_prompt_instructions_and_cfg_together(self, voxcpm2_server, mock_build_voxcpm2_prompt): + """Both features compose: text prefixed AND cfg_value stashed.""" + request = OpenAICreateSpeechRequest( + input="hello", + instructions="excited", + extra_params={"cfg_value": 2.5}, + ) + prompt = asyncio.run(voxcpm2_server._build_voxcpm2_prompt(request)) + + assert mock_build_voxcpm2_prompt.call_args.kwargs["text"] == "(excited)hello" + assert prompt["additional_information"]["cfg_value"] == 2.5 + + def test_build_prompt_hifi_cloning_ref_audio_ref_text_cfg( + self, voxcpm2_server, mock_build_voxcpm2_prompt, mocker: MockerFixture + ): + """Hi-Fi Cloning (`ref_audio` + `ref_text`) resolves audio and threads cfg_value.""" + voxcpm2_server._resolve_ref_audio = AsyncMock(return_value=([0.1, -0.1, 0.2], 16000)) + + request = OpenAICreateSpeechRequest( + input="clone me", + ref_audio="data:audio/wav;base64,QUJD", + ref_text="reference transcript", + extra_params={"cfg_value": 2.7}, + ) + prompt = asyncio.run(voxcpm2_server._build_voxcpm2_prompt(request)) + + kwargs = mock_build_voxcpm2_prompt.call_args.kwargs + assert kwargs["text"] == "clone me" + assert kwargs["ref_audio"] == [0.1, -0.1, 0.2] + assert kwargs["ref_sr"] == 16000 + assert kwargs["ref_text"] == "reference transcript" + assert prompt["additional_information"]["cfg_value"] == 2.7 + voxcpm2_server._resolve_ref_audio.assert_awaited_once_with("data:audio/wav;base64,QUJD") + + def test_validate_rejects_instructions_with_ref_text(self, voxcpm2_server): + """Hi-Fi Cloning (ref_audio + ref_text) + instructions returns a 400-level + error string from the validator instead of silently dropping `instructions`. + + Matches upstream voxcpm CLI's hard error on the same combo + (src/voxcpm/cli.py:128-131: --control + --prompt-text). + """ + request = OpenAICreateSpeechRequest( + input="hello", + ref_audio="data:audio/wav;base64,QUJD", + ref_text="reference transcript", + instructions="this is incompatible with Hi-Fi mode", + ) + error = voxcpm2_server._validate_tts_request(request) + assert error is not None + assert "Hi-Fi" in error + assert "ref_text" in error + + def test_validate_accepts_instructions_without_ref_text(self, voxcpm2_server): + """`instructions` is fine in Voice Design and Controllable Cloning modes + (i.e. whenever `ref_text` is not also set).""" + request = OpenAICreateSpeechRequest(input="hello", instructions="warm tone") + assert voxcpm2_server._validate_tts_request(request) is None + + def test_validate_accepts_ref_text_without_instructions(self, voxcpm2_server): + """Hi-Fi Cloning (ref_audio + ref_text, no instructions) is fine.""" + request = OpenAICreateSpeechRequest( + input="hello", + ref_audio="data:audio/wav;base64,QUJD", + ref_text="reference transcript", + ) + assert voxcpm2_server._validate_tts_request(request) is None + + def test_validate_rejects_overlong_instructions(self, voxcpm2_server): + """Instructions longer than `_max_instructions_length` (500 default) are rejected + with an error message that includes the actual length, the cap, and an + upstream-context hint so users coming from the voxcpm CLI understand why + the limit exists. + """ + oversize = "x" * (voxcpm2_server._max_instructions_length + 1) + request = OpenAICreateSpeechRequest(input="hello", instructions=oversize) + error = voxcpm2_server._validate_tts_request(request) + assert error is not None + assert str(len(oversize)) in error + assert str(voxcpm2_server._max_instructions_length) in error + assert "Upstream voxcpm has no cap" in error + + def test_validate_accepts_at_limit_instructions(self, voxcpm2_server): + """Instructions exactly at `_max_instructions_length` are accepted.""" + at_limit = "x" * voxcpm2_server._max_instructions_length + request = OpenAICreateSpeechRequest(input="hello", instructions=at_limit) + assert voxcpm2_server._validate_tts_request(request) is None + + def test_prepare_speech_generation_runs_validator_for_voxcpm2_length( + self, voxcpm2_server, mock_build_voxcpm2_prompt, mocker: MockerFixture + ): + """Single-request `/v1/audio/speech` path must invoke `_validate_tts_request` + before building the prompt, so the instructions-length cap actually fires + for normal (non-batch) requests too. Regression guard for #3118 P2. + """ + oversize = "x" * (voxcpm2_server._max_instructions_length + 1) + request = OpenAICreateSpeechRequest(input="hello", instructions=oversize) + with pytest.raises(ValueError, match=str(voxcpm2_server._max_instructions_length)): + asyncio.run(voxcpm2_server._prepare_speech_generation(request)) + # The prompt builder must NOT be reached when validation fails. + mock_build_voxcpm2_prompt.assert_not_called() + + def test_prepare_speech_generation_runs_validator_for_voxcpm2_hifi( + self, voxcpm2_server, mock_build_voxcpm2_prompt, mocker: MockerFixture + ): + """Single-request path must also surface the Hi-Fi mode rejection + (instructions + ref_text), not silently strip `instructions`. + """ + request = OpenAICreateSpeechRequest( + input="hello", + ref_audio="data:audio/wav;base64,QUJD", + ref_text="reference transcript", + instructions="incompatible with Hi-Fi mode", + ) + with pytest.raises(ValueError, match="Hi-Fi"): + asyncio.run(voxcpm2_server._prepare_speech_generation(request)) + mock_build_voxcpm2_prompt.assert_not_called() + + @pytest.mark.parametrize("cfg", [0.1, 0.5, 1.5, 2.0, 2.7, 5.0, 10.0]) + def test_cfg_value_accepts_range(self, voxcpm2_server, mock_build_voxcpm2_prompt, cfg): + """`_build_voxcpm2_prompt` accepts cfg_value within 0.1-10.0 inclusive.""" + request = OpenAICreateSpeechRequest(input="hello", extra_params={"cfg_value": cfg}) + prompt = asyncio.run(voxcpm2_server._build_voxcpm2_prompt(request)) + assert prompt["additional_information"]["cfg_value"] == cfg + + @pytest.mark.parametrize("cfg", [0.0, -1.0, 10.5, 100.0]) + def test_cfg_value_rejects_out_of_range(self, voxcpm2_server, mock_build_voxcpm2_prompt, cfg): + """Out-of-range `cfg_value` (validated in `_build_voxcpm2_prompt`) raises ValueError.""" + request = OpenAICreateSpeechRequest(input="hello", extra_params={"cfg_value": cfg}) + with pytest.raises(ValueError, match="out of range"): + asyncio.run(voxcpm2_server._build_voxcpm2_prompt(request)) + + @pytest.mark.parametrize("bad", ["abc", None, [1, 2], {"x": 1}]) + def test_cfg_value_rejects_non_numeric(self, voxcpm2_server, mock_build_voxcpm2_prompt, bad): + """Non-numeric `extra_params['cfg_value']` raises ValueError instead of crashing the talker.""" + request = OpenAICreateSpeechRequest(input="hello", extra_params={"cfg_value": bad}) + with pytest.raises(ValueError, match="must be a number|out of range"): + asyncio.run(voxcpm2_server._build_voxcpm2_prompt(request)) + + +class TestVoxCPM2DecodeStepCfgPropagation: + """Static AST regression guard for #3118 P1: ensure both `_finish_prefill` + and `_finish_decode` invoke `_run_cfm` with the per-request `cfg_value` + kwarg. Originally only `_finish_prefill` threaded the kwarg, so requests + longer than the prefill patch fell back to `self._cfg_value = 2.0` for + every decode step. This test parses the talker source and asserts the + kwarg is present at every `_run_cfm` call site inside the two methods, + so a future refactor that drops the kwarg from either method fails CI + even before the behavioral cfg-sweep tests run. + """ + + @pytest.fixture(scope="class") + def talker_class_ast(self): + import ast + import inspect + + from vllm_omni.model_executor.models.voxcpm2 import voxcpm2_talker + + src = inspect.getsource(voxcpm2_talker) + tree = ast.parse(src) + cls = next( + n + for n in ast.walk(tree) + if isinstance(n, ast.ClassDef) and n.name == "VoxCPM2TalkerForConditionalGeneration" + ) + return cls + + def _run_cfm_calls_in(self, cls_ast, method_name): + import ast + + method = next(n for n in cls_ast.body if isinstance(n, ast.FunctionDef) and n.name == method_name) + return [ + n + for n in ast.walk(method) + if isinstance(n, ast.Call) and isinstance(n.func, ast.Attribute) and n.func.attr == "_run_cfm" + ] + + @pytest.mark.parametrize("method_name", ["_finish_prefill", "_finish_decode"]) + def test_run_cfm_called_with_cfg_value_kwarg(self, talker_class_ast, method_name): + calls = self._run_cfm_calls_in(talker_class_ast, method_name) + assert calls, f"{method_name} must call self._run_cfm at least once" + for call in calls: + kwarg_names = {kw.arg for kw in call.keywords if kw.arg} + assert "cfg_value" in kwarg_names, ( + f"{method_name}: every self._run_cfm call must thread the " + f"per-request `cfg_value` kwarg (the bug fixed in #3118 P1 was " + f"that `_finish_decode` dropped this kwarg, so all decode steps " + f"silently fell back to the talker default `self._cfg_value`)." + ) diff --git a/vllm_omni/entrypoints/openai/protocol/audio.py b/vllm_omni/entrypoints/openai/protocol/audio.py index 5144e0744..ec3b4516e 100644 --- a/vllm_omni/entrypoints/openai/protocol/audio.py +++ b/vllm_omni/entrypoints/openai/protocol/audio.py @@ -83,7 +83,12 @@ class OpenAICreateSpeechRequest(BaseModel): ) extra_params: dict[str, Any] | None = Field( default=None, - description=("Optional model-specific parameters passed directly to the model's extra_args."), + description=( + "Optional model-specific parameters passed directly to the model's extra_args. " + "VoxCPM2 reads `cfg_value` (float in 0.1-10.0; default 2.0) from this dict to set " + "the classifier-free guidance scale per request; Voxtral TTS reads `cfg_alpha` here " + "the same way." + ), ) @field_validator("stream_format") diff --git a/vllm_omni/entrypoints/openai/serving_speech.py b/vllm_omni/entrypoints/openai/serving_speech.py index e1e165637..d17b13516 100644 --- a/vllm_omni/entrypoints/openai/serving_speech.py +++ b/vllm_omni/entrypoints/openai/serving_speech.py @@ -492,15 +492,39 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): ref_sr = None if request.ref_audio is not None: ref_audio, ref_sr = await self._resolve_ref_audio(request.ref_audio) - return build_voxcpm2_prompt( + # VoxCPM2 uses a text-prefix convention for style/voice instructions: + # the native CLI wraps the instruction in parens and prepends it. + # Mirror that here so `instructions` works through the OpenAI endpoint. + # The Hi-Fi Cloning mode (ref_audio + ref_text) + instructions combo + # is rejected upstream in `_validate_tts_request`, so by the time we + # reach this builder we can assume the combination is valid. + text = request.input + if request.instructions: + text = f"({request.instructions.strip()}){text}" + prompt = build_voxcpm2_prompt( hf_config=self.engine_client.model_config.hf_config, tokenizer=self._voxcpm2_tokenizer, split_map=self._voxcpm2_split_map, - text=request.input, + text=text, ref_audio=ref_audio, ref_sr=ref_sr, ref_text=request.ref_text, ) + # Per-request cfg_value threads via additional_information; the talker + # lifts it onto _RequestState in preprocess() and uses it in _run_cfm. + # Following the convention from #2338 (Voxtral TTS / cfg_alpha), the + # value is provided via the shared `extra_params` dict rather than as + # a top-level protocol field, keeping OpenAICreateSpeechRequest free + # of model-specific knobs. + if request.extra_params and "cfg_value" in request.extra_params: + try: + cfg_value = float(request.extra_params["cfg_value"]) + except (TypeError, ValueError) as e: + raise ValueError(f"extra_params['cfg_value'] must be a number: {e}") from e + if not 0.1 <= cfg_value <= 10.0: + raise ValueError(f"extra_params['cfg_value']={cfg_value} out of range (0.1-10.0)") + prompt.setdefault("additional_information", {})["cfg_value"] = cfg_value + return prompt def _get_uploaded_audio_data(self, voice_name: str) -> str | None: """Get base64 encoded audio data for uploaded voice.""" @@ -856,7 +880,27 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): if self._tts_model_type == "voxcpm": return self._validate_voxcpm_request(request) if self._tts_model_type == "voxcpm2": - return None # VoxCPM2 accepts any text input + # Hi-Fi Cloning mode (`ref_audio` + `ref_text`) does not use the + # control instruction. Upstream voxcpm CLI errors out on the same + # combination (src/voxcpm/cli.py:128-131: --control cannot be used + # together with --prompt-text); we surface a 400 to match instead + # of silently dropping user input. + if request.instructions and request.ref_text: + return ( + "`instructions` is incompatible with `ref_text` in Hi-Fi Cloning " + "mode (the control instruction is ignored when ref_text is set)." + ) + # Cap `instructions` length for parity with qwen3_tts / voxtral / + # fish_tts (prevents blowing the prefill budget via an unbounded + # parenthetical prefix). Upstream voxcpm has no cap; this is a + # serving-side safety net, see PR #3118 discussion. + if request.instructions and len(request.instructions) > self._max_instructions_length: + return ( + f"Instructions too long ({len(request.instructions)}/" + f"{self._max_instructions_length} chars). Upstream voxcpm has no cap; " + f"this limit prevents exceeding the prefill budget." + ) + return None if self._tts_model_type == "ming_flash_omni_tts": return self._validate_ming_tts_request(request) if self._tts_model_type == "moss_tts_nano": @@ -1717,6 +1761,12 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): if request.instructions: prompt["instruct"] = request.instructions elif self._tts_model_type == "voxcpm2": + # Run the same validator the batch endpoint uses so single-request + # /v1/audio/speech enforces the instructions-length cap and any + # other voxcpm2-specific checks (e.g. Hi-Fi mode + instructions). + validation_error = self._validate_tts_request(request) + if validation_error: + raise ValueError(validation_error) prompt = await self._build_voxcpm2_prompt(request) tts_params = {} elif self._is_tts: diff --git a/vllm_omni/model_executor/models/voxcpm2/voxcpm2_talker.py b/vllm_omni/model_executor/models/voxcpm2/voxcpm2_talker.py index 0a9246251..a3c98842a 100644 --- a/vllm_omni/model_executor/models/voxcpm2/voxcpm2_talker.py +++ b/vllm_omni/model_executor/models/voxcpm2/voxcpm2_talker.py @@ -185,6 +185,8 @@ class _RequestState: prefill_masks: tuple | None = None is_stopping: bool = False last_decoded_audio: torch.Tensor | None = None + # Per-request classifier-free guidance override (None -> talker default). + cfg_value: float | None = None @dataclasses.dataclass @@ -890,7 +892,8 @@ class VoxCPM2TalkerForConditionalGeneration(nn.Module): res_input = tts.fusion_concat_proj(torch.cat([lm_h, prev], dim=-1)) return res_input, {"new_lm_hidden": lm_h} - def _run_cfm(self, dit_h: torch.Tensor, cond: torch.Tensor) -> torch.Tensor: + def _run_cfm(self, dit_h: torch.Tensor, cond: torch.Tensor, cfg_value: float | None = None) -> torch.Tensor: + cfg = self._cfg_value if cfg_value is None else float(cfg_value) if self._cfm_buffers is not None: return _optimized_solve_euler( self.tts.feat_decoder, @@ -898,7 +901,7 @@ class VoxCPM2TalkerForConditionalGeneration(nn.Module): self._patch_size, cond, self._inference_timesteps, - self._cfg_value, + cfg, self._cfm_buffers, cfg_cutoff_ratio=self._cfg_cutoff_ratio, perf=self._perf, @@ -908,7 +911,7 @@ class VoxCPM2TalkerForConditionalGeneration(nn.Module): patch_size=self._patch_size, cond=cond, n_timesteps=self._inference_timesteps, - cfg_value=self._cfg_value, + cfg_value=cfg, ).transpose(1, 2) def _finish_prefill(self, state: _RequestState, meta: dict, res_out: torch.Tensor, dev: Any): @@ -924,7 +927,7 @@ class VoxCPM2TalkerForConditionalGeneration(nn.Module): if self._enable_torch_compile: self._setup_torch_compile() - pred_feat = self._run_cfm(dit_h, prefix_feat_cond.transpose(1, 2).contiguous()) + pred_feat = self._run_cfm(dit_h, prefix_feat_cond.transpose(1, 2).contiguous(), cfg_value=state.cfg_value) with torch.no_grad(): curr_embed = tts.enc_to_lm_proj(tts.feat_encoder(pred_feat.unsqueeze(1))).squeeze(1) @@ -957,7 +960,7 @@ class VoxCPM2TalkerForConditionalGeneration(nn.Module): if pfc.ndim == 2: pfc = pfc.unsqueeze(0) - pred_feat = self._run_cfm(dit_h, pfc.transpose(1, 2).contiguous()) + pred_feat = self._run_cfm(dit_h, pfc.transpose(1, 2).contiguous(), cfg_value=state.cfg_value) next_embed = tts.enc_to_lm_proj(tts.feat_encoder(pred_feat.unsqueeze(1))).squeeze(1) state.precomputed_stop_logits = stop_fn(lm_h).detach() @@ -1140,6 +1143,9 @@ class VoxCPM2TalkerForConditionalGeneration(nn.Module): state.is_stopping = False state.last_decoded_audio = None + # Per-request cfg_value override (threaded via additional_information). + state.cfg_value = info_dict.get("cfg_value") + # Voice clone / continuation ref_audio = info_dict.get("reference_audio") or info_dict.get("ref_audio") prompt_audio = info_dict.get("prompt_audio")