From c481ccee2b405e2a580b4f050cbc795cdb1e10ba Mon Sep 17 00:00:00 2001 From: Dan <416947747@qq.com> Date: Sat, 9 May 2026 06:44:19 +0800 Subject: [PATCH] [Perf] Optimize VoxCPM2 first-request latency via startup warmup (#3424) Signed-off-by: Dan250124 <416947747@qq.com> --- .../openai_api/test_serving_speech_voxcpm.py | 49 +++++++++++++++++++ vllm_omni/entrypoints/openai/api_server.py | 4 ++ .../entrypoints/openai/serving_speech.py | 42 ++++++++++++++++ 3 files changed, 95 insertions(+) diff --git a/tests/entrypoints/openai_api/test_serving_speech_voxcpm.py b/tests/entrypoints/openai_api/test_serving_speech_voxcpm.py index ebd4ef1ab..1f7279483 100644 --- a/tests/entrypoints/openai_api/test_serving_speech_voxcpm.py +++ b/tests/entrypoints/openai_api/test_serving_speech_voxcpm.py @@ -51,6 +51,40 @@ def voxcpm_server(mocker: MockerFixture): ) +@pytest.fixture +def voxcpm2_server(mocker: MockerFixture): + mocker.patch.object(OmniOpenAIServingSpeech, "_load_codec_frame_rate", return_value=None) + + mock_engine_client = mocker.MagicMock() + mock_engine_client.errored = False + mock_engine_client.model_config = mocker.MagicMock(model="OpenBMB/VoxCPM2") + mock_engine_client.default_sampling_params_list = [SimpleNamespace(max_tokens=2048)] + mock_engine_client.tts_batch_max_items = 32 + mock_engine_client.generate = mocker.MagicMock(return_value="generator") + mock_engine_client.stage_configs = [ + SimpleNamespace( + engine_args=SimpleNamespace( + model_stage="latent_generator", + model_arch="VoxCPM2TalkerForConditionalGeneration", + ), + tts_args={}, + ), + SimpleNamespace( + engine_args=SimpleNamespace(model_stage="vae"), + tts_args={}, + ), + ] + + mock_models = mocker.MagicMock() + mock_models.is_base_model.return_value = True + + return OmniOpenAIServingSpeech( + engine_client=mock_engine_client, + models=mock_models, + request_logger=mocker.MagicMock(), + ) + + class TestVoxCPMServing: def test_voxcpm_model_type_detection(self, voxcpm_server): assert voxcpm_server._tts_model_type == "voxcpm" @@ -143,3 +177,18 @@ class TestVoxCPMServing: "additional_information": tts_params, "type": "token", } + + +class TestVoxCPM2Serving: + """Regression tests for VoxCPM2 serving behavior.""" + + def test_voxcpm2_model_type_detection(self, voxcpm2_server): + assert voxcpm2_server._tts_model_type == "voxcpm2" + assert voxcpm2_server._is_tts is True + + def test_voxcpm2_default_voice_is_supported(self, voxcpm2_server): + """VoxCPM2 should accept voice='default' (zero-shot mode, no voice cloning). + + Regression: previously returned 400 Invalid voice 'default'. + """ + assert "default" in voxcpm2_server.supported_speakers diff --git a/vllm_omni/entrypoints/openai/api_server.py b/vllm_omni/entrypoints/openai/api_server.py index e58b490a7..380dde2e5 100644 --- a/vllm_omni/entrypoints/openai/api_server.py +++ b/vllm_omni/entrypoints/openai/api_server.py @@ -916,6 +916,10 @@ async def omni_init_app_state( engine_client, state.openai_serving_models, request_logger=request_logger, model_name=model_name ) + # Warm up speech pipeline (CUDA Graph capture, torch.compile) so the first + # real user request is fast instead of paying a 100s compilation tax. + await state.openai_serving_speech.warmup() + state.openai_serving_audio_generate = OmniOpenAIServingAudioGenerate( engine_client, state.openai_serving_models, request_logger=request_logger, model_name=model_name ) diff --git a/vllm_omni/entrypoints/openai/serving_speech.py b/vllm_omni/entrypoints/openai/serving_speech.py index 9b9a7f205..28b06601f 100644 --- a/vllm_omni/entrypoints/openai/serving_speech.py +++ b/vllm_omni/entrypoints/openai/serving_speech.py @@ -359,6 +359,46 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): self._build_fish_speech_prompt_async = make_async(self._build_fish_speech_prompt, executor=self._tts_executor) self._estimate_prompt_len_async = make_async(self._estimate_prompt_len, executor=self._tts_executor) + async def warmup(self) -> None: + """Run a synthetic speech request to trigger all first-request warmup. + + Unlike qwen3-tts, whose CUDA Graph warmup targets a standalone tokenizer + decoder (no vLLM dependencies) and can complete entirely at model-init + time, VoxCPM2 needs to warm up PagedAttention scaffold/residual LLMs. + Their CUDA Graph capture requires a vLLM ``ForwardContext`` + (attn_metadata, slot_mapping, etc.) that only exists during real + inference steps. The same request also pays the one-time torch.compile + JIT tax for the LocDiT estimator, feat_encoder, AudioVAE decoder, and + projection helpers. + + For VoxCPM2 this shifts ~15s of torch.compile + CUDA Graph capture from + the first user request to server startup. + """ + if self._tts_model_type != "voxcpm2": + return + + t0 = time.time() + logger.info("Running warmup speech request for model_type=%s", self._tts_model_type) + # VoxCPM2 has no predefined speaker presets — "default" means zero-shot + # mode (no voice cloning). The voice field is required by the OpenAI + # API schema but semantically ignored by the model. + warmup_req = OpenAICreateSpeechRequest( + input="Warmup.", + voice="default", + response_format="wav", + speed=1.0, + stream=False, + model=self.model_name, + ) + try: + _audio_bytes, _media_type = await self._generate_audio_bytes(warmup_req, request_id="speech-warmup") + except Exception as exc: + logger.warning("Speech warmup failed (non-fatal): %s", exc) + return + + elapsed = time.time() - t0 + logger.info("Speech warmup complete in %.1fs", elapsed) + def _get_qwen_tts_expected_speaker_embedding_dim(self) -> int | None: """Return the loaded Qwen3-TTS speaker embedding dim, if known. @@ -491,6 +531,8 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin): try: if self._tts_model_type == "voxcpm": return set() + if self._tts_model_type == "voxcpm2": + return {"default"} if self._tts_model_type == "voxtral_tts": config = self.engine_client.model_config.hf_config.audio_config else: