[Perf] Optimize VoxCPM2 first-request latency via startup warmup (#3424)
Signed-off-by: Dan250124 <416947747@qq.com>
This commit is contained in:
3 files changed
+95
No files matched your search
@@ -51,6 +51,40 @@ def voxcpm_server(mocker: MockerFixture):
|
|||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture
|
||||||
|
def voxcpm2_server(mocker: MockerFixture):
|
||||||
|
mocker.patch.object(OmniOpenAIServingSpeech, "_load_codec_frame_rate", return_value=None)
|
||||||
|
|
||||||
|
mock_engine_client = mocker.MagicMock()
|
||||||
|
mock_engine_client.errored = False
|
||||||
|
mock_engine_client.model_config = mocker.MagicMock(model="OpenBMB/VoxCPM2")
|
||||||
|
mock_engine_client.default_sampling_params_list = [SimpleNamespace(max_tokens=2048)]
|
||||||
|
mock_engine_client.tts_batch_max_items = 32
|
||||||
|
mock_engine_client.generate = mocker.MagicMock(return_value="generator")
|
||||||
|
mock_engine_client.stage_configs = [
|
||||||
|
SimpleNamespace(
|
||||||
|
engine_args=SimpleNamespace(
|
||||||
|
model_stage="latent_generator",
|
||||||
|
model_arch="VoxCPM2TalkerForConditionalGeneration",
|
||||||
|
),
|
||||||
|
tts_args={},
|
||||||
|
),
|
||||||
|
SimpleNamespace(
|
||||||
|
engine_args=SimpleNamespace(model_stage="vae"),
|
||||||
|
tts_args={},
|
||||||
|
),
|
||||||
|
]
|
||||||
|
|
||||||
|
mock_models = mocker.MagicMock()
|
||||||
|
mock_models.is_base_model.return_value = True
|
||||||
|
|
||||||
|
return OmniOpenAIServingSpeech(
|
||||||
|
engine_client=mock_engine_client,
|
||||||
|
models=mock_models,
|
||||||
|
request_logger=mocker.MagicMock(),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
class TestVoxCPMServing:
|
class TestVoxCPMServing:
|
||||||
def test_voxcpm_model_type_detection(self, voxcpm_server):
|
def test_voxcpm_model_type_detection(self, voxcpm_server):
|
||||||
assert voxcpm_server._tts_model_type == "voxcpm"
|
assert voxcpm_server._tts_model_type == "voxcpm"
|
||||||
@@ -143,3 +177,18 @@ class TestVoxCPMServing:
|
|||||||
"additional_information": tts_params,
|
"additional_information": tts_params,
|
||||||
"type": "token",
|
"type": "token",
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
class TestVoxCPM2Serving:
|
||||||
|
"""Regression tests for VoxCPM2 serving behavior."""
|
||||||
|
|
||||||
|
def test_voxcpm2_model_type_detection(self, voxcpm2_server):
|
||||||
|
assert voxcpm2_server._tts_model_type == "voxcpm2"
|
||||||
|
assert voxcpm2_server._is_tts is True
|
||||||
|
|
||||||
|
def test_voxcpm2_default_voice_is_supported(self, voxcpm2_server):
|
||||||
|
"""VoxCPM2 should accept voice='default' (zero-shot mode, no voice cloning).
|
||||||
|
|
||||||
|
Regression: previously returned 400 Invalid voice 'default'.
|
||||||
|
"""
|
||||||
|
assert "default" in voxcpm2_server.supported_speakers
|
||||||
@@ -916,6 +916,10 @@ async def omni_init_app_state(
|
|||||||
engine_client, state.openai_serving_models, request_logger=request_logger, model_name=model_name
|
engine_client, state.openai_serving_models, request_logger=request_logger, model_name=model_name
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# Warm up speech pipeline (CUDA Graph capture, torch.compile) so the first
|
||||||
|
# real user request is fast instead of paying a 100s compilation tax.
|
||||||
|
await state.openai_serving_speech.warmup()
|
||||||
|
|
||||||
state.openai_serving_audio_generate = OmniOpenAIServingAudioGenerate(
|
state.openai_serving_audio_generate = OmniOpenAIServingAudioGenerate(
|
||||||
engine_client, state.openai_serving_models, request_logger=request_logger, model_name=model_name
|
engine_client, state.openai_serving_models, request_logger=request_logger, model_name=model_name
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -359,6 +359,46 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
|
|||||||
self._build_fish_speech_prompt_async = make_async(self._build_fish_speech_prompt, executor=self._tts_executor)
|
self._build_fish_speech_prompt_async = make_async(self._build_fish_speech_prompt, executor=self._tts_executor)
|
||||||
self._estimate_prompt_len_async = make_async(self._estimate_prompt_len, executor=self._tts_executor)
|
self._estimate_prompt_len_async = make_async(self._estimate_prompt_len, executor=self._tts_executor)
|
||||||
|
|
||||||
|
async def warmup(self) -> None:
|
||||||
|
"""Run a synthetic speech request to trigger all first-request warmup.
|
||||||
|
|
||||||
|
Unlike qwen3-tts, whose CUDA Graph warmup targets a standalone tokenizer
|
||||||
|
decoder (no vLLM dependencies) and can complete entirely at model-init
|
||||||
|
time, VoxCPM2 needs to warm up PagedAttention scaffold/residual LLMs.
|
||||||
|
Their CUDA Graph capture requires a vLLM ``ForwardContext``
|
||||||
|
(attn_metadata, slot_mapping, etc.) that only exists during real
|
||||||
|
inference steps. The same request also pays the one-time torch.compile
|
||||||
|
JIT tax for the LocDiT estimator, feat_encoder, AudioVAE decoder, and
|
||||||
|
projection helpers.
|
||||||
|
|
||||||
|
For VoxCPM2 this shifts ~15s of torch.compile + CUDA Graph capture from
|
||||||
|
the first user request to server startup.
|
||||||
|
"""
|
||||||
|
if self._tts_model_type != "voxcpm2":
|
||||||
|
return
|
||||||
|
|
||||||
|
t0 = time.time()
|
||||||
|
logger.info("Running warmup speech request for model_type=%s", self._tts_model_type)
|
||||||
|
# VoxCPM2 has no predefined speaker presets — "default" means zero-shot
|
||||||
|
# mode (no voice cloning). The voice field is required by the OpenAI
|
||||||
|
# API schema but semantically ignored by the model.
|
||||||
|
warmup_req = OpenAICreateSpeechRequest(
|
||||||
|
input="Warmup.",
|
||||||
|
voice="default",
|
||||||
|
response_format="wav",
|
||||||
|
speed=1.0,
|
||||||
|
stream=False,
|
||||||
|
model=self.model_name,
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
_audio_bytes, _media_type = await self._generate_audio_bytes(warmup_req, request_id="speech-warmup")
|
||||||
|
except Exception as exc:
|
||||||
|
logger.warning("Speech warmup failed (non-fatal): %s", exc)
|
||||||
|
return
|
||||||
|
|
||||||
|
elapsed = time.time() - t0
|
||||||
|
logger.info("Speech warmup complete in %.1fs", elapsed)
|
||||||
|
|
||||||
def _get_qwen_tts_expected_speaker_embedding_dim(self) -> int | None:
|
def _get_qwen_tts_expected_speaker_embedding_dim(self) -> int | None:
|
||||||
"""Return the loaded Qwen3-TTS speaker embedding dim, if known.
|
"""Return the loaded Qwen3-TTS speaker embedding dim, if known.
|
||||||
|
|
||||||
@@ -491,6 +531,8 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
|
|||||||
try:
|
try:
|
||||||
if self._tts_model_type == "voxcpm":
|
if self._tts_model_type == "voxcpm":
|
||||||
return set()
|
return set()
|
||||||
|
if self._tts_model_type == "voxcpm2":
|
||||||
|
return {"default"}
|
||||||
if self._tts_model_type == "voxtral_tts":
|
if self._tts_model_type == "voxtral_tts":
|
||||||
config = self.engine_client.model_config.hf_config.audio_config
|
config = self.engine_client.model_config.hf_config.audio_config
|
||||||
else:
|
else:
|
||||||
|
|||||||
Reference in new issue
Block a user