anodyine/vllm-omni
1
0 Fork 0
Code Issues 4 Pull Requests 2 Actions Packages Projects Releases Wiki Activity

[Doc][TTS] CosyVoice3 online docs + residual TTS yaml cleanup + remove VoxCPM v1 (#3748)

Browse Source 
Signed-off-by: Yueqian Lin <linyueqian@outlook.com>
This commit is contained in:
Yueqian Lin authored and GitHub committed 2026-05-20 15:24:06 +08:00
1 parent 1c530d984d
commit fc8486cf20
35 files changed
+248 -4147
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

tests/e2e/offline_inference/test_voxcpm_expansion.py
-99
View File
@@ -1,99 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""E2E test for VoxCPM offline inference."""
from typing import Any
import numpy as np
import pytest
import torch
from tests.helpers.mark import hardware_test
from tests.helpers.runtime import OmniRunner
from tests.helpers.stage_config import get_deploy_config_path
VOXCPM_MODEL = "OpenBMB/VoxCPM1.5"
STAGE_CONFIG = get_deploy_config_path("voxcpm.yaml")
SAMPLE_RATE = 24000
# (model, stage_config_path) for ``@pytest.mark.parametrize("omni_runner", ..., indirect=True)``
_OMNI_RUNNER_PARAM = (VOXCPM_MODEL, STAGE_CONFIG)
pytestmark = [
pytest.mark.full_model,
pytest.mark.tts,
pytest.mark.parametrize("omni_runner", [_OMNI_RUNNER_PARAM], indirect=True),
]
def _build_prompt(text: str) -> dict[str, Any]:
return {
"prompt_token_ids": [1],
"additional_information": {
"text": [text],
"cfg_value": [2.0],
"inference_timesteps": [10],
"min_len": [2],
"max_new_tokens": [1024],
},
}
def _extract_audio_tensor(multimodal_output: dict[str, Any]) -> torch.Tensor:
audio = multimodal_output.get("audio", multimodal_output.get("model_outputs"))
assert audio is not None, f"No audio output found, keys={list(multimodal_output.keys())}"
if isinstance(audio, list):
parts: list[torch.Tensor] = []
for item in audio:
if item is None:
continue
tensor = torch.as_tensor(item)
if tensor.numel() == 0:
continue
parts.append(tensor.float().cpu().reshape(-1))
return torch.cat(parts, dim=-1) if parts else torch.zeros((0,), dtype=torch.float32)
return torch.as_tensor(audio).float().cpu().reshape(-1)
def _extract_final_multimodal_output(outputs) -> dict[str, Any]:
for item in reversed(outputs):
request_output = getattr(item, "request_output", None)
if request_output is not None:
multimodal_output = getattr(request_output, "multimodal_output", None)
if isinstance(multimodal_output, dict):
return multimodal_output
completions = getattr(request_output, "outputs", None) or []
for completion in completions:
multimodal_output = getattr(completion, "multimodal_output", None)
if isinstance(multimodal_output, dict):
return multimodal_output
multimodal_output = getattr(item, "multimodal_output", None)
if isinstance(multimodal_output, dict):
return multimodal_output
raise AssertionError("No multimodal audio output found in VoxCPM generate results")
@hardware_test(res={"cuda": "L4"}, num_cards=1)
def test_voxcpm_zero_shot_001(omni_runner: OmniRunner) -> None:
outputs = list(omni_runner.omni.generate(_build_prompt("Hello, this is a VoxCPM offline inference test.")))
assert outputs, "No outputs returned"
multimodal_output = _extract_final_multimodal_output(outputs)
audio = _extract_audio_tensor(multimodal_output)
assert audio.numel() > SAMPLE_RATE // 2, f"Audio too short: {audio.numel()} samples"
duration_s = audio.shape[0] / SAMPLE_RATE
assert 0.5 < duration_s < 30.0, f"Audio duration out of range: {duration_s:.2f}s"
peak = float(torch.max(torch.abs(audio)).item()) if audio.numel() > 0 else 0.0
assert peak > 0.01, "Generated audio appears to be silence"
audio_np = audio.numpy()
rms = float(np.sqrt(np.mean(np.square(audio_np)))) if audio_np.size else 0.0
assert rms > 1e-4, "Generated audio RMS too low"
tests/engine/test_arg_utils.py
-18
View File
@@ -232,24 +232,6 @@ def test_stage_configs_path_field():
assert args.stage_configs_path == "/some/path.yaml"
def test_voxcpm_model_arch_injects_model_type_override(mocker):
"""Ensure VoxCPM model_arch injects hf_overrides for config resolution."""
mocker.patch.object(OmniEngineArgs, "_ensure_omni_models_registered", return_value=True)
mocker.patch.object(OmniEngineArgs, "_patch_empty_hf_config")
mocker.patch.object(EngineArgs, "create_model_config", return_value=Mock())
mocker.patch.object(OmniModelConfig, "from_vllm_model_config", return_value=Mock())
args = OmniEngineArgs(
model="OpenBMB/VoxCPM1.5",
model_arch="VoxCPMForConditionalGeneration",
)
args.create_model_config()
assert args.hf_overrides["architectures"] == ["VoxCPMForConditionalGeneration"]
assert args.hf_overrides["model_type"] == "voxcpm"
args._patch_empty_hf_config.assert_called_once_with("voxcpm")
def test_strip_single_engine_args():
"""_strip_single_engine_args should remove EngineArgs fields but keep omni fields."""
kwargs = {
tests/entrypoints/openai_api/test_serving_speech_voxcpm.py
-275
View File
@@ -1,275 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""UTs for VoxCPM OpenAI speech serving behavior."""
import asyncio
from types import SimpleNamespace
from unittest.mock import AsyncMock
import pytest
from pytest_mock import MockerFixture
from vllm_omni.entrypoints.openai.protocol.audio import OpenAICreateSpeechRequest
from vllm_omni.entrypoints.openai.serving_speech import OmniOpenAIServingSpeech
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
@pytest.fixture
def voxcpm_server(mocker: MockerFixture):
mocker.patch.object(OmniOpenAIServingSpeech, "_load_supported_speakers", return_value=set())
mocker.patch.object(OmniOpenAIServingSpeech, "_load_codec_frame_rate", return_value=None)
mock_engine_client = mocker.MagicMock()
mock_engine_client.errored = False
mock_engine_client.model_config = mocker.MagicMock(model="OpenBMB/VoxCPM1.5")
mock_engine_client.default_sampling_params_list = [SimpleNamespace(max_tokens=2048)]
mock_engine_client.tts_batch_max_items = 32
mock_engine_client.generate = mocker.MagicMock(return_value="generator")
mock_engine_client.stage_configs = [
SimpleNamespace(
engine_args=SimpleNamespace(
model_stage="latent_generator",
model_arch="VoxCPMForConditionalGeneration",
),
tts_args={},
),
SimpleNamespace(
engine_args=SimpleNamespace(model_stage="vae"),
tts_args={},
),
]
mock_models = mocker.MagicMock()
mock_models.is_base_model.return_value = True
return OmniOpenAIServingSpeech(
engine_client=mock_engine_client,
models=mock_models,
request_logger=mocker.MagicMock(),
)
@pytest.fixture
def voxcpm2_server(mocker: MockerFixture):
mocker.patch.object(OmniOpenAIServingSpeech, "_load_codec_frame_rate", return_value=None)
mock_engine_client = mocker.MagicMock()
mock_engine_client.errored = False
mock_engine_client.model_config = mocker.MagicMock(model="OpenBMB/VoxCPM2")
mock_engine_client.default_sampling_params_list = [SimpleNamespace(max_tokens=2048)]
mock_engine_client.tts_batch_max_items = 32
mock_engine_client.generate = mocker.MagicMock(return_value="generator")
mock_engine_client.stage_configs = [
SimpleNamespace(
engine_args=SimpleNamespace(
model_stage="latent_generator",
model_arch="VoxCPM2TalkerForConditionalGeneration",
),
tts_args={},
),
SimpleNamespace(
engine_args=SimpleNamespace(model_stage="vae"),
tts_args={},
),
]
mock_models = mocker.MagicMock()
mock_models.is_base_model.return_value = True
return OmniOpenAIServingSpeech(
engine_client=mock_engine_client,
models=mock_models,
request_logger=mocker.MagicMock(),
)
class TestVoxCPMServing:
def test_voxcpm_model_type_detection(self, voxcpm_server):
assert voxcpm_server._tts_model_type == "voxcpm"
assert voxcpm_server._is_tts is True
assert voxcpm_server.supported_speakers == set()
@pytest.mark.parametrize(
("request_kwargs", "expected_substring"),
[
({"voice": "alice"}, "voice"),
({"instructions": "whisper"}, "instructions"),
({"language": "en"}, "language"),
({"task_type": "CustomVoice"}, "plain tts"),
({"x_vector_only_mode": True}, "x_vector_only_mode"),
({"speaker_embedding": [0.1, 0.2]}, "speaker_embedding"),
({"initial_codec_chunk_frames": 4}, "initial_codec_chunk_frames"),
({"ref_text": "reference"}, "ref_audio"),
],
)
def test_validate_voxcpm_rejects_unsupported_fields(self, voxcpm_server, request_kwargs, expected_substring):
request = OpenAICreateSpeechRequest(input="hello voxcpm", **request_kwargs)
error = voxcpm_server._validate_voxcpm_request(request)
assert error is not None
assert expected_substring in error.lower()
def test_validate_voxcpm_accepts_plain_tts_request(self, voxcpm_server):
request = OpenAICreateSpeechRequest(input="hello voxcpm", max_new_tokens=256)
assert voxcpm_server._validate_voxcpm_request(request) is None
def test_validate_voxcpm_accepts_voice_clone_request(self, voxcpm_server):
request = OpenAICreateSpeechRequest(
input="clone this voice",
ref_audio="data:audio/wav;base64,QUJD",
ref_text="reference transcript",
max_new_tokens=256,
)
assert voxcpm_server._validate_voxcpm_request(request) is None
def test_prepare_speech_generation_voxcpm_text_only(self, voxcpm_server):
request = OpenAICreateSpeechRequest(input="hello voxcpm", max_new_tokens=321)
request_id, generator, tts_params = asyncio.run(voxcpm_server._prepare_speech_generation(request))
assert request_id.startswith("speech-")
assert generator == "generator"
assert tts_params == {
"text": ["hello voxcpm"],
"cfg_value": [2.0],
"inference_timesteps": [10],
"min_len": [2],
"max_new_tokens": [321],
}
voxcpm_server.engine_client.generate.assert_called_once()
call = voxcpm_server.engine_client.generate.call_args
assert call.kwargs["prompt"] == {
"prompt_token_ids": [1],
"additional_information": tts_params,
"type": "token",
}
assert call.kwargs["output_modalities"] == ["audio"]
def test_prepare_speech_generation_voxcpm_voice_clone_resolves_ref_audio(self, voxcpm_server):
voxcpm_server._resolve_ref_audio = AsyncMock(return_value=([0.1, -0.1, 0.2], 16000))
request = OpenAICreateSpeechRequest(
input="clone this voice",
ref_audio="data:audio/wav;base64,QUJD",
ref_text="reference transcript",
max_new_tokens=512,
)
request_id, generator, tts_params = asyncio.run(voxcpm_server._prepare_speech_generation(request))
assert request_id.startswith("speech-")
assert generator == "generator"
assert tts_params == {
"text": ["clone this voice"],
"cfg_value": [2.0],
"inference_timesteps": [10],
"min_len": [2],
"max_new_tokens": [512],
"ref_text": ["reference transcript"],
"ref_audio": [[[0.1, -0.1, 0.2], 16000]],
}
voxcpm_server._resolve_ref_audio.assert_awaited_once_with("data:audio/wav;base64,QUJD")
call = voxcpm_server.engine_client.generate.call_args
assert call.kwargs["prompt"] == {
"prompt_token_ids": [1],
"additional_information": tts_params,
"type": "token",
}
class TestVoxCPM2Serving:
"""Regression tests for VoxCPM2 serving behavior."""
def test_voxcpm2_model_type_detection(self, voxcpm2_server):
assert voxcpm2_server._tts_model_type == "voxcpm2"
assert voxcpm2_server._is_tts is True
def test_voxcpm2_default_voice_is_supported(self, voxcpm2_server):
"""VoxCPM2 should accept voice='default' (zero-shot mode, no voice cloning).
Regression: previously returned 400 Invalid voice 'default'.
"""
assert "default" in voxcpm2_server.supported_speakers
def test_voxcpm2_inline_ref_audio_does_not_set_voice_name(self, voxcpm2_server):
"""Inline ref_audio must not populate voice_name in additional_information.
Regression: the speaker cache keys on voice_name, so two requests with
the same voice but different inline ref_audio would collide — the second
request reused stale cached features from the first, causing a prefill
length mismatch and engine crash.
"""
mock_prompt = {"prompt_token_ids": [1, 1, 1], "additional_information": {}, "type": "token"}
voxcpm2_server._build_voxcpm2_prompt = AsyncMock(return_value=mock_prompt)
request = OpenAICreateSpeechRequest(
input="test",
voice="default",
ref_audio="data:audio/wav;base64,QUJD",
)
asyncio.run(voxcpm2_server._prepare_speech_generation(request))
additional = mock_prompt["additional_information"]
assert "voice_name" not in additional, (
"voice_name must not be set for inline ref_audio — "
"it causes speaker cache collisions across different audio clips"
)
def test_voxcpm2_uploaded_voice_sets_voice_name(self, voxcpm2_server):
"""Uploaded voice without inline ref_audio must populate voice_name.
This ensures the speaker cache still works for the intended use case:
a voice uploaded once and referenced by name in subsequent requests.
"""
mock_prompt = {"prompt_token_ids": [1, 1, 1], "additional_information": {}, "type": "token"}
voxcpm2_server._build_voxcpm2_prompt = AsyncMock(return_value=mock_prompt)
voxcpm2_server.uploaded_speakers["my_voice"] = {
"name": "my_voice",
"file_path": "/tmp/fake.wav",
"created_at": 12345,
}
voxcpm2_server.supported_speakers.add("my_voice")
request = OpenAICreateSpeechRequest(
input="test",
voice="my_voice",
)
asyncio.run(voxcpm2_server._prepare_speech_generation(request))
additional = mock_prompt["additional_information"]
assert additional.get("voice_name") == "my_voice"
assert "voice_created_at" in additional
def test_voxcpm2_uploaded_voice_with_inline_override_does_not_set_voice_name(self, voxcpm2_server):
"""Uploaded voice with inline ref_audio override must not populate voice_name.
Prevents cache poisoning: an attacker could send an uploaded voice name
with crafted ref_audio, caching malicious features under the uploaded
voice's name for subsequent legitimate requests.
"""
mock_prompt = {"prompt_token_ids": [1, 1, 1], "additional_information": {}, "type": "token"}
voxcpm2_server._build_voxcpm2_prompt = AsyncMock(return_value=mock_prompt)
voxcpm2_server.uploaded_speakers["my_voice"] = {
"name": "my_voice",
"file_path": "/tmp/fake.wav",
"created_at": 12345,
}
voxcpm2_server.supported_speakers.add("my_voice")
request = OpenAICreateSpeechRequest(
input="test",
voice="my_voice",
ref_audio="data:audio/wav;base64,QUJD",
)
asyncio.run(voxcpm2_server._prepare_speech_generation(request))
additional = mock_prompt["additional_information"]
assert "voice_name" not in additional, (
"voice_name must not be set when inline ref_audio overrides an "
"uploaded voice — it would poison the cache for future requests"
)
tests/entrypoints/test_utils.py
-33
View File
@@ -313,39 +313,6 @@ class TestResolveModelConfigPath:
assert result is not None
assert "glm_image.yaml" in result
def test_voxcpm_transformers_format_resolution(self, mocker: MockerFixture):
"""Test VoxCPM transformers config resolves to the voxcpm stage config."""
mocker.patch(
"vllm_omni.entrypoints.utils.get_config",
side_effect=ValueError("missing transformers config"),
)
mocker.patch(
"vllm_omni.entrypoints.utils.file_or_path_exists",
side_effect=lambda _model, filename, revision=None: filename == "config.json",
)
mocker.patch(
"vllm_omni.entrypoints.utils.get_hf_file_to_dict",
return_value={"model_type": "voxcpm"},
)
mocker.patch(
"vllm_omni.entrypoints.utils.current_omni_platform.get_default_stage_config_path",
return_value="vllm_omni/deploy",
)
original_exists = os.path.exists
def mock_exists(path):
if "voxcpm.yaml" in str(path):
return True
return original_exists(path)
mocker.patch("os.path.exists", side_effect=mock_exists)
result = resolve_model_config_path("OpenBMB/VoxCPM1.5")
assert result is not None
assert "voxcpm.yaml" in result
class TestLoadAndResolveStageConfigs:
def test_load_and_resolve_with_kwargs(self):
tests/model_executor/stage_input_processors/test_voxcpm_async_chunk.py
-85
View File
@@ -1,85 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""UTs for VoxCPM async-chunk stage input processing."""
from types import SimpleNamespace
import pytest
import torch
from vllm_omni.model_executor.stage_input_processors.voxcpm import (
_VOXCPM_LATENT_MAGIC,
_coerce_finished_flag,
latent2vae_async_chunk,
)
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
def _request(*, finished):
return SimpleNamespace(is_finished=lambda: finished)
def _decode_serialized_latent(codes: list[int]) -> torch.Tensor:
assert codes[0] == _VOXCPM_LATENT_MAGIC
latent_dim = codes[1]
time_dim = codes[2]
payload = torch.tensor(codes[3:], dtype=torch.int32).to(torch.uint16)
return payload.view(torch.bfloat16).to(torch.float32).reshape(1, latent_dim, time_dim)
@pytest.mark.parametrize(
("value", "expected"),
[
(None, False),
(False, False),
(True, True),
(torch.tensor(False), False),
(torch.tensor(True), True),
([torch.tensor(True)], True),
(([True],), True),
([], False),
],
)
def test_coerce_finished_flag(value, expected):
assert _coerce_finished_flag(value) is expected
def test_latent2vae_async_chunk_serializes_latent_payload():
latent = torch.arange(6, dtype=torch.float32).reshape(2, 3)
payload = latent2vae_async_chunk(
transfer_manager=None,
pooling_output={"latent_audio_feat": latent},
request=_request(finished=False),
is_finished=torch.tensor(False),
)
assert payload is not None
assert torch.equal(payload.meta.finished, torch.tensor(False, dtype=torch.bool))
recovered = _decode_serialized_latent(payload.codes.audio.tolist())
torch.testing.assert_close(recovered, latent.to(torch.bfloat16).to(torch.float32).unsqueeze(0))
def test_latent2vae_async_chunk_returns_terminal_marker_without_latent():
payload = latent2vae_async_chunk(
transfer_manager=None,
pooling_output=None,
request=_request(finished=[torch.tensor(True)]),
is_finished=False,
)
assert payload.codes.audio.tolist() == []
assert torch.equal(payload.meta.finished, torch.tensor(True, dtype=torch.bool))
def test_latent2vae_async_chunk_returns_none_for_nonterminal_empty_chunk():
payload = latent2vae_async_chunk(
transfer_manager=None,
pooling_output={"latent_audio_feat": torch.zeros((0,), dtype=torch.float32)},
request=_request(finished=False),
is_finished=False,
)
assert payload is None