Add voxcpm model support. (#2467)
Signed-off-by: Celeste-jq <591998922@qq.com> Signed-off-by: lyj-jjj <liuyingjun5@huawei.com> Signed-off-by: IsleOfDawnlight <stellamou@qq.com> Signed-off-by: Yueqian Lin <linyueqian@outlook.com> Co-authored-by: Celeste-jq <591998922@qq.com> Co-authored-by: lyj-jjj <liuyingjun5@huawei.com> Co-authored-by: Yueqian Lin <linyueqian@outlook.com>
This commit is contained in:
31 files changed
+4727
-2
No files matched your search
@@ -0,0 +1,156 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""E2E test for VoxCPM offline inference."""
|
||||
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
import tests.conftest as omni_test_conftest
|
||||
from tests.conftest import OmniRunner
|
||||
from tests.utils import hardware_test
|
||||
from vllm_omni.model_executor.models.voxcpm.voxcpm_runtime_utils import (
|
||||
prepare_voxcpm_hf_config_dir,
|
||||
resolve_voxcpm_model_dir,
|
||||
)
|
||||
|
||||
VOXCPM_MODEL = os.environ.get("VOXCPM_MODEL", "OpenBMB/VoxCPM1.5")
|
||||
STAGE_CONFIG = str(
|
||||
Path(__file__).parent.parent.parent.parent / "vllm_omni" / "model_executor" / "stage_configs" / "voxcpm.yaml"
|
||||
)
|
||||
SAMPLE_RATE = 24000
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _patch_npu_cleanup_for_voxcpm(monkeypatch: pytest.MonkeyPatch):
|
||||
"""Limit the NPU cleanup workaround to this VoxCPM test module only."""
|
||||
original_cleanup = omni_test_conftest.cleanup_dist_env_and_memory
|
||||
|
||||
def _safe_cleanup() -> None:
|
||||
try:
|
||||
original_cleanup()
|
||||
except RuntimeError as exc:
|
||||
if "Allocator for npu is not a DeviceAllocator" in str(exc):
|
||||
return
|
||||
raise
|
||||
|
||||
monkeypatch.setattr(omni_test_conftest, "cleanup_dist_env_and_memory", _safe_cleanup)
|
||||
|
||||
|
||||
def _build_prompt(text: str) -> dict[str, Any]:
|
||||
return {
|
||||
"prompt_token_ids": [1],
|
||||
"additional_information": {
|
||||
"text": [text],
|
||||
"cfg_value": [2.0],
|
||||
"inference_timesteps": [10],
|
||||
"min_len": [2],
|
||||
"max_new_tokens": [1024],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _extract_audio_tensor(multimodal_output: dict[str, Any]) -> torch.Tensor:
|
||||
audio = multimodal_output.get("audio", multimodal_output.get("model_outputs"))
|
||||
assert audio is not None, f"No audio output found, keys={list(multimodal_output.keys())}"
|
||||
|
||||
if isinstance(audio, list):
|
||||
parts: list[torch.Tensor] = []
|
||||
for item in audio:
|
||||
if item is None:
|
||||
continue
|
||||
tensor = torch.as_tensor(item)
|
||||
if tensor.numel() == 0:
|
||||
continue
|
||||
parts.append(tensor.float().cpu().reshape(-1))
|
||||
return torch.cat(parts, dim=-1) if parts else torch.zeros((0,), dtype=torch.float32)
|
||||
|
||||
return torch.as_tensor(audio).float().cpu().reshape(-1)
|
||||
|
||||
|
||||
def _extract_final_multimodal_output(outputs) -> dict[str, Any]:
|
||||
for item in reversed(outputs):
|
||||
request_output = getattr(item, "request_output", None)
|
||||
if request_output is not None:
|
||||
multimodal_output = getattr(request_output, "multimodal_output", None)
|
||||
if isinstance(multimodal_output, dict):
|
||||
return multimodal_output
|
||||
completions = getattr(request_output, "outputs", None) or []
|
||||
for completion in completions:
|
||||
multimodal_output = getattr(completion, "multimodal_output", None)
|
||||
if isinstance(multimodal_output, dict):
|
||||
return multimodal_output
|
||||
|
||||
multimodal_output = getattr(item, "multimodal_output", None)
|
||||
if isinstance(multimodal_output, dict):
|
||||
return multimodal_output
|
||||
|
||||
raise AssertionError("No multimodal audio output found in VoxCPM generate results")
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def voxcpm_model_path(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> str:
|
||||
model_dir = resolve_voxcpm_model_dir(VOXCPM_MODEL)
|
||||
|
||||
hf_config_env = os.environ.get("VLLM_OMNI_VOXCPM_HF_CONFIG_PATH")
|
||||
if hf_config_env:
|
||||
hf_config_dir = Path(hf_config_env).expanduser()
|
||||
else:
|
||||
hf_config_dir = tmp_path / "voxcpm_hf_config"
|
||||
|
||||
if not (hf_config_dir / "config.json").exists():
|
||||
prepare_voxcpm_hf_config_dir(model_dir, hf_config_dir)
|
||||
|
||||
monkeypatch.setenv("VLLM_OMNI_VOXCPM_HF_CONFIG_PATH", str(hf_config_dir))
|
||||
return str(model_dir)
|
||||
|
||||
|
||||
def test_prepare_voxcpm_hf_config_dir(tmp_path: Path):
|
||||
model_dir = tmp_path / "model"
|
||||
model_dir.mkdir()
|
||||
(model_dir / "config.json").write_text(json.dumps({"hidden_size": 1024}), encoding="utf-8")
|
||||
(model_dir / "generation_config.json").write_text(json.dumps({"do_sample": False}), encoding="utf-8")
|
||||
|
||||
hf_config_dir = prepare_voxcpm_hf_config_dir(model_dir, tmp_path / "voxcpm_hf_config")
|
||||
|
||||
prepared_config = json.loads((hf_config_dir / "config.json").read_text(encoding="utf-8"))
|
||||
assert prepared_config["model_type"] == "voxcpm"
|
||||
assert prepared_config["architectures"] == ["VoxCPMForConditionalGeneration"]
|
||||
assert (hf_config_dir / "generation_config.json").exists()
|
||||
|
||||
|
||||
def test_resolve_voxcpm_model_dir_local_path(tmp_path: Path):
|
||||
model_dir = tmp_path / "OpenBMB" / "VoxCPM1.5"
|
||||
model_dir.mkdir(parents=True)
|
||||
|
||||
assert resolve_voxcpm_model_dir(str(model_dir)) == model_dir
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "L4"}, num_cards=1)
|
||||
def test_voxcpm_zero_shot_001(voxcpm_model_path: str):
|
||||
with OmniRunner(voxcpm_model_path, stage_configs_path=STAGE_CONFIG) as runner:
|
||||
outputs = list(runner.omni.generate(_build_prompt("Hello, this is a VoxCPM offline inference test.")))
|
||||
|
||||
assert outputs, "No outputs returned"
|
||||
|
||||
multimodal_output = _extract_final_multimodal_output(outputs)
|
||||
audio = _extract_audio_tensor(multimodal_output)
|
||||
assert audio.numel() > SAMPLE_RATE // 2, f"Audio too short: {audio.numel()} samples"
|
||||
|
||||
duration_s = audio.shape[0] / SAMPLE_RATE
|
||||
assert 0.5 < duration_s < 30.0, f"Audio duration out of range: {duration_s:.2f}s"
|
||||
|
||||
peak = float(torch.max(torch.abs(audio)).item()) if audio.numel() > 0 else 0.0
|
||||
assert peak > 0.01, "Generated audio appears to be silence"
|
||||
|
||||
audio_np = audio.numpy()
|
||||
rms = float(np.sqrt(np.mean(np.square(audio_np)))) if audio_np.size else 0.0
|
||||
assert rms > 1e-4, "Generated audio RMS too low"
|
||||
@@ -7,6 +7,7 @@ explicitly patch values that differ from vLLM.
|
||||
import argparse
|
||||
import inspect
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import Mock
|
||||
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
@@ -166,6 +167,24 @@ def test_stage_configs_path_field():
|
||||
assert args.stage_configs_path == "/some/path.yaml"
|
||||
|
||||
|
||||
def test_voxcpm_model_arch_injects_model_type_override(mocker):
|
||||
"""Ensure VoxCPM model_arch injects hf_overrides for config resolution."""
|
||||
mocker.patch.object(OmniEngineArgs, "_ensure_omni_models_registered", return_value=True)
|
||||
mocker.patch.object(OmniEngineArgs, "_patch_empty_hf_config")
|
||||
mocker.patch.object(EngineArgs, "create_model_config", return_value=Mock())
|
||||
mocker.patch.object(OmniModelConfig, "from_vllm_model_config", return_value=Mock())
|
||||
|
||||
args = OmniEngineArgs(
|
||||
model="OpenBMB/VoxCPM1.5",
|
||||
model_arch="VoxCPMForConditionalGeneration",
|
||||
)
|
||||
args.create_model_config()
|
||||
|
||||
assert args.hf_overrides["architectures"] == ["VoxCPMForConditionalGeneration"]
|
||||
assert args.hf_overrides["model_type"] == "voxcpm"
|
||||
args._patch_empty_hf_config.assert_called_once_with("voxcpm")
|
||||
|
||||
|
||||
def test_strip_single_engine_args():
|
||||
"""_strip_single_engine_args should remove EngineArgs fields but keep omni fields."""
|
||||
kwargs = {
|
||||
|
||||
@@ -0,0 +1,143 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""UTs for VoxCPM OpenAI speech serving behavior."""
|
||||
|
||||
import asyncio
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
import pytest
|
||||
from pytest_mock import MockerFixture
|
||||
|
||||
from vllm_omni.entrypoints.openai.protocol.audio import OpenAICreateSpeechRequest
|
||||
from vllm_omni.entrypoints.openai.serving_speech import OmniOpenAIServingSpeech
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def voxcpm_server(mocker: MockerFixture):
|
||||
mocker.patch.object(OmniOpenAIServingSpeech, "_load_supported_speakers", return_value=set())
|
||||
mocker.patch.object(OmniOpenAIServingSpeech, "_load_codec_frame_rate", return_value=None)
|
||||
|
||||
mock_engine_client = mocker.MagicMock()
|
||||
mock_engine_client.errored = False
|
||||
mock_engine_client.model_config = mocker.MagicMock(model="OpenBMB/VoxCPM1.5")
|
||||
mock_engine_client.default_sampling_params_list = [SimpleNamespace(max_tokens=2048)]
|
||||
mock_engine_client.tts_batch_max_items = 32
|
||||
mock_engine_client.generate = mocker.MagicMock(return_value="generator")
|
||||
mock_engine_client.stage_configs = [
|
||||
SimpleNamespace(
|
||||
engine_args=SimpleNamespace(
|
||||
model_stage="latent_generator",
|
||||
model_arch="VoxCPMForConditionalGeneration",
|
||||
),
|
||||
tts_args={},
|
||||
),
|
||||
SimpleNamespace(
|
||||
engine_args=SimpleNamespace(model_stage="vae"),
|
||||
tts_args={},
|
||||
),
|
||||
]
|
||||
|
||||
mock_models = mocker.MagicMock()
|
||||
mock_models.is_base_model.return_value = True
|
||||
|
||||
return OmniOpenAIServingSpeech(
|
||||
engine_client=mock_engine_client,
|
||||
models=mock_models,
|
||||
request_logger=mocker.MagicMock(),
|
||||
)
|
||||
|
||||
|
||||
class TestVoxCPMServing:
|
||||
def test_voxcpm_model_type_detection(self, voxcpm_server):
|
||||
assert voxcpm_server._tts_model_type == "voxcpm"
|
||||
assert voxcpm_server._is_tts is True
|
||||
assert voxcpm_server.supported_speakers == set()
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("request_kwargs", "expected_substring"),
|
||||
[
|
||||
({"voice": "alice"}, "voice"),
|
||||
({"instructions": "whisper"}, "instructions"),
|
||||
({"language": "en"}, "language"),
|
||||
({"task_type": "CustomVoice"}, "plain tts"),
|
||||
({"x_vector_only_mode": True}, "x_vector_only_mode"),
|
||||
({"speaker_embedding": [0.1, 0.2]}, "speaker_embedding"),
|
||||
({"initial_codec_chunk_frames": 4}, "initial_codec_chunk_frames"),
|
||||
({"ref_text": "reference"}, "ref_audio"),
|
||||
],
|
||||
)
|
||||
def test_validate_voxcpm_rejects_unsupported_fields(self, voxcpm_server, request_kwargs, expected_substring):
|
||||
request = OpenAICreateSpeechRequest(input="hello voxcpm", **request_kwargs)
|
||||
error = voxcpm_server._validate_voxcpm_request(request)
|
||||
assert error is not None
|
||||
assert expected_substring in error.lower()
|
||||
|
||||
def test_validate_voxcpm_accepts_plain_tts_request(self, voxcpm_server):
|
||||
request = OpenAICreateSpeechRequest(input="hello voxcpm", max_new_tokens=256)
|
||||
assert voxcpm_server._validate_voxcpm_request(request) is None
|
||||
|
||||
def test_validate_voxcpm_accepts_voice_clone_request(self, voxcpm_server):
|
||||
request = OpenAICreateSpeechRequest(
|
||||
input="clone this voice",
|
||||
ref_audio="data:audio/wav;base64,QUJD",
|
||||
ref_text="reference transcript",
|
||||
max_new_tokens=256,
|
||||
)
|
||||
assert voxcpm_server._validate_voxcpm_request(request) is None
|
||||
|
||||
def test_prepare_speech_generation_voxcpm_text_only(self, voxcpm_server):
|
||||
request = OpenAICreateSpeechRequest(input="hello voxcpm", max_new_tokens=321)
|
||||
|
||||
request_id, generator, tts_params = asyncio.run(voxcpm_server._prepare_speech_generation(request))
|
||||
|
||||
assert request_id.startswith("speech-")
|
||||
assert generator == "generator"
|
||||
assert tts_params == {
|
||||
"text": ["hello voxcpm"],
|
||||
"cfg_value": [2.0],
|
||||
"inference_timesteps": [10],
|
||||
"min_len": [2],
|
||||
"max_new_tokens": [321],
|
||||
}
|
||||
|
||||
voxcpm_server.engine_client.generate.assert_called_once()
|
||||
call = voxcpm_server.engine_client.generate.call_args
|
||||
assert call.kwargs["prompt"] == {
|
||||
"prompt_token_ids": [1],
|
||||
"additional_information": tts_params,
|
||||
}
|
||||
assert call.kwargs["output_modalities"] == ["audio"]
|
||||
|
||||
def test_prepare_speech_generation_voxcpm_voice_clone_resolves_ref_audio(self, voxcpm_server):
|
||||
voxcpm_server._resolve_ref_audio = AsyncMock(return_value=([0.1, -0.1, 0.2], 16000))
|
||||
request = OpenAICreateSpeechRequest(
|
||||
input="clone this voice",
|
||||
ref_audio="data:audio/wav;base64,QUJD",
|
||||
ref_text="reference transcript",
|
||||
max_new_tokens=512,
|
||||
)
|
||||
|
||||
request_id, generator, tts_params = asyncio.run(voxcpm_server._prepare_speech_generation(request))
|
||||
|
||||
assert request_id.startswith("speech-")
|
||||
assert generator == "generator"
|
||||
assert tts_params == {
|
||||
"text": ["clone this voice"],
|
||||
"cfg_value": [2.0],
|
||||
"inference_timesteps": [10],
|
||||
"min_len": [2],
|
||||
"max_new_tokens": [512],
|
||||
"ref_text": ["reference transcript"],
|
||||
"ref_audio": [[[0.1, -0.1, 0.2], 16000]],
|
||||
}
|
||||
|
||||
voxcpm_server._resolve_ref_audio.assert_awaited_once_with("data:audio/wav;base64,QUJD")
|
||||
call = voxcpm_server.engine_client.generate.call_args
|
||||
assert call.kwargs["prompt"] == {
|
||||
"prompt_token_ids": [1],
|
||||
"additional_information": tts_params,
|
||||
}
|
||||
@@ -310,6 +310,39 @@ class TestResolveModelConfigPath:
|
||||
assert result is not None
|
||||
assert "glm_image.yaml" in result
|
||||
|
||||
def test_voxcpm_transformers_format_resolution(self, mocker: MockerFixture):
|
||||
"""Test VoxCPM transformers config resolves to the voxcpm stage config."""
|
||||
mocker.patch(
|
||||
"vllm_omni.entrypoints.utils.get_config",
|
||||
side_effect=ValueError("missing transformers config"),
|
||||
)
|
||||
mocker.patch(
|
||||
"vllm_omni.entrypoints.utils.file_or_path_exists",
|
||||
side_effect=lambda _model, filename, revision=None: filename == "config.json",
|
||||
)
|
||||
mocker.patch(
|
||||
"vllm_omni.entrypoints.utils.get_hf_file_to_dict",
|
||||
return_value={"model_type": "voxcpm"},
|
||||
)
|
||||
mocker.patch(
|
||||
"vllm_omni.entrypoints.utils.current_omni_platform.get_default_stage_config_path",
|
||||
return_value="vllm_omni/model_executor/stage_configs",
|
||||
)
|
||||
|
||||
original_exists = os.path.exists
|
||||
|
||||
def mock_exists(path):
|
||||
if "voxcpm.yaml" in str(path):
|
||||
return True
|
||||
return original_exists(path)
|
||||
|
||||
mocker.patch("os.path.exists", side_effect=mock_exists)
|
||||
|
||||
result = resolve_model_config_path("OpenBMB/VoxCPM1.5")
|
||||
|
||||
assert result is not None
|
||||
assert "voxcpm.yaml" in result
|
||||
|
||||
|
||||
class TestLoadAndResolveStageConfigs:
|
||||
def test_load_and_resolve_with_kwargs(self):
|
||||
|
||||
@@ -0,0 +1,87 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""UTs for VoxCPM async-chunk stage input processing."""
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm_omni.model_executor.stage_input_processors.voxcpm import (
|
||||
_VOXCPM_LATENT_MAGIC,
|
||||
_coerce_finished_flag,
|
||||
latent2vae_async_chunk,
|
||||
)
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
def _request(*, finished):
|
||||
return SimpleNamespace(is_finished=lambda: finished)
|
||||
|
||||
|
||||
def _decode_serialized_latent(codes: list[int]) -> torch.Tensor:
|
||||
assert codes[0] == _VOXCPM_LATENT_MAGIC
|
||||
latent_dim = codes[1]
|
||||
time_dim = codes[2]
|
||||
payload = torch.tensor(codes[3:], dtype=torch.int32).to(torch.uint16)
|
||||
return payload.view(torch.bfloat16).to(torch.float32).reshape(1, latent_dim, time_dim)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("value", "expected"),
|
||||
[
|
||||
(None, False),
|
||||
(False, False),
|
||||
(True, True),
|
||||
(torch.tensor(False), False),
|
||||
(torch.tensor(True), True),
|
||||
([torch.tensor(True)], True),
|
||||
(([True],), True),
|
||||
([], False),
|
||||
],
|
||||
)
|
||||
def test_coerce_finished_flag(value, expected):
|
||||
assert _coerce_finished_flag(value) is expected
|
||||
|
||||
|
||||
def test_latent2vae_async_chunk_serializes_latent_payload():
|
||||
latent = torch.arange(6, dtype=torch.float32).reshape(2, 3)
|
||||
|
||||
payload = latent2vae_async_chunk(
|
||||
transfer_manager=None,
|
||||
pooling_output={"latent_audio_feat": latent},
|
||||
request=_request(finished=False),
|
||||
is_finished=torch.tensor(False),
|
||||
)
|
||||
|
||||
assert payload is not None
|
||||
assert torch.equal(payload["finished"], torch.tensor(False, dtype=torch.bool))
|
||||
recovered = _decode_serialized_latent(payload["code_predictor_codes"])
|
||||
torch.testing.assert_close(recovered, latent.to(torch.bfloat16).to(torch.float32).unsqueeze(0))
|
||||
|
||||
|
||||
def test_latent2vae_async_chunk_returns_terminal_marker_without_latent():
|
||||
payload = latent2vae_async_chunk(
|
||||
transfer_manager=None,
|
||||
pooling_output=None,
|
||||
request=_request(finished=[torch.tensor(True)]),
|
||||
is_finished=False,
|
||||
)
|
||||
|
||||
assert payload == {
|
||||
"code_predictor_codes": [],
|
||||
"finished": torch.tensor(True, dtype=torch.bool),
|
||||
}
|
||||
|
||||
|
||||
def test_latent2vae_async_chunk_returns_none_for_nonterminal_empty_chunk():
|
||||
payload = latent2vae_async_chunk(
|
||||
transfer_manager=None,
|
||||
pooling_output={"latent_audio_feat": torch.zeros((0,), dtype=torch.float32)},
|
||||
request=_request(finished=False),
|
||||
is_finished=False,
|
||||
)
|
||||
|
||||
assert payload is None
|
||||
Reference in new issue
Block a user