[CosyVoice3] Add online serving support, fix stage config, and add CI tests (#2431)
Signed-off-by: linyueqian <linyueqian@outlook.com>
This commit is contained in:
1 parent
f50c5a413f
commit
4c031580cf
10 files changed
+497
-28
No files matched your search
@@ -390,6 +390,16 @@ steps:
|
||||
export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
pytest -s -v tests/e2e/online_serving/test_voxtral_tts.py tests/e2e/offline_inference/test_voxtral_tts.py -m "advanced_model" --run-level "advanced_model"
|
||||
'
|
||||
|
||||
- label: "CosyVoice3-TTS E2E Test"
|
||||
timeout_in_minutes: 20
|
||||
depends_on: upload-merge-pipeline
|
||||
commands:
|
||||
- |
|
||||
timeout 20m bash -c '
|
||||
export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
pytest -s -v tests/e2e/online_serving/test_cosyvoice3_tts.py -m "advanced_model" --run-level "advanced_model"
|
||||
'
|
||||
agents:
|
||||
queue: "mithril-h100-pool"
|
||||
plugins:
|
||||
@@ -408,6 +418,11 @@ steps:
|
||||
env:
|
||||
- name: HF_HOME
|
||||
value: /root/.cache/huggingface
|
||||
- name: HF_TOKEN
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: hf-token-secret
|
||||
key: token
|
||||
nodeSelector:
|
||||
node.kubernetes.io/instance-type: gpu-h100-sxm
|
||||
volumes:
|
||||
|
||||
@@ -548,3 +548,46 @@ steps:
|
||||
hostPath:
|
||||
path: /mnt/hf-cache
|
||||
type: DirectoryOrCreate
|
||||
|
||||
- label: "CosyVoice3-TTS E2E Test"
|
||||
timeout_in_minutes: 20
|
||||
depends_on: upload-ready-pipeline
|
||||
commands:
|
||||
- |
|
||||
timeout 20m bash -c '
|
||||
export VLLM_WORKER_MULTIPROC_METHOD=spawn
|
||||
pytest -s -v tests/e2e/online_serving/test_cosyvoice3_tts.py -m "core_model" --run-level "core_model"
|
||||
'
|
||||
agents:
|
||||
queue: "mithril-h100-pool"
|
||||
plugins:
|
||||
- kubernetes:
|
||||
podSpec:
|
||||
containers:
|
||||
- image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
|
||||
resources:
|
||||
limits:
|
||||
nvidia.com/gpu: 1
|
||||
volumeMounts:
|
||||
- name: devshm
|
||||
mountPath: /dev/shm
|
||||
- name: hf-cache
|
||||
mountPath: /root/.cache/huggingface
|
||||
env:
|
||||
- name: HF_HOME
|
||||
value: /root/.cache/huggingface
|
||||
- name: HF_TOKEN
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: hf-token-secret
|
||||
key: token
|
||||
nodeSelector:
|
||||
node.kubernetes.io/instance-type: gpu-h100-sxm
|
||||
volumes:
|
||||
- name: devshm
|
||||
emptyDir:
|
||||
medium: Memory
|
||||
- name: hf-cache
|
||||
hostPath:
|
||||
path: /mnt/hf-cache
|
||||
type: DirectoryOrCreate
|
||||
@@ -0,0 +1,124 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
E2E Online tests for CosyVoice3 TTS model with voice cloning.
|
||||
|
||||
These tests verify the /v1/audio/speech endpoint works correctly with
|
||||
the CosyVoice3 model, which requires reference audio for voice cloning.
|
||||
"""
|
||||
|
||||
import os
|
||||
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from tests.conftest import OmniServerParams
|
||||
from tests.utils import hardware_test
|
||||
|
||||
MODEL = "FunAudioLLM/Fun-CosyVoice3-0.5B-2512"
|
||||
|
||||
# Official CosyVoice zero-shot prompt audio and its transcript
|
||||
REF_AUDIO_URL = "https://raw.githubusercontent.com/FunAudioLLM/CosyVoice/main/asset/zero_shot_prompt.wav"
|
||||
REF_TEXT = "希望你以后能够做的比我还好呦。"
|
||||
|
||||
|
||||
def get_stage_config(name: str = "cosyvoice3.yaml"):
|
||||
"""Get the stage config path from vllm_omni model_executor stage_configs."""
|
||||
return str(Path(__file__).parent.parent.parent.parent / "vllm_omni" / "model_executor" / "stage_configs" / name)
|
||||
|
||||
|
||||
def get_prompt(prompt_type="zh"):
|
||||
prompts = {
|
||||
"zh": "收到好友从远方寄来的生日礼物,那份意外的惊喜与深深的感动让我热泪盈眶。",
|
||||
"en": "Hello, this is a voice cloning test with English text.",
|
||||
}
|
||||
return prompts.get(prompt_type, prompts["zh"])
|
||||
|
||||
|
||||
tts_server_params = [
|
||||
pytest.param(
|
||||
OmniServerParams(
|
||||
model=MODEL,
|
||||
stage_config_path=get_stage_config(),
|
||||
server_args=["--trust-remote-code", "--disable-log-stats"],
|
||||
),
|
||||
id="cosyvoice3",
|
||||
)
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "H100"}, num_cards=1)
|
||||
@pytest.mark.parametrize("omni_server", tts_server_params, indirect=True)
|
||||
def test_voice_clone_zh_001(omni_server, openai_client) -> None:
|
||||
"""
|
||||
Test voice cloning TTS with Chinese text via OpenAI API.
|
||||
Deploy Setting: default yaml
|
||||
Input Modal: text + ref_audio + ref_text
|
||||
Output Modal: audio
|
||||
Input Setting: stream=False
|
||||
Datasets: single request
|
||||
"""
|
||||
request_config = {
|
||||
"model": omni_server.model,
|
||||
"input": get_prompt("zh"),
|
||||
"stream": False,
|
||||
"response_format": "wav",
|
||||
"ref_audio": REF_AUDIO_URL,
|
||||
"ref_text": REF_TEXT,
|
||||
}
|
||||
openai_client.send_audio_speech_request(request_config)
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "H100"}, num_cards=1)
|
||||
@pytest.mark.parametrize("omni_server", tts_server_params, indirect=True)
|
||||
def test_voice_clone_zh_002(omni_server, openai_client) -> None:
|
||||
"""
|
||||
Test voice cloning TTS with Chinese text via OpenAI API.
|
||||
Deploy Setting: default yaml
|
||||
Input Modal: text + ref_audio + ref_text
|
||||
Output Modal: audio
|
||||
Input Setting: stream=True
|
||||
Datasets: single request
|
||||
"""
|
||||
request_config = {
|
||||
"model": omni_server.model,
|
||||
"input": get_prompt("zh"),
|
||||
"stream": True,
|
||||
"response_format": "wav",
|
||||
"ref_audio": REF_AUDIO_URL,
|
||||
"ref_text": REF_TEXT,
|
||||
}
|
||||
openai_client.send_audio_speech_request(request_config)
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "H100"}, num_cards=1)
|
||||
@pytest.mark.parametrize("omni_server", tts_server_params, indirect=True)
|
||||
def test_voice_clone_en_001(omni_server, openai_client) -> None:
|
||||
"""
|
||||
Test voice cloning TTS with English text via OpenAI API.
|
||||
Deploy Setting: default yaml
|
||||
Input Modal: text + ref_audio + ref_text
|
||||
Output Modal: audio
|
||||
Input Setting: stream=False
|
||||
Datasets: single request
|
||||
"""
|
||||
request_config = {
|
||||
"model": omni_server.model,
|
||||
"input": get_prompt("en"),
|
||||
"stream": False,
|
||||
"response_format": "wav",
|
||||
"ref_audio": REF_AUDIO_URL,
|
||||
"ref_text": REF_TEXT,
|
||||
}
|
||||
openai_client.send_audio_speech_request(request_config)
|
||||
@@ -1872,3 +1872,115 @@ class TestWAVStreaming:
|
||||
for fmt in unsupported_formats:
|
||||
response = client.post("/v1/audio/speech", json={"input": "Hello", "stream": True, "response_format": fmt})
|
||||
assert response.status_code == 422
|
||||
|
||||
|
||||
# ---- CosyVoice3 Serving Tests ----
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def cosyvoice3_server(mocker: MockerFixture):
|
||||
mocker.patch.object(OmniOpenAIServingSpeech, "_load_supported_speakers", return_value=set())
|
||||
mocker.patch.object(OmniOpenAIServingSpeech, "_load_codec_frame_rate", return_value=None)
|
||||
|
||||
mock_engine_client = mocker.MagicMock()
|
||||
mock_engine_client.errored = False
|
||||
mock_engine_client.model_config = mocker.MagicMock(model="FunAudioLLM/Fun-CosyVoice3-0.5B-2512")
|
||||
mock_engine_client.default_sampling_params_list = [SimpleNamespace(max_tokens=2048)]
|
||||
mock_engine_client.tts_batch_max_items = 32
|
||||
mock_engine_client.generate = mocker.MagicMock(return_value="generator")
|
||||
mock_engine_client.stage_configs = [
|
||||
SimpleNamespace(
|
||||
engine_args=SimpleNamespace(model_stage="cosyvoice3_talker"),
|
||||
tts_args={},
|
||||
)
|
||||
]
|
||||
|
||||
mock_models = mocker.MagicMock()
|
||||
mock_models.is_base_model.return_value = True
|
||||
|
||||
return OmniOpenAIServingSpeech(
|
||||
engine_client=mock_engine_client,
|
||||
models=mock_models,
|
||||
request_logger=mocker.MagicMock(),
|
||||
)
|
||||
|
||||
|
||||
class TestCosyVoice3Serving:
|
||||
def test_cosyvoice3_model_type_detection(self, cosyvoice3_server):
|
||||
assert cosyvoice3_server._tts_model_type == "cosyvoice3"
|
||||
assert cosyvoice3_server._is_tts is True
|
||||
assert cosyvoice3_server._is_cosyvoice3 is True
|
||||
|
||||
def test_cosyvoice3_stage_registered(self):
|
||||
from vllm_omni.entrypoints.openai.serving_speech import (
|
||||
_COSYVOICE3_TTS_MODEL_STAGES,
|
||||
_TTS_MODEL_STAGES,
|
||||
)
|
||||
|
||||
assert "cosyvoice3_talker" in _COSYVOICE3_TTS_MODEL_STAGES
|
||||
assert "cosyvoice3_talker" in _TTS_MODEL_STAGES
|
||||
|
||||
def test_validate_cosyvoice3_empty_input(self, cosyvoice3_server):
|
||||
request = OpenAICreateSpeechRequest(input="", ref_audio="data:audio/wav;base64,abc", ref_text="hello")
|
||||
error = cosyvoice3_server._validate_cosyvoice3_request(request)
|
||||
assert error is not None
|
||||
assert "empty" in error.lower()
|
||||
|
||||
def test_validate_cosyvoice3_missing_ref_audio(self, cosyvoice3_server):
|
||||
request = OpenAICreateSpeechRequest(input="Hello", ref_text="hello")
|
||||
error = cosyvoice3_server._validate_cosyvoice3_request(request)
|
||||
assert error is not None
|
||||
assert "ref_audio" in error.lower()
|
||||
|
||||
def test_validate_cosyvoice3_missing_ref_text(self, cosyvoice3_server):
|
||||
request = OpenAICreateSpeechRequest(input="Hello", ref_audio="data:audio/wav;base64,abc")
|
||||
error = cosyvoice3_server._validate_cosyvoice3_request(request)
|
||||
assert error is not None
|
||||
assert "ref_text" in error.lower()
|
||||
|
||||
def test_validate_cosyvoice3_invalid_ref_audio_format(self, cosyvoice3_server):
|
||||
request = OpenAICreateSpeechRequest(input="Hello", ref_audio="/local/path.wav", ref_text="hello")
|
||||
error = cosyvoice3_server._validate_cosyvoice3_request(request)
|
||||
assert error is not None
|
||||
assert "url" in error.lower() or "format" in error.lower()
|
||||
|
||||
def test_validate_cosyvoice3_valid_request(self, cosyvoice3_server):
|
||||
request = OpenAICreateSpeechRequest(
|
||||
input="Hello world",
|
||||
ref_audio="data:audio/wav;base64,abc123",
|
||||
ref_text="Reference transcript",
|
||||
)
|
||||
error = cosyvoice3_server._validate_cosyvoice3_request(request)
|
||||
assert error is None
|
||||
|
||||
def test_validate_cosyvoice3_max_new_tokens_range(self, cosyvoice3_server):
|
||||
request = OpenAICreateSpeechRequest(
|
||||
input="Hello",
|
||||
ref_audio="data:audio/wav;base64,abc",
|
||||
ref_text="hello",
|
||||
max_new_tokens=0,
|
||||
)
|
||||
error = cosyvoice3_server._validate_cosyvoice3_request(request)
|
||||
assert error is not None
|
||||
assert "max_new_tokens" in error
|
||||
|
||||
def test_prepare_speech_generation_cosyvoice3(self, cosyvoice3_server):
|
||||
cosyvoice3_server._build_cosyvoice3_prompt = AsyncMock(
|
||||
return_value={
|
||||
"prompt": "Hello",
|
||||
"multi_modal_data": {"audio": (np.zeros(24000), 24000)},
|
||||
"mm_processor_kwargs": {"prompt_text": "ref text", "sample_rate": 24000},
|
||||
}
|
||||
)
|
||||
|
||||
request = OpenAICreateSpeechRequest(
|
||||
input="Hello",
|
||||
ref_audio="data:audio/wav;base64,abc",
|
||||
ref_text="Reference text",
|
||||
)
|
||||
request_id, generator, tts_params = asyncio.run(cosyvoice3_server._prepare_speech_generation(request))
|
||||
|
||||
assert request_id.startswith("speech-")
|
||||
assert generator == "generator"
|
||||
assert tts_params == {}
|
||||
cosyvoice3_server._build_cosyvoice3_prompt.assert_awaited_once()
|
||||
@@ -1,5 +1,6 @@
|
||||
import argparse
|
||||
import dataclasses
|
||||
import os
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
@@ -12,6 +13,18 @@ from vllm_omni.plugins import load_omni_general_plugins
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# Maps model architecture names to their HuggingFace model_type values.
|
||||
# Used when auto-injecting hf_overrides for models with missing config.json.
|
||||
_ARCH_TO_MODEL_TYPE: dict[str, str] = {
|
||||
"CosyVoice3Model": "cosyvoice3",
|
||||
"OmniVoiceModel": "omnivoice",
|
||||
}
|
||||
|
||||
# Maps model architecture names to tokenizer subfolder paths within HF repos.
|
||||
_TOKENIZER_SUBFOLDER_MAP: dict[str, str] = {
|
||||
"CosyVoice3Model": "CosyVoice-BlankEN",
|
||||
}
|
||||
|
||||
|
||||
def _register_omni_hf_configs() -> None:
|
||||
try:
|
||||
@@ -29,6 +42,14 @@ def _register_omni_hf_configs() -> None:
|
||||
logger.warning("Skipping omni HF config registration due to import error: %s", exc)
|
||||
return
|
||||
|
||||
# Register with both transformers AutoConfig and vLLM's config registry
|
||||
# so models with empty/missing config.json (e.g. CosyVoice3) can be
|
||||
# resolved when model_type is injected via hf_overrides.
|
||||
try:
|
||||
from vllm.transformers_utils.config import _CONFIG_REGISTRY
|
||||
except ImportError:
|
||||
_CONFIG_REGISTRY = None
|
||||
|
||||
for model_type, config_cls in [
|
||||
("qwen3_tts", Qwen3TTSConfig),
|
||||
("cosyvoice3", CosyVoice3Config),
|
||||
@@ -40,6 +61,8 @@ def _register_omni_hf_configs() -> None:
|
||||
except ValueError:
|
||||
# Already registered elsewhere; ignore.
|
||||
pass
|
||||
if _CONFIG_REGISTRY is not None and model_type not in _CONFIG_REGISTRY:
|
||||
_CONFIG_REGISTRY[model_type] = config_cls
|
||||
|
||||
|
||||
def register_omni_models_to_vllm():
|
||||
@@ -129,11 +152,52 @@ class OmniEngineArgs(EngineArgs):
|
||||
|
||||
# If model_arch is specified, inject it into hf_overrides so vLLM can
|
||||
# resolve the architecture even when config.json lacks 'architectures'.
|
||||
# Also inject model_type so AutoConfig can resolve the correct config
|
||||
# class for models with empty or missing config.json (e.g. CosyVoice3).
|
||||
if self.model_arch:
|
||||
if self.hf_overrides is None:
|
||||
self.hf_overrides = {}
|
||||
if isinstance(self.hf_overrides, dict):
|
||||
self.hf_overrides.setdefault("architectures", [self.model_arch])
|
||||
if "model_type" not in self.hf_overrides:
|
||||
model_type = _ARCH_TO_MODEL_TYPE.get(self.model_arch)
|
||||
if model_type is not None:
|
||||
self.hf_overrides.setdefault("model_type", model_type)
|
||||
|
||||
# Auto-detect tokenizer for models that store it in a subdirectory
|
||||
# rather than the root (e.g. CosyVoice3 uses CosyVoice-BlankEN/).
|
||||
if not self.tokenizer and self.model:
|
||||
model_path = self.model
|
||||
if os.path.isdir(model_path) and not os.path.isfile(os.path.join(model_path, "tokenizer_config.json")):
|
||||
for subfolder in sorted(os.listdir(model_path)):
|
||||
candidate = os.path.join(model_path, subfolder)
|
||||
if os.path.isdir(candidate) and os.path.isfile(os.path.join(candidate, "tokenizer_config.json")):
|
||||
self.tokenizer = candidate
|
||||
logger.info("Auto-detected tokenizer at %s", candidate)
|
||||
break
|
||||
elif not os.path.isdir(model_path):
|
||||
subfolder = _TOKENIZER_SUBFOLDER_MAP.get(self.model_arch)
|
||||
if subfolder:
|
||||
# Download just the tokenizer files from the subfolder
|
||||
try:
|
||||
from huggingface_hub import snapshot_download
|
||||
|
||||
local_dir = snapshot_download(
|
||||
model_path,
|
||||
allow_patterns=[
|
||||
f"{subfolder}/tokenizer*",
|
||||
f"{subfolder}/special_tokens*",
|
||||
f"{subfolder}/vocab*",
|
||||
f"{subfolder}/merges*",
|
||||
f"{subfolder}/added_tokens*",
|
||||
],
|
||||
)
|
||||
candidate = os.path.join(local_dir, subfolder)
|
||||
if os.path.isdir(candidate):
|
||||
self.tokenizer = candidate
|
||||
logger.info("Downloaded tokenizer from %s/%s", model_path, subfolder)
|
||||
except Exception as e:
|
||||
logger.warning("Failed to download tokenizer subfolder: %s", e)
|
||||
|
||||
# Build the vLLM config first, then use it to create the Omni config.
|
||||
model_config = super().create_model_config()
|
||||
|
||||
@@ -73,6 +73,16 @@ from vllm_omni.platforms import current_omni_platform
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def _patch_generation_config_if_needed(model_config: Any) -> None:
|
||||
"""Ensure try_get_generation_config won't crash for models whose HF
|
||||
config.json lacks model_type (e.g. CosyVoice3). We probe it once;
|
||||
if it raises, we monkey-patch the method to return None."""
|
||||
try:
|
||||
model_config.try_get_generation_config()
|
||||
except Exception:
|
||||
model_config.try_get_generation_config = lambda: {}
|
||||
|
||||
|
||||
def _inject_kv_stage_info(stage_cfg: Any, stage_id: int) -> None:
|
||||
"""Inject stage_id and engine_input_source into omni_kv_config.
|
||||
|
||||
@@ -409,6 +419,12 @@ class AsyncOmniEngine:
|
||||
)
|
||||
input_processor = None
|
||||
if started.stage_id == 0:
|
||||
# Some omni models (e.g. CosyVoice3) have an empty HF
|
||||
# config.json without model_type, which causes
|
||||
# try_get_generation_config -> AutoConfig.from_pretrained
|
||||
# to raise ValueError. Patch it to return None so
|
||||
# InputProcessor doesn't crash.
|
||||
_patch_generation_config_if_needed(started.vllm_config.model_config)
|
||||
input_processor = InputProcessor(vllm_config=started.vllm_config)
|
||||
# Use omni preprocessor so text-only prompts with
|
||||
# mm_processor_kwargs (e.g. GLM-Image t2i target_h/target_w)
|
||||
|
||||
@@ -46,9 +46,14 @@ logger = init_logger(__name__)
|
||||
_VOXTRAL_TTS_MODEL_STAGES = {"audio_generation"}
|
||||
_QWEN3_TTS_MODEL_STAGES = {"qwen3_tts"}
|
||||
_FISH_TTS_MODEL_STAGES = {"fish_speech_slow_ar"}
|
||||
_COSYVOICE3_TTS_MODEL_STAGES = {"cosyvoice3_talker"}
|
||||
_OMNIVOICE_TTS_MODEL_STAGES = {"omnivoice_generator"}
|
||||
_TTS_MODEL_STAGES: set[str] = (
|
||||
_VOXTRAL_TTS_MODEL_STAGES | _QWEN3_TTS_MODEL_STAGES | _FISH_TTS_MODEL_STAGES | _OMNIVOICE_TTS_MODEL_STAGES
|
||||
_VOXTRAL_TTS_MODEL_STAGES
|
||||
| _QWEN3_TTS_MODEL_STAGES
|
||||
| _FISH_TTS_MODEL_STAGES
|
||||
| _COSYVOICE3_TTS_MODEL_STAGES
|
||||
| _OMNIVOICE_TTS_MODEL_STAGES
|
||||
)
|
||||
_TTS_LANGUAGES: set[str] = {
|
||||
"Auto",
|
||||
@@ -184,6 +189,13 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
|
||||
)
|
||||
self._fish_speech_tokenizer = None
|
||||
|
||||
self._is_cosyvoice3 = (
|
||||
self._tts_stage is not None
|
||||
and getattr(getattr(self._tts_stage, "engine_args", None), "model_stage", None)
|
||||
in _COSYVOICE3_TTS_MODEL_STAGES
|
||||
)
|
||||
self._cosyvoice3_tokenizer = None
|
||||
|
||||
# Determine TTS model type or None
|
||||
self._tts_model_type = self._detect_tts_model_type()
|
||||
|
||||
@@ -258,6 +270,8 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
|
||||
return "voxtral_tts"
|
||||
if model_stage in _FISH_TTS_MODEL_STAGES:
|
||||
return "fish_tts"
|
||||
if model_stage in _COSYVOICE3_TTS_MODEL_STAGES:
|
||||
return "cosyvoice3"
|
||||
if model_stage in _OMNIVOICE_TTS_MODEL_STAGES:
|
||||
return "omnivoice"
|
||||
return None
|
||||
@@ -713,6 +727,8 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
|
||||
return self._validate_voxtral_tts_request(request)
|
||||
if self._tts_model_type == "fish_tts":
|
||||
return self._validate_fish_tts_request(request)
|
||||
if self._tts_model_type == "cosyvoice3":
|
||||
return self._validate_cosyvoice3_request(request)
|
||||
return self._validate_qwen_tts_request(request)
|
||||
|
||||
def _validate_ref_audio_format(self, ref_audio: str) -> str | None:
|
||||
@@ -884,6 +900,30 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
|
||||
|
||||
return None
|
||||
|
||||
def _validate_cosyvoice3_request(self, request: OpenAICreateSpeechRequest) -> str | None:
|
||||
"""Validate CosyVoice3 request parameters. Returns error message or None."""
|
||||
if not request.input or not request.input.strip():
|
||||
return "Input text cannot be empty"
|
||||
|
||||
# CosyVoice3 requires reference audio for voice cloning
|
||||
if request.ref_audio is None:
|
||||
return "CosyVoice3 requires 'ref_audio' (reference audio for voice cloning)"
|
||||
|
||||
fmt_err = self._validate_ref_audio_format(request.ref_audio)
|
||||
if fmt_err:
|
||||
return fmt_err
|
||||
|
||||
if not request.ref_text or not request.ref_text.strip():
|
||||
return "CosyVoice3 requires 'ref_text' (transcript of the reference audio)"
|
||||
|
||||
if request.max_new_tokens is not None:
|
||||
if request.max_new_tokens < _TTS_MAX_NEW_TOKENS_MIN:
|
||||
return f"max_new_tokens must be at least {_TTS_MAX_NEW_TOKENS_MIN}"
|
||||
if request.max_new_tokens > _TTS_MAX_NEW_TOKENS_MAX:
|
||||
return f"max_new_tokens cannot exceed {_TTS_MAX_NEW_TOKENS_MAX}"
|
||||
|
||||
return None
|
||||
|
||||
async def _resolve_ref_audio(self, ref_audio_str: str) -> tuple[list[float], int]:
|
||||
"""Resolve ref_audio to (wav_samples, sample_rate).
|
||||
|
||||
@@ -1194,6 +1234,33 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
|
||||
"additional_information": additional_information,
|
||||
}
|
||||
|
||||
# ---- CosyVoice3 helpers ----
|
||||
|
||||
async def _build_cosyvoice3_prompt(
|
||||
self,
|
||||
request: OpenAICreateSpeechRequest,
|
||||
) -> dict[str, Any]:
|
||||
"""Build prompt for CosyVoice3.
|
||||
|
||||
CosyVoice3 uses multimodal input with reference audio for voice cloning.
|
||||
The prompt format matches the offline example: text prompt + audio data
|
||||
+ mm_processor_kwargs with prompt_text.
|
||||
"""
|
||||
# Resolve reference audio
|
||||
wav_samples, sr = await self._resolve_ref_audio(request.ref_audio)
|
||||
audio_data = (np.asarray(wav_samples, dtype=np.float32), sr)
|
||||
|
||||
return {
|
||||
"prompt": request.input,
|
||||
"multi_modal_data": {
|
||||
"audio": audio_data,
|
||||
},
|
||||
"mm_processor_kwargs": {
|
||||
"prompt_text": request.ref_text,
|
||||
"sample_rate": sr,
|
||||
},
|
||||
}
|
||||
|
||||
# ---- Common speech generation helpers ----
|
||||
|
||||
async def _prepare_speech_generation(
|
||||
@@ -1224,6 +1291,9 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
|
||||
if self._tts_model_type == "voxtral_tts":
|
||||
prompt = await self._build_voxtral_prompt(request)
|
||||
tts_params = {}
|
||||
elif self._tts_model_type == "cosyvoice3":
|
||||
prompt = await self._build_cosyvoice3_prompt(request)
|
||||
tts_params = {}
|
||||
else:
|
||||
tts_params = self._build_tts_params(request)
|
||||
# Resolve ref_audio (explicit or auto-set for uploaded voices)
|
||||
@@ -1247,6 +1317,8 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
|
||||
model_type = "fish_speech"
|
||||
elif self._tts_model_type == "voxtral_tts":
|
||||
model_type = "voxtral_tts"
|
||||
elif self._tts_model_type == "cosyvoice3":
|
||||
model_type = "cosyvoice3"
|
||||
elif self._is_tts:
|
||||
model_type = tts_params.get("task_type", ["unknown"])[0]
|
||||
else:
|
||||
|
||||
@@ -66,6 +66,12 @@ class CosyVoice3MultiModalProcessor(BaseMultiModalProcessor[CosyVoice3MultiModal
|
||||
if cached_model_dir == model_dir:
|
||||
return
|
||||
|
||||
# If model_dir is an HF repo ID (not a local path), resolve to cache
|
||||
if not os.path.isdir(model_dir):
|
||||
from huggingface_hub import snapshot_download
|
||||
|
||||
model_dir = snapshot_download(model_dir)
|
||||
|
||||
import onnxruntime
|
||||
|
||||
from vllm_omni.model_executor.models.cosyvoice3.tokenizer import get_qwen_tokenizer
|
||||
@@ -266,9 +272,14 @@ class CosyVoice3Model(
|
||||
self.config = vllm_config.model_config.hf_config
|
||||
self.have_multimodal_outputs = True
|
||||
self.model_stage = vllm_config.model_config.model_stage
|
||||
self.model_dir = vllm_config.model_config.model
|
||||
model_dir = vllm_config.model_config.model
|
||||
if not os.path.isdir(model_dir):
|
||||
from huggingface_hub import snapshot_download
|
||||
|
||||
model_dir = snapshot_download(model_dir)
|
||||
self.model_dir = model_dir
|
||||
self.model = None
|
||||
if self.model_stage == "talker":
|
||||
if self.model_stage == "cosyvoice3_talker":
|
||||
# Initialize talker stage (text to speech tokens)
|
||||
from vllm_omni.model_executor.models.cosyvoice3.cosyvoice3_talker import CosyVoice3LM, VLLMQwen2Encoder
|
||||
|
||||
@@ -286,7 +297,7 @@ class CosyVoice3Model(
|
||||
# KV cache is now managed externally by vLLM's PagedAttention
|
||||
# No need for self.llm_cache
|
||||
self.model = self.talker
|
||||
elif self.model_stage == "code2wav":
|
||||
elif self.model_stage == "cosyvoice3_code2wav":
|
||||
# Initialize code2wav stage (flow matching + vocoder)
|
||||
from vllm_omni.model_executor.models.cosyvoice3.cosyvoice3_code2wav import CosyVoice3Code2Wav
|
||||
|
||||
@@ -322,7 +333,7 @@ class CosyVoice3Model(
|
||||
def compute_logits(self, hidden_states: torch.Tensor | OmniOutput) -> torch.Tensor | None:
|
||||
if isinstance(hidden_states, OmniOutput):
|
||||
hidden_states = hidden_states.text_hidden_states
|
||||
if self.model_stage == "talker":
|
||||
if self.model_stage == "cosyvoice3_talker":
|
||||
logits = self.model.llm_decoder(hidden_states)
|
||||
vocab_size = self.config.vocab_size
|
||||
pad_size = vocab_size - logits.size(-1)
|
||||
@@ -337,7 +348,7 @@ class CosyVoice3Model(
|
||||
raise RuntimeError(f"compute_logits is only valid for {self.model_stage}.")
|
||||
|
||||
def embed_multimodal(self, **kwargs: object) -> torch.Tensor:
|
||||
if self.model_stage == "talker":
|
||||
if self.model_stage == "cosyvoice3_talker":
|
||||
speech_token = kwargs["speech_token"]
|
||||
speech_token_emb = self.model.speech_embedding(speech_token)
|
||||
return speech_token_emb
|
||||
@@ -350,7 +361,7 @@ class CosyVoice3Model(
|
||||
multimodal_embeddings=None,
|
||||
is_multimodal=None,
|
||||
) -> torch.Tensor:
|
||||
if self.model_stage == "talker":
|
||||
if self.model_stage == "cosyvoice3_talker":
|
||||
if is_multimodal is not None and any(is_multimodal):
|
||||
embed_tokens = self.model.llm.model.embed_tokens(input_ids)
|
||||
sos = self.model.speech_embedding.weight[self.model.sos].reshape(1, -1)
|
||||
@@ -363,7 +374,7 @@ class CosyVoice3Model(
|
||||
else:
|
||||
embed_tokens = self.model.speech_embedding.weight[input_ids]
|
||||
return embed_tokens
|
||||
elif self.model_stage == "code2wav":
|
||||
elif self.model_stage == "cosyvoice3_code2wav":
|
||||
assert input_ids.dim() == 1
|
||||
hidden = int(self.config.hidden_size)
|
||||
return torch.zeros(
|
||||
@@ -381,7 +392,7 @@ class CosyVoice3Model(
|
||||
additional_information: dict[str, object] | None = None,
|
||||
**kwargs: object,
|
||||
) -> OmniOutput:
|
||||
if self.model_stage == "talker":
|
||||
if self.model_stage == "cosyvoice3_talker":
|
||||
if inputs_embeds is None:
|
||||
inputs_embeds = self.embed_input_ids(input_ids)
|
||||
|
||||
@@ -399,7 +410,7 @@ class CosyVoice3Model(
|
||||
}
|
||||
|
||||
return OmniOutput(text_hidden_states=hidden_states, multimodal_outputs=multimodal_outputs)
|
||||
elif self.model_stage == "code2wav":
|
||||
elif self.model_stage == "cosyvoice3_code2wav":
|
||||
runtime_info = kwargs.get("runtime_additional_information", [])
|
||||
if not runtime_info:
|
||||
length = 30 * 24000
|
||||
@@ -420,13 +431,13 @@ class CosyVoice3Model(
|
||||
|
||||
return OmniOutput(
|
||||
text_hidden_states=None,
|
||||
multimodal_outputs={"audio": tts_speech},
|
||||
multimodal_outputs={"audio": tts_speech, "sr": 22050},
|
||||
)
|
||||
else:
|
||||
raise ValueError(f"Unsupported model_stage: {self.model_stage}")
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
if self.model_stage == "talker":
|
||||
if self.model_stage == "cosyvoice3_talker":
|
||||
# Load weights for text to speech LM stage using vLLM's weight loading
|
||||
llm_weight_path = os.path.join(self.model_dir, "llm.pt")
|
||||
device = next(self.parameters()).device
|
||||
@@ -460,7 +471,7 @@ class CosyVoice3Model(
|
||||
self.model.llm_decoder.load_state_dict(llm_decoder_state)
|
||||
|
||||
self.model.to(device).eval()
|
||||
elif self.model_stage == "code2wav":
|
||||
elif self.model_stage == "cosyvoice3_code2wav":
|
||||
# Load weights for code2wav stage (flow + hift)
|
||||
device = next(self.parameters()).device
|
||||
self.code2wav.load_weights(self.model_dir, device)
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import logging
|
||||
import os
|
||||
from functools import cache, lru_cache
|
||||
|
||||
@@ -10,6 +11,8 @@ import torchaudio
|
||||
import torchaudio.compliance.kaldi as kaldi
|
||||
from librosa.filters import mel as librosa_mel_fn
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
IGNORE_ID = -1
|
||||
|
||||
|
||||
@@ -134,15 +137,24 @@ def mel_filters(device, n_mels: int) -> torch.Tensor:
|
||||
filters_path = os.path.join(os.path.dirname(__file__), "assets", "mel_filters.npz")
|
||||
if not os.path.exists(filters_path):
|
||||
source_url = "https://raw.githubusercontent.com/openai/whisper/main/whisper/assets/mel_filters.npz"
|
||||
raise FileNotFoundError(
|
||||
"Missing CosyVoice3 mel filter asset:\n"
|
||||
f" {filters_path}\n"
|
||||
"Download it manually from:\n"
|
||||
f" {source_url}\n"
|
||||
"Example:\n"
|
||||
f" mkdir -p {os.path.dirname(filters_path)} && "
|
||||
f"curl -L {source_url} -o {filters_path}"
|
||||
)
|
||||
os.makedirs(os.path.dirname(filters_path), exist_ok=True)
|
||||
try:
|
||||
import urllib.request
|
||||
|
||||
with urllib.request.urlopen(source_url, timeout=30) as resp:
|
||||
with open(filters_path, "wb") as f_out:
|
||||
f_out.write(resp.read())
|
||||
logger.info("Downloaded mel_filters.npz from %s", source_url)
|
||||
except Exception as e:
|
||||
raise FileNotFoundError(
|
||||
"Missing CosyVoice3 mel filter asset:\n"
|
||||
f" {filters_path}\n"
|
||||
"Auto-download failed. Download it manually from:\n"
|
||||
f" {source_url}\n"
|
||||
"Example:\n"
|
||||
f" mkdir -p {os.path.dirname(filters_path)} && "
|
||||
f"curl -L {source_url} -o {filters_path}"
|
||||
) from e
|
||||
|
||||
with np.load(filters_path, allow_pickle=False) as f:
|
||||
return torch.from_numpy(f[f"mel_{n_mels}"]).to(device)
|
||||
|
||||
@@ -9,16 +9,16 @@ stage_args:
|
||||
runtime:
|
||||
devices: 0
|
||||
engine_args:
|
||||
model_stage: talker
|
||||
model_stage: cosyvoice3_talker
|
||||
worker_cls: vllm_omni.worker.gpu_ar_worker.GPUARWorker
|
||||
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
|
||||
model_arch: CosyVoice3Model
|
||||
trust_remote_code: true
|
||||
gpu_memory_utilization: 0.4
|
||||
gpu_memory_utilization: 0.2
|
||||
engine_output_type: latent # Output speech tokens for chunk aware flow matching
|
||||
disable_hybrid_kv_cache_manager: true
|
||||
enable_prefix_caching: false
|
||||
enforce_eager: false
|
||||
enforce_eager: true
|
||||
mm_processor_cache_gb: 0
|
||||
skip_mm_profiling: true
|
||||
dtype: "float32"
|
||||
@@ -27,14 +27,14 @@ stage_args:
|
||||
runtime:
|
||||
devices: 0
|
||||
engine_args:
|
||||
model_stage: code2wav
|
||||
model_stage: cosyvoice3_code2wav
|
||||
model_arch: CosyVoice3Model
|
||||
trust_remote_code: true
|
||||
worker_cls: vllm_omni.worker.gpu_generation_worker.GPUGenerationWorker
|
||||
scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler
|
||||
engine_output_type: latent
|
||||
gpu_memory_utilization: 0.2
|
||||
enforce_eager: false # CUDA graphs don't work with dynamic runtime_info access
|
||||
gpu_memory_utilization: 0.1
|
||||
enforce_eager: true # CUDA graphs don't work with dynamic conv shapes in code2wav
|
||||
disable_hybrid_kv_cache_manager: true
|
||||
enable_prefix_caching: false
|
||||
skip_mm_profiling: true
|
||||
|
||||
Reference in new issue
Block a user