[CosyVoice3] Add online serving support, fix stage config, and add CI tests (#2431)

Signed-off-by: linyueqian <linyueqian@outlook.com>
This commit is contained in:
Yueqian Lin authored and GitHub committed 2026-04-04 14:55:15 +13:00
1 parent f50c5a413f
commit 4c031580cf
10 files changed
+497 -28

No files matched your search

+15
View File
@@ -390,6 +390,16 @@ steps:
export VLLM_WORKER_MULTIPROC_METHOD=spawn
pytest -s -v tests/e2e/online_serving/test_voxtral_tts.py tests/e2e/offline_inference/test_voxtral_tts.py -m "advanced_model" --run-level "advanced_model"
'
- label: "CosyVoice3-TTS E2E Test"
timeout_in_minutes: 20
depends_on: upload-merge-pipeline
commands:
- |
timeout 20m bash -c '
export VLLM_WORKER_MULTIPROC_METHOD=spawn
pytest -s -v tests/e2e/online_serving/test_cosyvoice3_tts.py -m "advanced_model" --run-level "advanced_model"
'
agents:
queue: "mithril-h100-pool"
plugins:
@@ -408,6 +418,11 @@ steps:
env:
- name: HF_HOME
value: /root/.cache/huggingface
- name: HF_TOKEN
valueFrom:
secretKeyRef:
name: hf-token-secret
key: token
nodeSelector:
node.kubernetes.io/instance-type: gpu-h100-sxm
volumes:
+43
View File
@@ -548,3 +548,46 @@ steps:
hostPath:
path: /mnt/hf-cache
type: DirectoryOrCreate
- label: "CosyVoice3-TTS E2E Test"
timeout_in_minutes: 20
depends_on: upload-ready-pipeline
commands:
- |
timeout 20m bash -c '
export VLLM_WORKER_MULTIPROC_METHOD=spawn
pytest -s -v tests/e2e/online_serving/test_cosyvoice3_tts.py -m "core_model" --run-level "core_model"
'
agents:
queue: "mithril-h100-pool"
plugins:
- kubernetes:
podSpec:
containers:
- image: 936637512419.dkr.ecr.us-west-2.amazonaws.com/vllm-ci-pull-through-cache/q9t5s3a7/vllm-ci-test-repo:$BUILDKITE_COMMIT
resources:
limits:
nvidia.com/gpu: 1
volumeMounts:
- name: devshm
mountPath: /dev/shm
- name: hf-cache
mountPath: /root/.cache/huggingface
env:
- name: HF_HOME
value: /root/.cache/huggingface
- name: HF_TOKEN
valueFrom:
secretKeyRef:
name: hf-token-secret
key: token
nodeSelector:
node.kubernetes.io/instance-type: gpu-h100-sxm
volumes:
- name: devshm
emptyDir:
medium: Memory
- name: hf-cache
hostPath:
path: /mnt/hf-cache
type: DirectoryOrCreate
@@ -0,0 +1,124 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
E2E Online tests for CosyVoice3 TTS model with voice cloning.
These tests verify the /v1/audio/speech endpoint works correctly with
the CosyVoice3 model, which requires reference audio for voice cloning.
"""
import os
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
os.environ["VLLM_TEST_CLEAN_GPU_MEMORY"] = "0"
from pathlib import Path
import pytest
from tests.conftest import OmniServerParams
from tests.utils import hardware_test
MODEL = "FunAudioLLM/Fun-CosyVoice3-0.5B-2512"
# Official CosyVoice zero-shot prompt audio and its transcript
REF_AUDIO_URL = "https://raw.githubusercontent.com/FunAudioLLM/CosyVoice/main/asset/zero_shot_prompt.wav"
REF_TEXT = "希望你以后能够做的比我还好呦。"
def get_stage_config(name: str = "cosyvoice3.yaml"):
"""Get the stage config path from vllm_omni model_executor stage_configs."""
return str(Path(__file__).parent.parent.parent.parent / "vllm_omni" / "model_executor" / "stage_configs" / name)
def get_prompt(prompt_type="zh"):
prompts = {
"zh": "收到好友从远方寄来的生日礼物,那份意外的惊喜与深深的感动让我热泪盈眶。",
"en": "Hello, this is a voice cloning test with English text.",
}
return prompts.get(prompt_type, prompts["zh"])
tts_server_params = [
pytest.param(
OmniServerParams(
model=MODEL,
stage_config_path=get_stage_config(),
server_args=["--trust-remote-code", "--disable-log-stats"],
),
id="cosyvoice3",
)
]
@pytest.mark.advanced_model
@pytest.mark.core_model
@pytest.mark.omni
@hardware_test(res={"cuda": "H100"}, num_cards=1)
@pytest.mark.parametrize("omni_server", tts_server_params, indirect=True)
def test_voice_clone_zh_001(omni_server, openai_client) -> None:
"""
Test voice cloning TTS with Chinese text via OpenAI API.
Deploy Setting: default yaml
Input Modal: text + ref_audio + ref_text
Output Modal: audio
Input Setting: stream=False
Datasets: single request
"""
request_config = {
"model": omni_server.model,
"input": get_prompt("zh"),
"stream": False,
"response_format": "wav",
"ref_audio": REF_AUDIO_URL,
"ref_text": REF_TEXT,
}
openai_client.send_audio_speech_request(request_config)
@pytest.mark.advanced_model
@pytest.mark.omni
@hardware_test(res={"cuda": "H100"}, num_cards=1)
@pytest.mark.parametrize("omni_server", tts_server_params, indirect=True)
def test_voice_clone_zh_002(omni_server, openai_client) -> None:
"""
Test voice cloning TTS with Chinese text via OpenAI API.
Deploy Setting: default yaml
Input Modal: text + ref_audio + ref_text
Output Modal: audio
Input Setting: stream=True
Datasets: single request
"""
request_config = {
"model": omni_server.model,
"input": get_prompt("zh"),
"stream": True,
"response_format": "wav",
"ref_audio": REF_AUDIO_URL,
"ref_text": REF_TEXT,
}
openai_client.send_audio_speech_request(request_config)
@pytest.mark.advanced_model
@pytest.mark.omni
@hardware_test(res={"cuda": "H100"}, num_cards=1)
@pytest.mark.parametrize("omni_server", tts_server_params, indirect=True)
def test_voice_clone_en_001(omni_server, openai_client) -> None:
"""
Test voice cloning TTS with English text via OpenAI API.
Deploy Setting: default yaml
Input Modal: text + ref_audio + ref_text
Output Modal: audio
Input Setting: stream=False
Datasets: single request
"""
request_config = {
"model": omni_server.model,
"input": get_prompt("en"),
"stream": False,
"response_format": "wav",
"ref_audio": REF_AUDIO_URL,
"ref_text": REF_TEXT,
}
openai_client.send_audio_speech_request(request_config)
@@ -1872,3 +1872,115 @@ class TestWAVStreaming:
for fmt in unsupported_formats:
response = client.post("/v1/audio/speech", json={"input": "Hello", "stream": True, "response_format": fmt})
assert response.status_code == 422
# ---- CosyVoice3 Serving Tests ----
@pytest.fixture
def cosyvoice3_server(mocker: MockerFixture):
mocker.patch.object(OmniOpenAIServingSpeech, "_load_supported_speakers", return_value=set())
mocker.patch.object(OmniOpenAIServingSpeech, "_load_codec_frame_rate", return_value=None)
mock_engine_client = mocker.MagicMock()
mock_engine_client.errored = False
mock_engine_client.model_config = mocker.MagicMock(model="FunAudioLLM/Fun-CosyVoice3-0.5B-2512")
mock_engine_client.default_sampling_params_list = [SimpleNamespace(max_tokens=2048)]
mock_engine_client.tts_batch_max_items = 32
mock_engine_client.generate = mocker.MagicMock(return_value="generator")
mock_engine_client.stage_configs = [
SimpleNamespace(
engine_args=SimpleNamespace(model_stage="cosyvoice3_talker"),
tts_args={},
)
]
mock_models = mocker.MagicMock()
mock_models.is_base_model.return_value = True
return OmniOpenAIServingSpeech(
engine_client=mock_engine_client,
models=mock_models,
request_logger=mocker.MagicMock(),
)
class TestCosyVoice3Serving:
def test_cosyvoice3_model_type_detection(self, cosyvoice3_server):
assert cosyvoice3_server._tts_model_type == "cosyvoice3"
assert cosyvoice3_server._is_tts is True
assert cosyvoice3_server._is_cosyvoice3 is True
def test_cosyvoice3_stage_registered(self):
from vllm_omni.entrypoints.openai.serving_speech import (
_COSYVOICE3_TTS_MODEL_STAGES,
_TTS_MODEL_STAGES,
)
assert "cosyvoice3_talker" in _COSYVOICE3_TTS_MODEL_STAGES
assert "cosyvoice3_talker" in _TTS_MODEL_STAGES
def test_validate_cosyvoice3_empty_input(self, cosyvoice3_server):
request = OpenAICreateSpeechRequest(input="", ref_audio="data:audio/wav;base64,abc", ref_text="hello")
error = cosyvoice3_server._validate_cosyvoice3_request(request)
assert error is not None
assert "empty" in error.lower()
def test_validate_cosyvoice3_missing_ref_audio(self, cosyvoice3_server):
request = OpenAICreateSpeechRequest(input="Hello", ref_text="hello")
error = cosyvoice3_server._validate_cosyvoice3_request(request)
assert error is not None
assert "ref_audio" in error.lower()
def test_validate_cosyvoice3_missing_ref_text(self, cosyvoice3_server):
request = OpenAICreateSpeechRequest(input="Hello", ref_audio="data:audio/wav;base64,abc")
error = cosyvoice3_server._validate_cosyvoice3_request(request)
assert error is not None
assert "ref_text" in error.lower()
def test_validate_cosyvoice3_invalid_ref_audio_format(self, cosyvoice3_server):
request = OpenAICreateSpeechRequest(input="Hello", ref_audio="/local/path.wav", ref_text="hello")
error = cosyvoice3_server._validate_cosyvoice3_request(request)
assert error is not None
assert "url" in error.lower() or "format" in error.lower()
def test_validate_cosyvoice3_valid_request(self, cosyvoice3_server):
request = OpenAICreateSpeechRequest(
input="Hello world",
ref_audio="data:audio/wav;base64,abc123",
ref_text="Reference transcript",
)
error = cosyvoice3_server._validate_cosyvoice3_request(request)
assert error is None
def test_validate_cosyvoice3_max_new_tokens_range(self, cosyvoice3_server):
request = OpenAICreateSpeechRequest(
input="Hello",
ref_audio="data:audio/wav;base64,abc",
ref_text="hello",
max_new_tokens=0,
)
error = cosyvoice3_server._validate_cosyvoice3_request(request)
assert error is not None
assert "max_new_tokens" in error
def test_prepare_speech_generation_cosyvoice3(self, cosyvoice3_server):
cosyvoice3_server._build_cosyvoice3_prompt = AsyncMock(
return_value={
"prompt": "Hello",
"multi_modal_data": {"audio": (np.zeros(24000), 24000)},
"mm_processor_kwargs": {"prompt_text": "ref text", "sample_rate": 24000},
}
)
request = OpenAICreateSpeechRequest(
input="Hello",
ref_audio="data:audio/wav;base64,abc",
ref_text="Reference text",
)
request_id, generator, tts_params = asyncio.run(cosyvoice3_server._prepare_speech_generation(request))
assert request_id.startswith("speech-")
assert generator == "generator"
assert tts_params == {}
cosyvoice3_server._build_cosyvoice3_prompt.assert_awaited_once()
+64
View File
@@ -1,5 +1,6 @@
import argparse
import dataclasses
import os
from dataclasses import dataclass, field
from typing import Any
@@ -12,6 +13,18 @@ from vllm_omni.plugins import load_omni_general_plugins
logger = init_logger(__name__)
# Maps model architecture names to their HuggingFace model_type values.
# Used when auto-injecting hf_overrides for models with missing config.json.
_ARCH_TO_MODEL_TYPE: dict[str, str] = {
"CosyVoice3Model": "cosyvoice3",
"OmniVoiceModel": "omnivoice",
}
# Maps model architecture names to tokenizer subfolder paths within HF repos.
_TOKENIZER_SUBFOLDER_MAP: dict[str, str] = {
"CosyVoice3Model": "CosyVoice-BlankEN",
}
def _register_omni_hf_configs() -> None:
try:
@@ -29,6 +42,14 @@ def _register_omni_hf_configs() -> None:
logger.warning("Skipping omni HF config registration due to import error: %s", exc)
return
# Register with both transformers AutoConfig and vLLM's config registry
# so models with empty/missing config.json (e.g. CosyVoice3) can be
# resolved when model_type is injected via hf_overrides.
try:
from vllm.transformers_utils.config import _CONFIG_REGISTRY
except ImportError:
_CONFIG_REGISTRY = None
for model_type, config_cls in [
("qwen3_tts", Qwen3TTSConfig),
("cosyvoice3", CosyVoice3Config),
@@ -40,6 +61,8 @@ def _register_omni_hf_configs() -> None:
except ValueError:
# Already registered elsewhere; ignore.
pass
if _CONFIG_REGISTRY is not None and model_type not in _CONFIG_REGISTRY:
_CONFIG_REGISTRY[model_type] = config_cls
def register_omni_models_to_vllm():
@@ -129,11 +152,52 @@ class OmniEngineArgs(EngineArgs):
# If model_arch is specified, inject it into hf_overrides so vLLM can
# resolve the architecture even when config.json lacks 'architectures'.
# Also inject model_type so AutoConfig can resolve the correct config
# class for models with empty or missing config.json (e.g. CosyVoice3).
if self.model_arch:
if self.hf_overrides is None:
self.hf_overrides = {}
if isinstance(self.hf_overrides, dict):
self.hf_overrides.setdefault("architectures", [self.model_arch])
if "model_type" not in self.hf_overrides:
model_type = _ARCH_TO_MODEL_TYPE.get(self.model_arch)
if model_type is not None:
self.hf_overrides.setdefault("model_type", model_type)
# Auto-detect tokenizer for models that store it in a subdirectory
# rather than the root (e.g. CosyVoice3 uses CosyVoice-BlankEN/).
if not self.tokenizer and self.model:
model_path = self.model
if os.path.isdir(model_path) and not os.path.isfile(os.path.join(model_path, "tokenizer_config.json")):
for subfolder in sorted(os.listdir(model_path)):
candidate = os.path.join(model_path, subfolder)
if os.path.isdir(candidate) and os.path.isfile(os.path.join(candidate, "tokenizer_config.json")):
self.tokenizer = candidate
logger.info("Auto-detected tokenizer at %s", candidate)
break
elif not os.path.isdir(model_path):
subfolder = _TOKENIZER_SUBFOLDER_MAP.get(self.model_arch)
if subfolder:
# Download just the tokenizer files from the subfolder
try:
from huggingface_hub import snapshot_download
local_dir = snapshot_download(
model_path,
allow_patterns=[
f"{subfolder}/tokenizer*",
f"{subfolder}/special_tokens*",
f"{subfolder}/vocab*",
f"{subfolder}/merges*",
f"{subfolder}/added_tokens*",
],
)
candidate = os.path.join(local_dir, subfolder)
if os.path.isdir(candidate):
self.tokenizer = candidate
logger.info("Downloaded tokenizer from %s/%s", model_path, subfolder)
except Exception as e:
logger.warning("Failed to download tokenizer subfolder: %s", e)
# Build the vLLM config first, then use it to create the Omni config.
model_config = super().create_model_config()
+16
View File
@@ -73,6 +73,16 @@ from vllm_omni.platforms import current_omni_platform
logger = init_logger(__name__)
def _patch_generation_config_if_needed(model_config: Any) -> None:
"""Ensure try_get_generation_config won't crash for models whose HF
config.json lacks model_type (e.g. CosyVoice3). We probe it once;
if it raises, we monkey-patch the method to return None."""
try:
model_config.try_get_generation_config()
except Exception:
model_config.try_get_generation_config = lambda: {}
def _inject_kv_stage_info(stage_cfg: Any, stage_id: int) -> None:
"""Inject stage_id and engine_input_source into omni_kv_config.
@@ -409,6 +419,12 @@ class AsyncOmniEngine:
)
input_processor = None
if started.stage_id == 0:
# Some omni models (e.g. CosyVoice3) have an empty HF
# config.json without model_type, which causes
# try_get_generation_config -> AutoConfig.from_pretrained
# to raise ValueError. Patch it to return None so
# InputProcessor doesn't crash.
_patch_generation_config_if_needed(started.vllm_config.model_config)
input_processor = InputProcessor(vllm_config=started.vllm_config)
# Use omni preprocessor so text-only prompts with
# mm_processor_kwargs (e.g. GLM-Image t2i target_h/target_w)
+73 -1
View File
@@ -46,9 +46,14 @@ logger = init_logger(__name__)
_VOXTRAL_TTS_MODEL_STAGES = {"audio_generation"}
_QWEN3_TTS_MODEL_STAGES = {"qwen3_tts"}
_FISH_TTS_MODEL_STAGES = {"fish_speech_slow_ar"}
_COSYVOICE3_TTS_MODEL_STAGES = {"cosyvoice3_talker"}
_OMNIVOICE_TTS_MODEL_STAGES = {"omnivoice_generator"}
_TTS_MODEL_STAGES: set[str] = (
_VOXTRAL_TTS_MODEL_STAGES | _QWEN3_TTS_MODEL_STAGES | _FISH_TTS_MODEL_STAGES | _OMNIVOICE_TTS_MODEL_STAGES
_VOXTRAL_TTS_MODEL_STAGES
| _QWEN3_TTS_MODEL_STAGES
| _FISH_TTS_MODEL_STAGES
| _COSYVOICE3_TTS_MODEL_STAGES
| _OMNIVOICE_TTS_MODEL_STAGES
)
_TTS_LANGUAGES: set[str] = {
"Auto",
@@ -184,6 +189,13 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
)
self._fish_speech_tokenizer = None
self._is_cosyvoice3 = (
self._tts_stage is not None
and getattr(getattr(self._tts_stage, "engine_args", None), "model_stage", None)
in _COSYVOICE3_TTS_MODEL_STAGES
)
self._cosyvoice3_tokenizer = None
# Determine TTS model type or None
self._tts_model_type = self._detect_tts_model_type()
@@ -258,6 +270,8 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
return "voxtral_tts"
if model_stage in _FISH_TTS_MODEL_STAGES:
return "fish_tts"
if model_stage in _COSYVOICE3_TTS_MODEL_STAGES:
return "cosyvoice3"
if model_stage in _OMNIVOICE_TTS_MODEL_STAGES:
return "omnivoice"
return None
@@ -713,6 +727,8 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
return self._validate_voxtral_tts_request(request)
if self._tts_model_type == "fish_tts":
return self._validate_fish_tts_request(request)
if self._tts_model_type == "cosyvoice3":
return self._validate_cosyvoice3_request(request)
return self._validate_qwen_tts_request(request)
def _validate_ref_audio_format(self, ref_audio: str) -> str | None:
@@ -884,6 +900,30 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
return None
def _validate_cosyvoice3_request(self, request: OpenAICreateSpeechRequest) -> str | None:
"""Validate CosyVoice3 request parameters. Returns error message or None."""
if not request.input or not request.input.strip():
return "Input text cannot be empty"
# CosyVoice3 requires reference audio for voice cloning
if request.ref_audio is None:
return "CosyVoice3 requires 'ref_audio' (reference audio for voice cloning)"
fmt_err = self._validate_ref_audio_format(request.ref_audio)
if fmt_err:
return fmt_err
if not request.ref_text or not request.ref_text.strip():
return "CosyVoice3 requires 'ref_text' (transcript of the reference audio)"
if request.max_new_tokens is not None:
if request.max_new_tokens < _TTS_MAX_NEW_TOKENS_MIN:
return f"max_new_tokens must be at least {_TTS_MAX_NEW_TOKENS_MIN}"
if request.max_new_tokens > _TTS_MAX_NEW_TOKENS_MAX:
return f"max_new_tokens cannot exceed {_TTS_MAX_NEW_TOKENS_MAX}"
return None
async def _resolve_ref_audio(self, ref_audio_str: str) -> tuple[list[float], int]:
"""Resolve ref_audio to (wav_samples, sample_rate).
@@ -1194,6 +1234,33 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
"additional_information": additional_information,
}
# ---- CosyVoice3 helpers ----
async def _build_cosyvoice3_prompt(
self,
request: OpenAICreateSpeechRequest,
) -> dict[str, Any]:
"""Build prompt for CosyVoice3.
CosyVoice3 uses multimodal input with reference audio for voice cloning.
The prompt format matches the offline example: text prompt + audio data
+ mm_processor_kwargs with prompt_text.
"""
# Resolve reference audio
wav_samples, sr = await self._resolve_ref_audio(request.ref_audio)
audio_data = (np.asarray(wav_samples, dtype=np.float32), sr)
return {
"prompt": request.input,
"multi_modal_data": {
"audio": audio_data,
},
"mm_processor_kwargs": {
"prompt_text": request.ref_text,
"sample_rate": sr,
},
}
# ---- Common speech generation helpers ----
async def _prepare_speech_generation(
@@ -1224,6 +1291,9 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
if self._tts_model_type == "voxtral_tts":
prompt = await self._build_voxtral_prompt(request)
tts_params = {}
elif self._tts_model_type == "cosyvoice3":
prompt = await self._build_cosyvoice3_prompt(request)
tts_params = {}
else:
tts_params = self._build_tts_params(request)
# Resolve ref_audio (explicit or auto-set for uploaded voices)
@@ -1247,6 +1317,8 @@ class OmniOpenAIServingSpeech(OpenAIServing, AudioMixin):
model_type = "fish_speech"
elif self._tts_model_type == "voxtral_tts":
model_type = "voxtral_tts"
elif self._tts_model_type == "cosyvoice3":
model_type = "cosyvoice3"
elif self._is_tts:
model_type = tts_params.get("task_type", ["unknown"])[0]
else:
@@ -66,6 +66,12 @@ class CosyVoice3MultiModalProcessor(BaseMultiModalProcessor[CosyVoice3MultiModal
if cached_model_dir == model_dir:
return
# If model_dir is an HF repo ID (not a local path), resolve to cache
if not os.path.isdir(model_dir):
from huggingface_hub import snapshot_download
model_dir = snapshot_download(model_dir)
import onnxruntime
from vllm_omni.model_executor.models.cosyvoice3.tokenizer import get_qwen_tokenizer
@@ -266,9 +272,14 @@ class CosyVoice3Model(
self.config = vllm_config.model_config.hf_config
self.have_multimodal_outputs = True
self.model_stage = vllm_config.model_config.model_stage
self.model_dir = vllm_config.model_config.model
model_dir = vllm_config.model_config.model
if not os.path.isdir(model_dir):
from huggingface_hub import snapshot_download
model_dir = snapshot_download(model_dir)
self.model_dir = model_dir
self.model = None
if self.model_stage == "talker":
if self.model_stage == "cosyvoice3_talker":
# Initialize talker stage (text to speech tokens)
from vllm_omni.model_executor.models.cosyvoice3.cosyvoice3_talker import CosyVoice3LM, VLLMQwen2Encoder
@@ -286,7 +297,7 @@ class CosyVoice3Model(
# KV cache is now managed externally by vLLM's PagedAttention
# No need for self.llm_cache
self.model = self.talker
elif self.model_stage == "code2wav":
elif self.model_stage == "cosyvoice3_code2wav":
# Initialize code2wav stage (flow matching + vocoder)
from vllm_omni.model_executor.models.cosyvoice3.cosyvoice3_code2wav import CosyVoice3Code2Wav
@@ -322,7 +333,7 @@ class CosyVoice3Model(
def compute_logits(self, hidden_states: torch.Tensor | OmniOutput) -> torch.Tensor | None:
if isinstance(hidden_states, OmniOutput):
hidden_states = hidden_states.text_hidden_states
if self.model_stage == "talker":
if self.model_stage == "cosyvoice3_talker":
logits = self.model.llm_decoder(hidden_states)
vocab_size = self.config.vocab_size
pad_size = vocab_size - logits.size(-1)
@@ -337,7 +348,7 @@ class CosyVoice3Model(
raise RuntimeError(f"compute_logits is only valid for {self.model_stage}.")
def embed_multimodal(self, **kwargs: object) -> torch.Tensor:
if self.model_stage == "talker":
if self.model_stage == "cosyvoice3_talker":
speech_token = kwargs["speech_token"]
speech_token_emb = self.model.speech_embedding(speech_token)
return speech_token_emb
@@ -350,7 +361,7 @@ class CosyVoice3Model(
multimodal_embeddings=None,
is_multimodal=None,
) -> torch.Tensor:
if self.model_stage == "talker":
if self.model_stage == "cosyvoice3_talker":
if is_multimodal is not None and any(is_multimodal):
embed_tokens = self.model.llm.model.embed_tokens(input_ids)
sos = self.model.speech_embedding.weight[self.model.sos].reshape(1, -1)
@@ -363,7 +374,7 @@ class CosyVoice3Model(
else:
embed_tokens = self.model.speech_embedding.weight[input_ids]
return embed_tokens
elif self.model_stage == "code2wav":
elif self.model_stage == "cosyvoice3_code2wav":
assert input_ids.dim() == 1
hidden = int(self.config.hidden_size)
return torch.zeros(
@@ -381,7 +392,7 @@ class CosyVoice3Model(
additional_information: dict[str, object] | None = None,
**kwargs: object,
) -> OmniOutput:
if self.model_stage == "talker":
if self.model_stage == "cosyvoice3_talker":
if inputs_embeds is None:
inputs_embeds = self.embed_input_ids(input_ids)
@@ -399,7 +410,7 @@ class CosyVoice3Model(
}
return OmniOutput(text_hidden_states=hidden_states, multimodal_outputs=multimodal_outputs)
elif self.model_stage == "code2wav":
elif self.model_stage == "cosyvoice3_code2wav":
runtime_info = kwargs.get("runtime_additional_information", [])
if not runtime_info:
length = 30 * 24000
@@ -420,13 +431,13 @@ class CosyVoice3Model(
return OmniOutput(
text_hidden_states=None,
multimodal_outputs={"audio": tts_speech},
multimodal_outputs={"audio": tts_speech, "sr": 22050},
)
else:
raise ValueError(f"Unsupported model_stage: {self.model_stage}")
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
if self.model_stage == "talker":
if self.model_stage == "cosyvoice3_talker":
# Load weights for text to speech LM stage using vLLM's weight loading
llm_weight_path = os.path.join(self.model_dir, "llm.pt")
device = next(self.parameters()).device
@@ -460,7 +471,7 @@ class CosyVoice3Model(
self.model.llm_decoder.load_state_dict(llm_decoder_state)
self.model.to(device).eval()
elif self.model_stage == "code2wav":
elif self.model_stage == "cosyvoice3_code2wav":
# Load weights for code2wav stage (flow + hift)
device = next(self.parameters()).device
self.code2wav.load_weights(self.model_dir, device)
@@ -1,5 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import logging
import os
from functools import cache, lru_cache
@@ -10,6 +11,8 @@ import torchaudio
import torchaudio.compliance.kaldi as kaldi
from librosa.filters import mel as librosa_mel_fn
logger = logging.getLogger(__name__)
IGNORE_ID = -1
@@ -134,15 +137,24 @@ def mel_filters(device, n_mels: int) -> torch.Tensor:
filters_path = os.path.join(os.path.dirname(__file__), "assets", "mel_filters.npz")
if not os.path.exists(filters_path):
source_url = "https://raw.githubusercontent.com/openai/whisper/main/whisper/assets/mel_filters.npz"
raise FileNotFoundError(
"Missing CosyVoice3 mel filter asset:\n"
f" {filters_path}\n"
"Download it manually from:\n"
f" {source_url}\n"
"Example:\n"
f" mkdir -p {os.path.dirname(filters_path)} && "
f"curl -L {source_url} -o {filters_path}"
)
os.makedirs(os.path.dirname(filters_path), exist_ok=True)
try:
import urllib.request
with urllib.request.urlopen(source_url, timeout=30) as resp:
with open(filters_path, "wb") as f_out:
f_out.write(resp.read())
logger.info("Downloaded mel_filters.npz from %s", source_url)
except Exception as e:
raise FileNotFoundError(
"Missing CosyVoice3 mel filter asset:\n"
f" {filters_path}\n"
"Auto-download failed. Download it manually from:\n"
f" {source_url}\n"
"Example:\n"
f" mkdir -p {os.path.dirname(filters_path)} && "
f"curl -L {source_url} -o {filters_path}"
) from e
with np.load(filters_path, allow_pickle=False) as f:
return torch.from_numpy(f[f"mel_{n_mels}"]).to(device)
@@ -9,16 +9,16 @@ stage_args:
runtime:
devices: 0
engine_args:
model_stage: talker
model_stage: cosyvoice3_talker
worker_cls: vllm_omni.worker.gpu_ar_worker.GPUARWorker
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
model_arch: CosyVoice3Model
trust_remote_code: true
gpu_memory_utilization: 0.4
gpu_memory_utilization: 0.2
engine_output_type: latent # Output speech tokens for chunk aware flow matching
disable_hybrid_kv_cache_manager: true
enable_prefix_caching: false
enforce_eager: false
enforce_eager: true
mm_processor_cache_gb: 0
skip_mm_profiling: true
dtype: "float32"
@@ -27,14 +27,14 @@ stage_args:
runtime:
devices: 0
engine_args:
model_stage: code2wav
model_stage: cosyvoice3_code2wav
model_arch: CosyVoice3Model
trust_remote_code: true
worker_cls: vllm_omni.worker.gpu_generation_worker.GPUGenerationWorker
scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler
engine_output_type: latent
gpu_memory_utilization: 0.2
enforce_eager: false # CUDA graphs don't work with dynamic runtime_info access
gpu_memory_utilization: 0.1
enforce_eager: true # CUDA graphs don't work with dynamic conv shapes in code2wav
disable_hybrid_kv_cache_manager: true
enable_prefix_caching: false
skip_mm_profiling: true