From d92439c155dd315a662a6fb7d7efe94103d84065 Mon Sep 17 00:00:00 2001 From: Yueqian Lin <70319226+linyueqian@users.noreply.github.com> Date: Sat, 4 Apr 2026 20:45:49 -0400 Subject: [PATCH] [CosyVoice3] Fix vLLM 0.19.0 compatibility issues (#2486) --- .../e2e/online_serving/test_cosyvoice3_tts.py | 1 + vllm_omni/entrypoints/utils.py | 28 ++++++++++++++++++- .../models/cosyvoice3/config.py | 2 ++ .../models/cosyvoice3/cosyvoice3.py | 2 +- .../models/cosyvoice3/cosyvoice3_code2wav.py | 6 ++-- .../stage_configs/cosyvoice3.yaml | 5 ++++ 6 files changed, 40 insertions(+), 4 deletions(-) diff --git a/tests/e2e/online_serving/test_cosyvoice3_tts.py b/tests/e2e/online_serving/test_cosyvoice3_tts.py index 976be805c..1845d7818 100644 --- a/tests/e2e/online_serving/test_cosyvoice3_tts.py +++ b/tests/e2e/online_serving/test_cosyvoice3_tts.py @@ -80,6 +80,7 @@ def test_voice_clone_zh_001(omni_server, openai_client) -> None: @pytest.mark.omni @hardware_test(res={"cuda": "H100"}, num_cards=1) @pytest.mark.parametrize("omni_server", tts_server_params, indirect=True) +@pytest.mark.skip(reason="CosyVoice3 does not support async_chunk streaming yet") def test_voice_clone_zh_002(omni_server, openai_client) -> None: """ Test voice cloning TTS with Chinese text via OpenAI API. diff --git a/vllm_omni/entrypoints/utils.py b/vllm_omni/entrypoints/utils.py index e29e9eea1..0e1000ec9 100644 --- a/vllm_omni/entrypoints/utils.py +++ b/vllm_omni/entrypoints/utils.py @@ -182,6 +182,28 @@ def _convert_dataclasses_to_dict(obj: Any) -> Any: return obj +def _try_resolve_omni_model_type(model: str) -> str | None: + """Try to resolve model_type for omni models with empty config.json. + + Checks if any registered omni stage config file name matches a substring + in the model name (e.g. 'cosyvoice3' in 'FunAudioLLM/Fun-CosyVoice3-0.5B-2512'). + When multiple configs match, the longest stem wins to avoid ambiguity + (e.g. 'bagel_single_stage' over 'bagel'). + """ + stage_configs_dir = PROJECT_ROOT / "vllm_omni" / "model_executor" / "stage_configs" + if not stage_configs_dir.exists(): + return None + model_lower = model.lower().replace("-", "").replace("_", "") + best_match: str | None = None + best_len = 0 + for config_file in sorted(stage_configs_dir.glob("*.yaml")): + candidate = config_file.stem.replace("-", "").replace("_", "") + if candidate in model_lower and len(candidate) > best_len: + best_match = config_file.stem + best_len = len(candidate) + return best_match + + def resolve_model_config_path(model: str) -> str: """Resolve the stage config file path from the model name. @@ -220,7 +242,11 @@ def resolve_model_config_path(model: str) -> str: if config_dict and "model_type" in config_dict: model_type = config_dict["model_type"] else: - raise ValueError(f"config.json found but missing 'model_type' for model: {model}") + # For models with empty config.json (e.g. CosyVoice3), + # try matching against registered omni stage configs. + model_type = _try_resolve_omni_model_type(model) + if model_type is None: + raise ValueError(f"config.json found but missing 'model_type' for model: {model}") except Exception as e: raise ValueError(f"Failed to read config.json for model: {model}. Error: {e}") from e else: diff --git a/vllm_omni/model_executor/models/cosyvoice3/config.py b/vllm_omni/model_executor/models/cosyvoice3/config.py index 0c9a28997..b4e44b7a8 100644 --- a/vllm_omni/model_executor/models/cosyvoice3/config.py +++ b/vllm_omni/model_executor/models/cosyvoice3/config.py @@ -7,6 +7,8 @@ class CosyVoice3Config(PretrainedConfig): model_type = "cosyvoice3" def __init__(self, **kwargs): + # Set speech EOS so vLLM stops generation at the right token + kwargs.setdefault("eos_token_id", 6562) super().__init__(**kwargs) self.sample_rate = 24000 self.llm_input_size = 896 diff --git a/vllm_omni/model_executor/models/cosyvoice3/cosyvoice3.py b/vllm_omni/model_executor/models/cosyvoice3/cosyvoice3.py index bc04aae33..18a16ba55 100644 --- a/vllm_omni/model_executor/models/cosyvoice3/cosyvoice3.py +++ b/vllm_omni/model_executor/models/cosyvoice3/cosyvoice3.py @@ -432,7 +432,7 @@ class CosyVoice3Model( return OmniOutput( text_hidden_states=None, - multimodal_outputs={"audio": tts_speech, "sr": 22050}, + multimodal_outputs={"audio": tts_speech, "sr": torch.tensor(22050)}, ) else: raise ValueError(f"Unsupported model_stage: {self.model_stage}") diff --git a/vllm_omni/model_executor/models/cosyvoice3/cosyvoice3_code2wav.py b/vllm_omni/model_executor/models/cosyvoice3/cosyvoice3_code2wav.py index f5e0d04a8..222d6d98a 100644 --- a/vllm_omni/model_executor/models/cosyvoice3/cosyvoice3_code2wav.py +++ b/vllm_omni/model_executor/models/cosyvoice3/cosyvoice3_code2wav.py @@ -192,8 +192,10 @@ class CosyVoice3Code2Wav(nn.Module): # Create mask mask = (~make_pad_mask(full_token_len)).unsqueeze(-1).to(embedding) - # Token embedding - token_emb = self.input_embedding(torch.clamp(full_token, min=0)) * mask + # Token embedding (clamp to valid codebook range; EOS/padding tokens may exceed vocab_size) + token_emb = ( + self.input_embedding(torch.clamp(full_token, min=0, max=self.input_embedding.num_embeddings - 1)) * mask + ) # Pre-lookahead processing h = self.pre_lookahead_layer(token_emb) diff --git a/vllm_omni/model_executor/stage_configs/cosyvoice3.yaml b/vllm_omni/model_executor/stage_configs/cosyvoice3.yaml index e215f5142..bfb847f5e 100644 --- a/vllm_omni/model_executor/stage_configs/cosyvoice3.yaml +++ b/vllm_omni/model_executor/stage_configs/cosyvoice3.yaml @@ -22,6 +22,9 @@ stage_args: mm_processor_cache_gb: 0 skip_mm_profiling: true dtype: "float32" + default_sampling_params: + max_tokens: 2048 + stop_token_ids: [6562] # speech EOS token - stage_id: 1 runtime: @@ -39,6 +42,8 @@ stage_args: enable_prefix_caching: false skip_mm_profiling: true dtype: "float32" + default_sampling_params: + max_tokens: 2048 engine_input_source: [0] custom_process_input_func: vllm_omni.model_executor.stage_input_processors.cosyvoice3.text2flow final_output: true