[Config Refactor] Migrate 5 TTS models (VoxCPM2 / CosyVoice3 / MiMo Audio / Voxtral TTS / Fish Speech S2 Pro) to Pipeline + Deploy schema (#2958)

Signed-off-by: Yueqian Lin <linyueqian@outlook.com>
Co-authored-by: SYLAR <125541396+lishunyang12@users.noreply.github.com>
This commit is contained in:
Yueqian LinandSYLAR authored and GitHub committed 2026-04-21 21:17:53 +08:00
1 parent c2578f7131
commit 3f504b45d2
47 files changed
+702 -844

No files matched your search

@@ -0,0 +1,50 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""CosyVoice3 pipeline topology (frozen).
Stage 0: Talker — text prompt → speech tokens (LLM autoregressive).
Stage 1: Code2Wav — flow-matching decoder → acoustic features → waveform.
* ``sync_process_input_func`` runs when ``deploy.async_chunk=false``:
stage 1 builds full-sequence flow input via ``text2flow``.
* ``async_chunk_process_next_stage_input_func`` runs when
``deploy.async_chunk=true``: stage 0 streams codec chunks to stage 1
through the shared-memory connector.
"""
from vllm_omni.config.stage_config import (
PipelineConfig,
StageExecutionType,
StagePipelineConfig,
)
_PROC = "vllm_omni.model_executor.stage_input_processors.cosyvoice3"
COSYVOICE3_PIPELINE = PipelineConfig(
model_type="cosyvoice3",
model_arch="CosyVoice3Model",
stages=(
StagePipelineConfig(
stage_id=0,
model_stage="cosyvoice3_talker",
execution_type=StageExecutionType.LLM_AR,
input_sources=(),
owns_tokenizer=True,
engine_output_type="latent",
async_chunk_process_next_stage_input_func=(f"{_PROC}.talker2code2wav_async_chunk"),
sampling_constraints={
# merged speech stop token (logsumexp of all 200 stop logits)
"stop_token_ids": [6562],
},
),
StagePipelineConfig(
stage_id=1,
model_stage="cosyvoice3_code2wav",
execution_type=StageExecutionType.LLM_GENERATION,
input_sources=(0,),
final_output=True,
final_output_type="audio",
engine_output_type="latent",
sync_process_input_func=f"{_PROC}.text2flow",
),
),
)
@@ -0,0 +1,51 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Fish Speech S2 Pro pipeline topology (frozen).
Stage 0: slow_ar — text → RVQ codec tokens (LLM autoregressive).
Stage 1: dac_decoder — RVQ tokens → audio waveform (LLM_GENERATION).
The HF config top-level reports ``model_type = "fish_qwen3_omni"`` (the
OmniConfig that bundles slow-AR and fast-AR sub-configs), which is why the
registry key follows the HF name rather than the human-readable "fish_speech".
"""
from vllm_omni.config.stage_config import (
PipelineConfig,
StageExecutionType,
StagePipelineConfig,
)
_PROC = "vllm_omni.model_executor.stage_input_processors.fish_speech"
FISH_SPEECH_PIPELINE = PipelineConfig(
model_type="fish_qwen3_omni",
model_arch="FishSpeechSlowARForConditionalGeneration",
stages=(
StagePipelineConfig(
stage_id=0,
model_stage="fish_speech_slow_ar",
execution_type=StageExecutionType.LLM_AR,
input_sources=(),
owns_tokenizer=True,
engine_output_type="latent",
async_chunk_process_next_stage_input_func=(f"{_PROC}.slow_ar_to_dac_decoder_async_chunk"),
sampling_constraints={
"detokenize": False,
# <|im_end|> — stop when the model emits end-of-turn.
"stop_token_ids": [151645],
},
),
StagePipelineConfig(
stage_id=1,
model_stage="dac_decoder",
model_arch="FishSpeechDACDecoder",
execution_type=StageExecutionType.LLM_GENERATION,
input_sources=(0,),
final_output=True,
final_output_type="audio",
engine_output_type="audio",
sampling_constraints={"detokenize": True},
),
),
)
@@ -0,0 +1,54 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""MiMo Audio pipeline topology (frozen).
Stage 0: fused thinker+talker — multimodal understanding + text + RVQ codes.
Stage 1: Code2Wav — RVQ codes → waveform.
MiMoAudioConfig inherits from Qwen2Config, so the HF ``model_type`` field
reports ``qwen2`` — the registry's model_type-based auto-detect can't
disambiguate. ``hf_architectures`` lets ``StageConfigFactory`` fall back to
matching ``hf_config.architectures`` instead.
"""
from vllm_omni.config.stage_config import (
PipelineConfig,
StageExecutionType,
StagePipelineConfig,
)
_PROC = "vllm_omni.model_executor.stage_input_processors.mimo_audio"
MIMO_AUDIO_PIPELINE = PipelineConfig(
model_type="mimo_audio",
# HF ``architectures: ["MiMoAudioModel"]`` is also the registry key in
# ``model_executor/models/registry.py``; it resolves to the internal
# class ``MiMoAudioForConditionalGeneration`` in ``mimo_audio.py``.
model_arch="MiMoAudioModel",
hf_architectures=("MiMoAudioModel",),
stages=(
StagePipelineConfig(
stage_id=0,
model_stage="fused_thinker_talker",
execution_type=StageExecutionType.LLM_AR,
input_sources=(),
final_output=True,
final_output_type="text",
owns_tokenizer=True,
engine_output_type="latent",
async_chunk_process_next_stage_input_func=(f"{_PROC}.llm2code2wav_async_chunk"),
sampling_constraints={"detokenize": True},
),
StagePipelineConfig(
stage_id=1,
model_stage="code2wav",
execution_type=StageExecutionType.LLM_GENERATION,
input_sources=(0,),
final_output=True,
final_output_type="audio",
engine_output_type="audio",
sync_process_input_func=f"{_PROC}.llm2code2wav",
sampling_constraints={"detokenize": False},
),
),
)
@@ -0,0 +1,35 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""VoxCPM2 pipeline topology (frozen).
Single-stage AR TTS: text → speech waveform in one pass.
Uses the native MiniCPM4 base_lm with a per-request StaticKVCache that the
talker restores into the paged attention layer at step boundaries.
"""
from vllm_omni.config.stage_config import (
PipelineConfig,
StageExecutionType,
StagePipelineConfig,
)
VOXCPM2_PIPELINE = PipelineConfig(
model_type="voxcpm2",
model_arch="VoxCPM2TalkerForConditionalGeneration",
stages=(
StagePipelineConfig(
stage_id=0,
model_stage="latent_generator",
execution_type=StageExecutionType.LLM_AR,
input_sources=(),
final_output=True,
final_output_type="audio",
owns_tokenizer=True,
engine_output_type="audio",
sampling_constraints={
"detokenize": False,
"stop_token_ids": [1],
},
),
),
)
Whitespace-only changes.
@@ -0,0 +1,45 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Voxtral TTS pipeline topology (frozen).
Stage 0: audio_generation — text → acoustic latents (LLM_AR, tokenizer owner).
Stage 1: audio_tokenizer — acoustic latents → waveform (LLM_GENERATION).
"""
from vllm_omni.config.stage_config import (
PipelineConfig,
StageExecutionType,
StagePipelineConfig,
)
_PROC = "vllm_omni.model_executor.stage_input_processors.voxtral_tts"
VOXTRAL_TTS_PIPELINE = PipelineConfig(
model_type="voxtral_tts",
model_arch="VoxtralTTSForConditionalGeneration",
stages=(
StagePipelineConfig(
stage_id=0,
model_stage="audio_generation",
execution_type=StageExecutionType.LLM_AR,
input_sources=(),
final_output=False,
final_output_type="text",
owns_tokenizer=True,
engine_output_type="latent",
async_chunk_process_next_stage_input_func=(f"{_PROC}.generator2tokenizer_async_chunk"),
sampling_constraints={"detokenize": True},
),
StagePipelineConfig(
stage_id=1,
model_stage="audio_tokenizer",
execution_type=StageExecutionType.LLM_GENERATION,
input_sources=(0,),
final_output=True,
final_output_type="audio",
engine_output_type="audio",
sampling_constraints={"detokenize": True},
extras={"tts_args": {"max_instructions_length": 500}},
),
),
)