[Config Refactor] Migrate 5 TTS models (VoxCPM2 / CosyVoice3 / MiMo Audio / Voxtral TTS / Fish Speech S2 Pro) to Pipeline + Deploy schema (#2958)
Signed-off-by: Yueqian Lin <linyueqian@outlook.com> Co-authored-by: SYLAR <125541396+lishunyang12@users.noreply.github.com>
This commit is contained in:
1 parent
c2578f7131
commit
3f504b45d2
47 files changed
+702
-844
No files matched your search
@@ -0,0 +1,50 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""CosyVoice3 pipeline topology (frozen).
|
||||
|
||||
Stage 0: Talker — text prompt → speech tokens (LLM autoregressive).
|
||||
Stage 1: Code2Wav — flow-matching decoder → acoustic features → waveform.
|
||||
* ``sync_process_input_func`` runs when ``deploy.async_chunk=false``:
|
||||
stage 1 builds full-sequence flow input via ``text2flow``.
|
||||
* ``async_chunk_process_next_stage_input_func`` runs when
|
||||
``deploy.async_chunk=true``: stage 0 streams codec chunks to stage 1
|
||||
through the shared-memory connector.
|
||||
"""
|
||||
|
||||
from vllm_omni.config.stage_config import (
|
||||
PipelineConfig,
|
||||
StageExecutionType,
|
||||
StagePipelineConfig,
|
||||
)
|
||||
|
||||
_PROC = "vllm_omni.model_executor.stage_input_processors.cosyvoice3"
|
||||
|
||||
COSYVOICE3_PIPELINE = PipelineConfig(
|
||||
model_type="cosyvoice3",
|
||||
model_arch="CosyVoice3Model",
|
||||
stages=(
|
||||
StagePipelineConfig(
|
||||
stage_id=0,
|
||||
model_stage="cosyvoice3_talker",
|
||||
execution_type=StageExecutionType.LLM_AR,
|
||||
input_sources=(),
|
||||
owns_tokenizer=True,
|
||||
engine_output_type="latent",
|
||||
async_chunk_process_next_stage_input_func=(f"{_PROC}.talker2code2wav_async_chunk"),
|
||||
sampling_constraints={
|
||||
# merged speech stop token (logsumexp of all 200 stop logits)
|
||||
"stop_token_ids": [6562],
|
||||
},
|
||||
),
|
||||
StagePipelineConfig(
|
||||
stage_id=1,
|
||||
model_stage="cosyvoice3_code2wav",
|
||||
execution_type=StageExecutionType.LLM_GENERATION,
|
||||
input_sources=(0,),
|
||||
final_output=True,
|
||||
final_output_type="audio",
|
||||
engine_output_type="latent",
|
||||
sync_process_input_func=f"{_PROC}.text2flow",
|
||||
),
|
||||
),
|
||||
)
|
||||
@@ -0,0 +1,51 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Fish Speech S2 Pro pipeline topology (frozen).
|
||||
|
||||
Stage 0: slow_ar — text → RVQ codec tokens (LLM autoregressive).
|
||||
Stage 1: dac_decoder — RVQ tokens → audio waveform (LLM_GENERATION).
|
||||
|
||||
The HF config top-level reports ``model_type = "fish_qwen3_omni"`` (the
|
||||
OmniConfig that bundles slow-AR and fast-AR sub-configs), which is why the
|
||||
registry key follows the HF name rather than the human-readable "fish_speech".
|
||||
"""
|
||||
|
||||
from vllm_omni.config.stage_config import (
|
||||
PipelineConfig,
|
||||
StageExecutionType,
|
||||
StagePipelineConfig,
|
||||
)
|
||||
|
||||
_PROC = "vllm_omni.model_executor.stage_input_processors.fish_speech"
|
||||
|
||||
FISH_SPEECH_PIPELINE = PipelineConfig(
|
||||
model_type="fish_qwen3_omni",
|
||||
model_arch="FishSpeechSlowARForConditionalGeneration",
|
||||
stages=(
|
||||
StagePipelineConfig(
|
||||
stage_id=0,
|
||||
model_stage="fish_speech_slow_ar",
|
||||
execution_type=StageExecutionType.LLM_AR,
|
||||
input_sources=(),
|
||||
owns_tokenizer=True,
|
||||
engine_output_type="latent",
|
||||
async_chunk_process_next_stage_input_func=(f"{_PROC}.slow_ar_to_dac_decoder_async_chunk"),
|
||||
sampling_constraints={
|
||||
"detokenize": False,
|
||||
# <|im_end|> — stop when the model emits end-of-turn.
|
||||
"stop_token_ids": [151645],
|
||||
},
|
||||
),
|
||||
StagePipelineConfig(
|
||||
stage_id=1,
|
||||
model_stage="dac_decoder",
|
||||
model_arch="FishSpeechDACDecoder",
|
||||
execution_type=StageExecutionType.LLM_GENERATION,
|
||||
input_sources=(0,),
|
||||
final_output=True,
|
||||
final_output_type="audio",
|
||||
engine_output_type="audio",
|
||||
sampling_constraints={"detokenize": True},
|
||||
),
|
||||
),
|
||||
)
|
||||
@@ -0,0 +1,54 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""MiMo Audio pipeline topology (frozen).
|
||||
|
||||
Stage 0: fused thinker+talker — multimodal understanding + text + RVQ codes.
|
||||
Stage 1: Code2Wav — RVQ codes → waveform.
|
||||
|
||||
MiMoAudioConfig inherits from Qwen2Config, so the HF ``model_type`` field
|
||||
reports ``qwen2`` — the registry's model_type-based auto-detect can't
|
||||
disambiguate. ``hf_architectures`` lets ``StageConfigFactory`` fall back to
|
||||
matching ``hf_config.architectures`` instead.
|
||||
"""
|
||||
|
||||
from vllm_omni.config.stage_config import (
|
||||
PipelineConfig,
|
||||
StageExecutionType,
|
||||
StagePipelineConfig,
|
||||
)
|
||||
|
||||
_PROC = "vllm_omni.model_executor.stage_input_processors.mimo_audio"
|
||||
|
||||
MIMO_AUDIO_PIPELINE = PipelineConfig(
|
||||
model_type="mimo_audio",
|
||||
# HF ``architectures: ["MiMoAudioModel"]`` is also the registry key in
|
||||
# ``model_executor/models/registry.py``; it resolves to the internal
|
||||
# class ``MiMoAudioForConditionalGeneration`` in ``mimo_audio.py``.
|
||||
model_arch="MiMoAudioModel",
|
||||
hf_architectures=("MiMoAudioModel",),
|
||||
stages=(
|
||||
StagePipelineConfig(
|
||||
stage_id=0,
|
||||
model_stage="fused_thinker_talker",
|
||||
execution_type=StageExecutionType.LLM_AR,
|
||||
input_sources=(),
|
||||
final_output=True,
|
||||
final_output_type="text",
|
||||
owns_tokenizer=True,
|
||||
engine_output_type="latent",
|
||||
async_chunk_process_next_stage_input_func=(f"{_PROC}.llm2code2wav_async_chunk"),
|
||||
sampling_constraints={"detokenize": True},
|
||||
),
|
||||
StagePipelineConfig(
|
||||
stage_id=1,
|
||||
model_stage="code2wav",
|
||||
execution_type=StageExecutionType.LLM_GENERATION,
|
||||
input_sources=(0,),
|
||||
final_output=True,
|
||||
final_output_type="audio",
|
||||
engine_output_type="audio",
|
||||
sync_process_input_func=f"{_PROC}.llm2code2wav",
|
||||
sampling_constraints={"detokenize": False},
|
||||
),
|
||||
),
|
||||
)
|
||||
@@ -0,0 +1,35 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""VoxCPM2 pipeline topology (frozen).
|
||||
|
||||
Single-stage AR TTS: text → speech waveform in one pass.
|
||||
Uses the native MiniCPM4 base_lm with a per-request StaticKVCache that the
|
||||
talker restores into the paged attention layer at step boundaries.
|
||||
"""
|
||||
|
||||
from vllm_omni.config.stage_config import (
|
||||
PipelineConfig,
|
||||
StageExecutionType,
|
||||
StagePipelineConfig,
|
||||
)
|
||||
|
||||
VOXCPM2_PIPELINE = PipelineConfig(
|
||||
model_type="voxcpm2",
|
||||
model_arch="VoxCPM2TalkerForConditionalGeneration",
|
||||
stages=(
|
||||
StagePipelineConfig(
|
||||
stage_id=0,
|
||||
model_stage="latent_generator",
|
||||
execution_type=StageExecutionType.LLM_AR,
|
||||
input_sources=(),
|
||||
final_output=True,
|
||||
final_output_type="audio",
|
||||
owns_tokenizer=True,
|
||||
engine_output_type="audio",
|
||||
sampling_constraints={
|
||||
"detokenize": False,
|
||||
"stop_token_ids": [1],
|
||||
},
|
||||
),
|
||||
),
|
||||
)
|
||||
Whitespace-only changes.
@@ -0,0 +1,45 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Voxtral TTS pipeline topology (frozen).
|
||||
|
||||
Stage 0: audio_generation — text → acoustic latents (LLM_AR, tokenizer owner).
|
||||
Stage 1: audio_tokenizer — acoustic latents → waveform (LLM_GENERATION).
|
||||
"""
|
||||
|
||||
from vllm_omni.config.stage_config import (
|
||||
PipelineConfig,
|
||||
StageExecutionType,
|
||||
StagePipelineConfig,
|
||||
)
|
||||
|
||||
_PROC = "vllm_omni.model_executor.stage_input_processors.voxtral_tts"
|
||||
|
||||
VOXTRAL_TTS_PIPELINE = PipelineConfig(
|
||||
model_type="voxtral_tts",
|
||||
model_arch="VoxtralTTSForConditionalGeneration",
|
||||
stages=(
|
||||
StagePipelineConfig(
|
||||
stage_id=0,
|
||||
model_stage="audio_generation",
|
||||
execution_type=StageExecutionType.LLM_AR,
|
||||
input_sources=(),
|
||||
final_output=False,
|
||||
final_output_type="text",
|
||||
owns_tokenizer=True,
|
||||
engine_output_type="latent",
|
||||
async_chunk_process_next_stage_input_func=(f"{_PROC}.generator2tokenizer_async_chunk"),
|
||||
sampling_constraints={"detokenize": True},
|
||||
),
|
||||
StagePipelineConfig(
|
||||
stage_id=1,
|
||||
model_stage="audio_tokenizer",
|
||||
execution_type=StageExecutionType.LLM_GENERATION,
|
||||
input_sources=(0,),
|
||||
final_output=True,
|
||||
final_output_type="audio",
|
||||
engine_output_type="audio",
|
||||
sampling_constraints={"detokenize": True},
|
||||
extras={"tts_args": {"max_instructions_length": 500}},
|
||||
),
|
||||
),
|
||||
)
|
||||
Reference in new issue
Block a user