[Config Refactor] Migrate 5 TTS models (VoxCPM2 / CosyVoice3 / MiMo Audio / Voxtral TTS / Fish Speech S2 Pro) to Pipeline + Deploy schema (#2958)
Signed-off-by: Yueqian Lin <linyueqian@outlook.com> Co-authored-by: SYLAR <125541396+lishunyang12@users.noreply.github.com>
This commit is contained in:
1 parent
c2578f7131
commit
3f504b45d2
47 files changed
+702
-844
No files matched your search
@@ -0,0 +1,50 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""CosyVoice3 pipeline topology (frozen).
|
||||
|
||||
Stage 0: Talker — text prompt → speech tokens (LLM autoregressive).
|
||||
Stage 1: Code2Wav — flow-matching decoder → acoustic features → waveform.
|
||||
* ``sync_process_input_func`` runs when ``deploy.async_chunk=false``:
|
||||
stage 1 builds full-sequence flow input via ``text2flow``.
|
||||
* ``async_chunk_process_next_stage_input_func`` runs when
|
||||
``deploy.async_chunk=true``: stage 0 streams codec chunks to stage 1
|
||||
through the shared-memory connector.
|
||||
"""
|
||||
|
||||
from vllm_omni.config.stage_config import (
|
||||
PipelineConfig,
|
||||
StageExecutionType,
|
||||
StagePipelineConfig,
|
||||
)
|
||||
|
||||
_PROC = "vllm_omni.model_executor.stage_input_processors.cosyvoice3"
|
||||
|
||||
COSYVOICE3_PIPELINE = PipelineConfig(
|
||||
model_type="cosyvoice3",
|
||||
model_arch="CosyVoice3Model",
|
||||
stages=(
|
||||
StagePipelineConfig(
|
||||
stage_id=0,
|
||||
model_stage="cosyvoice3_talker",
|
||||
execution_type=StageExecutionType.LLM_AR,
|
||||
input_sources=(),
|
||||
owns_tokenizer=True,
|
||||
engine_output_type="latent",
|
||||
async_chunk_process_next_stage_input_func=(f"{_PROC}.talker2code2wav_async_chunk"),
|
||||
sampling_constraints={
|
||||
# merged speech stop token (logsumexp of all 200 stop logits)
|
||||
"stop_token_ids": [6562],
|
||||
},
|
||||
),
|
||||
StagePipelineConfig(
|
||||
stage_id=1,
|
||||
model_stage="cosyvoice3_code2wav",
|
||||
execution_type=StageExecutionType.LLM_GENERATION,
|
||||
input_sources=(0,),
|
||||
final_output=True,
|
||||
final_output_type="audio",
|
||||
engine_output_type="latent",
|
||||
sync_process_input_func=f"{_PROC}.text2flow",
|
||||
),
|
||||
),
|
||||
)
|
||||
@@ -0,0 +1,51 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Fish Speech S2 Pro pipeline topology (frozen).
|
||||
|
||||
Stage 0: slow_ar — text → RVQ codec tokens (LLM autoregressive).
|
||||
Stage 1: dac_decoder — RVQ tokens → audio waveform (LLM_GENERATION).
|
||||
|
||||
The HF config top-level reports ``model_type = "fish_qwen3_omni"`` (the
|
||||
OmniConfig that bundles slow-AR and fast-AR sub-configs), which is why the
|
||||
registry key follows the HF name rather than the human-readable "fish_speech".
|
||||
"""
|
||||
|
||||
from vllm_omni.config.stage_config import (
|
||||
PipelineConfig,
|
||||
StageExecutionType,
|
||||
StagePipelineConfig,
|
||||
)
|
||||
|
||||
_PROC = "vllm_omni.model_executor.stage_input_processors.fish_speech"
|
||||
|
||||
FISH_SPEECH_PIPELINE = PipelineConfig(
|
||||
model_type="fish_qwen3_omni",
|
||||
model_arch="FishSpeechSlowARForConditionalGeneration",
|
||||
stages=(
|
||||
StagePipelineConfig(
|
||||
stage_id=0,
|
||||
model_stage="fish_speech_slow_ar",
|
||||
execution_type=StageExecutionType.LLM_AR,
|
||||
input_sources=(),
|
||||
owns_tokenizer=True,
|
||||
engine_output_type="latent",
|
||||
async_chunk_process_next_stage_input_func=(f"{_PROC}.slow_ar_to_dac_decoder_async_chunk"),
|
||||
sampling_constraints={
|
||||
"detokenize": False,
|
||||
# <|im_end|> — stop when the model emits end-of-turn.
|
||||
"stop_token_ids": [151645],
|
||||
},
|
||||
),
|
||||
StagePipelineConfig(
|
||||
stage_id=1,
|
||||
model_stage="dac_decoder",
|
||||
model_arch="FishSpeechDACDecoder",
|
||||
execution_type=StageExecutionType.LLM_GENERATION,
|
||||
input_sources=(0,),
|
||||
final_output=True,
|
||||
final_output_type="audio",
|
||||
engine_output_type="audio",
|
||||
sampling_constraints={"detokenize": True},
|
||||
),
|
||||
),
|
||||
)
|
||||
@@ -0,0 +1,54 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""MiMo Audio pipeline topology (frozen).
|
||||
|
||||
Stage 0: fused thinker+talker — multimodal understanding + text + RVQ codes.
|
||||
Stage 1: Code2Wav — RVQ codes → waveform.
|
||||
|
||||
MiMoAudioConfig inherits from Qwen2Config, so the HF ``model_type`` field
|
||||
reports ``qwen2`` — the registry's model_type-based auto-detect can't
|
||||
disambiguate. ``hf_architectures`` lets ``StageConfigFactory`` fall back to
|
||||
matching ``hf_config.architectures`` instead.
|
||||
"""
|
||||
|
||||
from vllm_omni.config.stage_config import (
|
||||
PipelineConfig,
|
||||
StageExecutionType,
|
||||
StagePipelineConfig,
|
||||
)
|
||||
|
||||
_PROC = "vllm_omni.model_executor.stage_input_processors.mimo_audio"
|
||||
|
||||
MIMO_AUDIO_PIPELINE = PipelineConfig(
|
||||
model_type="mimo_audio",
|
||||
# HF ``architectures: ["MiMoAudioModel"]`` is also the registry key in
|
||||
# ``model_executor/models/registry.py``; it resolves to the internal
|
||||
# class ``MiMoAudioForConditionalGeneration`` in ``mimo_audio.py``.
|
||||
model_arch="MiMoAudioModel",
|
||||
hf_architectures=("MiMoAudioModel",),
|
||||
stages=(
|
||||
StagePipelineConfig(
|
||||
stage_id=0,
|
||||
model_stage="fused_thinker_talker",
|
||||
execution_type=StageExecutionType.LLM_AR,
|
||||
input_sources=(),
|
||||
final_output=True,
|
||||
final_output_type="text",
|
||||
owns_tokenizer=True,
|
||||
engine_output_type="latent",
|
||||
async_chunk_process_next_stage_input_func=(f"{_PROC}.llm2code2wav_async_chunk"),
|
||||
sampling_constraints={"detokenize": True},
|
||||
),
|
||||
StagePipelineConfig(
|
||||
stage_id=1,
|
||||
model_stage="code2wav",
|
||||
execution_type=StageExecutionType.LLM_GENERATION,
|
||||
input_sources=(0,),
|
||||
final_output=True,
|
||||
final_output_type="audio",
|
||||
engine_output_type="audio",
|
||||
sync_process_input_func=f"{_PROC}.llm2code2wav",
|
||||
sampling_constraints={"detokenize": False},
|
||||
),
|
||||
),
|
||||
)
|
||||
@@ -0,0 +1,35 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""VoxCPM2 pipeline topology (frozen).
|
||||
|
||||
Single-stage AR TTS: text → speech waveform in one pass.
|
||||
Uses the native MiniCPM4 base_lm with a per-request StaticKVCache that the
|
||||
talker restores into the paged attention layer at step boundaries.
|
||||
"""
|
||||
|
||||
from vllm_omni.config.stage_config import (
|
||||
PipelineConfig,
|
||||
StageExecutionType,
|
||||
StagePipelineConfig,
|
||||
)
|
||||
|
||||
VOXCPM2_PIPELINE = PipelineConfig(
|
||||
model_type="voxcpm2",
|
||||
model_arch="VoxCPM2TalkerForConditionalGeneration",
|
||||
stages=(
|
||||
StagePipelineConfig(
|
||||
stage_id=0,
|
||||
model_stage="latent_generator",
|
||||
execution_type=StageExecutionType.LLM_AR,
|
||||
input_sources=(),
|
||||
final_output=True,
|
||||
final_output_type="audio",
|
||||
owns_tokenizer=True,
|
||||
engine_output_type="audio",
|
||||
sampling_constraints={
|
||||
"detokenize": False,
|
||||
"stop_token_ids": [1],
|
||||
},
|
||||
),
|
||||
),
|
||||
)
|
||||
Whitespace-only changes.
@@ -0,0 +1,45 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Voxtral TTS pipeline topology (frozen).
|
||||
|
||||
Stage 0: audio_generation — text → acoustic latents (LLM_AR, tokenizer owner).
|
||||
Stage 1: audio_tokenizer — acoustic latents → waveform (LLM_GENERATION).
|
||||
"""
|
||||
|
||||
from vllm_omni.config.stage_config import (
|
||||
PipelineConfig,
|
||||
StageExecutionType,
|
||||
StagePipelineConfig,
|
||||
)
|
||||
|
||||
_PROC = "vllm_omni.model_executor.stage_input_processors.voxtral_tts"
|
||||
|
||||
VOXTRAL_TTS_PIPELINE = PipelineConfig(
|
||||
model_type="voxtral_tts",
|
||||
model_arch="VoxtralTTSForConditionalGeneration",
|
||||
stages=(
|
||||
StagePipelineConfig(
|
||||
stage_id=0,
|
||||
model_stage="audio_generation",
|
||||
execution_type=StageExecutionType.LLM_AR,
|
||||
input_sources=(),
|
||||
final_output=False,
|
||||
final_output_type="text",
|
||||
owns_tokenizer=True,
|
||||
engine_output_type="latent",
|
||||
async_chunk_process_next_stage_input_func=(f"{_PROC}.generator2tokenizer_async_chunk"),
|
||||
sampling_constraints={"detokenize": True},
|
||||
),
|
||||
StagePipelineConfig(
|
||||
stage_id=1,
|
||||
model_stage="audio_tokenizer",
|
||||
execution_type=StageExecutionType.LLM_GENERATION,
|
||||
input_sources=(0,),
|
||||
final_output=True,
|
||||
final_output_type="audio",
|
||||
engine_output_type="audio",
|
||||
sampling_constraints={"detokenize": True},
|
||||
extras={"tts_args": {"max_instructions_length": 500}},
|
||||
),
|
||||
),
|
||||
)
|
||||
@@ -1,53 +0,0 @@
|
||||
# Stage config for running CosyVoice3 with 2 stage architecture
|
||||
# Stage 0: Talker (text prompt → speech tokens)
|
||||
# Stage 1: Code2Wav (flow matching → acoustic features → waveform)
|
||||
# Right now I have coded up stage 2 in stage 1 only; can be split in future
|
||||
|
||||
stage_args:
|
||||
- stage_id: 0
|
||||
is_comprehension: true
|
||||
runtime:
|
||||
devices: 0
|
||||
engine_args:
|
||||
model_stage: cosyvoice3_talker
|
||||
worker_cls: vllm_omni.worker.gpu_ar_worker.GPUARWorker
|
||||
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
|
||||
model_arch: CosyVoice3Model
|
||||
trust_remote_code: true
|
||||
gpu_memory_utilization: 0.2
|
||||
engine_output_type: latent # Output speech tokens for chunk aware flow matching
|
||||
disable_hybrid_kv_cache_manager: true
|
||||
enable_prefix_caching: false
|
||||
enforce_eager: true
|
||||
mm_processor_cache_gb: 0
|
||||
skip_mm_profiling: true
|
||||
dtype: "float32"
|
||||
default_sampling_params:
|
||||
max_tokens: 2048
|
||||
stop_token_ids: [6562] # merged speech stop token (logsumexp of all 200 stop logits)
|
||||
top_k: 25
|
||||
top_p: 0.8
|
||||
repetition_penalty: 1.0001 # near-identity; forces vLLM to track output_token_ids for RAS
|
||||
|
||||
- stage_id: 1
|
||||
runtime:
|
||||
devices: 0
|
||||
engine_args:
|
||||
model_stage: cosyvoice3_code2wav
|
||||
model_arch: CosyVoice3Model
|
||||
trust_remote_code: true
|
||||
worker_cls: vllm_omni.worker.gpu_generation_worker.GPUGenerationWorker
|
||||
scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler
|
||||
engine_output_type: latent
|
||||
gpu_memory_utilization: 0.1
|
||||
enforce_eager: true # CUDA graphs don't work with dynamic conv shapes in code2wav
|
||||
disable_hybrid_kv_cache_manager: true
|
||||
enable_prefix_caching: false
|
||||
skip_mm_profiling: true
|
||||
dtype: "float32"
|
||||
default_sampling_params:
|
||||
max_tokens: 2048
|
||||
engine_input_source: [0]
|
||||
custom_process_input_func: vllm_omni.model_executor.stage_input_processors.cosyvoice3.text2flow
|
||||
final_output: true
|
||||
final_output_type: audio
|
||||
@@ -1,81 +0,0 @@
|
||||
# Stage config for running CosyVoice3 with async_chunk architecture
|
||||
# Stage 0: Talker (text prompt -> speech tokens streamed by chunks)
|
||||
# Stage 1: Code2Wav (flow matching -> acoustic features -> waveform)
|
||||
async_chunk: true
|
||||
|
||||
stage_args:
|
||||
- stage_id: 0
|
||||
is_comprehension: true
|
||||
runtime:
|
||||
devices: 0
|
||||
max_batch_size: 1
|
||||
engine_args:
|
||||
model_stage: cosyvoice3_talker
|
||||
model_arch: CosyVoice3Model
|
||||
worker_type: ar
|
||||
worker_cls: vllm_omni.worker.gpu_ar_worker.GPUARWorker
|
||||
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
|
||||
custom_process_next_stage_input_func: vllm_omni.model_executor.stage_input_processors.cosyvoice3.talker2code2wav_async_chunk
|
||||
trust_remote_code: true
|
||||
gpu_memory_utilization: 0.4
|
||||
engine_output_type: latent
|
||||
disable_hybrid_kv_cache_manager: true
|
||||
enable_prefix_caching: false
|
||||
enforce_eager: true
|
||||
mm_processor_cache_gb: 0
|
||||
skip_mm_profiling: true
|
||||
dtype: "float32"
|
||||
default_sampling_params:
|
||||
max_tokens: 2048
|
||||
stop_token_ids: [6562] # merged speech stop token (logsumexp of all 200 stop logits)
|
||||
top_k: 25
|
||||
top_p: 0.8
|
||||
repetition_penalty: 1.0001 # near-identity; forces vLLM to track output_token_ids for RAS
|
||||
output_connectors:
|
||||
to_stage_1: connector_of_shared_memory
|
||||
|
||||
- stage_id: 1
|
||||
runtime:
|
||||
devices: 0
|
||||
max_batch_size: 1
|
||||
engine_args:
|
||||
model_stage: cosyvoice3_code2wav
|
||||
model_arch: CosyVoice3Model
|
||||
worker_type: generation
|
||||
worker_cls: vllm_omni.worker.gpu_generation_worker.GPUGenerationWorker
|
||||
scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler
|
||||
trust_remote_code: true
|
||||
engine_output_type: latent
|
||||
gpu_memory_utilization: 0.2
|
||||
enforce_eager: true
|
||||
disable_hybrid_kv_cache_manager: true
|
||||
enable_prefix_caching: false
|
||||
skip_mm_profiling: true
|
||||
max_model_len: 32768
|
||||
dtype: "float32"
|
||||
default_sampling_params:
|
||||
max_tokens: 2048
|
||||
engine_input_source: [0]
|
||||
final_output: true
|
||||
final_output_type: audio
|
||||
input_connectors:
|
||||
from_stage_0: connector_of_shared_memory
|
||||
|
||||
runtime:
|
||||
enabled: true
|
||||
|
||||
connectors:
|
||||
connector_of_shared_memory:
|
||||
name: SharedMemoryConnector
|
||||
extra:
|
||||
codec_streaming: true
|
||||
connector_get_sleep_s: 0.01
|
||||
connector_get_max_wait_first_chunk: 3000
|
||||
connector_get_max_wait: 300
|
||||
codec_chunk_frames: 25
|
||||
codec_left_context_frames: 25
|
||||
codec_vocab_size: 6561
|
||||
|
||||
edges:
|
||||
- from: 0
|
||||
to: 1
|
||||
@@ -1,91 +0,0 @@
|
||||
async_chunk: true
|
||||
stage_args:
|
||||
- stage_id: 0
|
||||
stage_type: llm
|
||||
is_comprehension: true
|
||||
runtime:
|
||||
devices: "0"
|
||||
max_batch_size: 16
|
||||
engine_args:
|
||||
max_num_seqs: 4
|
||||
model_stage: fish_speech_slow_ar
|
||||
model_arch: FishSpeechSlowARForConditionalGeneration
|
||||
worker_type: ar
|
||||
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
|
||||
enforce_eager: false
|
||||
trust_remote_code: true
|
||||
async_scheduling: false
|
||||
enable_prefix_caching: false
|
||||
engine_output_type: latent
|
||||
gpu_memory_utilization: 0.6
|
||||
distributed_executor_backend: "mp"
|
||||
max_num_batched_tokens: 3072
|
||||
max_model_len: 16384
|
||||
custom_process_next_stage_input_func: vllm_omni.model_executor.stage_input_processors.fish_speech.slow_ar_to_dac_decoder_async_chunk
|
||||
output_connectors:
|
||||
to_stage_1: connector_of_shared_memory
|
||||
default_sampling_params:
|
||||
temperature: 0.8
|
||||
top_k: 30
|
||||
top_p: 0.9
|
||||
max_tokens: 2048
|
||||
seed: 42
|
||||
detokenize: false
|
||||
repetition_penalty: 1.0
|
||||
# <|im_end|> token -- stop when model emits end-of-turn.
|
||||
stop_token_ids: [151645]
|
||||
|
||||
- stage_id: 1
|
||||
stage_type: llm
|
||||
runtime:
|
||||
devices: "0"
|
||||
max_batch_size: 16
|
||||
engine_args:
|
||||
max_num_seqs: 1
|
||||
model_stage: dac_decoder
|
||||
model_arch: FishSpeechDACDecoder
|
||||
worker_type: generation
|
||||
scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler
|
||||
enforce_eager: true
|
||||
trust_remote_code: true
|
||||
async_scheduling: false
|
||||
enable_prefix_caching: false
|
||||
engine_output_type: audio
|
||||
gpu_memory_utilization: 0.1
|
||||
distributed_executor_backend: "mp"
|
||||
max_num_batched_tokens: 8192
|
||||
max_model_len: 16384
|
||||
engine_input_source: [0]
|
||||
final_output: true
|
||||
final_output_type: audio
|
||||
input_connectors:
|
||||
from_stage_0: connector_of_shared_memory
|
||||
default_sampling_params:
|
||||
temperature: 0.0
|
||||
top_p: 1.0
|
||||
top_k: -1
|
||||
max_tokens: 65536
|
||||
seed: 42
|
||||
detokenize: true
|
||||
repetition_penalty: 1.0
|
||||
|
||||
runtime:
|
||||
enabled: true
|
||||
connectors:
|
||||
connector_of_shared_memory:
|
||||
name: SharedMemoryConnector
|
||||
extra:
|
||||
shm_threshold_bytes: 65536
|
||||
codec_streaming: true
|
||||
connector_get_sleep_s: 0.01
|
||||
connector_get_max_wait_first_chunk: 3000
|
||||
connector_get_max_wait: 300
|
||||
# Chunk sizes for streaming -- ~21 Hz codec.
|
||||
# 25 frames ≈ 1.16s of audio at 21.5 Hz.
|
||||
codec_chunk_frames: 25
|
||||
codec_left_context_frames: 25
|
||||
initial_codec_chunk_frames: 4
|
||||
|
||||
edges:
|
||||
- from: 0
|
||||
to: 1
|
||||
@@ -1,69 +0,0 @@
|
||||
# stage config for running mimo-audio for multi-stage omni runtime.
|
||||
|
||||
# The following config has been verified on 1x H20-96G GPU.
|
||||
async_chunk: false
|
||||
stage_args:
|
||||
- stage_id: 0
|
||||
stage_type: llm
|
||||
runtime:
|
||||
process: true # Run this stage in a separate process
|
||||
devices: "0" # Visible devices for this stage (CUDA_VISIBLE_DEVICES/torch.cuda.set_device)
|
||||
engine_args:
|
||||
dtype: bfloat16
|
||||
max_num_seqs: 1
|
||||
model_stage: fused_thinker_talker
|
||||
model_arch: MiMoAudioForConditionalGeneration
|
||||
worker_type: ar
|
||||
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
|
||||
tensor_parallel_size: 1 # Change to desired TP size for multi-GPU inference (e.g., 4 for 4 GPUs)
|
||||
gpu_memory_utilization: 0.3
|
||||
enforce_eager: true # need to discuss
|
||||
trust_remote_code: true
|
||||
enable_prefix_caching: false
|
||||
engine_output_type: latent # change the param name,such as pooling_output
|
||||
max_model_len: 8192
|
||||
max_num_batched_tokens: 8192
|
||||
is_comprehension: true
|
||||
final_output: true
|
||||
final_output_type: text
|
||||
default_sampling_params:
|
||||
temperature: 0.6
|
||||
top_p: 0.95
|
||||
top_k: 50
|
||||
max_tokens: 18192
|
||||
seed: 42
|
||||
detokenize: True
|
||||
repetition_penalty: 1.1
|
||||
|
||||
- stage_id: 1
|
||||
stage_type: llm
|
||||
runtime:
|
||||
process: true # Run this stage in a separate process
|
||||
devices: "1" # Visible devices for this stage (CUDA_VISIBLE_DEVICES/torch.cuda.set_device)
|
||||
engine_args:
|
||||
model_stage: code2wav
|
||||
max_num_seqs: 1
|
||||
model_arch: MiMoAudioForConditionalGeneration
|
||||
worker_type: generation
|
||||
scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler
|
||||
tensor_parallel_size: 1 # Change to desired TP size for multi-GPU inference (e.g., 4 for 4 GPUs)
|
||||
gpu_memory_utilization: 0.2
|
||||
enforce_eager: true
|
||||
trust_remote_code: true
|
||||
enable_prefix_caching: false
|
||||
engine_output_type: audio
|
||||
max_model_len: 18192
|
||||
max_num_batched_tokens: 18192
|
||||
async_scheduling: false
|
||||
engine_input_source: [ 0 ]
|
||||
is_comprehension: false
|
||||
custom_process_input_func: vllm_omni.model_executor.stage_input_processors.mimo_audio.llm2code2wav
|
||||
final_output: true
|
||||
final_output_type: audio
|
||||
default_sampling_params:
|
||||
temperature: 0.0
|
||||
top_p: 1.0
|
||||
top_k: -1
|
||||
max_tokens: 18192
|
||||
seed: 42
|
||||
detokenize: false
|
||||
@@ -1,91 +0,0 @@
|
||||
# stage config for running mimo-audio for multi-stage omni runtime.
|
||||
|
||||
# The following config has been verified on 1x H20-96G GPU.
|
||||
async_chunk: true
|
||||
stage_args:
|
||||
- stage_id: 0
|
||||
stage_type: llm
|
||||
runtime:
|
||||
process: true # Run this stage in a separate process
|
||||
devices: "0" # Visible devices for this stage (CUDA_VISIBLE_DEVICES/torch.cuda.set_device)
|
||||
engine_args:
|
||||
dtype: bfloat16
|
||||
max_num_seqs: 1
|
||||
model_stage: fused_thinker_talker
|
||||
model_arch: MiMoAudioForConditionalGeneration
|
||||
worker_type: ar
|
||||
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
|
||||
tensor_parallel_size: 1 # Change to desired TP size for multi-GPU inference (e.g., 4 for 4 GPUs)
|
||||
gpu_memory_utilization: 0.3
|
||||
enforce_eager: true # need to discuss
|
||||
trust_remote_code: true
|
||||
enable_prefix_caching: false
|
||||
engine_output_type: latent # change the param name,such as pooling_output
|
||||
max_model_len: 8192
|
||||
max_num_batched_tokens: 8192
|
||||
custom_process_next_stage_input_func: vllm_omni.model_executor.stage_input_processors.mimo_audio.llm2code2wav_async_chunk
|
||||
output_connectors:
|
||||
to_stage_1: connector_of_shared_memory
|
||||
is_comprehension: true
|
||||
final_output: true
|
||||
final_output_type: text
|
||||
default_sampling_params:
|
||||
temperature: 0.6
|
||||
top_p: 0.95
|
||||
top_k: 50
|
||||
max_tokens: 18192
|
||||
seed: 42
|
||||
detokenize: True
|
||||
repetition_penalty: 1.1
|
||||
|
||||
- stage_id: 1
|
||||
stage_type: llm
|
||||
runtime:
|
||||
process: true # Run this stage in a separate process
|
||||
devices: "0" # Visible devices for this stage (CUDA_VISIBLE_DEVICES/torch.cuda.set_device)
|
||||
engine_args:
|
||||
model_stage: code2wav
|
||||
max_num_seqs: 1
|
||||
model_arch: MiMoAudioForConditionalGeneration
|
||||
worker_type: generation
|
||||
scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler
|
||||
tensor_parallel_size: 1 # Change to desired TP size for multi-GPU inference (e.g., 4 for 4 GPUs)
|
||||
gpu_memory_utilization: 0.2
|
||||
enforce_eager: true
|
||||
trust_remote_code: true
|
||||
async_scheduling: false
|
||||
enable_prefix_caching: false
|
||||
engine_output_type: audio
|
||||
max_model_len: 8192
|
||||
max_num_batched_tokens: 8192
|
||||
engine_input_source: [ 0 ]
|
||||
input_connectors:
|
||||
from_stage_0: connector_of_shared_memory
|
||||
is_comprehension: false
|
||||
final_output: true
|
||||
final_output_type: audio
|
||||
default_sampling_params:
|
||||
temperature: 0.0
|
||||
top_p: 1.0
|
||||
top_k: -1
|
||||
max_tokens: 18192
|
||||
seed: 42
|
||||
detokenize: false
|
||||
|
||||
runtime:
|
||||
enabled: true
|
||||
connectors:
|
||||
connector_of_shared_memory:
|
||||
name: SharedMemoryConnector
|
||||
extra:
|
||||
shm_threshold_bytes: 365536
|
||||
codec_streaming: true
|
||||
connector_get_sleep_s: 0.001
|
||||
connector_get_max_wait_first_chunk: 3000
|
||||
connector_get_max_wait: 300
|
||||
codec_chunk_frames: 3
|
||||
codec_left_context_frames: 3
|
||||
|
||||
edges:
|
||||
- from: 0
|
||||
to: 1
|
||||
@@ -1,36 +0,0 @@
|
||||
# VoxCPM2 AR pipeline with per-request state batching.
|
||||
# Uses native MiniCPM4 base_lm + per-request StaticKVCache.
|
||||
# max_batch_size > 1 supported via KV cache save/restore.
|
||||
stage_args:
|
||||
- stage_id: 0
|
||||
stage_type: llm
|
||||
is_comprehension: true
|
||||
runtime:
|
||||
devices: "0"
|
||||
max_batch_size: 4
|
||||
engine_args:
|
||||
dtype: bfloat16
|
||||
model_stage: latent_generator
|
||||
model_arch: VoxCPM2TalkerForConditionalGeneration
|
||||
worker_type: ar
|
||||
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
|
||||
enforce_eager: true
|
||||
trust_remote_code: true
|
||||
async_scheduling: true
|
||||
enable_prefix_caching: false
|
||||
engine_output_type: audio
|
||||
gpu_memory_utilization: 0.9
|
||||
distributed_executor_backend: "mp"
|
||||
max_num_batched_tokens: 4096
|
||||
max_model_len: 4096
|
||||
default_sampling_params:
|
||||
temperature: 0.0
|
||||
top_p: 1.0
|
||||
top_k: -1
|
||||
max_tokens: 4096
|
||||
seed: 42
|
||||
detokenize: false
|
||||
repetition_penalty: 1.0
|
||||
stop_token_ids: [1]
|
||||
final_output: true
|
||||
final_output_type: audio
|
||||
@@ -1,100 +0,0 @@
|
||||
async_chunk: true
|
||||
stage_args:
|
||||
- stage_id: 0
|
||||
stage_type: llm # Use llm stage type to launch OmniLLM
|
||||
runtime:
|
||||
process: true # Run this stage in a separate process
|
||||
devices: "0" # Visible devices for this stage (CUDA_VISIBLE_DEVICES/torch.cuda.set_device)
|
||||
engine_args:
|
||||
max_num_seqs: 32
|
||||
model_stage: audio_generation
|
||||
model_arch: VoxtralTTSForConditionalGeneration
|
||||
worker_type: ar
|
||||
worker_cls: vllm_omni.worker.gpu_ar_worker.GPUARWorker
|
||||
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
|
||||
gpu_memory_utilization: 0.8
|
||||
enforce_eager: false
|
||||
trust_remote_code: true
|
||||
async_scheduling: true
|
||||
engine_output_type: latent
|
||||
enable_prefix_caching: false
|
||||
tokenizer_mode: mistral
|
||||
config_format: mistral
|
||||
load_format: mistral
|
||||
skip_mm_profiling: true
|
||||
enable_chunked_prefill: false
|
||||
max_model_len: 4096
|
||||
custom_process_next_stage_input_func: vllm_omni.model_executor.stage_input_processors.voxtral_tts.generator2tokenizer_async_chunk
|
||||
output_connectors:
|
||||
to_stage_1: connector_of_shared_memory
|
||||
is_comprehension: true
|
||||
final_output: false
|
||||
final_output_type: text
|
||||
default_sampling_params:
|
||||
temperature: 0.0
|
||||
top_p: 1.0
|
||||
top_k: -1
|
||||
max_tokens: 2048
|
||||
seed: 42
|
||||
detokenize: True
|
||||
repetition_penalty: 1.1
|
||||
- stage_id: 1
|
||||
stage_type: llm # Use llm stage type to launch OmniLLM
|
||||
runtime:
|
||||
process: true
|
||||
devices: "0"
|
||||
engine_args:
|
||||
max_num_seqs: 32
|
||||
model_stage: audio_tokenizer
|
||||
model_arch: VoxtralTTSForConditionalGeneration
|
||||
worker_type: generation
|
||||
worker_cls: vllm_omni.worker.gpu_generation_worker.GPUGenerationWorker
|
||||
scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler
|
||||
async_scheduling: false
|
||||
gpu_memory_utilization: 0.1
|
||||
enforce_eager: true
|
||||
trust_remote_code: true
|
||||
enable_prefix_caching: false
|
||||
skip_mm_profiling: true
|
||||
engine_output_type: audio
|
||||
tokenizer_mode: mistral
|
||||
config_format: mistral
|
||||
load_format: mistral
|
||||
max_num_batched_tokens: 65536
|
||||
max_model_len: 65536
|
||||
engine_input_source: [0]
|
||||
is_comprehension: false
|
||||
final_output: true
|
||||
final_output_type: audio
|
||||
input_connectors:
|
||||
from_stage_0: connector_of_shared_memory
|
||||
tts_args:
|
||||
max_instructions_length: 500
|
||||
default_sampling_params:
|
||||
temperature: 0.9
|
||||
top_p: 0.8
|
||||
top_k: 40
|
||||
max_tokens: 2048
|
||||
seed: 42
|
||||
detokenize: True
|
||||
repetition_penalty: 1.05
|
||||
|
||||
# Top-level runtime config (concise): default windows and stage edges
|
||||
runtime:
|
||||
enabled: true
|
||||
connectors:
|
||||
connector_of_shared_memory:
|
||||
name: SharedMemoryConnector
|
||||
extra:
|
||||
shm_threshold_bytes: 65536
|
||||
codec_streaming: true
|
||||
connector_get_sleep_s: 0.01
|
||||
connector_get_max_wait_first_chunk: 3000
|
||||
connector_get_max_wait: 300
|
||||
codec_chunk_frames: 25
|
||||
codec_chunk_frames_at_begin: 5
|
||||
codec_left_context_frames: 25
|
||||
|
||||
edges:
|
||||
- from: 0 # language_model → acoustic_transformer: trigger only after receiving full input (-1)
|
||||
to: 1
|
||||
Reference in new issue
Block a user