[Config Refactor] Migrate Ming and Ming-TTS deploy/pipline configs (#3154)

Signed-off-by: Yuanheng Zhao <jonathan.zhaoyh@gmail.com>
Signed-off-by: yuanheng <jonathan.zhaoyh@gmail.com>
Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
This commit is contained in:
Yuanheng ZhaoandHongsheng Liu authored and GitHub committed 2026-05-03 14:43:49 +12:00
1 parent 6a837849dd
commit d49356e4ca
25 files changed
+518 -398

No files matched your search

@@ -104,8 +104,8 @@ class MingFlashOmniForConditionalGeneration(
"model_arch='MingFlashOmniTalkerForConditionalGeneration' "
"directly — the talker has a self-contained LLM that "
"tokenises text itself and does not need the multimodal "
"preprocess path. See stage_configs/ming_flash_omni.yaml "
"stage 1 and stage_configs/ming_flash_omni_tts.yaml."
"preprocess path. See vllm_omni/deploy/ming_flash_omni.yaml "
"stage 1 and vllm_omni/deploy/ming_flash_omni_tts.yaml."
)
if self.model_stage == "thinker":
@@ -0,0 +1,106 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Ming-flash-omni-2.0 pipeline topology (frozen).
Stage 0: Thinker — multimodal understanding + text generation.
Stage 1: Talker — text -> audio waveform via CFM + AudioVAE.
The thinker -> talker bridge passes the detokenized text rather than
hidden states through `ming_flash_omni.thinker2talker`; the talker has a
self-contained Qwen2 LLM that retokenizes the string itself.
"""
from vllm_omni.config.stage_config import (
PipelineConfig,
StageExecutionType,
StagePipelineConfig,
)
_PROC = "vllm_omni.model_executor.stage_input_processors.ming_flash_omni"
MING_FLASH_OMNI_PIPELINE = PipelineConfig(
model_type="ming_flash_omni",
model_arch="MingFlashOmniForConditionalGeneration",
# Upstream HF config applies model_type="bailingmm_moe_v2_lite"
# (the thinker sub-config name) rather than "ming_flash_omni".
# Declare the architectures here explicitly for routing
hf_architectures=(
"MingFlashOmniForConditionalGeneration",
"BailingMM2NativeForConditionalGeneration",
),
stages=(
StagePipelineConfig(
stage_id=0,
model_stage="thinker",
execution_type=StageExecutionType.LLM_AR,
input_sources=(),
final_output=True,
final_output_type="text",
owns_tokenizer=True,
requires_multimodal_data=True,
# Thinker reads the LLM sub-config of BailingMM2Config
hf_config_name="llm_config",
engine_output_type="text",
sampling_constraints={"detokenize": True},
),
StagePipelineConfig(
stage_id=1,
model_stage="ming_tts",
# Talker is a self-contained Qwen2 LLM + CFM + AudioVAE;
# it does not share the thinker backbone, so it gets its own arch.
model_arch="MingFlashOmniTalkerForConditionalGeneration",
execution_type=StageExecutionType.LLM_GENERATION,
input_sources=(0,),
final_output=True,
final_output_type="audio",
hf_config_name="talker_config",
engine_output_type="audio",
tokenizer_subdir="talker/llm",
custom_process_input_func=f"{_PROC}.thinker2talker",
),
),
)
# Standalone TTS variant: talker only.
MING_FLASH_OMNI_TTS_PIPELINE = PipelineConfig(
model_type="ming_flash_omni_tts",
model_arch="MingFlashOmniTalkerForConditionalGeneration",
stages=(
StagePipelineConfig(
stage_id=0,
model_stage="ming_tts",
execution_type=StageExecutionType.LLM_GENERATION,
input_sources=(),
final_output=True,
final_output_type="audio",
owns_tokenizer=True,
hf_config_name="talker_config",
engine_output_type="audio",
tokenizer_subdir="talker/llm",
),
),
)
# Thinker-only variant: multimodal understanding with text output
MING_FLASH_OMNI_THINKER_ONLY_PIPELINE = PipelineConfig(
model_type="ming_flash_omni_thinker_only",
model_arch="MingFlashOmniForConditionalGeneration",
stages=(
StagePipelineConfig(
stage_id=0,
model_stage="thinker",
execution_type=StageExecutionType.LLM_AR,
input_sources=(),
final_output=True,
final_output_type="text",
owns_tokenizer=True,
requires_multimodal_data=True,
hf_config_name="llm_config",
engine_output_type="text",
sampling_constraints={"detokenize": True},
),
),
)
@@ -963,7 +963,7 @@ class MingAudioGenerator:
)
if step > min_new_token and stop_prob > 0.5:
logger.info("Stopping at step %d with stop_prob=%.4f", step, stop_prob)
logger.debug("Stopping at step %d with stop_prob=%.4f", step, stop_prob)
break
return all_latents