[Model] Ming-flash-omni-2.0 Omni-Speech and TTS (#2890)
Signed-off-by: Yuanheng Zhao <jonathan.zhaoyh@gmail.com> Signed-off-by: yuanheng <jonathan.zhaoyh@gmail.com> Signed-off-by: LHXuuu <xulianhao.xlh@antgroup.com> Co-authored-by: LHXuuu <xulianhao.xlh@antgroup.com> Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
This commit is contained in:
32 files changed
+4620
-261
No files matched your search
@@ -36,6 +36,21 @@ def build_prompt(user_text: str) -> str:
|
||||
|
||||
|
||||
def get_eager_config():
|
||||
path = modify_stage_config(
|
||||
str(Path(__file__).parent.parent / "stage_configs" / "bailingmm_moe_v2_lite_thinker_only_ci.yaml"),
|
||||
updates={
|
||||
"stage_args": {
|
||||
0: {
|
||||
"engine_args.enforce_eager": "true",
|
||||
},
|
||||
},
|
||||
},
|
||||
)
|
||||
return path
|
||||
|
||||
|
||||
def get_eager_tts_config():
|
||||
"""Thinker+talker CI config with enforce_eager on the thinker stage."""
|
||||
path = modify_stage_config(
|
||||
str(Path(__file__).parent.parent / "stage_configs" / "bailingmm_moe_v2_lite_ci.yaml"),
|
||||
updates={
|
||||
@@ -49,9 +64,14 @@ def get_eager_config():
|
||||
return path
|
||||
|
||||
|
||||
# Thinker-only config — used by text-output tests.
|
||||
stage_configs = [get_eager_config()]
|
||||
test_params = [(model, stage_config) for model in models for stage_config in stage_configs]
|
||||
|
||||
# Thinker+talker config — used by audio-output tests.
|
||||
stage_configs_tts = [get_eager_tts_config()]
|
||||
test_params_tts = [(model, stage_config) for model in models for stage_config in stage_configs_tts]
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.omni
|
||||
@@ -140,3 +160,36 @@ def test_mixed_to_text(omni_runner, omni_runner_handler) -> None:
|
||||
request_config = {"prompts": prompt, "images": image, "audios": audio, "modalities": ["text"]}
|
||||
|
||||
omni_runner_handler.send_request(request_config)
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "H100"}, num_cards=4)
|
||||
@pytest.mark.parametrize("omni_runner", test_params_tts, indirect=True)
|
||||
def test_text_to_audio(omni_runner, omni_runner_handler) -> None:
|
||||
"""
|
||||
Test text input with audio output via the thinker+talker pipeline.
|
||||
Input Modal: text
|
||||
Output Modal: audio
|
||||
"""
|
||||
prompt = build_prompt("请简单介绍一下北京。")
|
||||
request_config = {"prompts": prompt, "modalities": ["audio"]}
|
||||
|
||||
omni_runner_handler.send_request(request_config)
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.omni
|
||||
@hardware_test(res={"cuda": "H100"}, num_cards=4)
|
||||
@pytest.mark.parametrize("omni_runner", test_params_tts, indirect=True)
|
||||
def test_image_to_audio(omni_runner, omni_runner_handler) -> None:
|
||||
"""
|
||||
Test image + text input with audio output via the thinker+talker pipeline.
|
||||
Input Modal: image + text
|
||||
Output Modal: audio
|
||||
"""
|
||||
image = generate_synthetic_image(224, 224)["np_array"]
|
||||
prompt = build_prompt(f"{IMAGE_TOKEN}Describe this image briefly.")
|
||||
request_config = {"prompts": prompt, "images": image, "modalities": ["audio"]}
|
||||
|
||||
omni_runner_handler.send_request(request_config)
|
||||
@@ -1,4 +1,8 @@
|
||||
# Thinker stage only
|
||||
# CI stage config for Ming-flash-omni-2.0 thinker+talker pipeline.
|
||||
# Stage 0: Thinker (multimodal understanding -> text generation)
|
||||
# Stage 1: Talker (text -> audio waveform via CFM + AudioVAE)
|
||||
#
|
||||
# The following config has been verified on 4x H100-80G GPUs
|
||||
stage_args:
|
||||
- stage_id: 0
|
||||
stage_type: llm
|
||||
@@ -10,10 +14,12 @@ stage_args:
|
||||
model_arch: MingFlashOmniForConditionalGeneration
|
||||
worker_type: ar
|
||||
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
|
||||
gpu_memory_utilization: 0.9
|
||||
gpu_memory_utilization: 0.74
|
||||
enforce_eager: false
|
||||
trust_remote_code: true
|
||||
engine_output_type: latent
|
||||
# Ming Thinker -> talker bridge reads detokenised text from
|
||||
# source_output.outputs[0].text (not hidden states).
|
||||
engine_output_type: text
|
||||
distributed_executor_backend: "mp"
|
||||
enable_prefix_caching: false
|
||||
max_num_batched_tokens: 32768
|
||||
@@ -22,14 +28,48 @@ stage_args:
|
||||
hf_config_name: llm_config
|
||||
load_format: dummy
|
||||
mm_processor_cache_gb: 0
|
||||
compilation_config:
|
||||
pass_config:
|
||||
# Disable fused all-reduce to avoid a vllm/flashinfer version mismatch.
|
||||
fuse_allreduce_rms: false
|
||||
final_output: true
|
||||
final_output_type: text
|
||||
is_comprehension: true
|
||||
default_sampling_params:
|
||||
temperature: 0.4
|
||||
temperature: 0.0
|
||||
top_p: 0.9
|
||||
max_tokens: 100
|
||||
repetition_penalty: 1.05
|
||||
seed: 42
|
||||
detokenize: true
|
||||
ignore_eos: false
|
||||
|
||||
- stage_id: 1
|
||||
stage_type: llm
|
||||
runtime:
|
||||
devices: "3"
|
||||
max_batch_size: 1
|
||||
engine_args:
|
||||
model_stage: ming_tts
|
||||
model_arch: MingFlashOmniTalkerForConditionalGeneration
|
||||
worker_cls: vllm_omni.worker.gpu_generation_worker.GPUGenerationWorker
|
||||
scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler
|
||||
gpu_memory_utilization: 0.18
|
||||
enforce_eager: true
|
||||
trust_remote_code: true
|
||||
engine_output_type: audio
|
||||
enable_prefix_caching: false
|
||||
max_num_batched_tokens: 1000000
|
||||
tokenizer_subdir: talker/llm
|
||||
# The HF repo ships BailingMM2Config (thinker-only) at root;
|
||||
# OmniModelConfig treats that as "stage does not share outer mrope".
|
||||
hf_config_name: talker_config
|
||||
load_format: dummy
|
||||
engine_input_source: [0]
|
||||
custom_process_input_func: vllm_omni.model_executor.stage_input_processors.ming_flash_omni.thinker2talker
|
||||
final_output: true
|
||||
final_output_type: audio
|
||||
default_sampling_params:
|
||||
temperature: 0.0
|
||||
max_tokens: 1
|
||||
seed: 42
|
||||
@@ -0,0 +1,35 @@
|
||||
# Thinker stage only
|
||||
stage_args:
|
||||
- stage_id: 0
|
||||
stage_type: llm
|
||||
runtime:
|
||||
devices: "0,1,2,3"
|
||||
max_batch_size: 1
|
||||
engine_args:
|
||||
model_stage: thinker
|
||||
model_arch: MingFlashOmniForConditionalGeneration
|
||||
worker_type: ar
|
||||
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
|
||||
gpu_memory_utilization: 0.9
|
||||
enforce_eager: false
|
||||
trust_remote_code: true
|
||||
engine_output_type: latent
|
||||
distributed_executor_backend: "mp"
|
||||
enable_prefix_caching: false
|
||||
max_num_batched_tokens: 32768
|
||||
max_model_len: 32768
|
||||
tensor_parallel_size: 4
|
||||
hf_config_name: llm_config
|
||||
load_format: dummy
|
||||
mm_processor_cache_gb: 0
|
||||
final_output: true
|
||||
final_output_type: text
|
||||
is_comprehension: true
|
||||
default_sampling_params:
|
||||
temperature: 0.4
|
||||
top_p: 0.9
|
||||
max_tokens: 100
|
||||
repetition_penalty: 1.05
|
||||
seed: 42
|
||||
detokenize: true
|
||||
ignore_eos: false
|
||||
Reference in new issue
Block a user