[Model] Ming-flash-omni-2.0 Omni-Speech and TTS (#2890)

Signed-off-by: Yuanheng Zhao <jonathan.zhaoyh@gmail.com>
Signed-off-by: yuanheng <jonathan.zhaoyh@gmail.com>
Signed-off-by: LHXuuu <xulianhao.xlh@antgroup.com>
Co-authored-by: LHXuuu <xulianhao.xlh@antgroup.com>
Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
This commit is contained in:
authored and GitHub committed 2026-04-23 12:04:52 +08:00
1 parent f327cd1c89
commit 3a32cd6780
32 files changed
+4620 -261

No files matched your search

@@ -36,6 +36,21 @@ def build_prompt(user_text: str) -> str:
def get_eager_config():
path = modify_stage_config(
str(Path(__file__).parent.parent / "stage_configs" / "bailingmm_moe_v2_lite_thinker_only_ci.yaml"),
updates={
"stage_args": {
0: {
"engine_args.enforce_eager": "true",
},
},
},
)
return path
def get_eager_tts_config():
"""Thinker+talker CI config with enforce_eager on the thinker stage."""
path = modify_stage_config(
str(Path(__file__).parent.parent / "stage_configs" / "bailingmm_moe_v2_lite_ci.yaml"),
updates={
@@ -49,9 +64,14 @@ def get_eager_config():
return path
# Thinker-only config — used by text-output tests.
stage_configs = [get_eager_config()]
test_params = [(model, stage_config) for model in models for stage_config in stage_configs]
# Thinker+talker config — used by audio-output tests.
stage_configs_tts = [get_eager_tts_config()]
test_params_tts = [(model, stage_config) for model in models for stage_config in stage_configs_tts]
@pytest.mark.core_model
@pytest.mark.omni
@@ -140,3 +160,36 @@ def test_mixed_to_text(omni_runner, omni_runner_handler) -> None:
request_config = {"prompts": prompt, "images": image, "audios": audio, "modalities": ["text"]}
omni_runner_handler.send_request(request_config)
@pytest.mark.advanced_model
@pytest.mark.omni
@hardware_test(res={"cuda": "H100"}, num_cards=4)
@pytest.mark.parametrize("omni_runner", test_params_tts, indirect=True)
def test_text_to_audio(omni_runner, omni_runner_handler) -> None:
"""
Test text input with audio output via the thinker+talker pipeline.
Input Modal: text
Output Modal: audio
"""
prompt = build_prompt("请简单介绍一下北京。")
request_config = {"prompts": prompt, "modalities": ["audio"]}
omni_runner_handler.send_request(request_config)
@pytest.mark.advanced_model
@pytest.mark.omni
@hardware_test(res={"cuda": "H100"}, num_cards=4)
@pytest.mark.parametrize("omni_runner", test_params_tts, indirect=True)
def test_image_to_audio(omni_runner, omni_runner_handler) -> None:
"""
Test image + text input with audio output via the thinker+talker pipeline.
Input Modal: image + text
Output Modal: audio
"""
image = generate_synthetic_image(224, 224)["np_array"]
prompt = build_prompt(f"{IMAGE_TOKEN}Describe this image briefly.")
request_config = {"prompts": prompt, "images": image, "modalities": ["audio"]}
omni_runner_handler.send_request(request_config)
@@ -1,4 +1,8 @@
# Thinker stage only
# CI stage config for Ming-flash-omni-2.0 thinker+talker pipeline.
# Stage 0: Thinker (multimodal understanding -> text generation)
# Stage 1: Talker (text -> audio waveform via CFM + AudioVAE)
#
# The following config has been verified on 4x H100-80G GPUs
stage_args:
- stage_id: 0
stage_type: llm
@@ -10,10 +14,12 @@ stage_args:
model_arch: MingFlashOmniForConditionalGeneration
worker_type: ar
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
gpu_memory_utilization: 0.9
gpu_memory_utilization: 0.74
enforce_eager: false
trust_remote_code: true
engine_output_type: latent
# Ming Thinker -> talker bridge reads detokenised text from
# source_output.outputs[0].text (not hidden states).
engine_output_type: text
distributed_executor_backend: "mp"
enable_prefix_caching: false
max_num_batched_tokens: 32768
@@ -22,14 +28,48 @@ stage_args:
hf_config_name: llm_config
load_format: dummy
mm_processor_cache_gb: 0
compilation_config:
pass_config:
# Disable fused all-reduce to avoid a vllm/flashinfer version mismatch.
fuse_allreduce_rms: false
final_output: true
final_output_type: text
is_comprehension: true
default_sampling_params:
temperature: 0.4
temperature: 0.0
top_p: 0.9
max_tokens: 100
repetition_penalty: 1.05
seed: 42
detokenize: true
ignore_eos: false
- stage_id: 1
stage_type: llm
runtime:
devices: "3"
max_batch_size: 1
engine_args:
model_stage: ming_tts
model_arch: MingFlashOmniTalkerForConditionalGeneration
worker_cls: vllm_omni.worker.gpu_generation_worker.GPUGenerationWorker
scheduler_cls: vllm_omni.core.sched.omni_generation_scheduler.OmniGenerationScheduler
gpu_memory_utilization: 0.18
enforce_eager: true
trust_remote_code: true
engine_output_type: audio
enable_prefix_caching: false
max_num_batched_tokens: 1000000
tokenizer_subdir: talker/llm
# The HF repo ships BailingMM2Config (thinker-only) at root;
# OmniModelConfig treats that as "stage does not share outer mrope".
hf_config_name: talker_config
load_format: dummy
engine_input_source: [0]
custom_process_input_func: vllm_omni.model_executor.stage_input_processors.ming_flash_omni.thinker2talker
final_output: true
final_output_type: audio
default_sampling_params:
temperature: 0.0
max_tokens: 1
seed: 42
@@ -0,0 +1,35 @@
# Thinker stage only
stage_args:
- stage_id: 0
stage_type: llm
runtime:
devices: "0,1,2,3"
max_batch_size: 1
engine_args:
model_stage: thinker
model_arch: MingFlashOmniForConditionalGeneration
worker_type: ar
scheduler_cls: vllm_omni.core.sched.omni_ar_scheduler.OmniARScheduler
gpu_memory_utilization: 0.9
enforce_eager: false
trust_remote_code: true
engine_output_type: latent
distributed_executor_backend: "mp"
enable_prefix_caching: false
max_num_batched_tokens: 32768
max_model_len: 32768
tensor_parallel_size: 4
hf_config_name: llm_config
load_format: dummy
mm_processor_cache_gb: 0
final_output: true
final_output_type: text
is_comprehension: true
default_sampling_params:
temperature: 0.4
top_p: 0.9
max_tokens: 100
repetition_penalty: 1.05
seed: 42
detokenize: true
ignore_eos: false
@@ -0,0 +1,146 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from __future__ import annotations
from types import SimpleNamespace
import pytest
from vllm_omni.model_executor.models.ming_flash_omni.talker_module import (
CFM,
Aggregator,
CFMGraphExecutor,
CFMGraphExecutorPool,
DiT,
)
torch = pytest.importorskip("torch")
pytest.importorskip("x_transformers")
pytestmark = [
pytest.mark.core_model,
pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA for graph capture"),
]
_LATENT_DIM = 8
_PATCH_SIZE = 4
_HIS_PATCH_SIZE = 8
_LLM_HIDDEN = 16
_DIT_HIDDEN = 32
_AGG_HIDDEN = 32
_NUM_HEADS = 4
_DEPTH = 2
_STEPS = 5
_DTYPE = torch.float32
def _warmup_pipeline(cfm: CFM, aggregator: Aggregator, stop_head: torch.nn.Linear, device: torch.device) -> None:
llm_cond = torch.randn(1, 1, _LLM_HIDDEN, device=device, dtype=_DTYPE)
lat_cond = torch.randn(1, _HIS_PATCH_SIZE, _LATENT_DIM, device=device, dtype=_DTYPE)
y0 = torch.randn(1, _PATCH_SIZE, _LATENT_DIM, device=device, dtype=_DTYPE)
t = torch.linspace(0.0, 1.0, _STEPS + 1, device=device, dtype=_DTYPE)
sde_args = torch.tensor([2.0, 0.25, 0.0], device=device, dtype=_DTYPE)
sde_rnd = torch.randn(_STEPS, 1, _PATCH_SIZE, _LATENT_DIM, device=device, dtype=_DTYPE)
with torch.no_grad():
gen_lat = cfm.sample(llm_cond, lat_cond, y0, t, sde_args, sde_rnd)
aggregator(gen_lat)
stop_head(llm_cond[:, -1, :]).softmax(dim=-1)
torch.cuda.synchronize(device)
def _build_pipeline():
device = torch.device("cuda")
dit = (
DiT(
in_channels=_LATENT_DIM,
hidden_size=_DIT_HIDDEN,
depth=_DEPTH,
num_heads=_NUM_HEADS,
mlp_ratio=2.0,
llm_cond_dim=_LLM_HIDDEN,
)
.to(device=device, dtype=_DTYPE)
.eval()
)
cfm = CFM(dit, steps=_STEPS, sway_sampling_coef=-1.0).to(device=device, dtype=_DTYPE).eval()
aggregator = (
Aggregator(
in_channels=_LATENT_DIM,
hidden_size=_AGG_HIDDEN,
depth=_DEPTH,
num_heads=_NUM_HEADS,
mlp_ratio=2.0,
llm_input_dim=_LLM_HIDDEN,
)
.to(device=device, dtype=_DTYPE)
.eval()
)
stop_head = torch.nn.Linear(_LLM_HIDDEN, 2).to(device=device, dtype=_DTYPE).eval()
config = SimpleNamespace(steps=_STEPS, patch_size=_PATCH_SIZE)
_warmup_pipeline(cfm, aggregator, stop_head, device)
return config, cfm, aggregator, stop_head, device
class TestCFMGraphExecutor:
"""Capture once, replay twice: outputs must stay consistently-shaped."""
def test_execute_shapes_and_replay(self) -> None:
config, cfm, aggregator, stop_head, device = _build_pipeline()
executor = CFMGraphExecutor(config, cfm, aggregator, stop_head)
bsz = 1
input_tensor = torch.randn(bsz, 1, _LLM_HIDDEN, device=device, dtype=_DTYPE)
his_lat = torch.randn(bsz, _HIS_PATCH_SIZE, _LATENT_DIM, device=device, dtype=_DTYPE)
gen_lat, inputs_embeds, stop_out = executor.execute(input_tensor, his_lat)
torch.cuda.synchronize()
assert gen_lat.shape == (bsz, _PATCH_SIZE, _LATENT_DIM)
assert inputs_embeds.shape == (bsz, 1, _LLM_HIDDEN)
assert stop_out.shape == (bsz, 2)
assert torch.isfinite(gen_lat).all()
assert torch.isfinite(inputs_embeds).all()
# stop_head output is softmax-normalized across the last dim.
assert torch.allclose(stop_out.sum(dim=-1), torch.ones(bsz, device=device, dtype=_DTYPE), atol=1e-4)
# Replay the captured graph with fresh inputs — shapes must match.
new_input = torch.randn_like(input_tensor)
new_his = torch.randn_like(his_lat)
gen_lat2, inputs_embeds2, stop_out2 = executor.execute(new_input, new_his)
torch.cuda.synchronize()
assert gen_lat2.shape == gen_lat.shape
assert inputs_embeds2.shape == inputs_embeds.shape
assert stop_out2.shape == stop_out.shape
assert executor.initialized is True
def test_execute_is_noninplace_on_inputs(self) -> None:
config, cfm, aggregator, stop_head, device = _build_pipeline()
executor = CFMGraphExecutor(config, cfm, aggregator, stop_head)
input_tensor = torch.randn(1, 1, _LLM_HIDDEN, device=device, dtype=_DTYPE)
his_lat = torch.randn(1, _HIS_PATCH_SIZE, _LATENT_DIM, device=device, dtype=_DTYPE)
snapshot_input = input_tensor.clone()
snapshot_his = his_lat.clone()
executor.execute(input_tensor, his_lat)
torch.cuda.synchronize()
assert torch.equal(input_tensor, snapshot_input)
assert torch.equal(his_lat, snapshot_his)
class TestCFMGraphExecutorPool:
def test_pool_acquires_and_releases(self) -> None:
config, cfm, aggregator, stop_head, device = _build_pipeline()
pool = CFMGraphExecutorPool(config, cfm, aggregator, stop_head, pool_size=2)
input_tensor = torch.randn(1, 1, _LLM_HIDDEN, device=device, dtype=_DTYPE)
his_lat = torch.randn(1, _HIS_PATCH_SIZE, _LATENT_DIM, device=device, dtype=_DTYPE)
gen_lat, inputs_embeds, stop_out = pool.execute(input_tensor, his_lat)
torch.cuda.synchronize()
assert gen_lat.shape == (1, _PATCH_SIZE, _LATENT_DIM)
assert inputs_embeds.shape == (1, 1, _LLM_HIDDEN)
assert stop_out.shape == (1, 2)
assert pool.pool.qsize() == 2
@@ -0,0 +1,162 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from __future__ import annotations
import pytest
from vllm_omni.model_executor.models.ming_flash_omni.talker_module import CFM, Aggregator, DiT
torch = pytest.importorskip("torch")
pytest.importorskip("x_transformers")
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
_LATENT_DIM = 8
_PATCH_SIZE = 4
_HIS_PATCH_SIZE = 8
_LLM_HIDDEN = 16
_DIT_HIDDEN = 32
_AGG_HIDDEN = 32
_NUM_HEADS = 4
_DEPTH = 2
_STEPS = 5
def _make_dit() -> DiT:
return DiT(
in_channels=_LATENT_DIM,
hidden_size=_DIT_HIDDEN,
depth=_DEPTH,
num_heads=_NUM_HEADS,
mlp_ratio=2.0,
llm_cond_dim=_LLM_HIDDEN,
)
def _make_aggregator() -> Aggregator:
return Aggregator(
in_channels=_LATENT_DIM,
hidden_size=_AGG_HIDDEN,
depth=_DEPTH,
num_heads=_NUM_HEADS,
mlp_ratio=2.0,
llm_input_dim=_LLM_HIDDEN,
)
class TestDiTDummyForward:
"""DiT with dummy weights runs forward + CFG-doubled forward."""
def test_forward_shape(self) -> None:
dit = _make_dit().eval()
bsz = 2
x = torch.randn(bsz, _PATCH_SIZE, _LATENT_DIM)
t = torch.zeros(bsz)
c = torch.randn(bsz, 1, _LLM_HIDDEN)
latent_history = torch.randn(bsz, _HIS_PATCH_SIZE, _LATENT_DIM)
with torch.no_grad():
out = dit(x, t, c, latent_history)
# Output preserves the concatenated (history + time/cond prefix + x)
# token axis: history + 1 (time+cond) + patch.
assert out.shape == (bsz, _HIS_PATCH_SIZE + 1 + _PATCH_SIZE, _LATENT_DIM)
def test_forward_with_cfg_trims_to_patch(self) -> None:
dit = _make_dit().eval()
bsz = 1
x = torch.randn(bsz, _PATCH_SIZE, _LATENT_DIM)
t = torch.zeros(())
c = torch.randn(bsz, 1, _LLM_HIDDEN)
latent_history = torch.randn(bsz, _HIS_PATCH_SIZE, _LATENT_DIM)
with torch.no_grad():
out = dit.forward_with_cfg(x, t, c, latent_history)
# CFG doubles the batch and trims the output to the patch window.
assert out.shape == (2 * bsz, _PATCH_SIZE, _LATENT_DIM)
class TestAggregatorDummyForward:
"""Aggregator with dummy weights maps latent patch -> LLM hidden."""
def test_forward_shape(self) -> None:
agg = _make_aggregator().eval()
bsz = 3
gen_lat = torch.randn(bsz, _PATCH_SIZE, _LATENT_DIM)
with torch.no_grad():
out = agg(gen_lat)
assert out.shape == (bsz, 1, _LLM_HIDDEN)
def test_forward_is_finite(self) -> None:
agg = _make_aggregator().eval()
gen_lat = torch.randn(1, _PATCH_SIZE, _LATENT_DIM)
with torch.no_grad():
out = agg(gen_lat)
assert torch.isfinite(out).all()
class TestCFMSampleDummy:
"""CFM.sample drives DiT.forward_with_cfg through the integration loop."""
def test_sample_shape_and_finite(self) -> None:
cfm = CFM(_make_dit(), steps=_STEPS, sway_sampling_coef=-1.0).eval()
bsz = 1
llm_cond = torch.randn(bsz, 1, _LLM_HIDDEN)
lat_cond = torch.randn(bsz, _HIS_PATCH_SIZE, _LATENT_DIM)
y0 = torch.randn(bsz, _PATCH_SIZE, _LATENT_DIM)
# Grid used by the talker; must span [0, 1] inclusive.
t = torch.linspace(0.0, 1.0, _STEPS + 1)
sde_args = torch.tensor([2.0, 0.0, 0.0]) # cfg=2.0, sigma=0, temp=0
sde_rnd = torch.zeros(_STEPS, bsz, _PATCH_SIZE, _LATENT_DIM)
with torch.no_grad():
out = cfm.sample(llm_cond, lat_cond, y0, t, sde_args, sde_rnd)
assert out.shape == y0.shape
assert torch.isfinite(out).all()
def test_sample_zero_cfg_reduces_to_unguided(self) -> None:
"""With cfg=0 the guidance term drops, but output shape is still valid."""
cfm = CFM(_make_dit(), steps=_STEPS, sway_sampling_coef=None).eval()
bsz = 2
llm_cond = torch.randn(bsz, 1, _LLM_HIDDEN)
lat_cond = torch.randn(bsz, _HIS_PATCH_SIZE, _LATENT_DIM)
y0 = torch.zeros(bsz, _PATCH_SIZE, _LATENT_DIM)
t = torch.linspace(0.0, 1.0, _STEPS + 1)
sde_args = torch.tensor([0.0, 0.0, 0.0])
sde_rnd = torch.zeros(_STEPS, bsz, _PATCH_SIZE, _LATENT_DIM)
with torch.no_grad():
out = cfm.sample(llm_cond, lat_cond, y0, t, sde_args, sde_rnd)
assert out.shape == (bsz, _PATCH_SIZE, _LATENT_DIM)
assert torch.isfinite(out).all()
class TestTalkerPipelineDummyWiring:
"""End-to-end wiring of DiT -> CFM.sample -> Aggregator with dummy weights."""
def test_cfm_then_aggregator(self) -> None:
dit = _make_dit().eval()
cfm = CFM(dit, steps=_STEPS, sway_sampling_coef=-1.0).eval()
agg = _make_aggregator().eval()
bsz = 1
llm_cond = torch.randn(bsz, 1, _LLM_HIDDEN)
lat_cond = torch.randn(bsz, _HIS_PATCH_SIZE, _LATENT_DIM)
y0 = torch.randn(bsz, _PATCH_SIZE, _LATENT_DIM)
t = torch.linspace(0.0, 1.0, _STEPS + 1)
sde_args = torch.tensor([2.0, 0.0, 0.0])
sde_rnd = torch.zeros(_STEPS, bsz, _PATCH_SIZE, _LATENT_DIM)
with torch.no_grad():
gen_lat = cfm.sample(llm_cond, lat_cond, y0, t, sde_args, sde_rnd)
agg_out = agg(gen_lat)
assert gen_lat.shape == (bsz, _PATCH_SIZE, _LATENT_DIM)
assert agg_out.shape == (bsz, 1, _LLM_HIDDEN)
assert torch.isfinite(agg_out).all()