[Model] Ming-flash-omni-2.0 Omni-Speech and TTS (#2890)

Signed-off-by: Yuanheng Zhao <jonathan.zhaoyh@gmail.com>
Signed-off-by: yuanheng <jonathan.zhaoyh@gmail.com>
Signed-off-by: LHXuuu <xulianhao.xlh@antgroup.com>
Co-authored-by: LHXuuu <xulianhao.xlh@antgroup.com>
Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
This commit is contained in:
authored and GitHub committed 2026-04-23 12:04:52 +08:00
1 parent f327cd1c89
commit 3a32cd6780
32 files changed
+4620 -261

No files matched your search

@@ -2,6 +2,7 @@
# Copyright 2025 The vLLM-Omni team.
from .ming_flash_omni import MingFlashOmniForConditionalGeneration
from .ming_flash_omni_talker import MingFlashOmniTalkerForConditionalGeneration
from .ming_flash_omni_thinker import (
MingFlashOmniThinkerDummyInputsBuilder,
MingFlashOmniThinkerForConditionalGeneration,
@@ -11,6 +12,7 @@ from .ming_flash_omni_thinker import (
__all__ = [
"MingFlashOmniForConditionalGeneration",
"MingFlashOmniTalkerForConditionalGeneration",
"MingFlashOmniThinkerForConditionalGeneration",
"MingFlashOmniThinkerProcessingInfo",
"MingFlashOmniThinkerMultiModalProcessor",
@@ -0,0 +1,390 @@
# SPDX-License-Identifier: Apache-2.0
# Copyright 2025 The vLLM-Omni team.
# Copyright (c) Ant Group. All rights reserved.
# Adapted from:
# https://github.com/inclusionAI/Ming/tree/e58533db227031990c5a6864dcf5f08fb53ed0d2/AudioVAE
from __future__ import annotations
import torch
import torch.nn as nn
import torch.nn.functional as F
from transformers import PretrainedConfig, PreTrainedModel, Qwen2Config, Qwen2Model
from vllm.logger import init_logger
logger = init_logger(__name__)
try:
import flash_attn # noqa: F401
except (ImportError, ModuleNotFoundError):
flash_attn = None
logger.warning(
"flash_attn is not available, the model may not yield the "
"exactly same result as the transformers implementation "
"in the audio tower part."
)
class AudioVAEConfig(PretrainedConfig):
model_type = "audio_vae"
def __init__(
self,
sample_rate: int = 44100,
enc_kwargs: dict | None = None,
dec_kwargs: dict | None = None,
init_method: str = "kaiming",
patch_size: int = 4,
**kwargs,
):
self.sample_rate = sample_rate
self.enc_kwargs = enc_kwargs or {}
self.dec_kwargs = dec_kwargs or {}
self.init_method = init_method
self.patch_size = patch_size
super().__init__(**kwargs)
class ISTFT(nn.Module):
def __init__(self, n_fft: int, hop_length: int, win_length: int, padding: str = "same"):
super().__init__()
if padding not in ["center", "same"]:
raise ValueError("Padding must be 'center' or 'same'.")
self.padding = padding
self.n_fft = n_fft
self.hop_length = hop_length
self.win_length = win_length
window = torch.hann_window(win_length)
self.register_buffer("window", window)
self.buffer_len = self.win_length - self.hop_length
def _buffer_process(self, x, buffer, pad, last_chunk=False, streaming=False):
if streaming:
if buffer is None:
x = x[:, pad:]
if buffer is not None:
x[:, : self.buffer_len] += buffer
buffer = x[:, -self.buffer_len :]
if not last_chunk:
x = x[:, : -self.buffer_len]
else:
x = x[:, :-pad]
else:
x = x[:, pad:-pad]
return x, buffer
def forward(self, spec, audio_buffer=None, window_buffer=None, streaming=False, last_chunk=False):
if self.padding == "center":
return torch.istft(spec, self.n_fft, self.hop_length, self.win_length, self.window, center=True)
elif self.padding == "same":
pad = (self.win_length - self.hop_length) // 2
else:
raise ValueError("Padding must be 'center' or 'same'.")
B, N, T = spec.shape
ifft = torch.fft.irfft(spec, self.n_fft, dim=1, norm="backward")
ifft = ifft * self.window[None, :, None]
output_size = (T - 1) * self.hop_length + self.win_length
y = torch.nn.functional.fold(
ifft,
output_size=(1, output_size),
kernel_size=(1, self.win_length),
stride=(1, self.hop_length),
)[:, 0, 0, :]
y, audio_buffer = self._buffer_process(y, audio_buffer, pad, last_chunk=last_chunk, streaming=streaming)
window_sq = self.window.square().expand(1, T, -1).transpose(1, 2)
window_envelope = (
torch.nn.functional.fold(
window_sq,
output_size=(1, output_size),
kernel_size=(1, self.win_length),
stride=(1, self.hop_length),
)
.squeeze(0)
.squeeze(0)
)
window_envelope, window_buffer = self._buffer_process(
window_envelope, window_buffer, pad, last_chunk=last_chunk, streaming=streaming
)
window_envelope = window_envelope.squeeze()
assert (window_envelope > 1e-11).all()
y = y / window_envelope
return y, audio_buffer, window_buffer
class ISTFTHead(nn.Module):
def __init__(self, dim: int, n_fft: int, hop_length: int, padding: str = "same"):
super().__init__()
out_dim = n_fft + 2
self.out = nn.Linear(dim, out_dim)
self.istft = ISTFT(n_fft=n_fft, hop_length=hop_length, win_length=n_fft, padding=padding)
def forward(self, x, audio_buffer=None, window_buffer=None, streaming=False, last_chunk=False):
x_pred = self.out(x)
x_pred = x_pred.transpose(1, 2)
mag, p = x_pred.chunk(2, dim=1)
mag = torch.exp(mag)
mag = torch.clip(mag, max=1e2)
x = torch.cos(p)
y = torch.sin(p)
S = mag * (x + 1j * y)
audio, audio_buffer, window_buffer = self.istft(
S, audio_buffer=audio_buffer, window_buffer=window_buffer, streaming=streaming, last_chunk=last_chunk
)
return audio.unsqueeze(1), x_pred, audio_buffer, window_buffer
class StreamingLinearUpsample(nn.Module):
def __init__(self, scale_factor=4):
super().__init__()
self.scale_factor = scale_factor
self.upsampler = nn.Upsample(scale_factor=scale_factor, mode="linear", align_corners=False)
def forward(self, x, state=None, is_last=False):
if state is None:
state = {"prev_chunk": None, "history_last": None, "is_first": True}
if x is None and not is_last:
return None, state
if state["is_first"] and is_last:
out = self.upsampler(x.transpose(1, 2)).transpose(1, 2)
return out, None
output_chunks = []
if state["is_first"]:
state["prev_chunk"] = x
state["is_first"] = False
if not is_last:
return None, state
if state["prev_chunk"] is not None:
p = state["prev_chunk"].transpose(1, 2)
if state["history_last"] is None:
lookahead = x[:, :1, :].transpose(1, 2)
inp = torch.cat([p, lookahead], dim=2)
up = self.upsampler(inp)
out_prev = up[:, :, : p.size(2) * self.scale_factor]
else:
lookahead = x[:, :1, :].transpose(1, 2)
inp = torch.cat([state["history_last"], p, lookahead], dim=2)
up = self.upsampler(inp)
start = self.scale_factor
end = start + p.size(2) * self.scale_factor
out_prev = up[:, :, start:end]
output_chunks.append(out_prev.transpose(1, 2))
state["history_last"] = p[:, :, -1:]
state["prev_chunk"] = x
if is_last:
p = state["prev_chunk"].transpose(1, 2)
inp = torch.cat([state["history_last"], p], dim=2)
up = self.upsampler(inp)
out_last = up[:, :, self.scale_factor :]
output_chunks.append(out_last.transpose(1, 2))
state = None
final_out = torch.cat(output_chunks, dim=1) if output_chunks else None
return final_out, state
class Decoder(nn.Module):
def __init__(self, decoder_args, output_dim=320, latent_dim=64, patch_size=-1):
super().__init__()
config = Qwen2Config.from_dict(config_dict=decoder_args)
if flash_attn is None:
config._attn_implementation = "sdpa"
self.decoder = Qwen2Model(config)
self.output_dim = output_dim
self.latent_dim = latent_dim
self.fc1 = nn.Linear(latent_dim, config.hidden_size)
self.hop_length = output_dim
self.head = ISTFTHead(
dim=config.hidden_size, n_fft=self.hop_length * 4, hop_length=self.hop_length, padding="same"
)
self.patch_size = patch_size
if self.patch_size != -1:
self.upsampling = StreamingLinearUpsample(scale_factor=patch_size)
def low_level_reconstruct(self, x, past_key_values=None, use_cache=False, stream_state=None, last_chunk=False):
upsample_state, audio_buffer, window_buffer = stream_state
bsz, device, dtype = x.size(0), x.device, x.dtype
x = self.fc1(x)
if self.patch_size != -1:
if use_cache:
x, upsample_state = self.upsampling(x, state=upsample_state, is_last=last_chunk)
if x is None:
stream_state = (upsample_state, audio_buffer, window_buffer)
return torch.empty(bsz, 1, 0, device=device, dtype=dtype), stream_state, past_key_values
else:
x = self.upsampling.upsampler(x.transpose(1, 2)).transpose(1, 2)
hidden_states_list = []
if use_cache and getattr(self.decoder.config, "sliding_window", None) is not None:
sw_size = self.decoder.config.sliding_window
target_len = sw_size - 1
if past_key_values is None:
past_len = 0
elif hasattr(past_key_values, "get_seq_length"):
past_len = past_key_values.get_seq_length()
elif isinstance(past_key_values, tuple) and len(past_key_values) > 0:
past_len = past_key_values[0][0].shape[-2]
else:
past_len = 0
curr_len = x.shape[1]
if past_len < target_len and (past_len + curr_len) >= sw_size:
fill_len = target_len - past_len
x_fill = x[:, :fill_len, :]
outputs = self.decoder(inputs_embeds=x_fill, past_key_values=past_key_values, use_cache=use_cache)
hidden_states_list.append(outputs.last_hidden_state)
past_key_values = outputs.past_key_values
x = x[:, fill_len:, :]
outputs = self.decoder(inputs_embeds=x, past_key_values=past_key_values, use_cache=use_cache)
hidden_states_list.append(outputs.last_hidden_state)
past_key_values = outputs.past_key_values
if len(hidden_states_list) > 1:
full_hidden_state = torch.cat(hidden_states_list, dim=1)
else:
full_hidden_state = hidden_states_list[0]
x_out, _, audio_buffer, window_buffer = self.head(
full_hidden_state,
streaming=use_cache,
audio_buffer=audio_buffer,
window_buffer=window_buffer,
last_chunk=last_chunk,
)
stream_state = (upsample_state, audio_buffer, window_buffer)
return x_out, stream_state, past_key_values
class Encoder(nn.Module):
def __init__(self, encoder_args, input_dim=320, hop_size=320, latent_dim=64, patch_size=-1):
super().__init__()
config = Qwen2Config.from_dict(config_dict=encoder_args)
if flash_attn is None:
config._attn_implementation = "sdpa"
self.encoder = Qwen2Model(config)
self.input_dim = input_dim
self.hop_size = hop_size
self.latent_dim = latent_dim
self.fc1 = nn.Linear(input_dim, config.hidden_size, bias=False)
self.fc2 = nn.Linear(config.hidden_size, config.hidden_size)
self.fc3 = nn.Linear(config.hidden_size, latent_dim * 2)
self.norm = nn.LayerNorm(config.hidden_size)
self.patch_size = patch_size
if patch_size != -1:
config.num_hidden_layers = 4
self.aggregator = Qwen2Model(config)
self.cls_embed = nn.Parameter(torch.rand(1, 1, config.hidden_size))
self.cls_embed.data.normal_(0, 0.02)
def get_frames(self, x):
num_frames_total = (x.size(-1) + self.hop_size - 1) // self.hop_size
expected_len = (num_frames_total - 1) * self.hop_size + self.input_dim
padding_needed = expected_len - x.size(-1)
waveform = F.pad(x, (0, padding_needed), value=0.0)
frames = waveform.unfold(dimension=-1, size=self.input_dim, step=self.hop_size)
return frames
def pad_patch_insert_cls(self, x):
bsz, _, dim = x.size()
num_frame = x.size(1)
r = num_frame % self.patch_size
pad_num = self.patch_size - r if r else 0
x = F.pad(x, (0, 0, 0, pad_num), value=0.0)
x = x.reshape(-1, self.patch_size, dim)
x = torch.cat((x, self.cls_embed.expand(x.size(0), -1, -1)), dim=1)
x = x.reshape(bsz, -1, dim)
return x
def forward(self, waveform):
x = self.get_frames(waveform)
x = self.fc1(x)
x = self.fc2(x)
x = self.encoder(inputs_embeds=x)
x = x.last_hidden_state
if self.patch_size != -1:
x = self.pad_patch_insert_cls(x)
x = self.aggregator(inputs_embeds=x)
x = x.last_hidden_state
bsz, _, dim = x.size()
x = x.reshape(-1, self.patch_size + 1, dim)
x = x[:, -1:, :].reshape(bsz, -1, dim)
x = self.fc3(x)
return x, waveform.unsqueeze(1)
class AudioVAE(PreTrainedModel):
config_class = AudioVAEConfig
def __init__(self, config: AudioVAEConfig):
super().__init__(config)
self.encoder = Encoder(
encoder_args=config.enc_kwargs["backbone"],
input_dim=config.enc_kwargs["input_dim"],
hop_size=config.enc_kwargs.get("hop_size", 320),
latent_dim=config.enc_kwargs["latent_dim"],
patch_size=config.patch_size,
)
self.decoder = Decoder(
decoder_args=config.dec_kwargs["backbone"],
output_dim=config.dec_kwargs["output_dim"],
latent_dim=config.dec_kwargs["latent_dim"],
patch_size=config.patch_size,
)
self.post_init()
def _init_weights(self, module):
std = 0.02
if isinstance(module, nn.Linear):
if self.config.init_method == "kaiming":
nn.init.kaiming_normal_(module.weight, mode="fan_in", nonlinearity="relu")
else:
module.weight.data.normal_(mean=0.0, std=std)
if module.bias is not None:
module.bias.data.zero_()
elif isinstance(module, nn.Embedding):
module.weight.data.normal_(mean=0.0, std=std)
if module.padding_idx is not None:
module.weight.data[module.padding_idx].zero_()
def encode_latent(self, waveform, waveform_length):
from diffusers.models.autoencoders.autoencoder_oobleck import OobleckDiagonalGaussianDistribution
frame_num = torch.ceil(waveform_length / self.config.enc_kwargs["input_dim"]).to(torch.int32)
if self.config.patch_size != -1:
frame_num = torch.ceil(frame_num / self.config.patch_size)
h, y = self.encoder(waveform)
h = h.transpose(1, 2)
posterior = OobleckDiagonalGaussianDistribution(h)
latent = posterior.sample()
latent = latent.transpose(1, 2)
return latent, frame_num
def decode(self, latent, past_key_values=None, use_cache=False, stream_state=(None, None, None), last_chunk=False):
waveform, stream_state, past_key_values = self.decoder.low_level_reconstruct(
latent,
past_key_values=past_key_values,
use_cache=use_cache,
stream_state=stream_state,
last_chunk=last_chunk,
)
return waveform, stream_state, past_key_values
@@ -16,7 +16,23 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""Ming-flash-omni-2.0 unified model (thinker + imagegen + talker)."""
"""Ming-flash-omni-2.0 thinker / image-gen wrapper.
This class is the multimodal-registered entry point for Ming stages that
share the thinker's backbone: comprehension / text generation (`thinker`)
and diffusion conditioning for image generation (`imagegen`, not yet
implemented).
The talker deliberately lives elsewhere. Upstream Ming hands text (not
hidden states) from the thinker to the talker, and the talker then
tokenises that string with its own Qwen2 tokenizer and runs an entirely
self-contained LLM + CFM + AudioVAE pipeline. Because it has no
multimodal inputs, it belongs in the non-MM-registered
`MingFlashOmniTalkerForConditionalGeneration` — routing it through
this wrapper would force it through vLLM's multimodal preprocess path
and trigger a hidden-size mismatch between the outer Ming config
(4096, thinker's LLM) and the talker's Qwen2 backbone (896).
"""
from collections.abc import Iterable
@@ -40,7 +56,10 @@ from vllm.sequence import IntermediateTensors
from vllm_omni.model_executor.custom_process_mixin import CustomProcessMixin
from vllm_omni.model_executor.models.output_templates import OmniOutput
from vllm_omni.model_executor.models.utils import add_prefix_to_loaded_weights
from vllm_omni.transformers_utils.configs.ming_flash_omni import BailingMM2Config, MingFlashOmniConfig
from vllm_omni.transformers_utils.configs.ming_flash_omni import (
BailingMM2Config,
MingFlashOmniConfig,
)
from .ming_flash_omni_thinker import (
MingFlashOmniThinkerDummyInputsBuilder,
@@ -63,7 +82,7 @@ class MingFlashOmniForConditionalGeneration(
SupportsMRoPE,
CustomProcessMixin,
):
"""Unified Ming-flash-omni-2.0 model combining thinker, imagegen, and talker."""
"""Ming-flash-omni-2.0 thinker + image-gen wrapper."""
supports_multimodal = True
requires_raw_input_tokens: bool = True
@@ -75,19 +94,26 @@ class MingFlashOmniForConditionalGeneration(
self.has_postprocess = False
config = vllm_config.model_config.hf_config
self.vllm_config = vllm_config
self.config = config
if isinstance(config, MingFlashOmniConfig):
thinker_config = config.thinker_config
else:
thinker_config = config
self.thinker_config: BailingMM2Config = thinker_config
self.model_stage = vllm_config.model_config.model_stage
if self.model_stage == "talker":
raise ValueError(
"MingFlashOmniForConditionalGeneration does not support "
"model_stage='talker'. Use "
"model_arch='MingFlashOmniTalkerForConditionalGeneration' "
"directly — the talker has a self-contained LLM that "
"tokenises text itself and does not need the multimodal "
"preprocess path. See stage_configs/ming_flash_omni.yaml "
"stage 1 and stage_configs/ming_flash_omni_tts.yaml."
)
if self.model_stage == "thinker":
if isinstance(config, MingFlashOmniConfig):
thinker_config: BailingMM2Config = config.thinker_config
else:
thinker_config: BailingMM2Config = config
thinker_vllm_config = vllm_config.with_hf_config(
thinker_config, architectures=["MingFlashOmniThinkerForConditionalGeneration"]
)
@@ -97,8 +123,7 @@ class MingFlashOmniForConditionalGeneration(
architectures=["MingFlashOmniThinkerForConditionalGeneration"],
)
self.model = self.thinker
self.imagegen = None
self.talker = None
self.make_empty_intermediate_tensors = self.thinker.make_empty_intermediate_tensors
elif self.model_stage == "imagegen":
# TODO: Implement image generator stage
@@ -106,22 +131,12 @@ class MingFlashOmniForConditionalGeneration(
"Image generation stage is not yet implemented. Please use model_stage='thinker' for now."
)
elif self.model_stage == "talker":
# TODO: Implement talker (TTS) stage
raise NotImplementedError(
"Talker (TTS) stage is not yet implemented. Please use model_stage='thinker' for now."
)
else:
raise ValueError(
f"Invalid model_stage: {self.model_stage}. Must be one of: 'thinker', 'imagegen', 'talker'"
f"Invalid model_stage: {self.model_stage!r}. Must be one of: 'thinker', 'imagegen'. "
f"For the talker stage, use MingFlashOmniTalkerForConditionalGeneration directly."
)
# Set up intermediate tensors
self.make_empty_intermediate_tensors = (
self.thinker.make_empty_intermediate_tensors if self.model_stage == "thinker" else lambda: None
)
def forward(
self,
input_ids: torch.Tensor,
@@ -154,7 +169,7 @@ class MingFlashOmniForConditionalGeneration(
):
if hasattr(self.model, "sample"):
return self.model.sample(logits, sampling_metadata)
raise NotImplementedError("sample method not available on current stage")
return None
def get_mrope_input_positions(self, *args, **kwargs):
if hasattr(self.model, "get_mrope_input_positions"):
@@ -162,36 +177,9 @@ class MingFlashOmniForConditionalGeneration(
raise NotImplementedError("get_mrope_input_positions not available on current stage")
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
loaded_weights = set()
thinker_weights = []
imagegen_weights = []
talker_weights = []
for name, value in weights:
if name.startswith("thinker."):
thinker_weights.append((name, value))
elif name.startswith("imagegen."):
imagegen_weights.append((name, value))
elif name.startswith("talker."):
talker_weights.append((name, value))
else:
# Weights without prefix go to thinker by default
thinker_weights.append((name, value))
if self.model_stage == "thinker" and thinker_weights:
# Remove "thinker." prefix before loading
thinker_weights_stripped = [
(name.replace("thinker.", "", 1) if name.startswith("thinker.") else name, value)
for name, value in thinker_weights
]
thinker_loaded = self.thinker.load_weights(thinker_weights_stripped)
thinker_loaded = add_prefix_to_loaded_weights(thinker_loaded, "thinker")
loaded_weights.update(thinker_loaded)
# TODO: Load imagegen weights when implemented
# TODO: Load talker weights when implemented
return loaded_weights
stripped = ((name.removeprefix("thinker."), value) for name, value in weights)
thinker_loaded = self.thinker.load_weights(stripped)
return add_prefix_to_loaded_weights(thinker_loaded, "thinker")
def get_mm_mapping(self) -> MultiModelKeys:
return MultiModelKeys.from_string_field(
@@ -202,9 +190,7 @@ class MingFlashOmniForConditionalGeneration(
@property
def sampler(self):
if hasattr(self.model, "sampler"):
return self.model.sampler
return None
return getattr(self.model, "sampler", None)
def embed_input_ids(
self,
@@ -0,0 +1,586 @@
# SPDX-License-Identifier: Apache-2.0
# Copyright 2025 The vLLM-Omni team.
# Copyright (c) Ant Group. All rights reserved.
# Adapted from:
# https://github.com/inclusionAI/Ming/blob/e58533db227031990c5a6864dcf5f08fb53ed0d2/modeling_bailing_talker.py
"""Ming-flash-omni-2.0 talker (TTS) stage model."""
from __future__ import annotations
import glob as glob_module
import os
from collections.abc import Iterable
from dataclasses import dataclass
from functools import cached_property
from typing import Any
import torch
import torch.nn as nn
from safetensors.torch import load_file
from transformers import AutoTokenizer, Qwen2Config, Qwen2Model
from transformers.utils.hub import cached_file
from vllm.config import VllmConfig
from vllm.logger import init_logger
from vllm.model_executor.models.utils import AutoWeightsLoader
from vllm.sequence import IntermediateTensors
from vllm_omni.model_executor.custom_process_mixin import CustomProcessMixin
from vllm_omni.model_executor.model_loader.weight_utils import download_weights_from_hf_specific
from vllm_omni.model_executor.models.output_templates import OmniOutput
from vllm_omni.transformers_utils.configs.ming_flash_omni import MingFlashOmniTalkerConfig
from .audio_vae import AudioVAE, AudioVAEConfig
from .prompt_utils import DEFAULT_PROMPT as MING_DEFAULT_PROMPT
from .talker_module import CFM, Aggregator, DiT, MingAudioGenerator, build_tts_input
from .text_processing import segment_and_normalize
from .voice_presets import VoicePresetRegistry
logger = init_logger(__name__)
@dataclass(slots=True)
class _GenerationParams:
"""Resolved sampling / decoding parameters for one forward call."""
prompt: str
instruction: str | None
cfg: float
sigma: float
temperature: float
max_steps: int
use_zero_spk_emb: bool
max_text_length: int
use_static_cache: bool
stream_decode: bool
@dataclass(slots=True)
class _VoiceContext:
"""Voice cloning inputs resolved from request info + presets."""
spk_emb: Any # list[Tensor] | Tensor | list[float] | None
prompt_text: str | None
prompt_wav_lat: torch.Tensor | None
prompt_wav_emb: torch.Tensor | None
already_projected: bool
class MingFlashOmniTalkerForConditionalGeneration(nn.Module, CustomProcessMixin):
"""Ming-flash-omni-2.0 talker stage: text -> audio waveform.
Uses Qwen2 LLM + CFM (Conditional Flow Matching with DiT) + Aggregator
in an autoregressive loop to produce continuous audio latents, then
AudioVAE decodes latents to waveforms.
"""
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
self.have_multimodal_outputs = True
self.has_preprocess = False
self.has_postprocess = False
self.vllm_config = vllm_config
root_config = vllm_config.model_config.hf_config
model_path = vllm_config.model_config.model
self._model_path = model_path
self.talker_dir = (
os.path.join(model_path, "talker") if os.path.isdir(os.path.join(model_path, "talker")) else model_path
)
# When used standalone (model_arch=MingFlashOmniTalkerForConditionalGeneration),
# the root hf_config may be BailingMM2Config (thinker-only) due to model file structure
# Resolve talker config from talker/config.json in that case.
config = (
root_config
if isinstance(root_config, MingFlashOmniTalkerConfig)
else self._resolve_talker_config(root_config, self.talker_dir, model_path)
)
self.config = config
self._standalone = prefix in ("", "talker")
if self._standalone:
self.allow_patterns_overrides = ["talker/model*.safetensors"]
self.fall_back_to_pt_during_load = False
# LLM
llm_config = self._resolve_llm_config(config, self.talker_dir, model_path)
llm_config._attn_implementation = "sdpa"
self.llm_config = llm_config
self.hidden_size = llm_config.hidden_size
self.latent_dim = config.latent_dim
self.patch_size = config.patch_size
self.his_patch_size = config.history_patch_size
self.cfg_strength = config.cfg_strength
self.model = Qwen2Model(llm_config)
self.cfm = CFM(
DiT(llm_input_dim=self.hidden_size, **config.flowmodel),
steps=config.steps,
)
self.aggregator = Aggregator(llm_input_dim=self.hidden_size, **config.aggregator)
self.stop_head = nn.Linear(self.hidden_size, 2, bias=True)
# CAMPPlus 192-dim -> hidden
self.spk_head = nn.Linear(192, self.hidden_size, bias=True)
# AudioVAE
self.audio_vae, self._vae_weight_source = self._init_audio_vae(config, self.talker_dir, model_path)
self._use_cuda_graphs = not vllm_config.model_config.enforce_eager
self.audio_generator = MingAudioGenerator(
config=self.config,
llm_config=self.llm_config,
model=self.model,
cfm=self.cfm,
aggregator=self.aggregator,
stop_head=self.stop_head,
audio_vae=self.audio_vae,
patch_size=self.patch_size,
his_patch_size=self.his_patch_size,
latent_dim=self.latent_dim,
cfg_strength=self.cfg_strength,
use_cuda_graphs=self._use_cuda_graphs,
)
self.voice_presets = VoicePresetRegistry(
talker_dir=self.talker_dir,
model_path=self._model_path,
download_dir=vllm_config.load_config.download_dir,
audio_vae=self.audio_vae,
aggregator=self.aggregator,
spk_head=self.spk_head,
patch_size=self.patch_size,
)
@property
def device(self) -> torch.device:
return next(self.model.parameters()).device
@property
def dtype(self) -> torch.dtype:
return next(self.model.parameters()).dtype
@cached_property
def tokenizer(self):
# Lazy Qwen2 tokenizer resolution:
# 1. Try local dirs first (talker/llm, talker, and then model root).
# 2. HF repo-id fallback: talker/llm is the canonical tokenizer location.
candidates = (os.path.join(self.talker_dir, "llm"), self.talker_dir, self._model_path)
for path in candidates:
if os.path.isdir(path):
try:
logger.debug("Resolving talker tokenizer from local dir %s", path)
return AutoTokenizer.from_pretrained(path, trust_remote_code=True)
except Exception:
continue
for subfolder in ("talker/llm", "llm"):
try:
logger.debug("Resolving talker tokenizer from HF subfolder %s", subfolder)
return AutoTokenizer.from_pretrained(self._model_path, subfolder=subfolder, trust_remote_code=True)
except Exception:
continue
logger.debug("Falling back to raw model_path tokenizer resolution")
return AutoTokenizer.from_pretrained(self._model_path, trust_remote_code=True)
@staticmethod
def _resolve_talker_config(config, talker_dir: str, model_path: str) -> MingFlashOmniTalkerConfig:
"""Resolve MingFlashOmniTalkerConfig when the root config is not one.
This happens in standalone TTS mode where hf_config is BailingMM2Config.
"""
# If the root config wraps a talker_config, use it
talker_config = getattr(config, "talker_config", None)
if isinstance(talker_config, MingFlashOmniTalkerConfig):
return talker_config
# Try loading from talker/config.json
if os.path.isdir(talker_dir):
try:
resolved = MingFlashOmniTalkerConfig.from_pretrained(talker_dir)
logger.info("Resolved talker config from %s", talker_dir)
return resolved
except Exception:
pass
try:
resolved = MingFlashOmniTalkerConfig.from_pretrained(model_path, subfolder="talker", trust_remote_code=True)
logger.info("Resolved talker config from %s/talker (HF hub)", model_path)
return resolved
except Exception as e:
raise ValueError(
f"Cannot resolve MingFlashOmniTalkerConfig. The root config "
f"is {type(config).__name__}, and talker/config.json was not "
f"found at {talker_dir} or via HF hub: {e}"
) from e
@staticmethod
def _resolve_llm_config(config: MingFlashOmniTalkerConfig, talker_dir: str, model_path: str) -> Qwen2Config:
"""Resolve the Qwen2 LLM config for the talker backbone."""
if config.llm_config is not None:
return Qwen2Config(**config.llm_config) if isinstance(config.llm_config, dict) else config.llm_config
# Try local talker/llm directory
llm_dir = os.path.join(talker_dir, "llm")
if os.path.isdir(llm_dir):
return Qwen2Config.from_pretrained(llm_dir)
# HF hub fallback
for subfolder in ("talker/llm", "llm"):
try:
return Qwen2Config.from_pretrained(model_path, subfolder=subfolder, trust_remote_code=True)
except Exception:
continue
raise ValueError(
f"Cannot find talker LLM config at {llm_dir}. "
"Either provide llm_config in MingFlashOmniTalkerConfig or "
"ensure the model path contains talker/llm/config.json."
)
@staticmethod
def _init_audio_vae(
config: MingFlashOmniTalkerConfig, talker_dir: str, model_path: str
) -> tuple[AudioVAE | None, str | tuple[str, str] | None]:
"""Initialize AudioVAE and return (vae, weight_source).
weight_source is either a local directory path (str) or an
(repo_id, subfolder) tuple for HF hub downloads, or None.
"""
vae_path = config.audio_vae_path or os.path.join(talker_dir, "vae")
# Try local directory first
if os.path.isdir(vae_path):
try:
vae_config = AudioVAEConfig.from_pretrained(vae_path)
vae = AudioVAE(vae_config)
logger.info("Initialized AudioVAE from %s (sr=%d)", vae_path, vae_config.sample_rate)
return vae, vae_path
except Exception as e:
logger.warning("Failed to initialize AudioVAE from %s: %s", vae_path, e)
return None, None
# HF hub fallback
for subfolder in ("talker/vae", "vae"):
try:
vae_config = AudioVAEConfig.from_pretrained(model_path, subfolder=subfolder, trust_remote_code=True)
vae = AudioVAE(vae_config)
logger.info(f"Initialized AudioVAE from {model_path}/{subfolder}")
return vae, (model_path, subfolder)
except Exception:
continue
logger.info("AudioVAE not found at %s; waveform decoding unavailable", vae_path)
return None, None
def compute_logits(self, hidden_states: torch.Tensor, sampling_metadata=None) -> torch.Tensor | None:
return None
def sample(self, logits: torch.Tensor, sampling_metadata):
return None
def embed_input_ids(
self,
input_ids: torch.Tensor,
multimodal_embeddings=None,
is_multimodal=None,
) -> torch.Tensor:
return self.model.get_input_embeddings()(input_ids)
def make_empty_intermediate_tensors(
self, batch_size: int, dtype: torch.dtype, device: torch.device
) -> IntermediateTensors | None:
return None
def get_dummy_runtime_additional_information(self, num_reqs: int) -> list[dict[str, object]]:
info: dict[str, object] = {"text": "dummy", "use_zero_spk_emb": True, "max_steps": 1}
return [info for _ in range(num_reqs)]
def forward(
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
runtime_additional_information: list[dict] | None = None,
**kwargs,
) -> OmniOutput:
"""Run TTS generation and return audio output.
The full autoregressive generation loop is executed inside this method.
"""
additional_info = self._extract_additional_info(runtime_additional_information)
params = self._resolve_generation_params(additional_info)
voice = self._resolve_voice(additional_info)
latents = self._generate_latents(
input_ids=input_ids,
inputs_embeds=inputs_embeds,
text=additional_info.get("text", ""),
params=params,
voice=voice,
)
return self._decode_to_output(latents, stream_decode=params.stream_decode)
@staticmethod
def _extract_additional_info(
runtime_additional_information: list[dict] | None,
) -> dict[str, Any]:
if runtime_additional_information and len(runtime_additional_information) > 0:
return runtime_additional_information[0] or {}
return {}
def _resolve_generation_params(self, additional_info: dict[str, Any]) -> _GenerationParams:
# "omni" : thinker -> talker hand-off with hardcoded defaults
# "instruct": standalone TTS with caller-supplied sampling knobs
ming_task = additional_info.get("ming_task", "instruct")
if ming_task == "omni":
prompt = MING_DEFAULT_PROMPT
instruction = None
use_zero_spk_emb = additional_info.get("spk_emb") is None
cfg = 2.0
sigma = 0.25
temperature = 0.0
max_steps = 200
else:
prompt = additional_info.get("prompt", MING_DEFAULT_PROMPT)
instruction = additional_info.get("instruction", None)
use_zero_spk_emb = additional_info.get("use_zero_spk_emb", False)
cfg = additional_info.get("cfg", self.cfg_strength)
sigma = additional_info.get("sigma", 0.25)
temperature = additional_info.get("temperature", 0.0)
max_steps = int(additional_info.get("max_steps", additional_info.get("max_decode_steps", 200)))
return _GenerationParams(
prompt=prompt,
instruction=instruction,
cfg=cfg,
sigma=sigma,
temperature=temperature,
max_steps=max_steps,
use_zero_spk_emb=use_zero_spk_emb,
max_text_length=int(additional_info.get("max_text_length", 50)),
use_static_cache=bool(additional_info.get("use_static_cache", True)),
stream_decode=bool(additional_info.get("stream_decode", True)),
)
def _resolve_voice(self, additional_info: dict[str, Any]) -> _VoiceContext:
spk_emb = additional_info.get("spk_emb", None)
prompt_text = additional_info.get("prompt_text", None)
prompt_wav_lat = additional_info.get("prompt_wav_lat", None)
prompt_wav_emb = additional_info.get("prompt_wav_emb", None)
already_projected = False
voice_name = additional_info.get("voice_name", None)
if voice_name and spk_emb is None and voice_name in self.voice_presets:
preset = self.voice_presets.get(voice_name) or {}
prompt_wav_lat = preset.get("prompt_wav_lat")
prompt_wav_emb = preset.get("prompt_wav_emb")
spk_emb = preset.get("spk_emb")
already_projected = True
if prompt_text is None:
prompt_text = preset.get("prompt_text")
return _VoiceContext(
spk_emb=spk_emb,
prompt_text=prompt_text,
prompt_wav_lat=prompt_wav_lat,
prompt_wav_emb=prompt_wav_emb,
already_projected=already_projected,
)
def _project_spk_emb(
self, spk_emb: Any, already_projected: bool, use_zero_spk_emb: bool
) -> list[torch.Tensor] | None:
if spk_emb is None:
if use_zero_spk_emb:
return [torch.zeros(1, self.hidden_size, device=self.device, dtype=self.dtype)]
return None
if already_projected:
return spk_emb if isinstance(spk_emb, list) else [spk_emb]
if isinstance(spk_emb, torch.Tensor):
tensors = [spk_emb]
elif isinstance(spk_emb, list) and spk_emb and isinstance(spk_emb[0], (int, float)):
tensors = [torch.tensor(spk_emb, dtype=self.dtype).unsqueeze(0)]
elif isinstance(spk_emb, list):
tensors = spk_emb
else:
tensors = [spk_emb]
return [self.spk_head(t.to(device=self.device, dtype=self.dtype)) for t in tensors]
def _generate_latents(
self,
*,
input_ids: torch.Tensor,
inputs_embeds: torch.Tensor | None,
text: str,
params: _GenerationParams,
voice: _VoiceContext,
) -> list[torch.Tensor]:
generator = self.audio_generator
if inputs_embeds is not None:
# Caller pre-built embeddings — run a single AR pass.
return generator.generate_latents(
inputs_embeds=inputs_embeds,
prompt_wav_lat=voice.prompt_wav_lat,
max_steps=params.max_steps,
cfg=params.cfg,
sigma=params.sigma,
temperature=params.temperature,
use_static_cache=params.use_static_cache,
)
spk_emb = self._project_spk_emb(voice.spk_emb, voice.already_projected, params.use_zero_spk_emb)
text_segments = segment_and_normalize(text, max_length=params.max_text_length) if text else []
if not text_segments:
# vLLM passes 1D input_ids; Qwen2Model expects (batch, seq).
inputs_embeds = self.model.get_input_embeddings()(input_ids.to(self.device)).unsqueeze(0)
return generator.generate_latents(
inputs_embeds=inputs_embeds,
prompt_wav_lat=voice.prompt_wav_lat,
max_steps=params.max_steps,
cfg=params.cfg,
sigma=params.sigma,
temperature=params.temperature,
use_static_cache=params.use_static_cache,
)
all_latents: list[torch.Tensor] = []
for segment in text_segments:
seg_embeds, _ = build_tts_input(
tokenizer=self.tokenizer,
embed_tokens=self.model.get_input_embeddings(),
device=self.device,
dtype=torch.bfloat16,
text=segment,
prompt=params.prompt,
spk_emb=spk_emb,
instruction=params.instruction,
prompt_text=voice.prompt_text,
prompt_wav_emb=voice.prompt_wav_emb,
)
effective_max_steps = generator.duration_capped_steps(len(segment), params.max_steps)
all_latents.extend(
generator.generate_latents(
inputs_embeds=seg_embeds,
prompt_wav_lat=voice.prompt_wav_lat,
max_steps=effective_max_steps,
cfg=params.cfg,
sigma=params.sigma,
temperature=params.temperature,
use_static_cache=params.use_static_cache,
)
)
return all_latents
def _decode_to_output(self, latents: list[torch.Tensor], *, stream_decode: bool) -> OmniOutput:
multimodal_outputs: dict[str, Any] = {}
if latents and self.audio_vae is not None:
waveform = self.audio_generator.decode_to_waveform(latents, stream_decode=stream_decode)
if not stream_decode:
waveform = self.audio_generator.trim_trailing_silence(waveform)
multimodal_outputs["audio"] = waveform.detach().float().cpu()
multimodal_outputs["sr"] = torch.tensor(self.audio_vae.config.sample_rate)
elif latents:
all_lat = torch.cat(latents, dim=1)
multimodal_outputs["audio_latents"] = all_lat.detach().float().cpu()
return OmniOutput(text_hidden_states=None, multimodal_outputs=multimodal_outputs)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
"""Load weights for all talker components.
The talker's HF checkpoint (talker/model.safetensors) stores
weights with prefixes matching this module's submodule names directly.
And AudioVAE weights live in a separate file under talker/vae/
"""
# Standalone: bypass the default loader's iterator (torch.load on
# .safetensors crashes) and read talker/model*.safetensors directly.
if self._standalone:
weights = self._iter_talker_safetensors()
loader = AutoWeightsLoader(
self,
skip_prefixes=["audio_vae."], # loaded separately
skip_substrs=["rotary_embed.inv_freq"], # non-persistent buffer
)
loaded = loader.load_weights(weights)
logger.info("Loaded %d talker weights from checkpoint", len(loaded))
if self.audio_vae is not None and self._vae_weight_source is not None:
loaded.update(self._load_vae_weights())
# Register voice presets after all weights (incl. VAE) are loaded.
try:
self.voice_presets.load_presets_from_manifest(device=self.device, dtype=self.dtype)
except Exception as e: # pragma: no cover — best-effort
logger.warning("Voice preset loading failed (non-fatal): %s", e)
return loaded
def _iter_talker_safetensors(self) -> Iterable[tuple[str, torch.Tensor]]:
"""Yield (name, tensor) pairs from talker/model*.safetensors."""
model_path = self._model_path
# Try local path first
for candidate in (os.path.join(model_path, "talker"), model_path):
sf_files = sorted(glob_module.glob(os.path.join(candidate, "model*.safetensors")))
if sf_files:
for sf_path in sf_files:
yield from load_file(sf_path, device="cpu").items()
return
# HF hub fallback: download only the talker checkpoint files
model_root = download_weights_from_hf_specific(
model_path,
self.vllm_config.load_config.download_dir,
allow_patterns=["talker/model*.safetensors"],
)
talker_dir = os.path.join(model_root, "talker")
sf_files = sorted(glob_module.glob(os.path.join(talker_dir, "model*.safetensors")))
if not sf_files:
raise RuntimeError(f"No talker safetensors found under {model_root}. Expected talker/model*.safetensors.")
for sf_path in sf_files:
yield from load_file(sf_path, device="cpu").items()
def _load_vae_weights(self) -> set[str]:
"""Load AudioVAE weights from talker/vae/model.safetensors."""
if self.audio_vae is None or self._vae_weight_source is None:
return set()
# Resolve safetensors file paths from the weight source
safetensors_files: list[str] = []
source = self._vae_weight_source
if isinstance(source, str):
# Local directory path
safetensors_files = sorted(glob_module.glob(os.path.join(source, "*.safetensors")))
elif isinstance(source, tuple):
# (repo_id, subfolder) for HF hub
repo_id, subfolder = source
for filename in ("model.safetensors", "diffusion_pytorch_model.safetensors"):
try:
cached = cached_file(repo_id, filename, subfolder=subfolder)
except Exception:
cached = None
if cached is not None:
safetensors_files.append(cached)
break
if not safetensors_files:
logger.warning("No AudioVAE safetensors files found for source=%s", source)
return set()
vae_state_keys = set(self.audio_vae.state_dict().keys())
vae_loader = AutoWeightsLoader(self.audio_vae)
loaded: set[str] = set()
for sf_path in safetensors_files:
file_weights = load_file(sf_path, device="cpu")
matched = ((name, tensor) for name, tensor in file_weights.items() if name in vae_state_keys)
loaded.update(f"audio_vae.{name}" for name in vae_loader.load_weights(matched))
logger.info("Loaded %d AudioVAE weights from %s", len(loaded), source)
return loaded
@@ -0,0 +1,50 @@
# SPDX-License-Identifier: Apache-2.0
# Copyright 2025 The vLLM-Omni team.
# Copyright (c) Ant Group. All rights reserved.
# Adapted from Ming repo's usage cookbook:
# https://github.com/inclusionAI/Ming/blob/3954fcb880ff5e61ff128bcf7f1ec344d46a6fe3/cookbook.ipynb
"""Shared prompt-building helpers for Ming-flash-omni standalone talker."""
import copy
import json
from typing import Any
DEFAULT_PROMPT = "Please generate speech based on the following description.\n"
BASE_CAPTION_TEMPLATE: dict[str, Any] = {
"audio_sequence": [
{
"序号": 1,
"说话人": "speaker_1",
"方言": None,
"风格": None,
"语速": None,
"基频": None,
"音量": None,
"情感": None,
"BGM": {
"Genre": None,
"Mood": None,
"Instrument": None,
"Theme": None,
"ENV": None,
"SNR": None,
},
"IP": None,
}
]
}
def create_instruction(user_input: dict[str, Any]) -> str:
"""Return a JSON caption string for ``audio_sequence[0]``.
Only keys already present on the base template are merged in; unknown
keys are silently ignored to keep the output schema stable.
"""
caption = copy.deepcopy(BASE_CAPTION_TEMPLATE)
item = caption["audio_sequence"][0]
for key, value in user_input.items():
if key in item:
item[key] = value
return json.dumps(caption, ensure_ascii=False)
@@ -0,0 +1,44 @@
# SPDX-License-Identifier: Apache-2.0
# Copyright 2025 The vLLM-Omni team.
# Copyright (c) Ant Group. All rights reserved.
# Ported from:
# https://github.com/inclusionAI/Ming/blob/e58533db227031990c5a6864dcf5f08fb53ed0d2/modeling_bailing_talker.py
from __future__ import annotations
import torch
class SpkembExtractor:
"""CAMPPlus ONNX-based speaker embedding extractor (runs on CPU)."""
def __init__(self, campplus_model: str, target_sr: int = 16000):
import onnxruntime
import torchaudio.compliance.kaldi as kaldi
self.kaldi = kaldi
option = onnxruntime.SessionOptions()
option.graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_ENABLE_ALL
option.intra_op_num_threads = 2
self.campplus_session = onnxruntime.InferenceSession(
campplus_model, sess_options=option, providers=["CPUExecutionProvider"]
)
self.target_sr = target_sr
def _extract_spk_embedding(self, speech):
feat = self.kaldi.fbank(speech, num_mel_bins=80, dither=0, sample_frequency=16000)
feat = feat - feat.mean(dim=0, keepdim=True)
embedding = (
self.campplus_session.run(
None,
{self.campplus_session.get_inputs()[0].name: feat.unsqueeze(dim=0).cpu().numpy()},
)[0]
.flatten()
.tolist()
)
embedding = torch.tensor([embedding])
return embedding
def __call__(self, waveform, **kwargs) -> torch.Tensor | None:
spk_emb = self._extract_spk_embedding(waveform)
return spk_emb
File diff suppressed because it is too large. Load diff
@@ -0,0 +1,535 @@
# SPDX-License-Identifier: Apache-2.0
# Copyright 2025 The vLLM-Omni team.
# Copyright (c) Ant Group. All rights reserved.
# Adapted from:
# https://github.com/inclusionAI/Ming/tree/e58533db227031990c5a6864dcf5f08fb53ed0d2/front
"""Text segmentation and normalization utilities for Ming TTS."""
from __future__ import annotations
import re
import string
from vllm.logger import init_logger
logger = init_logger(__name__)
# Ported from
# https://github.com/inclusionAI/Ming/blob/e58533db227031990c5a6864dcf5f08fb53ed0d2/front/toolkit.py
_TOKENIZE_PATTERN = re.compile(r"(?:[a-zA-Z]\.)+|[a-zA-Z]+(?:['\-][a-zA-Z]+)*|\d+(?:\.\d+)?|[\u4e00-\u9fff]|\s+|\S")
def tokenize_mixed_text(text: str) -> list[str]:
return re.findall(_TOKENIZE_PATTERN, text)
def tokenize_mixed_text_iterator(text: str):
for match in _TOKENIZE_PATTERN.finditer(text):
yield match.group(0)
# Ported from
# https://github.com/inclusionAI/Ming/blob/e58533db227031990c5a6864dcf5f08fb53ed0d2/front/text_segment_cut.py
def is_chinese(text: str) -> bool:
return bool(re.search(r"[\u4e00-\u9fff]", text))
def get_semantic_length(text: str) -> int:
"""1 CJK char = 1 unit; 1 contiguous English word = 1 unit."""
chinese_char_count = len(re.findall(r"[\u4e00-\u9fa5]", text))
english_word_count = len(re.findall(r"[a-zA-Z]+", text))
return chinese_char_count + english_word_count
def has_valid_content(text: str) -> bool:
punctuation_and_whitespace = string.punctuation + string.whitespace
for char in text:
if char not in punctuation_and_whitespace:
return True
return False
def append_text_fragment(
fragments: list[str],
new_text: str,
max_len: int,
min_tail_length: int,
) -> list[str]:
new_text = new_text.lstrip(",,:;" + string.whitespace)
if not has_valid_content(new_text):
return fragments
if not fragments:
fragments.append(new_text)
return fragments
last_fragment = fragments[-1]
last_semantic_len = get_semantic_length(last_fragment)
new_semantic_len = get_semantic_length(new_text)
if last_semantic_len + new_semantic_len <= max_len:
if last_fragment.endswith(("。", "!", "?")) and new_semantic_len < min_tail_length:
fragments.append(new_text)
else:
separator = ""
if not last_fragment.endswith(" ") and re.match(r"^[a-zA-Z0-9]", new_text):
separator = " "
fragments[-1] += separator + new_text
else:
fragments.append(new_text)
return fragments
def split_long_fragment(text_fragment: str, max_len: int) -> list[str]:
if get_semantic_length(text_fragment) <= max_len:
return [text_fragment]
fragments: list[str] = []
current_fragment = ""
semantic_units = re.findall(r"([\u4e00-\u9fa5]|[a-zA-Z]+|[^a-zA-Z\u4e00-\u9fa5]+)", text_fragment)
for unit in semantic_units:
unit_len = get_semantic_length(unit)
current_len = get_semantic_length(current_fragment)
if current_len + unit_len <= max_len:
current_fragment += unit
else:
if current_fragment:
fragments.append(current_fragment)
if unit_len > max_len:
fragments.append(unit)
current_fragment = ""
else:
current_fragment = unit
if current_fragment:
fragments.append(current_fragment)
return fragments
_DOT_PLACEHOLDER = "##DOT##"
# default soft cap on fragment length in semantic units
_DEFAULT_MAX_SEMANTIC_LENGTH: int = 50
# default tail length controls when a short trailing fragment is
# merged with the previous one to avoid leaving an awkward stub.
_DEFAULT_MIN_TAIL_LENGTH: int = 5
def cut_text_by_semantic_length(
text: str,
max_semantic_length: int = _DEFAULT_MAX_SEMANTIC_LENGTH,
min_tail_length: int = _DEFAULT_MIN_TAIL_LENGTH,
) -> list[str]:
"""Segment text into fragments respecting semantic length limits.
Ported from upstream Ming's `front/text_segment_cut.py`.
Position tracking is omitted (not needed for non-streaming VAE decode).
"""
if not has_valid_content(text):
return []
processed = re.sub(r"(\d)\.(\d)", r"\1" + _DOT_PLACEHOLDER + r"\2", text)
for _ in range(3):
processed = re.sub(r"([A-Z])\.([A-Z])", r"\1" + _DOT_PLACEHOLDER + r"\2", processed)
processed = processed.replace("\n", " ").replace("。,", "。")
if get_semantic_length(processed) <= max_semantic_length:
return [processed.replace(_DOT_PLACEHOLDER, ".")]
normalized = processed.replace(".", "。").replace("!", "!").replace("?", "?").replace(",", ",")
# Phase 1: split into sentences on 。!?
sentences: list[str] = []
current: list[str] = []
for char in normalized:
current.append(char)
if char in "。!?":
s = "".join(current).strip()
if s:
sentences.append(s)
current = []
if current:
s = "".join(current).strip()
if s:
if not s.endswith(("。", "!", "?")):
s += "。"
sentences.append(s)
# Phase 2: merge whole sentences; only clause-split oversized ones.
# This ensures split points land on sentence boundaries (。!?)
# rather than mid-sentence commas.
result_fragments: list[str] = []
for sentence in sentences:
sent_len = get_semantic_length(sentence)
if sent_len > max_semantic_length:
# Oversized sentence: fall back to clause-level splitting
clauses: list[str] = []
clause_buf: list[str] = []
for char in sentence:
clause_buf.append(char)
if char in ",;;":
cl = "".join(clause_buf).strip()
if cl and has_valid_content(cl):
clauses.append(cl)
elif cl and clauses:
clauses[-1] += cl
clause_buf = []
if clause_buf:
cl = "".join(clause_buf).strip()
if cl and has_valid_content(cl):
clauses.append(cl)
elif cl and clauses:
clauses[-1] += cl
i = 0
while i < len(clauses):
clause = clauses[i]
clause_len = get_semantic_length(clause)
if clause_len < min_tail_length and i + 1 < len(clauses):
combined = clause + clauses[i + 1]
if get_semantic_length(combined) <= max_semantic_length:
result_fragments = append_text_fragment(
result_fragments, combined, max_semantic_length, min_tail_length
)
i += 2
continue
if clause_len > max_semantic_length:
for frag in split_long_fragment(clause, max_semantic_length):
result_fragments = append_text_fragment(
result_fragments, frag, max_semantic_length, min_tail_length
)
else:
result_fragments = append_text_fragment(
result_fragments, clause, max_semantic_length, min_tail_length
)
i += 1
else:
# Normal sentence: merge at sentence level
if not result_fragments:
result_fragments.append(sentence)
else:
last_len = get_semantic_length(result_fragments[-1])
if last_len + sent_len <= max_semantic_length:
result_fragments[-1] += sentence
else:
result_fragments.append(sentence)
return [f.replace(_DOT_PLACEHOLDER, ".") for f in result_fragments]
# Streaming sentence boundary detection
_RE_CJK = re.compile(r"[\u4e00-\u9fff]")
_RE_DIGIT_LAST = re.compile(r"[0-9]")
# Left for reference for now
# Alternative to `cut_text_by_semantic_length`
def detect_sentence_boundaries(
text: str,
max_length: int = 50,
) -> list[str]:
"""Accumulate tokens and flush at sentence boundaries.
Ported from the streaming sentence detection loop from the Ming repo
TTS branch, but operates on the full text since we have it available upfront.
"""
sentences: list[str] = []
streaming_text: list[str] = []
count = 0
for ele in tokenize_mixed_text_iterator(text):
if len(ele) == 0:
continue
should_process = False
min_tokens = 12 if count == 0 else 8
if ele[-1] in "!?。,!?":
if len(streaming_text) >= min_tokens:
should_process = True
streaming_text.append(ele)
elif ele[-1] == ".":
if (
len(streaming_text) >= min_tokens
and streaming_text
and not _RE_DIGIT_LAST.search(streaming_text[-1][-1])
):
should_process = True
streaming_text.append(ele)
elif ele[-1] == "\n":
if streaming_text:
joined = "".join(streaming_text)
if _RE_CJK.search(joined):
if _RE_CJK.search(streaming_text[-1][-1]):
ele = ","
streaming_text.append(ele)
else:
if len(ele) > 1 and re.search(r"[a-zA-Z]", ele[-2]):
ele = ele[:-1] + "."
else:
ele = ele[:-1]
streaming_text.append(ele)
if len(streaming_text) >= min_tokens:
should_process = True
else:
streaming_text.append(ele)
continue
if should_process:
joined = "".join(streaming_text)
fragments = cut_text_by_semantic_length(joined, max_length)
sentences.extend(fragments)
streaming_text = []
count += 1
# Flush remaining
if streaming_text and re.search(r"[a-zA-Z\u4e00-\u9fff1-9]", "".join(streaming_text)):
joined = "".join(streaming_text)
fragments = cut_text_by_semantic_length(joined, max_length)
sentences.extend(fragments)
return sentences
# number normalization for English. Ported from
# https://github.com/inclusionAI/Ming/blob/e58533db227031990c5a6864dcf5f08fb53ed0d2/front/number_en.py
_inflect_engine = None
def _get_inflect():
global _inflect_engine
if _inflect_engine is not None:
return _inflect_engine
try:
import inflect
_inflect_engine = inflect.engine()
except ImportError:
logger.warning(
"Package 'inflect' not installed - English number normalization "
"will be skipped. Install with: pip install inflect"
)
_inflect_engine = None
return _inflect_engine
_comma_number_re = re.compile(r"([0-9][0-9,]+[0-9])")
_percent_number_re = re.compile(r"(-?[0-9.,]*[0-9]+)%")
_pounds_re = re.compile(r"£(-?[0-9,]*[0-9]+(?:\.[0-9]+)?)")
_dollars_re = re.compile(r"\$(-?[0-9.,]*[0-9]+(?:\.[0-9]+)?)")
_fraction_re = re.compile(r"([0-9]+)\/([0-9]+)")
_ordinal_re = re.compile(r"\b[0-9]+(st|[nr]d|th)\b")
_number_re = re.compile(r"\b-?[0-9]+(?:\.[0-9]+)?\b")
_unit_re = re.compile(
r"\b(-?\d+(?:\.\d+)?)\s*"
r"(ms|s|Hz|kHz|MHz|GHz|kb|mb|gb|tb|KB|MB|GB|TB|bps|kbps|Mbps|Gbps|cm|km|kg|V|A|W|°C|°F)\b",
re.IGNORECASE,
)
_version_re = re.compile(r"\b([a-zA-Z]+)([-]?)([0-9]+(?:\.[0-9]+)?)\b")
_whitespace_re = re.compile(r"\s+")
_unit_mapping = {
"ms": "milliseconds",
"s": "seconds",
"hz": "hertz",
"khz": "kilohertz",
"mhz": "megahertz",
"ghz": "gigahertz",
"kb": "kilobytes",
"mb": "megabytes",
"gb": "gigabytes",
"tb": "terabytes",
"kbps": "kilobits per second",
"mbps": "megabits per second",
"gbps": "gigabits per second",
"bps": "bits per second",
"cm": "centimeters",
"km": "kilometers",
"kg": "kilograms",
"v": "volts",
"a": "amperes",
"w": "watts",
"°c": "degrees celsius",
"°f": "degrees fahrenheit",
}
def _num_to_words(n: int) -> str:
p = _get_inflect()
if p is None:
return str(n)
return p.number_to_words(n, andword="")
def _expand_decimal(num_str: str) -> str:
"""Expand a decimal number string like '3.14' -> 'three point one four'."""
is_negative = num_str.startswith("-")
clean = num_str.lstrip("-") or "0"
if "." in clean:
parts = clean.split(".", 1)
integer_part = parts[0] or "0"
decimal_part = parts[1]
if not integer_part.isdigit() or not decimal_part.isdigit():
return num_str
int_word = _num_to_words(int(integer_part)) if integer_part != "0" else "zero"
dec_words = " ".join(_num_to_words(int(d)) for d in decimal_part if d.isdigit())
word = f"{int_word} point {dec_words}"
else:
if not clean.isdigit():
return num_str
word = _num_to_words(int(clean))
if is_negative:
word = f"minus {word}"
return word
def _remove_commas(m: re.Match) -> str:
return m.group(1).replace(",", "")
_NUM_PARSE_EXC: tuple[type[BaseException], ...] = (ValueError, TypeError)
def _expand_unit(m: re.Match) -> str:
num_str, unit = m.group(1), m.group(2).lower()
unit_word = _unit_mapping.get(unit, unit)
try:
return f" {_expand_decimal(num_str)} {unit_word} "
except _NUM_PARSE_EXC:
return f" {num_str} {unit} "
def _expand_percent(m: re.Match) -> str:
try:
return f" {_expand_decimal(m.group(1))} percent "
except _NUM_PARSE_EXC:
return f" {m.group(1)} percent "
def _expand_dollars(m: re.Match) -> str:
raw = m.group(1)
clean = raw.lstrip("-") or "0"
try:
word = _expand_decimal(raw)
value = float(clean)
unit = "dollar" if abs(value) == 1.0 else "dollars"
return f" {word} {unit} "
except _NUM_PARSE_EXC:
return f" {clean} dollars "
def _expand_pounds(m: re.Match) -> str:
raw = m.group(1)
clean = raw.lstrip("-") or "0"
try:
word = _expand_decimal(raw)
value = float(clean)
unit = "pound" if abs(value) == 1.0 else "pounds"
return f" {word} {unit} "
except _NUM_PARSE_EXC:
return f" {clean} pounds "
def _expand_fraction(m: re.Match) -> str:
p = _get_inflect()
if p is None:
return m.group(0)
try:
num, den = int(m.group(1)), int(m.group(2))
if num == 1 and den == 2:
return " one half "
if num == 1 and den == 4:
return " one quarter "
if den == 2:
plural = " half" if num == 1 else " halves"
return f" {p.number_to_words(num)}{plural} "
if den == 4:
plural = " quarter" if num == 1 else " quarters"
return f" {p.number_to_words(num)}{plural} "
ordinal = p.ordinal(p.number_to_words(den))
return f" {p.number_to_words(num)} {ordinal} "
except _NUM_PARSE_EXC:
return f" {m.group(1)} over {m.group(2)} "
def _expand_ordinal(m: re.Match) -> str:
try:
num = int(re.sub(r"(st|and|rd|th)", "", m.group(0)))
return f" {_num_to_words(num)} "
except _NUM_PARSE_EXC:
return m.group(0)
def _expand_number(m: re.Match) -> str:
try:
return f" {_expand_decimal(m.group(0))} "
except _NUM_PARSE_EXC:
return f" {m.group(0)} "
def _expand_version(m: re.Match) -> str:
prefix, _, num_str = m.group(1), m.group(2), m.group(3)
try:
word = _expand_decimal(num_str)
except _NUM_PARSE_EXC:
return m.group(0)
return f"{prefix} {word}"
def normalize_numbers(text: str) -> str:
"""Expand English numbers, currencies, units, etc. to words.
Returns text unchanged if `inflect` package is not installed.
"""
if _get_inflect() is None:
return text
text = re.sub(_comma_number_re, _remove_commas, text)
text = re.sub(_unit_re, _expand_unit, text)
text = re.sub(_pounds_re, _expand_pounds, text)
text = re.sub(_dollars_re, _expand_dollars, text)
text = re.sub(_fraction_re, _expand_fraction, text)
text = re.sub(_percent_number_re, _expand_percent, text)
text = re.sub(_ordinal_re, _expand_ordinal, text)
text = re.sub(_version_re, _expand_version, text)
text = re.sub(_number_re, _expand_number, text)
text = re.sub(_whitespace_re, " ", text)
return text.strip()
# Top-level API
def segment_and_normalize(
text: str,
max_length: int = _DEFAULT_MAX_SEMANTIC_LENGTH,
) -> list[str]:
"""Segment text into fragments and expand English numbers for Ming TTS.
This function cuts text by semantic length directly rather than following
the streaming algorithm to detect sentence boundaries in the upstream
Ming repo (which is more aggressively splitting at commas). It produces
fewer and larger segments at natural sentence boundaries.
"""
if not text or not text.strip():
return []
segments = cut_text_by_semantic_length(text.strip(), max_length)
normalized: list[str] = []
for seg in segments:
if not is_chinese(seg):
seg = normalize_numbers(seg)
if seg and seg[0] == ",":
seg = seg[1:]
seg = seg.strip()
if seg:
normalized.append(seg)
return normalized if normalized else [text.strip()]
@@ -0,0 +1,289 @@
# SPDX-License-Identifier: Apache-2.0
# Copyright 2025 The vLLM-Omni team.
# Copyright (c) Ant Group. All rights reserved.
# Adapted from:
# https://github.com/inclusionAI/Ming/blob/e58533db227031990c5a6864dcf5f08fb53ed0d2/modeling_bailing_talker.py
from __future__ import annotations
import json
import os
from functools import cached_property
from typing import TYPE_CHECKING, Any
import soundfile as sf
import torch
from transformers.utils.hub import cached_file
from vllm.logger import init_logger
from vllm_omni.model_executor.model_loader.weight_utils import download_weights_from_hf_specific
from .spk_embedding import SpkembExtractor
from .talker_module import resample
if TYPE_CHECKING:
from .audio_vae import AudioVAE
from .talker_module import Aggregator
logger = init_logger(__name__)
class InvalidPromptWavError(ValueError):
"""Prompt wav failed local validation and can be skipped in list mode."""
class VoicePresetRegistry:
"""Loader and registry for Ming voice presets."""
def __init__(
self,
*,
talker_dir: str,
model_path: str,
download_dir: str | None,
audio_vae: AudioVAE | None,
aggregator: Aggregator,
spk_head: torch.nn.Module,
patch_size: int,
) -> None:
self._talker_dir = talker_dir
self._model_path = model_path
self._download_dir = download_dir
self._audio_vae = audio_vae
self._aggregator = aggregator
self._spk_head = spk_head
self._patch_size = patch_size
self.registered: dict[str, dict[str, Any]] = {}
def __contains__(self, voice_name: str) -> bool:
return voice_name in self.registered
def get(self, voice_name: str) -> dict[str, Any] | None:
return self.registered.get(voice_name)
def register(
self,
voice_name: str,
prompt_wav_path: str | list[str],
*,
device: torch.device,
dtype: torch.dtype,
) -> None:
"""Register a voice preset from one or more reference wav files.
Args:
voice_name: Key under which to store the preset.
prompt_wav_path: Single wav path or a list (multi-clip mode skips
invalid entries with a warning instead of raising).
device: Target device for cached prompt latents / projected
speaker embeddings.
dtype: Target dtype for the projected speaker embedding head.
"""
paths = self._normalize_paths(voice_name, prompt_wav_path)
allow_partial = len(paths) > 1
vae_sr = int(self._audio_vae.config.sample_rate) if self._audio_vae else 44100
if self._audio_vae is None:
logger.warning(
"Voice preset '%s' being registered without AudioVAE features",
voice_name,
)
speech_chunks: list[torch.Tensor] = []
spk_emb_list: list[torch.Tensor] = []
for wav_path in paths:
try:
speech_for_vae, raw_emb = self._load_single_wav(voice_name, wav_path, vae_sr)
except (FileNotFoundError, InvalidPromptWavError) as e:
if allow_partial:
logger.warning(
"Voice preset '%s': skipping invalid prompt wav %s: %s",
voice_name,
wav_path,
e,
)
continue
raise
speech_chunks.append(speech_for_vae)
if raw_emb is not None:
projected = self._spk_head(raw_emb.to(device=device, dtype=dtype))
spk_emb_list.append(projected)
if not speech_chunks:
raise RuntimeError(f"Failed to register voice preset '{voice_name}': no valid prompt wavs remained")
if not spk_emb_list and self._audio_vae is None:
raise RuntimeError(
f"Failed to register voice preset '{voice_name}': neither speaker "
"embeddings nor AudioVAE prompt features are available"
)
prompt_wav_lat, prompt_wav_emb = self._build_wav_embeddings(
voice_name, torch.cat(speech_chunks, dim=-1), device=device
)
if voice_name in self.registered:
logger.warning("Voice preset '%s' is being overwritten", voice_name)
self.registered[voice_name] = {
"prompt_wav_lat": prompt_wav_lat,
"prompt_wav_emb": prompt_wav_emb,
"spk_emb": spk_emb_list,
}
logger.info("Registered voice preset '%s' from %s", voice_name, paths)
def load_presets_from_manifest(self, *, device: torch.device, dtype: torch.dtype) -> None:
"""Resolve voice_name.json on disk or HF hub and register all entries.
Each entry is registered onto the supplied device and dtype.
"""
voice_json_path, base_dir = self._locate_manifest()
if voice_json_path is None:
logger.info("No voice_name.json found; voice presets unavailable")
return
with open(voice_json_path) as f:
voice_dict = json.load(f)
for name, info in voice_dict.items():
wav_path = info.get("prompt_wav_path", "")
prompt_text = info.get("prompt_text", "")
if not wav_path:
logger.warning("Voice preset '%s' has no prompt_wav_path, skipping", name)
continue
if not os.path.isabs(wav_path):
wav_path = os.path.join(base_dir, wav_path)
if not os.path.isfile(wav_path):
logger.warning("Voice preset '%s': wav not found at %s, skipping", name, wav_path)
continue
try:
self.register(name, wav_path, device=device, dtype=dtype)
self.registered[name]["prompt_text"] = prompt_text
except Exception as e: # pragma: no cover — manifest is best-effort
logger.warning("Failed to register voice preset '%s': %s", name, e)
@cached_property
def _spkemb_extractor(self) -> SpkembExtractor:
"""Lazily resolve the CAMPPlus ONNX extractor."""
for candidate in (self._talker_dir, self._model_path):
path = os.path.join(candidate, "campplus.onnx")
if os.path.isfile(path):
extractor = SpkembExtractor(path)
logger.info("Initialized SpkembExtractor from %s", path)
return extractor
try:
path = cached_file(self._model_path, "campplus.onnx", subfolder="talker")
except Exception as e:
raise RuntimeError("campplus.onnx not found. Expected at <model_path>/talker/campplus.onnx") from e
extractor = SpkembExtractor(path)
logger.info("Initialized SpkembExtractor from %s", path)
return extractor
@staticmethod
def _normalize_paths(voice_name: str, prompt_wav_path: str | list[str]) -> list[str]:
if not isinstance(voice_name, str) or not voice_name.strip():
raise ValueError("voice_name must be a non-empty string")
if isinstance(prompt_wav_path, str):
paths = [prompt_wav_path]
elif isinstance(prompt_wav_path, list):
paths = list(prompt_wav_path)
else:
raise TypeError("prompt_wav_path must be a string path or a list of string paths")
paths = [p.strip() for p in paths]
if not paths or any(not p for p in paths):
raise ValueError("Provided audio path is invalid")
return paths
def _load_single_wav(self, voice_name: str, wav_path: str, vae_sr: int) -> tuple[torch.Tensor, torch.Tensor | None]:
"""Return ``(speech_for_vae, raw_spk_emb_or_none)``.
Stays device-agnostic — both returned tensors live on CPU; the caller
moves them to the target device when projecting / encoding.
"""
if not os.path.isfile(wav_path):
raise FileNotFoundError(f"prompt wav not found: {wav_path}")
data, sample_rate = sf.read(wav_path, dtype="float32")
speech_tmp = torch.from_numpy(data)
if speech_tmp.ndim == 1:
speech_tmp = speech_tmp.unsqueeze(0)
elif speech_tmp.ndim == 2:
num_channels = speech_tmp.shape[1]
if num_channels > 1:
logger.warning(
"Voice preset '%s': downmixing %d-channel audio at %s to mono",
voice_name,
num_channels,
wav_path,
)
speech_tmp = speech_tmp.mean(dim=1, keepdim=True).T
else:
raise InvalidPromptWavError(f"unsupported audio shape {tuple(speech_tmp.shape)} for {wav_path}")
if not torch.isfinite(speech_tmp).all():
raise InvalidPromptWavError(f"audio file contains NaN or Inf samples: {wav_path}")
speech_for_vae = resample(speech_tmp, sample_rate, vae_sr)
# Speaker embedding (16 kHz CAMPPlus). If the extractor fails to
# resolve (missing ONNX model), skip embedding extraction rather than
# blocking VAE-only registration.
raw_emb: torch.Tensor | None = None
try:
extractor = self._spkemb_extractor
speech_for_spk = resample(speech_tmp, sample_rate, 16000)
raw_emb = extractor(speech_for_spk)
except RuntimeError:
raw_emb = None
return speech_for_vae, raw_emb
def _build_wav_embeddings(
self,
voice_name: str,
speech: torch.Tensor,
*,
device: torch.device,
) -> tuple[torch.Tensor | None, torch.Tensor | None]:
if self._audio_vae is None:
return None, None
patch_pt = self._audio_vae.encoder.hop_size * max(1, self._audio_vae.encoder.patch_size) * self._patch_size
if speech.shape[-1] % patch_pt != 0:
pad_len = (speech.shape[-1] + patch_pt - 1) // patch_pt * patch_pt
pad_speech = torch.zeros((speech.shape[0], pad_len), dtype=speech.dtype, device=speech.device)
pad_speech[:, -speech.shape[-1] :] = speech
speech = pad_speech
prompt_wav_lat, _ = self._audio_vae.encode_latent(
speech.to(dtype=torch.bfloat16, device=device),
torch.tensor([speech.size(1)], dtype=torch.long, device=device),
)
assert prompt_wav_lat.shape[1] % self._patch_size == 0, (
f"AudioVAE latent length is incompatible with patch_size for voice preset '{voice_name}'"
)
prompt_wav_lat = prompt_wav_lat.reshape(-1, self._patch_size, prompt_wav_lat.shape[-1])
prompt_wav_emb = self._aggregator(prompt_wav_lat)
prompt_wav_lat = prompt_wav_lat.reshape(1, -1, prompt_wav_lat.shape[-1])
prompt_wav_emb = prompt_wav_emb.reshape(1, -1, prompt_wav_emb.shape[-1])
return prompt_wav_lat, prompt_wav_emb
def _locate_manifest(self) -> tuple[str | None, str | None]:
for candidate in (self._talker_dir, self._model_path):
path = os.path.join(candidate, "data", "voice_name.json")
if os.path.isfile(path):
return path, candidate
if not os.path.isdir(self._model_path):
try:
hf_root = download_weights_from_hf_specific(
self._model_path,
self._download_dir,
allow_patterns=["talker/data/**"],
require_all=True,
)
candidate = os.path.join(hf_root, "talker", "data", "voice_name.json")
if os.path.isfile(candidate):
return candidate, os.path.join(hf_root, "talker")
except Exception as e: # pragma: no cover
logger.info("Could not download voice presets from HF: %s", e)
return None, None
@@ -185,6 +185,11 @@ _OMNI_MODELS = {
"ming_flash_omni_thinker",
"MingFlashOmniThinkerForConditionalGeneration",
),
"MingFlashOmniTalkerForConditionalGeneration": (
"ming_flash_omni",
"ming_flash_omni_talker",
"MingFlashOmniTalkerForConditionalGeneration",
),
# Alias: HF repo currently ships this architecture name in config.json
"BailingMM2NativeForConditionalGeneration": (
"ming_flash_omni",