[Model] Ming-flash-omni-2.0 Omni-Speech and TTS (#2890)
Signed-off-by: Yuanheng Zhao <jonathan.zhaoyh@gmail.com> Signed-off-by: yuanheng <jonathan.zhaoyh@gmail.com> Signed-off-by: LHXuuu <xulianhao.xlh@antgroup.com> Co-authored-by: LHXuuu <xulianhao.xlh@antgroup.com> Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
This commit is contained in:
32 files changed
+4620
-261
No files matched your search
@@ -2,6 +2,7 @@
|
||||
# Copyright 2025 The vLLM-Omni team.
|
||||
|
||||
from .ming_flash_omni import MingFlashOmniForConditionalGeneration
|
||||
from .ming_flash_omni_talker import MingFlashOmniTalkerForConditionalGeneration
|
||||
from .ming_flash_omni_thinker import (
|
||||
MingFlashOmniThinkerDummyInputsBuilder,
|
||||
MingFlashOmniThinkerForConditionalGeneration,
|
||||
@@ -11,6 +12,7 @@ from .ming_flash_omni_thinker import (
|
||||
|
||||
__all__ = [
|
||||
"MingFlashOmniForConditionalGeneration",
|
||||
"MingFlashOmniTalkerForConditionalGeneration",
|
||||
"MingFlashOmniThinkerForConditionalGeneration",
|
||||
"MingFlashOmniThinkerProcessingInfo",
|
||||
"MingFlashOmniThinkerMultiModalProcessor",
|
||||
|
||||
@@ -0,0 +1,390 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Copyright 2025 The vLLM-Omni team.
|
||||
# Copyright (c) Ant Group. All rights reserved.
|
||||
# Adapted from:
|
||||
# https://github.com/inclusionAI/Ming/tree/e58533db227031990c5a6864dcf5f08fb53ed0d2/AudioVAE
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
from transformers import PretrainedConfig, PreTrainedModel, Qwen2Config, Qwen2Model
|
||||
from vllm.logger import init_logger
|
||||
|
||||
logger = init_logger(__name__)
|
||||
try:
|
||||
import flash_attn # noqa: F401
|
||||
except (ImportError, ModuleNotFoundError):
|
||||
flash_attn = None
|
||||
logger.warning(
|
||||
"flash_attn is not available, the model may not yield the "
|
||||
"exactly same result as the transformers implementation "
|
||||
"in the audio tower part."
|
||||
)
|
||||
|
||||
|
||||
class AudioVAEConfig(PretrainedConfig):
|
||||
model_type = "audio_vae"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
sample_rate: int = 44100,
|
||||
enc_kwargs: dict | None = None,
|
||||
dec_kwargs: dict | None = None,
|
||||
init_method: str = "kaiming",
|
||||
patch_size: int = 4,
|
||||
**kwargs,
|
||||
):
|
||||
self.sample_rate = sample_rate
|
||||
self.enc_kwargs = enc_kwargs or {}
|
||||
self.dec_kwargs = dec_kwargs or {}
|
||||
self.init_method = init_method
|
||||
self.patch_size = patch_size
|
||||
super().__init__(**kwargs)
|
||||
|
||||
|
||||
class ISTFT(nn.Module):
|
||||
def __init__(self, n_fft: int, hop_length: int, win_length: int, padding: str = "same"):
|
||||
super().__init__()
|
||||
if padding not in ["center", "same"]:
|
||||
raise ValueError("Padding must be 'center' or 'same'.")
|
||||
self.padding = padding
|
||||
self.n_fft = n_fft
|
||||
self.hop_length = hop_length
|
||||
self.win_length = win_length
|
||||
window = torch.hann_window(win_length)
|
||||
self.register_buffer("window", window)
|
||||
self.buffer_len = self.win_length - self.hop_length
|
||||
|
||||
def _buffer_process(self, x, buffer, pad, last_chunk=False, streaming=False):
|
||||
if streaming:
|
||||
if buffer is None:
|
||||
x = x[:, pad:]
|
||||
if buffer is not None:
|
||||
x[:, : self.buffer_len] += buffer
|
||||
buffer = x[:, -self.buffer_len :]
|
||||
if not last_chunk:
|
||||
x = x[:, : -self.buffer_len]
|
||||
else:
|
||||
x = x[:, :-pad]
|
||||
else:
|
||||
x = x[:, pad:-pad]
|
||||
return x, buffer
|
||||
|
||||
def forward(self, spec, audio_buffer=None, window_buffer=None, streaming=False, last_chunk=False):
|
||||
if self.padding == "center":
|
||||
return torch.istft(spec, self.n_fft, self.hop_length, self.win_length, self.window, center=True)
|
||||
elif self.padding == "same":
|
||||
pad = (self.win_length - self.hop_length) // 2
|
||||
else:
|
||||
raise ValueError("Padding must be 'center' or 'same'.")
|
||||
|
||||
B, N, T = spec.shape
|
||||
ifft = torch.fft.irfft(spec, self.n_fft, dim=1, norm="backward")
|
||||
ifft = ifft * self.window[None, :, None]
|
||||
|
||||
output_size = (T - 1) * self.hop_length + self.win_length
|
||||
y = torch.nn.functional.fold(
|
||||
ifft,
|
||||
output_size=(1, output_size),
|
||||
kernel_size=(1, self.win_length),
|
||||
stride=(1, self.hop_length),
|
||||
)[:, 0, 0, :]
|
||||
|
||||
y, audio_buffer = self._buffer_process(y, audio_buffer, pad, last_chunk=last_chunk, streaming=streaming)
|
||||
|
||||
window_sq = self.window.square().expand(1, T, -1).transpose(1, 2)
|
||||
window_envelope = (
|
||||
torch.nn.functional.fold(
|
||||
window_sq,
|
||||
output_size=(1, output_size),
|
||||
kernel_size=(1, self.win_length),
|
||||
stride=(1, self.hop_length),
|
||||
)
|
||||
.squeeze(0)
|
||||
.squeeze(0)
|
||||
)
|
||||
|
||||
window_envelope, window_buffer = self._buffer_process(
|
||||
window_envelope, window_buffer, pad, last_chunk=last_chunk, streaming=streaming
|
||||
)
|
||||
window_envelope = window_envelope.squeeze()
|
||||
|
||||
assert (window_envelope > 1e-11).all()
|
||||
y = y / window_envelope
|
||||
|
||||
return y, audio_buffer, window_buffer
|
||||
|
||||
|
||||
class ISTFTHead(nn.Module):
|
||||
def __init__(self, dim: int, n_fft: int, hop_length: int, padding: str = "same"):
|
||||
super().__init__()
|
||||
out_dim = n_fft + 2
|
||||
self.out = nn.Linear(dim, out_dim)
|
||||
self.istft = ISTFT(n_fft=n_fft, hop_length=hop_length, win_length=n_fft, padding=padding)
|
||||
|
||||
def forward(self, x, audio_buffer=None, window_buffer=None, streaming=False, last_chunk=False):
|
||||
x_pred = self.out(x)
|
||||
x_pred = x_pred.transpose(1, 2)
|
||||
mag, p = x_pred.chunk(2, dim=1)
|
||||
mag = torch.exp(mag)
|
||||
mag = torch.clip(mag, max=1e2)
|
||||
x = torch.cos(p)
|
||||
y = torch.sin(p)
|
||||
S = mag * (x + 1j * y)
|
||||
audio, audio_buffer, window_buffer = self.istft(
|
||||
S, audio_buffer=audio_buffer, window_buffer=window_buffer, streaming=streaming, last_chunk=last_chunk
|
||||
)
|
||||
return audio.unsqueeze(1), x_pred, audio_buffer, window_buffer
|
||||
|
||||
|
||||
class StreamingLinearUpsample(nn.Module):
|
||||
def __init__(self, scale_factor=4):
|
||||
super().__init__()
|
||||
self.scale_factor = scale_factor
|
||||
self.upsampler = nn.Upsample(scale_factor=scale_factor, mode="linear", align_corners=False)
|
||||
|
||||
def forward(self, x, state=None, is_last=False):
|
||||
if state is None:
|
||||
state = {"prev_chunk": None, "history_last": None, "is_first": True}
|
||||
|
||||
if x is None and not is_last:
|
||||
return None, state
|
||||
|
||||
if state["is_first"] and is_last:
|
||||
out = self.upsampler(x.transpose(1, 2)).transpose(1, 2)
|
||||
return out, None
|
||||
|
||||
output_chunks = []
|
||||
|
||||
if state["is_first"]:
|
||||
state["prev_chunk"] = x
|
||||
state["is_first"] = False
|
||||
if not is_last:
|
||||
return None, state
|
||||
|
||||
if state["prev_chunk"] is not None:
|
||||
p = state["prev_chunk"].transpose(1, 2)
|
||||
|
||||
if state["history_last"] is None:
|
||||
lookahead = x[:, :1, :].transpose(1, 2)
|
||||
inp = torch.cat([p, lookahead], dim=2)
|
||||
up = self.upsampler(inp)
|
||||
out_prev = up[:, :, : p.size(2) * self.scale_factor]
|
||||
else:
|
||||
lookahead = x[:, :1, :].transpose(1, 2)
|
||||
inp = torch.cat([state["history_last"], p, lookahead], dim=2)
|
||||
up = self.upsampler(inp)
|
||||
start = self.scale_factor
|
||||
end = start + p.size(2) * self.scale_factor
|
||||
out_prev = up[:, :, start:end]
|
||||
|
||||
output_chunks.append(out_prev.transpose(1, 2))
|
||||
state["history_last"] = p[:, :, -1:]
|
||||
state["prev_chunk"] = x
|
||||
|
||||
if is_last:
|
||||
p = state["prev_chunk"].transpose(1, 2)
|
||||
inp = torch.cat([state["history_last"], p], dim=2)
|
||||
up = self.upsampler(inp)
|
||||
out_last = up[:, :, self.scale_factor :]
|
||||
output_chunks.append(out_last.transpose(1, 2))
|
||||
state = None
|
||||
|
||||
final_out = torch.cat(output_chunks, dim=1) if output_chunks else None
|
||||
return final_out, state
|
||||
|
||||
|
||||
class Decoder(nn.Module):
|
||||
def __init__(self, decoder_args, output_dim=320, latent_dim=64, patch_size=-1):
|
||||
super().__init__()
|
||||
config = Qwen2Config.from_dict(config_dict=decoder_args)
|
||||
if flash_attn is None:
|
||||
config._attn_implementation = "sdpa"
|
||||
self.decoder = Qwen2Model(config)
|
||||
self.output_dim = output_dim
|
||||
self.latent_dim = latent_dim
|
||||
self.fc1 = nn.Linear(latent_dim, config.hidden_size)
|
||||
self.hop_length = output_dim
|
||||
self.head = ISTFTHead(
|
||||
dim=config.hidden_size, n_fft=self.hop_length * 4, hop_length=self.hop_length, padding="same"
|
||||
)
|
||||
self.patch_size = patch_size
|
||||
if self.patch_size != -1:
|
||||
self.upsampling = StreamingLinearUpsample(scale_factor=patch_size)
|
||||
|
||||
def low_level_reconstruct(self, x, past_key_values=None, use_cache=False, stream_state=None, last_chunk=False):
|
||||
upsample_state, audio_buffer, window_buffer = stream_state
|
||||
bsz, device, dtype = x.size(0), x.device, x.dtype
|
||||
x = self.fc1(x)
|
||||
if self.patch_size != -1:
|
||||
if use_cache:
|
||||
x, upsample_state = self.upsampling(x, state=upsample_state, is_last=last_chunk)
|
||||
if x is None:
|
||||
stream_state = (upsample_state, audio_buffer, window_buffer)
|
||||
return torch.empty(bsz, 1, 0, device=device, dtype=dtype), stream_state, past_key_values
|
||||
else:
|
||||
x = self.upsampling.upsampler(x.transpose(1, 2)).transpose(1, 2)
|
||||
|
||||
hidden_states_list = []
|
||||
|
||||
if use_cache and getattr(self.decoder.config, "sliding_window", None) is not None:
|
||||
sw_size = self.decoder.config.sliding_window
|
||||
target_len = sw_size - 1
|
||||
if past_key_values is None:
|
||||
past_len = 0
|
||||
elif hasattr(past_key_values, "get_seq_length"):
|
||||
past_len = past_key_values.get_seq_length()
|
||||
elif isinstance(past_key_values, tuple) and len(past_key_values) > 0:
|
||||
past_len = past_key_values[0][0].shape[-2]
|
||||
else:
|
||||
past_len = 0
|
||||
|
||||
curr_len = x.shape[1]
|
||||
|
||||
if past_len < target_len and (past_len + curr_len) >= sw_size:
|
||||
fill_len = target_len - past_len
|
||||
x_fill = x[:, :fill_len, :]
|
||||
outputs = self.decoder(inputs_embeds=x_fill, past_key_values=past_key_values, use_cache=use_cache)
|
||||
hidden_states_list.append(outputs.last_hidden_state)
|
||||
past_key_values = outputs.past_key_values
|
||||
x = x[:, fill_len:, :]
|
||||
|
||||
outputs = self.decoder(inputs_embeds=x, past_key_values=past_key_values, use_cache=use_cache)
|
||||
hidden_states_list.append(outputs.last_hidden_state)
|
||||
past_key_values = outputs.past_key_values
|
||||
|
||||
if len(hidden_states_list) > 1:
|
||||
full_hidden_state = torch.cat(hidden_states_list, dim=1)
|
||||
else:
|
||||
full_hidden_state = hidden_states_list[0]
|
||||
|
||||
x_out, _, audio_buffer, window_buffer = self.head(
|
||||
full_hidden_state,
|
||||
streaming=use_cache,
|
||||
audio_buffer=audio_buffer,
|
||||
window_buffer=window_buffer,
|
||||
last_chunk=last_chunk,
|
||||
)
|
||||
|
||||
stream_state = (upsample_state, audio_buffer, window_buffer)
|
||||
return x_out, stream_state, past_key_values
|
||||
|
||||
|
||||
class Encoder(nn.Module):
|
||||
def __init__(self, encoder_args, input_dim=320, hop_size=320, latent_dim=64, patch_size=-1):
|
||||
super().__init__()
|
||||
config = Qwen2Config.from_dict(config_dict=encoder_args)
|
||||
if flash_attn is None:
|
||||
config._attn_implementation = "sdpa"
|
||||
self.encoder = Qwen2Model(config)
|
||||
self.input_dim = input_dim
|
||||
self.hop_size = hop_size
|
||||
self.latent_dim = latent_dim
|
||||
self.fc1 = nn.Linear(input_dim, config.hidden_size, bias=False)
|
||||
self.fc2 = nn.Linear(config.hidden_size, config.hidden_size)
|
||||
self.fc3 = nn.Linear(config.hidden_size, latent_dim * 2)
|
||||
self.norm = nn.LayerNorm(config.hidden_size)
|
||||
self.patch_size = patch_size
|
||||
if patch_size != -1:
|
||||
config.num_hidden_layers = 4
|
||||
self.aggregator = Qwen2Model(config)
|
||||
self.cls_embed = nn.Parameter(torch.rand(1, 1, config.hidden_size))
|
||||
self.cls_embed.data.normal_(0, 0.02)
|
||||
|
||||
def get_frames(self, x):
|
||||
num_frames_total = (x.size(-1) + self.hop_size - 1) // self.hop_size
|
||||
expected_len = (num_frames_total - 1) * self.hop_size + self.input_dim
|
||||
padding_needed = expected_len - x.size(-1)
|
||||
waveform = F.pad(x, (0, padding_needed), value=0.0)
|
||||
frames = waveform.unfold(dimension=-1, size=self.input_dim, step=self.hop_size)
|
||||
return frames
|
||||
|
||||
def pad_patch_insert_cls(self, x):
|
||||
bsz, _, dim = x.size()
|
||||
num_frame = x.size(1)
|
||||
r = num_frame % self.patch_size
|
||||
pad_num = self.patch_size - r if r else 0
|
||||
x = F.pad(x, (0, 0, 0, pad_num), value=0.0)
|
||||
x = x.reshape(-1, self.patch_size, dim)
|
||||
x = torch.cat((x, self.cls_embed.expand(x.size(0), -1, -1)), dim=1)
|
||||
x = x.reshape(bsz, -1, dim)
|
||||
return x
|
||||
|
||||
def forward(self, waveform):
|
||||
x = self.get_frames(waveform)
|
||||
x = self.fc1(x)
|
||||
x = self.fc2(x)
|
||||
x = self.encoder(inputs_embeds=x)
|
||||
x = x.last_hidden_state
|
||||
|
||||
if self.patch_size != -1:
|
||||
x = self.pad_patch_insert_cls(x)
|
||||
x = self.aggregator(inputs_embeds=x)
|
||||
x = x.last_hidden_state
|
||||
bsz, _, dim = x.size()
|
||||
x = x.reshape(-1, self.patch_size + 1, dim)
|
||||
x = x[:, -1:, :].reshape(bsz, -1, dim)
|
||||
|
||||
x = self.fc3(x)
|
||||
return x, waveform.unsqueeze(1)
|
||||
|
||||
|
||||
class AudioVAE(PreTrainedModel):
|
||||
config_class = AudioVAEConfig
|
||||
|
||||
def __init__(self, config: AudioVAEConfig):
|
||||
super().__init__(config)
|
||||
self.encoder = Encoder(
|
||||
encoder_args=config.enc_kwargs["backbone"],
|
||||
input_dim=config.enc_kwargs["input_dim"],
|
||||
hop_size=config.enc_kwargs.get("hop_size", 320),
|
||||
latent_dim=config.enc_kwargs["latent_dim"],
|
||||
patch_size=config.patch_size,
|
||||
)
|
||||
self.decoder = Decoder(
|
||||
decoder_args=config.dec_kwargs["backbone"],
|
||||
output_dim=config.dec_kwargs["output_dim"],
|
||||
latent_dim=config.dec_kwargs["latent_dim"],
|
||||
patch_size=config.patch_size,
|
||||
)
|
||||
self.post_init()
|
||||
|
||||
def _init_weights(self, module):
|
||||
std = 0.02
|
||||
if isinstance(module, nn.Linear):
|
||||
if self.config.init_method == "kaiming":
|
||||
nn.init.kaiming_normal_(module.weight, mode="fan_in", nonlinearity="relu")
|
||||
else:
|
||||
module.weight.data.normal_(mean=0.0, std=std)
|
||||
if module.bias is not None:
|
||||
module.bias.data.zero_()
|
||||
elif isinstance(module, nn.Embedding):
|
||||
module.weight.data.normal_(mean=0.0, std=std)
|
||||
if module.padding_idx is not None:
|
||||
module.weight.data[module.padding_idx].zero_()
|
||||
|
||||
def encode_latent(self, waveform, waveform_length):
|
||||
from diffusers.models.autoencoders.autoencoder_oobleck import OobleckDiagonalGaussianDistribution
|
||||
|
||||
frame_num = torch.ceil(waveform_length / self.config.enc_kwargs["input_dim"]).to(torch.int32)
|
||||
if self.config.patch_size != -1:
|
||||
frame_num = torch.ceil(frame_num / self.config.patch_size)
|
||||
h, y = self.encoder(waveform)
|
||||
h = h.transpose(1, 2)
|
||||
|
||||
posterior = OobleckDiagonalGaussianDistribution(h)
|
||||
latent = posterior.sample()
|
||||
latent = latent.transpose(1, 2)
|
||||
return latent, frame_num
|
||||
|
||||
def decode(self, latent, past_key_values=None, use_cache=False, stream_state=(None, None, None), last_chunk=False):
|
||||
waveform, stream_state, past_key_values = self.decoder.low_level_reconstruct(
|
||||
latent,
|
||||
past_key_values=past_key_values,
|
||||
use_cache=use_cache,
|
||||
stream_state=stream_state,
|
||||
last_chunk=last_chunk,
|
||||
)
|
||||
return waveform, stream_state, past_key_values
|
||||
@@ -16,7 +16,23 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Ming-flash-omni-2.0 unified model (thinker + imagegen + talker)."""
|
||||
"""Ming-flash-omni-2.0 thinker / image-gen wrapper.
|
||||
|
||||
This class is the multimodal-registered entry point for Ming stages that
|
||||
share the thinker's backbone: comprehension / text generation (`thinker`)
|
||||
and diffusion conditioning for image generation (`imagegen`, not yet
|
||||
implemented).
|
||||
|
||||
The talker deliberately lives elsewhere. Upstream Ming hands text (not
|
||||
hidden states) from the thinker to the talker, and the talker then
|
||||
tokenises that string with its own Qwen2 tokenizer and runs an entirely
|
||||
self-contained LLM + CFM + AudioVAE pipeline. Because it has no
|
||||
multimodal inputs, it belongs in the non-MM-registered
|
||||
`MingFlashOmniTalkerForConditionalGeneration` — routing it through
|
||||
this wrapper would force it through vLLM's multimodal preprocess path
|
||||
and trigger a hidden-size mismatch between the outer Ming config
|
||||
(4096, thinker's LLM) and the talker's Qwen2 backbone (896).
|
||||
"""
|
||||
|
||||
from collections.abc import Iterable
|
||||
|
||||
@@ -40,7 +56,10 @@ from vllm.sequence import IntermediateTensors
|
||||
from vllm_omni.model_executor.custom_process_mixin import CustomProcessMixin
|
||||
from vllm_omni.model_executor.models.output_templates import OmniOutput
|
||||
from vllm_omni.model_executor.models.utils import add_prefix_to_loaded_weights
|
||||
from vllm_omni.transformers_utils.configs.ming_flash_omni import BailingMM2Config, MingFlashOmniConfig
|
||||
from vllm_omni.transformers_utils.configs.ming_flash_omni import (
|
||||
BailingMM2Config,
|
||||
MingFlashOmniConfig,
|
||||
)
|
||||
|
||||
from .ming_flash_omni_thinker import (
|
||||
MingFlashOmniThinkerDummyInputsBuilder,
|
||||
@@ -63,7 +82,7 @@ class MingFlashOmniForConditionalGeneration(
|
||||
SupportsMRoPE,
|
||||
CustomProcessMixin,
|
||||
):
|
||||
"""Unified Ming-flash-omni-2.0 model combining thinker, imagegen, and talker."""
|
||||
"""Ming-flash-omni-2.0 thinker + image-gen wrapper."""
|
||||
|
||||
supports_multimodal = True
|
||||
requires_raw_input_tokens: bool = True
|
||||
@@ -75,19 +94,26 @@ class MingFlashOmniForConditionalGeneration(
|
||||
self.has_postprocess = False
|
||||
|
||||
config = vllm_config.model_config.hf_config
|
||||
|
||||
self.vllm_config = vllm_config
|
||||
self.config = config
|
||||
|
||||
if isinstance(config, MingFlashOmniConfig):
|
||||
thinker_config = config.thinker_config
|
||||
else:
|
||||
thinker_config = config
|
||||
|
||||
self.thinker_config: BailingMM2Config = thinker_config
|
||||
self.model_stage = vllm_config.model_config.model_stage
|
||||
|
||||
if self.model_stage == "talker":
|
||||
raise ValueError(
|
||||
"MingFlashOmniForConditionalGeneration does not support "
|
||||
"model_stage='talker'. Use "
|
||||
"model_arch='MingFlashOmniTalkerForConditionalGeneration' "
|
||||
"directly — the talker has a self-contained LLM that "
|
||||
"tokenises text itself and does not need the multimodal "
|
||||
"preprocess path. See stage_configs/ming_flash_omni.yaml "
|
||||
"stage 1 and stage_configs/ming_flash_omni_tts.yaml."
|
||||
)
|
||||
|
||||
if self.model_stage == "thinker":
|
||||
if isinstance(config, MingFlashOmniConfig):
|
||||
thinker_config: BailingMM2Config = config.thinker_config
|
||||
else:
|
||||
thinker_config: BailingMM2Config = config
|
||||
|
||||
thinker_vllm_config = vllm_config.with_hf_config(
|
||||
thinker_config, architectures=["MingFlashOmniThinkerForConditionalGeneration"]
|
||||
)
|
||||
@@ -97,8 +123,7 @@ class MingFlashOmniForConditionalGeneration(
|
||||
architectures=["MingFlashOmniThinkerForConditionalGeneration"],
|
||||
)
|
||||
self.model = self.thinker
|
||||
self.imagegen = None
|
||||
self.talker = None
|
||||
self.make_empty_intermediate_tensors = self.thinker.make_empty_intermediate_tensors
|
||||
|
||||
elif self.model_stage == "imagegen":
|
||||
# TODO: Implement image generator stage
|
||||
@@ -106,22 +131,12 @@ class MingFlashOmniForConditionalGeneration(
|
||||
"Image generation stage is not yet implemented. Please use model_stage='thinker' for now."
|
||||
)
|
||||
|
||||
elif self.model_stage == "talker":
|
||||
# TODO: Implement talker (TTS) stage
|
||||
raise NotImplementedError(
|
||||
"Talker (TTS) stage is not yet implemented. Please use model_stage='thinker' for now."
|
||||
)
|
||||
|
||||
else:
|
||||
raise ValueError(
|
||||
f"Invalid model_stage: {self.model_stage}. Must be one of: 'thinker', 'imagegen', 'talker'"
|
||||
f"Invalid model_stage: {self.model_stage!r}. Must be one of: 'thinker', 'imagegen'. "
|
||||
f"For the talker stage, use MingFlashOmniTalkerForConditionalGeneration directly."
|
||||
)
|
||||
|
||||
# Set up intermediate tensors
|
||||
self.make_empty_intermediate_tensors = (
|
||||
self.thinker.make_empty_intermediate_tensors if self.model_stage == "thinker" else lambda: None
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
@@ -154,7 +169,7 @@ class MingFlashOmniForConditionalGeneration(
|
||||
):
|
||||
if hasattr(self.model, "sample"):
|
||||
return self.model.sample(logits, sampling_metadata)
|
||||
raise NotImplementedError("sample method not available on current stage")
|
||||
return None
|
||||
|
||||
def get_mrope_input_positions(self, *args, **kwargs):
|
||||
if hasattr(self.model, "get_mrope_input_positions"):
|
||||
@@ -162,36 +177,9 @@ class MingFlashOmniForConditionalGeneration(
|
||||
raise NotImplementedError("get_mrope_input_positions not available on current stage")
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
loaded_weights = set()
|
||||
thinker_weights = []
|
||||
imagegen_weights = []
|
||||
talker_weights = []
|
||||
|
||||
for name, value in weights:
|
||||
if name.startswith("thinker."):
|
||||
thinker_weights.append((name, value))
|
||||
elif name.startswith("imagegen."):
|
||||
imagegen_weights.append((name, value))
|
||||
elif name.startswith("talker."):
|
||||
talker_weights.append((name, value))
|
||||
else:
|
||||
# Weights without prefix go to thinker by default
|
||||
thinker_weights.append((name, value))
|
||||
|
||||
if self.model_stage == "thinker" and thinker_weights:
|
||||
# Remove "thinker." prefix before loading
|
||||
thinker_weights_stripped = [
|
||||
(name.replace("thinker.", "", 1) if name.startswith("thinker.") else name, value)
|
||||
for name, value in thinker_weights
|
||||
]
|
||||
thinker_loaded = self.thinker.load_weights(thinker_weights_stripped)
|
||||
thinker_loaded = add_prefix_to_loaded_weights(thinker_loaded, "thinker")
|
||||
loaded_weights.update(thinker_loaded)
|
||||
|
||||
# TODO: Load imagegen weights when implemented
|
||||
# TODO: Load talker weights when implemented
|
||||
|
||||
return loaded_weights
|
||||
stripped = ((name.removeprefix("thinker."), value) for name, value in weights)
|
||||
thinker_loaded = self.thinker.load_weights(stripped)
|
||||
return add_prefix_to_loaded_weights(thinker_loaded, "thinker")
|
||||
|
||||
def get_mm_mapping(self) -> MultiModelKeys:
|
||||
return MultiModelKeys.from_string_field(
|
||||
@@ -202,9 +190,7 @@ class MingFlashOmniForConditionalGeneration(
|
||||
|
||||
@property
|
||||
def sampler(self):
|
||||
if hasattr(self.model, "sampler"):
|
||||
return self.model.sampler
|
||||
return None
|
||||
return getattr(self.model, "sampler", None)
|
||||
|
||||
def embed_input_ids(
|
||||
self,
|
||||
|
||||
@@ -0,0 +1,586 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Copyright 2025 The vLLM-Omni team.
|
||||
# Copyright (c) Ant Group. All rights reserved.
|
||||
# Adapted from:
|
||||
# https://github.com/inclusionAI/Ming/blob/e58533db227031990c5a6864dcf5f08fb53ed0d2/modeling_bailing_talker.py
|
||||
"""Ming-flash-omni-2.0 talker (TTS) stage model."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import glob as glob_module
|
||||
import os
|
||||
from collections.abc import Iterable
|
||||
from dataclasses import dataclass
|
||||
from functools import cached_property
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from safetensors.torch import load_file
|
||||
from transformers import AutoTokenizer, Qwen2Config, Qwen2Model
|
||||
from transformers.utils.hub import cached_file
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.models.utils import AutoWeightsLoader
|
||||
from vllm.sequence import IntermediateTensors
|
||||
|
||||
from vllm_omni.model_executor.custom_process_mixin import CustomProcessMixin
|
||||
from vllm_omni.model_executor.model_loader.weight_utils import download_weights_from_hf_specific
|
||||
from vllm_omni.model_executor.models.output_templates import OmniOutput
|
||||
from vllm_omni.transformers_utils.configs.ming_flash_omni import MingFlashOmniTalkerConfig
|
||||
|
||||
from .audio_vae import AudioVAE, AudioVAEConfig
|
||||
from .prompt_utils import DEFAULT_PROMPT as MING_DEFAULT_PROMPT
|
||||
from .talker_module import CFM, Aggregator, DiT, MingAudioGenerator, build_tts_input
|
||||
from .text_processing import segment_and_normalize
|
||||
from .voice_presets import VoicePresetRegistry
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class _GenerationParams:
|
||||
"""Resolved sampling / decoding parameters for one forward call."""
|
||||
|
||||
prompt: str
|
||||
instruction: str | None
|
||||
cfg: float
|
||||
sigma: float
|
||||
temperature: float
|
||||
max_steps: int
|
||||
use_zero_spk_emb: bool
|
||||
max_text_length: int
|
||||
use_static_cache: bool
|
||||
stream_decode: bool
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class _VoiceContext:
|
||||
"""Voice cloning inputs resolved from request info + presets."""
|
||||
|
||||
spk_emb: Any # list[Tensor] | Tensor | list[float] | None
|
||||
prompt_text: str | None
|
||||
prompt_wav_lat: torch.Tensor | None
|
||||
prompt_wav_emb: torch.Tensor | None
|
||||
already_projected: bool
|
||||
|
||||
|
||||
class MingFlashOmniTalkerForConditionalGeneration(nn.Module, CustomProcessMixin):
|
||||
"""Ming-flash-omni-2.0 talker stage: text -> audio waveform.
|
||||
|
||||
Uses Qwen2 LLM + CFM (Conditional Flow Matching with DiT) + Aggregator
|
||||
in an autoregressive loop to produce continuous audio latents, then
|
||||
AudioVAE decodes latents to waveforms.
|
||||
"""
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
self.have_multimodal_outputs = True
|
||||
self.has_preprocess = False
|
||||
self.has_postprocess = False
|
||||
|
||||
self.vllm_config = vllm_config
|
||||
root_config = vllm_config.model_config.hf_config
|
||||
|
||||
model_path = vllm_config.model_config.model
|
||||
self._model_path = model_path
|
||||
self.talker_dir = (
|
||||
os.path.join(model_path, "talker") if os.path.isdir(os.path.join(model_path, "talker")) else model_path
|
||||
)
|
||||
|
||||
# When used standalone (model_arch=MingFlashOmniTalkerForConditionalGeneration),
|
||||
# the root hf_config may be BailingMM2Config (thinker-only) due to model file structure
|
||||
# Resolve talker config from talker/config.json in that case.
|
||||
config = (
|
||||
root_config
|
||||
if isinstance(root_config, MingFlashOmniTalkerConfig)
|
||||
else self._resolve_talker_config(root_config, self.talker_dir, model_path)
|
||||
)
|
||||
self.config = config
|
||||
|
||||
self._standalone = prefix in ("", "talker")
|
||||
if self._standalone:
|
||||
self.allow_patterns_overrides = ["talker/model*.safetensors"]
|
||||
self.fall_back_to_pt_during_load = False
|
||||
|
||||
# LLM
|
||||
llm_config = self._resolve_llm_config(config, self.talker_dir, model_path)
|
||||
llm_config._attn_implementation = "sdpa"
|
||||
self.llm_config = llm_config
|
||||
self.hidden_size = llm_config.hidden_size
|
||||
self.latent_dim = config.latent_dim
|
||||
self.patch_size = config.patch_size
|
||||
self.his_patch_size = config.history_patch_size
|
||||
self.cfg_strength = config.cfg_strength
|
||||
|
||||
self.model = Qwen2Model(llm_config)
|
||||
self.cfm = CFM(
|
||||
DiT(llm_input_dim=self.hidden_size, **config.flowmodel),
|
||||
steps=config.steps,
|
||||
)
|
||||
self.aggregator = Aggregator(llm_input_dim=self.hidden_size, **config.aggregator)
|
||||
self.stop_head = nn.Linear(self.hidden_size, 2, bias=True)
|
||||
# CAMPPlus 192-dim -> hidden
|
||||
self.spk_head = nn.Linear(192, self.hidden_size, bias=True)
|
||||
|
||||
# AudioVAE
|
||||
self.audio_vae, self._vae_weight_source = self._init_audio_vae(config, self.talker_dir, model_path)
|
||||
|
||||
self._use_cuda_graphs = not vllm_config.model_config.enforce_eager
|
||||
|
||||
self.audio_generator = MingAudioGenerator(
|
||||
config=self.config,
|
||||
llm_config=self.llm_config,
|
||||
model=self.model,
|
||||
cfm=self.cfm,
|
||||
aggregator=self.aggregator,
|
||||
stop_head=self.stop_head,
|
||||
audio_vae=self.audio_vae,
|
||||
patch_size=self.patch_size,
|
||||
his_patch_size=self.his_patch_size,
|
||||
latent_dim=self.latent_dim,
|
||||
cfg_strength=self.cfg_strength,
|
||||
use_cuda_graphs=self._use_cuda_graphs,
|
||||
)
|
||||
self.voice_presets = VoicePresetRegistry(
|
||||
talker_dir=self.talker_dir,
|
||||
model_path=self._model_path,
|
||||
download_dir=vllm_config.load_config.download_dir,
|
||||
audio_vae=self.audio_vae,
|
||||
aggregator=self.aggregator,
|
||||
spk_head=self.spk_head,
|
||||
patch_size=self.patch_size,
|
||||
)
|
||||
|
||||
@property
|
||||
def device(self) -> torch.device:
|
||||
return next(self.model.parameters()).device
|
||||
|
||||
@property
|
||||
def dtype(self) -> torch.dtype:
|
||||
return next(self.model.parameters()).dtype
|
||||
|
||||
@cached_property
|
||||
def tokenizer(self):
|
||||
# Lazy Qwen2 tokenizer resolution:
|
||||
# 1. Try local dirs first (talker/llm, talker, and then model root).
|
||||
# 2. HF repo-id fallback: talker/llm is the canonical tokenizer location.
|
||||
candidates = (os.path.join(self.talker_dir, "llm"), self.talker_dir, self._model_path)
|
||||
for path in candidates:
|
||||
if os.path.isdir(path):
|
||||
try:
|
||||
logger.debug("Resolving talker tokenizer from local dir %s", path)
|
||||
return AutoTokenizer.from_pretrained(path, trust_remote_code=True)
|
||||
except Exception:
|
||||
continue
|
||||
for subfolder in ("talker/llm", "llm"):
|
||||
try:
|
||||
logger.debug("Resolving talker tokenizer from HF subfolder %s", subfolder)
|
||||
return AutoTokenizer.from_pretrained(self._model_path, subfolder=subfolder, trust_remote_code=True)
|
||||
except Exception:
|
||||
continue
|
||||
logger.debug("Falling back to raw model_path tokenizer resolution")
|
||||
return AutoTokenizer.from_pretrained(self._model_path, trust_remote_code=True)
|
||||
|
||||
@staticmethod
|
||||
def _resolve_talker_config(config, talker_dir: str, model_path: str) -> MingFlashOmniTalkerConfig:
|
||||
"""Resolve MingFlashOmniTalkerConfig when the root config is not one.
|
||||
|
||||
This happens in standalone TTS mode where hf_config is BailingMM2Config.
|
||||
"""
|
||||
# If the root config wraps a talker_config, use it
|
||||
talker_config = getattr(config, "talker_config", None)
|
||||
if isinstance(talker_config, MingFlashOmniTalkerConfig):
|
||||
return talker_config
|
||||
|
||||
# Try loading from talker/config.json
|
||||
if os.path.isdir(talker_dir):
|
||||
try:
|
||||
resolved = MingFlashOmniTalkerConfig.from_pretrained(talker_dir)
|
||||
logger.info("Resolved talker config from %s", talker_dir)
|
||||
return resolved
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
try:
|
||||
resolved = MingFlashOmniTalkerConfig.from_pretrained(model_path, subfolder="talker", trust_remote_code=True)
|
||||
logger.info("Resolved talker config from %s/talker (HF hub)", model_path)
|
||||
return resolved
|
||||
except Exception as e:
|
||||
raise ValueError(
|
||||
f"Cannot resolve MingFlashOmniTalkerConfig. The root config "
|
||||
f"is {type(config).__name__}, and talker/config.json was not "
|
||||
f"found at {talker_dir} or via HF hub: {e}"
|
||||
) from e
|
||||
|
||||
@staticmethod
|
||||
def _resolve_llm_config(config: MingFlashOmniTalkerConfig, talker_dir: str, model_path: str) -> Qwen2Config:
|
||||
"""Resolve the Qwen2 LLM config for the talker backbone."""
|
||||
|
||||
if config.llm_config is not None:
|
||||
return Qwen2Config(**config.llm_config) if isinstance(config.llm_config, dict) else config.llm_config
|
||||
|
||||
# Try local talker/llm directory
|
||||
llm_dir = os.path.join(talker_dir, "llm")
|
||||
if os.path.isdir(llm_dir):
|
||||
return Qwen2Config.from_pretrained(llm_dir)
|
||||
|
||||
# HF hub fallback
|
||||
for subfolder in ("talker/llm", "llm"):
|
||||
try:
|
||||
return Qwen2Config.from_pretrained(model_path, subfolder=subfolder, trust_remote_code=True)
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
raise ValueError(
|
||||
f"Cannot find talker LLM config at {llm_dir}. "
|
||||
"Either provide llm_config in MingFlashOmniTalkerConfig or "
|
||||
"ensure the model path contains talker/llm/config.json."
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _init_audio_vae(
|
||||
config: MingFlashOmniTalkerConfig, talker_dir: str, model_path: str
|
||||
) -> tuple[AudioVAE | None, str | tuple[str, str] | None]:
|
||||
"""Initialize AudioVAE and return (vae, weight_source).
|
||||
|
||||
weight_source is either a local directory path (str) or an
|
||||
(repo_id, subfolder) tuple for HF hub downloads, or None.
|
||||
"""
|
||||
vae_path = config.audio_vae_path or os.path.join(talker_dir, "vae")
|
||||
|
||||
# Try local directory first
|
||||
if os.path.isdir(vae_path):
|
||||
try:
|
||||
vae_config = AudioVAEConfig.from_pretrained(vae_path)
|
||||
vae = AudioVAE(vae_config)
|
||||
logger.info("Initialized AudioVAE from %s (sr=%d)", vae_path, vae_config.sample_rate)
|
||||
return vae, vae_path
|
||||
except Exception as e:
|
||||
logger.warning("Failed to initialize AudioVAE from %s: %s", vae_path, e)
|
||||
return None, None
|
||||
|
||||
# HF hub fallback
|
||||
for subfolder in ("talker/vae", "vae"):
|
||||
try:
|
||||
vae_config = AudioVAEConfig.from_pretrained(model_path, subfolder=subfolder, trust_remote_code=True)
|
||||
vae = AudioVAE(vae_config)
|
||||
logger.info(f"Initialized AudioVAE from {model_path}/{subfolder}")
|
||||
return vae, (model_path, subfolder)
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
logger.info("AudioVAE not found at %s; waveform decoding unavailable", vae_path)
|
||||
return None, None
|
||||
|
||||
def compute_logits(self, hidden_states: torch.Tensor, sampling_metadata=None) -> torch.Tensor | None:
|
||||
return None
|
||||
|
||||
def sample(self, logits: torch.Tensor, sampling_metadata):
|
||||
return None
|
||||
|
||||
def embed_input_ids(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
multimodal_embeddings=None,
|
||||
is_multimodal=None,
|
||||
) -> torch.Tensor:
|
||||
return self.model.get_input_embeddings()(input_ids)
|
||||
|
||||
def make_empty_intermediate_tensors(
|
||||
self, batch_size: int, dtype: torch.dtype, device: torch.device
|
||||
) -> IntermediateTensors | None:
|
||||
return None
|
||||
|
||||
def get_dummy_runtime_additional_information(self, num_reqs: int) -> list[dict[str, object]]:
|
||||
info: dict[str, object] = {"text": "dummy", "use_zero_spk_emb": True, "max_steps": 1}
|
||||
return [info for _ in range(num_reqs)]
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
intermediate_tensors: IntermediateTensors | None = None,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
runtime_additional_information: list[dict] | None = None,
|
||||
**kwargs,
|
||||
) -> OmniOutput:
|
||||
"""Run TTS generation and return audio output.
|
||||
|
||||
The full autoregressive generation loop is executed inside this method.
|
||||
"""
|
||||
additional_info = self._extract_additional_info(runtime_additional_information)
|
||||
params = self._resolve_generation_params(additional_info)
|
||||
voice = self._resolve_voice(additional_info)
|
||||
|
||||
latents = self._generate_latents(
|
||||
input_ids=input_ids,
|
||||
inputs_embeds=inputs_embeds,
|
||||
text=additional_info.get("text", ""),
|
||||
params=params,
|
||||
voice=voice,
|
||||
)
|
||||
return self._decode_to_output(latents, stream_decode=params.stream_decode)
|
||||
|
||||
@staticmethod
|
||||
def _extract_additional_info(
|
||||
runtime_additional_information: list[dict] | None,
|
||||
) -> dict[str, Any]:
|
||||
if runtime_additional_information and len(runtime_additional_information) > 0:
|
||||
return runtime_additional_information[0] or {}
|
||||
return {}
|
||||
|
||||
def _resolve_generation_params(self, additional_info: dict[str, Any]) -> _GenerationParams:
|
||||
# "omni" : thinker -> talker hand-off with hardcoded defaults
|
||||
# "instruct": standalone TTS with caller-supplied sampling knobs
|
||||
ming_task = additional_info.get("ming_task", "instruct")
|
||||
|
||||
if ming_task == "omni":
|
||||
prompt = MING_DEFAULT_PROMPT
|
||||
instruction = None
|
||||
use_zero_spk_emb = additional_info.get("spk_emb") is None
|
||||
cfg = 2.0
|
||||
sigma = 0.25
|
||||
temperature = 0.0
|
||||
max_steps = 200
|
||||
else:
|
||||
prompt = additional_info.get("prompt", MING_DEFAULT_PROMPT)
|
||||
instruction = additional_info.get("instruction", None)
|
||||
use_zero_spk_emb = additional_info.get("use_zero_spk_emb", False)
|
||||
cfg = additional_info.get("cfg", self.cfg_strength)
|
||||
sigma = additional_info.get("sigma", 0.25)
|
||||
temperature = additional_info.get("temperature", 0.0)
|
||||
max_steps = int(additional_info.get("max_steps", additional_info.get("max_decode_steps", 200)))
|
||||
|
||||
return _GenerationParams(
|
||||
prompt=prompt,
|
||||
instruction=instruction,
|
||||
cfg=cfg,
|
||||
sigma=sigma,
|
||||
temperature=temperature,
|
||||
max_steps=max_steps,
|
||||
use_zero_spk_emb=use_zero_spk_emb,
|
||||
max_text_length=int(additional_info.get("max_text_length", 50)),
|
||||
use_static_cache=bool(additional_info.get("use_static_cache", True)),
|
||||
stream_decode=bool(additional_info.get("stream_decode", True)),
|
||||
)
|
||||
|
||||
def _resolve_voice(self, additional_info: dict[str, Any]) -> _VoiceContext:
|
||||
spk_emb = additional_info.get("spk_emb", None)
|
||||
prompt_text = additional_info.get("prompt_text", None)
|
||||
prompt_wav_lat = additional_info.get("prompt_wav_lat", None)
|
||||
prompt_wav_emb = additional_info.get("prompt_wav_emb", None)
|
||||
already_projected = False
|
||||
|
||||
voice_name = additional_info.get("voice_name", None)
|
||||
if voice_name and spk_emb is None and voice_name in self.voice_presets:
|
||||
preset = self.voice_presets.get(voice_name) or {}
|
||||
prompt_wav_lat = preset.get("prompt_wav_lat")
|
||||
prompt_wav_emb = preset.get("prompt_wav_emb")
|
||||
spk_emb = preset.get("spk_emb")
|
||||
already_projected = True
|
||||
if prompt_text is None:
|
||||
prompt_text = preset.get("prompt_text")
|
||||
|
||||
return _VoiceContext(
|
||||
spk_emb=spk_emb,
|
||||
prompt_text=prompt_text,
|
||||
prompt_wav_lat=prompt_wav_lat,
|
||||
prompt_wav_emb=prompt_wav_emb,
|
||||
already_projected=already_projected,
|
||||
)
|
||||
|
||||
def _project_spk_emb(
|
||||
self, spk_emb: Any, already_projected: bool, use_zero_spk_emb: bool
|
||||
) -> list[torch.Tensor] | None:
|
||||
if spk_emb is None:
|
||||
if use_zero_spk_emb:
|
||||
return [torch.zeros(1, self.hidden_size, device=self.device, dtype=self.dtype)]
|
||||
return None
|
||||
|
||||
if already_projected:
|
||||
return spk_emb if isinstance(spk_emb, list) else [spk_emb]
|
||||
|
||||
if isinstance(spk_emb, torch.Tensor):
|
||||
tensors = [spk_emb]
|
||||
elif isinstance(spk_emb, list) and spk_emb and isinstance(spk_emb[0], (int, float)):
|
||||
tensors = [torch.tensor(spk_emb, dtype=self.dtype).unsqueeze(0)]
|
||||
elif isinstance(spk_emb, list):
|
||||
tensors = spk_emb
|
||||
else:
|
||||
tensors = [spk_emb]
|
||||
return [self.spk_head(t.to(device=self.device, dtype=self.dtype)) for t in tensors]
|
||||
|
||||
def _generate_latents(
|
||||
self,
|
||||
*,
|
||||
input_ids: torch.Tensor,
|
||||
inputs_embeds: torch.Tensor | None,
|
||||
text: str,
|
||||
params: _GenerationParams,
|
||||
voice: _VoiceContext,
|
||||
) -> list[torch.Tensor]:
|
||||
generator = self.audio_generator
|
||||
|
||||
if inputs_embeds is not None:
|
||||
# Caller pre-built embeddings — run a single AR pass.
|
||||
return generator.generate_latents(
|
||||
inputs_embeds=inputs_embeds,
|
||||
prompt_wav_lat=voice.prompt_wav_lat,
|
||||
max_steps=params.max_steps,
|
||||
cfg=params.cfg,
|
||||
sigma=params.sigma,
|
||||
temperature=params.temperature,
|
||||
use_static_cache=params.use_static_cache,
|
||||
)
|
||||
|
||||
spk_emb = self._project_spk_emb(voice.spk_emb, voice.already_projected, params.use_zero_spk_emb)
|
||||
text_segments = segment_and_normalize(text, max_length=params.max_text_length) if text else []
|
||||
|
||||
if not text_segments:
|
||||
# vLLM passes 1D input_ids; Qwen2Model expects (batch, seq).
|
||||
inputs_embeds = self.model.get_input_embeddings()(input_ids.to(self.device)).unsqueeze(0)
|
||||
return generator.generate_latents(
|
||||
inputs_embeds=inputs_embeds,
|
||||
prompt_wav_lat=voice.prompt_wav_lat,
|
||||
max_steps=params.max_steps,
|
||||
cfg=params.cfg,
|
||||
sigma=params.sigma,
|
||||
temperature=params.temperature,
|
||||
use_static_cache=params.use_static_cache,
|
||||
)
|
||||
|
||||
all_latents: list[torch.Tensor] = []
|
||||
for segment in text_segments:
|
||||
seg_embeds, _ = build_tts_input(
|
||||
tokenizer=self.tokenizer,
|
||||
embed_tokens=self.model.get_input_embeddings(),
|
||||
device=self.device,
|
||||
dtype=torch.bfloat16,
|
||||
text=segment,
|
||||
prompt=params.prompt,
|
||||
spk_emb=spk_emb,
|
||||
instruction=params.instruction,
|
||||
prompt_text=voice.prompt_text,
|
||||
prompt_wav_emb=voice.prompt_wav_emb,
|
||||
)
|
||||
effective_max_steps = generator.duration_capped_steps(len(segment), params.max_steps)
|
||||
all_latents.extend(
|
||||
generator.generate_latents(
|
||||
inputs_embeds=seg_embeds,
|
||||
prompt_wav_lat=voice.prompt_wav_lat,
|
||||
max_steps=effective_max_steps,
|
||||
cfg=params.cfg,
|
||||
sigma=params.sigma,
|
||||
temperature=params.temperature,
|
||||
use_static_cache=params.use_static_cache,
|
||||
)
|
||||
)
|
||||
return all_latents
|
||||
|
||||
def _decode_to_output(self, latents: list[torch.Tensor], *, stream_decode: bool) -> OmniOutput:
|
||||
multimodal_outputs: dict[str, Any] = {}
|
||||
if latents and self.audio_vae is not None:
|
||||
waveform = self.audio_generator.decode_to_waveform(latents, stream_decode=stream_decode)
|
||||
if not stream_decode:
|
||||
waveform = self.audio_generator.trim_trailing_silence(waveform)
|
||||
multimodal_outputs["audio"] = waveform.detach().float().cpu()
|
||||
multimodal_outputs["sr"] = torch.tensor(self.audio_vae.config.sample_rate)
|
||||
elif latents:
|
||||
all_lat = torch.cat(latents, dim=1)
|
||||
multimodal_outputs["audio_latents"] = all_lat.detach().float().cpu()
|
||||
|
||||
return OmniOutput(text_hidden_states=None, multimodal_outputs=multimodal_outputs)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
"""Load weights for all talker components.
|
||||
|
||||
The talker's HF checkpoint (talker/model.safetensors) stores
|
||||
weights with prefixes matching this module's submodule names directly.
|
||||
And AudioVAE weights live in a separate file under talker/vae/
|
||||
"""
|
||||
# Standalone: bypass the default loader's iterator (torch.load on
|
||||
# .safetensors crashes) and read talker/model*.safetensors directly.
|
||||
if self._standalone:
|
||||
weights = self._iter_talker_safetensors()
|
||||
|
||||
loader = AutoWeightsLoader(
|
||||
self,
|
||||
skip_prefixes=["audio_vae."], # loaded separately
|
||||
skip_substrs=["rotary_embed.inv_freq"], # non-persistent buffer
|
||||
)
|
||||
loaded = loader.load_weights(weights)
|
||||
logger.info("Loaded %d talker weights from checkpoint", len(loaded))
|
||||
|
||||
if self.audio_vae is not None and self._vae_weight_source is not None:
|
||||
loaded.update(self._load_vae_weights())
|
||||
|
||||
# Register voice presets after all weights (incl. VAE) are loaded.
|
||||
try:
|
||||
self.voice_presets.load_presets_from_manifest(device=self.device, dtype=self.dtype)
|
||||
except Exception as e: # pragma: no cover — best-effort
|
||||
logger.warning("Voice preset loading failed (non-fatal): %s", e)
|
||||
|
||||
return loaded
|
||||
|
||||
def _iter_talker_safetensors(self) -> Iterable[tuple[str, torch.Tensor]]:
|
||||
"""Yield (name, tensor) pairs from talker/model*.safetensors."""
|
||||
model_path = self._model_path
|
||||
# Try local path first
|
||||
for candidate in (os.path.join(model_path, "talker"), model_path):
|
||||
sf_files = sorted(glob_module.glob(os.path.join(candidate, "model*.safetensors")))
|
||||
if sf_files:
|
||||
for sf_path in sf_files:
|
||||
yield from load_file(sf_path, device="cpu").items()
|
||||
return
|
||||
|
||||
# HF hub fallback: download only the talker checkpoint files
|
||||
model_root = download_weights_from_hf_specific(
|
||||
model_path,
|
||||
self.vllm_config.load_config.download_dir,
|
||||
allow_patterns=["talker/model*.safetensors"],
|
||||
)
|
||||
talker_dir = os.path.join(model_root, "talker")
|
||||
sf_files = sorted(glob_module.glob(os.path.join(talker_dir, "model*.safetensors")))
|
||||
if not sf_files:
|
||||
raise RuntimeError(f"No talker safetensors found under {model_root}. Expected talker/model*.safetensors.")
|
||||
for sf_path in sf_files:
|
||||
yield from load_file(sf_path, device="cpu").items()
|
||||
|
||||
def _load_vae_weights(self) -> set[str]:
|
||||
"""Load AudioVAE weights from talker/vae/model.safetensors."""
|
||||
if self.audio_vae is None or self._vae_weight_source is None:
|
||||
return set()
|
||||
|
||||
# Resolve safetensors file paths from the weight source
|
||||
safetensors_files: list[str] = []
|
||||
source = self._vae_weight_source
|
||||
if isinstance(source, str):
|
||||
# Local directory path
|
||||
safetensors_files = sorted(glob_module.glob(os.path.join(source, "*.safetensors")))
|
||||
elif isinstance(source, tuple):
|
||||
# (repo_id, subfolder) for HF hub
|
||||
repo_id, subfolder = source
|
||||
for filename in ("model.safetensors", "diffusion_pytorch_model.safetensors"):
|
||||
try:
|
||||
cached = cached_file(repo_id, filename, subfolder=subfolder)
|
||||
except Exception:
|
||||
cached = None
|
||||
if cached is not None:
|
||||
safetensors_files.append(cached)
|
||||
break
|
||||
|
||||
if not safetensors_files:
|
||||
logger.warning("No AudioVAE safetensors files found for source=%s", source)
|
||||
return set()
|
||||
|
||||
vae_state_keys = set(self.audio_vae.state_dict().keys())
|
||||
vae_loader = AutoWeightsLoader(self.audio_vae)
|
||||
loaded: set[str] = set()
|
||||
for sf_path in safetensors_files:
|
||||
file_weights = load_file(sf_path, device="cpu")
|
||||
matched = ((name, tensor) for name, tensor in file_weights.items() if name in vae_state_keys)
|
||||
loaded.update(f"audio_vae.{name}" for name in vae_loader.load_weights(matched))
|
||||
|
||||
logger.info("Loaded %d AudioVAE weights from %s", len(loaded), source)
|
||||
return loaded
|
||||
@@ -0,0 +1,50 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Copyright 2025 The vLLM-Omni team.
|
||||
# Copyright (c) Ant Group. All rights reserved.
|
||||
# Adapted from Ming repo's usage cookbook:
|
||||
# https://github.com/inclusionAI/Ming/blob/3954fcb880ff5e61ff128bcf7f1ec344d46a6fe3/cookbook.ipynb
|
||||
"""Shared prompt-building helpers for Ming-flash-omni standalone talker."""
|
||||
|
||||
import copy
|
||||
import json
|
||||
from typing import Any
|
||||
|
||||
DEFAULT_PROMPT = "Please generate speech based on the following description.\n"
|
||||
|
||||
BASE_CAPTION_TEMPLATE: dict[str, Any] = {
|
||||
"audio_sequence": [
|
||||
{
|
||||
"序号": 1,
|
||||
"说话人": "speaker_1",
|
||||
"方言": None,
|
||||
"风格": None,
|
||||
"语速": None,
|
||||
"基频": None,
|
||||
"音量": None,
|
||||
"情感": None,
|
||||
"BGM": {
|
||||
"Genre": None,
|
||||
"Mood": None,
|
||||
"Instrument": None,
|
||||
"Theme": None,
|
||||
"ENV": None,
|
||||
"SNR": None,
|
||||
},
|
||||
"IP": None,
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
|
||||
def create_instruction(user_input: dict[str, Any]) -> str:
|
||||
"""Return a JSON caption string for ``audio_sequence[0]``.
|
||||
|
||||
Only keys already present on the base template are merged in; unknown
|
||||
keys are silently ignored to keep the output schema stable.
|
||||
"""
|
||||
caption = copy.deepcopy(BASE_CAPTION_TEMPLATE)
|
||||
item = caption["audio_sequence"][0]
|
||||
for key, value in user_input.items():
|
||||
if key in item:
|
||||
item[key] = value
|
||||
return json.dumps(caption, ensure_ascii=False)
|
||||
@@ -0,0 +1,44 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Copyright 2025 The vLLM-Omni team.
|
||||
# Copyright (c) Ant Group. All rights reserved.
|
||||
# Ported from:
|
||||
# https://github.com/inclusionAI/Ming/blob/e58533db227031990c5a6864dcf5f08fb53ed0d2/modeling_bailing_talker.py
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import torch
|
||||
|
||||
|
||||
class SpkembExtractor:
|
||||
"""CAMPPlus ONNX-based speaker embedding extractor (runs on CPU)."""
|
||||
|
||||
def __init__(self, campplus_model: str, target_sr: int = 16000):
|
||||
import onnxruntime
|
||||
import torchaudio.compliance.kaldi as kaldi
|
||||
|
||||
self.kaldi = kaldi
|
||||
option = onnxruntime.SessionOptions()
|
||||
option.graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_ENABLE_ALL
|
||||
option.intra_op_num_threads = 2
|
||||
self.campplus_session = onnxruntime.InferenceSession(
|
||||
campplus_model, sess_options=option, providers=["CPUExecutionProvider"]
|
||||
)
|
||||
self.target_sr = target_sr
|
||||
|
||||
def _extract_spk_embedding(self, speech):
|
||||
feat = self.kaldi.fbank(speech, num_mel_bins=80, dither=0, sample_frequency=16000)
|
||||
feat = feat - feat.mean(dim=0, keepdim=True)
|
||||
embedding = (
|
||||
self.campplus_session.run(
|
||||
None,
|
||||
{self.campplus_session.get_inputs()[0].name: feat.unsqueeze(dim=0).cpu().numpy()},
|
||||
)[0]
|
||||
.flatten()
|
||||
.tolist()
|
||||
)
|
||||
embedding = torch.tensor([embedding])
|
||||
return embedding
|
||||
|
||||
def __call__(self, waveform, **kwargs) -> torch.Tensor | None:
|
||||
spk_emb = self._extract_spk_embedding(waveform)
|
||||
return spk_emb
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,535 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Copyright 2025 The vLLM-Omni team.
|
||||
# Copyright (c) Ant Group. All rights reserved.
|
||||
# Adapted from:
|
||||
# https://github.com/inclusionAI/Ming/tree/e58533db227031990c5a6864dcf5f08fb53ed0d2/front
|
||||
|
||||
"""Text segmentation and normalization utilities for Ming TTS."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import string
|
||||
|
||||
from vllm.logger import init_logger
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# Ported from
|
||||
# https://github.com/inclusionAI/Ming/blob/e58533db227031990c5a6864dcf5f08fb53ed0d2/front/toolkit.py
|
||||
_TOKENIZE_PATTERN = re.compile(r"(?:[a-zA-Z]\.)+|[a-zA-Z]+(?:['\-][a-zA-Z]+)*|\d+(?:\.\d+)?|[\u4e00-\u9fff]|\s+|\S")
|
||||
|
||||
|
||||
def tokenize_mixed_text(text: str) -> list[str]:
|
||||
return re.findall(_TOKENIZE_PATTERN, text)
|
||||
|
||||
|
||||
def tokenize_mixed_text_iterator(text: str):
|
||||
for match in _TOKENIZE_PATTERN.finditer(text):
|
||||
yield match.group(0)
|
||||
|
||||
|
||||
# Ported from
|
||||
# https://github.com/inclusionAI/Ming/blob/e58533db227031990c5a6864dcf5f08fb53ed0d2/front/text_segment_cut.py
|
||||
def is_chinese(text: str) -> bool:
|
||||
return bool(re.search(r"[\u4e00-\u9fff]", text))
|
||||
|
||||
|
||||
def get_semantic_length(text: str) -> int:
|
||||
"""1 CJK char = 1 unit; 1 contiguous English word = 1 unit."""
|
||||
chinese_char_count = len(re.findall(r"[\u4e00-\u9fa5]", text))
|
||||
english_word_count = len(re.findall(r"[a-zA-Z]+", text))
|
||||
return chinese_char_count + english_word_count
|
||||
|
||||
|
||||
def has_valid_content(text: str) -> bool:
|
||||
punctuation_and_whitespace = string.punctuation + string.whitespace
|
||||
for char in text:
|
||||
if char not in punctuation_and_whitespace:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def append_text_fragment(
|
||||
fragments: list[str],
|
||||
new_text: str,
|
||||
max_len: int,
|
||||
min_tail_length: int,
|
||||
) -> list[str]:
|
||||
new_text = new_text.lstrip(",,:;" + string.whitespace)
|
||||
if not has_valid_content(new_text):
|
||||
return fragments
|
||||
if not fragments:
|
||||
fragments.append(new_text)
|
||||
return fragments
|
||||
|
||||
last_fragment = fragments[-1]
|
||||
last_semantic_len = get_semantic_length(last_fragment)
|
||||
new_semantic_len = get_semantic_length(new_text)
|
||||
|
||||
if last_semantic_len + new_semantic_len <= max_len:
|
||||
if last_fragment.endswith(("。", "!", "?")) and new_semantic_len < min_tail_length:
|
||||
fragments.append(new_text)
|
||||
else:
|
||||
separator = ""
|
||||
if not last_fragment.endswith(" ") and re.match(r"^[a-zA-Z0-9]", new_text):
|
||||
separator = " "
|
||||
fragments[-1] += separator + new_text
|
||||
else:
|
||||
fragments.append(new_text)
|
||||
return fragments
|
||||
|
||||
|
||||
def split_long_fragment(text_fragment: str, max_len: int) -> list[str]:
|
||||
if get_semantic_length(text_fragment) <= max_len:
|
||||
return [text_fragment]
|
||||
|
||||
fragments: list[str] = []
|
||||
current_fragment = ""
|
||||
semantic_units = re.findall(r"([\u4e00-\u9fa5]|[a-zA-Z]+|[^a-zA-Z\u4e00-\u9fa5]+)", text_fragment)
|
||||
for unit in semantic_units:
|
||||
unit_len = get_semantic_length(unit)
|
||||
current_len = get_semantic_length(current_fragment)
|
||||
if current_len + unit_len <= max_len:
|
||||
current_fragment += unit
|
||||
else:
|
||||
if current_fragment:
|
||||
fragments.append(current_fragment)
|
||||
if unit_len > max_len:
|
||||
fragments.append(unit)
|
||||
current_fragment = ""
|
||||
else:
|
||||
current_fragment = unit
|
||||
if current_fragment:
|
||||
fragments.append(current_fragment)
|
||||
return fragments
|
||||
|
||||
|
||||
_DOT_PLACEHOLDER = "##DOT##"
|
||||
# default soft cap on fragment length in semantic units
|
||||
_DEFAULT_MAX_SEMANTIC_LENGTH: int = 50
|
||||
# default tail length controls when a short trailing fragment is
|
||||
# merged with the previous one to avoid leaving an awkward stub.
|
||||
_DEFAULT_MIN_TAIL_LENGTH: int = 5
|
||||
|
||||
|
||||
def cut_text_by_semantic_length(
|
||||
text: str,
|
||||
max_semantic_length: int = _DEFAULT_MAX_SEMANTIC_LENGTH,
|
||||
min_tail_length: int = _DEFAULT_MIN_TAIL_LENGTH,
|
||||
) -> list[str]:
|
||||
"""Segment text into fragments respecting semantic length limits.
|
||||
|
||||
Ported from upstream Ming's `front/text_segment_cut.py`.
|
||||
Position tracking is omitted (not needed for non-streaming VAE decode).
|
||||
"""
|
||||
if not has_valid_content(text):
|
||||
return []
|
||||
|
||||
processed = re.sub(r"(\d)\.(\d)", r"\1" + _DOT_PLACEHOLDER + r"\2", text)
|
||||
for _ in range(3):
|
||||
processed = re.sub(r"([A-Z])\.([A-Z])", r"\1" + _DOT_PLACEHOLDER + r"\2", processed)
|
||||
processed = processed.replace("\n", " ").replace("。,", "。")
|
||||
|
||||
if get_semantic_length(processed) <= max_semantic_length:
|
||||
return [processed.replace(_DOT_PLACEHOLDER, ".")]
|
||||
|
||||
normalized = processed.replace(".", "。").replace("!", "!").replace("?", "?").replace(",", ",")
|
||||
|
||||
# Phase 1: split into sentences on 。!?
|
||||
sentences: list[str] = []
|
||||
current: list[str] = []
|
||||
for char in normalized:
|
||||
current.append(char)
|
||||
if char in "。!?":
|
||||
s = "".join(current).strip()
|
||||
if s:
|
||||
sentences.append(s)
|
||||
current = []
|
||||
if current:
|
||||
s = "".join(current).strip()
|
||||
if s:
|
||||
if not s.endswith(("。", "!", "?")):
|
||||
s += "。"
|
||||
sentences.append(s)
|
||||
|
||||
# Phase 2: merge whole sentences; only clause-split oversized ones.
|
||||
# This ensures split points land on sentence boundaries (。!?)
|
||||
# rather than mid-sentence commas.
|
||||
result_fragments: list[str] = []
|
||||
for sentence in sentences:
|
||||
sent_len = get_semantic_length(sentence)
|
||||
|
||||
if sent_len > max_semantic_length:
|
||||
# Oversized sentence: fall back to clause-level splitting
|
||||
clauses: list[str] = []
|
||||
clause_buf: list[str] = []
|
||||
for char in sentence:
|
||||
clause_buf.append(char)
|
||||
if char in ",;;":
|
||||
cl = "".join(clause_buf).strip()
|
||||
if cl and has_valid_content(cl):
|
||||
clauses.append(cl)
|
||||
elif cl and clauses:
|
||||
clauses[-1] += cl
|
||||
clause_buf = []
|
||||
if clause_buf:
|
||||
cl = "".join(clause_buf).strip()
|
||||
if cl and has_valid_content(cl):
|
||||
clauses.append(cl)
|
||||
elif cl and clauses:
|
||||
clauses[-1] += cl
|
||||
|
||||
i = 0
|
||||
while i < len(clauses):
|
||||
clause = clauses[i]
|
||||
clause_len = get_semantic_length(clause)
|
||||
|
||||
if clause_len < min_tail_length and i + 1 < len(clauses):
|
||||
combined = clause + clauses[i + 1]
|
||||
if get_semantic_length(combined) <= max_semantic_length:
|
||||
result_fragments = append_text_fragment(
|
||||
result_fragments, combined, max_semantic_length, min_tail_length
|
||||
)
|
||||
i += 2
|
||||
continue
|
||||
|
||||
if clause_len > max_semantic_length:
|
||||
for frag in split_long_fragment(clause, max_semantic_length):
|
||||
result_fragments = append_text_fragment(
|
||||
result_fragments, frag, max_semantic_length, min_tail_length
|
||||
)
|
||||
else:
|
||||
result_fragments = append_text_fragment(
|
||||
result_fragments, clause, max_semantic_length, min_tail_length
|
||||
)
|
||||
i += 1
|
||||
else:
|
||||
# Normal sentence: merge at sentence level
|
||||
if not result_fragments:
|
||||
result_fragments.append(sentence)
|
||||
else:
|
||||
last_len = get_semantic_length(result_fragments[-1])
|
||||
if last_len + sent_len <= max_semantic_length:
|
||||
result_fragments[-1] += sentence
|
||||
else:
|
||||
result_fragments.append(sentence)
|
||||
|
||||
return [f.replace(_DOT_PLACEHOLDER, ".") for f in result_fragments]
|
||||
|
||||
|
||||
# Streaming sentence boundary detection
|
||||
|
||||
_RE_CJK = re.compile(r"[\u4e00-\u9fff]")
|
||||
_RE_DIGIT_LAST = re.compile(r"[0-9]")
|
||||
|
||||
|
||||
# Left for reference for now
|
||||
# Alternative to `cut_text_by_semantic_length`
|
||||
def detect_sentence_boundaries(
|
||||
text: str,
|
||||
max_length: int = 50,
|
||||
) -> list[str]:
|
||||
"""Accumulate tokens and flush at sentence boundaries.
|
||||
|
||||
Ported from the streaming sentence detection loop from the Ming repo
|
||||
TTS branch, but operates on the full text since we have it available upfront.
|
||||
"""
|
||||
sentences: list[str] = []
|
||||
streaming_text: list[str] = []
|
||||
count = 0
|
||||
|
||||
for ele in tokenize_mixed_text_iterator(text):
|
||||
if len(ele) == 0:
|
||||
continue
|
||||
|
||||
should_process = False
|
||||
min_tokens = 12 if count == 0 else 8
|
||||
|
||||
if ele[-1] in "!?。,!?":
|
||||
if len(streaming_text) >= min_tokens:
|
||||
should_process = True
|
||||
streaming_text.append(ele)
|
||||
|
||||
elif ele[-1] == ".":
|
||||
if (
|
||||
len(streaming_text) >= min_tokens
|
||||
and streaming_text
|
||||
and not _RE_DIGIT_LAST.search(streaming_text[-1][-1])
|
||||
):
|
||||
should_process = True
|
||||
streaming_text.append(ele)
|
||||
|
||||
elif ele[-1] == "\n":
|
||||
if streaming_text:
|
||||
joined = "".join(streaming_text)
|
||||
if _RE_CJK.search(joined):
|
||||
if _RE_CJK.search(streaming_text[-1][-1]):
|
||||
ele = ","
|
||||
streaming_text.append(ele)
|
||||
else:
|
||||
if len(ele) > 1 and re.search(r"[a-zA-Z]", ele[-2]):
|
||||
ele = ele[:-1] + "."
|
||||
else:
|
||||
ele = ele[:-1]
|
||||
streaming_text.append(ele)
|
||||
|
||||
if len(streaming_text) >= min_tokens:
|
||||
should_process = True
|
||||
else:
|
||||
streaming_text.append(ele)
|
||||
continue
|
||||
|
||||
if should_process:
|
||||
joined = "".join(streaming_text)
|
||||
fragments = cut_text_by_semantic_length(joined, max_length)
|
||||
sentences.extend(fragments)
|
||||
streaming_text = []
|
||||
count += 1
|
||||
|
||||
# Flush remaining
|
||||
if streaming_text and re.search(r"[a-zA-Z\u4e00-\u9fff1-9]", "".join(streaming_text)):
|
||||
joined = "".join(streaming_text)
|
||||
fragments = cut_text_by_semantic_length(joined, max_length)
|
||||
sentences.extend(fragments)
|
||||
|
||||
return sentences
|
||||
|
||||
|
||||
# number normalization for English. Ported from
|
||||
# https://github.com/inclusionAI/Ming/blob/e58533db227031990c5a6864dcf5f08fb53ed0d2/front/number_en.py
|
||||
|
||||
|
||||
_inflect_engine = None
|
||||
|
||||
|
||||
def _get_inflect():
|
||||
global _inflect_engine
|
||||
if _inflect_engine is not None:
|
||||
return _inflect_engine
|
||||
try:
|
||||
import inflect
|
||||
|
||||
_inflect_engine = inflect.engine()
|
||||
except ImportError:
|
||||
logger.warning(
|
||||
"Package 'inflect' not installed - English number normalization "
|
||||
"will be skipped. Install with: pip install inflect"
|
||||
)
|
||||
_inflect_engine = None
|
||||
return _inflect_engine
|
||||
|
||||
|
||||
_comma_number_re = re.compile(r"([0-9][0-9,]+[0-9])")
|
||||
_percent_number_re = re.compile(r"(-?[0-9.,]*[0-9]+)%")
|
||||
_pounds_re = re.compile(r"£(-?[0-9,]*[0-9]+(?:\.[0-9]+)?)")
|
||||
_dollars_re = re.compile(r"\$(-?[0-9.,]*[0-9]+(?:\.[0-9]+)?)")
|
||||
_fraction_re = re.compile(r"([0-9]+)\/([0-9]+)")
|
||||
_ordinal_re = re.compile(r"\b[0-9]+(st|[nr]d|th)\b")
|
||||
_number_re = re.compile(r"\b-?[0-9]+(?:\.[0-9]+)?\b")
|
||||
_unit_re = re.compile(
|
||||
r"\b(-?\d+(?:\.\d+)?)\s*"
|
||||
r"(ms|s|Hz|kHz|MHz|GHz|kb|mb|gb|tb|KB|MB|GB|TB|bps|kbps|Mbps|Gbps|cm|km|kg|V|A|W|°C|°F)\b",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
_version_re = re.compile(r"\b([a-zA-Z]+)([-]?)([0-9]+(?:\.[0-9]+)?)\b")
|
||||
_whitespace_re = re.compile(r"\s+")
|
||||
|
||||
_unit_mapping = {
|
||||
"ms": "milliseconds",
|
||||
"s": "seconds",
|
||||
"hz": "hertz",
|
||||
"khz": "kilohertz",
|
||||
"mhz": "megahertz",
|
||||
"ghz": "gigahertz",
|
||||
"kb": "kilobytes",
|
||||
"mb": "megabytes",
|
||||
"gb": "gigabytes",
|
||||
"tb": "terabytes",
|
||||
"kbps": "kilobits per second",
|
||||
"mbps": "megabits per second",
|
||||
"gbps": "gigabits per second",
|
||||
"bps": "bits per second",
|
||||
"cm": "centimeters",
|
||||
"km": "kilometers",
|
||||
"kg": "kilograms",
|
||||
"v": "volts",
|
||||
"a": "amperes",
|
||||
"w": "watts",
|
||||
"°c": "degrees celsius",
|
||||
"°f": "degrees fahrenheit",
|
||||
}
|
||||
|
||||
|
||||
def _num_to_words(n: int) -> str:
|
||||
p = _get_inflect()
|
||||
if p is None:
|
||||
return str(n)
|
||||
return p.number_to_words(n, andword="")
|
||||
|
||||
|
||||
def _expand_decimal(num_str: str) -> str:
|
||||
"""Expand a decimal number string like '3.14' -> 'three point one four'."""
|
||||
is_negative = num_str.startswith("-")
|
||||
clean = num_str.lstrip("-") or "0"
|
||||
|
||||
if "." in clean:
|
||||
parts = clean.split(".", 1)
|
||||
integer_part = parts[0] or "0"
|
||||
decimal_part = parts[1]
|
||||
if not integer_part.isdigit() or not decimal_part.isdigit():
|
||||
return num_str
|
||||
int_word = _num_to_words(int(integer_part)) if integer_part != "0" else "zero"
|
||||
dec_words = " ".join(_num_to_words(int(d)) for d in decimal_part if d.isdigit())
|
||||
word = f"{int_word} point {dec_words}"
|
||||
else:
|
||||
if not clean.isdigit():
|
||||
return num_str
|
||||
word = _num_to_words(int(clean))
|
||||
|
||||
if is_negative:
|
||||
word = f"minus {word}"
|
||||
return word
|
||||
|
||||
|
||||
def _remove_commas(m: re.Match) -> str:
|
||||
return m.group(1).replace(",", "")
|
||||
|
||||
|
||||
_NUM_PARSE_EXC: tuple[type[BaseException], ...] = (ValueError, TypeError)
|
||||
|
||||
|
||||
def _expand_unit(m: re.Match) -> str:
|
||||
num_str, unit = m.group(1), m.group(2).lower()
|
||||
unit_word = _unit_mapping.get(unit, unit)
|
||||
try:
|
||||
return f" {_expand_decimal(num_str)} {unit_word} "
|
||||
except _NUM_PARSE_EXC:
|
||||
return f" {num_str} {unit} "
|
||||
|
||||
|
||||
def _expand_percent(m: re.Match) -> str:
|
||||
try:
|
||||
return f" {_expand_decimal(m.group(1))} percent "
|
||||
except _NUM_PARSE_EXC:
|
||||
return f" {m.group(1)} percent "
|
||||
|
||||
|
||||
def _expand_dollars(m: re.Match) -> str:
|
||||
raw = m.group(1)
|
||||
clean = raw.lstrip("-") or "0"
|
||||
try:
|
||||
word = _expand_decimal(raw)
|
||||
value = float(clean)
|
||||
unit = "dollar" if abs(value) == 1.0 else "dollars"
|
||||
return f" {word} {unit} "
|
||||
except _NUM_PARSE_EXC:
|
||||
return f" {clean} dollars "
|
||||
|
||||
|
||||
def _expand_pounds(m: re.Match) -> str:
|
||||
raw = m.group(1)
|
||||
clean = raw.lstrip("-") or "0"
|
||||
try:
|
||||
word = _expand_decimal(raw)
|
||||
value = float(clean)
|
||||
unit = "pound" if abs(value) == 1.0 else "pounds"
|
||||
return f" {word} {unit} "
|
||||
except _NUM_PARSE_EXC:
|
||||
return f" {clean} pounds "
|
||||
|
||||
|
||||
def _expand_fraction(m: re.Match) -> str:
|
||||
p = _get_inflect()
|
||||
if p is None:
|
||||
return m.group(0)
|
||||
try:
|
||||
num, den = int(m.group(1)), int(m.group(2))
|
||||
if num == 1 and den == 2:
|
||||
return " one half "
|
||||
if num == 1 and den == 4:
|
||||
return " one quarter "
|
||||
if den == 2:
|
||||
plural = " half" if num == 1 else " halves"
|
||||
return f" {p.number_to_words(num)}{plural} "
|
||||
if den == 4:
|
||||
plural = " quarter" if num == 1 else " quarters"
|
||||
return f" {p.number_to_words(num)}{plural} "
|
||||
ordinal = p.ordinal(p.number_to_words(den))
|
||||
return f" {p.number_to_words(num)} {ordinal} "
|
||||
except _NUM_PARSE_EXC:
|
||||
return f" {m.group(1)} over {m.group(2)} "
|
||||
|
||||
|
||||
def _expand_ordinal(m: re.Match) -> str:
|
||||
try:
|
||||
num = int(re.sub(r"(st|and|rd|th)", "", m.group(0)))
|
||||
return f" {_num_to_words(num)} "
|
||||
except _NUM_PARSE_EXC:
|
||||
return m.group(0)
|
||||
|
||||
|
||||
def _expand_number(m: re.Match) -> str:
|
||||
try:
|
||||
return f" {_expand_decimal(m.group(0))} "
|
||||
except _NUM_PARSE_EXC:
|
||||
return f" {m.group(0)} "
|
||||
|
||||
|
||||
def _expand_version(m: re.Match) -> str:
|
||||
prefix, _, num_str = m.group(1), m.group(2), m.group(3)
|
||||
try:
|
||||
word = _expand_decimal(num_str)
|
||||
except _NUM_PARSE_EXC:
|
||||
return m.group(0)
|
||||
return f"{prefix} {word}"
|
||||
|
||||
|
||||
def normalize_numbers(text: str) -> str:
|
||||
"""Expand English numbers, currencies, units, etc. to words.
|
||||
|
||||
Returns text unchanged if `inflect` package is not installed.
|
||||
"""
|
||||
if _get_inflect() is None:
|
||||
return text
|
||||
text = re.sub(_comma_number_re, _remove_commas, text)
|
||||
text = re.sub(_unit_re, _expand_unit, text)
|
||||
text = re.sub(_pounds_re, _expand_pounds, text)
|
||||
text = re.sub(_dollars_re, _expand_dollars, text)
|
||||
text = re.sub(_fraction_re, _expand_fraction, text)
|
||||
text = re.sub(_percent_number_re, _expand_percent, text)
|
||||
text = re.sub(_ordinal_re, _expand_ordinal, text)
|
||||
text = re.sub(_version_re, _expand_version, text)
|
||||
text = re.sub(_number_re, _expand_number, text)
|
||||
text = re.sub(_whitespace_re, " ", text)
|
||||
return text.strip()
|
||||
|
||||
|
||||
# Top-level API
|
||||
def segment_and_normalize(
|
||||
text: str,
|
||||
max_length: int = _DEFAULT_MAX_SEMANTIC_LENGTH,
|
||||
) -> list[str]:
|
||||
"""Segment text into fragments and expand English numbers for Ming TTS.
|
||||
|
||||
This function cuts text by semantic length directly rather than following
|
||||
the streaming algorithm to detect sentence boundaries in the upstream
|
||||
Ming repo (which is more aggressively splitting at commas). It produces
|
||||
fewer and larger segments at natural sentence boundaries.
|
||||
"""
|
||||
if not text or not text.strip():
|
||||
return []
|
||||
|
||||
segments = cut_text_by_semantic_length(text.strip(), max_length)
|
||||
|
||||
normalized: list[str] = []
|
||||
for seg in segments:
|
||||
if not is_chinese(seg):
|
||||
seg = normalize_numbers(seg)
|
||||
if seg and seg[0] == ",":
|
||||
seg = seg[1:]
|
||||
seg = seg.strip()
|
||||
if seg:
|
||||
normalized.append(seg)
|
||||
|
||||
return normalized if normalized else [text.strip()]
|
||||
@@ -0,0 +1,289 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Copyright 2025 The vLLM-Omni team.
|
||||
# Copyright (c) Ant Group. All rights reserved.
|
||||
# Adapted from:
|
||||
# https://github.com/inclusionAI/Ming/blob/e58533db227031990c5a6864dcf5f08fb53ed0d2/modeling_bailing_talker.py
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
from functools import cached_property
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import soundfile as sf
|
||||
import torch
|
||||
from transformers.utils.hub import cached_file
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from vllm_omni.model_executor.model_loader.weight_utils import download_weights_from_hf_specific
|
||||
|
||||
from .spk_embedding import SpkembExtractor
|
||||
from .talker_module import resample
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .audio_vae import AudioVAE
|
||||
from .talker_module import Aggregator
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class InvalidPromptWavError(ValueError):
|
||||
"""Prompt wav failed local validation and can be skipped in list mode."""
|
||||
|
||||
|
||||
class VoicePresetRegistry:
|
||||
"""Loader and registry for Ming voice presets."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
talker_dir: str,
|
||||
model_path: str,
|
||||
download_dir: str | None,
|
||||
audio_vae: AudioVAE | None,
|
||||
aggregator: Aggregator,
|
||||
spk_head: torch.nn.Module,
|
||||
patch_size: int,
|
||||
) -> None:
|
||||
self._talker_dir = talker_dir
|
||||
self._model_path = model_path
|
||||
self._download_dir = download_dir
|
||||
self._audio_vae = audio_vae
|
||||
self._aggregator = aggregator
|
||||
self._spk_head = spk_head
|
||||
self._patch_size = patch_size
|
||||
|
||||
self.registered: dict[str, dict[str, Any]] = {}
|
||||
|
||||
def __contains__(self, voice_name: str) -> bool:
|
||||
return voice_name in self.registered
|
||||
|
||||
def get(self, voice_name: str) -> dict[str, Any] | None:
|
||||
return self.registered.get(voice_name)
|
||||
|
||||
def register(
|
||||
self,
|
||||
voice_name: str,
|
||||
prompt_wav_path: str | list[str],
|
||||
*,
|
||||
device: torch.device,
|
||||
dtype: torch.dtype,
|
||||
) -> None:
|
||||
"""Register a voice preset from one or more reference wav files.
|
||||
|
||||
Args:
|
||||
voice_name: Key under which to store the preset.
|
||||
prompt_wav_path: Single wav path or a list (multi-clip mode skips
|
||||
invalid entries with a warning instead of raising).
|
||||
device: Target device for cached prompt latents / projected
|
||||
speaker embeddings.
|
||||
dtype: Target dtype for the projected speaker embedding head.
|
||||
"""
|
||||
paths = self._normalize_paths(voice_name, prompt_wav_path)
|
||||
allow_partial = len(paths) > 1
|
||||
|
||||
vae_sr = int(self._audio_vae.config.sample_rate) if self._audio_vae else 44100
|
||||
if self._audio_vae is None:
|
||||
logger.warning(
|
||||
"Voice preset '%s' being registered without AudioVAE features",
|
||||
voice_name,
|
||||
)
|
||||
|
||||
speech_chunks: list[torch.Tensor] = []
|
||||
spk_emb_list: list[torch.Tensor] = []
|
||||
for wav_path in paths:
|
||||
try:
|
||||
speech_for_vae, raw_emb = self._load_single_wav(voice_name, wav_path, vae_sr)
|
||||
except (FileNotFoundError, InvalidPromptWavError) as e:
|
||||
if allow_partial:
|
||||
logger.warning(
|
||||
"Voice preset '%s': skipping invalid prompt wav %s: %s",
|
||||
voice_name,
|
||||
wav_path,
|
||||
e,
|
||||
)
|
||||
continue
|
||||
raise
|
||||
speech_chunks.append(speech_for_vae)
|
||||
if raw_emb is not None:
|
||||
projected = self._spk_head(raw_emb.to(device=device, dtype=dtype))
|
||||
spk_emb_list.append(projected)
|
||||
|
||||
if not speech_chunks:
|
||||
raise RuntimeError(f"Failed to register voice preset '{voice_name}': no valid prompt wavs remained")
|
||||
if not spk_emb_list and self._audio_vae is None:
|
||||
raise RuntimeError(
|
||||
f"Failed to register voice preset '{voice_name}': neither speaker "
|
||||
"embeddings nor AudioVAE prompt features are available"
|
||||
)
|
||||
|
||||
prompt_wav_lat, prompt_wav_emb = self._build_wav_embeddings(
|
||||
voice_name, torch.cat(speech_chunks, dim=-1), device=device
|
||||
)
|
||||
|
||||
if voice_name in self.registered:
|
||||
logger.warning("Voice preset '%s' is being overwritten", voice_name)
|
||||
self.registered[voice_name] = {
|
||||
"prompt_wav_lat": prompt_wav_lat,
|
||||
"prompt_wav_emb": prompt_wav_emb,
|
||||
"spk_emb": spk_emb_list,
|
||||
}
|
||||
logger.info("Registered voice preset '%s' from %s", voice_name, paths)
|
||||
|
||||
def load_presets_from_manifest(self, *, device: torch.device, dtype: torch.dtype) -> None:
|
||||
"""Resolve voice_name.json on disk or HF hub and register all entries.
|
||||
|
||||
Each entry is registered onto the supplied device and dtype.
|
||||
"""
|
||||
voice_json_path, base_dir = self._locate_manifest()
|
||||
if voice_json_path is None:
|
||||
logger.info("No voice_name.json found; voice presets unavailable")
|
||||
return
|
||||
|
||||
with open(voice_json_path) as f:
|
||||
voice_dict = json.load(f)
|
||||
|
||||
for name, info in voice_dict.items():
|
||||
wav_path = info.get("prompt_wav_path", "")
|
||||
prompt_text = info.get("prompt_text", "")
|
||||
if not wav_path:
|
||||
logger.warning("Voice preset '%s' has no prompt_wav_path, skipping", name)
|
||||
continue
|
||||
if not os.path.isabs(wav_path):
|
||||
wav_path = os.path.join(base_dir, wav_path)
|
||||
if not os.path.isfile(wav_path):
|
||||
logger.warning("Voice preset '%s': wav not found at %s, skipping", name, wav_path)
|
||||
continue
|
||||
try:
|
||||
self.register(name, wav_path, device=device, dtype=dtype)
|
||||
self.registered[name]["prompt_text"] = prompt_text
|
||||
except Exception as e: # pragma: no cover — manifest is best-effort
|
||||
logger.warning("Failed to register voice preset '%s': %s", name, e)
|
||||
|
||||
@cached_property
|
||||
def _spkemb_extractor(self) -> SpkembExtractor:
|
||||
"""Lazily resolve the CAMPPlus ONNX extractor."""
|
||||
for candidate in (self._talker_dir, self._model_path):
|
||||
path = os.path.join(candidate, "campplus.onnx")
|
||||
if os.path.isfile(path):
|
||||
extractor = SpkembExtractor(path)
|
||||
logger.info("Initialized SpkembExtractor from %s", path)
|
||||
return extractor
|
||||
try:
|
||||
path = cached_file(self._model_path, "campplus.onnx", subfolder="talker")
|
||||
except Exception as e:
|
||||
raise RuntimeError("campplus.onnx not found. Expected at <model_path>/talker/campplus.onnx") from e
|
||||
extractor = SpkembExtractor(path)
|
||||
logger.info("Initialized SpkembExtractor from %s", path)
|
||||
return extractor
|
||||
|
||||
@staticmethod
|
||||
def _normalize_paths(voice_name: str, prompt_wav_path: str | list[str]) -> list[str]:
|
||||
if not isinstance(voice_name, str) or not voice_name.strip():
|
||||
raise ValueError("voice_name must be a non-empty string")
|
||||
if isinstance(prompt_wav_path, str):
|
||||
paths = [prompt_wav_path]
|
||||
elif isinstance(prompt_wav_path, list):
|
||||
paths = list(prompt_wav_path)
|
||||
else:
|
||||
raise TypeError("prompt_wav_path must be a string path or a list of string paths")
|
||||
paths = [p.strip() for p in paths]
|
||||
if not paths or any(not p for p in paths):
|
||||
raise ValueError("Provided audio path is invalid")
|
||||
return paths
|
||||
|
||||
def _load_single_wav(self, voice_name: str, wav_path: str, vae_sr: int) -> tuple[torch.Tensor, torch.Tensor | None]:
|
||||
"""Return ``(speech_for_vae, raw_spk_emb_or_none)``.
|
||||
|
||||
Stays device-agnostic — both returned tensors live on CPU; the caller
|
||||
moves them to the target device when projecting / encoding.
|
||||
"""
|
||||
if not os.path.isfile(wav_path):
|
||||
raise FileNotFoundError(f"prompt wav not found: {wav_path}")
|
||||
|
||||
data, sample_rate = sf.read(wav_path, dtype="float32")
|
||||
speech_tmp = torch.from_numpy(data)
|
||||
if speech_tmp.ndim == 1:
|
||||
speech_tmp = speech_tmp.unsqueeze(0)
|
||||
elif speech_tmp.ndim == 2:
|
||||
num_channels = speech_tmp.shape[1]
|
||||
if num_channels > 1:
|
||||
logger.warning(
|
||||
"Voice preset '%s': downmixing %d-channel audio at %s to mono",
|
||||
voice_name,
|
||||
num_channels,
|
||||
wav_path,
|
||||
)
|
||||
speech_tmp = speech_tmp.mean(dim=1, keepdim=True).T
|
||||
else:
|
||||
raise InvalidPromptWavError(f"unsupported audio shape {tuple(speech_tmp.shape)} for {wav_path}")
|
||||
|
||||
if not torch.isfinite(speech_tmp).all():
|
||||
raise InvalidPromptWavError(f"audio file contains NaN or Inf samples: {wav_path}")
|
||||
|
||||
speech_for_vae = resample(speech_tmp, sample_rate, vae_sr)
|
||||
|
||||
# Speaker embedding (16 kHz CAMPPlus). If the extractor fails to
|
||||
# resolve (missing ONNX model), skip embedding extraction rather than
|
||||
# blocking VAE-only registration.
|
||||
raw_emb: torch.Tensor | None = None
|
||||
try:
|
||||
extractor = self._spkemb_extractor
|
||||
speech_for_spk = resample(speech_tmp, sample_rate, 16000)
|
||||
raw_emb = extractor(speech_for_spk)
|
||||
except RuntimeError:
|
||||
raw_emb = None
|
||||
return speech_for_vae, raw_emb
|
||||
|
||||
def _build_wav_embeddings(
|
||||
self,
|
||||
voice_name: str,
|
||||
speech: torch.Tensor,
|
||||
*,
|
||||
device: torch.device,
|
||||
) -> tuple[torch.Tensor | None, torch.Tensor | None]:
|
||||
if self._audio_vae is None:
|
||||
return None, None
|
||||
|
||||
patch_pt = self._audio_vae.encoder.hop_size * max(1, self._audio_vae.encoder.patch_size) * self._patch_size
|
||||
if speech.shape[-1] % patch_pt != 0:
|
||||
pad_len = (speech.shape[-1] + patch_pt - 1) // patch_pt * patch_pt
|
||||
pad_speech = torch.zeros((speech.shape[0], pad_len), dtype=speech.dtype, device=speech.device)
|
||||
pad_speech[:, -speech.shape[-1] :] = speech
|
||||
speech = pad_speech
|
||||
|
||||
prompt_wav_lat, _ = self._audio_vae.encode_latent(
|
||||
speech.to(dtype=torch.bfloat16, device=device),
|
||||
torch.tensor([speech.size(1)], dtype=torch.long, device=device),
|
||||
)
|
||||
assert prompt_wav_lat.shape[1] % self._patch_size == 0, (
|
||||
f"AudioVAE latent length is incompatible with patch_size for voice preset '{voice_name}'"
|
||||
)
|
||||
prompt_wav_lat = prompt_wav_lat.reshape(-1, self._patch_size, prompt_wav_lat.shape[-1])
|
||||
prompt_wav_emb = self._aggregator(prompt_wav_lat)
|
||||
prompt_wav_lat = prompt_wav_lat.reshape(1, -1, prompt_wav_lat.shape[-1])
|
||||
prompt_wav_emb = prompt_wav_emb.reshape(1, -1, prompt_wav_emb.shape[-1])
|
||||
return prompt_wav_lat, prompt_wav_emb
|
||||
|
||||
def _locate_manifest(self) -> tuple[str | None, str | None]:
|
||||
for candidate in (self._talker_dir, self._model_path):
|
||||
path = os.path.join(candidate, "data", "voice_name.json")
|
||||
if os.path.isfile(path):
|
||||
return path, candidate
|
||||
|
||||
if not os.path.isdir(self._model_path):
|
||||
try:
|
||||
hf_root = download_weights_from_hf_specific(
|
||||
self._model_path,
|
||||
self._download_dir,
|
||||
allow_patterns=["talker/data/**"],
|
||||
require_all=True,
|
||||
)
|
||||
candidate = os.path.join(hf_root, "talker", "data", "voice_name.json")
|
||||
if os.path.isfile(candidate):
|
||||
return candidate, os.path.join(hf_root, "talker")
|
||||
except Exception as e: # pragma: no cover
|
||||
logger.info("Could not download voice presets from HF: %s", e)
|
||||
|
||||
return None, None
|
||||
@@ -185,6 +185,11 @@ _OMNI_MODELS = {
|
||||
"ming_flash_omni_thinker",
|
||||
"MingFlashOmniThinkerForConditionalGeneration",
|
||||
),
|
||||
"MingFlashOmniTalkerForConditionalGeneration": (
|
||||
"ming_flash_omni",
|
||||
"ming_flash_omni_talker",
|
||||
"MingFlashOmniTalkerForConditionalGeneration",
|
||||
),
|
||||
# Alias: HF repo currently ships this architecture name in config.json
|
||||
"BailingMM2NativeForConditionalGeneration": (
|
||||
"ming_flash_omni",
|
||||
|
||||
Reference in new issue
Block a user