[Model] Add Ming-omni-tts dense 0.5B pipeline (#2906)

Signed-off-by: akshatvishu <akshatnayak197@gmail.com>
Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
This commit is contained in:
akshatvishuandHongsheng Liu authored and GitHub committed 2026-06-10 20:11:04 +08:00
1 parent 39ae6e12f8
commit 2fa4c7a4a9
55 files changed
+6263 -549

No files matched your search

@@ -1,392 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# Copyright 2025 The vLLM-Omni team.
# Copyright (c) Ant Group. All rights reserved.
# Adapted from:
# https://github.com/inclusionAI/Ming/tree/e58533db227031990c5a6864dcf5f08fb53ed0d2/AudioVAE
from __future__ import annotations
import torch
import torch.nn as nn
import torch.nn.functional as F
from transformers import PretrainedConfig, PreTrainedModel, Qwen2Config, Qwen2Model
from transformers.utils import is_flash_attn_2_available
from vllm.logger import init_logger
logger = init_logger(__name__)
class AudioVAEConfig(PretrainedConfig):
model_type = "audio_vae"
def __init__(
self,
sample_rate: int = 44100,
enc_kwargs: dict | None = None,
dec_kwargs: dict | None = None,
init_method: str = "kaiming",
patch_size: int = 4,
**kwargs,
):
self.sample_rate = sample_rate
self.enc_kwargs = enc_kwargs or {}
self.dec_kwargs = dec_kwargs or {}
self.init_method = init_method
self.patch_size = patch_size
super().__init__(**kwargs)
class ISTFT(nn.Module):
def __init__(self, n_fft: int, hop_length: int, win_length: int, padding: str = "same"):
super().__init__()
if padding not in ["center", "same"]:
raise ValueError("Padding must be 'center' or 'same'.")
self.padding = padding
self.n_fft = n_fft
self.hop_length = hop_length
self.win_length = win_length
window = torch.hann_window(win_length)
self.register_buffer("window", window)
self.buffer_len = self.win_length - self.hop_length
def _buffer_process(self, x, buffer, pad, last_chunk=False, streaming=False):
if streaming:
if buffer is None:
x = x[:, pad:]
if buffer is not None:
x[:, : self.buffer_len] += buffer
buffer = x[:, -self.buffer_len :]
if not last_chunk:
x = x[:, : -self.buffer_len]
else:
x = x[:, :-pad]
else:
x = x[:, pad:-pad]
return x, buffer
def forward(self, spec, audio_buffer=None, window_buffer=None, streaming=False, last_chunk=False):
if self.padding == "center":
return torch.istft(spec, self.n_fft, self.hop_length, self.win_length, self.window, center=True)
elif self.padding == "same":
pad = (self.win_length - self.hop_length) // 2
else:
raise ValueError("Padding must be 'center' or 'same'.")
B, N, T = spec.shape
ifft = torch.fft.irfft(spec, self.n_fft, dim=1, norm="backward")
ifft = ifft * self.window[None, :, None]
output_size = (T - 1) * self.hop_length + self.win_length
y = torch.nn.functional.fold(
ifft,
output_size=(1, output_size),
kernel_size=(1, self.win_length),
stride=(1, self.hop_length),
)[:, 0, 0, :]
y, audio_buffer = self._buffer_process(y, audio_buffer, pad, last_chunk=last_chunk, streaming=streaming)
window_sq = self.window.square().expand(1, T, -1).transpose(1, 2)
window_envelope = (
torch.nn.functional.fold(
window_sq,
output_size=(1, output_size),
kernel_size=(1, self.win_length),
stride=(1, self.hop_length),
)
.squeeze(0)
.squeeze(0)
)
window_envelope, window_buffer = self._buffer_process(
window_envelope, window_buffer, pad, last_chunk=last_chunk, streaming=streaming
)
window_envelope = window_envelope.squeeze()
assert (window_envelope > 1e-11).all()
y = y / window_envelope
return y, audio_buffer, window_buffer
class ISTFTHead(nn.Module):
def __init__(self, dim: int, n_fft: int, hop_length: int, padding: str = "same"):
super().__init__()
out_dim = n_fft + 2
self.out = nn.Linear(dim, out_dim)
self.istft = ISTFT(n_fft=n_fft, hop_length=hop_length, win_length=n_fft, padding=padding)
def forward(self, x, audio_buffer=None, window_buffer=None, streaming=False, last_chunk=False):
x_pred = self.out(x)
x_pred = x_pred.transpose(1, 2)
mag, p = x_pred.chunk(2, dim=1)
mag = torch.exp(mag)
mag = torch.clip(mag, max=1e2)
x = torch.cos(p)
y = torch.sin(p)
S = mag * (x + 1j * y)
audio, audio_buffer, window_buffer = self.istft(
S, audio_buffer=audio_buffer, window_buffer=window_buffer, streaming=streaming, last_chunk=last_chunk
)
return audio.unsqueeze(1), x_pred, audio_buffer, window_buffer
class StreamingLinearUpsample(nn.Module):
def __init__(self, scale_factor=4):
super().__init__()
self.scale_factor = scale_factor
self.upsampler = nn.Upsample(scale_factor=scale_factor, mode="linear", align_corners=False)
def forward(self, x, state=None, is_last=False):
if state is None:
state = {"prev_chunk": None, "history_last": None, "is_first": True}
if x is None and not is_last:
return None, state
if state["is_first"] and is_last:
out = self.upsampler(x.transpose(1, 2)).transpose(1, 2)
return out, None
output_chunks = []
if state["is_first"]:
state["prev_chunk"] = x
state["is_first"] = False
if not is_last:
return None, state
if state["prev_chunk"] is not None:
p = state["prev_chunk"].transpose(1, 2)
if state["history_last"] is None:
lookahead = x[:, :1, :].transpose(1, 2)
inp = torch.cat([p, lookahead], dim=2)
up = self.upsampler(inp)
out_prev = up[:, :, : p.size(2) * self.scale_factor]
else:
lookahead = x[:, :1, :].transpose(1, 2)
inp = torch.cat([state["history_last"], p, lookahead], dim=2)
up = self.upsampler(inp)
start = self.scale_factor
end = start + p.size(2) * self.scale_factor
out_prev = up[:, :, start:end]
output_chunks.append(out_prev.transpose(1, 2))
state["history_last"] = p[:, :, -1:]
state["prev_chunk"] = x
if is_last:
p = state["prev_chunk"].transpose(1, 2)
inp = torch.cat([state["history_last"], p], dim=2)
up = self.upsampler(inp)
out_last = up[:, :, self.scale_factor :]
output_chunks.append(out_last.transpose(1, 2))
state = None
final_out = torch.cat(output_chunks, dim=1) if output_chunks else None
return final_out, state
class Decoder(nn.Module):
def __init__(self, decoder_args, output_dim=320, latent_dim=64, patch_size=-1):
super().__init__()
config = Qwen2Config.from_dict(config_dict=decoder_args)
if is_flash_attn_2_available():
config._attn_implementation_autoset = True
config._attn_implementation = "flash_attention_2"
else:
config._attn_implementation = "sdpa"
logger.info("AudioVAE Decoder: using attn_implementation=%r", config._attn_implementation)
self.decoder = Qwen2Model(config)
self.output_dim = output_dim
self.latent_dim = latent_dim
self.fc1 = nn.Linear(latent_dim, config.hidden_size)
self.hop_length = output_dim
self.head = ISTFTHead(
dim=config.hidden_size, n_fft=self.hop_length * 4, hop_length=self.hop_length, padding="same"
)
self.patch_size = patch_size
if self.patch_size != -1:
self.upsampling = StreamingLinearUpsample(scale_factor=patch_size)
def low_level_reconstruct(self, x, past_key_values=None, use_cache=False, stream_state=None, last_chunk=False):
upsample_state, audio_buffer, window_buffer = stream_state
bsz, device, dtype = x.size(0), x.device, x.dtype
x = self.fc1(x)
if self.patch_size != -1:
if use_cache:
x, upsample_state = self.upsampling(x, state=upsample_state, is_last=last_chunk)
if x is None:
stream_state = (upsample_state, audio_buffer, window_buffer)
return torch.empty(bsz, 1, 0, device=device, dtype=dtype), stream_state, past_key_values
else:
x = self.upsampling.upsampler(x.transpose(1, 2)).transpose(1, 2)
hidden_states_list = []
if use_cache and getattr(self.decoder.config, "sliding_window", None) is not None:
sw_size = self.decoder.config.sliding_window
target_len = sw_size - 1
if past_key_values is None:
past_len = 0
elif hasattr(past_key_values, "get_seq_length"):
past_len = past_key_values.get_seq_length()
elif isinstance(past_key_values, tuple) and len(past_key_values) > 0:
past_len = past_key_values[0][0].shape[-2]
else:
past_len = 0
curr_len = x.shape[1]
if past_len < target_len and (past_len + curr_len) >= sw_size:
fill_len = target_len - past_len
x_fill = x[:, :fill_len, :]
outputs = self.decoder(inputs_embeds=x_fill, past_key_values=past_key_values, use_cache=use_cache)
hidden_states_list.append(outputs.last_hidden_state)
past_key_values = outputs.past_key_values
x = x[:, fill_len:, :]
outputs = self.decoder(inputs_embeds=x, past_key_values=past_key_values, use_cache=use_cache)
hidden_states_list.append(outputs.last_hidden_state)
past_key_values = outputs.past_key_values
if len(hidden_states_list) > 1:
full_hidden_state = torch.cat(hidden_states_list, dim=1)
else:
full_hidden_state = hidden_states_list[0]
x_out, _, audio_buffer, window_buffer = self.head(
full_hidden_state,
streaming=use_cache,
audio_buffer=audio_buffer,
window_buffer=window_buffer,
last_chunk=last_chunk,
)
stream_state = (upsample_state, audio_buffer, window_buffer)
return x_out, stream_state, past_key_values
class Encoder(nn.Module):
def __init__(self, encoder_args, input_dim=320, hop_size=320, latent_dim=64, patch_size=-1):
super().__init__()
config = Qwen2Config.from_dict(config_dict=encoder_args)
if is_flash_attn_2_available():
config._attn_implementation_autoset = True
config._attn_implementation = "flash_attention_2"
else:
config._attn_implementation = "sdpa"
logger.info("AudioVAE Encoder: using attn_implementation=%r", config._attn_implementation)
self.encoder = Qwen2Model(config)
self.input_dim = input_dim
self.hop_size = hop_size
self.latent_dim = latent_dim
self.fc1 = nn.Linear(input_dim, config.hidden_size, bias=False)
self.fc2 = nn.Linear(config.hidden_size, config.hidden_size)
self.fc3 = nn.Linear(config.hidden_size, latent_dim * 2)
self.norm = nn.LayerNorm(config.hidden_size)
self.patch_size = patch_size
if patch_size != -1:
config.num_hidden_layers = 4
self.aggregator = Qwen2Model(config)
self.cls_embed = nn.Parameter(torch.rand(1, 1, config.hidden_size))
self.cls_embed.data.normal_(0, 0.02)
def get_frames(self, x):
num_frames_total = (x.size(-1) + self.hop_size - 1) // self.hop_size
expected_len = (num_frames_total - 1) * self.hop_size + self.input_dim
padding_needed = expected_len - x.size(-1)
waveform = F.pad(x, (0, padding_needed), value=0.0)
frames = waveform.unfold(dimension=-1, size=self.input_dim, step=self.hop_size)
return frames
def pad_patch_insert_cls(self, x):
bsz, _, dim = x.size()
num_frame = x.size(1)
r = num_frame % self.patch_size
pad_num = self.patch_size - r if r else 0
x = F.pad(x, (0, 0, 0, pad_num), value=0.0)
x = x.reshape(-1, self.patch_size, dim)
x = torch.cat((x, self.cls_embed.expand(x.size(0), -1, -1)), dim=1)
x = x.reshape(bsz, -1, dim)
return x
def forward(self, waveform):
x = self.get_frames(waveform)
x = self.fc1(x)
x = self.fc2(x)
x = self.encoder(inputs_embeds=x)
x = x.last_hidden_state
if self.patch_size != -1:
x = self.pad_patch_insert_cls(x)
x = self.aggregator(inputs_embeds=x)
x = x.last_hidden_state
bsz, _, dim = x.size()
x = x.reshape(-1, self.patch_size + 1, dim)
x = x[:, -1:, :].reshape(bsz, -1, dim)
x = self.fc3(x)
return x, waveform.unsqueeze(1)
class AudioVAE(PreTrainedModel):
config_class = AudioVAEConfig
def __init__(self, config: AudioVAEConfig):
super().__init__(config)
self.encoder = Encoder(
encoder_args=config.enc_kwargs["backbone"],
input_dim=config.enc_kwargs["input_dim"],
hop_size=config.enc_kwargs.get("hop_size", 320),
latent_dim=config.enc_kwargs["latent_dim"],
patch_size=config.patch_size,
)
self.decoder = Decoder(
decoder_args=config.dec_kwargs["backbone"],
output_dim=config.dec_kwargs["output_dim"],
latent_dim=config.dec_kwargs["latent_dim"],
patch_size=config.patch_size,
)
self.post_init()
def _init_weights(self, module):
std = 0.02
if isinstance(module, nn.Linear):
if self.config.init_method == "kaiming":
nn.init.kaiming_normal_(module.weight, mode="fan_in", nonlinearity="relu")
else:
module.weight.data.normal_(mean=0.0, std=std)
if module.bias is not None:
module.bias.data.zero_()
elif isinstance(module, nn.Embedding):
module.weight.data.normal_(mean=0.0, std=std)
if module.padding_idx is not None:
module.weight.data[module.padding_idx].zero_()
def encode_latent(self, waveform, waveform_length):
from diffusers.models.autoencoders.autoencoder_oobleck import OobleckDiagonalGaussianDistribution
frame_num = torch.ceil(waveform_length / self.config.enc_kwargs["input_dim"]).to(torch.int32)
if self.config.patch_size != -1:
frame_num = torch.ceil(frame_num / self.config.patch_size)
h, y = self.encoder(waveform)
h = h.transpose(1, 2)
posterior = OobleckDiagonalGaussianDistribution(h)
latent = posterior.sample()
latent = latent.transpose(1, 2)
return latent, frame_num
def decode(self, latent, past_key_values=None, use_cache=False, stream_state=(None, None, None), last_chunk=False):
waveform, stream_state, past_key_values = self.decoder.low_level_reconstruct(
latent,
past_key_values=past_key_values,
use_cache=use_cache,
stream_state=stream_state,
last_chunk=last_chunk,
)
return waveform, stream_state, past_key_values
@@ -26,10 +26,10 @@ from vllm.sequence import IntermediateTensors
from vllm_omni.model_executor.custom_process_mixin import CustomProcessMixin
from vllm_omni.model_executor.model_loader.weight_utils import download_weights_from_hf_specific
from vllm_omni.model_executor.models.common.ming.audio_vae import AudioVAE, AudioVAEConfig
from vllm_omni.model_executor.models.output_templates import OmniOutput
from vllm_omni.transformers_utils.configs.ming_flash_omni import MingFlashOmniTalkerConfig
from .audio_vae import AudioVAE, AudioVAEConfig
from .prompt_utils import DEFAULT_PROMPT as MING_DEFAULT_PROMPT
from .talker_module import CFM, Aggregator, DiT, MingAudioGenerator, build_tts_input
from .text_processing import segment_and_normalize
@@ -697,6 +697,9 @@ class MingFlashOmniThinkerForConditionalGeneration(
f"image_gen_scales={list(self.query_tokens_dict.keys())}"
)
def get_language_model(self):
return self.language_model
def _load_image_gen_query_tokens(self, model_path: str) -> None:
"""Load learnable query tokens used for image generation.
@@ -110,12 +110,18 @@ BASE_CAPTION_TEMPLATE: dict[str, Any] = {
}
def create_instruction(user_input: dict[str, Any]) -> str:
def create_instruction(user_input: Any) -> str | None:
"""Return a JSON caption string for ``audio_sequence[0]``.
Only keys already present on the base template are merged in; unknown
keys are silently ignored to keep the output schema stable.
"""
if user_input is None:
return None
if isinstance(user_input, str):
return user_input
if not isinstance(user_input, dict):
raise ValueError(f"Ming instruction must be str, dict, or None; got {type(user_input).__name__}")
caption = copy.deepcopy(BASE_CAPTION_TEMPLATE)
item = caption["audio_sequence"][0]
for key, value in user_input.items():
@@ -27,223 +27,19 @@ from threading import Lock
import torch
import torch.nn as nn
import torch.nn.functional as F
from transformers import PreTrainedTokenizerBase, Qwen2Config, Qwen2Model, StaticCache
from vllm.logger import init_logger
from vllm.platforms import current_platform
from x_transformers.x_transformers import RotaryEmbedding, apply_rotary_pos_emb
from x_transformers.x_transformers import RotaryEmbedding
from vllm_omni.model_executor.layers.timestep_embedding import DiTTimestepEmbedding
from .audio_vae import AudioVAE
from vllm_omni.model_executor.models.common.ming.audio_vae import AudioVAE
from vllm_omni.model_executor.models.common.ming.dit import CondEmbedder, DiTBlock, FinalLayer, get_epss_timesteps
from vllm_omni.model_executor.models.common.ming.fm import apply_sway_sampling, integrate_cfm_steps
logger = init_logger(__name__)
########################################################################
# DiT Modules
# Ported from:
# https://github.com/inclusionAI/Ming/blob/e58533db227031990c5a6864dcf5f08fb53ed0d2/talker_module/modules.py
# Ported from:
# https://github.com/inclusionAI/Ming/blob/e58533db227031990c5a6864dcf5f08fb53ed0d2/talker_module/dit.py
########################################################################
class RMSNorm(nn.Module):
def __init__(self, dim: int, eps: float = 1e-6):
super().__init__()
self.eps = eps
self.weight = nn.Parameter(torch.ones(dim))
def forward(self, x: torch.Tensor) -> torch.Tensor:
if self.weight.dtype in [torch.float16, torch.bfloat16]:
x = x.to(self.weight.dtype)
x = F.rms_norm(x, normalized_shape=(x.shape[-1],), weight=self.weight, eps=self.eps)
return x
class FeedForward(nn.Module):
def __init__(
self, dim: int, dim_out: int | None = None, mult: float = 4, dropout: float = 0.0, approximate: str = "none"
):
super().__init__()
inner_dim = int(dim * mult)
dim_out = dim_out if dim_out is not None else dim
activation = nn.GELU(approximate=approximate)
project_in = nn.Sequential(nn.Linear(dim, inner_dim), activation)
self.ff = nn.Sequential(project_in, nn.Dropout(dropout), nn.Linear(inner_dim, dim_out))
def forward(self, x: torch.Tensor) -> torch.Tensor:
return self.ff(x)
class Attention(nn.Module):
def __init__(
self,
dim: int,
heads: int = 8,
dim_head: int = 64,
dropout: float = 0.0,
qk_norm: str | None = None,
pe_attn_head: int | None = None,
attn_mask_enabled: bool = True,
):
super().__init__()
self.dim = dim
self.heads = heads
self.inner_dim = dim_head * heads
self.dropout = dropout
self.to_q = nn.Linear(dim, self.inner_dim)
self.to_k = nn.Linear(dim, self.inner_dim)
self.to_v = nn.Linear(dim, self.inner_dim)
if qk_norm is None:
self.q_norm = None
self.k_norm = None
elif qk_norm == "rms_norm":
self.q_norm = RMSNorm(dim_head)
self.k_norm = RMSNorm(dim_head)
else:
raise ValueError(f"Unimplemented qk_norm: {qk_norm}")
self.to_out = nn.ModuleList([])
self.to_out.append(nn.Linear(self.inner_dim, dim))
self.to_out.append(nn.Dropout(dropout))
self.pe_attn_head = pe_attn_head
self.attn_mask_enabled = attn_mask_enabled
def forward(
self,
x: torch.Tensor,
mask: torch.Tensor | None = None,
rope: tuple[torch.Tensor, torch.Tensor | None] | None = None,
) -> torch.Tensor:
batch_size = x.shape[0]
query = self.to_q(x)
key = self.to_k(x)
value = self.to_v(x)
inner_dim = key.shape[-1]
head_dim = inner_dim // self.heads
query = query.view(batch_size, -1, self.heads, head_dim).transpose(1, 2)
key = key.view(batch_size, -1, self.heads, head_dim).transpose(1, 2)
value = value.view(batch_size, -1, self.heads, head_dim).transpose(1, 2)
if self.q_norm is not None:
query = self.q_norm(query)
if self.k_norm is not None:
key = self.k_norm(key)
if rope is not None:
freqs, xpos_scale = rope
q_xpos_scale, k_xpos_scale = (xpos_scale, xpos_scale**-1.0) if xpos_scale is not None else (1.0, 1.0)
if self.pe_attn_head is not None:
on = self.pe_attn_head
query[:, :on, :, :] = apply_rotary_pos_emb(query[:, :on, :, :], freqs, q_xpos_scale)
key[:, :on, :, :] = apply_rotary_pos_emb(key[:, :on, :, :], freqs, k_xpos_scale)
else:
query = apply_rotary_pos_emb(query, freqs, q_xpos_scale)
key = apply_rotary_pos_emb(key, freqs, k_xpos_scale)
if self.attn_mask_enabled and mask is not None:
valid_sample_indices = mask.any(dim=1)
final_output = torch.zeros_like(query).to(query.device)
attn_mask = mask[valid_sample_indices]
query = query[valid_sample_indices]
key = key[valid_sample_indices]
value = value[valid_sample_indices]
attn_mask = attn_mask.unsqueeze(1).unsqueeze(1)
attn_mask = attn_mask.expand(valid_sample_indices.sum().item(), self.heads, query.shape[-2], key.shape[-2])
else:
attn_mask = None
x = F.scaled_dot_product_attention(query, key, value, attn_mask=attn_mask, dropout_p=0.0, is_causal=False)
if self.attn_mask_enabled and mask is not None:
final_output[valid_sample_indices] = x
x = final_output
x = x.transpose(1, 2).reshape(batch_size, -1, self.heads * head_dim)
x = x.to(query.dtype)
x = self.to_out[0](x)
x = self.to_out[1](x)
if mask is not None:
mask = mask.unsqueeze(-1)
x = x.masked_fill(~mask, 0.0)
return x
class DiTBlock(nn.Module):
"""A DiT block with pre-norm and residual connections."""
def __init__(
self,
hidden_size: int,
num_heads: int,
mlp_ratio: float = 4.0,
dropout: float = 0.1,
qk_norm: str | None = None,
pe_attn_head: int | None = None,
attn_mask_enabled: bool = True,
**kwargs,
):
super().__init__()
self.norm1 = RMSNorm(hidden_size)
self.attn = Attention(
dim=hidden_size,
heads=num_heads,
dim_head=hidden_size // num_heads,
dropout=dropout,
qk_norm=qk_norm,
pe_attn_head=pe_attn_head,
attn_mask_enabled=attn_mask_enabled,
)
self.norm2 = RMSNorm(hidden_size)
self.mlp = FeedForward(dim=hidden_size, mult=mlp_ratio, dropout=dropout, approximate="tanh")
def forward(
self,
x: torch.Tensor,
mask: torch.Tensor | None,
rope: tuple[torch.Tensor, torch.Tensor | None] | None,
) -> torch.Tensor:
x = x + self.attn(self.norm1(x), mask=mask, rope=rope)
x = x + self.mlp(self.norm2(x))
return x
class FinalLayer(nn.Module):
"""The final layer of DiT."""
def __init__(self, hidden_size: int, out_channels: int):
super().__init__()
self.norm_final = RMSNorm(hidden_size)
self.linear = nn.Linear(hidden_size, out_channels, bias=True)
def forward(self, x: torch.Tensor) -> torch.Tensor:
x = self.norm_final(x)
x = self.linear(x)
return x
class CondEmbedder(nn.Module):
"""Embeds LLM hidden states with optional CFG dropout."""
def __init__(self, input_feature_size: int, hidden_size: int):
super().__init__()
self.cond_embedder = nn.Linear(input_feature_size, hidden_size)
def forward(self, llm_cond: torch.Tensor) -> torch.Tensor:
return self.cond_embedder(llm_cond)
class DiT(nn.Module):
"""Diffusion model with a Transformer backbone for audio latent generation."""
@@ -324,29 +120,6 @@ class DiT(nn.Module):
return model_out[:, -x.shape[1] :, :]
#########################################################################################
# CFM
# Ported from:
# https://github.com/inclusionAI/Ming/blob/e58533db227031990c5a6864dcf5f08fb53ed0d2/talker_module/cfm.py
#########################################################################################
def get_epss_timesteps(n, device, dtype):
dt = 1 / 32
predefined_timesteps = {
5: [0, 2, 4, 8, 16, 32],
6: [0, 2, 4, 6, 8, 16, 32],
7: [0, 2, 4, 6, 8, 16, 24, 32],
10: [0, 2, 4, 6, 8, 12, 16, 20, 24, 28, 32],
12: [0, 2, 4, 6, 8, 10, 12, 14, 16, 20, 24, 28, 32],
16: [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 12, 14, 16, 20, 24, 28, 32],
}
t = predefined_timesteps.get(n, [])
if not t:
return torch.linspace(0, 1, n + 1, device=device, dtype=dtype)
return dt * torch.tensor(t, device=device, dtype=dtype)
class CFM(nn.Module):
"""Conditional Flow Matching module for audio latent generation."""
@@ -391,15 +164,8 @@ class CFM(nn.Module):
pred, null_pred = torch.chunk(pred_cfg, 2, dim=0)
return pred + (pred - null_pred) * sde_args[0]
if self.sway_sampling_coef is not None:
t = t + self.sway_sampling_coef * (torch.cos(torch.pi / 2 * t) - 1 + t)
for step in range(self.steps):
dt = t[step + 1] - t[step]
y0 = y0 + fn(t[step], y0) * dt
y0 = y0 + sde_args[1] * (sde_args[2] ** 0.5) * (dt.abs() ** 0.5) * sde_rnd[step]
return y0
t = apply_sway_sampling(t, self.sway_sampling_coef)
return integrate_cfm_steps(fn, y0, t, sde_args, sde_rnd, self.steps)
class CFMGraphExecutor:
@@ -22,7 +22,8 @@ from .spk_embedding import SpkembExtractor
from .talker_module import resample
if TYPE_CHECKING:
from .audio_vae import AudioVAE
from vllm_omni.model_executor.models.common.ming.audio_vae import AudioVAE
from .talker_module import Aggregator
logger = init_logger(__name__)