[Model] Add Ming-omni-tts dense 0.5B pipeline (#2906)
Signed-off-by: akshatvishu <akshatnayak197@gmail.com> Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
This commit is contained in:
1 parent
39ae6e12f8
commit
2fa4c7a4a9
55 files changed
+6263
-549
No files matched your search
@@ -1,392 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Copyright 2025 The vLLM-Omni team.
|
||||
# Copyright (c) Ant Group. All rights reserved.
|
||||
# Adapted from:
|
||||
# https://github.com/inclusionAI/Ming/tree/e58533db227031990c5a6864dcf5f08fb53ed0d2/AudioVAE
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
from transformers import PretrainedConfig, PreTrainedModel, Qwen2Config, Qwen2Model
|
||||
from transformers.utils import is_flash_attn_2_available
|
||||
from vllm.logger import init_logger
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class AudioVAEConfig(PretrainedConfig):
|
||||
model_type = "audio_vae"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
sample_rate: int = 44100,
|
||||
enc_kwargs: dict | None = None,
|
||||
dec_kwargs: dict | None = None,
|
||||
init_method: str = "kaiming",
|
||||
patch_size: int = 4,
|
||||
**kwargs,
|
||||
):
|
||||
self.sample_rate = sample_rate
|
||||
self.enc_kwargs = enc_kwargs or {}
|
||||
self.dec_kwargs = dec_kwargs or {}
|
||||
self.init_method = init_method
|
||||
self.patch_size = patch_size
|
||||
super().__init__(**kwargs)
|
||||
|
||||
|
||||
class ISTFT(nn.Module):
|
||||
def __init__(self, n_fft: int, hop_length: int, win_length: int, padding: str = "same"):
|
||||
super().__init__()
|
||||
if padding not in ["center", "same"]:
|
||||
raise ValueError("Padding must be 'center' or 'same'.")
|
||||
self.padding = padding
|
||||
self.n_fft = n_fft
|
||||
self.hop_length = hop_length
|
||||
self.win_length = win_length
|
||||
window = torch.hann_window(win_length)
|
||||
self.register_buffer("window", window)
|
||||
self.buffer_len = self.win_length - self.hop_length
|
||||
|
||||
def _buffer_process(self, x, buffer, pad, last_chunk=False, streaming=False):
|
||||
if streaming:
|
||||
if buffer is None:
|
||||
x = x[:, pad:]
|
||||
if buffer is not None:
|
||||
x[:, : self.buffer_len] += buffer
|
||||
buffer = x[:, -self.buffer_len :]
|
||||
if not last_chunk:
|
||||
x = x[:, : -self.buffer_len]
|
||||
else:
|
||||
x = x[:, :-pad]
|
||||
else:
|
||||
x = x[:, pad:-pad]
|
||||
return x, buffer
|
||||
|
||||
def forward(self, spec, audio_buffer=None, window_buffer=None, streaming=False, last_chunk=False):
|
||||
if self.padding == "center":
|
||||
return torch.istft(spec, self.n_fft, self.hop_length, self.win_length, self.window, center=True)
|
||||
elif self.padding == "same":
|
||||
pad = (self.win_length - self.hop_length) // 2
|
||||
else:
|
||||
raise ValueError("Padding must be 'center' or 'same'.")
|
||||
|
||||
B, N, T = spec.shape
|
||||
ifft = torch.fft.irfft(spec, self.n_fft, dim=1, norm="backward")
|
||||
ifft = ifft * self.window[None, :, None]
|
||||
|
||||
output_size = (T - 1) * self.hop_length + self.win_length
|
||||
y = torch.nn.functional.fold(
|
||||
ifft,
|
||||
output_size=(1, output_size),
|
||||
kernel_size=(1, self.win_length),
|
||||
stride=(1, self.hop_length),
|
||||
)[:, 0, 0, :]
|
||||
|
||||
y, audio_buffer = self._buffer_process(y, audio_buffer, pad, last_chunk=last_chunk, streaming=streaming)
|
||||
|
||||
window_sq = self.window.square().expand(1, T, -1).transpose(1, 2)
|
||||
window_envelope = (
|
||||
torch.nn.functional.fold(
|
||||
window_sq,
|
||||
output_size=(1, output_size),
|
||||
kernel_size=(1, self.win_length),
|
||||
stride=(1, self.hop_length),
|
||||
)
|
||||
.squeeze(0)
|
||||
.squeeze(0)
|
||||
)
|
||||
|
||||
window_envelope, window_buffer = self._buffer_process(
|
||||
window_envelope, window_buffer, pad, last_chunk=last_chunk, streaming=streaming
|
||||
)
|
||||
window_envelope = window_envelope.squeeze()
|
||||
|
||||
assert (window_envelope > 1e-11).all()
|
||||
y = y / window_envelope
|
||||
|
||||
return y, audio_buffer, window_buffer
|
||||
|
||||
|
||||
class ISTFTHead(nn.Module):
|
||||
def __init__(self, dim: int, n_fft: int, hop_length: int, padding: str = "same"):
|
||||
super().__init__()
|
||||
out_dim = n_fft + 2
|
||||
self.out = nn.Linear(dim, out_dim)
|
||||
self.istft = ISTFT(n_fft=n_fft, hop_length=hop_length, win_length=n_fft, padding=padding)
|
||||
|
||||
def forward(self, x, audio_buffer=None, window_buffer=None, streaming=False, last_chunk=False):
|
||||
x_pred = self.out(x)
|
||||
x_pred = x_pred.transpose(1, 2)
|
||||
mag, p = x_pred.chunk(2, dim=1)
|
||||
mag = torch.exp(mag)
|
||||
mag = torch.clip(mag, max=1e2)
|
||||
x = torch.cos(p)
|
||||
y = torch.sin(p)
|
||||
S = mag * (x + 1j * y)
|
||||
audio, audio_buffer, window_buffer = self.istft(
|
||||
S, audio_buffer=audio_buffer, window_buffer=window_buffer, streaming=streaming, last_chunk=last_chunk
|
||||
)
|
||||
return audio.unsqueeze(1), x_pred, audio_buffer, window_buffer
|
||||
|
||||
|
||||
class StreamingLinearUpsample(nn.Module):
|
||||
def __init__(self, scale_factor=4):
|
||||
super().__init__()
|
||||
self.scale_factor = scale_factor
|
||||
self.upsampler = nn.Upsample(scale_factor=scale_factor, mode="linear", align_corners=False)
|
||||
|
||||
def forward(self, x, state=None, is_last=False):
|
||||
if state is None:
|
||||
state = {"prev_chunk": None, "history_last": None, "is_first": True}
|
||||
|
||||
if x is None and not is_last:
|
||||
return None, state
|
||||
|
||||
if state["is_first"] and is_last:
|
||||
out = self.upsampler(x.transpose(1, 2)).transpose(1, 2)
|
||||
return out, None
|
||||
|
||||
output_chunks = []
|
||||
|
||||
if state["is_first"]:
|
||||
state["prev_chunk"] = x
|
||||
state["is_first"] = False
|
||||
if not is_last:
|
||||
return None, state
|
||||
|
||||
if state["prev_chunk"] is not None:
|
||||
p = state["prev_chunk"].transpose(1, 2)
|
||||
|
||||
if state["history_last"] is None:
|
||||
lookahead = x[:, :1, :].transpose(1, 2)
|
||||
inp = torch.cat([p, lookahead], dim=2)
|
||||
up = self.upsampler(inp)
|
||||
out_prev = up[:, :, : p.size(2) * self.scale_factor]
|
||||
else:
|
||||
lookahead = x[:, :1, :].transpose(1, 2)
|
||||
inp = torch.cat([state["history_last"], p, lookahead], dim=2)
|
||||
up = self.upsampler(inp)
|
||||
start = self.scale_factor
|
||||
end = start + p.size(2) * self.scale_factor
|
||||
out_prev = up[:, :, start:end]
|
||||
|
||||
output_chunks.append(out_prev.transpose(1, 2))
|
||||
state["history_last"] = p[:, :, -1:]
|
||||
state["prev_chunk"] = x
|
||||
|
||||
if is_last:
|
||||
p = state["prev_chunk"].transpose(1, 2)
|
||||
inp = torch.cat([state["history_last"], p], dim=2)
|
||||
up = self.upsampler(inp)
|
||||
out_last = up[:, :, self.scale_factor :]
|
||||
output_chunks.append(out_last.transpose(1, 2))
|
||||
state = None
|
||||
|
||||
final_out = torch.cat(output_chunks, dim=1) if output_chunks else None
|
||||
return final_out, state
|
||||
|
||||
|
||||
class Decoder(nn.Module):
|
||||
def __init__(self, decoder_args, output_dim=320, latent_dim=64, patch_size=-1):
|
||||
super().__init__()
|
||||
config = Qwen2Config.from_dict(config_dict=decoder_args)
|
||||
if is_flash_attn_2_available():
|
||||
config._attn_implementation_autoset = True
|
||||
config._attn_implementation = "flash_attention_2"
|
||||
else:
|
||||
config._attn_implementation = "sdpa"
|
||||
|
||||
logger.info("AudioVAE Decoder: using attn_implementation=%r", config._attn_implementation)
|
||||
self.decoder = Qwen2Model(config)
|
||||
self.output_dim = output_dim
|
||||
self.latent_dim = latent_dim
|
||||
self.fc1 = nn.Linear(latent_dim, config.hidden_size)
|
||||
self.hop_length = output_dim
|
||||
self.head = ISTFTHead(
|
||||
dim=config.hidden_size, n_fft=self.hop_length * 4, hop_length=self.hop_length, padding="same"
|
||||
)
|
||||
self.patch_size = patch_size
|
||||
if self.patch_size != -1:
|
||||
self.upsampling = StreamingLinearUpsample(scale_factor=patch_size)
|
||||
|
||||
def low_level_reconstruct(self, x, past_key_values=None, use_cache=False, stream_state=None, last_chunk=False):
|
||||
upsample_state, audio_buffer, window_buffer = stream_state
|
||||
bsz, device, dtype = x.size(0), x.device, x.dtype
|
||||
x = self.fc1(x)
|
||||
if self.patch_size != -1:
|
||||
if use_cache:
|
||||
x, upsample_state = self.upsampling(x, state=upsample_state, is_last=last_chunk)
|
||||
if x is None:
|
||||
stream_state = (upsample_state, audio_buffer, window_buffer)
|
||||
return torch.empty(bsz, 1, 0, device=device, dtype=dtype), stream_state, past_key_values
|
||||
else:
|
||||
x = self.upsampling.upsampler(x.transpose(1, 2)).transpose(1, 2)
|
||||
|
||||
hidden_states_list = []
|
||||
|
||||
if use_cache and getattr(self.decoder.config, "sliding_window", None) is not None:
|
||||
sw_size = self.decoder.config.sliding_window
|
||||
target_len = sw_size - 1
|
||||
if past_key_values is None:
|
||||
past_len = 0
|
||||
elif hasattr(past_key_values, "get_seq_length"):
|
||||
past_len = past_key_values.get_seq_length()
|
||||
elif isinstance(past_key_values, tuple) and len(past_key_values) > 0:
|
||||
past_len = past_key_values[0][0].shape[-2]
|
||||
else:
|
||||
past_len = 0
|
||||
|
||||
curr_len = x.shape[1]
|
||||
|
||||
if past_len < target_len and (past_len + curr_len) >= sw_size:
|
||||
fill_len = target_len - past_len
|
||||
x_fill = x[:, :fill_len, :]
|
||||
outputs = self.decoder(inputs_embeds=x_fill, past_key_values=past_key_values, use_cache=use_cache)
|
||||
hidden_states_list.append(outputs.last_hidden_state)
|
||||
past_key_values = outputs.past_key_values
|
||||
x = x[:, fill_len:, :]
|
||||
|
||||
outputs = self.decoder(inputs_embeds=x, past_key_values=past_key_values, use_cache=use_cache)
|
||||
hidden_states_list.append(outputs.last_hidden_state)
|
||||
past_key_values = outputs.past_key_values
|
||||
|
||||
if len(hidden_states_list) > 1:
|
||||
full_hidden_state = torch.cat(hidden_states_list, dim=1)
|
||||
else:
|
||||
full_hidden_state = hidden_states_list[0]
|
||||
|
||||
x_out, _, audio_buffer, window_buffer = self.head(
|
||||
full_hidden_state,
|
||||
streaming=use_cache,
|
||||
audio_buffer=audio_buffer,
|
||||
window_buffer=window_buffer,
|
||||
last_chunk=last_chunk,
|
||||
)
|
||||
|
||||
stream_state = (upsample_state, audio_buffer, window_buffer)
|
||||
return x_out, stream_state, past_key_values
|
||||
|
||||
|
||||
class Encoder(nn.Module):
|
||||
def __init__(self, encoder_args, input_dim=320, hop_size=320, latent_dim=64, patch_size=-1):
|
||||
super().__init__()
|
||||
config = Qwen2Config.from_dict(config_dict=encoder_args)
|
||||
if is_flash_attn_2_available():
|
||||
config._attn_implementation_autoset = True
|
||||
config._attn_implementation = "flash_attention_2"
|
||||
else:
|
||||
config._attn_implementation = "sdpa"
|
||||
|
||||
logger.info("AudioVAE Encoder: using attn_implementation=%r", config._attn_implementation)
|
||||
self.encoder = Qwen2Model(config)
|
||||
self.input_dim = input_dim
|
||||
self.hop_size = hop_size
|
||||
self.latent_dim = latent_dim
|
||||
self.fc1 = nn.Linear(input_dim, config.hidden_size, bias=False)
|
||||
self.fc2 = nn.Linear(config.hidden_size, config.hidden_size)
|
||||
self.fc3 = nn.Linear(config.hidden_size, latent_dim * 2)
|
||||
self.norm = nn.LayerNorm(config.hidden_size)
|
||||
self.patch_size = patch_size
|
||||
if patch_size != -1:
|
||||
config.num_hidden_layers = 4
|
||||
self.aggregator = Qwen2Model(config)
|
||||
self.cls_embed = nn.Parameter(torch.rand(1, 1, config.hidden_size))
|
||||
self.cls_embed.data.normal_(0, 0.02)
|
||||
|
||||
def get_frames(self, x):
|
||||
num_frames_total = (x.size(-1) + self.hop_size - 1) // self.hop_size
|
||||
expected_len = (num_frames_total - 1) * self.hop_size + self.input_dim
|
||||
padding_needed = expected_len - x.size(-1)
|
||||
waveform = F.pad(x, (0, padding_needed), value=0.0)
|
||||
frames = waveform.unfold(dimension=-1, size=self.input_dim, step=self.hop_size)
|
||||
return frames
|
||||
|
||||
def pad_patch_insert_cls(self, x):
|
||||
bsz, _, dim = x.size()
|
||||
num_frame = x.size(1)
|
||||
r = num_frame % self.patch_size
|
||||
pad_num = self.patch_size - r if r else 0
|
||||
x = F.pad(x, (0, 0, 0, pad_num), value=0.0)
|
||||
x = x.reshape(-1, self.patch_size, dim)
|
||||
x = torch.cat((x, self.cls_embed.expand(x.size(0), -1, -1)), dim=1)
|
||||
x = x.reshape(bsz, -1, dim)
|
||||
return x
|
||||
|
||||
def forward(self, waveform):
|
||||
x = self.get_frames(waveform)
|
||||
x = self.fc1(x)
|
||||
x = self.fc2(x)
|
||||
x = self.encoder(inputs_embeds=x)
|
||||
x = x.last_hidden_state
|
||||
|
||||
if self.patch_size != -1:
|
||||
x = self.pad_patch_insert_cls(x)
|
||||
x = self.aggregator(inputs_embeds=x)
|
||||
x = x.last_hidden_state
|
||||
bsz, _, dim = x.size()
|
||||
x = x.reshape(-1, self.patch_size + 1, dim)
|
||||
x = x[:, -1:, :].reshape(bsz, -1, dim)
|
||||
|
||||
x = self.fc3(x)
|
||||
return x, waveform.unsqueeze(1)
|
||||
|
||||
|
||||
class AudioVAE(PreTrainedModel):
|
||||
config_class = AudioVAEConfig
|
||||
|
||||
def __init__(self, config: AudioVAEConfig):
|
||||
super().__init__(config)
|
||||
self.encoder = Encoder(
|
||||
encoder_args=config.enc_kwargs["backbone"],
|
||||
input_dim=config.enc_kwargs["input_dim"],
|
||||
hop_size=config.enc_kwargs.get("hop_size", 320),
|
||||
latent_dim=config.enc_kwargs["latent_dim"],
|
||||
patch_size=config.patch_size,
|
||||
)
|
||||
self.decoder = Decoder(
|
||||
decoder_args=config.dec_kwargs["backbone"],
|
||||
output_dim=config.dec_kwargs["output_dim"],
|
||||
latent_dim=config.dec_kwargs["latent_dim"],
|
||||
patch_size=config.patch_size,
|
||||
)
|
||||
self.post_init()
|
||||
|
||||
def _init_weights(self, module):
|
||||
std = 0.02
|
||||
if isinstance(module, nn.Linear):
|
||||
if self.config.init_method == "kaiming":
|
||||
nn.init.kaiming_normal_(module.weight, mode="fan_in", nonlinearity="relu")
|
||||
else:
|
||||
module.weight.data.normal_(mean=0.0, std=std)
|
||||
if module.bias is not None:
|
||||
module.bias.data.zero_()
|
||||
elif isinstance(module, nn.Embedding):
|
||||
module.weight.data.normal_(mean=0.0, std=std)
|
||||
if module.padding_idx is not None:
|
||||
module.weight.data[module.padding_idx].zero_()
|
||||
|
||||
def encode_latent(self, waveform, waveform_length):
|
||||
from diffusers.models.autoencoders.autoencoder_oobleck import OobleckDiagonalGaussianDistribution
|
||||
|
||||
frame_num = torch.ceil(waveform_length / self.config.enc_kwargs["input_dim"]).to(torch.int32)
|
||||
if self.config.patch_size != -1:
|
||||
frame_num = torch.ceil(frame_num / self.config.patch_size)
|
||||
h, y = self.encoder(waveform)
|
||||
h = h.transpose(1, 2)
|
||||
|
||||
posterior = OobleckDiagonalGaussianDistribution(h)
|
||||
latent = posterior.sample()
|
||||
latent = latent.transpose(1, 2)
|
||||
return latent, frame_num
|
||||
|
||||
def decode(self, latent, past_key_values=None, use_cache=False, stream_state=(None, None, None), last_chunk=False):
|
||||
waveform, stream_state, past_key_values = self.decoder.low_level_reconstruct(
|
||||
latent,
|
||||
past_key_values=past_key_values,
|
||||
use_cache=use_cache,
|
||||
stream_state=stream_state,
|
||||
last_chunk=last_chunk,
|
||||
)
|
||||
return waveform, stream_state, past_key_values
|
||||
@@ -26,10 +26,10 @@ from vllm.sequence import IntermediateTensors
|
||||
|
||||
from vllm_omni.model_executor.custom_process_mixin import CustomProcessMixin
|
||||
from vllm_omni.model_executor.model_loader.weight_utils import download_weights_from_hf_specific
|
||||
from vllm_omni.model_executor.models.common.ming.audio_vae import AudioVAE, AudioVAEConfig
|
||||
from vllm_omni.model_executor.models.output_templates import OmniOutput
|
||||
from vllm_omni.transformers_utils.configs.ming_flash_omni import MingFlashOmniTalkerConfig
|
||||
|
||||
from .audio_vae import AudioVAE, AudioVAEConfig
|
||||
from .prompt_utils import DEFAULT_PROMPT as MING_DEFAULT_PROMPT
|
||||
from .talker_module import CFM, Aggregator, DiT, MingAudioGenerator, build_tts_input
|
||||
from .text_processing import segment_and_normalize
|
||||
|
||||
@@ -697,6 +697,9 @@ class MingFlashOmniThinkerForConditionalGeneration(
|
||||
f"image_gen_scales={list(self.query_tokens_dict.keys())}"
|
||||
)
|
||||
|
||||
def get_language_model(self):
|
||||
return self.language_model
|
||||
|
||||
def _load_image_gen_query_tokens(self, model_path: str) -> None:
|
||||
"""Load learnable query tokens used for image generation.
|
||||
|
||||
|
||||
@@ -110,12 +110,18 @@ BASE_CAPTION_TEMPLATE: dict[str, Any] = {
|
||||
}
|
||||
|
||||
|
||||
def create_instruction(user_input: dict[str, Any]) -> str:
|
||||
def create_instruction(user_input: Any) -> str | None:
|
||||
"""Return a JSON caption string for ``audio_sequence[0]``.
|
||||
|
||||
Only keys already present on the base template are merged in; unknown
|
||||
keys are silently ignored to keep the output schema stable.
|
||||
"""
|
||||
if user_input is None:
|
||||
return None
|
||||
if isinstance(user_input, str):
|
||||
return user_input
|
||||
if not isinstance(user_input, dict):
|
||||
raise ValueError(f"Ming instruction must be str, dict, or None; got {type(user_input).__name__}")
|
||||
caption = copy.deepcopy(BASE_CAPTION_TEMPLATE)
|
||||
item = caption["audio_sequence"][0]
|
||||
for key, value in user_input.items():
|
||||
|
||||
@@ -27,223 +27,19 @@ from threading import Lock
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
from transformers import PreTrainedTokenizerBase, Qwen2Config, Qwen2Model, StaticCache
|
||||
from vllm.logger import init_logger
|
||||
from vllm.platforms import current_platform
|
||||
from x_transformers.x_transformers import RotaryEmbedding, apply_rotary_pos_emb
|
||||
from x_transformers.x_transformers import RotaryEmbedding
|
||||
|
||||
from vllm_omni.model_executor.layers.timestep_embedding import DiTTimestepEmbedding
|
||||
|
||||
from .audio_vae import AudioVAE
|
||||
from vllm_omni.model_executor.models.common.ming.audio_vae import AudioVAE
|
||||
from vllm_omni.model_executor.models.common.ming.dit import CondEmbedder, DiTBlock, FinalLayer, get_epss_timesteps
|
||||
from vllm_omni.model_executor.models.common.ming.fm import apply_sway_sampling, integrate_cfm_steps
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
########################################################################
|
||||
# DiT Modules
|
||||
# Ported from:
|
||||
# https://github.com/inclusionAI/Ming/blob/e58533db227031990c5a6864dcf5f08fb53ed0d2/talker_module/modules.py
|
||||
# Ported from:
|
||||
# https://github.com/inclusionAI/Ming/blob/e58533db227031990c5a6864dcf5f08fb53ed0d2/talker_module/dit.py
|
||||
########################################################################
|
||||
|
||||
|
||||
class RMSNorm(nn.Module):
|
||||
def __init__(self, dim: int, eps: float = 1e-6):
|
||||
super().__init__()
|
||||
self.eps = eps
|
||||
self.weight = nn.Parameter(torch.ones(dim))
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
if self.weight.dtype in [torch.float16, torch.bfloat16]:
|
||||
x = x.to(self.weight.dtype)
|
||||
x = F.rms_norm(x, normalized_shape=(x.shape[-1],), weight=self.weight, eps=self.eps)
|
||||
return x
|
||||
|
||||
|
||||
class FeedForward(nn.Module):
|
||||
def __init__(
|
||||
self, dim: int, dim_out: int | None = None, mult: float = 4, dropout: float = 0.0, approximate: str = "none"
|
||||
):
|
||||
super().__init__()
|
||||
inner_dim = int(dim * mult)
|
||||
dim_out = dim_out if dim_out is not None else dim
|
||||
|
||||
activation = nn.GELU(approximate=approximate)
|
||||
project_in = nn.Sequential(nn.Linear(dim, inner_dim), activation)
|
||||
self.ff = nn.Sequential(project_in, nn.Dropout(dropout), nn.Linear(inner_dim, dim_out))
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
return self.ff(x)
|
||||
|
||||
|
||||
class Attention(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
dim: int,
|
||||
heads: int = 8,
|
||||
dim_head: int = 64,
|
||||
dropout: float = 0.0,
|
||||
qk_norm: str | None = None,
|
||||
pe_attn_head: int | None = None,
|
||||
attn_mask_enabled: bool = True,
|
||||
):
|
||||
super().__init__()
|
||||
self.dim = dim
|
||||
self.heads = heads
|
||||
self.inner_dim = dim_head * heads
|
||||
self.dropout = dropout
|
||||
|
||||
self.to_q = nn.Linear(dim, self.inner_dim)
|
||||
self.to_k = nn.Linear(dim, self.inner_dim)
|
||||
self.to_v = nn.Linear(dim, self.inner_dim)
|
||||
if qk_norm is None:
|
||||
self.q_norm = None
|
||||
self.k_norm = None
|
||||
elif qk_norm == "rms_norm":
|
||||
self.q_norm = RMSNorm(dim_head)
|
||||
self.k_norm = RMSNorm(dim_head)
|
||||
else:
|
||||
raise ValueError(f"Unimplemented qk_norm: {qk_norm}")
|
||||
|
||||
self.to_out = nn.ModuleList([])
|
||||
self.to_out.append(nn.Linear(self.inner_dim, dim))
|
||||
self.to_out.append(nn.Dropout(dropout))
|
||||
|
||||
self.pe_attn_head = pe_attn_head
|
||||
self.attn_mask_enabled = attn_mask_enabled
|
||||
|
||||
def forward(
|
||||
self,
|
||||
x: torch.Tensor,
|
||||
mask: torch.Tensor | None = None,
|
||||
rope: tuple[torch.Tensor, torch.Tensor | None] | None = None,
|
||||
) -> torch.Tensor:
|
||||
batch_size = x.shape[0]
|
||||
|
||||
query = self.to_q(x)
|
||||
key = self.to_k(x)
|
||||
value = self.to_v(x)
|
||||
|
||||
inner_dim = key.shape[-1]
|
||||
head_dim = inner_dim // self.heads
|
||||
query = query.view(batch_size, -1, self.heads, head_dim).transpose(1, 2)
|
||||
key = key.view(batch_size, -1, self.heads, head_dim).transpose(1, 2)
|
||||
value = value.view(batch_size, -1, self.heads, head_dim).transpose(1, 2)
|
||||
|
||||
if self.q_norm is not None:
|
||||
query = self.q_norm(query)
|
||||
if self.k_norm is not None:
|
||||
key = self.k_norm(key)
|
||||
|
||||
if rope is not None:
|
||||
freqs, xpos_scale = rope
|
||||
q_xpos_scale, k_xpos_scale = (xpos_scale, xpos_scale**-1.0) if xpos_scale is not None else (1.0, 1.0)
|
||||
|
||||
if self.pe_attn_head is not None:
|
||||
on = self.pe_attn_head
|
||||
query[:, :on, :, :] = apply_rotary_pos_emb(query[:, :on, :, :], freqs, q_xpos_scale)
|
||||
key[:, :on, :, :] = apply_rotary_pos_emb(key[:, :on, :, :], freqs, k_xpos_scale)
|
||||
else:
|
||||
query = apply_rotary_pos_emb(query, freqs, q_xpos_scale)
|
||||
key = apply_rotary_pos_emb(key, freqs, k_xpos_scale)
|
||||
|
||||
if self.attn_mask_enabled and mask is not None:
|
||||
valid_sample_indices = mask.any(dim=1)
|
||||
final_output = torch.zeros_like(query).to(query.device)
|
||||
|
||||
attn_mask = mask[valid_sample_indices]
|
||||
query = query[valid_sample_indices]
|
||||
key = key[valid_sample_indices]
|
||||
value = value[valid_sample_indices]
|
||||
attn_mask = attn_mask.unsqueeze(1).unsqueeze(1)
|
||||
attn_mask = attn_mask.expand(valid_sample_indices.sum().item(), self.heads, query.shape[-2], key.shape[-2])
|
||||
else:
|
||||
attn_mask = None
|
||||
|
||||
x = F.scaled_dot_product_attention(query, key, value, attn_mask=attn_mask, dropout_p=0.0, is_causal=False)
|
||||
if self.attn_mask_enabled and mask is not None:
|
||||
final_output[valid_sample_indices] = x
|
||||
x = final_output
|
||||
|
||||
x = x.transpose(1, 2).reshape(batch_size, -1, self.heads * head_dim)
|
||||
x = x.to(query.dtype)
|
||||
|
||||
x = self.to_out[0](x)
|
||||
x = self.to_out[1](x)
|
||||
|
||||
if mask is not None:
|
||||
mask = mask.unsqueeze(-1)
|
||||
x = x.masked_fill(~mask, 0.0)
|
||||
|
||||
return x
|
||||
|
||||
|
||||
class DiTBlock(nn.Module):
|
||||
"""A DiT block with pre-norm and residual connections."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
hidden_size: int,
|
||||
num_heads: int,
|
||||
mlp_ratio: float = 4.0,
|
||||
dropout: float = 0.1,
|
||||
qk_norm: str | None = None,
|
||||
pe_attn_head: int | None = None,
|
||||
attn_mask_enabled: bool = True,
|
||||
**kwargs,
|
||||
):
|
||||
super().__init__()
|
||||
self.norm1 = RMSNorm(hidden_size)
|
||||
self.attn = Attention(
|
||||
dim=hidden_size,
|
||||
heads=num_heads,
|
||||
dim_head=hidden_size // num_heads,
|
||||
dropout=dropout,
|
||||
qk_norm=qk_norm,
|
||||
pe_attn_head=pe_attn_head,
|
||||
attn_mask_enabled=attn_mask_enabled,
|
||||
)
|
||||
self.norm2 = RMSNorm(hidden_size)
|
||||
self.mlp = FeedForward(dim=hidden_size, mult=mlp_ratio, dropout=dropout, approximate="tanh")
|
||||
|
||||
def forward(
|
||||
self,
|
||||
x: torch.Tensor,
|
||||
mask: torch.Tensor | None,
|
||||
rope: tuple[torch.Tensor, torch.Tensor | None] | None,
|
||||
) -> torch.Tensor:
|
||||
x = x + self.attn(self.norm1(x), mask=mask, rope=rope)
|
||||
x = x + self.mlp(self.norm2(x))
|
||||
return x
|
||||
|
||||
|
||||
class FinalLayer(nn.Module):
|
||||
"""The final layer of DiT."""
|
||||
|
||||
def __init__(self, hidden_size: int, out_channels: int):
|
||||
super().__init__()
|
||||
self.norm_final = RMSNorm(hidden_size)
|
||||
self.linear = nn.Linear(hidden_size, out_channels, bias=True)
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
x = self.norm_final(x)
|
||||
x = self.linear(x)
|
||||
return x
|
||||
|
||||
|
||||
class CondEmbedder(nn.Module):
|
||||
"""Embeds LLM hidden states with optional CFG dropout."""
|
||||
|
||||
def __init__(self, input_feature_size: int, hidden_size: int):
|
||||
super().__init__()
|
||||
self.cond_embedder = nn.Linear(input_feature_size, hidden_size)
|
||||
|
||||
def forward(self, llm_cond: torch.Tensor) -> torch.Tensor:
|
||||
return self.cond_embedder(llm_cond)
|
||||
|
||||
|
||||
class DiT(nn.Module):
|
||||
"""Diffusion model with a Transformer backbone for audio latent generation."""
|
||||
|
||||
@@ -324,29 +120,6 @@ class DiT(nn.Module):
|
||||
return model_out[:, -x.shape[1] :, :]
|
||||
|
||||
|
||||
#########################################################################################
|
||||
# CFM
|
||||
# Ported from:
|
||||
# https://github.com/inclusionAI/Ming/blob/e58533db227031990c5a6864dcf5f08fb53ed0d2/talker_module/cfm.py
|
||||
#########################################################################################
|
||||
|
||||
|
||||
def get_epss_timesteps(n, device, dtype):
|
||||
dt = 1 / 32
|
||||
predefined_timesteps = {
|
||||
5: [0, 2, 4, 8, 16, 32],
|
||||
6: [0, 2, 4, 6, 8, 16, 32],
|
||||
7: [0, 2, 4, 6, 8, 16, 24, 32],
|
||||
10: [0, 2, 4, 6, 8, 12, 16, 20, 24, 28, 32],
|
||||
12: [0, 2, 4, 6, 8, 10, 12, 14, 16, 20, 24, 28, 32],
|
||||
16: [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 12, 14, 16, 20, 24, 28, 32],
|
||||
}
|
||||
t = predefined_timesteps.get(n, [])
|
||||
if not t:
|
||||
return torch.linspace(0, 1, n + 1, device=device, dtype=dtype)
|
||||
return dt * torch.tensor(t, device=device, dtype=dtype)
|
||||
|
||||
|
||||
class CFM(nn.Module):
|
||||
"""Conditional Flow Matching module for audio latent generation."""
|
||||
|
||||
@@ -391,15 +164,8 @@ class CFM(nn.Module):
|
||||
pred, null_pred = torch.chunk(pred_cfg, 2, dim=0)
|
||||
return pred + (pred - null_pred) * sde_args[0]
|
||||
|
||||
if self.sway_sampling_coef is not None:
|
||||
t = t + self.sway_sampling_coef * (torch.cos(torch.pi / 2 * t) - 1 + t)
|
||||
|
||||
for step in range(self.steps):
|
||||
dt = t[step + 1] - t[step]
|
||||
y0 = y0 + fn(t[step], y0) * dt
|
||||
y0 = y0 + sde_args[1] * (sde_args[2] ** 0.5) * (dt.abs() ** 0.5) * sde_rnd[step]
|
||||
|
||||
return y0
|
||||
t = apply_sway_sampling(t, self.sway_sampling_coef)
|
||||
return integrate_cfm_steps(fn, y0, t, sde_args, sde_rnd, self.steps)
|
||||
|
||||
|
||||
class CFMGraphExecutor:
|
||||
|
||||
@@ -22,7 +22,8 @@ from .spk_embedding import SpkembExtractor
|
||||
from .talker_module import resample
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from .audio_vae import AudioVAE
|
||||
from vllm_omni.model_executor.models.common.ming.audio_vae import AudioVAE
|
||||
|
||||
from .talker_module import Aggregator
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
Reference in new issue
Block a user