[Ming] Cleanup Ming-family shared modules (#6119)

Signed-off-by: Yuanheng <jonathan.zhaoyh@gmail.com>
Signed-off-by: Yuanheng Zhao <jonathan.zhaoyh@gmail.com>
This commit is contained in:
Yuanheng Zhao authored and GitHub committed 2026-08-19 14:13:40 +08:00
1 parent adf6d0c1ea
commit a630b0c2ec
22 files changed
+380 -192

No files matched your search

@@ -1,2 +1,13 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Components shared by the Ming-family model packages.
Naming: The flow-matching pieces all use the cfm_ prefix,
because "flow matching", "fm" and "cfm" were three names for one concept here:
* dit_blocks - DiT building blocks (DiTBlock, FinalLayer, CondEmbedder)
* cfm_dit - the DiT used as the CFM velocity model
* cfm_solver - timestep schedules and the ODE/SDE integrator
* cfm_head - inference-only CFM wrapper preserving cfm.model.* names
* cfm_cudagraph - batch-1 CUDA-graph executor for the sampling loop
"""
@@ -7,7 +7,7 @@ import torch
import torch.nn as nn
from x_transformers.x_transformers import RotaryEmbedding
from vllm_omni.model_executor.models.common.ming.dit import DiTBlock, FinalLayer
from vllm_omni.model_executor.models.common.ming.dit_blocks import DiTBlock, FinalLayer
class Aggregator(nn.Module):
@@ -17,7 +17,7 @@ from __future__ import annotations
import torch
import torch.nn as nn
from vllm_omni.model_executor.models.common.ming.fm import build_timesteps
from vllm_omni.model_executor.models.common.ming.cfm_solver import build_timesteps
class CFMSampler(nn.Module):
@@ -7,7 +7,7 @@ import torch.nn as nn
from x_transformers.x_transformers import RotaryEmbedding
from vllm_omni.model_executor.layers.timestep_embedding import DiTTimestepEmbedding
from vllm_omni.model_executor.models.common.ming.dit import CondEmbedder, DiTBlock, FinalLayer
from vllm_omni.model_executor.models.common.ming.dit_blocks import CondEmbedder, DiTBlock, FinalLayer
class DiT(nn.Module):
@@ -5,9 +5,8 @@
import torch
import torch.nn as nn
from vllm_omni.model_executor.models.common.ming.fm import Solver, build_timesteps
from .fm.dit import DiT
from vllm_omni.model_executor.models.common.ming.cfm_dit import DiT
from vllm_omni.model_executor.models.common.ming.cfm_solver import Solver, build_timesteps
class FlowLoss(nn.Module):
@@ -3,7 +3,21 @@
import torch
from .dit import get_epss_timesteps
def get_epss_timesteps(n, device, dtype):
dt = 1 / 32
predefined_timesteps = {
5: [0, 2, 4, 8, 16, 32],
6: [0, 2, 4, 6, 8, 16, 32],
7: [0, 2, 4, 6, 8, 16, 24, 32],
10: [0, 2, 4, 6, 8, 12, 16, 20, 24, 28, 32],
12: [0, 2, 4, 6, 8, 10, 12, 14, 16, 20, 24, 28, 32],
16: [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 12, 14, 16, 20, 24, 28, 32],
}
t = predefined_timesteps.get(n, [])
if not t:
return torch.linspace(0, 1, n + 1, device=device, dtype=dtype)
return dt * torch.tensor(t, device=device, dtype=dtype)
def apply_sway_sampling(t, sway_sampling_coef):
@@ -216,19 +216,3 @@ class CondEmbedder(nn.Module):
def forward(self, llm_cond):
return self.cond_embedder(llm_cond)
def get_epss_timesteps(n, device, dtype):
dt = 1 / 32
predefined_timesteps = {
5: [0, 2, 4, 8, 16, 32],
6: [0, 2, 4, 6, 8, 16, 32],
7: [0, 2, 4, 6, 8, 16, 24, 32],
10: [0, 2, 4, 6, 8, 12, 16, 20, 24, 28, 32],
12: [0, 2, 4, 6, 8, 10, 12, 14, 16, 20, 24, 28, 32],
16: [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 12, 14, 16, 20, 24, 28, 32],
}
t = predefined_timesteps.get(n, [])
if not t:
return torch.linspace(0, 1, n + 1, device=device, dtype=dtype)
return dt * torch.tensor(t, device=device, dtype=dtype)
@@ -0,0 +1,56 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# Adopted from https://github.com/inclusionAI/Ming-omni-tts/blob/main/spkemb_extractor.py
"""CAM++ speaker-embedding extraction, shared by the Ming model packages."""
from __future__ import annotations
import os
import torch
import torchaudio
from vllm.multimodal.media.audio import load_audio
from vllm_omni.model_executor.model_loader.weight_utils import resolve_model_to_local_path
from vllm_omni.model_executor.models.common.ming.spk_embedding import SpkembExtractor
class SpeakerEmbeddingExtractor:
"""CAM++ ONNX speaker embeddings, resampled to the extractor's rate."""
def __init__(self, model, target_sr=16000, *, allow_download: bool = False):
local_model_path = resolve_model_to_local_path(
model,
allow_download=allow_download,
allow_patterns=["campplus.onnx"],
)
campplus_path = os.path.join(local_model_path, "campplus.onnx")
if not os.path.exists(campplus_path):
raise RuntimeError(f"Missing Ming speaker extractor model: {campplus_path}")
self.target_sr = int(target_sr)
self._core = SpkembExtractor(campplus_path, target_sr=self.target_sr)
def extract_from_waveform(self, waveform, sample_rate):
if not isinstance(waveform, torch.Tensor):
waveform = torch.as_tensor(waveform)
tensor = waveform.detach().to(torch.float32)
if tensor.ndim == 1:
tensor = tensor.unsqueeze(0)
if int(sample_rate) != self.target_sr:
tensor = torchaudio.transforms.Resample(orig_freq=int(sample_rate), new_freq=self.target_sr)(tensor)
embedding = self._core._extract_spk_embedding(tensor)
return embedding.squeeze(0).to(dtype=torch.float32)
def extract_from_file(self, audio_path):
audio, sample_rate = load_audio(audio_path, sr=None, mono=False)
waveform = torch.from_numpy(audio).contiguous()
return self.extract_from_waveform(waveform, sample_rate)
def extract_many(self, audio_paths):
return [self.extract_from_file(path) for path in audio_paths]
__all__ = ["SpeakerEmbeddingExtractor"]
@@ -25,14 +25,15 @@ from vllm.model_executor.models.utils import AutoWeightsLoader
from vllm.sequence import IntermediateTensors
from vllm_omni.model_executor.custom_process_mixin import CustomProcessMixin
from vllm_omni.model_executor.model_loader.weight_utils import download_weights_from_hf_specific
from vllm_omni.model_executor.model_loader.weight_utils import resolve_model_to_local_path
from vllm_omni.model_executor.models.common.ming.aggregator import Aggregator
from vllm_omni.model_executor.models.common.ming.audio_vae import AudioVAE, AudioVAEConfig
from vllm_omni.model_executor.models.ming_tts.constants import SPEAKER_EMBEDDING_DIM
from vllm_omni.model_executor.models.output_templates import OmniOutput
from vllm_omni.transformers_utils.configs.ming_flash_omni import MingFlashOmniTalkerConfig
from .prompt_utils import DEFAULT_PROMPT as MING_DEFAULT_PROMPT
from .talker_module import CFM, Aggregator, DiT, MingAudioGenerator, build_tts_input
from .talker_module import CFM, DiT, MingAudioGenerator, build_tts_input
from .text_processing import segment_and_normalize
from .voice_presets import VoicePresetRegistry
@@ -119,7 +120,11 @@ class MingFlashOmniTalkerForConditionalGeneration(nn.Module, CustomProcessMixin)
DiT(llm_input_dim=self.hidden_size, **config.flowmodel),
steps=config.steps,
)
self.aggregator = Aggregator(llm_input_dim=self.hidden_size, **config.aggregator)
# config.aggregator still wins `in_channels` if the checkpoint states it.
self.aggregator = Aggregator(
llm_input_dim=self.hidden_size,
**{"in_channels": self.latent_dim, **config.aggregator},
)
self.stop_head = nn.Linear(self.hidden_size, 2, bias=True)
self.spk_head = nn.Linear(SPEAKER_EMBEDDING_DIM, self.hidden_size, bias=True)
@@ -525,27 +530,22 @@ class MingFlashOmniTalkerForConditionalGeneration(nn.Module, CustomProcessMixin)
def _iter_talker_safetensors(self) -> Iterable[tuple[str, torch.Tensor]]:
"""Yield (name, tensor) pairs from talker/model*.safetensors."""
model_path = self._model_path
# Try local path first
for candidate in (os.path.join(model_path, "talker"), model_path):
model_root = resolve_model_to_local_path(
self._model_path,
allow_download=True,
allow_patterns=["talker/model*.safetensors"],
cache_dir=self.vllm_config.load_config.download_dir,
)
# Nested talker/ layout first, then a flat checkpoint at the root.
for candidate in (os.path.join(model_root, "talker"), model_root):
sf_files = sorted(glob_module.glob(os.path.join(candidate, "model*.safetensors")))
if sf_files:
for sf_path in sf_files:
yield from load_file(sf_path, device="cpu").items()
return
# HF hub fallback: download only the talker checkpoint files
model_root = download_weights_from_hf_specific(
model_path,
self.vllm_config.load_config.download_dir,
allow_patterns=["talker/model*.safetensors"],
)
talker_dir = os.path.join(model_root, "talker")
sf_files = sorted(glob_module.glob(os.path.join(talker_dir, "model*.safetensors")))
if not sf_files:
raise RuntimeError(f"No talker safetensors found under {model_root}. Expected talker/model*.safetensors.")
for sf_path in sf_files:
yield from load_file(sf_path, device="cpu").items()
raise RuntimeError(f"No talker safetensors found under {model_root}. Expected talker/model*.safetensors.")
def _load_vae_weights(self) -> set[str]:
"""Load AudioVAE weights from talker/vae/model.safetensors."""
@@ -64,6 +64,7 @@ from vllm.sequence import IntermediateTensors
from vllm.utils.tensor_schema import TensorSchema, TensorShape
from vllm_omni.model_executor.custom_process_mixin import CustomProcessMixin
from vllm_omni.model_executor.model_loader.weight_utils import resolve_model_to_local_path
from vllm_omni.model_executor.models.output_templates import OmniOutput
from vllm_omni.transformers_utils.configs.ming_flash_omni import BailingMM2Config
from vllm_omni.transformers_utils.processors.ming import (
@@ -715,12 +716,8 @@ class MingFlashOmniThinkerForConditionalGeneration(
p = Path(model_path) / "mlp" / "model.safetensors"
if not p.exists() and not os.path.exists(model_path):
from vllm_omni.model_executor.model_loader.weight_utils import (
download_weights_from_hf_specific,
)
try:
local_path = download_weights_from_hf_specific(model_path, None, ["mlp/*"])
local_path = resolve_model_to_local_path(model_path, allow_download=True, allow_patterns=["mlp/*"])
p = Path(local_path) / "mlp" / "model.safetensors"
except Exception:
logger.exception("[MingFlashOmniThinker] failed to download mlp/ from %s", model_path)
@@ -33,9 +33,14 @@ from vllm.platforms import current_platform
from x_transformers.x_transformers import RotaryEmbedding
from vllm_omni.model_executor.layers.timestep_embedding import DiTTimestepEmbedding
from vllm_omni.model_executor.models.common.ming.aggregator import Aggregator
from vllm_omni.model_executor.models.common.ming.audio_vae import AudioVAE
from vllm_omni.model_executor.models.common.ming.dit import CondEmbedder, DiTBlock, FinalLayer, get_epss_timesteps
from vllm_omni.model_executor.models.common.ming.fm import apply_sway_sampling, integrate_cfm_steps
from vllm_omni.model_executor.models.common.ming.cfm_solver import (
apply_sway_sampling,
get_epss_timesteps,
integrate_cfm_steps,
)
from vllm_omni.model_executor.models.common.ming.dit_blocks import CondEmbedder, DiTBlock, FinalLayer
logger = init_logger(__name__)
@@ -458,58 +463,6 @@ def silence_holder(
return speech_out, sil_cache
########################################################################
# Audio Postprocess
# Ported from:
# https://github.com/inclusionAI/Ming/blob/e58533db227031990c5a6864dcf5f08fb53ed0d2/talker_module/aggregator.py
########################################################################
class Aggregator(nn.Module):
"""Maps generated audio latent patches back to LLM embedding space."""
def __init__(
self,
in_channels: int = 64,
hidden_size: int = 1152,
depth: int = 28,
num_heads: int = 16,
mlp_ratio: float = 4.0,
llm_input_dim: int = 896,
**kwargs,
):
super().__init__()
self.in_channels = in_channels
self.out_channels = in_channels
self.num_heads = num_heads
self.word_embedder = nn.Embedding(1, hidden_size)
self.x_embedder = nn.Linear(in_channels, hidden_size)
self.hidden_size = hidden_size
self.rotary_embed = RotaryEmbedding(hidden_size // num_heads)
self.blocks = nn.ModuleList(
[DiTBlock(hidden_size, num_heads, mlp_ratio=mlp_ratio, **kwargs) for _ in range(depth)]
)
self.final_layer = FinalLayer(hidden_size, llm_input_dim)
def forward(self, x: torch.Tensor, mask: torch.Tensor | None = None) -> torch.Tensor:
x = self.x_embedder(x)
cls_embed = self.word_embedder(torch.zeros((x.shape[0], 1), dtype=torch.long, device=x.device))
x = torch.cat([cls_embed, x], dim=1)
rope = self.rotary_embed.forward_from_seq_len(x.shape[1])
if mask is not None:
mask_pad = mask.clone().detach()[:, :1]
mask = torch.cat([mask_pad, mask], dim=-1)
for block in self.blocks:
x = block(x, mask, rope)
x = self.final_layer(x)
x = x[:, :1, :]
return x
########################################################################
# Prompt Builder
# Adapted from:
@@ -16,16 +16,15 @@ import torch
from transformers.utils.hub import cached_file
from vllm.logger import init_logger
from vllm_omni.model_executor.model_loader.weight_utils import download_weights_from_hf_specific
from vllm_omni.model_executor.model_loader.weight_utils import resolve_model_to_local_path
from vllm_omni.model_executor.models.common.ming.spk_embedding import SpkembExtractor
from .spk_embedding import SpkembExtractor
from .talker_module import resample
if TYPE_CHECKING:
from vllm_omni.model_executor.models.common.ming.aggregator import Aggregator
from vllm_omni.model_executor.models.common.ming.audio_vae import AudioVAE
from .talker_module import Aggregator
logger = init_logger(__name__)
@@ -273,18 +272,19 @@ class VoicePresetRegistry:
if os.path.isfile(path):
return path, candidate
if not os.path.isdir(self._model_path):
try:
hf_root = download_weights_from_hf_specific(
self._model_path,
self._download_dir,
allow_patterns=["talker/data/**"],
require_all=True,
)
candidate = os.path.join(hf_root, "talker", "data", "voice_name.json")
if os.path.isfile(candidate):
return candidate, os.path.join(hf_root, "talker")
except Exception as e: # pragma: no cover
logger.info("Could not download voice presets from HF: %s", e)
try:
hf_root = resolve_model_to_local_path(
self._model_path,
allow_download=True,
allow_patterns=["talker/data/**"],
cache_dir=self._download_dir,
)
except Exception as e:
logger.info("Could not download voice presets from HF: %s", e)
return None, None
candidate = os.path.join(hf_root, "talker", "data", "voice_name.json")
if os.path.isfile(candidate):
return candidate, os.path.join(hf_root, "talker")
return None, None
@@ -1,2 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
@@ -11,15 +11,21 @@ import torch
import torch.nn as nn
from vllm.config import VllmConfig
from vllm.logger import init_logger
from vllm.model_executor.models.bailing_moe import BailingMoeModel
from vllm.model_executor.models.qwen2 import Qwen2Model
from vllm.model_executor.models.utils import AutoWeightsLoader, WeightsMapper, maybe_prefix
from vllm.sequence import IntermediateTensors
from vllm.v1.outputs import SamplerOutput
from vllm.v1.sample.metadata import SamplingMetadata
from vllm_omni.model_executor.models.common.ming.aggregator import Aggregator
from vllm_omni.model_executor.models.common.ming.cfm_cudagraph import (
CFMGraphExecutor,
CFMSampler,
)
from vllm_omni.model_executor.models.common.ming.cfm_head import FlowLoss
from vllm_omni.model_executor.models.output_templates import OmniOutput
from .aggregator import Aggregator
from .config_ming_tts import (
KEY_CFG,
KEY_DECODE_STEP,
@@ -34,7 +40,6 @@ from .config_ming_tts import (
MingTTSConfig,
)
from .constants import SPEAKER_EMBEDDING_DIM
from .flowloss_head import FlowLoss
from .patch_emission import (
MING_STOP_REASON_CODES,
MING_STOP_REASON_CONTINUE,
@@ -82,8 +87,6 @@ class MingLLMModel(nn.Module):
self.prefix = prefix
self.fm_dtype = _resolve_ming_runtime_dtype(vllm_config)
if self.ming_config.model_variant == "moe":
from vllm.model_executor.models.bailing_moe import BailingMoeModel
# BailingMoeModel reads ``vllm_config.model_config.hf_config`` directly
# (no get_text_config()), so re-wrap with the nested bailing_moe config.
llm_config = vllm_config.model_config.hf_config.get_text_config()
@@ -335,8 +338,6 @@ class MingLLMModel(nn.Module):
if self._cfm_graph is not None:
return self._cfm_graph
try:
from .fm.cfm_cudagraph import CFMGraphExecutor, CFMSampler
# TODO(perf):
# (1) compile per-block (dit_model.blocks) instead of the
# whole forward for tighter fusion / fewer graph breaks;
@@ -12,7 +12,7 @@ import torch
from safetensors import safe_open
from vllm.logger import init_logger
from vllm_omni.engine.stage_init_utils import _resolve_model_to_local_path
from vllm_omni.model_executor.model_loader.weight_utils import resolve_model_to_local_path
from vllm_omni.model_executor.models.common.ming.audio_vae import AudioVAE
from .audio_prep import (
@@ -78,7 +78,7 @@ def _load_prompt_encoder(wrapper: Any) -> AudioVAE:
loaded_encoder_params = set()
with torch.no_grad():
for shard_path in _iter_model_safetensors(
_resolve_model_to_local_path(str(wrapper.vllm_config.model_config.model))
resolve_model_to_local_path(str(wrapper.vllm_config.model_config.model))
):
with safe_open(str(shard_path), framework="pt", device="cpu") as handle:
for key in handle.keys():
@@ -3,16 +3,13 @@
# Adopted from https://github.com/inclusionAI/Ming-omni-tts/blob/main/spkemb_extractor.py
from __future__ import annotations
import os
import threading
from typing import TYPE_CHECKING, Any
import torch
import torchaudio
from vllm.logger import init_logger
from vllm.multimodal.media.audio import load_audio
from vllm_omni.model_executor.models.ming_flash_omni.spk_embedding import SpkembExtractor
from vllm_omni.model_executor.models.common.ming.speaker_extractor import SpeakerEmbeddingExtractor
from .audio_prep import coerce_prompt_waveform, coerce_speaker_embeddings
from .constants import (
@@ -29,48 +26,9 @@ logger = init_logger(__name__)
_EXTRACTOR_LOAD_LOCK = threading.Lock()
def resolve_model_to_local_path(model):
if os.path.isdir(model):
return model
from huggingface_hub import snapshot_download
return snapshot_download(model)
class MingSpeakerEmbeddingExtractor:
def __init__(self, model, target_sr=16000):
local_model_path = resolve_model_to_local_path(model)
campplus_path = os.path.join(local_model_path, "campplus.onnx")
if not os.path.exists(campplus_path):
raise RuntimeError(f"Missing Ming speaker extractor model: {campplus_path}")
self.target_sr = int(target_sr)
self._core = SpkembExtractor(campplus_path, target_sr=self.target_sr)
def extract_from_waveform(self, waveform, sample_rate):
if not isinstance(waveform, torch.Tensor):
waveform = torch.as_tensor(waveform)
tensor = waveform.detach().to(torch.float32)
if tensor.ndim == 1:
tensor = tensor.unsqueeze(0)
if int(sample_rate) != self.target_sr:
tensor = torchaudio.transforms.Resample(orig_freq=int(sample_rate), new_freq=self.target_sr)(tensor)
embedding = self._core._extract_spk_embedding(tensor)
return embedding.squeeze(0).to(dtype=torch.float32)
def extract_from_file(self, audio_path):
audio, sample_rate = load_audio(audio_path, sr=None, mono=False)
waveform = torch.from_numpy(audio).contiguous()
return self.extract_from_waveform(waveform, sample_rate)
def extract_many(self, audio_paths):
return [self.extract_from_file(path) for path in audio_paths]
def _load_speaker_extractor(wrapper: MingTTSForConditionalGeneration) -> MingSpeakerEmbeddingExtractor:
# Ming's bundle ships the CAM++ artifact as `campplus.onnx` in the model
# directory, which is the shared extractor's default.
def _load_speaker_extractor(wrapper: MingTTSForConditionalGeneration) -> SpeakerEmbeddingExtractor:
extractor = getattr(wrapper, "_speaker_extractor", None)
if extractor is not None:
return extractor
@@ -79,7 +37,7 @@ def _load_speaker_extractor(wrapper: MingTTSForConditionalGeneration) -> MingSpe
extractor = getattr(wrapper, "_speaker_extractor", None)
if extractor is None:
model_path = wrapper.vllm_config.model_config.model
extractor = MingSpeakerEmbeddingExtractor(model_path, target_sr=16000)
extractor = SpeakerEmbeddingExtractor(model_path, target_sr=16000)
wrapper._speaker_extractor = extractor
return extractor
@@ -174,8 +132,6 @@ def _resolve_speaker_embeddings(
__all__ = [
"MingSpeakerEmbeddingExtractor",
"_load_speaker_extractor",
"_resolve_speaker_embeddings",
"resolve_model_to_local_path",
]