[Ming] Cleanup Ming-family shared modules (#6119)
Signed-off-by: Yuanheng <jonathan.zhaoyh@gmail.com> Signed-off-by: Yuanheng Zhao <jonathan.zhaoyh@gmail.com>
This commit is contained in:
1 parent
adf6d0c1ea
commit
a630b0c2ec
22 files changed
+380
-192
No files matched your search
@@ -1,2 +1,13 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Components shared by the Ming-family model packages.
|
||||
|
||||
Naming: The flow-matching pieces all use the cfm_ prefix,
|
||||
because "flow matching", "fm" and "cfm" were three names for one concept here:
|
||||
|
||||
* dit_blocks - DiT building blocks (DiTBlock, FinalLayer, CondEmbedder)
|
||||
* cfm_dit - the DiT used as the CFM velocity model
|
||||
* cfm_solver - timestep schedules and the ODE/SDE integrator
|
||||
* cfm_head - inference-only CFM wrapper preserving cfm.model.* names
|
||||
* cfm_cudagraph - batch-1 CUDA-graph executor for the sampling loop
|
||||
"""
|
||||
+1
-1
@@ -7,7 +7,7 @@ import torch
|
||||
import torch.nn as nn
|
||||
from x_transformers.x_transformers import RotaryEmbedding
|
||||
|
||||
from vllm_omni.model_executor.models.common.ming.dit import DiTBlock, FinalLayer
|
||||
from vllm_omni.model_executor.models.common.ming.dit_blocks import DiTBlock, FinalLayer
|
||||
|
||||
|
||||
class Aggregator(nn.Module):
|
||||
+1
-1
@@ -17,7 +17,7 @@ from __future__ import annotations
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm_omni.model_executor.models.common.ming.fm import build_timesteps
|
||||
from vllm_omni.model_executor.models.common.ming.cfm_solver import build_timesteps
|
||||
|
||||
|
||||
class CFMSampler(nn.Module):
|
||||
+1
-1
@@ -7,7 +7,7 @@ import torch.nn as nn
|
||||
from x_transformers.x_transformers import RotaryEmbedding
|
||||
|
||||
from vllm_omni.model_executor.layers.timestep_embedding import DiTTimestepEmbedding
|
||||
from vllm_omni.model_executor.models.common.ming.dit import CondEmbedder, DiTBlock, FinalLayer
|
||||
from vllm_omni.model_executor.models.common.ming.dit_blocks import CondEmbedder, DiTBlock, FinalLayer
|
||||
|
||||
|
||||
class DiT(nn.Module):
|
||||
+2
-3
@@ -5,9 +5,8 @@
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm_omni.model_executor.models.common.ming.fm import Solver, build_timesteps
|
||||
|
||||
from .fm.dit import DiT
|
||||
from vllm_omni.model_executor.models.common.ming.cfm_dit import DiT
|
||||
from vllm_omni.model_executor.models.common.ming.cfm_solver import Solver, build_timesteps
|
||||
|
||||
|
||||
class FlowLoss(nn.Module):
|
||||
+15
-1
@@ -3,7 +3,21 @@
|
||||
|
||||
import torch
|
||||
|
||||
from .dit import get_epss_timesteps
|
||||
|
||||
def get_epss_timesteps(n, device, dtype):
|
||||
dt = 1 / 32
|
||||
predefined_timesteps = {
|
||||
5: [0, 2, 4, 8, 16, 32],
|
||||
6: [0, 2, 4, 6, 8, 16, 32],
|
||||
7: [0, 2, 4, 6, 8, 16, 24, 32],
|
||||
10: [0, 2, 4, 6, 8, 12, 16, 20, 24, 28, 32],
|
||||
12: [0, 2, 4, 6, 8, 10, 12, 14, 16, 20, 24, 28, 32],
|
||||
16: [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 12, 14, 16, 20, 24, 28, 32],
|
||||
}
|
||||
t = predefined_timesteps.get(n, [])
|
||||
if not t:
|
||||
return torch.linspace(0, 1, n + 1, device=device, dtype=dtype)
|
||||
return dt * torch.tensor(t, device=device, dtype=dtype)
|
||||
|
||||
|
||||
def apply_sway_sampling(t, sway_sampling_coef):
|
||||
-16
@@ -216,19 +216,3 @@ class CondEmbedder(nn.Module):
|
||||
|
||||
def forward(self, llm_cond):
|
||||
return self.cond_embedder(llm_cond)
|
||||
|
||||
|
||||
def get_epss_timesteps(n, device, dtype):
|
||||
dt = 1 / 32
|
||||
predefined_timesteps = {
|
||||
5: [0, 2, 4, 8, 16, 32],
|
||||
6: [0, 2, 4, 6, 8, 16, 32],
|
||||
7: [0, 2, 4, 6, 8, 16, 24, 32],
|
||||
10: [0, 2, 4, 6, 8, 12, 16, 20, 24, 28, 32],
|
||||
12: [0, 2, 4, 6, 8, 10, 12, 14, 16, 20, 24, 28, 32],
|
||||
16: [0, 1, 2, 3, 4, 5, 6, 7, 8, 10, 12, 14, 16, 20, 24, 28, 32],
|
||||
}
|
||||
t = predefined_timesteps.get(n, [])
|
||||
if not t:
|
||||
return torch.linspace(0, 1, n + 1, device=device, dtype=dtype)
|
||||
return dt * torch.tensor(t, device=device, dtype=dtype)
|
||||
@@ -0,0 +1,56 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# Adopted from https://github.com/inclusionAI/Ming-omni-tts/blob/main/spkemb_extractor.py
|
||||
"""CAM++ speaker-embedding extraction, shared by the Ming model packages."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
|
||||
import torch
|
||||
import torchaudio
|
||||
from vllm.multimodal.media.audio import load_audio
|
||||
|
||||
from vllm_omni.model_executor.model_loader.weight_utils import resolve_model_to_local_path
|
||||
from vllm_omni.model_executor.models.common.ming.spk_embedding import SpkembExtractor
|
||||
|
||||
|
||||
class SpeakerEmbeddingExtractor:
|
||||
"""CAM++ ONNX speaker embeddings, resampled to the extractor's rate."""
|
||||
|
||||
def __init__(self, model, target_sr=16000, *, allow_download: bool = False):
|
||||
local_model_path = resolve_model_to_local_path(
|
||||
model,
|
||||
allow_download=allow_download,
|
||||
allow_patterns=["campplus.onnx"],
|
||||
)
|
||||
campplus_path = os.path.join(local_model_path, "campplus.onnx")
|
||||
if not os.path.exists(campplus_path):
|
||||
raise RuntimeError(f"Missing Ming speaker extractor model: {campplus_path}")
|
||||
|
||||
self.target_sr = int(target_sr)
|
||||
self._core = SpkembExtractor(campplus_path, target_sr=self.target_sr)
|
||||
|
||||
def extract_from_waveform(self, waveform, sample_rate):
|
||||
if not isinstance(waveform, torch.Tensor):
|
||||
waveform = torch.as_tensor(waveform)
|
||||
|
||||
tensor = waveform.detach().to(torch.float32)
|
||||
if tensor.ndim == 1:
|
||||
tensor = tensor.unsqueeze(0)
|
||||
if int(sample_rate) != self.target_sr:
|
||||
tensor = torchaudio.transforms.Resample(orig_freq=int(sample_rate), new_freq=self.target_sr)(tensor)
|
||||
|
||||
embedding = self._core._extract_spk_embedding(tensor)
|
||||
return embedding.squeeze(0).to(dtype=torch.float32)
|
||||
|
||||
def extract_from_file(self, audio_path):
|
||||
audio, sample_rate = load_audio(audio_path, sr=None, mono=False)
|
||||
waveform = torch.from_numpy(audio).contiguous()
|
||||
return self.extract_from_waveform(waveform, sample_rate)
|
||||
|
||||
def extract_many(self, audio_paths):
|
||||
return [self.extract_from_file(path) for path in audio_paths]
|
||||
|
||||
|
||||
__all__ = ["SpeakerEmbeddingExtractor"]
|
||||
File renamed without changes.
@@ -25,14 +25,15 @@ from vllm.model_executor.models.utils import AutoWeightsLoader
|
||||
from vllm.sequence import IntermediateTensors
|
||||
|
||||
from vllm_omni.model_executor.custom_process_mixin import CustomProcessMixin
|
||||
from vllm_omni.model_executor.model_loader.weight_utils import download_weights_from_hf_specific
|
||||
from vllm_omni.model_executor.model_loader.weight_utils import resolve_model_to_local_path
|
||||
from vllm_omni.model_executor.models.common.ming.aggregator import Aggregator
|
||||
from vllm_omni.model_executor.models.common.ming.audio_vae import AudioVAE, AudioVAEConfig
|
||||
from vllm_omni.model_executor.models.ming_tts.constants import SPEAKER_EMBEDDING_DIM
|
||||
from vllm_omni.model_executor.models.output_templates import OmniOutput
|
||||
from vllm_omni.transformers_utils.configs.ming_flash_omni import MingFlashOmniTalkerConfig
|
||||
|
||||
from .prompt_utils import DEFAULT_PROMPT as MING_DEFAULT_PROMPT
|
||||
from .talker_module import CFM, Aggregator, DiT, MingAudioGenerator, build_tts_input
|
||||
from .talker_module import CFM, DiT, MingAudioGenerator, build_tts_input
|
||||
from .text_processing import segment_and_normalize
|
||||
from .voice_presets import VoicePresetRegistry
|
||||
|
||||
@@ -119,7 +120,11 @@ class MingFlashOmniTalkerForConditionalGeneration(nn.Module, CustomProcessMixin)
|
||||
DiT(llm_input_dim=self.hidden_size, **config.flowmodel),
|
||||
steps=config.steps,
|
||||
)
|
||||
self.aggregator = Aggregator(llm_input_dim=self.hidden_size, **config.aggregator)
|
||||
# config.aggregator still wins `in_channels` if the checkpoint states it.
|
||||
self.aggregator = Aggregator(
|
||||
llm_input_dim=self.hidden_size,
|
||||
**{"in_channels": self.latent_dim, **config.aggregator},
|
||||
)
|
||||
self.stop_head = nn.Linear(self.hidden_size, 2, bias=True)
|
||||
self.spk_head = nn.Linear(SPEAKER_EMBEDDING_DIM, self.hidden_size, bias=True)
|
||||
|
||||
@@ -525,27 +530,22 @@ class MingFlashOmniTalkerForConditionalGeneration(nn.Module, CustomProcessMixin)
|
||||
|
||||
def _iter_talker_safetensors(self) -> Iterable[tuple[str, torch.Tensor]]:
|
||||
"""Yield (name, tensor) pairs from talker/model*.safetensors."""
|
||||
model_path = self._model_path
|
||||
# Try local path first
|
||||
for candidate in (os.path.join(model_path, "talker"), model_path):
|
||||
model_root = resolve_model_to_local_path(
|
||||
self._model_path,
|
||||
allow_download=True,
|
||||
allow_patterns=["talker/model*.safetensors"],
|
||||
cache_dir=self.vllm_config.load_config.download_dir,
|
||||
)
|
||||
|
||||
# Nested talker/ layout first, then a flat checkpoint at the root.
|
||||
for candidate in (os.path.join(model_root, "talker"), model_root):
|
||||
sf_files = sorted(glob_module.glob(os.path.join(candidate, "model*.safetensors")))
|
||||
if sf_files:
|
||||
for sf_path in sf_files:
|
||||
yield from load_file(sf_path, device="cpu").items()
|
||||
return
|
||||
|
||||
# HF hub fallback: download only the talker checkpoint files
|
||||
model_root = download_weights_from_hf_specific(
|
||||
model_path,
|
||||
self.vllm_config.load_config.download_dir,
|
||||
allow_patterns=["talker/model*.safetensors"],
|
||||
)
|
||||
talker_dir = os.path.join(model_root, "talker")
|
||||
sf_files = sorted(glob_module.glob(os.path.join(talker_dir, "model*.safetensors")))
|
||||
if not sf_files:
|
||||
raise RuntimeError(f"No talker safetensors found under {model_root}. Expected talker/model*.safetensors.")
|
||||
for sf_path in sf_files:
|
||||
yield from load_file(sf_path, device="cpu").items()
|
||||
raise RuntimeError(f"No talker safetensors found under {model_root}. Expected talker/model*.safetensors.")
|
||||
|
||||
def _load_vae_weights(self) -> set[str]:
|
||||
"""Load AudioVAE weights from talker/vae/model.safetensors."""
|
||||
|
||||
@@ -64,6 +64,7 @@ from vllm.sequence import IntermediateTensors
|
||||
from vllm.utils.tensor_schema import TensorSchema, TensorShape
|
||||
|
||||
from vllm_omni.model_executor.custom_process_mixin import CustomProcessMixin
|
||||
from vllm_omni.model_executor.model_loader.weight_utils import resolve_model_to_local_path
|
||||
from vllm_omni.model_executor.models.output_templates import OmniOutput
|
||||
from vllm_omni.transformers_utils.configs.ming_flash_omni import BailingMM2Config
|
||||
from vllm_omni.transformers_utils.processors.ming import (
|
||||
@@ -715,12 +716,8 @@ class MingFlashOmniThinkerForConditionalGeneration(
|
||||
|
||||
p = Path(model_path) / "mlp" / "model.safetensors"
|
||||
if not p.exists() and not os.path.exists(model_path):
|
||||
from vllm_omni.model_executor.model_loader.weight_utils import (
|
||||
download_weights_from_hf_specific,
|
||||
)
|
||||
|
||||
try:
|
||||
local_path = download_weights_from_hf_specific(model_path, None, ["mlp/*"])
|
||||
local_path = resolve_model_to_local_path(model_path, allow_download=True, allow_patterns=["mlp/*"])
|
||||
p = Path(local_path) / "mlp" / "model.safetensors"
|
||||
except Exception:
|
||||
logger.exception("[MingFlashOmniThinker] failed to download mlp/ from %s", model_path)
|
||||
|
||||
@@ -33,9 +33,14 @@ from vllm.platforms import current_platform
|
||||
from x_transformers.x_transformers import RotaryEmbedding
|
||||
|
||||
from vllm_omni.model_executor.layers.timestep_embedding import DiTTimestepEmbedding
|
||||
from vllm_omni.model_executor.models.common.ming.aggregator import Aggregator
|
||||
from vllm_omni.model_executor.models.common.ming.audio_vae import AudioVAE
|
||||
from vllm_omni.model_executor.models.common.ming.dit import CondEmbedder, DiTBlock, FinalLayer, get_epss_timesteps
|
||||
from vllm_omni.model_executor.models.common.ming.fm import apply_sway_sampling, integrate_cfm_steps
|
||||
from vllm_omni.model_executor.models.common.ming.cfm_solver import (
|
||||
apply_sway_sampling,
|
||||
get_epss_timesteps,
|
||||
integrate_cfm_steps,
|
||||
)
|
||||
from vllm_omni.model_executor.models.common.ming.dit_blocks import CondEmbedder, DiTBlock, FinalLayer
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -458,58 +463,6 @@ def silence_holder(
|
||||
return speech_out, sil_cache
|
||||
|
||||
|
||||
########################################################################
|
||||
# Audio Postprocess
|
||||
# Ported from:
|
||||
# https://github.com/inclusionAI/Ming/blob/e58533db227031990c5a6864dcf5f08fb53ed0d2/talker_module/aggregator.py
|
||||
########################################################################
|
||||
|
||||
|
||||
class Aggregator(nn.Module):
|
||||
"""Maps generated audio latent patches back to LLM embedding space."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
in_channels: int = 64,
|
||||
hidden_size: int = 1152,
|
||||
depth: int = 28,
|
||||
num_heads: int = 16,
|
||||
mlp_ratio: float = 4.0,
|
||||
llm_input_dim: int = 896,
|
||||
**kwargs,
|
||||
):
|
||||
super().__init__()
|
||||
self.in_channels = in_channels
|
||||
self.out_channels = in_channels
|
||||
self.num_heads = num_heads
|
||||
|
||||
self.word_embedder = nn.Embedding(1, hidden_size)
|
||||
self.x_embedder = nn.Linear(in_channels, hidden_size)
|
||||
self.hidden_size = hidden_size
|
||||
|
||||
self.rotary_embed = RotaryEmbedding(hidden_size // num_heads)
|
||||
|
||||
self.blocks = nn.ModuleList(
|
||||
[DiTBlock(hidden_size, num_heads, mlp_ratio=mlp_ratio, **kwargs) for _ in range(depth)]
|
||||
)
|
||||
self.final_layer = FinalLayer(hidden_size, llm_input_dim)
|
||||
|
||||
def forward(self, x: torch.Tensor, mask: torch.Tensor | None = None) -> torch.Tensor:
|
||||
x = self.x_embedder(x)
|
||||
cls_embed = self.word_embedder(torch.zeros((x.shape[0], 1), dtype=torch.long, device=x.device))
|
||||
x = torch.cat([cls_embed, x], dim=1)
|
||||
|
||||
rope = self.rotary_embed.forward_from_seq_len(x.shape[1])
|
||||
if mask is not None:
|
||||
mask_pad = mask.clone().detach()[:, :1]
|
||||
mask = torch.cat([mask_pad, mask], dim=-1)
|
||||
for block in self.blocks:
|
||||
x = block(x, mask, rope)
|
||||
x = self.final_layer(x)
|
||||
x = x[:, :1, :]
|
||||
return x
|
||||
|
||||
|
||||
########################################################################
|
||||
# Prompt Builder
|
||||
# Adapted from:
|
||||
|
||||
@@ -16,16 +16,15 @@ import torch
|
||||
from transformers.utils.hub import cached_file
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from vllm_omni.model_executor.model_loader.weight_utils import download_weights_from_hf_specific
|
||||
from vllm_omni.model_executor.model_loader.weight_utils import resolve_model_to_local_path
|
||||
from vllm_omni.model_executor.models.common.ming.spk_embedding import SpkembExtractor
|
||||
|
||||
from .spk_embedding import SpkembExtractor
|
||||
from .talker_module import resample
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm_omni.model_executor.models.common.ming.aggregator import Aggregator
|
||||
from vllm_omni.model_executor.models.common.ming.audio_vae import AudioVAE
|
||||
|
||||
from .talker_module import Aggregator
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@@ -273,18 +272,19 @@ class VoicePresetRegistry:
|
||||
if os.path.isfile(path):
|
||||
return path, candidate
|
||||
|
||||
if not os.path.isdir(self._model_path):
|
||||
try:
|
||||
hf_root = download_weights_from_hf_specific(
|
||||
self._model_path,
|
||||
self._download_dir,
|
||||
allow_patterns=["talker/data/**"],
|
||||
require_all=True,
|
||||
)
|
||||
candidate = os.path.join(hf_root, "talker", "data", "voice_name.json")
|
||||
if os.path.isfile(candidate):
|
||||
return candidate, os.path.join(hf_root, "talker")
|
||||
except Exception as e: # pragma: no cover
|
||||
logger.info("Could not download voice presets from HF: %s", e)
|
||||
try:
|
||||
hf_root = resolve_model_to_local_path(
|
||||
self._model_path,
|
||||
allow_download=True,
|
||||
allow_patterns=["talker/data/**"],
|
||||
cache_dir=self._download_dir,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.info("Could not download voice presets from HF: %s", e)
|
||||
return None, None
|
||||
|
||||
candidate = os.path.join(hf_root, "talker", "data", "voice_name.json")
|
||||
if os.path.isfile(candidate):
|
||||
return candidate, os.path.join(hf_root, "talker")
|
||||
|
||||
return None, None
|
||||
@@ -1,2 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
@@ -11,15 +11,21 @@ import torch
|
||||
import torch.nn as nn
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.models.bailing_moe import BailingMoeModel
|
||||
from vllm.model_executor.models.qwen2 import Qwen2Model
|
||||
from vllm.model_executor.models.utils import AutoWeightsLoader, WeightsMapper, maybe_prefix
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.v1.outputs import SamplerOutput
|
||||
from vllm.v1.sample.metadata import SamplingMetadata
|
||||
|
||||
from vllm_omni.model_executor.models.common.ming.aggregator import Aggregator
|
||||
from vllm_omni.model_executor.models.common.ming.cfm_cudagraph import (
|
||||
CFMGraphExecutor,
|
||||
CFMSampler,
|
||||
)
|
||||
from vllm_omni.model_executor.models.common.ming.cfm_head import FlowLoss
|
||||
from vllm_omni.model_executor.models.output_templates import OmniOutput
|
||||
|
||||
from .aggregator import Aggregator
|
||||
from .config_ming_tts import (
|
||||
KEY_CFG,
|
||||
KEY_DECODE_STEP,
|
||||
@@ -34,7 +40,6 @@ from .config_ming_tts import (
|
||||
MingTTSConfig,
|
||||
)
|
||||
from .constants import SPEAKER_EMBEDDING_DIM
|
||||
from .flowloss_head import FlowLoss
|
||||
from .patch_emission import (
|
||||
MING_STOP_REASON_CODES,
|
||||
MING_STOP_REASON_CONTINUE,
|
||||
@@ -82,8 +87,6 @@ class MingLLMModel(nn.Module):
|
||||
self.prefix = prefix
|
||||
self.fm_dtype = _resolve_ming_runtime_dtype(vllm_config)
|
||||
if self.ming_config.model_variant == "moe":
|
||||
from vllm.model_executor.models.bailing_moe import BailingMoeModel
|
||||
|
||||
# BailingMoeModel reads ``vllm_config.model_config.hf_config`` directly
|
||||
# (no get_text_config()), so re-wrap with the nested bailing_moe config.
|
||||
llm_config = vllm_config.model_config.hf_config.get_text_config()
|
||||
@@ -335,8 +338,6 @@ class MingLLMModel(nn.Module):
|
||||
if self._cfm_graph is not None:
|
||||
return self._cfm_graph
|
||||
try:
|
||||
from .fm.cfm_cudagraph import CFMGraphExecutor, CFMSampler
|
||||
|
||||
# TODO(perf):
|
||||
# (1) compile per-block (dit_model.blocks) instead of the
|
||||
# whole forward for tighter fusion / fewer graph breaks;
|
||||
|
||||
@@ -12,7 +12,7 @@ import torch
|
||||
from safetensors import safe_open
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from vllm_omni.engine.stage_init_utils import _resolve_model_to_local_path
|
||||
from vllm_omni.model_executor.model_loader.weight_utils import resolve_model_to_local_path
|
||||
from vllm_omni.model_executor.models.common.ming.audio_vae import AudioVAE
|
||||
|
||||
from .audio_prep import (
|
||||
@@ -78,7 +78,7 @@ def _load_prompt_encoder(wrapper: Any) -> AudioVAE:
|
||||
loaded_encoder_params = set()
|
||||
with torch.no_grad():
|
||||
for shard_path in _iter_model_safetensors(
|
||||
_resolve_model_to_local_path(str(wrapper.vllm_config.model_config.model))
|
||||
resolve_model_to_local_path(str(wrapper.vllm_config.model_config.model))
|
||||
):
|
||||
with safe_open(str(shard_path), framework="pt", device="cpu") as handle:
|
||||
for key in handle.keys():
|
||||
|
||||
@@ -3,16 +3,13 @@
|
||||
# Adopted from https://github.com/inclusionAI/Ming-omni-tts/blob/main/spkemb_extractor.py
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import threading
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import torch
|
||||
import torchaudio
|
||||
from vllm.logger import init_logger
|
||||
from vllm.multimodal.media.audio import load_audio
|
||||
|
||||
from vllm_omni.model_executor.models.ming_flash_omni.spk_embedding import SpkembExtractor
|
||||
from vllm_omni.model_executor.models.common.ming.speaker_extractor import SpeakerEmbeddingExtractor
|
||||
|
||||
from .audio_prep import coerce_prompt_waveform, coerce_speaker_embeddings
|
||||
from .constants import (
|
||||
@@ -29,48 +26,9 @@ logger = init_logger(__name__)
|
||||
_EXTRACTOR_LOAD_LOCK = threading.Lock()
|
||||
|
||||
|
||||
def resolve_model_to_local_path(model):
|
||||
if os.path.isdir(model):
|
||||
return model
|
||||
|
||||
from huggingface_hub import snapshot_download
|
||||
|
||||
return snapshot_download(model)
|
||||
|
||||
|
||||
class MingSpeakerEmbeddingExtractor:
|
||||
def __init__(self, model, target_sr=16000):
|
||||
local_model_path = resolve_model_to_local_path(model)
|
||||
campplus_path = os.path.join(local_model_path, "campplus.onnx")
|
||||
if not os.path.exists(campplus_path):
|
||||
raise RuntimeError(f"Missing Ming speaker extractor model: {campplus_path}")
|
||||
|
||||
self.target_sr = int(target_sr)
|
||||
self._core = SpkembExtractor(campplus_path, target_sr=self.target_sr)
|
||||
|
||||
def extract_from_waveform(self, waveform, sample_rate):
|
||||
if not isinstance(waveform, torch.Tensor):
|
||||
waveform = torch.as_tensor(waveform)
|
||||
|
||||
tensor = waveform.detach().to(torch.float32)
|
||||
if tensor.ndim == 1:
|
||||
tensor = tensor.unsqueeze(0)
|
||||
if int(sample_rate) != self.target_sr:
|
||||
tensor = torchaudio.transforms.Resample(orig_freq=int(sample_rate), new_freq=self.target_sr)(tensor)
|
||||
|
||||
embedding = self._core._extract_spk_embedding(tensor)
|
||||
return embedding.squeeze(0).to(dtype=torch.float32)
|
||||
|
||||
def extract_from_file(self, audio_path):
|
||||
audio, sample_rate = load_audio(audio_path, sr=None, mono=False)
|
||||
waveform = torch.from_numpy(audio).contiguous()
|
||||
return self.extract_from_waveform(waveform, sample_rate)
|
||||
|
||||
def extract_many(self, audio_paths):
|
||||
return [self.extract_from_file(path) for path in audio_paths]
|
||||
|
||||
|
||||
def _load_speaker_extractor(wrapper: MingTTSForConditionalGeneration) -> MingSpeakerEmbeddingExtractor:
|
||||
# Ming's bundle ships the CAM++ artifact as `campplus.onnx` in the model
|
||||
# directory, which is the shared extractor's default.
|
||||
def _load_speaker_extractor(wrapper: MingTTSForConditionalGeneration) -> SpeakerEmbeddingExtractor:
|
||||
extractor = getattr(wrapper, "_speaker_extractor", None)
|
||||
if extractor is not None:
|
||||
return extractor
|
||||
@@ -79,7 +37,7 @@ def _load_speaker_extractor(wrapper: MingTTSForConditionalGeneration) -> MingSpe
|
||||
extractor = getattr(wrapper, "_speaker_extractor", None)
|
||||
if extractor is None:
|
||||
model_path = wrapper.vllm_config.model_config.model
|
||||
extractor = MingSpeakerEmbeddingExtractor(model_path, target_sr=16000)
|
||||
extractor = SpeakerEmbeddingExtractor(model_path, target_sr=16000)
|
||||
wrapper._speaker_extractor = extractor
|
||||
return extractor
|
||||
|
||||
@@ -174,8 +132,6 @@ def _resolve_speaker_embeddings(
|
||||
|
||||
|
||||
__all__ = [
|
||||
"MingSpeakerEmbeddingExtractor",
|
||||
"_load_speaker_extractor",
|
||||
"_resolve_speaker_embeddings",
|
||||
"resolve_model_to_local_path",
|
||||
]
|
||||
Reference in new issue
Block a user