[Model] add soulx-singer support (#3862)
Signed-off-by: Mr.Dong <1042542469@qq.com> Signed-off-by: Yueqian Lin <linyueqian@outlook.com> Co-authored-by: Mr.Dong <1042542469@qq.com> Co-authored-by: Yueqian Lin <70319226+linyueqian@users.noreply.github.com> Co-authored-by: Yueqian Lin <linyueqian@outlook.com>
This commit is contained in:
50 files changed
+9533
-13
No files matched your search
@@ -0,0 +1,145 @@
|
||||
"""Lazy-loaded SoulX preprocess model tree."""
|
||||
|
||||
import os
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from typing import ClassVar
|
||||
|
||||
import numpy as np
|
||||
import soundfile as sf
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from vllm_omni.diffusion.models.interface import SupportsComponentDiscovery
|
||||
|
||||
from .asr import LyricModel
|
||||
from .mel_grid_f0 import extract_f0_file
|
||||
from .rmvpe import RMVPE
|
||||
from .rosvot import RosvotModel
|
||||
from .segmenter import VocalSegmenter
|
||||
from .utils import resample_mono
|
||||
from .vocal_sep import MelBandRoformerSeparator
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class SoulXPreprocessStack(nn.Module, SupportsComponentDiscovery):
|
||||
"""Unified nn.Module tree for SoulX-Singer audio preprocess."""
|
||||
|
||||
_dit_modules: ClassVar[list[str]] = []
|
||||
_encoder_modules: ClassVar[list[str]] = ["vocal_sep", "lyric", "rosvot"]
|
||||
_vae_modules: ClassVar[list[str]] = []
|
||||
_resident_modules: ClassVar[list[str]] = ["rmvpe"]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
weights: dict[str, str],
|
||||
device: str,
|
||||
*,
|
||||
target_sr: int = 24000,
|
||||
hop_size: int = 480,
|
||||
verbose: bool = False,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.weights = weights
|
||||
self.device_str = device
|
||||
self.target_sr = target_sr
|
||||
self.hop_size = hop_size
|
||||
self.verbose = verbose
|
||||
self._rmvpe: nn.Module | None = None
|
||||
self._vocal_sep: nn.Module | None = None
|
||||
self._segmenter: nn.Module | None = None
|
||||
self._lyric: nn.Module | None = None
|
||||
self._rosvot: nn.Module | None = None
|
||||
|
||||
@property
|
||||
def device(self) -> torch.device:
|
||||
return torch.device(self.device_str)
|
||||
|
||||
def ensure_rmvpe(self):
|
||||
if self._rmvpe is None:
|
||||
if self.verbose:
|
||||
logger.info("[rmvpe] loading shared pitch model")
|
||||
self._rmvpe = RMVPE(self.weights["rmvpe"], device=self.device_str)
|
||||
self.add_module("rmvpe", self._rmvpe)
|
||||
return self._rmvpe
|
||||
|
||||
def ensure_vocal_sep(self):
|
||||
if self._vocal_sep is None:
|
||||
if self.verbose:
|
||||
logger.info("[vocal extraction] loading model")
|
||||
self._vocal_sep = MelBandRoformerSeparator(
|
||||
self.weights["sep_config"],
|
||||
self.weights["sep_ckpt"],
|
||||
use_half=False,
|
||||
).to(self.device)
|
||||
self.add_module("vocal_sep", self._vocal_sep)
|
||||
if self.verbose:
|
||||
logger.info("[vocal extraction] ready on %s, sr=%s", self.device_str, self._vocal_sep.sample_rate)
|
||||
return self._vocal_sep
|
||||
|
||||
def ensure_segmenter(self):
|
||||
if self._segmenter is None:
|
||||
self._segmenter = VocalSegmenter(verbose=self.verbose)
|
||||
self.add_module("segmenter", self._segmenter)
|
||||
return self._segmenter
|
||||
|
||||
def ensure_lyric(self):
|
||||
if self._lyric is None:
|
||||
if self.verbose:
|
||||
logger.info("[lyric transcription] loading ASR")
|
||||
self._lyric = LyricModel(
|
||||
self.weights["asr_zh"],
|
||||
self.weights["asr_en"],
|
||||
device=self.device_str,
|
||||
target_sr=self.target_sr,
|
||||
hop_size=self.hop_size,
|
||||
)
|
||||
self.add_module("lyric", self._lyric)
|
||||
return self._lyric
|
||||
|
||||
def ensure_rosvot(self):
|
||||
if self._rosvot is None:
|
||||
if self.verbose:
|
||||
logger.info("[note transcription] loading ROSVOT")
|
||||
rosvot_src = os.environ.get("ROSVOT_SOURCE_DIR")
|
||||
self._rosvot = RosvotModel(
|
||||
self.weights["rosvot"],
|
||||
pe=self.ensure_rmvpe(),
|
||||
verbose=self.verbose,
|
||||
rosvot_source_dir=rosvot_src,
|
||||
).to(self.device)
|
||||
self.add_module("rosvot", self._rosvot)
|
||||
return self._rosvot
|
||||
|
||||
def extract_f0(self, vocal, sample_rate: int, *, f0_path: str | None = None) -> np.ndarray:
|
||||
"""Extract F0 on the mel grid; ndarray inputs are spooled to a temp wav."""
|
||||
rmvpe = self.ensure_rmvpe()
|
||||
kwargs = dict(
|
||||
target_sr=self.target_sr,
|
||||
hop_size=self.hop_size,
|
||||
f0_path=f0_path,
|
||||
verbose=self.verbose,
|
||||
)
|
||||
if isinstance(vocal, str):
|
||||
return extract_f0_file(rmvpe, vocal, **kwargs)
|
||||
tmp_path: str | None = None
|
||||
try:
|
||||
with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as tmp:
|
||||
tmp_path = tmp.name
|
||||
sf.write(tmp_path, np.asarray(vocal, dtype=np.float32).squeeze(), int(sample_rate))
|
||||
return extract_f0_file(rmvpe, tmp_path, **kwargs)
|
||||
finally:
|
||||
if tmp_path is not None:
|
||||
Path(tmp_path).unlink(missing_ok=True)
|
||||
|
||||
def extract_vocal(self, audio: str | tuple[np.ndarray, int]) -> tuple[np.ndarray, int]:
|
||||
sep = self.ensure_vocal_sep()
|
||||
if isinstance(audio, tuple):
|
||||
mix, sr = audio
|
||||
mix = np.asarray(mix, dtype=np.float32).squeeze()
|
||||
if sr != sep.sample_rate:
|
||||
mix = resample_mono(mix, orig_sr=sr, target_sr=sep.sample_rate)
|
||||
return sep.separate_mono(mix, self.device), sep.sample_rate
|
||||
return sep.separate_path(str(audio), self.device)
|
||||
Reference in new issue
Block a user