vllm-omni/vllm_omni/diffusion/models/soulx_singer/modules/preprocess/stack.py

"""Lazy-loaded SoulX preprocess model tree."""

import os
import tempfile
from pathlib import Path
from typing import ClassVar

import numpy as np
import soundfile as sf
import torch
import torch.nn as nn
from vllm.logger import init_logger

from vllm_omni.diffusion.models.interface import SupportsComponentDiscovery

from .asr import LyricModel
from .mel_grid_f0 import extract_f0_file
from .rmvpe import RMVPE
from .rosvot import RosvotModel
from .segmenter import VocalSegmenter
from .utils import resample_mono
from .vocal_sep import MelBandRoformerSeparator

logger = init_logger(__name__)


class SoulXPreprocessStack(nn.Module, SupportsComponentDiscovery):
    """Unified nn.Module tree for SoulX-Singer audio preprocess."""

    _dit_modules: ClassVar[list[str]] = []
    _encoder_modules: ClassVar[list[str]] = ["vocal_sep", "lyric", "rosvot"]
    _vae_modules: ClassVar[list[str]] = []
    _resident_modules: ClassVar[list[str]] = ["rmvpe"]

    def __init__(
        self,
        weights: dict[str, str],
        device: str,
        *,
        target_sr: int = 24000,
        hop_size: int = 480,
        verbose: bool = False,
    ) -> None:
        super().__init__()
        self.weights = weights
        self.device_str = device
        self.target_sr = target_sr
        self.hop_size = hop_size
        self.verbose = verbose
        self._rmvpe: nn.Module | None = None
        self._vocal_sep: nn.Module | None = None
        self._segmenter: nn.Module | None = None
        self._lyric: nn.Module | None = None
        self._rosvot: nn.Module | None = None

    @property
    def device(self) -> torch.device:
        return torch.device(self.device_str)

    def ensure_rmvpe(self):
        if self._rmvpe is None:
            if self.verbose:
                logger.info("[rmvpe] loading shared pitch model")
            self._rmvpe = RMVPE(self.weights["rmvpe"], device=self.device_str)
            self.add_module("rmvpe", self._rmvpe)
        return self._rmvpe

    def ensure_vocal_sep(self):
        if self._vocal_sep is None:
            if self.verbose:
                logger.info("[vocal extraction] loading model")
            self._vocal_sep = MelBandRoformerSeparator(
                self.weights["sep_config"],
                self.weights["sep_ckpt"],
                use_half=False,
            ).to(self.device)
            self.add_module("vocal_sep", self._vocal_sep)
            if self.verbose:
                logger.info("[vocal extraction] ready on %s, sr=%s", self.device_str, self._vocal_sep.sample_rate)
        return self._vocal_sep

    def ensure_segmenter(self):
        if self._segmenter is None:
            self._segmenter = VocalSegmenter(verbose=self.verbose)
            self.add_module("segmenter", self._segmenter)
        return self._segmenter

    def ensure_lyric(self):
        if self._lyric is None:
            if self.verbose:
                logger.info("[lyric transcription] loading ASR")
            self._lyric = LyricModel(
                self.weights["asr_zh"],
                self.weights["asr_en"],
                device=self.device_str,
                target_sr=self.target_sr,
                hop_size=self.hop_size,
            )
            self.add_module("lyric", self._lyric)
        return self._lyric

    def ensure_rosvot(self):
        if self._rosvot is None:
            if self.verbose:
                logger.info("[note transcription] loading ROSVOT")
            rosvot_src = os.environ.get("ROSVOT_SOURCE_DIR")
            self._rosvot = RosvotModel(
                self.weights["rosvot"],
                pe=self.ensure_rmvpe(),
                verbose=self.verbose,
                rosvot_source_dir=rosvot_src,
            ).to(self.device)
            self.add_module("rosvot", self._rosvot)
        return self._rosvot

    def extract_f0(self, vocal, sample_rate: int, *, f0_path: str | None = None) -> np.ndarray:
        """Extract F0 on the mel grid; ndarray inputs are spooled to a temp wav."""
        rmvpe = self.ensure_rmvpe()
        kwargs = dict(
            target_sr=self.target_sr,
            hop_size=self.hop_size,
            f0_path=f0_path,
            verbose=self.verbose,
        )
        if isinstance(vocal, str):
            return extract_f0_file(rmvpe, vocal, **kwargs)
        tmp_path: str | None = None
        try:
            with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as tmp:
                tmp_path = tmp.name
            sf.write(tmp_path, np.asarray(vocal, dtype=np.float32).squeeze(), int(sample_rate))
            return extract_f0_file(rmvpe, tmp_path, **kwargs)
        finally:
            if tmp_path is not None:
                Path(tmp_path).unlink(missing_ok=True)

    def extract_vocal(self, audio: str | tuple[np.ndarray, int]) -> tuple[np.ndarray, int]:
        sep = self.ensure_vocal_sep()
        if isinstance(audio, tuple):
            mix, sr = audio
            mix = np.asarray(mix, dtype=np.float32).squeeze()
            if sr != sep.sample_rate:
                mix = resample_mono(mix, orig_sr=sr, target_sr=sep.sample_rate)
            return sep.separate_mono(mix, self.device), sep.sample_rate
        return sep.separate_path(str(audio), self.device)