vllm-omni/vllm_omni/model_executor/models/voxcpm/voxcpm_loader.py

from __future__ import annotations

import importlib
import json
import os
import shutil
import sys
import tempfile
from contextlib import contextmanager
from hashlib import sha256
from pathlib import Path
from typing import Any
from unittest.mock import patch

import numpy as np
import torch
from vllm.config import VllmConfig
from vllm.logger import init_logger

logger = init_logger(__name__)


def _iter_voxcpm_src_candidates() -> list[Path]:
    candidates: list[Path] = []
    env_path = os.environ.get("VLLM_OMNI_VOXCPM_CODE_PATH")
    if env_path:
        candidates.append(Path(env_path).expanduser())

    repo_root = Path(__file__).resolve().parents[4]
    candidates.append(repo_root.parent / "VoxCPM" / "src")

    unique_candidates: list[Path] = []
    seen: set[str] = set()
    for candidate in candidates:
        candidate_key = str(candidate)
        if candidate_key in seen:
            continue
        seen.add(candidate_key)
        unique_candidates.append(candidate)
    return unique_candidates


def _prepend_voxcpm_src(candidate: Path) -> None:
    candidate_str = str(candidate)
    if candidate_str not in sys.path:
        sys.path.insert(0, candidate_str)


def _import_voxcpm_attrs(module_name: str, *attr_names: str) -> tuple[Any, ...]:
    last_exc: ImportError | None = None
    for candidate in _iter_voxcpm_src_candidates():
        if not candidate.exists():
            continue
        _prepend_voxcpm_src(candidate)
        try:
            module = importlib.import_module(module_name)
            return tuple(getattr(module, attr_name) for attr_name in attr_names)
        except ImportError as exc:
            last_exc = exc

    try:
        module = importlib.import_module(module_name)
        return tuple(getattr(module, attr_name) for attr_name in attr_names)
    except ImportError as exc:
        last_exc = exc

    raise ImportError(f"Failed to import {module_name}.") from last_exc


def _import_voxcpm_base_model_class():
    """Import upstream ``VoxCPMModel`` from ``VoxCPM/src/voxcpm`` (env, sibling tree, or pip)."""
    try:
        (VoxCPMModel,) = _import_voxcpm_attrs("voxcpm.model.voxcpm", "VoxCPMModel")
        return VoxCPMModel
    except ImportError as exc:
        raise ImportError(
            "Failed to import VoxCPMModel. Install the `voxcpm` package or set "
            "`VLLM_OMNI_VOXCPM_CODE_PATH` to the VoxCPM repository `src` directory "
            "(the parent of the `voxcpm` package that contains `model/` and `modules/`)."
        ) from exc


def _import_voxcpm_audio_vae_classes():
    try:
        return _import_voxcpm_attrs("voxcpm.modules.audiovae", "AudioVAE", "AudioVAEConfig")
    except ImportError as exc:
        raise ImportError(
            "Failed to import VoxCPM AudioVAE. Install the `voxcpm` package or set "
            "`VLLM_OMNI_VOXCPM_CODE_PATH` to the VoxCPM repository `src` directory."
        ) from exc


def _device_to_string(device: torch.device) -> str:
    if device.index is None:
        return device.type
    return f"{device.type}:{device.index}"


def _normalize_dtype_name(dtype: Any) -> str | None:
    if dtype is None:
        return None
    if isinstance(dtype, torch.dtype):
        mapping = {
            torch.bfloat16: "bfloat16",
            torch.float16: "float16",
            torch.float32: "float32",
        }
        return mapping.get(dtype, str(dtype).removeprefix("torch."))
    dtype_str = str(dtype)
    return dtype_str.removeprefix("torch.")


def _resolve_runtime_device(vllm_config: VllmConfig) -> torch.device:
    try:
        from vllm_omni.platforms import current_omni_platform

        return current_omni_platform.get_torch_device()
    except Exception:
        pass

    device = getattr(getattr(vllm_config, "device_config", None), "device", None)
    if isinstance(device, torch.device):
        return device
    if device:
        return torch.device(device)
    return torch.device("cpu")


def _prepare_runtime_model_dir(
    model_path: str | Path,
    *,
    target_device: torch.device,
    target_dtype: str | None,
) -> str:
    source_dir = Path(model_path)
    config_path = source_dir / "config.json"
    if not config_path.exists():
        return str(source_dir)

    config_text = config_path.read_text()
    config_dict = json.loads(config_text)
    desired_device = target_device.type
    desired_dtype = target_dtype or config_dict.get("dtype")

    if config_dict.get("device") == desired_device and config_dict.get("dtype") == desired_dtype:
        return str(source_dir)

    digest = sha256(f"{source_dir.resolve()}:{config_text}:{desired_device}:{desired_dtype}".encode()).hexdigest()[:16]
    runtime_dir = Path(tempfile.gettempdir()) / "vllm_omni_voxcpm_runtime" / digest
    runtime_dir.mkdir(parents=True, exist_ok=True)

    for entry in source_dir.iterdir():
        target = runtime_dir / entry.name
        if entry.name == "config.json" or target.exists():
            continue
        try:
            target.symlink_to(entry, target_is_directory=entry.is_dir())
        except OSError as exc:
            logger.warning(
                "Falling back to copying VoxCPM runtime artifact %s into %s because symlink creation failed: %s",
                entry,
                runtime_dir,
                exc,
            )
            if entry.is_dir():
                shutil.copytree(entry, target, dirs_exist_ok=True)
            else:
                shutil.copy2(entry, target)

    patched_config = dict(config_dict)
    patched_config["device"] = desired_device
    if desired_dtype is not None:
        patched_config["dtype"] = desired_dtype
    (runtime_dir / "config.json").write_text(json.dumps(patched_config, indent=2, sort_keys=True))
    return str(runtime_dir)


@contextmanager
def _force_cuda_available_for_npu(device: torch.device):
    if device.type != "npu":
        yield
        return

    with patch("torch.cuda.is_available", return_value=True):
        yield


def _is_torchcodec_load_error(exc: BaseException) -> bool:
    message = str(exc).lower()
    return "torchcodec" in message or "load_with_torchcodec" in message


def _load_audio_with_soundfile(
    prompt_wav_path: str,
    *,
    sample_rate: int,
) -> torch.Tensor:
    try:
        import soundfile as sf
    except ImportError:
        raise

    audio_np, source_sr = sf.read(prompt_wav_path, dtype="float32", always_2d=True)
    audio = torch.from_numpy(np.ascontiguousarray(audio_np.T))

    if audio.size(0) > 1:
        audio = audio.mean(dim=0, keepdim=True)

    if int(source_sr) != int(sample_rate):
        try:
            import torchaudio
        except ImportError as exc:
            raise ImportError("torchaudio is required for resampling prompt audio.") from exc
        audio = torchaudio.functional.resample(audio, int(source_sr), int(sample_rate))

    return audio


def _build_prompt_cache_with_soundfile(model: Any, *args: Any, **kwargs: Any) -> dict[str, Any]:
    if args:
        prompt_text = args[0]
        prompt_wav_path = args[1] if len(args) > 1 else kwargs.get("prompt_wav_path")
    else:
        prompt_text = kwargs.get("prompt_text")
        prompt_wav_path = kwargs.get("prompt_wav_path")

    if not prompt_text or not prompt_wav_path:
        raise ValueError("prompt_text and prompt_wav_path are required")

    audio = _load_audio_with_soundfile(prompt_wav_path, sample_rate=int(model.sample_rate))

    patch_len = model.patch_size * model.chunk_size
    if audio.size(1) % patch_len != 0:
        padding_size = patch_len - audio.size(1) % patch_len
        audio = torch.nn.functional.pad(audio, (padding_size, 0))

    audio_feat = model.audio_vae.encode(audio.to(model.device), model.sample_rate).cpu()
    audio_feat = audio_feat.view(
        model.audio_vae.latent_dim,
        -1,
        model.patch_size,
    ).permute(1, 2, 0)

    return {
        "prompt_text": prompt_text,
        "audio_feat": audio_feat,
    }