[Model] Add VoxCPM2 startup LoRA adapter support (#7108)

Signed-off-by: bezdarnost <urumbekovaman@gmail.com>
Co-authored-by: bezdarnost <urumbekovaman@gmail.com>
This commit is contained in:
Amanandbezdarnost authored and GitHub committed 2026-09-14 15:25:42 -07:00
1 parent 1b7e854b05
commit a22a7dd68f
6 files changed
+432 -1

No files matched your search

@@ -0,0 +1,95 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Merge a native VoxCPM2 LoRA checkpoint at startup for all requests."""
from __future__ import annotations
import json
import math
from pathlib import Path
import torch
from safetensors.torch import load_file
from torch import nn
def merge_voxcpm2_lora(
adapter_path: str,
*,
base_lm: nn.Module,
residual_lm: nn.Module,
tts: nn.Module,
) -> int:
"""Validate and merge one local native safetensors adapter.
Call once after base weight loading and before fusion or graph capture.
Validate all checkpoint keys and tensors before changing weights.
"""
path = Path(adapter_path)
with (path / "lora_config.json").open(encoding="utf-8") as f:
metadata = json.load(f)
config = metadata.get("lora_config") if isinstance(metadata, dict) else None
if not isinstance(config, dict):
raise ValueError("VoxCPM2 lora_config.json must contain a lora_config object")
rank, alpha = config.get("r"), config.get("alpha")
if type(rank) is not int or rank <= 0:
raise ValueError("VoxCPM2 LoRA r must be a positive integer")
if isinstance(alpha, bool) or not isinstance(alpha, (int, float)) or not math.isfinite(alpha):
raise ValueError("VoxCPM2 LoRA alpha must be finite")
targets: dict[str, nn.Linear] = {}
groups = (
("lm", {"base_lm": base_lm, "residual_lm": residual_lm}, "target_modules_lm"),
("dit", {"feat_decoder.estimator": tts.feat_decoder.estimator}, "target_modules_dit"),
("proj", {"": tts}, "target_proj_modules"),
)
for group, roots, target_key in groups:
enabled = config.get(f"enable_{group}", False)
if type(enabled) is not bool:
raise ValueError(f"VoxCPM2 LoRA enable_{group} must be a boolean")
if not enabled:
continue
defaults = (
["enc_to_lm_proj", "lm_to_dit_proj", "res_to_dit_proj", "fusion_concat_proj"]
if group == "proj"
else ["q_proj", "v_proj", "k_proj", "o_proj"]
)
names = config.get(target_key, defaults)
if not isinstance(names, list) or not names or any(not isinstance(n, str) or not n for n in names):
raise ValueError(f"VoxCPM2 LoRA {target_key} must be a non-empty list of names")
for prefix, root in roots.items():
matched = set()
for name, module in root.named_modules():
selector = name if group == "proj" else name.rsplit(".", 1)[-1]
if selector in names and isinstance(module, nn.Linear):
targets[f"{prefix}.{name}" if prefix else name] = module
matched.add(selector)
if missing := set(names) - matched:
raise ValueError(f"VoxCPM2 LoRA targets not found in {prefix or 'tts'}: {sorted(missing)}")
if not targets:
raise ValueError("VoxCPM2 LoRA config enables no target layers")
state = load_file(str(path / "lora_weights.safetensors"), device="cpu")
expected = {f"{name}.lora_{suffix}" for name in targets for suffix in ("A", "B")}
if set(state) != expected:
raise ValueError(
"VoxCPM2 LoRA checkpoint keys do not match the configured targets: "
f"missing={sorted(expected - set(state))}, unexpected={sorted(set(state) - expected)}"
)
for name, module in targets.items():
if not module.weight.is_floating_point() or module.weight.is_meta:
raise ValueError(f"VoxCPM2 LoRA requires materialized floating-point weights: {name}")
for suffix, shape in (("A", (rank, module.in_features)), ("B", (module.out_features, rank))):
key = f"{name}.lora_{suffix}"
tensor = state[key]
if tensor.shape != shape or not tensor.is_floating_point() or not torch.isfinite(tensor).all():
raise ValueError(f"Invalid VoxCPM2 LoRA tensor {key}: expected finite floating-point shape {shape}")
with torch.no_grad():
for name, module in targets.items():
# Merge one layer at a time in fp32, preserving the base Parameter.
a = state[f"{name}.lora_A"].to(device=module.weight.device, dtype=torch.float32)
b = state[f"{name}.lora_B"].to(device=module.weight.device, dtype=torch.float32)
merged = torch.addmm(module.weight.float(), b, a, alpha=alpha / rank)
module.weight.copy_(merged)
return len(targets)
@@ -1,5 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
from __future__ import annotations
@@ -13,6 +13,7 @@ logger = init_logger(__name__)
@dataclasses.dataclass(frozen=True)
class _VoxCPM2RuntimeConfig:
startup_lora_path: str | None = None
enable_profiling: bool = False
enable_nvtx_profile: bool = False
enable_loc_dit_layer_nvtx: bool = False
@@ -105,6 +106,10 @@ class _VoxCPM2RuntimeConfig:
@staticmethod
def _coerce_value(key: str, value: Any, default: Any) -> Any:
if key == "startup_lora_path":
if value is not None and (not isinstance(value, str) or not value.strip()):
raise ValueError("VoxCPM2 startup_lora_path must be a non-empty local directory path or null")
return value
if isinstance(default, bool):
if isinstance(value, str):
return value.strip().lower() in ("1", "true", "yes", "on")
@@ -46,6 +46,7 @@ from vllm_omni.utils.speaker_cache import (
)
from vllm_omni.worker.runner_assisted_metadata import RunnerAssistedFullAttentionMetadataRequest
from .lora import merge_voxcpm2_lora
from .minicpm4_paged import MiniCPM4PagedForVoxCPM2, MiniCPM4PagedResidualLM
from .runtime_config import _VoxCPM2RuntimeConfig
from .voxcpm2_import_utils import import_voxcpm2_core
@@ -834,6 +835,9 @@ class VoxCPM2TalkerForConditionalGeneration(nn.Module):
self.vllm_config = vllm_config
self.config = vllm_config.model_config.hf_config
self._runtime_config = _VoxCPM2RuntimeConfig.from_vllm_config(vllm_config)
self._startup_lora_applied = False
if self._runtime_config.startup_lora_path and vllm_config.load_config.load_format == "dummy":
raise ValueError("VoxCPM2 startup LoRA requires real base weights; load_format=dummy is unsupported")
global _ENABLE_NVTX_PROFILE
_ENABLE_NVTX_PROFILE = self._runtime_config.enable_nvtx_profile
@@ -3284,6 +3288,9 @@ class VoxCPM2TalkerForConditionalGeneration(nn.Module):
hf_to_vllm_mapper = WeightsMapper(orig_to_new_prefix={"base_lm.": "model."})
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
if self._startup_lora_applied:
raise ValueError("Reloading VoxCPM2 weights with a startup LoRA is unsupported; restart the server")
def _base_lm_only(ws):
for name, tensor in ws:
if name.startswith("base_lm."):
@@ -3298,6 +3305,13 @@ class VoxCPM2TalkerForConditionalGeneration(nn.Module):
# them as missing from the checkpoint.
loaded |= {name for name, _ in self.named_parameters() if name.startswith(("_tts.", "residual_model."))}
if adapter_path := self._runtime_config.startup_lora_path:
merged = merge_voxcpm2_lora(
adapter_path, base_lm=self.model, residual_lm=self.residual_model, tts=self._tts
)
self._startup_lora_applied = True
logger.info("Merged VoxCPM2 startup LoRA from %s into %d linear layers", adapter_path, merged)
logger.info(
"Loaded VoxCPM2 (patch=%d, feat_dim=%d, dtype=%s)", self._patch_size, self._feat_dim, self._side_dtype
)