[Model] Add VoxCPM2 startup LoRA adapter support (#7108)
Signed-off-by: bezdarnost <urumbekovaman@gmail.com> Co-authored-by: bezdarnost <urumbekovaman@gmail.com>
This commit is contained in:
1 parent
1b7e854b05
commit
a22a7dd68f
6 files changed
+432
-1
No files matched your search
@@ -0,0 +1,106 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""Serve a native VoxCPM2 adapter through the speech API."""
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import yaml
|
||||
from huggingface_hub import hf_hub_download
|
||||
from safetensors import safe_open
|
||||
from safetensors.torch import save_file
|
||||
|
||||
from tests.helpers.fixtures.runtime import omni_fixture_lock
|
||||
from tests.helpers.mark import hardware_test
|
||||
from tests.helpers.runtime import OmniServerParams, get_model_prefix, iter_omni_server
|
||||
from tests.helpers.stage_config import get_deploy_config_path
|
||||
|
||||
pytestmark = [pytest.mark.slow, pytest.mark.tts]
|
||||
MODEL = "openbmb/VoxCPM2"
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def omni_server(request, run_level, tmp_path_factory):
|
||||
directory = tmp_path_factory.mktemp("voxcpm2-lora")
|
||||
model = get_model_prefix() + MODEL
|
||||
weights = Path(model) / "model.safetensors"
|
||||
if not weights.is_file():
|
||||
weights = Path(hf_hub_download(MODEL, "model.safetensors"))
|
||||
projections = ["enc_to_lm_proj", "lm_to_dit_proj", "res_to_dit_proj", "fusion_concat_proj"]
|
||||
generator = torch.Generator().manual_seed(42)
|
||||
tensors = {}
|
||||
with safe_open(weights, framework="pt", device="cpu") as checkpoint:
|
||||
for key in checkpoint.keys():
|
||||
if not key.endswith(".weight"):
|
||||
continue
|
||||
name = key.removesuffix(".weight")
|
||||
attention = name.startswith(("base_lm.", "residual_lm.", "feat_decoder.estimator.")) and name.rsplit(
|
||||
".", 1
|
||||
)[-1] in {"q_proj", "k_proj", "v_proj", "o_proj"}
|
||||
if not attention and name not in projections:
|
||||
continue
|
||||
out_features, in_features = checkpoint.get_slice(key).get_shape()
|
||||
# Small nonzero deltas exercise every group without retraining a voice.
|
||||
tensors[f"{name}.lora_A"] = torch.randn(2, in_features, generator=generator) * 0.001
|
||||
tensors[f"{name}.lora_B"] = torch.randn(out_features, 2, generator=generator) * 0.001
|
||||
assert tensors
|
||||
save_file(tensors, str(directory / "lora_weights.safetensors"))
|
||||
(directory / "lora_config.json").write_text(
|
||||
json.dumps({"lora_config": {"r": 2, "alpha": 2, "enable_lm": True, "enable_dit": True, "enable_proj": True}})
|
||||
)
|
||||
|
||||
with open(get_deploy_config_path("voxcpm2.yaml")) as stream:
|
||||
config = yaml.safe_load(stream)
|
||||
stage = config["stages"][0]
|
||||
stage.update(
|
||||
max_num_seqs=4,
|
||||
kv_cache_memory_bytes=1024**3,
|
||||
gpu_memory_utilization=0.65,
|
||||
max_model_len=2048,
|
||||
max_num_batched_tokens=2048,
|
||||
)
|
||||
stage["default_sampling_params"]["max_tokens"] = 256
|
||||
runtime = stage["engine_extras"]["hf_overrides"]["voxcpm2_runtime_config"]
|
||||
runtime.update(startup_lora_path=str(directory), unified_decode_graph_max_batch_size=4)
|
||||
deploy_path = directory / "deploy.yaml"
|
||||
deploy_path.write_text(yaml.safe_dump(config))
|
||||
request.param = OmniServerParams(
|
||||
model=MODEL,
|
||||
stage_config_path=str(deploy_path),
|
||||
server_args=["--trust-remote-code", "--disable-log-stats"],
|
||||
)
|
||||
yield from iter_omni_server(request, run_level, omni_fixture_lock)
|
||||
|
||||
|
||||
@hardware_test(res={"cuda": "L4"}, num_cards=1)
|
||||
def test_startup_lora_concurrent_speech(omni_server, online_client):
|
||||
online_client.send_audio_speech_request(
|
||||
{
|
||||
"model": omni_server.model,
|
||||
"input": "The weather is nice today, perfect for a walk in the park.",
|
||||
"voice": "default",
|
||||
"response_format": "wav",
|
||||
"stream": False,
|
||||
"timeout": 300,
|
||||
"min_audio_bytes": 40_000,
|
||||
},
|
||||
request_num=4,
|
||||
)
|
||||
|
||||
|
||||
@hardware_test(res={"cuda": "L4"}, num_cards=1)
|
||||
def test_startup_lora_streaming_speech(omni_server, online_client):
|
||||
online_client.send_audio_speech_request(
|
||||
{
|
||||
"model": omni_server.model,
|
||||
"input": "The weather is nice today, perfect for a walk in the park.",
|
||||
"voice": "default",
|
||||
"response_format": "wav",
|
||||
"stream": True,
|
||||
"stream_format": "audio",
|
||||
"timeout": 300,
|
||||
"min_audio_bytes": 40_000,
|
||||
}
|
||||
)
|
||||
@@ -0,0 +1,177 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""Native VoxCPM2 adapter validation and merge parity, without model weights."""
|
||||
|
||||
import importlib.util
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from safetensors.torch import save_file
|
||||
from torch import nn
|
||||
from torch.nn import functional as F
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def merge_lora():
|
||||
# Keep these L1 checks independent of vLLM custom-op/platform imports.
|
||||
path = Path(__file__).resolve().parents[4] / "vllm_omni/model_executor/models/voxcpm2/lora.py"
|
||||
spec = importlib.util.spec_from_file_location("voxcpm2_startup_lora", path)
|
||||
assert spec is not None and spec.loader is not None
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
return module.merge_voxcpm2_lora
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def adapter(tmp_path):
|
||||
def attention():
|
||||
return nn.ModuleDict({name: nn.Linear(3, 4, bias=True) for name in ("q_proj", "k_proj", "v_proj", "o_proj")})
|
||||
|
||||
base_lm = nn.ModuleDict({"layers": nn.ModuleList([attention()])})
|
||||
residual_lm = nn.ModuleDict({"layers": nn.ModuleList([attention()])})
|
||||
tts = nn.ModuleDict(
|
||||
{
|
||||
"feat_decoder": nn.ModuleDict({"estimator": nn.ModuleDict({"decoder": attention()})}),
|
||||
"fusion_concat_proj": nn.Linear(3, 4),
|
||||
"stop_head": nn.Linear(3, 2),
|
||||
}
|
||||
)
|
||||
roots = {"base_lm": base_lm, "residual_lm": residual_lm, "tts": tts}
|
||||
native = nn.ModuleDict({"base_lm": base_lm, "residual_lm": residual_lm, **dict(tts.items())})
|
||||
targets = {name: module for name, module in native.named_modules() if isinstance(module, nn.Linear)}
|
||||
targets.pop("stop_head")
|
||||
generator = torch.Generator().manual_seed(42)
|
||||
tensors = {
|
||||
f"{name}.lora_{suffix}": torch.randn(shape, generator=generator)
|
||||
for name in targets
|
||||
for suffix, shape in (("A", (2, 3)), ("B", (4, 2)))
|
||||
}
|
||||
config = {
|
||||
"r": 2,
|
||||
"alpha": 3,
|
||||
"enable_lm": True,
|
||||
"enable_dit": True,
|
||||
"enable_proj": True,
|
||||
"target_proj_modules": ["fusion_concat_proj"],
|
||||
}
|
||||
|
||||
def save():
|
||||
(tmp_path / "lora_config.json").write_text(json.dumps({"lora_config": config}))
|
||||
save_file(tensors, str(tmp_path / "lora_weights.safetensors"))
|
||||
|
||||
save()
|
||||
return tmp_path, roots, targets, tensors, config, save
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dtype", [torch.float32, torch.bfloat16])
|
||||
def test_merge_matches_native_linear_output(merge_lora, adapter, dtype):
|
||||
path, roots, targets, tensors, _, _ = adapter
|
||||
inputs = torch.tensor([[0.5, -0.2, 0.1]], dtype=dtype)
|
||||
expected = {}
|
||||
identities = {}
|
||||
for name, layer in targets.items():
|
||||
layer.to(dtype=dtype)
|
||||
identities[name] = id(layer.weight)
|
||||
# Native LoRALinear: linear(x, W, bias) + linear(linear(x, A), B) * alpha/r.
|
||||
expected[name] = F.linear(inputs.float(), layer.weight.float(), layer.bias.float()) + 1.5 * F.linear(
|
||||
F.linear(inputs.float(), tensors[f"{name}.lora_A"]), tensors[f"{name}.lora_B"]
|
||||
)
|
||||
untouched = roots["tts"].stop_head.weight.detach().clone()
|
||||
assert merge_lora(str(path), **roots) == 13
|
||||
for name, layer in targets.items():
|
||||
tolerance = 0.025 if dtype == torch.bfloat16 else 1e-6
|
||||
torch.testing.assert_close(layer(inputs).float(), expected[name], atol=tolerance, rtol=tolerance)
|
||||
assert id(layer.weight) == identities[name]
|
||||
assert layer.weight.dtype == dtype
|
||||
assert set(dict(layer.named_parameters())) == {"weight", "bias"}
|
||||
torch.testing.assert_close(roots["tts"].stop_head.weight, untouched, rtol=0, atol=0)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("group", ["lm", "dit", "proj"])
|
||||
def test_independently_enabled_groups(merge_lora, adapter, group):
|
||||
path, roots, targets, tensors, config, save = adapter
|
||||
for key in ("lm", "dit", "proj"):
|
||||
config[f"enable_{key}"] = key == group
|
||||
prefixes = {"lm": ("base_lm.", "residual_lm."), "dit": ("feat_decoder.",), "proj": ("fusion_concat_proj.",)}
|
||||
for key in list(tensors):
|
||||
if not key.startswith(prefixes[group]):
|
||||
del tensors[key]
|
||||
original = {name: layer.weight.detach().clone() for name, layer in targets.items()}
|
||||
save()
|
||||
assert merge_lora(str(path), **roots) == {"lm": 8, "dit": 4, "proj": 1}[group]
|
||||
for name, layer in targets.items():
|
||||
if not (name + ".").startswith(prefixes[group]):
|
||||
torch.testing.assert_close(layer.weight, original[name], rtol=0, atol=0)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"corruption", ["missing", "extra", "shape", "rank", "alpha", "nonfinite", "dtype", "target", "flag"]
|
||||
)
|
||||
def test_rejects_invalid_checkpoint_before_mutating_weights(merge_lora, adapter, corruption):
|
||||
path, roots, targets, tensors, config, save = adapter
|
||||
key = "fusion_concat_proj.lora_B" # Validate even the last component before merging any layer.
|
||||
if corruption == "missing":
|
||||
del tensors[key]
|
||||
elif corruption == "extra":
|
||||
tensors["audio_vae.lora_A"] = torch.ones(2, 3)
|
||||
elif corruption == "shape":
|
||||
tensors[key] = torch.ones(5, 2)
|
||||
elif corruption == "rank":
|
||||
config["r"] = 0
|
||||
elif corruption == "alpha":
|
||||
config["alpha"] = float("nan")
|
||||
elif corruption == "nonfinite":
|
||||
tensors[key][0, 0] = float("nan")
|
||||
elif corruption == "dtype":
|
||||
tensors[key] = tensors[key].long()
|
||||
elif corruption == "target":
|
||||
config["target_modules_lm"] = ["does_not_exist"]
|
||||
elif corruption == "flag":
|
||||
config["enable_dit"] = "false"
|
||||
save()
|
||||
original = {name: layer.weight.detach().clone() for name, layer in targets.items()}
|
||||
with pytest.raises(ValueError, match="VoxCPM2"):
|
||||
merge_lora(str(path), **roots)
|
||||
for name, layer in targets.items():
|
||||
torch.testing.assert_close(layer.weight, original[name], rtol=0, atol=0)
|
||||
|
||||
|
||||
def test_requires_native_metadata_and_safetensors(merge_lora, adapter):
|
||||
path, roots, _, _, _, save = adapter
|
||||
(path / "lora_config.json").write_text(json.dumps({"r": 2, "alpha": 3}))
|
||||
with pytest.raises(ValueError, match="lora_config object"):
|
||||
merge_lora(str(path), **roots)
|
||||
save()
|
||||
(path / "lora_weights.safetensors").unlink()
|
||||
with pytest.raises(FileNotFoundError):
|
||||
merge_lora(str(path), **roots)
|
||||
|
||||
|
||||
def test_talker_loads_base_before_merge_and_rejects_reload(adapter):
|
||||
pytest.importorskip("vllm")
|
||||
from vllm_omni.model_executor.models.voxcpm2.runtime_config import _VoxCPM2RuntimeConfig
|
||||
from vllm_omni.model_executor.models.voxcpm2.voxcpm2_talker import VoxCPM2TalkerForConditionalGeneration
|
||||
|
||||
path, roots, targets, tensors, _, _ = adapter
|
||||
talker = VoxCPM2TalkerForConditionalGeneration.__new__(VoxCPM2TalkerForConditionalGeneration)
|
||||
nn.Module.__init__(talker)
|
||||
talker.model = roots["base_lm"]
|
||||
talker.residual_model = roots["residual_lm"]
|
||||
talker._tts = roots["tts"]
|
||||
talker._runtime_config = _VoxCPM2RuntimeConfig(startup_lora_path=str(path))
|
||||
talker._startup_lora_applied = False
|
||||
talker._patch_size, talker._feat_dim, talker._side_dtype = 4, 64, torch.float32
|
||||
weights = [(f"base_lm.{name}", torch.ones_like(param)) for name, param in talker.model.named_parameters()]
|
||||
loaded = talker.load_weights(iter(weights))
|
||||
name = "base_lm.layers.0.q_proj"
|
||||
expected = 1 + 1.5 * tensors[f"{name}.lora_B"] @ tensors[f"{name}.lora_A"]
|
||||
torch.testing.assert_close(targets[name].weight, expected)
|
||||
assert "model.layers.0.q_proj.weight" in loaded
|
||||
assert talker._startup_lora_applied
|
||||
with pytest.raises(ValueError, match="restart the server"):
|
||||
talker.load_weights(iter(weights))
|
||||
torch.testing.assert_close(targets[name].weight, expected)
|
||||
Reference in new issue
Block a user