[Feature] Enable LoRA adapter injection for BAGEL (#2490)
Signed-off-by: Zhengyuan Su <su.zhengyuan@u.nus.edu> Signed-off-by: Claude <noreply@anthropic.com> Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
1 parent
328de586aa
commit
486d77d797
6 files changed
+392
-31
No files matched your search
@@ -7,8 +7,12 @@ import pytest
|
||||
import torch
|
||||
from vllm.lora.lora_weights import LoRALayerWeights
|
||||
from vllm.lora.utils import get_supported_lora_modules
|
||||
from vllm.model_executor.layers.linear import LinearBase
|
||||
|
||||
from tests.diffusion.lora.conftest import (
|
||||
DummyBaseLayerWithLoRA,
|
||||
FakeLinearBase,
|
||||
fake_replace_submodule,
|
||||
)
|
||||
from vllm_omni.diffusion.lora.manager import DiffusionLoRAManager
|
||||
from vllm_omni.lora.request import LoRARequest
|
||||
|
||||
@@ -33,35 +37,9 @@ class _DummyLoRALayer:
|
||||
self.reset_calls += 1
|
||||
|
||||
|
||||
class _FakeLinearBase(LinearBase):
|
||||
def __init__(self):
|
||||
torch.nn.Module.__init__(self)
|
||||
|
||||
|
||||
class _DummyBaseLayerWithLoRA(torch.nn.Module):
|
||||
def __init__(self, base_layer: torch.nn.Module):
|
||||
super().__init__()
|
||||
self.base_layer = base_layer
|
||||
|
||||
self.set_calls: list[
|
||||
tuple[list[torch.Tensor | None] | torch.Tensor, list[torch.Tensor | None] | torch.Tensor]
|
||||
] = []
|
||||
self.reset_calls: int = 0
|
||||
self.create_calls: int = 0
|
||||
|
||||
def set_lora(self, index: int, lora_a, lora_b):
|
||||
assert index == 0
|
||||
self.set_calls.append((lora_a, lora_b))
|
||||
|
||||
def reset_lora(self, index: int):
|
||||
assert index == 0
|
||||
self.reset_calls += 1
|
||||
|
||||
def create_lora_weights(self, max_loras, lora_config, model_config):
|
||||
# Needs to be callable for scale test when rank changes, but not
|
||||
# actually used since we mock everything and check everything based
|
||||
# on set calls.
|
||||
self.create_calls += 1
|
||||
# Aliases for backward compatibility within this file
|
||||
_FakeLinearBase = FakeLinearBase
|
||||
_DummyBaseLayerWithLoRA = DummyBaseLayerWithLoRA
|
||||
|
||||
|
||||
class _DummyPipeline(torch.nn.Module):
|
||||
@@ -555,3 +533,45 @@ def test_lora_manager_max_rank_validation(monkeypatch, rank):
|
||||
req1 = _dummy_lora_request(1)
|
||||
with pytest.raises(ValueError):
|
||||
manager.add_adapter(req1)
|
||||
|
||||
|
||||
def test_lora_manager_discovers_bagel_component(monkeypatch):
|
||||
"""Verify that _replace_layers_with_lora finds layers under 'bagel'."""
|
||||
import vllm_omni.diffusion.lora.manager as manager_mod
|
||||
|
||||
monkeypatch.setattr(manager_mod, "BaseLayerWithLoRA", _DummyBaseLayerWithLoRA)
|
||||
|
||||
def _fake_from_layer_diffusion(*, layer: torch.nn.Module, **_kwargs):
|
||||
if isinstance(layer, _FakeLinearBase):
|
||||
return _DummyBaseLayerWithLoRA(layer)
|
||||
return layer
|
||||
|
||||
replace_calls: list[str] = []
|
||||
|
||||
monkeypatch.setattr(manager_mod, "from_layer_diffusion", _fake_from_layer_diffusion)
|
||||
monkeypatch.setattr(
|
||||
manager_mod,
|
||||
"replace_submodule",
|
||||
lambda root, name, sub: fake_replace_submodule(root, name, sub, replace_calls),
|
||||
)
|
||||
|
||||
# Pipeline with a 'bagel' component (no 'transformer')
|
||||
pipeline = torch.nn.Module()
|
||||
pipeline.bagel = torch.nn.Module()
|
||||
pipeline.bagel.language_model = torch.nn.Module()
|
||||
pipeline.bagel.language_model.qkv_proj = _FakeLinearBase()
|
||||
|
||||
manager = DiffusionLoRAManager(
|
||||
pipeline=pipeline,
|
||||
device=torch.device("cpu"),
|
||||
dtype=torch.bfloat16,
|
||||
max_cached_adapters=1,
|
||||
)
|
||||
|
||||
peft_helper = type("_PH", (), {"r": 1})()
|
||||
manager._replace_layers_with_lora(peft_helper)
|
||||
|
||||
assert "language_model.qkv_proj" in replace_calls
|
||||
assert "bagel.language_model.qkv_proj" in manager._lora_modules
|
||||
# Verify the module was actually replaced in the tree (not just recorded)
|
||||
assert isinstance(pipeline.bagel.language_model.qkv_proj, _DummyBaseLayerWithLoRA)
|
||||
Reference in new issue
Block a user