[Feature] Enable LoRA adapter injection for BAGEL (#2490)

Signed-off-by: Zhengyuan Su <su.zhengyuan@u.nus.edu>
Signed-off-by: Claude <noreply@anthropic.com>
Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
Zhengyuan Su (苏政渊)andClaude authored and GitHub committed 2026-04-06 19:30:52 +08:00
1 parent 328de586aa
commit 486d77d797
6 files changed
+392 -31

No files matched your search

+50 -30
View File
@@ -7,8 +7,12 @@ import pytest
import torch
from vllm.lora.lora_weights import LoRALayerWeights
from vllm.lora.utils import get_supported_lora_modules
from vllm.model_executor.layers.linear import LinearBase
from tests.diffusion.lora.conftest import (
DummyBaseLayerWithLoRA,
FakeLinearBase,
fake_replace_submodule,
)
from vllm_omni.diffusion.lora.manager import DiffusionLoRAManager
from vllm_omni.lora.request import LoRARequest
@@ -33,35 +37,9 @@ class _DummyLoRALayer:
self.reset_calls += 1
class _FakeLinearBase(LinearBase):
def __init__(self):
torch.nn.Module.__init__(self)
class _DummyBaseLayerWithLoRA(torch.nn.Module):
def __init__(self, base_layer: torch.nn.Module):
super().__init__()
self.base_layer = base_layer
self.set_calls: list[
tuple[list[torch.Tensor | None] | torch.Tensor, list[torch.Tensor | None] | torch.Tensor]
] = []
self.reset_calls: int = 0
self.create_calls: int = 0
def set_lora(self, index: int, lora_a, lora_b):
assert index == 0
self.set_calls.append((lora_a, lora_b))
def reset_lora(self, index: int):
assert index == 0
self.reset_calls += 1
def create_lora_weights(self, max_loras, lora_config, model_config):
# Needs to be callable for scale test when rank changes, but not
# actually used since we mock everything and check everything based
# on set calls.
self.create_calls += 1
# Aliases for backward compatibility within this file
_FakeLinearBase = FakeLinearBase
_DummyBaseLayerWithLoRA = DummyBaseLayerWithLoRA
class _DummyPipeline(torch.nn.Module):
@@ -555,3 +533,45 @@ def test_lora_manager_max_rank_validation(monkeypatch, rank):
req1 = _dummy_lora_request(1)
with pytest.raises(ValueError):
manager.add_adapter(req1)
def test_lora_manager_discovers_bagel_component(monkeypatch):
"""Verify that _replace_layers_with_lora finds layers under 'bagel'."""
import vllm_omni.diffusion.lora.manager as manager_mod
monkeypatch.setattr(manager_mod, "BaseLayerWithLoRA", _DummyBaseLayerWithLoRA)
def _fake_from_layer_diffusion(*, layer: torch.nn.Module, **_kwargs):
if isinstance(layer, _FakeLinearBase):
return _DummyBaseLayerWithLoRA(layer)
return layer
replace_calls: list[str] = []
monkeypatch.setattr(manager_mod, "from_layer_diffusion", _fake_from_layer_diffusion)
monkeypatch.setattr(
manager_mod,
"replace_submodule",
lambda root, name, sub: fake_replace_submodule(root, name, sub, replace_calls),
)
# Pipeline with a 'bagel' component (no 'transformer')
pipeline = torch.nn.Module()
pipeline.bagel = torch.nn.Module()
pipeline.bagel.language_model = torch.nn.Module()
pipeline.bagel.language_model.qkv_proj = _FakeLinearBase()
manager = DiffusionLoRAManager(
pipeline=pipeline,
device=torch.device("cpu"),
dtype=torch.bfloat16,
max_cached_adapters=1,
)
peft_helper = type("_PH", (), {"r": 1})()
manager._replace_layers_with_lora(peft_helper)
assert "language_model.qkv_proj" in replace_calls
assert "bagel.language_model.qkv_proj" in manager._lora_modules
# Verify the module was actually replaced in the tree (not just recorded)
assert isinstance(pipeline.bagel.language_model.qkv_proj, _DummyBaseLayerWithLoRA)