[Feature] Support MiniMax-H3 Turbo LoRA with DLO (#6550)
Signed-off-by: mglyn <1203789601@qq.com> Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
This commit is contained in:
1 parent
7439ed41c2
commit
59cf3add40
7 files changed
+124
-9
No files matched your search
@@ -1,5 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -22,6 +22,19 @@ class DiffusionBaseLinearLayerWithLoRA(BaseLinearLayerWithLoRA):
|
||||
is inherited from vLLM's BaseLinearLayerWithLoRA.
|
||||
"""
|
||||
|
||||
lora_a_stacked: tuple[torch.Tensor, ...]
|
||||
lora_b_stacked: tuple[torch.Tensor, ...]
|
||||
|
||||
def _move_lora_buffers(self, device: torch.device) -> None:
|
||||
self.lora_a_stacked = tuple(tensor.to(device=device) for tensor in self.lora_a_stacked)
|
||||
self.lora_b_stacked = tuple(tensor.to(device=device) for tensor in self.lora_b_stacked)
|
||||
|
||||
def _set_diffusion_lora_buffer_device(self, device: torch.device) -> None:
|
||||
"""Keep dynamic LoRA sidecars resident on the compute device."""
|
||||
|
||||
self._diffusion_lora_buffer_device = device
|
||||
self._move_lora_buffers(device)
|
||||
|
||||
def create_lora_weights(
|
||||
self,
|
||||
max_loras: int,
|
||||
@@ -29,6 +42,9 @@ class DiffusionBaseLinearLayerWithLoRA(BaseLinearLayerWithLoRA):
|
||||
model_config=None,
|
||||
) -> None:
|
||||
super().create_lora_weights(max_loras, lora_config, model_config)
|
||||
buffer_device = getattr(self, "_diffusion_lora_buffer_device", None)
|
||||
if buffer_device is not None:
|
||||
self._move_lora_buffers(buffer_device)
|
||||
# Keep a direct reference for attribute forwarding: `base_layer` is a
|
||||
# registered submodule (stored under `_modules`), so direct access via
|
||||
# `object.__getattribute__` will not find it. We stash a ref in
|
||||
|
||||
Reference in new issue
Block a user