[Feat] support layerwise offload for Bagel (#2734)
Signed-off-by: Nick Cao <ncao@redhat.com> Signed-off-by: siyuan.lei <siyuanlei37@gmail.com> Signed-off-by: 汪志鹏 <wangzhipeng628@gmail.com> Co-authored-by: Nick Cao <ncao@redhat.com> Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: SYLAR <125541396+lishunyang12@users.noreply.github.com> Co-authored-by: 汪志鹏 <wangzhipeng628@gmail.com>
This commit is contained in:
5 files changed
+17
-3
No files matched your search
@@ -738,6 +738,8 @@ class Qwen2MoTDecoderLayer(nn.Module):
|
||||
|
||||
|
||||
class Qwen2MoTModel(Qwen2PreTrainedModel):
|
||||
_layerwise_offload_blocks_attrs = ["layers"]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config,
|
||||
|
||||
@@ -288,7 +288,7 @@ class BagelPipeline(nn.Module, SupportsModuleOffload, DiffusionPipelineProfilerM
|
||||
# device until the weight loader materializes them. Calling
|
||||
# .to(device) would fail on those meta tensors, so we skip it
|
||||
# entirely and let the weight loader handle device placement.
|
||||
if quant_config is None:
|
||||
if quant_config is None and not od_config.enable_layerwise_offload:
|
||||
self.to(self.device)
|
||||
self.setup_diffusion_pipeline_profiler(
|
||||
enable_diffusion_pipeline_profiler=self.od_config.enable_diffusion_pipeline_profiler
|
||||
|
||||
Reference in new issue
Block a user