[Feat] support for multi-block layerwise offloading, fix top-level parameters/buffers staying on CPU (#1486)

Signed-off-by: Lancer <maruixiang6688@gmail.com>
Signed-off-by: Lancer <402430575@qq.com>
Co-authored-by: Didan Deng <33117903+wtomin@users.noreply.github.com>
This commit is contained in:
LancerandDidan Deng authored and GitHub committed 2026-04-06 11:49:35 +08:00
1 parent e23b2634d1
commit 328de586aa
13 files changed
+288 -38

No files matched your search

@@ -510,6 +510,7 @@ class FluxTransformer2DModel(nn.Module):
# -- typically a transformer layer
# used for torch compile optimizations
_repeated_blocks = ["FluxTransformerBlock"]
_layerwise_offload_blocks_attrs = ["transformer_blocks", "single_transformer_blocks"]
@staticmethod
def _is_transformer_block(name: str, module) -> bool:
@@ -742,6 +742,7 @@ class Flux2Transformer2DModel(nn.Module):
"""
_repeated_blocks = ["Flux2TransformerBlock", "Flux2SingleTransformerBlock"]
_layerwise_offload_blocks_attrs = ["transformer_blocks", "single_transformer_blocks"]
@staticmethod
def _is_transformer_block(name: str, module) -> bool:
@@ -576,7 +576,7 @@ class HeliosTransformer3DModel(nn.Module):
"""
_repeated_blocks = ["HeliosTransformerBlock"]
_layerwise_offload_blocks_attr = "blocks"
_layerwise_offload_blocks_attrs = ["blocks"]
packed_modules_mapping = {
"to_qkv": ["to_q", "to_k", "to_v"],
}
@@ -539,7 +539,7 @@ class HunyuanVideo15Transformer3DModel(nn.Module):
"""
_repeated_blocks = ["HunyuanVideo15TransformerBlock"]
_layerwise_offload_blocks_attr = "transformer_blocks"
_layerwise_offload_blocks_attrs = ["transformer_blocks"]
packed_modules_mapping = {
"to_qkv": ["to_q", "to_k", "to_v"],
"add_kv_proj": ["add_q_proj", "add_k_proj", "add_v_proj"],
@@ -882,7 +882,7 @@ class QwenImageTransformer2DModel(CachedTransformer):
# -- typically a transformer layer
# used for torch compile optimizations
_repeated_blocks = ["QwenImageTransformerBlock"]
_layerwise_offload_blocks_attr = "transformer_blocks"
_layerwise_offload_blocks_attrs = ["transformer_blocks"]
packed_modules_mapping = {
"to_qkv": ["to_q", "to_k", "to_v"],
"add_kv_proj": ["add_q_proj", "add_k_proj", "add_v_proj"],
@@ -725,7 +725,7 @@ class WanTransformer3DModel(nn.Module):
"""
_repeated_blocks = ["WanTransformerBlock"]
_layerwise_offload_blocks_attr = "blocks"
_layerwise_offload_blocks_attrs = ["blocks"]
packed_modules_mapping = {
"to_qkv": ["to_q", "to_k", "to_v"],
}
@@ -579,6 +579,7 @@ class ZImageTransformer2DModel(CachedTransformer):
"""
_repeated_blocks = ["ZImageTransformerBlock"]
_layerwise_offload_blocks_attrs = ["layers"]
@staticmethod
def _is_transformer_block(name: str, module) -> bool: