[Feat] support for multi-block layerwise offloading, fix top-level parameters/buffers staying on CPU (#1486)
Signed-off-by: Lancer <maruixiang6688@gmail.com> Signed-off-by: Lancer <402430575@qq.com> Co-authored-by: Didan Deng <33117903+wtomin@users.noreply.github.com>
This commit is contained in:
1 parent
e23b2634d1
commit
328de586aa
13 files changed
+288
-38
No files matched your search
@@ -510,6 +510,7 @@ class FluxTransformer2DModel(nn.Module):
|
||||
# -- typically a transformer layer
|
||||
# used for torch compile optimizations
|
||||
_repeated_blocks = ["FluxTransformerBlock"]
|
||||
_layerwise_offload_blocks_attrs = ["transformer_blocks", "single_transformer_blocks"]
|
||||
|
||||
@staticmethod
|
||||
def _is_transformer_block(name: str, module) -> bool:
|
||||
|
||||
@@ -742,6 +742,7 @@ class Flux2Transformer2DModel(nn.Module):
|
||||
"""
|
||||
|
||||
_repeated_blocks = ["Flux2TransformerBlock", "Flux2SingleTransformerBlock"]
|
||||
_layerwise_offload_blocks_attrs = ["transformer_blocks", "single_transformer_blocks"]
|
||||
|
||||
@staticmethod
|
||||
def _is_transformer_block(name: str, module) -> bool:
|
||||
|
||||
@@ -576,7 +576,7 @@ class HeliosTransformer3DModel(nn.Module):
|
||||
"""
|
||||
|
||||
_repeated_blocks = ["HeliosTransformerBlock"]
|
||||
_layerwise_offload_blocks_attr = "blocks"
|
||||
_layerwise_offload_blocks_attrs = ["blocks"]
|
||||
packed_modules_mapping = {
|
||||
"to_qkv": ["to_q", "to_k", "to_v"],
|
||||
}
|
||||
|
||||
@@ -539,7 +539,7 @@ class HunyuanVideo15Transformer3DModel(nn.Module):
|
||||
"""
|
||||
|
||||
_repeated_blocks = ["HunyuanVideo15TransformerBlock"]
|
||||
_layerwise_offload_blocks_attr = "transformer_blocks"
|
||||
_layerwise_offload_blocks_attrs = ["transformer_blocks"]
|
||||
packed_modules_mapping = {
|
||||
"to_qkv": ["to_q", "to_k", "to_v"],
|
||||
"add_kv_proj": ["add_q_proj", "add_k_proj", "add_v_proj"],
|
||||
|
||||
@@ -882,7 +882,7 @@ class QwenImageTransformer2DModel(CachedTransformer):
|
||||
# -- typically a transformer layer
|
||||
# used for torch compile optimizations
|
||||
_repeated_blocks = ["QwenImageTransformerBlock"]
|
||||
_layerwise_offload_blocks_attr = "transformer_blocks"
|
||||
_layerwise_offload_blocks_attrs = ["transformer_blocks"]
|
||||
packed_modules_mapping = {
|
||||
"to_qkv": ["to_q", "to_k", "to_v"],
|
||||
"add_kv_proj": ["add_q_proj", "add_k_proj", "add_v_proj"],
|
||||
|
||||
@@ -725,7 +725,7 @@ class WanTransformer3DModel(nn.Module):
|
||||
"""
|
||||
|
||||
_repeated_blocks = ["WanTransformerBlock"]
|
||||
_layerwise_offload_blocks_attr = "blocks"
|
||||
_layerwise_offload_blocks_attrs = ["blocks"]
|
||||
packed_modules_mapping = {
|
||||
"to_qkv": ["to_q", "to_k", "to_v"],
|
||||
}
|
||||
|
||||
@@ -579,6 +579,7 @@ class ZImageTransformer2DModel(CachedTransformer):
|
||||
"""
|
||||
|
||||
_repeated_blocks = ["ZImageTransformerBlock"]
|
||||
_layerwise_offload_blocks_attrs = ["layers"]
|
||||
|
||||
@staticmethod
|
||||
def _is_transformer_block(name: str, module) -> bool:
|
||||
|
||||
Reference in new issue
Block a user