[Feature] Support HSDP for diffusion models (#1339)
Signed-off-by: gcanlin <canlinguosdu@gmail.com> Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
This commit is contained in:
1 parent
66c3e04263
commit
f53152d340
17 files changed
+1023
-46
No files matched your search
@@ -725,6 +725,13 @@ class WanTransformer3DModel(nn.Module):
|
||||
"to_qkv": ["to_q", "to_k", "to_v"],
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _is_transformer_block(name: str, module) -> bool:
|
||||
"""Match transformer blocks for HSDP sharding (e.g., blocks.0, blocks.1)."""
|
||||
return "blocks" in name and name.split(".")[-1].isdigit()
|
||||
|
||||
_hsdp_shard_conditions = [_is_transformer_block]
|
||||
|
||||
# Sequence Parallelism for Wan (following diffusers' _cp_plan pattern)
|
||||
#
|
||||
# The _sp_plan specifies sharding/gathering at module boundaries:
|
||||
|
||||
Reference in new issue
Block a user