[Feature] Support HSDP for diffusion models (#1339)

Signed-off-by: gcanlin <canlinguosdu@gmail.com>
Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
This commit is contained in:
Canlin GuoandHongsheng Liu authored and GitHub committed 2026-02-26 21:31:10 +08:00
1 parent 66c3e04263
commit f53152d340
17 files changed
+1023 -46

No files matched your search

@@ -725,6 +725,13 @@ class WanTransformer3DModel(nn.Module):
"to_qkv": ["to_q", "to_k", "to_v"],
}
@staticmethod
def _is_transformer_block(name: str, module) -> bool:
"""Match transformer blocks for HSDP sharding (e.g., blocks.0, blocks.1)."""
return "blocks" in name and name.split(".")[-1].isdigit()
_hsdp_shard_conditions = [_is_transformer_block]
# Sequence Parallelism for Wan (following diffusers' _cp_plan pattern)
#
# The _sp_plan specifies sharding/gathering at module boundaries: