[WAN_S2V] Enable HSDP for wan s2v (#4458)
Signed-off-by: Chendi Xue <chendi.xue@intel.com>
This commit is contained in:
1 parent
92f7208559
commit
60ccadfbb6
5 files changed
+384
-14
No files matched your search
@@ -1288,7 +1288,7 @@ class Wan22S2VPipeline(
|
||||
)
|
||||
|
||||
# ---- Decode this clip ----
|
||||
if self.od_config.enable_cpu_offload:
|
||||
if self.od_config.enable_cpu_offload and not getattr(self.od_config.parallel_config, "use_hsdp", False):
|
||||
self.transformer.to("cpu")
|
||||
current_omni_platform.empty_cache()
|
||||
|
||||
|
||||
@@ -1499,18 +1499,27 @@ class WanS2VTransformer3DModel(nn.Module):
|
||||
dict with 'audio_emb' (and optionally 'audio_emb_global' when
|
||||
enable_adain is True).
|
||||
"""
|
||||
audio_input = torch.cat(
|
||||
[audio_input[..., 0:1].repeat(1, 1, 1, motion_frames[0]), audio_input],
|
||||
dim=-1,
|
||||
)
|
||||
audio_emb_res = self.casual_audio_encoder(audio_input)
|
||||
result = {}
|
||||
if self.enable_adain:
|
||||
audio_emb_global, audio_emb = audio_emb_res
|
||||
result["audio_emb_global"] = audio_emb_global[:, motion_frames[1] :]
|
||||
else:
|
||||
audio_emb = audio_emb_res
|
||||
result["audio_emb"] = audio_emb[:, motion_frames[1] :, :]
|
||||
# Under HSDP, this method is called outside the FSDP forward hook.
|
||||
# Unshard root-managed params so casual_audio_encoder can run.
|
||||
is_fsdp = hasattr(self, "unshard") and hasattr(self, "reshard")
|
||||
if is_fsdp:
|
||||
self.unshard()
|
||||
try:
|
||||
audio_input = torch.cat(
|
||||
[audio_input[..., 0:1].repeat(1, 1, 1, motion_frames[0]), audio_input],
|
||||
dim=-1,
|
||||
)
|
||||
audio_emb_res = self.casual_audio_encoder(audio_input)
|
||||
result = {}
|
||||
if self.enable_adain:
|
||||
audio_emb_global, audio_emb = audio_emb_res
|
||||
result["audio_emb_global"] = audio_emb_global[:, motion_frames[1] :]
|
||||
else:
|
||||
audio_emb = audio_emb_res
|
||||
result["audio_emb"] = audio_emb[:, motion_frames[1] :, :]
|
||||
finally:
|
||||
if is_fsdp:
|
||||
self.reshard()
|
||||
return result
|
||||
|
||||
def forward(
|
||||
|
||||
Reference in new issue
Block a user