[Bugfix][Diffusion] Skip AllGather DLO weight restore on worker shutdown (#8103)

Signed-off-by: specture724 <specture724@gmail.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Gao Han <hgaoaf@connect.ust.hk>
This commit is contained in:
authored and GitHub committed 2026-09-25 08:33:32 +00:00
1 parent 3bf96080f3
commit 95781c31e0
5 files changed
+49 -7

No files matched your search

@@ -1889,10 +1889,11 @@ class DistributedLayerwiseOffloadBackend(OffloadBackend):
skipped_allgather = bool(allgather_hooks) and not restore_allgather_weights
if skipped_allgather:
# Startup rollback cannot safely enter a collective that a failed
# peer may never reach. Those blocks cannot be reconstructed, so
# make accidental reuse explicit instead of accepting zero weights.
# peer may never reach, and shutdown has no later user. Those blocks
# cannot be reconstructed, so make accidental reuse explicit
# instead of accepting zero weights.
self._poisoned_reason = (
"Distributed layerwise offload startup skipped AllGather weight restoration; "
"Distributed layerwise offload skipped AllGather weight restoration; "
"recreate the backend and reload the pipeline before retrying"
)
@@ -1981,6 +1982,11 @@ class DistributedLayerwiseOffloadBackend(OffloadBackend):
def disable(self) -> None:
self._disable(restore_allgather_weights=True)
def shutdown(self) -> None:
# Restoring AllGather blocks rebuilds the full DiT in every rank's host
# memory, which can outlast the executor's shutdown grace period.
self._disable(restore_allgather_weights=False)
@staticmethod
def _allocate_shared_buffers(
hooks: list[DistributedLayerwiseOffloadHook],