[Diffusion]: Diffusion Ring Attention support (#273)

Signed-off-by: mxuax <mxuax@connect.ust.hk>
Signed-off-by: Hongsheng Liu <liuhongsheng4@huawei.com>
Co-authored-by: Jiangyun Zhu <riverclouds.zhu@qq.com>
Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
This commit is contained in:
authored and GitHub committed 2025-12-31 14:26:30 +00:00
1 parent 9c7de5373a
commit 680742bc33
23 files changed
+1888 -186

No files matched your search

+53
View File
@@ -8,6 +8,8 @@ import torch
import torch.distributed as dist
from torch import Tensor
__all__ = ["all_to_all_4D", "all_to_all_5D", "SeqAllToAll4D", "SeqAllToAll5D", "RingComm"]
def all_to_all_4D(
input: torch.tensor, scatter_idx: int = 2, gather_idx: int = 1, group=None, use_sync: bool = False
@@ -219,3 +221,54 @@ class SeqAllToAll5D(torch.autograd.Function):
ctx.use_sync = use_sync
return all_to_all_5D(input, scatter_idx, gather_idx, group=group, use_sync=use_sync)
class RingComm:
"""Ring communication utility for Ring Attention P2P communication."""
def __init__(self, process_group: dist.ProcessGroup):
self._process_group = process_group
self._ops = []
self.rank = dist.get_rank(self._process_group)
self.world_size = dist.get_world_size(self._process_group)
self._reqs = None
self.send_rank = (self.rank + 1) % self.world_size
self.recv_rank = (self.rank - 1) % self.world_size
if process_group is not None:
self.send_rank = dist.get_global_rank(self._process_group, self.send_rank)
self.recv_rank = dist.get_global_rank(self._process_group, self.recv_rank)
def send_recv(self, to_send: torch.Tensor, recv_tensor: torch.Tensor | None = None) -> torch.Tensor:
# Ensure to_send is contiguous for P2P
if not to_send.is_contiguous():
to_send = to_send.contiguous()
if recv_tensor is None:
# Create a contiguous buffer for receiving
res = torch.empty_like(to_send, memory_format=torch.contiguous_format)
# print(f"send_recv: empty_like {to_send.shape}")
else:
res = recv_tensor
if not res.is_contiguous():
res = res.contiguous()
send_op = dist.P2POp(dist.isend, to_send, self.send_rank, group=self._process_group)
recv_op = dist.P2POp(dist.irecv, res, self.recv_rank, group=self._process_group)
self._ops.append(send_op)
self._ops.append(recv_op)
return res
def commit(self):
if self._reqs is not None:
raise RuntimeError("commit called twice")
self._reqs = dist.batch_isend_irecv(self._ops)
def wait(self):
if self._reqs is None:
raise RuntimeError("wait called before commit")
for req in self._reqs:
req.wait()
self._reqs = None
self._ops = []