[Diffusion]: Diffusion Ring Attention support (#273)
Signed-off-by: mxuax <mxuax@connect.ust.hk> Signed-off-by: Hongsheng Liu <liuhongsheng4@huawei.com> Co-authored-by: Jiangyun Zhu <riverclouds.zhu@qq.com> Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
This commit is contained in:
23 files changed
+1888
-186
No files matched your search
@@ -8,6 +8,8 @@ import torch
|
||||
import torch.distributed as dist
|
||||
from torch import Tensor
|
||||
|
||||
__all__ = ["all_to_all_4D", "all_to_all_5D", "SeqAllToAll4D", "SeqAllToAll5D", "RingComm"]
|
||||
|
||||
|
||||
def all_to_all_4D(
|
||||
input: torch.tensor, scatter_idx: int = 2, gather_idx: int = 1, group=None, use_sync: bool = False
|
||||
@@ -219,3 +221,54 @@ class SeqAllToAll5D(torch.autograd.Function):
|
||||
ctx.use_sync = use_sync
|
||||
|
||||
return all_to_all_5D(input, scatter_idx, gather_idx, group=group, use_sync=use_sync)
|
||||
|
||||
|
||||
class RingComm:
|
||||
"""Ring communication utility for Ring Attention P2P communication."""
|
||||
|
||||
def __init__(self, process_group: dist.ProcessGroup):
|
||||
self._process_group = process_group
|
||||
self._ops = []
|
||||
self.rank = dist.get_rank(self._process_group)
|
||||
self.world_size = dist.get_world_size(self._process_group)
|
||||
self._reqs = None
|
||||
|
||||
self.send_rank = (self.rank + 1) % self.world_size
|
||||
self.recv_rank = (self.rank - 1) % self.world_size
|
||||
|
||||
if process_group is not None:
|
||||
self.send_rank = dist.get_global_rank(self._process_group, self.send_rank)
|
||||
self.recv_rank = dist.get_global_rank(self._process_group, self.recv_rank)
|
||||
|
||||
def send_recv(self, to_send: torch.Tensor, recv_tensor: torch.Tensor | None = None) -> torch.Tensor:
|
||||
# Ensure to_send is contiguous for P2P
|
||||
if not to_send.is_contiguous():
|
||||
to_send = to_send.contiguous()
|
||||
|
||||
if recv_tensor is None:
|
||||
# Create a contiguous buffer for receiving
|
||||
res = torch.empty_like(to_send, memory_format=torch.contiguous_format)
|
||||
# print(f"send_recv: empty_like {to_send.shape}")
|
||||
else:
|
||||
res = recv_tensor
|
||||
if not res.is_contiguous():
|
||||
res = res.contiguous()
|
||||
|
||||
send_op = dist.P2POp(dist.isend, to_send, self.send_rank, group=self._process_group)
|
||||
recv_op = dist.P2POp(dist.irecv, res, self.recv_rank, group=self._process_group)
|
||||
self._ops.append(send_op)
|
||||
self._ops.append(recv_op)
|
||||
return res
|
||||
|
||||
def commit(self):
|
||||
if self._reqs is not None:
|
||||
raise RuntimeError("commit called twice")
|
||||
self._reqs = dist.batch_isend_irecv(self._ops)
|
||||
|
||||
def wait(self):
|
||||
if self._reqs is None:
|
||||
raise RuntimeError("wait called before commit")
|
||||
for req in self._reqs:
|
||||
req.wait()
|
||||
self._reqs = None
|
||||
self._ops = []
|
||||
Reference in new issue
Block a user