[Diffusion]: Diffusion Ulysses-Sequence-Parallelism support (#189)
Signed-off-by: Didan Deng <33117903+wtomin@users.noreply.github.com>
This commit is contained in:
1 parent
f785a5d401
commit
85f0c1177f
21 files changed
+3513
-61
No files matched your search
@@ -0,0 +1,221 @@
|
||||
# Copyright (c) Microsoft Corporation and Jiarui Fang
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# DeepSpeed Team & Jiarui Fang
|
||||
# from https://github.com/feifeibear/long-context-attention/blob/main/yunchang/comm/all_to_all.py
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
import torch.distributed as dist
|
||||
from torch import Tensor
|
||||
|
||||
|
||||
def all_to_all_4D(
|
||||
input: torch.tensor, scatter_idx: int = 2, gather_idx: int = 1, group=None, use_sync: bool = False
|
||||
) -> torch.tensor:
|
||||
"""
|
||||
all-to-all for QKV
|
||||
|
||||
Args:
|
||||
input (torch.tensor): a tensor sharded along dim scatter dim
|
||||
scatter_idx (int): default 1
|
||||
gather_idx (int): default 2
|
||||
group (torch.distributed.ProcessGroup): torch process group
|
||||
use_sync (bool): whether to synchronize after all-to-all
|
||||
|
||||
Returns:
|
||||
torch.tensor: resharded tensor (bs, seqlen/P, hc, hs)
|
||||
"""
|
||||
assert input.dim() == 4, f"input must be 4D tensor, got {input.dim()} and shape {input.shape}"
|
||||
|
||||
seq_world_size = dist.get_world_size(group)
|
||||
|
||||
if scatter_idx == 2 and gather_idx == 1:
|
||||
# input (torch.tensor): a tensor sharded along dim 1 (bs, seqlen/P, hc, hs) output: (bs, seqlen, hc/P, hs)
|
||||
bs, shard_seqlen, hc, hs = input.shape
|
||||
seqlen = shard_seqlen * seq_world_size
|
||||
shard_hc = hc // seq_world_size
|
||||
|
||||
# transpose groups of heads with the seq-len parallel dimension, so that we can scatter them!
|
||||
# (bs, seqlen/P, hc, hs) -reshape-> (bs, seq_len/P, P, hc/P, hs) -transpose(0,2)-> (P, seq_len/P, bs, hc/P, hs)
|
||||
input_t = input.reshape(bs, shard_seqlen, seq_world_size, shard_hc, hs).transpose(0, 2).contiguous()
|
||||
|
||||
output = torch.empty_like(input_t)
|
||||
# https://pytorch.org/docs/stable/distributed.html#torch.distributed.all_to_all_single
|
||||
# (P, seq_len/P, bs, hc/P, hs) scatter seqlen -all2all-> (P, seq_len/P, bs, hc/P, hs) scatter head
|
||||
|
||||
if seq_world_size > 1:
|
||||
dist.all_to_all_single(output, input_t, group=group)
|
||||
if use_sync:
|
||||
torch.cuda.synchronize()
|
||||
else:
|
||||
output = input_t
|
||||
# if scattering the seq-dim, transpose the heads back to the original dimension
|
||||
output = output.reshape(seqlen, bs, shard_hc, hs)
|
||||
|
||||
# (seq_len, bs, hc/P, hs) -reshape-> (bs, seq_len, hc/P, hs)
|
||||
output = output.transpose(0, 1).contiguous().reshape(bs, seqlen, shard_hc, hs)
|
||||
|
||||
return output
|
||||
|
||||
elif scatter_idx == 1 and gather_idx == 2:
|
||||
# input (torch.tensor): a tensor sharded along dim 1 (bs, seqlen, hc/P, hs) output: (bs, seqlen/P, hc, hs)
|
||||
bs, seqlen, shard_hc, hs = input.shape
|
||||
hc = shard_hc * seq_world_size
|
||||
shard_seqlen = seqlen // seq_world_size
|
||||
seq_world_size = dist.get_world_size(group)
|
||||
|
||||
# transpose groups of heads with the seq-len parallel dimension, so that we can scatter them!
|
||||
# (bs, seqlen, hc/P, hs) -reshape-> (bs, P, seq_len/P, hc/P, hs) -transpose(0, 3)->
|
||||
# (hc/P, P, seqlen/P, bs, hs) -transpose(0, 1) -> (P, hc/P, seqlen/P, bs, hs)
|
||||
input_t = (
|
||||
input.reshape(bs, seq_world_size, shard_seqlen, shard_hc, hs)
|
||||
.transpose(0, 3)
|
||||
.transpose(0, 1)
|
||||
.contiguous()
|
||||
.reshape(seq_world_size, shard_hc, shard_seqlen, bs, hs)
|
||||
)
|
||||
|
||||
output = torch.empty_like(input_t)
|
||||
# https://pytorch.org/docs/stable/distributed.html#torch.distributed.all_to_all_single
|
||||
# (P, bs x hc/P, seqlen/P, hs) scatter seqlen -all2all-> (P, bs x seq_len/P, hc/P, hs) scatter head
|
||||
if seq_world_size > 1:
|
||||
dist.all_to_all_single(output, input_t, group=group)
|
||||
if use_sync:
|
||||
torch.cuda.synchronize()
|
||||
else:
|
||||
output = input_t
|
||||
|
||||
# if scattering the seq-dim, transpose the heads back to the original dimension
|
||||
output = output.reshape(hc, shard_seqlen, bs, hs)
|
||||
|
||||
# (hc, seqlen/N, bs, hs) -transpose(0,2)-> (bs, seqlen/N, hc, hs)
|
||||
output = output.transpose(0, 2).contiguous().reshape(bs, shard_seqlen, hc, hs)
|
||||
|
||||
return output
|
||||
else:
|
||||
raise RuntimeError("scatter_idx must be 1 or 2 and gather_idx must be 1 or 2")
|
||||
|
||||
|
||||
class SeqAllToAll4D(torch.autograd.Function):
|
||||
@staticmethod
|
||||
def forward(
|
||||
ctx: Any,
|
||||
group: dist.ProcessGroup,
|
||||
input: Tensor,
|
||||
scatter_idx: int,
|
||||
gather_idx: int,
|
||||
use_sync: bool = False,
|
||||
) -> Tensor:
|
||||
ctx.group = group
|
||||
ctx.scatter_idx = scatter_idx
|
||||
ctx.gather_idx = gather_idx
|
||||
ctx.use_sync = use_sync
|
||||
return all_to_all_4D(input, scatter_idx, gather_idx, group=group, use_sync=use_sync)
|
||||
|
||||
|
||||
def all_to_all_5D(
|
||||
input: torch.tensor, scatter_idx: int = 3, gather_idx: int = 1, group=None, use_sync: bool = False
|
||||
) -> torch.tensor:
|
||||
"""
|
||||
all-to-all for QKV
|
||||
forward (bs, seqlen/N, 3, hc, hs) -> (bs, seqlen, 3, hc/N, hs)
|
||||
|
||||
Args:
|
||||
input (torch.tensor): a tensor sharded along dim scatter dim
|
||||
scatter_idx (int): default 1
|
||||
gather_idx (int): default 2
|
||||
group (torch.distributed.ProcessGroup): torch process group
|
||||
use_sync (bool): whether to synchronize after all-to-all
|
||||
|
||||
Returns:
|
||||
torch.tensor: resharded tensor (bs, seqlen/P, 3, hc, hs)
|
||||
"""
|
||||
assert input.dim() == 5, f"input must be 5D tensor, got {input.dim()} and shape {input.shape}"
|
||||
|
||||
seq_world_size = dist.get_world_size(group)
|
||||
|
||||
if scatter_idx == 3 and gather_idx == 1:
|
||||
# input (torch.tensor): a tensor sharded along dim 1 (bs, seqlen/P, 3, hc, hs) output: (bs, seqlen, 3, hc/P, hs)
|
||||
bs, shard_seqlen, t_cnt, hc, hs = input.shape
|
||||
|
||||
assert t_cnt == 3
|
||||
seqlen = shard_seqlen * seq_world_size
|
||||
shard_hc = hc // seq_world_size
|
||||
|
||||
# transpose groups of heads with the seq-len parallel dimension, so that we can scatter them!
|
||||
# (bs, seqlen/P, 3, hc, hs) -reshape-> (bs, seq_len/P, 3, P, hc/P, hs) -transpose(0,3)->
|
||||
# (P, seq_len/P, 3, bs, hc/P, hs)
|
||||
input_t = input.reshape(bs, shard_seqlen, 3, seq_world_size, shard_hc, hs).transpose(0, 3).contiguous()
|
||||
|
||||
output = torch.empty_like(input_t)
|
||||
# https://pytorch.org/docs/stable/distributed.html#torch.distributed.all_to_all_single
|
||||
# (P, seq_len/P, 3, bs, hc/P, hs) scatter seqlen -all2all-> (P, seq_len/P, 3, bs, hc/P, hs) scatter head
|
||||
if seq_world_size > 1:
|
||||
dist.all_to_all_single(output, input_t, group=group)
|
||||
if use_sync:
|
||||
torch.cuda.synchronize()
|
||||
else:
|
||||
output = input_t
|
||||
|
||||
# if scattering the seq-dim, transpose the heads back to the original dimension
|
||||
output = output.reshape(seqlen, 3, bs, shard_hc, hs)
|
||||
|
||||
# (seq_len, 3, bs, hc/P, hs) -trans-> (bs, seq_len, 3, hc/P, hs)
|
||||
output = output.transpose(0, 2).transpose(1, 2).contiguous()
|
||||
|
||||
return output.reshape(bs, seqlen, 3, shard_hc, hs).contiguous()
|
||||
elif scatter_idx == 1 and gather_idx == 3:
|
||||
# input (torch.tensor): a tensor sharded along dim 1 (bs, seqlen, hc/P, hs) output: (bs, seqlen/P, hc, hs)
|
||||
bs, seqlen, _, shard_hc, hs = input.shape
|
||||
hc = shard_hc * seq_world_size
|
||||
shard_seqlen = seqlen // seq_world_size
|
||||
seq_world_size = dist.get_world_size(group)
|
||||
|
||||
# transpose groups of heads with the seq-len parallel dimension, so that we can scatter them!
|
||||
# (bs, seqlen, 3, hc/P, hs) -reshape-> (bs, P, seq_len/P, 3, hc/P, hs) -transpose(0, 4)->
|
||||
# (hc/P, P, seqlen/P, 3, bs, hs) -transpose(0, 1) -> (P, hc/P, seqlen/P, 3, bs, hs)
|
||||
input_t = (
|
||||
input.reshape(bs, seq_world_size, shard_seqlen, 3, shard_hc, hs)
|
||||
.transpose(0, 4)
|
||||
.transpose(0, 1)
|
||||
.contiguous()
|
||||
.reshape(seq_world_size, shard_hc, shard_seqlen, 3, bs, hs)
|
||||
)
|
||||
|
||||
output = torch.empty_like(input_t)
|
||||
# https://pytorch.org/docs/stable/distributed.html#torch.distributed.all_to_all_single
|
||||
# (P, bs x hc/P, seqlen/P, hs) scatter seqlen -all2all-> (P, bs x seq_len/P, hc/P, hs) scatter head
|
||||
if seq_world_size > 1:
|
||||
dist.all_to_all_single(output, input_t, group=group)
|
||||
if use_sync:
|
||||
torch.cuda.synchronize()
|
||||
else:
|
||||
output = input_t
|
||||
|
||||
# if scattering the seq-dim, transpose the heads back to the original dimension
|
||||
output = output.reshape(hc, shard_seqlen, 3, bs, hs)
|
||||
|
||||
# (hc, seqlen/N, bs, hs) -transpose(0,2)-> (bs, seqlen/N, hc, hs)
|
||||
output = output.transpose(0, 3).contiguous()
|
||||
|
||||
return output.reshape(bs, shard_seqlen, 3, hc, hs).contiguous()
|
||||
else:
|
||||
raise RuntimeError("scatter_idx must be 1 or 3 and gather_idx must be 1 or 3")
|
||||
|
||||
|
||||
class SeqAllToAll5D(torch.autograd.Function):
|
||||
@staticmethod
|
||||
def forward(
|
||||
ctx: Any,
|
||||
group: dist.ProcessGroup,
|
||||
input: Tensor,
|
||||
scatter_idx: int = 3,
|
||||
gather_idx: int = 1,
|
||||
use_sync: bool = False,
|
||||
) -> Tensor:
|
||||
ctx.group = group
|
||||
ctx.scatter_idx = scatter_idx
|
||||
ctx.gather_idx = gather_idx
|
||||
ctx.use_sync = use_sync
|
||||
|
||||
return all_to_all_5D(input, scatter_idx, gather_idx, group=group, use_sync=use_sync)
|
||||
@@ -0,0 +1,942 @@
|
||||
# Copyright 2024 xDiT team.
|
||||
# Adapted from
|
||||
# https://github.com/vllm-project/vllm/blob/main/vllm/distributed/parallel_state.py
|
||||
# Copyright 2023 The vLLM team.
|
||||
# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
|
||||
import pickle
|
||||
from collections import namedtuple
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
import torch.distributed
|
||||
from torch.cuda import synchronize
|
||||
from torch.distributed import Backend, ProcessGroup
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from vllm_omni.diffusion import envs
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
if envs._is_npu():
|
||||
logger.info("torch.npu synchronize")
|
||||
from torch.npu import synchronize
|
||||
|
||||
|
||||
TensorMetadata = namedtuple("TensorMetadata", ["device", "dtype", "size"])
|
||||
|
||||
env_info = envs.PACKAGES_CHECKER.get_packages_info()
|
||||
|
||||
|
||||
def _split_tensor_dict(
|
||||
tensor_dict: dict[str, torch.Tensor | Any], prefix: str = ""
|
||||
) -> tuple[list[tuple[str, Any]], list[torch.Tensor]]:
|
||||
"""Split the tensor dictionary into two parts:
|
||||
1. A list of (key, value) pairs. If the value is a tensor, it is replaced
|
||||
by its metadata.
|
||||
2. A list of tensors.
|
||||
|
||||
If the Tensor is nested under `tensor_dict["key1"]["key2"]`, the key of its
|
||||
metadata will be "key1%key2".
|
||||
"""
|
||||
metadata_list: list[tuple[str, Any]] = []
|
||||
tensor_list = []
|
||||
for key, value in tensor_dict.items():
|
||||
assert "%" not in key, "Avoid having '%' in key as it is used as a separator for nested entries."
|
||||
if isinstance(value, torch.Tensor):
|
||||
# Note: we cannot use `value.device` here,
|
||||
# because it contains not only the device type but also the device
|
||||
# index (e.g. "cuda:0"). We only need the device type.
|
||||
# receiving side will set the device index.
|
||||
device = value.device.type
|
||||
metadata_list.append((prefix + key, TensorMetadata(device, value.dtype, value.size())))
|
||||
tensor_list.append(value)
|
||||
elif isinstance(value, dict):
|
||||
if len(value) == 0:
|
||||
metadata_list.append((prefix + key, value))
|
||||
inner_metadata_list, inner_tensor_list = _split_tensor_dict(value, prefix + key + "%")
|
||||
metadata_list.extend(inner_metadata_list)
|
||||
tensor_list.extend(inner_tensor_list)
|
||||
else:
|
||||
metadata_list.append((prefix + key, value))
|
||||
return metadata_list, tensor_list
|
||||
|
||||
|
||||
def _update_nested_dict(nested_dict, flattened_key, value):
|
||||
key_splits = flattened_key.split("%")
|
||||
cur_dict = nested_dict
|
||||
for k in key_splits[:-1]:
|
||||
if k not in cur_dict:
|
||||
cur_dict[k] = {}
|
||||
cur_dict = cur_dict[k]
|
||||
cur_dict[key_splits[-1]] = value
|
||||
|
||||
|
||||
class GroupCoordinator:
|
||||
"""
|
||||
PyTorch ProcessGroup wrapper for a group of processes.
|
||||
PyTorch ProcessGroup is bound to one specific communication backend,
|
||||
e.g. NCCL, Gloo, MPI, etc.
|
||||
GroupCoordinator takes charge of all the communication operations among
|
||||
the processes in the group. It can route the communication to
|
||||
a specific implementation (e.g. switch allreduce implementation
|
||||
based on the tensor size and cuda graph mode).
|
||||
"""
|
||||
|
||||
# available attributes:
|
||||
rank: int # global rank
|
||||
ranks: list[int] # global ranks in the group
|
||||
world_size: int # size of the group
|
||||
# difference between `local_rank` and `rank_in_group`:
|
||||
# if we have a group of size 4 across two nodes:
|
||||
# Process | Node | Rank | Local Rank | Rank in Group
|
||||
# 0 | 0 | 0 | 0 | 0
|
||||
# 1 | 0 | 1 | 1 | 1
|
||||
# 2 | 1 | 2 | 0 | 2
|
||||
# 3 | 1 | 3 | 1 | 3
|
||||
local_rank: int # local rank used to assign devices
|
||||
rank_in_group: int # rank inside the group
|
||||
cpu_group: ProcessGroup # group for CPU communication
|
||||
device_group: ProcessGroup # group for device communication
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
group_ranks: list[list[int]],
|
||||
local_rank: int,
|
||||
torch_distributed_backend: str | Backend,
|
||||
):
|
||||
self.rank = torch.distributed.get_rank()
|
||||
self.local_rank = local_rank
|
||||
self.device_group = None
|
||||
self.cpu_group = None
|
||||
|
||||
for ranks in group_ranks:
|
||||
device_group = torch.distributed.new_group(ranks, backend=torch_distributed_backend)
|
||||
# a group with `gloo` backend, to allow direct coordination between
|
||||
# processes through the CPU.
|
||||
cpu_group = torch.distributed.new_group(ranks, backend="gloo")
|
||||
if self.rank in ranks:
|
||||
self.ranks = ranks
|
||||
self.world_size = len(ranks)
|
||||
self.rank_in_group = ranks.index(self.rank)
|
||||
self.device_group = device_group
|
||||
self.cpu_group = cpu_group
|
||||
|
||||
assert self.cpu_group is not None
|
||||
assert self.device_group is not None
|
||||
|
||||
self.device = envs.get_device(local_rank)
|
||||
|
||||
@property
|
||||
def first_rank(self):
|
||||
"""Return the global rank of the first process in the group"""
|
||||
return self.ranks[0]
|
||||
|
||||
@property
|
||||
def last_rank(self):
|
||||
"""Return the global rank of the last process in the group"""
|
||||
return self.ranks[-1]
|
||||
|
||||
@property
|
||||
def is_first_rank(self):
|
||||
"""Return whether the caller is the first process in the group"""
|
||||
return self.rank == self.first_rank
|
||||
|
||||
@property
|
||||
def is_last_rank(self):
|
||||
"""Return whether the caller is the last process in the group"""
|
||||
return self.rank == self.last_rank
|
||||
|
||||
@property
|
||||
def next_rank(self):
|
||||
"""Return the global rank of the process that follows the caller"""
|
||||
rank_in_group = self.rank_in_group
|
||||
world_size = self.world_size
|
||||
return self.ranks[(rank_in_group + 1) % world_size]
|
||||
|
||||
@property
|
||||
def prev_rank(self):
|
||||
"""Return the global rank of the process that precedes the caller"""
|
||||
rank_in_group = self.rank_in_group
|
||||
world_size = self.world_size
|
||||
return self.ranks[(rank_in_group - 1) % world_size]
|
||||
|
||||
@property
|
||||
def group_next_rank(self):
|
||||
"""Return the group rank of the process that follows the caller"""
|
||||
rank_in_group = self.rank_in_group
|
||||
world_size = self.world_size
|
||||
return (rank_in_group + 1) % world_size
|
||||
|
||||
@property
|
||||
def group_prev_rank(self):
|
||||
"""Return the group rank of the process that precedes the caller"""
|
||||
rank_in_group = self.rank_in_group
|
||||
world_size = self.world_size
|
||||
return (rank_in_group - 1) % world_size
|
||||
|
||||
@property
|
||||
def skip_rank(self):
|
||||
"""Return the global rank of the process that skip connects with the caller"""
|
||||
rank_in_group = self.rank_in_group
|
||||
world_size = self.world_size
|
||||
return self.ranks[(world_size - rank_in_group - 1) % world_size]
|
||||
|
||||
@property
|
||||
def group_skip_rank(self):
|
||||
"""Return the group rank of the process that skip connects with the caller"""
|
||||
rank_in_group = self.rank_in_group
|
||||
world_size = self.world_size
|
||||
return (world_size - rank_in_group - 1) % world_size
|
||||
|
||||
def all_reduce(self, input_: torch.Tensor, op=torch._C._distributed_c10d.ReduceOp.SUM) -> torch.Tensor:
|
||||
"""
|
||||
NOTE: This operation will be applied in-place or out-of-place.
|
||||
Always assume this function modifies its input, but use the return
|
||||
value as the output.
|
||||
"""
|
||||
# Bypass the function if we are using only 1 GPU.
|
||||
if self.world_size == 1:
|
||||
return input_
|
||||
else:
|
||||
torch.distributed.all_reduce(input_, op=op, group=self.device_group)
|
||||
return input_
|
||||
|
||||
def all_gather(
|
||||
self, input_: torch.Tensor, dim: int = 0, separate_tensors: bool = False
|
||||
) -> torch.Tensor | list[torch.Tensor]:
|
||||
world_size = self.world_size
|
||||
# Bypass the function if we are using only 1 GPU.
|
||||
if world_size == 1:
|
||||
return input_
|
||||
assert -input_.dim() <= dim < input_.dim(), f"Invalid dim ({dim}) for input tensor with shape {input_.size()}"
|
||||
if dim < 0:
|
||||
# Convert negative dim to positive.
|
||||
dim += input_.dim()
|
||||
# Allocate output tensor.
|
||||
input_size = list(input_.size())
|
||||
input_size[0] *= world_size
|
||||
output_tensor = torch.empty(input_size, dtype=input_.dtype, device=input_.device)
|
||||
# All-gather.
|
||||
torch.distributed.all_gather_into_tensor(output_tensor, input_, group=self.device_group)
|
||||
if dim != 0:
|
||||
input_size[0] //= world_size
|
||||
output_tensor = output_tensor.reshape(
|
||||
[
|
||||
world_size,
|
||||
]
|
||||
+ input_size
|
||||
)
|
||||
output_tensor = output_tensor.movedim(0, dim)
|
||||
|
||||
if separate_tensors:
|
||||
tensor_list = [
|
||||
output_tensor.view(-1).narrow(0, input_.numel() * i, input_.numel()).view_as(input_)
|
||||
for i in range(world_size)
|
||||
]
|
||||
return tensor_list
|
||||
else:
|
||||
input_size = list(input_.size())
|
||||
input_size[dim] = input_size[dim] * world_size
|
||||
# Reshape
|
||||
output_tensor = output_tensor.reshape(input_size)
|
||||
return output_tensor
|
||||
|
||||
def gather(self, input_: torch.Tensor, dst: int = 0, dim: int = -1) -> torch.Tensor:
|
||||
"""
|
||||
NOTE: We assume that the input tensor is on the same device across
|
||||
all the ranks.
|
||||
NOTE: `dst` is the local rank of the destination rank.
|
||||
"""
|
||||
world_size = self.world_size
|
||||
# Bypass the function if we are using only 1 GPU.
|
||||
if world_size == 1:
|
||||
return input_
|
||||
assert -input_.dim() <= dim < input_.dim(), f"Invalid dim ({dim}) for input tensor with shape {input_.size()}"
|
||||
if dim < 0:
|
||||
# Convert negative dim to positive.
|
||||
dim += input_.dim()
|
||||
# Allocate output tensor.
|
||||
if self.rank_in_group == dst:
|
||||
gather_list = [torch.empty_like(input_) for _ in range(world_size)]
|
||||
else:
|
||||
gather_list = None
|
||||
# Gather.
|
||||
torch.distributed.gather(input_, gather_list, dst=self.ranks[dst], group=self.device_group)
|
||||
if self.rank_in_group == dst:
|
||||
output_tensor = torch.cat(gather_list, dim=dim)
|
||||
else:
|
||||
output_tensor = None
|
||||
return output_tensor
|
||||
|
||||
def broadcast(self, input_: torch.Tensor, src: int = 0):
|
||||
"""Broadcast the input tensor.
|
||||
NOTE: `src` is the local rank of the source rank.
|
||||
"""
|
||||
assert src < self.world_size, f"Invalid src rank ({src})"
|
||||
|
||||
# Bypass the function if we are using only 1 GPU.
|
||||
if self.world_size == 1:
|
||||
return input_
|
||||
# Broadcast.
|
||||
torch.distributed.broadcast(input_, src=self.ranks[src], group=self.device_group)
|
||||
return input_
|
||||
|
||||
def broadcast_object(self, obj: Any | None = None, src: int = 0):
|
||||
"""Broadcast the input object.
|
||||
NOTE: `src` is the local rank of the source rank.
|
||||
"""
|
||||
assert src < self.world_size, f"Invalid src rank ({src})"
|
||||
|
||||
# Bypass the function if we are using only 1 GPU.
|
||||
if self.world_size == 1:
|
||||
return obj
|
||||
if self.shm_broadcaster is not None:
|
||||
assert src == 0, "Shared memory broadcaster only supports src=0"
|
||||
return self.shm_broadcaster.broadcast_object(obj)
|
||||
if self.rank_in_group == src:
|
||||
torch.distributed.broadcast_object_list([obj], src=self.ranks[src], group=self.cpu_group)
|
||||
return obj
|
||||
else:
|
||||
recv = [None]
|
||||
torch.distributed.broadcast_object_list(recv, src=self.ranks[src], group=self.cpu_group)
|
||||
return recv[0]
|
||||
|
||||
def broadcast_object_list(self, obj_list: list[Any], src: int = 0, group: ProcessGroup | None = None):
|
||||
"""Broadcast the input object list.
|
||||
NOTE: `src` is the local rank of the source rank.
|
||||
"""
|
||||
assert src < self.world_size, f"Invalid src rank ({src})"
|
||||
|
||||
# Bypass the function if we are using only 1 GPU.
|
||||
if self.world_size == 1:
|
||||
return obj_list
|
||||
# Broadcast.
|
||||
torch.distributed.broadcast_object_list(obj_list, src=self.ranks[src], group=self.device_group)
|
||||
return obj_list
|
||||
|
||||
def send_object(self, obj: Any, dst: int) -> None:
|
||||
"""Send the input object list to the destination rank."""
|
||||
"""NOTE: `dst` is the local rank of the destination rank."""
|
||||
|
||||
assert dst < self.world_size, f"Invalid dst rank ({dst})"
|
||||
|
||||
assert dst != self.rank, "Invalid destination rank. Destination rank is the same as the current rank."
|
||||
|
||||
# Serialize object to tensor and get the size as well
|
||||
object_tensor = torch.frombuffer(pickle.dumps(obj), dtype=torch.uint8)
|
||||
|
||||
size_tensor = torch.tensor([object_tensor.numel()], dtype=torch.long, device="cpu")
|
||||
|
||||
# Send object size
|
||||
|
||||
torch.distributed.send(size_tensor, dst=self.ranks[dst], group=self.cpu_group)
|
||||
|
||||
# Send object
|
||||
torch.distributed.send(object_tensor, dst=self.ranks[dst], group=self.cpu_group)
|
||||
|
||||
return None
|
||||
|
||||
def recv_object(self, src: int) -> Any:
|
||||
"""Receive the input object list from the source rank."""
|
||||
"""NOTE: `src` is the local rank of the source rank."""
|
||||
|
||||
assert src < self.world_size, f"Invalid src rank ({src})"
|
||||
|
||||
assert src != self.rank, "Invalid source rank. Source rank is the same as the current rank."
|
||||
|
||||
size_tensor = torch.empty(1, dtype=torch.long, device="cpu")
|
||||
|
||||
# Receive object size
|
||||
rank_size = torch.distributed.recv(size_tensor, src=self.ranks[src], group=self.cpu_group)
|
||||
|
||||
# Tensor to receive serialized objects into.
|
||||
object_tensor = torch.empty( # type: ignore[call-overload]
|
||||
size_tensor.item(), # type: ignore[arg-type]
|
||||
dtype=torch.uint8,
|
||||
device="cpu",
|
||||
)
|
||||
|
||||
rank_object = torch.distributed.recv(object_tensor, src=self.ranks[src], group=self.cpu_group)
|
||||
|
||||
assert rank_object == rank_size, "Received object sender rank does not match the size sender rank."
|
||||
|
||||
obj = pickle.loads(object_tensor.numpy().tobytes())
|
||||
|
||||
return obj
|
||||
|
||||
def broadcast_tensor_dict(
|
||||
self,
|
||||
tensor_dict: dict[str, torch.Tensor | Any] | None = None,
|
||||
src: int = 0,
|
||||
group: ProcessGroup | None = None,
|
||||
metadata_group: ProcessGroup | None = None,
|
||||
) -> dict[str, torch.Tensor | Any] | None:
|
||||
"""Broadcast the input tensor dictionary.
|
||||
NOTE: `src` is the local rank of the source rank.
|
||||
"""
|
||||
# Bypass the function if we are using only 1 GPU.
|
||||
if not torch.distributed.is_initialized() or self.world_size == 1:
|
||||
return tensor_dict
|
||||
|
||||
group = self.device_group
|
||||
metadata_group = self.cpu_group
|
||||
assert src < self.world_size, f"Invalid src rank ({src})"
|
||||
src = self.ranks[src]
|
||||
|
||||
rank = self.rank
|
||||
if rank == src:
|
||||
metadata_list: list[tuple[Any, Any]] = []
|
||||
assert isinstance(tensor_dict, dict), f"Expecting a dictionary, got {type(tensor_dict)}"
|
||||
metadata_list, tensor_list = _split_tensor_dict(tensor_dict)
|
||||
# `metadata_list` lives in CPU memory.
|
||||
# `broadcast_object_list` has serialization & deserialization,
|
||||
# all happening on CPU. Therefore, we can use the CPU group.
|
||||
self.broadcast_object(metadata_list, src=src)
|
||||
async_handles = []
|
||||
for tensor in tensor_list:
|
||||
if tensor.numel() == 0:
|
||||
# Skip broadcasting empty tensors.
|
||||
continue
|
||||
if tensor.is_cpu:
|
||||
# use metadata_group for CPU tensors
|
||||
handle = torch.distributed.broadcast(tensor, src=src, group=metadata_group, async_op=True)
|
||||
else:
|
||||
# use group for GPU tensors
|
||||
handle = torch.distributed.broadcast(tensor, src=src, group=group, async_op=True)
|
||||
async_handles.append(handle)
|
||||
for async_handle in async_handles:
|
||||
async_handle.wait()
|
||||
|
||||
else:
|
||||
metadata_list = self.broadcast_object(None, src=src)
|
||||
tensor_dict = {}
|
||||
async_handles = []
|
||||
for key, value in metadata_list:
|
||||
if isinstance(value, TensorMetadata):
|
||||
tensor = torch.empty(value.size, dtype=value.dtype, device=value.device)
|
||||
if tensor.numel() == 0:
|
||||
# Skip broadcasting empty tensors.
|
||||
_update_nested_dict(tensor_dict, key, tensor)
|
||||
continue
|
||||
if tensor.is_cpu:
|
||||
# use metadata_group for CPU tensors
|
||||
handle = torch.distributed.broadcast(tensor, src=src, group=metadata_group, async_op=True)
|
||||
else:
|
||||
# use group for GPU tensors
|
||||
handle = torch.distributed.broadcast(tensor, src=src, group=group, async_op=True)
|
||||
async_handles.append(handle)
|
||||
_update_nested_dict(tensor_dict, key, tensor)
|
||||
else:
|
||||
_update_nested_dict(tensor_dict, key, value)
|
||||
for async_handle in async_handles:
|
||||
async_handle.wait()
|
||||
return tensor_dict
|
||||
|
||||
def send_tensor_dict(
|
||||
self,
|
||||
tensor_dict: dict[str, torch.Tensor | Any],
|
||||
dst: int | None = None,
|
||||
) -> dict[str, torch.Tensor | Any] | None:
|
||||
"""Send the input tensor dictionary.
|
||||
NOTE: `dst` is the local rank of the source rank.
|
||||
"""
|
||||
# Bypass the function if we are using only 1 GPU.
|
||||
if not torch.distributed.is_initialized() or self.world_size == 1:
|
||||
return tensor_dict
|
||||
|
||||
group = self.device_group
|
||||
metadata_group = self.cpu_group
|
||||
|
||||
if dst is None:
|
||||
dst = self.group_next_rank
|
||||
assert dst < self.world_size, f"Invalid dst rank ({dst})"
|
||||
|
||||
metadata_list: list[tuple[Any, Any]] = []
|
||||
assert isinstance(tensor_dict, dict), f"Expecting a dictionary, got {type(tensor_dict)}"
|
||||
metadata_list, tensor_list = _split_tensor_dict(tensor_dict)
|
||||
# `metadata_list` lives in CPU memory.
|
||||
# `send_object_list` has serialization & deserialization,
|
||||
# all happening on CPU. Therefore, we can use the CPU group.
|
||||
self.send_object(metadata_list, dst=dst)
|
||||
for tensor in tensor_list:
|
||||
if tensor.numel() == 0:
|
||||
# Skip sending empty tensors.
|
||||
continue
|
||||
if tensor.is_cpu:
|
||||
# use metadata_group for CPU tensors
|
||||
torch.distributed.send(tensor, dst=self.ranks[dst], group=metadata_group)
|
||||
else:
|
||||
# use group for GPU tensors
|
||||
torch.distributed.send(tensor, dst=self.ranks[dst], group=group)
|
||||
return None
|
||||
|
||||
def recv_tensor_dict(self, src: int | None = None) -> dict[str, torch.Tensor | Any] | None:
|
||||
"""Recv the input tensor dictionary.
|
||||
NOTE: `src` is the local rank of the source rank.
|
||||
"""
|
||||
# Bypass the function if we are using only 1 GPU.
|
||||
if not torch.distributed.is_initialized() or self.world_size == 1:
|
||||
return None
|
||||
|
||||
group = self.device_group
|
||||
metadata_group = self.cpu_group
|
||||
|
||||
if src is None:
|
||||
src = self.group_prev_rank
|
||||
assert src < self.world_size, f"Invalid src rank ({src})"
|
||||
|
||||
recv_metadata_list = self.recv_object(src=src)
|
||||
tensor_dict: dict[str, Any] = {}
|
||||
for key, value in recv_metadata_list:
|
||||
if isinstance(value, TensorMetadata):
|
||||
tensor = torch.empty(value.size, dtype=value.dtype, device=value.device)
|
||||
if tensor.numel() == 0:
|
||||
# Skip broadcasting empty tensors.
|
||||
_update_nested_dict(tensor_dict, key, tensor)
|
||||
continue
|
||||
if tensor.is_cpu:
|
||||
# use metadata_group for CPU tensors
|
||||
torch.distributed.recv(tensor, src=self.ranks[src], group=metadata_group)
|
||||
else:
|
||||
# use group for GPU tensors
|
||||
torch.distributed.recv(tensor, src=self.ranks[src], group=group)
|
||||
_update_nested_dict(tensor_dict, key, tensor)
|
||||
else:
|
||||
_update_nested_dict(tensor_dict, key, value)
|
||||
return tensor_dict
|
||||
|
||||
def barrier(self):
|
||||
"""Barrier synchronization among the group.
|
||||
NOTE: don't use `device_group` here! `barrier` in NCCL is
|
||||
terrible because it is internally a broadcast operation with
|
||||
secretly created GPU tensors. It is easy to mess up the current
|
||||
device. Use the CPU group instead.
|
||||
"""
|
||||
torch.distributed.barrier(group=self.cpu_group)
|
||||
|
||||
def send(self, tensor: torch.Tensor, dst: int | None = None) -> None:
|
||||
"""Sends a tensor to the destination rank in a non-blocking way"""
|
||||
"""NOTE: `dst` is the rank_in_group of the destination rank."""
|
||||
if dst is None:
|
||||
dst = self.group_next_rank
|
||||
|
||||
torch.distributed.send(
|
||||
tensor,
|
||||
self.ranks[dst],
|
||||
group=(self.device_groups[self.rank_in_group % 2] if self.world_size == 2 else self.device_group),
|
||||
)
|
||||
|
||||
def recv(self, size: torch.Size, dtype: torch.dtype, src: int | None = None) -> torch.Tensor:
|
||||
"""Receives a tensor from the src rank."""
|
||||
"""NOTE: `src` is the rank_in_group of the source rank."""
|
||||
if src is None:
|
||||
src = self.group_prev_rank
|
||||
|
||||
tensor = torch.empty(size, dtype=dtype, device=self.device)
|
||||
torch.distributed.recv(
|
||||
tensor,
|
||||
self.ranks[src],
|
||||
(self.device_groups[(self.rank_in_group + 1) % 2] if self.world_size == 2 else self.device_group),
|
||||
)
|
||||
return tensor
|
||||
|
||||
def destroy(self):
|
||||
if self.device_group is not None:
|
||||
torch.distributed.destroy_process_group(self.device_group)
|
||||
self.device_group = None
|
||||
if self.cpu_group is not None:
|
||||
torch.distributed.destroy_process_group(self.cpu_group)
|
||||
self.cpu_group = None
|
||||
|
||||
|
||||
class PipelineGroupCoordinator(GroupCoordinator):
|
||||
"""
|
||||
available attributes:
|
||||
rank: int # global rank
|
||||
ranks: list[int] # global ranks in the group
|
||||
world_size: int # size of the group
|
||||
difference between `local_rank` and `rank_in_group`:
|
||||
if we have a group of size 4 across two nodes:
|
||||
Process | Node | Rank | Local Rank | Rank in Group
|
||||
0 | 0 | 0 | 0 | 0
|
||||
1 | 0 | 1 | 1 | 1
|
||||
2 | 1 | 2 | 0 | 2
|
||||
3 | 1 | 3 | 1 | 3
|
||||
local_rank: int # local rank used to assign devices
|
||||
rank_in_group: int # rank inside the group
|
||||
cpu_group: ProcessGroup # group for CPU communication
|
||||
device_group: ProcessGroup # group for device communication
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
group_ranks: list[list[int]],
|
||||
local_rank: int,
|
||||
torch_distributed_backend: str | Backend,
|
||||
):
|
||||
self.rank = torch.distributed.get_rank()
|
||||
self.local_rank = local_rank
|
||||
self.device_group = None
|
||||
self.cpu_group = None
|
||||
self.cpu_groups = []
|
||||
self.device_groups = []
|
||||
if len(group_ranks[0]) > 2 or len(group_ranks[0]) == 1:
|
||||
for ranks in group_ranks:
|
||||
device_group = torch.distributed.new_group(ranks, backend=torch_distributed_backend)
|
||||
# a group with `gloo` backend, to allow direct coordination between
|
||||
# processes through the CPU.
|
||||
cpu_group = torch.distributed.new_group(ranks, backend="gloo")
|
||||
if self.rank in ranks:
|
||||
self.ranks = ranks
|
||||
self.world_size = len(ranks)
|
||||
self.rank_in_group = ranks.index(self.rank)
|
||||
self.device_group = device_group
|
||||
self.cpu_group = cpu_group
|
||||
# when pipeline parallelism is 2, we need to create two groups to avoid
|
||||
# communication stall.
|
||||
# *_group_0_1 represents the group for communication from device 0 to
|
||||
# device 1.
|
||||
# *_group_1_0 represents the group for communication from device 1 to
|
||||
# device 0.
|
||||
elif len(group_ranks[0]) == 2:
|
||||
for ranks in group_ranks:
|
||||
device_group_0_1 = torch.distributed.new_group(ranks, backend=torch_distributed_backend)
|
||||
device_group_1_0 = torch.distributed.new_group(ranks, backend=torch_distributed_backend)
|
||||
# a group with `gloo` backend, to allow direct coordination between
|
||||
# processes through the CPU.
|
||||
cpu_group_0_1 = torch.distributed.new_group(ranks, backend="gloo")
|
||||
cpu_group_1_0 = torch.distributed.new_group(ranks, backend="gloo")
|
||||
if self.rank in ranks:
|
||||
self.ranks = ranks
|
||||
self.world_size = len(ranks)
|
||||
self.rank_in_group = ranks.index(self.rank)
|
||||
self.device_groups = [device_group_0_1, device_group_1_0]
|
||||
self.cpu_groups = [cpu_group_0_1, cpu_group_1_0]
|
||||
self.device_group = device_group_0_1
|
||||
self.cpu_group = cpu_group_0_1
|
||||
|
||||
assert self.cpu_group is not None
|
||||
assert self.device_group is not None
|
||||
|
||||
self.device = envs.get_device(local_rank)
|
||||
|
||||
self.recv_buffer_set: bool = False
|
||||
self.recv_tasks_queue: list[tuple[str, int]] = []
|
||||
self.receiving_tasks: list[tuple[torch.distributed.Work, str, int]] = []
|
||||
self.dtype: torch.dtype | None = None
|
||||
self.num_pipefusion_patches: int | None = None
|
||||
|
||||
self.recv_shape: dict[str, dict[int, torch.Size]] = {}
|
||||
self.send_shape: dict[str, dict[int, torch.Size]] = {}
|
||||
self.recv_buffer: dict[str, dict[int, torch.Size]] = {}
|
||||
|
||||
self.skip_tensor_recv_buffer_set: bool = False
|
||||
self.recv_skip_tasks_queue: list[int | tuple[str, int]] = []
|
||||
self.receiving_skip_tasks: list[tuple[torch.distributed.Work, str, int]] = []
|
||||
self.skip_tensor_recv_buffer: list[torch.Tensor] | torch.Tensor | None = None
|
||||
self.skip_device_group = None
|
||||
for ranks in group_ranks:
|
||||
skip_device_group = torch.distributed.new_group(ranks, backend=torch_distributed_backend)
|
||||
if self.rank in ranks:
|
||||
self.skip_device_group = skip_device_group
|
||||
assert self.skip_device_group is not None
|
||||
|
||||
def reset_buffer(self):
|
||||
self.recv_tasks_queue = []
|
||||
self.receiving_tasks = []
|
||||
self.recv_shape = {}
|
||||
self.send_shape = {}
|
||||
self.recv_buffer = {}
|
||||
|
||||
self.recv_skip_tasks_queue = []
|
||||
self.receiving_skip_tasks = []
|
||||
self.skip_tensor_recv_buffer = {}
|
||||
|
||||
def set_config(self, dtype: torch.dtype):
|
||||
self.dtype = dtype
|
||||
|
||||
def set_recv_buffer(
|
||||
self,
|
||||
num_pipefusion_patches: int,
|
||||
patches_shape_list: list[list[int]],
|
||||
feature_map_shape: list[int],
|
||||
dtype: torch.dtype,
|
||||
):
|
||||
assert isinstance(dtype, torch.dtype), "dtype must be a torch.dtype object"
|
||||
assert isinstance(num_pipefusion_patches, int) and num_pipefusion_patches >= 1, (
|
||||
"num_pipefusion_patches must be greater than or equal to 1"
|
||||
)
|
||||
self.dtype = dtype
|
||||
self.num_pipefusion_patches = num_pipefusion_patches
|
||||
self.recv_buffer = [torch.zeros(*shape, dtype=self.dtype, device=self.device) for shape in patches_shape_list]
|
||||
self.recv_buffer.append(torch.zeros(*feature_map_shape, dtype=self.dtype, device=self.device))
|
||||
self.recv_buffer_set = True
|
||||
|
||||
def set_extra_tensors_recv_buffer(
|
||||
self,
|
||||
name: str,
|
||||
shape: list[int],
|
||||
num_buffers: int = 1,
|
||||
dtype: torch.dtype = torch.float16,
|
||||
):
|
||||
self.extra_tensors_recv_buffer[name] = [
|
||||
torch.zeros(*shape, dtype=dtype, device=self.device) for _ in range(num_buffers)
|
||||
]
|
||||
|
||||
def _check_shape_and_buffer(
|
||||
self,
|
||||
tensor_send_to_next=None,
|
||||
recv_prev=False,
|
||||
name: str | None = None,
|
||||
segment_idx: int = 0,
|
||||
):
|
||||
send_flag = False
|
||||
name = name or "latent"
|
||||
if tensor_send_to_next is not None:
|
||||
shape_list = self.send_shape.get(name, None)
|
||||
if shape_list is None:
|
||||
self.send_shape[name] = {segment_idx: tensor_send_to_next.shape}
|
||||
send_flag = True
|
||||
elif shape_list.get(segment_idx, None) is None:
|
||||
self.send_shape[name][segment_idx] = tensor_send_to_next.shape
|
||||
send_flag = True
|
||||
|
||||
recv_flag = False
|
||||
if recv_prev:
|
||||
shape_list = self.recv_shape.get(name, None)
|
||||
if shape_list is None:
|
||||
recv_flag = True
|
||||
elif shape_list.get(segment_idx, None) is None:
|
||||
recv_flag = True
|
||||
|
||||
recv_prev_shape = self._communicate_shapes(
|
||||
tensor_send_to_next=tensor_send_to_next if send_flag else None,
|
||||
recv_prev=recv_flag,
|
||||
)
|
||||
|
||||
if recv_flag:
|
||||
if self.recv_shape.get(name, None) is None:
|
||||
self.recv_shape[name] = {segment_idx: recv_prev_shape}
|
||||
else:
|
||||
self.recv_shape[name][segment_idx] = recv_prev_shape
|
||||
|
||||
if self.recv_buffer.get(name, None) is None:
|
||||
self.recv_buffer[name] = {
|
||||
segment_idx: torch.zeros(recv_prev_shape, device=self.device, dtype=self.dtype)
|
||||
}
|
||||
else:
|
||||
if self.recv_buffer[name].get(segment_idx, None) is not None:
|
||||
logger.warning(f"Recv buffer [name: {name}, segment_idx: {segment_idx}] already exist. updating...")
|
||||
self.recv_buffer[name][segment_idx] = torch.zeros(recv_prev_shape, device=self.device, dtype=self.dtype)
|
||||
|
||||
def _communicate_shapes(self, tensor_send_to_next=None, recv_prev=False):
|
||||
"""Communicate tensor shapes between stages. Used to communicate
|
||||
tensor shapes before the actual tensor communication happens.
|
||||
|
||||
Args:
|
||||
tensor_send_next: tensor to send to next rank (no tensor sent if
|
||||
set to None).
|
||||
recv_prev: boolean for whether tensor should be received from
|
||||
previous rank.
|
||||
"""
|
||||
|
||||
ops = []
|
||||
if recv_prev:
|
||||
recv_prev_dim_tensor = torch.empty((1), device=self.device, dtype=torch.int64)
|
||||
recv_prev_dim_op = torch.distributed.P2POp(
|
||||
torch.distributed.irecv,
|
||||
recv_prev_dim_tensor,
|
||||
self.prev_rank,
|
||||
self.device_group,
|
||||
)
|
||||
ops.append(recv_prev_dim_op)
|
||||
|
||||
if tensor_send_to_next is not None:
|
||||
send_next_dim_tensor = torch.tensor(tensor_send_to_next.dim(), device=self.device, dtype=torch.int64)
|
||||
send_next_dim_op = torch.distributed.P2POp(
|
||||
torch.distributed.isend,
|
||||
send_next_dim_tensor,
|
||||
self.next_rank,
|
||||
self.device_group,
|
||||
)
|
||||
ops.append(send_next_dim_op)
|
||||
|
||||
if len(ops) > 0:
|
||||
reqs = torch.distributed.batch_isend_irecv(ops)
|
||||
for req in reqs:
|
||||
req.wait()
|
||||
|
||||
# To protect against race condition when using batch_isend_irecv().
|
||||
# should take this out once the bug with batch_isend_irecv is resolved.
|
||||
synchronize()
|
||||
|
||||
ops = []
|
||||
recv_prev_shape_tensor = None
|
||||
if recv_prev:
|
||||
recv_prev_shape_tensor = torch.empty(
|
||||
torch.Size(recv_prev_dim_tensor), device=self.device, dtype=torch.int64
|
||||
)
|
||||
recv_prev_shape_op = torch.distributed.P2POp(
|
||||
torch.distributed.irecv,
|
||||
recv_prev_shape_tensor,
|
||||
self.prev_rank,
|
||||
self.device_group,
|
||||
)
|
||||
ops.append(recv_prev_shape_op)
|
||||
|
||||
if tensor_send_to_next is not None:
|
||||
send_next_shape_tensor = torch.tensor(tensor_send_to_next.size(), device=self.device, dtype=torch.int64)
|
||||
send_next_shape_op = torch.distributed.P2POp(
|
||||
torch.distributed.isend,
|
||||
send_next_shape_tensor,
|
||||
self.next_rank,
|
||||
self.device_group,
|
||||
)
|
||||
ops.append(send_next_shape_op)
|
||||
|
||||
if len(ops) > 0:
|
||||
reqs = torch.distributed.batch_isend_irecv(ops)
|
||||
for req in reqs:
|
||||
req.wait()
|
||||
|
||||
synchronize()
|
||||
|
||||
recv_prev_shape = [0, 0, 0]
|
||||
if recv_prev_shape_tensor is not None:
|
||||
recv_prev_shape = recv_prev_shape_tensor
|
||||
return torch.Size(recv_prev_shape)
|
||||
|
||||
def pipeline_send(self, tensor: torch.Tensor, name: str = "latent", segment_idx: int = -1) -> None:
|
||||
tensor = tensor.contiguous()
|
||||
self._check_shape_and_buffer(tensor_send_to_next=tensor, name=name, segment_idx=segment_idx)
|
||||
self._pipeline_isend(tensor).wait()
|
||||
|
||||
def pipeline_isend(self, tensor: torch.Tensor, name: str = "latent", segment_idx: int = -1) -> None:
|
||||
tensor = tensor.contiguous()
|
||||
self._check_shape_and_buffer(tensor_send_to_next=tensor, name=name, segment_idx=segment_idx)
|
||||
self._pipeline_isend(tensor)
|
||||
|
||||
def pipeline_recv(self, idx: int = -1, name: str = "latent") -> torch.Tensor:
|
||||
name = name or "latent"
|
||||
self._check_shape_and_buffer(recv_prev=True, name=name, segment_idx=idx)
|
||||
self._pipeline_irecv(self.recv_buffer[name][idx]).wait()
|
||||
return self.recv_buffer[name][idx]
|
||||
|
||||
def add_pipeline_recv_task(self, idx: int = -1, name: str = "latent"):
|
||||
name = name or "latent"
|
||||
self.recv_tasks_queue.append((name, idx))
|
||||
|
||||
def recv_next(self):
|
||||
if len(self.recv_tasks_queue) == 0:
|
||||
raise ValueError("No more tasks to receive")
|
||||
elif len(self.recv_tasks_queue) > 0:
|
||||
name, idx = self.recv_tasks_queue.pop(0)
|
||||
self._check_shape_and_buffer(recv_prev=True, name=name, segment_idx=idx)
|
||||
self.receiving_tasks.append((self._pipeline_irecv(self.recv_buffer[name][idx]), name, idx))
|
||||
|
||||
def get_pipeline_recv_data(self, idx: int = -1, name: str = "latent") -> torch.Tensor:
|
||||
assert len(self.receiving_tasks) > 0, "No tasks to receive, call add_pipeline_recv_task first"
|
||||
receiving_task = self.receiving_tasks.pop(0)
|
||||
receiving_task[0].wait()
|
||||
assert receiving_task[1] == name and receiving_task[2] == idx, "Received tensor does not match the requested"
|
||||
return self.recv_buffer[name][idx]
|
||||
|
||||
def _pipeline_irecv(self, tensor: torch.tensor):
|
||||
return torch.distributed.irecv(
|
||||
tensor,
|
||||
src=self.prev_rank,
|
||||
group=(self.device_groups[(self.rank_in_group + 1) % 2] if self.world_size == 2 else self.device_group),
|
||||
)
|
||||
|
||||
def _pipeline_isend(self, tensor: torch.tensor):
|
||||
return torch.distributed.isend(
|
||||
tensor,
|
||||
dst=self.next_rank,
|
||||
group=(self.device_groups[self.rank_in_group % 2] if self.world_size == 2 else self.device_group),
|
||||
)
|
||||
|
||||
def set_skip_tensor_recv_buffer(
|
||||
self,
|
||||
patches_shape_list: list[list[int]],
|
||||
feature_map_shape: list[int],
|
||||
):
|
||||
self.skip_tensor_recv_buffer = [
|
||||
torch.zeros(*shape, dtype=self.dtype, device=self.device) for shape in patches_shape_list
|
||||
]
|
||||
self.skip_tensor_recv_buffer.append(torch.zeros(*feature_map_shape, dtype=self.dtype, device=self.device))
|
||||
self.skip_tensor_recv_buffer_set = True
|
||||
|
||||
def pipeline_send_skip(self, tensor: torch.Tensor) -> None:
|
||||
tensor = tensor.contiguous()
|
||||
self._pipeline_isend_skip(tensor).wait()
|
||||
|
||||
def pipeline_isend_skip(self, tensor: torch.Tensor) -> None:
|
||||
tensor = tensor.contiguous()
|
||||
self._pipeline_isend_skip(tensor)
|
||||
|
||||
def pipeline_recv_skip(self, idx: int = -1) -> torch.Tensor:
|
||||
self._pipeline_irecv_skip(self.skip_tensor_recv_buffer[idx]).wait()
|
||||
return self.skip_tensor_recv_buffer[idx]
|
||||
|
||||
def add_pipeline_recv_skip_task(self, idx: int = -1):
|
||||
self.recv_skip_tasks_queue.append(idx)
|
||||
|
||||
def get_pipeline_recv_skip_data(self, idx: int = -1) -> torch.Tensor:
|
||||
assert len(self.receiving_skip_tasks) > 0, "No tasks to receive, call add_pipeline_recv_skip_task first"
|
||||
receiving_skip_task = self.receiving_skip_tasks.pop(0)
|
||||
receiving_skip_task[0].wait()
|
||||
assert receiving_skip_task[2] == idx, "Received tensor does not match the requested"
|
||||
return self.skip_tensor_recv_buffer[idx]
|
||||
|
||||
def recv_skip_next(self):
|
||||
if len(self.recv_skip_tasks_queue) == 0:
|
||||
raise ValueError("No more tasks to receive")
|
||||
elif len(self.recv_skip_tasks_queue) > 0:
|
||||
task = self.recv_skip_tasks_queue.pop(0)
|
||||
idx = task
|
||||
self.receiving_skip_tasks.append(
|
||||
(
|
||||
self._pipeline_irecv_skip(self.skip_tensor_recv_buffer[idx]),
|
||||
None,
|
||||
idx,
|
||||
)
|
||||
)
|
||||
|
||||
def _pipeline_irecv_skip(self, tensor: torch.tensor):
|
||||
return torch.distributed.irecv(tensor, src=self.skip_rank, group=self.skip_device_group)
|
||||
|
||||
def _pipeline_isend_skip(self, tensor: torch.tensor):
|
||||
return torch.distributed.isend(tensor, dst=self.skip_rank, group=self.skip_device_group)
|
||||
|
||||
|
||||
class SequenceParallelGroupCoordinator(GroupCoordinator):
|
||||
def __init__(
|
||||
self,
|
||||
group_ranks: list[list[int]],
|
||||
local_rank: int,
|
||||
torch_distributed_backend: str | Backend,
|
||||
**kwargs,
|
||||
):
|
||||
super().__init__(
|
||||
group_ranks=group_ranks,
|
||||
local_rank=local_rank,
|
||||
torch_distributed_backend=torch_distributed_backend,
|
||||
)
|
||||
|
||||
ulysses_group = kwargs.get("ulysses_group", None)
|
||||
ring_group = kwargs.get("ring_group", None)
|
||||
if ulysses_group is None:
|
||||
raise RuntimeError(
|
||||
"Please pass argument 'ulysses_group' when calling init func of SequenceParallelGroupCoordinator"
|
||||
)
|
||||
if ring_group is None:
|
||||
raise RuntimeError(
|
||||
"Please pass argument 'ring_group' when calling init func of SequenceParallelGroupCoordinator"
|
||||
)
|
||||
self.ulysses_group = ulysses_group
|
||||
self.ring_group = ring_group
|
||||
|
||||
self.ulysses_world_size = torch.distributed.get_world_size(self.ulysses_group)
|
||||
self.ulysses_rank = torch.distributed.get_rank(self.ulysses_group)
|
||||
self.ring_world_size = torch.distributed.get_world_size(self.ring_group)
|
||||
self.ring_rank = torch.distributed.get_rank(self.ring_group)
|
||||
@@ -0,0 +1,760 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# Adapted from: https://github.com/vllm-project/vllm/blob/v0.7.3/vllm/distributed/parallel_state.py
|
||||
# Copyright 2023 The vLLM team.
|
||||
# Adapted from
|
||||
# https://github.com/NVIDIA/Megatron-LM/blob/main/megatron/core/parallel_state.py
|
||||
# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
|
||||
# Copyright 2024 xDiT team.
|
||||
# Adapted from
|
||||
# https://github.com/xdit-project/xDiT/blob/main/xfuser/core/distributed/utils.py
|
||||
# https://github.com/vllm-project/vllm/blob/main/vllm/distributed/parallel_state.py
|
||||
# Copyright 2023 The vLLM team.
|
||||
# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
|
||||
"""vLLM-Omni distributed state.
|
||||
|
||||
It takes over the control of the distributed environment from PyTorch.
|
||||
The typical workflow is:
|
||||
|
||||
- call `init_distributed_environment` to initialize the distributed environment.
|
||||
- call `initialize_model_parallel` or `ensure_model_parallel_initialized` to
|
||||
initialize the model parallel groups.
|
||||
|
||||
- any code dealing with the distributed stuff
|
||||
|
||||
- call `destroy_model_parallel` to destroy the model parallel groups.
|
||||
- call `destroy_distributed_environment` to destroy the distributed environment.
|
||||
|
||||
If you only need to use the distributed environment without model parallelism,
|
||||
you can skip the model parallel initialization and destruction steps.
|
||||
"""
|
||||
|
||||
import torch
|
||||
import torch.distributed
|
||||
import vllm.distributed.parallel_state as vllm_parallel_state
|
||||
from vllm.distributed.parallel_state import get_tensor_model_parallel_world_size
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from vllm_omni.diffusion import envs
|
||||
|
||||
from .group_coordinator import (
|
||||
GroupCoordinator,
|
||||
PipelineGroupCoordinator,
|
||||
SequenceParallelGroupCoordinator,
|
||||
)
|
||||
|
||||
if envs._is_npu():
|
||||
from torch.npu import device_count, set_device
|
||||
elif envs._is_musa():
|
||||
from torch_musa.core.device import device_count, set_device
|
||||
else:
|
||||
from torch.cuda import device_count, set_device
|
||||
|
||||
|
||||
env_info = envs.PACKAGES_CHECKER.get_packages_info()
|
||||
|
||||
HAS_FLASH_ATTN = env_info["has_flash_attn"]
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
_WORLD: GroupCoordinator | None = None
|
||||
# get _TP from vllm.distributed.parallel_state
|
||||
_SP: SequenceParallelGroupCoordinator | None = None
|
||||
_PP: PipelineGroupCoordinator | None = None
|
||||
_CFG: GroupCoordinator | None = None
|
||||
_DP: GroupCoordinator | None = None
|
||||
_DIT: GroupCoordinator | None = None
|
||||
_VAE: GroupCoordinator | None = None
|
||||
|
||||
|
||||
def generate_masked_orthogonal_rank_groups(
|
||||
world_size: int, parallel_size: list[int], mask: list[bool]
|
||||
) -> list[list[int]]:
|
||||
r"""Generate orthogonal parallel groups based on the parallel size and mask.
|
||||
|
||||
Arguments:
|
||||
world_size (int): world size
|
||||
|
||||
parallel_size (list[int]):
|
||||
The parallel size of each orthogonal parallel type. For example, if
|
||||
tensor_parallel_size = 2, pipeline_model_parallel_group = 3, data_parallel_size = 4,
|
||||
and the parallel mapping order is tp-pp-dp, then the parallel_size = [2, 3, 4].
|
||||
|
||||
mask (list[bool]):
|
||||
The mask controls which parallel methods the generated groups represent. If mask[i] is
|
||||
True, it means the generated group contains the i-th parallelism method. For example,
|
||||
if parallel_size = [tp_size, pp_size, dp_size], and mask = [True, False , True], then
|
||||
the generated group is the `tp-dp` group, if the mask = [False, True, False], then the
|
||||
generated group is the `pp` group.
|
||||
|
||||
Algorithm:
|
||||
For orthogonal parallelism, such as tp/dp/pp/cp, the global_rank and
|
||||
local_rank satisfy the following equation:
|
||||
global_rank = tp_rank + dp_rank * tp_size + pp_rank * tp_size * dp_size (1)
|
||||
tp_rank \in [0, tp_size)
|
||||
dp_rank \in [0, dp_size)
|
||||
pp_rank \in [0, pp_size)
|
||||
|
||||
If we want to get the `dp_group` (tp_size * pp_size groups of dp_size ranks each.
|
||||
For example, if the gpu size is 8 and order is 'tp-pp-dp', size is '2-2-2', and the
|
||||
dp_group here is [[0, 4], [1, 5], [2, 6], [3, 7]].)
|
||||
The tp_rank and pp_rank will be combined to form the `dp_group_index`.
|
||||
dp_group_index = tp_rank + pp_rank * tp_size (2)
|
||||
|
||||
So, Given that tp_rank and pp_rank satisfy equation (2), and dp_rank in
|
||||
range(0, dp_size), the ranks in dp_group[dp_group_index] satisfies the
|
||||
equation (1).
|
||||
|
||||
This function solve this math problem.
|
||||
|
||||
For example, if the parallel_size = [tp_size, dp_size, pp_size] = [2, 3, 4],
|
||||
and the mask = [False, True, False]. Then,
|
||||
dp_group_index(0) = tp_rank(0) + pp_rank(0) * 2
|
||||
dp_group_index(1) = tp_rank(1) + pp_rank(0) * 2
|
||||
...
|
||||
dp_group_index(7) = tp_rank(1) + pp_rank(3) * 2
|
||||
|
||||
dp_group[0] = 0 + range(0, 3) * 2 + 0 = [0, 2, 4]
|
||||
dp_group[1] = 1 + range(0, 3) * 2 + 0 = [1, 3, 5]
|
||||
...
|
||||
dp_group[7] = 1 + range(0, 3) * 2 + 3 * 2 * 3 = [19, 21, 23]
|
||||
"""
|
||||
|
||||
def prefix_product(a: list[int], init=1) -> list[int]:
|
||||
r = [init]
|
||||
for v in a:
|
||||
init = init * v
|
||||
r.append(init)
|
||||
return r
|
||||
|
||||
def inner_product(a: list[int], b: list[int]) -> int:
|
||||
return sum([x * y for x, y in zip(a, b)])
|
||||
|
||||
def decompose(index, shape, stride=None):
|
||||
"""
|
||||
This function solve the math problem below:
|
||||
There is an equation:
|
||||
index = sum(idx[i] * stride[i])
|
||||
And given the value of index, stride.
|
||||
Return the idx.
|
||||
This function will used to get the pp/dp/pp_rank
|
||||
from group_index and rank_in_group.
|
||||
"""
|
||||
if stride is None:
|
||||
stride = prefix_product(shape)
|
||||
idx = [(index // d) % s for s, d in zip(shape, stride)]
|
||||
# stride is a prefix_product result. And the value of stride[-1]
|
||||
# is not used.
|
||||
assert sum([x * y for x, y in zip(idx, stride[:-1])]) == index, (
|
||||
f"idx {index} with shape {shape} mismatch the return idx {idx}"
|
||||
)
|
||||
return idx
|
||||
|
||||
masked_shape = [s for s, m in zip(parallel_size, mask) if m]
|
||||
unmasked_shape = [s for s, m in zip(parallel_size, mask) if not m]
|
||||
|
||||
global_stride = prefix_product(parallel_size)
|
||||
masked_stride = [d for d, m in zip(global_stride, mask) if m]
|
||||
unmasked_stride = [d for d, m in zip(global_stride, mask) if not m]
|
||||
|
||||
group_size = prefix_product(masked_shape)[-1]
|
||||
num_of_group = world_size // group_size
|
||||
|
||||
ranks = []
|
||||
for group_index in range(num_of_group):
|
||||
# get indices from unmaksed for group_index.
|
||||
decomposed_group_idx = decompose(group_index, unmasked_shape)
|
||||
rank = []
|
||||
for rank_in_group in range(group_size):
|
||||
# get indices from masked for rank_in_group.
|
||||
decomposed_rank_idx = decompose(rank_in_group, masked_shape)
|
||||
rank.append(
|
||||
inner_product(decomposed_rank_idx, masked_stride) + inner_product(decomposed_group_idx, unmasked_stride)
|
||||
)
|
||||
ranks.append(rank)
|
||||
return ranks
|
||||
|
||||
|
||||
class RankGenerator:
|
||||
def __init__(
|
||||
self,
|
||||
tp: int,
|
||||
sp: int,
|
||||
pp: int,
|
||||
cfg: int,
|
||||
dp: int,
|
||||
order: str,
|
||||
rank_offset: int = 0,
|
||||
) -> None:
|
||||
self.tp = tp
|
||||
self.sp = sp
|
||||
self.pp = pp
|
||||
self.cfg = cfg
|
||||
self.dp = dp
|
||||
self.rank_offset = rank_offset
|
||||
self.world_size = tp * sp * pp * cfg * dp
|
||||
|
||||
self.name_to_size = {
|
||||
"tp": self.tp,
|
||||
"sp": self.sp,
|
||||
"pp": self.pp,
|
||||
"cfg": self.cfg,
|
||||
"dp": self.dp,
|
||||
}
|
||||
order = order.lower()
|
||||
|
||||
for name in self.name_to_size.keys():
|
||||
if name not in order and self.name_to_size[name] != 1:
|
||||
raise RuntimeError(
|
||||
f"The size of ({name}) is ({self.name_to_size[name]}), "
|
||||
f"but you haven't specified the order ({self.order})."
|
||||
)
|
||||
elif name not in order:
|
||||
order = order + "-" + name
|
||||
|
||||
self.order = order
|
||||
self.ordered_size = []
|
||||
|
||||
for token in order.split("-"):
|
||||
self.ordered_size.append(self.name_to_size[token])
|
||||
|
||||
def get_mask(self, order: str, token: str):
|
||||
ordered_token = order.split("-")
|
||||
token = token.split("-")
|
||||
mask = [False] * len(ordered_token)
|
||||
for t in token:
|
||||
mask[ordered_token.index(t)] = True
|
||||
return mask
|
||||
|
||||
def get_ranks(self, token):
|
||||
"""Get rank group by input token.
|
||||
|
||||
Arguments:
|
||||
token (str):
|
||||
Specify the ranks type that want to get. If we want
|
||||
to obtain multiple parallel types, we can use a hyphen
|
||||
'-' to separate them. For example, if we want to obtain
|
||||
the TP_DP group, the token should be 'tp-dp'.
|
||||
"""
|
||||
mask = self.get_mask(self.order, token)
|
||||
ranks = generate_masked_orthogonal_rank_groups(self.world_size, self.ordered_size, mask)
|
||||
if self.rank_offset > 0:
|
||||
for rank_group in ranks:
|
||||
for i in range(len(rank_group)):
|
||||
rank_group[i] += self.rank_offset
|
||||
return ranks
|
||||
|
||||
|
||||
# * QUERY
|
||||
def get_world_group() -> GroupCoordinator:
|
||||
assert _WORLD is not None, "world group is not initialized"
|
||||
return _WORLD
|
||||
|
||||
|
||||
# SP
|
||||
def get_sp_group() -> SequenceParallelGroupCoordinator:
|
||||
assert _SP is not None, "pipeline model parallel group is not initialized"
|
||||
return _SP
|
||||
|
||||
|
||||
def get_sequence_parallel_world_size():
|
||||
"""Return world size for the sequence parallel group."""
|
||||
return get_sp_group().world_size
|
||||
|
||||
|
||||
def get_sequence_parallel_rank():
|
||||
"""Return my rank for the sequence parallel group."""
|
||||
return get_sp_group().rank_in_group
|
||||
|
||||
|
||||
def get_ulysses_parallel_world_size():
|
||||
return get_sp_group().ulysses_world_size
|
||||
|
||||
|
||||
def get_ulysses_parallel_rank():
|
||||
return get_sp_group().ulysses_rank
|
||||
|
||||
|
||||
def get_ring_parallel_world_size():
|
||||
return get_sp_group().ring_world_size
|
||||
|
||||
|
||||
def get_ring_parallel_rank():
|
||||
return get_sp_group().ring_rank
|
||||
|
||||
|
||||
# PP
|
||||
def get_pp_group() -> PipelineGroupCoordinator:
|
||||
assert _PP is not None, "pipeline model parallel group is not initialized"
|
||||
return _PP
|
||||
|
||||
|
||||
def get_pipeline_parallel_world_size():
|
||||
"""Return world size for the pipeline model parallel group."""
|
||||
return get_pp_group().world_size
|
||||
|
||||
|
||||
def get_pipeline_parallel_rank():
|
||||
"""Return my rank for the pipeline model parallel group."""
|
||||
return get_pp_group().rank_in_group
|
||||
|
||||
|
||||
def is_pipeline_first_stage():
|
||||
"""Return True if in the first pipeline model parallel stage, False otherwise."""
|
||||
return get_pipeline_parallel_rank() == 0
|
||||
|
||||
|
||||
def is_pipeline_last_stage():
|
||||
"""Return True if in the last pipeline model parallel stage, False otherwise."""
|
||||
return get_pipeline_parallel_rank() == (get_pipeline_parallel_world_size() - 1)
|
||||
|
||||
|
||||
# CFG
|
||||
def get_cfg_group() -> GroupCoordinator:
|
||||
assert _CFG is not None, "classifier_free_guidance parallel group is not initialized"
|
||||
return _CFG
|
||||
|
||||
|
||||
def get_classifier_free_guidance_world_size():
|
||||
"""Return world size for the classifier_free_guidance parallel group."""
|
||||
return get_cfg_group().world_size
|
||||
|
||||
|
||||
def get_classifier_free_guidance_rank():
|
||||
"""Return my rank for the classifier_free_guidance parallel group."""
|
||||
return get_cfg_group().rank_in_group
|
||||
|
||||
|
||||
# DP
|
||||
def get_dp_group() -> GroupCoordinator:
|
||||
assert _DP is not None, "pipeline model parallel group is not initialized"
|
||||
return _DP
|
||||
|
||||
|
||||
def get_data_parallel_world_size():
|
||||
"""Return world size for the data parallel group."""
|
||||
return get_dp_group().world_size
|
||||
|
||||
|
||||
def get_data_parallel_rank():
|
||||
"""Return my rank for the data parallel group."""
|
||||
return get_dp_group().rank_in_group
|
||||
|
||||
|
||||
def is_dp_last_group():
|
||||
"""Return True if in the last data parallel group, False otherwise."""
|
||||
return (
|
||||
get_sequence_parallel_rank() == (get_sequence_parallel_world_size() - 1)
|
||||
and get_classifier_free_guidance_rank() == (get_classifier_free_guidance_world_size() - 1)
|
||||
and get_pipeline_parallel_rank() == (get_pipeline_parallel_world_size() - 1)
|
||||
)
|
||||
|
||||
|
||||
def get_dit_world_size():
|
||||
"""Return world size for the DiT model (excluding VAE)."""
|
||||
return (
|
||||
get_data_parallel_world_size()
|
||||
* get_classifier_free_guidance_world_size()
|
||||
* get_sequence_parallel_world_size()
|
||||
* get_pipeline_parallel_world_size()
|
||||
* get_tensor_model_parallel_world_size()
|
||||
)
|
||||
|
||||
|
||||
# Add VAE getter functions
|
||||
def get_vae_parallel_group() -> GroupCoordinator:
|
||||
assert _VAE is not None, "VAE parallel group is not initialized"
|
||||
return _VAE
|
||||
|
||||
|
||||
def get_vae_parallel_world_size():
|
||||
"""Return world size for the VAE parallel group."""
|
||||
return get_vae_parallel_group().world_size
|
||||
|
||||
|
||||
def get_vae_parallel_rank():
|
||||
"""Return my rank for the VAE parallel group."""
|
||||
return get_vae_parallel_group().rank_in_group
|
||||
|
||||
|
||||
# * SET
|
||||
|
||||
|
||||
def init_world_group(ranks: list[int], local_rank: int, backend: str) -> GroupCoordinator:
|
||||
return GroupCoordinator(
|
||||
group_ranks=[ranks],
|
||||
local_rank=local_rank,
|
||||
torch_distributed_backend=backend,
|
||||
)
|
||||
|
||||
|
||||
def init_distributed_environment(
|
||||
world_size: int = -1,
|
||||
rank: int = -1,
|
||||
distributed_init_method: str = "env://",
|
||||
local_rank: int = -1,
|
||||
backend: str | None = None,
|
||||
):
|
||||
if backend is None:
|
||||
backend = envs.get_torch_distributed_backend()
|
||||
logger.debug(
|
||||
"world_size=%d rank=%d local_rank=%d distributed_init_method=%s backend=%s",
|
||||
world_size,
|
||||
rank,
|
||||
local_rank,
|
||||
distributed_init_method,
|
||||
backend,
|
||||
)
|
||||
if not torch.distributed.is_initialized():
|
||||
assert distributed_init_method is not None, (
|
||||
"distributed_init_method must be provided when initializing distributed environment"
|
||||
)
|
||||
# this backend is used for WORLD
|
||||
torch.distributed.init_process_group(
|
||||
backend=backend,
|
||||
init_method=distributed_init_method,
|
||||
world_size=world_size,
|
||||
rank=rank,
|
||||
)
|
||||
set_device(torch.distributed.get_rank() % device_count())
|
||||
# set the local rank
|
||||
# local_rank is not available in torch ProcessGroup,
|
||||
# see https://github.com/pytorch/pytorch/issues/122816
|
||||
if local_rank == -1:
|
||||
# local rank not set, this usually happens in single-node
|
||||
# setting, where we can use rank as local rank
|
||||
if distributed_init_method == "env://":
|
||||
local_rank = envs.LOCAL_RANK
|
||||
else:
|
||||
local_rank = rank
|
||||
global _WORLD
|
||||
if _WORLD is None:
|
||||
ranks = list(range(torch.distributed.get_world_size()))
|
||||
_WORLD = init_world_group(ranks, local_rank, backend)
|
||||
else:
|
||||
assert _WORLD.world_size == torch.distributed.get_world_size(), (
|
||||
"world group already initialized with a different world size"
|
||||
)
|
||||
|
||||
|
||||
def model_parallel_is_initialized():
|
||||
"""Check if tensor and pipeline parallel groups are initialized."""
|
||||
return (
|
||||
_DP is not None
|
||||
and _CFG is not None
|
||||
and _SP is not None
|
||||
and _PP is not None
|
||||
and vllm_parallel_state._TP is not None
|
||||
)
|
||||
|
||||
|
||||
def init_model_parallel_group(
|
||||
group_ranks: list[list[int]],
|
||||
local_rank: int,
|
||||
backend: str,
|
||||
parallel_mode: str,
|
||||
**kwargs,
|
||||
) -> GroupCoordinator:
|
||||
assert parallel_mode in [
|
||||
"data",
|
||||
"pipeline",
|
||||
"tensor",
|
||||
"sequence",
|
||||
"classifier_free_guidance",
|
||||
], f"parallel_mode {parallel_mode} is not supported"
|
||||
if parallel_mode == "pipeline":
|
||||
return PipelineGroupCoordinator(
|
||||
group_ranks=group_ranks,
|
||||
local_rank=local_rank,
|
||||
torch_distributed_backend=backend,
|
||||
)
|
||||
elif parallel_mode == "sequence":
|
||||
return SequenceParallelGroupCoordinator(
|
||||
group_ranks=group_ranks,
|
||||
local_rank=local_rank,
|
||||
torch_distributed_backend=backend,
|
||||
**kwargs,
|
||||
)
|
||||
else:
|
||||
return GroupCoordinator(
|
||||
group_ranks=group_ranks,
|
||||
local_rank=local_rank,
|
||||
torch_distributed_backend=backend,
|
||||
)
|
||||
|
||||
|
||||
def init_dit_group(
|
||||
dit_parallel_size: int,
|
||||
backend: str,
|
||||
):
|
||||
global _DIT
|
||||
_DIT = torch.distributed.new_group(ranks=list(range(dit_parallel_size)), backend=backend)
|
||||
|
||||
|
||||
def get_dit_group():
|
||||
assert _DIT is not None, "DIT group is not initialized"
|
||||
return _DIT
|
||||
|
||||
|
||||
def init_vae_group(
|
||||
dit_parallel_size: int,
|
||||
vae_parallel_size: int,
|
||||
backend: str,
|
||||
):
|
||||
# Initialize VAE group first
|
||||
global _VAE
|
||||
assert _VAE is None, "VAE parallel group is already initialized"
|
||||
vae_ranks = list(range(dit_parallel_size, dit_parallel_size + vae_parallel_size))
|
||||
_VAE = torch.distributed.new_group(ranks=vae_ranks, backend=backend)
|
||||
|
||||
|
||||
# adapted from https://github.com/feifeibear/long-context-attention/blob/main/yunchang/globals.py
|
||||
def set_seq_parallel_pg(sp_ulysses_degree, sp_ring_degree, rank, world_size, use_ulysses_low=True):
|
||||
"""
|
||||
sp_ulysses_degree x sp_ring_degree = seq_parallel_size
|
||||
(ulysses_degree, dp_size)
|
||||
"""
|
||||
sp_size = sp_ring_degree * sp_ulysses_degree
|
||||
dp_size = world_size // sp_size
|
||||
|
||||
assert world_size % sp_size == 0, f"world_size {world_size} % sp_size {sp_ulysses_degree} == 0"
|
||||
|
||||
num_ulysses_pgs = sp_ring_degree # world_size // sp_ulysses_degree
|
||||
num_ring_pgs = sp_ulysses_degree # world_size // sp_ring_degree
|
||||
|
||||
if use_ulysses_low:
|
||||
for dp_rank in range(dp_size):
|
||||
offset = dp_rank * sp_size
|
||||
for i in range(num_ulysses_pgs):
|
||||
ulysses_ranks = list(
|
||||
range(
|
||||
i * sp_ulysses_degree + offset,
|
||||
(i + 1) * sp_ulysses_degree + offset,
|
||||
)
|
||||
)
|
||||
group = torch.distributed.new_group(ulysses_ranks)
|
||||
if rank in ulysses_ranks:
|
||||
ulyssess_pg = group
|
||||
|
||||
for i in range(num_ring_pgs):
|
||||
ring_ranks = list(range(i + offset, sp_size + offset, num_ring_pgs))
|
||||
group = torch.distributed.new_group(ring_ranks)
|
||||
if rank in ring_ranks:
|
||||
ring_pg = group
|
||||
|
||||
else:
|
||||
for dp_rank in range(dp_size):
|
||||
offset = dp_rank * sp_size
|
||||
for i in range(num_ring_pgs):
|
||||
ring_ranks = list(range(i * sp_ring_degree + offset, (i + 1) * sp_ring_degree + offset))
|
||||
group = torch.distributed.new_group(ring_ranks)
|
||||
if rank in ring_ranks:
|
||||
ring_pg = group
|
||||
|
||||
for i in range(num_ulysses_pgs):
|
||||
ulysses_ranks = list(range(i + offset, sp_size + offset, num_ulysses_pgs))
|
||||
group = torch.distributed.new_group(ulysses_ranks)
|
||||
if rank in ulysses_ranks:
|
||||
ulyssess_pg = group
|
||||
|
||||
return ulyssess_pg, ring_pg
|
||||
|
||||
|
||||
def initialize_model_parallel(
|
||||
data_parallel_size: int = 1,
|
||||
cfg_parallel_size: int = 1,
|
||||
sequence_parallel_size: int | None = None,
|
||||
ulysses_degree: int = 1,
|
||||
ring_degree: int = 1,
|
||||
tensor_parallel_size: int = 1,
|
||||
pipeline_parallel_size: int = 1,
|
||||
vae_parallel_size: int = 0,
|
||||
backend: str | None = None,
|
||||
) -> None:
|
||||
if backend is None:
|
||||
backend = envs.get_torch_distributed_backend()
|
||||
"""
|
||||
Initialize model parallel groups.
|
||||
|
||||
Arguments:
|
||||
data_parallel_size: number of data parallelism groups.
|
||||
cfg_parallel_size: number of GPUs used for Classifier Free Guidance (CFG) parallelism.
|
||||
sequence_parallel_size: number of GPUs used for sequence parallelism.
|
||||
sequence_parallel_size = ulysses_degree * ring_degree
|
||||
ulysses_degree: number of GPUs used for ulysses sequence parallelism.
|
||||
ring_degree: number of GPUs used for ring sequence parallelism.
|
||||
tensor_parallel_size: number of GPUs used for tensor parallelism.
|
||||
pipeline_parallel_size: number of GPUs used for pipeline parallelism.
|
||||
backend: distributed backend of pytorch collective comm.
|
||||
|
||||
Let's say we have a total of 16 GPUs denoted by g0 ... g15 and we
|
||||
use 2 groups to parallelize the batch dim(dp), 2 groups to parallelize
|
||||
split batch caused by CFG, and 2 GPUs to parallelize sequence.
|
||||
|
||||
dp_size (2) * cfg_size (2) * sp_size (2) * pp_size (2) = 16.
|
||||
|
||||
The present function will create 8 data-parallel groups,
|
||||
8 CFG group, 8 pipeline-parallel group, and
|
||||
8 sequence-parallel groups:
|
||||
8 data-parallel groups:
|
||||
[g0, g8], [g1, g9], [g2, g10], [g3, g11],
|
||||
[g4, g12], [g5, g13], [g6, g14], [g7, g15]
|
||||
8 CFG-parallel groups:
|
||||
[g0, g4], [g1, g5], [g2, g6], [g3, g7],
|
||||
[g8, g12], [g9, g13], [g10, g14], [g11, g15]
|
||||
8 sequence-parallel groups:
|
||||
[g0, g1], [g2, g3], [g4, g5], [g6, g7],
|
||||
[g8, g9], [g10, g11], [g12, g13], [g14, g15]
|
||||
8 pipeline-parallel groups:
|
||||
[g0, g2], [g4, g6], [g8, g10], [g12, g14],
|
||||
[g1, g3], [g5, g7], [g9, g11], [g13, g15]
|
||||
Note that for efficiency, the caller should make sure adjacent ranks
|
||||
are on the same DGX box. For example if we are using 2 DGX-1 boxes
|
||||
with a total of 16 GPUs, rank 0 to 7 belong to the first box and
|
||||
ranks 8 to 15 belong to the second box.
|
||||
"""
|
||||
# Get world size and rank. Ensure some consistencies.
|
||||
assert torch.distributed.is_initialized()
|
||||
world_size: int = torch.distributed.get_world_size()
|
||||
backend = backend or torch.distributed.get_backend(get_world_group().device_group)
|
||||
|
||||
if sequence_parallel_size is None:
|
||||
sequence_parallel_size = ring_degree * ulysses_degree
|
||||
logger.info(
|
||||
f"sequence_parallel_size is not provided, using ring_degree * ulysses_degree = {sequence_parallel_size}"
|
||||
)
|
||||
|
||||
if sequence_parallel_size != ring_degree * ulysses_degree:
|
||||
raise ValueError(
|
||||
"sequence_parallel_size is not equal to ring_degree * ulysses_degree,"
|
||||
f" but got {sequence_parallel_size} != {ring_degree} * {ulysses_degree}"
|
||||
)
|
||||
|
||||
# FIXME: Since the async p2p communication operation of NPU is not same as cuda in torch,
|
||||
# the pipefusion is not ready for npu yet
|
||||
if envs._is_npu():
|
||||
assert pipeline_parallel_size == 1, "Current pipefusion is not ready for NPU"
|
||||
|
||||
dit_parallel_size = (
|
||||
data_parallel_size * cfg_parallel_size * sequence_parallel_size * pipeline_parallel_size * tensor_parallel_size
|
||||
)
|
||||
|
||||
if world_size < dit_parallel_size:
|
||||
raise RuntimeError(
|
||||
f"world_size ({world_size}) is less than "
|
||||
f"tensor_parallel_size ({tensor_parallel_size}) x "
|
||||
f"pipeline_parallel_size ({pipeline_parallel_size}) x"
|
||||
f"sequence_parallel_size ({sequence_parallel_size}) x"
|
||||
f"cfg_parallel_size "
|
||||
f"({cfg_parallel_size}) x"
|
||||
f"data_parallel_size ({data_parallel_size})"
|
||||
)
|
||||
|
||||
rank_generator: RankGenerator = RankGenerator(
|
||||
tensor_parallel_size,
|
||||
sequence_parallel_size,
|
||||
pipeline_parallel_size,
|
||||
cfg_parallel_size,
|
||||
data_parallel_size,
|
||||
"tp-sp-pp-cfg-dp",
|
||||
)
|
||||
global _DP
|
||||
assert _DP is None, "data parallel group is already initialized"
|
||||
_DP = init_model_parallel_group(
|
||||
group_ranks=rank_generator.get_ranks("dp"),
|
||||
local_rank=get_world_group().local_rank,
|
||||
backend=backend,
|
||||
parallel_mode="data",
|
||||
)
|
||||
|
||||
global _CFG
|
||||
assert _CFG is None, "classifier_free_guidance group is already initialized"
|
||||
_CFG = init_model_parallel_group(
|
||||
group_ranks=rank_generator.get_ranks("cfg"),
|
||||
local_rank=get_world_group().local_rank,
|
||||
backend=backend,
|
||||
parallel_mode="classifier_free_guidance",
|
||||
)
|
||||
global _PP
|
||||
assert _PP is None, "pipeline model parallel group is already initialized"
|
||||
_PP = init_model_parallel_group(
|
||||
group_ranks=rank_generator.get_ranks("pp"),
|
||||
local_rank=get_world_group().local_rank,
|
||||
backend=backend,
|
||||
parallel_mode="pipeline",
|
||||
)
|
||||
|
||||
global _SP
|
||||
assert _SP is None, "sequence parallel group is already initialized"
|
||||
ulysses_pg, ring_pg = set_seq_parallel_pg(
|
||||
sp_ulysses_degree=ulysses_degree,
|
||||
sp_ring_degree=ring_degree,
|
||||
rank=get_world_group().rank_in_group,
|
||||
world_size=dit_parallel_size,
|
||||
)
|
||||
_SP = init_model_parallel_group(
|
||||
group_ranks=rank_generator.get_ranks("sp"),
|
||||
local_rank=get_world_group().local_rank,
|
||||
backend=backend,
|
||||
parallel_mode="sequence",
|
||||
ulysses_group=ulysses_pg,
|
||||
ring_group=ring_pg,
|
||||
)
|
||||
|
||||
assert vllm_parallel_state._TP is None, "Tensor parallel group is already initialized"
|
||||
vllm_parallel_state._TP = init_model_parallel_group(
|
||||
group_ranks=rank_generator.get_ranks("tp"),
|
||||
local_rank=get_world_group().local_rank,
|
||||
backend=backend,
|
||||
parallel_mode="tensor",
|
||||
)
|
||||
if vae_parallel_size > 0:
|
||||
init_vae_group(dit_parallel_size, vae_parallel_size, backend)
|
||||
init_dit_group(dit_parallel_size, backend)
|
||||
|
||||
|
||||
def destroy_model_parallel():
|
||||
"""Set the groups to none and destroy them."""
|
||||
global _DP
|
||||
if _DP:
|
||||
_DP.destroy()
|
||||
_DP = None
|
||||
|
||||
global _CFG
|
||||
if _CFG:
|
||||
_CFG.destroy()
|
||||
_CFG = None
|
||||
|
||||
global _SP
|
||||
if _SP:
|
||||
_SP.destroy()
|
||||
_SP = None
|
||||
|
||||
if vllm_parallel_state._TP:
|
||||
vllm_parallel_state._TP.destroy()
|
||||
vllm_parallel_state._TP = None
|
||||
|
||||
global _PP
|
||||
if _PP:
|
||||
_PP.destroy()
|
||||
_PP = None
|
||||
|
||||
global _VAE
|
||||
if _VAE:
|
||||
_VAE.destroy()
|
||||
_VAE = None
|
||||
|
||||
|
||||
def destroy_distributed_environment():
|
||||
global _WORLD
|
||||
if _WORLD:
|
||||
_WORLD.destroy()
|
||||
_WORLD = None
|
||||
if torch.distributed.is_initialized():
|
||||
torch.distributed.destroy_process_group()
|
||||
|
||||
|
||||
def destroy_distributed_env():
|
||||
if model_parallel_is_initialized():
|
||||
destroy_model_parallel()
|
||||
destroy_distributed_environment()
|
||||
Reference in new issue
Block a user