[Feature] Support HSDP for diffusion models (#1339)
Signed-off-by: gcanlin <canlinguosdu@gmail.com> Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
This commit is contained in:
1 parent
66c3e04263
commit
f53152d340
17 files changed
+1023
-46
No files matched your search
@@ -2,6 +2,15 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Distributed utilities for vLLM-Omni diffusion models."""
|
||||
|
||||
from vllm_omni.diffusion.distributed.hsdp import (
|
||||
HSDPInferenceConfig,
|
||||
apply_hsdp_to_model,
|
||||
)
|
||||
from vllm_omni.diffusion.distributed.parallel_state import (
|
||||
get_fs_group,
|
||||
get_fully_shard_rank,
|
||||
get_fully_shard_world_size,
|
||||
)
|
||||
from vllm_omni.diffusion.distributed.sp_plan import (
|
||||
SequenceParallelConfig,
|
||||
SequenceParallelInput,
|
||||
@@ -35,4 +44,10 @@ __all__ = [
|
||||
"sp_shard_with_padding",
|
||||
"ShardingValidator",
|
||||
"get_sharding_validator",
|
||||
"HSDPInferenceConfig",
|
||||
"apply_hsdp_to_model",
|
||||
# FS (Fully Shard) utilities
|
||||
"get_fs_group",
|
||||
"get_fully_shard_rank",
|
||||
"get_fully_shard_world_size",
|
||||
]
|
||||
@@ -0,0 +1,202 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
import torch.distributed
|
||||
from torch import nn
|
||||
from torch.distributed import DeviceMesh, init_device_mesh
|
||||
from torch.distributed.fsdp import (
|
||||
MixedPrecisionPolicy,
|
||||
fully_shard,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from vllm_omni.diffusion.distributed.parallel_state import (
|
||||
get_fs_group,
|
||||
get_fully_shard_rank,
|
||||
get_fully_shard_world_size,
|
||||
get_world_group,
|
||||
)
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class HSDPInferenceConfig:
|
||||
"""Configuration for HSDP inference.
|
||||
|
||||
This is a runtime config created from DiffusionParallelConfig's HSDP settings.
|
||||
"""
|
||||
|
||||
enabled: bool = False
|
||||
hsdp_replicate_size: int = 1
|
||||
hsdp_shard_size: int = -1 # -1 = auto (shard across entire world)
|
||||
param_dtype: torch.dtype = torch.bfloat16
|
||||
reduce_dtype: torch.dtype = torch.float32
|
||||
output_dtype: torch.dtype | None = None
|
||||
reshard_after_forward: bool = True
|
||||
|
||||
|
||||
def _create_hsdp_mesh(
|
||||
device_type: str,
|
||||
replicate_size: int,
|
||||
shard_pg: torch.distributed.ProcessGroup,
|
||||
) -> DeviceMesh:
|
||||
"""Create a 2D DeviceMesh for HSDP using an existing ProcessGroup for the shard dimension.
|
||||
|
||||
Args:
|
||||
device_type: The device type (e.g., "cuda", "npu")
|
||||
replicate_size: Number of replica groups
|
||||
shard_pg: The ProcessGroup for the shard dimension (from FS GroupCoordinator)
|
||||
|
||||
Returns:
|
||||
A 2D DeviceMesh with dimensions ("replicate", "shard")
|
||||
"""
|
||||
shard_size = torch.distributed.get_world_size(shard_pg)
|
||||
world_size = replicate_size * shard_size
|
||||
|
||||
# Build 2D mesh tensor: shape (replicate_size, shard_size)
|
||||
# Ranks are arranged so that each row is a shard group
|
||||
mesh_tensor = torch.arange(world_size).reshape(replicate_size, shard_size)
|
||||
|
||||
# Create DeviceMesh with the shard ProcessGroup
|
||||
# For the shard dimension, we reuse the existing FS ProcessGroup
|
||||
device_mesh = init_device_mesh(
|
||||
device_type,
|
||||
mesh_shape=(replicate_size, shard_size),
|
||||
mesh_dim_names=("replicate", "shard"),
|
||||
)
|
||||
|
||||
# Note: init_device_mesh creates new ProcessGroups internally.
|
||||
# For consistency, we verify the mesh structure matches our FS group.
|
||||
# In a future optimization, we could pass the existing ProcessGroups directly.
|
||||
logger.debug(
|
||||
"Created HSDP mesh: replicate_size=%d, shard_size=%d, mesh=%s",
|
||||
replicate_size,
|
||||
shard_size,
|
||||
mesh_tensor.tolist(),
|
||||
)
|
||||
|
||||
return device_mesh
|
||||
|
||||
|
||||
def apply_hsdp_to_model(
|
||||
model: nn.Module,
|
||||
hsdp_config: HSDPInferenceConfig,
|
||||
) -> nn.Module:
|
||||
"""
|
||||
Apply HSDP sharding to a model that already has weights loaded.
|
||||
|
||||
This function redistributes the model's parameters across GPUs using HSDP.
|
||||
The model should already have its weights loaded via the standard load_weights method.
|
||||
|
||||
Args:
|
||||
model: Model instance with weights already loaded
|
||||
hsdp_config: HSDP configuration with HSDP mesh dimensions
|
||||
|
||||
Returns:
|
||||
HSDP-wrapped model ready for inference
|
||||
"""
|
||||
if not hsdp_config.enabled:
|
||||
raise ValueError("HSDP is not enabled in config")
|
||||
|
||||
# Use GroupCoordinator for distributed info
|
||||
world_group = get_world_group()
|
||||
fs_group = get_fs_group()
|
||||
|
||||
world_size = world_group.world_size
|
||||
rank = world_group.rank_in_group
|
||||
fs_world_size = get_fully_shard_world_size()
|
||||
fs_rank = get_fully_shard_rank()
|
||||
|
||||
hsdp_replicate_size = hsdp_config.hsdp_replicate_size
|
||||
hsdp_shard_size = hsdp_config.hsdp_shard_size
|
||||
|
||||
# Validate that the FS group matches the HSDP shard size
|
||||
if fs_world_size != hsdp_shard_size:
|
||||
raise ValueError(
|
||||
f"FS group world_size ({fs_world_size}) does not match "
|
||||
f"HSDP shard_size ({hsdp_shard_size}). "
|
||||
"Ensure fully_shard_degree is set correctly in initialize_model_parallel."
|
||||
)
|
||||
|
||||
logger.info(
|
||||
"HSDP Inference: replicate_size=%d, shard_size=%d, world_size=%d, rank=%d, fs_world_size=%d, fs_rank=%d",
|
||||
hsdp_replicate_size,
|
||||
hsdp_shard_size,
|
||||
world_size,
|
||||
rank,
|
||||
fs_world_size,
|
||||
fs_rank,
|
||||
)
|
||||
|
||||
mp_policy = MixedPrecisionPolicy(
|
||||
param_dtype=hsdp_config.param_dtype,
|
||||
reduce_dtype=hsdp_config.reduce_dtype,
|
||||
output_dtype=hsdp_config.output_dtype,
|
||||
cast_forward_inputs=False,
|
||||
)
|
||||
|
||||
device_type = current_omni_platform.device_type
|
||||
|
||||
# Create 2D DeviceMesh for HSDP using the FS group's ProcessGroup for shard dimension
|
||||
# The mesh shape is (replicate, shard) where:
|
||||
# - replicate: groups of ranks that hold the same shard (for gradient all-reduce in training)
|
||||
# - shard: groups of ranks that each hold different shards (for parameter all-gather)
|
||||
device_mesh = _create_hsdp_mesh(
|
||||
device_type=device_type,
|
||||
replicate_size=hsdp_replicate_size,
|
||||
shard_pg=fs_group.device_group,
|
||||
)
|
||||
|
||||
hsdp_shard_conditions = getattr(model, "_hsdp_shard_conditions", None)
|
||||
if not hsdp_shard_conditions or len(hsdp_shard_conditions) == 0:
|
||||
raise ValueError(f"Model {type(model).__name__} has no _hsdp_shard_conditions defined")
|
||||
|
||||
# Apply HSDP sharding, this will automatically handle weight distribution
|
||||
shard_model(
|
||||
model,
|
||||
reshard_after_forward=hsdp_config.reshard_after_forward,
|
||||
mp_policy=mp_policy,
|
||||
mesh=device_mesh,
|
||||
hsdp_shard_conditions=hsdp_shard_conditions,
|
||||
)
|
||||
|
||||
for param in model.parameters():
|
||||
param.requires_grad = False
|
||||
|
||||
logger.info("HSDP applied to model: %s", type(model).__name__)
|
||||
return model
|
||||
|
||||
|
||||
def shard_model(
|
||||
model: nn.Module,
|
||||
*,
|
||||
reshard_after_forward: bool = True,
|
||||
mp_policy: MixedPrecisionPolicy | None = None,
|
||||
mesh: DeviceMesh | None = None,
|
||||
hsdp_shard_conditions: list[Callable[[str, nn.Module], bool]],
|
||||
) -> None:
|
||||
"""Apply HSDP sharding to model modules based on shard conditions."""
|
||||
hsdp_kwargs: dict[str, Any] = {
|
||||
"reshard_after_forward": reshard_after_forward,
|
||||
"mesh": mesh,
|
||||
"mp_policy": mp_policy,
|
||||
}
|
||||
|
||||
num_sharded = 0
|
||||
for name, module in reversed(list(model.named_modules())):
|
||||
if any(cond(name, module) for cond in hsdp_shard_conditions):
|
||||
fully_shard(module, **hsdp_kwargs)
|
||||
num_sharded += 1
|
||||
|
||||
if num_sharded == 0:
|
||||
raise ValueError("No modules were sharded")
|
||||
|
||||
fully_shard(model, **hsdp_kwargs)
|
||||
logger.info("Sharded %d modules + root", num_sharded)
|
||||
@@ -56,6 +56,7 @@ _SP: SequenceParallelGroupCoordinator | None = None
|
||||
_PP: PipelineGroupCoordinator | None = None
|
||||
_CFG: GroupCoordinator | None = None
|
||||
_DP: GroupCoordinator | None = None
|
||||
_FS: GroupCoordinator | None = None # Fully Sharded (HSDP shard dimension)
|
||||
_DIT: GroupCoordinator | None = None
|
||||
|
||||
|
||||
@@ -175,7 +176,8 @@ class RankGenerator:
|
||||
pp: int,
|
||||
cfg: int,
|
||||
dp: int,
|
||||
order: str,
|
||||
fs: int = 1,
|
||||
order: str = "tp-sp-pp-cfg-dp",
|
||||
rank_offset: int = 0,
|
||||
) -> None:
|
||||
self.tp = tp
|
||||
@@ -183,6 +185,7 @@ class RankGenerator:
|
||||
self.pp = pp
|
||||
self.cfg = cfg
|
||||
self.dp = dp
|
||||
self.fs = fs
|
||||
self.rank_offset = rank_offset
|
||||
self.world_size = tp * sp * pp * cfg * dp
|
||||
|
||||
@@ -192,14 +195,19 @@ class RankGenerator:
|
||||
"pp": self.pp,
|
||||
"cfg": self.cfg,
|
||||
"dp": self.dp,
|
||||
"fs": self.fs,
|
||||
}
|
||||
order = order.lower()
|
||||
|
||||
for name in self.name_to_size.keys():
|
||||
# Skip 'fs' validation - it's handled separately with independent_ranks=True
|
||||
# and doesn't participate in the main orthogonal rank generation
|
||||
if name == "fs":
|
||||
continue
|
||||
if name not in order and self.name_to_size[name] != 1:
|
||||
raise RuntimeError(
|
||||
f"The size of ({name}) is ({self.name_to_size[name]}), "
|
||||
f"but you haven't specified the order ({self.order})."
|
||||
f"but you haven't specified the order ({order})."
|
||||
)
|
||||
elif name not in order:
|
||||
order = order + "-" + name
|
||||
@@ -218,7 +226,7 @@ class RankGenerator:
|
||||
mask[ordered_token.index(t)] = True
|
||||
return mask
|
||||
|
||||
def get_ranks(self, token):
|
||||
def get_ranks(self, token, independent_ranks: bool = False):
|
||||
"""Get rank group by input token.
|
||||
|
||||
Arguments:
|
||||
@@ -227,7 +235,21 @@ class RankGenerator:
|
||||
to obtain multiple parallel types, we can use a hyphen
|
||||
'-' to separate them. For example, if we want to obtain
|
||||
the TP_DP group, the token should be 'tp-dp'.
|
||||
independent_ranks (bool):
|
||||
If True, generate independent rank groups that divide the world
|
||||
into groups of the specified size. Used for FS (fully shard) groups
|
||||
which operate independently from the main parallelism hierarchy.
|
||||
"""
|
||||
if independent_ranks and token == "fs":
|
||||
# FS groups divide world into groups of size fs
|
||||
# e.g., world_size=8, fs=4 -> [[0,1,2,3], [4,5,6,7]]
|
||||
ranks = []
|
||||
num_groups = self.world_size // self.fs
|
||||
for i in range(num_groups):
|
||||
group = list(range(i * self.fs + self.rank_offset, (i + 1) * self.fs + self.rank_offset))
|
||||
ranks.append(group)
|
||||
return ranks
|
||||
|
||||
mask = self.get_mask(self.order, token)
|
||||
ranks = generate_masked_orthogonal_rank_groups(self.world_size, self.ordered_size, mask)
|
||||
if self.rank_offset > 0:
|
||||
@@ -333,6 +355,22 @@ def get_data_parallel_rank():
|
||||
return get_dp_group().rank_in_group
|
||||
|
||||
|
||||
# FS (Fully Shard / HSDP shard dimension)
|
||||
def get_fs_group() -> GroupCoordinator:
|
||||
assert _FS is not None, "fully shard group is not initialized"
|
||||
return _FS
|
||||
|
||||
|
||||
def get_fully_shard_world_size():
|
||||
"""Return world size for the fully shard group."""
|
||||
return get_fs_group().world_size
|
||||
|
||||
|
||||
def get_fully_shard_rank():
|
||||
"""Return my rank for the fully shard group."""
|
||||
return get_fs_group().rank_in_group
|
||||
|
||||
|
||||
def is_dp_last_group():
|
||||
"""Return True if in the last data parallel group, False otherwise."""
|
||||
return (
|
||||
@@ -438,6 +476,7 @@ def init_model_parallel_group(
|
||||
"tensor",
|
||||
"sequence",
|
||||
"classifier_free_guidance",
|
||||
"fully_shard",
|
||||
], f"parallel_mode {parallel_mode} is not supported"
|
||||
if parallel_mode == "pipeline":
|
||||
return PipelineGroupCoordinator(
|
||||
@@ -629,6 +668,7 @@ def initialize_model_parallel(
|
||||
ring_degree: int = 1,
|
||||
tensor_parallel_size: int = 1,
|
||||
pipeline_parallel_size: int = 1,
|
||||
fully_shard_degree: int = 1,
|
||||
backend: str | None = None,
|
||||
) -> None:
|
||||
if backend is None:
|
||||
@@ -645,6 +685,7 @@ def initialize_model_parallel(
|
||||
ring_degree: number of GPUs used for ring sequence parallelism.
|
||||
tensor_parallel_size: number of GPUs used for tensor parallelism.
|
||||
pipeline_parallel_size: number of GPUs used for pipeline parallelism.
|
||||
fully_shard_degree: number of GPUs used for fully sharded data parallelism (HSDP shard dimension).
|
||||
backend: distributed backend of pytorch collective comm.
|
||||
|
||||
Let's say we have a total of 16 GPUs denoted by g0 ... g15 and we
|
||||
@@ -716,7 +757,8 @@ def initialize_model_parallel(
|
||||
pipeline_parallel_size,
|
||||
cfg_parallel_size,
|
||||
data_parallel_size,
|
||||
"tp-sp-pp-cfg-dp",
|
||||
fs=fully_shard_degree,
|
||||
order="tp-sp-pp-cfg-dp",
|
||||
)
|
||||
sp_group_ranks = rank_generator.get_ranks("sp")
|
||||
global _DP
|
||||
@@ -772,6 +814,16 @@ def initialize_model_parallel(
|
||||
backend=backend,
|
||||
parallel_mode="tensor",
|
||||
)
|
||||
|
||||
global _FS
|
||||
assert _FS is None, "fully shard group is already initialized"
|
||||
_FS = init_model_parallel_group(
|
||||
group_ranks=rank_generator.get_ranks("fs", independent_ranks=True),
|
||||
local_rank=get_world_group().local_rank,
|
||||
backend=backend,
|
||||
parallel_mode="fully_shard",
|
||||
)
|
||||
|
||||
init_dit_group(dit_parallel_size, backend)
|
||||
|
||||
|
||||
@@ -801,6 +853,11 @@ def destroy_model_parallel():
|
||||
_PP.destroy()
|
||||
_PP = None
|
||||
|
||||
global _FS
|
||||
if _FS:
|
||||
_FS.destroy()
|
||||
_FS = None
|
||||
|
||||
|
||||
def destroy_distributed_environment():
|
||||
global _WORLD
|
||||
|
||||
Reference in new issue
Block a user