[Feature] Support HSDP for diffusion models (#1339)

Signed-off-by: gcanlin <canlinguosdu@gmail.com>
Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
This commit is contained in:
Canlin GuoandHongsheng Liu authored and GitHub committed 2026-02-26 21:31:10 +08:00
1 parent 66c3e04263
commit f53152d340
17 files changed
+1023 -46

No files matched your search

@@ -2,6 +2,15 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Distributed utilities for vLLM-Omni diffusion models."""
from vllm_omni.diffusion.distributed.hsdp import (
HSDPInferenceConfig,
apply_hsdp_to_model,
)
from vllm_omni.diffusion.distributed.parallel_state import (
get_fs_group,
get_fully_shard_rank,
get_fully_shard_world_size,
)
from vllm_omni.diffusion.distributed.sp_plan import (
SequenceParallelConfig,
SequenceParallelInput,
@@ -35,4 +44,10 @@ __all__ = [
"sp_shard_with_padding",
"ShardingValidator",
"get_sharding_validator",
"HSDPInferenceConfig",
"apply_hsdp_to_model",
# FS (Fully Shard) utilities
"get_fs_group",
"get_fully_shard_rank",
"get_fully_shard_world_size",
]
+202
View File
@@ -0,0 +1,202 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Callable
from dataclasses import dataclass
from typing import Any
import torch
import torch.distributed
from torch import nn
from torch.distributed import DeviceMesh, init_device_mesh
from torch.distributed.fsdp import (
MixedPrecisionPolicy,
fully_shard,
)
from vllm.logger import init_logger
from vllm_omni.diffusion.distributed.parallel_state import (
get_fs_group,
get_fully_shard_rank,
get_fully_shard_world_size,
get_world_group,
)
from vllm_omni.platforms import current_omni_platform
logger = init_logger(__name__)
@dataclass
class HSDPInferenceConfig:
"""Configuration for HSDP inference.
This is a runtime config created from DiffusionParallelConfig's HSDP settings.
"""
enabled: bool = False
hsdp_replicate_size: int = 1
hsdp_shard_size: int = -1 # -1 = auto (shard across entire world)
param_dtype: torch.dtype = torch.bfloat16
reduce_dtype: torch.dtype = torch.float32
output_dtype: torch.dtype | None = None
reshard_after_forward: bool = True
def _create_hsdp_mesh(
device_type: str,
replicate_size: int,
shard_pg: torch.distributed.ProcessGroup,
) -> DeviceMesh:
"""Create a 2D DeviceMesh for HSDP using an existing ProcessGroup for the shard dimension.
Args:
device_type: The device type (e.g., "cuda", "npu")
replicate_size: Number of replica groups
shard_pg: The ProcessGroup for the shard dimension (from FS GroupCoordinator)
Returns:
A 2D DeviceMesh with dimensions ("replicate", "shard")
"""
shard_size = torch.distributed.get_world_size(shard_pg)
world_size = replicate_size * shard_size
# Build 2D mesh tensor: shape (replicate_size, shard_size)
# Ranks are arranged so that each row is a shard group
mesh_tensor = torch.arange(world_size).reshape(replicate_size, shard_size)
# Create DeviceMesh with the shard ProcessGroup
# For the shard dimension, we reuse the existing FS ProcessGroup
device_mesh = init_device_mesh(
device_type,
mesh_shape=(replicate_size, shard_size),
mesh_dim_names=("replicate", "shard"),
)
# Note: init_device_mesh creates new ProcessGroups internally.
# For consistency, we verify the mesh structure matches our FS group.
# In a future optimization, we could pass the existing ProcessGroups directly.
logger.debug(
"Created HSDP mesh: replicate_size=%d, shard_size=%d, mesh=%s",
replicate_size,
shard_size,
mesh_tensor.tolist(),
)
return device_mesh
def apply_hsdp_to_model(
model: nn.Module,
hsdp_config: HSDPInferenceConfig,
) -> nn.Module:
"""
Apply HSDP sharding to a model that already has weights loaded.
This function redistributes the model's parameters across GPUs using HSDP.
The model should already have its weights loaded via the standard load_weights method.
Args:
model: Model instance with weights already loaded
hsdp_config: HSDP configuration with HSDP mesh dimensions
Returns:
HSDP-wrapped model ready for inference
"""
if not hsdp_config.enabled:
raise ValueError("HSDP is not enabled in config")
# Use GroupCoordinator for distributed info
world_group = get_world_group()
fs_group = get_fs_group()
world_size = world_group.world_size
rank = world_group.rank_in_group
fs_world_size = get_fully_shard_world_size()
fs_rank = get_fully_shard_rank()
hsdp_replicate_size = hsdp_config.hsdp_replicate_size
hsdp_shard_size = hsdp_config.hsdp_shard_size
# Validate that the FS group matches the HSDP shard size
if fs_world_size != hsdp_shard_size:
raise ValueError(
f"FS group world_size ({fs_world_size}) does not match "
f"HSDP shard_size ({hsdp_shard_size}). "
"Ensure fully_shard_degree is set correctly in initialize_model_parallel."
)
logger.info(
"HSDP Inference: replicate_size=%d, shard_size=%d, world_size=%d, rank=%d, fs_world_size=%d, fs_rank=%d",
hsdp_replicate_size,
hsdp_shard_size,
world_size,
rank,
fs_world_size,
fs_rank,
)
mp_policy = MixedPrecisionPolicy(
param_dtype=hsdp_config.param_dtype,
reduce_dtype=hsdp_config.reduce_dtype,
output_dtype=hsdp_config.output_dtype,
cast_forward_inputs=False,
)
device_type = current_omni_platform.device_type
# Create 2D DeviceMesh for HSDP using the FS group's ProcessGroup for shard dimension
# The mesh shape is (replicate, shard) where:
# - replicate: groups of ranks that hold the same shard (for gradient all-reduce in training)
# - shard: groups of ranks that each hold different shards (for parameter all-gather)
device_mesh = _create_hsdp_mesh(
device_type=device_type,
replicate_size=hsdp_replicate_size,
shard_pg=fs_group.device_group,
)
hsdp_shard_conditions = getattr(model, "_hsdp_shard_conditions", None)
if not hsdp_shard_conditions or len(hsdp_shard_conditions) == 0:
raise ValueError(f"Model {type(model).__name__} has no _hsdp_shard_conditions defined")
# Apply HSDP sharding, this will automatically handle weight distribution
shard_model(
model,
reshard_after_forward=hsdp_config.reshard_after_forward,
mp_policy=mp_policy,
mesh=device_mesh,
hsdp_shard_conditions=hsdp_shard_conditions,
)
for param in model.parameters():
param.requires_grad = False
logger.info("HSDP applied to model: %s", type(model).__name__)
return model
def shard_model(
model: nn.Module,
*,
reshard_after_forward: bool = True,
mp_policy: MixedPrecisionPolicy | None = None,
mesh: DeviceMesh | None = None,
hsdp_shard_conditions: list[Callable[[str, nn.Module], bool]],
) -> None:
"""Apply HSDP sharding to model modules based on shard conditions."""
hsdp_kwargs: dict[str, Any] = {
"reshard_after_forward": reshard_after_forward,
"mesh": mesh,
"mp_policy": mp_policy,
}
num_sharded = 0
for name, module in reversed(list(model.named_modules())):
if any(cond(name, module) for cond in hsdp_shard_conditions):
fully_shard(module, **hsdp_kwargs)
num_sharded += 1
if num_sharded == 0:
raise ValueError("No modules were sharded")
fully_shard(model, **hsdp_kwargs)
logger.info("Sharded %d modules + root", num_sharded)
@@ -56,6 +56,7 @@ _SP: SequenceParallelGroupCoordinator | None = None
_PP: PipelineGroupCoordinator | None = None
_CFG: GroupCoordinator | None = None
_DP: GroupCoordinator | None = None
_FS: GroupCoordinator | None = None # Fully Sharded (HSDP shard dimension)
_DIT: GroupCoordinator | None = None
@@ -175,7 +176,8 @@ class RankGenerator:
pp: int,
cfg: int,
dp: int,
order: str,
fs: int = 1,
order: str = "tp-sp-pp-cfg-dp",
rank_offset: int = 0,
) -> None:
self.tp = tp
@@ -183,6 +185,7 @@ class RankGenerator:
self.pp = pp
self.cfg = cfg
self.dp = dp
self.fs = fs
self.rank_offset = rank_offset
self.world_size = tp * sp * pp * cfg * dp
@@ -192,14 +195,19 @@ class RankGenerator:
"pp": self.pp,
"cfg": self.cfg,
"dp": self.dp,
"fs": self.fs,
}
order = order.lower()
for name in self.name_to_size.keys():
# Skip 'fs' validation - it's handled separately with independent_ranks=True
# and doesn't participate in the main orthogonal rank generation
if name == "fs":
continue
if name not in order and self.name_to_size[name] != 1:
raise RuntimeError(
f"The size of ({name}) is ({self.name_to_size[name]}), "
f"but you haven't specified the order ({self.order})."
f"but you haven't specified the order ({order})."
)
elif name not in order:
order = order + "-" + name
@@ -218,7 +226,7 @@ class RankGenerator:
mask[ordered_token.index(t)] = True
return mask
def get_ranks(self, token):
def get_ranks(self, token, independent_ranks: bool = False):
"""Get rank group by input token.
Arguments:
@@ -227,7 +235,21 @@ class RankGenerator:
to obtain multiple parallel types, we can use a hyphen
'-' to separate them. For example, if we want to obtain
the TP_DP group, the token should be 'tp-dp'.
independent_ranks (bool):
If True, generate independent rank groups that divide the world
into groups of the specified size. Used for FS (fully shard) groups
which operate independently from the main parallelism hierarchy.
"""
if independent_ranks and token == "fs":
# FS groups divide world into groups of size fs
# e.g., world_size=8, fs=4 -> [[0,1,2,3], [4,5,6,7]]
ranks = []
num_groups = self.world_size // self.fs
for i in range(num_groups):
group = list(range(i * self.fs + self.rank_offset, (i + 1) * self.fs + self.rank_offset))
ranks.append(group)
return ranks
mask = self.get_mask(self.order, token)
ranks = generate_masked_orthogonal_rank_groups(self.world_size, self.ordered_size, mask)
if self.rank_offset > 0:
@@ -333,6 +355,22 @@ def get_data_parallel_rank():
return get_dp_group().rank_in_group
# FS (Fully Shard / HSDP shard dimension)
def get_fs_group() -> GroupCoordinator:
assert _FS is not None, "fully shard group is not initialized"
return _FS
def get_fully_shard_world_size():
"""Return world size for the fully shard group."""
return get_fs_group().world_size
def get_fully_shard_rank():
"""Return my rank for the fully shard group."""
return get_fs_group().rank_in_group
def is_dp_last_group():
"""Return True if in the last data parallel group, False otherwise."""
return (
@@ -438,6 +476,7 @@ def init_model_parallel_group(
"tensor",
"sequence",
"classifier_free_guidance",
"fully_shard",
], f"parallel_mode {parallel_mode} is not supported"
if parallel_mode == "pipeline":
return PipelineGroupCoordinator(
@@ -629,6 +668,7 @@ def initialize_model_parallel(
ring_degree: int = 1,
tensor_parallel_size: int = 1,
pipeline_parallel_size: int = 1,
fully_shard_degree: int = 1,
backend: str | None = None,
) -> None:
if backend is None:
@@ -645,6 +685,7 @@ def initialize_model_parallel(
ring_degree: number of GPUs used for ring sequence parallelism.
tensor_parallel_size: number of GPUs used for tensor parallelism.
pipeline_parallel_size: number of GPUs used for pipeline parallelism.
fully_shard_degree: number of GPUs used for fully sharded data parallelism (HSDP shard dimension).
backend: distributed backend of pytorch collective comm.
Let's say we have a total of 16 GPUs denoted by g0 ... g15 and we
@@ -716,7 +757,8 @@ def initialize_model_parallel(
pipeline_parallel_size,
cfg_parallel_size,
data_parallel_size,
"tp-sp-pp-cfg-dp",
fs=fully_shard_degree,
order="tp-sp-pp-cfg-dp",
)
sp_group_ranks = rank_generator.get_ranks("sp")
global _DP
@@ -772,6 +814,16 @@ def initialize_model_parallel(
backend=backend,
parallel_mode="tensor",
)
global _FS
assert _FS is None, "fully shard group is already initialized"
_FS = init_model_parallel_group(
group_ranks=rank_generator.get_ranks("fs", independent_ranks=True),
local_rank=get_world_group().local_rank,
backend=backend,
parallel_mode="fully_shard",
)
init_dit_group(dit_parallel_size, backend)
@@ -801,6 +853,11 @@ def destroy_model_parallel():
_PP.destroy()
_PP = None
global _FS
if _FS:
_FS.destroy()
_FS = None
def destroy_distributed_environment():
global _WORLD