[Core] Restore topology-aware diffusion worker titles
Signed-off-by: hsliuustc0106 <hsliuustc0106@users.noreply.github.com>
This commit is contained in:
1 parent
d5aced9296
commit
65ac6794f9
3 files changed
+321
-8
No files matched your search
@@ -1,4 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
# Adapted from: https://github.com/vllm-project/vllm/blob/v0.7.3/vllm/distributed/parallel_state.py
|
||||
# Copyright 2023 The vLLM team.
|
||||
# Adapted from
|
||||
@@ -59,6 +60,7 @@ _SP: SequenceParallelGroupCoordinator | None = None
|
||||
_PP: PipelineGroupCoordinator | None = None
|
||||
_CFG: GroupCoordinator | None = None
|
||||
_DP: GroupCoordinator | None = None
|
||||
_FS: GroupCoordinator | None = None # Fully Sharded (HSDP shard dimension)
|
||||
|
||||
# Rank-layout metadata for expert parallelism. This is not a process group;
|
||||
# it is reused by platform-specific runtimes that must build companion groups
|
||||
@@ -350,6 +352,12 @@ def get_data_parallel_rank():
|
||||
return get_dp_group().rank_in_group
|
||||
|
||||
|
||||
# FS (Fully Shard / HSDP shard dimension)
|
||||
def get_fs_group() -> GroupCoordinator:
|
||||
assert _FS is not None, "fully shard group is not initialized"
|
||||
return _FS
|
||||
|
||||
|
||||
def is_dp_last_group():
|
||||
"""Return True if in the last data parallel group, False otherwise."""
|
||||
return (
|
||||
@@ -445,6 +453,7 @@ def init_model_parallel_group(
|
||||
"expert",
|
||||
"sequence",
|
||||
"classifier_free_guidance",
|
||||
"fully_shard",
|
||||
], f"parallel_mode {parallel_mode} is not supported"
|
||||
if parallel_mode == "pipeline":
|
||||
return PipelineGroupCoordinator(
|
||||
@@ -679,10 +688,13 @@ def _initialize_model_parallel(
|
||||
allgather_degree: int = 1,
|
||||
tensor_parallel_size: int = 1,
|
||||
pipeline_parallel_size: int = 1,
|
||||
fully_shard_degree: int = 1,
|
||||
enable_expert_parallel: bool = False,
|
||||
use_hsdp: bool = False,
|
||||
backend: str | None = None,
|
||||
) -> None:
|
||||
global _FS
|
||||
|
||||
if backend is None:
|
||||
backend = current_omni_platform.dist_backend
|
||||
"""
|
||||
@@ -700,6 +712,7 @@ def _initialize_model_parallel(
|
||||
(causal=False only). Mutually exclusive with ulysses/ring in v1.
|
||||
tensor_parallel_size: number of GPUs used for tensor parallelism.
|
||||
pipeline_parallel_size: number of GPUs used for pipeline parallelism.
|
||||
fully_shard_degree: number of GPUs used for the HSDP shard dimension.
|
||||
backend: distributed backend of pytorch collective comm.
|
||||
|
||||
Let's say we have a total of 16 GPUs denoted by g0 ... g15 and we
|
||||
@@ -881,6 +894,27 @@ def _initialize_model_parallel(
|
||||
group_name="dp",
|
||||
)
|
||||
|
||||
if use_hsdp:
|
||||
if fully_shard_degree <= 0:
|
||||
raise ValueError(f"fully_shard_degree must be positive, got {fully_shard_degree}")
|
||||
if world_size % fully_shard_degree != 0:
|
||||
raise ValueError(
|
||||
f"WORLD size ({world_size}) must be divisible by fully_shard_degree ({fully_shard_degree})"
|
||||
)
|
||||
|
||||
assert _FS is None, "fully shard group is already initialized"
|
||||
# HSDP builds its mesh from arange(world_size).reshape(replicate, shard),
|
||||
# so each consecutive rank run is one fully-sharded group.
|
||||
fs_group_ranks = [
|
||||
list(range(start, start + fully_shard_degree)) for start in range(0, world_size, fully_shard_degree)
|
||||
]
|
||||
_FS = init_model_parallel_group(
|
||||
group_ranks=fs_group_ranks,
|
||||
local_rank=get_world_group().local_rank,
|
||||
backend=backend,
|
||||
parallel_mode="fully_shard",
|
||||
)
|
||||
|
||||
global _EXPERT_PARALLEL_GROUP_RANKS
|
||||
_EXPERT_PARALLEL_GROUP_RANKS = get_rank_groups("tp-sp-cfg-dp")
|
||||
if use_moe_parallel_mapping:
|
||||
@@ -902,6 +936,7 @@ def initialize_model_parallel(
|
||||
allgather_degree: int = 1,
|
||||
tensor_parallel_size: int = 1,
|
||||
pipeline_parallel_size: int = 1,
|
||||
fully_shard_degree: int = 1,
|
||||
enable_expert_parallel: bool = False,
|
||||
use_hsdp: bool = False,
|
||||
backend: str | None = None,
|
||||
@@ -916,6 +951,7 @@ def initialize_model_parallel(
|
||||
"cfg": _CFG,
|
||||
"sp": _SP,
|
||||
"pp": _PP,
|
||||
"fs": _FS,
|
||||
"vllm_tp": vllm_parallel_state._TP,
|
||||
"vllm_dp": vllm_parallel_state._DP,
|
||||
"vllm_pp": vllm_parallel_state._PP,
|
||||
@@ -937,6 +973,7 @@ def initialize_model_parallel(
|
||||
allgather_degree=allgather_degree,
|
||||
tensor_parallel_size=tensor_parallel_size,
|
||||
pipeline_parallel_size=pipeline_parallel_size,
|
||||
fully_shard_degree=fully_shard_degree,
|
||||
enable_expert_parallel=enable_expert_parallel,
|
||||
use_hsdp=use_hsdp,
|
||||
backend=backend,
|
||||
@@ -948,7 +985,7 @@ def initialize_model_parallel(
|
||||
|
||||
def destroy_model_parallel():
|
||||
"""Set the groups to none and destroy them."""
|
||||
global _DP, _CFG, _SP, _PP, _EXPERT_PARALLEL_GROUP_RANKS
|
||||
global _DP, _CFG, _SP, _PP, _FS, _EXPERT_PARALLEL_GROUP_RANKS
|
||||
|
||||
if vllm_parallel_state._DP and vllm_parallel_state._DP is not _DP:
|
||||
vllm_parallel_state._DP.destroy()
|
||||
@@ -958,6 +995,10 @@ def destroy_model_parallel():
|
||||
_DP.destroy()
|
||||
_DP = None
|
||||
|
||||
if _FS:
|
||||
_FS.destroy()
|
||||
_FS = None
|
||||
|
||||
if _CFG:
|
||||
_CFG.destroy()
|
||||
_CFG = None
|
||||
|
||||
Reference in new issue
Block a user