[Core] Identify HSDP replica ranks in worker titles
Signed-off-by: hsliuustc0106 <hsliuustc0106@users.noreply.github.com>
This commit is contained in:
1 parent
65ac6794f9
commit
5c8e0e85d8
3 files changed
+71
-11
No files matched your search
@@ -61,6 +61,7 @@ _PP: PipelineGroupCoordinator | None = None
|
||||
_CFG: GroupCoordinator | None = None
|
||||
_DP: GroupCoordinator | None = None
|
||||
_FS: GroupCoordinator | None = None # Fully Sharded (HSDP shard dimension)
|
||||
_HSDP_REPLICATE: GroupCoordinator | None = None # HSDP replica dimension
|
||||
|
||||
# Rank-layout metadata for expert parallelism. This is not a process group;
|
||||
# it is reused by platform-specific runtimes that must build companion groups
|
||||
@@ -358,6 +359,11 @@ def get_fs_group() -> GroupCoordinator:
|
||||
return _FS
|
||||
|
||||
|
||||
def get_hsdp_replicate_group() -> GroupCoordinator:
|
||||
assert _HSDP_REPLICATE is not None, "HSDP replicate group is not initialized"
|
||||
return _HSDP_REPLICATE
|
||||
|
||||
|
||||
def is_dp_last_group():
|
||||
"""Return True if in the last data parallel group, False otherwise."""
|
||||
return (
|
||||
@@ -693,7 +699,7 @@ def _initialize_model_parallel(
|
||||
use_hsdp: bool = False,
|
||||
backend: str | None = None,
|
||||
) -> None:
|
||||
global _FS
|
||||
global _FS, _HSDP_REPLICATE
|
||||
|
||||
if backend is None:
|
||||
backend = current_omni_platform.dist_backend
|
||||
@@ -914,6 +920,18 @@ def _initialize_model_parallel(
|
||||
backend=backend,
|
||||
parallel_mode="fully_shard",
|
||||
)
|
||||
if world_size > fully_shard_degree:
|
||||
# The HSDP mesh's columns are replica groups: each column contains
|
||||
# the same shard position across all replica rows.
|
||||
hsdp_replicate_group_ranks = [
|
||||
list(range(offset, world_size, fully_shard_degree)) for offset in range(fully_shard_degree)
|
||||
]
|
||||
_HSDP_REPLICATE = init_model_parallel_group(
|
||||
group_ranks=hsdp_replicate_group_ranks,
|
||||
local_rank=get_world_group().local_rank,
|
||||
backend=backend,
|
||||
parallel_mode="fully_shard",
|
||||
)
|
||||
|
||||
global _EXPERT_PARALLEL_GROUP_RANKS
|
||||
_EXPERT_PARALLEL_GROUP_RANKS = get_rank_groups("tp-sp-cfg-dp")
|
||||
@@ -952,6 +970,7 @@ def initialize_model_parallel(
|
||||
"sp": _SP,
|
||||
"pp": _PP,
|
||||
"fs": _FS,
|
||||
"hsdp_replicate": _HSDP_REPLICATE,
|
||||
"vllm_tp": vllm_parallel_state._TP,
|
||||
"vllm_dp": vllm_parallel_state._DP,
|
||||
"vllm_pp": vllm_parallel_state._PP,
|
||||
@@ -985,7 +1004,7 @@ def initialize_model_parallel(
|
||||
|
||||
def destroy_model_parallel():
|
||||
"""Set the groups to none and destroy them."""
|
||||
global _DP, _CFG, _SP, _PP, _FS, _EXPERT_PARALLEL_GROUP_RANKS
|
||||
global _DP, _CFG, _SP, _PP, _FS, _HSDP_REPLICATE, _EXPERT_PARALLEL_GROUP_RANKS
|
||||
|
||||
if vllm_parallel_state._DP and vllm_parallel_state._DP is not _DP:
|
||||
vllm_parallel_state._DP.destroy()
|
||||
@@ -999,6 +1018,10 @@ def destroy_model_parallel():
|
||||
_FS.destroy()
|
||||
_FS = None
|
||||
|
||||
if _HSDP_REPLICATE:
|
||||
_HSDP_REPLICATE.destroy()
|
||||
_HSDP_REPLICATE = None
|
||||
|
||||
if _CFG:
|
||||
_CFG.destroy()
|
||||
_CFG = None
|
||||
|
||||
@@ -51,6 +51,7 @@ from vllm_omni.diffusion.distributed.parallel_state import (
|
||||
get_cfg_group,
|
||||
get_dp_group,
|
||||
get_fs_group,
|
||||
get_hsdp_replicate_group,
|
||||
get_pp_group,
|
||||
get_sp_group,
|
||||
init_distributed_environment,
|
||||
@@ -126,7 +127,11 @@ def _run_and_gather_rank_values(operation: str, func: Callable[[], Any]) -> list
|
||||
return [result for _, result in rank_results]
|
||||
|
||||
|
||||
def _setup_diffusion_worker_proc_title_and_log_prefix(enable_ep: bool, use_hsdp: bool) -> None:
|
||||
def _setup_diffusion_worker_proc_title_and_log_prefix(
|
||||
enable_ep: bool,
|
||||
use_hsdp: bool,
|
||||
hsdp_replicate_size: int = 1,
|
||||
) -> None:
|
||||
"""Set the worker process title and log prefix from initialized groups."""
|
||||
process_name = "DiffusionWorker"
|
||||
if model_parallel_is_initialized():
|
||||
@@ -150,6 +155,10 @@ def _setup_diffusion_worker_proc_title_and_log_prefix(enable_ep: bool, use_hsdp:
|
||||
fs_group = get_fs_group()
|
||||
if fs_group.world_size > 1:
|
||||
process_name += f"_FS{fs_group.rank_in_group}"
|
||||
if hsdp_replicate_size > 1:
|
||||
replicate_group = get_hsdp_replicate_group()
|
||||
if replicate_group.world_size > 1:
|
||||
process_name += f"_RP{replicate_group.rank_in_group}"
|
||||
if enable_ep:
|
||||
ep_group = get_ep_group()
|
||||
if ep_group.world_size > 1:
|
||||
@@ -342,6 +351,7 @@ class DiffusionWorker:
|
||||
_setup_diffusion_worker_proc_title_and_log_prefix(
|
||||
enable_ep=parallel_config.enable_expert_parallel,
|
||||
use_hsdp=parallel_config.use_hsdp,
|
||||
hsdp_replicate_size=parallel_config.hsdp_replicate_size,
|
||||
)
|
||||
if (
|
||||
getattr(self.od_config, "diffusion_kv_mode", DiffusionKVCacheMode.DENSE_LEGACY)
|
||||
@@ -1448,6 +1458,7 @@ class WorkerProc:
|
||||
_setup_diffusion_worker_proc_title_and_log_prefix(
|
||||
enable_ep=od_config.parallel_config.enable_expert_parallel,
|
||||
use_hsdp=od_config.parallel_config.use_hsdp,
|
||||
hsdp_replicate_size=od_config.parallel_config.hsdp_replicate_size,
|
||||
)
|
||||
|
||||
load_omni_general_plugins()
|
||||
|
||||
Reference in new issue
Block a user