[Core] Restore topology-aware diffusion worker titles (#6722)
Signed-off-by: hsliuustc0106 <hsliuustc0106@users.noreply.github.com> Co-authored-by: hsliuustc0106 <hsliuustc0106@users.noreply.github.com>
This commit is contained in:
1 parent
6b9ce05feb
commit
8bfad23f98
4 files changed
+422
-9
No files matched your search
@@ -25,10 +25,12 @@ import torch.distributed as dist
|
||||
import zmq
|
||||
from vllm.config import VllmConfig, set_current_vllm_config
|
||||
from vllm.distributed.device_communicators.shm_broadcast import MessageQueue
|
||||
from vllm.distributed.parallel_state import get_ep_group, get_tp_group
|
||||
from vllm.logger import init_logger
|
||||
from vllm.profiler.wrapper import CudaProfilerWrapper, WorkerProfiler
|
||||
from vllm.utils.import_utils import resolve_obj_by_qualname
|
||||
from vllm.utils.mem_utils import GiB_bytes, MemorySnapshot, format_gib, memory_profiling
|
||||
from vllm.utils.system_utils import decorate_logs, set_process_title
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig, KVCacheSpec
|
||||
from vllm.v1.worker.utils import request_memory
|
||||
from vllm.v1.worker.workspace import init_workspace_manager
|
||||
@@ -46,8 +48,15 @@ from vllm_omni.diffusion.diffusion_kv.config import DiffusionKVCacheMode
|
||||
from vllm_omni.diffusion.diffusion_kv.metadata import DiffusionKVMetadata
|
||||
from vllm_omni.diffusion.distributed.parallel_state import (
|
||||
destroy_distributed_env,
|
||||
get_cfg_group,
|
||||
get_dp_group,
|
||||
get_fs_group,
|
||||
get_hsdp_replicate_group,
|
||||
get_pp_group,
|
||||
get_sp_group,
|
||||
init_distributed_environment,
|
||||
initialize_model_parallel,
|
||||
model_parallel_is_initialized,
|
||||
)
|
||||
from vllm_omni.diffusion.forward_context import set_forward_context
|
||||
from vllm_omni.diffusion.ipc import DIFFUSION_RPC_RESULT_ENVELOPE, pack_diffusion_output_shm
|
||||
@@ -118,6 +127,47 @@ def _run_and_gather_rank_values(operation: str, func: Callable[[], Any]) -> list
|
||||
return [result for _, result in rank_results]
|
||||
|
||||
|
||||
def _setup_diffusion_worker_proc_title_and_log_prefix(
|
||||
enable_ep: bool,
|
||||
use_hsdp: bool,
|
||||
hsdp_replicate_size: int = 1,
|
||||
) -> None:
|
||||
"""Set the worker process title and log prefix from initialized groups."""
|
||||
process_name = "DiffusionWorker"
|
||||
if model_parallel_is_initialized():
|
||||
dp_group = get_dp_group()
|
||||
pp_group = get_pp_group()
|
||||
sp_group = get_sp_group()
|
||||
cfg_group = get_cfg_group()
|
||||
tp_group = get_tp_group()
|
||||
|
||||
if dp_group.world_size > 1:
|
||||
process_name += f"_DP{dp_group.rank_in_group}"
|
||||
if pp_group.world_size > 1:
|
||||
process_name += f"_PP{pp_group.rank_in_group}"
|
||||
if sp_group.world_size > 1:
|
||||
process_name += f"_SP{sp_group.rank_in_group}"
|
||||
if cfg_group.world_size > 1:
|
||||
process_name += f"_CFG{cfg_group.rank_in_group}"
|
||||
if tp_group.world_size > 1:
|
||||
process_name += f"_TP{tp_group.rank_in_group}"
|
||||
if use_hsdp:
|
||||
fs_group = get_fs_group()
|
||||
if fs_group.world_size > 1:
|
||||
process_name += f"_FS{fs_group.rank_in_group}"
|
||||
if hsdp_replicate_size > 1:
|
||||
replicate_group = get_hsdp_replicate_group()
|
||||
if replicate_group.world_size > 1:
|
||||
process_name += f"_RP{replicate_group.rank_in_group}"
|
||||
if enable_ep:
|
||||
ep_group = get_ep_group()
|
||||
if ep_group.world_size > 1:
|
||||
process_name += f"_EP{ep_group.rank_in_group}"
|
||||
|
||||
set_process_title(name=process_name, prefix="vLLM-Omni")
|
||||
decorate_logs(process_name)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _force_cutlass_fp8_linear_kernel(quant_config: object | None) -> Iterator[None]:
|
||||
import vllm.model_executor.layers.quantization.modelopt as vllm_modelopt
|
||||
@@ -294,9 +344,15 @@ class DiffusionWorker:
|
||||
allgather_degree=parallel_config.allgather_degree,
|
||||
tensor_parallel_size=parallel_config.tensor_parallel_size,
|
||||
pipeline_parallel_size=parallel_config.pipeline_parallel_size,
|
||||
fully_shard_degree=parallel_config.hsdp_shard_size if parallel_config.use_hsdp else 1,
|
||||
enable_expert_parallel=parallel_config.enable_expert_parallel,
|
||||
use_hsdp=parallel_config.use_hsdp,
|
||||
)
|
||||
_setup_diffusion_worker_proc_title_and_log_prefix(
|
||||
enable_ep=parallel_config.enable_expert_parallel,
|
||||
use_hsdp=parallel_config.use_hsdp,
|
||||
hsdp_replicate_size=parallel_config.hsdp_replicate_size,
|
||||
)
|
||||
if (
|
||||
getattr(self.od_config, "diffusion_kv_mode", DiffusionKVCacheMode.DENSE_LEGACY)
|
||||
is DiffusionKVCacheMode.PAGED_SCHEDULER
|
||||
@@ -1399,13 +1455,11 @@ class WorkerProc:
|
||||
|
||||
set_death_signal(signal.SIGTERM)
|
||||
|
||||
# Set process title for visibility in nvidia-smi / htop (optional, non-fatal)
|
||||
try:
|
||||
import setproctitle
|
||||
|
||||
setproctitle.setproctitle(f"vLLM-Omni::DiffusionWorker-{rank}")
|
||||
except ImportError:
|
||||
pass # setproctitle not installed, skip process title setting
|
||||
_setup_diffusion_worker_proc_title_and_log_prefix(
|
||||
enable_ep=od_config.parallel_config.enable_expert_parallel,
|
||||
use_hsdp=od_config.parallel_config.use_hsdp,
|
||||
hsdp_replicate_size=od_config.parallel_config.hsdp_replicate_size,
|
||||
)
|
||||
|
||||
load_omni_general_plugins()
|
||||
worker_proc = None
|
||||
|
||||
Reference in new issue
Block a user