[FEAT] support multi-stage deployment (#2396)
Signed-off-by: ZhengWG <zwg0606@gmail.com> Signed-off-by: Zheng Wengang <zwg0606@gmail.com> Signed-off-by: Peiqi Yin <60515999+yinpeiqi@users.noreply.github.com> Signed-off-by: yinpe <11810305@mail.sustech.edu.cn> Signed-off-by: yinpeiqi <yinpeiqi809@gmail.com> Co-authored-by: Peiqi Yin <60515999+yinpeiqi@users.noreply.github.com> Co-authored-by: yinpe <11810305@mail.sustech.edu.cn> Co-authored-by: yinpeiqi <yinpeiqi809@gmail.com> Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com> Co-authored-by: Gao Han <hgaoaf@connect.ust.hk> Co-authored-by: Chenguang Zheng <645327136@qq.com>
This commit is contained in:
55 files changed
+4684
-3407
No files matched your search
@@ -187,6 +187,12 @@ class OmniServeCommand(CLISubcommand):
|
||||
default=None,
|
||||
help="Select and launch a single stage by stage_id.",
|
||||
)
|
||||
omni_config_group.add_argument(
|
||||
"--replica-id",
|
||||
type=int,
|
||||
default=0,
|
||||
help="Replica id to register when launching a single headless stage.",
|
||||
)
|
||||
omni_config_group.add_argument(
|
||||
"--stage-init-timeout",
|
||||
type=int,
|
||||
@@ -529,14 +535,18 @@ def run_headless(args: argparse.Namespace) -> None:
|
||||
|
||||
model = args.model
|
||||
stage_id: int | None = args.stage_id
|
||||
replica_id: int = args.replica_id
|
||||
omni_master_address: str | None = args.omni_master_address
|
||||
omni_master_port: int | None = args.omni_master_port
|
||||
|
||||
if stage_id is None:
|
||||
raise ValueError("--stage-id is required in headless mode")
|
||||
if replica_id < 0:
|
||||
raise ValueError("--replica-id must be >= 0 in headless mode")
|
||||
if omni_master_address is None or omni_master_port is None:
|
||||
raise ValueError("--omni-master-address and --omni-master-port are required in headless mode")
|
||||
if getattr(args, "api_server_count", 0) and args.api_server_count > 1:
|
||||
api_server_count = args.api_server_count or 0
|
||||
if api_server_count > 1:
|
||||
raise ValueError("api_server_count can't be set in headless mode")
|
||||
if args.worker_backend != "multi_process":
|
||||
raise ValueError("headless mode requires worker_backend=multi_process")
|
||||
@@ -552,20 +562,19 @@ def run_headless(args: argparse.Namespace) -> None:
|
||||
# Locate the stage config that matches stage_id.
|
||||
stage_cfg = None
|
||||
for cfg in stage_configs:
|
||||
if getattr(cfg, "stage_id", None) == stage_id:
|
||||
if cfg.stage_id == stage_id:
|
||||
stage_cfg = cfg
|
||||
break
|
||||
if stage_cfg is None:
|
||||
raise ValueError(
|
||||
f"No stage config found for stage_id={stage_id}. "
|
||||
f"Available stage ids: {[getattr(c, 'stage_id', None) for c in stage_configs]}"
|
||||
f"No stage config found for stage_id={stage_id}. Available stage ids: {[c.stage_id for c in stage_configs]}"
|
||||
)
|
||||
|
||||
prepare_engine_environment()
|
||||
omni_transfer_config = load_omni_transfer_config_for_model(model, config_path)
|
||||
omni_conn_cfg, omni_from, omni_to = resolve_omni_kv_config_for_stage(omni_transfer_config, stage_id)
|
||||
|
||||
if getattr(stage_cfg, "stage_type", "llm") == "diffusion":
|
||||
if stage_cfg.stage_type == "diffusion":
|
||||
metadata = extract_stage_metadata(stage_cfg)
|
||||
if omni_conn_cfg:
|
||||
inject_omni_kv_config(stage_cfg, omni_conn_cfg, omni_from, omni_to)
|
||||
@@ -573,8 +582,9 @@ def run_headless(args: argparse.Namespace) -> None:
|
||||
od_config = build_diffusion_config(model, stage_cfg, metadata)
|
||||
|
||||
logger.info(
|
||||
"[Headless] Launching diffusion stage %d via OmniMasterServer at %s:%d",
|
||||
"[Headless] Launching diffusion stage %d replica %d via OmniMasterServer at %s:%d",
|
||||
stage_id,
|
||||
replica_id,
|
||||
omni_master_address,
|
||||
omni_master_port,
|
||||
)
|
||||
@@ -587,6 +597,7 @@ def run_headless(args: argparse.Namespace) -> None:
|
||||
omni_stage_id=stage_id,
|
||||
omni_stage_config=stage_cfg,
|
||||
return_addresses=True,
|
||||
replica_id=replica_id,
|
||||
)
|
||||
proc, _, _, _ = spawn_diffusion_proc(
|
||||
model,
|
||||
@@ -595,13 +606,13 @@ def run_headless(args: argparse.Namespace) -> None:
|
||||
request_address=request_address,
|
||||
response_address=response_address,
|
||||
)
|
||||
complete_diffusion_handshake(proc, handshake_address)
|
||||
complete_diffusion_handshake(proc, handshake_address, args.stage_init_timeout)
|
||||
proc.join()
|
||||
if proc.exitcode not in (None, 0):
|
||||
raise RuntimeError(f"Diffusion stage {stage_id} exited with code {proc.exitcode}")
|
||||
raise RuntimeError(f"Diffusion stage {stage_id} replica {replica_id} exited with code {proc.exitcode}")
|
||||
return
|
||||
finally:
|
||||
logger.info("[Headless] Shutting down stage %d.", stage_id)
|
||||
logger.info("[Headless] Shutting down stage %d replica %d.", stage_id, replica_id)
|
||||
if proc is not None and proc.is_alive():
|
||||
terminate_alive_proc(proc)
|
||||
|
||||
@@ -677,15 +688,17 @@ def run_headless(args: argparse.Namespace) -> None:
|
||||
enable_wave_coordination=vllm_config.model_config.is_moe,
|
||||
)
|
||||
logger.info(
|
||||
"[Headless] Started DP Coordinator process for stage %d (PID: %d)",
|
||||
"[Headless] Started DP Coordinator process for stage %d replica %d (PID: %d)",
|
||||
stage_id,
|
||||
replica_id,
|
||||
coordinator.proc.pid,
|
||||
)
|
||||
|
||||
logger.info(
|
||||
"[Headless] Launching %d engine core(s) for stage %d via OmniMasterServer at %s:%d",
|
||||
"[Headless] Launching %d engine core(s) for stage %d replica %d via OmniMasterServer at %s:%d",
|
||||
local_engine_count,
|
||||
stage_id,
|
||||
replica_id,
|
||||
omni_master_address,
|
||||
omni_master_port,
|
||||
)
|
||||
@@ -700,11 +713,12 @@ def run_headless(args: argparse.Namespace) -> None:
|
||||
omni_stage_id=stage_id,
|
||||
omni_stage_config=stage_cfg,
|
||||
coordinator=coordinator,
|
||||
replica_id=replica_id,
|
||||
)
|
||||
|
||||
engine_manager = None
|
||||
log_stats = bool(getattr(args, "log_stats", False))
|
||||
if getattr(args, "disable_log_stats", False):
|
||||
log_stats = bool(args.log_stats)
|
||||
if args.disable_log_stats:
|
||||
log_stats = False
|
||||
|
||||
try:
|
||||
|
||||
Reference in new issue
Block a user