[FEAT] support multi-stage deployment (#2396)

Signed-off-by: ZhengWG <zwg0606@gmail.com>
Signed-off-by: Zheng Wengang <zwg0606@gmail.com>
Signed-off-by: Peiqi Yin <60515999+yinpeiqi@users.noreply.github.com>
Signed-off-by: yinpe <11810305@mail.sustech.edu.cn>
Signed-off-by: yinpeiqi <yinpeiqi809@gmail.com>
Co-authored-by: Peiqi Yin <60515999+yinpeiqi@users.noreply.github.com>
Co-authored-by: yinpe <11810305@mail.sustech.edu.cn>
Co-authored-by: yinpeiqi <yinpeiqi809@gmail.com>
Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
Co-authored-by: Gao Han <hgaoaf@connect.ust.hk>
Co-authored-by: Chenguang Zheng <645327136@qq.com>
This commit is contained in:
authored and GitHub committed 2026-05-06 19:33:18 +08:00
1 parent e969d2e99f
commit 1e5f288a91
55 files changed
+4684 -3407

No files matched your search

+27 -13
View File
@@ -187,6 +187,12 @@ class OmniServeCommand(CLISubcommand):
default=None,
help="Select and launch a single stage by stage_id.",
)
omni_config_group.add_argument(
"--replica-id",
type=int,
default=0,
help="Replica id to register when launching a single headless stage.",
)
omni_config_group.add_argument(
"--stage-init-timeout",
type=int,
@@ -529,14 +535,18 @@ def run_headless(args: argparse.Namespace) -> None:
model = args.model
stage_id: int | None = args.stage_id
replica_id: int = args.replica_id
omni_master_address: str | None = args.omni_master_address
omni_master_port: int | None = args.omni_master_port
if stage_id is None:
raise ValueError("--stage-id is required in headless mode")
if replica_id < 0:
raise ValueError("--replica-id must be >= 0 in headless mode")
if omni_master_address is None or omni_master_port is None:
raise ValueError("--omni-master-address and --omni-master-port are required in headless mode")
if getattr(args, "api_server_count", 0) and args.api_server_count > 1:
api_server_count = args.api_server_count or 0
if api_server_count > 1:
raise ValueError("api_server_count can't be set in headless mode")
if args.worker_backend != "multi_process":
raise ValueError("headless mode requires worker_backend=multi_process")
@@ -552,20 +562,19 @@ def run_headless(args: argparse.Namespace) -> None:
# Locate the stage config that matches stage_id.
stage_cfg = None
for cfg in stage_configs:
if getattr(cfg, "stage_id", None) == stage_id:
if cfg.stage_id == stage_id:
stage_cfg = cfg
break
if stage_cfg is None:
raise ValueError(
f"No stage config found for stage_id={stage_id}. "
f"Available stage ids: {[getattr(c, 'stage_id', None) for c in stage_configs]}"
f"No stage config found for stage_id={stage_id}. Available stage ids: {[c.stage_id for c in stage_configs]}"
)
prepare_engine_environment()
omni_transfer_config = load_omni_transfer_config_for_model(model, config_path)
omni_conn_cfg, omni_from, omni_to = resolve_omni_kv_config_for_stage(omni_transfer_config, stage_id)
if getattr(stage_cfg, "stage_type", "llm") == "diffusion":
if stage_cfg.stage_type == "diffusion":
metadata = extract_stage_metadata(stage_cfg)
if omni_conn_cfg:
inject_omni_kv_config(stage_cfg, omni_conn_cfg, omni_from, omni_to)
@@ -573,8 +582,9 @@ def run_headless(args: argparse.Namespace) -> None:
od_config = build_diffusion_config(model, stage_cfg, metadata)
logger.info(
"[Headless] Launching diffusion stage %d via OmniMasterServer at %s:%d",
"[Headless] Launching diffusion stage %d replica %d via OmniMasterServer at %s:%d",
stage_id,
replica_id,
omni_master_address,
omni_master_port,
)
@@ -587,6 +597,7 @@ def run_headless(args: argparse.Namespace) -> None:
omni_stage_id=stage_id,
omni_stage_config=stage_cfg,
return_addresses=True,
replica_id=replica_id,
)
proc, _, _, _ = spawn_diffusion_proc(
model,
@@ -595,13 +606,13 @@ def run_headless(args: argparse.Namespace) -> None:
request_address=request_address,
response_address=response_address,
)
complete_diffusion_handshake(proc, handshake_address)
complete_diffusion_handshake(proc, handshake_address, args.stage_init_timeout)
proc.join()
if proc.exitcode not in (None, 0):
raise RuntimeError(f"Diffusion stage {stage_id} exited with code {proc.exitcode}")
raise RuntimeError(f"Diffusion stage {stage_id} replica {replica_id} exited with code {proc.exitcode}")
return
finally:
logger.info("[Headless] Shutting down stage %d.", stage_id)
logger.info("[Headless] Shutting down stage %d replica %d.", stage_id, replica_id)
if proc is not None and proc.is_alive():
terminate_alive_proc(proc)
@@ -677,15 +688,17 @@ def run_headless(args: argparse.Namespace) -> None:
enable_wave_coordination=vllm_config.model_config.is_moe,
)
logger.info(
"[Headless] Started DP Coordinator process for stage %d (PID: %d)",
"[Headless] Started DP Coordinator process for stage %d replica %d (PID: %d)",
stage_id,
replica_id,
coordinator.proc.pid,
)
logger.info(
"[Headless] Launching %d engine core(s) for stage %d via OmniMasterServer at %s:%d",
"[Headless] Launching %d engine core(s) for stage %d replica %d via OmniMasterServer at %s:%d",
local_engine_count,
stage_id,
replica_id,
omni_master_address,
omni_master_port,
)
@@ -700,11 +713,12 @@ def run_headless(args: argparse.Namespace) -> None:
omni_stage_id=stage_id,
omni_stage_config=stage_cfg,
coordinator=coordinator,
replica_id=replica_id,
)
engine_manager = None
log_stats = bool(getattr(args, "log_stats", False))
if getattr(args, "disable_log_stats", False):
log_stats = bool(args.log_stats)
if args.disable_log_stats:
log_stats = False
try: