[FEAT] support multi-stage deployment (#2396)

Signed-off-by: ZhengWG <zwg0606@gmail.com>
Signed-off-by: Zheng Wengang <zwg0606@gmail.com>
Signed-off-by: Peiqi Yin <60515999+yinpeiqi@users.noreply.github.com>
Signed-off-by: yinpe <11810305@mail.sustech.edu.cn>
Signed-off-by: yinpeiqi <yinpeiqi809@gmail.com>
Co-authored-by: Peiqi Yin <60515999+yinpeiqi@users.noreply.github.com>
Co-authored-by: yinpe <11810305@mail.sustech.edu.cn>
Co-authored-by: yinpeiqi <yinpeiqi809@gmail.com>
Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
Co-authored-by: Gao Han <hgaoaf@connect.ust.hk>
Co-authored-by: Chenguang Zheng <645327136@qq.com>
This commit is contained in:
authored and GitHub committed 2026-05-06 19:33:18 +08:00
1 parent e969d2e99f
commit 1e5f288a91
55 files changed
+4684 -3407

No files matched your search

@@ -35,6 +35,7 @@ class InlineStageDiffusionClient:
"""Runs DiffusionEngine in a thread executor inside the Orchestrator."""
stage_type: str = "diffusion"
replica_id: int = 0
def __init__(
self,
@@ -46,9 +47,11 @@ class InlineStageDiffusionClient:
self.model = model
self.od_config = od_config
self.stage_id = metadata.stage_id
self.replica_id = metadata.replica_id
self.final_output = metadata.final_output
self.final_output_type = metadata.final_output_type
self.default_sampling_params = metadata.default_sampling_params
self.requires_multimodal_data = metadata.requires_multimodal_data
self.custom_process_input_func = metadata.custom_process_input_func
self.engine_input_source = metadata.engine_input_source
self.batch_size = batch_size
@@ -62,8 +65,9 @@ class InlineStageDiffusionClient:
self._shutting_down = False
logger.info(
"[InlineStageDiffusionClient] Stage-%s initialized inline (batch_size=%d)",
"[InlineStageDiffusionClient] stage-%s [rep-%s] initialized inline (batch_size=%d)",
self.stage_id,
self.replica_id,
self.batch_size,
)
@@ -82,6 +86,12 @@ class InlineStageDiffusionClient:
sampling_params: OmniDiffusionSamplingParams,
kv_sender_info: dict[int, dict[str, Any]] | None = None,
) -> None:
logger.info(
"[InlineStageDiffusionClient] stage-%s [rep-%s] add request: %s",
self.stage_id,
self.replica_id,
request_id,
)
task = asyncio.create_task(
self._dispatch_request(
request_id,
@@ -134,6 +144,13 @@ class InlineStageDiffusionClient:
sampling_params: OmniDiffusionSamplingParams,
kv_sender_info: dict[int, dict[str, Any]] | None = None,
) -> None:
logger.info(
"[InlineStageDiffusionClient] stage-%s [rep-%s] add batch request: %s (%d prompts)",
self.stage_id,
self.replica_id,
request_id,
len(prompts),
)
task = asyncio.create_task(
self._dispatch_batch(
request_id,
@@ -252,7 +269,7 @@ class InlineStageDiffusionClient:
is_start = args[0] if args else True
profile_prefix = args[1] if len(args) > 1 else None
if is_start and profile_prefix is None:
profile_prefix = f"stage_{self.stage_id}_diffusion_{int(time.time())}"
profile_prefix = f"stage_{self.stage_id}_rep_{self.replica_id}_diffusion_{int(time.time())}"
return await loop.run_in_executor(
self._executor,
self._engine.profile,
+29 -8
View File
@@ -67,6 +67,7 @@ class StageDiffusionClient:
"""
stage_type: str = "diffusion"
replica_id: int = 0
def __init__(
self,
@@ -112,11 +113,13 @@ class StageDiffusionClient:
batch_size: int,
) -> None:
self.stage_id = metadata.stage_id
self.replica_id = metadata.replica_id
self.final_output = metadata.final_output
self.final_output_type = metadata.final_output_type
self.default_sampling_params = metadata.default_sampling_params
self.custom_process_input_func = metadata.custom_process_input_func
self.engine_input_source = metadata.engine_input_source
self.requires_multimodal_data = getattr(metadata, "requires_multimodal_data", False)
self.custom_process_input_func = getattr(metadata, "custom_process_input_func", None)
self.engine_input_source = getattr(metadata, "engine_input_source", [])
self._proc = proc
self._owns_process = proc is not None
@@ -142,8 +145,9 @@ class StageDiffusionClient:
self._start_proc_monitor()
logger.info(
"[StageDiffusionClient] Stage-%s initialized (owns_process=%s, batch_size=%d)",
"[StageDiffusionClient] stage-%s [rep-%s] initialized (owns_process=%s, batch_size=%d)",
self.stage_id,
self.replica_id,
self._owns_process,
batch_size,
)
@@ -172,8 +176,9 @@ class StageDiffusionClient:
return
client._engine_dead = True
logger.error(
"[StageDiffusionClient] Stage-%s StageDiffusionProc died unexpectedly (exit code %s).",
"[StageDiffusionClient] stage-%s [rep-%s] StageDiffusionProc died unexpectedly (exit code %s).",
client.stage_id,
client.replica_id,
proc.exitcode,
)
@@ -195,8 +200,9 @@ class StageDiffusionClient:
if raw == StageDiffusionProc.DIFFUSION_PROC_DEAD:
self._engine_dead = True
logger.error(
"[StageDiffusionClient] Stage-%s received DIFFUSION_PROC_DEAD sentinel from subprocess.",
"[StageDiffusionClient] stage-%s [rep-%s] received DIFFUSION_PROC_DEAD sentinel from subprocess.",
self.stage_id,
self.replica_id,
)
break
@@ -212,8 +218,9 @@ class StageDiffusionClient:
rpc_id = msg.get("rpc_id")
error_msg = msg.get("error")
logger.error(
"[StageDiffusionClient] Stage-%s subprocess error for %s: %s",
"[StageDiffusionClient] stage-%s [rep-%s] subprocess error for %s: %s",
self.stage_id,
self.replica_id,
rpc_id or req_id,
error_msg,
)
@@ -292,6 +299,12 @@ class StageDiffusionClient:
) -> None:
if self._engine_dead:
raise EngineDeadError()
logger.info(
"[StageDiffusionClient] stage-%s [rep-%s] add request: %s",
self.stage_id,
self.replica_id,
request_id,
)
self._request_socket.send(
self._encoder.encode(
{
@@ -321,6 +334,13 @@ class StageDiffusionClient:
"""
if self._engine_dead:
raise EngineDeadError()
logger.info(
"[StageDiffusionClient] stage-%s [rep-%s] add batch request: %s (%d prompts)",
self.stage_id,
self.replica_id,
request_id,
len(prompts),
)
task = asyncio.create_task(
self._run_batch(
request_id,
@@ -353,8 +373,9 @@ class StageDiffusionClient:
)
except Exception as e:
logger.exception(
"[StageDiffusionClient] Stage-%s batch req=%s failed: %s",
"[StageDiffusionClient] stage-%s [rep-%s] batch req=%s failed: %s",
self.stage_id,
self.replica_id,
request_id,
e,
)
@@ -414,7 +435,7 @@ class StageDiffusionClient:
is_start = args_list[0] if args_list else True
profile_prefix = args_list[1] if len(args_list) > 1 else None
if is_start and profile_prefix is None:
profile_prefix = f"stage_{self.stage_id}_diffusion_{int(time.time())}"
profile_prefix = f"stage_{self.stage_id}_rep_{self.replica_id}_diffusion_{int(time.time())}"
if len(args_list) > 1:
args_list[1] = profile_prefix
else: