[FEAT] support multi-stage deployment (#2396)
Signed-off-by: ZhengWG <zwg0606@gmail.com> Signed-off-by: Zheng Wengang <zwg0606@gmail.com> Signed-off-by: Peiqi Yin <60515999+yinpeiqi@users.noreply.github.com> Signed-off-by: yinpe <11810305@mail.sustech.edu.cn> Signed-off-by: yinpeiqi <yinpeiqi809@gmail.com> Co-authored-by: Peiqi Yin <60515999+yinpeiqi@users.noreply.github.com> Co-authored-by: yinpe <11810305@mail.sustech.edu.cn> Co-authored-by: yinpeiqi <yinpeiqi809@gmail.com> Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com> Co-authored-by: Gao Han <hgaoaf@connect.ust.hk> Co-authored-by: Chenguang Zheng <645327136@qq.com>
This commit is contained in:
55 files changed
+4684
-3407
No files matched your search
@@ -35,6 +35,7 @@ class InlineStageDiffusionClient:
|
||||
"""Runs DiffusionEngine in a thread executor inside the Orchestrator."""
|
||||
|
||||
stage_type: str = "diffusion"
|
||||
replica_id: int = 0
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -46,9 +47,11 @@ class InlineStageDiffusionClient:
|
||||
self.model = model
|
||||
self.od_config = od_config
|
||||
self.stage_id = metadata.stage_id
|
||||
self.replica_id = metadata.replica_id
|
||||
self.final_output = metadata.final_output
|
||||
self.final_output_type = metadata.final_output_type
|
||||
self.default_sampling_params = metadata.default_sampling_params
|
||||
self.requires_multimodal_data = metadata.requires_multimodal_data
|
||||
self.custom_process_input_func = metadata.custom_process_input_func
|
||||
self.engine_input_source = metadata.engine_input_source
|
||||
self.batch_size = batch_size
|
||||
@@ -62,8 +65,9 @@ class InlineStageDiffusionClient:
|
||||
self._shutting_down = False
|
||||
|
||||
logger.info(
|
||||
"[InlineStageDiffusionClient] Stage-%s initialized inline (batch_size=%d)",
|
||||
"[InlineStageDiffusionClient] stage-%s [rep-%s] initialized inline (batch_size=%d)",
|
||||
self.stage_id,
|
||||
self.replica_id,
|
||||
self.batch_size,
|
||||
)
|
||||
|
||||
@@ -82,6 +86,12 @@ class InlineStageDiffusionClient:
|
||||
sampling_params: OmniDiffusionSamplingParams,
|
||||
kv_sender_info: dict[int, dict[str, Any]] | None = None,
|
||||
) -> None:
|
||||
logger.info(
|
||||
"[InlineStageDiffusionClient] stage-%s [rep-%s] add request: %s",
|
||||
self.stage_id,
|
||||
self.replica_id,
|
||||
request_id,
|
||||
)
|
||||
task = asyncio.create_task(
|
||||
self._dispatch_request(
|
||||
request_id,
|
||||
@@ -134,6 +144,13 @@ class InlineStageDiffusionClient:
|
||||
sampling_params: OmniDiffusionSamplingParams,
|
||||
kv_sender_info: dict[int, dict[str, Any]] | None = None,
|
||||
) -> None:
|
||||
logger.info(
|
||||
"[InlineStageDiffusionClient] stage-%s [rep-%s] add batch request: %s (%d prompts)",
|
||||
self.stage_id,
|
||||
self.replica_id,
|
||||
request_id,
|
||||
len(prompts),
|
||||
)
|
||||
task = asyncio.create_task(
|
||||
self._dispatch_batch(
|
||||
request_id,
|
||||
@@ -252,7 +269,7 @@ class InlineStageDiffusionClient:
|
||||
is_start = args[0] if args else True
|
||||
profile_prefix = args[1] if len(args) > 1 else None
|
||||
if is_start and profile_prefix is None:
|
||||
profile_prefix = f"stage_{self.stage_id}_diffusion_{int(time.time())}"
|
||||
profile_prefix = f"stage_{self.stage_id}_rep_{self.replica_id}_diffusion_{int(time.time())}"
|
||||
return await loop.run_in_executor(
|
||||
self._executor,
|
||||
self._engine.profile,
|
||||
|
||||
@@ -67,6 +67,7 @@ class StageDiffusionClient:
|
||||
"""
|
||||
|
||||
stage_type: str = "diffusion"
|
||||
replica_id: int = 0
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -112,11 +113,13 @@ class StageDiffusionClient:
|
||||
batch_size: int,
|
||||
) -> None:
|
||||
self.stage_id = metadata.stage_id
|
||||
self.replica_id = metadata.replica_id
|
||||
self.final_output = metadata.final_output
|
||||
self.final_output_type = metadata.final_output_type
|
||||
self.default_sampling_params = metadata.default_sampling_params
|
||||
self.custom_process_input_func = metadata.custom_process_input_func
|
||||
self.engine_input_source = metadata.engine_input_source
|
||||
self.requires_multimodal_data = getattr(metadata, "requires_multimodal_data", False)
|
||||
self.custom_process_input_func = getattr(metadata, "custom_process_input_func", None)
|
||||
self.engine_input_source = getattr(metadata, "engine_input_source", [])
|
||||
self._proc = proc
|
||||
self._owns_process = proc is not None
|
||||
|
||||
@@ -142,8 +145,9 @@ class StageDiffusionClient:
|
||||
self._start_proc_monitor()
|
||||
|
||||
logger.info(
|
||||
"[StageDiffusionClient] Stage-%s initialized (owns_process=%s, batch_size=%d)",
|
||||
"[StageDiffusionClient] stage-%s [rep-%s] initialized (owns_process=%s, batch_size=%d)",
|
||||
self.stage_id,
|
||||
self.replica_id,
|
||||
self._owns_process,
|
||||
batch_size,
|
||||
)
|
||||
@@ -172,8 +176,9 @@ class StageDiffusionClient:
|
||||
return
|
||||
client._engine_dead = True
|
||||
logger.error(
|
||||
"[StageDiffusionClient] Stage-%s StageDiffusionProc died unexpectedly (exit code %s).",
|
||||
"[StageDiffusionClient] stage-%s [rep-%s] StageDiffusionProc died unexpectedly (exit code %s).",
|
||||
client.stage_id,
|
||||
client.replica_id,
|
||||
proc.exitcode,
|
||||
)
|
||||
|
||||
@@ -195,8 +200,9 @@ class StageDiffusionClient:
|
||||
if raw == StageDiffusionProc.DIFFUSION_PROC_DEAD:
|
||||
self._engine_dead = True
|
||||
logger.error(
|
||||
"[StageDiffusionClient] Stage-%s received DIFFUSION_PROC_DEAD sentinel from subprocess.",
|
||||
"[StageDiffusionClient] stage-%s [rep-%s] received DIFFUSION_PROC_DEAD sentinel from subprocess.",
|
||||
self.stage_id,
|
||||
self.replica_id,
|
||||
)
|
||||
break
|
||||
|
||||
@@ -212,8 +218,9 @@ class StageDiffusionClient:
|
||||
rpc_id = msg.get("rpc_id")
|
||||
error_msg = msg.get("error")
|
||||
logger.error(
|
||||
"[StageDiffusionClient] Stage-%s subprocess error for %s: %s",
|
||||
"[StageDiffusionClient] stage-%s [rep-%s] subprocess error for %s: %s",
|
||||
self.stage_id,
|
||||
self.replica_id,
|
||||
rpc_id or req_id,
|
||||
error_msg,
|
||||
)
|
||||
@@ -292,6 +299,12 @@ class StageDiffusionClient:
|
||||
) -> None:
|
||||
if self._engine_dead:
|
||||
raise EngineDeadError()
|
||||
logger.info(
|
||||
"[StageDiffusionClient] stage-%s [rep-%s] add request: %s",
|
||||
self.stage_id,
|
||||
self.replica_id,
|
||||
request_id,
|
||||
)
|
||||
self._request_socket.send(
|
||||
self._encoder.encode(
|
||||
{
|
||||
@@ -321,6 +334,13 @@ class StageDiffusionClient:
|
||||
"""
|
||||
if self._engine_dead:
|
||||
raise EngineDeadError()
|
||||
logger.info(
|
||||
"[StageDiffusionClient] stage-%s [rep-%s] add batch request: %s (%d prompts)",
|
||||
self.stage_id,
|
||||
self.replica_id,
|
||||
request_id,
|
||||
len(prompts),
|
||||
)
|
||||
task = asyncio.create_task(
|
||||
self._run_batch(
|
||||
request_id,
|
||||
@@ -353,8 +373,9 @@ class StageDiffusionClient:
|
||||
)
|
||||
except Exception as e:
|
||||
logger.exception(
|
||||
"[StageDiffusionClient] Stage-%s batch req=%s failed: %s",
|
||||
"[StageDiffusionClient] stage-%s [rep-%s] batch req=%s failed: %s",
|
||||
self.stage_id,
|
||||
self.replica_id,
|
||||
request_id,
|
||||
e,
|
||||
)
|
||||
@@ -414,7 +435,7 @@ class StageDiffusionClient:
|
||||
is_start = args_list[0] if args_list else True
|
||||
profile_prefix = args_list[1] if len(args_list) > 1 else None
|
||||
if is_start and profile_prefix is None:
|
||||
profile_prefix = f"stage_{self.stage_id}_diffusion_{int(time.time())}"
|
||||
profile_prefix = f"stage_{self.stage_id}_rep_{self.replica_id}_diffusion_{int(time.time())}"
|
||||
if len(args_list) > 1:
|
||||
args_list[1] = profile_prefix
|
||||
else:
|
||||
|
||||
Reference in new issue
Block a user