fix:#issue 432 (#517)
Signed-off-by: GG-li <3226868735@qq.com> Signed-off-by: Sihao Li <111170255+GG-li@users.noreply.github.com> Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
This commit is contained in:
1 parent
61ddeef9f9
commit
2c92b63cce
1 file changed
+27
-3
@@ -585,9 +585,18 @@ def _stage_worker(
|
|||||||
|
|
||||||
lock_files = acquired_lock_fds
|
lock_files = acquired_lock_fds
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.debug("[Stage-%s] Failed to set up sequential initialization lock: %s", stage_id, e)
|
logger.debug(
|
||||||
|
"[Stage-%s] Failed to set up sequential initialization lock: %s",
|
||||||
|
stage_id,
|
||||||
|
e,
|
||||||
|
)
|
||||||
# Init engine based on stage_type
|
# Init engine based on stage_type
|
||||||
logger.debug("[Stage-%s] Initializing %s engine with args keys=%s", stage_id, stage_type, list(engine_args.keys()))
|
logger.debug(
|
||||||
|
"[Stage-%s] Initializing %s engine with args keys=%s",
|
||||||
|
stage_id,
|
||||||
|
stage_type,
|
||||||
|
list(engine_args.keys()),
|
||||||
|
)
|
||||||
try:
|
try:
|
||||||
if stage_type == "diffusion":
|
if stage_type == "diffusion":
|
||||||
engine_args.pop("model_stage")
|
engine_args.pop("model_stage")
|
||||||
@@ -1111,6 +1120,20 @@ async def _stage_worker_async(
|
|||||||
except (OSError, ValueError):
|
except (OSError, ValueError):
|
||||||
pass
|
pass
|
||||||
omni_stage.set_async_engine(stage_engine)
|
omni_stage.set_async_engine(stage_engine)
|
||||||
|
if hasattr(omni_stage.async_engine, "log_stats") and omni_stage.async_engine.log_stats:
|
||||||
|
|
||||||
|
async def _force_log():
|
||||||
|
try:
|
||||||
|
while True:
|
||||||
|
await asyncio.sleep(10.0)
|
||||||
|
await omni_stage.async_engine.do_log_stats()
|
||||||
|
except asyncio.CancelledError:
|
||||||
|
pass
|
||||||
|
|
||||||
|
log_stats_task = asyncio.create_task(_force_log())
|
||||||
|
else:
|
||||||
|
log_stats_task = None
|
||||||
|
|
||||||
# Don't keep the dummy data in memory (only for LLM engines)
|
# Don't keep the dummy data in memory (only for LLM engines)
|
||||||
if stage_type != "diffusion":
|
if stage_type != "diffusion":
|
||||||
await stage_engine.reset_mm_cache()
|
await stage_engine.reset_mm_cache()
|
||||||
@@ -1335,7 +1358,8 @@ async def _stage_worker_async(
|
|||||||
}
|
}
|
||||||
)
|
)
|
||||||
logger.debug("Enqueued result for request %s to downstream", rid)
|
logger.debug("Enqueued result for request %s to downstream", rid)
|
||||||
|
if log_stats_task is not None:
|
||||||
|
log_stats_task.cancel()
|
||||||
logger.info("Stage worker exiting")
|
logger.info("Stage worker exiting")
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
Reference in new issue
Block a user