[Bugfix] Restore user config/runtime stage init timeout (#2519)

Signed-off-by: yuanheng <jonathan.zhaoyh@gmail.com>
Signed-off-by: Yuanheng Zhao <jonathan.zhaoyh@gmail.com>
Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
Co-authored-by: SYLAR <125541396+lishunyang12@users.noreply.github.com>
This commit is contained in:
authored and GitHub committed 2026-04-12 00:48:47 +08:00
1 parent 6e93595822
commit c20cac86ce
16 files changed
+207 -29

No files matched your search

@@ -50,11 +50,12 @@ class StageDiffusionClient:
model: str,
od_config: OmniDiffusionConfig,
metadata: StageMetadata,
stage_init_timeout: int,
batch_size: int = 1,
) -> None:
# Spawn StageDiffusionProc subprocess and wait for READY.
proc, handshake_address, request_address, response_address = spawn_diffusion_proc(model, od_config)
complete_diffusion_handshake(proc, handshake_address)
complete_diffusion_handshake(proc, handshake_address, stage_init_timeout)
self._initialize_client(metadata, request_address, response_address, proc=proc, batch_size=batch_size)
@classmethod
+9 -5
View File
@@ -39,8 +39,6 @@ if TYPE_CHECKING:
logger = init_logger(__name__)
_HANDSHAKE_POLL_TIMEOUT_S = 600
class StageDiffusionProc:
"""Subprocess entry point for diffusion inference.
@@ -619,13 +617,14 @@ def spawn_diffusion_proc(
def complete_diffusion_handshake(
proc: BaseProcess,
handshake_address: str,
handshake_timeout: int,
) -> None:
"""Wait for the diffusion subprocess to signal READY.
On failure the process is terminated before re-raising.
"""
try:
_perform_diffusion_handshake(proc, handshake_address)
_perform_diffusion_handshake(proc, handshake_address, handshake_timeout)
except Exception:
shutdown([proc])
raise
@@ -634,6 +633,7 @@ def complete_diffusion_handshake(
def _perform_diffusion_handshake(
proc: BaseProcess,
handshake_address: str,
handshake_timeout: int,
) -> None:
"""Run the handshake with the diffusion subprocess."""
with zmq_socket_ctx(handshake_address, zmq.ROUTER, bind=True) as handshake_socket:
@@ -641,11 +641,15 @@ def _perform_diffusion_handshake(
poller.register(handshake_socket, zmq.POLLIN)
poller.register(proc.sentinel, zmq.POLLIN)
timeout_ms = _HANDSHAKE_POLL_TIMEOUT_S * 1000
timeout_ms = handshake_timeout * 1000
while True:
events = dict(poller.poll(timeout=timeout_ms))
if not events:
raise TimeoutError("Timed out waiting for READY from StageDiffusionProc")
raise TimeoutError(
f"Timed out waiting for READY from StageDiffusionProc after {handshake_timeout}s. "
f"This typically indicates model loading or warmup is taking too long. "
f"Consider increasing `stage_init_timeout` for large models."
)
if handshake_socket in events:
identity, raw = handshake_socket.recv_multipart()
msg = msgspec.msgpack.decode(raw)