[Bugfix] Fix lost async diffusion outputs in request-level batching (#6023)
Signed-off-by: samithuang <285365963@qq.com>
This commit is contained in:
1 parent
ce6ab0ca46
commit
136cb27b05
9 files changed
+475
-55
No files matched your search
@@ -641,8 +641,23 @@ class MultiprocDiffusionExecutor(DiffusionExecutor):
|
||||
async_output_id=per_req_id,
|
||||
)
|
||||
)
|
||||
# The worker's background D2H/SHM thread can publish OUTPUT_READY
|
||||
# before this call returns. In that case the pump found no split
|
||||
# map and cached the whole batch output under batch_id, so adopt it
|
||||
# here instead of registering a map nothing will ever consume.
|
||||
with self._futures_lock:
|
||||
self._batch_split_map[batch_id] = per_req_map
|
||||
early = self._completed_outputs.pop(batch_id, None)
|
||||
if early is None:
|
||||
self._batch_split_map[batch_id] = per_req_map
|
||||
if early is not None:
|
||||
logger.debug("Batch %s output arrived before split map; splitting now", batch_id)
|
||||
try:
|
||||
batch_output = early.result()
|
||||
error = None
|
||||
except Exception as exc:
|
||||
batch_output = None
|
||||
error = str(exc)
|
||||
self._deliver_batch_split(per_req_map, batch_output, error)
|
||||
return BatchRunnerOutput.from_list(runner_outputs)
|
||||
if not isinstance(result, BatchRunnerOutput):
|
||||
raise RuntimeError(f"Unexpected response type for execute_batch: {type(result)!r}")
|
||||
@@ -868,52 +883,76 @@ class MultiprocDiffusionExecutor(DiffusionExecutor):
|
||||
unpack_diffusion_output_shm(msg.output)
|
||||
except Exception:
|
||||
logger.exception("SHM unpack failed for batch %s", batch_id)
|
||||
batch_output = msg.output
|
||||
for per_req_id, req_id in per_req_map.items():
|
||||
req_output = batch_output.get_request_output(req_id)
|
||||
per_req_result: DiffusionOutput
|
||||
if req_output is not None and req_output.result is not None:
|
||||
per_req_result = req_output.result
|
||||
elif msg.error:
|
||||
per_req_result = DiffusionOutput(error=msg.error)
|
||||
else:
|
||||
per_req_result = DiffusionOutput(error="No output result for batch request")
|
||||
fut: concurrent.futures.Future = concurrent.futures.Future()
|
||||
fut.set_result(per_req_result)
|
||||
with self._futures_lock:
|
||||
pending = self._output_futures.pop(per_req_id, None)
|
||||
if pending is not None and not pending.done():
|
||||
pending.set_result(per_req_result)
|
||||
else:
|
||||
self._completed_outputs[per_req_id] = fut
|
||||
self._deliver_batch_split(per_req_map, msg.output, msg.error)
|
||||
else:
|
||||
with self._futures_lock:
|
||||
fut = self._output_futures.pop(batch_id, None) if batch_id else None
|
||||
if fut is not None and not fut.done():
|
||||
if msg.error:
|
||||
fut.set_exception(RuntimeError(msg.error))
|
||||
else:
|
||||
try:
|
||||
unpack_diffusion_output_shm(msg.output)
|
||||
except Exception as e:
|
||||
logger.exception("SHM unpack failed in result pump")
|
||||
fut.set_exception(e)
|
||||
continue
|
||||
fut.set_result(msg.output)
|
||||
elif batch_id:
|
||||
fut = concurrent.futures.Future()
|
||||
if msg.error:
|
||||
fut.set_exception(RuntimeError(msg.error))
|
||||
else:
|
||||
try:
|
||||
unpack_diffusion_output_shm(msg.output)
|
||||
except Exception as e:
|
||||
logger.exception("SHM unpack failed in result pump (cached)")
|
||||
fut.set_exception(e)
|
||||
else:
|
||||
fut.set_result(msg.output)
|
||||
# Single-request result: unpack SHM first, then resolve or cache atomically.
|
||||
output_result: DiffusionOutput | None = None
|
||||
exc: Exception | None = None
|
||||
if msg.error:
|
||||
exc = RuntimeError(msg.error)
|
||||
else:
|
||||
try:
|
||||
unpack_diffusion_output_shm(msg.output)
|
||||
output_result = msg.output
|
||||
except Exception as e:
|
||||
logger.exception("SHM unpack failed in result pump")
|
||||
exc = e
|
||||
|
||||
if batch_id:
|
||||
with self._futures_lock:
|
||||
self._completed_outputs[batch_id] = fut
|
||||
pending = self._output_futures.pop(batch_id, None)
|
||||
if pending is not None and not pending.done():
|
||||
if exc is not None:
|
||||
pending.set_exception(exc)
|
||||
else:
|
||||
pending.set_result(output_result)
|
||||
else:
|
||||
fut = concurrent.futures.Future()
|
||||
if exc is not None:
|
||||
fut.set_exception(exc)
|
||||
else:
|
||||
fut.set_result(output_result)
|
||||
self._completed_outputs[batch_id] = fut
|
||||
|
||||
def _deliver_batch_split(
|
||||
self,
|
||||
per_req_map: dict[str, str],
|
||||
batch_output: Any,
|
||||
error: str | None = None,
|
||||
) -> None:
|
||||
"""Resolve per-request futures from one batch-level output."""
|
||||
for per_req_id, req_id in per_req_map.items():
|
||||
req_output = batch_output.get_request_output(req_id) if batch_output is not None else None
|
||||
per_req_result: DiffusionOutput
|
||||
if req_output is not None and req_output.result is not None:
|
||||
per_req_result = req_output.result
|
||||
elif error:
|
||||
per_req_result = DiffusionOutput(error=error)
|
||||
else:
|
||||
per_req_result = DiffusionOutput(error="No output result for batch request")
|
||||
with self._futures_lock:
|
||||
pending = self._output_futures.pop(per_req_id, None)
|
||||
if pending is not None and not pending.done():
|
||||
pending.set_result(per_req_result)
|
||||
else:
|
||||
fut: concurrent.futures.Future = concurrent.futures.Future()
|
||||
fut.set_result(per_req_result)
|
||||
self._completed_outputs[per_req_id] = fut
|
||||
|
||||
def describe_pending_state(self, async_output_id: str | None = None) -> str:
|
||||
"""Summarize async-output bookkeeping for diagnosing stuck waits."""
|
||||
with self._futures_lock:
|
||||
waiting = list(self._output_futures)
|
||||
cached = list(self._completed_outputs)
|
||||
batches = list(self._batch_split_map)
|
||||
rpcs = list(self._rpc_futures)
|
||||
return (
|
||||
f"async_output_id={async_output_id} "
|
||||
f"waiting={len(waiting)}{waiting[:5]} "
|
||||
f"cached={len(cached)}{cached[:5]} "
|
||||
f"unsplit_batches={len(batches)}{batches[:5]} "
|
||||
f"pending_rpcs={len(rpcs)}{rpcs[:5]}"
|
||||
)
|
||||
|
||||
def _next_rpc_id(self) -> str:
|
||||
with self._rpc_id_lock:
|
||||
|
||||
Reference in new issue
Block a user