Address diffusion RPC review feedback

Signed-off-by: hsliuustc0106 <liuhongsheng4@huawei.com>
This commit is contained in:
hsliuustc0106 committed 2026-06-13 14:39:53 +08:00
1 parent 170f98c847
commit 08b2f694f3
3 files changed
+82 -9

No files matched your search

@@ -352,6 +352,35 @@ class TestSerialEngineOperations:
assert engine.collective_rpc("remove_lora") == [False]
def test_collective_rpc_collects_rank_status_only_for_control_plane_all_rank_rpc(self):
executor, req_q, res_q = _make_executor(num_gpus=2)
res_q.put(_tagged_output("forward"))
result = executor.collective_rpc(
"execute_stepwise",
unique_reply_rank=0,
exec_all_ranks=True,
)
forward_rpc = req_q.get_nowait()
assert result.error == "forward"
assert forward_rpc["exec_all_ranks"] is True
assert forward_rpc["collect_rank_status"] is False
res_q.put(
{
"type": DIFFUSION_RPC_RESULT_ENVELOPE,
"method": "remove_lora",
"result": True,
"rank_statuses": [{"rank": 0, "ok": True, "bool_result": True}],
}
)
assert executor.collective_rpc("remove_lora") == [True]
control_rpc = req_q.get_nowait()
assert control_rpc["exec_all_ranks"] is True
assert control_rpc["collect_rank_status"] is True
def test_serial_add_req_then_collective_rpc(self):
engine, _, req_q, res_q = _make_engine()
wt = _start_worker(req_q, res_q, count=2)
@@ -467,6 +496,42 @@ class TestWorkerProcRpcRankStatus:
assert status["bool_result"] is None
assert "local boom" in status["traceback"]
def test_execute_rpc_rejects_collect_rank_status_without_all_ranks(self):
proc = self._make_worker_proc()
with pytest.raises(ValueError, match="collect_rank_status requires exec_all_ranks=True"):
proc.execute_rpc(
{
"method": "ping",
"args": (),
"kwargs": {},
"output_rank": 0,
"exec_all_ranks": False,
"collect_rank_status": True,
}
)
proc.worker.execute_method.assert_not_called()
def test_execute_rpc_non_collect_exception_preserves_original_type(self):
proc = self._make_worker_proc()
original = ValueError("local boom")
proc.worker.execute_method = Mock(side_effect=original)
with pytest.raises(ValueError) as excinfo:
proc.execute_rpc(
{
"method": "bad",
"args": (),
"kwargs": {},
"output_rank": 0,
"exec_all_ranks": False,
"collect_rank_status": False,
}
)
assert excinfo.value is original
# ───────── error handling: EngineDeadError propagation through layers ─────
@@ -376,19 +376,20 @@ class MultiprocDiffusionExecutor(DiffusionExecutor):
deadline = None if timeout is None else time.monotonic() + timeout
kwargs = kwargs or {}
# Prepare RPC request message
# When unique_reply_rank is None, all workers must execute the RPC
# but only rank 0 can reply (it's the only one with a result_mq).
collect_rank_status = unique_reply_rank is None or exec_all_ranks
# Prepare RPC request message. When unique_reply_rank is None, all
# workers must execute the RPC but only rank 0 can reply (it's the
# only one with a result_mq). Collect detailed rank statuses only for
# this control-plane all-rank path; forward-path exec_all_ranks RPCs
# avoid the per-step host object gather.
execute_all_ranks = unique_reply_rank is None or exec_all_ranks
collect_rank_status = unique_reply_rank is None
rpc_request = {
"type": "rpc",
"method": method,
"args": args,
"kwargs": kwargs,
"output_rank": unique_reply_rank if unique_reply_rank is not None else 0,
"exec_all_ranks": unique_reply_rank is None or exec_all_ranks,
# This intentionally includes forward-path exec_all_ranks RPCs:
# the host status gather prevents silent non-reply-rank failures.
"exec_all_ranks": execute_all_ranks,
"collect_rank_status": collect_rank_status,
}
@@ -762,6 +762,11 @@ class WorkerProc:
exec_all_ranks = rpc_request.get("exec_all_ranks", False)
collect_rank_status = rpc_request.get("collect_rank_status", False)
if collect_rank_status and not exec_all_ranks:
raise ValueError(
"collect_rank_status requires exec_all_ranks=True so all ranks enter the status gather"
)
should_execute = exec_all_ranks or output_rank is None or output_rank == self.gpu_id
should_reply = (output_rank is None or output_rank == self.gpu_id) and self.result_mq is not None
@@ -769,6 +774,7 @@ class WorkerProc:
return None, False
result = None
rpc_exception: Exception | None = None
status: dict[str, Any] = {
"rank": self.gpu_id,
"ok": True,
@@ -783,6 +789,7 @@ class WorkerProc:
result = self.worker.execute_method(method, *args, **kwargs)
except Exception as e:
logger.error(f"Error executing RPC: {e}", exc_info=True)
rpc_exception = e
status.update(
{
"ok": False,
@@ -809,8 +816,8 @@ class WorkerProc:
)
return None, False
if not status["ok"]:
raise RuntimeError(status["error"])
if rpc_exception is not None:
raise rpc_exception
return result, should_reply
def worker_busy_loop(self) -> None: