[Orchestrator] Implement cache reset RPC forwarding to AR stage engine cores (#6766)

Signed-off-by: Asthenia <asthenia0412@gmail.com>
Signed-off-by: Gao Han <hgaoaf@connect.ust.hk>
Co-authored-by: Asthenia <asthenia0412@gmail.com>
Co-authored-by: Gao Han <hgaoaf@connect.ust.hk>
This commit is contained in:
authored and GitHub committed 2026-09-24 15:32:07 +00:00
1 parent 535bd35b42
commit 0295eeb6f3
7 files changed
+497 -65

No files matched your search

@@ -1206,3 +1206,110 @@ async def test_rpc_failure_capture_is_limited_to_pause_and_resume() -> None:
finally:
for q in queues:
q.close()
@pytest.mark.asyncio
@pytest.mark.parametrize("method", ["reset_mm_cache", "reset_encoder_cache", "reset_prefix_cache"])
async def test_cache_reset_rpc_failure_is_nonfatal_and_visits_all_replicas(method):
clients = [FakeStageClient(stage_type="llm") for _ in range(2)]
setattr(clients[0], f"{method}_async", AsyncMock(side_effect=RuntimeError("reset failed")))
ok = AsyncMock(return_value=True)
setattr(clients[1], f"{method}_async", ok)
orchestrator, queues = _build_bare_orchestrator(_build_stage_pools([clients]))
try:
await orchestrator._handle_collective_rpc(
CollectiveRPCRequestMessage(rpc_id="reset", method=method, args=(), kwargs={}, stage_ids=[0], timeout=1.0)
)
result = queues[2].async_q.get_nowait()
assert result.results[0]["supported"] is False
assert "reset failed" in result.results[0]["error"]
assert result.results[1] is True
ok.assert_awaited_once_with()
# Errors before StagePool dispatch (e.g. a missing route) must also
# be serialized at the orchestrator boundary, then allow another RPC.
orchestrator.stage_pools[0].collective_rpc = AsyncMock(side_effect=RuntimeError("route failed"))
await orchestrator._handle_collective_rpc(
CollectiveRPCRequestMessage(rpc_id="route", method=method, args=(), kwargs={}, stage_ids=[0])
)
assert all("route failed" in r["error"] for r in queues[2].async_q.get_nowait().results)
orchestrator.stage_pools[0].collective_rpc = AsyncMock(return_value=True)
await orchestrator._handle_collective_rpc(
CollectiveRPCRequestMessage(rpc_id="next", method=method, args=(), kwargs={}, stage_ids=[0])
)
assert queues[2].async_q.get_nowait().results == [True, True]
finally:
for q in queues:
q.close()
@pytest.mark.asyncio
@pytest.mark.parametrize(
"method, kwargs, clears",
[
("reset_mm_cache", {}, True),
("sleep", {}, True),
("pause_scheduler", {"clear_cache": True}, True),
("pause_scheduler", {"clear_cache": False}, False),
("reset_prefix_cache", {}, False),
("reset_encoder_cache", {}, False),
],
)
async def test_control_rpc_clears_only_selected_downstream_sender(method, kwargs, clears):
pools = _build_stage_pools([[FakeStageClient(stage_type="llm") for _ in range(2)] for _ in range(2)])
orchestrator, queues = _build_bare_orchestrator(pools)
events = []
renderers = [SimpleNamespace(clear_mm_cache_async=AsyncMock()) for _ in range(2)]
renderers[1].clear_mm_cache_async.side_effect = lambda: events.append("sender")
orchestrator._stage_input_processors = {i: SimpleNamespace(renderer=r) for i, r in enumerate(renderers)}
async def rpc(**kwargs):
events.append("receiver")
return True
pools[0].collective_rpc = AsyncMock()
pools[1].collective_rpc = AsyncMock(side_effect=rpc)
try:
await orchestrator._handle_collective_rpc(
CollectiveRPCRequestMessage(
rpc_id="reset", method=method, args=(), kwargs=kwargs, stage_ids=[1], timeout=1.0
)
)
renderers[0].clear_mm_cache_async.assert_not_awaited()
pools[0].collective_rpc.assert_not_awaited()
assert renderers[1].clear_mm_cache_async.await_count == int(clears)
assert events == (["sender"] if clears else []) + ["receiver", "receiver"]
assert queues[2].async_q.get_nowait().results == [True, True]
finally:
for q in queues:
q.close()
@pytest.mark.asyncio
@pytest.mark.parametrize("failure", ["error", "timeout"])
async def test_downstream_sender_clear_failure_is_nonfatal(failure):
pools = _build_stage_pools([[FakeStageClient(stage_type="llm")]])
orchestrator, queues = _build_bare_orchestrator(pools)
async def clear():
if failure == "timeout":
await asyncio.Event().wait()
raise RuntimeError("sender cache failed")
renderer = SimpleNamespace(clear_mm_cache_async=AsyncMock(side_effect=clear))
orchestrator._stage_input_processors[0] = SimpleNamespace(renderer=renderer)
pools[0].collective_rpc = AsyncMock(return_value=None)
message = CollectiveRPCRequestMessage(
rpc_id="reset", method="reset_mm_cache", args=(), kwargs={}, stage_ids=[0], timeout=0.01
)
try:
await orchestrator._handle_collective_rpc(message)
result = queues[2].async_q.get_nowait().results[0]
assert result["supported"] is False
assert ("TimeoutError" if failure == "timeout" else "sender cache failed") in result["error"]
pools[0].collective_rpc.assert_not_awaited()
renderer.clear_mm_cache_async = AsyncMock()
await orchestrator._handle_collective_rpc(message)
assert queues[2].async_q.get_nowait().results == [None]
finally:
for q in queues:
q.close()