[Orchestrator] Implement cache reset RPC forwarding to AR stage engine cores (#6766)
Signed-off-by: Asthenia <asthenia0412@gmail.com> Signed-off-by: Gao Han <hgaoaf@connect.ust.hk> Co-authored-by: Asthenia <asthenia0412@gmail.com> Co-authored-by: Gao Han <hgaoaf@connect.ust.hk>
This commit is contained in:
7 files changed
+497
-65
No files matched your search
@@ -1206,3 +1206,110 @@ async def test_rpc_failure_capture_is_limited_to_pause_and_resume() -> None:
|
||||
finally:
|
||||
for q in queues:
|
||||
q.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("method", ["reset_mm_cache", "reset_encoder_cache", "reset_prefix_cache"])
|
||||
async def test_cache_reset_rpc_failure_is_nonfatal_and_visits_all_replicas(method):
|
||||
clients = [FakeStageClient(stage_type="llm") for _ in range(2)]
|
||||
setattr(clients[0], f"{method}_async", AsyncMock(side_effect=RuntimeError("reset failed")))
|
||||
ok = AsyncMock(return_value=True)
|
||||
setattr(clients[1], f"{method}_async", ok)
|
||||
orchestrator, queues = _build_bare_orchestrator(_build_stage_pools([clients]))
|
||||
try:
|
||||
await orchestrator._handle_collective_rpc(
|
||||
CollectiveRPCRequestMessage(rpc_id="reset", method=method, args=(), kwargs={}, stage_ids=[0], timeout=1.0)
|
||||
)
|
||||
result = queues[2].async_q.get_nowait()
|
||||
assert result.results[0]["supported"] is False
|
||||
assert "reset failed" in result.results[0]["error"]
|
||||
assert result.results[1] is True
|
||||
ok.assert_awaited_once_with()
|
||||
# Errors before StagePool dispatch (e.g. a missing route) must also
|
||||
# be serialized at the orchestrator boundary, then allow another RPC.
|
||||
orchestrator.stage_pools[0].collective_rpc = AsyncMock(side_effect=RuntimeError("route failed"))
|
||||
await orchestrator._handle_collective_rpc(
|
||||
CollectiveRPCRequestMessage(rpc_id="route", method=method, args=(), kwargs={}, stage_ids=[0])
|
||||
)
|
||||
assert all("route failed" in r["error"] for r in queues[2].async_q.get_nowait().results)
|
||||
orchestrator.stage_pools[0].collective_rpc = AsyncMock(return_value=True)
|
||||
await orchestrator._handle_collective_rpc(
|
||||
CollectiveRPCRequestMessage(rpc_id="next", method=method, args=(), kwargs={}, stage_ids=[0])
|
||||
)
|
||||
assert queues[2].async_q.get_nowait().results == [True, True]
|
||||
finally:
|
||||
for q in queues:
|
||||
q.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"method, kwargs, clears",
|
||||
[
|
||||
("reset_mm_cache", {}, True),
|
||||
("sleep", {}, True),
|
||||
("pause_scheduler", {"clear_cache": True}, True),
|
||||
("pause_scheduler", {"clear_cache": False}, False),
|
||||
("reset_prefix_cache", {}, False),
|
||||
("reset_encoder_cache", {}, False),
|
||||
],
|
||||
)
|
||||
async def test_control_rpc_clears_only_selected_downstream_sender(method, kwargs, clears):
|
||||
pools = _build_stage_pools([[FakeStageClient(stage_type="llm") for _ in range(2)] for _ in range(2)])
|
||||
orchestrator, queues = _build_bare_orchestrator(pools)
|
||||
events = []
|
||||
renderers = [SimpleNamespace(clear_mm_cache_async=AsyncMock()) for _ in range(2)]
|
||||
renderers[1].clear_mm_cache_async.side_effect = lambda: events.append("sender")
|
||||
orchestrator._stage_input_processors = {i: SimpleNamespace(renderer=r) for i, r in enumerate(renderers)}
|
||||
|
||||
async def rpc(**kwargs):
|
||||
events.append("receiver")
|
||||
return True
|
||||
|
||||
pools[0].collective_rpc = AsyncMock()
|
||||
pools[1].collective_rpc = AsyncMock(side_effect=rpc)
|
||||
try:
|
||||
await orchestrator._handle_collective_rpc(
|
||||
CollectiveRPCRequestMessage(
|
||||
rpc_id="reset", method=method, args=(), kwargs=kwargs, stage_ids=[1], timeout=1.0
|
||||
)
|
||||
)
|
||||
renderers[0].clear_mm_cache_async.assert_not_awaited()
|
||||
pools[0].collective_rpc.assert_not_awaited()
|
||||
assert renderers[1].clear_mm_cache_async.await_count == int(clears)
|
||||
assert events == (["sender"] if clears else []) + ["receiver", "receiver"]
|
||||
assert queues[2].async_q.get_nowait().results == [True, True]
|
||||
finally:
|
||||
for q in queues:
|
||||
q.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("failure", ["error", "timeout"])
|
||||
async def test_downstream_sender_clear_failure_is_nonfatal(failure):
|
||||
pools = _build_stage_pools([[FakeStageClient(stage_type="llm")]])
|
||||
orchestrator, queues = _build_bare_orchestrator(pools)
|
||||
|
||||
async def clear():
|
||||
if failure == "timeout":
|
||||
await asyncio.Event().wait()
|
||||
raise RuntimeError("sender cache failed")
|
||||
|
||||
renderer = SimpleNamespace(clear_mm_cache_async=AsyncMock(side_effect=clear))
|
||||
orchestrator._stage_input_processors[0] = SimpleNamespace(renderer=renderer)
|
||||
pools[0].collective_rpc = AsyncMock(return_value=None)
|
||||
message = CollectiveRPCRequestMessage(
|
||||
rpc_id="reset", method="reset_mm_cache", args=(), kwargs={}, stage_ids=[0], timeout=0.01
|
||||
)
|
||||
try:
|
||||
await orchestrator._handle_collective_rpc(message)
|
||||
result = queues[2].async_q.get_nowait().results[0]
|
||||
assert result["supported"] is False
|
||||
assert ("TimeoutError" if failure == "timeout" else "sender cache failed") in result["error"]
|
||||
pools[0].collective_rpc.assert_not_awaited()
|
||||
renderer.clear_mm_cache_async = AsyncMock()
|
||||
await orchestrator._handle_collective_rpc(message)
|
||||
assert queues[2].async_q.get_nowait().results == [None]
|
||||
finally:
|
||||
for q in queues:
|
||||
q.close()
|
||||
Reference in new issue
Block a user