[Orchestrator] Implement cache reset RPC forwarding to AR stage engine cores (#6766)

Signed-off-by: Asthenia <asthenia0412@gmail.com>
Signed-off-by: Gao Han <hgaoaf@connect.ust.hk>
Co-authored-by: Asthenia <asthenia0412@gmail.com>
Co-authored-by: Gao Han <hgaoaf@connect.ust.hk>
This commit is contained in:
authored and GitHub committed 2026-09-24 15:32:07 +00:00
1 parent 535bd35b42
commit 0295eeb6f3
7 files changed
+497 -65

No files matched your search

@@ -1206,3 +1206,110 @@ async def test_rpc_failure_capture_is_limited_to_pause_and_resume() -> None:
finally:
for q in queues:
q.close()
@pytest.mark.asyncio
@pytest.mark.parametrize("method", ["reset_mm_cache", "reset_encoder_cache", "reset_prefix_cache"])
async def test_cache_reset_rpc_failure_is_nonfatal_and_visits_all_replicas(method):
clients = [FakeStageClient(stage_type="llm") for _ in range(2)]
setattr(clients[0], f"{method}_async", AsyncMock(side_effect=RuntimeError("reset failed")))
ok = AsyncMock(return_value=True)
setattr(clients[1], f"{method}_async", ok)
orchestrator, queues = _build_bare_orchestrator(_build_stage_pools([clients]))
try:
await orchestrator._handle_collective_rpc(
CollectiveRPCRequestMessage(rpc_id="reset", method=method, args=(), kwargs={}, stage_ids=[0], timeout=1.0)
)
result = queues[2].async_q.get_nowait()
assert result.results[0]["supported"] is False
assert "reset failed" in result.results[0]["error"]
assert result.results[1] is True
ok.assert_awaited_once_with()
# Errors before StagePool dispatch (e.g. a missing route) must also
# be serialized at the orchestrator boundary, then allow another RPC.
orchestrator.stage_pools[0].collective_rpc = AsyncMock(side_effect=RuntimeError("route failed"))
await orchestrator._handle_collective_rpc(
CollectiveRPCRequestMessage(rpc_id="route", method=method, args=(), kwargs={}, stage_ids=[0])
)
assert all("route failed" in r["error"] for r in queues[2].async_q.get_nowait().results)
orchestrator.stage_pools[0].collective_rpc = AsyncMock(return_value=True)
await orchestrator._handle_collective_rpc(
CollectiveRPCRequestMessage(rpc_id="next", method=method, args=(), kwargs={}, stage_ids=[0])
)
assert queues[2].async_q.get_nowait().results == [True, True]
finally:
for q in queues:
q.close()
@pytest.mark.asyncio
@pytest.mark.parametrize(
"method, kwargs, clears",
[
("reset_mm_cache", {}, True),
("sleep", {}, True),
("pause_scheduler", {"clear_cache": True}, True),
("pause_scheduler", {"clear_cache": False}, False),
("reset_prefix_cache", {}, False),
("reset_encoder_cache", {}, False),
],
)
async def test_control_rpc_clears_only_selected_downstream_sender(method, kwargs, clears):
pools = _build_stage_pools([[FakeStageClient(stage_type="llm") for _ in range(2)] for _ in range(2)])
orchestrator, queues = _build_bare_orchestrator(pools)
events = []
renderers = [SimpleNamespace(clear_mm_cache_async=AsyncMock()) for _ in range(2)]
renderers[1].clear_mm_cache_async.side_effect = lambda: events.append("sender")
orchestrator._stage_input_processors = {i: SimpleNamespace(renderer=r) for i, r in enumerate(renderers)}
async def rpc(**kwargs):
events.append("receiver")
return True
pools[0].collective_rpc = AsyncMock()
pools[1].collective_rpc = AsyncMock(side_effect=rpc)
try:
await orchestrator._handle_collective_rpc(
CollectiveRPCRequestMessage(
rpc_id="reset", method=method, args=(), kwargs=kwargs, stage_ids=[1], timeout=1.0
)
)
renderers[0].clear_mm_cache_async.assert_not_awaited()
pools[0].collective_rpc.assert_not_awaited()
assert renderers[1].clear_mm_cache_async.await_count == int(clears)
assert events == (["sender"] if clears else []) + ["receiver", "receiver"]
assert queues[2].async_q.get_nowait().results == [True, True]
finally:
for q in queues:
q.close()
@pytest.mark.asyncio
@pytest.mark.parametrize("failure", ["error", "timeout"])
async def test_downstream_sender_clear_failure_is_nonfatal(failure):
pools = _build_stage_pools([[FakeStageClient(stage_type="llm")]])
orchestrator, queues = _build_bare_orchestrator(pools)
async def clear():
if failure == "timeout":
await asyncio.Event().wait()
raise RuntimeError("sender cache failed")
renderer = SimpleNamespace(clear_mm_cache_async=AsyncMock(side_effect=clear))
orchestrator._stage_input_processors[0] = SimpleNamespace(renderer=renderer)
pools[0].collective_rpc = AsyncMock(return_value=None)
message = CollectiveRPCRequestMessage(
rpc_id="reset", method="reset_mm_cache", args=(), kwargs={}, stage_ids=[0], timeout=0.01
)
try:
await orchestrator._handle_collective_rpc(message)
result = queues[2].async_q.get_nowait().results[0]
assert result["supported"] is False
assert ("TimeoutError" if failure == "timeout" else "sender cache failed") in result["error"]
pools[0].collective_rpc.assert_not_awaited()
renderer.clear_mm_cache_async = AsyncMock()
await orchestrator._handle_collective_rpc(message)
assert queues[2].async_q.get_nowait().results == [None]
finally:
for q in queues:
q.close()
+60 -5
View File
@@ -1,3 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Unit tests for StagePool.collective_rpc EngineCore control dispatch."""
from __future__ import annotations
@@ -40,14 +43,14 @@ def test_collective_rpc_normalizes_none_args_on_control_helper():
def test_collective_rpc_unrelated_async_helper_uses_collective_path():
async def run() -> None:
other = AsyncMock(return_value="should-not-run")
pool, client = _make_pool(reset_prefix_cache_async=other)
pool, client = _make_pool(is_sleeping_async=other)
result = await pool.collective_rpc(0, "reset_prefix_cache", timeout=1.5, args=("x",), kwargs={"k": 1})
result = await pool.collective_rpc(0, "is_sleeping", timeout=1.5, args=("x",), kwargs={"k": 1})
assert result == {"via": "collective"}
other.assert_not_awaited()
client.collective_rpc_async.assert_awaited_once_with(
method="reset_prefix_cache",
method="is_sleeping",
timeout=1.5,
args=("x",),
kwargs={"k": 1},
@@ -63,7 +66,7 @@ def test_collective_rpc_control_helper_honors_timeout():
await asyncio.sleep(1.0)
return "slept"
pool, _client = _make_pool(sleep_async=slow_sleep)
pool, _client = _make_pool(sleep_async=AsyncMock(side_effect=slow_sleep))
with pytest.raises(asyncio.TimeoutError):
await pool.collective_rpc(0, "sleep", timeout=0.01, args=(1,))
@@ -80,11 +83,38 @@ def test_collective_rpc_control_method_reraises_worker_error():
asyncio.run(run())
@pytest.mark.cpu
def test_collective_rpc_reset_caches_use_control_helpers():
async def run() -> None:
reset_prefix = AsyncMock(return_value=True)
reset_encoder = AsyncMock(return_value=None)
reset_mm = AsyncMock(return_value=None)
pool, client = _make_pool(
reset_prefix_cache_async=reset_prefix,
reset_encoder_cache_async=reset_encoder,
reset_mm_cache_async=reset_mm,
)
result = await pool.collective_rpc(0, "reset_prefix_cache", args=(True, True), timeout=2.0)
assert result is True
reset_prefix.assert_awaited_once_with(True, True)
await pool.collective_rpc(0, "reset_encoder_cache", timeout=2.0)
reset_encoder.assert_awaited_once()
await pool.collective_rpc(0, "reset_mm_cache", timeout=2.0)
reset_mm.assert_awaited_once()
client.collective_rpc_async.assert_not_awaited()
asyncio.run(run())
@pytest.mark.cpu
def test_collective_rpc_non_control_method_still_returns_error_dict():
async def run() -> None:
pool, _client = _make_pool(collective_rpc_async=AsyncMock(side_effect=RuntimeError("probe failed")))
result = await pool.collective_rpc(0, "reset_prefix_cache")
result = await pool.collective_rpc(0, "is_sleeping")
assert result["supported"] is False
assert "probe failed" in result["error"]
@@ -122,3 +152,28 @@ def test_abort_requests_does_not_commit_op_state_when_engine_abort_fails():
abort.assert_awaited_once()
asyncio.run(run())
@pytest.mark.cpu
@pytest.mark.parametrize("method", ["reset_prefix_cache", "reset_encoder_cache", "reset_mm_cache"])
@pytest.mark.parametrize("failure", ["error", "timeout", "missing"])
def test_cache_reset_failure_is_serialized_and_pool_remains_usable(method, failure):
async def run():
async def reset():
if failure == "timeout":
await asyncio.Event().wait()
raise RuntimeError("cache reset failed")
pool, client = _make_pool()
if failure != "missing":
setattr(client, f"{method}_async", AsyncMock(side_effect=reset))
result = await pool.collective_rpc(0, method, timeout=0.01)
assert result["supported"] is False
assert result["error"]
if failure == "timeout":
assert "timed out" in result["error"]
client.collective_rpc_async.assert_not_awaited()
# A subsequent control request still completes on the same pool.
assert await pool.collective_rpc(0, "is_sleeping") == {"via": "collective"}
asyncio.run(run())