[Orchestrator] Implement cache reset RPC forwarding to AR stage engine cores (#6766)
Signed-off-by: Asthenia <asthenia0412@gmail.com> Signed-off-by: Gao Han <hgaoaf@connect.ust.hk> Co-authored-by: Asthenia <asthenia0412@gmail.com> Co-authored-by: Gao Han <hgaoaf@connect.ust.hk>
This commit is contained in:
7 files changed
+497
-65
No files matched your search
@@ -1206,3 +1206,110 @@ async def test_rpc_failure_capture_is_limited_to_pause_and_resume() -> None:
|
||||
finally:
|
||||
for q in queues:
|
||||
q.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("method", ["reset_mm_cache", "reset_encoder_cache", "reset_prefix_cache"])
|
||||
async def test_cache_reset_rpc_failure_is_nonfatal_and_visits_all_replicas(method):
|
||||
clients = [FakeStageClient(stage_type="llm") for _ in range(2)]
|
||||
setattr(clients[0], f"{method}_async", AsyncMock(side_effect=RuntimeError("reset failed")))
|
||||
ok = AsyncMock(return_value=True)
|
||||
setattr(clients[1], f"{method}_async", ok)
|
||||
orchestrator, queues = _build_bare_orchestrator(_build_stage_pools([clients]))
|
||||
try:
|
||||
await orchestrator._handle_collective_rpc(
|
||||
CollectiveRPCRequestMessage(rpc_id="reset", method=method, args=(), kwargs={}, stage_ids=[0], timeout=1.0)
|
||||
)
|
||||
result = queues[2].async_q.get_nowait()
|
||||
assert result.results[0]["supported"] is False
|
||||
assert "reset failed" in result.results[0]["error"]
|
||||
assert result.results[1] is True
|
||||
ok.assert_awaited_once_with()
|
||||
# Errors before StagePool dispatch (e.g. a missing route) must also
|
||||
# be serialized at the orchestrator boundary, then allow another RPC.
|
||||
orchestrator.stage_pools[0].collective_rpc = AsyncMock(side_effect=RuntimeError("route failed"))
|
||||
await orchestrator._handle_collective_rpc(
|
||||
CollectiveRPCRequestMessage(rpc_id="route", method=method, args=(), kwargs={}, stage_ids=[0])
|
||||
)
|
||||
assert all("route failed" in r["error"] for r in queues[2].async_q.get_nowait().results)
|
||||
orchestrator.stage_pools[0].collective_rpc = AsyncMock(return_value=True)
|
||||
await orchestrator._handle_collective_rpc(
|
||||
CollectiveRPCRequestMessage(rpc_id="next", method=method, args=(), kwargs={}, stage_ids=[0])
|
||||
)
|
||||
assert queues[2].async_q.get_nowait().results == [True, True]
|
||||
finally:
|
||||
for q in queues:
|
||||
q.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
"method, kwargs, clears",
|
||||
[
|
||||
("reset_mm_cache", {}, True),
|
||||
("sleep", {}, True),
|
||||
("pause_scheduler", {"clear_cache": True}, True),
|
||||
("pause_scheduler", {"clear_cache": False}, False),
|
||||
("reset_prefix_cache", {}, False),
|
||||
("reset_encoder_cache", {}, False),
|
||||
],
|
||||
)
|
||||
async def test_control_rpc_clears_only_selected_downstream_sender(method, kwargs, clears):
|
||||
pools = _build_stage_pools([[FakeStageClient(stage_type="llm") for _ in range(2)] for _ in range(2)])
|
||||
orchestrator, queues = _build_bare_orchestrator(pools)
|
||||
events = []
|
||||
renderers = [SimpleNamespace(clear_mm_cache_async=AsyncMock()) for _ in range(2)]
|
||||
renderers[1].clear_mm_cache_async.side_effect = lambda: events.append("sender")
|
||||
orchestrator._stage_input_processors = {i: SimpleNamespace(renderer=r) for i, r in enumerate(renderers)}
|
||||
|
||||
async def rpc(**kwargs):
|
||||
events.append("receiver")
|
||||
return True
|
||||
|
||||
pools[0].collective_rpc = AsyncMock()
|
||||
pools[1].collective_rpc = AsyncMock(side_effect=rpc)
|
||||
try:
|
||||
await orchestrator._handle_collective_rpc(
|
||||
CollectiveRPCRequestMessage(
|
||||
rpc_id="reset", method=method, args=(), kwargs=kwargs, stage_ids=[1], timeout=1.0
|
||||
)
|
||||
)
|
||||
renderers[0].clear_mm_cache_async.assert_not_awaited()
|
||||
pools[0].collective_rpc.assert_not_awaited()
|
||||
assert renderers[1].clear_mm_cache_async.await_count == int(clears)
|
||||
assert events == (["sender"] if clears else []) + ["receiver", "receiver"]
|
||||
assert queues[2].async_q.get_nowait().results == [True, True]
|
||||
finally:
|
||||
for q in queues:
|
||||
q.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("failure", ["error", "timeout"])
|
||||
async def test_downstream_sender_clear_failure_is_nonfatal(failure):
|
||||
pools = _build_stage_pools([[FakeStageClient(stage_type="llm")]])
|
||||
orchestrator, queues = _build_bare_orchestrator(pools)
|
||||
|
||||
async def clear():
|
||||
if failure == "timeout":
|
||||
await asyncio.Event().wait()
|
||||
raise RuntimeError("sender cache failed")
|
||||
|
||||
renderer = SimpleNamespace(clear_mm_cache_async=AsyncMock(side_effect=clear))
|
||||
orchestrator._stage_input_processors[0] = SimpleNamespace(renderer=renderer)
|
||||
pools[0].collective_rpc = AsyncMock(return_value=None)
|
||||
message = CollectiveRPCRequestMessage(
|
||||
rpc_id="reset", method="reset_mm_cache", args=(), kwargs={}, stage_ids=[0], timeout=0.01
|
||||
)
|
||||
try:
|
||||
await orchestrator._handle_collective_rpc(message)
|
||||
result = queues[2].async_q.get_nowait().results[0]
|
||||
assert result["supported"] is False
|
||||
assert ("TimeoutError" if failure == "timeout" else "sender cache failed") in result["error"]
|
||||
pools[0].collective_rpc.assert_not_awaited()
|
||||
renderer.clear_mm_cache_async = AsyncMock()
|
||||
await orchestrator._handle_collective_rpc(message)
|
||||
assert queues[2].async_q.get_nowait().results == [None]
|
||||
finally:
|
||||
for q in queues:
|
||||
q.close()
|
||||
@@ -1,3 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
"""Unit tests for StagePool.collective_rpc EngineCore control dispatch."""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -40,14 +43,14 @@ def test_collective_rpc_normalizes_none_args_on_control_helper():
|
||||
def test_collective_rpc_unrelated_async_helper_uses_collective_path():
|
||||
async def run() -> None:
|
||||
other = AsyncMock(return_value="should-not-run")
|
||||
pool, client = _make_pool(reset_prefix_cache_async=other)
|
||||
pool, client = _make_pool(is_sleeping_async=other)
|
||||
|
||||
result = await pool.collective_rpc(0, "reset_prefix_cache", timeout=1.5, args=("x",), kwargs={"k": 1})
|
||||
result = await pool.collective_rpc(0, "is_sleeping", timeout=1.5, args=("x",), kwargs={"k": 1})
|
||||
|
||||
assert result == {"via": "collective"}
|
||||
other.assert_not_awaited()
|
||||
client.collective_rpc_async.assert_awaited_once_with(
|
||||
method="reset_prefix_cache",
|
||||
method="is_sleeping",
|
||||
timeout=1.5,
|
||||
args=("x",),
|
||||
kwargs={"k": 1},
|
||||
@@ -63,7 +66,7 @@ def test_collective_rpc_control_helper_honors_timeout():
|
||||
await asyncio.sleep(1.0)
|
||||
return "slept"
|
||||
|
||||
pool, _client = _make_pool(sleep_async=slow_sleep)
|
||||
pool, _client = _make_pool(sleep_async=AsyncMock(side_effect=slow_sleep))
|
||||
with pytest.raises(asyncio.TimeoutError):
|
||||
await pool.collective_rpc(0, "sleep", timeout=0.01, args=(1,))
|
||||
|
||||
@@ -80,11 +83,38 @@ def test_collective_rpc_control_method_reraises_worker_error():
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
@pytest.mark.cpu
|
||||
def test_collective_rpc_reset_caches_use_control_helpers():
|
||||
async def run() -> None:
|
||||
reset_prefix = AsyncMock(return_value=True)
|
||||
reset_encoder = AsyncMock(return_value=None)
|
||||
reset_mm = AsyncMock(return_value=None)
|
||||
pool, client = _make_pool(
|
||||
reset_prefix_cache_async=reset_prefix,
|
||||
reset_encoder_cache_async=reset_encoder,
|
||||
reset_mm_cache_async=reset_mm,
|
||||
)
|
||||
|
||||
result = await pool.collective_rpc(0, "reset_prefix_cache", args=(True, True), timeout=2.0)
|
||||
assert result is True
|
||||
reset_prefix.assert_awaited_once_with(True, True)
|
||||
|
||||
await pool.collective_rpc(0, "reset_encoder_cache", timeout=2.0)
|
||||
reset_encoder.assert_awaited_once()
|
||||
|
||||
await pool.collective_rpc(0, "reset_mm_cache", timeout=2.0)
|
||||
reset_mm.assert_awaited_once()
|
||||
|
||||
client.collective_rpc_async.assert_not_awaited()
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
@pytest.mark.cpu
|
||||
def test_collective_rpc_non_control_method_still_returns_error_dict():
|
||||
async def run() -> None:
|
||||
pool, _client = _make_pool(collective_rpc_async=AsyncMock(side_effect=RuntimeError("probe failed")))
|
||||
result = await pool.collective_rpc(0, "reset_prefix_cache")
|
||||
result = await pool.collective_rpc(0, "is_sleeping")
|
||||
assert result["supported"] is False
|
||||
assert "probe failed" in result["error"]
|
||||
|
||||
@@ -122,3 +152,28 @@ def test_abort_requests_does_not_commit_op_state_when_engine_abort_fails():
|
||||
abort.assert_awaited_once()
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
@pytest.mark.cpu
|
||||
@pytest.mark.parametrize("method", ["reset_prefix_cache", "reset_encoder_cache", "reset_mm_cache"])
|
||||
@pytest.mark.parametrize("failure", ["error", "timeout", "missing"])
|
||||
def test_cache_reset_failure_is_serialized_and_pool_remains_usable(method, failure):
|
||||
async def run():
|
||||
async def reset():
|
||||
if failure == "timeout":
|
||||
await asyncio.Event().wait()
|
||||
raise RuntimeError("cache reset failed")
|
||||
|
||||
pool, client = _make_pool()
|
||||
if failure != "missing":
|
||||
setattr(client, f"{method}_async", AsyncMock(side_effect=reset))
|
||||
result = await pool.collective_rpc(0, method, timeout=0.01)
|
||||
assert result["supported"] is False
|
||||
assert result["error"]
|
||||
if failure == "timeout":
|
||||
assert "timed out" in result["error"]
|
||||
client.collective_rpc_async.assert_not_awaited()
|
||||
# A subsequent control request still completes on the same pool.
|
||||
assert await pool.collective_rpc(0, "is_sleeping") == {"via": "collective"}
|
||||
|
||||
asyncio.run(run())
|
||||
Reference in new issue
Block a user