[Orchestrator] Implement cache reset RPC forwarding to AR stage engine cores (#6766)

Signed-off-by: Asthenia <asthenia0412@gmail.com>
Signed-off-by: Gao Han <hgaoaf@connect.ust.hk>
Co-authored-by: Asthenia <asthenia0412@gmail.com>
Co-authored-by: Gao Han <hgaoaf@connect.ust.hk>
This commit is contained in:
authored and GitHub committed 2026-09-24 15:32:07 +00:00
1 parent 535bd35b42
commit 0295eeb6f3
7 files changed
+497 -65

No files matched your search

@@ -0,0 +1,77 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Real EngineCore cache invalidation through the AsyncOmni control plane."""
import pytest
from PIL import Image
from vllm import SamplingParams
from tests.helpers.stage_config import get_deploy_config_path, modify_stage_config
from vllm_omni.entrypoints.async_omni import AsyncOmni
@pytest.mark.advanced_model
@pytest.mark.omni
@pytest.mark.cuda
@pytest.mark.asyncio
async def test_cache_resets_invalidate_real_engine_and_allow_repeated_multimodal_inputs():
deploy_config = modify_stage_config(
get_deploy_config_path("ci/qwen2_5_omni_thinker_only.yaml"),
updates={
"stages": {
0: {
"enable_prefix_caching": True,
"mm_processor_cache_gb": 0.1,
"max_model_len": 2048,
"max_num_batched_tokens": 2048,
"num_gpu_blocks_override": 256,
"enforce_eager": True,
}
}
},
)
engine = AsyncOmni(
model="Qwen/Qwen2.5-Omni-7B",
deploy_config=deploy_config,
stage_init_timeout=600,
)
async def generate(prompt, request_id):
final = None
async for output in engine.generate(
prompt,
request_id=request_id,
sampling_params_list=[SamplingParams(max_tokens=4, temperature=0)],
):
final = output
assert final is not None and final.finished
assert final.outputs and final.outputs[0].token_ids
return final
try:
prompt = "Explain how a cache works. " * 40
cold = await generate(prompt, "cold")
warm = await generate(prompt, "warm")
assert cold.num_cached_tokens == 0
assert warm.num_cached_tokens > 0
assert await engine.reset_prefix_cache(reset_running_requests=True, reset_connector=True)
await engine.reset_encoder_cache()
await engine.reset_mm_cache()
reset = await generate(prompt, "reset")
assert reset.num_cached_tokens == 0
assert reset.outputs[0].token_ids == cold.outputs[0].token_ids
image_prompt = {
"prompt": "<|im_start|>user\n<|vision_start|><|IMAGE|><|vision_end|>Describe this image.<|im_end|>\n<|im_start|>assistant\n",
"multi_modal_data": {"image": Image.new("RGB", (56, 56), "red")},
}
before = await generate(image_prompt, "image-before")
await engine.reset_mm_cache()
await engine.reset_encoder_cache()
assert await engine.reset_prefix_cache()
after = await generate(image_prompt, "image-after")
assert after.outputs[0].token_ids == before.outputs[0].token_ids
finally:
engine.shutdown()
@@ -14,7 +14,7 @@ from vllm.renderers import BaseRenderer
from vllm.v1.engine.input_processor import InputProcessor
from vllm_omni.diffusion.data import CuMemTag, OmniACK
from vllm_omni.entrypoints.async_omni import AsyncOmni
from vllm_omni.entrypoints.async_omni import CACHE_RESET_TIMEOUT_S, AsyncOmni
pytestmark = [pytest.mark.core_model]
@@ -31,7 +31,7 @@ def _make_omni(*, stage_types: list[str]) -> AsyncOmni:
omni._level2_sleeping = False
omni.event_resolver = SimpleNamespace(watch_task=lambda *a, **k: None, resolve=AsyncMock())
omni._final_output_handler = lambda: None
omni.reset_mm_cache = AsyncMock()
omni._clear_frontend_mm_cache = AsyncMock()
stage_configs = [SimpleNamespace(stage_type=stage_type) for stage_type in stage_types]
omni.engine = SimpleNamespace(
@@ -46,7 +46,8 @@ def _make_omni(*, stage_types: list[str]) -> AsyncOmni:
@pytest.mark.cpu
def test_reset_mm_cache_routes_through_renderer(mocker):
async def run() -> None:
omni = object.__new__(AsyncOmni)
omni = _make_omni(stage_types=["llm"])
del omni._clear_frontend_mm_cache
renderer = mocker.create_autospec(BaseRenderer, instance=True)
input_processor = mocker.create_autospec(InputProcessor, instance=True)
input_processor.renderer = renderer
@@ -115,9 +116,10 @@ def test_pause_generation_still_rpcs_when_already_paused():
kwargs={"mode": "abort", "clear_cache": True},
stage_ids=[0],
)
omni.reset_prefix_cache.assert_awaited_once()
omni.reset_mm_cache.assert_awaited_once()
omni.reset_encoder_cache.assert_awaited_once()
omni.reset_prefix_cache.assert_not_awaited()
omni.reset_mm_cache.assert_not_awaited()
omni.reset_encoder_cache.assert_not_awaited()
omni._clear_frontend_mm_cache.assert_awaited_once_with()
asyncio.run(run())
@@ -142,6 +144,84 @@ def test_resume_generation_resumes_ar_then_clears_frontend_pause():
asyncio.run(run())
@pytest.mark.cpu
def test_reset_prefix_cache_forwards_to_ar_stages():
async def run() -> None:
omni = _make_omni(stage_types=["llm", "diffusion"])
result = await omni.reset_prefix_cache(reset_running_requests=True, reset_connector=True)
assert result is True
omni.collective_rpc.assert_awaited_once_with(
method="reset_prefix_cache",
args=(True, True),
kwargs=None,
stage_ids=[0],
timeout=CACHE_RESET_TIMEOUT_S,
)
asyncio.run(run())
@pytest.mark.cpu
def test_reset_prefix_cache_diffusion_only_skips_rpc():
async def run() -> None:
omni = _make_omni(stage_types=["diffusion"])
result = await omni.reset_prefix_cache()
assert result is True
omni.collective_rpc.assert_not_awaited()
asyncio.run(run())
@pytest.mark.cpu
def test_reset_encoder_cache_forwards_to_ar_stages():
async def run() -> None:
omni = _make_omni(stage_types=["llm", "diffusion"])
await omni.reset_encoder_cache()
omni.collective_rpc.assert_awaited_once_with(
method="reset_encoder_cache",
args=(),
kwargs=None,
stage_ids=[0],
timeout=CACHE_RESET_TIMEOUT_S,
)
asyncio.run(run())
@pytest.mark.cpu
def test_reset_mm_cache_clears_frontend_then_forwards_to_ar_stages():
async def run() -> None:
omni = _make_omni(stage_types=["llm", "diffusion"])
calls = []
renderer = SimpleNamespace(clear_mm_cache_async=AsyncMock(side_effect=lambda: calls.append("frontend")))
omni.input_processor = SimpleNamespace(renderer=renderer)
del omni._clear_frontend_mm_cache
async def rpc(**kwargs):
calls.append("engine")
return [None]
omni.collective_rpc.side_effect = rpc
await omni.reset_mm_cache()
assert calls == ["frontend", "engine"]
omni.collective_rpc.assert_awaited_once_with(
method="reset_mm_cache",
args=(),
kwargs=None,
stage_ids=[0],
timeout=CACHE_RESET_TIMEOUT_S,
)
asyncio.run(run())
@pytest.mark.cpu
def test_sleep_routes_ar_via_collective_rpc_and_diffusion_to_worker_rpc():
async def run() -> None:
@@ -539,8 +619,9 @@ def test_mixed_engine_keep_sends_engine_core_and_diffusion_pauses():
"kwargs": {"mode": "keep"},
"stage_ids": [1],
}
# The frontend cache reset still runs after the backend pauses.
omni.reset_prefix_cache.assert_awaited_once()
# Only P0 needs clearing here: pause_scheduler already resets P1.
omni.reset_prefix_cache.assert_not_awaited()
omni._clear_frontend_mm_cache.assert_awaited_once_with()
await omni.resume_generation()
assert _rpc_methods(omni)[2:] == [("resume_scheduler", [0]), ("resume_scheduler", [1])]
@@ -579,9 +660,11 @@ def test_keep_then_sleep_wake_resume_keeps_admission_paused_until_resume():
omni.reset_prefix_cache = AsyncMock(return_value=True)
omni.reset_encoder_cache = AsyncMock()
omni.collective_rpc = AsyncMock(
side_effect=lambda **kw: [OmniACK(task_id="t", status="SUCCESS", stage_id=0, rank=0)]
if kw["method"] in {"handle_sleep_task", "handle_wake_task"}
else [None]
side_effect=lambda **kw: (
[OmniACK(task_id="t", status="SUCCESS", stage_id=0, rank=0)]
if kw["method"] in {"handle_sleep_task", "handle_wake_task"}
else [None]
)
)
await omni.pause_generation(mode="keep", clear_cache=False)
@@ -683,3 +766,63 @@ def test_two_ar_stages_partial_resume_keeps_admission_paused():
assert await omni.is_paused() is False
asyncio.run(run())
@pytest.mark.cpu
@pytest.mark.parametrize("method", ["reset_prefix_cache", "reset_encoder_cache", "reset_mm_cache"])
@pytest.mark.parametrize("stage_ids, targets", [(None, [0, 2]), ([2], [2]), ([1], []), ([], [])])
def test_cache_resets_respect_stage_selection(method, stage_ids, targets):
async def run():
omni = _make_omni(stage_types=["llm", "diffusion", "llm"])
await getattr(omni, method)(stage_ids=stage_ids, timeout=2.0)
if targets:
assert omni.collective_rpc.await_args.kwargs["stage_ids"] == targets
assert omni.collective_rpc.await_args.kwargs["timeout"] == 2.0
else:
omni.collective_rpc.assert_not_awaited()
assert omni._clear_frontend_mm_cache.await_count == int(method == "reset_mm_cache" and 0 in targets)
asyncio.run(run())
@pytest.mark.cpu
@pytest.mark.parametrize("method", ["reset_prefix_cache", "reset_encoder_cache", "reset_mm_cache"])
@pytest.mark.parametrize("failure", [{"todo": "unsupported"}, {"supported": False}, {"error": "reset failed"}])
def test_cache_resets_reject_failure_from_any_replica(method, failure):
async def run():
omni = _make_omni(stage_types=["llm"])
# Exercise the actual collective_rpc too: a single stage can have
# multiple replicas, so result indexes are not stage IDs.
del omni.collective_rpc
omni.engine.collective_rpc_async.return_value = [True, failure]
with pytest.raises(RuntimeError, match=method):
await getattr(omni, method)()
asyncio.run(run())
@pytest.mark.cpu
def test_prefix_reset_returns_false_if_any_replica_refuses():
async def run():
omni = _make_omni(stage_types=["llm"])
omni.collective_rpc.return_value = [True, [True, False]]
assert await omni.reset_prefix_cache() is False
asyncio.run(run())
@pytest.mark.cpu
@pytest.mark.parametrize("operation", ["pause_generation", "sleep"])
@pytest.mark.parametrize("stage_ids", [[0], [2]])
def test_pause_and_sleep_do_not_reset_unselected_stages_or_reset_twice(operation, stage_ids):
async def run():
omni = _make_omni(stage_types=["llm", "diffusion", "llm"])
kwargs = {"clear_cache": True} if operation == "pause_generation" else {}
await getattr(omni, operation)(stage_ids=stage_ids, **kwargs)
assert omni.collective_rpc.await_count == 1
rpc = omni.collective_rpc.await_args.kwargs
assert rpc["method"] == ("pause_scheduler" if operation == "pause_generation" else "sleep")
assert rpc["stage_ids"] == stage_ids
assert omni._clear_frontend_mm_cache.await_count == int(0 in stage_ids)
asyncio.run(run())