[Orchestrator] Implement cache reset RPC forwarding to AR stage engine cores (#6766)
Signed-off-by: Asthenia <asthenia0412@gmail.com> Signed-off-by: Gao Han <hgaoaf@connect.ust.hk> Co-authored-by: Asthenia <asthenia0412@gmail.com> Co-authored-by: Gao Han <hgaoaf@connect.ust.hk>
This commit is contained in:
7 files changed
+497
-65
No files matched your search
@@ -0,0 +1,77 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
"""Real EngineCore cache invalidation through the AsyncOmni control plane."""
|
||||
|
||||
import pytest
|
||||
from PIL import Image
|
||||
from vllm import SamplingParams
|
||||
|
||||
from tests.helpers.stage_config import get_deploy_config_path, modify_stage_config
|
||||
from vllm_omni.entrypoints.async_omni import AsyncOmni
|
||||
|
||||
|
||||
@pytest.mark.advanced_model
|
||||
@pytest.mark.omni
|
||||
@pytest.mark.cuda
|
||||
@pytest.mark.asyncio
|
||||
async def test_cache_resets_invalidate_real_engine_and_allow_repeated_multimodal_inputs():
|
||||
deploy_config = modify_stage_config(
|
||||
get_deploy_config_path("ci/qwen2_5_omni_thinker_only.yaml"),
|
||||
updates={
|
||||
"stages": {
|
||||
0: {
|
||||
"enable_prefix_caching": True,
|
||||
"mm_processor_cache_gb": 0.1,
|
||||
"max_model_len": 2048,
|
||||
"max_num_batched_tokens": 2048,
|
||||
"num_gpu_blocks_override": 256,
|
||||
"enforce_eager": True,
|
||||
}
|
||||
}
|
||||
},
|
||||
)
|
||||
engine = AsyncOmni(
|
||||
model="Qwen/Qwen2.5-Omni-7B",
|
||||
deploy_config=deploy_config,
|
||||
stage_init_timeout=600,
|
||||
)
|
||||
|
||||
async def generate(prompt, request_id):
|
||||
final = None
|
||||
async for output in engine.generate(
|
||||
prompt,
|
||||
request_id=request_id,
|
||||
sampling_params_list=[SamplingParams(max_tokens=4, temperature=0)],
|
||||
):
|
||||
final = output
|
||||
assert final is not None and final.finished
|
||||
assert final.outputs and final.outputs[0].token_ids
|
||||
return final
|
||||
|
||||
try:
|
||||
prompt = "Explain how a cache works. " * 40
|
||||
cold = await generate(prompt, "cold")
|
||||
warm = await generate(prompt, "warm")
|
||||
assert cold.num_cached_tokens == 0
|
||||
assert warm.num_cached_tokens > 0
|
||||
|
||||
assert await engine.reset_prefix_cache(reset_running_requests=True, reset_connector=True)
|
||||
await engine.reset_encoder_cache()
|
||||
await engine.reset_mm_cache()
|
||||
reset = await generate(prompt, "reset")
|
||||
assert reset.num_cached_tokens == 0
|
||||
assert reset.outputs[0].token_ids == cold.outputs[0].token_ids
|
||||
|
||||
image_prompt = {
|
||||
"prompt": "<|im_start|>user\n<|vision_start|><|IMAGE|><|vision_end|>Describe this image.<|im_end|>\n<|im_start|>assistant\n",
|
||||
"multi_modal_data": {"image": Image.new("RGB", (56, 56), "red")},
|
||||
}
|
||||
before = await generate(image_prompt, "image-before")
|
||||
await engine.reset_mm_cache()
|
||||
await engine.reset_encoder_cache()
|
||||
assert await engine.reset_prefix_cache()
|
||||
after = await generate(image_prompt, "image-after")
|
||||
assert after.outputs[0].token_ids == before.outputs[0].token_ids
|
||||
finally:
|
||||
engine.shutdown()
|
||||
@@ -14,7 +14,7 @@ from vllm.renderers import BaseRenderer
|
||||
from vllm.v1.engine.input_processor import InputProcessor
|
||||
|
||||
from vllm_omni.diffusion.data import CuMemTag, OmniACK
|
||||
from vllm_omni.entrypoints.async_omni import AsyncOmni
|
||||
from vllm_omni.entrypoints.async_omni import CACHE_RESET_TIMEOUT_S, AsyncOmni
|
||||
|
||||
pytestmark = [pytest.mark.core_model]
|
||||
|
||||
@@ -31,7 +31,7 @@ def _make_omni(*, stage_types: list[str]) -> AsyncOmni:
|
||||
omni._level2_sleeping = False
|
||||
omni.event_resolver = SimpleNamespace(watch_task=lambda *a, **k: None, resolve=AsyncMock())
|
||||
omni._final_output_handler = lambda: None
|
||||
omni.reset_mm_cache = AsyncMock()
|
||||
omni._clear_frontend_mm_cache = AsyncMock()
|
||||
|
||||
stage_configs = [SimpleNamespace(stage_type=stage_type) for stage_type in stage_types]
|
||||
omni.engine = SimpleNamespace(
|
||||
@@ -46,7 +46,8 @@ def _make_omni(*, stage_types: list[str]) -> AsyncOmni:
|
||||
@pytest.mark.cpu
|
||||
def test_reset_mm_cache_routes_through_renderer(mocker):
|
||||
async def run() -> None:
|
||||
omni = object.__new__(AsyncOmni)
|
||||
omni = _make_omni(stage_types=["llm"])
|
||||
del omni._clear_frontend_mm_cache
|
||||
renderer = mocker.create_autospec(BaseRenderer, instance=True)
|
||||
input_processor = mocker.create_autospec(InputProcessor, instance=True)
|
||||
input_processor.renderer = renderer
|
||||
@@ -115,9 +116,10 @@ def test_pause_generation_still_rpcs_when_already_paused():
|
||||
kwargs={"mode": "abort", "clear_cache": True},
|
||||
stage_ids=[0],
|
||||
)
|
||||
omni.reset_prefix_cache.assert_awaited_once()
|
||||
omni.reset_mm_cache.assert_awaited_once()
|
||||
omni.reset_encoder_cache.assert_awaited_once()
|
||||
omni.reset_prefix_cache.assert_not_awaited()
|
||||
omni.reset_mm_cache.assert_not_awaited()
|
||||
omni.reset_encoder_cache.assert_not_awaited()
|
||||
omni._clear_frontend_mm_cache.assert_awaited_once_with()
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
@@ -142,6 +144,84 @@ def test_resume_generation_resumes_ar_then_clears_frontend_pause():
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
@pytest.mark.cpu
|
||||
def test_reset_prefix_cache_forwards_to_ar_stages():
|
||||
async def run() -> None:
|
||||
omni = _make_omni(stage_types=["llm", "diffusion"])
|
||||
|
||||
result = await omni.reset_prefix_cache(reset_running_requests=True, reset_connector=True)
|
||||
|
||||
assert result is True
|
||||
omni.collective_rpc.assert_awaited_once_with(
|
||||
method="reset_prefix_cache",
|
||||
args=(True, True),
|
||||
kwargs=None,
|
||||
stage_ids=[0],
|
||||
timeout=CACHE_RESET_TIMEOUT_S,
|
||||
)
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
@pytest.mark.cpu
|
||||
def test_reset_prefix_cache_diffusion_only_skips_rpc():
|
||||
async def run() -> None:
|
||||
omni = _make_omni(stage_types=["diffusion"])
|
||||
|
||||
result = await omni.reset_prefix_cache()
|
||||
|
||||
assert result is True
|
||||
omni.collective_rpc.assert_not_awaited()
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
@pytest.mark.cpu
|
||||
def test_reset_encoder_cache_forwards_to_ar_stages():
|
||||
async def run() -> None:
|
||||
omni = _make_omni(stage_types=["llm", "diffusion"])
|
||||
|
||||
await omni.reset_encoder_cache()
|
||||
|
||||
omni.collective_rpc.assert_awaited_once_with(
|
||||
method="reset_encoder_cache",
|
||||
args=(),
|
||||
kwargs=None,
|
||||
stage_ids=[0],
|
||||
timeout=CACHE_RESET_TIMEOUT_S,
|
||||
)
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
@pytest.mark.cpu
|
||||
def test_reset_mm_cache_clears_frontend_then_forwards_to_ar_stages():
|
||||
async def run() -> None:
|
||||
omni = _make_omni(stage_types=["llm", "diffusion"])
|
||||
calls = []
|
||||
renderer = SimpleNamespace(clear_mm_cache_async=AsyncMock(side_effect=lambda: calls.append("frontend")))
|
||||
omni.input_processor = SimpleNamespace(renderer=renderer)
|
||||
del omni._clear_frontend_mm_cache
|
||||
|
||||
async def rpc(**kwargs):
|
||||
calls.append("engine")
|
||||
return [None]
|
||||
|
||||
omni.collective_rpc.side_effect = rpc
|
||||
await omni.reset_mm_cache()
|
||||
|
||||
assert calls == ["frontend", "engine"]
|
||||
omni.collective_rpc.assert_awaited_once_with(
|
||||
method="reset_mm_cache",
|
||||
args=(),
|
||||
kwargs=None,
|
||||
stage_ids=[0],
|
||||
timeout=CACHE_RESET_TIMEOUT_S,
|
||||
)
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
@pytest.mark.cpu
|
||||
def test_sleep_routes_ar_via_collective_rpc_and_diffusion_to_worker_rpc():
|
||||
async def run() -> None:
|
||||
@@ -539,8 +619,9 @@ def test_mixed_engine_keep_sends_engine_core_and_diffusion_pauses():
|
||||
"kwargs": {"mode": "keep"},
|
||||
"stage_ids": [1],
|
||||
}
|
||||
# The frontend cache reset still runs after the backend pauses.
|
||||
omni.reset_prefix_cache.assert_awaited_once()
|
||||
# Only P0 needs clearing here: pause_scheduler already resets P1.
|
||||
omni.reset_prefix_cache.assert_not_awaited()
|
||||
omni._clear_frontend_mm_cache.assert_awaited_once_with()
|
||||
|
||||
await omni.resume_generation()
|
||||
assert _rpc_methods(omni)[2:] == [("resume_scheduler", [0]), ("resume_scheduler", [1])]
|
||||
@@ -579,9 +660,11 @@ def test_keep_then_sleep_wake_resume_keeps_admission_paused_until_resume():
|
||||
omni.reset_prefix_cache = AsyncMock(return_value=True)
|
||||
omni.reset_encoder_cache = AsyncMock()
|
||||
omni.collective_rpc = AsyncMock(
|
||||
side_effect=lambda **kw: [OmniACK(task_id="t", status="SUCCESS", stage_id=0, rank=0)]
|
||||
if kw["method"] in {"handle_sleep_task", "handle_wake_task"}
|
||||
else [None]
|
||||
side_effect=lambda **kw: (
|
||||
[OmniACK(task_id="t", status="SUCCESS", stage_id=0, rank=0)]
|
||||
if kw["method"] in {"handle_sleep_task", "handle_wake_task"}
|
||||
else [None]
|
||||
)
|
||||
)
|
||||
|
||||
await omni.pause_generation(mode="keep", clear_cache=False)
|
||||
@@ -683,3 +766,63 @@ def test_two_ar_stages_partial_resume_keeps_admission_paused():
|
||||
assert await omni.is_paused() is False
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
@pytest.mark.cpu
|
||||
@pytest.mark.parametrize("method", ["reset_prefix_cache", "reset_encoder_cache", "reset_mm_cache"])
|
||||
@pytest.mark.parametrize("stage_ids, targets", [(None, [0, 2]), ([2], [2]), ([1], []), ([], [])])
|
||||
def test_cache_resets_respect_stage_selection(method, stage_ids, targets):
|
||||
async def run():
|
||||
omni = _make_omni(stage_types=["llm", "diffusion", "llm"])
|
||||
await getattr(omni, method)(stage_ids=stage_ids, timeout=2.0)
|
||||
if targets:
|
||||
assert omni.collective_rpc.await_args.kwargs["stage_ids"] == targets
|
||||
assert omni.collective_rpc.await_args.kwargs["timeout"] == 2.0
|
||||
else:
|
||||
omni.collective_rpc.assert_not_awaited()
|
||||
assert omni._clear_frontend_mm_cache.await_count == int(method == "reset_mm_cache" and 0 in targets)
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
@pytest.mark.cpu
|
||||
@pytest.mark.parametrize("method", ["reset_prefix_cache", "reset_encoder_cache", "reset_mm_cache"])
|
||||
@pytest.mark.parametrize("failure", [{"todo": "unsupported"}, {"supported": False}, {"error": "reset failed"}])
|
||||
def test_cache_resets_reject_failure_from_any_replica(method, failure):
|
||||
async def run():
|
||||
omni = _make_omni(stage_types=["llm"])
|
||||
# Exercise the actual collective_rpc too: a single stage can have
|
||||
# multiple replicas, so result indexes are not stage IDs.
|
||||
del omni.collective_rpc
|
||||
omni.engine.collective_rpc_async.return_value = [True, failure]
|
||||
with pytest.raises(RuntimeError, match=method):
|
||||
await getattr(omni, method)()
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
@pytest.mark.cpu
|
||||
def test_prefix_reset_returns_false_if_any_replica_refuses():
|
||||
async def run():
|
||||
omni = _make_omni(stage_types=["llm"])
|
||||
omni.collective_rpc.return_value = [True, [True, False]]
|
||||
assert await omni.reset_prefix_cache() is False
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
@pytest.mark.cpu
|
||||
@pytest.mark.parametrize("operation", ["pause_generation", "sleep"])
|
||||
@pytest.mark.parametrize("stage_ids", [[0], [2]])
|
||||
def test_pause_and_sleep_do_not_reset_unselected_stages_or_reset_twice(operation, stage_ids):
|
||||
async def run():
|
||||
omni = _make_omni(stage_types=["llm", "diffusion", "llm"])
|
||||
kwargs = {"clear_cache": True} if operation == "pause_generation" else {}
|
||||
await getattr(omni, operation)(stage_ids=stage_ids, **kwargs)
|
||||
assert omni.collective_rpc.await_count == 1
|
||||
rpc = omni.collective_rpc.await_args.kwargs
|
||||
assert rpc["method"] == ("pause_scheduler" if operation == "pause_generation" else "sleep")
|
||||
assert rpc["stage_ids"] == stage_ids
|
||||
assert omni._clear_frontend_mm_cache.await_count == int(0 in stage_ids)
|
||||
|
||||
asyncio.run(run())
|
||||
Reference in new issue
Block a user