[Feature]: Support pause / resume and sleep / wake for AR stages in AsyncOmni (#6084)
Signed-off-by: knlnguyen1802 <knlnguyen1802@gmail.com> Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
1 parent
072bfc02dd
commit
d15b7ca3e1
12 files changed
+1024
-69
No files matched your search
@@ -0,0 +1,156 @@
|
||||
"""Unit tests for acknowledged abort on AsyncOmniEngine."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import queue
|
||||
import threading
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
from pytest_mock import MockerFixture
|
||||
|
||||
from vllm_omni.engine.async_omni_engine import AsyncOmniEngine
|
||||
from vllm_omni.engine.messages import AbortRequestMessage, AbortResultMessage
|
||||
from vllm_omni.engine.rpc_result_router import CorrelatedRpcClient
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
|
||||
def _make_engine(request_q: queue.Queue, rpc_q: queue.Queue) -> AsyncOmniEngine:
|
||||
engine = object.__new__(AsyncOmniEngine)
|
||||
engine.request_queue = SimpleNamespace(sync_q=request_q)
|
||||
engine.rpc_output_queue = SimpleNamespace(sync_q=rpc_q)
|
||||
engine._correlated_rpc_client = CorrelatedRpcClient(request_q, rpc_q)
|
||||
return engine
|
||||
|
||||
|
||||
def test_abort_result_message_correlation_key():
|
||||
msg = AbortResultMessage(rpc_id="abc", success=True)
|
||||
assert msg.rpc_correlation_key == ("abort", "abc")
|
||||
|
||||
|
||||
def test_sync_abort_is_fire_and_forget():
|
||||
request_q: queue.Queue = queue.Queue()
|
||||
rpc_q: queue.Queue = queue.Queue()
|
||||
engine = _make_engine(request_q, rpc_q)
|
||||
|
||||
engine.abort(["req-1"])
|
||||
|
||||
msg = request_q.get_nowait()
|
||||
assert isinstance(msg, AbortRequestMessage)
|
||||
assert msg.request_ids == ["req-1"]
|
||||
assert msg.rpc_id is None
|
||||
assert rpc_q.empty()
|
||||
engine._correlated_rpc_client.close()
|
||||
|
||||
|
||||
def test_abort_async_waits_for_ack(mocker: MockerFixture):
|
||||
request_q: queue.Queue = queue.Queue()
|
||||
rpc_q: queue.Queue = queue.Queue()
|
||||
engine = _make_engine(request_q, rpc_q)
|
||||
mocker.patch(
|
||||
"vllm_omni.engine.async_omni_engine.uuid.uuid4",
|
||||
return_value=SimpleNamespace(hex="abort-rpc-1"),
|
||||
)
|
||||
|
||||
async def _run() -> None:
|
||||
task = asyncio.create_task(engine.abort_async(["req-a", "req-b"], timeout=1))
|
||||
msg = await asyncio.to_thread(request_q.get, True, 1)
|
||||
assert isinstance(msg, AbortRequestMessage)
|
||||
assert msg.request_ids == ["req-a", "req-b"]
|
||||
assert msg.rpc_id == "abort-rpc-1"
|
||||
assert not task.done()
|
||||
|
||||
rpc_q.put(AbortResultMessage(rpc_id="abort-rpc-1", success=True))
|
||||
await task
|
||||
|
||||
try:
|
||||
asyncio.run(_run())
|
||||
finally:
|
||||
engine._correlated_rpc_client.close()
|
||||
|
||||
|
||||
def test_abort_async_raises_on_orchestrator_error(mocker: MockerFixture):
|
||||
request_q: queue.Queue = queue.Queue()
|
||||
rpc_q: queue.Queue = queue.Queue()
|
||||
engine = _make_engine(request_q, rpc_q)
|
||||
mocker.patch(
|
||||
"vllm_omni.engine.async_omni_engine.uuid.uuid4",
|
||||
return_value=SimpleNamespace(hex="abort-rpc-err"),
|
||||
)
|
||||
|
||||
async def _run() -> None:
|
||||
task = asyncio.create_task(engine.abort_async(["req-x"], timeout=1))
|
||||
await asyncio.to_thread(request_q.get, True, 1)
|
||||
rpc_q.put(
|
||||
AbortResultMessage(
|
||||
rpc_id="abort-rpc-err",
|
||||
success=False,
|
||||
error="stage abort failed",
|
||||
)
|
||||
)
|
||||
with pytest.raises(RuntimeError, match="stage abort failed"):
|
||||
await task
|
||||
|
||||
try:
|
||||
asyncio.run(_run())
|
||||
finally:
|
||||
engine._correlated_rpc_client.close()
|
||||
|
||||
|
||||
def test_abort_async_times_out_without_result(mocker: MockerFixture):
|
||||
request_q: queue.Queue = queue.Queue()
|
||||
rpc_q: queue.Queue = queue.Queue()
|
||||
engine = _make_engine(request_q, rpc_q)
|
||||
mocker.patch(
|
||||
"vllm_omni.engine.async_omni_engine.uuid.uuid4",
|
||||
return_value=SimpleNamespace(hex="abort-rpc-timeout"),
|
||||
)
|
||||
|
||||
async def _run() -> None:
|
||||
with pytest.raises(TimeoutError, match="abort timed out"):
|
||||
await engine.abort_async(["req-timeout"], timeout=0.05)
|
||||
|
||||
try:
|
||||
asyncio.run(_run())
|
||||
finally:
|
||||
engine._correlated_rpc_client.close()
|
||||
|
||||
|
||||
def test_abort_async_preserves_request_queue_backpressure(mocker: MockerFixture):
|
||||
class SignallingQueue(queue.Queue):
|
||||
def __init__(self) -> None:
|
||||
super().__init__(maxsize=1)
|
||||
self.put_attempted = threading.Event()
|
||||
|
||||
def put(self, item, block=True, timeout=None):
|
||||
self.put_attempted.set()
|
||||
return super().put(item, block=block, timeout=timeout)
|
||||
|
||||
request_q = SignallingQueue()
|
||||
request_q.put("queue-is-full")
|
||||
request_q.put_attempted.clear()
|
||||
rpc_q: queue.Queue = queue.Queue()
|
||||
engine = _make_engine(request_q, rpc_q)
|
||||
mocker.patch(
|
||||
"vllm_omni.engine.async_omni_engine.uuid.uuid4",
|
||||
return_value=SimpleNamespace(hex="blocked-abort"),
|
||||
)
|
||||
|
||||
async def _run() -> None:
|
||||
task = asyncio.create_task(engine.abort_async(["req-blocked"], timeout=1))
|
||||
assert await asyncio.to_thread(request_q.put_attempted.wait, 1)
|
||||
assert not task.done()
|
||||
|
||||
assert request_q.get(timeout=1) == "queue-is-full"
|
||||
msg = await asyncio.to_thread(request_q.get, True, 1)
|
||||
assert isinstance(msg, AbortRequestMessage)
|
||||
assert msg.rpc_id == "blocked-abort"
|
||||
rpc_q.put(AbortResultMessage(rpc_id="blocked-abort", success=True))
|
||||
await task
|
||||
|
||||
try:
|
||||
asyncio.run(_run())
|
||||
finally:
|
||||
engine._correlated_rpc_client.close()
|
||||
@@ -19,6 +19,7 @@ from vllm.v1.metrics.stats import IterationStats
|
||||
|
||||
from vllm_omni.engine.messages import (
|
||||
AbortRequestMessage,
|
||||
AbortResultMessage,
|
||||
AddCompanionRequestMessage,
|
||||
CollectiveRPCRequestMessage,
|
||||
CollectiveRPCResultMessage,
|
||||
@@ -805,10 +806,74 @@ async def test_run_abort(orchestrator_factory) -> None:
|
||||
assert stages[0].abort_calls == [["req-abort"]]
|
||||
assert stages[1].abort_calls == []
|
||||
assert "req-abort" not in orchestrator_fixture.orchestrator.request_states
|
||||
# Fire-and-forget abort must not emit an RPC result.
|
||||
assert orchestrator_fixture.queues[2].sync_q.empty()
|
||||
finally:
|
||||
await _shutdown_orchestrator(orchestrator_fixture)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_abort_emits_result_when_rpc_id_set(orchestrator_factory) -> None:
|
||||
stages = [
|
||||
FakeStageClient(stage_type="llm", final_output=False),
|
||||
FakeStageClient(stage_type="llm", final_output=True),
|
||||
]
|
||||
processors = [
|
||||
FakeOutputProcessor(request_outputs=[_build_request_output("req-ack", token_ids=[1], finished=True)]),
|
||||
FakeOutputProcessor(request_outputs=[_build_request_output("req-ack", token_ids=[2], finished=True)]),
|
||||
]
|
||||
orchestrator_fixture = orchestrator_factory(stages, output_processors=processors)
|
||||
request = SimpleNamespace(request_id="req-ack", prompt_token_ids=[1, 2, 3])
|
||||
|
||||
try:
|
||||
await _enqueue_add_request(
|
||||
orchestrator_fixture,
|
||||
request_id="req-ack",
|
||||
prompt=request,
|
||||
original_prompt={"prompt": "cancel with ack"},
|
||||
sampling_params_list=[_sampling_params(), _sampling_params()],
|
||||
final_stage_id=1,
|
||||
)
|
||||
await _wait_for(lambda: len(stages[0].add_request_calls) == 1)
|
||||
|
||||
orchestrator_fixture.request_sync_q.put_nowait(AbortRequestMessage(request_ids=["req-ack"], rpc_id="abort-1"))
|
||||
result = await _get_rpc_message(orchestrator_fixture)
|
||||
assert isinstance(result, AbortResultMessage)
|
||||
assert result.rpc_id == "abort-1"
|
||||
assert result.success is True
|
||||
assert result.error is None
|
||||
assert result.rpc_correlation_key == ("abort", "abort-1")
|
||||
assert stages[0].abort_calls == [["req-ack"]]
|
||||
assert "req-ack" not in orchestrator_fixture.orchestrator.request_states
|
||||
finally:
|
||||
await _shutdown_orchestrator(orchestrator_fixture)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_handle_abort_emits_error_result_on_failure() -> None:
|
||||
rpc_queue: asyncio.Queue = asyncio.Queue()
|
||||
orchestrator = Orchestrator(
|
||||
request_async_queue=asyncio.Queue(),
|
||||
output_async_queue=asyncio.Queue(),
|
||||
rpc_async_queue=rpc_queue,
|
||||
stage_pools=[],
|
||||
)
|
||||
|
||||
async def boom(_request_ids, *, abort=False):
|
||||
del _request_ids, abort
|
||||
raise RuntimeError("cleanup exploded")
|
||||
|
||||
orchestrator._cleanup_request_ids = boom # type: ignore[method-assign]
|
||||
|
||||
await orchestrator._handle_abort(AbortRequestMessage(request_ids=["req-fail"], rpc_id="abort-err"))
|
||||
|
||||
result = rpc_queue.get_nowait()
|
||||
assert isinstance(result, AbortResultMessage)
|
||||
assert result.rpc_id == "abort-err"
|
||||
assert result.success is False
|
||||
assert "cleanup exploded" in (result.error or "")
|
||||
|
||||
|
||||
def _duplex_open_message(
|
||||
session_id: str,
|
||||
*,
|
||||
|
||||
@@ -4,6 +4,7 @@ from typing import Literal
|
||||
import pytest
|
||||
|
||||
from vllm_omni.engine.messages import (
|
||||
AbortResultMessage,
|
||||
CollectiveRPCResultMessage,
|
||||
EngineQueueMessage,
|
||||
ErrorMessage,
|
||||
@@ -93,6 +94,19 @@ def test_rpc_result_router_does_not_broadcast_uncorrelated_nonfatal_errors():
|
||||
router.close()
|
||||
|
||||
|
||||
def test_rpc_result_router_routes_abort_results_by_correlation_id():
|
||||
source: queue.Queue = queue.Queue()
|
||||
router = RpcResultRouter(source)
|
||||
waiter = router.register(("abort", "abort-42"))
|
||||
|
||||
source.put(AbortResultMessage(rpc_id="abort-42", success=True))
|
||||
|
||||
result = waiter.get(timeout=1)
|
||||
assert isinstance(result, AbortResultMessage)
|
||||
assert result.rpc_id == "abort-42"
|
||||
router.close()
|
||||
|
||||
|
||||
def test_rpc_result_router_close_unblocks_waiters_and_stops_consumer():
|
||||
source: queue.Queue = queue.Queue()
|
||||
router = RpcResultRouter(source)
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
"""Unit tests for StagePool.collective_rpc EngineCore control dispatch."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm_omni.engine.stage_pool import StagePool
|
||||
|
||||
pytestmark = [pytest.mark.core_model]
|
||||
|
||||
|
||||
def _make_pool(*, stage_type: str = "llm", **client_methods: AsyncMock) -> tuple[StagePool, SimpleNamespace]:
|
||||
client = SimpleNamespace(stage_type=stage_type, **client_methods)
|
||||
if "collective_rpc_async" not in client_methods:
|
||||
client.collective_rpc_async = AsyncMock(return_value={"via": "collective"})
|
||||
pool = StagePool(0, [client]) # type: ignore[arg-type]
|
||||
return pool, client
|
||||
|
||||
|
||||
@pytest.mark.cpu
|
||||
def test_collective_rpc_normalizes_none_args_on_control_helper():
|
||||
async def run() -> None:
|
||||
pause = AsyncMock(return_value="paused")
|
||||
pool, client = _make_pool(pause_scheduler_async=pause)
|
||||
|
||||
result = await pool.collective_rpc(0, "pause_scheduler", args=None, kwargs={"mode": "abort"})
|
||||
|
||||
assert result == "paused"
|
||||
pause.assert_awaited_once_with(mode="abort")
|
||||
client.collective_rpc_async.assert_not_awaited()
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
@pytest.mark.cpu
|
||||
def test_collective_rpc_unrelated_async_helper_uses_collective_path():
|
||||
async def run() -> None:
|
||||
other = AsyncMock(return_value="should-not-run")
|
||||
pool, client = _make_pool(reset_prefix_cache_async=other)
|
||||
|
||||
result = await pool.collective_rpc(0, "reset_prefix_cache", timeout=1.5, args=("x",), kwargs={"k": 1})
|
||||
|
||||
assert result == {"via": "collective"}
|
||||
other.assert_not_awaited()
|
||||
client.collective_rpc_async.assert_awaited_once_with(
|
||||
method="reset_prefix_cache",
|
||||
timeout=1.5,
|
||||
args=("x",),
|
||||
kwargs={"k": 1},
|
||||
)
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
@pytest.mark.cpu
|
||||
def test_collective_rpc_control_helper_honors_timeout():
|
||||
async def run() -> None:
|
||||
async def slow_sleep(*_args, **_kwargs):
|
||||
await asyncio.sleep(1.0)
|
||||
return "slept"
|
||||
|
||||
pool, _client = _make_pool(sleep_async=slow_sleep)
|
||||
result = await pool.collective_rpc(0, "sleep", timeout=0.01, args=(1,))
|
||||
assert result != "slept"
|
||||
assert result["supported"] is False
|
||||
|
||||
asyncio.run(run())
|
||||
@@ -185,6 +185,58 @@ def test_abort_handles_internal_request_mapping(req_ids: list[str], cancel_prefi
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
@pytest.mark.cpu
|
||||
def test_abort_pops_request_states_only_after_ack():
|
||||
async def run():
|
||||
release = asyncio.Event()
|
||||
seen_during_wait: list[int] = []
|
||||
|
||||
async def slow_abort_async(request_ids):
|
||||
del request_ids
|
||||
seen_during_wait.append(len(omni.request_states))
|
||||
await release.wait()
|
||||
|
||||
omni = get_async_omni_instance(fake_abort_request=slow_abort_async)
|
||||
omni.request_states["req-1-aaaa"] = SimpleNamespace(
|
||||
request_id="req-1-aaaa",
|
||||
external_request_id="req-1",
|
||||
input_stream_task=None,
|
||||
)
|
||||
|
||||
task = asyncio.create_task(omni.abort("req-1"))
|
||||
await asyncio.sleep(0)
|
||||
assert not task.done()
|
||||
assert "req-1-aaaa" in omni.request_states
|
||||
assert seen_during_wait == [1]
|
||||
|
||||
release.set()
|
||||
await task
|
||||
assert "req-1-aaaa" not in omni.request_states
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
@pytest.mark.cpu
|
||||
def test_abort_propagates_engine_errors_without_popping_state():
|
||||
async def run():
|
||||
async def failing_abort_async(request_ids):
|
||||
del request_ids
|
||||
raise RuntimeError("orchestrator abort failed")
|
||||
|
||||
omni = get_async_omni_instance(fake_abort_request=failing_abort_async)
|
||||
omni.request_states["req-1-bbbb"] = SimpleNamespace(
|
||||
request_id="req-1-bbbb",
|
||||
external_request_id="req-1",
|
||||
input_stream_task=None,
|
||||
)
|
||||
|
||||
with pytest.raises(RuntimeError, match="orchestrator abort failed"):
|
||||
await omni.abort("req-1")
|
||||
assert "req-1-bbbb" in omni.request_states
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
@pytest.mark.cpu
|
||||
def test_generate_accepts_request_after_repeated_cancellations():
|
||||
async def run_test():
|
||||
|
||||
@@ -0,0 +1,310 @@
|
||||
"""Unit tests for AR EngineCore vs diffusion worker pause/sleep routing."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock, call
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm_omni.diffusion.data import CuMemTag, OmniACK
|
||||
from vllm_omni.entrypoints.async_omni import AsyncOmni
|
||||
|
||||
pytestmark = [pytest.mark.core_model]
|
||||
|
||||
|
||||
def _make_omni(*, stage_types: list[str]) -> AsyncOmni:
|
||||
omni = object.__new__(AsyncOmni)
|
||||
omni._pause_cond = asyncio.Condition()
|
||||
omni._paused = False
|
||||
omni._hold_admission_until_resume = False
|
||||
omni._sleeping_tags = set()
|
||||
omni._level2_sleeping = False
|
||||
omni.event_resolver = SimpleNamespace(watch_task=lambda *a, **k: None, resolve=AsyncMock())
|
||||
omni._final_output_handler = lambda: None
|
||||
|
||||
stage_clients = [SimpleNamespace(stage_type=stage_type) for stage_type in stage_types]
|
||||
omni.engine = SimpleNamespace(
|
||||
stage_clients=stage_clients,
|
||||
stage_vllm_configs=[None] * len(stage_types),
|
||||
collective_rpc_async=AsyncMock(return_value=[True]),
|
||||
)
|
||||
omni.collective_rpc = AsyncMock(return_value=[True])
|
||||
return omni
|
||||
|
||||
|
||||
@pytest.mark.cpu
|
||||
def test_split_stage_ids_by_type():
|
||||
omni = _make_omni(stage_types=["llm", "diffusion", "llm"])
|
||||
ar_ids, diff_ids = omni._split_stage_ids_by_type()
|
||||
assert ar_ids == [0, 2]
|
||||
assert diff_ids == [1]
|
||||
|
||||
|
||||
@pytest.mark.cpu
|
||||
def test_split_stage_ids_by_type_rejects_out_of_range():
|
||||
omni = _make_omni(stage_types=["llm", "diffusion"])
|
||||
with pytest.raises(ValueError, match=r"Invalid stage_ids \[2\].*0\.\.1"):
|
||||
omni._split_stage_ids_by_type([0, 2])
|
||||
with pytest.raises(ValueError, match=r"Invalid stage_ids \[-1\]"):
|
||||
omni._split_stage_ids_by_type([-1])
|
||||
|
||||
|
||||
@pytest.mark.cpu
|
||||
def test_pause_generation_routes_ar_via_collective_rpc():
|
||||
async def run() -> None:
|
||||
omni = _make_omni(stage_types=["llm", "diffusion"])
|
||||
omni.reset_prefix_cache = AsyncMock(return_value=True)
|
||||
omni.reset_mm_cache = AsyncMock()
|
||||
omni.reset_encoder_cache = AsyncMock()
|
||||
|
||||
await omni.pause_generation(mode="abort", clear_cache=True)
|
||||
|
||||
assert omni._paused is True
|
||||
assert omni._hold_admission_until_resume is True
|
||||
omni.collective_rpc.assert_awaited_once_with(
|
||||
method="pause_scheduler",
|
||||
args=(),
|
||||
kwargs={"mode": "abort", "clear_cache": True},
|
||||
stage_ids=[0],
|
||||
)
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
@pytest.mark.cpu
|
||||
def test_pause_generation_still_rpcs_when_already_paused():
|
||||
async def run() -> None:
|
||||
omni = _make_omni(stage_types=["llm", "diffusion"])
|
||||
omni.reset_prefix_cache = AsyncMock(return_value=True)
|
||||
omni.reset_mm_cache = AsyncMock()
|
||||
omni.reset_encoder_cache = AsyncMock()
|
||||
omni._paused = True
|
||||
|
||||
await omni.pause_generation(mode="abort", clear_cache=True, stage_ids=[0])
|
||||
|
||||
omni.collective_rpc.assert_awaited_once_with(
|
||||
method="pause_scheduler",
|
||||
args=(),
|
||||
kwargs={"mode": "abort", "clear_cache": True},
|
||||
stage_ids=[0],
|
||||
)
|
||||
omni.reset_prefix_cache.assert_awaited_once()
|
||||
omni.reset_mm_cache.assert_awaited_once()
|
||||
omni.reset_encoder_cache.assert_awaited_once()
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
@pytest.mark.cpu
|
||||
def test_resume_generation_resumes_ar_then_clears_frontend_pause():
|
||||
async def run() -> None:
|
||||
omni = _make_omni(stage_types=["llm", "diffusion"])
|
||||
omni._paused = True
|
||||
|
||||
await omni.resume_generation()
|
||||
|
||||
omni.collective_rpc.assert_awaited_once_with(
|
||||
method="resume_scheduler",
|
||||
args=(),
|
||||
kwargs=None,
|
||||
stage_ids=[0],
|
||||
)
|
||||
assert omni._paused is False
|
||||
assert omni._hold_admission_until_resume is False
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
@pytest.mark.cpu
|
||||
def test_sleep_routes_ar_via_collective_rpc_and_diffusion_to_worker_rpc():
|
||||
async def run() -> None:
|
||||
omni = _make_omni(stage_types=["llm", "diffusion"])
|
||||
diffusion_ack = OmniACK(task_id="diff", status="SUCCESS", stage_id=1, rank=0)
|
||||
omni._sleep_diffusion = AsyncMock(return_value=[diffusion_ack])
|
||||
|
||||
acks = await omni.sleep(stage_ids=[0, 1], level=1, mode="abort")
|
||||
|
||||
omni.collective_rpc.assert_awaited_once_with(
|
||||
method="sleep",
|
||||
args=(1, "abort"),
|
||||
kwargs=None,
|
||||
stage_ids=[0],
|
||||
)
|
||||
omni._sleep_diffusion.assert_awaited_once_with([1], 1)
|
||||
assert {ack.stage_id for ack in acks} == {0, 1}
|
||||
assert any(ack.metadata.get("path") == "engine_core" for ack in acks if ack.stage_id == 0)
|
||||
assert CuMemTag.WEIGHTS.value in omni._sleeping_tags
|
||||
assert CuMemTag.KV_CACHE.value in omni._sleeping_tags
|
||||
# Sleep gates frontend admission for the trainer resume contract.
|
||||
assert omni._paused is True
|
||||
assert omni._hold_admission_until_resume is True
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
@pytest.mark.cpu
|
||||
def test_sleep_blocks_admission_before_engine_core_rpc():
|
||||
"""Problem 3: _paused must be set before awaiting sleep RPC."""
|
||||
|
||||
async def run() -> None:
|
||||
omni = _make_omni(stage_types=["llm"])
|
||||
paused_at_rpc: list[bool] = []
|
||||
|
||||
async def rpc_side_effect(**kwargs):
|
||||
if kwargs.get("method") == "sleep":
|
||||
paused_at_rpc.append(omni._paused)
|
||||
return [True]
|
||||
|
||||
omni.collective_rpc = AsyncMock(side_effect=rpc_side_effect)
|
||||
|
||||
assert omni._paused is False
|
||||
await omni.sleep(level=1, mode="abort")
|
||||
|
||||
assert paused_at_rpc == [True]
|
||||
assert omni._paused is True
|
||||
assert omni._hold_admission_until_resume is True
|
||||
# Sleep must not also call pause_scheduler (EngineCore.sleep pauses).
|
||||
assert all(c.kwargs.get("method") != "pause_scheduler" for c in omni.collective_rpc.await_args_list)
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
@pytest.mark.cpu
|
||||
def test_wake_up_routes_ar_via_collective_rpc_and_diffusion_to_worker_rpc():
|
||||
async def run() -> None:
|
||||
omni = _make_omni(stage_types=["llm", "diffusion"])
|
||||
omni._paused = True
|
||||
omni._hold_admission_until_resume = True
|
||||
omni._sleeping_tags = {CuMemTag.WEIGHTS.value, CuMemTag.KV_CACHE.value}
|
||||
diffusion_ack = OmniACK(task_id="diff", status="SUCCESS", stage_id=1, rank=0)
|
||||
omni._wake_diffusion = AsyncMock(return_value=[diffusion_ack])
|
||||
|
||||
acks = await omni.wake_up(stage_ids=[0, 1])
|
||||
|
||||
omni.collective_rpc.assert_awaited_once()
|
||||
wake_kwargs = omni.collective_rpc.await_args.kwargs
|
||||
assert wake_kwargs["method"] == "wake_up"
|
||||
assert wake_kwargs["stage_ids"] == [0]
|
||||
assert set(wake_kwargs["kwargs"]["tags"]) == {
|
||||
CuMemTag.WEIGHTS.value,
|
||||
CuMemTag.KV_CACHE.value,
|
||||
}
|
||||
omni._wake_diffusion.assert_awaited_once()
|
||||
assert {ack.stage_id for ack in acks} == {0, 1}
|
||||
assert not omni._sleeping_tags
|
||||
# Mixed/AR wake restores memory but does not resume frontend admission.
|
||||
assert omni._paused is True
|
||||
assert omni._hold_admission_until_resume is True
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
@pytest.mark.cpu
|
||||
def test_wake_up_does_not_resume_frontend_admission():
|
||||
async def run() -> None:
|
||||
omni = _make_omni(stage_types=["llm"])
|
||||
await omni.sleep(level=1, mode="abort")
|
||||
assert omni._paused is True
|
||||
assert omni._hold_admission_until_resume is True
|
||||
|
||||
await omni.wake_up()
|
||||
|
||||
assert omni._paused is True
|
||||
assert omni._hold_admission_until_resume is True
|
||||
assert not omni._sleeping_tags
|
||||
# Explicit resume is required after AR sleep/wake.
|
||||
await omni.resume_generation()
|
||||
assert omni._paused is False
|
||||
assert omni._hold_admission_until_resume is False
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
@pytest.mark.cpu
|
||||
def test_sleep_level1_wake_without_tags_clears_all_sleeping_tags():
|
||||
"""sleep(level=1) tracks WEIGHTS+KV; untagged wake_up must clear both."""
|
||||
|
||||
async def run() -> None:
|
||||
omni = _make_omni(stage_types=["llm"])
|
||||
|
||||
await omni.sleep(level=1, mode="abort")
|
||||
assert CuMemTag.WEIGHTS.value in omni._sleeping_tags
|
||||
assert CuMemTag.KV_CACHE.value in omni._sleeping_tags
|
||||
|
||||
await omni.wake_up()
|
||||
assert not omni._sleeping_tags
|
||||
assert omni._paused is True
|
||||
assert omni._hold_admission_until_resume is True
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
@pytest.mark.cpu
|
||||
def test_sleep_diffusion_only_skips_engine_core_collective_rpc():
|
||||
async def run() -> None:
|
||||
omni = _make_omni(stage_types=["diffusion"])
|
||||
omni._sleep_diffusion = AsyncMock(return_value=[OmniACK(task_id="d", status="SUCCESS", stage_id=0, rank=0)])
|
||||
|
||||
await omni.sleep(level=1)
|
||||
|
||||
# AR EngineCore sleep path must not run for diffusion-only.
|
||||
assert call(method="sleep", args=(1, "abort"), kwargs=None, stage_ids=[0]) not in (
|
||||
omni.collective_rpc.await_args_list
|
||||
)
|
||||
omni._sleep_diffusion.assert_awaited_once_with([0], 1)
|
||||
assert omni._paused is True
|
||||
assert omni._hold_admission_until_resume is False
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
@pytest.mark.cpu
|
||||
def test_wake_up_restores_admission_for_diffusion_only():
|
||||
"""Pure diffusion must keep sleep → wake → generate (no resume)."""
|
||||
|
||||
async def run() -> None:
|
||||
omni = _make_omni(stage_types=["diffusion"])
|
||||
omni._sleep_diffusion = AsyncMock(return_value=[OmniACK(task_id="d", status="SUCCESS", stage_id=0, rank=0)])
|
||||
omni._wake_diffusion = AsyncMock(return_value=[OmniACK(task_id="w", status="SUCCESS", stage_id=0, rank=0)])
|
||||
|
||||
await omni.sleep(level=1)
|
||||
assert omni._paused is True
|
||||
|
||||
async def wait_for_admission() -> None:
|
||||
async with omni._pause_cond:
|
||||
await omni._pause_cond.wait_for(lambda: not omni._paused)
|
||||
|
||||
waiter = asyncio.create_task(wait_for_admission())
|
||||
await omni.wake_up()
|
||||
await asyncio.wait_for(waiter, timeout=1.0)
|
||||
assert omni._paused is False
|
||||
assert omni._hold_admission_until_resume is False
|
||||
assert not omni._sleeping_tags
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
@pytest.mark.cpu
|
||||
def test_pause_then_sleep_wake_keeps_admission_paused_for_diffusion():
|
||||
"""Explicit pause_generation still requires resume after diffusion wake."""
|
||||
|
||||
async def run() -> None:
|
||||
omni = _make_omni(stage_types=["diffusion"])
|
||||
omni.reset_prefix_cache = AsyncMock(return_value=True)
|
||||
omni.reset_mm_cache = AsyncMock()
|
||||
omni.reset_encoder_cache = AsyncMock()
|
||||
omni._sleep_diffusion = AsyncMock(return_value=[OmniACK(task_id="d", status="SUCCESS", stage_id=0, rank=0)])
|
||||
omni._wake_diffusion = AsyncMock(return_value=[OmniACK(task_id="w", status="SUCCESS", stage_id=0, rank=0)])
|
||||
|
||||
await omni.pause_generation()
|
||||
await omni.sleep(level=1)
|
||||
await omni.wake_up()
|
||||
|
||||
assert omni._paused is True
|
||||
assert omni._hold_admission_until_resume is True
|
||||
await omni.resume_generation()
|
||||
assert omni._paused is False
|
||||
|
||||
asyncio.run(run())
|
||||
@@ -114,6 +114,10 @@ async def _ensure_awake(engine: AsyncOmni, stage_ids: list[int]) -> None:
|
||||
await engine.wake_up(stage_ids=stage_ids)
|
||||
except Exception as e:
|
||||
logger.warning("ensure_awake failed (stage_ids=%s): %s", stage_ids, e)
|
||||
try:
|
||||
await engine.resume_generation(stage_ids=stage_ids)
|
||||
except Exception as e:
|
||||
logger.warning("ensure_resume failed (stage_ids=%s): %s", stage_ids, e)
|
||||
|
||||
|
||||
def _build_llm_stages() -> tuple[list[dict], list[dict]]:
|
||||
@@ -240,6 +244,9 @@ class TestOmniLlmSleepMode:
|
||||
try:
|
||||
await llm_engine.sleep(stage_ids=[0], level=1)
|
||||
await llm_engine.wake_up(stage_ids=[0], tags=["weights"])
|
||||
# wake_up restores weights but keeps the AR admission hold.
|
||||
# Resume so generate() can reach the sleeping-tags rejection.
|
||||
await llm_engine.resume_generation(stage_ids=[0])
|
||||
with pytest.raises(RuntimeError, match="partially or fully asleep"):
|
||||
async for _ in llm_engine.generate("test", sampling_params=SamplingParams(max_tokens=4)):
|
||||
pass
|
||||
@@ -490,6 +497,7 @@ async def test_multistage_llm_diffusion_sleep_wake(tp_size: int):
|
||||
assert len(acks) == 2
|
||||
|
||||
await engine.wake_up(stage_ids=[0, 1])
|
||||
await engine.resume_generation(stage_ids=[0, 1])
|
||||
async for _ in engine.generate("verify", sampling_params_list=[SamplingParams(), sp]):
|
||||
pass
|
||||
finally:
|
||||
|
||||
Reference in new issue
Block a user