[Core][Diffusion] Add pause_generation(mode="keep") for diffusion stages (#7685)

Signed-off-by: cr-gao <gaochenrui@sjtu.edu.cn>
This commit is contained in:
vOv authored and GitHub committed 2026-09-22 01:31:22 +08:00
1 parent 62b15142bd
commit d8d162d810
17 files changed
+1383 -65

No files matched your search

@@ -0,0 +1,81 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""
Two-rank ``pause_generation(mode="keep")``: every rank runs the pause barrier
before the ACK, and a barrier failure on a non-zero rank fails the pause
without aborting or dropping the running batch.
"""
from __future__ import annotations
import asyncio
from collections import defaultdict
from contextlib import ExitStack
import pytest
from tests.e2e.features.custom_pipeline.test_diffusion_pause_keep import (
CUSTOM_PIPELINE_CLASS,
MODEL,
WORKER_EXTENSION_CLASS,
_generate,
_pause_round,
_wait_for,
)
from tests.helpers.mark import hardware_test
from vllm_omni.entrypoints.async_omni import AsyncOmni
def _all_true(results) -> bool:
flat = []
stack = list(results)
while stack:
item = stack.pop()
if isinstance(item, list):
stack.extend(item)
else:
flat.append(item)
return bool(flat) and all(item is True for item in flat)
@pytest.mark.core_model
@pytest.mark.diffusion
@hardware_test(res={"cuda": "L4"}, num_cards=2)
@pytest.mark.asyncio
async def test_keep_pause_two_ranks_ack_and_rank_failure():
with ExitStack() as after:
engine = AsyncOmni(
model=MODEL,
custom_pipeline_args={"pipeline_class": CUSTOM_PIPELINE_CLASS},
worker_extension_cls=WORKER_EXTENSION_CLASS,
enforce_eager=True,
enable_sleep_mode=True,
num_gpus=2,
parallel_config={"ulysses_degree": 2, "ulysses_mode": "advanced_uaa"},
)
after.callback(engine.shutdown)
events: dict[str, dict[str, float]] = defaultdict(dict)
assert _all_true(await engine.collective_rpc(method="start_counting_synchronize_device"))
pause_latencies = [await _pause_round(engine, events, round_index) for round_index in range(3)]
# The result is the all-rank AND, so True means both ranks ran the barrier.
assert _all_true(await engine.collective_rpc(method="synchronize_device_seen"))
assert _all_true(await engine.collective_rpc(method="set_synchronize_device_failure", args=(1, True)))
a = asyncio.create_task(_generate(engine, "a-fail", events))
await _wait_for(lambda: "started" in events["a-fail"])
with pytest.raises(RuntimeError, match="pause_scheduler failed"):
await engine.pause_generation(mode="keep", clear_cache=False)
assert await engine.is_paused()
a_output = await asyncio.wait_for(a, 60.0)
assert a_output.images, "a failed barrier must not abort the running batch"
assert _all_true(await engine.collective_rpc(method="set_synchronize_device_failure", args=(1, False)))
await engine.resume_generation()
assert not await engine.is_paused()
recovered = await asyncio.wait_for(_generate(engine, "after-recovery", events), 60.0)
assert recovered.images
assert not engine.request_states
print(f"[sp2] rounds=3 pause_latency_s min/max={min(pause_latencies):.3f}/{max(pause_latencies):.3f}")
@@ -497,3 +497,99 @@ class TestWorkerProcShutdown:
result_mq.shutdown.assert_called_once_with()
assert proc.mq is None
assert proc.result_mq is None
class TestSynchronizeDeviceBarrier:
"""The pause barrier must not report a quiet device while outputs still pack."""
@staticmethod
def _rpc(timeout):
return {
"method": "synchronize_device",
"args": (),
"kwargs": {"timeout": timeout},
"output_rank": 0,
"exec_all_ranks": False,
"collect_rank_status": False,
}
def test_rejects_undrained_output(self, mocker):
proc = _make_worker_proc(step_execution=False)
proc.worker = MagicMock()
mocker.patch.object(proc, "drain_async_outputs", return_value=False)
with pytest.raises(TimeoutError, match="did not drain"):
proc._execute_rpc(self._rpc(0.01))
proc.worker.execute_method.assert_not_called()
@pytest.mark.parametrize("timeout", [0.5, None])
def test_drains_with_the_call_timeout_then_synchronizes(self, mocker, timeout):
proc = _make_worker_proc(step_execution=False)
proc.worker = MagicMock()
call_order: list = []
mocker.patch.object(
proc,
"drain_async_outputs",
side_effect=lambda **kw: call_order.append(("drain", kw)) or True,
)
proc.worker.execute_method.side_effect = lambda *a, **kw: call_order.append(("sync", a, kw))
_, should_reply = proc._execute_rpc(self._rpc(timeout))
assert call_order == [
("drain", {"timeout": timeout}),
("sync", ("synchronize_device",), {"timeout": timeout}),
]
assert should_reply is True
def test_completes_only_after_output_ready_is_published(self, mocker):
proc = _make_worker_proc(step_execution=False)
proc.worker = MagicMock()
mock_platform = mocker.MagicMock()
mock_platform.record_device_event.return_value = MagicMock()
mocker.patch(
"vllm_omni.diffusion.worker.diffusion_worker.current_omni_platform",
mock_platform,
)
mock_torch = mocker.MagicMock()
mock_torch.accelerator.current_accelerator.return_value.type = "cpu"
mocker.patch("vllm_omni.diffusion.worker.diffusion_worker.torch", mock_torch)
release = threading.Event()
mocker.patch(
"vllm_omni.diffusion.worker.diffusion_worker.pack_diffusion_output_shm",
side_effect=lambda *a, **kw: release.wait(timeout=5.0),
)
proc._return_result(DiffusionOutput(output="data"), rpc_id="1")
loop_thread = threading.Thread(target=proc._async_output_loop, daemon=True)
loop_thread.start()
barrier_done = threading.Event()
def run_barrier():
proc._execute_rpc(self._rpc(5.0))
barrier_done.set()
barrier_thread = threading.Thread(target=run_barrier, daemon=True)
barrier_thread.start()
try:
assert not barrier_done.wait(timeout=0.2)
proc.worker.execute_method.assert_not_called()
release.set()
assert barrier_done.wait(timeout=5.0)
kinds = [
call.args[0].kind
for call in proc.result_mq.enqueue.call_args_list
if isinstance(call.args[0], AsyncDiffusionOutput)
]
assert kinds == [AsyncOutputKind.COMPUTE_DONE, AsyncOutputKind.OUTPUT_READY]
proc.worker.execute_method.assert_called_once_with("synchronize_device", timeout=5.0)
finally:
release.set()
proc._async_output_queue.put(None)
loop_thread.join(timeout=2.0)
barrier_thread.join(timeout=2.0)
@@ -1,5 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Regression tests for the busy-loop-based RPC routing in DiffusionEngine.
These tests guard the invariants established when ``collective_rpc`` was
@@ -60,6 +60,12 @@ class _ConcurrencyTrackingExecutor:
self.is_failed = False
self._closed = False
self.od_config = SimpleNamespace()
# Pause tests: a call whose gate is registered (by request id, or by
# method name for ``synchronize_device``) blocks inside the executor
# until the gate is set; ``version`` tags each call so a test can tell
# which "weights" a request ran on.
self.gates: dict[str, threading.Event] = {}
self.version = "v1"
def collective_rpc(
self,
@@ -81,16 +87,25 @@ class _ConcurrencyTrackingExecutor:
"kwargs": kwargs,
"unique_reply_rank": unique_reply_rank,
"thread": threading.get_ident(),
"version": self.version,
}
)
try:
if self.rpc_delay:
time.sleep(self.rpc_delay)
if method == "synchronize_device":
gate = self.gates.get(method)
if gate is not None:
gate.wait(timeout=5.0)
return [None]
# Distinguish per-request execution from raw RPC by method name.
if method in {"execute_model", "execute_stepwise", "generate"}:
# args[0] is the request-like object for execute_model.
req = args[0] if args else None
tag = req.request_id if req is not None and hasattr(req, "request_id") else "unknown"
gate = self.gates.get(tag)
if gate is not None:
gate.wait(timeout=5.0)
return DiffusionOutput(error=f"result_for_{tag}", finished=True)
tag = args[0] if args else method
return DiffusionOutput(error=f"rpc_result_for_{tag}")
@@ -133,21 +148,27 @@ def _make_request(tag: str):
def _make_engine_with_loop(
loop: asyncio.AbstractEventLoop,
loop: asyncio.AbstractEventLoop | None,
rpc_delay: float = 0.0,
*,
scheduler: RequestScheduler | None = None,
start_loop: bool = True,
):
"""Construct a ``DiffusionEngine`` skeleton with a real busy loop.
The engine is wired with a fake executor that asserts no concurrent
calls and a real ``RequestScheduler``.
calls and a real ``RequestScheduler``. ``start_loop=False`` leaves the
engine in its bootstrap state (no busy-loop thread).
"""
engine = DiffusionEngine.__new__(DiffusionEngine)
engine._closed = False
engine.od_config = SimpleNamespace(streaming_output=False)
engine.executor = _ConcurrencyTrackingExecutor(rpc_delay=rpc_delay)
sched = RequestScheduler()
sched.initialize(SimpleNamespace(max_num_seqs=1, request_batch_max_wait_ms=0.0))
sched = scheduler
if sched is None:
sched = RequestScheduler()
sched.initialize(SimpleNamespace(max_num_seqs=1, request_batch_max_wait_ms=0.0))
engine.scheduler = sched
engine.step_execution = False
engine.supports_request_batch = False
@@ -160,9 +181,14 @@ def _make_engine_with_loop(
engine._closed = False
engine.abort_queue = queue.Queue()
engine._rpc_queue = queue.Queue()
engine._scheduling_paused = False
engine.main_loop = loop
engine.stop_event = threading.Event()
if not start_loop:
engine._loop_started = False
engine.worker_thread = None
return engine
engine._loop_started = True
engine.worker_thread = threading.Thread(target=engine._busy_loop, daemon=True)
engine.worker_thread.start()
@@ -170,6 +196,10 @@ def _make_engine_with_loop(
def _stop_engine(engine: DiffusionEngine) -> None:
if engine.worker_thread is None:
return
for gate in engine.executor.gates.values():
gate.set()
with engine._cv:
engine.stop_event.set()
engine._cv.notify_all()
@@ -565,3 +595,319 @@ async def test_rpc_and_request_results_do_not_swap():
assert req_res.error == "result_for_req1"
# Belt and braces: never overlapped on the executor.
assert engine.executor.max_concurrent == 1
# ───────────────────────── scheduler pause (keep) ─────────────────────────
class _GatedAdmissionScheduler(RequestScheduler):
"""Batching wait that only ends when the test opens admission."""
def __init__(self) -> None:
super().__init__()
self.admission_open = threading.Event()
def get_admission_wait_decision(self, *, now: float, dp_concurrent: bool = False):
decision = super().get_admission_wait_decision(now=now, dp_concurrent=dp_concurrent)
return decision.__class__(should_wait=True, deadline=None, max_batch=self.max_num_running_reqs)
def should_end_admission_wait(self, decision, *, now: float, stable_since: float) -> bool:
return self.admission_open.is_set()
def _pause(engine: DiffusionEngine, timeout: float | None = None):
return engine.async_collective_rpc("pause_scheduler", timeout=timeout, kwargs={"mode": "keep"})
def _resume(engine: DiffusionEngine):
return engine.async_collective_rpc("resume_scheduler")
def _methods(engine: DiffusionEngine) -> list[str]:
return [call["method"] for call in engine.executor.calls]
def _executed(engine: DiffusionEngine) -> list[tuple[str, str]]:
return [
(call["args"][0].request_id, call["version"])
for call in engine.executor.calls
if call["method"] == "execute_model"
]
def _submit(engine: DiffusionEngine, tag: str) -> asyncio.Task:
return asyncio.create_task(_consume_final_output(engine.async_add_req_and_stream_response(_make_request(tag))))
async def _wait_until(predicate, timeout: float = 3.0) -> None:
deadline = time.monotonic() + timeout
while not predicate():
if time.monotonic() > deadline:
raise AssertionError("condition not reached in time")
await asyncio.sleep(0.005)
@pytest.mark.asyncio
async def test_keep_pause_retains_waiting_request_and_serves_control_rpcs():
loop = asyncio.get_running_loop()
engine = _make_engine_with_loop(loop)
try:
assert await _pause(engine) is None
queued = _submit(engine, "queued")
result = await engine.async_collective_rpc("list_loras", unique_reply_rank=0)
assert result.error == "rpc_result_for_list_loras"
await asyncio.sleep(0.05)
assert engine.scheduler.num_waiting_requests() == 1
assert _executed(engine) == []
assert not queued.done()
await _resume(engine)
assert (await asyncio.wait_for(queued, 3.0)).error == "result_for_queued"
assert _methods(engine) == ["synchronize_device", "list_loras", "execute_model"]
finally:
_stop_engine(engine)
@pytest.mark.asyncio
async def test_keep_pause_acks_after_running_batch_and_freezes_queue():
loop = asyncio.get_running_loop()
engine = _make_engine_with_loop(loop)
gate = engine.executor.gates["A"] = threading.Event()
try:
a = _submit(engine, "A")
await _wait_until(lambda: _executed(engine) == [("A", "v1")])
b = _submit(engine, "B")
await _wait_until(lambda: engine.scheduler.num_waiting_requests() == 1)
pause = asyncio.create_task(_pause(engine))
await asyncio.sleep(0.1)
assert not pause.done()
gate.set()
await asyncio.wait_for(pause, 3.0)
assert _methods(engine) == ["execute_model", "synchronize_device"]
assert (await asyncio.wait_for(a, 3.0)).error == "result_for_A"
assert engine.scheduler.num_waiting_requests() == 1
assert not b.done()
engine.executor.version = "v2"
await _resume(engine)
assert (await asyncio.wait_for(b, 3.0)).error == "result_for_B"
assert _executed(engine) == [("A", "v1"), ("B", "v2")]
finally:
_stop_engine(engine)
@pytest.mark.asyncio
async def test_keep_pause_ack_does_not_release_later_rpcs_before_the_batch_is_emitted():
"""A pause ACK must not open the queue to RPCs that touch device memory
(sleep) before the batch that just ran has been delivered.
"""
loop = asyncio.get_running_loop()
engine = _make_engine_with_loop(loop)
barrier_gate = engine.executor.gates["synchronize_device"] = threading.Event()
methods_at_emit: list[list[str]] = []
emit_outputs = engine._emit_outputs
def _recording_emit(*args, **kwargs):
methods_at_emit.append(_methods(engine))
return emit_outputs(*args, **kwargs)
engine._emit_outputs = _recording_emit
try:
gate = engine.executor.gates["A"] = threading.Event()
a = _submit(engine, "A")
await _wait_until(lambda: _executed(engine) == [("A", "v1")])
pause = asyncio.create_task(_pause(engine))
await _wait_until(lambda: engine._scheduling_paused)
gate.set()
# The barrier holds the busy loop inside the post-execute RPC drain,
# so the sleep is queued before the pause ACK is published.
await _wait_until(lambda: "synchronize_device" in _methods(engine))
sleep_rpc = asyncio.create_task(engine.async_collective_rpc("handle_sleep_task", args=("t",)))
await _wait_until(lambda: not engine._rpc_queue.empty())
barrier_gate.set()
await asyncio.wait_for(pause, 3.0)
assert (await asyncio.wait_for(a, 3.0)).error == "result_for_A"
assert (await asyncio.wait_for(sleep_rpc, 3.0)).error == "rpc_result_for_t"
finally:
_stop_engine(engine)
assert _methods(engine) == ["execute_model", "synchronize_device", "handle_sleep_task"]
assert methods_at_emit and "handle_sleep_task" not in methods_at_emit[0]
@pytest.mark.asyncio
async def test_timed_out_pause_still_ends_the_drain_before_the_batch_is_emitted():
"""A pause whose caller already gave up is still a drain barrier: its
cancelled task must not let a later RPC run before the batch is emitted.
"""
loop = asyncio.get_running_loop()
engine = _make_engine_with_loop(loop)
methods_at_emit: list[list[str]] = []
emit_outputs = engine._emit_outputs
def _recording_emit(*args, **kwargs):
methods_at_emit.append(_methods(engine))
return emit_outputs(*args, **kwargs)
engine._emit_outputs = _recording_emit
try:
gate = engine.executor.gates["A"] = threading.Event()
a = _submit(engine, "A")
await _wait_until(lambda: _executed(engine) == [("A", "v1")])
# The caller times out while A is still executing: the pause future is
# cancelled, but its task stays queued ahead of the sleep.
with pytest.raises(TimeoutError):
await _pause(engine, timeout=0.1)
sleep_rpc = asyncio.create_task(engine.async_collective_rpc("handle_sleep_task", args=("t",)))
await _wait_until(lambda: engine._rpc_queue.qsize() == 2)
gate.set()
assert (await asyncio.wait_for(a, 3.0)).error == "result_for_A"
assert (await asyncio.wait_for(sleep_rpc, 3.0)).error == "rpc_result_for_t"
finally:
_stop_engine(engine)
assert _methods(engine) == ["execute_model", "handle_sleep_task"]
assert methods_at_emit and "handle_sleep_task" not in methods_at_emit[0]
@pytest.mark.asyncio
async def test_pause_rejects_unsupported_modes_without_closing_the_gate():
loop = asyncio.get_running_loop()
engine = _make_engine_with_loop(loop)
try:
for kwargs in ({"mode": "abort"}, {"mode": "wait"}, {}):
with pytest.raises(ValueError):
await engine.async_collective_rpc("pause_scheduler", kwargs=kwargs)
engine.execution_mode = DiffusionExecutionMode.STEP_BATCH
with pytest.raises(NotImplementedError):
await _pause(engine)
engine.execution_mode = DiffusionExecutionMode.REQUEST_BATCH
assert (await asyncio.wait_for(_submit(engine, "still-runs"), 3.0)).error == "result_for_still-runs"
assert _methods(engine) == ["execute_model"]
finally:
_stop_engine(engine)
@pytest.mark.asyncio
async def test_keep_pause_during_batching_wait_holds_the_batch():
loop = asyncio.get_running_loop()
sched = _GatedAdmissionScheduler()
sched.initialize(SimpleNamespace(max_num_seqs=4, request_batch_max_wait_ms=1000.0))
engine = _make_engine_with_loop(loop, scheduler=sched)
try:
waiting = _submit(engine, "W")
await _wait_until(lambda: engine.scheduler.num_waiting_requests() == 1)
await asyncio.sleep(0.05)
await asyncio.wait_for(_pause(engine), 3.0)
assert _methods(engine) == ["synchronize_device"]
assert engine.scheduler.num_waiting_requests() == 1
await _resume(engine)
await asyncio.sleep(0.05)
assert _executed(engine) == []
sched.admission_open.set()
assert (await asyncio.wait_for(waiting, 3.0)).error == "result_for_W"
finally:
sched.admission_open.set()
_stop_engine(engine)
@pytest.mark.asyncio
async def test_pause_timeout_leaves_gate_closed_until_explicit_resume():
loop = asyncio.get_running_loop()
engine = _make_engine_with_loop(loop)
gate = engine.executor.gates["A"] = threading.Event()
try:
a = _submit(engine, "A")
await _wait_until(lambda: _executed(engine) == [("A", "v1")])
b = _submit(engine, "B")
await _wait_until(lambda: engine.scheduler.num_waiting_requests() == 1)
with pytest.raises(TimeoutError):
await _pause(engine, timeout=0.1)
gate.set()
assert (await asyncio.wait_for(a, 3.0)).error == "result_for_A"
await asyncio.sleep(0.1)
assert _executed(engine) == [("A", "v1")]
assert not b.done()
await _resume(engine)
assert (await asyncio.wait_for(b, 3.0)).error == "result_for_B"
assert "synchronize_device" not in _methods(engine)
finally:
_stop_engine(engine)
@pytest.mark.asyncio
async def test_repeated_pause_and_resume_are_idempotent():
loop = asyncio.get_running_loop()
engine = _make_engine_with_loop(loop)
try:
await _resume(engine)
await _pause(engine)
await _pause(engine)
queued = _submit(engine, "Q")
await asyncio.sleep(0.05)
assert _executed(engine) == []
await _resume(engine)
await _resume(engine)
assert (await asyncio.wait_for(queued, 3.0)).error == "result_for_Q"
assert _methods(engine) == ["synchronize_device", "synchronize_device", "execute_model"]
finally:
_stop_engine(engine)
@pytest.mark.asyncio
async def test_abort_of_waiting_request_while_paused_delivers_terminal():
loop = asyncio.get_running_loop()
engine = _make_engine_with_loop(loop)
try:
await _pause(engine)
b = _submit(engine, "B")
await _wait_until(lambda: engine.scheduler.num_waiting_requests() == 1)
engine.abort("B")
result = await asyncio.wait_for(b, 3.0)
assert result.aborted is True
assert engine.scheduler.get_request_state("B") is None
assert _executed(engine) == []
await _resume(engine)
assert (await asyncio.wait_for(_submit(engine, "C"), 3.0)).error == "result_for_C"
assert _executed(engine) == [("C", "v1")]
assert engine.worker_thread.is_alive()
finally:
_stop_engine(engine)
def test_pause_and_resume_before_loop_starts_use_engine_local_path():
engine = _make_engine_with_loop(None, start_loop=False)
assert engine.collective_rpc("pause_scheduler", kwargs={"mode": "keep"}) is None
assert _methods(engine) == ["synchronize_device"]
assert engine.collective_rpc("resume_scheduler") is None
assert engine.add_req_and_wait_for_response(_make_request("X")).error == "result_for_X"
assert _executed(engine) == [("X", "v1")]
@pytest.mark.asyncio
async def test_requests_without_pause_never_trigger_the_barrier():
loop = asyncio.get_running_loop()
engine = _make_engine_with_loop(loop)
try:
first = await asyncio.wait_for(_submit(engine, "r1"), 3.0)
await engine.async_collective_rpc("ping", args=("p",), unique_reply_rank=0)
second = await asyncio.wait_for(_submit(engine, "r2"), 3.0)
finally:
_stop_engine(engine)
assert (first.error, second.error) == ("result_for_r1", "result_for_r2")
assert _methods(engine) == ["execute_model", "ping", "execute_model"]
@@ -2092,3 +2092,19 @@ class TestStepScheduler:
with pytest.raises(ValueError):
self.scheduler.add_request(request)
class TestPendingFinishedRequestIds:
def test_reports_finished_ids_that_still_hold_state_without_clearing_them(self):
sched = RequestScheduler()
sched.initialize(SimpleNamespace(max_num_seqs=1, request_batch_max_wait_ms=0.0))
sched.add_request(_make_request("a"))
sched.add_request(_make_request("b"))
sched.finish_requests("a", DiffusionRequestStatus.FINISHED_ABORTED)
assert sched.pending_finished_request_ids() == {"a"}
sched.pop_request_state("a")
assert sched.pending_finished_request_ids() == set()
# The next wave still ships the id to the worker for its own cleanup.
assert sched.schedule().finished_req_ids == {"a"}
@@ -0,0 +1,53 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Unit tests for the device step of the diffusion pause barrier."""
from types import SimpleNamespace
from unittest.mock import MagicMock
import pytest
from vllm_omni.diffusion.worker.diffusion_worker import DiffusionWorker
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
def _make_worker(runner: object) -> DiffusionWorker:
worker = object.__new__(DiffusionWorker)
worker.model_runner = runner
worker.rank = 0
return worker
def test_synchronize_device_waits_for_prefetch_before_device_sync(mocker):
platform = mocker.patch("vllm_omni.diffusion.worker.diffusion_worker.current_omni_platform")
order: list = []
manager = MagicMock()
manager.wait_prefetch.side_effect = lambda timeout: order.append(("prefetch", timeout)) or True
platform.synchronize.side_effect = lambda: order.append("sync")
worker = _make_worker(SimpleNamespace(kv_transfer_manager=manager))
worker.synchronize_device(timeout=2.5)
assert order == [("prefetch", 2.5), "sync"]
def test_synchronize_device_fails_when_prefetch_is_still_outstanding(mocker):
platform = mocker.patch("vllm_omni.diffusion.worker.diffusion_worker.current_omni_platform")
manager = MagicMock()
manager.wait_prefetch.return_value = False
worker = _make_worker(SimpleNamespace(kv_transfer_manager=manager))
with pytest.raises(TimeoutError, match="prefetch"):
worker.synchronize_device(timeout=0.01)
platform.synchronize.assert_not_called()
def test_synchronize_device_without_transfer_manager_only_syncs(mocker):
platform = mocker.patch("vllm_omni.diffusion.worker.diffusion_worker.current_omni_platform")
worker = _make_worker(SimpleNamespace(kv_transfer_manager=None))
worker.synchronize_device(timeout=None)
platform.synchronize.assert_called_once_with()
@@ -5,6 +5,7 @@ mock connector, dedup, per-call sender_info isolation, role classification, and
miss/abort paths. GPU H2D and D2D paths need CUDA (integration tests).
"""
import threading
from types import SimpleNamespace
import pytest
@@ -354,3 +355,41 @@ def test_consume_then_apply_attaches_payload():
receiver._record_stream_for_prefetched(data)
receiver.apply_kv_cache_to_request(req, data)
assert req.past_key_values is not None
# --------------------------------------------------------------------------- #
# wait_prefetch: the pause barrier's view of outstanding background loads
# --------------------------------------------------------------------------- #
def test_wait_prefetch_is_true_when_nothing_outstanding():
_, receiver, _ = _make_sender_receiver()
assert receiver.wait_prefetch(timeout=0.01) is True
def test_wait_prefetch_is_true_when_prefetch_disabled():
_, receiver, _ = _make_sender_receiver(async_prefetch=False)
assert receiver.wait_prefetch(timeout=0.01) is True
def test_wait_prefetch_reports_a_running_prefetch_and_keeps_its_payload():
sender, receiver, connector = _make_sender_receiver()
_seed_payload(sender, "rid-w")
gate = threading.Event()
real_get = connector.get
def slow_get(*args, **kwargs):
gate.wait(timeout=5.0)
return real_get(*args, **kwargs)
connector.get = slow_get
receiver.start_prefetch({"request_id": "rid-w", "kv_sender_info": _SENDER_INFO})
try:
assert receiver.wait_prefetch(timeout=0.05) is False
gate.set()
assert receiver.wait_prefetch(timeout=5.0) is True
data, _ = receiver.consume_prefetched_kv(_req("rid-w"))
assert data is not None
finally:
gate.set()
receiver.shutdown_prefetch()
@@ -0,0 +1,183 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""
Lifecycle test for ``AsyncOmni.pause_generation(mode="keep")`` on a diffusion stage.
Per round, request A is started, request B is queued behind it, and the pause
is issued while A is still running. The pause must return only after A has
been delivered, B must stay queued through ``sleep`` / ``wake_up``, and B may
start only after ``resume_generation``.
A second test follows the order the docs recommend for a weight update: sleep
is issued as soon as the pause returns, without awaiting the request that was
running, which must still be delivered intact.
Both single-GPU executor backends are covered explicitly: ``uni`` (in-process
worker) and ``mp`` (worker process with the asynchronous D2H output thread).
Usage:
pytest tests/e2e/features/custom_pipeline/test_diffusion_pause_keep.py -v -s
"""
from __future__ import annotations
import asyncio
import os
import time
from collections import defaultdict
from contextlib import ExitStack
import pytest
from tests.helpers.mark import hardware_test
from vllm_omni.diffusion.data import is_diffusion_request_started_output
from vllm_omni.entrypoints.async_omni import AsyncOmni
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
MODEL = "tiny-random/Qwen-Image"
CUSTOM_PIPELINE_CLASS = "tests.e2e.features.helpers.custom_pipeline.QwenImagePipelineWithLogProbForTest"
WORKER_EXTENSION_CLASS = "tests.e2e.features.helpers.custom_pipeline.vLLMOmniColocateWorkerExtensionForTest"
# CI runs a couple of rounds; set PAUSE_KEEP_ROUNDS=10 for a soak run.
ROUNDS = int(os.environ.get("PAUSE_KEEP_ROUNDS", "2"))
# Enough denoising steps that the pause lands while A is still executing.
NUM_INFERENCE_STEPS = 40
def _sampling_params() -> OmniDiffusionSamplingParams:
return OmniDiffusionSamplingParams(
num_inference_steps=NUM_INFERENCE_STEPS,
guidance_scale=0.0,
height=256,
width=256,
seed=42,
emit_request_lifecycle=True,
)
async def _generate(engine: AsyncOmni, request_id: str, events: dict[str, dict[str, float]]):
final = None
async for output in engine.generate(
prompt={"prompt_ids": list(range(50))},
request_id=request_id,
sampling_params_list=[_sampling_params()],
output_modalities=["image"],
):
if is_diffusion_request_started_output(output):
events[request_id]["started"] = time.monotonic()
continue
final = output
events[request_id]["finished"] = time.monotonic()
return final
async def _wait_for(predicate, timeout: float = 30.0) -> None:
deadline = time.monotonic() + timeout
while not predicate():
assert time.monotonic() < deadline, "condition not reached in time"
await asyncio.sleep(0.005)
async def _pause_round(engine: AsyncOmni, events: dict[str, dict[str, float]], round_index: int) -> float:
"""A runs, B queues, keep-pause lands while A runs; B starts only after resume.
Returns the pause latency.
"""
a_id, b_id = f"a{round_index}", f"b{round_index}"
a = asyncio.create_task(_generate(engine, a_id, events))
await _wait_for(lambda: "started" in events[a_id])
b = asyncio.create_task(_generate(engine, b_id, events))
await asyncio.sleep(0.05)
assert not a.done(), "A finished before the pause was issued; raise NUM_INFERENCE_STEPS"
pause_start = time.monotonic()
await engine.pause_generation(mode="keep", clear_cache=False)
pause_latency = time.monotonic() - pause_start
assert await engine.is_paused()
a_output = await asyncio.wait_for(a, 60.0)
assert a_output.images, "A must be delivered, not aborted, by a keep pause"
assert "started" not in events[b_id], "B must not start while paused"
# Control RPCs and the sleep cycle stay available on the paused stage.
assert await engine.list_loras() == []
await engine.sleep(level=1)
assert await engine.is_sleeping()
await engine.wake_up()
assert not await engine.is_sleeping()
assert await engine.is_paused()
assert not b.done() and "started" not in events[b_id]
resume_at = time.monotonic()
await engine.resume_generation()
b_output = await asyncio.wait_for(b, 60.0)
assert b_output.images, "B must run after resume"
assert events[b_id]["started"] >= resume_at
return pause_latency
@pytest.mark.core_model
@pytest.mark.diffusion
@hardware_test(res={"cuda": "L4"}, num_cards=1)
@pytest.mark.asyncio
@pytest.mark.parametrize("backend", ["uni", "mp"])
async def test_keep_pause_sleep_wake_resume_lifecycle(backend: str):
with ExitStack() as after:
engine = AsyncOmni(
model=MODEL,
custom_pipeline_args={"pipeline_class": CUSTOM_PIPELINE_CLASS},
worker_extension_cls=WORKER_EXTENSION_CLASS,
enforce_eager=True,
enable_sleep_mode=True,
distributed_executor_backend=backend,
)
after.callback(engine.shutdown)
events: dict[str, dict[str, float]] = defaultdict(dict)
pause_latencies = [await _pause_round(engine, events, round_index) for round_index in range(ROUNDS)]
assert not engine.request_states, "every request must be finalized"
print(
f"[{backend}] rounds={ROUNDS} steps={NUM_INFERENCE_STEPS} "
f"pause_latency_s min/median/max="
f"{min(pause_latencies):.3f}/{sorted(pause_latencies)[ROUNDS // 2]:.3f}/{max(pause_latencies):.3f}"
)
@pytest.mark.core_model
@pytest.mark.diffusion
@hardware_test(res={"cuda": "L4"}, num_cards=1)
@pytest.mark.asyncio
@pytest.mark.parametrize("backend", ["uni", "mp"])
async def test_sleep_right_after_keep_pause_keeps_the_running_batch(backend: str):
"""The documented weight-update order: sleep is issued as soon as the pause
returns, with the request that was running still un-awaited.
"""
with ExitStack() as after:
engine = AsyncOmni(
model=MODEL,
custom_pipeline_args={"pipeline_class": CUSTOM_PIPELINE_CLASS},
worker_extension_cls=WORKER_EXTENSION_CLASS,
enforce_eager=True,
enable_sleep_mode=True,
distributed_executor_backend=backend,
)
after.callback(engine.shutdown)
events: dict[str, dict[str, float]] = defaultdict(dict)
a = asyncio.create_task(_generate(engine, "a", events))
await _wait_for(lambda: "started" in events["a"])
await asyncio.sleep(0.05)
assert not a.done(), "A finished before the pause was issued; raise NUM_INFERENCE_STEPS"
await engine.pause_generation(mode="keep", clear_cache=False)
await engine.sleep(level=1)
a_output = await asyncio.wait_for(a, 60.0)
assert a_output.images, "A must survive a sleep issued right after the pause ACK"
await engine.wake_up()
await engine.resume_generation()
b_output = await asyncio.wait_for(_generate(engine, "b", events), 120.0)
assert b_output.images, "the stage must generate again after wake and resume"
+31 -1
View File
@@ -1,5 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Test doubles for custom-pipeline E2E and RLHF integration.
@@ -290,6 +290,36 @@ class vLLMOmniColocateWorkerExtensionForTest(CustomPipelineWorkerExtension):
"""Return a stable identifier for assertions in unit tests."""
return "vllm-omni-colocate-worker-extension-for-test"
def start_counting_synchronize_device(self) -> bool:
"""Count pause-barrier runs on this rank (test-only)."""
self._synchronize_device_calls = 0
inner = type(self).synchronize_device
def _counted(timeout=None):
self._synchronize_device_calls += 1
return inner(self, timeout=timeout)
self.synchronize_device = _counted
return True
def synchronize_device_seen(self) -> bool:
"""True from every rank only when each rank ran the barrier at least once."""
return getattr(self, "_synchronize_device_calls", 0) > 0
def set_synchronize_device_failure(self, rank: int, fail: bool) -> bool:
"""Make (or stop making) the pause barrier fail on ``rank`` (test-only)."""
if self.rank != rank:
return True
if fail:
def _fail(timeout=None):
raise RuntimeError(f"injected barrier failure on rank {rank}")
self.synchronize_device = _fail
else:
self.__dict__.pop("synchronize_device", None)
return True
# ---------------------------------------------------------------------------
# Pipeline layer
@@ -13,6 +13,7 @@ import asyncio
import queue
import time
from types import SimpleNamespace
from unittest.mock import AsyncMock
import janus
import pytest
@@ -22,6 +23,7 @@ from vllm.v1.serial_utils import MsgpackEncoder
from vllm_omni.engine.messages import (
AddCompanionRequestMessage,
CollectiveRPCRequestMessage,
EngineQueueMessage,
ErrorMessage,
ShutdownRequestMessage,
@@ -1143,3 +1145,64 @@ async def test_diffusion_client_error_output_propagates_non_400_status(
finally:
orchestrator_fixture.request_sync_q.put_nowait(ShutdownRequestMessage())
orchestrator_fixture.thread.join(timeout=5)
@pytest.mark.asyncio
async def test_control_rpc_failure_is_reported_to_the_caller_not_fatal() -> None:
"""A control RPC that fails on one replica must come back as that
replica's error result instead of raising out of _request_handler.
"""
class _FailingClient(FakeStageClient):
async def collective_rpc_async(self, *args, **kwargs):
raise TimeoutError("outputs did not drain")
ok = FakeStageClient(stage_type="diffusion")
ok.collective_rpc_async = AsyncMock(return_value=None)
orchestrator, queues = _build_bare_orchestrator(
_build_stage_pools([[ok], [_FailingClient(stage_type="diffusion")]])
)
try:
await orchestrator._handle_collective_rpc(
CollectiveRPCRequestMessage(
rpc_id="rpc-1",
method="pause_scheduler",
args=(),
kwargs={"mode": "keep"},
stage_ids=[0, 1],
)
)
result = queues[2].async_q.get_nowait()
assert result.rpc_id == "rpc-1"
assert result.stage_ids == [0, 1]
assert result.results[0] is None
assert result.results[1]["supported"] is False
assert "outputs did not drain" in result.results[1]["error"]
finally:
for q in queues:
q.close()
@pytest.mark.asyncio
async def test_rpc_failure_capture_is_limited_to_pause_and_resume() -> None:
"""Only pause/resume failures are reported as a result; every other RPC
keeps propagating the way it does today.
"""
orchestrator, queues = _build_bare_orchestrator(_build_stage_pools([[FakeStageClient(stage_type="diffusion")]]))
orchestrator.stage_pools[0].collective_rpc = AsyncMock(side_effect=TimeoutError("worker died"))
def _msg(rpc_id: str, method: str) -> CollectiveRPCRequestMessage:
return CollectiveRPCRequestMessage(rpc_id=rpc_id, method=method, args=(), kwargs={}, stage_ids=[0])
try:
await orchestrator._handle_collective_rpc(_msg("rpc-resume", "resume_scheduler"))
result = queues[2].async_q.get_nowait()
assert result.results[0]["supported"] is False
assert "worker died" in result.results[0]["error"]
with pytest.raises(TimeoutError):
await orchestrator._handle_collective_rpc(_msg("rpc-sleep", "sleep"))
finally:
for q in queues:
q.close()
@@ -25,6 +25,7 @@ def _make_omni(*, stage_types: list[str]) -> AsyncOmni:
omni._paused = False
omni._admitting = 0
omni._hold_admission_until_resume = False
omni._paused_stage_ids = set()
omni._sleeping_tags = set()
omni._stage_sleeping_tags = {}
omni._level2_sleeping = False
@@ -460,3 +461,225 @@ def test_partial_wake_does_not_skip_remaining_sleeping_stage():
assert not omni._stage_sleeping_tags
asyncio.run(run())
# ───────────────────── diffusion pause_generation(mode="keep") ─────────────────────
def _rpc_methods(omni: AsyncOmni) -> list[tuple[str, list[int] | None]]:
return [(call.kwargs["method"], call.kwargs.get("stage_ids")) for call in omni.collective_rpc.await_args_list]
@pytest.mark.cpu
def test_diffusion_keep_routes_to_backend():
async def run() -> None:
omni = _make_omni(stage_types=["diffusion"])
await omni.pause_generation(mode="keep", clear_cache=False)
omni.collective_rpc.assert_awaited_once_with(
method="pause_scheduler",
args=(),
kwargs={"mode": "keep"},
stage_ids=[0],
)
assert await omni.is_paused() is True
assert omni._hold_admission_until_resume is True
asyncio.run(run())
@pytest.mark.cpu
@pytest.mark.parametrize(
"pause_kwargs",
[
{},
{"mode": "abort"},
{"mode": "wait"},
{"mode": "keep", "wait_for_inflight_requests": True},
],
)
def test_diffusion_non_keep_modes_pause_frontend_only(pause_kwargs):
async def run() -> None:
omni = _make_omni(stage_types=["diffusion"])
omni.reset_prefix_cache = AsyncMock(return_value=True)
omni.reset_mm_cache = AsyncMock()
omni.reset_encoder_cache = AsyncMock()
await omni.pause_generation(**pause_kwargs)
assert await omni.is_paused() is True
omni.collective_rpc.assert_not_awaited()
await omni.resume_generation()
omni.collective_rpc.assert_not_awaited()
assert await omni.is_paused() is False
asyncio.run(run())
@pytest.mark.cpu
def test_mixed_engine_keep_sends_engine_core_and_diffusion_pauses():
async def run() -> None:
omni = _make_omni(stage_types=["llm", "diffusion"])
omni.reset_prefix_cache = AsyncMock(return_value=True)
omni.reset_mm_cache = AsyncMock()
omni.reset_encoder_cache = AsyncMock()
await omni.pause_generation(mode="keep", clear_cache=True)
assert omni.collective_rpc.await_args_list[0].kwargs == {
"method": "pause_scheduler",
"args": (),
"kwargs": {"mode": "keep", "clear_cache": True},
"stage_ids": [0],
}
assert omni.collective_rpc.await_args_list[1].kwargs == {
"method": "pause_scheduler",
"args": (),
"kwargs": {"mode": "keep"},
"stage_ids": [1],
}
# The frontend cache reset still runs after the backend pauses.
omni.reset_prefix_cache.assert_awaited_once()
await omni.resume_generation()
assert _rpc_methods(omni)[2:] == [("resume_scheduler", [0]), ("resume_scheduler", [1])]
asyncio.run(run())
@pytest.mark.cpu
def test_keep_targets_only_requested_diffusion_stages():
async def run() -> None:
omni = _make_omni(stage_types=["diffusion", "diffusion"])
await omni.pause_generation(mode="keep", clear_cache=False, stage_ids=[1])
assert _rpc_methods(omni) == [("pause_scheduler", [1])]
await omni.resume_generation(stage_ids=[0])
assert _rpc_methods(omni) == [("pause_scheduler", [1])]
# Stage 1's scheduler is still closed: admission must stay closed too,
# otherwise new requests queue on a stage that will not schedule them.
assert await omni.is_paused() is True
await omni.resume_generation(stage_ids=[1])
assert _rpc_methods(omni) == [("pause_scheduler", [1]), ("resume_scheduler", [1])]
assert await omni.is_paused() is False
await omni.resume_generation()
assert len(_rpc_methods(omni)) == 2
asyncio.run(run())
@pytest.mark.cpu
def test_keep_then_sleep_wake_resume_keeps_admission_paused_until_resume():
async def run() -> None:
omni = _make_omni(stage_types=["diffusion"])
omni.reset_prefix_cache = AsyncMock(return_value=True)
omni.reset_encoder_cache = AsyncMock()
omni.collective_rpc = AsyncMock(
side_effect=lambda **kw: [OmniACK(task_id="t", status="SUCCESS", stage_id=0, rank=0)]
if kw["method"] in {"handle_sleep_task", "handle_wake_task"}
else [None]
)
await omni.pause_generation(mode="keep", clear_cache=False)
await omni.sleep(level=1)
await omni.wake_up()
assert await omni.is_paused() is True
await omni.resume_generation()
assert await omni.is_paused() is False
assert [m for m, _ in _rpc_methods(omni)] == [
"pause_scheduler",
"handle_sleep_task",
"handle_wake_task",
"resume_scheduler",
]
asyncio.run(run())
@pytest.mark.cpu
@pytest.mark.parametrize(
"failure",
[
AsyncMock(side_effect=NotImplementedError("request-level execution only")),
AsyncMock(return_value=[{"supported": False, "error": "TimeoutError: outputs did not drain"}]),
],
)
def test_keep_failure_surfaces_and_leaves_explicit_resume_possible(failure):
async def run() -> None:
omni = _make_omni(stage_types=["diffusion"])
omni.collective_rpc = failure
with pytest.raises((NotImplementedError, RuntimeError)):
await omni.pause_generation(mode="keep", clear_cache=False)
assert await omni.is_paused() is True
assert omni._paused_stage_ids == {0}
omni.collective_rpc = AsyncMock(return_value=[None])
await omni.resume_generation()
omni.collective_rpc.assert_awaited_once_with(
method="resume_scheduler",
args=(),
kwargs=None,
stage_ids=[0],
)
assert await omni.is_paused() is False
assert omni._paused_stage_ids == set()
asyncio.run(run())
@pytest.mark.cpu
def test_resume_failure_keeps_admission_paused_and_target_recorded():
async def run() -> None:
omni = _make_omni(stage_types=["diffusion"])
await omni.pause_generation(mode="keep", clear_cache=False)
omni.collective_rpc = AsyncMock(return_value=[{"error": "replica gone"}])
with pytest.raises(RuntimeError, match="resume_scheduler failed"):
await omni.resume_generation()
assert await omni.is_paused() is True
assert omni._paused_stage_ids == {0}
asyncio.run(run())
@pytest.mark.cpu
@pytest.mark.parametrize("first, second", [([1], [0]), ([0], [1])])
def test_mixed_engine_partial_resume_keeps_admission_paused(first, second):
async def run() -> None:
omni = _make_omni(stage_types=["llm", "diffusion"])
await omni.pause_generation(mode="keep", clear_cache=False)
await omni.resume_generation(stage_ids=first)
assert await omni.is_paused() is True
await omni.resume_generation(stage_ids=second)
assert await omni.is_paused() is False
assert _rpc_methods(omni)[2:] == [("resume_scheduler", first), ("resume_scheduler", second)]
asyncio.run(run())
@pytest.mark.cpu
def test_two_ar_stages_partial_resume_keeps_admission_paused():
async def run() -> None:
omni = _make_omni(stage_types=["llm", "llm"])
omni.reset_prefix_cache = AsyncMock(return_value=True)
omni.reset_mm_cache = AsyncMock()
omni.reset_encoder_cache = AsyncMock()
await omni.pause_generation(mode="abort")
await omni.resume_generation(stage_ids=[0])
assert await omni.is_paused() is True
await omni.resume_generation(stage_ids=[1])
assert await omni.is_paused() is False
asyncio.run(run())