vllm-omni/tests/diffusion/test_diffusion_worker_broadcast.py

# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

import threading
import time
from unittest.mock import Mock

import pytest
from vllm.distributed.device_communicators.shm_broadcast import MessageQueue

import vllm_omni.diffusion.worker.diffusion_worker as diffusion_worker_module
from vllm_omni.diffusion.worker.diffusion_worker import WorkerProc

pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cpu]


class _DelayedMultipartSocket:
    def __init__(
        self,
        socket,
        send_started: threading.Event,
        release_send: threading.Event,
    ):
        self._socket = socket
        self._send_started = send_started
        self._release_send = release_send
        self._delay_next_send = True

    def __getattr__(self, name):
        return getattr(self._socket, name)

    def send_multipart(self, *args, **kwargs):
        if self._delay_next_send:
            self._delay_next_send = False
            self._send_started.set()
            if not self._release_send.wait(timeout=2):
                raise TimeoutError("Timed out waiting to release delayed overflow payload")
        return self._socket.send_multipart(*args, **kwargs)


class _ObservedPollSocket:
    def __init__(self, socket, poll_started: threading.Event):
        self._socket = socket
        self._poll_started = poll_started

    def __getattr__(self, name):
        return getattr(self._socket, name)

    def poll(self, *args, **kwargs):
        self._poll_started.set()
        return self._socket.poll(*args, **kwargs)


class _RecordingWorker:
    def __init__(self):
        self.calls: list[str] = []

    def execute_method(self, method, *args, **kwargs):
        assert method == "record"
        self.calls.append(args[0])


def test_recv_message_waits_indefinitely():
    worker_proc = WorkerProc.__new__(WorkerProc)
    worker_proc.mq = Mock()
    worker_proc.mq.dequeue.return_value = {"type": "rpc"}

    result = worker_proc.recv_message()

    assert result == {"type": "rpc"}
    worker_proc.mq.dequeue.assert_called_once_with(indefinite=True)


def test_busy_loop_releases_failed_rpc_traceback_and_device_cache(monkeypatch):
    original = RuntimeError("model execution failed")
    worker_proc = WorkerProc.__new__(WorkerProc)
    worker_proc.gpu_id = 0
    worker_proc.mq = Mock()
    worker_proc.mq.dequeue.side_effect = [
        {
            "type": "rpc",
            "method": "execute_model",
            "args": (),
            "kwargs": {},
            "output_rank": 0,
            "exec_all_ranks": False,
            "collect_rank_status": False,
        },
        {"type": "shutdown"},
    ]
    worker_proc.worker = Mock()
    worker_proc.worker.execute_method.side_effect = original
    worker_proc.result_mq = object()
    worker_proc._return_result = Mock()
    worker_proc.wake_event = None
    worker_proc._running = True

    gc_collect = Mock()
    mock_platform = Mock()
    monkeypatch.setattr(diffusion_worker_module.gc, "collect", gc_collect)
    monkeypatch.setattr(diffusion_worker_module, "current_omni_platform", mock_platform)

    worker_proc._worker_busy_loop()

    worker_proc._return_result.assert_called_once_with(
        {"status": "error", "error": "model execution failed", "wave_id": None}
    )
    assert original.__traceback__ is None
    gc_collect.assert_called_once_with()
    mock_platform.empty_cache.assert_called_once_with()


def test_busy_loop_preserves_order_when_overflow_payload_is_delayed(monkeypatch):
    """A consumed overflow marker must not be abandoned on a socket timeout."""
    writer = MessageQueue(
        n_reader=1,
        n_local_reader=1,
        local_reader_ranks=[0],
        max_chunk_bytes=1024,
        max_chunks=4,
    )
    reader = MessageQueue.create_from_handle(writer.export_handle(), rank=0)
    writer.wait_until_ready()
    reader.wait_until_ready()

    overflow_send_started = threading.Event()
    release_overflow_send = threading.Event()
    overflow_poll_started = threading.Event()
    writer.local_socket = _DelayedMultipartSocket(
        writer.local_socket,
        overflow_send_started,
        release_overflow_send,
    )
    reader.local_socket = _ObservedPollSocket(reader.local_socket, overflow_poll_started)

    recording_worker = _RecordingWorker()
    worker_proc = WorkerProc.__new__(WorkerProc)
    worker_proc.gpu_id = 0
    worker_proc.mq = reader
    worker_proc.worker = recording_worker
    worker_proc.result_mq = None
    worker_proc.wake_event = None
    worker_proc._running = True

    # Make the pre-fix implementation fail quickly: it read this module-level
    # value directly from _worker_busy_loop. The fixed implementation ignores it
    # and delegates to recv_message(), which waits indefinitely.
    monkeypatch.setattr(
        diffusion_worker_module,
        "_BROADCAST_DEQUEUE_TIMEOUT_S",
        0.02,
        raising=False,
    )

    overflow_message = {
        "type": "rpc",
        "method": "record",
        "args": ("overflow-1",),
        "padding": b"x" * 2048,
    }
    inline_message = {
        "type": "rpc",
        "method": "record",
        "args": ("inline-2",),
    }
    shutdown_message = {"type": "shutdown"}
    thread_errors: list[BaseException] = []

    def send_messages():
        try:
            writer.enqueue(overflow_message)
            writer.enqueue(inline_message)
            writer.enqueue(shutdown_message)
        except BaseException as exc:
            thread_errors.append(exc)

    def run_busy_loop():
        try:
            worker_proc._worker_busy_loop()
        except BaseException as exc:
            thread_errors.append(exc)

    writer_thread = threading.Thread(target=send_messages, daemon=True)
    worker_thread = threading.Thread(target=run_busy_loop, daemon=True)

    try:
        writer_thread.start()
        worker_thread.start()

        assert overflow_send_started.wait(timeout=2)
        assert overflow_poll_started.wait(timeout=2)

        # Keep the multipart unavailable beyond the pre-fix 20 ms timeout.
        time.sleep(0.1)
        release_overflow_send.set()

        writer_thread.join(timeout=2)
        worker_thread.join(timeout=2)

        assert not writer_thread.is_alive()
        assert not worker_thread.is_alive()
        assert thread_errors == []
        assert recording_worker.calls == ["overflow-1", "inline-2"]
    finally:
        release_overflow_send.set()
        worker_proc._running = False
        reader.shutdown()
        writer.shutdown()
        writer_thread.join(timeout=1)
        worker_thread.join(timeout=1)