vllm-omni/vllm_omni/worker_v2/delivery.py

# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project

from __future__ import annotations

import threading
import time
from enum import Enum, auto


class DeliveryState(Enum):
    QUEUED = auto()
    IN_FLIGHT = auto()
    DELIVERED = auto()
    FAILED = auto()
    CANCELLED = auto()


class DeliveryTimeoutError(TimeoutError):
    pass


class DeliveryCancelledError(RuntimeError):
    pass


_TERMINAL_STATES = {
    DeliveryState.DELIVERED,
    DeliveryState.FAILED,
    DeliveryState.CANCELLED,
}


class DeliveryTicket:
    """Exactly-once completion for one MRv2 connector delivery."""

    def __init__(
        self,
        manager: OmniDeliveryManager,
        *,
        request_id: str,
        put_key: str,
        deadline: float,
    ) -> None:
        self._manager = manager
        self.request_id = request_id
        self.put_key = put_key
        self.deadline = deadline
        self._state = DeliveryState.QUEUED
        self._error: BaseException | None = None
        self._event = threading.Event()

    @property
    def state(self) -> DeliveryState:
        with self._manager._lock:
            return self._state

    def mark_in_flight(self) -> bool:
        return self._manager._mark_in_flight(self)

    def set_delivered(self) -> bool:
        return self._manager._set_delivered(self)

    def set_failed(self, error: BaseException) -> bool:
        return self._manager._set_failed(self, error)

    # Compatibility with the connector mixin's completion interface.
    def set_result(self) -> bool:
        return self.set_delivered()

    def set_error(self, error: BaseException) -> bool:
        return self.set_failed(error)

    def wait(self) -> None:
        while True:
            with self._manager._lock:
                state = self._state
                error = self._error
                remaining = self.deadline - time.monotonic()
            if state is DeliveryState.DELIVERED:
                return
            if state in (DeliveryState.FAILED, DeliveryState.CANCELLED):
                assert error is not None
                raise error
            if remaining <= 0:
                self._manager._timeout(self)
                continue
            self._event.wait(timeout=remaining)


class OmniDeliveryManager:
    """MRv2-only connector delivery state machine.

    A timeout or permanent delivery failure quarantines the whole manager.
    This is a stage-fatal condition: no new payload may be accepted after the
    transport can no longer prove ordered, exactly-once completion.
    """

    def __init__(
        self,
        *,
        delivery_timeout_s: float,
        shutdown_timeout_s: float = 5.0,
    ) -> None:
        if delivery_timeout_s <= 0:
            raise ValueError("delivery_timeout_s must be positive")
        if shutdown_timeout_s <= 0:
            raise ValueError("shutdown_timeout_s must be positive")
        self.delivery_timeout_s = float(delivery_timeout_s)
        self.shutdown_timeout_s = float(shutdown_timeout_s)
        self._lock = threading.Lock()
        self._tickets: set[DeliveryTicket] = set()
        self._quarantine_error: BaseException | None = None
        self._closed = False

    @property
    def is_quarantined(self) -> bool:
        with self._lock:
            return self._quarantine_error is not None

    def create_ticket(self, *, request_id: str, put_key: str) -> DeliveryTicket:
        with self._lock:
            if self._closed:
                raise RuntimeError("MRv2 delivery manager is closed")
            if self._quarantine_error is not None:
                raise RuntimeError("MRv2 delivery manager is quarantined") from self._quarantine_error
            ticket = DeliveryTicket(
                self,
                request_id=request_id,
                put_key=put_key,
                deadline=time.monotonic() + self.delivery_timeout_s,
            )
            self._tickets.add(ticket)
            return ticket

    def quarantine(self, error: BaseException) -> None:
        with self._lock:
            self._quarantine_locked(error, primary_ticket=None)

    def shutdown(self, cause: BaseException | None = None) -> None:
        with self._lock:
            if self._closed:
                return
            self._closed = True
            if self._quarantine_error is None:
                self._quarantine_error = cause or RuntimeError("MRv2 stage shutdown")
            for ticket in tuple(self._tickets):
                if ticket._state in _TERMINAL_STATES:
                    continue
                message = f"connector delivery cancelled for request {ticket.request_id!r}, key {ticket.put_key!r}"
                if cause is not None:
                    message += f": {cause}"
                self._finish_locked(
                    ticket,
                    DeliveryState.CANCELLED,
                    DeliveryCancelledError(message),
                )

    def _mark_in_flight(self, ticket: DeliveryTicket) -> bool:
        with self._lock:
            if ticket._state is not DeliveryState.QUEUED:
                return False
            ticket._state = DeliveryState.IN_FLIGHT
            return True

    def _set_delivered(self, ticket: DeliveryTicket) -> bool:
        with self._lock:
            if ticket._state in _TERMINAL_STATES:
                return False
            return self._finish_locked(ticket, DeliveryState.DELIVERED, None)

    def _set_failed(self, ticket: DeliveryTicket, error: BaseException) -> bool:
        with self._lock:
            if ticket._state in _TERMINAL_STATES:
                return False
            self._quarantine_locked(error, primary_ticket=ticket)
            return True

    def _timeout(self, ticket: DeliveryTicket) -> bool:
        error = DeliveryTimeoutError(
            "connector delivery timed out after "
            f"{self.delivery_timeout_s:.3f}s for request "
            f"{ticket.request_id!r}, key {ticket.put_key!r}"
        )
        with self._lock:
            if ticket._state in _TERMINAL_STATES:
                return False
            self._quarantine_locked(error, primary_ticket=ticket)
            return True

    def _quarantine_locked(
        self,
        error: BaseException,
        *,
        primary_ticket: DeliveryTicket | None,
    ) -> None:
        if self._quarantine_error is None:
            self._quarantine_error = error
        stage_error = self._quarantine_error
        for ticket in tuple(self._tickets):
            if ticket._state in _TERMINAL_STATES:
                continue
            ticket_error = error if ticket is primary_ticket else stage_error
            self._finish_locked(ticket, DeliveryState.FAILED, ticket_error)

    def _finish_locked(
        self,
        ticket: DeliveryTicket,
        state: DeliveryState,
        error: BaseException | None,
    ) -> bool:
        if ticket._state in _TERMINAL_STATES:
            return False
        ticket._state = state
        ticket._error = error
        self._tickets.discard(ticket)
        ticket._event.set()
        return True