vllm-omni/tests/entrypoints/duplex/test_duplex_serving.py

# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project

"""The thin websocket handler: handshake, command translation, event pump, resume/takeover."""

from __future__ import annotations

import asyncio
import base64
import json
from collections.abc import Awaitable, Callable
from contextlib import suppress
from typing import Any

import pytest
from fastapi import WebSocketDisconnect

from vllm_omni.config.stage_config import DuplexSessionRuntimeConfig
from vllm_omni.engine.duplex import commands
from vllm_omni.engine.duplex.config import DuplexCapabilities
from vllm_omni.engine.duplex.events import AudioDelta, DuplexEvent, SessionClosed, SessionCreated
from vllm_omni.engine.duplex.messages import DuplexSessionError
from vllm_omni.entrypoints.duplex.realtime_input import RealtimeEnvelope, parse_resume_request
from vllm_omni.entrypoints.duplex.serving import OmniDuplexSessionHandler
from vllm_omni.entrypoints.duplex.websocket import MAX_EVENT_BYTES

pytestmark = [pytest.mark.core_model, pytest.mark.cpu]

_DISCONNECT = object()


class FakeWebSocket:
    def __init__(self, query: dict[str, str] | None = None) -> None:
        self.query_params = dict(query or {})
        self.sent: list[dict[str, Any]] = []
        self.accepted = False
        self.closed: list[tuple[int, str]] = []
        self._send_failure: str | None = None
        self._inbound: asyncio.Queue[Any] = asyncio.Queue()

    async def accept(self) -> None:
        self.accepted = True

    async def send_json(self, payload: dict[str, Any]) -> None:
        if self.closed or self._send_failure is not None:
            raise RuntimeError(
                self._send_failure or "Unexpected ASGI message 'websocket.send', after sending 'websocket.close'."
            )
        self.sent.append(json.loads(json.dumps(payload)))

    async def receive_text(self) -> str:
        item = await self._inbound.get()
        if item is _DISCONNECT:
            raise WebSocketDisconnect(code=1000)
        return item

    async def close(self, code: int = 1000, reason: str = "") -> None:
        self.closed.append((code, reason))
        # A closed socket ends the reader exactly like a client disconnect.
        self._inbound.put_nowait(_DISCONNECT)

    # ---- test helpers ----

    def feed(self, payload: dict[str, Any] | str) -> None:
        self._inbound.put_nowait(payload if isinstance(payload, str) else json.dumps(payload))

    def disconnect(self) -> None:
        self._inbound.put_nowait(_DISCONNECT)

    def break_sends(self) -> None:
        """Kill the write half only: the reader stays parked, as it does in practice."""
        self._send_failure = "Unexpected ASGI message 'websocket.send', after sending 'websocket.close'."

    def types(self) -> list[str]:
        return [payload["type"] for payload in self.sent]

    async def wait_for(self, wire_type: str, *, timeout_s: float = 2.0) -> dict[str, Any]:
        loop = asyncio.get_running_loop()
        deadline = loop.time() + timeout_s
        while loop.time() < deadline:
            for payload in self.sent:
                if payload["type"] == wire_type:
                    return payload
            await asyncio.sleep(0.005)
        raise AssertionError(f"no {wire_type!r} in {self.types()}")


class FakeHandle:
    def __init__(self, session_id: str, capabilities: DuplexCapabilities, *, idle_timeout_s: float = 300) -> None:
        self.session_id = session_id
        self.capabilities = capabilities
        self.public_session: dict[str, Any] = {"id": session_id, "idle_timeout_s": idle_timeout_s}
        self.lease_generation = 0
        self.closed = False
        self.close_reasons: list[str] = []
        self.commands: list[commands.DuplexCommand] = []
        self._outbox: asyncio.Queue[DuplexEvent] = asyncio.Queue()

    def deliver(self, event: DuplexEvent) -> None:
        self._outbox.put_nowait(event)

    async def submit(self, command: commands.DuplexCommand) -> None:
        if self.closed:
            raise DuplexSessionError("closed", code="session_closed", session_id=self.session_id)
        self.commands.append(command)

    async def events(self):
        while True:
            event = await self._outbox.get()
            yield event
            if event.is_terminal:
                return

    async def close(self, *, reason: str = "client_close", timeout: float | None = None) -> None:
        self.close_reasons.append(reason)
        if self.closed:
            return
        self.closed = True
        self.deliver(SessionClosed(session_id=self.session_id, reason=reason))


class FakeOmni:
    def __init__(
        self, *, resumable: bool = True, replay_max_bytes: int = 64 * 1024, idle_timeout_s: float = 300
    ) -> None:
        self.duplex_session_config = DuplexSessionRuntimeConfig(
            resume_replay_ttl_s=60.0, resume_replay_max_bytes_per_session=replay_max_bytes
        )
        self.capabilities = DuplexCapabilities(supports_session_resume=resumable)
        self.idle_timeout_s = idle_timeout_s
        self.opened: list[dict[str, Any]] = []
        self.handles: dict[str, FakeHandle] = {}
        self.resumed: list[tuple[str, int]] = []
        self.detached: list[str] = []
        #: Detaches the engine refused because the caller's lease generation was stale.
        self.detach_refused: list[tuple[str, int]] = []
        self.open_error: DuplexSessionError | None = None
        #: When set, ``resume_session`` parks on it once the engine applied the
        #: resume: the lease generation is bumped, the result is still in flight.
        self.resume_gate: asyncio.Event | None = None
        self.resume_started = asyncio.Event()
        #: Settlements of resumes whose caller was cancelled mid-RPC (DuplexOmni's contract).
        self.compensations: list[asyncio.Task[None]] = []

    async def open_session(self, config: Any) -> FakeHandle:
        self.opened.append(dict(config))
        if self.open_error is not None:
            raise self.open_error
        session_id = f"duplex-{len(self.handles) + 1:032x}"
        handle = FakeHandle(session_id, self.capabilities, idle_timeout_s=self.idle_timeout_s)
        self.handles[session_id] = handle
        handle.deliver(SessionCreated(session_id=session_id, session={"id": session_id, "model": config.get("model")}))
        return handle

    def get_session(self, session_id: str) -> FakeHandle | None:
        return self.handles.get(session_id)

    async def resume_session(
        self,
        session_id: str,
        *,
        expected_lease_generation: int,
        on_abandoned: Callable[[int], Awaitable[None]] | None = None,
    ) -> FakeHandle:
        self.resumed.append((session_id, expected_lease_generation))
        handle = self.handles[session_id]
        lease_generation = expected_lease_generation + 1
        handle.lease_generation = lease_generation
        if self.resume_gate is not None:
            self.resume_started.set()
            gate = self.resume_gate
            try:
                await gate.wait()
            except asyncio.CancelledError:
                # DuplexOmni's contract: the engine applied the resume; once
                # its answer arrives the landed generation is settled off the
                # cancelled task, through the caller's callback if it gave one.
                async def settle() -> None:
                    await gate.wait()
                    if on_abandoned is not None:
                        await on_abandoned(lease_generation)
                    else:
                        with suppress(DuplexSessionError):
                            await self.detach_session(session_id, expected_lease_generation=lease_generation)

                self.compensations.append(asyncio.create_task(settle()))
                raise
        return handle

    async def detach_session(self, session_id: str, *, expected_lease_generation: int | None = None) -> None:
        handle = self.handles[session_id]
        if expected_lease_generation is not None and expected_lease_generation != handle.lease_generation:
            # The engine's fence: a detach of a lease the caller no longer holds is refused.
            self.detach_refused.append((session_id, expected_lease_generation))
            raise DuplexSessionError(
                "duplex lease generation mismatch", code="session_resume_conflict", session_id=session_id
            )
        self.detached.append(session_id)


def _handler(omni: FakeOmni, **kwargs: Any) -> OmniDuplexSessionHandler:
    kwargs.setdefault("config_timeout_s", 1.0)
    kwargs.setdefault("idle_timeout_s", 5.0)
    return OmniDuplexSessionHandler(duplex_omni=omni, **kwargs)


def _session_update(**session: Any) -> dict[str, Any]:
    return {"type": "session.update", "session": {"model": "test-model", "modalities": ["audio", "text"], **session}}


async def _open(
    handler: OmniDuplexSessionHandler, omni: FakeOmni, **session: Any
) -> tuple[FakeWebSocket, FakeHandle, asyncio.Task]:
    ws = FakeWebSocket({"duplex": "1", "autostart": "0"})
    task = asyncio.create_task(handler.handle_realtime_session(ws))
    ws.feed(_session_update(**session))
    created = await ws.wait_for("session.created")
    handle = omni.handles[created["session"]["id"]]
    return ws, handle, task


# --------------------------------------------------------------------------- #
# Handshake and commands                                                      #
# --------------------------------------------------------------------------- #


@pytest.mark.asyncio
async def test_session_update_opens_a_session_and_announces_the_server_id() -> None:
    omni = FakeOmni()
    handler = _handler(omni)
    ws, handle, task = await _open(handler, omni, session_id="mine", instructions="hi")

    assert ws.accepted
    created = ws.sent[0]
    assert created["type"] == "session.created"
    assert created["session"]["id"] == handle.session_id != "mine"
    assert created["attachment_generation"] == 1
    assert isinstance(created["resume_token"], str) and created["resume_token"]
    assert "incarnation" not in created
    assert "server_event_seq" not in created
    # The whole session object went to open_session; the id inside it is ignored there.
    assert omni.opened == [
        {"model": "test-model", "modalities": ["audio", "text"], "session_id": "mine", "instructions": "hi"}
    ]

    ws.feed({"type": "input_audio_buffer.append", "audio": base64.b64encode(b"\x00\x01" * 8).decode("ascii")})
    ws.feed({"type": "input_audio_buffer.commit", "event_id": "evt-c"})
    await asyncio.sleep(0.05)
    assert [type(command) for command in handle.commands] == [commands.AppendAudio, commands.Commit]
    assert handle.commands[1].event_id == "evt-c"

    ws.disconnect()
    await asyncio.wait_for(task, timeout=2.0)
    # A resumable session is detached (engine-owned grace), not closed.
    assert omni.detached == [handle.session_id]
    assert handle.close_reasons == []


@pytest.mark.asyncio
async def test_session_events_are_journaled_and_sent_in_order() -> None:
    omni = FakeOmni()
    handler = _handler(omni)
    ws, handle, task = await _open(handler, omni)

    handle.deliver(AudioDelta(session_id=handle.session_id, response_id="r1", delta="aGk="))
    delta = await ws.wait_for("response.output_audio.delta")
    assert delta["delta"] == "aGk=" and delta["server_event_seq"] == 1

    ws.feed({"type": "session.event_ack", "server_event_seq": 1})
    ws.feed({"type": "session.event_ack", "server_event_seq": 9, "event_id": "evt-ack"})
    error = await ws.wait_for("error")
    assert error["error"]["code"] == "invalid_event_ack" and error["error"]["event_id"] == "evt-ack"

    await handle.close(reason="client_close")
    closed = await ws.wait_for("session.closed")
    assert closed["reason"] == "client_close"
    await asyncio.wait_for(task, timeout=2.0)
    assert ws.closed[0][1] == "client_close"


@pytest.mark.asyncio
async def test_envelope_errors_are_answered_locally_without_touching_the_session() -> None:
    omni = FakeOmni()
    handler = _handler(omni)
    ws, handle, task = await _open(handler, omni)

    ws.feed("not json")
    ws.feed("x" * (MAX_EVENT_BYTES + 1))
    ws.feed({"type": "totally.unknown", "event_id": "evt-u"})
    ws.feed({"type": "session.resume", "event_id": "evt-r"})
    ws.feed({"type": "playback.ack", "event_id": "evt-p"})  # played_ms missing
    await asyncio.sleep(0.1)

    codes = [payload["error"]["code"] for payload in ws.sent if payload["type"] == "error"]
    assert codes == ["invalid_json", "event_too_large", "unknown_event", "unsupported_session_resume", "bad_event"]
    errors = [payload for payload in ws.sent if payload["type"] == "error"]
    assert errors[2]["error"]["event_id"] == "evt-u"
    assert errors[4]["error"]["event_id"] == "evt-p"
    assert handle.commands == []
    ws.disconnect()
    await asyncio.wait_for(task, timeout=2.0)


@pytest.mark.asyncio
async def test_open_rejection_is_reported_and_the_socket_ends() -> None:
    omni = FakeOmni()
    omni.open_error = DuplexSessionError("no room", code="resource_exhausted", retryable=True)
    handler = _handler(omni)
    ws = FakeWebSocket({"duplex": "1"})
    task = asyncio.create_task(handler.handle_realtime_session(ws))
    ws.feed(_session_update())
    await asyncio.wait_for(task, timeout=2.0)
    assert ws.types() == ["error"]
    assert ws.sent[0]["error"]["code"] == "resource_exhausted"


@pytest.mark.asyncio
async def test_config_timeout_and_first_message_validation() -> None:
    omni = FakeOmni()
    handler = _handler(omni, config_timeout_s=0.05)
    ws = FakeWebSocket({"duplex": "1"})
    await asyncio.wait_for(handler.handle_realtime_session(ws), timeout=2.0)
    assert ws.sent[0]["error"]["code"] == "config_timeout"

    ws = FakeWebSocket({"duplex": "1"})
    task = asyncio.create_task(handler.handle_realtime_session(ws))
    ws.feed("{not json")
    await asyncio.wait_for(task, timeout=2.0)
    assert ws.sent[0]["error"]["code"] == "invalid_json"
    assert omni.opened == []


@pytest.mark.asyncio
async def test_idle_timeout_detaches_a_resumable_session_like_a_disconnect() -> None:
    omni = FakeOmni(idle_timeout_s=0.05)
    handler = _handler(omni, idle_timeout_s=0.05)
    ws, handle, task = await _open(handler, omni)

    await asyncio.wait_for(task, timeout=2.0)
    # Serving makes no session-lifetime decision: the engine lease decides
    # whether a silent, detached session expires.
    assert handle.close_reasons == []
    assert omni.detached == [handle.session_id]


@pytest.mark.asyncio
async def test_idle_timeout_closes_a_non_resumable_session() -> None:
    omni = FakeOmni(idle_timeout_s=0.05, resumable=False)
    handler = _handler(omni, idle_timeout_s=0.05)
    ws, handle, task = await _open(handler, omni)

    await asyncio.wait_for(task, timeout=2.0)
    assert handle.close_reasons == ["disconnect"]
    assert omni.detached == []


@pytest.mark.asyncio
async def test_transport_send_failure_detaches_the_session_instead_of_closing_it() -> None:
    omni = FakeOmni()
    handler = _handler(omni)
    ws, handle, task = await _open(handler, omni)

    # Only the write half dies, so the pump's send is the one and only report
    # of the broken socket (the reader is still parked on receive).
    ws.break_sends()
    handle.deliver(AudioDelta(session_id=handle.session_id, response_id="r1", delta="aGk="))
    await asyncio.sleep(0.05)

    # The engine session is alive and resumable, so this is a disconnect of
    # the current attachment, not a session close; the pump keeps journaling.
    assert handle.close_reasons == []
    assert omni.detached == [handle.session_id]
    assert not handler._pumps[handle.session_id].done()

    # The reader sees the same broken socket a moment later. The attachment is
    # already gone, so this must not detach a second time: another detach would
    # restart the engine's disconnect grace window.
    ws.disconnect()
    await asyncio.wait_for(task, timeout=2.0)
    assert omni.detached == [handle.session_id]
    assert handle.close_reasons == []

    handle.deliver(SessionClosed(session_id=handle.session_id, reason="client_close"))
    await asyncio.sleep(0.05)
    assert handle.session_id not in handler._pumps


@pytest.mark.asyncio
async def test_non_resumable_session_is_closed_on_disconnect() -> None:
    omni = FakeOmni(resumable=False)
    handler = _handler(omni)
    ws, handle, task = await _open(handler, omni)
    assert "resume_token" not in ws.sent[0]

    ws.disconnect()
    await asyncio.wait_for(task, timeout=2.0)
    assert handle.close_reasons == ["disconnect"]
    assert omni.detached == []


@pytest.mark.asyncio
async def test_a_client_close_still_delivers_session_closed_before_the_socket_goes() -> None:
    """The read loop must not outrun the pump's terminal event.

    ``DuplexSessionHandle._deliver`` queues ``SessionClosed`` and marks the
    handle closed in one synchronous step, so ``handle.closed`` is already true
    while the event is still sitting in the outbox. ``_read_loop`` loops on
    exactly that flag: it returned, the endpoint returned, and the ASGI server
    tore the socket down with ``session.closed`` unsent -- the client saw an
    abrupt close (no close frame) instead of the terminal event it was waiting
    for.

    The teardown is what makes this observable, so it is modelled here: a real
    connection stops accepting writes the moment the endpoint returns, which is
    why a pump that is merely *still scheduled* is already too late.
    """
    omni = FakeOmni()
    handler = _handler(omni)
    ws = FakeWebSocket({"duplex": "1", "autostart": "0"})

    async def serve() -> None:
        try:
            await handler.handle_realtime_session(ws)
        finally:
            # The ASGI server drops the connection when the endpoint returns;
            # anything the pump writes after this never reaches the client.
            ws.break_sends()

    task = asyncio.create_task(serve())
    ws.feed(_session_update())
    created = await ws.wait_for("session.created")
    handle = omni.handles[created["session"]["id"]]
    submit = handle.submit

    async def closing_submit(command: commands.DuplexCommand) -> None:
        await submit(command)
        if isinstance(command, commands.CloseSession):
            # Same order as the real handle: queue the event, then flip the flag.
            handle.deliver(SessionClosed(session_id=handle.session_id, reason="client_close"))
            handle.closed = True

    handle.submit = closing_submit  # type: ignore[method-assign]

    ws.feed({"type": "session.close"})
    await asyncio.wait_for(task, timeout=2.0)

    assert ws.types()[-1] == "session.closed", f"terminal event never reached the wire: {ws.types()}"
    assert ws.closed and ws.closed[0][0] == 1000, "the socket must close with a normal close frame"


# --------------------------------------------------------------------------- #
# Resume and takeover                                                         #
# --------------------------------------------------------------------------- #


async def _resume(
    handler: OmniDuplexSessionHandler,
    session_id: str,
    token: str,
    *,
    last_seq: int = 0,
) -> tuple[FakeWebSocket, asyncio.Task]:
    ws = FakeWebSocket({"duplex": "1", "resume": "1"})
    task = asyncio.create_task(handler.handle_realtime_session(ws))
    ws.feed(
        {
            "type": "session.resume",
            "session_id": session_id,
            "resume_token": token,
            "last_received_server_event_seq": last_seq,
        }
    )
    return ws, task


@pytest.mark.asyncio
async def test_resume_after_disconnect_replays_missed_events_and_rotates_the_token() -> None:
    omni = FakeOmni()
    handler = _handler(omni)
    ws, handle, task = await _open(handler, omni)
    token = ws.sent[0]["resume_token"]
    handle.deliver(AudioDelta(session_id=handle.session_id, response_id="r1", delta="one"))
    await ws.wait_for("response.output_audio.delta")
    ws.disconnect()
    await asyncio.wait_for(task, timeout=2.0)
    assert omni.detached == [handle.session_id]
    # Emitted while detached: journaled for replay only.
    handle.deliver(AudioDelta(session_id=handle.session_id, response_id="r1", delta="two"))
    await asyncio.sleep(0.05)

    ws2, task2 = await _resume(handler, handle.session_id, token, last_seq=1)
    resumed = await ws2.wait_for("session.resumed")
    assert resumed["session_id"] == handle.session_id
    assert resumed["attachment_generation"] == 2
    assert resumed["resume_token"] != token
    assert "incarnation" not in resumed
    replayed = await ws2.wait_for("response.output_audio.delta")
    assert replayed["delta"] == "two" and replayed["server_event_seq"] == 2
    assert omni.resumed == [(handle.session_id, 0)]

    # The old token is revoked by the rotation.
    ws3, task3 = await _resume(handler, handle.session_id, token)
    await asyncio.wait_for(task3, timeout=2.0)
    assert ws3.sent[0]["error"]["code"] == "invalid_resume_token"

    ws2.feed({"type": "session.close"})
    await asyncio.sleep(0.05)
    ws2.disconnect()
    await asyncio.wait_for(task2, timeout=2.0)


@pytest.mark.asyncio
async def test_resume_takes_over_a_live_attachment() -> None:
    omni = FakeOmni()
    handler = _handler(omni)
    ws, handle, task = await _open(handler, omni)
    token = ws.sent[0]["resume_token"]

    ws2, task2 = await _resume(handler, handle.session_id, token)
    await ws2.wait_for("session.resumed")
    replaced = await ws.wait_for("session.replaced")
    assert replaced["attachment_generation"] == 1
    assert ws.closed and ws.closed[0][1] == "session_replaced"
    # The replaced socket's later input is ignored; the new one drives the session.
    ws.feed({"type": "input_audio_buffer.clear"})
    ws2.feed({"type": "input_audio_buffer.clear", "event_id": "evt-new"})
    await asyncio.sleep(0.05)
    assert [command.event_id for command in handle.commands] == ["evt-new"]
    await asyncio.wait_for(task, timeout=2.0)

    ws2.disconnect()
    await asyncio.wait_for(task2, timeout=2.0)
    assert omni.detached == [handle.session_id]


@pytest.mark.asyncio
async def test_resume_validation_errors() -> None:
    omni = FakeOmni()
    handler = _handler(omni)
    ws = FakeWebSocket({"duplex": "1", "resume": "1"})
    task = asyncio.create_task(handler.handle_realtime_session(ws))
    ws.feed({"type": "session.resume", "session_id": "duplex-x"})
    await asyncio.wait_for(task, timeout=2.0)
    assert ws.sent[0]["error"]["code"] == "invalid_session_resume"

    ws, task = await _resume(handler, "duplex-unknown", "token")
    await asyncio.wait_for(task, timeout=2.0)
    assert ws.sent[0]["error"]["code"] == "session_resume_expired"


@pytest.mark.asyncio
async def test_journal_overflow_degrades_to_live_delivery_with_resync_required() -> None:
    omni = FakeOmni(replay_max_bytes=256)
    handler = _handler(omni)
    ws, handle, task = await _open(handler, omni)

    handle.deliver(AudioDelta(session_id=handle.session_id, response_id="r1", delta="x" * 400))
    resync = await ws.wait_for("session.resync_required")
    assert resync["reason"] == "journal_overflow"
    delta = await ws.wait_for("response.output_audio.delta")
    assert "server_event_seq" not in delta
    assert ws.types().index("session.resync_required") < ws.types().index("response.output_audio.delta")

    ws.disconnect()
    await asyncio.wait_for(task, timeout=2.0)


# --------------------------------------------------------------------------- #
# Envelope helpers                                                            #
# --------------------------------------------------------------------------- #


def test_realtime_envelope_query_rules() -> None:
    envelope = RealtimeEnvelope.from_query_params({"model": "m"})
    assert envelope.initial_open_payload() == {"model": "m"}
    assert envelope.initial_open_payload() is None  # autostart happens once

    envelope = RealtimeEnvelope.from_query_params({"model": "m", "autostart": "0"})
    assert envelope.resume_only is True
    assert envelope.initial_open_payload() is None
    assert RealtimeEnvelope.from_query_params({"resume": "1"}).resume_only is True
    assert RealtimeEnvelope.from_query_params({"session_id": "mine"}).default_session_payload() == {"model": None}


def test_realtime_envelope_first_message_classification() -> None:
    envelope = RealtimeEnvelope.from_query_params({"model": "m", "autostart": "0"})
    resume = envelope.first_message({"type": "session.resume", "session_id": "s"})
    assert resume.kind == "resume" and resume.resume_payload["session_id"] == "s"

    envelope = RealtimeEnvelope.from_query_params({"model": "m", "autostart": "0"})
    opened = envelope.first_message({"type": "session.update", "session": {"model": "x", "instructions": "hi"}})
    assert opened.kind == "open" and opened.session_payload == {"model": "x", "instructions": "hi"}
    assert opened.pending_command_payload is None

    envelope = RealtimeEnvelope.from_query_params({"model": "m", "autostart": "0"})
    autostarted = envelope.first_message({"type": "input_audio_buffer.commit"})
    assert autostarted.kind == "open" and autostarted.session_payload == {"model": "m"}
    assert autostarted.pending_command_payload == {"type": "input_audio_buffer.commit"}


def test_parse_resume_request_requires_the_three_fields_only() -> None:
    request = parse_resume_request({"session_id": "s", "resume_token": "t", "last_received_server_event_seq": 3})
    assert request is not None
    assert (request.session_id, request.resume_token, request.last_received_server_event_seq) == ("s", "t", 3)
    assert parse_resume_request({"session_id": "s", "resume_token": "t"}).last_received_server_event_seq == 0
    assert parse_resume_request({"session_id": "s", "resume_token": "t", "incarnation": 1}) is not None
    assert parse_resume_request({"session_id": "s"}) is None
    assert parse_resume_request({"session_id": "s", "resume_token": "t", "last_received_server_event_seq": -1}) is None


@pytest.mark.asyncio
async def test_a_resume_that_fails_to_activate_does_not_detach_the_live_attachment() -> None:
    """A rejected resume must roll back only what it owns.

    Two reconnects can both authenticate and both complete the engine resume;
    only one activates. The loser's activation raises, and the rollback used to
    detach the *session*, starting the engine's disconnect grace for the winner
    -- a grace ordinary heartbeats do not clear.

    The activation failure is injected rather than raced for: what is under test
    is the rollback's precondition, not the window that produces it.
    """
    omni = FakeOmni()
    handler = _handler(omni)
    ws, handle, task = await _open(handler, omni)
    token = ws.sent[0]["resume_token"]
    registry = handler._attachment_registry
    try:
        assert await registry.has_attachment(handle.session_id), "the opener is attached"
        detached_before = list(omni.detached)

        async def failing_resume(*args, **kwargs):
            raise RuntimeError("transport activation lost the race")

        original_resume = registry.resume
        registry.resume = failing_resume  # type: ignore[method-assign]
        try:
            ws_lose, task_lose = await _resume(handler, handle.session_id, token)
            error = await ws_lose.wait_for("error")
        finally:
            registry.resume = original_resume  # type: ignore[method-assign]

        assert error["error"]["code"] == "session_resume_conflict"
        assert omni.detached == detached_before, "the loser must not detach the winner"
        assert await registry.has_attachment(handle.session_id), "the winner is still attached"
    finally:
        for pending in (task, locals().get("task_lose")):
            if pending is not None:
                pending.cancel()
                with suppress(asyncio.CancelledError, Exception):
                    await pending


# --------------------------------------------------------------------------- #
# Cancelled resume and the engine disconnect grace (#7636 Issue 3)            #
# --------------------------------------------------------------------------- #


@pytest.mark.asyncio
async def test_a_resume_cancelled_during_activation_puts_the_engine_lease_back_into_grace() -> None:
    """``_resume`` resumes the engine lease first; a cancel mid-activation must detach it again.

    The rollback used to catch ``Exception`` only. A handler task cancelled
    while sending ``session.resumed`` or a replay entry left the engine with
    ``detached_at=None`` and the outer handler with no attachment to clean up:
    the reaper reclaimed nothing after the grace, and with one slot the next
    open was refused with ``resource_exhausted``.
    """
    omni = FakeOmni()
    handler = _handler(omni)
    ws, handle, task = await _open(handler, omni)
    token = ws.sent[0]["resume_token"]
    ws.disconnect()
    await asyncio.wait_for(task, timeout=2.0)
    assert omni.detached == [handle.session_id]
    registry = handler._attachment_registry

    activating = asyncio.Event()

    async def parked_resume(*args, **kwargs):
        activating.set()
        await asyncio.Event().wait()

    original_resume = registry.resume
    registry.resume = parked_resume  # type: ignore[method-assign]
    try:
        ws2, task2 = await _resume(handler, handle.session_id, token)
        await asyncio.wait_for(activating.wait(), timeout=2.0)
        assert omni.resumed == [(handle.session_id, 0)], "the engine lease was resumed before activation"

        task2.cancel()
        with suppress(asyncio.CancelledError):
            await task2
    finally:
        registry.resume = original_resume  # type: ignore[method-assign]

    # The lease is detached again, so the disconnect grace runs for it.
    assert omni.detached == [handle.session_id, handle.session_id]
    assert not await registry.has_attachment(handle.session_id)
    assert handle.close_reasons == []
    # And the session is still resumable with the token the cancelled attempt presented.
    ws3, task3 = await _resume(handler, handle.session_id, token)
    resumed = await ws3.wait_for("session.resumed")
    assert resumed["session_id"] == handle.session_id
    ws3.disconnect()
    await asyncio.wait_for(task3, timeout=2.0)


@pytest.mark.asyncio
async def test_a_resume_cancelled_after_activation_detaches_the_attachment_it_made() -> None:
    """A cancel after the registry activated the new attachment must undo that attachment too.

    Otherwise the session stays attached to a socket nobody serves, with the
    engine lease resumed: no grace, no expiry, until the idle TTL.
    """
    omni = FakeOmni()
    handler = _handler(omni)
    ws, handle, task = await _open(handler, omni)
    token = ws.sent[0]["resume_token"]
    registry = handler._attachment_registry

    # The takeover notifies the replaced socket after activation; park there.
    notifying = asyncio.Event()

    async def parked_send(payload: dict[str, Any]) -> None:
        notifying.set()
        await asyncio.Event().wait()

    ws.send_json = parked_send  # type: ignore[method-assign]
    ws2, task2 = await _resume(handler, handle.session_id, token)
    await asyncio.wait_for(notifying.wait(), timeout=2.0)
    assert await registry.is_current_attachment(handle.session_id, 2), "the new socket is attached"

    task2.cancel()
    with suppress(asyncio.CancelledError):
        await task2

    assert not await registry.has_attachment(handle.session_id)
    assert omni.detached == [handle.session_id]
    assert handle.close_reasons == []

    task.cancel()
    with suppress(asyncio.CancelledError, Exception):
        await task


@pytest.mark.asyncio
async def test_an_abandoned_resume_never_detaches_the_lease_a_later_resume_owns() -> None:
    """Two connections resuming the same session: the loser's rollback must not touch the winner's lease.

    Interleaving: A has activated and is notifying the socket it replaced; B,
    holding A's rotated token, has completed its engine resume but its RPC
    result has not reached the handler yet, so nothing is attached for B.
    Cancelling A detaches A's registry generation, which used to make the
    engine detach unconditional: it landed on the lease B had just resumed,
    and the reaper expired B after the grace even though it heartbeats. The
    engine now refuses a detach fenced on a generation it has moved past.
    """
    omni = FakeOmni()
    handler = _handler(omni)
    ws, handle, task = await _open(handler, omni)
    token = ws.sent[0]["resume_token"]
    registry = handler._attachment_registry

    # A takes over and parks while notifying the replaced socket (after activation).
    notifying = asyncio.Event()

    async def parked_send(payload: dict[str, Any]) -> None:
        notifying.set()
        await asyncio.Event().wait()

    ws.send_json = parked_send  # type: ignore[method-assign]
    ws_a, task_a = await _resume(handler, handle.session_id, token)
    await asyncio.wait_for(notifying.wait(), timeout=2.0)
    resumed_a = await ws_a.wait_for("session.resumed")
    assert handle.lease_generation == 1

    # B resumes with A's rotated token: the engine applies it, the result is still in flight.
    omni.resume_gate = asyncio.Event()
    ws_b, task_b = await _resume(handler, handle.session_id, resumed_a["resume_token"])
    await asyncio.wait_for(omni.resume_started.wait(), timeout=2.0)
    assert handle.lease_generation == 2

    task_a.cancel()
    with suppress(asyncio.CancelledError):
        await task_a
    assert omni.detached == [], "A's rollback must not detach the lease B resumed"
    assert omni.detach_refused == [(handle.session_id, 1)]
    assert not await registry.has_attachment(handle.session_id), "A's attachment is gone"

    omni.resume_gate.set()
    resumed_b = await ws_b.wait_for("session.resumed")
    assert resumed_b["attachment_generation"] == 3
    assert await registry.is_current_attachment(handle.session_id, 3)
    assert omni.detached == []

    # B's own disconnect detaches the lease B holds.
    ws_b.disconnect()
    await asyncio.wait_for(task_b, timeout=2.0)
    assert omni.detached == [handle.session_id]
    assert omni.detach_refused == [(handle.session_id, 1)]
    task.cancel()
    with suppress(asyncio.CancelledError, Exception):
        await task


@pytest.mark.asyncio
async def test_an_abandoned_takeover_leaves_the_live_connection_serving_and_owning_the_lease() -> None:
    """A takeover cancelled mid-RPC must not put the connection it never replaced into disconnect grace.

    A is attached at generation 0. B's engine resume lands generation 1 and B
    is cancelled before it reaches the registry, so A stays the current
    attachment. Settling B's generation must hand it to A rather than detach
    it: A keeps serving, and A's own disconnect later detaches generation 1,
    which is the lease the engine actually holds.
    """
    omni = FakeOmni()
    handler = _handler(omni)
    ws, handle, task = await _open(handler, omni)
    token = ws.sent[0]["resume_token"]
    registry = handler._attachment_registry

    omni.resume_gate = asyncio.Event()
    ws_b, task_b = await _resume(handler, handle.session_id, token)
    await asyncio.wait_for(omni.resume_started.wait(), timeout=2.0)
    assert handle.lease_generation == 1

    task_b.cancel()
    with suppress(asyncio.CancelledError):
        await task_b
    omni.resume_gate.set()
    await asyncio.gather(*omni.compensations)

    assert omni.detached == [], "A is still serving: its lease must not enter disconnect grace"
    assert omni.detach_refused == []
    assert await registry.is_current_attachment(handle.session_id, 1), "A is still the attachment"
    handle.deliver(AudioDelta(session_id=handle.session_id, response_id="r1", delta="aGk="))
    await ws.wait_for("response.output_audio.delta")

    # A now owns generation 1: its disconnect detaches the lease the engine holds.
    ws.disconnect()
    await asyncio.wait_for(task, timeout=2.0)
    assert omni.detached == [handle.session_id]
    assert omni.detach_refused == []


@pytest.mark.asyncio
async def test_a_send_failure_detaches_the_failed_sockets_own_lease_not_a_pending_resumes() -> None:
    """The pump's detach is fenced on the lease of the socket whose send failed, captured with it.

    The pump holds the registry's outbound lock while sending to A. B's
    engine resume lands generation 1 meanwhile and B waits for that lock to
    activate. When A's send fails, the pump drops A and must detach A's
    generation 0, which the engine refuses, not B's generation 1, which would
    put B into disconnect grace the moment it activates.
    """
    omni = FakeOmni()
    handler = _handler(omni)
    ws, handle, task = await _open(handler, omni)
    token = ws.sent[0]["resume_token"]
    registry = handler._attachment_registry

    sending = asyncio.Event()
    fail_send = asyncio.Event()

    async def parked_failing_send(payload: dict[str, Any]) -> None:
        sending.set()
        await fail_send.wait()
        raise RuntimeError("socket closed")

    ws.send_json = parked_failing_send  # type: ignore[method-assign]
    handle.deliver(AudioDelta(session_id=handle.session_id, response_id="r1", delta="aGk="))
    await asyncio.wait_for(sending.wait(), timeout=2.0)

    # B: the engine resume lands; activation waits for the outbound lock the pump holds.
    ws_b, task_b = await _resume(handler, handle.session_id, token)
    await asyncio.sleep(0.05)
    assert omni.resumed == [(handle.session_id, 0)]
    assert handle.lease_generation == 1
    assert await registry.is_current_attachment(handle.session_id, 1), "B has not activated yet"

    fail_send.set()
    resumed_b = await ws_b.wait_for("session.resumed")
    assert resumed_b["attachment_generation"] == 2
    assert omni.detached == [], "A's lease (generation 0) is refused, B's lease is untouched"
    assert omni.detach_refused == [(handle.session_id, 0)]
    assert await registry.is_current_attachment(handle.session_id, 2)

    # B serves; its own disconnect detaches the lease it resumed.
    ws_b.disconnect()
    await asyncio.wait_for(task_b, timeout=2.0)
    assert omni.detached == [handle.session_id]
    ws.disconnect()
    with suppress(asyncio.CancelledError, Exception):
        await asyncio.wait_for(task, timeout=2.0)


@pytest.mark.asyncio
async def test_an_abandoned_resume_goes_to_the_resume_waiting_to_activate_not_the_socket_it_replaces() -> None:
    """Three connections: A sending, B resumed and waiting for the outbound lock, C abandoned mid-RPC.

    C's generation 2 must be parked for B's activation. If it were handed to
    A instead, A's send failure would detach generation 2 (accepted by the
    engine), and B would then activate on a lease already in disconnect grace
    and expire despite heartbeating.
    """
    omni = FakeOmni()
    handler = _handler(omni)
    ws, handle, task = await _open(handler, omni)
    token = ws.sent[0]["resume_token"]
    registry = handler._attachment_registry

    sending = asyncio.Event()
    fail_send = asyncio.Event()

    async def parked_failing_send(payload: dict[str, Any]) -> None:
        sending.set()
        await fail_send.wait()
        raise RuntimeError("socket closed")

    ws.send_json = parked_failing_send  # type: ignore[method-assign]
    handle.deliver(AudioDelta(session_id=handle.session_id, response_id="r1", delta="aGk="))
    await asyncio.wait_for(sending.wait(), timeout=2.0)

    # B: engine resume lands generation 1; activation waits for the lock the pump holds.
    ws_b, task_b = await _resume(handler, handle.session_id, token)
    await asyncio.sleep(0.05)
    assert handle.lease_generation == 1

    # C: same unrotated token, engine resume lands generation 2, cancelled mid-RPC.
    omni.resume_gate = asyncio.Event()
    ws_c, task_c = await _resume(handler, handle.session_id, token)
    await asyncio.wait_for(omni.resume_started.wait(), timeout=2.0)
    assert handle.lease_generation == 2
    task_c.cancel()
    with suppress(asyncio.CancelledError):
        await task_c
    omni.resume_gate.set()
    await asyncio.gather(*omni.compensations)
    assert omni.detached == [] and omni.detach_refused == [], "generation 2 is parked for B"

    # A's send fails: the pump drops A and detaches A's own generation 0, refused.
    fail_send.set()
    resumed_b = await ws_b.wait_for("session.resumed")
    assert resumed_b["attachment_generation"] == 2
    assert omni.detach_refused == [(handle.session_id, 0)]
    assert omni.detached == [], "the lease B serves is untouched"
    assert await registry.is_current_attachment(handle.session_id, 2)

    # B serves generation 2 (inherited from C): its disconnect detaches that lease.
    ws_b.disconnect()
    await asyncio.wait_for(task_b, timeout=2.0)
    assert omni.detached == [handle.session_id]
    assert omni.detach_refused == [(handle.session_id, 0)]
    ws.disconnect()
    with suppress(asyncio.CancelledError, Exception):
        await asyncio.wait_for(task, timeout=2.0)


@pytest.mark.asyncio
async def test_a_resume_cancelled_during_replay_settles_the_newer_lease_it_was_handed() -> None:
    """Activation rollback must give back the lease the provisional attachment actually owned.

    A resumed generation 1, delivered ``session.resumed`` and is blocked
    replaying events. B, with A's rotated token, resumed generation 2 and was
    cancelled mid-RPC; its settlement handed 2 to A. When A is then cancelled
    during replay, the registry rolls A back; detaching A's own generation 1
    would be refused and generation 2 would never enter disconnect grace.
    """
    omni = FakeOmni()
    handler = _handler(omni)
    ws, handle, task = await _open(handler, omni)
    token = ws.sent[0]["resume_token"]
    registry = handler._attachment_registry
    ws.disconnect()
    await asyncio.wait_for(task, timeout=2.0)
    assert omni.detached == [handle.session_id]
    # Journaled while detached: A's activation has something to replay.
    handle.deliver(AudioDelta(session_id=handle.session_id, response_id="r1", delta="one"))
    await asyncio.sleep(0.05)

    # A: session.resumed goes out, the replay entry parks.
    ws_a = FakeWebSocket({"duplex": "1", "resume": "1"})
    replaying = asyncio.Event()
    sends = 0
    original_send = ws_a.send_json

    async def send_parking_on_replay(payload: dict[str, Any]) -> None:
        nonlocal sends
        sends += 1
        if sends == 2:
            replaying.set()
            await asyncio.Event().wait()
        await original_send(payload)

    ws_a.send_json = send_parking_on_replay  # type: ignore[method-assign]
    task_a = asyncio.create_task(handler.handle_realtime_session(ws_a))
    ws_a.feed(
        {
            "type": "session.resume",
            "session_id": handle.session_id,
            "resume_token": token,
            "last_received_server_event_seq": 0,
        }
    )
    await asyncio.wait_for(replaying.wait(), timeout=2.0)
    resumed_a = await ws_a.wait_for("session.resumed")
    assert handle.lease_generation == 1

    # B: A's rotated token, engine resume lands generation 2, cancelled mid-RPC.
    omni.resume_gate = asyncio.Event()
    ws_b, task_b = await _resume(handler, handle.session_id, resumed_a["resume_token"])
    await asyncio.wait_for(omni.resume_started.wait(), timeout=2.0)
    assert handle.lease_generation == 2
    task_b.cancel()
    with suppress(asyncio.CancelledError):
        await task_b
    omni.resume_gate.set()
    await asyncio.gather(*omni.compensations)
    assert omni.detached == [handle.session_id], "generation 2 went to A's pending activation"

    # A is cancelled during replay: its rollback settles generation 2, the lease A owned.
    task_a.cancel()
    with suppress(asyncio.CancelledError):
        await task_a
    assert not await registry.has_attachment(handle.session_id)
    assert omni.detached == [handle.session_id, handle.session_id]
    assert omni.detach_refused == [], "generation 1 was never detached: the engine would refuse it"