[Bugfix] Compensate cancelled duplex open, append and resume; fix serving README drift (#7636 Issues 1, 2, 3, 21) (#7785)
Signed-off-by: chickeyton <ngton2014@gmail.com> Signed-off-by: Yueqian Lin <70319226+linyueqian@users.noreply.github.com> Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com> Co-authored-by: Yueqian Lin <70319226+linyueqian@users.noreply.github.com>
This commit is contained in:
19 files changed
+1697
-96
No files matched your search
@@ -85,6 +85,32 @@ def test_detach_grace_and_resume_advance_the_lease_generation() -> None:
|
||||
session.resume_lease(expected_lease_generation=0)
|
||||
|
||||
|
||||
def test_a_resume_replayed_under_its_control_id_answers_without_resuming_again() -> None:
|
||||
"""A caller that lost the answer (cancelled or timed-out waiter) asks again with the same id.
|
||||
|
||||
The engine must tell it the generation that resume produced, not bump the
|
||||
lease a second time, and must still refuse any other resume against the
|
||||
old generation.
|
||||
"""
|
||||
clock = FakeMonotonicClock(10.0)
|
||||
session = _session("sid-replay", clock)
|
||||
session.detach_lease()
|
||||
|
||||
assert session.resume_lease(expected_lease_generation=0, control_id="rpc-a") == 1
|
||||
clock.advance(5.0)
|
||||
session.detach_lease()
|
||||
|
||||
assert session.resume_lease(expected_lease_generation=0, control_id="rpc-a") == 1
|
||||
assert session.lease_generation == 1
|
||||
assert session.lease.detached_at == 15.0, "a replay is an answer, not a new resume"
|
||||
with pytest.raises(ValueError, match="lease generation mismatch"):
|
||||
session.resume_lease(expected_lease_generation=0, control_id="rpc-b")
|
||||
|
||||
assert session.resume_lease(expected_lease_generation=1, control_id="rpc-b") == 2
|
||||
with pytest.raises(ValueError, match="lease generation mismatch"):
|
||||
session.resume_lease(expected_lease_generation=0, control_id="rpc-a")
|
||||
|
||||
|
||||
def test_active_operation_prevents_mid_transaction_expiry() -> None:
|
||||
clock = FakeMonotonicClock(0.0)
|
||||
session = _session("sid-operation", clock, idle_ttl_s=5.0)
|
||||
|
||||
@@ -296,6 +296,7 @@ class Harness:
|
||||
clock: FakeClock
|
||||
output_sink: asyncio.Queue = field(default_factory=asyncio.Queue)
|
||||
result_sink: asyncio.Queue = field(default_factory=asyncio.Queue)
|
||||
_control_seq: int = 0
|
||||
|
||||
@classmethod
|
||||
def create(
|
||||
@@ -344,20 +345,30 @@ class Harness:
|
||||
)
|
||||
return await self.result()
|
||||
|
||||
async def resume(self, session_id: str, *, expected_lease_generation: int) -> DuplexControlResultMessage:
|
||||
async def resume(
|
||||
self, session_id: str, *, expected_lease_generation: int, control_id: str | None = None
|
||||
) -> DuplexControlResultMessage:
|
||||
# Distinct by default: the engine answers a repeated control id as a
|
||||
# replay of the same resume, which only the replay test wants.
|
||||
self._control_seq += 1
|
||||
await self.manager.handle(
|
||||
ResumeDuplexSessionMessage(
|
||||
control_id=f"resume-{session_id}-{expected_lease_generation}",
|
||||
control_id=control_id or f"resume-{session_id}-{expected_lease_generation}-{self._control_seq}",
|
||||
session_id=session_id,
|
||||
expected_lease_generation=expected_lease_generation,
|
||||
)
|
||||
)
|
||||
return await self.result()
|
||||
|
||||
async def touch(self, session_id: str, activity: str) -> DuplexControlResultMessage:
|
||||
async def touch(
|
||||
self, session_id: str, activity: str, *, expected_lease_generation: int | None = None
|
||||
) -> DuplexControlResultMessage:
|
||||
await self.manager.handle(
|
||||
TouchDuplexSessionMessage(
|
||||
control_id=f"touch-{session_id}-{activity}", session_id=session_id, activity=activity
|
||||
control_id=f"touch-{session_id}-{activity}-{expected_lease_generation}",
|
||||
session_id=session_id,
|
||||
activity=activity,
|
||||
expected_lease_generation=expected_lease_generation,
|
||||
)
|
||||
)
|
||||
return await self.result()
|
||||
@@ -1656,6 +1667,168 @@ async def test_reaper_loop_survives_one_cleanup_failure(first_cleanup_delay: flo
|
||||
await asyncio.wait_for(task, timeout=5.0)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Cancelled open (#7636 Issue 1) #
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
class _BlockingResultSink(asyncio.Queue):
|
||||
"""A result sink whose ``put`` parks, so an open can be cancelled after it admitted the session."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.entered = asyncio.Event()
|
||||
self.gate = asyncio.Event()
|
||||
|
||||
async def put(self, item: object) -> None:
|
||||
self.entered.set()
|
||||
await self.gate.wait()
|
||||
await super().put(item)
|
||||
|
||||
|
||||
async def test_an_open_cancelled_while_awaiting_the_plugin_frees_the_admission_slot() -> None:
|
||||
"""``CancelledError`` is not an ``Exception``: the rollback used to be skipped.
|
||||
|
||||
Admission counts ``runners | closing | admitting``, so every open cancelled
|
||||
mid-flight (engine teardown, task cancellation) burned one slot for good;
|
||||
after ``max_sessions`` of them every new session was refused.
|
||||
"""
|
||||
async with Harness.create(max_sessions=1) as harness:
|
||||
harness.plugin.blocked_session_ids.add("blocked")
|
||||
config = DuplexSessionConfig(model="fake-model", instructions="blocked")
|
||||
open_task = asyncio.create_task(harness.open("sid-cancelled", config))
|
||||
await asyncio.wait_for(harness.plugin.runtime_config_started.wait(), timeout=1.0)
|
||||
|
||||
open_task.cancel()
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await open_task
|
||||
|
||||
assert harness.manager.get("sid-cancelled") is None
|
||||
assert harness.manager.active_count() == 0
|
||||
assert harness.result_sink.empty(), "a cancelled open answers nobody"
|
||||
assert (await harness.open("sid-replacement")).ok is True
|
||||
|
||||
|
||||
async def test_an_open_cancelled_after_admission_releases_the_runner_and_its_stage_reservation() -> None:
|
||||
"""The worst landing spot for the cancel: the runner is registered and Stage0 is reserved.
|
||||
|
||||
Both have to be undone, or the runner stays in ``runners`` (one slot gone)
|
||||
and the Stage0 request stays reserved in the orchestrator.
|
||||
"""
|
||||
async with Harness.create(max_sessions=1) as harness:
|
||||
sink = _BlockingResultSink()
|
||||
original_sink = harness.manager._result_sink
|
||||
harness.manager._result_sink = sink
|
||||
open_task = asyncio.create_task(
|
||||
harness.manager.handle(
|
||||
OpenDuplexSessionMessage(
|
||||
control_id="open-cancelled",
|
||||
session_id="sid-cancelled",
|
||||
session_config=DuplexSessionConfig(model="fake-model"),
|
||||
)
|
||||
)
|
||||
)
|
||||
await asyncio.wait_for(sink.entered.wait(), timeout=1.0)
|
||||
assert "sid-cancelled" in harness.manager.runners
|
||||
assert [context.request_id for context in harness.stage_port.ensure_calls] == [
|
||||
stage0_request_id("sid-cancelled")
|
||||
]
|
||||
|
||||
open_task.cancel()
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await open_task
|
||||
harness.manager._result_sink = original_sink
|
||||
|
||||
assert "sid-cancelled" not in harness.manager.runners
|
||||
assert harness.manager.active_count() == 0
|
||||
assert harness.stage_port.cleanup_calls == [([stage0_request_id("sid-cancelled")], False)]
|
||||
assert stage0_request_id("sid-cancelled") not in harness.manager._request_index
|
||||
assert (await harness.open("sid-replacement")).ok is True
|
||||
|
||||
|
||||
async def test_an_open_cancelled_again_during_its_rollback_leaves_the_stage_cleanup_to_the_reaper() -> None:
|
||||
"""A second cancellation landing in the rollback's awaits must not orphan the Stage0 request.
|
||||
|
||||
The runner, the reservations and the request index are already gone by
|
||||
then, so nothing would ever clean the orchestrator's request state up.
|
||||
The rollback records the ids as a pending request cleanup before it
|
||||
awaits anything, and the reaper's retry finishes the job.
|
||||
"""
|
||||
async with Harness.create(max_sessions=1) as harness:
|
||||
sink = _BlockingResultSink()
|
||||
original_sink = harness.manager._result_sink
|
||||
harness.manager._result_sink = sink
|
||||
open_task = asyncio.create_task(
|
||||
harness.manager.handle(
|
||||
OpenDuplexSessionMessage(
|
||||
control_id="open-cancelled-twice",
|
||||
session_id="sid-cancelled",
|
||||
session_config=DuplexSessionConfig(model="fake-model"),
|
||||
)
|
||||
)
|
||||
)
|
||||
await asyncio.wait_for(sink.entered.wait(), timeout=1.0)
|
||||
runner = harness.manager.runners["sid-cancelled"]
|
||||
shutdown_started = asyncio.Event()
|
||||
|
||||
async def parked_shutdown() -> None:
|
||||
shutdown_started.set()
|
||||
await asyncio.Event().wait()
|
||||
|
||||
runner.shutdown = parked_shutdown # type: ignore[method-assign]
|
||||
|
||||
open_task.cancel()
|
||||
await asyncio.wait_for(shutdown_started.wait(), timeout=1.0)
|
||||
open_task.cancel() # lands inside the rollback, before the stage cleanup
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await open_task
|
||||
harness.manager._result_sink = original_sink
|
||||
|
||||
assert "sid-cancelled" not in harness.manager.runners
|
||||
assert harness.manager.active_count() == 0
|
||||
assert harness.stage_port.cleanup_calls == [], "the cleanup await was never reached"
|
||||
key = ("sid-cancelled", 0)
|
||||
assert key in harness.manager._pending_request_cleanups
|
||||
|
||||
await harness.manager.reap_expired()
|
||||
assert harness.stage_port.cleanup_calls == [([stage0_request_id("sid-cancelled")], False)]
|
||||
assert key not in harness.manager._pending_request_cleanups
|
||||
assert (await harness.open("sid-replacement")).ok is True
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Detach fenced on the lease generation #
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
async def test_detach_is_refused_for_a_lease_the_caller_no_longer_holds() -> None:
|
||||
"""A connection giving up its lease must not start the grace for the lease a later resume owns.
|
||||
|
||||
Resume is a CAS on the lease generation; detach is now fenced the same
|
||||
way, so a stale caller (its resume was superseded by another connection's)
|
||||
is refused instead of detaching the winner.
|
||||
"""
|
||||
async with Harness.create() as harness:
|
||||
await harness.open("sid-fence")
|
||||
session = harness.session("sid-fence")
|
||||
assert (await harness.resume("sid-fence", expected_lease_generation=0)).ok is True
|
||||
assert session.lease_generation == 1
|
||||
|
||||
stale = await harness.touch("sid-fence", "detach", expected_lease_generation=0)
|
||||
assert stale.ok is False
|
||||
assert stale.error_code == "session_resume_conflict"
|
||||
assert session.lease.detached_at is None, "the current lease is untouched"
|
||||
|
||||
current = await harness.touch("sid-fence", "detach", expected_lease_generation=1)
|
||||
assert current.ok is True
|
||||
assert session.lease.detached_at is not None
|
||||
|
||||
# An unfenced detach keeps its meaning: whatever generation is current.
|
||||
assert (await harness.resume("sid-fence", expected_lease_generation=1)).ok is True
|
||||
assert (await harness.touch("sid-fence", "detach")).ok is True
|
||||
assert session.lease.detached_at is not None
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Cancellation waits #
|
||||
# --------------------------------------------------------------------------- #
|
||||
@@ -1685,3 +1858,20 @@ async def test_cancel_append_tasks_absorbs_a_task_that_outlives_the_wait() -> No
|
||||
assert await tasks.cancel_append_tasks(timeout_s=0.05) is True
|
||||
assert tasks.append_tail is None
|
||||
await asyncio.wait_for(asyncio.gather(task, return_exceptions=True), timeout=5.0)
|
||||
|
||||
|
||||
async def test_a_resume_replayed_under_its_control_id_reports_the_generation_it_produced() -> None:
|
||||
"""The client side of an abandoned resume: it lost the answer and asks again with the same id."""
|
||||
async with Harness.create() as harness:
|
||||
await harness.open("sid-replay")
|
||||
session = harness.session("sid-replay")
|
||||
first = await harness.resume("sid-replay", expected_lease_generation=0, control_id="rpc-a")
|
||||
assert first.ok is True and first.lease_generation == 1
|
||||
|
||||
replay = await harness.resume("sid-replay", expected_lease_generation=0, control_id="rpc-a")
|
||||
assert replay.ok is True
|
||||
assert replay.lease_generation == 1
|
||||
assert session.lease_generation == 1, "a replay does not resume again"
|
||||
|
||||
other = await harness.resume("sid-replay", expected_lease_generation=0, control_id="rpc-b")
|
||||
assert other.ok is False and other.error_code == "session_resume_conflict"
|
||||
@@ -73,6 +73,9 @@ class RecordingStagePort(DuplexStagePort):
|
||||
self.cleanups: list[tuple[list[str], bool]] = []
|
||||
self.aborts: list[list[str]] = []
|
||||
self.fail_submit: Exception | None = None
|
||||
#: When set, ``submit`` parks on it after signalling ``submit_started``.
|
||||
self.submit_gate: asyncio.Event | None = None
|
||||
self.submit_started = asyncio.Event()
|
||||
|
||||
@property
|
||||
def stage_count(self) -> int:
|
||||
@@ -87,6 +90,9 @@ class RecordingStagePort(DuplexStagePort):
|
||||
async def submit(self, submission: DuplexStageSubmission) -> DuplexStageSubmissionResult:
|
||||
if self.fail_submit is not None:
|
||||
raise self.fail_submit
|
||||
if self.submit_gate is not None:
|
||||
self.submit_started.set()
|
||||
await self.submit_gate.wait()
|
||||
self.submissions.append(submission)
|
||||
return DuplexStageSubmissionResult(
|
||||
request_id=submission.context.request_id,
|
||||
@@ -163,7 +169,7 @@ class Harness:
|
||||
|
||||
def deliver(
|
||||
self,
|
||||
output: object,
|
||||
output: SimpleNamespace,
|
||||
*,
|
||||
stage_id: int = 1,
|
||||
segment_finished: bool = False,
|
||||
@@ -186,7 +192,7 @@ class Harness:
|
||||
raise AssertionError("stage output is missing request_id")
|
||||
return self.runner.on_stage_output(stage_id, output, metrics, request_id=request_id, context=context)
|
||||
|
||||
async def deliver_and_settle(self, output: object, **kwargs: Any) -> list[DuplexEvent]:
|
||||
async def deliver_and_settle(self, output: SimpleNamespace, **kwargs: Any) -> list[DuplexEvent]:
|
||||
self.deliver(output, **kwargs)
|
||||
return await self.settle()
|
||||
|
||||
@@ -1624,3 +1630,133 @@ async def test_server_vad_speech_stopped_still_commits_a_turn_mode_session() ->
|
||||
assert len(_final_submissions(h)) == 1, "the detector's stop commits the turn and starts the response"
|
||||
finally:
|
||||
await close_harness(h)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Cancelled append and its precreated response (#7636 Issue 2) #
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
async def _commit_with_pending_response(h: Harness) -> str:
|
||||
"""Commit a turn whose final append parks in the stage port; return the precreated response id."""
|
||||
await h.run(append_audio())
|
||||
h.port.submit_gate = asyncio.Event()
|
||||
h.submit(commands.Commit(create_response=True))
|
||||
await asyncio.wait_for(h.port.submit_started.wait(), timeout=2.0)
|
||||
# The parked append keeps the runner "busy": settle only until the mailbox is quiet.
|
||||
events = await h.settle(timeout_s=0.3)
|
||||
assert "response.created" in types(events)
|
||||
response_id = h.session.active_response_id
|
||||
assert response_id is not None
|
||||
assert h.runner.tasks.append_tail is not None and not h.runner.tasks.append_tail.done()
|
||||
return response_id
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_an_append_cancelled_from_outside_fails_the_response_it_precreated() -> None:
|
||||
"""A cancelled append gives back the response it precreated, as its docstring promises.
|
||||
|
||||
The ``CancelledError`` branch used to roll back only the PCM reservation,
|
||||
so ``active_response_id`` stayed pinned to a response nobody would ever
|
||||
finish: the client had seen ``response.created`` and waited forever.
|
||||
"""
|
||||
h = await open_harness(auto_response=False)
|
||||
try:
|
||||
response_id = await _commit_with_pending_response(h)
|
||||
|
||||
pending = h.runner.tasks.append_tail
|
||||
assert pending is not None
|
||||
pending.cancel()
|
||||
events = await h.settle()
|
||||
|
||||
done = find(events, "response.done")
|
||||
assert done.response_id == response_id
|
||||
assert done.status == "failed"
|
||||
assert done.response["status_details"]["reason"] == "append_cancelled"
|
||||
assert h.session.active_response_id is None
|
||||
assert h.session.state == DuplexSessionState.OPEN
|
||||
|
||||
# The cancelled append counts as a failed one: the chain behind it is
|
||||
# refused with the documented error, not left hanging.
|
||||
h.port.submit_gate = None
|
||||
await h.run(append_audio())
|
||||
events = await h.run(commands.Commit(create_response=True))
|
||||
assert "response.created" not in types(events)
|
||||
assert find(events, "error").code == "commit_aborted"
|
||||
assert h.session.active_response_id is None
|
||||
finally:
|
||||
await close_harness(h)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_response_cancel_of_a_pending_append_ends_the_response_once_as_cancelled() -> None:
|
||||
"""When the runner cancels the append itself, it owns the response: one terminal, status cancelled.
|
||||
|
||||
The append must not also fail the response on its way out, or the client
|
||||
would get a failed ``response.done`` for a response it cancelled.
|
||||
"""
|
||||
h = await open_harness(auto_response=False)
|
||||
try:
|
||||
response_id = await _commit_with_pending_response(h)
|
||||
|
||||
events = await h.run(commands.CancelResponse())
|
||||
|
||||
terminals = [event for event in events if event.type == "response.done"]
|
||||
assert [event.response_id for event in terminals] == [response_id]
|
||||
assert terminals[0].status == "cancelled"
|
||||
assert h.session.active_response_id is None
|
||||
assert h.session.state == DuplexSessionState.OPEN
|
||||
finally:
|
||||
await close_harness(h)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_closing_the_session_with_a_pending_append_emits_no_failed_response() -> None:
|
||||
"""A close ends the active response itself; the append it cancels stays quiet."""
|
||||
h = await open_harness(auto_response=False)
|
||||
try:
|
||||
await _commit_with_pending_response(h)
|
||||
|
||||
events = await h.run(commands.CloseSession(reason="client_close"))
|
||||
events += await h.settle()
|
||||
|
||||
assert [event.type for event in events if event.type == "response.done"] == []
|
||||
assert "session.closed" in types(events)
|
||||
assert h.session.active_response_id is None
|
||||
finally:
|
||||
await close_harness(h)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_input_clear_racing_a_queued_append_drops_it_as_the_clients_doing() -> None:
|
||||
"""An ``input_audio_buffer.clear`` that lands while an append waits its turn is not a runtime failure.
|
||||
|
||||
The clear deactivates the queued append's reservation; when its turn
|
||||
comes it gives everything back and stops, without an error and without
|
||||
failing the session, and its reason is the client's clear rather than
|
||||
``runtime_append_failed``.
|
||||
"""
|
||||
h = await open_harness()
|
||||
try:
|
||||
h.port.submit_gate = asyncio.Event()
|
||||
h.submit(append_audio())
|
||||
await asyncio.wait_for(h.port.submit_started.wait(), timeout=2.0)
|
||||
h.submit(append_audio())
|
||||
await h.settle(timeout_s=0.3)
|
||||
assert len(h.runner.tasks.append_tasks) == 2, "the second append is queued behind the parked one"
|
||||
queued = h.runner.tasks.append_tail
|
||||
assert queued is not None and not queued.done()
|
||||
|
||||
h.submit(commands.ClearInput())
|
||||
events = await h.settle(timeout_s=0.3)
|
||||
assert types(events) == ["input_audio_buffer.cleared"]
|
||||
|
||||
h.port.submit_gate.set()
|
||||
events = await h.settle()
|
||||
assert queued.done() and queued.result() is False, "the cleared append never ran"
|
||||
assert len(h.port.submissions) == 1, "only the append that was already in flight reached the stage"
|
||||
assert [event.type for event in events if event.type in {"error", "response.done", "session.closed"}] == []
|
||||
assert h.session.pending_input_bytes == 0
|
||||
assert h.session.state == DuplexSessionState.OPEN
|
||||
finally:
|
||||
await close_harness(h)
|
||||
@@ -8,6 +8,7 @@ from __future__ import annotations
|
||||
import asyncio
|
||||
import base64
|
||||
import json
|
||||
from collections.abc import Awaitable, Callable
|
||||
from contextlib import suppress
|
||||
from typing import Any
|
||||
|
||||
@@ -131,7 +132,15 @@ class FakeOmni:
|
||||
self.handles: dict[str, FakeHandle] = {}
|
||||
self.resumed: list[tuple[str, int]] = []
|
||||
self.detached: list[str] = []
|
||||
#: Detaches the engine refused because the caller's lease generation was stale.
|
||||
self.detach_refused: list[tuple[str, int]] = []
|
||||
self.open_error: DuplexSessionError | None = None
|
||||
#: When set, ``resume_session`` parks on it once the engine applied the
|
||||
#: resume: the lease generation is bumped, the result is still in flight.
|
||||
self.resume_gate: asyncio.Event | None = None
|
||||
self.resume_started = asyncio.Event()
|
||||
#: Settlements of resumes whose caller was cancelled mid-RPC (DuplexOmni's contract).
|
||||
self.compensations: list[asyncio.Task[None]] = []
|
||||
|
||||
async def open_session(self, config: Any) -> FakeHandle:
|
||||
self.opened.append(dict(config))
|
||||
@@ -146,13 +155,46 @@ class FakeOmni:
|
||||
def get_session(self, session_id: str) -> FakeHandle | None:
|
||||
return self.handles.get(session_id)
|
||||
|
||||
async def resume_session(self, session_id: str, *, expected_lease_generation: int) -> FakeHandle:
|
||||
async def resume_session(
|
||||
self,
|
||||
session_id: str,
|
||||
*,
|
||||
expected_lease_generation: int,
|
||||
on_abandoned: Callable[[int], Awaitable[None]] | None = None,
|
||||
) -> FakeHandle:
|
||||
self.resumed.append((session_id, expected_lease_generation))
|
||||
handle = self.handles[session_id]
|
||||
handle.lease_generation = expected_lease_generation + 1
|
||||
lease_generation = expected_lease_generation + 1
|
||||
handle.lease_generation = lease_generation
|
||||
if self.resume_gate is not None:
|
||||
self.resume_started.set()
|
||||
gate = self.resume_gate
|
||||
try:
|
||||
await gate.wait()
|
||||
except asyncio.CancelledError:
|
||||
# DuplexOmni's contract: the engine applied the resume; once
|
||||
# its answer arrives the landed generation is settled off the
|
||||
# cancelled task, through the caller's callback if it gave one.
|
||||
async def settle() -> None:
|
||||
await gate.wait()
|
||||
if on_abandoned is not None:
|
||||
await on_abandoned(lease_generation)
|
||||
else:
|
||||
with suppress(DuplexSessionError):
|
||||
await self.detach_session(session_id, expected_lease_generation=lease_generation)
|
||||
|
||||
self.compensations.append(asyncio.create_task(settle()))
|
||||
raise
|
||||
return handle
|
||||
|
||||
async def detach_session(self, session_id: str) -> None:
|
||||
async def detach_session(self, session_id: str, *, expected_lease_generation: int | None = None) -> None:
|
||||
handle = self.handles[session_id]
|
||||
if expected_lease_generation is not None and expected_lease_generation != handle.lease_generation:
|
||||
# The engine's fence: a detach of a lease the caller no longer holds is refused.
|
||||
self.detach_refused.append((session_id, expected_lease_generation))
|
||||
raise DuplexSessionError(
|
||||
"duplex lease generation mismatch", code="session_resume_conflict", session_id=session_id
|
||||
)
|
||||
self.detached.append(session_id)
|
||||
|
||||
|
||||
@@ -607,3 +649,380 @@ async def test_a_resume_that_fails_to_activate_does_not_detach_the_live_attachme
|
||||
pending.cancel()
|
||||
with suppress(asyncio.CancelledError, Exception):
|
||||
await pending
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Cancelled resume and the engine disconnect grace (#7636 Issue 3) #
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_resume_cancelled_during_activation_puts_the_engine_lease_back_into_grace() -> None:
|
||||
"""``_resume`` resumes the engine lease first; a cancel mid-activation must detach it again.
|
||||
|
||||
The rollback used to catch ``Exception`` only. A handler task cancelled
|
||||
while sending ``session.resumed`` or a replay entry left the engine with
|
||||
``detached_at=None`` and the outer handler with no attachment to clean up:
|
||||
the reaper reclaimed nothing after the grace, and with one slot the next
|
||||
open was refused with ``resource_exhausted``.
|
||||
"""
|
||||
omni = FakeOmni()
|
||||
handler = _handler(omni)
|
||||
ws, handle, task = await _open(handler, omni)
|
||||
token = ws.sent[0]["resume_token"]
|
||||
ws.disconnect()
|
||||
await asyncio.wait_for(task, timeout=2.0)
|
||||
assert omni.detached == [handle.session_id]
|
||||
registry = handler._attachment_registry
|
||||
|
||||
activating = asyncio.Event()
|
||||
|
||||
async def parked_resume(*args, **kwargs):
|
||||
activating.set()
|
||||
await asyncio.Event().wait()
|
||||
|
||||
original_resume = registry.resume
|
||||
registry.resume = parked_resume # type: ignore[method-assign]
|
||||
try:
|
||||
ws2, task2 = await _resume(handler, handle.session_id, token)
|
||||
await asyncio.wait_for(activating.wait(), timeout=2.0)
|
||||
assert omni.resumed == [(handle.session_id, 0)], "the engine lease was resumed before activation"
|
||||
|
||||
task2.cancel()
|
||||
with suppress(asyncio.CancelledError):
|
||||
await task2
|
||||
finally:
|
||||
registry.resume = original_resume # type: ignore[method-assign]
|
||||
|
||||
# The lease is detached again, so the disconnect grace runs for it.
|
||||
assert omni.detached == [handle.session_id, handle.session_id]
|
||||
assert not await registry.has_attachment(handle.session_id)
|
||||
assert handle.close_reasons == []
|
||||
# And the session is still resumable with the token the cancelled attempt presented.
|
||||
ws3, task3 = await _resume(handler, handle.session_id, token)
|
||||
resumed = await ws3.wait_for("session.resumed")
|
||||
assert resumed["session_id"] == handle.session_id
|
||||
ws3.disconnect()
|
||||
await asyncio.wait_for(task3, timeout=2.0)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_resume_cancelled_after_activation_detaches_the_attachment_it_made() -> None:
|
||||
"""A cancel after the registry activated the new attachment must undo that attachment too.
|
||||
|
||||
Otherwise the session stays attached to a socket nobody serves, with the
|
||||
engine lease resumed: no grace, no expiry, until the idle TTL.
|
||||
"""
|
||||
omni = FakeOmni()
|
||||
handler = _handler(omni)
|
||||
ws, handle, task = await _open(handler, omni)
|
||||
token = ws.sent[0]["resume_token"]
|
||||
registry = handler._attachment_registry
|
||||
|
||||
# The takeover notifies the replaced socket after activation; park there.
|
||||
notifying = asyncio.Event()
|
||||
|
||||
async def parked_send(payload: dict[str, Any]) -> None:
|
||||
notifying.set()
|
||||
await asyncio.Event().wait()
|
||||
|
||||
ws.send_json = parked_send # type: ignore[method-assign]
|
||||
ws2, task2 = await _resume(handler, handle.session_id, token)
|
||||
await asyncio.wait_for(notifying.wait(), timeout=2.0)
|
||||
assert await registry.is_current_attachment(handle.session_id, 2), "the new socket is attached"
|
||||
|
||||
task2.cancel()
|
||||
with suppress(asyncio.CancelledError):
|
||||
await task2
|
||||
|
||||
assert not await registry.has_attachment(handle.session_id)
|
||||
assert omni.detached == [handle.session_id]
|
||||
assert handle.close_reasons == []
|
||||
|
||||
task.cancel()
|
||||
with suppress(asyncio.CancelledError, Exception):
|
||||
await task
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_an_abandoned_resume_never_detaches_the_lease_a_later_resume_owns() -> None:
|
||||
"""Two connections resuming the same session: the loser's rollback must not touch the winner's lease.
|
||||
|
||||
Interleaving: A has activated and is notifying the socket it replaced; B,
|
||||
holding A's rotated token, has completed its engine resume but its RPC
|
||||
result has not reached the handler yet, so nothing is attached for B.
|
||||
Cancelling A detaches A's registry generation, which used to make the
|
||||
engine detach unconditional: it landed on the lease B had just resumed,
|
||||
and the reaper expired B after the grace even though it heartbeats. The
|
||||
engine now refuses a detach fenced on a generation it has moved past.
|
||||
"""
|
||||
omni = FakeOmni()
|
||||
handler = _handler(omni)
|
||||
ws, handle, task = await _open(handler, omni)
|
||||
token = ws.sent[0]["resume_token"]
|
||||
registry = handler._attachment_registry
|
||||
|
||||
# A takes over and parks while notifying the replaced socket (after activation).
|
||||
notifying = asyncio.Event()
|
||||
|
||||
async def parked_send(payload: dict[str, Any]) -> None:
|
||||
notifying.set()
|
||||
await asyncio.Event().wait()
|
||||
|
||||
ws.send_json = parked_send # type: ignore[method-assign]
|
||||
ws_a, task_a = await _resume(handler, handle.session_id, token)
|
||||
await asyncio.wait_for(notifying.wait(), timeout=2.0)
|
||||
resumed_a = await ws_a.wait_for("session.resumed")
|
||||
assert handle.lease_generation == 1
|
||||
|
||||
# B resumes with A's rotated token: the engine applies it, the result is still in flight.
|
||||
omni.resume_gate = asyncio.Event()
|
||||
ws_b, task_b = await _resume(handler, handle.session_id, resumed_a["resume_token"])
|
||||
await asyncio.wait_for(omni.resume_started.wait(), timeout=2.0)
|
||||
assert handle.lease_generation == 2
|
||||
|
||||
task_a.cancel()
|
||||
with suppress(asyncio.CancelledError):
|
||||
await task_a
|
||||
assert omni.detached == [], "A's rollback must not detach the lease B resumed"
|
||||
assert omni.detach_refused == [(handle.session_id, 1)]
|
||||
assert not await registry.has_attachment(handle.session_id), "A's attachment is gone"
|
||||
|
||||
omni.resume_gate.set()
|
||||
resumed_b = await ws_b.wait_for("session.resumed")
|
||||
assert resumed_b["attachment_generation"] == 3
|
||||
assert await registry.is_current_attachment(handle.session_id, 3)
|
||||
assert omni.detached == []
|
||||
|
||||
# B's own disconnect detaches the lease B holds.
|
||||
ws_b.disconnect()
|
||||
await asyncio.wait_for(task_b, timeout=2.0)
|
||||
assert omni.detached == [handle.session_id]
|
||||
assert omni.detach_refused == [(handle.session_id, 1)]
|
||||
task.cancel()
|
||||
with suppress(asyncio.CancelledError, Exception):
|
||||
await task
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_an_abandoned_takeover_leaves_the_live_connection_serving_and_owning_the_lease() -> None:
|
||||
"""A takeover cancelled mid-RPC must not put the connection it never replaced into disconnect grace.
|
||||
|
||||
A is attached at generation 0. B's engine resume lands generation 1 and B
|
||||
is cancelled before it reaches the registry, so A stays the current
|
||||
attachment. Settling B's generation must hand it to A rather than detach
|
||||
it: A keeps serving, and A's own disconnect later detaches generation 1,
|
||||
which is the lease the engine actually holds.
|
||||
"""
|
||||
omni = FakeOmni()
|
||||
handler = _handler(omni)
|
||||
ws, handle, task = await _open(handler, omni)
|
||||
token = ws.sent[0]["resume_token"]
|
||||
registry = handler._attachment_registry
|
||||
|
||||
omni.resume_gate = asyncio.Event()
|
||||
ws_b, task_b = await _resume(handler, handle.session_id, token)
|
||||
await asyncio.wait_for(omni.resume_started.wait(), timeout=2.0)
|
||||
assert handle.lease_generation == 1
|
||||
|
||||
task_b.cancel()
|
||||
with suppress(asyncio.CancelledError):
|
||||
await task_b
|
||||
omni.resume_gate.set()
|
||||
await asyncio.gather(*omni.compensations)
|
||||
|
||||
assert omni.detached == [], "A is still serving: its lease must not enter disconnect grace"
|
||||
assert omni.detach_refused == []
|
||||
assert await registry.is_current_attachment(handle.session_id, 1), "A is still the attachment"
|
||||
handle.deliver(AudioDelta(session_id=handle.session_id, response_id="r1", delta="aGk="))
|
||||
await ws.wait_for("response.output_audio.delta")
|
||||
|
||||
# A now owns generation 1: its disconnect detaches the lease the engine holds.
|
||||
ws.disconnect()
|
||||
await asyncio.wait_for(task, timeout=2.0)
|
||||
assert omni.detached == [handle.session_id]
|
||||
assert omni.detach_refused == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_send_failure_detaches_the_failed_sockets_own_lease_not_a_pending_resumes() -> None:
|
||||
"""The pump's detach is fenced on the lease of the socket whose send failed, captured with it.
|
||||
|
||||
The pump holds the registry's outbound lock while sending to A. B's
|
||||
engine resume lands generation 1 meanwhile and B waits for that lock to
|
||||
activate. When A's send fails, the pump drops A and must detach A's
|
||||
generation 0, which the engine refuses, not B's generation 1, which would
|
||||
put B into disconnect grace the moment it activates.
|
||||
"""
|
||||
omni = FakeOmni()
|
||||
handler = _handler(omni)
|
||||
ws, handle, task = await _open(handler, omni)
|
||||
token = ws.sent[0]["resume_token"]
|
||||
registry = handler._attachment_registry
|
||||
|
||||
sending = asyncio.Event()
|
||||
fail_send = asyncio.Event()
|
||||
|
||||
async def parked_failing_send(payload: dict[str, Any]) -> None:
|
||||
sending.set()
|
||||
await fail_send.wait()
|
||||
raise RuntimeError("socket closed")
|
||||
|
||||
ws.send_json = parked_failing_send # type: ignore[method-assign]
|
||||
handle.deliver(AudioDelta(session_id=handle.session_id, response_id="r1", delta="aGk="))
|
||||
await asyncio.wait_for(sending.wait(), timeout=2.0)
|
||||
|
||||
# B: the engine resume lands; activation waits for the outbound lock the pump holds.
|
||||
ws_b, task_b = await _resume(handler, handle.session_id, token)
|
||||
await asyncio.sleep(0.05)
|
||||
assert omni.resumed == [(handle.session_id, 0)]
|
||||
assert handle.lease_generation == 1
|
||||
assert await registry.is_current_attachment(handle.session_id, 1), "B has not activated yet"
|
||||
|
||||
fail_send.set()
|
||||
resumed_b = await ws_b.wait_for("session.resumed")
|
||||
assert resumed_b["attachment_generation"] == 2
|
||||
assert omni.detached == [], "A's lease (generation 0) is refused, B's lease is untouched"
|
||||
assert omni.detach_refused == [(handle.session_id, 0)]
|
||||
assert await registry.is_current_attachment(handle.session_id, 2)
|
||||
|
||||
# B serves; its own disconnect detaches the lease it resumed.
|
||||
ws_b.disconnect()
|
||||
await asyncio.wait_for(task_b, timeout=2.0)
|
||||
assert omni.detached == [handle.session_id]
|
||||
ws.disconnect()
|
||||
with suppress(asyncio.CancelledError, Exception):
|
||||
await asyncio.wait_for(task, timeout=2.0)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_an_abandoned_resume_goes_to_the_resume_waiting_to_activate_not_the_socket_it_replaces() -> None:
|
||||
"""Three connections: A sending, B resumed and waiting for the outbound lock, C abandoned mid-RPC.
|
||||
|
||||
C's generation 2 must be parked for B's activation. If it were handed to
|
||||
A instead, A's send failure would detach generation 2 (accepted by the
|
||||
engine), and B would then activate on a lease already in disconnect grace
|
||||
and expire despite heartbeating.
|
||||
"""
|
||||
omni = FakeOmni()
|
||||
handler = _handler(omni)
|
||||
ws, handle, task = await _open(handler, omni)
|
||||
token = ws.sent[0]["resume_token"]
|
||||
registry = handler._attachment_registry
|
||||
|
||||
sending = asyncio.Event()
|
||||
fail_send = asyncio.Event()
|
||||
|
||||
async def parked_failing_send(payload: dict[str, Any]) -> None:
|
||||
sending.set()
|
||||
await fail_send.wait()
|
||||
raise RuntimeError("socket closed")
|
||||
|
||||
ws.send_json = parked_failing_send # type: ignore[method-assign]
|
||||
handle.deliver(AudioDelta(session_id=handle.session_id, response_id="r1", delta="aGk="))
|
||||
await asyncio.wait_for(sending.wait(), timeout=2.0)
|
||||
|
||||
# B: engine resume lands generation 1; activation waits for the lock the pump holds.
|
||||
ws_b, task_b = await _resume(handler, handle.session_id, token)
|
||||
await asyncio.sleep(0.05)
|
||||
assert handle.lease_generation == 1
|
||||
|
||||
# C: same unrotated token, engine resume lands generation 2, cancelled mid-RPC.
|
||||
omni.resume_gate = asyncio.Event()
|
||||
ws_c, task_c = await _resume(handler, handle.session_id, token)
|
||||
await asyncio.wait_for(omni.resume_started.wait(), timeout=2.0)
|
||||
assert handle.lease_generation == 2
|
||||
task_c.cancel()
|
||||
with suppress(asyncio.CancelledError):
|
||||
await task_c
|
||||
omni.resume_gate.set()
|
||||
await asyncio.gather(*omni.compensations)
|
||||
assert omni.detached == [] and omni.detach_refused == [], "generation 2 is parked for B"
|
||||
|
||||
# A's send fails: the pump drops A and detaches A's own generation 0, refused.
|
||||
fail_send.set()
|
||||
resumed_b = await ws_b.wait_for("session.resumed")
|
||||
assert resumed_b["attachment_generation"] == 2
|
||||
assert omni.detach_refused == [(handle.session_id, 0)]
|
||||
assert omni.detached == [], "the lease B serves is untouched"
|
||||
assert await registry.is_current_attachment(handle.session_id, 2)
|
||||
|
||||
# B serves generation 2 (inherited from C): its disconnect detaches that lease.
|
||||
ws_b.disconnect()
|
||||
await asyncio.wait_for(task_b, timeout=2.0)
|
||||
assert omni.detached == [handle.session_id]
|
||||
assert omni.detach_refused == [(handle.session_id, 0)]
|
||||
ws.disconnect()
|
||||
with suppress(asyncio.CancelledError, Exception):
|
||||
await asyncio.wait_for(task, timeout=2.0)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_resume_cancelled_during_replay_settles_the_newer_lease_it_was_handed() -> None:
|
||||
"""Activation rollback must give back the lease the provisional attachment actually owned.
|
||||
|
||||
A resumed generation 1, delivered ``session.resumed`` and is blocked
|
||||
replaying events. B, with A's rotated token, resumed generation 2 and was
|
||||
cancelled mid-RPC; its settlement handed 2 to A. When A is then cancelled
|
||||
during replay, the registry rolls A back; detaching A's own generation 1
|
||||
would be refused and generation 2 would never enter disconnect grace.
|
||||
"""
|
||||
omni = FakeOmni()
|
||||
handler = _handler(omni)
|
||||
ws, handle, task = await _open(handler, omni)
|
||||
token = ws.sent[0]["resume_token"]
|
||||
registry = handler._attachment_registry
|
||||
ws.disconnect()
|
||||
await asyncio.wait_for(task, timeout=2.0)
|
||||
assert omni.detached == [handle.session_id]
|
||||
# Journaled while detached: A's activation has something to replay.
|
||||
handle.deliver(AudioDelta(session_id=handle.session_id, response_id="r1", delta="one"))
|
||||
await asyncio.sleep(0.05)
|
||||
|
||||
# A: session.resumed goes out, the replay entry parks.
|
||||
ws_a = FakeWebSocket({"duplex": "1", "resume": "1"})
|
||||
replaying = asyncio.Event()
|
||||
sends = 0
|
||||
original_send = ws_a.send_json
|
||||
|
||||
async def send_parking_on_replay(payload: dict[str, Any]) -> None:
|
||||
nonlocal sends
|
||||
sends += 1
|
||||
if sends == 2:
|
||||
replaying.set()
|
||||
await asyncio.Event().wait()
|
||||
await original_send(payload)
|
||||
|
||||
ws_a.send_json = send_parking_on_replay # type: ignore[method-assign]
|
||||
task_a = asyncio.create_task(handler.handle_realtime_session(ws_a))
|
||||
ws_a.feed(
|
||||
{
|
||||
"type": "session.resume",
|
||||
"session_id": handle.session_id,
|
||||
"resume_token": token,
|
||||
"last_received_server_event_seq": 0,
|
||||
}
|
||||
)
|
||||
await asyncio.wait_for(replaying.wait(), timeout=2.0)
|
||||
resumed_a = await ws_a.wait_for("session.resumed")
|
||||
assert handle.lease_generation == 1
|
||||
|
||||
# B: A's rotated token, engine resume lands generation 2, cancelled mid-RPC.
|
||||
omni.resume_gate = asyncio.Event()
|
||||
ws_b, task_b = await _resume(handler, handle.session_id, resumed_a["resume_token"])
|
||||
await asyncio.wait_for(omni.resume_started.wait(), timeout=2.0)
|
||||
assert handle.lease_generation == 2
|
||||
task_b.cancel()
|
||||
with suppress(asyncio.CancelledError):
|
||||
await task_b
|
||||
omni.resume_gate.set()
|
||||
await asyncio.gather(*omni.compensations)
|
||||
assert omni.detached == [handle.session_id], "generation 2 went to A's pending activation"
|
||||
|
||||
# A is cancelled during replay: its rollback settles generation 2, the lease A owned.
|
||||
task_a.cancel()
|
||||
with suppress(asyncio.CancelledError):
|
||||
await task_a
|
||||
assert not await registry.has_attachment(handle.session_id)
|
||||
assert omni.detached == [handle.session_id, handle.session_id]
|
||||
assert omni.detach_refused == [], "generation 1 was never detached: the engine would refuse it"
|
||||
@@ -552,3 +552,90 @@ async def test_registry_resume_cancelled_mid_delivery_rolls_back_like_a_failure(
|
||||
close=close,
|
||||
)
|
||||
assert recovered.attachment_generation == 3
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_release_hands_back_the_lease_generation_the_dropped_connection_was_serving(mocker) -> None:
|
||||
"""The lease generation travels with the attachment, so a detach is fenced on the right lease."""
|
||||
registry = DuplexSessionAttachmentRegistry(replay_ttl_s=60.0, replay_max_bytes_per_session=4096)
|
||||
send = mocker.AsyncMock()
|
||||
created = await registry.create("sid-lease", send=send, close=mocker.AsyncMock(), lease_generation=0)
|
||||
|
||||
# An abandoned takeover hands its generation to the connection still attached.
|
||||
assert await registry.settle_lease_generation("sid-lease", 1) is None
|
||||
assert await registry.settle_lease_generation("sid-lease", 0) is None, "generations only move forward"
|
||||
|
||||
released = await registry.release_attachment("sid-lease", attachment_generation=created.attachment_generation)
|
||||
assert released is not None
|
||||
assert (released.attachment_generation, released.lease_generation) == (1, 1)
|
||||
assert await registry.release_attachment("sid-lease", attachment_generation=None) is None
|
||||
assert await registry.settle_lease_generation("sid-lease", 2) == 2, "nobody is attached: the caller detaches it"
|
||||
|
||||
resumed = await registry.resume(
|
||||
"sid-lease",
|
||||
resume_token=created.resume_token.plaintext,
|
||||
last_received_server_event_seq=0,
|
||||
send=send,
|
||||
close=mocker.AsyncMock(),
|
||||
lease_generation=3,
|
||||
)
|
||||
released = await registry.release_attachment("sid-lease", attachment_generation=resumed.attachment_generation)
|
||||
assert released is not None and released.lease_generation == 3
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_an_orphaned_generation_waits_for_the_resume_that_is_about_to_activate(mocker) -> None:
|
||||
"""While a resume is pending, an orphan goes to its activation, not to the socket it will replace."""
|
||||
registry = DuplexSessionAttachmentRegistry(replay_ttl_s=60.0, replay_max_bytes_per_session=4096)
|
||||
send = mocker.AsyncMock()
|
||||
created = await registry.create("sid-pending", send=send, close=mocker.AsyncMock(), lease_generation=0)
|
||||
|
||||
await registry.begin_resume("sid-pending")
|
||||
assert await registry.settle_lease_generation("sid-pending", 2) is None, "parked for the pending activation"
|
||||
released = await registry.release_attachment("sid-pending", attachment_generation=created.attachment_generation)
|
||||
assert released is not None and released.lease_generation == 0, "the replaced socket keeps its own lease"
|
||||
|
||||
resumed = await registry.resume(
|
||||
"sid-pending",
|
||||
resume_token=created.resume_token.plaintext,
|
||||
last_received_server_event_seq=0,
|
||||
send=send,
|
||||
close=mocker.AsyncMock(),
|
||||
lease_generation=1,
|
||||
)
|
||||
assert await registry.end_resume("sid-pending") is None, "the activation absorbed the orphan"
|
||||
released = await registry.release_attachment("sid-pending", attachment_generation=resumed.attachment_generation)
|
||||
assert released is not None and released.lease_generation == 2
|
||||
|
||||
# A pending resume that never activates leaves the orphan to whoever ends the claim.
|
||||
await registry.begin_resume("sid-pending")
|
||||
assert await registry.settle_lease_generation("sid-pending", 3) is None
|
||||
assert await registry.end_resume("sid-pending") == 3
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_rolled_back_activation_orphans_the_lease_it_was_handed(mocker) -> None:
|
||||
"""A provisional attachment may own a newer generation than its own resume produced."""
|
||||
registry = DuplexSessionAttachmentRegistry(replay_ttl_s=60.0, replay_max_bytes_per_session=4096)
|
||||
created = await registry.create("sid-rollback", send=mocker.AsyncMock(), close=mocker.AsyncMock())
|
||||
await registry.detach("sid-rollback", attachment_generation=created.attachment_generation)
|
||||
|
||||
async def failing_activation(payload):
|
||||
raise RuntimeError("socket closed during replay")
|
||||
|
||||
await registry.begin_resume("sid-rollback")
|
||||
with pytest.raises(RuntimeError):
|
||||
await registry.resume(
|
||||
"sid-rollback",
|
||||
resume_token=created.resume_token.plaintext,
|
||||
last_received_server_event_seq=0,
|
||||
send=failing_activation,
|
||||
close=mocker.AsyncMock(),
|
||||
activation_payload_factory=lambda token, generation: {"type": "session.resumed"},
|
||||
lease_generation=1,
|
||||
)
|
||||
# Meanwhile a resume abandoned mid-RPC handed generation 2 to the provisional attachment.
|
||||
# (Order does not matter: both land in the orphan slot while the claim is open.)
|
||||
assert await registry.settle_lease_generation("sid-rollback", 2) is None
|
||||
assert await registry.settle_lease_generation("sid-rollback", 1) is None
|
||||
assert await registry.end_resume("sid-rollback") == 2, "the newest lease is the one to detach"
|
||||
@@ -43,7 +43,20 @@ class FakeDuplexEngine(FakeAsyncOmniEngine):
|
||||
self.opened: list[tuple[str, DuplexSessionConfig]] = []
|
||||
self.closed: list[tuple[str, str]] = []
|
||||
self.resumed: list[tuple[str, int]] = []
|
||||
#: The control id of every resume RPC, in order (a replay repeats one).
|
||||
self.resume_control_ids: list[str | None] = []
|
||||
#: Per session: the control id that produced the current lease generation, and that generation.
|
||||
self.lease: dict[str, tuple[str | None, int]] = {}
|
||||
#: How many resume RPCs, after applying the resume, answer with a
|
||||
#: timeout: the engine landed it but the waiter never saw the result.
|
||||
self.resume_timeouts_remaining = 0
|
||||
self.touched: list[tuple[str, str]] = []
|
||||
#: The lease-generation fence of every detach, in order (``None`` = unfenced).
|
||||
self.detach_generations: list[int | None] = []
|
||||
#: When set, ``resume_session_async`` parks on it after signalling ``resume_started``:
|
||||
#: the engine has applied the resume, its result has not reached the caller.
|
||||
self.resume_gate: asyncio.Event | None = None
|
||||
self.resume_started = asyncio.Event()
|
||||
self.commands: list[tuple[str, commands.DuplexCommand]] = []
|
||||
self.open_error: DuplexSessionError | None = None
|
||||
self.close_error: DuplexSessionError | None = None
|
||||
@@ -71,12 +84,33 @@ class FakeDuplexEngine(FakeAsyncOmniEngine):
|
||||
self.emit(session_id, SessionClosed(reason=reason))
|
||||
return self._result("close", session_id)
|
||||
|
||||
async def resume_session_async(self, session_id, *, expected_lease_generation, timeout=None):
|
||||
async def resume_session_async(self, session_id, *, expected_lease_generation, control_id=None, timeout=None):
|
||||
self.resumed.append((session_id, expected_lease_generation))
|
||||
return self._result("resume", session_id, lease_generation=expected_lease_generation + 1)
|
||||
self.resume_control_ids.append(control_id)
|
||||
resumed_by, generation = self.lease.get(session_id, (None, 0))
|
||||
if control_id is not None and control_id == resumed_by:
|
||||
# The engine's replay answer: the generation this resume produced.
|
||||
# Its answer can be lost to a timeout exactly like the first one.
|
||||
if self.resume_timeouts_remaining > 0:
|
||||
self.resume_timeouts_remaining -= 1
|
||||
raise DuplexSessionError("duplex resume timed out", code="timeout", retryable=True)
|
||||
return self._result("resume", session_id, lease_generation=generation)
|
||||
if expected_lease_generation != generation:
|
||||
raise DuplexSessionError("duplex lease generation mismatch", code="session_resume_conflict")
|
||||
generation += 1
|
||||
self.lease[session_id] = (control_id, generation)
|
||||
if self.resume_gate is not None:
|
||||
self.resume_started.set()
|
||||
await self.resume_gate.wait()
|
||||
if self.resume_timeouts_remaining > 0:
|
||||
self.resume_timeouts_remaining -= 1
|
||||
raise DuplexSessionError("duplex resume timed out", code="timeout", retryable=True)
|
||||
return self._result("resume", session_id, lease_generation=generation)
|
||||
|
||||
async def touch_session_async(self, session_id, *, activity, timeout=None):
|
||||
async def touch_session_async(self, session_id, *, activity, expected_lease_generation=None, timeout=None):
|
||||
self.touched.append((session_id, activity))
|
||||
if activity == "detach":
|
||||
self.detach_generations.append(expected_lease_generation)
|
||||
return self._result("touch", session_id)
|
||||
|
||||
async def submit_command_async(self, session_id, command):
|
||||
@@ -324,3 +358,141 @@ async def test_shutdown_closes_every_handle(monkeypatch) -> None:
|
||||
assert first.close_reason == "shutdown"
|
||||
assert omni.sessions == {}
|
||||
assert engine.shutdown_called
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_detach_session_forwards_the_lease_generation_fence(monkeypatch) -> None:
|
||||
omni, engine = _make_omni(monkeypatch)
|
||||
try:
|
||||
handle = await omni.open_session()
|
||||
await omni.detach_session(handle.session_id, expected_lease_generation=3)
|
||||
await omni.detach_session(handle.session_id)
|
||||
assert engine.detach_generations == [3, None]
|
||||
finally:
|
||||
omni.shutdown()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_resume_cancelled_mid_rpc_is_adopted_and_detached_once_it_lands(monkeypatch) -> None:
|
||||
"""The resume RPC runs in an executor: cancelling its waiter does not stop the engine.
|
||||
|
||||
Without compensation the engine ends up with the lease resumed (no
|
||||
disconnect grace) for a connection that is gone, and the handle keeps the
|
||||
old generation, so a later reconnect resumes against a stale lease. The
|
||||
outcome has to be observed off the cancelled task: adopt the generation
|
||||
the resume produced and detach exactly that lease.
|
||||
"""
|
||||
omni, engine = _make_omni(monkeypatch)
|
||||
try:
|
||||
handle = await omni.open_session()
|
||||
engine.resume_gate = asyncio.Event()
|
||||
resume = asyncio.create_task(omni.resume_session(handle.session_id, expected_lease_generation=0))
|
||||
await asyncio.wait_for(engine.resume_started.wait(), timeout=2.0)
|
||||
|
||||
resume.cancel()
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await resume
|
||||
# Nothing to give back yet: the engine has not answered.
|
||||
assert handle.lease_generation == 0
|
||||
assert engine.detach_generations == []
|
||||
|
||||
engine.resume_gate.set()
|
||||
await asyncio.gather(*list(omni._resume_compensations))
|
||||
assert handle.lease_generation == 1, "the generation the resume produced is adopted"
|
||||
assert engine.touched == [(handle.session_id, "detach")]
|
||||
assert engine.detach_generations == [1], "the detach is fenced on that generation"
|
||||
finally:
|
||||
omni.shutdown()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_resume_cancelled_then_timed_out_is_settled_by_replaying_it(monkeypatch) -> None:
|
||||
"""A timeout after the cancel leaves the outcome unknown: the resume is replayed under its id.
|
||||
|
||||
The engine may have applied the resume after the waiter gave up, and the
|
||||
RPC router drops the late answer. Replaying the same control id makes the
|
||||
engine report the generation that resume produced (or land it now), so
|
||||
the handle adopts the real generation and the lease is detached again.
|
||||
"""
|
||||
omni, engine = _make_omni(monkeypatch)
|
||||
try:
|
||||
handle = await omni.open_session()
|
||||
engine.resume_gate = asyncio.Event()
|
||||
engine.resume_timeouts_remaining = 1
|
||||
resume = asyncio.create_task(omni.resume_session(handle.session_id, expected_lease_generation=0))
|
||||
await asyncio.wait_for(engine.resume_started.wait(), timeout=2.0)
|
||||
|
||||
resume.cancel()
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await resume
|
||||
engine.resume_gate.set()
|
||||
await asyncio.gather(*list(omni._resume_compensations))
|
||||
|
||||
assert len(engine.resume_control_ids) == 2, "the resume was replayed once"
|
||||
assert engine.resume_control_ids[0] == engine.resume_control_ids[1] is not None
|
||||
assert engine.lease[handle.session_id][1] == 1, "the replay did not resume a second time"
|
||||
assert handle.lease_generation == 1
|
||||
assert engine.detach_generations == [1]
|
||||
finally:
|
||||
omni.shutdown()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_an_abandoned_resume_is_settled_through_the_callers_callback(monkeypatch) -> None:
|
||||
"""A caller that owns attachments decides what the landed generation means; DuplexOmni does not detach."""
|
||||
omni, engine = _make_omni(monkeypatch)
|
||||
try:
|
||||
handle = await omni.open_session()
|
||||
settled: list[int] = []
|
||||
|
||||
async def on_abandoned(lease_generation: int) -> None:
|
||||
settled.append(lease_generation)
|
||||
|
||||
engine.resume_gate = asyncio.Event()
|
||||
resume = asyncio.create_task(
|
||||
omni.resume_session(handle.session_id, expected_lease_generation=0, on_abandoned=on_abandoned)
|
||||
)
|
||||
await asyncio.wait_for(engine.resume_started.wait(), timeout=2.0)
|
||||
resume.cancel()
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await resume
|
||||
engine.resume_gate.set()
|
||||
await asyncio.gather(*list(omni._resume_compensations))
|
||||
|
||||
assert settled == [1]
|
||||
assert handle.lease_generation == 1
|
||||
assert engine.detach_generations == [], "the callback owns the decision"
|
||||
finally:
|
||||
omni.shutdown()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_an_abandoned_resume_keeps_replaying_across_repeated_timeouts(monkeypatch) -> None:
|
||||
"""A replay can time out too; giving up then would reopen the lost-answer window.
|
||||
|
||||
The engine applied the resume, the abandoned RPC and the first replay both
|
||||
lose their answers to timeouts, the second replay is answered: the handle
|
||||
adopts the generation and the lease is settled, with the lease bumped once.
|
||||
"""
|
||||
omni, engine = _make_omni(monkeypatch)
|
||||
try:
|
||||
omni._resume_replay_backoff_s = (0.01, 0.02)
|
||||
handle = await omni.open_session()
|
||||
engine.resume_gate = asyncio.Event()
|
||||
engine.resume_timeouts_remaining = 2
|
||||
resume = asyncio.create_task(omni.resume_session(handle.session_id, expected_lease_generation=0))
|
||||
await asyncio.wait_for(engine.resume_started.wait(), timeout=2.0)
|
||||
|
||||
resume.cancel()
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await resume
|
||||
engine.resume_gate.set()
|
||||
await asyncio.wait_for(asyncio.gather(*list(omni._resume_compensations)), timeout=5.0)
|
||||
|
||||
assert len(engine.resume_control_ids) == 3, "the abandoned RPC, a replay that timed out, the replay answered"
|
||||
assert len(set(engine.resume_control_ids)) == 1
|
||||
assert engine.lease[handle.session_id][1] == 1, "the lease was bumped once"
|
||||
assert handle.lease_generation == 1
|
||||
assert engine.detach_generations == [1]
|
||||
finally:
|
||||
omni.shutdown()
|
||||
Reference in new issue
Block a user