[Core][Frontend] Unified Full-duplex Framework (#7413)
Signed-off-by: chickeyton <ngton2014@gmail.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
9005d78903
commit
99ff4f3070
182 files changed
+26951
-34743
No files matched your search
@@ -0,0 +1,161 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
"""Shared state for :class:`DuplexSessionRunner` and its components.
|
||||
|
||||
The runner is a state machine that several components read and write. Splitting
|
||||
it into modules only helps if the shared part stops being implicit: a component
|
||||
holding a back-reference to the runner is a mixin with extra steps, and one
|
||||
holding its own copy of a flag diverges from the others.
|
||||
|
||||
So the mutable part is named. ``DuplexRunState`` is the small set of flags that
|
||||
more than one component touches -- ``grep "\\.run\\."`` finds every mutation --
|
||||
and ``DuplexSessionContext`` is the read-mostly collaborator bundle everything
|
||||
is constructed with. Anything a component needs from the runner that is neither
|
||||
of those is infrastructure, and goes through :class:`RunnerServices`.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from collections.abc import Awaitable, Callable
|
||||
from dataclasses import dataclass, field
|
||||
from typing import TYPE_CHECKING, Protocol, TypeVar
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.outputs import RequestOutput
|
||||
|
||||
from vllm_omni.engine.duplex.contracts import (
|
||||
DuplexOutputContext,
|
||||
DuplexOutputDecision,
|
||||
DuplexStagePort,
|
||||
)
|
||||
from vllm_omni.engine.duplex.plugin import DuplexModelPlugin, DuplexModelSessionState
|
||||
from vllm_omni.engine.duplex.session.engine_session import DuplexEngineSession
|
||||
from vllm_omni.engine.duplex.session.manager import DuplexSessionManager
|
||||
from vllm_omni.metrics.stats import StageRequestStats
|
||||
|
||||
_OffloadT = TypeVar("_OffloadT")
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class DuplexRunState:
|
||||
"""Mutable per-run flags shared by the runner and its components.
|
||||
|
||||
Every field here is written by one component and read by another; that is
|
||||
the criterion for being in this object rather than private to a component.
|
||||
"""
|
||||
|
||||
#: An irreversible close has begun: commands and control ops are refused.
|
||||
closing: bool = False
|
||||
#: ``session.closed`` / ``session.expired`` already left the runner.
|
||||
closed_emitted: bool = False
|
||||
#: The terminal event is deferred to the manager, which emits it after the
|
||||
#: stage cleanup so it also means "the admission slot is free again".
|
||||
closed_deferred: bool = False
|
||||
close_reason: str | None = None
|
||||
#: The model runtime reported a close; further data-plane work is pointless.
|
||||
runtime_closed: bool = False
|
||||
#: Request id of the resumable data-plane stream currently bound to the session.
|
||||
stream_request_id: str | None = None
|
||||
|
||||
|
||||
class RunnerServices(Protocol):
|
||||
"""The only things a component may ask the runner for.
|
||||
|
||||
Deliberately two methods. A component that needs more than task scheduling
|
||||
from the runner is reaching for orchestration that belongs in the runner.
|
||||
"""
|
||||
|
||||
def spawn(self, coro: Awaitable[None], *, name: str) -> None:
|
||||
"""Run ``coro`` as a tracked background task on the session's loop."""
|
||||
...
|
||||
|
||||
async def offload(self, fn: Callable[..., _OffloadT], *args: object, **kwargs: object) -> _OffloadT:
|
||||
"""Run a blocking call off the orchestrator loop."""
|
||||
...
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class DuplexSessionContext:
|
||||
"""Collaborators one session's components are built with.
|
||||
|
||||
Read-mostly: the session and the model state are mutated through their own
|
||||
APIs, not by rebinding these fields. The mutable flags live in ``run``.
|
||||
"""
|
||||
|
||||
session: DuplexEngineSession
|
||||
model_state: DuplexModelSessionState
|
||||
plugin: DuplexModelPlugin
|
||||
stage_port: DuplexStagePort
|
||||
manager: DuplexSessionManager
|
||||
tasks: DuplexSessionTasks
|
||||
run: DuplexRunState
|
||||
services: RunnerServices
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Task bookkeeping #
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class DuplexAppendTaskMeta:
|
||||
epoch: int
|
||||
final: bool
|
||||
response_bound: bool
|
||||
|
||||
|
||||
@dataclass
|
||||
class DuplexSessionTasks:
|
||||
"""Tracked task handles of one session (append tail, active response, pending silence)."""
|
||||
|
||||
append_tasks: dict[asyncio.Task[bool], DuplexAppendTaskMeta] = field(default_factory=dict)
|
||||
append_tail: asyncio.Task[bool] | None = None
|
||||
active_response_task: asyncio.Task[None] | None = None
|
||||
|
||||
def track_append_task(
|
||||
self,
|
||||
task: asyncio.Task[bool],
|
||||
*,
|
||||
epoch: int,
|
||||
final: bool,
|
||||
response_bound: bool,
|
||||
) -> None:
|
||||
self.append_tasks[task] = DuplexAppendTaskMeta(epoch, final, response_bound)
|
||||
task.add_done_callback(self.append_tasks.pop)
|
||||
|
||||
def has_response_bound_append_tasks(self) -> bool:
|
||||
return any(meta.response_bound for meta in self.append_tasks.values())
|
||||
|
||||
async def cancel_append_tasks(self, timeout_s: float = 0.25, *, response_bound_only: bool = False) -> bool:
|
||||
tasks = [task for task, meta in self.append_tasks.items() if not response_bound_only or meta.response_bound]
|
||||
if not tasks:
|
||||
return False
|
||||
cancelled_tail = self.append_tail if self.append_tail in tasks else None
|
||||
for task in tasks:
|
||||
task.cancel()
|
||||
try:
|
||||
await asyncio.wait_for(asyncio.gather(*tasks, return_exceptions=True), timeout=timeout_s)
|
||||
except TimeoutError:
|
||||
pass
|
||||
if cancelled_tail is not None and self.append_tail is cancelled_tail:
|
||||
self.append_tail = None
|
||||
return True
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Stage output (the mailbox item the orchestrator hands the runner) #
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class StageOutput:
|
||||
"""One stage result, queued for the session that owns the request."""
|
||||
|
||||
stage_id: int
|
||||
output: RequestOutput
|
||||
metrics: StageRequestStats | None
|
||||
request_id: str
|
||||
context: DuplexOutputContext
|
||||
decision: DuplexOutputDecision | None
|
||||
Reference in new issue
Block a user