e241005a71
Signed-off-by: lishunyang12 <125541396+lishunyang12@users.noreply.github.com> Signed-off-by: Yueqian Lin <linyueqian@outlook.com> Signed-off-by: lishunyang <lishunyang12@163.com> Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com> Co-authored-by: Yueqian Lin <linyueqian@outlook.com>
73 lines
2.8 KiB
Python
73 lines
2.8 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
|
|
from __future__ import annotations
|
|
|
|
from collections.abc import AsyncIterator, Awaitable, Callable
|
|
from typing import Any
|
|
|
|
from vllm_omni.experimental.fullduplex.core.adapter import DuplexAdapter, DuplexCapability, OutputChunk
|
|
from vllm_omni.experimental.fullduplex.core.session import DuplexSession
|
|
from vllm_omni.experimental.fullduplex.joyvl.bridges.delegation import DelegationBridge, StubDelegationBridge
|
|
from vllm_omni.experimental.fullduplex.joyvl.decision.output_parser import Action
|
|
from vllm_omni.experimental.fullduplex.joyvl.decision.policy import JoyVLPolicy, sample_frames
|
|
|
|
GenerateFn = Callable[[list[dict[str, Any]]], Awaitable[str]]
|
|
|
|
|
|
class JoyVLDuplexAdapter(DuplexAdapter):
|
|
def __init__(
|
|
self,
|
|
generate: GenerateFn,
|
|
*,
|
|
persona: str = "default",
|
|
num_frames: int = 4,
|
|
chunk_frames: int = 200,
|
|
frame_seconds: float = 1.0,
|
|
delegation: DelegationBridge | None = None,
|
|
) -> None:
|
|
self._generate = generate
|
|
self._policy = JoyVLPolicy(
|
|
persona=persona,
|
|
num_frames=num_frames,
|
|
chunk_frames=chunk_frames,
|
|
frame_seconds=frame_seconds,
|
|
delegation=delegation or StubDelegationBridge(),
|
|
)
|
|
self._frames: list[str] = []
|
|
self._pending_query: str | None = None
|
|
|
|
def capabilities(self) -> DuplexCapability:
|
|
return DuplexCapability(frozenset({"video", "text"}), frozenset({"text"}), proactive=True)
|
|
|
|
async def on_input(self, session: DuplexSession, modality: str, data: Any) -> None:
|
|
if modality == "video":
|
|
self._policy.tick()
|
|
self._frames.append(data)
|
|
elif modality == "text":
|
|
self._pending_query = data
|
|
|
|
def should_respond(self, session: DuplexSession) -> bool:
|
|
return bool(self._frames)
|
|
|
|
async def respond(self, session: DuplexSession) -> AsyncIterator[OutputChunk]:
|
|
policy = self._policy
|
|
await policy.fold_delegations()
|
|
policy.set_query(self._pending_query)
|
|
self._pending_query = None
|
|
|
|
parts = [{"type": "image_url", "image_url": {"url": f}} for f in sample_frames(self._frames, policy.num_frames)]
|
|
messages, _ = policy.build_messages(parts)
|
|
action = policy.commit(await self._generate(messages))
|
|
|
|
frames = [(str(i), f) for i, f in enumerate(self._frames)]
|
|
await policy.submit_if_delegate(action, frames)
|
|
if action.action is not Action.SILENCE and action.text:
|
|
yield OutputChunk("text", action.text)
|
|
if policy.needs_flush():
|
|
await policy.flush(frames)
|
|
self._frames.clear()
|
|
|
|
async def on_barge_in(self, session: DuplexSession) -> None:
|
|
self._pending_query = None
|