[Feature] JoyAI-VL-Interaction streaming interaction serving layer (#4575)
Signed-off-by: lishunyang12 <125541396+lishunyang12@users.noreply.github.com> Signed-off-by: Yueqian Lin <linyueqian@outlook.com> Signed-off-by: lishunyang <lishunyang12@163.com> Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com> Co-authored-by: Yueqian Lin <linyueqian@outlook.com>
This commit is contained in:
45 files changed
+3699
-1
No files matched your search
@@ -0,0 +1,32 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
|
||||
from stream_client import stream
|
||||
|
||||
ICON = {"response": "SPEAK ", "silence": "silence ", "delegate": "DELEGATE"}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("video")
|
||||
parser.add_argument("--server", default="http://127.0.0.1:8070")
|
||||
parser.add_argument("--query", default=None)
|
||||
parser.add_argument("--fps", type=float, default=1.0)
|
||||
parser.add_argument("--session-id", default="cli")
|
||||
args = parser.parse_args()
|
||||
|
||||
for tick in stream(args.video, args.server, args.session_id, args.query, args.fps):
|
||||
if tick.action == "silence":
|
||||
continue
|
||||
line = f"[{tick.t:6.1f}s] {ICON.get(tick.action)} {tick.text}"
|
||||
if tick.delegation:
|
||||
line += f" (-> {tick.delegation.get('question', '')})"
|
||||
print(line, flush=True)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,79 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import time
|
||||
from collections.abc import Iterator
|
||||
from dataclasses import dataclass
|
||||
|
||||
import cv2
|
||||
import requests
|
||||
|
||||
|
||||
@dataclass
|
||||
class Tick:
|
||||
index: int
|
||||
t: float
|
||||
action: str
|
||||
text: str
|
||||
skipped: bool
|
||||
latency_ms: float
|
||||
delegation: dict | None
|
||||
|
||||
|
||||
def iter_frames(video_path: str, fps: float) -> Iterator[tuple[float, bytes]]:
|
||||
cap = cv2.VideoCapture(video_path)
|
||||
src_fps = cap.get(cv2.CAP_PROP_FPS) or 30.0
|
||||
step = max(1, round(src_fps / fps))
|
||||
idx = 0
|
||||
while True:
|
||||
ok, frame = cap.read()
|
||||
if not ok:
|
||||
break
|
||||
if idx % step == 0:
|
||||
ok, buf = cv2.imencode(".jpg", frame)
|
||||
if ok:
|
||||
yield idx / src_fps, buf.tobytes()
|
||||
idx += 1
|
||||
cap.release()
|
||||
|
||||
|
||||
def stream(
|
||||
video_path: str,
|
||||
server: str,
|
||||
session_id: str,
|
||||
query: str | None = None,
|
||||
fps: float = 1.0,
|
||||
realtime: bool = False,
|
||||
) -> Iterator[Tick]:
|
||||
"""Send each sampled frame as one chat completion; yield the parsed action.
|
||||
|
||||
``query`` is attached to the first tick to arm the session; inject later
|
||||
queries by posting another frame turn with a new text part.
|
||||
"""
|
||||
url = server.rstrip("/") + "/v1/chat/completions"
|
||||
requests.post(server.rstrip("/") + "/reset", json={"session_id": session_id}, timeout=10)
|
||||
for i, (t, jpeg) in enumerate(iter_frames(video_path, fps)):
|
||||
data_url = "data:image/jpeg;base64," + base64.b64encode(jpeg).decode()
|
||||
content = [{"type": "image_url", "image_url": {"url": data_url}}]
|
||||
if i == 0 and query:
|
||||
content.insert(0, {"type": "text", "text": query})
|
||||
resp = requests.post(
|
||||
url,
|
||||
json={"session_id": session_id, "messages": [{"role": "user", "content": content}]},
|
||||
timeout=120,
|
||||
).json()
|
||||
info = resp.get("interaction", {})
|
||||
yield Tick(
|
||||
index=info.get("frame_index", i + 1),
|
||||
t=t,
|
||||
action=info.get("action", "?"),
|
||||
text=resp["choices"][0]["message"]["content"],
|
||||
skipped=info.get("inference_skipped", False),
|
||||
latency_ms=info.get("latency_ms", 0.0),
|
||||
delegation=info.get("delegation"),
|
||||
)
|
||||
if realtime:
|
||||
time.sleep(1.0 / fps)
|
||||
Reference in new issue
Block a user