[Feature] JoyAI-VL-Interaction streaming interaction serving layer (#4575)

Signed-off-by: lishunyang12 <125541396+lishunyang12@users.noreply.github.com>
Signed-off-by: Yueqian Lin <linyueqian@outlook.com>
Signed-off-by: lishunyang <lishunyang12@163.com>
Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
Co-authored-by: Yueqian Lin <linyueqian@outlook.com>
This commit is contained in:
authored and GitHub committed 2026-06-21 07:42:51 +00:00
1 parent 950e66a44c
commit e241005a71
45 files changed
+3699 -1

No files matched your search

@@ -0,0 +1,32 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from __future__ import annotations
import argparse
from stream_client import stream
ICON = {"response": "SPEAK ", "silence": "silence ", "delegate": "DELEGATE"}
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("video")
parser.add_argument("--server", default="http://127.0.0.1:8070")
parser.add_argument("--query", default=None)
parser.add_argument("--fps", type=float, default=1.0)
parser.add_argument("--session-id", default="cli")
args = parser.parse_args()
for tick in stream(args.video, args.server, args.session_id, args.query, args.fps):
if tick.action == "silence":
continue
line = f"[{tick.t:6.1f}s] {ICON.get(tick.action)} {tick.text}"
if tick.delegation:
line += f" (-> {tick.delegation.get('question', '')})"
print(line, flush=True)
if __name__ == "__main__":
main()
@@ -0,0 +1,79 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from __future__ import annotations
import base64
import time
from collections.abc import Iterator
from dataclasses import dataclass
import cv2
import requests
@dataclass
class Tick:
index: int
t: float
action: str
text: str
skipped: bool
latency_ms: float
delegation: dict | None
def iter_frames(video_path: str, fps: float) -> Iterator[tuple[float, bytes]]:
cap = cv2.VideoCapture(video_path)
src_fps = cap.get(cv2.CAP_PROP_FPS) or 30.0
step = max(1, round(src_fps / fps))
idx = 0
while True:
ok, frame = cap.read()
if not ok:
break
if idx % step == 0:
ok, buf = cv2.imencode(".jpg", frame)
if ok:
yield idx / src_fps, buf.tobytes()
idx += 1
cap.release()
def stream(
video_path: str,
server: str,
session_id: str,
query: str | None = None,
fps: float = 1.0,
realtime: bool = False,
) -> Iterator[Tick]:
"""Send each sampled frame as one chat completion; yield the parsed action.
``query`` is attached to the first tick to arm the session; inject later
queries by posting another frame turn with a new text part.
"""
url = server.rstrip("/") + "/v1/chat/completions"
requests.post(server.rstrip("/") + "/reset", json={"session_id": session_id}, timeout=10)
for i, (t, jpeg) in enumerate(iter_frames(video_path, fps)):
data_url = "data:image/jpeg;base64," + base64.b64encode(jpeg).decode()
content = [{"type": "image_url", "image_url": {"url": data_url}}]
if i == 0 and query:
content.insert(0, {"type": "text", "text": query})
resp = requests.post(
url,
json={"session_id": session_id, "messages": [{"role": "user", "content": content}]},
timeout=120,
).json()
info = resp.get("interaction", {})
yield Tick(
index=info.get("frame_index", i + 1),
t=t,
action=info.get("action", "?"),
text=resp["choices"][0]["message"]["content"],
skipped=info.get("inference_skipped", False),
latency_ms=info.get("latency_ms", 0.0),
delegation=info.get("delegation"),
)
if realtime:
time.sleep(1.0 / fps)