[Entrypoint] Add realtime OpenPI robot serving API (#3673)
Signed-off-by: Yangshen Deng <yangshen.d@outlook.com> Signed-off-by: amy-why-3459 <wuhaiyan17@huawei.com> Co-authored-by: Meng <meng_chen99@163.com> Co-authored-by: amy-why-3459 <wuhaiyan17@huawei.com>
This commit is contained in:
6 files changed
+966
No files matched your search
@@ -0,0 +1,156 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""WebSocket connection for robot policy inference (OpenPI protocol).
|
||||
|
||||
Protocol (compatible with OpenPI policy clients):
|
||||
Connect -> server sends msgpack(PolicyServerConfig fields)
|
||||
Infer -> client sends msgpack(obs), server sends msgpack(ndarray)
|
||||
Reset -> client sends msgpack({endpoint:reset}), server sends msgpack(status)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from typing import Any
|
||||
|
||||
from fastapi import WebSocket
|
||||
from starlette.websockets import WebSocketDisconnect
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from vllm_omni.entrypoints.openpi.serving import (
|
||||
ServingRealtimeRobotOpenPI,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
_DEFAULT_IDLE_TIMEOUT = 30.0
|
||||
MAX_OPENPI_PAYLOAD_BYTES = 64 * 1024 * 1024
|
||||
|
||||
|
||||
def _get_msgpack_numpy() -> Any:
|
||||
try:
|
||||
from openpi_client import msgpack_numpy
|
||||
except ImportError as exc:
|
||||
raise ImportError(
|
||||
"The `/v1/realtime/robot/openpi` endpoint requires the optional "
|
||||
"`openpi-client` dependency. Install it with `pip install openpi-client`."
|
||||
) from exc
|
||||
|
||||
return msgpack_numpy
|
||||
|
||||
|
||||
def _pack(obj: Any) -> bytes:
|
||||
return _get_msgpack_numpy().packb(obj)
|
||||
|
||||
|
||||
def _unpack(data: bytes) -> Any:
|
||||
return _get_msgpack_numpy().unpackb(data)
|
||||
|
||||
|
||||
class RobotRealtimeConnection:
|
||||
"""WebSocket connection for robot policy inference."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
websocket: WebSocket,
|
||||
serving: ServingRealtimeRobotOpenPI,
|
||||
idle_timeout: float = _DEFAULT_IDLE_TIMEOUT,
|
||||
) -> None:
|
||||
self.websocket = websocket
|
||||
self.serving = serving
|
||||
self._idle_timeout = idle_timeout
|
||||
self._current_session_id: str | None = None
|
||||
self._call_count = 0
|
||||
|
||||
def reset(self) -> None:
|
||||
self._current_session_id = None
|
||||
self._call_count = 0
|
||||
|
||||
async def _send_error(self, message: str) -> None:
|
||||
await self.websocket.send_bytes(_pack({"type": "error", "message": message}))
|
||||
|
||||
def _unpack_request(self, data: bytes) -> dict[str, Any]:
|
||||
if len(data) > MAX_OPENPI_PAYLOAD_BYTES:
|
||||
raise ValueError("OpenPI request payload too large")
|
||||
obs = _unpack(data)
|
||||
if not isinstance(obs, dict):
|
||||
raise ValueError("Invalid request payload")
|
||||
return obs
|
||||
|
||||
async def handle_connection(self) -> None:
|
||||
"""Main loop for OpenPI-compatible policy serving."""
|
||||
await self.websocket.accept()
|
||||
|
||||
try:
|
||||
# Send model-specific PolicyServerConfig resolved by serving from
|
||||
# diffusion od_config.model_config.
|
||||
metadata = self.serving.policy_server_config.to_dict()
|
||||
await self.websocket.send_bytes(_pack(metadata))
|
||||
|
||||
while True:
|
||||
try:
|
||||
msg = await asyncio.wait_for(
|
||||
self.websocket.receive(),
|
||||
timeout=self._idle_timeout,
|
||||
)
|
||||
except asyncio.TimeoutError:
|
||||
logger.info("Robot OpenPI connection idle timeout after %.1f seconds", self._idle_timeout)
|
||||
try:
|
||||
await self.websocket.close()
|
||||
except Exception:
|
||||
logger.debug("Failed to close idle robot OpenPI websocket", exc_info=True)
|
||||
return
|
||||
|
||||
if msg.get("type") == "websocket.disconnect":
|
||||
break
|
||||
|
||||
if "bytes" not in msg or not msg["bytes"]:
|
||||
continue
|
||||
|
||||
try:
|
||||
obs = self._unpack_request(msg["bytes"])
|
||||
except Exception:
|
||||
logger.exception("Invalid robot OpenPI request payload")
|
||||
try:
|
||||
await self._send_error("Invalid request payload")
|
||||
except Exception:
|
||||
break
|
||||
continue
|
||||
|
||||
try:
|
||||
endpoint = obs.pop("endpoint", "infer")
|
||||
|
||||
if endpoint == "reset":
|
||||
self.reset()
|
||||
self.serving.reset(obs)
|
||||
await self.websocket.send_bytes(_pack({"status": "reset successful"}))
|
||||
else:
|
||||
session_id = str(obs.get("session_id") or self._current_session_id or "default")
|
||||
if session_id != self._current_session_id:
|
||||
if self._current_session_id is not None:
|
||||
logger.info(
|
||||
"Robot OpenPI session changed %s -> %s",
|
||||
self._current_session_id,
|
||||
session_id,
|
||||
)
|
||||
self._current_session_id = session_id
|
||||
self._call_count = 0
|
||||
|
||||
self._call_count += 1
|
||||
actions = await self.serving.infer(
|
||||
obs,
|
||||
session_id=session_id,
|
||||
reset=self._call_count <= 1,
|
||||
)
|
||||
await self.websocket.send_bytes(_pack(actions))
|
||||
except Exception:
|
||||
logger.exception("Error handling request")
|
||||
try:
|
||||
await self._send_error("Internal inference error")
|
||||
except Exception:
|
||||
break
|
||||
|
||||
except WebSocketDisconnect:
|
||||
pass
|
||||
except Exception:
|
||||
logger.exception("Connection error")
|
||||
Reference in new issue
Block a user