[P0][RFC #3747] RL rollout serving: session management + world_model_env (#4770)

Signed-off-by: srinivas_oo7 <sklinkedin0120@gmail.com>
Co-authored-by: srinivas_oo7 <sklinkedin0120@gmail.com>
Co-authored-by: Zhou Taichang <tzhouam@connect.ust.hk>
This commit is contained in:
authored and GitHub committed 2026-09-14 09:26:44 +08:00
1 parent 6505577459
commit 8e738ce21b
11 files changed
+1421 -1

No files matched your search

@@ -138,6 +138,10 @@ from vllm_omni.entrypoints.openai.protocol.images import (
ImageGenerationResponse,
ResponseFormat,
)
from vllm_omni.entrypoints.openai.protocol.rollout import (
CreateSessionRequest,
RolloutStepRequest,
)
from vllm_omni.entrypoints.openai.protocol.videos import (
VideoDeleteResponse,
VideoGenerationRequest,
@@ -146,8 +150,14 @@ from vllm_omni.entrypoints.openai.protocol.videos import (
VideoResponse,
)
from vllm_omni.entrypoints.openai.realtime_connection import RealtimeConnection
from vllm_omni.entrypoints.openai.rollout_session import (
RolloutSessionCapacityError,
RolloutSessionClosedError,
RolloutSessionNotFoundError,
)
from vllm_omni.entrypoints.openai.serving_audio_generate import OmniOpenAIServingAudioGenerate
from vllm_omni.entrypoints.openai.serving_chat import OmniOpenAIServingChat
from vllm_omni.entrypoints.openai.serving_rl_rollout import ServingRLRollout
from vllm_omni.entrypoints.openai.serving_speech import OmniOpenAIServingSpeech
from vllm_omni.entrypoints.openai.serving_speech_stream import OmniStreamingSpeechHandler
from vllm_omni.entrypoints.openai.serving_video import (
@@ -604,6 +614,10 @@ async def omni_init_app_state(
engine_client=engine_client,
model_name=model_name,
)
if state.openai_serving_realtime_robot is not None:
state.rl_rollout_serving = ServingRLRollout(state.openai_serving_realtime_robot)
else:
state.rl_rollout_serving = None
state.enable_server_load_tracking = getattr(args, "enable_server_load_tracking", False)
state.server_load_metrics = 0
@@ -949,6 +963,7 @@ async def omni_init_app_state(
stage_configs=state.stage_configs,
)
state.openai_serving_realtime_robot = None
state.rl_rollout_serving = None
state.enable_server_load_tracking = args.enable_server_load_tracking
state.server_load_metrics = 0
@@ -1529,6 +1544,78 @@ async def duplex_websocket(websocket: WebSocket):
await handler.handle_session(websocket)
# RL Rollout serving (RFC #3747, P0)
def _rl_rollout_serving(request: Request) -> ServingRLRollout:
serving = getattr(request.app.state, "rl_rollout_serving", None)
if serving is None:
raise HTTPException(status_code=501, detail="RL rollout serving not available for this model.")
return serving
@router.post("/v1/realtime/sessions")
async def create_rollout_session(body: CreateSessionRequest, request: Request):
serving = _rl_rollout_serving(request)
try:
return (await serving.create_session(body)).model_dump()
except ValueError as exc:
raise HTTPException(status_code=400, detail=str(exc)) from exc
except RolloutSessionCapacityError as exc:
raise HTTPException(status_code=503, detail=str(exc)) from exc
@router.post("/v1/realtime/sessions/{session_id}/step")
async def rollout_step(session_id: str, body: RolloutStepRequest, request: Request):
serving = _rl_rollout_serving(request)
response = await serving.step(session_id, body)
if response.error is not None:
status_code = None
if response.error.code == "session_not_found":
status_code = 404
elif response.error.code == "session_closed":
status_code = 410
elif response.error.code in {"invalid_request", "step_already_committed", "step_out_of_order"}:
status_code = 400
if status_code is not None:
return JSONResponse(content=response.model_dump(), status_code=status_code)
return response.model_dump()
@router.post("/v1/realtime/sessions/{session_id}/reset")
async def reset_rollout_session(session_id: str, request: Request):
serving = _rl_rollout_serving(request)
try:
return (await serving.reset_session(session_id)).model_dump()
except RolloutSessionNotFoundError:
raise HTTPException(status_code=404, detail=f"Session {session_id!r} not found.")
except RolloutSessionClosedError:
raise HTTPException(status_code=410, detail=f"Session {session_id!r} is closed.")
@router.post("/v1/realtime/sessions/{session_id}/close")
async def close_rollout_session(session_id: str, request: Request):
serving = _rl_rollout_serving(request)
try:
await serving.close_session(session_id)
return {"session_id": session_id, "closed": True}
except RolloutSessionNotFoundError:
raise HTTPException(status_code=404, detail=f"Session {session_id!r} not found.")
except RolloutSessionClosedError:
return {"session_id": session_id, "closed": True}
@router.get("/v1/realtime/sessions/{session_id}/status")
async def rollout_session_status(session_id: str, request: Request):
serving = _rl_rollout_serving(request)
try:
return (await serving.get_status(session_id)).model_dump()
except RolloutSessionNotFoundError:
raise HTTPException(status_code=404, detail=f"Session {session_id!r} not found.")
except RolloutSessionClosedError:
raise HTTPException(status_code=410, detail=f"Session {session_id!r} is closed.")
# Health and Model endpoints for diffusion mode