[P0][RFC #3747] RL rollout serving: session management + world_model_env (#4770)

Signed-off-by: srinivas_oo7 <sklinkedin0120@gmail.com>
Co-authored-by: srinivas_oo7 <sklinkedin0120@gmail.com>
Co-authored-by: Zhou Taichang <tzhouam@connect.ust.hk>
This commit is contained in:
authored and GitHub committed 2026-09-14 09:26:44 +08:00
1 parent 6505577459
commit 8e738ce21b
11 files changed
+1421 -1

No files matched your search

@@ -80,6 +80,11 @@ _EXPECTED_ROUTER_ROUTES = {
("WEBSOCKET", "/v1/realtime"),
("WEBSOCKET", "/v1/realtime/robot/openpi"),
("WEBSOCKET", "/v1/duplex"),
("POST", "/v1/realtime/sessions"),
("POST", "/v1/realtime/sessions/{session_id}/step"),
("POST", "/v1/realtime/sessions/{session_id}/reset"),
("POST", "/v1/realtime/sessions/{session_id}/close"),
("GET", "/v1/realtime/sessions/{session_id}/status"),
("GET", "/health"),
("GET", "/v1/models"),
("POST", "/v1/images/generations"),
@@ -120,6 +125,7 @@ _DIFFUSION_APP_STATE_KEYS = {
"openai_streaming_speech",
"openai_streaming_video",
"openai_serving_realtime_robot",
"rl_rollout_serving",
"enable_server_load_tracking",
"server_load_metrics",
}
@@ -151,12 +157,14 @@ _MULTISTAGE_APP_STATE_KEYS = {
"openai_serving_realtime",
"openai_serving_video",
"openai_serving_realtime_robot",
"rl_rollout_serving",
"enable_server_load_tracking",
"server_load_metrics",
}
_MULTISTAGE_MUST_BE_NONE = {
"openai_serving_duplex",
"openai_serving_realtime_robot",
"rl_rollout_serving",
}
_MULTISTAGE_MUST_BE_WIRED = _MULTISTAGE_APP_STATE_KEYS - _MULTISTAGE_MUST_BE_NONE
@@ -440,3 +440,17 @@ def test_extract_actions_does_not_iterate_result_object():
actions = serving._extract_actions(IterableResult())
np.testing.assert_allclose(actions, np.array([[1.0, 2.0, 3.0]], dtype=np.float32))
def test_drop_session_calls_dreamzero_close_hook():
closed: list[str] = []
pipeline = SimpleNamespace(close_ar_diffusion_session=lambda session_id: closed.append(session_id))
engine_client = SimpleNamespace(
model_config={"policy_server_config": TEST_POLICY_SERVER_CONFIG},
model_runner=SimpleNamespace(pipeline=pipeline),
)
serving = openpi_serving.ServingRealtimeRobotOpenPI(engine_client=engine_client)
serving.drop_session("rollout-1")
assert closed == ["rollout-1"]
+2
View File
@@ -0,0 +1,2 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
+167
View File
@@ -0,0 +1,167 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Route-level tests for RL rollout serving endpoints."""
from __future__ import annotations
import pytest
from fastapi import FastAPI
from fastapi.testclient import TestClient
from vllm_omni.entrypoints.openai.api_server import router
from vllm_omni.entrypoints.openai.protocol.rollout import (
ErrorObject,
RolloutStepResponse,
SessionMetadata,
)
from vllm_omni.entrypoints.openai.rollout_session import (
RolloutSessionClosedError,
RolloutSessionNotFoundError,
)
class _FakeRolloutServing:
def __init__(self, close_error: Exception | None = None, step_error_code: str | None = None) -> None:
self.close_error = close_error
self.step_error_code = step_error_code
self.step_request = None
async def step(self, session_id, body):
self.step_request = body
error = None
if self.step_error_code is not None:
error = ErrorObject(
code=self.step_error_code,
message=f"{self.step_error_code}: {session_id}",
step_id=body.step_id,
committed_step_id=-1,
)
return RolloutStepResponse(
step_id=body.step_id,
next_observation=None,
model_metadata=SessionMetadata(
latency_ms=0.0,
steps_generated=0,
context_length=0,
committed_step_id=-1,
),
error=error,
)
async def close_session(self, session_id):
if self.close_error is not None:
raise self.close_error
def _client(serving: _FakeRolloutServing | None = None) -> TestClient:
app = FastAPI()
app.include_router(router)
if serving is not None:
app.state.rl_rollout_serving = serving
return TestClient(app)
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
def test_step_route_rejects_base64_image_payload():
serving = _FakeRolloutServing()
client = _client(serving)
response = client.post(
"/v1/realtime/sessions/s1/step",
json={
"step_id": 0,
"observation": {"images": {"front": "base64-frames"}},
"action": {},
},
)
assert response.status_code == 422
assert serving.step_request is None
def test_step_route_accepts_nested_image_payload():
serving = _FakeRolloutServing()
client = _client(serving)
response = client.post(
"/v1/realtime/sessions/s1/step",
json={
"step_id": 0,
"observation": {"images": {"observation/exterior_image_1_left": [[[0, 0, 0]]]}},
"action": {},
},
)
assert response.status_code == 200
assert serving.step_request.observation.images == {"observation/exterior_image_1_left": [[[0, 0, 0]]]}
def test_step_route_requires_action():
client = _client(_FakeRolloutServing())
response = client.post(
"/v1/realtime/sessions/s1/step",
json={"step_id": 0, "observation": {}},
)
assert response.status_code == 422
def test_step_route_maps_missing_session_to_404():
client = _client(_FakeRolloutServing(step_error_code="session_not_found"))
response = client.post(
"/v1/realtime/sessions/s1/step",
json={"step_id": 0, "observation": {}, "action": {}},
)
assert response.status_code == 404
def test_step_route_maps_closed_session_to_410():
client = _client(_FakeRolloutServing(step_error_code="session_closed"))
response = client.post(
"/v1/realtime/sessions/s1/step",
json={"step_id": 0, "observation": {}, "action": {}},
)
assert response.status_code == 410
def test_step_route_maps_invalid_request_to_400():
client = _client(_FakeRolloutServing(step_error_code="invalid_request"))
response = client.post(
"/v1/realtime/sessions/s1/step",
json={"step_id": 0, "observation": {}, "action": {}},
)
assert response.status_code == 400
def test_close_route_maps_missing_session_to_404():
client = _client(_FakeRolloutServing(close_error=RolloutSessionNotFoundError("s1")))
response = client.post("/v1/realtime/sessions/s1/close")
assert response.status_code == 404
def test_close_route_is_idempotent_for_closed_session():
client = _client(_FakeRolloutServing(close_error=RolloutSessionClosedError("s1")))
response = client.post("/v1/realtime/sessions/s1/close")
assert response.status_code == 200
assert response.json()["closed"] is True
def test_rollout_routes_return_501_when_serving_is_unavailable():
client = _client()
response = client.post("/v1/realtime/sessions/s1/close")
assert response.status_code == 501
+157
View File
@@ -0,0 +1,157 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Unit tests for RolloutSessionStore (RFC #3747, P0)."""
import asyncio
import pytest
from vllm_omni.entrypoints.openai.rollout_session import (
RolloutSession,
RolloutSessionClosedError,
RolloutSessionNotFoundError,
RolloutSessionStepError,
RolloutSessionStore,
)
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
@pytest.fixture
def store() -> RolloutSessionStore:
return RolloutSessionStore()
@pytest.mark.asyncio
async def test_create_returns_session(store):
s = await store.create("s1", model="dreamzero", mode="world_model_env")
assert isinstance(s, RolloutSession)
assert s.session_id == "s1"
assert s.committed_step_id == -1
assert s.context_length == 0
assert not s.closed
@pytest.mark.asyncio
async def test_get_existing(store):
await store.create("s1", model="dreamzero", mode="world_model_env")
s = await store.get("s1")
assert s.session_id == "s1"
@pytest.mark.asyncio
async def test_get_missing_raises(store):
with pytest.raises(RolloutSessionNotFoundError):
await store.get("does-not-exist")
@pytest.mark.asyncio
async def test_get_closed_raises(store):
await store.create("s1", model="dreamzero", mode="world_model_env")
await store.close("s1")
with pytest.raises(RolloutSessionClosedError):
await store.get("s1")
@pytest.mark.asyncio
async def test_close_missing_raises(store):
with pytest.raises(RolloutSessionNotFoundError):
await store.close("does-not-exist")
@pytest.mark.asyncio
async def test_close_is_idempotent(store):
await store.create("s1", model="dreamzero", mode="world_model_env")
await store.close("s1")
await store.close("s1")
s = await store.get("s1", include_closed=True)
assert s.closed
@pytest.mark.asyncio
async def test_advance_updates_committed_step_id(store):
await store.create("s1", model="dreamzero", mode="world_model_env")
await store.advance("s1", step_id=0)
s = await store.get("s1")
assert s.committed_step_id == 0
assert s.context_length == 1
await store.advance("s1", step_id=1)
s = await store.get("s1")
assert s.committed_step_id == 1
assert s.context_length == 2
@pytest.mark.asyncio
async def test_advance_rejects_duplicate_step_id(store):
await store.create("s1", model="dreamzero", mode="world_model_env")
await store.advance("s1", step_id=0)
with pytest.raises(RolloutSessionStepError):
await store.advance("s1", step_id=0)
s = await store.get("s1")
assert s.committed_step_id == 0
assert s.context_length == 1
@pytest.mark.asyncio
async def test_advance_rejects_out_of_order_step_id(store):
await store.create("s1", model="dreamzero", mode="world_model_env")
with pytest.raises(RolloutSessionStepError):
await store.advance("s1", step_id=2)
s = await store.get("s1")
assert s.committed_step_id == -1
assert s.context_length == 0
@pytest.mark.asyncio
async def test_reset_clears_state(store):
await store.create("s1", model="dreamzero", mode="world_model_env")
await store.advance("s1", step_id=0)
await store.advance("s1", step_id=1)
await store.advance("s1", step_id=2)
await store.advance("s1", step_id=3)
await store.reset("s1")
s = await store.get("s1")
assert s.committed_step_id == -1
assert s.context_length == 0
@pytest.mark.asyncio
async def test_reset_closed_raises(store):
await store.create("s1", model="dreamzero", mode="world_model_env")
await store.close("s1")
with pytest.raises(RolloutSessionClosedError):
await store.reset("s1")
@pytest.mark.asyncio
async def test_failed_step_does_not_advance(store):
"""committed_step_id must not change when a step raises (RFC section 6.5 invariant)."""
await store.create("s1", model="dreamzero", mode="world_model_env")
await store.advance("s1", step_id=0)
# Simulate a failed step: advance is NOT called
s = await store.get("s1")
assert s.committed_step_id == 0 # unchanged
@pytest.mark.asyncio
async def test_concurrent_steps_are_serialised(store):
"""Per-session lock must prevent concurrent context mutations."""
await store.create("s1", model="dreamzero", mode="world_model_env")
s = await store.get("s1")
results: list[int] = []
async def slow_advance(step_id: int, delay: float) -> None:
await asyncio.sleep(delay)
async with s.lock:
await asyncio.sleep(0.01)
await store.advance("s1", step_id=step_id)
results.append(step_id)
await asyncio.gather(slow_advance(0, 0.0), slow_advance(1, 0.01))
# Both steps ran under the per-session lock and committed in order.
assert sorted(results) == [0, 1]
+371
View File
@@ -0,0 +1,371 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Unit tests for ServingRLRollout (RFC #3747, P0).
The DreamZero engine is mocked. Tests verify session lifecycle, step_id
semantics, committed_step_id invariants, and error paths without a GPU.
"""
from __future__ import annotations
from typing import Any
from unittest.mock import MagicMock
import numpy as np
import pytest
from vllm_omni.entrypoints.openai.protocol.rollout import (
Action,
CreateSessionRequest,
Observation,
RolloutStepRequest,
)
from vllm_omni.entrypoints.openai.serving_rl_rollout import ServingRLRollout
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
def _make_fake_result(video: Any = None, *, formatter_shaped: bool = True) -> MagicMock:
result = MagicMock()
if video is None:
video = np.zeros((1, 3, 64, 64), dtype=np.float32)
if formatter_shaped:
# Matches output_formatter: video is peeled into images; mm only has actions.
result.images = video
result.multimodal_output = {"actions": np.zeros((1, 8), dtype=np.float32)}
else:
result.images = None
result.multimodal_output = {"video": video}
return result
def _make_openpi(video: Any = None, raises: Exception | None = None, *, formatter_shaped: bool = True):
openpi = MagicMock()
openpi.engine_client = MagicMock()
openpi.model_name = None
async def _generate(*_, **__):
if raises:
raise raises
yield _make_fake_result(video, formatter_shaped=formatter_shaped)
openpi.engine_client.generate = _generate
openpi.build_request = MagicMock(
return_value=MagicMock(
prompt="",
prompts=[""],
request_id="test-req",
sampling_params=MagicMock(),
)
)
return openpi
@pytest.fixture
def serving() -> ServingRLRollout:
return ServingRLRollout(_make_openpi())
# ---------------------------------------------------------------------------
# Session lifecycle
# ---------------------------------------------------------------------------
@pytest.mark.asyncio
async def test_create_session(serving):
resp = await serving.create_session(CreateSessionRequest(model="dreamzero", mode="world_model_env"))
assert resp.session_id
assert resp.mode == "world_model_env"
@pytest.mark.asyncio
async def test_get_status_after_create(serving):
resp = await serving.create_session(CreateSessionRequest(model="dreamzero", mode="world_model_env"))
status = await serving.get_status(resp.session_id)
assert status.committed_step_id == -1
assert status.context_length == 0
assert not status.closed
@pytest.mark.asyncio
async def test_close_session(serving):
resp = await serving.create_session(CreateSessionRequest(model="dreamzero", mode="world_model_env"))
await serving.close_session(resp.session_id)
status = await serving.get_status(resp.session_id)
assert status.closed
assert status.committed_step_id == -1
serving._openpi.drop_session.assert_any_call(resp.session_id)
serving._openpi.drop_session.assert_any_call(f"{resp.session_id}:stateless")
# ---------------------------------------------------------------------------
# Step - success path
# ---------------------------------------------------------------------------
@pytest.mark.asyncio
async def test_successful_step_advances_committed_step_id(serving):
sess = await serving.create_session(CreateSessionRequest(model="dreamzero", mode="world_model_env"))
req = RolloutStepRequest(
step_id=0,
observation=Observation(state=[0.1, 0.2]),
action=Action(joint_positions=[0.0] * 7),
)
resp = await serving.step(sess.session_id, req)
assert resp.error is None
assert resp.model_metadata.committed_step_id == 0
status = await serving.get_status(sess.session_id)
assert status.committed_step_id == 0
assert status.context_length == 1
@pytest.mark.asyncio
async def test_repeated_steps_advance_monotonically(serving):
sess = await serving.create_session(CreateSessionRequest(model="dreamzero", mode="world_model_env"))
for i in range(3):
req = RolloutStepRequest(
step_id=i,
observation=Observation(state=[float(i)]),
action=Action(joint_positions=[0.0] * 7),
)
resp = await serving.step(sess.session_id, req)
assert resp.error is None
assert resp.model_metadata.committed_step_id == i
status = await serving.get_status(sess.session_id)
assert status.committed_step_id == 2
assert status.context_length == 3
@pytest.mark.asyncio
async def test_next_observation_present_on_success(serving):
sess = await serving.create_session(CreateSessionRequest(model="dreamzero", mode="world_model_env"))
req = RolloutStepRequest(
step_id=0,
observation=Observation(),
action=Action(),
)
resp = await serving.step(sess.session_id, req)
assert resp.next_observation is not None
assert "video_latent" in resp.next_observation
assert resp.next_observation["dtype"] == "float32"
@pytest.mark.asyncio
async def test_duplicate_step_does_not_rerun_or_advance(serving):
sess = await serving.create_session(CreateSessionRequest(model="dreamzero", mode="world_model_env"))
req = RolloutStepRequest(step_id=0, observation=Observation(), action=Action())
assert (await serving.step(sess.session_id, req)).error is None
resp = await serving.step(sess.session_id, req)
assert resp.error is not None
assert resp.error.code == "step_already_committed"
assert resp.model_metadata.committed_step_id == 0
assert resp.model_metadata.context_length == 1
serving._openpi.build_request.assert_called_once()
@pytest.mark.asyncio
async def test_out_of_order_step_does_not_rerun_or_advance(serving):
sess = await serving.create_session(CreateSessionRequest(model="dreamzero", mode="world_model_env"))
req = RolloutStepRequest(step_id=2, observation=Observation(), action=Action())
resp = await serving.step(sess.session_id, req)
assert resp.error is not None
assert resp.error.code == "step_out_of_order"
assert resp.model_metadata.committed_step_id == -1
assert resp.model_metadata.context_length == 0
serving._openpi.build_request.assert_not_called()
@pytest.mark.asyncio
async def test_stateless_step_does_not_commit_context(serving):
sess = await serving.create_session(CreateSessionRequest(model="dreamzero", mode="world_model_env"))
req = RolloutStepRequest(
step_id=7,
observation=Observation(),
action=Action(),
use_session_context=False,
)
resp = await serving.step(sess.session_id, req)
assert resp.error is None
assert resp.model_metadata.committed_step_id == -1
assert resp.model_metadata.context_length == 0
serving._openpi.build_request.assert_called_once()
kwargs = serving._openpi.build_request.call_args.kwargs
assert kwargs["reset"] is True
assert kwargs["session_id"] == f"{sess.session_id}:stateless"
# ---------------------------------------------------------------------------
# Step - failure path: committed_step_id must NOT advance
# ---------------------------------------------------------------------------
@pytest.mark.asyncio
async def test_failed_step_does_not_advance_committed_step_id():
serving = ServingRLRollout(_make_openpi(raises=RuntimeError("engine dead")))
sess = await serving.create_session(CreateSessionRequest(model="dreamzero", mode="world_model_env"))
req = RolloutStepRequest(step_id=0, observation=Observation(), action=Action())
resp = await serving.step(sess.session_id, req)
assert resp.error is not None
assert resp.error.code == "inference_error"
assert resp.model_metadata.committed_step_id == -1 # must not have advanced
status = await serving.get_status(sess.session_id)
assert status.committed_step_id == -1
@pytest.mark.asyncio
async def test_output_encoding_failure_does_not_advance_committed_step_id():
class BadVideo:
def detach(self):
return self
def float(self):
return self
def cpu(self):
return self
def numpy(self):
raise TypeError("cannot encode")
serving = ServingRLRollout(_make_openpi(video=BadVideo()))
sess = await serving.create_session(CreateSessionRequest(model="dreamzero", mode="world_model_env"))
req = RolloutStepRequest(step_id=0, observation=Observation(), action=Action())
resp = await serving.step(sess.session_id, req)
assert resp.error is not None
assert resp.error.code == "inference_error"
assert resp.model_metadata.committed_step_id == -1
status = await serving.get_status(sess.session_id)
assert status.committed_step_id == -1
@pytest.mark.asyncio
async def test_failed_step_after_success_preserves_last_committed():
good_serving = ServingRLRollout(_make_openpi())
sess = await good_serving.create_session(CreateSessionRequest(model="dreamzero", mode="world_model_env"))
# Step 0 succeeds
await good_serving.step(sess.session_id, RolloutStepRequest(step_id=0, observation=Observation(), action=Action()))
# Swap engine to failing one - reuse the same store via internal reference
good_serving._openpi = _make_openpi(raises=RuntimeError("oops"))
resp = await good_serving.step(
sess.session_id, RolloutStepRequest(step_id=1, observation=Observation(), action=Action())
)
assert resp.error is not None
assert resp.model_metadata.committed_step_id == 0 # still at last good step
assert resp.model_metadata.context_length == 1
# ---------------------------------------------------------------------------
# Step - unknown session
# ---------------------------------------------------------------------------
@pytest.mark.asyncio
async def test_step_unknown_session_returns_error(serving):
req = RolloutStepRequest(step_id=0, observation=Observation(), action=Action())
resp = await serving.step("no-such-session", req)
assert resp.error is not None
assert resp.error.code == "session_not_found"
# ---------------------------------------------------------------------------
# Reset
# ---------------------------------------------------------------------------
@pytest.mark.asyncio
async def test_reset_clears_committed_step_id(serving):
sess = await serving.create_session(CreateSessionRequest(model="dreamzero", mode="world_model_env"))
await serving.step(sess.session_id, RolloutStepRequest(step_id=0, observation=Observation(), action=Action()))
await serving.reset_session(sess.session_id)
status = await serving.get_status(sess.session_id)
assert status.committed_step_id == -1
assert status.context_length == 0
serving._openpi.drop_session.assert_called_with(sess.session_id)
# ---------------------------------------------------------------------------
# action -> obs folding (module assumption)
# ---------------------------------------------------------------------------
def test_merge_action_into_obs_emits_dreamzero_state_keys():
from vllm_omni.entrypoints.openai.serving_rl_rollout import _merge_action_into_obs
obs = Observation(state=[9.0] * 8, prompt="test")
action = Action(joint_positions=[1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0], gripper_position=0.5)
result = _merge_action_into_obs(obs, action)
assert result["prompt"] == "test"
np.testing.assert_array_almost_equal(result["observation/joint_position"], [1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0])
np.testing.assert_array_almost_equal(result["observation/gripper_position"], [0.5])
def test_merge_action_none_splits_obs_state_for_dreamzero():
from vllm_omni.entrypoints.openai.serving_rl_rollout import _merge_action_into_obs
obs = Observation(state=[1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 0.25])
result = _merge_action_into_obs(obs, None)
np.testing.assert_array_almost_equal(result["observation/joint_position"], [1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0])
np.testing.assert_array_almost_equal(result["observation/gripper_position"], [0.25])
def test_merge_no_state_no_action_omits_state_key():
from vllm_omni.entrypoints.openai.serving_rl_rollout import _merge_action_into_obs
obs = Observation()
result = _merge_action_into_obs(obs, None)
assert "observation/joint_position" not in result
assert "observation/gripper_position" not in result
def test_merge_rejects_wrong_dreamzero_joint_dimension():
from vllm_omni.entrypoints.openai.serving_rl_rollout import _merge_action_into_obs
with pytest.raises(ValueError, match="joint_positions must have 7 values"):
_merge_action_into_obs(Observation(), Action(joint_positions=[1.0, 2.0]))
@pytest.mark.asyncio
async def test_step_reads_formatter_images_not_multimodal_video():
video = np.ones((1, 3, 8, 8), dtype=np.float32)
serving = ServingRLRollout(_make_openpi(video=video, formatter_shaped=True))
sess = await serving.create_session(CreateSessionRequest(model="dreamzero", mode="world_model_env"))
resp = await serving.step(
sess.session_id,
RolloutStepRequest(step_id=0, observation=Observation(), action=Action()),
)
assert resp.error is None
assert resp.next_observation is not None
assert "video_latent" in resp.next_observation
assert resp.next_observation["shape"] == [1, 3, 8, 8]
@pytest.mark.asyncio
async def test_step_falls_back_to_multimodal_video():
video = np.ones((2, 4, 4), dtype=np.float32)
serving = ServingRLRollout(_make_openpi(video=video, formatter_shaped=False))
sess = await serving.create_session(CreateSessionRequest(model="dreamzero", mode="world_model_env"))
resp = await serving.step(
sess.session_id,
RolloutStepRequest(step_id=0, observation=Observation(), action=Action()),
)
assert resp.error is None
assert resp.next_observation is not None
assert resp.next_observation["shape"] == [2, 4, 4]