[Diffusion] DreamZero world model integration with CFG parallel + OpenPI serving (#2162)

Signed-off-by: Meng <meng_chen99@163.com>
Signed-off-by: Yangshen Deng <yangshen.d@outlook.com>
Signed-off-by: Suli Wang <50828270+wsl2000@users.noreply.github.com>
Co-authored-by: Meng <meng_chen99@163.com>
Co-authored-by: Suli Wang <50828270+wsl2000@users.noreply.github.com>
Co-authored-by: amy-why-3459 <wuhaiyan17@huawei.com>

examples/offline_inference/dreamzero/export_prediction_video.py

@@ -0,0 +1,304 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+from __future__ import annotations
+
+import argparse
+import uuid
+from pathlib import Path
+
+import cv2
+import numpy as np
+import torch
+from PIL import Image
+
+from vllm_omni import Omni
+from vllm_omni.inputs.data import OmniDiffusionSamplingParams
+from vllm_omni.outputs import OmniRequestOutput
+
+WORKER_EXTENSION = "vllm_omni.diffusion.models.dreamzero.video_export_worker.DreamZeroVideoExportWorkerExtension"
+ASSET_REPO_ID = "YangshenDeng/vllm-omni-dreamzero-assets"
+DEFAULT_SESSION_PREFIX = "dreamzero-export"
+RELATIVE_OFFSETS = [-23, -16, -8, 0]
+ACTION_HORIZON = 24
+CAMERA_FILES = {
+    "observation/exterior_image_0_left": "exterior_image_1_left.mp4",
+    "observation/exterior_image_1_left": "exterior_image_2_left.mp4",
+    "observation/wrist_image_left": "wrist_image_left.mp4",
+}
+
+
+def _parse_args() -> argparse.Namespace:
+    repo_root = Path(__file__).resolve().parents[3]
+    parser = argparse.ArgumentParser(description="Export DreamZero prediction video from downloaded example inputs.")
+    parser.add_argument("--model", default="GEAR-Dreams/DreamZero-DROID")
+    parser.add_argument("--deploy-config", type=Path, required=True)
+    parser.add_argument(
+        "--video-dir",
+        type=Path,
+        default=repo_root / "outputs" / "dreamzero" / "assets",
+        help="Directory containing the three camera MP4 files.",
+    )
+    parser.add_argument(
+        "--output-dir", type=Path, default=repo_root / "outputs" / "dreamzero" / "generated_predictions"
+    )
+    parser.add_argument("--output-stem", default="dreamzero_prediction")
+    parser.add_argument(
+        "--prompt",
+        default="Move the pan forward and use the brush in the middle of the plates to brush the inside of the pan",
+    )
+    parser.add_argument("--session-id", default=None)
+    parser.add_argument("--save-input-video", action="store_true")
+    parser.add_argument("--save-gif", action="store_true")
+    parser.add_argument("--save-actions", action="store_true")
+    parser.add_argument("--fps", type=int, default=5)
+    return parser.parse_args()
+
+
+def _load_all_frames(video_path: Path) -> np.ndarray:
+    cap = cv2.VideoCapture(str(video_path))
+    frames = []
+    while True:
+        ok, frame = cap.read()
+        if not ok:
+            break
+        frames.append(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB))
+    cap.release()
+    if not frames:
+        raise RuntimeError(f"No frames loaded from {video_path}")
+    return np.stack(frames, axis=0)
+
+
+def _load_camera_frames(video_dir: Path) -> dict[str, np.ndarray]:
+    camera_frames: dict[str, np.ndarray] = {}
+    for camera_key, file_name in CAMERA_FILES.items():
+        video_path = video_dir / file_name
+        if not video_path.exists():
+            raise FileNotFoundError(
+                f"Missing DreamZero example asset: {video_path}. "
+                "Download the example videos with: "
+                f"`hf download {ASSET_REPO_ID} --repo-type dataset --local-dir {video_dir}`"
+            )
+        camera_frames[camera_key] = _load_all_frames(video_path)
+    return camera_frames
+
+
+def _build_frame_schedule(total_frames: int, num_chunks: int) -> list[list[int]]:
+    chunks: list[list[int]] = []
+    current_frame = 23
+    for _ in range(num_chunks):
+        indices = [max(current_frame + offset, 0) for offset in RELATIVE_OFFSETS]
+        if indices[-1] >= total_frames:
+            break
+        chunks.append(indices)
+        current_frame += ACTION_HORIZON
+    return chunks
+
+
+def _make_obs_from_video(
+    camera_frames: dict[str, np.ndarray],
+    frame_indices: list[int],
+    *,
+    prompt: str,
+    session_id: str,
+) -> dict:
+    obs: dict = {}
+    for camera_key, all_frames in camera_frames.items():
+        selected = all_frames[frame_indices]
+        obs[camera_key] = selected[0] if len(frame_indices) == 1 else selected
+
+    obs["observation/joint_position"] = np.zeros(7, dtype=np.float32)
+    obs["observation/cartesian_position"] = np.zeros(6, dtype=np.float32)
+    obs["observation/gripper_position"] = np.zeros(1, dtype=np.float32)
+    obs["prompt"] = prompt
+    obs["session_id"] = session_id
+    return obs
+
+
+def _build_observations(video_dir: Path, prompt: str, session_id: str) -> tuple[dict[str, np.ndarray], list[dict]]:
+    camera_frames = _load_camera_frames(video_dir)
+    total_frames = min(frames.shape[0] for frames in camera_frames.values())
+    chunks = _build_frame_schedule(total_frames, 1)
+    observations = [
+        _make_obs_from_video(camera_frames, [0], prompt=prompt, session_id=session_id),
+    ]
+    if chunks:
+        observations.append(
+            _make_obs_from_video(
+                camera_frames,
+                chunks[0],
+                prompt=prompt,
+                session_id=session_id,
+            )
+        )
+    if len(observations) < 2:
+        raise RuntimeError("Need at least two DreamZero example observations to export a prediction video.")
+    return camera_frames, observations[:2]
+
+
+def _extract_latents(output: OmniRequestOutput) -> torch.Tensor:
+    if not isinstance(output, OmniRequestOutput):
+        raise TypeError(f"Expected OmniRequestOutput, got {type(output)!r}")
+    if not output.images:
+        raise RuntimeError("DreamZero output does not contain video latents in `images`.")
+
+    latents = output.images[0]
+    if not isinstance(latents, torch.Tensor):
+        raise TypeError(f"Expected tensor latents, got {type(latents)!r}")
+
+    latents = latents.detach().cpu()
+    if latents.dim() == 4:
+        latents = latents.unsqueeze(0)
+    if latents.dim() != 5:
+        raise ValueError(f"Unexpected latent shape: {tuple(latents.shape)}")
+
+    if latents.shape[1] < latents.shape[2]:
+        latents = latents.transpose(1, 2).contiguous()
+    return latents
+
+
+def _write_mp4(path: Path, frames: np.ndarray, fps: int) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    height, width = frames.shape[1:3]
+    writer = cv2.VideoWriter(
+        str(path),
+        cv2.VideoWriter_fourcc(*"mp4v"),
+        float(fps),
+        (width, height),
+    )
+    if not writer.isOpened():
+        raise RuntimeError(f"Failed to open video writer for {path}")
+    try:
+        for frame in frames:
+            writer.write(cv2.cvtColor(frame, cv2.COLOR_RGB2BGR))
+    finally:
+        writer.release()
+
+
+def _write_gif(path: Path, frames: np.ndarray, fps: int) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    images = [Image.fromarray(frame) for frame in frames]
+    duration_ms = max(int(round(1000 / max(fps, 1))), 1)
+    images[0].save(
+        path,
+        save_all=True,
+        append_images=images[1:],
+        duration=duration_ms,
+        loop=0,
+    )
+
+
+def _stitch_input_frames(camera_frames: dict[str, np.ndarray]) -> np.ndarray:
+    total_frames = min(frames.shape[0] for frames in camera_frames.values())
+    stitched = []
+    for frame_index in range(total_frames):
+        left = camera_frames["observation/exterior_image_0_left"][frame_index]
+        right = camera_frames["observation/exterior_image_1_left"][frame_index]
+        wrist = camera_frames["observation/wrist_image_left"][frame_index]
+        pad = np.zeros((left.shape[0], left.shape[1], 3), dtype=np.uint8)
+        canvas = np.concatenate([left, right], axis=1)
+        bottom = np.concatenate([wrist, pad], axis=1)
+        stitched.append(np.concatenate([canvas, bottom], axis=0))
+    return np.stack(stitched, axis=0)
+
+
+def _run_generation(
+    model: str, deploy_config_path: Path, observations: list[dict]
+) -> tuple[Omni, list[OmniRequestOutput]]:
+    omni = Omni(
+        model=model,
+        deploy_config=str(deploy_config_path),
+        enforce_eager=True,
+        worker_extension_cls=WORKER_EXTENSION,
+    )
+
+    outputs: list[OmniRequestOutput] = []
+    for index, obs in enumerate(observations):
+        sampling_params = OmniDiffusionSamplingParams(
+            extra_args={
+                "reset": index == 0,
+                "session_id": obs["session_id"],
+                "robot_obs": obs,
+            }
+        )
+        result = omni.generate(obs["prompt"], sampling_params_list=[sampling_params])
+        if not result:
+            raise RuntimeError(f"No output returned for DreamZero request {index}")
+        outputs.append(result[0])
+    return omni, outputs
+
+
+def _decode_with_worker(omni: Omni, full_latents: torch.Tensor) -> np.ndarray:
+    stage_client = omni.engine.stage_clients[0]
+    engine = getattr(stage_client, "_engine", None)
+    if engine is None:
+        raise RuntimeError("DreamZero export requires inline diffusion stage access.")
+
+    decoded = engine.executor.collective_rpc(
+        "decode_video_latents_to_uint8",
+        args=(full_latents,),
+        unique_reply_rank=0,
+        exec_all_ranks=True,
+    )
+    if isinstance(decoded, torch.Tensor):
+        decoded = decoded.numpy()
+    if not isinstance(decoded, np.ndarray):
+        raise TypeError(f"Unexpected decoded output type: {type(decoded)!r}")
+    return decoded
+
+
+def main() -> None:
+    args = _parse_args()
+    session_id = args.session_id or f"{DEFAULT_SESSION_PREFIX}-{uuid.uuid4()}"
+
+    camera_frames, observations = _build_observations(
+        video_dir=args.video_dir,
+        prompt=args.prompt,
+        session_id=session_id,
+    )
+
+    args.output_dir.mkdir(parents=True, exist_ok=True)
+
+    if args.save_input_video:
+        input_frames = _stitch_input_frames(camera_frames)
+        _write_mp4(args.output_dir / f"{args.output_stem}_input.mp4", input_frames, fps=15)
+        if args.save_gif:
+            _write_gif(args.output_dir / f"{args.output_stem}_input.gif", input_frames[::3], fps=5)
+
+    omni = None
+    try:
+        omni, outputs = _run_generation(
+            model=args.model,
+            deploy_config_path=args.deploy_config,
+            observations=observations,
+        )
+        latent_steps = [_extract_latents(output) for output in outputs]
+        full_latents = torch.cat(latent_steps, dim=2)
+        frames = _decode_with_worker(omni, full_latents)
+    finally:
+        if omni is not None:
+            omni.close()
+
+    mp4_path = args.output_dir / f"{args.output_stem}.mp4"
+
+    _write_mp4(mp4_path, frames, fps=args.fps)
+    print(f"SAVED_MP4={mp4_path}")
+
+    if args.save_gif:
+        gif_path = args.output_dir / f"{args.output_stem}.gif"
+        _write_gif(gif_path, frames, fps=args.fps)
+        print(f"SAVED_GIF={gif_path}")
+
+    if args.save_actions:
+        npz_path = args.output_dir / f"{args.output_stem}_actions.npz"
+        np.savez(
+            npz_path,
+            step0=np.asarray(outputs[0].multimodal_output.get("actions")),
+            step1=np.asarray(outputs[1].multimodal_output.get("actions")),
+        )
+        print(f"SAVED_ACTIONS={npz_path}")
+
+
+if __name__ == "__main__":
+    main()

examples/online_serving/dreamzero/README.md

@@ -0,0 +1,132 @@
+# DreamZero
+
+DreamZero is the robot-policy serving example for the OpenPI-compatible `/v1/realtime/robot/openpi` endpoint.
+
+## Supported checkpoints
+
+| Model | HuggingFace repo |
+|---|---|
+| DreamZero-DROID | `GEAR-Dreams/DreamZero-DROID` |
+
+## Quick start
+
+### Start the server
+
+```bash
+bash examples/online_serving/dreamzero/run_server.sh
+```
+
+By default this launches:
+
+```bash
+vllm serve GEAR-Dreams/DreamZero-DROID --omni --port 8000 \
+    --served-model-name dreamzero-droid \
+    --deploy-config vllm_omni/deploy/dreamzero_tp1_cfg2.yaml \
+    --enforce-eager --disable-log-stats
+```
+
+Override `MODEL`, `PORT`, `HOST`, `DEPLOY_CONFIG`, or `SERVED_MODEL_NAME` through the script environment if needed.
+
+### Install example dependencies
+
+The core `pip install -e .` setup does not include the extra packages used by the DreamZero example scripts.
+
+- `openpi_client.py`:
+  `openpi-client`, `websockets`, `opencv-python`
+- `droid_sim_eval_client.py`:
+  `mediapy`, `websockets`, `openpi-client`
+
+The DROID sim-eval script also needs an Isaac Lab environment that provides `isaaclab`, `isaaclab_tasks`, `sim_evals`, and `gymnasium`.
+
+If you run the DROID client on Python < 3.12, also install `typing-extensions`.
+
+### Configure TP and CFG parallelism
+
+The bundled DreamZero configs intentionally keep only:
+
+| Config | Purpose |
+|---|---|
+| `vllm_omni/deploy/dreamzero.yaml` | Default TP=1, CFG parallel disabled |
+| `vllm_omni/deploy/dreamzero_tp1_cfg2.yaml` | TP=1, CFG parallel size=2 |
+
+For other topologies, use CLI parallelism flags and update stage 0 `devices` with `--stage-overrides`. The number of listed devices must match `tensor_parallel_size * cfg_parallel_size`.
+
+TP=2 with CFG parallel disabled:
+
+```bash
+vllm serve GEAR-Dreams/DreamZero-DROID --omni --port 8000 \
+    --served-model-name dreamzero-droid \
+    --deploy-config vllm_omni/deploy/dreamzero.yaml \
+    --tensor-parallel-size 2 \
+    --stage-overrides '{"0": {"devices": "0,1"}}' \
+    --enforce-eager --disable-log-stats
+```
+
+TP=2 with CFG parallel size=2:
+
+```bash
+vllm serve GEAR-Dreams/DreamZero-DROID --omni --port 8000 \
+    --served-model-name dreamzero-droid \
+    --deploy-config vllm_omni/deploy/dreamzero.yaml \
+    --tensor-parallel-size 2 \
+    --cfg-parallel-size 2 \
+    --stage-overrides '{"0": {"devices": "0,1,2,3"}}' \
+    --enforce-eager --disable-log-stats
+```
+
+### Download example assets
+
+The OpenPI client and DROID sim-eval example expect the three camera MP4 files in `outputs/dreamzero/assets`.
+
+```bash
+hf download YangshenDeng/vllm-omni-dreamzero-assets --repo-type dataset --local-dir outputs/dreamzero/assets
+```
+
+### Run the OpenPI client
+
+```bash
+python examples/online_serving/dreamzero/openpi_client.py \
+    --host 127.0.0.1 \
+    --port 8000 \
+    --video-dir outputs/dreamzero/assets
+```
+
+This client uses downloaded example videos and talks to the OpenPI websocket server.
+
+### Run DROID sim eval
+
+```bash
+${ISAACLAB_LAUNCHER} -p examples/online_serving/dreamzero/droid_sim_eval_client.py \
+    --host 127.0.0.1 \
+    --port 8000 \
+    --scene 1 \
+    --episodes 1 \
+    --headless \
+    --device cuda:0
+```
+
+Set `ISAACLAB_LAUNCHER=path/to/isaaclab.sh` from the vLLM-Omni repository root before running the command.
+This launches Isaac Lab / sim-evals and runs the DROID benchmark loop against the same websocket endpoint.
+
+### Export comparison videos offline
+
+```bash
+python examples/offline_inference/dreamzero/export_prediction_video.py \
+    --deploy-config vllm_omni/deploy/dreamzero_tp1_cfg2.yaml \
+    --save-input-video \
+    --save-gif
+```
+
+The export script uses local `Omni` inference, not the websocket server. It writes the input rollout video and the predicted output video artifacts for side-by-side inspection.
+
+### MolmoSpace demo
+
+```bash
+python examples/online_serving/dreamzero/molmospace_dreamzero_eval_demo.py \
+    --host 127.0.0.1 \
+    --port 8000 \
+    --benchmark_dir /path/to/benchmark \
+    --output_dir /path/to/output
+```
+
+This demo adapts DreamZero to the MolmoSpace-style remote policy eval loop.

examples/online_serving/dreamzero/droid_sim_eval_client.py

@@ -0,0 +1,819 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+"""Run one or more DROID sim-eval rollouts against the vLLM DreamZero server.
+
+This script is the vLLM/OpenPI adaptation of the upstream DreamZero
+`eval_utils/run_sim_eval.py` sim-eval client.
+
+Behavior intentionally kept close to upstream:
+- same DROID observation extraction (`external_cam`, `external_cam_2`,
+  `wrist_cam`, joint position, gripper position)
+- same resize-with-pad preprocessing to `(180, 320)`
+- same `open_loop_horizon=8`
+- same gripper binarization rule (`> 0.5 -> 1`, else `0`)
+- same per-scene language prompts
+
+Unlike upstream DreamZero, vLLM serves the compatible websocket policy endpoint
+at `/v1/realtime/robot/openpi`, so this script includes the path suffix in the
+client URI.
+
+Run this script through Isaac Lab's launcher from the vLLM-Omni repository
+root, for example:
+
+    "${ISAACLAB_LAUNCHER}" -p \
+      examples/online_serving/dreamzero/droid_sim_eval_client.py \
+      --host 127.0.0.1 \
+      --port 8000 \
+      --scene 1 \
+      --episodes 1 \
+      --headless \
+      --device cuda:1
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import logging
+import sys
+import time
+import uuid
+from dataclasses import dataclass
+from datetime import UTC, datetime
+from pathlib import Path
+from typing import Any
+
+import numpy as np
+import torch
+
+try:
+    import mediapy
+except ImportError as exc:  # pragma: no cover - runtime dependency guard
+    raise ImportError("Optional DROID sim-eval client requires `mediapy`.") from exc
+
+try:
+    from typing import override
+except ImportError:
+    try:
+        from typing_extensions import override
+    except ImportError as exc:  # pragma: no cover - runtime dependency guard
+        raise ImportError("Optional DROID sim-eval client requires `typing-extensions` on Python < 3.12.") from exc
+
+try:
+    import websockets.sync.client
+except ImportError as exc:  # pragma: no cover - runtime dependency guard
+    raise ImportError("Optional DROID sim-eval client requires `websockets`.") from exc
+
+# NOTE:
+# This directory already contains a local file named `openpi_client.py`.
+# However, what we want here is the *installed* `openpi_client` package from
+# upstream OpenPI, not the sibling example file. When a script is executed
+# directly, Python often puts the script directory at `sys.path[0]`, which
+# would cause `import openpi_client` to resolve to the local example file and
+# create a circular import.
+#
+# To avoid that ambiguity, temporarily remove the current example directory
+# from the front of `sys.path`, import the real package, and then restore the
+# path afterwards.
+example_dir = str(Path(__file__).resolve().parent)
+removed_path = False
+if sys.path and sys.path[0] == example_dir:
+    sys.path.pop(0)
+    removed_path = True
+try:
+    from openpi_client import image_tools, msgpack_numpy
+    from openpi_client.base_policy import BasePolicy
+except ImportError as exc:  # pragma: no cover - runtime dependency guard
+    raise ImportError("Optional DROID sim-eval client requires the `openpi-client` package.") from exc
+finally:
+    if removed_path:
+        sys.path.insert(0, example_dir)
+
+# -----------------------------------------------------------------------------
+# Constants
+# -----------------------------------------------------------------------------
+#
+# These values intentionally mirror the upstream DreamZero sim-eval client
+# where possible. The important distinction is:
+#
+# - ACTION_HORIZON = 24
+#     The model returns 24 future actions per inference call.
+# - DEFAULT_OPEN_LOOP_HORIZON = 8
+#     The sim client only executes the first 8 actions locally before asking
+#     the server to replan from a fresh observation.
+#
+# So a single server call predicts 24x8 actions, but the rollout consumes only
+# 8 of them before replanning.
+PING_INTERVAL_SECS = 60
+PING_TIMEOUT_SECS = 600
+DEFAULT_PATH = "/v1/realtime/robot/openpi"
+DEFAULT_OPEN_LOOP_HORIZON = 8
+ACTION_HORIZON = 24
+ACTION_DIM = 8
+DEFAULT_OUTPUT_ROOT = Path("runs") / "dreamzero_sim_eval"
+SCENE_PROMPTS = {
+    1: "put the cube in the bowl",
+    2: "pick up the can and put it in the mug",
+    3: "put the banana in the bin",
+}
+
+
+def _decode_action_response(response: bytes | str) -> np.ndarray:
+    if isinstance(response, str):
+        raise RuntimeError(f"Error in inference server:\n{response}")
+    decoded = msgpack_numpy.unpackb(response)
+    if isinstance(decoded, dict) and decoded.get("type") == "error":
+        message = decoded.get("message", decoded)
+        raise RuntimeError(f"Error in inference server:\n{message}")
+    return np.asarray(decoded, dtype=np.float32)
+
+
+@dataclass(frozen=True)
+class StepRecord:
+    """One fully materialized rollout step for later JSON export.
+
+    The `episode_00.json` artifact is intended to be human-readable and
+    post-process-friendly. Instead of keeping raw tensors around, each step is
+    flattened into plain Python types so it can be serialized directly.
+    """
+
+    # Index of this control step within the episode.
+    step_index: int
+    # Whether this step triggered a fresh model call (as opposed to reusing the
+    # cached open-loop chunk from the previous server response).
+    used_server_call: bool
+    # End-to-end latency of the server call that produced the current chunk.
+    # This is `None` on steps that only reuse cached actions.
+    chunk_latency_s: float | None
+    # The concrete 8-D action sent into the simulator at this step.
+    action: list[float]
+    # Observed 7-DoF arm joint positions before the next environment step.
+    joint_position: list[float]
+    # Observed gripper scalar before the next environment step.
+    gripper_position: list[float]
+    # Reward and termination signals directly returned by the simulator.
+    reward: float
+    terminated: bool
+    truncated: bool
+    # Optional scene object positions for downstream debugging / success
+    # heuristics. This may be empty if the environment does not expose them.
+    scene_objects: dict[str, list[float]]
+
+
+class OpenPIWebsocketClientPolicy(BasePolicy):
+    """Minimal websocket client for the DreamZero/OpenPI policy protocol.
+
+    Protocol shape:
+    - connect -> server immediately sends a metadata dict
+    - infer   -> send msgpack observation, receive action chunk
+    - reset   -> send msgpack reset command, receive confirmation string
+
+    This class intentionally stays very small because the more interesting
+    DreamZero-specific behavior lives one layer above, in
+    `DreamZeroJointPosClient`.
+    """
+
+    def __init__(
+        self,
+        host: str = "127.0.0.1",
+        port: int = 8000,
+        path: str = DEFAULT_PATH,
+    ) -> None:
+        # vLLM serves the robot endpoint under `/v1/realtime/robot/openpi`.
+        self._uri = f"ws://{host}:{port}{path}"
+        # Upstream protocol uses msgpack with numpy support, not JSON.
+        self._packer = msgpack_numpy.Packer()
+        # Connect immediately and cache the server handshake metadata.
+        self._ws, self._server_metadata = self._wait_for_server()
+
+    def get_server_metadata(self) -> dict[str, Any]:
+        """Return a copy of the server handshake metadata."""
+
+        return dict(self._server_metadata)
+
+    def _wait_for_server(self):
+        """Connect to the websocket server and read the initial metadata frame."""
+
+        logging.info("Connecting to %s", self._uri)
+        conn = websockets.sync.client.connect(
+            self._uri,
+            compression=None,
+            max_size=None,
+            ping_interval=PING_INTERVAL_SECS,
+            ping_timeout=PING_TIMEOUT_SECS,
+        )
+        metadata = msgpack_numpy.unpackb(conn.recv())
+        if not isinstance(metadata, dict):
+            raise TypeError(f"Expected dict metadata from server, got {type(metadata)!r}")
+        return conn, metadata
+
+    @override
+    def infer(self, obs: dict[str, Any]) -> np.ndarray:
+        """Send an inference request and return the decoded action chunk."""
+
+        # Keep the upstream DreamZero/OpenPI convention that the request itself
+        # tells the server which logical endpoint is being called.
+        payload = dict(obs)
+        payload["endpoint"] = "infer"
+        self._ws.send(self._packer.pack(payload))
+        response = self._ws.recv()
+        return _decode_action_response(response)
+
+    @override
+    def reset(self, reset_info: dict[str, Any] | None = None) -> str:
+        """Tell the server to reset its session-side state."""
+
+        payload = dict(reset_info or {})
+        payload["endpoint"] = "reset"
+        self._ws.send(self._packer.pack(payload))
+        response = self._ws.recv()
+        if isinstance(response, str):
+            return response
+        decoded = msgpack_numpy.unpackb(response)
+        if not isinstance(decoded, dict) or decoded.get("status") != "reset successful":
+            raise RuntimeError(f"Unexpected reset response: {decoded!r}")
+        return str(decoded["status"])
+
+    def close(self) -> None:
+        """Close the websocket connection explicitly."""
+
+        self._ws.close()
+
+
+class DreamZeroJointPosClient:
+    """DROID sim-eval client that talks to the vLLM OpenPI websocket server.
+
+    This is the main compatibility layer between:
+    - Isaac Lab DROID observations (`obs["policy"][...]`)
+    - DreamZero/OpenPI websocket payloads
+    - local open-loop action reuse across several simulator steps
+
+    In other words:
+        simulator obs -> websocket request -> action chunk -> one action per step
+    """
+
+    def __init__(
+        self,
+        remote_host: str = "127.0.0.1",
+        remote_port: int = 8000,
+        path: str = DEFAULT_PATH,
+        open_loop_horizon: int = DEFAULT_OPEN_LOOP_HORIZON,
+    ) -> None:
+        # Low-level transport client.
+        self.client = OpenPIWebsocketClientPolicy(remote_host, remote_port, path=path)
+        # Number of actions to execute locally before replanning.
+        self.open_loop_horizon = open_loop_horizon
+        # Cursor into the currently cached action chunk.
+        self.actions_from_chunk_completed = 0
+        # Most recent `(ACTION_HORIZON, ACTION_DIM)` server response.
+        self.pred_action_chunk: np.ndarray | None = None
+        # Session id is part of the DreamZero serving contract. Changing it
+        # causes the server side to treat the rollout as a fresh episode.
+        self.session_id = str(uuid.uuid4())
+        # Simple runtime stats for reporting.
+        self.server_calls = 0
+        self.last_chunk_latency_s: float | None = None
+        self.last_used_server_call = False
+
+    def metadata(self) -> dict[str, Any]:
+        """Expose the server metadata to callers / logs."""
+
+        return self.client.get_server_metadata()
+
+    def reset(self) -> str:
+        """Reset local chunk state and remote session state.
+
+        Local reset:
+        - drop cached action chunk
+        - rewind chunk cursor
+        - allocate a fresh session id
+
+        Remote reset:
+        - send a websocket `reset` message so the server can clear any
+          request/session-side state it associates with this client
+        """
+
+        self.actions_from_chunk_completed = 0
+        self.pred_action_chunk = None
+        self.session_id = str(uuid.uuid4())
+        self.last_chunk_latency_s = None
+        self.last_used_server_call = False
+        return self.client.reset({})
+
+    def infer(self, obs: dict[str, Any], instruction: str) -> dict[str, Any]:
+        """Turn one simulator observation into one executable 8-D action.
+
+        Key behavior:
+        - call the server only when the local chunk cache is empty/exhausted
+        - otherwise, keep consuming the cached chunk open-loop
+        - always return exactly one 8-D action for the current simulator step
+        """
+
+        # Convert Isaac Lab observation structure into a plain numpy-friendly
+        # record that is easier to serialize and visualize.
+        curr_obs = self._extract_observation(obs)
+        self.last_used_server_call = False
+
+        # Replan if:
+        # 1. this is the first step of a rollout / chunk
+        # 2. we already consumed `open_loop_horizon` actions from the current chunk
+        # 3. no cached chunk is currently available
+        if (
+            self.actions_from_chunk_completed == 0
+            or self.actions_from_chunk_completed >= self.open_loop_horizon
+            or self.pred_action_chunk is None
+        ):
+            self.actions_from_chunk_completed = 0
+            # Build the exact DreamZero/OpenPI payload expected by the server.
+            #
+            # Notes:
+            # - images are resized/padded to the serving contract's 180x320
+            # - proprio is cast to float64 to match upstream client behavior
+            # - cartesian_position is currently unused by DreamZero DROID, so
+            #   a dummy zero vector is sent for protocol completeness
+            request_data = {
+                "observation/exterior_image_0_left": image_tools.resize_with_pad(curr_obs["right_image"], 180, 320),
+                "observation/exterior_image_1_left": image_tools.resize_with_pad(curr_obs["left_image"], 180, 320),
+                "observation/wrist_image_left": image_tools.resize_with_pad(curr_obs["wrist_image"], 180, 320),
+                "observation/joint_position": curr_obs["joint_position"].astype(np.float64),
+                "observation/cartesian_position": np.zeros((6,), dtype=np.float64),
+                "observation/gripper_position": curr_obs["gripper_position"].astype(np.float64),
+                "prompt": instruction,
+                "session_id": self.session_id,
+            }
+
+            # Measure end-to-end server latency for this chunk request.
+            start = time.perf_counter()
+            actions = self.client.infer(request_data)
+            self.last_chunk_latency_s = time.perf_counter() - start
+            self.last_used_server_call = True
+            self.server_calls += 1
+
+            # DreamZero DROID serving is expected to return an action chunk with
+            # 24 future actions, each action being 8-D.
+            if actions.ndim != 2:
+                raise AssertionError(f"Expected 2D action array, got shape {actions.shape}")
+            if actions.shape != (ACTION_HORIZON, ACTION_DIM):
+                raise AssertionError(f"Expected action shape {(ACTION_HORIZON, ACTION_DIM)}, got {actions.shape}")
+            self.pred_action_chunk = actions
+
+        # Consume exactly one action row from the cached chunk for this
+        # simulator step.
+        action = np.array(self.pred_action_chunk[self.actions_from_chunk_completed], copy=True)
+        self.actions_from_chunk_completed += 1
+
+        # Upstream DreamZero sim-eval binarizes the gripper command.
+        action[-1] = 1.0 if action[-1].item() > 0.5 else 0.0
+
+        # Produce a human-friendly visualization strip for videos:
+        # right external | wrist | left external
+        img1 = image_tools.resize_with_pad(curr_obs["right_image"], 224, 224)
+        img2 = image_tools.resize_with_pad(curr_obs["wrist_image"], 224, 224)
+        img3 = image_tools.resize_with_pad(curr_obs["left_image"], 224, 224)
+        viz = np.concatenate([img1, img2, img3], axis=1)
+
+        # Return both the executable action and auxiliary debug info.
+        return {
+            "action": action,
+            "viz": viz,
+            "joint_position": curr_obs["joint_position"],
+            "gripper_position": curr_obs["gripper_position"],
+            "used_server_call": self.last_used_server_call,
+            "chunk_latency_s": self.last_chunk_latency_s if self.last_used_server_call else None,
+        }
+
+    @staticmethod
+    def _extract_observation(obs_dict: dict[str, Any]) -> dict[str, np.ndarray]:
+        """Extract the pieces DreamZero cares about from Isaac Lab observations.
+
+        `sim-evals` exposes camera frames and robot state inside the
+        `obs["policy"]` group. This helper converts those tensors into numpy
+        arrays so they can be fed into image preprocessing / websocket packing.
+        """
+
+        policy = obs_dict["policy"]
+        # Isaac Lab stores camera observations as batched tensors; use env 0.
+        right_image = policy["external_cam"][0].clone().detach().cpu().numpy()
+        left_image = policy["external_cam_2"][0].clone().detach().cpu().numpy()
+        wrist_image = policy["wrist_cam"][0].clone().detach().cpu().numpy()
+        # Robot proprioception.
+        joint_position = policy["arm_joint_pos"].clone().detach().cpu().numpy()
+        gripper_position = policy["gripper_pos"].clone().detach().cpu().numpy()
+
+        return {
+            "right_image": right_image,
+            "left_image": left_image,
+            "wrist_image": wrist_image,
+            "joint_position": joint_position,
+            "gripper_position": gripper_position,
+        }
+
+
+def _scene_instruction(scene: int) -> str:
+    """Map a numeric scene id onto the fixed language prompt used for rollout."""
+
+    try:
+        return SCENE_PROMPTS[scene]
+    except KeyError as exc:
+        raise ValueError(f"Unsupported scene {scene}. Available scenes: {sorted(SCENE_PROMPTS)}") from exc
+
+
+def _capture_scene_objects(env: Any) -> dict[str, list[float]]:
+    """Best-effort extraction of scene object root positions.
+
+    This is only for debugging / reporting. The rollout logic itself does not
+    depend on these positions.
+    """
+
+    objects: dict[str, list[float]] = {}
+    scene = getattr(env, "scene", None)
+    if scene is None:
+        return objects
+
+    # Skip non-task entities such as cameras, the robot, and lighting.
+    for name in scene.keys():
+        if name in {"robot", "external_cam", "external_cam_2", "wrist_cam", "sphere_light", "scene"}:
+            continue
+        entity = scene[name]
+        data = getattr(entity, "data", None)
+        root_pos_w = getattr(data, "root_pos_w", None)
+        if root_pos_w is None:
+            continue
+        # Convert tensors to plain lists for JSON serialization.
+        value = root_pos_w[0].detach().cpu().to(torch.float32).tolist()
+        objects[str(name)] = [float(x) for x in value]
+    return objects
+
+
+def _maybe_infer_success(scene: int, final_objects: dict[str, list[float]]) -> dict[str, Any]:
+    """Best-effort geometric heuristic.
+
+    The simulator itself does not expose a built-in success term; this function
+    provides a transparent fallback for human-readable reporting only.
+    """
+
+    task_pairs = {
+        1: ("cube", "bowl"),
+        2: ("can", "mug"),
+        3: ("banana", "bin"),
+    }
+    source_name, target_name = task_pairs.get(scene, (None, None))
+    if source_name not in final_objects or target_name not in final_objects:
+        return {
+            "has_builtin_success": False,
+            "heuristic_success": None,
+            "reason": "scene object names unavailable for heuristic",
+        }
+
+    source = np.asarray(final_objects[source_name], dtype=np.float32)
+    target = np.asarray(final_objects[target_name], dtype=np.float32)
+    xy_distance = float(np.linalg.norm(source[:2] - target[:2]))
+    z_delta = float(source[2] - target[2])
+    heuristic_success = bool(xy_distance < 0.12 and z_delta > -0.08)
+    return {
+        "has_builtin_success": False,
+        "heuristic_success": heuristic_success,
+        "xy_distance": xy_distance,
+        "z_delta": z_delta,
+        "source_object": source_name,
+        "target_object": target_name,
+        "reason": "no built-in env success flag; using final object pose heuristic",
+    }
+
+
+def _dump_json(path: Path, payload: dict[str, Any]) -> None:
+    """Write a JSON file with stable UTF-8 formatting."""
+
+    path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
+
+
+def _scalar_from_env_value(value: Any) -> float:
+    """Normalize simulator scalar outputs into a plain Python float.
+
+    Isaac Lab / Gym values may come back as tensors, numpy arrays, tuples, or
+    direct Python scalars depending on the wrapper stack. Centralizing the
+    conversion here makes the rollout loop cleaner and more robust.
+    """
+
+    if isinstance(value, torch.Tensor):
+        return float(value.reshape(-1)[0].detach().cpu().item())
+    if isinstance(value, np.ndarray):
+        return float(value.reshape(-1)[0])
+    if isinstance(value, (list, tuple)):
+        return float(np.asarray(value).reshape(-1)[0])
+    return float(value)
+
+
+def _bool_from_env_value(value: Any) -> bool:
+    """Normalize simulator boolean-like outputs into a plain Python bool."""
+
+    if isinstance(value, torch.Tensor):
+        return bool(value.reshape(-1)[0].detach().cpu().item())
+    if isinstance(value, np.ndarray):
+        return bool(value.reshape(-1)[0])
+    if isinstance(value, (list, tuple)):
+        return bool(np.asarray(value).reshape(-1)[0])
+    return bool(value)
+
+
+def _make_output_dir(output_root: Path, scene: int) -> Path:
+    """Create a timestamped output directory for one sim-eval run."""
+
+    timestamp = datetime.now(UTC).strftime("%Y%m%dT%H%M%SZ")
+    output_dir = output_root / f"scene{scene}_{timestamp}"
+    output_dir.mkdir(parents=True, exist_ok=True)
+    return output_dir
+
+
+def main() -> None:
+    """Entry point for one or more DROID simulation rollouts.
+
+    High-level flow:
+    1. parse command-line flags
+    2. bootstrap Isaac Lab / sim-evals imports
+    3. create the DROID environment
+    4. connect the DreamZero websocket client
+    5. run `episodes` rollouts
+    6. export videos + JSON summaries
+    """
+
+    # Script-level arguments.
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--episodes", type=int, default=1, help="Number of episodes to run.")
+    parser.add_argument("--scene", type=int, default=1, help="DROID scene id (1/2/3).")
+    parser.add_argument("--host", type=str, default="127.0.0.1", help="vLLM DreamZero server host.")
+    parser.add_argument("--port", type=int, default=8000, help="vLLM DreamZero server port.")
+    parser.add_argument("--path", type=str, default=DEFAULT_PATH, help="Websocket path suffix.")
+    parser.add_argument(
+        "--open-loop-horizon",
+        type=int,
+        default=DEFAULT_OPEN_LOOP_HORIZON,
+        help="How many actions to consume locally before requesting the next chunk.",
+    )
+    parser.add_argument(
+        "--output-root",
+        type=Path,
+        default=DEFAULT_OUTPUT_ROOT,
+        help="Directory where videos and trajectory logs are stored.",
+    )
+
+    try:
+        from isaaclab.app import AppLauncher
+    except ImportError as exc:  # pragma: no cover - runtime dependency guard
+        raise ImportError(
+            "Optional DROID sim-eval client requires Isaac Lab (`isaaclab`). "
+            "Launch it from an Isaac Lab environment, e.g. via `isaaclab.sh -p`."
+        ) from exc
+
+    # Let Isaac Lab inject its own runtime flags (e.g. headless, device).
+    AppLauncher.add_app_launcher_args(parser)
+    args = parser.parse_args()
+
+    # DreamZero sim-eval always needs camera observations enabled.
+    args.enable_cameras = True
+    # Boot Isaac Sim / Isaac Lab.
+    app_launcher = AppLauncher(args)
+    simulation_app = app_launcher.app
+    # Set defaults so the `finally` block can clean up safely even if an
+    # earlier step fails.
+    env = None
+    client = None
+
+    # Import simulator modules only *after* the app is launched. This matches
+    # Isaac Lab's required import ordering.
+    try:
+        import gymnasium as gym
+    except ImportError as exc:  # pragma: no cover - runtime dependency guard
+        raise ImportError("Optional DROID sim-eval client requires `gymnasium`.") from exc
+
+    try:
+        import sim_evals.environments  # noqa: F401
+    except ImportError as exc:  # pragma: no cover - runtime dependency guard
+        raise ImportError(
+            "Optional DROID sim-eval client requires the external `sim-evals` package or checkout to be importable."
+        ) from exc
+
+    try:
+        from isaaclab_tasks.utils import parse_env_cfg
+    except ImportError as exc:  # pragma: no cover - runtime dependency guard
+        raise ImportError("Optional DROID sim-eval client requires `isaaclab_tasks`.") from exc
+
+    # Resolve output location and scene prompt.
+    output_dir = _make_output_dir(args.output_root.expanduser().resolve(), args.scene)
+    instruction = _scene_instruction(args.scene)
+
+    # Build the DROID environment configuration from `sim-evals`.
+    env_cfg = parse_env_cfg(
+        "DROID",
+        device=args.device,
+        num_envs=1,
+        use_fabric=True,
+    )
+    # Select one of the pre-authored scenes/tasks.
+    env_cfg.set_scene(args.scene)
+    env = gym.make("DROID", cfg=env_cfg)
+
+    # Upstream sim-evals resets twice so materials / cameras are fully ready.
+    obs, _ = env.reset()
+    obs, _ = env.reset()
+
+    # Connect the websocket policy client.
+    client = DreamZeroJointPosClient(
+        remote_host=args.host,
+        remote_port=args.port,
+        path=args.path,
+        open_loop_horizon=args.open_loop_horizon,
+    )
+
+    # Aggregated per-run results.
+    all_episode_summaries: list[dict[str, Any]] = []
+    max_steps = int(env.env.max_episode_length)
+    logging.info("DreamZero metadata: %s", client.metadata())
+    logging.info("Scene %s prompt: %s", args.scene, instruction)
+    logging.info("Writing outputs to %s", output_dir)
+
+    try:
+        # No gradients are needed in inference-only rollout mode.
+        with torch.no_grad():
+            for episode_index in range(args.episodes):
+                # Per-episode collectors.
+                frames: list[np.ndarray] = []
+                step_records: list[StepRecord] = []
+                episode_start = time.perf_counter()
+                server_time_s = 0.0
+                final_reward = 0.0
+                terminated = False
+                truncated = False
+
+                for step_index in range(max_steps):
+                    # Ask the policy for the next action. Internally this may or
+                    # may not trigger a real server request depending on whether
+                    # the local chunk cache has been exhausted.
+                    logging.debug("Episode %d step %d: requesting action", episode_index, step_index)
+                    result = client.infer(obs, instruction)
+                    logging.debug(
+                        "Episode %d step %d: got action (server_call=%s latency=%s)",
+                        episode_index,
+                        step_index,
+                        result["used_server_call"],
+                        result["chunk_latency_s"],
+                    )
+                    # Save one visualization frame per simulator step.
+                    frames.append(result["viz"])
+
+                    # Isaac Lab expects batched actions, hence `[None]`.
+                    action_tensor = torch.tensor(result["action"], dtype=torch.float32)[None]
+                    logging.debug("Episode %d step %d: stepping env", episode_index, step_index)
+                    obs, reward, term, trunc, info = env.step(action_tensor)
+                    logging.debug("Episode %d step %d: env.step returned", episode_index, step_index)
+                    logging.debug("Episode %d step %d: parsing reward/flags", episode_index, step_index)
+                    logging.debug(
+                        "Episode %d step %d: raw types reward=%s term=%s trunc=%s",
+                        episode_index,
+                        step_index,
+                        type(reward).__name__,
+                        type(term).__name__,
+                        type(trunc).__name__,
+                    )
+                    # Normalize environment outputs into plain Python scalars so
+                    # the rest of the code does not depend on wrapper-specific types.
+                    reward_value = _scalar_from_env_value(reward)
+                    term_value = _bool_from_env_value(term)
+                    trunc_value = _bool_from_env_value(trunc)
+                    logging.debug(
+                        "Episode %d step %d: parsed reward=%s term=%s trunc=%s",
+                        episode_index,
+                        step_index,
+                        reward_value,
+                        term_value,
+                        trunc_value,
+                    )
+                    # Keep scene-object capture optional. It is useful for
+                    # debugging / success heuristics, but the rollout should not
+                    # fail if the environment does not expose object roots.
+                    scene_objects = _capture_scene_objects(env)
+
+                    # Accumulate total server-side time only on steps that
+                    # triggered a fresh chunk inference.
+                    if result["chunk_latency_s"] is not None:
+                        server_time_s += float(result["chunk_latency_s"])
+
+                    # Materialize one JSON-serializable trajectory record.
+                    logging.debug("Episode %d step %d: appending trajectory", episode_index, step_index)
+                    step_records.append(
+                        StepRecord(
+                            step_index=step_index,
+                            used_server_call=bool(result["used_server_call"]),
+                            chunk_latency_s=(
+                                float(result["chunk_latency_s"]) if result["chunk_latency_s"] is not None else None
+                            ),
+                            action=[float(x) for x in np.asarray(result["action"], dtype=np.float32).tolist()],
+                            joint_position=[
+                                float(x) for x in np.asarray(result["joint_position"], dtype=np.float32).tolist()
+                            ],
+                            gripper_position=[
+                                float(x) for x in np.asarray(result["gripper_position"], dtype=np.float32).tolist()
+                            ],
+                            reward=reward_value,
+                            terminated=term_value,
+                            truncated=trunc_value,
+                            scene_objects=scene_objects,
+                        )
+                    )
+                    logging.debug("Episode %d step %d: trajectory appended", episode_index, step_index)
+
+                    # Track final status for summary export.
+                    final_reward = reward_value
+                    terminated = term_value
+                    truncated = trunc_value
+                    if term_value or trunc_value:
+                        # End the rollout early if the environment terminates.
+                        break
+
+                # Episode-level timing and video export.
+                episode_wall_time_s = time.perf_counter() - episode_start
+                video_path = output_dir / f"episode_{episode_index:02d}.mp4"
+                logging.info("Episode %d: writing video to %s", episode_index, video_path)
+                mediapy.write_video(video_path, frames, fps=15)
+
+                # Reset the policy server between episodes.
+                logging.info("Episode %d: sending reset", episode_index)
+                reset_response = client.reset()
+                final_objects = step_records[-1].scene_objects if step_records else {}
+                success_report = _maybe_infer_success(args.scene, final_objects)
+
+                # Assemble the per-episode summary that is written to
+                # `episode_XX.json`.
+                episode_summary = {
+                    "episode_index": episode_index,
+                    "prompt": instruction,
+                    "video_path": str(video_path),
+                    "steps_executed": len(step_records),
+                    "max_steps": max_steps,
+                    "terminated": terminated,
+                    "truncated": truncated,
+                    "final_reward": final_reward,
+                    "server_calls": client.server_calls,
+                    "server_time_s": server_time_s,
+                    "episode_wall_time_s": episode_wall_time_s,
+                    "avg_server_time_per_call_s": (
+                        server_time_s / client.server_calls if client.server_calls else None
+                    ),
+                    "reset_response": reset_response,
+                    "success_report": success_report,
+                    "server_metadata": client.metadata(),
+                    "trajectory": [record.__dict__ for record in step_records],
+                }
+                _dump_json(output_dir / f"episode_{episode_index:02d}.json", episode_summary)
+                all_episode_summaries.append(episode_summary)
+
+                logging.info(
+                    "Episode %d done: steps=%d wall=%.2fs server_calls=%d heuristic_success=%s",
+                    episode_index,
+                    len(step_records),
+                    episode_wall_time_s,
+                    client.server_calls,
+                    success_report.get("heuristic_success"),
+                )
+
+                # Reset per-episode counters while keeping the client alive.
+                client.server_calls = 0
+
+        # Top-level run summary across all episodes.
+        summary = {
+            "scene": args.scene,
+            "prompt": instruction,
+            "episodes": args.episodes,
+            "host": args.host,
+            "port": args.port,
+            "path": args.path,
+            "device": args.device,
+            "output_dir": str(output_dir),
+            "server_metadata": client.metadata(),
+            "episodes_summary": all_episode_summaries,
+        }
+        _dump_json(output_dir / "summary.json", summary)
+        # Also print the summary to stdout so the caller can capture it in logs.
+        print(json.dumps(summary, ensure_ascii=False, indent=2))
+    finally:
+        # Best-effort cleanup. Avoid masking the main error if cleanup fails.
+        try:
+            if client is not None:
+                client.client.close()
+        except Exception:
+            pass
+        if env is not None:
+            env.close()
+        simulation_app.close()
+
+
+if __name__ == "__main__":
+    # Keep the script-level logs readable. Per-step rollout details are still
+    # available via `DEBUG` if needed, but websocket / asyncio internals are
+    # usually too noisy for normal usage.
+    logging.basicConfig(level=logging.INFO)
+    logging.getLogger("websockets").setLevel(logging.WARNING)
+    logging.getLogger("asyncio").setLevel(logging.WARNING)
+    main()

examples/online_serving/dreamzero/molmospace_dreamzero_eval_demo.py

@@ -0,0 +1,86 @@
+from __future__ import annotations
+
+import argparse
+import os
+import sys
+from pathlib import Path
+
+os.environ.setdefault("MUJOCO_GL", "egl")
+os.environ.setdefault("PYOPENGL_PLATFORM", "egl")
+
+# Import base configs at module top level so the subclasses below are pickle-
+# resolvable (worker processes import this module fresh via __main__).
+from molmo_spaces.configs.policy_configs_baselines import (  # noqa: E402
+    DreamZeroPolicyConfig,
+)
+from molmo_spaces.evaluation.configs.evaluation_configs import (  # noqa: E402
+    DreamZeroPolicyEvalConfig,
+)
+
+
+# We only need to change the backend host and port to the vllm-host!
+class DreamZeroVllmOmniPolicyConfig(DreamZeroPolicyConfig):
+    remote_config: dict = dict(host="127.0.0.1", port=8000)
+
+
+class DreamZeroVllmOmniEvalConfig(DreamZeroPolicyEvalConfig):
+    policy_config: DreamZeroVllmOmniPolicyConfig = DreamZeroVllmOmniPolicyConfig()
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--host", default="127.0.0.1")
+    parser.add_argument("--port", type=int, default=8000)
+    parser.add_argument(
+        "--benchmark_dir",
+        required=True,
+        help=(
+            "Path to a MolmoSpaces benchmark directory, for example "
+            "$MOLMOSPACES_BENCHMARK_DIR/20260327/ithor/FrankaCloseHardBench/"
+            "FrankaCloseHardBench_20260206_json_benchmark"
+        ),
+    )
+    parser.add_argument("--max_episodes", type=int, default=1)
+    parser.add_argument("--task_horizon_steps", type=int, default=80)
+    parser.add_argument(
+        "--output_dir",
+        required=True,
+        help="Directory to write evaluation outputs (created if missing).",
+    )
+    parser.add_argument("--episode_idx", type=int, default=None)
+    args = parser.parse_args()
+
+    policy_config = DreamZeroVllmOmniPolicyConfig(remote_config=dict(host=args.host, port=args.port))
+    DreamZeroVllmOmniPolicyConfig.model_fields["remote_config"].default = policy_config.remote_config
+    DreamZeroVllmOmniEvalConfig.model_fields["policy_config"].default = policy_config
+
+    # Import after env vars are set so MuJoCo picks EGL.
+    from molmo_spaces.evaluation import run_evaluation
+
+    cfg_cls = DreamZeroVllmOmniEvalConfig
+
+    output_dir = args.output_dir
+    Path(output_dir).mkdir(parents=True, exist_ok=True)
+
+    print(f"[eval] benchmark_dir={args.benchmark_dir}")
+    print(f"[eval] max_episodes={args.max_episodes} task_horizon_steps={args.task_horizon_steps}")
+    print(f"[eval] remote policy: ws://{args.host}:{args.port}/v1/realtime/robot/openpi")
+
+    results = run_evaluation(
+        eval_config_cls=cfg_cls,
+        benchmark_dir=Path(args.benchmark_dir),
+        max_episodes=args.max_episodes,
+        task_horizon_steps=args.task_horizon_steps,
+        num_workers=1,
+        use_wandb=False,
+        output_dir=output_dir,
+        episode_idx=args.episode_idx,
+    )
+
+    print(f"[eval] success={results.success_count}/{results.total_count} ({results.success_rate:.1%})")
+    print(f"[eval] output_dir={results.output_dir}")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

examples/online_serving/dreamzero/openpi_client.py

@@ -0,0 +1,311 @@
+#!/usr/bin/env python3
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+from __future__ import annotations
+
+import argparse
+import json
+import logging
+import sys
+import uuid
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+import numpy as np
+
+try:
+    import cv2
+except ImportError as exc:  # pragma: no cover - runtime dependency guard
+    raise ImportError("DreamZero OpenPI example requires `opencv-python`.") from exc
+
+try:
+    example_dir = str(Path(__file__).resolve().parent)
+    removed_path = False
+    if sys.path and sys.path[0] == example_dir:
+        sys.path.pop(0)
+        removed_path = True
+    try:
+        from openpi_client.websocket_client_policy import WebsocketClientPolicy
+    finally:
+        if removed_path:
+            sys.path.insert(0, example_dir)
+except ImportError as exc:  # pragma: no cover - runtime dependency guard
+    raise ImportError("DreamZero OpenPI example requires `openpi-client`.") from exc
+
+DEFAULT_HOST = "127.0.0.1"
+DEFAULT_PORT = 8000
+DEFAULT_PATH = "/v1/realtime/robot/openpi"
+DEFAULT_PROMPT = "Move the pan forward and use the brush in the middle of the plates to brush the inside of the pan"
+ACTION_HORIZON = 24
+DEFAULT_ACTION_DIM = 8
+RELATIVE_OFFSETS = [-23, -16, -8, 0]
+REPO_ROOT = Path(__file__).resolve().parents[3]
+ASSET_REPO_ID = "YangshenDeng/vllm-omni-dreamzero-assets"
+DEFAULT_VIDEO_DIR = REPO_ROOT / "outputs" / "dreamzero" / "assets"
+CAMERA_FILES = {
+    "observation/exterior_image_0_left": "exterior_image_1_left.mp4",
+    "observation/exterior_image_1_left": "exterior_image_2_left.mp4",
+    "observation/wrist_image_left": "wrist_image_left.mp4",
+}
+
+
+def _as_action_array(response: Any) -> np.ndarray:
+    if isinstance(response, dict) and response.get("type") == "error":
+        message = response.get("message", response)
+        raise RuntimeError(f"Inference failed: {message}")
+    return np.asarray(response, dtype=np.float32)
+
+
+@dataclass(frozen=True)
+class DreamZeroServerMetadata:
+    image_resolution: tuple[int, int]
+    n_external_cameras: int
+    needs_wrist_camera: bool
+    needs_stereo_camera: bool
+    needs_session_id: bool
+    action_space: str
+
+    @classmethod
+    def from_dict(cls, payload: dict[str, Any]) -> DreamZeroServerMetadata:
+        required_keys = (
+            "image_resolution",
+            "n_external_cameras",
+            "needs_wrist_camera",
+            "needs_stereo_camera",
+            "needs_session_id",
+            "action_space",
+        )
+        missing_keys = [key for key in required_keys if key not in payload]
+        if missing_keys:
+            raise ValueError(f"Missing DreamZero metadata keys: {missing_keys}")
+
+        image_resolution = payload["image_resolution"]
+        if not isinstance(image_resolution, (list, tuple)) or len(image_resolution) != 2:
+            raise ValueError(f"Invalid image_resolution: {image_resolution!r}")
+
+        return cls(
+            image_resolution=(int(image_resolution[0]), int(image_resolution[1])),
+            n_external_cameras=int(payload["n_external_cameras"]),
+            needs_wrist_camera=bool(payload["needs_wrist_camera"]),
+            needs_stereo_camera=bool(payload["needs_stereo_camera"]),
+            needs_session_id=bool(payload["needs_session_id"]),
+            action_space=str(payload["action_space"]),
+        )
+
+
+def _openpi_uri(host: str, port: int, path: str) -> str:
+    if host.startswith(("ws://", "wss://")):
+        return host
+    return f"ws://{host}:{port}{path}"
+
+
+def _close_policy_client(client: WebsocketClientPolicy) -> None:
+    ws = getattr(client, "_ws", None)
+    if ws is not None:
+        ws.close()
+
+
+def load_all_frames(video_path: Path) -> np.ndarray:
+    cap = cv2.VideoCapture(str(video_path))
+    frames = []
+    while True:
+        ok, frame = cap.read()
+        if not ok:
+            break
+        frames.append(cv2.cvtColor(frame, cv2.COLOR_BGR2RGB))
+    cap.release()
+    if not frames:
+        raise RuntimeError(f"No frames loaded from {video_path}")
+    return np.stack(frames, axis=0)
+
+
+def load_camera_frames(video_dir: Path) -> dict[str, np.ndarray]:
+    camera_frames: dict[str, np.ndarray] = {}
+    for camera_key, file_name in CAMERA_FILES.items():
+        video_path = video_dir / file_name
+        if not video_path.exists():
+            raise FileNotFoundError(
+                f"Missing DreamZero example asset: {video_path}. "
+                "Download the example videos with: "
+                f"`hf download {ASSET_REPO_ID} --repo-type dataset --local-dir {video_dir}`"
+            )
+        camera_frames[camera_key] = load_all_frames(video_path)
+    return camera_frames
+
+
+def build_frame_schedule(total_frames: int, num_chunks: int) -> list[list[int]]:
+    chunks: list[list[int]] = []
+    current_frame = 23
+    for _ in range(num_chunks):
+        indices = [max(current_frame + offset, 0) for offset in RELATIVE_OFFSETS]
+        if indices[-1] >= total_frames:
+            break
+        chunks.append(indices)
+        current_frame += ACTION_HORIZON
+    return chunks
+
+
+def make_obs_from_video(
+    camera_frames: dict[str, np.ndarray],
+    frame_indices: list[int],
+    *,
+    prompt: str,
+    session_id: str,
+) -> dict[str, Any]:
+    obs: dict[str, Any] = {}
+    for camera_key, all_frames in camera_frames.items():
+        selected = all_frames[frame_indices]
+        obs[camera_key] = selected[0] if len(frame_indices) == 1 else selected
+
+    obs["observation/joint_position"] = np.zeros(7, dtype=np.float32)
+    obs["observation/cartesian_position"] = np.zeros(6, dtype=np.float32)
+    obs["observation/gripper_position"] = np.zeros(1, dtype=np.float32)
+    obs["prompt"] = prompt
+    obs["session_id"] = session_id
+    return obs
+
+
+def build_demo_observations(
+    camera_frames: dict[str, np.ndarray],
+    *,
+    prompt: str,
+    session_id: str,
+    num_chunks: int = 2,
+) -> list[dict[str, Any]]:
+    if num_chunks < 1:
+        raise ValueError("num_chunks must be at least 1")
+
+    total_frames = min(frames.shape[0] for frames in camera_frames.values())
+    observations = [
+        make_obs_from_video(
+            camera_frames,
+            [0],
+            prompt=prompt,
+            session_id=session_id,
+        )
+    ]
+    for indices in build_frame_schedule(total_frames, num_chunks - 1):
+        observations.append(
+            make_obs_from_video(
+                camera_frames,
+                indices,
+                prompt=prompt,
+                session_id=session_id,
+            )
+        )
+    return observations
+
+
+def validate_session_result(
+    result: dict[str, Any],
+    *,
+    expected_action_horizon: int = ACTION_HORIZON,
+    expected_action_dim: int = DEFAULT_ACTION_DIM,
+    expected_action_count: int | None = None,
+) -> None:
+    metadata = DreamZeroServerMetadata.from_dict(result["metadata"])
+    if metadata.image_resolution != (180, 320):
+        raise AssertionError(f"Unexpected image_resolution: {metadata.image_resolution}")
+    if metadata.n_external_cameras != 2:
+        raise AssertionError(f"Unexpected n_external_cameras: {metadata.n_external_cameras}")
+    if not metadata.needs_wrist_camera:
+        raise AssertionError("DreamZero example expects wrist camera metadata")
+    if metadata.action_space != "joint_position":
+        raise AssertionError(f"Unexpected action_space: {metadata.action_space}")
+
+    actions = result["actions"]
+    if expected_action_count is not None and len(actions) != expected_action_count:
+        raise AssertionError(f"Expected {expected_action_count} action tensors, got {len(actions)}")
+    if not actions:
+        raise AssertionError("Expected at least one action tensor")
+    for index, action in enumerate(actions):
+        if action.shape != (expected_action_horizon, expected_action_dim):
+            raise AssertionError(
+                f"Action {index} shape mismatch: expected "
+                f"{(expected_action_horizon, expected_action_dim)}, got {action.shape}"
+            )
+        if not np.isfinite(action).all():
+            raise AssertionError(f"Action {index} contains non-finite values")
+
+
+def run_policy_session(
+    *,
+    host: str = DEFAULT_HOST,
+    port: int = DEFAULT_PORT,
+    path: str = DEFAULT_PATH,
+    video_dir: Path = DEFAULT_VIDEO_DIR,
+    prompt: str = DEFAULT_PROMPT,
+    session_id: str | None = None,
+    num_chunks: int = 2,
+) -> dict[str, Any]:
+    session_id = session_id or str(uuid.uuid4())
+    camera_frames = load_camera_frames(video_dir)
+    observations = build_demo_observations(
+        camera_frames,
+        prompt=prompt,
+        session_id=session_id,
+        num_chunks=num_chunks,
+    )
+
+    uri = _openpi_uri(host, port, path)
+    logging.info("Connecting to %s", uri)
+    client = WebsocketClientPolicy(host=uri)
+    try:
+        metadata = dict(client.get_server_metadata())
+        actions = [_as_action_array(client.infer(obs)) for obs in observations]
+        return {
+            "metadata": metadata,
+            "actions": actions,
+            "session_id": session_id,
+        }
+    finally:
+        _close_policy_client(client)
+
+
+def format_action_summary(index: int, action: np.ndarray) -> str:
+    return (
+        f"Action {index}: shape={tuple(action.shape)} dtype={action.dtype} "
+        f"min={action.min():.6f} max={action.max():.6f}"
+    )
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(description="DreamZero OpenPI client example with downloaded real videos.")
+    parser.add_argument("--host", default=DEFAULT_HOST)
+    parser.add_argument("--port", type=int, default=DEFAULT_PORT)
+    parser.add_argument("--path", default=DEFAULT_PATH)
+    parser.add_argument(
+        "--video-dir", type=Path, default=DEFAULT_VIDEO_DIR, help="Directory containing the three camera MP4 files."
+    )
+    parser.add_argument("--prompt", default=DEFAULT_PROMPT)
+    parser.add_argument("--session-id", default=None)
+    parser.add_argument("--num-chunks", type=int, default=2)
+    return parser.parse_args()
+
+
+def main() -> int:
+    args = parse_args()
+    logging.basicConfig(level=logging.INFO)
+
+    result = run_policy_session(
+        host=args.host,
+        port=args.port,
+        path=args.path,
+        video_dir=args.video_dir,
+        prompt=args.prompt,
+        session_id=args.session_id,
+        num_chunks=args.num_chunks,
+    )
+    validate_session_result(result, expected_action_count=args.num_chunks)
+
+    print("Server metadata:", json.dumps(result["metadata"], sort_keys=True))
+    for index, action in enumerate(result["actions"]):
+        print(format_action_summary(index, action))
+    print("Session ID:", result["session_id"])
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

examples/online_serving/dreamzero/run_server.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+MODEL="${MODEL:-GEAR-Dreams/DreamZero-DROID}"
+HOST="${HOST:-127.0.0.1}"
+PORT="${PORT:-8000}"
+DEPLOY_CONFIG="${DEPLOY_CONFIG:-vllm_omni/deploy/dreamzero_tp1_cfg2.yaml}"
+SERVED_MODEL_NAME="${SERVED_MODEL_NAME:-dreamzero-droid}"
+
+args=(
+  serve
+  "$MODEL"
+  --omni
+  --host "$HOST"
+  --port "$PORT"
+  --served-model-name "$SERVED_MODEL_NAME"
+  --deploy-config "$DEPLOY_CONFIG"
+  --enforce-eager
+  --disable-log-stats
+)
+
+ATTENTION_BACKEND="${ATTENTION_BACKEND:-torch}" \
+DIFFUSION_ATTENTION_BACKEND="${DIFFUSION_ATTENTION_BACKEND:-TORCH_SDPA}" \
+vllm "${args[@]}"