vllm-omni/vllm_omni/diffusion/models/dreamzero/transform/droid.py

# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project

"""DROID dataset transform.

DROID uses 1-indexed exterior cameras, 3 views total (OXE_DROID embodiment).
Stitching layout (same as RoboArena — both are OXE_DROID):
    ┌─────────────────────────┐
    │   wrist (2x width)      │  ← pixel-repeat along width
    ├────────────┬────────────┤
    │  left ext  │ right ext  │
    └────────────┴────────────┘
"""

from __future__ import annotations

import numpy as np
import torch
import torchvision.transforms.v2 as T

from vllm_omni.diffusion.models.dreamzero.transform.base import (
    RobotPolicyTransform,
    register_transform,
)


class DroidTransform(RobotPolicyTransform):
    """Transform for DROID dataset (OXE_DROID embodiment).

    DROID observation keys (1-indexed exterior cameras):
        observation/exterior_image_1_left  → left exterior
        observation/exterior_image_2_left  → right exterior
        observation/wrist_image_left       → wrist
    """

    IMAGE_KEY_MAP = {
        "observation/exterior_image_1_left": "images/exterior_0",
        "observation/exterior_image_2_left": "images/exterior_1",
        "observation/wrist_image_left": "images/wrist",
    }
    EMBODIMENT_NAME = "oxe_droid"
    ACTION_DIM = 8  # 7 joint + 1 gripper
    _VIDEO_CROP_SCALE = 0.95
    _VIDEO_RESIZE_HW = (176, 320)

    @classmethod
    def _preprocess_view(cls, arr: np.ndarray) -> np.ndarray:
        """Apply per-view crop and resize before stitching."""
        frames = torch.from_numpy(arr).to(torch.float32).permute(0, 3, 1, 2) / 255.0
        crop_h = int(arr.shape[1] * cls._VIDEO_CROP_SCALE)
        crop_w = int(arr.shape[2] * cls._VIDEO_CROP_SCALE)
        frames = T.CenterCrop((crop_h, crop_w))(frames)
        frames = T.Resize(
            cls._VIDEO_RESIZE_HW,
            interpolation=T.InterpolationMode.BILINEAR,
            antialias=True,
        )(frames)
        return (frames.permute(0, 2, 3, 1) * 255.0).to(torch.uint8).cpu().numpy()

    def _stitch_views(self, images: dict[str, np.ndarray]) -> np.ndarray:
        """OXE_DROID 2x2 stitching: wrist top (2x wide), exteriors bottom."""
        left_ext = images.get("images/exterior_0")
        right_ext = images.get("images/exterior_1")
        wrist = images.get("images/wrist")

        # Ensure 4D: (T, H, W, C)
        def ensure_4d(arr: np.ndarray | None) -> np.ndarray | None:
            if arr is None:
                return None
            return arr if arr.ndim == 4 else arr[np.newaxis]

        left_ext = ensure_4d(left_ext)
        right_ext = ensure_4d(right_ext)
        wrist = ensure_4d(wrist)

        # Determine shape from first available view.
        ref = next((v for v in [wrist, left_ext, right_ext] if v is not None), None)
        if ref is None:
            return np.zeros((1, 352, 640, 3), dtype=np.uint8)

        # Apply per-view crop + resize before stitching.
        def maybe_preprocess(arr: np.ndarray | None) -> np.ndarray | None:
            if arr is None:
                return None
            return self._preprocess_view(arr)

        left_ext = maybe_preprocess(left_ext)
        right_ext = maybe_preprocess(right_ext)
        wrist = maybe_preprocess(wrist)
        ref = next((v for v in [wrist, left_ext, right_ext] if v is not None), None)
        if ref is None:
            raise RuntimeError("Expected at least one DROID camera view after preprocessing.")
        t, h, w, c = ref.shape

        out = np.zeros((t, 2 * h, 2 * w, c), dtype=ref.dtype)  # (T, 2H, 2W, C)

        # Top row: wrist repeated 2x along width.
        if wrist is not None:
            wrist_wide = np.repeat(wrist, 2, axis=2)  # (T, H, 2W, C)
            out[:, :h, :] = wrist_wide

        # Bottom row: left exterior | right exterior.
        if left_ext is not None:
            out[:, h:, :w] = left_ext
        if right_ext is not None:
            out[:, h:, w:] = right_ext

        return out

    def _language_template(self, prompt: str) -> str:
        """Expand the language prompt for the OXE_DROID multi-view format."""
        prompt = (prompt or "Perform the default behavior.").strip()
        prompt_lower = prompt.lower()
        return (
            "A multi-view video shows that a robot "
            + prompt_lower
            + " The video is split into three views: The top view shows the "
            + "camera view from the robot's wrist, the bottom-left view shows "
            + "the camera view from the left exterior camera, and the "
            + "bottom-right view shows the camera view from the right exterior "
            + "camera. During training, one of the two bottom exterior views "
            + "may be a black screen (dropped view). The robot "
            + prompt_lower
        )

    def _extract_raw_state(self, obs: dict) -> np.ndarray:
        """OXE_DROID state: 7 joint + 1 gripper = 8 dims."""
        parts = []
        if "observation/joint_position" in obs:
            parts.append(np.asarray(obs["observation/joint_position"], dtype=np.float64).flatten())
        if "observation/gripper_position" in obs:
            parts.append(np.asarray(obs["observation/gripper_position"], dtype=np.float64).flatten())
        if parts:
            return np.concatenate(parts)
        return np.zeros(8, dtype=np.float64)


register_transform("droid", DroidTransform())