[Diffusion] add GR00T-N1.7 pipeline with OpenPI serving (#3798)

Signed-off-by: Zhengyuan Su <su.zhengyuan@u.nus.edu>
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Zhengyuan Su (苏政渊)andClaude Opus 4.8 authored and GitHub committed 2026-06-20 11:45:26 +08:00
1 parent e70822ed54
commit 2d0b1d8a29
34 files changed
+5495

No files matched your search

@@ -0,0 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from vllm_omni.diffusion.models.gr00t.modeling.gr00t_n1d7 import Gr00tN1d7
from vllm_omni.diffusion.models.gr00t.modeling.processing_gr00t_n1d7 import Gr00tN1d7Processor
__all__ = ["Gr00tN1d7", "Gr00tN1d7Processor"]
@@ -0,0 +1,491 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import logging
import os
from typing import Any
import torch
from torch import nn
from transformers import AutoConfig, AutoModel, PreTrainedModel
from transformers.feature_extraction_utils import BatchFeature
from vllm_omni.diffusion.models.gr00t.configs.gr00t_n1d7 import Gr00tN1d7Config
from vllm_omni.diffusion.models.gr00t.modeling.modules.dit import AlternateVLDiT, DiT, SelfAttentionTransformer
from vllm_omni.diffusion.models.gr00t.modeling.modules.embodiment_conditioned_mlp import (
CategorySpecificMLP,
MultiEmbodimentActionEncoder,
)
from vllm_omni.diffusion.models.gr00t.modeling.processing_gr00t_n1d7 import Gr00tN1d7DataCollator
logger = logging.getLogger(__name__)
class Gr00tN1d7ActionHead(nn.Module):
"""Action head component for flow matching diffusion policy."""
supports_gradient_checkpointing = True
def __init__(self, config: Gr00tN1d7Config):
super().__init__()
self.config = config
self.hidden_size = config.hidden_size
self.input_embedding_dim = config.input_embedding_dim
if config.use_alternate_vl_dit:
self.model = AlternateVLDiT(
**config.diffusion_model_cfg,
cross_attention_dim=config.backbone_embedding_dim,
attend_text_every_n_blocks=config.attend_text_every_n_blocks,
)
logger.info("Using AlternateVLDiT for diffusion model")
else:
self.model = DiT(
**config.diffusion_model_cfg,
cross_attention_dim=config.backbone_embedding_dim,
)
logger.info("Using DiT for diffusion model")
self.action_dim = config.max_action_dim
self.action_horizon = config.action_horizon
self.num_inference_timesteps = config.num_inference_timesteps
self.state_encoder = CategorySpecificMLP(
num_categories=config.max_num_embodiments,
input_dim=config.max_state_dim * config.state_history_length,
hidden_dim=self.hidden_size,
output_dim=self.input_embedding_dim,
)
self.action_encoder = MultiEmbodimentActionEncoder(
action_dim=self.action_dim,
hidden_size=self.input_embedding_dim,
num_embodiments=config.max_num_embodiments,
)
self.action_decoder = CategorySpecificMLP(
num_categories=config.max_num_embodiments,
input_dim=self.hidden_size,
hidden_dim=self.hidden_size,
output_dim=self.action_dim,
)
self.vlln = nn.LayerNorm(config.backbone_embedding_dim) if config.use_vlln else nn.Identity()
vl_self_attention_cfg = config.vl_self_attention_cfg
if vl_self_attention_cfg and vl_self_attention_cfg.get("num_layers", 0) > 0:
self.vl_self_attention = SelfAttentionTransformer(**vl_self_attention_cfg)
else:
self.vl_self_attention = nn.Identity()
if config.add_pos_embed:
self.position_embedding = nn.Embedding(config.max_seq_len, self.input_embedding_dim)
nn.init.normal_(self.position_embedding.weight, mean=0.0, std=0.02)
self.num_timestep_buckets = config.num_timestep_buckets
def process_backbone_output(self, backbone_output: BatchFeature) -> BatchFeature:
backbone_features = backbone_output["backbone_features"]
backbone_features = self.vlln(backbone_features)
backbone_features = self.vl_self_attention(backbone_features)
backbone_output["backbone_features"] = backbone_features
return backbone_output
def _encode_features(self, backbone_output: BatchFeature, action_input: BatchFeature) -> BatchFeature:
"""
Encode features for the action head.
Args:
backbone_output: Output from the backbone model containing:
- backbone_features: [B, seq_len, backbone_embedding_dim]
- backbone_attention_mask: [B, seq_len]
action_input: Input containing:
- state: [B, state_history_length, max_state_dim]
- embodiment_id: [B] (embodiment IDs)
Returns:
BatchFeature containing:
- backbone_features: [B, seq_len, backbone_embedding_dim]
- state_features: [B, 1, input_embedding_dim]
"""
backbone_output = self.process_backbone_output(backbone_output)
vl_embeds = backbone_output.backbone_features
embodiment_id = action_input.embodiment_id
state = action_input.state
current_T = state.shape[1]
assert current_T == self.config.state_history_length, "current_T != state_history_length"
# [B, state_history_length, max_state_dim] -> [B, 1, state_history_length * max_state_dim]
state = state.view(state.shape[0], 1, -1)
state_features = self.state_encoder(state, embodiment_id)
return BatchFeature(data={"backbone_features": vl_embeds, "state_features": state_features})
@torch.no_grad()
def get_action_with_features(
self,
backbone_features: torch.Tensor,
state_features: torch.Tensor,
embodiment_id: torch.Tensor,
backbone_output: BatchFeature,
action_input: BatchFeature,
options: dict[str, Any] | None = None,
) -> BatchFeature:
"""
Generate actions using the flow matching diffusion process.
Args:
backbone_features: [B, seq_len, backbone_embedding_dim]
state_features: [B, state_horizon, input_embedding_dim]
embodiment_id: [B] (embodiment IDs)
backbone_output: Output from the backbone model
"""
vl_embeds = backbone_features
batch_size = vl_embeds.shape[0]
device = vl_embeds.device
_seed_env = os.environ.get("GR00T_NOISE_SEED")
if _seed_env is not None:
_gen = torch.Generator(device=device).manual_seed(int(_seed_env))
actions = torch.randn(
size=(batch_size, self.config.action_horizon, self.action_dim),
dtype=vl_embeds.dtype,
device=device,
generator=_gen,
)
else:
actions = torch.randn(
size=(batch_size, self.config.action_horizon, self.action_dim),
dtype=vl_embeds.dtype,
device=device,
)
dt = 1.0 / self.num_inference_timesteps
vel_strength = torch.ones_like(actions)
if "action" in action_input:
# If action in input when doing get action, it means we want to use RTC.
# action_horizon is the action horizon of the input action.
# rtc_overlap_steps is the number of steps to overlap with the previous action chunks.
# rtc_frozen_steps is the policy inference latency expressed as frozen action steps.
# rtc_ramp_rate is the rate of the ramp of denoising the actions.
assert options is not None, "options is not None"
assert "action_horizon" in options, "action_horizon is not in options"
assert "rtc_overlap_steps" in options, "rtc_overlap_steps is not in options"
assert "rtc_frozen_steps" in options, "rtc_frozen_steps is not in options"
assert "rtc_ramp_rate" in options, "rtc_ramp_rate is not in options"
action_horizon_before_padding = options["action_horizon"]
# Use previous action instead of pure noise to do inpainting
actions[:, : options["rtc_overlap_steps"], :] = action_input["action"][
:,
action_horizon_before_padding - options["rtc_overlap_steps"] : action_horizon_before_padding,
:,
]
vel_strength[:, : options["rtc_frozen_steps"], :] = 0.0
# NOTE: use an exponential ramp strength to set the remaining unfrozen rtc_steps
intermediate_steps = options["rtc_overlap_steps"] - options["rtc_frozen_steps"]
# Create exponential ramp from 0 to 1 over intermediate steps
t = torch.linspace(0.0, 1.0, intermediate_steps + 2, device=device)
ramp = 1 - torch.exp(-options["rtc_ramp_rate"] * t)
ramp = ramp / ramp[-1].clamp_min(1e-8) # normalize to [0,1]
ramp = ramp[1:-1] # we will only take the middle part of the ramp, ignore the 0.0 and 1.0
# Apply ramp to the intermediate steps [batch, intermediate_steps, action_dim]
vel_strength[
:,
options["rtc_frozen_steps"] : options["rtc_overlap_steps"],
:,
] = ramp[None, :, None].to(device)
# Run denoising steps.
for t in range(self.num_inference_timesteps):
t_discretized = t * self.num_timestep_buckets // self.num_inference_timesteps
# Embed noised action trajectory.
timesteps_tensor = torch.full(size=(batch_size,), fill_value=t_discretized, device=device)
action_features = self.action_encoder(actions, timesteps_tensor, embodiment_id)
# Add position embedding.
if self.config.add_pos_embed:
pos_ids = torch.arange(action_features.shape[1], dtype=torch.long, device=device)
pos_embs = self.position_embedding(pos_ids).unsqueeze(0)
action_features = action_features + pos_embs
# Join vision, language, state and action embedding along sequence dimension.
sa_embs = torch.cat((state_features, action_features), dim=1)
# Run model forward.
if self.config.use_alternate_vl_dit:
model_output = self.model(
hidden_states=sa_embs,
encoder_hidden_states=vl_embeds,
timestep=timesteps_tensor,
image_mask=backbone_output.image_mask,
backbone_attention_mask=backbone_output.backbone_attention_mask,
)
else:
model_output = self.model(
hidden_states=sa_embs,
encoder_hidden_states=vl_embeds,
timestep=timesteps_tensor,
)
pred = self.action_decoder(model_output, embodiment_id)
pred_velocity = pred[:, -self.action_horizon :]
# Update actions using euler integration.
actions = actions + dt * pred_velocity * vel_strength
return BatchFeature(
data={
"action_pred": actions,
"backbone_features": vl_embeds,
"state_features": state_features,
}
)
@torch.no_grad()
def get_action(
self,
backbone_output: BatchFeature,
action_input: BatchFeature,
options: dict[str, Any] | None = None,
) -> BatchFeature:
"""
Generate actions using the flow matching diffusion process.
Args:
backbone_output: Output from the backbone model containing:
- backbone_features: [B, seq_len, backbone_embedding_dim]
- backbone_attention_mask: [B, seq_len]
action_input: Input containing:
- state: [B, state_dim]
- embodiment_id: [B] (embodiment IDs)
Returns:
BatchFeature containing:
- action_pred: [B, action_horizon, action_dim] predicted actions
"""
features = self._encode_features(backbone_output, action_input)
return self.get_action_with_features(
backbone_features=features.backbone_features,
state_features=features.state_features,
embodiment_id=action_input.embodiment_id,
backbone_output=backbone_output,
action_input=action_input,
options=options,
)
@property
def device(self):
return next(iter(self.parameters())).device
@property
def dtype(self):
return next(iter(self.parameters())).dtype
def prepare_input(self, batch: dict) -> BatchFeature:
"""Prepare input batch for the action head."""
return BatchFeature(data=batch)
def _load_qwen3_vl_cls() -> type:
"""Import Qwen3-VL lazily so the missing-dep failure is clear and deferred."""
try:
from transformers.models.qwen3_vl.modeling_qwen3_vl import Qwen3VLForConditionalGeneration
except ImportError as exc:
raise ImportError(
"GR00T-N1.7 requires transformers>=4.57.1 for transformers.models.qwen3_vl "
"(the repo's declared floor is 4.56.0). Please upgrade transformers."
) from exc
return Qwen3VLForConditionalGeneration
class _Qwen3VLBackbone(nn.Module):
"""GR00T adapter around the shared Qwen3-VL implementation."""
def __init__(
self,
model_name: str,
select_layer: int,
backbone_embedding_dim: int,
load_bf16: bool,
transformers_loading_kwargs: dict[str, Any] | None = None,
):
super().__init__()
backbone_config = AutoConfig.from_pretrained(
model_name, **(transformers_loading_kwargs or {"trust_remote_code": True})
)
backbone_config.text_config._attn_implementation = "sdpa"
backbone_config.vision_config._attn_implementation = "sdpa"
self.model = _load_qwen3_vl_cls()(backbone_config).eval()
if load_bf16:
self.model.to(dtype=torch.bfloat16)
target_layers = select_layer if select_layer >= 0 else len(self.model.model.language_model.layers)
while len(self.model.model.language_model.layers) > target_layers:
self.model.model.language_model.layers.pop(-1)
# Drop the unused lm_head: feature-extractor only, and .to(dtype) above unties it (~0.58 GiB).
self.model.lm_head = None
def set_frozen_modules_to_eval_mode(self) -> None:
self.eval()
def prepare_input(self, batch: dict) -> BatchFeature:
return BatchFeature(data=batch)
def forward(self, vl_input: BatchFeature) -> BatchFeature:
self.set_frozen_modules_to_eval_mode()
keys_to_use = [
"input_ids",
"attention_mask",
"pixel_values",
"image_grid_thw",
"mm_token_type_ids",
]
vl_input = {key: vl_input[key] for key in keys_to_use if key in vl_input}
# GR00T was trained against the **pre-final-RMSNorm** output of the last
# decoder layer (this was `outer.hidden_states[-1]` under HF 4.57.x semantics,
# where the @check_model_inputs tie-logic skipped overwriting because
# Qwen3VLCausalLMOutputWithPast has no `last_hidden_state` field).
# HF >=5.x changed the output_hidden_states tuple plumbing so that
# `hidden_states[-1]` is post-norm, which is a different tensor and breaks
# the trained action head. Capture the pre-norm tensor via a hook on the
# final RMSNorm regardless of transformers version.
_captured: list[torch.Tensor] = []
def _pre_norm_hook(_module, args, _out):
_captured.append(args[0])
norm = self.model.model.language_model.norm
handle = norm.register_forward_hook(_pre_norm_hook)
try:
self.model.model(**vl_input, return_dict=True)
finally:
handle.remove()
if not _captured:
raise RuntimeError("Failed to capture pre-norm hidden states from Qwen3-VL backbone")
backbone_features = _captured[-1]
image_mask = vl_input["input_ids"] == self.model.config.image_token_id
attention_mask = vl_input["attention_mask"] == 1
return BatchFeature(
data={
"backbone_features": backbone_features,
"backbone_attention_mask": attention_mask,
"image_mask": image_mask,
}
)
def get_backbone_cls(config: Gr00tN1d7Config):
if "nvidia/Cosmos-Reason2" in config.model_name:
return _Qwen3VLBackbone
raise ValueError(f"Unsupported model name: {config.model_name}")
class Gr00tN1d7(PreTrainedModel):
"""Gr00tN1d7: VLA model with Cosmos-Reason2-2B (Qwen3-VL) backbone."""
config_class = Gr00tN1d7Config
supports_gradient_checkpointing = True
_tp_plan = {}
@property
def all_tied_weights_keys(self) -> dict[str, Any]:
return {}
def __init__(
self,
config: Gr00tN1d7Config,
transformers_loading_kwargs: dict | None = None,
):
super().__init__(config)
if transformers_loading_kwargs is None:
transformers_loading_kwargs = {"trust_remote_code": True}
self.config = config
backbone_cls = get_backbone_cls(config)
self.backbone = backbone_cls(
model_name=config.model_name,
select_layer=config.select_layer,
backbone_embedding_dim=config.backbone_embedding_dim,
load_bf16=config.load_bf16,
transformers_loading_kwargs=transformers_loading_kwargs,
)
# Initialize action head
self.action_head = Gr00tN1d7ActionHead(config)
self.collator = Gr00tN1d7DataCollator(
model_name=config.model_name,
model_type=config.backbone_model_type,
transformers_loading_kwargs=transformers_loading_kwargs,
)
def prepare_input(self, inputs: dict) -> tuple[BatchFeature, BatchFeature]:
"""Prepare inputs for backbone and action head."""
# NOTE -- currently the eval code doesn't use collator, so we need to add it here
if "vlm_content" in inputs:
# Fix for n_envs > 1: Process all environments' VLM content, not just the first
vlm_content_list = inputs["vlm_content"]
# Ensure vlm_content_list is always a list for consistent processing
if not isinstance(vlm_content_list, list):
vlm_content_list = [vlm_content_list]
# Process all VLM contents through the collator
prep = self.collator([{"vlm_content": vlm} for vlm in vlm_content_list])["inputs"]
inputs.pop("vlm_content")
inputs.update(prep)
backbone_inputs = self.backbone.prepare_input(inputs)
action_inputs = self.action_head.prepare_input(inputs)
backbone_inputs = backbone_inputs.to(device=self.device, dtype=self.dtype)
action_inputs = action_inputs.to(device=self.device, dtype=self.dtype)
return backbone_inputs, action_inputs
def forward(self, inputs: dict) -> BatchFeature:
"""
Forward pass through the complete model.
Args:
inputs: Dictionary containing:
- Action inputs (state, action, embodiment_id, etc.)
Returns:
BatchFeature containing loss and other outputs
"""
# Prepare inputs for backbone and action head
backbone_inputs, action_inputs = self.prepare_input(inputs)
backbone_outputs = self.backbone(backbone_inputs)
action_outputs = self.action_head(backbone_outputs, action_inputs)
return action_outputs
def get_action(self, inputs: dict, options: dict[str, Any] | None = None) -> BatchFeature:
"""
Generate actions using the complete model.
"""
# Prepare inputs for backbone and action head
backbone_inputs, action_inputs = self.prepare_input(inputs)
# Forward through backbone
backbone_outputs = self.backbone(backbone_inputs)
action_outputs = self.action_head.get_action(backbone_outputs, action_inputs, options)
return action_outputs
@property
def device(self):
return next(iter(self.parameters())).device
@property
def dtype(self):
return next(iter(self.parameters())).dtype
# Register the model with HuggingFace
AutoConfig.register("Gr00tN1d7", Gr00tN1d7Config)
AutoModel.register(Gr00tN1d7Config, Gr00tN1d7)
@@ -0,0 +1,2 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
@@ -0,0 +1,344 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
import torch.nn.functional as F
from diffusers import ConfigMixin, ModelMixin
from diffusers.configuration_utils import register_to_config
from diffusers.models.attention import Attention, FeedForward
from diffusers.models.embeddings import SinusoidalPositionalEmbedding, TimestepEmbedding, Timesteps
from torch import nn
class AdaLayerNorm(nn.Module):
def __init__(
self,
embedding_dim: int,
norm_elementwise_affine: bool = False,
norm_eps: float = 1e-5,
chunk_dim: int = 0,
):
super().__init__()
self.chunk_dim = chunk_dim
output_dim = embedding_dim * 2
self.silu = nn.SiLU()
self.linear = nn.Linear(embedding_dim, output_dim)
self.norm = nn.LayerNorm(output_dim // 2, norm_eps, norm_elementwise_affine)
def forward(
self,
x: torch.Tensor,
temb: torch.Tensor | None = None,
) -> torch.Tensor:
temb = self.linear(self.silu(temb))
scale, shift = temb.chunk(2, dim=1)
x = self.norm(x) * (1 + scale[:, None]) + shift[:, None]
return x
class BasicTransformerBlock(nn.Module):
def __init__(
self,
dim: int,
num_attention_heads: int,
attention_head_dim: int,
dropout=0.0,
cross_attention_dim: int | None = None,
activation_fn: str = "geglu",
attention_bias: bool = False,
upcast_attention: bool = False,
norm_elementwise_affine: bool = True,
norm_type: str = "layer_norm",
norm_eps: float = 1e-5,
final_dropout: bool = False,
attention_type: str = "default",
positional_embeddings: str | None = None,
num_positional_embeddings: int | None = None,
ff_inner_dim: int | None = None,
ff_bias: bool = True,
attention_out_bias: bool = True,
):
super().__init__()
self.norm_type = norm_type
if positional_embeddings and (num_positional_embeddings is None):
raise ValueError(
"If `positional_embedding` type is defined, `num_position_embeddings` must also be defined."
)
if positional_embeddings == "sinusoidal":
self.pos_embed = SinusoidalPositionalEmbedding(dim, max_seq_length=num_positional_embeddings)
else:
self.pos_embed = None
if norm_type == "ada_norm":
self.norm1 = AdaLayerNorm(dim)
else:
self.norm1 = nn.LayerNorm(dim, elementwise_affine=norm_elementwise_affine, eps=norm_eps)
self.attn1 = Attention(
query_dim=dim,
heads=num_attention_heads,
dim_head=attention_head_dim,
dropout=dropout,
bias=attention_bias,
cross_attention_dim=cross_attention_dim,
upcast_attention=upcast_attention,
out_bias=attention_out_bias,
)
self.norm3 = nn.LayerNorm(dim, norm_eps, norm_elementwise_affine)
self.ff = FeedForward(
dim,
dropout=dropout,
activation_fn=activation_fn,
final_dropout=final_dropout,
inner_dim=ff_inner_dim,
bias=ff_bias,
)
self.final_dropout = nn.Dropout(dropout) if final_dropout else None
def forward(
self,
hidden_states: torch.Tensor,
attention_mask: torch.Tensor | None = None,
encoder_hidden_states: torch.Tensor | None = None,
encoder_attention_mask: torch.Tensor | None = None,
temb: torch.LongTensor | None = None,
) -> torch.Tensor:
if self.norm_type == "ada_norm":
norm_hidden_states = self.norm1(hidden_states, temb)
else:
norm_hidden_states = self.norm1(hidden_states)
if self.pos_embed is not None:
norm_hidden_states = self.pos_embed(norm_hidden_states)
attn_output = self.attn1(
norm_hidden_states,
encoder_hidden_states=encoder_hidden_states,
attention_mask=(encoder_attention_mask if encoder_hidden_states is not None else attention_mask),
)
if self.final_dropout:
attn_output = self.final_dropout(attn_output)
hidden_states = attn_output + hidden_states
if hidden_states.ndim == 4:
hidden_states = hidden_states.squeeze(1)
norm_hidden_states = self.norm3(hidden_states)
ff_output = self.ff(norm_hidden_states)
hidden_states = ff_output + hidden_states
if hidden_states.ndim == 4:
hidden_states = hidden_states.squeeze(1)
return hidden_states
class TimestepEncoder(nn.Module):
def __init__(self, embedding_dim: int):
super().__init__()
self.time_proj = Timesteps(num_channels=256, flip_sin_to_cos=True, downscale_freq_shift=1)
self.timestep_embedder = TimestepEmbedding(in_channels=256, time_embed_dim=embedding_dim)
def forward(self, timesteps: torch.Tensor) -> torch.Tensor:
dtype = next(self.parameters()).dtype
return self.timestep_embedder(self.time_proj(timesteps).to(dtype))
class DiT(ModelMixin, ConfigMixin):
_supports_gradient_checkpointing = True
@register_to_config
def __init__(
self,
num_attention_heads: int = 8,
attention_head_dim: int = 64,
output_dim: int = 26,
num_layers: int = 12,
dropout: float = 0.1,
attention_bias: bool = True,
activation_fn: str = "gelu-approximate",
num_embeds_ada_norm: int | None = 1000,
upcast_attention: bool = False,
norm_type: str = "ada_norm",
norm_elementwise_affine: bool = False,
norm_eps: float = 1e-5,
max_num_positional_embeddings: int = 512,
compute_dtype=torch.float32,
final_dropout: bool = True,
positional_embeddings: str | None = "sinusoidal",
interleave_self_attention=False,
cross_attention_dim: int | None = None,
):
super().__init__()
self.inner_dim = num_attention_heads * attention_head_dim
self.gradient_checkpointing = False
self.timestep_encoder = TimestepEncoder(embedding_dim=self.inner_dim)
all_blocks = []
for idx in range(num_layers):
use_self_attn = idx % 2 == 1 and interleave_self_attention
curr_cross_attention_dim = cross_attention_dim if not use_self_attn else None
all_blocks.append(
BasicTransformerBlock(
self.inner_dim,
num_attention_heads,
attention_head_dim,
dropout=dropout,
activation_fn=activation_fn,
attention_bias=attention_bias,
upcast_attention=upcast_attention,
norm_type=norm_type,
norm_elementwise_affine=norm_elementwise_affine,
norm_eps=norm_eps,
positional_embeddings=positional_embeddings,
num_positional_embeddings=max_num_positional_embeddings,
final_dropout=final_dropout,
cross_attention_dim=curr_cross_attention_dim,
)
)
self.transformer_blocks = nn.ModuleList(all_blocks)
self.norm_out = nn.LayerNorm(self.inner_dim, elementwise_affine=False, eps=1e-6)
self.proj_out_1 = nn.Linear(self.inner_dim, 2 * self.inner_dim)
self.proj_out_2 = nn.Linear(self.inner_dim, output_dim)
def forward(
self,
hidden_states: torch.Tensor,
encoder_hidden_states: torch.Tensor,
timestep: torch.LongTensor | None = None,
encoder_attention_mask: torch.Tensor | None = None,
return_all_hidden_states: bool = False,
):
temb = self.timestep_encoder(timestep)
hidden_states = hidden_states.contiguous()
encoder_hidden_states = encoder_hidden_states.contiguous()
all_hidden_states = [hidden_states]
for idx, block in enumerate(self.transformer_blocks):
if idx % 2 == 1 and self.config.interleave_self_attention:
hidden_states = block(hidden_states, temb=temb)
else:
hidden_states = block(
hidden_states,
encoder_hidden_states=encoder_hidden_states,
encoder_attention_mask=encoder_attention_mask,
temb=temb,
)
all_hidden_states.append(hidden_states)
shift, scale = self.proj_out_1(F.silu(temb)).chunk(2, dim=1)
hidden_states = self.norm_out(hidden_states) * (1 + scale[:, None]) + shift[:, None]
if return_all_hidden_states:
return self.proj_out_2(hidden_states), all_hidden_states
return self.proj_out_2(hidden_states)
class AlternateVLDiT(DiT):
def __init__(self, *args, attend_text_every_n_blocks: int = 2, **kwargs):
super().__init__(*args, **kwargs)
self.attend_text_every_n_blocks = attend_text_every_n_blocks
def forward(
self,
hidden_states: torch.Tensor,
encoder_hidden_states: torch.Tensor,
timestep: torch.LongTensor | None = None,
encoder_attention_mask: torch.Tensor | None = None,
return_all_hidden_states: bool = False,
image_mask: torch.Tensor | None = None,
backbone_attention_mask: torch.Tensor | None = None,
):
assert image_mask is not None, "Image mask is required"
assert self.config.interleave_self_attention, "Interleave self attention must be enabled"
temb = self.timestep_encoder(timestep)
hidden_states = hidden_states.contiguous()
encoder_hidden_states = encoder_hidden_states.contiguous()
image_attention_mask = image_mask & backbone_attention_mask
non_image_attention_mask = (~image_mask) & backbone_attention_mask
all_hidden_states = [hidden_states]
for idx, block in enumerate(self.transformer_blocks):
if idx % 2 == 1:
hidden_states = block(hidden_states, temb=temb)
else:
curr_encoder_attention_mask = (
non_image_attention_mask
if idx % (2 * self.attend_text_every_n_blocks) == 0
else image_attention_mask
)
hidden_states = block(
hidden_states,
encoder_hidden_states=encoder_hidden_states,
encoder_attention_mask=curr_encoder_attention_mask,
temb=temb,
)
all_hidden_states.append(hidden_states)
shift, scale = self.proj_out_1(F.silu(temb)).chunk(2, dim=1)
hidden_states = self.norm_out(hidden_states) * (1 + scale[:, None]) + shift[:, None]
if return_all_hidden_states:
return self.proj_out_2(hidden_states), all_hidden_states
return self.proj_out_2(hidden_states)
class SelfAttentionTransformer(ModelMixin, ConfigMixin):
_supports_gradient_checkpointing = True
@register_to_config
def __init__(
self,
num_attention_heads: int = 8,
attention_head_dim: int = 64,
output_dim: int = 26,
num_layers: int = 12,
dropout: float = 0.1,
attention_bias: bool = True,
activation_fn: str = "gelu-approximate",
num_embeds_ada_norm: int | None = 1000,
upcast_attention: bool = False,
max_num_positional_embeddings: int = 512,
compute_dtype=torch.float32,
final_dropout: bool = True,
positional_embeddings: str | None = "sinusoidal",
interleave_self_attention=False,
):
super().__init__()
self.inner_dim = num_attention_heads * attention_head_dim
self.gradient_checkpointing = False
self.transformer_blocks = nn.ModuleList(
[
BasicTransformerBlock(
self.inner_dim,
num_attention_heads,
attention_head_dim,
dropout=dropout,
activation_fn=activation_fn,
attention_bias=attention_bias,
upcast_attention=upcast_attention,
positional_embeddings=positional_embeddings,
num_positional_embeddings=max_num_positional_embeddings,
final_dropout=final_dropout,
)
for _ in range(num_layers)
]
)
def forward(
self,
hidden_states: torch.Tensor,
return_all_hidden_states: bool = False,
):
hidden_states = hidden_states.contiguous()
all_hidden_states = [hidden_states]
for block in self.transformer_blocks:
hidden_states = block(hidden_states)
all_hidden_states.append(hidden_states)
if return_all_hidden_states:
return hidden_states, all_hidden_states
return hidden_states
@@ -0,0 +1,178 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
import torch.nn.functional as F
from torch import nn
from vllm_omni.diffusion.utils.flow_matching import SinusoidalPositionalEncoding, swish
class CategorySpecificLinear(nn.Module):
"""Linear layer with category-specific weights and biases for multi-embodiment support."""
def __init__(self, num_categories: int, input_dim: int, hidden_dim: int) -> None:
super().__init__()
self.num_categories = num_categories
# For each category, we have separate weights and biases.
self.W = nn.Parameter(0.02 * torch.randn(num_categories, input_dim, hidden_dim))
self.b = nn.Parameter(torch.zeros(num_categories, hidden_dim))
def forward(self, x: torch.Tensor, cat_ids: torch.Tensor) -> torch.Tensor:
"""Apply per-category linear projection.
Args:
x: ``[B, T, input_dim]`` input tensor.
cat_ids: ``[B]`` category / embodiment IDs.
Returns:
``[B, T, hidden_dim]`` output tensor.
"""
selected_W = self.W[cat_ids]
selected_b = self.b[cat_ids]
return torch.bmm(x, selected_W) + selected_b.unsqueeze(1)
def expand_action_dimension(
self,
old_action_dim: int,
new_action_dim: int,
expand_input: bool = False,
expand_output: bool = False,
) -> None:
"""Safely expand action dimension with explicit targeting.
Args:
old_action_dim: Original action dimension.
new_action_dim: New (larger) action dimension.
expand_input: Whether to expand input dimension (dim=1).
expand_output: Whether to expand output dimension (dim=2).
"""
if new_action_dim <= old_action_dim:
raise ValueError(f"New action dim {new_action_dim} must be larger than old action dim {old_action_dim}")
# Expand input dimension (dim=1) only if explicitly requested AND dimensions match
if expand_input and self.W.shape[1] == old_action_dim:
repeat_times = new_action_dim // old_action_dim
remainder = new_action_dim % old_action_dim
new_W_parts = [self.W] * repeat_times
if remainder > 0:
new_W_parts.append(self.W[:, :remainder, :])
new_W = torch.cat(new_W_parts, dim=1)
self.W = nn.Parameter(new_W)
# Expand output dimension (dim=2) only if explicitly requested AND dimensions match
if expand_output and self.W.shape[2] == old_action_dim:
repeat_times = new_action_dim // old_action_dim
remainder = new_action_dim % old_action_dim
new_W_parts = [self.W] * repeat_times
if remainder > 0:
new_W_parts.append(self.W[:, :, :remainder])
new_W = torch.cat(new_W_parts, dim=2)
self.W = nn.Parameter(new_W)
# Expand bias for output dimension
if self.b.shape[1] == old_action_dim:
new_b_parts = [self.b] * repeat_times
if remainder > 0:
new_b_parts.append(self.b[:, :remainder])
new_b = torch.cat(new_b_parts, dim=1)
self.b = nn.Parameter(new_b)
class CategorySpecificMLP(nn.Module):
"""Two-layer MLP with category-specific weights for multi-embodiment support."""
def __init__(self, num_categories: int, input_dim: int, hidden_dim: int, output_dim: int) -> None:
super().__init__()
self.num_categories = num_categories
self.layer1 = CategorySpecificLinear(num_categories, input_dim, hidden_dim)
self.layer2 = CategorySpecificLinear(num_categories, hidden_dim, output_dim)
def forward(self, x: torch.Tensor, cat_ids: torch.Tensor) -> torch.Tensor:
"""Apply two-layer per-category MLP.
Args:
x: ``[B, T, input_dim]`` input tensor.
cat_ids: ``[B]`` category / embodiment IDs.
Returns:
``[B, T, output_dim]`` output tensor.
"""
hidden = F.relu(self.layer1(x, cat_ids))
return self.layer2(hidden, cat_ids)
def expand_action_dimension(self, old_action_dim: int, new_action_dim: int) -> None:
"""Expand action dimension by copying weights from existing dimensions.
Args:
old_action_dim: Original action dimension.
new_action_dim: New (larger) action dimension.
"""
# self.layer1 does not take action_dim as input, so no expansion needed
self.layer2.expand_action_dimension(old_action_dim, new_action_dim, expand_input=False, expand_output=True)
class MultiEmbodimentActionEncoder(nn.Module):
"""Action encoder with multi-embodiment support and sinusoidal positional encoding."""
def __init__(self, action_dim: int, hidden_size: int, num_embodiments: int) -> None:
super().__init__()
self.hidden_size = hidden_size
self.num_embodiments = num_embodiments
# W1: R^{w x d}, W2: R^{w x 2w}, W3: R^{w x w}
self.W1 = CategorySpecificLinear(num_embodiments, action_dim, hidden_size) # (d -> w)
self.W2 = CategorySpecificLinear(num_embodiments, 2 * hidden_size, hidden_size) # (2w -> w)
self.W3 = CategorySpecificLinear(num_embodiments, hidden_size, hidden_size) # (w -> w)
self.pos_encoding = SinusoidalPositionalEncoding(hidden_size)
def forward(self, actions: torch.Tensor, timesteps: torch.Tensor, cat_ids: torch.Tensor) -> torch.Tensor:
"""Encode actions with embodiment-conditioned MLP and time embedding.
Args:
actions: ``[B, T, action_dim]`` action tensor.
timesteps: ``[B]`` timesteps (one scalar per batch item).
cat_ids: ``[B]`` category / embodiment IDs.
Returns:
``[B, T, hidden_size]`` encoded action features.
"""
B, T, _ = actions.shape
# 1) Expand each batch's single scalar time 'tau' across all T steps
# so that shape => (B, T)
# e.g. if timesteps is (B,), replicate across T
if timesteps.dim() == 1 and timesteps.shape[0] == B:
# shape (B,) => (B,T)
timesteps = timesteps.unsqueeze(1).expand(-1, T)
else:
raise ValueError("Expected `timesteps` to have shape (B,) so we can replicate across T.")
# 2) Standard action MLP step for shape => (B, T, w)
a_emb = self.W1(actions, cat_ids)
# 3) Get the sinusoidal encoding (B, T, w)
tau_emb = self.pos_encoding(timesteps).to(dtype=a_emb.dtype)
# 4) Concat along last dim => (B, T, 2w), then W2 => (B, T, w), swish
x = torch.cat([a_emb, tau_emb], dim=-1)
x = swish(self.W2(x, cat_ids))
# 5) Finally W3 => (B, T, w)
x = self.W3(x, cat_ids)
return x
def expand_action_dimension(self, old_action_dim: int, new_action_dim: int) -> None:
"""Expand action dimension by copying weights from existing dimensions.
Args:
old_action_dim: Original action dimension.
new_action_dim: New (larger) action dimension.
"""
# Only W1 takes action_dim as input, so only expand its input dimension
self.W1.expand_action_dimension(old_action_dim, new_action_dim, expand_input=True, expand_output=False)
@@ -0,0 +1,555 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
import os
import re
import warnings
from copy import deepcopy
from pathlib import Path
from typing import Any
import numpy as np
import torch
import torchvision.transforms.v2 as transforms
from PIL import Image
from transformers import AutoProcessor, ProcessorMixin
from transformers.feature_extraction_utils import BatchFeature
from transformers.utils import cached_file
from vllm.logger import init_logger
from vllm_omni.diffusion.models.gr00t.configs.embodiment_configs import ModalityConfig
from vllm_omni.diffusion.models.gr00t.dataio.embodiment_tags import EmbodimentTag
from vllm_omni.diffusion.models.gr00t.dataio.state_action.state_action_processor import StateActionProcessor
from vllm_omni.diffusion.models.gr00t.dataio.utils import parse_modality_configs, to_json_serializable
class LetterBoxTransform:
"""Pad image to square dimensions by adding black bars to the smaller side."""
def __call__(self, img: torch.Tensor) -> torch.Tensor:
import math
*leading_dims, c, h, w = img.shape
if h == w:
return img
max_dim = max(h, w)
pad_h, pad_w = max_dim - h, max_dim - w
pad_top, pad_left = pad_h // 2, pad_w // 2
pad_bottom, pad_right = pad_h - pad_top, pad_w - pad_left
if leading_dims:
batch_size = math.prod(leading_dims)
img_r = img.reshape(batch_size, c, h, w)
padded = transforms.functional.pad(img_r, padding=[pad_left, pad_top, pad_right, pad_bottom], fill=0)
return padded.reshape(leading_dims + [c, max_dim, max_dim])
return transforms.functional.pad(img, padding=[pad_left, pad_top, pad_right, pad_bottom], fill=0)
def _build_eval_image_transform(
image_target_size: list[int],
image_crop_size: list[int],
) -> transforms.Compose:
"""Deterministic eval/inference image transform (letterbox → resize → centercrop → resize)."""
return transforms.Compose(
[
transforms.ToImage(),
LetterBoxTransform(),
transforms.Resize(size=image_target_size),
transforms.CenterCrop(size=image_crop_size),
transforms.Resize(size=image_target_size),
]
)
logger = init_logger(__name__)
# Suppress protobuf deprecation warnings
warnings.filterwarnings("ignore", category=DeprecationWarning, module="google.protobuf")
EMBODIMENT_TAG_TO_PROJECTOR_INDEX = {
# Pretrain embodiment ids
"oxe_droid_relative_eef_relative_joint": 24,
"xdof_relative_eef_relative_joint": 27,
"xdof_relative_eef_relative_joint_subtask": 27,
"real_g1_relative_eef_relative_joints": 25,
"real_r1_pro_sharpa_relative_eef": 26,
"real_r1_pro_sharpa_relative_eef_human": 26,
"real_r1_pro_sharpa_relative_eef_maxinsights": 26,
"real_r1_pro_sharpa_relative_eef_mecka": 26,
# Posttrain embodiment ids
"unitree_g1_full_body_with_waist_height_nav_cmd": 25,
"unitree_g1_sonic": 11,
"simpler_env_google": 0,
"simpler_env_widowx": 1,
"libero_sim": 2,
"new_embodiment": 10,
}
QWEN3_VL_2B_PROCESSOR = "Qwen/Qwen3-VL-2B-Instruct"
def build_processor(model_name: str, transformers_loading_kwargs: dict) -> ProcessorMixin:
try:
from transformers import Qwen3VLProcessor
except ImportError as exc:
raise ImportError(
"GR00T-N1.7 requires transformers>=4.57.1 for Qwen3VLProcessor "
"(the repo's declared floor is 4.56.0). Please upgrade transformers."
) from exc
if model_name == "nvidia/Cosmos-Reason2-2B":
# Cosmos-Reason2-2B lacks a Qwen3VLProcessor; fall back to upstream artifacts.
logger.warning_once(
"Substituting processor from %s because %s does not ship one. "
"If you fine-tune Cosmos-Reason2-2B's tokenizer/image processor, "
"load the processor explicitly instead of relying on this fallback.",
QWEN3_VL_2B_PROCESSOR,
model_name,
)
model_name = QWEN3_VL_2B_PROCESSOR
return Qwen3VLProcessor.from_pretrained(model_name, **transformers_loading_kwargs)
class Gr00tN1d7DataCollator:
def __init__(
self,
model_name: str,
model_type: str = "qwen",
transformers_loading_kwargs: dict | None = None,
):
if transformers_loading_kwargs is None:
transformers_loading_kwargs = {}
self.processor = build_processor(model_name, transformers_loading_kwargs)
self.processor.tokenizer.padding_side = "left"
self.model_type = model_type
self.model_name = model_name
def __call__(self, features: list[dict[str, Any]]) -> BatchFeature:
batch = {}
keys = list(set().union(*(elem.keys() for elem in features)))
for key in keys:
values = [elem[key] for elem in features if key in elem]
if key == "vlm_content":
text_list = []
image_inputs = []
for v in values:
curr_text_list = [v["text"]]
text_list += curr_text_list
curr_image_inputs = v["images"]
image_inputs += curr_image_inputs
vlm_inputs = self.processor(
text=text_list,
images=image_inputs,
return_tensors="pt",
padding=True,
)
for k, v in vlm_inputs.items():
batch[k] = v
elif key in (
"pixel_values",
"image_grid_thw",
"attention_mask",
"input_ids",
):
raise Exception("Not implemented")
else:
batch[key] = torch.from_numpy(np.stack(values))
return BatchFeature(data={"inputs": batch})
def __str__(self):
return f"Gr00tN1d7DataCollator(model_name={self.model_name}, model_type={self.model_type})"
class Gr00tN1d7Processor(ProcessorMixin):
data_collator_class = Gr00tN1d7DataCollator
def __init__(
self,
modality_configs: dict[str, dict[str, ModalityConfig]],
statistics: (dict[str, dict[str, dict[str, dict[str, list[float]]]]] | None) = None,
use_percentiles: bool = False,
clip_outliers: bool = True,
image_crop_size: list[int] = None,
image_target_size: list[int] = None,
shortest_image_edge: int = 256,
crop_fraction: float = 0.95,
random_rotation_angle: int | None = None,
color_jitter_params: dict[str, float] | None = None,
formalize_language: bool = True,
model_name: str = "nvidia/Cosmos-Reason2-2B",
model_type: str = "qwen",
max_state_dim: int = 29,
max_action_dim: int = 29,
max_action_horizon: int = 50,
apply_sincos_state_encoding: bool = False,
use_relative_action: bool = False,
embodiment_id_mapping: dict[str, int] | None = None,
transformers_loading_kwargs: dict | None = None,
exclude_state: bool = False,
# Normalization
use_mean_std: bool = False,
**kwargs, # absorb deprecated training-only keys from saved processor_config.json
):
if transformers_loading_kwargs is None:
transformers_loading_kwargs = {"trust_remote_code": True}
if kwargs:
logger.debug("Gr00tN1d7Processor: ignoring unknown keys: %s", list(kwargs))
self.modality_configs = parse_modality_configs(modality_configs)
# Initialize StateActionProcessor for state/action normalization
self.state_action_processor = StateActionProcessor(
modality_configs=modality_configs,
statistics=statistics,
use_percentiles=use_percentiles,
clip_outliers=clip_outliers,
apply_sincos_state_encoding=apply_sincos_state_encoding,
use_relative_action=use_relative_action,
)
self.use_percentiles = use_percentiles
self.use_mean_std = use_mean_std
self.clip_outliers = clip_outliers
self.apply_sincos_state_encoding = apply_sincos_state_encoding
self.use_relative_action = use_relative_action
self.exclude_state = exclude_state
self.formalize_language = formalize_language
self.model_name = model_name
self.model_type = model_type
self.max_state_dim = max_state_dim
self.max_action_dim = max_action_dim
self.max_action_horizon = max_action_horizon
self.image_crop_size = image_crop_size
self.image_target_size = image_target_size
self.random_rotation_angle = random_rotation_angle
self.color_jitter_params = color_jitter_params
self.processor = build_processor(model_name, transformers_loading_kwargs)
self.processor.tokenizer.padding_side = "left"
self.embodiment_id_mapping = embodiment_id_mapping or EMBODIMENT_TAG_TO_PROJECTOR_INDEX
for k, v in EMBODIMENT_TAG_TO_PROJECTOR_INDEX.items():
if k not in self.embodiment_id_mapping:
self.embodiment_id_mapping[k] = v
self.shortest_image_edge = shortest_image_edge
self.crop_fraction = crop_fraction
self.statistics: dict[str, dict[str, dict[str, dict[str, list[float]]]]] = {}
# Eval/inference image transform
self.eval_image_transform = _build_eval_image_transform(
image_target_size,
image_crop_size,
)
self._collator = self.data_collator_class(
model_name=model_name,
model_type=model_type,
transformers_loading_kwargs=transformers_loading_kwargs,
)
@property
def collator(self):
return self._collator
def set_statistics(
self,
statistics: dict[str, dict[str, dict[str, dict[str, list[float]]]]],
override: bool = False,
) -> None:
"""Set dataset statistics for normalization."""
for key in statistics:
if key not in self.statistics or override:
if override:
logger.info("Overriding statistics for %s", key)
self.statistics[key] = deepcopy(statistics[key])
else:
logger.debug("Embodiment tag %s already in statistics, skipping update", key)
self.state_action_processor.set_statistics(statistics, override=override)
# Compute action dimensions for convenience
self.action_dim = {}
for embodiment_tag in self.state_action_processor.statistics:
self.action_dim[embodiment_tag] = self.state_action_processor.get_action_dim(embodiment_tag)
def decode_action(
self,
action: np.ndarray,
embodiment_tag: EmbodimentTag,
state: dict[str, np.ndarray] | None = None,
):
"""Undo action normalization and convert relative actions to absolute."""
# Split concatenated action into joint groups
out_dict = {}
start_idx = 0
joint_groups = self.modality_configs[embodiment_tag.value]["action"].modality_keys
action_horizon = len(self.modality_configs[embodiment_tag.value]["action"].delta_indices)
for key in joint_groups:
joint_dim = self.state_action_processor.norm_params[embodiment_tag.value]["action"][key]["dim"].item()
out_dict[key] = action[..., :action_horizon, start_idx : start_idx + joint_dim]
start_idx += joint_dim
# Use StateActionProcessor to unnormalize and convert to absolute
return self.state_action_processor.unapply_action(out_dict, embodiment_tag.value, state=state)
def _apply_vlm_processing(self, images: np.ndarray, language: str) -> BatchFeature:
pil_images = [Image.fromarray(np.transpose(v, (1, 2, 0))) for v in images]
conversation = [
{
"role": "user",
"content": [
*[{"type": "image", "image": img} for img in pil_images],
{"type": "text", "text": language},
],
}
]
text = self.processor.apply_chat_template(conversation, tokenize=False, add_generation_prompt=False)
return {
"vlm_content": {
"text": text,
"images": pil_images,
"conversation": conversation,
}
}
def __call__(
self,
messages: list[dict[str, Any]],
):
assert len(messages) == 1
content = messages[0]["content"]
embodiment_tag = content.embodiment
action_data = content.actions
state_data = content.states
norm_state_dict, normalized_actions = self.state_action_processor.apply(
state=state_data,
action=action_data,
embodiment_tag=embodiment_tag.value,
)
if normalized_actions:
action_keys = self.modality_configs[embodiment_tag.value]["action"].modality_keys
normalized_actions = torch.cat(
[torch.from_numpy(normalized_actions[key]) for key in action_keys],
dim=-1,
) # (t, d)
action_dim = normalized_actions.shape[1]
normalized_actions = torch.cat(
[
normalized_actions,
torch.zeros(
normalized_actions.shape[0],
self.max_action_dim - normalized_actions.shape[1],
),
],
dim=-1,
) # (t, max_action_dim)
action_horizon = normalized_actions.shape[0]
assert action_horizon <= self.max_action_horizon, (
f"Action sequence length {action_horizon} exceeds max_action_horizon"
f" {self.max_action_horizon}. Increase model config action_horizon to"
f" >= {action_horizon}."
)
normalized_actions = torch.cat(
[
normalized_actions,
torch.zeros(
self.max_action_horizon - normalized_actions.shape[0],
self.max_action_dim,
),
],
dim=0,
) # (max_action_horizon, max_action_dim)
action_mask = torch.ones_like(normalized_actions)
action_mask[action_horizon:] = 0
action_mask[:, action_dim:] = 0
else:
normalized_actions = None
action_mask = None
state_keys = self.modality_configs[embodiment_tag.value]["state"].modality_keys
exclude_state = self.exclude_state or getattr(
self.modality_configs[embodiment_tag.value]["state"], "exclude_state", False
)
if exclude_state:
normalized_states = torch.cat(
[torch.from_numpy(np.zeros_like(state_data[key])) for key in state_keys], dim=-1
)
else:
normalized_states = torch.cat([torch.from_numpy(norm_state_dict[key]) for key in state_keys], dim=-1)
normalized_states = torch.cat(
[
normalized_states,
torch.zeros(
normalized_states.shape[0],
self.max_state_dim - normalized_states.shape[1],
),
],
dim=-1,
)
image_transform = self.eval_image_transform
image_keys = self.modality_configs[embodiment_tag.value]["video"].modality_keys
if self.formalize_language:
language = content.text.lower()
language = re.sub(r"[^\w\s]", "", language)
else:
language = content.text
vlm_inputs = self._get_vlm_inputs(
image_keys=image_keys,
images=content.images,
masks=content.masks,
image_transform=image_transform,
language=language,
)
transformed_inputs = {
"state": normalized_states.to(torch.get_default_dtype()),
}
if normalized_actions is not None:
transformed_inputs["action"] = normalized_actions.to(torch.get_default_dtype())
# Add VLM inputs
transformed_inputs.update(vlm_inputs)
if action_mask is not None:
transformed_inputs["action_mask"] = action_mask
transformed_inputs["embodiment_id"] = self.embodiment_id_mapping[embodiment_tag.value]
return transformed_inputs
def _get_vlm_inputs(
self,
image_keys: list[str],
images: list[Image.Image],
masks: dict[str, list[np.ndarray]] | None,
image_transform: transforms.Compose,
language: str,
):
temporal_stacked_images = {}
if masks is not None:
raise ValueError("Mask-based transforms are not supported at inference.")
for view in image_keys:
assert view in images, f"{view} not in {images}"
temporal_stacked_images[view] = torch.stack([image_transform(img) for img in images[view]]) # (T, C, H, W)
for k, v in temporal_stacked_images.items():
assert isinstance(k, str), f"{k} is not a string"
assert isinstance(v, torch.Tensor), f"{v} is not a torch tensor"
assert v.ndim == 4, f"{v} is not a 4D tensor"
assert v.dtype == torch.uint8, f"{v} is not a uint8 tensor"
assert v.shape[1] == 3, f"{v} is not a 3 channel tensor"
stacked_images = (
torch.stack([temporal_stacked_images[view] for view in image_keys], dim=1).flatten(0, 1).numpy()
) # (T*V, C, H, W), processor expects numpy array
vlm_inputs = self._apply_vlm_processing(stacked_images, language)
return vlm_inputs
def save_pretrained(self, save_directory: str | Path) -> list[Path]:
save_directory = Path(save_directory)
save_directory.mkdir(parents=True, exist_ok=True)
main_config_file = save_directory / "processor_config.json"
statistics_file = save_directory / "statistics.json"
embodiment_id_file = save_directory / "embodiment_id.json"
config = {
"processor_class": self.__class__.__name__,
"processor_kwargs": {
"modality_configs": to_json_serializable(self.modality_configs),
"image_crop_size": self.image_crop_size,
"image_target_size": self.image_target_size,
"random_rotation_angle": self.random_rotation_angle,
"color_jitter_params": self.color_jitter_params,
"shortest_image_edge": self.shortest_image_edge,
"crop_fraction": self.crop_fraction,
"model_name": self.model_name,
"model_type": self.model_type,
"formalize_language": self.formalize_language,
"max_state_dim": self.max_state_dim,
"max_action_dim": self.max_action_dim,
"max_action_horizon": self.max_action_horizon,
"use_percentiles": self.use_percentiles,
"use_mean_std": self.use_mean_std,
"clip_outliers": self.clip_outliers,
"apply_sincos_state_encoding": self.apply_sincos_state_encoding,
"use_relative_action": self.use_relative_action,
"exclude_state": self.exclude_state,
},
}
with open(main_config_file, "w") as f:
json.dump(config, f, indent=2)
with open(statistics_file, "w") as f:
json.dump(
to_json_serializable(self.state_action_processor.statistics),
f,
indent=2,
)
with open(embodiment_id_file, "w") as f:
json.dump(self.embodiment_id_mapping, f, indent=2)
return [main_config_file, statistics_file, embodiment_id_file]
@classmethod
def from_pretrained(cls, pretrained_model_name_or_path: str | Path, **kwargs):
transformers_loading_kwargs = kwargs.pop("transformers_loading_kwargs", {"trust_remote_code": True})
pretrained_model_name_or_path = Path(pretrained_model_name_or_path)
config_file = pretrained_model_name_or_path / "processor_config.json"
statistics_file = pretrained_model_name_or_path / "statistics.json"
embodiment_id_file = pretrained_model_name_or_path / "embodiment_id.json"
is_local = os.path.isdir(pretrained_model_name_or_path)
if not is_local:
config_file = Path(cached_file(pretrained_model_name_or_path, "processor_config.json"))
statistics_file = Path(cached_file(pretrained_model_name_or_path, "statistics.json"))
embodiment_id_file = Path(cached_file(pretrained_model_name_or_path, "embodiment_id.json"))
with open(config_file) as f:
config = json.load(f)
with open(statistics_file) as f:
statistics = json.load(f)
if embodiment_id_file.exists():
with open(embodiment_id_file) as f:
embodiment_id_mapping = json.load(f)
else:
embodiment_id_mapping = None
processor_kwargs = config["processor_kwargs"]
processor_kwargs["statistics"] = statistics
processor_kwargs["embodiment_id_mapping"] = embodiment_id_mapping
# Backfill missing fields from older checkpoints.
processor_kwargs.setdefault("model_name", "nvidia/Cosmos-Reason2-2B")
processor_kwargs.setdefault("model_type", "qwen")
processor_kwargs.setdefault("clip_outliers", True)
# Directly override other processor kwargs
if kwargs:
# Override modality configs while keeping pretrained embodiment configs
modality_configs = kwargs.pop("modality_configs", {})
for embodiment_tag, modality_config in modality_configs.items():
processor_kwargs["modality_configs"][embodiment_tag] = modality_config
override_keys = [
"random_rotation_angle",
"color_jitter_params",
"use_relative_action",
"exclude_state",
"use_mean_std",
"model_name",
"model_type",
"max_action_horizon",
"max_state_dim",
"max_action_dim",
]
for key in override_keys:
if key in kwargs:
override = kwargs.pop(key)
if override is not None:
processor_kwargs[key] = override
return cls(**processor_kwargs, transformers_loading_kwargs=transformers_loading_kwargs)
AutoProcessor.register("Gr00tN1d7", Gr00tN1d7Processor)