[Diffusion] add GR00T-N1.7 pipeline with OpenPI serving (#3798)
Signed-off-by: Zhengyuan Su <su.zhengyuan@u.nus.edu> Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
e70822ed54
commit
2d0b1d8a29
34 files changed
+5495
No files matched your search
@@ -0,0 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from vllm_omni.diffusion.models.gr00t.modeling.gr00t_n1d7 import Gr00tN1d7
|
||||
from vllm_omni.diffusion.models.gr00t.modeling.processing_gr00t_n1d7 import Gr00tN1d7Processor
|
||||
|
||||
__all__ = ["Gr00tN1d7", "Gr00tN1d7Processor"]
|
||||
@@ -0,0 +1,491 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import logging
|
||||
import os
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
from transformers import AutoConfig, AutoModel, PreTrainedModel
|
||||
from transformers.feature_extraction_utils import BatchFeature
|
||||
|
||||
from vllm_omni.diffusion.models.gr00t.configs.gr00t_n1d7 import Gr00tN1d7Config
|
||||
from vllm_omni.diffusion.models.gr00t.modeling.modules.dit import AlternateVLDiT, DiT, SelfAttentionTransformer
|
||||
from vllm_omni.diffusion.models.gr00t.modeling.modules.embodiment_conditioned_mlp import (
|
||||
CategorySpecificMLP,
|
||||
MultiEmbodimentActionEncoder,
|
||||
)
|
||||
from vllm_omni.diffusion.models.gr00t.modeling.processing_gr00t_n1d7 import Gr00tN1d7DataCollator
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class Gr00tN1d7ActionHead(nn.Module):
|
||||
"""Action head component for flow matching diffusion policy."""
|
||||
|
||||
supports_gradient_checkpointing = True
|
||||
|
||||
def __init__(self, config: Gr00tN1d7Config):
|
||||
super().__init__()
|
||||
self.config = config
|
||||
self.hidden_size = config.hidden_size
|
||||
self.input_embedding_dim = config.input_embedding_dim
|
||||
|
||||
if config.use_alternate_vl_dit:
|
||||
self.model = AlternateVLDiT(
|
||||
**config.diffusion_model_cfg,
|
||||
cross_attention_dim=config.backbone_embedding_dim,
|
||||
attend_text_every_n_blocks=config.attend_text_every_n_blocks,
|
||||
)
|
||||
logger.info("Using AlternateVLDiT for diffusion model")
|
||||
else:
|
||||
self.model = DiT(
|
||||
**config.diffusion_model_cfg,
|
||||
cross_attention_dim=config.backbone_embedding_dim,
|
||||
)
|
||||
logger.info("Using DiT for diffusion model")
|
||||
self.action_dim = config.max_action_dim
|
||||
self.action_horizon = config.action_horizon
|
||||
self.num_inference_timesteps = config.num_inference_timesteps
|
||||
|
||||
self.state_encoder = CategorySpecificMLP(
|
||||
num_categories=config.max_num_embodiments,
|
||||
input_dim=config.max_state_dim * config.state_history_length,
|
||||
hidden_dim=self.hidden_size,
|
||||
output_dim=self.input_embedding_dim,
|
||||
)
|
||||
self.action_encoder = MultiEmbodimentActionEncoder(
|
||||
action_dim=self.action_dim,
|
||||
hidden_size=self.input_embedding_dim,
|
||||
num_embodiments=config.max_num_embodiments,
|
||||
)
|
||||
self.action_decoder = CategorySpecificMLP(
|
||||
num_categories=config.max_num_embodiments,
|
||||
input_dim=self.hidden_size,
|
||||
hidden_dim=self.hidden_size,
|
||||
output_dim=self.action_dim,
|
||||
)
|
||||
|
||||
self.vlln = nn.LayerNorm(config.backbone_embedding_dim) if config.use_vlln else nn.Identity()
|
||||
|
||||
vl_self_attention_cfg = config.vl_self_attention_cfg
|
||||
if vl_self_attention_cfg and vl_self_attention_cfg.get("num_layers", 0) > 0:
|
||||
self.vl_self_attention = SelfAttentionTransformer(**vl_self_attention_cfg)
|
||||
else:
|
||||
self.vl_self_attention = nn.Identity()
|
||||
|
||||
if config.add_pos_embed:
|
||||
self.position_embedding = nn.Embedding(config.max_seq_len, self.input_embedding_dim)
|
||||
nn.init.normal_(self.position_embedding.weight, mean=0.0, std=0.02)
|
||||
|
||||
self.num_timestep_buckets = config.num_timestep_buckets
|
||||
|
||||
def process_backbone_output(self, backbone_output: BatchFeature) -> BatchFeature:
|
||||
backbone_features = backbone_output["backbone_features"]
|
||||
backbone_features = self.vlln(backbone_features)
|
||||
backbone_features = self.vl_self_attention(backbone_features)
|
||||
backbone_output["backbone_features"] = backbone_features
|
||||
return backbone_output
|
||||
|
||||
def _encode_features(self, backbone_output: BatchFeature, action_input: BatchFeature) -> BatchFeature:
|
||||
"""
|
||||
Encode features for the action head.
|
||||
|
||||
Args:
|
||||
backbone_output: Output from the backbone model containing:
|
||||
- backbone_features: [B, seq_len, backbone_embedding_dim]
|
||||
- backbone_attention_mask: [B, seq_len]
|
||||
action_input: Input containing:
|
||||
- state: [B, state_history_length, max_state_dim]
|
||||
- embodiment_id: [B] (embodiment IDs)
|
||||
|
||||
Returns:
|
||||
BatchFeature containing:
|
||||
- backbone_features: [B, seq_len, backbone_embedding_dim]
|
||||
- state_features: [B, 1, input_embedding_dim]
|
||||
"""
|
||||
backbone_output = self.process_backbone_output(backbone_output)
|
||||
|
||||
vl_embeds = backbone_output.backbone_features
|
||||
embodiment_id = action_input.embodiment_id
|
||||
|
||||
state = action_input.state
|
||||
current_T = state.shape[1]
|
||||
assert current_T == self.config.state_history_length, "current_T != state_history_length"
|
||||
# [B, state_history_length, max_state_dim] -> [B, 1, state_history_length * max_state_dim]
|
||||
state = state.view(state.shape[0], 1, -1)
|
||||
|
||||
state_features = self.state_encoder(state, embodiment_id)
|
||||
|
||||
return BatchFeature(data={"backbone_features": vl_embeds, "state_features": state_features})
|
||||
|
||||
@torch.no_grad()
|
||||
def get_action_with_features(
|
||||
self,
|
||||
backbone_features: torch.Tensor,
|
||||
state_features: torch.Tensor,
|
||||
embodiment_id: torch.Tensor,
|
||||
backbone_output: BatchFeature,
|
||||
action_input: BatchFeature,
|
||||
options: dict[str, Any] | None = None,
|
||||
) -> BatchFeature:
|
||||
"""
|
||||
Generate actions using the flow matching diffusion process.
|
||||
|
||||
Args:
|
||||
backbone_features: [B, seq_len, backbone_embedding_dim]
|
||||
state_features: [B, state_horizon, input_embedding_dim]
|
||||
embodiment_id: [B] (embodiment IDs)
|
||||
backbone_output: Output from the backbone model
|
||||
"""
|
||||
vl_embeds = backbone_features
|
||||
|
||||
batch_size = vl_embeds.shape[0]
|
||||
device = vl_embeds.device
|
||||
_seed_env = os.environ.get("GR00T_NOISE_SEED")
|
||||
if _seed_env is not None:
|
||||
_gen = torch.Generator(device=device).manual_seed(int(_seed_env))
|
||||
actions = torch.randn(
|
||||
size=(batch_size, self.config.action_horizon, self.action_dim),
|
||||
dtype=vl_embeds.dtype,
|
||||
device=device,
|
||||
generator=_gen,
|
||||
)
|
||||
else:
|
||||
actions = torch.randn(
|
||||
size=(batch_size, self.config.action_horizon, self.action_dim),
|
||||
dtype=vl_embeds.dtype,
|
||||
device=device,
|
||||
)
|
||||
|
||||
dt = 1.0 / self.num_inference_timesteps
|
||||
vel_strength = torch.ones_like(actions)
|
||||
|
||||
if "action" in action_input:
|
||||
# If action in input when doing get action, it means we want to use RTC.
|
||||
# action_horizon is the action horizon of the input action.
|
||||
# rtc_overlap_steps is the number of steps to overlap with the previous action chunks.
|
||||
# rtc_frozen_steps is the policy inference latency expressed as frozen action steps.
|
||||
# rtc_ramp_rate is the rate of the ramp of denoising the actions.
|
||||
assert options is not None, "options is not None"
|
||||
assert "action_horizon" in options, "action_horizon is not in options"
|
||||
assert "rtc_overlap_steps" in options, "rtc_overlap_steps is not in options"
|
||||
assert "rtc_frozen_steps" in options, "rtc_frozen_steps is not in options"
|
||||
assert "rtc_ramp_rate" in options, "rtc_ramp_rate is not in options"
|
||||
|
||||
action_horizon_before_padding = options["action_horizon"]
|
||||
|
||||
# Use previous action instead of pure noise to do inpainting
|
||||
actions[:, : options["rtc_overlap_steps"], :] = action_input["action"][
|
||||
:,
|
||||
action_horizon_before_padding - options["rtc_overlap_steps"] : action_horizon_before_padding,
|
||||
:,
|
||||
]
|
||||
vel_strength[:, : options["rtc_frozen_steps"], :] = 0.0
|
||||
# NOTE: use an exponential ramp strength to set the remaining unfrozen rtc_steps
|
||||
intermediate_steps = options["rtc_overlap_steps"] - options["rtc_frozen_steps"]
|
||||
# Create exponential ramp from 0 to 1 over intermediate steps
|
||||
t = torch.linspace(0.0, 1.0, intermediate_steps + 2, device=device)
|
||||
ramp = 1 - torch.exp(-options["rtc_ramp_rate"] * t)
|
||||
ramp = ramp / ramp[-1].clamp_min(1e-8) # normalize to [0,1]
|
||||
ramp = ramp[1:-1] # we will only take the middle part of the ramp, ignore the 0.0 and 1.0
|
||||
# Apply ramp to the intermediate steps [batch, intermediate_steps, action_dim]
|
||||
vel_strength[
|
||||
:,
|
||||
options["rtc_frozen_steps"] : options["rtc_overlap_steps"],
|
||||
:,
|
||||
] = ramp[None, :, None].to(device)
|
||||
|
||||
# Run denoising steps.
|
||||
for t in range(self.num_inference_timesteps):
|
||||
t_discretized = t * self.num_timestep_buckets // self.num_inference_timesteps
|
||||
|
||||
# Embed noised action trajectory.
|
||||
timesteps_tensor = torch.full(size=(batch_size,), fill_value=t_discretized, device=device)
|
||||
action_features = self.action_encoder(actions, timesteps_tensor, embodiment_id)
|
||||
# Add position embedding.
|
||||
if self.config.add_pos_embed:
|
||||
pos_ids = torch.arange(action_features.shape[1], dtype=torch.long, device=device)
|
||||
pos_embs = self.position_embedding(pos_ids).unsqueeze(0)
|
||||
action_features = action_features + pos_embs
|
||||
|
||||
# Join vision, language, state and action embedding along sequence dimension.
|
||||
sa_embs = torch.cat((state_features, action_features), dim=1)
|
||||
|
||||
# Run model forward.
|
||||
if self.config.use_alternate_vl_dit:
|
||||
model_output = self.model(
|
||||
hidden_states=sa_embs,
|
||||
encoder_hidden_states=vl_embeds,
|
||||
timestep=timesteps_tensor,
|
||||
image_mask=backbone_output.image_mask,
|
||||
backbone_attention_mask=backbone_output.backbone_attention_mask,
|
||||
)
|
||||
else:
|
||||
model_output = self.model(
|
||||
hidden_states=sa_embs,
|
||||
encoder_hidden_states=vl_embeds,
|
||||
timestep=timesteps_tensor,
|
||||
)
|
||||
pred = self.action_decoder(model_output, embodiment_id)
|
||||
|
||||
pred_velocity = pred[:, -self.action_horizon :]
|
||||
|
||||
# Update actions using euler integration.
|
||||
actions = actions + dt * pred_velocity * vel_strength
|
||||
|
||||
return BatchFeature(
|
||||
data={
|
||||
"action_pred": actions,
|
||||
"backbone_features": vl_embeds,
|
||||
"state_features": state_features,
|
||||
}
|
||||
)
|
||||
|
||||
@torch.no_grad()
|
||||
def get_action(
|
||||
self,
|
||||
backbone_output: BatchFeature,
|
||||
action_input: BatchFeature,
|
||||
options: dict[str, Any] | None = None,
|
||||
) -> BatchFeature:
|
||||
"""
|
||||
Generate actions using the flow matching diffusion process.
|
||||
|
||||
Args:
|
||||
backbone_output: Output from the backbone model containing:
|
||||
- backbone_features: [B, seq_len, backbone_embedding_dim]
|
||||
- backbone_attention_mask: [B, seq_len]
|
||||
action_input: Input containing:
|
||||
- state: [B, state_dim]
|
||||
- embodiment_id: [B] (embodiment IDs)
|
||||
|
||||
Returns:
|
||||
BatchFeature containing:
|
||||
- action_pred: [B, action_horizon, action_dim] predicted actions
|
||||
"""
|
||||
features = self._encode_features(backbone_output, action_input)
|
||||
return self.get_action_with_features(
|
||||
backbone_features=features.backbone_features,
|
||||
state_features=features.state_features,
|
||||
embodiment_id=action_input.embodiment_id,
|
||||
backbone_output=backbone_output,
|
||||
action_input=action_input,
|
||||
options=options,
|
||||
)
|
||||
|
||||
@property
|
||||
def device(self):
|
||||
return next(iter(self.parameters())).device
|
||||
|
||||
@property
|
||||
def dtype(self):
|
||||
return next(iter(self.parameters())).dtype
|
||||
|
||||
def prepare_input(self, batch: dict) -> BatchFeature:
|
||||
"""Prepare input batch for the action head."""
|
||||
return BatchFeature(data=batch)
|
||||
|
||||
|
||||
def _load_qwen3_vl_cls() -> type:
|
||||
"""Import Qwen3-VL lazily so the missing-dep failure is clear and deferred."""
|
||||
try:
|
||||
from transformers.models.qwen3_vl.modeling_qwen3_vl import Qwen3VLForConditionalGeneration
|
||||
except ImportError as exc:
|
||||
raise ImportError(
|
||||
"GR00T-N1.7 requires transformers>=4.57.1 for transformers.models.qwen3_vl "
|
||||
"(the repo's declared floor is 4.56.0). Please upgrade transformers."
|
||||
) from exc
|
||||
return Qwen3VLForConditionalGeneration
|
||||
|
||||
|
||||
class _Qwen3VLBackbone(nn.Module):
|
||||
"""GR00T adapter around the shared Qwen3-VL implementation."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
model_name: str,
|
||||
select_layer: int,
|
||||
backbone_embedding_dim: int,
|
||||
load_bf16: bool,
|
||||
transformers_loading_kwargs: dict[str, Any] | None = None,
|
||||
):
|
||||
super().__init__()
|
||||
backbone_config = AutoConfig.from_pretrained(
|
||||
model_name, **(transformers_loading_kwargs or {"trust_remote_code": True})
|
||||
)
|
||||
backbone_config.text_config._attn_implementation = "sdpa"
|
||||
backbone_config.vision_config._attn_implementation = "sdpa"
|
||||
self.model = _load_qwen3_vl_cls()(backbone_config).eval()
|
||||
if load_bf16:
|
||||
self.model.to(dtype=torch.bfloat16)
|
||||
|
||||
target_layers = select_layer if select_layer >= 0 else len(self.model.model.language_model.layers)
|
||||
while len(self.model.model.language_model.layers) > target_layers:
|
||||
self.model.model.language_model.layers.pop(-1)
|
||||
|
||||
# Drop the unused lm_head: feature-extractor only, and .to(dtype) above unties it (~0.58 GiB).
|
||||
self.model.lm_head = None
|
||||
|
||||
def set_frozen_modules_to_eval_mode(self) -> None:
|
||||
self.eval()
|
||||
|
||||
def prepare_input(self, batch: dict) -> BatchFeature:
|
||||
return BatchFeature(data=batch)
|
||||
|
||||
def forward(self, vl_input: BatchFeature) -> BatchFeature:
|
||||
self.set_frozen_modules_to_eval_mode()
|
||||
keys_to_use = [
|
||||
"input_ids",
|
||||
"attention_mask",
|
||||
"pixel_values",
|
||||
"image_grid_thw",
|
||||
"mm_token_type_ids",
|
||||
]
|
||||
vl_input = {key: vl_input[key] for key in keys_to_use if key in vl_input}
|
||||
# GR00T was trained against the **pre-final-RMSNorm** output of the last
|
||||
# decoder layer (this was `outer.hidden_states[-1]` under HF 4.57.x semantics,
|
||||
# where the @check_model_inputs tie-logic skipped overwriting because
|
||||
# Qwen3VLCausalLMOutputWithPast has no `last_hidden_state` field).
|
||||
# HF >=5.x changed the output_hidden_states tuple plumbing so that
|
||||
# `hidden_states[-1]` is post-norm, which is a different tensor and breaks
|
||||
# the trained action head. Capture the pre-norm tensor via a hook on the
|
||||
# final RMSNorm regardless of transformers version.
|
||||
_captured: list[torch.Tensor] = []
|
||||
|
||||
def _pre_norm_hook(_module, args, _out):
|
||||
_captured.append(args[0])
|
||||
|
||||
norm = self.model.model.language_model.norm
|
||||
handle = norm.register_forward_hook(_pre_norm_hook)
|
||||
try:
|
||||
self.model.model(**vl_input, return_dict=True)
|
||||
finally:
|
||||
handle.remove()
|
||||
if not _captured:
|
||||
raise RuntimeError("Failed to capture pre-norm hidden states from Qwen3-VL backbone")
|
||||
backbone_features = _captured[-1]
|
||||
image_mask = vl_input["input_ids"] == self.model.config.image_token_id
|
||||
attention_mask = vl_input["attention_mask"] == 1
|
||||
return BatchFeature(
|
||||
data={
|
||||
"backbone_features": backbone_features,
|
||||
"backbone_attention_mask": attention_mask,
|
||||
"image_mask": image_mask,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def get_backbone_cls(config: Gr00tN1d7Config):
|
||||
if "nvidia/Cosmos-Reason2" in config.model_name:
|
||||
return _Qwen3VLBackbone
|
||||
raise ValueError(f"Unsupported model name: {config.model_name}")
|
||||
|
||||
|
||||
class Gr00tN1d7(PreTrainedModel):
|
||||
"""Gr00tN1d7: VLA model with Cosmos-Reason2-2B (Qwen3-VL) backbone."""
|
||||
|
||||
config_class = Gr00tN1d7Config
|
||||
supports_gradient_checkpointing = True
|
||||
_tp_plan = {}
|
||||
|
||||
@property
|
||||
def all_tied_weights_keys(self) -> dict[str, Any]:
|
||||
return {}
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config: Gr00tN1d7Config,
|
||||
transformers_loading_kwargs: dict | None = None,
|
||||
):
|
||||
super().__init__(config)
|
||||
if transformers_loading_kwargs is None:
|
||||
transformers_loading_kwargs = {"trust_remote_code": True}
|
||||
self.config = config
|
||||
|
||||
backbone_cls = get_backbone_cls(config)
|
||||
self.backbone = backbone_cls(
|
||||
model_name=config.model_name,
|
||||
select_layer=config.select_layer,
|
||||
backbone_embedding_dim=config.backbone_embedding_dim,
|
||||
load_bf16=config.load_bf16,
|
||||
transformers_loading_kwargs=transformers_loading_kwargs,
|
||||
)
|
||||
|
||||
# Initialize action head
|
||||
self.action_head = Gr00tN1d7ActionHead(config)
|
||||
|
||||
self.collator = Gr00tN1d7DataCollator(
|
||||
model_name=config.model_name,
|
||||
model_type=config.backbone_model_type,
|
||||
transformers_loading_kwargs=transformers_loading_kwargs,
|
||||
)
|
||||
|
||||
def prepare_input(self, inputs: dict) -> tuple[BatchFeature, BatchFeature]:
|
||||
"""Prepare inputs for backbone and action head."""
|
||||
|
||||
# NOTE -- currently the eval code doesn't use collator, so we need to add it here
|
||||
if "vlm_content" in inputs:
|
||||
# Fix for n_envs > 1: Process all environments' VLM content, not just the first
|
||||
vlm_content_list = inputs["vlm_content"]
|
||||
# Ensure vlm_content_list is always a list for consistent processing
|
||||
if not isinstance(vlm_content_list, list):
|
||||
vlm_content_list = [vlm_content_list]
|
||||
|
||||
# Process all VLM contents through the collator
|
||||
prep = self.collator([{"vlm_content": vlm} for vlm in vlm_content_list])["inputs"]
|
||||
inputs.pop("vlm_content")
|
||||
inputs.update(prep)
|
||||
|
||||
backbone_inputs = self.backbone.prepare_input(inputs)
|
||||
action_inputs = self.action_head.prepare_input(inputs)
|
||||
|
||||
backbone_inputs = backbone_inputs.to(device=self.device, dtype=self.dtype)
|
||||
action_inputs = action_inputs.to(device=self.device, dtype=self.dtype)
|
||||
|
||||
return backbone_inputs, action_inputs
|
||||
|
||||
def forward(self, inputs: dict) -> BatchFeature:
|
||||
"""
|
||||
Forward pass through the complete model.
|
||||
|
||||
Args:
|
||||
inputs: Dictionary containing:
|
||||
- Action inputs (state, action, embodiment_id, etc.)
|
||||
|
||||
Returns:
|
||||
BatchFeature containing loss and other outputs
|
||||
"""
|
||||
# Prepare inputs for backbone and action head
|
||||
backbone_inputs, action_inputs = self.prepare_input(inputs)
|
||||
backbone_outputs = self.backbone(backbone_inputs)
|
||||
action_outputs = self.action_head(backbone_outputs, action_inputs)
|
||||
|
||||
return action_outputs
|
||||
|
||||
def get_action(self, inputs: dict, options: dict[str, Any] | None = None) -> BatchFeature:
|
||||
"""
|
||||
Generate actions using the complete model.
|
||||
"""
|
||||
# Prepare inputs for backbone and action head
|
||||
backbone_inputs, action_inputs = self.prepare_input(inputs)
|
||||
|
||||
# Forward through backbone
|
||||
backbone_outputs = self.backbone(backbone_inputs)
|
||||
action_outputs = self.action_head.get_action(backbone_outputs, action_inputs, options)
|
||||
|
||||
return action_outputs
|
||||
|
||||
@property
|
||||
def device(self):
|
||||
return next(iter(self.parameters())).device
|
||||
|
||||
@property
|
||||
def dtype(self):
|
||||
return next(iter(self.parameters())).dtype
|
||||
|
||||
|
||||
# Register the model with HuggingFace
|
||||
AutoConfig.register("Gr00tN1d7", Gr00tN1d7Config)
|
||||
AutoModel.register(Gr00tN1d7Config, Gr00tN1d7)
|
||||
@@ -0,0 +1,2 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
@@ -0,0 +1,344 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
from diffusers import ConfigMixin, ModelMixin
|
||||
from diffusers.configuration_utils import register_to_config
|
||||
from diffusers.models.attention import Attention, FeedForward
|
||||
from diffusers.models.embeddings import SinusoidalPositionalEmbedding, TimestepEmbedding, Timesteps
|
||||
from torch import nn
|
||||
|
||||
|
||||
class AdaLayerNorm(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
embedding_dim: int,
|
||||
norm_elementwise_affine: bool = False,
|
||||
norm_eps: float = 1e-5,
|
||||
chunk_dim: int = 0,
|
||||
):
|
||||
super().__init__()
|
||||
self.chunk_dim = chunk_dim
|
||||
output_dim = embedding_dim * 2
|
||||
self.silu = nn.SiLU()
|
||||
self.linear = nn.Linear(embedding_dim, output_dim)
|
||||
self.norm = nn.LayerNorm(output_dim // 2, norm_eps, norm_elementwise_affine)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
x: torch.Tensor,
|
||||
temb: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
temb = self.linear(self.silu(temb))
|
||||
scale, shift = temb.chunk(2, dim=1)
|
||||
x = self.norm(x) * (1 + scale[:, None]) + shift[:, None]
|
||||
return x
|
||||
|
||||
|
||||
class BasicTransformerBlock(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
dim: int,
|
||||
num_attention_heads: int,
|
||||
attention_head_dim: int,
|
||||
dropout=0.0,
|
||||
cross_attention_dim: int | None = None,
|
||||
activation_fn: str = "geglu",
|
||||
attention_bias: bool = False,
|
||||
upcast_attention: bool = False,
|
||||
norm_elementwise_affine: bool = True,
|
||||
norm_type: str = "layer_norm",
|
||||
norm_eps: float = 1e-5,
|
||||
final_dropout: bool = False,
|
||||
attention_type: str = "default",
|
||||
positional_embeddings: str | None = None,
|
||||
num_positional_embeddings: int | None = None,
|
||||
ff_inner_dim: int | None = None,
|
||||
ff_bias: bool = True,
|
||||
attention_out_bias: bool = True,
|
||||
):
|
||||
super().__init__()
|
||||
self.norm_type = norm_type
|
||||
|
||||
if positional_embeddings and (num_positional_embeddings is None):
|
||||
raise ValueError(
|
||||
"If `positional_embedding` type is defined, `num_position_embeddings` must also be defined."
|
||||
)
|
||||
|
||||
if positional_embeddings == "sinusoidal":
|
||||
self.pos_embed = SinusoidalPositionalEmbedding(dim, max_seq_length=num_positional_embeddings)
|
||||
else:
|
||||
self.pos_embed = None
|
||||
|
||||
if norm_type == "ada_norm":
|
||||
self.norm1 = AdaLayerNorm(dim)
|
||||
else:
|
||||
self.norm1 = nn.LayerNorm(dim, elementwise_affine=norm_elementwise_affine, eps=norm_eps)
|
||||
|
||||
self.attn1 = Attention(
|
||||
query_dim=dim,
|
||||
heads=num_attention_heads,
|
||||
dim_head=attention_head_dim,
|
||||
dropout=dropout,
|
||||
bias=attention_bias,
|
||||
cross_attention_dim=cross_attention_dim,
|
||||
upcast_attention=upcast_attention,
|
||||
out_bias=attention_out_bias,
|
||||
)
|
||||
|
||||
self.norm3 = nn.LayerNorm(dim, norm_eps, norm_elementwise_affine)
|
||||
self.ff = FeedForward(
|
||||
dim,
|
||||
dropout=dropout,
|
||||
activation_fn=activation_fn,
|
||||
final_dropout=final_dropout,
|
||||
inner_dim=ff_inner_dim,
|
||||
bias=ff_bias,
|
||||
)
|
||||
self.final_dropout = nn.Dropout(dropout) if final_dropout else None
|
||||
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
attention_mask: torch.Tensor | None = None,
|
||||
encoder_hidden_states: torch.Tensor | None = None,
|
||||
encoder_attention_mask: torch.Tensor | None = None,
|
||||
temb: torch.LongTensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
if self.norm_type == "ada_norm":
|
||||
norm_hidden_states = self.norm1(hidden_states, temb)
|
||||
else:
|
||||
norm_hidden_states = self.norm1(hidden_states)
|
||||
|
||||
if self.pos_embed is not None:
|
||||
norm_hidden_states = self.pos_embed(norm_hidden_states)
|
||||
|
||||
attn_output = self.attn1(
|
||||
norm_hidden_states,
|
||||
encoder_hidden_states=encoder_hidden_states,
|
||||
attention_mask=(encoder_attention_mask if encoder_hidden_states is not None else attention_mask),
|
||||
)
|
||||
if self.final_dropout:
|
||||
attn_output = self.final_dropout(attn_output)
|
||||
|
||||
hidden_states = attn_output + hidden_states
|
||||
if hidden_states.ndim == 4:
|
||||
hidden_states = hidden_states.squeeze(1)
|
||||
|
||||
norm_hidden_states = self.norm3(hidden_states)
|
||||
ff_output = self.ff(norm_hidden_states)
|
||||
|
||||
hidden_states = ff_output + hidden_states
|
||||
if hidden_states.ndim == 4:
|
||||
hidden_states = hidden_states.squeeze(1)
|
||||
return hidden_states
|
||||
|
||||
|
||||
class TimestepEncoder(nn.Module):
|
||||
def __init__(self, embedding_dim: int):
|
||||
super().__init__()
|
||||
self.time_proj = Timesteps(num_channels=256, flip_sin_to_cos=True, downscale_freq_shift=1)
|
||||
self.timestep_embedder = TimestepEmbedding(in_channels=256, time_embed_dim=embedding_dim)
|
||||
|
||||
def forward(self, timesteps: torch.Tensor) -> torch.Tensor:
|
||||
dtype = next(self.parameters()).dtype
|
||||
return self.timestep_embedder(self.time_proj(timesteps).to(dtype))
|
||||
|
||||
|
||||
class DiT(ModelMixin, ConfigMixin):
|
||||
_supports_gradient_checkpointing = True
|
||||
|
||||
@register_to_config
|
||||
def __init__(
|
||||
self,
|
||||
num_attention_heads: int = 8,
|
||||
attention_head_dim: int = 64,
|
||||
output_dim: int = 26,
|
||||
num_layers: int = 12,
|
||||
dropout: float = 0.1,
|
||||
attention_bias: bool = True,
|
||||
activation_fn: str = "gelu-approximate",
|
||||
num_embeds_ada_norm: int | None = 1000,
|
||||
upcast_attention: bool = False,
|
||||
norm_type: str = "ada_norm",
|
||||
norm_elementwise_affine: bool = False,
|
||||
norm_eps: float = 1e-5,
|
||||
max_num_positional_embeddings: int = 512,
|
||||
compute_dtype=torch.float32,
|
||||
final_dropout: bool = True,
|
||||
positional_embeddings: str | None = "sinusoidal",
|
||||
interleave_self_attention=False,
|
||||
cross_attention_dim: int | None = None,
|
||||
):
|
||||
super().__init__()
|
||||
|
||||
self.inner_dim = num_attention_heads * attention_head_dim
|
||||
self.gradient_checkpointing = False
|
||||
self.timestep_encoder = TimestepEncoder(embedding_dim=self.inner_dim)
|
||||
|
||||
all_blocks = []
|
||||
for idx in range(num_layers):
|
||||
use_self_attn = idx % 2 == 1 and interleave_self_attention
|
||||
curr_cross_attention_dim = cross_attention_dim if not use_self_attn else None
|
||||
all_blocks.append(
|
||||
BasicTransformerBlock(
|
||||
self.inner_dim,
|
||||
num_attention_heads,
|
||||
attention_head_dim,
|
||||
dropout=dropout,
|
||||
activation_fn=activation_fn,
|
||||
attention_bias=attention_bias,
|
||||
upcast_attention=upcast_attention,
|
||||
norm_type=norm_type,
|
||||
norm_elementwise_affine=norm_elementwise_affine,
|
||||
norm_eps=norm_eps,
|
||||
positional_embeddings=positional_embeddings,
|
||||
num_positional_embeddings=max_num_positional_embeddings,
|
||||
final_dropout=final_dropout,
|
||||
cross_attention_dim=curr_cross_attention_dim,
|
||||
)
|
||||
)
|
||||
self.transformer_blocks = nn.ModuleList(all_blocks)
|
||||
self.norm_out = nn.LayerNorm(self.inner_dim, elementwise_affine=False, eps=1e-6)
|
||||
self.proj_out_1 = nn.Linear(self.inner_dim, 2 * self.inner_dim)
|
||||
self.proj_out_2 = nn.Linear(self.inner_dim, output_dim)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
encoder_hidden_states: torch.Tensor,
|
||||
timestep: torch.LongTensor | None = None,
|
||||
encoder_attention_mask: torch.Tensor | None = None,
|
||||
return_all_hidden_states: bool = False,
|
||||
):
|
||||
temb = self.timestep_encoder(timestep)
|
||||
hidden_states = hidden_states.contiguous()
|
||||
encoder_hidden_states = encoder_hidden_states.contiguous()
|
||||
all_hidden_states = [hidden_states]
|
||||
|
||||
for idx, block in enumerate(self.transformer_blocks):
|
||||
if idx % 2 == 1 and self.config.interleave_self_attention:
|
||||
hidden_states = block(hidden_states, temb=temb)
|
||||
else:
|
||||
hidden_states = block(
|
||||
hidden_states,
|
||||
encoder_hidden_states=encoder_hidden_states,
|
||||
encoder_attention_mask=encoder_attention_mask,
|
||||
temb=temb,
|
||||
)
|
||||
all_hidden_states.append(hidden_states)
|
||||
|
||||
shift, scale = self.proj_out_1(F.silu(temb)).chunk(2, dim=1)
|
||||
hidden_states = self.norm_out(hidden_states) * (1 + scale[:, None]) + shift[:, None]
|
||||
if return_all_hidden_states:
|
||||
return self.proj_out_2(hidden_states), all_hidden_states
|
||||
return self.proj_out_2(hidden_states)
|
||||
|
||||
|
||||
class AlternateVLDiT(DiT):
|
||||
def __init__(self, *args, attend_text_every_n_blocks: int = 2, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
self.attend_text_every_n_blocks = attend_text_every_n_blocks
|
||||
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
encoder_hidden_states: torch.Tensor,
|
||||
timestep: torch.LongTensor | None = None,
|
||||
encoder_attention_mask: torch.Tensor | None = None,
|
||||
return_all_hidden_states: bool = False,
|
||||
image_mask: torch.Tensor | None = None,
|
||||
backbone_attention_mask: torch.Tensor | None = None,
|
||||
):
|
||||
assert image_mask is not None, "Image mask is required"
|
||||
assert self.config.interleave_self_attention, "Interleave self attention must be enabled"
|
||||
|
||||
temb = self.timestep_encoder(timestep)
|
||||
hidden_states = hidden_states.contiguous()
|
||||
encoder_hidden_states = encoder_hidden_states.contiguous()
|
||||
|
||||
image_attention_mask = image_mask & backbone_attention_mask
|
||||
non_image_attention_mask = (~image_mask) & backbone_attention_mask
|
||||
|
||||
all_hidden_states = [hidden_states]
|
||||
for idx, block in enumerate(self.transformer_blocks):
|
||||
if idx % 2 == 1:
|
||||
hidden_states = block(hidden_states, temb=temb)
|
||||
else:
|
||||
curr_encoder_attention_mask = (
|
||||
non_image_attention_mask
|
||||
if idx % (2 * self.attend_text_every_n_blocks) == 0
|
||||
else image_attention_mask
|
||||
)
|
||||
hidden_states = block(
|
||||
hidden_states,
|
||||
encoder_hidden_states=encoder_hidden_states,
|
||||
encoder_attention_mask=curr_encoder_attention_mask,
|
||||
temb=temb,
|
||||
)
|
||||
all_hidden_states.append(hidden_states)
|
||||
|
||||
shift, scale = self.proj_out_1(F.silu(temb)).chunk(2, dim=1)
|
||||
hidden_states = self.norm_out(hidden_states) * (1 + scale[:, None]) + shift[:, None]
|
||||
if return_all_hidden_states:
|
||||
return self.proj_out_2(hidden_states), all_hidden_states
|
||||
return self.proj_out_2(hidden_states)
|
||||
|
||||
|
||||
class SelfAttentionTransformer(ModelMixin, ConfigMixin):
|
||||
_supports_gradient_checkpointing = True
|
||||
|
||||
@register_to_config
|
||||
def __init__(
|
||||
self,
|
||||
num_attention_heads: int = 8,
|
||||
attention_head_dim: int = 64,
|
||||
output_dim: int = 26,
|
||||
num_layers: int = 12,
|
||||
dropout: float = 0.1,
|
||||
attention_bias: bool = True,
|
||||
activation_fn: str = "gelu-approximate",
|
||||
num_embeds_ada_norm: int | None = 1000,
|
||||
upcast_attention: bool = False,
|
||||
max_num_positional_embeddings: int = 512,
|
||||
compute_dtype=torch.float32,
|
||||
final_dropout: bool = True,
|
||||
positional_embeddings: str | None = "sinusoidal",
|
||||
interleave_self_attention=False,
|
||||
):
|
||||
super().__init__()
|
||||
|
||||
self.inner_dim = num_attention_heads * attention_head_dim
|
||||
self.gradient_checkpointing = False
|
||||
self.transformer_blocks = nn.ModuleList(
|
||||
[
|
||||
BasicTransformerBlock(
|
||||
self.inner_dim,
|
||||
num_attention_heads,
|
||||
attention_head_dim,
|
||||
dropout=dropout,
|
||||
activation_fn=activation_fn,
|
||||
attention_bias=attention_bias,
|
||||
upcast_attention=upcast_attention,
|
||||
positional_embeddings=positional_embeddings,
|
||||
num_positional_embeddings=max_num_positional_embeddings,
|
||||
final_dropout=final_dropout,
|
||||
)
|
||||
for _ in range(num_layers)
|
||||
]
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
return_all_hidden_states: bool = False,
|
||||
):
|
||||
hidden_states = hidden_states.contiguous()
|
||||
all_hidden_states = [hidden_states]
|
||||
for block in self.transformer_blocks:
|
||||
hidden_states = block(hidden_states)
|
||||
all_hidden_states.append(hidden_states)
|
||||
if return_all_hidden_states:
|
||||
return hidden_states, all_hidden_states
|
||||
return hidden_states
|
||||
@@ -0,0 +1,178 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
from torch import nn
|
||||
|
||||
from vllm_omni.diffusion.utils.flow_matching import SinusoidalPositionalEncoding, swish
|
||||
|
||||
|
||||
class CategorySpecificLinear(nn.Module):
|
||||
"""Linear layer with category-specific weights and biases for multi-embodiment support."""
|
||||
|
||||
def __init__(self, num_categories: int, input_dim: int, hidden_dim: int) -> None:
|
||||
super().__init__()
|
||||
self.num_categories = num_categories
|
||||
# For each category, we have separate weights and biases.
|
||||
self.W = nn.Parameter(0.02 * torch.randn(num_categories, input_dim, hidden_dim))
|
||||
self.b = nn.Parameter(torch.zeros(num_categories, hidden_dim))
|
||||
|
||||
def forward(self, x: torch.Tensor, cat_ids: torch.Tensor) -> torch.Tensor:
|
||||
"""Apply per-category linear projection.
|
||||
|
||||
Args:
|
||||
x: ``[B, T, input_dim]`` input tensor.
|
||||
cat_ids: ``[B]`` category / embodiment IDs.
|
||||
|
||||
Returns:
|
||||
``[B, T, hidden_dim]`` output tensor.
|
||||
"""
|
||||
selected_W = self.W[cat_ids]
|
||||
selected_b = self.b[cat_ids]
|
||||
return torch.bmm(x, selected_W) + selected_b.unsqueeze(1)
|
||||
|
||||
def expand_action_dimension(
|
||||
self,
|
||||
old_action_dim: int,
|
||||
new_action_dim: int,
|
||||
expand_input: bool = False,
|
||||
expand_output: bool = False,
|
||||
) -> None:
|
||||
"""Safely expand action dimension with explicit targeting.
|
||||
|
||||
Args:
|
||||
old_action_dim: Original action dimension.
|
||||
new_action_dim: New (larger) action dimension.
|
||||
expand_input: Whether to expand input dimension (dim=1).
|
||||
expand_output: Whether to expand output dimension (dim=2).
|
||||
"""
|
||||
if new_action_dim <= old_action_dim:
|
||||
raise ValueError(f"New action dim {new_action_dim} must be larger than old action dim {old_action_dim}")
|
||||
|
||||
# Expand input dimension (dim=1) only if explicitly requested AND dimensions match
|
||||
if expand_input and self.W.shape[1] == old_action_dim:
|
||||
repeat_times = new_action_dim // old_action_dim
|
||||
remainder = new_action_dim % old_action_dim
|
||||
|
||||
new_W_parts = [self.W] * repeat_times
|
||||
if remainder > 0:
|
||||
new_W_parts.append(self.W[:, :remainder, :])
|
||||
|
||||
new_W = torch.cat(new_W_parts, dim=1)
|
||||
self.W = nn.Parameter(new_W)
|
||||
|
||||
# Expand output dimension (dim=2) only if explicitly requested AND dimensions match
|
||||
if expand_output and self.W.shape[2] == old_action_dim:
|
||||
repeat_times = new_action_dim // old_action_dim
|
||||
remainder = new_action_dim % old_action_dim
|
||||
|
||||
new_W_parts = [self.W] * repeat_times
|
||||
if remainder > 0:
|
||||
new_W_parts.append(self.W[:, :, :remainder])
|
||||
|
||||
new_W = torch.cat(new_W_parts, dim=2)
|
||||
self.W = nn.Parameter(new_W)
|
||||
|
||||
# Expand bias for output dimension
|
||||
if self.b.shape[1] == old_action_dim:
|
||||
new_b_parts = [self.b] * repeat_times
|
||||
if remainder > 0:
|
||||
new_b_parts.append(self.b[:, :remainder])
|
||||
|
||||
new_b = torch.cat(new_b_parts, dim=1)
|
||||
self.b = nn.Parameter(new_b)
|
||||
|
||||
|
||||
class CategorySpecificMLP(nn.Module):
|
||||
"""Two-layer MLP with category-specific weights for multi-embodiment support."""
|
||||
|
||||
def __init__(self, num_categories: int, input_dim: int, hidden_dim: int, output_dim: int) -> None:
|
||||
super().__init__()
|
||||
self.num_categories = num_categories
|
||||
self.layer1 = CategorySpecificLinear(num_categories, input_dim, hidden_dim)
|
||||
self.layer2 = CategorySpecificLinear(num_categories, hidden_dim, output_dim)
|
||||
|
||||
def forward(self, x: torch.Tensor, cat_ids: torch.Tensor) -> torch.Tensor:
|
||||
"""Apply two-layer per-category MLP.
|
||||
|
||||
Args:
|
||||
x: ``[B, T, input_dim]`` input tensor.
|
||||
cat_ids: ``[B]`` category / embodiment IDs.
|
||||
|
||||
Returns:
|
||||
``[B, T, output_dim]`` output tensor.
|
||||
"""
|
||||
hidden = F.relu(self.layer1(x, cat_ids))
|
||||
return self.layer2(hidden, cat_ids)
|
||||
|
||||
def expand_action_dimension(self, old_action_dim: int, new_action_dim: int) -> None:
|
||||
"""Expand action dimension by copying weights from existing dimensions.
|
||||
|
||||
Args:
|
||||
old_action_dim: Original action dimension.
|
||||
new_action_dim: New (larger) action dimension.
|
||||
"""
|
||||
# self.layer1 does not take action_dim as input, so no expansion needed
|
||||
self.layer2.expand_action_dimension(old_action_dim, new_action_dim, expand_input=False, expand_output=True)
|
||||
|
||||
|
||||
class MultiEmbodimentActionEncoder(nn.Module):
|
||||
"""Action encoder with multi-embodiment support and sinusoidal positional encoding."""
|
||||
|
||||
def __init__(self, action_dim: int, hidden_size: int, num_embodiments: int) -> None:
|
||||
super().__init__()
|
||||
self.hidden_size = hidden_size
|
||||
self.num_embodiments = num_embodiments
|
||||
|
||||
# W1: R^{w x d}, W2: R^{w x 2w}, W3: R^{w x w}
|
||||
self.W1 = CategorySpecificLinear(num_embodiments, action_dim, hidden_size) # (d -> w)
|
||||
self.W2 = CategorySpecificLinear(num_embodiments, 2 * hidden_size, hidden_size) # (2w -> w)
|
||||
self.W3 = CategorySpecificLinear(num_embodiments, hidden_size, hidden_size) # (w -> w)
|
||||
self.pos_encoding = SinusoidalPositionalEncoding(hidden_size)
|
||||
|
||||
def forward(self, actions: torch.Tensor, timesteps: torch.Tensor, cat_ids: torch.Tensor) -> torch.Tensor:
|
||||
"""Encode actions with embodiment-conditioned MLP and time embedding.
|
||||
|
||||
Args:
|
||||
actions: ``[B, T, action_dim]`` action tensor.
|
||||
timesteps: ``[B]`` timesteps (one scalar per batch item).
|
||||
cat_ids: ``[B]`` category / embodiment IDs.
|
||||
|
||||
Returns:
|
||||
``[B, T, hidden_size]`` encoded action features.
|
||||
"""
|
||||
B, T, _ = actions.shape
|
||||
|
||||
# 1) Expand each batch's single scalar time 'tau' across all T steps
|
||||
# so that shape => (B, T)
|
||||
# e.g. if timesteps is (B,), replicate across T
|
||||
if timesteps.dim() == 1 and timesteps.shape[0] == B:
|
||||
# shape (B,) => (B,T)
|
||||
timesteps = timesteps.unsqueeze(1).expand(-1, T)
|
||||
else:
|
||||
raise ValueError("Expected `timesteps` to have shape (B,) so we can replicate across T.")
|
||||
|
||||
# 2) Standard action MLP step for shape => (B, T, w)
|
||||
a_emb = self.W1(actions, cat_ids)
|
||||
|
||||
# 3) Get the sinusoidal encoding (B, T, w)
|
||||
tau_emb = self.pos_encoding(timesteps).to(dtype=a_emb.dtype)
|
||||
|
||||
# 4) Concat along last dim => (B, T, 2w), then W2 => (B, T, w), swish
|
||||
x = torch.cat([a_emb, tau_emb], dim=-1)
|
||||
x = swish(self.W2(x, cat_ids))
|
||||
|
||||
# 5) Finally W3 => (B, T, w)
|
||||
x = self.W3(x, cat_ids)
|
||||
return x
|
||||
|
||||
def expand_action_dimension(self, old_action_dim: int, new_action_dim: int) -> None:
|
||||
"""Expand action dimension by copying weights from existing dimensions.
|
||||
|
||||
Args:
|
||||
old_action_dim: Original action dimension.
|
||||
new_action_dim: New (larger) action dimension.
|
||||
"""
|
||||
# Only W1 takes action_dim as input, so only expand its input dimension
|
||||
self.W1.expand_action_dimension(old_action_dim, new_action_dim, expand_input=True, expand_output=False)
|
||||
@@ -0,0 +1,555 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import warnings
|
||||
from copy import deepcopy
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
import torchvision.transforms.v2 as transforms
|
||||
from PIL import Image
|
||||
from transformers import AutoProcessor, ProcessorMixin
|
||||
from transformers.feature_extraction_utils import BatchFeature
|
||||
from transformers.utils import cached_file
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from vllm_omni.diffusion.models.gr00t.configs.embodiment_configs import ModalityConfig
|
||||
from vllm_omni.diffusion.models.gr00t.dataio.embodiment_tags import EmbodimentTag
|
||||
from vllm_omni.diffusion.models.gr00t.dataio.state_action.state_action_processor import StateActionProcessor
|
||||
from vllm_omni.diffusion.models.gr00t.dataio.utils import parse_modality_configs, to_json_serializable
|
||||
|
||||
|
||||
class LetterBoxTransform:
|
||||
"""Pad image to square dimensions by adding black bars to the smaller side."""
|
||||
|
||||
def __call__(self, img: torch.Tensor) -> torch.Tensor:
|
||||
import math
|
||||
|
||||
*leading_dims, c, h, w = img.shape
|
||||
if h == w:
|
||||
return img
|
||||
max_dim = max(h, w)
|
||||
pad_h, pad_w = max_dim - h, max_dim - w
|
||||
pad_top, pad_left = pad_h // 2, pad_w // 2
|
||||
pad_bottom, pad_right = pad_h - pad_top, pad_w - pad_left
|
||||
if leading_dims:
|
||||
batch_size = math.prod(leading_dims)
|
||||
img_r = img.reshape(batch_size, c, h, w)
|
||||
padded = transforms.functional.pad(img_r, padding=[pad_left, pad_top, pad_right, pad_bottom], fill=0)
|
||||
return padded.reshape(leading_dims + [c, max_dim, max_dim])
|
||||
return transforms.functional.pad(img, padding=[pad_left, pad_top, pad_right, pad_bottom], fill=0)
|
||||
|
||||
|
||||
def _build_eval_image_transform(
|
||||
image_target_size: list[int],
|
||||
image_crop_size: list[int],
|
||||
) -> transforms.Compose:
|
||||
"""Deterministic eval/inference image transform (letterbox → resize → centercrop → resize)."""
|
||||
return transforms.Compose(
|
||||
[
|
||||
transforms.ToImage(),
|
||||
LetterBoxTransform(),
|
||||
transforms.Resize(size=image_target_size),
|
||||
transforms.CenterCrop(size=image_crop_size),
|
||||
transforms.Resize(size=image_target_size),
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# Suppress protobuf deprecation warnings
|
||||
warnings.filterwarnings("ignore", category=DeprecationWarning, module="google.protobuf")
|
||||
|
||||
EMBODIMENT_TAG_TO_PROJECTOR_INDEX = {
|
||||
# Pretrain embodiment ids
|
||||
"oxe_droid_relative_eef_relative_joint": 24,
|
||||
"xdof_relative_eef_relative_joint": 27,
|
||||
"xdof_relative_eef_relative_joint_subtask": 27,
|
||||
"real_g1_relative_eef_relative_joints": 25,
|
||||
"real_r1_pro_sharpa_relative_eef": 26,
|
||||
"real_r1_pro_sharpa_relative_eef_human": 26,
|
||||
"real_r1_pro_sharpa_relative_eef_maxinsights": 26,
|
||||
"real_r1_pro_sharpa_relative_eef_mecka": 26,
|
||||
# Posttrain embodiment ids
|
||||
"unitree_g1_full_body_with_waist_height_nav_cmd": 25,
|
||||
"unitree_g1_sonic": 11,
|
||||
"simpler_env_google": 0,
|
||||
"simpler_env_widowx": 1,
|
||||
"libero_sim": 2,
|
||||
"new_embodiment": 10,
|
||||
}
|
||||
|
||||
QWEN3_VL_2B_PROCESSOR = "Qwen/Qwen3-VL-2B-Instruct"
|
||||
|
||||
|
||||
def build_processor(model_name: str, transformers_loading_kwargs: dict) -> ProcessorMixin:
|
||||
try:
|
||||
from transformers import Qwen3VLProcessor
|
||||
except ImportError as exc:
|
||||
raise ImportError(
|
||||
"GR00T-N1.7 requires transformers>=4.57.1 for Qwen3VLProcessor "
|
||||
"(the repo's declared floor is 4.56.0). Please upgrade transformers."
|
||||
) from exc
|
||||
if model_name == "nvidia/Cosmos-Reason2-2B":
|
||||
# Cosmos-Reason2-2B lacks a Qwen3VLProcessor; fall back to upstream artifacts.
|
||||
logger.warning_once(
|
||||
"Substituting processor from %s because %s does not ship one. "
|
||||
"If you fine-tune Cosmos-Reason2-2B's tokenizer/image processor, "
|
||||
"load the processor explicitly instead of relying on this fallback.",
|
||||
QWEN3_VL_2B_PROCESSOR,
|
||||
model_name,
|
||||
)
|
||||
model_name = QWEN3_VL_2B_PROCESSOR
|
||||
return Qwen3VLProcessor.from_pretrained(model_name, **transformers_loading_kwargs)
|
||||
|
||||
|
||||
class Gr00tN1d7DataCollator:
|
||||
def __init__(
|
||||
self,
|
||||
model_name: str,
|
||||
model_type: str = "qwen",
|
||||
transformers_loading_kwargs: dict | None = None,
|
||||
):
|
||||
if transformers_loading_kwargs is None:
|
||||
transformers_loading_kwargs = {}
|
||||
self.processor = build_processor(model_name, transformers_loading_kwargs)
|
||||
self.processor.tokenizer.padding_side = "left"
|
||||
self.model_type = model_type
|
||||
self.model_name = model_name
|
||||
|
||||
def __call__(self, features: list[dict[str, Any]]) -> BatchFeature:
|
||||
batch = {}
|
||||
keys = list(set().union(*(elem.keys() for elem in features)))
|
||||
|
||||
for key in keys:
|
||||
values = [elem[key] for elem in features if key in elem]
|
||||
if key == "vlm_content":
|
||||
text_list = []
|
||||
image_inputs = []
|
||||
for v in values:
|
||||
curr_text_list = [v["text"]]
|
||||
|
||||
text_list += curr_text_list
|
||||
curr_image_inputs = v["images"]
|
||||
image_inputs += curr_image_inputs
|
||||
|
||||
vlm_inputs = self.processor(
|
||||
text=text_list,
|
||||
images=image_inputs,
|
||||
return_tensors="pt",
|
||||
padding=True,
|
||||
)
|
||||
for k, v in vlm_inputs.items():
|
||||
batch[k] = v
|
||||
elif key in (
|
||||
"pixel_values",
|
||||
"image_grid_thw",
|
||||
"attention_mask",
|
||||
"input_ids",
|
||||
):
|
||||
raise Exception("Not implemented")
|
||||
else:
|
||||
batch[key] = torch.from_numpy(np.stack(values))
|
||||
return BatchFeature(data={"inputs": batch})
|
||||
|
||||
def __str__(self):
|
||||
return f"Gr00tN1d7DataCollator(model_name={self.model_name}, model_type={self.model_type})"
|
||||
|
||||
|
||||
class Gr00tN1d7Processor(ProcessorMixin):
|
||||
data_collator_class = Gr00tN1d7DataCollator
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
modality_configs: dict[str, dict[str, ModalityConfig]],
|
||||
statistics: (dict[str, dict[str, dict[str, dict[str, list[float]]]]] | None) = None,
|
||||
use_percentiles: bool = False,
|
||||
clip_outliers: bool = True,
|
||||
image_crop_size: list[int] = None,
|
||||
image_target_size: list[int] = None,
|
||||
shortest_image_edge: int = 256,
|
||||
crop_fraction: float = 0.95,
|
||||
random_rotation_angle: int | None = None,
|
||||
color_jitter_params: dict[str, float] | None = None,
|
||||
formalize_language: bool = True,
|
||||
model_name: str = "nvidia/Cosmos-Reason2-2B",
|
||||
model_type: str = "qwen",
|
||||
max_state_dim: int = 29,
|
||||
max_action_dim: int = 29,
|
||||
max_action_horizon: int = 50,
|
||||
apply_sincos_state_encoding: bool = False,
|
||||
use_relative_action: bool = False,
|
||||
embodiment_id_mapping: dict[str, int] | None = None,
|
||||
transformers_loading_kwargs: dict | None = None,
|
||||
exclude_state: bool = False,
|
||||
# Normalization
|
||||
use_mean_std: bool = False,
|
||||
**kwargs, # absorb deprecated training-only keys from saved processor_config.json
|
||||
):
|
||||
if transformers_loading_kwargs is None:
|
||||
transformers_loading_kwargs = {"trust_remote_code": True}
|
||||
if kwargs:
|
||||
logger.debug("Gr00tN1d7Processor: ignoring unknown keys: %s", list(kwargs))
|
||||
self.modality_configs = parse_modality_configs(modality_configs)
|
||||
|
||||
# Initialize StateActionProcessor for state/action normalization
|
||||
self.state_action_processor = StateActionProcessor(
|
||||
modality_configs=modality_configs,
|
||||
statistics=statistics,
|
||||
use_percentiles=use_percentiles,
|
||||
clip_outliers=clip_outliers,
|
||||
apply_sincos_state_encoding=apply_sincos_state_encoding,
|
||||
use_relative_action=use_relative_action,
|
||||
)
|
||||
|
||||
self.use_percentiles = use_percentiles
|
||||
self.use_mean_std = use_mean_std
|
||||
self.clip_outliers = clip_outliers
|
||||
self.apply_sincos_state_encoding = apply_sincos_state_encoding
|
||||
self.use_relative_action = use_relative_action
|
||||
|
||||
self.exclude_state = exclude_state
|
||||
|
||||
self.formalize_language = formalize_language
|
||||
self.model_name = model_name
|
||||
self.model_type = model_type
|
||||
|
||||
self.max_state_dim = max_state_dim
|
||||
self.max_action_dim = max_action_dim
|
||||
self.max_action_horizon = max_action_horizon
|
||||
|
||||
self.image_crop_size = image_crop_size
|
||||
self.image_target_size = image_target_size
|
||||
self.random_rotation_angle = random_rotation_angle
|
||||
self.color_jitter_params = color_jitter_params
|
||||
self.processor = build_processor(model_name, transformers_loading_kwargs)
|
||||
self.processor.tokenizer.padding_side = "left"
|
||||
self.embodiment_id_mapping = embodiment_id_mapping or EMBODIMENT_TAG_TO_PROJECTOR_INDEX
|
||||
for k, v in EMBODIMENT_TAG_TO_PROJECTOR_INDEX.items():
|
||||
if k not in self.embodiment_id_mapping:
|
||||
self.embodiment_id_mapping[k] = v
|
||||
self.shortest_image_edge = shortest_image_edge
|
||||
self.crop_fraction = crop_fraction
|
||||
|
||||
self.statistics: dict[str, dict[str, dict[str, dict[str, list[float]]]]] = {}
|
||||
|
||||
# Eval/inference image transform
|
||||
self.eval_image_transform = _build_eval_image_transform(
|
||||
image_target_size,
|
||||
image_crop_size,
|
||||
)
|
||||
self._collator = self.data_collator_class(
|
||||
model_name=model_name,
|
||||
model_type=model_type,
|
||||
transformers_loading_kwargs=transformers_loading_kwargs,
|
||||
)
|
||||
|
||||
@property
|
||||
def collator(self):
|
||||
return self._collator
|
||||
|
||||
def set_statistics(
|
||||
self,
|
||||
statistics: dict[str, dict[str, dict[str, dict[str, list[float]]]]],
|
||||
override: bool = False,
|
||||
) -> None:
|
||||
"""Set dataset statistics for normalization."""
|
||||
for key in statistics:
|
||||
if key not in self.statistics or override:
|
||||
if override:
|
||||
logger.info("Overriding statistics for %s", key)
|
||||
self.statistics[key] = deepcopy(statistics[key])
|
||||
else:
|
||||
logger.debug("Embodiment tag %s already in statistics, skipping update", key)
|
||||
|
||||
self.state_action_processor.set_statistics(statistics, override=override)
|
||||
|
||||
# Compute action dimensions for convenience
|
||||
self.action_dim = {}
|
||||
for embodiment_tag in self.state_action_processor.statistics:
|
||||
self.action_dim[embodiment_tag] = self.state_action_processor.get_action_dim(embodiment_tag)
|
||||
|
||||
def decode_action(
|
||||
self,
|
||||
action: np.ndarray,
|
||||
embodiment_tag: EmbodimentTag,
|
||||
state: dict[str, np.ndarray] | None = None,
|
||||
):
|
||||
"""Undo action normalization and convert relative actions to absolute."""
|
||||
# Split concatenated action into joint groups
|
||||
out_dict = {}
|
||||
start_idx = 0
|
||||
joint_groups = self.modality_configs[embodiment_tag.value]["action"].modality_keys
|
||||
action_horizon = len(self.modality_configs[embodiment_tag.value]["action"].delta_indices)
|
||||
for key in joint_groups:
|
||||
joint_dim = self.state_action_processor.norm_params[embodiment_tag.value]["action"][key]["dim"].item()
|
||||
out_dict[key] = action[..., :action_horizon, start_idx : start_idx + joint_dim]
|
||||
start_idx += joint_dim
|
||||
|
||||
# Use StateActionProcessor to unnormalize and convert to absolute
|
||||
return self.state_action_processor.unapply_action(out_dict, embodiment_tag.value, state=state)
|
||||
|
||||
def _apply_vlm_processing(self, images: np.ndarray, language: str) -> BatchFeature:
|
||||
pil_images = [Image.fromarray(np.transpose(v, (1, 2, 0))) for v in images]
|
||||
conversation = [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
*[{"type": "image", "image": img} for img in pil_images],
|
||||
{"type": "text", "text": language},
|
||||
],
|
||||
}
|
||||
]
|
||||
|
||||
text = self.processor.apply_chat_template(conversation, tokenize=False, add_generation_prompt=False)
|
||||
return {
|
||||
"vlm_content": {
|
||||
"text": text,
|
||||
"images": pil_images,
|
||||
"conversation": conversation,
|
||||
}
|
||||
}
|
||||
|
||||
def __call__(
|
||||
self,
|
||||
messages: list[dict[str, Any]],
|
||||
):
|
||||
assert len(messages) == 1
|
||||
content = messages[0]["content"]
|
||||
embodiment_tag = content.embodiment
|
||||
action_data = content.actions
|
||||
state_data = content.states
|
||||
|
||||
norm_state_dict, normalized_actions = self.state_action_processor.apply(
|
||||
state=state_data,
|
||||
action=action_data,
|
||||
embodiment_tag=embodiment_tag.value,
|
||||
)
|
||||
|
||||
if normalized_actions:
|
||||
action_keys = self.modality_configs[embodiment_tag.value]["action"].modality_keys
|
||||
normalized_actions = torch.cat(
|
||||
[torch.from_numpy(normalized_actions[key]) for key in action_keys],
|
||||
dim=-1,
|
||||
) # (t, d)
|
||||
action_dim = normalized_actions.shape[1]
|
||||
normalized_actions = torch.cat(
|
||||
[
|
||||
normalized_actions,
|
||||
torch.zeros(
|
||||
normalized_actions.shape[0],
|
||||
self.max_action_dim - normalized_actions.shape[1],
|
||||
),
|
||||
],
|
||||
dim=-1,
|
||||
) # (t, max_action_dim)
|
||||
action_horizon = normalized_actions.shape[0]
|
||||
assert action_horizon <= self.max_action_horizon, (
|
||||
f"Action sequence length {action_horizon} exceeds max_action_horizon"
|
||||
f" {self.max_action_horizon}. Increase model config action_horizon to"
|
||||
f" >= {action_horizon}."
|
||||
)
|
||||
normalized_actions = torch.cat(
|
||||
[
|
||||
normalized_actions,
|
||||
torch.zeros(
|
||||
self.max_action_horizon - normalized_actions.shape[0],
|
||||
self.max_action_dim,
|
||||
),
|
||||
],
|
||||
dim=0,
|
||||
) # (max_action_horizon, max_action_dim)
|
||||
action_mask = torch.ones_like(normalized_actions)
|
||||
action_mask[action_horizon:] = 0
|
||||
action_mask[:, action_dim:] = 0
|
||||
else:
|
||||
normalized_actions = None
|
||||
action_mask = None
|
||||
|
||||
state_keys = self.modality_configs[embodiment_tag.value]["state"].modality_keys
|
||||
exclude_state = self.exclude_state or getattr(
|
||||
self.modality_configs[embodiment_tag.value]["state"], "exclude_state", False
|
||||
)
|
||||
if exclude_state:
|
||||
normalized_states = torch.cat(
|
||||
[torch.from_numpy(np.zeros_like(state_data[key])) for key in state_keys], dim=-1
|
||||
)
|
||||
else:
|
||||
normalized_states = torch.cat([torch.from_numpy(norm_state_dict[key]) for key in state_keys], dim=-1)
|
||||
normalized_states = torch.cat(
|
||||
[
|
||||
normalized_states,
|
||||
torch.zeros(
|
||||
normalized_states.shape[0],
|
||||
self.max_state_dim - normalized_states.shape[1],
|
||||
),
|
||||
],
|
||||
dim=-1,
|
||||
)
|
||||
|
||||
image_transform = self.eval_image_transform
|
||||
image_keys = self.modality_configs[embodiment_tag.value]["video"].modality_keys
|
||||
|
||||
if self.formalize_language:
|
||||
language = content.text.lower()
|
||||
language = re.sub(r"[^\w\s]", "", language)
|
||||
else:
|
||||
language = content.text
|
||||
|
||||
vlm_inputs = self._get_vlm_inputs(
|
||||
image_keys=image_keys,
|
||||
images=content.images,
|
||||
masks=content.masks,
|
||||
image_transform=image_transform,
|
||||
language=language,
|
||||
)
|
||||
|
||||
transformed_inputs = {
|
||||
"state": normalized_states.to(torch.get_default_dtype()),
|
||||
}
|
||||
if normalized_actions is not None:
|
||||
transformed_inputs["action"] = normalized_actions.to(torch.get_default_dtype())
|
||||
# Add VLM inputs
|
||||
transformed_inputs.update(vlm_inputs)
|
||||
if action_mask is not None:
|
||||
transformed_inputs["action_mask"] = action_mask
|
||||
transformed_inputs["embodiment_id"] = self.embodiment_id_mapping[embodiment_tag.value]
|
||||
return transformed_inputs
|
||||
|
||||
def _get_vlm_inputs(
|
||||
self,
|
||||
image_keys: list[str],
|
||||
images: list[Image.Image],
|
||||
masks: dict[str, list[np.ndarray]] | None,
|
||||
image_transform: transforms.Compose,
|
||||
language: str,
|
||||
):
|
||||
temporal_stacked_images = {}
|
||||
|
||||
if masks is not None:
|
||||
raise ValueError("Mask-based transforms are not supported at inference.")
|
||||
for view in image_keys:
|
||||
assert view in images, f"{view} not in {images}"
|
||||
temporal_stacked_images[view] = torch.stack([image_transform(img) for img in images[view]]) # (T, C, H, W)
|
||||
|
||||
for k, v in temporal_stacked_images.items():
|
||||
assert isinstance(k, str), f"{k} is not a string"
|
||||
assert isinstance(v, torch.Tensor), f"{v} is not a torch tensor"
|
||||
assert v.ndim == 4, f"{v} is not a 4D tensor"
|
||||
assert v.dtype == torch.uint8, f"{v} is not a uint8 tensor"
|
||||
assert v.shape[1] == 3, f"{v} is not a 3 channel tensor"
|
||||
|
||||
stacked_images = (
|
||||
torch.stack([temporal_stacked_images[view] for view in image_keys], dim=1).flatten(0, 1).numpy()
|
||||
) # (T*V, C, H, W), processor expects numpy array
|
||||
|
||||
vlm_inputs = self._apply_vlm_processing(stacked_images, language)
|
||||
return vlm_inputs
|
||||
|
||||
def save_pretrained(self, save_directory: str | Path) -> list[Path]:
|
||||
save_directory = Path(save_directory)
|
||||
save_directory.mkdir(parents=True, exist_ok=True)
|
||||
main_config_file = save_directory / "processor_config.json"
|
||||
statistics_file = save_directory / "statistics.json"
|
||||
embodiment_id_file = save_directory / "embodiment_id.json"
|
||||
|
||||
config = {
|
||||
"processor_class": self.__class__.__name__,
|
||||
"processor_kwargs": {
|
||||
"modality_configs": to_json_serializable(self.modality_configs),
|
||||
"image_crop_size": self.image_crop_size,
|
||||
"image_target_size": self.image_target_size,
|
||||
"random_rotation_angle": self.random_rotation_angle,
|
||||
"color_jitter_params": self.color_jitter_params,
|
||||
"shortest_image_edge": self.shortest_image_edge,
|
||||
"crop_fraction": self.crop_fraction,
|
||||
"model_name": self.model_name,
|
||||
"model_type": self.model_type,
|
||||
"formalize_language": self.formalize_language,
|
||||
"max_state_dim": self.max_state_dim,
|
||||
"max_action_dim": self.max_action_dim,
|
||||
"max_action_horizon": self.max_action_horizon,
|
||||
"use_percentiles": self.use_percentiles,
|
||||
"use_mean_std": self.use_mean_std,
|
||||
"clip_outliers": self.clip_outliers,
|
||||
"apply_sincos_state_encoding": self.apply_sincos_state_encoding,
|
||||
"use_relative_action": self.use_relative_action,
|
||||
"exclude_state": self.exclude_state,
|
||||
},
|
||||
}
|
||||
with open(main_config_file, "w") as f:
|
||||
json.dump(config, f, indent=2)
|
||||
with open(statistics_file, "w") as f:
|
||||
json.dump(
|
||||
to_json_serializable(self.state_action_processor.statistics),
|
||||
f,
|
||||
indent=2,
|
||||
)
|
||||
with open(embodiment_id_file, "w") as f:
|
||||
json.dump(self.embodiment_id_mapping, f, indent=2)
|
||||
return [main_config_file, statistics_file, embodiment_id_file]
|
||||
|
||||
@classmethod
|
||||
def from_pretrained(cls, pretrained_model_name_or_path: str | Path, **kwargs):
|
||||
transformers_loading_kwargs = kwargs.pop("transformers_loading_kwargs", {"trust_remote_code": True})
|
||||
pretrained_model_name_or_path = Path(pretrained_model_name_or_path)
|
||||
config_file = pretrained_model_name_or_path / "processor_config.json"
|
||||
statistics_file = pretrained_model_name_or_path / "statistics.json"
|
||||
embodiment_id_file = pretrained_model_name_or_path / "embodiment_id.json"
|
||||
is_local = os.path.isdir(pretrained_model_name_or_path)
|
||||
if not is_local:
|
||||
config_file = Path(cached_file(pretrained_model_name_or_path, "processor_config.json"))
|
||||
statistics_file = Path(cached_file(pretrained_model_name_or_path, "statistics.json"))
|
||||
embodiment_id_file = Path(cached_file(pretrained_model_name_or_path, "embodiment_id.json"))
|
||||
|
||||
with open(config_file) as f:
|
||||
config = json.load(f)
|
||||
with open(statistics_file) as f:
|
||||
statistics = json.load(f)
|
||||
if embodiment_id_file.exists():
|
||||
with open(embodiment_id_file) as f:
|
||||
embodiment_id_mapping = json.load(f)
|
||||
else:
|
||||
embodiment_id_mapping = None
|
||||
processor_kwargs = config["processor_kwargs"]
|
||||
processor_kwargs["statistics"] = statistics
|
||||
processor_kwargs["embodiment_id_mapping"] = embodiment_id_mapping
|
||||
|
||||
# Backfill missing fields from older checkpoints.
|
||||
processor_kwargs.setdefault("model_name", "nvidia/Cosmos-Reason2-2B")
|
||||
processor_kwargs.setdefault("model_type", "qwen")
|
||||
processor_kwargs.setdefault("clip_outliers", True)
|
||||
|
||||
# Directly override other processor kwargs
|
||||
if kwargs:
|
||||
# Override modality configs while keeping pretrained embodiment configs
|
||||
modality_configs = kwargs.pop("modality_configs", {})
|
||||
for embodiment_tag, modality_config in modality_configs.items():
|
||||
processor_kwargs["modality_configs"][embodiment_tag] = modality_config
|
||||
override_keys = [
|
||||
"random_rotation_angle",
|
||||
"color_jitter_params",
|
||||
"use_relative_action",
|
||||
"exclude_state",
|
||||
"use_mean_std",
|
||||
"model_name",
|
||||
"model_type",
|
||||
"max_action_horizon",
|
||||
"max_state_dim",
|
||||
"max_action_dim",
|
||||
]
|
||||
for key in override_keys:
|
||||
if key in kwargs:
|
||||
override = kwargs.pop(key)
|
||||
if override is not None:
|
||||
processor_kwargs[key] = override
|
||||
return cls(**processor_kwargs, transformers_loading_kwargs=transformers_loading_kwargs)
|
||||
|
||||
|
||||
AutoProcessor.register("Gr00tN1d7", Gr00tN1d7Processor)
|
||||
Reference in new issue
Block a user