Cosmos3 action policy improvements (#6460)
Signed-off-by: Maciej Bala <mbala@nvidia.com> Signed-off-by: MaciejBalaNV <mbala@nvidia.com> Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
This commit is contained in:
1 parent
c8bef008eb
commit
05ab509269
21 files changed
+863
-105
No files matched your search
@@ -38,13 +38,24 @@ EMBODIMENT_TO_DOMAIN_ID: dict[str, int] = {
|
||||
"bridge_orig_lerobot": 7,
|
||||
"droid_lerobot": 8,
|
||||
"robomind-franka": 8,
|
||||
"embodiment_b": 9,
|
||||
# Backward-compatible alias for the pre-generator Cosmos Framework name.
|
||||
"galbot": 9,
|
||||
"robomind-franka-dual": 12,
|
||||
"robomind-ur": 13,
|
||||
"agibotworld": 15,
|
||||
"embodiment_c_gripper": 15,
|
||||
"embodiment_c_gripper_ext": 15,
|
||||
# Backward-compatible aliases for the pre-generator Cosmos Framework names.
|
||||
"agibot_gear_gripper": 15,
|
||||
"agibot_gear_gripper_ext": 15,
|
||||
"xdof_yam": 16,
|
||||
"molmoact2_yam": 16,
|
||||
"abc_yam": 16,
|
||||
"fractal": 20,
|
||||
"drawanything": 21,
|
||||
"behavior1k_lerobot": 22,
|
||||
"maniparena": 23,
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -138,6 +138,7 @@ from .utils import (
|
||||
ensure_gripper_array,
|
||||
extract_robolab_image,
|
||||
extract_robolab_prompt_image,
|
||||
get_robolab_domain_id,
|
||||
lazy_action_transform_pipeline,
|
||||
make_robolab_action_postprocess_inputs,
|
||||
next_robolab_seed,
|
||||
@@ -1075,7 +1076,7 @@ class Cosmos3OmniDiffusersPipeline(
|
||||
self._guidance_scale = None
|
||||
self._num_timesteps = None
|
||||
self._cosmos3_branch_caches: dict[str, tuple[Any, Any]] | None = None
|
||||
self._robolab_transform = None
|
||||
self._robolab_transforms: dict[bool, Any] = {}
|
||||
|
||||
# Set True by ``enable_cache_for_cosmos3`` when cache-dit is enabled on
|
||||
# this pipeline. Tells the sequential-CFG loop to keep paired
|
||||
@@ -1394,11 +1395,18 @@ class Cosmos3OmniDiffusersPipeline(
|
||||
return val
|
||||
return default
|
||||
|
||||
def _get_robolab_transform(self):
|
||||
if self._robolab_transform is None:
|
||||
def _get_robolab_transform(self, *, format_prompt_as_json: bool = False):
|
||||
transforms = getattr(self, "_robolab_transforms", None)
|
||||
if transforms is None:
|
||||
transforms = {}
|
||||
self._robolab_transforms = transforms
|
||||
if format_prompt_as_json not in transforms:
|
||||
action_dim = int(getattr(self.transformer, "action_dim", 64))
|
||||
self._robolab_transform = lazy_action_transform_pipeline(action_dim)
|
||||
return self._robolab_transform
|
||||
transforms[format_prompt_as_json] = lazy_action_transform_pipeline(
|
||||
action_dim,
|
||||
format_prompt_as_json=format_prompt_as_json,
|
||||
)
|
||||
return transforms[format_prompt_as_json]
|
||||
|
||||
def _build_robolab_policy_inputs(
|
||||
self,
|
||||
@@ -1443,7 +1451,8 @@ class Cosmos3OmniDiffusersPipeline(
|
||||
resolution = str(extra_param("resolution", ROBOLAB_DEFAULT_RESOLUTION))
|
||||
fps = float(extra_param("conditioning_fps", ROBOLAB_DEFAULT_CONDITIONING_FPS))
|
||||
domain_name = str(extra_param("domain_name", ROBOLAB_DEFAULT_DOMAIN_NAME))
|
||||
domain_id = resolve_domain_id(domain_name=domain_name, require_explicit=True)
|
||||
domain_id = get_robolab_domain_id(domain_name)
|
||||
format_prompt_as_json = self._truthy(extra_param("format_prompt_as_json", False))
|
||||
|
||||
if use_state and history_length < 1:
|
||||
raise ValueError("RoboLab history_length must be >= 1 when use_state is true.")
|
||||
@@ -1506,7 +1515,9 @@ class Cosmos3OmniDiffusersPipeline(
|
||||
"action": action,
|
||||
# Cosmos Framework consumes this as an integer conditioning bucket.
|
||||
"conditioning_fps": torch.tensor(fps, dtype=torch.long),
|
||||
"mode": ACTION_MODE_POLICY,
|
||||
# Cosmos Framework 1.2 renamed the internal policy transform mode to
|
||||
# ``wam``. The public vLLM action_mode remains ``policy``.
|
||||
"mode": "wam",
|
||||
"domain_id": torch.tensor(domain_id, dtype=torch.long),
|
||||
"viewpoint": "concat_view",
|
||||
"additional_view_description": ROBOLAB_CONCAT_VIEW_DESCRIPTION,
|
||||
@@ -1514,7 +1525,9 @@ class Cosmos3OmniDiffusersPipeline(
|
||||
if history_action is not None:
|
||||
sample["history_action"] = history_action
|
||||
|
||||
sample = self._get_robolab_transform()(sample, resolution)
|
||||
sample = self._get_robolab_transform(format_prompt_as_json=format_prompt_as_json)(sample, resolution)
|
||||
if isinstance(sample.get("ai_caption"), dict):
|
||||
sample["ai_caption"] = json.dumps(sample["ai_caption"])
|
||||
sequence_plan = sample["sequence_plan"]
|
||||
video_tensor = sample["video"].float() / 127.5 - 1.0
|
||||
raw_action_dim_tensor = sample.get("raw_action_dim")
|
||||
|
||||
@@ -78,6 +78,12 @@ ROBOLAB_CONCAT_VIEW_DESCRIPTION = (
|
||||
"sides, with the robot visible."
|
||||
)
|
||||
|
||||
_LEGACY_EMBODIMENT_ALIASES = {
|
||||
"galbot": "embodiment_b",
|
||||
"agibot_gear_gripper": "embodiment_c_gripper",
|
||||
"agibot_gear_gripper_ext": "embodiment_c_gripper_ext",
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RoboLabPolicyInputs:
|
||||
@@ -252,33 +258,83 @@ def extract_robolab_prompt_image(prompt_data: Any | None) -> np.ndarray | None:
|
||||
return ensure_rgb_uint8_image(image, "multi_modal_data.image")
|
||||
|
||||
|
||||
_ROBOLAB_FRAMEWORK_IMPORT_ERROR = (
|
||||
"Cosmos3 RoboLab policy serving requires a compatible cosmos_framework source on PYTHONPATH."
|
||||
)
|
||||
_ROBOLAB_FRAMEWORK_IMPORTS: dict[str, tuple[str, str]] = {
|
||||
"ActionTransformPipeline": (
|
||||
"cosmos_framework.data.generator.action.utils.transforms",
|
||||
"ActionTransformPipeline",
|
||||
),
|
||||
"convert_rotation": (
|
||||
"cosmos_framework.data.generator.action.utils.pose_utils",
|
||||
"convert_rotation",
|
||||
),
|
||||
"pose_abs_to_rel": (
|
||||
"cosmos_framework.data.generator.action.utils.pose_utils",
|
||||
"pose_abs_to_rel",
|
||||
),
|
||||
"pose_rel_to_abs": (
|
||||
"cosmos_framework.data.generator.action.utils.pose_utils",
|
||||
"pose_rel_to_abs",
|
||||
),
|
||||
"build_abs_pose_from_components": (
|
||||
"cosmos_framework.data.generator.action.utils.pose_utils",
|
||||
"build_abs_pose_from_components",
|
||||
),
|
||||
"FlowUniPCMultistepScheduler": (
|
||||
"cosmos_framework.model.generator.diffusion.samplers.fm_solvers_unipc",
|
||||
"FlowUniPCMultistepScheduler",
|
||||
),
|
||||
"get_domain_id": (
|
||||
"cosmos_framework.data.generator.action.utils.domain_utils",
|
||||
"get_domain_id",
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def lazy_import(module_name: str, symbol_name: str, error_message: str):
|
||||
try:
|
||||
module = import_module(module_name)
|
||||
except ModuleNotFoundError as exc:
|
||||
raise ModuleNotFoundError(error_message) from exc
|
||||
return getattr(module, symbol_name)
|
||||
return getattr(module, symbol_name)
|
||||
except (ImportError, AttributeError) as exc:
|
||||
raise ImportError(
|
||||
f"{error_message} Could not import required symbol {symbol_name!r} from {module_name!r}."
|
||||
) from exc
|
||||
|
||||
|
||||
def lazy_action_transform_pipeline(max_action_dim: int):
|
||||
ActionTransformPipeline = lazy_import(
|
||||
"cosmos_framework.data.vfm.action.transforms",
|
||||
"ActionTransformPipeline",
|
||||
"Cosmos3 RoboLab policy serving requires cosmos_framework on PYTHONPATH so the "
|
||||
"golden ActionTransformPipeline can be reused.",
|
||||
def _load_robolab_framework_symbol(symbol_name: str):
|
||||
module_name, exported_name = _ROBOLAB_FRAMEWORK_IMPORTS[symbol_name]
|
||||
return lazy_import(
|
||||
module_name,
|
||||
exported_name,
|
||||
_ROBOLAB_FRAMEWORK_IMPORT_ERROR,
|
||||
)
|
||||
return ActionTransformPipeline(max_action_dim=max_action_dim, cfg_dropout_rate=0.0)
|
||||
|
||||
|
||||
def preflight_cosmos3_action_framework_imports() -> None:
|
||||
"""Validate every Cosmos Framework symbol used by action policy serving."""
|
||||
for symbol_name in _ROBOLAB_FRAMEWORK_IMPORTS:
|
||||
_load_robolab_framework_symbol(symbol_name)
|
||||
|
||||
|
||||
def lazy_action_transform_pipeline(max_action_dim: int, *, format_prompt_as_json: bool = False):
|
||||
ActionTransformPipeline = _load_robolab_framework_symbol("ActionTransformPipeline")
|
||||
return ActionTransformPipeline(
|
||||
max_action_dim=max_action_dim,
|
||||
cfg_dropout_rate=0.0,
|
||||
format_prompt_as_json=format_prompt_as_json,
|
||||
)
|
||||
|
||||
|
||||
def get_robolab_domain_id(domain_name: str) -> int:
|
||||
get_domain_id = _load_robolab_framework_symbol("get_domain_id")
|
||||
canonical_name = _LEGACY_EMBODIMENT_ALIASES.get(domain_name.lower().strip(), domain_name)
|
||||
return int(get_domain_id(canonical_name))
|
||||
|
||||
|
||||
def build_robolab_unipc_scheduler(num_steps: int, shift: float, device: torch.device):
|
||||
FlowUniPCMultistepScheduler = lazy_import(
|
||||
"cosmos_framework.model.vfm.diffusion.samplers.fm_solvers_unipc",
|
||||
"FlowUniPCMultistepScheduler",
|
||||
(
|
||||
"Cosmos3 RoboLab policy serving requires cosmos_framework on PYTHONPATH so the "
|
||||
"golden FlowUniPCMultistepScheduler can be reused."
|
||||
),
|
||||
)
|
||||
FlowUniPCMultistepScheduler = _load_robolab_framework_symbol("FlowUniPCMultistepScheduler")
|
||||
|
||||
scheduler = FlowUniPCMultistepScheduler(
|
||||
num_train_timesteps=1000,
|
||||
@@ -290,38 +346,22 @@ def build_robolab_unipc_scheduler(num_steps: int, shift: float, device: torch.de
|
||||
|
||||
|
||||
def convert_midtrain_rotation(value: Any, src: str, dst: str) -> np.ndarray:
|
||||
convert_rotation = lazy_import(
|
||||
"cosmos_framework.data.vfm.action.pose_utils",
|
||||
"convert_rotation",
|
||||
"Cosmos3 RoboLab midtrain action serving requires cosmos_framework pose_utils on PYTHONPATH.",
|
||||
)
|
||||
convert_rotation = _load_robolab_framework_symbol("convert_rotation")
|
||||
return convert_rotation(value, src, dst)
|
||||
|
||||
|
||||
def pose_abs_to_rel(*args, **kwargs) -> np.ndarray:
|
||||
pose_abs_to_rel_func = lazy_import(
|
||||
"cosmos_framework.data.vfm.action.pose_utils",
|
||||
"pose_abs_to_rel",
|
||||
"Cosmos3 RoboLab midtrain action serving requires cosmos_framework pose_utils on PYTHONPATH.",
|
||||
)
|
||||
pose_abs_to_rel_func = _load_robolab_framework_symbol("pose_abs_to_rel")
|
||||
return pose_abs_to_rel_func(*args, **kwargs)
|
||||
|
||||
|
||||
def pose_rel_to_abs(*args, **kwargs) -> np.ndarray:
|
||||
pose_rel_to_abs_func = lazy_import(
|
||||
"cosmos_framework.data.vfm.action.pose_utils",
|
||||
"pose_rel_to_abs",
|
||||
"Cosmos3 RoboLab midtrain action serving requires cosmos_framework pose_utils on PYTHONPATH.",
|
||||
)
|
||||
pose_rel_to_abs_func = _load_robolab_framework_symbol("pose_rel_to_abs")
|
||||
return pose_rel_to_abs_func(*args, **kwargs)
|
||||
|
||||
|
||||
def build_abs_pose_from_components(*args, **kwargs) -> np.ndarray:
|
||||
build_abs_pose_from_components_func = lazy_import(
|
||||
"cosmos_framework.data.vfm.action.pose_utils",
|
||||
"build_abs_pose_from_components",
|
||||
"Cosmos3 RoboLab midtrain action serving requires cosmos_framework pose_utils on PYTHONPATH.",
|
||||
)
|
||||
build_abs_pose_from_components_func = _load_robolab_framework_symbol("build_abs_pose_from_components")
|
||||
return build_abs_pose_from_components_func(*args, **kwargs)
|
||||
|
||||
|
||||
|
||||
Reference in new issue
Block a user