Cosmos3 action policy improvements (#6460)
Signed-off-by: Maciej Bala <mbala@nvidia.com> Signed-off-by: MaciejBalaNV <mbala@nvidia.com> Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
This commit is contained in:
1 parent
c8bef008eb
commit
05ab509269
21 files changed
+863
-105
No files matched your search
@@ -78,6 +78,12 @@ ROBOLAB_CONCAT_VIEW_DESCRIPTION = (
|
||||
"sides, with the robot visible."
|
||||
)
|
||||
|
||||
_LEGACY_EMBODIMENT_ALIASES = {
|
||||
"galbot": "embodiment_b",
|
||||
"agibot_gear_gripper": "embodiment_c_gripper",
|
||||
"agibot_gear_gripper_ext": "embodiment_c_gripper_ext",
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RoboLabPolicyInputs:
|
||||
@@ -252,33 +258,83 @@ def extract_robolab_prompt_image(prompt_data: Any | None) -> np.ndarray | None:
|
||||
return ensure_rgb_uint8_image(image, "multi_modal_data.image")
|
||||
|
||||
|
||||
_ROBOLAB_FRAMEWORK_IMPORT_ERROR = (
|
||||
"Cosmos3 RoboLab policy serving requires a compatible cosmos_framework source on PYTHONPATH."
|
||||
)
|
||||
_ROBOLAB_FRAMEWORK_IMPORTS: dict[str, tuple[str, str]] = {
|
||||
"ActionTransformPipeline": (
|
||||
"cosmos_framework.data.generator.action.utils.transforms",
|
||||
"ActionTransformPipeline",
|
||||
),
|
||||
"convert_rotation": (
|
||||
"cosmos_framework.data.generator.action.utils.pose_utils",
|
||||
"convert_rotation",
|
||||
),
|
||||
"pose_abs_to_rel": (
|
||||
"cosmos_framework.data.generator.action.utils.pose_utils",
|
||||
"pose_abs_to_rel",
|
||||
),
|
||||
"pose_rel_to_abs": (
|
||||
"cosmos_framework.data.generator.action.utils.pose_utils",
|
||||
"pose_rel_to_abs",
|
||||
),
|
||||
"build_abs_pose_from_components": (
|
||||
"cosmos_framework.data.generator.action.utils.pose_utils",
|
||||
"build_abs_pose_from_components",
|
||||
),
|
||||
"FlowUniPCMultistepScheduler": (
|
||||
"cosmos_framework.model.generator.diffusion.samplers.fm_solvers_unipc",
|
||||
"FlowUniPCMultistepScheduler",
|
||||
),
|
||||
"get_domain_id": (
|
||||
"cosmos_framework.data.generator.action.utils.domain_utils",
|
||||
"get_domain_id",
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def lazy_import(module_name: str, symbol_name: str, error_message: str):
|
||||
try:
|
||||
module = import_module(module_name)
|
||||
except ModuleNotFoundError as exc:
|
||||
raise ModuleNotFoundError(error_message) from exc
|
||||
return getattr(module, symbol_name)
|
||||
return getattr(module, symbol_name)
|
||||
except (ImportError, AttributeError) as exc:
|
||||
raise ImportError(
|
||||
f"{error_message} Could not import required symbol {symbol_name!r} from {module_name!r}."
|
||||
) from exc
|
||||
|
||||
|
||||
def lazy_action_transform_pipeline(max_action_dim: int):
|
||||
ActionTransformPipeline = lazy_import(
|
||||
"cosmos_framework.data.vfm.action.transforms",
|
||||
"ActionTransformPipeline",
|
||||
"Cosmos3 RoboLab policy serving requires cosmos_framework on PYTHONPATH so the "
|
||||
"golden ActionTransformPipeline can be reused.",
|
||||
def _load_robolab_framework_symbol(symbol_name: str):
|
||||
module_name, exported_name = _ROBOLAB_FRAMEWORK_IMPORTS[symbol_name]
|
||||
return lazy_import(
|
||||
module_name,
|
||||
exported_name,
|
||||
_ROBOLAB_FRAMEWORK_IMPORT_ERROR,
|
||||
)
|
||||
return ActionTransformPipeline(max_action_dim=max_action_dim, cfg_dropout_rate=0.0)
|
||||
|
||||
|
||||
def preflight_cosmos3_action_framework_imports() -> None:
|
||||
"""Validate every Cosmos Framework symbol used by action policy serving."""
|
||||
for symbol_name in _ROBOLAB_FRAMEWORK_IMPORTS:
|
||||
_load_robolab_framework_symbol(symbol_name)
|
||||
|
||||
|
||||
def lazy_action_transform_pipeline(max_action_dim: int, *, format_prompt_as_json: bool = False):
|
||||
ActionTransformPipeline = _load_robolab_framework_symbol("ActionTransformPipeline")
|
||||
return ActionTransformPipeline(
|
||||
max_action_dim=max_action_dim,
|
||||
cfg_dropout_rate=0.0,
|
||||
format_prompt_as_json=format_prompt_as_json,
|
||||
)
|
||||
|
||||
|
||||
def get_robolab_domain_id(domain_name: str) -> int:
|
||||
get_domain_id = _load_robolab_framework_symbol("get_domain_id")
|
||||
canonical_name = _LEGACY_EMBODIMENT_ALIASES.get(domain_name.lower().strip(), domain_name)
|
||||
return int(get_domain_id(canonical_name))
|
||||
|
||||
|
||||
def build_robolab_unipc_scheduler(num_steps: int, shift: float, device: torch.device):
|
||||
FlowUniPCMultistepScheduler = lazy_import(
|
||||
"cosmos_framework.model.vfm.diffusion.samplers.fm_solvers_unipc",
|
||||
"FlowUniPCMultistepScheduler",
|
||||
(
|
||||
"Cosmos3 RoboLab policy serving requires cosmos_framework on PYTHONPATH so the "
|
||||
"golden FlowUniPCMultistepScheduler can be reused."
|
||||
),
|
||||
)
|
||||
FlowUniPCMultistepScheduler = _load_robolab_framework_symbol("FlowUniPCMultistepScheduler")
|
||||
|
||||
scheduler = FlowUniPCMultistepScheduler(
|
||||
num_train_timesteps=1000,
|
||||
@@ -290,38 +346,22 @@ def build_robolab_unipc_scheduler(num_steps: int, shift: float, device: torch.de
|
||||
|
||||
|
||||
def convert_midtrain_rotation(value: Any, src: str, dst: str) -> np.ndarray:
|
||||
convert_rotation = lazy_import(
|
||||
"cosmos_framework.data.vfm.action.pose_utils",
|
||||
"convert_rotation",
|
||||
"Cosmos3 RoboLab midtrain action serving requires cosmos_framework pose_utils on PYTHONPATH.",
|
||||
)
|
||||
convert_rotation = _load_robolab_framework_symbol("convert_rotation")
|
||||
return convert_rotation(value, src, dst)
|
||||
|
||||
|
||||
def pose_abs_to_rel(*args, **kwargs) -> np.ndarray:
|
||||
pose_abs_to_rel_func = lazy_import(
|
||||
"cosmos_framework.data.vfm.action.pose_utils",
|
||||
"pose_abs_to_rel",
|
||||
"Cosmos3 RoboLab midtrain action serving requires cosmos_framework pose_utils on PYTHONPATH.",
|
||||
)
|
||||
pose_abs_to_rel_func = _load_robolab_framework_symbol("pose_abs_to_rel")
|
||||
return pose_abs_to_rel_func(*args, **kwargs)
|
||||
|
||||
|
||||
def pose_rel_to_abs(*args, **kwargs) -> np.ndarray:
|
||||
pose_rel_to_abs_func = lazy_import(
|
||||
"cosmos_framework.data.vfm.action.pose_utils",
|
||||
"pose_rel_to_abs",
|
||||
"Cosmos3 RoboLab midtrain action serving requires cosmos_framework pose_utils on PYTHONPATH.",
|
||||
)
|
||||
pose_rel_to_abs_func = _load_robolab_framework_symbol("pose_rel_to_abs")
|
||||
return pose_rel_to_abs_func(*args, **kwargs)
|
||||
|
||||
|
||||
def build_abs_pose_from_components(*args, **kwargs) -> np.ndarray:
|
||||
build_abs_pose_from_components_func = lazy_import(
|
||||
"cosmos_framework.data.vfm.action.pose_utils",
|
||||
"build_abs_pose_from_components",
|
||||
"Cosmos3 RoboLab midtrain action serving requires cosmos_framework pose_utils on PYTHONPATH.",
|
||||
)
|
||||
build_abs_pose_from_components_func = _load_robolab_framework_symbol("build_abs_pose_from_components")
|
||||
return build_abs_pose_from_components_func(*args, **kwargs)
|
||||
|
||||
|
||||
|
||||
Reference in new issue
Block a user