Cosmos3 action policy improvements (#6460)

Signed-off-by: Maciej Bala <mbala@nvidia.com>
Signed-off-by: MaciejBalaNV <mbala@nvidia.com>
Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
This commit is contained in:
MaciejBalaNVandHongsheng Liu authored and GitHub committed 2026-09-10 22:32:50 +08:00
1 parent c8bef008eb
commit 05ab509269
21 files changed
+863 -105

No files matched your search

+71
View File
@@ -197,6 +197,25 @@ class TestStageConfig:
omega_config = config.to_omegaconf()
assert omega_config.engine_args.max_num_seqs == 32
def test_to_omegaconf_dict_override_deep_merges_yaml_dict(self):
"""A partial dict CLI override (e.g. --no-guardrails riding on
model_config) must layer onto the YAML dict, not clobber siblings."""
config = StageConfig(
stage_id=0,
model_stage="diffusion",
stage_type=StageType.DIFFUSION,
yaml_engine_args={
"model_config": {
"guardrails": True,
"policy_server_config": {"action_space": "joint_position"},
}
},
runtime_overrides={"model_config": {"guardrails": False}},
)
omega_config = config.to_omegaconf()
assert omega_config.engine_args.model_config.guardrails is False
assert omega_config.engine_args.model_config.policy_server_config.action_space == "joint_position"
def test_to_omegaconf_diffusion_parallel_overrides_replace_nested_values(self):
config = StageConfig(
stage_id=1,
@@ -623,6 +642,58 @@ class TestPipelineDiscovery:
assert p.hf_architectures == ("SomeCollidingArch",)
class TestCosmos3PolicyPipeline:
"""Cosmos3 policy serving resolves via deploy yaml selection (π0 precedent)."""
def test_registered_without_capturing_base_cosmos3_checkpoints(self):
assert "cosmos3_policy" in OMNI_PIPELINES
# T2I/video Cosmos3 checkpoints report model_type=cosmos3_omni and the
# same model_index.json _class_name as policy checkpoints; they must
# keep resolving through the single-stage diffusion fallback, so the
# policy pipeline must not be reachable by auto-detection.
assert "cosmos3_omni" not in OMNI_PIPELINES
pipeline = OMNI_PIPELINES["cosmos3_policy"]
assert pipeline.hf_architectures == ()
assert pipeline.diffusers_class_name is None
def test_droid_deploy_yaml_carries_policy_server_config(self):
deploy = load_deploy_config(get_deploy_config_path("cosmos3_policy_droid.yaml"))
assert deploy.pipeline == "cosmos3_policy"
stages = merge_pipeline_deploy(OMNI_PIPELINES["cosmos3_policy"], deploy)
assert len(stages) == 1
stage = stages[0].to_omegaconf()
assert stage.stage_type == "diffusion"
assert stage.final_output_type == "action"
assert stage.engine_args.model_class_name == "Cosmos3OmniDiffusersPipeline"
# OpenPI websocket handshake metadata must reach the serving layer via
# engine_args.model_config (ServingRealtimeRobotOpenPI reads it there).
psc = stage.engine_args.model_config.policy_server_config
assert list(psc.image_resolution) == [540, 640]
assert psc.n_external_cameras == 2
assert psc.needs_wrist_camera is True
assert psc.needs_stereo_camera is False
assert psc.needs_session_id is True
assert psc.action_space == "joint_position"
# The DROID checkpoint's training recipe expects JSON-formatted prompts.
assert stage.default_sampling_params.extra_args.format_prompt_as_json is True
def test_no_guardrails_cli_flag_keeps_policy_server_config(self):
"""--no-guardrails becomes a partial model_config CLI override; it must
deep-merge with the deploy yaml's model_config, not clobber the
policy_server_config the OpenPI handshake depends on."""
deploy = load_deploy_config(get_deploy_config_path("cosmos3_policy_droid.yaml"))
stages, _ = StageConfigFactory._create_legacy_from_registry(
OMNI_PIPELINES["cosmos3_policy"],
{"model_config": {"guardrails": False}},
user_deploy_config=deploy,
)
stage = stages[0].to_omegaconf()
assert stage.engine_args.model_config.guardrails is False
assert stage.engine_args.model_config.policy_server_config.action_space == "joint_position"
class TestStagePipelineConfig:
def test_frozen(self):
s = StagePipelineConfig(stage_id=0, model_stage="a")
@@ -0,0 +1,144 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from __future__ import annotations
from importlib.util import find_spec
import pytest
import torch
pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.diffusion]
_FRAMEWORK_AVAILABLE = find_spec("cosmos_framework") is not None
requires_cosmos_framework = pytest.mark.skipif(
not _FRAMEWORK_AVAILABLE,
reason="cosmos_framework source is not present on PYTHONPATH",
)
def test_framework_preflight_checks_every_policy_symbol(monkeypatch) -> None:
from vllm_omni.diffusion.models.cosmos3 import utils as cosmos3_utils
imported = []
def record_import(module_name: str, symbol_name: str, error_message: str):
imported.append((module_name, symbol_name, error_message))
return object()
monkeypatch.setattr(cosmos3_utils, "lazy_import", record_import)
cosmos3_utils.preflight_cosmos3_action_framework_imports()
expected_symbols = {
"ActionTransformPipeline",
"FlowUniPCMultistepScheduler",
"build_abs_pose_from_components",
"convert_rotation",
"get_domain_id",
"pose_abs_to_rel",
"pose_rel_to_abs",
}
assert {symbol_name for _, symbol_name, _ in imported} == expected_symbols
assert [(module_name, symbol_name) for module_name, symbol_name, _ in imported] == list(
cosmos3_utils._ROBOLAB_FRAMEWORK_IMPORTS.values()
)
def test_lazy_import_reports_framework_symbol_skew(monkeypatch) -> None:
from vllm_omni.diffusion.models.cosmos3 import utils as cosmos3_utils
monkeypatch.setattr(cosmos3_utils, "import_module", lambda _module_name: object())
with pytest.raises(ImportError, match="MissingSymbol") as exc_info:
cosmos3_utils.lazy_import(
"cosmos_framework.fake_module",
"MissingSymbol",
"Cosmos Framework compatibility error.",
)
assert isinstance(exc_info.value.__cause__, AttributeError)
@requires_cosmos_framework
def test_action_domain_table_matches_current_framework_and_legacy_aliases() -> None:
from cosmos_framework.data.generator.action.utils.domain_utils import (
EMBODIMENT_TO_DOMAIN_ID as FRAMEWORK_DOMAIN_IDS,
)
from vllm_omni.diffusion.models.cosmos3.action import EMBODIMENT_TO_DOMAIN_ID as VLLM_DOMAIN_IDS
from vllm_omni.diffusion.models.cosmos3.utils import _LEGACY_EMBODIMENT_ALIASES
canonical_vllm_domain_ids = {
name: domain_id for name, domain_id in VLLM_DOMAIN_IDS.items() if name not in _LEGACY_EMBODIMENT_ALIASES
}
assert canonical_vllm_domain_ids == FRAMEWORK_DOMAIN_IDS
assert {alias: VLLM_DOMAIN_IDS[alias] for alias in _LEGACY_EMBODIMENT_ALIASES} == {
alias: FRAMEWORK_DOMAIN_IDS[canonical_name] for alias, canonical_name in _LEGACY_EMBODIMENT_ALIASES.items()
}
@requires_cosmos_framework
def test_current_cosmos_framework_policy_modules_import_and_instantiate() -> None:
from cosmos_framework.data.generator.action.utils.domain_utils import get_domain_id
from cosmos_framework.data.generator.action.utils.pose_utils import (
build_abs_pose_from_components,
convert_rotation,
pose_abs_to_rel,
pose_rel_to_abs,
)
from cosmos_framework.data.generator.action.utils.transforms import ActionTransformPipeline
from cosmos_framework.model.generator.diffusion.samplers.fm_solvers_unipc import (
FlowUniPCMultistepScheduler,
)
transform = ActionTransformPipeline(
max_action_dim=64,
cfg_dropout_rate=0.0,
format_prompt_as_json=True,
)
scheduler = FlowUniPCMultistepScheduler(
num_train_timesteps=1000,
shift=1.0,
use_dynamic_shifting=False,
)
scheduler.set_timesteps(4, device=torch.device("cpu"), shift=5.0)
assert transform.prompt_json_formatter is not None
assert len(scheduler.timesteps) == 4
assert get_domain_id("droid_lerobot") == 8
assert convert_rotation([0.0, 0.0, 0.0, 1.0], "quat_xyzw", "rot6d").shape == (6,)
assert callable(pose_abs_to_rel)
assert callable(pose_rel_to_abs)
assert callable(build_abs_pose_from_components)
@requires_cosmos_framework
def test_real_action_transform_pipeline_matches_droid_policy_contract() -> None:
from cosmos_framework.data.generator.action.utils.transforms import ActionTransformPipeline
transform = ActionTransformPipeline(
max_action_dim=64,
cfg_dropout_rate=0.0,
format_prompt_as_json=True,
)
sample = {
"ai_caption": "Pick up the cube.",
"video": torch.zeros((3, 3, 192, 320), dtype=torch.uint8),
"action": torch.zeros((3, 8), dtype=torch.float32),
"conditioning_fps": torch.tensor(15, dtype=torch.long),
"mode": "wam",
"domain_id": torch.tensor(8, dtype=torch.long),
"viewpoint": "concat_view",
"additional_view_description": "Wrist view above two exterior views.",
}
transformed = transform(sample, "256")
assert isinstance(transformed["ai_caption"], dict)
assert transformed["action"].shape == (3, 64)
assert int(transformed["raw_action_dim"].item()) == 8
assert transformed["sequence_plan"].has_action is True
assert transformed["mode"] == "wam"
assert int(transformed["domain_id"].item()) == 8
@@ -3,6 +3,7 @@
from __future__ import annotations
import json
import sys
import types
from dataclasses import dataclass
@@ -21,6 +22,20 @@ from vllm_omni.experimental.world_models.session_state import SessionStateManage
pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.diffusion]
@pytest.mark.parametrize(
("alias", "canonical_name"),
[
("galbot", "embodiment_b"),
("agibot_gear_gripper", "embodiment_c_gripper"),
("agibot_gear_gripper_ext", "embodiment_c_gripper_ext"),
],
)
def test_action_domain_table_preserves_legacy_aliases(alias: str, canonical_name: str) -> None:
from vllm_omni.diffusion.models.cosmos3.action import resolve_domain_id
assert resolve_domain_id(domain_name=alias) == resolve_domain_id(domain_name=canonical_name)
def test_pipeline_declares_layerwise_offload_components() -> None:
from vllm_omni.diffusion.models.cosmos3.pipeline_cosmos3 import Cosmos3OmniDiffusersPipeline
@@ -479,6 +494,69 @@ def test_forward_threads_request_id_to_robolab(make_cosmos3_pipeline) -> None:
assert captured["session_id"] == "robolab-request-7"
@pytest.mark.parametrize("format_prompt_as_json", [False, True])
def test_robolab_input_builder_threads_prompt_format_and_uses_wam(
make_cosmos3_pipeline,
monkeypatch: pytest.MonkeyPatch,
format_prompt_as_json: bool,
) -> None:
from vllm_omni.diffusion.models.cosmos3 import pipeline_cosmos3
pipeline = make_cosmos3_pipeline()
pipeline.transformer = StubCosmos3Transformer(action_gen=True, action_dim=64)
captured: dict[str, Any] = {}
def fake_transform(sample, resolution):
captured["sample_mode"] = sample["mode"]
captured["resolution"] = resolution
sample["sequence_plan"] = SimpleNamespace(
condition_frame_indexes_action=[0],
action_start_frame_offset=1,
)
sample["raw_action_dim"] = torch.tensor(8)
sample["image_size"] = torch.tensor([16, 16, 16, 16])
if format_prompt_as_json:
sample["ai_caption"] = {"actions": {"instruction": sample["ai_caption"]}}
return sample
def fake_get_transform(*, format_prompt_as_json: bool):
captured["format_prompt_as_json"] = format_prompt_as_json
return fake_transform
pipeline._get_robolab_transform = fake_get_transform
monkeypatch.setattr(pipeline_cosmos3, "get_robolab_domain_id", lambda name: 8)
obs = {
"prompt": "Pick up the cube.",
"observation/image": np.zeros((16, 16, 3), dtype=np.uint8),
"observation/joint_position": np.zeros(7, dtype=np.float32),
"observation/gripper_position": np.zeros(1, dtype=np.float32),
}
sampling_params = make_sampling_params(
extra_args={
"robot_obs": obs,
"action_chunk_size": 2,
"image_height": 16,
"image_width": 16,
"format_prompt_as_json": format_prompt_as_json,
}
)
inputs = pipeline._build_robolab_policy_inputs(sampling_params, request_id="request-1")
assert inputs is not None
assert captured == {
"sample_mode": "wam",
"resolution": "480",
"format_prompt_as_json": format_prompt_as_json,
}
assert inputs.domain_id == 8
assert inputs.raw_action_dim == 8
if format_prompt_as_json:
assert json.loads(inputs.prompt) == {"actions": {"instruction": "Pick up the cube."}}
else:
assert inputs.prompt == "Pick up the cube."
@pytest.mark.parametrize(
("prompt", "sampling_params", "message"),
[
@@ -310,6 +310,29 @@ def test_handle_connection_closes_websocket_on_idle_timeout(monkeypatch):
serving.infer.assert_not_called()
def test_handle_connection_can_disable_idle_timeout(monkeypatch):
monkeypatch.setattr(openpi_connection, "_pack", lambda obj: obj)
async def fail_wait_for(*_args, **_kwargs):
pytest.fail("asyncio.wait_for should not be used when the idle timeout is disabled")
monkeypatch.setattr(openpi_connection.asyncio, "wait_for", fail_wait_for)
websocket = FakeWebSocket([{"type": "websocket.disconnect"}])
serving = _serving_mock()
asyncio.run(
openpi_connection.RobotRealtimeConnection(
websocket,
serving,
idle_timeout=None,
).handle_connection()
)
assert websocket.accepted is True
assert websocket.closed is False
serving.infer.assert_not_called()
def test_handle_connection_keeps_session_state_per_websocket(monkeypatch):
monkeypatch.setattr(openpi_connection, "_pack", lambda obj: obj)
requests = {
@@ -343,6 +366,73 @@ def test_handle_connection_keeps_session_state_per_websocket(monkeypatch):
assert calls[2].kwargs == {"session_id": "session-b", "reset": True}
def test_handle_connection_keeps_interleaved_parallel_session_state(monkeypatch):
monkeypatch.setattr(openpi_connection, "_pack", lambda obj: obj)
requests = {
b"a1": {"prompt": "first-a", "session_id": "session-a"},
b"b1": {"prompt": "first-b", "session_id": "session-b"},
b"a2": {"prompt": "second-a", "session_id": "session-a"},
b"b2": {"prompt": "second-b", "session_id": "session-b"},
}
monkeypatch.setattr(openpi_connection, "_unpack", lambda data: dict(requests[data]))
serving = _serving_mock()
websocket = FakeWebSocket(
[{"type": "websocket.receive", "bytes": key} for key in (b"a1", b"b1", b"a2", b"b2")]
+ [{"type": "websocket.disconnect"}]
)
asyncio.run(openpi_connection.RobotRealtimeConnection(websocket, serving).handle_connection())
assert [call.kwargs for call in serving.infer.await_args_list] == [
{"session_id": "session-a", "reset": True},
{"session_id": "session-b", "reset": True},
{"session_id": "session-a", "reset": False},
{"session_id": "session-b", "reset": False},
]
def test_handle_connection_bounds_tracked_sessions_without_resetting_live_ones(monkeypatch):
"""A client may hold one socket across many episodes, minting a fresh
session id per episode and never sending ``reset`` (RoboLab does exactly
this). Session tracking must stay bounded, and evicting finished episodes
must not disturb the sessions that are still running.
"""
monkeypatch.setattr(openpi_connection, "_pack", lambda obj: obj)
monkeypatch.setattr(openpi_connection, "MAX_TRACKED_SESSIONS", 8)
# One long-lived session interleaved with far more short episodes than the cap.
episodes = 40
keys = []
requests = {}
for episode in range(episodes):
for session_id in (f"episode-{episode}", "live-env"):
key = f"{session_id}@{episode}".encode()
keys.append(key)
requests[key] = {"prompt": "pick", "session_id": session_id}
monkeypatch.setattr(openpi_connection, "_unpack", lambda data: dict(requests[data]))
serving = _serving_mock()
websocket = FakeWebSocket(
[{"type": "websocket.receive", "bytes": key} for key in keys] + [{"type": "websocket.disconnect"}]
)
connection = openpi_connection.RobotRealtimeConnection(websocket, serving)
asyncio.run(connection.handle_connection())
assert len(connection._seen_sessions) <= 8
resets_by_session: dict[str, list[bool]] = {}
for call in serving.infer.await_args_list:
resets_by_session.setdefault(call.kwargs["session_id"], []).append(call.kwargs["reset"])
# Each new episode resets exactly once, on its first observation.
for episode in range(episodes):
assert resets_by_session[f"episode-{episode}"] == [True]
# The continuously active session is never evicted, so it resets only once
# despite ``episodes`` other sessions churning past the cap.
assert resets_by_session["live-env"] == [True] + [False] * (episodes - 1)
def test_handle_connection_reset_endpoint_resets_next_infer(monkeypatch):
monkeypatch.setattr(openpi_connection, "_pack", lambda obj: obj)
requests = {
@@ -15,6 +15,7 @@ from starlette.testclient import TestClient
from vllm_omni.entrypoints.openpi import connection as openpi_connection
from vllm_omni.entrypoints.openpi import serving as openpi_serving
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
@@ -177,6 +178,22 @@ def test_create_policy_server_returns_none_without_policy_config():
assert serving is None
def test_legacy_policy_request_defaults_key_no_longer_controls_serving():
od_config = SimpleNamespace(
model_config={
"policy_server_config": {},
"policy_request_defaults": "malformed legacy value",
}
)
serving = openpi_serving.ServingRealtimeRobotOpenPI.create_policy_server(
engine_client=SimpleNamespace(get_diffusion_od_config=lambda: od_config),
model_name="generic-policy",
)
assert serving is not None
def test_policy_server_config_allows_explicit_empty_config():
serving = openpi_serving.ServingRealtimeRobotOpenPI(
engine_client=_engine_with_policy_config(policy_config={}),
@@ -221,6 +238,44 @@ def test_build_request_uses_unique_engine_request_id_per_inference():
assert request_a.request_id != request_b.request_id
def test_build_request_clones_stage_defaults_before_protocol_fields():
default_params = OmniDiffusionSamplingParams(
extra_args={
"format_prompt_as_json": True,
"session_id": "configured-session-must-not-win",
"reset": False,
"nested": {"values": []},
}
)
od_config = SimpleNamespace(model_config={"policy_server_config": {}})
engine_client = SimpleNamespace(
get_diffusion_od_config=lambda: od_config,
default_sampling_params_list=[default_params],
)
serving = openpi_serving.ServingRealtimeRobotOpenPI(
engine_client=engine_client,
model_name="custom-policy",
)
request_a = serving._build_request(
{"prompt": "pick up the object"},
session_id="wire-session",
reset=True,
)
request_b = serving._build_request(
{"prompt": "pick up the object"},
session_id="wire-session",
reset=False,
)
assert request_a.sampling_params.extra_args["format_prompt_as_json"] is True
assert request_a.sampling_params.extra_args["session_id"] == "wire-session"
assert request_a.sampling_params.extra_args["reset"] is True
request_a.sampling_params.extra_args["nested"]["values"].append("request-a")
assert request_b.sampling_params.extra_args["nested"] == {"values": []}
assert default_params.extra_args["nested"] == {"values": []}
def test_build_request_forwards_seed_to_sampling_params():
"""``seed`` in the inference message is the engine-level seed; omitted, the request auto-seeds."""
serving = openpi_serving.ServingRealtimeRobotOpenPI(engine_client=_engine_with_policy_config())
@@ -257,6 +312,29 @@ def test_infer_keeps_session_state_but_uses_unique_engine_request_ids():
assert sampling_params_b.extra_args["reset"] is False
def test_infer_yields_event_loop_while_engine_is_running():
class DelayedEngine(RecordingEngine):
def generate(self, *, prompt, request_id, sampling_params_list):
async def _generate():
await asyncio.sleep(0.03)
yield SimpleNamespace(multimodal_output={"actions": [0.0]})
return _generate()
serving = openpi_serving.ServingRealtimeRobotOpenPI(engine_client=DelayedEngine())
async def run_test():
infer_task = asyncio.create_task(serving.infer({"prompt": "pick"}, session_id="session-a", reset=True))
ticks = 0
while not infer_task.done():
await asyncio.sleep(0.005)
ticks += 1
await infer_task
return ticks
assert asyncio.run(run_test()) >= 2
def test_two_websocket_clients_without_session_id_do_not_conflict(monkeypatch):
monkeypatch.setattr(openpi_connection, "_pack", _json_pack)
monkeypatch.setattr(openpi_connection, "_unpack", _json_unpack)
+20
View File
@@ -112,6 +112,26 @@ def test_serve_parser_accepts_four_way_cfg_parallelism() -> None:
assert args.cfg_parallel_size == 4
def test_serve_parser_accepts_robot_openpi_idle_timeout() -> None:
parser = TrackingArgumentParser()
subparsers = parser.add_subparsers(dest="subcommand")
OmniServeCommand().subparser_init(subparsers)
args = parser.parse_args(["serve", "fake-model", "--omni", "--robot-openpi-idle-timeout", "0"])
assert args.robot_openpi_idle_timeout == 0
assert args.get_explicit_kwargs_dict()["robot_openpi_idle_timeout"] == 0
def test_serve_parser_rejects_negative_robot_openpi_idle_timeout() -> None:
parser = TrackingArgumentParser()
subparsers = parser.add_subparsers(dest="subcommand")
OmniServeCommand().subparser_init(subparsers)
with pytest.raises(SystemExit):
parser.parse_args(["serve", "fake-model", "--omni", "--robot-openpi-idle-timeout", "-1"])
def test_serve_parser_accepts_ulysses_a2a_permute() -> None:
parser = TrackingArgumentParser()
subparsers = parser.add_subparsers(dest="subcommand")
+1
View File
@@ -177,6 +177,7 @@ def test_cosmos3_extra_registry_declares_request_and_response_params(pipeline_na
"resolution",
"image_size",
"use_state",
"format_prompt_as_json",
"observation",
"robot_obs",
"deterministic_seed",