Add Cosmos3 action modality (#4102)
Signed-off-by: Maciej Bala <mbala@nvidia.com> Signed-off-by: Bartosz Stefaniak <bstefaniak@nvidia.com> Signed-off-by: lishunyang12 <lishunyang12@163.com> Co-authored-by: Maciej Bala <mbala@nvidia.com> Co-authored-by: lishunyang12 <lishunyang12@163.com> Co-authored-by: SYLAR <125541396+lishunyang12@users.noreply.github.com>
This commit is contained in:
15 files changed
+1314
-90
No files matched your search
@@ -0,0 +1,214 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Action-token helpers for Cosmos3 UVA/action generation."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
ACTION_MODE_POLICY = "policy"
|
||||
ACTION_MODE_FORWARD_DYNAMICS = "forward_dynamics"
|
||||
ACTION_MODE_INVERSE_DYNAMICS = "inverse_dynamics"
|
||||
ACTION_MODES = {
|
||||
ACTION_MODE_POLICY,
|
||||
ACTION_MODE_FORWARD_DYNAMICS,
|
||||
ACTION_MODE_INVERSE_DYNAMICS,
|
||||
}
|
||||
|
||||
|
||||
EMBODIMENT_TO_DOMAIN_ID: dict[str, int] = {
|
||||
"no_action": 0,
|
||||
"av": 1,
|
||||
"camera_pose": 2,
|
||||
"hand_pose": 3,
|
||||
"pusht": 4,
|
||||
"libero": 5,
|
||||
"umi": 6,
|
||||
"bridge_orig_lerobot": 7,
|
||||
"droid_lerobot": 8,
|
||||
"robomind-franka": 8,
|
||||
"galbot": 9,
|
||||
"robomind-franka-dual": 12,
|
||||
"robomind-ur": 13,
|
||||
"agibotworld": 15,
|
||||
"agibot_gear_gripper": 15,
|
||||
"agibot_gear_gripper_ext": 15,
|
||||
"fractal": 20,
|
||||
}
|
||||
|
||||
|
||||
VIDEO_RES_SIZE_INFO: dict[str, dict[str, tuple[int, int]]] = {
|
||||
"256": {
|
||||
"1,1": (256, 256),
|
||||
"4,3": (320, 256),
|
||||
"3,4": (256, 320),
|
||||
"16,9": (320, 192),
|
||||
"9,16": (192, 320),
|
||||
},
|
||||
"480": {
|
||||
"1,1": (640, 640),
|
||||
"4,3": (736, 544),
|
||||
"3,4": (544, 736),
|
||||
"16,9": (832, 480),
|
||||
"9,16": (480, 832),
|
||||
},
|
||||
"704": {
|
||||
"1,1": (960, 960),
|
||||
"4,3": (1088, 832),
|
||||
"3,4": (832, 1088),
|
||||
"16,9": (1280, 704),
|
||||
"9,16": (704, 1280),
|
||||
},
|
||||
"720": {
|
||||
"1,1": (960, 960),
|
||||
"4,3": (1104, 832),
|
||||
"3,4": (832, 1104),
|
||||
"16,9": (1280, 720),
|
||||
"9,16": (720, 1280),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def normalize_action_mode(mode: Any) -> str | None:
|
||||
if mode is None:
|
||||
return None
|
||||
normalized = str(mode).strip().lower()
|
||||
if not normalized:
|
||||
return None
|
||||
if normalized not in ACTION_MODES:
|
||||
raise ValueError(f"Unsupported Cosmos3 action_mode={mode!r}; expected one of {sorted(ACTION_MODES)}.")
|
||||
return normalized
|
||||
|
||||
|
||||
def resolve_domain_id(
|
||||
*,
|
||||
domain_id: Any = None,
|
||||
domain_name: Any = None,
|
||||
require_explicit: bool = False,
|
||||
) -> int:
|
||||
if domain_id is not None:
|
||||
resolved = int(domain_id)
|
||||
if resolved < 0:
|
||||
raise ValueError(f"Cosmos3 domain_id must be non-negative, got {resolved}.")
|
||||
return resolved
|
||||
|
||||
if domain_name is None or str(domain_name).strip() == "":
|
||||
if require_explicit:
|
||||
raise ValueError(
|
||||
"Cosmos3 action generation requires extra_args['domain_id'] or non-empty extra_args['domain_name']."
|
||||
)
|
||||
return 0
|
||||
|
||||
key = str(domain_name).strip().lower()
|
||||
if key not in EMBODIMENT_TO_DOMAIN_ID:
|
||||
raise ValueError(
|
||||
f"Unknown Cosmos3 action domain_name={domain_name!r}; "
|
||||
f"expected one of {sorted(EMBODIMENT_TO_DOMAIN_ID)} or pass domain_id directly."
|
||||
)
|
||||
return EMBODIMENT_TO_DOMAIN_ID[key]
|
||||
|
||||
|
||||
def action_condition_indexes(mode: str, action_length: int) -> list[int]:
|
||||
mode = normalize_action_mode(mode)
|
||||
if mode == ACTION_MODE_FORWARD_DYNAMICS:
|
||||
return list(range(action_length))
|
||||
if mode in {ACTION_MODE_POLICY, ACTION_MODE_INVERSE_DYNAMICS}:
|
||||
return []
|
||||
raise AssertionError(f"Unexpected action mode: {mode!r}")
|
||||
|
||||
|
||||
def vision_condition_indexes(mode: str, video_length: int, temporal_compression_factor: int) -> list[int]:
|
||||
mode = normalize_action_mode(mode)
|
||||
latent_frames = (video_length - 1) // temporal_compression_factor + 1
|
||||
if mode in {ACTION_MODE_POLICY, ACTION_MODE_FORWARD_DYNAMICS}:
|
||||
return [0]
|
||||
if mode == ACTION_MODE_INVERSE_DYNAMICS:
|
||||
return list(range(latent_frames))
|
||||
raise AssertionError(f"Unexpected action mode: {mode!r}")
|
||||
|
||||
|
||||
def action_start_frame_offset(mode: str, action_length: int, video_length: int) -> int:
|
||||
del mode
|
||||
if action_length == video_length - 1:
|
||||
return 1
|
||||
if action_length == video_length:
|
||||
return 0
|
||||
raise ValueError(
|
||||
"Cosmos3 action_chunk_size must equal num_frames - 1 or num_frames; "
|
||||
f"got action_chunk_size={action_length}, num_frames={video_length}."
|
||||
)
|
||||
|
||||
|
||||
def build_action_condition_mask(
|
||||
mode: str,
|
||||
action_length: int,
|
||||
*,
|
||||
device: torch.device,
|
||||
dtype: torch.dtype,
|
||||
) -> torch.Tensor:
|
||||
mask = torch.zeros(1, action_length, 1, device=device, dtype=dtype)
|
||||
for idx in action_condition_indexes(mode, action_length):
|
||||
mask[:, idx, :] = 1.0
|
||||
return mask
|
||||
|
||||
|
||||
def build_vision_condition_mask(
|
||||
mode: str,
|
||||
video_length: int,
|
||||
temporal_compression_factor: int,
|
||||
*,
|
||||
device: torch.device,
|
||||
dtype: torch.dtype,
|
||||
) -> torch.Tensor:
|
||||
latent_frames = (video_length - 1) // temporal_compression_factor + 1
|
||||
mask = torch.zeros(1, 1, latent_frames, 1, 1, device=device, dtype=dtype)
|
||||
for idx in vision_condition_indexes(mode, video_length, temporal_compression_factor):
|
||||
mask[:, :, idx, :, :] = 1.0
|
||||
return mask
|
||||
|
||||
|
||||
def pad_action_to_dim(action: torch.Tensor, action_dim: int) -> torch.Tensor:
|
||||
if action.shape[-1] > action_dim:
|
||||
raise ValueError(f"Cosmos3 action dimension {action.shape[-1]} exceeds model action_dim={action_dim}.")
|
||||
if action.shape[-1] == action_dim:
|
||||
return action
|
||||
padding = torch.zeros(*action.shape[:-1], action_dim - action.shape[-1], dtype=action.dtype, device=action.device)
|
||||
return torch.cat([action, padding], dim=-1)
|
||||
|
||||
|
||||
def load_action_tensor(action: Any = None) -> torch.Tensor:
|
||||
if action is None:
|
||||
raise ValueError(
|
||||
"Cosmos3 forward_dynamics action mode requires extra_args['action'] "
|
||||
"(a JSON array / nested list / tensor of shape [T, D])."
|
||||
)
|
||||
if isinstance(action, torch.Tensor):
|
||||
tensor = action.detach().to(dtype=torch.float32)
|
||||
else:
|
||||
tensor = torch.as_tensor(np.asarray(action), dtype=torch.float32)
|
||||
if tensor.ndim == 3 and tensor.shape[0] == 1:
|
||||
tensor = tensor.squeeze(0)
|
||||
if tensor.ndim != 2:
|
||||
raise ValueError(f"Cosmos3 action must have shape [T, D], got {tuple(tensor.shape)}.")
|
||||
return tensor
|
||||
|
||||
|
||||
def find_closest_target_size(h: int, w: int, resolution: str | int) -> tuple[int, int]:
|
||||
key = str(resolution)
|
||||
if key not in VIDEO_RES_SIZE_INFO:
|
||||
raise ValueError(
|
||||
f"Unknown Cosmos3 action resolution={resolution!r}; expected one of {sorted(VIDEO_RES_SIZE_INFO)}."
|
||||
)
|
||||
input_ratio = h / w
|
||||
best_size = None
|
||||
best_diff = float("inf")
|
||||
for cand_w, cand_h in VIDEO_RES_SIZE_INFO[key].values():
|
||||
diff = abs(input_ratio - cand_h / cand_w)
|
||||
if diff < best_diff:
|
||||
best_diff = diff
|
||||
best_size = (cand_w, cand_h)
|
||||
assert best_size is not None
|
||||
return best_size
|
||||
@@ -46,6 +46,19 @@ from vllm_omni.diffusion.profiler.diffusion_pipeline_profiler import DiffusionPi
|
||||
from vllm_omni.diffusion.request import OmniDiffusionRequest
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
|
||||
from .action import (
|
||||
ACTION_MODE_FORWARD_DYNAMICS,
|
||||
ACTION_MODE_INVERSE_DYNAMICS,
|
||||
ACTION_MODE_POLICY,
|
||||
action_start_frame_offset,
|
||||
build_action_condition_mask,
|
||||
build_vision_condition_mask,
|
||||
find_closest_target_size,
|
||||
load_action_tensor,
|
||||
normalize_action_mode,
|
||||
pad_action_to_dim,
|
||||
resolve_domain_id,
|
||||
)
|
||||
from .transformer_cosmos3 import Cosmos3VFMTransformer, resolve_sound_gen
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -98,14 +111,67 @@ def get_cosmos3_pre_process_func(od_config: OmniDiffusionConfig):
|
||||
if is_guardrails_enabled(od_config):
|
||||
ensure_initialized(od_config)
|
||||
|
||||
def _extra_args(request: OmniDiffusionRequest) -> dict[str, Any]:
|
||||
extra = getattr(getattr(request, "sampling_params", None), "extra_args", None)
|
||||
return extra if isinstance(extra, dict) else {}
|
||||
|
||||
def _request_action_mode(request: OmniDiffusionRequest) -> str | None:
|
||||
return normalize_action_mode(_extra_args(request).get("action_mode"))
|
||||
|
||||
def _set_action_size_from_image(request: OmniDiffusionRequest, image: PIL.Image.Image) -> tuple[int, int]:
|
||||
sp = request.sampling_params
|
||||
if sp.height is not None and sp.width is not None:
|
||||
return int(sp.height), int(sp.width)
|
||||
|
||||
extra = _extra_args(request)
|
||||
resolution = extra.get("resolution", extra.get("image_size", 480))
|
||||
target_w, target_h = find_closest_target_size(image.height, image.width, resolution)
|
||||
if sp.height is None:
|
||||
sp.height = target_h
|
||||
if sp.width is None:
|
||||
sp.width = target_w
|
||||
return int(sp.height), int(sp.width)
|
||||
|
||||
def _pil_to_rgb(value: Any) -> PIL.Image.Image:
|
||||
if isinstance(value, str):
|
||||
return PIL.Image.open(value).convert("RGB")
|
||||
if isinstance(value, PIL.Image.Image):
|
||||
return value.convert("RGB")
|
||||
raise TypeError(f"Cosmos3 preprocessing expected PIL image or image path, got {type(value)!r}.")
|
||||
raise TypeError(f"Cosmos3 action preprocessing expected PIL image or image path, got {type(value)!r}.")
|
||||
|
||||
def _resize_and_pad_action_image(image: PIL.Image.Image, target_h: int, target_w: int) -> PIL.Image.Image:
|
||||
scale = min(target_w / image.width, target_h / image.height, 1.0)
|
||||
resize_w = max(1, int(scale * image.width + 0.5))
|
||||
resize_h = max(1, int(scale * image.height + 0.5))
|
||||
if (resize_w, resize_h) != image.size:
|
||||
image = image.resize((resize_w, resize_h), PIL.Image.Resampling.BICUBIC)
|
||||
|
||||
array = np.asarray(image)
|
||||
pad_h = target_h - resize_h
|
||||
pad_w = target_w - resize_w
|
||||
if pad_h < 0 or pad_w < 0:
|
||||
raise ValueError(
|
||||
f"Cosmos3 action image resize exceeded target size: resized={(resize_h, resize_w)}, "
|
||||
f"target={(target_h, target_w)}."
|
||||
)
|
||||
if pad_h == 0 and pad_w == 0:
|
||||
return image
|
||||
pad_mode = "reflect" if pad_h < resize_h and pad_w < resize_w else "edge"
|
||||
padded = np.pad(array, ((0, pad_h), (0, pad_w), (0, 0)), mode=pad_mode)
|
||||
return PIL.Image.fromarray(padded)
|
||||
|
||||
def _preprocess_action_image(image: PIL.Image.Image, target_h: int, target_w: int) -> torch.Tensor:
|
||||
image = _resize_and_pad_action_image(image, target_h, target_w)
|
||||
return video_processor.preprocess(image, height=target_h, width=target_w)
|
||||
|
||||
def _preprocess_action_video(frames: list[Any], target_h: int, target_w: int) -> torch.Tensor:
|
||||
if not frames:
|
||||
raise ValueError("Cosmos3 action video input must contain at least one frame.")
|
||||
processed = [_preprocess_action_image(_pil_to_rgb(frame), target_h, target_w).squeeze(0) for frame in frames]
|
||||
return torch.stack(processed, dim=1).unsqueeze(0).contiguous()
|
||||
|
||||
def pre_process_func(request: OmniDiffusionRequest) -> OmniDiffusionRequest:
|
||||
action_mode = _request_action_mode(request)
|
||||
if is_guardrails_enabled(od_config, request.sampling_params):
|
||||
for prompt in request.prompts:
|
||||
text = prompt if isinstance(prompt, str) else prompt.get("prompt", "")
|
||||
@@ -116,39 +182,63 @@ def get_cosmos3_pre_process_func(od_config: OmniDiffusionConfig):
|
||||
continue
|
||||
multi_modal_data = prompt.get("multi_modal_data", {}) or {}
|
||||
raw_image = multi_modal_data.get("image")
|
||||
if raw_image is None:
|
||||
raw_video = multi_modal_data.get("video")
|
||||
if raw_image is None and not (action_mode is not None and raw_video is not None):
|
||||
continue
|
||||
|
||||
if "additional_information" not in prompt:
|
||||
prompt["additional_information"] = {}
|
||||
|
||||
image = _pil_to_rgb(raw_image)
|
||||
if raw_image is None:
|
||||
if not isinstance(raw_video, list) or not raw_video:
|
||||
raise TypeError("Cosmos3 action video input must be a non-empty list of PIL images or image paths.")
|
||||
image = _pil_to_rgb(raw_video[0])
|
||||
else:
|
||||
image = _pil_to_rgb(raw_image)
|
||||
|
||||
# Auto-calculate H/W from aspect ratio (720p max area)
|
||||
if request.sampling_params.height is None or request.sampling_params.width is None:
|
||||
max_area = 720 * 1280
|
||||
aspect_ratio = image.height / image.width
|
||||
mod_value = 16
|
||||
height = round(np.sqrt(max_area * aspect_ratio)) // mod_value * mod_value
|
||||
width = round(np.sqrt(max_area / aspect_ratio)) // mod_value * mod_value
|
||||
if request.sampling_params.height is None:
|
||||
request.sampling_params.height = height
|
||||
if request.sampling_params.width is None:
|
||||
request.sampling_params.width = width
|
||||
if action_mode is not None:
|
||||
_set_action_size_from_image(request, image)
|
||||
else:
|
||||
max_area = 720 * 1280
|
||||
aspect_ratio = image.height / image.width
|
||||
mod_value = 16
|
||||
height = round(np.sqrt(max_area * aspect_ratio)) // mod_value * mod_value
|
||||
width = round(np.sqrt(max_area / aspect_ratio)) // mod_value * mod_value
|
||||
if request.sampling_params.height is None:
|
||||
request.sampling_params.height = height
|
||||
if request.sampling_params.width is None:
|
||||
request.sampling_params.width = width
|
||||
|
||||
target_w = request.sampling_params.width
|
||||
target_h = request.sampling_params.height
|
||||
scale = max(target_w / image.width, target_h / image.height)
|
||||
resize_w = int(np.ceil(scale * image.width))
|
||||
resize_h = int(np.ceil(scale * image.height))
|
||||
image = image.resize((resize_w, resize_h), PIL.Image.Resampling.LANCZOS)
|
||||
left = (resize_w - target_w) // 2
|
||||
top = (resize_h - target_h) // 2
|
||||
image = image.crop((left, top, left + target_w, top + target_h))
|
||||
if action_mode is not None:
|
||||
prompt["additional_information"]["preprocessed_image"] = _preprocess_action_image(
|
||||
image,
|
||||
int(target_h),
|
||||
int(target_w),
|
||||
)
|
||||
else:
|
||||
scale = max(target_w / image.width, target_h / image.height)
|
||||
resize_w = int(np.ceil(scale * image.width))
|
||||
resize_h = int(np.ceil(scale * image.height))
|
||||
image = image.resize((resize_w, resize_h), PIL.Image.Resampling.LANCZOS)
|
||||
left = (resize_w - target_w) // 2
|
||||
top = (resize_h - target_h) // 2
|
||||
image = image.crop((left, top, left + target_w, top + target_h))
|
||||
|
||||
prompt["additional_information"]["preprocessed_image"] = video_processor.preprocess(
|
||||
image, height=target_h, width=target_w
|
||||
)
|
||||
prompt["additional_information"]["preprocessed_image"] = video_processor.preprocess(
|
||||
image, height=target_h, width=target_w
|
||||
)
|
||||
if action_mode is not None and raw_video is not None:
|
||||
if not isinstance(raw_video, list):
|
||||
raise TypeError("Cosmos3 action video input must be a list of PIL images or image paths.")
|
||||
prompt["additional_information"]["preprocessed_video"] = _preprocess_action_video(
|
||||
raw_video,
|
||||
int(target_h),
|
||||
int(target_w),
|
||||
)
|
||||
request.prompts[i] = prompt
|
||||
|
||||
return request
|
||||
@@ -421,11 +511,17 @@ class Cosmos3OmniDiffusersPipeline(
|
||||
"time_embedder.",
|
||||
"audio_proj_in.",
|
||||
"audio_proj_out.",
|
||||
"action_proj_in.",
|
||||
"action_proj_out.",
|
||||
)
|
||||
):
|
||||
return f"transformer.{k}"
|
||||
if k in ("audio_modality_embed", "audio_modality_embed.weight"):
|
||||
return "transformer.audio_modality_embed"
|
||||
if k in ("action_modality_embed", "action_modality_embed.weight"):
|
||||
return "transformer.action_modality_embed"
|
||||
if k.startswith("action_pos_embed."):
|
||||
return None
|
||||
|
||||
# Skip lm_head
|
||||
if k.startswith("lm_head."):
|
||||
@@ -528,13 +624,22 @@ class Cosmos3OmniDiffusersPipeline(
|
||||
f"the checkpoint is missing sound weights for {missing}. "
|
||||
"Use a sound-capable transformer checkpoint."
|
||||
)
|
||||
if getattr(self.transformer, "action_gen", False):
|
||||
action_markers = ("action_proj_in.", "action_proj_out.", "action_modality_embed")
|
||||
missing = [marker.rstrip(".") for marker in action_markers if not any(marker in name for name in loaded)]
|
||||
if missing:
|
||||
raise ValueError(
|
||||
"Cosmos3 transformer config enables action generation, but "
|
||||
f"the checkpoint is missing action weights for {missing}. "
|
||||
"Use an action-capable transformer checkpoint."
|
||||
)
|
||||
return loaded
|
||||
|
||||
def predict_noise(self, **kwargs) -> torch.Tensor | tuple[torch.Tensor, ...]:
|
||||
"""Override CFGParallelMixin.predict_noise for Cosmos3.
|
||||
|
||||
The transformer returns the raw prediction: video-only as a tensor,
|
||||
or a tuple in video, sound order for sound generation.
|
||||
or a tuple in video, action, sound order for multimodal generation.
|
||||
"""
|
||||
return self.transformer(**kwargs)
|
||||
|
||||
@@ -611,6 +716,12 @@ class Cosmos3OmniDiffusersPipeline(
|
||||
return True
|
||||
return False
|
||||
|
||||
@classmethod
|
||||
def _get_action_mode(cls, prompt_data, sp) -> str | None:
|
||||
return normalize_action_mode(
|
||||
cls._get_sp_param(sp, "action_mode", cls._get_prompt_param(prompt_data, "action_mode", None))
|
||||
)
|
||||
|
||||
def _get_sound_tokenizer(self):
|
||||
if self._sound_tokenizer is None:
|
||||
from .sound_tokenizer import Cosmos3SoundTokenizer
|
||||
@@ -1017,6 +1128,30 @@ class Cosmos3OmniDiffusersPipeline(
|
||||
|
||||
return latent.to(self.dtype)
|
||||
|
||||
def _encode_video_tensor(self, video_tensor: torch.Tensor) -> torch.Tensor:
|
||||
"""VAE-encode a preprocessed pixel video [1, 3, T, H, W]."""
|
||||
if video_tensor.ndim == 4:
|
||||
video_tensor = video_tensor.unsqueeze(0)
|
||||
if video_tensor.ndim != 5:
|
||||
raise ValueError(f"Cosmos3 video tensor must have shape [1, 3, T, H, W], got {tuple(video_tensor.shape)}.")
|
||||
if video_tensor.shape[0] != 1 or video_tensor.shape[1] != 3:
|
||||
raise ValueError(f"Cosmos3 video tensor must have shape [1, 3, T, H, W], got {tuple(video_tensor.shape)}.")
|
||||
|
||||
video = video_tensor.to(device=self.device, dtype=self.vae.dtype)
|
||||
latent = self.vae.encode(video).latent_dist.mode()
|
||||
|
||||
if hasattr(self.vae.config, "latents_mean") and hasattr(self.vae.config, "latents_std"):
|
||||
latents_mean = (
|
||||
torch.tensor(self.vae.config.latents_mean).view(1, -1, 1, 1, 1).to(latent.device, latent.dtype)
|
||||
)
|
||||
latents_std = torch.tensor(self.vae.config.latents_std).view(1, -1, 1, 1, 1).to(latent.device, latent.dtype)
|
||||
latent = (latent - latents_mean) / latents_std
|
||||
else:
|
||||
scaling_factor = getattr(self.vae.config, "scaling_factor", 1.0)
|
||||
latent = latent * scaling_factor
|
||||
|
||||
return latent.to(self.dtype)
|
||||
|
||||
def _prepare_latents_i2v(
|
||||
self,
|
||||
image_tensor: torch.Tensor,
|
||||
@@ -1053,6 +1188,95 @@ class Cosmos3OmniDiffusersPipeline(
|
||||
velocity_mask = 1.0 - condition_mask
|
||||
return latents, velocity_mask, image_latent
|
||||
|
||||
def _prepare_latents_action_video(
|
||||
self,
|
||||
video_tensor: torch.Tensor,
|
||||
mode: str,
|
||||
height: int,
|
||||
width: int,
|
||||
num_frames: int,
|
||||
generator: torch.Generator,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""Prepare video latents for action modes with mode-specific conditioning."""
|
||||
del height, width
|
||||
C = self.transformer.latent_channel_size
|
||||
T_lat = (num_frames - 1) // self.vae_scale_factor_temporal + 1
|
||||
H_lat = video_tensor.shape[-2] // self.vae_scale_factor_spatial
|
||||
W_lat = video_tensor.shape[-1] // self.vae_scale_factor_spatial
|
||||
|
||||
noise = randn_tensor(
|
||||
(1, C, T_lat, H_lat, W_lat),
|
||||
generator=generator,
|
||||
device=self.device,
|
||||
dtype=self.dtype,
|
||||
)
|
||||
cond_latent = self._encode_video_tensor(video_tensor)
|
||||
if cond_latent.shape[2:] != noise.shape[2:]:
|
||||
raise ValueError(
|
||||
"Cosmos3 action video latent shape mismatch: "
|
||||
f"encoded={tuple(cond_latent.shape)}, expected={tuple(noise.shape)}."
|
||||
)
|
||||
condition_mask = build_vision_condition_mask(
|
||||
mode,
|
||||
num_frames,
|
||||
self.vae_scale_factor_temporal,
|
||||
device=self.device,
|
||||
dtype=self.dtype,
|
||||
)
|
||||
latents = condition_mask * cond_latent + (1.0 - condition_mask) * noise
|
||||
velocity_mask = 1.0 - condition_mask
|
||||
return latents, velocity_mask, cond_latent
|
||||
|
||||
def _prepare_action_latents(
|
||||
self,
|
||||
*,
|
||||
mode: str,
|
||||
action_chunk_size: int,
|
||||
raw_action_dim: int | None,
|
||||
generator: torch.Generator,
|
||||
sp,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, int]:
|
||||
action_dim = int(getattr(self.transformer, "action_dim", 64))
|
||||
if mode == ACTION_MODE_FORWARD_DYNAMICS:
|
||||
action = load_action_tensor(self._get_sp_param(sp, "action", None))
|
||||
if action.shape[0] < action_chunk_size:
|
||||
pad = action[-1:].repeat(action_chunk_size - action.shape[0], 1)
|
||||
action = torch.cat([action, pad], dim=0)
|
||||
elif action.shape[0] > action_chunk_size:
|
||||
action = action[:action_chunk_size]
|
||||
if raw_action_dim is None:
|
||||
raw_action_dim = int(action.shape[-1])
|
||||
clean_action = pad_action_to_dim(action, action_dim)
|
||||
else:
|
||||
if raw_action_dim is None:
|
||||
raise ValueError(
|
||||
"Cosmos3 action_mode='policy' and 'inverse_dynamics' require extra_args['raw_action_dim']."
|
||||
)
|
||||
clean_action = torch.zeros(action_chunk_size, action_dim, dtype=torch.float32)
|
||||
|
||||
raw_action_dim = int(raw_action_dim)
|
||||
if raw_action_dim <= 0 or raw_action_dim > action_dim:
|
||||
raise ValueError(f"Cosmos3 raw_action_dim must be in [1, {action_dim}], got {raw_action_dim}.")
|
||||
|
||||
clean_action = clean_action.to(device=self.device, dtype=self.dtype).unsqueeze(0)
|
||||
condition_mask = build_action_condition_mask(
|
||||
mode,
|
||||
action_chunk_size,
|
||||
device=self.device,
|
||||
dtype=self.dtype,
|
||||
)
|
||||
noise = randn_tensor(
|
||||
(1, action_chunk_size, action_dim),
|
||||
generator=generator,
|
||||
device=self.device,
|
||||
dtype=self.dtype,
|
||||
)
|
||||
noise[:, :, raw_action_dim:] = 0
|
||||
clean_action[:, :, raw_action_dim:] = 0
|
||||
action_latents = condition_mask * clean_action + (1.0 - condition_mask) * noise
|
||||
action_velocity_mask = 1.0 - condition_mask
|
||||
return action_latents, action_velocity_mask, clean_action, raw_action_dim
|
||||
|
||||
# -- Denoising loop (shared by T2V and I2V) -----------------------------
|
||||
|
||||
def diffuse(
|
||||
@@ -1066,11 +1290,15 @@ class Cosmos3OmniDiffusersPipeline(
|
||||
guidance_scale: float,
|
||||
shared_kwargs: dict,
|
||||
*,
|
||||
action_latents: torch.Tensor | None = None,
|
||||
action_velocity_mask: torch.Tensor | None = None,
|
||||
action_condition_latents: torch.Tensor | None = None,
|
||||
sound_latents: torch.Tensor | None = None,
|
||||
velocity_mask: torch.Tensor | None = None,
|
||||
image_latent: torch.Tensor | None = None,
|
||||
condition_latents: torch.Tensor | None = None,
|
||||
guidance_interval: tuple[float, float] | None = None,
|
||||
raw_action_dim: int | None = None,
|
||||
) -> torch.Tensor | tuple[torch.Tensor, ...]:
|
||||
"""Denoising loop with 3-mode CFG support (parallel, sequential, none).
|
||||
|
||||
@@ -1115,10 +1343,13 @@ class Cosmos3OmniDiffusersPipeline(
|
||||
# scheduler with cross-element dependencies (e.g. per-modality timestep).
|
||||
def _pack_joint(
|
||||
video_tensor: torch.Tensor,
|
||||
action_tensor: torch.Tensor | None = None,
|
||||
sound_tensor: torch.Tensor | None = None,
|
||||
):
|
||||
batch = video_tensor.shape[0]
|
||||
tensors = [video_tensor]
|
||||
if action_tensor is not None:
|
||||
tensors.append(action_tensor)
|
||||
if sound_tensor is not None:
|
||||
tensors.append(sound_tensor)
|
||||
flats = [tensor.reshape(batch, -1) for tensor in tensors]
|
||||
@@ -1138,57 +1369,86 @@ class Cosmos3OmniDiffusersPipeline(
|
||||
|
||||
def _split_noise_pred(
|
||||
noise_pred: torch.Tensor | tuple[torch.Tensor, ...],
|
||||
) -> tuple[torch.Tensor, torch.Tensor | None]:
|
||||
) -> tuple[torch.Tensor, torch.Tensor | None, torch.Tensor | None]:
|
||||
has_action = action_latents is not None
|
||||
has_sound = sound_latents is not None
|
||||
if not has_sound:
|
||||
if not has_action and not has_sound:
|
||||
if isinstance(noise_pred, tuple):
|
||||
raise ValueError("Cosmos3 video-only diffusion received tuple predictions.")
|
||||
return noise_pred, None
|
||||
return noise_pred, None, None
|
||||
if not isinstance(noise_pred, tuple):
|
||||
raise ValueError("Cosmos3 multimodal diffusion expects transformer predictions as a tuple.")
|
||||
if len(noise_pred) != 2:
|
||||
raise ValueError(f"Cosmos3 sound diffusion expected 2 predictions, got {len(noise_pred)}.")
|
||||
return noise_pred[0], noise_pred[1]
|
||||
expected = 1 + int(has_action) + int(has_sound)
|
||||
if len(noise_pred) != expected:
|
||||
raise ValueError(
|
||||
f"Cosmos3 multimodal diffusion expected {expected} predictions, got {len(noise_pred)}."
|
||||
)
|
||||
video_pred = noise_pred[0]
|
||||
idx = 1
|
||||
action_pred = noise_pred[idx] if has_action else None
|
||||
if has_action:
|
||||
idx += 1
|
||||
sound_pred = noise_pred[idx] if has_sound else None
|
||||
return video_pred, action_pred, sound_pred
|
||||
|
||||
def _step(
|
||||
noise_pred: torch.Tensor | tuple[torch.Tensor, ...],
|
||||
t: torch.Tensor,
|
||||
latents: torch.Tensor,
|
||||
action_latents: torch.Tensor | None,
|
||||
sound_latents: torch.Tensor | None,
|
||||
) -> torch.Tensor | tuple[torch.Tensor, ...]:
|
||||
video_pred, sound_pred = _split_noise_pred(noise_pred)
|
||||
video_pred, action_pred, sound_pred = _split_noise_pred(noise_pred)
|
||||
if velocity_mask is not None:
|
||||
video_pred = video_pred * velocity_mask
|
||||
if sound_latents is None:
|
||||
if action_pred is not None and action_velocity_mask is not None:
|
||||
action_pred = action_pred * action_velocity_mask
|
||||
if raw_action_dim is not None and 0 < raw_action_dim < action_pred.shape[-1]:
|
||||
action_pred[..., raw_action_dim:] = 0
|
||||
if action_latents is None and sound_latents is None:
|
||||
latents = self.scheduler.step(video_pred, t, latents, return_dict=False)[0]
|
||||
else:
|
||||
packed_noise, shapes, numels = _pack_joint(video_pred, sound_pred)
|
||||
packed_latents, _, _ = _pack_joint(latents, sound_latents)
|
||||
packed_noise, shapes, numels = _pack_joint(video_pred, action_pred, sound_pred)
|
||||
packed_latents, _, _ = _pack_joint(latents, action_latents, sound_latents)
|
||||
packed_next = self.scheduler.step(packed_noise, t, packed_latents, return_dict=False)[0]
|
||||
unpacked = _unpack_joint(packed_next, shapes, numels)
|
||||
latents = unpacked[0]
|
||||
idx = 1
|
||||
if action_latents is not None:
|
||||
action_latents = unpacked[idx]
|
||||
idx += 1
|
||||
if sound_latents is not None:
|
||||
sound_latents = unpacked[1]
|
||||
sound_latents = unpacked[idx]
|
||||
if condition_latents is not None and velocity_mask is not None:
|
||||
latents = velocity_mask * latents + (1.0 - velocity_mask) * condition_latents
|
||||
elif image_latent is not None:
|
||||
latents[:, :, 0:1, :, :] = image_latent
|
||||
if action_latents is not None and action_condition_latents is not None and action_velocity_mask is not None:
|
||||
action_latents = (
|
||||
action_velocity_mask * action_latents + (1.0 - action_velocity_mask) * action_condition_latents
|
||||
)
|
||||
outputs = [latents]
|
||||
if action_latents is not None:
|
||||
outputs.append(action_latents)
|
||||
if sound_latents is not None:
|
||||
outputs.append(sound_latents)
|
||||
return outputs[0] if len(outputs) == 1 else tuple(outputs)
|
||||
|
||||
def _assign_step_out(step_out: torch.Tensor | tuple[torch.Tensor, ...]) -> None:
|
||||
nonlocal latents, sound_latents
|
||||
if sound_latents is None:
|
||||
nonlocal latents, action_latents, sound_latents
|
||||
if action_latents is None and sound_latents is None:
|
||||
assert isinstance(step_out, torch.Tensor)
|
||||
latents = step_out
|
||||
return
|
||||
if not isinstance(step_out, tuple):
|
||||
raise ValueError("Cosmos3 multimodal diffusion step returned a non-tuple result.")
|
||||
latents = step_out[0]
|
||||
idx = 1
|
||||
if action_latents is not None:
|
||||
action_latents = step_out[idx]
|
||||
idx += 1
|
||||
if sound_latents is not None:
|
||||
sound_latents = step_out[1]
|
||||
sound_latents = step_out[idx]
|
||||
|
||||
if cfg_parallel:
|
||||
for t in self.progress_bar(timesteps):
|
||||
@@ -1206,6 +1466,7 @@ class Cosmos3OmniDiffusersPipeline(
|
||||
timestep=timestep,
|
||||
text_ids=cond_ids,
|
||||
text_mask=cond_mask,
|
||||
action_latents=action_latents,
|
||||
sound_latents=sound_latents,
|
||||
**shared_kwargs,
|
||||
),
|
||||
@@ -1214,12 +1475,13 @@ class Cosmos3OmniDiffusersPipeline(
|
||||
timestep=timestep,
|
||||
text_ids=uncond_ids,
|
||||
text_mask=uncond_mask,
|
||||
action_latents=action_latents,
|
||||
sound_latents=sound_latents,
|
||||
**shared_kwargs,
|
||||
),
|
||||
cfg_normalize=False,
|
||||
)
|
||||
_assign_step_out(_step(noise_pred, t, latents, sound_latents))
|
||||
_assign_step_out(_step(noise_pred, t, latents, action_latents, sound_latents))
|
||||
|
||||
elif do_cfg:
|
||||
cond_cache: tuple = (None, None)
|
||||
@@ -1237,6 +1499,7 @@ class Cosmos3OmniDiffusersPipeline(
|
||||
timestep=timestep,
|
||||
text_ids=cond_ids,
|
||||
text_mask=cond_mask,
|
||||
action_latents=action_latents,
|
||||
sound_latents=sound_latents,
|
||||
**shared_kwargs,
|
||||
)
|
||||
@@ -1250,6 +1513,7 @@ class Cosmos3OmniDiffusersPipeline(
|
||||
timestep=timestep,
|
||||
text_ids=uncond_ids,
|
||||
text_mask=uncond_mask,
|
||||
action_latents=action_latents,
|
||||
sound_latents=sound_latents,
|
||||
**shared_kwargs,
|
||||
)
|
||||
@@ -1263,7 +1527,7 @@ class Cosmos3OmniDiffusersPipeline(
|
||||
else:
|
||||
noise_pred = noise_cond
|
||||
|
||||
_assign_step_out(_step(noise_pred, t, latents, sound_latents))
|
||||
_assign_step_out(_step(noise_pred, t, latents, action_latents, sound_latents))
|
||||
|
||||
else:
|
||||
for t in self.progress_bar(timesteps):
|
||||
@@ -1273,12 +1537,15 @@ class Cosmos3OmniDiffusersPipeline(
|
||||
timestep=timestep,
|
||||
text_ids=cond_ids,
|
||||
text_mask=cond_mask,
|
||||
action_latents=action_latents,
|
||||
sound_latents=sound_latents,
|
||||
**shared_kwargs,
|
||||
)
|
||||
_assign_step_out(_step(noise_pred, t, latents, sound_latents))
|
||||
_assign_step_out(_step(noise_pred, t, latents, action_latents, sound_latents))
|
||||
|
||||
outputs = [latents]
|
||||
if action_latents is not None:
|
||||
outputs.append(action_latents)
|
||||
if sound_latents is not None:
|
||||
outputs.append(sound_latents)
|
||||
return outputs[0] if len(outputs) == 1 else tuple(outputs)
|
||||
@@ -1300,15 +1567,29 @@ class Cosmos3OmniDiffusersPipeline(
|
||||
prompt = prompt_data
|
||||
negative_prompt = None
|
||||
image_tensor = None
|
||||
action_video_tensor = None
|
||||
else:
|
||||
prompt = prompt_data.get("prompt", "")
|
||||
negative_prompt = prompt_data.get("negative_prompt")
|
||||
additional_info = prompt_data.get("additional_information", {}) or {}
|
||||
image_tensor = additional_info.get("preprocessed_image")
|
||||
action_video_tensor = additional_info.get("preprocessed_video")
|
||||
|
||||
sp = req.sampling_params
|
||||
is_t2i = self._is_t2i_request(req)
|
||||
sound_enabled = self._is_sound_request(prompt_data, sp)
|
||||
action_mode = self._get_action_mode(prompt_data, sp)
|
||||
action_enabled = action_mode is not None
|
||||
if action_enabled and is_t2i:
|
||||
raise ValueError("Cosmos3 action generation is supported only for video outputs.")
|
||||
if action_enabled and sound_enabled:
|
||||
raise ValueError("Cosmos3 action+sound joint generation is not supported in this phase.")
|
||||
if action_enabled and not getattr(self.transformer, "action_gen", False):
|
||||
raise ValueError(
|
||||
"Cosmos3 action generation was requested, but the transformer was "
|
||||
"initialized without action modules. Check that the checkpoint config "
|
||||
"enables action_gen and includes action weights."
|
||||
)
|
||||
if sound_enabled and is_t2i:
|
||||
raise ValueError(
|
||||
"Cosmos3 sound generation is supported only for video outputs in "
|
||||
@@ -1349,6 +1630,36 @@ class Cosmos3OmniDiffusersPipeline(
|
||||
default_guidance_interval = None
|
||||
batch_size = 1 # Existing video pipeline assumes B=1.
|
||||
|
||||
if action_enabled:
|
||||
action_chunk_param = self._get_sp_param(sp, "action_chunk_size", None)
|
||||
if action_chunk_param is not None:
|
||||
action_chunk_size = int(action_chunk_param)
|
||||
if sp.num_frames is None:
|
||||
num_frames = action_chunk_size + 1
|
||||
elif sp.num_frames is None:
|
||||
action_chunk_size = 16
|
||||
num_frames = action_chunk_size + 1
|
||||
else:
|
||||
action_chunk_size = int(num_frames) - 1
|
||||
if action_chunk_size <= 0:
|
||||
raise ValueError(f"Cosmos3 action_chunk_size must be positive, got {action_chunk_size}.")
|
||||
if num_frames not in (action_chunk_size, action_chunk_size + 1):
|
||||
raise ValueError(
|
||||
"Cosmos3 action requests require num_frames to equal action_chunk_size "
|
||||
f"or action_chunk_size + 1; got num_frames={num_frames}, action_chunk_size={action_chunk_size}."
|
||||
)
|
||||
num_inference_steps = sp.num_inference_steps or 30
|
||||
guidance_scale = sp.guidance_scale if sp.guidance_scale is not None else 1.0
|
||||
default_flow_shift = 5.0
|
||||
|
||||
domain_id = None
|
||||
if action_enabled:
|
||||
domain_id = resolve_domain_id(
|
||||
domain_id=self._get_sp_param(sp, "domain_id", None),
|
||||
domain_name=self._get_sp_param(sp, "domain_name", None),
|
||||
require_explicit=True,
|
||||
)
|
||||
|
||||
# Runtime controls: prefer ``extra_args`` (OpenAI endpoints write
|
||||
# there) over direct attrs.
|
||||
flow_shift_target = float(self._get_sp_param(sp, "flow_shift", default_flow_shift))
|
||||
@@ -1361,6 +1672,23 @@ class Cosmos3OmniDiffusersPipeline(
|
||||
)
|
||||
use_system_prompt = bool(self._get_sp_param(sp, "use_system_prompt", False))
|
||||
|
||||
if action_enabled and action_video_tensor is None:
|
||||
extra_action_video = self._get_sp_param(sp, "action_video", None)
|
||||
if isinstance(extra_action_video, torch.Tensor):
|
||||
action_video_tensor = extra_action_video
|
||||
if action_enabled and isinstance(action_video_tensor, torch.Tensor):
|
||||
if action_video_tensor.ndim == 4:
|
||||
action_video_tensor = action_video_tensor.unsqueeze(0)
|
||||
if action_video_tensor.ndim != 5:
|
||||
raise ValueError(
|
||||
"Cosmos3 extra_args['action_video'] must have shape [1, 3, T, H, W] "
|
||||
f"or [3, T, H, W], got {tuple(action_video_tensor.shape)}."
|
||||
)
|
||||
if sp.height is None:
|
||||
height = int(action_video_tensor.shape[-2])
|
||||
if sp.width is None:
|
||||
width = int(action_video_tensor.shape[-1])
|
||||
|
||||
self._guidance_scale = guidance_scale
|
||||
self._num_timesteps = num_inference_steps
|
||||
|
||||
@@ -1396,7 +1724,58 @@ class Cosmos3OmniDiffusersPipeline(
|
||||
# batching B=N together would require expanding text K/V (UND
|
||||
# pathway is text-only and cached) and is left as a future
|
||||
# optimization.
|
||||
if image_tensor is not None and not is_t2i:
|
||||
action_latents = None
|
||||
action_velocity_mask = None
|
||||
action_condition_latents = None
|
||||
raw_action_dim = None
|
||||
action_offset = 1
|
||||
if action_enabled:
|
||||
if action_video_tensor is not None and action_video_tensor.ndim == 4:
|
||||
action_video_tensor = action_video_tensor.unsqueeze(0)
|
||||
if action_video_tensor is not None and action_video_tensor.ndim != 5:
|
||||
raise ValueError(
|
||||
"Cosmos3 action video tensor must have shape [1, 3, T, H, W] "
|
||||
f"or [3, T, H, W], got {tuple(action_video_tensor.shape)}."
|
||||
)
|
||||
if action_video_tensor is not None and action_video_tensor.shape[2] < num_frames:
|
||||
pad = action_video_tensor[:, :, -1:].repeat(1, 1, num_frames - action_video_tensor.shape[2], 1, 1)
|
||||
action_video_tensor = torch.cat([action_video_tensor, pad], dim=2)
|
||||
elif action_video_tensor is not None and action_video_tensor.shape[2] > num_frames:
|
||||
action_video_tensor = action_video_tensor[:, :, :num_frames]
|
||||
|
||||
if action_mode == ACTION_MODE_INVERSE_DYNAMICS and action_video_tensor is None:
|
||||
raise ValueError("Cosmos3 inverse_dynamics action mode requires multi_modal_data['video'].")
|
||||
if action_mode in {ACTION_MODE_POLICY, ACTION_MODE_FORWARD_DYNAMICS} and image_tensor is None:
|
||||
if action_video_tensor is None:
|
||||
raise ValueError(
|
||||
f"Cosmos3 action_mode={action_mode!r} requires multi_modal_data['image'] "
|
||||
"or multi_modal_data['video']."
|
||||
)
|
||||
image_tensor = action_video_tensor[:, :, 0]
|
||||
|
||||
raw_action_dim_param = self._get_sp_param(sp, "raw_action_dim", None)
|
||||
raw_action_dim = int(raw_action_dim_param) if raw_action_dim_param is not None else None
|
||||
action_prepared = self._prepare_action_latents(
|
||||
mode=action_mode,
|
||||
action_chunk_size=action_chunk_size,
|
||||
raw_action_dim=raw_action_dim,
|
||||
generator=generator,
|
||||
sp=sp,
|
||||
)
|
||||
action_latents, action_velocity_mask, action_condition_latents, raw_action_dim = action_prepared
|
||||
action_offset = action_start_frame_offset(action_mode, action_chunk_size, num_frames)
|
||||
|
||||
if action_enabled and action_video_tensor is not None:
|
||||
latents, velocity_mask, condition_latents = self._prepare_latents_action_video(
|
||||
action_video_tensor,
|
||||
action_mode,
|
||||
height,
|
||||
width,
|
||||
num_frames,
|
||||
generator,
|
||||
)
|
||||
image_latent = condition_latents[:, :, 0:1]
|
||||
elif image_tensor is not None and not is_t2i:
|
||||
latents, velocity_mask, image_latent = self._prepare_latents_i2v(
|
||||
image_tensor,
|
||||
height,
|
||||
@@ -1427,6 +1806,13 @@ class Cosmos3OmniDiffusersPipeline(
|
||||
shared_kwargs = dict(video_shape=video_shape, fps=frame_rate)
|
||||
if velocity_mask is not None:
|
||||
shared_kwargs["noisy_frame_mask"] = velocity_mask
|
||||
if action_enabled:
|
||||
shared_kwargs.update(
|
||||
action_domain_ids=torch.tensor([domain_id], dtype=torch.long, device=self.device),
|
||||
action_noisy_mask=action_velocity_mask,
|
||||
action_start_frame_offset=action_offset,
|
||||
action_fps=float(self._get_sp_param(sp, "action_fps", frame_rate) or frame_rate),
|
||||
)
|
||||
|
||||
def _run_diffusion(start_latents):
|
||||
self.scheduler.set_timesteps(num_inference_steps, device=self.device)
|
||||
@@ -1439,11 +1825,15 @@ class Cosmos3OmniDiffusersPipeline(
|
||||
uncond_mask=uncond_mask,
|
||||
guidance_scale=guidance_scale,
|
||||
shared_kwargs=shared_kwargs,
|
||||
action_latents=action_latents,
|
||||
action_velocity_mask=action_velocity_mask,
|
||||
action_condition_latents=action_condition_latents,
|
||||
sound_latents=sound_latents,
|
||||
velocity_mask=velocity_mask,
|
||||
image_latent=image_latent,
|
||||
condition_latents=condition_latents,
|
||||
guidance_interval=guidance_interval,
|
||||
raw_action_dim=raw_action_dim,
|
||||
)
|
||||
|
||||
if is_t2i and batch_size > 1:
|
||||
@@ -1461,7 +1851,11 @@ class Cosmos3OmniDiffusersPipeline(
|
||||
latents = torch.cat(samples, dim=0)
|
||||
else:
|
||||
diffusion_output = _run_diffusion(latents)
|
||||
if sound_enabled:
|
||||
if action_enabled and sound_enabled:
|
||||
latents, action_latents, sound_latents = diffusion_output
|
||||
elif action_enabled:
|
||||
latents, action_latents = diffusion_output
|
||||
elif sound_enabled:
|
||||
latents, sound_latents = diffusion_output
|
||||
else:
|
||||
latents = diffusion_output
|
||||
@@ -1483,4 +1877,18 @@ class Cosmos3OmniDiffusersPipeline(
|
||||
audio = self._decode_sound_latents(sound_latents, target_audio_samples)
|
||||
return DiffusionOutput(output={"video": video, "audio": audio, "audio_sample_rate": sound_sample_rate})
|
||||
|
||||
if action_enabled:
|
||||
if action_latents is None or raw_action_dim is None or domain_id is None:
|
||||
raise ValueError("Cosmos3 action generation finished without action latents.")
|
||||
action = action_latents[:, :, :raw_action_dim].detach().cpu()
|
||||
return DiffusionOutput(
|
||||
output={"video": video},
|
||||
custom_output={
|
||||
"action": action,
|
||||
"raw_action_dim": raw_action_dim,
|
||||
"action_mode": action_mode,
|
||||
"domain_id": domain_id,
|
||||
},
|
||||
)
|
||||
|
||||
return DiffusionOutput(output={"image": video} if is_t2i else {"video": video})
|
||||
@@ -136,6 +136,47 @@ def resolve_sound_gen(od_config: Any) -> bool:
|
||||
return False
|
||||
|
||||
|
||||
class DomainAwareLinear(nn.Module):
|
||||
"""Linear projection with one weight/bias pair per action embodiment domain."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
input_size: int,
|
||||
output_size: int,
|
||||
num_domains: int,
|
||||
*,
|
||||
dtype: torch.dtype = torch.bfloat16,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.input_size = int(input_size)
|
||||
self.output_size = int(output_size)
|
||||
self.num_domains = int(num_domains)
|
||||
self.fc = nn.Embedding(self.num_domains, self.output_size * self.input_size, dtype=dtype)
|
||||
self.bias = nn.Embedding(self.num_domains, self.output_size, dtype=dtype)
|
||||
nn.init.xavier_uniform_(self.fc.weight)
|
||||
nn.init.zeros_(self.bias.weight)
|
||||
|
||||
def forward(self, x: torch.Tensor, domain_id: torch.Tensor) -> torch.Tensor:
|
||||
if domain_id.ndim == 0:
|
||||
domain_id = domain_id.unsqueeze(0)
|
||||
domain_id = domain_id.to(device=x.device, dtype=torch.long).reshape(-1)
|
||||
if x.shape[0] != domain_id.shape[0]:
|
||||
raise ValueError(
|
||||
"Cosmos3 action domain_id batch size must match action tokens: "
|
||||
f"tokens={x.shape[0]}, domain_id={domain_id.shape[0]}."
|
||||
)
|
||||
if torch.any((domain_id < 0) | (domain_id >= self.num_domains)):
|
||||
raise ValueError(f"Cosmos3 action domain_id must be in [0, {self.num_domains}), got {domain_id.tolist()}.")
|
||||
|
||||
weight = self.fc(domain_id).view(domain_id.shape[0], self.input_size, self.output_size)
|
||||
bias = self.bias(domain_id).view(domain_id.shape[0], self.output_size)
|
||||
if x.ndim == 2:
|
||||
return torch.bmm(x.unsqueeze(1), weight).squeeze(1) + bias
|
||||
if x.ndim == 3:
|
||||
return torch.bmm(x, weight) + bias.unsqueeze(1)
|
||||
raise ValueError(f"Cosmos3 DomainAwareLinear expected rank-2 or rank-3 input, got {tuple(x.shape)}.")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Rotary Position Embeddings (mRoPE)
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -163,6 +204,7 @@ def compute_mrope_position_ids_vision(
|
||||
temporal_compression_factor: int = 4,
|
||||
base_temporal_compression_factor: int | None = None,
|
||||
enable_fps_modulation: bool = True,
|
||||
start_frame_offset: int = 0,
|
||||
) -> tuple[torch.Tensor, int | float]:
|
||||
"""Generate 3D mRoPE position IDs for vision tokens.
|
||||
|
||||
@@ -180,10 +222,17 @@ def compute_mrope_position_ids_vision(
|
||||
)
|
||||
base_tps = base_fps / effective_base_tcf
|
||||
frame_indices = torch.arange(grid_t, dtype=torch.float32)
|
||||
t_index = (frame_indices / tps * base_tps + temporal_offset).view(-1, 1).expand(-1, grid_h * grid_w).flatten()
|
||||
t_index = (
|
||||
((frame_indices + start_frame_offset) / tps * base_tps + temporal_offset)
|
||||
.view(-1, 1)
|
||||
.expand(-1, grid_h * grid_w)
|
||||
.flatten()
|
||||
)
|
||||
else:
|
||||
t_index = torch.arange(grid_t, dtype=torch.long).view(-1, 1).expand(-1, grid_h * grid_w).flatten() + int(
|
||||
temporal_offset
|
||||
t_index = (
|
||||
torch.arange(grid_t, dtype=torch.long).view(-1, 1).expand(-1, grid_h * grid_w).flatten()
|
||||
+ int(temporal_offset)
|
||||
+ start_frame_offset
|
||||
)
|
||||
|
||||
h_index = torch.arange(grid_h, dtype=torch.long).view(1, -1, 1).expand(grid_t, -1, grid_w).flatten()
|
||||
@@ -220,6 +269,30 @@ def compute_mrope_position_ids_sound(
|
||||
)
|
||||
|
||||
|
||||
def compute_mrope_position_ids_action(
|
||||
grid_t: int,
|
||||
temporal_offset: int | float,
|
||||
action_fps: float | None,
|
||||
base_fps: float = 24.0,
|
||||
base_temporal_compression_factor: int = 4,
|
||||
enable_fps_modulation: bool = True,
|
||||
start_frame_offset: int = 1,
|
||||
) -> tuple[torch.Tensor, int | float]:
|
||||
"""Generate mRoPE IDs for action tokens as a frame-rate (T, 1, 1) grid."""
|
||||
return compute_mrope_position_ids_vision(
|
||||
grid_t=grid_t,
|
||||
grid_h=1,
|
||||
grid_w=1,
|
||||
temporal_offset=temporal_offset,
|
||||
fps=action_fps,
|
||||
base_fps=base_fps,
|
||||
temporal_compression_factor=1,
|
||||
base_temporal_compression_factor=base_temporal_compression_factor,
|
||||
enable_fps_modulation=enable_fps_modulation,
|
||||
start_frame_offset=start_frame_offset,
|
||||
)
|
||||
|
||||
|
||||
class Qwen3VLTextRotaryEmbedding(nn.Module):
|
||||
"""Multi-dimensional rotary position embedding for Qwen3-VL."""
|
||||
|
||||
@@ -947,19 +1020,28 @@ class Cosmos3VFMTransformer(nn.Module):
|
||||
self.sound_gen = sound_gen
|
||||
self.sound_dim = sound_dim
|
||||
self.sound_latent_fps = sound_latent_fps
|
||||
|
||||
if self.sound_gen and (sound_dim is None or sound_latent_fps is None):
|
||||
raise ValueError(
|
||||
"Cosmos3VFMTransformer requires an explicit sound_dim and sound_latent_fps when sound_gen is True; "
|
||||
"the pipeline must pass Cosmos3SoundTokenizer.latent_ch so the audio projection "
|
||||
"layers are sized from the authoritative AVAE latent width."
|
||||
)
|
||||
action_gen_value = _od_config_get(od_config, "action_gen", None)
|
||||
action_dim_value = _od_config_get(od_config, "action_dim", None)
|
||||
if action_dim_value is None:
|
||||
action_dim_value = _od_config_get(od_config, "max_action_dim", None)
|
||||
self.action_gen = _as_bool(action_gen_value) if action_gen_value is not None else False
|
||||
self.action_dim = int(action_dim_value if action_dim_value is not None else 64)
|
||||
self.num_embodiment_domains = int(_od_config_get(od_config, "num_embodiment_domains", 32))
|
||||
if temporal_compression_factor is None:
|
||||
temporal_compression_factor = _tf_config_get(model_config, "temporal_compression_factor", 4)
|
||||
self.temporal_compression_factor = int(temporal_compression_factor)
|
||||
self.temporal_compression_factor_sound = int(
|
||||
_tf_config_get(model_config, "temporal_compression_factor_sound", 1)
|
||||
)
|
||||
self.temporal_compression_factor_sound = int(
|
||||
_tf_config_get(model_config, "temporal_compression_factor_sound", 1)
|
||||
)
|
||||
self.enable_fps_modulation = bool(_tf_config_get(model_config, "enable_fps_modulation", True))
|
||||
self.temporal_modality_margin = int(
|
||||
_tf_config_get(
|
||||
@@ -992,6 +1074,20 @@ class Cosmos3VFMTransformer(nn.Module):
|
||||
self.proj_in = nn.Linear(self.patch_latent_dim, self.hidden_size)
|
||||
self.proj_out = nn.Linear(self.hidden_size, self.patch_latent_dim)
|
||||
self.time_embedder = TimestepEmbedder(self.hidden_size, target_dtype=dtype)
|
||||
if self.action_gen:
|
||||
self.action_proj_in = DomainAwareLinear(
|
||||
self.action_dim,
|
||||
self.hidden_size,
|
||||
self.num_embodiment_domains,
|
||||
dtype=dtype,
|
||||
)
|
||||
self.action_proj_out = DomainAwareLinear(
|
||||
self.hidden_size,
|
||||
self.action_dim,
|
||||
self.num_embodiment_domains,
|
||||
dtype=dtype,
|
||||
)
|
||||
self.action_modality_embed = nn.Parameter(torch.zeros(self.hidden_size, dtype=dtype))
|
||||
if self.sound_gen:
|
||||
self.audio_proj_in = nn.Linear(self.sound_dim, self.hidden_size)
|
||||
self.audio_proj_out = nn.Linear(self.hidden_size, self.sound_dim)
|
||||
@@ -1079,6 +1175,21 @@ class Cosmos3VFMTransformer(nn.Module):
|
||||
"""[B, T_sound, C_sound] -> [B, C_sound, T_sound]."""
|
||||
return tokens.permute(0, 2, 1).contiguous()
|
||||
|
||||
def pack_action(self, action_latents: torch.Tensor) -> torch.Tensor:
|
||||
"""Validate and return action latents as [B, T_action, D_action] tokens."""
|
||||
if action_latents.ndim != 3:
|
||||
raise ValueError(f"Cosmos3 action latents must have shape [B, T, D], got {tuple(action_latents.shape)}.")
|
||||
if action_latents.shape[-1] != self.action_dim:
|
||||
raise ValueError(
|
||||
f"Cosmos3 action latent dimension mismatch: expected {self.action_dim}, got {action_latents.shape[-1]}."
|
||||
)
|
||||
return action_latents.contiguous()
|
||||
|
||||
@staticmethod
|
||||
def unpack_action(tokens: torch.Tensor) -> torch.Tensor:
|
||||
"""Return [B, T_action, D_action] action predictions."""
|
||||
return tokens.contiguous()
|
||||
|
||||
# -- RoPE computation ----------------------------------------------------
|
||||
|
||||
def _compute_rope_freqs(
|
||||
@@ -1090,6 +1201,9 @@ class Cosmos3VFMTransformer(nn.Module):
|
||||
fps: float | None,
|
||||
device: torch.device,
|
||||
dtype: torch.dtype,
|
||||
t_action: int | None = None,
|
||||
action_start_frame_offset: int = 1,
|
||||
action_fps: float | None = None,
|
||||
t_sound: int | None = None,
|
||||
) -> tuple[tuple[torch.Tensor, torch.Tensor], tuple[torch.Tensor, torch.Tensor]]:
|
||||
"""Compute mRoPE cos/sin for UND text and GEN media pathways."""
|
||||
@@ -1097,6 +1211,7 @@ class Cosmos3VFMTransformer(nn.Module):
|
||||
S_text = text_mask.shape[1]
|
||||
text_lengths = text_mask.sum(dim=1).long()
|
||||
effective_fps = fps if fps is not None and t > 1 else None
|
||||
action_frames = int(t_action or 0)
|
||||
sound_frames = int(t_sound or 0)
|
||||
|
||||
text_pos_list = []
|
||||
@@ -1116,6 +1231,17 @@ class Cosmos3VFMTransformer(nn.Module):
|
||||
enable_fps_modulation=self.enable_fps_modulation,
|
||||
)
|
||||
gen_positions = [v_pos]
|
||||
if action_frames > 0:
|
||||
a_pos, _ = compute_mrope_position_ids_action(
|
||||
action_frames,
|
||||
temporal_offset=media_temporal_offset,
|
||||
action_fps=action_fps if action_fps is not None else fps,
|
||||
base_fps=self.base_fps,
|
||||
base_temporal_compression_factor=self.temporal_compression_factor,
|
||||
enable_fps_modulation=self.enable_fps_modulation,
|
||||
start_frame_offset=action_start_frame_offset,
|
||||
)
|
||||
gen_positions.append(a_pos)
|
||||
if sound_frames > 0:
|
||||
s_pos, _ = compute_mrope_position_ids_sound(
|
||||
sound_frames,
|
||||
@@ -1161,7 +1287,9 @@ class Cosmos3VFMTransformer(nn.Module):
|
||||
*,
|
||||
s_gen: int,
|
||||
s_video: int,
|
||||
s_action: int,
|
||||
s_sound: int,
|
||||
has_action: bool,
|
||||
has_sound: bool,
|
||||
ulysses_size: int,
|
||||
) -> None:
|
||||
@@ -1169,14 +1297,16 @@ class Cosmos3VFMTransformer(nn.Module):
|
||||
return
|
||||
|
||||
detail_parts = [f"video tokens {s_video}"]
|
||||
if has_action:
|
||||
detail_parts.append(f"action tokens {s_action}")
|
||||
if has_sound:
|
||||
detail_parts.append(f"sound tokens {s_sound}")
|
||||
detail = " = " + " + ".join(detail_parts) if len(detail_parts) > 1 else ""
|
||||
adjust_detail = (
|
||||
"Adjust the spatial resolution, frame count, sound duration, "
|
||||
"or sound latent FPS so the combined media sequence is a "
|
||||
"Adjust the spatial resolution, frame count, action chunk size, "
|
||||
"sound duration, or sound latent FPS so the combined media sequence is a "
|
||||
"multiple of ulysses_degree."
|
||||
if has_sound
|
||||
if has_action or has_sound
|
||||
else (
|
||||
"Adjust the spatial resolution so that "
|
||||
"t * ceil(h/patch) * ceil(w/patch) is a multiple "
|
||||
@@ -1198,6 +1328,11 @@ class Cosmos3VFMTransformer(nn.Module):
|
||||
text_mask: torch.Tensor,
|
||||
video_shape: tuple[int, int, int],
|
||||
fps: float | None = None,
|
||||
action_latents: torch.Tensor | None = None,
|
||||
action_domain_ids: torch.Tensor | None = None,
|
||||
action_noisy_mask: torch.Tensor | None = None,
|
||||
action_start_frame_offset: int = 1,
|
||||
action_fps: float | None = None,
|
||||
sound_latents: torch.Tensor | None = None,
|
||||
noisy_frame_mask: torch.Tensor | None = None,
|
||||
**kwargs,
|
||||
@@ -1210,6 +1345,10 @@ class Cosmos3VFMTransformer(nn.Module):
|
||||
text_mask: [B, S_text] attention mask (1=real, 0=pad)
|
||||
video_shape: (t, h, w) in latent space
|
||||
fps: video frame rate for temporal mRoPE modulation
|
||||
action_latents: Optional [B, T_action, D_action] noisy action latents.
|
||||
action_domain_ids: Optional [B] embodiment domain IDs for action projections.
|
||||
action_noisy_mask: Optional [B, T_action, 1] mask where 1=noisy
|
||||
action token and 0=clean conditioned token.
|
||||
sound_latents: Optional [B, C_sound, T_sound] noisy sound latents.
|
||||
noisy_frame_mask: Optional [B, 1, t, 1, 1] mask where 1=noisy (add
|
||||
timestep embedding, predict velocity) and 0=conditioned (clean
|
||||
@@ -1218,7 +1357,7 @@ class Cosmos3VFMTransformer(nn.Module):
|
||||
|
||||
Returns:
|
||||
[B, C, t, h, w] velocity prediction, or
|
||||
tuple outputs in video, sound order when sound latents are provided.
|
||||
tuple outputs in video, action, sound order when extra modalities are provided.
|
||||
"""
|
||||
t, h, w = video_shape
|
||||
hp, wp, _, _ = self._pad_to_patch_size(h, w)
|
||||
@@ -1230,7 +1369,14 @@ class Cosmos3VFMTransformer(nn.Module):
|
||||
f"Cosmos3 requires identical real text lengths within a batch "
|
||||
f"(got min={min_real_len}, max={max_real_len})."
|
||||
)
|
||||
has_action = action_latents is not None
|
||||
has_sound = sound_latents is not None
|
||||
if has_action and not self.action_gen:
|
||||
raise ValueError(
|
||||
"Cosmos3 action generation was requested, but this transformer "
|
||||
"was initialized without action modules. Check that the "
|
||||
"transformer config enables action_gen."
|
||||
)
|
||||
if has_sound and not self.sound_gen:
|
||||
raise ValueError(
|
||||
"Cosmos3 sound generation was requested, but this transformer "
|
||||
@@ -1244,8 +1390,21 @@ class Cosmos3VFMTransformer(nn.Module):
|
||||
# Patchify latents and project to hidden space
|
||||
hidden_video = self.proj_in(self.patchify(hidden_states, t, h, w))
|
||||
s_video = hidden_video.shape[1]
|
||||
s_action = 0
|
||||
hidden_action = None
|
||||
s_sound = 0
|
||||
hidden_sound = None
|
||||
if action_latents is not None:
|
||||
if action_latents.shape[0] != hidden_states.shape[0]:
|
||||
raise ValueError(
|
||||
"Cosmos3 action and video batch sizes must match: "
|
||||
f"video={hidden_states.shape[0]}, action={action_latents.shape[0]}."
|
||||
)
|
||||
if action_domain_ids is None:
|
||||
action_domain_ids = torch.zeros(action_latents.shape[0], dtype=torch.long, device=action_latents.device)
|
||||
hidden_action = self.action_proj_in(self.pack_action(action_latents), action_domain_ids)
|
||||
hidden_action = hidden_action + self.action_modality_embed.to(hidden_action.dtype)
|
||||
s_action = hidden_action.shape[1]
|
||||
if sound_latents is not None:
|
||||
if sound_latents.shape[0] != hidden_states.shape[0]:
|
||||
raise ValueError(
|
||||
@@ -1277,9 +1436,22 @@ class Cosmos3VFMTransformer(nn.Module):
|
||||
else:
|
||||
hidden_video = hidden_video + time_embed.unsqueeze(1)
|
||||
|
||||
if hidden_action is not None:
|
||||
if action_noisy_mask is None:
|
||||
hidden_action = hidden_action + time_embed.unsqueeze(1)
|
||||
else:
|
||||
if action_noisy_mask.shape != (hidden_action.shape[0], hidden_action.shape[1], 1):
|
||||
raise ValueError(
|
||||
"Cosmos3 action_noisy_mask must have shape [B, T_action, 1], "
|
||||
f"got {tuple(action_noisy_mask.shape)}."
|
||||
)
|
||||
hidden_action = hidden_action + time_embed.unsqueeze(1) * action_noisy_mask.to(hidden_action.dtype)
|
||||
|
||||
if hidden_sound is not None:
|
||||
hidden_sound = hidden_sound + time_embed.unsqueeze(1)
|
||||
hidden_parts = [hidden_video]
|
||||
if hidden_action is not None:
|
||||
hidden_parts.append(hidden_action)
|
||||
if hidden_sound is not None:
|
||||
hidden_parts.append(hidden_sound)
|
||||
hidden_gen = torch.cat(hidden_parts, dim=1)
|
||||
@@ -1294,6 +1466,9 @@ class Cosmos3VFMTransformer(nn.Module):
|
||||
fps,
|
||||
hidden_states.device,
|
||||
hidden_states.dtype,
|
||||
t_action=s_action,
|
||||
action_start_frame_offset=action_start_frame_offset,
|
||||
action_fps=action_fps,
|
||||
t_sound=s_sound,
|
||||
)
|
||||
cached_kv_full = self.language_model(text_ids, freqs_und)
|
||||
@@ -1311,7 +1486,9 @@ class Cosmos3VFMTransformer(nn.Module):
|
||||
self._validate_gen_sequence_parallel(
|
||||
s_gen=hidden_gen.shape[1],
|
||||
s_video=s_video,
|
||||
s_action=s_action,
|
||||
s_sound=s_sound,
|
||||
has_action=has_action,
|
||||
has_sound=has_sound,
|
||||
ulysses_size=ulysses_size,
|
||||
)
|
||||
@@ -1346,10 +1523,12 @@ class Cosmos3VFMTransformer(nn.Module):
|
||||
|
||||
# Final norm and project back to latent space
|
||||
hidden_gen = self.norm_moe_gen(hidden_gen)
|
||||
if not has_sound:
|
||||
if not has_action and not has_sound:
|
||||
return self.unpatchify(self.proj_out(hidden_gen), t, h, w)
|
||||
|
||||
split_sizes = [s_video]
|
||||
if has_action:
|
||||
split_sizes.append(s_action)
|
||||
if has_sound:
|
||||
split_sizes.append(s_sound)
|
||||
split_hidden = hidden_gen.split(split_sizes, dim=1)
|
||||
@@ -1357,6 +1536,11 @@ class Cosmos3VFMTransformer(nn.Module):
|
||||
video_pred = self.unpatchify(self.proj_out(hidden_video), t, h, w)
|
||||
outputs: list[torch.Tensor] = [video_pred]
|
||||
split_idx = 1
|
||||
if has_action:
|
||||
hidden_action = split_hidden[split_idx]
|
||||
split_idx += 1
|
||||
assert action_domain_ids is not None
|
||||
outputs.append(self.unpack_action(self.action_proj_out(hidden_action, action_domain_ids)))
|
||||
if has_sound:
|
||||
hidden_sound = split_hidden[split_idx]
|
||||
outputs.append(self.unpack_sound(self.audio_proj_out(hidden_sound)))
|
||||
|
||||
Reference in new issue
Block a user