Cosmos3 transfer (#4379)

Signed-off-by: Maciej Bala <mbala@nvidia.com>
Signed-off-by: MaciejBalaNV <mbala@nvidia.com>
This commit is contained in:
MaciejBalaNV authored and GitHub committed 2026-06-21 09:59:18 +08:00
1 parent cead615288
commit a1e1ac0155
12 files changed
+2003 -58

No files matched your search

@@ -20,7 +20,7 @@ from __future__ import annotations
import math
import os
import time
from collections.abc import Iterable
from collections.abc import Iterable, Mapping
from dataclasses import fields
from typing import Any, ClassVar
@@ -50,6 +50,7 @@ from vllm_omni.diffusion.models.interface import (
from vllm_omni.diffusion.models.progress_bar import ProgressBarMixin, _is_rank_zero
from vllm_omni.diffusion.profiler.diffusion_pipeline_profiler import DiffusionPipelineProfilerMixin
from vllm_omni.diffusion.request import OmniDiffusionRequest
from vllm_omni.entrypoints.openai.video_api_utils import positive_float
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
from .action import (
@@ -65,6 +66,19 @@ from .action import (
pad_action_to_dim,
resolve_domain_id,
)
from .transfer import (
Cosmos3TransferConfig,
has_transfer_hints,
load_or_compute_control_frames,
media_hw,
media_to_uint8_cthw,
normalized_video_to_uint8_cthw,
pad_temporal_frames,
resize_center_crop_uint8_cthw,
resolve_transfer_config,
transfer_max_frames_from_extra_args,
uint8_cthw_to_normalized_5d,
)
from .transformer_cosmos3 import Cosmos3VFMTransformer, resolve_sound_gen
from .utils import (
COSMOS3_DEFAULT_CONDITION_FRAME_INDEXES_VISION,
@@ -166,6 +180,14 @@ def get_cosmos3_pre_process_func(od_config: OmniDiffusionConfig):
def _request_action_mode(request: OmniDiffusionRequest) -> str | None:
return normalize_action_mode(_extra_args(request).get("action_mode"))
def _set_transfer_size_from_image(request: OmniDiffusionRequest, image: PIL.Image.Image) -> tuple[int, int]:
extra = _extra_args(request)
resolution = extra.get("resolution", extra.get("image_size", 720))
target_w, target_h = find_closest_target_size(image.height, image.width, resolution)
request.sampling_params.height = target_h
request.sampling_params.width = target_w
return int(target_h), int(target_w)
def _set_action_size_from_image(request: OmniDiffusionRequest, image: PIL.Image.Image) -> tuple[int, int]:
sp = request.sampling_params
if sp.height is not None and sp.width is not None:
@@ -248,7 +270,43 @@ def get_cosmos3_pre_process_func(od_config: OmniDiffusionConfig):
image = image.crop((left, top, left + target_w, top + target_h))
return video_processor.preprocess(image, height=target_h, width=target_w)
def _video_payload_value(video: Any, key: str) -> Any:
if isinstance(video, Mapping):
return video.get(key)
return getattr(video, key, None)
def _video_payload_fps(video: Any) -> float | None:
for key in ("fps", "frame_rate", "source_fps", "input_fps", "avg_fps", "average_fps"):
fps = positive_float(_video_payload_value(video, key))
if fps is not None:
return fps
for key in ("metadata", "info"):
metadata = _video_payload_value(video, key)
if metadata is None or metadata is video:
continue
fps = _video_payload_fps(metadata)
if fps is not None:
return fps
if isinstance(video, Mapping):
for key in ("frames", "data", "video"):
nested = video.get(key)
if nested is None or nested is video:
continue
fps = _video_payload_fps(nested)
if fps is not None:
return fps
return None
def _unwrap_video_payload(video: Any) -> Any:
if isinstance(video, Mapping):
for key in ("frames", "data", "video"):
nested = video.get(key)
if nested is not None:
return nested
return video
def _video_payload_to_frames(video: Any) -> list[Any]:
video = _unwrap_video_payload(video)
if isinstance(video, list):
return video
if isinstance(video, torch.Tensor):
@@ -315,7 +373,9 @@ def get_cosmos3_pre_process_func(od_config: OmniDiffusionConfig):
prompt["additional_information"] = {}
raw_video_frames: list[Any] | None = None
transfer_input_fps: float | None = None
if raw_video is not None:
transfer_input_fps = _video_payload_fps(raw_video)
raw_video_frames = _video_payload_to_frames(raw_video)
if not raw_video_frames:
raise TypeError("Cosmos3 video input must be a non-empty list of PIL images or image paths.")
@@ -325,9 +385,13 @@ def get_cosmos3_pre_process_func(od_config: OmniDiffusionConfig):
image = _pil_to_rgb(raw_video_frames[0])
else:
image = _pil_to_rgb(raw_image)
extra = _extra_args(request)
transfer_requested = action_mode is None and has_transfer_hints(extra)
# Auto-calculate H/W from aspect ratio (720p max area)
if request.sampling_params.height is None or request.sampling_params.width is None:
if transfer_requested:
_set_transfer_size_from_image(request, image)
elif request.sampling_params.height is None or request.sampling_params.width is None:
if action_mode is not None:
_set_action_size_from_image(request, image)
else:
@@ -357,27 +421,40 @@ def get_cosmos3_pre_process_func(od_config: OmniDiffusionConfig):
)
else:
assert raw_video_frames is not None
extra = _extra_args(request)
condition_frame_indexes_vision = normalize_condition_frame_indexes_vision(
extra.get(
"condition_frame_indexes_vision",
prompt.get("condition_frame_indexes_vision"),
if transfer_requested:
if transfer_input_fps is not None:
prompt["additional_information"]["transfer_input_fps"] = transfer_input_fps
transfer_frames = media_to_uint8_cthw(
raw_video_frames,
height=int(target_h),
width=int(target_w),
max_frames=transfer_max_frames_from_extra_args(extra),
)
prompt["additional_information"]["preprocessed_transfer_video"] = uint8_cthw_to_normalized_5d(
transfer_frames,
dtype=torch.float32,
)
else:
condition_frame_indexes_vision = normalize_condition_frame_indexes_vision(
extra.get(
"condition_frame_indexes_vision",
prompt.get("condition_frame_indexes_vision"),
)
)
keep = normalize_condition_video_keep(
extra.get("condition_video_keep", prompt.get("condition_video_keep"))
)
max_frames = condition_pixel_frame_count(condition_frame_indexes_vision)
prompt["additional_information"]["preprocessed_video"] = _preprocess_condition_video(
raw_video_frames,
int(target_h),
int(target_w),
max_frames,
keep,
)
prompt["additional_information"]["condition_frame_indexes_vision"] = list(
condition_frame_indexes_vision
)
)
keep = normalize_condition_video_keep(
extra.get("condition_video_keep", prompt.get("condition_video_keep"))
)
max_frames = condition_pixel_frame_count(condition_frame_indexes_vision)
prompt["additional_information"]["preprocessed_video"] = _preprocess_condition_video(
raw_video_frames,
int(target_h),
int(target_w),
max_frames,
keep,
)
prompt["additional_information"]["condition_frame_indexes_vision"] = list(
condition_frame_indexes_vision
)
if action_mode is not None and raw_video_frames is not None:
prompt["additional_information"]["preprocessed_video"] = _preprocess_action_video(
raw_video_frames,
@@ -558,6 +635,12 @@ class Cosmos3OmniDiffusersPipeline(
extra_args: dict[str, Any] | None = None,
) -> ReferenceVideoDecodeSpec:
extra_args = extra_args if isinstance(extra_args, dict) else {}
if has_transfer_hints(extra_args):
max_frames = transfer_max_frames_from_extra_args(extra_args)
if num_frames is not None:
max_frames = min(max_frames, int(num_frames))
return ReferenceVideoDecodeSpec(max_frames=max_frames, keep="first")
action_mode = normalize_action_mode(extra_args.get("action_mode"))
if action_mode is not None:
if num_frames is not None:
@@ -677,6 +760,7 @@ class Cosmos3OmniDiffusersPipeline(
self._guidance_scale = None
self._num_timesteps = None
self._cosmos3_branch_caches: dict[str, tuple[Any, Any]] | None = None
self._robolab_transform = None
# Set True by ``enable_cache_for_cosmos3`` when cache-dit is enabled on
@@ -850,7 +934,63 @@ class Cosmos3OmniDiffusersPipeline(
The transformer returns the raw prediction: video-only as a tensor,
or a tuple in video, action, sound order for multimodal generation.
"""
return self.transformer(**kwargs)
cache_key = kwargs.pop("_cosmos3_cache_key", None)
if cache_key is None:
return self.transformer(**kwargs)
branch_caches = self._cosmos3_branch_caches
if branch_caches is None:
return self.transformer(**kwargs)
cache_key = str(cache_key)
self.transformer.cached_kv, self.transformer.cached_freqs_gen = branch_caches.get(cache_key, (None, None))
prediction = self.transformer(**kwargs)
branch_caches[cache_key] = (self.transformer.cached_kv, self.transformer.cached_freqs_gen)
return prediction
def combine_multi_branch_cfg_noise(
self,
predictions: list[torch.Tensor | tuple[torch.Tensor, ...]],
true_cfg_scale: float | dict[str, float],
cfg_normalize: bool = False,
) -> torch.Tensor | tuple[torch.Tensor, ...]:
if not isinstance(true_cfg_scale, dict) or true_cfg_scale.get("mode") != "cosmos3_transfer":
return super().combine_multi_branch_cfg_noise(predictions, true_cfg_scale, cfg_normalize)
mode = str(true_cfg_scale.get("branch_mode", "control_and_text"))
guidance_scale = float(true_cfg_scale.get("guidance_scale", 1.0))
control_guidance = float(true_cfg_scale.get("control_guidance", 1.0))
if mode == "control_only":
if len(predictions) != 2:
raise ValueError(f"Cosmos3 transfer control-only CFG expects 2 branches, got {len(predictions)}.")
cond_full, cond_no_control = predictions
if isinstance(cond_full, tuple) or isinstance(cond_no_control, tuple):
raise ValueError("Cosmos3 transfer control-only CFG expects video-only tensor predictions.")
cfg_reference = cond_full
combined = cond_no_control + control_guidance * (cond_full - cond_no_control)
elif mode == "text_only":
if len(predictions) != 2:
raise ValueError(f"Cosmos3 transfer text-only CFG expects 2 branches, got {len(predictions)}.")
cond_full, uncond_full = predictions
if isinstance(cond_full, tuple) or isinstance(uncond_full, tuple):
raise ValueError("Cosmos3 transfer text-only CFG expects video-only tensor predictions.")
cfg_reference = cond_full
combined = uncond_full + guidance_scale * (cond_full - uncond_full)
elif mode == "control_and_text":
if len(predictions) != 3:
raise ValueError(f"Cosmos3 transfer control+text CFG expects 3 branches, got {len(predictions)}.")
cond_full, cond_no_control, uncond_full = predictions
if isinstance(cond_full, tuple) or isinstance(cond_no_control, tuple) or isinstance(uncond_full, tuple):
raise ValueError("Cosmos3 transfer control+text CFG expects video-only tensor predictions.")
cfg_reference = cond_full
control_cond = cond_no_control + control_guidance * (cond_full - cond_no_control)
combined = uncond_full + guidance_scale * (control_cond - uncond_full)
else:
raise ValueError(f"Unknown Cosmos3 transfer CFG branch_mode={mode!r}.")
if cfg_normalize:
combined = self.cfg_normalize_function(cfg_reference, combined)
return combined
@staticmethod
def _cfg_parallel_active() -> bool:
@@ -2176,6 +2316,444 @@ class Cosmos3OmniDiffusersPipeline(
outputs.append(sound_latents)
return outputs[0] if len(outputs) == 1 else tuple(outputs)
@staticmethod
def _get_transfer_num_chunks(
total_frames: int,
frames_per_chunk: int,
conditional_frames: int,
) -> tuple[int, int]:
if frames_per_chunk <= 0:
raise ValueError("Cosmos3 transfer frames_per_chunk must be positive.")
if total_frames <= frames_per_chunk:
return 1, frames_per_chunk
stride = frames_per_chunk - conditional_frames
if stride <= 0:
raise ValueError("Cosmos3 transfer num_conditional_frames must be smaller than num_video_frames_per_chunk.")
remaining = total_frames - frames_per_chunk
extra_chunks = remaining // stride + (1 if remaining % stride else 0)
return 1 + extra_chunks, stride
def _prepare_transfer_latents(
self,
target_video: torch.Tensor,
current_conditional_frames: int,
generator: torch.Generator,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
condition_latents = self._encode_video_tensor(target_video)
noise = randn_tensor(
condition_latents.shape,
generator=generator,
device=self.device,
dtype=self.dtype,
)
condition_mask = torch.zeros(
1,
1,
condition_latents.shape[2],
1,
1,
device=self.device,
dtype=self.dtype,
)
if current_conditional_frames > 0:
latent_frames = (current_conditional_frames - 1) // self.vae_scale_factor_temporal + 1
condition_mask[:, :, :latent_frames] = 1.0
latents = condition_mask * condition_latents + (1.0 - condition_mask) * noise
velocity_mask = 1.0 - condition_mask
return latents, velocity_mask, condition_mask * condition_latents
def _transfer_bucket_size(
self,
sp: OmniDiffusionSamplingParams,
source_hw: tuple[int, int] | None,
) -> tuple[int, int]:
resolution = self._get_sp_param(sp, "resolution", self._get_sp_param(sp, "image_size", 720))
source_h, source_w = source_hw or (COSMOS3_T2V_DEFAULT_HEIGHT, COSMOS3_T2V_DEFAULT_WIDTH)
target_w, target_h = find_closest_target_size(int(source_h), int(source_w), resolution)
return int(target_h), int(target_w)
@staticmethod
def _first_transfer_control_hw(transfer_config: Cosmos3TransferConfig) -> tuple[int, int] | None:
for hint in transfer_config.ordered_hints:
if hint.control is not None:
detected = media_hw(hint.control)
if detected is not None:
return detected
if hint.control_path is not None:
detected = media_hw(hint.control_path)
if detected is not None:
return detected
return None
def diffuse_transfer(
self,
latents: torch.Tensor,
timesteps: torch.Tensor,
cond_ids: torch.Tensor,
cond_mask: torch.Tensor,
uncond_ids: torch.Tensor,
uncond_mask: torch.Tensor,
guidance_scale: float,
control_guidance: float,
control_guidance_interval: tuple[float, float] | None,
control_latents: list[torch.Tensor],
shared_kwargs: dict[str, Any],
*,
velocity_mask: torch.Tensor,
condition_latents: torch.Tensor,
guidance_interval: tuple[float, float] | None = None,
) -> torch.Tensor:
def _active_at(t: torch.Tensor, interval: tuple[float, float] | None) -> bool:
if interval is None:
return True
t_scalar = float(t.item()) if torch.is_tensor(t) else float(t)
lo, hi = interval
return lo <= t_scalar <= hi
self.transformer.reset_cache()
self._cosmos3_branch_caches = {}
try:
for t in self.progress_bar(timesteps):
timestep = t.unsqueeze(0)
step_guidance = guidance_scale if _active_at(t, guidance_interval) else 1.0
step_control = control_guidance if _active_at(t, control_guidance_interval) else 1.0
needs_text_cfg = step_guidance > 1.0
needs_control_cfg = step_control != 1.0
cond_full_kwargs = dict(
hidden_states=latents,
timestep=timestep,
text_ids=cond_ids,
text_mask=cond_mask,
control_latents=control_latents,
_cosmos3_cache_key="transfer_cond_full",
**shared_kwargs,
)
if needs_control_cfg and needs_text_cfg:
branches_kwargs = [
cond_full_kwargs,
dict(
hidden_states=latents,
timestep=timestep,
text_ids=cond_ids,
text_mask=cond_mask,
control_latents=None,
_cosmos3_cache_key="transfer_cond_no_control",
**shared_kwargs,
),
dict(
hidden_states=latents,
timestep=timestep,
text_ids=uncond_ids,
text_mask=uncond_mask,
control_latents=control_latents,
_cosmos3_cache_key="transfer_uncond_full",
**shared_kwargs,
),
]
noise_pred = self.predict_noise_with_multi_branch_cfg(
do_true_cfg=True,
true_cfg_scale={
"mode": "cosmos3_transfer",
"branch_mode": "control_and_text",
"guidance_scale": step_guidance,
"control_guidance": step_control,
},
branches_kwargs=branches_kwargs,
cfg_normalize=False,
)
elif needs_control_cfg:
branches_kwargs = [
cond_full_kwargs,
dict(
hidden_states=latents,
timestep=timestep,
text_ids=cond_ids,
text_mask=cond_mask,
control_latents=None,
_cosmos3_cache_key="transfer_cond_no_control",
**shared_kwargs,
),
]
noise_pred = self.predict_noise_with_multi_branch_cfg(
do_true_cfg=True,
true_cfg_scale={
"mode": "cosmos3_transfer",
"branch_mode": "control_only",
"control_guidance": step_control,
},
branches_kwargs=branches_kwargs,
cfg_normalize=False,
)
elif needs_text_cfg:
branches_kwargs = [
cond_full_kwargs,
dict(
hidden_states=latents,
timestep=timestep,
text_ids=uncond_ids,
text_mask=uncond_mask,
control_latents=control_latents,
_cosmos3_cache_key="transfer_uncond_full",
**shared_kwargs,
),
]
noise_pred = self.predict_noise_with_multi_branch_cfg(
do_true_cfg=True,
true_cfg_scale={
"mode": "cosmos3_transfer",
"branch_mode": "text_only",
"guidance_scale": step_guidance,
},
branches_kwargs=branches_kwargs,
cfg_normalize=False,
)
else:
noise_pred = self.predict_noise(**cond_full_kwargs)
if isinstance(noise_pred, tuple):
raise ValueError("Cosmos3 transfer diffusion expects video-only tensor predictions.")
noise_pred = noise_pred * velocity_mask
latents = self.scheduler.step(noise_pred, t, latents, return_dict=False)[0]
latents = velocity_mask * latents + (1.0 - velocity_mask) * condition_latents
finally:
self._cosmos3_branch_caches = None
self.transformer.reset_cache()
return latents
def _forward_transfer(
self,
*,
prompt: str,
negative_prompt: str,
sp: OmniDiffusionSamplingParams,
transfer_config: Cosmos3TransferConfig,
transfer_video_tensor: torch.Tensor | None,
transfer_input_fps: float | None,
) -> DiffusionOutput:
input_frames = None
if transfer_video_tensor is not None:
input_frames = normalized_video_to_uint8_cthw(transfer_video_tensor)
source_hw = (int(input_frames.shape[-2]), int(input_frames.shape[-1]))
else:
source_hw = self._first_transfer_control_hw(transfer_config)
height, width = self._transfer_bucket_size(sp, source_hw)
if input_frames is not None:
if tuple(input_frames.shape[-2:]) != (height, width):
input_frames = resize_center_crop_uint8_cthw(input_frames, height, width)
input_frames = input_frames[:, : transfer_config.max_frames]
per_hint_frames: dict[str, torch.Tensor] = {}
for hint in transfer_config.ordered_hints:
frames = load_or_compute_control_frames(
hint,
height=height,
width=width,
max_frames=transfer_config.max_frames,
input_frames=input_frames,
)
if frames.shape[1] < 1:
raise ValueError(f"Cosmos3 transfer hint '{hint.key}' produced no frames.")
per_hint_frames[hint.key] = frames
if not per_hint_frames:
raise ValueError("Cosmos3 transfer requires at least one control hint.")
total_frames = next(iter(per_hint_frames.values())).shape[1]
if transfer_config.num_frames is not None:
total_frames = min(total_frames, int(transfer_config.num_frames))
total_frames = max(1, total_frames)
per_hint_frames = {key: pad_temporal_frames(frames, total_frames) for key, frames in per_hint_frames.items()}
if input_frames is not None:
input_frames = pad_temporal_frames(input_frames, total_frames)
temporal_compression = self.vae_scale_factor_temporal
chunk_frames = 1 if total_frames == 1 else transfer_config.num_video_frames_per_chunk
chunk_frames = math.ceil((chunk_frames - 1) / temporal_compression) * temporal_compression + 1
num_chunks, stride = self._get_transfer_num_chunks(
total_frames,
chunk_frames,
transfer_config.num_conditional_frames,
)
padded_frames = max(total_frames, chunk_frames)
per_hint_frames = {key: pad_temporal_frames(frames, padded_frames) for key, frames in per_hint_frames.items()}
if input_frames is not None:
input_frames = pad_temporal_frames(input_frames, padded_frames)
configured_frame_rate = positive_float(transfer_config.fps)
input_frame_rate = positive_float(transfer_input_fps)
sampling_frame_rate = (
positive_float(self._get_sp_param(sp, "resolved_frame_rate"))
or positive_float(self._get_sp_param(sp, "frame_rate"))
or positive_float(self._get_sp_param(sp, "fps"))
)
is_wsm_only = len(transfer_config.hints) == 1 and "wsm" in transfer_config.hints
if is_wsm_only:
frame_rate = configured_frame_rate or input_frame_rate or sampling_frame_rate or 24.0
else:
frame_rate = input_frame_rate or configured_frame_rate or sampling_frame_rate or 24.0
num_inference_steps = sp.num_inference_steps or COSMOS3_T2V_DEFAULT_NUM_INFERENCE_STEPS
guidance_scale = (
float(transfer_config.guidance_scale)
if transfer_config.guidance_scale is not None
else float(sp.guidance_scale or COSMOS3_T2V_DEFAULT_GUIDANCE_SCALE)
)
flow_shift_target = float(
transfer_config.flow_shift
if transfer_config.flow_shift is not None
else self._get_sp_param(sp, "flow_shift", COSMOS3_V2V_DEFAULT_FLOW_SHIFT)
)
max_sequence_length = (
self._get_sp_param(sp, "max_sequence_length", COSMOS3_DEFAULT_MAX_SEQUENCE_LENGTH)
or COSMOS3_DEFAULT_MAX_SEQUENCE_LENGTH
)
use_system_prompt = bool(self._get_sp_param(sp, "use_system_prompt", False))
self._guidance_scale = guidance_scale
self._num_timesteps = num_inference_steps
self._set_flow_shift(flow_shift_target)
generator = sp.generator
if generator is None:
seed = sp.seed if sp.seed is not None else 42
generator = torch.Generator(device=self.device).manual_seed(seed)
cond_ids, cond_mask, uncond_ids, uncond_mask = self._format_and_tokenize_prompts(
prompt,
negative_prompt,
chunk_frames,
frame_rate,
height,
width,
max_sequence_length,
sp,
use_system_prompt,
is_t2i=False,
)
output_chunks: list[torch.Tensor] = []
control_chunks_per_hint: dict[str, list[torch.Tensor]] = {key: [] for key in per_hint_frames}
previous_output: torch.Tensor | None = None
for chunk_id in range(num_chunks):
start_frame = chunk_id * stride
end_frame = min(start_frame + chunk_frames, total_frames)
control_norms = {
key: uint8_cthw_to_normalized_5d(
pad_temporal_frames(frames[:, start_frame:end_frame], chunk_frames),
dtype=self.dtype,
)
for key, frames in per_hint_frames.items()
}
target_norm = torch.zeros_like(next(iter(control_norms.values())))
current_conditional_frames = 0
if chunk_id == 0 and transfer_config.num_first_chunk_conditional_frames > 0:
if input_frames is None:
raise ValueError("Cosmos3 transfer num_first_chunk_conditional_frames > 0 requires a video input.")
current_conditional_frames = min(
transfer_config.num_first_chunk_conditional_frames,
input_frames.shape[1],
chunk_frames,
)
if current_conditional_frames > 0:
input_cond = uint8_cthw_to_normalized_5d(
input_frames[:, :current_conditional_frames],
dtype=self.dtype,
)
target_norm[:, :, :current_conditional_frames] = input_cond
if current_conditional_frames < chunk_frames:
fill = target_norm[:, :, current_conditional_frames - 1 : current_conditional_frames]
target_norm[:, :, current_conditional_frames:] = fill.expand(
-1,
-1,
chunk_frames - current_conditional_frames,
-1,
-1,
)
elif chunk_id > 0 and previous_output is not None:
current_conditional_frames = min(
transfer_config.num_conditional_frames,
previous_output.shape[2],
chunk_frames,
)
if current_conditional_frames > 0:
target_norm[:, :, :current_conditional_frames] = previous_output[
:, :, -current_conditional_frames:
].to(target_norm)
if current_conditional_frames < chunk_frames:
fill = target_norm[:, :, current_conditional_frames - 1 : current_conditional_frames]
target_norm[:, :, current_conditional_frames:] = fill.expand(
-1,
-1,
chunk_frames - current_conditional_frames,
-1,
-1,
)
control_latents = [self._encode_video_tensor(video) for video in control_norms.values()]
latents, velocity_mask, condition_latents = self._prepare_transfer_latents(
target_norm,
current_conditional_frames,
generator,
)
video_shape = (latents.shape[2], latents.shape[3], latents.shape[4])
shared_kwargs = dict(
video_shape=video_shape,
fps=frame_rate,
noisy_frame_mask=velocity_mask,
transfer_share_vision_temporal_positions=transfer_config.share_vision_temporal_positions,
)
self.scheduler.set_timesteps(num_inference_steps, device=self.device)
latents = self.diffuse_transfer(
latents=latents,
timesteps=self.scheduler.timesteps,
cond_ids=cond_ids,
cond_mask=cond_mask,
uncond_ids=uncond_ids,
uncond_mask=uncond_mask,
guidance_scale=guidance_scale,
control_guidance=transfer_config.control_guidance,
control_guidance_interval=transfer_config.control_guidance_interval,
control_latents=control_latents,
shared_kwargs=shared_kwargs,
velocity_mask=velocity_mask,
condition_latents=condition_latents,
)
output_video = self._decode_latents(latents).clamp(-1, 1)
previous_output = output_video
if chunk_id == 0:
output_chunks.append(output_video)
for key, control in control_norms.items():
control_chunks_per_hint[key].append(control)
else:
output_chunks.append(output_video[:, :, current_conditional_frames:])
for key, control in control_norms.items():
control_chunks_per_hint[key].append(control[:, :, current_conditional_frames:])
full_output = torch.cat(output_chunks, dim=2)[:, :, :total_frames]
full_controls = {
key: torch.cat(chunks, dim=2)[:, :, :total_frames] for key, chunks in control_chunks_per_hint.items()
}
if transfer_config.show_control_condition:
all_controls = torch.cat([full_controls[key] for key in per_hint_frames], dim=-1)
all_controls = all_controls.to(full_output)
full_output = torch.cat([all_controls, full_output], dim=-1)
if transfer_config.show_input and input_frames is not None:
normalized_input = uint8_cthw_to_normalized_5d(input_frames[:, :total_frames], dtype=torch.float32)
full_output = torch.cat([normalized_input.to(full_output), full_output], dim=-1)
return DiffusionOutput(
output={"video": full_output},
custom_output={
"transfer_controls": full_controls,
"transfer_hints": list(per_hint_frames),
"fps": frame_rate,
},
)
# -- Forward (main generation entry point) -------------------------------
def forward(
@@ -2199,18 +2777,30 @@ class Cosmos3OmniDiffusersPipeline(
negative_prompt = None
image_tensor = None
video_tensor = None
transfer_video_tensor = None
transfer_input_fps = None
else:
prompt = prompt_data.get("prompt", "")
negative_prompt = prompt_data.get("negative_prompt")
additional_info = prompt_data.get("additional_information", {}) or {}
image_tensor = additional_info.get("preprocessed_image")
video_tensor = additional_info.get("preprocessed_video")
transfer_video_tensor = additional_info.get("preprocessed_transfer_video")
transfer_input_fps = positive_float(additional_info.get("transfer_input_fps"))
is_t2i = self._is_t2i_request(req)
sound_enabled = self._is_sound_request(prompt_data, sp)
action_mode = self._get_action_mode(prompt_data, sp)
action_enabled = action_mode is not None
transfer_config = resolve_transfer_config(sp, prompt_data)
action_video_tensor = video_tensor if action_enabled else None
if transfer_config is not None:
if is_t2i:
raise ValueError("Cosmos3 transfer inference is supported only for video outputs.")
if action_enabled:
raise ValueError("Cosmos3 transfer inference cannot be combined with action generation.")
if sound_enabled:
raise ValueError("Cosmos3 transfer inference cannot be combined with sound generation.")
if action_enabled and is_t2i:
raise ValueError("Cosmos3 action generation is supported only for video outputs.")
if action_enabled and sound_enabled:
@@ -2234,6 +2824,19 @@ class Cosmos3OmniDiffusersPipeline(
)
if negative_prompt is None:
negative_prompt = ""
if transfer_config is not None:
if image_tensor is not None:
raise ValueError("Cosmos3 transfer inference accepts video inputs or control_path values, not images.")
if transfer_video_tensor is None and video_tensor is not None:
transfer_video_tensor = video_tensor
return self._forward_transfer(
prompt=prompt,
negative_prompt=negative_prompt,
sp=sp,
transfer_config=transfer_config,
transfer_video_tensor=transfer_video_tensor,
transfer_input_fps=transfer_input_fps,
)
if image_tensor is not None and video_tensor is not None and not action_enabled:
raise ValueError("Cosmos3 non-action generation accepts either image or video input, not both.")
if video_tensor is not None and is_t2i:
@@ -2525,14 +3128,19 @@ class Cosmos3OmniDiffusersPipeline(
video = self._decode_latents(latents)
if _is_rank_zero():
logger.info("Video decoded in %.2fs", time.time() - decode_start)
logger.info("Total pipeline time: %.2fs", time.time() - pipeline_start)
if not sound_enabled:
logger.info("Total pipeline time: %.2fs", time.time() - pipeline_start)
if sound_enabled:
if sound_latents is None or target_audio_samples is None or sound_sample_rate is None:
raise ValueError("Cosmos3 sound generation finished without sound latents.")
if _is_rank_zero():
logger.info("Decoding sound...")
sound_decode_start = time.time()
audio = self._decode_sound_latents(sound_latents, target_audio_samples)
if _is_rank_zero():
logger.info("Sound tokenizer decoded in %.2fs", time.time() - sound_decode_start)
logger.info("Total pipeline time: %.2fs", time.time() - pipeline_start)
return DiffusionOutput(output={"video": video, "audio": audio, "audio_sample_rate": sound_sample_rate})
if action_enabled:
@@ -0,0 +1,625 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Cosmos3 transfer inference helpers.
The reference Cosmos Framework transfer path accepts one or more control hints
and packs them before the noisy target video. This module keeps the vLLM Omni
pipeline-specific pieces small: request parsing, media normalization, and
optional on-the-fly edge/blur control generation.
"""
from __future__ import annotations
import importlib
from collections.abc import Mapping
from dataclasses import dataclass, field
from pathlib import Path
from typing import Any
import numpy as np
import PIL.Image
import torch
import torch.nn.functional as F
TRANSFER_HINT_KEYS: tuple[str, ...] = ("edge", "blur", "depth", "seg", "wsm")
TRANSFER_SAMPLE_DEFAULTS: dict[str, Any] = {
"num_video_frames_per_chunk": 93,
"num_conditional_frames": 1,
"max_frames": 5000,
"show_control_condition": False,
"show_input": False,
"num_first_chunk_conditional_frames": 0,
"share_vision_temporal_positions": True,
}
TRANSFER_DEFAULTS: dict[str, dict[str, Any]] = {
"edge": {"guidance_scale": 3.0, "control_guidance": 1.5, "flow_shift": 10.0},
"blur": {"guidance_scale": 3.0, "control_guidance": 1.5, "flow_shift": 10.0},
"depth": {"guidance_scale": 3.0, "control_guidance": 1.5, "flow_shift": 10.0},
"seg": {"guidance_scale": 3.0, "control_guidance": 2.0, "flow_shift": 10.0},
"wsm": {
"guidance_scale": 1.0,
"control_guidance": 3.0,
"flow_shift": 10.0,
"num_frames": 101,
"fps": 10,
"num_video_frames_per_chunk": 101,
},
}
EDGE_PRESETS: dict[str, tuple[int, int]] = {
"none": (20, 50),
"very_low": (20, 50),
"low": (50, 100),
"medium": (100, 200),
"high": (200, 300),
"very_high": (300, 400),
}
BLUR_DOWNUP_PRESETS: dict[str, int] = {
"none": 1,
"very_low": 4,
"low": 4,
"medium": 10,
"high": 16,
"very_high": 16,
}
BLUR_PRE_BLUR_DOWNSCALE_PRESETS: dict[str, int] = {
"none": 1,
"very_low": 1,
"low": 4,
"medium": 2,
"high": 1,
"very_high": 4,
}
BILATERAL_REFERENCE_RESOLUTION = 720
BILATERAL_D = 30
BILATERAL_SIGMA_COLOR = 150
BILATERAL_SIGMA_SPACE = 100
BILATERAL_ITERATIONS = 1
IMAGE_EXTENSIONS = {".bmp", ".gif", ".jpg", ".jpeg", ".png", ".tif", ".tiff", ".webp"}
@dataclass
class Cosmos3TransferHint:
key: str
control_path: str | None = None
control: Any | None = None
preset_edge_threshold: str = "medium"
preset_blur_strength: str = "medium"
@dataclass
class Cosmos3TransferConfig:
hints: dict[str, Cosmos3TransferHint] = field(default_factory=dict)
guidance_scale: float | None = None
control_guidance: float = 1.0
control_guidance_interval: tuple[float, float] | None = None
flow_shift: float | None = None
num_video_frames_per_chunk: int = 93
num_conditional_frames: int = 1
max_frames: int = 5000
show_control_condition: bool = False
show_input: bool = False
num_first_chunk_conditional_frames: int = 0
share_vision_temporal_positions: bool = True
num_frames: int | None = None
fps: float | None = None
@property
def ordered_hints(self) -> list[Cosmos3TransferHint]:
return [self.hints[key] for key in TRANSFER_HINT_KEYS if key in self.hints]
def _extra_args(sp: Any) -> Mapping[str, Any]:
extra = getattr(sp, "extra_args", None)
return extra if isinstance(extra, Mapping) else {}
def _prompt_value(prompt_data: Any, key: str) -> Any:
if not isinstance(prompt_data, Mapping):
return None
if prompt_data.get(key) is not None:
return prompt_data[key]
additional = prompt_data.get("additional_information")
if isinstance(additional, Mapping):
return additional.get(key)
return None
def _param(extra: Mapping[str, Any], sp: Any, prompt_data: Any, key: str, default: Any = None) -> Any:
if extra.get(key) is not None:
return extra[key]
value = getattr(sp, key, None)
if value is not None:
return value
value = _prompt_value(prompt_data, key)
return default if value is None else value
def _param_any(extra: Mapping[str, Any], sp: Any, prompt_data: Any, keys: tuple[str, ...], default: Any = None) -> Any:
for key in keys:
value = _param(extra, sp, prompt_data, key, None)
if value is not None:
return value
return default
def _is_user_field(extra: Mapping[str, Any], sp: Any, prompt_data: Any, *keys: str) -> bool:
for key in keys:
if key in extra and extra[key] is not None:
return True
if getattr(sp, key, None) is not None:
return True
if _prompt_value(prompt_data, key) is not None:
return True
return False
# fps, frame_rate, and resolved_frame_rate are aliases for the same request-level
# control. The serving layer leaves all of them ``None`` on the sampling params when
# the user did not provide fps, so a non-``None`` value here unambiguously means the
# user set it.
_FPS_KEYS: tuple[str, ...] = ("resolved_frame_rate", "frame_rate", "fps")
def _transfer_fps_param(extra: Mapping[str, Any], sp: Any, prompt_data: Any) -> Any:
# extra_args is a model-specific override and wins over the generic sampling params.
for key in _FPS_KEYS:
if extra.get(key) is not None:
return extra[key]
return _param_any(extra, sp, prompt_data, _FPS_KEYS)
def _as_bool(value: Any, default: bool = False) -> bool:
if value is None:
return default
if isinstance(value, str):
return value.strip().lower() in {"1", "true", "yes", "on"}
return bool(value)
def _as_interval(value: Any) -> tuple[float, float] | None:
if value is None:
return None
if isinstance(value, str):
value = [item.strip() for item in value.split(",") if item.strip()]
if not isinstance(value, (list, tuple)) or len(value) != 2:
raise ValueError("Cosmos3 transfer control_guidance_interval must contain exactly two values.")
lo, hi = float(value[0]), float(value[1])
if lo > hi:
raise ValueError(f"Cosmos3 transfer control_guidance_interval must be ordered as [lo, hi], got {(lo, hi)}.")
return lo, hi
def has_transfer_hints(extra_args: Mapping[str, Any] | None) -> bool:
if not isinstance(extra_args, Mapping):
return False
return any(extra_args.get(key) is not None for key in TRANSFER_HINT_KEYS)
def transfer_max_frames_from_extra_args(extra_args: Mapping[str, Any] | None) -> int:
if not isinstance(extra_args, Mapping):
return int(TRANSFER_SAMPLE_DEFAULTS["max_frames"])
value = extra_args.get("max_frames", TRANSFER_SAMPLE_DEFAULTS["max_frames"])
return max(1, int(value))
def resolve_transfer_config(sp: Any, prompt_data: Any = None) -> Cosmos3TransferConfig | None:
extra = _extra_args(sp)
hints: dict[str, Cosmos3TransferHint] = {}
for key in TRANSFER_HINT_KEYS:
raw = _param(extra, sp, prompt_data, key, None)
if raw is None:
continue
if raw is True:
raw = {}
elif isinstance(raw, str | Path):
raw = {"control_path": str(raw)}
if not isinstance(raw, Mapping):
raise TypeError(
f"Cosmos3 transfer hint '{key}' must be an object, path string, or true; got {type(raw)!r}."
)
hints[key] = Cosmos3TransferHint(
key=key,
control_path=str(raw["control_path"]) if raw.get("control_path") is not None else None,
control=raw.get("control"),
preset_edge_threshold=str(raw.get("preset_edge_threshold") or "medium").lower(),
preset_blur_strength=str(raw.get("preset_blur_strength") or "medium").lower(),
)
if not hints:
transfer_only = (
"control_guidance",
"control_guidance_interval",
"num_video_frames_per_chunk",
"num_conditional_frames",
"max_frames",
"show_control_condition",
"show_input",
"num_first_chunk_conditional_frames",
"share_vision_temporal_positions",
)
if any(_is_user_field(extra, sp, prompt_data, key) for key in transfer_only):
raise ValueError("Cosmos3 transfer options were provided, but no transfer hint was selected.")
return None
fps_value = _transfer_fps_param(extra, sp, prompt_data)
config = Cosmos3TransferConfig(
hints=hints,
guidance_scale=(
float(_param_any(extra, sp, prompt_data, ("guidance_scale", "guidance")))
if _param_any(extra, sp, prompt_data, ("guidance_scale", "guidance")) is not None
else None
),
control_guidance=float(_param(extra, sp, prompt_data, "control_guidance", 1.0)),
control_guidance_interval=_as_interval(_param(extra, sp, prompt_data, "control_guidance_interval", None)),
flow_shift=(
float(_param_any(extra, sp, prompt_data, ("flow_shift", "shift")))
if _param_any(extra, sp, prompt_data, ("flow_shift", "shift")) is not None
else None
),
num_video_frames_per_chunk=int(
_param(
extra,
sp,
prompt_data,
"num_video_frames_per_chunk",
TRANSFER_SAMPLE_DEFAULTS["num_video_frames_per_chunk"],
)
),
num_conditional_frames=int(
_param(extra, sp, prompt_data, "num_conditional_frames", TRANSFER_SAMPLE_DEFAULTS["num_conditional_frames"])
),
max_frames=int(_param(extra, sp, prompt_data, "max_frames", TRANSFER_SAMPLE_DEFAULTS["max_frames"])),
show_control_condition=_as_bool(
_param(extra, sp, prompt_data, "show_control_condition", None),
bool(TRANSFER_SAMPLE_DEFAULTS["show_control_condition"]),
),
show_input=_as_bool(
_param(extra, sp, prompt_data, "show_input", None),
bool(TRANSFER_SAMPLE_DEFAULTS["show_input"]),
),
num_first_chunk_conditional_frames=int(
_param(
extra,
sp,
prompt_data,
"num_first_chunk_conditional_frames",
TRANSFER_SAMPLE_DEFAULTS["num_first_chunk_conditional_frames"],
)
),
share_vision_temporal_positions=_as_bool(
_param(extra, sp, prompt_data, "share_vision_temporal_positions", None),
bool(TRANSFER_SAMPLE_DEFAULTS["share_vision_temporal_positions"]),
),
num_frames=(
int(_param(extra, sp, prompt_data, "num_frames"))
if _param(extra, sp, prompt_data, "num_frames", None) is not None
else None
),
fps=(float(fps_value) if fps_value is not None else None),
)
if len(hints) == 1:
hint_key = next(iter(hints))
for field_name, default_value in TRANSFER_DEFAULTS[hint_key].items():
if field_name == "guidance_scale":
user_set = _is_user_field(extra, sp, prompt_data, "guidance_scale", "guidance")
elif field_name == "flow_shift":
user_set = _is_user_field(extra, sp, prompt_data, "flow_shift", "shift")
elif field_name == "fps":
user_set = _is_user_field(extra, sp, prompt_data, *_FPS_KEYS)
else:
user_set = _is_user_field(extra, sp, prompt_data, field_name)
if not user_set:
setattr(config, field_name, default_value)
if config.num_video_frames_per_chunk <= 0:
raise ValueError("Cosmos3 transfer num_video_frames_per_chunk must be positive.")
if config.num_conditional_frames < 0:
raise ValueError("Cosmos3 transfer num_conditional_frames must be non-negative.")
if config.max_frames <= 0:
raise ValueError("Cosmos3 transfer max_frames must be positive.")
if config.num_first_chunk_conditional_frames < 0:
raise ValueError("Cosmos3 transfer num_first_chunk_conditional_frames must be non-negative.")
for hint in hints.values():
if hint.key == "edge" and hint.preset_edge_threshold not in EDGE_PRESETS:
raise ValueError(f"Unsupported Cosmos3 edge preset: {hint.preset_edge_threshold!r}.")
if hint.key == "blur" and hint.preset_blur_strength not in BLUR_DOWNUP_PRESETS:
raise ValueError(f"Unsupported Cosmos3 blur preset: {hint.preset_blur_strength!r}.")
return config
def _import_cv2(hint_key: str):
try:
return importlib.import_module("cv2")
except ImportError as exc:
raise ImportError(
f"Cosmos3 transfer hint '{hint_key}' requires opencv-python for on-the-fly control generation. "
"Install opencv-python in the environment, or provide a precomputed control_path for this hint."
) from exc
def pad_temporal_frames(frames: torch.Tensor, target_frames: int) -> torch.Tensor:
if frames.ndim != 4:
raise ValueError(f"Cosmos3 transfer frames must have shape [C, T, H, W], got {tuple(frames.shape)}.")
target_frames = int(target_frames)
if target_frames <= 0:
raise ValueError("Cosmos3 transfer target frame count must be positive.")
if frames.shape[1] >= target_frames:
return frames
if frames.shape[1] == 0:
raise ValueError("Cannot pad an empty Cosmos3 transfer frame tensor.")
padded = frames
while padded.shape[1] < target_frames:
pad_len = min(padded.shape[1] - 1, target_frames - padded.shape[1])
if pad_len <= 0:
pad_frame = padded[:, -1:].repeat(1, target_frames - padded.shape[1], 1, 1)
padded = torch.cat([padded, pad_frame], dim=1)
break
padded = torch.cat([padded, padded.flip(dims=[1])[:, :pad_len]], dim=1)
return padded
def normalized_video_to_uint8_cthw(video: torch.Tensor) -> torch.Tensor:
tensor = video.detach().cpu()
if tensor.ndim == 5:
if tensor.shape[0] != 1:
raise ValueError("Cosmos3 transfer supports only batch size 1 video inputs.")
tensor = tensor[0]
if tensor.ndim != 4:
raise ValueError(
f"Cosmos3 transfer video must have shape [1, 3, T, H, W] or [3, T, H, W], got {tuple(video.shape)}."
)
if tensor.shape[0] != 3:
if tensor.shape[-1] in (3, 4):
tensor = tensor[..., :3].permute(3, 0, 1, 2)
else:
raise ValueError(f"Cosmos3 transfer video must have 3 RGB channels, got {tuple(video.shape)}.")
if tensor.is_floating_point():
if tensor.numel() and (tensor.min().item() < 0.0 or tensor.max().item() > 1.0):
tensor = tensor.clamp(-1.0, 1.0).mul(0.5).add(0.5)
tensor = tensor.clamp(0.0, 1.0).mul(255.0).round().to(torch.uint8)
else:
tensor = tensor.clamp(0, 255).to(torch.uint8)
return tensor.contiguous()
def uint8_cthw_to_normalized_5d(frames: torch.Tensor, *, dtype: torch.dtype) -> torch.Tensor:
if frames.ndim != 4 or frames.shape[0] != 3:
raise ValueError(f"Cosmos3 transfer frames must have shape [3, T, H, W], got {tuple(frames.shape)}.")
return frames.to(dtype=dtype).div(127.5).sub(1.0).unsqueeze(0).contiguous()
def _pil_to_uint8_rgb(value: Any) -> np.ndarray:
if isinstance(value, PIL.Image.Image):
return np.asarray(value.convert("RGB"), dtype=np.uint8)
if isinstance(value, str | Path):
return np.asarray(PIL.Image.open(value).convert("RGB"), dtype=np.uint8)
if isinstance(value, torch.Tensor):
tensor = value.detach().cpu()
if tensor.ndim == 3 and tensor.shape[0] in (3, 4):
tensor = tensor[:3].permute(1, 2, 0)
if tensor.is_floating_point():
if tensor.numel() and (tensor.min().item() < 0.0 or tensor.max().item() > 1.0):
tensor = tensor.clamp(-1.0, 1.0).mul(0.5).add(0.5)
tensor = tensor.clamp(0.0, 1.0).mul(255.0).round().to(torch.uint8)
return tensor[..., :3].numpy().astype(np.uint8)
if isinstance(value, np.ndarray):
array = value
if array.ndim == 3 and array.shape[0] in (3, 4) and array.shape[-1] not in (3, 4):
array = np.transpose(array[:3], (1, 2, 0))
if np.issubdtype(array.dtype, np.floating):
if array.size and (array.min() < 0.0 or array.max() > 1.0):
array = np.clip(array, -1.0, 1.0) * 0.5 + 0.5
array = (np.clip(array, 0.0, 1.0) * 255.0).round().astype(np.uint8)
return array[..., :3].astype(np.uint8)
raise TypeError(f"Cosmos3 transfer expected an RGB frame, got {type(value)!r}.")
def media_hw(value: Any) -> tuple[int, int] | None:
if isinstance(value, PIL.Image.Image):
return value.height, value.width
if isinstance(value, str | Path):
media_path = Path(value)
if media_path.suffix.lower() in IMAGE_EXTENSIONS:
with PIL.Image.open(media_path) as image:
return image.height, image.width
try:
import imageio.v3 as iio
except ImportError:
return None
for frame in iio.imiter(media_path):
array = np.asarray(frame)
if array.ndim >= 2:
return int(array.shape[0]), int(array.shape[1])
return None
if isinstance(value, torch.Tensor):
tensor = value.detach()
if tensor.ndim == 5:
tensor = tensor[0]
if tensor.ndim == 4:
if tensor.shape[0] in (3, 4) and tensor.shape[-1] not in (3, 4):
return int(tensor.shape[-2]), int(tensor.shape[-1])
if tensor.shape[-1] in (3, 4):
return int(tensor.shape[1]), int(tensor.shape[2])
if tensor.ndim == 3:
if tensor.shape[0] in (3, 4) and tensor.shape[-1] not in (3, 4):
return int(tensor.shape[-2]), int(tensor.shape[-1])
if tensor.shape[-1] in (3, 4):
return int(tensor.shape[0]), int(tensor.shape[1])
return None
if isinstance(value, np.ndarray):
array = value
if array.ndim == 5:
array = array[0]
if array.ndim == 4:
if array.shape[0] in (3, 4) and array.shape[-1] not in (3, 4):
return int(array.shape[-2]), int(array.shape[-1])
if array.shape[-1] in (3, 4):
return int(array.shape[1]), int(array.shape[2])
if array.ndim == 3:
if array.shape[0] in (3, 4) and array.shape[-1] not in (3, 4):
return int(array.shape[-2]), int(array.shape[-1])
if array.shape[-1] in (3, 4):
return int(array.shape[0]), int(array.shape[1])
return None
if isinstance(value, list | tuple) and value:
return media_hw(value[0])
return None
def resize_center_crop_uint8_cthw(frames: torch.Tensor, height: int, width: int) -> torch.Tensor:
if frames.ndim != 4 or frames.shape[0] != 3:
raise ValueError(f"Cosmos3 transfer frames must have shape [3, T, H, W], got {tuple(frames.shape)}.")
orig_h, orig_w = int(frames.shape[2]), int(frames.shape[3])
scale = max(width / orig_w, height / orig_h)
resize_h = int(np.ceil(scale * orig_h))
resize_w = int(np.ceil(scale * orig_w))
frames_tchw = frames.permute(1, 0, 2, 3).to(dtype=torch.float32)
resized = F.interpolate(frames_tchw, size=(resize_h, resize_w), mode="bilinear", align_corners=False)
top = (resize_h - height) // 2
left = (resize_w - width) // 2
cropped = resized[:, :, top : top + height, left : left + width]
return cropped.round().clamp(0, 255).to(torch.uint8).permute(1, 0, 2, 3).contiguous()
def _path_media_to_uint8_cthw(path: str | Path, max_frames: int | None) -> torch.Tensor:
media_path = Path(path)
if not media_path.exists():
raise FileNotFoundError(f"Missing Cosmos3 transfer control_path: {media_path}")
if media_path.suffix.lower() in IMAGE_EXTENSIONS:
array = _pil_to_uint8_rgb(media_path)
return torch.from_numpy(array).permute(2, 0, 1).unsqueeze(1).contiguous()
try:
import imageio.v3 as iio
except ImportError as exc:
raise ImportError(
"Cosmos3 transfer video control_path loading requires imageio. "
"Install imageio[ffmpeg] or provide decoded control frames."
) from exc
frames: list[torch.Tensor] = []
limit = max_frames if max_frames is not None else None
for frame in iio.imiter(media_path):
frames.append(torch.from_numpy(_pil_to_uint8_rgb(frame)).permute(2, 0, 1))
if limit is not None and len(frames) >= int(limit):
break
if not frames:
raise ValueError(f"Cosmos3 transfer control_path produced no frames: {media_path}")
return torch.stack(frames, dim=1).contiguous()
def media_to_uint8_cthw(value: Any, *, height: int, width: int, max_frames: int | None = None) -> torch.Tensor:
if isinstance(value, str | Path):
frames = _path_media_to_uint8_cthw(value, max_frames=max_frames)
elif isinstance(value, torch.Tensor):
frames = normalized_video_to_uint8_cthw(value)
elif isinstance(value, np.ndarray):
frames = normalized_video_to_uint8_cthw(torch.from_numpy(value))
elif isinstance(value, list | tuple):
if not value:
raise ValueError("Cosmos3 transfer control frames cannot be empty.")
selected = value[:max_frames] if max_frames is not None else value
tensors = [torch.from_numpy(_pil_to_uint8_rgb(frame)).permute(2, 0, 1) for frame in selected]
frames = torch.stack(tensors, dim=1).contiguous()
else:
raise TypeError(f"Unsupported Cosmos3 transfer control payload type: {type(value)!r}.")
if max_frames is not None:
frames = frames[:, : int(max_frames)]
return resize_center_crop_uint8_cthw(frames, int(height), int(width))
def make_edge_control(frames: torch.Tensor, preset: str) -> torch.Tensor:
cv2 = _import_cv2("edge")
try:
lower, upper = EDGE_PRESETS[preset]
except KeyError as exc:
raise ValueError(f"Unsupported Cosmos3 edge preset: {preset!r}.") from exc
frames_np = frames.detach().cpu().numpy().astype(np.uint8)
edge_maps = []
for idx in range(frames_np.shape[1]):
frame = np.ascontiguousarray(np.transpose(frames_np[:, idx], (1, 2, 0)))
edge_maps.append(cv2.Canny(frame, lower, upper))
edge = np.stack(edge_maps, axis=0)[None]
return torch.from_numpy(edge).expand(3, -1, -1, -1).contiguous()
def _scale_for_bilateral_resolution(value: float, longest_side: int) -> float:
if longest_side <= 0:
return value
return value * (longest_side / BILATERAL_REFERENCE_RESOLUTION)
def _scaled_bilateral_params(height: int, width: int) -> tuple[int, float, float]:
longest_side = int(max(height, width))
diameter = max(1, int(round(_scale_for_bilateral_resolution(float(BILATERAL_D), longest_side))))
if diameter % 2 == 0:
diameter += 1
sigma_color = max(1.0, _scale_for_bilateral_resolution(float(BILATERAL_SIGMA_COLOR), longest_side))
sigma_space = max(1.0, _scale_for_bilateral_resolution(float(BILATERAL_SIGMA_SPACE), longest_side))
return diameter, sigma_color, sigma_space
def make_blur_control(frames: torch.Tensor, preset: str) -> torch.Tensor:
cv2 = _import_cv2("blur")
preset = preset.lower()
if preset not in BLUR_DOWNUP_PRESETS:
raise ValueError(f"Unsupported Cosmos3 blur preset: {preset!r}.")
if preset == "none":
return frames.clone()
frames_np = frames.detach().cpu().numpy().astype(np.uint8)
_, t, h, w = frames_np.shape
pre_blur_factor = max(1, int(BLUR_PRE_BLUR_DOWNSCALE_PRESETS[preset]))
downup_factor = max(1, int(BLUR_DOWNUP_PRESETS[preset]))
result = np.empty_like(frames_np)
for idx in range(t):
frame = np.ascontiguousarray(np.transpose(frames_np[:, idx], (1, 2, 0)))
if pre_blur_factor > 1:
small_w = max(1, w // pre_blur_factor)
small_h = max(1, h // pre_blur_factor)
frame = cv2.resize(frame, (small_w, small_h), interpolation=cv2.INTER_AREA)
diameter, sigma_color, sigma_space = _scaled_bilateral_params(frame.shape[0], frame.shape[1])
for _ in range(BILATERAL_ITERATIONS):
frame = cv2.bilateralFilter(frame, diameter, sigma_color, sigma_space)
if pre_blur_factor > 1:
frame = cv2.resize(frame, (w, h), interpolation=cv2.INTER_LINEAR)
if downup_factor > 1:
small_w = max(1, w // downup_factor)
small_h = max(1, h // downup_factor)
frame = cv2.resize(frame, (small_w, small_h), interpolation=cv2.INTER_CUBIC)
frame = cv2.resize(frame, (w, h), interpolation=cv2.INTER_CUBIC)
result[:, idx] = np.transpose(frame, (2, 0, 1))
return torch.from_numpy(result).contiguous()
def load_or_compute_control_frames(
hint: Cosmos3TransferHint,
*,
height: int,
width: int,
max_frames: int,
input_frames: torch.Tensor | None,
) -> torch.Tensor:
if hint.control is not None:
return media_to_uint8_cthw(hint.control, height=height, width=width, max_frames=max_frames)
if hint.control_path is not None:
return media_to_uint8_cthw(hint.control_path, height=height, width=width, max_frames=max_frames)
if hint.key == "edge":
if input_frames is None:
raise ValueError(
"Cosmos3 transfer hint 'edge' requires either a video input for on-the-fly control generation "
"or a precomputed control_path."
)
return make_edge_control(input_frames[:, :max_frames], hint.preset_edge_threshold)
if hint.key == "blur":
if input_frames is None:
raise ValueError(
"Cosmos3 transfer hint 'blur' requires either a video input for on-the-fly control generation "
"or a precomputed control_path."
)
return make_blur_control(input_frames[:, :max_frames], hint.preset_blur_strength)
raise FileNotFoundError(
f"Cosmos3 transfer hint '{hint.key}' requires a precomputed control_path; "
"on-the-fly generation is supported only for edge and blur."
)
@@ -1234,8 +1234,12 @@ class Cosmos3VFMTransformer(nn.Module):
action_start_frame_offset: int = 1,
action_fps: float | None = None,
t_sound: int | None = None,
num_vision_items: int = 1,
share_vision_temporal_positions: bool = False,
) -> tuple[tuple[torch.Tensor, torch.Tensor], tuple[torch.Tensor, torch.Tensor]]:
"""Compute mRoPE cos/sin for UND text and GEN media pathways."""
if num_vision_items <= 0:
raise ValueError(f"Cosmos3 num_vision_items must be positive, got {num_vision_items}.")
B = text_mask.shape[0]
S_text = text_mask.shape[1]
text_lengths = text_mask.sum(dim=1).long()
@@ -1249,17 +1253,33 @@ class Cosmos3VFMTransformer(nn.Module):
real_len = int(text_lengths[b].item())
t_pos, t_offset = compute_mrope_position_ids_text(real_len, temporal_offset=0)
media_temporal_offset = t_offset + self.temporal_modality_margin
v_pos, _ = compute_mrope_position_ids_vision(
t,
hp,
wp,
temporal_offset=media_temporal_offset,
fps=effective_fps,
base_fps=self.base_fps,
temporal_compression_factor=self.temporal_compression_factor,
enable_fps_modulation=self.enable_fps_modulation,
)
gen_positions = [v_pos]
gen_positions = []
if num_vision_items == 1 or share_vision_temporal_positions:
v_pos, _ = compute_mrope_position_ids_vision(
t,
hp,
wp,
temporal_offset=media_temporal_offset,
fps=effective_fps,
base_fps=self.base_fps,
temporal_compression_factor=self.temporal_compression_factor,
enable_fps_modulation=self.enable_fps_modulation,
)
gen_positions.extend([v_pos] * num_vision_items)
else:
vision_offset: int | float = media_temporal_offset
for _ in range(num_vision_items):
v_pos, vision_offset = compute_mrope_position_ids_vision(
t,
hp,
wp,
temporal_offset=vision_offset,
fps=effective_fps,
base_fps=self.base_fps,
temporal_compression_factor=self.temporal_compression_factor,
enable_fps_modulation=self.enable_fps_modulation,
)
gen_positions.append(v_pos)
if action_frames > 0:
a_pos, _ = compute_mrope_position_ids_action(
action_frames,
@@ -1316,16 +1336,21 @@ class Cosmos3VFMTransformer(nn.Module):
*,
s_gen: int,
s_video: int,
s_control: int,
s_action: int,
s_sound: int,
has_action: bool,
has_sound: bool,
has_control: bool,
ulysses_size: int,
) -> None:
if ulysses_size <= 1 or s_gen % ulysses_size == 0:
return
detail_parts = [f"video tokens {s_video}"]
detail_parts = []
if has_control:
detail_parts.append(f"control tokens {s_control}")
detail_parts.append(f"video tokens {s_video}")
if has_action:
detail_parts.append(f"action tokens {s_action}")
if has_sound:
@@ -1335,7 +1360,7 @@ class Cosmos3VFMTransformer(nn.Module):
"Adjust the spatial resolution, frame count, action chunk size, "
"sound duration, or sound latent FPS so the combined media sequence is a "
"multiple of ulysses_degree."
if has_action or has_sound
if has_control or has_action or has_sound
else (
"Adjust the spatial resolution so that "
"t * ceil(h/patch) * ceil(w/patch) is a multiple "
@@ -1364,6 +1389,8 @@ class Cosmos3VFMTransformer(nn.Module):
action_fps: float | None = None,
sound_latents: torch.Tensor | None = None,
noisy_frame_mask: torch.Tensor | None = None,
control_latents: list[torch.Tensor] | tuple[torch.Tensor, ...] | torch.Tensor | None = None,
transfer_share_vision_temporal_positions: bool = True,
**kwargs,
) -> torch.Tensor | tuple[torch.Tensor, ...]:
"""
@@ -1383,11 +1410,15 @@ class Cosmos3VFMTransformer(nn.Module):
timestep embedding, predict velocity) and 0=conditioned (clean
context, skip timestep embedding). None means all frames noisy
(T2V mode).
control_latents: Optional transfer-control latents. Controls are
clean vision context and are packed before the noisy target.
Returns:
[B, C, t, h, w] velocity prediction, or
tuple outputs in video, action, sound order when extra modalities are provided.
"""
if kwargs:
raise TypeError(f"Unexpected Cosmos3 transformer kwargs: {sorted(kwargs)}")
t, h, w = video_shape
hp, wp, _, _ = self._pad_to_patch_size(h, w)
text_lengths = text_mask.sum(dim=1)
@@ -1400,6 +1431,15 @@ class Cosmos3VFMTransformer(nn.Module):
)
has_action = action_latents is not None
has_sound = sound_latents is not None
if control_latents is None:
control_latent_list: list[torch.Tensor] = []
elif isinstance(control_latents, torch.Tensor):
control_latent_list = [control_latents]
else:
control_latent_list = list(control_latents)
has_control = len(control_latent_list) > 0
if has_control and (has_action or has_sound):
raise ValueError("Cosmos3 transfer control latents cannot be combined with action or sound latents.")
if has_action and not self.action_gen:
raise ValueError(
"Cosmos3 action generation was requested, but this transformer "
@@ -1419,6 +1459,19 @@ class Cosmos3VFMTransformer(nn.Module):
# Patchify latents and project to hidden space
hidden_video = self.proj_in(self.patchify(hidden_states, t, h, w))
s_video = hidden_video.shape[1]
s_control = 0
hidden_controls: list[torch.Tensor] = []
for idx, control in enumerate(control_latent_list):
if control.shape != hidden_states.shape:
raise ValueError(
"Cosmos3 transfer control latent shape must match target latent shape: "
f"control[{idx}]={tuple(control.shape)}, target={tuple(hidden_states.shape)}."
)
hidden_control = self.proj_in(
self.patchify(control.to(device=hidden_states.device, dtype=hidden_states.dtype), t, h, w)
)
hidden_controls.append(hidden_control)
s_control += hidden_control.shape[1]
s_action = 0
hidden_action = None
s_sound = 0
@@ -1479,7 +1532,7 @@ class Cosmos3VFMTransformer(nn.Module):
if hidden_sound is not None:
hidden_sound = hidden_sound + time_embed.unsqueeze(1)
hidden_parts = [hidden_video]
hidden_parts = [*hidden_controls, hidden_video]
if hidden_action is not None:
hidden_parts.append(hidden_action)
if hidden_sound is not None:
@@ -1500,6 +1553,8 @@ class Cosmos3VFMTransformer(nn.Module):
action_start_frame_offset=action_start_frame_offset,
action_fps=action_fps,
t_sound=s_sound,
num_vision_items=len(control_latent_list) + 1,
share_vision_temporal_positions=transfer_share_vision_temporal_positions,
)
cached_kv_full = self.language_model(text_ids, freqs_und)
self.cached_freqs_gen = freqs_gen
@@ -1516,10 +1571,12 @@ class Cosmos3VFMTransformer(nn.Module):
self._validate_gen_sequence_parallel(
s_gen=hidden_gen.shape[1],
s_video=s_video,
s_control=s_control,
s_action=s_action,
s_sound=s_sound,
has_action=has_action,
has_sound=has_sound,
has_control=has_control,
ulysses_size=ulysses_size,
)
freqs_cos, freqs_sin = self.cached_freqs_gen
@@ -1553,19 +1610,27 @@ class Cosmos3VFMTransformer(nn.Module):
# Final norm and project back to latent space
hidden_gen = self.norm_moe_gen(hidden_gen)
if not has_action and not has_sound:
if not has_action and not has_sound and not has_control:
return self.unpatchify(self.proj_out(hidden_gen), t, h, w)
split_sizes = [s_video]
split_sizes = []
if has_control:
split_sizes.append(s_control)
split_sizes.append(s_video)
if has_action:
split_sizes.append(s_action)
if has_sound:
split_sizes.append(s_sound)
split_hidden = hidden_gen.split(split_sizes, dim=1)
hidden_video = split_hidden[0]
split_idx = 0
if has_control:
split_idx += 1
hidden_video = split_hidden[split_idx]
split_idx += 1
video_pred = self.unpatchify(self.proj_out(hidden_video), t, h, w)
if has_control:
return video_pred
outputs: list[torch.Tensor] = [video_pred]
split_idx = 1
if has_action:
hidden_action = split_hidden[split_idx]
split_idx += 1