[Perf][Cosmos3] Add SeaCache support for Cosmos3 (#6922)

Signed-off-by: Yuliya Zhautouskaya <yzhautouskay@nvidia.com>
This commit is contained in:
yzhautouskay authored and GitHub committed 2026-09-15 14:57:15 -07:00
1 parent c17126c219
commit 6040bb97e2
18 files changed
+1966 -160

No files matched your search

@@ -1,5 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Cosmos3 text/image/video/sound/action pipeline for vllm-omni.
One pipeline class serves the Cosmos3 family modes. Output modality is selected
@@ -34,6 +34,7 @@ import math
import os
import time
from collections.abc import Iterable, Mapping
from contextlib import nullcontext
from dataclasses import fields
from typing import Any, ClassVar
@@ -1075,6 +1076,8 @@ class Cosmos3OmniDiffusersPipeline(
self._guidance_scale = None
self._num_timesteps = None
self._current_step_index = None
self._current_sigma = None
self._cosmos3_branch_caches: dict[str, tuple[Any, Any]] | None = None
self._robolab_transforms: dict[bool, Any] = {}
@@ -1291,17 +1294,24 @@ class Cosmos3OmniDiffusersPipeline(
or a tuple in video, action, sound order for multimodal generation.
"""
cache_key = kwargs.pop("_cosmos3_cache_key", None)
if cache_key is None:
return self.transformer(**kwargs)
context_name = str(kwargs.pop("_cache_context", "cond"))
branch_caches = self._cosmos3_branch_caches
if branch_caches is None:
return self.transformer(**kwargs)
cache_key = str(cache_key)
self.transformer.cached_kv, self.transformer.cached_freqs_gen = branch_caches.get(cache_key, (None, None))
prediction = self.transformer(**kwargs)
branch_caches[cache_key] = (self.transformer.cached_kv, self.transformer.cached_freqs_gen)
context_factory = getattr(self, "_cache_context_factory", None)
context = context_factory(context_name) if callable(context_factory) else nullcontext()
with context:
if cache_key is None:
prediction = self.transformer(**kwargs)
else:
branch_caches = self._cosmos3_branch_caches
if branch_caches is None:
prediction = self.transformer(**kwargs)
else:
cache_key = str(cache_key)
self.transformer.cached_kv, self.transformer.cached_freqs_gen = branch_caches.get(
cache_key, (None, None)
)
prediction = self.transformer(**kwargs)
branch_caches[cache_key] = (self.transformer.cached_kv, self.transformer.cached_freqs_gen)
return prediction
def combine_multi_branch_cfg_noise(
@@ -1840,6 +1850,29 @@ class Cosmos3OmniDiffusersPipeline(
def num_timesteps(self):
return self._num_timesteps
@property
def current_step_index(self):
return self._current_step_index
@property
def current_sigma(self):
return self._current_sigma
def _set_denoise_step_metadata(
self,
step_index: int,
timesteps: torch.Tensor,
scheduler: Any,
) -> None:
self._current_step_index = step_index
self._num_timesteps = len(timesteps)
sigmas = getattr(scheduler, "sigmas", None)
self._current_sigma = sigmas[step_index] if sigmas is not None and step_index < len(sigmas) else None
def _clear_denoise_step_metadata(self) -> None:
self._current_step_index = None
self._current_sigma = None
def _set_mixed_precision_step(self, step_index: int, num_steps: int) -> None:
setter = getattr(self.transformer, "set_mixed_precision_step", None)
if setter is not None:
@@ -2848,6 +2881,7 @@ class Cosmos3OmniDiffusersPipeline(
# else uncond), so session keying loads/stores only this rank's branch.
cfg_rank_is_negative = get_classifier_free_guidance_rank() != 0
for step_index, t in enumerate(self.progress_bar(timesteps)):
self._set_denoise_step_metadata(step_index, timesteps, step_scheduler)
self._set_mixed_precision_step(step_index, len(timesteps))
timestep = t.unsqueeze(0)
# Outside the interval, scale=1 makes the combined output equal
@@ -2859,6 +2893,7 @@ class Cosmos3OmniDiffusersPipeline(
do_true_cfg=True,
true_cfg_scale=step_scale,
positive_kwargs=dict(
_cache_context="cond",
hidden_states=latents,
timestep=timestep,
text_ids=cond_ids,
@@ -2868,6 +2903,7 @@ class Cosmos3OmniDiffusersPipeline(
**shared_kwargs,
),
negative_kwargs=dict(
_cache_context="uncond",
hidden_states=latents,
timestep=timestep,
text_ids=uncond_ids,
@@ -2888,13 +2924,15 @@ class Cosmos3OmniDiffusersPipeline(
keep_uncond_for_cache = self._cache_requires_paired_cfg()
for step_index, t in enumerate(self.progress_bar(timesteps)):
self._set_denoise_step_metadata(step_index, timesteps, step_scheduler)
self._set_mixed_precision_step(step_index, len(timesteps))
timestep = t.unsqueeze(0)
cfg_active = _cfg_active_at(t)
if not self._kv_load_und(kv_state, is_negative=False):
self.transformer.cached_kv, self.transformer.cached_freqs_gen = cond_cache
noise_cond = self.transformer(
noise_cond = self.predict_noise(
_cache_context="cond",
hidden_states=latents,
timestep=timestep,
text_ids=cond_ids,
@@ -2911,7 +2949,8 @@ class Cosmos3OmniDiffusersPipeline(
if cfg_active or keep_uncond_for_cache:
if not self._kv_load_und(kv_state, is_negative=True):
self.transformer.cached_kv, self.transformer.cached_freqs_gen = uncond_cache
noise_uncond = self.transformer(
noise_uncond = self.predict_noise(
_cache_context="uncond",
hidden_states=latents,
timestep=timestep,
text_ids=uncond_ids,
@@ -2943,10 +2982,12 @@ class Cosmos3OmniDiffusersPipeline(
# No CFG: a single cond branch per step. Bespoke (state None) keeps
# using the transformer-instance cache exactly as before.
for step_index, t in enumerate(self.progress_bar(timesteps)):
self._set_denoise_step_metadata(step_index, timesteps, step_scheduler)
self._set_mixed_precision_step(step_index, len(timesteps))
timestep = t.unsqueeze(0)
self._kv_load_und(kv_state, is_negative=False)
noise_pred = self.transformer(
noise_pred = self.predict_noise(
_cache_context="cond",
hidden_states=latents,
timestep=timestep,
text_ids=cond_ids,
@@ -2959,6 +3000,7 @@ class Cosmos3OmniDiffusersPipeline(
self._kv_capture_und(kv_state, is_negative=False)
_assign_step_out(_step(noise_pred, t, latents, action_latents, sound_latents))
finally:
self._clear_denoise_step_metadata()
self._reset_mixed_precision()
# Cosmos3 currently receives a unique request_id rather than a
# reusable rollout session id. Retaining its state would only pin
@@ -3145,6 +3187,7 @@ class Cosmos3OmniDiffusersPipeline(
self._cosmos3_branch_caches = {}
try:
for step_index, t in enumerate(self.progress_bar(timesteps)):
self._set_denoise_step_metadata(step_index, timesteps, self.scheduler)
self._set_mixed_precision_step(step_index, len(timesteps))
timestep = t.unsqueeze(0)
step_guidance = guidance_scale if _active_at(t, guidance_interval) else 1.0
@@ -3153,6 +3196,7 @@ class Cosmos3OmniDiffusersPipeline(
needs_control_cfg = step_control != 1.0
cond_full_kwargs = dict(
_cache_context="cond",
hidden_states=latents,
timestep=timestep,
text_ids=cond_ids,
@@ -3166,6 +3210,7 @@ class Cosmos3OmniDiffusersPipeline(
branches_kwargs = [
cond_full_kwargs,
dict(
_cache_context="cond_no_control",
hidden_states=latents,
timestep=timestep,
text_ids=cond_ids,
@@ -3175,6 +3220,7 @@ class Cosmos3OmniDiffusersPipeline(
**shared_kwargs,
),
dict(
_cache_context="uncond",
hidden_states=latents,
timestep=timestep,
text_ids=uncond_ids,
@@ -3200,6 +3246,7 @@ class Cosmos3OmniDiffusersPipeline(
branches_kwargs = [
cond_full_kwargs,
dict(
_cache_context="cond_no_control",
hidden_states=latents,
timestep=timestep,
text_ids=cond_ids,
@@ -3223,6 +3270,7 @@ class Cosmos3OmniDiffusersPipeline(
branches_kwargs = [
cond_full_kwargs,
dict(
_cache_context="uncond",
hidden_states=latents,
timestep=timestep,
text_ids=uncond_ids,
@@ -3257,6 +3305,7 @@ class Cosmos3OmniDiffusersPipeline(
)[0]
latents = velocity_mask * latents + (1.0 - velocity_mask) * condition_latents
finally:
self._clear_denoise_step_metadata()
self._reset_mixed_precision()
self._cosmos3_branch_caches = None
self.transformer.reset_cache()
@@ -1,5 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Cosmos3 VFM Transformer for vllm-omni.
Implements the Mixture-of-Transformers architecture with two pathways:
@@ -14,7 +14,7 @@ from __future__ import annotations
import math
from collections.abc import Iterator
from contextlib import AbstractContextManager, contextmanager, nullcontext
from typing import TYPE_CHECKING, Any
from typing import TYPE_CHECKING, Any, NamedTuple
import torch
import torch.distributed as dist
@@ -427,7 +427,7 @@ class TimestepEmbedder(nn.Module):
max_period: int = 10000,
) -> None:
super().__init__()
# Following diffusers naming pattern here for checkpoint compatibility.
# Preserve checkpoint-compatible parameter names.
self.linear_1 = nn.Linear(frequency_embedding_size, hidden_size, bias=True)
self.act = nn.SiLU()
self.linear_2 = nn.Linear(hidden_size, hidden_size, bias=True)
@@ -1105,6 +1105,28 @@ class Cosmos3GenSPPrepare(nn.Module):
return hidden_gen, freqs_cos, freqs_sin
class _GenPrepared(NamedTuple):
"""GEN-pathway state shared by normal and cached execution."""
hidden_gen: torch.Tensor
time_embed: torch.Tensor
t: int
h: int
w: int
s_video: int
s_control: int
s_action: int
s_sound: int
has_control: bool
has_action: bool
has_sound: bool
action_domain_ids: torch.Tensor | None
ulysses_size: int
use_multi_control_attention: bool
multi_control_token_sizes: tuple[int, ...] | None
multi_control_weights: tuple[float, ...] | None
class Cosmos3VFMTransformer(nn.Module):
"""Cosmos3 VFM Transformer: UND language model + GEN denoising layers.
@@ -1675,6 +1697,61 @@ class Cosmos3VFMTransformer(nn.Module):
pad = (-(base + sound_frames)) % ulysses_size
return sound_frames + pad
def _run_gen_layers(
self,
hidden_gen: torch.Tensor,
*,
s_video: int,
s_control: int,
s_action: int,
s_sound: int,
has_action: bool,
has_sound: bool,
has_control: bool,
ulysses_size: int,
use_multi_control_attention: bool,
multi_control_token_sizes: tuple[int, ...] | None,
multi_control_weights: tuple[float, ...] | None,
) -> torch.Tensor:
"""Run the complete GEN decoder between full-layout boundaries."""
if self.cached_kv is None or self.cached_freqs_gen is None:
raise RuntimeError("Cosmos3 GEN cache was not initialized before running GEN layers.")
freqs_cos, freqs_sin = self.cached_freqs_gen
if not use_multi_control_attention:
hidden_gen, freqs_cos, freqs_sin = self.gen_sp_prepare(hidden_gen, freqs_cos, freqs_sin)
freqs_gen = (freqs_cos, freqs_sin)
if len(self.gen_layers) == len(self.cached_kv):
for layer, (k_und, v_und) in zip(self.gen_layers, self.cached_kv, strict=True):
hidden_gen = layer(
hidden_gen,
k_und=k_und,
v_und=v_und,
freqs_cos=freqs_cos,
freqs_sin=freqs_sin,
control_token_sizes=multi_control_token_sizes,
control_weights=multi_control_weights,
)
# Cache-dit's block wrapper may return a tuple; unwrap it.
if isinstance(hidden_gen, tuple):
hidden_gen = hidden_gen[0]
else:
# Cache-dit patches gen_layers to a grouped wrapper.
for layer in self.gen_layers:
hidden_gen = layer(
hidden_gen,
cached_kv=self.cached_kv,
freqs_gen=freqs_gen,
control_token_sizes=multi_control_token_sizes,
control_weights=multi_control_weights,
)
if isinstance(hidden_gen, tuple):
hidden_gen = hidden_gen[0]
if not use_multi_control_attention:
hidden_gen = self.gen_sp_gather(hidden_gen)
return hidden_gen
# -- Forward -------------------------------------------------------------
def forward(
@@ -1697,7 +1774,51 @@ class Cosmos3VFMTransformer(nn.Module):
transfer_share_vision_temporal_positions: bool = True,
**kwargs,
) -> torch.Tensor | tuple[torch.Tensor, ...]:
"""Run the shared Cosmos3 GEN preprocess, stack, and postprocess path."""
if kwargs:
raise TypeError(f"Unexpected Cosmos3 transformer kwargs: {sorted(kwargs)}")
prep = self._gen_preprocess(
hidden_states,
timestep,
text_ids,
text_mask,
video_shape,
fps=fps,
action_latents=action_latents,
action_domain_ids=action_domain_ids,
action_noisy_mask=action_noisy_mask,
action_start_frame_offset=action_start_frame_offset,
action_fps=action_fps,
sound_latents=sound_latents,
noisy_frame_mask=noisy_frame_mask,
control_latents=control_latents,
control_weights=control_weights,
transfer_share_vision_temporal_positions=transfer_share_vision_temporal_positions,
)
return self._gen_postprocess(self._run_gen_stack(prep), prep)
def _gen_preprocess(
self,
hidden_states: torch.Tensor,
timestep: torch.Tensor,
text_ids: torch.Tensor,
text_mask: torch.Tensor,
video_shape: tuple[int, int, int],
fps: float | None = None,
action_latents: torch.Tensor | None = None,
action_domain_ids: torch.Tensor | None = None,
action_noisy_mask: torch.Tensor | None = None,
action_start_frame_offset: int = 1,
action_fps: float | None = None,
sound_latents: torch.Tensor | None = None,
noisy_frame_mask: torch.Tensor | None = None,
control_latents: list[torch.Tensor] | tuple[torch.Tensor, ...] | torch.Tensor | None = None,
control_weights: list[float] | tuple[float, ...] | torch.Tensor | None = None,
transfer_share_vision_temporal_positions: bool = True,
) -> _GenPrepared:
"""
Prepare the packed GEN sequence before its cacheable execution region.
Args:
hidden_states: [B, C, t, h, w] noisy latents
timestep: [B] diffusion timestep
@@ -1720,13 +1841,9 @@ class Cosmos3VFMTransformer(nn.Module):
transfer controls. Values are normalized to sum to one.
Returns:
[B, C, t, h, w] velocity prediction, or
tuple outputs in video, action, sound order when action/sound streams
are provided. Transfer-control streams condition the video prediction
and are not returned.
Packed GEN inputs and metadata consumed by the shared execution and
postprocessing helpers.
"""
if kwargs:
raise TypeError(f"Unexpected Cosmos3 transformer kwargs: {sorted(kwargs)}")
t, h, w = video_shape
hp, wp, _, _ = self._pad_to_patch_size(h, w)
text_lengths = text_mask.sum(dim=1)
@@ -1926,78 +2043,81 @@ class Cosmos3VFMTransformer(nn.Module):
hidden_parts.append(hidden_sound)
hidden_gen = torch.cat(hidden_parts, dim=1)
# Run GEN layers. UND K/V (replicated) is passed to each layer;
# the Cosmos3CrossAttention forwards them as joint_key/value so the
# framework Attention handles the Ulysses head-slicing internally.
if self.cached_kv is None or self.cached_freqs_gen is None:
raise RuntimeError("Cosmos3 GEN cache was not initialized before running GEN layers.")
freqs_cos, freqs_sin = self.cached_freqs_gen
if not use_multi_control_attention:
hidden_gen, freqs_cos, freqs_sin = self.gen_sp_prepare(hidden_gen, freqs_cos, freqs_sin)
freqs_gen = (freqs_cos, freqs_sin)
return _GenPrepared(
hidden_gen=hidden_gen,
time_embed=time_embed,
t=t,
h=h,
w=w,
s_video=s_video,
s_control=s_control,
s_action=s_action,
s_sound=s_sound,
has_control=has_control,
has_action=has_action,
has_sound=has_sound,
action_domain_ids=action_domain_ids,
ulysses_size=ulysses_size,
use_multi_control_attention=use_multi_control_attention,
multi_control_token_sizes=multi_control_token_sizes,
multi_control_weights=multi_control_weights,
)
if len(self.gen_layers) == len(self.cached_kv):
for layer, (k_und, v_und) in zip(self.gen_layers, self.cached_kv, strict=True):
hidden_gen = layer(
hidden_gen,
k_und=k_und,
v_und=v_und,
freqs_cos=freqs_cos,
freqs_sin=freqs_sin,
control_token_sizes=multi_control_token_sizes,
control_weights=multi_control_weights,
)
# Cache-dit's block wrapper may return a tuple; unwrap it.
if isinstance(hidden_gen, tuple):
hidden_gen = hidden_gen[0]
else:
# Cache-dit patches gen_layers to a grouped wrapper.
for layer in self.gen_layers:
hidden_gen = layer(
hidden_gen,
cached_kv=self.cached_kv,
freqs_gen=freqs_gen,
control_token_sizes=multi_control_token_sizes,
control_weights=multi_control_weights,
)
if isinstance(hidden_gen, tuple):
hidden_gen = hidden_gen[0]
def _run_gen_stack(self, prep: _GenPrepared) -> torch.Tensor:
"""Execute the cacheable full-layout GEN stack, including final norm."""
hidden_gen = self._run_gen_layers(
prep.hidden_gen,
s_video=prep.s_video,
s_control=prep.s_control,
s_action=prep.s_action,
s_sound=prep.s_sound,
has_action=prep.has_action,
has_sound=prep.has_sound,
has_control=prep.has_control,
ulysses_size=prep.ulysses_size,
use_multi_control_attention=prep.use_multi_control_attention,
multi_control_token_sizes=prep.multi_control_token_sizes,
multi_control_weights=prep.multi_control_weights,
)
return self.norm_moe_gen(hidden_gen)
if not use_multi_control_attention:
hidden_gen = self.gen_sp_gather(hidden_gen)
def _gen_postprocess(
self,
hidden_gen: torch.Tensor,
prep: _GenPrepared,
) -> torch.Tensor | tuple[torch.Tensor, ...]:
"""Project an already-normalized packed GEN state to model outputs."""
if not prep.has_action and not prep.has_sound and not prep.has_control:
return self.unpatchify(self.proj_out(hidden_gen), prep.t, prep.h, prep.w)
# Final norm and project back to latent space
hidden_gen = self.norm_moe_gen(hidden_gen)
if not has_action and not has_sound and not has_control:
return self.unpatchify(self.proj_out(hidden_gen), t, h, w)
split_sizes = []
if has_control:
split_sizes.append(s_control)
split_sizes.append(s_video)
if has_action:
split_sizes.append(s_action)
if has_sound:
split_sizes.append(s_sound)
split_hidden = hidden_gen.split(split_sizes, dim=1)
split_idx = 0
if has_control:
split_idx += 1
hidden_video = split_hidden[split_idx]
split_sizes = []
if prep.has_control:
split_sizes.append(prep.s_control)
split_sizes.append(prep.s_video)
if prep.has_action:
split_sizes.append(prep.s_action)
if prep.has_sound:
split_sizes.append(prep.s_sound)
split_hidden = hidden_gen.split(split_sizes, dim=1)
split_idx = 0
if prep.has_control:
split_idx += 1
video_pred = self.unpatchify(self.proj_out(hidden_video), t, h, w)
if has_control:
return video_pred
outputs: list[torch.Tensor] = [video_pred]
if has_action:
hidden_action = split_hidden[split_idx]
split_idx += 1
assert action_domain_ids is not None
outputs.append(self.unpack_action(self.action_proj_out(hidden_action, action_domain_ids)))
if has_sound:
hidden_sound = split_hidden[split_idx]
outputs.append(self.unpack_sound(self.audio_proj_out(hidden_sound)))
return tuple(outputs)
hidden_video = split_hidden[split_idx]
split_idx += 1
video_pred = self.unpatchify(self.proj_out(hidden_video), prep.t, prep.h, prep.w)
if prep.has_control:
return video_pred
outputs: list[torch.Tensor] = [video_pred]
if prep.has_action:
hidden_action = split_hidden[split_idx]
split_idx += 1
assert prep.action_domain_ids is not None
outputs.append(self.unpack_action(self.action_proj_out(hidden_action, prep.action_domain_ids)))
if prep.has_sound:
hidden_sound = split_hidden[split_idx]
outputs.append(self.unpack_sound(self.audio_proj_out(hidden_sound)))
return tuple(outputs)
def post_load_weights(self) -> None:
"""Post-load processing: ensure correct dtypes."""