[Perf][Cosmos3] Add SeaCache support for Cosmos3 (#6922)
Signed-off-by: Yuliya Zhautouskaya <yzhautouskay@nvidia.com>
This commit is contained in:
1 parent
c17126c219
commit
6040bb97e2
18 files changed
+1966
-160
No files matched your search
@@ -1,5 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""Cosmos3 text/image/video/sound/action pipeline for vllm-omni.
|
||||
|
||||
One pipeline class serves the Cosmos3 family modes. Output modality is selected
|
||||
@@ -34,6 +34,7 @@ import math
|
||||
import os
|
||||
import time
|
||||
from collections.abc import Iterable, Mapping
|
||||
from contextlib import nullcontext
|
||||
from dataclasses import fields
|
||||
from typing import Any, ClassVar
|
||||
|
||||
@@ -1075,6 +1076,8 @@ class Cosmos3OmniDiffusersPipeline(
|
||||
|
||||
self._guidance_scale = None
|
||||
self._num_timesteps = None
|
||||
self._current_step_index = None
|
||||
self._current_sigma = None
|
||||
self._cosmos3_branch_caches: dict[str, tuple[Any, Any]] | None = None
|
||||
self._robolab_transforms: dict[bool, Any] = {}
|
||||
|
||||
@@ -1291,17 +1294,24 @@ class Cosmos3OmniDiffusersPipeline(
|
||||
or a tuple in video, action, sound order for multimodal generation.
|
||||
"""
|
||||
cache_key = kwargs.pop("_cosmos3_cache_key", None)
|
||||
if cache_key is None:
|
||||
return self.transformer(**kwargs)
|
||||
context_name = str(kwargs.pop("_cache_context", "cond"))
|
||||
|
||||
branch_caches = self._cosmos3_branch_caches
|
||||
if branch_caches is None:
|
||||
return self.transformer(**kwargs)
|
||||
|
||||
cache_key = str(cache_key)
|
||||
self.transformer.cached_kv, self.transformer.cached_freqs_gen = branch_caches.get(cache_key, (None, None))
|
||||
prediction = self.transformer(**kwargs)
|
||||
branch_caches[cache_key] = (self.transformer.cached_kv, self.transformer.cached_freqs_gen)
|
||||
context_factory = getattr(self, "_cache_context_factory", None)
|
||||
context = context_factory(context_name) if callable(context_factory) else nullcontext()
|
||||
with context:
|
||||
if cache_key is None:
|
||||
prediction = self.transformer(**kwargs)
|
||||
else:
|
||||
branch_caches = self._cosmos3_branch_caches
|
||||
if branch_caches is None:
|
||||
prediction = self.transformer(**kwargs)
|
||||
else:
|
||||
cache_key = str(cache_key)
|
||||
self.transformer.cached_kv, self.transformer.cached_freqs_gen = branch_caches.get(
|
||||
cache_key, (None, None)
|
||||
)
|
||||
prediction = self.transformer(**kwargs)
|
||||
branch_caches[cache_key] = (self.transformer.cached_kv, self.transformer.cached_freqs_gen)
|
||||
return prediction
|
||||
|
||||
def combine_multi_branch_cfg_noise(
|
||||
@@ -1840,6 +1850,29 @@ class Cosmos3OmniDiffusersPipeline(
|
||||
def num_timesteps(self):
|
||||
return self._num_timesteps
|
||||
|
||||
@property
|
||||
def current_step_index(self):
|
||||
return self._current_step_index
|
||||
|
||||
@property
|
||||
def current_sigma(self):
|
||||
return self._current_sigma
|
||||
|
||||
def _set_denoise_step_metadata(
|
||||
self,
|
||||
step_index: int,
|
||||
timesteps: torch.Tensor,
|
||||
scheduler: Any,
|
||||
) -> None:
|
||||
self._current_step_index = step_index
|
||||
self._num_timesteps = len(timesteps)
|
||||
sigmas = getattr(scheduler, "sigmas", None)
|
||||
self._current_sigma = sigmas[step_index] if sigmas is not None and step_index < len(sigmas) else None
|
||||
|
||||
def _clear_denoise_step_metadata(self) -> None:
|
||||
self._current_step_index = None
|
||||
self._current_sigma = None
|
||||
|
||||
def _set_mixed_precision_step(self, step_index: int, num_steps: int) -> None:
|
||||
setter = getattr(self.transformer, "set_mixed_precision_step", None)
|
||||
if setter is not None:
|
||||
@@ -2848,6 +2881,7 @@ class Cosmos3OmniDiffusersPipeline(
|
||||
# else uncond), so session keying loads/stores only this rank's branch.
|
||||
cfg_rank_is_negative = get_classifier_free_guidance_rank() != 0
|
||||
for step_index, t in enumerate(self.progress_bar(timesteps)):
|
||||
self._set_denoise_step_metadata(step_index, timesteps, step_scheduler)
|
||||
self._set_mixed_precision_step(step_index, len(timesteps))
|
||||
timestep = t.unsqueeze(0)
|
||||
# Outside the interval, scale=1 makes the combined output equal
|
||||
@@ -2859,6 +2893,7 @@ class Cosmos3OmniDiffusersPipeline(
|
||||
do_true_cfg=True,
|
||||
true_cfg_scale=step_scale,
|
||||
positive_kwargs=dict(
|
||||
_cache_context="cond",
|
||||
hidden_states=latents,
|
||||
timestep=timestep,
|
||||
text_ids=cond_ids,
|
||||
@@ -2868,6 +2903,7 @@ class Cosmos3OmniDiffusersPipeline(
|
||||
**shared_kwargs,
|
||||
),
|
||||
negative_kwargs=dict(
|
||||
_cache_context="uncond",
|
||||
hidden_states=latents,
|
||||
timestep=timestep,
|
||||
text_ids=uncond_ids,
|
||||
@@ -2888,13 +2924,15 @@ class Cosmos3OmniDiffusersPipeline(
|
||||
keep_uncond_for_cache = self._cache_requires_paired_cfg()
|
||||
|
||||
for step_index, t in enumerate(self.progress_bar(timesteps)):
|
||||
self._set_denoise_step_metadata(step_index, timesteps, step_scheduler)
|
||||
self._set_mixed_precision_step(step_index, len(timesteps))
|
||||
timestep = t.unsqueeze(0)
|
||||
cfg_active = _cfg_active_at(t)
|
||||
|
||||
if not self._kv_load_und(kv_state, is_negative=False):
|
||||
self.transformer.cached_kv, self.transformer.cached_freqs_gen = cond_cache
|
||||
noise_cond = self.transformer(
|
||||
noise_cond = self.predict_noise(
|
||||
_cache_context="cond",
|
||||
hidden_states=latents,
|
||||
timestep=timestep,
|
||||
text_ids=cond_ids,
|
||||
@@ -2911,7 +2949,8 @@ class Cosmos3OmniDiffusersPipeline(
|
||||
if cfg_active or keep_uncond_for_cache:
|
||||
if not self._kv_load_und(kv_state, is_negative=True):
|
||||
self.transformer.cached_kv, self.transformer.cached_freqs_gen = uncond_cache
|
||||
noise_uncond = self.transformer(
|
||||
noise_uncond = self.predict_noise(
|
||||
_cache_context="uncond",
|
||||
hidden_states=latents,
|
||||
timestep=timestep,
|
||||
text_ids=uncond_ids,
|
||||
@@ -2943,10 +2982,12 @@ class Cosmos3OmniDiffusersPipeline(
|
||||
# No CFG: a single cond branch per step. Bespoke (state None) keeps
|
||||
# using the transformer-instance cache exactly as before.
|
||||
for step_index, t in enumerate(self.progress_bar(timesteps)):
|
||||
self._set_denoise_step_metadata(step_index, timesteps, step_scheduler)
|
||||
self._set_mixed_precision_step(step_index, len(timesteps))
|
||||
timestep = t.unsqueeze(0)
|
||||
self._kv_load_und(kv_state, is_negative=False)
|
||||
noise_pred = self.transformer(
|
||||
noise_pred = self.predict_noise(
|
||||
_cache_context="cond",
|
||||
hidden_states=latents,
|
||||
timestep=timestep,
|
||||
text_ids=cond_ids,
|
||||
@@ -2959,6 +3000,7 @@ class Cosmos3OmniDiffusersPipeline(
|
||||
self._kv_capture_und(kv_state, is_negative=False)
|
||||
_assign_step_out(_step(noise_pred, t, latents, action_latents, sound_latents))
|
||||
finally:
|
||||
self._clear_denoise_step_metadata()
|
||||
self._reset_mixed_precision()
|
||||
# Cosmos3 currently receives a unique request_id rather than a
|
||||
# reusable rollout session id. Retaining its state would only pin
|
||||
@@ -3145,6 +3187,7 @@ class Cosmos3OmniDiffusersPipeline(
|
||||
self._cosmos3_branch_caches = {}
|
||||
try:
|
||||
for step_index, t in enumerate(self.progress_bar(timesteps)):
|
||||
self._set_denoise_step_metadata(step_index, timesteps, self.scheduler)
|
||||
self._set_mixed_precision_step(step_index, len(timesteps))
|
||||
timestep = t.unsqueeze(0)
|
||||
step_guidance = guidance_scale if _active_at(t, guidance_interval) else 1.0
|
||||
@@ -3153,6 +3196,7 @@ class Cosmos3OmniDiffusersPipeline(
|
||||
needs_control_cfg = step_control != 1.0
|
||||
|
||||
cond_full_kwargs = dict(
|
||||
_cache_context="cond",
|
||||
hidden_states=latents,
|
||||
timestep=timestep,
|
||||
text_ids=cond_ids,
|
||||
@@ -3166,6 +3210,7 @@ class Cosmos3OmniDiffusersPipeline(
|
||||
branches_kwargs = [
|
||||
cond_full_kwargs,
|
||||
dict(
|
||||
_cache_context="cond_no_control",
|
||||
hidden_states=latents,
|
||||
timestep=timestep,
|
||||
text_ids=cond_ids,
|
||||
@@ -3175,6 +3220,7 @@ class Cosmos3OmniDiffusersPipeline(
|
||||
**shared_kwargs,
|
||||
),
|
||||
dict(
|
||||
_cache_context="uncond",
|
||||
hidden_states=latents,
|
||||
timestep=timestep,
|
||||
text_ids=uncond_ids,
|
||||
@@ -3200,6 +3246,7 @@ class Cosmos3OmniDiffusersPipeline(
|
||||
branches_kwargs = [
|
||||
cond_full_kwargs,
|
||||
dict(
|
||||
_cache_context="cond_no_control",
|
||||
hidden_states=latents,
|
||||
timestep=timestep,
|
||||
text_ids=cond_ids,
|
||||
@@ -3223,6 +3270,7 @@ class Cosmos3OmniDiffusersPipeline(
|
||||
branches_kwargs = [
|
||||
cond_full_kwargs,
|
||||
dict(
|
||||
_cache_context="uncond",
|
||||
hidden_states=latents,
|
||||
timestep=timestep,
|
||||
text_ids=uncond_ids,
|
||||
@@ -3257,6 +3305,7 @@ class Cosmos3OmniDiffusersPipeline(
|
||||
)[0]
|
||||
latents = velocity_mask * latents + (1.0 - velocity_mask) * condition_latents
|
||||
finally:
|
||||
self._clear_denoise_step_metadata()
|
||||
self._reset_mixed_precision()
|
||||
self._cosmos3_branch_caches = None
|
||||
self.transformer.reset_cache()
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""Cosmos3 VFM Transformer for vllm-omni.
|
||||
|
||||
Implements the Mixture-of-Transformers architecture with two pathways:
|
||||
@@ -14,7 +14,7 @@ from __future__ import annotations
|
||||
import math
|
||||
from collections.abc import Iterator
|
||||
from contextlib import AbstractContextManager, contextmanager, nullcontext
|
||||
from typing import TYPE_CHECKING, Any
|
||||
from typing import TYPE_CHECKING, Any, NamedTuple
|
||||
|
||||
import torch
|
||||
import torch.distributed as dist
|
||||
@@ -427,7 +427,7 @@ class TimestepEmbedder(nn.Module):
|
||||
max_period: int = 10000,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
# Following diffusers naming pattern here for checkpoint compatibility.
|
||||
# Preserve checkpoint-compatible parameter names.
|
||||
self.linear_1 = nn.Linear(frequency_embedding_size, hidden_size, bias=True)
|
||||
self.act = nn.SiLU()
|
||||
self.linear_2 = nn.Linear(hidden_size, hidden_size, bias=True)
|
||||
@@ -1105,6 +1105,28 @@ class Cosmos3GenSPPrepare(nn.Module):
|
||||
return hidden_gen, freqs_cos, freqs_sin
|
||||
|
||||
|
||||
class _GenPrepared(NamedTuple):
|
||||
"""GEN-pathway state shared by normal and cached execution."""
|
||||
|
||||
hidden_gen: torch.Tensor
|
||||
time_embed: torch.Tensor
|
||||
t: int
|
||||
h: int
|
||||
w: int
|
||||
s_video: int
|
||||
s_control: int
|
||||
s_action: int
|
||||
s_sound: int
|
||||
has_control: bool
|
||||
has_action: bool
|
||||
has_sound: bool
|
||||
action_domain_ids: torch.Tensor | None
|
||||
ulysses_size: int
|
||||
use_multi_control_attention: bool
|
||||
multi_control_token_sizes: tuple[int, ...] | None
|
||||
multi_control_weights: tuple[float, ...] | None
|
||||
|
||||
|
||||
class Cosmos3VFMTransformer(nn.Module):
|
||||
"""Cosmos3 VFM Transformer: UND language model + GEN denoising layers.
|
||||
|
||||
@@ -1675,6 +1697,61 @@ class Cosmos3VFMTransformer(nn.Module):
|
||||
pad = (-(base + sound_frames)) % ulysses_size
|
||||
return sound_frames + pad
|
||||
|
||||
def _run_gen_layers(
|
||||
self,
|
||||
hidden_gen: torch.Tensor,
|
||||
*,
|
||||
s_video: int,
|
||||
s_control: int,
|
||||
s_action: int,
|
||||
s_sound: int,
|
||||
has_action: bool,
|
||||
has_sound: bool,
|
||||
has_control: bool,
|
||||
ulysses_size: int,
|
||||
use_multi_control_attention: bool,
|
||||
multi_control_token_sizes: tuple[int, ...] | None,
|
||||
multi_control_weights: tuple[float, ...] | None,
|
||||
) -> torch.Tensor:
|
||||
"""Run the complete GEN decoder between full-layout boundaries."""
|
||||
if self.cached_kv is None or self.cached_freqs_gen is None:
|
||||
raise RuntimeError("Cosmos3 GEN cache was not initialized before running GEN layers.")
|
||||
freqs_cos, freqs_sin = self.cached_freqs_gen
|
||||
if not use_multi_control_attention:
|
||||
hidden_gen, freqs_cos, freqs_sin = self.gen_sp_prepare(hidden_gen, freqs_cos, freqs_sin)
|
||||
freqs_gen = (freqs_cos, freqs_sin)
|
||||
|
||||
if len(self.gen_layers) == len(self.cached_kv):
|
||||
for layer, (k_und, v_und) in zip(self.gen_layers, self.cached_kv, strict=True):
|
||||
hidden_gen = layer(
|
||||
hidden_gen,
|
||||
k_und=k_und,
|
||||
v_und=v_und,
|
||||
freqs_cos=freqs_cos,
|
||||
freqs_sin=freqs_sin,
|
||||
control_token_sizes=multi_control_token_sizes,
|
||||
control_weights=multi_control_weights,
|
||||
)
|
||||
# Cache-dit's block wrapper may return a tuple; unwrap it.
|
||||
if isinstance(hidden_gen, tuple):
|
||||
hidden_gen = hidden_gen[0]
|
||||
else:
|
||||
# Cache-dit patches gen_layers to a grouped wrapper.
|
||||
for layer in self.gen_layers:
|
||||
hidden_gen = layer(
|
||||
hidden_gen,
|
||||
cached_kv=self.cached_kv,
|
||||
freqs_gen=freqs_gen,
|
||||
control_token_sizes=multi_control_token_sizes,
|
||||
control_weights=multi_control_weights,
|
||||
)
|
||||
if isinstance(hidden_gen, tuple):
|
||||
hidden_gen = hidden_gen[0]
|
||||
|
||||
if not use_multi_control_attention:
|
||||
hidden_gen = self.gen_sp_gather(hidden_gen)
|
||||
return hidden_gen
|
||||
|
||||
# -- Forward -------------------------------------------------------------
|
||||
|
||||
def forward(
|
||||
@@ -1697,7 +1774,51 @@ class Cosmos3VFMTransformer(nn.Module):
|
||||
transfer_share_vision_temporal_positions: bool = True,
|
||||
**kwargs,
|
||||
) -> torch.Tensor | tuple[torch.Tensor, ...]:
|
||||
"""Run the shared Cosmos3 GEN preprocess, stack, and postprocess path."""
|
||||
if kwargs:
|
||||
raise TypeError(f"Unexpected Cosmos3 transformer kwargs: {sorted(kwargs)}")
|
||||
prep = self._gen_preprocess(
|
||||
hidden_states,
|
||||
timestep,
|
||||
text_ids,
|
||||
text_mask,
|
||||
video_shape,
|
||||
fps=fps,
|
||||
action_latents=action_latents,
|
||||
action_domain_ids=action_domain_ids,
|
||||
action_noisy_mask=action_noisy_mask,
|
||||
action_start_frame_offset=action_start_frame_offset,
|
||||
action_fps=action_fps,
|
||||
sound_latents=sound_latents,
|
||||
noisy_frame_mask=noisy_frame_mask,
|
||||
control_latents=control_latents,
|
||||
control_weights=control_weights,
|
||||
transfer_share_vision_temporal_positions=transfer_share_vision_temporal_positions,
|
||||
)
|
||||
return self._gen_postprocess(self._run_gen_stack(prep), prep)
|
||||
|
||||
def _gen_preprocess(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
timestep: torch.Tensor,
|
||||
text_ids: torch.Tensor,
|
||||
text_mask: torch.Tensor,
|
||||
video_shape: tuple[int, int, int],
|
||||
fps: float | None = None,
|
||||
action_latents: torch.Tensor | None = None,
|
||||
action_domain_ids: torch.Tensor | None = None,
|
||||
action_noisy_mask: torch.Tensor | None = None,
|
||||
action_start_frame_offset: int = 1,
|
||||
action_fps: float | None = None,
|
||||
sound_latents: torch.Tensor | None = None,
|
||||
noisy_frame_mask: torch.Tensor | None = None,
|
||||
control_latents: list[torch.Tensor] | tuple[torch.Tensor, ...] | torch.Tensor | None = None,
|
||||
control_weights: list[float] | tuple[float, ...] | torch.Tensor | None = None,
|
||||
transfer_share_vision_temporal_positions: bool = True,
|
||||
) -> _GenPrepared:
|
||||
"""
|
||||
Prepare the packed GEN sequence before its cacheable execution region.
|
||||
|
||||
Args:
|
||||
hidden_states: [B, C, t, h, w] noisy latents
|
||||
timestep: [B] diffusion timestep
|
||||
@@ -1720,13 +1841,9 @@ class Cosmos3VFMTransformer(nn.Module):
|
||||
transfer controls. Values are normalized to sum to one.
|
||||
|
||||
Returns:
|
||||
[B, C, t, h, w] velocity prediction, or
|
||||
tuple outputs in video, action, sound order when action/sound streams
|
||||
are provided. Transfer-control streams condition the video prediction
|
||||
and are not returned.
|
||||
Packed GEN inputs and metadata consumed by the shared execution and
|
||||
postprocessing helpers.
|
||||
"""
|
||||
if kwargs:
|
||||
raise TypeError(f"Unexpected Cosmos3 transformer kwargs: {sorted(kwargs)}")
|
||||
t, h, w = video_shape
|
||||
hp, wp, _, _ = self._pad_to_patch_size(h, w)
|
||||
text_lengths = text_mask.sum(dim=1)
|
||||
@@ -1926,78 +2043,81 @@ class Cosmos3VFMTransformer(nn.Module):
|
||||
hidden_parts.append(hidden_sound)
|
||||
hidden_gen = torch.cat(hidden_parts, dim=1)
|
||||
|
||||
# Run GEN layers. UND K/V (replicated) is passed to each layer;
|
||||
# the Cosmos3CrossAttention forwards them as joint_key/value so the
|
||||
# framework Attention handles the Ulysses head-slicing internally.
|
||||
if self.cached_kv is None or self.cached_freqs_gen is None:
|
||||
raise RuntimeError("Cosmos3 GEN cache was not initialized before running GEN layers.")
|
||||
freqs_cos, freqs_sin = self.cached_freqs_gen
|
||||
if not use_multi_control_attention:
|
||||
hidden_gen, freqs_cos, freqs_sin = self.gen_sp_prepare(hidden_gen, freqs_cos, freqs_sin)
|
||||
freqs_gen = (freqs_cos, freqs_sin)
|
||||
return _GenPrepared(
|
||||
hidden_gen=hidden_gen,
|
||||
time_embed=time_embed,
|
||||
t=t,
|
||||
h=h,
|
||||
w=w,
|
||||
s_video=s_video,
|
||||
s_control=s_control,
|
||||
s_action=s_action,
|
||||
s_sound=s_sound,
|
||||
has_control=has_control,
|
||||
has_action=has_action,
|
||||
has_sound=has_sound,
|
||||
action_domain_ids=action_domain_ids,
|
||||
ulysses_size=ulysses_size,
|
||||
use_multi_control_attention=use_multi_control_attention,
|
||||
multi_control_token_sizes=multi_control_token_sizes,
|
||||
multi_control_weights=multi_control_weights,
|
||||
)
|
||||
|
||||
if len(self.gen_layers) == len(self.cached_kv):
|
||||
for layer, (k_und, v_und) in zip(self.gen_layers, self.cached_kv, strict=True):
|
||||
hidden_gen = layer(
|
||||
hidden_gen,
|
||||
k_und=k_und,
|
||||
v_und=v_und,
|
||||
freqs_cos=freqs_cos,
|
||||
freqs_sin=freqs_sin,
|
||||
control_token_sizes=multi_control_token_sizes,
|
||||
control_weights=multi_control_weights,
|
||||
)
|
||||
# Cache-dit's block wrapper may return a tuple; unwrap it.
|
||||
if isinstance(hidden_gen, tuple):
|
||||
hidden_gen = hidden_gen[0]
|
||||
else:
|
||||
# Cache-dit patches gen_layers to a grouped wrapper.
|
||||
for layer in self.gen_layers:
|
||||
hidden_gen = layer(
|
||||
hidden_gen,
|
||||
cached_kv=self.cached_kv,
|
||||
freqs_gen=freqs_gen,
|
||||
control_token_sizes=multi_control_token_sizes,
|
||||
control_weights=multi_control_weights,
|
||||
)
|
||||
if isinstance(hidden_gen, tuple):
|
||||
hidden_gen = hidden_gen[0]
|
||||
def _run_gen_stack(self, prep: _GenPrepared) -> torch.Tensor:
|
||||
"""Execute the cacheable full-layout GEN stack, including final norm."""
|
||||
hidden_gen = self._run_gen_layers(
|
||||
prep.hidden_gen,
|
||||
s_video=prep.s_video,
|
||||
s_control=prep.s_control,
|
||||
s_action=prep.s_action,
|
||||
s_sound=prep.s_sound,
|
||||
has_action=prep.has_action,
|
||||
has_sound=prep.has_sound,
|
||||
has_control=prep.has_control,
|
||||
ulysses_size=prep.ulysses_size,
|
||||
use_multi_control_attention=prep.use_multi_control_attention,
|
||||
multi_control_token_sizes=prep.multi_control_token_sizes,
|
||||
multi_control_weights=prep.multi_control_weights,
|
||||
)
|
||||
return self.norm_moe_gen(hidden_gen)
|
||||
|
||||
if not use_multi_control_attention:
|
||||
hidden_gen = self.gen_sp_gather(hidden_gen)
|
||||
def _gen_postprocess(
|
||||
self,
|
||||
hidden_gen: torch.Tensor,
|
||||
prep: _GenPrepared,
|
||||
) -> torch.Tensor | tuple[torch.Tensor, ...]:
|
||||
"""Project an already-normalized packed GEN state to model outputs."""
|
||||
if not prep.has_action and not prep.has_sound and not prep.has_control:
|
||||
return self.unpatchify(self.proj_out(hidden_gen), prep.t, prep.h, prep.w)
|
||||
|
||||
# Final norm and project back to latent space
|
||||
hidden_gen = self.norm_moe_gen(hidden_gen)
|
||||
if not has_action and not has_sound and not has_control:
|
||||
return self.unpatchify(self.proj_out(hidden_gen), t, h, w)
|
||||
|
||||
split_sizes = []
|
||||
if has_control:
|
||||
split_sizes.append(s_control)
|
||||
split_sizes.append(s_video)
|
||||
if has_action:
|
||||
split_sizes.append(s_action)
|
||||
if has_sound:
|
||||
split_sizes.append(s_sound)
|
||||
split_hidden = hidden_gen.split(split_sizes, dim=1)
|
||||
split_idx = 0
|
||||
if has_control:
|
||||
split_idx += 1
|
||||
hidden_video = split_hidden[split_idx]
|
||||
split_sizes = []
|
||||
if prep.has_control:
|
||||
split_sizes.append(prep.s_control)
|
||||
split_sizes.append(prep.s_video)
|
||||
if prep.has_action:
|
||||
split_sizes.append(prep.s_action)
|
||||
if prep.has_sound:
|
||||
split_sizes.append(prep.s_sound)
|
||||
split_hidden = hidden_gen.split(split_sizes, dim=1)
|
||||
split_idx = 0
|
||||
if prep.has_control:
|
||||
split_idx += 1
|
||||
video_pred = self.unpatchify(self.proj_out(hidden_video), t, h, w)
|
||||
if has_control:
|
||||
return video_pred
|
||||
outputs: list[torch.Tensor] = [video_pred]
|
||||
if has_action:
|
||||
hidden_action = split_hidden[split_idx]
|
||||
split_idx += 1
|
||||
assert action_domain_ids is not None
|
||||
outputs.append(self.unpack_action(self.action_proj_out(hidden_action, action_domain_ids)))
|
||||
if has_sound:
|
||||
hidden_sound = split_hidden[split_idx]
|
||||
outputs.append(self.unpack_sound(self.audio_proj_out(hidden_sound)))
|
||||
return tuple(outputs)
|
||||
hidden_video = split_hidden[split_idx]
|
||||
split_idx += 1
|
||||
video_pred = self.unpatchify(self.proj_out(hidden_video), prep.t, prep.h, prep.w)
|
||||
if prep.has_control:
|
||||
return video_pred
|
||||
|
||||
outputs: list[torch.Tensor] = [video_pred]
|
||||
if prep.has_action:
|
||||
hidden_action = split_hidden[split_idx]
|
||||
split_idx += 1
|
||||
assert prep.action_domain_ids is not None
|
||||
outputs.append(self.unpack_action(self.action_proj_out(hidden_action, prep.action_domain_ids)))
|
||||
if prep.has_sound:
|
||||
hidden_sound = split_hidden[split_idx]
|
||||
outputs.append(self.unpack_sound(self.audio_proj_out(hidden_sound)))
|
||||
return tuple(outputs)
|
||||
|
||||
def post_load_weights(self) -> None:
|
||||
"""Post-load processing: ensure correct dtypes."""
|
||||
|
||||
Reference in new issue
Block a user