Merge branch 'main' into feat/diffusion-prefetch-protection
This commit is contained in:
74 files changed
+9185
-350
No files matched your search
@@ -0,0 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from .avae import Cosmos3AVAEAudioTokenizer
|
||||
|
||||
__all__ = ["Cosmos3AVAEAudioTokenizer"]
|
||||
@@ -0,0 +1,323 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Diffusers-format AVAE audio tokenizer used by Cosmos3 sound generation."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import math
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
from torch.nn.utils import weight_norm
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from vllm_omni.diffusion.models.progress_bar import _is_rank_zero
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def _default_avae_config(
|
||||
*,
|
||||
sample_rate: int,
|
||||
audio_channels: int,
|
||||
io_channels: int,
|
||||
hop_size: int,
|
||||
) -> dict[str, Any]:
|
||||
return {
|
||||
"sampling_rate": sample_rate,
|
||||
"hop_size": hop_size,
|
||||
"dec_dim": 320,
|
||||
"dec_c_mults": [1, 2, 4, 8, 16],
|
||||
"dec_strides": [2, 4, 5, 6, 8],
|
||||
"dec_out_channels": audio_channels,
|
||||
"vocoder_input_dim": io_channels,
|
||||
"normalization_type": "none",
|
||||
"normalize_latents": False,
|
||||
"tanh_input_scale": 1.5,
|
||||
"tanh_output_scale": 3.5,
|
||||
"tanh_clamp": 0.995,
|
||||
}
|
||||
|
||||
|
||||
def _config_get(config: dict[str, Any], *keys: str, default: Any = None) -> Any:
|
||||
for key in keys:
|
||||
value = config.get(key)
|
||||
if value is not None:
|
||||
return value
|
||||
return default
|
||||
|
||||
|
||||
def _load_config(
|
||||
config_path: str | Path | None,
|
||||
*,
|
||||
sample_rate: int,
|
||||
audio_channels: int,
|
||||
io_channels: int,
|
||||
hop_size: int,
|
||||
) -> dict[str, Any]:
|
||||
if config_path:
|
||||
with open(config_path, encoding="utf-8") as f:
|
||||
config = json.load(f)
|
||||
if not isinstance(config, dict):
|
||||
raise TypeError(f"Cosmos3 AVAE config must be a JSON object, got {type(config)!r}.")
|
||||
return config
|
||||
return _default_avae_config(
|
||||
sample_rate=sample_rate,
|
||||
audio_channels=audio_channels,
|
||||
io_channels=io_channels,
|
||||
hop_size=hop_size,
|
||||
)
|
||||
|
||||
|
||||
def _load_checkpoint(path: str | Path, map_location: torch.device | str) -> dict[str, torch.Tensor]:
|
||||
path = Path(path)
|
||||
if path.suffix == ".safetensors":
|
||||
try:
|
||||
from safetensors.torch import load_file
|
||||
except ImportError as exc:
|
||||
raise ImportError("Loading AVAE .safetensors checkpoints requires safetensors.") from exc
|
||||
checkpoint = load_file(str(path), device=str(map_location))
|
||||
else:
|
||||
checkpoint = torch.load(path, map_location=map_location)
|
||||
|
||||
if not isinstance(checkpoint, dict):
|
||||
raise TypeError(f"AVAE checkpoint must be a flat state dict, got {type(checkpoint)!r}.")
|
||||
if not all(isinstance(value, torch.Tensor) for value in checkpoint.values()):
|
||||
raise TypeError("AVAE checkpoint must be a flat tensor state dict.")
|
||||
return checkpoint
|
||||
|
||||
|
||||
def _validate_diffusers_state_dict(state_dict: dict[str, torch.Tensor]) -> None:
|
||||
if not state_dict:
|
||||
raise RuntimeError("AVAE checkpoint is empty.")
|
||||
|
||||
if not any(key.startswith("decoder.") for key in state_dict):
|
||||
raise RuntimeError("Cosmos3 AVAE checkpoint must contain diffusers-format decoder.* keys.")
|
||||
|
||||
|
||||
class Snake1d(nn.Module):
|
||||
"""One-dimensional Snake activation matching diffusers' Oobleck layout."""
|
||||
|
||||
def __init__(self, hidden_dim: int, logscale: bool = True) -> None:
|
||||
super().__init__()
|
||||
self.alpha = nn.Parameter(torch.zeros(1, hidden_dim, 1))
|
||||
self.beta = nn.Parameter(torch.zeros(1, hidden_dim, 1))
|
||||
self.logscale = logscale
|
||||
|
||||
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
shape = hidden_states.shape
|
||||
alpha = torch.exp(self.alpha) if self.logscale else self.alpha
|
||||
beta = torch.exp(self.beta) if self.logscale else self.beta
|
||||
hidden_states = hidden_states.reshape(shape[0], shape[1], -1)
|
||||
hidden_states = hidden_states + (beta + 1e-9).reciprocal() * torch.sin(alpha * hidden_states).pow(2)
|
||||
return hidden_states.reshape(shape)
|
||||
|
||||
|
||||
class OobleckResidualUnit(nn.Module):
|
||||
"""Residual unit used by the diffusers Oobleck decoder."""
|
||||
|
||||
def __init__(self, dimension: int = 16, dilation: int = 1) -> None:
|
||||
super().__init__()
|
||||
pad = ((7 - 1) * dilation) // 2
|
||||
self.snake1 = Snake1d(dimension)
|
||||
self.conv1 = weight_norm(nn.Conv1d(dimension, dimension, kernel_size=7, dilation=dilation, padding=pad))
|
||||
self.snake2 = Snake1d(dimension)
|
||||
self.conv2 = weight_norm(nn.Conv1d(dimension, dimension, kernel_size=1))
|
||||
|
||||
def forward(self, hidden_state: torch.Tensor) -> torch.Tensor:
|
||||
output_tensor = self.conv1(self.snake1(hidden_state))
|
||||
output_tensor = self.conv2(self.snake2(output_tensor))
|
||||
padding = (hidden_state.shape[-1] - output_tensor.shape[-1]) // 2
|
||||
if padding > 0:
|
||||
hidden_state = hidden_state[..., padding:-padding]
|
||||
return hidden_state + output_tensor
|
||||
|
||||
|
||||
class OobleckDecoderBlock(nn.Module):
|
||||
"""Decoder block used by the diffusers Oobleck decoder."""
|
||||
|
||||
def __init__(self, input_dim: int, output_dim: int, stride: int = 1, output_padding: int = 0) -> None:
|
||||
super().__init__()
|
||||
self.snake1 = Snake1d(input_dim)
|
||||
self.conv_t1 = weight_norm(
|
||||
nn.ConvTranspose1d(
|
||||
input_dim,
|
||||
output_dim,
|
||||
kernel_size=2 * stride,
|
||||
stride=stride,
|
||||
padding=math.ceil(stride / 2),
|
||||
output_padding=output_padding,
|
||||
)
|
||||
)
|
||||
self.res_unit1 = OobleckResidualUnit(output_dim, dilation=1)
|
||||
self.res_unit2 = OobleckResidualUnit(output_dim, dilation=3)
|
||||
self.res_unit3 = OobleckResidualUnit(output_dim, dilation=9)
|
||||
|
||||
def forward(self, hidden_state: torch.Tensor) -> torch.Tensor:
|
||||
hidden_state = self.snake1(hidden_state)
|
||||
hidden_state = self.conv_t1(hidden_state)
|
||||
hidden_state = self.res_unit1(hidden_state)
|
||||
hidden_state = self.res_unit2(hidden_state)
|
||||
return self.res_unit3(hidden_state)
|
||||
|
||||
|
||||
class OobleckDecoder(nn.Module):
|
||||
"""Diffusers-compatible Oobleck decoder for Cosmos3 AVAE latents."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
channels: int,
|
||||
input_channels: int,
|
||||
audio_channels: int,
|
||||
upsampling_ratios: list[int],
|
||||
channel_multiples: list[int],
|
||||
) -> None:
|
||||
super().__init__()
|
||||
strides = upsampling_ratios
|
||||
channel_multiples = [1] + channel_multiples
|
||||
|
||||
self.conv1 = weight_norm(nn.Conv1d(input_channels, channels * channel_multiples[-1], kernel_size=7, padding=3))
|
||||
|
||||
block = []
|
||||
for stride_index, stride in enumerate(strides):
|
||||
block.append(
|
||||
OobleckDecoderBlock(
|
||||
input_dim=channels * channel_multiples[len(strides) - stride_index],
|
||||
output_dim=channels * channel_multiples[len(strides) - stride_index - 1],
|
||||
stride=stride,
|
||||
output_padding=stride % 2,
|
||||
)
|
||||
)
|
||||
self.block = nn.ModuleList(block)
|
||||
self.snake1 = Snake1d(channels)
|
||||
self.conv2 = weight_norm(nn.Conv1d(channels, audio_channels, kernel_size=7, padding=3, bias=False))
|
||||
|
||||
def forward(self, hidden_state: torch.Tensor) -> torch.Tensor:
|
||||
hidden_state = self.conv1(hidden_state)
|
||||
for layer in self.block:
|
||||
hidden_state = layer(hidden_state)
|
||||
hidden_state = self.snake1(hidden_state)
|
||||
return self.conv2(hidden_state)
|
||||
|
||||
|
||||
class Cosmos3AVAEAudioTokenizer(nn.Module):
|
||||
"""Decoder-only AVAE tokenizer for Cosmos3 audio latents."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
checkpoint_path: str | Path,
|
||||
config_path: str | Path | None = None,
|
||||
sample_rate: int = 48000,
|
||||
audio_channels: int = 2,
|
||||
io_channels: int = 64,
|
||||
hop_size: int = 1920,
|
||||
normalize_latents: bool = False,
|
||||
normalization_type: str = "none",
|
||||
tanh_input_scale: float = 1.5,
|
||||
tanh_output_scale: float = 3.5,
|
||||
tanh_clamp: float = 0.995,
|
||||
dtype: torch.dtype = torch.bfloat16,
|
||||
device: torch.device | str = "cuda",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.dtype = dtype
|
||||
self.device = torch.device(device)
|
||||
|
||||
config = _load_config(
|
||||
config_path,
|
||||
sample_rate=sample_rate,
|
||||
audio_channels=audio_channels,
|
||||
io_channels=io_channels,
|
||||
hop_size=hop_size,
|
||||
)
|
||||
self.sample_rate = int(_config_get(config, "sampling_rate", "sample_rate", default=sample_rate))
|
||||
self.audio_channels = int(
|
||||
_config_get(
|
||||
config,
|
||||
"dec_out_channels",
|
||||
"audio_channels",
|
||||
default=2 if bool(config.get("stereo", audio_channels == 2)) else 1,
|
||||
)
|
||||
)
|
||||
self.latent_ch = int(_config_get(config, "vocoder_input_dim", "io_channels", "latent_ch", default=io_channels))
|
||||
dec_strides = [int(stride) for stride in _config_get(config, "dec_strides", default=[2, 4, 5, 6, 8])]
|
||||
self.hop_size = int(
|
||||
_config_get(config, "hop_size", default=math.prod(dec_strides) if dec_strides else hop_size)
|
||||
)
|
||||
dec_stride_product = math.prod(dec_strides)
|
||||
if dec_stride_product != self.hop_size:
|
||||
raise ValueError(
|
||||
"Cosmos3 AVAE config dec_strides product must equal hop_size "
|
||||
f"for correct latent/audio duration math: product={dec_stride_product}, hop_size={self.hop_size}."
|
||||
)
|
||||
|
||||
normalization_type = str(_config_get(config, "normalization_type", default=normalization_type))
|
||||
normalize_latents = bool(_config_get(config, "normalize_latents", default=normalize_latents))
|
||||
if normalization_type == "none" and normalize_latents:
|
||||
normalization_type = "tanh"
|
||||
self.normalization_type = normalization_type
|
||||
self.tanh_input_scale = float(_config_get(config, "tanh_input_scale", default=tanh_input_scale))
|
||||
self.tanh_output_scale = float(_config_get(config, "tanh_output_scale", default=tanh_output_scale))
|
||||
self.tanh_clamp = float(_config_get(config, "tanh_clamp", default=tanh_clamp))
|
||||
|
||||
self.decoder = OobleckDecoder(
|
||||
channels=int(_config_get(config, "dec_dim", default=320)),
|
||||
input_channels=self.latent_ch,
|
||||
audio_channels=self.audio_channels,
|
||||
upsampling_ratios=list(reversed(dec_strides)),
|
||||
channel_multiples=list(_config_get(config, "dec_c_mults", default=[1, 2, 4, 8, 16])),
|
||||
)
|
||||
state_dict = _load_checkpoint(checkpoint_path, self.device)
|
||||
_validate_diffusers_state_dict(state_dict)
|
||||
|
||||
# The checkpoint also contains encoder weights, which we do not support here, hence strict=False
|
||||
self.load_state_dict(state_dict, strict=False)
|
||||
|
||||
self.eval()
|
||||
for param in self.parameters():
|
||||
param.requires_grad = False
|
||||
self.to(device=self.device, dtype=self.dtype)
|
||||
if _is_rank_zero():
|
||||
logger.info("Loaded diffusers-format Cosmos3 AVAE checkpoint from %s", checkpoint_path)
|
||||
|
||||
@property
|
||||
def temporal_compression_factor(self) -> int:
|
||||
return self.hop_size
|
||||
|
||||
def get_latent_num_samples(self, num_audio_samples: int) -> int:
|
||||
return int(num_audio_samples) // self.temporal_compression_factor
|
||||
|
||||
def get_audio_num_samples(self, num_latent_samples: int) -> int:
|
||||
return int(num_latent_samples) * self.temporal_compression_factor
|
||||
|
||||
def _denormalize_latent(self, latent: torch.Tensor) -> torch.Tensor:
|
||||
if self.normalization_type == "tanh":
|
||||
in_dtype = latent.dtype
|
||||
latent = torch.clamp(
|
||||
latent.float() / self.tanh_output_scale,
|
||||
-self.tanh_clamp,
|
||||
self.tanh_clamp,
|
||||
)
|
||||
return (torch.atanh(latent) * self.tanh_input_scale).to(in_dtype)
|
||||
if self.normalization_type != "none":
|
||||
raise ValueError(f"Unsupported AVAE normalization_type={self.normalization_type!r}.")
|
||||
return latent
|
||||
|
||||
@torch.no_grad()
|
||||
def encode(self, audio: torch.Tensor, force_pad: bool = False) -> torch.Tensor:
|
||||
del audio, force_pad
|
||||
raise NotImplementedError("Cosmos3AVAEAudioTokenizer is decoder-only for diffusers-format sound_tokenizer/.")
|
||||
|
||||
@torch.no_grad()
|
||||
def decode(self, latent: torch.Tensor) -> torch.Tensor:
|
||||
in_dtype = latent.dtype
|
||||
squeeze = latent.ndim == 2
|
||||
if squeeze:
|
||||
latent = latent.unsqueeze(0)
|
||||
z = self._denormalize_latent(latent.to(self.device)).to(self.dtype)
|
||||
audio = self.decoder(z).clamp(-1.0, 1.0).to(in_dtype)
|
||||
return audio.squeeze(0) if squeeze else audio
|
||||
@@ -15,6 +15,7 @@ runtime by:
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
import os
|
||||
import time
|
||||
from collections.abc import Iterable
|
||||
@@ -45,7 +46,7 @@ from vllm_omni.diffusion.profiler.diffusion_pipeline_profiler import DiffusionPi
|
||||
from vllm_omni.diffusion.request import OmniDiffusionRequest
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
|
||||
from .transformer_cosmos3 import Cosmos3VFMTransformer
|
||||
from .transformer_cosmos3 import Cosmos3VFMTransformer, resolve_sound_gen
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -160,6 +161,28 @@ def get_cosmos3_post_process_func(od_config: OmniDiffusionConfig):
|
||||
|
||||
video_processor = VideoProcessor(vae_scale_factor=16)
|
||||
|
||||
def _sampling_param(sampling_params, key: str, default=None):
|
||||
extra = getattr(sampling_params, "extra_args", None)
|
||||
if isinstance(extra, dict) and extra.get(key) is not None:
|
||||
return extra[key]
|
||||
value = getattr(sampling_params, key, None)
|
||||
return default if value is None else value
|
||||
|
||||
def _resolve_output_fps(sampling_params):
|
||||
fps = (
|
||||
_sampling_param(sampling_params, "resolved_frame_rate")
|
||||
or _sampling_param(sampling_params, "frame_rate")
|
||||
or _sampling_param(sampling_params, "fps")
|
||||
or 24.0
|
||||
)
|
||||
try:
|
||||
fps_value = float(fps)
|
||||
except (TypeError, ValueError):
|
||||
fps_value = 24.0
|
||||
if fps_value <= 0:
|
||||
fps_value = 24.0
|
||||
return int(fps_value) if fps_value.is_integer() else fps_value
|
||||
|
||||
def post_process_func(
|
||||
output: torch.Tensor | dict[str, torch.Tensor] | tuple,
|
||||
output_type: str = "np",
|
||||
@@ -168,6 +191,8 @@ def get_cosmos3_post_process_func(od_config: OmniDiffusionConfig):
|
||||
if output_type == "latent":
|
||||
return output
|
||||
|
||||
audio = None
|
||||
audio_sample_rate = None
|
||||
if isinstance(output, dict):
|
||||
if "image" in output and "video" in output:
|
||||
raise ValueError("Cosmos3 output cannot contain both image and video payloads.")
|
||||
@@ -177,10 +202,23 @@ def get_cosmos3_post_process_func(od_config: OmniDiffusionConfig):
|
||||
video = output["video"]
|
||||
else:
|
||||
raise ValueError("Cosmos3 postprocess expected an 'image' or 'video' output payload.")
|
||||
audio = output.get("audio")
|
||||
audio_sample_rate = output.get("audio_sample_rate")
|
||||
elif isinstance(output, tuple):
|
||||
if len(output) == 3:
|
||||
video, audio, audio_sample_rate = output
|
||||
elif len(output) == 2:
|
||||
video, audio = output
|
||||
else:
|
||||
raise ValueError(
|
||||
"Cosmos3 postprocess expects output tensor, output dict, or (video, audio[, sample_rate]) tuple."
|
||||
)
|
||||
else:
|
||||
video = output
|
||||
|
||||
if isinstance(output, dict) and "image" in output:
|
||||
if audio is not None:
|
||||
raise ValueError("Cosmos3 text-to-image postprocess does not support audio output.")
|
||||
if video.ndim != 5 or video.shape[2] != 1:
|
||||
raise ValueError(
|
||||
"Cosmos3 text-to-image postprocess expects decoded output "
|
||||
@@ -194,7 +232,19 @@ def get_cosmos3_post_process_func(od_config: OmniDiffusionConfig):
|
||||
return video_processor.postprocess(image, output_type="pil")
|
||||
if is_guardrails_enabled(od_config, sampling_params):
|
||||
video = check_video_safety(video)
|
||||
return video_processor.postprocess_video(video, output_type=output_type)
|
||||
processed_video = video_processor.postprocess_video(video, output_type=output_type)
|
||||
if audio is None:
|
||||
return processed_video
|
||||
if isinstance(audio, torch.Tensor):
|
||||
audio = audio.detach().cpu()
|
||||
result = {
|
||||
"video": processed_video,
|
||||
"audio": audio,
|
||||
"fps": _resolve_output_fps(sampling_params),
|
||||
}
|
||||
if audio_sample_rate is not None:
|
||||
result["audio_sample_rate"] = int(audio_sample_rate)
|
||||
return result
|
||||
|
||||
return post_process_func
|
||||
|
||||
@@ -274,10 +324,22 @@ class Cosmos3OmniDiffusersPipeline(
|
||||
self.vae_scale_factor_temporal = int(self.vae.config.scale_factor_temporal)
|
||||
self.vae_scale_factor_spatial = getattr(self.vae.config, "scale_factor_spatial", 16)
|
||||
|
||||
sound_gen = resolve_sound_gen(od_config)
|
||||
sound_dim = None
|
||||
sound_latent_fps = None
|
||||
self._sound_tokenizer = None
|
||||
if sound_gen:
|
||||
self._sound_tokenizer = self._get_sound_tokenizer()
|
||||
sound_dim = self._sound_tokenizer.latent_ch
|
||||
sound_latent_fps = self._sound_tokenizer.latent_fps
|
||||
|
||||
# --- Transformer (weights loaded later via weights_sources) ---
|
||||
self.transformer = Cosmos3VFMTransformer(
|
||||
od_config=od_config,
|
||||
temporal_compression_factor=self.vae_scale_factor_temporal,
|
||||
sound_gen=sound_gen,
|
||||
sound_dim=sound_dim,
|
||||
sound_latent_fps=sound_latent_fps,
|
||||
)
|
||||
|
||||
# --- Scheduler ---
|
||||
@@ -357,9 +419,13 @@ class Cosmos3OmniDiffusersPipeline(
|
||||
"proj_in.",
|
||||
"proj_out.",
|
||||
"time_embedder.",
|
||||
"audio_proj_in.",
|
||||
"audio_proj_out.",
|
||||
)
|
||||
):
|
||||
return f"transformer.{k}"
|
||||
if k in ("audio_modality_embed", "audio_modality_embed.weight"):
|
||||
return "transformer.audio_modality_embed"
|
||||
|
||||
# Skip lm_head
|
||||
if k.startswith("lm_head."):
|
||||
@@ -453,12 +519,22 @@ class Cosmos3OmniDiffusersPipeline(
|
||||
loaded = loader.load_weights(_remapped_weights())
|
||||
self.transformer.post_load_weights()
|
||||
self.transformer.eval()
|
||||
if getattr(self.transformer, "sound_gen", False):
|
||||
sound_markers = ("audio_proj_in.", "audio_proj_out.", "audio_modality_embed")
|
||||
missing = [marker.rstrip(".") for marker in sound_markers if not any(marker in name for name in loaded)]
|
||||
if missing:
|
||||
raise ValueError(
|
||||
"Cosmos3 transformer config enables sound generation, but "
|
||||
f"the checkpoint is missing sound weights for {missing}. "
|
||||
"Use a sound-capable transformer checkpoint."
|
||||
)
|
||||
return loaded
|
||||
|
||||
def predict_noise(self, **kwargs) -> torch.Tensor | tuple[torch.Tensor, ...]:
|
||||
"""Override CFGParallelMixin.predict_noise for Cosmos3.
|
||||
|
||||
The transformer returns the raw video noise prediction.
|
||||
The transformer returns the raw prediction: video-only as a tensor,
|
||||
or a tuple in video, sound order for sound generation.
|
||||
"""
|
||||
return self.transformer(**kwargs)
|
||||
|
||||
@@ -509,6 +585,39 @@ class Cosmos3OmniDiffusersPipeline(
|
||||
return val
|
||||
return default
|
||||
|
||||
@staticmethod
|
||||
def _truthy(value) -> bool:
|
||||
if isinstance(value, str):
|
||||
return value.strip().lower() in {"1", "true", "yes", "on"}
|
||||
return bool(value)
|
||||
|
||||
@classmethod
|
||||
def _get_prompt_param(cls, prompt_data, key: str, default=None):
|
||||
if not isinstance(prompt_data, dict):
|
||||
return default
|
||||
if prompt_data.get(key) is not None:
|
||||
return prompt_data[key]
|
||||
additional = prompt_data.get("additional_information")
|
||||
if isinstance(additional, dict) and additional.get(key) is not None:
|
||||
return additional[key]
|
||||
return default
|
||||
|
||||
@classmethod
|
||||
def _is_sound_request(cls, prompt_data, sp) -> bool:
|
||||
for key in ("generate_sound", "sound_gen"):
|
||||
if cls._truthy(cls._get_prompt_param(prompt_data, key, None)):
|
||||
return True
|
||||
if cls._truthy(cls._get_sp_param(sp, key, None)):
|
||||
return True
|
||||
return False
|
||||
|
||||
def _get_sound_tokenizer(self):
|
||||
if self._sound_tokenizer is None:
|
||||
from .sound_tokenizer import Cosmos3SoundTokenizer
|
||||
|
||||
self._sound_tokenizer = Cosmos3SoundTokenizer.from_config(self.od_config)
|
||||
return self._sound_tokenizer
|
||||
|
||||
@staticmethod
|
||||
def _is_t2i_request(req: OmniDiffusionRequest) -> bool:
|
||||
"""Detect text-to-image mode from request-level prompt modalities."""
|
||||
@@ -721,6 +830,47 @@ class Cosmos3OmniDiffusersPipeline(
|
||||
)
|
||||
return randn_tensor(shape, generator=generator, device=self.device, dtype=self.dtype)
|
||||
|
||||
def _prepare_sound_latents(
|
||||
self,
|
||||
target_audio_samples: int,
|
||||
generator: torch.Generator,
|
||||
) -> tuple[torch.Tensor, int]:
|
||||
sound_tokenizer = self._get_sound_tokenizer()
|
||||
hop_size = int(
|
||||
getattr(sound_tokenizer, "hop_size", None) or getattr(sound_tokenizer, "temporal_compression_factor")
|
||||
)
|
||||
latent_frames = max(1, math.ceil(max(1, int(target_audio_samples)) / hop_size))
|
||||
sound_dim = int(getattr(sound_tokenizer, "latent_ch", 64))
|
||||
transformer_sound_dim = int(getattr(self.transformer, "sound_dim", sound_dim))
|
||||
if sound_dim != transformer_sound_dim:
|
||||
raise ValueError(
|
||||
"Cosmos3 sound tokenizer latent channels do not match transformer "
|
||||
f"sound_dim: tokenizer={sound_dim}, transformer={transformer_sound_dim}."
|
||||
)
|
||||
latents = randn_tensor(
|
||||
(1, sound_dim, latent_frames),
|
||||
generator=generator,
|
||||
device=self.device,
|
||||
dtype=self.dtype,
|
||||
)
|
||||
return latents, latent_frames
|
||||
|
||||
def _resolve_sound_target_samples(
|
||||
self,
|
||||
sp,
|
||||
num_frames: int,
|
||||
frame_rate: float,
|
||||
) -> tuple[int, float, int]:
|
||||
sound_tokenizer = self._get_sound_tokenizer()
|
||||
duration = self._get_sp_param(sp, "sound_duration", None)
|
||||
if duration is None:
|
||||
duration = self._get_sp_param(sp, "audio_duration", None)
|
||||
if duration is None:
|
||||
duration = num_frames / frame_rate
|
||||
duration = max(float(duration), 1.0 / max(float(frame_rate), 1.0))
|
||||
sample_rate = int(getattr(sound_tokenizer, "sample_rate", 48000))
|
||||
return max(1, int(round(duration * sample_rate))), duration, sample_rate
|
||||
|
||||
# -- VAE decode ----------------------------------------------------------
|
||||
|
||||
def _decode_latents(self, latents: torch.Tensor) -> torch.Tensor:
|
||||
@@ -742,6 +892,19 @@ class Cosmos3OmniDiffusersPipeline(
|
||||
video = self.vae.decode(latents, return_dict=False)[0]
|
||||
return video
|
||||
|
||||
def _decode_sound_latents(
|
||||
self,
|
||||
sound_latents: torch.Tensor,
|
||||
target_audio_samples: int,
|
||||
) -> torch.Tensor:
|
||||
sound_tokenizer = self._get_sound_tokenizer()
|
||||
audio = sound_tokenizer.decode(sound_latents.to(self.dtype))
|
||||
if audio.shape[-1] > target_audio_samples:
|
||||
audio = audio[..., :target_audio_samples]
|
||||
elif audio.shape[-1] < target_audio_samples:
|
||||
audio = torch.nn.functional.pad(audio, (0, target_audio_samples - audio.shape[-1]))
|
||||
return audio.detach().cpu()
|
||||
|
||||
# -- Prompt formatting + tokenization (shared by T2V and I2V) ------------
|
||||
|
||||
def _format_and_tokenize_prompts(
|
||||
@@ -903,11 +1066,12 @@ class Cosmos3OmniDiffusersPipeline(
|
||||
guidance_scale: float,
|
||||
shared_kwargs: dict,
|
||||
*,
|
||||
sound_latents: torch.Tensor | None = None,
|
||||
velocity_mask: torch.Tensor | None = None,
|
||||
image_latent: torch.Tensor | None = None,
|
||||
condition_latents: torch.Tensor | None = None,
|
||||
guidance_interval: tuple[float, float] | None = None,
|
||||
) -> torch.Tensor:
|
||||
) -> torch.Tensor | tuple[torch.Tensor, ...]:
|
||||
"""Denoising loop with 3-mode CFG support (parallel, sequential, none).
|
||||
|
||||
Cosmos3's UND pathway is text-dependent, so CFG needs separate K/V
|
||||
@@ -946,21 +1110,85 @@ class Cosmos3OmniDiffusersPipeline(
|
||||
lo, hi = guidance_interval
|
||||
return lo <= t_scalar <= hi
|
||||
|
||||
# Joint scheduler step over multiple modalities. Safe for flow-matching schedulers
|
||||
# because the update is linear per element; revisit this if Cosmos3 adopts a
|
||||
# scheduler with cross-element dependencies (e.g. per-modality timestep).
|
||||
def _pack_joint(
|
||||
video_tensor: torch.Tensor,
|
||||
sound_tensor: torch.Tensor | None = None,
|
||||
):
|
||||
batch = video_tensor.shape[0]
|
||||
tensors = [video_tensor]
|
||||
if sound_tensor is not None:
|
||||
tensors.append(sound_tensor)
|
||||
flats = [tensor.reshape(batch, -1) for tensor in tensors]
|
||||
return torch.cat(flats, dim=1), [tensor.shape for tensor in tensors], [flat.shape[1] for flat in flats]
|
||||
|
||||
def _unpack_joint(
|
||||
packed: torch.Tensor,
|
||||
shapes: list[torch.Size],
|
||||
numels: list[int],
|
||||
) -> tuple[torch.Tensor, ...]:
|
||||
outputs = []
|
||||
offset = 0
|
||||
for shape, numel in zip(shapes, numels, strict=True):
|
||||
outputs.append(packed[:, offset : offset + numel].reshape(shape))
|
||||
offset += numel
|
||||
return tuple(outputs)
|
||||
|
||||
def _split_noise_pred(
|
||||
noise_pred: torch.Tensor | tuple[torch.Tensor, ...],
|
||||
) -> tuple[torch.Tensor, torch.Tensor | None]:
|
||||
has_sound = sound_latents is not None
|
||||
if not has_sound:
|
||||
if isinstance(noise_pred, tuple):
|
||||
raise ValueError("Cosmos3 video-only diffusion received tuple predictions.")
|
||||
return noise_pred, None
|
||||
if not isinstance(noise_pred, tuple):
|
||||
raise ValueError("Cosmos3 multimodal diffusion expects transformer predictions as a tuple.")
|
||||
if len(noise_pred) != 2:
|
||||
raise ValueError(f"Cosmos3 sound diffusion expected 2 predictions, got {len(noise_pred)}.")
|
||||
return noise_pred[0], noise_pred[1]
|
||||
|
||||
def _step(
|
||||
noise_pred: torch.Tensor,
|
||||
noise_pred: torch.Tensor | tuple[torch.Tensor, ...],
|
||||
t: torch.Tensor,
|
||||
latents: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
if isinstance(noise_pred, tuple):
|
||||
raise ValueError("Cosmos3 noise prediction must be a single tensor; got a tuple.")
|
||||
sound_latents: torch.Tensor | None,
|
||||
) -> torch.Tensor | tuple[torch.Tensor, ...]:
|
||||
video_pred, sound_pred = _split_noise_pred(noise_pred)
|
||||
if velocity_mask is not None:
|
||||
noise_pred = noise_pred * velocity_mask
|
||||
latents = self.scheduler.step(noise_pred, t, latents, return_dict=False)[0]
|
||||
video_pred = video_pred * velocity_mask
|
||||
if sound_latents is None:
|
||||
latents = self.scheduler.step(video_pred, t, latents, return_dict=False)[0]
|
||||
else:
|
||||
packed_noise, shapes, numels = _pack_joint(video_pred, sound_pred)
|
||||
packed_latents, _, _ = _pack_joint(latents, sound_latents)
|
||||
packed_next = self.scheduler.step(packed_noise, t, packed_latents, return_dict=False)[0]
|
||||
unpacked = _unpack_joint(packed_next, shapes, numels)
|
||||
latents = unpacked[0]
|
||||
if sound_latents is not None:
|
||||
sound_latents = unpacked[1]
|
||||
if condition_latents is not None and velocity_mask is not None:
|
||||
latents = velocity_mask * latents + (1.0 - velocity_mask) * condition_latents
|
||||
elif image_latent is not None:
|
||||
latents[:, :, 0:1, :, :] = image_latent
|
||||
return latents
|
||||
outputs = [latents]
|
||||
if sound_latents is not None:
|
||||
outputs.append(sound_latents)
|
||||
return outputs[0] if len(outputs) == 1 else tuple(outputs)
|
||||
|
||||
def _assign_step_out(step_out: torch.Tensor | tuple[torch.Tensor, ...]) -> None:
|
||||
nonlocal latents, sound_latents
|
||||
if sound_latents is None:
|
||||
assert isinstance(step_out, torch.Tensor)
|
||||
latents = step_out
|
||||
return
|
||||
if not isinstance(step_out, tuple):
|
||||
raise ValueError("Cosmos3 multimodal diffusion step returned a non-tuple result.")
|
||||
latents = step_out[0]
|
||||
if sound_latents is not None:
|
||||
sound_latents = step_out[1]
|
||||
|
||||
if cfg_parallel:
|
||||
for t in self.progress_bar(timesteps):
|
||||
@@ -978,6 +1206,7 @@ class Cosmos3OmniDiffusersPipeline(
|
||||
timestep=timestep,
|
||||
text_ids=cond_ids,
|
||||
text_mask=cond_mask,
|
||||
sound_latents=sound_latents,
|
||||
**shared_kwargs,
|
||||
),
|
||||
negative_kwargs=dict(
|
||||
@@ -985,11 +1214,12 @@ class Cosmos3OmniDiffusersPipeline(
|
||||
timestep=timestep,
|
||||
text_ids=uncond_ids,
|
||||
text_mask=uncond_mask,
|
||||
sound_latents=sound_latents,
|
||||
**shared_kwargs,
|
||||
),
|
||||
cfg_normalize=False,
|
||||
)
|
||||
latents = _step(noise_pred, t, latents)
|
||||
_assign_step_out(_step(noise_pred, t, latents, sound_latents))
|
||||
|
||||
elif do_cfg:
|
||||
cond_cache: tuple = (None, None)
|
||||
@@ -1007,6 +1237,7 @@ class Cosmos3OmniDiffusersPipeline(
|
||||
timestep=timestep,
|
||||
text_ids=cond_ids,
|
||||
text_mask=cond_mask,
|
||||
sound_latents=sound_latents,
|
||||
**shared_kwargs,
|
||||
)
|
||||
if cond_cache[0] is None:
|
||||
@@ -1019,6 +1250,7 @@ class Cosmos3OmniDiffusersPipeline(
|
||||
timestep=timestep,
|
||||
text_ids=uncond_ids,
|
||||
text_mask=uncond_mask,
|
||||
sound_latents=sound_latents,
|
||||
**shared_kwargs,
|
||||
)
|
||||
if uncond_cache[0] is None:
|
||||
@@ -1031,7 +1263,7 @@ class Cosmos3OmniDiffusersPipeline(
|
||||
else:
|
||||
noise_pred = noise_cond
|
||||
|
||||
latents = _step(noise_pred, t, latents)
|
||||
_assign_step_out(_step(noise_pred, t, latents, sound_latents))
|
||||
|
||||
else:
|
||||
for t in self.progress_bar(timesteps):
|
||||
@@ -1041,11 +1273,15 @@ class Cosmos3OmniDiffusersPipeline(
|
||||
timestep=timestep,
|
||||
text_ids=cond_ids,
|
||||
text_mask=cond_mask,
|
||||
sound_latents=sound_latents,
|
||||
**shared_kwargs,
|
||||
)
|
||||
latents = _step(noise_pred, t, latents)
|
||||
_assign_step_out(_step(noise_pred, t, latents, sound_latents))
|
||||
|
||||
return latents
|
||||
outputs = [latents]
|
||||
if sound_latents is not None:
|
||||
outputs.append(sound_latents)
|
||||
return outputs[0] if len(outputs) == 1 else tuple(outputs)
|
||||
|
||||
# -- Forward (main generation entry point) -------------------------------
|
||||
|
||||
@@ -1072,6 +1308,18 @@ class Cosmos3OmniDiffusersPipeline(
|
||||
|
||||
sp = req.sampling_params
|
||||
is_t2i = self._is_t2i_request(req)
|
||||
sound_enabled = self._is_sound_request(prompt_data, sp)
|
||||
if sound_enabled and is_t2i:
|
||||
raise ValueError(
|
||||
"Cosmos3 sound generation is supported only for video outputs in "
|
||||
"this phase; text-to-image with sound is unsupported."
|
||||
)
|
||||
if sound_enabled and not getattr(self.transformer, "sound_gen", False):
|
||||
raise ValueError(
|
||||
"Cosmos3 sound generation was requested, but the transformer was "
|
||||
"initialized without sound modules. Check that the checkpoint config "
|
||||
"enables sound_gen or defines sound_dim and includes sound weights."
|
||||
)
|
||||
if negative_prompt is None:
|
||||
negative_prompt = ""
|
||||
|
||||
@@ -1163,6 +1411,13 @@ class Cosmos3OmniDiffusersPipeline(
|
||||
image_latent = None
|
||||
condition_latents = None
|
||||
|
||||
sound_latents = None
|
||||
target_audio_samples = None
|
||||
sound_sample_rate = None
|
||||
if sound_enabled:
|
||||
target_audio_samples, _, sound_sample_rate = self._resolve_sound_target_samples(sp, num_frames, frame_rate)
|
||||
sound_latents, _ = self._prepare_sound_latents(target_audio_samples, generator)
|
||||
|
||||
T_latent = latents.shape[2]
|
||||
H_latent = latents.shape[3]
|
||||
W_latent = latents.shape[4]
|
||||
@@ -1184,6 +1439,7 @@ class Cosmos3OmniDiffusersPipeline(
|
||||
uncond_mask=uncond_mask,
|
||||
guidance_scale=guidance_scale,
|
||||
shared_kwargs=shared_kwargs,
|
||||
sound_latents=sound_latents,
|
||||
velocity_mask=velocity_mask,
|
||||
image_latent=image_latent,
|
||||
condition_latents=condition_latents,
|
||||
@@ -1204,7 +1460,11 @@ class Cosmos3OmniDiffusersPipeline(
|
||||
samples.append(_run_diffusion(next_latents))
|
||||
latents = torch.cat(samples, dim=0)
|
||||
else:
|
||||
latents = _run_diffusion(latents)
|
||||
diffusion_output = _run_diffusion(latents)
|
||||
if sound_enabled:
|
||||
latents, sound_latents = diffusion_output
|
||||
else:
|
||||
latents = diffusion_output
|
||||
|
||||
# --- Decode ---
|
||||
if _is_rank_zero():
|
||||
@@ -1215,4 +1475,12 @@ class Cosmos3OmniDiffusersPipeline(
|
||||
logger.info("Video decoded in %.2fs", time.time() - decode_start)
|
||||
logger.info("Total pipeline time: %.2fs", time.time() - pipeline_start)
|
||||
|
||||
if sound_enabled:
|
||||
if sound_latents is None or target_audio_samples is None or sound_sample_rate is None:
|
||||
raise ValueError("Cosmos3 sound generation finished without sound latents.")
|
||||
if _is_rank_zero():
|
||||
logger.info("Decoding sound...")
|
||||
audio = self._decode_sound_latents(sound_latents, target_audio_samples)
|
||||
return DiffusionOutput(output={"video": video, "audio": audio, "audio_sample_rate": sound_sample_rate})
|
||||
|
||||
return DiffusionOutput(output={"image": video} if is_t2i else {"video": video})
|
||||
@@ -0,0 +1,504 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Cosmos3 sound tokenizer integration."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from vllm_omni.diffusion.data import OmniDiffusionConfig
|
||||
from vllm_omni.diffusion.distributed.utils import get_local_device
|
||||
from vllm_omni.diffusion.models.progress_bar import _is_rank_zero
|
||||
|
||||
from .audio_tokenizer import Cosmos3AVAEAudioTokenizer
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
DEFAULT_SOUND_SAMPLE_RATE = 48000
|
||||
DEFAULT_SOUND_CHANNELS = 2
|
||||
DEFAULT_SOUND_DIM = 64
|
||||
DEFAULT_SOUND_HOP_SIZE = 1920
|
||||
DEFAULT_SOUND_LATENT_FPS = DEFAULT_SOUND_SAMPLE_RATE / DEFAULT_SOUND_HOP_SIZE
|
||||
DEFAULT_SOUND_NORMALIZE_LATENTS = False
|
||||
DEFAULT_SOUND_NORMALIZATION_TYPE = "none"
|
||||
DEFAULT_SOUND_TANH_INPUT_SCALE = 1.5
|
||||
DEFAULT_SOUND_TANH_OUTPUT_SCALE = 3.5
|
||||
DEFAULT_SOUND_TANH_CLAMP = 0.995
|
||||
SOUND_TOKENIZER_COMPONENT_NAME = "sound_tokenizer"
|
||||
SOUND_TOKENIZER_CHECKPOINT_NAME = "diffusion_pytorch_model.safetensors"
|
||||
|
||||
|
||||
def _pipeline_args(od_config: OmniDiffusionConfig) -> dict[str, Any]:
|
||||
return dict(getattr(od_config, "custom_pipeline_args", None) or {})
|
||||
|
||||
|
||||
def _config_get(config: Any, key: str, default: Any = None) -> Any:
|
||||
if config is None:
|
||||
return default
|
||||
if isinstance(config, dict):
|
||||
return config.get(key, default)
|
||||
if hasattr(config, "get"):
|
||||
value = config.get(key, None)
|
||||
return default if value is None else value
|
||||
return getattr(config, key, default)
|
||||
|
||||
|
||||
def _config_path_get(config: Any, *keys: str) -> Any:
|
||||
value = config
|
||||
for key in keys:
|
||||
value = _config_get(value, key, None)
|
||||
if value is None:
|
||||
return None
|
||||
return value
|
||||
|
||||
|
||||
def _sound_tokenizer_config_from(config: Any) -> Any:
|
||||
"""Return nested ``sound_tokenizer`` config from Cosmos3 config shapes."""
|
||||
for path in (
|
||||
("sound_tokenizer",),
|
||||
("model", "config", "sound_tokenizer"),
|
||||
("config", "sound_tokenizer"),
|
||||
("model_config", "sound_tokenizer"),
|
||||
):
|
||||
value = _config_path_get(config, *path)
|
||||
if value is not None:
|
||||
return value
|
||||
return None
|
||||
|
||||
|
||||
def _nested_sound_tokenizer_configs(od_config: OmniDiffusionConfig | None) -> tuple[Any, ...]:
|
||||
if od_config is None:
|
||||
return ()
|
||||
configs = []
|
||||
for source in (
|
||||
getattr(od_config, "model_config", None),
|
||||
getattr(od_config, "tf_model_config", None),
|
||||
):
|
||||
config = _sound_tokenizer_config_from(source)
|
||||
if config is not None:
|
||||
configs.append(config)
|
||||
return tuple(configs)
|
||||
|
||||
|
||||
def _first_value_from_configs(configs: tuple[Any, ...], keys: tuple[str, ...]) -> Any:
|
||||
for config in configs:
|
||||
for key in keys:
|
||||
value = _config_get(config, key, None)
|
||||
if value is not None:
|
||||
return value
|
||||
return None
|
||||
|
||||
|
||||
def _top_level_model_value(od_config: OmniDiffusionConfig | None, keys: tuple[str, ...]) -> Any:
|
||||
if od_config is None:
|
||||
return None
|
||||
for source in (
|
||||
getattr(od_config, "model_config", None),
|
||||
getattr(od_config, "tf_model_config", None),
|
||||
):
|
||||
for key in keys:
|
||||
for path in ((key,), ("model", "config", key), ("config", key), ("model_config", key)):
|
||||
value = _config_path_get(source, *path)
|
||||
if value is not None:
|
||||
return value
|
||||
return None
|
||||
|
||||
|
||||
def _custom_arg_value(args: dict[str, Any], keys: tuple[str, ...]) -> Any:
|
||||
for key in keys:
|
||||
value = args.get(key)
|
||||
if value is not None:
|
||||
return value
|
||||
return None
|
||||
|
||||
|
||||
def _as_bool(value: Any) -> bool:
|
||||
if isinstance(value, str):
|
||||
return value.strip().lower() in {"1", "true", "yes", "on"}
|
||||
return bool(value)
|
||||
|
||||
|
||||
def _as_audio_channels(value: Any) -> int:
|
||||
if isinstance(value, bool):
|
||||
return 2 if value else 1
|
||||
if isinstance(value, str) and value.strip().lower() in {
|
||||
"1",
|
||||
"0",
|
||||
"true",
|
||||
"false",
|
||||
"yes",
|
||||
"no",
|
||||
"on",
|
||||
"off",
|
||||
}:
|
||||
return 2 if _as_bool(value) else 1
|
||||
return int(value)
|
||||
|
||||
|
||||
def _resolve_model_file(path: Any, model_root: str | None) -> str | None:
|
||||
if not path:
|
||||
return None
|
||||
path = str(path)
|
||||
if "://" in path or os.path.isabs(path) or os.path.exists(path) or not model_root:
|
||||
return path
|
||||
return str(Path(model_root) / path)
|
||||
|
||||
|
||||
def _load_sound_tokenizer_component_config(config_path: str | None) -> dict[str, Any]:
|
||||
if not config_path:
|
||||
return {}
|
||||
with open(config_path, encoding="utf-8") as f:
|
||||
config = json.load(f)
|
||||
if not isinstance(config, dict):
|
||||
raise TypeError(f"Cosmos3 sound tokenizer config must be a JSON object, got {type(config)!r}.")
|
||||
return config
|
||||
|
||||
|
||||
def _component_audio_channels(config: dict[str, Any]) -> Any:
|
||||
if config.get("dec_out_channels") is not None:
|
||||
return config["dec_out_channels"]
|
||||
if config.get("audio_channels") is not None:
|
||||
return config["audio_channels"]
|
||||
if config.get("stereo") is not None:
|
||||
return 2 if _as_bool(config["stereo"]) else 1
|
||||
return None
|
||||
|
||||
|
||||
def _component_arch_values(config: dict[str, Any]) -> dict[str, Any]:
|
||||
values = {
|
||||
"sample_rate": config.get("sampling_rate", config.get("sample_rate")),
|
||||
"audio_channels": _component_audio_channels(config),
|
||||
"io_channels": config.get("vocoder_input_dim", config.get("io_channels", config.get("latent_ch"))),
|
||||
"hop_size": config.get("hop_size"),
|
||||
}
|
||||
return {key: value for key, value in values.items() if value is not None}
|
||||
|
||||
|
||||
def _resolve_arch_value(
|
||||
od_config: OmniDiffusionConfig,
|
||||
args: dict[str, Any],
|
||||
component_values: dict[str, Any],
|
||||
*,
|
||||
field: str,
|
||||
custom_keys: tuple[str, ...],
|
||||
nested_keys: tuple[str, ...],
|
||||
top_level_keys: tuple[str, ...],
|
||||
default: Any,
|
||||
cast,
|
||||
) -> Any:
|
||||
custom_value = _custom_arg_value(args, custom_keys)
|
||||
component_value = component_values.get(field)
|
||||
if component_value is not None:
|
||||
resolved = cast(component_value)
|
||||
if custom_value is not None and cast(custom_value) != resolved:
|
||||
raise ValueError(
|
||||
"Conflicting Cosmos3 sound tokenizer architecture override for "
|
||||
f"{field}: component config has {resolved!r}, custom args have {cast(custom_value)!r}."
|
||||
)
|
||||
return resolved
|
||||
|
||||
if custom_value is not None:
|
||||
return cast(custom_value)
|
||||
|
||||
nested_value = _first_value_from_configs(_nested_sound_tokenizer_configs(od_config), nested_keys)
|
||||
if nested_value is not None:
|
||||
return cast(nested_value)
|
||||
|
||||
top_value = _top_level_model_value(od_config, top_level_keys)
|
||||
if top_value is not None:
|
||||
return cast(top_value)
|
||||
|
||||
return cast(default)
|
||||
|
||||
|
||||
def _resolve_normalization_value(
|
||||
od_config: OmniDiffusionConfig,
|
||||
args: dict[str, Any],
|
||||
*,
|
||||
name: str,
|
||||
default: Any,
|
||||
aliases: tuple[str, ...] = (),
|
||||
) -> Any:
|
||||
keys = (f"sound_{name}", name, *aliases)
|
||||
custom_value = _custom_arg_value(args, keys)
|
||||
if custom_value is not None:
|
||||
return custom_value
|
||||
nested_value = _first_value_from_configs(_nested_sound_tokenizer_configs(od_config), (name, *aliases))
|
||||
return default if nested_value is None else nested_value
|
||||
|
||||
|
||||
def get_sound_config_value(
|
||||
od_config: OmniDiffusionConfig,
|
||||
name: str,
|
||||
default: Any,
|
||||
aliases: tuple[str, ...] = (),
|
||||
) -> Any:
|
||||
# Backward-compatible generic accessor. Prefer the more specific helpers
|
||||
# below for Cosmos3 sound tokenizer fields so precedence stays explicit.
|
||||
keys = (name, *aliases)
|
||||
for config in (
|
||||
_pipeline_args(od_config),
|
||||
getattr(od_config, "model_config", None),
|
||||
getattr(od_config, "tf_model_config", None),
|
||||
):
|
||||
if config is None:
|
||||
continue
|
||||
for key in keys:
|
||||
if hasattr(config, "get"):
|
||||
value = config.get(key, None)
|
||||
else:
|
||||
value = getattr(config, key, None)
|
||||
if value is not None:
|
||||
return value
|
||||
return default
|
||||
|
||||
|
||||
def get_sound_sample_rate(od_config: OmniDiffusionConfig) -> int:
|
||||
args = _pipeline_args(od_config)
|
||||
return _resolve_arch_value(
|
||||
od_config,
|
||||
args,
|
||||
{},
|
||||
field="sample_rate",
|
||||
custom_keys=("sound_sample_rate", "sample_rate"),
|
||||
nested_keys=("sample_rate", "sampling_rate"),
|
||||
top_level_keys=("sound_sample_rate", "sample_rate"),
|
||||
default=DEFAULT_SOUND_SAMPLE_RATE,
|
||||
cast=int,
|
||||
)
|
||||
|
||||
|
||||
def get_sound_channels(od_config: OmniDiffusionConfig) -> int:
|
||||
args = _pipeline_args(od_config)
|
||||
return _resolve_arch_value(
|
||||
od_config,
|
||||
args,
|
||||
{},
|
||||
field="audio_channels",
|
||||
custom_keys=("sound_audio_channels", "audio_channels", "stereo"),
|
||||
nested_keys=("audio_channels", "dec_out_channels", "stereo"),
|
||||
top_level_keys=("sound_audio_channels", "audio_channels", "stereo"),
|
||||
default=DEFAULT_SOUND_CHANNELS,
|
||||
cast=_as_audio_channels,
|
||||
)
|
||||
|
||||
|
||||
def get_sound_hop_size(od_config: OmniDiffusionConfig) -> int:
|
||||
args = _pipeline_args(od_config)
|
||||
return _resolve_arch_value(
|
||||
od_config,
|
||||
args,
|
||||
{},
|
||||
field="hop_size",
|
||||
custom_keys=("sound_hop_size", "hop_size"),
|
||||
nested_keys=("hop_size",),
|
||||
top_level_keys=("sound_hop_size", "hop_size"),
|
||||
default=DEFAULT_SOUND_HOP_SIZE,
|
||||
cast=int,
|
||||
)
|
||||
|
||||
|
||||
class Cosmos3SoundTokenizer:
|
||||
"""Thin adapter around the local AVAE tokenizer implementation."""
|
||||
|
||||
def __init__(self, tokenizer: Any) -> None:
|
||||
self.tokenizer = tokenizer
|
||||
self.sample_rate = int(getattr(tokenizer, "sample_rate", DEFAULT_SOUND_SAMPLE_RATE))
|
||||
self.audio_channels = int(getattr(tokenizer, "audio_channels", DEFAULT_SOUND_CHANNELS))
|
||||
self.latent_ch = int(getattr(tokenizer, "latent_ch", DEFAULT_SOUND_DIM))
|
||||
self.hop_size = int(getattr(tokenizer, "temporal_compression_factor", DEFAULT_SOUND_HOP_SIZE))
|
||||
if self.hop_size <= 0:
|
||||
raise ValueError(f"Cosmos3 sound tokenizer hop_size must be positive, got {self.hop_size}.")
|
||||
self.latent_fps = float(self.sample_rate) / float(self.hop_size)
|
||||
|
||||
@classmethod
|
||||
def from_config(cls, od_config: OmniDiffusionConfig) -> Cosmos3SoundTokenizer:
|
||||
args = _pipeline_args(od_config)
|
||||
model_path = getattr(od_config, "model", None)
|
||||
explicit_avae_path = (
|
||||
args.get("sound_tokenizer_path")
|
||||
or args.get("avae_path")
|
||||
or args.get("cosmos3_avae_path")
|
||||
or os.environ.get("COSMOS3_SOUND_TOKENIZER_PATH")
|
||||
)
|
||||
explicit_config_path = args.get("sound_tokenizer_config_path") or os.environ.get(
|
||||
"COSMOS3_SOUND_TOKENIZER_CONFIG_PATH"
|
||||
)
|
||||
|
||||
model_root = str(model_path) if model_path and os.path.isdir(model_path) else None
|
||||
if model_root is None and model_path and not explicit_avae_path:
|
||||
from huggingface_hub import snapshot_download
|
||||
|
||||
model_root = snapshot_download(
|
||||
repo_id=str(model_path),
|
||||
revision=getattr(od_config, "revision", None),
|
||||
allow_patterns=[
|
||||
f"{SOUND_TOKENIZER_COMPONENT_NAME}/config.json",
|
||||
f"{SOUND_TOKENIZER_COMPONENT_NAME}/{SOUND_TOKENIZER_CHECKPOINT_NAME}",
|
||||
],
|
||||
)
|
||||
|
||||
if explicit_avae_path:
|
||||
avae_path = _resolve_model_file(explicit_avae_path, model_root)
|
||||
else:
|
||||
tokenizer_dir = Path(model_root) / SOUND_TOKENIZER_COMPONENT_NAME if model_root else None
|
||||
candidate = tokenizer_dir / SOUND_TOKENIZER_CHECKPOINT_NAME if tokenizer_dir else None
|
||||
avae_path = str(candidate) if candidate and candidate.exists() else None
|
||||
|
||||
if not avae_path:
|
||||
raise ValueError(
|
||||
"Cosmos3 sound generation was requested, but no AVAE sound "
|
||||
"tokenizer checkpoint was provided. Set "
|
||||
"custom_pipeline_args['sound_tokenizer_path'] or "
|
||||
"COSMOS3_SOUND_TOKENIZER_PATH, or include "
|
||||
f"{SOUND_TOKENIZER_COMPONENT_NAME}/{SOUND_TOKENIZER_CHECKPOINT_NAME} under the model path."
|
||||
)
|
||||
|
||||
config_path = _resolve_model_file(explicit_config_path, model_root)
|
||||
if config_path is None and model_root:
|
||||
candidate = Path(model_root) / SOUND_TOKENIZER_COMPONENT_NAME / "config.json"
|
||||
config_path = str(candidate) if candidate.exists() else None
|
||||
component_config = _load_sound_tokenizer_component_config(config_path)
|
||||
component_values = _component_arch_values(component_config)
|
||||
|
||||
sample_rate = _resolve_arch_value(
|
||||
od_config,
|
||||
args,
|
||||
component_values,
|
||||
field="sample_rate",
|
||||
custom_keys=("sound_sample_rate", "sample_rate"),
|
||||
nested_keys=("sample_rate", "sampling_rate"),
|
||||
top_level_keys=("sound_sample_rate", "sample_rate"),
|
||||
default=DEFAULT_SOUND_SAMPLE_RATE,
|
||||
cast=int,
|
||||
)
|
||||
audio_channels = _resolve_arch_value(
|
||||
od_config,
|
||||
args,
|
||||
component_values,
|
||||
field="audio_channels",
|
||||
custom_keys=("sound_audio_channels", "audio_channels", "stereo"),
|
||||
nested_keys=("audio_channels", "dec_out_channels", "stereo"),
|
||||
top_level_keys=("sound_audio_channels", "audio_channels", "stereo"),
|
||||
default=DEFAULT_SOUND_CHANNELS,
|
||||
cast=_as_audio_channels,
|
||||
)
|
||||
sound_dim = _resolve_arch_value(
|
||||
od_config,
|
||||
args,
|
||||
component_values,
|
||||
field="io_channels",
|
||||
custom_keys=("sound_dim", "io_channels", "latent_ch"),
|
||||
nested_keys=("io_channels", "vocoder_input_dim", "latent_ch"),
|
||||
top_level_keys=("sound_dim",),
|
||||
default=DEFAULT_SOUND_DIM,
|
||||
cast=int,
|
||||
)
|
||||
hop_size = _resolve_arch_value(
|
||||
od_config,
|
||||
args,
|
||||
component_values,
|
||||
field="hop_size",
|
||||
custom_keys=("sound_hop_size", "hop_size"),
|
||||
nested_keys=("hop_size",),
|
||||
top_level_keys=("sound_hop_size", "hop_size"),
|
||||
default=DEFAULT_SOUND_HOP_SIZE,
|
||||
cast=int,
|
||||
)
|
||||
normalize_latents = _as_bool(
|
||||
_resolve_normalization_value(
|
||||
od_config,
|
||||
args,
|
||||
name="normalize_latents",
|
||||
default=DEFAULT_SOUND_NORMALIZE_LATENTS,
|
||||
)
|
||||
)
|
||||
normalization_type = str(
|
||||
_resolve_normalization_value(
|
||||
od_config,
|
||||
args,
|
||||
name="normalization_type",
|
||||
default=DEFAULT_SOUND_NORMALIZATION_TYPE,
|
||||
)
|
||||
)
|
||||
tanh_input_scale = float(
|
||||
_resolve_normalization_value(
|
||||
od_config,
|
||||
args,
|
||||
name="tanh_input_scale",
|
||||
default=DEFAULT_SOUND_TANH_INPUT_SCALE,
|
||||
)
|
||||
)
|
||||
tanh_output_scale = float(
|
||||
_resolve_normalization_value(
|
||||
od_config,
|
||||
args,
|
||||
name="tanh_output_scale",
|
||||
default=DEFAULT_SOUND_TANH_OUTPUT_SCALE,
|
||||
)
|
||||
)
|
||||
tanh_clamp = float(
|
||||
_resolve_normalization_value(
|
||||
od_config,
|
||||
args,
|
||||
name="tanh_clamp",
|
||||
default=DEFAULT_SOUND_TANH_CLAMP,
|
||||
)
|
||||
)
|
||||
tokenizer = Cosmos3AVAEAudioTokenizer(
|
||||
checkpoint_path=str(avae_path),
|
||||
config_path=config_path,
|
||||
sample_rate=sample_rate,
|
||||
audio_channels=audio_channels,
|
||||
io_channels=sound_dim,
|
||||
hop_size=hop_size,
|
||||
normalize_latents=normalize_latents,
|
||||
normalization_type=normalization_type,
|
||||
tanh_input_scale=tanh_input_scale,
|
||||
tanh_output_scale=tanh_output_scale,
|
||||
tanh_clamp=tanh_clamp,
|
||||
dtype=getattr(od_config, "dtype", torch.bfloat16),
|
||||
device=get_local_device(),
|
||||
)
|
||||
if _is_rank_zero():
|
||||
logger.info(
|
||||
"Loaded Cosmos3 AVAE sound tokenizer from %s "
|
||||
"(sr=%d, channels=%d, latent_ch=%d, hop=%d, latent_fps=%.3f)",
|
||||
avae_path,
|
||||
sample_rate,
|
||||
audio_channels,
|
||||
sound_dim,
|
||||
hop_size,
|
||||
float(sample_rate) / float(hop_size),
|
||||
)
|
||||
return cls(tokenizer)
|
||||
|
||||
def get_latent_num_samples(self, num_audio_samples: int) -> int:
|
||||
return int(self.tokenizer.get_latent_num_samples(num_audio_samples))
|
||||
|
||||
def get_audio_num_samples(self, num_latent_samples: int) -> int:
|
||||
return int(self.tokenizer.get_audio_num_samples(num_latent_samples))
|
||||
|
||||
@torch.no_grad()
|
||||
def decode(self, latents: torch.Tensor) -> torch.Tensor:
|
||||
"""Decode sound latents.
|
||||
|
||||
Args:
|
||||
latents: ``[B, C, T]`` or ``[C, T]`` tensor.
|
||||
|
||||
Returns:
|
||||
``[B, audio_channels, N]`` tensor for batched input, or
|
||||
``[audio_channels, N]`` for unbatched input.
|
||||
"""
|
||||
squeeze = latents.ndim == 2
|
||||
if squeeze:
|
||||
latents = latents.unsqueeze(0)
|
||||
audio = self.tokenizer.decode(latents)
|
||||
audio = audio.clamp(-1.0, 1.0)
|
||||
return audio.squeeze(0) if squeeze else audio
|
||||
@@ -76,6 +76,66 @@ def _tf_config_get(config: Any, key: str, default: Any) -> Any:
|
||||
return getattr(config, key, default)
|
||||
|
||||
|
||||
def _nested_get(value: Any, key: str) -> Any:
|
||||
if isinstance(value, dict):
|
||||
if key in value:
|
||||
return value[key]
|
||||
for child in value.values():
|
||||
found = _nested_get(child, key)
|
||||
if found is not None:
|
||||
return found
|
||||
elif isinstance(value, list | tuple):
|
||||
for child in value:
|
||||
found = _nested_get(child, key)
|
||||
if found is not None:
|
||||
return found
|
||||
return None
|
||||
|
||||
|
||||
def _od_config_get(od_config: Any, key: str, default: Any = None) -> Any:
|
||||
"""Read Cosmos3 options from runtime, model, or transformer config."""
|
||||
if od_config is None:
|
||||
return default
|
||||
for attr in ("custom_pipeline_args", "model_config"):
|
||||
source = getattr(od_config, attr, None) or {}
|
||||
if isinstance(source, dict):
|
||||
if key in source:
|
||||
return source[key]
|
||||
found = _nested_get(source, key)
|
||||
if found is not None:
|
||||
return found
|
||||
tf_model_config = getattr(od_config, "tf_model_config", None)
|
||||
if isinstance(tf_model_config, dict):
|
||||
if key in tf_model_config:
|
||||
return tf_model_config[key]
|
||||
found = _nested_get(tf_model_config, key)
|
||||
if found is not None:
|
||||
return found
|
||||
value = _tf_config_get(tf_model_config, key, None)
|
||||
return default if value is None else value
|
||||
|
||||
|
||||
def _as_bool(value: Any) -> bool:
|
||||
if isinstance(value, str):
|
||||
return value.strip().lower() in {"1", "true", "yes", "on"}
|
||||
return bool(value)
|
||||
|
||||
|
||||
def resolve_sound_gen(od_config: Any) -> bool:
|
||||
"""Capability gate shared by the pipeline and transformer.
|
||||
|
||||
Explicit ``sound_gen`` flag wins (including an explicit False);
|
||||
otherwise infer from the presence of any sound-width key in od_config.
|
||||
"""
|
||||
sound_gen_value = _od_config_get(od_config, "sound_gen", None)
|
||||
if sound_gen_value is not None:
|
||||
return _as_bool(sound_gen_value)
|
||||
for key in ("sound_dim", "io_channels", "vocoder_input_dim", "latent_ch"):
|
||||
if _od_config_get(od_config, key, None) is not None:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Rotary Position Embeddings (mRoPE)
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -138,6 +198,28 @@ def compute_mrope_position_ids_vision(
|
||||
return mrope_ids, next_offset
|
||||
|
||||
|
||||
def compute_mrope_position_ids_sound(
|
||||
grid_t: int,
|
||||
temporal_offset: int | float,
|
||||
sound_latent_fps: float,
|
||||
base_fps: float = 24.0,
|
||||
temporal_compression_factor_sound: int = 1,
|
||||
enable_fps_modulation: bool = True,
|
||||
) -> tuple[torch.Tensor, int | float]:
|
||||
"""Generate mRoPE IDs for sound tokens as a (T, 1, 1) grid."""
|
||||
return compute_mrope_position_ids_vision(
|
||||
grid_t=grid_t,
|
||||
grid_h=1,
|
||||
grid_w=1,
|
||||
temporal_offset=temporal_offset,
|
||||
fps=sound_latent_fps,
|
||||
base_fps=base_fps,
|
||||
temporal_compression_factor=temporal_compression_factor_sound,
|
||||
base_temporal_compression_factor=temporal_compression_factor_sound,
|
||||
enable_fps_modulation=enable_fps_modulation,
|
||||
)
|
||||
|
||||
|
||||
class Qwen3VLTextRotaryEmbedding(nn.Module):
|
||||
"""Multi-dimensional rotary position embedding for Qwen3-VL."""
|
||||
|
||||
@@ -839,6 +921,9 @@ class Cosmos3VFMTransformer(nn.Module):
|
||||
od_config: OmniDiffusionConfig,
|
||||
*,
|
||||
temporal_compression_factor: int | None = None,
|
||||
sound_gen: bool = False,
|
||||
sound_dim: int | None = None,
|
||||
sound_latent_fps: float | None = None,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
model_config = od_config.tf_model_config
|
||||
@@ -859,9 +944,22 @@ class Cosmos3VFMTransformer(nn.Module):
|
||||
self.latent_channel_size = int(_tf_config_get(model_config, "latent_channel", 48))
|
||||
self.timestep_scale = float(_tf_config_get(model_config, "timestep_scale", 0.001))
|
||||
self.base_fps = float(_tf_config_get(model_config, "base_fps", 24.0))
|
||||
self.sound_gen = sound_gen
|
||||
self.sound_dim = sound_dim
|
||||
self.sound_latent_fps = sound_latent_fps
|
||||
|
||||
if self.sound_gen and (sound_dim is None or sound_latent_fps is None):
|
||||
raise ValueError(
|
||||
"Cosmos3VFMTransformer requires an explicit sound_dim and sound_latent_fps when sound_gen is True; "
|
||||
"the pipeline must pass Cosmos3SoundTokenizer.latent_ch so the audio projection "
|
||||
"layers are sized from the authoritative AVAE latent width."
|
||||
)
|
||||
if temporal_compression_factor is None:
|
||||
temporal_compression_factor = _tf_config_get(model_config, "temporal_compression_factor", 4)
|
||||
self.temporal_compression_factor = int(temporal_compression_factor)
|
||||
self.temporal_compression_factor_sound = int(
|
||||
_tf_config_get(model_config, "temporal_compression_factor_sound", 1)
|
||||
)
|
||||
self.enable_fps_modulation = bool(_tf_config_get(model_config, "enable_fps_modulation", True))
|
||||
self.temporal_modality_margin = int(
|
||||
_tf_config_get(
|
||||
@@ -894,6 +992,10 @@ class Cosmos3VFMTransformer(nn.Module):
|
||||
self.proj_in = nn.Linear(self.patch_latent_dim, self.hidden_size)
|
||||
self.proj_out = nn.Linear(self.hidden_size, self.patch_latent_dim)
|
||||
self.time_embedder = TimestepEmbedder(self.hidden_size, target_dtype=dtype)
|
||||
if self.sound_gen:
|
||||
self.audio_proj_in = nn.Linear(self.sound_dim, self.hidden_size)
|
||||
self.audio_proj_out = nn.Linear(self.hidden_size, self.sound_dim)
|
||||
self.audio_modality_embed = nn.Parameter(torch.zeros(self.hidden_size))
|
||||
|
||||
self.gen_layers = nn.ModuleList(
|
||||
[
|
||||
@@ -962,6 +1064,21 @@ class Cosmos3VFMTransformer(nn.Module):
|
||||
x = x[:, :, :, :h, :w]
|
||||
return x
|
||||
|
||||
def pack_sound(self, sound_latents: torch.Tensor) -> torch.Tensor:
|
||||
"""[B, C_sound, T_sound] -> [B, T_sound, C_sound]."""
|
||||
if sound_latents.ndim != 3:
|
||||
raise ValueError(f"Cosmos3 sound latents must have shape [B, C, T], got {tuple(sound_latents.shape)}.")
|
||||
if sound_latents.shape[1] != self.sound_dim:
|
||||
raise ValueError(
|
||||
f"Cosmos3 sound latent channel mismatch: expected {self.sound_dim}, got {sound_latents.shape[1]}."
|
||||
)
|
||||
return sound_latents.permute(0, 2, 1).contiguous()
|
||||
|
||||
@staticmethod
|
||||
def unpack_sound(tokens: torch.Tensor) -> torch.Tensor:
|
||||
"""[B, T_sound, C_sound] -> [B, C_sound, T_sound]."""
|
||||
return tokens.permute(0, 2, 1).contiguous()
|
||||
|
||||
# -- RoPE computation ----------------------------------------------------
|
||||
|
||||
def _compute_rope_freqs(
|
||||
@@ -973,12 +1090,14 @@ class Cosmos3VFMTransformer(nn.Module):
|
||||
fps: float | None,
|
||||
device: torch.device,
|
||||
dtype: torch.dtype,
|
||||
t_sound: int | None = None,
|
||||
) -> tuple[tuple[torch.Tensor, torch.Tensor], tuple[torch.Tensor, torch.Tensor]]:
|
||||
"""Compute mRoPE cos/sin for UND text and GEN media pathways."""
|
||||
B = text_mask.shape[0]
|
||||
S_text = text_mask.shape[1]
|
||||
text_lengths = text_mask.sum(dim=1).long()
|
||||
effective_fps = fps if fps is not None and t > 1 else None
|
||||
sound_frames = int(t_sound or 0)
|
||||
|
||||
text_pos_list = []
|
||||
gen_pos_list = []
|
||||
@@ -996,6 +1115,21 @@ class Cosmos3VFMTransformer(nn.Module):
|
||||
temporal_compression_factor=self.temporal_compression_factor,
|
||||
enable_fps_modulation=self.enable_fps_modulation,
|
||||
)
|
||||
gen_positions = [v_pos]
|
||||
if sound_frames > 0:
|
||||
s_pos, _ = compute_mrope_position_ids_sound(
|
||||
sound_frames,
|
||||
temporal_offset=media_temporal_offset,
|
||||
sound_latent_fps=self.sound_latent_fps,
|
||||
base_fps=self.base_fps,
|
||||
temporal_compression_factor_sound=getattr(self, "temporal_compression_factor_sound", 1),
|
||||
enable_fps_modulation=self.enable_fps_modulation,
|
||||
)
|
||||
gen_positions.append(s_pos)
|
||||
pos_dtype = gen_positions[0].dtype
|
||||
for pos in gen_positions[1:]:
|
||||
pos_dtype = torch.promote_types(pos_dtype, pos.dtype)
|
||||
v_pos = torch.cat([pos.to(pos_dtype) for pos in gen_positions], dim=1)
|
||||
if real_len < S_text:
|
||||
t_pos = torch.cat(
|
||||
[t_pos, torch.zeros(3, S_text - real_len, dtype=t_pos.dtype)],
|
||||
@@ -1026,16 +1160,31 @@ class Cosmos3VFMTransformer(nn.Module):
|
||||
def _validate_gen_sequence_parallel(
|
||||
*,
|
||||
s_gen: int,
|
||||
s_video: int,
|
||||
s_sound: int,
|
||||
has_sound: bool,
|
||||
ulysses_size: int,
|
||||
) -> None:
|
||||
if ulysses_size <= 1 or s_gen % ulysses_size == 0:
|
||||
return
|
||||
|
||||
detail_parts = [f"video tokens {s_video}"]
|
||||
if has_sound:
|
||||
detail_parts.append(f"sound tokens {s_sound}")
|
||||
detail = " = " + " + ".join(detail_parts) if len(detail_parts) > 1 else ""
|
||||
adjust_detail = (
|
||||
"Adjust the spatial resolution so that t * ceil(h/patch) * ceil(w/patch) is a multiple of ulysses_degree."
|
||||
"Adjust the spatial resolution, frame count, sound duration, "
|
||||
"or sound latent FPS so the combined media sequence is a "
|
||||
"multiple of ulysses_degree."
|
||||
if has_sound
|
||||
else (
|
||||
"Adjust the spatial resolution so that "
|
||||
"t * ceil(h/patch) * ceil(w/patch) is a multiple "
|
||||
"of ulysses_degree."
|
||||
)
|
||||
)
|
||||
raise ValueError(
|
||||
f"GEN sequence length ({s_gen} video tokens) must be divisible by "
|
||||
f"GEN sequence length ({s_gen}{detail}) must be divisible by "
|
||||
f"ulysses_degree ({ulysses_size}). {adjust_detail}"
|
||||
)
|
||||
|
||||
@@ -1049,9 +1198,10 @@ class Cosmos3VFMTransformer(nn.Module):
|
||||
text_mask: torch.Tensor,
|
||||
video_shape: tuple[int, int, int],
|
||||
fps: float | None = None,
|
||||
sound_latents: torch.Tensor | None = None,
|
||||
noisy_frame_mask: torch.Tensor | None = None,
|
||||
**kwargs,
|
||||
) -> torch.Tensor:
|
||||
) -> torch.Tensor | tuple[torch.Tensor, ...]:
|
||||
"""
|
||||
Args:
|
||||
hidden_states: [B, C, t, h, w] noisy latents
|
||||
@@ -1060,13 +1210,15 @@ class Cosmos3VFMTransformer(nn.Module):
|
||||
text_mask: [B, S_text] attention mask (1=real, 0=pad)
|
||||
video_shape: (t, h, w) in latent space
|
||||
fps: video frame rate for temporal mRoPE modulation
|
||||
sound_latents: Optional [B, C_sound, T_sound] noisy sound latents.
|
||||
noisy_frame_mask: Optional [B, 1, t, 1, 1] mask where 1=noisy (add
|
||||
timestep embedding, predict velocity) and 0=conditioned (clean
|
||||
context, skip timestep embedding). None means all frames noisy
|
||||
(T2V mode).
|
||||
|
||||
Returns:
|
||||
[B, C, t, h, w] velocity prediction.
|
||||
[B, C, t, h, w] velocity prediction, or
|
||||
tuple outputs in video, sound order when sound latents are provided.
|
||||
"""
|
||||
t, h, w = video_shape
|
||||
hp, wp, _, _ = self._pad_to_patch_size(h, w)
|
||||
@@ -1078,12 +1230,31 @@ class Cosmos3VFMTransformer(nn.Module):
|
||||
f"Cosmos3 requires identical real text lengths within a batch "
|
||||
f"(got min={min_real_len}, max={max_real_len})."
|
||||
)
|
||||
has_sound = sound_latents is not None
|
||||
if has_sound and not self.sound_gen:
|
||||
raise ValueError(
|
||||
"Cosmos3 sound generation was requested, but this transformer "
|
||||
"was initialized without sound modules. Check that the "
|
||||
"transformer config enables sound_gen or defines sound_dim."
|
||||
)
|
||||
|
||||
# Query Ulysses state at runtime
|
||||
ulysses_size, _, _ = _get_ulysses_state()
|
||||
|
||||
# Patchify latents and project to hidden space
|
||||
hidden_video = self.proj_in(self.patchify(hidden_states, t, h, w))
|
||||
s_video = hidden_video.shape[1]
|
||||
s_sound = 0
|
||||
hidden_sound = None
|
||||
if sound_latents is not None:
|
||||
if sound_latents.shape[0] != hidden_states.shape[0]:
|
||||
raise ValueError(
|
||||
"Cosmos3 sound and video batch sizes must match: "
|
||||
f"video={hidden_states.shape[0]}, sound={sound_latents.shape[0]}."
|
||||
)
|
||||
hidden_sound = self.audio_proj_in(self.pack_sound(sound_latents))
|
||||
hidden_sound = hidden_sound + self.audio_modality_embed.to(hidden_sound.dtype)
|
||||
s_sound = hidden_sound.shape[1]
|
||||
|
||||
# Timestep embedding (fp32 for precision).
|
||||
# For I2V: only add to noisy tokens, not conditioned ones.
|
||||
@@ -1106,7 +1277,12 @@ class Cosmos3VFMTransformer(nn.Module):
|
||||
else:
|
||||
hidden_video = hidden_video + time_embed.unsqueeze(1)
|
||||
|
||||
hidden_gen = hidden_video
|
||||
if hidden_sound is not None:
|
||||
hidden_sound = hidden_sound + time_embed.unsqueeze(1)
|
||||
hidden_parts = [hidden_video]
|
||||
if hidden_sound is not None:
|
||||
hidden_parts.append(hidden_sound)
|
||||
hidden_gen = torch.cat(hidden_parts, dim=1)
|
||||
|
||||
# Run UND pathway once and cache K/V (replicated across all ranks)
|
||||
if self.cached_kv is None:
|
||||
@@ -1118,6 +1294,7 @@ class Cosmos3VFMTransformer(nn.Module):
|
||||
fps,
|
||||
hidden_states.device,
|
||||
hidden_states.dtype,
|
||||
t_sound=s_sound,
|
||||
)
|
||||
cached_kv_full = self.language_model(text_ids, freqs_und)
|
||||
self.cached_freqs_gen = freqs_gen
|
||||
@@ -1133,6 +1310,9 @@ class Cosmos3VFMTransformer(nn.Module):
|
||||
raise RuntimeError("Cosmos3 GEN cache was not initialized before running GEN layers.")
|
||||
self._validate_gen_sequence_parallel(
|
||||
s_gen=hidden_gen.shape[1],
|
||||
s_video=s_video,
|
||||
s_sound=s_sound,
|
||||
has_sound=has_sound,
|
||||
ulysses_size=ulysses_size,
|
||||
)
|
||||
freqs_cos, freqs_sin = self.cached_freqs_gen
|
||||
@@ -1166,7 +1346,21 @@ class Cosmos3VFMTransformer(nn.Module):
|
||||
|
||||
# Final norm and project back to latent space
|
||||
hidden_gen = self.norm_moe_gen(hidden_gen)
|
||||
return self.unpatchify(self.proj_out(hidden_gen), t, h, w)
|
||||
if not has_sound:
|
||||
return self.unpatchify(self.proj_out(hidden_gen), t, h, w)
|
||||
|
||||
split_sizes = [s_video]
|
||||
if has_sound:
|
||||
split_sizes.append(s_sound)
|
||||
split_hidden = hidden_gen.split(split_sizes, dim=1)
|
||||
hidden_video = split_hidden[0]
|
||||
video_pred = self.unpatchify(self.proj_out(hidden_video), t, h, w)
|
||||
outputs: list[torch.Tensor] = [video_pred]
|
||||
split_idx = 1
|
||||
if has_sound:
|
||||
hidden_sound = split_hidden[split_idx]
|
||||
outputs.append(self.unpack_sound(self.audio_proj_out(hidden_sound)))
|
||||
return tuple(outputs)
|
||||
|
||||
def post_load_weights(self) -> None:
|
||||
"""Post-load processing: ensure correct dtypes."""
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
@@ -0,0 +1,100 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""Action encoder/decoder for DreamZero."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
|
||||
|
||||
def swish(x: torch.Tensor) -> torch.Tensor:
|
||||
"""swish activation: x * sigmoid(x)"""
|
||||
return x * torch.sigmoid(x)
|
||||
|
||||
|
||||
class SinusoidalPositionalEncoding(nn.Module):
|
||||
"""Sinusoidal encoding: (B, T) timesteps → (B, T, dim)"""
|
||||
|
||||
def __init__(self, embedding_dim: int) -> None:
|
||||
super().__init__()
|
||||
self.embedding_dim = embedding_dim
|
||||
|
||||
def forward(self, timesteps: torch.Tensor) -> torch.Tensor:
|
||||
timesteps = timesteps.float()
|
||||
half_dim = self.embedding_dim // 2
|
||||
exponent = -torch.arange(half_dim, dtype=torch.float, device=timesteps.device) * (
|
||||
torch.log(torch.tensor(10000.0)) / half_dim
|
||||
)
|
||||
freqs = timesteps.unsqueeze(-1) * exponent.exp()
|
||||
return torch.cat([torch.sin(freqs), torch.cos(freqs)], dim=-1)
|
||||
|
||||
|
||||
class CategorySpecificLinear(nn.Module):
|
||||
"""Per-category linear: W[cat_id] @ x + b[cat_id]
|
||||
|
||||
Attributes:
|
||||
W: (num_categories, input_dim, hidden_dim) — note: 0.02 * randn init
|
||||
b: (num_categories, hidden_dim) — zero init
|
||||
"""
|
||||
|
||||
def __init__(self, num_categories: int, input_dim: int, hidden_dim: int) -> None:
|
||||
super().__init__()
|
||||
self.W = nn.Parameter(0.02 * torch.randn(num_categories, input_dim, hidden_dim))
|
||||
self.b = nn.Parameter(torch.zeros(num_categories, hidden_dim))
|
||||
|
||||
def forward(self, x: torch.Tensor, cat_ids: torch.Tensor) -> torch.Tensor:
|
||||
selected_W = self.W[cat_ids]
|
||||
selected_b = self.b[cat_ids]
|
||||
return torch.bmm(x, selected_W) + selected_b.unsqueeze(1)
|
||||
|
||||
|
||||
class CategorySpecificMLP(nn.Module):
|
||||
"""Two-layer MLP: layer1 (relu) → layer2"""
|
||||
|
||||
def __init__(self, num_categories: int, input_dim: int, hidden_dim: int, output_dim: int) -> None:
|
||||
super().__init__()
|
||||
self.layer1 = CategorySpecificLinear(num_categories, input_dim, hidden_dim)
|
||||
self.layer2 = CategorySpecificLinear(num_categories, hidden_dim, output_dim)
|
||||
|
||||
def forward(self, x: torch.Tensor, cat_ids: torch.Tensor) -> torch.Tensor:
|
||||
hidden = F.relu(self.layer1(x, cat_ids))
|
||||
return self.layer2(hidden, cat_ids)
|
||||
|
||||
|
||||
class MultiEmbodimentActionEncoder(nn.Module):
|
||||
"""Encode actions with embodiment-specific weights + sinusoidal timestep.
|
||||
|
||||
Flow: actions → W1 → concat(a_emb, pos_enc(timesteps)) → W2 (swish) → W3
|
||||
|
||||
Args:
|
||||
action_dim: action vector dimension (e.g. 32)
|
||||
hidden_size: output/hidden dimension (e.g. 5120 = model dim)
|
||||
num_embodiments: number of robot types (e.g. 32)
|
||||
"""
|
||||
|
||||
def __init__(self, action_dim: int, hidden_size: int, num_embodiments: int) -> None:
|
||||
super().__init__()
|
||||
self.hidden_size = hidden_size
|
||||
self.W1 = CategorySpecificLinear(num_embodiments, action_dim, hidden_size)
|
||||
self.W2 = CategorySpecificLinear(num_embodiments, 2 * hidden_size, hidden_size)
|
||||
self.W3 = CategorySpecificLinear(num_embodiments, hidden_size, hidden_size)
|
||||
self.pos_encoding = SinusoidalPositionalEncoding(hidden_size)
|
||||
|
||||
def forward(self, actions: torch.Tensor, timesteps: torch.Tensor, cat_ids: torch.Tensor) -> torch.Tensor:
|
||||
"""
|
||||
Args:
|
||||
actions: (B, T, action_dim)
|
||||
timesteps: (B, T) — per-token timestep
|
||||
cat_ids: (B,) — embodiment id per sample
|
||||
Returns:
|
||||
(B, T, hidden_size)
|
||||
"""
|
||||
a_emb = self.W1(actions, cat_ids)
|
||||
tau_emb = self.pos_encoding(timesteps).to(dtype=a_emb.dtype)
|
||||
x = torch.cat([a_emb, tau_emb], dim=-1)
|
||||
x = swish(self.W2(x, cat_ids))
|
||||
x = self.W3(x, cat_ids)
|
||||
return x
|
||||
@@ -0,0 +1,954 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""CausalWanModel — 40-layer DiT with causal attention and KV cache.
|
||||
|
||||
Key differences from WanTransformer3DModel:
|
||||
- Causal self-attention (new frames only see history)
|
||||
- KV cache for streaming inference
|
||||
- Action/state token support (appended after video tokens)
|
||||
- Extended RoPE with action/state-specific frequencies
|
||||
- Inference-only forward with KV cache
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
from vllm.distributed import (
|
||||
get_tensor_model_parallel_rank,
|
||||
get_tensor_model_parallel_world_size,
|
||||
get_tp_group,
|
||||
)
|
||||
from vllm.model_executor.layers.conv import Conv3dLayer
|
||||
from vllm.model_executor.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
RowParallelLinear,
|
||||
)
|
||||
from vllm.model_executor.utils import set_weight_attrs
|
||||
|
||||
from vllm_omni.diffusion.attention.layer import Attention
|
||||
from vllm_omni.diffusion.models.dreamzero.action_encoder import (
|
||||
CategorySpecificMLP,
|
||||
MultiEmbodimentActionEncoder,
|
||||
)
|
||||
|
||||
# ── RoPE utilities ──────────────────────────────────────────────────
|
||||
|
||||
|
||||
def sinusoidal_embedding_1d(dim: int, position: torch.Tensor) -> torch.Tensor:
|
||||
"""Sinusoidal positional embedding for timesteps."""
|
||||
if dim % 2 != 0:
|
||||
raise ValueError(f"dim must be even, got {dim}.")
|
||||
half = dim // 2
|
||||
position = position.type(torch.float64)
|
||||
sinusoid = torch.outer(
|
||||
position,
|
||||
torch.pow(10000, -torch.arange(half, dtype=position.dtype, device=position.device).div(half)),
|
||||
)
|
||||
x = torch.cat([torch.cos(sinusoid), torch.sin(sinusoid)], dim=1)
|
||||
return x
|
||||
|
||||
|
||||
def rope_params(max_seq_len: int, dim: int) -> torch.Tensor:
|
||||
"""Precompute complex-valued RoPE frequencies (polar form).
|
||||
Returns: complex tensor [max_seq_len, dim // 2]
|
||||
"""
|
||||
if dim % 2 != 0:
|
||||
raise ValueError(f"dim must be even, got {dim}.")
|
||||
freqs = torch.outer(
|
||||
torch.arange(max_seq_len),
|
||||
1.0 / torch.pow(10000, torch.arange(0, dim, 2).to(torch.float64).div(dim)),
|
||||
)
|
||||
freqs = torch.polar(torch.ones_like(freqs), freqs)
|
||||
return freqs
|
||||
|
||||
|
||||
def rope_apply(x: torch.Tensor, freqs: torch.Tensor) -> torch.Tensor:
|
||||
"""Apply RoPE to x using precomputed complex freqs."""
|
||||
B, seq_len, n, _ = x.shape
|
||||
x = torch.view_as_complex(x.to(torch.float64).reshape(B, seq_len, n, -1, 2))
|
||||
freqs = freqs.unsqueeze(0)
|
||||
x = torch.view_as_real(x * freqs).flatten(3)
|
||||
return x
|
||||
|
||||
|
||||
def rope_action_apply(
|
||||
x: torch.Tensor,
|
||||
freqs: torch.Tensor,
|
||||
freqs_action: torch.Tensor,
|
||||
freqs_state: torch.Tensor,
|
||||
action_register_length: int | None,
|
||||
num_action_per_block: int = 32,
|
||||
num_state_per_block: int = 1,
|
||||
) -> torch.Tensor:
|
||||
"""RoPE with action/state frequency tables for multi-step sequences."""
|
||||
B, seq_len, n, _ = x.shape
|
||||
x = torch.view_as_complex(x.to(torch.float64).reshape(B, seq_len, n, -1, 2))
|
||||
if action_register_length is not None:
|
||||
if num_action_per_block is None:
|
||||
raise ValueError("num_action_per_block is required when action_register_length is set.")
|
||||
if num_state_per_block is None:
|
||||
raise ValueError("num_state_per_block is required when action_register_length is set.")
|
||||
chunk_size = action_register_length // (num_action_per_block + num_state_per_block)
|
||||
freqs_1d_action = freqs_action[: chunk_size * num_action_per_block].view(
|
||||
chunk_size * num_action_per_block, 1, -1
|
||||
)
|
||||
freqs_1d_state = freqs_state[: chunk_size * num_state_per_block].view(chunk_size * num_state_per_block, 1, -1)
|
||||
freqs = torch.cat([freqs, freqs_1d_action, freqs_1d_state], dim=0)
|
||||
freqs = freqs.unsqueeze(0)
|
||||
x = torch.view_as_real(x * freqs).flatten(3)
|
||||
return x
|
||||
|
||||
|
||||
def causal_rope_action_apply(
|
||||
x: torch.Tensor,
|
||||
freqs: torch.Tensor,
|
||||
freqs_action: torch.Tensor,
|
||||
freqs_state: torch.Tensor,
|
||||
action_register_length: int | None,
|
||||
num_action_per_block: int,
|
||||
num_state_per_block: int,
|
||||
action_state_index: int,
|
||||
) -> torch.Tensor:
|
||||
"""RoPE for single inference step (causal / KV-cache mode)."""
|
||||
B, seq_len, n, _ = x.shape
|
||||
x = torch.view_as_complex(x.to(torch.float64).reshape(B, seq_len, n, -1, 2))
|
||||
if action_register_length is not None:
|
||||
expected_length = num_action_per_block + num_state_per_block
|
||||
if action_register_length != expected_length:
|
||||
raise ValueError(
|
||||
f"action_register_length must equal num_action_per_block + num_state_per_block "
|
||||
f"({expected_length}), got {action_register_length}."
|
||||
)
|
||||
freqs_action = freqs_action[
|
||||
action_state_index * num_action_per_block : (action_state_index + 1) * num_action_per_block
|
||||
]
|
||||
freqs_state = freqs_state[
|
||||
action_state_index * num_state_per_block : (action_state_index + 1) * num_state_per_block
|
||||
]
|
||||
freqs_1d = torch.cat([freqs_action, freqs_state], dim=0).view(action_register_length, 1, -1)
|
||||
freqs = torch.cat([freqs, freqs_1d], dim=0)
|
||||
freqs = freqs.unsqueeze(0)
|
||||
x = torch.view_as_real(x * freqs).flatten(3)
|
||||
return x
|
||||
|
||||
|
||||
# ── Normalization ───────────────────────────────────────────────────
|
||||
|
||||
|
||||
class WanLayerNorm(nn.LayerNorm):
|
||||
"""LayerNorm wrapper used by DreamZero blocks."""
|
||||
|
||||
def __init__(self, dim: int, eps: float = 1e-6, elementwise_affine: bool = False) -> None:
|
||||
super().__init__(dim, eps=eps, elementwise_affine=elementwise_affine)
|
||||
|
||||
|
||||
class DistributedRMSNorm(nn.Module):
|
||||
"""RMSNorm that computes global RMS across tensor parallel ranks."""
|
||||
|
||||
def __init__(self, hidden_size: int, eps: float = 1e-6):
|
||||
super().__init__()
|
||||
self.eps = eps
|
||||
self.weight = nn.Parameter(torch.ones(hidden_size))
|
||||
set_weight_attrs(self.weight, {"weight_loader": self.weight_loader})
|
||||
|
||||
def weight_loader(self, param: torch.Tensor, loaded_weight: torch.Tensor) -> None:
|
||||
if param.shape == loaded_weight.shape:
|
||||
param.data.copy_(loaded_weight)
|
||||
return
|
||||
|
||||
tp_size = get_tensor_model_parallel_world_size()
|
||||
if loaded_weight.shape[0] % tp_size != 0:
|
||||
raise ValueError(
|
||||
f"Cannot shard RMSNorm weight of shape {tuple(loaded_weight.shape)} across tp_size={tp_size}."
|
||||
)
|
||||
|
||||
shard_size = loaded_weight.shape[0] // tp_size
|
||||
start_idx = get_tensor_model_parallel_rank() * shard_size
|
||||
shard = loaded_weight.narrow(0, start_idx, shard_size)
|
||||
if param.shape != shard.shape:
|
||||
raise ValueError(f"RMSNorm shard shape mismatch: param={tuple(param.shape)}, shard={tuple(shard.shape)}.")
|
||||
param.data.copy_(shard)
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
tp_size = get_tensor_model_parallel_world_size()
|
||||
x_float = x.float()
|
||||
local_sum_sq = x_float.pow(2).sum(dim=-1, keepdim=True)
|
||||
local_count = x.shape[-1]
|
||||
|
||||
if tp_size > 1:
|
||||
global_sum_sq = local_sum_sq.clone()
|
||||
torch.distributed.all_reduce(global_sum_sq, group=get_tp_group().device_group)
|
||||
global_count = local_count * tp_size
|
||||
else:
|
||||
global_sum_sq = local_sum_sq
|
||||
global_count = local_count
|
||||
|
||||
mean_sq = global_sum_sq / global_count
|
||||
return (x_float * torch.rsqrt(mean_sq + self.eps)).type_as(x) * self.weight
|
||||
|
||||
|
||||
# ── Projections ─────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class MLPProj(nn.Module):
|
||||
"""CLIP feature projection for i2v.
|
||||
Uses ColumnParallelLinear + RowParallelLinear (Qwen3_VisionMLP pattern).
|
||||
"""
|
||||
|
||||
def __init__(self, in_dim: int, out_dim: int) -> None:
|
||||
super().__init__()
|
||||
self.norm1 = nn.LayerNorm(in_dim)
|
||||
self.fc1 = ColumnParallelLinear(
|
||||
in_dim,
|
||||
in_dim,
|
||||
bias=True,
|
||||
return_bias=False,
|
||||
)
|
||||
self.act = nn.GELU()
|
||||
self.fc2 = RowParallelLinear(
|
||||
in_dim,
|
||||
out_dim,
|
||||
bias=True,
|
||||
return_bias=False,
|
||||
)
|
||||
self.norm2 = nn.LayerNorm(out_dim)
|
||||
|
||||
def forward(self, image_embeds: torch.Tensor) -> torch.Tensor:
|
||||
x = self.norm1(image_embeds)
|
||||
x = self.fc1(x)
|
||||
x = self.act(x)
|
||||
x = self.fc2(x)
|
||||
x = self.norm2(x)
|
||||
return x
|
||||
|
||||
|
||||
# ── Cross-Attention ─────────────────────────────────────────────────
|
||||
# T2V and I2V cross-attention variants
|
||||
|
||||
|
||||
class WanT2VCrossAttention(nn.Module):
|
||||
"""Text-to-video cross-attention.
|
||||
Uses vllm-omni Attention for FlashAttn backend.
|
||||
"""
|
||||
|
||||
def __init__(self, dim: int, num_heads: int, window_size=(-1, -1), qk_norm: bool = True, eps: float = 1e-6) -> None:
|
||||
super().__init__()
|
||||
if dim % num_heads != 0:
|
||||
raise ValueError(f"dim={dim} must be divisible by num_heads={num_heads}.")
|
||||
self.dim = dim
|
||||
self.num_heads = num_heads
|
||||
self.head_dim = dim // num_heads
|
||||
tp_size = get_tensor_model_parallel_world_size()
|
||||
if num_heads % tp_size != 0:
|
||||
raise ValueError(f"num_heads={num_heads} must be divisible by tp_size={tp_size}.")
|
||||
self.tp_num_heads = num_heads // tp_size
|
||||
self.tp_inner_dim = self.tp_num_heads * self.head_dim
|
||||
self.q = ColumnParallelLinear(dim, dim, bias=True, gather_output=False, return_bias=False)
|
||||
self.k = ColumnParallelLinear(dim, dim, bias=True, gather_output=False, return_bias=False)
|
||||
self.v = ColumnParallelLinear(dim, dim, bias=True, gather_output=False, return_bias=False)
|
||||
self.o = RowParallelLinear(dim, dim, bias=True, input_is_parallel=True, return_bias=False)
|
||||
self.norm_q = DistributedRMSNorm(self.tp_inner_dim, eps=eps) if qk_norm else nn.Identity()
|
||||
self.norm_k = DistributedRMSNorm(self.tp_inner_dim, eps=eps) if qk_norm else nn.Identity()
|
||||
self.attn = Attention(
|
||||
self.tp_num_heads,
|
||||
self.head_dim,
|
||||
causal=False,
|
||||
softmax_scale=self.head_dim**-0.5,
|
||||
skip_sequence_parallel=True,
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
x: torch.Tensor,
|
||||
context: torch.Tensor,
|
||||
context_lens: torch.Tensor | None = None,
|
||||
crossattn_cache: dict | None = None,
|
||||
) -> torch.Tensor:
|
||||
del context_lens
|
||||
n, d = self.tp_num_heads, self.head_dim
|
||||
q = self.norm_q(self.q(x)).unflatten(2, (n, d))
|
||||
if crossattn_cache is not None:
|
||||
if not crossattn_cache["is_init"]:
|
||||
crossattn_cache["is_init"] = True
|
||||
k = self.norm_k(self.k(context)).unflatten(2, (n, d))
|
||||
v = self.v(context).unflatten(2, (n, d))
|
||||
crossattn_cache["k"] = k
|
||||
crossattn_cache["v"] = v
|
||||
else:
|
||||
k = crossattn_cache["k"]
|
||||
v = crossattn_cache["v"]
|
||||
else:
|
||||
k = self.norm_k(self.k(context)).unflatten(2, (n, d))
|
||||
v = self.v(context).unflatten(2, (n, d))
|
||||
x = self.attn(q, k, v)
|
||||
x = x.flatten(2)
|
||||
x = self.o(x)
|
||||
return x
|
||||
|
||||
|
||||
class WanI2VCrossAttention(nn.Module):
|
||||
"""Image-to-video cross-attention (splits first 257 image tokens).
|
||||
Uses vllm-omni Attention for FlashAttn backend.
|
||||
"""
|
||||
|
||||
def __init__(self, dim: int, num_heads: int, window_size=(-1, -1), qk_norm: bool = True, eps: float = 1e-6) -> None:
|
||||
super().__init__()
|
||||
if dim % num_heads != 0:
|
||||
raise ValueError(f"dim={dim} must be divisible by num_heads={num_heads}.")
|
||||
self.dim = dim
|
||||
self.num_heads = num_heads
|
||||
self.head_dim = dim // num_heads
|
||||
tp_size = get_tensor_model_parallel_world_size()
|
||||
if num_heads % tp_size != 0:
|
||||
raise ValueError(f"num_heads={num_heads} must be divisible by tp_size={tp_size}.")
|
||||
self.tp_num_heads = num_heads // tp_size
|
||||
self.tp_inner_dim = self.tp_num_heads * self.head_dim
|
||||
self.q = ColumnParallelLinear(dim, dim, bias=True, gather_output=False, return_bias=False)
|
||||
self.k = ColumnParallelLinear(dim, dim, bias=True, gather_output=False, return_bias=False)
|
||||
self.v = ColumnParallelLinear(dim, dim, bias=True, gather_output=False, return_bias=False)
|
||||
self.o = RowParallelLinear(dim, dim, bias=True, input_is_parallel=True, return_bias=False)
|
||||
self.norm_q = DistributedRMSNorm(self.tp_inner_dim, eps=eps) if qk_norm else nn.Identity()
|
||||
self.norm_k = DistributedRMSNorm(self.tp_inner_dim, eps=eps) if qk_norm else nn.Identity()
|
||||
self.k_img = ColumnParallelLinear(dim, dim, bias=True, gather_output=False, return_bias=False)
|
||||
self.v_img = ColumnParallelLinear(dim, dim, bias=True, gather_output=False, return_bias=False)
|
||||
self.norm_k_img = DistributedRMSNorm(self.tp_inner_dim, eps=eps) if qk_norm else nn.Identity()
|
||||
self.attn = Attention(
|
||||
self.tp_num_heads,
|
||||
self.head_dim,
|
||||
causal=False,
|
||||
softmax_scale=self.head_dim**-0.5,
|
||||
skip_sequence_parallel=True,
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
x: torch.Tensor,
|
||||
context: torch.Tensor,
|
||||
context_lens: torch.Tensor | None = None,
|
||||
crossattn_cache: dict | None = None,
|
||||
) -> torch.Tensor:
|
||||
del context_lens
|
||||
context_img = context[:, :257]
|
||||
context = context[:, 257:]
|
||||
n, d = self.tp_num_heads, self.head_dim
|
||||
q = self.norm_q(self.q(x)).unflatten(2, (n, d))
|
||||
if crossattn_cache is not None:
|
||||
if not crossattn_cache["is_init"]:
|
||||
crossattn_cache["is_init"] = True
|
||||
k = self.norm_k(self.k(context)).unflatten(2, (n, d))
|
||||
v = self.v(context).unflatten(2, (n, d))
|
||||
crossattn_cache["k"] = k
|
||||
crossattn_cache["v"] = v
|
||||
else:
|
||||
k = crossattn_cache["k"]
|
||||
v = crossattn_cache["v"]
|
||||
else:
|
||||
k = self.norm_k(self.k(context)).unflatten(2, (n, d))
|
||||
v = self.v(context).unflatten(2, (n, d))
|
||||
x = self.attn(q, k, v)
|
||||
k_img = self.norm_k_img(self.k_img(context_img)).unflatten(2, (n, d))
|
||||
v_img = self.v_img(context_img).unflatten(2, (n, d))
|
||||
img_x = self.attn(q, k_img, v_img)
|
||||
x = x.flatten(2)
|
||||
img_x = img_x.flatten(2)
|
||||
x = x + img_x
|
||||
x = self.o(x)
|
||||
return x
|
||||
|
||||
|
||||
WAN_CROSSATTENTION_CLASSES = {
|
||||
"t2v_cross_attn": WanT2VCrossAttention,
|
||||
"i2v_cross_attn": WanI2VCrossAttention,
|
||||
}
|
||||
|
||||
|
||||
# ── Self-Attention with causal masking + KV cache ───────────────────
|
||||
|
||||
|
||||
class CausalWanSelfAttention(nn.Module):
|
||||
"""Causal self-attention with KV cache + action/state tokens."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
dim: int,
|
||||
num_heads: int,
|
||||
frame_seqlen: int,
|
||||
local_attn_size: int = -1,
|
||||
sink_size: int = 0,
|
||||
num_frame_per_block: int = 1,
|
||||
qk_norm: bool = True,
|
||||
eps: float = 1e-6,
|
||||
num_action_per_block: int = 32,
|
||||
num_state_per_block: int = 1,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
if dim % num_heads != 0:
|
||||
raise ValueError(f"dim={dim} must be divisible by num_heads={num_heads}.")
|
||||
self.dim = dim
|
||||
self.num_heads = num_heads
|
||||
self.head_dim = dim // num_heads
|
||||
tp_size = get_tensor_model_parallel_world_size()
|
||||
if num_heads % tp_size != 0:
|
||||
raise ValueError(f"num_heads={num_heads} must be divisible by tp_size={tp_size}.")
|
||||
self.tp_num_heads = num_heads // tp_size
|
||||
self.tp_inner_dim = self.tp_num_heads * self.head_dim
|
||||
self.local_attn_size = local_attn_size
|
||||
self.num_frame_per_block = num_frame_per_block
|
||||
self.frame_seqlen = frame_seqlen
|
||||
self.num_action_per_block = num_action_per_block
|
||||
self.num_state_per_block = num_state_per_block
|
||||
self.max_attention_size = 21 * frame_seqlen if local_attn_size == -1 else local_attn_size * frame_seqlen
|
||||
self.q = ColumnParallelLinear(dim, dim, bias=True, gather_output=False, return_bias=False)
|
||||
self.k = ColumnParallelLinear(dim, dim, bias=True, gather_output=False, return_bias=False)
|
||||
self.v = ColumnParallelLinear(dim, dim, bias=True, gather_output=False, return_bias=False)
|
||||
self.o = RowParallelLinear(dim, dim, bias=True, input_is_parallel=True, return_bias=False)
|
||||
self.norm_q = DistributedRMSNorm(self.tp_inner_dim, eps=eps) if qk_norm else nn.Identity()
|
||||
self.norm_k = DistributedRMSNorm(self.tp_inner_dim, eps=eps) if qk_norm else nn.Identity()
|
||||
self.attn = Attention(
|
||||
self.tp_num_heads,
|
||||
self.head_dim,
|
||||
causal=False,
|
||||
softmax_scale=self.head_dim**-0.5,
|
||||
skip_sequence_parallel=True,
|
||||
)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
x: torch.Tensor,
|
||||
freqs: torch.Tensor,
|
||||
freqs_action: torch.Tensor,
|
||||
freqs_state: torch.Tensor,
|
||||
action_register_length: int | None,
|
||||
kv_cache: torch.Tensor | None = None,
|
||||
current_start_frame: int = 0,
|
||||
is_tf: bool = True,
|
||||
) -> tuple[torch.Tensor, torch.Tensor | None]:
|
||||
"""Inference-only forward (KV cache path)."""
|
||||
n, d = self.tp_num_heads, self.head_dim
|
||||
|
||||
q = self.norm_q(self.q(x)).unflatten(2, (n, d))
|
||||
k = self.norm_k(self.k(x)).unflatten(2, (n, d))
|
||||
v = self.v(x).unflatten(2, (n, d))
|
||||
|
||||
updated_kv_cache: torch.Tensor | None = None
|
||||
|
||||
if kv_cache is None:
|
||||
raise RuntimeError("Inference only: kv_cache is required.")
|
||||
|
||||
action_state_index = max(0, (current_start_frame - 1) // self.num_frame_per_block)
|
||||
|
||||
roped_query = causal_rope_action_apply(
|
||||
q,
|
||||
freqs,
|
||||
freqs_action,
|
||||
freqs_state,
|
||||
action_register_length,
|
||||
self.num_action_per_block,
|
||||
self.num_state_per_block,
|
||||
action_state_index,
|
||||
).type_as(v)
|
||||
roped_key = causal_rope_action_apply(
|
||||
k,
|
||||
freqs,
|
||||
freqs_action,
|
||||
freqs_state,
|
||||
action_register_length,
|
||||
self.num_action_per_block,
|
||||
self.num_state_per_block,
|
||||
action_state_index,
|
||||
).type_as(v)
|
||||
|
||||
roped_action_query = None
|
||||
roped_action_key = None
|
||||
action_v = None
|
||||
|
||||
if action_register_length is not None:
|
||||
roped_action_query = roped_query[:, -action_register_length:]
|
||||
roped_query = roped_query[:, :-action_register_length]
|
||||
roped_action_key = roped_key[:, -action_register_length:]
|
||||
roped_key = roped_key[:, :-action_register_length]
|
||||
action_v = v[:, -action_register_length:]
|
||||
v = v[:, :-action_register_length]
|
||||
|
||||
updated_k = kv_cache[0]
|
||||
updated_v = kv_cache[1]
|
||||
new_k = torch.cat([updated_k, roped_key], dim=1)
|
||||
new_v = torch.cat([updated_v, v], dim=1)
|
||||
new_k = new_k[:, -self.max_attention_size :]
|
||||
new_v = new_v[:, -self.max_attention_size :]
|
||||
|
||||
if action_register_length is not None:
|
||||
q_cat = torch.cat([roped_query, roped_action_query], dim=1)
|
||||
k_cat = torch.cat([new_k, roped_action_key], dim=1)
|
||||
v_cat = torch.cat([new_v, action_v], dim=1)
|
||||
else:
|
||||
q_cat = roped_query
|
||||
k_cat = new_k
|
||||
v_cat = new_v
|
||||
|
||||
x = self.attn(q_cat, k_cat, v_cat)
|
||||
updated_kv_cache = torch.stack([new_k, new_v], dim=0)
|
||||
|
||||
x = x.flatten(2)
|
||||
x = self.o(x)
|
||||
return x, updated_kv_cache
|
||||
|
||||
|
||||
# ── Attention Block ─────────────────────────────────────────────────
|
||||
|
||||
|
||||
class CausalWanAttentionBlock(nn.Module):
|
||||
"""Transformer block: self-attn + cross-attn + FFN with 6-param modulation."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
cross_attn_type: str,
|
||||
dim: int,
|
||||
ffn_dim: int,
|
||||
num_heads: int,
|
||||
frame_seqlen: int,
|
||||
local_attn_size: int = -1,
|
||||
sink_size: int = 0,
|
||||
num_frame_per_block: int = 1,
|
||||
qk_norm: bool = True,
|
||||
cross_attn_norm: bool = False,
|
||||
eps: float = 1e-6,
|
||||
num_action_per_block: int = 32,
|
||||
num_state_per_block: int = 1,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.norm1 = WanLayerNorm(dim, eps)
|
||||
self.self_attn = CausalWanSelfAttention(
|
||||
dim=dim,
|
||||
num_heads=num_heads,
|
||||
frame_seqlen=frame_seqlen,
|
||||
local_attn_size=local_attn_size,
|
||||
sink_size=sink_size,
|
||||
num_frame_per_block=num_frame_per_block,
|
||||
qk_norm=qk_norm,
|
||||
eps=eps,
|
||||
num_action_per_block=num_action_per_block,
|
||||
num_state_per_block=num_state_per_block,
|
||||
)
|
||||
self.norm3 = WanLayerNorm(dim, eps, elementwise_affine=True) if cross_attn_norm else nn.Identity()
|
||||
self.cross_attn = WAN_CROSSATTENTION_CLASSES[cross_attn_type](dim, num_heads, (-1, -1), qk_norm, eps)
|
||||
self.norm2 = WanLayerNorm(dim, eps)
|
||||
self.ffn = nn.Sequential(
|
||||
ColumnParallelLinear(dim, ffn_dim, bias=True, gather_output=False, return_bias=False),
|
||||
nn.GELU(approximate="tanh"),
|
||||
RowParallelLinear(ffn_dim, dim, bias=True, input_is_parallel=True, return_bias=False),
|
||||
)
|
||||
self.modulation = nn.Parameter(torch.randn(1, 6, dim) / dim**0.5)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
x: torch.Tensor,
|
||||
e: torch.Tensor,
|
||||
freqs: torch.Tensor,
|
||||
freqs_action: torch.Tensor,
|
||||
freqs_state: torch.Tensor,
|
||||
context: torch.Tensor,
|
||||
action_register_length: int | None = None,
|
||||
kv_cache: torch.Tensor | None = None,
|
||||
crossattn_cache: dict | None = None,
|
||||
current_start_frame: int = 0,
|
||||
is_tf: bool = True,
|
||||
) -> tuple[torch.Tensor, torch.Tensor | None]:
|
||||
e = (self.modulation.unsqueeze(1) + e).chunk(6, dim=2)
|
||||
|
||||
y, updated_kv_cache = self.self_attn(
|
||||
x=(self.norm1(x) * (1 + e[1].squeeze(2)) + e[0].squeeze(2)),
|
||||
freqs=freqs,
|
||||
freqs_action=freqs_action,
|
||||
freqs_state=freqs_state,
|
||||
action_register_length=action_register_length,
|
||||
kv_cache=kv_cache,
|
||||
is_tf=is_tf,
|
||||
current_start_frame=current_start_frame,
|
||||
)
|
||||
x = x + (y * e[2].squeeze(2))
|
||||
|
||||
x = x + self.cross_attn(self.norm3(x), context, crossattn_cache=crossattn_cache)
|
||||
y = self.ffn(self.norm2(x) * (1 + e[4].squeeze(2)) + e[3].squeeze(2))
|
||||
x = x + (y * e[5].squeeze(2))
|
||||
return x, updated_kv_cache
|
||||
|
||||
|
||||
# ── Output Head ─────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class CausalHead(nn.Module):
|
||||
"""Output norm + linear with 2-param modulation.
|
||||
Runs once per step (not TP-critical), uses nn.Linear.
|
||||
"""
|
||||
|
||||
def __init__(self, dim: int, out_dim: int, patch_size: tuple, eps: float = 1e-6) -> None:
|
||||
super().__init__()
|
||||
self.dim = dim
|
||||
self.out_dim = out_dim
|
||||
self.patch_size = patch_size
|
||||
out_channels = math.prod(patch_size) * out_dim
|
||||
self.norm = WanLayerNorm(dim, eps)
|
||||
self.head = nn.Linear(dim, out_channels)
|
||||
self.modulation = nn.Parameter(torch.randn(1, 2, dim) / dim**0.5)
|
||||
|
||||
def forward(self, x: torch.Tensor, e: torch.Tensor) -> torch.Tensor:
|
||||
"""
|
||||
Args:
|
||||
x: [B, L1, C]
|
||||
e: [B, F, 1, C] (time embedding, unsqueezed)
|
||||
"""
|
||||
e = (self.modulation.unsqueeze(1) + e).chunk(2, dim=2)
|
||||
x = self.head(self.norm(x) * (1 + e[1].squeeze(2)) + e[0].squeeze(2))
|
||||
return x
|
||||
|
||||
|
||||
# ── Main Model ──────────────────────────────────────────────────────
|
||||
|
||||
|
||||
class CausalWanModel(nn.Module):
|
||||
"""Causal video diffusion transformer for DreamZero.
|
||||
|
||||
Architecture (14B): 40 layers, dim=5120, heads=40, ffn=13824
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
model_type: str = "t2v",
|
||||
patch_size: tuple[int, int, int] = (1, 2, 2),
|
||||
frame_seqlen: int = 220,
|
||||
text_len: int = 512,
|
||||
in_dim: int = 16,
|
||||
dim: int = 2048,
|
||||
ffn_dim: int = 8192,
|
||||
freq_dim: int = 256,
|
||||
text_dim: int = 4096,
|
||||
out_dim: int = 16,
|
||||
num_heads: int = 16,
|
||||
num_layers: int = 32,
|
||||
max_chunk_size: int = -1,
|
||||
sink_size: int = 0,
|
||||
qk_norm: bool = True,
|
||||
cross_attn_norm: bool = True,
|
||||
eps: float = 1e-6,
|
||||
num_frame_per_block: int = 1,
|
||||
action_dim: int = 32,
|
||||
num_registers: int = 8,
|
||||
max_state_dim: int = 64,
|
||||
max_num_embodiments: int = 32,
|
||||
hidden_size: int = 1024,
|
||||
diffusion_model_pretrained_path: str | None = None,
|
||||
num_action_per_block: int = 32,
|
||||
num_state_per_block: int = 1,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
if model_type not in ["t2v", "i2v", "ti2v"]:
|
||||
raise ValueError(f"Unsupported model_type={model_type!r}; expected one of ['t2v', 'i2v', 'ti2v'].")
|
||||
self.model_type = model_type
|
||||
self.patch_size = patch_size
|
||||
self.frame_seqlen = frame_seqlen
|
||||
self.text_len = text_len
|
||||
self.dim = dim
|
||||
self.freq_dim = freq_dim
|
||||
self.out_dim = out_dim
|
||||
self.num_heads = num_heads
|
||||
self.num_layers = num_layers
|
||||
self.local_attn_size = max_chunk_size * num_frame_per_block + 1 if max_chunk_size != -1 else -1
|
||||
self.num_frame_per_block = num_frame_per_block
|
||||
self.action_dim = action_dim
|
||||
self.num_action_per_block = num_action_per_block
|
||||
self.num_state_per_block = num_state_per_block
|
||||
|
||||
max_num_embodiments_local = 1
|
||||
self.state_encoder = CategorySpecificMLP(
|
||||
num_categories=max_num_embodiments_local,
|
||||
input_dim=max_state_dim,
|
||||
hidden_dim=hidden_size,
|
||||
output_dim=dim,
|
||||
)
|
||||
self.action_encoder = MultiEmbodimentActionEncoder(
|
||||
action_dim=action_dim,
|
||||
hidden_size=dim,
|
||||
num_embodiments=max_num_embodiments_local,
|
||||
)
|
||||
self.action_decoder = CategorySpecificMLP(
|
||||
num_categories=max_num_embodiments_local,
|
||||
input_dim=dim,
|
||||
hidden_dim=hidden_size,
|
||||
output_dim=action_dim,
|
||||
)
|
||||
|
||||
# Disable the Conv3d GEMM rewrite for patch embedding.
|
||||
self.patch_embedding = Conv3dLayer(
|
||||
in_dim,
|
||||
dim,
|
||||
kernel_size=patch_size,
|
||||
stride=patch_size,
|
||||
)
|
||||
self.patch_embedding.enable_linear = False
|
||||
self.text_embedding = nn.Sequential(
|
||||
nn.Linear(text_dim, dim),
|
||||
nn.GELU(approximate="tanh"),
|
||||
nn.Linear(dim, dim),
|
||||
)
|
||||
self.time_embedding = nn.Sequential(
|
||||
nn.Linear(freq_dim, dim),
|
||||
nn.SiLU(),
|
||||
nn.Linear(dim, dim),
|
||||
)
|
||||
self.time_projection = nn.Sequential(
|
||||
nn.SiLU(),
|
||||
nn.Linear(dim, dim * 6),
|
||||
)
|
||||
|
||||
cross_attn_type = "t2v_cross_attn" if model_type == "t2v" else "i2v_cross_attn"
|
||||
self.blocks = nn.ModuleList(
|
||||
[
|
||||
CausalWanAttentionBlock(
|
||||
cross_attn_type,
|
||||
dim,
|
||||
ffn_dim,
|
||||
num_heads,
|
||||
frame_seqlen,
|
||||
self.local_attn_size,
|
||||
sink_size,
|
||||
num_frame_per_block,
|
||||
qk_norm,
|
||||
cross_attn_norm,
|
||||
eps,
|
||||
num_action_per_block,
|
||||
num_state_per_block,
|
||||
)
|
||||
for _ in range(num_layers)
|
||||
]
|
||||
)
|
||||
|
||||
self.head = CausalHead(dim, out_dim, patch_size, eps)
|
||||
|
||||
if dim % num_heads != 0:
|
||||
raise ValueError(f"dim={dim} must be divisible by num_heads={num_heads}.")
|
||||
if (dim // num_heads) % 2 != 0:
|
||||
raise ValueError(f"dim // num_heads must be even, got {dim // num_heads}.")
|
||||
d = dim // num_heads
|
||||
self.freqs_action = rope_params(1024 * 10, d)
|
||||
self.freqs_state = rope_params(1024, d)
|
||||
self.freqs = [
|
||||
rope_params(1024, d - 4 * (d // 6)),
|
||||
rope_params(1024, 2 * (d // 6)),
|
||||
rope_params(1024, 2 * (d // 6)),
|
||||
]
|
||||
|
||||
if model_type == "i2v":
|
||||
self.img_emb = MLPProj(1280, dim)
|
||||
|
||||
self.init_weights()
|
||||
|
||||
def init_weights(self) -> None:
|
||||
"""Initialize parameters."""
|
||||
|
||||
def _init_linear_like(module: nn.Module) -> None:
|
||||
if isinstance(module, nn.Linear):
|
||||
nn.init.xavier_uniform_(module.weight)
|
||||
if module.bias is not None:
|
||||
nn.init.zeros_(module.bias)
|
||||
return
|
||||
|
||||
if isinstance(module, (ColumnParallelLinear, RowParallelLinear)):
|
||||
fan_in = module.input_size
|
||||
fan_out = module.output_size
|
||||
bound = math.sqrt(6.0 / float(fan_in + fan_out))
|
||||
nn.init.uniform_(module.weight, -bound, bound)
|
||||
if module.bias is not None:
|
||||
nn.init.zeros_(module.bias)
|
||||
|
||||
for module in self.modules():
|
||||
_init_linear_like(module)
|
||||
|
||||
nn.init.xavier_uniform_(self.patch_embedding.weight.flatten(1))
|
||||
if self.patch_embedding.bias is not None:
|
||||
fan_in = self.patch_embedding.in_channels * math.prod(self.patch_embedding.kernel_size)
|
||||
bound = 1 / math.sqrt(fan_in)
|
||||
nn.init.uniform_(self.patch_embedding.bias, -bound, bound)
|
||||
|
||||
for module in self.text_embedding.modules():
|
||||
if isinstance(module, nn.Linear):
|
||||
nn.init.normal_(module.weight, std=0.02)
|
||||
|
||||
for module in self.time_embedding.modules():
|
||||
if isinstance(module, nn.Linear):
|
||||
nn.init.normal_(module.weight, std=0.02)
|
||||
|
||||
nn.init.zeros_(self.head.head.weight)
|
||||
|
||||
def _create_freqs(self, grid_size: torch.Tensor, start_frame: int) -> torch.Tensor:
|
||||
"""Create 3D RoPE frequency tensor."""
|
||||
device = self.patch_embedding.weight.device
|
||||
if any(freq.device != device for freq in self.freqs):
|
||||
self.freqs = [freq.to(device) for freq in self.freqs]
|
||||
if self.freqs_action.device != device:
|
||||
self.freqs_action = self.freqs_action.to(device)
|
||||
if self.freqs_state.device != device:
|
||||
self.freqs_state = self.freqs_state.to(device)
|
||||
|
||||
f, h, w = grid_size.tolist()
|
||||
freqs = torch.cat(
|
||||
[
|
||||
self.freqs[0][start_frame : start_frame + f].view(f, 1, 1, -1).expand(f, h, w, -1),
|
||||
self.freqs[1][:h].view(1, h, 1, -1).expand(f, h, w, -1),
|
||||
self.freqs[2][:w].view(1, 1, w, -1).expand(f, h, w, -1),
|
||||
],
|
||||
dim=-1,
|
||||
).reshape(f * h * w, 1, -1)
|
||||
return freqs
|
||||
|
||||
def unpatchify(self, x: torch.Tensor, grid_size: torch.Tensor) -> torch.Tensor:
|
||||
"""Reconstruct video from patch embeddings."""
|
||||
B = x.shape[0]
|
||||
c = self.out_dim
|
||||
grid_size = grid_size.tolist()
|
||||
expected_seq_len = math.prod(grid_size)
|
||||
if x.shape[1] != expected_seq_len:
|
||||
raise ValueError(f"x sequence length must equal product(grid_size)={expected_seq_len}, got {x.shape[1]}.")
|
||||
x = x.view(B, *grid_size, *self.patch_size, c)
|
||||
x = torch.einsum("bfhwpqrc->bcfphqwr", x)
|
||||
x = x.reshape(B, c, *[i * j for i, j in zip(grid_size, self.patch_size)])
|
||||
return x
|
||||
|
||||
def _forward_blocks(
|
||||
self,
|
||||
x: torch.Tensor,
|
||||
seq_len: int,
|
||||
freqs: torch.Tensor,
|
||||
timestep: torch.Tensor,
|
||||
context: torch.Tensor,
|
||||
clip_feature: torch.Tensor | None,
|
||||
embodiment_id: torch.Tensor | None,
|
||||
action: torch.Tensor | None,
|
||||
timestep_action: torch.Tensor | None,
|
||||
state: torch.Tensor | None,
|
||||
kv_cache: list[torch.Tensor],
|
||||
current_start_frame: int,
|
||||
) -> tuple[torch.Tensor, torch.Tensor | None, list[torch.Tensor]]:
|
||||
x = x.flatten(start_dim=2).transpose(1, 2)
|
||||
B = x.shape[0]
|
||||
F_t = timestep.shape[1]
|
||||
|
||||
if action is not None:
|
||||
# Current DreamZero checkpoints have one local action/state adapter.
|
||||
# Global embodiment IDs are used by transforms and normalization.
|
||||
adapter_category_id = torch.zeros(B, dtype=torch.long, device=x.device)
|
||||
action_features = self.action_encoder(action, timestep_action, adapter_category_id)
|
||||
state_features = self.state_encoder(state, adapter_category_id)
|
||||
action_register = torch.cat([action_features, state_features], dim=1)
|
||||
action_length = action_features.shape[1]
|
||||
action_register_length = action_register.shape[1]
|
||||
x = torch.cat([x, action_register], dim=1)
|
||||
else:
|
||||
action_length = 0
|
||||
action_register_length = None
|
||||
|
||||
timestep = timestep.unsqueeze(-1).expand(B, F_t, seq_len // F_t).reshape(B, -1)
|
||||
if action is not None:
|
||||
if timestep_action is None or state is None:
|
||||
raise RuntimeError("timestep_action and state are required when action is provided.")
|
||||
state_features_t = self.state_encoder(state, adapter_category_id)
|
||||
stride = timestep_action.shape[1] // state_features_t.shape[1]
|
||||
timestep_state = timestep_action[:, ::stride]
|
||||
timestep = torch.cat([timestep, timestep_action, timestep_state], dim=1)
|
||||
|
||||
e = self.time_embedding(sinusoidal_embedding_1d(self.freq_dim, timestep.flatten()).type_as(x))
|
||||
e = e.unflatten(dim=0, sizes=(B, -1))
|
||||
e0 = self.time_projection(e)
|
||||
e0 = e0.unflatten(dim=2, sizes=(6, self.dim))
|
||||
|
||||
context = self.text_embedding(context)
|
||||
if clip_feature is not None:
|
||||
clip_embedding = self.img_emb(clip_feature)
|
||||
context = torch.cat([clip_embedding, context], dim=1)
|
||||
|
||||
updated_kv_caches: list[torch.Tensor] = []
|
||||
for block_index, block in enumerate(self.blocks):
|
||||
x, updated_kv_cache = block(
|
||||
x=x,
|
||||
e=e0,
|
||||
freqs=freqs,
|
||||
freqs_action=self.freqs_action,
|
||||
freqs_state=self.freqs_state,
|
||||
context=context,
|
||||
action_register_length=action_register_length,
|
||||
kv_cache=kv_cache[block_index] if kv_cache else None,
|
||||
current_start_frame=current_start_frame,
|
||||
)
|
||||
updated_kv_caches.append(updated_kv_cache)
|
||||
|
||||
if action is not None:
|
||||
action_noise_pred = x[:, seq_len : seq_len + action_length]
|
||||
action_noise_pred = self.action_decoder(action_noise_pred, adapter_category_id)
|
||||
else:
|
||||
action_noise_pred = None
|
||||
|
||||
x_video = x[:, :seq_len]
|
||||
e_video = e[:, :seq_len]
|
||||
x_video = self.head(x_video, e_video.unsqueeze(2))
|
||||
|
||||
return x_video, action_noise_pred, updated_kv_caches
|
||||
|
||||
def _forward_inference(
|
||||
self,
|
||||
x: torch.Tensor,
|
||||
timestep: torch.Tensor,
|
||||
context: torch.Tensor,
|
||||
seq_len: int,
|
||||
kv_cache: list[torch.Tensor],
|
||||
crossattn_cache: list[torch.Tensor],
|
||||
current_start_frame: int,
|
||||
y: torch.Tensor | None = None,
|
||||
clip_feature: torch.Tensor | None = None,
|
||||
action: torch.Tensor | None = None,
|
||||
timestep_action: torch.Tensor | None = None,
|
||||
state: torch.Tensor | None = None,
|
||||
embodiment_id: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor | None, list[torch.Tensor]]:
|
||||
if self.model_type == "i2v":
|
||||
if clip_feature is None or y is None:
|
||||
raise RuntimeError("clip_feature and y are required for i2v inference.")
|
||||
if context.shape[1] != self.text_len:
|
||||
raise ValueError(f"context length must be {self.text_len}, got {context.shape[1]}.")
|
||||
|
||||
if y is not None:
|
||||
x = torch.cat([x, y.to(dtype=x.dtype)], dim=1)
|
||||
|
||||
x = self.patch_embedding(x)
|
||||
grid_size = torch.tensor(x.shape[2:], dtype=torch.long)
|
||||
freqs = self._create_freqs(grid_size, current_start_frame)
|
||||
|
||||
x_video, action_noise_pred, updated_kv_caches = self._forward_blocks(
|
||||
x=x,
|
||||
seq_len=seq_len,
|
||||
freqs=freqs,
|
||||
timestep=timestep,
|
||||
context=context,
|
||||
clip_feature=clip_feature,
|
||||
embodiment_id=embodiment_id,
|
||||
action=action,
|
||||
timestep_action=timestep_action,
|
||||
state=state,
|
||||
kv_cache=kv_cache,
|
||||
current_start_frame=current_start_frame,
|
||||
)
|
||||
|
||||
x_video = x_video.clone()
|
||||
if action_noise_pred is not None:
|
||||
action_noise_pred = action_noise_pred.clone()
|
||||
|
||||
video_noise_pred = self.unpatchify(x_video, grid_size)
|
||||
return video_noise_pred, action_noise_pred, updated_kv_caches
|
||||
|
||||
def forward(self, *args: Any, **kwargs: Any):
|
||||
"""Inference only. Requires kv_cache."""
|
||||
return self._forward_inference(*args, **kwargs)
|
||||
@@ -0,0 +1,243 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""DreamZero image encoder.
|
||||
|
||||
Only the visual tower used by DreamZero I2V inference is ported here. The
|
||||
checkpoint keys under `action_head.image_encoder.*` load via simple prefix
|
||||
stripping.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import math
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
import torchvision.transforms as T
|
||||
|
||||
|
||||
class DreamZeroLayerNorm(nn.LayerNorm):
|
||||
"""LayerNorm that preserves the input dtype."""
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
return super().forward(x).type_as(x)
|
||||
|
||||
|
||||
class DreamZeroVisionSelfAttention(nn.Module):
|
||||
"""Self-attention for the vision tower."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
dim: int,
|
||||
num_heads: int,
|
||||
proj_dropout: float = 0.0,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
if dim % num_heads != 0:
|
||||
raise ValueError(f"dim={dim} must be divisible by num_heads={num_heads}.")
|
||||
self.dim = dim
|
||||
self.num_heads = num_heads
|
||||
self.head_dim = dim // num_heads
|
||||
self.proj_dropout = proj_dropout
|
||||
|
||||
self.to_qkv = nn.Linear(dim, dim * 3)
|
||||
self.proj = nn.Linear(dim, dim)
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
batch_size, seq_len, _ = x.shape
|
||||
q, k, v = self.to_qkv(x).chunk(3, dim=-1)
|
||||
q = q.view(batch_size, seq_len, self.num_heads, self.head_dim).permute(0, 2, 1, 3)
|
||||
k = k.view(batch_size, seq_len, self.num_heads, self.head_dim).permute(0, 2, 1, 3)
|
||||
v = v.view(batch_size, seq_len, self.num_heads, self.head_dim).permute(0, 2, 1, 3)
|
||||
x = F.scaled_dot_product_attention(q, k, v)
|
||||
x = x.permute(0, 2, 1, 3).reshape(batch_size, seq_len, self.dim)
|
||||
x = self.proj(x)
|
||||
return F.dropout(x, self.proj_dropout, self.training)
|
||||
|
||||
|
||||
class DreamZeroVisionAttentionBlock(nn.Module):
|
||||
"""Attention block for the vision tower."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
dim: int,
|
||||
mlp_ratio: float,
|
||||
num_heads: int,
|
||||
post_norm: bool = False,
|
||||
activation: str = "gelu",
|
||||
proj_dropout: float = 0.0,
|
||||
norm_eps: float = 1e-5,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
if activation != "gelu":
|
||||
raise ValueError(f"DreamZero image encoder uses GELU; got activation={activation!r}.")
|
||||
self.post_norm = post_norm
|
||||
hidden_dim = int(dim * mlp_ratio)
|
||||
|
||||
self.norm1 = DreamZeroLayerNorm(dim, eps=norm_eps)
|
||||
self.attn = DreamZeroVisionSelfAttention(
|
||||
dim,
|
||||
num_heads,
|
||||
proj_dropout=proj_dropout,
|
||||
)
|
||||
self.norm2 = DreamZeroLayerNorm(dim, eps=norm_eps)
|
||||
self.mlp = nn.Sequential(
|
||||
nn.Linear(dim, hidden_dim),
|
||||
nn.GELU(),
|
||||
nn.Linear(hidden_dim, dim),
|
||||
nn.Dropout(proj_dropout),
|
||||
)
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
if self.post_norm:
|
||||
x = x + self.norm1(self.attn(x))
|
||||
x = x + self.norm2(self.mlp(x))
|
||||
else:
|
||||
x = x + self.attn(self.norm1(x))
|
||||
x = x + self.mlp(self.norm2(x))
|
||||
return x
|
||||
|
||||
|
||||
class DreamZeroVisionTransformer(nn.Module):
|
||||
"""Vision transformer used by the image encoder."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
image_size: int = 224,
|
||||
patch_size: int = 14,
|
||||
dim: int = 1280,
|
||||
mlp_ratio: float = 4.0,
|
||||
out_dim: int = 1024,
|
||||
num_heads: int = 16,
|
||||
num_layers: int = 32,
|
||||
pool_type: str = "token",
|
||||
pre_norm: bool = True,
|
||||
post_norm: bool = False,
|
||||
activation: str = "gelu",
|
||||
proj_dropout: float = 0.0,
|
||||
embedding_dropout: float = 0.0,
|
||||
norm_eps: float = 1e-5,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
if pool_type != "token":
|
||||
raise ValueError(f"DreamZero image encoder only supports pool_type='token', got {pool_type!r}.")
|
||||
self.image_size = image_size
|
||||
self.patch_size = patch_size
|
||||
self.num_patches = (image_size // patch_size) ** 2
|
||||
self.dim = dim
|
||||
self.num_heads = num_heads
|
||||
self.num_layers = num_layers
|
||||
self.pool_type = pool_type
|
||||
|
||||
gain = 1.0 / math.sqrt(dim)
|
||||
self.patch_embedding = nn.Conv2d(
|
||||
3,
|
||||
dim,
|
||||
kernel_size=patch_size,
|
||||
stride=patch_size,
|
||||
bias=not pre_norm,
|
||||
)
|
||||
self.cls_embedding = nn.Parameter(gain * torch.randn(1, 1, dim))
|
||||
self.pos_embedding = nn.Parameter(
|
||||
gain * torch.randn(1, self.num_patches + 1, dim),
|
||||
)
|
||||
self.dropout = nn.Dropout(embedding_dropout)
|
||||
self.pre_norm = DreamZeroLayerNorm(dim, eps=norm_eps) if pre_norm else None
|
||||
self.transformer = nn.Sequential(
|
||||
*[
|
||||
DreamZeroVisionAttentionBlock(
|
||||
dim=dim,
|
||||
mlp_ratio=mlp_ratio,
|
||||
num_heads=num_heads,
|
||||
post_norm=post_norm,
|
||||
activation=activation,
|
||||
proj_dropout=proj_dropout,
|
||||
norm_eps=norm_eps,
|
||||
)
|
||||
for _ in range(num_layers)
|
||||
]
|
||||
)
|
||||
self.post_norm = DreamZeroLayerNorm(dim, eps=norm_eps)
|
||||
self.head = nn.Parameter(gain * torch.randn(dim, out_dim))
|
||||
|
||||
def forward(self, x: torch.Tensor, use_31_block: bool = False) -> torch.Tensor:
|
||||
batch_size = x.shape[0]
|
||||
x = self.patch_embedding(x).flatten(2).permute(0, 2, 1)
|
||||
x = torch.cat(
|
||||
[
|
||||
self.cls_embedding.expand(batch_size, -1, -1).to(dtype=x.dtype, device=x.device),
|
||||
x,
|
||||
],
|
||||
dim=1,
|
||||
)
|
||||
x = self.dropout(x + self.pos_embedding.to(dtype=x.dtype, device=x.device))
|
||||
if self.pre_norm is not None:
|
||||
x = self.pre_norm(x)
|
||||
|
||||
if use_31_block:
|
||||
return self.transformer[:-1](x)
|
||||
return self.transformer(x)
|
||||
|
||||
|
||||
class _DreamZeroCLIPContainer(nn.Module):
|
||||
"""Container matching checkpoint names under `model.visual.*`."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.log_scale = nn.Parameter(torch.ones(()))
|
||||
self.visual = DreamZeroVisionTransformer(
|
||||
image_size=224,
|
||||
patch_size=14,
|
||||
dim=1280,
|
||||
mlp_ratio=4.0,
|
||||
out_dim=1024,
|
||||
num_heads=16,
|
||||
num_layers=32,
|
||||
pool_type="token",
|
||||
pre_norm=True,
|
||||
post_norm=False,
|
||||
activation="gelu",
|
||||
proj_dropout=0.0,
|
||||
embedding_dropout=0.0,
|
||||
norm_eps=1e-5,
|
||||
)
|
||||
|
||||
|
||||
class DreamZeroImageEncoder(nn.Module):
|
||||
"""Image encoder wrapper."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.model = _DreamZeroCLIPContainer()
|
||||
# returns a composed transform whose last stage is CLIP normalization.
|
||||
self.transforms = T.Compose(
|
||||
[
|
||||
T.Normalize(
|
||||
mean=[0.48145466, 0.4578275, 0.40821073],
|
||||
std=[0.26862954, 0.26130258, 0.27577711],
|
||||
),
|
||||
]
|
||||
)
|
||||
|
||||
def encode_image(self, videos: torch.Tensor) -> torch.Tensor:
|
||||
"""Encode images for I2V conditioning."""
|
||||
size = (self.model.visual.image_size,) * 2
|
||||
videos = torch.cat(
|
||||
[
|
||||
F.interpolate(
|
||||
frame_batch,
|
||||
size=size,
|
||||
mode="bicubic",
|
||||
align_corners=False,
|
||||
)
|
||||
for frame_batch in videos
|
||||
]
|
||||
)
|
||||
videos = self.transforms.transforms[-1](videos.mul_(0.5).add_(0.5))
|
||||
|
||||
param_dtype = next(iter(self.model.visual.parameters())).dtype
|
||||
videos = videos.to(dtype=param_dtype)
|
||||
out = self.model.visual(videos, use_31_block=True)
|
||||
return out.clone()
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,161 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""DreamZero pipeline persistent state."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from collections import deque
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Number of frames per chunk for subsequent calls (first call uses 1)
|
||||
FRAMES_PER_CHUNK = 4
|
||||
|
||||
|
||||
class DreamZeroState:
|
||||
"""Pipeline persistent state across forward() calls.
|
||||
|
||||
Lifecycle:
|
||||
- Created once in DreamZeroPipeline.__init__()
|
||||
- Mutated every forward() call (frame append, KV cache grow)
|
||||
- reset() on new session / language change / local_attn_size exceeded
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.reset()
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Frame accumulation (single stitched buffer)
|
||||
# Transform outputs stitched single frame per call.
|
||||
# We accumulate here to build multi-frame video for AR inference.
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def accumulate_frames(self, stitched: np.ndarray) -> np.ndarray:
|
||||
"""Accumulate stitched frames and return multi-frame video.
|
||||
|
||||
Args:
|
||||
stitched: (H, W, C) single frame or (T, H, W, C) multi-frame,
|
||||
already stitched by transform.
|
||||
|
||||
Returns:
|
||||
(T, H, W, C) ndarray. T=1 for first call, T=FRAMES_PER_CHUNK(4) after.
|
||||
"""
|
||||
if stitched.ndim == 3:
|
||||
self.stitched_buffer.append(stitched)
|
||||
elif stitched.ndim == 4:
|
||||
self.stitched_buffer.extend(list(stitched))
|
||||
else:
|
||||
raise ValueError(f"Expected 3D or 4D stitched, got {stitched.ndim}D")
|
||||
|
||||
num_frames = 1 if self.call_count == 0 else FRAMES_PER_CHUNK
|
||||
|
||||
buffer_frames = list(self.stitched_buffer)
|
||||
if len(buffer_frames) >= num_frames:
|
||||
frames = buffer_frames[-num_frames:]
|
||||
else:
|
||||
frames = buffer_frames
|
||||
while len(frames) < num_frames:
|
||||
frames.insert(0, buffer_frames[0])
|
||||
|
||||
self.call_count += 1
|
||||
return np.stack(frames, axis=0) # (T, H, W, C)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Reset / should_reset
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def reset(self) -> None:
|
||||
"""Clear all state."""
|
||||
self.stitched_buffer: deque[np.ndarray] = deque(maxlen=FRAMES_PER_CHUNK)
|
||||
self.call_count: int = 0
|
||||
|
||||
# KV cache once robot-policy diffusion supports that integration.
|
||||
self.kv_cache: list[torch.Tensor] | None = None
|
||||
self.kv_cache_neg: list[torch.Tensor] | None = None
|
||||
self.crossattn_cache: list[dict[str, bool | torch.Tensor | None]] | None = None
|
||||
self.crossattn_cache_neg: list[dict[str, bool | torch.Tensor | None]] | None = None
|
||||
self.current_start_frame: int = 0
|
||||
|
||||
self.clip_feas: torch.Tensor | None = None
|
||||
self.ys: torch.Tensor | None = None
|
||||
self.language: torch.Tensor | None = None
|
||||
|
||||
def should_reset(self, text_tokens: torch.Tensor | None, num_video_frames: int, local_attn_size: int) -> bool:
|
||||
"""Determine if state should be reset before this forward()."""
|
||||
if self.language is None:
|
||||
logger.info("language is None, resetting")
|
||||
return True
|
||||
|
||||
if text_tokens is not None and not torch.equal(self.language, text_tokens):
|
||||
logger.info("language changed, resetting")
|
||||
return True
|
||||
|
||||
# NOTE: after accumulate_frames, num_video_frames is the accumulated T
|
||||
# (1 for first call, 4 for subsequent). Only reset on true single-frame
|
||||
# which happens when the stitched_buffer was cleared externally.
|
||||
if num_video_frames == 1 and self.call_count > 1:
|
||||
logger.info("single frame input after first call, resetting")
|
||||
return True
|
||||
|
||||
if local_attn_size != -1 and self.current_start_frame >= local_attn_size:
|
||||
logger.info(
|
||||
"current_start_frame %d >= local_attn_size %d, resetting", self.current_start_frame, local_attn_size
|
||||
)
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# KV cache management
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def create_kv_caches(
|
||||
self,
|
||||
batch_size: int,
|
||||
dtype: torch.dtype,
|
||||
device: torch.device,
|
||||
num_layers: int,
|
||||
num_heads: int,
|
||||
head_dim: int,
|
||||
) -> None:
|
||||
"""Initialize empty KV caches and cross-attention caches."""
|
||||
self.kv_cache = [
|
||||
torch.zeros(2, batch_size, 0, num_heads, head_dim, dtype=dtype, device=device) for _ in range(num_layers)
|
||||
]
|
||||
self.kv_cache_neg = [
|
||||
torch.zeros(2, batch_size, 0, num_heads, head_dim, dtype=dtype, device=device) for _ in range(num_layers)
|
||||
]
|
||||
|
||||
self.crossattn_cache = [{"is_init": False, "k": None, "v": None} for _ in range(num_layers)]
|
||||
self.crossattn_cache_neg = [{"is_init": False, "k": None, "v": None} for _ in range(num_layers)]
|
||||
|
||||
def update_kv_cache(
|
||||
self,
|
||||
layer_index: int,
|
||||
updated_kv: torch.Tensor,
|
||||
is_negative: bool = False,
|
||||
) -> None:
|
||||
"""Update a single layer's KV cache after prefill."""
|
||||
cache = self.kv_cache_neg if is_negative else self.kv_cache
|
||||
if cache is None:
|
||||
raise RuntimeError("KV caches not initialized, call create_kv_caches first.")
|
||||
cache[layer_index] = updated_kv.clone()
|
||||
|
||||
def get_kv_caches(self, is_negative: bool = False) -> list[torch.Tensor]:
|
||||
"""Get KV caches for the specified branch."""
|
||||
cache = self.kv_cache_neg if is_negative else self.kv_cache
|
||||
if cache is None:
|
||||
raise RuntimeError("KV caches not initialized.")
|
||||
return cache
|
||||
|
||||
def get_crossattn_caches(self, is_negative: bool = False) -> list[dict[str, bool | torch.Tensor | None]]:
|
||||
"""Get cross-attention caches for the specified branch."""
|
||||
cache = self.crossattn_cache_neg if is_negative else self.crossattn_cache
|
||||
if cache is None:
|
||||
raise RuntimeError("Cross-attn caches not initialized.")
|
||||
return cache
|
||||
@@ -0,0 +1,31 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from vllm_omni.diffusion.models.dreamzero.transform.base import TRANSFORMS
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
DEFAULT_EMBODIMENT = "roboarena"
|
||||
_BUILTIN_TRANSFORM_MODULES = (
|
||||
"vllm_omni.diffusion.models.dreamzero.transform.droid",
|
||||
"vllm_omni.diffusion.models.dreamzero.transform.roboarena",
|
||||
)
|
||||
|
||||
|
||||
def ensure_transforms_loaded() -> None:
|
||||
"""Import DreamZero transform modules and verify registration."""
|
||||
for module_name in _BUILTIN_TRANSFORM_MODULES:
|
||||
try:
|
||||
importlib.import_module(module_name)
|
||||
except Exception as exc:
|
||||
logger.exception("Failed to import DreamZero transform module %s", module_name)
|
||||
raise RuntimeError(f"Failed to import DreamZero transform module '{module_name}'.") from exc
|
||||
|
||||
if DEFAULT_EMBODIMENT not in TRANSFORMS:
|
||||
raise RuntimeError(f"Built-in DreamZero transform '{DEFAULT_EMBODIMENT}' is not registered after import.")
|
||||
@@ -0,0 +1,126 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""Base transform interface for DreamZero robot policy serving.
|
||||
|
||||
Transforms handle dataset-specific concerns ONLY:
|
||||
- Observation key mapping
|
||||
- Multi-view stitching (embodiment-specific layout)
|
||||
- Language template wrapping (embodiment-specific)
|
||||
- Raw state extraction (dataset-specific keys)
|
||||
- Output action slicing (to actual action_dim)
|
||||
|
||||
Model-specific concerns belong in the pipeline:
|
||||
- Tokenization (pipeline owns tokenizer)
|
||||
- State padding (pipeline knows MAX_STATE_DIM)
|
||||
- Negative prompt (pipeline owns the string)
|
||||
- Noise generation, encoding, decoding
|
||||
|
||||
Flow:
|
||||
raw obs (dataset format)
|
||||
→ DreamZeroPipeline selects transform by embodiment
|
||||
→ unified dict (stitched video, templated prompt str, raw state)
|
||||
→ tokenize, pad, encode, denoise
|
||||
→ transform_action_output()
|
||||
→ ndarray (N, action_dim)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
|
||||
|
||||
class RobotPolicyTransform:
|
||||
"""Base class for dataset-specific observation transforms.
|
||||
|
||||
Subclasses MUST define:
|
||||
IMAGE_KEY_MAP: dict — dataset obs keys → unified keys
|
||||
EMBODIMENT_NAME: str — embodiment identity (pipeline maps to numeric ID)
|
||||
ACTION_DIM: int — actual action dimensions (for output slicing)
|
||||
|
||||
Subclasses MUST override:
|
||||
_stitch_views() — multi-view → single stitched image
|
||||
_language_template() — prompt → embodiment-aware template
|
||||
_extract_raw_state() — obs → raw state ndarray
|
||||
"""
|
||||
|
||||
IMAGE_KEY_MAP: dict[str, str]
|
||||
EMBODIMENT_NAME: str
|
||||
ACTION_DIM: int
|
||||
|
||||
def transform_input(self, obs: dict) -> dict:
|
||||
"""Dataset-specific transform: key map → stitch → template → state."""
|
||||
# 1. Map image keys → unified keys
|
||||
images: dict[str, np.ndarray] = {}
|
||||
for src_key, dst_key in self.IMAGE_KEY_MAP.items():
|
||||
if src_key in obs:
|
||||
images[dst_key] = np.asarray(obs[src_key])
|
||||
|
||||
# 2. Multi-view stitching
|
||||
stitched = self._stitch_views(images)
|
||||
|
||||
# 3. Language template (string only, pipeline tokenizes)
|
||||
prompt = obs.get("prompt", "")
|
||||
templated_prompt = self._language_template(prompt)
|
||||
|
||||
# 4. Raw state extraction (pipeline pads)
|
||||
raw_state = self._extract_raw_state(obs)
|
||||
|
||||
# 5. Build unified output
|
||||
unified: dict[str, Any] = {
|
||||
"images": stitched, # ndarray (T, H_out, W_out, 3)
|
||||
"prompt": templated_prompt, # str (templated, not tokenized)
|
||||
"state": raw_state, # ndarray (state_dim,) — pipeline pads
|
||||
"embodiment_name": self.EMBODIMENT_NAME,
|
||||
}
|
||||
if "session_id" in obs:
|
||||
unified["session_id"] = obs["session_id"]
|
||||
return unified
|
||||
|
||||
def transform_action_output(self, actions: Any) -> np.ndarray:
|
||||
"""Adapt model action output to this transform's action dimensions."""
|
||||
actions = np.asarray(actions, dtype=np.float32)
|
||||
# Handle any remaining batch dims: squeeze to 2D (horizon, dim)
|
||||
while actions.ndim > 2:
|
||||
actions = actions[0]
|
||||
# Slice padded dim to actual ACTION_DIM
|
||||
if actions.ndim == 2 and actions.shape[-1] > self.ACTION_DIM:
|
||||
actions = actions[:, : self.ACTION_DIM]
|
||||
return actions
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Subclass MUST override
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _stitch_views(self, images: dict[str, np.ndarray]) -> np.ndarray:
|
||||
"""Stitch camera views into single image.
|
||||
Input: unified key → ndarray (H,W,3) or (T,H,W,3).
|
||||
Output: ndarray (T, H_out, W_out, 3).
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
def _language_template(self, prompt: str) -> str:
|
||||
"""Wrap prompt in embodiment-specific template string."""
|
||||
raise NotImplementedError
|
||||
|
||||
def _extract_raw_state(self, obs: dict) -> np.ndarray:
|
||||
"""Extract raw state vector from obs.
|
||||
Returns: ndarray (state_dim,) float64. Pipeline handles padding.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
|
||||
|
||||
# Transform registry — keyed by embodiment/dataset name
|
||||
TRANSFORMS: dict[str, RobotPolicyTransform] = {}
|
||||
|
||||
|
||||
def register_transform(name: str, transform: RobotPolicyTransform) -> None:
|
||||
TRANSFORMS[name] = transform
|
||||
|
||||
|
||||
def get_transform(name: str) -> RobotPolicyTransform:
|
||||
if name not in TRANSFORMS:
|
||||
raise KeyError(f"Unknown transform '{name}'. Available: {list(TRANSFORMS.keys())}")
|
||||
return TRANSFORMS[name]
|
||||
@@ -0,0 +1,138 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""DROID dataset transform.
|
||||
|
||||
DROID uses 1-indexed exterior cameras, 3 views total (OXE_DROID embodiment).
|
||||
Stitching layout (same as RoboArena — both are OXE_DROID):
|
||||
┌─────────────────────────┐
|
||||
│ wrist (2x width) │ ← pixel-repeat along width
|
||||
├────────────┬────────────┤
|
||||
│ left ext │ right ext │
|
||||
└────────────┴────────────┘
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
import torchvision.transforms.v2 as T
|
||||
|
||||
from vllm_omni.diffusion.models.dreamzero.transform.base import (
|
||||
RobotPolicyTransform,
|
||||
register_transform,
|
||||
)
|
||||
|
||||
|
||||
class DroidTransform(RobotPolicyTransform):
|
||||
"""Transform for DROID dataset (OXE_DROID embodiment).
|
||||
|
||||
DROID observation keys (1-indexed exterior cameras):
|
||||
observation/exterior_image_1_left → left exterior
|
||||
observation/exterior_image_2_left → right exterior
|
||||
observation/wrist_image_left → wrist
|
||||
"""
|
||||
|
||||
IMAGE_KEY_MAP = {
|
||||
"observation/exterior_image_1_left": "images/exterior_0",
|
||||
"observation/exterior_image_2_left": "images/exterior_1",
|
||||
"observation/wrist_image_left": "images/wrist",
|
||||
}
|
||||
EMBODIMENT_NAME = "oxe_droid"
|
||||
ACTION_DIM = 8 # 7 joint + 1 gripper
|
||||
_VIDEO_CROP_SCALE = 0.95
|
||||
_VIDEO_RESIZE_HW = (176, 320)
|
||||
|
||||
@classmethod
|
||||
def _preprocess_view(cls, arr: np.ndarray) -> np.ndarray:
|
||||
"""Apply per-view crop and resize before stitching."""
|
||||
frames = torch.from_numpy(arr).to(torch.float32).permute(0, 3, 1, 2) / 255.0
|
||||
crop_h = int(arr.shape[1] * cls._VIDEO_CROP_SCALE)
|
||||
crop_w = int(arr.shape[2] * cls._VIDEO_CROP_SCALE)
|
||||
frames = T.CenterCrop((crop_h, crop_w))(frames)
|
||||
frames = T.Resize(
|
||||
cls._VIDEO_RESIZE_HW,
|
||||
interpolation=T.InterpolationMode.BILINEAR,
|
||||
antialias=True,
|
||||
)(frames)
|
||||
return (frames.permute(0, 2, 3, 1) * 255.0).to(torch.uint8).cpu().numpy()
|
||||
|
||||
def _stitch_views(self, images: dict[str, np.ndarray]) -> np.ndarray:
|
||||
"""OXE_DROID 2x2 stitching: wrist top (2x wide), exteriors bottom."""
|
||||
left_ext = images.get("images/exterior_0")
|
||||
right_ext = images.get("images/exterior_1")
|
||||
wrist = images.get("images/wrist")
|
||||
|
||||
# Ensure 4D: (T, H, W, C)
|
||||
def ensure_4d(arr: np.ndarray | None) -> np.ndarray | None:
|
||||
if arr is None:
|
||||
return None
|
||||
return arr if arr.ndim == 4 else arr[np.newaxis]
|
||||
|
||||
left_ext = ensure_4d(left_ext)
|
||||
right_ext = ensure_4d(right_ext)
|
||||
wrist = ensure_4d(wrist)
|
||||
|
||||
# Determine shape from first available view.
|
||||
ref = next((v for v in [wrist, left_ext, right_ext] if v is not None), None)
|
||||
if ref is None:
|
||||
return np.zeros((1, 352, 640, 3), dtype=np.uint8)
|
||||
|
||||
# Apply per-view crop + resize before stitching.
|
||||
def maybe_preprocess(arr: np.ndarray | None) -> np.ndarray | None:
|
||||
if arr is None:
|
||||
return None
|
||||
return self._preprocess_view(arr)
|
||||
|
||||
left_ext = maybe_preprocess(left_ext)
|
||||
right_ext = maybe_preprocess(right_ext)
|
||||
wrist = maybe_preprocess(wrist)
|
||||
ref = next((v for v in [wrist, left_ext, right_ext] if v is not None), None)
|
||||
if ref is None:
|
||||
raise RuntimeError("Expected at least one DROID camera view after preprocessing.")
|
||||
t, h, w, c = ref.shape
|
||||
|
||||
out = np.zeros((t, 2 * h, 2 * w, c), dtype=ref.dtype) # (T, 2H, 2W, C)
|
||||
|
||||
# Top row: wrist repeated 2x along width.
|
||||
if wrist is not None:
|
||||
wrist_wide = np.repeat(wrist, 2, axis=2) # (T, H, 2W, C)
|
||||
out[:, :h, :] = wrist_wide
|
||||
|
||||
# Bottom row: left exterior | right exterior.
|
||||
if left_ext is not None:
|
||||
out[:, h:, :w] = left_ext
|
||||
if right_ext is not None:
|
||||
out[:, h:, w:] = right_ext
|
||||
|
||||
return out
|
||||
|
||||
def _language_template(self, prompt: str) -> str:
|
||||
"""Expand the language prompt for the OXE_DROID multi-view format."""
|
||||
prompt = (prompt or "Perform the default behavior.").strip()
|
||||
prompt_lower = prompt.lower()
|
||||
return (
|
||||
"A multi-view video shows that a robot "
|
||||
+ prompt_lower
|
||||
+ " The video is split into three views: The top view shows the "
|
||||
+ "camera view from the robot's wrist, the bottom-left view shows "
|
||||
+ "the camera view from the left exterior camera, and the "
|
||||
+ "bottom-right view shows the camera view from the right exterior "
|
||||
+ "camera. During training, one of the two bottom exterior views "
|
||||
+ "may be a black screen (dropped view). The robot "
|
||||
+ prompt_lower
|
||||
)
|
||||
|
||||
def _extract_raw_state(self, obs: dict) -> np.ndarray:
|
||||
"""OXE_DROID state: 7 joint + 1 gripper = 8 dims."""
|
||||
parts = []
|
||||
if "observation/joint_position" in obs:
|
||||
parts.append(np.asarray(obs["observation/joint_position"], dtype=np.float64).flatten())
|
||||
if "observation/gripper_position" in obs:
|
||||
parts.append(np.asarray(obs["observation/gripper_position"], dtype=np.float64).flatten())
|
||||
if parts:
|
||||
return np.concatenate(parts)
|
||||
return np.zeros(8, dtype=np.float64)
|
||||
|
||||
|
||||
register_transform("droid", DroidTransform())
|
||||
@@ -0,0 +1,41 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""RoboArena dataset transform.
|
||||
|
||||
RoboArena uses 0-indexed exterior cameras, 3 views total (OXE_DROID embodiment).
|
||||
Same stitching layout as DROID — both map to OXE_DROID in DreamZero.
|
||||
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from vllm_omni.diffusion.models.dreamzero.transform.base import (
|
||||
register_transform,
|
||||
)
|
||||
from vllm_omni.diffusion.models.dreamzero.transform.droid import (
|
||||
DroidTransform,
|
||||
)
|
||||
|
||||
|
||||
class RoboArenaTransform(DroidTransform):
|
||||
"""Transform for RoboArena dataset.
|
||||
|
||||
Same embodiment as DROID (OXE_DROID), same stitching and template.
|
||||
Only difference: 0-indexed exterior camera keys.
|
||||
|
||||
RoboArena observation keys (0-indexed):
|
||||
observation/exterior_image_0_left → left exterior
|
||||
observation/exterior_image_1_left → right exterior
|
||||
observation/wrist_image_left → wrist
|
||||
|
||||
"""
|
||||
|
||||
IMAGE_KEY_MAP = {
|
||||
"observation/exterior_image_0_left": "images/exterior_0",
|
||||
"observation/exterior_image_1_left": "images/exterior_1",
|
||||
"observation/wrist_image_left": "images/wrist",
|
||||
}
|
||||
|
||||
|
||||
register_transform("roboarena", RoboArenaTransform())
|
||||
@@ -0,0 +1,29 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""DreamZero model constants shared by the pipeline."""
|
||||
|
||||
DEFAULT_NUM_INFERENCE_STEPS = 16
|
||||
DEFAULT_CFG_SCALE = 5.0
|
||||
DEFAULT_SIGMA_SHIFT = 5.0
|
||||
DEFAULT_SEED = 1140
|
||||
|
||||
DEFAULT_NEGATIVE_PROMPT = (
|
||||
"Vibrant colors, overexposed, static, blurry details, text, subtitles, "
|
||||
"style, artwork, painting, image, still, grayscale, dull, worst quality, "
|
||||
"low quality, JPEG artifacts, ugly, mutilated, extra fingers, bad hands, "
|
||||
"bad face, deformed, disfigured, mutated limbs, fused fingers, stagnant "
|
||||
"image, cluttered background, three legs, many people in the background, "
|
||||
"walking backwards."
|
||||
)
|
||||
|
||||
DEFAULT_EMBODIMENT_NAME_TO_ID = {
|
||||
"oxe_droid": 17,
|
||||
"agibot": 26,
|
||||
"gr1_unified": 24,
|
||||
"xdof": 22,
|
||||
"yam": 32,
|
||||
"mecka_hands": 27,
|
||||
"lapa": 27,
|
||||
"dream": 31,
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import torch
|
||||
|
||||
|
||||
class DreamZeroVideoExportWorkerExtension:
|
||||
"""DreamZero worker RPCs used by offline example video export."""
|
||||
|
||||
def decode_video_latents_to_uint8(self, video_latents: torch.Tensor) -> torch.Tensor:
|
||||
if self.model_runner is None or self.model_runner.pipeline is None:
|
||||
raise RuntimeError("DreamZero pipeline is not initialized on this worker.")
|
||||
|
||||
with torch.inference_mode():
|
||||
decoded = self.model_runner.pipeline.decode_video_latents(video_latents)
|
||||
decoded = decoded.squeeze(0).permute(1, 2, 3, 0).contiguous()
|
||||
decoded = decoded.clamp(-1, 1) * 0.5 + 0.5
|
||||
decoded = (decoded * 255.0).round().to(torch.uint8).cpu()
|
||||
return decoded
|
||||
Reference in new issue
Block a user