Merge branch 'main' into feat/diffusion-prefetch-protection

This commit is contained in:
Gao Han authored and GitHub committed 2026-06-03 14:59:14 +08:00
commit f0758f5cbe
74 files changed
+9185 -350

No files matched your search

@@ -0,0 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from .avae import Cosmos3AVAEAudioTokenizer
__all__ = ["Cosmos3AVAEAudioTokenizer"]
@@ -0,0 +1,323 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Diffusers-format AVAE audio tokenizer used by Cosmos3 sound generation."""
from __future__ import annotations
import json
import math
from pathlib import Path
from typing import Any
import torch
from torch import nn
from torch.nn.utils import weight_norm
from vllm.logger import init_logger
from vllm_omni.diffusion.models.progress_bar import _is_rank_zero
logger = init_logger(__name__)
def _default_avae_config(
*,
sample_rate: int,
audio_channels: int,
io_channels: int,
hop_size: int,
) -> dict[str, Any]:
return {
"sampling_rate": sample_rate,
"hop_size": hop_size,
"dec_dim": 320,
"dec_c_mults": [1, 2, 4, 8, 16],
"dec_strides": [2, 4, 5, 6, 8],
"dec_out_channels": audio_channels,
"vocoder_input_dim": io_channels,
"normalization_type": "none",
"normalize_latents": False,
"tanh_input_scale": 1.5,
"tanh_output_scale": 3.5,
"tanh_clamp": 0.995,
}
def _config_get(config: dict[str, Any], *keys: str, default: Any = None) -> Any:
for key in keys:
value = config.get(key)
if value is not None:
return value
return default
def _load_config(
config_path: str | Path | None,
*,
sample_rate: int,
audio_channels: int,
io_channels: int,
hop_size: int,
) -> dict[str, Any]:
if config_path:
with open(config_path, encoding="utf-8") as f:
config = json.load(f)
if not isinstance(config, dict):
raise TypeError(f"Cosmos3 AVAE config must be a JSON object, got {type(config)!r}.")
return config
return _default_avae_config(
sample_rate=sample_rate,
audio_channels=audio_channels,
io_channels=io_channels,
hop_size=hop_size,
)
def _load_checkpoint(path: str | Path, map_location: torch.device | str) -> dict[str, torch.Tensor]:
path = Path(path)
if path.suffix == ".safetensors":
try:
from safetensors.torch import load_file
except ImportError as exc:
raise ImportError("Loading AVAE .safetensors checkpoints requires safetensors.") from exc
checkpoint = load_file(str(path), device=str(map_location))
else:
checkpoint = torch.load(path, map_location=map_location)
if not isinstance(checkpoint, dict):
raise TypeError(f"AVAE checkpoint must be a flat state dict, got {type(checkpoint)!r}.")
if not all(isinstance(value, torch.Tensor) for value in checkpoint.values()):
raise TypeError("AVAE checkpoint must be a flat tensor state dict.")
return checkpoint
def _validate_diffusers_state_dict(state_dict: dict[str, torch.Tensor]) -> None:
if not state_dict:
raise RuntimeError("AVAE checkpoint is empty.")
if not any(key.startswith("decoder.") for key in state_dict):
raise RuntimeError("Cosmos3 AVAE checkpoint must contain diffusers-format decoder.* keys.")
class Snake1d(nn.Module):
"""One-dimensional Snake activation matching diffusers' Oobleck layout."""
def __init__(self, hidden_dim: int, logscale: bool = True) -> None:
super().__init__()
self.alpha = nn.Parameter(torch.zeros(1, hidden_dim, 1))
self.beta = nn.Parameter(torch.zeros(1, hidden_dim, 1))
self.logscale = logscale
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
shape = hidden_states.shape
alpha = torch.exp(self.alpha) if self.logscale else self.alpha
beta = torch.exp(self.beta) if self.logscale else self.beta
hidden_states = hidden_states.reshape(shape[0], shape[1], -1)
hidden_states = hidden_states + (beta + 1e-9).reciprocal() * torch.sin(alpha * hidden_states).pow(2)
return hidden_states.reshape(shape)
class OobleckResidualUnit(nn.Module):
"""Residual unit used by the diffusers Oobleck decoder."""
def __init__(self, dimension: int = 16, dilation: int = 1) -> None:
super().__init__()
pad = ((7 - 1) * dilation) // 2
self.snake1 = Snake1d(dimension)
self.conv1 = weight_norm(nn.Conv1d(dimension, dimension, kernel_size=7, dilation=dilation, padding=pad))
self.snake2 = Snake1d(dimension)
self.conv2 = weight_norm(nn.Conv1d(dimension, dimension, kernel_size=1))
def forward(self, hidden_state: torch.Tensor) -> torch.Tensor:
output_tensor = self.conv1(self.snake1(hidden_state))
output_tensor = self.conv2(self.snake2(output_tensor))
padding = (hidden_state.shape[-1] - output_tensor.shape[-1]) // 2
if padding > 0:
hidden_state = hidden_state[..., padding:-padding]
return hidden_state + output_tensor
class OobleckDecoderBlock(nn.Module):
"""Decoder block used by the diffusers Oobleck decoder."""
def __init__(self, input_dim: int, output_dim: int, stride: int = 1, output_padding: int = 0) -> None:
super().__init__()
self.snake1 = Snake1d(input_dim)
self.conv_t1 = weight_norm(
nn.ConvTranspose1d(
input_dim,
output_dim,
kernel_size=2 * stride,
stride=stride,
padding=math.ceil(stride / 2),
output_padding=output_padding,
)
)
self.res_unit1 = OobleckResidualUnit(output_dim, dilation=1)
self.res_unit2 = OobleckResidualUnit(output_dim, dilation=3)
self.res_unit3 = OobleckResidualUnit(output_dim, dilation=9)
def forward(self, hidden_state: torch.Tensor) -> torch.Tensor:
hidden_state = self.snake1(hidden_state)
hidden_state = self.conv_t1(hidden_state)
hidden_state = self.res_unit1(hidden_state)
hidden_state = self.res_unit2(hidden_state)
return self.res_unit3(hidden_state)
class OobleckDecoder(nn.Module):
"""Diffusers-compatible Oobleck decoder for Cosmos3 AVAE latents."""
def __init__(
self,
channels: int,
input_channels: int,
audio_channels: int,
upsampling_ratios: list[int],
channel_multiples: list[int],
) -> None:
super().__init__()
strides = upsampling_ratios
channel_multiples = [1] + channel_multiples
self.conv1 = weight_norm(nn.Conv1d(input_channels, channels * channel_multiples[-1], kernel_size=7, padding=3))
block = []
for stride_index, stride in enumerate(strides):
block.append(
OobleckDecoderBlock(
input_dim=channels * channel_multiples[len(strides) - stride_index],
output_dim=channels * channel_multiples[len(strides) - stride_index - 1],
stride=stride,
output_padding=stride % 2,
)
)
self.block = nn.ModuleList(block)
self.snake1 = Snake1d(channels)
self.conv2 = weight_norm(nn.Conv1d(channels, audio_channels, kernel_size=7, padding=3, bias=False))
def forward(self, hidden_state: torch.Tensor) -> torch.Tensor:
hidden_state = self.conv1(hidden_state)
for layer in self.block:
hidden_state = layer(hidden_state)
hidden_state = self.snake1(hidden_state)
return self.conv2(hidden_state)
class Cosmos3AVAEAudioTokenizer(nn.Module):
"""Decoder-only AVAE tokenizer for Cosmos3 audio latents."""
def __init__(
self,
*,
checkpoint_path: str | Path,
config_path: str | Path | None = None,
sample_rate: int = 48000,
audio_channels: int = 2,
io_channels: int = 64,
hop_size: int = 1920,
normalize_latents: bool = False,
normalization_type: str = "none",
tanh_input_scale: float = 1.5,
tanh_output_scale: float = 3.5,
tanh_clamp: float = 0.995,
dtype: torch.dtype = torch.bfloat16,
device: torch.device | str = "cuda",
) -> None:
super().__init__()
self.dtype = dtype
self.device = torch.device(device)
config = _load_config(
config_path,
sample_rate=sample_rate,
audio_channels=audio_channels,
io_channels=io_channels,
hop_size=hop_size,
)
self.sample_rate = int(_config_get(config, "sampling_rate", "sample_rate", default=sample_rate))
self.audio_channels = int(
_config_get(
config,
"dec_out_channels",
"audio_channels",
default=2 if bool(config.get("stereo", audio_channels == 2)) else 1,
)
)
self.latent_ch = int(_config_get(config, "vocoder_input_dim", "io_channels", "latent_ch", default=io_channels))
dec_strides = [int(stride) for stride in _config_get(config, "dec_strides", default=[2, 4, 5, 6, 8])]
self.hop_size = int(
_config_get(config, "hop_size", default=math.prod(dec_strides) if dec_strides else hop_size)
)
dec_stride_product = math.prod(dec_strides)
if dec_stride_product != self.hop_size:
raise ValueError(
"Cosmos3 AVAE config dec_strides product must equal hop_size "
f"for correct latent/audio duration math: product={dec_stride_product}, hop_size={self.hop_size}."
)
normalization_type = str(_config_get(config, "normalization_type", default=normalization_type))
normalize_latents = bool(_config_get(config, "normalize_latents", default=normalize_latents))
if normalization_type == "none" and normalize_latents:
normalization_type = "tanh"
self.normalization_type = normalization_type
self.tanh_input_scale = float(_config_get(config, "tanh_input_scale", default=tanh_input_scale))
self.tanh_output_scale = float(_config_get(config, "tanh_output_scale", default=tanh_output_scale))
self.tanh_clamp = float(_config_get(config, "tanh_clamp", default=tanh_clamp))
self.decoder = OobleckDecoder(
channels=int(_config_get(config, "dec_dim", default=320)),
input_channels=self.latent_ch,
audio_channels=self.audio_channels,
upsampling_ratios=list(reversed(dec_strides)),
channel_multiples=list(_config_get(config, "dec_c_mults", default=[1, 2, 4, 8, 16])),
)
state_dict = _load_checkpoint(checkpoint_path, self.device)
_validate_diffusers_state_dict(state_dict)
# The checkpoint also contains encoder weights, which we do not support here, hence strict=False
self.load_state_dict(state_dict, strict=False)
self.eval()
for param in self.parameters():
param.requires_grad = False
self.to(device=self.device, dtype=self.dtype)
if _is_rank_zero():
logger.info("Loaded diffusers-format Cosmos3 AVAE checkpoint from %s", checkpoint_path)
@property
def temporal_compression_factor(self) -> int:
return self.hop_size
def get_latent_num_samples(self, num_audio_samples: int) -> int:
return int(num_audio_samples) // self.temporal_compression_factor
def get_audio_num_samples(self, num_latent_samples: int) -> int:
return int(num_latent_samples) * self.temporal_compression_factor
def _denormalize_latent(self, latent: torch.Tensor) -> torch.Tensor:
if self.normalization_type == "tanh":
in_dtype = latent.dtype
latent = torch.clamp(
latent.float() / self.tanh_output_scale,
-self.tanh_clamp,
self.tanh_clamp,
)
return (torch.atanh(latent) * self.tanh_input_scale).to(in_dtype)
if self.normalization_type != "none":
raise ValueError(f"Unsupported AVAE normalization_type={self.normalization_type!r}.")
return latent
@torch.no_grad()
def encode(self, audio: torch.Tensor, force_pad: bool = False) -> torch.Tensor:
del audio, force_pad
raise NotImplementedError("Cosmos3AVAEAudioTokenizer is decoder-only for diffusers-format sound_tokenizer/.")
@torch.no_grad()
def decode(self, latent: torch.Tensor) -> torch.Tensor:
in_dtype = latent.dtype
squeeze = latent.ndim == 2
if squeeze:
latent = latent.unsqueeze(0)
z = self._denormalize_latent(latent.to(self.device)).to(self.dtype)
audio = self.decoder(z).clamp(-1.0, 1.0).to(in_dtype)
return audio.squeeze(0) if squeeze else audio
@@ -15,6 +15,7 @@ runtime by:
from __future__ import annotations
import math
import os
import time
from collections.abc import Iterable
@@ -45,7 +46,7 @@ from vllm_omni.diffusion.profiler.diffusion_pipeline_profiler import DiffusionPi
from vllm_omni.diffusion.request import OmniDiffusionRequest
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
from .transformer_cosmos3 import Cosmos3VFMTransformer
from .transformer_cosmos3 import Cosmos3VFMTransformer, resolve_sound_gen
logger = init_logger(__name__)
@@ -160,6 +161,28 @@ def get_cosmos3_post_process_func(od_config: OmniDiffusionConfig):
video_processor = VideoProcessor(vae_scale_factor=16)
def _sampling_param(sampling_params, key: str, default=None):
extra = getattr(sampling_params, "extra_args", None)
if isinstance(extra, dict) and extra.get(key) is not None:
return extra[key]
value = getattr(sampling_params, key, None)
return default if value is None else value
def _resolve_output_fps(sampling_params):
fps = (
_sampling_param(sampling_params, "resolved_frame_rate")
or _sampling_param(sampling_params, "frame_rate")
or _sampling_param(sampling_params, "fps")
or 24.0
)
try:
fps_value = float(fps)
except (TypeError, ValueError):
fps_value = 24.0
if fps_value <= 0:
fps_value = 24.0
return int(fps_value) if fps_value.is_integer() else fps_value
def post_process_func(
output: torch.Tensor | dict[str, torch.Tensor] | tuple,
output_type: str = "np",
@@ -168,6 +191,8 @@ def get_cosmos3_post_process_func(od_config: OmniDiffusionConfig):
if output_type == "latent":
return output
audio = None
audio_sample_rate = None
if isinstance(output, dict):
if "image" in output and "video" in output:
raise ValueError("Cosmos3 output cannot contain both image and video payloads.")
@@ -177,10 +202,23 @@ def get_cosmos3_post_process_func(od_config: OmniDiffusionConfig):
video = output["video"]
else:
raise ValueError("Cosmos3 postprocess expected an 'image' or 'video' output payload.")
audio = output.get("audio")
audio_sample_rate = output.get("audio_sample_rate")
elif isinstance(output, tuple):
if len(output) == 3:
video, audio, audio_sample_rate = output
elif len(output) == 2:
video, audio = output
else:
raise ValueError(
"Cosmos3 postprocess expects output tensor, output dict, or (video, audio[, sample_rate]) tuple."
)
else:
video = output
if isinstance(output, dict) and "image" in output:
if audio is not None:
raise ValueError("Cosmos3 text-to-image postprocess does not support audio output.")
if video.ndim != 5 or video.shape[2] != 1:
raise ValueError(
"Cosmos3 text-to-image postprocess expects decoded output "
@@ -194,7 +232,19 @@ def get_cosmos3_post_process_func(od_config: OmniDiffusionConfig):
return video_processor.postprocess(image, output_type="pil")
if is_guardrails_enabled(od_config, sampling_params):
video = check_video_safety(video)
return video_processor.postprocess_video(video, output_type=output_type)
processed_video = video_processor.postprocess_video(video, output_type=output_type)
if audio is None:
return processed_video
if isinstance(audio, torch.Tensor):
audio = audio.detach().cpu()
result = {
"video": processed_video,
"audio": audio,
"fps": _resolve_output_fps(sampling_params),
}
if audio_sample_rate is not None:
result["audio_sample_rate"] = int(audio_sample_rate)
return result
return post_process_func
@@ -274,10 +324,22 @@ class Cosmos3OmniDiffusersPipeline(
self.vae_scale_factor_temporal = int(self.vae.config.scale_factor_temporal)
self.vae_scale_factor_spatial = getattr(self.vae.config, "scale_factor_spatial", 16)
sound_gen = resolve_sound_gen(od_config)
sound_dim = None
sound_latent_fps = None
self._sound_tokenizer = None
if sound_gen:
self._sound_tokenizer = self._get_sound_tokenizer()
sound_dim = self._sound_tokenizer.latent_ch
sound_latent_fps = self._sound_tokenizer.latent_fps
# --- Transformer (weights loaded later via weights_sources) ---
self.transformer = Cosmos3VFMTransformer(
od_config=od_config,
temporal_compression_factor=self.vae_scale_factor_temporal,
sound_gen=sound_gen,
sound_dim=sound_dim,
sound_latent_fps=sound_latent_fps,
)
# --- Scheduler ---
@@ -357,9 +419,13 @@ class Cosmos3OmniDiffusersPipeline(
"proj_in.",
"proj_out.",
"time_embedder.",
"audio_proj_in.",
"audio_proj_out.",
)
):
return f"transformer.{k}"
if k in ("audio_modality_embed", "audio_modality_embed.weight"):
return "transformer.audio_modality_embed"
# Skip lm_head
if k.startswith("lm_head."):
@@ -453,12 +519,22 @@ class Cosmos3OmniDiffusersPipeline(
loaded = loader.load_weights(_remapped_weights())
self.transformer.post_load_weights()
self.transformer.eval()
if getattr(self.transformer, "sound_gen", False):
sound_markers = ("audio_proj_in.", "audio_proj_out.", "audio_modality_embed")
missing = [marker.rstrip(".") for marker in sound_markers if not any(marker in name for name in loaded)]
if missing:
raise ValueError(
"Cosmos3 transformer config enables sound generation, but "
f"the checkpoint is missing sound weights for {missing}. "
"Use a sound-capable transformer checkpoint."
)
return loaded
def predict_noise(self, **kwargs) -> torch.Tensor | tuple[torch.Tensor, ...]:
"""Override CFGParallelMixin.predict_noise for Cosmos3.
The transformer returns the raw video noise prediction.
The transformer returns the raw prediction: video-only as a tensor,
or a tuple in video, sound order for sound generation.
"""
return self.transformer(**kwargs)
@@ -509,6 +585,39 @@ class Cosmos3OmniDiffusersPipeline(
return val
return default
@staticmethod
def _truthy(value) -> bool:
if isinstance(value, str):
return value.strip().lower() in {"1", "true", "yes", "on"}
return bool(value)
@classmethod
def _get_prompt_param(cls, prompt_data, key: str, default=None):
if not isinstance(prompt_data, dict):
return default
if prompt_data.get(key) is not None:
return prompt_data[key]
additional = prompt_data.get("additional_information")
if isinstance(additional, dict) and additional.get(key) is not None:
return additional[key]
return default
@classmethod
def _is_sound_request(cls, prompt_data, sp) -> bool:
for key in ("generate_sound", "sound_gen"):
if cls._truthy(cls._get_prompt_param(prompt_data, key, None)):
return True
if cls._truthy(cls._get_sp_param(sp, key, None)):
return True
return False
def _get_sound_tokenizer(self):
if self._sound_tokenizer is None:
from .sound_tokenizer import Cosmos3SoundTokenizer
self._sound_tokenizer = Cosmos3SoundTokenizer.from_config(self.od_config)
return self._sound_tokenizer
@staticmethod
def _is_t2i_request(req: OmniDiffusionRequest) -> bool:
"""Detect text-to-image mode from request-level prompt modalities."""
@@ -721,6 +830,47 @@ class Cosmos3OmniDiffusersPipeline(
)
return randn_tensor(shape, generator=generator, device=self.device, dtype=self.dtype)
def _prepare_sound_latents(
self,
target_audio_samples: int,
generator: torch.Generator,
) -> tuple[torch.Tensor, int]:
sound_tokenizer = self._get_sound_tokenizer()
hop_size = int(
getattr(sound_tokenizer, "hop_size", None) or getattr(sound_tokenizer, "temporal_compression_factor")
)
latent_frames = max(1, math.ceil(max(1, int(target_audio_samples)) / hop_size))
sound_dim = int(getattr(sound_tokenizer, "latent_ch", 64))
transformer_sound_dim = int(getattr(self.transformer, "sound_dim", sound_dim))
if sound_dim != transformer_sound_dim:
raise ValueError(
"Cosmos3 sound tokenizer latent channels do not match transformer "
f"sound_dim: tokenizer={sound_dim}, transformer={transformer_sound_dim}."
)
latents = randn_tensor(
(1, sound_dim, latent_frames),
generator=generator,
device=self.device,
dtype=self.dtype,
)
return latents, latent_frames
def _resolve_sound_target_samples(
self,
sp,
num_frames: int,
frame_rate: float,
) -> tuple[int, float, int]:
sound_tokenizer = self._get_sound_tokenizer()
duration = self._get_sp_param(sp, "sound_duration", None)
if duration is None:
duration = self._get_sp_param(sp, "audio_duration", None)
if duration is None:
duration = num_frames / frame_rate
duration = max(float(duration), 1.0 / max(float(frame_rate), 1.0))
sample_rate = int(getattr(sound_tokenizer, "sample_rate", 48000))
return max(1, int(round(duration * sample_rate))), duration, sample_rate
# -- VAE decode ----------------------------------------------------------
def _decode_latents(self, latents: torch.Tensor) -> torch.Tensor:
@@ -742,6 +892,19 @@ class Cosmos3OmniDiffusersPipeline(
video = self.vae.decode(latents, return_dict=False)[0]
return video
def _decode_sound_latents(
self,
sound_latents: torch.Tensor,
target_audio_samples: int,
) -> torch.Tensor:
sound_tokenizer = self._get_sound_tokenizer()
audio = sound_tokenizer.decode(sound_latents.to(self.dtype))
if audio.shape[-1] > target_audio_samples:
audio = audio[..., :target_audio_samples]
elif audio.shape[-1] < target_audio_samples:
audio = torch.nn.functional.pad(audio, (0, target_audio_samples - audio.shape[-1]))
return audio.detach().cpu()
# -- Prompt formatting + tokenization (shared by T2V and I2V) ------------
def _format_and_tokenize_prompts(
@@ -903,11 +1066,12 @@ class Cosmos3OmniDiffusersPipeline(
guidance_scale: float,
shared_kwargs: dict,
*,
sound_latents: torch.Tensor | None = None,
velocity_mask: torch.Tensor | None = None,
image_latent: torch.Tensor | None = None,
condition_latents: torch.Tensor | None = None,
guidance_interval: tuple[float, float] | None = None,
) -> torch.Tensor:
) -> torch.Tensor | tuple[torch.Tensor, ...]:
"""Denoising loop with 3-mode CFG support (parallel, sequential, none).
Cosmos3's UND pathway is text-dependent, so CFG needs separate K/V
@@ -946,21 +1110,85 @@ class Cosmos3OmniDiffusersPipeline(
lo, hi = guidance_interval
return lo <= t_scalar <= hi
# Joint scheduler step over multiple modalities. Safe for flow-matching schedulers
# because the update is linear per element; revisit this if Cosmos3 adopts a
# scheduler with cross-element dependencies (e.g. per-modality timestep).
def _pack_joint(
video_tensor: torch.Tensor,
sound_tensor: torch.Tensor | None = None,
):
batch = video_tensor.shape[0]
tensors = [video_tensor]
if sound_tensor is not None:
tensors.append(sound_tensor)
flats = [tensor.reshape(batch, -1) for tensor in tensors]
return torch.cat(flats, dim=1), [tensor.shape for tensor in tensors], [flat.shape[1] for flat in flats]
def _unpack_joint(
packed: torch.Tensor,
shapes: list[torch.Size],
numels: list[int],
) -> tuple[torch.Tensor, ...]:
outputs = []
offset = 0
for shape, numel in zip(shapes, numels, strict=True):
outputs.append(packed[:, offset : offset + numel].reshape(shape))
offset += numel
return tuple(outputs)
def _split_noise_pred(
noise_pred: torch.Tensor | tuple[torch.Tensor, ...],
) -> tuple[torch.Tensor, torch.Tensor | None]:
has_sound = sound_latents is not None
if not has_sound:
if isinstance(noise_pred, tuple):
raise ValueError("Cosmos3 video-only diffusion received tuple predictions.")
return noise_pred, None
if not isinstance(noise_pred, tuple):
raise ValueError("Cosmos3 multimodal diffusion expects transformer predictions as a tuple.")
if len(noise_pred) != 2:
raise ValueError(f"Cosmos3 sound diffusion expected 2 predictions, got {len(noise_pred)}.")
return noise_pred[0], noise_pred[1]
def _step(
noise_pred: torch.Tensor,
noise_pred: torch.Tensor | tuple[torch.Tensor, ...],
t: torch.Tensor,
latents: torch.Tensor,
) -> torch.Tensor:
if isinstance(noise_pred, tuple):
raise ValueError("Cosmos3 noise prediction must be a single tensor; got a tuple.")
sound_latents: torch.Tensor | None,
) -> torch.Tensor | tuple[torch.Tensor, ...]:
video_pred, sound_pred = _split_noise_pred(noise_pred)
if velocity_mask is not None:
noise_pred = noise_pred * velocity_mask
latents = self.scheduler.step(noise_pred, t, latents, return_dict=False)[0]
video_pred = video_pred * velocity_mask
if sound_latents is None:
latents = self.scheduler.step(video_pred, t, latents, return_dict=False)[0]
else:
packed_noise, shapes, numels = _pack_joint(video_pred, sound_pred)
packed_latents, _, _ = _pack_joint(latents, sound_latents)
packed_next = self.scheduler.step(packed_noise, t, packed_latents, return_dict=False)[0]
unpacked = _unpack_joint(packed_next, shapes, numels)
latents = unpacked[0]
if sound_latents is not None:
sound_latents = unpacked[1]
if condition_latents is not None and velocity_mask is not None:
latents = velocity_mask * latents + (1.0 - velocity_mask) * condition_latents
elif image_latent is not None:
latents[:, :, 0:1, :, :] = image_latent
return latents
outputs = [latents]
if sound_latents is not None:
outputs.append(sound_latents)
return outputs[0] if len(outputs) == 1 else tuple(outputs)
def _assign_step_out(step_out: torch.Tensor | tuple[torch.Tensor, ...]) -> None:
nonlocal latents, sound_latents
if sound_latents is None:
assert isinstance(step_out, torch.Tensor)
latents = step_out
return
if not isinstance(step_out, tuple):
raise ValueError("Cosmos3 multimodal diffusion step returned a non-tuple result.")
latents = step_out[0]
if sound_latents is not None:
sound_latents = step_out[1]
if cfg_parallel:
for t in self.progress_bar(timesteps):
@@ -978,6 +1206,7 @@ class Cosmos3OmniDiffusersPipeline(
timestep=timestep,
text_ids=cond_ids,
text_mask=cond_mask,
sound_latents=sound_latents,
**shared_kwargs,
),
negative_kwargs=dict(
@@ -985,11 +1214,12 @@ class Cosmos3OmniDiffusersPipeline(
timestep=timestep,
text_ids=uncond_ids,
text_mask=uncond_mask,
sound_latents=sound_latents,
**shared_kwargs,
),
cfg_normalize=False,
)
latents = _step(noise_pred, t, latents)
_assign_step_out(_step(noise_pred, t, latents, sound_latents))
elif do_cfg:
cond_cache: tuple = (None, None)
@@ -1007,6 +1237,7 @@ class Cosmos3OmniDiffusersPipeline(
timestep=timestep,
text_ids=cond_ids,
text_mask=cond_mask,
sound_latents=sound_latents,
**shared_kwargs,
)
if cond_cache[0] is None:
@@ -1019,6 +1250,7 @@ class Cosmos3OmniDiffusersPipeline(
timestep=timestep,
text_ids=uncond_ids,
text_mask=uncond_mask,
sound_latents=sound_latents,
**shared_kwargs,
)
if uncond_cache[0] is None:
@@ -1031,7 +1263,7 @@ class Cosmos3OmniDiffusersPipeline(
else:
noise_pred = noise_cond
latents = _step(noise_pred, t, latents)
_assign_step_out(_step(noise_pred, t, latents, sound_latents))
else:
for t in self.progress_bar(timesteps):
@@ -1041,11 +1273,15 @@ class Cosmos3OmniDiffusersPipeline(
timestep=timestep,
text_ids=cond_ids,
text_mask=cond_mask,
sound_latents=sound_latents,
**shared_kwargs,
)
latents = _step(noise_pred, t, latents)
_assign_step_out(_step(noise_pred, t, latents, sound_latents))
return latents
outputs = [latents]
if sound_latents is not None:
outputs.append(sound_latents)
return outputs[0] if len(outputs) == 1 else tuple(outputs)
# -- Forward (main generation entry point) -------------------------------
@@ -1072,6 +1308,18 @@ class Cosmos3OmniDiffusersPipeline(
sp = req.sampling_params
is_t2i = self._is_t2i_request(req)
sound_enabled = self._is_sound_request(prompt_data, sp)
if sound_enabled and is_t2i:
raise ValueError(
"Cosmos3 sound generation is supported only for video outputs in "
"this phase; text-to-image with sound is unsupported."
)
if sound_enabled and not getattr(self.transformer, "sound_gen", False):
raise ValueError(
"Cosmos3 sound generation was requested, but the transformer was "
"initialized without sound modules. Check that the checkpoint config "
"enables sound_gen or defines sound_dim and includes sound weights."
)
if negative_prompt is None:
negative_prompt = ""
@@ -1163,6 +1411,13 @@ class Cosmos3OmniDiffusersPipeline(
image_latent = None
condition_latents = None
sound_latents = None
target_audio_samples = None
sound_sample_rate = None
if sound_enabled:
target_audio_samples, _, sound_sample_rate = self._resolve_sound_target_samples(sp, num_frames, frame_rate)
sound_latents, _ = self._prepare_sound_latents(target_audio_samples, generator)
T_latent = latents.shape[2]
H_latent = latents.shape[3]
W_latent = latents.shape[4]
@@ -1184,6 +1439,7 @@ class Cosmos3OmniDiffusersPipeline(
uncond_mask=uncond_mask,
guidance_scale=guidance_scale,
shared_kwargs=shared_kwargs,
sound_latents=sound_latents,
velocity_mask=velocity_mask,
image_latent=image_latent,
condition_latents=condition_latents,
@@ -1204,7 +1460,11 @@ class Cosmos3OmniDiffusersPipeline(
samples.append(_run_diffusion(next_latents))
latents = torch.cat(samples, dim=0)
else:
latents = _run_diffusion(latents)
diffusion_output = _run_diffusion(latents)
if sound_enabled:
latents, sound_latents = diffusion_output
else:
latents = diffusion_output
# --- Decode ---
if _is_rank_zero():
@@ -1215,4 +1475,12 @@ class Cosmos3OmniDiffusersPipeline(
logger.info("Video decoded in %.2fs", time.time() - decode_start)
logger.info("Total pipeline time: %.2fs", time.time() - pipeline_start)
if sound_enabled:
if sound_latents is None or target_audio_samples is None or sound_sample_rate is None:
raise ValueError("Cosmos3 sound generation finished without sound latents.")
if _is_rank_zero():
logger.info("Decoding sound...")
audio = self._decode_sound_latents(sound_latents, target_audio_samples)
return DiffusionOutput(output={"video": video, "audio": audio, "audio_sample_rate": sound_sample_rate})
return DiffusionOutput(output={"image": video} if is_t2i else {"video": video})
@@ -0,0 +1,504 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Cosmos3 sound tokenizer integration."""
from __future__ import annotations
import json
import os
from pathlib import Path
from typing import Any
import torch
from vllm.logger import init_logger
from vllm_omni.diffusion.data import OmniDiffusionConfig
from vllm_omni.diffusion.distributed.utils import get_local_device
from vllm_omni.diffusion.models.progress_bar import _is_rank_zero
from .audio_tokenizer import Cosmos3AVAEAudioTokenizer
logger = init_logger(__name__)
DEFAULT_SOUND_SAMPLE_RATE = 48000
DEFAULT_SOUND_CHANNELS = 2
DEFAULT_SOUND_DIM = 64
DEFAULT_SOUND_HOP_SIZE = 1920
DEFAULT_SOUND_LATENT_FPS = DEFAULT_SOUND_SAMPLE_RATE / DEFAULT_SOUND_HOP_SIZE
DEFAULT_SOUND_NORMALIZE_LATENTS = False
DEFAULT_SOUND_NORMALIZATION_TYPE = "none"
DEFAULT_SOUND_TANH_INPUT_SCALE = 1.5
DEFAULT_SOUND_TANH_OUTPUT_SCALE = 3.5
DEFAULT_SOUND_TANH_CLAMP = 0.995
SOUND_TOKENIZER_COMPONENT_NAME = "sound_tokenizer"
SOUND_TOKENIZER_CHECKPOINT_NAME = "diffusion_pytorch_model.safetensors"
def _pipeline_args(od_config: OmniDiffusionConfig) -> dict[str, Any]:
return dict(getattr(od_config, "custom_pipeline_args", None) or {})
def _config_get(config: Any, key: str, default: Any = None) -> Any:
if config is None:
return default
if isinstance(config, dict):
return config.get(key, default)
if hasattr(config, "get"):
value = config.get(key, None)
return default if value is None else value
return getattr(config, key, default)
def _config_path_get(config: Any, *keys: str) -> Any:
value = config
for key in keys:
value = _config_get(value, key, None)
if value is None:
return None
return value
def _sound_tokenizer_config_from(config: Any) -> Any:
"""Return nested ``sound_tokenizer`` config from Cosmos3 config shapes."""
for path in (
("sound_tokenizer",),
("model", "config", "sound_tokenizer"),
("config", "sound_tokenizer"),
("model_config", "sound_tokenizer"),
):
value = _config_path_get(config, *path)
if value is not None:
return value
return None
def _nested_sound_tokenizer_configs(od_config: OmniDiffusionConfig | None) -> tuple[Any, ...]:
if od_config is None:
return ()
configs = []
for source in (
getattr(od_config, "model_config", None),
getattr(od_config, "tf_model_config", None),
):
config = _sound_tokenizer_config_from(source)
if config is not None:
configs.append(config)
return tuple(configs)
def _first_value_from_configs(configs: tuple[Any, ...], keys: tuple[str, ...]) -> Any:
for config in configs:
for key in keys:
value = _config_get(config, key, None)
if value is not None:
return value
return None
def _top_level_model_value(od_config: OmniDiffusionConfig | None, keys: tuple[str, ...]) -> Any:
if od_config is None:
return None
for source in (
getattr(od_config, "model_config", None),
getattr(od_config, "tf_model_config", None),
):
for key in keys:
for path in ((key,), ("model", "config", key), ("config", key), ("model_config", key)):
value = _config_path_get(source, *path)
if value is not None:
return value
return None
def _custom_arg_value(args: dict[str, Any], keys: tuple[str, ...]) -> Any:
for key in keys:
value = args.get(key)
if value is not None:
return value
return None
def _as_bool(value: Any) -> bool:
if isinstance(value, str):
return value.strip().lower() in {"1", "true", "yes", "on"}
return bool(value)
def _as_audio_channels(value: Any) -> int:
if isinstance(value, bool):
return 2 if value else 1
if isinstance(value, str) and value.strip().lower() in {
"1",
"0",
"true",
"false",
"yes",
"no",
"on",
"off",
}:
return 2 if _as_bool(value) else 1
return int(value)
def _resolve_model_file(path: Any, model_root: str | None) -> str | None:
if not path:
return None
path = str(path)
if "://" in path or os.path.isabs(path) or os.path.exists(path) or not model_root:
return path
return str(Path(model_root) / path)
def _load_sound_tokenizer_component_config(config_path: str | None) -> dict[str, Any]:
if not config_path:
return {}
with open(config_path, encoding="utf-8") as f:
config = json.load(f)
if not isinstance(config, dict):
raise TypeError(f"Cosmos3 sound tokenizer config must be a JSON object, got {type(config)!r}.")
return config
def _component_audio_channels(config: dict[str, Any]) -> Any:
if config.get("dec_out_channels") is not None:
return config["dec_out_channels"]
if config.get("audio_channels") is not None:
return config["audio_channels"]
if config.get("stereo") is not None:
return 2 if _as_bool(config["stereo"]) else 1
return None
def _component_arch_values(config: dict[str, Any]) -> dict[str, Any]:
values = {
"sample_rate": config.get("sampling_rate", config.get("sample_rate")),
"audio_channels": _component_audio_channels(config),
"io_channels": config.get("vocoder_input_dim", config.get("io_channels", config.get("latent_ch"))),
"hop_size": config.get("hop_size"),
}
return {key: value for key, value in values.items() if value is not None}
def _resolve_arch_value(
od_config: OmniDiffusionConfig,
args: dict[str, Any],
component_values: dict[str, Any],
*,
field: str,
custom_keys: tuple[str, ...],
nested_keys: tuple[str, ...],
top_level_keys: tuple[str, ...],
default: Any,
cast,
) -> Any:
custom_value = _custom_arg_value(args, custom_keys)
component_value = component_values.get(field)
if component_value is not None:
resolved = cast(component_value)
if custom_value is not None and cast(custom_value) != resolved:
raise ValueError(
"Conflicting Cosmos3 sound tokenizer architecture override for "
f"{field}: component config has {resolved!r}, custom args have {cast(custom_value)!r}."
)
return resolved
if custom_value is not None:
return cast(custom_value)
nested_value = _first_value_from_configs(_nested_sound_tokenizer_configs(od_config), nested_keys)
if nested_value is not None:
return cast(nested_value)
top_value = _top_level_model_value(od_config, top_level_keys)
if top_value is not None:
return cast(top_value)
return cast(default)
def _resolve_normalization_value(
od_config: OmniDiffusionConfig,
args: dict[str, Any],
*,
name: str,
default: Any,
aliases: tuple[str, ...] = (),
) -> Any:
keys = (f"sound_{name}", name, *aliases)
custom_value = _custom_arg_value(args, keys)
if custom_value is not None:
return custom_value
nested_value = _first_value_from_configs(_nested_sound_tokenizer_configs(od_config), (name, *aliases))
return default if nested_value is None else nested_value
def get_sound_config_value(
od_config: OmniDiffusionConfig,
name: str,
default: Any,
aliases: tuple[str, ...] = (),
) -> Any:
# Backward-compatible generic accessor. Prefer the more specific helpers
# below for Cosmos3 sound tokenizer fields so precedence stays explicit.
keys = (name, *aliases)
for config in (
_pipeline_args(od_config),
getattr(od_config, "model_config", None),
getattr(od_config, "tf_model_config", None),
):
if config is None:
continue
for key in keys:
if hasattr(config, "get"):
value = config.get(key, None)
else:
value = getattr(config, key, None)
if value is not None:
return value
return default
def get_sound_sample_rate(od_config: OmniDiffusionConfig) -> int:
args = _pipeline_args(od_config)
return _resolve_arch_value(
od_config,
args,
{},
field="sample_rate",
custom_keys=("sound_sample_rate", "sample_rate"),
nested_keys=("sample_rate", "sampling_rate"),
top_level_keys=("sound_sample_rate", "sample_rate"),
default=DEFAULT_SOUND_SAMPLE_RATE,
cast=int,
)
def get_sound_channels(od_config: OmniDiffusionConfig) -> int:
args = _pipeline_args(od_config)
return _resolve_arch_value(
od_config,
args,
{},
field="audio_channels",
custom_keys=("sound_audio_channels", "audio_channels", "stereo"),
nested_keys=("audio_channels", "dec_out_channels", "stereo"),
top_level_keys=("sound_audio_channels", "audio_channels", "stereo"),
default=DEFAULT_SOUND_CHANNELS,
cast=_as_audio_channels,
)
def get_sound_hop_size(od_config: OmniDiffusionConfig) -> int:
args = _pipeline_args(od_config)
return _resolve_arch_value(
od_config,
args,
{},
field="hop_size",
custom_keys=("sound_hop_size", "hop_size"),
nested_keys=("hop_size",),
top_level_keys=("sound_hop_size", "hop_size"),
default=DEFAULT_SOUND_HOP_SIZE,
cast=int,
)
class Cosmos3SoundTokenizer:
"""Thin adapter around the local AVAE tokenizer implementation."""
def __init__(self, tokenizer: Any) -> None:
self.tokenizer = tokenizer
self.sample_rate = int(getattr(tokenizer, "sample_rate", DEFAULT_SOUND_SAMPLE_RATE))
self.audio_channels = int(getattr(tokenizer, "audio_channels", DEFAULT_SOUND_CHANNELS))
self.latent_ch = int(getattr(tokenizer, "latent_ch", DEFAULT_SOUND_DIM))
self.hop_size = int(getattr(tokenizer, "temporal_compression_factor", DEFAULT_SOUND_HOP_SIZE))
if self.hop_size <= 0:
raise ValueError(f"Cosmos3 sound tokenizer hop_size must be positive, got {self.hop_size}.")
self.latent_fps = float(self.sample_rate) / float(self.hop_size)
@classmethod
def from_config(cls, od_config: OmniDiffusionConfig) -> Cosmos3SoundTokenizer:
args = _pipeline_args(od_config)
model_path = getattr(od_config, "model", None)
explicit_avae_path = (
args.get("sound_tokenizer_path")
or args.get("avae_path")
or args.get("cosmos3_avae_path")
or os.environ.get("COSMOS3_SOUND_TOKENIZER_PATH")
)
explicit_config_path = args.get("sound_tokenizer_config_path") or os.environ.get(
"COSMOS3_SOUND_TOKENIZER_CONFIG_PATH"
)
model_root = str(model_path) if model_path and os.path.isdir(model_path) else None
if model_root is None and model_path and not explicit_avae_path:
from huggingface_hub import snapshot_download
model_root = snapshot_download(
repo_id=str(model_path),
revision=getattr(od_config, "revision", None),
allow_patterns=[
f"{SOUND_TOKENIZER_COMPONENT_NAME}/config.json",
f"{SOUND_TOKENIZER_COMPONENT_NAME}/{SOUND_TOKENIZER_CHECKPOINT_NAME}",
],
)
if explicit_avae_path:
avae_path = _resolve_model_file(explicit_avae_path, model_root)
else:
tokenizer_dir = Path(model_root) / SOUND_TOKENIZER_COMPONENT_NAME if model_root else None
candidate = tokenizer_dir / SOUND_TOKENIZER_CHECKPOINT_NAME if tokenizer_dir else None
avae_path = str(candidate) if candidate and candidate.exists() else None
if not avae_path:
raise ValueError(
"Cosmos3 sound generation was requested, but no AVAE sound "
"tokenizer checkpoint was provided. Set "
"custom_pipeline_args['sound_tokenizer_path'] or "
"COSMOS3_SOUND_TOKENIZER_PATH, or include "
f"{SOUND_TOKENIZER_COMPONENT_NAME}/{SOUND_TOKENIZER_CHECKPOINT_NAME} under the model path."
)
config_path = _resolve_model_file(explicit_config_path, model_root)
if config_path is None and model_root:
candidate = Path(model_root) / SOUND_TOKENIZER_COMPONENT_NAME / "config.json"
config_path = str(candidate) if candidate.exists() else None
component_config = _load_sound_tokenizer_component_config(config_path)
component_values = _component_arch_values(component_config)
sample_rate = _resolve_arch_value(
od_config,
args,
component_values,
field="sample_rate",
custom_keys=("sound_sample_rate", "sample_rate"),
nested_keys=("sample_rate", "sampling_rate"),
top_level_keys=("sound_sample_rate", "sample_rate"),
default=DEFAULT_SOUND_SAMPLE_RATE,
cast=int,
)
audio_channels = _resolve_arch_value(
od_config,
args,
component_values,
field="audio_channels",
custom_keys=("sound_audio_channels", "audio_channels", "stereo"),
nested_keys=("audio_channels", "dec_out_channels", "stereo"),
top_level_keys=("sound_audio_channels", "audio_channels", "stereo"),
default=DEFAULT_SOUND_CHANNELS,
cast=_as_audio_channels,
)
sound_dim = _resolve_arch_value(
od_config,
args,
component_values,
field="io_channels",
custom_keys=("sound_dim", "io_channels", "latent_ch"),
nested_keys=("io_channels", "vocoder_input_dim", "latent_ch"),
top_level_keys=("sound_dim",),
default=DEFAULT_SOUND_DIM,
cast=int,
)
hop_size = _resolve_arch_value(
od_config,
args,
component_values,
field="hop_size",
custom_keys=("sound_hop_size", "hop_size"),
nested_keys=("hop_size",),
top_level_keys=("sound_hop_size", "hop_size"),
default=DEFAULT_SOUND_HOP_SIZE,
cast=int,
)
normalize_latents = _as_bool(
_resolve_normalization_value(
od_config,
args,
name="normalize_latents",
default=DEFAULT_SOUND_NORMALIZE_LATENTS,
)
)
normalization_type = str(
_resolve_normalization_value(
od_config,
args,
name="normalization_type",
default=DEFAULT_SOUND_NORMALIZATION_TYPE,
)
)
tanh_input_scale = float(
_resolve_normalization_value(
od_config,
args,
name="tanh_input_scale",
default=DEFAULT_SOUND_TANH_INPUT_SCALE,
)
)
tanh_output_scale = float(
_resolve_normalization_value(
od_config,
args,
name="tanh_output_scale",
default=DEFAULT_SOUND_TANH_OUTPUT_SCALE,
)
)
tanh_clamp = float(
_resolve_normalization_value(
od_config,
args,
name="tanh_clamp",
default=DEFAULT_SOUND_TANH_CLAMP,
)
)
tokenizer = Cosmos3AVAEAudioTokenizer(
checkpoint_path=str(avae_path),
config_path=config_path,
sample_rate=sample_rate,
audio_channels=audio_channels,
io_channels=sound_dim,
hop_size=hop_size,
normalize_latents=normalize_latents,
normalization_type=normalization_type,
tanh_input_scale=tanh_input_scale,
tanh_output_scale=tanh_output_scale,
tanh_clamp=tanh_clamp,
dtype=getattr(od_config, "dtype", torch.bfloat16),
device=get_local_device(),
)
if _is_rank_zero():
logger.info(
"Loaded Cosmos3 AVAE sound tokenizer from %s "
"(sr=%d, channels=%d, latent_ch=%d, hop=%d, latent_fps=%.3f)",
avae_path,
sample_rate,
audio_channels,
sound_dim,
hop_size,
float(sample_rate) / float(hop_size),
)
return cls(tokenizer)
def get_latent_num_samples(self, num_audio_samples: int) -> int:
return int(self.tokenizer.get_latent_num_samples(num_audio_samples))
def get_audio_num_samples(self, num_latent_samples: int) -> int:
return int(self.tokenizer.get_audio_num_samples(num_latent_samples))
@torch.no_grad()
def decode(self, latents: torch.Tensor) -> torch.Tensor:
"""Decode sound latents.
Args:
latents: ``[B, C, T]`` or ``[C, T]`` tensor.
Returns:
``[B, audio_channels, N]`` tensor for batched input, or
``[audio_channels, N]`` for unbatched input.
"""
squeeze = latents.ndim == 2
if squeeze:
latents = latents.unsqueeze(0)
audio = self.tokenizer.decode(latents)
audio = audio.clamp(-1.0, 1.0)
return audio.squeeze(0) if squeeze else audio
@@ -76,6 +76,66 @@ def _tf_config_get(config: Any, key: str, default: Any) -> Any:
return getattr(config, key, default)
def _nested_get(value: Any, key: str) -> Any:
if isinstance(value, dict):
if key in value:
return value[key]
for child in value.values():
found = _nested_get(child, key)
if found is not None:
return found
elif isinstance(value, list | tuple):
for child in value:
found = _nested_get(child, key)
if found is not None:
return found
return None
def _od_config_get(od_config: Any, key: str, default: Any = None) -> Any:
"""Read Cosmos3 options from runtime, model, or transformer config."""
if od_config is None:
return default
for attr in ("custom_pipeline_args", "model_config"):
source = getattr(od_config, attr, None) or {}
if isinstance(source, dict):
if key in source:
return source[key]
found = _nested_get(source, key)
if found is not None:
return found
tf_model_config = getattr(od_config, "tf_model_config", None)
if isinstance(tf_model_config, dict):
if key in tf_model_config:
return tf_model_config[key]
found = _nested_get(tf_model_config, key)
if found is not None:
return found
value = _tf_config_get(tf_model_config, key, None)
return default if value is None else value
def _as_bool(value: Any) -> bool:
if isinstance(value, str):
return value.strip().lower() in {"1", "true", "yes", "on"}
return bool(value)
def resolve_sound_gen(od_config: Any) -> bool:
"""Capability gate shared by the pipeline and transformer.
Explicit ``sound_gen`` flag wins (including an explicit False);
otherwise infer from the presence of any sound-width key in od_config.
"""
sound_gen_value = _od_config_get(od_config, "sound_gen", None)
if sound_gen_value is not None:
return _as_bool(sound_gen_value)
for key in ("sound_dim", "io_channels", "vocoder_input_dim", "latent_ch"):
if _od_config_get(od_config, key, None) is not None:
return True
return False
# ---------------------------------------------------------------------------
# Rotary Position Embeddings (mRoPE)
# ---------------------------------------------------------------------------
@@ -138,6 +198,28 @@ def compute_mrope_position_ids_vision(
return mrope_ids, next_offset
def compute_mrope_position_ids_sound(
grid_t: int,
temporal_offset: int | float,
sound_latent_fps: float,
base_fps: float = 24.0,
temporal_compression_factor_sound: int = 1,
enable_fps_modulation: bool = True,
) -> tuple[torch.Tensor, int | float]:
"""Generate mRoPE IDs for sound tokens as a (T, 1, 1) grid."""
return compute_mrope_position_ids_vision(
grid_t=grid_t,
grid_h=1,
grid_w=1,
temporal_offset=temporal_offset,
fps=sound_latent_fps,
base_fps=base_fps,
temporal_compression_factor=temporal_compression_factor_sound,
base_temporal_compression_factor=temporal_compression_factor_sound,
enable_fps_modulation=enable_fps_modulation,
)
class Qwen3VLTextRotaryEmbedding(nn.Module):
"""Multi-dimensional rotary position embedding for Qwen3-VL."""
@@ -839,6 +921,9 @@ class Cosmos3VFMTransformer(nn.Module):
od_config: OmniDiffusionConfig,
*,
temporal_compression_factor: int | None = None,
sound_gen: bool = False,
sound_dim: int | None = None,
sound_latent_fps: float | None = None,
) -> None:
super().__init__()
model_config = od_config.tf_model_config
@@ -859,9 +944,22 @@ class Cosmos3VFMTransformer(nn.Module):
self.latent_channel_size = int(_tf_config_get(model_config, "latent_channel", 48))
self.timestep_scale = float(_tf_config_get(model_config, "timestep_scale", 0.001))
self.base_fps = float(_tf_config_get(model_config, "base_fps", 24.0))
self.sound_gen = sound_gen
self.sound_dim = sound_dim
self.sound_latent_fps = sound_latent_fps
if self.sound_gen and (sound_dim is None or sound_latent_fps is None):
raise ValueError(
"Cosmos3VFMTransformer requires an explicit sound_dim and sound_latent_fps when sound_gen is True; "
"the pipeline must pass Cosmos3SoundTokenizer.latent_ch so the audio projection "
"layers are sized from the authoritative AVAE latent width."
)
if temporal_compression_factor is None:
temporal_compression_factor = _tf_config_get(model_config, "temporal_compression_factor", 4)
self.temporal_compression_factor = int(temporal_compression_factor)
self.temporal_compression_factor_sound = int(
_tf_config_get(model_config, "temporal_compression_factor_sound", 1)
)
self.enable_fps_modulation = bool(_tf_config_get(model_config, "enable_fps_modulation", True))
self.temporal_modality_margin = int(
_tf_config_get(
@@ -894,6 +992,10 @@ class Cosmos3VFMTransformer(nn.Module):
self.proj_in = nn.Linear(self.patch_latent_dim, self.hidden_size)
self.proj_out = nn.Linear(self.hidden_size, self.patch_latent_dim)
self.time_embedder = TimestepEmbedder(self.hidden_size, target_dtype=dtype)
if self.sound_gen:
self.audio_proj_in = nn.Linear(self.sound_dim, self.hidden_size)
self.audio_proj_out = nn.Linear(self.hidden_size, self.sound_dim)
self.audio_modality_embed = nn.Parameter(torch.zeros(self.hidden_size))
self.gen_layers = nn.ModuleList(
[
@@ -962,6 +1064,21 @@ class Cosmos3VFMTransformer(nn.Module):
x = x[:, :, :, :h, :w]
return x
def pack_sound(self, sound_latents: torch.Tensor) -> torch.Tensor:
"""[B, C_sound, T_sound] -> [B, T_sound, C_sound]."""
if sound_latents.ndim != 3:
raise ValueError(f"Cosmos3 sound latents must have shape [B, C, T], got {tuple(sound_latents.shape)}.")
if sound_latents.shape[1] != self.sound_dim:
raise ValueError(
f"Cosmos3 sound latent channel mismatch: expected {self.sound_dim}, got {sound_latents.shape[1]}."
)
return sound_latents.permute(0, 2, 1).contiguous()
@staticmethod
def unpack_sound(tokens: torch.Tensor) -> torch.Tensor:
"""[B, T_sound, C_sound] -> [B, C_sound, T_sound]."""
return tokens.permute(0, 2, 1).contiguous()
# -- RoPE computation ----------------------------------------------------
def _compute_rope_freqs(
@@ -973,12 +1090,14 @@ class Cosmos3VFMTransformer(nn.Module):
fps: float | None,
device: torch.device,
dtype: torch.dtype,
t_sound: int | None = None,
) -> tuple[tuple[torch.Tensor, torch.Tensor], tuple[torch.Tensor, torch.Tensor]]:
"""Compute mRoPE cos/sin for UND text and GEN media pathways."""
B = text_mask.shape[0]
S_text = text_mask.shape[1]
text_lengths = text_mask.sum(dim=1).long()
effective_fps = fps if fps is not None and t > 1 else None
sound_frames = int(t_sound or 0)
text_pos_list = []
gen_pos_list = []
@@ -996,6 +1115,21 @@ class Cosmos3VFMTransformer(nn.Module):
temporal_compression_factor=self.temporal_compression_factor,
enable_fps_modulation=self.enable_fps_modulation,
)
gen_positions = [v_pos]
if sound_frames > 0:
s_pos, _ = compute_mrope_position_ids_sound(
sound_frames,
temporal_offset=media_temporal_offset,
sound_latent_fps=self.sound_latent_fps,
base_fps=self.base_fps,
temporal_compression_factor_sound=getattr(self, "temporal_compression_factor_sound", 1),
enable_fps_modulation=self.enable_fps_modulation,
)
gen_positions.append(s_pos)
pos_dtype = gen_positions[0].dtype
for pos in gen_positions[1:]:
pos_dtype = torch.promote_types(pos_dtype, pos.dtype)
v_pos = torch.cat([pos.to(pos_dtype) for pos in gen_positions], dim=1)
if real_len < S_text:
t_pos = torch.cat(
[t_pos, torch.zeros(3, S_text - real_len, dtype=t_pos.dtype)],
@@ -1026,16 +1160,31 @@ class Cosmos3VFMTransformer(nn.Module):
def _validate_gen_sequence_parallel(
*,
s_gen: int,
s_video: int,
s_sound: int,
has_sound: bool,
ulysses_size: int,
) -> None:
if ulysses_size <= 1 or s_gen % ulysses_size == 0:
return
detail_parts = [f"video tokens {s_video}"]
if has_sound:
detail_parts.append(f"sound tokens {s_sound}")
detail = " = " + " + ".join(detail_parts) if len(detail_parts) > 1 else ""
adjust_detail = (
"Adjust the spatial resolution so that t * ceil(h/patch) * ceil(w/patch) is a multiple of ulysses_degree."
"Adjust the spatial resolution, frame count, sound duration, "
"or sound latent FPS so the combined media sequence is a "
"multiple of ulysses_degree."
if has_sound
else (
"Adjust the spatial resolution so that "
"t * ceil(h/patch) * ceil(w/patch) is a multiple "
"of ulysses_degree."
)
)
raise ValueError(
f"GEN sequence length ({s_gen} video tokens) must be divisible by "
f"GEN sequence length ({s_gen}{detail}) must be divisible by "
f"ulysses_degree ({ulysses_size}). {adjust_detail}"
)
@@ -1049,9 +1198,10 @@ class Cosmos3VFMTransformer(nn.Module):
text_mask: torch.Tensor,
video_shape: tuple[int, int, int],
fps: float | None = None,
sound_latents: torch.Tensor | None = None,
noisy_frame_mask: torch.Tensor | None = None,
**kwargs,
) -> torch.Tensor:
) -> torch.Tensor | tuple[torch.Tensor, ...]:
"""
Args:
hidden_states: [B, C, t, h, w] noisy latents
@@ -1060,13 +1210,15 @@ class Cosmos3VFMTransformer(nn.Module):
text_mask: [B, S_text] attention mask (1=real, 0=pad)
video_shape: (t, h, w) in latent space
fps: video frame rate for temporal mRoPE modulation
sound_latents: Optional [B, C_sound, T_sound] noisy sound latents.
noisy_frame_mask: Optional [B, 1, t, 1, 1] mask where 1=noisy (add
timestep embedding, predict velocity) and 0=conditioned (clean
context, skip timestep embedding). None means all frames noisy
(T2V mode).
Returns:
[B, C, t, h, w] velocity prediction.
[B, C, t, h, w] velocity prediction, or
tuple outputs in video, sound order when sound latents are provided.
"""
t, h, w = video_shape
hp, wp, _, _ = self._pad_to_patch_size(h, w)
@@ -1078,12 +1230,31 @@ class Cosmos3VFMTransformer(nn.Module):
f"Cosmos3 requires identical real text lengths within a batch "
f"(got min={min_real_len}, max={max_real_len})."
)
has_sound = sound_latents is not None
if has_sound and not self.sound_gen:
raise ValueError(
"Cosmos3 sound generation was requested, but this transformer "
"was initialized without sound modules. Check that the "
"transformer config enables sound_gen or defines sound_dim."
)
# Query Ulysses state at runtime
ulysses_size, _, _ = _get_ulysses_state()
# Patchify latents and project to hidden space
hidden_video = self.proj_in(self.patchify(hidden_states, t, h, w))
s_video = hidden_video.shape[1]
s_sound = 0
hidden_sound = None
if sound_latents is not None:
if sound_latents.shape[0] != hidden_states.shape[0]:
raise ValueError(
"Cosmos3 sound and video batch sizes must match: "
f"video={hidden_states.shape[0]}, sound={sound_latents.shape[0]}."
)
hidden_sound = self.audio_proj_in(self.pack_sound(sound_latents))
hidden_sound = hidden_sound + self.audio_modality_embed.to(hidden_sound.dtype)
s_sound = hidden_sound.shape[1]
# Timestep embedding (fp32 for precision).
# For I2V: only add to noisy tokens, not conditioned ones.
@@ -1106,7 +1277,12 @@ class Cosmos3VFMTransformer(nn.Module):
else:
hidden_video = hidden_video + time_embed.unsqueeze(1)
hidden_gen = hidden_video
if hidden_sound is not None:
hidden_sound = hidden_sound + time_embed.unsqueeze(1)
hidden_parts = [hidden_video]
if hidden_sound is not None:
hidden_parts.append(hidden_sound)
hidden_gen = torch.cat(hidden_parts, dim=1)
# Run UND pathway once and cache K/V (replicated across all ranks)
if self.cached_kv is None:
@@ -1118,6 +1294,7 @@ class Cosmos3VFMTransformer(nn.Module):
fps,
hidden_states.device,
hidden_states.dtype,
t_sound=s_sound,
)
cached_kv_full = self.language_model(text_ids, freqs_und)
self.cached_freqs_gen = freqs_gen
@@ -1133,6 +1310,9 @@ class Cosmos3VFMTransformer(nn.Module):
raise RuntimeError("Cosmos3 GEN cache was not initialized before running GEN layers.")
self._validate_gen_sequence_parallel(
s_gen=hidden_gen.shape[1],
s_video=s_video,
s_sound=s_sound,
has_sound=has_sound,
ulysses_size=ulysses_size,
)
freqs_cos, freqs_sin = self.cached_freqs_gen
@@ -1166,7 +1346,21 @@ class Cosmos3VFMTransformer(nn.Module):
# Final norm and project back to latent space
hidden_gen = self.norm_moe_gen(hidden_gen)
return self.unpatchify(self.proj_out(hidden_gen), t, h, w)
if not has_sound:
return self.unpatchify(self.proj_out(hidden_gen), t, h, w)
split_sizes = [s_video]
if has_sound:
split_sizes.append(s_sound)
split_hidden = hidden_gen.split(split_sizes, dim=1)
hidden_video = split_hidden[0]
video_pred = self.unpatchify(self.proj_out(hidden_video), t, h, w)
outputs: list[torch.Tensor] = [video_pred]
split_idx = 1
if has_sound:
hidden_sound = split_hidden[split_idx]
outputs.append(self.unpack_sound(self.audio_proj_out(hidden_sound)))
return tuple(outputs)
def post_load_weights(self) -> None:
"""Post-load processing: ensure correct dtypes."""
@@ -0,0 +1,2 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
@@ -0,0 +1,100 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Action encoder/decoder for DreamZero."""
from __future__ import annotations
import torch
import torch.nn as nn
import torch.nn.functional as F
def swish(x: torch.Tensor) -> torch.Tensor:
"""swish activation: x * sigmoid(x)"""
return x * torch.sigmoid(x)
class SinusoidalPositionalEncoding(nn.Module):
"""Sinusoidal encoding: (B, T) timesteps → (B, T, dim)"""
def __init__(self, embedding_dim: int) -> None:
super().__init__()
self.embedding_dim = embedding_dim
def forward(self, timesteps: torch.Tensor) -> torch.Tensor:
timesteps = timesteps.float()
half_dim = self.embedding_dim // 2
exponent = -torch.arange(half_dim, dtype=torch.float, device=timesteps.device) * (
torch.log(torch.tensor(10000.0)) / half_dim
)
freqs = timesteps.unsqueeze(-1) * exponent.exp()
return torch.cat([torch.sin(freqs), torch.cos(freqs)], dim=-1)
class CategorySpecificLinear(nn.Module):
"""Per-category linear: W[cat_id] @ x + b[cat_id]
Attributes:
W: (num_categories, input_dim, hidden_dim) — note: 0.02 * randn init
b: (num_categories, hidden_dim) — zero init
"""
def __init__(self, num_categories: int, input_dim: int, hidden_dim: int) -> None:
super().__init__()
self.W = nn.Parameter(0.02 * torch.randn(num_categories, input_dim, hidden_dim))
self.b = nn.Parameter(torch.zeros(num_categories, hidden_dim))
def forward(self, x: torch.Tensor, cat_ids: torch.Tensor) -> torch.Tensor:
selected_W = self.W[cat_ids]
selected_b = self.b[cat_ids]
return torch.bmm(x, selected_W) + selected_b.unsqueeze(1)
class CategorySpecificMLP(nn.Module):
"""Two-layer MLP: layer1 (relu) → layer2"""
def __init__(self, num_categories: int, input_dim: int, hidden_dim: int, output_dim: int) -> None:
super().__init__()
self.layer1 = CategorySpecificLinear(num_categories, input_dim, hidden_dim)
self.layer2 = CategorySpecificLinear(num_categories, hidden_dim, output_dim)
def forward(self, x: torch.Tensor, cat_ids: torch.Tensor) -> torch.Tensor:
hidden = F.relu(self.layer1(x, cat_ids))
return self.layer2(hidden, cat_ids)
class MultiEmbodimentActionEncoder(nn.Module):
"""Encode actions with embodiment-specific weights + sinusoidal timestep.
Flow: actions → W1 → concat(a_emb, pos_enc(timesteps)) → W2 (swish) → W3
Args:
action_dim: action vector dimension (e.g. 32)
hidden_size: output/hidden dimension (e.g. 5120 = model dim)
num_embodiments: number of robot types (e.g. 32)
"""
def __init__(self, action_dim: int, hidden_size: int, num_embodiments: int) -> None:
super().__init__()
self.hidden_size = hidden_size
self.W1 = CategorySpecificLinear(num_embodiments, action_dim, hidden_size)
self.W2 = CategorySpecificLinear(num_embodiments, 2 * hidden_size, hidden_size)
self.W3 = CategorySpecificLinear(num_embodiments, hidden_size, hidden_size)
self.pos_encoding = SinusoidalPositionalEncoding(hidden_size)
def forward(self, actions: torch.Tensor, timesteps: torch.Tensor, cat_ids: torch.Tensor) -> torch.Tensor:
"""
Args:
actions: (B, T, action_dim)
timesteps: (B, T) — per-token timestep
cat_ids: (B,) — embodiment id per sample
Returns:
(B, T, hidden_size)
"""
a_emb = self.W1(actions, cat_ids)
tau_emb = self.pos_encoding(timesteps).to(dtype=a_emb.dtype)
x = torch.cat([a_emb, tau_emb], dim=-1)
x = swish(self.W2(x, cat_ids))
x = self.W3(x, cat_ids)
return x
@@ -0,0 +1,954 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""CausalWanModel — 40-layer DiT with causal attention and KV cache.
Key differences from WanTransformer3DModel:
- Causal self-attention (new frames only see history)
- KV cache for streaming inference
- Action/state token support (appended after video tokens)
- Extended RoPE with action/state-specific frequencies
- Inference-only forward with KV cache
"""
from __future__ import annotations
import math
from typing import Any
import torch
import torch.nn as nn
from vllm.distributed import (
get_tensor_model_parallel_rank,
get_tensor_model_parallel_world_size,
get_tp_group,
)
from vllm.model_executor.layers.conv import Conv3dLayer
from vllm.model_executor.layers.linear import (
ColumnParallelLinear,
RowParallelLinear,
)
from vllm.model_executor.utils import set_weight_attrs
from vllm_omni.diffusion.attention.layer import Attention
from vllm_omni.diffusion.models.dreamzero.action_encoder import (
CategorySpecificMLP,
MultiEmbodimentActionEncoder,
)
# ── RoPE utilities ──────────────────────────────────────────────────
def sinusoidal_embedding_1d(dim: int, position: torch.Tensor) -> torch.Tensor:
"""Sinusoidal positional embedding for timesteps."""
if dim % 2 != 0:
raise ValueError(f"dim must be even, got {dim}.")
half = dim // 2
position = position.type(torch.float64)
sinusoid = torch.outer(
position,
torch.pow(10000, -torch.arange(half, dtype=position.dtype, device=position.device).div(half)),
)
x = torch.cat([torch.cos(sinusoid), torch.sin(sinusoid)], dim=1)
return x
def rope_params(max_seq_len: int, dim: int) -> torch.Tensor:
"""Precompute complex-valued RoPE frequencies (polar form).
Returns: complex tensor [max_seq_len, dim // 2]
"""
if dim % 2 != 0:
raise ValueError(f"dim must be even, got {dim}.")
freqs = torch.outer(
torch.arange(max_seq_len),
1.0 / torch.pow(10000, torch.arange(0, dim, 2).to(torch.float64).div(dim)),
)
freqs = torch.polar(torch.ones_like(freqs), freqs)
return freqs
def rope_apply(x: torch.Tensor, freqs: torch.Tensor) -> torch.Tensor:
"""Apply RoPE to x using precomputed complex freqs."""
B, seq_len, n, _ = x.shape
x = torch.view_as_complex(x.to(torch.float64).reshape(B, seq_len, n, -1, 2))
freqs = freqs.unsqueeze(0)
x = torch.view_as_real(x * freqs).flatten(3)
return x
def rope_action_apply(
x: torch.Tensor,
freqs: torch.Tensor,
freqs_action: torch.Tensor,
freqs_state: torch.Tensor,
action_register_length: int | None,
num_action_per_block: int = 32,
num_state_per_block: int = 1,
) -> torch.Tensor:
"""RoPE with action/state frequency tables for multi-step sequences."""
B, seq_len, n, _ = x.shape
x = torch.view_as_complex(x.to(torch.float64).reshape(B, seq_len, n, -1, 2))
if action_register_length is not None:
if num_action_per_block is None:
raise ValueError("num_action_per_block is required when action_register_length is set.")
if num_state_per_block is None:
raise ValueError("num_state_per_block is required when action_register_length is set.")
chunk_size = action_register_length // (num_action_per_block + num_state_per_block)
freqs_1d_action = freqs_action[: chunk_size * num_action_per_block].view(
chunk_size * num_action_per_block, 1, -1
)
freqs_1d_state = freqs_state[: chunk_size * num_state_per_block].view(chunk_size * num_state_per_block, 1, -1)
freqs = torch.cat([freqs, freqs_1d_action, freqs_1d_state], dim=0)
freqs = freqs.unsqueeze(0)
x = torch.view_as_real(x * freqs).flatten(3)
return x
def causal_rope_action_apply(
x: torch.Tensor,
freqs: torch.Tensor,
freqs_action: torch.Tensor,
freqs_state: torch.Tensor,
action_register_length: int | None,
num_action_per_block: int,
num_state_per_block: int,
action_state_index: int,
) -> torch.Tensor:
"""RoPE for single inference step (causal / KV-cache mode)."""
B, seq_len, n, _ = x.shape
x = torch.view_as_complex(x.to(torch.float64).reshape(B, seq_len, n, -1, 2))
if action_register_length is not None:
expected_length = num_action_per_block + num_state_per_block
if action_register_length != expected_length:
raise ValueError(
f"action_register_length must equal num_action_per_block + num_state_per_block "
f"({expected_length}), got {action_register_length}."
)
freqs_action = freqs_action[
action_state_index * num_action_per_block : (action_state_index + 1) * num_action_per_block
]
freqs_state = freqs_state[
action_state_index * num_state_per_block : (action_state_index + 1) * num_state_per_block
]
freqs_1d = torch.cat([freqs_action, freqs_state], dim=0).view(action_register_length, 1, -1)
freqs = torch.cat([freqs, freqs_1d], dim=0)
freqs = freqs.unsqueeze(0)
x = torch.view_as_real(x * freqs).flatten(3)
return x
# ── Normalization ───────────────────────────────────────────────────
class WanLayerNorm(nn.LayerNorm):
"""LayerNorm wrapper used by DreamZero blocks."""
def __init__(self, dim: int, eps: float = 1e-6, elementwise_affine: bool = False) -> None:
super().__init__(dim, eps=eps, elementwise_affine=elementwise_affine)
class DistributedRMSNorm(nn.Module):
"""RMSNorm that computes global RMS across tensor parallel ranks."""
def __init__(self, hidden_size: int, eps: float = 1e-6):
super().__init__()
self.eps = eps
self.weight = nn.Parameter(torch.ones(hidden_size))
set_weight_attrs(self.weight, {"weight_loader": self.weight_loader})
def weight_loader(self, param: torch.Tensor, loaded_weight: torch.Tensor) -> None:
if param.shape == loaded_weight.shape:
param.data.copy_(loaded_weight)
return
tp_size = get_tensor_model_parallel_world_size()
if loaded_weight.shape[0] % tp_size != 0:
raise ValueError(
f"Cannot shard RMSNorm weight of shape {tuple(loaded_weight.shape)} across tp_size={tp_size}."
)
shard_size = loaded_weight.shape[0] // tp_size
start_idx = get_tensor_model_parallel_rank() * shard_size
shard = loaded_weight.narrow(0, start_idx, shard_size)
if param.shape != shard.shape:
raise ValueError(f"RMSNorm shard shape mismatch: param={tuple(param.shape)}, shard={tuple(shard.shape)}.")
param.data.copy_(shard)
def forward(self, x: torch.Tensor) -> torch.Tensor:
tp_size = get_tensor_model_parallel_world_size()
x_float = x.float()
local_sum_sq = x_float.pow(2).sum(dim=-1, keepdim=True)
local_count = x.shape[-1]
if tp_size > 1:
global_sum_sq = local_sum_sq.clone()
torch.distributed.all_reduce(global_sum_sq, group=get_tp_group().device_group)
global_count = local_count * tp_size
else:
global_sum_sq = local_sum_sq
global_count = local_count
mean_sq = global_sum_sq / global_count
return (x_float * torch.rsqrt(mean_sq + self.eps)).type_as(x) * self.weight
# ── Projections ─────────────────────────────────────────────────────
class MLPProj(nn.Module):
"""CLIP feature projection for i2v.
Uses ColumnParallelLinear + RowParallelLinear (Qwen3_VisionMLP pattern).
"""
def __init__(self, in_dim: int, out_dim: int) -> None:
super().__init__()
self.norm1 = nn.LayerNorm(in_dim)
self.fc1 = ColumnParallelLinear(
in_dim,
in_dim,
bias=True,
return_bias=False,
)
self.act = nn.GELU()
self.fc2 = RowParallelLinear(
in_dim,
out_dim,
bias=True,
return_bias=False,
)
self.norm2 = nn.LayerNorm(out_dim)
def forward(self, image_embeds: torch.Tensor) -> torch.Tensor:
x = self.norm1(image_embeds)
x = self.fc1(x)
x = self.act(x)
x = self.fc2(x)
x = self.norm2(x)
return x
# ── Cross-Attention ─────────────────────────────────────────────────
# T2V and I2V cross-attention variants
class WanT2VCrossAttention(nn.Module):
"""Text-to-video cross-attention.
Uses vllm-omni Attention for FlashAttn backend.
"""
def __init__(self, dim: int, num_heads: int, window_size=(-1, -1), qk_norm: bool = True, eps: float = 1e-6) -> None:
super().__init__()
if dim % num_heads != 0:
raise ValueError(f"dim={dim} must be divisible by num_heads={num_heads}.")
self.dim = dim
self.num_heads = num_heads
self.head_dim = dim // num_heads
tp_size = get_tensor_model_parallel_world_size()
if num_heads % tp_size != 0:
raise ValueError(f"num_heads={num_heads} must be divisible by tp_size={tp_size}.")
self.tp_num_heads = num_heads // tp_size
self.tp_inner_dim = self.tp_num_heads * self.head_dim
self.q = ColumnParallelLinear(dim, dim, bias=True, gather_output=False, return_bias=False)
self.k = ColumnParallelLinear(dim, dim, bias=True, gather_output=False, return_bias=False)
self.v = ColumnParallelLinear(dim, dim, bias=True, gather_output=False, return_bias=False)
self.o = RowParallelLinear(dim, dim, bias=True, input_is_parallel=True, return_bias=False)
self.norm_q = DistributedRMSNorm(self.tp_inner_dim, eps=eps) if qk_norm else nn.Identity()
self.norm_k = DistributedRMSNorm(self.tp_inner_dim, eps=eps) if qk_norm else nn.Identity()
self.attn = Attention(
self.tp_num_heads,
self.head_dim,
causal=False,
softmax_scale=self.head_dim**-0.5,
skip_sequence_parallel=True,
)
def forward(
self,
x: torch.Tensor,
context: torch.Tensor,
context_lens: torch.Tensor | None = None,
crossattn_cache: dict | None = None,
) -> torch.Tensor:
del context_lens
n, d = self.tp_num_heads, self.head_dim
q = self.norm_q(self.q(x)).unflatten(2, (n, d))
if crossattn_cache is not None:
if not crossattn_cache["is_init"]:
crossattn_cache["is_init"] = True
k = self.norm_k(self.k(context)).unflatten(2, (n, d))
v = self.v(context).unflatten(2, (n, d))
crossattn_cache["k"] = k
crossattn_cache["v"] = v
else:
k = crossattn_cache["k"]
v = crossattn_cache["v"]
else:
k = self.norm_k(self.k(context)).unflatten(2, (n, d))
v = self.v(context).unflatten(2, (n, d))
x = self.attn(q, k, v)
x = x.flatten(2)
x = self.o(x)
return x
class WanI2VCrossAttention(nn.Module):
"""Image-to-video cross-attention (splits first 257 image tokens).
Uses vllm-omni Attention for FlashAttn backend.
"""
def __init__(self, dim: int, num_heads: int, window_size=(-1, -1), qk_norm: bool = True, eps: float = 1e-6) -> None:
super().__init__()
if dim % num_heads != 0:
raise ValueError(f"dim={dim} must be divisible by num_heads={num_heads}.")
self.dim = dim
self.num_heads = num_heads
self.head_dim = dim // num_heads
tp_size = get_tensor_model_parallel_world_size()
if num_heads % tp_size != 0:
raise ValueError(f"num_heads={num_heads} must be divisible by tp_size={tp_size}.")
self.tp_num_heads = num_heads // tp_size
self.tp_inner_dim = self.tp_num_heads * self.head_dim
self.q = ColumnParallelLinear(dim, dim, bias=True, gather_output=False, return_bias=False)
self.k = ColumnParallelLinear(dim, dim, bias=True, gather_output=False, return_bias=False)
self.v = ColumnParallelLinear(dim, dim, bias=True, gather_output=False, return_bias=False)
self.o = RowParallelLinear(dim, dim, bias=True, input_is_parallel=True, return_bias=False)
self.norm_q = DistributedRMSNorm(self.tp_inner_dim, eps=eps) if qk_norm else nn.Identity()
self.norm_k = DistributedRMSNorm(self.tp_inner_dim, eps=eps) if qk_norm else nn.Identity()
self.k_img = ColumnParallelLinear(dim, dim, bias=True, gather_output=False, return_bias=False)
self.v_img = ColumnParallelLinear(dim, dim, bias=True, gather_output=False, return_bias=False)
self.norm_k_img = DistributedRMSNorm(self.tp_inner_dim, eps=eps) if qk_norm else nn.Identity()
self.attn = Attention(
self.tp_num_heads,
self.head_dim,
causal=False,
softmax_scale=self.head_dim**-0.5,
skip_sequence_parallel=True,
)
def forward(
self,
x: torch.Tensor,
context: torch.Tensor,
context_lens: torch.Tensor | None = None,
crossattn_cache: dict | None = None,
) -> torch.Tensor:
del context_lens
context_img = context[:, :257]
context = context[:, 257:]
n, d = self.tp_num_heads, self.head_dim
q = self.norm_q(self.q(x)).unflatten(2, (n, d))
if crossattn_cache is not None:
if not crossattn_cache["is_init"]:
crossattn_cache["is_init"] = True
k = self.norm_k(self.k(context)).unflatten(2, (n, d))
v = self.v(context).unflatten(2, (n, d))
crossattn_cache["k"] = k
crossattn_cache["v"] = v
else:
k = crossattn_cache["k"]
v = crossattn_cache["v"]
else:
k = self.norm_k(self.k(context)).unflatten(2, (n, d))
v = self.v(context).unflatten(2, (n, d))
x = self.attn(q, k, v)
k_img = self.norm_k_img(self.k_img(context_img)).unflatten(2, (n, d))
v_img = self.v_img(context_img).unflatten(2, (n, d))
img_x = self.attn(q, k_img, v_img)
x = x.flatten(2)
img_x = img_x.flatten(2)
x = x + img_x
x = self.o(x)
return x
WAN_CROSSATTENTION_CLASSES = {
"t2v_cross_attn": WanT2VCrossAttention,
"i2v_cross_attn": WanI2VCrossAttention,
}
# ── Self-Attention with causal masking + KV cache ───────────────────
class CausalWanSelfAttention(nn.Module):
"""Causal self-attention with KV cache + action/state tokens."""
def __init__(
self,
dim: int,
num_heads: int,
frame_seqlen: int,
local_attn_size: int = -1,
sink_size: int = 0,
num_frame_per_block: int = 1,
qk_norm: bool = True,
eps: float = 1e-6,
num_action_per_block: int = 32,
num_state_per_block: int = 1,
) -> None:
super().__init__()
if dim % num_heads != 0:
raise ValueError(f"dim={dim} must be divisible by num_heads={num_heads}.")
self.dim = dim
self.num_heads = num_heads
self.head_dim = dim // num_heads
tp_size = get_tensor_model_parallel_world_size()
if num_heads % tp_size != 0:
raise ValueError(f"num_heads={num_heads} must be divisible by tp_size={tp_size}.")
self.tp_num_heads = num_heads // tp_size
self.tp_inner_dim = self.tp_num_heads * self.head_dim
self.local_attn_size = local_attn_size
self.num_frame_per_block = num_frame_per_block
self.frame_seqlen = frame_seqlen
self.num_action_per_block = num_action_per_block
self.num_state_per_block = num_state_per_block
self.max_attention_size = 21 * frame_seqlen if local_attn_size == -1 else local_attn_size * frame_seqlen
self.q = ColumnParallelLinear(dim, dim, bias=True, gather_output=False, return_bias=False)
self.k = ColumnParallelLinear(dim, dim, bias=True, gather_output=False, return_bias=False)
self.v = ColumnParallelLinear(dim, dim, bias=True, gather_output=False, return_bias=False)
self.o = RowParallelLinear(dim, dim, bias=True, input_is_parallel=True, return_bias=False)
self.norm_q = DistributedRMSNorm(self.tp_inner_dim, eps=eps) if qk_norm else nn.Identity()
self.norm_k = DistributedRMSNorm(self.tp_inner_dim, eps=eps) if qk_norm else nn.Identity()
self.attn = Attention(
self.tp_num_heads,
self.head_dim,
causal=False,
softmax_scale=self.head_dim**-0.5,
skip_sequence_parallel=True,
)
def forward(
self,
x: torch.Tensor,
freqs: torch.Tensor,
freqs_action: torch.Tensor,
freqs_state: torch.Tensor,
action_register_length: int | None,
kv_cache: torch.Tensor | None = None,
current_start_frame: int = 0,
is_tf: bool = True,
) -> tuple[torch.Tensor, torch.Tensor | None]:
"""Inference-only forward (KV cache path)."""
n, d = self.tp_num_heads, self.head_dim
q = self.norm_q(self.q(x)).unflatten(2, (n, d))
k = self.norm_k(self.k(x)).unflatten(2, (n, d))
v = self.v(x).unflatten(2, (n, d))
updated_kv_cache: torch.Tensor | None = None
if kv_cache is None:
raise RuntimeError("Inference only: kv_cache is required.")
action_state_index = max(0, (current_start_frame - 1) // self.num_frame_per_block)
roped_query = causal_rope_action_apply(
q,
freqs,
freqs_action,
freqs_state,
action_register_length,
self.num_action_per_block,
self.num_state_per_block,
action_state_index,
).type_as(v)
roped_key = causal_rope_action_apply(
k,
freqs,
freqs_action,
freqs_state,
action_register_length,
self.num_action_per_block,
self.num_state_per_block,
action_state_index,
).type_as(v)
roped_action_query = None
roped_action_key = None
action_v = None
if action_register_length is not None:
roped_action_query = roped_query[:, -action_register_length:]
roped_query = roped_query[:, :-action_register_length]
roped_action_key = roped_key[:, -action_register_length:]
roped_key = roped_key[:, :-action_register_length]
action_v = v[:, -action_register_length:]
v = v[:, :-action_register_length]
updated_k = kv_cache[0]
updated_v = kv_cache[1]
new_k = torch.cat([updated_k, roped_key], dim=1)
new_v = torch.cat([updated_v, v], dim=1)
new_k = new_k[:, -self.max_attention_size :]
new_v = new_v[:, -self.max_attention_size :]
if action_register_length is not None:
q_cat = torch.cat([roped_query, roped_action_query], dim=1)
k_cat = torch.cat([new_k, roped_action_key], dim=1)
v_cat = torch.cat([new_v, action_v], dim=1)
else:
q_cat = roped_query
k_cat = new_k
v_cat = new_v
x = self.attn(q_cat, k_cat, v_cat)
updated_kv_cache = torch.stack([new_k, new_v], dim=0)
x = x.flatten(2)
x = self.o(x)
return x, updated_kv_cache
# ── Attention Block ─────────────────────────────────────────────────
class CausalWanAttentionBlock(nn.Module):
"""Transformer block: self-attn + cross-attn + FFN with 6-param modulation."""
def __init__(
self,
cross_attn_type: str,
dim: int,
ffn_dim: int,
num_heads: int,
frame_seqlen: int,
local_attn_size: int = -1,
sink_size: int = 0,
num_frame_per_block: int = 1,
qk_norm: bool = True,
cross_attn_norm: bool = False,
eps: float = 1e-6,
num_action_per_block: int = 32,
num_state_per_block: int = 1,
) -> None:
super().__init__()
self.norm1 = WanLayerNorm(dim, eps)
self.self_attn = CausalWanSelfAttention(
dim=dim,
num_heads=num_heads,
frame_seqlen=frame_seqlen,
local_attn_size=local_attn_size,
sink_size=sink_size,
num_frame_per_block=num_frame_per_block,
qk_norm=qk_norm,
eps=eps,
num_action_per_block=num_action_per_block,
num_state_per_block=num_state_per_block,
)
self.norm3 = WanLayerNorm(dim, eps, elementwise_affine=True) if cross_attn_norm else nn.Identity()
self.cross_attn = WAN_CROSSATTENTION_CLASSES[cross_attn_type](dim, num_heads, (-1, -1), qk_norm, eps)
self.norm2 = WanLayerNorm(dim, eps)
self.ffn = nn.Sequential(
ColumnParallelLinear(dim, ffn_dim, bias=True, gather_output=False, return_bias=False),
nn.GELU(approximate="tanh"),
RowParallelLinear(ffn_dim, dim, bias=True, input_is_parallel=True, return_bias=False),
)
self.modulation = nn.Parameter(torch.randn(1, 6, dim) / dim**0.5)
def forward(
self,
x: torch.Tensor,
e: torch.Tensor,
freqs: torch.Tensor,
freqs_action: torch.Tensor,
freqs_state: torch.Tensor,
context: torch.Tensor,
action_register_length: int | None = None,
kv_cache: torch.Tensor | None = None,
crossattn_cache: dict | None = None,
current_start_frame: int = 0,
is_tf: bool = True,
) -> tuple[torch.Tensor, torch.Tensor | None]:
e = (self.modulation.unsqueeze(1) + e).chunk(6, dim=2)
y, updated_kv_cache = self.self_attn(
x=(self.norm1(x) * (1 + e[1].squeeze(2)) + e[0].squeeze(2)),
freqs=freqs,
freqs_action=freqs_action,
freqs_state=freqs_state,
action_register_length=action_register_length,
kv_cache=kv_cache,
is_tf=is_tf,
current_start_frame=current_start_frame,
)
x = x + (y * e[2].squeeze(2))
x = x + self.cross_attn(self.norm3(x), context, crossattn_cache=crossattn_cache)
y = self.ffn(self.norm2(x) * (1 + e[4].squeeze(2)) + e[3].squeeze(2))
x = x + (y * e[5].squeeze(2))
return x, updated_kv_cache
# ── Output Head ─────────────────────────────────────────────────────
class CausalHead(nn.Module):
"""Output norm + linear with 2-param modulation.
Runs once per step (not TP-critical), uses nn.Linear.
"""
def __init__(self, dim: int, out_dim: int, patch_size: tuple, eps: float = 1e-6) -> None:
super().__init__()
self.dim = dim
self.out_dim = out_dim
self.patch_size = patch_size
out_channels = math.prod(patch_size) * out_dim
self.norm = WanLayerNorm(dim, eps)
self.head = nn.Linear(dim, out_channels)
self.modulation = nn.Parameter(torch.randn(1, 2, dim) / dim**0.5)
def forward(self, x: torch.Tensor, e: torch.Tensor) -> torch.Tensor:
"""
Args:
x: [B, L1, C]
e: [B, F, 1, C] (time embedding, unsqueezed)
"""
e = (self.modulation.unsqueeze(1) + e).chunk(2, dim=2)
x = self.head(self.norm(x) * (1 + e[1].squeeze(2)) + e[0].squeeze(2))
return x
# ── Main Model ──────────────────────────────────────────────────────
class CausalWanModel(nn.Module):
"""Causal video diffusion transformer for DreamZero.
Architecture (14B): 40 layers, dim=5120, heads=40, ffn=13824
"""
def __init__(
self,
model_type: str = "t2v",
patch_size: tuple[int, int, int] = (1, 2, 2),
frame_seqlen: int = 220,
text_len: int = 512,
in_dim: int = 16,
dim: int = 2048,
ffn_dim: int = 8192,
freq_dim: int = 256,
text_dim: int = 4096,
out_dim: int = 16,
num_heads: int = 16,
num_layers: int = 32,
max_chunk_size: int = -1,
sink_size: int = 0,
qk_norm: bool = True,
cross_attn_norm: bool = True,
eps: float = 1e-6,
num_frame_per_block: int = 1,
action_dim: int = 32,
num_registers: int = 8,
max_state_dim: int = 64,
max_num_embodiments: int = 32,
hidden_size: int = 1024,
diffusion_model_pretrained_path: str | None = None,
num_action_per_block: int = 32,
num_state_per_block: int = 1,
) -> None:
super().__init__()
if model_type not in ["t2v", "i2v", "ti2v"]:
raise ValueError(f"Unsupported model_type={model_type!r}; expected one of ['t2v', 'i2v', 'ti2v'].")
self.model_type = model_type
self.patch_size = patch_size
self.frame_seqlen = frame_seqlen
self.text_len = text_len
self.dim = dim
self.freq_dim = freq_dim
self.out_dim = out_dim
self.num_heads = num_heads
self.num_layers = num_layers
self.local_attn_size = max_chunk_size * num_frame_per_block + 1 if max_chunk_size != -1 else -1
self.num_frame_per_block = num_frame_per_block
self.action_dim = action_dim
self.num_action_per_block = num_action_per_block
self.num_state_per_block = num_state_per_block
max_num_embodiments_local = 1
self.state_encoder = CategorySpecificMLP(
num_categories=max_num_embodiments_local,
input_dim=max_state_dim,
hidden_dim=hidden_size,
output_dim=dim,
)
self.action_encoder = MultiEmbodimentActionEncoder(
action_dim=action_dim,
hidden_size=dim,
num_embodiments=max_num_embodiments_local,
)
self.action_decoder = CategorySpecificMLP(
num_categories=max_num_embodiments_local,
input_dim=dim,
hidden_dim=hidden_size,
output_dim=action_dim,
)
# Disable the Conv3d GEMM rewrite for patch embedding.
self.patch_embedding = Conv3dLayer(
in_dim,
dim,
kernel_size=patch_size,
stride=patch_size,
)
self.patch_embedding.enable_linear = False
self.text_embedding = nn.Sequential(
nn.Linear(text_dim, dim),
nn.GELU(approximate="tanh"),
nn.Linear(dim, dim),
)
self.time_embedding = nn.Sequential(
nn.Linear(freq_dim, dim),
nn.SiLU(),
nn.Linear(dim, dim),
)
self.time_projection = nn.Sequential(
nn.SiLU(),
nn.Linear(dim, dim * 6),
)
cross_attn_type = "t2v_cross_attn" if model_type == "t2v" else "i2v_cross_attn"
self.blocks = nn.ModuleList(
[
CausalWanAttentionBlock(
cross_attn_type,
dim,
ffn_dim,
num_heads,
frame_seqlen,
self.local_attn_size,
sink_size,
num_frame_per_block,
qk_norm,
cross_attn_norm,
eps,
num_action_per_block,
num_state_per_block,
)
for _ in range(num_layers)
]
)
self.head = CausalHead(dim, out_dim, patch_size, eps)
if dim % num_heads != 0:
raise ValueError(f"dim={dim} must be divisible by num_heads={num_heads}.")
if (dim // num_heads) % 2 != 0:
raise ValueError(f"dim // num_heads must be even, got {dim // num_heads}.")
d = dim // num_heads
self.freqs_action = rope_params(1024 * 10, d)
self.freqs_state = rope_params(1024, d)
self.freqs = [
rope_params(1024, d - 4 * (d // 6)),
rope_params(1024, 2 * (d // 6)),
rope_params(1024, 2 * (d // 6)),
]
if model_type == "i2v":
self.img_emb = MLPProj(1280, dim)
self.init_weights()
def init_weights(self) -> None:
"""Initialize parameters."""
def _init_linear_like(module: nn.Module) -> None:
if isinstance(module, nn.Linear):
nn.init.xavier_uniform_(module.weight)
if module.bias is not None:
nn.init.zeros_(module.bias)
return
if isinstance(module, (ColumnParallelLinear, RowParallelLinear)):
fan_in = module.input_size
fan_out = module.output_size
bound = math.sqrt(6.0 / float(fan_in + fan_out))
nn.init.uniform_(module.weight, -bound, bound)
if module.bias is not None:
nn.init.zeros_(module.bias)
for module in self.modules():
_init_linear_like(module)
nn.init.xavier_uniform_(self.patch_embedding.weight.flatten(1))
if self.patch_embedding.bias is not None:
fan_in = self.patch_embedding.in_channels * math.prod(self.patch_embedding.kernel_size)
bound = 1 / math.sqrt(fan_in)
nn.init.uniform_(self.patch_embedding.bias, -bound, bound)
for module in self.text_embedding.modules():
if isinstance(module, nn.Linear):
nn.init.normal_(module.weight, std=0.02)
for module in self.time_embedding.modules():
if isinstance(module, nn.Linear):
nn.init.normal_(module.weight, std=0.02)
nn.init.zeros_(self.head.head.weight)
def _create_freqs(self, grid_size: torch.Tensor, start_frame: int) -> torch.Tensor:
"""Create 3D RoPE frequency tensor."""
device = self.patch_embedding.weight.device
if any(freq.device != device for freq in self.freqs):
self.freqs = [freq.to(device) for freq in self.freqs]
if self.freqs_action.device != device:
self.freqs_action = self.freqs_action.to(device)
if self.freqs_state.device != device:
self.freqs_state = self.freqs_state.to(device)
f, h, w = grid_size.tolist()
freqs = torch.cat(
[
self.freqs[0][start_frame : start_frame + f].view(f, 1, 1, -1).expand(f, h, w, -1),
self.freqs[1][:h].view(1, h, 1, -1).expand(f, h, w, -1),
self.freqs[2][:w].view(1, 1, w, -1).expand(f, h, w, -1),
],
dim=-1,
).reshape(f * h * w, 1, -1)
return freqs
def unpatchify(self, x: torch.Tensor, grid_size: torch.Tensor) -> torch.Tensor:
"""Reconstruct video from patch embeddings."""
B = x.shape[0]
c = self.out_dim
grid_size = grid_size.tolist()
expected_seq_len = math.prod(grid_size)
if x.shape[1] != expected_seq_len:
raise ValueError(f"x sequence length must equal product(grid_size)={expected_seq_len}, got {x.shape[1]}.")
x = x.view(B, *grid_size, *self.patch_size, c)
x = torch.einsum("bfhwpqrc->bcfphqwr", x)
x = x.reshape(B, c, *[i * j for i, j in zip(grid_size, self.patch_size)])
return x
def _forward_blocks(
self,
x: torch.Tensor,
seq_len: int,
freqs: torch.Tensor,
timestep: torch.Tensor,
context: torch.Tensor,
clip_feature: torch.Tensor | None,
embodiment_id: torch.Tensor | None,
action: torch.Tensor | None,
timestep_action: torch.Tensor | None,
state: torch.Tensor | None,
kv_cache: list[torch.Tensor],
current_start_frame: int,
) -> tuple[torch.Tensor, torch.Tensor | None, list[torch.Tensor]]:
x = x.flatten(start_dim=2).transpose(1, 2)
B = x.shape[0]
F_t = timestep.shape[1]
if action is not None:
# Current DreamZero checkpoints have one local action/state adapter.
# Global embodiment IDs are used by transforms and normalization.
adapter_category_id = torch.zeros(B, dtype=torch.long, device=x.device)
action_features = self.action_encoder(action, timestep_action, adapter_category_id)
state_features = self.state_encoder(state, adapter_category_id)
action_register = torch.cat([action_features, state_features], dim=1)
action_length = action_features.shape[1]
action_register_length = action_register.shape[1]
x = torch.cat([x, action_register], dim=1)
else:
action_length = 0
action_register_length = None
timestep = timestep.unsqueeze(-1).expand(B, F_t, seq_len // F_t).reshape(B, -1)
if action is not None:
if timestep_action is None or state is None:
raise RuntimeError("timestep_action and state are required when action is provided.")
state_features_t = self.state_encoder(state, adapter_category_id)
stride = timestep_action.shape[1] // state_features_t.shape[1]
timestep_state = timestep_action[:, ::stride]
timestep = torch.cat([timestep, timestep_action, timestep_state], dim=1)
e = self.time_embedding(sinusoidal_embedding_1d(self.freq_dim, timestep.flatten()).type_as(x))
e = e.unflatten(dim=0, sizes=(B, -1))
e0 = self.time_projection(e)
e0 = e0.unflatten(dim=2, sizes=(6, self.dim))
context = self.text_embedding(context)
if clip_feature is not None:
clip_embedding = self.img_emb(clip_feature)
context = torch.cat([clip_embedding, context], dim=1)
updated_kv_caches: list[torch.Tensor] = []
for block_index, block in enumerate(self.blocks):
x, updated_kv_cache = block(
x=x,
e=e0,
freqs=freqs,
freqs_action=self.freqs_action,
freqs_state=self.freqs_state,
context=context,
action_register_length=action_register_length,
kv_cache=kv_cache[block_index] if kv_cache else None,
current_start_frame=current_start_frame,
)
updated_kv_caches.append(updated_kv_cache)
if action is not None:
action_noise_pred = x[:, seq_len : seq_len + action_length]
action_noise_pred = self.action_decoder(action_noise_pred, adapter_category_id)
else:
action_noise_pred = None
x_video = x[:, :seq_len]
e_video = e[:, :seq_len]
x_video = self.head(x_video, e_video.unsqueeze(2))
return x_video, action_noise_pred, updated_kv_caches
def _forward_inference(
self,
x: torch.Tensor,
timestep: torch.Tensor,
context: torch.Tensor,
seq_len: int,
kv_cache: list[torch.Tensor],
crossattn_cache: list[torch.Tensor],
current_start_frame: int,
y: torch.Tensor | None = None,
clip_feature: torch.Tensor | None = None,
action: torch.Tensor | None = None,
timestep_action: torch.Tensor | None = None,
state: torch.Tensor | None = None,
embodiment_id: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor | None, list[torch.Tensor]]:
if self.model_type == "i2v":
if clip_feature is None or y is None:
raise RuntimeError("clip_feature and y are required for i2v inference.")
if context.shape[1] != self.text_len:
raise ValueError(f"context length must be {self.text_len}, got {context.shape[1]}.")
if y is not None:
x = torch.cat([x, y.to(dtype=x.dtype)], dim=1)
x = self.patch_embedding(x)
grid_size = torch.tensor(x.shape[2:], dtype=torch.long)
freqs = self._create_freqs(grid_size, current_start_frame)
x_video, action_noise_pred, updated_kv_caches = self._forward_blocks(
x=x,
seq_len=seq_len,
freqs=freqs,
timestep=timestep,
context=context,
clip_feature=clip_feature,
embodiment_id=embodiment_id,
action=action,
timestep_action=timestep_action,
state=state,
kv_cache=kv_cache,
current_start_frame=current_start_frame,
)
x_video = x_video.clone()
if action_noise_pred is not None:
action_noise_pred = action_noise_pred.clone()
video_noise_pred = self.unpatchify(x_video, grid_size)
return video_noise_pred, action_noise_pred, updated_kv_caches
def forward(self, *args: Any, **kwargs: Any):
"""Inference only. Requires kv_cache."""
return self._forward_inference(*args, **kwargs)
@@ -0,0 +1,243 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""DreamZero image encoder.
Only the visual tower used by DreamZero I2V inference is ported here. The
checkpoint keys under `action_head.image_encoder.*` load via simple prefix
stripping.
"""
from __future__ import annotations
import math
import torch
import torch.nn as nn
import torch.nn.functional as F
import torchvision.transforms as T
class DreamZeroLayerNorm(nn.LayerNorm):
"""LayerNorm that preserves the input dtype."""
def forward(self, x: torch.Tensor) -> torch.Tensor:
return super().forward(x).type_as(x)
class DreamZeroVisionSelfAttention(nn.Module):
"""Self-attention for the vision tower."""
def __init__(
self,
dim: int,
num_heads: int,
proj_dropout: float = 0.0,
) -> None:
super().__init__()
if dim % num_heads != 0:
raise ValueError(f"dim={dim} must be divisible by num_heads={num_heads}.")
self.dim = dim
self.num_heads = num_heads
self.head_dim = dim // num_heads
self.proj_dropout = proj_dropout
self.to_qkv = nn.Linear(dim, dim * 3)
self.proj = nn.Linear(dim, dim)
def forward(self, x: torch.Tensor) -> torch.Tensor:
batch_size, seq_len, _ = x.shape
q, k, v = self.to_qkv(x).chunk(3, dim=-1)
q = q.view(batch_size, seq_len, self.num_heads, self.head_dim).permute(0, 2, 1, 3)
k = k.view(batch_size, seq_len, self.num_heads, self.head_dim).permute(0, 2, 1, 3)
v = v.view(batch_size, seq_len, self.num_heads, self.head_dim).permute(0, 2, 1, 3)
x = F.scaled_dot_product_attention(q, k, v)
x = x.permute(0, 2, 1, 3).reshape(batch_size, seq_len, self.dim)
x = self.proj(x)
return F.dropout(x, self.proj_dropout, self.training)
class DreamZeroVisionAttentionBlock(nn.Module):
"""Attention block for the vision tower."""
def __init__(
self,
dim: int,
mlp_ratio: float,
num_heads: int,
post_norm: bool = False,
activation: str = "gelu",
proj_dropout: float = 0.0,
norm_eps: float = 1e-5,
) -> None:
super().__init__()
if activation != "gelu":
raise ValueError(f"DreamZero image encoder uses GELU; got activation={activation!r}.")
self.post_norm = post_norm
hidden_dim = int(dim * mlp_ratio)
self.norm1 = DreamZeroLayerNorm(dim, eps=norm_eps)
self.attn = DreamZeroVisionSelfAttention(
dim,
num_heads,
proj_dropout=proj_dropout,
)
self.norm2 = DreamZeroLayerNorm(dim, eps=norm_eps)
self.mlp = nn.Sequential(
nn.Linear(dim, hidden_dim),
nn.GELU(),
nn.Linear(hidden_dim, dim),
nn.Dropout(proj_dropout),
)
def forward(self, x: torch.Tensor) -> torch.Tensor:
if self.post_norm:
x = x + self.norm1(self.attn(x))
x = x + self.norm2(self.mlp(x))
else:
x = x + self.attn(self.norm1(x))
x = x + self.mlp(self.norm2(x))
return x
class DreamZeroVisionTransformer(nn.Module):
"""Vision transformer used by the image encoder."""
def __init__(
self,
image_size: int = 224,
patch_size: int = 14,
dim: int = 1280,
mlp_ratio: float = 4.0,
out_dim: int = 1024,
num_heads: int = 16,
num_layers: int = 32,
pool_type: str = "token",
pre_norm: bool = True,
post_norm: bool = False,
activation: str = "gelu",
proj_dropout: float = 0.0,
embedding_dropout: float = 0.0,
norm_eps: float = 1e-5,
) -> None:
super().__init__()
if pool_type != "token":
raise ValueError(f"DreamZero image encoder only supports pool_type='token', got {pool_type!r}.")
self.image_size = image_size
self.patch_size = patch_size
self.num_patches = (image_size // patch_size) ** 2
self.dim = dim
self.num_heads = num_heads
self.num_layers = num_layers
self.pool_type = pool_type
gain = 1.0 / math.sqrt(dim)
self.patch_embedding = nn.Conv2d(
3,
dim,
kernel_size=patch_size,
stride=patch_size,
bias=not pre_norm,
)
self.cls_embedding = nn.Parameter(gain * torch.randn(1, 1, dim))
self.pos_embedding = nn.Parameter(
gain * torch.randn(1, self.num_patches + 1, dim),
)
self.dropout = nn.Dropout(embedding_dropout)
self.pre_norm = DreamZeroLayerNorm(dim, eps=norm_eps) if pre_norm else None
self.transformer = nn.Sequential(
*[
DreamZeroVisionAttentionBlock(
dim=dim,
mlp_ratio=mlp_ratio,
num_heads=num_heads,
post_norm=post_norm,
activation=activation,
proj_dropout=proj_dropout,
norm_eps=norm_eps,
)
for _ in range(num_layers)
]
)
self.post_norm = DreamZeroLayerNorm(dim, eps=norm_eps)
self.head = nn.Parameter(gain * torch.randn(dim, out_dim))
def forward(self, x: torch.Tensor, use_31_block: bool = False) -> torch.Tensor:
batch_size = x.shape[0]
x = self.patch_embedding(x).flatten(2).permute(0, 2, 1)
x = torch.cat(
[
self.cls_embedding.expand(batch_size, -1, -1).to(dtype=x.dtype, device=x.device),
x,
],
dim=1,
)
x = self.dropout(x + self.pos_embedding.to(dtype=x.dtype, device=x.device))
if self.pre_norm is not None:
x = self.pre_norm(x)
if use_31_block:
return self.transformer[:-1](x)
return self.transformer(x)
class _DreamZeroCLIPContainer(nn.Module):
"""Container matching checkpoint names under `model.visual.*`."""
def __init__(self) -> None:
super().__init__()
self.log_scale = nn.Parameter(torch.ones(()))
self.visual = DreamZeroVisionTransformer(
image_size=224,
patch_size=14,
dim=1280,
mlp_ratio=4.0,
out_dim=1024,
num_heads=16,
num_layers=32,
pool_type="token",
pre_norm=True,
post_norm=False,
activation="gelu",
proj_dropout=0.0,
embedding_dropout=0.0,
norm_eps=1e-5,
)
class DreamZeroImageEncoder(nn.Module):
"""Image encoder wrapper."""
def __init__(self) -> None:
super().__init__()
self.model = _DreamZeroCLIPContainer()
# returns a composed transform whose last stage is CLIP normalization.
self.transforms = T.Compose(
[
T.Normalize(
mean=[0.48145466, 0.4578275, 0.40821073],
std=[0.26862954, 0.26130258, 0.27577711],
),
]
)
def encode_image(self, videos: torch.Tensor) -> torch.Tensor:
"""Encode images for I2V conditioning."""
size = (self.model.visual.image_size,) * 2
videos = torch.cat(
[
F.interpolate(
frame_batch,
size=size,
mode="bicubic",
align_corners=False,
)
for frame_batch in videos
]
)
videos = self.transforms.transforms[-1](videos.mul_(0.5).add_(0.5))
param_dtype = next(iter(self.model.visual.parameters())).dtype
videos = videos.to(dtype=param_dtype)
out = self.model.visual(videos, use_31_block=True)
return out.clone()
File diff suppressed because it is too large. Load diff
@@ -0,0 +1,161 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""DreamZero pipeline persistent state."""
from __future__ import annotations
import logging
from collections import deque
import numpy as np
import torch
logger = logging.getLogger(__name__)
# Number of frames per chunk for subsequent calls (first call uses 1)
FRAMES_PER_CHUNK = 4
class DreamZeroState:
"""Pipeline persistent state across forward() calls.
Lifecycle:
- Created once in DreamZeroPipeline.__init__()
- Mutated every forward() call (frame append, KV cache grow)
- reset() on new session / language change / local_attn_size exceeded
"""
def __init__(self) -> None:
self.reset()
# ------------------------------------------------------------------
# Frame accumulation (single stitched buffer)
# Transform outputs stitched single frame per call.
# We accumulate here to build multi-frame video for AR inference.
# ------------------------------------------------------------------
def accumulate_frames(self, stitched: np.ndarray) -> np.ndarray:
"""Accumulate stitched frames and return multi-frame video.
Args:
stitched: (H, W, C) single frame or (T, H, W, C) multi-frame,
already stitched by transform.
Returns:
(T, H, W, C) ndarray. T=1 for first call, T=FRAMES_PER_CHUNK(4) after.
"""
if stitched.ndim == 3:
self.stitched_buffer.append(stitched)
elif stitched.ndim == 4:
self.stitched_buffer.extend(list(stitched))
else:
raise ValueError(f"Expected 3D or 4D stitched, got {stitched.ndim}D")
num_frames = 1 if self.call_count == 0 else FRAMES_PER_CHUNK
buffer_frames = list(self.stitched_buffer)
if len(buffer_frames) >= num_frames:
frames = buffer_frames[-num_frames:]
else:
frames = buffer_frames
while len(frames) < num_frames:
frames.insert(0, buffer_frames[0])
self.call_count += 1
return np.stack(frames, axis=0) # (T, H, W, C)
# ------------------------------------------------------------------
# Reset / should_reset
# ------------------------------------------------------------------
def reset(self) -> None:
"""Clear all state."""
self.stitched_buffer: deque[np.ndarray] = deque(maxlen=FRAMES_PER_CHUNK)
self.call_count: int = 0
# KV cache once robot-policy diffusion supports that integration.
self.kv_cache: list[torch.Tensor] | None = None
self.kv_cache_neg: list[torch.Tensor] | None = None
self.crossattn_cache: list[dict[str, bool | torch.Tensor | None]] | None = None
self.crossattn_cache_neg: list[dict[str, bool | torch.Tensor | None]] | None = None
self.current_start_frame: int = 0
self.clip_feas: torch.Tensor | None = None
self.ys: torch.Tensor | None = None
self.language: torch.Tensor | None = None
def should_reset(self, text_tokens: torch.Tensor | None, num_video_frames: int, local_attn_size: int) -> bool:
"""Determine if state should be reset before this forward()."""
if self.language is None:
logger.info("language is None, resetting")
return True
if text_tokens is not None and not torch.equal(self.language, text_tokens):
logger.info("language changed, resetting")
return True
# NOTE: after accumulate_frames, num_video_frames is the accumulated T
# (1 for first call, 4 for subsequent). Only reset on true single-frame
# which happens when the stitched_buffer was cleared externally.
if num_video_frames == 1 and self.call_count > 1:
logger.info("single frame input after first call, resetting")
return True
if local_attn_size != -1 and self.current_start_frame >= local_attn_size:
logger.info(
"current_start_frame %d >= local_attn_size %d, resetting", self.current_start_frame, local_attn_size
)
return True
return False
# ------------------------------------------------------------------
# KV cache management
# ------------------------------------------------------------------
def create_kv_caches(
self,
batch_size: int,
dtype: torch.dtype,
device: torch.device,
num_layers: int,
num_heads: int,
head_dim: int,
) -> None:
"""Initialize empty KV caches and cross-attention caches."""
self.kv_cache = [
torch.zeros(2, batch_size, 0, num_heads, head_dim, dtype=dtype, device=device) for _ in range(num_layers)
]
self.kv_cache_neg = [
torch.zeros(2, batch_size, 0, num_heads, head_dim, dtype=dtype, device=device) for _ in range(num_layers)
]
self.crossattn_cache = [{"is_init": False, "k": None, "v": None} for _ in range(num_layers)]
self.crossattn_cache_neg = [{"is_init": False, "k": None, "v": None} for _ in range(num_layers)]
def update_kv_cache(
self,
layer_index: int,
updated_kv: torch.Tensor,
is_negative: bool = False,
) -> None:
"""Update a single layer's KV cache after prefill."""
cache = self.kv_cache_neg if is_negative else self.kv_cache
if cache is None:
raise RuntimeError("KV caches not initialized, call create_kv_caches first.")
cache[layer_index] = updated_kv.clone()
def get_kv_caches(self, is_negative: bool = False) -> list[torch.Tensor]:
"""Get KV caches for the specified branch."""
cache = self.kv_cache_neg if is_negative else self.kv_cache
if cache is None:
raise RuntimeError("KV caches not initialized.")
return cache
def get_crossattn_caches(self, is_negative: bool = False) -> list[dict[str, bool | torch.Tensor | None]]:
"""Get cross-attention caches for the specified branch."""
cache = self.crossattn_cache_neg if is_negative else self.crossattn_cache
if cache is None:
raise RuntimeError("Cross-attn caches not initialized.")
return cache
@@ -0,0 +1,31 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from __future__ import annotations
import importlib
from vllm.logger import init_logger
from vllm_omni.diffusion.models.dreamzero.transform.base import TRANSFORMS
logger = init_logger(__name__)
DEFAULT_EMBODIMENT = "roboarena"
_BUILTIN_TRANSFORM_MODULES = (
"vllm_omni.diffusion.models.dreamzero.transform.droid",
"vllm_omni.diffusion.models.dreamzero.transform.roboarena",
)
def ensure_transforms_loaded() -> None:
"""Import DreamZero transform modules and verify registration."""
for module_name in _BUILTIN_TRANSFORM_MODULES:
try:
importlib.import_module(module_name)
except Exception as exc:
logger.exception("Failed to import DreamZero transform module %s", module_name)
raise RuntimeError(f"Failed to import DreamZero transform module '{module_name}'.") from exc
if DEFAULT_EMBODIMENT not in TRANSFORMS:
raise RuntimeError(f"Built-in DreamZero transform '{DEFAULT_EMBODIMENT}' is not registered after import.")
@@ -0,0 +1,126 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Base transform interface for DreamZero robot policy serving.
Transforms handle dataset-specific concerns ONLY:
- Observation key mapping
- Multi-view stitching (embodiment-specific layout)
- Language template wrapping (embodiment-specific)
- Raw state extraction (dataset-specific keys)
- Output action slicing (to actual action_dim)
Model-specific concerns belong in the pipeline:
- Tokenization (pipeline owns tokenizer)
- State padding (pipeline knows MAX_STATE_DIM)
- Negative prompt (pipeline owns the string)
- Noise generation, encoding, decoding
Flow:
raw obs (dataset format)
→ DreamZeroPipeline selects transform by embodiment
→ unified dict (stitched video, templated prompt str, raw state)
→ tokenize, pad, encode, denoise
→ transform_action_output()
→ ndarray (N, action_dim)
"""
from __future__ import annotations
from typing import Any
import numpy as np
class RobotPolicyTransform:
"""Base class for dataset-specific observation transforms.
Subclasses MUST define:
IMAGE_KEY_MAP: dict — dataset obs keys → unified keys
EMBODIMENT_NAME: str — embodiment identity (pipeline maps to numeric ID)
ACTION_DIM: int — actual action dimensions (for output slicing)
Subclasses MUST override:
_stitch_views() — multi-view → single stitched image
_language_template() — prompt → embodiment-aware template
_extract_raw_state() — obs → raw state ndarray
"""
IMAGE_KEY_MAP: dict[str, str]
EMBODIMENT_NAME: str
ACTION_DIM: int
def transform_input(self, obs: dict) -> dict:
"""Dataset-specific transform: key map → stitch → template → state."""
# 1. Map image keys → unified keys
images: dict[str, np.ndarray] = {}
for src_key, dst_key in self.IMAGE_KEY_MAP.items():
if src_key in obs:
images[dst_key] = np.asarray(obs[src_key])
# 2. Multi-view stitching
stitched = self._stitch_views(images)
# 3. Language template (string only, pipeline tokenizes)
prompt = obs.get("prompt", "")
templated_prompt = self._language_template(prompt)
# 4. Raw state extraction (pipeline pads)
raw_state = self._extract_raw_state(obs)
# 5. Build unified output
unified: dict[str, Any] = {
"images": stitched, # ndarray (T, H_out, W_out, 3)
"prompt": templated_prompt, # str (templated, not tokenized)
"state": raw_state, # ndarray (state_dim,) — pipeline pads
"embodiment_name": self.EMBODIMENT_NAME,
}
if "session_id" in obs:
unified["session_id"] = obs["session_id"]
return unified
def transform_action_output(self, actions: Any) -> np.ndarray:
"""Adapt model action output to this transform's action dimensions."""
actions = np.asarray(actions, dtype=np.float32)
# Handle any remaining batch dims: squeeze to 2D (horizon, dim)
while actions.ndim > 2:
actions = actions[0]
# Slice padded dim to actual ACTION_DIM
if actions.ndim == 2 and actions.shape[-1] > self.ACTION_DIM:
actions = actions[:, : self.ACTION_DIM]
return actions
# ------------------------------------------------------------------
# Subclass MUST override
# ------------------------------------------------------------------
def _stitch_views(self, images: dict[str, np.ndarray]) -> np.ndarray:
"""Stitch camera views into single image.
Input: unified key → ndarray (H,W,3) or (T,H,W,3).
Output: ndarray (T, H_out, W_out, 3).
"""
raise NotImplementedError
def _language_template(self, prompt: str) -> str:
"""Wrap prompt in embodiment-specific template string."""
raise NotImplementedError
def _extract_raw_state(self, obs: dict) -> np.ndarray:
"""Extract raw state vector from obs.
Returns: ndarray (state_dim,) float64. Pipeline handles padding.
"""
raise NotImplementedError
# Transform registry — keyed by embodiment/dataset name
TRANSFORMS: dict[str, RobotPolicyTransform] = {}
def register_transform(name: str, transform: RobotPolicyTransform) -> None:
TRANSFORMS[name] = transform
def get_transform(name: str) -> RobotPolicyTransform:
if name not in TRANSFORMS:
raise KeyError(f"Unknown transform '{name}'. Available: {list(TRANSFORMS.keys())}")
return TRANSFORMS[name]
@@ -0,0 +1,138 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""DROID dataset transform.
DROID uses 1-indexed exterior cameras, 3 views total (OXE_DROID embodiment).
Stitching layout (same as RoboArena — both are OXE_DROID):
┌─────────────────────────┐
│ wrist (2x width) │ ← pixel-repeat along width
├────────────┬────────────┤
│ left ext │ right ext │
└────────────┴────────────┘
"""
from __future__ import annotations
import numpy as np
import torch
import torchvision.transforms.v2 as T
from vllm_omni.diffusion.models.dreamzero.transform.base import (
RobotPolicyTransform,
register_transform,
)
class DroidTransform(RobotPolicyTransform):
"""Transform for DROID dataset (OXE_DROID embodiment).
DROID observation keys (1-indexed exterior cameras):
observation/exterior_image_1_left → left exterior
observation/exterior_image_2_left → right exterior
observation/wrist_image_left → wrist
"""
IMAGE_KEY_MAP = {
"observation/exterior_image_1_left": "images/exterior_0",
"observation/exterior_image_2_left": "images/exterior_1",
"observation/wrist_image_left": "images/wrist",
}
EMBODIMENT_NAME = "oxe_droid"
ACTION_DIM = 8 # 7 joint + 1 gripper
_VIDEO_CROP_SCALE = 0.95
_VIDEO_RESIZE_HW = (176, 320)
@classmethod
def _preprocess_view(cls, arr: np.ndarray) -> np.ndarray:
"""Apply per-view crop and resize before stitching."""
frames = torch.from_numpy(arr).to(torch.float32).permute(0, 3, 1, 2) / 255.0
crop_h = int(arr.shape[1] * cls._VIDEO_CROP_SCALE)
crop_w = int(arr.shape[2] * cls._VIDEO_CROP_SCALE)
frames = T.CenterCrop((crop_h, crop_w))(frames)
frames = T.Resize(
cls._VIDEO_RESIZE_HW,
interpolation=T.InterpolationMode.BILINEAR,
antialias=True,
)(frames)
return (frames.permute(0, 2, 3, 1) * 255.0).to(torch.uint8).cpu().numpy()
def _stitch_views(self, images: dict[str, np.ndarray]) -> np.ndarray:
"""OXE_DROID 2x2 stitching: wrist top (2x wide), exteriors bottom."""
left_ext = images.get("images/exterior_0")
right_ext = images.get("images/exterior_1")
wrist = images.get("images/wrist")
# Ensure 4D: (T, H, W, C)
def ensure_4d(arr: np.ndarray | None) -> np.ndarray | None:
if arr is None:
return None
return arr if arr.ndim == 4 else arr[np.newaxis]
left_ext = ensure_4d(left_ext)
right_ext = ensure_4d(right_ext)
wrist = ensure_4d(wrist)
# Determine shape from first available view.
ref = next((v for v in [wrist, left_ext, right_ext] if v is not None), None)
if ref is None:
return np.zeros((1, 352, 640, 3), dtype=np.uint8)
# Apply per-view crop + resize before stitching.
def maybe_preprocess(arr: np.ndarray | None) -> np.ndarray | None:
if arr is None:
return None
return self._preprocess_view(arr)
left_ext = maybe_preprocess(left_ext)
right_ext = maybe_preprocess(right_ext)
wrist = maybe_preprocess(wrist)
ref = next((v for v in [wrist, left_ext, right_ext] if v is not None), None)
if ref is None:
raise RuntimeError("Expected at least one DROID camera view after preprocessing.")
t, h, w, c = ref.shape
out = np.zeros((t, 2 * h, 2 * w, c), dtype=ref.dtype) # (T, 2H, 2W, C)
# Top row: wrist repeated 2x along width.
if wrist is not None:
wrist_wide = np.repeat(wrist, 2, axis=2) # (T, H, 2W, C)
out[:, :h, :] = wrist_wide
# Bottom row: left exterior | right exterior.
if left_ext is not None:
out[:, h:, :w] = left_ext
if right_ext is not None:
out[:, h:, w:] = right_ext
return out
def _language_template(self, prompt: str) -> str:
"""Expand the language prompt for the OXE_DROID multi-view format."""
prompt = (prompt or "Perform the default behavior.").strip()
prompt_lower = prompt.lower()
return (
"A multi-view video shows that a robot "
+ prompt_lower
+ " The video is split into three views: The top view shows the "
+ "camera view from the robot's wrist, the bottom-left view shows "
+ "the camera view from the left exterior camera, and the "
+ "bottom-right view shows the camera view from the right exterior "
+ "camera. During training, one of the two bottom exterior views "
+ "may be a black screen (dropped view). The robot "
+ prompt_lower
)
def _extract_raw_state(self, obs: dict) -> np.ndarray:
"""OXE_DROID state: 7 joint + 1 gripper = 8 dims."""
parts = []
if "observation/joint_position" in obs:
parts.append(np.asarray(obs["observation/joint_position"], dtype=np.float64).flatten())
if "observation/gripper_position" in obs:
parts.append(np.asarray(obs["observation/gripper_position"], dtype=np.float64).flatten())
if parts:
return np.concatenate(parts)
return np.zeros(8, dtype=np.float64)
register_transform("droid", DroidTransform())
@@ -0,0 +1,41 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""RoboArena dataset transform.
RoboArena uses 0-indexed exterior cameras, 3 views total (OXE_DROID embodiment).
Same stitching layout as DROID — both map to OXE_DROID in DreamZero.
"""
from __future__ import annotations
from vllm_omni.diffusion.models.dreamzero.transform.base import (
register_transform,
)
from vllm_omni.diffusion.models.dreamzero.transform.droid import (
DroidTransform,
)
class RoboArenaTransform(DroidTransform):
"""Transform for RoboArena dataset.
Same embodiment as DROID (OXE_DROID), same stitching and template.
Only difference: 0-indexed exterior camera keys.
RoboArena observation keys (0-indexed):
observation/exterior_image_0_left → left exterior
observation/exterior_image_1_left → right exterior
observation/wrist_image_left → wrist
"""
IMAGE_KEY_MAP = {
"observation/exterior_image_0_left": "images/exterior_0",
"observation/exterior_image_1_left": "images/exterior_1",
"observation/wrist_image_left": "images/wrist",
}
register_transform("roboarena", RoboArenaTransform())
@@ -0,0 +1,29 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""DreamZero model constants shared by the pipeline."""
DEFAULT_NUM_INFERENCE_STEPS = 16
DEFAULT_CFG_SCALE = 5.0
DEFAULT_SIGMA_SHIFT = 5.0
DEFAULT_SEED = 1140
DEFAULT_NEGATIVE_PROMPT = (
"Vibrant colors, overexposed, static, blurry details, text, subtitles, "
"style, artwork, painting, image, still, grayscale, dull, worst quality, "
"low quality, JPEG artifacts, ugly, mutilated, extra fingers, bad hands, "
"bad face, deformed, disfigured, mutated limbs, fused fingers, stagnant "
"image, cluttered background, three legs, many people in the background, "
"walking backwards."
)
DEFAULT_EMBODIMENT_NAME_TO_ID = {
"oxe_droid": 17,
"agibot": 26,
"gr1_unified": 24,
"xdof": 22,
"yam": 32,
"mecka_hands": 27,
"lapa": 27,
"dream": 31,
}
@@ -0,0 +1,21 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from __future__ import annotations
import torch
class DreamZeroVideoExportWorkerExtension:
"""DreamZero worker RPCs used by offline example video export."""
def decode_video_latents_to_uint8(self, video_latents: torch.Tensor) -> torch.Tensor:
if self.model_runner is None or self.model_runner.pipeline is None:
raise RuntimeError("DreamZero pipeline is not initialized on this worker.")
with torch.inference_mode():
decoded = self.model_runner.pipeline.decode_video_latents(video_latents)
decoded = decoded.squeeze(0).permute(1, 2, 3, 0).contiguous()
decoded = decoded.clamp(-1, 1) * 0.5 + 0.5
decoded = (decoded * 255.0).round().to(torch.uint8).cpu()
return decoded