[Perf][DreamZero]dreamzero supports CUDA graph, torch.compile, and DiT Caching. (#4213)

Signed-off-by: amy-why-3459 <wuhaiyan17@huawei.com>
This commit is contained in:
amy-why-3459 authored and GitHub committed 2026-06-19 14:24:37 +08:00
1 parent 63ba11b8b1
commit f05e03aea8
22 files changed
+1204 -177

No files matched your search

@@ -23,6 +23,10 @@ from huggingface_hub import hf_hub_download
from transformers import AutoTokenizer, UMT5Config, UMT5EncoderModel
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
from vllm_omni.diffusion.cache.stepcache import (
get_stepcache_state,
is_stepcache_active,
)
from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig
from vllm_omni.diffusion.distributed.autoencoders.autoencoder_kl_wan import (
DistributedAutoencoderKLWan,
@@ -55,10 +59,6 @@ logger = logging.getLogger(__name__)
MAX_DREAMZERO_SESSIONS = 64
# ---------------------------------------------------------------------------
# ---------------------------------------------------------------------------
class VideoActionScheduler:
"""Wraps video + action schedulers into single .step() interface."""
@@ -205,6 +205,9 @@ class DreamZeroPipeline(nn.Module, CFGParallelMixin):
self._max_session_states = MAX_DREAMZERO_SESSIONS
self.state = self._get_or_create_state("default")
# DiT step cache is configured by StepCacheBackend
# (cache_backend="step_cache") via pipeline._stepcache_config.
# Keep runtime inference settings separate from the training-time config.
self.num_inference_steps: int = model_config.get(
"num_inference_steps",
@@ -306,6 +309,27 @@ class DreamZeroPipeline(nn.Module, CFGParallelMixin):
def predict_noise(self, **kwargs) -> tuple[torch.Tensor, torch.Tensor]:
"""Call CausalWanModel, return (video_pred, action_pred)."""
video_pred, action_pred = self._predict_noise_eager(kwargs)
if is_stepcache_active(self):
video_pred = video_pred.clone()
if action_pred is not None:
action_pred = action_pred.clone()
if action_pred is None:
batch_size = kwargs["hidden_states"].shape[0]
action_pred = torch.empty(
batch_size,
0,
self.transformer.action_dim,
device=video_pred.device,
dtype=video_pred.dtype,
)
return (video_pred, action_pred)
def _predict_noise_eager(self, kwargs: dict) -> tuple[torch.Tensor, torch.Tensor | None]:
"""Eager DiT forward; also handles KV-cache write-back on prefill."""
self._cudagraph_mark_step_begin()
video_pred, action_pred, updated_kv_caches = self.transformer(
x=kwargs["hidden_states"],
timestep=kwargs["timestep_video"],
@@ -327,19 +351,118 @@ class DreamZeroPipeline(nn.Module, CFGParallelMixin):
for i, kv in enumerate(updated_kv_caches):
state.update_kv_cache(i, kv, is_negative=is_neg)
video_pred = video_pred.clone()
if action_pred is not None:
action_pred = action_pred.clone()
else:
batch_size = kwargs["hidden_states"].shape[0]
action_pred = torch.empty(
batch_size,
0,
self.transformer.action_dim,
device=video_pred.device,
dtype=video_pred.dtype,
) # CFG-parallel-safe dummy action pred
return (video_pred, action_pred)
return video_pred, action_pred
# -----------------------------------------------------------------------
# torch.compile setup (paper D.2 extended: encoders + VAE + DiT blocks)
# -----------------------------------------------------------------------
def setup_compile(self) -> None:
"""Compile DreamZero encoders, VAE, and per-block DiT for inference.
Paper D.2 uses ``mode=reduce-overhead``, ``fullgraph=True``, ``dynamic=False``
on text/image/VAE and DiT. VAE decode uses a tensor feat_cache patch and
compiles ``decoder.forward`` (not ``_decode``, which has a Python frame loop).
DiT blocks use per-block ``fullgraph=True``.
"""
if not torch.cuda.is_available():
logger.info("DreamZero setup_compile skipped: CUDA not available.")
return
from vllm_omni.diffusion.models.dreamzero.wan_vae_feat_cache_patch import (
apply_wan_vae_feat_cache_tensor_patch,
)
apply_wan_vae_feat_cache_tensor_patch()
compile_ro = {"mode": "reduce-overhead", "fullgraph": True, "dynamic": False}
# DiT blocks: default avoids CUDAGraph overwrite on modulation tensors; encoders use reduce-overhead.
dit_compile = {"mode": "default", "fullgraph": True, "dynamic": False}
logger.info(
"DreamZero: torch.compile text/image/VAE encode + per-block DiT (encoders reduce-overhead, DiT default)."
)
try:
self.text_encoder.forward = torch.compile(self.text_encoder.forward, **compile_ro)
except Exception as exc:
logger.warning("DreamZero: text_encoder compile failed (%s); skipping.", exc)
try:
self.image_encoder.model.visual.forward = torch.compile(
self.image_encoder.model.visual.forward,
**compile_ro,
)
except Exception as exc:
logger.warning("DreamZero: image_encoder compile failed (%s); skipping.", exc)
try:
self.vae.encode = torch.compile(self.vae.encode, **compile_ro)
except Exception as exc:
logger.warning("DreamZero: vae.encode compile failed (%s); skipping.", exc)
compiled_blocks = 0
for block in self.transformer.blocks:
try:
block.forward = torch.compile(block.forward, **dit_compile)
compiled_blocks += 1
except Exception as exc:
logger.warning(
"DreamZero: transformer block %d compile failed (%s); leaving remaining eager.",
compiled_blocks,
exc,
)
break
if compiled_blocks:
logger.info(
"DreamZero: compiled %d/%d transformer blocks.",
compiled_blocks,
len(self.transformer.blocks),
)
self.warmup_compile()
def warmup_compile(self) -> None:
"""Warm up compiled text/image/VAE paths before timed inference."""
if not torch.cuda.is_available():
return
device = next(self.text_encoder.parameters()).device
with torch.inference_mode():
try:
text_tokens = torch.zeros(1, 16, dtype=torch.long, device=device)
attention_mask = torch.ones_like(text_tokens)
self._encode_text(text_tokens, attention_mask)
except Exception as exc:
logger.warning("DreamZero compile warmup (text_encoder) skipped: %s", exc)
try:
image = torch.zeros(1, 1, 3, 180, 320, dtype=torch.bfloat16, device=device)
self._encode_image(image, self.num_frames, 180, 320)
except Exception as exc:
logger.warning("DreamZero compile warmup (image_encoder) skipped: %s", exc)
try:
latent_h, latent_w = 180 // 8, 320 // 8
dummy_latent = torch.zeros(
1,
16,
self.num_frame_per_block,
latent_h * 2,
latent_w * 2,
dtype=self.vae.dtype,
device=device,
)
mean = self.vae_latents_mean.to(device=device, dtype=self.vae.dtype)
inv_std = self.vae_latents_inv_std.to(device=device, dtype=self.vae.dtype)
dummy_latent_denorm = dummy_latent / inv_std + mean
self._cudagraph_mark_step_begin()
self.vae.decode(dummy_latent_denorm, return_dict=False)
except Exception as exc:
logger.warning("DreamZero compile warmup (vae decode) skipped: %s", exc)
torch.accelerator.synchronize(device)
logger.info("DreamZero compile warmup finished (text / image / vae decode).")
def combine_cfg_noise(
self,
@@ -388,12 +511,20 @@ class DreamZeroPipeline(nn.Module, CFGParallelMixin):
videos = videos.reshape(b, t, c, h, w).permute(0, 2, 1, 3, 4)
return videos.to(dtype=torch.bfloat16)
@staticmethod
def _cudagraph_mark_step_begin() -> None:
try:
torch.compiler.cudagraph_mark_step_begin()
except Exception:
pass
# -----------------------------------------------------------------------
# Text encoding
# -----------------------------------------------------------------------
def _encode_text(self, text_tokens: torch.Tensor, attention_mask: torch.Tensor) -> torch.Tensor:
"""Encode text prompt via UMT5."""
self._cudagraph_mark_step_begin()
seq_lens = attention_mask.gt(0).sum(dim=1).long()
prompt_emb = self.text_encoder(
text_tokens,
@@ -422,6 +553,7 @@ class DreamZeroPipeline(nn.Module, CFGParallelMixin):
batch_size = image.shape[0]
with torch.amp.autocast(dtype=torch.bfloat16, device_type=device.type):
self._cudagraph_mark_step_begin()
clip_context = self.image_encoder.encode_image(image)
msk = torch.ones(batch_size, num_frames, height // 8, width // 8, device=device)
@@ -469,8 +601,23 @@ class DreamZeroPipeline(nn.Module, CFGParallelMixin):
inv_std = self.vae_latents_inv_std.to(device=vae_device, dtype=vae_dtype)
latents = latents / inv_std + mean
with torch.no_grad():
self._cudagraph_mark_step_begin()
return self.vae.decode(latents, return_dict=False)[0]
def decode_accumulated_video_latents(self, session_id: str | None = None) -> torch.Tensor:
"""Decode all AR-chunk latents accumulated for ``session_id``."""
state = self._get_or_create_state(session_id)
latents = state.get_concatenated_video_latents()
if latents is None:
session_key = str(session_id or "default")
raise RuntimeError(f"No accumulated video latents for session {session_key!r}.")
return self.decode_video_latents(latents)
def clear_accumulated_video_latents(self, session_id: str | None = None) -> None:
"""Clear accumulated video latents for ``session_id`` without resetting KV state."""
state = self._get_or_create_state(session_id)
state.clear_video_latents()
# -----------------------------------------------------------------------
# KV cache prefill
# -----------------------------------------------------------------------
@@ -635,6 +782,12 @@ class DreamZeroPipeline(nn.Module, CFGParallelMixin):
noisy_input = video_latents
noisy_input_action = action_latents
# ---- step_cache (StepCacheBackend / dreamzero.git) ----
_cached_flow_pred: torch.Tensor | None = None
_cached_flow_pred_action: torch.Tensor | None = None
_prev_predictions: list[tuple[torch.Tensor]] = []
_step_cache = get_stepcache_state(self) if is_stepcache_active(self) else None
for index in range(len(timesteps_video)):
video_timestep = timesteps_video[index]
action_timestep = timesteps_action[index]
@@ -663,46 +816,58 @@ class DreamZeroPipeline(nn.Module, CFGParallelMixin):
else:
y = state.ys[:, :, -self.num_frame_per_block :]
positive_kwargs = dict(
hidden_states=noisy_input.transpose(1, 2),
timestep_video=timestep,
encoder_hidden_states=prompt_embeds,
kv_cache=state.get_kv_caches(False),
crossattn_cache=state.get_crossattn_caches(False),
y=y,
clip_feature=state.clip_feas,
action=noisy_input_action,
timestep_action=timestep_action,
is_negative=False,
**common_kwargs,
)
# Negative (uncond) kwargs
if do_true_cfg and negative_prompt_embeds is not None:
negative_kwargs = dict(
run_dit = _step_cache is None or _step_cache.should_run_step(_prev_predictions)
if run_dit:
positive_kwargs = dict(
hidden_states=noisy_input.transpose(1, 2),
timestep_video=timestep,
encoder_hidden_states=negative_prompt_embeds,
kv_cache=state.get_kv_caches(True),
crossattn_cache=state.get_crossattn_caches(True),
encoder_hidden_states=prompt_embeds,
kv_cache=state.get_kv_caches(False),
crossattn_cache=state.get_crossattn_caches(False),
y=y,
clip_feature=state.clip_feas,
action=noisy_input_action,
timestep_action=timestep_action,
is_negative=True,
is_negative=False,
**common_kwargs,
)
else:
negative_kwargs = None
noise_pred = self.predict_noise_maybe_with_cfg(
positive_kwargs=positive_kwargs,
negative_kwargs=negative_kwargs,
do_true_cfg=do_true_cfg,
true_cfg_scale=self.cfg_scale,
cfg_normalize=False,
)
flow_pred, flow_pred_action = noise_pred
if do_true_cfg and negative_prompt_embeds is not None:
negative_kwargs = dict(
hidden_states=noisy_input.transpose(1, 2),
timestep_video=timestep,
encoder_hidden_states=negative_prompt_embeds,
kv_cache=state.get_kv_caches(True),
crossattn_cache=state.get_crossattn_caches(True),
y=y,
clip_feature=state.clip_feas,
action=noisy_input_action,
timestep_action=timestep_action,
is_negative=True,
**common_kwargs,
)
else:
negative_kwargs = None
noise_pred = self.predict_noise_maybe_with_cfg(
positive_kwargs=positive_kwargs,
negative_kwargs=negative_kwargs,
do_true_cfg=do_true_cfg,
true_cfg_scale=self.cfg_scale,
cfg_normalize=False,
)
flow_pred, flow_pred_action = noise_pred
_cached_flow_pred = flow_pred
_cached_flow_pred_action = flow_pred_action
_prev_predictions.append((flow_pred,))
if _step_cache is not None:
_step_cache.trim_history(_prev_predictions)
else:
# Reuse previous prediction (DiT step-skipping cache).
assert _cached_flow_pred is not None, "DiT cache: no cached prediction available for step skip."
flow_pred = _cached_flow_pred
flow_pred_action = _cached_flow_pred_action
latents = (noisy_input, noisy_input_action)
t = (video_timestep, action_timestep)
@@ -811,9 +976,12 @@ class DreamZeroPipeline(nn.Module, CFGParallelMixin):
# on the next inference request after websocket reset/session switch.
if extra_args.get("reset", False):
state.reset()
# Auto-reset based on model state (before accumulation)
if state.should_reset(text_tokens, 0, self.transformer.local_attn_size):
state.reset()
else:
reset_reason = state.reset_reason(text_tokens, 0, self.transformer.local_attn_size)
if reset_reason == "session":
state.reset()
elif reset_reason == "inference":
state.reset_inference_state()
state.language = text_tokens
# Frame accumulation: stitched single frame → multi-frame video
@@ -959,6 +1127,8 @@ class DreamZeroPipeline(nn.Module, CFGParallelMixin):
video_out = torch.cat([image, video_out], dim=1)
state.current_start_frame += self.num_frame_per_block
state.append_video_latents(video_out)
# q99 denorm: [-1,1] → real values
action_out = self._denormalize_action(action_out.float(), embodiment_name)
@@ -23,7 +23,8 @@ class DreamZeroState:
Lifecycle:
- Created once in DreamZeroPipeline.__init__()
- Mutated every forward() call (frame append, KV cache grow)
- reset() on new session / language change / local_attn_size exceeded
- reset() on new session / language change
- reset_inference_state() on local_attn_size exceeded (KV only)
"""
def __init__(self) -> None:
@@ -65,50 +66,106 @@ class DreamZeroState:
self.call_count += 1
return np.stack(frames, axis=0) # (T, H, W, C)
def append_video_latents(self, video_out: torch.Tensor) -> None:
"""Append one AR chunk of normalized video latents for later decode.
Args:
video_out: ``(B, T, C, H, W)`` tensor returned by the denoise loop
(before the ``transpose(1, 2)`` stored in ``DiffusionOutput``).
"""
if video_out.dim() != 5:
raise ValueError(f"Expected 5D video_out, got shape {tuple(video_out.shape)}")
# Upstream ``torch.cat(self.video_across_time, dim=2)`` uses (B, C, T, H, W).
chunk = video_out.transpose(1, 2).detach().cpu()
self.video_latents_across_time.append(chunk)
logger.info(
"append_video_latents: chunk_shape=%s total_chunks=%d total_latent_t=%d",
tuple(chunk.shape),
len(self.video_latents_across_time),
int(sum(c.shape[2] for c in self.video_latents_across_time)),
)
def get_concatenated_video_latents(self) -> torch.Tensor | None:
"""Return all accumulated chunks concatenated along the time dimension."""
if not self.video_latents_across_time:
return None
if len(self.video_latents_across_time) == 1:
return self.video_latents_across_time[0]
return torch.cat(self.video_latents_across_time, dim=2)
def clear_video_latents(self) -> None:
"""Drop accumulated video latents without resetting KV/frame state."""
self.video_latents_across_time = []
# ------------------------------------------------------------------
# Reset / should_reset
# ------------------------------------------------------------------
def reset(self) -> None:
"""Clear all state."""
self.stitched_buffer: deque[np.ndarray] = deque(maxlen=FRAMES_PER_CHUNK)
self.call_count: int = 0
def reset(self, *, clear_video_latents: bool = True) -> None:
"""Clear session state.
Args:
clear_video_latents: When ``False``, keep ``video_latents_across_time``
so offline/online export can decode the full AR rollout.
"""
saved_video_latents = [] if clear_video_latents else list(self.video_latents_across_time)
self.stitched_buffer = deque(maxlen=FRAMES_PER_CHUNK)
self.call_count = 0
# Normalized VAE latents per AR chunk, each (B, C, T, H, W) on CPU.
# Concatenated along T before decode (matches upstream video_across_time).
self.video_latents_across_time = saved_video_latents
# KV cache once robot-policy diffusion supports that integration.
self.kv_cache: list[torch.Tensor] | None = None
self.kv_cache_neg: list[torch.Tensor] | None = None
self.crossattn_cache: list[dict[str, bool | torch.Tensor | None]] | None = None
self.crossattn_cache_neg: list[dict[str, bool | torch.Tensor | None]] | None = None
self.current_start_frame: int = 0
self.kv_cache = None
self.kv_cache_neg = None
self.crossattn_cache = None
self.crossattn_cache_neg = None
self.current_start_frame = 0
self.clip_feas: torch.Tensor | None = None
self.ys: torch.Tensor | None = None
self.language: torch.Tensor | None = None
self.clip_feas = None
self.ys = None
if clear_video_latents:
self.language = None
def should_reset(self, text_tokens: torch.Tensor | None, num_video_frames: int, local_attn_size: int) -> bool:
"""Determine if state should be reset before this forward()."""
def reset_inference_state(self) -> None:
"""Reset KV/frame state after local attention rolls without dropping video latents."""
self.reset(clear_video_latents=False)
def reset_reason(
self,
text_tokens: torch.Tensor | None,
num_video_frames: int,
local_attn_size: int,
) -> str | None:
"""Return why state should reset before the next forward(), if any."""
if self.language is None:
logger.info("language is None, resetting")
return True
return "session"
if text_tokens is not None and not torch.equal(self.language, text_tokens):
logger.info("language changed, resetting")
return True
return "session"
# NOTE: after accumulate_frames, num_video_frames is the accumulated T
# (1 for first call, 4 for subsequent). Only reset on true single-frame
# which happens when the stitched_buffer was cleared externally.
if num_video_frames == 1 and self.call_count > 1:
logger.info("single frame input after first call, resetting")
return True
return "session"
if local_attn_size != -1 and self.current_start_frame >= local_attn_size:
logger.info(
"current_start_frame %d >= local_attn_size %d, resetting", self.current_start_frame, local_attn_size
"current_start_frame %d >= local_attn_size %d, resetting inference state",
self.current_start_frame,
local_attn_size,
)
return True
return "inference"
return False
return None
def should_reset(self, text_tokens: torch.Tensor | None, num_video_frames: int, local_attn_size: int) -> bool:
"""Determine if state should be reset before this forward()."""
return self.reset_reason(text_tokens, num_video_frames, local_attn_size) is not None
# ------------------------------------------------------------------
# KV cache management
@@ -9,13 +9,30 @@ import torch
class DreamZeroVideoExportWorkerExtension:
"""DreamZero worker RPCs used by offline example video export."""
@staticmethod
def _latents_to_uint8_frames(decoded: torch.Tensor) -> torch.Tensor:
decoded = decoded.squeeze(0).permute(1, 2, 3, 0).contiguous()
decoded = decoded.clamp(-1, 1) * 0.5 + 0.5
return (decoded * 255.0).round().to(torch.uint8).cpu()
def decode_video_latents_to_uint8(self, video_latents: torch.Tensor) -> torch.Tensor:
if self.model_runner is None or self.model_runner.pipeline is None:
raise RuntimeError("DreamZero pipeline is not initialized on this worker.")
with torch.inference_mode():
decoded = self.model_runner.pipeline.decode_video_latents(video_latents)
decoded = decoded.squeeze(0).permute(1, 2, 3, 0).contiguous()
decoded = decoded.clamp(-1, 1) * 0.5 + 0.5
decoded = (decoded * 255.0).round().to(torch.uint8).cpu()
return decoded
return self._latents_to_uint8_frames(decoded)
def decode_accumulated_video_latents_to_uint8(self, session_id: str = "default") -> torch.Tensor:
"""Decode all AR-chunk latents accumulated on the server for one session."""
if self.model_runner is None or self.model_runner.pipeline is None:
raise RuntimeError("DreamZero pipeline is not initialized on this worker.")
with torch.inference_mode():
decoded = self.model_runner.pipeline.decode_accumulated_video_latents(session_id)
return self._latents_to_uint8_frames(decoded)
def clear_accumulated_video_latents(self, session_id: str = "default") -> None:
if self.model_runner is None or self.model_runner.pipeline is None:
raise RuntimeError("DreamZero pipeline is not initialized on this worker.")
self.model_runner.pipeline.clear_accumulated_video_latents(session_id)
@@ -0,0 +1,92 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Make diffusers Wan VAE feat_cache compile-friendly.
diffusers ``WanResample`` stores the string sentinel ``"Rep"`` in feat_cache slots.
``tensor == "Rep"`` returns a non-Tensor and breaks ``torch.compile(fullgraph=True)``.
We replace it with a scalar int8 tensor marker (value 0).
"""
from __future__ import annotations
from typing import Any
import torch
from vllm.logger import init_logger
logger = init_logger(__name__)
_PATCHED = False
def _is_feat_rep(entry: Any) -> bool:
if not isinstance(entry, torch.Tensor):
return False
return entry.numel() == 1 and entry.dtype == torch.int8
def _make_feat_rep(x: torch.Tensor) -> torch.Tensor:
return torch.zeros(1, device=x.device, dtype=torch.int8)
def _wan_resample_forward_tensorized(self, x, feat_cache=None, feat_idx=[0]):
"""``WanResample.forward`` with tensor feat_cache sentinel (no ``"Rep"`` string)."""
from diffusers.models.autoencoders.autoencoder_kl_wan import CACHE_T
b, c, t, h, w = x.size()
if self.mode == "upsample3d":
if feat_cache is not None:
idx = feat_idx[0]
if feat_cache[idx] is None:
feat_cache[idx] = _make_feat_rep(x)
feat_idx[0] += 1
else:
cache_x = x[:, :, -CACHE_T:, :, :].clone()
if cache_x.shape[2] < 2 and feat_cache[idx] is not None and not _is_feat_rep(feat_cache[idx]):
cache_x = torch.cat(
[feat_cache[idx][:, :, -1, :, :].unsqueeze(2).to(cache_x.device), cache_x],
dim=2,
)
if cache_x.shape[2] < 2 and feat_cache[idx] is not None and _is_feat_rep(feat_cache[idx]):
cache_x = torch.cat([torch.zeros_like(cache_x).to(cache_x.device), cache_x], dim=2)
if _is_feat_rep(feat_cache[idx]):
x = self.time_conv(x)
else:
x = self.time_conv(x, feat_cache[idx])
feat_cache[idx] = cache_x
feat_idx[0] += 1
x = x.reshape(b, 2, c, t, h, w)
x = torch.stack((x[:, 0, :, :, :, :], x[:, 1, :, :, :, :]), 3)
x = x.reshape(b, c, t * 2, h, w)
t = x.shape[2]
x = x.permute(0, 2, 1, 3, 4).reshape(b * t, c, h, w)
x = self.resample(x)
x = x.view(b, t, x.size(1), x.size(2), x.size(3)).permute(0, 2, 1, 3, 4)
if self.mode == "downsample3d":
if feat_cache is not None:
idx = feat_idx[0]
if feat_cache[idx] is None:
feat_cache[idx] = x.clone()
feat_idx[0] += 1
else:
cache_x = x[:, :, -1:, :, :].clone()
x = self.time_conv(torch.cat([feat_cache[idx][:, :, -1:, :, :], x], 2))
feat_cache[idx] = cache_x
feat_idx[0] += 1
return x
def apply_wan_vae_feat_cache_tensor_patch() -> None:
"""Monkeypatch diffusers WanResample once per process."""
global _PATCHED
if _PATCHED:
return
from diffusers.models.autoencoders.autoencoder_kl_wan import WanResample
WanResample.forward = _wan_resample_forward_tensorized
_PATCHED = True
logger.info("DreamZero: applied Wan VAE feat_cache tensor sentinel patch.")