[Perf][DreamZero]dreamzero supports CUDA graph, torch.compile, and DiT Caching. (#4213)

Signed-off-by: amy-why-3459 <wuhaiyan17@huawei.com>
This commit is contained in:
amy-why-3459 authored and GitHub committed 2026-06-19 14:24:37 +08:00
1 parent 63ba11b8b1
commit f05e03aea8
22 files changed
+1204 -177

No files matched your search

@@ -23,7 +23,8 @@ class DreamZeroState:
Lifecycle:
- Created once in DreamZeroPipeline.__init__()
- Mutated every forward() call (frame append, KV cache grow)
- reset() on new session / language change / local_attn_size exceeded
- reset() on new session / language change
- reset_inference_state() on local_attn_size exceeded (KV only)
"""
def __init__(self) -> None:
@@ -65,50 +66,106 @@ class DreamZeroState:
self.call_count += 1
return np.stack(frames, axis=0) # (T, H, W, C)
def append_video_latents(self, video_out: torch.Tensor) -> None:
"""Append one AR chunk of normalized video latents for later decode.
Args:
video_out: ``(B, T, C, H, W)`` tensor returned by the denoise loop
(before the ``transpose(1, 2)`` stored in ``DiffusionOutput``).
"""
if video_out.dim() != 5:
raise ValueError(f"Expected 5D video_out, got shape {tuple(video_out.shape)}")
# Upstream ``torch.cat(self.video_across_time, dim=2)`` uses (B, C, T, H, W).
chunk = video_out.transpose(1, 2).detach().cpu()
self.video_latents_across_time.append(chunk)
logger.info(
"append_video_latents: chunk_shape=%s total_chunks=%d total_latent_t=%d",
tuple(chunk.shape),
len(self.video_latents_across_time),
int(sum(c.shape[2] for c in self.video_latents_across_time)),
)
def get_concatenated_video_latents(self) -> torch.Tensor | None:
"""Return all accumulated chunks concatenated along the time dimension."""
if not self.video_latents_across_time:
return None
if len(self.video_latents_across_time) == 1:
return self.video_latents_across_time[0]
return torch.cat(self.video_latents_across_time, dim=2)
def clear_video_latents(self) -> None:
"""Drop accumulated video latents without resetting KV/frame state."""
self.video_latents_across_time = []
# ------------------------------------------------------------------
# Reset / should_reset
# ------------------------------------------------------------------
def reset(self) -> None:
"""Clear all state."""
self.stitched_buffer: deque[np.ndarray] = deque(maxlen=FRAMES_PER_CHUNK)
self.call_count: int = 0
def reset(self, *, clear_video_latents: bool = True) -> None:
"""Clear session state.
Args:
clear_video_latents: When ``False``, keep ``video_latents_across_time``
so offline/online export can decode the full AR rollout.
"""
saved_video_latents = [] if clear_video_latents else list(self.video_latents_across_time)
self.stitched_buffer = deque(maxlen=FRAMES_PER_CHUNK)
self.call_count = 0
# Normalized VAE latents per AR chunk, each (B, C, T, H, W) on CPU.
# Concatenated along T before decode (matches upstream video_across_time).
self.video_latents_across_time = saved_video_latents
# KV cache once robot-policy diffusion supports that integration.
self.kv_cache: list[torch.Tensor] | None = None
self.kv_cache_neg: list[torch.Tensor] | None = None
self.crossattn_cache: list[dict[str, bool | torch.Tensor | None]] | None = None
self.crossattn_cache_neg: list[dict[str, bool | torch.Tensor | None]] | None = None
self.current_start_frame: int = 0
self.kv_cache = None
self.kv_cache_neg = None
self.crossattn_cache = None
self.crossattn_cache_neg = None
self.current_start_frame = 0
self.clip_feas: torch.Tensor | None = None
self.ys: torch.Tensor | None = None
self.language: torch.Tensor | None = None
self.clip_feas = None
self.ys = None
if clear_video_latents:
self.language = None
def should_reset(self, text_tokens: torch.Tensor | None, num_video_frames: int, local_attn_size: int) -> bool:
"""Determine if state should be reset before this forward()."""
def reset_inference_state(self) -> None:
"""Reset KV/frame state after local attention rolls without dropping video latents."""
self.reset(clear_video_latents=False)
def reset_reason(
self,
text_tokens: torch.Tensor | None,
num_video_frames: int,
local_attn_size: int,
) -> str | None:
"""Return why state should reset before the next forward(), if any."""
if self.language is None:
logger.info("language is None, resetting")
return True
return "session"
if text_tokens is not None and not torch.equal(self.language, text_tokens):
logger.info("language changed, resetting")
return True
return "session"
# NOTE: after accumulate_frames, num_video_frames is the accumulated T
# (1 for first call, 4 for subsequent). Only reset on true single-frame
# which happens when the stitched_buffer was cleared externally.
if num_video_frames == 1 and self.call_count > 1:
logger.info("single frame input after first call, resetting")
return True
return "session"
if local_attn_size != -1 and self.current_start_frame >= local_attn_size:
logger.info(
"current_start_frame %d >= local_attn_size %d, resetting", self.current_start_frame, local_attn_size
"current_start_frame %d >= local_attn_size %d, resetting inference state",
self.current_start_frame,
local_attn_size,
)
return True
return "inference"
return False
return None
def should_reset(self, text_tokens: torch.Tensor | None, num_video_frames: int, local_attn_size: int) -> bool:
"""Determine if state should be reset before this forward()."""
return self.reset_reason(text_tokens, num_video_frames, local_attn_size) is not None
# ------------------------------------------------------------------
# KV cache management