[Perf][DreamZero]dreamzero supports CUDA graph, torch.compile, and DiT Caching. (#4213)
Signed-off-by: amy-why-3459 <wuhaiyan17@huawei.com>
This commit is contained in:
1 parent
63ba11b8b1
commit
f05e03aea8
22 files changed
+1204
-177
No files matched your search
@@ -23,7 +23,8 @@ class DreamZeroState:
|
||||
Lifecycle:
|
||||
- Created once in DreamZeroPipeline.__init__()
|
||||
- Mutated every forward() call (frame append, KV cache grow)
|
||||
- reset() on new session / language change / local_attn_size exceeded
|
||||
- reset() on new session / language change
|
||||
- reset_inference_state() on local_attn_size exceeded (KV only)
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
@@ -65,50 +66,106 @@ class DreamZeroState:
|
||||
self.call_count += 1
|
||||
return np.stack(frames, axis=0) # (T, H, W, C)
|
||||
|
||||
def append_video_latents(self, video_out: torch.Tensor) -> None:
|
||||
"""Append one AR chunk of normalized video latents for later decode.
|
||||
|
||||
Args:
|
||||
video_out: ``(B, T, C, H, W)`` tensor returned by the denoise loop
|
||||
(before the ``transpose(1, 2)`` stored in ``DiffusionOutput``).
|
||||
"""
|
||||
if video_out.dim() != 5:
|
||||
raise ValueError(f"Expected 5D video_out, got shape {tuple(video_out.shape)}")
|
||||
# Upstream ``torch.cat(self.video_across_time, dim=2)`` uses (B, C, T, H, W).
|
||||
chunk = video_out.transpose(1, 2).detach().cpu()
|
||||
self.video_latents_across_time.append(chunk)
|
||||
logger.info(
|
||||
"append_video_latents: chunk_shape=%s total_chunks=%d total_latent_t=%d",
|
||||
tuple(chunk.shape),
|
||||
len(self.video_latents_across_time),
|
||||
int(sum(c.shape[2] for c in self.video_latents_across_time)),
|
||||
)
|
||||
|
||||
def get_concatenated_video_latents(self) -> torch.Tensor | None:
|
||||
"""Return all accumulated chunks concatenated along the time dimension."""
|
||||
if not self.video_latents_across_time:
|
||||
return None
|
||||
if len(self.video_latents_across_time) == 1:
|
||||
return self.video_latents_across_time[0]
|
||||
return torch.cat(self.video_latents_across_time, dim=2)
|
||||
|
||||
def clear_video_latents(self) -> None:
|
||||
"""Drop accumulated video latents without resetting KV/frame state."""
|
||||
self.video_latents_across_time = []
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Reset / should_reset
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def reset(self) -> None:
|
||||
"""Clear all state."""
|
||||
self.stitched_buffer: deque[np.ndarray] = deque(maxlen=FRAMES_PER_CHUNK)
|
||||
self.call_count: int = 0
|
||||
def reset(self, *, clear_video_latents: bool = True) -> None:
|
||||
"""Clear session state.
|
||||
|
||||
Args:
|
||||
clear_video_latents: When ``False``, keep ``video_latents_across_time``
|
||||
so offline/online export can decode the full AR rollout.
|
||||
"""
|
||||
saved_video_latents = [] if clear_video_latents else list(self.video_latents_across_time)
|
||||
self.stitched_buffer = deque(maxlen=FRAMES_PER_CHUNK)
|
||||
self.call_count = 0
|
||||
# Normalized VAE latents per AR chunk, each (B, C, T, H, W) on CPU.
|
||||
# Concatenated along T before decode (matches upstream video_across_time).
|
||||
self.video_latents_across_time = saved_video_latents
|
||||
|
||||
# KV cache once robot-policy diffusion supports that integration.
|
||||
self.kv_cache: list[torch.Tensor] | None = None
|
||||
self.kv_cache_neg: list[torch.Tensor] | None = None
|
||||
self.crossattn_cache: list[dict[str, bool | torch.Tensor | None]] | None = None
|
||||
self.crossattn_cache_neg: list[dict[str, bool | torch.Tensor | None]] | None = None
|
||||
self.current_start_frame: int = 0
|
||||
self.kv_cache = None
|
||||
self.kv_cache_neg = None
|
||||
self.crossattn_cache = None
|
||||
self.crossattn_cache_neg = None
|
||||
self.current_start_frame = 0
|
||||
|
||||
self.clip_feas: torch.Tensor | None = None
|
||||
self.ys: torch.Tensor | None = None
|
||||
self.language: torch.Tensor | None = None
|
||||
self.clip_feas = None
|
||||
self.ys = None
|
||||
if clear_video_latents:
|
||||
self.language = None
|
||||
|
||||
def should_reset(self, text_tokens: torch.Tensor | None, num_video_frames: int, local_attn_size: int) -> bool:
|
||||
"""Determine if state should be reset before this forward()."""
|
||||
def reset_inference_state(self) -> None:
|
||||
"""Reset KV/frame state after local attention rolls without dropping video latents."""
|
||||
self.reset(clear_video_latents=False)
|
||||
|
||||
def reset_reason(
|
||||
self,
|
||||
text_tokens: torch.Tensor | None,
|
||||
num_video_frames: int,
|
||||
local_attn_size: int,
|
||||
) -> str | None:
|
||||
"""Return why state should reset before the next forward(), if any."""
|
||||
if self.language is None:
|
||||
logger.info("language is None, resetting")
|
||||
return True
|
||||
return "session"
|
||||
|
||||
if text_tokens is not None and not torch.equal(self.language, text_tokens):
|
||||
logger.info("language changed, resetting")
|
||||
return True
|
||||
return "session"
|
||||
|
||||
# NOTE: after accumulate_frames, num_video_frames is the accumulated T
|
||||
# (1 for first call, 4 for subsequent). Only reset on true single-frame
|
||||
# which happens when the stitched_buffer was cleared externally.
|
||||
if num_video_frames == 1 and self.call_count > 1:
|
||||
logger.info("single frame input after first call, resetting")
|
||||
return True
|
||||
return "session"
|
||||
|
||||
if local_attn_size != -1 and self.current_start_frame >= local_attn_size:
|
||||
logger.info(
|
||||
"current_start_frame %d >= local_attn_size %d, resetting", self.current_start_frame, local_attn_size
|
||||
"current_start_frame %d >= local_attn_size %d, resetting inference state",
|
||||
self.current_start_frame,
|
||||
local_attn_size,
|
||||
)
|
||||
return True
|
||||
return "inference"
|
||||
|
||||
return False
|
||||
return None
|
||||
|
||||
def should_reset(self, text_tokens: torch.Tensor | None, num_video_frames: int, local_attn_size: int) -> bool:
|
||||
"""Determine if state should be reset before this forward()."""
|
||||
return self.reset_reason(text_tokens, num_video_frames, local_attn_size) is not None
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# KV cache management
|
||||
|
||||
Reference in new issue
Block a user