[Perf][DreamZero]dreamzero supports CUDA graph, torch.compile, and DiT Caching. (#4213)
Signed-off-by: amy-why-3459 <wuhaiyan17@huawei.com>
This commit is contained in:
1 parent
63ba11b8b1
commit
f05e03aea8
22 files changed
+1204
-177
No files matched your search
@@ -23,6 +23,10 @@ from huggingface_hub import hf_hub_download
|
||||
from transformers import AutoTokenizer, UMT5Config, UMT5EncoderModel
|
||||
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
|
||||
|
||||
from vllm_omni.diffusion.cache.stepcache import (
|
||||
get_stepcache_state,
|
||||
is_stepcache_active,
|
||||
)
|
||||
from vllm_omni.diffusion.data import DiffusionOutput, OmniDiffusionConfig
|
||||
from vllm_omni.diffusion.distributed.autoencoders.autoencoder_kl_wan import (
|
||||
DistributedAutoencoderKLWan,
|
||||
@@ -55,10 +59,6 @@ logger = logging.getLogger(__name__)
|
||||
MAX_DREAMZERO_SESSIONS = 64
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class VideoActionScheduler:
|
||||
"""Wraps video + action schedulers into single .step() interface."""
|
||||
|
||||
@@ -205,6 +205,9 @@ class DreamZeroPipeline(nn.Module, CFGParallelMixin):
|
||||
self._max_session_states = MAX_DREAMZERO_SESSIONS
|
||||
self.state = self._get_or_create_state("default")
|
||||
|
||||
# DiT step cache is configured by StepCacheBackend
|
||||
# (cache_backend="step_cache") via pipeline._stepcache_config.
|
||||
|
||||
# Keep runtime inference settings separate from the training-time config.
|
||||
self.num_inference_steps: int = model_config.get(
|
||||
"num_inference_steps",
|
||||
@@ -306,6 +309,27 @@ class DreamZeroPipeline(nn.Module, CFGParallelMixin):
|
||||
|
||||
def predict_noise(self, **kwargs) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Call CausalWanModel, return (video_pred, action_pred)."""
|
||||
video_pred, action_pred = self._predict_noise_eager(kwargs)
|
||||
|
||||
if is_stepcache_active(self):
|
||||
video_pred = video_pred.clone()
|
||||
if action_pred is not None:
|
||||
action_pred = action_pred.clone()
|
||||
|
||||
if action_pred is None:
|
||||
batch_size = kwargs["hidden_states"].shape[0]
|
||||
action_pred = torch.empty(
|
||||
batch_size,
|
||||
0,
|
||||
self.transformer.action_dim,
|
||||
device=video_pred.device,
|
||||
dtype=video_pred.dtype,
|
||||
)
|
||||
return (video_pred, action_pred)
|
||||
|
||||
def _predict_noise_eager(self, kwargs: dict) -> tuple[torch.Tensor, torch.Tensor | None]:
|
||||
"""Eager DiT forward; also handles KV-cache write-back on prefill."""
|
||||
self._cudagraph_mark_step_begin()
|
||||
video_pred, action_pred, updated_kv_caches = self.transformer(
|
||||
x=kwargs["hidden_states"],
|
||||
timestep=kwargs["timestep_video"],
|
||||
@@ -327,19 +351,118 @@ class DreamZeroPipeline(nn.Module, CFGParallelMixin):
|
||||
for i, kv in enumerate(updated_kv_caches):
|
||||
state.update_kv_cache(i, kv, is_negative=is_neg)
|
||||
|
||||
video_pred = video_pred.clone()
|
||||
if action_pred is not None:
|
||||
action_pred = action_pred.clone()
|
||||
else:
|
||||
batch_size = kwargs["hidden_states"].shape[0]
|
||||
action_pred = torch.empty(
|
||||
batch_size,
|
||||
0,
|
||||
self.transformer.action_dim,
|
||||
device=video_pred.device,
|
||||
dtype=video_pred.dtype,
|
||||
) # CFG-parallel-safe dummy action pred
|
||||
return (video_pred, action_pred)
|
||||
return video_pred, action_pred
|
||||
|
||||
# -----------------------------------------------------------------------
|
||||
# torch.compile setup (paper D.2 extended: encoders + VAE + DiT blocks)
|
||||
# -----------------------------------------------------------------------
|
||||
|
||||
def setup_compile(self) -> None:
|
||||
"""Compile DreamZero encoders, VAE, and per-block DiT for inference.
|
||||
|
||||
Paper D.2 uses ``mode=reduce-overhead``, ``fullgraph=True``, ``dynamic=False``
|
||||
on text/image/VAE and DiT. VAE decode uses a tensor feat_cache patch and
|
||||
compiles ``decoder.forward`` (not ``_decode``, which has a Python frame loop).
|
||||
DiT blocks use per-block ``fullgraph=True``.
|
||||
"""
|
||||
if not torch.cuda.is_available():
|
||||
logger.info("DreamZero setup_compile skipped: CUDA not available.")
|
||||
return
|
||||
|
||||
from vllm_omni.diffusion.models.dreamzero.wan_vae_feat_cache_patch import (
|
||||
apply_wan_vae_feat_cache_tensor_patch,
|
||||
)
|
||||
|
||||
apply_wan_vae_feat_cache_tensor_patch()
|
||||
|
||||
compile_ro = {"mode": "reduce-overhead", "fullgraph": True, "dynamic": False}
|
||||
# DiT blocks: default avoids CUDAGraph overwrite on modulation tensors; encoders use reduce-overhead.
|
||||
dit_compile = {"mode": "default", "fullgraph": True, "dynamic": False}
|
||||
|
||||
logger.info(
|
||||
"DreamZero: torch.compile text/image/VAE encode + per-block DiT (encoders reduce-overhead, DiT default)."
|
||||
)
|
||||
|
||||
try:
|
||||
self.text_encoder.forward = torch.compile(self.text_encoder.forward, **compile_ro)
|
||||
except Exception as exc:
|
||||
logger.warning("DreamZero: text_encoder compile failed (%s); skipping.", exc)
|
||||
|
||||
try:
|
||||
self.image_encoder.model.visual.forward = torch.compile(
|
||||
self.image_encoder.model.visual.forward,
|
||||
**compile_ro,
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.warning("DreamZero: image_encoder compile failed (%s); skipping.", exc)
|
||||
|
||||
try:
|
||||
self.vae.encode = torch.compile(self.vae.encode, **compile_ro)
|
||||
except Exception as exc:
|
||||
logger.warning("DreamZero: vae.encode compile failed (%s); skipping.", exc)
|
||||
|
||||
compiled_blocks = 0
|
||||
for block in self.transformer.blocks:
|
||||
try:
|
||||
block.forward = torch.compile(block.forward, **dit_compile)
|
||||
compiled_blocks += 1
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
"DreamZero: transformer block %d compile failed (%s); leaving remaining eager.",
|
||||
compiled_blocks,
|
||||
exc,
|
||||
)
|
||||
break
|
||||
if compiled_blocks:
|
||||
logger.info(
|
||||
"DreamZero: compiled %d/%d transformer blocks.",
|
||||
compiled_blocks,
|
||||
len(self.transformer.blocks),
|
||||
)
|
||||
|
||||
self.warmup_compile()
|
||||
|
||||
def warmup_compile(self) -> None:
|
||||
"""Warm up compiled text/image/VAE paths before timed inference."""
|
||||
if not torch.cuda.is_available():
|
||||
return
|
||||
|
||||
device = next(self.text_encoder.parameters()).device
|
||||
with torch.inference_mode():
|
||||
try:
|
||||
text_tokens = torch.zeros(1, 16, dtype=torch.long, device=device)
|
||||
attention_mask = torch.ones_like(text_tokens)
|
||||
self._encode_text(text_tokens, attention_mask)
|
||||
except Exception as exc:
|
||||
logger.warning("DreamZero compile warmup (text_encoder) skipped: %s", exc)
|
||||
|
||||
try:
|
||||
image = torch.zeros(1, 1, 3, 180, 320, dtype=torch.bfloat16, device=device)
|
||||
self._encode_image(image, self.num_frames, 180, 320)
|
||||
except Exception as exc:
|
||||
logger.warning("DreamZero compile warmup (image_encoder) skipped: %s", exc)
|
||||
|
||||
try:
|
||||
latent_h, latent_w = 180 // 8, 320 // 8
|
||||
dummy_latent = torch.zeros(
|
||||
1,
|
||||
16,
|
||||
self.num_frame_per_block,
|
||||
latent_h * 2,
|
||||
latent_w * 2,
|
||||
dtype=self.vae.dtype,
|
||||
device=device,
|
||||
)
|
||||
mean = self.vae_latents_mean.to(device=device, dtype=self.vae.dtype)
|
||||
inv_std = self.vae_latents_inv_std.to(device=device, dtype=self.vae.dtype)
|
||||
dummy_latent_denorm = dummy_latent / inv_std + mean
|
||||
self._cudagraph_mark_step_begin()
|
||||
self.vae.decode(dummy_latent_denorm, return_dict=False)
|
||||
except Exception as exc:
|
||||
logger.warning("DreamZero compile warmup (vae decode) skipped: %s", exc)
|
||||
|
||||
torch.accelerator.synchronize(device)
|
||||
logger.info("DreamZero compile warmup finished (text / image / vae decode).")
|
||||
|
||||
def combine_cfg_noise(
|
||||
self,
|
||||
@@ -388,12 +511,20 @@ class DreamZeroPipeline(nn.Module, CFGParallelMixin):
|
||||
videos = videos.reshape(b, t, c, h, w).permute(0, 2, 1, 3, 4)
|
||||
return videos.to(dtype=torch.bfloat16)
|
||||
|
||||
@staticmethod
|
||||
def _cudagraph_mark_step_begin() -> None:
|
||||
try:
|
||||
torch.compiler.cudagraph_mark_step_begin()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# -----------------------------------------------------------------------
|
||||
# Text encoding
|
||||
# -----------------------------------------------------------------------
|
||||
|
||||
def _encode_text(self, text_tokens: torch.Tensor, attention_mask: torch.Tensor) -> torch.Tensor:
|
||||
"""Encode text prompt via UMT5."""
|
||||
self._cudagraph_mark_step_begin()
|
||||
seq_lens = attention_mask.gt(0).sum(dim=1).long()
|
||||
prompt_emb = self.text_encoder(
|
||||
text_tokens,
|
||||
@@ -422,6 +553,7 @@ class DreamZeroPipeline(nn.Module, CFGParallelMixin):
|
||||
batch_size = image.shape[0]
|
||||
|
||||
with torch.amp.autocast(dtype=torch.bfloat16, device_type=device.type):
|
||||
self._cudagraph_mark_step_begin()
|
||||
clip_context = self.image_encoder.encode_image(image)
|
||||
|
||||
msk = torch.ones(batch_size, num_frames, height // 8, width // 8, device=device)
|
||||
@@ -469,8 +601,23 @@ class DreamZeroPipeline(nn.Module, CFGParallelMixin):
|
||||
inv_std = self.vae_latents_inv_std.to(device=vae_device, dtype=vae_dtype)
|
||||
latents = latents / inv_std + mean
|
||||
with torch.no_grad():
|
||||
self._cudagraph_mark_step_begin()
|
||||
return self.vae.decode(latents, return_dict=False)[0]
|
||||
|
||||
def decode_accumulated_video_latents(self, session_id: str | None = None) -> torch.Tensor:
|
||||
"""Decode all AR-chunk latents accumulated for ``session_id``."""
|
||||
state = self._get_or_create_state(session_id)
|
||||
latents = state.get_concatenated_video_latents()
|
||||
if latents is None:
|
||||
session_key = str(session_id or "default")
|
||||
raise RuntimeError(f"No accumulated video latents for session {session_key!r}.")
|
||||
return self.decode_video_latents(latents)
|
||||
|
||||
def clear_accumulated_video_latents(self, session_id: str | None = None) -> None:
|
||||
"""Clear accumulated video latents for ``session_id`` without resetting KV state."""
|
||||
state = self._get_or_create_state(session_id)
|
||||
state.clear_video_latents()
|
||||
|
||||
# -----------------------------------------------------------------------
|
||||
# KV cache prefill
|
||||
# -----------------------------------------------------------------------
|
||||
@@ -635,6 +782,12 @@ class DreamZeroPipeline(nn.Module, CFGParallelMixin):
|
||||
|
||||
noisy_input = video_latents
|
||||
noisy_input_action = action_latents
|
||||
# ---- step_cache (StepCacheBackend / dreamzero.git) ----
|
||||
_cached_flow_pred: torch.Tensor | None = None
|
||||
_cached_flow_pred_action: torch.Tensor | None = None
|
||||
_prev_predictions: list[tuple[torch.Tensor]] = []
|
||||
_step_cache = get_stepcache_state(self) if is_stepcache_active(self) else None
|
||||
|
||||
for index in range(len(timesteps_video)):
|
||||
video_timestep = timesteps_video[index]
|
||||
action_timestep = timesteps_action[index]
|
||||
@@ -663,46 +816,58 @@ class DreamZeroPipeline(nn.Module, CFGParallelMixin):
|
||||
else:
|
||||
y = state.ys[:, :, -self.num_frame_per_block :]
|
||||
|
||||
positive_kwargs = dict(
|
||||
hidden_states=noisy_input.transpose(1, 2),
|
||||
timestep_video=timestep,
|
||||
encoder_hidden_states=prompt_embeds,
|
||||
kv_cache=state.get_kv_caches(False),
|
||||
crossattn_cache=state.get_crossattn_caches(False),
|
||||
y=y,
|
||||
clip_feature=state.clip_feas,
|
||||
action=noisy_input_action,
|
||||
timestep_action=timestep_action,
|
||||
is_negative=False,
|
||||
**common_kwargs,
|
||||
)
|
||||
|
||||
# Negative (uncond) kwargs
|
||||
if do_true_cfg and negative_prompt_embeds is not None:
|
||||
negative_kwargs = dict(
|
||||
run_dit = _step_cache is None or _step_cache.should_run_step(_prev_predictions)
|
||||
if run_dit:
|
||||
positive_kwargs = dict(
|
||||
hidden_states=noisy_input.transpose(1, 2),
|
||||
timestep_video=timestep,
|
||||
encoder_hidden_states=negative_prompt_embeds,
|
||||
kv_cache=state.get_kv_caches(True),
|
||||
crossattn_cache=state.get_crossattn_caches(True),
|
||||
encoder_hidden_states=prompt_embeds,
|
||||
kv_cache=state.get_kv_caches(False),
|
||||
crossattn_cache=state.get_crossattn_caches(False),
|
||||
y=y,
|
||||
clip_feature=state.clip_feas,
|
||||
action=noisy_input_action,
|
||||
timestep_action=timestep_action,
|
||||
is_negative=True,
|
||||
is_negative=False,
|
||||
**common_kwargs,
|
||||
)
|
||||
else:
|
||||
negative_kwargs = None
|
||||
|
||||
noise_pred = self.predict_noise_maybe_with_cfg(
|
||||
positive_kwargs=positive_kwargs,
|
||||
negative_kwargs=negative_kwargs,
|
||||
do_true_cfg=do_true_cfg,
|
||||
true_cfg_scale=self.cfg_scale,
|
||||
cfg_normalize=False,
|
||||
)
|
||||
flow_pred, flow_pred_action = noise_pred
|
||||
if do_true_cfg and negative_prompt_embeds is not None:
|
||||
negative_kwargs = dict(
|
||||
hidden_states=noisy_input.transpose(1, 2),
|
||||
timestep_video=timestep,
|
||||
encoder_hidden_states=negative_prompt_embeds,
|
||||
kv_cache=state.get_kv_caches(True),
|
||||
crossattn_cache=state.get_crossattn_caches(True),
|
||||
y=y,
|
||||
clip_feature=state.clip_feas,
|
||||
action=noisy_input_action,
|
||||
timestep_action=timestep_action,
|
||||
is_negative=True,
|
||||
**common_kwargs,
|
||||
)
|
||||
else:
|
||||
negative_kwargs = None
|
||||
|
||||
noise_pred = self.predict_noise_maybe_with_cfg(
|
||||
positive_kwargs=positive_kwargs,
|
||||
negative_kwargs=negative_kwargs,
|
||||
do_true_cfg=do_true_cfg,
|
||||
true_cfg_scale=self.cfg_scale,
|
||||
cfg_normalize=False,
|
||||
)
|
||||
flow_pred, flow_pred_action = noise_pred
|
||||
_cached_flow_pred = flow_pred
|
||||
_cached_flow_pred_action = flow_pred_action
|
||||
|
||||
_prev_predictions.append((flow_pred,))
|
||||
if _step_cache is not None:
|
||||
_step_cache.trim_history(_prev_predictions)
|
||||
else:
|
||||
# Reuse previous prediction (DiT step-skipping cache).
|
||||
assert _cached_flow_pred is not None, "DiT cache: no cached prediction available for step skip."
|
||||
flow_pred = _cached_flow_pred
|
||||
flow_pred_action = _cached_flow_pred_action
|
||||
|
||||
latents = (noisy_input, noisy_input_action)
|
||||
t = (video_timestep, action_timestep)
|
||||
@@ -811,9 +976,12 @@ class DreamZeroPipeline(nn.Module, CFGParallelMixin):
|
||||
# on the next inference request after websocket reset/session switch.
|
||||
if extra_args.get("reset", False):
|
||||
state.reset()
|
||||
# Auto-reset based on model state (before accumulation)
|
||||
if state.should_reset(text_tokens, 0, self.transformer.local_attn_size):
|
||||
state.reset()
|
||||
else:
|
||||
reset_reason = state.reset_reason(text_tokens, 0, self.transformer.local_attn_size)
|
||||
if reset_reason == "session":
|
||||
state.reset()
|
||||
elif reset_reason == "inference":
|
||||
state.reset_inference_state()
|
||||
state.language = text_tokens
|
||||
|
||||
# Frame accumulation: stitched single frame → multi-frame video
|
||||
@@ -959,6 +1127,8 @@ class DreamZeroPipeline(nn.Module, CFGParallelMixin):
|
||||
video_out = torch.cat([image, video_out], dim=1)
|
||||
state.current_start_frame += self.num_frame_per_block
|
||||
|
||||
state.append_video_latents(video_out)
|
||||
|
||||
# q99 denorm: [-1,1] → real values
|
||||
action_out = self._denormalize_action(action_out.float(), embodiment_name)
|
||||
|
||||
|
||||
@@ -23,7 +23,8 @@ class DreamZeroState:
|
||||
Lifecycle:
|
||||
- Created once in DreamZeroPipeline.__init__()
|
||||
- Mutated every forward() call (frame append, KV cache grow)
|
||||
- reset() on new session / language change / local_attn_size exceeded
|
||||
- reset() on new session / language change
|
||||
- reset_inference_state() on local_attn_size exceeded (KV only)
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
@@ -65,50 +66,106 @@ class DreamZeroState:
|
||||
self.call_count += 1
|
||||
return np.stack(frames, axis=0) # (T, H, W, C)
|
||||
|
||||
def append_video_latents(self, video_out: torch.Tensor) -> None:
|
||||
"""Append one AR chunk of normalized video latents for later decode.
|
||||
|
||||
Args:
|
||||
video_out: ``(B, T, C, H, W)`` tensor returned by the denoise loop
|
||||
(before the ``transpose(1, 2)`` stored in ``DiffusionOutput``).
|
||||
"""
|
||||
if video_out.dim() != 5:
|
||||
raise ValueError(f"Expected 5D video_out, got shape {tuple(video_out.shape)}")
|
||||
# Upstream ``torch.cat(self.video_across_time, dim=2)`` uses (B, C, T, H, W).
|
||||
chunk = video_out.transpose(1, 2).detach().cpu()
|
||||
self.video_latents_across_time.append(chunk)
|
||||
logger.info(
|
||||
"append_video_latents: chunk_shape=%s total_chunks=%d total_latent_t=%d",
|
||||
tuple(chunk.shape),
|
||||
len(self.video_latents_across_time),
|
||||
int(sum(c.shape[2] for c in self.video_latents_across_time)),
|
||||
)
|
||||
|
||||
def get_concatenated_video_latents(self) -> torch.Tensor | None:
|
||||
"""Return all accumulated chunks concatenated along the time dimension."""
|
||||
if not self.video_latents_across_time:
|
||||
return None
|
||||
if len(self.video_latents_across_time) == 1:
|
||||
return self.video_latents_across_time[0]
|
||||
return torch.cat(self.video_latents_across_time, dim=2)
|
||||
|
||||
def clear_video_latents(self) -> None:
|
||||
"""Drop accumulated video latents without resetting KV/frame state."""
|
||||
self.video_latents_across_time = []
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Reset / should_reset
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def reset(self) -> None:
|
||||
"""Clear all state."""
|
||||
self.stitched_buffer: deque[np.ndarray] = deque(maxlen=FRAMES_PER_CHUNK)
|
||||
self.call_count: int = 0
|
||||
def reset(self, *, clear_video_latents: bool = True) -> None:
|
||||
"""Clear session state.
|
||||
|
||||
Args:
|
||||
clear_video_latents: When ``False``, keep ``video_latents_across_time``
|
||||
so offline/online export can decode the full AR rollout.
|
||||
"""
|
||||
saved_video_latents = [] if clear_video_latents else list(self.video_latents_across_time)
|
||||
self.stitched_buffer = deque(maxlen=FRAMES_PER_CHUNK)
|
||||
self.call_count = 0
|
||||
# Normalized VAE latents per AR chunk, each (B, C, T, H, W) on CPU.
|
||||
# Concatenated along T before decode (matches upstream video_across_time).
|
||||
self.video_latents_across_time = saved_video_latents
|
||||
|
||||
# KV cache once robot-policy diffusion supports that integration.
|
||||
self.kv_cache: list[torch.Tensor] | None = None
|
||||
self.kv_cache_neg: list[torch.Tensor] | None = None
|
||||
self.crossattn_cache: list[dict[str, bool | torch.Tensor | None]] | None = None
|
||||
self.crossattn_cache_neg: list[dict[str, bool | torch.Tensor | None]] | None = None
|
||||
self.current_start_frame: int = 0
|
||||
self.kv_cache = None
|
||||
self.kv_cache_neg = None
|
||||
self.crossattn_cache = None
|
||||
self.crossattn_cache_neg = None
|
||||
self.current_start_frame = 0
|
||||
|
||||
self.clip_feas: torch.Tensor | None = None
|
||||
self.ys: torch.Tensor | None = None
|
||||
self.language: torch.Tensor | None = None
|
||||
self.clip_feas = None
|
||||
self.ys = None
|
||||
if clear_video_latents:
|
||||
self.language = None
|
||||
|
||||
def should_reset(self, text_tokens: torch.Tensor | None, num_video_frames: int, local_attn_size: int) -> bool:
|
||||
"""Determine if state should be reset before this forward()."""
|
||||
def reset_inference_state(self) -> None:
|
||||
"""Reset KV/frame state after local attention rolls without dropping video latents."""
|
||||
self.reset(clear_video_latents=False)
|
||||
|
||||
def reset_reason(
|
||||
self,
|
||||
text_tokens: torch.Tensor | None,
|
||||
num_video_frames: int,
|
||||
local_attn_size: int,
|
||||
) -> str | None:
|
||||
"""Return why state should reset before the next forward(), if any."""
|
||||
if self.language is None:
|
||||
logger.info("language is None, resetting")
|
||||
return True
|
||||
return "session"
|
||||
|
||||
if text_tokens is not None and not torch.equal(self.language, text_tokens):
|
||||
logger.info("language changed, resetting")
|
||||
return True
|
||||
return "session"
|
||||
|
||||
# NOTE: after accumulate_frames, num_video_frames is the accumulated T
|
||||
# (1 for first call, 4 for subsequent). Only reset on true single-frame
|
||||
# which happens when the stitched_buffer was cleared externally.
|
||||
if num_video_frames == 1 and self.call_count > 1:
|
||||
logger.info("single frame input after first call, resetting")
|
||||
return True
|
||||
return "session"
|
||||
|
||||
if local_attn_size != -1 and self.current_start_frame >= local_attn_size:
|
||||
logger.info(
|
||||
"current_start_frame %d >= local_attn_size %d, resetting", self.current_start_frame, local_attn_size
|
||||
"current_start_frame %d >= local_attn_size %d, resetting inference state",
|
||||
self.current_start_frame,
|
||||
local_attn_size,
|
||||
)
|
||||
return True
|
||||
return "inference"
|
||||
|
||||
return False
|
||||
return None
|
||||
|
||||
def should_reset(self, text_tokens: torch.Tensor | None, num_video_frames: int, local_attn_size: int) -> bool:
|
||||
"""Determine if state should be reset before this forward()."""
|
||||
return self.reset_reason(text_tokens, num_video_frames, local_attn_size) is not None
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# KV cache management
|
||||
|
||||
@@ -9,13 +9,30 @@ import torch
|
||||
class DreamZeroVideoExportWorkerExtension:
|
||||
"""DreamZero worker RPCs used by offline example video export."""
|
||||
|
||||
@staticmethod
|
||||
def _latents_to_uint8_frames(decoded: torch.Tensor) -> torch.Tensor:
|
||||
decoded = decoded.squeeze(0).permute(1, 2, 3, 0).contiguous()
|
||||
decoded = decoded.clamp(-1, 1) * 0.5 + 0.5
|
||||
return (decoded * 255.0).round().to(torch.uint8).cpu()
|
||||
|
||||
def decode_video_latents_to_uint8(self, video_latents: torch.Tensor) -> torch.Tensor:
|
||||
if self.model_runner is None or self.model_runner.pipeline is None:
|
||||
raise RuntimeError("DreamZero pipeline is not initialized on this worker.")
|
||||
|
||||
with torch.inference_mode():
|
||||
decoded = self.model_runner.pipeline.decode_video_latents(video_latents)
|
||||
decoded = decoded.squeeze(0).permute(1, 2, 3, 0).contiguous()
|
||||
decoded = decoded.clamp(-1, 1) * 0.5 + 0.5
|
||||
decoded = (decoded * 255.0).round().to(torch.uint8).cpu()
|
||||
return decoded
|
||||
return self._latents_to_uint8_frames(decoded)
|
||||
|
||||
def decode_accumulated_video_latents_to_uint8(self, session_id: str = "default") -> torch.Tensor:
|
||||
"""Decode all AR-chunk latents accumulated on the server for one session."""
|
||||
if self.model_runner is None or self.model_runner.pipeline is None:
|
||||
raise RuntimeError("DreamZero pipeline is not initialized on this worker.")
|
||||
|
||||
with torch.inference_mode():
|
||||
decoded = self.model_runner.pipeline.decode_accumulated_video_latents(session_id)
|
||||
return self._latents_to_uint8_frames(decoded)
|
||||
|
||||
def clear_accumulated_video_latents(self, session_id: str = "default") -> None:
|
||||
if self.model_runner is None or self.model_runner.pipeline is None:
|
||||
raise RuntimeError("DreamZero pipeline is not initialized on this worker.")
|
||||
self.model_runner.pipeline.clear_accumulated_video_latents(session_id)
|
||||
@@ -0,0 +1,92 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""Make diffusers Wan VAE feat_cache compile-friendly.
|
||||
|
||||
diffusers ``WanResample`` stores the string sentinel ``"Rep"`` in feat_cache slots.
|
||||
``tensor == "Rep"`` returns a non-Tensor and breaks ``torch.compile(fullgraph=True)``.
|
||||
We replace it with a scalar int8 tensor marker (value 0).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
from vllm.logger import init_logger
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
_PATCHED = False
|
||||
|
||||
|
||||
def _is_feat_rep(entry: Any) -> bool:
|
||||
if not isinstance(entry, torch.Tensor):
|
||||
return False
|
||||
return entry.numel() == 1 and entry.dtype == torch.int8
|
||||
|
||||
|
||||
def _make_feat_rep(x: torch.Tensor) -> torch.Tensor:
|
||||
return torch.zeros(1, device=x.device, dtype=torch.int8)
|
||||
|
||||
|
||||
def _wan_resample_forward_tensorized(self, x, feat_cache=None, feat_idx=[0]):
|
||||
"""``WanResample.forward`` with tensor feat_cache sentinel (no ``"Rep"`` string)."""
|
||||
from diffusers.models.autoencoders.autoencoder_kl_wan import CACHE_T
|
||||
|
||||
b, c, t, h, w = x.size()
|
||||
if self.mode == "upsample3d":
|
||||
if feat_cache is not None:
|
||||
idx = feat_idx[0]
|
||||
if feat_cache[idx] is None:
|
||||
feat_cache[idx] = _make_feat_rep(x)
|
||||
feat_idx[0] += 1
|
||||
else:
|
||||
cache_x = x[:, :, -CACHE_T:, :, :].clone()
|
||||
if cache_x.shape[2] < 2 and feat_cache[idx] is not None and not _is_feat_rep(feat_cache[idx]):
|
||||
cache_x = torch.cat(
|
||||
[feat_cache[idx][:, :, -1, :, :].unsqueeze(2).to(cache_x.device), cache_x],
|
||||
dim=2,
|
||||
)
|
||||
if cache_x.shape[2] < 2 and feat_cache[idx] is not None and _is_feat_rep(feat_cache[idx]):
|
||||
cache_x = torch.cat([torch.zeros_like(cache_x).to(cache_x.device), cache_x], dim=2)
|
||||
if _is_feat_rep(feat_cache[idx]):
|
||||
x = self.time_conv(x)
|
||||
else:
|
||||
x = self.time_conv(x, feat_cache[idx])
|
||||
feat_cache[idx] = cache_x
|
||||
feat_idx[0] += 1
|
||||
|
||||
x = x.reshape(b, 2, c, t, h, w)
|
||||
x = torch.stack((x[:, 0, :, :, :, :], x[:, 1, :, :, :, :]), 3)
|
||||
x = x.reshape(b, c, t * 2, h, w)
|
||||
t = x.shape[2]
|
||||
x = x.permute(0, 2, 1, 3, 4).reshape(b * t, c, h, w)
|
||||
x = self.resample(x)
|
||||
x = x.view(b, t, x.size(1), x.size(2), x.size(3)).permute(0, 2, 1, 3, 4)
|
||||
|
||||
if self.mode == "downsample3d":
|
||||
if feat_cache is not None:
|
||||
idx = feat_idx[0]
|
||||
if feat_cache[idx] is None:
|
||||
feat_cache[idx] = x.clone()
|
||||
feat_idx[0] += 1
|
||||
else:
|
||||
cache_x = x[:, :, -1:, :, :].clone()
|
||||
x = self.time_conv(torch.cat([feat_cache[idx][:, :, -1:, :, :], x], 2))
|
||||
feat_cache[idx] = cache_x
|
||||
feat_idx[0] += 1
|
||||
return x
|
||||
|
||||
|
||||
def apply_wan_vae_feat_cache_tensor_patch() -> None:
|
||||
"""Monkeypatch diffusers WanResample once per process."""
|
||||
global _PATCHED
|
||||
if _PATCHED:
|
||||
return
|
||||
|
||||
from diffusers.models.autoencoders.autoencoder_kl_wan import WanResample
|
||||
|
||||
WanResample.forward = _wan_resample_forward_tensorized
|
||||
_PATCHED = True
|
||||
logger.info("DreamZero: applied Wan VAE feat_cache tensor sentinel patch.")
|
||||
Reference in new issue
Block a user