[Feature] Add experimental LingBot World last-step KV reuse (#7816)
Signed-off-by: wtz2333 <2955110911@qq.com>
This commit is contained in:
1 parent
fb2e3535af
commit
8efd3c2f6d
4 files changed
+94
-22
No files matched your search
@@ -7,9 +7,10 @@ over every block, stepwise execution calls ``probe_step`` / ``apply_transition``
|
||||
one denoise step at a time and ``commit_block_kv`` from ``post_decode`` -- so the
|
||||
math lives here once and cannot drift between the two modes.
|
||||
|
||||
A block is four probes that must not touch KV, then one clean-x0 commit that
|
||||
does. Whether a call sees paged (session-bound) or request-local KV is decided
|
||||
by the ``ar`` argument, so this class never reaches back into the pipeline.
|
||||
By default a block is four probes followed by one clean-x0 KV commit.
|
||||
The experimental last-step reuse mode instead commits the fourth noisy probe.
|
||||
Whether a call sees paged (session-bound) or request-local KV is decided by
|
||||
the ``ar`` argument, so this class never reaches back into the pipeline.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -19,6 +20,7 @@ from typing import TYPE_CHECKING, Any, cast
|
||||
|
||||
import torch
|
||||
from diffusers.utils.torch_utils import randn_tensor
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from vllm_omni.diffusion.forward_context import set_forward_context_denoise_step_idx
|
||||
from vllm_omni.diffusion.models.lingbot_world.transformer import (
|
||||
@@ -33,6 +35,9 @@ if TYPE_CHECKING:
|
||||
from vllm_omni.experimental.ar_diffusion.kv_cache.state import ARDiffusionKVState
|
||||
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ARBlockContext:
|
||||
"""The bound AR-Diffusion session one block runs against.
|
||||
@@ -55,10 +60,14 @@ class LingBotDMDBlockRunner:
|
||||
*,
|
||||
device: torch.device,
|
||||
enforce_eager: bool,
|
||||
reuse_last_step_kv: bool = False,
|
||||
) -> None:
|
||||
self.transformer = transformer
|
||||
self.device = device
|
||||
self.enforce_eager = enforce_eager
|
||||
# Experimental approximation: cache x_t/t_last features instead of x0/t=0.
|
||||
self.reuse_last_step_kv = reuse_last_step_kv
|
||||
logger.info("LingBot experimental last-step KV reuse: %s", self.reuse_last_step_kv)
|
||||
|
||||
# ── cache selection ──────────────────────────────────────────────────
|
||||
|
||||
@@ -114,8 +123,8 @@ class LingBotDMDBlockRunner:
|
||||
) -> torch.Tensor:
|
||||
"""Predict flow for one denoise step.
|
||||
|
||||
A probe never writes KV: only the clean x0 of a finished block may
|
||||
enter the cache, which is ``commit_block_kv``'s job.
|
||||
Normally only clean x0 enters KV. The experimental reuse mode commits
|
||||
the final noisy probe instead; the fixed DMD schedule has four probes.
|
||||
|
||||
``camera_cache`` is the block's camera-modulation cache when the caller
|
||||
holds it across separate calls (the stepwise path); ``generate_block``
|
||||
@@ -128,14 +137,15 @@ class LingBotDMDBlockRunner:
|
||||
# Checkpoint channel contract:
|
||||
# [noise/x_t(16), temporal_mask(4), image_latent(16)] -> 36.
|
||||
model_input = torch.cat((current_latents.to(dtype=condition.dtype), condition), dim=1)
|
||||
commit_probe = self.reuse_last_step_kv and step_index == 3
|
||||
flow_prediction = self.transformer(
|
||||
hidden_states=model_input,
|
||||
timestep=timestep,
|
||||
encoder_hidden_states=prompt_embeds,
|
||||
camera_hidden_states=camera,
|
||||
cache=self._block_cache(condition=condition, cache=cache, ar=ar, commit_current=False),
|
||||
cache=self._block_cache(condition=condition, cache=cache, ar=ar, commit_current=commit_probe),
|
||||
start_frame=start_frame,
|
||||
update_cache=False,
|
||||
update_cache=commit_probe,
|
||||
camera_modulation_cache=camera_cache,
|
||||
)
|
||||
if flow_prediction.shape != current_latents.shape:
|
||||
@@ -183,10 +193,15 @@ class LingBotDMDBlockRunner:
|
||||
) -> None:
|
||||
"""Write the finished block's clean x0 into KV and commit its pages.
|
||||
|
||||
The fifth transformer call of a block, deliberately not a denoise step:
|
||||
on the stepwise path it belongs to ``post_decode()``. ``camera_cache``
|
||||
is as in ``probe_step``.
|
||||
By default this is the fifth transformer call, at t=0. Experimental
|
||||
reuse only finalizes the pages written by the fourth probe. On the
|
||||
stepwise path both modes finalize from ``post_decode()``.
|
||||
``camera_cache`` is as in ``probe_step``.
|
||||
"""
|
||||
if self.reuse_last_step_kv:
|
||||
if ar is not None:
|
||||
ar.state.commit_paged_context(ar.branch)
|
||||
return
|
||||
# Commit K/V only for the final clean block, never for noisy probes.
|
||||
cache_input = torch.cat((latents.to(dtype=condition.dtype), condition), dim=1)
|
||||
if not self.enforce_eager:
|
||||
|
||||
@@ -581,6 +581,11 @@ class LingBotWorldCausalDMDPipeline(
|
||||
del prefix
|
||||
_validate_parallel_config(od_config)
|
||||
self.od_config = od_config
|
||||
model_config = getattr(od_config, "model_config", None) or {}
|
||||
reuse_last_step_kv = model_config.get("lingbot_reuse_last_step_kv", False)
|
||||
if not isinstance(reuse_last_step_kv, bool):
|
||||
raise ValueError("model_config.lingbot_reuse_last_step_kv must be a bool.")
|
||||
self._reuse_last_step_kv = reuse_last_step_kv
|
||||
self.device = get_local_device()
|
||||
dtype = getattr(od_config, "dtype", torch.bfloat16)
|
||||
model = od_config.model
|
||||
@@ -666,7 +671,6 @@ class LingBotWorldCausalDMDPipeline(
|
||||
|
||||
self.vae_scale_factor_temporal = int(getattr(self.vae.config, "scale_factor_temporal", 4))
|
||||
self.vae_scale_factor_spatial = int(getattr(self.vae.config, "scale_factor_spatial", 8))
|
||||
model_config = getattr(od_config, "model_config", None) or {}
|
||||
self._ar_height = int(model_config.get("ar_diffusion_height", 480))
|
||||
self._ar_width = int(model_config.get("ar_diffusion_width", 832))
|
||||
self._ar_diffusion_kv_state: ARDiffusionKVState | None = None
|
||||
@@ -1337,6 +1341,7 @@ class LingBotWorldCausalDMDPipeline(
|
||||
self.transformer,
|
||||
device=self.device,
|
||||
enforce_eager=bool(self.od_config.enforce_eager),
|
||||
reuse_last_step_kv=self._reuse_last_step_kv,
|
||||
)
|
||||
self._dmd_block_runner = runner
|
||||
return runner
|
||||
|
||||
Reference in new issue
Block a user