[Feature] Add experimental LingBot World last-step KV reuse (#7816)

Signed-off-by: wtz2333 <2955110911@qq.com>
This commit is contained in:
wtz2333 authored and GitHub committed 2026-09-21 17:02:15 +08:00
1 parent fb2e3535af
commit 8efd3c2f6d
4 files changed
+94 -22

No files matched your search

@@ -7,9 +7,10 @@ over every block, stepwise execution calls ``probe_step`` / ``apply_transition``
one denoise step at a time and ``commit_block_kv`` from ``post_decode`` -- so the
math lives here once and cannot drift between the two modes.
A block is four probes that must not touch KV, then one clean-x0 commit that
does. Whether a call sees paged (session-bound) or request-local KV is decided
by the ``ar`` argument, so this class never reaches back into the pipeline.
By default a block is four probes followed by one clean-x0 KV commit.
The experimental last-step reuse mode instead commits the fourth noisy probe.
Whether a call sees paged (session-bound) or request-local KV is decided by
the ``ar`` argument, so this class never reaches back into the pipeline.
"""
from __future__ import annotations
@@ -19,6 +20,7 @@ from typing import TYPE_CHECKING, Any, cast
import torch
from diffusers.utils.torch_utils import randn_tensor
from vllm.logger import init_logger
from vllm_omni.diffusion.forward_context import set_forward_context_denoise_step_idx
from vllm_omni.diffusion.models.lingbot_world.transformer import (
@@ -33,6 +35,9 @@ if TYPE_CHECKING:
from vllm_omni.experimental.ar_diffusion.kv_cache.state import ARDiffusionKVState
logger = init_logger(__name__)
@dataclass(frozen=True)
class ARBlockContext:
"""The bound AR-Diffusion session one block runs against.
@@ -55,10 +60,14 @@ class LingBotDMDBlockRunner:
*,
device: torch.device,
enforce_eager: bool,
reuse_last_step_kv: bool = False,
) -> None:
self.transformer = transformer
self.device = device
self.enforce_eager = enforce_eager
# Experimental approximation: cache x_t/t_last features instead of x0/t=0.
self.reuse_last_step_kv = reuse_last_step_kv
logger.info("LingBot experimental last-step KV reuse: %s", self.reuse_last_step_kv)
# ── cache selection ──────────────────────────────────────────────────
@@ -114,8 +123,8 @@ class LingBotDMDBlockRunner:
) -> torch.Tensor:
"""Predict flow for one denoise step.
A probe never writes KV: only the clean x0 of a finished block may
enter the cache, which is ``commit_block_kv``'s job.
Normally only clean x0 enters KV. The experimental reuse mode commits
the final noisy probe instead; the fixed DMD schedule has four probes.
``camera_cache`` is the block's camera-modulation cache when the caller
holds it across separate calls (the stepwise path); ``generate_block``
@@ -128,14 +137,15 @@ class LingBotDMDBlockRunner:
# Checkpoint channel contract:
# [noise/x_t(16), temporal_mask(4), image_latent(16)] -> 36.
model_input = torch.cat((current_latents.to(dtype=condition.dtype), condition), dim=1)
commit_probe = self.reuse_last_step_kv and step_index == 3
flow_prediction = self.transformer(
hidden_states=model_input,
timestep=timestep,
encoder_hidden_states=prompt_embeds,
camera_hidden_states=camera,
cache=self._block_cache(condition=condition, cache=cache, ar=ar, commit_current=False),
cache=self._block_cache(condition=condition, cache=cache, ar=ar, commit_current=commit_probe),
start_frame=start_frame,
update_cache=False,
update_cache=commit_probe,
camera_modulation_cache=camera_cache,
)
if flow_prediction.shape != current_latents.shape:
@@ -183,10 +193,15 @@ class LingBotDMDBlockRunner:
) -> None:
"""Write the finished block's clean x0 into KV and commit its pages.
The fifth transformer call of a block, deliberately not a denoise step:
on the stepwise path it belongs to ``post_decode()``. ``camera_cache``
is as in ``probe_step``.
By default this is the fifth transformer call, at t=0. Experimental
reuse only finalizes the pages written by the fourth probe. On the
stepwise path both modes finalize from ``post_decode()``.
``camera_cache`` is as in ``probe_step``.
"""
if self.reuse_last_step_kv:
if ar is not None:
ar.state.commit_paged_context(ar.branch)
return
# Commit K/V only for the final clean block, never for noisy probes.
cache_input = torch.cat((latents.to(dtype=condition.dtype), condition), dim=1)
if not self.enforce_eager:
@@ -581,6 +581,11 @@ class LingBotWorldCausalDMDPipeline(
del prefix
_validate_parallel_config(od_config)
self.od_config = od_config
model_config = getattr(od_config, "model_config", None) or {}
reuse_last_step_kv = model_config.get("lingbot_reuse_last_step_kv", False)
if not isinstance(reuse_last_step_kv, bool):
raise ValueError("model_config.lingbot_reuse_last_step_kv must be a bool.")
self._reuse_last_step_kv = reuse_last_step_kv
self.device = get_local_device()
dtype = getattr(od_config, "dtype", torch.bfloat16)
model = od_config.model
@@ -666,7 +671,6 @@ class LingBotWorldCausalDMDPipeline(
self.vae_scale_factor_temporal = int(getattr(self.vae.config, "scale_factor_temporal", 4))
self.vae_scale_factor_spatial = int(getattr(self.vae.config, "scale_factor_spatial", 8))
model_config = getattr(od_config, "model_config", None) or {}
self._ar_height = int(model_config.get("ar_diffusion_height", 480))
self._ar_width = int(model_config.get("ar_diffusion_width", 832))
self._ar_diffusion_kv_state: ARDiffusionKVState | None = None
@@ -1337,6 +1341,7 @@ class LingBotWorldCausalDMDPipeline(
self.transformer,
device=self.device,
enforce_eager=bool(self.od_config.enforce_eager),
reuse_last_step_kv=self._reuse_last_step_kv,
)
self._dmd_block_runner = runner
return runner