[LingBot World] Eleven bit-identical removals of repeated work in the realtime path (#7648)

Signed-off-by: tzhouam <tzhouam@connect.ust.hk>
Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
Zhou TaichangandClaude Fable 5.1 authored and GitHub committed 2026-09-21 06:12:45 +00:00
1 parent a9e2117f5e
commit 86fcfe95f1
23 files changed
+1957 -200

No files matched your search

@@ -1,10 +1,24 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Configuration for AR-Diffusion engine-level KV cache management."""
from __future__ import annotations
import os
from dataclasses import dataclass
KV_GATHER_ENV = "VLLM_OMNI_AR_DIFFUSION_KV_GATHER"
def contiguous_kv_gather_enabled() -> bool:
"""Whether the experimental contiguous-K/V gather attention path is on (``VLLM_OMNI_AR_DIFFUSION_KV_GATHER=1``).
The one place the switch is read: the KV manager consults it when it budgets and allocates the history
staging buffers, and the attention dispatch when it picks the path, so the two cannot disagree. It is the
only consumer of ``ARDiffusionKVConfig.reuse_history_staging``.
"""
return os.environ.get(KV_GATHER_ENV, "0") == "1"
@dataclass
class ARDiffusionKVConfig:
@@ -30,6 +44,12 @@ class ARDiffusionKVConfig:
# DiT graphs for every window-fill shape at load time via a synthetic rollout,
# so the serving run is fast from the first chunk. No effect when eager.
warmup_cudagraph: bool = True
# Keep one contiguous K/V staging buffer per layer and refresh only the tokens that changed.
# Within one AR block every forward attends the same history and differs only in the current
# chunk, so re-gathering the whole visible window per forward re-copies bytes that did not move.
# Off by default: it costs two max-sequence buffers per layer, per rank. See
# ARDiffusionPagedForwardContext.history_staging.
reuse_history_staging: bool = False
# Also capture the post-window-boundary (reset-cycle) forward during warm-up.
warmup_capture_reset: bool = False