[LingBot World] Eleven bit-identical removals of repeated work in the realtime path (#7648)
Signed-off-by: tzhouam <tzhouam@connect.ust.hk> Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
1 parent
a9e2117f5e
commit
86fcfe95f1
23 files changed
+1957
-200
No files matched your search
@@ -1,10 +1,24 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
"""Configuration for AR-Diffusion engine-level KV cache management."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from dataclasses import dataclass
|
||||
|
||||
KV_GATHER_ENV = "VLLM_OMNI_AR_DIFFUSION_KV_GATHER"
|
||||
|
||||
|
||||
def contiguous_kv_gather_enabled() -> bool:
|
||||
"""Whether the experimental contiguous-K/V gather attention path is on (``VLLM_OMNI_AR_DIFFUSION_KV_GATHER=1``).
|
||||
|
||||
The one place the switch is read: the KV manager consults it when it budgets and allocates the history
|
||||
staging buffers, and the attention dispatch when it picks the path, so the two cannot disagree. It is the
|
||||
only consumer of ``ARDiffusionKVConfig.reuse_history_staging``.
|
||||
"""
|
||||
return os.environ.get(KV_GATHER_ENV, "0") == "1"
|
||||
|
||||
|
||||
@dataclass
|
||||
class ARDiffusionKVConfig:
|
||||
@@ -30,6 +44,12 @@ class ARDiffusionKVConfig:
|
||||
# DiT graphs for every window-fill shape at load time via a synthetic rollout,
|
||||
# so the serving run is fast from the first chunk. No effect when eager.
|
||||
warmup_cudagraph: bool = True
|
||||
# Keep one contiguous K/V staging buffer per layer and refresh only the tokens that changed.
|
||||
# Within one AR block every forward attends the same history and differs only in the current
|
||||
# chunk, so re-gathering the whole visible window per forward re-copies bytes that did not move.
|
||||
# Off by default: it costs two max-sequence buffers per layer, per rank. See
|
||||
# ARDiffusionPagedForwardContext.history_staging.
|
||||
reuse_history_staging: bool = False
|
||||
# Also capture the post-window-boundary (reset-cycle) forward during warm-up.
|
||||
warmup_capture_reset: bool = False
|
||||
|
||||
|
||||
Reference in new issue
Block a user