[Refactor] Declare model-local KV held outside the paged manager (#6171)

Signed-off-by: Yueqian Lin <linyueqian@outlook.com>
This commit is contained in:
Yueqian Lin authored and GitHub committed 2026-09-11 08:44:32 +08:00
1 parent 5a386d789f
commit 5c69fc2a8a
12 files changed
+1283 -3

No files matched your search

@@ -58,6 +58,12 @@ from vllm.sequence import IntermediateTensors
from vllm.utils.tensor_schema import TensorSchema
from vllm_omni.model_executor.models.mimo_audio.config_mimo_audio import MiMoAudioConfig
from vllm_omni.model_executor.models.model_local_kv import (
ModelLocalKVScope,
ModelLocalKVSpec,
RowDriver,
spec_from_hf_config,
)
from vllm_omni.platforms import current_omni_platform
logger = logging.getLogger(__name__)
@@ -744,6 +750,67 @@ class MiMoAudioLLMForConditionalGeneration(nn.Module, SupportsMultiModal, Suppor
def get_language_model(self) -> torch.nn.Module:
return self.model
def model_local_kv_specs(self) -> list[ModelLocalKVSpec]:
"""Declare the local transformer's decode cache and its graph copies.
The cache is created per call inside ``base_local_forward`` and is
bounded by the decode loop, not by sequence length: the loop runs
``group_size + max(delay_pattern)`` steps regardless of how long the
utterance is.
The second entry is the one that matters. ``base_local_forward`` runs
inside ``torch.cuda.graph`` during capture, so each captured bucket
leaves a cache's worth of tensors pinned in the graph pool for the
process lifetime. There is no Python object left to inspect -- the
cache is a local in the captured frame -- so the width comes from the
buckets that captured successfully, which is why it reads
``local_forward_cg_by_bs`` rather than MIMO_CUDAGRAPH_BATCH_SIZES: a
bucket whose capture raised is logged and skipped.
Both entries are declared, and summing them over-states a single step:
a given call either replays a captured bucket or runs eagerly, never
both. The graph pool is resident regardless, so the sum is the right
ceiling for the process and the wrong number for one request.
"""
delay_iters = self.group_size + max(self.delay_pattern)
dtype = next(self.local_transformer.parameters()).dtype
bound = f"group_size({self.group_size}) + max(delay_pattern)({max(self.delay_pattern)})"
specs = [
spec_from_hf_config(
self.local_config,
name="local_transformer",
dtype=dtype,
physical_capacity_positions=delay_iters,
capacity_source=bound,
scope=ModelLocalKVScope.INVOCATION,
# One cache, B rows wide -- not B caches. base_local_forward
# builds a single DynamicCache whose batch dimension is the
# number of requests in the group.
rows=RowDriver.MAX_NUM_SEQS,
allocation_note="one batched allocation per eager call; graph replay uses the pool entry instead",
)
]
captured_rows = sum(self.local_forward_cg_by_bs)
if captured_rows:
specs.append(
spec_from_hf_config(
self.local_config,
name="local_transformer_graph_pool",
dtype=dtype,
physical_capacity_positions=delay_iters,
capacity_source=bound,
scope=ModelLocalKVScope.MODEL,
rows=RowDriver.FIXED,
rows_fixed=captured_rows,
rows_reason=f"sum of captured buckets {sorted(self.local_forward_cg_by_bs)}",
allocation_note=(
"pinned in the CUDA graph pool; capture is gated only on cuda.is_available(), "
"so this stays resident under enforce_eager too"
),
)
)
return specs
def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings:
if kwargs.get("modality_preprocess") is None:
mm_dummy_embeddings = []
@@ -158,6 +158,17 @@ class MingFlashOmniTalkerForConditionalGeneration(nn.Module, CustomProcessMixin)
patch_size=self.patch_size,
)
def model_local_kv_specs(self):
"""Forward to the generator, which owns the talker's StaticCache.
``MingAudioGenerator`` is a plain object rather than a submodule, so
the module-tree walk cannot reach it on its own.
"""
generator = getattr(self, "audio_generator", None)
if generator is None:
return []
return generator.model_local_kv_specs()
@property
def device(self) -> torch.device:
return next(self.model.parameters()).device
@@ -41,6 +41,12 @@ from vllm_omni.model_executor.models.common.ming.cfm_solver import (
integrate_cfm_steps,
)
from vllm_omni.model_executor.models.common.ming.dit_blocks import CondEmbedder, DiTBlock, FinalLayer
from vllm_omni.model_executor.models.model_local_kv import (
ModelLocalKVScope,
ModelLocalKVSpec,
RowDriver,
spec_from_hf_config,
)
logger = init_logger(__name__)
@@ -575,6 +581,13 @@ class MingAudioGenerator:
for a single TTS request. The generator is stateless across requests.
"""
_STATIC_CACHE_LEN = 2048
"""Positions the talker's StaticCache is preallocated for.
Read by both ``_init_kv_cache`` and ``model_local_kv_specs`` so the
declaration cannot drift from the allocation.
"""
def __init__(
self,
config,
@@ -609,6 +622,49 @@ class MingAudioGenerator:
# trailing latent frames prepended on each decode call
self._vae_decode_pad_frames = 32
def model_local_kv_specs(self) -> list[ModelLocalKVSpec]:
"""Declare the talker's preallocated decode cache.
This is the case that a static table cannot hold. ``self._llm_config``
is a ``Qwen2Config`` resolved from the checkpoint at load time, so
layers, kv-heads and head-dim do not exist anywhere in this repo, and
the dtype is whatever the weights loaded as. Reading them off the same
objects ``_init_kv_cache`` builds from is the only way to get a real
number.
Unlike the other three this is not a ceiling that a short utterance
stays under. transformers v5 initializes ``StaticLayer`` lazily, but
the first write to a layer allocates all ``max_cache_len`` positions at
once rather than growing, so a prefill brings the full extent into
existence on step 0 of every generation.
Width is fixed at one row, not ``max_num_seqs``. ``forward`` takes only
``runtime_additional_information[0]`` and runs the whole AR loop inline
before returning, so one worker holds one cache no matter how many
sequences the scheduler admits. Declaring this per-sequence -- which an
earlier revision did, by deriving the multiplier from scope -- reported
``max_num_seqs`` times the real figure.
The caveat is that nothing enforces the serialization. ``StaticCache``
is built outside ``_sampler_pool``'s lock, so overlapping calls to
``generate_latents`` would coexist. If this model ever gains a
concurrent entry point, this row count is the line to revisit.
"""
return [
spec_from_hf_config(
self._llm_config,
name="talker_llm_static_cache",
dtype=next(self._model.parameters()).dtype,
physical_capacity_positions=self._STATIC_CACHE_LEN,
capacity_source=f"hardcoded max_cache_len={self._STATIC_CACHE_LEN} in _init_kv_cache",
scope=ModelLocalKVScope.INVOCATION,
rows=RowDriver.FIXED,
rows_fixed=1,
rows_reason="forward() handles one request and runs the AR loop inline",
allocation_note="full extent on first write per layer, not grown; rebuilt per text segment",
)
]
@cached_property
def _sampler_pool(self) -> CFMGraphExecutorPool | None:
device = next(self._model.parameters()).device
@@ -790,7 +846,7 @@ class MingAudioGenerator:
def _init_kv_cache(
self, use_static_cache: bool, device: torch.device, dtype: torch.dtype
) -> tuple[StaticCache | None, int]:
max_cache_len = 2048
max_cache_len = self._STATIC_CACHE_LEN
if not use_static_cache:
return None, max_cache_len
cache = StaticCache(
@@ -136,6 +136,13 @@ def _encode_tokens(tokenizer: Any, prompt: str) -> list[int]:
from vllm.utils.tensor_schema import TensorSchema, TensorShape
from vllm_omni.model_executor.models.model_local_kv import (
ModelLocalKVScope,
ModelLocalKVSpec,
RowDriver,
spec_from_hf_config,
)
logger = init_logger(__name__)
hf_logger = logging.get_logger(__name__)
@@ -2621,6 +2628,51 @@ class MiniCPMWhisperEncoder(WhisperEncoder):
[MiniCPMWhisperEncoderLayer(config, layer_idx=i) for i in range(config.encoder_layers)]
)
def model_local_kv_specs(self) -> list[ModelLocalKVSpec]:
"""Declare the streaming encoder's self-attention cache.
Bounded by learned position embeddings, not by ``max_model_len``: the
cache cannot outgrow ``embed_positions``, and past that point the
forward pass repeats the last position rather than extending.
Whisper is encoder-only self-attention here, so kv-heads equal
attention heads. Layer count is read off the built ``self.layers``
rather than the config, so a partially built encoder reports what it
actually has.
Declared per session, not per sequence. Each streaming session state
owns its own ``audio_past_key_values`` and the encoder runs at batch
size one, so ``max_num_seqs`` is the wrong number here.
One session is the unit rather than the configured cap, because the cap
is not visible from the model and the setting that carries it cannot
distinguish "duplex off" from "one session" -- so a cap-scaled figure
would report a cache that a non-duplex deployment never allocates. An
earlier revision grew a capacity driver and an engine-side field to
carry that number for this one declarer; the arithmetic is a
multiplication the reader can do when they need the ceiling.
"""
return [
spec_from_hf_config(
self.config,
name="whisper_encoder_self_attn",
dtype=self.conv1.weight.dtype,
layers=len(self.layers),
kv_heads=self.config.encoder_attention_heads,
head_dim=self.config.d_model // self.config.encoder_attention_heads,
physical_capacity_positions=int(self.embed_positions.weight.shape[0]),
capacity_source="embed_positions rows (max_source_positions)",
scope=ModelLocalKVScope.SESSION,
rows=RowDriver.FIXED,
rows_fixed=1,
rows_reason="one EncoderDecoderCache per streaming session, encoder runs at batch size 1",
allocation_note=(
"only allocated on the streaming path, where use_cache is set; scales with the "
"configured duplex session cap"
),
)
]
def forward(
self,
input_features,
@@ -0,0 +1,347 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Declaration protocol for attention KV kept outside the paged manager.
Several models hold their own attention KV as HuggingFace ``transformers`` cache
objects rather than using the engine's paged KV manager. That memory is
allocated after the profiling run that decides how much KV the engine may claim,
so no per-stage footprint number accounts for it, and nothing catches a cache
that silently stops working.
This module lets a model *declare* what it holds. It describes; it never
allocates. Allocation for the diffusion path is owned by the DiT KV manager
(RFC #5244 / PR #6094) and this protocol must not grow into a second allocator.
Why a runtime query rather than a static table: geometry is frequently not
knowable until weights are loaded. ``ming_flash_omni``'s talker builds its cache
from a ``Qwen2Config`` that comes from the checkpoint, so layers, kv-heads,
head-dim and dtype simply do not exist in this repo. A table would have a hole
in it; a post-load query does not.
Why capacity is a resolved number rather than a formula: the four known caches
are each bounded by a different mechanism -- a sliding window, a decode-loop
trip count, an encoder frame limit, a hardcoded constant -- and not one of them
is ``max_model_len``. Only the model knows its own bound.
"""
from __future__ import annotations
from collections.abc import Sequence
from dataclasses import dataclass
from enum import Enum
from typing import Protocol, runtime_checkable
import torch
from vllm.logger import init_logger
logger = init_logger(__name__)
__all__ = [
"HasModelLocalKV",
"ModelLocalKVScope",
"ModelLocalKVSpec",
"RowDriver",
"collect_model_local_kv_specs",
"spec_from_hf_config",
]
class ModelLocalKVScope(str, Enum):
"""How long one allocation lives.
Diagnostic only. Lifetime and size are independent: a per-call cache can be
the widest thing a model owns, and a process-lifetime one can be a single
row. ``ModelLocalKVSpec.rows`` carries the size; this carries only the
reader's mental model of when the memory appears and goes.
"""
INVOCATION = "invocation"
"""Dies when the call returns (e.g. a per-step working copy)."""
REQUEST = "request"
"""Retained across steps of one request, released when it finishes."""
SESSION = "session"
"""Outlives a request; belongs to a duplex/streaming session."""
MODEL = "model"
"""Lives as long as the model (e.g. captured into a CUDA-graph pool)."""
class RowDriver(str, Enum):
"""What sets how many rows of a cache are live at peak.
Two values, because two are all the known caches need. A third belongs
here only when a model actually has a cache the engine widens by some
other number -- adding one speculatively costs a branch everywhere and
buys nothing.
"""
FIXED = "fixed"
"""A count the model controls: a captured-bucket sum, a serialized call path."""
MAX_NUM_SEQS = "max_num_seqs"
"""One row per in-flight sequence. Only the engine knows the value."""
@dataclass(frozen=True)
class ModelLocalKVSpec:
"""One declared KV allocation.
A model returns one entry per *distinct lifetime*, not per cache object: if
the same geometry exists as both a retained per-request cache and a
short-lived working copy, that is two entries.
"""
name: str
layers: int
kv_heads: int
head_dim: int
dtype: torch.dtype
physical_capacity_positions: int
"""Positions the tensor can physically hold.
Not the logical sequence position. A sliding-window cache truncates on every
write, so a stream of thousands of tokens may only ever occupy
``sliding_window - 1`` slots.
"""
capacity_source: str
"""Free-text note on where the bound comes from, for diagnostics only.
Never branch on this. It exists so a reader can tell "71 because sliding
window" from "2048 because someone typed 2048", which is otherwise
invisible.
"""
scope: ModelLocalKVScope
"""How long one allocation lives. Diagnostic only.
Deliberately not the multiplier. An earlier revision derived scaling from
scope, which made every non-``MODEL`` cache multiply by ``max_num_seqs``.
That over-reported ming by that factor (its calls are serialized) and used
the wrong driver entirely for MiniCPM-o (duplex sessions, not sequences).
Lifetime does not imply replication topology; ``rows`` says what does.
"""
rows: RowDriver
"""What sets how many rows of this shape are live at peak.
A row is one batch entry's worth of the geometry above. Deliberately does
not distinguish "one allocation N rows wide" from "N allocations of one
row": those cost the same, and an earlier revision that tried to carry both
got the object topology wrong in three of four declarers. Bytes are what
this protocol reports; describe the layout in ``allocation_note``.
Required, because defaulting it silently understates every cache whose
multiplicity is not 1.
"""
rows_fixed: int = 1
"""The count, when ``rows`` is ``FIXED``. Ignored otherwise."""
rows_reason: str = ""
"""Why ``rows_fixed`` is that number. Required when ``rows`` is ``FIXED``.
A bare constant cannot be reviewed. ``rows_fixed=1`` on a cache that looks
per-request is exactly the claim a reader has to be able to check.
"""
allocation_note: str | None = None
"""Optional detail such as "rebuilt per text segment". Diagnostic only."""
def __post_init__(self) -> None:
if self.rows is RowDriver.FIXED:
if self.rows_fixed < 1:
raise ValueError(f"rows_fixed must be >= 1, got {self.rows_fixed}")
if not self.rows_reason.strip():
raise ValueError("a FIXED row count needs rows_reason; an unexplained constant cannot be reviewed")
@property
def bytes_per_row(self) -> int:
"""Bytes for a single batch row of this cache."""
return (
2 # K and V
* self.layers
* self.kv_heads
* self.head_dim
* self.physical_capacity_positions
* torch.empty((), dtype=self.dtype).element_size()
)
def row_count(self, max_num_seqs: int = 1) -> int:
"""Rows live at peak, given the engine's concurrency."""
if self.rows is RowDriver.FIXED:
return self.rows_fixed
return max(1, max_num_seqs)
def peak_bytes(self, max_num_seqs: int = 1) -> int:
"""Peak bytes once the engine's concurrency is known.
The model supplies geometry and names the driver; only the engine knows
the number behind it.
"""
return self.bytes_per_row * self.row_count(max_num_seqs)
def _resolve(config: object, candidates: Sequence[str], what: str) -> int:
for attr in candidates:
value = getattr(config, attr, None)
if value is not None:
return int(value)
raise ValueError(
f"Cannot determine {what} from {type(config).__name__}: tried "
f"{', '.join(candidates)}. Pass it explicitly to spec_from_hf_config()."
)
def spec_from_hf_config(
config: object,
*,
name: str,
physical_capacity_positions: int,
capacity_source: str,
scope: ModelLocalKVScope,
dtype: torch.dtype,
rows: RowDriver,
rows_fixed: int = 1,
rows_reason: str = "",
allocation_note: str | None = None,
layers: int | None = None,
kv_heads: int | None = None,
head_dim: int | None = None,
) -> ModelLocalKVSpec:
"""Build a spec from the same config object the cache is built from.
Every known model-local cache is constructed from an HF config, so the
geometry half of a declaration is the same three lookups each time. Pass
``config`` and this derives them; pass ``layers``/``kv_heads``/``head_dim``
explicitly to override when the config uses encoder-decoder naming or when
the built module is a more truthful source than the config.
Raises rather than guessing when an attribute is absent: a silently wrong
geometry would under-report, which is the failure this protocol exists to
prevent. ``rows`` is keyword-required for the same reason -- an earlier
revision defaulted the batch extent to 1 here while documenting it as
required on the dataclass, so every declarer that used this helper got the
default and the guarantee was decorative.
"""
if layers is None:
layers = _resolve(config, ("num_hidden_layers", "encoder_layers", "num_layers"), "layer count")
if kv_heads is None:
kv_heads = _resolve(
config,
("num_key_value_heads", "num_attention_heads", "encoder_attention_heads"),
"kv head count",
)
if head_dim is None:
explicit = getattr(config, "head_dim", None)
if explicit is not None:
head_dim = int(explicit)
else:
hidden = _resolve(config, ("hidden_size", "d_model"), "hidden size")
heads = _resolve(config, ("num_attention_heads", "encoder_attention_heads"), "attention head count")
head_dim = hidden // heads
return ModelLocalKVSpec(
name=name,
layers=layers,
kv_heads=kv_heads,
head_dim=head_dim,
dtype=dtype,
physical_capacity_positions=physical_capacity_positions,
capacity_source=capacity_source,
scope=scope,
rows=rows,
rows_fixed=rows_fixed,
rows_reason=rows_reason,
allocation_note=allocation_note,
)
@runtime_checkable
class HasModelLocalKV(Protocol):
"""Implemented by whichever object owns the cache.
Implement it on the owner, not on the registered model. The owner is
usually several levels down (a codec decoder, a talker backbone), and
``collect_model_local_kv_specs`` finds it by walking the module tree, so
no intermediate class has to forward anything.
"""
def model_local_kv_specs(self) -> Sequence[ModelLocalKVSpec]: ...
def _unwrap_to_module(model: object) -> object:
"""Unwrap runner-side wrappers down to the real module tree.
``vllm.compilation.cuda_graph.CUDAGraphWrapper`` is a plain callable, not
an ``nn.Module``, but its ``__getattr__`` forwards anything the runnable
has -- including ``named_modules``. So a check for ``named_modules`` does
not detect the wrapper, and the wrapper and its runnable both answer to the
same attributes. That matters twice over: it means walking the wrapper
happens to work, and it means a naive walk can enter the tree as both the
wrapper and the root module and count a root-level declarer twice.
Prefer the supported ``unwrap()`` accessor, which returns the runnable
directly.
"""
seen: set[int] = set()
current = model
while current is not None and id(current) not in seen:
seen.add(id(current))
unwrap = getattr(current, "unwrap", None)
if not callable(unwrap):
return current
try:
nxt = unwrap()
except Exception:
return current
if nxt is None or nxt is current:
return current
current = nxt
return model
def collect_model_local_kv_specs(model: object) -> list[tuple[str, ModelLocalKVSpec]]:
"""Collect every declaration in a loaded model, with the owner's path.
Walks ``named_modules()`` because the cache owner is an inner module in all
known cases. Requiring each registered model to forward the call would put
the burden on classes that do not own a cache and would silently report
zero the moment someone forgot -- exactly the failure mode this is meant to
surface.
A raising declaration is logged and skipped rather than propagated:
reporting memory must not be able to break model load.
"""
root = _unwrap_to_module(model)
owners: list[tuple[str, object]] = []
seen_ids: set[int] = set()
if isinstance(root, HasModelLocalKV):
owners.append(("", root))
seen_ids.add(id(root))
named_modules = getattr(root, "named_modules", None)
if callable(named_modules):
for path, module in named_modules():
if id(module) in seen_ids or not isinstance(module, HasModelLocalKV):
continue
seen_ids.add(id(module))
owners.append((path, module))
collected: list[tuple[str, ModelLocalKVSpec]] = []
for path, owner in owners:
try:
specs = owner.model_local_kv_specs()
except Exception:
logger.warning(
"model_local_kv_specs() raised on %s; skipping its declaration",
type(owner).__name__,
exc_info=True,
)
continue
collected.extend((path, spec) for spec in specs)
return collected
@@ -17,6 +17,13 @@ from transformers.cache_utils import DynamicCache
from vllm.logger import init_logger
from vllm.platforms import current_platform
from vllm_omni.model_executor.models.model_local_kv import (
ModelLocalKVScope,
ModelLocalKVSpec,
RowDriver,
spec_from_hf_config,
)
logger = init_logger(__name__)
@@ -166,6 +173,85 @@ class CUDAGraphDecoderWrapper:
if self._stats_log_every > 0 and self._stats_total_requests % self._stats_log_every == 0:
self.log_decode_stats()
@staticmethod
def _retained_kv_caches(states: dict) -> list[tuple[int, DynamicCache]]:
"""Find the KV caches a capture kept alive, with their batch rows.
Walks the stored ``cache`` payload instead of assuming which capture
path populates it: ``_decode_icl_first_chunk`` fills its dict on the
decoder side, so the only reliable count is the one taken from the
objects themselves after warmup.
"""
found: list[tuple[int, DynamicCache]] = []
for state in states.values():
payload = state.get("cache")
if not isinstance(payload, dict):
continue
for value in payload.values():
if not isinstance(value, DynamicCache):
continue
layers = getattr(value, "layers", None)
keys = getattr(layers[0], "keys", None) if layers else None
if keys is None:
continue
found.append((int(keys.shape[0]), value))
return found
def model_local_kv_specs(self) -> list[ModelLocalKVSpec]:
"""Declare only the graph-resident copies this wrapper retains.
The per-decode cache is declared by the decoder itself, because it
exists whether or not graphs were captured. What is specific to this
wrapper is that every captured shape keeps its dummy ``DynamicCache``
alive for replay (``combined_states``, ``icl_prefix_states`` and
``xvec_prefix_states`` all store ``cache``), for the process lifetime.
Rows come from the state dicts rather than the configured capture
lists: a capture that raised is logged and skipped, so the configured
shape count would over-report. The trade is that a capture whose
tensors transformers has not yet materialized contributes nothing, so
this is a floor rather than a contract.
"""
config = self.decoder.config
try:
dtype = next(self.decoder.parameters()).dtype
except StopIteration: # parameterless stub, only reachable in tests
return []
# Physical, not logical: the sliding window truncates on every write,
# so a stream of thousands of frames never occupies more than this.
physical = self.prefix_length - 1
retained = [
*self._retained_kv_caches(self.combined_states),
*self._retained_kv_caches(self.icl_prefix_states),
*self._retained_kv_caches(self.xvec_prefix_states),
]
batched_captures = sum(rows for rows, _ in retained)
if not batched_captures:
return []
return [
spec_from_hf_config(
config,
name="codec_decoder_graph_pool",
dtype=dtype,
physical_capacity_positions=physical,
capacity_source=f"sliding_window({self.prefix_length}) - 1",
scope=ModelLocalKVScope.MODEL,
rows=RowDriver.FIXED,
rows_fixed=batched_captures,
rows_reason=(
f"rows across {len(retained)} retained caches in "
f"{len(self.combined_states)} suffix / {len(self.icl_prefix_states)} icl-prefix / "
f"{len(self.xvec_prefix_states)} xvec-prefix captures"
),
allocation_note=(
"counted from caches transformers has materialized; captures whose tensors are still "
"unallocated report nothing, so this is a floor for the xvec path"
),
)
]
def log_decode_stats(self) -> None:
if not getattr(self, "_stats_enabled", False) or self._stats_total_requests == 0:
return
@@ -854,6 +854,75 @@ class Qwen3TTSTokenizerV2Decoder(Qwen3TTSTokenizerV2DecoderPreTrainedModel):
self._cudagraph_enabled = False
self._cudagraph_wrapper = None
def model_local_kv_specs(self):
"""Declare this decoder's attention KV.
Only the async-chunk path holds KV. In stateless mode the decoder runs
``_forward_exact``, which calls ``pre_transformer`` without
``use_cache`` and allocates no cache at all, so declaring a per-request
cache there invents memory that does not exist. The previous revision
did exactly that and reported over a gigabyte for stateless
deployments; the one before it hung the declaration off the CUDA-graph
wrapper and so reported zero under ``enforce_eager``, where the eager
async-chunk path does allocate. Both mistakes come from tying this to
the wrong condition.
Two entries when async-chunk is on: the retained per-request cache, and
the working copy ``_decode_icl_first_chunk`` deep-copies from it, which
is live at the same time.
The wrapper, when present, adds the graph-resident copies. It is a
plain object rather than a submodule, so the module-tree walk cannot
reach it; this is the hop it needs.
"""
from vllm_omni.model_executor.models.model_local_kv import (
ModelLocalKVScope,
RowDriver,
spec_from_hf_config,
)
config = self.config
try:
dtype = next(self.parameters()).dtype
except StopIteration: # parameterless stub, only reachable in tests
return []
window = int(getattr(config, "sliding_window", 0) or 0)
if window <= 1:
return []
specs = []
wrapper = getattr(self, "_cudagraph_wrapper", None)
# async_chunk is the wrapper's record of the mode; without a wrapper the
# decoder has not been told, and only the stateless path is reachable.
if wrapper is not None and getattr(wrapper, "async_chunk", False):
common = dict(
config=config,
dtype=dtype,
physical_capacity_positions=window - 1,
capacity_source=f"sliding_window({window}) - 1",
rows=RowDriver.MAX_NUM_SEQS,
)
specs.append(
spec_from_hf_config(
name="codec_decoder_request",
scope=ModelLocalKVScope.REQUEST,
allocation_note="retained across chunks of one request",
**common,
)
)
specs.append(
spec_from_hf_config(
name="codec_decoder_working_copy",
scope=ModelLocalKVScope.INVOCATION,
allocation_note="deep-copied from the retained cache each chunk; live alongside it",
**common,
)
)
if wrapper is not None:
specs.extend(wrapper.model_local_kv_specs())
return specs
def precompute_snake_caches(self):
"""Precompute exp(alpha) and 1/(exp(beta)+eps) for all SnakeBeta modules."""
count = 0