[Refactor] Declare model-local KV held outside the paged manager (#6171)
Signed-off-by: Yueqian Lin <linyueqian@outlook.com>
This commit is contained in:
1 parent
5a386d789f
commit
5c69fc2a8a
12 files changed
+1283
-3
No files matched your search
@@ -58,6 +58,12 @@ from vllm.sequence import IntermediateTensors
|
||||
from vllm.utils.tensor_schema import TensorSchema
|
||||
|
||||
from vllm_omni.model_executor.models.mimo_audio.config_mimo_audio import MiMoAudioConfig
|
||||
from vllm_omni.model_executor.models.model_local_kv import (
|
||||
ModelLocalKVScope,
|
||||
ModelLocalKVSpec,
|
||||
RowDriver,
|
||||
spec_from_hf_config,
|
||||
)
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -744,6 +750,67 @@ class MiMoAudioLLMForConditionalGeneration(nn.Module, SupportsMultiModal, Suppor
|
||||
def get_language_model(self) -> torch.nn.Module:
|
||||
return self.model
|
||||
|
||||
def model_local_kv_specs(self) -> list[ModelLocalKVSpec]:
|
||||
"""Declare the local transformer's decode cache and its graph copies.
|
||||
|
||||
The cache is created per call inside ``base_local_forward`` and is
|
||||
bounded by the decode loop, not by sequence length: the loop runs
|
||||
``group_size + max(delay_pattern)`` steps regardless of how long the
|
||||
utterance is.
|
||||
|
||||
The second entry is the one that matters. ``base_local_forward`` runs
|
||||
inside ``torch.cuda.graph`` during capture, so each captured bucket
|
||||
leaves a cache's worth of tensors pinned in the graph pool for the
|
||||
process lifetime. There is no Python object left to inspect -- the
|
||||
cache is a local in the captured frame -- so the width comes from the
|
||||
buckets that captured successfully, which is why it reads
|
||||
``local_forward_cg_by_bs`` rather than MIMO_CUDAGRAPH_BATCH_SIZES: a
|
||||
bucket whose capture raised is logged and skipped.
|
||||
|
||||
Both entries are declared, and summing them over-states a single step:
|
||||
a given call either replays a captured bucket or runs eagerly, never
|
||||
both. The graph pool is resident regardless, so the sum is the right
|
||||
ceiling for the process and the wrong number for one request.
|
||||
"""
|
||||
delay_iters = self.group_size + max(self.delay_pattern)
|
||||
dtype = next(self.local_transformer.parameters()).dtype
|
||||
bound = f"group_size({self.group_size}) + max(delay_pattern)({max(self.delay_pattern)})"
|
||||
specs = [
|
||||
spec_from_hf_config(
|
||||
self.local_config,
|
||||
name="local_transformer",
|
||||
dtype=dtype,
|
||||
physical_capacity_positions=delay_iters,
|
||||
capacity_source=bound,
|
||||
scope=ModelLocalKVScope.INVOCATION,
|
||||
# One cache, B rows wide -- not B caches. base_local_forward
|
||||
# builds a single DynamicCache whose batch dimension is the
|
||||
# number of requests in the group.
|
||||
rows=RowDriver.MAX_NUM_SEQS,
|
||||
allocation_note="one batched allocation per eager call; graph replay uses the pool entry instead",
|
||||
)
|
||||
]
|
||||
captured_rows = sum(self.local_forward_cg_by_bs)
|
||||
if captured_rows:
|
||||
specs.append(
|
||||
spec_from_hf_config(
|
||||
self.local_config,
|
||||
name="local_transformer_graph_pool",
|
||||
dtype=dtype,
|
||||
physical_capacity_positions=delay_iters,
|
||||
capacity_source=bound,
|
||||
scope=ModelLocalKVScope.MODEL,
|
||||
rows=RowDriver.FIXED,
|
||||
rows_fixed=captured_rows,
|
||||
rows_reason=f"sum of captured buckets {sorted(self.local_forward_cg_by_bs)}",
|
||||
allocation_note=(
|
||||
"pinned in the CUDA graph pool; capture is gated only on cuda.is_available(), "
|
||||
"so this stays resident under enforce_eager too"
|
||||
),
|
||||
)
|
||||
)
|
||||
return specs
|
||||
|
||||
def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings:
|
||||
if kwargs.get("modality_preprocess") is None:
|
||||
mm_dummy_embeddings = []
|
||||
|
||||
Reference in new issue
Block a user