[Diffusion] Add native KV cache initialization and Scheduler-managed block allocation (#6094)
Signed-off-by: zwhzzz0821 <zwhzzz0821@users.noreply.github.com> Co-authored-by: zwhzzz0821 <zwhzzz0821@users.noreply.github.com>
This commit is contained in:
1 parent
b283215c28
commit
0a1846aa45
33 files changed
+2609
-265
No files matched your search
@@ -11,6 +11,7 @@ model-related operations.
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import gc
|
||||
import time
|
||||
from collections.abc import Callable
|
||||
from contextlib import AbstractContextManager, nullcontext
|
||||
@@ -21,6 +22,7 @@ from torch.profiler import record_function
|
||||
from vllm.config import LoadConfig, VllmConfig
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.mem_utils import DeviceMemoryProfiler, GiB_bytes
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig, KVCacheSpec
|
||||
|
||||
from vllm_omni.diffusion.cache.cachedit import cache_summary
|
||||
from vllm_omni.diffusion.cache.prompt_embed_cache import (
|
||||
@@ -45,8 +47,10 @@ from vllm_omni.diffusion.offloader import get_offload_backend
|
||||
from vllm_omni.diffusion.registry import _NO_CACHE_ACCELERATION
|
||||
from vllm_omni.diffusion.request import OmniDiffusionRequest
|
||||
from vllm_omni.diffusion.sched.interface import (
|
||||
CachedRequestData,
|
||||
DiffusionSchedulerOutput,
|
||||
KVPrefetchJob,
|
||||
NewRequestData,
|
||||
validate_new_request_data_identity,
|
||||
)
|
||||
from vllm_omni.diffusion.worker.input_batch import InputBatch, scatter_latents
|
||||
@@ -139,6 +143,11 @@ class DiffusionModelRunner(OmniConnectorModelRunnerMixin):
|
||||
self.offload_backend: Any | None = None
|
||||
self.prompt_embed_cache: Any | None = None
|
||||
self.input_batch: InputBatch | None = None
|
||||
self.model_memory_usage = 0
|
||||
# Rank-local native config generated by the Engine. Physical tensors
|
||||
# and BlockTable binding remain the responsibility of the paged
|
||||
# data-plane initialization path.
|
||||
self.kv_cache_config: KVCacheConfig | None = None
|
||||
|
||||
# Cache for per-request stepwise state.
|
||||
self.state_cache: dict[str, StepRequestState] = {}
|
||||
@@ -167,8 +176,8 @@ class DiffusionModelRunner(OmniConnectorModelRunnerMixin):
|
||||
metadata: DiffusionKVMetadata | None,
|
||||
) -> None:
|
||||
cache_mode = getattr(self.od_config, "diffusion_kv_mode", DiffusionKVCacheMode.DENSE_LEGACY)
|
||||
# W0 does not wire the Scheduler allocator or Worker installer yet, so
|
||||
# validate an allocation snapshot when present without requiring one.
|
||||
if cache_mode is DiffusionKVCacheMode.PAGED_SCHEDULER and metadata is None:
|
||||
raise ValueError(f"paged_scheduler request {request_id!r} requires Diffusion KV metadata")
|
||||
if cache_mode is not DiffusionKVCacheMode.PAGED_SCHEDULER and metadata is not None:
|
||||
raise ValueError(f"{cache_mode.value} request {request_id!r} must not carry Diffusion KV metadata")
|
||||
|
||||
@@ -273,6 +282,7 @@ class DiffusionModelRunner(OmniConnectorModelRunnerMixin):
|
||||
m.consumed_memory / GiB_bytes,
|
||||
time_after_load - time_before_load,
|
||||
)
|
||||
self.model_memory_usage = int(m.consumed_memory)
|
||||
logger.info("Model runner: Model loaded successfully.")
|
||||
|
||||
if self.od_config.streaming_output and not getattr(self.od_config, "step_execution", False):
|
||||
@@ -368,6 +378,45 @@ class DiffusionModelRunner(OmniConnectorModelRunnerMixin):
|
||||
|
||||
logger.info("Model runner: Initialization complete.")
|
||||
|
||||
def get_kv_cache_spec(self) -> dict[str, KVCacheSpec]:
|
||||
"""Collect native specs from cache-enabled loaded attention modules."""
|
||||
|
||||
from vllm_omni.diffusion.attention.layer import Attention
|
||||
from vllm_omni.diffusion.diffusion_kv.config import DiffusionKVCacheMode
|
||||
|
||||
if self.od_config.diffusion_kv_mode is not DiffusionKVCacheMode.PAGED_SCHEDULER:
|
||||
return {}
|
||||
if self.pipeline is None:
|
||||
raise RuntimeError("Model must be loaded before collecting Diffusion KV cache specs")
|
||||
|
||||
specs: dict[str, KVCacheSpec] = {}
|
||||
for layer_name, module in self.pipeline.named_modules():
|
||||
if not isinstance(module, Attention):
|
||||
continue
|
||||
spec = module.get_kv_cache_spec(self.vllm_config)
|
||||
if spec is not None:
|
||||
specs[layer_name] = spec
|
||||
if not specs:
|
||||
raise RuntimeError(
|
||||
"paged_scheduler Diffusion KV found no cache-enabled Attention modules "
|
||||
f"in {type(self.pipeline).__name__}"
|
||||
)
|
||||
return specs
|
||||
|
||||
def set_kv_cache_config(self, kv_cache_config: KVCacheConfig) -> None:
|
||||
"""Retain the Engine-generated rank-local config for the data plane."""
|
||||
|
||||
configured_layers = {
|
||||
layer_name for group in kv_cache_config.kv_cache_groups for layer_name in group.layer_names
|
||||
}
|
||||
expected_layers = set(self.get_kv_cache_spec())
|
||||
if configured_layers != expected_layers:
|
||||
raise ValueError(
|
||||
"Rank-local Diffusion KVCacheConfig layer mismatch: "
|
||||
f"expected={sorted(expected_layers)}, configured={sorted(configured_layers)}"
|
||||
)
|
||||
self.kv_cache_config = kv_cache_config
|
||||
|
||||
def clear_prompt_embed_cache(self) -> None:
|
||||
"""Evict all cached text-encoder outputs (e.g. between training epochs).
|
||||
|
||||
@@ -516,6 +565,7 @@ class DiffusionModelRunner(OmniConnectorModelRunnerMixin):
|
||||
require_request_batch_support: bool,
|
||||
kv_prefetch_job: KVPrefetchJob | None = None,
|
||||
record_name: str,
|
||||
record_output_peak_memory: bool = True,
|
||||
) -> BatchRunnerOutput:
|
||||
assert self.pipeline is not None, "Model not loaded. Call load_model() first."
|
||||
if not reqs:
|
||||
@@ -545,7 +595,7 @@ class DiffusionModelRunner(OmniConnectorModelRunnerMixin):
|
||||
|
||||
batch = DiffusionRequestBatch(requests=reqs)
|
||||
is_primary = not torch.distributed.is_initialized() or torch.distributed.get_rank() == 0
|
||||
if is_primary:
|
||||
if is_primary and record_output_peak_memory:
|
||||
current_omni_platform.reset_peak_memory_stats()
|
||||
|
||||
with set_forward_context(vllm_config=self.vllm_config, omni_diffusion_config=od_config):
|
||||
@@ -558,7 +608,7 @@ class DiffusionModelRunner(OmniConnectorModelRunnerMixin):
|
||||
pipeline_name=type(self.pipeline).__name__,
|
||||
)
|
||||
|
||||
if is_primary and outputs:
|
||||
if is_primary and outputs and record_output_peak_memory:
|
||||
batch_peak_memory_mb = self._sample_peak_memory_mb()
|
||||
for output in outputs:
|
||||
output.peak_memory_mb = max(output.peak_memory_mb, batch_peak_memory_mb)
|
||||
@@ -626,6 +676,59 @@ class DiffusionModelRunner(OmniConnectorModelRunnerMixin):
|
||||
assert output is not None
|
||||
return output
|
||||
|
||||
def profile_run(self, requests: list[OmniDiffusionRequest]) -> None:
|
||||
"""Run the maximum per-rank request batch for memory profiling.
|
||||
|
||||
This deliberately bypasses Scheduler admission and Diffusion KV
|
||||
metadata validation because cache capacity has not been sized yet.
|
||||
It otherwise uses the normal execution-mode path so model inputs,
|
||||
collective communication, backend workspaces, denoising, and decode
|
||||
allocations contribute to the observed peak. Step execution profiles
|
||||
one fused ``InputBatch`` instead of sequential single-request forwards.
|
||||
"""
|
||||
|
||||
if not requests:
|
||||
raise ValueError("Diffusion memory profiling requires at least one request.")
|
||||
|
||||
runner_output: BatchRunnerOutput | None = None
|
||||
request_ids = [request.request_id for request in requests]
|
||||
try:
|
||||
if getattr(self.od_config, "step_execution", False):
|
||||
scheduler_output = DiffusionSchedulerOutput(
|
||||
step_id=0,
|
||||
scheduled_new_reqs=[
|
||||
NewRequestData(request_id=request.request_id, req=request) for request in requests
|
||||
],
|
||||
scheduled_cached_reqs=CachedRequestData.make_empty(),
|
||||
finished_req_ids=set(),
|
||||
num_running_reqs=len(requests),
|
||||
num_waiting_reqs=0,
|
||||
)
|
||||
runner_output = self._execute_stepwise(
|
||||
scheduler_output,
|
||||
validate_kv_metadata=False,
|
||||
record_output_peak_memory=False,
|
||||
)
|
||||
else:
|
||||
runner_output = self._execute_request_list(
|
||||
requests,
|
||||
od_config=self.od_config,
|
||||
allow_single_output=len(requests) == 1,
|
||||
require_request_batch_support=len(requests) > 1,
|
||||
record_name="pipeline_memory_profile",
|
||||
# The enclosing native memory_profiling context owns the
|
||||
# peak counters. Resetting them here would discard request
|
||||
# preparation allocations and understate the budget.
|
||||
record_output_peak_memory=False,
|
||||
)
|
||||
current_omni_platform.synchronize()
|
||||
finally:
|
||||
for request_id in request_ids:
|
||||
self.state_cache.pop(request_id, None)
|
||||
self.input_batch = None
|
||||
del runner_output
|
||||
gc.collect()
|
||||
|
||||
def execute_model_batch(
|
||||
self,
|
||||
scheduler_output: DiffusionSchedulerOutput,
|
||||
@@ -750,13 +853,29 @@ class DiffusionModelRunner(OmniConnectorModelRunnerMixin):
|
||||
|
||||
def execute_stepwise(self, scheduler_output: DiffusionSchedulerOutput) -> BatchRunnerOutput:
|
||||
"""Execute one step for one scheduled request and return runner output."""
|
||||
return self._execute_stepwise(
|
||||
scheduler_output,
|
||||
validate_kv_metadata=True,
|
||||
record_output_peak_memory=True,
|
||||
)
|
||||
|
||||
def _execute_stepwise(
|
||||
self,
|
||||
scheduler_output: DiffusionSchedulerOutput,
|
||||
*,
|
||||
validate_kv_metadata: bool,
|
||||
record_output_peak_memory: bool,
|
||||
) -> BatchRunnerOutput:
|
||||
"""Execute one step with explicit validation and profiling policy."""
|
||||
|
||||
assert self.pipeline is not None, "Model not loaded. Call load_model() first."
|
||||
for new_req in scheduler_output.scheduled_new_reqs:
|
||||
validate_new_request_data_identity(new_req)
|
||||
self._validate_diffusion_kv_metadata(
|
||||
request_id=new_req.req.request_id,
|
||||
metadata=new_req.diffusion_kv_metadata,
|
||||
)
|
||||
if validate_kv_metadata:
|
||||
self._validate_diffusion_kv_metadata(
|
||||
request_id=new_req.req.request_id,
|
||||
metadata=new_req.diffusion_kv_metadata,
|
||||
)
|
||||
if not self._supports_step_mode():
|
||||
raise ValueError("Current pipeline does not support step execution.")
|
||||
# Stepwise mode only supports the basic state-driven denoise path for now.
|
||||
@@ -771,7 +890,13 @@ class DiffusionModelRunner(OmniConnectorModelRunnerMixin):
|
||||
had_active_states = bool(self.state_cache)
|
||||
states, new_request_ids = self._update_states(scheduler_output)
|
||||
is_primary = not torch.distributed.is_initialized() or torch.distributed.get_rank() == 0
|
||||
if new_request_ids and not had_active_states and is_primary and current_omni_platform.is_available():
|
||||
if (
|
||||
record_output_peak_memory
|
||||
and new_request_ids
|
||||
and not had_active_states
|
||||
and is_primary
|
||||
and current_omni_platform.is_available()
|
||||
):
|
||||
current_omni_platform.reset_peak_memory_stats()
|
||||
input_batch = self._prepare_batch_inputs(states, new_request_ids)
|
||||
attn_metadata = {}
|
||||
@@ -864,7 +989,7 @@ class DiffusionModelRunner(OmniConnectorModelRunnerMixin):
|
||||
f"but batched noise_pred has {noise_pred.shape[0]} rows."
|
||||
)
|
||||
|
||||
if is_primary:
|
||||
if is_primary and record_output_peak_memory:
|
||||
batch_peak_memory_mb = self._sample_peak_memory_mb()
|
||||
states_by_id = {state.request_id: state for state in states}
|
||||
for state in states:
|
||||
|
||||
Reference in new issue
Block a user