[Diffusion] Add native KV cache initialization and Scheduler-managed block allocation (#6094)

Signed-off-by: zwhzzz0821 <zwhzzz0821@users.noreply.github.com>
Co-authored-by: zwhzzz0821 <zwhzzz0821@users.noreply.github.com>
This commit is contained in:
Yukim1andzwhzzz0821 authored and GitHub committed 2026-08-18 07:00:24 +00:00
1 parent b283215c28
commit 0a1846aa45
33 files changed
+2609 -265

No files matched your search

@@ -11,6 +11,7 @@ model-related operations.
from __future__ import annotations
import copy
import gc
import time
from collections.abc import Callable
from contextlib import AbstractContextManager, nullcontext
@@ -21,6 +22,7 @@ from torch.profiler import record_function
from vllm.config import LoadConfig, VllmConfig
from vllm.logger import init_logger
from vllm.utils.mem_utils import DeviceMemoryProfiler, GiB_bytes
from vllm.v1.kv_cache_interface import KVCacheConfig, KVCacheSpec
from vllm_omni.diffusion.cache.cachedit import cache_summary
from vllm_omni.diffusion.cache.prompt_embed_cache import (
@@ -45,8 +47,10 @@ from vllm_omni.diffusion.offloader import get_offload_backend
from vllm_omni.diffusion.registry import _NO_CACHE_ACCELERATION
from vllm_omni.diffusion.request import OmniDiffusionRequest
from vllm_omni.diffusion.sched.interface import (
CachedRequestData,
DiffusionSchedulerOutput,
KVPrefetchJob,
NewRequestData,
validate_new_request_data_identity,
)
from vllm_omni.diffusion.worker.input_batch import InputBatch, scatter_latents
@@ -139,6 +143,11 @@ class DiffusionModelRunner(OmniConnectorModelRunnerMixin):
self.offload_backend: Any | None = None
self.prompt_embed_cache: Any | None = None
self.input_batch: InputBatch | None = None
self.model_memory_usage = 0
# Rank-local native config generated by the Engine. Physical tensors
# and BlockTable binding remain the responsibility of the paged
# data-plane initialization path.
self.kv_cache_config: KVCacheConfig | None = None
# Cache for per-request stepwise state.
self.state_cache: dict[str, StepRequestState] = {}
@@ -167,8 +176,8 @@ class DiffusionModelRunner(OmniConnectorModelRunnerMixin):
metadata: DiffusionKVMetadata | None,
) -> None:
cache_mode = getattr(self.od_config, "diffusion_kv_mode", DiffusionKVCacheMode.DENSE_LEGACY)
# W0 does not wire the Scheduler allocator or Worker installer yet, so
# validate an allocation snapshot when present without requiring one.
if cache_mode is DiffusionKVCacheMode.PAGED_SCHEDULER and metadata is None:
raise ValueError(f"paged_scheduler request {request_id!r} requires Diffusion KV metadata")
if cache_mode is not DiffusionKVCacheMode.PAGED_SCHEDULER and metadata is not None:
raise ValueError(f"{cache_mode.value} request {request_id!r} must not carry Diffusion KV metadata")
@@ -273,6 +282,7 @@ class DiffusionModelRunner(OmniConnectorModelRunnerMixin):
m.consumed_memory / GiB_bytes,
time_after_load - time_before_load,
)
self.model_memory_usage = int(m.consumed_memory)
logger.info("Model runner: Model loaded successfully.")
if self.od_config.streaming_output and not getattr(self.od_config, "step_execution", False):
@@ -368,6 +378,45 @@ class DiffusionModelRunner(OmniConnectorModelRunnerMixin):
logger.info("Model runner: Initialization complete.")
def get_kv_cache_spec(self) -> dict[str, KVCacheSpec]:
"""Collect native specs from cache-enabled loaded attention modules."""
from vllm_omni.diffusion.attention.layer import Attention
from vllm_omni.diffusion.diffusion_kv.config import DiffusionKVCacheMode
if self.od_config.diffusion_kv_mode is not DiffusionKVCacheMode.PAGED_SCHEDULER:
return {}
if self.pipeline is None:
raise RuntimeError("Model must be loaded before collecting Diffusion KV cache specs")
specs: dict[str, KVCacheSpec] = {}
for layer_name, module in self.pipeline.named_modules():
if not isinstance(module, Attention):
continue
spec = module.get_kv_cache_spec(self.vllm_config)
if spec is not None:
specs[layer_name] = spec
if not specs:
raise RuntimeError(
"paged_scheduler Diffusion KV found no cache-enabled Attention modules "
f"in {type(self.pipeline).__name__}"
)
return specs
def set_kv_cache_config(self, kv_cache_config: KVCacheConfig) -> None:
"""Retain the Engine-generated rank-local config for the data plane."""
configured_layers = {
layer_name for group in kv_cache_config.kv_cache_groups for layer_name in group.layer_names
}
expected_layers = set(self.get_kv_cache_spec())
if configured_layers != expected_layers:
raise ValueError(
"Rank-local Diffusion KVCacheConfig layer mismatch: "
f"expected={sorted(expected_layers)}, configured={sorted(configured_layers)}"
)
self.kv_cache_config = kv_cache_config
def clear_prompt_embed_cache(self) -> None:
"""Evict all cached text-encoder outputs (e.g. between training epochs).
@@ -516,6 +565,7 @@ class DiffusionModelRunner(OmniConnectorModelRunnerMixin):
require_request_batch_support: bool,
kv_prefetch_job: KVPrefetchJob | None = None,
record_name: str,
record_output_peak_memory: bool = True,
) -> BatchRunnerOutput:
assert self.pipeline is not None, "Model not loaded. Call load_model() first."
if not reqs:
@@ -545,7 +595,7 @@ class DiffusionModelRunner(OmniConnectorModelRunnerMixin):
batch = DiffusionRequestBatch(requests=reqs)
is_primary = not torch.distributed.is_initialized() or torch.distributed.get_rank() == 0
if is_primary:
if is_primary and record_output_peak_memory:
current_omni_platform.reset_peak_memory_stats()
with set_forward_context(vllm_config=self.vllm_config, omni_diffusion_config=od_config):
@@ -558,7 +608,7 @@ class DiffusionModelRunner(OmniConnectorModelRunnerMixin):
pipeline_name=type(self.pipeline).__name__,
)
if is_primary and outputs:
if is_primary and outputs and record_output_peak_memory:
batch_peak_memory_mb = self._sample_peak_memory_mb()
for output in outputs:
output.peak_memory_mb = max(output.peak_memory_mb, batch_peak_memory_mb)
@@ -626,6 +676,59 @@ class DiffusionModelRunner(OmniConnectorModelRunnerMixin):
assert output is not None
return output
def profile_run(self, requests: list[OmniDiffusionRequest]) -> None:
"""Run the maximum per-rank request batch for memory profiling.
This deliberately bypasses Scheduler admission and Diffusion KV
metadata validation because cache capacity has not been sized yet.
It otherwise uses the normal execution-mode path so model inputs,
collective communication, backend workspaces, denoising, and decode
allocations contribute to the observed peak. Step execution profiles
one fused ``InputBatch`` instead of sequential single-request forwards.
"""
if not requests:
raise ValueError("Diffusion memory profiling requires at least one request.")
runner_output: BatchRunnerOutput | None = None
request_ids = [request.request_id for request in requests]
try:
if getattr(self.od_config, "step_execution", False):
scheduler_output = DiffusionSchedulerOutput(
step_id=0,
scheduled_new_reqs=[
NewRequestData(request_id=request.request_id, req=request) for request in requests
],
scheduled_cached_reqs=CachedRequestData.make_empty(),
finished_req_ids=set(),
num_running_reqs=len(requests),
num_waiting_reqs=0,
)
runner_output = self._execute_stepwise(
scheduler_output,
validate_kv_metadata=False,
record_output_peak_memory=False,
)
else:
runner_output = self._execute_request_list(
requests,
od_config=self.od_config,
allow_single_output=len(requests) == 1,
require_request_batch_support=len(requests) > 1,
record_name="pipeline_memory_profile",
# The enclosing native memory_profiling context owns the
# peak counters. Resetting them here would discard request
# preparation allocations and understate the budget.
record_output_peak_memory=False,
)
current_omni_platform.synchronize()
finally:
for request_id in request_ids:
self.state_cache.pop(request_id, None)
self.input_batch = None
del runner_output
gc.collect()
def execute_model_batch(
self,
scheduler_output: DiffusionSchedulerOutput,
@@ -750,13 +853,29 @@ class DiffusionModelRunner(OmniConnectorModelRunnerMixin):
def execute_stepwise(self, scheduler_output: DiffusionSchedulerOutput) -> BatchRunnerOutput:
"""Execute one step for one scheduled request and return runner output."""
return self._execute_stepwise(
scheduler_output,
validate_kv_metadata=True,
record_output_peak_memory=True,
)
def _execute_stepwise(
self,
scheduler_output: DiffusionSchedulerOutput,
*,
validate_kv_metadata: bool,
record_output_peak_memory: bool,
) -> BatchRunnerOutput:
"""Execute one step with explicit validation and profiling policy."""
assert self.pipeline is not None, "Model not loaded. Call load_model() first."
for new_req in scheduler_output.scheduled_new_reqs:
validate_new_request_data_identity(new_req)
self._validate_diffusion_kv_metadata(
request_id=new_req.req.request_id,
metadata=new_req.diffusion_kv_metadata,
)
if validate_kv_metadata:
self._validate_diffusion_kv_metadata(
request_id=new_req.req.request_id,
metadata=new_req.diffusion_kv_metadata,
)
if not self._supports_step_mode():
raise ValueError("Current pipeline does not support step execution.")
# Stepwise mode only supports the basic state-driven denoise path for now.
@@ -771,7 +890,13 @@ class DiffusionModelRunner(OmniConnectorModelRunnerMixin):
had_active_states = bool(self.state_cache)
states, new_request_ids = self._update_states(scheduler_output)
is_primary = not torch.distributed.is_initialized() or torch.distributed.get_rank() == 0
if new_request_ids and not had_active_states and is_primary and current_omni_platform.is_available():
if (
record_output_peak_memory
and new_request_ids
and not had_active_states
and is_primary
and current_omni_platform.is_available()
):
current_omni_platform.reset_peak_memory_stats()
input_batch = self._prepare_batch_inputs(states, new_request_ids)
attn_metadata = {}
@@ -864,7 +989,7 @@ class DiffusionModelRunner(OmniConnectorModelRunnerMixin):
f"but batched noise_pred has {noise_pred.shape[0]} rows."
)
if is_primary:
if is_primary and record_output_peak_memory:
batch_peak_memory_mb = self._sample_peak_memory_mb()
states_by_id = {state.request_id: state for state in states}
for state in states: