[Feat] Add prefetch for Mooncake cross-stage paged KV transfer in HunyuanImage3 (#7637)
Signed-off-by: Acerak01-fy <wfy2003324@163.com>
This commit is contained in:
1 parent
f7e28348ad
commit
9ebef4b1d3
11 files changed
+909
-23
No files matched your search
@@ -6,6 +6,7 @@ from __future__ import annotations
|
||||
import time
|
||||
from abc import ABC, abstractmethod
|
||||
from collections import deque
|
||||
from collections.abc import Iterable
|
||||
from dataclasses import fields
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
@@ -19,7 +20,9 @@ from vllm_omni.diffusion.diffusion_kv.config import DiffusionKVCacheMode
|
||||
from vllm_omni.diffusion.diffusion_kv.kv_connector import (
|
||||
KVTransferRegistrationError,
|
||||
commit_kv_load,
|
||||
native_prefetch_enabled,
|
||||
prepare_kv_requests,
|
||||
validate_kv_transfer_boundaries,
|
||||
)
|
||||
from vllm_omni.diffusion.diffusion_kv.manager import DiffusionKVCacheManager
|
||||
from vllm_omni.diffusion.diffusion_kv.metadata import DiffusionKVMetadata
|
||||
@@ -71,6 +74,8 @@ class BaseScheduler(ABC):
|
||||
self._kv_request_generations: dict[str, int] = {}
|
||||
self._kv_draining_requests: dict[str, SchedulerRequestState] = {}
|
||||
self._kv_received_request_ids: set[str] = set()
|
||||
self._native_prefetch_enabled = False
|
||||
self._native_prefetch_request_id: str | None = None
|
||||
|
||||
def initialize(
|
||||
self,
|
||||
@@ -82,6 +87,8 @@ class BaseScheduler(ABC):
|
||||
kv_vllm_config: VllmConfig | None = None,
|
||||
) -> None:
|
||||
self.od_config = od_config
|
||||
self._native_prefetch_enabled = native_prefetch_enabled(od_config)
|
||||
self._native_prefetch_request_id = None
|
||||
self._request_states.clear()
|
||||
self._step_id = 0
|
||||
self._waiting.clear()
|
||||
@@ -227,6 +234,8 @@ class BaseScheduler(ABC):
|
||||
sequence.num_computed_tokens = request.num_computed_tokens
|
||||
|
||||
self._waiting.popleft()
|
||||
if request_id == self._native_prefetch_request_id:
|
||||
self._native_prefetch_request_id = None
|
||||
was_new_request = state.status == DiffusionRequestStatus.WAITING
|
||||
if not self._running:
|
||||
self._running_sampling_params_key = state.sampling_params_key
|
||||
@@ -277,6 +286,23 @@ class BaseScheduler(ABC):
|
||||
scheduler_output.kv_connector_metadata = self._kv_connector.build_connector_meta(scheduler_output)
|
||||
scheduler_output.kv_transfer_request_ids = self._kv_transfer_request_ids
|
||||
self._kv_transfer_request_ids = set()
|
||||
if self._native_prefetch_enabled:
|
||||
scheduler_output.kv_required_request_ids = self._loading_sequence_ids(self._running)
|
||||
# Flush current-request metadata before staging B. Mooncake
|
||||
# batches ready requests from the same producer into one write;
|
||||
# combining A+B would make A's completion depend on B's bytes.
|
||||
self._try_native_prefetch()
|
||||
prefetch_finished_ids = self._kv_finished_request_ids - scheduler_output.kv_finished_request_ids
|
||||
if self._kv_transfer_request_ids or prefetch_finished_ids:
|
||||
scheduler_output.kv_prefetch_connector_metadata = self._kv_connector.build_connector_meta(
|
||||
scheduler_output
|
||||
)
|
||||
scheduler_output.kv_prefetch_request_ids = self._kv_transfer_request_ids
|
||||
scheduler_output.kv_transfer_request_ids |= self._kv_transfer_request_ids
|
||||
self._kv_transfer_request_ids = set()
|
||||
scheduler_output.finished_req_ids.update(self._finished_req_ids)
|
||||
scheduler_output.kv_finished_request_ids.update(prefetch_finished_ids)
|
||||
scheduler_output.num_waiting_reqs = len(self._waiting)
|
||||
|
||||
# update after schedule
|
||||
self._step_id += 1
|
||||
@@ -284,6 +310,92 @@ class BaseScheduler(ABC):
|
||||
self._kv_finished_request_ids.clear()
|
||||
return scheduler_output
|
||||
|
||||
def _try_native_prefetch(self) -> None:
|
||||
if not self._running or not self._waiting or self._native_prefetch_request_id is not None:
|
||||
return
|
||||
request_id = self._waiting[0]
|
||||
state = self._request_states.get(request_id)
|
||||
if state is None or state.is_finished() or not state.diffusion_kv_requests:
|
||||
return
|
||||
manager = self._diffusion_kv_manager
|
||||
if manager.has_request(request_id):
|
||||
return
|
||||
matched_tokens = []
|
||||
for request in state.diffusion_kv_requests:
|
||||
params = request.kv_transfer_params or {}
|
||||
if (
|
||||
not params.get("do_remote_prefill")
|
||||
or not all(params.get(key) for key in ("remote_engine_id", "remote_bootstrap_addr", "transfer_id"))
|
||||
or type(params.get("num_transfer_tokens")) is not int
|
||||
or params["num_transfer_tokens"] <= 0
|
||||
):
|
||||
return
|
||||
num_tokens, _ = self._kv_connector.get_num_new_matched_tokens(request, 0)
|
||||
if num_tokens is None:
|
||||
return
|
||||
matched_tokens.append(num_tokens)
|
||||
try:
|
||||
validate_kv_transfer_boundaries(state.diffusion_kv_requests, matched_tokens)
|
||||
except KVTransferRegistrationError as exc:
|
||||
# Do not reserve B's pages for a transfer that cannot be registered.
|
||||
# Normal admission will report the request-scoped error later.
|
||||
logger.debug("Native KV prefetch skipped for %s: invalid boundary (%s)", request_id, exc)
|
||||
return
|
||||
# The manager atomically reserves all CFG rows, including capacity
|
||||
# needed when B eventually executes. B remains in WAITING.
|
||||
try:
|
||||
allocation = manager.reserve_request(request_id, state.diffusion_kv_requests)
|
||||
except Exception as exc:
|
||||
# reserve_request rolls back atomically. Normal admission will
|
||||
# surface B's error later; speculative allocation must not fail A.
|
||||
logger.debug("Native KV prefetch skipped for %s: reservation failed (%s)", request_id, exc)
|
||||
return
|
||||
if allocation is None:
|
||||
logger.debug("Native KV prefetch skipped for %s: insufficient blocks", request_id)
|
||||
return
|
||||
self._kv_request_generations[request_id] = allocation.allocation_generation
|
||||
try:
|
||||
transfer_ids = commit_kv_load(
|
||||
self._kv_connector, manager.native_manager, state.diffusion_kv_requests, matched_tokens
|
||||
)
|
||||
except KVTransferRegistrationError as exc:
|
||||
# Rollback may already notify the producer to release its pages.
|
||||
# B cannot retry that transfer ticket, but A can still proceed.
|
||||
self._finish_requests({request_id: DiffusionRequestStatus.FINISHED_ERROR}, {request_id: str(exc)})
|
||||
return
|
||||
self._kv_transfer_request_ids.update(transfer_ids)
|
||||
if transfer_ids:
|
||||
self._kv_loading_request_ids.add(request_id)
|
||||
for sequence, request in zip(allocation.sequences, state.diffusion_kv_requests, strict=True):
|
||||
sequence.num_computed_tokens = request.num_computed_tokens
|
||||
self._native_prefetch_request_id = request_id
|
||||
logger.debug("Native KV prefetch submitted for %s: %s", request_id, sorted(transfer_ids))
|
||||
|
||||
def _loading_sequence_ids(self, request_ids: Iterable[str]) -> set[str]:
|
||||
sequence_ids: set[str] = set()
|
||||
for request_id in request_ids:
|
||||
if request_id not in self._kv_loading_request_ids:
|
||||
continue
|
||||
state = self._request_states.get(request_id) or self._kv_draining_requests.get(request_id)
|
||||
if state is not None:
|
||||
sequence_ids.update(request.request_id for request in state.diffusion_kv_requests)
|
||||
return sequence_ids
|
||||
|
||||
def native_kv_poll_output(self, *, drain_request_ids: list[str] | None = None) -> DiffusionSchedulerOutput | None:
|
||||
"""Poll after compute; cancellation/close may wait for selected loads."""
|
||||
if not self._native_prefetch_enabled or not self._kv_loading_request_ids:
|
||||
return None
|
||||
return DiffusionSchedulerOutput(
|
||||
step_id=self._step_id,
|
||||
scheduled_new_reqs=[],
|
||||
scheduled_cached_reqs=CachedRequestData.make_empty(),
|
||||
finished_req_ids=set(),
|
||||
num_running_reqs=len(self._running),
|
||||
num_waiting_reqs=len(self._waiting),
|
||||
kv_required_request_ids=self._loading_sequence_ids(drain_request_ids or []),
|
||||
kv_poll_only=True,
|
||||
)
|
||||
|
||||
@abstractmethod
|
||||
def update_from_output(self, sched_output: DiffusionSchedulerOutput, output: RunnerOutput) -> set[str]:
|
||||
pass
|
||||
@@ -426,6 +538,7 @@ class BaseScheduler(ABC):
|
||||
self._kv_draining_requests.clear()
|
||||
self._kv_received_request_ids.clear()
|
||||
self._kv_request_generations.clear()
|
||||
self._native_prefetch_request_id = None
|
||||
|
||||
def _shutdown_diffusion_kv(self) -> None:
|
||||
"""Stop transfers before releasing their cache and request state."""
|
||||
@@ -471,6 +584,8 @@ class BaseScheduler(ABC):
|
||||
for request_id in finished_req_ids:
|
||||
state = self._request_states[request_id]
|
||||
status = statuses[request_id]
|
||||
if request_id == self._native_prefetch_request_id:
|
||||
self._native_prefetch_request_id = None
|
||||
if request_id in self._kv_loading_request_ids:
|
||||
self._kv_draining_requests[request_id] = state
|
||||
else:
|
||||
|
||||
@@ -256,9 +256,23 @@ class DiffusionSchedulerOutput:
|
||||
kv_prefetch_job: KVPrefetchJob | None = None
|
||||
kv_connector_metadata: KVConnectorMetadata | None = None
|
||||
kv_transfer_request_ids: set[str] = field(default_factory=set)
|
||||
# None preserves the synchronous path; an empty set means submit/poll
|
||||
# without waiting. Transfer IDs above are submitted only once.
|
||||
kv_required_request_ids: set[str] | None = None
|
||||
kv_poll_only: bool = False
|
||||
# Keep background transfers out of the current request's Mooncake batch.
|
||||
kv_prefetch_connector_metadata: KVConnectorMetadata | None = None
|
||||
kv_prefetch_request_ids: set[str] = field(default_factory=set)
|
||||
# Connector lifecycle uses per-sequence IDs, not public request IDs.
|
||||
kv_finished_request_ids: set[str] = field(default_factory=set)
|
||||
|
||||
@property
|
||||
def has_sync_kv_loads(self) -> bool:
|
||||
# vLLM defers async submission until post_forward. The synchronous
|
||||
# receive loop must start its loads in pre_forward before polling.
|
||||
required = self.kv_required_request_ids
|
||||
return bool(self.kv_transfer_request_ids if required is None else required)
|
||||
|
||||
@cached_property
|
||||
def scheduled_request_ids(self) -> list[str]:
|
||||
"""
|
||||
@@ -275,4 +289,10 @@ class DiffusionSchedulerOutput:
|
||||
|
||||
@property
|
||||
def is_empty(self) -> bool:
|
||||
return self.num_scheduled_reqs == 0 and self.kv_connector_metadata is None
|
||||
return (
|
||||
self.num_scheduled_reqs == 0
|
||||
and self.kv_connector_metadata is None
|
||||
and self.kv_prefetch_connector_metadata is None
|
||||
and not self.kv_required_request_ids
|
||||
and not self.kv_poll_only
|
||||
)
|
||||
Reference in new issue
Block a user