[Feat] Add native Mooncake KV transfer from AR to DiT (#7166)

Signed-off-by: asukaqaq-s <1311722138@qq.com>
Signed-off-by: Acerak01-fy <wfy2003324@163.com>
Co-authored-by: Acerak01-fy <wfy2003324@163.com>
Co-authored-by: amy-why-3459 <wuhaiyan17@huawei.com>
This commit is contained in:
authored and GitHub committed 2026-09-20 14:44:55 +00:00
1 parent 2d0b03f717
commit 102ba716bf
53 files changed
+2564 -204

No files matched your search

+32 -8
View File
@@ -59,6 +59,7 @@ from vllm_omni.diffusion.distributed.parallel_state import (
get_hsdp_replicate_group,
get_pp_group,
get_sp_group,
get_world_group,
init_distributed_environment,
initialize_model_parallel,
model_parallel_is_initialized,
@@ -116,7 +117,11 @@ def _all_gather_rank_values(value: Any) -> list[Any]:
if not dist.is_available() or not dist.is_initialized():
return [value]
values: list[Any] = [None] * dist.get_world_size()
dist.all_gather_object(values, value)
# Object collectives are control-plane traffic. Using the default NCCL
# group serializes them through temporary CUDA tensors, adding pointless
# H2D/DtoH copies to every rank-wide status check. Reuse the world
# coordinator's Gloo group, as vLLM does for CPU metadata collectives.
dist.all_gather_object(values, value, group=get_world_group().cpu_group)
return values
@@ -446,15 +451,13 @@ class DiffusionWorker:
raise RuntimeError("Diffusion KV memory snapshot was not captured before model loading")
override = self.vllm_config.cache_config.kv_cache_memory_bytes
if override:
# Match native vLLM: an explicit cache budget skips automatic
# capacity derivation, but still runs the maximum-shape model
# request so lazy kernels and communication buffers initialize.
self.model_runner.profile_run(profile_requests)
# Post-allocation warmup avoids retaining a second activation arena.
logger.info(
"Worker %d: Initial free memory %s GiB, reserved %s GiB memory for "
"Diffusion KV Cache as specified by kv_cache_memory_bytes config and "
"skipped automatic memory profiling. This does not respect the "
"gpu_memory_utilization config. A profile warmup was still executed.",
"gpu_memory_utilization config. Model kernels will be initialized by "
"the post-allocation startup warmup.",
self.rank,
format_gib(self.init_snapshot.free_memory),
format_gib(int(override)),
@@ -467,6 +470,8 @@ class DiffusionWorker:
) as profile_result:
self.model_runner.profile_run(profile_requests)
current_omni_platform.empty_cache()
available_memory = self.requested_memory - profile_result.non_kv_cache_memory
if available_memory <= 0:
raise RuntimeError(
@@ -534,12 +539,18 @@ class DiffusionWorker:
with self._maybe_get_memory_pool_context("kv_cache"):
self.model_runner.set_kv_cache_config(kv_cache_config)
def remove_diffusion_kv_requests(self, request_ids: list[str]) -> int:
def remove_diffusion_kv_requests(self, request_ids: list[str | tuple[str, int]]) -> int:
"""Clear Worker-local rows without freeing Scheduler-owned blocks."""
assert self.model_runner is not None, "Model runner not initialized"
return self.model_runner.remove_diffusion_kv_requests(request_ids)
def prepare_kv_for_forward(self, scheduler_output: DiffusionSchedulerOutput):
return _run_and_gather_rank_values(
"Diffusion KV receive",
lambda: self.model_runner.prepare_kv_for_forward(scheduler_output),
)
def init_lora_manager(self) -> None:
"""Initialize the LoRA manager for this worker."""
pipeline = self.model_runner.pipeline
@@ -790,6 +801,15 @@ class DiffusionWorker:
Args:
level: Sleep level. Level 1 offloads weights, level 2 also saves buffers.
"""
# The config validator rejects sleep for the native paged path. Keep
# this worker-side guard precise as well: test doubles and legacy
# configs may expose arbitrary attributes through Mock/getattr.
if (
getattr(self.od_config, "diffusion_kv_mode", DiffusionKVCacheMode.DENSE_LEGACY)
is DiffusionKVCacheMode.PAGED_SCHEDULER
and getattr(self.od_config, "kv_transfer_config", None) is not None
):
raise ValueError("Cannot sleep while native KV connector memory is registered")
CuMemAllocator = _get_cumem_allocator_class()
allocator = CuMemAllocator.get_instance()
@@ -1254,7 +1274,11 @@ class WorkerProc:
world_size = torch.distributed.get_world_size()
statuses: list[dict[str, Any] | None] = [None] * world_size
torch.distributed.all_gather_object(statuses, status)
torch.distributed.all_gather_object(
statuses,
status,
group=get_world_group().cpu_group,
)
missing_ranks = [rank for rank, rank_status in enumerate(statuses) if rank_status is None]
if missing_ranks:
logger.warning("RPC rank status gather returned missing entries for ranks: %s", missing_ranks)