[Feat] Add native Mooncake KV transfer from AR to DiT (#7166)

Signed-off-by: asukaqaq-s <1311722138@qq.com>
Signed-off-by: Acerak01-fy <wfy2003324@163.com>
Co-authored-by: Acerak01-fy <wfy2003324@163.com>
Co-authored-by: amy-why-3459 <wuhaiyan17@huawei.com>
This commit is contained in:
authored and GitHub committed 2026-09-20 14:44:55 +00:00
1 parent 2d0b03f717
commit 102ba716bf
53 files changed
+2564 -204

No files matched your search

@@ -1,16 +1,23 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
from contextlib import nullcontext
from types import SimpleNamespace
from unittest import mock
import pytest
import torch
from vllm.config import KVTransferConfig
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
from vllm.v1.outputs import KVConnectorOutput
from vllm_omni.diffusion.data import OmniDiffusionConfig
from vllm_omni.diffusion.diffusion_kv.kv_connector import (
KVTransferRegistrationError,
commit_kv_load,
parse_kv_transfer_config,
shutdown_kv_connector,
wait_for_kv_load,
)
from vllm_omni.diffusion.sched.base_scheduler import BaseScheduler
from vllm_omni.diffusion.vllm_config import create_diffusion_vllm_config
@@ -138,3 +145,205 @@ def test_shutdown_kv_connector_is_idempotent() -> None:
assert ensure_shutdown.call_count == 2
scheduler_connector.shutdown.assert_called_once()
def test_commit_aligns_cfg_transport_pages_without_extending_computed_prefix() -> None:
requests = tuple(
SimpleNamespace(
request_id=f"req/{sequence_id}",
kv_transfer_params={"num_transfer_tokens": 9},
num_tokens=16,
num_computed_tokens=0,
)
for sequence_id in range(2)
)
manager = SimpleNamespace(
get_blocks=lambda _request_id: KVCacheBlocks(([0, 1, 2, 3],)),
kv_cache_config=SimpleNamespace(kv_cache_groups=[SimpleNamespace(kv_cache_spec=SimpleNamespace(block_size=4))]),
)
connector = mock.Mock()
expected = commit_kv_load(connector, manager, requests, [7, 5])
assert expected == {"req/0", "req/1"}
calls = connector.update_state_after_alloc.call_args_list
assert [len(call.args[1].blocks[0]) for call in calls] == [3, 3]
assert [call.args[2] for call in calls] == [7, 5]
assert [request.num_computed_tokens for request in requests] == [7, 5]
def test_wait_without_pending_load_uses_vllm_post_forward_once() -> None:
connector = mock.Mock()
output = mock.Mock(invalid_block_ids=set())
active_connector = mock.Mock(kv_connector=connector)
active_connector.post_forward.return_value = output
scheduler_output = SimpleNamespace(
kv_transfer_request_ids=set(),
finished_req_ids={"finished"},
kv_finished_request_ids={"finished/diffusion-kv/0", "finished/diffusion-kv/1"},
)
result = wait_for_kv_load(active_connector, scheduler_output, timeout=1.0)
assert result is output
active_connector.pre_forward.assert_called_once_with(scheduler_output)
active_connector.post_forward.assert_called_once_with({"finished/diffusion-kv/0", "finished/diffusion-kv/1"})
connector.get_finished.assert_not_called()
def test_timeout_returns_partial_completion_without_releasing_pages():
active = mock.Mock()
active.kv_connector.get_finished.return_value = (set(), {"cfg0"})
active.kv_connector.get_block_ids_with_load_errors.return_value = set()
active.post_forward.return_value = KVConnectorOutput()
scheduled = SimpleNamespace(kv_transfer_request_ids={"cfg0", "cfg1"}, kv_finished_request_ids=set())
output = wait_for_kv_load(active, scheduled, timeout=0)
assert output.finished_recving == {"cfg0"}
active.kv_connector.request_finished.assert_not_called()
active.post_forward.assert_called_once_with(set())
@pytest.fixture
def cfg_registration():
requests = tuple(
SimpleNamespace(
request_id=f"cfg{i}",
status=None,
num_tokens=8,
kv_transfer_params={"num_transfer_tokens": 4, "transfer_id": "ticket", "do_remote_prefill": True},
)
for i in range(2)
)
manager = SimpleNamespace(
get_blocks=lambda _: KVCacheBlocks(([0, 1],)),
kv_cache_config=SimpleNamespace(kv_cache_groups=[SimpleNamespace(kv_cache_spec=SimpleNamespace(block_size=4))]),
)
return requests, manager
def test_cfg_validation_is_atomic_before_connector_registration(cfg_registration):
requests, manager = cfg_registration
requests[1].kv_transfer_params["num_transfer_tokens"] = 9
connector = mock.Mock()
with pytest.raises(KVTransferRegistrationError, match="boundary"):
commit_kv_load(connector, manager, requests, [4, 4])
connector.update_state_after_alloc.assert_not_called()
@pytest.mark.parametrize("fail_init", [False, True])
def test_sp_connector_init_snapshots_topology_and_restores_model_tp(monkeypatch, fail_init):
import vllm.distributed.parallel_state as parallel_state
import vllm_omni.diffusion.diffusion_kv.kv_connector as module
import vllm_omni.diffusion.distributed.parallel_state as omni_parallel
tp = SimpleNamespace(world_size=1, rank_in_group=0)
sp = SimpleNamespace(world_size=2, rank_in_group=1)
monkeypatch.setattr(parallel_state, "_TP", tp)
monkeypatch.setattr(omni_parallel, "get_sp_group", lambda: sp)
snapshot: dict[str, int] = {}
def initialize(*args):
snapshot.update(
rank=parallel_state.get_tensor_model_parallel_rank(),
size=parallel_state.get_tensor_model_parallel_world_size(),
)
if fail_init:
raise RuntimeError("initialization failed")
monkeypatch.setattr(module, "ensure_kv_transfer_initialized", initialize)
config = SimpleNamespace(
kv_transfer_config=SimpleNamespace(kv_connector="MooncakeConnector", engine_id="dit"),
parallel_config=SimpleNamespace(prefill_context_parallel_size=2),
)
with pytest.raises(RuntimeError, match="initialization failed") if fail_init else nullcontext():
module.init_worker_kv_connector(config, object())
assert parallel_state._TP is tp
assert snapshot == {"rank": 1, "size": 2}
def test_partial_mooncake_registration_is_replaced_by_empty_notifications(cfg_registration):
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.mooncake_connector import (
MooncakeConnector,
MooncakeConnectorScheduler,
)
scheduler = object.__new__(MooncakeConnectorScheduler)
scheduler.is_kv_producer = False
scheduler.is_kv_consumer = True
scheduler._reqs_need_recv = {}
connector = object.__new__(MooncakeConnector)
connector.connector_scheduler = scheduler
requests, manager = cfg_registration
for request in requests:
request.kv_transfer_params.update(remote_engine_id="ar", remote_bootstrap_addr="http://localhost:8998")
def register(request, blocks, count):
scheduler._reqs_need_recv[request.request_id] = (request, blocks)
request.kv_transfer_params["do_remote_prefill"] = False
if request.request_id == "cfg1":
raise RuntimeError("partial registration")
connector.update_state_after_alloc = register
with pytest.raises(KVTransferRegistrationError, match="partial registration"):
commit_kv_load(connector, manager, requests, [4, 4])
assert set(scheduler._reqs_need_recv) == {"cfg0", "cfg1"}
assert all(blocks == [] for _, blocks in scheduler._reqs_need_recv.values())
metadata = scheduler.build_connector_meta(None)
# Even partial registration failure sends every row in one per-rank batch,
# preserving the producer's complete fan-out count for the shared ticket.
assert set(metadata.reqs_to_recv["ar"]) == {"cfg0", "cfg1"}
assert all(
meta.transfer_id == "ticket" and meta.local_block_ids == [] for meta in metadata.reqs_to_recv["ar"].values()
)
@pytest.mark.parametrize("timeout", [0, -1, float("inf"), "60"])
def test_native_transfer_timeout_must_be_positive_finite(timeout):
with pytest.raises(ValueError, match="transfer_timeout"):
parse_kv_transfer_config(
{
**KV_TRANSFER_CONFIG,
"kv_connector_extra_config": {"transfer_timeout": timeout},
}
)
def test_receive_completion_requires_all_ranks_across_polls():
from vllm_omni.diffusion.executor.abstract import DiffusionExecutor
from vllm_omni.diffusion.sched.interface import CachedRequestData, DiffusionSchedulerOutput
executor = SimpleNamespace(od_config=SimpleNamespace(num_gpus=2))
executor.collective_rpc = mock.Mock(
side_effect=[
[KVConnectorOutput(finished_recving={"cfg0", "cfg1"}), KVConnectorOutput(finished_recving={"cfg0"})],
[KVConnectorOutput(), KVConnectorOutput(finished_recving={"cfg1"})],
]
)
scheduled = DiffusionSchedulerOutput(
step_id=0,
scheduled_new_reqs=[],
scheduled_cached_reqs=CachedRequestData.make_empty(),
finished_req_ids=set(),
num_running_reqs=0,
num_waiting_reqs=0,
kv_connector_metadata=object(),
kv_transfer_request_ids={"cfg0", "cfg1"},
)
result = DiffusionExecutor.prepare_kv_for_forward(executor, scheduled)
assert result.finished_recving == {"cfg0"}
scheduled.kv_transfer_request_ids = set()
result = DiffusionExecutor.prepare_kv_for_forward(executor, scheduled)
assert result.finished_recving == {"cfg1"}
assert not executor._kv_receive_completed_ranks
def test_native_transfer_rejects_sleep_without_changing_legacy_sleep():
with pytest.raises(ValueError, match="registered pages must remain mapped"):
OmniDiffusionConfig.from_kwargs(
diffusion_kv_mode="paged_scheduler",
diffusion_kv_max_rows_per_request=2,
kv_transfer_config=dict(KV_TRANSFER_CONFIG),
enable_sleep_mode=True,
)
assert OmniDiffusionConfig.from_kwargs(enable_sleep_mode=True).enable_sleep_mode