[Feat] Add prefetch for Mooncake cross-stage paged KV transfer in HunyuanImage3 (#7637)
Signed-off-by: Acerak01-fy <wfy2003324@163.com>
This commit is contained in:
1 parent
f7e28348ad
commit
9ebef4b1d3
11 files changed
+909
-23
No files matched your search
@@ -0,0 +1,552 @@
|
|||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||||
|
|
||||||
|
from types import SimpleNamespace
|
||||||
|
from unittest.mock import Mock
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
import torch
|
||||||
|
import vllm.v1.core.single_type_kv_cache_manager as native_kv_managers
|
||||||
|
from vllm.config import KVTransferConfig
|
||||||
|
from vllm.v1.kv_cache_interface import FullAttentionSpec, KVCacheConfig, KVCacheGroupSpec, KVCacheTensor
|
||||||
|
from vllm.v1.outputs import KVConnectorOutput
|
||||||
|
|
||||||
|
from vllm_omni.diffusion.diffusion_engine import DiffusionEngine
|
||||||
|
from vllm_omni.diffusion.diffusion_kv.config import DiffusionKVCacheMode
|
||||||
|
from vllm_omni.diffusion.diffusion_kv.kv_connector import KVReceiveProgress, native_prefetch_enabled
|
||||||
|
from vllm_omni.diffusion.diffusion_kv.request import DiffusionKVRequest
|
||||||
|
from vllm_omni.diffusion.executor.abstract import DiffusionExecutor
|
||||||
|
from vllm_omni.diffusion.request import OmniDiffusionRequest
|
||||||
|
from vllm_omni.diffusion.sched import DiffusionRequestStatus, RequestScheduler
|
||||||
|
from vllm_omni.diffusion.sched.interface import CachedRequestData, DiffusionSchedulerOutput
|
||||||
|
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||||
|
|
||||||
|
pytestmark = [pytest.mark.core_model, pytest.mark.cpu, pytest.mark.diffusion]
|
||||||
|
|
||||||
|
|
||||||
|
def _output(*, submit=(), required=(), retire=(), metadata=None):
|
||||||
|
return DiffusionSchedulerOutput(
|
||||||
|
step_id=0,
|
||||||
|
scheduled_new_reqs=[],
|
||||||
|
scheduled_cached_reqs=CachedRequestData.make_empty(),
|
||||||
|
finished_req_ids=set(),
|
||||||
|
num_running_reqs=0,
|
||||||
|
num_waiting_reqs=0,
|
||||||
|
kv_connector_metadata=metadata,
|
||||||
|
kv_transfer_request_ids=set(submit),
|
||||||
|
kv_required_request_ids=set(required),
|
||||||
|
kv_finished_request_ids=set(retire),
|
||||||
|
kv_poll_only=metadata is None,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _completed(*ids, invalid=()):
|
||||||
|
return KVConnectorOutput(finished_recving=set(ids), invalid_block_ids=set(invalid))
|
||||||
|
|
||||||
|
|
||||||
|
def test_prefetch_survives_post_forward_and_is_not_submitted_twice():
|
||||||
|
progress = KVReceiveProgress()
|
||||||
|
active = Mock()
|
||||||
|
# B finishes before A, in a different get_finished/post_forward call.
|
||||||
|
active.post_forward.side_effect = [_completed("B0"), _completed("A0"), _completed(), _completed()]
|
||||||
|
first = _output(submit=["A0", "B0"], required=["A0"], metadata=object())
|
||||||
|
assert progress.prepare(active, first, 1).finished_recving == {"A0", "B0"}
|
||||||
|
second = _output(required=["B0"], retire=["A0"])
|
||||||
|
assert progress.prepare(active, second, 1).finished_recving == {"B0"}
|
||||||
|
active.pre_forward.assert_called_once_with(first)
|
||||||
|
progress.prepare(active, _output(retire=["B0"]), 1)
|
||||||
|
assert not progress.submitted
|
||||||
|
assert not progress.received
|
||||||
|
assert not progress.deadlines
|
||||||
|
|
||||||
|
|
||||||
|
def test_submit_b_does_not_wait_for_b():
|
||||||
|
progress = KVReceiveProgress()
|
||||||
|
active = Mock()
|
||||||
|
active.post_forward.return_value = _completed("A0")
|
||||||
|
result = progress.prepare(active, _output(submit=["A0", "B0"], required=["A0"], metadata=object()), 10)
|
||||||
|
assert result.finished_recving == {"A0"}
|
||||||
|
assert progress.submitted == {"A0", "B0"}
|
||||||
|
active.post_forward.assert_called_once()
|
||||||
|
active.post_forward.side_effect = [_completed(), _completed("B0")]
|
||||||
|
progress.prepare(active, _output(required=["B0"]), 10)
|
||||||
|
assert progress.received == {"A0", "B0"}
|
||||||
|
assert active.pre_forward.call_count == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_duplicate_submission_and_unknown_required_fail_before_transfer():
|
||||||
|
progress = KVReceiveProgress(submitted={"B0"})
|
||||||
|
active = Mock()
|
||||||
|
with pytest.raises(RuntimeError, match="Duplicate"):
|
||||||
|
progress.prepare(active, _output(submit=["B0"], metadata=object()), 1)
|
||||||
|
with pytest.raises(RuntimeError, match="never submitted"):
|
||||||
|
progress.prepare(active, _output(required=["C0"]), 1)
|
||||||
|
active.pre_forward.assert_not_called()
|
||||||
|
|
||||||
|
|
||||||
|
def test_poll_checks_timeout_without_resetting_it(monkeypatch):
|
||||||
|
now = [1.0]
|
||||||
|
monkeypatch.setattr("vllm_omni.diffusion.diffusion_kv.kv_connector.time.monotonic", lambda: now[0])
|
||||||
|
progress = KVReceiveProgress()
|
||||||
|
active = Mock()
|
||||||
|
active.post_forward.side_effect = lambda *_: _completed()
|
||||||
|
progress.prepare(active, _output(submit=["B0"], metadata=object()), 5)
|
||||||
|
now[0] = 7
|
||||||
|
with pytest.raises(TimeoutError, match="B0"):
|
||||||
|
progress.prepare(active, _output(), 5)
|
||||||
|
assert progress.submitted == {"B0"}
|
||||||
|
|
||||||
|
|
||||||
|
def test_poll_checks_load_errors_and_refuses_early_retirement():
|
||||||
|
progress = KVReceiveProgress(submitted={"B0"})
|
||||||
|
active = Mock()
|
||||||
|
active.post_forward.return_value = _completed(invalid=[42])
|
||||||
|
with pytest.raises(RuntimeError, match="invalid remote pages"):
|
||||||
|
progress.prepare(active, _output(), 1)
|
||||||
|
active.post_forward.return_value = _completed()
|
||||||
|
with pytest.raises(RuntimeError, match="unfinished"):
|
||||||
|
progress.prepare(active, _output(retire=["B0"]), 1)
|
||||||
|
assert progress.submitted == {"B0"}
|
||||||
|
|
||||||
|
|
||||||
|
def test_late_unsubmitted_cancellation_ack_is_not_retained():
|
||||||
|
progress = KVReceiveProgress()
|
||||||
|
active = Mock()
|
||||||
|
active.post_forward.return_value = _completed("cancelled-before-admission")
|
||||||
|
result = progress.prepare(active, _output(), 1)
|
||||||
|
assert not result.finished_recving
|
||||||
|
assert not progress.received
|
||||||
|
|
||||||
|
|
||||||
|
def test_executor_intersects_rank_completions_and_waits_only_required():
|
||||||
|
executor = SimpleNamespace(od_config=SimpleNamespace(num_gpus=2), collective_rpc=Mock())
|
||||||
|
output = _output(submit=["A0", "B0"], required=["A0"], metadata=object())
|
||||||
|
executor.collective_rpc.return_value = [_completed("A0", "B0"), _completed("A0")]
|
||||||
|
result = DiffusionExecutor.prepare_kv_for_forward(executor, output)
|
||||||
|
assert result.finished_recving == {"A0"}
|
||||||
|
assert executor.collective_rpc.call_args.kwargs["exec_all_ranks"] is True
|
||||||
|
executor.collective_rpc.return_value = [_completed("A0", "B0"), _completed("A0", "B0")]
|
||||||
|
assert DiffusionExecutor.prepare_kv_for_forward(executor, _output()).finished_recving == {"A0", "B0"}
|
||||||
|
executor.collective_rpc.return_value = [_completed("B0"), _completed()]
|
||||||
|
# Background polling can return early, but compute B requires every rank.
|
||||||
|
assert not DiffusionExecutor.prepare_kv_for_forward(executor, _output()).finished_recving
|
||||||
|
with pytest.raises(RuntimeError, match="every rank"):
|
||||||
|
DiffusionExecutor.prepare_kv_for_forward(executor, _output(required=["B0"]))
|
||||||
|
|
||||||
|
|
||||||
|
def test_executor_submits_prefetch_after_current_rank_barrier():
|
||||||
|
calls = []
|
||||||
|
|
||||||
|
def rpc(method, **kwargs):
|
||||||
|
output = kwargs["args"][0]
|
||||||
|
calls.append(output)
|
||||||
|
if len(calls) == 1:
|
||||||
|
assert output.kv_transfer_request_ids == output.kv_required_request_ids == {"A0"}
|
||||||
|
assert output.has_sync_kv_loads
|
||||||
|
else:
|
||||||
|
assert output.kv_transfer_request_ids == {"B0"}
|
||||||
|
assert not output.kv_required_request_ids
|
||||||
|
assert not output.has_sync_kv_loads
|
||||||
|
assert output.kv_connector_metadata == "B metadata"
|
||||||
|
# B is still in flight on both ranks. This must not block compute A.
|
||||||
|
return [_completed("A0"), _completed("A0")]
|
||||||
|
|
||||||
|
executor = SimpleNamespace(od_config=SimpleNamespace(num_gpus=2), collective_rpc=rpc)
|
||||||
|
executor.prepare_kv_for_forward = lambda out: DiffusionExecutor.prepare_kv_for_forward(executor, out)
|
||||||
|
output = _output(submit=["A0", "B0"], required=["A0"], metadata="A metadata")
|
||||||
|
output.kv_prefetch_connector_metadata = "B metadata"
|
||||||
|
output.kv_prefetch_request_ids = {"B0"}
|
||||||
|
assert executor.prepare_kv_for_forward(output).finished_recving == {"A0"}
|
||||||
|
assert len(calls) == 2
|
||||||
|
|
||||||
|
|
||||||
|
def _config():
|
||||||
|
return SimpleNamespace(
|
||||||
|
kv_transfer_config=KVTransferConfig(
|
||||||
|
kv_connector="MooncakeConnector",
|
||||||
|
kv_role="kv_consumer",
|
||||||
|
engine_id="prefetch-test",
|
||||||
|
kv_connector_extra_config={"mooncake_protocol": "tcp", "enable_kv_async_prefetch": True},
|
||||||
|
),
|
||||||
|
diffusion_kv_mode=DiffusionKVCacheMode.PAGED_SCHEDULER,
|
||||||
|
max_num_seqs=1,
|
||||||
|
model_class_name="TestDiffusionPipeline",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_mock_config_does_not_enable_native_prefetch():
|
||||||
|
assert not native_prefetch_enabled(Mock())
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("enabled", ["true", "false", 0, 1, None])
|
||||||
|
def test_native_prefetch_rejects_non_boolean_flag(enabled):
|
||||||
|
config = _config()
|
||||||
|
config.kv_transfer_config.kv_connector_extra_config["enable_kv_async_prefetch"] = enabled
|
||||||
|
with pytest.raises(ValueError, match="must be a boolean"):
|
||||||
|
native_prefetch_enabled(config)
|
||||||
|
|
||||||
|
|
||||||
|
def test_opt_in_and_platform_validation(monkeypatch):
|
||||||
|
monkeypatch.setattr("vllm_omni.platforms.current_omni_platform.is_cuda", lambda: True)
|
||||||
|
config = _config()
|
||||||
|
assert native_prefetch_enabled(config)
|
||||||
|
config.model_class_name = "AnotherDiffusionPipeline"
|
||||||
|
assert native_prefetch_enabled(config)
|
||||||
|
config.max_num_seqs = 2
|
||||||
|
with pytest.raises(ValueError, match="max_num_seqs=1"):
|
||||||
|
native_prefetch_enabled(config)
|
||||||
|
config.max_num_seqs = 1
|
||||||
|
monkeypatch.setattr("vllm_omni.platforms.current_omni_platform.is_cuda", lambda: False)
|
||||||
|
with pytest.raises(ValueError, match="CUDA"):
|
||||||
|
native_prefetch_enabled(config)
|
||||||
|
config.kv_transfer_config.kv_connector_extra_config["enable_kv_async_prefetch"] = False
|
||||||
|
assert not native_prefetch_enabled(config)
|
||||||
|
assert not native_prefetch_enabled(SimpleNamespace())
|
||||||
|
|
||||||
|
|
||||||
|
class _Connector:
|
||||||
|
def __init__(self):
|
||||||
|
self.commits = []
|
||||||
|
self.metas = []
|
||||||
|
|
||||||
|
def get_num_new_matched_tokens(self, request, computed):
|
||||||
|
return 4, True
|
||||||
|
|
||||||
|
def update_state_after_alloc(self, request, blocks, tokens):
|
||||||
|
self.commits.append(request.request_id)
|
||||||
|
self.metas.append(request.request_id)
|
||||||
|
# Upstream Mooncake mutates this flag after staging the receive.
|
||||||
|
request.kv_transfer_params["do_remote_prefill"] = False
|
||||||
|
|
||||||
|
def build_connector_meta(self, output):
|
||||||
|
meta, self.metas = tuple(self.metas), []
|
||||||
|
return meta
|
||||||
|
|
||||||
|
def update_connector_output(self, output):
|
||||||
|
pass
|
||||||
|
|
||||||
|
def request_finished(self, request, blocks):
|
||||||
|
return False, None
|
||||||
|
|
||||||
|
|
||||||
|
def _scheduler(monkeypatch, num_blocks=24):
|
||||||
|
monkeypatch.setattr("vllm_omni.platforms.current_omni_platform.is_cuda", lambda: True)
|
||||||
|
connector = _Connector()
|
||||||
|
monkeypatch.setattr(
|
||||||
|
"vllm_omni.diffusion.diffusion_kv.kv_connector.create_scheduler_kv_connector", lambda *args: connector
|
||||||
|
)
|
||||||
|
native_kv_managers.register_all_kvcache_specs(None)
|
||||||
|
spec = FullAttentionSpec(block_size=4, num_kv_heads=2, head_size=8, dtype=torch.bfloat16)
|
||||||
|
config = KVCacheConfig(
|
||||||
|
num_blocks=num_blocks,
|
||||||
|
kv_cache_tensors=[
|
||||||
|
KVCacheTensor(
|
||||||
|
size=spec.page_size_bytes * num_blocks,
|
||||||
|
layers=["layer0"],
|
||||||
|
layer_stride=spec.page_size_bytes,
|
||||||
|
block_stride=spec.page_size_bytes,
|
||||||
|
)
|
||||||
|
],
|
||||||
|
kv_cache_groups=[KVCacheGroupSpec(layer_names=["layer0"], kv_cache_spec=spec)],
|
||||||
|
)
|
||||||
|
scheduler = RequestScheduler()
|
||||||
|
scheduler.initialize(
|
||||||
|
_config(),
|
||||||
|
kv_cache_config=config,
|
||||||
|
scheduler_block_size=4,
|
||||||
|
hash_block_size=4,
|
||||||
|
kv_vllm_config=SimpleNamespace(model_config=SimpleNamespace(max_model_len=64), max_in_flight_tokens=64),
|
||||||
|
)
|
||||||
|
return scheduler, connector
|
||||||
|
|
||||||
|
|
||||||
|
def _request(rid):
|
||||||
|
return OmniDiffusionRequest(
|
||||||
|
prompt=rid,
|
||||||
|
request_id=rid,
|
||||||
|
sampling_params=OmniDiffusionSamplingParams(num_inference_steps=1),
|
||||||
|
diffusion_kv_requests=tuple(
|
||||||
|
DiffusionKVRequest(
|
||||||
|
f"{rid}/{row}", sequence_id=row, prefix_len=4, target_len=4, seq_len=8, prompt_token_ids=[1] * 4
|
||||||
|
)
|
||||||
|
for row in range(2)
|
||||||
|
),
|
||||||
|
kv_transfer_params={
|
||||||
|
"do_remote_prefill": True,
|
||||||
|
"transfer_id": f"xfer-{rid}",
|
||||||
|
"remote_engine_id": "ar",
|
||||||
|
"remote_bootstrap_addr": "localhost:8998",
|
||||||
|
"num_transfer_tokens": 4,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def test_scheduler_prefetches_one_waiting_request_and_reuses_allocation(monkeypatch):
|
||||||
|
scheduler, connector = _scheduler(monkeypatch)
|
||||||
|
for rid in ("A", "B", "C"):
|
||||||
|
scheduler.add_request(_request(rid))
|
||||||
|
output = scheduler.schedule()
|
||||||
|
assert output.scheduled_request_ids == ["A"]
|
||||||
|
assert output.kv_transfer_request_ids == {"A/0", "A/1", "B/0", "B/1"}
|
||||||
|
assert output.kv_required_request_ids == {"A/0", "A/1"}
|
||||||
|
assert output.kv_connector_metadata == ("A/0", "A/1")
|
||||||
|
assert output.kv_prefetch_connector_metadata == ("B/0", "B/1")
|
||||||
|
assert scheduler.get_request_state("B").status == DiffusionRequestStatus.WAITING
|
||||||
|
manager = scheduler._diffusion_kv_manager
|
||||||
|
b_meta = manager.get_metadata("B")
|
||||||
|
assert not manager.has_request("C")
|
||||||
|
a_blocks = {b for row in manager.get_metadata("A").sequences for group in row.block_ids for b in group}
|
||||||
|
b_blocks = {b for row in b_meta.sequences for group in row.block_ids for b in group}
|
||||||
|
assert a_blocks.isdisjoint(b_blocks)
|
||||||
|
scheduler.update_kv_connector_output(_completed("A/0", "A/1", "B/0", "B/1"))
|
||||||
|
scheduler.finish_requests("A", DiffusionRequestStatus.FINISHED_COMPLETED)
|
||||||
|
output = scheduler.schedule()
|
||||||
|
assert output.scheduled_request_ids == ["B"]
|
||||||
|
assert manager.get_metadata("B") is b_meta
|
||||||
|
assert output.kv_transfer_request_ids == {"C/0", "C/1"}
|
||||||
|
assert not output.kv_required_request_ids
|
||||||
|
assert output.kv_finished_request_ids == {"A/0", "A/1"}
|
||||||
|
assert connector.commits.count("B/0") == connector.commits.count("B/1") == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_prefetch_capacity_failure_rolls_back_all_cfg_rows(monkeypatch):
|
||||||
|
# One sentinel + A's four pages + only one of B's two CFG rows.
|
||||||
|
scheduler, connector = _scheduler(monkeypatch, num_blocks=7)
|
||||||
|
for rid in ("A", "B"):
|
||||||
|
scheduler.add_request(_request(rid))
|
||||||
|
output = scheduler.schedule()
|
||||||
|
assert output.kv_transfer_request_ids == {"A/0", "A/1"}
|
||||||
|
assert not scheduler._diffusion_kv_manager.has_request("B")
|
||||||
|
assert scheduler._native_prefetch_request_id is None
|
||||||
|
assert connector.commits == ["A/0", "A/1"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_abort_drains_b_before_releasing_pages(monkeypatch):
|
||||||
|
scheduler, _ = _scheduler(monkeypatch)
|
||||||
|
for rid in ("A", "B"):
|
||||||
|
scheduler.add_request(_request(rid))
|
||||||
|
scheduler.schedule()
|
||||||
|
scheduler.update_kv_connector_output(_completed("A/0", "A/1"))
|
||||||
|
scheduler.finish_requests("B", DiffusionRequestStatus.FINISHED_ABORTED)
|
||||||
|
assert "B" in scheduler._kv_draining_requests
|
||||||
|
assert scheduler._diffusion_kv_manager.has_request("B")
|
||||||
|
engine = object.__new__(DiffusionEngine)
|
||||||
|
engine.scheduler = scheduler
|
||||||
|
engine._remove_diffusion_kv_requests = Mock()
|
||||||
|
engine.abort_queue = SimpleNamespace(empty=lambda: True)
|
||||||
|
|
||||||
|
def drain(output):
|
||||||
|
assert output.kv_required_request_ids == {"B/0", "B/1"}
|
||||||
|
assert scheduler._diffusion_kv_manager.has_request("B")
|
||||||
|
engine._remove_diffusion_kv_requests.assert_not_called()
|
||||||
|
return _completed("A/0", "A/1", "B/0", "B/1")
|
||||||
|
|
||||||
|
engine.executor = SimpleNamespace(prepare_kv_for_forward=drain)
|
||||||
|
engine._abort_requests(["B"])
|
||||||
|
assert not scheduler._diffusion_kv_manager.has_request("B")
|
||||||
|
assert scheduler.get_request_state("B").status == DiffusionRequestStatus.FINISHED_ABORTED
|
||||||
|
assert scheduler._native_prefetch_request_id is None
|
||||||
|
|
||||||
|
|
||||||
|
def test_pending_prefetch_reuses_transfer_when_admitted(monkeypatch):
|
||||||
|
scheduler, connector = _scheduler(monkeypatch)
|
||||||
|
for rid in ("A", "B"):
|
||||||
|
scheduler.add_request(_request(rid))
|
||||||
|
scheduler.schedule()
|
||||||
|
scheduler.update_kv_connector_output(_completed("A/0", "A/1", "B/0"))
|
||||||
|
assert "B" in scheduler._kv_loading_request_ids
|
||||||
|
scheduler.finish_requests("A", DiffusionRequestStatus.FINISHED_COMPLETED)
|
||||||
|
output = scheduler.schedule()
|
||||||
|
assert output.scheduled_request_ids == ["B"]
|
||||||
|
assert output.kv_required_request_ids == {"B/0", "B/1"}
|
||||||
|
assert not output.kv_transfer_request_ids
|
||||||
|
assert len(connector.commits) == 4
|
||||||
|
|
||||||
|
|
||||||
|
def test_missing_source_skips_prefetch_without_allocating(monkeypatch):
|
||||||
|
scheduler, connector = _scheduler(monkeypatch)
|
||||||
|
scheduler.add_request(_request("A"))
|
||||||
|
b = _request("B")
|
||||||
|
del b.kv_transfer_params["remote_bootstrap_addr"]
|
||||||
|
scheduler.add_request(b)
|
||||||
|
scheduler.schedule()
|
||||||
|
assert not scheduler._diffusion_kv_manager.has_request("B")
|
||||||
|
assert connector.commits == ["A/0", "A/1"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_prefetch_reservation_error_does_not_fail_current_request(monkeypatch):
|
||||||
|
scheduler, connector = _scheduler(monkeypatch)
|
||||||
|
for rid in ("A", "B"):
|
||||||
|
scheduler.add_request(_request(rid))
|
||||||
|
manager = scheduler._diffusion_kv_manager
|
||||||
|
reserve = manager.reserve_request
|
||||||
|
|
||||||
|
def fail_b(request_id, requests):
|
||||||
|
if request_id == "B":
|
||||||
|
raise ValueError("invalid B layout")
|
||||||
|
return reserve(request_id, requests)
|
||||||
|
|
||||||
|
monkeypatch.setattr(manager, "reserve_request", fail_b)
|
||||||
|
output = scheduler.schedule()
|
||||||
|
assert output.scheduled_request_ids == ["A"]
|
||||||
|
assert connector.commits == ["A/0", "A/1"]
|
||||||
|
assert not manager.has_request("B")
|
||||||
|
scheduler.update_kv_connector_output(_completed("A/0", "A/1"))
|
||||||
|
scheduler.finish_requests("A", DiffusionRequestStatus.FINISHED_COMPLETED)
|
||||||
|
output = scheduler.schedule()
|
||||||
|
assert scheduler.get_request_state("B").status == DiffusionRequestStatus.FINISHED_ERROR
|
||||||
|
assert "B" in output.finished_req_ids
|
||||||
|
|
||||||
|
|
||||||
|
def test_invalid_prefetch_boundary_is_rejected_before_reservation(monkeypatch):
|
||||||
|
scheduler, connector = _scheduler(monkeypatch)
|
||||||
|
for rid in ("A", "B"):
|
||||||
|
scheduler.add_request(_request(rid))
|
||||||
|
b_rows = scheduler.get_request_state("B").diffusion_kv_requests
|
||||||
|
b_rows[1].kv_transfer_params["num_transfer_tokens"] = b_rows[1].num_tokens + 1
|
||||||
|
manager = scheduler._diffusion_kv_manager
|
||||||
|
reserve = manager.reserve_request
|
||||||
|
|
||||||
|
def fail_if_b_reserved(request_id, requests):
|
||||||
|
if request_id == "B":
|
||||||
|
raise AssertionError("invalid B must be rejected before reservation")
|
||||||
|
return reserve(request_id, requests)
|
||||||
|
|
||||||
|
monkeypatch.setattr(manager, "reserve_request", fail_if_b_reserved)
|
||||||
|
output = scheduler.schedule()
|
||||||
|
assert output.scheduled_request_ids == ["A"]
|
||||||
|
assert scheduler.get_request_state("B").status == DiffusionRequestStatus.WAITING
|
||||||
|
assert not manager.has_request("B")
|
||||||
|
|
||||||
|
|
||||||
|
def test_abort_timeout_keeps_target_pages(monkeypatch):
|
||||||
|
scheduler, _ = _scheduler(monkeypatch)
|
||||||
|
for rid in ("A", "B"):
|
||||||
|
scheduler.add_request(_request(rid))
|
||||||
|
scheduler.schedule()
|
||||||
|
engine = object.__new__(DiffusionEngine)
|
||||||
|
engine.scheduler = scheduler
|
||||||
|
engine.executor = SimpleNamespace(prepare_kv_for_forward=Mock(side_effect=TimeoutError("receive")))
|
||||||
|
engine._fail_engine = Mock()
|
||||||
|
engine._remove_diffusion_kv_requests = Mock()
|
||||||
|
with pytest.raises(TimeoutError):
|
||||||
|
engine._abort_requests(["B"])
|
||||||
|
engine._fail_engine.assert_called_once()
|
||||||
|
engine._remove_diffusion_kv_requests.assert_not_called()
|
||||||
|
assert scheduler._diffusion_kv_manager.has_request("B")
|
||||||
|
|
||||||
|
|
||||||
|
def test_sleep_rejects_live_prefetch_before_touching_allocator():
|
||||||
|
from vllm_omni.diffusion.worker.diffusion_worker import DiffusionWorker
|
||||||
|
|
||||||
|
worker = SimpleNamespace(model_runner=SimpleNamespace(_kv_receive_progress=KVReceiveProgress(submitted={"B/0"})))
|
||||||
|
with pytest.raises(RuntimeError, match="Cannot sleep"):
|
||||||
|
DiffusionWorker.sleep(worker)
|
||||||
|
|
||||||
|
|
||||||
|
def test_registration_failure_retires_b_and_flushes_cancellation(monkeypatch):
|
||||||
|
from vllm_omni.diffusion.diffusion_kv import kv_connector
|
||||||
|
|
||||||
|
scheduler, connector = _scheduler(monkeypatch)
|
||||||
|
for rid in ("A", "B"):
|
||||||
|
scheduler.add_request(_request(rid))
|
||||||
|
commit = kv_connector.commit_kv_load
|
||||||
|
|
||||||
|
def fail_b(conn, manager, requests, matched):
|
||||||
|
if requests[0].request_id.startswith("B/"):
|
||||||
|
conn.metas.append("B cancellation")
|
||||||
|
raise kv_connector.KVTransferRegistrationError("registration rollback")
|
||||||
|
return commit(conn, manager, requests, matched)
|
||||||
|
|
||||||
|
monkeypatch.setattr("vllm_omni.diffusion.sched.base_scheduler.commit_kv_load", fail_b)
|
||||||
|
output = scheduler.schedule()
|
||||||
|
assert output.scheduled_request_ids == ["A"]
|
||||||
|
assert output.finished_req_ids == {"B"}
|
||||||
|
assert output.kv_finished_request_ids == {"B/0", "B/1"}
|
||||||
|
assert output.kv_prefetch_connector_metadata == ("B cancellation",)
|
||||||
|
assert not output.kv_prefetch_request_ids
|
||||||
|
assert output.num_waiting_reqs == 0
|
||||||
|
assert scheduler.get_request_state("B").status == DiffusionRequestStatus.FINISHED_ERROR
|
||||||
|
assert not scheduler._diffusion_kv_manager.has_request("B")
|
||||||
|
assert scheduler._diffusion_kv_manager.has_request("A")
|
||||||
|
|
||||||
|
|
||||||
|
def test_partial_cfg_registration_rolls_back_every_row(monkeypatch):
|
||||||
|
from vllm_omni.diffusion.diffusion_kv import kv_connector
|
||||||
|
|
||||||
|
class FakeMooncakeConnector:
|
||||||
|
def __init__(self):
|
||||||
|
self.staged = []
|
||||||
|
self.cancelled = []
|
||||||
|
|
||||||
|
def update_state_after_alloc(self, request, blocks, tokens):
|
||||||
|
self.staged.append(request.request_id)
|
||||||
|
if request.request_id.endswith("/1"):
|
||||||
|
raise RuntimeError("second CFG row failed")
|
||||||
|
|
||||||
|
def request_finished(self, request, blocks):
|
||||||
|
self.cancelled.append(request.request_id)
|
||||||
|
return False, None
|
||||||
|
|
||||||
|
monkeypatch.setattr(
|
||||||
|
"vllm.distributed.kv_transfer.kv_connector.v1.mooncake.mooncake_connector.MooncakeConnector",
|
||||||
|
FakeMooncakeConnector,
|
||||||
|
)
|
||||||
|
scheduler, _ = _scheduler(monkeypatch)
|
||||||
|
scheduler.add_request(_request("B"))
|
||||||
|
state = scheduler.get_request_state("B")
|
||||||
|
rows = state.diffusion_kv_requests
|
||||||
|
for row in rows:
|
||||||
|
row.kv_transfer_params = dict(state.req.kv_transfer_params)
|
||||||
|
allocation = scheduler._diffusion_kv_manager.reserve_request("B", rows)
|
||||||
|
connector = FakeMooncakeConnector()
|
||||||
|
|
||||||
|
with pytest.raises(kv_connector.KVTransferRegistrationError, match="Could not register"):
|
||||||
|
kv_connector.commit_kv_load(
|
||||||
|
connector,
|
||||||
|
scheduler._diffusion_kv_manager.native_manager,
|
||||||
|
rows,
|
||||||
|
[4, 4],
|
||||||
|
)
|
||||||
|
assert allocation is not None
|
||||||
|
assert connector.staged == ["B/0", "B/1"]
|
||||||
|
assert connector.cancelled == ["B/0", "B/1"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_drain_after_terminal_request_state_was_popped(monkeypatch):
|
||||||
|
scheduler, _ = _scheduler(monkeypatch)
|
||||||
|
for rid in ("A", "B"):
|
||||||
|
scheduler.add_request(_request(rid))
|
||||||
|
scheduler.schedule()
|
||||||
|
scheduler.finish_requests("B", DiffusionRequestStatus.FINISHED_ABORTED)
|
||||||
|
scheduler.pop_request_state("B")
|
||||||
|
output = scheduler.native_kv_poll_output(drain_request_ids=["B"])
|
||||||
|
assert output.kv_required_request_ids == {"B/0", "B/1"}
|
||||||
|
assert scheduler._diffusion_kv_manager.has_request("B")
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("prefetch", [False, True])
|
||||||
|
def test_real_active_connector_starts_required_load_before_polling(prefetch):
|
||||||
|
from vllm.v1.worker.gpu.kv_connector import ActiveKVConnector
|
||||||
|
|
||||||
|
from vllm_omni.diffusion.diffusion_kv.kv_connector import wait_for_kv_load
|
||||||
|
|
||||||
|
active = object.__new__(ActiveKVConnector)
|
||||||
|
active._disabled = False
|
||||||
|
active._pending_load_start = False
|
||||||
|
active.kv_connector = Mock()
|
||||||
|
active.kv_connector.get_block_ids_with_load_errors.return_value = set()
|
||||||
|
active._start_load_kv = Mock()
|
||||||
|
|
||||||
|
def finished(_):
|
||||||
|
active._start_load_kv.assert_called_once()
|
||||||
|
return set(), {"A0"}
|
||||||
|
|
||||||
|
active.kv_connector.get_finished.side_effect = finished
|
||||||
|
output = _output(submit=["A0"], required=["A0"], metadata=object())
|
||||||
|
if prefetch:
|
||||||
|
result = KVReceiveProgress().prepare(active, output, 1)
|
||||||
|
else:
|
||||||
|
output.kv_required_request_ids = None
|
||||||
|
result = wait_for_kv_load(active, output, 1)
|
||||||
|
assert result.finished_recving == {"A0"}
|
||||||
@@ -38,6 +38,7 @@ def _make_engine() -> DiffusionEngine:
|
|||||||
engine._cv = threading.Condition(engine._rpc_lock)
|
engine._cv = threading.Condition(engine._rpc_lock)
|
||||||
engine._out_streams = {}
|
engine._out_streams = {}
|
||||||
engine._closed = False
|
engine._closed = False
|
||||||
|
engine._shutting_down = False
|
||||||
engine._shutdown_complete = False
|
engine._shutdown_complete = False
|
||||||
engine.abort_queue = queue.Queue()
|
engine.abort_queue = queue.Queue()
|
||||||
engine._loop_started = False
|
engine._loop_started = False
|
||||||
@@ -379,3 +380,22 @@ def test_close_defers_resource_shutdown_until_worker_thread_stops() -> None:
|
|||||||
engine.executor.shutdown.assert_called_once()
|
engine.executor.shutdown.assert_called_once()
|
||||||
assert engine._shutdown_complete is True
|
assert engine._shutdown_complete is True
|
||||||
assert engine._loop_started is False
|
assert engine._loop_started is False
|
||||||
|
|
||||||
|
|
||||||
|
def test_fail_engine_does_not_reenter_when_executor_shutdown_fails() -> None:
|
||||||
|
engine = _make_engine()
|
||||||
|
shutdown_error = RuntimeError("worker shutdown failed")
|
||||||
|
engine.executor.shutdown.side_effect = shutdown_error
|
||||||
|
engine.scheduler.close = Mock()
|
||||||
|
engine._fail_pending_rpcs = Mock()
|
||||||
|
|
||||||
|
with pytest.raises(RuntimeError, match="worker shutdown failed"):
|
||||||
|
engine._fail_engine(RuntimeError("engine failed"))
|
||||||
|
|
||||||
|
engine._fail_engine(RuntimeError("follow-up failure"))
|
||||||
|
|
||||||
|
engine.executor.shutdown.assert_called_once_with()
|
||||||
|
engine.scheduler.close.assert_not_called()
|
||||||
|
assert engine._closed is True
|
||||||
|
assert engine._shutting_down is True
|
||||||
|
assert engine._shutdown_complete is False
|
||||||
@@ -220,6 +220,7 @@ def test_paged_kv_profile_requests_match_per_rank_batch(
|
|||||||
width=1024,
|
width=1024,
|
||||||
guidance_scale=5.0,
|
guidance_scale=5.0,
|
||||||
num_image_inputs=3,
|
num_image_inputs=3,
|
||||||
|
num_frames=1,
|
||||||
)
|
)
|
||||||
engine._prepare_request_for_admission.assert_called_once_with(request)
|
engine._prepare_request_for_admission.assert_called_once_with(request)
|
||||||
|
|
||||||
|
|||||||
@@ -48,6 +48,7 @@ def mock_gpu_worker(mocker: MockerFixture, mock_od_config):
|
|||||||
worker = DiffusionWorker(local_rank=0, rank=0, od_config=mock_od_config)
|
worker = DiffusionWorker(local_rank=0, rank=0, od_config=mock_od_config)
|
||||||
# Mock the model_runner with pipeline
|
# Mock the model_runner with pipeline
|
||||||
worker.model_runner = mocker.Mock()
|
worker.model_runner = mocker.Mock()
|
||||||
|
worker.model_runner._kv_receive_progress = None
|
||||||
worker.model_runner.pipeline = mocker.Mock()
|
worker.model_runner.pipeline = mocker.Mock()
|
||||||
worker.device = torch.device("cuda", 0)
|
worker.device = torch.device("cuda", 0)
|
||||||
worker._sleep_saved_buffers = {}
|
worker._sleep_saved_buffers = {}
|
||||||
|
|||||||
@@ -385,6 +385,7 @@ class DiffusionEngine:
|
|||||||
self._cv = threading.Condition(self._rpc_lock)
|
self._cv = threading.Condition(self._rpc_lock)
|
||||||
self._out_streams: dict[str, asyncio.Queue[DiffusionOutput]] = {}
|
self._out_streams: dict[str, asyncio.Queue[DiffusionOutput]] = {}
|
||||||
self._closed = False
|
self._closed = False
|
||||||
|
self._shutting_down = False
|
||||||
self._shutdown_complete = False
|
self._shutdown_complete = False
|
||||||
self.abort_queue: queue.Queue[str] = queue.Queue()
|
self.abort_queue: queue.Queue[str] = queue.Queue()
|
||||||
self._rpc_queue: queue.Queue[_RpcTask] = queue.Queue()
|
self._rpc_queue: queue.Queue[_RpcTask] = queue.Queue()
|
||||||
@@ -635,6 +636,7 @@ class DiffusionEngine:
|
|||||||
if sched_output.scheduled_request_ids
|
if sched_output.scheduled_request_ids
|
||||||
else BatchRunnerOutput.from_list([])
|
else BatchRunnerOutput.from_list([])
|
||||||
)
|
)
|
||||||
|
self._poll_native_kv()
|
||||||
worker_execution_completed = True
|
worker_execution_completed = True
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
if self._closed:
|
if self._closed:
|
||||||
@@ -845,7 +847,12 @@ class DiffusionEngine:
|
|||||||
raise
|
raise
|
||||||
|
|
||||||
def _prepare_kv_for_forward(self, sched_output: DiffusionSchedulerOutput) -> None:
|
def _prepare_kv_for_forward(self, sched_output: DiffusionSchedulerOutput) -> None:
|
||||||
if getattr(sched_output, "kv_connector_metadata", None) is None:
|
if (
|
||||||
|
getattr(sched_output, "kv_connector_metadata", None) is None
|
||||||
|
and getattr(sched_output, "kv_prefetch_connector_metadata", None) is None
|
||||||
|
and not getattr(sched_output, "kv_required_request_ids", None)
|
||||||
|
and not getattr(sched_output, "kv_poll_only", False)
|
||||||
|
):
|
||||||
return
|
return
|
||||||
try:
|
try:
|
||||||
output = self.executor.prepare_kv_for_forward(sched_output)
|
output = self.executor.prepare_kv_for_forward(sched_output)
|
||||||
@@ -853,7 +860,10 @@ class DiffusionEngine:
|
|||||||
self.scheduler.update_kv_connector_output(output)
|
self.scheduler.update_kv_connector_output(output)
|
||||||
if not self.abort_queue.empty():
|
if not self.abort_queue.empty():
|
||||||
self._process_aborts_queue()
|
self._process_aborts_queue()
|
||||||
incomplete = sched_output.kv_transfer_request_ids - (output.finished_recving or set())
|
required_ids = sched_output.kv_required_request_ids
|
||||||
|
if required_ids is None:
|
||||||
|
required_ids = sched_output.kv_transfer_request_ids
|
||||||
|
incomplete = required_ids - (output.finished_recving or set())
|
||||||
sched_output.finished_req_ids.update(self.scheduler.fail_incomplete_kv_loads(incomplete))
|
sched_output.finished_req_ids.update(self.scheduler.fail_incomplete_kv_loads(incomplete))
|
||||||
# Timed-out/cancelled requests must never reach model execution.
|
# Timed-out/cancelled requests must never reach model execution.
|
||||||
terminal = {
|
terminal = {
|
||||||
@@ -879,26 +889,37 @@ class DiffusionEngine:
|
|||||||
self._fail_engine(exc)
|
self._fail_engine(exc)
|
||||||
raise
|
raise
|
||||||
|
|
||||||
def _fail_engine(self, exc: Exception) -> None:
|
def _poll_native_kv(self, *, drain_request_ids: list[str] | None = None) -> None:
|
||||||
if getattr(self, "_shutdown_complete", False):
|
poll = getattr(self.scheduler, "native_kv_poll_output", None)
|
||||||
|
if poll is None:
|
||||||
return
|
return
|
||||||
logger.error("Diffusion engine failed; stopping workers before releasing KV pages", exc_info=exc)
|
output = poll(drain_request_ids=drain_request_ids)
|
||||||
|
if output is not None:
|
||||||
|
self._prepare_kv_for_forward(output)
|
||||||
|
|
||||||
|
def _fail_engine(self, exc: Exception) -> None:
|
||||||
with self._cv:
|
with self._cv:
|
||||||
|
if getattr(self, "_shutdown_complete", False) or getattr(self, "_shutting_down", False):
|
||||||
|
return
|
||||||
|
# Mark shutdown before invoking any component callbacks. A
|
||||||
|
# failing shutdown can re-enter this method from another error
|
||||||
|
# path, and repeating executor.shutdown() is unsafe.
|
||||||
|
self._shutting_down = True
|
||||||
self._closed = True
|
self._closed = True
|
||||||
if self.stop_event is not None:
|
if self.stop_event is not None:
|
||||||
self.stop_event.set()
|
self.stop_event.set()
|
||||||
streams = list(self._out_streams.values())
|
streams = list(self._out_streams.values())
|
||||||
self._cv.notify_all()
|
self._cv.notify_all()
|
||||||
|
|
||||||
|
logger.error("Diffusion engine failed; stopping workers before releasing KV pages", exc_info=exc)
|
||||||
for stream in streams:
|
for stream in streams:
|
||||||
self._put_queue_output(stream, DiffusionOutput.from_exception(exc))
|
self._put_queue_output(stream, DiffusionOutput.from_exception(exc))
|
||||||
self._fail_pending_rpcs(exc)
|
self._fail_pending_rpcs(exc)
|
||||||
try:
|
# If Worker shutdown fails, retain the Scheduler reservations. A
|
||||||
self.executor.shutdown()
|
# remote producer may still be writing into those allocations.
|
||||||
finally:
|
self.executor.shutdown()
|
||||||
try:
|
self.scheduler.close()
|
||||||
self.scheduler.close()
|
self._shutdown_complete = True
|
||||||
finally:
|
|
||||||
self._shutdown_complete = True
|
|
||||||
|
|
||||||
def _emit_finished_outputs(
|
def _emit_finished_outputs(
|
||||||
self,
|
self,
|
||||||
@@ -1150,6 +1171,7 @@ class DiffusionEngine:
|
|||||||
if sched_output.scheduled_request_ids
|
if sched_output.scheduled_request_ids
|
||||||
else BatchRunnerOutput.from_list([])
|
else BatchRunnerOutput.from_list([])
|
||||||
)
|
)
|
||||||
|
self._poll_native_kv()
|
||||||
except EngineDeadError:
|
except EngineDeadError:
|
||||||
raise
|
raise
|
||||||
except Exception as exc:
|
except Exception as exc:
|
||||||
@@ -1225,6 +1247,7 @@ class DiffusionEngine:
|
|||||||
guidance_scale: float,
|
guidance_scale: float,
|
||||||
num_image_inputs: int = 1,
|
num_image_inputs: int = 1,
|
||||||
num_inference_steps: int = 1,
|
num_inference_steps: int = 1,
|
||||||
|
num_frames: int | None = None,
|
||||||
) -> OmniDiffusionRequest | None:
|
) -> OmniDiffusionRequest | None:
|
||||||
"""Build a minimal model request for startup profiling or warmup."""
|
"""Build a minimal model request for startup profiling or warmup."""
|
||||||
prompt = OmniTextPrompt(prompt="dummy run")
|
prompt = OmniTextPrompt(prompt="dummy run")
|
||||||
@@ -1241,7 +1264,8 @@ class DiffusionEngine:
|
|||||||
audio_sr = 16000
|
audio_sr = 16000
|
||||||
prompt.setdefault("multi_modal_data", {})["audio"] = np.random.randn(audio_sr * 2).astype(np.float32)
|
prompt.setdefault("multi_modal_data", {})["audio"] = np.random.randn(audio_sr * 2).astype(np.float32)
|
||||||
|
|
||||||
num_frames = get_dummy_run_num_frames(model_class_name, supports_audio_input)
|
if num_frames is None:
|
||||||
|
num_frames = get_dummy_run_num_frames(model_class_name, supports_audio_input)
|
||||||
if num_frames <= 0:
|
if num_frames <= 0:
|
||||||
return None
|
return None
|
||||||
return OmniDiffusionRequest(
|
return OmniDiffusionRequest(
|
||||||
@@ -1293,6 +1317,9 @@ class DiffusionEngine:
|
|||||||
width=profile_width,
|
width=profile_width,
|
||||||
guidance_scale=5.0,
|
guidance_scale=5.0,
|
||||||
num_image_inputs=get_dummy_run_num_image_inputs(model_class_name),
|
num_image_inputs=get_dummy_run_num_image_inputs(model_class_name),
|
||||||
|
# Hunyuan skips generic warmup, but paged KV still needs its
|
||||||
|
# prepared image request to measure the startup memory envelope.
|
||||||
|
num_frames=1,
|
||||||
)
|
)
|
||||||
if request is None:
|
if request is None:
|
||||||
raise RuntimeError("paged_scheduler requires a runnable Diffusion KV memory profile request")
|
raise RuntimeError("paged_scheduler requires a runnable Diffusion KV memory profile request")
|
||||||
@@ -1495,8 +1522,15 @@ class DiffusionEngine:
|
|||||||
else:
|
else:
|
||||||
self._loop_started = False
|
self._loop_started = False
|
||||||
|
|
||||||
self.scheduler.close()
|
if getattr(self.scheduler, "_native_prefetch_enabled", False):
|
||||||
self.executor.shutdown()
|
# No new schedules after the busy loop exits. Finish outstanding
|
||||||
|
# writes before shutting down consumers and releasing reservations.
|
||||||
|
self._poll_native_kv(drain_request_ids=list(self.scheduler._kv_loading_request_ids))
|
||||||
|
self.executor.shutdown()
|
||||||
|
self.scheduler.close()
|
||||||
|
else:
|
||||||
|
self.scheduler.close()
|
||||||
|
self.executor.shutdown()
|
||||||
self._shutdown_complete = True
|
self._shutdown_complete = True
|
||||||
|
|
||||||
def abort(self, request_id: str | Iterable[str]) -> None:
|
def abort(self, request_id: str | Iterable[str]) -> None:
|
||||||
@@ -1532,6 +1566,8 @@ class DiffusionEngine:
|
|||||||
request_ids = [request_ids] if isinstance(request_ids, str) else list(request_ids)
|
request_ids = [request_ids] if isinstance(request_ids, str) else list(request_ids)
|
||||||
request_ids = list(dict.fromkeys(request_ids))
|
request_ids = list(dict.fromkeys(request_ids))
|
||||||
|
|
||||||
|
self._poll_native_kv(drain_request_ids=request_ids)
|
||||||
|
|
||||||
for request_id in request_ids:
|
for request_id in request_ids:
|
||||||
if self.scheduler.get_request_state(request_id) is not None:
|
if self.scheduler.get_request_state(request_id) is not None:
|
||||||
self.scheduler.finish_requests(request_id, DiffusionRequestStatus.FINISHED_ABORTED)
|
self.scheduler.finish_requests(request_id, DiffusionRequestStatus.FINISHED_ABORTED)
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ import math
|
|||||||
import time
|
import time
|
||||||
from collections import Counter
|
from collections import Counter
|
||||||
from collections.abc import Mapping
|
from collections.abc import Mapping
|
||||||
|
from dataclasses import dataclass, field
|
||||||
from typing import TYPE_CHECKING, Any
|
from typing import TYPE_CHECKING, Any
|
||||||
|
|
||||||
from vllm.config import KVTransferConfig, VllmConfig
|
from vllm.config import KVTransferConfig, VllmConfig
|
||||||
@@ -35,6 +36,96 @@ class KVTransferRegistrationError(ValueError):
|
|||||||
"""A failed handoff whose destination pages have not been dispatched."""
|
"""A failed handoff whose destination pages have not been dispatched."""
|
||||||
|
|
||||||
|
|
||||||
|
def native_prefetch_enabled(od_config: OmniDiffusionConfig) -> bool:
|
||||||
|
"""Validate the opt-in GPU consumer implementation before allocating pages."""
|
||||||
|
config = getattr(od_config, "kv_transfer_config", None)
|
||||||
|
if not isinstance(config, KVTransferConfig):
|
||||||
|
return False
|
||||||
|
extra = config.kv_connector_extra_config or {}
|
||||||
|
enabled = extra.get("enable_kv_async_prefetch", False)
|
||||||
|
if not isinstance(enabled, bool):
|
||||||
|
raise ValueError("enable_kv_async_prefetch must be a boolean")
|
||||||
|
if not enabled:
|
||||||
|
return False
|
||||||
|
from vllm_omni.diffusion.diffusion_kv.config import DiffusionKVCacheMode
|
||||||
|
from vllm_omni.platforms import current_omni_platform
|
||||||
|
|
||||||
|
if (
|
||||||
|
not current_omni_platform.is_cuda()
|
||||||
|
or config.kv_connector != "MooncakeConnector"
|
||||||
|
or config.kv_role != "kv_consumer"
|
||||||
|
or extra.get("mooncake_protocol", "rdma") != "tcp"
|
||||||
|
or od_config.diffusion_kv_mode is not DiffusionKVCacheMode.PAGED_SCHEDULER
|
||||||
|
or od_config.max_num_seqs != 1
|
||||||
|
or getattr(od_config, "cfg_kv_collect_func", None) is not None
|
||||||
|
):
|
||||||
|
raise ValueError(
|
||||||
|
"Native KV prefetch requires CUDA, MooncakeConnector TCP consumer, "
|
||||||
|
"paged_scheduler, max_num_seqs=1 and no CFG companion collector"
|
||||||
|
)
|
||||||
|
return True
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class KVReceiveProgress:
|
||||||
|
"""Rank-local state survives consumption of Mooncake completion events."""
|
||||||
|
|
||||||
|
submitted: set[str] = field(default_factory=set)
|
||||||
|
received: set[str] = field(default_factory=set)
|
||||||
|
sent: set[str] = field(default_factory=set)
|
||||||
|
deadlines: dict[str, float] = field(default_factory=dict)
|
||||||
|
|
||||||
|
def prepare(
|
||||||
|
self, active_connector: ActiveKVConnector, output: DiffusionSchedulerOutput, timeout: float
|
||||||
|
) -> KVConnectorOutput:
|
||||||
|
new_ids = output.kv_transfer_request_ids
|
||||||
|
if new_ids & self.submitted:
|
||||||
|
raise RuntimeError("Duplicate native KV receive submission")
|
||||||
|
required = output.kv_required_request_ids or set()
|
||||||
|
if not required.issubset(self.submitted | new_ids):
|
||||||
|
raise RuntimeError("Required native KV receive was never submitted")
|
||||||
|
if output.kv_connector_metadata is not None:
|
||||||
|
active_connector.pre_forward(output)
|
||||||
|
elif new_ids:
|
||||||
|
raise RuntimeError("Native KV submission requires connector metadata")
|
||||||
|
self.submitted.update(new_ids)
|
||||||
|
self.deadlines.update(dict.fromkeys(new_ids, time.monotonic() + timeout))
|
||||||
|
|
||||||
|
while True:
|
||||||
|
# post_forward also consumes get_finished(); always retain its
|
||||||
|
# events, including completions for a request not required yet.
|
||||||
|
result = active_connector.post_forward(output.kv_finished_request_ids)
|
||||||
|
# Cancelling a request before admission can send a zero-page
|
||||||
|
# notification to release its producer. Its late acknowledgement
|
||||||
|
# has no receive reservation and must not become a leaked record.
|
||||||
|
self.received.update(self.submitted.intersection(result.finished_recving or ()))
|
||||||
|
self.sent.update(self.submitted.intersection(result.finished_sending or ()))
|
||||||
|
if result.invalid_block_ids:
|
||||||
|
raise RuntimeError("Diffusion KV connector reported invalid remote pages")
|
||||||
|
expired = {
|
||||||
|
rid
|
||||||
|
for rid, deadline in self.deadlines.items()
|
||||||
|
if rid not in self.received and time.monotonic() >= deadline
|
||||||
|
}
|
||||||
|
if expired:
|
||||||
|
raise TimeoutError(f"Timed out receiving diffusion KV for {sorted(expired)}")
|
||||||
|
if required.issubset(self.received):
|
||||||
|
break
|
||||||
|
time.sleep(0.001)
|
||||||
|
|
||||||
|
retired = output.kv_finished_request_ids
|
||||||
|
if (retired & self.submitted) - self.received:
|
||||||
|
raise RuntimeError("Cannot retire an unfinished native KV receive")
|
||||||
|
self.submitted.difference_update(retired)
|
||||||
|
self.received.difference_update(retired)
|
||||||
|
self.sent.difference_update(retired)
|
||||||
|
for rid in retired:
|
||||||
|
self.deadlines.pop(rid, None)
|
||||||
|
result.finished_recving = set(self.received)
|
||||||
|
result.finished_sending = set(self.sent)
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
def mint_transfer_id(request_id: str) -> str:
|
def mint_transfer_id(request_id: str) -> str:
|
||||||
"""Return the stable ticket shared by one AR -> Diffusion handoff."""
|
"""Return the stable ticket shared by one AR -> Diffusion handoff."""
|
||||||
|
|
||||||
@@ -235,6 +326,21 @@ def install_mooncake_cfg_fanout(connector: KVConnectorBase_V1) -> None:
|
|||||||
setattr(worker, "_omni_cfg_fanout_installed", True)
|
setattr(worker, "_omni_cfg_fanout_installed", True)
|
||||||
|
|
||||||
|
|
||||||
|
def validate_kv_transfer_boundaries(requests: tuple[DiffusionKVRequest, ...], matched_tokens: list[int]) -> None:
|
||||||
|
"""Validate every CFG row before reserving or registering destination pages."""
|
||||||
|
for request, num_tokens in zip(requests, matched_tokens, strict=True):
|
||||||
|
params = request.kv_transfer_params
|
||||||
|
if num_tokens <= 0 or params is None:
|
||||||
|
continue
|
||||||
|
transfer_tokens = params.get("num_transfer_tokens")
|
||||||
|
if type(transfer_tokens) is not int or not num_tokens <= transfer_tokens <= request.num_tokens:
|
||||||
|
raise KVTransferRegistrationError(
|
||||||
|
"Diffusion KV transfer boundary must cover the reusable prefix "
|
||||||
|
f"without exceeding the allocated sequence: reusable={num_tokens}, "
|
||||||
|
f"transfer={transfer_tokens!r}, allocated={request.num_tokens}"
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def commit_kv_load(
|
def commit_kv_load(
|
||||||
connector: KVConnectorBase_V1,
|
connector: KVConnectorBase_V1,
|
||||||
manager: KVCacheManager,
|
manager: KVCacheManager,
|
||||||
@@ -246,6 +352,7 @@ def commit_kv_load(
|
|||||||
# All rows sharing a transfer_id must reach the same connector metadata:
|
# All rows sharing a transfer_id must reach the same connector metadata:
|
||||||
# the producer uses this complete per-rank row set for fan-out accounting.
|
# the producer uses this complete per-rank row set for fan-out accounting.
|
||||||
# Validate every CFG row before mutating any connector state.
|
# Validate every CFG row before mutating any connector state.
|
||||||
|
validate_kv_transfer_boundaries(requests, matched_tokens)
|
||||||
for request, num_tokens in zip(requests, matched_tokens, strict=True):
|
for request, num_tokens in zip(requests, matched_tokens, strict=True):
|
||||||
blocks = manager.get_blocks(request.request_id)
|
blocks = manager.get_blocks(request.request_id)
|
||||||
# Mooncake's producer advertises complete physical blocks. Keep every
|
# Mooncake's producer advertises complete physical blocks. Keep every
|
||||||
@@ -256,12 +363,6 @@ def commit_kv_load(
|
|||||||
transfer_tokens = num_tokens
|
transfer_tokens = num_tokens
|
||||||
if num_tokens > 0 and request.kv_transfer_params is not None:
|
if num_tokens > 0 and request.kv_transfer_params is not None:
|
||||||
transfer_tokens = request.kv_transfer_params["num_transfer_tokens"]
|
transfer_tokens = request.kv_transfer_params["num_transfer_tokens"]
|
||||||
if type(transfer_tokens) is not int or not num_tokens <= transfer_tokens <= request.num_tokens:
|
|
||||||
raise KVTransferRegistrationError(
|
|
||||||
"Diffusion KV transfer boundary must cover the reusable prefix "
|
|
||||||
f"without exceeding the allocated sequence: reusable={num_tokens}, "
|
|
||||||
f"transfer={transfer_tokens!r}, allocated={request.num_tokens}"
|
|
||||||
)
|
|
||||||
prefix_blocks = KVCacheBlocks(
|
prefix_blocks = KVCacheBlocks(
|
||||||
tuple(
|
tuple(
|
||||||
group[: (transfer_tokens + spec.kv_cache_spec.block_size - 1) // spec.kv_cache_spec.block_size]
|
group[: (transfer_tokens + spec.kv_cache_spec.block_size - 1) // spec.kv_cache_spec.block_size]
|
||||||
|
|||||||
@@ -177,7 +177,31 @@ class DiffusionExecutor(ABC):
|
|||||||
)
|
)
|
||||||
|
|
||||||
def prepare_kv_for_forward(self, scheduler_output: DiffusionSchedulerOutput) -> KVConnectorOutput | None:
|
def prepare_kv_for_forward(self, scheduler_output: DiffusionSchedulerOutput) -> KVConnectorOutput | None:
|
||||||
if scheduler_output.kv_connector_metadata is None:
|
if scheduler_output.kv_prefetch_connector_metadata is not None:
|
||||||
|
current = replace(
|
||||||
|
scheduler_output,
|
||||||
|
kv_transfer_request_ids=scheduler_output.kv_transfer_request_ids
|
||||||
|
- scheduler_output.kv_prefetch_request_ids,
|
||||||
|
kv_prefetch_connector_metadata=None,
|
||||||
|
kv_prefetch_request_ids=set(),
|
||||||
|
)
|
||||||
|
# First complete the current request on every rank. Only then
|
||||||
|
# submit B, preventing Mooncake from coalescing its bytes with A.
|
||||||
|
self.prepare_kv_for_forward(current)
|
||||||
|
return self.prepare_kv_for_forward(
|
||||||
|
replace(
|
||||||
|
current,
|
||||||
|
kv_connector_metadata=scheduler_output.kv_prefetch_connector_metadata,
|
||||||
|
kv_transfer_request_ids=scheduler_output.kv_prefetch_request_ids,
|
||||||
|
kv_required_request_ids=set(),
|
||||||
|
kv_finished_request_ids=set(),
|
||||||
|
)
|
||||||
|
)
|
||||||
|
if (
|
||||||
|
scheduler_output.kv_connector_metadata is None
|
||||||
|
and not scheduler_output.kv_required_request_ids
|
||||||
|
and not scheduler_output.kv_poll_only
|
||||||
|
):
|
||||||
return
|
return
|
||||||
transfer_output = replace(
|
transfer_output = replace(
|
||||||
scheduler_output,
|
scheduler_output,
|
||||||
@@ -194,6 +218,14 @@ class DiffusionExecutor(ABC):
|
|||||||
if len(outputs) != self.od_config.num_gpus or any(output.invalid_block_ids for output in outputs):
|
if len(outputs) != self.od_config.num_gpus or any(output.invalid_block_ids for output in outputs):
|
||||||
# Missing ranks / invalid pages cannot establish safe ownership.
|
# Missing ranks / invalid pages cannot establish safe ownership.
|
||||||
raise RuntimeError("Diffusion KV receive failed on one or more ranks")
|
raise RuntimeError("Diffusion KV receive failed on one or more ranks")
|
||||||
|
if scheduler_output.kv_required_request_ids is not None:
|
||||||
|
# Prefetch workers retain completion events until retirement.
|
||||||
|
# Their cumulative snapshots can be intersected directly.
|
||||||
|
finished = set.intersection(*(set(output.finished_recving or ()) for output in outputs))
|
||||||
|
if not scheduler_output.kv_required_request_ids.issubset(finished):
|
||||||
|
raise RuntimeError("Required diffusion KV receive did not complete on every rank")
|
||||||
|
outputs[0].finished_recving = finished
|
||||||
|
return outputs[0]
|
||||||
completed = getattr(self, "_kv_receive_completed_ranks", {})
|
completed = getattr(self, "_kv_receive_completed_ranks", {})
|
||||||
for rank, output in enumerate(outputs):
|
for rank, output in enumerate(outputs):
|
||||||
for request_id in output.finished_recving or ():
|
for request_id in output.finished_recving or ():
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ from __future__ import annotations
|
|||||||
import time
|
import time
|
||||||
from abc import ABC, abstractmethod
|
from abc import ABC, abstractmethod
|
||||||
from collections import deque
|
from collections import deque
|
||||||
|
from collections.abc import Iterable
|
||||||
from dataclasses import fields
|
from dataclasses import fields
|
||||||
|
|
||||||
from vllm.config import VllmConfig
|
from vllm.config import VllmConfig
|
||||||
@@ -19,7 +20,9 @@ from vllm_omni.diffusion.diffusion_kv.config import DiffusionKVCacheMode
|
|||||||
from vllm_omni.diffusion.diffusion_kv.kv_connector import (
|
from vllm_omni.diffusion.diffusion_kv.kv_connector import (
|
||||||
KVTransferRegistrationError,
|
KVTransferRegistrationError,
|
||||||
commit_kv_load,
|
commit_kv_load,
|
||||||
|
native_prefetch_enabled,
|
||||||
prepare_kv_requests,
|
prepare_kv_requests,
|
||||||
|
validate_kv_transfer_boundaries,
|
||||||
)
|
)
|
||||||
from vllm_omni.diffusion.diffusion_kv.manager import DiffusionKVCacheManager
|
from vllm_omni.diffusion.diffusion_kv.manager import DiffusionKVCacheManager
|
||||||
from vllm_omni.diffusion.diffusion_kv.metadata import DiffusionKVMetadata
|
from vllm_omni.diffusion.diffusion_kv.metadata import DiffusionKVMetadata
|
||||||
@@ -71,6 +74,8 @@ class BaseScheduler(ABC):
|
|||||||
self._kv_request_generations: dict[str, int] = {}
|
self._kv_request_generations: dict[str, int] = {}
|
||||||
self._kv_draining_requests: dict[str, SchedulerRequestState] = {}
|
self._kv_draining_requests: dict[str, SchedulerRequestState] = {}
|
||||||
self._kv_received_request_ids: set[str] = set()
|
self._kv_received_request_ids: set[str] = set()
|
||||||
|
self._native_prefetch_enabled = False
|
||||||
|
self._native_prefetch_request_id: str | None = None
|
||||||
|
|
||||||
def initialize(
|
def initialize(
|
||||||
self,
|
self,
|
||||||
@@ -82,6 +87,8 @@ class BaseScheduler(ABC):
|
|||||||
kv_vllm_config: VllmConfig | None = None,
|
kv_vllm_config: VllmConfig | None = None,
|
||||||
) -> None:
|
) -> None:
|
||||||
self.od_config = od_config
|
self.od_config = od_config
|
||||||
|
self._native_prefetch_enabled = native_prefetch_enabled(od_config)
|
||||||
|
self._native_prefetch_request_id = None
|
||||||
self._request_states.clear()
|
self._request_states.clear()
|
||||||
self._step_id = 0
|
self._step_id = 0
|
||||||
self._waiting.clear()
|
self._waiting.clear()
|
||||||
@@ -227,6 +234,8 @@ class BaseScheduler(ABC):
|
|||||||
sequence.num_computed_tokens = request.num_computed_tokens
|
sequence.num_computed_tokens = request.num_computed_tokens
|
||||||
|
|
||||||
self._waiting.popleft()
|
self._waiting.popleft()
|
||||||
|
if request_id == self._native_prefetch_request_id:
|
||||||
|
self._native_prefetch_request_id = None
|
||||||
was_new_request = state.status == DiffusionRequestStatus.WAITING
|
was_new_request = state.status == DiffusionRequestStatus.WAITING
|
||||||
if not self._running:
|
if not self._running:
|
||||||
self._running_sampling_params_key = state.sampling_params_key
|
self._running_sampling_params_key = state.sampling_params_key
|
||||||
@@ -277,6 +286,23 @@ class BaseScheduler(ABC):
|
|||||||
scheduler_output.kv_connector_metadata = self._kv_connector.build_connector_meta(scheduler_output)
|
scheduler_output.kv_connector_metadata = self._kv_connector.build_connector_meta(scheduler_output)
|
||||||
scheduler_output.kv_transfer_request_ids = self._kv_transfer_request_ids
|
scheduler_output.kv_transfer_request_ids = self._kv_transfer_request_ids
|
||||||
self._kv_transfer_request_ids = set()
|
self._kv_transfer_request_ids = set()
|
||||||
|
if self._native_prefetch_enabled:
|
||||||
|
scheduler_output.kv_required_request_ids = self._loading_sequence_ids(self._running)
|
||||||
|
# Flush current-request metadata before staging B. Mooncake
|
||||||
|
# batches ready requests from the same producer into one write;
|
||||||
|
# combining A+B would make A's completion depend on B's bytes.
|
||||||
|
self._try_native_prefetch()
|
||||||
|
prefetch_finished_ids = self._kv_finished_request_ids - scheduler_output.kv_finished_request_ids
|
||||||
|
if self._kv_transfer_request_ids or prefetch_finished_ids:
|
||||||
|
scheduler_output.kv_prefetch_connector_metadata = self._kv_connector.build_connector_meta(
|
||||||
|
scheduler_output
|
||||||
|
)
|
||||||
|
scheduler_output.kv_prefetch_request_ids = self._kv_transfer_request_ids
|
||||||
|
scheduler_output.kv_transfer_request_ids |= self._kv_transfer_request_ids
|
||||||
|
self._kv_transfer_request_ids = set()
|
||||||
|
scheduler_output.finished_req_ids.update(self._finished_req_ids)
|
||||||
|
scheduler_output.kv_finished_request_ids.update(prefetch_finished_ids)
|
||||||
|
scheduler_output.num_waiting_reqs = len(self._waiting)
|
||||||
|
|
||||||
# update after schedule
|
# update after schedule
|
||||||
self._step_id += 1
|
self._step_id += 1
|
||||||
@@ -284,6 +310,92 @@ class BaseScheduler(ABC):
|
|||||||
self._kv_finished_request_ids.clear()
|
self._kv_finished_request_ids.clear()
|
||||||
return scheduler_output
|
return scheduler_output
|
||||||
|
|
||||||
|
def _try_native_prefetch(self) -> None:
|
||||||
|
if not self._running or not self._waiting or self._native_prefetch_request_id is not None:
|
||||||
|
return
|
||||||
|
request_id = self._waiting[0]
|
||||||
|
state = self._request_states.get(request_id)
|
||||||
|
if state is None or state.is_finished() or not state.diffusion_kv_requests:
|
||||||
|
return
|
||||||
|
manager = self._diffusion_kv_manager
|
||||||
|
if manager.has_request(request_id):
|
||||||
|
return
|
||||||
|
matched_tokens = []
|
||||||
|
for request in state.diffusion_kv_requests:
|
||||||
|
params = request.kv_transfer_params or {}
|
||||||
|
if (
|
||||||
|
not params.get("do_remote_prefill")
|
||||||
|
or not all(params.get(key) for key in ("remote_engine_id", "remote_bootstrap_addr", "transfer_id"))
|
||||||
|
or type(params.get("num_transfer_tokens")) is not int
|
||||||
|
or params["num_transfer_tokens"] <= 0
|
||||||
|
):
|
||||||
|
return
|
||||||
|
num_tokens, _ = self._kv_connector.get_num_new_matched_tokens(request, 0)
|
||||||
|
if num_tokens is None:
|
||||||
|
return
|
||||||
|
matched_tokens.append(num_tokens)
|
||||||
|
try:
|
||||||
|
validate_kv_transfer_boundaries(state.diffusion_kv_requests, matched_tokens)
|
||||||
|
except KVTransferRegistrationError as exc:
|
||||||
|
# Do not reserve B's pages for a transfer that cannot be registered.
|
||||||
|
# Normal admission will report the request-scoped error later.
|
||||||
|
logger.debug("Native KV prefetch skipped for %s: invalid boundary (%s)", request_id, exc)
|
||||||
|
return
|
||||||
|
# The manager atomically reserves all CFG rows, including capacity
|
||||||
|
# needed when B eventually executes. B remains in WAITING.
|
||||||
|
try:
|
||||||
|
allocation = manager.reserve_request(request_id, state.diffusion_kv_requests)
|
||||||
|
except Exception as exc:
|
||||||
|
# reserve_request rolls back atomically. Normal admission will
|
||||||
|
# surface B's error later; speculative allocation must not fail A.
|
||||||
|
logger.debug("Native KV prefetch skipped for %s: reservation failed (%s)", request_id, exc)
|
||||||
|
return
|
||||||
|
if allocation is None:
|
||||||
|
logger.debug("Native KV prefetch skipped for %s: insufficient blocks", request_id)
|
||||||
|
return
|
||||||
|
self._kv_request_generations[request_id] = allocation.allocation_generation
|
||||||
|
try:
|
||||||
|
transfer_ids = commit_kv_load(
|
||||||
|
self._kv_connector, manager.native_manager, state.diffusion_kv_requests, matched_tokens
|
||||||
|
)
|
||||||
|
except KVTransferRegistrationError as exc:
|
||||||
|
# Rollback may already notify the producer to release its pages.
|
||||||
|
# B cannot retry that transfer ticket, but A can still proceed.
|
||||||
|
self._finish_requests({request_id: DiffusionRequestStatus.FINISHED_ERROR}, {request_id: str(exc)})
|
||||||
|
return
|
||||||
|
self._kv_transfer_request_ids.update(transfer_ids)
|
||||||
|
if transfer_ids:
|
||||||
|
self._kv_loading_request_ids.add(request_id)
|
||||||
|
for sequence, request in zip(allocation.sequences, state.diffusion_kv_requests, strict=True):
|
||||||
|
sequence.num_computed_tokens = request.num_computed_tokens
|
||||||
|
self._native_prefetch_request_id = request_id
|
||||||
|
logger.debug("Native KV prefetch submitted for %s: %s", request_id, sorted(transfer_ids))
|
||||||
|
|
||||||
|
def _loading_sequence_ids(self, request_ids: Iterable[str]) -> set[str]:
|
||||||
|
sequence_ids: set[str] = set()
|
||||||
|
for request_id in request_ids:
|
||||||
|
if request_id not in self._kv_loading_request_ids:
|
||||||
|
continue
|
||||||
|
state = self._request_states.get(request_id) or self._kv_draining_requests.get(request_id)
|
||||||
|
if state is not None:
|
||||||
|
sequence_ids.update(request.request_id for request in state.diffusion_kv_requests)
|
||||||
|
return sequence_ids
|
||||||
|
|
||||||
|
def native_kv_poll_output(self, *, drain_request_ids: list[str] | None = None) -> DiffusionSchedulerOutput | None:
|
||||||
|
"""Poll after compute; cancellation/close may wait for selected loads."""
|
||||||
|
if not self._native_prefetch_enabled or not self._kv_loading_request_ids:
|
||||||
|
return None
|
||||||
|
return DiffusionSchedulerOutput(
|
||||||
|
step_id=self._step_id,
|
||||||
|
scheduled_new_reqs=[],
|
||||||
|
scheduled_cached_reqs=CachedRequestData.make_empty(),
|
||||||
|
finished_req_ids=set(),
|
||||||
|
num_running_reqs=len(self._running),
|
||||||
|
num_waiting_reqs=len(self._waiting),
|
||||||
|
kv_required_request_ids=self._loading_sequence_ids(drain_request_ids or []),
|
||||||
|
kv_poll_only=True,
|
||||||
|
)
|
||||||
|
|
||||||
@abstractmethod
|
@abstractmethod
|
||||||
def update_from_output(self, sched_output: DiffusionSchedulerOutput, output: RunnerOutput) -> set[str]:
|
def update_from_output(self, sched_output: DiffusionSchedulerOutput, output: RunnerOutput) -> set[str]:
|
||||||
pass
|
pass
|
||||||
@@ -426,6 +538,7 @@ class BaseScheduler(ABC):
|
|||||||
self._kv_draining_requests.clear()
|
self._kv_draining_requests.clear()
|
||||||
self._kv_received_request_ids.clear()
|
self._kv_received_request_ids.clear()
|
||||||
self._kv_request_generations.clear()
|
self._kv_request_generations.clear()
|
||||||
|
self._native_prefetch_request_id = None
|
||||||
|
|
||||||
def _shutdown_diffusion_kv(self) -> None:
|
def _shutdown_diffusion_kv(self) -> None:
|
||||||
"""Stop transfers before releasing their cache and request state."""
|
"""Stop transfers before releasing their cache and request state."""
|
||||||
@@ -471,6 +584,8 @@ class BaseScheduler(ABC):
|
|||||||
for request_id in finished_req_ids:
|
for request_id in finished_req_ids:
|
||||||
state = self._request_states[request_id]
|
state = self._request_states[request_id]
|
||||||
status = statuses[request_id]
|
status = statuses[request_id]
|
||||||
|
if request_id == self._native_prefetch_request_id:
|
||||||
|
self._native_prefetch_request_id = None
|
||||||
if request_id in self._kv_loading_request_ids:
|
if request_id in self._kv_loading_request_ids:
|
||||||
self._kv_draining_requests[request_id] = state
|
self._kv_draining_requests[request_id] = state
|
||||||
else:
|
else:
|
||||||
|
|||||||
@@ -256,9 +256,23 @@ class DiffusionSchedulerOutput:
|
|||||||
kv_prefetch_job: KVPrefetchJob | None = None
|
kv_prefetch_job: KVPrefetchJob | None = None
|
||||||
kv_connector_metadata: KVConnectorMetadata | None = None
|
kv_connector_metadata: KVConnectorMetadata | None = None
|
||||||
kv_transfer_request_ids: set[str] = field(default_factory=set)
|
kv_transfer_request_ids: set[str] = field(default_factory=set)
|
||||||
|
# None preserves the synchronous path; an empty set means submit/poll
|
||||||
|
# without waiting. Transfer IDs above are submitted only once.
|
||||||
|
kv_required_request_ids: set[str] | None = None
|
||||||
|
kv_poll_only: bool = False
|
||||||
|
# Keep background transfers out of the current request's Mooncake batch.
|
||||||
|
kv_prefetch_connector_metadata: KVConnectorMetadata | None = None
|
||||||
|
kv_prefetch_request_ids: set[str] = field(default_factory=set)
|
||||||
# Connector lifecycle uses per-sequence IDs, not public request IDs.
|
# Connector lifecycle uses per-sequence IDs, not public request IDs.
|
||||||
kv_finished_request_ids: set[str] = field(default_factory=set)
|
kv_finished_request_ids: set[str] = field(default_factory=set)
|
||||||
|
|
||||||
|
@property
|
||||||
|
def has_sync_kv_loads(self) -> bool:
|
||||||
|
# vLLM defers async submission until post_forward. The synchronous
|
||||||
|
# receive loop must start its loads in pre_forward before polling.
|
||||||
|
required = self.kv_required_request_ids
|
||||||
|
return bool(self.kv_transfer_request_ids if required is None else required)
|
||||||
|
|
||||||
@cached_property
|
@cached_property
|
||||||
def scheduled_request_ids(self) -> list[str]:
|
def scheduled_request_ids(self) -> list[str]:
|
||||||
"""
|
"""
|
||||||
@@ -275,4 +289,10 @@ class DiffusionSchedulerOutput:
|
|||||||
|
|
||||||
@property
|
@property
|
||||||
def is_empty(self) -> bool:
|
def is_empty(self) -> bool:
|
||||||
return self.num_scheduled_reqs == 0 and self.kv_connector_metadata is None
|
return (
|
||||||
|
self.num_scheduled_reqs == 0
|
||||||
|
and self.kv_connector_metadata is None
|
||||||
|
and self.kv_prefetch_connector_metadata is None
|
||||||
|
and not self.kv_required_request_ids
|
||||||
|
and not self.kv_poll_only
|
||||||
|
)
|
||||||
@@ -200,6 +200,9 @@ class DiffusionModelRunner(OmniConnectorModelRunnerMixin):
|
|||||||
else None
|
else None
|
||||||
)
|
)
|
||||||
self._kv_connector = None
|
self._kv_connector = None
|
||||||
|
from vllm_omni.diffusion.diffusion_kv.kv_connector import KVReceiveProgress, native_prefetch_enabled
|
||||||
|
|
||||||
|
self._kv_receive_progress = KVReceiveProgress() if native_prefetch_enabled(od_config) else None
|
||||||
|
|
||||||
# Prefetch covers TP / SP / CFG-Parallel / HSDP. Disabled when a CFG
|
# Prefetch covers TP / SP / CFG-Parallel / HSDP. Disabled when a CFG
|
||||||
# companion KV collector is set (that KV is not backgrounded).
|
# companion KV collector is set (that KV is not backgrounded).
|
||||||
@@ -487,6 +490,8 @@ class DiffusionModelRunner(OmniConnectorModelRunnerMixin):
|
|||||||
from vllm_omni.diffusion.diffusion_kv.kv_connector import wait_for_kv_load
|
from vllm_omni.diffusion.diffusion_kv.kv_connector import wait_for_kv_load
|
||||||
|
|
||||||
timeout = self.od_config.kv_transfer_config.kv_connector_extra_config.get("transfer_timeout", 60.0)
|
timeout = self.od_config.kv_transfer_config.kv_connector_extra_config.get("transfer_timeout", 60.0)
|
||||||
|
if self._kv_receive_progress is not None:
|
||||||
|
return self._kv_receive_progress.prepare(self._kv_connector, scheduler_output, timeout)
|
||||||
return wait_for_kv_load(self._kv_connector, scheduler_output, timeout)
|
return wait_for_kv_load(self._kv_connector, scheduler_output, timeout)
|
||||||
|
|
||||||
def install_diffusion_kv_metadata(self, metadata: DiffusionKVMetadata) -> bool:
|
def install_diffusion_kv_metadata(self, metadata: DiffusionKVMetadata) -> bool:
|
||||||
|
|||||||
@@ -801,6 +801,9 @@ class DiffusionWorker:
|
|||||||
Args:
|
Args:
|
||||||
level: Sleep level. Level 1 offloads weights, level 2 also saves buffers.
|
level: Sleep level. Level 1 offloads weights, level 2 also saves buffers.
|
||||||
"""
|
"""
|
||||||
|
progress = getattr(getattr(self, "model_runner", None), "_kv_receive_progress", None)
|
||||||
|
if progress is not None and progress.submitted:
|
||||||
|
raise RuntimeError("Cannot sleep with live native KV prefetch reservations; finish requests first")
|
||||||
# The config validator rejects sleep for the native paged path. Keep
|
# The config validator rejects sleep for the native paged path. Keep
|
||||||
# this worker-side guard precise as well: test doubles and legacy
|
# this worker-side guard precise as well: test doubles and legacy
|
||||||
# configs may expose arbitrary attributes through Mock/getattr.
|
# configs may expose arbitrary attributes through Mock/getattr.
|
||||||
|
|||||||
Reference in new issue
Block a user