[Core] hunyuan-image prefetch kv (#4448)
Signed-off-by: z00806815 <zhuhonghan@huawei.com> Signed-off-by: zhuhh97 <zhuhonghan@huawei.com> Signed-off-by: dph97 <nenbaying@163.com> Signed-off-by: zhh <zhh@users.noreply.github.com> Co-authored-by: Claude Opus 4.7 <noreply@anthropic.com> Co-authored-by: zhuhh97 <zhuhh97@users.noreply.github.com>
This commit is contained in:
19 files changed
+1214
-218
No files matched your search
@@ -0,0 +1,356 @@
|
||||
"""Background KV prefetch — CPU-testable mechanism.
|
||||
|
||||
Covers opt-in gating, the start_prefetch / consume_prefetched_kv round trip over a
|
||||
mock connector, dedup, per-call sender_info isolation, role classification, and
|
||||
miss/abort paths. GPU H2D and D2D paths need CUDA (integration tests).
|
||||
"""
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm_omni.diffusion.request import OmniDiffusionRequest
|
||||
from vllm_omni.distributed.omni_connectors.kv_transfer_manager import (
|
||||
OmniKVCacheConfig,
|
||||
OmniKVTransferManager,
|
||||
ReceiveRole,
|
||||
)
|
||||
from vllm_omni.inputs.data import OmniDiffusionSamplingParams
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
# start_prefetch() skips stubs without a sender endpoint, so submissions carry it.
|
||||
_SENDER_INFO = {"host": "127.0.0.1", "zmq_port": 50051}
|
||||
|
||||
|
||||
class MockConnector:
|
||||
def __init__(self):
|
||||
self.store = {}
|
||||
self.get_calls = []
|
||||
|
||||
def put(self, from_stage, to_stage, put_key, data):
|
||||
self.store[f"{from_stage}->{to_stage}:{put_key}"] = data
|
||||
return True, len(str(data)), None
|
||||
|
||||
def get(self, from_stage, to_stage, get_key, metadata=None):
|
||||
self.get_calls.append({"get_key": get_key, "metadata": metadata})
|
||||
key = f"{from_stage}->{to_stage}:{get_key}"
|
||||
if key in self.store:
|
||||
return self.store[key], len(str(self.store[key]))
|
||||
return None
|
||||
|
||||
|
||||
def _make_sender_receiver(*, async_prefetch=True):
|
||||
"""Build sender+receiver managers sharing one mock connector, seed one KV."""
|
||||
connector = MockConnector()
|
||||
|
||||
sender = OmniKVTransferManager(
|
||||
OmniKVCacheConfig(
|
||||
connector_config={"type": "mock"}, from_stage="sender", to_stage="receiver", need_send_cache=True
|
||||
)
|
||||
)
|
||||
sender._connector = connector
|
||||
|
||||
receiver = OmniKVTransferManager(
|
||||
OmniKVCacheConfig(
|
||||
connector_config={"type": "mock"},
|
||||
from_stage="sender",
|
||||
stage_id="receiver",
|
||||
need_recv_cache=True,
|
||||
recv_timeout=1.0,
|
||||
),
|
||||
async_prefetch=async_prefetch,
|
||||
)
|
||||
receiver._connector = connector
|
||||
return sender, receiver, connector
|
||||
|
||||
|
||||
def _seed_payload(sender, req_id, *, num_layers=2, block_size=4, num_heads=4, head_dim=8):
|
||||
kv_caches = [torch.randn(2, 5, block_size, num_heads, head_dim) for _ in range(num_layers)]
|
||||
sender.handle_finished_requests_kv_transfer(
|
||||
{req_id: {"block_ids": [0, 1], "seq_len": 10}}, kv_caches, block_size, "float32"
|
||||
)
|
||||
|
||||
|
||||
def _req(rid):
|
||||
"""A minimal request carrying just ``request_id`` (consume_prefetched_kv only needs that)."""
|
||||
return SimpleNamespace(request_id=rid)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Opt-in gating
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def test_async_prefetch_defaults_false():
|
||||
mgr = OmniKVTransferManager(OmniKVCacheConfig())
|
||||
assert mgr._async_prefetch is False
|
||||
|
||||
|
||||
def test_from_od_config_enables_prefetch_flag():
|
||||
od = SimpleNamespace(omni_kv_config={"enable_kv_async_prefetch": True})
|
||||
assert OmniKVTransferManager.from_od_config(od)._async_prefetch is True
|
||||
od2 = SimpleNamespace(omni_kv_config=None)
|
||||
assert OmniKVTransferManager.from_od_config(od2)._async_prefetch is False
|
||||
od3 = SimpleNamespace(omni_kv_config={"enable_kv_async_prefetch": False})
|
||||
assert OmniKVTransferManager.from_od_config(od3)._async_prefetch is False
|
||||
|
||||
|
||||
def test_prefetch_preserved_with_connector_config():
|
||||
# The flag must survive merging with a populated connector config.
|
||||
od = SimpleNamespace(
|
||||
omni_kv_config={
|
||||
"connector_config": {"type": "mock"},
|
||||
"need_recv_cache": True,
|
||||
"enable_kv_async_prefetch": True,
|
||||
},
|
||||
)
|
||||
assert OmniKVTransferManager.from_od_config(od)._async_prefetch is True
|
||||
|
||||
|
||||
def test_prefetch_disabled_with_cfg_companion_collector():
|
||||
# A CFG companion collector forces the synchronous path (see _resolve_async_prefetch).
|
||||
od = SimpleNamespace(
|
||||
omni_kv_config={"enable_kv_async_prefetch": True},
|
||||
cfg_kv_collect_func=lambda *a, **k: {},
|
||||
)
|
||||
assert OmniKVTransferManager.from_od_config(od)._async_prefetch is False
|
||||
|
||||
|
||||
def test_from_vllm_config_ar_path_is_always_sync():
|
||||
# The AR entry point never enables prefetch.
|
||||
model_config = SimpleNamespace(omni_kv_config=None)
|
||||
vllm_config = SimpleNamespace(kv_transfer_config=None)
|
||||
assert OmniKVTransferManager.from_vllm_config(vllm_config, model_config)._async_prefetch is False
|
||||
|
||||
|
||||
def test_start_prefetch_noop_when_disabled():
|
||||
_, receiver, _ = _make_sender_receiver(async_prefetch=False)
|
||||
receiver.start_prefetch({"request_id": "r1", "kv_sender_info": None})
|
||||
assert receiver._prefetch_futures == {}
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Round trip
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def test_prefetch_round_trip_returns_data():
|
||||
sender, receiver, _ = _make_sender_receiver()
|
||||
_seed_payload(sender, "rid-1")
|
||||
|
||||
receiver.start_prefetch({"request_id": "rid-1", "kv_sender_info": _SENDER_INFO})
|
||||
assert "rid-1" in receiver._prefetch_futures
|
||||
|
||||
data, size = receiver.consume_prefetched_kv(_req("rid-1"))
|
||||
assert data is not None
|
||||
assert "layer_blocks" in data
|
||||
assert data["metadata"]["seq_len"] == 10
|
||||
assert size > 0
|
||||
# Future is consumed (popped) on retrieval.
|
||||
assert "rid-1" not in receiver._prefetch_futures
|
||||
|
||||
|
||||
def test_consume_prefetched_kv_miss_without_prefetch_returns_none():
|
||||
_, receiver, _ = _make_sender_receiver(async_prefetch=False)
|
||||
assert receiver.consume_prefetched_kv(_req("never-prefetched")) == (None, 0)
|
||||
|
||||
|
||||
def test_start_prefetch_dedups_same_request():
|
||||
sender, receiver, _ = _make_sender_receiver()
|
||||
_seed_payload(sender, "rid-dup")
|
||||
receiver.start_prefetch({"request_id": "rid-dup", "kv_sender_info": _SENDER_INFO})
|
||||
fut1 = receiver._prefetch_futures["rid-dup"]
|
||||
receiver.start_prefetch({"request_id": "rid-dup", "kv_sender_info": _SENDER_INFO})
|
||||
assert receiver._prefetch_futures["rid-dup"] is fut1
|
||||
|
||||
|
||||
def test_start_prefetch_skips_without_sender_info():
|
||||
# No endpoint => no submission; the sync path handles it.
|
||||
sender, receiver, _ = _make_sender_receiver()
|
||||
_seed_payload(sender, "rid-nosi")
|
||||
receiver.start_prefetch({"request_id": "rid-nosi", "kv_sender_info": None})
|
||||
assert receiver._prefetch_futures == {}
|
||||
|
||||
|
||||
def test_consume_prefetched_kv_sweeps_stale_entries():
|
||||
# Consuming a slot drops every other tracked prefetch.
|
||||
sender, receiver, _ = _make_sender_receiver()
|
||||
_seed_payload(sender, "rid-stale")
|
||||
receiver.start_prefetch({"request_id": "rid-stale", "kv_sender_info": _SENDER_INFO})
|
||||
assert "rid-stale" in receiver._prefetch_futures
|
||||
# Miss on the current request still sweeps stale entries.
|
||||
receiver._async_prefetch = False
|
||||
assert receiver.consume_prefetched_kv(_req("rid-current")) == (None, 0)
|
||||
assert receiver._prefetch_futures == {}
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# consume_prefetched_kv: miss returns None, stale sweep
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def test_start_prefetch_sweeps_orphan():
|
||||
# An orphan prefetch (e.g. aborted) is dropped when the next prefetch starts.
|
||||
sender, receiver, _ = _make_sender_receiver()
|
||||
_seed_payload(sender, "rid-orphan")
|
||||
_seed_payload(sender, "rid-next")
|
||||
receiver.start_prefetch({"request_id": "rid-orphan", "kv_sender_info": _SENDER_INFO})
|
||||
receiver.consume_prefetched_kv(_req("rid-orphan")) # complete the future
|
||||
receiver.start_prefetch({"request_id": "rid-orphan", "kv_sender_info": _SENDER_INFO}) # re-add as leftover
|
||||
receiver.start_prefetch({"request_id": "rid-next", "kv_sender_info": _SENDER_INFO})
|
||||
assert "rid-orphan" not in receiver._prefetch_futures
|
||||
assert "rid-next" in receiver._prefetch_futures
|
||||
|
||||
|
||||
def test_shutdown_prefetch_clears_state():
|
||||
sender, receiver, _ = _make_sender_receiver()
|
||||
_seed_payload(sender, "rid-sd")
|
||||
receiver.start_prefetch({"request_id": "rid-sd", "kv_sender_info": _SENDER_INFO})
|
||||
receiver.shutdown_prefetch()
|
||||
assert receiver._prefetch_futures == {}
|
||||
assert receiver._prefetch_executor is None
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Correctness: per-call sender_info must not touch instance state
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def test_receive_with_sender_info_does_not_mutate_instance_state():
|
||||
sender, receiver, _ = _make_sender_receiver()
|
||||
_seed_payload(sender, "rid-si")
|
||||
assert receiver._sender_base_host is None
|
||||
assert receiver._sender_base_zmq_port is None
|
||||
|
||||
receiver.receive_kv_cache_for_request(
|
||||
"rid-si",
|
||||
target_device=None,
|
||||
sender_info={"host": "1.2.3.4", "zmq_port": 50051},
|
||||
)
|
||||
# The per-call endpoint must not leak into instance fields.
|
||||
assert receiver._sender_base_host is None
|
||||
assert receiver._sender_base_zmq_port is None
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Receive-role classification (recv_role) and the follower prefetch guard
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
import vllm_omni.diffusion.distributed.parallel_state as ps # noqa: E402
|
||||
|
||||
|
||||
def _patch_topo(monkeypatch, *, world_size, world_rank=0, cfg_size=1, cfg_rank=0, sp_size=1, sp_rank=0):
|
||||
monkeypatch.setattr(ps, "get_world_group", lambda: SimpleNamespace(world_size=world_size, rank_in_group=world_rank))
|
||||
monkeypatch.setattr(ps, "get_classifier_free_guidance_world_size", lambda: cfg_size)
|
||||
monkeypatch.setattr(ps, "get_classifier_free_guidance_rank", lambda: cfg_rank)
|
||||
monkeypatch.setattr(ps, "get_cfg_group", lambda: None)
|
||||
monkeypatch.setattr(ps, "get_sequence_parallel_world_size", lambda: sp_size)
|
||||
monkeypatch.setattr(ps, "get_sequence_parallel_rank", lambda: sp_rank)
|
||||
monkeypatch.setattr(ps, "get_sp_group", lambda: None)
|
||||
|
||||
|
||||
def _set_tp(mgr, src, tgt):
|
||||
mgr._tp_topo = SimpleNamespace(source_tp_size=src, target_tp_size=tgt, local_rank=0)
|
||||
|
||||
|
||||
def _role(mgr):
|
||||
"""Recompute the receive role from the (patched) parallel state, bypassing the cache."""
|
||||
mgr._topo_config = None
|
||||
return mgr.topo_config.role
|
||||
|
||||
|
||||
def test_recv_role_world1_is_local(monkeypatch):
|
||||
mgr = OmniKVTransferManager(OmniKVCacheConfig())
|
||||
_set_tp(mgr, 1, 1)
|
||||
_patch_topo(monkeypatch, world_size=1)
|
||||
assert _role(mgr) is ReceiveRole.LOCAL
|
||||
|
||||
|
||||
def test_recv_role_pure_tp_is_local(monkeypatch):
|
||||
mgr = OmniKVTransferManager(OmniKVCacheConfig())
|
||||
_set_tp(mgr, 2, 2)
|
||||
_patch_topo(monkeypatch, world_size=2)
|
||||
assert _role(mgr) is ReceiveRole.LOCAL
|
||||
|
||||
|
||||
def test_recv_role_cfg_leader_and_follower(monkeypatch):
|
||||
mgr = OmniKVTransferManager(OmniKVCacheConfig())
|
||||
_set_tp(mgr, 2, 2)
|
||||
_patch_topo(monkeypatch, world_size=4, cfg_size=2, cfg_rank=0)
|
||||
assert _role(mgr) is ReceiveRole.LEADER
|
||||
_patch_topo(monkeypatch, world_size=4, cfg_size=2, cfg_rank=1)
|
||||
assert _role(mgr) is ReceiveRole.FOLLOWER
|
||||
|
||||
|
||||
def test_recv_role_sp_leader_and_follower(monkeypatch):
|
||||
mgr = OmniKVTransferManager(OmniKVCacheConfig())
|
||||
_set_tp(mgr, 2, 2)
|
||||
_patch_topo(monkeypatch, world_size=4, sp_size=2, sp_rank=0)
|
||||
assert _role(mgr) is ReceiveRole.LEADER
|
||||
_patch_topo(monkeypatch, world_size=4, sp_size=2, sp_rank=1)
|
||||
assert _role(mgr) is ReceiveRole.FOLLOWER
|
||||
|
||||
|
||||
def test_recv_role_tp_and_sp_leader_per_tp_rank(monkeypatch):
|
||||
# TP=2 + SP=2: sp_rank 0 is leader (pulls), sp_rank 1 is follower (gets broadcast).
|
||||
mgr = OmniKVTransferManager(OmniKVCacheConfig())
|
||||
_set_tp(mgr, 2, 2)
|
||||
_patch_topo(monkeypatch, world_size=4, sp_size=2, sp_rank=0)
|
||||
assert _role(mgr) is ReceiveRole.LEADER
|
||||
_patch_topo(monkeypatch, world_size=4, sp_size=2, sp_rank=1)
|
||||
assert _role(mgr) is ReceiveRole.FOLLOWER
|
||||
|
||||
|
||||
def test_recv_role_hsdp_world_broadcast(monkeypatch):
|
||||
# TP inactive, world > 1 (HSDP): rank-0 leader, others follower.
|
||||
mgr = OmniKVTransferManager(OmniKVCacheConfig())
|
||||
_set_tp(mgr, 1, 1)
|
||||
_patch_topo(monkeypatch, world_size=2, world_rank=0)
|
||||
assert _role(mgr) is ReceiveRole.LEADER
|
||||
_patch_topo(monkeypatch, world_size=2, world_rank=1)
|
||||
assert _role(mgr) is ReceiveRole.FOLLOWER
|
||||
|
||||
|
||||
def test_recv_role_uninitialized_defaults_local(monkeypatch):
|
||||
mgr = OmniKVTransferManager(OmniKVCacheConfig())
|
||||
|
||||
def _boom():
|
||||
raise AssertionError("world group is not initialized")
|
||||
|
||||
monkeypatch.setattr(ps, "get_world_group", _boom)
|
||||
assert _role(mgr) is ReceiveRole.LOCAL
|
||||
|
||||
|
||||
def test_start_prefetch_skips_follower(monkeypatch):
|
||||
# A follower never pulls, so start_prefetch must not submit.
|
||||
sender, receiver, _ = _make_sender_receiver()
|
||||
_seed_payload(sender, "rid-fol")
|
||||
_set_tp(receiver, 2, 2)
|
||||
_patch_topo(monkeypatch, world_size=4, cfg_size=2, cfg_rank=1)
|
||||
receiver.start_prefetch({"request_id": "rid-fol", "kv_sender_info": _SENDER_INFO})
|
||||
assert receiver._prefetch_futures == {}
|
||||
|
||||
|
||||
def test_start_prefetch_submits_for_owner(monkeypatch):
|
||||
sender, receiver, _ = _make_sender_receiver()
|
||||
_seed_payload(sender, "rid-own")
|
||||
_set_tp(receiver, 2, 2)
|
||||
_patch_topo(monkeypatch, world_size=4, cfg_size=2, cfg_rank=0)
|
||||
receiver.start_prefetch({"request_id": "rid-own", "kv_sender_info": _SENDER_INFO})
|
||||
assert "rid-own" in receiver._prefetch_futures
|
||||
|
||||
|
||||
def test_consume_then_apply_attaches_payload():
|
||||
sender, receiver, _ = _make_sender_receiver()
|
||||
_seed_payload(sender, "rid-apply")
|
||||
receiver.start_prefetch({"request_id": "rid-apply", "kv_sender_info": _SENDER_INFO})
|
||||
data, _ = receiver.consume_prefetched_kv(_req("rid-apply"))
|
||||
assert data is not None
|
||||
|
||||
req = OmniDiffusionRequest(prompts=["p"], sampling_params=OmniDiffusionSamplingParams(), request_id="rid-apply")
|
||||
# Mirror consume_and_distribute_kv_cache's LOCAL apply path (CPU: record_stream no-op).
|
||||
receiver._record_stream_for_prefetched(data)
|
||||
receiver.apply_kv_cache_to_request(req, data)
|
||||
assert req.past_key_values is not None
|
||||
@@ -1,7 +1,6 @@
|
||||
import json
|
||||
import struct
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
@@ -206,17 +205,6 @@ def test_update_sender_info_uses_configured_source_stage():
|
||||
assert manager.config.connector_config["sender_zmq_port"] == 50152
|
||||
|
||||
|
||||
def test_clone_received_payload_tensors_breaks_buffer_alias():
|
||||
payload, key_tensor = _make_serialized_payload()
|
||||
raw = np.frombuffer(bytearray(payload), dtype=np.uint8)
|
||||
data = KVCacheTransferData.from_bytes(memoryview(raw))
|
||||
|
||||
OmniKVTransferManager._clone_received_payload_tensors(data)
|
||||
raw[:] = 0
|
||||
|
||||
assert torch.equal(data["layer_blocks"]["key_cache"][0], key_tensor)
|
||||
|
||||
|
||||
def test_receive_kv_cache_uses_exponential_backoff(monkeypatch):
|
||||
config = OmniKVCacheConfig(
|
||||
connector_config={"type": "mock"},
|
||||
@@ -526,6 +514,57 @@ def test_integration_flow(common_constants):
|
||||
assert req.kv_metadata["seq_len"] == 10
|
||||
|
||||
|
||||
def test_deferred_release_isolates_data_from_pool_buffer():
|
||||
payload, key_tensor = _make_serialized_payload()
|
||||
|
||||
# ManagedBuffer-like object whose release() zeros storage (simulates reuse).
|
||||
raw_array = bytearray(payload)
|
||||
buf_tensor = torch.frombuffer(raw_array, dtype=torch.uint8)
|
||||
|
||||
class _CorruptingManagedBuffer:
|
||||
def __init__(self, storage, tensor):
|
||||
self._storage = storage
|
||||
self._tensor = tensor
|
||||
|
||||
@property
|
||||
def tensor(self):
|
||||
return self._tensor
|
||||
|
||||
def release(self):
|
||||
for i in range(len(self._storage)):
|
||||
self._storage[i] = 0
|
||||
|
||||
managed_buf = _CorruptingManagedBuffer(raw_array, buf_tensor)
|
||||
|
||||
class _ManagedBufConnector:
|
||||
def __init__(self, buf, from_stage, to_stage, req_id):
|
||||
self._buf = buf
|
||||
self._key = f"{from_stage}->{to_stage}:omni_{from_stage}_to_{to_stage}_kv_cache_{req_id}"
|
||||
|
||||
def get(self, from_stage, to_stage, get_key, metadata=None):
|
||||
if f"{from_stage}->{to_stage}:{get_key}" == self._key:
|
||||
return self._buf, len(self._buf._storage)
|
||||
return None
|
||||
|
||||
config = OmniKVCacheConfig(
|
||||
connector_config={"type": "mock"},
|
||||
from_stage="sender",
|
||||
stage_id="receiver",
|
||||
need_recv_cache=True,
|
||||
recv_timeout=1.0,
|
||||
)
|
||||
manager = OmniKVTransferManager(config)
|
||||
req_id = "req-payload"
|
||||
manager._connector = _ManagedBufConnector(managed_buf, "sender", "receiver", req_id)
|
||||
|
||||
data, size = manager.receive_kv_cache_for_request(req_id, target_device=torch.device("cpu"))
|
||||
|
||||
assert data is not None
|
||||
assert size > 0
|
||||
# Returned data must survive the pool buffer being zeroed by release().
|
||||
assert torch.equal(data["layer_blocks"]["key_cache"][0], key_tensor)
|
||||
|
||||
|
||||
def test_manager_extraction_no_connector(kv_config, common_constants):
|
||||
"""Test extraction when connector is unavailable (should still return IDs)."""
|
||||
block_size = common_constants["block_size"]
|
||||
|
||||
@@ -586,8 +586,17 @@ class TestDistributedReceive:
|
||||
return_value=0,
|
||||
),
|
||||
patch("vllm_omni.diffusion.distributed.parallel_state.get_cfg_group", return_value=cfg_group),
|
||||
patch(
|
||||
"vllm_omni.diffusion.distributed.parallel_state.get_sequence_parallel_world_size",
|
||||
return_value=1,
|
||||
),
|
||||
patch(
|
||||
"vllm_omni.diffusion.distributed.parallel_state.get_sequence_parallel_rank",
|
||||
return_value=0,
|
||||
),
|
||||
patch("vllm_omni.diffusion.distributed.parallel_state.get_sp_group", return_value=None),
|
||||
):
|
||||
assert mgr.receive_multi_kv_cache_distributed(req) is True
|
||||
assert mgr.receive_multi_kv_cache_distributed(req, target_device=torch.device("cpu")) is True
|
||||
|
||||
mgr.receive_multi_kv_cache.assert_called_once()
|
||||
assert mgr.receive_multi_kv_cache.call_args.args[2] == torch.device("cpu")
|
||||
@@ -642,6 +651,15 @@ class TestDistributedReceive:
|
||||
return_value=1,
|
||||
),
|
||||
patch("vllm_omni.diffusion.distributed.parallel_state.get_cfg_group", return_value=cfg_group),
|
||||
patch(
|
||||
"vllm_omni.diffusion.distributed.parallel_state.get_sequence_parallel_world_size",
|
||||
return_value=1,
|
||||
),
|
||||
patch(
|
||||
"vllm_omni.diffusion.distributed.parallel_state.get_sequence_parallel_rank",
|
||||
return_value=0,
|
||||
),
|
||||
patch("vllm_omni.diffusion.distributed.parallel_state.get_sp_group", return_value=None),
|
||||
):
|
||||
assert mgr.receive_multi_kv_cache_distributed(req) is True
|
||||
|
||||
@@ -666,7 +684,27 @@ class TestDistributedReceive:
|
||||
world_group = _MockBroadcastGroup(world_size=2, rank_in_group=1)
|
||||
mgr.receive_multi_kv_cache = MagicMock(return_value=True)
|
||||
|
||||
with patch("vllm_omni.diffusion.distributed.parallel_state.get_world_group", return_value=world_group):
|
||||
with (
|
||||
patch("vllm_omni.diffusion.distributed.parallel_state.get_world_group", return_value=world_group),
|
||||
patch(
|
||||
"vllm_omni.diffusion.distributed.parallel_state.get_classifier_free_guidance_world_size",
|
||||
return_value=1,
|
||||
),
|
||||
patch(
|
||||
"vllm_omni.diffusion.distributed.parallel_state.get_classifier_free_guidance_rank",
|
||||
return_value=0,
|
||||
),
|
||||
patch("vllm_omni.diffusion.distributed.parallel_state.get_cfg_group", return_value=None),
|
||||
patch(
|
||||
"vllm_omni.diffusion.distributed.parallel_state.get_sequence_parallel_world_size",
|
||||
return_value=1,
|
||||
),
|
||||
patch(
|
||||
"vllm_omni.diffusion.distributed.parallel_state.get_sequence_parallel_rank",
|
||||
return_value=0,
|
||||
),
|
||||
patch("vllm_omni.diffusion.distributed.parallel_state.get_sp_group", return_value=None),
|
||||
):
|
||||
assert mgr.receive_multi_kv_cache_distributed(req, target_device=torch.device("cpu")) is True
|
||||
|
||||
mgr.receive_multi_kv_cache.assert_called_once_with(req, None, torch.device("cpu"))
|
||||
@@ -722,7 +760,7 @@ class TestDistributedReceive:
|
||||
),
|
||||
patch("vllm_omni.diffusion.distributed.parallel_state.get_sp_group", return_value=sp_group),
|
||||
):
|
||||
assert mgr.receive_multi_kv_cache_distributed(req) is True
|
||||
assert mgr.receive_multi_kv_cache_distributed(req, target_device=torch.device("cpu")) is True
|
||||
|
||||
# Verify behavior based on role
|
||||
if is_owner:
|
||||
@@ -817,7 +855,7 @@ class TestDistributedReceive:
|
||||
),
|
||||
patch("vllm_omni.diffusion.distributed.parallel_state.get_sp_group", return_value=sp_group),
|
||||
):
|
||||
assert mgr.receive_multi_kv_cache_distributed(req) is True
|
||||
assert mgr.receive_multi_kv_cache_distributed(req, target_device=torch.device("cpu")) is True
|
||||
|
||||
# Verify behavior based on role
|
||||
if role == "owner":
|
||||
|
||||
Reference in new issue
Block a user