[Feature] Transfer AR-to-diffusion payloads over NIXL (#6264)
Signed-off-by: yuanwu <yuan.wu@intel.com> Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
This commit is contained in:
1 parent
8953d082c0
commit
cb5f508212
60 files changed
+4397
-641
No files matched your search
@@ -205,6 +205,25 @@ def test_start_prefetch_sweeps_orphan():
|
||||
assert "rid-next" in receiver._prefetch_futures
|
||||
|
||||
|
||||
def test_manager_close_stops_prefetch_before_connector_and_is_idempotent():
|
||||
receiver = object.__new__(OmniKVTransferManager)
|
||||
events = []
|
||||
receiver.shutdown_prefetch = lambda: events.append("prefetch")
|
||||
receiver._connector = SimpleNamespace(close=lambda: events.append("connector"))
|
||||
receiver.close()
|
||||
receiver.close()
|
||||
assert events == ["prefetch", "connector", "prefetch"]
|
||||
assert receiver.connector is None
|
||||
|
||||
|
||||
def test_manager_close_does_not_initialize_unused_connector():
|
||||
receiver = object.__new__(OmniKVTransferManager)
|
||||
receiver.shutdown_prefetch = lambda: None
|
||||
receiver._connector = None
|
||||
receiver.close()
|
||||
assert receiver.connector is None
|
||||
|
||||
|
||||
def test_shutdown_prefetch_clears_state():
|
||||
sender, receiver, _ = _make_sender_receiver()
|
||||
_seed_payload(sender, "rid-sd")
|
||||
|
||||
@@ -132,6 +132,7 @@ def test_manager_receiver_does_not_bind_shared_producer_port(producer, need_recv
|
||||
"role": "receiver",
|
||||
"host": "127.0.0.1",
|
||||
"zmq_port": PORT,
|
||||
"metadata_query_timeout_ms": 5000,
|
||||
"backends": ["UCX"],
|
||||
},
|
||||
from_stage="0",
|
||||
@@ -280,14 +281,15 @@ def test_abandoned_claim_survives_close_and_late_completion(producer, consumer,
|
||||
consumer._notify_transfer_done("abandoned", claimed)
|
||||
assert not producer._pending
|
||||
assert not producer._agent.registered
|
||||
assert producer._listener_thread.is_alive()
|
||||
producer.close()
|
||||
producer._close_thread.join(timeout=2.0)
|
||||
assert not producer._close_thread.is_alive()
|
||||
assert producer._listener_thread is None
|
||||
assert not producer._agent.registered
|
||||
assert producer._closed
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def copying_native_agent(consumer, monkeypatch):
|
||||
def ownership_copying_agent(consumer, monkeypatch):
|
||||
"""Strict fake descriptors and actual CPU copies; not native NIXL evidence."""
|
||||
calls = []
|
||||
|
||||
@@ -320,7 +322,7 @@ def copying_native_agent(consumer, monkeypatch):
|
||||
|
||||
@pytest.mark.parametrize("case", ["empty", "all_empty", "mixed", "scalar", "structured_empty", "structured_mixed"])
|
||||
@pytest.mark.usefixtures("reliable_claim_queries")
|
||||
def test_zero_byte_leaves_roundtrip_without_native_descriptors(producer, consumer, copying_native_agent, case):
|
||||
def test_zero_byte_leaves_complete_source_claim(producer, consumer, ownership_copying_agent, case):
|
||||
empty = torch.empty((2, 0, 3), dtype=torch.float64)
|
||||
other = torch.empty((0,), dtype=torch.int64)
|
||||
scalar = torch.tensor(7)
|
||||
@@ -345,7 +347,7 @@ def test_zero_byte_leaves_roundtrip_without_native_descriptors(producer, consume
|
||||
torch.testing.assert_close(actual, payload, rtol=0, atol=0)
|
||||
assert size == received_size
|
||||
if case in ("empty", "all_empty"):
|
||||
assert not copying_native_agent
|
||||
assert not ownership_copying_agent
|
||||
assert not producer._pending and not producer._agent.registered
|
||||
assert not consumer._agent.registered
|
||||
|
||||
@@ -354,7 +356,7 @@ def test_zero_byte_leaves_roundtrip_without_native_descriptors(producer, consume
|
||||
@pytest.mark.parametrize("outcome", ["done", "error", "timeout", "unknown"])
|
||||
@pytest.mark.usefixtures("reliable_claim_queries")
|
||||
def test_read_ownership_through_terminal_and_deferred_paths(
|
||||
producer, consumer, copying_native_agent, monkeypatch, direct, outcome
|
||||
producer, consumer, ownership_copying_agent, monkeypatch, direct, outcome
|
||||
):
|
||||
import threading
|
||||
|
||||
@@ -411,6 +413,7 @@ class _FakeNixlAgent:
|
||||
self.registered = []
|
||||
|
||||
def get_reg_descs(self, regions, memory_type):
|
||||
assert regions and all(region[0] > 0 and region[1] > 0 for region in regions)
|
||||
return ("reg", tuple(regions), memory_type)
|
||||
|
||||
def register_memory(self, descs, backends=None):
|
||||
@@ -548,21 +551,150 @@ def test_source_endpoint_metadata_overrides_configured_sender(consumer, monkeypa
|
||||
assert queried == [("req-tp4-rank3", "10.0.0.3", PORT + 3 * 16)]
|
||||
|
||||
|
||||
def test_unknown_key_is_queried_once(producer, consumer, monkeypatch):
|
||||
@pytest.mark.parametrize("force_timeout", [False, True], ids=["real-query", "recv-timeout"])
|
||||
def test_unknown_key_is_queried_once(producer, consumer, monkeypatch, force_timeout):
|
||||
socket = consumer._get_req_socket(f"tcp://127.0.0.1:{PORT}")
|
||||
send_count = 0
|
||||
original_send = socket.send
|
||||
|
||||
def count_send(message):
|
||||
nonlocal send_count
|
||||
assert socket.getsockopt(zmq.RCVTIMEO) == 10
|
||||
assert socket.getsockopt(zmq.SNDTIMEO) == 10
|
||||
send_count += 1
|
||||
return original_send(message)
|
||||
|
||||
monkeypatch.setattr(socket, "send", count_send)
|
||||
if force_timeout:
|
||||
|
||||
def timeout_recv():
|
||||
raise zmq.Again()
|
||||
|
||||
monkeypatch.setattr(socket, "recv", timeout_recv)
|
||||
|
||||
assert consumer._resolve_metadata("never-published", None) is None
|
||||
assert send_count == 1
|
||||
assert socket.getsockopt(zmq.RCVTIMEO) == 10
|
||||
if force_timeout:
|
||||
assert socket.closed
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("reliable_claim_queries")
|
||||
def test_deadline_get_retains_active_dma_then_releases(producer, consumer, ownership_copying_agent, monkeypatch):
|
||||
producer.put("0", "1", "deadline-read", torch.ones(2))
|
||||
consumer._stop_event.set()
|
||||
consumer._transfer_wakeup.set()
|
||||
consumer._transfer_thread.join(timeout=2)
|
||||
state = ["PROC"]
|
||||
monkeypatch.setattr(consumer._agent, "check_xfer_state", lambda handle: state[0])
|
||||
started = time.monotonic()
|
||||
assert consumer.get_with_deadline("0", "1", "deadline-read", deadline=started + 0.03) is None
|
||||
assert time.monotonic() - started < 0.5
|
||||
assert producer._pending["deadline-read"].claims
|
||||
assert producer._agent.registered
|
||||
assert consumer._agent.registered
|
||||
assert consumer._deferred_transfers
|
||||
state[0] = "DONE"
|
||||
consumer._reap_deferred_transfers()
|
||||
assert not producer._pending
|
||||
assert not producer._agent.registered
|
||||
assert not consumer._agent.registered
|
||||
assert not consumer._deferred_transfers
|
||||
|
||||
|
||||
def test_receive_deadline_overrides_dma_timeout(consumer, monkeypatch):
|
||||
monkeypatch.setattr(consumer._agent, "check_xfer_state", lambda handle: "PROC", raising=False)
|
||||
consumer._transfer_timeout_s = 300
|
||||
consumer._req_local.deadline = time.monotonic() + 0.02
|
||||
started = time.monotonic()
|
||||
try:
|
||||
with pytest.raises(TimeoutError):
|
||||
consumer._wait_for_transfer(1, "bounded")
|
||||
assert time.monotonic() - started < 0.5
|
||||
finally:
|
||||
consumer._req_local.deadline = None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("closing", [False, True])
|
||||
def test_abandoned_metadata_query_recovers_same_claim_without_read(producer, consumer, monkeypatch, closing):
|
||||
from vllm_omni.distributed.omni_connectors.connectors.nixl_connector import _GET_META_MSG
|
||||
|
||||
consumer._stop_event.set()
|
||||
consumer._transfer_wakeup.set()
|
||||
consumer._transfer_thread.join(timeout=2)
|
||||
producer.put("0", "1", "abandoned-query", torch.ones(1))
|
||||
messages = []
|
||||
|
||||
def send(message):
|
||||
messages.append(message)
|
||||
|
||||
def receive():
|
||||
reply = producer._handle_handshake_message(messages[-1])
|
||||
if len(messages) == 1:
|
||||
raise zmq.Again()
|
||||
return reply
|
||||
|
||||
socket = types.SimpleNamespace(send=send, recv=receive)
|
||||
monkeypatch.setattr(consumer, "_get_req_socket", lambda *args: socket)
|
||||
monkeypatch.setattr(consumer, "_invalidate_req_socket", lambda *args: None)
|
||||
assert consumer.get_with_deadline("0", "1", "abandoned-query", deadline=time.monotonic() + 1) is None
|
||||
assert not consumer._abandoned_queries
|
||||
consumer.abandon_get("abandoned-query")
|
||||
claim = next(iter(producer._pending["abandoned-query"].claims))
|
||||
if closing:
|
||||
producer._closing = True
|
||||
consumer._reap_deferred_transfers()
|
||||
assert not consumer._abandoned_queries
|
||||
assert not producer._pending
|
||||
claims = [
|
||||
msgspec.msgpack.decode(message[len(_GET_META_MSG) :])["claim_id"]
|
||||
for message in messages
|
||||
if message.startswith(_GET_META_MSG)
|
||||
]
|
||||
assert claims == [claim, claim]
|
||||
|
||||
|
||||
def test_expired_ack_is_deferred_instead_of_raising(consumer):
|
||||
consumer._req_local.deadline = time.monotonic() - 1
|
||||
try:
|
||||
assert consumer._notify_transfer_done("expired", {"source_host": "127.0.0.1", "source_port": PORT}) is False
|
||||
finally:
|
||||
consumer._req_local.deadline = None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("direct", [False, True])
|
||||
@pytest.mark.parametrize("lost_replies", [1, 3])
|
||||
def test_lost_metadata_reply_retry_releases_source(producer, consumer, monkeypatch, direct, lost_replies):
|
||||
_, _, metadata = producer.put("0", "1", "lost-reply", torch.ones(1))
|
||||
replies = []
|
||||
requests = []
|
||||
|
||||
def send(message):
|
||||
requests.append(message)
|
||||
replies.append(producer._handle_handshake_message(message))
|
||||
|
||||
def recv():
|
||||
reply = replies.pop(0)
|
||||
if len(requests) <= lost_replies:
|
||||
raise zmq.Again()
|
||||
return reply
|
||||
|
||||
socket = types.SimpleNamespace(send=send, recv=recv)
|
||||
monkeypatch.setattr(consumer, "_get_req_socket", lambda *args: socket)
|
||||
forwarded = metadata if direct else None
|
||||
for attempt in range(lost_replies):
|
||||
assert consumer._resolve_metadata("lost-reply", forwarded) is None
|
||||
resolved = consumer._resolve_metadata("lost-reply", forwarded)
|
||||
assert resolved is not None
|
||||
assert producer._pending["lost-reply"].claims == {resolved["claim_id"]}
|
||||
independent = consumer._resolve_metadata("lost-reply", forwarded)
|
||||
assert independent["claim_id"] != resolved["claim_id"]
|
||||
consumer._notify_transfer_done("lost-reply", resolved)
|
||||
consumer._notify_transfer_done("lost-reply", resolved)
|
||||
assert producer._pending["lost-reply"].claims == {independent["claim_id"]}
|
||||
consumer._notify_transfer_done("lost-reply", independent)
|
||||
assert producer._pending == {}
|
||||
assert producer._registered_descs == []
|
||||
assert consumer._req_local.claims == {}
|
||||
|
||||
|
||||
@pytest.mark.usefixtures("reliable_claim_queries")
|
||||
@@ -681,6 +813,77 @@ def test_deferred_transfer_releases_exactly_once_after_done(nixl_connector_cls):
|
||||
connector.close()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("poll_raises", [False, True], ids=["stuck", "poll-error"])
|
||||
def test_close_returns_without_releasing_active_dma(nixl_connector_cls, poll_raises):
|
||||
import threading
|
||||
|
||||
from vllm_omni.distributed.omni_connectors.connectors.nixl_connector import (
|
||||
_RETAINED_PRODUCERS,
|
||||
_DeferredTransfer,
|
||||
)
|
||||
|
||||
connector = nixl_connector_cls({"role": "receiver"})
|
||||
released = []
|
||||
|
||||
def check_state(handle):
|
||||
if poll_raises:
|
||||
raise RuntimeError("device unavailable")
|
||||
return "PROC"
|
||||
|
||||
connector._agent.check_xfer_state = check_state
|
||||
connector._agent.release_xfer_handle = lambda handle: released.append(("handle", handle))
|
||||
connector._agent.release_dlist_handle = lambda handle: released.append(("dlist", handle))
|
||||
connector._agent.remove_remote_agent = lambda agent: released.append(("agent", agent))
|
||||
connector._agent.deregister_memory = lambda descs: released.append(("registration", descs))
|
||||
transfer = _DeferredTransfer(
|
||||
tensors=[torch.zeros(1)],
|
||||
registrations=["registration"],
|
||||
dlists=["local", "remote"],
|
||||
handles=["transfer"],
|
||||
remote_agent="producer",
|
||||
)
|
||||
connector._defer_transfer(transfer)
|
||||
closing = threading.Thread(target=connector.close, daemon=True)
|
||||
try:
|
||||
closing.start()
|
||||
closing.join(timeout=2.0)
|
||||
assert not closing.is_alive(), "close must not wait forever for DMA completion"
|
||||
assert released == []
|
||||
assert transfer in connector._deferred_transfers
|
||||
assert transfer.tensors
|
||||
assert connector._closing
|
||||
assert connector in _RETAINED_PRODUCERS
|
||||
assert connector.health()["status"] == "unhealthy"
|
||||
cleanup_thread = connector._close_thread
|
||||
assert cleanup_thread is not None
|
||||
connector.close()
|
||||
assert connector._close_thread is cleanup_thread
|
||||
connector._agent.check_xfer_state = lambda handle: "DONE"
|
||||
cleanup_thread.join(timeout=2.0)
|
||||
assert not cleanup_thread.is_alive()
|
||||
assert connector._closed
|
||||
assert not connector._deferred_transfers
|
||||
assert not transfer.tensors
|
||||
assert connector not in _RETAINED_PRODUCERS
|
||||
finally:
|
||||
connector._agent.check_xfer_state = lambda handle: "DONE"
|
||||
closing.join(timeout=2.0)
|
||||
connector.close()
|
||||
|
||||
assert released == [
|
||||
("handle", "transfer"),
|
||||
("dlist", "local"),
|
||||
("dlist", "remote"),
|
||||
("agent", "producer"),
|
||||
("registration", "registration"),
|
||||
]
|
||||
assert not connector._deferred_transfers
|
||||
assert not transfer.tensors
|
||||
assert connector not in _RETAINED_PRODUCERS
|
||||
connector.close()
|
||||
assert len(released) == 5
|
||||
|
||||
|
||||
def test_active_sibling_transfer_requires_deferred_ownership(nixl_connector_cls):
|
||||
connector = nixl_connector_cls({"role": "receiver"})
|
||||
connector._agent.check_xfer_state = lambda handle: {"failed": "ERR", "active": "PROC"}[handle]
|
||||
@@ -691,7 +894,8 @@ def test_active_sibling_transfer_requires_deferred_ownership(nixl_connector_cls)
|
||||
connector.close()
|
||||
|
||||
|
||||
def test_get_defers_complete_ownership_when_sibling_transfer_is_active(nixl_connector_cls):
|
||||
@pytest.mark.parametrize("with_empty", [False, True])
|
||||
def test_get_defers_complete_ownership_when_sibling_transfer_is_active(nixl_connector_cls, with_empty):
|
||||
connector = nixl_connector_cls({"role": "receiver"})
|
||||
connector._agent.add_remote_agent = lambda metadata: "producer"
|
||||
connector._agent.get_xfer_descs = lambda regions, memory_type: (regions, memory_type)
|
||||
@@ -719,6 +923,9 @@ def test_get_defers_complete_ownership_when_sibling_transfer_is_active(nixl_conn
|
||||
],
|
||||
"size": 8,
|
||||
}
|
||||
if with_empty:
|
||||
metadata["tensor_specs"].insert(1, {"shape": [2, 0, 3], "dtype": "torch.int64", "device": "cpu", "size": 0})
|
||||
metadata["descriptor_groups"][1]["tensor_indices"] = [2]
|
||||
|
||||
try:
|
||||
assert connector.get("0", "1", "req-active-sibling", metadata) is None
|
||||
@@ -729,7 +936,9 @@ def test_get_defers_complete_ownership_when_sibling_transfer_is_active(nixl_conn
|
||||
assert len(deferred.registrations) == 2
|
||||
assert len(deferred.dlists) == 4
|
||||
assert deferred.remote_agent == "producer"
|
||||
assert len(deferred.tensors) == 2
|
||||
assert len(deferred.tensors) == 2 + with_empty
|
||||
if with_empty:
|
||||
assert deferred.tensors[1].shape == (2, 0, 3)
|
||||
finally:
|
||||
connector._agent.check_xfer_state = lambda handle: "DONE"
|
||||
connector.close()
|
||||
@@ -893,3 +1102,198 @@ def test_receive_device_config_wins(nixl_connector_cls):
|
||||
assert connector._resolve_receive_device("cuda:3") == torch.device("cpu")
|
||||
finally:
|
||||
connector.close()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def copying_native_agent(consumer, monkeypatch):
|
||||
"""Enforce native nonzero descriptors and copy only the submitted regions."""
|
||||
agent = consumer._agent
|
||||
calls = {"agents": [], "descs": [], "transfers": [], "released": []}
|
||||
|
||||
def add_remote(metadata):
|
||||
calls["agents"].append(metadata)
|
||||
return "producer"
|
||||
|
||||
def descriptors(regions, memory_type):
|
||||
assert regions and all(len(region) == 3 and region[0] > 0 and region[1] > 0 for region in regions)
|
||||
calls["descs"].append((regions, memory_type))
|
||||
return regions
|
||||
|
||||
def prepare(operation, local, local_ids, remote, remote_ids):
|
||||
assert operation == "READ"
|
||||
assert local_ids == remote_ids == list(range(len(local)))
|
||||
assert len(local) == len(remote)
|
||||
pairs = list(zip(local, remote, strict=True))
|
||||
for destination, source in pairs:
|
||||
assert destination[1] == source[1] > 0
|
||||
calls["transfers"].append(pairs)
|
||||
return pairs
|
||||
|
||||
def transfer(pairs):
|
||||
for destination, source in pairs:
|
||||
ctypes.memmove(destination[0], source[0], source[1])
|
||||
|
||||
monkeypatch.setattr(agent, "add_remote_agent", add_remote, raising=False)
|
||||
monkeypatch.setattr(agent, "get_xfer_descs", descriptors, raising=False)
|
||||
monkeypatch.setattr(agent, "prep_xfer_dlist", lambda agent, descs: descs, raising=False)
|
||||
monkeypatch.setattr(agent, "make_prepped_xfer", prepare, raising=False)
|
||||
monkeypatch.setattr(agent, "transfer", transfer, raising=False)
|
||||
monkeypatch.setattr(agent, "check_xfer_state", lambda handle: "DONE", raising=False)
|
||||
for method in ("release_xfer_handle", "release_dlist_handle", "remove_remote_agent"):
|
||||
monkeypatch.setattr(
|
||||
agent, method, lambda handle, method=method: calls["released"].append((method, handle)), raising=False
|
||||
)
|
||||
return calls
|
||||
|
||||
|
||||
@pytest.mark.parametrize("case", ["empty", "all_empty", "mixed", "scalar", "structured_empty", "structured_mixed"])
|
||||
@pytest.mark.usefixtures("reliable_claim_queries")
|
||||
def test_zero_byte_leaves_roundtrip_without_native_descriptors(
|
||||
producer, consumer, copying_native_agent, monkeypatch, case
|
||||
):
|
||||
empty = torch.empty((2, 0, 3), dtype=torch.float64)
|
||||
other_empty = torch.empty((0,), dtype=torch.int64)
|
||||
scalar = torch.tensor(7, dtype=torch.int64)
|
||||
vector = torch.arange(6, dtype=torch.float32).reshape(2, 3)
|
||||
payloads = {
|
||||
"empty": empty,
|
||||
"all_empty": [empty, other_empty],
|
||||
"mixed": [empty, scalar, other_empty, vector, empty],
|
||||
"scalar": scalar,
|
||||
"structured_empty": {"leaves": (empty, [other_empty]), "label": "empty slots"},
|
||||
"structured_mixed": {"leaves": (empty, [scalar, other_empty, vector, empty]), "label": "mixed slots"},
|
||||
}
|
||||
payload = payloads[case]
|
||||
# Simulate mixed source memory groups without requiring a GPU in unit tests.
|
||||
monkeypatch.setattr(producer, "_resolve_memory_type", lambda t: "VRAM" if t.dtype == torch.int64 else "DRAM")
|
||||
# Splitting the producer's DRAM group on receipt also exercises local IDs:
|
||||
# IDs are group-relative, whereas spec/skeleton indices remain global.
|
||||
monkeypatch.setattr(consumer, "_resolve_memory_type", lambda t: "VRAM" if t.dtype == torch.float32 else "DRAM")
|
||||
consumer._receive_device = torch.device("cpu")
|
||||
success, size, metadata = producer.put("0", "1", "req-empty-slots", payload)
|
||||
assert success
|
||||
tensors = producer._pending["req-empty-slots"].tensors
|
||||
specs = metadata["tensor_specs"]
|
||||
assert len(specs) == len(tensors)
|
||||
assert size == sum(t.numel() * t.element_size() for t in tensors)
|
||||
for tensor, spec in zip(tensors, specs, strict=True):
|
||||
assert spec["shape"] == list(tensor.shape)
|
||||
assert spec["dtype"] == str(tensor.dtype)
|
||||
assert spec["device"] == str(tensor.device)
|
||||
indices = [i for group in metadata["descriptor_groups"] for i in group["tensor_indices"]]
|
||||
assert sorted(indices) == [i for i, tensor in enumerate(tensors) if tensor.numel()]
|
||||
# Verify receiver override applies to empty specs too, without trusting a
|
||||
# producer's device index. Real CUDA placement is covered by the native suite.
|
||||
for spec in specs:
|
||||
if spec["size"] == 0:
|
||||
spec["device"] = "cuda:7"
|
||||
received = consumer.get("0", "1", "req-empty-slots", metadata)
|
||||
assert received is not None
|
||||
actual, received_size = received
|
||||
assert received_size == size
|
||||
if isinstance(payload, dict):
|
||||
assert actual.keys() == payload.keys()
|
||||
assert actual["label"] == payload["label"]
|
||||
assert isinstance(actual["leaves"], tuple)
|
||||
assert isinstance(actual["leaves"][1], list)
|
||||
torch.testing.assert_close(actual["leaves"], payload["leaves"], rtol=0, atol=0)
|
||||
else:
|
||||
torch.testing.assert_close(actual, payload, rtol=0, atol=0)
|
||||
calls = copying_native_agent
|
||||
assert sum(len(pairs) for pairs in calls["transfers"]) == len(indices)
|
||||
if case in ("empty", "all_empty"):
|
||||
assert metadata["descriptor_groups"] == []
|
||||
assert calls == {"agents": [], "descs": [], "transfers": [], "released": []}
|
||||
assert size == 0
|
||||
else:
|
||||
assert len(calls["agents"]) == 1
|
||||
assert len(calls["released"]) == 3 * len(calls["transfers"]) + 1
|
||||
assert producer._pending == producer._published == {}
|
||||
assert producer._registered_descs == producer._agent.registered == []
|
||||
assert consumer._agent.registered == consumer._remote_agents == consumer._deferred_transfers == []
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"defect",
|
||||
[
|
||||
"missing_groups",
|
||||
"missing_nonempty",
|
||||
"duplicate",
|
||||
"empty_index",
|
||||
"zero_region",
|
||||
"wrong_region_size",
|
||||
"null_pointer",
|
||||
"negative_device",
|
||||
"short_region",
|
||||
"bad_region_type",
|
||||
"bool_index",
|
||||
"float_index",
|
||||
"empty_group",
|
||||
"unequal_lengths",
|
||||
"missing_memory_type",
|
||||
"fake_empty_size",
|
||||
"fake_nonempty_size",
|
||||
"negative_shape",
|
||||
"float_shape",
|
||||
"bad_dtype",
|
||||
"bad_spec",
|
||||
],
|
||||
)
|
||||
@pytest.mark.usefixtures("reliable_claim_queries")
|
||||
def test_invalid_empty_tensor_metadata_rejected_before_native_calls(producer, consumer, copying_native_agent, defect):
|
||||
success, _, metadata = producer.put(
|
||||
"0", "1", "req-invalid-empty", [torch.empty((0, 2)), torch.tensor(3.0), torch.empty((1, 0))]
|
||||
)
|
||||
assert success
|
||||
group = metadata["descriptor_groups"][0]
|
||||
region = group["regions"][0]
|
||||
if defect == "missing_groups":
|
||||
metadata.pop("descriptor_groups")
|
||||
elif defect == "missing_nonempty":
|
||||
metadata["descriptor_groups"] = []
|
||||
elif defect == "duplicate":
|
||||
group["tensor_indices"] *= 2
|
||||
group["regions"] *= 2
|
||||
elif defect == "empty_index":
|
||||
group["tensor_indices"] = [0]
|
||||
elif defect in ("zero_region", "wrong_region_size"):
|
||||
group["regions"] = [(region[0], 0 if defect == "zero_region" else 8, 0, "")]
|
||||
elif defect == "null_pointer":
|
||||
group["regions"] = [(0, 4, 0, "")]
|
||||
elif defect == "negative_device":
|
||||
group["regions"] = [(region[0], 4, -1, "")]
|
||||
elif defect == "short_region":
|
||||
group["regions"] = [(region[0], 4)]
|
||||
elif defect == "bad_region_type":
|
||||
group["regions"] = [None]
|
||||
elif defect in ("bool_index", "float_index"):
|
||||
group["tensor_indices"] = [True if defect == "bool_index" else 1.0]
|
||||
elif defect == "empty_group":
|
||||
metadata["descriptor_groups"].append({"memory_type": "DRAM", "tensor_indices": [], "regions": []})
|
||||
elif defect == "unequal_lengths":
|
||||
group["regions"] = []
|
||||
elif defect == "missing_memory_type":
|
||||
group.pop("memory_type")
|
||||
elif defect == "fake_empty_size":
|
||||
metadata["tensor_specs"][1]["size"] = 0
|
||||
metadata["descriptor_groups"] = []
|
||||
elif defect == "fake_nonempty_size":
|
||||
metadata["tensor_specs"][0]["size"] = 4
|
||||
elif defect == "negative_shape":
|
||||
metadata["tensor_specs"][0]["shape"] = [-1, 0]
|
||||
elif defect == "float_shape":
|
||||
metadata["tensor_specs"][0]["shape"] = [0.0, 2]
|
||||
elif defect == "bad_dtype":
|
||||
metadata["tensor_specs"][0]["dtype"] = "torch.not_a_dtype"
|
||||
elif defect == "bad_spec":
|
||||
metadata["tensor_specs"][0] = None
|
||||
assert consumer.get("0", "1", "req-invalid-empty", metadata) is None
|
||||
assert copying_native_agent == {"agents": [], "descs": [], "transfers": [], "released": []}
|
||||
assert consumer._agent.registered == []
|
||||
# Metadata acquisition now claims the source before validation. With no
|
||||
# submitted DMA, failure safely completes that claim instead of leaking it;
|
||||
# it must still report no successful payload receive.
|
||||
assert producer._pending == producer._published == {}
|
||||
assert producer._registered_descs == producer._agent.registered == []
|
||||
assert consumer._metrics["gets"] == 0
|
||||
assert consumer._metrics["errors"] == 1
|
||||
@@ -4,12 +4,13 @@
|
||||
import multiprocessing as mp
|
||||
import socket
|
||||
import time
|
||||
from dataclasses import fields
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm_omni.data_entry_keys import HiddenStatesStruct, MetaStruct, OmniPayloadStruct
|
||||
from vllm_omni.data_entry_keys import EmbeddingsStruct, HiddenStatesStruct, MetaStruct, OmniPayloadStruct
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.cuda, pytest.mark.parallel]
|
||||
@@ -29,7 +30,83 @@ def _free_port() -> int:
|
||||
return int(sock.getsockname()[1])
|
||||
|
||||
|
||||
def _producer(port: int, ready: Any, consumed: Any, result: Any, claimed: Any, proceed: Any, wire: Any) -> None:
|
||||
def _h3_conditioning(payload_kind: str, device: str = "cpu"):
|
||||
from vllm_omni.model_executor.models.minimax_h3.conditioning import MiniMaxH3EncoderConditioning
|
||||
|
||||
media = {}
|
||||
if payload_kind == "h3_ref2va":
|
||||
media = {
|
||||
"visual_condition": torch.arange(5 * 96, dtype=torch.float32).reshape(5, 96) / 8,
|
||||
"visual_condition_shapes": ((1, 2, 2), (2, 2, 4)),
|
||||
"audio_condition": torch.arange(10 * 32, dtype=torch.float32, device=device).reshape(10, 32) / 16,
|
||||
"audio_condition_lengths": (2, 3),
|
||||
"ref_blocks": (
|
||||
{"kind": "image", "latent_t": 1, "latent_h": 2, "latent_w": 2},
|
||||
{"kind": "video_audio", "ref_audio_t": 3, "latent_t": 2, "latent_h": 2, "latent_w": 4},
|
||||
),
|
||||
}
|
||||
elif payload_kind == "h3_fl2va":
|
||||
media = {
|
||||
"visual_condition": torch.arange(2 * 96, dtype=torch.float32).reshape(2, 96) / 8,
|
||||
"visual_condition_shapes": ((1, 2, 2), (1, 2, 2)),
|
||||
"keyframe_frame_indices": (0, 16),
|
||||
}
|
||||
return MiniMaxH3EncoderConditioning(
|
||||
hidden_states=(torch.arange(3 * 5120, device=device).reshape(3, 5120) % 127).to(torch.bfloat16),
|
||||
token_tags=torch.tensor([1, 0, 1], dtype=torch.int64),
|
||||
task=payload_kind.removeprefix("h3_"),
|
||||
height=256,
|
||||
width=448,
|
||||
num_frames=17,
|
||||
latent_t=5,
|
||||
audio_t=10,
|
||||
**media,
|
||||
)
|
||||
|
||||
|
||||
def _h3_structured_payload(payload_kind: str) -> OmniPayloadStruct:
|
||||
# Only adapt to the existing public Struct; NIXL owns all serialization.
|
||||
wire = _h3_conditioning(payload_kind, "cuda:0").to_omni_payload()
|
||||
# Exercise both CPU and CUDA empty leaves (T2VA has two empty slots).
|
||||
wire["embed"]["speech_feat"] = wire["embed"]["speech_feat"].to("cuda:0")
|
||||
return OmniPayloadStruct(
|
||||
hidden_states=HiddenStatesStruct(**wire["hidden_states"]),
|
||||
meta=MetaStruct(**wire["meta"]),
|
||||
embed=EmbeddingsStruct(**wire["embed"]),
|
||||
kv_metadata=wire["kv_metadata"],
|
||||
request_id="native-smoke",
|
||||
)
|
||||
|
||||
|
||||
def _assert_h3_roundtrip(payload: dict[str, Any], payload_kind: str) -> tuple:
|
||||
from vllm_omni.model_executor.models.minimax_h3.conditioning import MiniMaxH3EncoderConditioning
|
||||
|
||||
expected = _h3_conditioning(payload_kind)
|
||||
expected_wire = expected.to_omni_payload()
|
||||
tensors = []
|
||||
for section, values in expected_wire.items():
|
||||
for key, reference in values.items():
|
||||
actual = payload[section][key]
|
||||
assert actual.device == torch.device("cuda:1"), (section, key, actual.device)
|
||||
# Exact values, shapes and dtypes, including empty wire slots and layout.
|
||||
torch.testing.assert_close(actual.cpu(), reference, rtol=0, atol=0)
|
||||
tensors.append((f"{section}.{key}", str(actual.dtype), list(actual.shape), str(actual.device)))
|
||||
actual = MiniMaxH3EncoderConditioning.from_omni_payload(payload)
|
||||
for field in fields(expected):
|
||||
reference = getattr(expected, field.name)
|
||||
value = getattr(actual, field.name)
|
||||
if isinstance(reference, torch.Tensor):
|
||||
assert value.device == torch.device("cuda:1"), field.name
|
||||
torch.testing.assert_close(value.cpu(), reference, rtol=0, atol=0)
|
||||
else:
|
||||
assert value == reference, field.name
|
||||
assert payload["request_id"] == "native-smoke"
|
||||
return (payload_kind, tensors)
|
||||
|
||||
|
||||
def _producer(
|
||||
port: int, ready: Any, consumed: Any, result: Any, claimed: Any, proceed: Any, wire: Any, payload_kind: str
|
||||
) -> None:
|
||||
from vllm_omni.distributed.omni_connectors.connectors.nixl_connector import NixlConnector
|
||||
|
||||
current_omni_platform.set_device(0)
|
||||
@@ -48,6 +125,8 @@ def _producer(port: int, ready: Any, consumed: Any, result: Any, claimed: Any, p
|
||||
kv_metadata={"empty": torch.empty((2, 0, 3), device="cuda:0")},
|
||||
request_id="native-smoke",
|
||||
)
|
||||
if payload_kind != "generic":
|
||||
payload = _h3_structured_payload(payload_kind)
|
||||
success, size, metadata = connector.put("0", "1", "native-smoke", payload)
|
||||
result.put(("put", success, size, metadata["kind"] if metadata else None))
|
||||
wire.put(metadata)
|
||||
@@ -76,7 +155,15 @@ def _producer(port: int, ready: Any, consumed: Any, result: Any, claimed: Any, p
|
||||
|
||||
|
||||
def _consumer(
|
||||
port: int, ready: Any, consumed: Any, result: Any, claimed: Any, proceed: Any, wire: Any, direct: bool
|
||||
port: int,
|
||||
ready: Any,
|
||||
consumed: Any,
|
||||
result: Any,
|
||||
claimed: Any,
|
||||
proceed: Any,
|
||||
wire: Any,
|
||||
direct: bool,
|
||||
payload_kind: str,
|
||||
) -> None:
|
||||
from vllm_omni.distributed.omni_connectors.kv_transfer_manager import (
|
||||
OmniKVCacheConfig,
|
||||
@@ -133,11 +220,15 @@ def _consumer(
|
||||
if received is None:
|
||||
raise TimeoutError("native NIXL transfer did not complete")
|
||||
payload, size = received
|
||||
result.put(("native_states", states))
|
||||
if payload_kind != "generic":
|
||||
result.put(("get", size, *_assert_h3_roundtrip(payload, payload_kind)))
|
||||
consumed.set()
|
||||
return
|
||||
hidden = payload["hidden_states"]["output"]
|
||||
token_roles = payload["meta"]["token_role_ids"]
|
||||
empty = payload["kv_metadata"]["empty"]
|
||||
assert empty.shape == (2, 0, 3) and str(empty.device) == "cuda:1"
|
||||
result.put(("native_states", states))
|
||||
result.put(
|
||||
(
|
||||
"get",
|
||||
@@ -161,8 +252,9 @@ def _consumer(
|
||||
|
||||
|
||||
@pytest.mark.skipif(not _native_nixl_available(), reason="requires NIXL and at least two CUDA devices")
|
||||
@pytest.mark.parametrize("payload_kind", ["generic", "h3_ref2va", "h3_fl2va", "h3_t2va"])
|
||||
@pytest.mark.parametrize("direct", [False, True])
|
||||
def test_native_two_process_structured_mixed_device_transfer(direct):
|
||||
def test_native_two_process_structured_mixed_device_transfer(direct, payload_kind):
|
||||
context = mp.get_context("spawn")
|
||||
ready = context.Event()
|
||||
consumed = context.Event()
|
||||
@@ -171,8 +263,12 @@ def test_native_two_process_structured_mixed_device_transfer(direct):
|
||||
wire = context.Queue()
|
||||
result = context.Queue()
|
||||
port = _free_port()
|
||||
producer = context.Process(target=_producer, args=(port, ready, consumed, result, claimed, proceed, wire))
|
||||
consumer = context.Process(target=_consumer, args=(port, ready, consumed, result, claimed, proceed, wire, direct))
|
||||
producer = context.Process(
|
||||
target=_producer, args=(port, ready, consumed, result, claimed, proceed, wire, payload_kind)
|
||||
)
|
||||
consumer = context.Process(
|
||||
target=_consumer, args=(port, ready, consumed, result, claimed, proceed, wire, direct, payload_kind)
|
||||
)
|
||||
|
||||
producer.start()
|
||||
consumer.start()
|
||||
@@ -180,8 +276,10 @@ def test_native_two_process_structured_mixed_device_transfer(direct):
|
||||
consumer.join(timeout=60)
|
||||
if producer.is_alive():
|
||||
producer.terminate()
|
||||
producer.join(timeout=10)
|
||||
if consumer.is_alive():
|
||||
consumer.terminate()
|
||||
consumer.join(timeout=10)
|
||||
|
||||
records = [result.get(timeout=5) for _ in range(5)]
|
||||
print("Native ownership records:", records)
|
||||
@@ -193,12 +291,17 @@ def test_native_two_process_structured_mixed_device_transfer(direct):
|
||||
assert records[0][3] == "structured"
|
||||
get_record = next(record for record in records if record[0] == "get")
|
||||
assert get_record[1] == records[0][2]
|
||||
assert get_record[2:6] == (
|
||||
list(range(8)),
|
||||
"torch.float32",
|
||||
[8],
|
||||
"cuda:1",
|
||||
)
|
||||
assert get_record[6:] == ([1, 2, 3], "torch.int64", [3], "cuda:1")
|
||||
if payload_kind == "generic":
|
||||
assert get_record[2:6] == (
|
||||
list(range(8)),
|
||||
"torch.float32",
|
||||
[8],
|
||||
"cuda:1",
|
||||
)
|
||||
assert get_record[6:] == ([1, 2, 3], "torch.int64", [3], "cuda:1")
|
||||
else:
|
||||
assert get_record[2] == payload_kind
|
||||
assert len(get_record[3]) == 5
|
||||
print(f"Native H3 exact wire/semantic roundtrip: {get_record}")
|
||||
assert next(record for record in records if record[0] == "cleanup") == ("cleanup", 0, 0)
|
||||
assert next(record for record in records if record[0] == "source_retained") == ("source_retained", 1)
|
||||
@@ -2,7 +2,9 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit tests for SharedMemoryConnector focusing on TP / CFG / metadata fallback."""
|
||||
|
||||
import fcntl
|
||||
import os
|
||||
import time
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
@@ -25,6 +27,15 @@ def connector():
|
||||
|
||||
|
||||
class TestKeyBasedReadWrite:
|
||||
def test_deadline_receive_does_not_wait_for_writer_lock(self, connector):
|
||||
key = "deadline_locked_payload"
|
||||
connector.put("0", "1", key, {"value": 7})
|
||||
with open(f"/dev/shm/shm_{key}_lockfile.lock", "rb+") as lock_file:
|
||||
fcntl.flock(lock_file, fcntl.LOCK_EX)
|
||||
assert connector.get_with_deadline("0", "1", key, deadline=time.monotonic() + 1) is None
|
||||
assert connector.get_with_deadline("0", "1", key, deadline=time.monotonic() - 1) is None
|
||||
assert connector.get_with_deadline("0", "1", key, deadline=time.monotonic() + 1)[0] == {"value": 7}
|
||||
|
||||
def test_put_then_get_by_key(self, connector):
|
||||
data = {"hello": "world", "n": 42}
|
||||
ok, size, meta = connector.put("s0", "s1", "test_key_1", data)
|
||||
|
||||
Reference in new issue
Block a user