[Core] Cleanup Omni connector runtime and transfer backends (#5096)

Signed-off-by: spencerr221 <liubingyu62@gmail.com>
This commit is contained in:
Bingyu (Spencer) Liu authored and GitHub committed 2026-07-22 09:44:15 +08:00
1 parent 2cf5f57b3e
commit dcf8b07d39
53 files changed
+349 -448

No files matched your search

-2
View File
@@ -302,7 +302,6 @@ def test_from_pipeline_config_maps_orchestrator_cli_overrides():
"omni_dp_size_local": 2,
"omni_lb_policy": "round_robin",
"omni_heartbeat_timeout": 9.5,
"shm_threshold_bytes": 4096,
"batch_timeout": 3,
},
)
@@ -317,7 +316,6 @@ def test_from_pipeline_config_maps_orchestrator_cli_overrides():
assert orchestrator_config.omni_dp_size_local == 2
assert orchestrator_config.omni_lb_policy == "round_robin"
assert orchestrator_config.omni_heartbeat_timeout == 9.5
assert orchestrator_config.shm_threshold_bytes == 4096
assert orchestrator_config.batch_timeout == 3
@@ -142,11 +142,10 @@ def test_recv_no_connector():
def test_shm_connector_flow(mocker: MockerFixture):
"""
Verify the full flow: Send -> Adapter -> Connector -> Adapter -> Recv.
Using real SharedMemoryConnector (inline mode for simplicity).
Using real SharedMemoryConnector and key-addressed SHM metadata.
"""
# 1. Setup Connector
config = {"shm_threshold_bytes": 1024, "inline_small_payloads": True}
connector = SharedMemoryConnector(config)
connector = SharedMemoryConnector({})
connectors_map = {("0", "1"): connector}
# 2. Setup Data
@@ -60,10 +60,9 @@ def test_ndarray_serialization():
def test_create_shm_connector():
"""Test creating SharedMemoryConnector via Factory."""
spec = ConnectorSpec(name="SharedMemoryConnector", extra={"shm_threshold_bytes": 1024})
spec = ConnectorSpec(name="SharedMemoryConnector")
connector = OmniConnectorFactory.create_connector(spec)
assert isinstance(connector, SharedMemoryConnector)
assert connector.threshold == 1024
def test_create_unknown_connector():
@@ -75,21 +74,20 @@ def test_create_unknown_connector():
@pytest.fixture
def shm_connector():
config = {"shm_threshold_bytes": 100, "inline_small_payloads": True}
return SharedMemoryConnector(config)
connector = SharedMemoryConnector({})
yield connector
connector.close()
def test_put_get_inline(shm_connector):
"""Test inline transfer for small data."""
def test_put_get_small_payload_uses_shm(shm_connector):
"""Small payloads use key-addressed SHM; inline metadata is retired."""
data = {"small": "data"}
success, size, metadata = shm_connector.put("stage_0", "stage_1", "req_1", data)
assert success is True
assert "inline_bytes" in metadata
assert "shm" not in metadata
assert "shm" in metadata
assert "inline_bytes" not in metadata
assert "size" in metadata
assert shm_connector._metrics["inline_writes"] == 1
assert shm_connector._metrics["shm_writes"] == 0
# Retrieve
retrieved_data, ret_size = shm_connector.get("stage_0", "stage_1", "req_1", metadata)
@@ -195,3 +193,42 @@ def test_mooncake_connector_defaults_missing_host_to_detected_ip(monkeypatch: py
assert connector.get_connection_info()["host"] == "10.20.30.40"
finally:
connector.close()
def test_mooncake_get_counts_zmq_timeout_separately(mocker: MockerFixture):
import vllm_omni.distributed.omni_connectors.connectors.mooncake_transfer_engine_connector as mooncake_module
connector = object.__new__(mooncake_module.MooncakeTransferEngineConnector)
connector._closed = False
connector._metrics = {"errors": 0, "timeouts": 0}
connector._resolve_metadata = mocker.Mock(
return_value={"source_host": "127.0.0.1", "source_port": 1234, "data_size": 8}
)
recv_buffer = mocker.Mock()
connector._alloc_recv_buffer = mocker.Mock(return_value=(recv_buffer, 1000))
connector._request_transfer = mocker.Mock(side_effect=mooncake_module.zmq.Again())
assert connector.get("s0", "s1", "req") is None
assert connector._metrics == {"errors": 0, "timeouts": 1}
recv_buffer.release.assert_called_once()
connector._closed = True
def test_mooncake_get_counts_cuda_sync_failure_as_error(mocker: MockerFixture):
import vllm_omni.distributed.omni_connectors.connectors.mooncake_transfer_engine_connector as mooncake_module
connector = object.__new__(mooncake_module.MooncakeTransferEngineConnector)
connector._closed = False
connector._metrics = {"errors": 0, "timeouts": 0}
connector._resolve_metadata = mocker.Mock(
return_value={"source_host": "127.0.0.1", "source_port": 1234, "data_size": 8}
)
recv_buffer = mocker.Mock()
connector._alloc_recv_buffer = mocker.Mock(return_value=(recv_buffer, 1000))
connector._request_transfer = mocker.Mock(return_value=mooncake_module.TRANS_DONE)
connector._sync_receive_buffer_if_needed = mocker.Mock(side_effect=RuntimeError("CUDA sync failed"))
assert connector.get("s0", "s1", "req") is None
assert connector._metrics == {"errors": 1, "timeouts": 0}
recv_buffer.release.assert_called_once()
connector._closed = True
@@ -48,7 +48,6 @@ def test_load_qwen_yaml_configs(yaml_file):
print(f"Testing config load: {yaml_file.name}")
try:
# Attempt to load the config
# default_shm_threshold doesn't matter much for loading correctness, using default
config = load_omni_transfer_config(yaml_file)
assert config is not None, "Config should not be None"
@@ -2,7 +2,10 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for SharedMemoryConnector focusing on TP / CFG / metadata fallback."""
import os
import pytest
import torch
from vllm_omni.distributed.omni_connectors.connectors.shm_connector import (
SharedMemoryConnector,
@@ -13,7 +16,7 @@ pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
@pytest.fixture()
def connector():
c = SharedMemoryConnector({"shm_threshold_bytes": 0})
c = SharedMemoryConnector({})
yield c
c.close()
@@ -36,6 +39,30 @@ class TestKeyBasedReadWrite:
assert obj == data
assert rsize == size
assert "test_key_1" not in connector._pending_keys
assert connector._metrics["gets"] == 1
def test_tensor_payload_removes_lock_file(self, connector):
key = "tensor_payload"
payload = torch.ones(2, 2)
ok, _, metadata = connector.put("s0", "s1", key, payload)
assert ok
result = connector.get("s0", "s1", key, metadata=metadata)
assert result is not None
assert torch.equal(result[0], payload)
assert not os.path.exists(f"/dev/shm/shm_{key}_lockfile.lock")
def test_falsey_payload_removes_lock_file(self, connector):
key = "falsey_payload"
ok, _, metadata = connector.put("s0", "s1", key, 0)
assert ok
result = connector.get("s0", "s1", key, metadata=metadata)
assert result is not None
assert result[0] == 0
assert not os.path.exists(f"/dev/shm/shm_{key}_lockfile.lock")
def test_get_nonexistent_key_returns_none(self, connector):
result = connector.get("s0", "s1", "no_such_key_xyz", metadata=None)
@@ -141,7 +141,6 @@ _DEPLOY_CONFIG = {
"connectors": {
"shared_memory_connector": {
"name": "SharedMemoryConnector",
"extra": {"shm_threshold_bytes": 65536},
},
},
"stages": [
@@ -1,6 +1,5 @@
import asyncio
import os
import sys
from contextlib import ExitStack
import pytest
@@ -67,11 +66,9 @@ async def generate(
@pytest.mark.asyncio
async def test_abort():
with ExitStack() as after:
# Avoid SHM IPC in tests to prevent /dev/shm exhaustion and SIGBUS.
engine = AsyncOmni(
model=model,
stage_configs_path=stage_config,
shm_threshold_bytes=sys.maxsize,
)
after.callback(engine.shutdown)
+1 -3
View File
@@ -121,9 +121,7 @@ class _FakeStage:
self._in_q = in_q
self._out_q = out_q
def init_stage_worker(
self, model: str, *, is_async=False, shm_threshold_bytes=65536, ctx=None, batch_timeout=10, **kwargs
):
def init_stage_worker(self, model: str, *, is_async=False, ctx=None, batch_timeout=10, **kwargs):
self._proc = _ns(
start=lambda: None,
join=lambda timeout=None: None,
-2
View File
@@ -2333,7 +2333,6 @@ class OmniRunner:
# production default in AsyncOmniEngine remains 600s; this only
# affects the test runner wrapper.
init_timeout: int = 1800,
shm_threshold_bytes: int = 65536,
log_stats: bool = False,
stage_configs_path: str | None = None,
**kwargs,
@@ -2353,7 +2352,6 @@ class OmniRunner:
stage_init_timeout=stage_init_timeout,
batch_timeout=batch_timeout,
init_timeout=init_timeout,
shm_threshold_bytes=shm_threshold_bytes,
stage_configs_path=stage_configs_path,
**kwargs,
)
+2 -27
View File
@@ -110,7 +110,6 @@ class TestMixinAsyncChunkSendRecv(unittest.TestCase):
sender = MixinHost()
sender.init_omni_connectors(
vllm_config=None,
model_config=_make_model_config(stage_id=0, async_chunk=True),
)
sender._omni_connector = connector
@@ -147,7 +146,6 @@ class TestMixinAsyncChunkSendRecv(unittest.TestCase):
sender = MixinHost()
sender.init_omni_connectors(
vllm_config=None,
model_config=_make_model_config(stage_id=0, async_chunk=True),
)
sender._omni_connector = connector
@@ -180,7 +178,6 @@ class TestMixinKVCacheTransfer(unittest.TestCase):
host = MixinHost()
host.init_omni_connectors(
vllm_config=None,
model_config=_make_model_config(),
kv_transfer_manager=mock_kvm,
)
@@ -202,7 +199,6 @@ class TestMixinKVCacheTransfer(unittest.TestCase):
host = MixinHost()
host.init_omni_connectors(
vllm_config=None,
model_config=_make_model_config(),
kv_transfer_manager=mock_kvm,
)
@@ -219,7 +215,6 @@ class TestMixinKVCacheTransfer(unittest.TestCase):
host = MixinHost()
host.init_omni_connectors(
vllm_config=None,
model_config=_make_model_config(),
kv_transfer_manager=mock_kvm,
)
@@ -257,7 +252,6 @@ class TestMixinKVCacheTransfer(unittest.TestCase):
host = MixinHost()
host.init_omni_connectors(
vllm_config=None,
model_config=_make_model_config(),
kv_transfer_manager=mock_kvm,
)
@@ -281,7 +275,6 @@ class TestOmniConnectorOutput(unittest.TestCase):
def test_output_aggregation(self):
host = MixinHost()
host.init_omni_connectors(
vllm_config=None,
model_config=_make_model_config(),
)
@@ -310,7 +303,6 @@ class TestMixinNoConnector(unittest.TestCase):
def test_no_connector(self):
host = MixinHost()
host.init_omni_connectors(
vllm_config=None,
model_config=_make_model_config(),
)
self.assertIsNone(host._omni_connector)
@@ -336,7 +328,6 @@ class TestFinishedLoadReqsDrain(unittest.TestCase):
def test_finished_load_reqs_flow_to_chunk_ready(self):
host = MixinHost()
host.init_omni_connectors(
vllm_config=None,
model_config=_make_model_config(),
)
@@ -389,7 +380,6 @@ class TestFullPayloadSendWithCustomFunc(unittest.TestCase):
host = MixinHost()
host.init_omni_connectors(
vllm_config=None,
model_config=_make_model_config(),
)
host._omni_connector = MockConnector(stage_id=0)
@@ -424,7 +414,6 @@ class TestFullPayloadSendWithCustomFunc(unittest.TestCase):
host = MixinHost()
host.init_omni_connectors(
vllm_config=None,
model_config=_make_model_config(),
)
host._omni_connector = MockConnector(stage_id=0)
@@ -453,7 +442,6 @@ class TestKVSentReqIdsAccumulation(unittest.TestCase):
host = MixinHost()
host.init_omni_connectors(
vllm_config=None,
model_config=_make_model_config(),
kv_transfer_manager=mock_kvm,
)
@@ -488,7 +476,6 @@ class TestChunkStreamCompletedGuard(unittest.TestCase):
def _make_host(self, stage_id: int = 1) -> MixinHost:
host = MixinHost()
host.init_omni_connectors(
vllm_config=None,
model_config=_make_model_config(stage_id=stage_id, async_chunk=True),
)
host._omni_connector = MockConnector(stage_id=stage_id)
@@ -587,7 +574,6 @@ class TestCleanupFinishedRequest(unittest.TestCase):
def _make_host(self, stage_id: int = 1) -> MixinHost:
host = MixinHost()
host.init_omni_connectors(
vllm_config=None,
model_config=_make_model_config(stage_id=stage_id, async_chunk=True),
)
host._omni_connector = MockConnector(stage_id=stage_id)
@@ -771,7 +757,6 @@ class TestSendChunkCachesMapping(unittest.TestCase):
"""send_chunk should cache the internal→external mapping."""
host = MixinHost()
host.init_omni_connectors(
vllm_config=None,
model_config=_make_model_config(stage_id=0, async_chunk=True),
)
host._omni_connector = MockConnector(stage_id=0)
@@ -802,7 +787,6 @@ class TestLocalPayloadCacheLifecycle(unittest.TestCase):
def _make_host(self) -> MixinHost:
host = MixinHost()
host.init_omni_connectors(
vllm_config=None,
model_config=_make_model_config(stage_id=0),
)
host._omni_connector = MockConnector(stage_id=0)
@@ -901,7 +885,6 @@ class TestTPAsyncChunkFanout(unittest.TestCase):
def _make_host(self, rank: int) -> MixinHost:
host = MixinHost()
host.init_omni_connectors(
vllm_config=None,
model_config=_make_model_config(stage_id=2, async_chunk=True, worker_type="gen"),
)
host._omni_connector = MagicMock()
@@ -984,7 +967,6 @@ class TestKVTransferLifecycle(unittest.TestCase):
def _make_host(self) -> MixinHost:
host = MixinHost()
host.init_omni_connectors(
vllm_config=None,
model_config=_make_model_config(stage_id=0),
)
return host
@@ -1039,7 +1021,6 @@ class TestAsyncPayloadLifecycle(unittest.TestCase):
def test_send_side_request_payload_not_cleared_before_payload_is_consumable(self):
host = MixinHost()
host.init_omni_connectors(
vllm_config=None,
model_config=_make_model_config(stage_id=1, async_chunk=True, worker_type="ar"),
)
host._request_ids_mapping["r1"] = "r1"
@@ -1063,7 +1044,6 @@ class TestAsyncPayloadLifecycle(unittest.TestCase):
def test_payload_consumable_ignores_token_horizon_only_updates(self):
host = MixinHost()
host.init_omni_connectors(
vllm_config=None,
model_config=_make_model_config(stage_id=1, async_chunk=True, worker_type="ar"),
)
payload = {
@@ -1084,7 +1064,6 @@ class TestAsyncPayloadLifecycle(unittest.TestCase):
def test_payload_consumable_accepts_decode_embeddings(self):
host = MixinHost()
host.init_omni_connectors(
vllm_config=None,
model_config=_make_model_config(stage_id=1, async_chunk=True, worker_type="ar"),
)
payload = {
@@ -1098,7 +1077,6 @@ class TestAsyncPayloadLifecycle(unittest.TestCase):
def test_ar_metadata_only_followup_chunk_does_not_rewake_request(self):
host = MixinHost()
host.init_omni_connectors(
vllm_config=None,
model_config=_make_model_config(stage_id=1, async_chunk=True, worker_type="ar"),
)
host._omni_connector = MagicMock()
@@ -1138,7 +1116,6 @@ class TestAsyncPayloadLifecycle(unittest.TestCase):
def test_non_ar_recv_does_not_overwrite_unconsumed_staged_chunk(self):
host = MixinHost()
host.init_omni_connectors(
vllm_config=None,
model_config=_make_model_config(stage_id=2, async_chunk=True, worker_type="gen"),
)
host._omni_connector = MagicMock()
@@ -1164,7 +1141,6 @@ class TestAsyncPayloadLifecycle(unittest.TestCase):
def test_non_ar_recv_waits_for_scheduler_handoff_before_fetching_next_chunk(self):
host = MixinHost()
host.init_omni_connectors(
vllm_config=None,
model_config=_make_model_config(stage_id=2, async_chunk=True, worker_type="gen"),
)
host._omni_connector = MagicMock()
@@ -1208,7 +1184,7 @@ class TestAsyncPayloadLifecycle(unittest.TestCase):
class TestRankAwareKVRouting(unittest.TestCase):
def _make_host(self, *, from_tp: int, to_tp: int, local_rank: int) -> MixinHost:
host = MixinHost()
host.init_omni_connectors(vllm_config=None, model_config=_make_model_config(stage_id=1))
host.init_omni_connectors(model_config=_make_model_config(stage_id=1))
host._from_tp = from_tp
host._to_tp = to_tp
host._local_rank = local_rank
@@ -1287,7 +1263,7 @@ class TestConnectorConfigValidation(unittest.TestCase):
model_config.stage_connector_config = {"name": " "}
with self.assertRaisesRegex(RuntimeError, "missing connector name"):
host.init_omni_connectors(vllm_config=None, model_config=model_config)
host.init_omni_connectors(model_config=model_config)
class _FailingConnector:
@@ -1319,7 +1295,6 @@ class TestSendRetry(unittest.TestCase):
def _make_sender(self, connector):
sender = MixinHost()
sender.init_omni_connectors(
vllm_config=None,
model_config=_make_model_config(stage_id=0, async_chunk=True),
)
sender._omni_connector = connector