[Core] Cleanup Omni connector runtime and transfer backends (#5096)
Signed-off-by: spencerr221 <liubingyu62@gmail.com>
This commit is contained in:
1 parent
2cf5f57b3e
commit
dcf8b07d39
53 files changed
+349
-448
No files matched your search
@@ -302,7 +302,6 @@ def test_from_pipeline_config_maps_orchestrator_cli_overrides():
|
||||
"omni_dp_size_local": 2,
|
||||
"omni_lb_policy": "round_robin",
|
||||
"omni_heartbeat_timeout": 9.5,
|
||||
"shm_threshold_bytes": 4096,
|
||||
"batch_timeout": 3,
|
||||
},
|
||||
)
|
||||
@@ -317,7 +316,6 @@ def test_from_pipeline_config_maps_orchestrator_cli_overrides():
|
||||
assert orchestrator_config.omni_dp_size_local == 2
|
||||
assert orchestrator_config.omni_lb_policy == "round_robin"
|
||||
assert orchestrator_config.omni_heartbeat_timeout == 9.5
|
||||
assert orchestrator_config.shm_threshold_bytes == 4096
|
||||
assert orchestrator_config.batch_timeout == 3
|
||||
|
||||
|
||||
|
||||
@@ -142,11 +142,10 @@ def test_recv_no_connector():
|
||||
def test_shm_connector_flow(mocker: MockerFixture):
|
||||
"""
|
||||
Verify the full flow: Send -> Adapter -> Connector -> Adapter -> Recv.
|
||||
Using real SharedMemoryConnector (inline mode for simplicity).
|
||||
Using real SharedMemoryConnector and key-addressed SHM metadata.
|
||||
"""
|
||||
# 1. Setup Connector
|
||||
config = {"shm_threshold_bytes": 1024, "inline_small_payloads": True}
|
||||
connector = SharedMemoryConnector(config)
|
||||
connector = SharedMemoryConnector({})
|
||||
connectors_map = {("0", "1"): connector}
|
||||
|
||||
# 2. Setup Data
|
||||
|
||||
@@ -60,10 +60,9 @@ def test_ndarray_serialization():
|
||||
|
||||
def test_create_shm_connector():
|
||||
"""Test creating SharedMemoryConnector via Factory."""
|
||||
spec = ConnectorSpec(name="SharedMemoryConnector", extra={"shm_threshold_bytes": 1024})
|
||||
spec = ConnectorSpec(name="SharedMemoryConnector")
|
||||
connector = OmniConnectorFactory.create_connector(spec)
|
||||
assert isinstance(connector, SharedMemoryConnector)
|
||||
assert connector.threshold == 1024
|
||||
|
||||
|
||||
def test_create_unknown_connector():
|
||||
@@ -75,21 +74,20 @@ def test_create_unknown_connector():
|
||||
|
||||
@pytest.fixture
|
||||
def shm_connector():
|
||||
config = {"shm_threshold_bytes": 100, "inline_small_payloads": True}
|
||||
return SharedMemoryConnector(config)
|
||||
connector = SharedMemoryConnector({})
|
||||
yield connector
|
||||
connector.close()
|
||||
|
||||
|
||||
def test_put_get_inline(shm_connector):
|
||||
"""Test inline transfer for small data."""
|
||||
def test_put_get_small_payload_uses_shm(shm_connector):
|
||||
"""Small payloads use key-addressed SHM; inline metadata is retired."""
|
||||
data = {"small": "data"}
|
||||
|
||||
success, size, metadata = shm_connector.put("stage_0", "stage_1", "req_1", data)
|
||||
assert success is True
|
||||
assert "inline_bytes" in metadata
|
||||
assert "shm" not in metadata
|
||||
assert "shm" in metadata
|
||||
assert "inline_bytes" not in metadata
|
||||
assert "size" in metadata
|
||||
assert shm_connector._metrics["inline_writes"] == 1
|
||||
assert shm_connector._metrics["shm_writes"] == 0
|
||||
|
||||
# Retrieve
|
||||
retrieved_data, ret_size = shm_connector.get("stage_0", "stage_1", "req_1", metadata)
|
||||
@@ -195,3 +193,42 @@ def test_mooncake_connector_defaults_missing_host_to_detected_ip(monkeypatch: py
|
||||
assert connector.get_connection_info()["host"] == "10.20.30.40"
|
||||
finally:
|
||||
connector.close()
|
||||
|
||||
|
||||
def test_mooncake_get_counts_zmq_timeout_separately(mocker: MockerFixture):
|
||||
import vllm_omni.distributed.omni_connectors.connectors.mooncake_transfer_engine_connector as mooncake_module
|
||||
|
||||
connector = object.__new__(mooncake_module.MooncakeTransferEngineConnector)
|
||||
connector._closed = False
|
||||
connector._metrics = {"errors": 0, "timeouts": 0}
|
||||
connector._resolve_metadata = mocker.Mock(
|
||||
return_value={"source_host": "127.0.0.1", "source_port": 1234, "data_size": 8}
|
||||
)
|
||||
recv_buffer = mocker.Mock()
|
||||
connector._alloc_recv_buffer = mocker.Mock(return_value=(recv_buffer, 1000))
|
||||
connector._request_transfer = mocker.Mock(side_effect=mooncake_module.zmq.Again())
|
||||
|
||||
assert connector.get("s0", "s1", "req") is None
|
||||
assert connector._metrics == {"errors": 0, "timeouts": 1}
|
||||
recv_buffer.release.assert_called_once()
|
||||
connector._closed = True
|
||||
|
||||
|
||||
def test_mooncake_get_counts_cuda_sync_failure_as_error(mocker: MockerFixture):
|
||||
import vllm_omni.distributed.omni_connectors.connectors.mooncake_transfer_engine_connector as mooncake_module
|
||||
|
||||
connector = object.__new__(mooncake_module.MooncakeTransferEngineConnector)
|
||||
connector._closed = False
|
||||
connector._metrics = {"errors": 0, "timeouts": 0}
|
||||
connector._resolve_metadata = mocker.Mock(
|
||||
return_value={"source_host": "127.0.0.1", "source_port": 1234, "data_size": 8}
|
||||
)
|
||||
recv_buffer = mocker.Mock()
|
||||
connector._alloc_recv_buffer = mocker.Mock(return_value=(recv_buffer, 1000))
|
||||
connector._request_transfer = mocker.Mock(return_value=mooncake_module.TRANS_DONE)
|
||||
connector._sync_receive_buffer_if_needed = mocker.Mock(side_effect=RuntimeError("CUDA sync failed"))
|
||||
|
||||
assert connector.get("s0", "s1", "req") is None
|
||||
assert connector._metrics == {"errors": 1, "timeouts": 0}
|
||||
recv_buffer.release.assert_called_once()
|
||||
connector._closed = True
|
||||
@@ -48,7 +48,6 @@ def test_load_qwen_yaml_configs(yaml_file):
|
||||
print(f"Testing config load: {yaml_file.name}")
|
||||
try:
|
||||
# Attempt to load the config
|
||||
# default_shm_threshold doesn't matter much for loading correctness, using default
|
||||
config = load_omni_transfer_config(yaml_file)
|
||||
|
||||
assert config is not None, "Config should not be None"
|
||||
|
||||
@@ -2,7 +2,10 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit tests for SharedMemoryConnector focusing on TP / CFG / metadata fallback."""
|
||||
|
||||
import os
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm_omni.distributed.omni_connectors.connectors.shm_connector import (
|
||||
SharedMemoryConnector,
|
||||
@@ -13,7 +16,7 @@ pytestmark = [pytest.mark.core_model, pytest.mark.cpu]
|
||||
|
||||
@pytest.fixture()
|
||||
def connector():
|
||||
c = SharedMemoryConnector({"shm_threshold_bytes": 0})
|
||||
c = SharedMemoryConnector({})
|
||||
yield c
|
||||
c.close()
|
||||
|
||||
@@ -36,6 +39,30 @@ class TestKeyBasedReadWrite:
|
||||
assert obj == data
|
||||
assert rsize == size
|
||||
assert "test_key_1" not in connector._pending_keys
|
||||
assert connector._metrics["gets"] == 1
|
||||
|
||||
def test_tensor_payload_removes_lock_file(self, connector):
|
||||
key = "tensor_payload"
|
||||
payload = torch.ones(2, 2)
|
||||
ok, _, metadata = connector.put("s0", "s1", key, payload)
|
||||
assert ok
|
||||
|
||||
result = connector.get("s0", "s1", key, metadata=metadata)
|
||||
|
||||
assert result is not None
|
||||
assert torch.equal(result[0], payload)
|
||||
assert not os.path.exists(f"/dev/shm/shm_{key}_lockfile.lock")
|
||||
|
||||
def test_falsey_payload_removes_lock_file(self, connector):
|
||||
key = "falsey_payload"
|
||||
ok, _, metadata = connector.put("s0", "s1", key, 0)
|
||||
assert ok
|
||||
|
||||
result = connector.get("s0", "s1", key, metadata=metadata)
|
||||
|
||||
assert result is not None
|
||||
assert result[0] == 0
|
||||
assert not os.path.exists(f"/dev/shm/shm_{key}_lockfile.lock")
|
||||
|
||||
def test_get_nonexistent_key_returns_none(self, connector):
|
||||
result = connector.get("s0", "s1", "no_such_key_xyz", metadata=None)
|
||||
|
||||
@@ -141,7 +141,6 @@ _DEPLOY_CONFIG = {
|
||||
"connectors": {
|
||||
"shared_memory_connector": {
|
||||
"name": "SharedMemoryConnector",
|
||||
"extra": {"shm_threshold_bytes": 65536},
|
||||
},
|
||||
},
|
||||
"stages": [
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
import asyncio
|
||||
import os
|
||||
import sys
|
||||
from contextlib import ExitStack
|
||||
|
||||
import pytest
|
||||
@@ -67,11 +66,9 @@ async def generate(
|
||||
@pytest.mark.asyncio
|
||||
async def test_abort():
|
||||
with ExitStack() as after:
|
||||
# Avoid SHM IPC in tests to prevent /dev/shm exhaustion and SIGBUS.
|
||||
engine = AsyncOmni(
|
||||
model=model,
|
||||
stage_configs_path=stage_config,
|
||||
shm_threshold_bytes=sys.maxsize,
|
||||
)
|
||||
after.callback(engine.shutdown)
|
||||
|
||||
|
||||
@@ -121,9 +121,7 @@ class _FakeStage:
|
||||
self._in_q = in_q
|
||||
self._out_q = out_q
|
||||
|
||||
def init_stage_worker(
|
||||
self, model: str, *, is_async=False, shm_threshold_bytes=65536, ctx=None, batch_timeout=10, **kwargs
|
||||
):
|
||||
def init_stage_worker(self, model: str, *, is_async=False, ctx=None, batch_timeout=10, **kwargs):
|
||||
self._proc = _ns(
|
||||
start=lambda: None,
|
||||
join=lambda timeout=None: None,
|
||||
|
||||
@@ -2333,7 +2333,6 @@ class OmniRunner:
|
||||
# production default in AsyncOmniEngine remains 600s; this only
|
||||
# affects the test runner wrapper.
|
||||
init_timeout: int = 1800,
|
||||
shm_threshold_bytes: int = 65536,
|
||||
log_stats: bool = False,
|
||||
stage_configs_path: str | None = None,
|
||||
**kwargs,
|
||||
@@ -2353,7 +2352,6 @@ class OmniRunner:
|
||||
stage_init_timeout=stage_init_timeout,
|
||||
batch_timeout=batch_timeout,
|
||||
init_timeout=init_timeout,
|
||||
shm_threshold_bytes=shm_threshold_bytes,
|
||||
stage_configs_path=stage_configs_path,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
@@ -110,7 +110,6 @@ class TestMixinAsyncChunkSendRecv(unittest.TestCase):
|
||||
|
||||
sender = MixinHost()
|
||||
sender.init_omni_connectors(
|
||||
vllm_config=None,
|
||||
model_config=_make_model_config(stage_id=0, async_chunk=True),
|
||||
)
|
||||
sender._omni_connector = connector
|
||||
@@ -147,7 +146,6 @@ class TestMixinAsyncChunkSendRecv(unittest.TestCase):
|
||||
|
||||
sender = MixinHost()
|
||||
sender.init_omni_connectors(
|
||||
vllm_config=None,
|
||||
model_config=_make_model_config(stage_id=0, async_chunk=True),
|
||||
)
|
||||
sender._omni_connector = connector
|
||||
@@ -180,7 +178,6 @@ class TestMixinKVCacheTransfer(unittest.TestCase):
|
||||
|
||||
host = MixinHost()
|
||||
host.init_omni_connectors(
|
||||
vllm_config=None,
|
||||
model_config=_make_model_config(),
|
||||
kv_transfer_manager=mock_kvm,
|
||||
)
|
||||
@@ -202,7 +199,6 @@ class TestMixinKVCacheTransfer(unittest.TestCase):
|
||||
|
||||
host = MixinHost()
|
||||
host.init_omni_connectors(
|
||||
vllm_config=None,
|
||||
model_config=_make_model_config(),
|
||||
kv_transfer_manager=mock_kvm,
|
||||
)
|
||||
@@ -219,7 +215,6 @@ class TestMixinKVCacheTransfer(unittest.TestCase):
|
||||
|
||||
host = MixinHost()
|
||||
host.init_omni_connectors(
|
||||
vllm_config=None,
|
||||
model_config=_make_model_config(),
|
||||
kv_transfer_manager=mock_kvm,
|
||||
)
|
||||
@@ -257,7 +252,6 @@ class TestMixinKVCacheTransfer(unittest.TestCase):
|
||||
|
||||
host = MixinHost()
|
||||
host.init_omni_connectors(
|
||||
vllm_config=None,
|
||||
model_config=_make_model_config(),
|
||||
kv_transfer_manager=mock_kvm,
|
||||
)
|
||||
@@ -281,7 +275,6 @@ class TestOmniConnectorOutput(unittest.TestCase):
|
||||
def test_output_aggregation(self):
|
||||
host = MixinHost()
|
||||
host.init_omni_connectors(
|
||||
vllm_config=None,
|
||||
model_config=_make_model_config(),
|
||||
)
|
||||
|
||||
@@ -310,7 +303,6 @@ class TestMixinNoConnector(unittest.TestCase):
|
||||
def test_no_connector(self):
|
||||
host = MixinHost()
|
||||
host.init_omni_connectors(
|
||||
vllm_config=None,
|
||||
model_config=_make_model_config(),
|
||||
)
|
||||
self.assertIsNone(host._omni_connector)
|
||||
@@ -336,7 +328,6 @@ class TestFinishedLoadReqsDrain(unittest.TestCase):
|
||||
def test_finished_load_reqs_flow_to_chunk_ready(self):
|
||||
host = MixinHost()
|
||||
host.init_omni_connectors(
|
||||
vllm_config=None,
|
||||
model_config=_make_model_config(),
|
||||
)
|
||||
|
||||
@@ -389,7 +380,6 @@ class TestFullPayloadSendWithCustomFunc(unittest.TestCase):
|
||||
|
||||
host = MixinHost()
|
||||
host.init_omni_connectors(
|
||||
vllm_config=None,
|
||||
model_config=_make_model_config(),
|
||||
)
|
||||
host._omni_connector = MockConnector(stage_id=0)
|
||||
@@ -424,7 +414,6 @@ class TestFullPayloadSendWithCustomFunc(unittest.TestCase):
|
||||
|
||||
host = MixinHost()
|
||||
host.init_omni_connectors(
|
||||
vllm_config=None,
|
||||
model_config=_make_model_config(),
|
||||
)
|
||||
host._omni_connector = MockConnector(stage_id=0)
|
||||
@@ -453,7 +442,6 @@ class TestKVSentReqIdsAccumulation(unittest.TestCase):
|
||||
|
||||
host = MixinHost()
|
||||
host.init_omni_connectors(
|
||||
vllm_config=None,
|
||||
model_config=_make_model_config(),
|
||||
kv_transfer_manager=mock_kvm,
|
||||
)
|
||||
@@ -488,7 +476,6 @@ class TestChunkStreamCompletedGuard(unittest.TestCase):
|
||||
def _make_host(self, stage_id: int = 1) -> MixinHost:
|
||||
host = MixinHost()
|
||||
host.init_omni_connectors(
|
||||
vllm_config=None,
|
||||
model_config=_make_model_config(stage_id=stage_id, async_chunk=True),
|
||||
)
|
||||
host._omni_connector = MockConnector(stage_id=stage_id)
|
||||
@@ -587,7 +574,6 @@ class TestCleanupFinishedRequest(unittest.TestCase):
|
||||
def _make_host(self, stage_id: int = 1) -> MixinHost:
|
||||
host = MixinHost()
|
||||
host.init_omni_connectors(
|
||||
vllm_config=None,
|
||||
model_config=_make_model_config(stage_id=stage_id, async_chunk=True),
|
||||
)
|
||||
host._omni_connector = MockConnector(stage_id=stage_id)
|
||||
@@ -771,7 +757,6 @@ class TestSendChunkCachesMapping(unittest.TestCase):
|
||||
"""send_chunk should cache the internal→external mapping."""
|
||||
host = MixinHost()
|
||||
host.init_omni_connectors(
|
||||
vllm_config=None,
|
||||
model_config=_make_model_config(stage_id=0, async_chunk=True),
|
||||
)
|
||||
host._omni_connector = MockConnector(stage_id=0)
|
||||
@@ -802,7 +787,6 @@ class TestLocalPayloadCacheLifecycle(unittest.TestCase):
|
||||
def _make_host(self) -> MixinHost:
|
||||
host = MixinHost()
|
||||
host.init_omni_connectors(
|
||||
vllm_config=None,
|
||||
model_config=_make_model_config(stage_id=0),
|
||||
)
|
||||
host._omni_connector = MockConnector(stage_id=0)
|
||||
@@ -901,7 +885,6 @@ class TestTPAsyncChunkFanout(unittest.TestCase):
|
||||
def _make_host(self, rank: int) -> MixinHost:
|
||||
host = MixinHost()
|
||||
host.init_omni_connectors(
|
||||
vllm_config=None,
|
||||
model_config=_make_model_config(stage_id=2, async_chunk=True, worker_type="gen"),
|
||||
)
|
||||
host._omni_connector = MagicMock()
|
||||
@@ -984,7 +967,6 @@ class TestKVTransferLifecycle(unittest.TestCase):
|
||||
def _make_host(self) -> MixinHost:
|
||||
host = MixinHost()
|
||||
host.init_omni_connectors(
|
||||
vllm_config=None,
|
||||
model_config=_make_model_config(stage_id=0),
|
||||
)
|
||||
return host
|
||||
@@ -1039,7 +1021,6 @@ class TestAsyncPayloadLifecycle(unittest.TestCase):
|
||||
def test_send_side_request_payload_not_cleared_before_payload_is_consumable(self):
|
||||
host = MixinHost()
|
||||
host.init_omni_connectors(
|
||||
vllm_config=None,
|
||||
model_config=_make_model_config(stage_id=1, async_chunk=True, worker_type="ar"),
|
||||
)
|
||||
host._request_ids_mapping["r1"] = "r1"
|
||||
@@ -1063,7 +1044,6 @@ class TestAsyncPayloadLifecycle(unittest.TestCase):
|
||||
def test_payload_consumable_ignores_token_horizon_only_updates(self):
|
||||
host = MixinHost()
|
||||
host.init_omni_connectors(
|
||||
vllm_config=None,
|
||||
model_config=_make_model_config(stage_id=1, async_chunk=True, worker_type="ar"),
|
||||
)
|
||||
payload = {
|
||||
@@ -1084,7 +1064,6 @@ class TestAsyncPayloadLifecycle(unittest.TestCase):
|
||||
def test_payload_consumable_accepts_decode_embeddings(self):
|
||||
host = MixinHost()
|
||||
host.init_omni_connectors(
|
||||
vllm_config=None,
|
||||
model_config=_make_model_config(stage_id=1, async_chunk=True, worker_type="ar"),
|
||||
)
|
||||
payload = {
|
||||
@@ -1098,7 +1077,6 @@ class TestAsyncPayloadLifecycle(unittest.TestCase):
|
||||
def test_ar_metadata_only_followup_chunk_does_not_rewake_request(self):
|
||||
host = MixinHost()
|
||||
host.init_omni_connectors(
|
||||
vllm_config=None,
|
||||
model_config=_make_model_config(stage_id=1, async_chunk=True, worker_type="ar"),
|
||||
)
|
||||
host._omni_connector = MagicMock()
|
||||
@@ -1138,7 +1116,6 @@ class TestAsyncPayloadLifecycle(unittest.TestCase):
|
||||
def test_non_ar_recv_does_not_overwrite_unconsumed_staged_chunk(self):
|
||||
host = MixinHost()
|
||||
host.init_omni_connectors(
|
||||
vllm_config=None,
|
||||
model_config=_make_model_config(stage_id=2, async_chunk=True, worker_type="gen"),
|
||||
)
|
||||
host._omni_connector = MagicMock()
|
||||
@@ -1164,7 +1141,6 @@ class TestAsyncPayloadLifecycle(unittest.TestCase):
|
||||
def test_non_ar_recv_waits_for_scheduler_handoff_before_fetching_next_chunk(self):
|
||||
host = MixinHost()
|
||||
host.init_omni_connectors(
|
||||
vllm_config=None,
|
||||
model_config=_make_model_config(stage_id=2, async_chunk=True, worker_type="gen"),
|
||||
)
|
||||
host._omni_connector = MagicMock()
|
||||
@@ -1208,7 +1184,7 @@ class TestAsyncPayloadLifecycle(unittest.TestCase):
|
||||
class TestRankAwareKVRouting(unittest.TestCase):
|
||||
def _make_host(self, *, from_tp: int, to_tp: int, local_rank: int) -> MixinHost:
|
||||
host = MixinHost()
|
||||
host.init_omni_connectors(vllm_config=None, model_config=_make_model_config(stage_id=1))
|
||||
host.init_omni_connectors(model_config=_make_model_config(stage_id=1))
|
||||
host._from_tp = from_tp
|
||||
host._to_tp = to_tp
|
||||
host._local_rank = local_rank
|
||||
@@ -1287,7 +1263,7 @@ class TestConnectorConfigValidation(unittest.TestCase):
|
||||
model_config.stage_connector_config = {"name": " "}
|
||||
|
||||
with self.assertRaisesRegex(RuntimeError, "missing connector name"):
|
||||
host.init_omni_connectors(vllm_config=None, model_config=model_config)
|
||||
host.init_omni_connectors(model_config=model_config)
|
||||
|
||||
|
||||
class _FailingConnector:
|
||||
@@ -1319,7 +1295,6 @@ class TestSendRetry(unittest.TestCase):
|
||||
def _make_sender(self, connector):
|
||||
sender = MixinHost()
|
||||
sender.init_omni_connectors(
|
||||
vllm_config=None,
|
||||
model_config=_make_model_config(stage_id=0, async_chunk=True),
|
||||
)
|
||||
sender._omni_connector = connector
|
||||
|
||||
Reference in new issue
Block a user