[Feature] Omni Connector + ray supported (#215)

Signed-off-by: wzliu <wzliu@connect.hku.hk>
This commit is contained in:
NATURE authored and GitHub committed 2025-12-10 20:15:03 +08:00
1 parent cafe52e8ec
commit f995211da0
34 files changed
+2673 -201

No files matched your search

@@ -0,0 +1,236 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from unittest.mock import MagicMock
import pytest
from vllm_omni.distributed.omni_connectors.adapter import try_recv_via_connector, try_send_via_connector
from vllm_omni.distributed.omni_connectors.connectors.shm_connector import SharedMemoryConnector
from vllm_omni.distributed.omni_connectors.utils.config import ConnectorSpec, OmniTransferConfig
from vllm_omni.distributed.omni_connectors.utils.initialization import get_connectors_config_for_stage
@pytest.fixture
def mock_objects():
return {"connector": MagicMock(), "metrics": MagicMock(), "queue_fn": MagicMock()}
def test_send_success(mock_objects):
"""Test try_send_via_connector success path."""
# Setup
mock_connector = mock_objects["connector"]
mock_metrics = mock_objects["metrics"]
mock_queue_fn = mock_objects["queue_fn"]
stage_id = 0
next_stage_id = 1
req_id = "req_123"
inputs = {"input_ids": [1, 2, 3]}
sampling_params = {"temperature": 0.7}
prompt = "test prompt"
# Mock connector.put return
# Returns: (success, size, metadata)
mock_metadata = {"handle": "xyz"}
mock_connector.put.return_value = (True, 100, mock_metadata)
# Execute
result = try_send_via_connector(
connector=mock_connector,
stage_id=stage_id,
next_stage_id=next_stage_id,
req_id=req_id,
next_inputs=inputs,
sampling_params=sampling_params,
original_prompt=prompt,
next_stage_queue_submit_fn=mock_queue_fn,
metrics=mock_metrics,
)
# Verify
assert result is True
# 1. Verify connector.put called correctly
mock_connector.put.assert_called_once()
args, _ = mock_connector.put.call_args
assert args[0] == "0" # from_stage
assert args[1] == "1" # to_stage
assert args[2] == req_id
# Verify payload structure in put
payload = args[3]
assert payload["engine_inputs"] == inputs
assert payload["sampling_params"] == sampling_params
# 2. Verify queue notification submitted
mock_queue_fn.assert_called_once()
notify_payload = mock_queue_fn.call_args[0][0]
assert notify_payload["request_id"] == req_id
assert notify_payload["from_connector"] is True
assert notify_payload["connector_metadata"] == mock_metadata
# 3. Verify metrics recorded
mock_metrics.on_forward.assert_called_once()
def test_send_fail(mock_objects):
"""Test try_send_via_connector when connector fails."""
mock_connector = mock_objects["connector"]
mock_metrics = mock_objects["metrics"]
mock_queue_fn = mock_objects["queue_fn"]
mock_connector.put.return_value = (False, 0, None)
result = try_send_via_connector(
connector=mock_connector,
stage_id=0,
next_stage_id=1,
req_id="req_fail",
next_inputs={},
sampling_params={},
original_prompt="",
next_stage_queue_submit_fn=mock_queue_fn,
metrics=mock_metrics,
)
assert result is False
mock_queue_fn.assert_not_called()
def test_recv_success(mock_objects):
"""Test try_recv_via_connector success path."""
mock_connector = mock_objects["connector"]
# Setup task received from queue
task = {
"request_id": "req_recv",
"from_connector": True,
"from_stage": "0",
"connector_metadata": {"handle": "xyz"},
}
# Setup connectors dict
connectors = {("0", "1"): mock_connector}
# Mock connector.get return
expected_data = {"engine_inputs": {"ids": [1]}}
# get returns: (data_obj, size)
mock_connector.get.return_value = (expected_data, 50)
# serialize_obj needed for metrics calculation if size not returned directly
mock_connector.serialize_obj.return_value = b"bytes"
# Execute
# We are stage 1 receiving from stage 0
inputs, rx_metrics = try_recv_via_connector(task, connectors, stage_id=1)
# Verify
assert inputs == expected_data["engine_inputs"]
assert rx_metrics is not None
mock_connector.get.assert_called_once_with("0", "1", "req_recv", metadata={"handle": "xyz"})
def test_recv_no_connector():
"""Test recv fails when no connector exists for edge."""
task = {"request_id": "req_missing", "from_connector": True, "from_stage": "0"}
connectors = {} # Empty connectors
inputs, _ = try_recv_via_connector(task, connectors, stage_id=1)
assert inputs is None
def test_shm_connector_flow():
"""
Verify the full flow: Send -> Adapter -> Connector -> Adapter -> Recv.
Using real SharedMemoryConnector (inline mode for simplicity).
"""
# 1. Setup Connector
config = {"shm_threshold_bytes": 1024} # Large threshold to use inline
connector = SharedMemoryConnector(config)
connectors_map = {("0", "1"): connector}
# 2. Setup Data
stage_id = 0
next_stage_id = 1
req_id = "flow_req"
inputs = {"tokens": [10, 20, 30]}
sampling_params = {"n": 1}
# Queue capture mechanism
queue_capture = []
def mock_submit(payload):
queue_capture.append(payload)
mock_metrics = MagicMock()
# 3. Send
success = try_send_via_connector(
connector=connector,
stage_id=stage_id,
next_stage_id=next_stage_id,
req_id=req_id,
next_inputs=inputs,
sampling_params=sampling_params,
original_prompt="prompt",
next_stage_queue_submit_fn=mock_submit,
metrics=mock_metrics,
)
assert success is True
assert len(queue_capture) == 1
# 4. Recv
# The 'task' is what would be popped from the queue
received_task = queue_capture[0]
# Verify queue payload contains what we expect
assert received_task["from_connector"] is True
assert received_task["from_stage"] == "0"
# Decode
decoded_inputs, _ = try_recv_via_connector(received_task, connectors_map, stage_id=1)
# 5. Verify Data Integrity
assert decoded_inputs == inputs
def test_get_connectors_for_stage():
"""Test filtering logic for stage config."""
# Config has edges: 0->1, 1->2
config = OmniTransferConfig(connectors={("0", "1"): ConnectorSpec(name="C1"), ("1", "2"): ConnectorSpec(name="C2")})
# Get config for Stage 1
# Stage 1 receives from 0 (input) and sends to 2 (output)
# get_connectors_config_for_stage ONLY returns INPUT connectors for the worker to initialize
stage_config = get_connectors_config_for_stage(config, stage_id=1)
# Should contain "from_stage_0"
assert "from_stage_0" in stage_config
assert stage_config["from_stage_0"]["spec"]["name"] == "C1"
# Should NOT contain "from_stage_1" or related to output
assert "from_stage_1" not in stage_config
# Verify Stage 2
stage_2_config = get_connectors_config_for_stage(config, stage_id=2)
assert "from_stage_1" in stage_2_config
assert stage_2_config["from_stage_1"]["spec"]["name"] == "C2"
def test_recv_with_missing_metadata():
"""Test recv when queue payload is malformed (missing metadata)."""
# Connector expects metadata but task doesn't have it
task = {
"request_id": "req_bad",
"from_connector": True,
"from_stage": "0",
# Missing "connector_metadata"
}
mock_conn = MagicMock()
# If get is called with None metadata, connector usually handles it or adapter handles exception
mock_conn.get.side_effect = Exception("Get failed")
connectors = {("0", "1"): mock_conn}
inputs, _ = try_recv_via_connector(task, connectors, stage_id=1)
assert inputs is None
@@ -0,0 +1,99 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from unittest.mock import MagicMock
import pytest
from vllm_omni.distributed.omni_connectors.connectors.shm_connector import SharedMemoryConnector
from vllm_omni.distributed.omni_connectors.factory import OmniConnectorFactory
from vllm_omni.distributed.omni_connectors.utils.config import ConnectorSpec
from vllm_omni.distributed.omni_connectors.utils.serialization import OmniSerializer
def test_pickle_serialization():
"""Test basic pickle serialization."""
data = {"key": "value", "list": [1, 2, 3]}
serialized = OmniSerializer.serialize(data, method="cloudpickle")
assert isinstance(serialized, bytes)
deserialized = OmniSerializer.deserialize(serialized, method="cloudpickle")
assert data == deserialized
def test_create_shm_connector():
"""Test creating SharedMemoryConnector via Factory."""
spec = ConnectorSpec(name="SharedMemoryConnector", extra={"shm_threshold_bytes": 1024})
connector = OmniConnectorFactory.create_connector(spec)
assert isinstance(connector, SharedMemoryConnector)
assert connector.threshold == 1024
def test_create_unknown_connector():
"""Test error when creating unknown connector."""
spec = ConnectorSpec(name="UnknownConnector")
with pytest.raises(ValueError):
OmniConnectorFactory.create_connector(spec)
@pytest.fixture
def shm_connector():
config = {"shm_threshold_bytes": 100} # Small threshold for testing
return SharedMemoryConnector(config)
def test_put_get_inline(shm_connector):
"""Test inline transfer for small data."""
data = {"small": "data"}
# Ensure data is smaller than threshold (100 bytes)
success, size, metadata = shm_connector.put("stage_0", "stage_1", "req_1", data)
assert success is True
assert "inline_bytes" in metadata
assert "shm" not in metadata
# Retrieve
retrieved_data, ret_size = shm_connector.get("stage_0", "stage_1", "req_1", metadata)
assert data == retrieved_data
assert size == ret_size
def test_put_get_shm(shm_connector, monkeypatch):
"""Test SHM transfer logic for large data (Mocked)."""
# Create data larger than 100 bytes
data = {"large": "x" * 200}
# Mock SHM return values
mock_handle = {"name": "test_shm", "size": 200}
mock_write = MagicMock(return_value=mock_handle)
monkeypatch.setattr("vllm_omni.distributed.omni_connectors.connectors.shm_connector.shm_write_bytes", mock_write)
# When reading, return the serialized bytes of the data
serialized_data = shm_connector.serialize_obj(data)
mock_read = MagicMock(return_value=serialized_data)
monkeypatch.setattr("vllm_omni.distributed.omni_connectors.connectors.shm_connector.shm_read_bytes", mock_read)
# Put
success, size, metadata = shm_connector.put("stage_0", "stage_1", "req_2", data)
assert success is True
# Should use SHM because data > threshold
assert "shm" in metadata
assert metadata["shm"] == mock_handle
assert "inline_bytes" not in metadata
mock_write.assert_called_once()
# Get
retrieved_data, ret_size = shm_connector.get("stage_0", "stage_1", "req_2", metadata)
assert data == retrieved_data
mock_read.assert_called_once_with(mock_handle)
def test_get_invalid_metadata(shm_connector):
"""Test get with invalid metadata."""
result = shm_connector.get("stage_0", "stage_1", "req_3", {})
assert result is None
result = shm_connector.get("stage_0", "stage_1", "req_3", {"unknown": "format"})
assert result is None
@@ -0,0 +1,64 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from pathlib import Path
import pytest
# Use the new import path for initialization utilities
from vllm_omni.distributed.omni_connectors.utils.initialization import load_omni_transfer_config
def get_config_files():
"""Helper to find config files."""
# Go up two levels from 'tests/distributed/omni_connectors' (approx) to 'vllm-omni' root
# Adjust based on file location: vllm-omni/tests/distributed/omni_connectors/test_omni_connector_configs.py
# This file is 4 levels deep from root if we count from tests?
# vllm-omni/tests/distributed/omni_connectors -> parent -> distributed -> parent -> tests -> parent -> vllm-omni
# Let's use resolve to be safe.
# Path(__file__) = .../vllm-omni/tests/distributed/omni_connectors/test_omni_connector_configs.py
# .parent = omni_connectors
# .parent = distributed
# .parent = tests
# .parent = vllm-omni
base_dir = Path(__file__).resolve().parent.parent.parent.parent
config_dir = base_dir / "vllm_omni" / "model_executor" / "stage_configs"
if not config_dir.exists():
return []
return list(config_dir.glob("qwen*.yaml"))
# Collect files at module level for parametrization
config_files = get_config_files()
@pytest.mark.skipif(len(config_files) == 0, reason="No config files found or directory missing")
@pytest.mark.parametrize("yaml_file", config_files, ids=lambda p: p.name)
def test_load_qwen_yaml_configs(yaml_file):
"""
Scan and test loading of all qwen*.yaml config files.
This ensures that existing stage configs are compatible with the OmniConnector system.
"""
print(f"Testing config load: {yaml_file.name}")
try:
# Attempt to load the config
# default_shm_threshold doesn't matter much for loading correctness, using default
config = load_omni_transfer_config(yaml_file)
assert config is not None, "Config should not be None"
# Basic validation
# Note: Some configs might not have 'runtime' or 'connectors' section if they rely on auto-shm
# but the load function should succeed regardless.
# If the config defines stages, we expect connectors to be populated (either explicit or auto SHM)
# We can't strictly assert len(config.connectors) > 0 because a single stage pipeline might have 0 edges.
print(f" -> Successfully loaded. Connectors: {len(config.connectors)}")
except Exception as e:
pytest.fail(f"Failed to load config {yaml_file.name}: {e}")