[Feature] Omni Connector + ray supported (#215)
Signed-off-by: wzliu <wzliu@connect.hku.hk>
This commit is contained in:
1 parent
cafe52e8ec
commit
f995211da0
34 files changed
+2673
-201
No files matched your search
@@ -0,0 +1,236 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm_omni.distributed.omni_connectors.adapter import try_recv_via_connector, try_send_via_connector
|
||||
from vllm_omni.distributed.omni_connectors.connectors.shm_connector import SharedMemoryConnector
|
||||
from vllm_omni.distributed.omni_connectors.utils.config import ConnectorSpec, OmniTransferConfig
|
||||
from vllm_omni.distributed.omni_connectors.utils.initialization import get_connectors_config_for_stage
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_objects():
|
||||
return {"connector": MagicMock(), "metrics": MagicMock(), "queue_fn": MagicMock()}
|
||||
|
||||
|
||||
def test_send_success(mock_objects):
|
||||
"""Test try_send_via_connector success path."""
|
||||
# Setup
|
||||
mock_connector = mock_objects["connector"]
|
||||
mock_metrics = mock_objects["metrics"]
|
||||
mock_queue_fn = mock_objects["queue_fn"]
|
||||
|
||||
stage_id = 0
|
||||
next_stage_id = 1
|
||||
req_id = "req_123"
|
||||
inputs = {"input_ids": [1, 2, 3]}
|
||||
sampling_params = {"temperature": 0.7}
|
||||
prompt = "test prompt"
|
||||
|
||||
# Mock connector.put return
|
||||
# Returns: (success, size, metadata)
|
||||
mock_metadata = {"handle": "xyz"}
|
||||
mock_connector.put.return_value = (True, 100, mock_metadata)
|
||||
|
||||
# Execute
|
||||
result = try_send_via_connector(
|
||||
connector=mock_connector,
|
||||
stage_id=stage_id,
|
||||
next_stage_id=next_stage_id,
|
||||
req_id=req_id,
|
||||
next_inputs=inputs,
|
||||
sampling_params=sampling_params,
|
||||
original_prompt=prompt,
|
||||
next_stage_queue_submit_fn=mock_queue_fn,
|
||||
metrics=mock_metrics,
|
||||
)
|
||||
|
||||
# Verify
|
||||
assert result is True
|
||||
|
||||
# 1. Verify connector.put called correctly
|
||||
mock_connector.put.assert_called_once()
|
||||
args, _ = mock_connector.put.call_args
|
||||
assert args[0] == "0" # from_stage
|
||||
assert args[1] == "1" # to_stage
|
||||
assert args[2] == req_id
|
||||
# Verify payload structure in put
|
||||
payload = args[3]
|
||||
assert payload["engine_inputs"] == inputs
|
||||
assert payload["sampling_params"] == sampling_params
|
||||
|
||||
# 2. Verify queue notification submitted
|
||||
mock_queue_fn.assert_called_once()
|
||||
notify_payload = mock_queue_fn.call_args[0][0]
|
||||
assert notify_payload["request_id"] == req_id
|
||||
assert notify_payload["from_connector"] is True
|
||||
assert notify_payload["connector_metadata"] == mock_metadata
|
||||
|
||||
# 3. Verify metrics recorded
|
||||
mock_metrics.on_forward.assert_called_once()
|
||||
|
||||
|
||||
def test_send_fail(mock_objects):
|
||||
"""Test try_send_via_connector when connector fails."""
|
||||
mock_connector = mock_objects["connector"]
|
||||
mock_metrics = mock_objects["metrics"]
|
||||
mock_queue_fn = mock_objects["queue_fn"]
|
||||
|
||||
mock_connector.put.return_value = (False, 0, None)
|
||||
|
||||
result = try_send_via_connector(
|
||||
connector=mock_connector,
|
||||
stage_id=0,
|
||||
next_stage_id=1,
|
||||
req_id="req_fail",
|
||||
next_inputs={},
|
||||
sampling_params={},
|
||||
original_prompt="",
|
||||
next_stage_queue_submit_fn=mock_queue_fn,
|
||||
metrics=mock_metrics,
|
||||
)
|
||||
|
||||
assert result is False
|
||||
mock_queue_fn.assert_not_called()
|
||||
|
||||
|
||||
def test_recv_success(mock_objects):
|
||||
"""Test try_recv_via_connector success path."""
|
||||
mock_connector = mock_objects["connector"]
|
||||
|
||||
# Setup task received from queue
|
||||
task = {
|
||||
"request_id": "req_recv",
|
||||
"from_connector": True,
|
||||
"from_stage": "0",
|
||||
"connector_metadata": {"handle": "xyz"},
|
||||
}
|
||||
|
||||
# Setup connectors dict
|
||||
connectors = {("0", "1"): mock_connector}
|
||||
|
||||
# Mock connector.get return
|
||||
expected_data = {"engine_inputs": {"ids": [1]}}
|
||||
# get returns: (data_obj, size)
|
||||
mock_connector.get.return_value = (expected_data, 50)
|
||||
# serialize_obj needed for metrics calculation if size not returned directly
|
||||
mock_connector.serialize_obj.return_value = b"bytes"
|
||||
|
||||
# Execute
|
||||
# We are stage 1 receiving from stage 0
|
||||
inputs, rx_metrics = try_recv_via_connector(task, connectors, stage_id=1)
|
||||
|
||||
# Verify
|
||||
assert inputs == expected_data["engine_inputs"]
|
||||
assert rx_metrics is not None
|
||||
mock_connector.get.assert_called_once_with("0", "1", "req_recv", metadata={"handle": "xyz"})
|
||||
|
||||
|
||||
def test_recv_no_connector():
|
||||
"""Test recv fails when no connector exists for edge."""
|
||||
task = {"request_id": "req_missing", "from_connector": True, "from_stage": "0"}
|
||||
connectors = {} # Empty connectors
|
||||
|
||||
inputs, _ = try_recv_via_connector(task, connectors, stage_id=1)
|
||||
assert inputs is None
|
||||
|
||||
|
||||
def test_shm_connector_flow():
|
||||
"""
|
||||
Verify the full flow: Send -> Adapter -> Connector -> Adapter -> Recv.
|
||||
Using real SharedMemoryConnector (inline mode for simplicity).
|
||||
"""
|
||||
# 1. Setup Connector
|
||||
config = {"shm_threshold_bytes": 1024} # Large threshold to use inline
|
||||
connector = SharedMemoryConnector(config)
|
||||
connectors_map = {("0", "1"): connector}
|
||||
|
||||
# 2. Setup Data
|
||||
stage_id = 0
|
||||
next_stage_id = 1
|
||||
req_id = "flow_req"
|
||||
inputs = {"tokens": [10, 20, 30]}
|
||||
sampling_params = {"n": 1}
|
||||
|
||||
# Queue capture mechanism
|
||||
queue_capture = []
|
||||
|
||||
def mock_submit(payload):
|
||||
queue_capture.append(payload)
|
||||
|
||||
mock_metrics = MagicMock()
|
||||
|
||||
# 3. Send
|
||||
success = try_send_via_connector(
|
||||
connector=connector,
|
||||
stage_id=stage_id,
|
||||
next_stage_id=next_stage_id,
|
||||
req_id=req_id,
|
||||
next_inputs=inputs,
|
||||
sampling_params=sampling_params,
|
||||
original_prompt="prompt",
|
||||
next_stage_queue_submit_fn=mock_submit,
|
||||
metrics=mock_metrics,
|
||||
)
|
||||
assert success is True
|
||||
assert len(queue_capture) == 1
|
||||
|
||||
# 4. Recv
|
||||
# The 'task' is what would be popped from the queue
|
||||
received_task = queue_capture[0]
|
||||
|
||||
# Verify queue payload contains what we expect
|
||||
assert received_task["from_connector"] is True
|
||||
assert received_task["from_stage"] == "0"
|
||||
|
||||
# Decode
|
||||
decoded_inputs, _ = try_recv_via_connector(received_task, connectors_map, stage_id=1)
|
||||
|
||||
# 5. Verify Data Integrity
|
||||
assert decoded_inputs == inputs
|
||||
|
||||
|
||||
def test_get_connectors_for_stage():
|
||||
"""Test filtering logic for stage config."""
|
||||
# Config has edges: 0->1, 1->2
|
||||
config = OmniTransferConfig(connectors={("0", "1"): ConnectorSpec(name="C1"), ("1", "2"): ConnectorSpec(name="C2")})
|
||||
|
||||
# Get config for Stage 1
|
||||
# Stage 1 receives from 0 (input) and sends to 2 (output)
|
||||
# get_connectors_config_for_stage ONLY returns INPUT connectors for the worker to initialize
|
||||
|
||||
stage_config = get_connectors_config_for_stage(config, stage_id=1)
|
||||
|
||||
# Should contain "from_stage_0"
|
||||
assert "from_stage_0" in stage_config
|
||||
assert stage_config["from_stage_0"]["spec"]["name"] == "C1"
|
||||
|
||||
# Should NOT contain "from_stage_1" or related to output
|
||||
assert "from_stage_1" not in stage_config
|
||||
|
||||
# Verify Stage 2
|
||||
stage_2_config = get_connectors_config_for_stage(config, stage_id=2)
|
||||
assert "from_stage_1" in stage_2_config
|
||||
assert stage_2_config["from_stage_1"]["spec"]["name"] == "C2"
|
||||
|
||||
|
||||
def test_recv_with_missing_metadata():
|
||||
"""Test recv when queue payload is malformed (missing metadata)."""
|
||||
# Connector expects metadata but task doesn't have it
|
||||
task = {
|
||||
"request_id": "req_bad",
|
||||
"from_connector": True,
|
||||
"from_stage": "0",
|
||||
# Missing "connector_metadata"
|
||||
}
|
||||
mock_conn = MagicMock()
|
||||
# If get is called with None metadata, connector usually handles it or adapter handles exception
|
||||
mock_conn.get.side_effect = Exception("Get failed")
|
||||
|
||||
connectors = {("0", "1"): mock_conn}
|
||||
|
||||
inputs, _ = try_recv_via_connector(task, connectors, stage_id=1)
|
||||
assert inputs is None
|
||||
@@ -0,0 +1,99 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm_omni.distributed.omni_connectors.connectors.shm_connector import SharedMemoryConnector
|
||||
from vllm_omni.distributed.omni_connectors.factory import OmniConnectorFactory
|
||||
from vllm_omni.distributed.omni_connectors.utils.config import ConnectorSpec
|
||||
from vllm_omni.distributed.omni_connectors.utils.serialization import OmniSerializer
|
||||
|
||||
|
||||
def test_pickle_serialization():
|
||||
"""Test basic pickle serialization."""
|
||||
data = {"key": "value", "list": [1, 2, 3]}
|
||||
serialized = OmniSerializer.serialize(data, method="cloudpickle")
|
||||
assert isinstance(serialized, bytes)
|
||||
|
||||
deserialized = OmniSerializer.deserialize(serialized, method="cloudpickle")
|
||||
assert data == deserialized
|
||||
|
||||
|
||||
def test_create_shm_connector():
|
||||
"""Test creating SharedMemoryConnector via Factory."""
|
||||
spec = ConnectorSpec(name="SharedMemoryConnector", extra={"shm_threshold_bytes": 1024})
|
||||
connector = OmniConnectorFactory.create_connector(spec)
|
||||
assert isinstance(connector, SharedMemoryConnector)
|
||||
assert connector.threshold == 1024
|
||||
|
||||
|
||||
def test_create_unknown_connector():
|
||||
"""Test error when creating unknown connector."""
|
||||
spec = ConnectorSpec(name="UnknownConnector")
|
||||
with pytest.raises(ValueError):
|
||||
OmniConnectorFactory.create_connector(spec)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def shm_connector():
|
||||
config = {"shm_threshold_bytes": 100} # Small threshold for testing
|
||||
return SharedMemoryConnector(config)
|
||||
|
||||
|
||||
def test_put_get_inline(shm_connector):
|
||||
"""Test inline transfer for small data."""
|
||||
data = {"small": "data"}
|
||||
# Ensure data is smaller than threshold (100 bytes)
|
||||
|
||||
success, size, metadata = shm_connector.put("stage_0", "stage_1", "req_1", data)
|
||||
assert success is True
|
||||
assert "inline_bytes" in metadata
|
||||
assert "shm" not in metadata
|
||||
|
||||
# Retrieve
|
||||
retrieved_data, ret_size = shm_connector.get("stage_0", "stage_1", "req_1", metadata)
|
||||
assert data == retrieved_data
|
||||
assert size == ret_size
|
||||
|
||||
|
||||
def test_put_get_shm(shm_connector, monkeypatch):
|
||||
"""Test SHM transfer logic for large data (Mocked)."""
|
||||
# Create data larger than 100 bytes
|
||||
data = {"large": "x" * 200}
|
||||
|
||||
# Mock SHM return values
|
||||
mock_handle = {"name": "test_shm", "size": 200}
|
||||
mock_write = MagicMock(return_value=mock_handle)
|
||||
monkeypatch.setattr("vllm_omni.distributed.omni_connectors.connectors.shm_connector.shm_write_bytes", mock_write)
|
||||
|
||||
# When reading, return the serialized bytes of the data
|
||||
serialized_data = shm_connector.serialize_obj(data)
|
||||
mock_read = MagicMock(return_value=serialized_data)
|
||||
monkeypatch.setattr("vllm_omni.distributed.omni_connectors.connectors.shm_connector.shm_read_bytes", mock_read)
|
||||
|
||||
# Put
|
||||
success, size, metadata = shm_connector.put("stage_0", "stage_1", "req_2", data)
|
||||
|
||||
assert success is True
|
||||
# Should use SHM because data > threshold
|
||||
assert "shm" in metadata
|
||||
assert metadata["shm"] == mock_handle
|
||||
assert "inline_bytes" not in metadata
|
||||
|
||||
mock_write.assert_called_once()
|
||||
|
||||
# Get
|
||||
retrieved_data, ret_size = shm_connector.get("stage_0", "stage_1", "req_2", metadata)
|
||||
|
||||
assert data == retrieved_data
|
||||
mock_read.assert_called_once_with(mock_handle)
|
||||
|
||||
|
||||
def test_get_invalid_metadata(shm_connector):
|
||||
"""Test get with invalid metadata."""
|
||||
result = shm_connector.get("stage_0", "stage_1", "req_3", {})
|
||||
assert result is None
|
||||
|
||||
result = shm_connector.get("stage_0", "stage_1", "req_3", {"unknown": "format"})
|
||||
assert result is None
|
||||
@@ -0,0 +1,64 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
# Use the new import path for initialization utilities
|
||||
from vllm_omni.distributed.omni_connectors.utils.initialization import load_omni_transfer_config
|
||||
|
||||
|
||||
def get_config_files():
|
||||
"""Helper to find config files."""
|
||||
# Go up two levels from 'tests/distributed/omni_connectors' (approx) to 'vllm-omni' root
|
||||
# Adjust based on file location: vllm-omni/tests/distributed/omni_connectors/test_omni_connector_configs.py
|
||||
# This file is 4 levels deep from root if we count from tests?
|
||||
# vllm-omni/tests/distributed/omni_connectors -> parent -> distributed -> parent -> tests -> parent -> vllm-omni
|
||||
# Let's use resolve to be safe.
|
||||
|
||||
# Path(__file__) = .../vllm-omni/tests/distributed/omni_connectors/test_omni_connector_configs.py
|
||||
# .parent = omni_connectors
|
||||
# .parent = distributed
|
||||
# .parent = tests
|
||||
# .parent = vllm-omni
|
||||
|
||||
base_dir = Path(__file__).resolve().parent.parent.parent.parent
|
||||
config_dir = base_dir / "vllm_omni" / "model_executor" / "stage_configs"
|
||||
|
||||
if not config_dir.exists():
|
||||
return []
|
||||
|
||||
return list(config_dir.glob("qwen*.yaml"))
|
||||
|
||||
|
||||
# Collect files at module level for parametrization
|
||||
config_files = get_config_files()
|
||||
|
||||
|
||||
@pytest.mark.skipif(len(config_files) == 0, reason="No config files found or directory missing")
|
||||
@pytest.mark.parametrize("yaml_file", config_files, ids=lambda p: p.name)
|
||||
def test_load_qwen_yaml_configs(yaml_file):
|
||||
"""
|
||||
Scan and test loading of all qwen*.yaml config files.
|
||||
This ensures that existing stage configs are compatible with the OmniConnector system.
|
||||
"""
|
||||
print(f"Testing config load: {yaml_file.name}")
|
||||
try:
|
||||
# Attempt to load the config
|
||||
# default_shm_threshold doesn't matter much for loading correctness, using default
|
||||
config = load_omni_transfer_config(yaml_file)
|
||||
|
||||
assert config is not None, "Config should not be None"
|
||||
|
||||
# Basic validation
|
||||
# Note: Some configs might not have 'runtime' or 'connectors' section if they rely on auto-shm
|
||||
# but the load function should succeed regardless.
|
||||
|
||||
# If the config defines stages, we expect connectors to be populated (either explicit or auto SHM)
|
||||
# We can't strictly assert len(config.connectors) > 0 because a single stage pipeline might have 0 edges.
|
||||
|
||||
print(f" -> Successfully loaded. Connectors: {len(config.connectors)}")
|
||||
|
||||
except Exception as e:
|
||||
pytest.fail(f"Failed to load config {yaml_file.name}: {e}")
|
||||
Reference in new issue
Block a user