[Core] Add NIXL omni connector (#6093)

Signed-off-by: yuanwu <yuan.wu@intel.com>
Signed-off-by: Yuan Wu <yuan.wu@intel.com>
This commit is contained in:
Yuan Wu authored and GitHub committed 2026-09-13 16:35:02 +08:00
1 parent 01a2f93256
commit 6753bbd18c
33 files changed
+3032 -66

No files matched your search

+2
View File
@@ -7,6 +7,7 @@ from .omni_connectors import (
MooncakeStoreConnector,
MooncakeTransferEngineConnector,
MoriTransferEngineConnector,
NixlConnector,
OmniConnectorBase,
OmniConnectorFactory,
OmniTransferConfig,
@@ -27,6 +28,7 @@ __all__ = [
"MooncakeStoreConnector",
"MooncakeTransferEngineConnector",
"MoriTransferEngineConnector",
"NixlConnector",
"SharedMemoryConnector",
"YuanrongConnector",
"YuanrongTransferEngineConnector",
@@ -22,6 +22,11 @@ try:
from .connectors.mori_transfer_engine_connector import MoriTransferEngineConnector
except ImportError:
MoriTransferEngineConnector = None # RDMA deps (msgspec/zmq/mori) not installed
try:
from .connectors.nixl_connector import NixlConnector
except ImportError:
NixlConnector = None # NIXL deps not installed
from .factory import OmniConnectorFactory
from .utils.config import ConnectorSpec, OmniTransferConfig
from .utils.initialization import (
@@ -50,6 +55,7 @@ __all__ = [
"MooncakeStoreConnector",
"MooncakeTransferEngineConnector",
"MoriTransferEngineConnector",
"NixlConnector",
"SharedMemoryConnector",
"YuanrongConnector",
"YuanrongTransferEngineConnector",
File diff suppressed because it is too large. Load diff
@@ -125,6 +125,17 @@ def _create_mori_transfer_engine_connector(config: dict[str, Any]) -> OmniConnec
return MoriTransferEngineConnector(config)
def _create_nixl_connector(config: dict[str, Any]) -> OmniConnectorBase:
try:
from .connectors.nixl_connector import NixlConnector
except ImportError:
import sys
sys.path.insert(0, os.path.dirname(os.path.dirname(__file__)))
from omni_connectors.connectors.nixl_connector import NixlConnector
return NixlConnector(config)
# Register connectors
OmniConnectorFactory.register_connector("MooncakeStoreConnector", _create_mooncake_store_connector)
OmniConnectorFactory.register_connector("MooncakeTransferEngineConnector", _create_mooncake_transfer_engine_connector)
@@ -132,5 +143,6 @@ OmniConnectorFactory.register_connector("SharedMemoryConnector", _create_shm_con
OmniConnectorFactory.register_connector("YuanrongConnector", _create_yuanrong_connector)
OmniConnectorFactory.register_connector("YuanrongTransferEngineConnector", _create_yuanrong_transfer_engine_connector)
OmniConnectorFactory.register_connector("MoriTransferEngineConnector", _create_mori_transfer_engine_connector)
OmniConnectorFactory.register_connector("NixlConnector", _create_nixl_connector)
# Backward-compatible aliases – will be removed in the future
OmniConnectorFactory.register_connector("MooncakeConnector", _create_mooncake_store_connector)
@@ -564,6 +564,13 @@ class OmniKVTransferManager:
# the wrong process. Explicit sender_* YAML values
# are still preserved for standalone connector use.
if c_type == "NixlConnector" and c_extra.get("role") == "receiver":
# This manager receives on the shared incoming edge;
# its YAML zmq_port belongs to the producer. NIXL treats
# an explicit zmq_port as a local listener, even for a
# receiver. Keep only the request-scoped sender_* endpoint.
c_extra.pop("zmq_port", None)
logger.info(
"Initializing OmniConnector type=%s role=%s",
c_type,
@@ -922,7 +929,9 @@ class OmniKVTransferManager:
The base host/port are also stored so that the receive path can
construct per-rank metadata for heterogeneous TP scenarios.
"""
if not self.config.need_recv_cache:
# A stage can sit on the receiving end of a payload-only edge, where the
# sender endpoint still has to be applied even though no KV is expected.
if not self.config.need_recv_cache and (self.config.connector_config or {}).get("role") != "receiver":
return
actual_info = self._resolve_sender_info(sender_info, sender_stage_id=sender_stage_id)
@@ -182,14 +182,19 @@ class _OmniConnectorPayloadTransportMixin(_OmniConnectorRuntimeMixin):
from_stage: str,
to_stage: str,
connector_get_key: str,
metadata: dict[str, Any] | None = None,
) -> Any:
"""Receive one ordinary non-KV stage payload on the local leader rank only."""
tp_group = self._get_local_tp_group()
if tp_group is None or getattr(tp_group, "world_size", 1) <= 1:
return connector.get(from_stage, to_stage, connector_get_key)
if metadata is None:
return connector.get(from_stage, to_stage, connector_get_key)
return connector.get(from_stage, to_stage, connector_get_key, metadata)
if not self.is_data_transfer_rank():
return None
return connector.get(from_stage, to_stage, connector_get_key)
if metadata is None:
return connector.get(from_stage, to_stage, connector_get_key)
return connector.get(from_stage, to_stage, connector_get_key, metadata)
def _recv_full_payload_result(
self,
@@ -197,6 +202,7 @@ class _OmniConnectorPayloadTransportMixin(_OmniConnectorRuntimeMixin):
from_stage: str,
to_stage: str,
connector_get_key: str,
metadata: dict[str, Any] | None = None,
) -> Any:
"""Receive one full-payload transfer on the local leader rank only."""
return self._recv_ordinary_stage_result(
@@ -204,6 +210,7 @@ class _OmniConnectorPayloadTransportMixin(_OmniConnectorRuntimeMixin):
from_stage,
to_stage,
connector_get_key,
metadata,
)
def _recv_async_chunk_result(
@@ -212,6 +219,7 @@ class _OmniConnectorPayloadTransportMixin(_OmniConnectorRuntimeMixin):
from_stage: str,
to_stage: str,
connector_get_key: str,
metadata: dict[str, Any] | None = None,
) -> Any:
"""Receive one ordinary async chunk on the local leader rank only."""
return self._recv_ordinary_stage_result(
@@ -219,6 +227,7 @@ class _OmniConnectorPayloadTransportMixin(_OmniConnectorRuntimeMixin):
from_stage,
to_stage,
connector_get_key,
metadata,
)
@staticmethod
@@ -1027,6 +1036,14 @@ class _OmniConnectorPayloadTransportMixin(_OmniConnectorRuntimeMixin):
chunk_id = self._get_req_chunk[req_id]
external_req_id = self._request_ids_mapping.get(req_id, req_id)
connector_get_key = f"{external_req_id}_{target_stage_id}_{chunk_id}"
request = self._pending_load_reqs.get(req_id)
sender_info = getattr(request, "payload_sender_info", None)
metadata = None
if isinstance(sender_info, dict):
host = sender_info.get("host")
port = sender_info.get("zmq_port")
if host and port:
metadata = {"source_host": str(host), "source_port": int(port)}
if self._async_chunk:
result = self._recv_async_chunk_result(
@@ -1034,6 +1051,7 @@ class _OmniConnectorPayloadTransportMixin(_OmniConnectorRuntimeMixin):
str(target_stage_id),
str(self._stage_id),
connector_get_key,
metadata,
)
else:
result = self._recv_full_payload_result(
@@ -1041,6 +1059,7 @@ class _OmniConnectorPayloadTransportMixin(_OmniConnectorRuntimeMixin):
str(target_stage_id),
str(self._stage_id),
connector_get_key,
metadata,
)
if result is None:
@@ -19,6 +19,8 @@ from vllm_omni.distributed.omni_connectors.utils.config import (
ConnectorSpec,
get_stage_connector_role,
)
from vllm_omni.distributed.omni_connectors.utils.initialization import resolve_connector_spec
from vllm_omni.distributed.omni_connectors.utils.kv_utils import get_local_tp_rank, get_omni_replica_id
logger = init_logger("vllm_omni.worker.omni_connector_model_runner_mixin")
@@ -186,7 +188,11 @@ class _OmniConnectorRuntimeMixin:
getattr(topology, "target_tp_size", None),
getattr(topology, "local_rank", None),
)
if all(isinstance(value, int) for value in effective_mapping):
if (
isinstance(effective_mapping[0], int)
and isinstance(effective_mapping[1], int)
and isinstance(effective_mapping[2], int)
):
rank_cfg = {
"from_tp": effective_mapping[0],
"to_tp": effective_mapping[1],
@@ -521,7 +527,15 @@ class _OmniConnectorRuntimeMixin:
elif not isinstance(extra, dict):
raise RuntimeError(f"Invalid extra config for connector {name}: expected dict, got {type(extra).__name__}")
spec = ConnectorSpec(name=name, extra=extra)
stage_id = int(getattr(model_config, "stage_id", extra.get("stage_id", 0)))
role = extra.get("role")
spec = resolve_connector_spec(
ConnectorSpec(name=name, extra=extra),
stage_id=stage_id,
role=str(role) if role is not None else None,
local_rank=get_local_tp_rank(),
replica_id=get_omni_replica_id(),
)
try:
return OmniConnectorFactory.create_connector(spec)
except Exception as exc:
@@ -20,6 +20,8 @@ from vllm_omni.data_entry_keys import MetaStruct, OmniPayloadStruct, unflatten_p
from ..adapter import construct_next_stage_streaming_input_prompt
from ..factory import OmniConnectorFactory
from ..utils.config import ConnectorSpec, stage_receives_chunks
from ..utils.initialization import resolve_connector_spec
from ..utils.kv_utils import get_local_tp_rank, get_omni_replica_id
from ..utils.logging import get_connector_logger
from .base import OmniTransferAdapterBase
@@ -47,10 +49,20 @@ class _SenderGeneration:
class _LoadEntry:
"""Identify one receiver registration across queue and I/O boundaries."""
__slots__ = ("request",)
__slots__ = ("request", "source_metadata")
def __init__(self, request: Request) -> None:
self.request = request
sender_info = getattr(request, "payload_sender_info", None)
self.source_metadata = None
if isinstance(sender_info, dict):
host = sender_info.get("host")
port = sender_info.get("zmq_port")
if host and port:
self.source_metadata = {
"source_host": str(host),
"source_port": int(port),
}
@property
def request_id(self) -> str:
@@ -280,11 +292,20 @@ class OmniChunkTransferAdapter(OmniTransferAdapterBase):
"extra": getattr(connector_config, "extra", {}),
}
connector_specs = ConnectorSpec(
connector_spec = ConnectorSpec(
name=connector_config.get("name", "SharedMemoryConnector"),
extra=connector_config.get("extra", {}),
)
return OmniConnectorFactory.create_connector(connector_specs)
stage_id = int(getattr(model_config, "stage_id", connector_spec.extra.get("stage_id", 0)))
role = connector_spec.extra.get("role")
resolved_spec = resolve_connector_spec(
connector_spec,
stage_id=stage_id,
role=str(role) if role is not None else None,
local_rank=get_local_tp_rank(),
replica_id=get_omni_replica_id(),
)
return OmniConnectorFactory.create_connector(resolved_spec)
def load_async(self, request: Request):
"""Register a request for asynchronous chunk retrieval.
@@ -484,6 +505,7 @@ class OmniChunkTransferAdapter(OmniTransferAdapterBase):
str(target_stage_id),
str(stage_id),
connector_get_key,
entry.source_metadata,
)
except Exception as e:
logger.error(f"SharedMemoryConnector get failed for req {connector_get_key}: {e}")
@@ -13,6 +13,7 @@ TRANSFER_ENGINE_CONNECTOR_NAMES = frozenset(
"MooncakeTransferEngineConnector",
"MoriTransferEngineConnector",
"YuanrongTransferEngineConnector",
"NixlConnector",
}
)
@@ -4,9 +4,10 @@
"""Utilities for OmniConnector configuration and validation."""
import json
import os
import warnings
from pathlib import Path
from typing import TYPE_CHECKING, Any
from typing import TYPE_CHECKING, Any, Literal
from ..factory import OmniConnectorFactory
from .config import TRANSFER_ENGINE_CONNECTOR_NAMES, ConnectorSpec, OmniTransferConfig
@@ -38,6 +39,91 @@ KV_RANK_PORT_STRIDE = 16
# comfortably sized block per replica for TP-rank and stage offsets.
KV_REPLICA_PORT_STRIDE = 1024
REQUEST_FORWARDING_PORT_OFFSET = 0
ORCHESTRATOR_PORT_OFFSET = 200
ConnectorPurpose = Literal["request_forwarding", "kv_transfer", "orchestrator"]
def compute_connector_zmq_port(
base_port: int,
*,
purpose: ConnectorPurpose,
from_stage: int,
local_rank: int = 0,
replica_id: int = 0,
) -> int:
"""Return a collision-free connector endpoint for an edge and worker."""
purpose_offsets = {
"request_forwarding": REQUEST_FORWARDING_PORT_OFFSET,
"kv_transfer": KV_TRANSFER_PORT_OFFSET,
"orchestrator": ORCHESTRATOR_PORT_OFFSET,
}
if purpose not in purpose_offsets:
raise ValueError(f"Unsupported connector purpose: {purpose}")
if not 0 <= from_stage < KV_RANK_PORT_STRIDE:
raise ValueError(f"from_stage must be in [0, {KV_RANK_PORT_STRIDE}), got {from_stage}")
if local_rank < 0:
raise ValueError(f"local_rank must be non-negative, got {local_rank}")
if replica_id < 0:
raise ValueError(f"replica_id must be non-negative, got {replica_id}")
port = (
base_port
+ purpose_offsets[purpose]
+ replica_id * KV_REPLICA_PORT_STRIDE
+ local_rank * KV_RANK_PORT_STRIDE
+ from_stage
)
if not 1 <= port <= 65535:
raise ValueError(f"Resolved connector port must be in [1, 65535], got {port}")
return port
def resolve_connector_spec(
spec: ConnectorSpec,
*,
stage_id: int,
role: str | None,
purpose: ConnectorPurpose = "request_forwarding",
local_rank: int = 0,
replica_id: int = 0,
) -> ConnectorSpec:
"""Copy and resolve a connector spec for one stage worker."""
extra = dict(spec.extra or {})
extra["stage_id"] = stage_id
if role is not None:
extra["role"] = role
if spec.name not in TRANSFER_ENGINE_CONNECTOR_NAMES:
return ConnectorSpec(name=spec.name, extra=extra)
resolved_port = None
if role != "receiver" or "sender_zmq_port" not in extra:
resolved_port = compute_connector_zmq_port(
expand_env_int(extra.get("zmq_port", 50051), "zmq_port"),
purpose=purpose,
from_stage=int(extra.get("from_stage", stage_id)),
local_rank=local_rank,
replica_id=replica_id,
)
if role == "receiver":
extra.setdefault("sender_host", extra.get("host", "127.0.0.1"))
extra.setdefault("sender_zmq_port", resolved_port)
extra.pop("zmq_port", None)
outgoing = extra.get("outgoing")
if spec.name == "NixlConnector" and isinstance(outgoing, dict):
extra["zmq_port"] = compute_connector_zmq_port(
expand_env_int(outgoing.get("zmq_port", 50051), "zmq_port"),
purpose=purpose,
from_stage=int(outgoing.get("from_stage", stage_id)),
local_rank=local_rank,
replica_id=replica_id,
)
extra["host"] = outgoing.get("host", "auto")
else:
extra["zmq_port"] = resolved_port
return ConnectorSpec(name=spec.name, extra=extra)
def initialize_connectors_from_config(
config_path: str | Path | None = None,
@@ -82,52 +168,40 @@ def create_connectors_from_config(
Returns:
A dictionary of connectors.
"""
purpose_port_offsets = {
"request_forwarding": 0,
"kv_transfer": KV_TRANSFER_PORT_OFFSET,
}
port_offset = purpose_port_offsets.get(purpose, 0)
orchestrator_port_offset = 200
connectors = {}
for edge_key, connector_spec in connectors_config.items():
from_stage, to_stage = edge_key
try:
if connector_spec.name in TRANSFER_ENGINE_CONNECTOR_NAMES:
extra = dict(connector_spec.extra) if connector_spec.extra else {}
base_port = expand_env_int(extra.get("zmq_port", 50051), "zmq_port")
try:
stage_offset = int(from_stage)
except (TypeError, ValueError):
stage_offset = 0
local_rank = 0 if str(caller_stage_id) == "orchestrator" else get_connector_local_rank()
if str(caller_stage_id) == "orchestrator":
rank0_port = base_port + orchestrator_port_offset + stage_offset
else:
rank0_port = base_port + port_offset + stage_offset
adjusted_port = rank0_port + local_rank * KV_RANK_PORT_STRIDE
extra["zmq_port"] = adjusted_port
from_stage_id = int(from_stage)
except (TypeError, ValueError) as exc:
raise ValueError(f"Transfer-engine stage id must be an integer, got {from_stage!r}") from exc
is_orchestrator = str(caller_stage_id) == "orchestrator"
local_rank = 0 if is_orchestrator else get_connector_local_rank()
resolved_purpose: ConnectorPurpose = "orchestrator" if is_orchestrator else purpose # type: ignore[assignment]
if is_sender is not None:
extra["role"] = "sender" if is_sender else "receiver"
if not is_sender:
extra.setdefault("sender_host", extra.get("host", "127.0.0.1"))
extra.setdefault("sender_zmq_port", adjusted_port)
elif caller_stage_id is not None:
caller_str = str(caller_stage_id)
if caller_str == from_stage:
extra["role"] = "sender"
elif caller_str == to_stage:
extra["role"] = "receiver"
extra.setdefault("sender_host", extra.get("host", "127.0.0.1"))
extra.setdefault("sender_zmq_port", adjusted_port)
else:
extra["role"] = "sender"
role = "sender" if is_sender else "receiver"
elif caller_stage_id is not None and str(caller_stage_id) == to_stage:
role = "receiver"
elif caller_stage_id is not None and str(caller_stage_id) == from_stage:
role = "sender"
else:
extra["role"] = extra.get("role", "auto")
connector = OmniConnectorFactory.create_connector(ConnectorSpec(name=connector_spec.name, extra=extra))
role = str(extra.get("role", "auto"))
extra["from_stage"] = from_stage_id
extra["to_stage"] = int(to_stage)
replica_id = max(int(os.environ.get("VLLM_OMNI_REPLICA_ID", "0")), 0)
resolved_spec = resolve_connector_spec(
ConnectorSpec(name=connector_spec.name, extra=extra),
stage_id=int(caller_stage_id) if str(caller_stage_id).isdigit() else from_stage_id,
role=role,
purpose=resolved_purpose,
local_rank=local_rank,
replica_id=replica_id,
)
connector = OmniConnectorFactory.create_connector(resolved_spec)
else:
connector = OmniConnectorFactory.create_connector(connector_spec)
connectors[edge_key] = connector
@@ -174,12 +248,16 @@ def get_connectors_config_for_stage(transfer_config: OmniTransferConfig | None,
# Incoming edge → this stage is the receiver
extra = dict(spec.extra) if spec.extra else {}
extra.setdefault("role", "receiver")
extra["from_stage"] = int(from_stage)
extra["to_stage"] = int(to_stage)
stage_connectors_config[f"from_stage_{from_stage}"] = {"spec": {"name": spec.name, "extra": extra}}
elif from_stage == target_stage and target_stage == "0":
# Outgoing edge for stage 0 — included for async_chunk spec
# extraction (omni_stage.py), NOT for connector instantiation.
extra = dict(spec.extra) if spec.extra else {}
extra.setdefault("role", "sender")
extra["from_stage"] = int(from_stage)
extra["to_stage"] = int(to_stage)
stage_connectors_config[f"to_stage_{to_stage}"] = {"spec": {"name": spec.name, "extra": extra}}
return stage_connectors_config
@@ -16,7 +16,7 @@ from vllm.distributed.parallel_state import (
)
from vllm.logger import init_logger
from .initialization import KV_RANK_PORT_STRIDE, KV_REPLICA_PORT_STRIDE, KV_TRANSFER_PORT_OFFSET
from .initialization import compute_connector_zmq_port
logger = init_logger(__name__)
@@ -123,12 +123,12 @@ def kv_zmq_port(
"""
replica = get_omni_replica_id() if replica_id is None else max(int(replica_id), 0)
return (
base_port
+ KV_TRANSFER_PORT_OFFSET
+ replica * KV_REPLICA_PORT_STRIDE
+ local_rank * KV_RANK_PORT_STRIDE
+ from_stage
return compute_connector_zmq_port(
base_port,
purpose="kv_transfer",
from_stage=from_stage,
local_rank=local_rank,
replica_id=replica,
)