[Core] Add NIXL omni connector (#6093)
Signed-off-by: yuanwu <yuan.wu@intel.com> Signed-off-by: Yuan Wu <yuan.wu@intel.com>
This commit is contained in:
1 parent
01a2f93256
commit
6753bbd18c
33 files changed
+3032
-66
No files matched your search
@@ -7,6 +7,7 @@ from .omni_connectors import (
|
||||
MooncakeStoreConnector,
|
||||
MooncakeTransferEngineConnector,
|
||||
MoriTransferEngineConnector,
|
||||
NixlConnector,
|
||||
OmniConnectorBase,
|
||||
OmniConnectorFactory,
|
||||
OmniTransferConfig,
|
||||
@@ -27,6 +28,7 @@ __all__ = [
|
||||
"MooncakeStoreConnector",
|
||||
"MooncakeTransferEngineConnector",
|
||||
"MoriTransferEngineConnector",
|
||||
"NixlConnector",
|
||||
"SharedMemoryConnector",
|
||||
"YuanrongConnector",
|
||||
"YuanrongTransferEngineConnector",
|
||||
|
||||
@@ -22,6 +22,11 @@ try:
|
||||
from .connectors.mori_transfer_engine_connector import MoriTransferEngineConnector
|
||||
except ImportError:
|
||||
MoriTransferEngineConnector = None # RDMA deps (msgspec/zmq/mori) not installed
|
||||
|
||||
try:
|
||||
from .connectors.nixl_connector import NixlConnector
|
||||
except ImportError:
|
||||
NixlConnector = None # NIXL deps not installed
|
||||
from .factory import OmniConnectorFactory
|
||||
from .utils.config import ConnectorSpec, OmniTransferConfig
|
||||
from .utils.initialization import (
|
||||
@@ -50,6 +55,7 @@ __all__ = [
|
||||
"MooncakeStoreConnector",
|
||||
"MooncakeTransferEngineConnector",
|
||||
"MoriTransferEngineConnector",
|
||||
"NixlConnector",
|
||||
"SharedMemoryConnector",
|
||||
"YuanrongConnector",
|
||||
"YuanrongTransferEngineConnector",
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
@@ -125,6 +125,17 @@ def _create_mori_transfer_engine_connector(config: dict[str, Any]) -> OmniConnec
|
||||
return MoriTransferEngineConnector(config)
|
||||
|
||||
|
||||
def _create_nixl_connector(config: dict[str, Any]) -> OmniConnectorBase:
|
||||
try:
|
||||
from .connectors.nixl_connector import NixlConnector
|
||||
except ImportError:
|
||||
import sys
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(__file__)))
|
||||
from omni_connectors.connectors.nixl_connector import NixlConnector
|
||||
return NixlConnector(config)
|
||||
|
||||
|
||||
# Register connectors
|
||||
OmniConnectorFactory.register_connector("MooncakeStoreConnector", _create_mooncake_store_connector)
|
||||
OmniConnectorFactory.register_connector("MooncakeTransferEngineConnector", _create_mooncake_transfer_engine_connector)
|
||||
@@ -132,5 +143,6 @@ OmniConnectorFactory.register_connector("SharedMemoryConnector", _create_shm_con
|
||||
OmniConnectorFactory.register_connector("YuanrongConnector", _create_yuanrong_connector)
|
||||
OmniConnectorFactory.register_connector("YuanrongTransferEngineConnector", _create_yuanrong_transfer_engine_connector)
|
||||
OmniConnectorFactory.register_connector("MoriTransferEngineConnector", _create_mori_transfer_engine_connector)
|
||||
OmniConnectorFactory.register_connector("NixlConnector", _create_nixl_connector)
|
||||
# Backward-compatible aliases – will be removed in the future
|
||||
OmniConnectorFactory.register_connector("MooncakeConnector", _create_mooncake_store_connector)
|
||||
@@ -564,6 +564,13 @@ class OmniKVTransferManager:
|
||||
# the wrong process. Explicit sender_* YAML values
|
||||
# are still preserved for standalone connector use.
|
||||
|
||||
if c_type == "NixlConnector" and c_extra.get("role") == "receiver":
|
||||
# This manager receives on the shared incoming edge;
|
||||
# its YAML zmq_port belongs to the producer. NIXL treats
|
||||
# an explicit zmq_port as a local listener, even for a
|
||||
# receiver. Keep only the request-scoped sender_* endpoint.
|
||||
c_extra.pop("zmq_port", None)
|
||||
|
||||
logger.info(
|
||||
"Initializing OmniConnector type=%s role=%s",
|
||||
c_type,
|
||||
@@ -922,7 +929,9 @@ class OmniKVTransferManager:
|
||||
The base host/port are also stored so that the receive path can
|
||||
construct per-rank metadata for heterogeneous TP scenarios.
|
||||
"""
|
||||
if not self.config.need_recv_cache:
|
||||
# A stage can sit on the receiving end of a payload-only edge, where the
|
||||
# sender endpoint still has to be applied even though no KV is expected.
|
||||
if not self.config.need_recv_cache and (self.config.connector_config or {}).get("role") != "receiver":
|
||||
return
|
||||
|
||||
actual_info = self._resolve_sender_info(sender_info, sender_stage_id=sender_stage_id)
|
||||
|
||||
+21
-2
@@ -182,14 +182,19 @@ class _OmniConnectorPayloadTransportMixin(_OmniConnectorRuntimeMixin):
|
||||
from_stage: str,
|
||||
to_stage: str,
|
||||
connector_get_key: str,
|
||||
metadata: dict[str, Any] | None = None,
|
||||
) -> Any:
|
||||
"""Receive one ordinary non-KV stage payload on the local leader rank only."""
|
||||
tp_group = self._get_local_tp_group()
|
||||
if tp_group is None or getattr(tp_group, "world_size", 1) <= 1:
|
||||
return connector.get(from_stage, to_stage, connector_get_key)
|
||||
if metadata is None:
|
||||
return connector.get(from_stage, to_stage, connector_get_key)
|
||||
return connector.get(from_stage, to_stage, connector_get_key, metadata)
|
||||
if not self.is_data_transfer_rank():
|
||||
return None
|
||||
return connector.get(from_stage, to_stage, connector_get_key)
|
||||
if metadata is None:
|
||||
return connector.get(from_stage, to_stage, connector_get_key)
|
||||
return connector.get(from_stage, to_stage, connector_get_key, metadata)
|
||||
|
||||
def _recv_full_payload_result(
|
||||
self,
|
||||
@@ -197,6 +202,7 @@ class _OmniConnectorPayloadTransportMixin(_OmniConnectorRuntimeMixin):
|
||||
from_stage: str,
|
||||
to_stage: str,
|
||||
connector_get_key: str,
|
||||
metadata: dict[str, Any] | None = None,
|
||||
) -> Any:
|
||||
"""Receive one full-payload transfer on the local leader rank only."""
|
||||
return self._recv_ordinary_stage_result(
|
||||
@@ -204,6 +210,7 @@ class _OmniConnectorPayloadTransportMixin(_OmniConnectorRuntimeMixin):
|
||||
from_stage,
|
||||
to_stage,
|
||||
connector_get_key,
|
||||
metadata,
|
||||
)
|
||||
|
||||
def _recv_async_chunk_result(
|
||||
@@ -212,6 +219,7 @@ class _OmniConnectorPayloadTransportMixin(_OmniConnectorRuntimeMixin):
|
||||
from_stage: str,
|
||||
to_stage: str,
|
||||
connector_get_key: str,
|
||||
metadata: dict[str, Any] | None = None,
|
||||
) -> Any:
|
||||
"""Receive one ordinary async chunk on the local leader rank only."""
|
||||
return self._recv_ordinary_stage_result(
|
||||
@@ -219,6 +227,7 @@ class _OmniConnectorPayloadTransportMixin(_OmniConnectorRuntimeMixin):
|
||||
from_stage,
|
||||
to_stage,
|
||||
connector_get_key,
|
||||
metadata,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
@@ -1027,6 +1036,14 @@ class _OmniConnectorPayloadTransportMixin(_OmniConnectorRuntimeMixin):
|
||||
chunk_id = self._get_req_chunk[req_id]
|
||||
external_req_id = self._request_ids_mapping.get(req_id, req_id)
|
||||
connector_get_key = f"{external_req_id}_{target_stage_id}_{chunk_id}"
|
||||
request = self._pending_load_reqs.get(req_id)
|
||||
sender_info = getattr(request, "payload_sender_info", None)
|
||||
metadata = None
|
||||
if isinstance(sender_info, dict):
|
||||
host = sender_info.get("host")
|
||||
port = sender_info.get("zmq_port")
|
||||
if host and port:
|
||||
metadata = {"source_host": str(host), "source_port": int(port)}
|
||||
|
||||
if self._async_chunk:
|
||||
result = self._recv_async_chunk_result(
|
||||
@@ -1034,6 +1051,7 @@ class _OmniConnectorPayloadTransportMixin(_OmniConnectorRuntimeMixin):
|
||||
str(target_stage_id),
|
||||
str(self._stage_id),
|
||||
connector_get_key,
|
||||
metadata,
|
||||
)
|
||||
else:
|
||||
result = self._recv_full_payload_result(
|
||||
@@ -1041,6 +1059,7 @@ class _OmniConnectorPayloadTransportMixin(_OmniConnectorRuntimeMixin):
|
||||
str(target_stage_id),
|
||||
str(self._stage_id),
|
||||
connector_get_key,
|
||||
metadata,
|
||||
)
|
||||
|
||||
if result is None:
|
||||
|
||||
@@ -19,6 +19,8 @@ from vllm_omni.distributed.omni_connectors.utils.config import (
|
||||
ConnectorSpec,
|
||||
get_stage_connector_role,
|
||||
)
|
||||
from vllm_omni.distributed.omni_connectors.utils.initialization import resolve_connector_spec
|
||||
from vllm_omni.distributed.omni_connectors.utils.kv_utils import get_local_tp_rank, get_omni_replica_id
|
||||
|
||||
logger = init_logger("vllm_omni.worker.omni_connector_model_runner_mixin")
|
||||
|
||||
@@ -186,7 +188,11 @@ class _OmniConnectorRuntimeMixin:
|
||||
getattr(topology, "target_tp_size", None),
|
||||
getattr(topology, "local_rank", None),
|
||||
)
|
||||
if all(isinstance(value, int) for value in effective_mapping):
|
||||
if (
|
||||
isinstance(effective_mapping[0], int)
|
||||
and isinstance(effective_mapping[1], int)
|
||||
and isinstance(effective_mapping[2], int)
|
||||
):
|
||||
rank_cfg = {
|
||||
"from_tp": effective_mapping[0],
|
||||
"to_tp": effective_mapping[1],
|
||||
@@ -521,7 +527,15 @@ class _OmniConnectorRuntimeMixin:
|
||||
elif not isinstance(extra, dict):
|
||||
raise RuntimeError(f"Invalid extra config for connector {name}: expected dict, got {type(extra).__name__}")
|
||||
|
||||
spec = ConnectorSpec(name=name, extra=extra)
|
||||
stage_id = int(getattr(model_config, "stage_id", extra.get("stage_id", 0)))
|
||||
role = extra.get("role")
|
||||
spec = resolve_connector_spec(
|
||||
ConnectorSpec(name=name, extra=extra),
|
||||
stage_id=stage_id,
|
||||
role=str(role) if role is not None else None,
|
||||
local_rank=get_local_tp_rank(),
|
||||
replica_id=get_omni_replica_id(),
|
||||
)
|
||||
try:
|
||||
return OmniConnectorFactory.create_connector(spec)
|
||||
except Exception as exc:
|
||||
|
||||
@@ -20,6 +20,8 @@ from vllm_omni.data_entry_keys import MetaStruct, OmniPayloadStruct, unflatten_p
|
||||
from ..adapter import construct_next_stage_streaming_input_prompt
|
||||
from ..factory import OmniConnectorFactory
|
||||
from ..utils.config import ConnectorSpec, stage_receives_chunks
|
||||
from ..utils.initialization import resolve_connector_spec
|
||||
from ..utils.kv_utils import get_local_tp_rank, get_omni_replica_id
|
||||
from ..utils.logging import get_connector_logger
|
||||
from .base import OmniTransferAdapterBase
|
||||
|
||||
@@ -47,10 +49,20 @@ class _SenderGeneration:
|
||||
class _LoadEntry:
|
||||
"""Identify one receiver registration across queue and I/O boundaries."""
|
||||
|
||||
__slots__ = ("request",)
|
||||
__slots__ = ("request", "source_metadata")
|
||||
|
||||
def __init__(self, request: Request) -> None:
|
||||
self.request = request
|
||||
sender_info = getattr(request, "payload_sender_info", None)
|
||||
self.source_metadata = None
|
||||
if isinstance(sender_info, dict):
|
||||
host = sender_info.get("host")
|
||||
port = sender_info.get("zmq_port")
|
||||
if host and port:
|
||||
self.source_metadata = {
|
||||
"source_host": str(host),
|
||||
"source_port": int(port),
|
||||
}
|
||||
|
||||
@property
|
||||
def request_id(self) -> str:
|
||||
@@ -280,11 +292,20 @@ class OmniChunkTransferAdapter(OmniTransferAdapterBase):
|
||||
"extra": getattr(connector_config, "extra", {}),
|
||||
}
|
||||
|
||||
connector_specs = ConnectorSpec(
|
||||
connector_spec = ConnectorSpec(
|
||||
name=connector_config.get("name", "SharedMemoryConnector"),
|
||||
extra=connector_config.get("extra", {}),
|
||||
)
|
||||
return OmniConnectorFactory.create_connector(connector_specs)
|
||||
stage_id = int(getattr(model_config, "stage_id", connector_spec.extra.get("stage_id", 0)))
|
||||
role = connector_spec.extra.get("role")
|
||||
resolved_spec = resolve_connector_spec(
|
||||
connector_spec,
|
||||
stage_id=stage_id,
|
||||
role=str(role) if role is not None else None,
|
||||
local_rank=get_local_tp_rank(),
|
||||
replica_id=get_omni_replica_id(),
|
||||
)
|
||||
return OmniConnectorFactory.create_connector(resolved_spec)
|
||||
|
||||
def load_async(self, request: Request):
|
||||
"""Register a request for asynchronous chunk retrieval.
|
||||
@@ -484,6 +505,7 @@ class OmniChunkTransferAdapter(OmniTransferAdapterBase):
|
||||
str(target_stage_id),
|
||||
str(stage_id),
|
||||
connector_get_key,
|
||||
entry.source_metadata,
|
||||
)
|
||||
except Exception as e:
|
||||
logger.error(f"SharedMemoryConnector get failed for req {connector_get_key}: {e}")
|
||||
|
||||
@@ -13,6 +13,7 @@ TRANSFER_ENGINE_CONNECTOR_NAMES = frozenset(
|
||||
"MooncakeTransferEngineConnector",
|
||||
"MoriTransferEngineConnector",
|
||||
"YuanrongTransferEngineConnector",
|
||||
"NixlConnector",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@@ -4,9 +4,10 @@
|
||||
"""Utilities for OmniConnector configuration and validation."""
|
||||
|
||||
import json
|
||||
import os
|
||||
import warnings
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, Any
|
||||
from typing import TYPE_CHECKING, Any, Literal
|
||||
|
||||
from ..factory import OmniConnectorFactory
|
||||
from .config import TRANSFER_ENGINE_CONNECTOR_NAMES, ConnectorSpec, OmniTransferConfig
|
||||
@@ -38,6 +39,91 @@ KV_RANK_PORT_STRIDE = 16
|
||||
# comfortably sized block per replica for TP-rank and stage offsets.
|
||||
KV_REPLICA_PORT_STRIDE = 1024
|
||||
|
||||
REQUEST_FORWARDING_PORT_OFFSET = 0
|
||||
ORCHESTRATOR_PORT_OFFSET = 200
|
||||
|
||||
ConnectorPurpose = Literal["request_forwarding", "kv_transfer", "orchestrator"]
|
||||
|
||||
|
||||
def compute_connector_zmq_port(
|
||||
base_port: int,
|
||||
*,
|
||||
purpose: ConnectorPurpose,
|
||||
from_stage: int,
|
||||
local_rank: int = 0,
|
||||
replica_id: int = 0,
|
||||
) -> int:
|
||||
"""Return a collision-free connector endpoint for an edge and worker."""
|
||||
purpose_offsets = {
|
||||
"request_forwarding": REQUEST_FORWARDING_PORT_OFFSET,
|
||||
"kv_transfer": KV_TRANSFER_PORT_OFFSET,
|
||||
"orchestrator": ORCHESTRATOR_PORT_OFFSET,
|
||||
}
|
||||
if purpose not in purpose_offsets:
|
||||
raise ValueError(f"Unsupported connector purpose: {purpose}")
|
||||
if not 0 <= from_stage < KV_RANK_PORT_STRIDE:
|
||||
raise ValueError(f"from_stage must be in [0, {KV_RANK_PORT_STRIDE}), got {from_stage}")
|
||||
if local_rank < 0:
|
||||
raise ValueError(f"local_rank must be non-negative, got {local_rank}")
|
||||
if replica_id < 0:
|
||||
raise ValueError(f"replica_id must be non-negative, got {replica_id}")
|
||||
|
||||
port = (
|
||||
base_port
|
||||
+ purpose_offsets[purpose]
|
||||
+ replica_id * KV_REPLICA_PORT_STRIDE
|
||||
+ local_rank * KV_RANK_PORT_STRIDE
|
||||
+ from_stage
|
||||
)
|
||||
if not 1 <= port <= 65535:
|
||||
raise ValueError(f"Resolved connector port must be in [1, 65535], got {port}")
|
||||
return port
|
||||
|
||||
|
||||
def resolve_connector_spec(
|
||||
spec: ConnectorSpec,
|
||||
*,
|
||||
stage_id: int,
|
||||
role: str | None,
|
||||
purpose: ConnectorPurpose = "request_forwarding",
|
||||
local_rank: int = 0,
|
||||
replica_id: int = 0,
|
||||
) -> ConnectorSpec:
|
||||
"""Copy and resolve a connector spec for one stage worker."""
|
||||
extra = dict(spec.extra or {})
|
||||
extra["stage_id"] = stage_id
|
||||
if role is not None:
|
||||
extra["role"] = role
|
||||
if spec.name not in TRANSFER_ENGINE_CONNECTOR_NAMES:
|
||||
return ConnectorSpec(name=spec.name, extra=extra)
|
||||
|
||||
resolved_port = None
|
||||
if role != "receiver" or "sender_zmq_port" not in extra:
|
||||
resolved_port = compute_connector_zmq_port(
|
||||
expand_env_int(extra.get("zmq_port", 50051), "zmq_port"),
|
||||
purpose=purpose,
|
||||
from_stage=int(extra.get("from_stage", stage_id)),
|
||||
local_rank=local_rank,
|
||||
replica_id=replica_id,
|
||||
)
|
||||
if role == "receiver":
|
||||
extra.setdefault("sender_host", extra.get("host", "127.0.0.1"))
|
||||
extra.setdefault("sender_zmq_port", resolved_port)
|
||||
extra.pop("zmq_port", None)
|
||||
outgoing = extra.get("outgoing")
|
||||
if spec.name == "NixlConnector" and isinstance(outgoing, dict):
|
||||
extra["zmq_port"] = compute_connector_zmq_port(
|
||||
expand_env_int(outgoing.get("zmq_port", 50051), "zmq_port"),
|
||||
purpose=purpose,
|
||||
from_stage=int(outgoing.get("from_stage", stage_id)),
|
||||
local_rank=local_rank,
|
||||
replica_id=replica_id,
|
||||
)
|
||||
extra["host"] = outgoing.get("host", "auto")
|
||||
else:
|
||||
extra["zmq_port"] = resolved_port
|
||||
return ConnectorSpec(name=spec.name, extra=extra)
|
||||
|
||||
|
||||
def initialize_connectors_from_config(
|
||||
config_path: str | Path | None = None,
|
||||
@@ -82,52 +168,40 @@ def create_connectors_from_config(
|
||||
Returns:
|
||||
A dictionary of connectors.
|
||||
"""
|
||||
purpose_port_offsets = {
|
||||
"request_forwarding": 0,
|
||||
"kv_transfer": KV_TRANSFER_PORT_OFFSET,
|
||||
}
|
||||
port_offset = purpose_port_offsets.get(purpose, 0)
|
||||
orchestrator_port_offset = 200
|
||||
|
||||
connectors = {}
|
||||
for edge_key, connector_spec in connectors_config.items():
|
||||
from_stage, to_stage = edge_key
|
||||
try:
|
||||
if connector_spec.name in TRANSFER_ENGINE_CONNECTOR_NAMES:
|
||||
extra = dict(connector_spec.extra) if connector_spec.extra else {}
|
||||
base_port = expand_env_int(extra.get("zmq_port", 50051), "zmq_port")
|
||||
try:
|
||||
stage_offset = int(from_stage)
|
||||
except (TypeError, ValueError):
|
||||
stage_offset = 0
|
||||
|
||||
local_rank = 0 if str(caller_stage_id) == "orchestrator" else get_connector_local_rank()
|
||||
if str(caller_stage_id) == "orchestrator":
|
||||
rank0_port = base_port + orchestrator_port_offset + stage_offset
|
||||
else:
|
||||
rank0_port = base_port + port_offset + stage_offset
|
||||
adjusted_port = rank0_port + local_rank * KV_RANK_PORT_STRIDE
|
||||
extra["zmq_port"] = adjusted_port
|
||||
from_stage_id = int(from_stage)
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise ValueError(f"Transfer-engine stage id must be an integer, got {from_stage!r}") from exc
|
||||
|
||||
is_orchestrator = str(caller_stage_id) == "orchestrator"
|
||||
local_rank = 0 if is_orchestrator else get_connector_local_rank()
|
||||
resolved_purpose: ConnectorPurpose = "orchestrator" if is_orchestrator else purpose # type: ignore[assignment]
|
||||
if is_sender is not None:
|
||||
extra["role"] = "sender" if is_sender else "receiver"
|
||||
if not is_sender:
|
||||
extra.setdefault("sender_host", extra.get("host", "127.0.0.1"))
|
||||
extra.setdefault("sender_zmq_port", adjusted_port)
|
||||
elif caller_stage_id is not None:
|
||||
caller_str = str(caller_stage_id)
|
||||
if caller_str == from_stage:
|
||||
extra["role"] = "sender"
|
||||
elif caller_str == to_stage:
|
||||
extra["role"] = "receiver"
|
||||
extra.setdefault("sender_host", extra.get("host", "127.0.0.1"))
|
||||
extra.setdefault("sender_zmq_port", adjusted_port)
|
||||
else:
|
||||
extra["role"] = "sender"
|
||||
role = "sender" if is_sender else "receiver"
|
||||
elif caller_stage_id is not None and str(caller_stage_id) == to_stage:
|
||||
role = "receiver"
|
||||
elif caller_stage_id is not None and str(caller_stage_id) == from_stage:
|
||||
role = "sender"
|
||||
else:
|
||||
extra["role"] = extra.get("role", "auto")
|
||||
|
||||
connector = OmniConnectorFactory.create_connector(ConnectorSpec(name=connector_spec.name, extra=extra))
|
||||
role = str(extra.get("role", "auto"))
|
||||
extra["from_stage"] = from_stage_id
|
||||
extra["to_stage"] = int(to_stage)
|
||||
replica_id = max(int(os.environ.get("VLLM_OMNI_REPLICA_ID", "0")), 0)
|
||||
resolved_spec = resolve_connector_spec(
|
||||
ConnectorSpec(name=connector_spec.name, extra=extra),
|
||||
stage_id=int(caller_stage_id) if str(caller_stage_id).isdigit() else from_stage_id,
|
||||
role=role,
|
||||
purpose=resolved_purpose,
|
||||
local_rank=local_rank,
|
||||
replica_id=replica_id,
|
||||
)
|
||||
connector = OmniConnectorFactory.create_connector(resolved_spec)
|
||||
else:
|
||||
connector = OmniConnectorFactory.create_connector(connector_spec)
|
||||
connectors[edge_key] = connector
|
||||
@@ -174,12 +248,16 @@ def get_connectors_config_for_stage(transfer_config: OmniTransferConfig | None,
|
||||
# Incoming edge → this stage is the receiver
|
||||
extra = dict(spec.extra) if spec.extra else {}
|
||||
extra.setdefault("role", "receiver")
|
||||
extra["from_stage"] = int(from_stage)
|
||||
extra["to_stage"] = int(to_stage)
|
||||
stage_connectors_config[f"from_stage_{from_stage}"] = {"spec": {"name": spec.name, "extra": extra}}
|
||||
elif from_stage == target_stage and target_stage == "0":
|
||||
# Outgoing edge for stage 0 — included for async_chunk spec
|
||||
# extraction (omni_stage.py), NOT for connector instantiation.
|
||||
extra = dict(spec.extra) if spec.extra else {}
|
||||
extra.setdefault("role", "sender")
|
||||
extra["from_stage"] = int(from_stage)
|
||||
extra["to_stage"] = int(to_stage)
|
||||
stage_connectors_config[f"to_stage_{to_stage}"] = {"spec": {"name": spec.name, "extra": extra}}
|
||||
|
||||
return stage_connectors_config
|
||||
|
||||
@@ -16,7 +16,7 @@ from vllm.distributed.parallel_state import (
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from .initialization import KV_RANK_PORT_STRIDE, KV_REPLICA_PORT_STRIDE, KV_TRANSFER_PORT_OFFSET
|
||||
from .initialization import compute_connector_zmq_port
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -123,12 +123,12 @@ def kv_zmq_port(
|
||||
|
||||
"""
|
||||
replica = get_omni_replica_id() if replica_id is None else max(int(replica_id), 0)
|
||||
return (
|
||||
base_port
|
||||
+ KV_TRANSFER_PORT_OFFSET
|
||||
+ replica * KV_REPLICA_PORT_STRIDE
|
||||
+ local_rank * KV_RANK_PORT_STRIDE
|
||||
+ from_stage
|
||||
return compute_connector_zmq_port(
|
||||
base_port,
|
||||
purpose="kv_transfer",
|
||||
from_stage=from_stage,
|
||||
local_rank=local_rank,
|
||||
replica_id=replica,
|
||||
)
|
||||
|
||||
|
||||
|
||||
Reference in new issue
Block a user