[Core] Add NIXL omni connector (#6093)

Signed-off-by: yuanwu <yuan.wu@intel.com>
Signed-off-by: Yuan Wu <yuan.wu@intel.com>
This commit is contained in:
Yuan Wu authored and GitHub committed 2026-09-13 16:35:02 +08:00
1 parent 01a2f93256
commit 6753bbd18c
33 files changed
+3032 -66

No files matched your search

+5
View File
@@ -79,6 +79,7 @@ class OmniEngineCoreRequest(EngineCoreRequest):
# GPUModelRunner.model_intermediate_buffer instead of using the deprecated
# additional_information request transport.
model_intermediate_buffer: dict[str, Any] | None = None
payload_sender_info: dict[str, Any] | None = None
@classmethod
def from_request(
@@ -88,6 +89,7 @@ class OmniEngineCoreRequest(EngineCoreRequest):
prompt_embeds: torch.Tensor | None = None,
additional_information: AdditionalInformationPayload | None = None,
model_intermediate_buffer: dict[str, Any] | None = None,
payload_sender_info: dict[str, Any] | None = None,
) -> "OmniEngineCoreRequest":
"""Clone an EngineCoreRequest into an OmniEngineCoreRequest with optional payload overrides."""
@@ -97,6 +99,8 @@ class OmniEngineCoreRequest(EngineCoreRequest):
additional_information = getattr(request, "additional_information", None)
if model_intermediate_buffer is None:
model_intermediate_buffer = getattr(request, "model_intermediate_buffer", None)
if payload_sender_info is None:
payload_sender_info = getattr(request, "payload_sender_info", None)
return cls(
request_id=request.request_id,
@@ -121,6 +125,7 @@ class OmniEngineCoreRequest(EngineCoreRequest):
abort_immediately=request.abort_immediately,
additional_information=additional_information,
model_intermediate_buffer=model_intermediate_buffer,
payload_sender_info=payload_sender_info,
)
+22
View File
@@ -2148,6 +2148,7 @@ class Orchestrator:
*,
mm_features: list | None = None,
resumable: bool = False,
payload_sender_info: dict[str, Any] | None = None,
) -> Any:
next_pool = self.stage_pools[next_stage_id]
if self._next_stage_input_is_tokens(next_input):
@@ -2160,6 +2161,7 @@ class Orchestrator:
resumable=resumable,
)
request.external_req_id = request.request_id
request.payload_sender_info = payload_sender_info
return request
processor = self._get_stage_input_processor(next_stage_id)
@@ -2181,6 +2183,7 @@ class Orchestrator:
)
request = self._upgrade_processed_stage_request(request, next_input)
request.external_req_id = req_id
request.payload_sender_info = payload_sender_info
return request
@staticmethod
@@ -2829,6 +2832,7 @@ class Orchestrator:
params=params,
mm_features=mm_features,
resumable=next_stage_resumable,
payload_sender_info=self._build_payload_sender_info(src_stage_id, request_id=req_id),
)
if already_submitted:
@@ -2967,6 +2971,7 @@ class Orchestrator:
model_config=next_pool.stage_vllm_config.model_config,
resumable=downstream_resumable,
)
request.payload_sender_info = self._build_payload_sender_info(next_stage_id - 1, request_id=request_id)
request.external_req_id = request.request_id
submitted = await self._dispatch_or_fail_request(
lambda: next_pool.submit_initial(
@@ -3043,6 +3048,23 @@ class Orchestrator:
return sender_infos or None
def _build_payload_sender_info(
self,
sender_stage_id: int,
*,
request_id: str | None = None,
) -> dict[str, Any] | None:
if sender_stage_id < 0 or sender_stage_id >= len(self.stage_pools):
return None
sender_pool = self.stage_pools[sender_stage_id]
sender_stage = sender_pool.get_bound_client(request_id) if request_id is not None else None
if sender_stage is None:
sender_stage = sender_pool.stage_client
get_sender_info = getattr(sender_stage, "get_payload_sender_info", None)
if not callable(get_sender_info):
return None
return get_sender_info()
# ---- Shutdown / lifecycle ----
def _shutdown_stages(self) -> None:
+2
View File
@@ -94,6 +94,8 @@ class StagePoolLLMClient(StagePoolClient, Protocol):
kv_transfer_port_offset: int = ...,
) -> dict[str, Any] | None: ...
def get_payload_sender_info(self) -> dict[str, Any] | None: ...
class StagePoolDiffusionClient(StagePoolClient, Protocol):
"""Pool-facing API for diffusion stages."""
@@ -1,3 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""
Stage Engine Core Client for vLLM-Omni multi-stage runtime.
@@ -210,6 +213,7 @@ class StageEngineCoreClientBase(StageClientBase):
)
raise
self._payload_sender_info = self._build_payload_sender_info()
self._initialize_kv_sender_endpoint()
logger.info(
@@ -291,6 +295,39 @@ class StageEngineCoreClientBase(StageClientBase):
return None
return connector_config
def _build_payload_sender_info(self) -> dict[str, Any] | None:
model_config = getattr(self.vllm_config, "model_config", None)
connector_config = getattr(model_config, "stage_connector_config", None)
if not isinstance(connector_config, dict):
return None
extra = connector_config.get("extra")
if not isinstance(extra, dict):
return None
if isinstance(extra.get("outgoing"), dict):
extra = extra["outgoing"]
elif extra.get("role") != "sender":
return None
base_port = extra.get("zmq_port", 50051)
if base_port is None:
return None
sender_host = self._resolve_sender_host_from_config(extra)
if sender_host is None:
return None
from vllm_omni.distributed.omni_connectors.utils.initialization import compute_connector_zmq_port
return {
"host": sender_host,
"zmq_port": compute_connector_zmq_port(
int(os.path.expandvars(str(base_port))),
purpose="request_forwarding",
from_stage=int(extra.get("from_stage", self.stage_id)),
replica_id=self.replica_id,
),
}
def get_payload_sender_info(self) -> dict[str, Any] | None:
return dict(self._payload_sender_info) if self._payload_sender_info is not None else None
def _resolve_sender_host_from_config(self, connector_config: dict[str, Any]) -> str | None:
host = connector_config.get("sender_host") or connector_config.get("host")
if host in {None, "", "auto", "*", "0.0.0.0", "::"}:
@@ -88,6 +88,7 @@ class StageEngineCoreProc(EngineCoreProc):
scheduler_request, current_wave = super().preprocess_add_request(request)
scheduler_request.additional_information = request.additional_information
scheduler_request.external_req_id = getattr(request, "external_req_id", request.request_id)
scheduler_request.payload_sender_info = request.payload_sender_info
return scheduler_request, current_wave
@staticmethod
+25 -2
View File
@@ -50,6 +50,9 @@ from vllm_omni.config.omni_config import (
)
from vllm_omni.config.stage_config import StageType
from vllm_omni.diffusion.data import OmniDiffusionConfig
from vllm_omni.distributed.omni_connectors.utils.config import (
TRANSFER_ENGINE_CONNECTOR_NAMES,
)
from vllm_omni.engine.arg_utils import OmniEngineArgs
from vllm_omni.entrypoints.stage_utils import _to_dict, set_stage_devices
from vllm_omni.entrypoints.utils import filter_dataclass_kwargs
@@ -1882,17 +1885,37 @@ def get_stage_connector_spec(
stage_connectors_cfg = get_stage_connector_config(omni_transfer_config, stage_id)
for cfg in stage_connectors_cfg.values():
return dict(cfg.get("spec", {}))
connector_spec = dict(cfg.get("spec", {}))
if connector_spec.get("name") == "NixlConnector":
# A middle worker uses one connector for both directions. Preserve
# the incoming edge and carry the outgoing bind config separately.
for (source, target), outgoing in getattr(omni_transfer_config, "connectors", {}).items():
if source == str(stage_id):
if outgoing.name != "NixlConnector":
raise ValueError("A NIXL middle stage requires NIXL on its outgoing edge")
extra = dict(connector_spec.get("extra", {}))
extra["outgoing"] = {**(outgoing.extra or {}), "from_stage": int(source), "to_stage": int(target)}
connector_spec["extra"] = extra
break
if connector_spec.get("name") not in TRANSFER_ENGINE_CONNECTOR_NAMES:
extra = dict(connector_spec.get("extra", {}))
extra.pop("from_stage", None)
extra.pop("to_stage", None)
connector_spec["extra"] = extra
return connector_spec
# A producer does not consume connector data itself. Keep its connector
# for both async-chunk and terminal full-payload sends, but mark it
# sender-only so the scheduler does not park orchestrator-provided inputs
# waiting for an upstream payload.
target_stage = str(stage_id)
for (from_stage, _to_stage), spec in getattr(omni_transfer_config, "connectors", {}).items():
for (from_stage, to_stage), spec in getattr(omni_transfer_config, "connectors", {}).items():
if from_stage == target_stage:
extra = dict(spec.extra or {})
extra.setdefault("role", "sender")
if spec.name in TRANSFER_ENGINE_CONNECTOR_NAMES:
extra["from_stage"] = int(from_stage)
extra["to_stage"] = int(to_stage)
return {"name": spec.name, "extra": extra}
return {}