[Core] Add NIXL omni connector (#6093)
Signed-off-by: yuanwu <yuan.wu@intel.com> Signed-off-by: Yuan Wu <yuan.wu@intel.com>
This commit is contained in:
1 parent
01a2f93256
commit
6753bbd18c
33 files changed
+3032
-66
No files matched your search
@@ -79,6 +79,7 @@ class OmniEngineCoreRequest(EngineCoreRequest):
|
||||
# GPUModelRunner.model_intermediate_buffer instead of using the deprecated
|
||||
# additional_information request transport.
|
||||
model_intermediate_buffer: dict[str, Any] | None = None
|
||||
payload_sender_info: dict[str, Any] | None = None
|
||||
|
||||
@classmethod
|
||||
def from_request(
|
||||
@@ -88,6 +89,7 @@ class OmniEngineCoreRequest(EngineCoreRequest):
|
||||
prompt_embeds: torch.Tensor | None = None,
|
||||
additional_information: AdditionalInformationPayload | None = None,
|
||||
model_intermediate_buffer: dict[str, Any] | None = None,
|
||||
payload_sender_info: dict[str, Any] | None = None,
|
||||
) -> "OmniEngineCoreRequest":
|
||||
"""Clone an EngineCoreRequest into an OmniEngineCoreRequest with optional payload overrides."""
|
||||
|
||||
@@ -97,6 +99,8 @@ class OmniEngineCoreRequest(EngineCoreRequest):
|
||||
additional_information = getattr(request, "additional_information", None)
|
||||
if model_intermediate_buffer is None:
|
||||
model_intermediate_buffer = getattr(request, "model_intermediate_buffer", None)
|
||||
if payload_sender_info is None:
|
||||
payload_sender_info = getattr(request, "payload_sender_info", None)
|
||||
|
||||
return cls(
|
||||
request_id=request.request_id,
|
||||
@@ -121,6 +125,7 @@ class OmniEngineCoreRequest(EngineCoreRequest):
|
||||
abort_immediately=request.abort_immediately,
|
||||
additional_information=additional_information,
|
||||
model_intermediate_buffer=model_intermediate_buffer,
|
||||
payload_sender_info=payload_sender_info,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -2148,6 +2148,7 @@ class Orchestrator:
|
||||
*,
|
||||
mm_features: list | None = None,
|
||||
resumable: bool = False,
|
||||
payload_sender_info: dict[str, Any] | None = None,
|
||||
) -> Any:
|
||||
next_pool = self.stage_pools[next_stage_id]
|
||||
if self._next_stage_input_is_tokens(next_input):
|
||||
@@ -2160,6 +2161,7 @@ class Orchestrator:
|
||||
resumable=resumable,
|
||||
)
|
||||
request.external_req_id = request.request_id
|
||||
request.payload_sender_info = payload_sender_info
|
||||
return request
|
||||
|
||||
processor = self._get_stage_input_processor(next_stage_id)
|
||||
@@ -2181,6 +2183,7 @@ class Orchestrator:
|
||||
)
|
||||
request = self._upgrade_processed_stage_request(request, next_input)
|
||||
request.external_req_id = req_id
|
||||
request.payload_sender_info = payload_sender_info
|
||||
return request
|
||||
|
||||
@staticmethod
|
||||
@@ -2829,6 +2832,7 @@ class Orchestrator:
|
||||
params=params,
|
||||
mm_features=mm_features,
|
||||
resumable=next_stage_resumable,
|
||||
payload_sender_info=self._build_payload_sender_info(src_stage_id, request_id=req_id),
|
||||
)
|
||||
|
||||
if already_submitted:
|
||||
@@ -2967,6 +2971,7 @@ class Orchestrator:
|
||||
model_config=next_pool.stage_vllm_config.model_config,
|
||||
resumable=downstream_resumable,
|
||||
)
|
||||
request.payload_sender_info = self._build_payload_sender_info(next_stage_id - 1, request_id=request_id)
|
||||
request.external_req_id = request.request_id
|
||||
submitted = await self._dispatch_or_fail_request(
|
||||
lambda: next_pool.submit_initial(
|
||||
@@ -3043,6 +3048,23 @@ class Orchestrator:
|
||||
|
||||
return sender_infos or None
|
||||
|
||||
def _build_payload_sender_info(
|
||||
self,
|
||||
sender_stage_id: int,
|
||||
*,
|
||||
request_id: str | None = None,
|
||||
) -> dict[str, Any] | None:
|
||||
if sender_stage_id < 0 or sender_stage_id >= len(self.stage_pools):
|
||||
return None
|
||||
sender_pool = self.stage_pools[sender_stage_id]
|
||||
sender_stage = sender_pool.get_bound_client(request_id) if request_id is not None else None
|
||||
if sender_stage is None:
|
||||
sender_stage = sender_pool.stage_client
|
||||
get_sender_info = getattr(sender_stage, "get_payload_sender_info", None)
|
||||
if not callable(get_sender_info):
|
||||
return None
|
||||
return get_sender_info()
|
||||
|
||||
# ---- Shutdown / lifecycle ----
|
||||
|
||||
def _shutdown_stages(self) -> None:
|
||||
|
||||
@@ -94,6 +94,8 @@ class StagePoolLLMClient(StagePoolClient, Protocol):
|
||||
kv_transfer_port_offset: int = ...,
|
||||
) -> dict[str, Any] | None: ...
|
||||
|
||||
def get_payload_sender_info(self) -> dict[str, Any] | None: ...
|
||||
|
||||
|
||||
class StagePoolDiffusionClient(StagePoolClient, Protocol):
|
||||
"""Pool-facing API for diffusion stages."""
|
||||
|
||||
@@ -1,3 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
"""
|
||||
Stage Engine Core Client for vLLM-Omni multi-stage runtime.
|
||||
|
||||
@@ -210,6 +213,7 @@ class StageEngineCoreClientBase(StageClientBase):
|
||||
)
|
||||
raise
|
||||
|
||||
self._payload_sender_info = self._build_payload_sender_info()
|
||||
self._initialize_kv_sender_endpoint()
|
||||
|
||||
logger.info(
|
||||
@@ -291,6 +295,39 @@ class StageEngineCoreClientBase(StageClientBase):
|
||||
return None
|
||||
return connector_config
|
||||
|
||||
def _build_payload_sender_info(self) -> dict[str, Any] | None:
|
||||
model_config = getattr(self.vllm_config, "model_config", None)
|
||||
connector_config = getattr(model_config, "stage_connector_config", None)
|
||||
if not isinstance(connector_config, dict):
|
||||
return None
|
||||
extra = connector_config.get("extra")
|
||||
if not isinstance(extra, dict):
|
||||
return None
|
||||
if isinstance(extra.get("outgoing"), dict):
|
||||
extra = extra["outgoing"]
|
||||
elif extra.get("role") != "sender":
|
||||
return None
|
||||
base_port = extra.get("zmq_port", 50051)
|
||||
if base_port is None:
|
||||
return None
|
||||
sender_host = self._resolve_sender_host_from_config(extra)
|
||||
if sender_host is None:
|
||||
return None
|
||||
from vllm_omni.distributed.omni_connectors.utils.initialization import compute_connector_zmq_port
|
||||
|
||||
return {
|
||||
"host": sender_host,
|
||||
"zmq_port": compute_connector_zmq_port(
|
||||
int(os.path.expandvars(str(base_port))),
|
||||
purpose="request_forwarding",
|
||||
from_stage=int(extra.get("from_stage", self.stage_id)),
|
||||
replica_id=self.replica_id,
|
||||
),
|
||||
}
|
||||
|
||||
def get_payload_sender_info(self) -> dict[str, Any] | None:
|
||||
return dict(self._payload_sender_info) if self._payload_sender_info is not None else None
|
||||
|
||||
def _resolve_sender_host_from_config(self, connector_config: dict[str, Any]) -> str | None:
|
||||
host = connector_config.get("sender_host") or connector_config.get("host")
|
||||
if host in {None, "", "auto", "*", "0.0.0.0", "::"}:
|
||||
|
||||
@@ -88,6 +88,7 @@ class StageEngineCoreProc(EngineCoreProc):
|
||||
scheduler_request, current_wave = super().preprocess_add_request(request)
|
||||
scheduler_request.additional_information = request.additional_information
|
||||
scheduler_request.external_req_id = getattr(request, "external_req_id", request.request_id)
|
||||
scheduler_request.payload_sender_info = request.payload_sender_info
|
||||
return scheduler_request, current_wave
|
||||
|
||||
@staticmethod
|
||||
|
||||
@@ -50,6 +50,9 @@ from vllm_omni.config.omni_config import (
|
||||
)
|
||||
from vllm_omni.config.stage_config import StageType
|
||||
from vllm_omni.diffusion.data import OmniDiffusionConfig
|
||||
from vllm_omni.distributed.omni_connectors.utils.config import (
|
||||
TRANSFER_ENGINE_CONNECTOR_NAMES,
|
||||
)
|
||||
from vllm_omni.engine.arg_utils import OmniEngineArgs
|
||||
from vllm_omni.entrypoints.stage_utils import _to_dict, set_stage_devices
|
||||
from vllm_omni.entrypoints.utils import filter_dataclass_kwargs
|
||||
@@ -1882,17 +1885,37 @@ def get_stage_connector_spec(
|
||||
|
||||
stage_connectors_cfg = get_stage_connector_config(omni_transfer_config, stage_id)
|
||||
for cfg in stage_connectors_cfg.values():
|
||||
return dict(cfg.get("spec", {}))
|
||||
connector_spec = dict(cfg.get("spec", {}))
|
||||
if connector_spec.get("name") == "NixlConnector":
|
||||
# A middle worker uses one connector for both directions. Preserve
|
||||
# the incoming edge and carry the outgoing bind config separately.
|
||||
for (source, target), outgoing in getattr(omni_transfer_config, "connectors", {}).items():
|
||||
if source == str(stage_id):
|
||||
if outgoing.name != "NixlConnector":
|
||||
raise ValueError("A NIXL middle stage requires NIXL on its outgoing edge")
|
||||
extra = dict(connector_spec.get("extra", {}))
|
||||
extra["outgoing"] = {**(outgoing.extra or {}), "from_stage": int(source), "to_stage": int(target)}
|
||||
connector_spec["extra"] = extra
|
||||
break
|
||||
if connector_spec.get("name") not in TRANSFER_ENGINE_CONNECTOR_NAMES:
|
||||
extra = dict(connector_spec.get("extra", {}))
|
||||
extra.pop("from_stage", None)
|
||||
extra.pop("to_stage", None)
|
||||
connector_spec["extra"] = extra
|
||||
return connector_spec
|
||||
|
||||
# A producer does not consume connector data itself. Keep its connector
|
||||
# for both async-chunk and terminal full-payload sends, but mark it
|
||||
# sender-only so the scheduler does not park orchestrator-provided inputs
|
||||
# waiting for an upstream payload.
|
||||
target_stage = str(stage_id)
|
||||
for (from_stage, _to_stage), spec in getattr(omni_transfer_config, "connectors", {}).items():
|
||||
for (from_stage, to_stage), spec in getattr(omni_transfer_config, "connectors", {}).items():
|
||||
if from_stage == target_stage:
|
||||
extra = dict(spec.extra or {})
|
||||
extra.setdefault("role", "sender")
|
||||
if spec.name in TRANSFER_ENGINE_CONNECTOR_NAMES:
|
||||
extra["from_stage"] = int(from_stage)
|
||||
extra["to_stage"] = int(to_stage)
|
||||
return {"name": spec.name, "extra": extra}
|
||||
return {}
|
||||
|
||||
|
||||
Reference in new issue
Block a user