[Feature] Transfer AR-to-diffusion payloads over NIXL (#6264)

Signed-off-by: yuanwu <yuan.wu@intel.com>
Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
This commit is contained in:
Yuan WuandHongsheng Liu authored and GitHub committed 2026-09-23 18:26:53 +08:00
1 parent 8953d082c0
commit cb5f508212
60 files changed
+4397 -641

No files matched your search

@@ -43,13 +43,16 @@ class _DummyDiffusionStage:
self.engine_input_source = engine_input_source or [0]
self.calls = []
async def add_request_async(self, request_id, prompt, sampling_params, kv_sender_info=None):
async def add_request_async(
self, request_id, prompt, sampling_params, kv_sender_info=None, payload_sender_info=None
):
self.calls.append(
{
"request_id": request_id,
"prompt": prompt,
"sampling_params": sampling_params,
"kv_sender_info": kv_sender_info,
"payload_sender_info": payload_sender_info,
}
)
@@ -154,6 +157,31 @@ def test_payload_sender_endpoint_matches_resolver_with_unequal_replicas(outgoing
assert sender_info["zmq_port"] != consumer.extra["sender_zmq_port"]
@pytest.mark.parametrize("initialized", [False, True])
def test_intermediate_payload_sender_ignores_incoming_kv_endpoint(initialized):
client = object.__new__(StageEngineCoreClient)
client.stage_id = 1
client.replica_id = 3
client.vllm_config = SimpleNamespace(
model_config=SimpleNamespace(
stage_connector_config={
"name": "NixlConnector",
"extra": {
"role": "receiver",
"host": "10.0.0.1",
"zmq_port": 47000,
"from_stage": 0,
"outgoing": {"host": "10.0.0.2", "zmq_port": 48000, "from_stage": 1},
},
}
)
)
client._omni_kv_config = {"connector_config": {"type": "NixlConnector", "role": "receiver", "zmq_port": 47100}}
client._kv_sender_info = {"host": "10.0.0.1", "zmq_port": 47100} if initialized else None
assert client._build_payload_sender_info() == {"host": "10.0.0.2", "zmq_port": 51073}
def test_stage_engine_core_client_builds_kv_sender_info_from_tcp_address():
client = object.__new__(StageEngineCoreClient)
client.stage_id = 0
@@ -235,6 +263,90 @@ def test_stage_engine_core_client_preserves_explicit_loopback_sender_host():
}
def test_stage_engine_core_client_payload_sender_uses_unresolved_base_port():
client = object.__new__(StageEngineCoreClient)
client.stage_id = 0
client.replica_id = 0
client.client_addresses = {"input_address": "tcp://10.20.30.40:1234"}
client.vllm_config = SimpleNamespace(
model_config=SimpleNamespace(
stage_connector_config={
"name": "NixlConnector",
"extra": {"role": "sender", "from_stage": 0, "zmq_port": 50171},
},
omni_kv_config={"connector_config": {"type": "NixlConnector", "role": "sender", "zmq_port": 50171}},
)
)
client._omni_kv_config = client.vllm_config.model_config.omni_kv_config
assert client._build_payload_sender_info() == {
"host": "10.20.30.40",
"zmq_port": 50071,
}
def test_stage_engine_core_client_payload_sender_preserves_stage_port_without_kv_config():
client = object.__new__(StageEngineCoreClient)
client.stage_id = 0
client.replica_id = 0
client.client_addresses = {"input_address": "tcp://10.20.30.40:1234"}
client.vllm_config = SimpleNamespace(
model_config=SimpleNamespace(
stage_connector_config={
"name": "NixlConnector",
"extra": {"role": "sender", "from_stage": 0, "zmq_port": 50071},
},
omni_kv_config=None,
)
)
client._omni_kv_config = None
assert client._build_payload_sender_info() == {
"host": "10.20.30.40",
"zmq_port": 50071,
}
def test_stage_engine_core_client_payload_sender_falls_back_to_kv_connector_config():
client = object.__new__(StageEngineCoreClient)
client.stage_id = 0
client.replica_id = 0
client.client_addresses = {"input_address": "tcp://10.20.30.40:1234"}
client.vllm_config = SimpleNamespace(
model_config=SimpleNamespace(
stage_connector_config=None,
omni_kv_config={
"connector_config": {
"type": "NixlConnector",
"role": "sender",
"zmq_port": 50171,
}
},
)
)
client._omni_kv_config = client.vllm_config.model_config.omni_kv_config
assert client._build_payload_sender_info() == {
"host": "10.20.30.40",
"zmq_port": 50071,
}
def test_stage_engine_core_client_payload_sender_falls_back_to_initialized_kv_sender():
client = object.__new__(StageEngineCoreClient)
client.stage_id = 0
client.replica_id = 0
client.client_addresses = {"input_address": "tcp://10.20.30.40:1234"}
client.vllm_config = SimpleNamespace(model_config=SimpleNamespace(stage_connector_config=None))
client._omni_kv_config = None
client._kv_sender_info = {"host": "10.20.30.40", "zmq_port": 50171}
assert client._build_payload_sender_info() == {
"host": "10.20.30.40",
"zmq_port": 50071,
}
def test_forward_to_diffusion_attaches_kv_sender_info():
orchestrator = object.__new__(Orchestrator)
diffusion_stage = _DummyDiffusionStage(engine_input_source=[0])
@@ -263,6 +375,36 @@ def test_forward_to_diffusion_attaches_kv_sender_info():
assert req_state.stage_submit_ts[1] > 0
def test_diffusion_resubmission_preserves_sender_endpoints():
orchestrator = object.__new__(Orchestrator)
diffusion_stage = _DummyDiffusionStage(engine_input_source=[0])
sender_pool = _build_sender_pool(0, {"host": "10.0.0.2", "zmq_port": 50151})
payload_endpoint = {"host": "10.0.0.2", "zmq_port": 50071}
sender_pool.stage_client.get_payload_sender_info = lambda: payload_endpoint
diffusion_pool = StagePool(1, diffusion_stage)
orchestrator.num_stages = 2
orchestrator.stage_pools = [sender_pool, diffusion_pool]
orchestrator._cfg_tracker = CfgCompanionTracker()
state = OrchestratorRequestState(
request_id="streaming",
prompt={"prompt": "hello"},
sampling_params_list=[SamplingParams(max_tokens=4), OmniDiffusionSamplingParams()],
final_stage_id=1,
)
output = SimpleNamespace(request_id="streaming", finished=True)
async def forward_twice():
await orchestrator._forward_to_next_stage("streaming", 0, output, state)
await orchestrator._forward_to_next_stage("streaming", 0, output, state)
asyncio.run(forward_twice())
assert len(diffusion_stage.calls) == 2
for call in diffusion_stage.calls:
assert call["kv_sender_info"] == {0: {"host": "10.0.0.2", "zmq_port": 50151}}
assert call["payload_sender_info"] == payload_endpoint
def test_forward_to_diffusion_uses_engine_input_source_for_kv_sender_info():
orchestrator = object.__new__(Orchestrator)
diffusion_stage = _DummyDiffusionStage(engine_input_source=[0])
@@ -293,7 +435,7 @@ def test_forward_to_diffusion_uses_engine_input_source_for_kv_sender_info():
def test_forward_to_diffusion_returns_terminal_error_for_empty_custom_inputs():
orchestrator = object.__new__(Orchestrator)
diffusion_stage = _DummyDiffusionStage(engine_input_source=[0])
diffusion_stage.custom_process_input_func = lambda *_args, **_kwargs: []
setattr(diffusion_stage, "custom_process_input_func", lambda *_args, **_kwargs: [])
sender_pool = _build_sender_pool(0, {"host": "10.0.0.2", "zmq_port": 50151})
diffusion_pool = StagePool(1, diffusion_stage)