[Feature] Transfer AR-to-diffusion payloads over NIXL (#6264)
Signed-off-by: yuanwu <yuan.wu@intel.com> Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
This commit is contained in:
1 parent
8953d082c0
commit
cb5f508212
60 files changed
+4397
-641
No files matched your search
@@ -43,13 +43,16 @@ class _DummyDiffusionStage:
|
||||
self.engine_input_source = engine_input_source or [0]
|
||||
self.calls = []
|
||||
|
||||
async def add_request_async(self, request_id, prompt, sampling_params, kv_sender_info=None):
|
||||
async def add_request_async(
|
||||
self, request_id, prompt, sampling_params, kv_sender_info=None, payload_sender_info=None
|
||||
):
|
||||
self.calls.append(
|
||||
{
|
||||
"request_id": request_id,
|
||||
"prompt": prompt,
|
||||
"sampling_params": sampling_params,
|
||||
"kv_sender_info": kv_sender_info,
|
||||
"payload_sender_info": payload_sender_info,
|
||||
}
|
||||
)
|
||||
|
||||
@@ -154,6 +157,31 @@ def test_payload_sender_endpoint_matches_resolver_with_unequal_replicas(outgoing
|
||||
assert sender_info["zmq_port"] != consumer.extra["sender_zmq_port"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("initialized", [False, True])
|
||||
def test_intermediate_payload_sender_ignores_incoming_kv_endpoint(initialized):
|
||||
client = object.__new__(StageEngineCoreClient)
|
||||
client.stage_id = 1
|
||||
client.replica_id = 3
|
||||
client.vllm_config = SimpleNamespace(
|
||||
model_config=SimpleNamespace(
|
||||
stage_connector_config={
|
||||
"name": "NixlConnector",
|
||||
"extra": {
|
||||
"role": "receiver",
|
||||
"host": "10.0.0.1",
|
||||
"zmq_port": 47000,
|
||||
"from_stage": 0,
|
||||
"outgoing": {"host": "10.0.0.2", "zmq_port": 48000, "from_stage": 1},
|
||||
},
|
||||
}
|
||||
)
|
||||
)
|
||||
client._omni_kv_config = {"connector_config": {"type": "NixlConnector", "role": "receiver", "zmq_port": 47100}}
|
||||
client._kv_sender_info = {"host": "10.0.0.1", "zmq_port": 47100} if initialized else None
|
||||
|
||||
assert client._build_payload_sender_info() == {"host": "10.0.0.2", "zmq_port": 51073}
|
||||
|
||||
|
||||
def test_stage_engine_core_client_builds_kv_sender_info_from_tcp_address():
|
||||
client = object.__new__(StageEngineCoreClient)
|
||||
client.stage_id = 0
|
||||
@@ -235,6 +263,90 @@ def test_stage_engine_core_client_preserves_explicit_loopback_sender_host():
|
||||
}
|
||||
|
||||
|
||||
def test_stage_engine_core_client_payload_sender_uses_unresolved_base_port():
|
||||
client = object.__new__(StageEngineCoreClient)
|
||||
client.stage_id = 0
|
||||
client.replica_id = 0
|
||||
client.client_addresses = {"input_address": "tcp://10.20.30.40:1234"}
|
||||
client.vllm_config = SimpleNamespace(
|
||||
model_config=SimpleNamespace(
|
||||
stage_connector_config={
|
||||
"name": "NixlConnector",
|
||||
"extra": {"role": "sender", "from_stage": 0, "zmq_port": 50171},
|
||||
},
|
||||
omni_kv_config={"connector_config": {"type": "NixlConnector", "role": "sender", "zmq_port": 50171}},
|
||||
)
|
||||
)
|
||||
client._omni_kv_config = client.vllm_config.model_config.omni_kv_config
|
||||
|
||||
assert client._build_payload_sender_info() == {
|
||||
"host": "10.20.30.40",
|
||||
"zmq_port": 50071,
|
||||
}
|
||||
|
||||
|
||||
def test_stage_engine_core_client_payload_sender_preserves_stage_port_without_kv_config():
|
||||
client = object.__new__(StageEngineCoreClient)
|
||||
client.stage_id = 0
|
||||
client.replica_id = 0
|
||||
client.client_addresses = {"input_address": "tcp://10.20.30.40:1234"}
|
||||
client.vllm_config = SimpleNamespace(
|
||||
model_config=SimpleNamespace(
|
||||
stage_connector_config={
|
||||
"name": "NixlConnector",
|
||||
"extra": {"role": "sender", "from_stage": 0, "zmq_port": 50071},
|
||||
},
|
||||
omni_kv_config=None,
|
||||
)
|
||||
)
|
||||
client._omni_kv_config = None
|
||||
|
||||
assert client._build_payload_sender_info() == {
|
||||
"host": "10.20.30.40",
|
||||
"zmq_port": 50071,
|
||||
}
|
||||
|
||||
|
||||
def test_stage_engine_core_client_payload_sender_falls_back_to_kv_connector_config():
|
||||
client = object.__new__(StageEngineCoreClient)
|
||||
client.stage_id = 0
|
||||
client.replica_id = 0
|
||||
client.client_addresses = {"input_address": "tcp://10.20.30.40:1234"}
|
||||
client.vllm_config = SimpleNamespace(
|
||||
model_config=SimpleNamespace(
|
||||
stage_connector_config=None,
|
||||
omni_kv_config={
|
||||
"connector_config": {
|
||||
"type": "NixlConnector",
|
||||
"role": "sender",
|
||||
"zmq_port": 50171,
|
||||
}
|
||||
},
|
||||
)
|
||||
)
|
||||
client._omni_kv_config = client.vllm_config.model_config.omni_kv_config
|
||||
|
||||
assert client._build_payload_sender_info() == {
|
||||
"host": "10.20.30.40",
|
||||
"zmq_port": 50071,
|
||||
}
|
||||
|
||||
|
||||
def test_stage_engine_core_client_payload_sender_falls_back_to_initialized_kv_sender():
|
||||
client = object.__new__(StageEngineCoreClient)
|
||||
client.stage_id = 0
|
||||
client.replica_id = 0
|
||||
client.client_addresses = {"input_address": "tcp://10.20.30.40:1234"}
|
||||
client.vllm_config = SimpleNamespace(model_config=SimpleNamespace(stage_connector_config=None))
|
||||
client._omni_kv_config = None
|
||||
client._kv_sender_info = {"host": "10.20.30.40", "zmq_port": 50171}
|
||||
|
||||
assert client._build_payload_sender_info() == {
|
||||
"host": "10.20.30.40",
|
||||
"zmq_port": 50071,
|
||||
}
|
||||
|
||||
|
||||
def test_forward_to_diffusion_attaches_kv_sender_info():
|
||||
orchestrator = object.__new__(Orchestrator)
|
||||
diffusion_stage = _DummyDiffusionStage(engine_input_source=[0])
|
||||
@@ -263,6 +375,36 @@ def test_forward_to_diffusion_attaches_kv_sender_info():
|
||||
assert req_state.stage_submit_ts[1] > 0
|
||||
|
||||
|
||||
def test_diffusion_resubmission_preserves_sender_endpoints():
|
||||
orchestrator = object.__new__(Orchestrator)
|
||||
diffusion_stage = _DummyDiffusionStage(engine_input_source=[0])
|
||||
sender_pool = _build_sender_pool(0, {"host": "10.0.0.2", "zmq_port": 50151})
|
||||
payload_endpoint = {"host": "10.0.0.2", "zmq_port": 50071}
|
||||
sender_pool.stage_client.get_payload_sender_info = lambda: payload_endpoint
|
||||
diffusion_pool = StagePool(1, diffusion_stage)
|
||||
orchestrator.num_stages = 2
|
||||
orchestrator.stage_pools = [sender_pool, diffusion_pool]
|
||||
orchestrator._cfg_tracker = CfgCompanionTracker()
|
||||
state = OrchestratorRequestState(
|
||||
request_id="streaming",
|
||||
prompt={"prompt": "hello"},
|
||||
sampling_params_list=[SamplingParams(max_tokens=4), OmniDiffusionSamplingParams()],
|
||||
final_stage_id=1,
|
||||
)
|
||||
output = SimpleNamespace(request_id="streaming", finished=True)
|
||||
|
||||
async def forward_twice():
|
||||
await orchestrator._forward_to_next_stage("streaming", 0, output, state)
|
||||
await orchestrator._forward_to_next_stage("streaming", 0, output, state)
|
||||
|
||||
asyncio.run(forward_twice())
|
||||
|
||||
assert len(diffusion_stage.calls) == 2
|
||||
for call in diffusion_stage.calls:
|
||||
assert call["kv_sender_info"] == {0: {"host": "10.0.0.2", "zmq_port": 50151}}
|
||||
assert call["payload_sender_info"] == payload_endpoint
|
||||
|
||||
|
||||
def test_forward_to_diffusion_uses_engine_input_source_for_kv_sender_info():
|
||||
orchestrator = object.__new__(Orchestrator)
|
||||
diffusion_stage = _DummyDiffusionStage(engine_input_source=[0])
|
||||
@@ -293,7 +435,7 @@ def test_forward_to_diffusion_uses_engine_input_source_for_kv_sender_info():
|
||||
def test_forward_to_diffusion_returns_terminal_error_for_empty_custom_inputs():
|
||||
orchestrator = object.__new__(Orchestrator)
|
||||
diffusion_stage = _DummyDiffusionStage(engine_input_source=[0])
|
||||
diffusion_stage.custom_process_input_func = lambda *_args, **_kwargs: []
|
||||
setattr(diffusion_stage, "custom_process_input_func", lambda *_args, **_kwargs: [])
|
||||
sender_pool = _build_sender_pool(0, {"host": "10.0.0.2", "zmq_port": 50151})
|
||||
diffusion_pool = StagePool(1, diffusion_stage)
|
||||
|
||||
|
||||
Reference in new issue
Block a user