[BugFix] Eliminate distributed test port race by switching to file:// rendezvous (#6468)
Signed-off-by: Nick Cao <ncao@redhat.com>
This commit is contained in:
1 parent
3ad64c6a12
commit
b2c9376539
11 files changed
+117
-237
No files matched your search
@@ -1,5 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
"""
|
||||
End-to-end test for Bagel text2img with the Mooncake inter-stage connector.
|
||||
@@ -22,7 +22,7 @@ import pytest
|
||||
from PIL import Image
|
||||
|
||||
from tests.helpers.mark import hardware_test
|
||||
from tests.helpers.runtime import OmniRunner
|
||||
from tests.helpers.runtime import OmniRunner, get_open_port
|
||||
from tests.helpers.stage_config import get_deploy_config_path, modify_stage_config
|
||||
from vllm_omni.entrypoints.omni import Omni
|
||||
from vllm_omni.outputs import OmniRequestOutput
|
||||
@@ -55,15 +55,6 @@ PIXEL_TOLERANCE = 10
|
||||
DEFAULT_PROMPT = "<|im_start|>A cute cat<|im_end|>"
|
||||
|
||||
|
||||
def _find_free_port() -> int:
|
||||
"""Find and return a free ephemeral port by binding to port 0."""
|
||||
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
|
||||
s.bind(("127.0.0.1", 0))
|
||||
s.listen(1)
|
||||
port = s.getsockname()[1]
|
||||
return port
|
||||
|
||||
|
||||
def _configure_sampling_params(omni: Omni, num_inference_steps: int = 15) -> list:
|
||||
"""Configure sampling parameters for Bagel text2img generation.
|
||||
|
||||
@@ -277,9 +268,9 @@ def test_bagel_text2img_mooncake_connector(run_level):
|
||||
"mooncake_master is not available or cannot execute (missing shared libraries like libibverbs)"
|
||||
)
|
||||
MOONCAKE_HOST = "127.0.0.1"
|
||||
MOONCAKE_RPC_PORT = _find_free_port()
|
||||
MOONCAKE_HTTP_PORT = _find_free_port()
|
||||
MOONCAKE_METRICS_PORT = _find_free_port()
|
||||
MOONCAKE_RPC_PORT = get_open_port()
|
||||
MOONCAKE_HTTP_PORT = get_open_port()
|
||||
MOONCAKE_METRICS_PORT = get_open_port()
|
||||
|
||||
mooncake_master_proc = None
|
||||
temp_config_file = None
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
"""
|
||||
End-to-end tests for Bagel with shared memory connector: img2img and text2img.
|
||||
@@ -13,7 +13,6 @@ End-to-end tests for Bagel with shared memory connector: img2img and text2img.
|
||||
import os
|
||||
|
||||
os.environ["VLLM_WORKER_MULTIPROC_METHOD"] = "spawn"
|
||||
import socket
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
@@ -82,15 +81,6 @@ def _load_input_image() -> Image.Image:
|
||||
return ImageAsset("2560px-Gfp-wisconsin-madison-the-nature-boardwalk").pil_image.convert("RGB")
|
||||
|
||||
|
||||
def _find_free_port() -> int:
|
||||
"""Find and return a free ephemeral port by binding to port 0."""
|
||||
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
|
||||
s.bind(("127.0.0.1", 0))
|
||||
s.listen(1)
|
||||
port = s.getsockname()[1]
|
||||
return port
|
||||
|
||||
|
||||
def _configure_sampling_params(omni: Omni, num_inference_steps: int = 15) -> list:
|
||||
"""Configure sampling parameters for Bagel img2img generation.
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
"""
|
||||
Integration tests for MooncakeTransferEngineConnector.
|
||||
@@ -18,6 +18,7 @@ import pytest
|
||||
import torch
|
||||
|
||||
from tests.helpers.mark import hardware_test
|
||||
from tests.helpers.runtime import get_open_port
|
||||
from vllm_omni.distributed.omni_connectors.connectors.mooncake_transfer_engine_connector import (
|
||||
ManagedBuffer,
|
||||
MooncakeTransferEngineConnector,
|
||||
@@ -164,14 +165,6 @@ RDMA_HOST = get_rdma_host()
|
||||
RDMA_DEVICE = _detect_rdma_device()
|
||||
|
||||
|
||||
def _free_port() -> int:
|
||||
import socket
|
||||
|
||||
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
|
||||
s.bind((RDMA_HOST, 0))
|
||||
return s.getsockname()[1]
|
||||
|
||||
|
||||
def _connector_config(
|
||||
zmq_port: int,
|
||||
pool_size: int = 16 * 1024 * 1024,
|
||||
@@ -204,7 +197,7 @@ class TestBasicConnector:
|
||||
"""Verify connector initialization, put, cleanup, and health check."""
|
||||
|
||||
def test_initialization(self):
|
||||
port = _free_port()
|
||||
port = get_open_port(RDMA_HOST)
|
||||
with MooncakeTransferEngineConnector(_connector_config(port, pool_size=1024 * 1024)) as c:
|
||||
assert c.rpc_port != 0
|
||||
assert c.pool_size == 1024 * 1024
|
||||
@@ -214,7 +207,7 @@ class TestBasicConnector:
|
||||
|
||||
def test_put_tensor_bytes_object(self):
|
||||
"""Put tensor / bytes / dict and verify metadata."""
|
||||
port = _free_port()
|
||||
port = get_open_port(RDMA_HOST)
|
||||
with MooncakeTransferEngineConnector(_connector_config(port)) as c:
|
||||
ok, sz, meta = c.put("s0", "s1", "t", torch.randn(100))
|
||||
assert ok
|
||||
@@ -229,7 +222,7 @@ class TestBasicConnector:
|
||||
assert not meta["is_fast_path"]
|
||||
|
||||
def test_cleanup_releases_buffer(self):
|
||||
port = _free_port()
|
||||
port = get_open_port(RDMA_HOST)
|
||||
with MooncakeTransferEngineConnector(_connector_config(port)) as c:
|
||||
c.put("s0", "s1", "r1", torch.randn(100))
|
||||
key = MooncakeTransferEngineConnector._make_key("r1", "s0", "s1")
|
||||
@@ -239,7 +232,7 @@ class TestBasicConnector:
|
||||
|
||||
def test_pool_exhaustion_and_recovery(self):
|
||||
"""Fill pool, verify failure, free, verify recovery."""
|
||||
port = _free_port()
|
||||
port = get_open_port(RDMA_HOST)
|
||||
with MooncakeTransferEngineConnector(_connector_config(port, pool_size=64 * 1024)) as c:
|
||||
ids = []
|
||||
for i in range(10):
|
||||
@@ -264,8 +257,8 @@ class TestEndToEnd:
|
||||
"""E2E RDMA transfer: tensor, bytes, object, zero-copy, large payload, mixed types."""
|
||||
|
||||
def _pair(self, pool_size=16 * 1024 * 1024):
|
||||
p = MooncakeTransferEngineConnector(_connector_config(_free_port(), pool_size))
|
||||
c = MooncakeTransferEngineConnector(_connector_config(_free_port(), pool_size))
|
||||
p = MooncakeTransferEngineConnector(_connector_config(get_open_port(RDMA_HOST), pool_size))
|
||||
c = MooncakeTransferEngineConnector(_connector_config(get_open_port(RDMA_HOST), pool_size))
|
||||
return p, c
|
||||
|
||||
def test_tensor_e2e(self):
|
||||
@@ -401,7 +394,7 @@ class TestEndToEnd:
|
||||
|
||||
def test_concurrent_put(self):
|
||||
"""10 concurrent puts should all succeed."""
|
||||
port = _free_port()
|
||||
port = get_open_port(RDMA_HOST)
|
||||
conn = MooncakeTransferEngineConnector(_connector_config(port, pool_size=64 * 1024 * 1024))
|
||||
errors: list[str] = []
|
||||
lock = threading.Lock()
|
||||
@@ -457,20 +450,20 @@ class TestLifecycle:
|
||||
"""Close, context manager, double-close safety."""
|
||||
|
||||
def test_close_releases_resources(self):
|
||||
c = MooncakeTransferEngineConnector(_connector_config(_free_port(), pool_size=1024 * 1024))
|
||||
c = MooncakeTransferEngineConnector(_connector_config(get_open_port(RDMA_HOST), pool_size=1024 * 1024))
|
||||
c.put("s0", "s1", "x", torch.randn(100))
|
||||
c.close()
|
||||
assert c._stop_event.is_set()
|
||||
assert len(c._local_buffers) == 0
|
||||
|
||||
def test_context_manager(self):
|
||||
with MooncakeTransferEngineConnector(_connector_config(_free_port())) as c:
|
||||
with MooncakeTransferEngineConnector(_connector_config(get_open_port(RDMA_HOST))) as c:
|
||||
ok, _, _ = c.put("s0", "s1", "ctx", torch.randn(50))
|
||||
assert ok
|
||||
assert c._stop_event.is_set()
|
||||
|
||||
def test_double_close_safe(self):
|
||||
c = MooncakeTransferEngineConnector(_connector_config(_free_port()))
|
||||
c = MooncakeTransferEngineConnector(_connector_config(get_open_port(RDMA_HOST)))
|
||||
c.close()
|
||||
c.close()
|
||||
|
||||
@@ -489,12 +482,12 @@ class TestGPUPool:
|
||||
return _connector_config(port, pool_size, pool_device="cuda:0")
|
||||
|
||||
def test_gpu_pool_init(self):
|
||||
with MooncakeTransferEngineConnector(self._gpu_cfg(_free_port())) as c:
|
||||
with MooncakeTransferEngineConnector(self._gpu_cfg(get_open_port(RDMA_HOST))) as c:
|
||||
assert c.pool_device == "cuda:0"
|
||||
assert c.pool.is_cuda
|
||||
|
||||
def test_gpu_pool_put_cpu_and_gpu_tensor(self):
|
||||
with MooncakeTransferEngineConnector(self._gpu_cfg(_free_port())) as c:
|
||||
with MooncakeTransferEngineConnector(self._gpu_cfg(get_open_port(RDMA_HOST))) as c:
|
||||
ok, _, meta = c.put("s0", "s1", "h2d", torch.randn(256, 256))
|
||||
assert ok
|
||||
assert meta["is_fast_path"]
|
||||
@@ -504,8 +497,8 @@ class TestGPUPool:
|
||||
assert meta["is_fast_path"]
|
||||
|
||||
def test_gpu_e2e_transfer(self):
|
||||
p = MooncakeTransferEngineConnector(self._gpu_cfg(_free_port()))
|
||||
c = MooncakeTransferEngineConnector(self._gpu_cfg(_free_port()))
|
||||
p = MooncakeTransferEngineConnector(self._gpu_cfg(get_open_port(RDMA_HOST)))
|
||||
c = MooncakeTransferEngineConnector(self._gpu_cfg(get_open_port(RDMA_HOST)))
|
||||
try:
|
||||
orig = torch.randn(512, 512, dtype=torch.float32, device="cuda:0")
|
||||
ok, _, meta = p.put("s0", "s1", "ge", orig)
|
||||
@@ -536,8 +529,8 @@ class TestStressCorrectness:
|
||||
"""
|
||||
|
||||
def _pair(self, pool_size=64 * 1024 * 1024):
|
||||
p = MooncakeTransferEngineConnector(_connector_config(_free_port(), pool_size))
|
||||
c = MooncakeTransferEngineConnector(_connector_config(_free_port(), pool_size))
|
||||
p = MooncakeTransferEngineConnector(_connector_config(get_open_port(RDMA_HOST), pool_size))
|
||||
c = MooncakeTransferEngineConnector(_connector_config(get_open_port(RDMA_HOST), pool_size))
|
||||
return p, c
|
||||
|
||||
# -- Concurrent put + get with data integrity --
|
||||
@@ -712,7 +705,7 @@ class TestStressCorrectness:
|
||||
|
||||
def test_empty_bytes_rejected(self):
|
||||
"""Connector should gracefully reject empty bytes payload."""
|
||||
port = _free_port()
|
||||
port = get_open_port(RDMA_HOST)
|
||||
with MooncakeTransferEngineConnector(_connector_config(port, pool_size=8 * 1024 * 1024)) as c:
|
||||
ok, sz, meta = c.put("s0", "s1", "empty_b", b"")
|
||||
assert not ok, "Empty bytes should be rejected by connector"
|
||||
@@ -744,7 +737,7 @@ class TestStressCorrectness:
|
||||
|
||||
def test_rapid_alloc_free_cycle(self):
|
||||
"""Put + cleanup in tight loop to stress allocator under real connector."""
|
||||
port = _free_port()
|
||||
port = get_open_port(RDMA_HOST)
|
||||
with MooncakeTransferEngineConnector(_connector_config(port, pool_size=8 * 1024 * 1024)) as c:
|
||||
for i in range(50):
|
||||
rid = f"cycle_{i}"
|
||||
|
||||
Reference in new issue
Block a user