[BugFix] Eliminate distributed test port race by switching to file:// rendezvous (#6468)
Signed-off-by: Nick Cao <ncao@redhat.com>
This commit is contained in:
1 parent
3ad64c6a12
commit
b2c9376539
11 files changed
+117
-237
No files matched your search
@@ -1,12 +1,10 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
"""Unit tests for DistributedLayerwiseOffloadHook and backend utilities."""
|
||||
|
||||
import gc
|
||||
import json
|
||||
import os
|
||||
import socket
|
||||
from contextlib import contextmanager
|
||||
from types import SimpleNamespace
|
||||
|
||||
@@ -18,6 +16,7 @@ from torch import nn
|
||||
from torch.distributed.tensor import DeviceMesh, DTensor, Replicate
|
||||
|
||||
import vllm_omni.diffusion.offloader.distributed_layerwise_backend as dist_backend_module
|
||||
from tests.helpers.runtime import get_distributed_init_method
|
||||
from vllm_omni.diffusion.model_loader.host_weight_plan import (
|
||||
HostWeightPlan,
|
||||
TensorBinding,
|
||||
@@ -65,27 +64,10 @@ def dummy_stream(_stream):
|
||||
yield None
|
||||
|
||||
|
||||
def _find_free_port() -> int:
|
||||
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
|
||||
s.bind(("127.0.0.1", 0))
|
||||
return int(s.getsockname()[1])
|
||||
|
||||
|
||||
def _set_dist_env(*, rank: int, world_size: int, master_port: int) -> None:
|
||||
os.environ["RANK"] = str(rank)
|
||||
os.environ["LOCAL_RANK"] = str(rank)
|
||||
os.environ["WORLD_SIZE"] = str(world_size)
|
||||
os.environ["MASTER_ADDR"] = "127.0.0.1"
|
||||
os.environ["MASTER_PORT"] = str(master_port)
|
||||
|
||||
|
||||
def _cleanup_distributed() -> None:
|
||||
if dist.is_initialized():
|
||||
dist.destroy_process_group()
|
||||
|
||||
for key in ["MASTER_ADDR", "MASTER_PORT", "RANK", "WORLD_SIZE", "LOCAL_RANK"]:
|
||||
os.environ.pop(key, None)
|
||||
|
||||
gc.collect()
|
||||
if current_omni_platform.is_available():
|
||||
current_omni_platform.empty_cache()
|
||||
@@ -94,10 +76,7 @@ def _cleanup_distributed() -> None:
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def dist_group():
|
||||
master_port = _find_free_port()
|
||||
_set_dist_env(rank=0, world_size=1, master_port=master_port)
|
||||
|
||||
dist.init_process_group("gloo", rank=0, world_size=1)
|
||||
dist.init_process_group("gloo", rank=0, world_size=1, init_method=get_distributed_init_method())
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
|
||||
Reference in new issue
Block a user