[BugFix] Eliminate distributed test port race by switching to file:// rendezvous (#6468)

Signed-off-by: Nick Cao <ncao@redhat.com>
This commit is contained in:
Nick Cao authored and GitHub committed 2026-08-23 07:08:44 +08:00
1 parent 3ad64c6a12
commit b2c9376539
11 files changed
+117 -237

No files matched your search

@@ -1,5 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Perf smoke tests for Ulysses advanced_uaa communication overhead.
This test is intended for CI monitoring only:
@@ -9,8 +9,6 @@ This test is intended for CI monitoring only:
from __future__ import annotations
import os
import socket
from dataclasses import dataclass
import pytest
@@ -18,6 +16,7 @@ import torch
import torch.distributed as dist
from tests.helpers.mark import hardware_test
from tests.helpers.runtime import get_distributed_init_method
from vllm_omni.diffusion.attention.parallel.ulysses import (
_all_gather_int,
_ulysses_all_to_all_any_o,
@@ -33,20 +32,6 @@ from vllm_omni.diffusion.distributed.parallel_state import (
from vllm_omni.platforms import current_omni_platform
def _find_free_port() -> int:
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
s.bind(("127.0.0.1", 0))
return int(s.getsockname()[1])
def _set_dist_env(*, rank: int, world_size: int, master_port: int) -> None:
os.environ["RANK"] = str(rank)
os.environ["LOCAL_RANK"] = str(rank)
os.environ["WORLD_SIZE"] = str(world_size)
os.environ["MASTER_ADDR"] = "127.0.0.1"
os.environ["MASTER_PORT"] = str(master_port)
def _max_all_reduce(pg: dist.ProcessGroup, value: float, *, device: torch.device) -> float:
t = torch.tensor([value], device=device, dtype=torch.float32)
dist.all_reduce(t, op=dist.ReduceOp.MAX, group=pg)
@@ -77,22 +62,20 @@ def test_ulysses_advanced_uaa_comm_overhead(case: _PerfCase) -> None:
if available_gpus < case.world_size:
pytest.skip(f"Requires {case.world_size} GPUs, got {available_gpus}")
master_port = _find_free_port()
init_method = get_distributed_init_method()
torch.multiprocessing.spawn(
_perf_worker,
args=(case.world_size, master_port, case.ulysses_degree, case.ring_degree),
args=(case.world_size, init_method, case.ulysses_degree, case.ring_degree),
nprocs=case.world_size,
)
def _perf_worker(local_rank: int, world_size: int, master_port: int, ulysses_degree: int, ring_degree: int) -> None:
def _perf_worker(local_rank: int, world_size: int, init_method: str, ulysses_degree: int, ring_degree: int) -> None:
device = torch.device(f"{current_omni_platform.device_type}:{local_rank}")
current_omni_platform.set_device(device)
_set_dist_env(rank=local_rank, world_size=world_size, master_port=master_port)
try:
init_distributed_environment(world_size=world_size, rank=local_rank)
init_distributed_environment(world_size=world_size, rank=local_rank, distributed_init_method=init_method)
initialize_model_parallel(
data_parallel_size=1,
cfg_parallel_size=1,