[BugFix] Eliminate distributed test port race by switching to file:// rendezvous (#6468)

Signed-off-by: Nick Cao <ncao@redhat.com>
This commit is contained in:
Nick Cao authored and GitHub committed 2026-08-23 07:08:44 +08:00
1 parent 3ad64c6a12
commit b2c9376539
11 files changed
+117 -237

No files matched your search

+24 -31
View File
@@ -1,10 +1,9 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
from __future__ import annotations
import os
import socket
import tempfile
from types import SimpleNamespace
@@ -12,6 +11,7 @@ import numpy as np
import pytest
import torch
from tests.helpers.runtime import get_distributed_init_method
from vllm_omni.diffusion.attention.layer import Attention
from vllm_omni.diffusion.attention.parallel.ulysses import UlyssesParallelAttention
from vllm_omni.diffusion.config import set_current_diffusion_config
@@ -27,20 +27,6 @@ from vllm_omni.platforms import current_omni_platform
pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cpu]
def _find_free_port() -> int:
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
s.bind(("127.0.0.1", 0))
return int(s.getsockname()[1])
def _set_dist_env(*, rank: int, world_size: int, master_port: int) -> None:
os.environ["RANK"] = str(rank)
os.environ["LOCAL_RANK"] = str(rank)
os.environ["WORLD_SIZE"] = str(world_size)
os.environ["MASTER_ADDR"] = "127.0.0.1"
os.environ["MASTER_PORT"] = str(master_port)
@pytest.mark.core_model
@pytest.mark.cpu
def test_advanced_uaa_hybrid_rejects_padded_mqa_heads(monkeypatch) -> None:
@@ -71,7 +57,7 @@ def test_advanced_uaa_hybrid_rejects_padded_mqa_heads(monkeypatch) -> None:
def _run_attention_case(
local_rank: int,
world_size: int,
master_port: int,
init_method: str,
input_file: str,
output_file: str,
num_heads: int,
@@ -85,10 +71,9 @@ def _run_attention_case(
device = torch.device(f"{current_omni_platform.device_type}:{local_rank}")
current_omni_platform.set_device(device)
_set_dist_env(rank=local_rank, world_size=world_size, master_port=master_port)
os.environ["DIFFUSION_ATTENTION_BACKEND"] = "TORCH_SDPA"
init_distributed_environment(world_size=world_size, rank=local_rank)
init_distributed_environment(world_size=world_size, rank=local_rank, distributed_init_method=init_method)
initialize_model_parallel(
data_parallel_size=1,
cfg_parallel_size=1,
@@ -202,10 +187,8 @@ def test_ulysses_uaa_matches_baseline(sp_world_size: int, seq_len: int, num_head
batch_size = 2
head_size = 8
base_port = _find_free_port()
sp_port = _find_free_port()
while sp_port == base_port:
sp_port = _find_free_port()
base_init_method = get_distributed_init_method()
sp_init_method = get_distributed_init_method()
with tempfile.NamedTemporaryFile(delete=False, suffix=".npz") as f_in:
input_file = f_in.name
@@ -224,7 +207,7 @@ def test_ulysses_uaa_matches_baseline(sp_world_size: int, seq_len: int, num_head
# Baseline (no SP)
torch.multiprocessing.spawn(
_run_attention_case,
args=(1, base_port, input_file, baseline_file, num_heads, head_size, 1, "strict"),
args=(1, base_init_method, input_file, baseline_file, num_heads, head_size, 1, "strict"),
nprocs=1,
)
@@ -233,7 +216,7 @@ def test_ulysses_uaa_matches_baseline(sp_world_size: int, seq_len: int, num_head
_run_attention_case,
args=(
sp_world_size,
sp_port,
sp_init_method,
input_file,
sp_file,
num_heads,
@@ -278,10 +261,8 @@ def test_ulysses_uaa_hybrid_ring_matches_baseline() -> None:
# rank0/1 -> 3+2=5, rank2/3 -> 3+2=5
split_sizes = [3, 2, 3, 2]
base_port = _find_free_port()
sp_port = _find_free_port()
while sp_port == base_port:
sp_port = _find_free_port()
base_init_method = get_distributed_init_method()
sp_init_method = get_distributed_init_method()
with tempfile.NamedTemporaryFile(delete=False, suffix=".npz") as f_in:
input_file = f_in.name
@@ -300,7 +281,19 @@ def test_ulysses_uaa_hybrid_ring_matches_baseline() -> None:
# Baseline (no SP)
torch.multiprocessing.spawn(
_run_attention_case,
args=(1, base_port, input_file, baseline_file, num_heads, head_size, 1, "strict", 1, None, "mem_efficient"),
args=(
1,
base_init_method,
input_file,
baseline_file,
num_heads,
head_size,
1,
"strict",
1,
None,
"mem_efficient",
),
nprocs=1,
)
@@ -309,7 +302,7 @@ def test_ulysses_uaa_hybrid_ring_matches_baseline() -> None:
_run_attention_case,
args=(
sp_world_size,
sp_port,
sp_init_method,
input_file,
sp_file,
num_heads,