[BugFix] Eliminate distributed test port race by switching to file:// rendezvous (#6468)
Signed-off-by: Nick Cao <ncao@redhat.com>
This commit is contained in:
1 parent
3ad64c6a12
commit
b2c9376539
11 files changed
+117
-237
No files matched your search
@@ -1,10 +1,9 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import socket
|
||||
import tempfile
|
||||
from types import SimpleNamespace
|
||||
|
||||
@@ -12,6 +11,7 @@ import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.helpers.runtime import get_distributed_init_method
|
||||
from vllm_omni.diffusion.attention.layer import Attention
|
||||
from vllm_omni.diffusion.attention.parallel.ulysses import UlyssesParallelAttention
|
||||
from vllm_omni.diffusion.config import set_current_diffusion_config
|
||||
@@ -27,20 +27,6 @@ from vllm_omni.platforms import current_omni_platform
|
||||
pytestmark = [pytest.mark.core_model, pytest.mark.diffusion, pytest.mark.cpu]
|
||||
|
||||
|
||||
def _find_free_port() -> int:
|
||||
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
|
||||
s.bind(("127.0.0.1", 0))
|
||||
return int(s.getsockname()[1])
|
||||
|
||||
|
||||
def _set_dist_env(*, rank: int, world_size: int, master_port: int) -> None:
|
||||
os.environ["RANK"] = str(rank)
|
||||
os.environ["LOCAL_RANK"] = str(rank)
|
||||
os.environ["WORLD_SIZE"] = str(world_size)
|
||||
os.environ["MASTER_ADDR"] = "127.0.0.1"
|
||||
os.environ["MASTER_PORT"] = str(master_port)
|
||||
|
||||
|
||||
@pytest.mark.core_model
|
||||
@pytest.mark.cpu
|
||||
def test_advanced_uaa_hybrid_rejects_padded_mqa_heads(monkeypatch) -> None:
|
||||
@@ -71,7 +57,7 @@ def test_advanced_uaa_hybrid_rejects_padded_mqa_heads(monkeypatch) -> None:
|
||||
def _run_attention_case(
|
||||
local_rank: int,
|
||||
world_size: int,
|
||||
master_port: int,
|
||||
init_method: str,
|
||||
input_file: str,
|
||||
output_file: str,
|
||||
num_heads: int,
|
||||
@@ -85,10 +71,9 @@ def _run_attention_case(
|
||||
device = torch.device(f"{current_omni_platform.device_type}:{local_rank}")
|
||||
current_omni_platform.set_device(device)
|
||||
|
||||
_set_dist_env(rank=local_rank, world_size=world_size, master_port=master_port)
|
||||
os.environ["DIFFUSION_ATTENTION_BACKEND"] = "TORCH_SDPA"
|
||||
|
||||
init_distributed_environment(world_size=world_size, rank=local_rank)
|
||||
init_distributed_environment(world_size=world_size, rank=local_rank, distributed_init_method=init_method)
|
||||
initialize_model_parallel(
|
||||
data_parallel_size=1,
|
||||
cfg_parallel_size=1,
|
||||
@@ -202,10 +187,8 @@ def test_ulysses_uaa_matches_baseline(sp_world_size: int, seq_len: int, num_head
|
||||
batch_size = 2
|
||||
head_size = 8
|
||||
|
||||
base_port = _find_free_port()
|
||||
sp_port = _find_free_port()
|
||||
while sp_port == base_port:
|
||||
sp_port = _find_free_port()
|
||||
base_init_method = get_distributed_init_method()
|
||||
sp_init_method = get_distributed_init_method()
|
||||
|
||||
with tempfile.NamedTemporaryFile(delete=False, suffix=".npz") as f_in:
|
||||
input_file = f_in.name
|
||||
@@ -224,7 +207,7 @@ def test_ulysses_uaa_matches_baseline(sp_world_size: int, seq_len: int, num_head
|
||||
# Baseline (no SP)
|
||||
torch.multiprocessing.spawn(
|
||||
_run_attention_case,
|
||||
args=(1, base_port, input_file, baseline_file, num_heads, head_size, 1, "strict"),
|
||||
args=(1, base_init_method, input_file, baseline_file, num_heads, head_size, 1, "strict"),
|
||||
nprocs=1,
|
||||
)
|
||||
|
||||
@@ -233,7 +216,7 @@ def test_ulysses_uaa_matches_baseline(sp_world_size: int, seq_len: int, num_head
|
||||
_run_attention_case,
|
||||
args=(
|
||||
sp_world_size,
|
||||
sp_port,
|
||||
sp_init_method,
|
||||
input_file,
|
||||
sp_file,
|
||||
num_heads,
|
||||
@@ -278,10 +261,8 @@ def test_ulysses_uaa_hybrid_ring_matches_baseline() -> None:
|
||||
# rank0/1 -> 3+2=5, rank2/3 -> 3+2=5
|
||||
split_sizes = [3, 2, 3, 2]
|
||||
|
||||
base_port = _find_free_port()
|
||||
sp_port = _find_free_port()
|
||||
while sp_port == base_port:
|
||||
sp_port = _find_free_port()
|
||||
base_init_method = get_distributed_init_method()
|
||||
sp_init_method = get_distributed_init_method()
|
||||
|
||||
with tempfile.NamedTemporaryFile(delete=False, suffix=".npz") as f_in:
|
||||
input_file = f_in.name
|
||||
@@ -300,7 +281,19 @@ def test_ulysses_uaa_hybrid_ring_matches_baseline() -> None:
|
||||
# Baseline (no SP)
|
||||
torch.multiprocessing.spawn(
|
||||
_run_attention_case,
|
||||
args=(1, base_port, input_file, baseline_file, num_heads, head_size, 1, "strict", 1, None, "mem_efficient"),
|
||||
args=(
|
||||
1,
|
||||
base_init_method,
|
||||
input_file,
|
||||
baseline_file,
|
||||
num_heads,
|
||||
head_size,
|
||||
1,
|
||||
"strict",
|
||||
1,
|
||||
None,
|
||||
"mem_efficient",
|
||||
),
|
||||
nprocs=1,
|
||||
)
|
||||
|
||||
@@ -309,7 +302,7 @@ def test_ulysses_uaa_hybrid_ring_matches_baseline() -> None:
|
||||
_run_attention_case,
|
||||
args=(
|
||||
sp_world_size,
|
||||
sp_port,
|
||||
sp_init_method,
|
||||
input_file,
|
||||
sp_file,
|
||||
num_heads,
|
||||
|
||||
Reference in new issue
Block a user