vllm-omni/tests/diffusion/distributed/test_parallel_state_sp_groups.py

# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM-Omni project
"""Unit tests for SP subgroup construction for diffusion parallel state."""

from __future__ import annotations

from types import SimpleNamespace

import pytest
import torch
import vllm.distributed.parallel_state as vllm_parallel_state

from tests.helpers.runtime import get_distributed_init_method
from vllm_omni.diffusion.distributed import parallel_state as omni_parallel_state
from vllm_omni.diffusion.distributed.parallel_state import RankGenerator, set_seq_parallel_pg

pytestmark = [pytest.mark.diffusion, pytest.mark.parallel, pytest.mark.core_model, pytest.mark.cpu]


def ensure_parallel_state_initialized():
    """Ensure torch distributed is initialized and that all world / count vars are set."""
    assert torch.distributed.is_initialized()
    assert omni_parallel_state._WORLD is not None
    assert vllm_parallel_state._WORLD is not None
    assert vllm_parallel_state._NODE_COUNT is not None


def ensure_parallel_state_not_initialized():
    """Ensure torch distributed is not initialized and that all world / count vars are unset."""
    assert not torch.distributed.is_initialized()
    assert omni_parallel_state._WORLD is None
    assert vllm_parallel_state._WORLD is None
    assert vllm_parallel_state._NODE_COUNT is None


def test_omni_manages_vllm_distributed_state(monkeypatch):
    """Verify Omni initializes and tears down vLLM's distributed state properly.

    This is a regression test for ensuring vLLM Omni also sets up vLLM's vars correctly
    to avoid potential misalignment in cases where Omni uses vLLM's native coordinator,
    e.g., MoE + diffusion.
    """
    ensure_parallel_state_not_initialized()
    monkeypatch.setattr(omni_parallel_state.current_omni_platform, "get_device_count", lambda: 1)
    monkeypatch.setattr(omni_parallel_state.current_omni_platform, "set_device", lambda _device: None)
    omni_parallel_state.init_distributed_environment(
        world_size=1,
        rank=0,
        local_rank=0,
        distributed_init_method=get_distributed_init_method(),
        backend="gloo",
    )
    try:
        ensure_parallel_state_initialized()
    finally:
        omni_parallel_state.destroy_distributed_env()
    ensure_parallel_state_not_initialized()


def _fake_new_group_factory(created_groups: list[SimpleNamespace]):
    def _fake_new_group(ranks, *args, **kwargs):
        group = SimpleNamespace(ranks=list(ranks))
        created_groups.append(group)
        return group

    return _fake_new_group


@pytest.mark.cpu
@pytest.mark.parametrize(
    "rank, expected_ulysses, expected_ring",
    [
        (0, [0, 2], [0]),
        (1, [1, 3], [1]),
        (2, [0, 2], [2]),
        (3, [1, 3], [3]),
    ],
)
def test_set_seq_parallel_pg_uses_explicit_sp_groups(rank, expected_ulysses, expected_ring, monkeypatch):
    created_groups: list[SimpleNamespace] = []
    monkeypatch.setattr(torch.distributed, "new_group", _fake_new_group_factory(created_groups))

    # tp=2, sp=2 -> SP groups are non-contiguous: [0,2] and [1,3]
    sp_group_ranks = RankGenerator(2, 2, 1, 1, 1, "tp-sp-pp-cfg-dp").get_ranks("sp")

    ulysses_pg, ring_pg, allgather_pg = set_seq_parallel_pg(
        sp_ulysses_degree=2,
        sp_ring_degree=1,
        rank=rank,
        world_size=4,
        sp_group_ranks=sp_group_ranks,
    )

    assert ulysses_pg.ranks == expected_ulysses
    assert ring_pg.ranks == expected_ring
    assert allgather_pg.ranks == [rank]


@pytest.mark.cpu
@pytest.mark.parametrize("rank", [0, 1, 2, 3])
def test_set_seq_parallel_pg_builds_independent_allgather_group(rank, monkeypatch):
    created_groups: list[SimpleNamespace] = []
    monkeypatch.setattr(torch.distributed, "new_group", _fake_new_group_factory(created_groups))
    sp_group_ranks = [[0, 2], [1, 3]]

    ulysses_pg, ring_pg, allgather_pg = set_seq_parallel_pg(
        sp_ulysses_degree=1,
        sp_ring_degree=1,
        sp_allgather_degree=2,
        rank=rank,
        world_size=4,
        sp_group_ranks=sp_group_ranks,
    )

    assert ulysses_pg.ranks == [rank]
    assert ring_pg.ranks == [rank]
    assert allgather_pg.ranks == ([0, 2] if rank % 2 == 0 else [1, 3])


@pytest.mark.cpu
def test_set_seq_parallel_pg_rejects_allgather_with_ulysses_or_ring(monkeypatch):
    created_groups: list[SimpleNamespace] = []
    monkeypatch.setattr(torch.distributed, "new_group", _fake_new_group_factory(created_groups))

    with pytest.raises(ValueError, match="mutually exclusive"):
        set_seq_parallel_pg(
            sp_ulysses_degree=2,
            sp_ring_degree=1,
            sp_allgather_degree=2,
            rank=0,
            world_size=4,
        )


@pytest.mark.cpu
def test_set_seq_parallel_pg_validates_sp_group_ranks(monkeypatch):
    created_groups: list[SimpleNamespace] = []
    monkeypatch.setattr(torch.distributed, "new_group", _fake_new_group_factory(created_groups))

    # world_size=4, sp_size=2 -> expect 2 groups, provide 1 to trigger validation
    with pytest.raises(ValueError, match="Invalid sp_group_ranks"):
        set_seq_parallel_pg(
            sp_ulysses_degree=2,
            sp_ring_degree=1,
            rank=0,
            world_size=4,
            sp_group_ranks=[[0, 2]],
        )