[Diffusion]: Diffusion Ulysses-Sequence-Parallelism support (#189)
Signed-off-by: Didan Deng <33117903+wtomin@users.noreply.github.com>
This commit is contained in:
1 parent
f785a5d401
commit
85f0c1177f
21 files changed
+3513
-61
No files matched your search
+122
-14
@@ -5,10 +5,15 @@ import enum
|
||||
import os
|
||||
import random
|
||||
from collections.abc import Callable
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import dataclass, field, fields
|
||||
from functools import lru_cache
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
from pydantic import model_validator
|
||||
from typing_extensions import Self
|
||||
from vllm.config.utils import config
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from vllm_omni.diffusion.utils.network_utils import is_port_available
|
||||
@@ -16,6 +21,61 @@ from vllm_omni.diffusion.utils.network_utils import is_port_available
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@config
|
||||
@dataclass
|
||||
class DiffusionParallelConfig:
|
||||
"""Configuration for diffusion model distributed execution."""
|
||||
|
||||
pipeline_parallel_size: int = 1
|
||||
"""Number of pipeline parallel stages."""
|
||||
|
||||
data_parallel_size: int = 1
|
||||
"""Number of data parallel groups."""
|
||||
|
||||
tensor_parallel_size: int = 1
|
||||
"""Number of tensor parallel groups."""
|
||||
|
||||
sequence_parallel_size: int | None = None
|
||||
"""Number of sequence parallel groups. sequence_parallel_size = ring_degree * ulysses_degree"""
|
||||
|
||||
ulysses_degree: int = 1
|
||||
"""Number of GPUs used for ulysses sequence parallelism."""
|
||||
|
||||
ring_degree: int = 1
|
||||
"""Number of GPUs used for ring sequence parallelism."""
|
||||
|
||||
cfg_parallel_size: int = 1
|
||||
"""Number of Classifier Free Guidance (CFG) parallel groups."""
|
||||
|
||||
@model_validator(mode="after")
|
||||
def _validate_parallel_config(self) -> Self:
|
||||
"""Validates the config relationships among the parallel strategies."""
|
||||
assert self.pipeline_parallel_size > 0, "Pipeline parallel size must be > 0"
|
||||
assert self.data_parallel_size > 0, "Data parallel size must be > 0"
|
||||
assert self.tensor_parallel_size > 0, "Tensor parallel size must be > 0"
|
||||
assert self.sequence_parallel_size > 0, "Sequence parallel size must be > 0"
|
||||
assert self.ulysses_degree > 0, "Ulysses degree must be > 0"
|
||||
assert self.ring_degree > 0, "Ring degree must be > 0"
|
||||
assert self.cfg_parallel_size > 0, "CFG parallel size must be > 0"
|
||||
assert self.sequence_parallel_size == self.ulysses_degree * self.ring_degree, (
|
||||
"Sequence parallel size must be equal to the product of ulysses degree and ring degree,"
|
||||
f" but got {self.sequence_parallel_size} != {self.ulysses_degree} * {self.ring_degree}"
|
||||
)
|
||||
return self
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
if self.sequence_parallel_size is None:
|
||||
self.sequence_parallel_size = self.ulysses_degree * self.ring_degree
|
||||
self.world_size = (
|
||||
self.pipeline_parallel_size
|
||||
* self.data_parallel_size
|
||||
* self.tensor_parallel_size
|
||||
* self.ulysses_degree
|
||||
* self.ring_degree
|
||||
* self.cfg_parallel_size
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class TransformerConfig:
|
||||
"""Container for raw transformer configuration dictionaries."""
|
||||
@@ -180,6 +240,7 @@ class OmniDiffusionConfig:
|
||||
|
||||
# Cache strategy (legacy)
|
||||
cache_strategy: str = "none"
|
||||
parallel_config: DiffusionParallelConfig = field(default_factory=DiffusionParallelConfig)
|
||||
|
||||
# Cache backend configuration (NEW)
|
||||
cache_backend: str = "none" # "tea_cache", "deep_cache", etc.
|
||||
@@ -193,20 +254,7 @@ class OmniDiffusionConfig:
|
||||
trust_remote_code: bool = False
|
||||
revision: str | None = None
|
||||
|
||||
# Parallelism
|
||||
num_gpus: int = 1
|
||||
tp_size: int = -1
|
||||
sp_degree: int = -1
|
||||
# sequence parallelism
|
||||
ulysses_degree: int | None = None
|
||||
ring_degree: int | None = None
|
||||
# data parallelism
|
||||
# number of data parallelism groups
|
||||
dp_size: int = 1
|
||||
# number of gpu in a dp group
|
||||
dp_degree: int = 1
|
||||
# cfg parallel
|
||||
enable_cfg_parallel: bool = False
|
||||
num_gpus: int | None = None
|
||||
|
||||
hsdp_replicate_dim: int = 1
|
||||
hsdp_shard_dim: int = -1
|
||||
@@ -329,6 +377,16 @@ class OmniDiffusionConfig:
|
||||
# TODO: remove hard code
|
||||
initial_master_port = (self.master_port or 30005) + random.randint(0, 100)
|
||||
self.master_port = self.settle_port(initial_master_port, 37)
|
||||
if self.num_gpus is None:
|
||||
if self.parallel_config is not None:
|
||||
self.num_gpus = self.parallel_config.world_size
|
||||
else:
|
||||
self.num_gpus = 1
|
||||
|
||||
if self.num_gpus < self.parallel_config.world_size:
|
||||
raise ValueError(
|
||||
f"num_gpus ({self.num_gpus}) < parallel_config.world_size ({self.parallel_config.world_size})"
|
||||
)
|
||||
|
||||
# Convert cache_config dict to DiffusionCacheConfig if needed
|
||||
if isinstance(self.cache_config, dict):
|
||||
@@ -347,6 +405,56 @@ class OmniDiffusionConfig:
|
||||
return cls(**kwargs)
|
||||
|
||||
|
||||
_current_omni_diffusion_config: OmniDiffusionConfig | None = None
|
||||
_current_prefix: str | None = None
|
||||
|
||||
|
||||
@contextmanager
|
||||
def set_current_omni_diffusion_config(
|
||||
omni_diffusion_config: OmniDiffusionConfig, check_compile=False, prefix: str | None = None
|
||||
):
|
||||
"""
|
||||
Temporarily set the current vLLM-Omni config.
|
||||
Used during model initialization.
|
||||
We save the current vLLM-Omni config in a global variable,
|
||||
so that all modules can access it, e.g. custom ops
|
||||
can access the vLLM-Omni config to determine how to dispatch.
|
||||
"""
|
||||
global _current_omni_diffusion_config, _current_prefix
|
||||
old_omni_diffusion_config = _current_omni_diffusion_config
|
||||
old_prefix = _current_prefix
|
||||
# from vllm.compilation.counter import compilation_counter
|
||||
|
||||
# num_models_seen = compilation_counter.num_models_seen
|
||||
try:
|
||||
_current_omni_diffusion_config = omni_diffusion_config
|
||||
_current_prefix = prefix
|
||||
yield
|
||||
except Exception:
|
||||
raise
|
||||
else:
|
||||
if check_compile:
|
||||
raise RuntimeError("Compilation is not yet supported for OmniDiffusion")
|
||||
finally:
|
||||
_current_omni_diffusion_config = old_omni_diffusion_config
|
||||
_current_prefix = old_prefix
|
||||
# Clear the compilation config cache when context changes
|
||||
get_cached_compilation_config.cache_clear()
|
||||
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def get_cached_compilation_config():
|
||||
"""Cache config to avoid repeated calls to get_current_omni_diffusion_config()"""
|
||||
return get_current_omni_diffusion_config().compilation_config
|
||||
|
||||
|
||||
def get_current_omni_diffusion_config() -> OmniDiffusionConfig:
|
||||
if _current_omni_diffusion_config is None:
|
||||
logger.warning("Current OmniDiffusionConfig is not set.")
|
||||
return OmniDiffusionConfig()
|
||||
return _current_omni_diffusion_config
|
||||
|
||||
|
||||
@dataclass
|
||||
class DiffusionOutput:
|
||||
"""
|
||||
|
||||
Reference in new issue
Block a user